]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2016 Intel Corporation. All rights reserved. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | #include <sys/types.h> | |
34 | #include <netinet/in.h> | |
35 | #include <netinet/ip.h> | |
36 | ||
37 | #include <rte_branch_prediction.h> | |
38 | #include <rte_log.h> | |
39 | #include <rte_crypto.h> | |
40 | #include <rte_cryptodev.h> | |
41 | #include <rte_mbuf.h> | |
42 | #include <rte_hash.h> | |
43 | ||
44 | #include "ipsec.h" | |
45 | #include "esp.h" | |
46 | ||
47 | static inline int | |
48 | create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) | |
49 | { | |
50 | unsigned long cdev_id_qp = 0; | |
51 | int32_t ret; | |
52 | struct cdev_key key = { 0 }; | |
53 | ||
54 | key.lcore_id = (uint8_t)rte_lcore_id(); | |
55 | ||
56 | key.cipher_algo = (uint8_t)sa->cipher_algo; | |
57 | key.auth_algo = (uint8_t)sa->auth_algo; | |
58 | ||
59 | ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key, | |
60 | (void **)&cdev_id_qp); | |
61 | if (ret < 0) { | |
62 | RTE_LOG(ERR, IPSEC, "No cryptodev: core %u, cipher_algo %u, " | |
63 | "auth_algo %u\n", key.lcore_id, key.cipher_algo, | |
64 | key.auth_algo); | |
65 | return -1; | |
66 | } | |
67 | ||
68 | RTE_LOG(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev " | |
69 | "%u qp %u\n", sa->spi, | |
70 | ipsec_ctx->tbl[cdev_id_qp].id, | |
71 | ipsec_ctx->tbl[cdev_id_qp].qp); | |
72 | ||
73 | sa->crypto_session = rte_cryptodev_sym_session_create( | |
74 | ipsec_ctx->tbl[cdev_id_qp].id, sa->xforms); | |
75 | ||
76 | sa->cdev_id_qp = cdev_id_qp; | |
77 | ||
78 | return 0; | |
79 | } | |
80 | ||
81 | static inline void | |
82 | enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop) | |
83 | { | |
84 | int32_t ret, i; | |
85 | ||
86 | cqp->buf[cqp->len++] = cop; | |
87 | ||
88 | if (cqp->len == MAX_PKT_BURST) { | |
89 | ret = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, | |
90 | cqp->buf, cqp->len); | |
91 | if (ret < cqp->len) { | |
92 | RTE_LOG(DEBUG, IPSEC, "Cryptodev %u queue %u:" | |
93 | " enqueued %u crypto ops out of %u\n", | |
94 | cqp->id, cqp->qp, | |
95 | ret, cqp->len); | |
96 | for (i = ret; i < cqp->len; i++) | |
97 | rte_pktmbuf_free(cqp->buf[i]->sym->m_src); | |
98 | } | |
99 | cqp->in_flight += ret; | |
100 | cqp->len = 0; | |
101 | } | |
102 | } | |
103 | ||
104 | static inline void | |
105 | ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, | |
106 | struct rte_mbuf *pkts[], struct ipsec_sa *sas[], | |
107 | uint16_t nb_pkts) | |
108 | { | |
109 | int32_t ret = 0, i; | |
110 | struct ipsec_mbuf_metadata *priv; | |
111 | struct ipsec_sa *sa; | |
112 | ||
113 | for (i = 0; i < nb_pkts; i++) { | |
114 | if (unlikely(sas[i] == NULL)) { | |
115 | rte_pktmbuf_free(pkts[i]); | |
116 | continue; | |
117 | } | |
118 | ||
119 | rte_prefetch0(sas[i]); | |
120 | rte_prefetch0(pkts[i]); | |
121 | ||
122 | priv = get_priv(pkts[i]); | |
123 | sa = sas[i]; | |
124 | priv->sa = sa; | |
125 | ||
126 | priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; | |
127 | priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; | |
128 | ||
129 | rte_prefetch0(&priv->sym_cop); | |
130 | priv->cop.sym = &priv->sym_cop; | |
131 | ||
132 | if ((unlikely(sa->crypto_session == NULL)) && | |
133 | create_session(ipsec_ctx, sa)) { | |
134 | rte_pktmbuf_free(pkts[i]); | |
135 | continue; | |
136 | } | |
137 | ||
138 | rte_crypto_op_attach_sym_session(&priv->cop, | |
139 | sa->crypto_session); | |
140 | ||
141 | ret = xform_func(pkts[i], sa, &priv->cop); | |
142 | if (unlikely(ret)) { | |
143 | rte_pktmbuf_free(pkts[i]); | |
144 | continue; | |
145 | } | |
146 | ||
147 | RTE_ASSERT(sa->cdev_id_qp < ipsec_ctx->nb_qps); | |
148 | enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop); | |
149 | } | |
150 | } | |
151 | ||
152 | static inline int | |
153 | ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, | |
154 | struct rte_mbuf *pkts[], uint16_t max_pkts) | |
155 | { | |
156 | int32_t nb_pkts = 0, ret = 0, i, j, nb_cops; | |
157 | struct ipsec_mbuf_metadata *priv; | |
158 | struct rte_crypto_op *cops[max_pkts]; | |
159 | struct ipsec_sa *sa; | |
160 | struct rte_mbuf *pkt; | |
161 | ||
162 | for (i = 0; i < ipsec_ctx->nb_qps && nb_pkts < max_pkts; i++) { | |
163 | struct cdev_qp *cqp; | |
164 | ||
165 | cqp = &ipsec_ctx->tbl[ipsec_ctx->last_qp++]; | |
166 | if (ipsec_ctx->last_qp == ipsec_ctx->nb_qps) | |
167 | ipsec_ctx->last_qp %= ipsec_ctx->nb_qps; | |
168 | ||
169 | if (cqp->in_flight == 0) | |
170 | continue; | |
171 | ||
172 | nb_cops = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, | |
173 | cops, max_pkts - nb_pkts); | |
174 | ||
175 | cqp->in_flight -= nb_cops; | |
176 | ||
177 | for (j = 0; j < nb_cops; j++) { | |
178 | pkt = cops[j]->sym->m_src; | |
179 | rte_prefetch0(pkt); | |
180 | ||
181 | priv = get_priv(pkt); | |
182 | sa = priv->sa; | |
183 | ||
184 | RTE_ASSERT(sa != NULL); | |
185 | ||
186 | ret = xform_func(pkt, sa, cops[j]); | |
187 | if (unlikely(ret)) | |
188 | rte_pktmbuf_free(pkt); | |
189 | else | |
190 | pkts[nb_pkts++] = pkt; | |
191 | } | |
192 | } | |
193 | ||
194 | /* return packets */ | |
195 | return nb_pkts; | |
196 | } | |
197 | ||
198 | uint16_t | |
199 | ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], | |
200 | uint16_t nb_pkts, uint16_t len) | |
201 | { | |
202 | struct ipsec_sa *sas[nb_pkts]; | |
203 | ||
204 | inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts); | |
205 | ||
206 | ipsec_enqueue(esp_inbound, ctx, pkts, sas, nb_pkts); | |
207 | ||
208 | return ipsec_dequeue(esp_inbound_post, ctx, pkts, len); | |
209 | } | |
210 | ||
211 | uint16_t | |
212 | ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], | |
213 | uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len) | |
214 | { | |
215 | struct ipsec_sa *sas[nb_pkts]; | |
216 | ||
217 | outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts); | |
218 | ||
219 | ipsec_enqueue(esp_outbound, ctx, pkts, sas, nb_pkts); | |
220 | ||
221 | return ipsec_dequeue(esp_outbound_post, ctx, pkts, len); | |
222 | } |