]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2016-2017 Intel Corporation | |
7c673cae FG |
3 | */ |
4 | ||
7c673cae | 5 | #include <rte_common.h> |
7c673cae FG |
6 | #include <rte_hexdump.h> |
7 | #include <rte_cryptodev.h> | |
8 | #include <rte_cryptodev_pmd.h> | |
9f95a23c | 9 | #include <rte_bus_vdev.h> |
7c673cae FG |
10 | #include <rte_malloc.h> |
11 | #include <rte_cpuflags.h> | |
11fdf7f2 | 12 | #include <rte_byteorder.h> |
7c673cae FG |
13 | |
14 | #include "aesni_gcm_pmd_private.h" | |
15 | ||
9f95a23c | 16 | static uint8_t cryptodev_driver_id; |
7c673cae FG |
17 | |
18 | /** Parse crypto xform chain and set private session parameters */ | |
19 | int | |
9f95a23c TL |
20 | aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, |
21 | struct aesni_gcm_session *sess, | |
7c673cae FG |
22 | const struct rte_crypto_sym_xform *xform) |
23 | { | |
11fdf7f2 | 24 | const struct rte_crypto_sym_xform *auth_xform; |
9f95a23c TL |
25 | const struct rte_crypto_sym_xform *aead_xform; |
26 | uint8_t key_length; | |
27 | uint8_t *key; | |
7c673cae | 28 | |
9f95a23c TL |
29 | /* AES-GMAC */ |
30 | if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { | |
7c673cae | 31 | auth_xform = xform; |
9f95a23c TL |
32 | if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) { |
33 | AESNI_GCM_LOG(ERR, "Only AES GMAC is supported as an " | |
34 | "authentication only algorithm"); | |
35 | return -ENOTSUP; | |
36 | } | |
37 | /* Set IV parameters */ | |
38 | sess->iv.offset = auth_xform->auth.iv.offset; | |
39 | sess->iv.length = auth_xform->auth.iv.length; | |
40 | ||
41 | /* Select Crypto operation */ | |
42 | if (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) | |
43 | sess->op = AESNI_GMAC_OP_GENERATE; | |
44 | else | |
45 | sess->op = AESNI_GMAC_OP_VERIFY; | |
46 | ||
47 | key_length = auth_xform->auth.key.length; | |
48 | key = auth_xform->auth.key.data; | |
49 | sess->req_digest_length = auth_xform->auth.digest_length; | |
50 | ||
51 | /* AES-GCM */ | |
52 | } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { | |
53 | aead_xform = xform; | |
54 | ||
55 | if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) { | |
56 | AESNI_GCM_LOG(ERR, "The only combined operation " | |
57 | "supported is AES GCM"); | |
58 | return -ENOTSUP; | |
59 | } | |
60 | ||
61 | /* Set IV parameters */ | |
62 | sess->iv.offset = aead_xform->aead.iv.offset; | |
63 | sess->iv.length = aead_xform->aead.iv.length; | |
64 | ||
65 | /* Select Crypto operation */ | |
66 | if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) | |
67 | sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; | |
68 | else | |
69 | sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; | |
70 | ||
71 | key_length = aead_xform->aead.key.length; | |
72 | key = aead_xform->aead.key.data; | |
73 | ||
74 | sess->aad_length = aead_xform->aead.aad_length; | |
75 | sess->req_digest_length = aead_xform->aead.digest_length; | |
11fdf7f2 | 76 | } else { |
9f95a23c TL |
77 | AESNI_GCM_LOG(ERR, "Wrong xform type, has to be AEAD or authentication"); |
78 | return -ENOTSUP; | |
7c673cae FG |
79 | } |
80 | ||
7c673cae | 81 | |
9f95a23c TL |
82 | /* IV check */ |
83 | if (sess->iv.length != 16 && sess->iv.length != 12 && | |
84 | sess->iv.length != 0) { | |
85 | AESNI_GCM_LOG(ERR, "Wrong IV length"); | |
7c673cae FG |
86 | return -EINVAL; |
87 | } | |
88 | ||
11fdf7f2 | 89 | /* Check key length and calculate GCM pre-compute. */ |
9f95a23c | 90 | switch (key_length) { |
11fdf7f2 | 91 | case 16: |
9f95a23c TL |
92 | sess->key = GCM_KEY_128; |
93 | break; | |
94 | case 24: | |
95 | sess->key = GCM_KEY_192; | |
11fdf7f2 TL |
96 | break; |
97 | case 32: | |
9f95a23c | 98 | sess->key = GCM_KEY_256; |
11fdf7f2 TL |
99 | break; |
100 | default: | |
9f95a23c TL |
101 | AESNI_GCM_LOG(ERR, "Invalid key length"); |
102 | return -EINVAL; | |
103 | } | |
104 | ||
105 | gcm_ops[sess->key].pre(key, &sess->gdata_key); | |
106 | ||
107 | /* Digest check */ | |
108 | if (sess->req_digest_length > 16) { | |
109 | AESNI_GCM_LOG(ERR, "Invalid digest length"); | |
11fdf7f2 TL |
110 | return -EINVAL; |
111 | } | |
9f95a23c TL |
112 | /* |
113 | * Multi-buffer lib supports digest sizes from 4 to 16 bytes | |
114 | * in version 0.50 and sizes of 8, 12 and 16 bytes, | |
115 | * in version 0.49. | |
116 | * If size requested is different, generate the full digest | |
117 | * (16 bytes) in a temporary location and then memcpy | |
118 | * the requested number of bytes. | |
119 | */ | |
120 | #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) | |
121 | if (sess->req_digest_length < 4) | |
122 | #else | |
123 | if (sess->req_digest_length != 16 && | |
124 | sess->req_digest_length != 12 && | |
125 | sess->req_digest_length != 8) | |
126 | #endif | |
127 | sess->gen_digest_length = 16; | |
128 | else | |
129 | sess->gen_digest_length = sess->req_digest_length; | |
7c673cae FG |
130 | |
131 | return 0; | |
132 | } | |
133 | ||
134 | /** Get gcm session */ | |
135 | static struct aesni_gcm_session * | |
9f95a23c | 136 | aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) |
7c673cae FG |
137 | { |
138 | struct aesni_gcm_session *sess = NULL; | |
9f95a23c TL |
139 | struct rte_crypto_sym_op *sym_op = op->sym; |
140 | ||
141 | if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { | |
142 | if (likely(sym_op->session != NULL)) | |
143 | sess = (struct aesni_gcm_session *) | |
144 | get_sym_session_private_data( | |
145 | sym_op->session, | |
146 | cryptodev_driver_id); | |
7c673cae FG |
147 | } else { |
148 | void *_sess; | |
9f95a23c TL |
149 | void *_sess_private_data = NULL; |
150 | ||
151 | if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) | |
152 | return NULL; | |
7c673cae | 153 | |
9f95a23c TL |
154 | if (rte_mempool_get(qp->sess_mp_priv, |
155 | (void **)&_sess_private_data)) | |
156 | return NULL; | |
7c673cae | 157 | |
9f95a23c | 158 | sess = (struct aesni_gcm_session *)_sess_private_data; |
7c673cae | 159 | |
9f95a23c TL |
160 | if (unlikely(aesni_gcm_set_session_parameters(qp->ops, |
161 | sess, sym_op->xform) != 0)) { | |
7c673cae | 162 | rte_mempool_put(qp->sess_mp, _sess); |
9f95a23c | 163 | rte_mempool_put(qp->sess_mp_priv, _sess_private_data); |
7c673cae FG |
164 | sess = NULL; |
165 | } | |
9f95a23c TL |
166 | sym_op->session = (struct rte_cryptodev_sym_session *)_sess; |
167 | set_sym_session_private_data(sym_op->session, | |
168 | cryptodev_driver_id, _sess_private_data); | |
7c673cae | 169 | } |
9f95a23c TL |
170 | |
171 | if (unlikely(sess == NULL)) | |
172 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
173 | ||
7c673cae FG |
174 | return sess; |
175 | } | |
176 | ||
177 | /** | |
9f95a23c TL |
178 | * Process a crypto operation, calling |
179 | * the GCM API from the multi buffer library. | |
7c673cae FG |
180 | * |
181 | * @param qp queue pair | |
182 | * @param op symmetric crypto operation | |
183 | * @param session GCM session | |
184 | * | |
185 | * @return | |
186 | * | |
187 | */ | |
188 | static int | |
9f95a23c | 189 | process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, |
7c673cae FG |
190 | struct aesni_gcm_session *session) |
191 | { | |
192 | uint8_t *src, *dst; | |
9f95a23c TL |
193 | uint8_t *iv_ptr; |
194 | struct rte_crypto_sym_op *sym_op = op->sym; | |
195 | struct rte_mbuf *m_src = sym_op->m_src; | |
196 | uint32_t offset, data_offset, data_length; | |
11fdf7f2 | 197 | uint32_t part_len, total_len, data_len; |
9f95a23c TL |
198 | uint8_t *tag; |
199 | ||
200 | if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION || | |
201 | session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { | |
202 | offset = sym_op->aead.data.offset; | |
203 | data_offset = offset; | |
204 | data_length = sym_op->aead.data.length; | |
205 | } else { | |
206 | offset = sym_op->auth.data.offset; | |
207 | data_offset = offset; | |
208 | data_length = sym_op->auth.data.length; | |
209 | } | |
11fdf7f2 TL |
210 | |
211 | RTE_ASSERT(m_src != NULL); | |
212 | ||
9f95a23c | 213 | while (offset >= m_src->data_len && data_length != 0) { |
11fdf7f2 TL |
214 | offset -= m_src->data_len; |
215 | m_src = m_src->next; | |
216 | ||
217 | RTE_ASSERT(m_src != NULL); | |
218 | } | |
219 | ||
220 | data_len = m_src->data_len - offset; | |
9f95a23c TL |
221 | part_len = (data_len < data_length) ? data_len : |
222 | data_length; | |
11fdf7f2 TL |
223 | |
224 | /* Destination buffer is required when segmented source buffer */ | |
9f95a23c TL |
225 | RTE_ASSERT((part_len == data_length) || |
226 | ((part_len != data_length) && | |
227 | (sym_op->m_dst != NULL))); | |
11fdf7f2 | 228 | /* Segmented destination buffer is not supported */ |
9f95a23c TL |
229 | RTE_ASSERT((sym_op->m_dst == NULL) || |
230 | ((sym_op->m_dst != NULL) && | |
231 | rte_pktmbuf_is_contiguous(sym_op->m_dst))); | |
11fdf7f2 | 232 | |
7c673cae | 233 | |
9f95a23c TL |
234 | dst = sym_op->m_dst ? |
235 | rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *, | |
236 | data_offset) : | |
237 | rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, | |
238 | data_offset); | |
7c673cae | 239 | |
11fdf7f2 TL |
240 | src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset); |
241 | ||
9f95a23c TL |
242 | iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, |
243 | session->iv.offset); | |
7c673cae FG |
244 | |
245 | if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) { | |
9f95a23c TL |
246 | qp->ops[session->key].init(&session->gdata_key, |
247 | &qp->gdata_ctx, | |
248 | iv_ptr, | |
249 | sym_op->aead.aad.data, | |
250 | (uint64_t)session->aad_length); | |
251 | ||
252 | qp->ops[session->key].update_enc(&session->gdata_key, | |
253 | &qp->gdata_ctx, dst, src, | |
11fdf7f2 | 254 | (uint64_t)part_len); |
9f95a23c | 255 | total_len = data_length - part_len; |
11fdf7f2 TL |
256 | |
257 | while (total_len) { | |
258 | dst += part_len; | |
259 | m_src = m_src->next; | |
260 | ||
261 | RTE_ASSERT(m_src != NULL); | |
262 | ||
263 | src = rte_pktmbuf_mtod(m_src, uint8_t *); | |
264 | part_len = (m_src->data_len < total_len) ? | |
265 | m_src->data_len : total_len; | |
266 | ||
9f95a23c TL |
267 | qp->ops[session->key].update_enc(&session->gdata_key, |
268 | &qp->gdata_ctx, dst, src, | |
11fdf7f2 TL |
269 | (uint64_t)part_len); |
270 | total_len -= part_len; | |
271 | } | |
272 | ||
9f95a23c TL |
273 | if (session->req_digest_length != session->gen_digest_length) |
274 | tag = qp->temp_digest; | |
275 | else | |
276 | tag = sym_op->aead.digest.data; | |
277 | ||
278 | qp->ops[session->key].finalize_enc(&session->gdata_key, | |
279 | &qp->gdata_ctx, | |
280 | tag, | |
281 | session->gen_digest_length); | |
282 | } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { | |
283 | qp->ops[session->key].init(&session->gdata_key, | |
284 | &qp->gdata_ctx, | |
285 | iv_ptr, | |
286 | sym_op->aead.aad.data, | |
287 | (uint64_t)session->aad_length); | |
288 | ||
289 | qp->ops[session->key].update_dec(&session->gdata_key, | |
290 | &qp->gdata_ctx, dst, src, | |
11fdf7f2 | 291 | (uint64_t)part_len); |
9f95a23c | 292 | total_len = data_length - part_len; |
11fdf7f2 TL |
293 | |
294 | while (total_len) { | |
295 | dst += part_len; | |
296 | m_src = m_src->next; | |
297 | ||
298 | RTE_ASSERT(m_src != NULL); | |
299 | ||
300 | src = rte_pktmbuf_mtod(m_src, uint8_t *); | |
301 | part_len = (m_src->data_len < total_len) ? | |
302 | m_src->data_len : total_len; | |
303 | ||
9f95a23c TL |
304 | qp->ops[session->key].update_dec(&session->gdata_key, |
305 | &qp->gdata_ctx, | |
11fdf7f2 TL |
306 | dst, src, |
307 | (uint64_t)part_len); | |
308 | total_len -= part_len; | |
309 | } | |
310 | ||
9f95a23c TL |
311 | tag = qp->temp_digest; |
312 | qp->ops[session->key].finalize_dec(&session->gdata_key, | |
313 | &qp->gdata_ctx, | |
314 | tag, | |
315 | session->gen_digest_length); | |
316 | } else if (session->op == AESNI_GMAC_OP_GENERATE) { | |
317 | qp->ops[session->key].init(&session->gdata_key, | |
318 | &qp->gdata_ctx, | |
319 | iv_ptr, | |
320 | src, | |
321 | (uint64_t)data_length); | |
322 | if (session->req_digest_length != session->gen_digest_length) | |
323 | tag = qp->temp_digest; | |
324 | else | |
325 | tag = sym_op->auth.digest.data; | |
326 | qp->ops[session->key].finalize_enc(&session->gdata_key, | |
327 | &qp->gdata_ctx, | |
328 | tag, | |
329 | session->gen_digest_length); | |
330 | } else { /* AESNI_GMAC_OP_VERIFY */ | |
331 | qp->ops[session->key].init(&session->gdata_key, | |
332 | &qp->gdata_ctx, | |
333 | iv_ptr, | |
334 | src, | |
335 | (uint64_t)data_length); | |
336 | ||
337 | /* | |
338 | * Generate always 16 bytes and later compare only | |
339 | * the bytes passed. | |
340 | */ | |
341 | tag = qp->temp_digest; | |
342 | qp->ops[session->key].finalize_enc(&session->gdata_key, | |
343 | &qp->gdata_ctx, | |
344 | tag, | |
345 | session->gen_digest_length); | |
7c673cae FG |
346 | } |
347 | ||
348 | return 0; | |
349 | } | |
350 | ||
351 | /** | |
352 | * Process a completed job and return rte_mbuf which job processed | |
353 | * | |
354 | * @param job JOB_AES_HMAC job to process | |
355 | * | |
356 | * @return | |
357 | * - Returns processed mbuf which is trimmed of output digest used in | |
358 | * verification of supplied digest in the case of a HASH_CIPHER operation | |
359 | * - Returns NULL on invalid job | |
360 | */ | |
361 | static void | |
9f95a23c TL |
362 | post_process_gcm_crypto_op(struct aesni_gcm_qp *qp, |
363 | struct rte_crypto_op *op, | |
364 | struct aesni_gcm_session *session) | |
7c673cae | 365 | { |
7c673cae FG |
366 | op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; |
367 | ||
368 | /* Verify digest if required */ | |
9f95a23c TL |
369 | if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION || |
370 | session->op == AESNI_GMAC_OP_VERIFY) { | |
371 | uint8_t *digest; | |
7c673cae | 372 | |
9f95a23c TL |
373 | uint8_t *tag = qp->temp_digest; |
374 | ||
375 | if (session->op == AESNI_GMAC_OP_VERIFY) | |
376 | digest = op->sym->auth.digest.data; | |
377 | else | |
378 | digest = op->sym->aead.digest.data; | |
7c673cae FG |
379 | |
380 | #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG | |
381 | rte_hexdump(stdout, "auth tag (orig):", | |
9f95a23c | 382 | digest, session->req_digest_length); |
7c673cae | 383 | rte_hexdump(stdout, "auth tag (calc):", |
9f95a23c | 384 | tag, session->req_digest_length); |
7c673cae FG |
385 | #endif |
386 | ||
9f95a23c | 387 | if (memcmp(tag, digest, session->req_digest_length) != 0) |
7c673cae | 388 | op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; |
9f95a23c TL |
389 | } else { |
390 | if (session->req_digest_length != session->gen_digest_length) { | |
391 | if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) | |
392 | memcpy(op->sym->aead.digest.data, qp->temp_digest, | |
393 | session->req_digest_length); | |
394 | else | |
395 | memcpy(op->sym->auth.digest.data, qp->temp_digest, | |
396 | session->req_digest_length); | |
397 | } | |
7c673cae FG |
398 | } |
399 | } | |
400 | ||
401 | /** | |
402 | * Process a completed GCM request | |
403 | * | |
404 | * @param qp Queue Pair to process | |
9f95a23c | 405 | * @param op Crypto operation |
7c673cae FG |
406 | * @param job JOB_AES_HMAC job |
407 | * | |
408 | * @return | |
409 | * - Number of processed jobs | |
410 | */ | |
411 | static void | |
412 | handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, | |
9f95a23c TL |
413 | struct rte_crypto_op *op, |
414 | struct aesni_gcm_session *sess) | |
7c673cae | 415 | { |
9f95a23c | 416 | post_process_gcm_crypto_op(qp, op, sess); |
7c673cae FG |
417 | |
418 | /* Free session if a session-less crypto op */ | |
9f95a23c TL |
419 | if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { |
420 | memset(sess, 0, sizeof(struct aesni_gcm_session)); | |
421 | memset(op->sym->session, 0, | |
422 | rte_cryptodev_sym_get_existing_header_session_size( | |
423 | op->sym->session)); | |
424 | rte_mempool_put(qp->sess_mp_priv, sess); | |
7c673cae FG |
425 | rte_mempool_put(qp->sess_mp, op->sym->session); |
426 | op->sym->session = NULL; | |
427 | } | |
7c673cae FG |
428 | } |
429 | ||
430 | static uint16_t | |
11fdf7f2 | 431 | aesni_gcm_pmd_dequeue_burst(void *queue_pair, |
7c673cae FG |
432 | struct rte_crypto_op **ops, uint16_t nb_ops) |
433 | { | |
434 | struct aesni_gcm_session *sess; | |
435 | struct aesni_gcm_qp *qp = queue_pair; | |
436 | ||
11fdf7f2 TL |
437 | int retval = 0; |
438 | unsigned int i, nb_dequeued; | |
439 | ||
440 | nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, | |
441 | (void **)ops, nb_ops, NULL); | |
7c673cae | 442 | |
11fdf7f2 | 443 | for (i = 0; i < nb_dequeued; i++) { |
7c673cae | 444 | |
9f95a23c | 445 | sess = aesni_gcm_get_session(qp, ops[i]); |
7c673cae FG |
446 | if (unlikely(sess == NULL)) { |
447 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
11fdf7f2 | 448 | qp->qp_stats.dequeue_err_count++; |
7c673cae FG |
449 | break; |
450 | } | |
451 | ||
9f95a23c | 452 | retval = process_gcm_crypto_op(qp, ops[i], sess); |
7c673cae FG |
453 | if (retval < 0) { |
454 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
11fdf7f2 | 455 | qp->qp_stats.dequeue_err_count++; |
7c673cae FG |
456 | break; |
457 | } | |
458 | ||
9f95a23c | 459 | handle_completed_gcm_crypto_op(qp, ops[i], sess); |
7c673cae | 460 | } |
11fdf7f2 TL |
461 | |
462 | qp->qp_stats.dequeued_count += i; | |
463 | ||
7c673cae FG |
464 | return i; |
465 | } | |
466 | ||
467 | static uint16_t | |
11fdf7f2 | 468 | aesni_gcm_pmd_enqueue_burst(void *queue_pair, |
7c673cae FG |
469 | struct rte_crypto_op **ops, uint16_t nb_ops) |
470 | { | |
471 | struct aesni_gcm_qp *qp = queue_pair; | |
472 | ||
11fdf7f2 | 473 | unsigned int nb_enqueued; |
7c673cae | 474 | |
11fdf7f2 TL |
475 | nb_enqueued = rte_ring_enqueue_burst(qp->processed_pkts, |
476 | (void **)ops, nb_ops, NULL); | |
477 | qp->qp_stats.enqueued_count += nb_enqueued; | |
7c673cae | 478 | |
11fdf7f2 | 479 | return nb_enqueued; |
7c673cae FG |
480 | } |
481 | ||
11fdf7f2 | 482 | static int aesni_gcm_remove(struct rte_vdev_device *vdev); |
7c673cae FG |
483 | |
484 | static int | |
485 | aesni_gcm_create(const char *name, | |
11fdf7f2 | 486 | struct rte_vdev_device *vdev, |
9f95a23c | 487 | struct rte_cryptodev_pmd_init_params *init_params) |
7c673cae FG |
488 | { |
489 | struct rte_cryptodev *dev; | |
7c673cae | 490 | struct aesni_gcm_private *internals; |
9f95a23c TL |
491 | enum aesni_gcm_vector_mode vector_mode; |
492 | MB_MGR *mb_mgr; | |
7c673cae FG |
493 | |
494 | /* Check CPU for support for AES instruction set */ | |
495 | if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { | |
9f95a23c | 496 | AESNI_GCM_LOG(ERR, "AES instructions not supported by CPU"); |
7c673cae FG |
497 | return -EFAULT; |
498 | } | |
9f95a23c | 499 | dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); |
7c673cae | 500 | if (dev == NULL) { |
9f95a23c TL |
501 | AESNI_GCM_LOG(ERR, "driver %s: create failed", |
502 | init_params->name); | |
503 | return -ENODEV; | |
7c673cae FG |
504 | } |
505 | ||
9f95a23c TL |
506 | /* Check CPU for supported vector instruction set */ |
507 | if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) | |
508 | vector_mode = RTE_AESNI_GCM_AVX512; | |
509 | else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) | |
510 | vector_mode = RTE_AESNI_GCM_AVX2; | |
511 | else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) | |
512 | vector_mode = RTE_AESNI_GCM_AVX; | |
513 | else | |
514 | vector_mode = RTE_AESNI_GCM_SSE; | |
515 | ||
516 | dev->driver_id = cryptodev_driver_id; | |
7c673cae FG |
517 | dev->dev_ops = rte_aesni_gcm_pmd_ops; |
518 | ||
519 | /* register rx/tx burst functions for data path */ | |
520 | dev->dequeue_burst = aesni_gcm_pmd_dequeue_burst; | |
521 | dev->enqueue_burst = aesni_gcm_pmd_enqueue_burst; | |
522 | ||
523 | dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | | |
524 | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | | |
11fdf7f2 | 525 | RTE_CRYPTODEV_FF_CPU_AESNI | |
9f95a23c TL |
526 | RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | |
527 | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; | |
528 | ||
529 | mb_mgr = alloc_mb_mgr(0); | |
530 | if (mb_mgr == NULL) | |
531 | return -ENOMEM; | |
532 | ||
533 | switch (vector_mode) { | |
534 | case RTE_AESNI_GCM_SSE: | |
535 | dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; | |
536 | init_mb_mgr_sse(mb_mgr); | |
537 | break; | |
538 | case RTE_AESNI_GCM_AVX: | |
539 | dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; | |
540 | init_mb_mgr_avx(mb_mgr); | |
541 | break; | |
542 | case RTE_AESNI_GCM_AVX2: | |
543 | dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; | |
544 | init_mb_mgr_avx2(mb_mgr); | |
545 | break; | |
546 | case RTE_AESNI_GCM_AVX512: | |
547 | dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; | |
548 | init_mb_mgr_avx512(mb_mgr); | |
549 | break; | |
550 | default: | |
551 | AESNI_GCM_LOG(ERR, "Unsupported vector mode %u\n", vector_mode); | |
552 | goto error_exit; | |
553 | } | |
7c673cae | 554 | |
7c673cae FG |
555 | internals = dev->data->dev_private; |
556 | ||
9f95a23c TL |
557 | internals->vector_mode = vector_mode; |
558 | internals->mb_mgr = mb_mgr; | |
559 | ||
560 | /* Set arch independent function pointers, based on key size */ | |
561 | internals->ops[GCM_KEY_128].enc = mb_mgr->gcm128_enc; | |
562 | internals->ops[GCM_KEY_128].dec = mb_mgr->gcm128_dec; | |
563 | internals->ops[GCM_KEY_128].pre = mb_mgr->gcm128_pre; | |
564 | internals->ops[GCM_KEY_128].init = mb_mgr->gcm128_init; | |
565 | internals->ops[GCM_KEY_128].update_enc = mb_mgr->gcm128_enc_update; | |
566 | internals->ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update; | |
567 | internals->ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize; | |
568 | internals->ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize; | |
569 | ||
570 | internals->ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc; | |
571 | internals->ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec; | |
572 | internals->ops[GCM_KEY_192].pre = mb_mgr->gcm192_pre; | |
573 | internals->ops[GCM_KEY_192].init = mb_mgr->gcm192_init; | |
574 | internals->ops[GCM_KEY_192].update_enc = mb_mgr->gcm192_enc_update; | |
575 | internals->ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update; | |
576 | internals->ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize; | |
577 | internals->ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize; | |
578 | ||
579 | internals->ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc; | |
580 | internals->ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec; | |
581 | internals->ops[GCM_KEY_256].pre = mb_mgr->gcm256_pre; | |
582 | internals->ops[GCM_KEY_256].init = mb_mgr->gcm256_init; | |
583 | internals->ops[GCM_KEY_256].update_enc = mb_mgr->gcm256_enc_update; | |
584 | internals->ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update; | |
585 | internals->ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize; | |
586 | internals->ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize; | |
587 | ||
7c673cae | 588 | internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; |
9f95a23c TL |
589 | |
590 | #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) | |
591 | AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: %s\n", | |
592 | imb_get_version_str()); | |
593 | #else | |
594 | AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: 0.49.0\n"); | |
595 | #endif | |
7c673cae FG |
596 | |
597 | return 0; | |
598 | ||
9f95a23c TL |
599 | error_exit: |
600 | if (mb_mgr) | |
601 | free_mb_mgr(mb_mgr); | |
7c673cae | 602 | |
9f95a23c TL |
603 | rte_cryptodev_pmd_destroy(dev); |
604 | ||
605 | return -1; | |
7c673cae FG |
606 | } |
607 | ||
608 | static int | |
11fdf7f2 | 609 | aesni_gcm_probe(struct rte_vdev_device *vdev) |
7c673cae | 610 | { |
9f95a23c TL |
611 | struct rte_cryptodev_pmd_init_params init_params = { |
612 | "", | |
613 | sizeof(struct aesni_gcm_private), | |
11fdf7f2 | 614 | rte_socket_id(), |
9f95a23c | 615 | RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS |
7c673cae | 616 | }; |
11fdf7f2 TL |
617 | const char *name; |
618 | const char *input_args; | |
7c673cae | 619 | |
11fdf7f2 TL |
620 | name = rte_vdev_device_name(vdev); |
621 | if (name == NULL) | |
622 | return -EINVAL; | |
623 | input_args = rte_vdev_device_args(vdev); | |
9f95a23c | 624 | rte_cryptodev_pmd_parse_input_args(&init_params, input_args); |
7c673cae | 625 | |
11fdf7f2 | 626 | return aesni_gcm_create(name, vdev, &init_params); |
7c673cae FG |
627 | } |
628 | ||
629 | static int | |
11fdf7f2 | 630 | aesni_gcm_remove(struct rte_vdev_device *vdev) |
7c673cae | 631 | { |
9f95a23c TL |
632 | struct rte_cryptodev *cryptodev; |
633 | struct aesni_gcm_private *internals; | |
11fdf7f2 TL |
634 | const char *name; |
635 | ||
636 | name = rte_vdev_device_name(vdev); | |
7c673cae FG |
637 | if (name == NULL) |
638 | return -EINVAL; | |
639 | ||
9f95a23c TL |
640 | cryptodev = rte_cryptodev_pmd_get_named_dev(name); |
641 | if (cryptodev == NULL) | |
642 | return -ENODEV; | |
7c673cae | 643 | |
9f95a23c TL |
644 | internals = cryptodev->data->dev_private; |
645 | ||
646 | free_mb_mgr(internals->mb_mgr); | |
647 | ||
648 | return rte_cryptodev_pmd_destroy(cryptodev); | |
7c673cae FG |
649 | } |
650 | ||
651 | static struct rte_vdev_driver aesni_gcm_pmd_drv = { | |
652 | .probe = aesni_gcm_probe, | |
653 | .remove = aesni_gcm_remove | |
654 | }; | |
655 | ||
9f95a23c TL |
656 | static struct cryptodev_driver aesni_gcm_crypto_drv; |
657 | ||
7c673cae FG |
658 | RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD, aesni_gcm_pmd_drv); |
659 | RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd); | |
660 | RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, | |
661 | "max_nb_queue_pairs=<int> " | |
7c673cae | 662 | "socket_id=<int>"); |
9f95a23c TL |
663 | RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv.driver, |
664 | cryptodev_driver_id); | |
665 | ||
666 | ||
667 | RTE_INIT(aesni_gcm_init_log) | |
668 | { | |
669 | aesni_gcm_logtype_driver = rte_log_register("pmd.crypto.aesni_gcm"); | |
670 | } |