]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / spdk / dpdk / drivers / crypto / aesni_mb / rte_aesni_mb_pmd.c
CommitLineData
11fdf7f2
TL		 1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2017 Intel Corporation
3 */
4
5#include <intel-ipsec-mb.h>
6
7#include <rte_common.h>
8#include <rte_hexdump.h>
9#include <rte_cryptodev.h>
10#include <rte_cryptodev_pmd.h>
11#include <rte_bus_vdev.h>
12#include <rte_malloc.h>
13#include <rte_cpuflags.h>
14
15#include "rte_aesni_mb_pmd_private.h"
16
9f95a23c
TL		 17#define AES_CCM_DIGEST_MIN_LEN 4
18#define AES_CCM_DIGEST_MAX_LEN 16
19#define HMAC_MAX_BLOCK_SIZE 128
11fdf7f2
TL		 20static uint8_t cryptodev_driver_id;
21
22typedef void (*hash_one_block_t)(const void *data, void *digest);
23typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
24
25/**
26 * Calculate the authentication pre-computes
27 *
28 * @param one_block_hash Function pointer to calculate digest on ipad/opad
29 * @param ipad Inner pad output byte array
30 * @param opad Outer pad output byte array
31 * @param hkey Authentication key
32 * @param hkey_len Authentication key length
33 * @param blocksize Block size of selected hash algo
34 */
35static void
36calculate_auth_precomputes(hash_one_block_t one_block_hash,
37 uint8_t *ipad, uint8_t *opad,
38 uint8_t *hkey, uint16_t hkey_len,
39 uint16_t blocksize)
40{
41 unsigned i, length;
42
43 uint8_t ipad_buf[blocksize] __rte_aligned(16);
44 uint8_t opad_buf[blocksize] __rte_aligned(16);
45
46 /* Setup inner and outer pads */
47 memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
48 memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
49
50 /* XOR hash key with inner and outer pads */
51 length = hkey_len > blocksize ? blocksize : hkey_len;
52
53 for (i = 0; i < length; i++) {
54 ipad_buf[i] ^= hkey[i];
55 opad_buf[i] ^= hkey[i];
56 }
57
58 /* Compute partial hashes */
59 (*one_block_hash)(ipad_buf, ipad);
60 (*one_block_hash)(opad_buf, opad);
61
62 /* Clean up stack */
63 memset(ipad_buf, 0, blocksize);
64 memset(opad_buf, 0, blocksize);
65}
66
67/** Get xform chain order */
68static enum aesni_mb_operation
69aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
70{
71 if (xform == NULL)
72 return AESNI_MB_OP_NOT_SUPPORTED;
73
74 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
75 if (xform->next == NULL)
76 return AESNI_MB_OP_CIPHER_ONLY;
77 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
78 return AESNI_MB_OP_CIPHER_HASH;
79 }
80
81 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
82 if (xform->next == NULL)
83 return AESNI_MB_OP_HASH_ONLY;
84 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
85 return AESNI_MB_OP_HASH_CIPHER;
86 }
87
88 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
9f95a23c
TL		 89 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
90 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
11fdf7f2
TL		 91 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
92 return AESNI_MB_OP_AEAD_CIPHER_HASH;
93 else
94 return AESNI_MB_OP_AEAD_HASH_CIPHER;
95 }
96 }
97
98 return AESNI_MB_OP_NOT_SUPPORTED;
99}
100
101/** Set session authentication parameters */
102static int
9f95a23c 103aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
11fdf7f2
TL		 104 struct aesni_mb_session *sess,
105 const struct rte_crypto_sym_xform *xform)
106{
107 hash_one_block_t hash_oneblock_fn;
9f95a23c
TL		 108 unsigned int key_larger_block_size = 0;
109 uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
110 uint32_t auth_precompute = 1;
11fdf7f2
TL		 111
112 if (xform == NULL) {
113 sess->auth.algo = NULL_HASH;
114 return 0;
115 }
116
117 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
118 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
119 return -1;
120 }
121
9f95a23c
TL		 122 /* Set the request digest size */
123 sess->auth.req_digest_len = xform->auth.digest_length;
124
11fdf7f2
TL		 125 /* Select auth generate/verify */
126 sess->auth.operation = xform->auth.op;
127
128 /* Set Authentication Parameters */
129 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
130 sess->auth.algo = AES_XCBC;
9f95a23c
TL		 131
132 uint16_t xcbc_mac_digest_len =
133 get_truncated_digest_byte_length(AES_XCBC);
134 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
135 AESNI_MB_LOG(ERR, "Invalid digest size\n");
136 return -EINVAL;
137 }
138 sess->auth.gen_digest_len = sess->auth.req_digest_len;
139
140 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
11fdf7f2
TL		 141 sess->auth.xcbc.k1_expanded,
142 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
143 return 0;
144 }
145
146 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
9f95a23c
TL		 147 uint32_t dust[4*15];
148
11fdf7f2 149 sess->auth.algo = AES_CMAC;
11fdf7f2 150
9f95a23c
TL		 151 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
152
153 if (sess->auth.req_digest_len > cmac_digest_len) {
154 AESNI_MB_LOG(ERR, "Invalid digest size\n");
155 return -EINVAL;
156 }
157 /*
158 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
159 * in version 0.50 and sizes of 12 and 16 bytes,
160 * in version 0.49.
161 * If size requested is different, generate the full digest
162 * (16 bytes) in a temporary location and then memcpy
163 * the requested number of bytes.
164 */
165 if (sess->auth.req_digest_len < 4)
166 sess->auth.gen_digest_len = cmac_digest_len;
167 else
168 sess->auth.gen_digest_len = sess->auth.req_digest_len;
169
170 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
171 sess->auth.cmac.expkey, dust);
172 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
11fdf7f2
TL		 173 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
174 return 0;
175 }
176
9f95a23c
TL		 177 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
178 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
179 sess->cipher.direction = ENCRYPT;
180 sess->chain_order = CIPHER_HASH;
181 } else
182 sess->cipher.direction = DECRYPT;
183
184 sess->auth.algo = AES_GMAC;
185 /*
186 * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
187 * If size requested is different, generate the full digest
188 * (16 bytes) in a temporary location and then memcpy
189 * the requested number of bytes.
190 */
191 if (sess->auth.req_digest_len != 16 &&
192 sess->auth.req_digest_len != 12 &&
193 sess->auth.req_digest_len != 8) {
194 sess->auth.gen_digest_len = 16;
195 } else {
196 sess->auth.gen_digest_len = sess->auth.req_digest_len;
197 }
198 sess->iv.length = xform->auth.iv.length;
199 sess->iv.offset = xform->auth.iv.offset;
200
201 switch (xform->auth.key.length) {
202 case AES_128_BYTES:
203 IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
204 &sess->cipher.gcm_key);
205 sess->cipher.key_length_in_bytes = AES_128_BYTES;
206 break;
207 case AES_192_BYTES:
208 IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
209 &sess->cipher.gcm_key);
210 sess->cipher.key_length_in_bytes = AES_192_BYTES;
211 break;
212 case AES_256_BYTES:
213 IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
214 &sess->cipher.gcm_key);
215 sess->cipher.key_length_in_bytes = AES_256_BYTES;
216 break;
217 default:
218 RTE_LOG(ERR, PMD, "failed to parse test type\n");
219 return -EINVAL;
220 }
221
222 return 0;
223 }
11fdf7f2
TL		 224
225 switch (xform->auth.algo) {
226 case RTE_CRYPTO_AUTH_MD5_HMAC:
227 sess->auth.algo = MD5;
9f95a23c 228 hash_oneblock_fn = mb_mgr->md5_one_block;
11fdf7f2
TL		 229 break;
230 case RTE_CRYPTO_AUTH_SHA1_HMAC:
231 sess->auth.algo = SHA1;
9f95a23c
TL		 232 hash_oneblock_fn = mb_mgr->sha1_one_block;
233 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
234 IMB_SHA1(mb_mgr,
235 xform->auth.key.data,
236 xform->auth.key.length,
237 hashed_key);
238 key_larger_block_size = 1;
239 }
240 break;
241 case RTE_CRYPTO_AUTH_SHA1:
242 sess->auth.algo = PLAIN_SHA1;
243 auth_precompute = 0;
11fdf7f2
TL		 244 break;
245 case RTE_CRYPTO_AUTH_SHA224_HMAC:
246 sess->auth.algo = SHA_224;
9f95a23c
TL		 247 hash_oneblock_fn = mb_mgr->sha224_one_block;
248 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
249 IMB_SHA224(mb_mgr,
250 xform->auth.key.data,
251 xform->auth.key.length,
252 hashed_key);
253 key_larger_block_size = 1;
254 }
255 break;
256 case RTE_CRYPTO_AUTH_SHA224:
257 sess->auth.algo = PLAIN_SHA_224;
258 auth_precompute = 0;
11fdf7f2
TL		 259 break;
260 case RTE_CRYPTO_AUTH_SHA256_HMAC:
261 sess->auth.algo = SHA_256;
9f95a23c
TL		 262 hash_oneblock_fn = mb_mgr->sha256_one_block;
263 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
264 IMB_SHA256(mb_mgr,
265 xform->auth.key.data,
266 xform->auth.key.length,
267 hashed_key);
268 key_larger_block_size = 1;
269 }
270 break;
271 case RTE_CRYPTO_AUTH_SHA256:
272 sess->auth.algo = PLAIN_SHA_256;
273 auth_precompute = 0;
11fdf7f2
TL		 274 break;
275 case RTE_CRYPTO_AUTH_SHA384_HMAC:
276 sess->auth.algo = SHA_384;
9f95a23c
TL		 277 hash_oneblock_fn = mb_mgr->sha384_one_block;
278 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
279 IMB_SHA384(mb_mgr,
280 xform->auth.key.data,
281 xform->auth.key.length,
282 hashed_key);
283 key_larger_block_size = 1;
284 }
285 break;
286 case RTE_CRYPTO_AUTH_SHA384:
287 sess->auth.algo = PLAIN_SHA_384;
288 auth_precompute = 0;
11fdf7f2
TL		 289 break;
290 case RTE_CRYPTO_AUTH_SHA512_HMAC:
291 sess->auth.algo = SHA_512;
9f95a23c
TL		 292 hash_oneblock_fn = mb_mgr->sha512_one_block;
293 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
294 IMB_SHA512(mb_mgr,
295 xform->auth.key.data,
296 xform->auth.key.length,
297 hashed_key);
298 key_larger_block_size = 1;
299 }
300 break;
301 case RTE_CRYPTO_AUTH_SHA512:
302 sess->auth.algo = PLAIN_SHA_512;
303 auth_precompute = 0;
11fdf7f2
TL		 304 break;
305 default:
306 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
307 return -ENOTSUP;
308 }
9f95a23c
TL		 309 uint16_t trunc_digest_size =
310 get_truncated_digest_byte_length(sess->auth.algo);
311 uint16_t full_digest_size =
312 get_digest_byte_length(sess->auth.algo);
313
314 if (sess->auth.req_digest_len > full_digest_size ||
315 sess->auth.req_digest_len == 0) {
316 AESNI_MB_LOG(ERR, "Invalid digest size\n");
317 return -EINVAL;
318 }
319
320 if (sess->auth.req_digest_len != trunc_digest_size &&
321 sess->auth.req_digest_len != full_digest_size)
322 sess->auth.gen_digest_len = full_digest_size;
323 else
324 sess->auth.gen_digest_len = sess->auth.req_digest_len;
325
326 /* Plain SHA does not require precompute key */
327 if (auth_precompute == 0)
328 return 0;
11fdf7f2
TL		 329
330 /* Calculate Authentication precomputes */
9f95a23c
TL		 331 if (key_larger_block_size) {
332 calculate_auth_precomputes(hash_oneblock_fn,
333 sess->auth.pads.inner, sess->auth.pads.outer,
334 hashed_key,
335 xform->auth.key.length,
336 get_auth_algo_blocksize(sess->auth.algo));
337 } else {
338 calculate_auth_precomputes(hash_oneblock_fn,
11fdf7f2
TL		 339 sess->auth.pads.inner, sess->auth.pads.outer,
340 xform->auth.key.data,
341 xform->auth.key.length,
342 get_auth_algo_blocksize(sess->auth.algo));
9f95a23c 343 }
11fdf7f2
TL		 344
345 return 0;
346}
347
348/** Set session cipher parameters */
349static int
9f95a23c 350aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
11fdf7f2
TL		 351 struct aesni_mb_session *sess,
352 const struct rte_crypto_sym_xform *xform)
353{
354 uint8_t is_aes = 0;
355 uint8_t is_3DES = 0;
11fdf7f2
TL		 356
357 if (xform == NULL) {
358 sess->cipher.mode = NULL_CIPHER;
359 return 0;
360 }
361
362 if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
363 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
364 return -EINVAL;
365 }
366
367 /* Select cipher direction */
368 switch (xform->cipher.op) {
369 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
370 sess->cipher.direction = ENCRYPT;
371 break;
372 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
373 sess->cipher.direction = DECRYPT;
374 break;
375 default:
376 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
377 return -EINVAL;
378 }
379
380 /* Select cipher mode */
381 switch (xform->cipher.algo) {
382 case RTE_CRYPTO_CIPHER_AES_CBC:
383 sess->cipher.mode = CBC;
384 is_aes = 1;
385 break;
386 case RTE_CRYPTO_CIPHER_AES_CTR:
387 sess->cipher.mode = CNTR;
388 is_aes = 1;
389 break;
390 case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
391 sess->cipher.mode = DOCSIS_SEC_BPI;
392 is_aes = 1;
393 break;
394 case RTE_CRYPTO_CIPHER_DES_CBC:
395 sess->cipher.mode = DES;
396 break;
397 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
398 sess->cipher.mode = DOCSIS_DES;
399 break;
400 case RTE_CRYPTO_CIPHER_3DES_CBC:
401 sess->cipher.mode = DES3;
402 is_3DES = 1;
403 break;
404 default:
405 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
406 return -ENOTSUP;
407 }
408
409 /* Set IV parameters */
410 sess->iv.offset = xform->cipher.iv.offset;
411 sess->iv.length = xform->cipher.iv.length;
412
413 /* Check key length and choose key expansion function for AES */
414 if (is_aes) {
415 switch (xform->cipher.key.length) {
416 case AES_128_BYTES:
417 sess->cipher.key_length_in_bytes = AES_128_BYTES;
9f95a23c
TL		 418 IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
419 sess->cipher.expanded_aes_keys.encode,
420 sess->cipher.expanded_aes_keys.decode);
11fdf7f2
TL		 421 break;
422 case AES_192_BYTES:
423 sess->cipher.key_length_in_bytes = AES_192_BYTES;
9f95a23c
TL		 424 IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
425 sess->cipher.expanded_aes_keys.encode,
426 sess->cipher.expanded_aes_keys.decode);
11fdf7f2
TL		 427 break;
428 case AES_256_BYTES:
429 sess->cipher.key_length_in_bytes = AES_256_BYTES;
9f95a23c
TL		 430 IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
431 sess->cipher.expanded_aes_keys.encode,
432 sess->cipher.expanded_aes_keys.decode);
11fdf7f2
TL		 433 break;
434 default:
435 AESNI_MB_LOG(ERR, "Invalid cipher key length");
436 return -EINVAL;
437 }
11fdf7f2
TL		 438 } else if (is_3DES) {
439 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
440 sess->cipher.exp_3des_keys.key[1],
441 sess->cipher.exp_3des_keys.key[2]};
442
443 switch (xform->cipher.key.length) {
444 case 24:
9f95a23c
TL		 445 IMB_DES_KEYSCHED(mb_mgr, keys[0],
446 xform->cipher.key.data);
447 IMB_DES_KEYSCHED(mb_mgr, keys[1],
448 xform->cipher.key.data + 8);
449 IMB_DES_KEYSCHED(mb_mgr, keys[2],
450 xform->cipher.key.data + 16);
11fdf7f2
TL		 451
452 /* Initialize keys - 24 bytes: [K1-K2-K3] */
453 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
454 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
455 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
456 break;
457 case 16:
9f95a23c
TL		 458 IMB_DES_KEYSCHED(mb_mgr, keys[0],
459 xform->cipher.key.data);
460 IMB_DES_KEYSCHED(mb_mgr, keys[1],
461 xform->cipher.key.data + 8);
11fdf7f2
TL		 462 /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
463 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
464 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
465 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
466 break;
467 case 8:
9f95a23c
TL		 468 IMB_DES_KEYSCHED(mb_mgr, keys[0],
469 xform->cipher.key.data);
11fdf7f2
TL		 470
471 /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
472 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
473 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
474 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
475 break;
476 default:
477 AESNI_MB_LOG(ERR, "Invalid cipher key length");
478 return -EINVAL;
479 }
480
11fdf7f2 481 sess->cipher.key_length_in_bytes = 24;
11fdf7f2
TL		 482 } else {
483 if (xform->cipher.key.length != 8) {
484 AESNI_MB_LOG(ERR, "Invalid cipher key length");
485 return -EINVAL;
486 }
487 sess->cipher.key_length_in_bytes = 8;
488
9f95a23c
TL		 489 IMB_DES_KEYSCHED(mb_mgr,
490 (uint64_t *)sess->cipher.expanded_aes_keys.encode,
11fdf7f2 491 xform->cipher.key.data);
9f95a23c
TL		 492 IMB_DES_KEYSCHED(mb_mgr,
493 (uint64_t *)sess->cipher.expanded_aes_keys.decode,
11fdf7f2
TL		 494 xform->cipher.key.data);
495 }
496
497 return 0;
498}
499
500static int
9f95a23c 501aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
11fdf7f2
TL		 502 struct aesni_mb_session *sess,
503 const struct rte_crypto_sym_xform *xform)
504{
11fdf7f2
TL		 505 switch (xform->aead.op) {
506 case RTE_CRYPTO_AEAD_OP_ENCRYPT:
507 sess->cipher.direction = ENCRYPT;
508 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
509 break;
510 case RTE_CRYPTO_AEAD_OP_DECRYPT:
511 sess->cipher.direction = DECRYPT;
512 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
513 break;
514 default:
515 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
516 return -EINVAL;
517 }
518
519 switch (xform->aead.algo) {
520 case RTE_CRYPTO_AEAD_AES_CCM:
521 sess->cipher.mode = CCM;
522 sess->auth.algo = AES_CCM;
9f95a23c
TL		 523
524 /* Check key length and choose key expansion function for AES */
525 switch (xform->aead.key.length) {
526 case AES_128_BYTES:
527 sess->cipher.key_length_in_bytes = AES_128_BYTES;
528 IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
529 sess->cipher.expanded_aes_keys.encode,
530 sess->cipher.expanded_aes_keys.decode);
531 break;
532 default:
533 AESNI_MB_LOG(ERR, "Invalid cipher key length");
534 return -EINVAL;
535 }
536
537 break;
538
539 case RTE_CRYPTO_AEAD_AES_GCM:
540 sess->cipher.mode = GCM;
541 sess->auth.algo = AES_GMAC;
542
543 switch (xform->aead.key.length) {
544 case AES_128_BYTES:
545 sess->cipher.key_length_in_bytes = AES_128_BYTES;
546 IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
547 &sess->cipher.gcm_key);
548 break;
549 case AES_192_BYTES:
550 sess->cipher.key_length_in_bytes = AES_192_BYTES;
551 IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
552 &sess->cipher.gcm_key);
553 break;
554 case AES_256_BYTES:
555 sess->cipher.key_length_in_bytes = AES_256_BYTES;
556 IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
557 &sess->cipher.gcm_key);
558 break;
559 default:
560 AESNI_MB_LOG(ERR, "Invalid cipher key length");
561 return -EINVAL;
562 }
563
11fdf7f2 564 break;
9f95a23c 565
11fdf7f2
TL		 566 default:
567 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
568 return -ENOTSUP;
569 }
570
571 /* Set IV parameters */
572 sess->iv.offset = xform->aead.iv.offset;
573 sess->iv.length = xform->aead.iv.length;
574
9f95a23c
TL		 575 sess->auth.req_digest_len = xform->aead.digest_length;
576 /* CCM digests must be between 4 and 16 and an even number */
577 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
578 sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
579 (sess->auth.req_digest_len & 1) == 1) {
580 AESNI_MB_LOG(ERR, "Invalid digest size\n");
11fdf7f2
TL		 581 return -EINVAL;
582 }
9f95a23c 583 sess->auth.gen_digest_len = sess->auth.req_digest_len;
11fdf7f2
TL		 584
585 return 0;
586}
587
588/** Parse crypto xform chain and set private session parameters */
589int
9f95a23c 590aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
11fdf7f2
TL		 591 struct aesni_mb_session *sess,
592 const struct rte_crypto_sym_xform *xform)
593{
594 const struct rte_crypto_sym_xform *auth_xform = NULL;
595 const struct rte_crypto_sym_xform *cipher_xform = NULL;
596 const struct rte_crypto_sym_xform *aead_xform = NULL;
597 int ret;
598
599 /* Select Crypto operation - hash then cipher / cipher then hash */
600 switch (aesni_mb_get_chain_order(xform)) {
601 case AESNI_MB_OP_HASH_CIPHER:
602 sess->chain_order = HASH_CIPHER;
603 auth_xform = xform;
604 cipher_xform = xform->next;
11fdf7f2
TL		 605 break;
606 case AESNI_MB_OP_CIPHER_HASH:
607 sess->chain_order = CIPHER_HASH;
608 auth_xform = xform->next;
609 cipher_xform = xform;
11fdf7f2
TL		 610 break;
611 case AESNI_MB_OP_HASH_ONLY:
612 sess->chain_order = HASH_CIPHER;
613 auth_xform = xform;
614 cipher_xform = NULL;
11fdf7f2
TL		 615 break;
616 case AESNI_MB_OP_CIPHER_ONLY:
617 /*
618 * Multi buffer library operates only at two modes,
619 * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
620 * chain order depends on cipher operation: encryption is always
621 * the first operation and decryption the last one.
622 */
623 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
624 sess->chain_order = CIPHER_HASH;
625 else
626 sess->chain_order = HASH_CIPHER;
627 auth_xform = NULL;
628 cipher_xform = xform;
629 break;
630 case AESNI_MB_OP_AEAD_CIPHER_HASH:
631 sess->chain_order = CIPHER_HASH;
632 sess->aead.aad_len = xform->aead.aad_length;
11fdf7f2
TL		 633 aead_xform = xform;
634 break;
635 case AESNI_MB_OP_AEAD_HASH_CIPHER:
636 sess->chain_order = HASH_CIPHER;
637 sess->aead.aad_len = xform->aead.aad_length;
11fdf7f2
TL		 638 aead_xform = xform;
639 break;
640 case AESNI_MB_OP_NOT_SUPPORTED:
641 default:
642 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
643 return -ENOTSUP;
644 }
645
646 /* Default IV length = 0 */
647 sess->iv.length = 0;
648
9f95a23c 649 ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
11fdf7f2
TL		 650 if (ret != 0) {
651 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
652 return ret;
653 }
654
9f95a23c 655 ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
11fdf7f2
TL		 656 cipher_xform);
657 if (ret != 0) {
658 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
659 return ret;
660 }
661
662 if (aead_xform) {
9f95a23c 663 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
11fdf7f2
TL		 664 aead_xform);
665 if (ret != 0) {
666 AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
667 return ret;
668 }
669 }
670
671 return 0;
672}
673
674/**
675 * burst enqueue, place crypto operations on ingress queue for processing.
676 *
677 * @param __qp Queue Pair to process
678 * @param ops Crypto operations for processing
679 * @param nb_ops Number of crypto operations for processing
680 *
681 * @return
682 * - Number of crypto operations enqueued
683 */
684static uint16_t
685aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
686 uint16_t nb_ops)
687{
688 struct aesni_mb_qp *qp = __qp;
689
690 unsigned int nb_enqueued;
691
692 nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
693 (void **)ops, nb_ops, NULL);
694
695 qp->stats.enqueued_count += nb_enqueued;
696
697 return nb_enqueued;
698}
699
700/** Get multi buffer session */
701static inline struct aesni_mb_session *
702get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
703{
704 struct aesni_mb_session *sess = NULL;
705
706 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
707 if (likely(op->sym->session != NULL))
708 sess = (struct aesni_mb_session *)
709 get_sym_session_private_data(
710 op->sym->session,
711 cryptodev_driver_id);
712 } else {
713 void *_sess = NULL;
714 void *_sess_private_data = NULL;
715
716 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
717 return NULL;
718
9f95a23c
TL		 719 if (rte_mempool_get(qp->sess_mp_priv,
720 (void **)&_sess_private_data))
11fdf7f2
TL		 721 return NULL;
722
723 sess = (struct aesni_mb_session *)_sess_private_data;
724
9f95a23c 725 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
11fdf7f2
TL		 726 sess, op->sym->xform) != 0)) {
727 rte_mempool_put(qp->sess_mp, _sess);
9f95a23c 728 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
11fdf7f2
TL		 729 sess = NULL;
730 }
731 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
732 set_sym_session_private_data(op->sym->session,
733 cryptodev_driver_id, _sess_private_data);
734 }
735
736 if (unlikely(sess == NULL))
737 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
738
739 return sess;
740}
741
9f95a23c
TL		 742static inline uint64_t
743auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
744 uint32_t oop)
745{
746 struct rte_mbuf *m_src, *m_dst;
747 uint8_t *p_src, *p_dst;
748 uintptr_t u_src, u_dst;
749 uint32_t cipher_end, auth_end;
750
751 /* Only cipher then hash needs special calculation. */
752 if (!oop || session->chain_order != CIPHER_HASH)
753 return op->sym->auth.data.offset;
754
755 m_src = op->sym->m_src;
756 m_dst = op->sym->m_dst;
757
758 p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
759 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
760 u_src = (uintptr_t)p_src;
761 u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
762
763 /**
764 * Copy the content between cipher offset and auth offset for generating
765 * correct digest.
766 */
767 if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
768 memcpy(p_dst + op->sym->auth.data.offset,
769 p_src + op->sym->auth.data.offset,
770 op->sym->cipher.data.offset -
771 op->sym->auth.data.offset);
772
773 /**
774 * Copy the content between (cipher offset + length) and (auth offset +
775 * length) for generating correct digest
776 */
777 cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
778 auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
779 if (cipher_end < auth_end)
780 memcpy(p_dst + cipher_end, p_src + cipher_end,
781 auth_end - cipher_end);
782
783 /**
784 * Since intel-ipsec-mb only supports positive values,
785 * we need to deduct the correct offset between src and dst.
786 */
787
788 return u_src < u_dst ? (u_dst - u_src) :
789 (UINT64_MAX - u_src + u_dst + 1);
790}
791
11fdf7f2
TL		 792/**
793 * Process a crypto operation and complete a JOB_AES_HMAC job structure for
794 * submission to the multi buffer library for processing.
795 *
796 * @param qp queue pair
797 * @param job JOB_AES_HMAC structure to fill
798 * @param m mbuf to process
799 *
800 * @return
801 * - Completed JOB_AES_HMAC structure pointer on success
802 * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
803 */
11fdf7f2
TL		 804static inline int
805set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
806 struct rte_crypto_op *op, uint8_t *digest_idx)
807{
11fdf7f2 808 struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
11fdf7f2 809 struct aesni_mb_session *session;
9f95a23c 810 uint32_t m_offset, oop;
11fdf7f2
TL		 811
812 session = get_session(qp, op);
813 if (session == NULL) {
814 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
815 return -1;
816 }
817
818 /* Set crypto operation */
819 job->chain_order = session->chain_order;
820
821 /* Set cipher parameters */
822 job->cipher_direction = session->cipher.direction;
823 job->cipher_mode = session->cipher.mode;
824
825 job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
826
11fdf7f2
TL		 827 /* Set authentication parameters */
828 job->hash_alg = session->auth.algo;
9f95a23c
TL		 829
830 switch (job->hash_alg) {
831 case AES_XCBC:
11fdf7f2
TL		 832 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
833 job->u.XCBC._k2 = session->auth.xcbc.k2;
834 job->u.XCBC._k3 = session->auth.xcbc.k3;
9f95a23c
TL		 835
836 job->aes_enc_key_expanded =
837 session->cipher.expanded_aes_keys.encode;
838 job->aes_dec_key_expanded =
839 session->cipher.expanded_aes_keys.decode;
840 break;
841
842 case AES_CCM:
11fdf7f2
TL		 843 job->u.CCM.aad = op->sym->aead.aad.data + 18;
844 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
9f95a23c
TL		 845 job->aes_enc_key_expanded =
846 session->cipher.expanded_aes_keys.encode;
847 job->aes_dec_key_expanded =
848 session->cipher.expanded_aes_keys.decode;
849 break;
850
851 case AES_CMAC:
11fdf7f2
TL		 852 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
853 job->u.CMAC._skey1 = session->auth.cmac.skey1;
854 job->u.CMAC._skey2 = session->auth.cmac.skey2;
9f95a23c
TL		 855 job->aes_enc_key_expanded =
856 session->cipher.expanded_aes_keys.encode;
857 job->aes_dec_key_expanded =
858 session->cipher.expanded_aes_keys.decode;
859 break;
11fdf7f2 860
9f95a23c
TL		 861 case AES_GMAC:
862 if (session->cipher.mode == GCM) {
863 job->u.GCM.aad = op->sym->aead.aad.data;
864 job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
865 } else {
866 /* For GMAC */
867 job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
868 uint8_t *, op->sym->auth.data.offset);
869 job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
870 job->cipher_mode = GCM;
871 }
872 job->aes_enc_key_expanded = &session->cipher.gcm_key;
873 job->aes_dec_key_expanded = &session->cipher.gcm_key;
874 break;
875
876 default:
11fdf7f2
TL		 877 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
878 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
11fdf7f2 879
9f95a23c
TL		 880 if (job->cipher_mode == DES3) {
881 job->aes_enc_key_expanded =
882 session->cipher.exp_3des_keys.ks_ptr;
883 job->aes_dec_key_expanded =
884 session->cipher.exp_3des_keys.ks_ptr;
885 } else {
886 job->aes_enc_key_expanded =
887 session->cipher.expanded_aes_keys.encode;
888 job->aes_dec_key_expanded =
889 session->cipher.expanded_aes_keys.decode;
11fdf7f2 890 }
9f95a23c 891 }
11fdf7f2 892
9f95a23c
TL		 893 if (!op->sym->m_dst) {
894 /* in-place operation */
11fdf7f2 895 m_dst = m_src;
9f95a23c
TL		 896 oop = 0;
897 } else if (op->sym->m_dst == op->sym->m_src) {
898 /* in-place operation */
899 m_dst = m_src;
900 oop = 0;
901 } else {
902 /* out-of-place operation */
903 m_dst = op->sym->m_dst;
904 oop = 1;
11fdf7f2 905 }
9f95a23c
TL		 906
907 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
908 session->cipher.mode == GCM))
909 m_offset = op->sym->aead.data.offset;
910 else
911 m_offset = op->sym->cipher.data.offset;
11fdf7f2
TL		 912
913 /* Set digest output location */
914 if (job->hash_alg != NULL_HASH &&
915 session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
916 job->auth_tag_output = qp->temp_digests[*digest_idx];
917 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
918 } else {
9f95a23c
TL		 919 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
920 session->cipher.mode == GCM))
11fdf7f2
TL		 921 job->auth_tag_output = op->sym->aead.digest.data;
922 else
923 job->auth_tag_output = op->sym->auth.digest.data;
11fdf7f2 924
9f95a23c
TL		 925 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
926 job->auth_tag_output = qp->temp_digests[*digest_idx];
927 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
928 }
929 }
11fdf7f2
TL		 930 /*
931 * Multi-buffer library current only support returning a truncated
932 * digest length as specified in the relevant IPsec RFCs
933 */
11fdf7f2 934
9f95a23c
TL		 935 /* Set digest length */
936 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
11fdf7f2
TL		 937
938 /* Set IV parameters */
11fdf7f2
TL		 939 job->iv_len_in_bytes = session->iv.length;
940
9f95a23c 941 /* Data Parameters */
11fdf7f2 942 job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
11fdf7f2 943 job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
11fdf7f2 944
9f95a23c
TL		 945 switch (job->hash_alg) {
946 case AES_CCM:
11fdf7f2
TL		 947 job->cipher_start_src_offset_in_bytes =
948 op->sym->aead.data.offset;
949 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
950 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
951 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
952
953 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
954 session->iv.offset + 1);
9f95a23c
TL		 955 break;
956
957 case AES_GMAC:
958 if (session->cipher.mode == GCM) {
959 job->cipher_start_src_offset_in_bytes =
960 op->sym->aead.data.offset;
961 job->hash_start_src_offset_in_bytes =
962 op->sym->aead.data.offset;
963 job->msg_len_to_cipher_in_bytes =
964 op->sym->aead.data.length;
965 job->msg_len_to_hash_in_bytes =
966 op->sym->aead.data.length;
967 } else {
968 job->cipher_start_src_offset_in_bytes =
969 op->sym->auth.data.offset;
970 job->hash_start_src_offset_in_bytes =
971 op->sym->auth.data.offset;
972 job->msg_len_to_cipher_in_bytes = 0;
973 job->msg_len_to_hash_in_bytes = 0;
974 }
975
976 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
977 session->iv.offset);
978 break;
979
980 default:
11fdf7f2
TL		 981 job->cipher_start_src_offset_in_bytes =
982 op->sym->cipher.data.offset;
983 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
984
9f95a23c
TL		 985 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
986 session, oop);
11fdf7f2
TL		 987 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
988
989 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
990 session->iv.offset);
991 }
992
993 /* Set user data to be crypto operation data struct */
994 job->user_data = op;
995
996 return 0;
997}
998
999static inline void
9f95a23c
TL		 1000verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1001{
11fdf7f2 1002 /* Verify digest if required */
9f95a23c
TL		 1003 if (memcmp(job->auth_tag_output, digest, len) != 0)
1004 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1005}
1006
1007static inline void
1008generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1009 struct aesni_mb_session *sess)
1010{
1011 /* No extra copy needed */
1012 if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1013 return;
1014
1015 /*
1016 * This can only happen for HMAC, so only digest
1017 * for authentication algos is required
1018 */
1019 memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1020 sess->auth.req_digest_len);
11fdf7f2
TL		 1021}
1022
1023/**
1024 * Process a completed job and return rte_mbuf which job processed
1025 *
1026 * @param qp Queue Pair to process
1027 * @param job JOB_AES_HMAC job to process
1028 *
1029 * @return
1030 * - Returns processed crypto operation.
1031 * - Returns NULL on invalid job
1032 */
1033static inline struct rte_crypto_op *
1034post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1035{
1036 struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1037 struct aesni_mb_session *sess = get_sym_session_private_data(
1038 op->sym->session,
1039 cryptodev_driver_id);
1040
1041 if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1042 switch (job->status) {
1043 case STS_COMPLETED:
1044 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1045
9f95a23c
TL		 1046 if (job->hash_alg == NULL_HASH)
1047 break;
1048
1049 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1050 if (job->hash_alg == AES_CCM ||
1051 (job->hash_alg == AES_GMAC &&
1052 sess->cipher.mode == GCM))
1053 verify_digest(job,
1054 op->sym->aead.digest.data,
1055 sess->auth.req_digest_len,
1056 &op->status);
1057 else
1058 verify_digest(job,
1059 op->sym->auth.digest.data,
1060 sess->auth.req_digest_len,
1061 &op->status);
1062 } else
1063 generate_digest(job, op, sess);
11fdf7f2
TL		 1064 break;
1065 default:
1066 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1067 }
1068 }
1069
1070 /* Free session if a session-less crypto op */
1071 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1072 memset(sess, 0, sizeof(struct aesni_mb_session));
1073 memset(op->sym->session, 0,
9f95a23c
TL		 1074 rte_cryptodev_sym_get_existing_header_session_size(
1075 op->sym->session));
1076 rte_mempool_put(qp->sess_mp_priv, sess);
11fdf7f2
TL		 1077 rte_mempool_put(qp->sess_mp, op->sym->session);
1078 op->sym->session = NULL;
1079 }
1080
1081 return op;
1082}
1083
1084/**
1085 * Process a completed JOB_AES_HMAC job and keep processing jobs until
1086 * get_completed_job return NULL
1087 *
1088 * @param qp Queue Pair to process
1089 * @param job JOB_AES_HMAC job
1090 *
1091 * @return
1092 * - Number of processed jobs
1093 */
1094static unsigned
1095handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1096 struct rte_crypto_op **ops, uint16_t nb_ops)
1097{
1098 struct rte_crypto_op *op = NULL;
1099 unsigned processed_jobs = 0;
1100
1101 while (job != NULL) {
1102 op = post_process_mb_job(qp, job);
1103
1104 if (op) {
1105 ops[processed_jobs++] = op;
1106 qp->stats.dequeued_count++;
1107 } else {
1108 qp->stats.dequeue_err_count++;
1109 break;
1110 }
1111 if (processed_jobs == nb_ops)
1112 break;
1113
9f95a23c 1114 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
11fdf7f2
TL		 1115 }
1116
1117 return processed_jobs;
1118}
1119
1120static inline uint16_t
1121flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1122 uint16_t nb_ops)
1123{
1124 int processed_ops = 0;
1125
1126 /* Flush the remaining jobs */
9f95a23c 1127 JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
11fdf7f2
TL		 1128
1129 if (job)
1130 processed_ops += handle_completed_jobs(qp, job,
1131 &ops[processed_ops], nb_ops - processed_ops);
1132
1133 return processed_ops;
1134}
1135
1136static inline JOB_AES_HMAC *
1137set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1138{
1139 job->chain_order = HASH_CIPHER;
1140 job->cipher_mode = NULL_CIPHER;
1141 job->hash_alg = NULL_HASH;
1142 job->cipher_direction = DECRYPT;
1143
1144 /* Set user data to be crypto operation data struct */
1145 job->user_data = op;
1146
1147 return job;
1148}
1149
1150static uint16_t
1151aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1152 uint16_t nb_ops)
1153{
1154 struct aesni_mb_qp *qp = queue_pair;
1155
1156 struct rte_crypto_op *op;
1157 JOB_AES_HMAC *job;
1158
1159 int retval, processed_jobs = 0;
1160
1161 if (unlikely(nb_ops == 0))
1162 return 0;
1163
1164 uint8_t digest_idx = qp->digest_idx;
1165 do {
11fdf7f2 1166 /* Get next free mb job struct from mb manager */
9f95a23c 1167 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
11fdf7f2
TL		 1168 if (unlikely(job == NULL)) {
1169 /* if no free mb job structs we need to flush mb_mgr */
1170 processed_jobs += flush_mb_mgr(qp,
1171 &ops[processed_jobs],
9f95a23c 1172 nb_ops - processed_jobs);
11fdf7f2 1173
9f95a23c
TL		 1174 if (nb_ops == processed_jobs)
1175 break;
1176
1177 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
11fdf7f2
TL		 1178 }
1179
9f95a23c
TL		 1180 /*
1181 * Get next operation to process from ingress queue.
1182 * There is no need to return the job to the MB_MGR
1183 * if there are no more operations to process, since the MB_MGR
1184 * can use that pointer again in next get_next calls.
1185 */
1186 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1187 if (retval < 0)
1188 break;
1189
11fdf7f2
TL		 1190 retval = set_mb_job_params(job, qp, op, &digest_idx);
1191 if (unlikely(retval != 0)) {
1192 qp->stats.dequeue_err_count++;
1193 set_job_null_op(job, op);
1194 }
1195
1196 /* Submit job to multi-buffer for processing */
9f95a23c
TL		 1197#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1198 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1199#else
1200 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1201#endif
11fdf7f2
TL		 1202 /*
1203 * If submit returns a processed job then handle it,
1204 * before submitting subsequent jobs
1205 */
1206 if (job)
1207 processed_jobs += handle_completed_jobs(qp, job,
1208 &ops[processed_jobs],
1209 nb_ops - processed_jobs);
1210
1211 } while (processed_jobs < nb_ops);
1212
1213 qp->digest_idx = digest_idx;
1214
1215 if (processed_jobs < 1)
1216 processed_jobs += flush_mb_mgr(qp,
1217 &ops[processed_jobs],
1218 nb_ops - processed_jobs);
1219
1220 return processed_jobs;
1221}
1222
1223static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1224
1225static int
1226cryptodev_aesni_mb_create(const char *name,
1227 struct rte_vdev_device *vdev,
1228 struct rte_cryptodev_pmd_init_params *init_params)
1229{
1230 struct rte_cryptodev *dev;
1231 struct aesni_mb_private *internals;
1232 enum aesni_mb_vector_mode vector_mode;
9f95a23c 1233 MB_MGR *mb_mgr;
11fdf7f2
TL		 1234
1235 /* Check CPU for support for AES instruction set */
1236 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
1237 AESNI_MB_LOG(ERR, "AES instructions not supported by CPU");
1238 return -EFAULT;
1239 }
1240
1241 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1242 if (dev == NULL) {
1243 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1244 return -ENODEV;
1245 }
1246
1247 /* Check CPU for supported vector instruction set */
1248 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1249 vector_mode = RTE_AESNI_MB_AVX512;
1250 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1251 vector_mode = RTE_AESNI_MB_AVX2;
1252 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1253 vector_mode = RTE_AESNI_MB_AVX;
1254 else
1255 vector_mode = RTE_AESNI_MB_SSE;
1256
1257 dev->driver_id = cryptodev_driver_id;
1258 dev->dev_ops = rte_aesni_mb_pmd_ops;
1259
1260 /* register rx/tx burst functions for data path */
1261 dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1262 dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1263
1264 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1265 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
9f95a23c
TL		 1266 RTE_CRYPTODEV_FF_CPU_AESNI |
1267 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
1268
1269
1270 mb_mgr = alloc_mb_mgr(0);
1271 if (mb_mgr == NULL)
1272 return -ENOMEM;
11fdf7f2
TL		 1273
1274 switch (vector_mode) {
1275 case RTE_AESNI_MB_SSE:
1276 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
9f95a23c 1277 init_mb_mgr_sse(mb_mgr);
11fdf7f2
TL		 1278 break;
1279 case RTE_AESNI_MB_AVX:
1280 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
9f95a23c 1281 init_mb_mgr_avx(mb_mgr);
11fdf7f2
TL		 1282 break;
1283 case RTE_AESNI_MB_AVX2:
1284 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
9f95a23c 1285 init_mb_mgr_avx2(mb_mgr);
11fdf7f2
TL		 1286 break;
1287 case RTE_AESNI_MB_AVX512:
1288 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
9f95a23c 1289 init_mb_mgr_avx512(mb_mgr);
11fdf7f2
TL		 1290 break;
1291 default:
9f95a23c
TL		 1292 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1293 goto error_exit;
11fdf7f2
TL		 1294 }
1295
1296 /* Set vector instructions mode supported */
1297 internals = dev->data->dev_private;
1298
1299 internals->vector_mode = vector_mode;
1300 internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
9f95a23c 1301 internals->mb_mgr = mb_mgr;
11fdf7f2 1302
11fdf7f2
TL		 1303 AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1304 imb_get_version_str());
11fdf7f2
TL		 1305
1306 return 0;
9f95a23c
TL		 1307
1308error_exit:
1309 if (mb_mgr)
1310 free_mb_mgr(mb_mgr);
1311
1312 rte_cryptodev_pmd_destroy(dev);
1313
1314 return -1;
11fdf7f2
TL		 1315}
1316
1317static int
1318cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1319{
1320 struct rte_cryptodev_pmd_init_params init_params = {
1321 "",
1322 sizeof(struct aesni_mb_private),
1323 rte_socket_id(),
1324 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1325 };
1326 const char *name, *args;
1327 int retval;
1328
1329 name = rte_vdev_device_name(vdev);
1330 if (name == NULL)
1331 return -EINVAL;
1332
1333 args = rte_vdev_device_args(vdev);
1334
1335 retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1336 if (retval) {
1337 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1338 args);
1339 return -EINVAL;
1340 }
1341
1342 return cryptodev_aesni_mb_create(name, vdev, &init_params);
1343}
1344
1345static int
1346cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1347{
1348 struct rte_cryptodev *cryptodev;
9f95a23c 1349 struct aesni_mb_private *internals;
11fdf7f2
TL		 1350 const char *name;
1351
1352 name = rte_vdev_device_name(vdev);
1353 if (name == NULL)
1354 return -EINVAL;
1355
1356 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1357 if (cryptodev == NULL)
1358 return -ENODEV;
1359
9f95a23c
TL		 1360 internals = cryptodev->data->dev_private;
1361
1362 free_mb_mgr(internals->mb_mgr);
1363
11fdf7f2
TL		 1364 return rte_cryptodev_pmd_destroy(cryptodev);
1365}
1366
1367static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1368 .probe = cryptodev_aesni_mb_probe,
1369 .remove = cryptodev_aesni_mb_remove
1370};
1371
1372static struct cryptodev_driver aesni_mb_crypto_drv;
1373
1374RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
1375RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1376RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1377 "max_nb_queue_pairs=<int> "
1378 "socket_id=<int>");
1379RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
1380 cryptodev_aesni_mb_pmd_drv.driver,
1381 cryptodev_driver_id);
1382
1383RTE_INIT(aesni_mb_init_log)
1384{
1385 aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb");
1386}
Ceph