]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/drivers/crypto/armv8/rte_armv8_pmd.c
projects / ceph.git / blob
? search:


0d4649adcd1d87a917e4fb80fb09fd54884203b5
[ceph.git] / ceph / src / spdk / dpdk / drivers / crypto / armv8 / rte_armv8_pmd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 */
4
5 #include <stdbool.h>
6
7 #include <rte_common.h>
8 #include <rte_hexdump.h>
9 #include <rte_cryptodev.h>
10 #include <rte_cryptodev_pmd.h>
11 #include <rte_bus_vdev.h>
12 #include <rte_malloc.h>
13 #include <rte_cpuflags.h>
14
15 #include "armv8_crypto_defs.h"
16
17 #include "rte_armv8_pmd_private.h"
18
19 static uint8_t cryptodev_driver_id;
20
21 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev);
22
23 /**
24 * Pointers to the supported combined mode crypto functions are stored
25 * in the static tables. Each combined (chained) cryptographic operation
26 * can be described by a set of numbers:
27 * - order: order of operations (cipher, auth) or (auth, cipher)
28 * - direction: encryption or decryption
29 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc.
30 * - aalg: authentication algorithm such as SHA1, SHA256, etc.
31 * - keyl: cipher key length, for example 128, 192, 256 bits
32 *
33 * In order to quickly acquire each function pointer based on those numbers,
34 * a hierarchy of arrays is maintained. The final level, 3D array is indexed
35 * by the combined mode function parameters only (cipher algorithm,
36 * authentication algorithm and key length).
37 *
38 * This gives 3 memory accesses to obtain a function pointer instead of
39 * traversing the array manually and comparing function parameters on each loop.
40 *
41 * +--+CRYPTO_FUNC
42 * +--+ENC|
43 * +--+CA|
44 * | +--+DEC
45 * ORDER|
46 * | +--+ENC
47 * +--+AC|
48 * +--+DEC
49 *
50 */
51
52 /**
53 * 3D array type for ARM Combined Mode crypto functions pointers.
54 * CRYPTO_CIPHER_MAX: max cipher ID number
55 * CRYPTO_AUTH_MAX: max auth ID number
56 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
57 */
58 typedef const crypto_func_t
59 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
60
61 /* Evaluate to key length definition */
62 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl)
63
64 /* Local aliases for supported ciphers */
65 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC
66 /* Local aliases for supported hashes */
67 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC
68 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC
69
70 /**
71 * Arrays containing pointers to particular cryptographic,
72 * combined mode functions.
73 * crypto_op_ca_encrypt: cipher (encrypt), authenticate
74 * crypto_op_ca_decrypt: cipher (decrypt), authenticate
75 * crypto_op_ac_encrypt: authenticate, cipher (encrypt)
76 * crypto_op_ac_decrypt: authenticate, cipher (decrypt)
77 */
78 static const crypto_func_tbl_t
79 crypto_op_ca_encrypt = {
80 /* [cipher alg][auth alg][key length] = crypto_function, */
81 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac,
82 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac,
83 };
84
85 static const crypto_func_tbl_t
86 crypto_op_ca_decrypt = {
87 NULL
88 };
89
90 static const crypto_func_tbl_t
91 crypto_op_ac_encrypt = {
92 NULL
93 };
94
95 static const crypto_func_tbl_t
96 crypto_op_ac_decrypt = {
97 /* [cipher alg][auth alg][key length] = crypto_function, */
98 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec,
99 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec,
100 };
101
102 /**
103 * Arrays containing pointers to particular cryptographic function sets,
104 * covering given cipher operation directions (encrypt, decrypt)
105 * for each order of cipher and authentication pairs.
106 */
107 static const crypto_func_tbl_t *
108 crypto_cipher_auth[] = {
109 &crypto_op_ca_encrypt,
110 &crypto_op_ca_decrypt,
111 NULL
112 };
113
114 static const crypto_func_tbl_t *
115 crypto_auth_cipher[] = {
116 &crypto_op_ac_encrypt,
117 &crypto_op_ac_decrypt,
118 NULL
119 };
120
121 /**
122 * Top level array containing pointers to particular cryptographic
123 * function sets, covering given order of chained operations.
124 * crypto_cipher_auth: cipher first, authenticate after
125 * crypto_auth_cipher: authenticate first, cipher after
126 */
127 static const crypto_func_tbl_t **
128 crypto_chain_order[] = {
129 crypto_cipher_auth,
130 crypto_auth_cipher,
131 NULL
132 };
133
134 /**
135 * Extract particular combined mode crypto function from the 3D array.
136 */
137 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \
138 ({ \
139 crypto_func_tbl_t *func_tbl = \
140 (crypto_chain_order[(order)])[(cop)]; \
141 \
142 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \
143 })
144
145 /*----------------------------------------------------------------------------*/
146
147 /**
148 * 2D array type for ARM key schedule functions pointers.
149 * CRYPTO_CIPHER_MAX: max cipher ID number
150 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
151 */
152 typedef const crypto_key_sched_t
153 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
154
155 static const crypto_key_sched_tbl_t
156 crypto_key_sched_encrypt = {
157 /* [cipher alg][key length] = key_expand_func, */
158 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc,
159 };
160
161 static const crypto_key_sched_tbl_t
162 crypto_key_sched_decrypt = {
163 /* [cipher alg][key length] = key_expand_func, */
164 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec,
165 };
166
167 /**
168 * Top level array containing pointers to particular key generation
169 * function sets, covering given operation direction.
170 * crypto_key_sched_encrypt: keys for encryption
171 * crypto_key_sched_decrypt: keys for decryption
172 */
173 static const crypto_key_sched_tbl_t *
174 crypto_key_sched_dir[] = {
175 &crypto_key_sched_encrypt,
176 &crypto_key_sched_decrypt,
177 NULL
178 };
179
180 /**
181 * Extract particular combined mode crypto function from the 3D array.
182 */
183 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \
184 ({ \
185 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \
186 \
187 ((*ks_tbl)[(calg)][KEYL(keyl)]); \
188 })
189
190 /*----------------------------------------------------------------------------*/
191
192 /*
193 *------------------------------------------------------------------------------
194 * Session Prepare
195 *------------------------------------------------------------------------------
196 */
197
198 /** Get xform chain order */
199 static enum armv8_crypto_chain_order
200 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
201 {
202
203 /*
204 * This driver currently covers only chained operations.
205 * Ignore only cipher or only authentication operations
206 * or chains longer than 2 xform structures.
207 */
208 if (xform->next == NULL || xform->next->next != NULL)
209 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
210
211 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
212 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
213 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER;
214 }
215
216 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
217 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
218 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH;
219 }
220
221 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
222 }
223
224 static inline void
225 auth_hmac_pad_prepare(struct armv8_crypto_session *sess,
226 const struct rte_crypto_sym_xform *xform)
227 {
228 size_t i;
229
230 /* Generate i_key_pad and o_key_pad */
231 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad));
232 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key,
233 xform->auth.key.length);
234 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad));
235 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key,
236 xform->auth.key.length);
237 /*
238 * XOR key with IPAD/OPAD values to obtain i_key_pad
239 * and o_key_pad.
240 * Byte-by-byte operation may seem to be the less efficient
241 * here but in fact it's the opposite.
242 * The result ASM code is likely operate on NEON registers
243 * (load auth key to Qx, load IPAD/OPAD to multiple
244 * elements of Qy, eor 128 bits at once).
245 */
246 for (i = 0; i < SHA_BLOCK_MAX; i++) {
247 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE;
248 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE;
249 }
250 }
251
252 static inline int
253 auth_set_prerequisites(struct armv8_crypto_session *sess,
254 const struct rte_crypto_sym_xform *xform)
255 {
256 uint8_t partial[64] = { 0 };
257 int error;
258
259 switch (xform->auth.algo) {
260 case RTE_CRYPTO_AUTH_SHA1_HMAC:
261 /*
262 * Generate authentication key, i_key_pad and o_key_pad.
263 */
264 /* Zero memory under key */
265 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE);
266
267 /*
268 * Now copy the given authentication key to the session
269 * key.
270 */
271 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
272 xform->auth.key.length);
273
274 /* Prepare HMAC padding: key|pattern */
275 auth_hmac_pad_prepare(sess, xform);
276 /*
277 * Calculate partial hash values for i_key_pad and o_key_pad.
278 * Will be used as initialization state for final HMAC.
279 */
280 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad,
281 partial, SHA1_BLOCK_SIZE);
282 if (error != 0)
283 return -1;
284 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE);
285
286 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad,
287 partial, SHA1_BLOCK_SIZE);
288 if (error != 0)
289 return -1;
290 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE);
291
292 break;
293 case RTE_CRYPTO_AUTH_SHA256_HMAC:
294 /*
295 * Generate authentication key, i_key_pad and o_key_pad.
296 */
297 /* Zero memory under key */
298 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE);
299
300 /*
301 * Now copy the given authentication key to the session
302 * key.
303 */
304 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
305 xform->auth.key.length);
306
307 /* Prepare HMAC padding: key|pattern */
308 auth_hmac_pad_prepare(sess, xform);
309 /*
310 * Calculate partial hash values for i_key_pad and o_key_pad.
311 * Will be used as initialization state for final HMAC.
312 */
313 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad,
314 partial, SHA256_BLOCK_SIZE);
315 if (error != 0)
316 return -1;
317 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE);
318
319 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad,
320 partial, SHA256_BLOCK_SIZE);
321 if (error != 0)
322 return -1;
323 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE);
324
325 break;
326 default:
327 break;
328 }
329
330 return 0;
331 }
332
333 static inline int
334 cipher_set_prerequisites(struct armv8_crypto_session *sess,
335 const struct rte_crypto_sym_xform *xform)
336 {
337 crypto_key_sched_t cipher_key_sched;
338
339 cipher_key_sched = sess->cipher.key_sched;
340 if (likely(cipher_key_sched != NULL)) {
341 /* Set up cipher session key */
342 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data);
343 }
344
345 return 0;
346 }
347
348 static int
349 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess,
350 const struct rte_crypto_sym_xform *cipher_xform,
351 const struct rte_crypto_sym_xform *auth_xform)
352 {
353 enum armv8_crypto_chain_order order;
354 enum armv8_crypto_cipher_operation cop;
355 enum rte_crypto_cipher_algorithm calg;
356 enum rte_crypto_auth_algorithm aalg;
357
358 /* Validate and prepare scratch order of combined operations */
359 switch (sess->chain_order) {
360 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
361 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
362 order = sess->chain_order;
363 break;
364 default:
365 return -ENOTSUP;
366 }
367 /* Select cipher direction */
368 sess->cipher.direction = cipher_xform->cipher.op;
369 /* Select cipher key */
370 sess->cipher.key.length = cipher_xform->cipher.key.length;
371 /* Set cipher direction */
372 cop = sess->cipher.direction;
373 /* Set cipher algorithm */
374 calg = cipher_xform->cipher.algo;
375
376 /* Select cipher algo */
377 switch (calg) {
378 /* Cover supported cipher algorithms */
379 case RTE_CRYPTO_CIPHER_AES_CBC:
380 sess->cipher.algo = calg;
381 /* IV len is always 16 bytes (block size) for AES CBC */
382 sess->cipher.iv.length = 16;
383 break;
384 default:
385 return -ENOTSUP;
386 }
387 /* Select auth generate/verify */
388 sess->auth.operation = auth_xform->auth.op;
389
390 /* Select auth algo */
391 switch (auth_xform->auth.algo) {
392 /* Cover supported hash algorithms */
393 case RTE_CRYPTO_AUTH_SHA1_HMAC:
394 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */
395 aalg = auth_xform->auth.algo;
396 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC;
397 break;
398 default:
399 return -ENOTSUP;
400 }
401
402 /* Set the digest length */
403 sess->auth.digest_length = auth_xform->auth.digest_length;
404
405 /* Verify supported key lengths and extract proper algorithm */
406 switch (cipher_xform->cipher.key.length << 3) {
407 case 128:
408 sess->crypto_func =
409 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128);
410 sess->cipher.key_sched =
411 CRYPTO_GET_KEY_SCHED(cop, calg, 128);
412 break;
413 case 192:
414 case 256:
415 /* These key lengths are not supported yet */
416 default: /* Fall through */
417 sess->crypto_func = NULL;
418 sess->cipher.key_sched = NULL;
419 return -ENOTSUP;
420 }
421
422 if (unlikely(sess->crypto_func == NULL)) {
423 /*
424 * If we got here that means that there must be a bug
425 * in the algorithms selection above. Nevertheless keep
426 * it here to catch bug immediately and avoid NULL pointer
427 * dereference in OPs processing.
428 */
429 ARMV8_CRYPTO_LOG_ERR(
430 "No appropriate crypto function for given parameters");
431 return -EINVAL;
432 }
433
434 /* Set up cipher session prerequisites */
435 if (cipher_set_prerequisites(sess, cipher_xform) != 0)
436 return -EINVAL;
437
438 /* Set up authentication session prerequisites */
439 if (auth_set_prerequisites(sess, auth_xform) != 0)
440 return -EINVAL;
441
442 return 0;
443 }
444
445 /** Parse crypto xform chain and set private session parameters */
446 int
447 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess,
448 const struct rte_crypto_sym_xform *xform)
449 {
450 const struct rte_crypto_sym_xform *cipher_xform = NULL;
451 const struct rte_crypto_sym_xform *auth_xform = NULL;
452 bool is_chained_op;
453 int ret;
454
455 /* Filter out spurious/broken requests */
456 if (xform == NULL)
457 return -EINVAL;
458
459 sess->chain_order = armv8_crypto_get_chain_order(xform);
460 switch (sess->chain_order) {
461 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
462 cipher_xform = xform;
463 auth_xform = xform->next;
464 is_chained_op = true;
465 break;
466 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
467 auth_xform = xform;
468 cipher_xform = xform->next;
469 is_chained_op = true;
470 break;
471 default:
472 is_chained_op = false;
473 return -ENOTSUP;
474 }
475
476 /* Set IV offset */
477 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset;
478
479 if (is_chained_op) {
480 ret = armv8_crypto_set_session_chained_parameters(sess,
481 cipher_xform, auth_xform);
482 if (unlikely(ret != 0)) {
483 ARMV8_CRYPTO_LOG_ERR(
484 "Invalid/unsupported chained (cipher/auth) parameters");
485 return ret;
486 }
487 } else {
488 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation");
489 return -ENOTSUP;
490 }
491
492 return 0;
493 }
494
495 /** Provide session for operation */
496 static inline struct armv8_crypto_session *
497 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
498 {
499 struct armv8_crypto_session *sess = NULL;
500
501 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
502 /* get existing session */
503 if (likely(op->sym->session != NULL)) {
504 sess = (struct armv8_crypto_session *)
505 get_sym_session_private_data(
506 op->sym->session,
507 cryptodev_driver_id);
508 }
509 } else {
510 /* provide internal session */
511 void *_sess = NULL;
512 void *_sess_private_data = NULL;
513
514 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
515 return NULL;
516
517 if (rte_mempool_get(qp->sess_mp_priv,
518 (void **)&_sess_private_data))
519 return NULL;
520
521 sess = (struct armv8_crypto_session *)_sess_private_data;
522
523 if (unlikely(armv8_crypto_set_session_parameters(sess,
524 op->sym->xform) != 0)) {
525 rte_mempool_put(qp->sess_mp, _sess);
526 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
527 sess = NULL;
528 }
529 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
530 set_sym_session_private_data(op->sym->session,
531 cryptodev_driver_id, _sess_private_data);
532 }
533
534 if (unlikely(sess == NULL))
535 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
536
537 return sess;
538 }
539
540 /*
541 *------------------------------------------------------------------------------
542 * Process Operations
543 *------------------------------------------------------------------------------
544 */
545
546 /*----------------------------------------------------------------------------*/
547
548 /** Process cipher operation */
549 static inline void
550 process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
551 struct armv8_crypto_session *sess,
552 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
553 {
554 crypto_func_t crypto_func;
555 crypto_arg_t arg;
556 struct rte_mbuf *m_asrc, *m_adst;
557 uint8_t *csrc, *cdst;
558 uint8_t *adst, *asrc;
559 uint64_t clen, alen;
560 int error;
561
562 clen = op->sym->cipher.data.length;
563 alen = op->sym->auth.data.length;
564
565 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
566 op->sym->cipher.data.offset);
567 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
568 op->sym->cipher.data.offset);
569
570 switch (sess->chain_order) {
571 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
572 m_asrc = m_adst = mbuf_dst;
573 break;
574 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
575 m_asrc = mbuf_src;
576 m_adst = mbuf_dst;
577 break;
578 default:
579 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
580 return;
581 }
582 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *,
583 op->sym->auth.data.offset);
584
585 switch (sess->auth.mode) {
586 case ARMV8_CRYPTO_AUTH_AS_AUTH:
587 /* Nothing to do here, just verify correct option */
588 break;
589 case ARMV8_CRYPTO_AUTH_AS_HMAC:
590 arg.digest.hmac.key = sess->auth.hmac.key;
591 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad;
592 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad;
593 break;
594 default:
595 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
596 return;
597 }
598
599 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) {
600 adst = op->sym->auth.digest.data;
601 if (adst == NULL) {
602 adst = rte_pktmbuf_mtod_offset(m_adst,
603 uint8_t *,
604 op->sym->auth.data.offset +
605 op->sym->auth.data.length);
606 }
607 } else {
608 adst = qp->temp_digest;
609 }
610
611 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *,
612 sess->cipher.iv.offset);
613 arg.cipher.key = sess->cipher.key.data;
614 /* Acquire combined mode function */
615 crypto_func = sess->crypto_func;
616 ARMV8_CRYPTO_ASSERT(crypto_func != NULL);
617 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg);
618 if (error != 0) {
619 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
620 return;
621 }
622
623 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
624 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
625 if (memcmp(adst, op->sym->auth.digest.data,
626 sess->auth.digest_length) != 0) {
627 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
628 }
629 }
630 }
631
632 /** Process crypto operation for mbuf */
633 static inline int
634 process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
635 struct armv8_crypto_session *sess)
636 {
637 struct rte_mbuf *msrc, *mdst;
638
639 msrc = op->sym->m_src;
640 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
641
642 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
643
644 switch (sess->chain_order) {
645 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
646 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */
647 process_armv8_chained_op(qp, op, sess, msrc, mdst);
648 break;
649 default:
650 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
651 break;
652 }
653
654 /* Free session if a session-less crypto op */
655 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
656 memset(sess, 0, sizeof(struct armv8_crypto_session));
657 memset(op->sym->session, 0,
658 rte_cryptodev_sym_get_existing_header_session_size(
659 op->sym->session));
660 rte_mempool_put(qp->sess_mp, sess);
661 rte_mempool_put(qp->sess_mp_priv, op->sym->session);
662 op->sym->session = NULL;
663 }
664
665 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
666 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
667
668 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR))
669 return -1;
670
671 return 0;
672 }
673
674 /*
675 *------------------------------------------------------------------------------
676 * PMD Framework
677 *------------------------------------------------------------------------------
678 */
679
680 /** Enqueue burst */
681 static uint16_t
682 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
683 uint16_t nb_ops)
684 {
685 struct armv8_crypto_session *sess;
686 struct armv8_crypto_qp *qp = queue_pair;
687 int i, retval;
688
689 for (i = 0; i < nb_ops; i++) {
690 sess = get_session(qp, ops[i]);
691 if (unlikely(sess == NULL))
692 goto enqueue_err;
693
694 retval = process_op(qp, ops[i], sess);
695 if (unlikely(retval < 0))
696 goto enqueue_err;
697 }
698
699 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
700 NULL);
701 qp->stats.enqueued_count += retval;
702
703 return retval;
704
705 enqueue_err:
706 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
707 NULL);
708 if (ops[i] != NULL)
709 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
710
711 qp->stats.enqueue_err_count++;
712 return retval;
713 }
714
715 /** Dequeue burst */
716 static uint16_t
717 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
718 uint16_t nb_ops)
719 {
720 struct armv8_crypto_qp *qp = queue_pair;
721
722 unsigned int nb_dequeued = 0;
723
724 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
725 (void **)ops, nb_ops, NULL);
726 qp->stats.dequeued_count += nb_dequeued;
727
728 return nb_dequeued;
729 }
730
731 /** Create ARMv8 crypto device */
732 static int
733 cryptodev_armv8_crypto_create(const char *name,
734 struct rte_vdev_device *vdev,
735 struct rte_cryptodev_pmd_init_params *init_params)
736 {
737 struct rte_cryptodev *dev;
738 struct armv8_crypto_private *internals;
739
740 /* Check CPU for support for AES instruction set */
741 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
742 ARMV8_CRYPTO_LOG_ERR(
743 "AES instructions not supported by CPU");
744 return -EFAULT;
745 }
746
747 /* Check CPU for support for SHA instruction set */
748 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) ||
749 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) {
750 ARMV8_CRYPTO_LOG_ERR(
751 "SHA1/SHA2 instructions not supported by CPU");
752 return -EFAULT;
753 }
754
755 /* Check CPU for support for Advance SIMD instruction set */
756 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
757 ARMV8_CRYPTO_LOG_ERR(
758 "Advanced SIMD instructions not supported by CPU");
759 return -EFAULT;
760 }
761
762 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
763 if (dev == NULL) {
764 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
765 goto init_error;
766 }
767
768 dev->driver_id = cryptodev_driver_id;
769 dev->dev_ops = rte_armv8_crypto_pmd_ops;
770
771 /* register rx/tx burst functions for data path */
772 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst;
773 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst;
774
775 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
776 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
777 RTE_CRYPTODEV_FF_CPU_NEON |
778 RTE_CRYPTODEV_FF_CPU_ARM_CE;
779
780 /* Set vector instructions mode supported */
781 internals = dev->data->dev_private;
782
783 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
784
785 return 0;
786
787 init_error:
788 ARMV8_CRYPTO_LOG_ERR(
789 "driver %s: cryptodev_armv8_crypto_create failed",
790 init_params->name);
791
792 cryptodev_armv8_crypto_uninit(vdev);
793 return -EFAULT;
794 }
795
796 /** Initialise ARMv8 crypto device */
797 static int
798 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
799 {
800 struct rte_cryptodev_pmd_init_params init_params = {
801 "",
802 sizeof(struct armv8_crypto_private),
803 rte_socket_id(),
804 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
805 };
806 const char *name;
807 const char *input_args;
808
809 name = rte_vdev_device_name(vdev);
810 if (name == NULL)
811 return -EINVAL;
812 input_args = rte_vdev_device_args(vdev);
813 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
814
815 return cryptodev_armv8_crypto_create(name, vdev, &init_params);
816 }
817
818 /** Uninitialise ARMv8 crypto device */
819 static int
820 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev)
821 {
822 struct rte_cryptodev *cryptodev;
823 const char *name;
824
825 name = rte_vdev_device_name(vdev);
826 if (name == NULL)
827 return -EINVAL;
828
829 RTE_LOG(INFO, PMD,
830 "Closing ARMv8 crypto device %s on numa socket %u\n",
831 name, rte_socket_id());
832
833 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
834 if (cryptodev == NULL)
835 return -ENODEV;
836
837 return rte_cryptodev_pmd_destroy(cryptodev);
838 }
839
840 static struct rte_vdev_driver armv8_crypto_pmd_drv = {
841 .probe = cryptodev_armv8_crypto_init,
842 .remove = cryptodev_armv8_crypto_uninit
843 };
844
845 static struct cryptodev_driver armv8_crypto_drv;
846
847 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv);
848 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
849 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
850 "max_nb_queue_pairs=<int> "
851 "socket_id=<int>");
852 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv.driver,
853 cryptodev_driver_id);
Ceph