4 * Copyright(c) 2016 Intel Corporation. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Intel Corporation nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include <rte_common.h>
34 #include <rte_hexdump.h>
35 #include <rte_cryptodev.h>
36 #include <rte_cryptodev_pmd.h>
38 #include <rte_malloc.h>
39 #include <rte_cpuflags.h>
41 #include <openssl/evp.h>
43 #include "rte_openssl_pmd_private.h"
45 #define DES_BLOCK_SIZE 8
47 static int cryptodev_openssl_remove(struct rte_vdev_device
*vdev
);
49 /*----------------------------------------------------------------------------*/
52 * Increment counter by 1
53 * Counter is 64 bit array, big-endian
58 uint64_t *ctr64
= (uint64_t *)ctr
;
60 *ctr64
= __builtin_bswap64(*ctr64
);
62 *ctr64
= __builtin_bswap64(*ctr64
);
66 *------------------------------------------------------------------------------
68 *------------------------------------------------------------------------------
71 /** Get xform chain order */
72 static enum openssl_chain_order
73 openssl_get_chain_order(const struct rte_crypto_sym_xform
*xform
)
75 enum openssl_chain_order res
= OPENSSL_CHAIN_NOT_SUPPORTED
;
78 if (xform
->type
== RTE_CRYPTO_SYM_XFORM_AUTH
) {
79 if (xform
->next
== NULL
)
80 res
= OPENSSL_CHAIN_ONLY_AUTH
;
81 else if (xform
->next
->type
==
82 RTE_CRYPTO_SYM_XFORM_CIPHER
)
83 res
= OPENSSL_CHAIN_AUTH_CIPHER
;
85 if (xform
->type
== RTE_CRYPTO_SYM_XFORM_CIPHER
) {
86 if (xform
->next
== NULL
)
87 res
= OPENSSL_CHAIN_ONLY_CIPHER
;
88 else if (xform
->next
->type
== RTE_CRYPTO_SYM_XFORM_AUTH
)
89 res
= OPENSSL_CHAIN_CIPHER_AUTH
;
96 /** Get session cipher key from input cipher key */
98 get_cipher_key(uint8_t *input_key
, int keylen
, uint8_t *session_key
)
100 memcpy(session_key
, input_key
, keylen
);
103 /** Get key ede 24 bytes standard from input key */
105 get_cipher_key_ede(uint8_t *key
, int keylen
, uint8_t *key_ede
)
109 /* Initialize keys - 24 bytes: [key1-key2-key3] */
112 memcpy(key_ede
, key
, 24);
116 memcpy(key_ede
, key
, 16);
117 memcpy(key_ede
+ 16, key
, 8);
120 /* K1 = K2 = K3 (DES compatibility) */
121 memcpy(key_ede
, key
, 8);
122 memcpy(key_ede
+ 8, key
, 8);
123 memcpy(key_ede
+ 16, key
, 8);
126 OPENSSL_LOG_ERR("Unsupported key size");
133 /** Get adequate openssl function for input cipher algorithm */
135 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo
, size_t keylen
,
136 const EVP_CIPHER
**algo
)
142 case RTE_CRYPTO_CIPHER_3DES_CBC
:
145 *algo
= EVP_des_ede_cbc();
148 *algo
= EVP_des_ede3_cbc();
154 case RTE_CRYPTO_CIPHER_3DES_CTR
:
156 case RTE_CRYPTO_CIPHER_AES_CBC
:
159 *algo
= EVP_aes_128_cbc();
162 *algo
= EVP_aes_192_cbc();
165 *algo
= EVP_aes_256_cbc();
171 case RTE_CRYPTO_CIPHER_AES_CTR
:
174 *algo
= EVP_aes_128_ctr();
177 *algo
= EVP_aes_192_ctr();
180 *algo
= EVP_aes_256_ctr();
186 case RTE_CRYPTO_CIPHER_AES_GCM
:
189 *algo
= EVP_aes_128_gcm();
192 *algo
= EVP_aes_192_gcm();
195 *algo
= EVP_aes_256_gcm();
212 /** Get adequate openssl function for input auth algorithm */
214 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo
,
221 case RTE_CRYPTO_AUTH_MD5
:
222 case RTE_CRYPTO_AUTH_MD5_HMAC
:
225 case RTE_CRYPTO_AUTH_SHA1
:
226 case RTE_CRYPTO_AUTH_SHA1_HMAC
:
229 case RTE_CRYPTO_AUTH_SHA224
:
230 case RTE_CRYPTO_AUTH_SHA224_HMAC
:
231 *algo
= EVP_sha224();
233 case RTE_CRYPTO_AUTH_SHA256
:
234 case RTE_CRYPTO_AUTH_SHA256_HMAC
:
235 *algo
= EVP_sha256();
237 case RTE_CRYPTO_AUTH_SHA384
:
238 case RTE_CRYPTO_AUTH_SHA384_HMAC
:
239 *algo
= EVP_sha384();
241 case RTE_CRYPTO_AUTH_SHA512
:
242 case RTE_CRYPTO_AUTH_SHA512_HMAC
:
243 *algo
= EVP_sha512();
256 /** Set session cipher parameters */
258 openssl_set_session_cipher_parameters(struct openssl_session
*sess
,
259 const struct rte_crypto_sym_xform
*xform
)
261 /* Select cipher direction */
262 sess
->cipher
.direction
= xform
->cipher
.op
;
263 /* Select cipher key */
264 sess
->cipher
.key
.length
= xform
->cipher
.key
.length
;
266 /* Select cipher algo */
267 switch (xform
->cipher
.algo
) {
268 case RTE_CRYPTO_CIPHER_3DES_CBC
:
269 case RTE_CRYPTO_CIPHER_AES_CBC
:
270 case RTE_CRYPTO_CIPHER_AES_CTR
:
271 case RTE_CRYPTO_CIPHER_AES_GCM
:
272 sess
->cipher
.mode
= OPENSSL_CIPHER_LIB
;
273 sess
->cipher
.algo
= xform
->cipher
.algo
;
274 sess
->cipher
.ctx
= EVP_CIPHER_CTX_new();
276 if (get_cipher_algo(sess
->cipher
.algo
, sess
->cipher
.key
.length
,
277 &sess
->cipher
.evp_algo
) != 0)
280 get_cipher_key(xform
->cipher
.key
.data
, sess
->cipher
.key
.length
,
281 sess
->cipher
.key
.data
);
285 case RTE_CRYPTO_CIPHER_3DES_CTR
:
286 sess
->cipher
.mode
= OPENSSL_CIPHER_DES3CTR
;
287 sess
->cipher
.ctx
= EVP_CIPHER_CTX_new();
289 if (get_cipher_key_ede(xform
->cipher
.key
.data
,
290 sess
->cipher
.key
.length
,
291 sess
->cipher
.key
.data
) != 0)
294 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI
:
295 sess
->cipher
.algo
= xform
->cipher
.algo
;
296 sess
->chain_order
= OPENSSL_CHAIN_CIPHER_BPI
;
297 sess
->cipher
.ctx
= EVP_CIPHER_CTX_new();
298 sess
->cipher
.evp_algo
= EVP_des_cbc();
300 sess
->cipher
.bpi_ctx
= EVP_CIPHER_CTX_new();
301 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
302 if (EVP_EncryptInit_ex(sess
->cipher
.bpi_ctx
, EVP_des_ecb(),
303 NULL
, xform
->cipher
.key
.data
, 0) != 1)
306 get_cipher_key(xform
->cipher
.key
.data
, sess
->cipher
.key
.length
,
307 sess
->cipher
.key
.data
);
310 sess
->cipher
.algo
= RTE_CRYPTO_CIPHER_NULL
;
317 /* Set session auth parameters */
319 openssl_set_session_auth_parameters(struct openssl_session
*sess
,
320 const struct rte_crypto_sym_xform
*xform
)
322 /* Select auth generate/verify */
323 sess
->auth
.operation
= xform
->auth
.op
;
324 sess
->auth
.algo
= xform
->auth
.algo
;
326 /* Select auth algo */
327 switch (xform
->auth
.algo
) {
328 case RTE_CRYPTO_AUTH_AES_GMAC
:
329 case RTE_CRYPTO_AUTH_AES_GCM
:
330 /* Check additional condition for AES_GMAC/GCM */
331 if (sess
->cipher
.algo
!= RTE_CRYPTO_CIPHER_AES_GCM
)
333 sess
->chain_order
= OPENSSL_CHAIN_COMBINED
;
336 case RTE_CRYPTO_AUTH_MD5
:
337 case RTE_CRYPTO_AUTH_SHA1
:
338 case RTE_CRYPTO_AUTH_SHA224
:
339 case RTE_CRYPTO_AUTH_SHA256
:
340 case RTE_CRYPTO_AUTH_SHA384
:
341 case RTE_CRYPTO_AUTH_SHA512
:
342 sess
->auth
.mode
= OPENSSL_AUTH_AS_AUTH
;
343 if (get_auth_algo(xform
->auth
.algo
,
344 &sess
->auth
.auth
.evp_algo
) != 0)
346 sess
->auth
.auth
.ctx
= EVP_MD_CTX_create();
349 case RTE_CRYPTO_AUTH_MD5_HMAC
:
350 case RTE_CRYPTO_AUTH_SHA1_HMAC
:
351 case RTE_CRYPTO_AUTH_SHA224_HMAC
:
352 case RTE_CRYPTO_AUTH_SHA256_HMAC
:
353 case RTE_CRYPTO_AUTH_SHA384_HMAC
:
354 case RTE_CRYPTO_AUTH_SHA512_HMAC
:
355 sess
->auth
.mode
= OPENSSL_AUTH_AS_HMAC
;
356 sess
->auth
.hmac
.ctx
= EVP_MD_CTX_create();
357 if (get_auth_algo(xform
->auth
.algo
,
358 &sess
->auth
.hmac
.evp_algo
) != 0)
360 sess
->auth
.hmac
.pkey
= EVP_PKEY_new_mac_key(EVP_PKEY_HMAC
, NULL
,
361 xform
->auth
.key
.data
, xform
->auth
.key
.length
);
371 /** Parse crypto xform chain and set private session parameters */
373 openssl_set_session_parameters(struct openssl_session
*sess
,
374 const struct rte_crypto_sym_xform
*xform
)
376 const struct rte_crypto_sym_xform
*cipher_xform
= NULL
;
377 const struct rte_crypto_sym_xform
*auth_xform
= NULL
;
379 sess
->chain_order
= openssl_get_chain_order(xform
);
380 switch (sess
->chain_order
) {
381 case OPENSSL_CHAIN_ONLY_CIPHER
:
382 cipher_xform
= xform
;
384 case OPENSSL_CHAIN_ONLY_AUTH
:
387 case OPENSSL_CHAIN_CIPHER_AUTH
:
388 cipher_xform
= xform
;
389 auth_xform
= xform
->next
;
391 case OPENSSL_CHAIN_AUTH_CIPHER
:
393 cipher_xform
= xform
->next
;
399 /* cipher_xform must be check before auth_xform */
401 if (openssl_set_session_cipher_parameters(
402 sess
, cipher_xform
)) {
404 "Invalid/unsupported cipher parameters");
410 if (openssl_set_session_auth_parameters(sess
, auth_xform
)) {
412 "Invalid/unsupported auth parameters");
420 /** Reset private session parameters */
422 openssl_reset_session(struct openssl_session
*sess
)
424 EVP_CIPHER_CTX_free(sess
->cipher
.ctx
);
426 if (sess
->chain_order
== OPENSSL_CHAIN_CIPHER_BPI
)
427 EVP_CIPHER_CTX_free(sess
->cipher
.bpi_ctx
);
429 switch (sess
->auth
.mode
) {
430 case OPENSSL_AUTH_AS_AUTH
:
431 EVP_MD_CTX_destroy(sess
->auth
.auth
.ctx
);
433 case OPENSSL_AUTH_AS_HMAC
:
434 EVP_PKEY_free(sess
->auth
.hmac
.pkey
);
435 EVP_MD_CTX_destroy(sess
->auth
.hmac
.ctx
);
442 /** Provide session for operation */
443 static struct openssl_session
*
444 get_session(struct openssl_qp
*qp
, struct rte_crypto_op
*op
)
446 struct openssl_session
*sess
= NULL
;
448 if (op
->sym
->sess_type
== RTE_CRYPTO_SYM_OP_WITH_SESSION
) {
449 /* get existing session */
450 if (likely(op
->sym
->session
!= NULL
&&
451 op
->sym
->session
->dev_type
==
452 RTE_CRYPTODEV_OPENSSL_PMD
))
453 sess
= (struct openssl_session
*)
454 op
->sym
->session
->_private
;
456 /* provide internal session */
459 if (!rte_mempool_get(qp
->sess_mp
, (void **)&_sess
)) {
460 sess
= (struct openssl_session
*)
461 ((struct rte_cryptodev_sym_session
*)_sess
)
464 if (unlikely(openssl_set_session_parameters(
465 sess
, op
->sym
->xform
) != 0)) {
466 rte_mempool_put(qp
->sess_mp
, _sess
);
469 op
->sym
->session
= _sess
;
474 op
->status
= RTE_CRYPTO_OP_STATUS_INVALID_SESSION
;
480 *------------------------------------------------------------------------------
482 *------------------------------------------------------------------------------
485 process_openssl_encryption_update(struct rte_mbuf
*mbuf_src
, int offset
,
486 uint8_t **dst
, int srclen
, EVP_CIPHER_CTX
*ctx
)
493 for (m
= mbuf_src
; m
!= NULL
&& offset
> rte_pktmbuf_data_len(m
);
495 offset
-= rte_pktmbuf_data_len(m
);
500 src
= rte_pktmbuf_mtod_offset(m
, uint8_t *, offset
);
502 l
= rte_pktmbuf_data_len(m
) - offset
;
504 if (EVP_EncryptUpdate(ctx
, *dst
, &dstlen
, src
, srclen
) <= 0)
510 if (EVP_EncryptUpdate(ctx
, *dst
, &dstlen
, src
, l
) <= 0)
516 for (m
= m
->next
; (m
!= NULL
) && (n
> 0); m
= m
->next
) {
517 src
= rte_pktmbuf_mtod(m
, uint8_t *);
518 l
= rte_pktmbuf_data_len(m
) < n
? rte_pktmbuf_data_len(m
) : n
;
519 if (EVP_EncryptUpdate(ctx
, *dst
, &dstlen
, src
, l
) <= 0)
529 process_openssl_decryption_update(struct rte_mbuf
*mbuf_src
, int offset
,
530 uint8_t **dst
, int srclen
, EVP_CIPHER_CTX
*ctx
)
537 for (m
= mbuf_src
; m
!= NULL
&& offset
> rte_pktmbuf_data_len(m
);
539 offset
-= rte_pktmbuf_data_len(m
);
544 src
= rte_pktmbuf_mtod_offset(m
, uint8_t *, offset
);
546 l
= rte_pktmbuf_data_len(m
) - offset
;
548 if (EVP_DecryptUpdate(ctx
, *dst
, &dstlen
, src
, srclen
) <= 0)
554 if (EVP_DecryptUpdate(ctx
, *dst
, &dstlen
, src
, l
) <= 0)
560 for (m
= m
->next
; (m
!= NULL
) && (n
> 0); m
= m
->next
) {
561 src
= rte_pktmbuf_mtod(m
, uint8_t *);
562 l
= rte_pktmbuf_data_len(m
) < n
? rte_pktmbuf_data_len(m
) : n
;
563 if (EVP_DecryptUpdate(ctx
, *dst
, &dstlen
, src
, l
) <= 0)
572 /** Process standard openssl cipher encryption */
574 process_openssl_cipher_encrypt(struct rte_mbuf
*mbuf_src
, uint8_t *dst
,
575 int offset
, uint8_t *iv
, uint8_t *key
, int srclen
,
576 EVP_CIPHER_CTX
*ctx
, const EVP_CIPHER
*algo
)
580 if (EVP_EncryptInit_ex(ctx
, algo
, NULL
, key
, iv
) <= 0)
581 goto process_cipher_encrypt_err
;
583 EVP_CIPHER_CTX_set_padding(ctx
, 0);
585 if (process_openssl_encryption_update(mbuf_src
, offset
, &dst
,
587 goto process_cipher_encrypt_err
;
589 if (EVP_EncryptFinal_ex(ctx
, dst
, &totlen
) <= 0)
590 goto process_cipher_encrypt_err
;
594 process_cipher_encrypt_err
:
595 OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
599 /** Process standard openssl cipher encryption */
601 process_openssl_cipher_bpi_encrypt(uint8_t *src
, uint8_t *dst
,
602 uint8_t *iv
, int srclen
,
606 uint8_t encrypted_iv
[DES_BLOCK_SIZE
];
609 if (EVP_EncryptUpdate(ctx
, encrypted_iv
, &encrypted_ivlen
,
610 iv
, DES_BLOCK_SIZE
) <= 0)
611 goto process_cipher_encrypt_err
;
613 for (i
= 0; i
< srclen
; i
++)
614 *(dst
+ i
) = *(src
+ i
) ^ (encrypted_iv
[i
]);
618 process_cipher_encrypt_err
:
619 OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
622 /** Process standard openssl cipher decryption */
624 process_openssl_cipher_decrypt(struct rte_mbuf
*mbuf_src
, uint8_t *dst
,
625 int offset
, uint8_t *iv
, uint8_t *key
, int srclen
,
626 EVP_CIPHER_CTX
*ctx
, const EVP_CIPHER
*algo
)
630 if (EVP_DecryptInit_ex(ctx
, algo
, NULL
, key
, iv
) <= 0)
631 goto process_cipher_decrypt_err
;
633 EVP_CIPHER_CTX_set_padding(ctx
, 0);
635 if (process_openssl_decryption_update(mbuf_src
, offset
, &dst
,
637 goto process_cipher_decrypt_err
;
639 if (EVP_DecryptFinal_ex(ctx
, dst
, &totlen
) <= 0)
640 goto process_cipher_decrypt_err
;
643 process_cipher_decrypt_err
:
644 OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
648 /** Process cipher des 3 ctr encryption, decryption algorithm */
650 process_openssl_cipher_des3ctr(struct rte_mbuf
*mbuf_src
, uint8_t *dst
,
651 int offset
, uint8_t *iv
, uint8_t *key
, int srclen
,
654 uint8_t ebuf
[8], ctr
[8];
660 for (m
= mbuf_src
; m
!= NULL
&& offset
> rte_pktmbuf_data_len(m
);
662 offset
-= rte_pktmbuf_data_len(m
);
665 goto process_cipher_des3ctr_err
;
667 src
= rte_pktmbuf_mtod_offset(m
, uint8_t *, offset
);
668 l
= rte_pktmbuf_data_len(m
) - offset
;
670 /* We use 3DES encryption also for decryption.
671 * IV is not important for 3DES ecb
673 if (EVP_EncryptInit_ex(ctx
, EVP_des_ede3_ecb(), NULL
, key
, NULL
) <= 0)
674 goto process_cipher_des3ctr_err
;
678 for (n
= 0; n
< srclen
; n
++) {
680 if (EVP_EncryptUpdate(ctx
,
681 (unsigned char *)&ebuf
, &unused
,
682 (const unsigned char *)&ctr
, 8) <= 0)
683 goto process_cipher_des3ctr_err
;
686 dst
[n
] = *(src
++) ^ ebuf
[n
% 8];
692 src
= rte_pktmbuf_mtod(m
, uint8_t *);
693 l
= rte_pktmbuf_data_len(m
);
700 process_cipher_des3ctr_err
:
701 OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
705 /** Process auth/encription aes-gcm algorithm */
707 process_openssl_auth_encryption_gcm(struct rte_mbuf
*mbuf_src
, int offset
,
708 int srclen
, uint8_t *aad
, int aadlen
, uint8_t *iv
, int ivlen
,
709 uint8_t *key
, uint8_t *dst
, uint8_t *tag
,
710 EVP_CIPHER_CTX
*ctx
, const EVP_CIPHER
*algo
)
712 int len
= 0, unused
= 0;
713 uint8_t empty
[] = {};
715 if (EVP_EncryptInit_ex(ctx
, algo
, NULL
, NULL
, NULL
) <= 0)
716 goto process_auth_encryption_gcm_err
;
718 if (EVP_CIPHER_CTX_ctrl(ctx
, EVP_CTRL_GCM_SET_IVLEN
, ivlen
, NULL
) <= 0)
719 goto process_auth_encryption_gcm_err
;
721 if (EVP_EncryptInit_ex(ctx
, NULL
, NULL
, key
, iv
) <= 0)
722 goto process_auth_encryption_gcm_err
;
725 if (EVP_EncryptUpdate(ctx
, NULL
, &len
, aad
, aadlen
) <= 0)
726 goto process_auth_encryption_gcm_err
;
729 if (process_openssl_encryption_update(mbuf_src
, offset
, &dst
,
731 goto process_auth_encryption_gcm_err
;
733 /* Workaround open ssl bug in version less then 1.0.1f */
734 if (EVP_EncryptUpdate(ctx
, empty
, &unused
, empty
, 0) <= 0)
735 goto process_auth_encryption_gcm_err
;
737 if (EVP_EncryptFinal_ex(ctx
, dst
, &len
) <= 0)
738 goto process_auth_encryption_gcm_err
;
740 if (EVP_CIPHER_CTX_ctrl(ctx
, EVP_CTRL_GCM_GET_TAG
, 16, tag
) <= 0)
741 goto process_auth_encryption_gcm_err
;
745 process_auth_encryption_gcm_err
:
746 OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
751 process_openssl_auth_decryption_gcm(struct rte_mbuf
*mbuf_src
, int offset
,
752 int srclen
, uint8_t *aad
, int aadlen
, uint8_t *iv
, int ivlen
,
753 uint8_t *key
, uint8_t *dst
, uint8_t *tag
, EVP_CIPHER_CTX
*ctx
,
754 const EVP_CIPHER
*algo
)
756 int len
= 0, unused
= 0;
757 uint8_t empty
[] = {};
759 if (EVP_DecryptInit_ex(ctx
, algo
, NULL
, NULL
, NULL
) <= 0)
760 goto process_auth_decryption_gcm_err
;
762 if (EVP_CIPHER_CTX_ctrl(ctx
, EVP_CTRL_GCM_SET_IVLEN
, ivlen
, NULL
) <= 0)
763 goto process_auth_decryption_gcm_err
;
765 if (EVP_CIPHER_CTX_ctrl(ctx
, EVP_CTRL_GCM_SET_TAG
, 16, tag
) <= 0)
766 goto process_auth_decryption_gcm_err
;
768 if (EVP_DecryptInit_ex(ctx
, NULL
, NULL
, key
, iv
) <= 0)
769 goto process_auth_decryption_gcm_err
;
772 if (EVP_DecryptUpdate(ctx
, NULL
, &len
, aad
, aadlen
) <= 0)
773 goto process_auth_decryption_gcm_err
;
776 if (process_openssl_decryption_update(mbuf_src
, offset
, &dst
,
778 goto process_auth_decryption_gcm_err
;
780 /* Workaround open ssl bug in version less then 1.0.1f */
781 if (EVP_DecryptUpdate(ctx
, empty
, &unused
, empty
, 0) <= 0)
782 goto process_auth_decryption_gcm_err
;
784 if (EVP_DecryptFinal_ex(ctx
, dst
, &len
) <= 0)
785 goto process_auth_decryption_gcm_final_err
;
789 process_auth_decryption_gcm_err
:
790 OPENSSL_LOG_ERR("Process openssl auth description gcm failed");
793 process_auth_decryption_gcm_final_err
:
797 /** Process standard openssl auth algorithms */
799 process_openssl_auth(struct rte_mbuf
*mbuf_src
, uint8_t *dst
, int offset
,
800 __rte_unused
uint8_t *iv
, __rte_unused EVP_PKEY
* pkey
,
801 int srclen
, EVP_MD_CTX
*ctx
, const EVP_MD
*algo
)
808 for (m
= mbuf_src
; m
!= NULL
&& offset
> rte_pktmbuf_data_len(m
);
810 offset
-= rte_pktmbuf_data_len(m
);
813 goto process_auth_err
;
815 if (EVP_DigestInit_ex(ctx
, algo
, NULL
) <= 0)
816 goto process_auth_err
;
818 src
= rte_pktmbuf_mtod_offset(m
, uint8_t *, offset
);
820 l
= rte_pktmbuf_data_len(m
) - offset
;
822 if (EVP_DigestUpdate(ctx
, (char *)src
, srclen
) <= 0)
823 goto process_auth_err
;
824 goto process_auth_final
;
827 if (EVP_DigestUpdate(ctx
, (char *)src
, l
) <= 0)
828 goto process_auth_err
;
832 for (m
= m
->next
; (m
!= NULL
) && (n
> 0); m
= m
->next
) {
833 src
= rte_pktmbuf_mtod(m
, uint8_t *);
834 l
= rte_pktmbuf_data_len(m
) < n
? rte_pktmbuf_data_len(m
) : n
;
835 if (EVP_DigestUpdate(ctx
, (char *)src
, l
) <= 0)
836 goto process_auth_err
;
841 if (EVP_DigestFinal_ex(ctx
, dst
, (unsigned int *)&dstlen
) <= 0)
842 goto process_auth_err
;
846 OPENSSL_LOG_ERR("Process openssl auth failed");
850 /** Process standard openssl auth algorithms with hmac */
852 process_openssl_auth_hmac(struct rte_mbuf
*mbuf_src
, uint8_t *dst
, int offset
,
853 __rte_unused
uint8_t *iv
, EVP_PKEY
*pkey
,
854 int srclen
, EVP_MD_CTX
*ctx
, const EVP_MD
*algo
)
861 for (m
= mbuf_src
; m
!= NULL
&& offset
> rte_pktmbuf_data_len(m
);
863 offset
-= rte_pktmbuf_data_len(m
);
866 goto process_auth_err
;
868 if (EVP_DigestSignInit(ctx
, NULL
, algo
, NULL
, pkey
) <= 0)
869 goto process_auth_err
;
871 src
= rte_pktmbuf_mtod_offset(m
, uint8_t *, offset
);
873 l
= rte_pktmbuf_data_len(m
) - offset
;
875 if (EVP_DigestSignUpdate(ctx
, (char *)src
, srclen
) <= 0)
876 goto process_auth_err
;
877 goto process_auth_final
;
880 if (EVP_DigestSignUpdate(ctx
, (char *)src
, l
) <= 0)
881 goto process_auth_err
;
885 for (m
= m
->next
; (m
!= NULL
) && (n
> 0); m
= m
->next
) {
886 src
= rte_pktmbuf_mtod(m
, uint8_t *);
887 l
= rte_pktmbuf_data_len(m
) < n
? rte_pktmbuf_data_len(m
) : n
;
888 if (EVP_DigestSignUpdate(ctx
, (char *)src
, l
) <= 0)
889 goto process_auth_err
;
894 if (EVP_DigestSignFinal(ctx
, dst
, &dstlen
) <= 0)
895 goto process_auth_err
;
900 OPENSSL_LOG_ERR("Process openssl auth failed");
904 /*----------------------------------------------------------------------------*/
906 /** Process auth/cipher combined operation */
908 process_openssl_combined_op
909 (struct rte_crypto_op
*op
, struct openssl_session
*sess
,
910 struct rte_mbuf
*mbuf_src
, struct rte_mbuf
*mbuf_dst
)
913 uint8_t *dst
= NULL
, *iv
, *tag
, *aad
;
914 int srclen
, ivlen
, aadlen
, status
= -1;
917 * Segmented destination buffer is not supported for
918 * encryption/decryption
920 if (!rte_pktmbuf_is_contiguous(mbuf_dst
)) {
921 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
925 iv
= op
->sym
->cipher
.iv
.data
;
926 ivlen
= op
->sym
->cipher
.iv
.length
;
927 aad
= op
->sym
->auth
.aad
.data
;
928 aadlen
= op
->sym
->auth
.aad
.length
;
930 tag
= op
->sym
->auth
.digest
.data
;
932 tag
= rte_pktmbuf_mtod_offset(mbuf_dst
, uint8_t *,
933 op
->sym
->cipher
.data
.offset
+
934 op
->sym
->cipher
.data
.length
);
936 if (sess
->auth
.algo
== RTE_CRYPTO_AUTH_AES_GMAC
)
939 srclen
= op
->sym
->cipher
.data
.length
;
940 dst
= rte_pktmbuf_mtod_offset(mbuf_dst
, uint8_t *,
941 op
->sym
->cipher
.data
.offset
);
944 if (sess
->cipher
.direction
== RTE_CRYPTO_CIPHER_OP_ENCRYPT
)
945 status
= process_openssl_auth_encryption_gcm(
946 mbuf_src
, op
->sym
->cipher
.data
.offset
, srclen
,
947 aad
, aadlen
, iv
, ivlen
, sess
->cipher
.key
.data
,
948 dst
, tag
, sess
->cipher
.ctx
,
949 sess
->cipher
.evp_algo
);
951 status
= process_openssl_auth_decryption_gcm(
952 mbuf_src
, op
->sym
->cipher
.data
.offset
, srclen
,
953 aad
, aadlen
, iv
, ivlen
, sess
->cipher
.key
.data
,
954 dst
, tag
, sess
->cipher
.ctx
,
955 sess
->cipher
.evp_algo
);
958 if (status
== (-EFAULT
) &&
959 sess
->auth
.operation
==
960 RTE_CRYPTO_AUTH_OP_VERIFY
)
961 op
->status
= RTE_CRYPTO_OP_STATUS_AUTH_FAILED
;
963 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
967 /** Process cipher operation */
969 process_openssl_cipher_op
970 (struct rte_crypto_op
*op
, struct openssl_session
*sess
,
971 struct rte_mbuf
*mbuf_src
, struct rte_mbuf
*mbuf_dst
)
977 * Segmented destination buffer is not supported for
978 * encryption/decryption
980 if (!rte_pktmbuf_is_contiguous(mbuf_dst
)) {
981 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
985 srclen
= op
->sym
->cipher
.data
.length
;
986 dst
= rte_pktmbuf_mtod_offset(mbuf_dst
, uint8_t *,
987 op
->sym
->cipher
.data
.offset
);
989 iv
= op
->sym
->cipher
.iv
.data
;
991 if (sess
->cipher
.mode
== OPENSSL_CIPHER_LIB
)
992 if (sess
->cipher
.direction
== RTE_CRYPTO_CIPHER_OP_ENCRYPT
)
993 status
= process_openssl_cipher_encrypt(mbuf_src
, dst
,
994 op
->sym
->cipher
.data
.offset
, iv
,
995 sess
->cipher
.key
.data
, srclen
,
997 sess
->cipher
.evp_algo
);
999 status
= process_openssl_cipher_decrypt(mbuf_src
, dst
,
1000 op
->sym
->cipher
.data
.offset
, iv
,
1001 sess
->cipher
.key
.data
, srclen
,
1003 sess
->cipher
.evp_algo
);
1005 status
= process_openssl_cipher_des3ctr(mbuf_src
, dst
,
1006 op
->sym
->cipher
.data
.offset
, iv
,
1007 sess
->cipher
.key
.data
, srclen
,
1011 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
1014 /** Process cipher operation */
1016 process_openssl_docsis_bpi_op(struct rte_crypto_op
*op
,
1017 struct openssl_session
*sess
, struct rte_mbuf
*mbuf_src
,
1018 struct rte_mbuf
*mbuf_dst
)
1020 uint8_t *src
, *dst
, *iv
;
1021 uint8_t block_size
, last_block_len
;
1022 int srclen
, status
= 0;
1024 srclen
= op
->sym
->cipher
.data
.length
;
1025 src
= rte_pktmbuf_mtod_offset(mbuf_src
, uint8_t *,
1026 op
->sym
->cipher
.data
.offset
);
1027 dst
= rte_pktmbuf_mtod_offset(mbuf_dst
, uint8_t *,
1028 op
->sym
->cipher
.data
.offset
);
1030 iv
= op
->sym
->cipher
.iv
.data
;
1032 block_size
= DES_BLOCK_SIZE
;
1034 last_block_len
= srclen
% block_size
;
1035 if (sess
->cipher
.direction
== RTE_CRYPTO_CIPHER_OP_ENCRYPT
) {
1036 /* Encrypt only with ECB mode XOR IV */
1037 if (srclen
< block_size
) {
1038 status
= process_openssl_cipher_bpi_encrypt(src
, dst
,
1040 sess
->cipher
.bpi_ctx
);
1042 srclen
-= last_block_len
;
1043 /* Encrypt with the block aligned stream with CBC mode */
1044 status
= process_openssl_cipher_encrypt(mbuf_src
, dst
,
1045 op
->sym
->cipher
.data
.offset
, iv
,
1046 sess
->cipher
.key
.data
, srclen
,
1047 sess
->cipher
.ctx
, sess
->cipher
.evp_algo
);
1048 if (last_block_len
) {
1049 /* Point at last block */
1052 * IV is the last encrypted block from
1053 * the previous operation
1055 iv
= dst
- block_size
;
1057 srclen
= last_block_len
;
1058 /* Encrypt the last frame with ECB mode */
1059 status
|= process_openssl_cipher_bpi_encrypt(src
,
1061 srclen
, sess
->cipher
.bpi_ctx
);
1065 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1066 if (srclen
< block_size
) {
1067 status
= process_openssl_cipher_bpi_encrypt(src
, dst
,
1070 sess
->cipher
.bpi_ctx
);
1072 if (last_block_len
) {
1073 /* Point at last block */
1074 dst
+= srclen
- last_block_len
;
1075 src
+= srclen
- last_block_len
;
1077 * IV is the last full block
1079 iv
= src
- block_size
;
1081 * Decrypt the last frame with ECB mode
1082 * (encrypt, as it is the same operation)
1084 status
= process_openssl_cipher_bpi_encrypt(src
,
1086 last_block_len
, sess
->cipher
.bpi_ctx
);
1087 /* Prepare parameters for CBC mode op */
1088 iv
= op
->sym
->cipher
.iv
.data
;
1089 dst
+= last_block_len
- srclen
;
1090 srclen
-= last_block_len
;
1093 /* Decrypt with CBC mode */
1094 status
|= process_openssl_cipher_decrypt(mbuf_src
, dst
,
1095 op
->sym
->cipher
.data
.offset
, iv
,
1096 sess
->cipher
.key
.data
, srclen
,
1098 sess
->cipher
.evp_algo
);
1103 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
1106 /** Process auth operation */
1108 process_openssl_auth_op
1109 (struct rte_crypto_op
*op
, struct openssl_session
*sess
,
1110 struct rte_mbuf
*mbuf_src
, struct rte_mbuf
*mbuf_dst
)
1115 srclen
= op
->sym
->auth
.data
.length
;
1117 if (sess
->auth
.operation
== RTE_CRYPTO_AUTH_OP_VERIFY
)
1118 dst
= (uint8_t *)rte_pktmbuf_append(mbuf_src
,
1119 op
->sym
->auth
.digest
.length
);
1121 dst
= op
->sym
->auth
.digest
.data
;
1123 dst
= rte_pktmbuf_mtod_offset(mbuf_dst
, uint8_t *,
1124 op
->sym
->auth
.data
.offset
+
1125 op
->sym
->auth
.data
.length
);
1128 switch (sess
->auth
.mode
) {
1129 case OPENSSL_AUTH_AS_AUTH
:
1130 status
= process_openssl_auth(mbuf_src
, dst
,
1131 op
->sym
->auth
.data
.offset
, NULL
, NULL
, srclen
,
1132 sess
->auth
.auth
.ctx
, sess
->auth
.auth
.evp_algo
);
1134 case OPENSSL_AUTH_AS_HMAC
:
1135 status
= process_openssl_auth_hmac(mbuf_src
, dst
,
1136 op
->sym
->auth
.data
.offset
, NULL
,
1137 sess
->auth
.hmac
.pkey
, srclen
,
1138 sess
->auth
.hmac
.ctx
, sess
->auth
.hmac
.evp_algo
);
1145 if (sess
->auth
.operation
== RTE_CRYPTO_AUTH_OP_VERIFY
) {
1146 if (memcmp(dst
, op
->sym
->auth
.digest
.data
,
1147 op
->sym
->auth
.digest
.length
) != 0) {
1148 op
->status
= RTE_CRYPTO_OP_STATUS_AUTH_FAILED
;
1150 /* Trim area used for digest from mbuf. */
1151 rte_pktmbuf_trim(mbuf_src
, op
->sym
->auth
.digest
.length
);
1155 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
1158 /** Process crypto operation for mbuf */
1160 process_op(const struct openssl_qp
*qp
, struct rte_crypto_op
*op
,
1161 struct openssl_session
*sess
)
1163 struct rte_mbuf
*msrc
, *mdst
;
1166 msrc
= op
->sym
->m_src
;
1167 mdst
= op
->sym
->m_dst
? op
->sym
->m_dst
: op
->sym
->m_src
;
1169 op
->status
= RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
;
1171 switch (sess
->chain_order
) {
1172 case OPENSSL_CHAIN_ONLY_CIPHER
:
1173 process_openssl_cipher_op(op
, sess
, msrc
, mdst
);
1175 case OPENSSL_CHAIN_ONLY_AUTH
:
1176 process_openssl_auth_op(op
, sess
, msrc
, mdst
);
1178 case OPENSSL_CHAIN_CIPHER_AUTH
:
1179 process_openssl_cipher_op(op
, sess
, msrc
, mdst
);
1180 process_openssl_auth_op(op
, sess
, mdst
, mdst
);
1182 case OPENSSL_CHAIN_AUTH_CIPHER
:
1183 process_openssl_auth_op(op
, sess
, msrc
, mdst
);
1184 process_openssl_cipher_op(op
, sess
, msrc
, mdst
);
1186 case OPENSSL_CHAIN_COMBINED
:
1187 process_openssl_combined_op(op
, sess
, msrc
, mdst
);
1189 case OPENSSL_CHAIN_CIPHER_BPI
:
1190 process_openssl_docsis_bpi_op(op
, sess
, msrc
, mdst
);
1193 op
->status
= RTE_CRYPTO_OP_STATUS_ERROR
;
1197 /* Free session if a session-less crypto op */
1198 if (op
->sym
->sess_type
== RTE_CRYPTO_SYM_OP_SESSIONLESS
) {
1199 openssl_reset_session(sess
);
1200 memset(sess
, 0, sizeof(struct openssl_session
));
1201 rte_mempool_put(qp
->sess_mp
, op
->sym
->session
);
1202 op
->sym
->session
= NULL
;
1205 if (op
->status
== RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
)
1206 op
->status
= RTE_CRYPTO_OP_STATUS_SUCCESS
;
1208 if (op
->status
!= RTE_CRYPTO_OP_STATUS_ERROR
)
1209 retval
= rte_ring_enqueue(qp
->processed_ops
, (void *)op
);
1217 *------------------------------------------------------------------------------
1219 *------------------------------------------------------------------------------
1222 /** Enqueue burst */
1224 openssl_pmd_enqueue_burst(void *queue_pair
, struct rte_crypto_op
**ops
,
1227 struct openssl_session
*sess
;
1228 struct openssl_qp
*qp
= queue_pair
;
1231 for (i
= 0; i
< nb_ops
; i
++) {
1232 sess
= get_session(qp
, ops
[i
]);
1233 if (unlikely(sess
== NULL
))
1236 retval
= process_op(qp
, ops
[i
], sess
);
1237 if (unlikely(retval
< 0))
1241 qp
->stats
.enqueued_count
+= i
;
1245 qp
->stats
.enqueue_err_count
++;
1249 /** Dequeue burst */
1251 openssl_pmd_dequeue_burst(void *queue_pair
, struct rte_crypto_op
**ops
,
1254 struct openssl_qp
*qp
= queue_pair
;
1256 unsigned int nb_dequeued
= 0;
1258 nb_dequeued
= rte_ring_dequeue_burst(qp
->processed_ops
,
1259 (void **)ops
, nb_ops
, NULL
);
1260 qp
->stats
.dequeued_count
+= nb_dequeued
;
1265 /** Create OPENSSL crypto device */
1267 cryptodev_openssl_create(const char *name
,
1268 struct rte_vdev_device
*vdev
,
1269 struct rte_crypto_vdev_init_params
*init_params
)
1271 struct rte_cryptodev
*dev
;
1272 struct openssl_private
*internals
;
1274 if (init_params
->name
[0] == '\0')
1275 snprintf(init_params
->name
, sizeof(init_params
->name
),
1278 dev
= rte_cryptodev_pmd_virtual_dev_init(init_params
->name
,
1279 sizeof(struct openssl_private
),
1280 init_params
->socket_id
);
1282 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1286 dev
->dev_type
= RTE_CRYPTODEV_OPENSSL_PMD
;
1287 dev
->dev_ops
= rte_openssl_pmd_ops
;
1289 /* register rx/tx burst functions for data path */
1290 dev
->dequeue_burst
= openssl_pmd_dequeue_burst
;
1291 dev
->enqueue_burst
= openssl_pmd_enqueue_burst
;
1293 dev
->feature_flags
= RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
|
1294 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
|
1295 RTE_CRYPTODEV_FF_CPU_AESNI
|
1296 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER
;
1298 /* Set vector instructions mode supported */
1299 internals
= dev
->data
->dev_private
;
1301 internals
->max_nb_qpairs
= init_params
->max_nb_queue_pairs
;
1302 internals
->max_nb_sessions
= init_params
->max_nb_sessions
;
1307 OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1310 cryptodev_openssl_remove(vdev
);
1314 /** Initialise OPENSSL crypto device */
1316 cryptodev_openssl_probe(struct rte_vdev_device
*vdev
)
1318 struct rte_crypto_vdev_init_params init_params
= {
1319 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS
,
1320 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS
,
1325 const char *input_args
;
1327 name
= rte_vdev_device_name(vdev
);
1330 input_args
= rte_vdev_device_args(vdev
);
1332 rte_cryptodev_parse_vdev_init_params(&init_params
, input_args
);
1334 RTE_LOG(INFO
, PMD
, "Initialising %s on NUMA node %d\n", name
,
1335 init_params
.socket_id
);
1336 if (init_params
.name
[0] != '\0')
1337 RTE_LOG(INFO
, PMD
, " User defined name = %s\n",
1339 RTE_LOG(INFO
, PMD
, " Max number of queue pairs = %d\n",
1340 init_params
.max_nb_queue_pairs
);
1341 RTE_LOG(INFO
, PMD
, " Max number of sessions = %d\n",
1342 init_params
.max_nb_sessions
);
1344 return cryptodev_openssl_create(name
, vdev
, &init_params
);
1347 /** Uninitialise OPENSSL crypto device */
1349 cryptodev_openssl_remove(struct rte_vdev_device
*vdev
)
1353 name
= rte_vdev_device_name(vdev
);
1358 "Closing OPENSSL crypto device %s on numa socket %u\n",
1359 name
, rte_socket_id());
1364 static struct rte_vdev_driver cryptodev_openssl_pmd_drv
= {
1365 .probe
= cryptodev_openssl_probe
,
1366 .remove
= cryptodev_openssl_remove
1369 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD
,
1370 cryptodev_openssl_pmd_drv
);
1371 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD
,
1372 "max_nb_queue_pairs=<int> "
1373 "max_nb_sessions=<int> "