1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
4 #include <rte_malloc.h>
8 #include <rte_cryptodev.h>
10 #include "rte_vhost_crypto.h"
12 #include "vhost_user.h"
13 #include "virtio_crypto.h"
15 #define INHDR_LEN (sizeof(struct virtio_crypto_inhdr))
16 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \
17 sizeof(struct rte_crypto_sym_op))
19 #ifdef RTE_LIBRTE_VHOST_DEBUG
20 #define VC_LOG_ERR(fmt, args...) \
21 RTE_LOG(ERR, USER1, "[%s] %s() line %u: " fmt "\n", \
22 "Vhost-Crypto", __func__, __LINE__, ## args)
23 #define VC_LOG_INFO(fmt, args...) \
24 RTE_LOG(INFO, USER1, "[%s] %s() line %u: " fmt "\n", \
25 "Vhost-Crypto", __func__, __LINE__, ## args)
27 #define VC_LOG_DBG(fmt, args...) \
28 RTE_LOG(DEBUG, USER1, "[%s] %s() line %u: " fmt "\n", \
29 "Vhost-Crypto", __func__, __LINE__, ## args)
31 #define VC_LOG_ERR(fmt, args...) \
32 RTE_LOG(ERR, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
33 #define VC_LOG_INFO(fmt, args...) \
34 RTE_LOG(INFO, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
35 #define VC_LOG_DBG(fmt, args...)
38 #define VIRTIO_CRYPTO_FEATURES ((1 << VIRTIO_F_NOTIFY_ON_EMPTY) | \
39 (1 << VIRTIO_RING_F_INDIRECT_DESC) | \
40 (1 << VIRTIO_RING_F_EVENT_IDX) | \
41 (1 << VIRTIO_CRYPTO_SERVICE_CIPHER) | \
42 (1 << VIRTIO_CRYPTO_SERVICE_MAC) | \
43 (1 << VIRTIO_NET_F_CTRL_VQ))
45 #define IOVA_TO_VVA(t, r, a, l, p) \
46 ((t)(uintptr_t)vhost_iova_to_vva(r->dev, r->vq, a, l, p))
49 cipher_algo_transform(uint32_t virtio_cipher_algo
)
53 switch (virtio_cipher_algo
) {
54 case VIRTIO_CRYPTO_CIPHER_AES_CBC
:
55 ret
= RTE_CRYPTO_CIPHER_AES_CBC
;
57 case VIRTIO_CRYPTO_CIPHER_AES_CTR
:
58 ret
= RTE_CRYPTO_CIPHER_AES_CTR
;
60 case VIRTIO_CRYPTO_CIPHER_DES_ECB
:
61 ret
= -VIRTIO_CRYPTO_NOTSUPP
;
63 case VIRTIO_CRYPTO_CIPHER_DES_CBC
:
64 ret
= RTE_CRYPTO_CIPHER_DES_CBC
;
66 case VIRTIO_CRYPTO_CIPHER_3DES_ECB
:
67 ret
= RTE_CRYPTO_CIPHER_3DES_ECB
;
69 case VIRTIO_CRYPTO_CIPHER_3DES_CBC
:
70 ret
= RTE_CRYPTO_CIPHER_3DES_CBC
;
72 case VIRTIO_CRYPTO_CIPHER_3DES_CTR
:
73 ret
= RTE_CRYPTO_CIPHER_3DES_CTR
;
75 case VIRTIO_CRYPTO_CIPHER_KASUMI_F8
:
76 ret
= RTE_CRYPTO_CIPHER_KASUMI_F8
;
78 case VIRTIO_CRYPTO_CIPHER_SNOW3G_UEA2
:
79 ret
= RTE_CRYPTO_CIPHER_SNOW3G_UEA2
;
81 case VIRTIO_CRYPTO_CIPHER_AES_F8
:
82 ret
= RTE_CRYPTO_CIPHER_AES_F8
;
84 case VIRTIO_CRYPTO_CIPHER_AES_XTS
:
85 ret
= RTE_CRYPTO_CIPHER_AES_XTS
;
87 case VIRTIO_CRYPTO_CIPHER_ZUC_EEA3
:
88 ret
= RTE_CRYPTO_CIPHER_ZUC_EEA3
;
91 ret
= -VIRTIO_CRYPTO_BADMSG
;
99 auth_algo_transform(uint32_t virtio_auth_algo
)
103 switch (virtio_auth_algo
) {
105 case VIRTIO_CRYPTO_NO_MAC
:
106 ret
= RTE_CRYPTO_AUTH_NULL
;
108 case VIRTIO_CRYPTO_MAC_HMAC_MD5
:
109 ret
= RTE_CRYPTO_AUTH_MD5_HMAC
;
111 case VIRTIO_CRYPTO_MAC_HMAC_SHA1
:
112 ret
= RTE_CRYPTO_AUTH_SHA1_HMAC
;
114 case VIRTIO_CRYPTO_MAC_HMAC_SHA_224
:
115 ret
= RTE_CRYPTO_AUTH_SHA224_HMAC
;
117 case VIRTIO_CRYPTO_MAC_HMAC_SHA_256
:
118 ret
= RTE_CRYPTO_AUTH_SHA256_HMAC
;
120 case VIRTIO_CRYPTO_MAC_HMAC_SHA_384
:
121 ret
= RTE_CRYPTO_AUTH_SHA384_HMAC
;
123 case VIRTIO_CRYPTO_MAC_HMAC_SHA_512
:
124 ret
= RTE_CRYPTO_AUTH_SHA512_HMAC
;
126 case VIRTIO_CRYPTO_MAC_CMAC_3DES
:
127 ret
= -VIRTIO_CRYPTO_NOTSUPP
;
129 case VIRTIO_CRYPTO_MAC_CMAC_AES
:
130 ret
= RTE_CRYPTO_AUTH_AES_CMAC
;
132 case VIRTIO_CRYPTO_MAC_KASUMI_F9
:
133 ret
= RTE_CRYPTO_AUTH_KASUMI_F9
;
135 case VIRTIO_CRYPTO_MAC_SNOW3G_UIA2
:
136 ret
= RTE_CRYPTO_AUTH_SNOW3G_UIA2
;
138 case VIRTIO_CRYPTO_MAC_GMAC_AES
:
139 ret
= RTE_CRYPTO_AUTH_AES_GMAC
;
141 case VIRTIO_CRYPTO_MAC_GMAC_TWOFISH
:
142 ret
= -VIRTIO_CRYPTO_NOTSUPP
;
144 case VIRTIO_CRYPTO_MAC_CBCMAC_AES
:
145 ret
= RTE_CRYPTO_AUTH_AES_CBC_MAC
;
147 case VIRTIO_CRYPTO_MAC_CBCMAC_KASUMI_F9
:
148 ret
= -VIRTIO_CRYPTO_NOTSUPP
;
150 case VIRTIO_CRYPTO_MAC_XCBC_AES
:
151 ret
= RTE_CRYPTO_AUTH_AES_XCBC_MAC
;
154 ret
= -VIRTIO_CRYPTO_BADMSG
;
161 static int get_iv_len(enum rte_crypto_cipher_algorithm algo
)
166 case RTE_CRYPTO_CIPHER_3DES_CBC
:
169 case RTE_CRYPTO_CIPHER_3DES_CTR
:
172 case RTE_CRYPTO_CIPHER_3DES_ECB
:
175 case RTE_CRYPTO_CIPHER_AES_CBC
:
179 /* TODO: add common algos */
190 * vhost_crypto struct is used to maintain a number of virtio_cryptos and
191 * one DPDK crypto device that deals with all crypto workloads. It is declared
192 * here and defined in vhost_crypto.c
194 struct vhost_crypto
{
195 /** Used to lookup DPDK Cryptodev Session based on VIRTIO crypto
198 struct rte_hash
*session_map
;
199 struct rte_mempool
*mbuf_pool
;
200 struct rte_mempool
*sess_pool
;
201 struct rte_mempool
*sess_priv_pool
;
202 struct rte_mempool
*wb_pool
;
204 /** DPDK cryptodev ID */
208 uint64_t last_session_id
;
210 uint64_t cache_session_id
;
211 struct rte_cryptodev_sym_session
*cache_session
;
212 /** socket id for the device */
215 struct virtio_net
*dev
;
218 } __rte_cache_aligned
;
220 struct vhost_crypto_writeback_data
{
224 struct vhost_crypto_writeback_data
*next
;
227 struct vhost_crypto_data_req
{
228 struct vring_desc
*head
;
229 struct virtio_net
*dev
;
230 struct virtio_crypto_inhdr
*inhdr
;
231 struct vhost_virtqueue
*vq
;
232 struct vhost_crypto_writeback_data
*wb
;
233 struct rte_mempool
*wb_pool
;
240 transform_cipher_param(struct rte_crypto_sym_xform
*xform
,
241 VhostUserCryptoSessionParam
*param
)
245 ret
= cipher_algo_transform(param
->cipher_algo
);
246 if (unlikely(ret
< 0))
249 xform
->type
= RTE_CRYPTO_SYM_XFORM_CIPHER
;
250 xform
->cipher
.algo
= (enum rte_crypto_cipher_algorithm
)ret
;
251 xform
->cipher
.key
.length
= param
->cipher_key_len
;
252 if (xform
->cipher
.key
.length
> 0)
253 xform
->cipher
.key
.data
= param
->cipher_key_buf
;
254 if (param
->dir
== VIRTIO_CRYPTO_OP_ENCRYPT
)
255 xform
->cipher
.op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
256 else if (param
->dir
== VIRTIO_CRYPTO_OP_DECRYPT
)
257 xform
->cipher
.op
= RTE_CRYPTO_CIPHER_OP_DECRYPT
;
259 VC_LOG_DBG("Bad operation type");
260 return -VIRTIO_CRYPTO_BADMSG
;
263 ret
= get_iv_len(xform
->cipher
.algo
);
264 if (unlikely(ret
< 0))
266 xform
->cipher
.iv
.length
= (uint16_t)ret
;
267 xform
->cipher
.iv
.offset
= IV_OFFSET
;
272 transform_chain_param(struct rte_crypto_sym_xform
*xforms
,
273 VhostUserCryptoSessionParam
*param
)
275 struct rte_crypto_sym_xform
*xform_cipher
, *xform_auth
;
278 switch (param
->chaining_dir
) {
279 case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER
:
281 xform_cipher
= xforms
->next
;
282 xform_cipher
->cipher
.op
= RTE_CRYPTO_CIPHER_OP_DECRYPT
;
283 xform_auth
->auth
.op
= RTE_CRYPTO_AUTH_OP_VERIFY
;
285 case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH
:
286 xform_cipher
= xforms
;
287 xform_auth
= xforms
->next
;
288 xform_cipher
->cipher
.op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
289 xform_auth
->auth
.op
= RTE_CRYPTO_AUTH_OP_GENERATE
;
292 return -VIRTIO_CRYPTO_BADMSG
;
296 ret
= cipher_algo_transform(param
->cipher_algo
);
297 if (unlikely(ret
< 0))
299 xform_cipher
->type
= RTE_CRYPTO_SYM_XFORM_CIPHER
;
300 xform_cipher
->cipher
.algo
= (enum rte_crypto_cipher_algorithm
)ret
;
301 xform_cipher
->cipher
.key
.length
= param
->cipher_key_len
;
302 xform_cipher
->cipher
.key
.data
= param
->cipher_key_buf
;
303 ret
= get_iv_len(xform_cipher
->cipher
.algo
);
304 if (unlikely(ret
< 0))
306 xform_cipher
->cipher
.iv
.length
= (uint16_t)ret
;
307 xform_cipher
->cipher
.iv
.offset
= IV_OFFSET
;
310 xform_auth
->type
= RTE_CRYPTO_SYM_XFORM_AUTH
;
311 ret
= auth_algo_transform(param
->hash_algo
);
312 if (unlikely(ret
< 0))
314 xform_auth
->auth
.algo
= (enum rte_crypto_auth_algorithm
)ret
;
315 xform_auth
->auth
.digest_length
= param
->digest_len
;
316 xform_auth
->auth
.key
.length
= param
->auth_key_len
;
317 xform_auth
->auth
.key
.data
= param
->auth_key_buf
;
323 vhost_crypto_create_sess(struct vhost_crypto
*vcrypto
,
324 VhostUserCryptoSessionParam
*sess_param
)
326 struct rte_crypto_sym_xform xform1
= {0}, xform2
= {0};
327 struct rte_cryptodev_sym_session
*session
;
330 switch (sess_param
->op_type
) {
331 case VIRTIO_CRYPTO_SYM_OP_NONE
:
332 case VIRTIO_CRYPTO_SYM_OP_CIPHER
:
333 ret
= transform_cipher_param(&xform1
, sess_param
);
335 VC_LOG_ERR("Error transform session msg (%i)", ret
);
336 sess_param
->session_id
= ret
;
340 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING
:
341 if (unlikely(sess_param
->hash_mode
!=
342 VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH
)) {
343 sess_param
->session_id
= -VIRTIO_CRYPTO_NOTSUPP
;
344 VC_LOG_ERR("Error transform session message (%i)",
345 -VIRTIO_CRYPTO_NOTSUPP
);
349 xform1
.next
= &xform2
;
351 ret
= transform_chain_param(&xform1
, sess_param
);
353 VC_LOG_ERR("Error transform session message (%i)", ret
);
354 sess_param
->session_id
= ret
;
360 VC_LOG_ERR("Algorithm not yet supported");
361 sess_param
->session_id
= -VIRTIO_CRYPTO_NOTSUPP
;
365 session
= rte_cryptodev_sym_session_create(vcrypto
->sess_pool
);
367 VC_LOG_ERR("Failed to create session");
368 sess_param
->session_id
= -VIRTIO_CRYPTO_ERR
;
372 if (rte_cryptodev_sym_session_init(vcrypto
->cid
, session
, &xform1
,
373 vcrypto
->sess_priv_pool
) < 0) {
374 VC_LOG_ERR("Failed to initialize session");
375 sess_param
->session_id
= -VIRTIO_CRYPTO_ERR
;
379 /* insert hash to map */
380 if (rte_hash_add_key_data(vcrypto
->session_map
,
381 &vcrypto
->last_session_id
, session
) < 0) {
382 VC_LOG_ERR("Failed to insert session to hash table");
384 if (rte_cryptodev_sym_session_clear(vcrypto
->cid
, session
) < 0)
385 VC_LOG_ERR("Failed to clear session");
387 if (rte_cryptodev_sym_session_free(session
) < 0)
388 VC_LOG_ERR("Failed to free session");
390 sess_param
->session_id
= -VIRTIO_CRYPTO_ERR
;
394 VC_LOG_INFO("Session %"PRIu64
" created for vdev %i.",
395 vcrypto
->last_session_id
, vcrypto
->dev
->vid
);
397 sess_param
->session_id
= vcrypto
->last_session_id
;
398 vcrypto
->last_session_id
++;
402 vhost_crypto_close_sess(struct vhost_crypto
*vcrypto
, uint64_t session_id
)
404 struct rte_cryptodev_sym_session
*session
;
405 uint64_t sess_id
= session_id
;
408 ret
= rte_hash_lookup_data(vcrypto
->session_map
, &sess_id
,
411 if (unlikely(ret
< 0)) {
412 VC_LOG_ERR("Failed to delete session %"PRIu64
".", session_id
);
413 return -VIRTIO_CRYPTO_INVSESS
;
416 if (rte_cryptodev_sym_session_clear(vcrypto
->cid
, session
) < 0) {
417 VC_LOG_DBG("Failed to clear session");
418 return -VIRTIO_CRYPTO_ERR
;
421 if (rte_cryptodev_sym_session_free(session
) < 0) {
422 VC_LOG_DBG("Failed to free session");
423 return -VIRTIO_CRYPTO_ERR
;
426 if (rte_hash_del_key(vcrypto
->session_map
, &sess_id
) < 0) {
427 VC_LOG_DBG("Failed to delete session from hash table.");
428 return -VIRTIO_CRYPTO_ERR
;
431 VC_LOG_INFO("Session %"PRIu64
" deleted for vdev %i.", sess_id
,
437 static enum rte_vhost_msg_result
438 vhost_crypto_msg_post_handler(int vid
, void *msg
)
440 struct virtio_net
*dev
= get_device(vid
);
441 struct vhost_crypto
*vcrypto
;
442 VhostUserMsg
*vmsg
= msg
;
443 enum rte_vhost_msg_result ret
= RTE_VHOST_MSG_RESULT_OK
;
446 VC_LOG_ERR("Invalid vid %i", vid
);
447 return RTE_VHOST_MSG_RESULT_ERR
;
450 vcrypto
= dev
->extern_data
;
451 if (vcrypto
== NULL
) {
452 VC_LOG_ERR("Cannot find required data, is it initialized?");
453 return RTE_VHOST_MSG_RESULT_ERR
;
456 switch (vmsg
->request
.master
) {
457 case VHOST_USER_CRYPTO_CREATE_SESS
:
458 vhost_crypto_create_sess(vcrypto
,
459 &vmsg
->payload
.crypto_session
);
461 ret
= RTE_VHOST_MSG_RESULT_REPLY
;
463 case VHOST_USER_CRYPTO_CLOSE_SESS
:
464 if (vhost_crypto_close_sess(vcrypto
, vmsg
->payload
.u64
))
465 ret
= RTE_VHOST_MSG_RESULT_ERR
;
468 ret
= RTE_VHOST_MSG_RESULT_NOT_HANDLED
;
475 static __rte_always_inline
struct vring_desc
*
476 find_write_desc(struct vring_desc
*head
, struct vring_desc
*desc
,
477 uint32_t *nb_descs
, uint32_t vq_size
)
479 if (desc
->flags
& VRING_DESC_F_WRITE
)
482 while (desc
->flags
& VRING_DESC_F_NEXT
) {
483 if (unlikely(*nb_descs
== 0 || desc
->next
>= vq_size
))
487 desc
= &head
[desc
->next
];
488 if (desc
->flags
& VRING_DESC_F_WRITE
)
495 static struct virtio_crypto_inhdr
*
496 reach_inhdr(struct vhost_crypto_data_req
*vc_req
, struct vring_desc
*desc
,
497 uint32_t *nb_descs
, uint32_t vq_size
)
500 struct virtio_crypto_inhdr
*inhdr
;
502 while (desc
->flags
& VRING_DESC_F_NEXT
) {
503 if (unlikely(*nb_descs
== 0 || desc
->next
>= vq_size
))
506 desc
= &vc_req
->head
[desc
->next
];
510 inhdr
= IOVA_TO_VVA(struct virtio_crypto_inhdr
*, vc_req
, desc
->addr
,
511 &dlen
, VHOST_ACCESS_WO
);
512 if (unlikely(!inhdr
|| dlen
!= desc
->len
))
518 static __rte_always_inline
int
519 move_desc(struct vring_desc
*head
, struct vring_desc
**cur_desc
,
520 uint32_t size
, uint32_t *nb_descs
, uint32_t vq_size
)
522 struct vring_desc
*desc
= *cur_desc
;
523 int left
= size
- desc
->len
;
525 while ((desc
->flags
& VRING_DESC_F_NEXT
) && left
> 0) {
527 if (unlikely(*nb_descs
== 0 || desc
->next
>= vq_size
))
530 desc
= &head
[desc
->next
];
531 rte_prefetch0(&head
[desc
->next
]);
535 if (unlikely(left
> 0))
538 if (unlikely(*nb_descs
== 0))
541 if (unlikely(desc
->next
>= vq_size
))
543 *cur_desc
= &head
[desc
->next
];
549 static __rte_always_inline
void *
550 get_data_ptr(struct vhost_crypto_data_req
*vc_req
, struct vring_desc
*cur_desc
,
554 uint64_t dlen
= cur_desc
->len
;
556 data
= IOVA_TO_VVA(void *, vc_req
, cur_desc
->addr
, &dlen
, perm
);
557 if (unlikely(!data
|| dlen
!= cur_desc
->len
)) {
558 VC_LOG_ERR("Failed to map object");
566 copy_data(void *dst_data
, struct vhost_crypto_data_req
*vc_req
,
567 struct vring_desc
**cur_desc
, uint32_t size
,
568 uint32_t *nb_descs
, uint32_t vq_size
)
570 struct vring_desc
*desc
= *cur_desc
;
571 uint64_t remain
, addr
, dlen
, len
;
573 uint8_t *data
= dst_data
;
577 to_copy
= RTE_MIN(desc
->len
, (uint32_t)left
);
579 src
= IOVA_TO_VVA(uint8_t *, vc_req
, desc
->addr
, &dlen
,
581 if (unlikely(!src
|| !dlen
))
584 rte_memcpy((uint8_t *)data
, src
, dlen
);
587 if (unlikely(dlen
< to_copy
)) {
588 remain
= to_copy
- dlen
;
589 addr
= desc
->addr
+ dlen
;
593 src
= IOVA_TO_VVA(uint8_t *, vc_req
, addr
, &len
,
595 if (unlikely(!src
|| !len
)) {
596 VC_LOG_ERR("Failed to map descriptor");
600 rte_memcpy(data
, src
, len
);
609 while ((desc
->flags
& VRING_DESC_F_NEXT
) && left
> 0) {
610 if (unlikely(*nb_descs
== 0 || desc
->next
>= vq_size
)) {
611 VC_LOG_ERR("Invalid descriptors");
616 desc
= &vc_req
->head
[desc
->next
];
617 rte_prefetch0(&vc_req
->head
[desc
->next
]);
618 to_copy
= RTE_MIN(desc
->len
, (uint32_t)left
);
620 src
= IOVA_TO_VVA(uint8_t *, vc_req
, desc
->addr
, &dlen
,
622 if (unlikely(!src
|| !dlen
)) {
623 VC_LOG_ERR("Failed to map descriptor");
627 rte_memcpy(data
, src
, dlen
);
630 if (unlikely(dlen
< to_copy
)) {
631 remain
= to_copy
- dlen
;
632 addr
= desc
->addr
+ dlen
;
636 src
= IOVA_TO_VVA(uint8_t *, vc_req
, addr
, &len
,
638 if (unlikely(!src
|| !len
)) {
639 VC_LOG_ERR("Failed to map descriptor");
643 rte_memcpy(data
, src
, len
);
653 if (unlikely(left
> 0)) {
654 VC_LOG_ERR("Incorrect virtio descriptor");
658 if (unlikely(*nb_descs
== 0))
661 if (unlikely(desc
->next
>= vq_size
))
663 *cur_desc
= &vc_req
->head
[desc
->next
];
670 write_back_data(struct vhost_crypto_data_req
*vc_req
)
672 struct vhost_crypto_writeback_data
*wb_data
= vc_req
->wb
, *wb_last
;
675 rte_memcpy(wb_data
->dst
, wb_data
->src
, wb_data
->len
);
677 wb_data
= wb_data
->next
;
678 rte_mempool_put(vc_req
->wb_pool
, wb_last
);
683 free_wb_data(struct vhost_crypto_writeback_data
*wb_data
,
684 struct rte_mempool
*mp
)
686 while (wb_data
->next
!= NULL
)
687 free_wb_data(wb_data
->next
, mp
);
689 rte_mempool_put(mp
, wb_data
);
693 * The function will allocate a vhost_crypto_writeback_data linked list
694 * containing the source and destination data pointers for the write back
695 * operation after dequeued from Cryptodev PMD queues.
698 * The vhost crypto data request pointer
700 * The pointer of the current in use descriptor pointer. The content of
701 * cur_desc is expected to be updated after the function execution.
703 * The last write back data element to be returned. It is used only in cipher
704 * and hash chain operations.
706 * The source data pointer
708 * The offset to both source and destination data. For source data the offset
709 * is the number of bytes between src and start point of cipher operation. For
710 * destination data the offset is the number of bytes from *cur_desc->addr
711 * to the point where the src will be written to.
712 * @param write_back_len
713 * The size of the write back length.
715 * The pointer to the start of the write back data linked list.
717 static struct vhost_crypto_writeback_data
*
718 prepare_write_back_data(struct vhost_crypto_data_req
*vc_req
,
719 struct vring_desc
**cur_desc
,
720 struct vhost_crypto_writeback_data
**end_wb_data
,
723 uint64_t write_back_len
,
724 uint32_t *nb_descs
, uint32_t vq_size
)
726 struct vhost_crypto_writeback_data
*wb_data
, *head
;
727 struct vring_desc
*desc
= *cur_desc
;
732 ret
= rte_mempool_get(vc_req
->wb_pool
, (void **)&head
);
733 if (unlikely(ret
< 0)) {
734 VC_LOG_ERR("no memory");
740 if (likely(desc
->len
> offset
)) {
741 wb_data
->src
= src
+ offset
;
743 dst
= IOVA_TO_VVA(uint8_t *, vc_req
, desc
->addr
,
744 &dlen
, VHOST_ACCESS_RW
) + offset
;
745 if (unlikely(!dst
|| dlen
!= desc
->len
)) {
746 VC_LOG_ERR("Failed to map descriptor");
751 wb_data
->len
= desc
->len
- offset
;
752 write_back_len
-= wb_data
->len
;
753 src
+= offset
+ wb_data
->len
;
756 if (unlikely(write_back_len
)) {
757 ret
= rte_mempool_get(vc_req
->wb_pool
,
758 (void **)&(wb_data
->next
));
759 if (unlikely(ret
< 0)) {
760 VC_LOG_ERR("no memory");
764 wb_data
= wb_data
->next
;
766 wb_data
->next
= NULL
;
770 while (write_back_len
) {
771 if (unlikely(*nb_descs
== 0 || desc
->next
>= vq_size
)) {
772 VC_LOG_ERR("Invalid descriptors");
777 desc
= &vc_req
->head
[desc
->next
];
778 if (unlikely(!(desc
->flags
& VRING_DESC_F_WRITE
))) {
779 VC_LOG_ERR("incorrect descriptor");
783 if (desc
->len
<= offset
) {
789 dst
= IOVA_TO_VVA(uint8_t *, vc_req
, desc
->addr
, &dlen
,
790 VHOST_ACCESS_RW
) + offset
;
791 if (unlikely(dst
== NULL
|| dlen
!= desc
->len
)) {
792 VC_LOG_ERR("Failed to map descriptor");
798 wb_data
->len
= RTE_MIN(desc
->len
- offset
, write_back_len
);
799 write_back_len
-= wb_data
->len
;
803 if (write_back_len
) {
804 ret
= rte_mempool_get(vc_req
->wb_pool
,
805 (void **)&(wb_data
->next
));
806 if (unlikely(ret
< 0)) {
807 VC_LOG_ERR("no memory");
811 wb_data
= wb_data
->next
;
813 wb_data
->next
= NULL
;
816 if (unlikely(*nb_descs
== 0))
819 if (unlikely(desc
->next
>= vq_size
))
821 *cur_desc
= &vc_req
->head
[desc
->next
];
824 *end_wb_data
= wb_data
;
830 free_wb_data(head
, vc_req
->wb_pool
);
836 prepare_sym_cipher_op(struct vhost_crypto
*vcrypto
, struct rte_crypto_op
*op
,
837 struct vhost_crypto_data_req
*vc_req
,
838 struct virtio_crypto_cipher_data_req
*cipher
,
839 struct vring_desc
*cur_desc
,
840 uint32_t *nb_descs
, uint32_t vq_size
)
842 struct vring_desc
*desc
= cur_desc
;
843 struct vhost_crypto_writeback_data
*ewb
= NULL
;
844 struct rte_mbuf
*m_src
= op
->sym
->m_src
, *m_dst
= op
->sym
->m_dst
;
845 uint8_t *iv_data
= rte_crypto_op_ctod_offset(op
, uint8_t *, IV_OFFSET
);
850 if (unlikely(copy_data(iv_data
, vc_req
, &desc
, cipher
->para
.iv_len
,
851 nb_descs
, vq_size
) < 0)) {
852 ret
= VIRTIO_CRYPTO_BADMSG
;
856 m_src
->data_len
= cipher
->para
.src_data_len
;
858 switch (vcrypto
->option
) {
859 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
860 m_src
->buf_iova
= gpa_to_hpa(vcrypto
->dev
, desc
->addr
,
861 cipher
->para
.src_data_len
);
862 m_src
->buf_addr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_RO
);
863 if (unlikely(m_src
->buf_iova
== 0 ||
864 m_src
->buf_addr
== NULL
)) {
865 VC_LOG_ERR("zero_copy may fail due to cross page data");
866 ret
= VIRTIO_CRYPTO_ERR
;
870 if (unlikely(move_desc(vc_req
->head
, &desc
,
871 cipher
->para
.src_data_len
, nb_descs
,
873 VC_LOG_ERR("Incorrect descriptor");
874 ret
= VIRTIO_CRYPTO_ERR
;
879 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
880 vc_req
->wb_pool
= vcrypto
->wb_pool
;
882 if (unlikely(cipher
->para
.src_data_len
>
883 RTE_MBUF_DEFAULT_BUF_SIZE
)) {
884 VC_LOG_ERR("Not enough space to do data copy");
885 ret
= VIRTIO_CRYPTO_ERR
;
888 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src
, uint8_t *),
889 vc_req
, &desc
, cipher
->para
.src_data_len
,
890 nb_descs
, vq_size
) < 0)) {
891 ret
= VIRTIO_CRYPTO_BADMSG
;
896 ret
= VIRTIO_CRYPTO_BADMSG
;
901 desc
= find_write_desc(vc_req
->head
, desc
, nb_descs
, vq_size
);
902 if (unlikely(!desc
)) {
903 VC_LOG_ERR("Cannot find write location");
904 ret
= VIRTIO_CRYPTO_BADMSG
;
908 switch (vcrypto
->option
) {
909 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
910 m_dst
->buf_iova
= gpa_to_hpa(vcrypto
->dev
,
911 desc
->addr
, cipher
->para
.dst_data_len
);
912 m_dst
->buf_addr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_RW
);
913 if (unlikely(m_dst
->buf_iova
== 0 || m_dst
->buf_addr
== NULL
)) {
914 VC_LOG_ERR("zero_copy may fail due to cross page data");
915 ret
= VIRTIO_CRYPTO_ERR
;
919 if (unlikely(move_desc(vc_req
->head
, &desc
,
920 cipher
->para
.dst_data_len
,
921 nb_descs
, vq_size
) < 0)) {
922 VC_LOG_ERR("Incorrect descriptor");
923 ret
= VIRTIO_CRYPTO_ERR
;
927 m_dst
->data_len
= cipher
->para
.dst_data_len
;
929 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
930 vc_req
->wb
= prepare_write_back_data(vc_req
, &desc
, &ewb
,
931 rte_pktmbuf_mtod(m_src
, uint8_t *), 0,
932 cipher
->para
.dst_data_len
, nb_descs
, vq_size
);
933 if (unlikely(vc_req
->wb
== NULL
)) {
934 ret
= VIRTIO_CRYPTO_ERR
;
940 ret
= VIRTIO_CRYPTO_BADMSG
;
945 op
->type
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
;
946 op
->sess_type
= RTE_CRYPTO_OP_WITH_SESSION
;
948 op
->sym
->cipher
.data
.offset
= 0;
949 op
->sym
->cipher
.data
.length
= cipher
->para
.src_data_len
;
951 vc_req
->inhdr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_WO
);
952 if (unlikely(vc_req
->inhdr
== NULL
)) {
953 ret
= VIRTIO_CRYPTO_BADMSG
;
957 vc_req
->inhdr
->status
= VIRTIO_CRYPTO_OK
;
958 vc_req
->len
= cipher
->para
.dst_data_len
+ INHDR_LEN
;
964 free_wb_data(vc_req
->wb
, vc_req
->wb_pool
);
966 vc_req
->len
= INHDR_LEN
;
971 prepare_sym_chain_op(struct vhost_crypto
*vcrypto
, struct rte_crypto_op
*op
,
972 struct vhost_crypto_data_req
*vc_req
,
973 struct virtio_crypto_alg_chain_data_req
*chain
,
974 struct vring_desc
*cur_desc
,
975 uint32_t *nb_descs
, uint32_t vq_size
)
977 struct vring_desc
*desc
= cur_desc
, *digest_desc
;
978 struct vhost_crypto_writeback_data
*ewb
= NULL
, *ewb2
= NULL
;
979 struct rte_mbuf
*m_src
= op
->sym
->m_src
, *m_dst
= op
->sym
->m_dst
;
980 uint8_t *iv_data
= rte_crypto_op_ctod_offset(op
, uint8_t *, IV_OFFSET
);
981 uint32_t digest_offset
;
987 if (unlikely(copy_data(iv_data
, vc_req
, &desc
,
988 chain
->para
.iv_len
, nb_descs
, vq_size
) < 0)) {
989 ret
= VIRTIO_CRYPTO_BADMSG
;
993 m_src
->data_len
= chain
->para
.src_data_len
;
995 switch (vcrypto
->option
) {
996 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
997 m_dst
->data_len
= chain
->para
.dst_data_len
;
999 m_src
->buf_iova
= gpa_to_hpa(vcrypto
->dev
, desc
->addr
,
1000 chain
->para
.src_data_len
);
1001 m_src
->buf_addr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_RO
);
1002 if (unlikely(m_src
->buf_iova
== 0 || m_src
->buf_addr
== NULL
)) {
1003 VC_LOG_ERR("zero_copy may fail due to cross page data");
1004 ret
= VIRTIO_CRYPTO_ERR
;
1008 if (unlikely(move_desc(vc_req
->head
, &desc
,
1009 chain
->para
.src_data_len
,
1010 nb_descs
, vq_size
) < 0)) {
1011 VC_LOG_ERR("Incorrect descriptor");
1012 ret
= VIRTIO_CRYPTO_ERR
;
1016 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
1017 vc_req
->wb_pool
= vcrypto
->wb_pool
;
1019 if (unlikely(chain
->para
.src_data_len
>
1020 RTE_MBUF_DEFAULT_BUF_SIZE
)) {
1021 VC_LOG_ERR("Not enough space to do data copy");
1022 ret
= VIRTIO_CRYPTO_ERR
;
1025 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src
, uint8_t *),
1026 vc_req
, &desc
, chain
->para
.src_data_len
,
1027 nb_descs
, vq_size
)) < 0) {
1028 ret
= VIRTIO_CRYPTO_BADMSG
;
1034 ret
= VIRTIO_CRYPTO_BADMSG
;
1039 desc
= find_write_desc(vc_req
->head
, desc
, nb_descs
, vq_size
);
1040 if (unlikely(!desc
)) {
1041 VC_LOG_ERR("Cannot find write location");
1042 ret
= VIRTIO_CRYPTO_BADMSG
;
1046 switch (vcrypto
->option
) {
1047 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
1048 m_dst
->buf_iova
= gpa_to_hpa(vcrypto
->dev
,
1049 desc
->addr
, chain
->para
.dst_data_len
);
1050 m_dst
->buf_addr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_RW
);
1051 if (unlikely(m_dst
->buf_iova
== 0 || m_dst
->buf_addr
== NULL
)) {
1052 VC_LOG_ERR("zero_copy may fail due to cross page data");
1053 ret
= VIRTIO_CRYPTO_ERR
;
1057 if (unlikely(move_desc(vc_req
->head
, &desc
,
1058 chain
->para
.dst_data_len
,
1059 nb_descs
, vq_size
) < 0)) {
1060 VC_LOG_ERR("Incorrect descriptor");
1061 ret
= VIRTIO_CRYPTO_ERR
;
1065 op
->sym
->auth
.digest
.phys_addr
= gpa_to_hpa(vcrypto
->dev
,
1066 desc
->addr
, chain
->para
.hash_result_len
);
1067 op
->sym
->auth
.digest
.data
= get_data_ptr(vc_req
, desc
,
1069 if (unlikely(op
->sym
->auth
.digest
.phys_addr
== 0)) {
1070 VC_LOG_ERR("zero_copy may fail due to cross page data");
1071 ret
= VIRTIO_CRYPTO_ERR
;
1075 if (unlikely(move_desc(vc_req
->head
, &desc
,
1076 chain
->para
.hash_result_len
,
1077 nb_descs
, vq_size
) < 0)) {
1078 VC_LOG_ERR("Incorrect descriptor");
1079 ret
= VIRTIO_CRYPTO_ERR
;
1084 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
1085 vc_req
->wb
= prepare_write_back_data(vc_req
, &desc
, &ewb
,
1086 rte_pktmbuf_mtod(m_src
, uint8_t *),
1087 chain
->para
.cipher_start_src_offset
,
1088 chain
->para
.dst_data_len
-
1089 chain
->para
.cipher_start_src_offset
,
1091 if (unlikely(vc_req
->wb
== NULL
)) {
1092 ret
= VIRTIO_CRYPTO_ERR
;
1096 digest_offset
= m_src
->data_len
;
1097 digest_addr
= rte_pktmbuf_mtod_offset(m_src
, void *,
1101 /** create a wb_data for digest */
1102 ewb
->next
= prepare_write_back_data(vc_req
, &desc
, &ewb2
,
1103 digest_addr
, 0, chain
->para
.hash_result_len
,
1105 if (unlikely(ewb
->next
== NULL
)) {
1106 ret
= VIRTIO_CRYPTO_ERR
;
1110 if (unlikely(copy_data(digest_addr
, vc_req
, &digest_desc
,
1111 chain
->para
.hash_result_len
,
1112 nb_descs
, vq_size
) < 0)) {
1113 ret
= VIRTIO_CRYPTO_BADMSG
;
1117 op
->sym
->auth
.digest
.data
= digest_addr
;
1118 op
->sym
->auth
.digest
.phys_addr
= rte_pktmbuf_iova_offset(m_src
,
1122 ret
= VIRTIO_CRYPTO_BADMSG
;
1127 vc_req
->inhdr
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_WO
);
1128 if (unlikely(vc_req
->inhdr
== NULL
)) {
1129 ret
= VIRTIO_CRYPTO_BADMSG
;
1133 vc_req
->inhdr
->status
= VIRTIO_CRYPTO_OK
;
1135 op
->type
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
;
1136 op
->sess_type
= RTE_CRYPTO_OP_WITH_SESSION
;
1138 op
->sym
->cipher
.data
.offset
= chain
->para
.cipher_start_src_offset
;
1139 op
->sym
->cipher
.data
.length
= chain
->para
.src_data_len
-
1140 chain
->para
.cipher_start_src_offset
;
1142 op
->sym
->auth
.data
.offset
= chain
->para
.hash_start_src_offset
;
1143 op
->sym
->auth
.data
.length
= chain
->para
.len_to_hash
;
1145 vc_req
->len
= chain
->para
.dst_data_len
+ chain
->para
.hash_result_len
+
1151 free_wb_data(vc_req
->wb
, vc_req
->wb_pool
);
1152 vc_req
->len
= INHDR_LEN
;
1157 * Process on descriptor
1159 static __rte_always_inline
int
1160 vhost_crypto_process_one_req(struct vhost_crypto
*vcrypto
,
1161 struct vhost_virtqueue
*vq
, struct rte_crypto_op
*op
,
1162 struct vring_desc
*head
, uint16_t desc_idx
)
1164 struct vhost_crypto_data_req
*vc_req
= rte_mbuf_to_priv(op
->sym
->m_src
);
1165 struct rte_cryptodev_sym_session
*session
;
1166 struct virtio_crypto_op_data_req
*req
, tmp_req
;
1167 struct virtio_crypto_inhdr
*inhdr
;
1168 struct vring_desc
*desc
= NULL
;
1169 uint64_t session_id
;
1171 uint32_t nb_descs
= vq
->size
;
1174 vc_req
->desc_idx
= desc_idx
;
1175 vc_req
->dev
= vcrypto
->dev
;
1178 if (likely(head
->flags
& VRING_DESC_F_INDIRECT
)) {
1180 nb_descs
= dlen
/ sizeof(struct vring_desc
);
1181 /* drop invalid descriptors */
1182 if (unlikely(nb_descs
> vq
->size
))
1184 desc
= IOVA_TO_VVA(struct vring_desc
*, vc_req
, head
->addr
,
1185 &dlen
, VHOST_ACCESS_RO
);
1186 if (unlikely(!desc
|| dlen
!= head
->len
))
1194 vc_req
->head
= head
;
1195 vc_req
->zero_copy
= vcrypto
->option
;
1197 req
= get_data_ptr(vc_req
, desc
, VHOST_ACCESS_RO
);
1198 if (unlikely(req
== NULL
)) {
1199 switch (vcrypto
->option
) {
1200 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
1201 err
= VIRTIO_CRYPTO_BADMSG
;
1202 VC_LOG_ERR("Invalid descriptor");
1204 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
1206 if (unlikely(copy_data(req
, vc_req
, &desc
, sizeof(*req
),
1207 &nb_descs
, vq
->size
) < 0)) {
1208 err
= VIRTIO_CRYPTO_BADMSG
;
1209 VC_LOG_ERR("Invalid descriptor");
1214 err
= VIRTIO_CRYPTO_ERR
;
1215 VC_LOG_ERR("Invalid option");
1219 if (unlikely(move_desc(vc_req
->head
, &desc
,
1220 sizeof(*req
), &nb_descs
, vq
->size
) < 0)) {
1221 VC_LOG_ERR("Incorrect descriptor");
1226 switch (req
->header
.opcode
) {
1227 case VIRTIO_CRYPTO_CIPHER_ENCRYPT
:
1228 case VIRTIO_CRYPTO_CIPHER_DECRYPT
:
1229 session_id
= req
->header
.session_id
;
1231 /* one branch to avoid unnecessary table lookup */
1232 if (vcrypto
->cache_session_id
!= session_id
) {
1233 err
= rte_hash_lookup_data(vcrypto
->session_map
,
1234 &session_id
, (void **)&session
);
1235 if (unlikely(err
< 0)) {
1236 err
= VIRTIO_CRYPTO_ERR
;
1237 VC_LOG_ERR("Failed to find session %"PRIu64
,
1242 vcrypto
->cache_session
= session
;
1243 vcrypto
->cache_session_id
= session_id
;
1246 session
= vcrypto
->cache_session
;
1248 err
= rte_crypto_op_attach_sym_session(op
, session
);
1249 if (unlikely(err
< 0)) {
1250 err
= VIRTIO_CRYPTO_ERR
;
1251 VC_LOG_ERR("Failed to attach session to op");
1255 switch (req
->u
.sym_req
.op_type
) {
1256 case VIRTIO_CRYPTO_SYM_OP_NONE
:
1257 err
= VIRTIO_CRYPTO_NOTSUPP
;
1259 case VIRTIO_CRYPTO_SYM_OP_CIPHER
:
1260 err
= prepare_sym_cipher_op(vcrypto
, op
, vc_req
,
1261 &req
->u
.sym_req
.u
.cipher
, desc
,
1262 &nb_descs
, vq
->size
);
1264 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING
:
1265 err
= prepare_sym_chain_op(vcrypto
, op
, vc_req
,
1266 &req
->u
.sym_req
.u
.chain
, desc
,
1267 &nb_descs
, vq
->size
);
1270 if (unlikely(err
!= 0)) {
1271 VC_LOG_ERR("Failed to process sym request");
1276 VC_LOG_ERR("Unsupported symmetric crypto request type %u",
1277 req
->header
.opcode
);
1285 inhdr
= reach_inhdr(vc_req
, desc
, &nb_descs
, vq
->size
);
1286 if (likely(inhdr
!= NULL
))
1287 inhdr
->status
= (uint8_t)err
;
1292 static __rte_always_inline
struct vhost_virtqueue
*
1293 vhost_crypto_finalize_one_request(struct rte_crypto_op
*op
,
1294 struct vhost_virtqueue
*old_vq
)
1296 struct rte_mbuf
*m_src
= op
->sym
->m_src
;
1297 struct rte_mbuf
*m_dst
= op
->sym
->m_dst
;
1298 struct vhost_crypto_data_req
*vc_req
= rte_mbuf_to_priv(m_src
);
1301 if (unlikely(!vc_req
)) {
1302 VC_LOG_ERR("Failed to retrieve vc_req");
1306 if (old_vq
&& (vc_req
->vq
!= old_vq
))
1309 desc_idx
= vc_req
->desc_idx
;
1311 if (unlikely(op
->status
!= RTE_CRYPTO_OP_STATUS_SUCCESS
))
1312 vc_req
->inhdr
->status
= VIRTIO_CRYPTO_ERR
;
1314 if (vc_req
->zero_copy
== 0)
1315 write_back_data(vc_req
);
1318 vc_req
->vq
->used
->ring
[desc_idx
].id
= desc_idx
;
1319 vc_req
->vq
->used
->ring
[desc_idx
].len
= vc_req
->len
;
1321 rte_mempool_put(m_src
->pool
, (void *)m_src
);
1324 rte_mempool_put(m_dst
->pool
, (void *)m_dst
);
1329 static __rte_always_inline
uint16_t
1330 vhost_crypto_complete_one_vm_requests(struct rte_crypto_op
**ops
,
1331 uint16_t nb_ops
, int *callfd
)
1333 uint16_t processed
= 1;
1334 struct vhost_virtqueue
*vq
, *tmp_vq
;
1336 if (unlikely(nb_ops
== 0))
1339 vq
= vhost_crypto_finalize_one_request(ops
[0], NULL
);
1340 if (unlikely(vq
== NULL
))
1344 while ((processed
< nb_ops
)) {
1345 tmp_vq
= vhost_crypto_finalize_one_request(ops
[processed
],
1348 if (unlikely(vq
!= tmp_vq
))
1354 *callfd
= vq
->callfd
;
1356 *(volatile uint16_t *)&vq
->used
->idx
+= processed
;
1361 int __rte_experimental
1362 rte_vhost_crypto_create(int vid
, uint8_t cryptodev_id
,
1363 struct rte_mempool
*sess_pool
,
1364 struct rte_mempool
*sess_priv_pool
,
1367 struct virtio_net
*dev
= get_device(vid
);
1368 struct rte_hash_parameters params
= {0};
1369 struct vhost_crypto
*vcrypto
;
1374 VC_LOG_ERR("Invalid vid %i", vid
);
1378 ret
= rte_vhost_driver_set_features(dev
->ifname
,
1379 VIRTIO_CRYPTO_FEATURES
);
1381 VC_LOG_ERR("Error setting features");
1385 vcrypto
= rte_zmalloc_socket(NULL
, sizeof(*vcrypto
),
1386 RTE_CACHE_LINE_SIZE
, socket_id
);
1388 VC_LOG_ERR("Insufficient memory");
1392 vcrypto
->sess_pool
= sess_pool
;
1393 vcrypto
->sess_priv_pool
= sess_priv_pool
;
1394 vcrypto
->cid
= cryptodev_id
;
1395 vcrypto
->cache_session_id
= UINT64_MAX
;
1396 vcrypto
->last_session_id
= 1;
1398 vcrypto
->option
= RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
;
1400 snprintf(name
, 127, "HASH_VHOST_CRYPT_%u", (uint32_t)vid
);
1402 params
.entries
= VHOST_CRYPTO_SESSION_MAP_ENTRIES
;
1403 params
.hash_func
= rte_jhash
;
1404 params
.key_len
= sizeof(uint64_t);
1405 params
.socket_id
= socket_id
;
1406 vcrypto
->session_map
= rte_hash_create(¶ms
);
1407 if (!vcrypto
->session_map
) {
1408 VC_LOG_ERR("Failed to creath session map");
1413 snprintf(name
, 127, "MBUF_POOL_VM_%u", (uint32_t)vid
);
1414 vcrypto
->mbuf_pool
= rte_pktmbuf_pool_create(name
,
1415 VHOST_CRYPTO_MBUF_POOL_SIZE
, 512,
1416 sizeof(struct vhost_crypto_data_req
),
1417 RTE_MBUF_DEFAULT_DATAROOM
* 2 + RTE_PKTMBUF_HEADROOM
,
1419 if (!vcrypto
->mbuf_pool
) {
1420 VC_LOG_ERR("Failed to creath mbuf pool");
1425 snprintf(name
, 127, "WB_POOL_VM_%u", (uint32_t)vid
);
1426 vcrypto
->wb_pool
= rte_mempool_create(name
,
1427 VHOST_CRYPTO_MBUF_POOL_SIZE
,
1428 sizeof(struct vhost_crypto_writeback_data
),
1429 128, 0, NULL
, NULL
, NULL
, NULL
,
1430 rte_socket_id(), 0);
1431 if (!vcrypto
->wb_pool
) {
1432 VC_LOG_ERR("Failed to creath mempool");
1437 dev
->extern_data
= vcrypto
;
1438 dev
->extern_ops
.pre_msg_handle
= NULL
;
1439 dev
->extern_ops
.post_msg_handle
= vhost_crypto_msg_post_handler
;
1444 if (vcrypto
->session_map
)
1445 rte_hash_free(vcrypto
->session_map
);
1446 if (vcrypto
->mbuf_pool
)
1447 rte_mempool_free(vcrypto
->mbuf_pool
);
1454 int __rte_experimental
1455 rte_vhost_crypto_free(int vid
)
1457 struct virtio_net
*dev
= get_device(vid
);
1458 struct vhost_crypto
*vcrypto
;
1460 if (unlikely(dev
== NULL
)) {
1461 VC_LOG_ERR("Invalid vid %i", vid
);
1465 vcrypto
= dev
->extern_data
;
1466 if (unlikely(vcrypto
== NULL
)) {
1467 VC_LOG_ERR("Cannot find required data, is it initialized?");
1471 rte_hash_free(vcrypto
->session_map
);
1472 rte_mempool_free(vcrypto
->mbuf_pool
);
1473 rte_mempool_free(vcrypto
->wb_pool
);
1476 dev
->extern_data
= NULL
;
1477 dev
->extern_ops
.pre_msg_handle
= NULL
;
1478 dev
->extern_ops
.post_msg_handle
= NULL
;
1483 int __rte_experimental
1484 rte_vhost_crypto_set_zero_copy(int vid
, enum rte_vhost_crypto_zero_copy option
)
1486 struct virtio_net
*dev
= get_device(vid
);
1487 struct vhost_crypto
*vcrypto
;
1489 if (unlikely(dev
== NULL
)) {
1490 VC_LOG_ERR("Invalid vid %i", vid
);
1494 if (unlikely((uint32_t)option
>=
1495 RTE_VHOST_CRYPTO_MAX_ZERO_COPY_OPTIONS
)) {
1496 VC_LOG_ERR("Invalid option %i", option
);
1500 vcrypto
= (struct vhost_crypto
*)dev
->extern_data
;
1501 if (unlikely(vcrypto
== NULL
)) {
1502 VC_LOG_ERR("Cannot find required data, is it initialized?");
1506 if (vcrypto
->option
== (uint8_t)option
)
1509 if (!(rte_mempool_full(vcrypto
->mbuf_pool
)) ||
1510 !(rte_mempool_full(vcrypto
->wb_pool
))) {
1511 VC_LOG_ERR("Cannot update zero copy as mempool is not full");
1515 if (option
== RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
) {
1518 snprintf(name
, 127, "WB_POOL_VM_%u", (uint32_t)vid
);
1519 vcrypto
->wb_pool
= rte_mempool_create(name
,
1520 VHOST_CRYPTO_MBUF_POOL_SIZE
,
1521 sizeof(struct vhost_crypto_writeback_data
),
1522 128, 0, NULL
, NULL
, NULL
, NULL
,
1523 rte_socket_id(), 0);
1524 if (!vcrypto
->wb_pool
) {
1525 VC_LOG_ERR("Failed to creath mbuf pool");
1529 rte_mempool_free(vcrypto
->wb_pool
);
1530 vcrypto
->wb_pool
= NULL
;
1533 vcrypto
->option
= (uint8_t)option
;
1538 uint16_t __rte_experimental
1539 rte_vhost_crypto_fetch_requests(int vid
, uint32_t qid
,
1540 struct rte_crypto_op
**ops
, uint16_t nb_ops
)
1542 struct rte_mbuf
*mbufs
[VHOST_CRYPTO_MAX_BURST_SIZE
* 2];
1543 struct virtio_net
*dev
= get_device(vid
);
1544 struct vhost_crypto
*vcrypto
;
1545 struct vhost_virtqueue
*vq
;
1551 if (unlikely(dev
== NULL
)) {
1552 VC_LOG_ERR("Invalid vid %i", vid
);
1556 if (unlikely(qid
>= VHOST_MAX_QUEUE_PAIRS
)) {
1557 VC_LOG_ERR("Invalid qid %u", qid
);
1561 vcrypto
= (struct vhost_crypto
*)dev
->extern_data
;
1562 if (unlikely(vcrypto
== NULL
)) {
1563 VC_LOG_ERR("Cannot find required data, is it initialized?");
1567 vq
= dev
->virtqueue
[qid
];
1569 avail_idx
= *((volatile uint16_t *)&vq
->avail
->idx
);
1570 start_idx
= vq
->last_used_idx
;
1571 count
= avail_idx
- start_idx
;
1572 count
= RTE_MIN(count
, VHOST_CRYPTO_MAX_BURST_SIZE
);
1573 count
= RTE_MIN(count
, nb_ops
);
1575 if (unlikely(count
== 0))
1578 /* for zero copy, we need 2 empty mbufs for src and dst, otherwise
1579 * we need only 1 mbuf as src and dst
1581 switch (vcrypto
->option
) {
1582 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE
:
1583 if (unlikely(rte_mempool_get_bulk(vcrypto
->mbuf_pool
,
1584 (void **)mbufs
, count
* 2) < 0)) {
1585 VC_LOG_ERR("Insufficient memory");
1589 for (i
= 0; i
< count
; i
++) {
1590 uint16_t used_idx
= (start_idx
+ i
) & (vq
->size
- 1);
1591 uint16_t desc_idx
= vq
->avail
->ring
[used_idx
];
1592 struct vring_desc
*head
= &vq
->desc
[desc_idx
];
1593 struct rte_crypto_op
*op
= ops
[i
];
1595 op
->sym
->m_src
= mbufs
[i
* 2];
1596 op
->sym
->m_dst
= mbufs
[i
* 2 + 1];
1597 op
->sym
->m_src
->data_off
= 0;
1598 op
->sym
->m_dst
->data_off
= 0;
1600 if (unlikely(vhost_crypto_process_one_req(vcrypto
, vq
,
1601 op
, head
, desc_idx
)) < 0)
1605 if (unlikely(i
< count
))
1606 rte_mempool_put_bulk(vcrypto
->mbuf_pool
,
1607 (void **)&mbufs
[i
* 2],
1612 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE
:
1613 if (unlikely(rte_mempool_get_bulk(vcrypto
->mbuf_pool
,
1614 (void **)mbufs
, count
) < 0)) {
1615 VC_LOG_ERR("Insufficient memory");
1619 for (i
= 0; i
< count
; i
++) {
1620 uint16_t used_idx
= (start_idx
+ i
) & (vq
->size
- 1);
1621 uint16_t desc_idx
= vq
->avail
->ring
[used_idx
];
1622 struct vring_desc
*head
= &vq
->desc
[desc_idx
];
1623 struct rte_crypto_op
*op
= ops
[i
];
1625 op
->sym
->m_src
= mbufs
[i
];
1626 op
->sym
->m_dst
= NULL
;
1627 op
->sym
->m_src
->data_off
= 0;
1629 if (unlikely(vhost_crypto_process_one_req(vcrypto
, vq
,
1630 op
, head
, desc_idx
) < 0))
1634 if (unlikely(i
< count
))
1635 rte_mempool_put_bulk(vcrypto
->mbuf_pool
,
1643 vq
->last_used_idx
+= i
;
1648 uint16_t __rte_experimental
1649 rte_vhost_crypto_finalize_requests(struct rte_crypto_op
**ops
,
1650 uint16_t nb_ops
, int *callfds
, uint16_t *nb_callfds
)
1652 struct rte_crypto_op
**tmp_ops
= ops
;
1653 uint16_t count
= 0, left
= nb_ops
;
1658 count
= vhost_crypto_complete_one_vm_requests(tmp_ops
, left
,
1660 if (unlikely(count
== 0))
1663 tmp_ops
= &tmp_ops
[count
];
1666 callfds
[idx
++] = callfd
;
1668 if (unlikely(idx
>= VIRTIO_CRYPTO_MAX_NUM_BURST_VQS
)) {
1669 VC_LOG_ERR("Too many vqs");
1676 return nb_ops
- left
;