1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Marvell International Ltd.
3 * Copyright(c) 2017 Semihalf.
9 #include <rte_common.h>
10 #include <rte_malloc.h>
11 #include <rte_cryptodev_pmd.h>
13 #include "mrvl_pmd_private.h"
16 * Capabilities list to be used in reporting to DPDK.
18 static const struct rte_cryptodev_capabilities
19 mrvl_crypto_pmd_capabilities
[] = {
21 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
23 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
25 .algo
= RTE_CRYPTO_AUTH_MD5_HMAC
,
41 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
43 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
45 .algo
= RTE_CRYPTO_AUTH_MD5
,
61 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
63 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
65 .algo
= RTE_CRYPTO_AUTH_SHA1_HMAC
,
81 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
83 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
85 .algo
= RTE_CRYPTO_AUTH_SHA1
,
102 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
104 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
106 .algo
= RTE_CRYPTO_AUTH_SHA224_HMAC
,
122 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
124 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
126 .algo
= RTE_CRYPTO_AUTH_SHA224
,
142 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
144 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
146 .algo
= RTE_CRYPTO_AUTH_SHA256_HMAC
,
162 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
164 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
166 .algo
= RTE_CRYPTO_AUTH_SHA256
,
182 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
184 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
186 .algo
= RTE_CRYPTO_AUTH_SHA384_HMAC
,
202 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
204 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
206 .algo
= RTE_CRYPTO_AUTH_SHA384
,
222 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
224 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
226 .algo
= RTE_CRYPTO_AUTH_SHA512_HMAC
,
242 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
244 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
246 .algo
= RTE_CRYPTO_AUTH_SHA512
,
262 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
264 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
266 .algo
= RTE_CRYPTO_CIPHER_AES_CBC
,
282 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
284 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
286 .algo
= RTE_CRYPTO_CIPHER_AES_CTR
,
302 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
304 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
306 .algo
= RTE_CRYPTO_CIPHER_AES_ECB
,
322 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
324 .xform_type
= RTE_CRYPTO_SYM_XFORM_AEAD
,
326 .algo
= RTE_CRYPTO_AEAD_AES_GCM
,
351 { /* AES GMAC (AUTH) */
352 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
354 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
356 .algo
= RTE_CRYPTO_AUTH_AES_GMAC
,
377 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
379 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
381 .algo
= RTE_CRYPTO_CIPHER_3DES_CBC
,
397 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
399 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
401 .algo
= RTE_CRYPTO_CIPHER_3DES_CTR
,
417 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
419 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
421 .algo
= RTE_CRYPTO_CIPHER_3DES_ECB
,
437 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
439 .xform_type
= RTE_CRYPTO_SYM_XFORM_AUTH
,
441 .algo
= RTE_CRYPTO_AUTH_NULL
,
461 { /* NULL (CIPHER) */
462 .op
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
464 .xform_type
= RTE_CRYPTO_SYM_XFORM_CIPHER
,
466 .algo
= RTE_CRYPTO_CIPHER_NULL
,
482 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
487 * Configure device (PMD ops callback).
489 * @param dev Pointer to the device structure.
490 * @param config Pointer to configuration structure.
491 * @returns 0. Always.
494 mrvl_crypto_pmd_config(__rte_unused
struct rte_cryptodev
*dev
,
495 __rte_unused
struct rte_cryptodev_config
*config
)
501 * Start device (PMD ops callback).
503 * @param dev Pointer to the device structure.
504 * @returns 0. Always.
507 mrvl_crypto_pmd_start(__rte_unused
struct rte_cryptodev
*dev
)
513 * Stop device (PMD ops callback).
515 * @param dev Pointer to the device structure.
516 * @returns 0. Always.
519 mrvl_crypto_pmd_stop(__rte_unused
struct rte_cryptodev
*dev
)
524 * Get device statistics (PMD ops callback).
526 * @param dev Pointer to the device structure.
527 * @param stats Pointer to statistics structure [out].
530 mrvl_crypto_pmd_stats_get(struct rte_cryptodev
*dev
,
531 struct rte_cryptodev_stats
*stats
)
535 for (qp_id
= 0; qp_id
< dev
->data
->nb_queue_pairs
; qp_id
++) {
536 struct mrvl_crypto_qp
*qp
= dev
->data
->queue_pairs
[qp_id
];
538 stats
->enqueued_count
+= qp
->stats
.enqueued_count
;
539 stats
->dequeued_count
+= qp
->stats
.dequeued_count
;
541 stats
->enqueue_err_count
+= qp
->stats
.enqueue_err_count
;
542 stats
->dequeue_err_count
+= qp
->stats
.dequeue_err_count
;
547 * Reset device statistics (PMD ops callback).
549 * @param dev Pointer to the device structure.
552 mrvl_crypto_pmd_stats_reset(struct rte_cryptodev
*dev
)
556 for (qp_id
= 0; qp_id
< dev
->data
->nb_queue_pairs
; qp_id
++) {
557 struct mrvl_crypto_qp
*qp
= dev
->data
->queue_pairs
[qp_id
];
559 memset(&qp
->stats
, 0, sizeof(qp
->stats
));
564 * Get device info (PMD ops callback).
566 * @param dev Pointer to the device structure.
567 * @param dev_info Pointer to the device info structure [out].
570 mrvl_crypto_pmd_info_get(struct rte_cryptodev
*dev
,
571 struct rte_cryptodev_info
*dev_info
)
573 struct mrvl_crypto_private
*internals
= dev
->data
->dev_private
;
575 if (dev_info
!= NULL
) {
576 dev_info
->driver_id
= dev
->driver_id
;
577 dev_info
->feature_flags
= dev
->feature_flags
;
578 dev_info
->capabilities
= mrvl_crypto_pmd_capabilities
;
579 dev_info
->max_nb_queue_pairs
= internals
->max_nb_qpairs
;
580 dev_info
->sym
.max_nb_sessions
= internals
->max_nb_sessions
;
585 * Release queue pair (PMD ops callback).
587 * @param dev Pointer to the device structure.
588 * @param qp_id ID of Queue Pair to release.
589 * @returns 0. Always.
592 mrvl_crypto_pmd_qp_release(struct rte_cryptodev
*dev
, uint16_t qp_id
)
594 struct mrvl_crypto_qp
*qp
=
595 (struct mrvl_crypto_qp
*)dev
->data
->queue_pairs
[qp_id
];
597 if (dev
->data
->queue_pairs
[qp_id
] != NULL
) {
598 sam_cio_flush(qp
->cio
);
599 sam_cio_deinit(qp
->cio
);
600 rte_free(dev
->data
->queue_pairs
[qp_id
]);
601 dev
->data
->queue_pairs
[qp_id
] = NULL
;
608 * Close device (PMD ops callback).
610 * @param dev Pointer to the device structure.
611 * @returns 0. Always.
614 mrvl_crypto_pmd_close(struct rte_cryptodev
*dev
)
618 for (qp_id
= 0; qp_id
< dev
->data
->nb_queue_pairs
; qp_id
++)
619 mrvl_crypto_pmd_qp_release(dev
, qp_id
);
625 * Setup a queue pair (PMD ops callback).
627 * @param dev Pointer to the device structure.
628 * @param qp_id ID of the Queue Pair.
629 * @param qp_conf Queue pair configuration (nb of descriptors).
630 * @param socket_id NUMA socket to allocate memory on.
631 * @returns 0 upon success, negative value otherwise.
634 mrvl_crypto_pmd_qp_setup(struct rte_cryptodev
*dev
, uint16_t qp_id
,
635 const struct rte_cryptodev_qp_conf
*qp_conf
,
638 struct mrvl_crypto_qp
*qp
= NULL
;
639 char match
[RTE_CRYPTODEV_NAME_MAX_LEN
];
642 /* Allocate the queue pair data structure. */
643 qp
= rte_zmalloc_socket("MRVL Crypto PMD Queue Pair", sizeof(*qp
),
644 RTE_CACHE_LINE_SIZE
, socket_id
);
648 /* Free old qp prior setup if needed. */
649 if (dev
->data
->queue_pairs
[qp_id
] != NULL
)
650 mrvl_crypto_pmd_qp_release(dev
, qp_id
);
652 do { /* Error handling block */
655 * This extra check is necessary due to a bug in
658 int num
= sam_get_num_inst();
660 MRVL_LOG(ERR
, "No crypto engines detected!");
665 * In case two crypto engines are enabled qps will
666 * be evenly spread among them. Even and odd qps will
667 * be handled by cio-0 and cio-1 respectively. qp-cio mapping
668 * will look as follows:
671 * cio-x:y: cio-0:0, cio-1:0, cio-0:1, cio-1:1
674 * cio-x:y: cio-0:2, cio-1:2, cio-0:3, cio-1:3
676 * In case just one engine is enabled mapping will look as
679 * cio-x:y: cio-0:0, cio-0:1, cio-0:2, cio-0:3
681 n
= snprintf(match
, sizeof(match
), "cio-%u:%u",
682 qp_id
% num
, qp_id
/ num
);
684 if (n
>= sizeof(match
))
687 qp
->cio_params
.match
= match
;
688 qp
->cio_params
.size
= qp_conf
->nb_descriptors
;
690 if (sam_cio_init(&qp
->cio_params
, &qp
->cio
) < 0)
693 qp
->sess_mp
= qp_conf
->mp_session
;
694 qp
->sess_mp_priv
= qp_conf
->mp_session_private
;
696 memset(&qp
->stats
, 0, sizeof(qp
->stats
));
697 dev
->data
->queue_pairs
[qp_id
] = qp
;
705 /** Returns the size of the session structure (PMD ops callback).
707 * @param dev Pointer to the device structure [Unused].
708 * @returns Size of Marvell crypto session.
711 mrvl_crypto_pmd_sym_session_get_size(__rte_unused
struct rte_cryptodev
*dev
)
713 return sizeof(struct mrvl_crypto_session
);
716 /** Configure the session from a crypto xform chain (PMD ops callback).
718 * @param dev Pointer to the device structure.
719 * @param xform Pointer to the crypto configuration structure.
720 * @param sess Pointer to the empty session structure.
721 * @returns 0 upon success, negative value otherwise.
724 mrvl_crypto_pmd_sym_session_configure(__rte_unused
struct rte_cryptodev
*dev
,
725 struct rte_crypto_sym_xform
*xform
,
726 struct rte_cryptodev_sym_session
*sess
,
727 struct rte_mempool
*mp
)
729 struct mrvl_crypto_session
*mrvl_sess
;
730 void *sess_private_data
;
734 MRVL_LOG(ERR
, "Invalid session struct!");
738 if (rte_mempool_get(mp
, &sess_private_data
)) {
739 CDEV_LOG_ERR("Couldn't get object from session mempool.");
743 memset(sess_private_data
, 0, sizeof(struct mrvl_crypto_session
));
745 ret
= mrvl_crypto_set_session_parameters(sess_private_data
, xform
);
747 MRVL_LOG(ERR
, "Failed to configure session parameters!");
749 /* Return session to mempool */
750 rte_mempool_put(mp
, sess_private_data
);
754 set_sym_session_private_data(sess
, dev
->driver_id
, sess_private_data
);
756 mrvl_sess
= (struct mrvl_crypto_session
*)sess_private_data
;
757 if (sam_session_create(&mrvl_sess
->sam_sess_params
,
758 &mrvl_sess
->sam_sess
) < 0) {
759 MRVL_LOG(DEBUG
, "Failed to create session!");
763 /* free the keys memory allocated for session creation */
764 if (mrvl_sess
->sam_sess_params
.cipher_key
!= NULL
)
765 free(mrvl_sess
->sam_sess_params
.cipher_key
);
766 if (mrvl_sess
->sam_sess_params
.auth_key
!= NULL
)
767 free(mrvl_sess
->sam_sess_params
.auth_key
);
773 * Clear the memory of session so it doesn't leave key material behind.
775 * @param dev Pointer to the device structure.
776 * @returns 0. Always.
779 mrvl_crypto_pmd_sym_session_clear(struct rte_cryptodev
*dev
,
780 struct rte_cryptodev_sym_session
*sess
)
783 uint8_t index
= dev
->driver_id
;
784 void *sess_priv
= get_sym_session_private_data(sess
, index
);
786 /* Zero out the whole structure */
788 struct mrvl_crypto_session
*mrvl_sess
=
789 (struct mrvl_crypto_session
*)sess_priv
;
791 if (mrvl_sess
->sam_sess
&&
792 sam_session_destroy(mrvl_sess
->sam_sess
) < 0) {
793 MRVL_LOG(ERR
, "Error while destroying session!");
796 memset(sess
, 0, sizeof(struct mrvl_crypto_session
));
797 struct rte_mempool
*sess_mp
= rte_mempool_from_obj(sess_priv
);
798 set_sym_session_private_data(sess
, index
, NULL
);
799 rte_mempool_put(sess_mp
, sess_priv
);
804 * PMD handlers for crypto ops.
806 static struct rte_cryptodev_ops mrvl_crypto_pmd_ops
= {
807 .dev_configure
= mrvl_crypto_pmd_config
,
808 .dev_start
= mrvl_crypto_pmd_start
,
809 .dev_stop
= mrvl_crypto_pmd_stop
,
810 .dev_close
= mrvl_crypto_pmd_close
,
812 .dev_infos_get
= mrvl_crypto_pmd_info_get
,
814 .stats_get
= mrvl_crypto_pmd_stats_get
,
815 .stats_reset
= mrvl_crypto_pmd_stats_reset
,
817 .queue_pair_setup
= mrvl_crypto_pmd_qp_setup
,
818 .queue_pair_release
= mrvl_crypto_pmd_qp_release
,
820 .sym_session_get_size
= mrvl_crypto_pmd_sym_session_get_size
,
821 .sym_session_configure
= mrvl_crypto_pmd_sym_session_configure
,
822 .sym_session_clear
= mrvl_crypto_pmd_sym_session_clear
825 struct rte_cryptodev_ops
*rte_mrvl_crypto_pmd_ops
= &mrvl_crypto_pmd_ops
;