[ceph.git] / ceph / src / spdk / dpdk / drivers / crypto / aesni_mb / rte_aesni_mb_pmd_ops.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2015-2017 Intel Corporation
 */

#include <string.h>

#include <rte_string_fns.h>
#include <rte_common.h>
#include <rte_malloc.h>
#include <rte_cryptodev_pmd.h>

#include "rte_aesni_mb_pmd_private.h"


static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
	{	/* MD5 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_MD5_HMAC,
				.block_size = 64,
				.key_size = {
					.min = 1,
					.max = 64,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 16,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA1 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
				.block_size = 64,
				.key_size = {
					.min = 1,
					.max = 65535,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 20,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA1 */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA1,
				.block_size = 64,
				.key_size = {
					.min = 0,
					.max = 0,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 20,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA224 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
				.block_size = 64,
				.key_size = {
					.min = 1,
					.max = 65535,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 28,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA224 */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA224,
				.block_size = 64,
				.key_size = {
					.min = 0,
					.max = 0,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 28,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA256 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
				.block_size = 64,
				.key_size = {
					.min = 1,
					.max = 65535,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 32,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA256 */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA256,
				.block_size = 64,
				.key_size = {
					.min = 0,
					.max = 0,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 32,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA384 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
				.block_size = 128,
				.key_size = {
					.min = 1,
					.max = 65535,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 48,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA384 */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA384,
				.block_size = 128,
				.key_size = {
					.min = 0,
					.max = 0,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 48,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA512 HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
				.block_size = 128,
				.key_size = {
					.min = 1,
					.max = 65535,
					.increment = 1
				},
				.digest_size = {
					.min = 1,
					.max = 64,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* SHA512  */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_SHA512,
				.block_size = 128,
				.key_size = {
					.min = 0,
					.max = 0,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 64,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* AES XCBC HMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				},
				.digest_size = {
					.min = 12,
					.max = 12,
					.increment = 0
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* AES CBC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_AES_CBC,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 32,
					.increment = 8
				},
				.iv_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/* AES CTR */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_AES_CTR,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 32,
					.increment = 8
				},
				.iv_size = {
					.min = 12,
					.max = 16,
					.increment = 4
				}
			}, }
		}, }
	},
	{	/* AES DOCSIS BPI */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				},
				.iv_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/* DES CBC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_DES_CBC,
				.block_size = 8,
				.key_size = {
					.min = 8,
					.max = 8,
					.increment = 0
				},
				.iv_size = {
					.min = 8,
					.max = 8,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/*  3DES CBC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
				.block_size = 8,
				.key_size = {
					.min = 8,
					.max = 24,
					.increment = 8
				},
				.iv_size = {
					.min = 8,
					.max = 8,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/* DES DOCSIS BPI */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
			{.cipher = {
				.algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
				.block_size = 8,
				.key_size = {
					.min = 8,
					.max = 8,
					.increment = 0
				},
				.iv_size = {
					.min = 8,
					.max = 8,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/* AES CCM */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
			{.aead = {
				.algo = RTE_CRYPTO_AEAD_AES_CCM,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				},
				.digest_size = {
					.min = 4,
					.max = 16,
					.increment = 2
				},
				.aad_size = {
					.min = 0,
					.max = 46,
					.increment = 1
				},
				.iv_size = {
					.min = 7,
					.max = 13,
					.increment = 1
				},
			}, }
		}, }
	},
	{	/* AES CMAC */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_AES_CMAC,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 16,
					.increment = 0
				},
				.digest_size = {
					.min = 1,
					.max = 16,
					.increment = 1
				},
				.iv_size = { 0 }
			}, }
		}, }
	},
	{	/* AES GCM */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
			{.aead = {
				.algo = RTE_CRYPTO_AEAD_AES_GCM,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 32,
					.increment = 8
				},
				.digest_size = {
					.min = 8,
					.max = 16,
					.increment = 4
				},
				.aad_size = {
					.min = 0,
					.max = 65535,
					.increment = 1
				},
				.iv_size = {
					.min = 12,
					.max = 12,
					.increment = 0
				}
			}, }
		}, }
	},
	{	/* AES GMAC (AUTH) */
		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
		{.sym = {
			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
			{.auth = {
				.algo = RTE_CRYPTO_AUTH_AES_GMAC,
				.block_size = 16,
				.key_size = {
					.min = 16,
					.max = 32,
					.increment = 8
				},
				.digest_size = {
					.min = 8,
					.max = 16,
					.increment = 4
				},
				.iv_size = {
					.min = 12,
					.max = 12,
					.increment = 0
				}
			}, }
		}, }
	},
	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};


/** Configure device */
static int
aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
		__rte_unused struct rte_cryptodev_config *config)
{
	return 0;
}

/** Start device */
static int
aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
{
	return 0;
}

/** Stop device */
static void
aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
{
}

/** Close device */
static int
aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
{
	return 0;
}


/** Get device statistics */
static void
aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_stats *stats)
{
	int qp_id;

	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
		struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];

		stats->enqueued_count += qp->stats.enqueued_count;
		stats->dequeued_count += qp->stats.dequeued_count;

		stats->enqueue_err_count += qp->stats.enqueue_err_count;
		stats->dequeue_err_count += qp->stats.dequeue_err_count;
	}
}

/** Reset device statistics */
static void
aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
{
	int qp_id;

	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
		struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];

		memset(&qp->stats, 0, sizeof(qp->stats));
	}
}


/** Get device info */
static void
aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_info *dev_info)
{
	struct aesni_mb_private *internals = dev->data->dev_private;

	if (dev_info != NULL) {
		dev_info->driver_id = dev->driver_id;
		dev_info->feature_flags = dev->feature_flags;
		dev_info->capabilities = aesni_mb_pmd_capabilities;
		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
		/* No limit of number of sessions */
		dev_info->sym.max_nb_sessions = 0;
	}
}

/** Release queue pair */
static int
aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
{
	struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
	struct rte_ring *r = NULL;

	if (qp != NULL) {
		r = rte_ring_lookup(qp->name);
		if (r)
			rte_ring_free(r);
		if (qp->mb_mgr)
			free_mb_mgr(qp->mb_mgr);
		rte_free(qp);
		dev->data->queue_pairs[qp_id] = NULL;
	}
	return 0;
}

/** set a unique name for the queue pair based on it's name, dev_id and qp_id */
static int
aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
		struct aesni_mb_qp *qp)
{
	unsigned n = snprintf(qp->name, sizeof(qp->name),
			"aesni_mb_pmd_%u_qp_%u",
			dev->data->dev_id, qp->id);

	if (n >= sizeof(qp->name))
		return -1;

	return 0;
}

/** Create a ring to place processed operations on */
static struct rte_ring *
aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
		unsigned int ring_size, int socket_id)
{
	struct rte_ring *r;
	char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];

	unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));

	if (n >= sizeof(ring_name))
		return NULL;

	r = rte_ring_lookup(ring_name);
	if (r) {
		if (rte_ring_get_size(r) >= ring_size) {
			AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
			ring_name);
			return r;
		}

		AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
			ring_name);
		return NULL;
	}

	return rte_ring_create(ring_name, ring_size, socket_id,
			RING_F_SP_ENQ | RING_F_SC_DEQ);
}

/** Setup a queue pair */
static int
aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
		const struct rte_cryptodev_qp_conf *qp_conf,
		int socket_id)
{
	struct aesni_mb_qp *qp = NULL;
	struct aesni_mb_private *internals = dev->data->dev_private;
	int ret = -1;

	/* Free memory prior to re-allocation if needed. */
	if (dev->data->queue_pairs[qp_id] != NULL)
		aesni_mb_pmd_qp_release(dev, qp_id);

	/* Allocate the queue pair data structure. */
	qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
					RTE_CACHE_LINE_SIZE, socket_id);
	if (qp == NULL)
		return -ENOMEM;

	qp->id = qp_id;
	dev->data->queue_pairs[qp_id] = qp;

	if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
		goto qp_setup_cleanup;


	qp->mb_mgr = alloc_mb_mgr(0);
	if (qp->mb_mgr == NULL) {
		ret = -ENOMEM;
		goto qp_setup_cleanup;
	}

	switch (internals->vector_mode) {
	case RTE_AESNI_MB_SSE:
		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
		init_mb_mgr_sse(qp->mb_mgr);
		break;
	case RTE_AESNI_MB_AVX:
		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
		init_mb_mgr_avx(qp->mb_mgr);
		break;
	case RTE_AESNI_MB_AVX2:
		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
		init_mb_mgr_avx2(qp->mb_mgr);
		break;
	case RTE_AESNI_MB_AVX512:
		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
		init_mb_mgr_avx512(qp->mb_mgr);
		break;
	default:
		AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
				internals->vector_mode);
		goto qp_setup_cleanup;
	}

	qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
			qp_conf->nb_descriptors, socket_id);
	if (qp->ingress_queue == NULL) {
		ret = -1;
		goto qp_setup_cleanup;
	}

	qp->sess_mp = qp_conf->mp_session;
	qp->sess_mp_priv = qp_conf->mp_session_private;

	memset(&qp->stats, 0, sizeof(qp->stats));

	char mp_name[RTE_MEMPOOL_NAMESIZE];

	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
				"digest_mp_%u_%u", dev->data->dev_id, qp_id);
	return 0;

qp_setup_cleanup:
	if (qp) {
		if (qp->mb_mgr)
			free_mb_mgr(qp->mb_mgr);
		rte_free(qp);
	}

	return ret;
}

/** Return the number of allocated queue pairs */
static uint32_t
aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
{
	return dev->data->nb_queue_pairs;
}

/** Returns the size of the aesni multi-buffer session structure */
static unsigned
aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
{
	return sizeof(struct aesni_mb_session);
}

/** Configure a aesni multi-buffer session from a crypto xform chain */
static int
aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
		struct rte_crypto_sym_xform *xform,
		struct rte_cryptodev_sym_session *sess,
		struct rte_mempool *mempool)
{
	void *sess_private_data;
	struct aesni_mb_private *internals = dev->data->dev_private;
	int ret;

	if (unlikely(sess == NULL)) {
		AESNI_MB_LOG(ERR, "invalid session struct");
		return -EINVAL;
	}

	if (rte_mempool_get(mempool, &sess_private_data)) {
		AESNI_MB_LOG(ERR,
				"Couldn't get object from session mempool");
		return -ENOMEM;
	}

	ret = aesni_mb_set_session_parameters(internals->mb_mgr,
			sess_private_data, xform);
	if (ret != 0) {
		AESNI_MB_LOG(ERR, "failed configure session parameters");

		/* Return session to mempool */
		rte_mempool_put(mempool, sess_private_data);
		return ret;
	}

	set_sym_session_private_data(sess, dev->driver_id,
			sess_private_data);

	return 0;
}

/** Clear the memory of session so it doesn't leave key material behind */
static void
aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
		struct rte_cryptodev_sym_session *sess)
{
	uint8_t index = dev->driver_id;
	void *sess_priv = get_sym_session_private_data(sess, index);

	/* Zero out the whole structure */
	if (sess_priv) {
		memset(sess_priv, 0, sizeof(struct aesni_mb_session));
		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
		set_sym_session_private_data(sess, index, NULL);
		rte_mempool_put(sess_mp, sess_priv);
	}
}

struct rte_cryptodev_ops aesni_mb_pmd_ops = {
		.dev_configure		= aesni_mb_pmd_config,
		.dev_start		= aesni_mb_pmd_start,
		.dev_stop		= aesni_mb_pmd_stop,
		.dev_close		= aesni_mb_pmd_close,

		.stats_get		= aesni_mb_pmd_stats_get,
		.stats_reset		= aesni_mb_pmd_stats_reset,

		.dev_infos_get		= aesni_mb_pmd_info_get,

		.queue_pair_setup	= aesni_mb_pmd_qp_setup,
		.queue_pair_release	= aesni_mb_pmd_qp_release,
		.queue_pair_count	= aesni_mb_pmd_qp_count,

		.sym_session_get_size	= aesni_mb_pmd_sym_session_get_size,
		.sym_session_configure	= aesni_mb_pmd_sym_session_configure,
		.sym_session_clear	= aesni_mb_pmd_sym_session_clear
};

struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
11fdf7f2 TL	2	* Copyright(c) 2015-2017 Intel Corporation
7c673cae FG	3	*/
	4
	5	#include <string.h>
	6
9f95a23c	7	#include <rte_string_fns.h>
7c673cae FG	8	#include <rte_common.h>
	9	#include <rte_malloc.h>
	10	#include <rte_cryptodev_pmd.h>
	11
	12	#include "rte_aesni_mb_pmd_private.h"
	13
	14
	15	static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
	16	{ /* MD5 HMAC */
	17	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	18	{.sym = {
	19	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	20	{.auth = {
	21	.algo = RTE_CRYPTO_AUTH_MD5_HMAC,
	22	.block_size = 64,
	23	.key_size = {
11fdf7f2	24	.min = 1,
7c673cae	25	.max = 64,
11fdf7f2	26	.increment = 1
7c673cae FG	27	},
7c673cae FG	28	.digest_size = {
9f95a23c TL	29	.min = 1,
	30	.max = 16,
	31	.increment = 1
7c673cae	32	},
11fdf7f2	33	.iv_size = { 0 }
7c673cae FG	34	}, }
	35	}, }
	36	},
	37	{ /* SHA1 HMAC */
	38	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	39	{.sym = {
	40	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	41	{.auth = {
	42	.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
	43	.block_size = 64,
	44	.key_size = {
11fdf7f2	45	.min = 1,
9f95a23c	46	.max = 65535,
11fdf7f2	47	.increment = 1
7c673cae FG	48	},
7c673cae FG	49	.digest_size = {
9f95a23c TL	50	.min = 1,
	51	.max = 20,
	52	.increment = 1
	53	},
	54	.iv_size = { 0 }
	55	}, }
	56	}, }
	57	},
	58	{ /* SHA1 */
	59	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	60	{.sym = {
	61	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	62	{.auth = {
	63	.algo = RTE_CRYPTO_AUTH_SHA1,
	64	.block_size = 64,
	65	.key_size = {
	66	.min = 0,
	67	.max = 0,
7c673cae FG	68	.increment = 0
7c673cae FG	69	},
9f95a23c TL	70	.digest_size = {
	71	.min = 1,
	72	.max = 20,
	73	.increment = 1
	74	},
11fdf7f2	75	.iv_size = { 0 }
7c673cae FG	76	}, }
	77	}, }
	78	},
	79	{ /* SHA224 HMAC */
	80	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	81	{.sym = {
	82	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	83	{.auth = {
	84	.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
	85	.block_size = 64,
	86	.key_size = {
11fdf7f2	87	.min = 1,
9f95a23c	88	.max = 65535,
11fdf7f2	89	.increment = 1
7c673cae FG	90	},
7c673cae FG	91	.digest_size = {
9f95a23c TL	92	.min = 1,
	93	.max = 28,
	94	.increment = 1
	95	},
	96	.iv_size = { 0 }
	97	}, }
	98	}, }
	99	},
	100	{ /* SHA224 */
	101	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	102	{.sym = {
	103	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	104	{.auth = {
	105	.algo = RTE_CRYPTO_AUTH_SHA224,
	106	.block_size = 64,
	107	.key_size = {
	108	.min = 0,
	109	.max = 0,
7c673cae FG	110	.increment = 0
7c673cae FG	111	},
9f95a23c TL	112	.digest_size = {
	113	.min = 1,
	114	.max = 28,
	115	.increment = 1
	116	},
11fdf7f2	117	.iv_size = { 0 }
7c673cae FG	118	}, }
	119	}, }
	120	},
	121	{ /* SHA256 HMAC */
	122	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	123	{.sym = {
	124	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	125	{.auth = {
	126	.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
	127	.block_size = 64,
	128	.key_size = {
11fdf7f2	129	.min = 1,
9f95a23c	130	.max = 65535,
11fdf7f2	131	.increment = 1
7c673cae FG	132	},
7c673cae FG	133	.digest_size = {
9f95a23c TL	134	.min = 1,
	135	.max = 32,
	136	.increment = 1
	137	},
	138	.iv_size = { 0 }
	139	}, }
	140	}, }
	141	},
	142	{ /* SHA256 */
	143	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	144	{.sym = {
	145	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	146	{.auth = {
	147	.algo = RTE_CRYPTO_AUTH_SHA256,
	148	.block_size = 64,
	149	.key_size = {
	150	.min = 0,
	151	.max = 0,
7c673cae FG	152	.increment = 0
7c673cae FG	153	},
9f95a23c TL	154	.digest_size = {
	155	.min = 1,
	156	.max = 32,
	157	.increment = 1
	158	},
11fdf7f2	159	.iv_size = { 0 }
7c673cae FG	160	}, }
	161	}, }
	162	},
	163	{ /* SHA384 HMAC */
	164	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	165	{.sym = {
	166	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	167	{.auth = {
	168	.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
	169	.block_size = 128,
	170	.key_size = {
11fdf7f2	171	.min = 1,
9f95a23c	172	.max = 65535,
11fdf7f2	173	.increment = 1
7c673cae FG	174	},
7c673cae FG	175	.digest_size = {
9f95a23c TL	176	.min = 1,
	177	.max = 48,
	178	.increment = 1
	179	},
	180	.iv_size = { 0 }
	181	}, }
	182	}, }
	183	},
	184	{ /* SHA384 */
	185	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	186	{.sym = {
	187	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	188	{.auth = {
	189	.algo = RTE_CRYPTO_AUTH_SHA384,
	190	.block_size = 128,
	191	.key_size = {
	192	.min = 0,
	193	.max = 0,
7c673cae FG	194	.increment = 0
7c673cae FG	195	},
9f95a23c TL	196	.digest_size = {
	197	.min = 1,
	198	.max = 48,
	199	.increment = 1
	200	},
11fdf7f2	201	.iv_size = { 0 }
7c673cae FG	202	}, }
	203	}, }
	204	},
	205	{ /* SHA512 HMAC */
	206	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	207	{.sym = {
	208	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	209	{.auth = {
	210	.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
	211	.block_size = 128,
	212	.key_size = {
11fdf7f2	213	.min = 1,
9f95a23c	214	.max = 65535,
11fdf7f2	215	.increment = 1
7c673cae FG	216	},
7c673cae FG	217	.digest_size = {
9f95a23c TL	218	.min = 1,
	219	.max = 64,
	220	.increment = 1
	221	},
	222	.iv_size = { 0 }
	223	}, }
	224	}, }
	225	},
	226	{ /* SHA512 */
	227	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	228	{.sym = {
	229	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	230	{.auth = {
	231	.algo = RTE_CRYPTO_AUTH_SHA512,
	232	.block_size = 128,
	233	.key_size = {
	234	.min = 0,
	235	.max = 0,
7c673cae FG	236	.increment = 0
7c673cae FG	237	},
9f95a23c TL	238	.digest_size = {
	239	.min = 1,
	240	.max = 64,
	241	.increment = 1
	242	},
11fdf7f2	243	.iv_size = { 0 }
7c673cae FG	244	}, }
	245	}, }
	246	},
	247	{ /* AES XCBC HMAC */
	248	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	249	{.sym = {
	250	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	251	{.auth = {
	252	.algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
	253	.block_size = 16,
	254	.key_size = {
	255	.min = 16,
	256	.max = 16,
	257	.increment = 0
	258	},
	259	.digest_size = {
	260	.min = 12,
	261	.max = 12,
	262	.increment = 0
	263	},
11fdf7f2	264	.iv_size = { 0 }
7c673cae FG	265	}, }
	266	}, }
	267	},
	268	{ /* AES CBC */
	269	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	270	{.sym = {
	271	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	272	{.cipher = {
	273	.algo = RTE_CRYPTO_CIPHER_AES_CBC,
	274	.block_size = 16,
	275	.key_size = {
	276	.min = 16,
	277	.max = 32,
	278	.increment = 8
	279	},
	280	.iv_size = {
	281	.min = 16,
	282	.max = 16,
	283	.increment = 0
	284	}
	285	}, }
	286	}, }
	287	},
	288	{ /* AES CTR */
	289	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	290	{.sym = {
	291	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	292	{.cipher = {
	293	.algo = RTE_CRYPTO_CIPHER_AES_CTR,
	294	.block_size = 16,
	295	.key_size = {
	296	.min = 16,
	297	.max = 32,
	298	.increment = 8
	299	},
11fdf7f2 TL	300	.iv_size = {
	301	.min = 12,
	302	.max = 16,
	303	.increment = 4
	304	}
	305	}, }
	306	}, }
	307	},
	308	{ /* AES DOCSIS BPI */
	309	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	310	{.sym = {
	311	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	312	{.cipher = {
	313	.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
	314	.block_size = 16,
	315	.key_size = {
	316	.min = 16,
	317	.max = 16,
	318	.increment = 0
	319	},
7c673cae FG	320	.iv_size = {
	321	.min = 16,
	322	.max = 16,
	323	.increment = 0
	324	}
	325	}, }
	326	}, }
	327	},
11fdf7f2 TL	328	{ /* DES CBC */
	329	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	330	{.sym = {
	331	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	332	{.cipher = {
	333	.algo = RTE_CRYPTO_CIPHER_DES_CBC,
	334	.block_size = 8,
	335	.key_size = {
	336	.min = 8,
	337	.max = 8,
	338	.increment = 0
	339	},
	340	.iv_size = {
	341	.min = 8,
	342	.max = 8,
	343	.increment = 0
	344	}
	345	}, }
	346	}, }
	347	},
	348	{ /* 3DES CBC */
	349	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	350	{.sym = {
	351	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	352	{.cipher = {
	353	.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
	354	.block_size = 8,
	355	.key_size = {
	356	.min = 8,
	357	.max = 24,
	358	.increment = 8
	359	},
	360	.iv_size = {
	361	.min = 8,
	362	.max = 8,
	363	.increment = 0
	364	}
	365	}, }
	366	}, }
	367	},
	368	{ /* DES DOCSIS BPI */
	369	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	370	{.sym = {
	371	.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
	372	{.cipher = {
	373	.algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
	374	.block_size = 8,
	375	.key_size = {
	376	.min = 8,
	377	.max = 8,
	378	.increment = 0
	379	},
	380	.iv_size = {
	381	.min = 8,
	382	.max = 8,
	383	.increment = 0
	384	}
	385	}, }
	386	}, }
	387	},
	388	{ /* AES CCM */
	389	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	390	{.sym = {
	391	.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
392	{.aead = {
393	.algo = RTE_CRYPTO_AEAD_AES_CCM,
394	.block_size = 16,
395	.key_size = {
396	.min = 16,
397	.max = 16,
398	.increment = 0
399	},
400	.digest_size = {
401	.min = 4,
402	.max = 16,
403	.increment = 2
404	},
405	.aad_size = {
406	.min = 0,
407	.max = 46,
408	.increment = 1
409	},
410	.iv_size = {
411	.min = 7,
412	.max = 13,
413	.increment = 1
414	},
415	}, }
416	}, }
417	},
418	{ /* AES CMAC */
419	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
420	{.sym = {
421	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
422	{.auth = {
423	.algo = RTE_CRYPTO_AUTH_AES_CMAC,
424	.block_size = 16,
425	.key_size = {
426	.min = 16,
427	.max = 16,
428	.increment = 0
429	},
430	.digest_size = {
9f95a23c TL	431	.min = 1,
	432	.max = 16,
	433	.increment = 1
	434	},
	435	.iv_size = { 0 }
	436	}, }
	437	}, }
	438	},
	439	{ /* AES GCM */
	440	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	441	{.sym = {
	442	.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
	443	{.aead = {
	444	.algo = RTE_CRYPTO_AEAD_AES_GCM,
	445	.block_size = 16,
	446	.key_size = {
	447	.min = 16,
	448	.max = 32,
	449	.increment = 8
	450	},
	451	.digest_size = {
	452	.min = 8,
	453	.max = 16,
	454	.increment = 4
	455	},
	456	.aad_size = {
	457	.min = 0,
	458	.max = 65535,
	459	.increment = 1
	460	},
	461	.iv_size = {
11fdf7f2	462	.min = 12,
9f95a23c TL	463	.max = 12,
	464	.increment = 0
	465	}
	466	}, }
	467	}, }
	468	},
	469	{ /* AES GMAC (AUTH) */
	470	.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	471	{.sym = {
	472	.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
	473	{.auth = {
	474	.algo = RTE_CRYPTO_AUTH_AES_GMAC,
	475	.block_size = 16,
	476	.key_size = {
	477	.min = 16,
	478	.max = 32,
	479	.increment = 8
	480	},
	481	.digest_size = {
	482	.min = 8,
11fdf7f2 TL	483	.max = 16,
	484	.increment = 4
	485	},
9f95a23c TL	486	.iv_size = {
	487	.min = 12,
	488	.max = 12,
	489	.increment = 0
	490	}
11fdf7f2 TL	491	}, }
	492	}, }
	493	},
7c673cae FG	494	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
	495	};
	496
	497
	498	/** Configure device */
	499	static int
11fdf7f2 TL	500	aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
11fdf7f2 TL	501	__rte_unused struct rte_cryptodev_config *config)
7c673cae FG	502	{
	503	return 0;
	504	}
	505
	506	/** Start device */
	507	static int
	508	aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
	509	{
	510	return 0;
	511	}
	512
	513	/** Stop device */
	514	static void
	515	aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
	516	{
	517	}
	518
	519	/** Close device */
	520	static int
	521	aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
	522	{
	523	return 0;
	524	}
	525
	526
	527	/** Get device statistics */
	528	static void
	529	aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
	530	struct rte_cryptodev_stats *stats)
	531	{
	532	int qp_id;
	533
	534	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
	535	struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
	536
	537	stats->enqueued_count += qp->stats.enqueued_count;
	538	stats->dequeued_count += qp->stats.dequeued_count;
	539
	540	stats->enqueue_err_count += qp->stats.enqueue_err_count;
	541	stats->dequeue_err_count += qp->stats.dequeue_err_count;
	542	}
	543	}
	544
	545	/** Reset device statistics */
	546	static void
	547	aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
	548	{
	549	int qp_id;
	550
	551	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
	552	struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
	553
	554	memset(&qp->stats, 0, sizeof(qp->stats));
	555	}
	556	}
	557
	558
	559	/** Get device info */
	560	static void
	561	aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
	562	struct rte_cryptodev_info *dev_info)
	563	{
	564	struct aesni_mb_private *internals = dev->data->dev_private;
	565
566	if (dev_info != NULL) {
11fdf7f2	567	dev_info->driver_id = dev->driver_id;
7c673cae FG	568	dev_info->feature_flags = dev->feature_flags;
	569	dev_info->capabilities = aesni_mb_pmd_capabilities;
	570	dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
11fdf7f2 TL	571	/* No limit of number of sessions */
11fdf7f2 TL	572	dev_info->sym.max_nb_sessions = 0;
7c673cae FG	573	}
	574	}
	575
	576	/** Release queue pair */
	577	static int
	578	aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
	579	{
	580	struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
	581	struct rte_ring *r = NULL;
	582
	583	if (qp != NULL) {
	584	r = rte_ring_lookup(qp->name);
	585	if (r)
	586	rte_ring_free(r);
9f95a23c TL	587	if (qp->mb_mgr)
9f95a23c TL	588	free_mb_mgr(qp->mb_mgr);
7c673cae FG	589	rte_free(qp);
	590	dev->data->queue_pairs[qp_id] = NULL;
	591	}
	592	return 0;
	593	}
	594
	595	/** set a unique name for the queue pair based on it's name, dev_id and qp_id */
	596	static int
	597	aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
	598	struct aesni_mb_qp *qp)
	599	{
	600	unsigned n = snprintf(qp->name, sizeof(qp->name),
	601	"aesni_mb_pmd_%u_qp_%u",
	602	dev->data->dev_id, qp->id);
	603
11fdf7f2	604	if (n >= sizeof(qp->name))
7c673cae FG	605	return -1;
	606
	607	return 0;
	608	}
	609
	610	/** Create a ring to place processed operations on */
	611	static struct rte_ring *
	612	aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
9f95a23c	613	unsigned int ring_size, int socket_id)
7c673cae FG	614	{
7c673cae FG	615	struct rte_ring *r;
11fdf7f2 TL	616	char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
11fdf7f2 TL	617
9f95a23c	618	unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
11fdf7f2 TL	619
	620	if (n >= sizeof(ring_name))
	621	return NULL;
7c673cae	622
11fdf7f2	623	r = rte_ring_lookup(ring_name);
7c673cae	624	if (r) {
11fdf7f2 TL	625	if (rte_ring_get_size(r) >= ring_size) {
	626	AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
	627	ring_name);
7c673cae FG	628	return r;
	629	}
	630
11fdf7f2 TL	631	AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
11fdf7f2 TL	632	ring_name);
7c673cae FG	633	return NULL;
	634	}
	635
11fdf7f2	636	return rte_ring_create(ring_name, ring_size, socket_id,
7c673cae FG	637	RING_F_SP_ENQ \| RING_F_SC_DEQ);
	638	}
	639
	640	/** Setup a queue pair */
	641	static int
	642	aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
	643	const struct rte_cryptodev_qp_conf *qp_conf,
9f95a23c	644	int socket_id)
7c673cae FG	645	{
	646	struct aesni_mb_qp *qp = NULL;
	647	struct aesni_mb_private *internals = dev->data->dev_private;
9f95a23c	648	int ret = -1;
7c673cae FG	649
	650	/* Free memory prior to re-allocation if needed. */
	651	if (dev->data->queue_pairs[qp_id] != NULL)
	652	aesni_mb_pmd_qp_release(dev, qp_id);
	653
	654	/* Allocate the queue pair data structure. */
	655	qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
	656	RTE_CACHE_LINE_SIZE, socket_id);
	657	if (qp == NULL)
	658	return -ENOMEM;
	659
	660	qp->id = qp_id;
	661	dev->data->queue_pairs[qp_id] = qp;
	662
	663	if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
	664	goto qp_setup_cleanup;
	665
7c673cae	666
9f95a23c TL	667	qp->mb_mgr = alloc_mb_mgr(0);
	668	if (qp->mb_mgr == NULL) {
	669	ret = -ENOMEM;
	670	goto qp_setup_cleanup;
	671	}
	672
	673	switch (internals->vector_mode) {
	674	case RTE_AESNI_MB_SSE:
	675	dev->feature_flags \|= RTE_CRYPTODEV_FF_CPU_SSE;
	676	init_mb_mgr_sse(qp->mb_mgr);
	677	break;
	678	case RTE_AESNI_MB_AVX:
	679	dev->feature_flags \|= RTE_CRYPTODEV_FF_CPU_AVX;
	680	init_mb_mgr_avx(qp->mb_mgr);
	681	break;
	682	case RTE_AESNI_MB_AVX2:
	683	dev->feature_flags \|= RTE_CRYPTODEV_FF_CPU_AVX2;
	684	init_mb_mgr_avx2(qp->mb_mgr);
	685	break;
	686	case RTE_AESNI_MB_AVX512:
	687	dev->feature_flags \|= RTE_CRYPTODEV_FF_CPU_AVX512;
	688	init_mb_mgr_avx512(qp->mb_mgr);
	689	break;
	690	default:
	691	AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
	692	internals->vector_mode);
	693	goto qp_setup_cleanup;
	694	}
11fdf7f2 TL	695
11fdf7f2 TL	696	qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
9f95a23c TL	697	qp_conf->nb_descriptors, socket_id);
	698	if (qp->ingress_queue == NULL) {
	699	ret = -1;
7c673cae	700	goto qp_setup_cleanup;
9f95a23c	701	}
7c673cae	702
9f95a23c TL	703	qp->sess_mp = qp_conf->mp_session;
9f95a23c TL	704	qp->sess_mp_priv = qp_conf->mp_session_private;
7c673cae FG	705
	706	memset(&qp->stats, 0, sizeof(qp->stats));
	707
11fdf7f2	708	char mp_name[RTE_MEMPOOL_NAMESIZE];
7c673cae	709
11fdf7f2 TL	710	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
11fdf7f2 TL	711	"digest_mp_%u_%u", dev->data->dev_id, qp_id);
7c673cae FG	712	return 0;
	713
	714	qp_setup_cleanup:
9f95a23c TL	715	if (qp) {
	716	if (qp->mb_mgr)
	717	free_mb_mgr(qp->mb_mgr);
7c673cae	718	rte_free(qp);
9f95a23c	719	}
7c673cae	720
9f95a23c	721	return ret;
7c673cae FG	722	}
7c673cae FG	723
7c673cae FG	724	/** Return the number of allocated queue pairs */
	725	static uint32_t
	726	aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
	727	{
	728	return dev->data->nb_queue_pairs;
	729	}
	730
	731	/** Returns the size of the aesni multi-buffer session structure */
	732	static unsigned
11fdf7f2	733	aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
7c673cae FG	734	{
	735	return sizeof(struct aesni_mb_session);
	736	}
	737
	738	/** Configure a aesni multi-buffer session from a crypto xform chain */
11fdf7f2 TL	739	static int
	740	aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
	741	struct rte_crypto_sym_xform *xform,
	742	struct rte_cryptodev_sym_session *sess,
	743	struct rte_mempool *mempool)
7c673cae	744	{
11fdf7f2	745	void *sess_private_data;
7c673cae	746	struct aesni_mb_private *internals = dev->data->dev_private;
11fdf7f2	747	int ret;
7c673cae FG	748
7c673cae FG	749	if (unlikely(sess == NULL)) {
11fdf7f2 TL	750	AESNI_MB_LOG(ERR, "invalid session struct");
11fdf7f2 TL	751	return -EINVAL;
7c673cae FG	752	}
7c673cae FG	753
11fdf7f2 TL	754	if (rte_mempool_get(mempool, &sess_private_data)) {
	755	AESNI_MB_LOG(ERR,
	756	"Couldn't get object from session mempool");
	757	return -ENOMEM;
	758	}
	759
9f95a23c	760	ret = aesni_mb_set_session_parameters(internals->mb_mgr,
11fdf7f2 TL	761	sess_private_data, xform);
	762	if (ret != 0) {
	763	AESNI_MB_LOG(ERR, "failed configure session parameters");
	764
	765	/* Return session to mempool */
	766	rte_mempool_put(mempool, sess_private_data);
	767	return ret;
7c673cae FG	768	}
7c673cae FG	769
11fdf7f2 TL	770	set_sym_session_private_data(sess, dev->driver_id,
	771	sess_private_data);
	772
	773	return 0;
7c673cae FG	774	}
	775
	776	/** Clear the memory of session so it doesn't leave key material behind */
	777	static void
11fdf7f2 TL	778	aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
11fdf7f2 TL	779	struct rte_cryptodev_sym_session *sess)
7c673cae	780	{
11fdf7f2 TL	781	uint8_t index = dev->driver_id;
	782	void *sess_priv = get_sym_session_private_data(sess, index);
	783
	784	/* Zero out the whole structure */
	785	if (sess_priv) {
	786	memset(sess_priv, 0, sizeof(struct aesni_mb_session));
	787	struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
	788	set_sym_session_private_data(sess, index, NULL);
	789	rte_mempool_put(sess_mp, sess_priv);
	790	}
7c673cae FG	791	}
	792
	793	struct rte_cryptodev_ops aesni_mb_pmd_ops = {
	794	.dev_configure = aesni_mb_pmd_config,
	795	.dev_start = aesni_mb_pmd_start,
	796	.dev_stop = aesni_mb_pmd_stop,
	797	.dev_close = aesni_mb_pmd_close,
	798
	799	.stats_get = aesni_mb_pmd_stats_get,
	800	.stats_reset = aesni_mb_pmd_stats_reset,
	801
	802	.dev_infos_get = aesni_mb_pmd_info_get,
	803
	804	.queue_pair_setup = aesni_mb_pmd_qp_setup,
	805	.queue_pair_release = aesni_mb_pmd_qp_release,
7c673cae FG	806	.queue_pair_count = aesni_mb_pmd_qp_count,
7c673cae FG	807
11fdf7f2 TL	808	.sym_session_get_size = aesni_mb_pmd_sym_session_get_size,
	809	.sym_session_configure = aesni_mb_pmd_sym_session_configure,
	810	.sym_session_clear = aesni_mb_pmd_sym_session_clear
7c673cae FG	811	};
	812
	813	struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;