[ceph.git] / ceph / src / spdk / dpdk / examples / vhost_crypto / main.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2017-2018 Intel Corporation
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stdbool.h>
#include <assert.h>
#include <getopt.h>

#include <rte_malloc.h>
#include <rte_cycles.h>
#include <rte_vhost.h>
#include <rte_cryptodev.h>
#include <rte_vhost_crypto.h>
#include <rte_string_fns.h>

#include <cmdline_rdline.h>
#include <cmdline_parse.h>
#include <cmdline_parse_string.h>
#include <cmdline.h>

#define NB_VIRTIO_QUEUES		(1)
#define MAX_PKT_BURST			(64)
#define MAX_IV_LEN			(32)
#define NB_MEMPOOL_OBJS			(8192)
#define NB_CRYPTO_DESCRIPTORS		(4096)
#define NB_CACHE_OBJS			(128)
#define SESSION_MAP_ENTRIES		(1024)
#define REFRESH_TIME_SEC		(3)

#define MAX_NB_SOCKETS			(4)
#define MAX_NB_WORKER_CORES		(16)

struct lcore_option {
	uint32_t lcore_id;
	char *socket_files[MAX_NB_SOCKETS];
	uint32_t nb_sockets;
	uint8_t cid;
	uint16_t qid;
};

struct vhost_crypto_info {
	int vids[MAX_NB_SOCKETS];
	uint32_t nb_vids;
	struct rte_mempool *sess_pool;
	struct rte_mempool *sess_priv_pool;
	struct rte_mempool *cop_pool;
	uint8_t cid;
	uint32_t qid;
	uint32_t nb_inflight_ops;
	volatile uint32_t initialized[MAX_NB_SOCKETS];
} __rte_cache_aligned;

struct vhost_crypto_options {
	struct lcore_option los[MAX_NB_WORKER_CORES];
	struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
	uint32_t nb_los;
	uint32_t zero_copy;
	uint32_t guest_polling;
} options;

#define CONFIG_KEYWORD		"config"
#define SOCKET_FILE_KEYWORD	"socket-file"
#define ZERO_COPY_KEYWORD	"zero-copy"
#define POLLING_KEYWORD		"guest-polling"

#define NB_SOCKET_FIELDS	(2)

static uint32_t
find_lo(uint32_t lcore_id)
{
	uint32_t i;

	for (i = 0; i < options.nb_los; i++)
		if (options.los[i].lcore_id == lcore_id)
			return i;

	return UINT32_MAX;
}

/** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
static int
parse_socket_arg(char *arg)
{
	uint32_t nb_sockets;
	uint32_t lcore_id;
	char *str_fld[NB_SOCKET_FIELDS];
	struct lcore_option *lo;
	uint32_t idx;
	char *end;

	if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
				NB_SOCKET_FIELDS) {
		RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
		return -EINVAL;
	}

	errno = 0;
	lcore_id = strtoul(str_fld[0], &end, 0);
	if (errno != 0 || end == str_fld[0] || lcore_id > 255)
		return -EINVAL;

	idx = find_lo(lcore_id);
	if (idx == UINT32_MAX) {
		if (options.nb_los == MAX_NB_WORKER_CORES)
			return -ENOMEM;
		lo = &options.los[options.nb_los];
		lo->lcore_id = lcore_id;
		options.nb_los++;
	} else
		lo = &options.los[idx];

	nb_sockets = lo->nb_sockets;

	if (nb_sockets >= MAX_NB_SOCKETS) {
		RTE_LOG(ERR, USER1, "Too many socket files!\n");
		return -ENOMEM;
	}

	lo->socket_files[nb_sockets] = strdup(str_fld[1]);
	if (!lo->socket_files[nb_sockets]) {
		RTE_LOG(ERR, USER1, "Insufficient memory\n");
		return -ENOMEM;
	}

	lo->nb_sockets++;

	return 0;
}

static int
parse_config(char *q_arg)
{
	struct lcore_option *lo;
	char s[256];
	const char *p, *p0 = q_arg;
	char *end;
	enum fieldnames {
		FLD_LCORE = 0,
		FLD_CID,
		FLD_QID,
		_NUM_FLD
	};
	uint32_t flds[_NUM_FLD];
	char *str_fld[_NUM_FLD];
	uint32_t i;
	uint32_t size;

	while ((p = strchr(p0, '(')) != NULL) {
		++p;
		p0 = strchr(p, ')');
		if (p0 == NULL)
			return -1;

		size = p0 - p;
		if (size >= sizeof(s))
			return -1;

		snprintf(s, sizeof(s), "%.*s", size, p);
		if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
				_NUM_FLD)
			return -1;
		for (i = 0; i < _NUM_FLD; i++) {
			errno = 0;
			flds[i] = strtoul(str_fld[i], &end, 0);
			if (errno != 0 || end == str_fld[i] || flds[i] > 255)
				return -EINVAL;
		}

		if (flds[FLD_LCORE] > RTE_MAX_LCORE)
			return -EINVAL;

		i = find_lo(flds[FLD_LCORE]);
		if (i == UINT32_MAX) {
			if (options.nb_los == MAX_NB_WORKER_CORES)
				return -ENOMEM;
			lo = &options.los[options.nb_los];
			options.nb_los++;
		} else
			lo = &options.los[i];

		lo->lcore_id = flds[FLD_LCORE];
		lo->cid = flds[FLD_CID];
		lo->qid = flds[FLD_QID];
	}

	return 0;
}

static void
vhost_crypto_usage(const char *prgname)
{
	printf("%s [EAL options] --\n"
		"  --%s <lcore>,SOCKET-FILE-PATH\n"
		"  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
		"  --%s: zero copy\n"
		"  --%s: guest polling\n",
		prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
		ZERO_COPY_KEYWORD, POLLING_KEYWORD);
}

static int
vhost_crypto_parse_args(int argc, char **argv)
{
	int opt, ret;
	char *prgname = argv[0];
	char **argvopt;
	int option_index;
	struct option lgopts[] = {
			{SOCKET_FILE_KEYWORD, required_argument, 0, 0},
			{CONFIG_KEYWORD, required_argument, 0, 0},
			{ZERO_COPY_KEYWORD, no_argument, 0, 0},
			{POLLING_KEYWORD, no_argument, 0, 0},
			{NULL, 0, 0, 0}
	};

	argvopt = argv;

	while ((opt = getopt_long(argc, argvopt, "s:",
				  lgopts, &option_index)) != EOF) {

		switch (opt) {
		case 0:
			if (strcmp(lgopts[option_index].name,
					SOCKET_FILE_KEYWORD) == 0) {
				ret = parse_socket_arg(optarg);
				if (ret < 0) {
					vhost_crypto_usage(prgname);
					return ret;
				}
			} else if (strcmp(lgopts[option_index].name,
					CONFIG_KEYWORD) == 0) {
				ret = parse_config(optarg);
				if (ret < 0) {
					vhost_crypto_usage(prgname);
					return ret;
				}
			} else if (strcmp(lgopts[option_index].name,
					ZERO_COPY_KEYWORD) == 0) {
				options.zero_copy =
					RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
			} else if (strcmp(lgopts[option_index].name,
					POLLING_KEYWORD) == 0) {
				options.guest_polling = 1;
			} else {
				vhost_crypto_usage(prgname);
				return -EINVAL;
			}
			break;
		default:
			return -1;
		}
	}

	return 0;
}

static int
new_device(int vid)
{
	struct vhost_crypto_info *info = NULL;
	char path[PATH_MAX];
	uint32_t i, j;
	int ret;

	ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
	if (ret) {
		RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
		return ret;
	}

	for (i = 0; i < options.nb_los; i++) {
		for (j = 0; j < options.los[i].nb_sockets; j++) {
			if (strcmp(path, options.los[i].socket_files[j]) == 0) {
				info = options.infos[i];
				break;
			}
		}

		if (info)
			break;
	}

	if (!info) {
		RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
		return -ENOENT;
	}

	ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
			info->sess_priv_pool,
			rte_lcore_to_socket_id(options.los[i].lcore_id));
	if (ret) {
		RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
		return ret;
	}

	ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
	if (ret) {
		RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
				options.zero_copy == 1 ? "enable" : "disable");
		return ret;
	}

	info->vids[j] = vid;
	info->initialized[j] = 1;

	rte_wmb();

	RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
			vid);
	return 0;
}

static void
destroy_device(int vid)
{
	struct vhost_crypto_info *info = NULL;
	uint32_t i, j;

	for (i = 0; i < options.nb_los; i++) {
		for (j = 0; j < options.los[i].nb_sockets; j++) {
			if (options.infos[i]->vids[j] == vid) {
				info = options.infos[i];
				break;
			}
		}
		if (info)
			break;
	}

	if (!info) {
		RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
		return;
	}

	do {

	} while (info->nb_inflight_ops);

	info->initialized[j] = 0;

	rte_wmb();

	rte_vhost_crypto_free(vid);

	RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
}

static const struct vhost_device_ops virtio_crypto_device_ops = {
	.new_device =  new_device,
	.destroy_device = destroy_device,
};

__attribute__((unused))
static void clrscr(void)
{
	system("@cls||clear");
}

static int
vhost_crypto_worker(void *arg)
{
	struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
	struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
	struct vhost_crypto_info *info = arg;
	uint16_t nb_callfds;
	int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
	uint32_t lcore_id = rte_lcore_id();
	uint32_t burst_size = MAX_PKT_BURST;
	uint32_t i, j, k;
	uint32_t to_fetch, fetched;

	int ret = 0;

	RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);

	for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
		if (rte_crypto_op_bulk_alloc(info->cop_pool,
				RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
				burst_size) < burst_size) {
			RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
			ret = -1;
			goto exit;
		}
	}

	while (1) {
		for (i = 0; i < info->nb_vids; i++) {
			if (unlikely(info->initialized[i] == 0))
				continue;

			for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
				to_fetch = RTE_MIN(burst_size,
						(NB_CRYPTO_DESCRIPTORS -
						info->nb_inflight_ops));
				fetched = rte_vhost_crypto_fetch_requests(
						info->vids[i], j, ops[j],
						to_fetch);
				info->nb_inflight_ops +=
						rte_cryptodev_enqueue_burst(
						info->cid, info->qid, ops[j],
						fetched);
				if (unlikely(rte_crypto_op_bulk_alloc(
						info->cop_pool,
						RTE_CRYPTO_OP_TYPE_SYMMETRIC,
						ops[j], fetched) < fetched)) {
					RTE_LOG(ERR, USER1, "Failed realloc\n");
					return -1;
				}

				fetched = rte_cryptodev_dequeue_burst(
						info->cid, info->qid,
						ops_deq[j], RTE_MIN(burst_size,
						info->nb_inflight_ops));
				fetched = rte_vhost_crypto_finalize_requests(
						ops_deq[j], fetched, callfds,
						&nb_callfds);

				info->nb_inflight_ops -= fetched;

				if (!options.guest_polling) {
					for (k = 0; k < nb_callfds; k++)
						eventfd_write(callfds[k],
								(eventfd_t)1);
				}

				rte_mempool_put_bulk(info->cop_pool,
						(void **)ops_deq[j], fetched);
			}
		}
	}
exit:
	return ret;
}

static void
free_resource(void)
{
	uint32_t i, j;

	for (i = 0; i < options.nb_los; i++) {
		struct lcore_option *lo = &options.los[i];
		struct vhost_crypto_info *info = options.infos[i];

		if (!info)
			continue;

		rte_mempool_free(info->cop_pool);
		rte_mempool_free(info->sess_pool);
		rte_mempool_free(info->sess_priv_pool);

		for (j = 0; j < lo->nb_sockets; j++) {
			rte_vhost_driver_unregister(lo->socket_files[i]);
			free(lo->socket_files[i]);
		}

		rte_free(info);
	}

	memset(&options, 0, sizeof(options));
}

int
main(int argc, char *argv[])
{
	struct rte_cryptodev_qp_conf qp_conf;
	struct rte_cryptodev_config config;
	struct rte_cryptodev_info dev_info;
	char name[128];
	uint32_t i, j, lcore;
	int ret;

	ret = rte_eal_init(argc, argv);
	if (ret < 0)
		return -1;
	argc -= ret;
	argv += ret;

	ret = vhost_crypto_parse_args(argc, argv);
	if (ret < 0)
		rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");

	for (i = 0; i < options.nb_los; i++) {
		struct lcore_option *lo = &options.los[i];
		struct vhost_crypto_info *info;

		info = rte_zmalloc_socket(NULL, sizeof(*info),
				RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
						lo->lcore_id));
		if (!info) {
			ret = -ENOMEM;
			goto error_exit;
		}

		info->cid = lo->cid;
		info->qid = lo->qid;
		info->nb_vids = lo->nb_sockets;

		rte_cryptodev_info_get(info->cid, &dev_info);
		if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
#define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD	crypto_aesni_mb
#define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD	crypto_aesni_gcm
			if (strstr(dev_info.driver_name,
				RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
				strstr(dev_info.driver_name,
				RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
				RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
					dev_info.driver_name);
				ret = -EPERM;
				goto error_exit;
			}
		}

		if (dev_info.max_nb_queue_pairs < info->qid + 1) {
			RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
					dev_info.max_nb_queue_pairs);
			goto error_exit;
		}

		config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
		config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);

		ret = rte_cryptodev_configure(info->cid, &config);
		if (ret < 0) {
			RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
					info->cid);
			goto error_exit;
		}

		snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
		info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
				SESSION_MAP_ENTRIES, 0, 0, 0,
				rte_lcore_to_socket_id(lo->lcore_id));

		snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
		info->sess_priv_pool = rte_mempool_create(name,
				SESSION_MAP_ENTRIES,
				rte_cryptodev_sym_get_private_session_size(
				info->cid), 64, 0, NULL, NULL, NULL, NULL,
				rte_lcore_to_socket_id(lo->lcore_id), 0);
		if (!info->sess_priv_pool || !info->sess_pool) {
			RTE_LOG(ERR, USER1, "Failed to create mempool");
			goto error_exit;
		}

		snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
		info->cop_pool = rte_crypto_op_pool_create(name,
				RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
				NB_CACHE_OBJS, 0,
				rte_lcore_to_socket_id(lo->lcore_id));

		if (!info->cop_pool) {
			RTE_LOG(ERR, USER1, "Failed to create crypto pool");
			ret = -ENOMEM;
			goto error_exit;
		}

		options.infos[i] = info;

		qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
		qp_conf.mp_session = info->sess_pool;
		qp_conf.mp_session_private = info->sess_priv_pool;

		for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
			ret = rte_cryptodev_queue_pair_setup(info->cid, j,
					&qp_conf, rte_lcore_to_socket_id(
							lo->lcore_id));
			if (ret < 0) {
				RTE_LOG(ERR, USER1, "Failed to configure qp\n");
				goto error_exit;
			}
		}
	}

	for (i = 0; i < options.nb_los; i++) {
		struct lcore_option *lo = &options.los[i];
		struct vhost_crypto_info *info = options.infos[i];

		ret = rte_cryptodev_start(lo->cid);
		if (ret < 0) {
			RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
			goto error_exit;
		}

		if (rte_eal_remote_launch(vhost_crypto_worker, info,
				lo->lcore_id) < 0) {
			RTE_LOG(ERR, USER1, "Failed to start worker lcore");
			goto error_exit;
		}

		for (j = 0; j < lo->nb_sockets; j++) {
			ret = rte_vhost_driver_register(lo->socket_files[j],
				RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
			if (ret < 0) {
				RTE_LOG(ERR, USER1, "socket %s already exists\n",
					lo->socket_files[j]);
				goto error_exit;
			}

			rte_vhost_driver_callback_register(lo->socket_files[j],
				&virtio_crypto_device_ops);

			ret = rte_vhost_driver_start(lo->socket_files[j]);
			if (ret < 0)  {
				RTE_LOG(ERR, USER1, "failed to start vhost.\n");
				goto error_exit;
			}
		}
	}

	RTE_LCORE_FOREACH(lcore)
		rte_eal_wait_lcore(lcore);

	free_resource();

	return 0;

error_exit:

	free_resource();

	return -1;
}
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
	2	* Copyright(c) 2017-2018 Intel Corporation
	3	*/
	4
	5	#include <stdio.h>
	6	#include <stdlib.h>
9f95a23c	7	#include <string.h>
11fdf7f2 TL	8	#include <unistd.h>
	9	#include <stdbool.h>
	10	#include <assert.h>
	11	#include <getopt.h>
	12
	13	#include <rte_malloc.h>
	14	#include <rte_cycles.h>
	15	#include <rte_vhost.h>
	16	#include <rte_cryptodev.h>
	17	#include <rte_vhost_crypto.h>
9f95a23c	18	#include <rte_string_fns.h>
11fdf7f2 TL	19
	20	#include <cmdline_rdline.h>
	21	#include <cmdline_parse.h>
	22	#include <cmdline_parse_string.h>
	23	#include <cmdline.h>
	24
	25	#define NB_VIRTIO_QUEUES (1)
	26	#define MAX_PKT_BURST (64)
	27	#define MAX_IV_LEN (32)
	28	#define NB_MEMPOOL_OBJS (8192)
	29	#define NB_CRYPTO_DESCRIPTORS (4096)
	30	#define NB_CACHE_OBJS (128)
	31	#define SESSION_MAP_ENTRIES (1024)
	32	#define REFRESH_TIME_SEC (3)
	33
9f95a23c TL	34	#define MAX_NB_SOCKETS (4)
9f95a23c TL	35	#define MAX_NB_WORKER_CORES (16)
11fdf7f2	36
9f95a23c TL	37	struct lcore_option {
9f95a23c TL	38	uint32_t lcore_id;
11fdf7f2 TL	39	char *socket_files[MAX_NB_SOCKETS];
	40	uint32_t nb_sockets;
	41	uint8_t cid;
	42	uint16_t qid;
9f95a23c	43	};
11fdf7f2 TL	44
	45	struct vhost_crypto_info {
	46	int vids[MAX_NB_SOCKETS];
9f95a23c	47	uint32_t nb_vids;
11fdf7f2	48	struct rte_mempool *sess_pool;
9f95a23c	49	struct rte_mempool *sess_priv_pool;
11fdf7f2	50	struct rte_mempool *cop_pool;
11fdf7f2 TL	51	uint8_t cid;
11fdf7f2 TL	52	uint32_t qid;
9f95a23c	53	uint32_t nb_inflight_ops;
11fdf7f2	54	volatile uint32_t initialized[MAX_NB_SOCKETS];
9f95a23c	55	} __rte_cache_aligned;
11fdf7f2	56
9f95a23c TL	57	struct vhost_crypto_options {
	58	struct lcore_option los[MAX_NB_WORKER_CORES];
	59	struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
	60	uint32_t nb_los;
	61	uint32_t zero_copy;
	62	uint32_t guest_polling;
	63	} options;
11fdf7f2	64
9f95a23c	65	#define CONFIG_KEYWORD "config"
11fdf7f2	66	#define SOCKET_FILE_KEYWORD "socket-file"
11fdf7f2 TL	67	#define ZERO_COPY_KEYWORD "zero-copy"
	68	#define POLLING_KEYWORD "guest-polling"
	69
9f95a23c TL	70	#define NB_SOCKET_FIELDS (2)
	71
	72	static uint32_t
	73	find_lo(uint32_t lcore_id)
	74	{
	75	uint32_t i;
	76
	77	for (i = 0; i < options.nb_los; i++)
	78	if (options.los[i].lcore_id == lcore_id)
	79	return i;
	80
	81	return UINT32_MAX;
	82	}
11fdf7f2 TL	83
	84	/** support SOCKET_FILE_PATH:CRYPTODEV_ID format */
	85	static int
	86	parse_socket_arg(char *arg)
	87	{
9f95a23c TL	88	uint32_t nb_sockets;
	89	uint32_t lcore_id;
	90	char *str_fld[NB_SOCKET_FIELDS];
	91	struct lcore_option *lo;
	92	uint32_t idx;
	93	char *end;
	94
	95	if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
	96	NB_SOCKET_FIELDS) {
	97	RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
	98	return -EINVAL;
	99	}
	100
	101	errno = 0;
	102	lcore_id = strtoul(str_fld[0], &end, 0);
	103	if (errno != 0 \|\| end == str_fld[0] \|\| lcore_id > 255)
	104	return -EINVAL;
	105
	106	idx = find_lo(lcore_id);
	107	if (idx == UINT32_MAX) {
	108	if (options.nb_los == MAX_NB_WORKER_CORES)
	109	return -ENOMEM;
	110	lo = &options.los[options.nb_los];
	111	lo->lcore_id = lcore_id;
	112	options.nb_los++;
	113	} else
	114	lo = &options.los[idx];
	115
	116	nb_sockets = lo->nb_sockets;
11fdf7f2 TL	117
	118	if (nb_sockets >= MAX_NB_SOCKETS) {
	119	RTE_LOG(ERR, USER1, "Too many socket files!\n");
	120	return -ENOMEM;
	121	}
	122
9f95a23c TL	123	lo->socket_files[nb_sockets] = strdup(str_fld[1]);
9f95a23c TL	124	if (!lo->socket_files[nb_sockets]) {
11fdf7f2 TL	125	RTE_LOG(ERR, USER1, "Insufficient memory\n");
	126	return -ENOMEM;
	127	}
	128
9f95a23c	129	lo->nb_sockets++;
11fdf7f2 TL	130
	131	return 0;
	132	}
	133
	134	static int
9f95a23c	135	parse_config(char *q_arg)
11fdf7f2	136	{
9f95a23c TL	137	struct lcore_option *lo;
	138	char s[256];
	139	const char p, p0 = q_arg;
	140	char *end;
	141	enum fieldnames {
	142	FLD_LCORE = 0,
	143	FLD_CID,
	144	FLD_QID,
	145	_NUM_FLD
	146	};
	147	uint32_t flds[_NUM_FLD];
	148	char *str_fld[_NUM_FLD];
	149	uint32_t i;
	150	uint32_t size;
11fdf7f2	151
9f95a23c TL	152	while ((p = strchr(p0, '(')) != NULL) {
	153	++p;
	154	p0 = strchr(p, ')');
	155	if (p0 == NULL)
	156	return -1;
11fdf7f2	157
9f95a23c TL	158	size = p0 - p;
	159	if (size >= sizeof(s))
	160	return -1;
11fdf7f2	161
9f95a23c TL	162	snprintf(s, sizeof(s), "%.*s", size, p);
	163	if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
	164	_NUM_FLD)
	165	return -1;
	166	for (i = 0; i < _NUM_FLD; i++) {
	167	errno = 0;
	168	flds[i] = strtoul(str_fld[i], &end, 0);
	169	if (errno != 0 \|\| end == str_fld[i] \|\| flds[i] > 255)
	170	return -EINVAL;
	171	}
11fdf7f2	172
9f95a23c TL	173	if (flds[FLD_LCORE] > RTE_MAX_LCORE)
	174	return -EINVAL;
	175
	176	i = find_lo(flds[FLD_LCORE]);
	177	if (i == UINT32_MAX) {
	178	if (options.nb_los == MAX_NB_WORKER_CORES)
	179	return -ENOMEM;
	180	lo = &options.los[options.nb_los];
	181	options.nb_los++;
	182	} else
	183	lo = &options.los[i];
	184
	185	lo->lcore_id = flds[FLD_LCORE];
	186	lo->cid = flds[FLD_CID];
	187	lo->qid = flds[FLD_QID];
11fdf7f2 TL	188	}
11fdf7f2 TL	189
11fdf7f2 TL	190	return 0;
	191	}
	192
	193	static void
	194	vhost_crypto_usage(const char *prgname)
	195	{
	196	printf("%s [EAL options] --\n"
9f95a23c TL	197	" --%s <lcore>,SOCKET-FILE-PATH\n"
9f95a23c TL	198	" --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
11fdf7f2 TL	199	" --%s: zero copy\n"
11fdf7f2 TL	200	" --%s: guest polling\n",
9f95a23c TL	201	prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
9f95a23c TL	202	ZERO_COPY_KEYWORD, POLLING_KEYWORD);
11fdf7f2 TL	203	}
	204
	205	static int
	206	vhost_crypto_parse_args(int argc, char **argv)
	207	{
	208	int opt, ret;
	209	char *prgname = argv[0];
	210	char **argvopt;
	211	int option_index;
	212	struct option lgopts[] = {
	213	{SOCKET_FILE_KEYWORD, required_argument, 0, 0},
9f95a23c	214	{CONFIG_KEYWORD, required_argument, 0, 0},
11fdf7f2 TL	215	{ZERO_COPY_KEYWORD, no_argument, 0, 0},
	216	{POLLING_KEYWORD, no_argument, 0, 0},
	217	{NULL, 0, 0, 0}
	218	};
	219
11fdf7f2 TL	220	argvopt = argv;
	221
	222	while ((opt = getopt_long(argc, argvopt, "s:",
	223	lgopts, &option_index)) != EOF) {
	224
	225	switch (opt) {
	226	case 0:
	227	if (strcmp(lgopts[option_index].name,
	228	SOCKET_FILE_KEYWORD) == 0) {
	229	ret = parse_socket_arg(optarg);
	230	if (ret < 0) {
	231	vhost_crypto_usage(prgname);
	232	return ret;
	233	}
	234	} else if (strcmp(lgopts[option_index].name,
9f95a23c TL	235	CONFIG_KEYWORD) == 0) {
9f95a23c TL	236	ret = parse_config(optarg);
11fdf7f2 TL	237	if (ret < 0) {
	238	vhost_crypto_usage(prgname);
	239	return ret;
	240	}
	241	} else if (strcmp(lgopts[option_index].name,
	242	ZERO_COPY_KEYWORD) == 0) {
	243	options.zero_copy =
	244	RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
	245	} else if (strcmp(lgopts[option_index].name,
	246	POLLING_KEYWORD) == 0) {
	247	options.guest_polling = 1;
	248	} else {
	249	vhost_crypto_usage(prgname);
	250	return -EINVAL;
	251	}
	252	break;
	253	default:
	254	return -1;
	255	}
	256	}
	257
11fdf7f2 TL	258	return 0;
	259	}
	260
	261	static int
	262	new_device(int vid)
	263	{
9f95a23c	264	struct vhost_crypto_info *info = NULL;
11fdf7f2	265	char path[PATH_MAX];
9f95a23c	266	uint32_t i, j;
11fdf7f2 TL	267	int ret;
	268
	269	ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
	270	if (ret) {
	271	RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
	272	return ret;
	273	}
	274
9f95a23c TL	275	for (i = 0; i < options.nb_los; i++) {
	276	for (j = 0; j < options.los[i].nb_sockets; j++) {
	277	if (strcmp(path, options.los[i].socket_files[j]) == 0) {
	278	info = options.infos[i];
	279	break;
	280	}
	281	}
	282
	283	if (info)
11fdf7f2 TL	284	break;
	285	}
	286
9f95a23c	287	if (!info) {
11fdf7f2 TL	288	RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
	289	return -ENOENT;
	290	}
	291
9f95a23c TL	292	ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
	293	info->sess_priv_pool,
	294	rte_lcore_to_socket_id(options.los[i].lcore_id));
11fdf7f2 TL	295	if (ret) {
	296	RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
	297	return ret;
	298	}
	299
	300	ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
	301	if (ret) {
	302	RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
	303	options.zero_copy == 1 ? "enable" : "disable");
	304	return ret;
	305	}
	306
9f95a23c TL	307	info->vids[j] = vid;
9f95a23c TL	308	info->initialized[j] = 1;
11fdf7f2 TL	309
	310	rte_wmb();
	311
	312	RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
	313	vid);
	314	return 0;
	315	}
	316
	317	static void
	318	destroy_device(int vid)
	319	{
9f95a23c TL	320	struct vhost_crypto_info *info = NULL;
	321	uint32_t i, j;
	322
	323	for (i = 0; i < options.nb_los; i++) {
	324	for (j = 0; j < options.los[i].nb_sockets; j++) {
	325	if (options.infos[i]->vids[j] == vid) {
	326	info = options.infos[i];
	327	break;
	328	}
	329	}
	330	if (info)
11fdf7f2 TL	331	break;
	332	}
	333
9f95a23c	334	if (!info) {
11fdf7f2 TL	335	RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
	336	return;
	337	}
	338
9f95a23c TL	339	do {
	340
	341	} while (info->nb_inflight_ops);
	342
	343	info->initialized[j] = 0;
11fdf7f2 TL	344
	345	rte_wmb();
	346
	347	rte_vhost_crypto_free(vid);
	348
	349	RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
	350	}
	351
	352	static const struct vhost_device_ops virtio_crypto_device_ops = {
	353	.new_device = new_device,
	354	.destroy_device = destroy_device,
	355	};
	356
	357	__attribute__((unused))
	358	static void clrscr(void)
	359	{
	360	system("@cls\|\|clear");
	361	}
	362
	363	static int
9f95a23c	364	vhost_crypto_worker(void *arg)
11fdf7f2 TL	365	{
	366	struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
	367	struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
9f95a23c	368	struct vhost_crypto_info *info = arg;
11fdf7f2 TL	369	uint16_t nb_callfds;
	370	int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
	371	uint32_t lcore_id = rte_lcore_id();
	372	uint32_t burst_size = MAX_PKT_BURST;
	373	uint32_t i, j, k;
	374	uint32_t to_fetch, fetched;
11fdf7f2 TL	375
	376	int ret = 0;
	377
	378	RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
	379
	380	for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
9f95a23c	381	if (rte_crypto_op_bulk_alloc(info->cop_pool,
11fdf7f2 TL	382	RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
	383	burst_size) < burst_size) {
	384	RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
	385	ret = -1;
	386	goto exit;
	387	}
	388	}
	389
	390	while (1) {
9f95a23c TL	391	for (i = 0; i < info->nb_vids; i++) {
9f95a23c TL	392	if (unlikely(info->initialized[i] == 0))
11fdf7f2 TL	393	continue;
	394
	395	for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
11fdf7f2 TL	396	to_fetch = RTE_MIN(burst_size,
11fdf7f2 TL	397	(NB_CRYPTO_DESCRIPTORS -
9f95a23c	398	info->nb_inflight_ops));
11fdf7f2	399	fetched = rte_vhost_crypto_fetch_requests(
9f95a23c	400	info->vids[i], j, ops[j],
11fdf7f2	401	to_fetch);
9f95a23c TL	402	info->nb_inflight_ops +=
	403	rte_cryptodev_enqueue_burst(
	404	info->cid, info->qid, ops[j],
11fdf7f2 TL	405	fetched);
11fdf7f2 TL	406	if (unlikely(rte_crypto_op_bulk_alloc(
9f95a23c	407	info->cop_pool,
11fdf7f2 TL	408	RTE_CRYPTO_OP_TYPE_SYMMETRIC,
	409	ops[j], fetched) < fetched)) {
	410	RTE_LOG(ERR, USER1, "Failed realloc\n");
	411	return -1;
	412	}
11fdf7f2	413
11fdf7f2	414	fetched = rte_cryptodev_dequeue_burst(
9f95a23c	415	info->cid, info->qid,
11fdf7f2	416	ops_deq[j], RTE_MIN(burst_size,
9f95a23c	417	info->nb_inflight_ops));
11fdf7f2 TL	418	fetched = rte_vhost_crypto_finalize_requests(
	419	ops_deq[j], fetched, callfds,
	420	&nb_callfds);
	421
9f95a23c	422	info->nb_inflight_ops -= fetched;
11fdf7f2 TL	423
	424	if (!options.guest_polling) {
	425	for (k = 0; k < nb_callfds; k++)
	426	eventfd_write(callfds[k],
	427	(eventfd_t)1);
	428	}
	429
9f95a23c	430	rte_mempool_put_bulk(info->cop_pool,
11fdf7f2	431	(void **)ops_deq[j], fetched);
11fdf7f2 TL	432	}
	433	}
	434	}
	435	exit:
	436	return ret;
	437	}
	438
11fdf7f2	439	static void
9f95a23c	440	free_resource(void)
11fdf7f2	441	{
9f95a23c TL	442	uint32_t i, j;
	443
	444	for (i = 0; i < options.nb_los; i++) {
	445	struct lcore_option *lo = &options.los[i];
	446	struct vhost_crypto_info *info = options.infos[i];
	447
	448	if (!info)
	449	continue;
	450
	451	rte_mempool_free(info->cop_pool);
	452	rte_mempool_free(info->sess_pool);
	453	rte_mempool_free(info->sess_priv_pool);
11fdf7f2	454
9f95a23c TL	455	for (j = 0; j < lo->nb_sockets; j++) {
	456	rte_vhost_driver_unregister(lo->socket_files[i]);
	457	free(lo->socket_files[i]);
	458	}
	459
	460	rte_free(info);
	461	}
	462
	463	memset(&options, 0, sizeof(options));
11fdf7f2 TL	464	}
	465
	466	int
	467	main(int argc, char *argv[])
	468	{
9f95a23c	469	struct rte_cryptodev_qp_conf qp_conf;
11fdf7f2 TL	470	struct rte_cryptodev_config config;
11fdf7f2 TL	471	struct rte_cryptodev_info dev_info;
11fdf7f2	472	char name[128];
9f95a23c	473	uint32_t i, j, lcore;
11fdf7f2 TL	474	int ret;
	475
	476	ret = rte_eal_init(argc, argv);
	477	if (ret < 0)
	478	return -1;
	479	argc -= ret;
	480	argv += ret;
	481
	482	ret = vhost_crypto_parse_args(argc, argv);
	483	if (ret < 0)
	484	rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
	485
9f95a23c TL	486	for (i = 0; i < options.nb_los; i++) {
	487	struct lcore_option *lo = &options.los[i];
	488	struct vhost_crypto_info *info;
11fdf7f2	489
9f95a23c TL	490	info = rte_zmalloc_socket(NULL, sizeof(*info),
	491	RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
	492	lo->lcore_id));
	493	if (!info) {
	494	ret = -ENOMEM;
	495	goto error_exit;
	496	}
11fdf7f2	497
9f95a23c TL	498	info->cid = lo->cid;
	499	info->qid = lo->qid;
	500	info->nb_vids = lo->nb_sockets;
	501
	502	rte_cryptodev_info_get(info->cid, &dev_info);
	503	if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
	504	#define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
	505	#define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
	506	if (strstr(dev_info.driver_name,
	507	RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) \|\|
	508	strstr(dev_info.driver_name,
	509	RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
	510	RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
	511	dev_info.driver_name);
	512	ret = -EPERM;
	513	goto error_exit;
	514	}
	515	}
11fdf7f2	516
9f95a23c TL	517	if (dev_info.max_nb_queue_pairs < info->qid + 1) {
	518	RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
	519	dev_info.max_nb_queue_pairs);
	520	goto error_exit;
	521	}
11fdf7f2	522
9f95a23c TL	523	config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
9f95a23c TL	524	config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
11fdf7f2	525
9f95a23c TL	526	ret = rte_cryptodev_configure(info->cid, &config);
	527	if (ret < 0) {
	528	RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
	529	info->cid);
	530	goto error_exit;
	531	}
11fdf7f2	532
9f95a23c TL	533	snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
	534	info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
	535	SESSION_MAP_ENTRIES, 0, 0, 0,
	536	rte_lcore_to_socket_id(lo->lcore_id));
	537
	538	snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
	539	info->sess_priv_pool = rte_mempool_create(name,
	540	SESSION_MAP_ENTRIES,
	541	rte_cryptodev_sym_get_private_session_size(
	542	info->cid), 64, 0, NULL, NULL, NULL, NULL,
	543	rte_lcore_to_socket_id(lo->lcore_id), 0);
	544	if (!info->sess_priv_pool \|\| !info->sess_pool) {
	545	RTE_LOG(ERR, USER1, "Failed to create mempool");
	546	goto error_exit;
	547	}
11fdf7f2	548
9f95a23c TL	549	snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
	550	info->cop_pool = rte_crypto_op_pool_create(name,
	551	RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
	552	NB_CACHE_OBJS, 0,
	553	rte_lcore_to_socket_id(lo->lcore_id));
11fdf7f2	554
9f95a23c TL	555	if (!info->cop_pool) {
	556	RTE_LOG(ERR, USER1, "Failed to create crypto pool");
	557	ret = -ENOMEM;
11fdf7f2 TL	558	goto error_exit;
11fdf7f2 TL	559	}
11fdf7f2	560
9f95a23c	561	options.infos[i] = info;
11fdf7f2	562
9f95a23c TL	563	qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
	564	qp_conf.mp_session = info->sess_pool;
	565	qp_conf.mp_session_private = info->sess_priv_pool;
11fdf7f2	566
9f95a23c TL	567	for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
	568	ret = rte_cryptodev_queue_pair_setup(info->cid, j,
	569	&qp_conf, rte_lcore_to_socket_id(
	570	lo->lcore_id));
	571	if (ret < 0) {
	572	RTE_LOG(ERR, USER1, "Failed to configure qp\n");
	573	goto error_exit;
	574	}
	575	}
11fdf7f2 TL	576	}
11fdf7f2 TL	577
9f95a23c TL	578	for (i = 0; i < options.nb_los; i++) {
	579	struct lcore_option *lo = &options.los[i];
	580	struct vhost_crypto_info *info = options.infos[i];
	581
	582	ret = rte_cryptodev_start(lo->cid);
	583	if (ret < 0) {
	584	RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
	585	goto error_exit;
	586	}
	587
	588	if (rte_eal_remote_launch(vhost_crypto_worker, info,
	589	lo->lcore_id) < 0) {
	590	RTE_LOG(ERR, USER1, "Failed to start worker lcore");
11fdf7f2 TL	591	goto error_exit;
	592	}
	593
9f95a23c TL	594	for (j = 0; j < lo->nb_sockets; j++) {
	595	ret = rte_vhost_driver_register(lo->socket_files[j],
	596	RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
	597	if (ret < 0) {
	598	RTE_LOG(ERR, USER1, "socket %s already exists\n",
	599	lo->socket_files[j]);
	600	goto error_exit;
	601	}
	602
	603	rte_vhost_driver_callback_register(lo->socket_files[j],
11fdf7f2 TL	604	&virtio_crypto_device_ops);
11fdf7f2 TL	605
9f95a23c TL	606	ret = rte_vhost_driver_start(lo->socket_files[j]);
	607	if (ret < 0) {
	608	RTE_LOG(ERR, USER1, "failed to start vhost.\n");
	609	goto error_exit;
	610	}
11fdf7f2 TL	611	}
	612	}
	613
9f95a23c TL	614	RTE_LCORE_FOREACH(lcore)
9f95a23c TL	615	rte_eal_wait_lcore(lcore);
11fdf7f2	616
9f95a23c	617	free_resource();
11fdf7f2 TL	618
	619	return 0;
	620
	621	error_exit:
11fdf7f2	622
9f95a23c	623	free_resource();
11fdf7f2 TL	624
	625	return -1;
	626	}