[ceph.git] / ceph / src / seastar / dpdk / drivers / crypto / scheduler / scheduler_pmd.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2017 Intel Corporation
 */
#include <rte_common.h>
#include <rte_hexdump.h>
#include <rte_cryptodev.h>
#include <rte_cryptodev_pmd.h>
#include <rte_bus_vdev.h>
#include <rte_malloc.h>
#include <rte_cpuflags.h>
#include <rte_reorder.h>
#include <rte_string_fns.h>

#include "rte_cryptodev_scheduler.h"
#include "scheduler_pmd_private.h"

uint8_t cryptodev_scheduler_driver_id;

struct scheduler_init_params {
	struct rte_cryptodev_pmd_init_params def_p;
	uint32_t nb_slaves;
	enum rte_cryptodev_scheduler_mode mode;
	char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
	uint32_t enable_ordering;
	uint16_t wc_pool[RTE_MAX_LCORE];
	uint16_t nb_wc;
	char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
			[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
};

#define RTE_CRYPTODEV_VDEV_NAME			("name")
#define RTE_CRYPTODEV_VDEV_SLAVE		("slave")
#define RTE_CRYPTODEV_VDEV_MODE			("mode")
#define RTE_CRYPTODEV_VDEV_MODE_PARAM		("mode_param")
#define RTE_CRYPTODEV_VDEV_ORDERING		("ordering")
#define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG	("max_nb_queue_pairs")
#define RTE_CRYPTODEV_VDEV_SOCKET_ID		("socket_id")
#define RTE_CRYPTODEV_VDEV_COREMASK		("coremask")
#define RTE_CRYPTODEV_VDEV_CORELIST		("corelist")

static const char * const scheduler_valid_params[] = {
	RTE_CRYPTODEV_VDEV_NAME,
	RTE_CRYPTODEV_VDEV_SLAVE,
	RTE_CRYPTODEV_VDEV_MODE,
	RTE_CRYPTODEV_VDEV_MODE_PARAM,
	RTE_CRYPTODEV_VDEV_ORDERING,
	RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
	RTE_CRYPTODEV_VDEV_SOCKET_ID,
	RTE_CRYPTODEV_VDEV_COREMASK,
	RTE_CRYPTODEV_VDEV_CORELIST
};

struct scheduler_parse_map {
	const char *name;
	uint32_t val;
};

const struct scheduler_parse_map scheduler_mode_map[] = {
	{RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
			CDEV_SCHED_MODE_ROUNDROBIN},
	{RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
			CDEV_SCHED_MODE_PKT_SIZE_DISTR},
	{RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
			CDEV_SCHED_MODE_FAILOVER},
	{RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
			CDEV_SCHED_MODE_MULTICORE}
};

const struct scheduler_parse_map scheduler_ordering_map[] = {
		{"enable", 1},
		{"disable", 0}
};

#define CDEV_SCHED_MODE_PARAM_SEP_CHAR		':'

static int
cryptodev_scheduler_create(const char *name,
		struct rte_vdev_device *vdev,
		struct scheduler_init_params *init_params)
{
	struct rte_cryptodev *dev;
	struct scheduler_ctx *sched_ctx;
	uint32_t i;
	int ret;

	dev = rte_cryptodev_pmd_create(name, &vdev->device,
			&init_params->def_p);
	if (dev == NULL) {
		CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
			name);
		return -EFAULT;
	}

	dev->driver_id = cryptodev_scheduler_driver_id;
	dev->dev_ops = rte_crypto_scheduler_pmd_ops;

	sched_ctx = dev->data->dev_private;
	sched_ctx->max_nb_queue_pairs =
			init_params->def_p.max_nb_queue_pairs;

	if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
		uint16_t i;

		sched_ctx->nb_wc = init_params->nb_wc;

		for (i = 0; i < sched_ctx->nb_wc; i++) {
			sched_ctx->wc_pool[i] = init_params->wc_pool[i];
			CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
				i, sched_ctx->wc_pool[i]);
		}
	}

	if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
			init_params->mode < CDEV_SCHED_MODE_COUNT) {
		union {
			struct rte_cryptodev_scheduler_threshold_option
					threshold_option;
		} option;
		enum rte_cryptodev_schedule_option_type option_type;
		char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
		char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
		char *s, *end;

		ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
			init_params->mode);
		if (ret < 0) {
			rte_cryptodev_pmd_release_device(dev);
			return ret;
		}

		for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
			if (scheduler_mode_map[i].val != sched_ctx->mode)
				continue;

			CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
					scheduler_mode_map[i].name);
			break;
		}

		if (strlen(init_params->mode_param_str) > 0) {
			s = strchr(init_params->mode_param_str,
					CDEV_SCHED_MODE_PARAM_SEP_CHAR);
			if (s == NULL) {
				CR_SCHED_LOG(ERR, "Invalid mode param");
				return -EINVAL;
			}

			strlcpy(param_name, init_params->mode_param_str,
					s - init_params->mode_param_str + 1);
			s++;
			strlcpy(param_val, s,
					RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);

			switch (init_params->mode) {
			case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
				if (strcmp(param_name,
					RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
						!= 0) {
					CR_SCHED_LOG(ERR, "Invalid mode param");
					return -EINVAL;
				}
				option_type = CDEV_SCHED_OPTION_THRESHOLD;

				option.threshold_option.threshold =
						strtoul(param_val, &end, 0);
				break;
			default:
				CR_SCHED_LOG(ERR, "Invalid mode param");
				return -EINVAL;
			}

			if (sched_ctx->ops.option_set(dev, option_type,
					(void *)&option) < 0) {
				CR_SCHED_LOG(ERR, "Invalid mode param");
				return -EINVAL;
			}

			RTE_LOG(INFO, PMD, "  Sched mode param (%s = %s)\n",
					param_name, param_val);
		}
	}

	sched_ctx->reordering_enabled = init_params->enable_ordering;

	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
		if (scheduler_ordering_map[i].val !=
				sched_ctx->reordering_enabled)
			continue;

		CR_SCHED_LOG(INFO, "  Packet ordering = %s",
				scheduler_ordering_map[i].name);

		break;
	}

	for (i = 0; i < init_params->nb_slaves; i++) {
		sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
			rte_zmalloc_socket(
				NULL,
				RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
				SOCKET_ID_ANY);

		if (!sched_ctx->init_slave_names[
				sched_ctx->nb_init_slaves]) {
			CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
					name);
			return -ENOMEM;
		}

		strncpy(sched_ctx->init_slave_names[
					sched_ctx->nb_init_slaves],
				init_params->slave_names[i],
				RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);

		sched_ctx->nb_init_slaves++;
	}

	/*
	 * Initialize capabilities structure as an empty structure,
	 * in case device information is requested when no slaves are attached
	 */
	sched_ctx->capabilities = rte_zmalloc_socket(NULL,
			sizeof(struct rte_cryptodev_capabilities),
			0, SOCKET_ID_ANY);

	if (!sched_ctx->capabilities) {
		CR_SCHED_LOG(ERR, "Not enough memory for capability "
				"information");
		return -ENOMEM;
	}

	return 0;
}

static int
cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
{
	const char *name;
	struct rte_cryptodev *dev;
	struct scheduler_ctx *sched_ctx;

	if (vdev == NULL)
		return -EINVAL;

	name = rte_vdev_device_name(vdev);
	dev = rte_cryptodev_pmd_get_named_dev(name);
	if (dev == NULL)
		return -EINVAL;

	sched_ctx = dev->data->dev_private;

	if (sched_ctx->nb_slaves) {
		uint32_t i;

		for (i = 0; i < sched_ctx->nb_slaves; i++)
			rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
					sched_ctx->slaves[i].dev_id);
	}

	return rte_cryptodev_pmd_destroy(dev);
}

/** Parse integer from integer argument */
static int
parse_integer_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	int *i = (int *) extra_args;

	*i = atoi(value);
	if (*i < 0) {
		CR_SCHED_LOG(ERR, "Argument has to be positive.");
		return -EINVAL;
	}

	return 0;
}

/** Parse integer from hexadecimal integer argument */
static int
parse_coremask_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	int i, j, val;
	uint16_t idx = 0;
	char c;
	struct scheduler_init_params *params = extra_args;

	params->nb_wc = 0;

	if (value == NULL)
		return -1;
	/* Remove all blank characters ahead and after .
	 * Remove 0x/0X if exists.
	 */
	while (isblank(*value))
		value++;
	if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
		value += 2;
	i = strlen(value);
	while ((i > 0) && isblank(value[i - 1]))
		i--;

	if (i == 0)
		return -1;

	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
		c = value[i];
		if (isxdigit(c) == 0) {
			/* invalid characters */
			return -1;
		}
		if (isdigit(c))
			val = c - '0';
		else if (isupper(c))
			val = c - 'A' + 10;
		else
			val = c - 'a' + 10;

		for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
			if ((1 << j) & val)
				params->wc_pool[params->nb_wc++] = idx;
		}
	}

	return 0;
}

/** Parse integer from list of integers argument */
static int
parse_corelist_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct scheduler_init_params *params = extra_args;

	params->nb_wc = 0;

	const char *token = value;

	while (isdigit(token[0])) {
		char *rval;
		unsigned int core = strtoul(token, &rval, 10);

		if (core >= RTE_MAX_LCORE) {
			CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
				   "than %u.", core, RTE_MAX_LCORE);
		}
		params->wc_pool[params->nb_wc++] = (uint16_t)core;
		token = (const char *)rval;
		if (token[0] == '\0')
			break;
		token++;
	}

	return 0;
}

/** Parse name */
static int
parse_name_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct rte_cryptodev_pmd_init_params *params = extra_args;

	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
		CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
				"%u bytes.", value,
				RTE_CRYPTODEV_NAME_MAX_LEN - 1);
		return -EINVAL;
	}

	strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);

	return 0;
}

/** Parse slave */
static int
parse_slave_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct scheduler_init_params *param = extra_args;

	if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
		CR_SCHED_LOG(ERR, "Too many slaves.");
		return -ENOMEM;
	}

	strncpy(param->slave_names[param->nb_slaves++], value,
			RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);

	return 0;
}

static int
parse_mode_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct scheduler_init_params *param = extra_args;
	uint32_t i;

	for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
		if (strcmp(value, scheduler_mode_map[i].name) == 0) {
			param->mode = (enum rte_cryptodev_scheduler_mode)
					scheduler_mode_map[i].val;

			break;
		}
	}

	if (i == RTE_DIM(scheduler_mode_map)) {
		CR_SCHED_LOG(ERR, "Unrecognized input.");
		return -EINVAL;
	}

	return 0;
}

static int
parse_mode_param_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct scheduler_init_params *param = extra_args;

	strlcpy(param->mode_param_str, value,
			RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);

	return 0;
}

static int
parse_ordering_arg(const char *key __rte_unused,
		const char *value, void *extra_args)
{
	struct scheduler_init_params *param = extra_args;
	uint32_t i;

	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
		if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
			param->enable_ordering =
					scheduler_ordering_map[i].val;
			break;
		}
	}

	if (i == RTE_DIM(scheduler_ordering_map)) {
		CR_SCHED_LOG(ERR, "Unrecognized input.");
		return -EINVAL;
	}

	return 0;
}

static int
scheduler_parse_init_params(struct scheduler_init_params *params,
		const char *input_args)
{
	struct rte_kvargs *kvlist = NULL;
	int ret = 0;

	if (params == NULL)
		return -EINVAL;

	if (input_args) {
		kvlist = rte_kvargs_parse(input_args,
				scheduler_valid_params);
		if (kvlist == NULL)
			return -1;

		ret = rte_kvargs_process(kvlist,
				RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
				&parse_integer_arg,
				&params->def_p.max_nb_queue_pairs);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
				&parse_integer_arg,
				&params->def_p.socket_id);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
				&parse_coremask_arg,
				params);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
				&parse_corelist_arg,
				params);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
				&parse_name_arg,
				&params->def_p);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
				&parse_slave_arg, params);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
				&parse_mode_arg, params);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
				&parse_mode_param_arg, params);
		if (ret < 0)
			goto free_kvlist;

		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
				&parse_ordering_arg, params);
		if (ret < 0)
			goto free_kvlist;
	}

free_kvlist:
	rte_kvargs_free(kvlist);
	return ret;
}

static int
cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
{
	struct scheduler_init_params init_params = {
		.def_p = {
			"",
			sizeof(struct scheduler_ctx),
			rte_socket_id(),
			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
		},
		.nb_slaves = 0,
		.mode = CDEV_SCHED_MODE_NOT_SET,
		.enable_ordering = 0,
		.slave_names = { {0} }
	};
	const char *name;

	name = rte_vdev_device_name(vdev);
	if (name == NULL)
		return -EINVAL;

	scheduler_parse_init_params(&init_params,
				    rte_vdev_device_args(vdev));


	return cryptodev_scheduler_create(name,
					vdev,
					&init_params);
}

static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
	.probe = cryptodev_scheduler_probe,
	.remove = cryptodev_scheduler_remove
};

static struct cryptodev_driver scheduler_crypto_drv;

RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
	cryptodev_scheduler_pmd_drv);
RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
	"max_nb_queue_pairs=<int> "
	"socket_id=<int> "
	"slave=<name>");
RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
		cryptodev_scheduler_pmd_drv.driver,
		cryptodev_scheduler_driver_id);
Commit	Line	Data
9f95a23c TL	1	/* SPDX-License-Identifier: BSD-3-Clause
9f95a23c TL	2	* Copyright(c) 2017 Intel Corporation
11fdf7f2 TL	3	*/
	4	#include <rte_common.h>
	5	#include <rte_hexdump.h>
	6	#include <rte_cryptodev.h>
	7	#include <rte_cryptodev_pmd.h>
9f95a23c	8	#include <rte_bus_vdev.h>
11fdf7f2 TL	9	#include <rte_malloc.h>
	10	#include <rte_cpuflags.h>
	11	#include <rte_reorder.h>
9f95a23c	12	#include <rte_string_fns.h>
11fdf7f2 TL	13
	14	#include "rte_cryptodev_scheduler.h"
	15	#include "scheduler_pmd_private.h"
	16
9f95a23c TL	17	uint8_t cryptodev_scheduler_driver_id;
9f95a23c TL	18
11fdf7f2	19	struct scheduler_init_params {
9f95a23c	20	struct rte_cryptodev_pmd_init_params def_p;
11fdf7f2 TL	21	uint32_t nb_slaves;
11fdf7f2 TL	22	enum rte_cryptodev_scheduler_mode mode;
9f95a23c	23	char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
11fdf7f2	24	uint32_t enable_ordering;
9f95a23c TL	25	uint16_t wc_pool[RTE_MAX_LCORE];
9f95a23c TL	26	uint16_t nb_wc;
11fdf7f2 TL	27	char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
	28	[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
	29	};
	30
	31	#define RTE_CRYPTODEV_VDEV_NAME ("name")
	32	#define RTE_CRYPTODEV_VDEV_SLAVE ("slave")
	33	#define RTE_CRYPTODEV_VDEV_MODE ("mode")
9f95a23c	34	#define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param")
11fdf7f2 TL	35	#define RTE_CRYPTODEV_VDEV_ORDERING ("ordering")
11fdf7f2 TL	36	#define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs")
11fdf7f2	37	#define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id")
9f95a23c TL	38	#define RTE_CRYPTODEV_VDEV_COREMASK ("coremask")
9f95a23c TL	39	#define RTE_CRYPTODEV_VDEV_CORELIST ("corelist")
11fdf7f2	40
9f95a23c	41	static const char * const scheduler_valid_params[] = {
11fdf7f2 TL	42	RTE_CRYPTODEV_VDEV_NAME,
	43	RTE_CRYPTODEV_VDEV_SLAVE,
	44	RTE_CRYPTODEV_VDEV_MODE,
9f95a23c	45	RTE_CRYPTODEV_VDEV_MODE_PARAM,
11fdf7f2 TL	46	RTE_CRYPTODEV_VDEV_ORDERING,
11fdf7f2 TL	47	RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
9f95a23c TL	48	RTE_CRYPTODEV_VDEV_SOCKET_ID,
	49	RTE_CRYPTODEV_VDEV_COREMASK,
	50	RTE_CRYPTODEV_VDEV_CORELIST
11fdf7f2 TL	51	};
	52
	53	struct scheduler_parse_map {
	54	const char *name;
	55	uint32_t val;
	56	};
	57
	58	const struct scheduler_parse_map scheduler_mode_map[] = {
	59	{RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
	60	CDEV_SCHED_MODE_ROUNDROBIN},
	61	{RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
	62	CDEV_SCHED_MODE_PKT_SIZE_DISTR},
	63	{RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
9f95a23c TL	64	CDEV_SCHED_MODE_FAILOVER},
	65	{RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
	66	CDEV_SCHED_MODE_MULTICORE}
11fdf7f2 TL	67	};
	68
	69	const struct scheduler_parse_map scheduler_ordering_map[] = {
	70	{"enable", 1},
	71	{"disable", 0}
	72	};
	73
9f95a23c TL	74	#define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':'
9f95a23c TL	75
11fdf7f2 TL	76	static int
11fdf7f2 TL	77	cryptodev_scheduler_create(const char *name,
9f95a23c TL	78	struct rte_vdev_device *vdev,
9f95a23c TL	79	struct scheduler_init_params *init_params)
11fdf7f2 TL	80	{
	81	struct rte_cryptodev *dev;
	82	struct scheduler_ctx *sched_ctx;
	83	uint32_t i;
	84	int ret;
	85
9f95a23c TL	86	dev = rte_cryptodev_pmd_create(name, &vdev->device,
9f95a23c TL	87	&init_params->def_p);
11fdf7f2	88	if (dev == NULL) {
9f95a23c	89	CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
11fdf7f2 TL	90	name);
	91	return -EFAULT;
	92	}
	93
9f95a23c	94	dev->driver_id = cryptodev_scheduler_driver_id;
11fdf7f2 TL	95	dev->dev_ops = rte_crypto_scheduler_pmd_ops;
	96
	97	sched_ctx = dev->data->dev_private;
	98	sched_ctx->max_nb_queue_pairs =
	99	init_params->def_p.max_nb_queue_pairs;
	100
9f95a23c TL	101	if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
	102	uint16_t i;
	103
	104	sched_ctx->nb_wc = init_params->nb_wc;
	105
	106	for (i = 0; i < sched_ctx->nb_wc; i++) {
	107	sched_ctx->wc_pool[i] = init_params->wc_pool[i];
	108	CR_SCHED_LOG(INFO, " Worker core[%u]=%u added",
	109	i, sched_ctx->wc_pool[i]);
	110	}
	111	}
	112
11fdf7f2 TL	113	if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
11fdf7f2 TL	114	init_params->mode < CDEV_SCHED_MODE_COUNT) {
9f95a23c TL	115	union {
	116	struct rte_cryptodev_scheduler_threshold_option
	117	threshold_option;
	118	} option;
	119	enum rte_cryptodev_schedule_option_type option_type;
	120	char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
	121	char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
	122	char s, end;
	123
11fdf7f2 TL	124	ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
	125	init_params->mode);
	126	if (ret < 0) {
	127	rte_cryptodev_pmd_release_device(dev);
	128	return ret;
	129	}
	130
	131	for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
	132	if (scheduler_mode_map[i].val != sched_ctx->mode)
	133	continue;
	134
9f95a23c	135	CR_SCHED_LOG(INFO, " Scheduling mode = %s",
11fdf7f2 TL	136	scheduler_mode_map[i].name);
	137	break;
	138	}
9f95a23c TL	139
	140	if (strlen(init_params->mode_param_str) > 0) {
	141	s = strchr(init_params->mode_param_str,
	142	CDEV_SCHED_MODE_PARAM_SEP_CHAR);
	143	if (s == NULL) {
	144	CR_SCHED_LOG(ERR, "Invalid mode param");
	145	return -EINVAL;
	146	}
	147
	148	strlcpy(param_name, init_params->mode_param_str,
	149	s - init_params->mode_param_str + 1);
	150	s++;
	151	strlcpy(param_val, s,
	152	RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
	153
	154	switch (init_params->mode) {
	155	case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
	156	if (strcmp(param_name,
	157	RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
	158	!= 0) {
	159	CR_SCHED_LOG(ERR, "Invalid mode param");
	160	return -EINVAL;
	161	}
	162	option_type = CDEV_SCHED_OPTION_THRESHOLD;
	163
	164	option.threshold_option.threshold =
	165	strtoul(param_val, &end, 0);
	166	break;
	167	default:
	168	CR_SCHED_LOG(ERR, "Invalid mode param");
	169	return -EINVAL;
	170	}
	171
	172	if (sched_ctx->ops.option_set(dev, option_type,
	173	(void *)&option) < 0) {
	174	CR_SCHED_LOG(ERR, "Invalid mode param");
	175	return -EINVAL;
	176	}
	177
	178	RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n",
	179	param_name, param_val);
	180	}
11fdf7f2 TL	181	}
	182
	183	sched_ctx->reordering_enabled = init_params->enable_ordering;
	184
	185	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
	186	if (scheduler_ordering_map[i].val !=
	187	sched_ctx->reordering_enabled)
	188	continue;
	189
9f95a23c	190	CR_SCHED_LOG(INFO, " Packet ordering = %s",
11fdf7f2 TL	191	scheduler_ordering_map[i].name);
	192
	193	break;
	194	}
	195
	196	for (i = 0; i < init_params->nb_slaves; i++) {
	197	sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
	198	rte_zmalloc_socket(
	199	NULL,
	200	RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
	201	SOCKET_ID_ANY);
	202
	203	if (!sched_ctx->init_slave_names[
	204	sched_ctx->nb_init_slaves]) {
9f95a23c	205	CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
11fdf7f2 TL	206	name);
	207	return -ENOMEM;
	208	}
	209
	210	strncpy(sched_ctx->init_slave_names[
	211	sched_ctx->nb_init_slaves],
	212	init_params->slave_names[i],
	213	RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
	214
	215	sched_ctx->nb_init_slaves++;
	216	}
	217
	218	/*
	219	* Initialize capabilities structure as an empty structure,
	220	* in case device information is requested when no slaves are attached
	221	*/
	222	sched_ctx->capabilities = rte_zmalloc_socket(NULL,
	223	sizeof(struct rte_cryptodev_capabilities),
	224	0, SOCKET_ID_ANY);
	225
	226	if (!sched_ctx->capabilities) {
9f95a23c TL	227	CR_SCHED_LOG(ERR, "Not enough memory for capability "
9f95a23c TL	228	"information");
11fdf7f2 TL	229	return -ENOMEM;
	230	}
	231
	232	return 0;
	233	}
	234
	235	static int
	236	cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
	237	{
	238	const char *name;
	239	struct rte_cryptodev *dev;
	240	struct scheduler_ctx *sched_ctx;
	241
	242	if (vdev == NULL)
	243	return -EINVAL;
	244
	245	name = rte_vdev_device_name(vdev);
	246	dev = rte_cryptodev_pmd_get_named_dev(name);
	247	if (dev == NULL)
	248	return -EINVAL;
	249
	250	sched_ctx = dev->data->dev_private;
	251
	252	if (sched_ctx->nb_slaves) {
	253	uint32_t i;
	254
	255	for (i = 0; i < sched_ctx->nb_slaves; i++)
	256	rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
	257	sched_ctx->slaves[i].dev_id);
	258	}
	259
9f95a23c TL	260	return rte_cryptodev_pmd_destroy(dev);
	261	}
	262
	263	/** Parse integer from integer argument */
	264	static int
	265	parse_integer_arg(const char *key __rte_unused,
	266	const char value, void extra_args)
	267	{
	268	int i = (int ) extra_args;
	269
	270	*i = atoi(value);
	271	if (*i < 0) {
	272	CR_SCHED_LOG(ERR, "Argument has to be positive.");
	273	return -EINVAL;
	274	}
11fdf7f2 TL	275
	276	return 0;
	277	}
	278
9f95a23c TL	279	/** Parse integer from hexadecimal integer argument */
	280	static int
	281	parse_coremask_arg(const char *key __rte_unused,
	282	const char value, void extra_args)
11fdf7f2	283	{
9f95a23c TL	284	int i, j, val;
	285	uint16_t idx = 0;
	286	char c;
	287	struct scheduler_init_params *params = extra_args;
11fdf7f2	288
9f95a23c TL	289	params->nb_wc = 0;
	290
	291	if (value == NULL)
	292	return -1;
	293	/* Remove all blank characters ahead and after .
	294	* Remove 0x/0X if exists.
	295	*/
	296	while (isblank(*value))
	297	value++;
	298	if (value[0] == '0' && ((value[1] == 'x') \|\| (value[1] == 'X')))
	299	value += 2;
	300	i = strlen(value);
	301	while ((i > 0) && isblank(value[i - 1]))
	302	i--;
	303
	304	if (i == 0)
	305	return -1;
	306
	307	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
	308	c = value[i];
	309	if (isxdigit(c) == 0) {
	310	/* invalid characters */
	311	return -1;
	312	}
	313	if (isdigit(c))
	314	val = c - '0';
	315	else if (isupper(c))
	316	val = c - 'A' + 10;
	317	else
	318	val = c - 'a' + 10;
	319
	320	for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
	321	if ((1 << j) & val)
	322	params->wc_pool[params->nb_wc++] = idx;
	323	}
11fdf7f2 TL	324	}
11fdf7f2 TL	325
9f95a23c	326	return 0;
11fdf7f2 TL	327	}
11fdf7f2 TL	328
9f95a23c	329	/** Parse integer from list of integers argument */
11fdf7f2	330	static int
9f95a23c	331	parse_corelist_arg(const char *key __rte_unused,
11fdf7f2 TL	332	const char value, void extra_args)
11fdf7f2 TL	333	{
9f95a23c	334	struct scheduler_init_params *params = extra_args;
11fdf7f2	335
9f95a23c TL	336	params->nb_wc = 0;
	337
	338	const char *token = value;
	339
	340	while (isdigit(token[0])) {
	341	char *rval;
	342	unsigned int core = strtoul(token, &rval, 10);
	343
	344	if (core >= RTE_MAX_LCORE) {
	345	CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
	346	"than %u.", core, RTE_MAX_LCORE);
	347	}
	348	params->wc_pool[params->nb_wc++] = (uint16_t)core;
	349	token = (const char *)rval;
	350	if (token[0] == '\0')
	351	break;
	352	token++;
11fdf7f2 TL	353	}
	354
	355	return 0;
	356	}
	357
	358	/** Parse name */
	359	static int
	360	parse_name_arg(const char *key __rte_unused,
	361	const char value, void extra_args)
	362	{
9f95a23c	363	struct rte_cryptodev_pmd_init_params *params = extra_args;
11fdf7f2 TL	364
11fdf7f2 TL	365	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
9f95a23c TL	366	CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
9f95a23c TL	367	"%u bytes.", value,
11fdf7f2 TL	368	RTE_CRYPTODEV_NAME_MAX_LEN - 1);
	369	return -EINVAL;
	370	}
	371
9f95a23c	372	strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
11fdf7f2 TL	373
	374	return 0;
	375	}
	376
	377	/** Parse slave */
	378	static int
	379	parse_slave_arg(const char *key __rte_unused,
	380	const char value, void extra_args)
	381	{
	382	struct scheduler_init_params *param = extra_args;
	383
9f95a23c TL	384	if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
9f95a23c TL	385	CR_SCHED_LOG(ERR, "Too many slaves.");
11fdf7f2 TL	386	return -ENOMEM;
	387	}
	388
	389	strncpy(param->slave_names[param->nb_slaves++], value,
	390	RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
	391
	392	return 0;
	393	}
	394
	395	static int
	396	parse_mode_arg(const char *key __rte_unused,
	397	const char value, void extra_args)
	398	{
	399	struct scheduler_init_params *param = extra_args;
	400	uint32_t i;
	401
	402	for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
	403	if (strcmp(value, scheduler_mode_map[i].name) == 0) {
	404	param->mode = (enum rte_cryptodev_scheduler_mode)
	405	scheduler_mode_map[i].val;
9f95a23c	406
11fdf7f2 TL	407	break;
	408	}
	409	}
	410
	411	if (i == RTE_DIM(scheduler_mode_map)) {
9f95a23c	412	CR_SCHED_LOG(ERR, "Unrecognized input.");
11fdf7f2 TL	413	return -EINVAL;
	414	}
	415
	416	return 0;
	417	}
	418
9f95a23c TL	419	static int
	420	parse_mode_param_arg(const char *key __rte_unused,
	421	const char value, void extra_args)
	422	{
	423	struct scheduler_init_params *param = extra_args;
	424
	425	strlcpy(param->mode_param_str, value,
	426	RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
	427
	428	return 0;
	429	}
	430
11fdf7f2 TL	431	static int
	432	parse_ordering_arg(const char *key __rte_unused,
	433	const char value, void extra_args)
	434	{
	435	struct scheduler_init_params *param = extra_args;
	436	uint32_t i;
	437
	438	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
	439	if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
	440	param->enable_ordering =
	441	scheduler_ordering_map[i].val;
	442	break;
	443	}
	444	}
	445
	446	if (i == RTE_DIM(scheduler_ordering_map)) {
9f95a23c	447	CR_SCHED_LOG(ERR, "Unrecognized input.");
11fdf7f2 TL	448	return -EINVAL;
	449	}
	450
	451	return 0;
	452	}
	453
	454	static int
	455	scheduler_parse_init_params(struct scheduler_init_params *params,
	456	const char *input_args)
	457	{
	458	struct rte_kvargs *kvlist = NULL;
	459	int ret = 0;
	460
	461	if (params == NULL)
	462	return -EINVAL;
	463
	464	if (input_args) {
	465	kvlist = rte_kvargs_parse(input_args,
	466	scheduler_valid_params);
	467	if (kvlist == NULL)
	468	return -1;
	469
	470	ret = rte_kvargs_process(kvlist,
	471	RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
	472	&parse_integer_arg,
	473	&params->def_p.max_nb_queue_pairs);
	474	if (ret < 0)
	475	goto free_kvlist;
	476
9f95a23c	477	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
11fdf7f2	478	&parse_integer_arg,
9f95a23c	479	&params->def_p.socket_id);
11fdf7f2 TL	480	if (ret < 0)
	481	goto free_kvlist;
	482
9f95a23c TL	483	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
	484	&parse_coremask_arg,
	485	params);
	486	if (ret < 0)
	487	goto free_kvlist;
	488
	489	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
	490	&parse_corelist_arg,
	491	params);
11fdf7f2 TL	492	if (ret < 0)
	493	goto free_kvlist;
	494
	495	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
	496	&parse_name_arg,
	497	&params->def_p);
	498	if (ret < 0)
	499	goto free_kvlist;
	500
	501	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
	502	&parse_slave_arg, params);
	503	if (ret < 0)
	504	goto free_kvlist;
	505
	506	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
	507	&parse_mode_arg, params);
	508	if (ret < 0)
	509	goto free_kvlist;
	510
9f95a23c TL	511	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
9f95a23c TL	512	&parse_mode_param_arg, params);
11fdf7f2 TL	513	if (ret < 0)
	514	goto free_kvlist;
	515
9f95a23c TL	516	ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
	517	&parse_ordering_arg, params);
	518	if (ret < 0)
11fdf7f2	519	goto free_kvlist;
11fdf7f2 TL	520	}
	521
	522	free_kvlist:
	523	rte_kvargs_free(kvlist);
	524	return ret;
	525	}
	526
	527	static int
	528	cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
	529	{
	530	struct scheduler_init_params init_params = {
	531	.def_p = {
9f95a23c TL	532	"",
9f95a23c TL	533	sizeof(struct scheduler_ctx),
11fdf7f2	534	rte_socket_id(),
9f95a23c	535	RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
11fdf7f2 TL	536	},
	537	.nb_slaves = 0,
	538	.mode = CDEV_SCHED_MODE_NOT_SET,
	539	.enable_ordering = 0,
	540	.slave_names = { {0} }
	541	};
	542	const char *name;
	543
	544	name = rte_vdev_device_name(vdev);
	545	if (name == NULL)
	546	return -EINVAL;
	547
	548	scheduler_parse_init_params(&init_params,
	549	rte_vdev_device_args(vdev));
	550
11fdf7f2 TL	551
11fdf7f2 TL	552	return cryptodev_scheduler_create(name,
9f95a23c	553	vdev,
11fdf7f2 TL	554	&init_params);
	555	}
	556
	557	static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
	558	.probe = cryptodev_scheduler_probe,
	559	.remove = cryptodev_scheduler_remove
	560	};
	561
9f95a23c TL	562	static struct cryptodev_driver scheduler_crypto_drv;
9f95a23c TL	563
11fdf7f2 TL	564	RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
	565	cryptodev_scheduler_pmd_drv);
	566	RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
	567	"max_nb_queue_pairs=<int> "
11fdf7f2 TL	568	"socket_id=<int> "
11fdf7f2 TL	569	"slave=<name>");
9f95a23c TL	570	RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
	571	cryptodev_scheduler_pmd_drv.driver,
	572	cryptodev_scheduler_driver_id);