[ceph.git] / ceph / src / seastar / dpdk / drivers / crypto / scheduler / scheduler_failover.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2017 Intel Corporation
 */

#include <rte_cryptodev.h>
#include <rte_malloc.h>

#include "rte_cryptodev_scheduler_operations.h"
#include "scheduler_pmd_private.h"

#define PRIMARY_SLAVE_IDX	0
#define SECONDARY_SLAVE_IDX	1
#define NB_FAILOVER_SLAVES	2
#define SLAVE_SWITCH_MASK	(0x01)

struct fo_scheduler_qp_ctx {
	struct scheduler_slave primary_slave;
	struct scheduler_slave secondary_slave;

	uint8_t deq_idx;
};

static __rte_always_inline uint16_t
failover_slave_enqueue(struct scheduler_slave *slave,
		struct rte_crypto_op **ops, uint16_t nb_ops)
{
	uint16_t i, processed_ops;

	for (i = 0; i < nb_ops && i < 4; i++)
		rte_prefetch0(ops[i]->sym->session);

	processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id,
			slave->qp_id, ops, nb_ops);
	slave->nb_inflight_cops += processed_ops;

	return processed_ops;
}

static uint16_t
schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
{
	struct fo_scheduler_qp_ctx *qp_ctx =
			((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
	uint16_t enqueued_ops;

	if (unlikely(nb_ops == 0))
		return 0;

	enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave,
			ops, nb_ops);

	if (enqueued_ops < nb_ops)
		enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave,
				&ops[enqueued_ops],
				nb_ops - enqueued_ops);

	return enqueued_ops;
}


static uint16_t
schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
		uint16_t nb_ops)
{
	struct rte_ring *order_ring =
			((struct scheduler_qp_ctx *)qp)->order_ring;
	uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
			nb_ops);
	uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
			nb_ops_to_enq);

	scheduler_order_insert(order_ring, ops, nb_ops_enqd);

	return nb_ops_enqd;
}

static uint16_t
schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
{
	struct fo_scheduler_qp_ctx *qp_ctx =
			((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
	struct scheduler_slave *slaves[NB_FAILOVER_SLAVES] = {
			&qp_ctx->primary_slave, &qp_ctx->secondary_slave};
	struct scheduler_slave *slave = slaves[qp_ctx->deq_idx];
	uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;

	if (slave->nb_inflight_cops) {
		nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id,
			slave->qp_id, ops, nb_ops);
		slave->nb_inflight_cops -= nb_deq_ops;
	}

	qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_SWITCH_MASK;

	if (nb_deq_ops == nb_ops)
		return nb_deq_ops;

	slave = slaves[qp_ctx->deq_idx];

	if (slave->nb_inflight_cops) {
		nb_deq_ops2 = rte_cryptodev_dequeue_burst(slave->dev_id,
			slave->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
		slave->nb_inflight_cops -= nb_deq_ops2;
	}

	return nb_deq_ops + nb_deq_ops2;
}

static uint16_t
schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
		uint16_t nb_ops)
{
	struct rte_ring *order_ring =
			((struct scheduler_qp_ctx *)qp)->order_ring;

	schedule_dequeue(qp, ops, nb_ops);

	return scheduler_order_drain(order_ring, ops, nb_ops);
}

static int
slave_attach(__rte_unused struct rte_cryptodev *dev,
		__rte_unused uint8_t slave_id)
{
	return 0;
}

static int
slave_detach(__rte_unused struct rte_cryptodev *dev,
		__rte_unused uint8_t slave_id)
{
	return 0;
}

static int
scheduler_start(struct rte_cryptodev *dev)
{
	struct scheduler_ctx *sched_ctx = dev->data->dev_private;
	uint16_t i;

	if (sched_ctx->nb_slaves < 2) {
		CR_SCHED_LOG(ERR, "Number of slaves shall no less than 2");
		return -ENOMEM;
	}

	if (sched_ctx->reordering_enabled) {
		dev->enqueue_burst = schedule_enqueue_ordering;
		dev->dequeue_burst = schedule_dequeue_ordering;
	} else {
		dev->enqueue_burst = schedule_enqueue;
		dev->dequeue_burst = schedule_dequeue;
	}

	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
		struct fo_scheduler_qp_ctx *qp_ctx =
			((struct scheduler_qp_ctx *)
				dev->data->queue_pairs[i])->private_qp_ctx;

		rte_memcpy(&qp_ctx->primary_slave,
				&sched_ctx->slaves[PRIMARY_SLAVE_IDX],
				sizeof(struct scheduler_slave));
		rte_memcpy(&qp_ctx->secondary_slave,
				&sched_ctx->slaves[SECONDARY_SLAVE_IDX],
				sizeof(struct scheduler_slave));
	}

	return 0;
}

static int
scheduler_stop(__rte_unused struct rte_cryptodev *dev)
{
	return 0;
}

static int
scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
{
	struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
	struct fo_scheduler_qp_ctx *fo_qp_ctx;

	fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
			rte_socket_id());
	if (!fo_qp_ctx) {
		CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
		return -ENOMEM;
	}

	qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;

	return 0;
}

static int
scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
{
	return 0;
}

static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
	slave_attach,
	slave_detach,
	scheduler_start,
	scheduler_stop,
	scheduler_config_qp,
	scheduler_create_private_ctx,
	NULL,	/* option_set */
	NULL	/*option_get */
};

static struct rte_cryptodev_scheduler fo_scheduler = {
		.name = "failover-scheduler",
		.description = "scheduler which enqueues to the primary slave, "
				"and only then enqueues to the secondary slave "
				"upon failing on enqueuing to primary",
		.mode = CDEV_SCHED_MODE_FAILOVER,
		.ops = &scheduler_fo_ops
};

struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;
Commit	Line	Data
9f95a23c TL	1	/* SPDX-License-Identifier: BSD-3-Clause
9f95a23c TL	2	* Copyright(c) 2017 Intel Corporation
11fdf7f2 TL	3	*/
	4
	5	#include <rte_cryptodev.h>
	6	#include <rte_malloc.h>
	7
	8	#include "rte_cryptodev_scheduler_operations.h"
	9	#include "scheduler_pmd_private.h"
	10
	11	#define PRIMARY_SLAVE_IDX 0
	12	#define SECONDARY_SLAVE_IDX 1
	13	#define NB_FAILOVER_SLAVES 2
	14	#define SLAVE_SWITCH_MASK (0x01)
	15
	16	struct fo_scheduler_qp_ctx {
	17	struct scheduler_slave primary_slave;
	18	struct scheduler_slave secondary_slave;
	19
	20	uint8_t deq_idx;
	21	};
	22
9f95a23c TL	23	static __rte_always_inline uint16_t
9f95a23c TL	24	failover_slave_enqueue(struct scheduler_slave *slave,
11fdf7f2 TL	25	struct rte_crypto_op **ops, uint16_t nb_ops)
	26	{
	27	uint16_t i, processed_ops;
11fdf7f2 TL	28
	29	for (i = 0; i < nb_ops && i < 4; i++)
	30	rte_prefetch0(ops[i]->sym->session);
	31
11fdf7f2 TL	32	processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id,
	33	slave->qp_id, ops, nb_ops);
	34	slave->nb_inflight_cops += processed_ops;
	35
11fdf7f2 TL	36	return processed_ops;
	37	}
	38
	39	static uint16_t
	40	schedule_enqueue(void qp, struct rte_crypto_op *ops, uint16_t nb_ops)
	41	{
	42	struct fo_scheduler_qp_ctx *qp_ctx =
	43	((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
	44	uint16_t enqueued_ops;
	45
	46	if (unlikely(nb_ops == 0))
	47	return 0;
	48
	49	enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave,
9f95a23c	50	ops, nb_ops);
11fdf7f2 TL	51
	52	if (enqueued_ops < nb_ops)
	53	enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave,
9f95a23c	54	&ops[enqueued_ops],
11fdf7f2 TL	55	nb_ops - enqueued_ops);
	56
	57	return enqueued_ops;
	58	}
	59
	60
	61	static uint16_t
	62	schedule_enqueue_ordering(void qp, struct rte_crypto_op *ops,
	63	uint16_t nb_ops)
	64	{
	65	struct rte_ring *order_ring =
	66	((struct scheduler_qp_ctx *)qp)->order_ring;
	67	uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
	68	nb_ops);
	69	uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
	70	nb_ops_to_enq);
	71
	72	scheduler_order_insert(order_ring, ops, nb_ops_enqd);
	73
	74	return nb_ops_enqd;
	75	}
	76
	77	static uint16_t
	78	schedule_dequeue(void qp, struct rte_crypto_op *ops, uint16_t nb_ops)
	79	{
	80	struct fo_scheduler_qp_ctx *qp_ctx =
	81	((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
	82	struct scheduler_slave *slaves[NB_FAILOVER_SLAVES] = {
	83	&qp_ctx->primary_slave, &qp_ctx->secondary_slave};
	84	struct scheduler_slave *slave = slaves[qp_ctx->deq_idx];
	85	uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;
	86
	87	if (slave->nb_inflight_cops) {
	88	nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id,
	89	slave->qp_id, ops, nb_ops);
	90	slave->nb_inflight_cops -= nb_deq_ops;
	91	}
	92
	93	qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_SWITCH_MASK;
	94
	95	if (nb_deq_ops == nb_ops)
	96	return nb_deq_ops;
	97
	98	slave = slaves[qp_ctx->deq_idx];
	99
	100	if (slave->nb_inflight_cops) {
	101	nb_deq_ops2 = rte_cryptodev_dequeue_burst(slave->dev_id,
	102	slave->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
	103	slave->nb_inflight_cops -= nb_deq_ops2;
	104	}
	105
	106	return nb_deq_ops + nb_deq_ops2;
	107	}
	108
	109	static uint16_t
	110	schedule_dequeue_ordering(void qp, struct rte_crypto_op *ops,
	111	uint16_t nb_ops)
	112	{
	113	struct rte_ring *order_ring =
	114	((struct scheduler_qp_ctx *)qp)->order_ring;
	115
	116	schedule_dequeue(qp, ops, nb_ops);
	117
	118	return scheduler_order_drain(order_ring, ops, nb_ops);
119	}
120
121	static int
122	slave_attach(__rte_unused struct rte_cryptodev *dev,
123	__rte_unused uint8_t slave_id)
124	{
125	return 0;
126	}
127
128	static int
129	slave_detach(__rte_unused struct rte_cryptodev *dev,
130	__rte_unused uint8_t slave_id)
131	{
132	return 0;
133	}
134
135	static int
136	scheduler_start(struct rte_cryptodev *dev)
137	{
138	struct scheduler_ctx *sched_ctx = dev->data->dev_private;
139	uint16_t i;
140
141	if (sched_ctx->nb_slaves < 2) {
9f95a23c	142	CR_SCHED_LOG(ERR, "Number of slaves shall no less than 2");
11fdf7f2 TL	143	return -ENOMEM;
	144	}
	145
	146	if (sched_ctx->reordering_enabled) {
	147	dev->enqueue_burst = schedule_enqueue_ordering;
	148	dev->dequeue_burst = schedule_dequeue_ordering;
	149	} else {
	150	dev->enqueue_burst = schedule_enqueue;
	151	dev->dequeue_burst = schedule_dequeue;
	152	}
	153
	154	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
	155	struct fo_scheduler_qp_ctx *qp_ctx =
	156	((struct scheduler_qp_ctx *)
	157	dev->data->queue_pairs[i])->private_qp_ctx;
	158
	159	rte_memcpy(&qp_ctx->primary_slave,
	160	&sched_ctx->slaves[PRIMARY_SLAVE_IDX],
	161	sizeof(struct scheduler_slave));
	162	rte_memcpy(&qp_ctx->secondary_slave,
	163	&sched_ctx->slaves[SECONDARY_SLAVE_IDX],
	164	sizeof(struct scheduler_slave));
	165	}
	166
	167	return 0;
	168	}
	169
	170	static int
	171	scheduler_stop(__rte_unused struct rte_cryptodev *dev)
	172	{
	173	return 0;
	174	}
	175
	176	static int
	177	scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
	178	{
	179	struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
	180	struct fo_scheduler_qp_ctx *fo_qp_ctx;
	181
	182	fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
	183	rte_socket_id());
	184	if (!fo_qp_ctx) {
9f95a23c	185	CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
11fdf7f2 TL	186	return -ENOMEM;
	187	}
	188
	189	qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;
	190
	191	return 0;
	192	}
	193
	194	static int
	195	scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
	196	{
	197	return 0;
	198	}
	199
9f95a23c	200	static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
11fdf7f2 TL	201	slave_attach,
	202	slave_detach,
	203	scheduler_start,
	204	scheduler_stop,
	205	scheduler_config_qp,
	206	scheduler_create_private_ctx,
	207	NULL, /* option_set */
	208	NULL /option_get /
	209	};
	210
9f95a23c	211	static struct rte_cryptodev_scheduler fo_scheduler = {
11fdf7f2 TL	212	.name = "failover-scheduler",
	213	.description = "scheduler which enqueues to the primary slave, "
	214	"and only then enqueues to the secondary slave "
	215	"upon failing on enqueuing to primary",
	216	.mode = CDEV_SCHED_MODE_FAILOVER,
	217	.ops = &scheduler_fo_ops
	218	};
	219
9f95a23c	220	struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;