[ceph.git] / ceph / src / spdk / dpdk / drivers / raw / dpaa2_cmdif / dpaa2_cmdif.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2018 NXP
 */

#include <stdio.h>
#include <errno.h>
#include <stdint.h>

#include <rte_bus_vdev.h>
#include <rte_atomic.h>
#include <rte_interrupts.h>
#include <rte_branch_prediction.h>
#include <rte_lcore.h>

#include <rte_rawdev.h>
#include <rte_rawdev_pmd.h>

#include <portal/dpaa2_hw_pvt.h>
#include <portal/dpaa2_hw_dpio.h>
#include "dpaa2_cmdif_logs.h"
#include "rte_pmd_dpaa2_cmdif.h"

/* Dynamic log type identifier */
int dpaa2_cmdif_logtype;

/* CMDIF driver name */
#define DPAA2_CMDIF_PMD_NAME dpaa2_dpci

/* CMDIF driver object */
static struct rte_vdev_driver dpaa2_cmdif_drv;

/*
 * This API provides the DPCI device ID in 'attr_value'.
 * The device ID shall be passed by GPP to the AIOP using CMDIF commands.
 */
static int
dpaa2_cmdif_get_attr(struct rte_rawdev *dev,
		     const char *attr_name,
		     uint64_t *attr_value)
{
	struct dpaa2_dpci_dev *cidev = dev->dev_private;

	DPAA2_CMDIF_FUNC_TRACE();

	RTE_SET_USED(attr_name);

	if (!attr_value) {
		DPAA2_CMDIF_ERR("Invalid arguments for getting attributes");
		return -EINVAL;
	}
	*attr_value = cidev->dpci_id;

	return 0;
}

static int
dpaa2_cmdif_enqueue_bufs(struct rte_rawdev *dev,
			 struct rte_rawdev_buf **buffers,
			 unsigned int count,
			 rte_rawdev_obj_t context)
{
	struct dpaa2_dpci_dev *cidev = dev->dev_private;
	struct rte_dpaa2_cmdif_context *cmdif_send_cnxt;
	struct dpaa2_queue *txq;
	struct qbman_fd fd;
	struct qbman_eq_desc eqdesc;
	struct qbman_swp *swp;
	int ret;

	RTE_SET_USED(count);

	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
		ret = dpaa2_affine_qbman_swp();
		if (ret) {
			DPAA2_CMDIF_ERR("Failure in affining portal\n");
			return 0;
		}
	}
	swp = DPAA2_PER_LCORE_PORTAL;

	cmdif_send_cnxt = (struct rte_dpaa2_cmdif_context *)(context);
	txq = &(cidev->tx_queue[cmdif_send_cnxt->priority]);

	/* Prepare enqueue descriptor */
	qbman_eq_desc_clear(&eqdesc);
	qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
	qbman_eq_desc_set_no_orp(&eqdesc, 0);
	qbman_eq_desc_set_response(&eqdesc, 0, 0);

	/* Set some of the FD parameters to i.
	 * For performance reasons do not memset
	 */
	fd.simple.bpid_offset = 0;
	fd.simple.ctrl = 0;

	DPAA2_SET_FD_ADDR(&fd, DPAA2_VADDR_TO_IOVA(buffers[0]->buf_addr));
	DPAA2_SET_FD_LEN(&fd, cmdif_send_cnxt->size);
	DPAA2_SET_FD_FRC(&fd, cmdif_send_cnxt->frc);
	DPAA2_SET_FD_FLC(&fd, cmdif_send_cnxt->flc);

	/* Enqueue a packet to the QBMAN */
	do {
		ret = qbman_swp_enqueue_multiple(swp, &eqdesc, &fd, NULL, 1);
		if (ret < 0 && ret != -EBUSY)
			DPAA2_CMDIF_ERR("Transmit failure with err: %d\n", ret);
	} while (ret == -EBUSY);

	DPAA2_CMDIF_DP_DEBUG("Successfully transmitted a packet\n");

	return 0;
}

static int
dpaa2_cmdif_dequeue_bufs(struct rte_rawdev *dev,
			 struct rte_rawdev_buf **buffers,
			 unsigned int count,
			 rte_rawdev_obj_t context)
{
	struct dpaa2_dpci_dev *cidev = dev->dev_private;
	struct rte_dpaa2_cmdif_context *cmdif_rcv_cnxt;
	struct dpaa2_queue *rxq;
	struct qbman_swp *swp;
	struct qbman_result *dq_storage;
	const struct qbman_fd *fd;
	struct qbman_pull_desc pulldesc;
	uint8_t status;
	int ret;

	RTE_SET_USED(count);

	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
		ret = dpaa2_affine_qbman_swp();
		if (ret) {
			DPAA2_CMDIF_ERR("Failure in affining portal\n");
			return 0;
		}
	}
	swp = DPAA2_PER_LCORE_PORTAL;

	cmdif_rcv_cnxt = (struct rte_dpaa2_cmdif_context *)(context);
	rxq = &(cidev->rx_queue[cmdif_rcv_cnxt->priority]);
	dq_storage = rxq->q_storage->dq_storage[0];

	qbman_pull_desc_clear(&pulldesc);
	qbman_pull_desc_set_fq(&pulldesc, rxq->fqid);
	qbman_pull_desc_set_numframes(&pulldesc, 1);
	qbman_pull_desc_set_storage(&pulldesc, dq_storage,
		(uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);

	while (1) {
		if (qbman_swp_pull(swp, &pulldesc)) {
			DPAA2_CMDIF_DP_WARN("VDQ cmd not issued. QBMAN is busy\n");
			/* Portal was busy, try again */
			continue;
		}
		break;
	}

	/* Check if previous issued command is completed. */
	while (!qbman_check_command_complete(dq_storage))
		;
	/* Loop until the dq_storage is updated with new token by QBMAN */
	while (!qbman_result_has_new_result(swp, dq_storage))
		;

	/* Check for valid frame. */
	status = (uint8_t)qbman_result_DQ_flags(dq_storage);
	if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
		DPAA2_CMDIF_DP_DEBUG("No frame is delivered\n");
		return 0;
	}

	fd = qbman_result_DQ_fd(dq_storage);

	buffers[0]->buf_addr = (void *)DPAA2_IOVA_TO_VADDR(
			DPAA2_GET_FD_ADDR(fd) +	DPAA2_GET_FD_OFFSET(fd));
	cmdif_rcv_cnxt->size = DPAA2_GET_FD_LEN(fd);
	cmdif_rcv_cnxt->flc = DPAA2_GET_FD_FLC(fd);
	cmdif_rcv_cnxt->frc = DPAA2_GET_FD_FRC(fd);

	DPAA2_CMDIF_DP_DEBUG("packet received\n");

	return 1;
}

static const struct rte_rawdev_ops dpaa2_cmdif_ops = {
	.attr_get = dpaa2_cmdif_get_attr,
	.enqueue_bufs = dpaa2_cmdif_enqueue_bufs,
	.dequeue_bufs = dpaa2_cmdif_dequeue_bufs,
};

static int
dpaa2_cmdif_create(const char *name,
		   struct rte_vdev_device *vdev,
		   int socket_id)
{
	struct rte_rawdev *rawdev;
	struct dpaa2_dpci_dev *cidev;

	/* Allocate device structure */
	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpaa2_dpci_dev),
					 socket_id);
	if (!rawdev) {
		DPAA2_CMDIF_ERR("Unable to allocate rawdevice");
		return -EINVAL;
	}

	rawdev->dev_ops = &dpaa2_cmdif_ops;
	rawdev->device = &vdev->device;

	/* For secondary processes, the primary has done all the work */
	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
		return 0;

	cidev = rte_dpaa2_alloc_dpci_dev();
	if (!cidev) {
		DPAA2_CMDIF_ERR("Unable to allocate CI device");
		rte_rawdev_pmd_release(rawdev);
		return -ENODEV;
	}

	rawdev->dev_private = cidev;

	return 0;
}

static int
dpaa2_cmdif_destroy(const char *name)
{
	int ret;
	struct rte_rawdev *rdev;

	rdev = rte_rawdev_pmd_get_named_dev(name);
	if (!rdev) {
		DPAA2_CMDIF_ERR("Invalid device name (%s)", name);
		return -EINVAL;
	}

	/* The primary process will only free the DPCI device */
	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
		rte_dpaa2_free_dpci_dev(rdev->dev_private);

	ret = rte_rawdev_pmd_release(rdev);
	if (ret)
		DPAA2_CMDIF_DEBUG("Device cleanup failed");

	return 0;
}

static int
dpaa2_cmdif_probe(struct rte_vdev_device *vdev)
{
	const char *name;
	int ret = 0;

	name = rte_vdev_device_name(vdev);

	DPAA2_CMDIF_INFO("Init %s on NUMA node %d", name, rte_socket_id());

	ret = dpaa2_cmdif_create(name, vdev, rte_socket_id());

	return ret;
}

static int
dpaa2_cmdif_remove(struct rte_vdev_device *vdev)
{
	const char *name;
	int ret;

	name = rte_vdev_device_name(vdev);
	if (name == NULL)
		return -1;

	DPAA2_CMDIF_INFO("Closing %s on NUMA node %d", name, rte_socket_id());

	ret = dpaa2_cmdif_destroy(name);

	return ret;
}

static struct rte_vdev_driver dpaa2_cmdif_drv = {
	.probe = dpaa2_cmdif_probe,
	.remove = dpaa2_cmdif_remove
};

RTE_PMD_REGISTER_VDEV(DPAA2_CMDIF_PMD_NAME, dpaa2_cmdif_drv);

RTE_INIT(dpaa2_cmdif_init_log)
{
	dpaa2_cmdif_logtype = rte_log_register("pmd.raw.dpaa2.cmdif");
	if (dpaa2_cmdif_logtype >= 0)
		rte_log_set_level(dpaa2_cmdif_logtype, RTE_LOG_INFO);
}
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
	2	* Copyright 2018 NXP
	3	*/
	4
	5	#include <stdio.h>
	6	#include <errno.h>
	7	#include <stdint.h>
	8
	9	#include <rte_bus_vdev.h>
	10	#include <rte_atomic.h>
	11	#include <rte_interrupts.h>
	12	#include <rte_branch_prediction.h>
	13	#include <rte_lcore.h>
	14
	15	#include <rte_rawdev.h>
	16	#include <rte_rawdev_pmd.h>
	17
	18	#include <portal/dpaa2_hw_pvt.h>
	19	#include <portal/dpaa2_hw_dpio.h>
	20	#include "dpaa2_cmdif_logs.h"
	21	#include "rte_pmd_dpaa2_cmdif.h"
	22
	23	/* Dynamic log type identifier */
	24	int dpaa2_cmdif_logtype;
	25
	26	/* CMDIF driver name */
	27	#define DPAA2_CMDIF_PMD_NAME dpaa2_dpci
	28
	29	/* CMDIF driver object */
	30	static struct rte_vdev_driver dpaa2_cmdif_drv;
	31
	32	/*
	33	* This API provides the DPCI device ID in 'attr_value'.
	34	* The device ID shall be passed by GPP to the AIOP using CMDIF commands.
	35	*/
	36	static int
	37	dpaa2_cmdif_get_attr(struct rte_rawdev *dev,
	38	const char *attr_name,
	39	uint64_t *attr_value)
	40	{
	41	struct dpaa2_dpci_dev *cidev = dev->dev_private;
	42
	43	DPAA2_CMDIF_FUNC_TRACE();
	44
	45	RTE_SET_USED(attr_name);
	46
	47	if (!attr_value) {
	48	DPAA2_CMDIF_ERR("Invalid arguments for getting attributes");
	49	return -EINVAL;
	50	}
	51	*attr_value = cidev->dpci_id;
	52
	53	return 0;
	54	}
	55
	56	static int
	57	dpaa2_cmdif_enqueue_bufs(struct rte_rawdev *dev,
	58	struct rte_rawdev_buf **buffers,
	59	unsigned int count,
	60	rte_rawdev_obj_t context)
	61	{
	62	struct dpaa2_dpci_dev *cidev = dev->dev_private;
	63	struct rte_dpaa2_cmdif_context *cmdif_send_cnxt;
	64	struct dpaa2_queue *txq;
65	struct qbman_fd fd;
66	struct qbman_eq_desc eqdesc;
67	struct qbman_swp *swp;
68	int ret;
69
11fdf7f2 TL	70	RTE_SET_USED(count);
	71
	72	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
	73	ret = dpaa2_affine_qbman_swp();
	74	if (ret) {
	75	DPAA2_CMDIF_ERR("Failure in affining portal\n");
	76	return 0;
	77	}
	78	}
	79	swp = DPAA2_PER_LCORE_PORTAL;
	80
	81	cmdif_send_cnxt = (struct rte_dpaa2_cmdif_context *)(context);
	82	txq = &(cidev->tx_queue[cmdif_send_cnxt->priority]);
	83
	84	/* Prepare enqueue descriptor */
	85	qbman_eq_desc_clear(&eqdesc);
	86	qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
	87	qbman_eq_desc_set_no_orp(&eqdesc, 0);
	88	qbman_eq_desc_set_response(&eqdesc, 0, 0);
	89
	90	/* Set some of the FD parameters to i.
	91	* For performance reasons do not memset
	92	*/
	93	fd.simple.bpid_offset = 0;
	94	fd.simple.ctrl = 0;
	95
	96	DPAA2_SET_FD_ADDR(&fd, DPAA2_VADDR_TO_IOVA(buffers[0]->buf_addr));
	97	DPAA2_SET_FD_LEN(&fd, cmdif_send_cnxt->size);
	98	DPAA2_SET_FD_FRC(&fd, cmdif_send_cnxt->frc);
	99	DPAA2_SET_FD_FLC(&fd, cmdif_send_cnxt->flc);
	100
	101	/* Enqueue a packet to the QBMAN */
	102	do {
	103	ret = qbman_swp_enqueue_multiple(swp, &eqdesc, &fd, NULL, 1);
	104	if (ret < 0 && ret != -EBUSY)
	105	DPAA2_CMDIF_ERR("Transmit failure with err: %d\n", ret);
	106	} while (ret == -EBUSY);
	107
	108	DPAA2_CMDIF_DP_DEBUG("Successfully transmitted a packet\n");
	109
	110	return 0;
	111	}
	112
	113	static int
	114	dpaa2_cmdif_dequeue_bufs(struct rte_rawdev *dev,
	115	struct rte_rawdev_buf **buffers,
	116	unsigned int count,
	117	rte_rawdev_obj_t context)
	118	{
	119	struct dpaa2_dpci_dev *cidev = dev->dev_private;
	120	struct rte_dpaa2_cmdif_context *cmdif_rcv_cnxt;
	121	struct dpaa2_queue *rxq;
	122	struct qbman_swp *swp;
	123	struct qbman_result *dq_storage;
	124	const struct qbman_fd *fd;
	125	struct qbman_pull_desc pulldesc;
	126	uint8_t status;
	127	int ret;
	128
11fdf7f2 TL	129	RTE_SET_USED(count);
	130
	131	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
	132	ret = dpaa2_affine_qbman_swp();
	133	if (ret) {
	134	DPAA2_CMDIF_ERR("Failure in affining portal\n");
	135	return 0;
	136	}
	137	}
	138	swp = DPAA2_PER_LCORE_PORTAL;
	139
	140	cmdif_rcv_cnxt = (struct rte_dpaa2_cmdif_context *)(context);
	141	rxq = &(cidev->rx_queue[cmdif_rcv_cnxt->priority]);
	142	dq_storage = rxq->q_storage->dq_storage[0];
	143
	144	qbman_pull_desc_clear(&pulldesc);
	145	qbman_pull_desc_set_fq(&pulldesc, rxq->fqid);
	146	qbman_pull_desc_set_numframes(&pulldesc, 1);
	147	qbman_pull_desc_set_storage(&pulldesc, dq_storage,
	148	(uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
	149
	150	while (1) {
	151	if (qbman_swp_pull(swp, &pulldesc)) {
	152	DPAA2_CMDIF_DP_WARN("VDQ cmd not issued. QBMAN is busy\n");
	153	/* Portal was busy, try again */
	154	continue;
	155	}
	156	break;
	157	}
	158
	159	/* Check if previous issued command is completed. */
	160	while (!qbman_check_command_complete(dq_storage))
	161	;
	162	/* Loop until the dq_storage is updated with new token by QBMAN */
	163	while (!qbman_result_has_new_result(swp, dq_storage))
	164	;
	165
	166	/* Check for valid frame. */
	167	status = (uint8_t)qbman_result_DQ_flags(dq_storage);
	168	if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
	169	DPAA2_CMDIF_DP_DEBUG("No frame is delivered\n");
	170	return 0;
	171	}
	172
	173	fd = qbman_result_DQ_fd(dq_storage);
	174
	175	buffers[0]->buf_addr = (void *)DPAA2_IOVA_TO_VADDR(
	176	DPAA2_GET_FD_ADDR(fd) + DPAA2_GET_FD_OFFSET(fd));
	177	cmdif_rcv_cnxt->size = DPAA2_GET_FD_LEN(fd);
	178	cmdif_rcv_cnxt->flc = DPAA2_GET_FD_FLC(fd);
	179	cmdif_rcv_cnxt->frc = DPAA2_GET_FD_FRC(fd);
	180
	181	DPAA2_CMDIF_DP_DEBUG("packet received\n");
	182
	183	return 1;
	184	}
	185
	186	static const struct rte_rawdev_ops dpaa2_cmdif_ops = {
	187	.attr_get = dpaa2_cmdif_get_attr,
	188	.enqueue_bufs = dpaa2_cmdif_enqueue_bufs,
	189	.dequeue_bufs = dpaa2_cmdif_dequeue_bufs,
	190	};
	191
	192	static int
193	dpaa2_cmdif_create(const char *name,
194	struct rte_vdev_device *vdev,
195	int socket_id)
196	{
197	struct rte_rawdev *rawdev;
198	struct dpaa2_dpci_dev *cidev;
199
200	/* Allocate device structure */
201	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpaa2_dpci_dev),
202	socket_id);
203	if (!rawdev) {
204	DPAA2_CMDIF_ERR("Unable to allocate rawdevice");
205	return -EINVAL;
206	}
207
208	rawdev->dev_ops = &dpaa2_cmdif_ops;
209	rawdev->device = &vdev->device;
11fdf7f2 TL	210
	211	/* For secondary processes, the primary has done all the work */
	212	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
	213	return 0;
	214
	215	cidev = rte_dpaa2_alloc_dpci_dev();
	216	if (!cidev) {
	217	DPAA2_CMDIF_ERR("Unable to allocate CI device");
	218	rte_rawdev_pmd_release(rawdev);
	219	return -ENODEV;
	220	}
	221
	222	rawdev->dev_private = cidev;
	223
	224	return 0;
	225	}
	226
	227	static int
	228	dpaa2_cmdif_destroy(const char *name)
	229	{
	230	int ret;
	231	struct rte_rawdev *rdev;
	232
	233	rdev = rte_rawdev_pmd_get_named_dev(name);
	234	if (!rdev) {
	235	DPAA2_CMDIF_ERR("Invalid device name (%s)", name);
	236	return -EINVAL;
	237	}
	238
	239	/* The primary process will only free the DPCI device */
	240	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
	241	rte_dpaa2_free_dpci_dev(rdev->dev_private);
	242
	243	ret = rte_rawdev_pmd_release(rdev);
	244	if (ret)
	245	DPAA2_CMDIF_DEBUG("Device cleanup failed");
	246
	247	return 0;
	248	}
	249
	250	static int
	251	dpaa2_cmdif_probe(struct rte_vdev_device *vdev)
	252	{
	253	const char *name;
	254	int ret = 0;
	255
	256	name = rte_vdev_device_name(vdev);
	257
	258	DPAA2_CMDIF_INFO("Init %s on NUMA node %d", name, rte_socket_id());
	259
	260	ret = dpaa2_cmdif_create(name, vdev, rte_socket_id());
	261
	262	return ret;
	263	}
	264
	265	static int
	266	dpaa2_cmdif_remove(struct rte_vdev_device *vdev)
	267	{
	268	const char *name;
	269	int ret;
	270
	271	name = rte_vdev_device_name(vdev);
9f95a23c TL	272	if (name == NULL)
9f95a23c TL	273	return -1;
11fdf7f2 TL	274
	275	DPAA2_CMDIF_INFO("Closing %s on NUMA node %d", name, rte_socket_id());
	276
	277	ret = dpaa2_cmdif_destroy(name);
	278
	279	return ret;
	280	}
	281
	282	static struct rte_vdev_driver dpaa2_cmdif_drv = {
	283	.probe = dpaa2_cmdif_probe,
	284	.remove = dpaa2_cmdif_remove
	285	};
	286
	287	RTE_PMD_REGISTER_VDEV(DPAA2_CMDIF_PMD_NAME, dpaa2_cmdif_drv);
	288
	289	RTE_INIT(dpaa2_cmdif_init_log)
	290	{
	291	dpaa2_cmdif_logtype = rte_log_register("pmd.raw.dpaa2.cmdif");
	292	if (dpaa2_cmdif_logtype >= 0)
	293	rte_log_set_level(dpaa2_cmdif_logtype, RTE_LOG_INFO);
	294	}