[ceph.git] / ceph / src / spdk / dpdk / drivers / net / sfc / sfc_intr.c

/* SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright(c) 2019-2020 Xilinx, Inc.
 * Copyright(c) 2016-2019 Solarflare Communications Inc.
 *
 * This software was jointly developed between OKTET Labs (under contract
 * for Solarflare) and Solarflare Communications, Inc.
 */

/*
 * At the momemt of writing DPDK v16.07 has notion of two types of
 * interrupts: LSC (link status change) and RXQ (receive indication).
 * It allows to register interrupt callback for entire device which is
 * not intended to be used for receive indication (i.e. link status
 * change indication only). The handler has no information which HW
 * interrupt has triggered it, so we don't know which event queue should
 * be polled/reprimed (except qmask in the case of legacy line interrupt).
 */

#include <rte_common.h>
#include <rte_interrupts.h>

#include "efx.h"

#include "sfc.h"
#include "sfc_log.h"
#include "sfc_ev.h"

static void
sfc_intr_handle_mgmt_evq(struct sfc_adapter *sa)
{
	struct sfc_evq *evq;

	rte_spinlock_lock(&sa->mgmt_evq_lock);

	evq = sa->mgmt_evq;

	if (!sa->mgmt_evq_running) {
		sfc_log_init(sa, "interrupt on not running management EVQ %u",
			     evq->evq_index);
	} else {
		sfc_ev_qpoll(evq);

		if (sfc_ev_qprime(evq) != 0)
			sfc_err(sa, "cannot prime EVQ %u", evq->evq_index);
	}

	rte_spinlock_unlock(&sa->mgmt_evq_lock);
}

static void
sfc_intr_line_handler(void *cb_arg)
{
	struct sfc_adapter *sa = (struct sfc_adapter *)cb_arg;
	efx_nic_t *enp = sa->nic;
	boolean_t fatal;
	uint32_t qmask;
	unsigned int lsc_seq = sa->port.lsc_seq;
	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);

	sfc_log_init(sa, "entry");

	if (sa->state != SFC_ADAPTER_STARTED &&
	    sa->state != SFC_ADAPTER_STARTING &&
	    sa->state != SFC_ADAPTER_STOPPING) {
		sfc_log_init(sa,
			     "interrupt on stopped adapter, don't reenable");
		goto exit;
	}

	efx_intr_status_line(enp, &fatal, &qmask);
	if (fatal) {
		(void)efx_intr_disable(enp);
		(void)efx_intr_fatal(enp);
		sfc_err(sa, "fatal, interrupts disabled");
		goto exit;
	}

	if (qmask & (1 << sa->mgmt_evq_index))
		sfc_intr_handle_mgmt_evq(sa);

	if (rte_intr_ack(&pci_dev->intr_handle) != 0)
		sfc_err(sa, "cannot reenable interrupts");

	sfc_log_init(sa, "done");

exit:
	if (lsc_seq != sa->port.lsc_seq) {
		sfc_notice(sa, "link status change event: link %s",
			 sa->eth_dev->data->dev_link.link_status ?
			 "UP" : "DOWN");
		_rte_eth_dev_callback_process(sa->eth_dev,
					      RTE_ETH_EVENT_INTR_LSC,
					      NULL);
	}
}

static void
sfc_intr_message_handler(void *cb_arg)
{
	struct sfc_adapter *sa = (struct sfc_adapter *)cb_arg;
	efx_nic_t *enp = sa->nic;
	boolean_t fatal;
	unsigned int lsc_seq = sa->port.lsc_seq;
	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);

	sfc_log_init(sa, "entry");

	if (sa->state != SFC_ADAPTER_STARTED &&
	    sa->state != SFC_ADAPTER_STARTING &&
	    sa->state != SFC_ADAPTER_STOPPING) {
		sfc_log_init(sa, "adapter not-started, don't reenable");
		goto exit;
	}

	efx_intr_status_message(enp, sa->mgmt_evq_index, &fatal);
	if (fatal) {
		(void)efx_intr_disable(enp);
		(void)efx_intr_fatal(enp);
		sfc_err(sa, "fatal, interrupts disabled");
		goto exit;
	}

	sfc_intr_handle_mgmt_evq(sa);

	if (rte_intr_ack(&pci_dev->intr_handle) != 0)
		sfc_err(sa, "cannot reenable interrupts");

	sfc_log_init(sa, "done");

exit:
	if (lsc_seq != sa->port.lsc_seq) {
		sfc_notice(sa, "link status change event");
		_rte_eth_dev_callback_process(sa->eth_dev,
					      RTE_ETH_EVENT_INTR_LSC,
					      NULL);
	}
}

int
sfc_intr_start(struct sfc_adapter *sa)
{
	struct sfc_intr *intr = &sa->intr;
	struct rte_intr_handle *intr_handle;
	struct rte_pci_device *pci_dev;
	int rc;

	sfc_log_init(sa, "entry");

	/*
	 * The EFX common code event queue module depends on the interrupt
	 * module. Ensure that the interrupt module is always initialized
	 * (even if interrupts are not used).  Status memory is required
	 * for Siena only and may be NULL for EF10.
	 */
	sfc_log_init(sa, "efx_intr_init");
	rc = efx_intr_init(sa->nic, intr->type, NULL);
	if (rc != 0)
		goto fail_intr_init;

	pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	intr_handle = &pci_dev->intr_handle;

	if (intr->handler != NULL) {
		if (intr->rxq_intr && rte_intr_cap_multiple(intr_handle)) {
			uint32_t intr_vector;

			intr_vector = sa->eth_dev->data->nb_rx_queues;
			rc = rte_intr_efd_enable(intr_handle, intr_vector);
			if (rc != 0)
				goto fail_rte_intr_efd_enable;
		}
		if (rte_intr_dp_is_en(intr_handle)) {
			intr_handle->intr_vec =
				rte_calloc("intr_vec",
				sa->eth_dev->data->nb_rx_queues, sizeof(int),
				0);
			if (intr_handle->intr_vec == NULL) {
				sfc_err(sa,
					"Failed to allocate %d rx_queues intr_vec",
					sa->eth_dev->data->nb_rx_queues);
				goto fail_intr_vector_alloc;
			}
		}

		sfc_log_init(sa, "rte_intr_callback_register");
		rc = rte_intr_callback_register(intr_handle, intr->handler,
						(void *)sa);
		if (rc != 0) {
			sfc_err(sa,
				"cannot register interrupt handler (rc=%d)",
				rc);
			/*
			 * Convert error code from negative returned by RTE API
			 * to positive used in the driver.
			 */
			rc = -rc;
			goto fail_rte_intr_cb_reg;
		}

		sfc_log_init(sa, "rte_intr_enable");
		rc = rte_intr_enable(intr_handle);
		if (rc != 0) {
			sfc_err(sa, "cannot enable interrupts (rc=%d)", rc);
			/*
			 * Convert error code from negative returned by RTE API
			 * to positive used in the driver.
			 */
			rc = -rc;
			goto fail_rte_intr_enable;
		}

		sfc_log_init(sa, "efx_intr_enable");
		efx_intr_enable(sa->nic);
	}

	sfc_log_init(sa, "done type=%u max_intr=%d nb_efd=%u vec=%p",
		     intr_handle->type, intr_handle->max_intr,
		     intr_handle->nb_efd, intr_handle->intr_vec);
	return 0;

fail_rte_intr_enable:
	rte_intr_callback_unregister(intr_handle, intr->handler, (void *)sa);

fail_rte_intr_cb_reg:
	rte_free(intr_handle->intr_vec);

fail_intr_vector_alloc:
	rte_intr_efd_disable(intr_handle);

fail_rte_intr_efd_enable:
	efx_intr_fini(sa->nic);

fail_intr_init:
	sfc_log_init(sa, "failed %d", rc);
	return rc;
}

void
sfc_intr_stop(struct sfc_adapter *sa)
{
	struct sfc_intr *intr = &sa->intr;
	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);

	sfc_log_init(sa, "entry");

	if (intr->handler != NULL) {
		struct rte_intr_handle *intr_handle;
		int rc;

		efx_intr_disable(sa->nic);

		intr_handle = &pci_dev->intr_handle;

		rte_free(intr_handle->intr_vec);
		rte_intr_efd_disable(intr_handle);

		if (rte_intr_disable(intr_handle) != 0)
			sfc_err(sa, "cannot disable interrupts");

		while ((rc = rte_intr_callback_unregister(intr_handle,
				intr->handler, (void *)sa)) == -EAGAIN)
			;
		if (rc != 1)
			sfc_err(sa,
				"cannot unregister interrupt handler %d",
				rc);
	}

	efx_intr_fini(sa->nic);

	sfc_log_init(sa, "done");
}

int
sfc_intr_configure(struct sfc_adapter *sa)
{
	struct sfc_intr *intr = &sa->intr;

	sfc_log_init(sa, "entry");

	intr->handler = NULL;
	intr->lsc_intr = (sa->eth_dev->data->dev_conf.intr_conf.lsc != 0);
	intr->rxq_intr = (sa->eth_dev->data->dev_conf.intr_conf.rxq != 0);

	if (!intr->lsc_intr && !intr->rxq_intr)
		goto done;

	switch (intr->type) {
	case EFX_INTR_MESSAGE:
		intr->handler = sfc_intr_message_handler;
		break;
	case EFX_INTR_LINE:
		intr->handler = sfc_intr_line_handler;
		break;
	case EFX_INTR_INVALID:
		sfc_warn(sa, "interrupts are not supported");
		break;
	default:
		sfc_panic(sa, "unexpected EFX interrupt type %u\n", intr->type);
		break;
	}

done:
	sfc_log_init(sa, "done");
	return 0;
}

void
sfc_intr_close(struct sfc_adapter *sa)
{
	sfc_log_init(sa, "entry");

	sfc_log_init(sa, "done");
}

int
sfc_intr_attach(struct sfc_adapter *sa)
{
	struct sfc_intr *intr = &sa->intr;
	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);

	sfc_log_init(sa, "entry");

	switch (pci_dev->intr_handle.type) {
#ifdef RTE_EXEC_ENV_LINUX
	case RTE_INTR_HANDLE_UIO_INTX:
	case RTE_INTR_HANDLE_VFIO_LEGACY:
		intr->type = EFX_INTR_LINE;
		break;
	case RTE_INTR_HANDLE_UIO:
	case RTE_INTR_HANDLE_VFIO_MSI:
	case RTE_INTR_HANDLE_VFIO_MSIX:
		intr->type = EFX_INTR_MESSAGE;
		break;
#endif
	default:
		intr->type = EFX_INTR_INVALID;
		break;
	}

	sfc_log_init(sa, "done");
	return 0;
}

void
sfc_intr_detach(struct sfc_adapter *sa)
{
	sfc_log_init(sa, "entry");

	sa->intr.type = EFX_INTR_INVALID;

	sfc_log_init(sa, "done");
}
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
11fdf7f2 TL	2	*
f67539c2 TL	3	* Copyright(c) 2019-2020 Xilinx, Inc.
f67539c2 TL	4	* Copyright(c) 2016-2019 Solarflare Communications Inc.
11fdf7f2 TL	5	*
	6	* This software was jointly developed between OKTET Labs (under contract
	7	* for Solarflare) and Solarflare Communications, Inc.
	8	*/
	9
	10	/*
	11	* At the momemt of writing DPDK v16.07 has notion of two types of
	12	* interrupts: LSC (link status change) and RXQ (receive indication).
	13	* It allows to register interrupt callback for entire device which is
	14	* not intended to be used for receive indication (i.e. link status
	15	* change indication only). The handler has no information which HW
	16	* interrupt has triggered it, so we don't know which event queue should
	17	* be polled/reprimed (except qmask in the case of legacy line interrupt).
	18	*/
	19
	20	#include <rte_common.h>
	21	#include <rte_interrupts.h>
	22
	23	#include "efx.h"
	24
	25	#include "sfc.h"
	26	#include "sfc_log.h"
	27	#include "sfc_ev.h"
	28
	29	static void
	30	sfc_intr_handle_mgmt_evq(struct sfc_adapter *sa)
	31	{
	32	struct sfc_evq *evq;
	33
	34	rte_spinlock_lock(&sa->mgmt_evq_lock);
	35
	36	evq = sa->mgmt_evq;
	37
	38	if (!sa->mgmt_evq_running) {
	39	sfc_log_init(sa, "interrupt on not running management EVQ %u",
	40	evq->evq_index);
	41	} else {
	42	sfc_ev_qpoll(evq);
	43
	44	if (sfc_ev_qprime(evq) != 0)
	45	sfc_err(sa, "cannot prime EVQ %u", evq->evq_index);
	46	}
	47
	48	rte_spinlock_unlock(&sa->mgmt_evq_lock);
	49	}
	50
	51	static void
	52	sfc_intr_line_handler(void *cb_arg)
	53	{
	54	struct sfc_adapter sa = (struct sfc_adapter )cb_arg;
	55	efx_nic_t *enp = sa->nic;
	56	boolean_t fatal;
	57	uint32_t qmask;
	58	unsigned int lsc_seq = sa->port.lsc_seq;
	59	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	60
	61	sfc_log_init(sa, "entry");
	62
	63	if (sa->state != SFC_ADAPTER_STARTED &&
	64	sa->state != SFC_ADAPTER_STARTING &&
	65	sa->state != SFC_ADAPTER_STOPPING) {
	66	sfc_log_init(sa,
	67	"interrupt on stopped adapter, don't reenable");
	68	goto exit;
69	}
70
71	efx_intr_status_line(enp, &fatal, &qmask);
72	if (fatal) {
73	(void)efx_intr_disable(enp);
74	(void)efx_intr_fatal(enp);
75	sfc_err(sa, "fatal, interrupts disabled");
76	goto exit;
77	}
78
79	if (qmask & (1 << sa->mgmt_evq_index))
80	sfc_intr_handle_mgmt_evq(sa);
81
f67539c2	82	if (rte_intr_ack(&pci_dev->intr_handle) != 0)
11fdf7f2 TL	83	sfc_err(sa, "cannot reenable interrupts");
	84
	85	sfc_log_init(sa, "done");
	86
	87	exit:
	88	if (lsc_seq != sa->port.lsc_seq) {
	89	sfc_notice(sa, "link status change event: link %s",
	90	sa->eth_dev->data->dev_link.link_status ?
	91	"UP" : "DOWN");
	92	_rte_eth_dev_callback_process(sa->eth_dev,
	93	RTE_ETH_EVENT_INTR_LSC,
	94	NULL);
	95	}
	96	}
	97
	98	static void
	99	sfc_intr_message_handler(void *cb_arg)
	100	{
	101	struct sfc_adapter sa = (struct sfc_adapter )cb_arg;
	102	efx_nic_t *enp = sa->nic;
	103	boolean_t fatal;
	104	unsigned int lsc_seq = sa->port.lsc_seq;
	105	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	106
	107	sfc_log_init(sa, "entry");
	108
	109	if (sa->state != SFC_ADAPTER_STARTED &&
	110	sa->state != SFC_ADAPTER_STARTING &&
	111	sa->state != SFC_ADAPTER_STOPPING) {
	112	sfc_log_init(sa, "adapter not-started, don't reenable");
	113	goto exit;
	114	}
	115
	116	efx_intr_status_message(enp, sa->mgmt_evq_index, &fatal);
	117	if (fatal) {
	118	(void)efx_intr_disable(enp);
	119	(void)efx_intr_fatal(enp);
	120	sfc_err(sa, "fatal, interrupts disabled");
	121	goto exit;
	122	}
	123
	124	sfc_intr_handle_mgmt_evq(sa);
	125
f67539c2	126	if (rte_intr_ack(&pci_dev->intr_handle) != 0)
11fdf7f2 TL	127	sfc_err(sa, "cannot reenable interrupts");
	128
	129	sfc_log_init(sa, "done");
	130
	131	exit:
	132	if (lsc_seq != sa->port.lsc_seq) {
	133	sfc_notice(sa, "link status change event");
	134	_rte_eth_dev_callback_process(sa->eth_dev,
	135	RTE_ETH_EVENT_INTR_LSC,
	136	NULL);
	137	}
	138	}
	139
	140	int
	141	sfc_intr_start(struct sfc_adapter *sa)
	142	{
	143	struct sfc_intr *intr = &sa->intr;
	144	struct rte_intr_handle *intr_handle;
	145	struct rte_pci_device *pci_dev;
	146	int rc;
	147
	148	sfc_log_init(sa, "entry");
	149
	150	/*
	151	* The EFX common code event queue module depends on the interrupt
	152	* module. Ensure that the interrupt module is always initialized
	153	* (even if interrupts are not used). Status memory is required
	154	* for Siena only and may be NULL for EF10.
	155	*/
	156	sfc_log_init(sa, "efx_intr_init");
	157	rc = efx_intr_init(sa->nic, intr->type, NULL);
	158	if (rc != 0)
	159	goto fail_intr_init;
	160
	161	pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	162	intr_handle = &pci_dev->intr_handle;
	163
	164	if (intr->handler != NULL) {
f67539c2 TL	165	if (intr->rxq_intr && rte_intr_cap_multiple(intr_handle)) {
	166	uint32_t intr_vector;
	167
	168	intr_vector = sa->eth_dev->data->nb_rx_queues;
	169	rc = rte_intr_efd_enable(intr_handle, intr_vector);
	170	if (rc != 0)
	171	goto fail_rte_intr_efd_enable;
	172	}
	173	if (rte_intr_dp_is_en(intr_handle)) {
	174	intr_handle->intr_vec =
	175	rte_calloc("intr_vec",
	176	sa->eth_dev->data->nb_rx_queues, sizeof(int),
	177	0);
	178	if (intr_handle->intr_vec == NULL) {
	179	sfc_err(sa,
	180	"Failed to allocate %d rx_queues intr_vec",
	181	sa->eth_dev->data->nb_rx_queues);
	182	goto fail_intr_vector_alloc;
	183	}
	184	}
	185
11fdf7f2 TL	186	sfc_log_init(sa, "rte_intr_callback_register");
	187	rc = rte_intr_callback_register(intr_handle, intr->handler,
	188	(void *)sa);
	189	if (rc != 0) {
	190	sfc_err(sa,
	191	"cannot register interrupt handler (rc=%d)",
	192	rc);
	193	/*
	194	* Convert error code from negative returned by RTE API
	195	* to positive used in the driver.
	196	*/
	197	rc = -rc;
	198	goto fail_rte_intr_cb_reg;
	199	}
	200
	201	sfc_log_init(sa, "rte_intr_enable");
	202	rc = rte_intr_enable(intr_handle);
	203	if (rc != 0) {
	204	sfc_err(sa, "cannot enable interrupts (rc=%d)", rc);
	205	/*
	206	* Convert error code from negative returned by RTE API
	207	* to positive used in the driver.
	208	*/
	209	rc = -rc;
	210	goto fail_rte_intr_enable;
	211	}
	212
	213	sfc_log_init(sa, "efx_intr_enable");
	214	efx_intr_enable(sa->nic);
	215	}
	216
	217	sfc_log_init(sa, "done type=%u max_intr=%d nb_efd=%u vec=%p",
	218	intr_handle->type, intr_handle->max_intr,
	219	intr_handle->nb_efd, intr_handle->intr_vec);
	220	return 0;
	221
	222	fail_rte_intr_enable:
	223	rte_intr_callback_unregister(intr_handle, intr->handler, (void *)sa);
	224
	225	fail_rte_intr_cb_reg:
f67539c2 TL	226	rte_free(intr_handle->intr_vec);
	227
	228	fail_intr_vector_alloc:
	229	rte_intr_efd_disable(intr_handle);
	230
	231	fail_rte_intr_efd_enable:
11fdf7f2 TL	232	efx_intr_fini(sa->nic);
	233
	234	fail_intr_init:
	235	sfc_log_init(sa, "failed %d", rc);
	236	return rc;
	237	}
	238
	239	void
	240	sfc_intr_stop(struct sfc_adapter *sa)
	241	{
	242	struct sfc_intr *intr = &sa->intr;
	243	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	244
	245	sfc_log_init(sa, "entry");
	246
	247	if (intr->handler != NULL) {
	248	struct rte_intr_handle *intr_handle;
	249	int rc;
	250
	251	efx_intr_disable(sa->nic);
	252
	253	intr_handle = &pci_dev->intr_handle;
f67539c2 TL	254
	255	rte_free(intr_handle->intr_vec);
	256	rte_intr_efd_disable(intr_handle);
	257
11fdf7f2 TL	258	if (rte_intr_disable(intr_handle) != 0)
	259	sfc_err(sa, "cannot disable interrupts");
	260
	261	while ((rc = rte_intr_callback_unregister(intr_handle,
	262	intr->handler, (void *)sa)) == -EAGAIN)
	263	;
	264	if (rc != 1)
	265	sfc_err(sa,
	266	"cannot unregister interrupt handler %d",
	267	rc);
	268	}
	269
	270	efx_intr_fini(sa->nic);
	271
	272	sfc_log_init(sa, "done");
	273	}
	274
	275	int
	276	sfc_intr_configure(struct sfc_adapter *sa)
	277	{
	278	struct sfc_intr *intr = &sa->intr;
	279
	280	sfc_log_init(sa, "entry");
	281
	282	intr->handler = NULL;
	283	intr->lsc_intr = (sa->eth_dev->data->dev_conf.intr_conf.lsc != 0);
f67539c2 TL	284	intr->rxq_intr = (sa->eth_dev->data->dev_conf.intr_conf.rxq != 0);
	285
	286	if (!intr->lsc_intr && !intr->rxq_intr)
11fdf7f2	287	goto done;
11fdf7f2 TL	288
	289	switch (intr->type) {
	290	case EFX_INTR_MESSAGE:
	291	intr->handler = sfc_intr_message_handler;
	292	break;
	293	case EFX_INTR_LINE:
	294	intr->handler = sfc_intr_line_handler;
	295	break;
	296	case EFX_INTR_INVALID:
	297	sfc_warn(sa, "interrupts are not supported");
	298	break;
	299	default:
	300	sfc_panic(sa, "unexpected EFX interrupt type %u\n", intr->type);
	301	break;
	302	}
	303
	304	done:
	305	sfc_log_init(sa, "done");
	306	return 0;
	307	}
	308
	309	void
	310	sfc_intr_close(struct sfc_adapter *sa)
	311	{
	312	sfc_log_init(sa, "entry");
	313
	314	sfc_log_init(sa, "done");
	315	}
	316
	317	int
	318	sfc_intr_attach(struct sfc_adapter *sa)
	319	{
	320	struct sfc_intr *intr = &sa->intr;
	321	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
	322
	323	sfc_log_init(sa, "entry");
	324
	325	switch (pci_dev->intr_handle.type) {
9f95a23c	326	#ifdef RTE_EXEC_ENV_LINUX
11fdf7f2 TL	327	case RTE_INTR_HANDLE_UIO_INTX:
	328	case RTE_INTR_HANDLE_VFIO_LEGACY:
	329	intr->type = EFX_INTR_LINE;
	330	break;
	331	case RTE_INTR_HANDLE_UIO:
	332	case RTE_INTR_HANDLE_VFIO_MSI:
	333	case RTE_INTR_HANDLE_VFIO_MSIX:
	334	intr->type = EFX_INTR_MESSAGE;
	335	break;
	336	#endif
	337	default:
	338	intr->type = EFX_INTR_INVALID;
	339	break;
	340	}
	341
	342	sfc_log_init(sa, "done");
	343	return 0;
	344	}
	345
	346	void
	347	sfc_intr_detach(struct sfc_adapter *sa)
	348	{
	349	sfc_log_init(sa, "entry");
	350
	351	sa->intr.type = EFX_INTR_INVALID;
	352
	353	sfc_log_init(sa, "done");
	354	}