[ceph.git] / ceph / src / seastar / dpdk / drivers / net / sfc / base / ef10_tx.c

/* SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (c) 2012-2018 Solarflare Communications Inc.
 * All rights reserved.
 */

#include "efx.h"
#include "efx_impl.h"


#if EFX_OPTS_EF10()

#if EFSYS_OPT_QSTATS
#define	EFX_TX_QSTAT_INCR(_etp, _stat)					\
	do {								\
		(_etp)->et_stat[_stat]++;				\
	_NOTE(CONSTANTCONDITION)					\
	} while (B_FALSE)
#else
#define	EFX_TX_QSTAT_INCR(_etp, _stat)
#endif

static	__checkReturn	efx_rc_t
efx_mcdi_init_txq(
	__in		efx_nic_t *enp,
	__in		uint32_t ndescs,
	__in		uint32_t target_evq,
	__in		uint32_t label,
	__in		uint32_t instance,
	__in		uint16_t flags,
	__in		efsys_mem_t *esmp)
{
	efx_mcdi_req_t req;
	EFX_MCDI_DECLARE_BUF(payload,
		MC_CMD_INIT_TXQ_IN_LEN(EF10_TXQ_MAXNBUFS),
		MC_CMD_INIT_TXQ_OUT_LEN);
	efx_qword_t *dma_addr;
	uint64_t addr;
	int npages;
	int i;
	efx_rc_t rc;

	EFSYS_ASSERT(EF10_TXQ_MAXNBUFS >=
	    efx_txq_nbufs(enp, enp->en_nic_cfg.enc_txq_max_ndescs));

	if ((esmp == NULL) ||
	    (EFSYS_MEM_SIZE(esmp) < efx_txq_size(enp, ndescs))) {
		rc = EINVAL;
		goto fail1;
	}

	npages = efx_txq_nbufs(enp, ndescs);
	if (MC_CMD_INIT_TXQ_IN_LEN(npages) > sizeof (payload)) {
		rc = EINVAL;
		goto fail2;
	}

	req.emr_cmd = MC_CMD_INIT_TXQ;
	req.emr_in_buf = payload;
	req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages);
	req.emr_out_buf = payload;
	req.emr_out_length = MC_CMD_INIT_TXQ_OUT_LEN;

	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_SIZE, ndescs);
	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_TARGET_EVQ, target_evq);
	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_LABEL, label);
	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_INSTANCE, instance);

	MCDI_IN_POPULATE_DWORD_9(req, INIT_TXQ_IN_FLAGS,
	    INIT_TXQ_IN_FLAG_BUFF_MODE, 0,
	    INIT_TXQ_IN_FLAG_IP_CSUM_DIS,
	    (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1,
	    INIT_TXQ_IN_FLAG_TCP_CSUM_DIS,
	    (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1,
	    INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN,
	    (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0,
	    INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN,
	    (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0,
	    INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, (flags & EFX_TXQ_FATSOV2) ? 1 : 0,
	    INIT_TXQ_IN_FLAG_TCP_UDP_ONLY, 0,
	    INIT_TXQ_IN_CRC_MODE, 0,
	    INIT_TXQ_IN_FLAG_TIMESTAMP, 0);

	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_OWNER_ID, 0);
	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);

	dma_addr = MCDI_IN2(req, efx_qword_t, INIT_TXQ_IN_DMA_ADDR);
	addr = EFSYS_MEM_ADDR(esmp);

	for (i = 0; i < npages; i++) {
		EFX_POPULATE_QWORD_2(*dma_addr,
		    EFX_DWORD_1, (uint32_t)(addr >> 32),
		    EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));

		dma_addr++;
		addr += EFX_BUF_SIZE;
	}

	efx_mcdi_execute(enp, &req);

	if (req.emr_rc != 0) {
		rc = req.emr_rc;
		goto fail3;
	}

	return (0);

fail3:
	EFSYS_PROBE(fail3);
fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

static	__checkReturn	efx_rc_t
efx_mcdi_fini_txq(
	__in		efx_nic_t *enp,
	__in		uint32_t instance)
{
	efx_mcdi_req_t req;
	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_TXQ_IN_LEN,
		MC_CMD_FINI_TXQ_OUT_LEN);
	efx_rc_t rc;

	req.emr_cmd = MC_CMD_FINI_TXQ;
	req.emr_in_buf = payload;
	req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN;
	req.emr_out_buf = payload;
	req.emr_out_length = MC_CMD_FINI_TXQ_OUT_LEN;

	MCDI_IN_SET_DWORD(req, FINI_TXQ_IN_INSTANCE, instance);

	efx_mcdi_execute_quiet(enp, &req);

	if (req.emr_rc != 0) {
		rc = req.emr_rc;
		goto fail1;
	}

	return (0);

fail1:
	/*
	 * EALREADY is not an error, but indicates that the MC has rebooted and
	 * that the TXQ has already been destroyed.
	 */
	if (rc != EALREADY)
		EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

	__checkReturn	efx_rc_t
ef10_tx_init(
	__in		efx_nic_t *enp)
{
	_NOTE(ARGUNUSED(enp))
	return (0);
}

			void
ef10_tx_fini(
	__in		efx_nic_t *enp)
{
	_NOTE(ARGUNUSED(enp))
}

	__checkReturn	efx_rc_t
ef10_tx_qcreate(
	__in		efx_nic_t *enp,
	__in		unsigned int index,
	__in		unsigned int label,
	__in		efsys_mem_t *esmp,
	__in		size_t ndescs,
	__in		uint32_t id,
	__in		uint16_t flags,
	__in		efx_evq_t *eep,
	__in		efx_txq_t *etp,
	__out		unsigned int *addedp)
{
	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
	uint16_t inner_csum;
	efx_desc_t desc;
	efx_rc_t rc;

	_NOTE(ARGUNUSED(id))

	inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
	if (((flags & inner_csum) != 0) &&
	    (encp->enc_tunnel_encapsulations_supported == 0)) {
		rc = EINVAL;
		goto fail1;
	}

	if ((rc = efx_mcdi_init_txq(enp, ndescs, eep->ee_index, label, index,
	    flags, esmp)) != 0)
		goto fail2;

	/*
	 * A previous user of this TX queue may have written a descriptor to the
	 * TX push collector, but not pushed the doorbell (e.g. after a crash).
	 * The next doorbell write would then push the stale descriptor.
	 *
	 * Ensure the (per network port) TX push collector is cleared by writing
	 * a no-op TX option descriptor. See bug29981 for details.
	 */
	*addedp = 1;
	ef10_tx_qdesc_checksum_create(etp, flags, &desc);

	EFSYS_MEM_WRITEQ(etp->et_esmp, 0, &desc.ed_eq);
	ef10_tx_qpush(etp, *addedp, 0);

	return (0);

fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

		void
ef10_tx_qdestroy(
	__in	efx_txq_t *etp)
{
	/* FIXME */
	_NOTE(ARGUNUSED(etp))
	/* FIXME */
}

	__checkReturn	efx_rc_t
ef10_tx_qpio_enable(
	__in		efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_piobuf_handle_t handle;
	efx_rc_t rc;

	if (etp->et_pio_size != 0) {
		rc = EALREADY;
		goto fail1;
	}

	/* Sub-allocate a PIO block from a piobuf */
	if ((rc = ef10_nic_pio_alloc(enp,
		    &etp->et_pio_bufnum,
		    &handle,
		    &etp->et_pio_blknum,
		    &etp->et_pio_offset,
		    &etp->et_pio_size)) != 0) {
		goto fail2;
	}
	EFSYS_ASSERT3U(etp->et_pio_size, !=, 0);

	/* Link the piobuf to this TXQ */
	if ((rc = ef10_nic_pio_link(enp, etp->et_index, handle)) != 0) {
		goto fail3;
	}

	/*
	 * et_pio_offset is the offset of the sub-allocated block within the
	 * hardware PIO buffer. It is used as the buffer address in the PIO
	 * option descriptor.
	 *
	 * et_pio_write_offset is the offset of the sub-allocated block from the
	 * start of the write-combined memory mapping, and is used for writing
	 * data into the PIO buffer.
	 */
	etp->et_pio_write_offset =
	    (etp->et_pio_bufnum * ER_DZ_TX_PIOBUF_STEP) +
	    ER_DZ_TX_PIOBUF_OFST + etp->et_pio_offset;

	return (0);

fail3:
	EFSYS_PROBE(fail3);
	(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum);
fail2:
	EFSYS_PROBE(fail2);
	etp->et_pio_size = 0;
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

			void
ef10_tx_qpio_disable(
	__in		efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;

	if (etp->et_pio_size != 0) {
		/* Unlink the piobuf from this TXQ */
		if (ef10_nic_pio_unlink(enp, etp->et_index) != 0)
			return;

		/* Free the sub-allocated PIO block */
		(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum,
		    etp->et_pio_blknum);
		etp->et_pio_size = 0;
		etp->et_pio_write_offset = 0;
	}
}

	__checkReturn	efx_rc_t
ef10_tx_qpio_write(
	__in			efx_txq_t *etp,
	__in_ecount(length)	uint8_t *buffer,
	__in			size_t length,
	__in			size_t offset)
{
	efx_nic_t *enp = etp->et_enp;
	efsys_bar_t *esbp = enp->en_esbp;
	uint32_t write_offset;
	uint32_t write_offset_limit;
	efx_qword_t *eqp;
	efx_rc_t rc;

	EFSYS_ASSERT(length % sizeof (efx_qword_t) == 0);

	if (etp->et_pio_size == 0) {
		rc = ENOENT;
		goto fail1;
	}
	if (offset + length > etp->et_pio_size)	{
		rc = ENOSPC;
		goto fail2;
	}

	/*
	 * Writes to PIO buffers must be 64 bit aligned, and multiples of
	 * 64 bits.
	 */
	write_offset = etp->et_pio_write_offset + offset;
	write_offset_limit = write_offset + length;
	eqp = (efx_qword_t *)buffer;
	while (write_offset < write_offset_limit) {
		EFSYS_BAR_WC_WRITEQ(esbp, write_offset, eqp);
		eqp++;
		write_offset += sizeof (efx_qword_t);
	}

	return (0);

fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

	__checkReturn	efx_rc_t
ef10_tx_qpio_post(
	__in			efx_txq_t *etp,
	__in			size_t pkt_length,
	__in			unsigned int completed,
	__inout			unsigned int *addedp)
{
	efx_qword_t pio_desc;
	unsigned int id;
	size_t offset;
	unsigned int added = *addedp;
	efx_rc_t rc;


	if (added - completed + 1 > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
		rc = ENOSPC;
		goto fail1;
	}

	if (etp->et_pio_size == 0) {
		rc = ENOENT;
		goto fail2;
	}

	id = added++ & etp->et_mask;
	offset = id * sizeof (efx_qword_t);

	EFSYS_PROBE4(tx_pio_post, unsigned int, etp->et_index,
		    unsigned int, id, uint32_t, etp->et_pio_offset,
		    size_t, pkt_length);

	EFX_POPULATE_QWORD_5(pio_desc,
			ESF_DZ_TX_DESC_IS_OPT, 1,
			ESF_DZ_TX_OPTION_TYPE, 1,
			ESF_DZ_TX_PIO_CONT, 0,
			ESF_DZ_TX_PIO_BYTE_CNT, pkt_length,
			ESF_DZ_TX_PIO_BUF_ADDR, etp->et_pio_offset);

	EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &pio_desc);

	EFX_TX_QSTAT_INCR(etp, TX_POST_PIO);

	*addedp = added;
	return (0);

fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

	__checkReturn		efx_rc_t
ef10_tx_qpost(
	__in			efx_txq_t *etp,
	__in_ecount(ndescs)	efx_buffer_t *eb,
	__in			unsigned int ndescs,
	__in			unsigned int completed,
	__inout			unsigned int *addedp)
{
	unsigned int added = *addedp;
	unsigned int i;
	efx_rc_t rc;

	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
		rc = ENOSPC;
		goto fail1;
	}

	for (i = 0; i < ndescs; i++) {
		efx_buffer_t *ebp = &eb[i];
		efsys_dma_addr_t addr = ebp->eb_addr;
		size_t size = ebp->eb_size;
		boolean_t eop = ebp->eb_eop;
		unsigned int id;
		size_t offset;
		efx_qword_t qword;

		/* No limitations on boundary crossing */
		EFSYS_ASSERT(size <=
		    etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max);

		id = added++ & etp->et_mask;
		offset = id * sizeof (efx_qword_t);

		EFSYS_PROBE5(tx_post, unsigned int, etp->et_index,
		    unsigned int, id, efsys_dma_addr_t, addr,
		    size_t, size, boolean_t, eop);

		EFX_POPULATE_QWORD_5(qword,
		    ESF_DZ_TX_KER_TYPE, 0,
		    ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1,
		    ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size),
		    ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff),
		    ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32));

		EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &qword);
	}

	EFX_TX_QSTAT_INCR(etp, TX_POST);

	*addedp = added;
	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

/*
 * This improves performance by, when possible, pushing a TX descriptor at the
 * same time as the doorbell. The descriptor must be added to the TXQ, so that
 * can be used if the hardware decides not to use the pushed descriptor.
 */
			void
ef10_tx_qpush(
	__in		efx_txq_t *etp,
	__in		unsigned int added,
	__in		unsigned int pushed)
{
	efx_nic_t *enp = etp->et_enp;
	unsigned int wptr;
	unsigned int id;
	size_t offset;
	efx_qword_t desc;
	efx_oword_t oword;

	wptr = added & etp->et_mask;
	id = pushed & etp->et_mask;
	offset = id * sizeof (efx_qword_t);

	EFSYS_MEM_READQ(etp->et_esmp, offset, &desc);

	/*
	 * Bug 65776: TSO option descriptors cannot be pushed if pacer bypass is
	 * enabled on the event queue this transmit queue is attached to.
	 *
	 * To ensure the code is safe, it is easiest to simply test the type of
	 * the descriptor to push, and only push it is if it not a TSO option
	 * descriptor.
	 */
	if ((EFX_QWORD_FIELD(desc, ESF_DZ_TX_DESC_IS_OPT) != 1) ||
	    (EFX_QWORD_FIELD(desc, ESF_DZ_TX_OPTION_TYPE) !=
	    ESE_DZ_TX_OPTION_DESC_TSO)) {
		/* Push the descriptor and update the wptr. */
		EFX_POPULATE_OWORD_3(oword, ERF_DZ_TX_DESC_WPTR, wptr,
		    ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1),
		    ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0));

		/* Ensure ordering of memory (descriptors) and PIO (doorbell) */
		EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
					    wptr, id);
		EFSYS_PIO_WRITE_BARRIER();
		EFX_BAR_VI_DOORBELL_WRITEO(enp, ER_DZ_TX_DESC_UPD_REG,
		    etp->et_index, &oword);
	} else {
		efx_dword_t dword;

		/*
		 * Only update the wptr. This is signalled to the hardware by
		 * only writing one DWORD of the doorbell register.
		 */
		EFX_POPULATE_OWORD_1(oword, ERF_DZ_TX_DESC_WPTR, wptr);
		dword = oword.eo_dword[2];

		/* Ensure ordering of memory (descriptors) and PIO (doorbell) */
		EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
					    wptr, id);
		EFSYS_PIO_WRITE_BARRIER();
		EFX_BAR_VI_WRITED2(enp, ER_DZ_TX_DESC_UPD_REG,
		    etp->et_index, &dword, B_FALSE);
	}
}

	__checkReturn		efx_rc_t
ef10_tx_qdesc_post(
	__in			efx_txq_t *etp,
	__in_ecount(ndescs)	efx_desc_t *ed,
	__in			unsigned int ndescs,
	__in			unsigned int completed,
	__inout			unsigned int *addedp)
{
	unsigned int added = *addedp;
	unsigned int i;

	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
		return (ENOSPC);

	for (i = 0; i < ndescs; i++) {
		efx_desc_t *edp = &ed[i];
		unsigned int id;
		size_t offset;

		id = added++ & etp->et_mask;
		offset = id * sizeof (efx_desc_t);

		EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
	}

	EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
		    unsigned int, added, unsigned int, ndescs);

	EFX_TX_QSTAT_INCR(etp, TX_POST);

	*addedp = added;
	return (0);
}

	void
ef10_tx_qdesc_dma_create(
	__in	efx_txq_t *etp,
	__in	efsys_dma_addr_t addr,
	__in	size_t size,
	__in	boolean_t eop,
	__out	efx_desc_t *edp)
{
	_NOTE(ARGUNUSED(etp))

	/* No limitations on boundary crossing */
	EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max);

	EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
		    efsys_dma_addr_t, addr,
		    size_t, size, boolean_t, eop);

	EFX_POPULATE_QWORD_5(edp->ed_eq,
		    ESF_DZ_TX_KER_TYPE, 0,
		    ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1,
		    ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size),
		    ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff),
		    ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32));
}

	void
ef10_tx_qdesc_tso_create(
	__in	efx_txq_t *etp,
	__in	uint16_t ipv4_id,
	__in	uint32_t tcp_seq,
	__in	uint8_t  tcp_flags,
	__out	efx_desc_t *edp)
{
	_NOTE(ARGUNUSED(etp))

	EFSYS_PROBE4(tx_desc_tso_create, unsigned int, etp->et_index,
		    uint16_t, ipv4_id, uint32_t, tcp_seq,
		    uint8_t, tcp_flags);

	EFX_POPULATE_QWORD_5(edp->ed_eq,
			    ESF_DZ_TX_DESC_IS_OPT, 1,
			    ESF_DZ_TX_OPTION_TYPE,
			    ESE_DZ_TX_OPTION_DESC_TSO,
			    ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags,
			    ESF_DZ_TX_TSO_IP_ID, ipv4_id,
			    ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
}

	void
ef10_tx_qdesc_tso2_create(
	__in			efx_txq_t *etp,
	__in			uint16_t ipv4_id,
	__in			uint16_t outer_ipv4_id,
	__in			uint32_t tcp_seq,
	__in			uint16_t tcp_mss,
	__out_ecount(count)	efx_desc_t *edp,
	__in			int count)
{
	_NOTE(ARGUNUSED(etp, count))

	EFSYS_PROBE4(tx_desc_tso2_create, unsigned int, etp->et_index,
		    uint16_t, ipv4_id, uint32_t, tcp_seq,
		    uint16_t, tcp_mss);

	EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS);

	EFX_POPULATE_QWORD_5(edp[0].ed_eq,
			    ESF_DZ_TX_DESC_IS_OPT, 1,
			    ESF_DZ_TX_OPTION_TYPE,
			    ESE_DZ_TX_OPTION_DESC_TSO,
			    ESF_DZ_TX_TSO_OPTION_TYPE,
			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
			    ESF_DZ_TX_TSO_IP_ID, ipv4_id,
			    ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
	EFX_POPULATE_QWORD_5(edp[1].ed_eq,
			    ESF_DZ_TX_DESC_IS_OPT, 1,
			    ESF_DZ_TX_OPTION_TYPE,
			    ESE_DZ_TX_OPTION_DESC_TSO,
			    ESF_DZ_TX_TSO_OPTION_TYPE,
			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
			    ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
			    ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
}

	void
ef10_tx_qdesc_vlantci_create(
	__in	efx_txq_t *etp,
	__in	uint16_t  tci,
	__out	efx_desc_t *edp)
{
	_NOTE(ARGUNUSED(etp))

	EFSYS_PROBE2(tx_desc_vlantci_create, unsigned int, etp->et_index,
		    uint16_t, tci);

	EFX_POPULATE_QWORD_4(edp->ed_eq,
			    ESF_DZ_TX_DESC_IS_OPT, 1,
			    ESF_DZ_TX_OPTION_TYPE,
			    ESE_DZ_TX_OPTION_DESC_VLAN,
			    ESF_DZ_TX_VLAN_OP, tci ? 1 : 0,
			    ESF_DZ_TX_VLAN_TAG1, tci);
}

	void
ef10_tx_qdesc_checksum_create(
	__in	efx_txq_t *etp,
	__in	uint16_t flags,
	__out	efx_desc_t *edp)
{
	_NOTE(ARGUNUSED(etp));

	EFSYS_PROBE2(tx_desc_checksum_create, unsigned int, etp->et_index,
		    uint32_t, flags);

	EFX_POPULATE_QWORD_6(edp->ed_eq,
	    ESF_DZ_TX_DESC_IS_OPT, 1,
	    ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_CRC_CSUM,
	    ESF_DZ_TX_OPTION_UDP_TCP_CSUM,
	    (flags & EFX_TXQ_CKSUM_TCPUDP) ? 1 : 0,
	    ESF_DZ_TX_OPTION_IP_CSUM,
	    (flags & EFX_TXQ_CKSUM_IPV4) ? 1 : 0,
	    ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM,
	    (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0,
	    ESF_DZ_TX_OPTION_INNER_IP_CSUM,
	    (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0);
}


	__checkReturn	efx_rc_t
ef10_tx_qpace(
	__in		efx_txq_t *etp,
	__in		unsigned int ns)
{
	efx_rc_t rc;

	/* FIXME */
	_NOTE(ARGUNUSED(etp, ns))
	_NOTE(CONSTANTCONDITION)
	if (B_FALSE) {
		rc = ENOTSUP;
		goto fail1;
	}
	/* FIXME */

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

	__checkReturn	efx_rc_t
ef10_tx_qflush(
	__in		efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_rc_t rc;

	if ((rc = efx_mcdi_fini_txq(enp, etp->et_index)) != 0)
		goto fail1;

	return (0);

fail1:
	/*
	 * EALREADY is not an error, but indicates that the MC has rebooted and
	 * that the TXQ has already been destroyed. Callers need to know that
	 * the TXQ flush has completed to avoid waiting until timeout for a
	 * flush done event that will not be delivered.
	 */
	if (rc != EALREADY)
		EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

			void
ef10_tx_qenable(
	__in		efx_txq_t *etp)
{
	/* FIXME */
	_NOTE(ARGUNUSED(etp))
	/* FIXME */
}

#if EFSYS_OPT_QSTATS
			void
ef10_tx_qstats_update(
	__in				efx_txq_t *etp,
	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
{
	unsigned int id;

	for (id = 0; id < TX_NQSTATS; id++) {
		efsys_stat_t *essp = &stat[id];

		EFSYS_STAT_INCR(essp, etp->et_stat[id]);
		etp->et_stat[id] = 0;
	}
}

#endif /* EFSYS_OPT_QSTATS */

#endif /* EFX_OPTS_EF10() */
Commit	Line	Data
9f95a23c	1	/* SPDX-License-Identifier: BSD-3-Clause
11fdf7f2	2	*
9f95a23c TL	3	* Copyright (c) 2012-2018 Solarflare Communications Inc.
9f95a23c TL	4	* All rights reserved.
11fdf7f2 TL	5	*/
	6
	7	#include "efx.h"
	8	#include "efx_impl.h"
	9
	10
9f95a23c	11	#if EFX_OPTS_EF10()
11fdf7f2 TL	12
	13	#if EFSYS_OPT_QSTATS
	14	#define EFX_TX_QSTAT_INCR(_etp, _stat) \
	15	do { \
	16	(_etp)->et_stat[_stat]++; \
	17	_NOTE(CONSTANTCONDITION) \
	18	} while (B_FALSE)
	19	#else
	20	#define EFX_TX_QSTAT_INCR(_etp, _stat)
	21	#endif
	22
	23	static __checkReturn efx_rc_t
	24	efx_mcdi_init_txq(
	25	__in efx_nic_t *enp,
9f95a23c	26	__in uint32_t ndescs,
11fdf7f2 TL	27	__in uint32_t target_evq,
	28	__in uint32_t label,
	29	__in uint32_t instance,
	30	__in uint16_t flags,
	31	__in efsys_mem_t *esmp)
	32	{
	33	efx_mcdi_req_t req;
9f95a23c TL	34	EFX_MCDI_DECLARE_BUF(payload,
	35	MC_CMD_INIT_TXQ_IN_LEN(EF10_TXQ_MAXNBUFS),
	36	MC_CMD_INIT_TXQ_OUT_LEN);
11fdf7f2 TL	37	efx_qword_t *dma_addr;
	38	uint64_t addr;
	39	int npages;
	40	int i;
	41	efx_rc_t rc;
	42
9f95a23c TL	43	EFSYS_ASSERT(EF10_TXQ_MAXNBUFS >=
9f95a23c TL	44	efx_txq_nbufs(enp, enp->en_nic_cfg.enc_txq_max_ndescs));
11fdf7f2	45
9f95a23c TL	46	if ((esmp == NULL) \|\|
9f95a23c TL	47	(EFSYS_MEM_SIZE(esmp) < efx_txq_size(enp, ndescs))) {
11fdf7f2 TL	48	rc = EINVAL;
	49	goto fail1;
	50	}
	51
9f95a23c TL	52	npages = efx_txq_nbufs(enp, ndescs);
	53	if (MC_CMD_INIT_TXQ_IN_LEN(npages) > sizeof (payload)) {
	54	rc = EINVAL;
	55	goto fail2;
	56	}
	57
11fdf7f2 TL	58	req.emr_cmd = MC_CMD_INIT_TXQ;
	59	req.emr_in_buf = payload;
	60	req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages);
	61	req.emr_out_buf = payload;
	62	req.emr_out_length = MC_CMD_INIT_TXQ_OUT_LEN;
	63
9f95a23c	64	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_SIZE, ndescs);
11fdf7f2 TL	65	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_TARGET_EVQ, target_evq);
	66	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_LABEL, label);
	67	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_INSTANCE, instance);
	68
9f95a23c	69	MCDI_IN_POPULATE_DWORD_9(req, INIT_TXQ_IN_FLAGS,
11fdf7f2 TL	70	INIT_TXQ_IN_FLAG_BUFF_MODE, 0,
	71	INIT_TXQ_IN_FLAG_IP_CSUM_DIS,
	72	(flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1,
	73	INIT_TXQ_IN_FLAG_TCP_CSUM_DIS,
	74	(flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1,
9f95a23c TL	75	INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN,
	76	(flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0,
	77	INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN,
	78	(flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0,
11fdf7f2 TL	79	INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, (flags & EFX_TXQ_FATSOV2) ? 1 : 0,
	80	INIT_TXQ_IN_FLAG_TCP_UDP_ONLY, 0,
	81	INIT_TXQ_IN_CRC_MODE, 0,
	82	INIT_TXQ_IN_FLAG_TIMESTAMP, 0);
	83
	84	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_OWNER_ID, 0);
	85	MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
	86
	87	dma_addr = MCDI_IN2(req, efx_qword_t, INIT_TXQ_IN_DMA_ADDR);
	88	addr = EFSYS_MEM_ADDR(esmp);
	89
	90	for (i = 0; i < npages; i++) {
	91	EFX_POPULATE_QWORD_2(*dma_addr,
	92	EFX_DWORD_1, (uint32_t)(addr >> 32),
	93	EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
	94
	95	dma_addr++;
	96	addr += EFX_BUF_SIZE;
	97	}
	98
	99	efx_mcdi_execute(enp, &req);
	100
	101	if (req.emr_rc != 0) {
	102	rc = req.emr_rc;
9f95a23c	103	goto fail3;
11fdf7f2 TL	104	}
	105
	106	return (0);
	107
9f95a23c TL	108	fail3:
9f95a23c TL	109	EFSYS_PROBE(fail3);
11fdf7f2 TL	110	fail2:
	111	EFSYS_PROBE(fail2);
	112	fail1:
	113	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	114
	115	return (rc);
	116	}
	117
	118	static __checkReturn efx_rc_t
	119	efx_mcdi_fini_txq(
	120	__in efx_nic_t *enp,
	121	__in uint32_t instance)
	122	{
	123	efx_mcdi_req_t req;
9f95a23c TL	124	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_TXQ_IN_LEN,
9f95a23c TL	125	MC_CMD_FINI_TXQ_OUT_LEN);
11fdf7f2 TL	126	efx_rc_t rc;
11fdf7f2 TL	127
11fdf7f2 TL	128	req.emr_cmd = MC_CMD_FINI_TXQ;
	129	req.emr_in_buf = payload;
	130	req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN;
	131	req.emr_out_buf = payload;
	132	req.emr_out_length = MC_CMD_FINI_TXQ_OUT_LEN;
	133
	134	MCDI_IN_SET_DWORD(req, FINI_TXQ_IN_INSTANCE, instance);
	135
	136	efx_mcdi_execute_quiet(enp, &req);
	137
9f95a23c	138	if (req.emr_rc != 0) {
11fdf7f2 TL	139	rc = req.emr_rc;
	140	goto fail1;
	141	}
	142
	143	return (0);
	144
	145	fail1:
9f95a23c TL	146	/*
	147	* EALREADY is not an error, but indicates that the MC has rebooted and
	148	* that the TXQ has already been destroyed.
	149	*/
	150	if (rc != EALREADY)
	151	EFSYS_PROBE1(fail1, efx_rc_t, rc);
11fdf7f2 TL	152
	153	return (rc);
	154	}
	155
	156	__checkReturn efx_rc_t
	157	ef10_tx_init(
	158	__in efx_nic_t *enp)
	159	{
	160	_NOTE(ARGUNUSED(enp))
	161	return (0);
	162	}
	163
	164	void
	165	ef10_tx_fini(
	166	__in efx_nic_t *enp)
	167	{
	168	_NOTE(ARGUNUSED(enp))
	169	}
	170
	171	__checkReturn efx_rc_t
	172	ef10_tx_qcreate(
	173	__in efx_nic_t *enp,
	174	__in unsigned int index,
	175	__in unsigned int label,
	176	__in efsys_mem_t *esmp,
9f95a23c	177	__in size_t ndescs,
11fdf7f2 TL	178	__in uint32_t id,
	179	__in uint16_t flags,
	180	__in efx_evq_t *eep,
	181	__in efx_txq_t *etp,
	182	__out unsigned int *addedp)
	183	{
9f95a23c TL	184	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
	185	uint16_t inner_csum;
	186	efx_desc_t desc;
11fdf7f2 TL	187	efx_rc_t rc;
	188
	189	_NOTE(ARGUNUSED(id))
	190
9f95a23c TL	191	inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 \| EFX_TXQ_CKSUM_INNER_TCPUDP;
	192	if (((flags & inner_csum) != 0) &&
	193	(encp->enc_tunnel_encapsulations_supported == 0)) {
	194	rc = EINVAL;
11fdf7f2	195	goto fail1;
9f95a23c TL	196	}
	197
	198	if ((rc = efx_mcdi_init_txq(enp, ndescs, eep->ee_index, label, index,
	199	flags, esmp)) != 0)
	200	goto fail2;
11fdf7f2 TL	201
	202	/*
	203	* A previous user of this TX queue may have written a descriptor to the
	204	* TX push collector, but not pushed the doorbell (e.g. after a crash).
	205	* The next doorbell write would then push the stale descriptor.
	206	*
	207	* Ensure the (per network port) TX push collector is cleared by writing
	208	* a no-op TX option descriptor. See bug29981 for details.
	209	*/
	210	*addedp = 1;
9f95a23c	211	ef10_tx_qdesc_checksum_create(etp, flags, &desc);
11fdf7f2	212
9f95a23c	213	EFSYS_MEM_WRITEQ(etp->et_esmp, 0, &desc.ed_eq);
11fdf7f2 TL	214	ef10_tx_qpush(etp, *addedp, 0);
	215
	216	return (0);
	217
9f95a23c TL	218	fail2:
9f95a23c TL	219	EFSYS_PROBE(fail2);
11fdf7f2 TL	220	fail1:
	221	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	222
	223	return (rc);
	224	}
	225
	226	void
	227	ef10_tx_qdestroy(
	228	__in efx_txq_t *etp)
	229	{
	230	/* FIXME */
	231	_NOTE(ARGUNUSED(etp))
	232	/* FIXME */
	233	}
	234
	235	__checkReturn efx_rc_t
	236	ef10_tx_qpio_enable(
	237	__in efx_txq_t *etp)
	238	{
	239	efx_nic_t *enp = etp->et_enp;
	240	efx_piobuf_handle_t handle;
	241	efx_rc_t rc;
	242
	243	if (etp->et_pio_size != 0) {
	244	rc = EALREADY;
	245	goto fail1;
	246	}
	247
	248	/* Sub-allocate a PIO block from a piobuf */
	249	if ((rc = ef10_nic_pio_alloc(enp,
	250	&etp->et_pio_bufnum,
	251	&handle,
	252	&etp->et_pio_blknum,
	253	&etp->et_pio_offset,
	254	&etp->et_pio_size)) != 0) {
	255	goto fail2;
	256	}
	257	EFSYS_ASSERT3U(etp->et_pio_size, !=, 0);
	258
	259	/* Link the piobuf to this TXQ */
	260	if ((rc = ef10_nic_pio_link(enp, etp->et_index, handle)) != 0) {
	261	goto fail3;
	262	}
	263
	264	/*
	265	* et_pio_offset is the offset of the sub-allocated block within the
	266	* hardware PIO buffer. It is used as the buffer address in the PIO
	267	* option descriptor.
	268	*
	269	* et_pio_write_offset is the offset of the sub-allocated block from the
	270	* start of the write-combined memory mapping, and is used for writing
	271	* data into the PIO buffer.
	272	*/
	273	etp->et_pio_write_offset =
	274	(etp->et_pio_bufnum * ER_DZ_TX_PIOBUF_STEP) +
	275	ER_DZ_TX_PIOBUF_OFST + etp->et_pio_offset;
	276
	277	return (0);
	278
	279	fail3:
	280	EFSYS_PROBE(fail3);
9f95a23c	281	(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum);
11fdf7f2 TL	282	fail2:
	283	EFSYS_PROBE(fail2);
	284	etp->et_pio_size = 0;
	285	fail1:
	286	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	287
	288	return (rc);
	289	}
	290
	291	void
	292	ef10_tx_qpio_disable(
	293	__in efx_txq_t *etp)
	294	{
	295	efx_nic_t *enp = etp->et_enp;
	296
	297	if (etp->et_pio_size != 0) {
	298	/* Unlink the piobuf from this TXQ */
9f95a23c TL	299	if (ef10_nic_pio_unlink(enp, etp->et_index) != 0)
9f95a23c TL	300	return;
11fdf7f2 TL	301
11fdf7f2 TL	302	/* Free the sub-allocated PIO block */
9f95a23c TL	303	(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum,
9f95a23c TL	304	etp->et_pio_blknum);
11fdf7f2 TL	305	etp->et_pio_size = 0;
	306	etp->et_pio_write_offset = 0;
	307	}
	308	}
	309
	310	__checkReturn efx_rc_t
	311	ef10_tx_qpio_write(
	312	__in efx_txq_t *etp,
	313	__in_ecount(length) uint8_t *buffer,
	314	__in size_t length,
	315	__in size_t offset)
	316	{
	317	efx_nic_t *enp = etp->et_enp;
	318	efsys_bar_t *esbp = enp->en_esbp;
	319	uint32_t write_offset;
	320	uint32_t write_offset_limit;
	321	efx_qword_t *eqp;
	322	efx_rc_t rc;
	323
	324	EFSYS_ASSERT(length % sizeof (efx_qword_t) == 0);
	325
	326	if (etp->et_pio_size == 0) {
	327	rc = ENOENT;
	328	goto fail1;
	329	}
	330	if (offset + length > etp->et_pio_size) {
	331	rc = ENOSPC;
	332	goto fail2;
	333	}
	334
	335	/*
	336	* Writes to PIO buffers must be 64 bit aligned, and multiples of
	337	* 64 bits.
	338	*/
	339	write_offset = etp->et_pio_write_offset + offset;
	340	write_offset_limit = write_offset + length;
	341	eqp = (efx_qword_t *)buffer;
	342	while (write_offset < write_offset_limit) {
	343	EFSYS_BAR_WC_WRITEQ(esbp, write_offset, eqp);
	344	eqp++;
	345	write_offset += sizeof (efx_qword_t);
	346	}
	347
	348	return (0);
	349
	350	fail2:
	351	EFSYS_PROBE(fail2);
	352	fail1:
	353	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	354
	355	return (rc);
	356	}
	357
	358	__checkReturn efx_rc_t
	359	ef10_tx_qpio_post(
	360	__in efx_txq_t *etp,
	361	__in size_t pkt_length,
	362	__in unsigned int completed,
	363	__inout unsigned int *addedp)
	364	{
	365	efx_qword_t pio_desc;
	366	unsigned int id;
	367	size_t offset;
	368	unsigned int added = *addedp;
369	efx_rc_t rc;
370
371
372	if (added - completed + 1 > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
373	rc = ENOSPC;
374	goto fail1;
375	}
376
377	if (etp->et_pio_size == 0) {
378	rc = ENOENT;
379	goto fail2;
380	}
381
382	id = added++ & etp->et_mask;
383	offset = id * sizeof (efx_qword_t);
384
385	EFSYS_PROBE4(tx_pio_post, unsigned int, etp->et_index,
386	unsigned int, id, uint32_t, etp->et_pio_offset,
387	size_t, pkt_length);
388
389	EFX_POPULATE_QWORD_5(pio_desc,
390	ESF_DZ_TX_DESC_IS_OPT, 1,
391	ESF_DZ_TX_OPTION_TYPE, 1,
392	ESF_DZ_TX_PIO_CONT, 0,
393	ESF_DZ_TX_PIO_BYTE_CNT, pkt_length,
394	ESF_DZ_TX_PIO_BUF_ADDR, etp->et_pio_offset);
395
396	EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &pio_desc);
397
398	EFX_TX_QSTAT_INCR(etp, TX_POST_PIO);
399
400	*addedp = added;
401	return (0);
402
403	fail2:
404	EFSYS_PROBE(fail2);
405	fail1:
406	EFSYS_PROBE1(fail1, efx_rc_t, rc);
407
408	return (rc);
409	}
410
9f95a23c	411	__checkReturn efx_rc_t
11fdf7f2	412	ef10_tx_qpost(
9f95a23c TL	413	__in efx_txq_t *etp,
	414	__in_ecount(ndescs) efx_buffer_t *eb,
	415	__in unsigned int ndescs,
	416	__in unsigned int completed,
	417	__inout unsigned int *addedp)
11fdf7f2 TL	418	{
	419	unsigned int added = *addedp;
	420	unsigned int i;
	421	efx_rc_t rc;
	422
9f95a23c	423	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
11fdf7f2 TL	424	rc = ENOSPC;
	425	goto fail1;
	426	}
	427
9f95a23c	428	for (i = 0; i < ndescs; i++) {
11fdf7f2 TL	429	efx_buffer_t *ebp = &eb[i];
	430	efsys_dma_addr_t addr = ebp->eb_addr;
	431	size_t size = ebp->eb_size;
	432	boolean_t eop = ebp->eb_eop;
	433	unsigned int id;
	434	size_t offset;
	435	efx_qword_t qword;
	436
	437	/* No limitations on boundary crossing */
	438	EFSYS_ASSERT(size <=
	439	etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max);
	440
	441	id = added++ & etp->et_mask;
	442	offset = id * sizeof (efx_qword_t);
	443
	444	EFSYS_PROBE5(tx_post, unsigned int, etp->et_index,
	445	unsigned int, id, efsys_dma_addr_t, addr,
	446	size_t, size, boolean_t, eop);
	447
	448	EFX_POPULATE_QWORD_5(qword,
	449	ESF_DZ_TX_KER_TYPE, 0,
	450	ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1,
	451	ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size),
	452	ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff),
	453	ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32));
	454
	455	EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &qword);
	456	}
	457
	458	EFX_TX_QSTAT_INCR(etp, TX_POST);
	459
	460	*addedp = added;
	461	return (0);
	462
	463	fail1:
	464	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	465
	466	return (rc);
	467	}
	468
	469	/*
9f95a23c TL	470	* This improves performance by, when possible, pushing a TX descriptor at the
	471	* same time as the doorbell. The descriptor must be added to the TXQ, so that
	472	* can be used if the hardware decides not to use the pushed descriptor.
11fdf7f2 TL	473	*/
	474	void
	475	ef10_tx_qpush(
	476	__in efx_txq_t *etp,
	477	__in unsigned int added,
	478	__in unsigned int pushed)
	479	{
	480	efx_nic_t *enp = etp->et_enp;
	481	unsigned int wptr;
	482	unsigned int id;
	483	size_t offset;
	484	efx_qword_t desc;
	485	efx_oword_t oword;
	486
	487	wptr = added & etp->et_mask;
	488	id = pushed & etp->et_mask;
	489	offset = id * sizeof (efx_qword_t);
	490
	491	EFSYS_MEM_READQ(etp->et_esmp, offset, &desc);
9f95a23c TL	492
	493	/*
	494	* Bug 65776: TSO option descriptors cannot be pushed if pacer bypass is
	495	* enabled on the event queue this transmit queue is attached to.
	496	*
	497	* To ensure the code is safe, it is easiest to simply test the type of
	498	* the descriptor to push, and only push it is if it not a TSO option
	499	* descriptor.
	500	*/
	501	if ((EFX_QWORD_FIELD(desc, ESF_DZ_TX_DESC_IS_OPT) != 1) \|\|
	502	(EFX_QWORD_FIELD(desc, ESF_DZ_TX_OPTION_TYPE) !=
	503	ESE_DZ_TX_OPTION_DESC_TSO)) {
	504	/* Push the descriptor and update the wptr. */
	505	EFX_POPULATE_OWORD_3(oword, ERF_DZ_TX_DESC_WPTR, wptr,
	506	ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1),
	507	ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0));
	508
	509	/* Ensure ordering of memory (descriptors) and PIO (doorbell) */
	510	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
	511	wptr, id);
	512	EFSYS_PIO_WRITE_BARRIER();
	513	EFX_BAR_VI_DOORBELL_WRITEO(enp, ER_DZ_TX_DESC_UPD_REG,
	514	etp->et_index, &oword);
	515	} else {
	516	efx_dword_t dword;
	517
	518	/*
	519	* Only update the wptr. This is signalled to the hardware by
	520	* only writing one DWORD of the doorbell register.
	521	*/
	522	EFX_POPULATE_OWORD_1(oword, ERF_DZ_TX_DESC_WPTR, wptr);
	523	dword = oword.eo_dword[2];
	524
	525	/* Ensure ordering of memory (descriptors) and PIO (doorbell) */
	526	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
	527	wptr, id);
	528	EFSYS_PIO_WRITE_BARRIER();
	529	EFX_BAR_VI_WRITED2(enp, ER_DZ_TX_DESC_UPD_REG,
	530	etp->et_index, &dword, B_FALSE);
	531	}
11fdf7f2 TL	532	}
11fdf7f2 TL	533
9f95a23c	534	__checkReturn efx_rc_t
11fdf7f2	535	ef10_tx_qdesc_post(
9f95a23c TL	536	__in efx_txq_t *etp,
	537	__in_ecount(ndescs) efx_desc_t *ed,
	538	__in unsigned int ndescs,
	539	__in unsigned int completed,
	540	__inout unsigned int *addedp)
11fdf7f2 TL	541	{
	542	unsigned int added = *addedp;
	543	unsigned int i;
11fdf7f2	544
9f95a23c TL	545	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
9f95a23c TL	546	return (ENOSPC);
11fdf7f2	547
9f95a23c	548	for (i = 0; i < ndescs; i++) {
11fdf7f2 TL	549	efx_desc_t *edp = &ed[i];
	550	unsigned int id;
	551	size_t offset;
	552
	553	id = added++ & etp->et_mask;
	554	offset = id * sizeof (efx_desc_t);
	555
	556	EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
	557	}
	558
	559	EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
9f95a23c	560	unsigned int, added, unsigned int, ndescs);
11fdf7f2 TL	561
	562	EFX_TX_QSTAT_INCR(etp, TX_POST);
	563
	564	*addedp = added;
	565	return (0);
11fdf7f2 TL	566	}
	567
	568	void
	569	ef10_tx_qdesc_dma_create(
	570	__in efx_txq_t *etp,
	571	__in efsys_dma_addr_t addr,
	572	__in size_t size,
	573	__in boolean_t eop,
	574	__out efx_desc_t *edp)
	575	{
9f95a23c TL	576	_NOTE(ARGUNUSED(etp))
9f95a23c TL	577
11fdf7f2 TL	578	/* No limitations on boundary crossing */
	579	EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max);
	580
	581	EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
	582	efsys_dma_addr_t, addr,
	583	size_t, size, boolean_t, eop);
	584
	585	EFX_POPULATE_QWORD_5(edp->ed_eq,
	586	ESF_DZ_TX_KER_TYPE, 0,
	587	ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1,
	588	ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size),
	589	ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff),
	590	ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32));
	591	}
	592
	593	void
	594	ef10_tx_qdesc_tso_create(
	595	__in efx_txq_t *etp,
	596	__in uint16_t ipv4_id,
	597	__in uint32_t tcp_seq,
	598	__in uint8_t tcp_flags,
	599	__out efx_desc_t *edp)
	600	{
9f95a23c TL	601	_NOTE(ARGUNUSED(etp))
9f95a23c TL	602
11fdf7f2 TL	603	EFSYS_PROBE4(tx_desc_tso_create, unsigned int, etp->et_index,
	604	uint16_t, ipv4_id, uint32_t, tcp_seq,
	605	uint8_t, tcp_flags);
	606
	607	EFX_POPULATE_QWORD_5(edp->ed_eq,
	608	ESF_DZ_TX_DESC_IS_OPT, 1,
	609	ESF_DZ_TX_OPTION_TYPE,
	610	ESE_DZ_TX_OPTION_DESC_TSO,
	611	ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags,
	612	ESF_DZ_TX_TSO_IP_ID, ipv4_id,
	613	ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
	614	}
	615
	616	void
	617	ef10_tx_qdesc_tso2_create(
	618	__in efx_txq_t *etp,
	619	__in uint16_t ipv4_id,
9f95a23c	620	__in uint16_t outer_ipv4_id,
11fdf7f2 TL	621	__in uint32_t tcp_seq,
	622	__in uint16_t tcp_mss,
	623	__out_ecount(count) efx_desc_t *edp,
	624	__in int count)
	625	{
9f95a23c TL	626	_NOTE(ARGUNUSED(etp, count))
9f95a23c TL	627
11fdf7f2 TL	628	EFSYS_PROBE4(tx_desc_tso2_create, unsigned int, etp->et_index,
	629	uint16_t, ipv4_id, uint32_t, tcp_seq,
	630	uint16_t, tcp_mss);
	631
	632	EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS);
	633
	634	EFX_POPULATE_QWORD_5(edp[0].ed_eq,
	635	ESF_DZ_TX_DESC_IS_OPT, 1,
	636	ESF_DZ_TX_OPTION_TYPE,
	637	ESE_DZ_TX_OPTION_DESC_TSO,
	638	ESF_DZ_TX_TSO_OPTION_TYPE,
	639	ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
	640	ESF_DZ_TX_TSO_IP_ID, ipv4_id,
	641	ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
9f95a23c	642	EFX_POPULATE_QWORD_5(edp[1].ed_eq,
11fdf7f2 TL	643	ESF_DZ_TX_DESC_IS_OPT, 1,
	644	ESF_DZ_TX_OPTION_TYPE,
	645	ESE_DZ_TX_OPTION_DESC_TSO,
	646	ESF_DZ_TX_TSO_OPTION_TYPE,
	647	ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
9f95a23c TL	648	ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
9f95a23c TL	649	ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
11fdf7f2 TL	650	}
	651
	652	void
	653	ef10_tx_qdesc_vlantci_create(
	654	__in efx_txq_t *etp,
	655	__in uint16_t tci,
	656	__out efx_desc_t *edp)
	657	{
9f95a23c TL	658	_NOTE(ARGUNUSED(etp))
9f95a23c TL	659
11fdf7f2 TL	660	EFSYS_PROBE2(tx_desc_vlantci_create, unsigned int, etp->et_index,
	661	uint16_t, tci);
	662
	663	EFX_POPULATE_QWORD_4(edp->ed_eq,
	664	ESF_DZ_TX_DESC_IS_OPT, 1,
	665	ESF_DZ_TX_OPTION_TYPE,
	666	ESE_DZ_TX_OPTION_DESC_VLAN,
	667	ESF_DZ_TX_VLAN_OP, tci ? 1 : 0,
	668	ESF_DZ_TX_VLAN_TAG1, tci);
	669	}
	670
9f95a23c TL	671	void
	672	ef10_tx_qdesc_checksum_create(
	673	__in efx_txq_t *etp,
	674	__in uint16_t flags,
	675	__out efx_desc_t *edp)
	676	{
	677	_NOTE(ARGUNUSED(etp));
	678
	679	EFSYS_PROBE2(tx_desc_checksum_create, unsigned int, etp->et_index,
	680	uint32_t, flags);
	681
	682	EFX_POPULATE_QWORD_6(edp->ed_eq,
	683	ESF_DZ_TX_DESC_IS_OPT, 1,
	684	ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_CRC_CSUM,
	685	ESF_DZ_TX_OPTION_UDP_TCP_CSUM,
	686	(flags & EFX_TXQ_CKSUM_TCPUDP) ? 1 : 0,
	687	ESF_DZ_TX_OPTION_IP_CSUM,
	688	(flags & EFX_TXQ_CKSUM_IPV4) ? 1 : 0,
	689	ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM,
	690	(flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0,
	691	ESF_DZ_TX_OPTION_INNER_IP_CSUM,
	692	(flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0);
	693	}
	694
11fdf7f2 TL	695
	696	__checkReturn efx_rc_t
	697	ef10_tx_qpace(
	698	__in efx_txq_t *etp,
	699	__in unsigned int ns)
	700	{
	701	efx_rc_t rc;
	702
	703	/* FIXME */
	704	_NOTE(ARGUNUSED(etp, ns))
	705	_NOTE(CONSTANTCONDITION)
	706	if (B_FALSE) {
	707	rc = ENOTSUP;
	708	goto fail1;
	709	}
	710	/* FIXME */
	711
	712	return (0);
	713
	714	fail1:
	715	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	716
	717	return (rc);
	718	}
	719
	720	__checkReturn efx_rc_t
	721	ef10_tx_qflush(
	722	__in efx_txq_t *etp)
	723	{
	724	efx_nic_t *enp = etp->et_enp;
	725	efx_rc_t rc;
	726
	727	if ((rc = efx_mcdi_fini_txq(enp, etp->et_index)) != 0)
	728	goto fail1;
	729
	730	return (0);
	731
	732	fail1:
9f95a23c TL	733	/*
	734	* EALREADY is not an error, but indicates that the MC has rebooted and
	735	* that the TXQ has already been destroyed. Callers need to know that
	736	* the TXQ flush has completed to avoid waiting until timeout for a
	737	* flush done event that will not be delivered.
	738	*/
	739	if (rc != EALREADY)
	740	EFSYS_PROBE1(fail1, efx_rc_t, rc);
11fdf7f2 TL	741
	742	return (rc);
	743	}
	744
	745	void
	746	ef10_tx_qenable(
	747	__in efx_txq_t *etp)
	748	{
	749	/* FIXME */
	750	_NOTE(ARGUNUSED(etp))
	751	/* FIXME */
	752	}
	753
	754	#if EFSYS_OPT_QSTATS
	755	void
	756	ef10_tx_qstats_update(
	757	__in efx_txq_t *etp,
	758	__inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
	759	{
	760	unsigned int id;
	761
	762	for (id = 0; id < TX_NQSTATS; id++) {
	763	efsys_stat_t *essp = &stat[id];
	764
	765	EFSYS_STAT_INCR(essp, etp->et_stat[id]);
	766	etp->et_stat[id] = 0;
	767	}
	768	}
	769
	770	#endif /* EFSYS_OPT_QSTATS */
	771
9f95a23c	772	#endif /* EFX_OPTS_EF10() */