[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_tx.c

/*
 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/tcp.h>
#include <linux/if_vlan.h>
#include "en.h"

#define MLX5E_SQ_NOPS_ROOM  MLX5_SEND_WQE_MAX_WQEBBS
#define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\
			    MLX5E_SQ_NOPS_ROOM)

void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw)
{
	struct mlx5_wq_cyc                *wq  = &sq->wq;

	u16 pi = sq->pc & wq->sz_m1;
	struct mlx5e_tx_wqe              *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);

	struct mlx5_wqe_ctrl_seg         *cseg = &wqe->ctrl;

	memset(cseg, 0, sizeof(*cseg));

	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | 0x01);

	sq->skb[pi] = NULL;
	sq->pc++;

	if (notify_hw) {
		cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
		mlx5e_tx_notify_hw(sq, wqe);
	}
}

static void mlx5e_dma_pop_last_pushed(struct mlx5e_sq *sq, dma_addr_t *addr,
				      u32 *size)
{
	sq->dma_fifo_pc--;
	*addr = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr;
	*size = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size;
}

static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq *sq, struct sk_buff *skb)
{
	dma_addr_t addr;
	u32 size;
	int i;

	for (i = 0; i < MLX5E_TX_SKB_CB(skb)->num_dma; i++) {
		mlx5e_dma_pop_last_pushed(sq, &addr, &size);
		dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE);
	}
}

static inline void mlx5e_dma_push(struct mlx5e_sq *sq, dma_addr_t addr,
				  u32 size)
{
	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr;
	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size;
	sq->dma_fifo_pc++;
}

static inline void mlx5e_dma_get(struct mlx5e_sq *sq, u32 i, dma_addr_t *addr,
				 u32 *size)
{
	*addr = sq->dma_fifo[i & sq->dma_fifo_mask].addr;
	*size = sq->dma_fifo[i & sq->dma_fifo_mask].size;
}

u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
		       void *accel_priv, select_queue_fallback_t fallback)
{
	struct mlx5e_priv *priv = netdev_priv(dev);
	int channel_ix = fallback(dev, skb);
	int up = skb_vlan_tag_present(skb)        ?
		 skb->vlan_tci >> VLAN_PRIO_SHIFT :
		 priv->default_vlan_prio;
	int tc = netdev_get_prio_tc_map(dev, up);

	return (tc << priv->order_base_2_num_channels) | channel_ix;
}

static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq,
					    struct sk_buff *skb)
{
#define MLX5E_MIN_INLINE 16 /* eth header with vlan (w/o next ethertype) */
	return MLX5E_MIN_INLINE;
}

static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
{
	struct mlx5_wq_cyc       *wq   = &sq->wq;

	u16 pi = sq->pc & wq->sz_m1;
	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);

	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
	struct mlx5_wqe_data_seg *dseg;

	u8  opcode = MLX5_OPCODE_SEND;
	dma_addr_t dma_addr = 0;
	u16 headlen;
	u16 ds_cnt;
	u16 ihs;
	int i;

	memset(wqe, 0, sizeof(*wqe));

	if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
		eseg->cs_flags	= MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
	else
		sq->stats.csum_offload_none++;

	if (skb_is_gso(skb)) {
		u32 payload_len;
		int num_pkts;

		eseg->mss    = cpu_to_be16(skb_shinfo(skb)->gso_size);
		opcode       = MLX5_OPCODE_LSO;
		ihs          = skb_transport_offset(skb) + tcp_hdrlen(skb);
		payload_len  = skb->len - ihs;
		num_pkts     =    (payload_len / skb_shinfo(skb)->gso_size) +
				!!(payload_len % skb_shinfo(skb)->gso_size);
		MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len +
						  (num_pkts - 1) * ihs;
		sq->stats.tso_packets++;
		sq->stats.tso_bytes += payload_len;
	} else {
		ihs             = mlx5e_get_inline_hdr_size(sq, skb);
		MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len,
							ETH_ZLEN);
	}

	skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs);
	skb_pull_inline(skb, ihs);

	eseg->inline_hdr_sz	= cpu_to_be16(ihs);

	ds_cnt  = sizeof(*wqe) / MLX5_SEND_WQE_DS;
	ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start),
			       MLX5_SEND_WQE_DS);
	dseg    = (struct mlx5_wqe_data_seg *)cseg + ds_cnt;

	MLX5E_TX_SKB_CB(skb)->num_dma = 0;

	headlen = skb_headlen(skb);
	if (headlen) {
		dma_addr = dma_map_single(sq->pdev, skb->data, headlen,
					  DMA_TO_DEVICE);
		if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
			goto dma_unmap_wqe_err;

		dseg->addr       = cpu_to_be64(dma_addr);
		dseg->lkey       = sq->mkey_be;
		dseg->byte_count = cpu_to_be32(headlen);

		mlx5e_dma_push(sq, dma_addr, headlen);
		MLX5E_TX_SKB_CB(skb)->num_dma++;

		dseg++;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
		int fsz = skb_frag_size(frag);

		dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
					    DMA_TO_DEVICE);
		if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
			goto dma_unmap_wqe_err;

		dseg->addr       = cpu_to_be64(dma_addr);
		dseg->lkey       = sq->mkey_be;
		dseg->byte_count = cpu_to_be32(fsz);

		mlx5e_dma_push(sq, dma_addr, fsz);
		MLX5E_TX_SKB_CB(skb)->num_dma++;

		dseg++;
	}

	ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma;

	cseg->opmod_idx_opcode	= cpu_to_be32((sq->pc << 8) | opcode);
	cseg->qpn_ds		= cpu_to_be32((sq->sqn << 8) | ds_cnt);
	cseg->fm_ce_se		= MLX5_WQE_CTRL_CQ_UPDATE;

	sq->skb[pi] = skb;

	MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt,
							MLX5_SEND_WQEBB_NUM_DS);
	sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;

	netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes);

	if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) {
		netif_tx_stop_queue(sq->txq);
		sq->stats.stopped++;
	}

	if (!skb->xmit_more || netif_xmit_stopped(sq->txq))
		mlx5e_tx_notify_hw(sq, wqe);

	/* fill sq edge with nops to avoid wqe wrap around */
	while ((sq->pc & wq->sz_m1) > sq->edge)
		mlx5e_send_nop(sq, false);

	sq->stats.packets++;
	return NETDEV_TX_OK;

dma_unmap_wqe_err:
	sq->stats.dropped++;
	mlx5e_dma_unmap_wqe_err(sq, skb);

	dev_kfree_skb_any(skb);

	return NETDEV_TX_OK;
}

netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct mlx5e_priv *priv = netdev_priv(dev);
	int ix = skb->queue_mapping;
	int tc = 0;
	struct mlx5e_channel *c = priv->channel[ix];
	struct mlx5e_sq *sq = &c->sq[tc];

	return mlx5e_sq_xmit(sq, skb);
}

netdev_tx_t mlx5e_xmit_multi_tc(struct sk_buff *skb, struct net_device *dev)
{
	struct mlx5e_priv *priv = netdev_priv(dev);
	int ix = skb->queue_mapping & priv->queue_mapping_channel_mask;
	int tc = skb->queue_mapping >> priv->order_base_2_num_channels;
	struct mlx5e_channel *c = priv->channel[ix];
	struct mlx5e_sq *sq = &c->sq[tc];

	return mlx5e_sq_xmit(sq, skb);
}

bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
{
	struct mlx5e_sq *sq;
	u32 dma_fifo_cc;
	u32 nbytes;
	u16 npkts;
	u16 sqcc;
	int i;

	/* avoid accessing cq (dma coherent memory) if not needed */
	if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags))
		return false;

	sq = cq->sqrq;

	npkts = 0;
	nbytes = 0;

	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	 * otherwise a cq overrun may occur
	 */
	sqcc = sq->cc;

	/* avoid dirtying sq cache line every cqe */
	dma_fifo_cc = sq->dma_fifo_cc;

	for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
		struct mlx5_cqe64 *cqe;
		struct sk_buff *skb;
		u16 ci;
		int j;

		cqe = mlx5e_get_cqe(cq);
		if (!cqe)
			break;

		ci = sqcc & sq->wq.sz_m1;
		skb = sq->skb[ci];

		if (unlikely(!skb)) { /* nop */
			sq->stats.nop++;
			sqcc++;
			goto free_skb;
		}

		for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) {
			dma_addr_t addr;
			u32 size;

			mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size);
			dma_fifo_cc++;
			dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE);
		}

		npkts++;
		nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes;
		sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;

free_skb:
		dev_kfree_skb(skb);
	}

	mlx5_cqwq_update_db_record(&cq->wq);

	/* ensure cq space is freed before enabling more cqes */
	wmb();

	sq->dma_fifo_cc = dma_fifo_cc;
	sq->cc = sqcc;

	netdev_tx_completed_queue(sq->txq, npkts, nbytes);

	if (netif_tx_queue_stopped(sq->txq) &&
	    mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) &&
	    likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) {
				netif_tx_wake_queue(sq->txq);
				sq->stats.wake++;
	}
	if (i == MLX5E_TX_CQ_POLL_BUDGET) {
		set_bit(MLX5E_CQ_HAS_CQES, &cq->flags);
		return true;
	}

	return false;
}
Commit	Line	Data
e586b3b0 AV	1	/*
	2	* Copyright (c) 2015, Mellanox Technologies. All rights reserved.
	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*/
	32
	33	#include <linux/tcp.h>
	34	#include <linux/if_vlan.h>
	35	#include "en.h"
	36
12be4b21 SM	37	#define MLX5E_SQ_NOPS_ROOM MLX5_SEND_WQE_MAX_WQEBBS
	38	#define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\
	39	MLX5E_SQ_NOPS_ROOM)
	40
	41	void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw)
	42	{
	43	struct mlx5_wq_cyc *wq = &sq->wq;
	44
	45	u16 pi = sq->pc & wq->sz_m1;
	46	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
	47
	48	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
	49
	50	memset(cseg, 0, sizeof(*cseg));
	51
	52	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_NOP);
	53	cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| 0x01);
	54
	55	sq->skb[pi] = NULL;
	56	sq->pc++;
	57
	58	if (notify_hw) {
	59	cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
	60	mlx5e_tx_notify_hw(sq, wqe);
	61	}
	62	}
	63
e586b3b0 AV	64	static void mlx5e_dma_pop_last_pushed(struct mlx5e_sq sq, dma_addr_t addr,
	65	u32 *size)
	66	{
	67	sq->dma_fifo_pc--;
	68	*addr = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr;
	69	*size = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size;
	70	}
	71
	72	static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq sq, struct sk_buff skb)
	73	{
	74	dma_addr_t addr;
	75	u32 size;
	76	int i;
	77
	78	for (i = 0; i < MLX5E_TX_SKB_CB(skb)->num_dma; i++) {
	79	mlx5e_dma_pop_last_pushed(sq, &addr, &size);
	80	dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE);
	81	}
	82	}
	83
	84	static inline void mlx5e_dma_push(struct mlx5e_sq *sq, dma_addr_t addr,
	85	u32 size)
	86	{
	87	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr;
	88	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size;
	89	sq->dma_fifo_pc++;
	90	}
	91
	92	static inline void mlx5e_dma_get(struct mlx5e_sq sq, u32 i, dma_addr_t addr,
	93	u32 *size)
	94	{
	95	*addr = sq->dma_fifo[i & sq->dma_fifo_mask].addr;
	96	*size = sq->dma_fifo[i & sq->dma_fifo_mask].size;
	97	}
	98
	99	u16 mlx5e_select_queue(struct net_device dev, struct sk_buff skb,
	100	void *accel_priv, select_queue_fallback_t fallback)
	101	{
	102	struct mlx5e_priv *priv = netdev_priv(dev);
	103	int channel_ix = fallback(dev, skb);
	104	int up = skb_vlan_tag_present(skb) ?
	105	skb->vlan_tci >> VLAN_PRIO_SHIFT :
	106	priv->default_vlan_prio;
	107	int tc = netdev_get_prio_tc_map(dev, up);
	108
	109	return (tc << priv->order_base_2_num_channels) \| channel_ix;
	110	}
	111
	112	static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq,
	113	struct sk_buff *skb)
	114	{
	115	#define MLX5E_MIN_INLINE 16 /* eth header with vlan (w/o next ethertype) */
	116	return MLX5E_MIN_INLINE;
	117	}
	118
e586b3b0 AV	119	static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq sq, struct sk_buff skb)
	120	{
	121	struct mlx5_wq_cyc *wq = &sq->wq;
	122
	123	u16 pi = sq->pc & wq->sz_m1;
	124	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
	125
	126	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
	127	struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
	128	struct mlx5_wqe_data_seg *dseg;
	129
	130	u8 opcode = MLX5_OPCODE_SEND;
	131	dma_addr_t dma_addr = 0;
	132	u16 headlen;
	133	u16 ds_cnt;
	134	u16 ihs;
	135	int i;
	136
	137	memset(wqe, 0, sizeof(*wqe));
	138
	139	if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
	140	eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM \| MLX5_ETH_WQE_L4_CSUM;
	141	else
	142	sq->stats.csum_offload_none++;
	143
	144	if (skb_is_gso(skb)) {
	145	u32 payload_len;
	146	int num_pkts;
	147
	148	eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
	149	opcode = MLX5_OPCODE_LSO;
	150	ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
	151	payload_len = skb->len - ihs;
	152	num_pkts = (payload_len / skb_shinfo(skb)->gso_size) +
	153	!!(payload_len % skb_shinfo(skb)->gso_size);
	154	MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len +
	155	(num_pkts - 1) * ihs;
	156	sq->stats.tso_packets++;
	157	sq->stats.tso_bytes += payload_len;
	158	} else {
	159	ihs = mlx5e_get_inline_hdr_size(sq, skb);
	160	MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len,
	161	ETH_ZLEN);
	162	}
	163
cd58c714 SM	164	skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs);
cd58c714 SM	165	skb_pull_inline(skb, ihs);
e586b3b0 AV	166
	167	eseg->inline_hdr_sz = cpu_to_be16(ihs);
	168
	169	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
	170	ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start),
	171	MLX5_SEND_WQE_DS);
	172	dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt;
	173
	174	MLX5E_TX_SKB_CB(skb)->num_dma = 0;
	175
	176	headlen = skb_headlen(skb);
	177	if (headlen) {
	178	dma_addr = dma_map_single(sq->pdev, skb->data, headlen,
	179	DMA_TO_DEVICE);
	180	if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
	181	goto dma_unmap_wqe_err;
	182
	183	dseg->addr = cpu_to_be64(dma_addr);
	184	dseg->lkey = sq->mkey_be;
	185	dseg->byte_count = cpu_to_be32(headlen);
	186
	187	mlx5e_dma_push(sq, dma_addr, headlen);
	188	MLX5E_TX_SKB_CB(skb)->num_dma++;
	189
	190	dseg++;
	191	}
	192
	193	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
	194	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
	195	int fsz = skb_frag_size(frag);
	196
	197	dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
	198	DMA_TO_DEVICE);
	199	if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
	200	goto dma_unmap_wqe_err;
	201
	202	dseg->addr = cpu_to_be64(dma_addr);
	203	dseg->lkey = sq->mkey_be;
	204	dseg->byte_count = cpu_to_be32(fsz);
	205
	206	mlx5e_dma_push(sq, dma_addr, fsz);
	207	MLX5E_TX_SKB_CB(skb)->num_dma++;
	208
	209	dseg++;
	210	}
	211
	212	ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma;
	213
	214	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| opcode);
	215	cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| ds_cnt);
	216	cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
	217
	218	sq->skb[pi] = skb;
	219
	220	MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt,
	221	MLX5_SEND_WQEBB_NUM_DS);
	222	sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
	223
	224	netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes);
	225
12be4b21	226	if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) {
e586b3b0 AV	227	netif_tx_stop_queue(sq->txq);
	228	sq->stats.stopped++;
	229	}
	230
	231	if (!skb->xmit_more \|\| netif_xmit_stopped(sq->txq))
	232	mlx5e_tx_notify_hw(sq, wqe);
	233
12be4b21 SM	234	/* fill sq edge with nops to avoid wqe wrap around */
	235	while ((sq->pc & wq->sz_m1) > sq->edge)
	236	mlx5e_send_nop(sq, false);
	237
e586b3b0 AV	238	sq->stats.packets++;
	239	return NETDEV_TX_OK;
	240
	241	dma_unmap_wqe_err:
	242	sq->stats.dropped++;
	243	mlx5e_dma_unmap_wqe_err(sq, skb);
	244
	245	dev_kfree_skb_any(skb);
	246
	247	return NETDEV_TX_OK;
	248	}
	249
	250	netdev_tx_t mlx5e_xmit(struct sk_buff skb, struct net_device dev)
	251	{
	252	struct mlx5e_priv *priv = netdev_priv(dev);
	253	int ix = skb->queue_mapping;
	254	int tc = 0;
	255	struct mlx5e_channel *c = priv->channel[ix];
	256	struct mlx5e_sq *sq = &c->sq[tc];
	257
	258	return mlx5e_sq_xmit(sq, skb);
	259	}
	260
	261	netdev_tx_t mlx5e_xmit_multi_tc(struct sk_buff skb, struct net_device dev)
	262	{
	263	struct mlx5e_priv *priv = netdev_priv(dev);
	264	int ix = skb->queue_mapping & priv->queue_mapping_channel_mask;
	265	int tc = skb->queue_mapping >> priv->order_base_2_num_channels;
	266	struct mlx5e_channel *c = priv->channel[ix];
	267	struct mlx5e_sq *sq = &c->sq[tc];
	268
	269	return mlx5e_sq_xmit(sq, skb);
	270	}
	271
	272	bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
	273	{
	274	struct mlx5e_sq *sq;
	275	u32 dma_fifo_cc;
	276	u32 nbytes;
	277	u16 npkts;
	278	u16 sqcc;
	279	int i;
	280
	281	/* avoid accessing cq (dma coherent memory) if not needed */
	282	if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags))
	283	return false;
	284
	285	sq = cq->sqrq;
	286
	287	npkts = 0;
	288	nbytes = 0;
	289
	290	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	291	* otherwise a cq overrun may occur
	292	*/
	293	sqcc = sq->cc;
	294
	295	/* avoid dirtying sq cache line every cqe */
	296	dma_fifo_cc = sq->dma_fifo_cc;
	297
	298	for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
	299	struct mlx5_cqe64 *cqe;
	300	struct sk_buff *skb;
	301	u16 ci;
302	int j;
303
304	cqe = mlx5e_get_cqe(cq);
305	if (!cqe)
306	break;
307
308	ci = sqcc & sq->wq.sz_m1;
309	skb = sq->skb[ci];
310
311	if (unlikely(!skb)) { /* nop */
312	sq->stats.nop++;
313	sqcc++;
314	goto free_skb;
315	}
316
317	for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) {
318	dma_addr_t addr;
319	u32 size;
320
321	mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size);
322	dma_fifo_cc++;
323	dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE);
324	}
325
326	npkts++;
327	nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes;
328	sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
329
330	free_skb:
331	dev_kfree_skb(skb);
332	}
333
334	mlx5_cqwq_update_db_record(&cq->wq);
335
336	/* ensure cq space is freed before enabling more cqes */
337	wmb();
338
339	sq->dma_fifo_cc = dma_fifo_cc;
340	sq->cc = sqcc;
341
342	netdev_tx_completed_queue(sq->txq, npkts, nbytes);
343
344	if (netif_tx_queue_stopped(sq->txq) &&
12be4b21	345	mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) &&
e586b3b0 AV	346	likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) {
	347	netif_tx_wake_queue(sq->txq);
	348	sq->stats.wake++;
	349	}
	350	if (i == MLX5E_TX_CQ_POLL_BUDGET) {
	351	set_bit(MLX5E_CQ_HAS_CQES, &cq->flags);
	352	return true;
	353	}
	354
	355	return false;
	356	}