2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <net/tc_act/tc_gact.h>
34 #include <net/pkt_cls.h>
35 #include <linux/mlx5/fs.h>
36 #include <net/vxlan.h>
37 #include <linux/bpf.h>
43 struct mlx5e_rq_param
{
44 u32 rqc
[MLX5_ST_SZ_DW(rqc
)];
45 struct mlx5_wq_param wq
;
49 struct mlx5e_sq_param
{
50 u32 sqc
[MLX5_ST_SZ_DW(sqc
)];
51 struct mlx5_wq_param wq
;
54 enum mlx5e_sq_type type
;
57 struct mlx5e_cq_param
{
58 u32 cqc
[MLX5_ST_SZ_DW(cqc
)];
59 struct mlx5_wq_param wq
;
64 struct mlx5e_channel_param
{
65 struct mlx5e_rq_param rq
;
66 struct mlx5e_sq_param sq
;
67 struct mlx5e_sq_param xdp_sq
;
68 struct mlx5e_sq_param icosq
;
69 struct mlx5e_cq_param rx_cq
;
70 struct mlx5e_cq_param tx_cq
;
71 struct mlx5e_cq_param icosq_cq
;
74 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev
*mdev
)
76 return MLX5_CAP_GEN(mdev
, striding_rq
) &&
77 MLX5_CAP_GEN(mdev
, umr_ptr_rlky
) &&
78 MLX5_CAP_ETH(mdev
, reg_umr_sq
);
81 static void mlx5e_set_rq_type_params(struct mlx5e_priv
*priv
, u8 rq_type
)
83 priv
->params
.rq_wq_type
= rq_type
;
84 switch (priv
->params
.rq_wq_type
) {
85 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
86 priv
->params
.log_rq_size
= MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW
;
87 priv
->params
.mpwqe_log_stride_sz
= priv
->params
.rx_cqe_compress
?
88 MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS
:
89 MLX5_MPWRQ_LOG_STRIDE_SIZE
;
90 priv
->params
.mpwqe_log_num_strides
= MLX5_MPWRQ_LOG_WQE_SZ
-
91 priv
->params
.mpwqe_log_stride_sz
;
93 default: /* MLX5_WQ_TYPE_LINKED_LIST */
94 priv
->params
.log_rq_size
= MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE
;
96 priv
->params
.min_rx_wqes
= mlx5_min_rx_wqes(priv
->params
.rq_wq_type
,
97 BIT(priv
->params
.log_rq_size
));
99 mlx5_core_info(priv
->mdev
,
100 "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
101 priv
->params
.rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
,
102 BIT(priv
->params
.log_rq_size
),
103 BIT(priv
->params
.mpwqe_log_stride_sz
),
104 priv
->params
.rx_cqe_compress_admin
);
107 static void mlx5e_set_rq_priv_params(struct mlx5e_priv
*priv
)
109 u8 rq_type
= mlx5e_check_fragmented_striding_rq_cap(priv
->mdev
) &&
111 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
112 MLX5_WQ_TYPE_LINKED_LIST
;
113 mlx5e_set_rq_type_params(priv
, rq_type
);
116 static void mlx5e_update_carrier(struct mlx5e_priv
*priv
)
118 struct mlx5_core_dev
*mdev
= priv
->mdev
;
121 port_state
= mlx5_query_vport_state(mdev
,
122 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT
, 0);
124 if (port_state
== VPORT_STATE_UP
) {
125 netdev_info(priv
->netdev
, "Link up\n");
126 netif_carrier_on(priv
->netdev
);
128 netdev_info(priv
->netdev
, "Link down\n");
129 netif_carrier_off(priv
->netdev
);
133 static void mlx5e_update_carrier_work(struct work_struct
*work
)
135 struct mlx5e_priv
*priv
= container_of(work
, struct mlx5e_priv
,
136 update_carrier_work
);
138 mutex_lock(&priv
->state_lock
);
139 if (test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
140 mlx5e_update_carrier(priv
);
141 mutex_unlock(&priv
->state_lock
);
144 static void mlx5e_tx_timeout_work(struct work_struct
*work
)
146 struct mlx5e_priv
*priv
= container_of(work
, struct mlx5e_priv
,
151 mutex_lock(&priv
->state_lock
);
152 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
154 mlx5e_close_locked(priv
->netdev
);
155 err
= mlx5e_open_locked(priv
->netdev
);
157 netdev_err(priv
->netdev
, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
160 mutex_unlock(&priv
->state_lock
);
164 static void mlx5e_update_sw_counters(struct mlx5e_priv
*priv
)
166 struct mlx5e_sw_stats
*s
= &priv
->stats
.sw
;
167 struct mlx5e_rq_stats
*rq_stats
;
168 struct mlx5e_sq_stats
*sq_stats
;
169 u64 tx_offload_none
= 0;
172 memset(s
, 0, sizeof(*s
));
173 for (i
= 0; i
< priv
->params
.num_channels
; i
++) {
174 rq_stats
= &priv
->channel
[i
]->rq
.stats
;
176 s
->rx_packets
+= rq_stats
->packets
;
177 s
->rx_bytes
+= rq_stats
->bytes
;
178 s
->rx_lro_packets
+= rq_stats
->lro_packets
;
179 s
->rx_lro_bytes
+= rq_stats
->lro_bytes
;
180 s
->rx_csum_none
+= rq_stats
->csum_none
;
181 s
->rx_csum_complete
+= rq_stats
->csum_complete
;
182 s
->rx_csum_unnecessary_inner
+= rq_stats
->csum_unnecessary_inner
;
183 s
->rx_xdp_drop
+= rq_stats
->xdp_drop
;
184 s
->rx_xdp_tx
+= rq_stats
->xdp_tx
;
185 s
->rx_xdp_tx_full
+= rq_stats
->xdp_tx_full
;
186 s
->rx_wqe_err
+= rq_stats
->wqe_err
;
187 s
->rx_mpwqe_filler
+= rq_stats
->mpwqe_filler
;
188 s
->rx_buff_alloc_err
+= rq_stats
->buff_alloc_err
;
189 s
->rx_cqe_compress_blks
+= rq_stats
->cqe_compress_blks
;
190 s
->rx_cqe_compress_pkts
+= rq_stats
->cqe_compress_pkts
;
191 s
->rx_cache_reuse
+= rq_stats
->cache_reuse
;
192 s
->rx_cache_full
+= rq_stats
->cache_full
;
193 s
->rx_cache_empty
+= rq_stats
->cache_empty
;
194 s
->rx_cache_busy
+= rq_stats
->cache_busy
;
196 for (j
= 0; j
< priv
->params
.num_tc
; j
++) {
197 sq_stats
= &priv
->channel
[i
]->sq
[j
].stats
;
199 s
->tx_packets
+= sq_stats
->packets
;
200 s
->tx_bytes
+= sq_stats
->bytes
;
201 s
->tx_tso_packets
+= sq_stats
->tso_packets
;
202 s
->tx_tso_bytes
+= sq_stats
->tso_bytes
;
203 s
->tx_tso_inner_packets
+= sq_stats
->tso_inner_packets
;
204 s
->tx_tso_inner_bytes
+= sq_stats
->tso_inner_bytes
;
205 s
->tx_queue_stopped
+= sq_stats
->stopped
;
206 s
->tx_queue_wake
+= sq_stats
->wake
;
207 s
->tx_queue_dropped
+= sq_stats
->dropped
;
208 s
->tx_xmit_more
+= sq_stats
->xmit_more
;
209 s
->tx_csum_partial_inner
+= sq_stats
->csum_partial_inner
;
210 tx_offload_none
+= sq_stats
->csum_none
;
214 /* Update calculated offload counters */
215 s
->tx_csum_partial
= s
->tx_packets
- tx_offload_none
- s
->tx_csum_partial_inner
;
216 s
->rx_csum_unnecessary
= s
->rx_packets
- s
->rx_csum_none
- s
->rx_csum_complete
;
218 s
->link_down_events_phy
= MLX5_GET(ppcnt_reg
,
219 priv
->stats
.pport
.phy_counters
,
220 counter_set
.phys_layer_cntrs
.link_down_events
);
223 static void mlx5e_update_vport_counters(struct mlx5e_priv
*priv
)
225 int outlen
= MLX5_ST_SZ_BYTES(query_vport_counter_out
);
226 u32
*out
= (u32
*)priv
->stats
.vport
.query_vport_out
;
227 u32 in
[MLX5_ST_SZ_DW(query_vport_counter_in
)] = {0};
228 struct mlx5_core_dev
*mdev
= priv
->mdev
;
230 MLX5_SET(query_vport_counter_in
, in
, opcode
,
231 MLX5_CMD_OP_QUERY_VPORT_COUNTER
);
232 MLX5_SET(query_vport_counter_in
, in
, op_mod
, 0);
233 MLX5_SET(query_vport_counter_in
, in
, other_vport
, 0);
235 memset(out
, 0, outlen
);
236 mlx5_cmd_exec(mdev
, in
, sizeof(in
), out
, outlen
);
239 static void mlx5e_update_pport_counters(struct mlx5e_priv
*priv
)
241 struct mlx5e_pport_stats
*pstats
= &priv
->stats
.pport
;
242 struct mlx5_core_dev
*mdev
= priv
->mdev
;
243 int sz
= MLX5_ST_SZ_BYTES(ppcnt_reg
);
248 in
= mlx5_vzalloc(sz
);
252 MLX5_SET(ppcnt_reg
, in
, local_port
, 1);
254 out
= pstats
->IEEE_802_3_counters
;
255 MLX5_SET(ppcnt_reg
, in
, grp
, MLX5_IEEE_802_3_COUNTERS_GROUP
);
256 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_PPCNT
, 0, 0);
258 out
= pstats
->RFC_2863_counters
;
259 MLX5_SET(ppcnt_reg
, in
, grp
, MLX5_RFC_2863_COUNTERS_GROUP
);
260 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_PPCNT
, 0, 0);
262 out
= pstats
->RFC_2819_counters
;
263 MLX5_SET(ppcnt_reg
, in
, grp
, MLX5_RFC_2819_COUNTERS_GROUP
);
264 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_PPCNT
, 0, 0);
266 out
= pstats
->phy_counters
;
267 MLX5_SET(ppcnt_reg
, in
, grp
, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP
);
268 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_PPCNT
, 0, 0);
270 MLX5_SET(ppcnt_reg
, in
, grp
, MLX5_PER_PRIORITY_COUNTERS_GROUP
);
271 for (prio
= 0; prio
< NUM_PPORT_PRIO
; prio
++) {
272 out
= pstats
->per_prio_counters
[prio
];
273 MLX5_SET(ppcnt_reg
, in
, prio_tc
, prio
);
274 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
,
275 MLX5_REG_PPCNT
, 0, 0);
282 static void mlx5e_update_q_counter(struct mlx5e_priv
*priv
)
284 struct mlx5e_qcounter_stats
*qcnt
= &priv
->stats
.qcnt
;
286 if (!priv
->q_counter
)
289 mlx5_core_query_out_of_buffer(priv
->mdev
, priv
->q_counter
,
290 &qcnt
->rx_out_of_buffer
);
293 static void mlx5e_update_pcie_counters(struct mlx5e_priv
*priv
)
295 struct mlx5e_pcie_stats
*pcie_stats
= &priv
->stats
.pcie
;
296 struct mlx5_core_dev
*mdev
= priv
->mdev
;
297 int sz
= MLX5_ST_SZ_BYTES(mpcnt_reg
);
301 in
= mlx5_vzalloc(sz
);
305 out
= pcie_stats
->pcie_perf_counters
;
306 MLX5_SET(mpcnt_reg
, in
, grp
, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP
);
307 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_MPCNT
, 0, 0);
309 out
= pcie_stats
->pcie_tas_counters
;
310 MLX5_SET(mpcnt_reg
, in
, grp
, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP
);
311 mlx5_core_access_reg(mdev
, in
, sz
, out
, sz
, MLX5_REG_MPCNT
, 0, 0);
316 void mlx5e_update_stats(struct mlx5e_priv
*priv
)
318 mlx5e_update_q_counter(priv
);
319 mlx5e_update_vport_counters(priv
);
320 mlx5e_update_pport_counters(priv
);
321 mlx5e_update_sw_counters(priv
);
322 mlx5e_update_pcie_counters(priv
);
325 void mlx5e_update_stats_work(struct work_struct
*work
)
327 struct delayed_work
*dwork
= to_delayed_work(work
);
328 struct mlx5e_priv
*priv
= container_of(dwork
, struct mlx5e_priv
,
330 mutex_lock(&priv
->state_lock
);
331 if (test_bit(MLX5E_STATE_OPENED
, &priv
->state
)) {
332 priv
->profile
->update_stats(priv
);
333 queue_delayed_work(priv
->wq
, dwork
,
334 msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL
));
336 mutex_unlock(&priv
->state_lock
);
339 static void mlx5e_async_event(struct mlx5_core_dev
*mdev
, void *vpriv
,
340 enum mlx5_dev_event event
, unsigned long param
)
342 struct mlx5e_priv
*priv
= vpriv
;
344 if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED
, &priv
->state
))
348 case MLX5_DEV_EVENT_PORT_UP
:
349 case MLX5_DEV_EVENT_PORT_DOWN
:
350 queue_work(priv
->wq
, &priv
->update_carrier_work
);
358 static void mlx5e_enable_async_events(struct mlx5e_priv
*priv
)
360 set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED
, &priv
->state
);
363 static void mlx5e_disable_async_events(struct mlx5e_priv
*priv
)
365 clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED
, &priv
->state
);
366 synchronize_irq(mlx5_get_msix_vec(priv
->mdev
, MLX5_EQ_VEC_ASYNC
));
369 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
370 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
372 static inline int mlx5e_get_wqe_mtt_sz(void)
374 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
375 * To avoid copying garbage after the mtt array, we allocate
378 return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE
* sizeof(__be64
),
379 MLX5_UMR_MTT_ALIGNMENT
);
382 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq
*rq
, struct mlx5e_sq
*sq
,
383 struct mlx5e_umr_wqe
*wqe
, u16 ix
)
385 struct mlx5_wqe_ctrl_seg
*cseg
= &wqe
->ctrl
;
386 struct mlx5_wqe_umr_ctrl_seg
*ucseg
= &wqe
->uctrl
;
387 struct mlx5_wqe_data_seg
*dseg
= &wqe
->data
;
388 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
389 u8 ds_cnt
= DIV_ROUND_UP(sizeof(*wqe
), MLX5_SEND_WQE_DS
);
390 u32 umr_wqe_mtt_offset
= mlx5e_get_wqe_mtt_offset(rq
, ix
);
392 cseg
->qpn_ds
= cpu_to_be32((sq
->sqn
<< MLX5_WQE_CTRL_QPN_SHIFT
) |
394 cseg
->fm_ce_se
= MLX5_WQE_CTRL_CQ_UPDATE
;
395 cseg
->imm
= rq
->mkey_be
;
397 ucseg
->flags
= MLX5_UMR_TRANSLATION_OFFSET_EN
;
398 ucseg
->klm_octowords
=
399 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE
));
400 ucseg
->bsf_octowords
=
401 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset
));
402 ucseg
->mkey_mask
= cpu_to_be64(MLX5_MKEY_MASK_FREE
);
404 dseg
->lkey
= sq
->mkey_be
;
405 dseg
->addr
= cpu_to_be64(wi
->umr
.mtt_addr
);
408 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq
*rq
,
409 struct mlx5e_channel
*c
)
411 int wq_sz
= mlx5_wq_ll_get_size(&rq
->wq
);
412 int mtt_sz
= mlx5e_get_wqe_mtt_sz();
413 int mtt_alloc
= mtt_sz
+ MLX5_UMR_ALIGN
- 1;
416 rq
->mpwqe
.info
= kzalloc_node(wq_sz
* sizeof(*rq
->mpwqe
.info
),
417 GFP_KERNEL
, cpu_to_node(c
->cpu
));
421 /* We allocate more than mtt_sz as we will align the pointer */
422 rq
->mpwqe
.mtt_no_align
= kzalloc_node(mtt_alloc
* wq_sz
, GFP_KERNEL
,
423 cpu_to_node(c
->cpu
));
424 if (unlikely(!rq
->mpwqe
.mtt_no_align
))
425 goto err_free_wqe_info
;
427 for (i
= 0; i
< wq_sz
; i
++) {
428 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[i
];
430 wi
->umr
.mtt
= PTR_ALIGN(rq
->mpwqe
.mtt_no_align
+ i
* mtt_alloc
,
432 wi
->umr
.mtt_addr
= dma_map_single(c
->pdev
, wi
->umr
.mtt
, mtt_sz
,
434 if (unlikely(dma_mapping_error(c
->pdev
, wi
->umr
.mtt_addr
)))
437 mlx5e_build_umr_wqe(rq
, &c
->icosq
, &wi
->umr
.wqe
, i
);
444 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[i
];
446 dma_unmap_single(c
->pdev
, wi
->umr
.mtt_addr
, mtt_sz
,
449 kfree(rq
->mpwqe
.mtt_no_align
);
451 kfree(rq
->mpwqe
.info
);
457 static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq
*rq
)
459 int wq_sz
= mlx5_wq_ll_get_size(&rq
->wq
);
460 int mtt_sz
= mlx5e_get_wqe_mtt_sz();
463 for (i
= 0; i
< wq_sz
; i
++) {
464 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[i
];
466 dma_unmap_single(rq
->pdev
, wi
->umr
.mtt_addr
, mtt_sz
,
469 kfree(rq
->mpwqe
.mtt_no_align
);
470 kfree(rq
->mpwqe
.info
);
473 static int mlx5e_create_rq(struct mlx5e_channel
*c
,
474 struct mlx5e_rq_param
*param
,
477 struct mlx5e_priv
*priv
= c
->priv
;
478 struct mlx5_core_dev
*mdev
= priv
->mdev
;
479 void *rqc
= param
->rqc
;
480 void *rqc_wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
488 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
490 err
= mlx5_wq_ll_create(mdev
, ¶m
->wq
, rqc_wq
, &rq
->wq
,
495 rq
->wq
.db
= &rq
->wq
.db
[MLX5_RCV_DBR
];
497 wq_sz
= mlx5_wq_ll_get_size(&rq
->wq
);
499 rq
->wq_type
= priv
->params
.rq_wq_type
;
501 rq
->netdev
= c
->netdev
;
502 rq
->tstamp
= &priv
->tstamp
;
507 rq
->xdp_prog
= priv
->xdp_prog
? bpf_prog_inc(priv
->xdp_prog
) : NULL
;
508 if (IS_ERR(rq
->xdp_prog
)) {
509 err
= PTR_ERR(rq
->xdp_prog
);
511 goto err_rq_wq_destroy
;
514 rq
->buff
.map_dir
= DMA_FROM_DEVICE
;
516 rq
->buff
.map_dir
= DMA_BIDIRECTIONAL
;
518 switch (priv
->params
.rq_wq_type
) {
519 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
520 if (mlx5e_is_vf_vport_rep(priv
)) {
522 goto err_rq_wq_destroy
;
525 rq
->handle_rx_cqe
= mlx5e_handle_rx_cqe_mpwrq
;
526 rq
->alloc_wqe
= mlx5e_alloc_rx_mpwqe
;
527 rq
->dealloc_wqe
= mlx5e_dealloc_rx_mpwqe
;
529 rq
->mpwqe
.mtt_offset
= c
->ix
*
530 MLX5E_REQUIRED_MTTS(1, BIT(priv
->params
.log_rq_size
));
532 rq
->mpwqe_stride_sz
= BIT(priv
->params
.mpwqe_log_stride_sz
);
533 rq
->mpwqe_num_strides
= BIT(priv
->params
.mpwqe_log_num_strides
);
535 rq
->buff
.wqe_sz
= rq
->mpwqe_stride_sz
* rq
->mpwqe_num_strides
;
536 byte_count
= rq
->buff
.wqe_sz
;
537 rq
->mkey_be
= cpu_to_be32(c
->priv
->umr_mkey
.key
);
538 err
= mlx5e_rq_alloc_mpwqe_info(rq
, c
);
540 goto err_rq_wq_destroy
;
542 default: /* MLX5_WQ_TYPE_LINKED_LIST */
543 rq
->dma_info
= kzalloc_node(wq_sz
* sizeof(*rq
->dma_info
),
544 GFP_KERNEL
, cpu_to_node(c
->cpu
));
547 goto err_rq_wq_destroy
;
550 if (mlx5e_is_vf_vport_rep(priv
))
551 rq
->handle_rx_cqe
= mlx5e_handle_rx_cqe_rep
;
553 rq
->handle_rx_cqe
= mlx5e_handle_rx_cqe
;
555 rq
->alloc_wqe
= mlx5e_alloc_rx_wqe
;
556 rq
->dealloc_wqe
= mlx5e_dealloc_rx_wqe
;
558 rq
->buff
.wqe_sz
= (priv
->params
.lro_en
) ?
559 priv
->params
.lro_wqe_sz
:
560 MLX5E_SW2HW_MTU(priv
->netdev
->mtu
);
561 byte_count
= rq
->buff
.wqe_sz
;
563 /* calc the required page order */
564 frag_sz
= MLX5_RX_HEADROOM
+
565 byte_count
/* packet data */ +
566 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
567 frag_sz
= SKB_DATA_ALIGN(frag_sz
);
569 npages
= DIV_ROUND_UP(frag_sz
, PAGE_SIZE
);
570 rq
->buff
.page_order
= order_base_2(npages
);
572 byte_count
|= MLX5_HW_START_PADDING
;
573 rq
->mkey_be
= c
->mkey_be
;
576 for (i
= 0; i
< wq_sz
; i
++) {
577 struct mlx5e_rx_wqe
*wqe
= mlx5_wq_ll_get_wqe(&rq
->wq
, i
);
579 wqe
->data
.byte_count
= cpu_to_be32(byte_count
);
580 wqe
->data
.lkey
= rq
->mkey_be
;
583 INIT_WORK(&rq
->am
.work
, mlx5e_rx_am_work
);
584 rq
->am
.mode
= priv
->params
.rx_cq_period_mode
;
586 rq
->page_cache
.head
= 0;
587 rq
->page_cache
.tail
= 0;
593 bpf_prog_put(rq
->xdp_prog
);
594 mlx5_wq_destroy(&rq
->wq_ctrl
);
599 static void mlx5e_destroy_rq(struct mlx5e_rq
*rq
)
604 bpf_prog_put(rq
->xdp_prog
);
606 switch (rq
->wq_type
) {
607 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
608 mlx5e_rq_free_mpwqe_info(rq
);
610 default: /* MLX5_WQ_TYPE_LINKED_LIST */
614 for (i
= rq
->page_cache
.head
; i
!= rq
->page_cache
.tail
;
615 i
= (i
+ 1) & (MLX5E_CACHE_SIZE
- 1)) {
616 struct mlx5e_dma_info
*dma_info
= &rq
->page_cache
.page_cache
[i
];
618 mlx5e_page_release(rq
, dma_info
, false);
620 mlx5_wq_destroy(&rq
->wq_ctrl
);
623 static int mlx5e_enable_rq(struct mlx5e_rq
*rq
, struct mlx5e_rq_param
*param
)
625 struct mlx5e_priv
*priv
= rq
->priv
;
626 struct mlx5_core_dev
*mdev
= priv
->mdev
;
634 inlen
= MLX5_ST_SZ_BYTES(create_rq_in
) +
635 sizeof(u64
) * rq
->wq_ctrl
.buf
.npages
;
636 in
= mlx5_vzalloc(inlen
);
640 rqc
= MLX5_ADDR_OF(create_rq_in
, in
, ctx
);
641 wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
643 memcpy(rqc
, param
->rqc
, sizeof(param
->rqc
));
645 MLX5_SET(rqc
, rqc
, cqn
, rq
->cq
.mcq
.cqn
);
646 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RST
);
647 MLX5_SET(rqc
, rqc
, vsd
, priv
->params
.vlan_strip_disable
);
648 MLX5_SET(wq
, wq
, log_wq_pg_sz
, rq
->wq_ctrl
.buf
.page_shift
-
649 MLX5_ADAPTER_PAGE_SHIFT
);
650 MLX5_SET64(wq
, wq
, dbr_addr
, rq
->wq_ctrl
.db
.dma
);
652 mlx5_fill_page_array(&rq
->wq_ctrl
.buf
,
653 (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
));
655 err
= mlx5_core_create_rq(mdev
, in
, inlen
, &rq
->rqn
);
662 static int mlx5e_modify_rq_state(struct mlx5e_rq
*rq
, int curr_state
,
665 struct mlx5e_channel
*c
= rq
->channel
;
666 struct mlx5e_priv
*priv
= c
->priv
;
667 struct mlx5_core_dev
*mdev
= priv
->mdev
;
674 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
675 in
= mlx5_vzalloc(inlen
);
679 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
681 MLX5_SET(modify_rq_in
, in
, rq_state
, curr_state
);
682 MLX5_SET(rqc
, rqc
, state
, next_state
);
684 err
= mlx5_core_modify_rq(mdev
, rq
->rqn
, in
, inlen
);
691 static int mlx5e_modify_rq_vsd(struct mlx5e_rq
*rq
, bool vsd
)
693 struct mlx5e_channel
*c
= rq
->channel
;
694 struct mlx5e_priv
*priv
= c
->priv
;
695 struct mlx5_core_dev
*mdev
= priv
->mdev
;
702 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
703 in
= mlx5_vzalloc(inlen
);
707 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
709 MLX5_SET(modify_rq_in
, in
, rq_state
, MLX5_RQC_STATE_RDY
);
710 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
711 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD
);
712 MLX5_SET(rqc
, rqc
, vsd
, vsd
);
713 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RDY
);
715 err
= mlx5_core_modify_rq(mdev
, rq
->rqn
, in
, inlen
);
722 static void mlx5e_disable_rq(struct mlx5e_rq
*rq
)
724 mlx5_core_destroy_rq(rq
->priv
->mdev
, rq
->rqn
);
727 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq
*rq
)
729 unsigned long exp_time
= jiffies
+ msecs_to_jiffies(20000);
730 struct mlx5e_channel
*c
= rq
->channel
;
731 struct mlx5e_priv
*priv
= c
->priv
;
732 struct mlx5_wq_ll
*wq
= &rq
->wq
;
734 while (time_before(jiffies
, exp_time
)) {
735 if (wq
->cur_sz
>= priv
->params
.min_rx_wqes
)
744 static void mlx5e_free_rx_descs(struct mlx5e_rq
*rq
)
746 struct mlx5_wq_ll
*wq
= &rq
->wq
;
747 struct mlx5e_rx_wqe
*wqe
;
751 /* UMR WQE (if in progress) is always at wq->head */
752 if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS
, &rq
->state
))
753 mlx5e_free_rx_mpwqe(rq
, &rq
->mpwqe
.info
[wq
->head
]);
755 while (!mlx5_wq_ll_is_empty(wq
)) {
756 wqe_ix_be
= *wq
->tail_next
;
757 wqe_ix
= be16_to_cpu(wqe_ix_be
);
758 wqe
= mlx5_wq_ll_get_wqe(&rq
->wq
, wqe_ix
);
759 rq
->dealloc_wqe(rq
, wqe_ix
);
760 mlx5_wq_ll_pop(&rq
->wq
, wqe_ix_be
,
761 &wqe
->next
.next_wqe_index
);
765 static int mlx5e_open_rq(struct mlx5e_channel
*c
,
766 struct mlx5e_rq_param
*param
,
769 struct mlx5e_sq
*sq
= &c
->icosq
;
770 u16 pi
= sq
->pc
& sq
->wq
.sz_m1
;
773 err
= mlx5e_create_rq(c
, param
, rq
);
777 err
= mlx5e_enable_rq(rq
, param
);
781 err
= mlx5e_modify_rq_state(rq
, MLX5_RQC_STATE_RST
, MLX5_RQC_STATE_RDY
);
785 if (param
->am_enabled
)
786 set_bit(MLX5E_RQ_STATE_AM
, &c
->rq
.state
);
788 sq
->db
.ico_wqe
[pi
].opcode
= MLX5_OPCODE_NOP
;
789 sq
->db
.ico_wqe
[pi
].num_wqebbs
= 1;
790 mlx5e_send_nop(sq
, true); /* trigger mlx5e_post_rx_wqes() */
795 mlx5e_disable_rq(rq
);
797 mlx5e_destroy_rq(rq
);
802 static void mlx5e_close_rq(struct mlx5e_rq
*rq
)
804 set_bit(MLX5E_RQ_STATE_FLUSH
, &rq
->state
);
805 napi_synchronize(&rq
->channel
->napi
); /* prevent mlx5e_post_rx_wqes */
806 cancel_work_sync(&rq
->am
.work
);
808 mlx5e_disable_rq(rq
);
809 mlx5e_free_rx_descs(rq
);
810 mlx5e_destroy_rq(rq
);
813 static void mlx5e_free_sq_xdp_db(struct mlx5e_sq
*sq
)
815 kfree(sq
->db
.xdp
.di
);
816 kfree(sq
->db
.xdp
.wqe_info
);
819 static int mlx5e_alloc_sq_xdp_db(struct mlx5e_sq
*sq
, int numa
)
821 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
823 sq
->db
.xdp
.di
= kzalloc_node(sizeof(*sq
->db
.xdp
.di
) * wq_sz
,
825 sq
->db
.xdp
.wqe_info
= kzalloc_node(sizeof(*sq
->db
.xdp
.wqe_info
) * wq_sz
,
827 if (!sq
->db
.xdp
.di
|| !sq
->db
.xdp
.wqe_info
) {
828 mlx5e_free_sq_xdp_db(sq
);
835 static void mlx5e_free_sq_ico_db(struct mlx5e_sq
*sq
)
837 kfree(sq
->db
.ico_wqe
);
840 static int mlx5e_alloc_sq_ico_db(struct mlx5e_sq
*sq
, int numa
)
842 u8 wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
844 sq
->db
.ico_wqe
= kzalloc_node(sizeof(*sq
->db
.ico_wqe
) * wq_sz
,
852 static void mlx5e_free_sq_txq_db(struct mlx5e_sq
*sq
)
854 kfree(sq
->db
.txq
.wqe_info
);
855 kfree(sq
->db
.txq
.dma_fifo
);
856 kfree(sq
->db
.txq
.skb
);
859 static int mlx5e_alloc_sq_txq_db(struct mlx5e_sq
*sq
, int numa
)
861 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
862 int df_sz
= wq_sz
* MLX5_SEND_WQEBB_NUM_DS
;
864 sq
->db
.txq
.skb
= kzalloc_node(wq_sz
* sizeof(*sq
->db
.txq
.skb
),
866 sq
->db
.txq
.dma_fifo
= kzalloc_node(df_sz
* sizeof(*sq
->db
.txq
.dma_fifo
),
868 sq
->db
.txq
.wqe_info
= kzalloc_node(wq_sz
* sizeof(*sq
->db
.txq
.wqe_info
),
870 if (!sq
->db
.txq
.skb
|| !sq
->db
.txq
.dma_fifo
|| !sq
->db
.txq
.wqe_info
) {
871 mlx5e_free_sq_txq_db(sq
);
875 sq
->dma_fifo_mask
= df_sz
- 1;
880 static void mlx5e_free_sq_db(struct mlx5e_sq
*sq
)
884 mlx5e_free_sq_txq_db(sq
);
887 mlx5e_free_sq_ico_db(sq
);
890 mlx5e_free_sq_xdp_db(sq
);
895 static int mlx5e_alloc_sq_db(struct mlx5e_sq
*sq
, int numa
)
899 return mlx5e_alloc_sq_txq_db(sq
, numa
);
901 return mlx5e_alloc_sq_ico_db(sq
, numa
);
903 return mlx5e_alloc_sq_xdp_db(sq
, numa
);
909 static int mlx5e_sq_get_max_wqebbs(u8 sq_type
)
913 return MLX5E_ICOSQ_MAX_WQEBBS
;
915 return MLX5E_XDP_TX_WQEBBS
;
917 return MLX5_SEND_WQE_MAX_WQEBBS
;
920 static int mlx5e_create_sq(struct mlx5e_channel
*c
,
922 struct mlx5e_sq_param
*param
,
925 struct mlx5e_priv
*priv
= c
->priv
;
926 struct mlx5_core_dev
*mdev
= priv
->mdev
;
928 void *sqc
= param
->sqc
;
929 void *sqc_wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
932 sq
->type
= param
->type
;
934 sq
->tstamp
= &priv
->tstamp
;
935 sq
->mkey_be
= c
->mkey_be
;
939 err
= mlx5_alloc_map_uar(mdev
, &sq
->uar
, !!MLX5_CAP_GEN(mdev
, bf
));
943 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
945 err
= mlx5_wq_cyc_create(mdev
, ¶m
->wq
, sqc_wq
, &sq
->wq
,
948 goto err_unmap_free_uar
;
950 sq
->wq
.db
= &sq
->wq
.db
[MLX5_SND_DBR
];
951 if (sq
->uar
.bf_map
) {
952 set_bit(MLX5E_SQ_STATE_BF_ENABLE
, &sq
->state
);
953 sq
->uar_map
= sq
->uar
.bf_map
;
955 sq
->uar_map
= sq
->uar
.map
;
957 sq
->bf_buf_size
= (1 << MLX5_CAP_GEN(mdev
, log_bf_reg_size
)) / 2;
958 sq
->max_inline
= param
->max_inline
;
959 sq
->min_inline_mode
=
960 MLX5_CAP_ETH(mdev
, wqe_inline_mode
) == MLX5E_INLINE_MODE_VPORT_CONTEXT
?
961 param
->min_inline_mode
: 0;
963 err
= mlx5e_alloc_sq_db(sq
, cpu_to_node(c
->cpu
));
965 goto err_sq_wq_destroy
;
967 if (sq
->type
== MLX5E_SQ_TXQ
) {
970 txq_ix
= c
->ix
+ tc
* priv
->params
.num_channels
;
971 sq
->txq
= netdev_get_tx_queue(priv
->netdev
, txq_ix
);
972 priv
->txq_to_sq_map
[txq_ix
] = sq
;
975 sq
->edge
= (sq
->wq
.sz_m1
+ 1) - mlx5e_sq_get_max_wqebbs(sq
->type
);
976 sq
->bf_budget
= MLX5E_SQ_BF_BUDGET
;
981 mlx5_wq_destroy(&sq
->wq_ctrl
);
984 mlx5_unmap_free_uar(mdev
, &sq
->uar
);
989 static void mlx5e_destroy_sq(struct mlx5e_sq
*sq
)
991 struct mlx5e_channel
*c
= sq
->channel
;
992 struct mlx5e_priv
*priv
= c
->priv
;
994 mlx5e_free_sq_db(sq
);
995 mlx5_wq_destroy(&sq
->wq_ctrl
);
996 mlx5_unmap_free_uar(priv
->mdev
, &sq
->uar
);
999 static int mlx5e_enable_sq(struct mlx5e_sq
*sq
, struct mlx5e_sq_param
*param
)
1001 struct mlx5e_channel
*c
= sq
->channel
;
1002 struct mlx5e_priv
*priv
= c
->priv
;
1003 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1011 inlen
= MLX5_ST_SZ_BYTES(create_sq_in
) +
1012 sizeof(u64
) * sq
->wq_ctrl
.buf
.npages
;
1013 in
= mlx5_vzalloc(inlen
);
1017 sqc
= MLX5_ADDR_OF(create_sq_in
, in
, ctx
);
1018 wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1020 memcpy(sqc
, param
->sqc
, sizeof(param
->sqc
));
1022 MLX5_SET(sqc
, sqc
, tis_num_0
, param
->type
== MLX5E_SQ_ICO
?
1023 0 : priv
->tisn
[sq
->tc
]);
1024 MLX5_SET(sqc
, sqc
, cqn
, sq
->cq
.mcq
.cqn
);
1025 MLX5_SET(sqc
, sqc
, min_wqe_inline_mode
, sq
->min_inline_mode
);
1026 MLX5_SET(sqc
, sqc
, state
, MLX5_SQC_STATE_RST
);
1027 MLX5_SET(sqc
, sqc
, tis_lst_sz
, param
->type
== MLX5E_SQ_ICO
? 0 : 1);
1028 MLX5_SET(sqc
, sqc
, flush_in_error_en
, 1);
1030 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_CYCLIC
);
1031 MLX5_SET(wq
, wq
, uar_page
, sq
->uar
.index
);
1032 MLX5_SET(wq
, wq
, log_wq_pg_sz
, sq
->wq_ctrl
.buf
.page_shift
-
1033 MLX5_ADAPTER_PAGE_SHIFT
);
1034 MLX5_SET64(wq
, wq
, dbr_addr
, sq
->wq_ctrl
.db
.dma
);
1036 mlx5_fill_page_array(&sq
->wq_ctrl
.buf
,
1037 (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
));
1039 err
= mlx5_core_create_sq(mdev
, in
, inlen
, &sq
->sqn
);
1046 static int mlx5e_modify_sq(struct mlx5e_sq
*sq
, int curr_state
,
1047 int next_state
, bool update_rl
, int rl_index
)
1049 struct mlx5e_channel
*c
= sq
->channel
;
1050 struct mlx5e_priv
*priv
= c
->priv
;
1051 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1058 inlen
= MLX5_ST_SZ_BYTES(modify_sq_in
);
1059 in
= mlx5_vzalloc(inlen
);
1063 sqc
= MLX5_ADDR_OF(modify_sq_in
, in
, ctx
);
1065 MLX5_SET(modify_sq_in
, in
, sq_state
, curr_state
);
1066 MLX5_SET(sqc
, sqc
, state
, next_state
);
1067 if (update_rl
&& next_state
== MLX5_SQC_STATE_RDY
) {
1068 MLX5_SET64(modify_sq_in
, in
, modify_bitmask
, 1);
1069 MLX5_SET(sqc
, sqc
, packet_pacing_rate_limit_index
, rl_index
);
1072 err
= mlx5_core_modify_sq(mdev
, sq
->sqn
, in
, inlen
);
1079 static void mlx5e_disable_sq(struct mlx5e_sq
*sq
)
1081 struct mlx5e_channel
*c
= sq
->channel
;
1082 struct mlx5e_priv
*priv
= c
->priv
;
1083 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1085 mlx5_core_destroy_sq(mdev
, sq
->sqn
);
1087 mlx5_rl_remove_rate(mdev
, sq
->rate_limit
);
1090 static int mlx5e_open_sq(struct mlx5e_channel
*c
,
1092 struct mlx5e_sq_param
*param
,
1093 struct mlx5e_sq
*sq
)
1097 err
= mlx5e_create_sq(c
, tc
, param
, sq
);
1101 err
= mlx5e_enable_sq(sq
, param
);
1103 goto err_destroy_sq
;
1105 err
= mlx5e_modify_sq(sq
, MLX5_SQC_STATE_RST
, MLX5_SQC_STATE_RDY
,
1108 goto err_disable_sq
;
1111 netdev_tx_reset_queue(sq
->txq
);
1112 netif_tx_start_queue(sq
->txq
);
1118 mlx5e_disable_sq(sq
);
1120 mlx5e_destroy_sq(sq
);
1125 static inline void netif_tx_disable_queue(struct netdev_queue
*txq
)
1127 __netif_tx_lock_bh(txq
);
1128 netif_tx_stop_queue(txq
);
1129 __netif_tx_unlock_bh(txq
);
1132 static void mlx5e_close_sq(struct mlx5e_sq
*sq
)
1134 set_bit(MLX5E_SQ_STATE_FLUSH
, &sq
->state
);
1135 /* prevent netif_tx_wake_queue */
1136 napi_synchronize(&sq
->channel
->napi
);
1139 netif_tx_disable_queue(sq
->txq
);
1141 /* last doorbell out, godspeed .. */
1142 if (mlx5e_sq_has_room_for(sq
, 1)) {
1143 sq
->db
.txq
.skb
[(sq
->pc
& sq
->wq
.sz_m1
)] = NULL
;
1144 mlx5e_send_nop(sq
, true);
1148 mlx5e_disable_sq(sq
);
1149 mlx5e_free_sq_descs(sq
);
1150 mlx5e_destroy_sq(sq
);
1153 static int mlx5e_create_cq(struct mlx5e_channel
*c
,
1154 struct mlx5e_cq_param
*param
,
1155 struct mlx5e_cq
*cq
)
1157 struct mlx5e_priv
*priv
= c
->priv
;
1158 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1159 struct mlx5_core_cq
*mcq
= &cq
->mcq
;
1165 param
->wq
.buf_numa_node
= cpu_to_node(c
->cpu
);
1166 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
1167 param
->eq_ix
= c
->ix
;
1169 err
= mlx5_cqwq_create(mdev
, ¶m
->wq
, param
->cqc
, &cq
->wq
,
1174 mlx5_vector2eqn(mdev
, param
->eq_ix
, &eqn_not_used
, &irqn
);
1176 cq
->napi
= &c
->napi
;
1179 mcq
->set_ci_db
= cq
->wq_ctrl
.db
.db
;
1180 mcq
->arm_db
= cq
->wq_ctrl
.db
.db
+ 1;
1181 *mcq
->set_ci_db
= 0;
1183 mcq
->vector
= param
->eq_ix
;
1184 mcq
->comp
= mlx5e_completion_event
;
1185 mcq
->event
= mlx5e_cq_error_event
;
1187 mcq
->uar
= &mdev
->mlx5e_res
.cq_uar
;
1189 for (i
= 0; i
< mlx5_cqwq_get_size(&cq
->wq
); i
++) {
1190 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(&cq
->wq
, i
);
1201 static void mlx5e_destroy_cq(struct mlx5e_cq
*cq
)
1203 mlx5_wq_destroy(&cq
->wq_ctrl
);
1206 static int mlx5e_enable_cq(struct mlx5e_cq
*cq
, struct mlx5e_cq_param
*param
)
1208 struct mlx5e_priv
*priv
= cq
->priv
;
1209 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1210 struct mlx5_core_cq
*mcq
= &cq
->mcq
;
1215 unsigned int irqn_not_used
;
1219 inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
1220 sizeof(u64
) * cq
->wq_ctrl
.buf
.npages
;
1221 in
= mlx5_vzalloc(inlen
);
1225 cqc
= MLX5_ADDR_OF(create_cq_in
, in
, cq_context
);
1227 memcpy(cqc
, param
->cqc
, sizeof(param
->cqc
));
1229 mlx5_fill_page_array(&cq
->wq_ctrl
.buf
,
1230 (__be64
*)MLX5_ADDR_OF(create_cq_in
, in
, pas
));
1232 mlx5_vector2eqn(mdev
, param
->eq_ix
, &eqn
, &irqn_not_used
);
1234 MLX5_SET(cqc
, cqc
, cq_period_mode
, param
->cq_period_mode
);
1235 MLX5_SET(cqc
, cqc
, c_eqn
, eqn
);
1236 MLX5_SET(cqc
, cqc
, uar_page
, mcq
->uar
->index
);
1237 MLX5_SET(cqc
, cqc
, log_page_size
, cq
->wq_ctrl
.buf
.page_shift
-
1238 MLX5_ADAPTER_PAGE_SHIFT
);
1239 MLX5_SET64(cqc
, cqc
, dbr_addr
, cq
->wq_ctrl
.db
.dma
);
1241 err
= mlx5_core_create_cq(mdev
, mcq
, in
, inlen
);
1253 static void mlx5e_disable_cq(struct mlx5e_cq
*cq
)
1255 struct mlx5e_priv
*priv
= cq
->priv
;
1256 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1258 mlx5_core_destroy_cq(mdev
, &cq
->mcq
);
1261 static int mlx5e_open_cq(struct mlx5e_channel
*c
,
1262 struct mlx5e_cq_param
*param
,
1263 struct mlx5e_cq
*cq
,
1264 struct mlx5e_cq_moder moderation
)
1267 struct mlx5e_priv
*priv
= c
->priv
;
1268 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1270 err
= mlx5e_create_cq(c
, param
, cq
);
1274 err
= mlx5e_enable_cq(cq
, param
);
1276 goto err_destroy_cq
;
1278 if (MLX5_CAP_GEN(mdev
, cq_moderation
))
1279 mlx5_core_modify_cq_moderation(mdev
, &cq
->mcq
,
1285 mlx5e_destroy_cq(cq
);
1290 static void mlx5e_close_cq(struct mlx5e_cq
*cq
)
1292 mlx5e_disable_cq(cq
);
1293 mlx5e_destroy_cq(cq
);
1296 static int mlx5e_get_cpu(struct mlx5e_priv
*priv
, int ix
)
1298 return cpumask_first(priv
->mdev
->priv
.irq_info
[ix
].mask
);
1301 static int mlx5e_open_tx_cqs(struct mlx5e_channel
*c
,
1302 struct mlx5e_channel_param
*cparam
)
1304 struct mlx5e_priv
*priv
= c
->priv
;
1308 for (tc
= 0; tc
< c
->num_tc
; tc
++) {
1309 err
= mlx5e_open_cq(c
, &cparam
->tx_cq
, &c
->sq
[tc
].cq
,
1310 priv
->params
.tx_cq_moderation
);
1312 goto err_close_tx_cqs
;
1318 for (tc
--; tc
>= 0; tc
--)
1319 mlx5e_close_cq(&c
->sq
[tc
].cq
);
1324 static void mlx5e_close_tx_cqs(struct mlx5e_channel
*c
)
1328 for (tc
= 0; tc
< c
->num_tc
; tc
++)
1329 mlx5e_close_cq(&c
->sq
[tc
].cq
);
1332 static int mlx5e_open_sqs(struct mlx5e_channel
*c
,
1333 struct mlx5e_channel_param
*cparam
)
1338 for (tc
= 0; tc
< c
->num_tc
; tc
++) {
1339 err
= mlx5e_open_sq(c
, tc
, &cparam
->sq
, &c
->sq
[tc
]);
1347 for (tc
--; tc
>= 0; tc
--)
1348 mlx5e_close_sq(&c
->sq
[tc
]);
1353 static void mlx5e_close_sqs(struct mlx5e_channel
*c
)
1357 for (tc
= 0; tc
< c
->num_tc
; tc
++)
1358 mlx5e_close_sq(&c
->sq
[tc
]);
1361 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv
*priv
, int ix
)
1365 for (i
= 0; i
< priv
->profile
->max_tc
; i
++)
1366 priv
->channeltc_to_txq_map
[ix
][i
] =
1367 ix
+ i
* priv
->params
.num_channels
;
1370 static int mlx5e_set_sq_maxrate(struct net_device
*dev
,
1371 struct mlx5e_sq
*sq
, u32 rate
)
1373 struct mlx5e_priv
*priv
= netdev_priv(dev
);
1374 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1378 if (rate
== sq
->rate_limit
)
1383 /* remove current rl index to free space to next ones */
1384 mlx5_rl_remove_rate(mdev
, sq
->rate_limit
);
1389 err
= mlx5_rl_add_rate(mdev
, rate
, &rl_index
);
1391 netdev_err(dev
, "Failed configuring rate %u: %d\n",
1397 err
= mlx5e_modify_sq(sq
, MLX5_SQC_STATE_RDY
,
1398 MLX5_SQC_STATE_RDY
, true, rl_index
);
1400 netdev_err(dev
, "Failed configuring rate %u: %d\n",
1402 /* remove the rate from the table */
1404 mlx5_rl_remove_rate(mdev
, rate
);
1408 sq
->rate_limit
= rate
;
1412 static int mlx5e_set_tx_maxrate(struct net_device
*dev
, int index
, u32 rate
)
1414 struct mlx5e_priv
*priv
= netdev_priv(dev
);
1415 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1416 struct mlx5e_sq
*sq
= priv
->txq_to_sq_map
[index
];
1419 if (!mlx5_rl_is_supported(mdev
)) {
1420 netdev_err(dev
, "Rate limiting is not supported on this device\n");
1424 /* rate is given in Mb/sec, HW config is in Kb/sec */
1427 /* Check whether rate in valid range, 0 is always valid */
1428 if (rate
&& !mlx5_rl_is_in_range(mdev
, rate
)) {
1429 netdev_err(dev
, "TX rate %u, is not in range\n", rate
);
1433 mutex_lock(&priv
->state_lock
);
1434 if (test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
1435 err
= mlx5e_set_sq_maxrate(dev
, sq
, rate
);
1437 priv
->tx_rates
[index
] = rate
;
1438 mutex_unlock(&priv
->state_lock
);
1443 static int mlx5e_open_channel(struct mlx5e_priv
*priv
, int ix
,
1444 struct mlx5e_channel_param
*cparam
,
1445 struct mlx5e_channel
**cp
)
1447 struct mlx5e_cq_moder icosq_cq_moder
= {0, 0};
1448 struct net_device
*netdev
= priv
->netdev
;
1449 struct mlx5e_cq_moder rx_cq_profile
;
1450 int cpu
= mlx5e_get_cpu(priv
, ix
);
1451 struct mlx5e_channel
*c
;
1452 struct mlx5e_sq
*sq
;
1456 c
= kzalloc_node(sizeof(*c
), GFP_KERNEL
, cpu_to_node(cpu
));
1463 c
->pdev
= &priv
->mdev
->pdev
->dev
;
1464 c
->netdev
= priv
->netdev
;
1465 c
->mkey_be
= cpu_to_be32(priv
->mdev
->mlx5e_res
.mkey
.key
);
1466 c
->num_tc
= priv
->params
.num_tc
;
1467 c
->xdp
= !!priv
->xdp_prog
;
1469 if (priv
->params
.rx_am_enabled
)
1470 rx_cq_profile
= mlx5e_am_get_def_profile(priv
->params
.rx_cq_period_mode
);
1472 rx_cq_profile
= priv
->params
.rx_cq_moderation
;
1474 mlx5e_build_channeltc_to_txq_map(priv
, ix
);
1476 netif_napi_add(netdev
, &c
->napi
, mlx5e_napi_poll
, 64);
1478 err
= mlx5e_open_cq(c
, &cparam
->icosq_cq
, &c
->icosq
.cq
, icosq_cq_moder
);
1482 err
= mlx5e_open_tx_cqs(c
, cparam
);
1484 goto err_close_icosq_cq
;
1486 err
= mlx5e_open_cq(c
, &cparam
->rx_cq
, &c
->rq
.cq
,
1489 goto err_close_tx_cqs
;
1491 /* XDP SQ CQ params are same as normal TXQ sq CQ params */
1492 err
= c
->xdp
? mlx5e_open_cq(c
, &cparam
->tx_cq
, &c
->xdp_sq
.cq
,
1493 priv
->params
.tx_cq_moderation
) : 0;
1495 goto err_close_rx_cq
;
1497 napi_enable(&c
->napi
);
1499 err
= mlx5e_open_sq(c
, 0, &cparam
->icosq
, &c
->icosq
);
1501 goto err_disable_napi
;
1503 err
= mlx5e_open_sqs(c
, cparam
);
1505 goto err_close_icosq
;
1507 for (i
= 0; i
< priv
->params
.num_tc
; i
++) {
1508 u32 txq_ix
= priv
->channeltc_to_txq_map
[ix
][i
];
1510 if (priv
->tx_rates
[txq_ix
]) {
1511 sq
= priv
->txq_to_sq_map
[txq_ix
];
1512 mlx5e_set_sq_maxrate(priv
->netdev
, sq
,
1513 priv
->tx_rates
[txq_ix
]);
1517 err
= c
->xdp
? mlx5e_open_sq(c
, 0, &cparam
->xdp_sq
, &c
->xdp_sq
) : 0;
1521 err
= mlx5e_open_rq(c
, &cparam
->rq
, &c
->rq
);
1523 goto err_close_xdp_sq
;
1525 netif_set_xps_queue(netdev
, get_cpu_mask(c
->cpu
), ix
);
1531 mlx5e_close_sq(&c
->xdp_sq
);
1537 mlx5e_close_sq(&c
->icosq
);
1540 napi_disable(&c
->napi
);
1542 mlx5e_close_cq(&c
->xdp_sq
.cq
);
1545 mlx5e_close_cq(&c
->rq
.cq
);
1548 mlx5e_close_tx_cqs(c
);
1551 mlx5e_close_cq(&c
->icosq
.cq
);
1554 netif_napi_del(&c
->napi
);
1560 static void mlx5e_close_channel(struct mlx5e_channel
*c
)
1562 mlx5e_close_rq(&c
->rq
);
1564 mlx5e_close_sq(&c
->xdp_sq
);
1566 mlx5e_close_sq(&c
->icosq
);
1567 napi_disable(&c
->napi
);
1569 mlx5e_close_cq(&c
->xdp_sq
.cq
);
1570 mlx5e_close_cq(&c
->rq
.cq
);
1571 mlx5e_close_tx_cqs(c
);
1572 mlx5e_close_cq(&c
->icosq
.cq
);
1573 netif_napi_del(&c
->napi
);
1578 static void mlx5e_build_rq_param(struct mlx5e_priv
*priv
,
1579 struct mlx5e_rq_param
*param
)
1581 void *rqc
= param
->rqc
;
1582 void *wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
1584 switch (priv
->params
.rq_wq_type
) {
1585 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
1586 MLX5_SET(wq
, wq
, log_wqe_num_of_strides
,
1587 priv
->params
.mpwqe_log_num_strides
- 9);
1588 MLX5_SET(wq
, wq
, log_wqe_stride_size
,
1589 priv
->params
.mpwqe_log_stride_sz
- 6);
1590 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
);
1592 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1593 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_LINKED_LIST
);
1596 MLX5_SET(wq
, wq
, end_padding_mode
, MLX5_WQ_END_PAD_MODE_ALIGN
);
1597 MLX5_SET(wq
, wq
, log_wq_stride
, ilog2(sizeof(struct mlx5e_rx_wqe
)));
1598 MLX5_SET(wq
, wq
, log_wq_sz
, priv
->params
.log_rq_size
);
1599 MLX5_SET(wq
, wq
, pd
, priv
->mdev
->mlx5e_res
.pdn
);
1600 MLX5_SET(rqc
, rqc
, counter_set_id
, priv
->q_counter
);
1602 param
->wq
.buf_numa_node
= dev_to_node(&priv
->mdev
->pdev
->dev
);
1603 param
->wq
.linear
= 1;
1605 param
->am_enabled
= priv
->params
.rx_am_enabled
;
1608 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param
*param
)
1610 void *rqc
= param
->rqc
;
1611 void *wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
1613 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_LINKED_LIST
);
1614 MLX5_SET(wq
, wq
, log_wq_stride
, ilog2(sizeof(struct mlx5e_rx_wqe
)));
1617 static void mlx5e_build_sq_param_common(struct mlx5e_priv
*priv
,
1618 struct mlx5e_sq_param
*param
)
1620 void *sqc
= param
->sqc
;
1621 void *wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1623 MLX5_SET(wq
, wq
, log_wq_stride
, ilog2(MLX5_SEND_WQE_BB
));
1624 MLX5_SET(wq
, wq
, pd
, priv
->mdev
->mlx5e_res
.pdn
);
1626 param
->wq
.buf_numa_node
= dev_to_node(&priv
->mdev
->pdev
->dev
);
1629 static void mlx5e_build_sq_param(struct mlx5e_priv
*priv
,
1630 struct mlx5e_sq_param
*param
)
1632 void *sqc
= param
->sqc
;
1633 void *wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1635 mlx5e_build_sq_param_common(priv
, param
);
1636 MLX5_SET(wq
, wq
, log_wq_sz
, priv
->params
.log_sq_size
);
1638 param
->max_inline
= priv
->params
.tx_max_inline
;
1639 param
->min_inline_mode
= priv
->params
.tx_min_inline_mode
;
1640 param
->type
= MLX5E_SQ_TXQ
;
1643 static void mlx5e_build_common_cq_param(struct mlx5e_priv
*priv
,
1644 struct mlx5e_cq_param
*param
)
1646 void *cqc
= param
->cqc
;
1648 MLX5_SET(cqc
, cqc
, uar_page
, priv
->mdev
->mlx5e_res
.cq_uar
.index
);
1651 static void mlx5e_build_rx_cq_param(struct mlx5e_priv
*priv
,
1652 struct mlx5e_cq_param
*param
)
1654 void *cqc
= param
->cqc
;
1657 switch (priv
->params
.rq_wq_type
) {
1658 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
1659 log_cq_size
= priv
->params
.log_rq_size
+
1660 priv
->params
.mpwqe_log_num_strides
;
1662 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1663 log_cq_size
= priv
->params
.log_rq_size
;
1666 MLX5_SET(cqc
, cqc
, log_cq_size
, log_cq_size
);
1667 if (priv
->params
.rx_cqe_compress
) {
1668 MLX5_SET(cqc
, cqc
, mini_cqe_res_format
, MLX5_CQE_FORMAT_CSUM
);
1669 MLX5_SET(cqc
, cqc
, cqe_comp_en
, 1);
1672 mlx5e_build_common_cq_param(priv
, param
);
1674 param
->cq_period_mode
= priv
->params
.rx_cq_period_mode
;
1677 static void mlx5e_build_tx_cq_param(struct mlx5e_priv
*priv
,
1678 struct mlx5e_cq_param
*param
)
1680 void *cqc
= param
->cqc
;
1682 MLX5_SET(cqc
, cqc
, log_cq_size
, priv
->params
.log_sq_size
);
1684 mlx5e_build_common_cq_param(priv
, param
);
1686 param
->cq_period_mode
= MLX5_CQ_PERIOD_MODE_START_FROM_EQE
;
1689 static void mlx5e_build_ico_cq_param(struct mlx5e_priv
*priv
,
1690 struct mlx5e_cq_param
*param
,
1693 void *cqc
= param
->cqc
;
1695 MLX5_SET(cqc
, cqc
, log_cq_size
, log_wq_size
);
1697 mlx5e_build_common_cq_param(priv
, param
);
1699 param
->cq_period_mode
= MLX5_CQ_PERIOD_MODE_START_FROM_EQE
;
1702 static void mlx5e_build_icosq_param(struct mlx5e_priv
*priv
,
1703 struct mlx5e_sq_param
*param
,
1706 void *sqc
= param
->sqc
;
1707 void *wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1709 mlx5e_build_sq_param_common(priv
, param
);
1711 MLX5_SET(wq
, wq
, log_wq_sz
, log_wq_size
);
1712 MLX5_SET(sqc
, sqc
, reg_umr
, MLX5_CAP_ETH(priv
->mdev
, reg_umr_sq
));
1714 param
->type
= MLX5E_SQ_ICO
;
1717 static void mlx5e_build_xdpsq_param(struct mlx5e_priv
*priv
,
1718 struct mlx5e_sq_param
*param
)
1720 void *sqc
= param
->sqc
;
1721 void *wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1723 mlx5e_build_sq_param_common(priv
, param
);
1724 MLX5_SET(wq
, wq
, log_wq_sz
, priv
->params
.log_sq_size
);
1726 param
->max_inline
= priv
->params
.tx_max_inline
;
1727 /* FOR XDP SQs will support only L2 inline mode */
1728 param
->min_inline_mode
= MLX5_INLINE_MODE_NONE
;
1729 param
->type
= MLX5E_SQ_XDP
;
1732 static void mlx5e_build_channel_param(struct mlx5e_priv
*priv
, struct mlx5e_channel_param
*cparam
)
1734 u8 icosq_log_wq_sz
= MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE
;
1736 mlx5e_build_rq_param(priv
, &cparam
->rq
);
1737 mlx5e_build_sq_param(priv
, &cparam
->sq
);
1738 mlx5e_build_xdpsq_param(priv
, &cparam
->xdp_sq
);
1739 mlx5e_build_icosq_param(priv
, &cparam
->icosq
, icosq_log_wq_sz
);
1740 mlx5e_build_rx_cq_param(priv
, &cparam
->rx_cq
);
1741 mlx5e_build_tx_cq_param(priv
, &cparam
->tx_cq
);
1742 mlx5e_build_ico_cq_param(priv
, &cparam
->icosq_cq
, icosq_log_wq_sz
);
1745 static int mlx5e_open_channels(struct mlx5e_priv
*priv
)
1747 struct mlx5e_channel_param
*cparam
;
1748 int nch
= priv
->params
.num_channels
;
1753 priv
->channel
= kcalloc(nch
, sizeof(struct mlx5e_channel
*),
1756 priv
->txq_to_sq_map
= kcalloc(nch
* priv
->params
.num_tc
,
1757 sizeof(struct mlx5e_sq
*), GFP_KERNEL
);
1759 cparam
= kzalloc(sizeof(struct mlx5e_channel_param
), GFP_KERNEL
);
1761 if (!priv
->channel
|| !priv
->txq_to_sq_map
|| !cparam
)
1762 goto err_free_txq_to_sq_map
;
1764 mlx5e_build_channel_param(priv
, cparam
);
1766 for (i
= 0; i
< nch
; i
++) {
1767 err
= mlx5e_open_channel(priv
, i
, cparam
, &priv
->channel
[i
]);
1769 goto err_close_channels
;
1772 for (j
= 0; j
< nch
; j
++) {
1773 err
= mlx5e_wait_for_min_rx_wqes(&priv
->channel
[j
]->rq
);
1775 goto err_close_channels
;
1778 /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1779 * polling for inactive tx queues.
1781 netif_tx_start_all_queues(priv
->netdev
);
1787 for (i
--; i
>= 0; i
--)
1788 mlx5e_close_channel(priv
->channel
[i
]);
1790 err_free_txq_to_sq_map
:
1791 kfree(priv
->txq_to_sq_map
);
1792 kfree(priv
->channel
);
1798 static void mlx5e_close_channels(struct mlx5e_priv
*priv
)
1802 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1803 * polling for inactive tx queues.
1805 netif_tx_stop_all_queues(priv
->netdev
);
1806 netif_tx_disable(priv
->netdev
);
1808 for (i
= 0; i
< priv
->params
.num_channels
; i
++)
1809 mlx5e_close_channel(priv
->channel
[i
]);
1811 kfree(priv
->txq_to_sq_map
);
1812 kfree(priv
->channel
);
1815 static int mlx5e_rx_hash_fn(int hfunc
)
1817 return (hfunc
== ETH_RSS_HASH_TOP
) ?
1818 MLX5_RX_HASH_FN_TOEPLITZ
:
1819 MLX5_RX_HASH_FN_INVERTED_XOR8
;
1822 static int mlx5e_bits_invert(unsigned long a
, int size
)
1827 for (i
= 0; i
< size
; i
++)
1828 inv
|= (test_bit(size
- i
- 1, &a
) ? 1 : 0) << i
;
1833 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv
*priv
, void *rqtc
)
1837 for (i
= 0; i
< MLX5E_INDIR_RQT_SIZE
; i
++) {
1841 if (priv
->params
.rss_hfunc
== ETH_RSS_HASH_XOR
)
1842 ix
= mlx5e_bits_invert(i
, MLX5E_LOG_INDIR_RQT_SIZE
);
1844 ix
= priv
->params
.indirection_rqt
[ix
];
1845 rqn
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
) ?
1846 priv
->channel
[ix
]->rq
.rqn
:
1848 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], rqn
);
1852 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv
*priv
, void *rqtc
,
1855 u32 rqn
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
) ?
1856 priv
->channel
[ix
]->rq
.rqn
:
1859 MLX5_SET(rqtc
, rqtc
, rq_num
[0], rqn
);
1862 static int mlx5e_create_rqt(struct mlx5e_priv
*priv
, int sz
,
1863 int ix
, struct mlx5e_rqt
*rqt
)
1865 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1871 inlen
= MLX5_ST_SZ_BYTES(create_rqt_in
) + sizeof(u32
) * sz
;
1872 in
= mlx5_vzalloc(inlen
);
1876 rqtc
= MLX5_ADDR_OF(create_rqt_in
, in
, rqt_context
);
1878 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
1879 MLX5_SET(rqtc
, rqtc
, rqt_max_size
, sz
);
1881 if (sz
> 1) /* RSS */
1882 mlx5e_fill_indir_rqt_rqns(priv
, rqtc
);
1884 mlx5e_fill_direct_rqt_rqn(priv
, rqtc
, ix
);
1886 err
= mlx5_core_create_rqt(mdev
, in
, inlen
, &rqt
->rqtn
);
1888 rqt
->enabled
= true;
1894 void mlx5e_destroy_rqt(struct mlx5e_priv
*priv
, struct mlx5e_rqt
*rqt
)
1896 rqt
->enabled
= false;
1897 mlx5_core_destroy_rqt(priv
->mdev
, rqt
->rqtn
);
1900 static int mlx5e_create_indirect_rqts(struct mlx5e_priv
*priv
)
1902 struct mlx5e_rqt
*rqt
= &priv
->indir_rqt
;
1904 return mlx5e_create_rqt(priv
, MLX5E_INDIR_RQT_SIZE
, 0, rqt
);
1907 int mlx5e_create_direct_rqts(struct mlx5e_priv
*priv
)
1909 struct mlx5e_rqt
*rqt
;
1913 for (ix
= 0; ix
< priv
->profile
->max_nch(priv
->mdev
); ix
++) {
1914 rqt
= &priv
->direct_tir
[ix
].rqt
;
1915 err
= mlx5e_create_rqt(priv
, 1 /*size */, ix
, rqt
);
1917 goto err_destroy_rqts
;
1923 for (ix
--; ix
>= 0; ix
--)
1924 mlx5e_destroy_rqt(priv
, &priv
->direct_tir
[ix
].rqt
);
1929 int mlx5e_redirect_rqt(struct mlx5e_priv
*priv
, u32 rqtn
, int sz
, int ix
)
1931 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1937 inlen
= MLX5_ST_SZ_BYTES(modify_rqt_in
) + sizeof(u32
) * sz
;
1938 in
= mlx5_vzalloc(inlen
);
1942 rqtc
= MLX5_ADDR_OF(modify_rqt_in
, in
, ctx
);
1944 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
1945 if (sz
> 1) /* RSS */
1946 mlx5e_fill_indir_rqt_rqns(priv
, rqtc
);
1948 mlx5e_fill_direct_rqt_rqn(priv
, rqtc
, ix
);
1950 MLX5_SET(modify_rqt_in
, in
, bitmask
.rqn_list
, 1);
1952 err
= mlx5_core_modify_rqt(mdev
, rqtn
, in
, inlen
);
1959 static void mlx5e_redirect_rqts(struct mlx5e_priv
*priv
)
1964 if (priv
->indir_rqt
.enabled
) {
1965 rqtn
= priv
->indir_rqt
.rqtn
;
1966 mlx5e_redirect_rqt(priv
, rqtn
, MLX5E_INDIR_RQT_SIZE
, 0);
1969 for (ix
= 0; ix
< priv
->params
.num_channels
; ix
++) {
1970 if (!priv
->direct_tir
[ix
].rqt
.enabled
)
1972 rqtn
= priv
->direct_tir
[ix
].rqt
.rqtn
;
1973 mlx5e_redirect_rqt(priv
, rqtn
, 1, ix
);
1977 static void mlx5e_build_tir_ctx_lro(void *tirc
, struct mlx5e_priv
*priv
)
1979 if (!priv
->params
.lro_en
)
1982 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1984 MLX5_SET(tirc
, tirc
, lro_enable_mask
,
1985 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO
|
1986 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO
);
1987 MLX5_SET(tirc
, tirc
, lro_max_ip_payload_size
,
1988 (priv
->params
.lro_wqe_sz
-
1989 ROUGH_MAX_L2_L3_HDR_SZ
) >> 8);
1990 MLX5_SET(tirc
, tirc
, lro_timeout_period_usecs
, priv
->params
.lro_timeout
);
1993 void mlx5e_build_tir_ctx_hash(void *tirc
, struct mlx5e_priv
*priv
)
1995 MLX5_SET(tirc
, tirc
, rx_hash_fn
,
1996 mlx5e_rx_hash_fn(priv
->params
.rss_hfunc
));
1997 if (priv
->params
.rss_hfunc
== ETH_RSS_HASH_TOP
) {
1998 void *rss_key
= MLX5_ADDR_OF(tirc
, tirc
,
1999 rx_hash_toeplitz_key
);
2000 size_t len
= MLX5_FLD_SZ_BYTES(tirc
,
2001 rx_hash_toeplitz_key
);
2003 MLX5_SET(tirc
, tirc
, rx_hash_symmetric
, 1);
2004 memcpy(rss_key
, priv
->params
.toeplitz_hash_key
, len
);
2008 static int mlx5e_modify_tirs_lro(struct mlx5e_priv
*priv
)
2010 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2019 inlen
= MLX5_ST_SZ_BYTES(modify_tir_in
);
2020 in
= mlx5_vzalloc(inlen
);
2024 MLX5_SET(modify_tir_in
, in
, bitmask
.lro
, 1);
2025 tirc
= MLX5_ADDR_OF(modify_tir_in
, in
, ctx
);
2027 mlx5e_build_tir_ctx_lro(tirc
, priv
);
2029 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
2030 err
= mlx5_core_modify_tir(mdev
, priv
->indir_tir
[tt
].tirn
, in
,
2036 for (ix
= 0; ix
< priv
->profile
->max_nch(priv
->mdev
); ix
++) {
2037 err
= mlx5_core_modify_tir(mdev
, priv
->direct_tir
[ix
].tirn
,
2049 static int mlx5e_set_mtu(struct mlx5e_priv
*priv
, u16 mtu
)
2051 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2052 u16 hw_mtu
= MLX5E_SW2HW_MTU(mtu
);
2055 err
= mlx5_set_port_mtu(mdev
, hw_mtu
, 1);
2059 /* Update vport context MTU */
2060 mlx5_modify_nic_vport_mtu(mdev
, hw_mtu
);
2064 static void mlx5e_query_mtu(struct mlx5e_priv
*priv
, u16
*mtu
)
2066 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2070 err
= mlx5_query_nic_vport_mtu(mdev
, &hw_mtu
);
2071 if (err
|| !hw_mtu
) /* fallback to port oper mtu */
2072 mlx5_query_port_oper_mtu(mdev
, &hw_mtu
, 1);
2074 *mtu
= MLX5E_HW2SW_MTU(hw_mtu
);
2077 static int mlx5e_set_dev_port_mtu(struct net_device
*netdev
)
2079 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2083 err
= mlx5e_set_mtu(priv
, netdev
->mtu
);
2087 mlx5e_query_mtu(priv
, &mtu
);
2088 if (mtu
!= netdev
->mtu
)
2089 netdev_warn(netdev
, "%s: VPort MTU %d is different than netdev mtu %d\n",
2090 __func__
, mtu
, netdev
->mtu
);
2096 static void mlx5e_netdev_set_tcs(struct net_device
*netdev
)
2098 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2099 int nch
= priv
->params
.num_channels
;
2100 int ntc
= priv
->params
.num_tc
;
2103 netdev_reset_tc(netdev
);
2108 netdev_set_num_tc(netdev
, ntc
);
2110 /* Map netdev TCs to offset 0
2111 * We have our own UP to TXQ mapping for QoS
2113 for (tc
= 0; tc
< ntc
; tc
++)
2114 netdev_set_tc_queue(netdev
, tc
, nch
, 0);
2117 int mlx5e_open_locked(struct net_device
*netdev
)
2119 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2120 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2124 set_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2126 mlx5e_netdev_set_tcs(netdev
);
2128 num_txqs
= priv
->params
.num_channels
* priv
->params
.num_tc
;
2129 netif_set_real_num_tx_queues(netdev
, num_txqs
);
2130 netif_set_real_num_rx_queues(netdev
, priv
->params
.num_channels
);
2132 err
= mlx5e_open_channels(priv
);
2134 netdev_err(netdev
, "%s: mlx5e_open_channels failed, %d\n",
2136 goto err_clear_state_opened_flag
;
2139 err
= mlx5e_refresh_tirs_self_loopback_enable(priv
->mdev
);
2141 netdev_err(netdev
, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
2143 goto err_close_channels
;
2146 mlx5e_redirect_rqts(priv
);
2147 mlx5e_update_carrier(priv
);
2148 mlx5e_timestamp_init(priv
);
2149 #ifdef CONFIG_RFS_ACCEL
2150 priv
->netdev
->rx_cpu_rmap
= priv
->mdev
->rmap
;
2152 if (priv
->profile
->update_stats
)
2153 queue_delayed_work(priv
->wq
, &priv
->update_stats_work
, 0);
2155 if (MLX5_CAP_GEN(mdev
, vport_group_manager
)) {
2156 err
= mlx5e_add_sqs_fwd_rules(priv
);
2158 goto err_close_channels
;
2163 mlx5e_close_channels(priv
);
2164 err_clear_state_opened_flag
:
2165 clear_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2169 int mlx5e_open(struct net_device
*netdev
)
2171 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2174 mutex_lock(&priv
->state_lock
);
2175 err
= mlx5e_open_locked(netdev
);
2176 mutex_unlock(&priv
->state_lock
);
2181 int mlx5e_close_locked(struct net_device
*netdev
)
2183 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2184 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2186 /* May already be CLOSED in case a previous configuration operation
2187 * (e.g RX/TX queue size change) that involves close&open failed.
2189 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
2192 clear_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2194 if (MLX5_CAP_GEN(mdev
, vport_group_manager
))
2195 mlx5e_remove_sqs_fwd_rules(priv
);
2197 mlx5e_timestamp_cleanup(priv
);
2198 netif_carrier_off(priv
->netdev
);
2199 mlx5e_redirect_rqts(priv
);
2200 mlx5e_close_channels(priv
);
2205 int mlx5e_close(struct net_device
*netdev
)
2207 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2210 if (!netif_device_present(netdev
))
2213 mutex_lock(&priv
->state_lock
);
2214 err
= mlx5e_close_locked(netdev
);
2215 mutex_unlock(&priv
->state_lock
);
2220 static int mlx5e_create_drop_rq(struct mlx5e_priv
*priv
,
2221 struct mlx5e_rq
*rq
,
2222 struct mlx5e_rq_param
*param
)
2224 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2225 void *rqc
= param
->rqc
;
2226 void *rqc_wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
2229 param
->wq
.db_numa_node
= param
->wq
.buf_numa_node
;
2231 err
= mlx5_wq_ll_create(mdev
, ¶m
->wq
, rqc_wq
, &rq
->wq
,
2241 static int mlx5e_create_drop_cq(struct mlx5e_priv
*priv
,
2242 struct mlx5e_cq
*cq
,
2243 struct mlx5e_cq_param
*param
)
2245 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2246 struct mlx5_core_cq
*mcq
= &cq
->mcq
;
2251 err
= mlx5_cqwq_create(mdev
, ¶m
->wq
, param
->cqc
, &cq
->wq
,
2256 mlx5_vector2eqn(mdev
, param
->eq_ix
, &eqn_not_used
, &irqn
);
2259 mcq
->set_ci_db
= cq
->wq_ctrl
.db
.db
;
2260 mcq
->arm_db
= cq
->wq_ctrl
.db
.db
+ 1;
2261 *mcq
->set_ci_db
= 0;
2263 mcq
->vector
= param
->eq_ix
;
2264 mcq
->comp
= mlx5e_completion_event
;
2265 mcq
->event
= mlx5e_cq_error_event
;
2267 mcq
->uar
= &mdev
->mlx5e_res
.cq_uar
;
2274 static int mlx5e_open_drop_rq(struct mlx5e_priv
*priv
)
2276 struct mlx5e_cq_param cq_param
;
2277 struct mlx5e_rq_param rq_param
;
2278 struct mlx5e_rq
*rq
= &priv
->drop_rq
;
2279 struct mlx5e_cq
*cq
= &priv
->drop_rq
.cq
;
2282 memset(&cq_param
, 0, sizeof(cq_param
));
2283 memset(&rq_param
, 0, sizeof(rq_param
));
2284 mlx5e_build_drop_rq_param(&rq_param
);
2286 err
= mlx5e_create_drop_cq(priv
, cq
, &cq_param
);
2290 err
= mlx5e_enable_cq(cq
, &cq_param
);
2292 goto err_destroy_cq
;
2294 err
= mlx5e_create_drop_rq(priv
, rq
, &rq_param
);
2296 goto err_disable_cq
;
2298 err
= mlx5e_enable_rq(rq
, &rq_param
);
2300 goto err_destroy_rq
;
2305 mlx5e_destroy_rq(&priv
->drop_rq
);
2308 mlx5e_disable_cq(&priv
->drop_rq
.cq
);
2311 mlx5e_destroy_cq(&priv
->drop_rq
.cq
);
2316 static void mlx5e_close_drop_rq(struct mlx5e_priv
*priv
)
2318 mlx5e_disable_rq(&priv
->drop_rq
);
2319 mlx5e_destroy_rq(&priv
->drop_rq
);
2320 mlx5e_disable_cq(&priv
->drop_rq
.cq
);
2321 mlx5e_destroy_cq(&priv
->drop_rq
.cq
);
2324 static int mlx5e_create_tis(struct mlx5e_priv
*priv
, int tc
)
2326 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2327 u32 in
[MLX5_ST_SZ_DW(create_tis_in
)] = {0};
2328 void *tisc
= MLX5_ADDR_OF(create_tis_in
, in
, ctx
);
2330 MLX5_SET(tisc
, tisc
, prio
, tc
<< 1);
2331 MLX5_SET(tisc
, tisc
, transport_domain
, mdev
->mlx5e_res
.td
.tdn
);
2333 if (mlx5_lag_is_lacp_owner(mdev
))
2334 MLX5_SET(tisc
, tisc
, strict_lag_tx_port_affinity
, 1);
2336 return mlx5_core_create_tis(mdev
, in
, sizeof(in
), &priv
->tisn
[tc
]);
2339 static void mlx5e_destroy_tis(struct mlx5e_priv
*priv
, int tc
)
2341 mlx5_core_destroy_tis(priv
->mdev
, priv
->tisn
[tc
]);
2344 int mlx5e_create_tises(struct mlx5e_priv
*priv
)
2349 for (tc
= 0; tc
< priv
->profile
->max_tc
; tc
++) {
2350 err
= mlx5e_create_tis(priv
, tc
);
2352 goto err_close_tises
;
2358 for (tc
--; tc
>= 0; tc
--)
2359 mlx5e_destroy_tis(priv
, tc
);
2364 void mlx5e_cleanup_nic_tx(struct mlx5e_priv
*priv
)
2368 for (tc
= 0; tc
< priv
->profile
->max_tc
; tc
++)
2369 mlx5e_destroy_tis(priv
, tc
);
2372 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv
*priv
, u32
*tirc
,
2373 enum mlx5e_traffic_types tt
)
2375 void *hfso
= MLX5_ADDR_OF(tirc
, tirc
, rx_hash_field_selector_outer
);
2377 MLX5_SET(tirc
, tirc
, transport_domain
, priv
->mdev
->mlx5e_res
.td
.tdn
);
2379 #define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
2380 MLX5_HASH_FIELD_SEL_DST_IP)
2382 #define MLX5_HASH_IP_L4PORTS (MLX5_HASH_FIELD_SEL_SRC_IP |\
2383 MLX5_HASH_FIELD_SEL_DST_IP |\
2384 MLX5_HASH_FIELD_SEL_L4_SPORT |\
2385 MLX5_HASH_FIELD_SEL_L4_DPORT)
2387 #define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\
2388 MLX5_HASH_FIELD_SEL_DST_IP |\
2389 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2391 mlx5e_build_tir_ctx_lro(tirc
, priv
);
2393 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_INDIRECT
);
2394 MLX5_SET(tirc
, tirc
, indirect_table
, priv
->indir_rqt
.rqtn
);
2395 mlx5e_build_tir_ctx_hash(tirc
, priv
);
2398 case MLX5E_TT_IPV4_TCP
:
2399 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2400 MLX5_L3_PROT_TYPE_IPV4
);
2401 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
2402 MLX5_L4_PROT_TYPE_TCP
);
2403 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2404 MLX5_HASH_IP_L4PORTS
);
2407 case MLX5E_TT_IPV6_TCP
:
2408 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2409 MLX5_L3_PROT_TYPE_IPV6
);
2410 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
2411 MLX5_L4_PROT_TYPE_TCP
);
2412 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2413 MLX5_HASH_IP_L4PORTS
);
2416 case MLX5E_TT_IPV4_UDP
:
2417 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2418 MLX5_L3_PROT_TYPE_IPV4
);
2419 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
2420 MLX5_L4_PROT_TYPE_UDP
);
2421 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2422 MLX5_HASH_IP_L4PORTS
);
2425 case MLX5E_TT_IPV6_UDP
:
2426 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2427 MLX5_L3_PROT_TYPE_IPV6
);
2428 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
2429 MLX5_L4_PROT_TYPE_UDP
);
2430 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2431 MLX5_HASH_IP_L4PORTS
);
2434 case MLX5E_TT_IPV4_IPSEC_AH
:
2435 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2436 MLX5_L3_PROT_TYPE_IPV4
);
2437 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2438 MLX5_HASH_IP_IPSEC_SPI
);
2441 case MLX5E_TT_IPV6_IPSEC_AH
:
2442 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2443 MLX5_L3_PROT_TYPE_IPV6
);
2444 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2445 MLX5_HASH_IP_IPSEC_SPI
);
2448 case MLX5E_TT_IPV4_IPSEC_ESP
:
2449 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2450 MLX5_L3_PROT_TYPE_IPV4
);
2451 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2452 MLX5_HASH_IP_IPSEC_SPI
);
2455 case MLX5E_TT_IPV6_IPSEC_ESP
:
2456 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2457 MLX5_L3_PROT_TYPE_IPV6
);
2458 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2459 MLX5_HASH_IP_IPSEC_SPI
);
2463 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2464 MLX5_L3_PROT_TYPE_IPV4
);
2465 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2470 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2471 MLX5_L3_PROT_TYPE_IPV6
);
2472 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2477 "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2481 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv
*priv
, u32
*tirc
,
2484 MLX5_SET(tirc
, tirc
, transport_domain
, priv
->mdev
->mlx5e_res
.td
.tdn
);
2486 mlx5e_build_tir_ctx_lro(tirc
, priv
);
2488 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_INDIRECT
);
2489 MLX5_SET(tirc
, tirc
, indirect_table
, rqtn
);
2490 MLX5_SET(tirc
, tirc
, rx_hash_fn
, MLX5_RX_HASH_FN_INVERTED_XOR8
);
2493 static int mlx5e_create_indirect_tirs(struct mlx5e_priv
*priv
)
2495 struct mlx5e_tir
*tir
;
2502 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
2503 in
= mlx5_vzalloc(inlen
);
2507 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
2508 memset(in
, 0, inlen
);
2509 tir
= &priv
->indir_tir
[tt
];
2510 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
2511 mlx5e_build_indir_tir_ctx(priv
, tirc
, tt
);
2512 err
= mlx5e_create_tir(priv
->mdev
, tir
, in
, inlen
);
2514 goto err_destroy_tirs
;
2522 for (tt
--; tt
>= 0; tt
--)
2523 mlx5e_destroy_tir(priv
->mdev
, &priv
->indir_tir
[tt
]);
2530 int mlx5e_create_direct_tirs(struct mlx5e_priv
*priv
)
2532 int nch
= priv
->profile
->max_nch(priv
->mdev
);
2533 struct mlx5e_tir
*tir
;
2540 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
2541 in
= mlx5_vzalloc(inlen
);
2545 for (ix
= 0; ix
< nch
; ix
++) {
2546 memset(in
, 0, inlen
);
2547 tir
= &priv
->direct_tir
[ix
];
2548 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
2549 mlx5e_build_direct_tir_ctx(priv
, tirc
,
2550 priv
->direct_tir
[ix
].rqt
.rqtn
);
2551 err
= mlx5e_create_tir(priv
->mdev
, tir
, in
, inlen
);
2553 goto err_destroy_ch_tirs
;
2560 err_destroy_ch_tirs
:
2561 for (ix
--; ix
>= 0; ix
--)
2562 mlx5e_destroy_tir(priv
->mdev
, &priv
->direct_tir
[ix
]);
2569 static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv
*priv
)
2573 for (i
= 0; i
< MLX5E_NUM_INDIR_TIRS
; i
++)
2574 mlx5e_destroy_tir(priv
->mdev
, &priv
->indir_tir
[i
]);
2577 void mlx5e_destroy_direct_tirs(struct mlx5e_priv
*priv
)
2579 int nch
= priv
->profile
->max_nch(priv
->mdev
);
2582 for (i
= 0; i
< nch
; i
++)
2583 mlx5e_destroy_tir(priv
->mdev
, &priv
->direct_tir
[i
]);
2586 int mlx5e_modify_rqs_vsd(struct mlx5e_priv
*priv
, bool vsd
)
2591 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
2594 for (i
= 0; i
< priv
->params
.num_channels
; i
++) {
2595 err
= mlx5e_modify_rq_vsd(&priv
->channel
[i
]->rq
, vsd
);
2603 static int mlx5e_setup_tc(struct net_device
*netdev
, u8 tc
)
2605 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2609 if (tc
&& tc
!= MLX5E_MAX_NUM_TC
)
2612 mutex_lock(&priv
->state_lock
);
2614 was_opened
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2616 mlx5e_close_locked(priv
->netdev
);
2618 priv
->params
.num_tc
= tc
? tc
: 1;
2621 err
= mlx5e_open_locked(priv
->netdev
);
2623 mutex_unlock(&priv
->state_lock
);
2628 static int mlx5e_ndo_setup_tc(struct net_device
*dev
, u32 handle
,
2629 __be16 proto
, struct tc_to_netdev
*tc
)
2631 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2633 if (TC_H_MAJ(handle
) != TC_H_MAJ(TC_H_INGRESS
))
2637 case TC_SETUP_CLSFLOWER
:
2638 switch (tc
->cls_flower
->command
) {
2639 case TC_CLSFLOWER_REPLACE
:
2640 return mlx5e_configure_flower(priv
, proto
, tc
->cls_flower
);
2641 case TC_CLSFLOWER_DESTROY
:
2642 return mlx5e_delete_flower(priv
, tc
->cls_flower
);
2643 case TC_CLSFLOWER_STATS
:
2644 return mlx5e_stats_flower(priv
, tc
->cls_flower
);
2651 if (tc
->type
!= TC_SETUP_MQPRIO
)
2654 return mlx5e_setup_tc(dev
, tc
->tc
);
2657 static struct rtnl_link_stats64
*
2658 mlx5e_get_stats(struct net_device
*dev
, struct rtnl_link_stats64
*stats
)
2660 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2661 struct mlx5e_sw_stats
*sstats
= &priv
->stats
.sw
;
2662 struct mlx5e_vport_stats
*vstats
= &priv
->stats
.vport
;
2663 struct mlx5e_pport_stats
*pstats
= &priv
->stats
.pport
;
2665 if (mlx5e_is_uplink_rep(priv
)) {
2666 stats
->rx_packets
= PPORT_802_3_GET(pstats
, a_frames_received_ok
);
2667 stats
->rx_bytes
= PPORT_802_3_GET(pstats
, a_octets_received_ok
);
2668 stats
->tx_packets
= PPORT_802_3_GET(pstats
, a_frames_transmitted_ok
);
2669 stats
->tx_bytes
= PPORT_802_3_GET(pstats
, a_octets_transmitted_ok
);
2671 stats
->rx_packets
= sstats
->rx_packets
;
2672 stats
->rx_bytes
= sstats
->rx_bytes
;
2673 stats
->tx_packets
= sstats
->tx_packets
;
2674 stats
->tx_bytes
= sstats
->tx_bytes
;
2675 stats
->tx_dropped
= sstats
->tx_queue_dropped
;
2678 stats
->rx_dropped
= priv
->stats
.qcnt
.rx_out_of_buffer
;
2680 stats
->rx_length_errors
=
2681 PPORT_802_3_GET(pstats
, a_in_range_length_errors
) +
2682 PPORT_802_3_GET(pstats
, a_out_of_range_length_field
) +
2683 PPORT_802_3_GET(pstats
, a_frame_too_long_errors
);
2684 stats
->rx_crc_errors
=
2685 PPORT_802_3_GET(pstats
, a_frame_check_sequence_errors
);
2686 stats
->rx_frame_errors
= PPORT_802_3_GET(pstats
, a_alignment_errors
);
2687 stats
->tx_aborted_errors
= PPORT_2863_GET(pstats
, if_out_discards
);
2688 stats
->tx_carrier_errors
=
2689 PPORT_802_3_GET(pstats
, a_symbol_error_during_carrier
);
2690 stats
->rx_errors
= stats
->rx_length_errors
+ stats
->rx_crc_errors
+
2691 stats
->rx_frame_errors
;
2692 stats
->tx_errors
= stats
->tx_aborted_errors
+ stats
->tx_carrier_errors
;
2694 /* vport multicast also counts packets that are dropped due to steering
2695 * or rx out of buffer
2698 VPORT_COUNTER_GET(vstats
, received_eth_multicast
.packets
);
2703 static void mlx5e_set_rx_mode(struct net_device
*dev
)
2705 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2707 queue_work(priv
->wq
, &priv
->set_rx_mode_work
);
2710 static int mlx5e_set_mac(struct net_device
*netdev
, void *addr
)
2712 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2713 struct sockaddr
*saddr
= addr
;
2715 if (!is_valid_ether_addr(saddr
->sa_data
))
2716 return -EADDRNOTAVAIL
;
2718 netif_addr_lock_bh(netdev
);
2719 ether_addr_copy(netdev
->dev_addr
, saddr
->sa_data
);
2720 netif_addr_unlock_bh(netdev
);
2722 queue_work(priv
->wq
, &priv
->set_rx_mode_work
);
2727 #define MLX5E_SET_FEATURE(netdev, feature, enable) \
2730 netdev->features |= feature; \
2732 netdev->features &= ~feature; \
2735 typedef int (*mlx5e_feature_handler
)(struct net_device
*netdev
, bool enable
);
2737 static int set_feature_lro(struct net_device
*netdev
, bool enable
)
2739 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2740 bool was_opened
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2743 mutex_lock(&priv
->state_lock
);
2745 if (was_opened
&& (priv
->params
.rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST
))
2746 mlx5e_close_locked(priv
->netdev
);
2748 priv
->params
.lro_en
= enable
;
2749 err
= mlx5e_modify_tirs_lro(priv
);
2751 netdev_err(netdev
, "lro modify failed, %d\n", err
);
2752 priv
->params
.lro_en
= !enable
;
2755 if (was_opened
&& (priv
->params
.rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST
))
2756 mlx5e_open_locked(priv
->netdev
);
2758 mutex_unlock(&priv
->state_lock
);
2763 static int set_feature_vlan_filter(struct net_device
*netdev
, bool enable
)
2765 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2768 mlx5e_enable_vlan_filter(priv
);
2770 mlx5e_disable_vlan_filter(priv
);
2775 static int set_feature_tc_num_filters(struct net_device
*netdev
, bool enable
)
2777 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2779 if (!enable
&& mlx5e_tc_num_filters(priv
)) {
2781 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2788 static int set_feature_rx_all(struct net_device
*netdev
, bool enable
)
2790 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2791 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2793 return mlx5_set_port_fcs(mdev
, !enable
);
2796 static int set_feature_rx_vlan(struct net_device
*netdev
, bool enable
)
2798 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2801 mutex_lock(&priv
->state_lock
);
2803 priv
->params
.vlan_strip_disable
= !enable
;
2804 err
= mlx5e_modify_rqs_vsd(priv
, !enable
);
2806 priv
->params
.vlan_strip_disable
= enable
;
2808 mutex_unlock(&priv
->state_lock
);
2813 #ifdef CONFIG_RFS_ACCEL
2814 static int set_feature_arfs(struct net_device
*netdev
, bool enable
)
2816 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2820 err
= mlx5e_arfs_enable(priv
);
2822 err
= mlx5e_arfs_disable(priv
);
2828 static int mlx5e_handle_feature(struct net_device
*netdev
,
2829 netdev_features_t wanted_features
,
2830 netdev_features_t feature
,
2831 mlx5e_feature_handler feature_handler
)
2833 netdev_features_t changes
= wanted_features
^ netdev
->features
;
2834 bool enable
= !!(wanted_features
& feature
);
2837 if (!(changes
& feature
))
2840 err
= feature_handler(netdev
, enable
);
2842 netdev_err(netdev
, "%s feature 0x%llx failed err %d\n",
2843 enable
? "Enable" : "Disable", feature
, err
);
2847 MLX5E_SET_FEATURE(netdev
, feature
, enable
);
2851 static int mlx5e_set_features(struct net_device
*netdev
,
2852 netdev_features_t features
)
2856 err
= mlx5e_handle_feature(netdev
, features
, NETIF_F_LRO
,
2858 err
|= mlx5e_handle_feature(netdev
, features
,
2859 NETIF_F_HW_VLAN_CTAG_FILTER
,
2860 set_feature_vlan_filter
);
2861 err
|= mlx5e_handle_feature(netdev
, features
, NETIF_F_HW_TC
,
2862 set_feature_tc_num_filters
);
2863 err
|= mlx5e_handle_feature(netdev
, features
, NETIF_F_RXALL
,
2864 set_feature_rx_all
);
2865 err
|= mlx5e_handle_feature(netdev
, features
, NETIF_F_HW_VLAN_CTAG_RX
,
2866 set_feature_rx_vlan
);
2867 #ifdef CONFIG_RFS_ACCEL
2868 err
|= mlx5e_handle_feature(netdev
, features
, NETIF_F_NTUPLE
,
2872 return err
? -EINVAL
: 0;
2875 static int mlx5e_change_mtu(struct net_device
*netdev
, int new_mtu
)
2877 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2882 mutex_lock(&priv
->state_lock
);
2884 reset
= !priv
->params
.lro_en
&&
2885 (priv
->params
.rq_wq_type
!=
2886 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
);
2888 was_opened
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2889 if (was_opened
&& reset
)
2890 mlx5e_close_locked(netdev
);
2892 netdev
->mtu
= new_mtu
;
2893 mlx5e_set_dev_port_mtu(netdev
);
2895 if (was_opened
&& reset
)
2896 err
= mlx5e_open_locked(netdev
);
2898 mutex_unlock(&priv
->state_lock
);
2903 static int mlx5e_ioctl(struct net_device
*dev
, struct ifreq
*ifr
, int cmd
)
2907 return mlx5e_hwstamp_set(dev
, ifr
);
2909 return mlx5e_hwstamp_get(dev
, ifr
);
2915 static int mlx5e_set_vf_mac(struct net_device
*dev
, int vf
, u8
*mac
)
2917 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2918 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2920 return mlx5_eswitch_set_vport_mac(mdev
->priv
.eswitch
, vf
+ 1, mac
);
2923 static int mlx5e_set_vf_vlan(struct net_device
*dev
, int vf
, u16 vlan
, u8 qos
,
2926 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2927 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2929 if (vlan_proto
!= htons(ETH_P_8021Q
))
2930 return -EPROTONOSUPPORT
;
2932 return mlx5_eswitch_set_vport_vlan(mdev
->priv
.eswitch
, vf
+ 1,
2936 static int mlx5e_set_vf_spoofchk(struct net_device
*dev
, int vf
, bool setting
)
2938 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2939 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2941 return mlx5_eswitch_set_vport_spoofchk(mdev
->priv
.eswitch
, vf
+ 1, setting
);
2944 static int mlx5e_set_vf_trust(struct net_device
*dev
, int vf
, bool setting
)
2946 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2947 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2949 return mlx5_eswitch_set_vport_trust(mdev
->priv
.eswitch
, vf
+ 1, setting
);
2952 static int mlx5e_set_vf_rate(struct net_device
*dev
, int vf
, int min_tx_rate
,
2955 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2956 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2961 return mlx5_eswitch_set_vport_rate(mdev
->priv
.eswitch
, vf
+ 1,
2965 static int mlx5_vport_link2ifla(u8 esw_link
)
2968 case MLX5_ESW_VPORT_ADMIN_STATE_DOWN
:
2969 return IFLA_VF_LINK_STATE_DISABLE
;
2970 case MLX5_ESW_VPORT_ADMIN_STATE_UP
:
2971 return IFLA_VF_LINK_STATE_ENABLE
;
2973 return IFLA_VF_LINK_STATE_AUTO
;
2976 static int mlx5_ifla_link2vport(u8 ifla_link
)
2978 switch (ifla_link
) {
2979 case IFLA_VF_LINK_STATE_DISABLE
:
2980 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN
;
2981 case IFLA_VF_LINK_STATE_ENABLE
:
2982 return MLX5_ESW_VPORT_ADMIN_STATE_UP
;
2984 return MLX5_ESW_VPORT_ADMIN_STATE_AUTO
;
2987 static int mlx5e_set_vf_link_state(struct net_device
*dev
, int vf
,
2990 struct mlx5e_priv
*priv
= netdev_priv(dev
);
2991 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2993 return mlx5_eswitch_set_vport_state(mdev
->priv
.eswitch
, vf
+ 1,
2994 mlx5_ifla_link2vport(link_state
));
2997 static int mlx5e_get_vf_config(struct net_device
*dev
,
2998 int vf
, struct ifla_vf_info
*ivi
)
3000 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3001 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3004 err
= mlx5_eswitch_get_vport_config(mdev
->priv
.eswitch
, vf
+ 1, ivi
);
3007 ivi
->linkstate
= mlx5_vport_link2ifla(ivi
->linkstate
);
3011 static int mlx5e_get_vf_stats(struct net_device
*dev
,
3012 int vf
, struct ifla_vf_stats
*vf_stats
)
3014 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3015 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3017 return mlx5_eswitch_get_vport_stats(mdev
->priv
.eswitch
, vf
+ 1,
3021 void mlx5e_add_vxlan_port(struct net_device
*netdev
,
3022 struct udp_tunnel_info
*ti
)
3024 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3026 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
3029 if (!mlx5e_vxlan_allowed(priv
->mdev
))
3032 mlx5e_vxlan_queue_work(priv
, ti
->sa_family
, be16_to_cpu(ti
->port
), 1);
3035 void mlx5e_del_vxlan_port(struct net_device
*netdev
,
3036 struct udp_tunnel_info
*ti
)
3038 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3040 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
3043 if (!mlx5e_vxlan_allowed(priv
->mdev
))
3046 mlx5e_vxlan_queue_work(priv
, ti
->sa_family
, be16_to_cpu(ti
->port
), 0);
3049 static netdev_features_t
mlx5e_vxlan_features_check(struct mlx5e_priv
*priv
,
3050 struct sk_buff
*skb
,
3051 netdev_features_t features
)
3053 struct udphdr
*udph
;
3057 switch (vlan_get_protocol(skb
)) {
3058 case htons(ETH_P_IP
):
3059 proto
= ip_hdr(skb
)->protocol
;
3061 case htons(ETH_P_IPV6
):
3062 proto
= ipv6_hdr(skb
)->nexthdr
;
3068 if (proto
== IPPROTO_UDP
) {
3069 udph
= udp_hdr(skb
);
3070 port
= be16_to_cpu(udph
->dest
);
3073 /* Verify if UDP port is being offloaded by HW */
3074 if (port
&& mlx5e_vxlan_lookup_port(priv
, port
))
3078 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
3079 return features
& ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
3082 static netdev_features_t
mlx5e_features_check(struct sk_buff
*skb
,
3083 struct net_device
*netdev
,
3084 netdev_features_t features
)
3086 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3088 features
= vlan_features_check(skb
, features
);
3089 features
= vxlan_features_check(skb
, features
);
3091 /* Validate if the tunneled packet is being offloaded by HW */
3092 if (skb
->encapsulation
&&
3093 (features
& NETIF_F_CSUM_MASK
|| features
& NETIF_F_GSO_MASK
))
3094 return mlx5e_vxlan_features_check(priv
, skb
, features
);
3099 static void mlx5e_tx_timeout(struct net_device
*dev
)
3101 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3102 bool sched_work
= false;
3105 netdev_err(dev
, "TX timeout detected\n");
3107 for (i
= 0; i
< priv
->params
.num_channels
* priv
->params
.num_tc
; i
++) {
3108 struct mlx5e_sq
*sq
= priv
->txq_to_sq_map
[i
];
3110 if (!netif_xmit_stopped(netdev_get_tx_queue(dev
, i
)))
3113 set_bit(MLX5E_SQ_STATE_FLUSH
, &sq
->state
);
3114 netdev_err(dev
, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
3115 i
, sq
->sqn
, sq
->cq
.mcq
.cqn
, sq
->cc
, sq
->pc
);
3118 if (sched_work
&& test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
3119 schedule_work(&priv
->tx_timeout_work
);
3122 static int mlx5e_xdp_set(struct net_device
*netdev
, struct bpf_prog
*prog
)
3124 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3125 struct bpf_prog
*old_prog
;
3127 bool reset
, was_opened
;
3130 mutex_lock(&priv
->state_lock
);
3132 if ((netdev
->features
& NETIF_F_LRO
) && prog
) {
3133 netdev_warn(netdev
, "can't set XDP while LRO is on, disable LRO first\n");
3138 was_opened
= test_bit(MLX5E_STATE_OPENED
, &priv
->state
);
3139 /* no need for full reset when exchanging programs */
3140 reset
= (!priv
->xdp_prog
|| !prog
);
3142 if (was_opened
&& reset
)
3143 mlx5e_close_locked(netdev
);
3144 if (was_opened
&& !reset
) {
3145 /* num_channels is invariant here, so we can take the
3146 * batched reference right upfront.
3148 prog
= bpf_prog_add(prog
, priv
->params
.num_channels
);
3150 err
= PTR_ERR(prog
);
3155 /* exchange programs, extra prog reference we got from caller
3156 * as long as we don't fail from this point onwards.
3158 old_prog
= xchg(&priv
->xdp_prog
, prog
);
3160 bpf_prog_put(old_prog
);
3162 if (reset
) /* change RQ type according to priv->xdp_prog */
3163 mlx5e_set_rq_priv_params(priv
);
3165 if (was_opened
&& reset
)
3166 mlx5e_open_locked(netdev
);
3168 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
) || reset
)
3171 /* exchanging programs w/o reset, we update ref counts on behalf
3172 * of the channels RQs here.
3174 for (i
= 0; i
< priv
->params
.num_channels
; i
++) {
3175 struct mlx5e_channel
*c
= priv
->channel
[i
];
3177 set_bit(MLX5E_RQ_STATE_FLUSH
, &c
->rq
.state
);
3178 napi_synchronize(&c
->napi
);
3179 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
3181 old_prog
= xchg(&c
->rq
.xdp_prog
, prog
);
3183 clear_bit(MLX5E_RQ_STATE_FLUSH
, &c
->rq
.state
);
3184 /* napi_schedule in case we have missed anything */
3185 set_bit(MLX5E_CHANNEL_NAPI_SCHED
, &c
->flags
);
3186 napi_schedule(&c
->napi
);
3189 bpf_prog_put(old_prog
);
3193 mutex_unlock(&priv
->state_lock
);
3197 static bool mlx5e_xdp_attached(struct net_device
*dev
)
3199 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3201 return !!priv
->xdp_prog
;
3204 static int mlx5e_xdp(struct net_device
*dev
, struct netdev_xdp
*xdp
)
3206 switch (xdp
->command
) {
3207 case XDP_SETUP_PROG
:
3208 return mlx5e_xdp_set(dev
, xdp
->prog
);
3209 case XDP_QUERY_PROG
:
3210 xdp
->prog_attached
= mlx5e_xdp_attached(dev
);
3217 #ifdef CONFIG_NET_POLL_CONTROLLER
3218 /* Fake "interrupt" called by netpoll (eg netconsole) to send skbs without
3219 * reenabling interrupts.
3221 static void mlx5e_netpoll(struct net_device
*dev
)
3223 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3226 for (i
= 0; i
< priv
->params
.num_channels
; i
++)
3227 napi_schedule(&priv
->channel
[i
]->napi
);
3231 static const struct net_device_ops mlx5e_netdev_ops_basic
= {
3232 .ndo_open
= mlx5e_open
,
3233 .ndo_stop
= mlx5e_close
,
3234 .ndo_start_xmit
= mlx5e_xmit
,
3235 .ndo_setup_tc
= mlx5e_ndo_setup_tc
,
3236 .ndo_select_queue
= mlx5e_select_queue
,
3237 .ndo_get_stats64
= mlx5e_get_stats
,
3238 .ndo_set_rx_mode
= mlx5e_set_rx_mode
,
3239 .ndo_set_mac_address
= mlx5e_set_mac
,
3240 .ndo_vlan_rx_add_vid
= mlx5e_vlan_rx_add_vid
,
3241 .ndo_vlan_rx_kill_vid
= mlx5e_vlan_rx_kill_vid
,
3242 .ndo_set_features
= mlx5e_set_features
,
3243 .ndo_change_mtu
= mlx5e_change_mtu
,
3244 .ndo_do_ioctl
= mlx5e_ioctl
,
3245 .ndo_set_tx_maxrate
= mlx5e_set_tx_maxrate
,
3246 #ifdef CONFIG_RFS_ACCEL
3247 .ndo_rx_flow_steer
= mlx5e_rx_flow_steer
,
3249 .ndo_tx_timeout
= mlx5e_tx_timeout
,
3250 .ndo_xdp
= mlx5e_xdp
,
3251 #ifdef CONFIG_NET_POLL_CONTROLLER
3252 .ndo_poll_controller
= mlx5e_netpoll
,
3256 static const struct net_device_ops mlx5e_netdev_ops_sriov
= {
3257 .ndo_open
= mlx5e_open
,
3258 .ndo_stop
= mlx5e_close
,
3259 .ndo_start_xmit
= mlx5e_xmit
,
3260 .ndo_setup_tc
= mlx5e_ndo_setup_tc
,
3261 .ndo_select_queue
= mlx5e_select_queue
,
3262 .ndo_get_stats64
= mlx5e_get_stats
,
3263 .ndo_set_rx_mode
= mlx5e_set_rx_mode
,
3264 .ndo_set_mac_address
= mlx5e_set_mac
,
3265 .ndo_vlan_rx_add_vid
= mlx5e_vlan_rx_add_vid
,
3266 .ndo_vlan_rx_kill_vid
= mlx5e_vlan_rx_kill_vid
,
3267 .ndo_set_features
= mlx5e_set_features
,
3268 .ndo_change_mtu
= mlx5e_change_mtu
,
3269 .ndo_do_ioctl
= mlx5e_ioctl
,
3270 .ndo_udp_tunnel_add
= mlx5e_add_vxlan_port
,
3271 .ndo_udp_tunnel_del
= mlx5e_del_vxlan_port
,
3272 .ndo_set_tx_maxrate
= mlx5e_set_tx_maxrate
,
3273 .ndo_features_check
= mlx5e_features_check
,
3274 #ifdef CONFIG_RFS_ACCEL
3275 .ndo_rx_flow_steer
= mlx5e_rx_flow_steer
,
3277 .ndo_set_vf_mac
= mlx5e_set_vf_mac
,
3278 .ndo_set_vf_vlan
= mlx5e_set_vf_vlan
,
3279 .ndo_set_vf_spoofchk
= mlx5e_set_vf_spoofchk
,
3280 .ndo_set_vf_trust
= mlx5e_set_vf_trust
,
3281 .ndo_set_vf_rate
= mlx5e_set_vf_rate
,
3282 .ndo_get_vf_config
= mlx5e_get_vf_config
,
3283 .ndo_set_vf_link_state
= mlx5e_set_vf_link_state
,
3284 .ndo_get_vf_stats
= mlx5e_get_vf_stats
,
3285 .ndo_tx_timeout
= mlx5e_tx_timeout
,
3286 .ndo_xdp
= mlx5e_xdp
,
3287 #ifdef CONFIG_NET_POLL_CONTROLLER
3288 .ndo_poll_controller
= mlx5e_netpoll
,
3290 .ndo_has_offload_stats
= mlx5e_has_offload_stats
,
3291 .ndo_get_offload_stats
= mlx5e_get_offload_stats
,
3294 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev
*mdev
)
3296 if (MLX5_CAP_GEN(mdev
, port_type
) != MLX5_CAP_PORT_TYPE_ETH
)
3298 if (!MLX5_CAP_GEN(mdev
, eth_net_offloads
) ||
3299 !MLX5_CAP_GEN(mdev
, nic_flow_table
) ||
3300 !MLX5_CAP_ETH(mdev
, csum_cap
) ||
3301 !MLX5_CAP_ETH(mdev
, max_lso_cap
) ||
3302 !MLX5_CAP_ETH(mdev
, vlan_cap
) ||
3303 !MLX5_CAP_ETH(mdev
, rss_ind_tbl_cap
) ||
3304 MLX5_CAP_FLOWTABLE(mdev
,
3305 flow_table_properties_nic_receive
.max_ft_level
)
3307 mlx5_core_warn(mdev
,
3308 "Not creating net device, some required device capabilities are missing\n");
3311 if (!MLX5_CAP_ETH(mdev
, self_lb_en_modifiable
))
3312 mlx5_core_warn(mdev
, "Self loop back prevention is not supported\n");
3313 if (!MLX5_CAP_GEN(mdev
, cq_moderation
))
3314 mlx5_core_warn(mdev
, "CQ modiration is not supported\n");
3319 u16
mlx5e_get_max_inline_cap(struct mlx5_core_dev
*mdev
)
3321 int bf_buf_size
= (1 << MLX5_CAP_GEN(mdev
, log_bf_reg_size
)) / 2;
3323 return bf_buf_size
-
3324 sizeof(struct mlx5e_tx_wqe
) +
3325 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3328 #ifdef CONFIG_MLX5_CORE_EN_DCB
3329 static void mlx5e_ets_init(struct mlx5e_priv
*priv
)
3333 priv
->params
.ets
.ets_cap
= mlx5_max_tc(priv
->mdev
) + 1;
3334 for (i
= 0; i
< priv
->params
.ets
.ets_cap
; i
++) {
3335 priv
->params
.ets
.tc_tx_bw
[i
] = MLX5E_MAX_BW_ALLOC
;
3336 priv
->params
.ets
.tc_tsa
[i
] = IEEE_8021QAZ_TSA_VENDOR
;
3337 priv
->params
.ets
.prio_tc
[i
] = i
;
3340 /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
3341 priv
->params
.ets
.prio_tc
[0] = 1;
3342 priv
->params
.ets
.prio_tc
[1] = 0;
3346 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev
*mdev
,
3347 u32
*indirection_rqt
, int len
,
3350 int node
= mdev
->priv
.numa_node
;
3351 int node_num_of_cores
;
3355 node
= first_online_node
;
3357 node_num_of_cores
= cpumask_weight(cpumask_of_node(node
));
3359 if (node_num_of_cores
)
3360 num_channels
= min_t(int, num_channels
, node_num_of_cores
);
3362 for (i
= 0; i
< len
; i
++)
3363 indirection_rqt
[i
] = i
% num_channels
;
3366 static int mlx5e_get_pci_bw(struct mlx5_core_dev
*mdev
, u32
*pci_bw
)
3368 enum pcie_link_width width
;
3369 enum pci_bus_speed speed
;
3372 err
= pcie_get_minimum_link(mdev
->pdev
, &speed
, &width
);
3376 if (speed
== PCI_SPEED_UNKNOWN
|| width
== PCIE_LNK_WIDTH_UNKNOWN
)
3380 case PCIE_SPEED_2_5GT
:
3381 *pci_bw
= 2500 * width
;
3383 case PCIE_SPEED_5_0GT
:
3384 *pci_bw
= 5000 * width
;
3386 case PCIE_SPEED_8_0GT
:
3387 *pci_bw
= 8000 * width
;
3396 static bool cqe_compress_heuristic(u32 link_speed
, u32 pci_bw
)
3398 return (link_speed
&& pci_bw
&&
3399 (pci_bw
< 40000) && (pci_bw
< link_speed
));
3402 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params
*params
, u8 cq_period_mode
)
3404 params
->rx_cq_period_mode
= cq_period_mode
;
3406 params
->rx_cq_moderation
.pkts
=
3407 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS
;
3408 params
->rx_cq_moderation
.usec
=
3409 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC
;
3411 if (cq_period_mode
== MLX5_CQ_PERIOD_MODE_START_FROM_CQE
)
3412 params
->rx_cq_moderation
.usec
=
3413 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE
;
3416 static void mlx5e_query_min_inline(struct mlx5_core_dev
*mdev
,
3417 u8
*min_inline_mode
)
3419 switch (MLX5_CAP_ETH(mdev
, wqe_inline_mode
)) {
3420 case MLX5E_INLINE_MODE_L2
:
3421 *min_inline_mode
= MLX5_INLINE_MODE_L2
;
3423 case MLX5E_INLINE_MODE_VPORT_CONTEXT
:
3424 mlx5_query_nic_vport_min_inline(mdev
,
3427 case MLX5_INLINE_MODE_NOT_REQUIRED
:
3428 *min_inline_mode
= MLX5_INLINE_MODE_NONE
;
3433 u32
mlx5e_choose_lro_timeout(struct mlx5_core_dev
*mdev
, u32 wanted_timeout
)
3437 /* The supported periods are organized in ascending order */
3438 for (i
= 0; i
< MLX5E_LRO_TIMEOUT_ARR_SIZE
- 1; i
++)
3439 if (MLX5_CAP_ETH(mdev
, lro_timer_supported_periods
[i
]) >= wanted_timeout
)
3442 return MLX5_CAP_ETH(mdev
, lro_timer_supported_periods
[i
]);
3445 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev
*mdev
,
3446 struct net_device
*netdev
,
3447 const struct mlx5e_profile
*profile
,
3450 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3453 u8 cq_period_mode
= MLX5_CAP_GEN(mdev
, cq_period_start_from_cqe
) ?
3454 MLX5_CQ_PERIOD_MODE_START_FROM_CQE
:
3455 MLX5_CQ_PERIOD_MODE_START_FROM_EQE
;
3458 priv
->netdev
= netdev
;
3459 priv
->params
.num_channels
= profile
->max_nch(mdev
);
3460 priv
->profile
= profile
;
3461 priv
->ppriv
= ppriv
;
3463 priv
->params
.lro_timeout
=
3464 mlx5e_choose_lro_timeout(mdev
, MLX5E_DEFAULT_LRO_TIMEOUT
);
3466 priv
->params
.log_sq_size
= MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE
;
3468 /* set CQE compression */
3469 priv
->params
.rx_cqe_compress_admin
= false;
3470 if (MLX5_CAP_GEN(mdev
, cqe_compression
) &&
3471 MLX5_CAP_GEN(mdev
, vport_group_manager
)) {
3472 mlx5e_get_max_linkspeed(mdev
, &link_speed
);
3473 mlx5e_get_pci_bw(mdev
, &pci_bw
);
3474 mlx5_core_dbg(mdev
, "Max link speed = %d, PCI BW = %d\n",
3475 link_speed
, pci_bw
);
3476 priv
->params
.rx_cqe_compress_admin
=
3477 cqe_compress_heuristic(link_speed
, pci_bw
);
3479 priv
->params
.rx_cqe_compress
= priv
->params
.rx_cqe_compress_admin
;
3481 mlx5e_set_rq_priv_params(priv
);
3482 if (priv
->params
.rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
)
3483 priv
->params
.lro_en
= true;
3485 priv
->params
.rx_am_enabled
= MLX5_CAP_GEN(mdev
, cq_moderation
);
3486 mlx5e_set_rx_cq_mode_params(&priv
->params
, cq_period_mode
);
3488 priv
->params
.tx_cq_moderation
.usec
=
3489 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC
;
3490 priv
->params
.tx_cq_moderation
.pkts
=
3491 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS
;
3492 priv
->params
.tx_max_inline
= mlx5e_get_max_inline_cap(mdev
);
3493 mlx5e_query_min_inline(mdev
, &priv
->params
.tx_min_inline_mode
);
3494 priv
->params
.num_tc
= 1;
3495 priv
->params
.rss_hfunc
= ETH_RSS_HASH_XOR
;
3497 netdev_rss_key_fill(priv
->params
.toeplitz_hash_key
,
3498 sizeof(priv
->params
.toeplitz_hash_key
));
3500 mlx5e_build_default_indir_rqt(mdev
, priv
->params
.indirection_rqt
,
3501 MLX5E_INDIR_RQT_SIZE
, profile
->max_nch(mdev
));
3503 priv
->params
.lro_wqe_sz
=
3504 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ
-
3505 /* Extra room needed for build_skb */
3507 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
3509 /* Initialize pflags */
3510 MLX5E_SET_PRIV_FLAG(priv
, MLX5E_PFLAG_RX_CQE_BASED_MODER
,
3511 priv
->params
.rx_cq_period_mode
== MLX5_CQ_PERIOD_MODE_START_FROM_CQE
);
3513 #ifdef CONFIG_MLX5_CORE_EN_DCB
3514 mlx5e_ets_init(priv
);
3517 mutex_init(&priv
->state_lock
);
3519 INIT_WORK(&priv
->update_carrier_work
, mlx5e_update_carrier_work
);
3520 INIT_WORK(&priv
->set_rx_mode_work
, mlx5e_set_rx_mode_work
);
3521 INIT_WORK(&priv
->tx_timeout_work
, mlx5e_tx_timeout_work
);
3522 INIT_DELAYED_WORK(&priv
->update_stats_work
, mlx5e_update_stats_work
);
3525 static void mlx5e_set_netdev_dev_addr(struct net_device
*netdev
)
3527 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3529 mlx5_query_nic_vport_mac_address(priv
->mdev
, 0, netdev
->dev_addr
);
3530 if (is_zero_ether_addr(netdev
->dev_addr
) &&
3531 !MLX5_CAP_GEN(priv
->mdev
, vport_group_manager
)) {
3532 eth_hw_addr_random(netdev
);
3533 mlx5_core_info(priv
->mdev
, "Assigned random MAC address %pM\n", netdev
->dev_addr
);
3537 static const struct switchdev_ops mlx5e_switchdev_ops
= {
3538 .switchdev_port_attr_get
= mlx5e_attr_get
,
3541 static void mlx5e_build_nic_netdev(struct net_device
*netdev
)
3543 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3544 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3548 SET_NETDEV_DEV(netdev
, &mdev
->pdev
->dev
);
3550 if (MLX5_CAP_GEN(mdev
, vport_group_manager
)) {
3551 netdev
->netdev_ops
= &mlx5e_netdev_ops_sriov
;
3552 #ifdef CONFIG_MLX5_CORE_EN_DCB
3553 netdev
->dcbnl_ops
= &mlx5e_dcbnl_ops
;
3556 netdev
->netdev_ops
= &mlx5e_netdev_ops_basic
;
3559 netdev
->watchdog_timeo
= 15 * HZ
;
3561 netdev
->ethtool_ops
= &mlx5e_ethtool_ops
;
3563 netdev
->vlan_features
|= NETIF_F_SG
;
3564 netdev
->vlan_features
|= NETIF_F_IP_CSUM
;
3565 netdev
->vlan_features
|= NETIF_F_IPV6_CSUM
;
3566 netdev
->vlan_features
|= NETIF_F_GRO
;
3567 netdev
->vlan_features
|= NETIF_F_TSO
;
3568 netdev
->vlan_features
|= NETIF_F_TSO6
;
3569 netdev
->vlan_features
|= NETIF_F_RXCSUM
;
3570 netdev
->vlan_features
|= NETIF_F_RXHASH
;
3572 if (!!MLX5_CAP_ETH(mdev
, lro_cap
))
3573 netdev
->vlan_features
|= NETIF_F_LRO
;
3575 netdev
->hw_features
= netdev
->vlan_features
;
3576 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_TX
;
3577 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_RX
;
3578 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_FILTER
;
3580 if (mlx5e_vxlan_allowed(mdev
)) {
3581 netdev
->hw_features
|= NETIF_F_GSO_UDP_TUNNEL
|
3582 NETIF_F_GSO_UDP_TUNNEL_CSUM
|
3583 NETIF_F_GSO_PARTIAL
;
3584 netdev
->hw_enc_features
|= NETIF_F_IP_CSUM
;
3585 netdev
->hw_enc_features
|= NETIF_F_IPV6_CSUM
;
3586 netdev
->hw_enc_features
|= NETIF_F_TSO
;
3587 netdev
->hw_enc_features
|= NETIF_F_TSO6
;
3588 netdev
->hw_enc_features
|= NETIF_F_GSO_UDP_TUNNEL
;
3589 netdev
->hw_enc_features
|= NETIF_F_GSO_UDP_TUNNEL_CSUM
|
3590 NETIF_F_GSO_PARTIAL
;
3591 netdev
->gso_partial_features
= NETIF_F_GSO_UDP_TUNNEL_CSUM
;
3594 mlx5_query_port_fcs(mdev
, &fcs_supported
, &fcs_enabled
);
3597 netdev
->hw_features
|= NETIF_F_RXALL
;
3599 netdev
->features
= netdev
->hw_features
;
3600 if (!priv
->params
.lro_en
)
3601 netdev
->features
&= ~NETIF_F_LRO
;
3604 netdev
->features
&= ~NETIF_F_RXALL
;
3606 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3607 if (FT_CAP(flow_modify_en
) &&
3608 FT_CAP(modify_root
) &&
3609 FT_CAP(identified_miss_table_mode
) &&
3610 FT_CAP(flow_table_modify
)) {
3611 netdev
->hw_features
|= NETIF_F_HW_TC
;
3612 #ifdef CONFIG_RFS_ACCEL
3613 netdev
->hw_features
|= NETIF_F_NTUPLE
;
3617 netdev
->features
|= NETIF_F_HIGHDMA
;
3619 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
3621 mlx5e_set_netdev_dev_addr(netdev
);
3623 #ifdef CONFIG_NET_SWITCHDEV
3624 if (MLX5_CAP_GEN(mdev
, vport_group_manager
))
3625 netdev
->switchdev_ops
= &mlx5e_switchdev_ops
;
3629 static void mlx5e_create_q_counter(struct mlx5e_priv
*priv
)
3631 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3634 err
= mlx5_core_alloc_q_counter(mdev
, &priv
->q_counter
);
3636 mlx5_core_warn(mdev
, "alloc queue counter failed, %d\n", err
);
3637 priv
->q_counter
= 0;
3641 static void mlx5e_destroy_q_counter(struct mlx5e_priv
*priv
)
3643 if (!priv
->q_counter
)
3646 mlx5_core_dealloc_q_counter(priv
->mdev
, priv
->q_counter
);
3649 static int mlx5e_create_umr_mkey(struct mlx5e_priv
*priv
)
3651 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3652 u64 npages
= MLX5E_REQUIRED_MTTS(priv
->profile
->max_nch(mdev
),
3653 BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW
));
3654 int inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
);
3659 in
= mlx5_vzalloc(inlen
);
3663 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
3665 npages
= min_t(u32
, ALIGN(U16_MAX
, 4) * 2, npages
);
3667 MLX5_SET(mkc
, mkc
, free
, 1);
3668 MLX5_SET(mkc
, mkc
, umr_en
, 1);
3669 MLX5_SET(mkc
, mkc
, lw
, 1);
3670 MLX5_SET(mkc
, mkc
, lr
, 1);
3671 MLX5_SET(mkc
, mkc
, access_mode
, MLX5_MKC_ACCESS_MODE_MTT
);
3673 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
3674 MLX5_SET(mkc
, mkc
, pd
, mdev
->mlx5e_res
.pdn
);
3675 MLX5_SET64(mkc
, mkc
, len
, npages
<< PAGE_SHIFT
);
3676 MLX5_SET(mkc
, mkc
, translations_octword_size
,
3677 MLX5_MTT_OCTW(npages
));
3678 MLX5_SET(mkc
, mkc
, log_page_size
, PAGE_SHIFT
);
3680 err
= mlx5_core_create_mkey(mdev
, &priv
->umr_mkey
, in
, inlen
);
3686 static void mlx5e_nic_init(struct mlx5_core_dev
*mdev
,
3687 struct net_device
*netdev
,
3688 const struct mlx5e_profile
*profile
,
3691 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3693 mlx5e_build_nic_netdev_priv(mdev
, netdev
, profile
, ppriv
);
3694 mlx5e_build_nic_netdev(netdev
);
3695 mlx5e_vxlan_init(priv
);
3698 static void mlx5e_nic_cleanup(struct mlx5e_priv
*priv
)
3700 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3701 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
3703 mlx5e_vxlan_cleanup(priv
);
3705 if (MLX5_CAP_GEN(mdev
, vport_group_manager
))
3706 mlx5_eswitch_unregister_vport_rep(esw
, 0);
3709 bpf_prog_put(priv
->xdp_prog
);
3712 static int mlx5e_init_nic_rx(struct mlx5e_priv
*priv
)
3714 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3718 err
= mlx5e_create_indirect_rqts(priv
);
3720 mlx5_core_warn(mdev
, "create indirect rqts failed, %d\n", err
);
3724 err
= mlx5e_create_direct_rqts(priv
);
3726 mlx5_core_warn(mdev
, "create direct rqts failed, %d\n", err
);
3727 goto err_destroy_indirect_rqts
;
3730 err
= mlx5e_create_indirect_tirs(priv
);
3732 mlx5_core_warn(mdev
, "create indirect tirs failed, %d\n", err
);
3733 goto err_destroy_direct_rqts
;
3736 err
= mlx5e_create_direct_tirs(priv
);
3738 mlx5_core_warn(mdev
, "create direct tirs failed, %d\n", err
);
3739 goto err_destroy_indirect_tirs
;
3742 err
= mlx5e_create_flow_steering(priv
);
3744 mlx5_core_warn(mdev
, "create flow steering failed, %d\n", err
);
3745 goto err_destroy_direct_tirs
;
3748 err
= mlx5e_tc_init(priv
);
3750 goto err_destroy_flow_steering
;
3754 err_destroy_flow_steering
:
3755 mlx5e_destroy_flow_steering(priv
);
3756 err_destroy_direct_tirs
:
3757 mlx5e_destroy_direct_tirs(priv
);
3758 err_destroy_indirect_tirs
:
3759 mlx5e_destroy_indirect_tirs(priv
);
3760 err_destroy_direct_rqts
:
3761 for (i
= 0; i
< priv
->profile
->max_nch(mdev
); i
++)
3762 mlx5e_destroy_rqt(priv
, &priv
->direct_tir
[i
].rqt
);
3763 err_destroy_indirect_rqts
:
3764 mlx5e_destroy_rqt(priv
, &priv
->indir_rqt
);
3768 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv
*priv
)
3772 mlx5e_tc_cleanup(priv
);
3773 mlx5e_destroy_flow_steering(priv
);
3774 mlx5e_destroy_direct_tirs(priv
);
3775 mlx5e_destroy_indirect_tirs(priv
);
3776 for (i
= 0; i
< priv
->profile
->max_nch(priv
->mdev
); i
++)
3777 mlx5e_destroy_rqt(priv
, &priv
->direct_tir
[i
].rqt
);
3778 mlx5e_destroy_rqt(priv
, &priv
->indir_rqt
);
3781 static int mlx5e_init_nic_tx(struct mlx5e_priv
*priv
)
3785 err
= mlx5e_create_tises(priv
);
3787 mlx5_core_warn(priv
->mdev
, "create tises failed, %d\n", err
);
3791 #ifdef CONFIG_MLX5_CORE_EN_DCB
3792 mlx5e_dcbnl_ieee_setets_core(priv
, &priv
->params
.ets
);
3797 static void mlx5e_nic_enable(struct mlx5e_priv
*priv
)
3799 struct net_device
*netdev
= priv
->netdev
;
3800 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3801 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
3802 struct mlx5_eswitch_rep rep
;
3804 mlx5_lag_add(mdev
, netdev
);
3806 if (mlx5e_vxlan_allowed(mdev
)) {
3808 udp_tunnel_get_rx_info(netdev
);
3812 mlx5e_enable_async_events(priv
);
3813 queue_work(priv
->wq
, &priv
->set_rx_mode_work
);
3815 if (MLX5_CAP_GEN(mdev
, vport_group_manager
)) {
3816 mlx5_query_nic_vport_mac_address(mdev
, 0, rep
.hw_id
);
3817 rep
.load
= mlx5e_nic_rep_load
;
3818 rep
.unload
= mlx5e_nic_rep_unload
;
3819 rep
.vport
= FDB_UPLINK_VPORT
;
3820 rep
.priv_data
= priv
;
3821 mlx5_eswitch_register_vport_rep(esw
, 0, &rep
);
3825 static void mlx5e_nic_disable(struct mlx5e_priv
*priv
)
3827 queue_work(priv
->wq
, &priv
->set_rx_mode_work
);
3828 mlx5e_disable_async_events(priv
);
3829 mlx5_lag_remove(priv
->mdev
);
3832 static const struct mlx5e_profile mlx5e_nic_profile
= {
3833 .init
= mlx5e_nic_init
,
3834 .cleanup
= mlx5e_nic_cleanup
,
3835 .init_rx
= mlx5e_init_nic_rx
,
3836 .cleanup_rx
= mlx5e_cleanup_nic_rx
,
3837 .init_tx
= mlx5e_init_nic_tx
,
3838 .cleanup_tx
= mlx5e_cleanup_nic_tx
,
3839 .enable
= mlx5e_nic_enable
,
3840 .disable
= mlx5e_nic_disable
,
3841 .update_stats
= mlx5e_update_stats
,
3842 .max_nch
= mlx5e_get_max_num_channels
,
3843 .max_tc
= MLX5E_MAX_NUM_TC
,
3846 struct net_device
*mlx5e_create_netdev(struct mlx5_core_dev
*mdev
,
3847 const struct mlx5e_profile
*profile
,
3850 int nch
= profile
->max_nch(mdev
);
3851 struct net_device
*netdev
;
3852 struct mlx5e_priv
*priv
;
3854 netdev
= alloc_etherdev_mqs(sizeof(struct mlx5e_priv
),
3855 nch
* profile
->max_tc
,
3858 mlx5_core_err(mdev
, "alloc_etherdev_mqs() failed\n");
3862 profile
->init(mdev
, netdev
, profile
, ppriv
);
3864 netif_carrier_off(netdev
);
3866 priv
= netdev_priv(netdev
);
3868 priv
->wq
= create_singlethread_workqueue("mlx5e");
3870 goto err_cleanup_nic
;
3875 profile
->cleanup(priv
);
3876 free_netdev(netdev
);
3881 int mlx5e_attach_netdev(struct mlx5_core_dev
*mdev
, struct net_device
*netdev
)
3883 const struct mlx5e_profile
*profile
;
3884 struct mlx5e_priv
*priv
;
3888 priv
= netdev_priv(netdev
);
3889 profile
= priv
->profile
;
3890 clear_bit(MLX5E_STATE_DESTROYING
, &priv
->state
);
3892 err
= mlx5e_create_umr_mkey(priv
);
3894 mlx5_core_err(mdev
, "create umr mkey failed, %d\n", err
);
3898 err
= profile
->init_tx(priv
);
3900 goto err_destroy_umr_mkey
;
3902 err
= mlx5e_open_drop_rq(priv
);
3904 mlx5_core_err(mdev
, "open drop rq failed, %d\n", err
);
3905 goto err_cleanup_tx
;
3908 err
= profile
->init_rx(priv
);
3910 goto err_close_drop_rq
;
3912 mlx5e_create_q_counter(priv
);
3914 mlx5e_init_l2_addr(priv
);
3916 /* MTU range: 68 - hw-specific max */
3917 netdev
->min_mtu
= ETH_MIN_MTU
;
3918 mlx5_query_port_max_mtu(priv
->mdev
, &max_mtu
, 1);
3919 netdev
->max_mtu
= MLX5E_HW2SW_MTU(max_mtu
);
3921 mlx5e_set_dev_port_mtu(netdev
);
3923 if (profile
->enable
)
3924 profile
->enable(priv
);
3927 if (netif_running(netdev
))
3929 netif_device_attach(netdev
);
3935 mlx5e_close_drop_rq(priv
);
3938 profile
->cleanup_tx(priv
);
3940 err_destroy_umr_mkey
:
3941 mlx5_core_destroy_mkey(mdev
, &priv
->umr_mkey
);
3947 static void mlx5e_register_vport_rep(struct mlx5_core_dev
*mdev
)
3949 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
3950 int total_vfs
= MLX5_TOTAL_VPORTS(mdev
);
3954 if (!MLX5_CAP_GEN(mdev
, vport_group_manager
))
3957 mlx5_query_nic_vport_mac_address(mdev
, 0, mac
);
3959 for (vport
= 1; vport
< total_vfs
; vport
++) {
3960 struct mlx5_eswitch_rep rep
;
3962 rep
.load
= mlx5e_vport_rep_load
;
3963 rep
.unload
= mlx5e_vport_rep_unload
;
3965 ether_addr_copy(rep
.hw_id
, mac
);
3966 mlx5_eswitch_register_vport_rep(esw
, vport
, &rep
);
3970 void mlx5e_detach_netdev(struct mlx5_core_dev
*mdev
, struct net_device
*netdev
)
3972 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3973 const struct mlx5e_profile
*profile
= priv
->profile
;
3975 set_bit(MLX5E_STATE_DESTROYING
, &priv
->state
);
3976 if (profile
->disable
)
3977 profile
->disable(priv
);
3979 flush_workqueue(priv
->wq
);
3982 if (netif_running(netdev
))
3983 mlx5e_close(netdev
);
3984 netif_device_detach(netdev
);
3987 mlx5e_destroy_q_counter(priv
);
3988 profile
->cleanup_rx(priv
);
3989 mlx5e_close_drop_rq(priv
);
3990 profile
->cleanup_tx(priv
);
3991 mlx5_core_destroy_mkey(priv
->mdev
, &priv
->umr_mkey
);
3992 cancel_delayed_work_sync(&priv
->update_stats_work
);
3995 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
3996 * hardware contexts and to connect it to the current netdev.
3998 static int mlx5e_attach(struct mlx5_core_dev
*mdev
, void *vpriv
)
4000 struct mlx5e_priv
*priv
= vpriv
;
4001 struct net_device
*netdev
= priv
->netdev
;
4004 if (netif_device_present(netdev
))
4007 err
= mlx5e_create_mdev_resources(mdev
);
4011 err
= mlx5e_attach_netdev(mdev
, netdev
);
4013 mlx5e_destroy_mdev_resources(mdev
);
4020 static void mlx5e_detach(struct mlx5_core_dev
*mdev
, void *vpriv
)
4022 struct mlx5e_priv
*priv
= vpriv
;
4023 struct net_device
*netdev
= priv
->netdev
;
4025 if (!netif_device_present(netdev
))
4028 mlx5e_detach_netdev(mdev
, netdev
);
4029 mlx5e_destroy_mdev_resources(mdev
);
4032 static void *mlx5e_add(struct mlx5_core_dev
*mdev
)
4034 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
4035 int total_vfs
= MLX5_TOTAL_VPORTS(mdev
);
4040 struct net_device
*netdev
;
4042 err
= mlx5e_check_required_hca_cap(mdev
);
4046 mlx5e_register_vport_rep(mdev
);
4048 if (MLX5_CAP_GEN(mdev
, vport_group_manager
))
4049 ppriv
= &esw
->offloads
.vport_reps
[0];
4051 netdev
= mlx5e_create_netdev(mdev
, &mlx5e_nic_profile
, ppriv
);
4053 mlx5_core_err(mdev
, "mlx5e_create_netdev failed\n");
4054 goto err_unregister_reps
;
4057 priv
= netdev_priv(netdev
);
4059 err
= mlx5e_attach(mdev
, priv
);
4061 mlx5_core_err(mdev
, "mlx5e_attach failed, %d\n", err
);
4062 goto err_destroy_netdev
;
4065 err
= register_netdev(netdev
);
4067 mlx5_core_err(mdev
, "register_netdev failed, %d\n", err
);
4074 mlx5e_detach(mdev
, priv
);
4077 mlx5e_destroy_netdev(mdev
, priv
);
4079 err_unregister_reps
:
4080 for (vport
= 1; vport
< total_vfs
; vport
++)
4081 mlx5_eswitch_unregister_vport_rep(esw
, vport
);
4086 void mlx5e_destroy_netdev(struct mlx5_core_dev
*mdev
, struct mlx5e_priv
*priv
)
4088 const struct mlx5e_profile
*profile
= priv
->profile
;
4089 struct net_device
*netdev
= priv
->netdev
;
4091 destroy_workqueue(priv
->wq
);
4092 if (profile
->cleanup
)
4093 profile
->cleanup(priv
);
4094 free_netdev(netdev
);
4097 static void mlx5e_remove(struct mlx5_core_dev
*mdev
, void *vpriv
)
4099 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
4100 int total_vfs
= MLX5_TOTAL_VPORTS(mdev
);
4101 struct mlx5e_priv
*priv
= vpriv
;
4104 for (vport
= 1; vport
< total_vfs
; vport
++)
4105 mlx5_eswitch_unregister_vport_rep(esw
, vport
);
4107 unregister_netdev(priv
->netdev
);
4108 mlx5e_detach(mdev
, vpriv
);
4109 mlx5e_destroy_netdev(mdev
, priv
);
4112 static void *mlx5e_get_netdev(void *vpriv
)
4114 struct mlx5e_priv
*priv
= vpriv
;
4116 return priv
->netdev
;
4119 static struct mlx5_interface mlx5e_interface
= {
4121 .remove
= mlx5e_remove
,
4122 .attach
= mlx5e_attach
,
4123 .detach
= mlx5e_detach
,
4124 .event
= mlx5e_async_event
,
4125 .protocol
= MLX5_INTERFACE_PROTOCOL_ETH
,
4126 .get_dev
= mlx5e_get_netdev
,
4129 void mlx5e_init(void)
4131 mlx5e_build_ptys2ethtool_map();
4132 mlx5_register_interface(&mlx5e_interface
);
4135 void mlx5e_cleanup(void)
4137 mlx5_unregister_interface(&mlx5e_interface
);