]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/net/ethernet/mellanox/mlx5/core/en_main.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 4.13-rc2
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c
CommitLineData
f62b8bb8 1/*
b3f63c3d 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
f62b8bb8
AV		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
e8f887ac
AV		 33#include <net/tc_act/tc_gact.h>
34#include <net/pkt_cls.h>
86d722ad 35#include <linux/mlx5/fs.h>
b3f63c3d 36#include <net/vxlan.h>
86994156 37#include <linux/bpf.h>
1d447a39 38#include "eswitch.h"
f62b8bb8 39#include "en.h"
e8f887ac 40#include "en_tc.h"
1d447a39 41#include "en_rep.h"
547eede0 42#include "en_accel/ipsec.h"
899a59d3
IT		 43#include "en_accel/ipsec_rxtx.h"
44#include "accel/ipsec.h"
b3f63c3d 45#include "vxlan.h"
f62b8bb8
AV		 46
47struct mlx5e_rq_param {
cb3c7fd4
GR		 48 u32 rqc[MLX5_ST_SZ_DW(rqc)];
49 struct mlx5_wq_param wq;
f62b8bb8
AV		 50};
51
52struct mlx5e_sq_param {
53 u32 sqc[MLX5_ST_SZ_DW(sqc)];
54 struct mlx5_wq_param wq;
55};
56
57struct mlx5e_cq_param {
58 u32 cqc[MLX5_ST_SZ_DW(cqc)];
59 struct mlx5_wq_param wq;
60 u16 eq_ix;
9908aa29 61 u8 cq_period_mode;
f62b8bb8
AV		 62};
63
64struct mlx5e_channel_param {
65 struct mlx5e_rq_param rq;
66 struct mlx5e_sq_param sq;
b5503b99 67 struct mlx5e_sq_param xdp_sq;
d3c9bc27 68 struct mlx5e_sq_param icosq;
f62b8bb8
AV		 69 struct mlx5e_cq_param rx_cq;
70 struct mlx5e_cq_param tx_cq;
d3c9bc27 71 struct mlx5e_cq_param icosq_cq;
f62b8bb8
AV		 72};
73
2fc4bfb7
SM		 74static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
75{
76 return MLX5_CAP_GEN(mdev, striding_rq) &&
77 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
78 MLX5_CAP_ETH(mdev, reg_umr_sq);
79}
80
6a9764ef
SM		 81void mlx5e_set_rq_type_params(struct mlx5_core_dev *mdev,
82 struct mlx5e_params *params, u8 rq_type)
2fc4bfb7 83{
6a9764ef
SM		 84 params->rq_wq_type = rq_type;
85 params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
86 switch (params->rq_wq_type) {
2fc4bfb7 87 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
6a9764ef 88 params->log_rq_size = is_kdump_kernel() ?
b4e029da
KH		 89 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW :
90 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
6a9764ef
SM		 91 params->mpwqe_log_stride_sz =
92 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) ?
93 MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) :
94 MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
95 params->mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
96 params->mpwqe_log_stride_sz;
2fc4bfb7
SM		 97 break;
98 default: /* MLX5_WQ_TYPE_LINKED_LIST */
6a9764ef 99 params->log_rq_size = is_kdump_kernel() ?
b4e029da
KH		 100 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
101 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
bce2b2bf
TT		 102 params->rq_headroom = params->xdp_prog ?
103 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
104 params->rq_headroom += NET_IP_ALIGN;
4078e637
TT		 105
106 /* Extra room needed for build_skb */
bce2b2bf 107 params->lro_wqe_sz -= params->rq_headroom +
4078e637 108 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2fc4bfb7 109 }
2fc4bfb7 110
6a9764ef
SM		 111 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
112 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
113 BIT(params->log_rq_size),
114 BIT(params->mpwqe_log_stride_sz),
115 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
2fc4bfb7
SM		 116}
117
6a9764ef 118static void mlx5e_set_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
2fc4bfb7 119{
6a9764ef 120 u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) &&
899a59d3 121 !params->xdp_prog && !MLX5_IPSEC_DEV(mdev) ?
2fc4bfb7
SM		 122 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
123 MLX5_WQ_TYPE_LINKED_LIST;
6a9764ef 124 mlx5e_set_rq_type_params(mdev, params, rq_type);
2fc4bfb7
SM		 125}
126
f62b8bb8
AV		 127static void mlx5e_update_carrier(struct mlx5e_priv *priv)
128{
129 struct mlx5_core_dev *mdev = priv->mdev;
130 u8 port_state;
131
132 port_state = mlx5_query_vport_state(mdev,
e53eef63
OG		 133 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT,
134 0);
f62b8bb8 135
87424ad5
SD		 136 if (port_state == VPORT_STATE_UP) {
137 netdev_info(priv->netdev, "Link up\n");
f62b8bb8 138 netif_carrier_on(priv->netdev);
87424ad5
SD		 139 } else {
140 netdev_info(priv->netdev, "Link down\n");
f62b8bb8 141 netif_carrier_off(priv->netdev);
87424ad5 142 }
f62b8bb8
AV		 143}
144
145static void mlx5e_update_carrier_work(struct work_struct *work)
146{
147 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
148 update_carrier_work);
149
150 mutex_lock(&priv->state_lock);
151 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
7ca42c80
ES		 152 if (priv->profile->update_carrier)
153 priv->profile->update_carrier(priv);
f62b8bb8
AV		 154 mutex_unlock(&priv->state_lock);
155}
156
3947ca18
DJ		 157static void mlx5e_tx_timeout_work(struct work_struct *work)
158{
159 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
160 tx_timeout_work);
161 int err;
162
163 rtnl_lock();
164 mutex_lock(&priv->state_lock);
165 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
166 goto unlock;
167 mlx5e_close_locked(priv->netdev);
168 err = mlx5e_open_locked(priv->netdev);
169 if (err)
170 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
171 err);
172unlock:
173 mutex_unlock(&priv->state_lock);
174 rtnl_unlock();
175}
176
9218b44d 177static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
f62b8bb8 178{
1510d728 179 struct mlx5e_sw_stats temp, *s = &temp;
f62b8bb8
AV		 180 struct mlx5e_rq_stats *rq_stats;
181 struct mlx5e_sq_stats *sq_stats;
9218b44d 182 u64 tx_offload_none = 0;
f62b8bb8
AV		 183 int i, j;
184
9218b44d 185 memset(s, 0, sizeof(*s));
ff9c852f
SM		 186 for (i = 0; i < priv->channels.num; i++) {
187 struct mlx5e_channel *c = priv->channels.c[i];
188
189 rq_stats = &c->rq.stats;
f62b8bb8 190
faf4478b
GP		 191 s->rx_packets += rq_stats->packets;
192 s->rx_bytes += rq_stats->bytes;
bfe6d8d1
GP		 193 s->rx_lro_packets += rq_stats->lro_packets;
194 s->rx_lro_bytes += rq_stats->lro_bytes;
f62b8bb8 195 s->rx_csum_none += rq_stats->csum_none;
bfe6d8d1
GP		 196 s->rx_csum_complete += rq_stats->csum_complete;
197 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
86994156 198 s->rx_xdp_drop += rq_stats->xdp_drop;
b5503b99
SM		 199 s->rx_xdp_tx += rq_stats->xdp_tx;
200 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
f62b8bb8 201 s->rx_wqe_err += rq_stats->wqe_err;
461017cb 202 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
54984407 203 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
7219ab34
TT		 204 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
205 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
accd5883 206 s->rx_page_reuse += rq_stats->page_reuse;
4415a031
TT		 207 s->rx_cache_reuse += rq_stats->cache_reuse;
208 s->rx_cache_full += rq_stats->cache_full;
209 s->rx_cache_empty += rq_stats->cache_empty;
210 s->rx_cache_busy += rq_stats->cache_busy;
f62b8bb8 211
6a9764ef 212 for (j = 0; j < priv->channels.params.num_tc; j++) {
ff9c852f 213 sq_stats = &c->sq[j].stats;
f62b8bb8 214
faf4478b
GP		 215 s->tx_packets += sq_stats->packets;
216 s->tx_bytes += sq_stats->bytes;
bfe6d8d1
GP		 217 s->tx_tso_packets += sq_stats->tso_packets;
218 s->tx_tso_bytes += sq_stats->tso_bytes;
219 s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
220 s->tx_tso_inner_bytes += sq_stats->tso_inner_bytes;
f62b8bb8
AV		 221 s->tx_queue_stopped += sq_stats->stopped;
222 s->tx_queue_wake += sq_stats->wake;
223 s->tx_queue_dropped += sq_stats->dropped;
c8cf78fe 224 s->tx_xmit_more += sq_stats->xmit_more;
bfe6d8d1
GP		 225 s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
226 tx_offload_none += sq_stats->csum_none;
f62b8bb8
AV		 227 }
228 }
229
9218b44d 230 /* Update calculated offload counters */
bfe6d8d1
GP		 231 s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
232 s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
121fcdc8 233
bfe6d8d1 234 s->link_down_events_phy = MLX5_GET(ppcnt_reg,
121fcdc8
GP		 235 priv->stats.pport.phy_counters,
236 counter_set.phys_layer_cntrs.link_down_events);
1510d728 237 memcpy(&priv->stats.sw, s, sizeof(*s));
9218b44d
GP		 238}
239
240static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
241{
242 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
243 u32 *out = (u32 *)priv->stats.vport.query_vport_out;
c4f287c4 244 u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
9218b44d
GP		 245 struct mlx5_core_dev *mdev = priv->mdev;
246
f62b8bb8
AV		 247 MLX5_SET(query_vport_counter_in, in, opcode,
248 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
249 MLX5_SET(query_vport_counter_in, in, op_mod, 0);
250 MLX5_SET(query_vport_counter_in, in, other_vport, 0);
251
9218b44d
GP		 252 mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
253}
254
3834a5e6 255static void mlx5e_update_pport_counters(struct mlx5e_priv *priv, bool full)
9218b44d
GP		 256{
257 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
258 struct mlx5_core_dev *mdev = priv->mdev;
0883b4f4 259 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
9218b44d 260 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
cf678570 261 int prio;
9218b44d 262 void *out;
f62b8bb8 263
9218b44d 264 MLX5_SET(ppcnt_reg, in, local_port, 1);
f62b8bb8 265
9218b44d
GP		 266 out = pstats->IEEE_802_3_counters;
267 MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
268 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
f62b8bb8 269
3834a5e6
GP		 270 if (!full)
271 return;
272
9218b44d
GP		 273 out = pstats->RFC_2863_counters;
274 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
275 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
276
277 out = pstats->RFC_2819_counters;
278 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
279 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
593cf338 280
121fcdc8
GP		 281 out = pstats->phy_counters;
282 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
283 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
284
5db0a4f6
GP		 285 if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) {
286 out = pstats->phy_statistical_counters;
287 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
288 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
289 }
290
cf678570
GP		 291 MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
292 for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
293 out = pstats->per_prio_counters[prio];
294 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
295 mlx5_core_access_reg(mdev, in, sz, out, sz,
296 MLX5_REG_PPCNT, 0, 0);
297 }
9218b44d
GP		 298}
299
300static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
301{
302 struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
432609a4
GP		 303 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
304 int err;
9218b44d
GP		 305
306 if (!priv->q_counter)
307 return;
308
432609a4
GP		 309 err = mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out, sizeof(out));
310 if (err)
311 return;
312
313 qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
9218b44d
GP		 314}
315
0f7f3481
GP		 316static void mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
317{
318 struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
319 struct mlx5_core_dev *mdev = priv->mdev;
0883b4f4 320 u32 in[MLX5_ST_SZ_DW(mpcnt_reg)] = {0};
0f7f3481
GP		 321 int sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
322 void *out;
0f7f3481
GP		 323
324 if (!MLX5_CAP_MCAM_FEATURE(mdev, pcie_performance_group))
325 return;
326
0f7f3481
GP		 327 out = pcie_stats->pcie_perf_counters;
328 MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
329 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
0f7f3481
GP		 330}
331
3834a5e6 332void mlx5e_update_stats(struct mlx5e_priv *priv, bool full)
9218b44d 333{
164f16f7 334 if (full) {
3834a5e6 335 mlx5e_update_pcie_counters(priv);
164f16f7
IT		 336 mlx5e_ipsec_update_stats(priv);
337 }
3834a5e6 338 mlx5e_update_pport_counters(priv, full);
3dd69e3d
SM		 339 mlx5e_update_vport_counters(priv);
340 mlx5e_update_q_counter(priv);
121fcdc8 341 mlx5e_update_sw_counters(priv);
f62b8bb8
AV		 342}
343
3834a5e6
GP		 344static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv)
345{
346 mlx5e_update_stats(priv, false);
347}
348
cb67b832 349void mlx5e_update_stats_work(struct work_struct *work)
f62b8bb8
AV		 350{
351 struct delayed_work *dwork = to_delayed_work(work);
352 struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
353 update_stats_work);
354 mutex_lock(&priv->state_lock);
355 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
6bfd390b 356 priv->profile->update_stats(priv);
7bb29755
MF		 357 queue_delayed_work(priv->wq, dwork,
358 msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
f62b8bb8
AV		 359 }
360 mutex_unlock(&priv->state_lock);
361}
362
daa21560
TT		 363static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
364 enum mlx5_dev_event event, unsigned long param)
f62b8bb8 365{
daa21560 366 struct mlx5e_priv *priv = vpriv;
ee7f1220
EE		 367 struct ptp_clock_event ptp_event;
368 struct mlx5_eqe *eqe = NULL;
daa21560 369
e0f46eb9 370 if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
daa21560
TT		 371 return;
372
f62b8bb8
AV		 373 switch (event) {
374 case MLX5_DEV_EVENT_PORT_UP:
375 case MLX5_DEV_EVENT_PORT_DOWN:
7bb29755 376 queue_work(priv->wq, &priv->update_carrier_work);
f62b8bb8 377 break;
ee7f1220
EE		 378 case MLX5_DEV_EVENT_PPS:
379 eqe = (struct mlx5_eqe *)param;
380 ptp_event.type = PTP_CLOCK_EXTTS;
381 ptp_event.index = eqe->data.pps.pin;
382 ptp_event.timestamp =
383 timecounter_cyc2time(&priv->tstamp.clock,
384 be64_to_cpu(eqe->data.pps.time_stamp));
385 mlx5e_pps_event_handler(vpriv, &ptp_event);
386 break;
f62b8bb8
AV		 387 default:
388 break;
389 }
390}
391
f62b8bb8
AV		 392static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
393{
e0f46eb9 394 set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
f62b8bb8
AV		 395}
396
397static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
398{
e0f46eb9 399 clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
daa21560 400 synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
f62b8bb8
AV		 401}
402
7e426671
TT		 403static inline int mlx5e_get_wqe_mtt_sz(void)
404{
405 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
406 * To avoid copying garbage after the mtt array, we allocate
407 * a little more.
408 */
409 return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
410 MLX5_UMR_MTT_ALIGNMENT);
411}
412
31391048
SM		 413static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
414 struct mlx5e_icosq *sq,
415 struct mlx5e_umr_wqe *wqe,
416 u16 ix)
7e426671
TT		 417{
418 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
419 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
420 struct mlx5_wqe_data_seg *dseg = &wqe->data;
21c59685 421 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
7e426671
TT		 422 u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
423 u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
424
425 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
426 ds_cnt);
427 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
428 cseg->imm = rq->mkey_be;
429
430 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
31616255 431 ucseg->xlt_octowords =
7e426671
TT		 432 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
433 ucseg->bsf_octowords =
434 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
435 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
436
437 dseg->lkey = sq->mkey_be;
438 dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
439}
440
441static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
442 struct mlx5e_channel *c)
443{
444 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
445 int mtt_sz = mlx5e_get_wqe_mtt_sz();
446 int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
447 int i;
448
21c59685
SM		 449 rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
450 GFP_KERNEL, cpu_to_node(c->cpu));
451 if (!rq->mpwqe.info)
7e426671
TT		 452 goto err_out;
453
454 /* We allocate more than mtt_sz as we will align the pointer */
21c59685 455 rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
7e426671 456 cpu_to_node(c->cpu));
21c59685 457 if (unlikely(!rq->mpwqe.mtt_no_align))
7e426671
TT		 458 goto err_free_wqe_info;
459
460 for (i = 0; i < wq_sz; i++) {
21c59685 461 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
7e426671 462
21c59685 463 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
7e426671
TT		 464 MLX5_UMR_ALIGN);
465 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
466 PCI_DMA_TODEVICE);
467 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
468 goto err_unmap_mtts;
469
470 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
471 }
472
473 return 0;
474
475err_unmap_mtts:
476 while (--i >= 0) {
21c59685 477 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
7e426671
TT		 478
479 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
480 PCI_DMA_TODEVICE);
481 }
21c59685 482 kfree(rq->mpwqe.mtt_no_align);
7e426671 483err_free_wqe_info:
21c59685 484 kfree(rq->mpwqe.info);
7e426671
TT		 485
486err_out:
487 return -ENOMEM;
488}
489
490static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
491{
492 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
493 int mtt_sz = mlx5e_get_wqe_mtt_sz();
494 int i;
495
496 for (i = 0; i < wq_sz; i++) {
21c59685 497 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
7e426671
TT		 498
499 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
500 PCI_DMA_TODEVICE);
501 }
21c59685
SM		 502 kfree(rq->mpwqe.mtt_no_align);
503 kfree(rq->mpwqe.info);
7e426671
TT		 504}
505
a43b25da 506static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
ec8b9981
TT		 507 u64 npages, u8 page_shift,
508 struct mlx5_core_mkey *umr_mkey)
3608ae77 509{
3608ae77
TT		 510 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
511 void *mkc;
512 u32 *in;
513 int err;
514
ec8b9981
TT		 515 if (!MLX5E_VALID_NUM_MTTS(npages))
516 return -EINVAL;
517
1b9a07ee 518 in = kvzalloc(inlen, GFP_KERNEL);
3608ae77
TT		 519 if (!in)
520 return -ENOMEM;
521
522 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
523
3608ae77
TT		 524 MLX5_SET(mkc, mkc, free, 1);
525 MLX5_SET(mkc, mkc, umr_en, 1);
526 MLX5_SET(mkc, mkc, lw, 1);
527 MLX5_SET(mkc, mkc, lr, 1);
528 MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
529
530 MLX5_SET(mkc, mkc, qpn, 0xffffff);
531 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
ec8b9981 532 MLX5_SET64(mkc, mkc, len, npages << page_shift);
3608ae77
TT		 533 MLX5_SET(mkc, mkc, translations_octword_size,
534 MLX5_MTT_OCTW(npages));
ec8b9981 535 MLX5_SET(mkc, mkc, log_page_size, page_shift);
3608ae77 536
ec8b9981 537 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
3608ae77
TT		 538
539 kvfree(in);
540 return err;
541}
542
a43b25da 543static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
ec8b9981 544{
6a9764ef 545 u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->wq));
ec8b9981 546
a43b25da 547 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
ec8b9981
TT		 548}
549
3b77235b 550static int mlx5e_alloc_rq(struct mlx5e_channel *c,
6a9764ef
SM		 551 struct mlx5e_params *params,
552 struct mlx5e_rq_param *rqp,
3b77235b 553 struct mlx5e_rq *rq)
f62b8bb8 554{
a43b25da 555 struct mlx5_core_dev *mdev = c->mdev;
6a9764ef 556 void *rqc = rqp->rqc;
f62b8bb8 557 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
461017cb 558 u32 byte_count;
1bfecfca 559 int npages;
f62b8bb8
AV		 560 int wq_sz;
561 int err;
562 int i;
563
6a9764ef 564 rqp->wq.db_numa_node = cpu_to_node(c->cpu);
311c7c71 565
6a9764ef 566 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->wq,
f62b8bb8
AV		 567 &rq->wq_ctrl);
568 if (err)
569 return err;
570
571 rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
572
573 wq_sz = mlx5_wq_ll_get_size(&rq->wq);
f62b8bb8 574
6a9764ef 575 rq->wq_type = params->rq_wq_type;
7e426671
TT		 576 rq->pdev = c->pdev;
577 rq->netdev = c->netdev;
a43b25da 578 rq->tstamp = c->tstamp;
7e426671
TT		 579 rq->channel = c;
580 rq->ix = c->ix;
a43b25da 581 rq->mdev = mdev;
97bc402d 582
6a9764ef 583 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
97bc402d
DB		 584 if (IS_ERR(rq->xdp_prog)) {
585 err = PTR_ERR(rq->xdp_prog);
586 rq->xdp_prog = NULL;
587 goto err_rq_wq_destroy;
588 }
7e426671 589
bce2b2bf
TT		 590 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
591 rq->rx_headroom = params->rq_headroom;
b5503b99 592
6a9764ef 593 switch (rq->wq_type) {
461017cb 594 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
f5f82476 595
461017cb 596 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
6cd392a0 597 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
461017cb 598
20fd0c19 599 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
899a59d3
IT		 600#ifdef CONFIG_MLX5_EN_IPSEC
601 if (MLX5_IPSEC_DEV(mdev)) {
602 err = -EINVAL;
603 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
604 goto err_rq_wq_destroy;
605 }
606#endif
20fd0c19
SM		 607 if (!rq->handle_rx_cqe) {
608 err = -EINVAL;
609 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err);
610 goto err_rq_wq_destroy;
611 }
612
6a9764ef
SM		 613 rq->mpwqe_stride_sz = BIT(params->mpwqe_log_stride_sz);
614 rq->mpwqe_num_strides = BIT(params->mpwqe_log_num_strides);
1bfecfca
SM		 615
616 rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
617 byte_count = rq->buff.wqe_sz;
ec8b9981 618
a43b25da 619 err = mlx5e_create_rq_umr_mkey(mdev, rq);
7e426671
TT		 620 if (err)
621 goto err_rq_wq_destroy;
ec8b9981
TT		 622 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
623
624 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
625 if (err)
626 goto err_destroy_umr_mkey;
461017cb
TT		 627 break;
628 default: /* MLX5_WQ_TYPE_LINKED_LIST */
accd5883
TT		 629 rq->wqe.frag_info =
630 kzalloc_node(wq_sz * sizeof(*rq->wqe.frag_info),
631 GFP_KERNEL, cpu_to_node(c->cpu));
632 if (!rq->wqe.frag_info) {
461017cb
TT		 633 err = -ENOMEM;
634 goto err_rq_wq_destroy;
635 }
461017cb 636 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
6cd392a0 637 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
461017cb 638
899a59d3
IT		 639#ifdef CONFIG_MLX5_EN_IPSEC
640 if (c->priv->ipsec)
641 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
642 else
643#endif
644 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
20fd0c19 645 if (!rq->handle_rx_cqe) {
accd5883 646 kfree(rq->wqe.frag_info);
20fd0c19
SM		 647 err = -EINVAL;
648 netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err);
649 goto err_rq_wq_destroy;
650 }
651
6a9764ef
SM		 652 rq->buff.wqe_sz = params->lro_en ?
653 params->lro_wqe_sz :
c139dbfd 654 MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu);
899a59d3
IT		 655#ifdef CONFIG_MLX5_EN_IPSEC
656 if (MLX5_IPSEC_DEV(mdev))
657 rq->buff.wqe_sz += MLX5E_METADATA_ETHER_LEN;
658#endif
accd5883 659 rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en;
1bfecfca
SM		 660 byte_count = rq->buff.wqe_sz;
661
662 /* calc the required page order */
accd5883
TT		 663 rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->rx_headroom + byte_count);
664 npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE);
1bfecfca
SM		 665 rq->buff.page_order = order_base_2(npages);
666
461017cb 667 byte_count |= MLX5_HW_START_PADDING;
7e426671 668 rq->mkey_be = c->mkey_be;
461017cb 669 }
f62b8bb8
AV		 670
671 for (i = 0; i < wq_sz; i++) {
672 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
673
461017cb 674 wqe->data.byte_count = cpu_to_be32(byte_count);
7e426671 675 wqe->data.lkey = rq->mkey_be;
f62b8bb8
AV		 676 }
677
cb3c7fd4 678 INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
6a9764ef 679 rq->am.mode = params->rx_cq_period_mode;
4415a031
TT		 680 rq->page_cache.head = 0;
681 rq->page_cache.tail = 0;
682
f62b8bb8
AV		 683 return 0;
684
ec8b9981
TT		 685err_destroy_umr_mkey:
686 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
687
f62b8bb8 688err_rq_wq_destroy:
97bc402d
DB		 689 if (rq->xdp_prog)
690 bpf_prog_put(rq->xdp_prog);
f62b8bb8
AV		 691 mlx5_wq_destroy(&rq->wq_ctrl);
692
693 return err;
694}
695
3b77235b 696static void mlx5e_free_rq(struct mlx5e_rq *rq)
f62b8bb8 697{
4415a031
TT		 698 int i;
699
86994156
RS		 700 if (rq->xdp_prog)
701 bpf_prog_put(rq->xdp_prog);
702
461017cb
TT		 703 switch (rq->wq_type) {
704 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
7e426671 705 mlx5e_rq_free_mpwqe_info(rq);
a43b25da 706 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
461017cb
TT		 707 break;
708 default: /* MLX5_WQ_TYPE_LINKED_LIST */
accd5883 709 kfree(rq->wqe.frag_info);
461017cb
TT		 710 }
711
4415a031
TT		 712 for (i = rq->page_cache.head; i != rq->page_cache.tail;
713 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
714 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
715
716 mlx5e_page_release(rq, dma_info, false);
717 }
f62b8bb8
AV		 718 mlx5_wq_destroy(&rq->wq_ctrl);
719}
720
6a9764ef
SM		 721static int mlx5e_create_rq(struct mlx5e_rq *rq,
722 struct mlx5e_rq_param *param)
f62b8bb8 723{
a43b25da 724 struct mlx5_core_dev *mdev = rq->mdev;
f62b8bb8
AV		 725
726 void *in;
727 void *rqc;
728 void *wq;
729 int inlen;
730 int err;
731
732 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
733 sizeof(u64) * rq->wq_ctrl.buf.npages;
1b9a07ee 734 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 735 if (!in)
736 return -ENOMEM;
737
738 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
739 wq = MLX5_ADDR_OF(rqc, rqc, wq);
740
741 memcpy(rqc, param->rqc, sizeof(param->rqc));
742
97de9f31 743 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
f62b8bb8 744 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
f62b8bb8 745 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
68cdf5d6 746 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 747 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
748
749 mlx5_fill_page_array(&rq->wq_ctrl.buf,
750 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
751
7db22ffb 752 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
f62b8bb8
AV		 753
754 kvfree(in);
755
756 return err;
757}
758
36350114
GP		 759static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
760 int next_state)
f62b8bb8
AV		 761{
762 struct mlx5e_channel *c = rq->channel;
a43b25da 763 struct mlx5_core_dev *mdev = c->mdev;
f62b8bb8
AV		 764
765 void *in;
766 void *rqc;
767 int inlen;
768 int err;
769
770 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 771 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 772 if (!in)
773 return -ENOMEM;
774
775 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
776
777 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
778 MLX5_SET(rqc, rqc, state, next_state);
779
7db22ffb 780 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
f62b8bb8
AV		 781
782 kvfree(in);
783
784 return err;
785}
786
102722fc
GE		 787static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
788{
789 struct mlx5e_channel *c = rq->channel;
790 struct mlx5e_priv *priv = c->priv;
791 struct mlx5_core_dev *mdev = priv->mdev;
792
793 void *in;
794 void *rqc;
795 int inlen;
796 int err;
797
798 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 799 in = kvzalloc(inlen, GFP_KERNEL);
102722fc
GE		 800 if (!in)
801 return -ENOMEM;
802
803 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
804
805 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
806 MLX5_SET64(modify_rq_in, in, modify_bitmask,
807 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
808 MLX5_SET(rqc, rqc, scatter_fcs, enable);
809 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
810
811 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
812
813 kvfree(in);
814
815 return err;
816}
817
36350114
GP		 818static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
819{
820 struct mlx5e_channel *c = rq->channel;
a43b25da 821 struct mlx5_core_dev *mdev = c->mdev;
36350114
GP		 822 void *in;
823 void *rqc;
824 int inlen;
825 int err;
826
827 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 828 in = kvzalloc(inlen, GFP_KERNEL);
36350114
GP		 829 if (!in)
830 return -ENOMEM;
831
832 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
833
834 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
83b502a1
AV		 835 MLX5_SET64(modify_rq_in, in, modify_bitmask,
836 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
36350114
GP		 837 MLX5_SET(rqc, rqc, vsd, vsd);
838 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
839
840 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
841
842 kvfree(in);
843
844 return err;
845}
846
3b77235b 847static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
f62b8bb8 848{
a43b25da 849 mlx5_core_destroy_rq(rq->mdev, rq->rqn);
f62b8bb8
AV		 850}
851
852static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
853{
01c196a2 854 unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
f62b8bb8 855 struct mlx5e_channel *c = rq->channel;
a43b25da 856
f62b8bb8 857 struct mlx5_wq_ll *wq = &rq->wq;
6a9764ef 858 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5_wq_ll_get_size(wq));
f62b8bb8 859
01c196a2 860 while (time_before(jiffies, exp_time)) {
6a9764ef 861 if (wq->cur_sz >= min_wqes)
f62b8bb8
AV		 862 return 0;
863
864 msleep(20);
865 }
866
a43b25da 867 netdev_warn(c->netdev, "Failed to get min RX wqes on RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
6a9764ef 868 rq->rqn, wq->cur_sz, min_wqes);
f62b8bb8
AV		 869 return -ETIMEDOUT;
870}
871
f2fde18c
SM		 872static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
873{
874 struct mlx5_wq_ll *wq = &rq->wq;
875 struct mlx5e_rx_wqe *wqe;
876 __be16 wqe_ix_be;
877 u16 wqe_ix;
878
8484f9ed
SM		 879 /* UMR WQE (if in progress) is always at wq->head */
880 if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
21c59685 881 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
8484f9ed 882
f2fde18c
SM		 883 while (!mlx5_wq_ll_is_empty(wq)) {
884 wqe_ix_be = *wq->tail_next;
885 wqe_ix = be16_to_cpu(wqe_ix_be);
886 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
887 rq->dealloc_wqe(rq, wqe_ix);
888 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
889 &wqe->next.next_wqe_index);
890 }
accd5883
TT		 891
892 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST && rq->wqe.page_reuse) {
893 /* Clean outstanding pages on handled WQEs that decided to do page-reuse,
894 * but yet to be re-posted.
895 */
896 int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
897
898 for (wqe_ix = 0; wqe_ix < wq_sz; wqe_ix++)
899 rq->dealloc_wqe(rq, wqe_ix);
900 }
f2fde18c
SM		 901}
902
f62b8bb8 903static int mlx5e_open_rq(struct mlx5e_channel *c,
6a9764ef 904 struct mlx5e_params *params,
f62b8bb8
AV		 905 struct mlx5e_rq_param *param,
906 struct mlx5e_rq *rq)
907{
908 int err;
909
6a9764ef 910 err = mlx5e_alloc_rq(c, params, param, rq);
f62b8bb8
AV		 911 if (err)
912 return err;
913
3b77235b 914 err = mlx5e_create_rq(rq, param);
f62b8bb8 915 if (err)
3b77235b 916 goto err_free_rq;
f62b8bb8 917
36350114 918 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
f62b8bb8 919 if (err)
3b77235b 920 goto err_destroy_rq;
f62b8bb8 921
6a9764ef 922 if (params->rx_am_enabled)
cb3c7fd4
GR		 923 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
924
f62b8bb8
AV		 925 return 0;
926
f62b8bb8
AV		 927err_destroy_rq:
928 mlx5e_destroy_rq(rq);
3b77235b
SM		 929err_free_rq:
930 mlx5e_free_rq(rq);
f62b8bb8
AV		 931
932 return err;
933}
934
acc6c595
SM		 935static void mlx5e_activate_rq(struct mlx5e_rq *rq)
936{
937 struct mlx5e_icosq *sq = &rq->channel->icosq;
938 u16 pi = sq->pc & sq->wq.sz_m1;
939 struct mlx5e_tx_wqe *nopwqe;
940
941 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
942 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
943 sq->db.ico_wqe[pi].num_wqebbs = 1;
944 nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
945 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
946}
947
948static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
f62b8bb8 949{
c0f1147d 950 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
f62b8bb8 951 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
acc6c595 952}
cb3c7fd4 953
acc6c595
SM		 954static void mlx5e_close_rq(struct mlx5e_rq *rq)
955{
956 cancel_work_sync(&rq->am.work);
f62b8bb8 957 mlx5e_destroy_rq(rq);
3b77235b
SM		 958 mlx5e_free_rx_descs(rq);
959 mlx5e_free_rq(rq);
f62b8bb8
AV		 960}
961
31391048 962static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
b5503b99 963{
31391048 964 kfree(sq->db.di);
b5503b99
SM		 965}
966
31391048 967static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
b5503b99
SM		 968{
969 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
970
31391048 971 sq->db.di = kzalloc_node(sizeof(*sq->db.di) * wq_sz,
b5503b99 972 GFP_KERNEL, numa);
31391048
SM		 973 if (!sq->db.di) {
974 mlx5e_free_xdpsq_db(sq);
b5503b99
SM		 975 return -ENOMEM;
976 }
977
978 return 0;
979}
980
31391048 981static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
6a9764ef 982 struct mlx5e_params *params,
31391048
SM		 983 struct mlx5e_sq_param *param,
984 struct mlx5e_xdpsq *sq)
985{
986 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 987 struct mlx5_core_dev *mdev = c->mdev;
31391048
SM		 988 int err;
989
990 sq->pdev = c->pdev;
991 sq->mkey_be = c->mkey_be;
992 sq->channel = c;
993 sq->uar_map = mdev->mlx5e_res.bfreg.map;
6a9764ef 994 sq->min_inline_mode = params->tx_min_inline_mode;
31391048
SM		 995
996 param->wq.db_numa_node = cpu_to_node(c->cpu);
997 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
998 if (err)
999 return err;
1000 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1001
1002 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1003 if (err)
1004 goto err_sq_wq_destroy;
1005
1006 return 0;
1007
1008err_sq_wq_destroy:
1009 mlx5_wq_destroy(&sq->wq_ctrl);
1010
1011 return err;
1012}
1013
1014static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1015{
1016 mlx5e_free_xdpsq_db(sq);
1017 mlx5_wq_destroy(&sq->wq_ctrl);
1018}
1019
1020static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
f62b8bb8 1021{
f10b7cc7 1022 kfree(sq->db.ico_wqe);
f62b8bb8
AV		 1023}
1024
31391048 1025static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
f10b7cc7
SM		 1026{
1027 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1028
1029 sq->db.ico_wqe = kzalloc_node(sizeof(*sq->db.ico_wqe) * wq_sz,
1030 GFP_KERNEL, numa);
1031 if (!sq->db.ico_wqe)
1032 return -ENOMEM;
1033
1034 return 0;
1035}
1036
31391048 1037static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
31391048
SM		 1038 struct mlx5e_sq_param *param,
1039 struct mlx5e_icosq *sq)
f10b7cc7 1040{
31391048 1041 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1042 struct mlx5_core_dev *mdev = c->mdev;
31391048 1043 int err;
f10b7cc7 1044
31391048
SM		 1045 sq->pdev = c->pdev;
1046 sq->mkey_be = c->mkey_be;
1047 sq->channel = c;
1048 sq->uar_map = mdev->mlx5e_res.bfreg.map;
f62b8bb8 1049
31391048
SM		 1050 param->wq.db_numa_node = cpu_to_node(c->cpu);
1051 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
1052 if (err)
1053 return err;
1054 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
f62b8bb8 1055
31391048
SM		 1056 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1057 if (err)
1058 goto err_sq_wq_destroy;
1059
1060 sq->edge = (sq->wq.sz_m1 + 1) - MLX5E_ICOSQ_MAX_WQEBBS;
f62b8bb8
AV		 1061
1062 return 0;
31391048
SM		 1063
1064err_sq_wq_destroy:
1065 mlx5_wq_destroy(&sq->wq_ctrl);
1066
1067 return err;
f62b8bb8
AV		 1068}
1069
31391048 1070static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
f10b7cc7 1071{
31391048
SM		 1072 mlx5e_free_icosq_db(sq);
1073 mlx5_wq_destroy(&sq->wq_ctrl);
f10b7cc7
SM		 1074}
1075
31391048 1076static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
f10b7cc7 1077{
31391048
SM		 1078 kfree(sq->db.wqe_info);
1079 kfree(sq->db.dma_fifo);
f10b7cc7
SM		 1080}
1081
31391048 1082static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
b5503b99 1083{
31391048
SM		 1084 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1085 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1086
31391048
SM		 1087 sq->db.dma_fifo = kzalloc_node(df_sz * sizeof(*sq->db.dma_fifo),
1088 GFP_KERNEL, numa);
1089 sq->db.wqe_info = kzalloc_node(wq_sz * sizeof(*sq->db.wqe_info),
1090 GFP_KERNEL, numa);
77bdf895 1091 if (!sq->db.dma_fifo || !sq->db.wqe_info) {
31391048
SM		 1092 mlx5e_free_txqsq_db(sq);
1093 return -ENOMEM;
b5503b99 1094 }
31391048
SM		 1095
1096 sq->dma_fifo_mask = df_sz - 1;
1097
1098 return 0;
b5503b99
SM		 1099}
1100
31391048 1101static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
acc6c595 1102 int txq_ix,
6a9764ef 1103 struct mlx5e_params *params,
31391048
SM		 1104 struct mlx5e_sq_param *param,
1105 struct mlx5e_txqsq *sq)
f62b8bb8 1106{
31391048 1107 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1108 struct mlx5_core_dev *mdev = c->mdev;
f62b8bb8
AV		 1109 int err;
1110
f10b7cc7 1111 sq->pdev = c->pdev;
a43b25da 1112 sq->tstamp = c->tstamp;
f10b7cc7
SM		 1113 sq->mkey_be = c->mkey_be;
1114 sq->channel = c;
acc6c595 1115 sq->txq_ix = txq_ix;
aff26157 1116 sq->uar_map = mdev->mlx5e_res.bfreg.map;
6a9764ef
SM		 1117 sq->max_inline = params->tx_max_inline;
1118 sq->min_inline_mode = params->tx_min_inline_mode;
2ac9cfe7
IT		 1119 if (MLX5_IPSEC_DEV(c->priv->mdev))
1120 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
f10b7cc7 1121
311c7c71 1122 param->wq.db_numa_node = cpu_to_node(c->cpu);
31391048 1123 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
f62b8bb8 1124 if (err)
aff26157 1125 return err;
31391048 1126 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
f62b8bb8 1127
31391048 1128 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
7ec0bb22 1129 if (err)
f62b8bb8
AV		 1130 goto err_sq_wq_destroy;
1131
31391048 1132 sq->edge = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
f62b8bb8
AV		 1133
1134 return 0;
1135
1136err_sq_wq_destroy:
1137 mlx5_wq_destroy(&sq->wq_ctrl);
1138
f62b8bb8
AV		 1139 return err;
1140}
1141
31391048 1142static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1143{
31391048 1144 mlx5e_free_txqsq_db(sq);
f62b8bb8 1145 mlx5_wq_destroy(&sq->wq_ctrl);
f62b8bb8
AV		 1146}
1147
33ad9711
SM		 1148struct mlx5e_create_sq_param {
1149 struct mlx5_wq_ctrl *wq_ctrl;
1150 u32 cqn;
1151 u32 tisn;
1152 u8 tis_lst_sz;
1153 u8 min_inline_mode;
1154};
1155
a43b25da 1156static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
33ad9711
SM		 1157 struct mlx5e_sq_param *param,
1158 struct mlx5e_create_sq_param *csp,
1159 u32 *sqn)
f62b8bb8 1160{
f62b8bb8
AV		 1161 void *in;
1162 void *sqc;
1163 void *wq;
1164 int inlen;
1165 int err;
1166
1167 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
33ad9711 1168 sizeof(u64) * csp->wq_ctrl->buf.npages;
1b9a07ee 1169 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1170 if (!in)
1171 return -ENOMEM;
1172
1173 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1174 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1175
1176 memcpy(sqc, param->sqc, sizeof(param->sqc));
33ad9711
SM		 1177 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz);
1178 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn);
1179 MLX5_SET(sqc, sqc, cqn, csp->cqn);
a6f402e4
SM		 1180
1181 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
33ad9711 1182 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode);
a6f402e4 1183
33ad9711 1184 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
f62b8bb8
AV		 1185
1186 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
a43b25da 1187 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index);
33ad9711 1188 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift -
68cdf5d6 1189 MLX5_ADAPTER_PAGE_SHIFT);
33ad9711 1190 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma);
f62b8bb8 1191
33ad9711 1192 mlx5_fill_page_array(&csp->wq_ctrl->buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
f62b8bb8 1193
33ad9711 1194 err = mlx5_core_create_sq(mdev, in, inlen, sqn);
f62b8bb8
AV		 1195
1196 kvfree(in);
1197
1198 return err;
1199}
1200
33ad9711
SM		 1201struct mlx5e_modify_sq_param {
1202 int curr_state;
1203 int next_state;
1204 bool rl_update;
1205 int rl_index;
1206};
1207
a43b25da 1208static int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
33ad9711 1209 struct mlx5e_modify_sq_param *p)
f62b8bb8 1210{
f62b8bb8
AV		 1211 void *in;
1212 void *sqc;
1213 int inlen;
1214 int err;
1215
1216 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 1217 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1218 if (!in)
1219 return -ENOMEM;
1220
1221 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1222
33ad9711
SM		 1223 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1224 MLX5_SET(sqc, sqc, state, p->next_state);
1225 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
507f0c81 1226 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
33ad9711 1227 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
507f0c81 1228 }
f62b8bb8 1229
33ad9711 1230 err = mlx5_core_modify_sq(mdev, sqn, in, inlen);
f62b8bb8
AV		 1231
1232 kvfree(in);
1233
1234 return err;
1235}
1236
a43b25da 1237static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
33ad9711 1238{
a43b25da 1239 mlx5_core_destroy_sq(mdev, sqn);
f62b8bb8
AV		 1240}
1241
a43b25da 1242static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
31391048
SM		 1243 struct mlx5e_sq_param *param,
1244 struct mlx5e_create_sq_param *csp,
1245 u32 *sqn)
f62b8bb8 1246{
33ad9711 1247 struct mlx5e_modify_sq_param msp = {0};
31391048
SM		 1248 int err;
1249
a43b25da 1250 err = mlx5e_create_sq(mdev, param, csp, sqn);
31391048
SM		 1251 if (err)
1252 return err;
1253
1254 msp.curr_state = MLX5_SQC_STATE_RST;
1255 msp.next_state = MLX5_SQC_STATE_RDY;
a43b25da 1256 err = mlx5e_modify_sq(mdev, *sqn, &msp);
31391048 1257 if (err)
a43b25da 1258 mlx5e_destroy_sq(mdev, *sqn);
31391048
SM		 1259
1260 return err;
1261}
1262
7f859ecf
SM		 1263static int mlx5e_set_sq_maxrate(struct net_device *dev,
1264 struct mlx5e_txqsq *sq, u32 rate);
1265
31391048 1266static int mlx5e_open_txqsq(struct mlx5e_channel *c,
a43b25da 1267 u32 tisn,
acc6c595 1268 int txq_ix,
6a9764ef 1269 struct mlx5e_params *params,
31391048
SM		 1270 struct mlx5e_sq_param *param,
1271 struct mlx5e_txqsq *sq)
1272{
1273 struct mlx5e_create_sq_param csp = {};
7f859ecf 1274 u32 tx_rate;
f62b8bb8
AV		 1275 int err;
1276
6a9764ef 1277 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq);
f62b8bb8
AV		 1278 if (err)
1279 return err;
1280
a43b25da 1281 csp.tisn = tisn;
31391048 1282 csp.tis_lst_sz = 1;
33ad9711
SM		 1283 csp.cqn = sq->cq.mcq.cqn;
1284 csp.wq_ctrl = &sq->wq_ctrl;
1285 csp.min_inline_mode = sq->min_inline_mode;
a43b25da 1286 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
f62b8bb8 1287 if (err)
31391048 1288 goto err_free_txqsq;
f62b8bb8 1289
a43b25da 1290 tx_rate = c->priv->tx_rates[sq->txq_ix];
7f859ecf 1291 if (tx_rate)
a43b25da 1292 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
7f859ecf 1293
f62b8bb8
AV		 1294 return 0;
1295
31391048 1296err_free_txqsq:
3b77235b 1297 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
31391048 1298 mlx5e_free_txqsq(sq);
f62b8bb8
AV		 1299
1300 return err;
1301}
1302
acc6c595
SM		 1303static void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1304{
a43b25da 1305 sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix);
acc6c595
SM		 1306 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1307 netdev_tx_reset_queue(sq->txq);
1308 netif_tx_start_queue(sq->txq);
1309}
1310
f62b8bb8
AV		 1311static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1312{
1313 __netif_tx_lock_bh(txq);
1314 netif_tx_stop_queue(txq);
1315 __netif_tx_unlock_bh(txq);
1316}
1317
acc6c595 1318static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1319{
33ad9711 1320 struct mlx5e_channel *c = sq->channel;
33ad9711 1321
c0f1147d 1322 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
6e8dd6d6 1323 /* prevent netif_tx_wake_queue */
33ad9711 1324 napi_synchronize(&c->napi);
29429f33 1325
31391048 1326 netif_tx_disable_queue(sq->txq);
f62b8bb8 1327
31391048
SM		 1328 /* last doorbell out, godspeed .. */
1329 if (mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1)) {
1330 struct mlx5e_tx_wqe *nop;
864b2d71 1331
77bdf895 1332 sq->db.wqe_info[(sq->pc & sq->wq.sz_m1)].skb = NULL;
31391048
SM		 1333 nop = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
1334 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nop->ctrl);
29429f33 1335 }
acc6c595
SM		 1336}
1337
1338static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1339{
1340 struct mlx5e_channel *c = sq->channel;
a43b25da 1341 struct mlx5_core_dev *mdev = c->mdev;
f62b8bb8 1342
a43b25da 1343 mlx5e_destroy_sq(mdev, sq->sqn);
33ad9711
SM		 1344 if (sq->rate_limit)
1345 mlx5_rl_remove_rate(mdev, sq->rate_limit);
31391048
SM		 1346 mlx5e_free_txqsq_descs(sq);
1347 mlx5e_free_txqsq(sq);
1348}
1349
1350static int mlx5e_open_icosq(struct mlx5e_channel *c,
6a9764ef 1351 struct mlx5e_params *params,
31391048
SM		 1352 struct mlx5e_sq_param *param,
1353 struct mlx5e_icosq *sq)
1354{
1355 struct mlx5e_create_sq_param csp = {};
1356 int err;
1357
6a9764ef 1358 err = mlx5e_alloc_icosq(c, param, sq);
31391048
SM		 1359 if (err)
1360 return err;
1361
1362 csp.cqn = sq->cq.mcq.cqn;
1363 csp.wq_ctrl = &sq->wq_ctrl;
6a9764ef 1364 csp.min_inline_mode = params->tx_min_inline_mode;
31391048 1365 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
a43b25da 1366 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
31391048
SM		 1367 if (err)
1368 goto err_free_icosq;
1369
1370 return 0;
1371
1372err_free_icosq:
1373 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1374 mlx5e_free_icosq(sq);
1375
1376 return err;
1377}
1378
1379static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1380{
1381 struct mlx5e_channel *c = sq->channel;
1382
1383 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1384 napi_synchronize(&c->napi);
1385
a43b25da 1386 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048
SM		 1387 mlx5e_free_icosq(sq);
1388}
1389
1390static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
6a9764ef 1391 struct mlx5e_params *params,
31391048
SM		 1392 struct mlx5e_sq_param *param,
1393 struct mlx5e_xdpsq *sq)
1394{
1395 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1396 struct mlx5e_create_sq_param csp = {};
31391048
SM		 1397 unsigned int inline_hdr_sz = 0;
1398 int err;
1399 int i;
1400
6a9764ef 1401 err = mlx5e_alloc_xdpsq(c, params, param, sq);
31391048
SM		 1402 if (err)
1403 return err;
1404
1405 csp.tis_lst_sz = 1;
a43b25da 1406 csp.tisn = c->priv->tisn[0]; /* tc = 0 */
31391048
SM		 1407 csp.cqn = sq->cq.mcq.cqn;
1408 csp.wq_ctrl = &sq->wq_ctrl;
1409 csp.min_inline_mode = sq->min_inline_mode;
1410 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
a43b25da 1411 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
31391048
SM		 1412 if (err)
1413 goto err_free_xdpsq;
1414
1415 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1416 inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1417 ds_cnt++;
1418 }
1419
1420 /* Pre initialize fixed WQE fields */
1421 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1422 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1423 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1424 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
1425 struct mlx5_wqe_data_seg *dseg;
1426
1427 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1428 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1429
1430 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1431 dseg->lkey = sq->mkey_be;
1432 }
1433
1434 return 0;
1435
1436err_free_xdpsq:
1437 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1438 mlx5e_free_xdpsq(sq);
1439
1440 return err;
1441}
1442
1443static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1444{
1445 struct mlx5e_channel *c = sq->channel;
1446
1447 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1448 napi_synchronize(&c->napi);
1449
a43b25da 1450 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048
SM		 1451 mlx5e_free_xdpsq_descs(sq);
1452 mlx5e_free_xdpsq(sq);
f62b8bb8
AV		 1453}
1454
95b6c6a5
EBE		 1455static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
1456 struct mlx5e_cq_param *param,
1457 struct mlx5e_cq *cq)
f62b8bb8 1458{
f62b8bb8
AV		 1459 struct mlx5_core_cq *mcq = &cq->mcq;
1460 int eqn_not_used;
0b6e26ce 1461 unsigned int irqn;
f62b8bb8
AV		 1462 int err;
1463 u32 i;
1464
f62b8bb8
AV		 1465 err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1466 &cq->wq_ctrl);
1467 if (err)
1468 return err;
1469
1470 mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1471
f62b8bb8
AV		 1472 mcq->cqe_sz = 64;
1473 mcq->set_ci_db = cq->wq_ctrl.db.db;
1474 mcq->arm_db = cq->wq_ctrl.db.db + 1;
1475 *mcq->set_ci_db = 0;
1476 *mcq->arm_db = 0;
1477 mcq->vector = param->eq_ix;
1478 mcq->comp = mlx5e_completion_event;
1479 mcq->event = mlx5e_cq_error_event;
1480 mcq->irqn = irqn;
f62b8bb8
AV		 1481
1482 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1483 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1484
1485 cqe->op_own = 0xf1;
1486 }
1487
a43b25da 1488 cq->mdev = mdev;
f62b8bb8
AV		 1489
1490 return 0;
1491}
1492
95b6c6a5
EBE		 1493static int mlx5e_alloc_cq(struct mlx5e_channel *c,
1494 struct mlx5e_cq_param *param,
1495 struct mlx5e_cq *cq)
1496{
1497 struct mlx5_core_dev *mdev = c->priv->mdev;
1498 int err;
1499
1500 param->wq.buf_numa_node = cpu_to_node(c->cpu);
1501 param->wq.db_numa_node = cpu_to_node(c->cpu);
1502 param->eq_ix = c->ix;
1503
1504 err = mlx5e_alloc_cq_common(mdev, param, cq);
1505
1506 cq->napi = &c->napi;
1507 cq->channel = c;
1508
1509 return err;
1510}
1511
3b77235b 1512static void mlx5e_free_cq(struct mlx5e_cq *cq)
f62b8bb8 1513{
1c1b5228 1514 mlx5_cqwq_destroy(&cq->wq_ctrl);
f62b8bb8
AV		 1515}
1516
3b77235b 1517static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
f62b8bb8 1518{
a43b25da 1519 struct mlx5_core_dev *mdev = cq->mdev;
f62b8bb8
AV		 1520 struct mlx5_core_cq *mcq = &cq->mcq;
1521
1522 void *in;
1523 void *cqc;
1524 int inlen;
0b6e26ce 1525 unsigned int irqn_not_used;
f62b8bb8
AV		 1526 int eqn;
1527 int err;
1528
1529 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1c1b5228 1530 sizeof(u64) * cq->wq_ctrl.frag_buf.npages;
1b9a07ee 1531 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1532 if (!in)
1533 return -ENOMEM;
1534
1535 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1536
1537 memcpy(cqc, param->cqc, sizeof(param->cqc));
1538
1c1b5228
TT		 1539 mlx5_fill_page_frag_array(&cq->wq_ctrl.frag_buf,
1540 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
f62b8bb8
AV		 1541
1542 mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1543
9908aa29 1544 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode);
f62b8bb8 1545 MLX5_SET(cqc, cqc, c_eqn, eqn);
30aa60b3 1546 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
1c1b5228 1547 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift -
68cdf5d6 1548 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 1549 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1550
1551 err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1552
1553 kvfree(in);
1554
1555 if (err)
1556 return err;
1557
1558 mlx5e_cq_arm(cq);
1559
1560 return 0;
1561}
1562
3b77235b 1563static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
f62b8bb8 1564{
a43b25da 1565 mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
f62b8bb8
AV		 1566}
1567
1568static int mlx5e_open_cq(struct mlx5e_channel *c,
6a9764ef 1569 struct mlx5e_cq_moder moder,
f62b8bb8 1570 struct mlx5e_cq_param *param,
6a9764ef 1571 struct mlx5e_cq *cq)
f62b8bb8 1572{
a43b25da 1573 struct mlx5_core_dev *mdev = c->mdev;
f62b8bb8 1574 int err;
f62b8bb8 1575
3b77235b 1576 err = mlx5e_alloc_cq(c, param, cq);
f62b8bb8
AV		 1577 if (err)
1578 return err;
1579
3b77235b 1580 err = mlx5e_create_cq(cq, param);
f62b8bb8 1581 if (err)
3b77235b 1582 goto err_free_cq;
f62b8bb8 1583
7524a5d8 1584 if (MLX5_CAP_GEN(mdev, cq_moderation))
6a9764ef 1585 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
f62b8bb8
AV		 1586 return 0;
1587
3b77235b
SM		 1588err_free_cq:
1589 mlx5e_free_cq(cq);
f62b8bb8
AV		 1590
1591 return err;
1592}
1593
1594static void mlx5e_close_cq(struct mlx5e_cq *cq)
1595{
f62b8bb8 1596 mlx5e_destroy_cq(cq);
3b77235b 1597 mlx5e_free_cq(cq);
f62b8bb8
AV		 1598}
1599
1600static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1601{
1602 return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1603}
1604
1605static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
6a9764ef 1606 struct mlx5e_params *params,
f62b8bb8
AV		 1607 struct mlx5e_channel_param *cparam)
1608{
f62b8bb8
AV		 1609 int err;
1610 int tc;
1611
1612 for (tc = 0; tc < c->num_tc; tc++) {
6a9764ef
SM		 1613 err = mlx5e_open_cq(c, params->tx_cq_moderation,
1614 &cparam->tx_cq, &c->sq[tc].cq);
f62b8bb8
AV		 1615 if (err)
1616 goto err_close_tx_cqs;
f62b8bb8
AV		 1617 }
1618
1619 return 0;
1620
1621err_close_tx_cqs:
1622 for (tc--; tc >= 0; tc--)
1623 mlx5e_close_cq(&c->sq[tc].cq);
1624
1625 return err;
1626}
1627
1628static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1629{
1630 int tc;
1631
1632 for (tc = 0; tc < c->num_tc; tc++)
1633 mlx5e_close_cq(&c->sq[tc].cq);
1634}
1635
1636static int mlx5e_open_sqs(struct mlx5e_channel *c,
6a9764ef 1637 struct mlx5e_params *params,
f62b8bb8
AV		 1638 struct mlx5e_channel_param *cparam)
1639{
1640 int err;
1641 int tc;
1642
6a9764ef
SM		 1643 for (tc = 0; tc < params->num_tc; tc++) {
1644 int txq_ix = c->ix + tc * params->num_channels;
acc6c595 1645
a43b25da
SM		 1646 err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,
1647 params, &cparam->sq, &c->sq[tc]);
f62b8bb8
AV		 1648 if (err)
1649 goto err_close_sqs;
1650 }
1651
1652 return 0;
1653
1654err_close_sqs:
1655 for (tc--; tc >= 0; tc--)
31391048 1656 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 1657
1658 return err;
1659}
1660
1661static void mlx5e_close_sqs(struct mlx5e_channel *c)
1662{
1663 int tc;
1664
1665 for (tc = 0; tc < c->num_tc; tc++)
31391048 1666 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 1667}
1668
507f0c81 1669static int mlx5e_set_sq_maxrate(struct net_device *dev,
31391048 1670 struct mlx5e_txqsq *sq, u32 rate)
507f0c81
YP		 1671{
1672 struct mlx5e_priv *priv = netdev_priv(dev);
1673 struct mlx5_core_dev *mdev = priv->mdev;
33ad9711 1674 struct mlx5e_modify_sq_param msp = {0};
507f0c81
YP		 1675 u16 rl_index = 0;
1676 int err;
1677
1678 if (rate == sq->rate_limit)
1679 /* nothing to do */
1680 return 0;
1681
1682 if (sq->rate_limit)
1683 /* remove current rl index to free space to next ones */
1684 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1685
1686 sq->rate_limit = 0;
1687
1688 if (rate) {
1689 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1690 if (err) {
1691 netdev_err(dev, "Failed configuring rate %u: %d\n",
1692 rate, err);
1693 return err;
1694 }
1695 }
1696
33ad9711
SM		 1697 msp.curr_state = MLX5_SQC_STATE_RDY;
1698 msp.next_state = MLX5_SQC_STATE_RDY;
1699 msp.rl_index = rl_index;
1700 msp.rl_update = true;
a43b25da 1701 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
507f0c81
YP		 1702 if (err) {
1703 netdev_err(dev, "Failed configuring rate %u: %d\n",
1704 rate, err);
1705 /* remove the rate from the table */
1706 if (rate)
1707 mlx5_rl_remove_rate(mdev, rate);
1708 return err;
1709 }
1710
1711 sq->rate_limit = rate;
1712 return 0;
1713}
1714
1715static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1716{
1717 struct mlx5e_priv *priv = netdev_priv(dev);
1718 struct mlx5_core_dev *mdev = priv->mdev;
acc6c595 1719 struct mlx5e_txqsq *sq = priv->txq2sq[index];
507f0c81
YP		 1720 int err = 0;
1721
1722 if (!mlx5_rl_is_supported(mdev)) {
1723 netdev_err(dev, "Rate limiting is not supported on this device\n");
1724 return -EINVAL;
1725 }
1726
1727 /* rate is given in Mb/sec, HW config is in Kb/sec */
1728 rate = rate << 10;
1729
1730 /* Check whether rate in valid range, 0 is always valid */
1731 if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1732 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1733 return -ERANGE;
1734 }
1735
1736 mutex_lock(&priv->state_lock);
1737 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1738 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1739 if (!err)
1740 priv->tx_rates[index] = rate;
1741 mutex_unlock(&priv->state_lock);
1742
1743 return err;
1744}
1745
f62b8bb8 1746static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
6a9764ef 1747 struct mlx5e_params *params,
f62b8bb8
AV		 1748 struct mlx5e_channel_param *cparam,
1749 struct mlx5e_channel **cp)
1750{
6a9764ef 1751 struct mlx5e_cq_moder icocq_moder = {0, 0};
f62b8bb8
AV		 1752 struct net_device *netdev = priv->netdev;
1753 int cpu = mlx5e_get_cpu(priv, ix);
1754 struct mlx5e_channel *c;
1755 int err;
1756
1757 c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1758 if (!c)
1759 return -ENOMEM;
1760
1761 c->priv = priv;
a43b25da
SM		 1762 c->mdev = priv->mdev;
1763 c->tstamp = &priv->tstamp;
f62b8bb8
AV		 1764 c->ix = ix;
1765 c->cpu = cpu;
1766 c->pdev = &priv->mdev->pdev->dev;
1767 c->netdev = priv->netdev;
b50d292b 1768 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
6a9764ef
SM		 1769 c->num_tc = params->num_tc;
1770 c->xdp = !!params->xdp_prog;
cb3c7fd4 1771
f62b8bb8
AV		 1772 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1773
6a9764ef 1774 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
f62b8bb8
AV		 1775 if (err)
1776 goto err_napi_del;
1777
6a9764ef 1778 err = mlx5e_open_tx_cqs(c, params, cparam);
d3c9bc27
TT		 1779 if (err)
1780 goto err_close_icosq_cq;
1781
6a9764ef 1782 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
f62b8bb8
AV		 1783 if (err)
1784 goto err_close_tx_cqs;
f62b8bb8 1785
d7a0ecab 1786 /* XDP SQ CQ params are same as normal TXQ sq CQ params */
6a9764ef
SM		 1787 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1788 &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
d7a0ecab
SM		 1789 if (err)
1790 goto err_close_rx_cq;
1791
f62b8bb8
AV		 1792 napi_enable(&c->napi);
1793
6a9764ef 1794 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
f62b8bb8
AV		 1795 if (err)
1796 goto err_disable_napi;
1797
6a9764ef 1798 err = mlx5e_open_sqs(c, params, cparam);
d3c9bc27
TT		 1799 if (err)
1800 goto err_close_icosq;
1801
6a9764ef 1802 err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq) : 0;
d7a0ecab
SM		 1803 if (err)
1804 goto err_close_sqs;
b5503b99 1805
6a9764ef 1806 err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);
f62b8bb8 1807 if (err)
b5503b99 1808 goto err_close_xdp_sq;
f62b8bb8 1809
f62b8bb8
AV		 1810 *cp = c;
1811
1812 return 0;
b5503b99 1813err_close_xdp_sq:
d7a0ecab 1814 if (c->xdp)
31391048 1815 mlx5e_close_xdpsq(&c->rq.xdpsq);
f62b8bb8
AV		 1816
1817err_close_sqs:
1818 mlx5e_close_sqs(c);
1819
d3c9bc27 1820err_close_icosq:
31391048 1821 mlx5e_close_icosq(&c->icosq);
d3c9bc27 1822
f62b8bb8
AV		 1823err_disable_napi:
1824 napi_disable(&c->napi);
d7a0ecab 1825 if (c->xdp)
31871f87 1826 mlx5e_close_cq(&c->rq.xdpsq.cq);
d7a0ecab
SM		 1827
1828err_close_rx_cq:
f62b8bb8
AV		 1829 mlx5e_close_cq(&c->rq.cq);
1830
1831err_close_tx_cqs:
1832 mlx5e_close_tx_cqs(c);
1833
d3c9bc27
TT		 1834err_close_icosq_cq:
1835 mlx5e_close_cq(&c->icosq.cq);
1836
f62b8bb8
AV		 1837err_napi_del:
1838 netif_napi_del(&c->napi);
1839 kfree(c);
1840
1841 return err;
1842}
1843
acc6c595
SM		 1844static void mlx5e_activate_channel(struct mlx5e_channel *c)
1845{
1846 int tc;
1847
1848 for (tc = 0; tc < c->num_tc; tc++)
1849 mlx5e_activate_txqsq(&c->sq[tc]);
1850 mlx5e_activate_rq(&c->rq);
a43b25da 1851 netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
acc6c595
SM		 1852}
1853
1854static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
1855{
1856 int tc;
1857
1858 mlx5e_deactivate_rq(&c->rq);
1859 for (tc = 0; tc < c->num_tc; tc++)
1860 mlx5e_deactivate_txqsq(&c->sq[tc]);
1861}
1862
f62b8bb8
AV		 1863static void mlx5e_close_channel(struct mlx5e_channel *c)
1864{
1865 mlx5e_close_rq(&c->rq);
b5503b99 1866 if (c->xdp)
31391048 1867 mlx5e_close_xdpsq(&c->rq.xdpsq);
f62b8bb8 1868 mlx5e_close_sqs(c);
31391048 1869 mlx5e_close_icosq(&c->icosq);
f62b8bb8 1870 napi_disable(&c->napi);
b5503b99 1871 if (c->xdp)
31871f87 1872 mlx5e_close_cq(&c->rq.xdpsq.cq);
f62b8bb8
AV		 1873 mlx5e_close_cq(&c->rq.cq);
1874 mlx5e_close_tx_cqs(c);
d3c9bc27 1875 mlx5e_close_cq(&c->icosq.cq);
f62b8bb8 1876 netif_napi_del(&c->napi);
7ae92ae5 1877
f62b8bb8
AV		 1878 kfree(c);
1879}
1880
1881static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
6a9764ef 1882 struct mlx5e_params *params,
f62b8bb8
AV		 1883 struct mlx5e_rq_param *param)
1884{
1885 void *rqc = param->rqc;
1886 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1887
6a9764ef 1888 switch (params->rq_wq_type) {
461017cb 1889 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
6a9764ef
SM		 1890 MLX5_SET(wq, wq, log_wqe_num_of_strides, params->mpwqe_log_num_strides - 9);
1891 MLX5_SET(wq, wq, log_wqe_stride_size, params->mpwqe_log_stride_sz - 6);
461017cb
TT		 1892 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1893 break;
1894 default: /* MLX5_WQ_TYPE_LINKED_LIST */
1895 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1896 }
1897
f62b8bb8
AV		 1898 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1899 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
6a9764ef 1900 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_size);
b50d292b 1901 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
593cf338 1902 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
6a9764ef 1903 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
102722fc 1904 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en);
f62b8bb8 1905
311c7c71 1906 param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
f62b8bb8
AV		 1907 param->wq.linear = 1;
1908}
1909
556dd1b9
TT		 1910static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1911{
1912 void *rqc = param->rqc;
1913 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1914
1915 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1916 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1917}
1918
d3c9bc27
TT		 1919static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1920 struct mlx5e_sq_param *param)
f62b8bb8
AV		 1921{
1922 void *sqc = param->sqc;
1923 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1924
f62b8bb8 1925 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
b50d292b 1926 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn);
f62b8bb8 1927
311c7c71 1928 param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
d3c9bc27
TT		 1929}
1930
1931static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
6a9764ef 1932 struct mlx5e_params *params,
d3c9bc27
TT		 1933 struct mlx5e_sq_param *param)
1934{
1935 void *sqc = param->sqc;
1936 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1937
1938 mlx5e_build_sq_param_common(priv, param);
6a9764ef 1939 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2ac9cfe7 1940 MLX5_SET(sqc, sqc, allow_swp, !!MLX5_IPSEC_DEV(priv->mdev));
f62b8bb8
AV		 1941}
1942
1943static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1944 struct mlx5e_cq_param *param)
1945{
1946 void *cqc = param->cqc;
1947
30aa60b3 1948 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
f62b8bb8
AV		 1949}
1950
1951static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
6a9764ef 1952 struct mlx5e_params *params,
f62b8bb8
AV		 1953 struct mlx5e_cq_param *param)
1954{
1955 void *cqc = param->cqc;
461017cb 1956 u8 log_cq_size;
f62b8bb8 1957
6a9764ef 1958 switch (params->rq_wq_type) {
461017cb 1959 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
6a9764ef 1960 log_cq_size = params->log_rq_size + params->mpwqe_log_num_strides;
461017cb
TT		 1961 break;
1962 default: /* MLX5_WQ_TYPE_LINKED_LIST */
6a9764ef 1963 log_cq_size = params->log_rq_size;
461017cb
TT		 1964 }
1965
1966 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
6a9764ef 1967 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
7219ab34
TT		 1968 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1969 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1970 }
f62b8bb8
AV		 1971
1972 mlx5e_build_common_cq_param(priv, param);
1973}
1974
1975static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
6a9764ef 1976 struct mlx5e_params *params,
f62b8bb8
AV		 1977 struct mlx5e_cq_param *param)
1978{
1979 void *cqc = param->cqc;
1980
6a9764ef 1981 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
f62b8bb8
AV		 1982
1983 mlx5e_build_common_cq_param(priv, param);
9908aa29
TT		 1984
1985 param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
f62b8bb8
AV		 1986}
1987
d3c9bc27 1988static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
6a9764ef
SM		 1989 u8 log_wq_size,
1990 struct mlx5e_cq_param *param)
d3c9bc27
TT		 1991{
1992 void *cqc = param->cqc;
1993
1994 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1995
1996 mlx5e_build_common_cq_param(priv, param);
9908aa29
TT		 1997
1998 param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
d3c9bc27
TT		 1999}
2000
2001static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
6a9764ef
SM		 2002 u8 log_wq_size,
2003 struct mlx5e_sq_param *param)
d3c9bc27
TT		 2004{
2005 void *sqc = param->sqc;
2006 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2007
2008 mlx5e_build_sq_param_common(priv, param);
2009
2010 MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
bc77b240 2011 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
d3c9bc27
TT		 2012}
2013
b5503b99 2014static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
6a9764ef 2015 struct mlx5e_params *params,
b5503b99
SM		 2016 struct mlx5e_sq_param *param)
2017{
2018 void *sqc = param->sqc;
2019 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2020
2021 mlx5e_build_sq_param_common(priv, param);
6a9764ef 2022 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
b5503b99
SM		 2023}
2024
6a9764ef
SM		 2025static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
2026 struct mlx5e_params *params,
2027 struct mlx5e_channel_param *cparam)
f62b8bb8 2028{
bc77b240 2029 u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
d3c9bc27 2030
6a9764ef
SM		 2031 mlx5e_build_rq_param(priv, params, &cparam->rq);
2032 mlx5e_build_sq_param(priv, params, &cparam->sq);
2033 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2034 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
2035 mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq);
2036 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
2037 mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);
f62b8bb8
AV		 2038}
2039
55c2503d
SM		 2040int mlx5e_open_channels(struct mlx5e_priv *priv,
2041 struct mlx5e_channels *chs)
f62b8bb8 2042{
6b87663f 2043 struct mlx5e_channel_param *cparam;
03289b88 2044 int err = -ENOMEM;
f62b8bb8 2045 int i;
f62b8bb8 2046
6a9764ef 2047 chs->num = chs->params.num_channels;
03289b88 2048
ff9c852f 2049 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
6b87663f 2050 cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
acc6c595
SM		 2051 if (!chs->c || !cparam)
2052 goto err_free;
f62b8bb8 2053
6a9764ef 2054 mlx5e_build_channel_param(priv, &chs->params, cparam);
ff9c852f 2055 for (i = 0; i < chs->num; i++) {
6a9764ef 2056 err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]);
f62b8bb8
AV		 2057 if (err)
2058 goto err_close_channels;
2059 }
2060
6b87663f 2061 kfree(cparam);
f62b8bb8
AV		 2062 return 0;
2063
2064err_close_channels:
2065 for (i--; i >= 0; i--)
ff9c852f 2066 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2067
acc6c595 2068err_free:
ff9c852f 2069 kfree(chs->c);
6b87663f 2070 kfree(cparam);
ff9c852f 2071 chs->num = 0;
f62b8bb8
AV		 2072 return err;
2073}
2074
acc6c595 2075static void mlx5e_activate_channels(struct mlx5e_channels *chs)
f62b8bb8
AV		 2076{
2077 int i;
2078
acc6c595
SM		 2079 for (i = 0; i < chs->num; i++)
2080 mlx5e_activate_channel(chs->c[i]);
2081}
2082
2083static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2084{
2085 int err = 0;
2086 int i;
2087
2088 for (i = 0; i < chs->num; i++) {
2089 err = mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq);
2090 if (err)
2091 break;
2092 }
2093
2094 return err;
2095}
2096
2097static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2098{
2099 int i;
2100
2101 for (i = 0; i < chs->num; i++)
2102 mlx5e_deactivate_channel(chs->c[i]);
2103}
2104
55c2503d 2105void mlx5e_close_channels(struct mlx5e_channels *chs)
acc6c595
SM		 2106{
2107 int i;
c3b7c5c9 2108
ff9c852f
SM		 2109 for (i = 0; i < chs->num; i++)
2110 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2111
ff9c852f
SM		 2112 kfree(chs->c);
2113 chs->num = 0;
f62b8bb8
AV		 2114}
2115
a5f97fee
SM		 2116static int
2117mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
f62b8bb8
AV		 2118{
2119 struct mlx5_core_dev *mdev = priv->mdev;
f62b8bb8
AV		 2120 void *rqtc;
2121 int inlen;
2122 int err;
1da36696 2123 u32 *in;
a5f97fee 2124 int i;
f62b8bb8 2125
f62b8bb8 2126 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 2127 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 2128 if (!in)
2129 return -ENOMEM;
2130
2131 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2132
2133 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2134 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2135
a5f97fee
SM		 2136 for (i = 0; i < sz; i++)
2137 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2be6967c 2138
398f3351
HHZ		 2139 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
2140 if (!err)
2141 rqt->enabled = true;
f62b8bb8
AV		 2142
2143 kvfree(in);
1da36696
TT		 2144 return err;
2145}
2146
cb67b832 2147void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1da36696 2148{
398f3351
HHZ		 2149 rqt->enabled = false;
2150 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1da36696
TT		 2151}
2152
8f493ffd 2153int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2154{
2155 struct mlx5e_rqt *rqt = &priv->indir_rqt;
8f493ffd 2156 int err;
6bfd390b 2157
8f493ffd
SM		 2158 err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
2159 if (err)
2160 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err);
2161 return err;
6bfd390b
HHZ		 2162}
2163
cb67b832 2164int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1da36696 2165{
398f3351 2166 struct mlx5e_rqt *rqt;
1da36696
TT		 2167 int err;
2168 int ix;
2169
6bfd390b 2170 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
398f3351 2171 rqt = &priv->direct_tir[ix].rqt;
a5f97fee 2172 err = mlx5e_create_rqt(priv, 1 /*size */, rqt);
1da36696
TT		 2173 if (err)
2174 goto err_destroy_rqts;
2175 }
2176
2177 return 0;
2178
2179err_destroy_rqts:
8f493ffd 2180 mlx5_core_warn(priv->mdev, "create direct rqts failed, %d\n", err);
1da36696 2181 for (ix--; ix >= 0; ix--)
398f3351 2182 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1da36696 2183
f62b8bb8
AV		 2184 return err;
2185}
2186
8f493ffd
SM		 2187void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv)
2188{
2189 int i;
2190
2191 for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
2192 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
2193}
2194
a5f97fee
SM		 2195static int mlx5e_rx_hash_fn(int hfunc)
2196{
2197 return (hfunc == ETH_RSS_HASH_TOP) ?
2198 MLX5_RX_HASH_FN_TOEPLITZ :
2199 MLX5_RX_HASH_FN_INVERTED_XOR8;
2200}
2201
2202static int mlx5e_bits_invert(unsigned long a, int size)
2203{
2204 int inv = 0;
2205 int i;
2206
2207 for (i = 0; i < size; i++)
2208 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
2209
2210 return inv;
2211}
2212
2213static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
2214 struct mlx5e_redirect_rqt_param rrp, void *rqtc)
2215{
2216 int i;
2217
2218 for (i = 0; i < sz; i++) {
2219 u32 rqn;
2220
2221 if (rrp.is_rss) {
2222 int ix = i;
2223
2224 if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
2225 ix = mlx5e_bits_invert(i, ilog2(sz));
2226
6a9764ef 2227 ix = priv->channels.params.indirection_rqt[ix];
a5f97fee
SM		 2228 rqn = rrp.rss.channels->c[ix]->rq.rqn;
2229 } else {
2230 rqn = rrp.rqn;
2231 }
2232 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
2233 }
2234}
2235
2236int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
2237 struct mlx5e_redirect_rqt_param rrp)
5c50368f
AS		 2238{
2239 struct mlx5_core_dev *mdev = priv->mdev;
5c50368f
AS		 2240 void *rqtc;
2241 int inlen;
1da36696 2242 u32 *in;
5c50368f
AS		 2243 int err;
2244
5c50368f 2245 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1b9a07ee 2246 in = kvzalloc(inlen, GFP_KERNEL);
5c50368f
AS		 2247 if (!in)
2248 return -ENOMEM;
2249
2250 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2251
2252 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5c50368f 2253 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
a5f97fee 2254 mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
1da36696 2255 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
5c50368f
AS		 2256
2257 kvfree(in);
5c50368f
AS		 2258 return err;
2259}
2260
a5f97fee
SM		 2261static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
2262 struct mlx5e_redirect_rqt_param rrp)
2263{
2264 if (!rrp.is_rss)
2265 return rrp.rqn;
2266
2267 if (ix >= rrp.rss.channels->num)
2268 return priv->drop_rq.rqn;
2269
2270 return rrp.rss.channels->c[ix]->rq.rqn;
2271}
2272
2273static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
2274 struct mlx5e_redirect_rqt_param rrp)
40ab6a6e 2275{
1da36696
TT		 2276 u32 rqtn;
2277 int ix;
2278
398f3351 2279 if (priv->indir_rqt.enabled) {
a5f97fee 2280 /* RSS RQ table */
398f3351 2281 rqtn = priv->indir_rqt.rqtn;
a5f97fee 2282 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
398f3351
HHZ		 2283 }
2284
a5f97fee
SM		 2285 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2286 struct mlx5e_redirect_rqt_param direct_rrp = {
2287 .is_rss = false,
95632791
AM		 2288 {
2289 .rqn = mlx5e_get_direct_rqn(priv, ix, rrp)
2290 },
a5f97fee
SM		 2291 };
2292
2293 /* Direct RQ Tables */
398f3351
HHZ		 2294 if (!priv->direct_tir[ix].rqt.enabled)
2295 continue;
a5f97fee 2296
398f3351 2297 rqtn = priv->direct_tir[ix].rqt.rqtn;
a5f97fee 2298 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
1da36696 2299 }
40ab6a6e
AS		 2300}
2301
a5f97fee
SM		 2302static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
2303 struct mlx5e_channels *chs)
2304{
2305 struct mlx5e_redirect_rqt_param rrp = {
2306 .is_rss = true,
95632791
AM		 2307 {
2308 .rss = {
2309 .channels = chs,
2310 .hfunc = chs->params.rss_hfunc,
2311 }
2312 },
a5f97fee
SM		 2313 };
2314
2315 mlx5e_redirect_rqts(priv, rrp);
2316}
2317
2318static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
2319{
2320 struct mlx5e_redirect_rqt_param drop_rrp = {
2321 .is_rss = false,
95632791
AM		 2322 {
2323 .rqn = priv->drop_rq.rqn,
2324 },
a5f97fee
SM		 2325 };
2326
2327 mlx5e_redirect_rqts(priv, drop_rrp);
2328}
2329
6a9764ef 2330static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
5c50368f 2331{
6a9764ef 2332 if (!params->lro_en)
5c50368f
AS		 2333 return;
2334
2335#define ROUGH_MAX_L2_L3_HDR_SZ 256
2336
2337 MLX5_SET(tirc, tirc, lro_enable_mask,
2338 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2339 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2340 MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
6a9764ef
SM		 2341 (params->lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2342 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
5c50368f
AS		 2343}
2344
6a9764ef
SM		 2345void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
2346 enum mlx5e_traffic_types tt,
2347 void *tirc)
bdfc028d 2348{
a100ff3e
GP		 2349 void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2350
2351#define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
2352 MLX5_HASH_FIELD_SEL_DST_IP)
2353
2354#define MLX5_HASH_IP_L4PORTS (MLX5_HASH_FIELD_SEL_SRC_IP |\
2355 MLX5_HASH_FIELD_SEL_DST_IP |\
2356 MLX5_HASH_FIELD_SEL_L4_SPORT |\
2357 MLX5_HASH_FIELD_SEL_L4_DPORT)
2358
2359#define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\
2360 MLX5_HASH_FIELD_SEL_DST_IP |\
2361 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2362
6a9764ef
SM		 2363 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(params->rss_hfunc));
2364 if (params->rss_hfunc == ETH_RSS_HASH_TOP) {
bdfc028d
TT		 2365 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2366 rx_hash_toeplitz_key);
2367 size_t len = MLX5_FLD_SZ_BYTES(tirc,
2368 rx_hash_toeplitz_key);
2369
2370 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
6a9764ef 2371 memcpy(rss_key, params->toeplitz_hash_key, len);
bdfc028d 2372 }
a100ff3e
GP		 2373
2374 switch (tt) {
2375 case MLX5E_TT_IPV4_TCP:
2376 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2377 MLX5_L3_PROT_TYPE_IPV4);
2378 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2379 MLX5_L4_PROT_TYPE_TCP);
2380 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2381 MLX5_HASH_IP_L4PORTS);
2382 break;
2383
2384 case MLX5E_TT_IPV6_TCP:
2385 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2386 MLX5_L3_PROT_TYPE_IPV6);
2387 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2388 MLX5_L4_PROT_TYPE_TCP);
2389 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2390 MLX5_HASH_IP_L4PORTS);
2391 break;
2392
2393 case MLX5E_TT_IPV4_UDP:
2394 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2395 MLX5_L3_PROT_TYPE_IPV4);
2396 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2397 MLX5_L4_PROT_TYPE_UDP);
2398 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2399 MLX5_HASH_IP_L4PORTS);
2400 break;
2401
2402 case MLX5E_TT_IPV6_UDP:
2403 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2404 MLX5_L3_PROT_TYPE_IPV6);
2405 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2406 MLX5_L4_PROT_TYPE_UDP);
2407 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2408 MLX5_HASH_IP_L4PORTS);
2409 break;
2410
2411 case MLX5E_TT_IPV4_IPSEC_AH:
2412 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2413 MLX5_L3_PROT_TYPE_IPV4);
2414 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2415 MLX5_HASH_IP_IPSEC_SPI);
2416 break;
2417
2418 case MLX5E_TT_IPV6_IPSEC_AH:
2419 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2420 MLX5_L3_PROT_TYPE_IPV6);
2421 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2422 MLX5_HASH_IP_IPSEC_SPI);
2423 break;
2424
2425 case MLX5E_TT_IPV4_IPSEC_ESP:
2426 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2427 MLX5_L3_PROT_TYPE_IPV4);
2428 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2429 MLX5_HASH_IP_IPSEC_SPI);
2430 break;
2431
2432 case MLX5E_TT_IPV6_IPSEC_ESP:
2433 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2434 MLX5_L3_PROT_TYPE_IPV6);
2435 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2436 MLX5_HASH_IP_IPSEC_SPI);
2437 break;
2438
2439 case MLX5E_TT_IPV4:
2440 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2441 MLX5_L3_PROT_TYPE_IPV4);
2442 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2443 MLX5_HASH_IP);
2444 break;
2445
2446 case MLX5E_TT_IPV6:
2447 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2448 MLX5_L3_PROT_TYPE_IPV6);
2449 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2450 MLX5_HASH_IP);
2451 break;
2452 default:
2453 WARN_ONCE(true, "%s: bad traffic type!\n", __func__);
2454 }
bdfc028d
TT		 2455}
2456
ab0394fe 2457static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
5c50368f
AS		 2458{
2459 struct mlx5_core_dev *mdev = priv->mdev;
2460
2461 void *in;
2462 void *tirc;
2463 int inlen;
2464 int err;
ab0394fe 2465 int tt;
1da36696 2466 int ix;
5c50368f
AS		 2467
2468 inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1b9a07ee 2469 in = kvzalloc(inlen, GFP_KERNEL);
5c50368f
AS		 2470 if (!in)
2471 return -ENOMEM;
2472
2473 MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2474 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2475
6a9764ef 2476 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
5c50368f 2477
1da36696 2478 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
724b2aa1 2479 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1da36696 2480 inlen);
ab0394fe 2481 if (err)
1da36696 2482 goto free_in;
ab0394fe 2483 }
5c50368f 2484
6bfd390b 2485 for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1da36696
TT		 2486 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
2487 in, inlen);
2488 if (err)
2489 goto free_in;
2490 }
2491
2492free_in:
5c50368f
AS		 2493 kvfree(in);
2494
2495 return err;
2496}
2497
cd255eff 2498static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
40ab6a6e 2499{
40ab6a6e 2500 struct mlx5_core_dev *mdev = priv->mdev;
c139dbfd 2501 u16 hw_mtu = MLX5E_SW2HW_MTU(priv, mtu);
40ab6a6e
AS		 2502 int err;
2503
cd255eff 2504 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
40ab6a6e
AS		 2505 if (err)
2506 return err;
2507
cd255eff
SM		 2508 /* Update vport context MTU */
2509 mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2510 return 0;
2511}
40ab6a6e 2512
cd255eff
SM		 2513static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
2514{
2515 struct mlx5_core_dev *mdev = priv->mdev;
2516 u16 hw_mtu = 0;
2517 int err;
40ab6a6e 2518
cd255eff
SM		 2519 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2520 if (err || !hw_mtu) /* fallback to port oper mtu */
2521 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2522
c139dbfd 2523 *mtu = MLX5E_HW2SW_MTU(priv, hw_mtu);
cd255eff
SM		 2524}
2525
2e20a151 2526static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
cd255eff 2527{
2e20a151 2528 struct net_device *netdev = priv->netdev;
cd255eff
SM		 2529 u16 mtu;
2530 int err;
2531
2532 err = mlx5e_set_mtu(priv, netdev->mtu);
2533 if (err)
2534 return err;
40ab6a6e 2535
cd255eff
SM		 2536 mlx5e_query_mtu(priv, &mtu);
2537 if (mtu != netdev->mtu)
2538 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2539 __func__, mtu, netdev->mtu);
40ab6a6e 2540
cd255eff 2541 netdev->mtu = mtu;
40ab6a6e
AS		 2542 return 0;
2543}
2544
08fb1dac
SM		 2545static void mlx5e_netdev_set_tcs(struct net_device *netdev)
2546{
2547 struct mlx5e_priv *priv = netdev_priv(netdev);
6a9764ef
SM		 2548 int nch = priv->channels.params.num_channels;
2549 int ntc = priv->channels.params.num_tc;
08fb1dac
SM		 2550 int tc;
2551
2552 netdev_reset_tc(netdev);
2553
2554 if (ntc == 1)
2555 return;
2556
2557 netdev_set_num_tc(netdev, ntc);
2558
7ccdd084
RS		 2559 /* Map netdev TCs to offset 0
2560 * We have our own UP to TXQ mapping for QoS
2561 */
08fb1dac 2562 for (tc = 0; tc < ntc; tc++)
7ccdd084 2563 netdev_set_tc_queue(netdev, tc, nch, 0);
08fb1dac
SM		 2564}
2565
acc6c595
SM		 2566static void mlx5e_build_channels_tx_maps(struct mlx5e_priv *priv)
2567{
2568 struct mlx5e_channel *c;
2569 struct mlx5e_txqsq *sq;
2570 int i, tc;
2571
2572 for (i = 0; i < priv->channels.num; i++)
2573 for (tc = 0; tc < priv->profile->max_tc; tc++)
2574 priv->channel_tc2txq[i][tc] = i + tc * priv->channels.num;
2575
2576 for (i = 0; i < priv->channels.num; i++) {
2577 c = priv->channels.c[i];
2578 for (tc = 0; tc < c->num_tc; tc++) {
2579 sq = &c->sq[tc];
2580 priv->txq2sq[sq->txq_ix] = sq;
2581 }
2582 }
2583}
2584
955bc480
SM		 2585static bool mlx5e_is_eswitch_vport_mngr(struct mlx5_core_dev *mdev)
2586{
2587 return (MLX5_CAP_GEN(mdev, vport_group_manager) &&
2588 MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH);
2589}
2590
603f4a45 2591void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
acc6c595 2592{
9008ae07
SM		 2593 int num_txqs = priv->channels.num * priv->channels.params.num_tc;
2594 struct net_device *netdev = priv->netdev;
2595
2596 mlx5e_netdev_set_tcs(netdev);
053ee0a7
TR		 2597 netif_set_real_num_tx_queues(netdev, num_txqs);
2598 netif_set_real_num_rx_queues(netdev, priv->channels.num);
9008ae07 2599
acc6c595
SM		 2600 mlx5e_build_channels_tx_maps(priv);
2601 mlx5e_activate_channels(&priv->channels);
2602 netif_tx_start_all_queues(priv->netdev);
9008ae07 2603
955bc480 2604 if (mlx5e_is_eswitch_vport_mngr(priv->mdev))
9008ae07
SM		 2605 mlx5e_add_sqs_fwd_rules(priv);
2606
acc6c595 2607 mlx5e_wait_channels_min_rx_wqes(&priv->channels);
9008ae07 2608 mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
acc6c595
SM		 2609}
2610
603f4a45 2611void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
acc6c595 2612{
9008ae07
SM		 2613 mlx5e_redirect_rqts_to_drop(priv);
2614
955bc480 2615 if (mlx5e_is_eswitch_vport_mngr(priv->mdev))
9008ae07
SM		 2616 mlx5e_remove_sqs_fwd_rules(priv);
2617
acc6c595
SM		 2618 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2619 * polling for inactive tx queues.
2620 */
2621 netif_tx_stop_all_queues(priv->netdev);
2622 netif_tx_disable(priv->netdev);
2623 mlx5e_deactivate_channels(&priv->channels);
2624}
2625
55c2503d 2626void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2e20a151
SM		 2627 struct mlx5e_channels *new_chs,
2628 mlx5e_fp_hw_modify hw_modify)
55c2503d
SM		 2629{
2630 struct net_device *netdev = priv->netdev;
2631 int new_num_txqs;
7ca42c80 2632 int carrier_ok;
55c2503d
SM		 2633 new_num_txqs = new_chs->num * new_chs->params.num_tc;
2634
7ca42c80 2635 carrier_ok = netif_carrier_ok(netdev);
55c2503d
SM		 2636 netif_carrier_off(netdev);
2637
2638 if (new_num_txqs < netdev->real_num_tx_queues)
2639 netif_set_real_num_tx_queues(netdev, new_num_txqs);
2640
2641 mlx5e_deactivate_priv_channels(priv);
2642 mlx5e_close_channels(&priv->channels);
2643
2644 priv->channels = *new_chs;
2645
2e20a151
SM		 2646 /* New channels are ready to roll, modify HW settings if needed */
2647 if (hw_modify)
2648 hw_modify(priv);
2649
55c2503d
SM		 2650 mlx5e_refresh_tirs(priv, false);
2651 mlx5e_activate_priv_channels(priv);
2652
7ca42c80
ES		 2653 /* return carrier back if needed */
2654 if (carrier_ok)
2655 netif_carrier_on(netdev);
55c2503d
SM		 2656}
2657
40ab6a6e
AS		 2658int mlx5e_open_locked(struct net_device *netdev)
2659{
2660 struct mlx5e_priv *priv = netdev_priv(netdev);
40ab6a6e
AS		 2661 int err;
2662
2663 set_bit(MLX5E_STATE_OPENED, &priv->state);
2664
ff9c852f 2665 err = mlx5e_open_channels(priv, &priv->channels);
acc6c595 2666 if (err)
343b29f3 2667 goto err_clear_state_opened_flag;
40ab6a6e 2668
b676f653 2669 mlx5e_refresh_tirs(priv, false);
acc6c595 2670 mlx5e_activate_priv_channels(priv);
7ca42c80
ES		 2671 if (priv->profile->update_carrier)
2672 priv->profile->update_carrier(priv);
ef9814de 2673 mlx5e_timestamp_init(priv);
be4891af 2674
cb67b832
HHZ		 2675 if (priv->profile->update_stats)
2676 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
40ab6a6e 2677
9b37b07f 2678 return 0;
343b29f3
AS		 2679
2680err_clear_state_opened_flag:
2681 clear_bit(MLX5E_STATE_OPENED, &priv->state);
2682 return err;
40ab6a6e
AS		 2683}
2684
cb67b832 2685int mlx5e_open(struct net_device *netdev)
40ab6a6e
AS		 2686{
2687 struct mlx5e_priv *priv = netdev_priv(netdev);
2688 int err;
2689
2690 mutex_lock(&priv->state_lock);
2691 err = mlx5e_open_locked(netdev);
2692 mutex_unlock(&priv->state_lock);
2693
2694 return err;
2695}
2696
2697int mlx5e_close_locked(struct net_device *netdev)
2698{
2699 struct mlx5e_priv *priv = netdev_priv(netdev);
2700
a1985740
AS		 2701 /* May already be CLOSED in case a previous configuration operation
2702 * (e.g RX/TX queue size change) that involves close&open failed.
2703 */
2704 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2705 return 0;
2706
40ab6a6e
AS		 2707 clear_bit(MLX5E_STATE_OPENED, &priv->state);
2708
ef9814de 2709 mlx5e_timestamp_cleanup(priv);
40ab6a6e 2710 netif_carrier_off(priv->netdev);
acc6c595
SM		 2711 mlx5e_deactivate_priv_channels(priv);
2712 mlx5e_close_channels(&priv->channels);
40ab6a6e
AS		 2713
2714 return 0;
2715}
2716
cb67b832 2717int mlx5e_close(struct net_device *netdev)
40ab6a6e
AS		 2718{
2719 struct mlx5e_priv *priv = netdev_priv(netdev);
2720 int err;
2721
26e59d80
MHY		 2722 if (!netif_device_present(netdev))
2723 return -ENODEV;
2724
40ab6a6e
AS		 2725 mutex_lock(&priv->state_lock);
2726 err = mlx5e_close_locked(netdev);
2727 mutex_unlock(&priv->state_lock);
2728
2729 return err;
2730}
2731
a43b25da 2732static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3b77235b
SM		 2733 struct mlx5e_rq *rq,
2734 struct mlx5e_rq_param *param)
40ab6a6e 2735{
40ab6a6e
AS		 2736 void *rqc = param->rqc;
2737 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2738 int err;
2739
2740 param->wq.db_numa_node = param->wq.buf_numa_node;
2741
2742 err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2743 &rq->wq_ctrl);
2744 if (err)
2745 return err;
2746
a43b25da 2747 rq->mdev = mdev;
40ab6a6e
AS		 2748
2749 return 0;
2750}
2751
a43b25da 2752static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev,
3b77235b
SM		 2753 struct mlx5e_cq *cq,
2754 struct mlx5e_cq_param *param)
40ab6a6e 2755{
95b6c6a5 2756 return mlx5e_alloc_cq_common(mdev, param, cq);
40ab6a6e
AS		 2757}
2758
a43b25da
SM		 2759static int mlx5e_open_drop_rq(struct mlx5_core_dev *mdev,
2760 struct mlx5e_rq *drop_rq)
40ab6a6e 2761{
a43b25da
SM		 2762 struct mlx5e_cq_param cq_param = {};
2763 struct mlx5e_rq_param rq_param = {};
2764 struct mlx5e_cq *cq = &drop_rq->cq;
40ab6a6e
AS		 2765 int err;
2766
556dd1b9 2767 mlx5e_build_drop_rq_param(&rq_param);
40ab6a6e 2768
a43b25da 2769 err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param);
40ab6a6e
AS		 2770 if (err)
2771 return err;
2772
3b77235b 2773 err = mlx5e_create_cq(cq, &cq_param);
40ab6a6e 2774 if (err)
3b77235b 2775 goto err_free_cq;
40ab6a6e 2776
a43b25da 2777 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
40ab6a6e 2778 if (err)
3b77235b 2779 goto err_destroy_cq;
40ab6a6e 2780
a43b25da 2781 err = mlx5e_create_rq(drop_rq, &rq_param);
40ab6a6e 2782 if (err)
3b77235b 2783 goto err_free_rq;
40ab6a6e
AS		 2784
2785 return 0;
2786
3b77235b 2787err_free_rq:
a43b25da 2788 mlx5e_free_rq(drop_rq);
40ab6a6e
AS		 2789
2790err_destroy_cq:
a43b25da 2791 mlx5e_destroy_cq(cq);
40ab6a6e 2792
3b77235b 2793err_free_cq:
a43b25da 2794 mlx5e_free_cq(cq);
3b77235b 2795
40ab6a6e
AS		 2796 return err;
2797}
2798
a43b25da 2799static void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
40ab6a6e 2800{
a43b25da
SM		 2801 mlx5e_destroy_rq(drop_rq);
2802 mlx5e_free_rq(drop_rq);
2803 mlx5e_destroy_cq(&drop_rq->cq);
2804 mlx5e_free_cq(&drop_rq->cq);
40ab6a6e
AS		 2805}
2806
5426a0b2
SM		 2807int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
2808 u32 underlay_qpn, u32 *tisn)
40ab6a6e 2809{
c4f287c4 2810 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
40ab6a6e
AS		 2811 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2812
08fb1dac 2813 MLX5_SET(tisc, tisc, prio, tc << 1);
5426a0b2 2814 MLX5_SET(tisc, tisc, underlay_qpn, underlay_qpn);
b50d292b 2815 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
db60b802
AH		 2816
2817 if (mlx5_lag_is_lacp_owner(mdev))
2818 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2819
5426a0b2 2820 return mlx5_core_create_tis(mdev, in, sizeof(in), tisn);
40ab6a6e
AS		 2821}
2822
5426a0b2 2823void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
40ab6a6e 2824{
5426a0b2 2825 mlx5_core_destroy_tis(mdev, tisn);
40ab6a6e
AS		 2826}
2827
cb67b832 2828int mlx5e_create_tises(struct mlx5e_priv *priv)
40ab6a6e
AS		 2829{
2830 int err;
2831 int tc;
2832
6bfd390b 2833 for (tc = 0; tc < priv->profile->max_tc; tc++) {
5426a0b2 2834 err = mlx5e_create_tis(priv->mdev, tc, 0, &priv->tisn[tc]);
40ab6a6e
AS		 2835 if (err)
2836 goto err_close_tises;
2837 }
2838
2839 return 0;
2840
2841err_close_tises:
2842 for (tc--; tc >= 0; tc--)
5426a0b2 2843 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
40ab6a6e
AS		 2844
2845 return err;
2846}
2847
cb67b832 2848void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
40ab6a6e
AS		 2849{
2850 int tc;
2851
6bfd390b 2852 for (tc = 0; tc < priv->profile->max_tc; tc++)
5426a0b2 2853 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
40ab6a6e
AS		 2854}
2855
6a9764ef
SM		 2856static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
2857 enum mlx5e_traffic_types tt,
2858 u32 *tirc)
f62b8bb8 2859{
b50d292b 2860 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3191e05f 2861
6a9764ef 2862 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
f62b8bb8 2863
4cbeaff5 2864 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
398f3351 2865 MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
6a9764ef 2866 mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc);
f62b8bb8
AV		 2867}
2868
6a9764ef 2869static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
f62b8bb8 2870{
b50d292b 2871 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
1da36696 2872
6a9764ef 2873 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
1da36696
TT		 2874
2875 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2876 MLX5_SET(tirc, tirc, indirect_table, rqtn);
2877 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2878}
2879
8f493ffd 2880int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
1da36696 2881{
724b2aa1 2882 struct mlx5e_tir *tir;
f62b8bb8
AV		 2883 void *tirc;
2884 int inlen;
2885 int err;
1da36696 2886 u32 *in;
1da36696 2887 int tt;
f62b8bb8
AV		 2888
2889 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 2890 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 2891 if (!in)
2892 return -ENOMEM;
2893
1da36696
TT		 2894 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2895 memset(in, 0, inlen);
724b2aa1 2896 tir = &priv->indir_tir[tt];
1da36696 2897 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
6a9764ef 2898 mlx5e_build_indir_tir_ctx(priv, tt, tirc);
724b2aa1 2899 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
f62b8bb8 2900 if (err)
40ab6a6e 2901 goto err_destroy_tirs;
f62b8bb8
AV		 2902 }
2903
6bfd390b
HHZ		 2904 kvfree(in);
2905
2906 return 0;
2907
2908err_destroy_tirs:
8f493ffd 2909 mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
6bfd390b
HHZ		 2910 for (tt--; tt >= 0; tt--)
2911 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2912
2913 kvfree(in);
2914
2915 return err;
2916}
2917
cb67b832 2918int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2919{
2920 int nch = priv->profile->max_nch(priv->mdev);
2921 struct mlx5e_tir *tir;
2922 void *tirc;
2923 int inlen;
2924 int err;
2925 u32 *in;
2926 int ix;
2927
2928 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 2929 in = kvzalloc(inlen, GFP_KERNEL);
6bfd390b
HHZ		 2930 if (!in)
2931 return -ENOMEM;
2932
1da36696
TT		 2933 for (ix = 0; ix < nch; ix++) {
2934 memset(in, 0, inlen);
724b2aa1 2935 tir = &priv->direct_tir[ix];
1da36696 2936 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
6a9764ef 2937 mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc);
724b2aa1 2938 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
1da36696
TT		 2939 if (err)
2940 goto err_destroy_ch_tirs;
2941 }
2942
2943 kvfree(in);
2944
f62b8bb8
AV		 2945 return 0;
2946
1da36696 2947err_destroy_ch_tirs:
8f493ffd 2948 mlx5_core_warn(priv->mdev, "create direct tirs failed, %d\n", err);
1da36696 2949 for (ix--; ix >= 0; ix--)
724b2aa1 2950 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
1da36696 2951
1da36696 2952 kvfree(in);
f62b8bb8
AV		 2953
2954 return err;
2955}
2956
8f493ffd 2957void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
f62b8bb8
AV		 2958{
2959 int i;
2960
1da36696 2961 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
724b2aa1 2962 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
f62b8bb8
AV		 2963}
2964
cb67b832 2965void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
6bfd390b
HHZ		 2966{
2967 int nch = priv->profile->max_nch(priv->mdev);
2968 int i;
2969
2970 for (i = 0; i < nch; i++)
2971 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2972}
2973
102722fc
GE		 2974static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
2975{
2976 int err = 0;
2977 int i;
2978
2979 for (i = 0; i < chs->num; i++) {
2980 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
2981 if (err)
2982 return err;
2983 }
2984
2985 return 0;
2986}
2987
f6d96a20 2988static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
36350114
GP		 2989{
2990 int err = 0;
2991 int i;
2992
ff9c852f
SM		 2993 for (i = 0; i < chs->num; i++) {
2994 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
36350114
GP		 2995 if (err)
2996 return err;
2997 }
2998
2999 return 0;
3000}
3001
08fb1dac
SM		 3002static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
3003{
3004 struct mlx5e_priv *priv = netdev_priv(netdev);
6f9485af 3005 struct mlx5e_channels new_channels = {};
08fb1dac
SM		 3006 int err = 0;
3007
3008 if (tc && tc != MLX5E_MAX_NUM_TC)
3009 return -EINVAL;
3010
3011 mutex_lock(&priv->state_lock);
3012
6f9485af
SM		 3013 new_channels.params = priv->channels.params;
3014 new_channels.params.num_tc = tc ? tc : 1;
08fb1dac 3015
20b6a1c7 3016 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
6f9485af
SM		 3017 priv->channels.params = new_channels.params;
3018 goto out;
3019 }
08fb1dac 3020
6f9485af
SM		 3021 err = mlx5e_open_channels(priv, &new_channels);
3022 if (err)
3023 goto out;
08fb1dac 3024
2e20a151 3025 mlx5e_switch_priv_channels(priv, &new_channels, NULL);
6f9485af 3026out:
08fb1dac 3027 mutex_unlock(&priv->state_lock);
08fb1dac
SM		 3028 return err;
3029}
3030
3031static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
a5fcf8a6
JP		 3032 u32 chain_index, __be16 proto,
3033 struct tc_to_netdev *tc)
08fb1dac 3034{
e8f887ac
AV		 3035 struct mlx5e_priv *priv = netdev_priv(dev);
3036
3037 if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
3038 goto mqprio;
3039
a5fcf8a6
JP		 3040 if (chain_index)
3041 return -EOPNOTSUPP;
3042
e8f887ac 3043 switch (tc->type) {
e3a2b7ed
AV		 3044 case TC_SETUP_CLSFLOWER:
3045 switch (tc->cls_flower->command) {
3046 case TC_CLSFLOWER_REPLACE:
3047 return mlx5e_configure_flower(priv, proto, tc->cls_flower);
3048 case TC_CLSFLOWER_DESTROY:
3049 return mlx5e_delete_flower(priv, tc->cls_flower);
aad7e08d
AV		 3050 case TC_CLSFLOWER_STATS:
3051 return mlx5e_stats_flower(priv, tc->cls_flower);
e3a2b7ed 3052 }
e8f887ac
AV		 3053 default:
3054 return -EOPNOTSUPP;
3055 }
3056
3057mqprio:
67ba422e 3058 if (tc->type != TC_SETUP_MQPRIO)
08fb1dac
SM		 3059 return -EINVAL;
3060
56f36acd
AN		 3061 tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3062
3063 return mlx5e_setup_tc(dev, tc->mqprio->num_tc);
08fb1dac
SM		 3064}
3065
bc1f4470 3066static void
f62b8bb8
AV		 3067mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3068{
3069 struct mlx5e_priv *priv = netdev_priv(dev);
9218b44d 3070 struct mlx5e_sw_stats *sstats = &priv->stats.sw;
f62b8bb8 3071 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
269e6b3a 3072 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
f62b8bb8 3073
370bad0f
OG		 3074 if (mlx5e_is_uplink_rep(priv)) {
3075 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3076 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok);
3077 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3078 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3079 } else {
3080 stats->rx_packets = sstats->rx_packets;
3081 stats->rx_bytes = sstats->rx_bytes;
3082 stats->tx_packets = sstats->tx_packets;
3083 stats->tx_bytes = sstats->tx_bytes;
3084 stats->tx_dropped = sstats->tx_queue_dropped;
3085 }
269e6b3a
GP		 3086
3087 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
269e6b3a
GP		 3088
3089 stats->rx_length_errors =
9218b44d
GP		 3090 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3091 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3092 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
269e6b3a 3093 stats->rx_crc_errors =
9218b44d
GP		 3094 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3095 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3096 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
269e6b3a
GP		 3097 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3098 stats->rx_frame_errors;
3099 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3100
3101 /* vport multicast also counts packets that are dropped due to steering
3102 * or rx out of buffer
3103 */
9218b44d
GP		 3104 stats->multicast =
3105 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
f62b8bb8
AV		 3106}
3107
3108static void mlx5e_set_rx_mode(struct net_device *dev)
3109{
3110 struct mlx5e_priv *priv = netdev_priv(dev);
3111
7bb29755 3112 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 3113}
3114
3115static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3116{
3117 struct mlx5e_priv *priv = netdev_priv(netdev);
3118 struct sockaddr *saddr = addr;
3119
3120 if (!is_valid_ether_addr(saddr->sa_data))
3121 return -EADDRNOTAVAIL;
3122
3123 netif_addr_lock_bh(netdev);
3124 ether_addr_copy(netdev->dev_addr, saddr->sa_data);
3125 netif_addr_unlock_bh(netdev);
3126
7bb29755 3127 queue_work(priv->wq, &priv->set_rx_mode_work);
f62b8bb8
AV		 3128
3129 return 0;
3130}
3131
0e405443
GP		 3132#define MLX5E_SET_FEATURE(netdev, feature, enable) \
3133 do { \
3134 if (enable) \
3135 netdev->features |= feature; \
3136 else \
3137 netdev->features &= ~feature; \
3138 } while (0)
3139
3140typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3141
3142static int set_feature_lro(struct net_device *netdev, bool enable)
f62b8bb8
AV		 3143{
3144 struct mlx5e_priv *priv = netdev_priv(netdev);
2e20a151
SM		 3145 struct mlx5e_channels new_channels = {};
3146 int err = 0;
3147 bool reset;
f62b8bb8
AV		 3148
3149 mutex_lock(&priv->state_lock);
f62b8bb8 3150
2e20a151
SM		 3151 reset = (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST);
3152 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
98e81b0a 3153
2e20a151
SM		 3154 new_channels.params = priv->channels.params;
3155 new_channels.params.lro_en = enable;
3156
3157 if (!reset) {
3158 priv->channels.params = new_channels.params;
3159 err = mlx5e_modify_tirs_lro(priv);
3160 goto out;
98e81b0a 3161 }
f62b8bb8 3162
2e20a151
SM		 3163 err = mlx5e_open_channels(priv, &new_channels);
3164 if (err)
3165 goto out;
0e405443 3166
2e20a151
SM		 3167 mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_modify_tirs_lro);
3168out:
9b37b07f 3169 mutex_unlock(&priv->state_lock);
0e405443
GP		 3170 return err;
3171}
3172
3173static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
3174{
3175 struct mlx5e_priv *priv = netdev_priv(netdev);
3176
3177 if (enable)
3178 mlx5e_enable_vlan_filter(priv);
3179 else
3180 mlx5e_disable_vlan_filter(priv);
3181
3182 return 0;
3183}
3184
3185static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
3186{
3187 struct mlx5e_priv *priv = netdev_priv(netdev);
f62b8bb8 3188
0e405443 3189 if (!enable && mlx5e_tc_num_filters(priv)) {
e8f887ac
AV		 3190 netdev_err(netdev,
3191 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3192 return -EINVAL;
3193 }
3194
0e405443
GP		 3195 return 0;
3196}
3197
94cb1ebb
EBE		 3198static int set_feature_rx_all(struct net_device *netdev, bool enable)
3199{
3200 struct mlx5e_priv *priv = netdev_priv(netdev);
3201 struct mlx5_core_dev *mdev = priv->mdev;
3202
3203 return mlx5_set_port_fcs(mdev, !enable);
3204}
3205
102722fc
GE		 3206static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3207{
3208 struct mlx5e_priv *priv = netdev_priv(netdev);
3209 int err;
3210
3211 mutex_lock(&priv->state_lock);
3212
3213 priv->channels.params.scatter_fcs_en = enable;
3214 err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable);
3215 if (err)
3216 priv->channels.params.scatter_fcs_en = !enable;
3217
3218 mutex_unlock(&priv->state_lock);
3219
3220 return err;
3221}
3222
36350114
GP		 3223static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3224{
3225 struct mlx5e_priv *priv = netdev_priv(netdev);
ff9c852f 3226 int err = 0;
36350114
GP		 3227
3228 mutex_lock(&priv->state_lock);
3229
6a9764ef 3230 priv->channels.params.vlan_strip_disable = !enable;
ff9c852f
SM		 3231 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3232 goto unlock;
3233
3234 err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
36350114 3235 if (err)
6a9764ef 3236 priv->channels.params.vlan_strip_disable = enable;
36350114 3237
ff9c852f 3238unlock:
36350114
GP		 3239 mutex_unlock(&priv->state_lock);
3240
3241 return err;
3242}
3243
45bf454a
MG		 3244#ifdef CONFIG_RFS_ACCEL
3245static int set_feature_arfs(struct net_device *netdev, bool enable)
3246{
3247 struct mlx5e_priv *priv = netdev_priv(netdev);
3248 int err;
3249
3250 if (enable)
3251 err = mlx5e_arfs_enable(priv);
3252 else
3253 err = mlx5e_arfs_disable(priv);
3254
3255 return err;
3256}
3257#endif
3258
0e405443
GP		 3259static int mlx5e_handle_feature(struct net_device *netdev,
3260 netdev_features_t wanted_features,
3261 netdev_features_t feature,
3262 mlx5e_feature_handler feature_handler)
3263{
3264 netdev_features_t changes = wanted_features ^ netdev->features;
3265 bool enable = !!(wanted_features & feature);
3266 int err;
3267
3268 if (!(changes & feature))
3269 return 0;
3270
3271 err = feature_handler(netdev, enable);
3272 if (err) {
3273 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
3274 enable ? "Enable" : "Disable", feature, err);
3275 return err;
3276 }
3277
3278 MLX5E_SET_FEATURE(netdev, feature, enable);
3279 return 0;
3280}
3281
3282static int mlx5e_set_features(struct net_device *netdev,
3283 netdev_features_t features)
3284{
3285 int err;
3286
3287 err = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
3288 set_feature_lro);
3289 err |= mlx5e_handle_feature(netdev, features,
3290 NETIF_F_HW_VLAN_CTAG_FILTER,
3291 set_feature_vlan_filter);
3292 err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
3293 set_feature_tc_num_filters);
94cb1ebb
EBE		 3294 err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
3295 set_feature_rx_all);
102722fc
GE		 3296 err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXFCS,
3297 set_feature_rx_fcs);
36350114
GP		 3298 err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
3299 set_feature_rx_vlan);
45bf454a
MG		 3300#ifdef CONFIG_RFS_ACCEL
3301 err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
3302 set_feature_arfs);
3303#endif
0e405443
GP		 3304
3305 return err ? -EINVAL : 0;
f62b8bb8
AV		 3306}
3307
3308static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
3309{
3310 struct mlx5e_priv *priv = netdev_priv(netdev);
2e20a151
SM		 3311 struct mlx5e_channels new_channels = {};
3312 int curr_mtu;
98e81b0a 3313 int err = 0;
506753b0 3314 bool reset;
f62b8bb8 3315
f62b8bb8 3316 mutex_lock(&priv->state_lock);
98e81b0a 3317
6a9764ef
SM		 3318 reset = !priv->channels.params.lro_en &&
3319 (priv->channels.params.rq_wq_type !=
506753b0
TT		 3320 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
3321
2e20a151 3322 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
98e81b0a 3323
2e20a151 3324 curr_mtu = netdev->mtu;
f62b8bb8 3325 netdev->mtu = new_mtu;
98e81b0a 3326
2e20a151
SM		 3327 if (!reset) {
3328 mlx5e_set_dev_port_mtu(priv);
3329 goto out;
3330 }
98e81b0a 3331
2e20a151
SM		 3332 new_channels.params = priv->channels.params;
3333 err = mlx5e_open_channels(priv, &new_channels);
3334 if (err) {
3335 netdev->mtu = curr_mtu;
3336 goto out;
3337 }
3338
3339 mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_set_dev_port_mtu);
f62b8bb8 3340
2e20a151
SM		 3341out:
3342 mutex_unlock(&priv->state_lock);
f62b8bb8
AV		 3343 return err;
3344}
3345
ef9814de
EBE		 3346static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3347{
1170fbd8
FD		 3348 struct mlx5e_priv *priv = netdev_priv(dev);
3349
ef9814de
EBE		 3350 switch (cmd) {
3351 case SIOCSHWTSTAMP:
1170fbd8 3352 return mlx5e_hwstamp_set(priv, ifr);
ef9814de 3353 case SIOCGHWTSTAMP:
1170fbd8 3354 return mlx5e_hwstamp_get(priv, ifr);
ef9814de
EBE		 3355 default:
3356 return -EOPNOTSUPP;
3357 }
3358}
3359
66e49ded
SM		 3360static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
3361{
3362 struct mlx5e_priv *priv = netdev_priv(dev);
3363 struct mlx5_core_dev *mdev = priv->mdev;
3364
3365 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
3366}
3367
79aab093
MS		 3368static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
3369 __be16 vlan_proto)
66e49ded
SM		 3370{
3371 struct mlx5e_priv *priv = netdev_priv(dev);
3372 struct mlx5_core_dev *mdev = priv->mdev;
3373
79aab093
MS		 3374 if (vlan_proto != htons(ETH_P_8021Q))
3375 return -EPROTONOSUPPORT;
3376
66e49ded
SM		 3377 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
3378 vlan, qos);
3379}
3380
f942380c
MHY		 3381static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
3382{
3383 struct mlx5e_priv *priv = netdev_priv(dev);
3384 struct mlx5_core_dev *mdev = priv->mdev;
3385
3386 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
3387}
3388
1edc57e2
MHY		 3389static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
3390{
3391 struct mlx5e_priv *priv = netdev_priv(dev);
3392 struct mlx5_core_dev *mdev = priv->mdev;
3393
3394 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
3395}
bd77bf1c
MHY		 3396
3397static int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
3398 int max_tx_rate)
3399{
3400 struct mlx5e_priv *priv = netdev_priv(dev);
3401 struct mlx5_core_dev *mdev = priv->mdev;
3402
bd77bf1c 3403 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
c9497c98 3404 max_tx_rate, min_tx_rate);
bd77bf1c
MHY		 3405}
3406
66e49ded
SM		 3407static int mlx5_vport_link2ifla(u8 esw_link)
3408{
3409 switch (esw_link) {
3410 case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
3411 return IFLA_VF_LINK_STATE_DISABLE;
3412 case MLX5_ESW_VPORT_ADMIN_STATE_UP:
3413 return IFLA_VF_LINK_STATE_ENABLE;
3414 }
3415 return IFLA_VF_LINK_STATE_AUTO;
3416}
3417
3418static int mlx5_ifla_link2vport(u8 ifla_link)
3419{
3420 switch (ifla_link) {
3421 case IFLA_VF_LINK_STATE_DISABLE:
3422 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
3423 case IFLA_VF_LINK_STATE_ENABLE:
3424 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
3425 }
3426 return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
3427}
3428
3429static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
3430 int link_state)
3431{
3432 struct mlx5e_priv *priv = netdev_priv(dev);
3433 struct mlx5_core_dev *mdev = priv->mdev;
3434
3435 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
3436 mlx5_ifla_link2vport(link_state));
3437}
3438
3439static int mlx5e_get_vf_config(struct net_device *dev,
3440 int vf, struct ifla_vf_info *ivi)
3441{
3442 struct mlx5e_priv *priv = netdev_priv(dev);
3443 struct mlx5_core_dev *mdev = priv->mdev;
3444 int err;
3445
3446 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
3447 if (err)
3448 return err;
3449 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
3450 return 0;
3451}
3452
3453static int mlx5e_get_vf_stats(struct net_device *dev,
3454 int vf, struct ifla_vf_stats *vf_stats)
3455{
3456 struct mlx5e_priv *priv = netdev_priv(dev);
3457 struct mlx5_core_dev *mdev = priv->mdev;
3458
3459 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
3460 vf_stats);
3461}
3462
1ad9a00a
PB		 3463static void mlx5e_add_vxlan_port(struct net_device *netdev,
3464 struct udp_tunnel_info *ti)
b3f63c3d
MF		 3465{
3466 struct mlx5e_priv *priv = netdev_priv(netdev);
3467
974c3f30
AD		 3468 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3469 return;
3470
b3f63c3d
MF		 3471 if (!mlx5e_vxlan_allowed(priv->mdev))
3472 return;
3473
974c3f30 3474 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
b3f63c3d
MF		 3475}
3476
1ad9a00a
PB		 3477static void mlx5e_del_vxlan_port(struct net_device *netdev,
3478 struct udp_tunnel_info *ti)
b3f63c3d
MF		 3479{
3480 struct mlx5e_priv *priv = netdev_priv(netdev);
3481
974c3f30
AD		 3482 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3483 return;
3484
b3f63c3d
MF		 3485 if (!mlx5e_vxlan_allowed(priv->mdev))
3486 return;
3487
974c3f30 3488 mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
b3f63c3d
MF		 3489}
3490
3491static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
3492 struct sk_buff *skb,
3493 netdev_features_t features)
3494{
3495 struct udphdr *udph;
3496 u16 proto;
3497 u16 port = 0;
3498
3499 switch (vlan_get_protocol(skb)) {
3500 case htons(ETH_P_IP):
3501 proto = ip_hdr(skb)->protocol;
3502 break;
3503 case htons(ETH_P_IPV6):
3504 proto = ipv6_hdr(skb)->nexthdr;
3505 break;
3506 default:
3507 goto out;
3508 }
3509
3510 if (proto == IPPROTO_UDP) {
3511 udph = udp_hdr(skb);
3512 port = be16_to_cpu(udph->dest);
3513 }
3514
3515 /* Verify if UDP port is being offloaded by HW */
3516 if (port && mlx5e_vxlan_lookup_port(priv, port))
3517 return features;
3518
3519out:
3520 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
3521 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3522}
3523
3524static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
3525 struct net_device *netdev,
3526 netdev_features_t features)
3527{
3528 struct mlx5e_priv *priv = netdev_priv(netdev);
3529
3530 features = vlan_features_check(skb, features);
3531 features = vxlan_features_check(skb, features);
3532
2ac9cfe7
IT		 3533#ifdef CONFIG_MLX5_EN_IPSEC
3534 if (mlx5e_ipsec_feature_check(skb, netdev, features))
3535 return features;
3536#endif
3537
b3f63c3d
MF		 3538 /* Validate if the tunneled packet is being offloaded by HW */
3539 if (skb->encapsulation &&
3540 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
3541 return mlx5e_vxlan_features_check(priv, skb, features);
3542
3543 return features;
3544}
3545
3947ca18
DJ		 3546static void mlx5e_tx_timeout(struct net_device *dev)
3547{
3548 struct mlx5e_priv *priv = netdev_priv(dev);
3549 bool sched_work = false;
3550 int i;
3551
3552 netdev_err(dev, "TX timeout detected\n");
3553
6a9764ef 3554 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
acc6c595 3555 struct mlx5e_txqsq *sq = priv->txq2sq[i];
3947ca18 3556
2c1ccc99 3557 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
3947ca18
DJ		 3558 continue;
3559 sched_work = true;
c0f1147d 3560 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
3947ca18
DJ		 3561 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
3562 i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
3563 }
3564
3565 if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
3566 schedule_work(&priv->tx_timeout_work);
3567}
3568
86994156
RS		 3569static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
3570{
3571 struct mlx5e_priv *priv = netdev_priv(netdev);
3572 struct bpf_prog *old_prog;
3573 int err = 0;
3574 bool reset, was_opened;
3575 int i;
3576
3577 mutex_lock(&priv->state_lock);
3578
3579 if ((netdev->features & NETIF_F_LRO) && prog) {
3580 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
3581 err = -EINVAL;
3582 goto unlock;
3583 }
3584
547eede0
IT		 3585 if ((netdev->features & NETIF_F_HW_ESP) && prog) {
3586 netdev_warn(netdev, "can't set XDP with IPSec offload\n");
3587 err = -EINVAL;
3588 goto unlock;
3589 }
3590
86994156
RS		 3591 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3592 /* no need for full reset when exchanging programs */
6a9764ef 3593 reset = (!priv->channels.params.xdp_prog || !prog);
86994156
RS		 3594
3595 if (was_opened && reset)
3596 mlx5e_close_locked(netdev);
c54c0629
DB		 3597 if (was_opened && !reset) {
3598 /* num_channels is invariant here, so we can take the
3599 * batched reference right upfront.
3600 */
6a9764ef 3601 prog = bpf_prog_add(prog, priv->channels.num);
c54c0629
DB		 3602 if (IS_ERR(prog)) {
3603 err = PTR_ERR(prog);
3604 goto unlock;
3605 }
3606 }
86994156 3607
c54c0629
DB		 3608 /* exchange programs, extra prog reference we got from caller
3609 * as long as we don't fail from this point onwards.
3610 */
6a9764ef 3611 old_prog = xchg(&priv->channels.params.xdp_prog, prog);
86994156
RS		 3612 if (old_prog)
3613 bpf_prog_put(old_prog);
3614
3615 if (reset) /* change RQ type according to priv->xdp_prog */
6a9764ef 3616 mlx5e_set_rq_params(priv->mdev, &priv->channels.params);
86994156
RS		 3617
3618 if (was_opened && reset)
3619 mlx5e_open_locked(netdev);
3620
3621 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
3622 goto unlock;
3623
3624 /* exchanging programs w/o reset, we update ref counts on behalf
3625 * of the channels RQs here.
3626 */
ff9c852f
SM		 3627 for (i = 0; i < priv->channels.num; i++) {
3628 struct mlx5e_channel *c = priv->channels.c[i];
86994156 3629
c0f1147d 3630 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
86994156
RS		 3631 napi_synchronize(&c->napi);
3632 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
3633
3634 old_prog = xchg(&c->rq.xdp_prog, prog);
3635
c0f1147d 3636 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
86994156
RS		 3637 /* napi_schedule in case we have missed anything */
3638 set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
3639 napi_schedule(&c->napi);
3640
3641 if (old_prog)
3642 bpf_prog_put(old_prog);
3643 }
3644
3645unlock:
3646 mutex_unlock(&priv->state_lock);
3647 return err;
3648}
3649
821b2e29 3650static u32 mlx5e_xdp_query(struct net_device *dev)
86994156
RS		 3651{
3652 struct mlx5e_priv *priv = netdev_priv(dev);
821b2e29
MKL		 3653 const struct bpf_prog *xdp_prog;
3654 u32 prog_id = 0;
86994156 3655
821b2e29
MKL		 3656 mutex_lock(&priv->state_lock);
3657 xdp_prog = priv->channels.params.xdp_prog;
3658 if (xdp_prog)
3659 prog_id = xdp_prog->aux->id;
3660 mutex_unlock(&priv->state_lock);
3661
3662 return prog_id;
86994156
RS		 3663}
3664
3665static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
3666{
3667 switch (xdp->command) {
3668 case XDP_SETUP_PROG:
3669 return mlx5e_xdp_set(dev, xdp->prog);
3670 case XDP_QUERY_PROG:
821b2e29
MKL		 3671 xdp->prog_id = mlx5e_xdp_query(dev);
3672 xdp->prog_attached = !!xdp->prog_id;
86994156
RS		 3673 return 0;
3674 default:
3675 return -EINVAL;
3676 }
3677}
3678
80378384
CO		 3679#ifdef CONFIG_NET_POLL_CONTROLLER
3680/* Fake "interrupt" called by netpoll (eg netconsole) to send skbs without
3681 * reenabling interrupts.
3682 */
3683static void mlx5e_netpoll(struct net_device *dev)
3684{
3685 struct mlx5e_priv *priv = netdev_priv(dev);
ff9c852f
SM		 3686 struct mlx5e_channels *chs = &priv->channels;
3687
80378384
CO		 3688 int i;
3689
ff9c852f
SM		 3690 for (i = 0; i < chs->num; i++)
3691 napi_schedule(&chs->c[i]->napi);
80378384
CO		 3692}
3693#endif
3694
b0eed40e 3695static const struct net_device_ops mlx5e_netdev_ops_basic = {
f62b8bb8
AV		 3696 .ndo_open = mlx5e_open,
3697 .ndo_stop = mlx5e_close,
3698 .ndo_start_xmit = mlx5e_xmit,
08fb1dac
SM		 3699 .ndo_setup_tc = mlx5e_ndo_setup_tc,
3700 .ndo_select_queue = mlx5e_select_queue,
f62b8bb8
AV		 3701 .ndo_get_stats64 = mlx5e_get_stats,
3702 .ndo_set_rx_mode = mlx5e_set_rx_mode,
3703 .ndo_set_mac_address = mlx5e_set_mac,
b0eed40e
SM		 3704 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
3705 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
f62b8bb8 3706 .ndo_set_features = mlx5e_set_features,
b0eed40e
SM		 3707 .ndo_change_mtu = mlx5e_change_mtu,
3708 .ndo_do_ioctl = mlx5e_ioctl,
507f0c81 3709 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
45bf454a
MG		 3710#ifdef CONFIG_RFS_ACCEL
3711 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
3712#endif
3947ca18 3713 .ndo_tx_timeout = mlx5e_tx_timeout,
86994156 3714 .ndo_xdp = mlx5e_xdp,
80378384
CO		 3715#ifdef CONFIG_NET_POLL_CONTROLLER
3716 .ndo_poll_controller = mlx5e_netpoll,
3717#endif
b0eed40e
SM		 3718};
3719
3720static const struct net_device_ops mlx5e_netdev_ops_sriov = {
3721 .ndo_open = mlx5e_open,
3722 .ndo_stop = mlx5e_close,
3723 .ndo_start_xmit = mlx5e_xmit,
08fb1dac
SM		 3724 .ndo_setup_tc = mlx5e_ndo_setup_tc,
3725 .ndo_select_queue = mlx5e_select_queue,
b0eed40e
SM		 3726 .ndo_get_stats64 = mlx5e_get_stats,
3727 .ndo_set_rx_mode = mlx5e_set_rx_mode,
3728 .ndo_set_mac_address = mlx5e_set_mac,
3729 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
3730 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
3731 .ndo_set_features = mlx5e_set_features,
3732 .ndo_change_mtu = mlx5e_change_mtu,
3733 .ndo_do_ioctl = mlx5e_ioctl,
974c3f30
AD		 3734 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port,
3735 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port,
507f0c81 3736 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
b3f63c3d 3737 .ndo_features_check = mlx5e_features_check,
45bf454a
MG		 3738#ifdef CONFIG_RFS_ACCEL
3739 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
3740#endif
b0eed40e
SM		 3741 .ndo_set_vf_mac = mlx5e_set_vf_mac,
3742 .ndo_set_vf_vlan = mlx5e_set_vf_vlan,
f942380c 3743 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk,
1edc57e2 3744 .ndo_set_vf_trust = mlx5e_set_vf_trust,
bd77bf1c 3745 .ndo_set_vf_rate = mlx5e_set_vf_rate,
b0eed40e
SM		 3746 .ndo_get_vf_config = mlx5e_get_vf_config,
3747 .ndo_set_vf_link_state = mlx5e_set_vf_link_state,
3748 .ndo_get_vf_stats = mlx5e_get_vf_stats,
3947ca18 3749 .ndo_tx_timeout = mlx5e_tx_timeout,
86994156 3750 .ndo_xdp = mlx5e_xdp,
80378384
CO		 3751#ifdef CONFIG_NET_POLL_CONTROLLER
3752 .ndo_poll_controller = mlx5e_netpoll,
3753#endif
370bad0f
OG		 3754 .ndo_has_offload_stats = mlx5e_has_offload_stats,
3755 .ndo_get_offload_stats = mlx5e_get_offload_stats,
f62b8bb8
AV		 3756};
3757
3758static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3759{
3760 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
9eb78923 3761 return -EOPNOTSUPP;
f62b8bb8
AV		 3762 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
3763 !MLX5_CAP_GEN(mdev, nic_flow_table) ||
3764 !MLX5_CAP_ETH(mdev, csum_cap) ||
3765 !MLX5_CAP_ETH(mdev, max_lso_cap) ||
3766 !MLX5_CAP_ETH(mdev, vlan_cap) ||
796a27ec
GP		 3767 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
3768 MLX5_CAP_FLOWTABLE(mdev,
3769 flow_table_properties_nic_receive.max_ft_level)
3770 < 3) {
f62b8bb8
AV		 3771 mlx5_core_warn(mdev,
3772 "Not creating net device, some required device capabilities are missing\n");
9eb78923 3773 return -EOPNOTSUPP;
f62b8bb8 3774 }
66189961
TT		 3775 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
3776 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
7524a5d8 3777 if (!MLX5_CAP_GEN(mdev, cq_moderation))
3e432ab6 3778 mlx5_core_warn(mdev, "CQ moderation is not supported\n");
66189961 3779
f62b8bb8
AV		 3780 return 0;
3781}
3782
58d52291
AS		 3783u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3784{
3785 int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
3786
3787 return bf_buf_size -
3788 sizeof(struct mlx5e_tx_wqe) +
3789 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3790}
3791
d8c9660d
TT		 3792void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3793 u32 *indirection_rqt, int len,
85082dba
TT		 3794 int num_channels)
3795{
d8c9660d
TT		 3796 int node = mdev->priv.numa_node;
3797 int node_num_of_cores;
85082dba
TT		 3798 int i;
3799
d8c9660d
TT		 3800 if (node == -1)
3801 node = first_online_node;
3802
3803 node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3804
3805 if (node_num_of_cores)
3806 num_channels = min_t(int, num_channels, node_num_of_cores);
3807
85082dba
TT		 3808 for (i = 0; i < len; i++)
3809 indirection_rqt[i] = i % num_channels;
3810}
3811
b797a684
SM		 3812static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3813{
3814 enum pcie_link_width width;
3815 enum pci_bus_speed speed;
3816 int err = 0;
3817
3818 err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3819 if (err)
3820 return err;
3821
3822 if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3823 return -EINVAL;
3824
3825 switch (speed) {
3826 case PCIE_SPEED_2_5GT:
3827 *pci_bw = 2500 * width;
3828 break;
3829 case PCIE_SPEED_5_0GT:
3830 *pci_bw = 5000 * width;
3831 break;
3832 case PCIE_SPEED_8_0GT:
3833 *pci_bw = 8000 * width;
3834 break;
3835 default:
3836 return -EINVAL;
3837 }
3838
3839 return 0;
3840}
3841
3842static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3843{
3844 return (link_speed && pci_bw &&
3845 (pci_bw < 40000) && (pci_bw < link_speed));
3846}
3847
0f6e4cf6
EBE		 3848static bool hw_lro_heuristic(u32 link_speed, u32 pci_bw)
3849{
3850 return !(link_speed && pci_bw &&
3851 (pci_bw <= 16000) && (pci_bw < link_speed));
3852}
3853
9908aa29
TT		 3854void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3855{
3856 params->rx_cq_period_mode = cq_period_mode;
3857
3858 params->rx_cq_moderation.pkts =
3859 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3860 params->rx_cq_moderation.usec =
3861 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3862
3863 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3864 params->rx_cq_moderation.usec =
3865 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
6a9764ef 3866
457fcd8a
SM		 3867 if (params->rx_am_enabled)
3868 params->rx_cq_moderation =
3869 mlx5e_am_get_def_profile(params->rx_cq_period_mode);
3870
6a9764ef
SM		 3871 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3872 params->rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
9908aa29
TT		 3873}
3874
2b029556
SM		 3875u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
3876{
3877 int i;
3878
3879 /* The supported periods are organized in ascending order */
3880 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
3881 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
3882 break;
3883
3884 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
3885}
3886
8f493ffd
SM		 3887void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
3888 struct mlx5e_params *params,
3889 u16 max_channels)
f62b8bb8 3890{
6a9764ef 3891 u8 cq_period_mode = 0;
b797a684
SM		 3892 u32 link_speed = 0;
3893 u32 pci_bw = 0;
2fc4bfb7 3894
6a9764ef
SM		 3895 params->num_channels = max_channels;
3896 params->num_tc = 1;
2b029556 3897
0f6e4cf6
EBE		 3898 mlx5e_get_max_linkspeed(mdev, &link_speed);
3899 mlx5e_get_pci_bw(mdev, &pci_bw);
3900 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3901 link_speed, pci_bw);
3902
6a9764ef
SM		 3903 /* SQ */
3904 params->log_sq_size = is_kdump_kernel() ?
b4e029da
KH		 3905 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
3906 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
461017cb 3907
b797a684 3908 /* set CQE compression */
6a9764ef 3909 params->rx_cqe_compress_def = false;
b797a684 3910 if (MLX5_CAP_GEN(mdev, cqe_compression) &&
e53eef63 3911 MLX5_CAP_GEN(mdev, vport_group_manager))
6a9764ef 3912 params->rx_cqe_compress_def = cqe_compress_heuristic(link_speed, pci_bw);
0f6e4cf6 3913
6a9764ef
SM		 3914 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
3915
3916 /* RQ */
3917 mlx5e_set_rq_params(mdev, params);
b797a684 3918
6a9764ef 3919 /* HW LRO */
c139dbfd 3920
5426a0b2 3921 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */
6a9764ef 3922 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
0f6e4cf6 3923 params->lro_en = hw_lro_heuristic(link_speed, pci_bw);
6a9764ef 3924 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
b0d4660b 3925
6a9764ef
SM		 3926 /* CQ moderation params */
3927 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3928 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3929 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
3930 params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3931 mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
9908aa29 3932
6a9764ef
SM		 3933 params->tx_cq_moderation.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3934 params->tx_cq_moderation.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
9908aa29 3935
6a9764ef
SM		 3936 /* TX inline */
3937 params->tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3938 mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
3939 if (params->tx_min_inline_mode == MLX5_INLINE_MODE_NONE &&
a6f402e4 3940 !MLX5_CAP_ETH(mdev, wqe_vlan_insert))
6a9764ef 3941 params->tx_min_inline_mode = MLX5_INLINE_MODE_L2;
a6f402e4 3942
6a9764ef
SM		 3943 /* RSS */
3944 params->rss_hfunc = ETH_RSS_HASH_XOR;
3945 netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
3946 mlx5e_build_default_indir_rqt(mdev, params->indirection_rqt,
3947 MLX5E_INDIR_RQT_SIZE, max_channels);
3948}
f62b8bb8 3949
6a9764ef
SM		 3950static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3951 struct net_device *netdev,
3952 const struct mlx5e_profile *profile,
3953 void *ppriv)
3954{
3955 struct mlx5e_priv *priv = netdev_priv(netdev);
57afead5 3956
6a9764ef
SM		 3957 priv->mdev = mdev;
3958 priv->netdev = netdev;
3959 priv->profile = profile;
3960 priv->ppriv = ppriv;
c139dbfd 3961 priv->hard_mtu = MLX5E_ETH_HARD_MTU;
2d75b2bc 3962
6a9764ef 3963 mlx5e_build_nic_params(mdev, &priv->channels.params, profile->max_nch(mdev));
9908aa29 3964
f62b8bb8
AV		 3965 mutex_init(&priv->state_lock);
3966
3967 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3968 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3947ca18 3969 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
f62b8bb8
AV		 3970 INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3971}
3972
3973static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3974{
3975 struct mlx5e_priv *priv = netdev_priv(netdev);
3976
e1d7d349 3977 mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
108805fc
SM		 3978 if (is_zero_ether_addr(netdev->dev_addr) &&
3979 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3980 eth_hw_addr_random(netdev);
3981 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3982 }
f62b8bb8
AV		 3983}
3984
cb67b832
HHZ		 3985static const struct switchdev_ops mlx5e_switchdev_ops = {
3986 .switchdev_port_attr_get = mlx5e_attr_get,
3987};
3988
6bfd390b 3989static void mlx5e_build_nic_netdev(struct net_device *netdev)
f62b8bb8
AV		 3990{
3991 struct mlx5e_priv *priv = netdev_priv(netdev);
3992 struct mlx5_core_dev *mdev = priv->mdev;
94cb1ebb
EBE		 3993 bool fcs_supported;
3994 bool fcs_enabled;
f62b8bb8
AV		 3995
3996 SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
3997
08fb1dac 3998 if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
b0eed40e 3999 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
08fb1dac 4000#ifdef CONFIG_MLX5_CORE_EN_DCB
80653f73
HN		 4001 if (MLX5_CAP_GEN(mdev, qos))
4002 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
08fb1dac
SM		 4003#endif
4004 } else {
b0eed40e 4005 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
08fb1dac 4006 }
66e49ded 4007
f62b8bb8
AV		 4008 netdev->watchdog_timeo = 15 * HZ;
4009
4010 netdev->ethtool_ops = &mlx5e_ethtool_ops;
4011
12be4b21 4012 netdev->vlan_features |= NETIF_F_SG;
f62b8bb8
AV		 4013 netdev->vlan_features |= NETIF_F_IP_CSUM;
4014 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
4015 netdev->vlan_features |= NETIF_F_GRO;
4016 netdev->vlan_features |= NETIF_F_TSO;
4017 netdev->vlan_features |= NETIF_F_TSO6;
4018 netdev->vlan_features |= NETIF_F_RXCSUM;
4019 netdev->vlan_features |= NETIF_F_RXHASH;
4020
4021 if (!!MLX5_CAP_ETH(mdev, lro_cap))
4022 netdev->vlan_features |= NETIF_F_LRO;
4023
4024 netdev->hw_features = netdev->vlan_features;
e4cf27bd 4025 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
f62b8bb8
AV		 4026 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4027 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4028
b3f63c3d 4029 if (mlx5e_vxlan_allowed(mdev)) {
b49663c8
AD		 4030 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
4031 NETIF_F_GSO_UDP_TUNNEL_CSUM |
4032 NETIF_F_GSO_PARTIAL;
b3f63c3d 4033 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
f3ed653c 4034 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
b3f63c3d
MF		 4035 netdev->hw_enc_features |= NETIF_F_TSO;
4036 netdev->hw_enc_features |= NETIF_F_TSO6;
b3f63c3d 4037 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
b49663c8
AD		 4038 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
4039 NETIF_F_GSO_PARTIAL;
4040 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
b3f63c3d
MF		 4041 }
4042
94cb1ebb
EBE		 4043 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
4044
4045 if (fcs_supported)
4046 netdev->hw_features |= NETIF_F_RXALL;
4047
102722fc
GE		 4048 if (MLX5_CAP_ETH(mdev, scatter_fcs))
4049 netdev->hw_features |= NETIF_F_RXFCS;
4050
f62b8bb8 4051 netdev->features = netdev->hw_features;
6a9764ef 4052 if (!priv->channels.params.lro_en)
f62b8bb8
AV		 4053 netdev->features &= ~NETIF_F_LRO;
4054
94cb1ebb
EBE		 4055 if (fcs_enabled)
4056 netdev->features &= ~NETIF_F_RXALL;
4057
102722fc
GE		 4058 if (!priv->channels.params.scatter_fcs_en)
4059 netdev->features &= ~NETIF_F_RXFCS;
4060
e8f887ac
AV		 4061#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
4062 if (FT_CAP(flow_modify_en) &&
4063 FT_CAP(modify_root) &&
4064 FT_CAP(identified_miss_table_mode) &&
1cabe6b0
MG		 4065 FT_CAP(flow_table_modify)) {
4066 netdev->hw_features |= NETIF_F_HW_TC;
4067#ifdef CONFIG_RFS_ACCEL
4068 netdev->hw_features |= NETIF_F_NTUPLE;
4069#endif
4070 }
e8f887ac 4071
f62b8bb8
AV		 4072 netdev->features |= NETIF_F_HIGHDMA;
4073
4074 netdev->priv_flags |= IFF_UNICAST_FLT;
4075
4076 mlx5e_set_netdev_dev_addr(netdev);
cb67b832
HHZ		 4077
4078#ifdef CONFIG_NET_SWITCHDEV
4079 if (MLX5_CAP_GEN(mdev, vport_group_manager))
4080 netdev->switchdev_ops = &mlx5e_switchdev_ops;
4081#endif
547eede0
IT		 4082
4083 mlx5e_ipsec_build_netdev(priv);
f62b8bb8
AV		 4084}
4085
593cf338
RS		 4086static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
4087{
4088 struct mlx5_core_dev *mdev = priv->mdev;
4089 int err;
4090
4091 err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
4092 if (err) {
4093 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
4094 priv->q_counter = 0;
4095 }
4096}
4097
4098static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
4099{
4100 if (!priv->q_counter)
4101 return;
4102
4103 mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
4104}
4105
6bfd390b
HHZ		 4106static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
4107 struct net_device *netdev,
127ea380
HHZ		 4108 const struct mlx5e_profile *profile,
4109 void *ppriv)
6bfd390b
HHZ		 4110{
4111 struct mlx5e_priv *priv = netdev_priv(netdev);
547eede0 4112 int err;
6bfd390b 4113
127ea380 4114 mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
547eede0
IT		 4115 err = mlx5e_ipsec_init(priv);
4116 if (err)
4117 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
6bfd390b
HHZ		 4118 mlx5e_build_nic_netdev(netdev);
4119 mlx5e_vxlan_init(priv);
4120}
4121
4122static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
4123{
547eede0 4124 mlx5e_ipsec_cleanup(priv);
6bfd390b 4125 mlx5e_vxlan_cleanup(priv);
127ea380 4126
6a9764ef
SM		 4127 if (priv->channels.params.xdp_prog)
4128 bpf_prog_put(priv->channels.params.xdp_prog);
6bfd390b
HHZ		 4129}
4130
4131static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
4132{
4133 struct mlx5_core_dev *mdev = priv->mdev;
4134 int err;
6bfd390b 4135
8f493ffd
SM		 4136 err = mlx5e_create_indirect_rqt(priv);
4137 if (err)
6bfd390b 4138 return err;
6bfd390b
HHZ		 4139
4140 err = mlx5e_create_direct_rqts(priv);
8f493ffd 4141 if (err)
6bfd390b 4142 goto err_destroy_indirect_rqts;
6bfd390b
HHZ		 4143
4144 err = mlx5e_create_indirect_tirs(priv);
8f493ffd 4145 if (err)
6bfd390b 4146 goto err_destroy_direct_rqts;
6bfd390b
HHZ		 4147
4148 err = mlx5e_create_direct_tirs(priv);
8f493ffd 4149 if (err)
6bfd390b 4150 goto err_destroy_indirect_tirs;
6bfd390b
HHZ		 4151
4152 err = mlx5e_create_flow_steering(priv);
4153 if (err) {
4154 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
4155 goto err_destroy_direct_tirs;
4156 }
4157
4158 err = mlx5e_tc_init(priv);
4159 if (err)
4160 goto err_destroy_flow_steering;
4161
4162 return 0;
4163
4164err_destroy_flow_steering:
4165 mlx5e_destroy_flow_steering(priv);
4166err_destroy_direct_tirs:
4167 mlx5e_destroy_direct_tirs(priv);
4168err_destroy_indirect_tirs:
4169 mlx5e_destroy_indirect_tirs(priv);
4170err_destroy_direct_rqts:
8f493ffd 4171 mlx5e_destroy_direct_rqts(priv);
6bfd390b
HHZ		 4172err_destroy_indirect_rqts:
4173 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4174 return err;
4175}
4176
4177static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
4178{
6bfd390b
HHZ		 4179 mlx5e_tc_cleanup(priv);
4180 mlx5e_destroy_flow_steering(priv);
4181 mlx5e_destroy_direct_tirs(priv);
4182 mlx5e_destroy_indirect_tirs(priv);
8f493ffd 4183 mlx5e_destroy_direct_rqts(priv);
6bfd390b
HHZ		 4184 mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4185}
4186
4187static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
4188{
4189 int err;
4190
4191 err = mlx5e_create_tises(priv);
4192 if (err) {
4193 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
4194 return err;
4195 }
4196
4197#ifdef CONFIG_MLX5_CORE_EN_DCB
e207b7e9 4198 mlx5e_dcbnl_initialize(priv);
6bfd390b
HHZ		 4199#endif
4200 return 0;
4201}
4202
4203static void mlx5e_nic_enable(struct mlx5e_priv *priv)
4204{
4205 struct net_device *netdev = priv->netdev;
4206 struct mlx5_core_dev *mdev = priv->mdev;
2c3b5bee
SM		 4207 u16 max_mtu;
4208
4209 mlx5e_init_l2_addr(priv);
4210
4211 /* MTU range: 68 - hw-specific max */
4212 netdev->min_mtu = ETH_MIN_MTU;
4213 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
c139dbfd 4214 netdev->max_mtu = MLX5E_HW2SW_MTU(priv, max_mtu);
2c3b5bee 4215 mlx5e_set_dev_port_mtu(priv);
6bfd390b 4216
7907f23a
AH		 4217 mlx5_lag_add(mdev, netdev);
4218
6bfd390b 4219 mlx5e_enable_async_events(priv);
127ea380 4220
1d447a39
SM		 4221 if (MLX5_CAP_GEN(mdev, vport_group_manager))
4222 mlx5e_register_vport_reps(priv);
2c3b5bee 4223
610e89e0
SM		 4224 if (netdev->reg_state != NETREG_REGISTERED)
4225 return;
4226
4227 /* Device already registered: sync netdev system state */
4228 if (mlx5e_vxlan_allowed(mdev)) {
4229 rtnl_lock();
4230 udp_tunnel_get_rx_info(netdev);
4231 rtnl_unlock();
4232 }
4233
4234 queue_work(priv->wq, &priv->set_rx_mode_work);
2c3b5bee
SM		 4235
4236 rtnl_lock();
4237 if (netif_running(netdev))
4238 mlx5e_open(netdev);
4239 netif_device_attach(netdev);
4240 rtnl_unlock();
6bfd390b
HHZ		 4241}
4242
4243static void mlx5e_nic_disable(struct mlx5e_priv *priv)
4244{
3deef8ce 4245 struct mlx5_core_dev *mdev = priv->mdev;
3deef8ce 4246
2c3b5bee
SM		 4247 rtnl_lock();
4248 if (netif_running(priv->netdev))
4249 mlx5e_close(priv->netdev);
4250 netif_device_detach(priv->netdev);
4251 rtnl_unlock();
4252
6bfd390b 4253 queue_work(priv->wq, &priv->set_rx_mode_work);
1d447a39 4254
3deef8ce 4255 if (MLX5_CAP_GEN(mdev, vport_group_manager))
1d447a39
SM		 4256 mlx5e_unregister_vport_reps(priv);
4257
6bfd390b 4258 mlx5e_disable_async_events(priv);
3deef8ce 4259 mlx5_lag_remove(mdev);
6bfd390b
HHZ		 4260}
4261
4262static const struct mlx5e_profile mlx5e_nic_profile = {
4263 .init = mlx5e_nic_init,
4264 .cleanup = mlx5e_nic_cleanup,
4265 .init_rx = mlx5e_init_nic_rx,
4266 .cleanup_rx = mlx5e_cleanup_nic_rx,
4267 .init_tx = mlx5e_init_nic_tx,
4268 .cleanup_tx = mlx5e_cleanup_nic_tx,
4269 .enable = mlx5e_nic_enable,
4270 .disable = mlx5e_nic_disable,
3834a5e6 4271 .update_stats = mlx5e_update_ndo_stats,
6bfd390b 4272 .max_nch = mlx5e_get_max_num_channels,
7ca42c80 4273 .update_carrier = mlx5e_update_carrier,
20fd0c19
SM		 4274 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe,
4275 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
6bfd390b
HHZ		 4276 .max_tc = MLX5E_MAX_NUM_TC,
4277};
4278
2c3b5bee
SM		 4279/* mlx5e generic netdev management API (move to en_common.c) */
4280
26e59d80
MHY		 4281struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
4282 const struct mlx5e_profile *profile,
4283 void *ppriv)
f62b8bb8 4284{
26e59d80 4285 int nch = profile->max_nch(mdev);
f62b8bb8
AV		 4286 struct net_device *netdev;
4287 struct mlx5e_priv *priv;
f62b8bb8 4288
08fb1dac 4289 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
6bfd390b 4290 nch * profile->max_tc,
08fb1dac 4291 nch);
f62b8bb8
AV		 4292 if (!netdev) {
4293 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
4294 return NULL;
4295 }
4296
be4891af
SM		 4297#ifdef CONFIG_RFS_ACCEL
4298 netdev->rx_cpu_rmap = mdev->rmap;
4299#endif
4300
127ea380 4301 profile->init(mdev, netdev, profile, ppriv);
f62b8bb8
AV		 4302
4303 netif_carrier_off(netdev);
4304
4305 priv = netdev_priv(netdev);
4306
7bb29755
MF		 4307 priv->wq = create_singlethread_workqueue("mlx5e");
4308 if (!priv->wq)
26e59d80
MHY		 4309 goto err_cleanup_nic;
4310
4311 return netdev;
4312
4313err_cleanup_nic:
31ac9338
OG		 4314 if (profile->cleanup)
4315 profile->cleanup(priv);
26e59d80
MHY		 4316 free_netdev(netdev);
4317
4318 return NULL;
4319}
4320
2c3b5bee 4321int mlx5e_attach_netdev(struct mlx5e_priv *priv)
26e59d80 4322{
2c3b5bee 4323 struct mlx5_core_dev *mdev = priv->mdev;
26e59d80 4324 const struct mlx5e_profile *profile;
26e59d80
MHY		 4325 int err;
4326
26e59d80
MHY		 4327 profile = priv->profile;
4328 clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
7bb29755 4329
6bfd390b
HHZ		 4330 err = profile->init_tx(priv);
4331 if (err)
ec8b9981 4332 goto out;
5c50368f 4333
a43b25da 4334 err = mlx5e_open_drop_rq(mdev, &priv->drop_rq);
5c50368f
AS		 4335 if (err) {
4336 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
6bfd390b 4337 goto err_cleanup_tx;
5c50368f
AS		 4338 }
4339
6bfd390b
HHZ		 4340 err = profile->init_rx(priv);
4341 if (err)
5c50368f 4342 goto err_close_drop_rq;
5c50368f 4343
593cf338
RS		 4344 mlx5e_create_q_counter(priv);
4345
6bfd390b
HHZ		 4346 if (profile->enable)
4347 profile->enable(priv);
f62b8bb8 4348
26e59d80 4349 return 0;
5c50368f
AS		 4350
4351err_close_drop_rq:
a43b25da 4352 mlx5e_close_drop_rq(&priv->drop_rq);
5c50368f 4353
6bfd390b
HHZ		 4354err_cleanup_tx:
4355 profile->cleanup_tx(priv);
5c50368f 4356
26e59d80
MHY		 4357out:
4358 return err;
f62b8bb8
AV		 4359}
4360
2c3b5bee 4361void mlx5e_detach_netdev(struct mlx5e_priv *priv)
26e59d80 4362{
26e59d80
MHY		 4363 const struct mlx5e_profile *profile = priv->profile;
4364
4365 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
26e59d80 4366
37f304d1
SM		 4367 if (profile->disable)
4368 profile->disable(priv);
4369 flush_workqueue(priv->wq);
4370
26e59d80
MHY		 4371 mlx5e_destroy_q_counter(priv);
4372 profile->cleanup_rx(priv);
a43b25da 4373 mlx5e_close_drop_rq(&priv->drop_rq);
26e59d80 4374 profile->cleanup_tx(priv);
26e59d80
MHY		 4375 cancel_delayed_work_sync(&priv->update_stats_work);
4376}
4377
2c3b5bee
SM		 4378void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
4379{
4380 const struct mlx5e_profile *profile = priv->profile;
4381 struct net_device *netdev = priv->netdev;
4382
4383 destroy_workqueue(priv->wq);
4384 if (profile->cleanup)
4385 profile->cleanup(priv);
4386 free_netdev(netdev);
4387}
4388
26e59d80
MHY		 4389/* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
4390 * hardware contexts and to connect it to the current netdev.
4391 */
4392static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
4393{
4394 struct mlx5e_priv *priv = vpriv;
4395 struct net_device *netdev = priv->netdev;
4396 int err;
4397
4398 if (netif_device_present(netdev))
4399 return 0;
4400
4401 err = mlx5e_create_mdev_resources(mdev);
4402 if (err)
4403 return err;
4404
2c3b5bee 4405 err = mlx5e_attach_netdev(priv);
26e59d80
MHY		 4406 if (err) {
4407 mlx5e_destroy_mdev_resources(mdev);
4408 return err;
4409 }
4410
4411 return 0;
4412}
4413
4414static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
4415{
4416 struct mlx5e_priv *priv = vpriv;
4417 struct net_device *netdev = priv->netdev;
4418
4419 if (!netif_device_present(netdev))
4420 return;
4421
2c3b5bee 4422 mlx5e_detach_netdev(priv);
26e59d80
MHY		 4423 mlx5e_destroy_mdev_resources(mdev);
4424}
4425
b50d292b
HHZ		 4426static void *mlx5e_add(struct mlx5_core_dev *mdev)
4427{
127ea380 4428 struct mlx5_eswitch *esw = mdev->priv.eswitch;
26e59d80 4429 int total_vfs = MLX5_TOTAL_VPORTS(mdev);
1d447a39 4430 struct mlx5e_rep_priv *rpriv = NULL;
26e59d80
MHY		 4431 void *priv;
4432 int vport;
4433 int err;
4434 struct net_device *netdev;
b50d292b 4435
26e59d80
MHY		 4436 err = mlx5e_check_required_hca_cap(mdev);
4437 if (err)
b50d292b
HHZ		 4438 return NULL;
4439
1d447a39
SM		 4440 if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
4441 rpriv = kzalloc(sizeof(*rpriv), GFP_KERNEL);
4442 if (!rpriv) {
4443 mlx5_core_warn(mdev,
4444 "Not creating net device, Failed to alloc rep priv data\n");
4445 return NULL;
4446 }
4447 rpriv->rep = &esw->offloads.vport_reps[0];
4448 }
127ea380 4449
1d447a39 4450 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, rpriv);
26e59d80
MHY		 4451 if (!netdev) {
4452 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
4453 goto err_unregister_reps;
4454 }
4455
4456 priv = netdev_priv(netdev);
4457
4458 err = mlx5e_attach(mdev, priv);
4459 if (err) {
4460 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
4461 goto err_destroy_netdev;
4462 }
4463
4464 err = register_netdev(netdev);
4465 if (err) {
4466 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
4467 goto err_detach;
b50d292b 4468 }
26e59d80
MHY		 4469
4470 return priv;
4471
4472err_detach:
4473 mlx5e_detach(mdev, priv);
4474
4475err_destroy_netdev:
2c3b5bee 4476 mlx5e_destroy_netdev(priv);
26e59d80
MHY		 4477
4478err_unregister_reps:
4479 for (vport = 1; vport < total_vfs; vport++)
4480 mlx5_eswitch_unregister_vport_rep(esw, vport);
4481
1d447a39 4482 kfree(rpriv);
26e59d80 4483 return NULL;
b50d292b
HHZ		 4484}
4485
b50d292b
HHZ		 4486static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
4487{
4488 struct mlx5e_priv *priv = vpriv;
1d447a39 4489 void *ppriv = priv->ppriv;
127ea380 4490
5e1e93c7 4491 unregister_netdev(priv->netdev);
26e59d80 4492 mlx5e_detach(mdev, vpriv);
2c3b5bee 4493 mlx5e_destroy_netdev(priv);
1d447a39 4494 kfree(ppriv);
b50d292b
HHZ		 4495}
4496
f62b8bb8
AV		 4497static void *mlx5e_get_netdev(void *vpriv)
4498{
4499 struct mlx5e_priv *priv = vpriv;
4500
4501 return priv->netdev;
4502}
4503
4504static struct mlx5_interface mlx5e_interface = {
b50d292b
HHZ		 4505 .add = mlx5e_add,
4506 .remove = mlx5e_remove,
26e59d80
MHY		 4507 .attach = mlx5e_attach,
4508 .detach = mlx5e_detach,
f62b8bb8
AV		 4509 .event = mlx5e_async_event,
4510 .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
4511 .get_dev = mlx5e_get_netdev,
4512};
4513
4514void mlx5e_init(void)
4515{
2ac9cfe7 4516 mlx5e_ipsec_build_inverse_table();
665bc539 4517 mlx5e_build_ptys2ethtool_map();
f62b8bb8
AV		 4518 mlx5_register_interface(&mlx5e_interface);
4519}
4520
4521void mlx5e_cleanup(void)
4522{
4523 mlx5_unregister_interface(&mlx5e_interface);
4524}
Ubuntu Jammy Linux kernel mirror