]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/net/ethernet/pensando/ionic/ionic_lif.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ionic: add SKBTX_IN_PROGRESS
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / pensando / ionic / ionic_lif.c
CommitLineData
1a58e196
SN		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
cc69837f 4#include <linux/ethtool.h>
011c7289
AB		 5#include <linux/printk.h>
6#include <linux/dynamic_debug.h>
1a58e196
SN		 7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>
4b03b273 9#include <linux/if_vlan.h>
8c15440b 10#include <linux/rtnetlink.h>
1a58e196
SN		 11#include <linux/interrupt.h>
12#include <linux/pci.h>
13#include <linux/cpumask.h>
14
15#include "ionic.h"
16#include "ionic_bus.h"
17#include "ionic_lif.h"
0f3154e6 18#include "ionic_txrx.h"
4d03e00a 19#include "ionic_ethtool.h"
1a58e196
SN		 20#include "ionic_debugfs.h"
21
5b3f3f2a
SN		 22/* queuetype support level */
23static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
24 [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
25 [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
26 [IONIC_QTYPE_RXQ] = 0, /* 0 = Base version with CQ+SG support */
27 [IONIC_QTYPE_TXQ] = 1, /* 0 = Base version with CQ+SG support
28 * 1 = ... with Tx SG version 1
29 */
30};
31
2a654540
SN		 32static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode);
33static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
34static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
8d61aad4 35static void ionic_link_status_check(struct ionic_lif *lif);
c672412f
SN		 36static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
37static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
38static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
2a654540 39
f053e1f8
SN		 40static void ionic_txrx_deinit(struct ionic_lif *lif);
41static int ionic_txrx_init(struct ionic_lif *lif);
49d3b493
SN		 42static int ionic_start_queues(struct ionic_lif *lif);
43static void ionic_stop_queues(struct ionic_lif *lif);
5b3f3f2a 44static void ionic_lif_queue_identify(struct ionic_lif *lif);
49d3b493 45
04a83459
SN		 46static void ionic_dim_work(struct work_struct *work)
47{
48 struct dim *dim = container_of(work, struct dim, work);
49 struct dim_cq_moder cur_moder;
50 struct ionic_qcq *qcq;
51 u32 new_coal;
52
53 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
54 qcq = container_of(dim, struct ionic_qcq, dim);
55 new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
56 qcq->intr.dim_coal_hw = new_coal ? new_coal : 1;
57 dim->state = DIM_START_MEASURE;
58}
59
2a654540
SN		 60static void ionic_lif_deferred_work(struct work_struct *work)
61{
62 struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
63 struct ionic_deferred *def = &lif->deferred;
64 struct ionic_deferred_work *w = NULL;
65
52733cff
SN		 66 do {
67 spin_lock_bh(&def->lock);
68 if (!list_empty(&def->list)) {
69 w = list_first_entry(&def->list,
70 struct ionic_deferred_work, list);
71 list_del(&w->list);
72 }
73 spin_unlock_bh(&def->lock);
74
75 if (!w)
76 break;
2a654540 77
2a654540
SN		 78 switch (w->type) {
79 case IONIC_DW_TYPE_RX_MODE:
80 ionic_lif_rx_mode(lif, w->rx_mode);
81 break;
82 case IONIC_DW_TYPE_RX_ADDR_ADD:
83 ionic_lif_addr_add(lif, w->addr);
84 break;
85 case IONIC_DW_TYPE_RX_ADDR_DEL:
86 ionic_lif_addr_del(lif, w->addr);
87 break;
8d61aad4
SN		 88 case IONIC_DW_TYPE_LINK_STATUS:
89 ionic_link_status_check(lif);
90 break;
c672412f
SN		 91 case IONIC_DW_TYPE_LIF_RESET:
92 if (w->fw_status)
93 ionic_lif_handle_fw_up(lif);
94 else
95 ionic_lif_handle_fw_down(lif);
96 break;
2a654540
SN		 97 default:
98 break;
99 }
100 kfree(w);
52733cff
SN		 101 w = NULL;
102 } while (true);
2a654540
SN		 103}
104
c672412f
SN		 105void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
106 struct ionic_deferred_work *work)
2a654540
SN		 107{
108 spin_lock_bh(&def->lock);
109 list_add_tail(&work->list, &def->list);
110 spin_unlock_bh(&def->lock);
111 schedule_work(&def->work);
112}
113
8d61aad4
SN		 114static void ionic_link_status_check(struct ionic_lif *lif)
115{
116 struct net_device *netdev = lif->netdev;
117 u16 link_status;
118 bool link_up;
119
0925e9db 120 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
49d3b493
SN		 121 return;
122
9e8eaf84
SN		 123 /* Don't put carrier back up if we're in a broken state */
124 if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
125 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
126 return;
127 }
128
8d61aad4
SN		 129 link_status = le16_to_cpu(lif->info->status.link_status);
130 link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
131
8d61aad4 132 if (link_up) {
9e8eaf84
SN		 133 int err = 0;
134
25cc5a5f 135 if (netdev->flags & IFF_UP && netif_running(netdev)) {
8f56bc4d 136 mutex_lock(&lif->queue_lock);
9e8eaf84
SN		 137 err = ionic_start_queues(lif);
138 if (err) {
139 netdev_err(lif->netdev,
140 "Failed to start queues: %d\n", err);
141 set_bit(IONIC_LIF_F_BROKEN, lif->state);
142 netif_carrier_off(lif->netdev);
143 }
8f56bc4d
SN		 144 mutex_unlock(&lif->queue_lock);
145 }
146
9e8eaf84 147 if (!err && !netif_carrier_ok(netdev)) {
aa47b540 148 ionic_port_identify(lif->ionic);
aa47b540 149 netdev_info(netdev, "Link up - %d Gbps\n",
25cc5a5f 150 le32_to_cpu(lif->info->status.link_speed) / 1000);
0f3154e6
SN		 151 netif_carrier_on(netdev);
152 }
8d61aad4 153 } else {
aa47b540
SN		 154 if (netif_carrier_ok(netdev)) {
155 netdev_info(netdev, "Link down\n");
156 netif_carrier_off(netdev);
157 }
8d61aad4 158
25cc5a5f 159 if (netdev->flags & IFF_UP && netif_running(netdev)) {
0925e9db 160 mutex_lock(&lif->queue_lock);
49d3b493 161 ionic_stop_queues(lif);
0925e9db
SN		 162 mutex_unlock(&lif->queue_lock);
163 }
8d61aad4
SN		 164 }
165
c6d3d73a 166 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
8d61aad4
SN		 167}
168
1800eee1 169void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
8d61aad4
SN		 170{
171 struct ionic_deferred_work *work;
172
173 /* we only need one request outstanding at a time */
c6d3d73a 174 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
8d61aad4
SN		 175 return;
176
1800eee1 177 if (!can_sleep) {
8d61aad4 178 work = kzalloc(sizeof(*work), GFP_ATOMIC);
2c580d77
SN		 179 if (!work) {
180 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
8d61aad4 181 return;
2c580d77 182 }
8d61aad4
SN		 183
184 work->type = IONIC_DW_TYPE_LINK_STATUS;
185 ionic_lif_deferred_enqueue(&lif->deferred, work);
186 } else {
187 ionic_link_status_check(lif);
188 }
189}
190
1d062b7b
SN		 191static irqreturn_t ionic_isr(int irq, void *data)
192{
193 struct napi_struct *napi = data;
194
195 napi_schedule_irqoff(napi);
196
197 return IRQ_HANDLED;
198}
199
200static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
201{
202 struct ionic_intr_info *intr = &qcq->intr;
203 struct device *dev = lif->ionic->dev;
204 struct ionic_queue *q = &qcq->q;
205 const char *name;
206
207 if (lif->registered)
208 name = lif->netdev->name;
209 else
210 name = dev_name(dev);
211
212 snprintf(intr->name, sizeof(intr->name),
213 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
214
215 return devm_request_irq(dev, intr->vector, ionic_isr,
216 0, intr->name, &qcq->napi);
217}
218
219static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
220{
221 struct ionic *ionic = lif->ionic;
222 int index;
223
224 index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
225 if (index == ionic->nintrs) {
226 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
227 __func__, index, ionic->nintrs);
228 return -ENOSPC;
229 }
230
231 set_bit(index, ionic->intrs);
232 ionic_intr_init(&ionic->idev, intr, index);
233
234 return 0;
235}
236
36ac2c50 237static void ionic_intr_free(struct ionic *ionic, int index)
1d062b7b 238{
c06107ca 239 if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
36ac2c50 240 clear_bit(index, ionic->intrs);
1d062b7b
SN		 241}
242
0f3154e6
SN		 243static int ionic_qcq_enable(struct ionic_qcq *qcq)
244{
245 struct ionic_queue *q = &qcq->q;
246 struct ionic_lif *lif = q->lif;
247 struct ionic_dev *idev;
248 struct device *dev;
249
250 struct ionic_admin_ctx ctx = {
251 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
252 .cmd.q_control = {
253 .opcode = IONIC_CMD_Q_CONTROL,
254 .lif_index = cpu_to_le16(lif->index),
255 .type = q->type,
256 .index = cpu_to_le32(q->index),
257 .oper = IONIC_Q_ENABLE,
258 },
259 };
260
261 idev = &lif->ionic->idev;
262 dev = lif->ionic->dev;
263
264 dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
265 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
266
267 if (qcq->flags & IONIC_QCQ_F_INTR) {
268 irq_set_affinity_hint(qcq->intr.vector,
269 &qcq->intr.affinity_mask);
270 napi_enable(&qcq->napi);
271 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
272 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
273 IONIC_INTR_MASK_CLEAR);
274 }
275
276 return ionic_adminq_post_wait(lif, &ctx);
277}
278
ba6ab8ac 279static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
0f3154e6 280{
7c737fc4
SN		 281 struct ionic_queue *q;
282 struct ionic_lif *lif;
ba6ab8ac 283 int err = 0;
0f3154e6
SN		 284
285 struct ionic_admin_ctx ctx = {
286 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
287 .cmd.q_control = {
288 .opcode = IONIC_CMD_Q_CONTROL,
0f3154e6
SN		 289 .oper = IONIC_Q_DISABLE,
290 },
291 };
292
7c737fc4
SN		 293 if (!qcq)
294 return -ENXIO;
0f3154e6 295
7c737fc4
SN		 296 q = &qcq->q;
297 lif = q->lif;
0f3154e6
SN		 298
299 if (qcq->flags & IONIC_QCQ_F_INTR) {
7c737fc4
SN		 300 struct ionic_dev *idev = &lif->ionic->idev;
301
04a83459 302 cancel_work_sync(&qcq->dim.work);
0f3154e6
SN		 303 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
304 IONIC_INTR_MASK_SET);
305 synchronize_irq(qcq->intr.vector);
306 irq_set_affinity_hint(qcq->intr.vector, NULL);
307 napi_disable(&qcq->napi);
308 }
309
ba6ab8ac
SN		 310 if (send_to_hw) {
311 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
312 ctx.cmd.q_control.type = q->type;
313 ctx.cmd.q_control.index = cpu_to_le32(q->index);
314 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
315 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
7c737fc4 316
ba6ab8ac
SN		 317 err = ionic_adminq_post_wait(lif, &ctx);
318 }
319
320 return err;
0f3154e6
SN		 321}
322
1d062b7b
SN		 323static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
324{
325 struct ionic_dev *idev = &lif->ionic->idev;
1d062b7b
SN		 326
327 if (!qcq)
328 return;
329
1d062b7b
SN		 330 if (!(qcq->flags & IONIC_QCQ_F_INITED))
331 return;
332
333 if (qcq->flags & IONIC_QCQ_F_INTR) {
334 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
335 IONIC_INTR_MASK_SET);
1d062b7b
SN		 336 netif_napi_del(&qcq->napi);
337 }
338
339 qcq->flags &= ~IONIC_QCQ_F_INITED;
340}
341
101b40a0
SN		 342static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
343{
344 if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
345 return;
346
347 irq_set_affinity_hint(qcq->intr.vector, NULL);
348 devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
349 qcq->intr.vector = 0;
350 ionic_intr_free(lif->ionic, qcq->intr.index);
351 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
352}
353
1d062b7b
SN		 354static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
355{
356 struct device *dev = lif->ionic->dev;
357
358 if (!qcq)
359 return;
360
2a8c2c1a
SN		 361 ionic_debugfs_del_qcq(qcq);
362
ea5a8b09
SN		 363 if (qcq->q_base) {
364 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
365 qcq->q_base = NULL;
366 qcq->q_base_pa = 0;
367 }
368
369 if (qcq->cq_base) {
370 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
371 qcq->cq_base = NULL;
372 qcq->cq_base_pa = 0;
373 }
374
375 if (qcq->sg_base) {
376 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
377 qcq->sg_base = NULL;
378 qcq->sg_base_pa = 0;
379 }
1d062b7b 380
101b40a0 381 ionic_qcq_intr_free(lif, qcq);
1d062b7b 382
a34e25ab
SN		 383 if (qcq->cq.info) {
384 devm_kfree(dev, qcq->cq.info);
385 qcq->cq.info = NULL;
386 }
387 if (qcq->q.info) {
388 devm_kfree(dev, qcq->q.info);
389 qcq->q.info = NULL;
390 }
1d062b7b
SN		 391}
392
393static void ionic_qcqs_free(struct ionic_lif *lif)
394{
0f3154e6 395 struct device *dev = lif->ionic->dev;
e768929d
SN		 396 struct ionic_qcq *adminqcq;
397 unsigned long irqflags;
0f3154e6 398
77ceb68e
SN		 399 if (lif->notifyqcq) {
400 ionic_qcq_free(lif, lif->notifyqcq);
101b40a0 401 devm_kfree(dev, lif->notifyqcq);
77ceb68e
SN		 402 lif->notifyqcq = NULL;
403 }
404
1d062b7b 405 if (lif->adminqcq) {
e768929d
SN		 406 spin_lock_irqsave(&lif->adminq_lock, irqflags);
407 adminqcq = READ_ONCE(lif->adminqcq);
1d062b7b 408 lif->adminqcq = NULL;
e768929d
SN		 409 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
410 if (adminqcq) {
411 ionic_qcq_free(lif, adminqcq);
412 devm_kfree(dev, adminqcq);
413 }
1d062b7b 414 }
0f3154e6 415
a4674f34 416 if (lif->rxqcqs) {
34dec947
SN		 417 devm_kfree(dev, lif->rxqstats);
418 lif->rxqstats = NULL;
a4674f34
SN		 419 devm_kfree(dev, lif->rxqcqs);
420 lif->rxqcqs = NULL;
421 }
0f3154e6 422
a4674f34 423 if (lif->txqcqs) {
34dec947
SN		 424 devm_kfree(dev, lif->txqstats);
425 lif->txqstats = NULL;
a4674f34
SN		 426 devm_kfree(dev, lif->txqcqs);
427 lif->txqcqs = NULL;
428 }
1d062b7b
SN		 429}
430
77ceb68e
SN		 431static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
432 struct ionic_qcq *n_qcq)
433{
434 if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
36ac2c50 435 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
77ceb68e
SN		 436 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
437 }
438
439 n_qcq->intr.vector = src_qcq->intr.vector;
440 n_qcq->intr.index = src_qcq->intr.index;
441}
442
101b40a0
SN		 443static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
444{
445 int err;
446
447 if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
448 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
449 return 0;
450 }
451
452 err = ionic_intr_alloc(lif, &qcq->intr);
453 if (err) {
454 netdev_warn(lif->netdev, "no intr for %s: %d\n",
455 qcq->q.name, err);
456 goto err_out;
457 }
458
459 err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
460 if (err < 0) {
461 netdev_warn(lif->netdev, "no vector for %s: %d\n",
462 qcq->q.name, err);
463 goto err_out_free_intr;
464 }
465 qcq->intr.vector = err;
466 ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
467 IONIC_INTR_MASK_SET);
468
469 err = ionic_request_irq(lif, qcq);
470 if (err) {
471 netdev_warn(lif->netdev, "irq request failed %d\n", err);
472 goto err_out_free_intr;
473 }
474
475 /* try to get the irq on the local numa node first */
476 qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
477 dev_to_node(lif->ionic->dev));
478 if (qcq->intr.cpu != -1)
479 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
480
481 netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
482 return 0;
483
484err_out_free_intr:
485 ionic_intr_free(lif->ionic, qcq->intr.index);
486err_out:
487 return err;
488}
489
1d062b7b
SN		 490static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
491 unsigned int index,
492 const char *name, unsigned int flags,
493 unsigned int num_descs, unsigned int desc_size,
494 unsigned int cq_desc_size,
495 unsigned int sg_desc_size,
496 unsigned int pid, struct ionic_qcq **qcq)
497{
498 struct ionic_dev *idev = &lif->ionic->idev;
1d062b7b
SN		 499 struct device *dev = lif->ionic->dev;
500 void *q_base, *cq_base, *sg_base;
501 dma_addr_t cq_base_pa = 0;
502 dma_addr_t sg_base_pa = 0;
503 dma_addr_t q_base_pa = 0;
504 struct ionic_qcq *new;
505 int err;
506
507 *qcq = NULL;
508
1d062b7b
SN		 509 new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
510 if (!new) {
511 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
512 err = -ENOMEM;
513 goto err_out;
514 }
515
f37bc346 516 new->q.dev = dev;
1d062b7b
SN		 517 new->flags = flags;
518
e7164200 519 new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
1d062b7b
SN		 520 GFP_KERNEL);
521 if (!new->q.info) {
522 netdev_err(lif->netdev, "Cannot allocate queue info\n");
523 err = -ENOMEM;
ea5a8b09 524 goto err_out_free_qcq;
1d062b7b
SN		 525 }
526
527 new->q.type = type;
f37bc346 528 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
1d062b7b
SN		 529
530 err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
531 desc_size, sg_desc_size, pid);
532 if (err) {
533 netdev_err(lif->netdev, "Cannot initialize queue\n");
ea5a8b09 534 goto err_out_free_q_info;
1d062b7b
SN		 535 }
536
101b40a0
SN		 537 err = ionic_alloc_qcq_interrupt(lif, new);
538 if (err)
539 goto err_out;
1d062b7b 540
e7164200 541 new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
1d062b7b
SN		 542 GFP_KERNEL);
543 if (!new->cq.info) {
544 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
545 err = -ENOMEM;
0b064100 546 goto err_out_free_irq;
1d062b7b
SN		 547 }
548
549 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
550 if (err) {
551 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
ea5a8b09 552 goto err_out_free_cq_info;
1d062b7b
SN		 553 }
554
9576a36c
SN		 555 if (flags & IONIC_QCQ_F_NOTIFYQ) {
556 int q_size, cq_size;
557
558 /* q & cq need to be contiguous in case of notifyq */
559 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
560 cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
561
562 new->q_size = PAGE_SIZE + q_size + cq_size;
563 new->q_base = dma_alloc_coherent(dev, new->q_size,
564 &new->q_base_pa, GFP_KERNEL);
565 if (!new->q_base) {
566 netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
567 err = -ENOMEM;
568 goto err_out_free_cq_info;
569 }
570 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
571 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
572 ionic_q_map(&new->q, q_base, q_base_pa);
573
574 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
575 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
576 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
577 ionic_cq_bind(&new->cq, &new->q);
578 } else {
579 new->q_size = PAGE_SIZE + (num_descs * desc_size);
580 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
581 GFP_KERNEL);
582 if (!new->q_base) {
583 netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
584 err = -ENOMEM;
585 goto err_out_free_cq_info;
586 }
587 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
588 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
589 ionic_q_map(&new->q, q_base, q_base_pa);
590
591 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
592 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
593 GFP_KERNEL);
594 if (!new->cq_base) {
595 netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
596 err = -ENOMEM;
597 goto err_out_free_q;
598 }
599 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
600 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
601 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
602 ionic_cq_bind(&new->cq, &new->q);
ea5a8b09 603 }
1d062b7b
SN		 604
605 if (flags & IONIC_QCQ_F_SG) {
ea5a8b09
SN		 606 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
607 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
608 GFP_KERNEL);
609 if (!new->sg_base) {
610 netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
611 err = -ENOMEM;
612 goto err_out_free_cq;
613 }
614 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
615 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
1d062b7b
SN		 616 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
617 }
618
04a83459
SN		 619 INIT_WORK(&new->dim.work, ionic_dim_work);
620 new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
621
1d062b7b
SN		 622 *qcq = new;
623
624 return 0;
625
ea5a8b09
SN		 626err_out_free_cq:
627 dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
628err_out_free_q:
629 dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
630err_out_free_cq_info:
631 devm_kfree(dev, new->cq.info);
0b064100 632err_out_free_irq:
101b40a0 633 if (flags & IONIC_QCQ_F_INTR) {
0b064100 634 devm_free_irq(dev, new->intr.vector, &new->napi);
36ac2c50 635 ionic_intr_free(lif->ionic, new->intr.index);
101b40a0 636 }
ea5a8b09
SN		 637err_out_free_q_info:
638 devm_kfree(dev, new->q.info);
639err_out_free_qcq:
640 devm_kfree(dev, new);
1d062b7b
SN		 641err_out:
642 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
643 return err;
644}
645
646static int ionic_qcqs_alloc(struct ionic_lif *lif)
647{
0f3154e6 648 struct device *dev = lif->ionic->dev;
1d062b7b
SN		 649 unsigned int flags;
650 int err;
651
652 flags = IONIC_QCQ_F_INTR;
653 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
654 IONIC_ADMINQ_LENGTH,
655 sizeof(struct ionic_admin_cmd),
656 sizeof(struct ionic_admin_comp),
657 0, lif->kern_pid, &lif->adminqcq);
658 if (err)
659 return err;
2a8c2c1a 660 ionic_debugfs_add_qcq(lif, lif->adminqcq);
1d062b7b 661
77ceb68e
SN		 662 if (lif->ionic->nnqs_per_lif) {
663 flags = IONIC_QCQ_F_NOTIFYQ;
664 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
665 flags, IONIC_NOTIFYQ_LENGTH,
666 sizeof(struct ionic_notifyq_cmd),
667 sizeof(union ionic_notifyq_comp),
668 0, lif->kern_pid, &lif->notifyqcq);
669 if (err)
34dec947 670 goto err_out;
2a8c2c1a 671 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
77ceb68e
SN		 672
673 /* Let the notifyq ride on the adminq interrupt */
674 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
675 }
676
0f3154e6 677 err = -ENOMEM;
ee205626 678 lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
230efff4 679 sizeof(*lif->txqcqs), GFP_KERNEL);
0f3154e6 680 if (!lif->txqcqs)
34dec947 681 goto err_out;
ee205626 682 lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
230efff4 683 sizeof(*lif->rxqcqs), GFP_KERNEL);
0f3154e6 684 if (!lif->rxqcqs)
34dec947 685 goto err_out;
0f3154e6 686
f0790bcd 687 lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
230efff4 688 sizeof(*lif->txqstats), GFP_KERNEL);
34dec947
SN		 689 if (!lif->txqstats)
690 goto err_out;
f0790bcd 691 lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
230efff4 692 sizeof(*lif->rxqstats), GFP_KERNEL);
34dec947
SN		 693 if (!lif->rxqstats)
694 goto err_out;
77ceb68e 695
34dec947 696 return 0;
77ceb68e 697
34dec947
SN		 698err_out:
699 ionic_qcqs_free(lif);
77ceb68e
SN		 700 return err;
701}
702
f053e1f8
SN		 703static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
704{
705 qcq->q.tail_idx = 0;
706 qcq->q.head_idx = 0;
707 qcq->cq.tail_idx = 0;
708 qcq->cq.done_color = 1;
709 memset(qcq->q_base, 0, qcq->q_size);
710 memset(qcq->cq_base, 0, qcq->cq_size);
711 memset(qcq->sg_base, 0, qcq->sg_size);
712}
713
0f3154e6
SN		 714static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
715{
716 struct device *dev = lif->ionic->dev;
717 struct ionic_queue *q = &qcq->q;
718 struct ionic_cq *cq = &qcq->cq;
719 struct ionic_admin_ctx ctx = {
720 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
721 .cmd.q_init = {
722 .opcode = IONIC_CMD_Q_INIT,
723 .lif_index = cpu_to_le16(lif->index),
724 .type = q->type,
5b3f3f2a 725 .ver = lif->qtype_info[q->type].version,
0f3154e6
SN		 726 .index = cpu_to_le32(q->index),
727 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
728 IONIC_QINIT_F_SG),
0f3154e6
SN		 729 .pid = cpu_to_le16(q->pid),
730 .ring_size = ilog2(q->num_descs),
731 .ring_base = cpu_to_le64(q->base_pa),
732 .cq_ring_base = cpu_to_le64(cq->base_pa),
733 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
57a3a98d 734 .features = cpu_to_le64(q->features),
0f3154e6
SN		 735 },
736 };
fe8c30b5 737 unsigned int intr_index;
0f3154e6
SN		 738 int err;
739
2103ed2f
SN		 740 intr_index = qcq->intr.index;
741
fe8c30b5
SN		 742 ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
743
0f3154e6
SN		 744 dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
745 dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
746 dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
747 dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
5b3f3f2a
SN		 748 dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
749 dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
fe8c30b5 750 dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
0f3154e6 751
f053e1f8 752 ionic_qcq_sanitize(qcq);
49d3b493 753
0f3154e6
SN		 754 err = ionic_adminq_post_wait(lif, &ctx);
755 if (err)
756 return err;
757
758 q->hw_type = ctx.comp.q_init.hw_type;
759 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
760 q->dbval = IONIC_DBELL_QID(q->hw_index);
761
762 dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
763 dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
764
fe8c30b5
SN		 765 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
766 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
767 NAPI_POLL_WEIGHT);
768
0f3154e6
SN		 769 qcq->flags |= IONIC_QCQ_F_INITED;
770
0f3154e6
SN		 771 return 0;
772}
773
774static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
775{
776 struct device *dev = lif->ionic->dev;
777 struct ionic_queue *q = &qcq->q;
778 struct ionic_cq *cq = &qcq->cq;
779 struct ionic_admin_ctx ctx = {
780 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
781 .cmd.q_init = {
782 .opcode = IONIC_CMD_Q_INIT,
783 .lif_index = cpu_to_le16(lif->index),
784 .type = q->type,
5b3f3f2a 785 .ver = lif->qtype_info[q->type].version,
0f3154e6 786 .index = cpu_to_le32(q->index),
08f2e4b2
SN		 787 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
788 IONIC_QINIT_F_SG),
0f3154e6
SN		 789 .intr_index = cpu_to_le16(cq->bound_intr->index),
790 .pid = cpu_to_le16(q->pid),
791 .ring_size = ilog2(q->num_descs),
792 .ring_base = cpu_to_le64(q->base_pa),
793 .cq_ring_base = cpu_to_le64(cq->base_pa),
08f2e4b2 794 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
57a3a98d 795 .features = cpu_to_le64(q->features),
0f3154e6
SN		 796 },
797 };
798 int err;
799
800 dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
801 dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
802 dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
803 dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
5b3f3f2a
SN		 804 dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
805 dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
fe8c30b5 806 dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
0f3154e6 807
f053e1f8 808 ionic_qcq_sanitize(qcq);
49d3b493 809
0f3154e6
SN		 810 err = ionic_adminq_post_wait(lif, &ctx);
811 if (err)
812 return err;
813
814 q->hw_type = ctx.comp.q_init.hw_type;
815 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
816 q->dbval = IONIC_DBELL_QID(q->hw_index);
817
818 dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
819 dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
820
fe8c30b5
SN		 821 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
822 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
823 NAPI_POLL_WEIGHT);
824 else
825 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
826 NAPI_POLL_WEIGHT);
0f3154e6 827
0f3154e6
SN		 828 qcq->flags |= IONIC_QCQ_F_INITED;
829
0f3154e6
SN		 830 return 0;
831}
832
61db421d
SN		 833int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
834{
f0790bcd
SN		 835 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
836 unsigned int txq_i, flags;
837 struct ionic_qcq *txq;
838 u64 features;
839 int err;
840
841 mutex_lock(&lif->queue_lock);
842
843 if (lif->hwstamp_txq)
844 goto out;
845
846 features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
847
848 num_desc = IONIC_MIN_TXRX_DESC;
849 desc_sz = sizeof(struct ionic_txq_desc);
850 comp_sz = 2 * sizeof(struct ionic_txq_comp);
851
852 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
853 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
854 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
855 else
856 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
857
858 txq_i = lif->ionic->ntxqs_per_lif;
859 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
860
861 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
862 num_desc, desc_sz, comp_sz, sg_desc_sz,
863 lif->kern_pid, &txq);
864 if (err)
865 goto err_qcq_alloc;
866
867 txq->q.features = features;
868
869 ionic_link_qcq_interrupts(lif->adminqcq, txq);
870 ionic_debugfs_add_qcq(lif, txq);
871
872 lif->hwstamp_txq = txq;
873
874 if (netif_running(lif->netdev)) {
875 err = ionic_lif_txq_init(lif, txq);
876 if (err)
877 goto err_qcq_init;
878
879 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
880 err = ionic_qcq_enable(txq);
881 if (err)
882 goto err_qcq_enable;
883 }
884 }
885
886out:
887 mutex_unlock(&lif->queue_lock);
888
61db421d 889 return 0;
f0790bcd
SN		 890
891err_qcq_enable:
892 ionic_lif_qcq_deinit(lif, txq);
893err_qcq_init:
894 lif->hwstamp_txq = NULL;
895 ionic_debugfs_del_qcq(txq);
896 ionic_qcq_free(lif, txq);
897 devm_kfree(lif->ionic->dev, txq);
898err_qcq_alloc:
899 mutex_unlock(&lif->queue_lock);
900 return err;
61db421d
SN		 901}
902
903int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
904{
f0790bcd
SN		 905 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
906 unsigned int rxq_i, flags;
907 struct ionic_qcq *rxq;
908 u64 features;
909 int err;
910
911 mutex_lock(&lif->queue_lock);
912
913 if (lif->hwstamp_rxq)
914 goto out;
915
916 features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
917
918 num_desc = IONIC_MIN_TXRX_DESC;
919 desc_sz = sizeof(struct ionic_rxq_desc);
920 comp_sz = 2 * sizeof(struct ionic_rxq_comp);
921 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
922
923 rxq_i = lif->ionic->nrxqs_per_lif;
924 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
925
926 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
927 num_desc, desc_sz, comp_sz, sg_desc_sz,
928 lif->kern_pid, &rxq);
929 if (err)
930 goto err_qcq_alloc;
931
932 rxq->q.features = features;
933
934 ionic_link_qcq_interrupts(lif->adminqcq, rxq);
935 ionic_debugfs_add_qcq(lif, rxq);
936
937 lif->hwstamp_rxq = rxq;
938
939 if (netif_running(lif->netdev)) {
940 err = ionic_lif_rxq_init(lif, rxq);
941 if (err)
942 goto err_qcq_init;
943
944 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
945 ionic_rx_fill(&rxq->q);
946 err = ionic_qcq_enable(rxq);
947 if (err)
948 goto err_qcq_enable;
949 }
950 }
951
952out:
953 mutex_unlock(&lif->queue_lock);
954
61db421d 955 return 0;
f0790bcd
SN		 956
957err_qcq_enable:
958 ionic_lif_qcq_deinit(lif, rxq);
959err_qcq_init:
960 lif->hwstamp_rxq = NULL;
961 ionic_debugfs_del_qcq(rxq);
962 ionic_qcq_free(lif, rxq);
963 devm_kfree(lif->ionic->dev, rxq);
964err_qcq_alloc:
965 mutex_unlock(&lif->queue_lock);
966 return err;
61db421d
SN		 967}
968
969int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
970{
f0790bcd
SN		 971 struct ionic_queue_params qparam;
972
973 ionic_init_queue_params(lif, &qparam);
974
975 if (rx_all)
976 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
977 else
978 qparam.rxq_features = 0;
979
980 /* if we're not running, just set the values and return */
981 if (!netif_running(lif->netdev)) {
982 lif->rxq_features = qparam.rxq_features;
983 return 0;
984 }
985
986 return ionic_reconfigure_queues(lif, &qparam);
61db421d
SN		 987}
988
989int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
990{
f0790bcd
SN		 991 struct ionic_admin_ctx ctx = {
992 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
993 .cmd.lif_setattr = {
994 .opcode = IONIC_CMD_LIF_SETATTR,
995 .index = cpu_to_le16(lif->index),
996 .attr = IONIC_LIF_ATTR_TXSTAMP,
997 .txstamp_mode = cpu_to_le16(txstamp_mode),
998 },
999 };
1000
1001 return ionic_adminq_post_wait(lif, &ctx);
1002}
1003
1004static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1005{
1006 struct ionic_admin_ctx ctx = {
1007 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1008 .cmd.rx_filter_del = {
1009 .opcode = IONIC_CMD_RX_FILTER_DEL,
1010 .lif_index = cpu_to_le16(lif->index),
1011 },
1012 };
1013 struct ionic_rx_filter *f;
1014 u32 filter_id;
1015 int err;
1016
1017 spin_lock_bh(&lif->rx_filters.lock);
1018
1019 f = ionic_rx_filter_rxsteer(lif);
1020 if (!f) {
1021 spin_unlock_bh(&lif->rx_filters.lock);
1022 return;
1023 }
1024
1025 filter_id = f->filter_id;
1026 ionic_rx_filter_free(lif, f);
1027
1028 spin_unlock_bh(&lif->rx_filters.lock);
1029
1030 netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1031
1032 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1033
1034 err = ionic_adminq_post_wait(lif, &ctx);
1035 if (err && err != -EEXIST)
1036 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1037}
1038
1039static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1040{
1041 struct ionic_admin_ctx ctx = {
1042 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1043 .cmd.rx_filter_add = {
1044 .opcode = IONIC_CMD_RX_FILTER_ADD,
1045 .lif_index = cpu_to_le16(lif->index),
1046 .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1047 .pkt_class = cpu_to_le64(pkt_class),
1048 },
1049 };
1050 u8 qtype;
1051 u32 qid;
1052 int err;
1053
1054 if (!lif->hwstamp_rxq)
1055 return -EINVAL;
1056
1057 qtype = lif->hwstamp_rxq->q.type;
1058 ctx.cmd.rx_filter_add.qtype = qtype;
1059
1060 qid = lif->hwstamp_rxq->q.index;
1061 ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1062
1063 netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1064 err = ionic_adminq_post_wait(lif, &ctx);
1065 if (err && err != -EEXIST)
1066 return err;
1067
1068 return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
61db421d
SN		 1069}
1070
1071int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1072{
f0790bcd
SN		 1073 ionic_lif_del_hwstamp_rxfilt(lif);
1074
1075 if (!pkt_class)
1076 return 0;
1077
1078 return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
61db421d
SN		 1079}
1080
77ceb68e
SN		 1081static bool ionic_notifyq_service(struct ionic_cq *cq,
1082 struct ionic_cq_info *cq_info)
1083{
1084 union ionic_notifyq_comp *comp = cq_info->cq_desc;
c672412f 1085 struct ionic_deferred_work *work;
77ceb68e
SN		 1086 struct net_device *netdev;
1087 struct ionic_queue *q;
1088 struct ionic_lif *lif;
1089 u64 eid;
1090
1091 q = cq->bound_q;
1092 lif = q->info[0].cb_arg;
1093 netdev = lif->netdev;
1094 eid = le64_to_cpu(comp->event.eid);
1095
1096 /* Have we run out of new completions to process? */
3fbc9bb6 1097 if ((s64)(eid - lif->last_eid) <= 0)
77ceb68e
SN		 1098 return false;
1099
1100 lif->last_eid = eid;
1101
1102 dev_dbg(lif->ionic->dev, "notifyq event:\n");
1103 dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1104 comp, sizeof(*comp), true);
1105
1106 switch (le16_to_cpu(comp->event.ecode)) {
1107 case IONIC_EVENT_LINK_CHANGE:
25cc5a5f 1108 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
77ceb68e
SN		 1109 break;
1110 case IONIC_EVENT_RESET:
c672412f
SN		 1111 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1112 if (!work) {
c0c682ee 1113 netdev_err(lif->netdev, "Reset event dropped\n");
c672412f
SN		 1114 } else {
1115 work->type = IONIC_DW_TYPE_LIF_RESET;
1116 ionic_lif_deferred_enqueue(&lif->deferred, work);
1117 }
77ceb68e
SN		 1118 break;
1119 default:
5b3f3f2a 1120 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
77ceb68e
SN		 1121 comp->event.ecode, eid);
1122 break;
1123 }
1124
1125 return true;
1126}
1127
1d062b7b
SN		 1128static bool ionic_adminq_service(struct ionic_cq *cq,
1129 struct ionic_cq_info *cq_info)
1130{
1131 struct ionic_admin_comp *comp = cq_info->cq_desc;
1132
1133 if (!color_match(comp->color, cq->done_color))
1134 return false;
1135
1136 ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1137
1138 return true;
1139}
1140
1141static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1142{
b4280948 1143 struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
77ceb68e 1144 struct ionic_lif *lif = napi_to_cq(napi)->lif;
b4280948 1145 struct ionic_dev *idev = &lif->ionic->idev;
e768929d 1146 unsigned long irqflags;
b4280948 1147 unsigned int flags = 0;
a8771bfe
SN		 1148 int rx_work = 0;
1149 int tx_work = 0;
77ceb68e
SN		 1150 int n_work = 0;
1151 int a_work = 0;
b4280948 1152 int work_done;
a8771bfe 1153 int credits;
b4280948
SN		 1154
1155 if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1156 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1157 ionic_notifyq_service, NULL, NULL);
77ceb68e 1158
e768929d 1159 spin_lock_irqsave(&lif->adminq_lock, irqflags);
b4280948
SN		 1160 if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1161 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1162 ionic_adminq_service, NULL, NULL);
e768929d 1163 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
b4280948 1164
a8771bfe
SN		 1165 if (lif->hwstamp_rxq)
1166 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1167 ionic_rx_service, NULL, NULL);
1168
1169 if (lif->hwstamp_txq)
1170 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1171 ionic_tx_service, NULL, NULL);
1172
1173 work_done = max(max(n_work, a_work), max(rx_work, tx_work));
b4280948
SN		 1174 if (work_done < budget && napi_complete_done(napi, work_done)) {
1175 flags |= IONIC_INTR_CRED_UNMASK;
9b761574 1176 intr->rearm_count++;
b4280948 1177 }
77ceb68e 1178
b4280948
SN		 1179 if (work_done || flags) {
1180 flags |= IONIC_INTR_CRED_RESET_COALESCE;
a8771bfe
SN		 1181 credits = n_work + a_work + rx_work + tx_work;
1182 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
b4280948
SN		 1183 }
1184
1185 return work_done;
1d062b7b
SN		 1186}
1187
f64e0c56
SN		 1188void ionic_get_stats64(struct net_device *netdev,
1189 struct rtnl_link_stats64 *ns)
8d61aad4
SN		 1190{
1191 struct ionic_lif *lif = netdev_priv(netdev);
1192 struct ionic_lif_stats *ls;
1193
1194 memset(ns, 0, sizeof(*ns));
1195 ls = &lif->info->stats;
1196
1197 ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1198 le64_to_cpu(ls->rx_mcast_packets) +
1199 le64_to_cpu(ls->rx_bcast_packets);
1200
1201 ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1202 le64_to_cpu(ls->tx_mcast_packets) +
1203 le64_to_cpu(ls->tx_bcast_packets);
1204
1205 ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1206 le64_to_cpu(ls->rx_mcast_bytes) +
1207 le64_to_cpu(ls->rx_bcast_bytes);
1208
1209 ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1210 le64_to_cpu(ls->tx_mcast_bytes) +
1211 le64_to_cpu(ls->tx_bcast_bytes);
1212
1213 ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1214 le64_to_cpu(ls->rx_mcast_drop_packets) +
1215 le64_to_cpu(ls->rx_bcast_drop_packets);
1216
1217 ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1218 le64_to_cpu(ls->tx_mcast_drop_packets) +
1219 le64_to_cpu(ls->tx_bcast_drop_packets);
1220
1221 ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1222
1223 ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1224
1225 ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1226 le64_to_cpu(ls->rx_queue_disabled) +
1227 le64_to_cpu(ls->rx_desc_fetch_error) +
1228 le64_to_cpu(ls->rx_desc_data_error);
1229
1230 ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1231 le64_to_cpu(ls->tx_queue_disabled) +
1232 le64_to_cpu(ls->tx_desc_fetch_error) +
1233 le64_to_cpu(ls->tx_desc_data_error);
1234
1235 ns->rx_errors = ns->rx_over_errors +
1236 ns->rx_missed_errors;
1237
1238 ns->tx_errors = ns->tx_aborted_errors;
1239}
1240
2a654540
SN		 1241static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1242{
1243 struct ionic_admin_ctx ctx = {
1244 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1245 .cmd.rx_filter_add = {
1246 .opcode = IONIC_CMD_RX_FILTER_ADD,
1247 .lif_index = cpu_to_le16(lif->index),
1248 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1249 },
1250 };
1251 struct ionic_rx_filter *f;
1252 int err;
1253
1254 /* don't bother if we already have it */
1255 spin_lock_bh(&lif->rx_filters.lock);
1256 f = ionic_rx_filter_by_addr(lif, addr);
1257 spin_unlock_bh(&lif->rx_filters.lock);
1258 if (f)
1259 return 0;
1260
cbec2153 1261 netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
2a654540
SN		 1262
1263 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1264 err = ionic_adminq_post_wait(lif, &ctx);
53faea3d 1265 if (err && err != -EEXIST)
2a654540
SN		 1266 return err;
1267
1268 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1269}
1270
1271static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1272{
1273 struct ionic_admin_ctx ctx = {
1274 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1275 .cmd.rx_filter_del = {
1276 .opcode = IONIC_CMD_RX_FILTER_DEL,
1277 .lif_index = cpu_to_le16(lif->index),
1278 },
1279 };
1280 struct ionic_rx_filter *f;
1281 int err;
1282
1283 spin_lock_bh(&lif->rx_filters.lock);
1284 f = ionic_rx_filter_by_addr(lif, addr);
1285 if (!f) {
1286 spin_unlock_bh(&lif->rx_filters.lock);
1287 return -ENOENT;
1288 }
1289
cbec2153
SN		 1290 netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1291 addr, f->filter_id);
1292
2a654540
SN		 1293 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1294 ionic_rx_filter_free(lif, f);
1295 spin_unlock_bh(&lif->rx_filters.lock);
1296
1297 err = ionic_adminq_post_wait(lif, &ctx);
53faea3d 1298 if (err && err != -EEXIST)
2a654540
SN		 1299 return err;
1300
2a654540
SN		 1301 return 0;
1302}
1303
1800eee1
SAS		 1304static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add,
1305 bool can_sleep)
2a654540 1306{
2a654540
SN		 1307 struct ionic_deferred_work *work;
1308 unsigned int nmfilters;
1309 unsigned int nufilters;
1310
1311 if (add) {
1312 /* Do we have space for this filter? We test the counters
1313 * here before checking the need for deferral so that we
1314 * can return an overflow error to the stack.
1315 */
bb9f80f3
SN		 1316 nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1317 nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
2a654540
SN		 1318
1319 if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1320 lif->nmcast++;
1321 else if (!is_multicast_ether_addr(addr) &&
1322 lif->nucast < nufilters)
1323 lif->nucast++;
1324 else
1325 return -ENOSPC;
1326 } else {
1327 if (is_multicast_ether_addr(addr) && lif->nmcast)
1328 lif->nmcast--;
1329 else if (!is_multicast_ether_addr(addr) && lif->nucast)
1330 lif->nucast--;
1331 }
1332
1800eee1 1333 if (!can_sleep) {
2a654540 1334 work = kzalloc(sizeof(*work), GFP_ATOMIC);
c0c682ee 1335 if (!work)
2a654540 1336 return -ENOMEM;
2a654540
SN		 1337 work->type = add ? IONIC_DW_TYPE_RX_ADDR_ADD :
1338 IONIC_DW_TYPE_RX_ADDR_DEL;
1339 memcpy(work->addr, addr, ETH_ALEN);
1340 netdev_dbg(lif->netdev, "deferred: rx_filter %s %pM\n",
1341 add ? "add" : "del", addr);
1342 ionic_lif_deferred_enqueue(&lif->deferred, work);
1343 } else {
1344 netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1345 add ? "add" : "del", addr);
1346 if (add)
1347 return ionic_lif_addr_add(lif, addr);
1348 else
1349 return ionic_lif_addr_del(lif, addr);
1350 }
1351
1352 return 0;
1353}
1354
1355static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1356{
7c8d008c 1357 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_SLEEP);
1800eee1
SAS		 1358}
1359
1360static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr)
1361{
7c8d008c 1362 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_NOT_SLEEP);
2a654540
SN		 1363}
1364
1365static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1366{
7c8d008c 1367 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_SLEEP);
1800eee1
SAS		 1368}
1369
1370static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr)
1371{
7c8d008c 1372 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_NOT_SLEEP);
2a654540
SN		 1373}
1374
1375static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode)
1376{
1377 struct ionic_admin_ctx ctx = {
1378 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1379 .cmd.rx_mode_set = {
1380 .opcode = IONIC_CMD_RX_MODE_SET,
1381 .lif_index = cpu_to_le16(lif->index),
1382 .rx_mode = cpu_to_le16(rx_mode),
1383 },
1384 };
1385 char buf[128];
1386 int err;
1387 int i;
1388#define REMAIN(__x) (sizeof(buf) - (__x))
1389
38e0f746
TI		 1390 i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1391 lif->rx_mode, rx_mode);
2a654540 1392 if (rx_mode & IONIC_RX_MODE_F_UNICAST)
38e0f746 1393 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
2a654540 1394 if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
38e0f746 1395 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
2a654540 1396 if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
38e0f746 1397 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
2a654540 1398 if (rx_mode & IONIC_RX_MODE_F_PROMISC)
38e0f746 1399 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
2a654540 1400 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
38e0f746 1401 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
2a654540
SN		 1402 netdev_dbg(lif->netdev, "lif%d %s\n", lif->index, buf);
1403
1404 err = ionic_adminq_post_wait(lif, &ctx);
1405 if (err)
1406 netdev_warn(lif->netdev, "set rx_mode 0x%04x failed: %d\n",
1407 rx_mode, err);
1408 else
1409 lif->rx_mode = rx_mode;
1410}
1411
81dbc241 1412static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
2a654540
SN		 1413{
1414 struct ionic_lif *lif = netdev_priv(netdev);
e94f76bb 1415 struct ionic_deferred_work *work;
2a654540
SN		 1416 unsigned int nfilters;
1417 unsigned int rx_mode;
1418
2a654540
SN		 1419 rx_mode = IONIC_RX_MODE_F_UNICAST;
1420 rx_mode |= (netdev->flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1421 rx_mode |= (netdev->flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1422 rx_mode |= (netdev->flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1423 rx_mode |= (netdev->flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1424
1425 /* sync unicast addresses
1426 * next check to see if we're in an overflow state
1427 * if so, we track that we overflowed and enable NIC PROMISC
1428 * else if the overflow is set and not needed
1429 * we remove our overflow flag and check the netdev flags
1430 * to see if we can disable NIC PROMISC
1431 */
81dbc241 1432 if (can_sleep)
e0243e19 1433 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
81dbc241
SN		 1434 else
1435 __dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
bb9f80f3 1436 nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
2a654540
SN		 1437 if (netdev_uc_count(netdev) + 1 > nfilters) {
1438 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1439 lif->uc_overflow = true;
1440 } else if (lif->uc_overflow) {
1441 lif->uc_overflow = false;
1442 if (!(netdev->flags & IFF_PROMISC))
1443 rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1444 }
1445
1446 /* same for multicast */
81dbc241 1447 if (can_sleep)
e0243e19 1448 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
81dbc241
SN		 1449 else
1450 __dev_mc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
bb9f80f3 1451 nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
2a654540
SN		 1452 if (netdev_mc_count(netdev) > nfilters) {
1453 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1454 lif->mc_overflow = true;
1455 } else if (lif->mc_overflow) {
1456 lif->mc_overflow = false;
1457 if (!(netdev->flags & IFF_ALLMULTI))
1458 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1459 }
1460
e94f76bb 1461 if (lif->rx_mode != rx_mode) {
81dbc241 1462 if (!can_sleep) {
e94f76bb
SN		 1463 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1464 if (!work) {
c0c682ee 1465 netdev_err(lif->netdev, "rxmode change dropped\n");
e94f76bb
SN		 1466 return;
1467 }
1468 work->type = IONIC_DW_TYPE_RX_MODE;
1469 work->rx_mode = rx_mode;
1470 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1471 ionic_lif_deferred_enqueue(&lif->deferred, work);
1472 } else {
1473 ionic_lif_rx_mode(lif, rx_mode);
1474 }
1475 }
1800eee1
SAS		 1476}
1477
1478static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1479{
7c8d008c 1480 ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
2a654540
SN		 1481}
1482
beead698
SN		 1483static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1484{
1485 u64 wanted = 0;
1486
1487 if (features & NETIF_F_HW_VLAN_CTAG_TX)
1488 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1489 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1490 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1491 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1492 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1493 if (features & NETIF_F_RXHASH)
1494 wanted |= IONIC_ETH_HW_RX_HASH;
1495 if (features & NETIF_F_RXCSUM)
1496 wanted |= IONIC_ETH_HW_RX_CSUM;
1497 if (features & NETIF_F_SG)
1498 wanted |= IONIC_ETH_HW_TX_SG;
1499 if (features & NETIF_F_HW_CSUM)
1500 wanted |= IONIC_ETH_HW_TX_CSUM;
1501 if (features & NETIF_F_TSO)
1502 wanted |= IONIC_ETH_HW_TSO;
1503 if (features & NETIF_F_TSO6)
1504 wanted |= IONIC_ETH_HW_TSO_IPV6;
1505 if (features & NETIF_F_TSO_ECN)
1506 wanted |= IONIC_ETH_HW_TSO_ECN;
1507 if (features & NETIF_F_GSO_GRE)
1508 wanted |= IONIC_ETH_HW_TSO_GRE;
1509 if (features & NETIF_F_GSO_GRE_CSUM)
1510 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1511 if (features & NETIF_F_GSO_IPXIP4)
1512 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1513 if (features & NETIF_F_GSO_IPXIP6)
1514 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1515 if (features & NETIF_F_GSO_UDP_TUNNEL)
1516 wanted |= IONIC_ETH_HW_TSO_UDP;
1517 if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1518 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1519
1520 return cpu_to_le64(wanted);
1521}
1522
1523static int ionic_set_nic_features(struct ionic_lif *lif,
1524 netdev_features_t features)
1525{
1526 struct device *dev = lif->ionic->dev;
1527 struct ionic_admin_ctx ctx = {
1528 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1529 .cmd.lif_setattr = {
1530 .opcode = IONIC_CMD_LIF_SETATTR,
1531 .index = cpu_to_le16(lif->index),
1532 .attr = IONIC_LIF_ATTR_FEATURES,
1533 },
1534 };
1535 u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1536 IONIC_ETH_HW_VLAN_RX_STRIP |
1537 IONIC_ETH_HW_VLAN_RX_FILTER;
75fcb75b 1538 u64 old_hw_features;
beead698
SN		 1539 int err;
1540
1541 ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
a8771bfe 1542
afeefec6
SN		 1543 if (lif->phc)
1544 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1545
beead698
SN		 1546 err = ionic_adminq_post_wait(lif, &ctx);
1547 if (err)
1548 return err;
1549
75fcb75b 1550 old_hw_features = lif->hw_features;
beead698
SN		 1551 lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1552 ctx.comp.lif_setattr.features);
1553
75fcb75b
SN		 1554 if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1555 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1556
beead698
SN		 1557 if ((vlan_flags & features) &&
1558 !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1559 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1560
1561 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1562 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1563 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1564 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1565 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1566 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1567 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1568 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1569 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1570 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1571 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1572 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1573 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1574 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1575 if (lif->hw_features & IONIC_ETH_HW_TSO)
1576 dev_dbg(dev, "feature ETH_HW_TSO\n");
1577 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1578 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1579 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1580 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1581 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1582 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1583 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1584 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1585 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1586 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1587 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1588 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1589 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1590 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1591 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1592 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
afeefec6
SN		 1593 if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1594 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
beead698
SN		 1595
1596 return 0;
1597}
1598
1599static int ionic_init_nic_features(struct ionic_lif *lif)
1600{
1601 struct net_device *netdev = lif->netdev;
1602 netdev_features_t features;
1603 int err;
1604
1605 /* set up what we expect to support by default */
1606 features = NETIF_F_HW_VLAN_CTAG_TX |
1607 NETIF_F_HW_VLAN_CTAG_RX |
1608 NETIF_F_HW_VLAN_CTAG_FILTER |
1609 NETIF_F_RXHASH |
1610 NETIF_F_SG |
1611 NETIF_F_HW_CSUM |
1612 NETIF_F_RXCSUM |
1613 NETIF_F_TSO |
1614 NETIF_F_TSO6 |
1615 NETIF_F_TSO_ECN;
1616
1617 err = ionic_set_nic_features(lif, features);
1618 if (err)
1619 return err;
1620
1621 /* tell the netdev what we actually can support */
1622 netdev->features |= NETIF_F_HIGHDMA;
1623
1624 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1625 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1626 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1627 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1628 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1629 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1630 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1631 netdev->hw_features |= NETIF_F_RXHASH;
1632 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1633 netdev->hw_features |= NETIF_F_SG;
1634
1635 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1636 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1637 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1638 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1639 if (lif->hw_features & IONIC_ETH_HW_TSO)
1640 netdev->hw_enc_features |= NETIF_F_TSO;
1641 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1642 netdev->hw_enc_features |= NETIF_F_TSO6;
1643 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1644 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1645 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1646 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1647 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1648 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1649 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1650 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1651 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1652 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1653 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1654 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1655 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1656 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1657
1658 netdev->hw_features |= netdev->hw_enc_features;
1659 netdev->features |= netdev->hw_features;
ef7232da 1660 netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
beead698 1661
c672412f
SN		 1662 netdev->priv_flags |= IFF_UNICAST_FLT |
1663 IFF_LIVE_ADDR_CHANGE;
beead698
SN		 1664
1665 return 0;
1666}
1667
1668static int ionic_set_features(struct net_device *netdev,
1669 netdev_features_t features)
1670{
1671 struct ionic_lif *lif = netdev_priv(netdev);
1672 int err;
1673
1674 netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1675 __func__, (u64)lif->netdev->features, (u64)features);
1676
1677 err = ionic_set_nic_features(lif, features);
1678
1679 return err;
1680}
1681
1682static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1683{
2a654540
SN		 1684 struct sockaddr *addr = sa;
1685 u8 *mac;
1686 int err;
1687
1688 mac = (u8 *)addr->sa_data;
1689 if (ether_addr_equal(netdev->dev_addr, mac))
1690 return 0;
1691
1692 err = eth_prepare_mac_addr_change(netdev, addr);
1693 if (err)
1694 return err;
1695
1696 if (!is_zero_ether_addr(netdev->dev_addr)) {
1697 netdev_info(netdev, "deleting mac addr %pM\n",
1698 netdev->dev_addr);
1699 ionic_addr_del(netdev, netdev->dev_addr);
1700 }
1701
1702 eth_commit_mac_addr_change(netdev, addr);
1703 netdev_info(netdev, "updating mac addr %pM\n", mac);
1704
1705 return ionic_addr_add(netdev, mac);
beead698
SN		 1706}
1707
f053e1f8
SN		 1708static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1709{
1710 /* Stop and clean the queues before reconfiguration */
1711 mutex_lock(&lif->queue_lock);
1712 netif_device_detach(lif->netdev);
1713 ionic_stop_queues(lif);
1714 ionic_txrx_deinit(lif);
1715}
1716
1717static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1718{
1719 int err;
1720
1721 /* Re-init the queues after reconfiguration */
1722
1723 /* The only way txrx_init can fail here is if communication
1724 * with FW is suddenly broken. There's not much we can do
1725 * at this point - error messages have already been printed,
1726 * so we can continue on and the user can eventually do a
1727 * DOWN and UP to try to reset and clear the issue.
1728 */
1729 err = ionic_txrx_init(lif);
1730 mutex_unlock(&lif->queue_lock);
25cc5a5f 1731 ionic_link_status_check_request(lif, CAN_SLEEP);
f053e1f8
SN		 1732 netif_device_attach(lif->netdev);
1733
1734 return err;
1735}
1736
beead698
SN		 1737static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1738{
1739 struct ionic_lif *lif = netdev_priv(netdev);
1740 struct ionic_admin_ctx ctx = {
1741 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1742 .cmd.lif_setattr = {
1743 .opcode = IONIC_CMD_LIF_SETATTR,
1744 .index = cpu_to_le16(lif->index),
1745 .attr = IONIC_LIF_ATTR_MTU,
1746 .mtu = cpu_to_le32(new_mtu),
1747 },
1748 };
1749 int err;
1750
1751 err = ionic_adminq_post_wait(lif, &ctx);
1752 if (err)
1753 return err;
1754
f053e1f8 1755 /* if we're not running, nothing more to do */
79ba55a3
SN		 1756 if (!netif_running(netdev)) {
1757 netdev->mtu = new_mtu;
f053e1f8 1758 return 0;
79ba55a3 1759 }
beead698 1760
f053e1f8 1761 ionic_stop_queues_reconfig(lif);
79ba55a3 1762 netdev->mtu = new_mtu;
f053e1f8 1763 return ionic_start_queues_reconfig(lif);
beead698
SN		 1764}
1765
8c15440b
SN		 1766static void ionic_tx_timeout_work(struct work_struct *ws)
1767{
1768 struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1769
8c775344
SN		 1770 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1771 return;
8c15440b 1772
6f7d6f0f
SN		 1773 /* if we were stopped before this scheduled job was launched,
1774 * don't bother the queues as they are already stopped.
1775 */
1776 if (!netif_running(lif->netdev))
1777 return;
1778
1779 ionic_stop_queues_reconfig(lif);
1780 ionic_start_queues_reconfig(lif);
8c15440b
SN		 1781}
1782
0290bd29 1783static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
beead698 1784{
8c15440b
SN		 1785 struct ionic_lif *lif = netdev_priv(netdev);
1786
8c775344 1787 netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
8c15440b 1788 schedule_work(&lif->tx_timeout_work);
beead698
SN		 1789}
1790
1791static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1792 u16 vid)
1793{
2a654540
SN		 1794 struct ionic_lif *lif = netdev_priv(netdev);
1795 struct ionic_admin_ctx ctx = {
1796 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1797 .cmd.rx_filter_add = {
1798 .opcode = IONIC_CMD_RX_FILTER_ADD,
1799 .lif_index = cpu_to_le16(lif->index),
1800 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1801 .vlan.vlan = cpu_to_le16(vid),
1802 },
1803 };
1804 int err;
1805
cbec2153 1806 netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
2a654540
SN		 1807 err = ionic_adminq_post_wait(lif, &ctx);
1808 if (err)
1809 return err;
1810
2a654540 1811 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
beead698
SN		 1812}
1813
1814static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1815 u16 vid)
1816{
2a654540
SN		 1817 struct ionic_lif *lif = netdev_priv(netdev);
1818 struct ionic_admin_ctx ctx = {
1819 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1820 .cmd.rx_filter_del = {
1821 .opcode = IONIC_CMD_RX_FILTER_DEL,
1822 .lif_index = cpu_to_le16(lif->index),
1823 },
1824 };
1825 struct ionic_rx_filter *f;
1826
1827 spin_lock_bh(&lif->rx_filters.lock);
1828
1829 f = ionic_rx_filter_by_vlan(lif, vid);
1830 if (!f) {
1831 spin_unlock_bh(&lif->rx_filters.lock);
1832 return -ENOENT;
1833 }
1834
cbec2153
SN		 1835 netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1836 vid, f->filter_id);
2a654540
SN		 1837
1838 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1839 ionic_rx_filter_free(lif, f);
1840 spin_unlock_bh(&lif->rx_filters.lock);
1841
1842 return ionic_adminq_post_wait(lif, &ctx);
beead698
SN		 1843}
1844
aa319881
SN		 1845int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1846 const u8 *key, const u32 *indir)
1847{
1848 struct ionic_admin_ctx ctx = {
1849 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1850 .cmd.lif_setattr = {
1851 .opcode = IONIC_CMD_LIF_SETATTR,
1852 .attr = IONIC_LIF_ATTR_RSS,
aa319881
SN		 1853 .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1854 },
1855 };
1856 unsigned int i, tbl_sz;
1857
75fcb75b
SN		 1858 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1859 lif->rss_types = types;
1860 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1861 }
aa319881
SN		 1862
1863 if (key)
1864 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1865
1866 if (indir) {
1867 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1868 for (i = 0; i < tbl_sz; i++)
1869 lif->rss_ind_tbl[i] = indir[i];
1870 }
1871
1872 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1873 IONIC_RSS_HASH_KEY_SIZE);
1874
1875 return ionic_adminq_post_wait(lif, &ctx);
1876}
1877
1878static int ionic_lif_rss_init(struct ionic_lif *lif)
1879{
aa319881
SN		 1880 unsigned int tbl_sz;
1881 unsigned int i;
1882
aa319881
SN		 1883 lif->rss_types = IONIC_RSS_TYPE_IPV4 |
1884 IONIC_RSS_TYPE_IPV4_TCP |
1885 IONIC_RSS_TYPE_IPV4_UDP |
1886 IONIC_RSS_TYPE_IPV6 |
1887 IONIC_RSS_TYPE_IPV6_TCP |
1888 IONIC_RSS_TYPE_IPV6_UDP;
1889
1890 /* Fill indirection table with 'default' values */
1891 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1892 for (i = 0; i < tbl_sz; i++)
1893 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1894
ffac2027 1895 return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
aa319881
SN		 1896}
1897
ffac2027 1898static void ionic_lif_rss_deinit(struct ionic_lif *lif)
aa319881 1899{
ffac2027
SN		 1900 int tbl_sz;
1901
1902 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1903 memset(lif->rss_ind_tbl, 0, tbl_sz);
1904 memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1905
1906 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
aa319881
SN		 1907}
1908
e7e8e087
SN		 1909static void ionic_lif_quiesce(struct ionic_lif *lif)
1910{
1911 struct ionic_admin_ctx ctx = {
1912 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1913 .cmd.lif_setattr = {
1914 .opcode = IONIC_CMD_LIF_SETATTR,
1915 .index = cpu_to_le16(lif->index),
1916 .attr = IONIC_LIF_ATTR_STATE,
1917 .state = IONIC_LIF_QUIESCE,
1918 },
1919 };
1920 int err;
1921
1922 err = ionic_adminq_post_wait(lif, &ctx);
1923 if (err)
1924 netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1925}
1926
0f3154e6
SN		 1927static void ionic_txrx_disable(struct ionic_lif *lif)
1928{
1929 unsigned int i;
ba6ab8ac 1930 int err = 0;
0f3154e6 1931
d5eddde5 1932 if (lif->txqcqs) {
ba6ab8ac
SN		 1933 for (i = 0; i < lif->nxqs; i++)
1934 err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
d5eddde5
SN		 1935 }
1936
f0790bcd
SN		 1937 if (lif->hwstamp_txq)
1938 err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1939
d5eddde5 1940 if (lif->rxqcqs) {
ba6ab8ac
SN		 1941 for (i = 0; i < lif->nxqs; i++)
1942 err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
0f3154e6 1943 }
e7e8e087 1944
f0790bcd
SN		 1945 if (lif->hwstamp_rxq)
1946 err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1947
e7e8e087 1948 ionic_lif_quiesce(lif);
0f3154e6
SN		 1949}
1950
1951static void ionic_txrx_deinit(struct ionic_lif *lif)
1952{
1953 unsigned int i;
1954
d5eddde5 1955 if (lif->txqcqs) {
101b40a0 1956 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
34dec947
SN		 1957 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1958 ionic_tx_flush(&lif->txqcqs[i]->cq);
1959 ionic_tx_empty(&lif->txqcqs[i]->q);
d5eddde5
SN		 1960 }
1961 }
0f3154e6 1962
d5eddde5 1963 if (lif->rxqcqs) {
101b40a0 1964 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
34dec947 1965 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
34dec947 1966 ionic_rx_empty(&lif->rxqcqs[i]->q);
d5eddde5 1967 }
0f3154e6 1968 }
49d3b493 1969 lif->rx_mode = 0;
a8771bfe
SN		 1970
1971 if (lif->hwstamp_txq) {
1972 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1973 ionic_tx_flush(&lif->hwstamp_txq->cq);
1974 ionic_tx_empty(&lif->hwstamp_txq->q);
1975 }
1976
1977 if (lif->hwstamp_rxq) {
1978 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1979 ionic_rx_empty(&lif->hwstamp_rxq->q);
1980 }
0f3154e6
SN		 1981}
1982
1983static void ionic_txrx_free(struct ionic_lif *lif)
1984{
1985 unsigned int i;
1986
d5eddde5 1987 if (lif->txqcqs) {
101b40a0 1988 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
34dec947 1989 ionic_qcq_free(lif, lif->txqcqs[i]);
101b40a0 1990 devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
34dec947 1991 lif->txqcqs[i] = NULL;
d5eddde5
SN		 1992 }
1993 }
0f3154e6 1994
d5eddde5 1995 if (lif->rxqcqs) {
101b40a0 1996 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
34dec947 1997 ionic_qcq_free(lif, lif->rxqcqs[i]);
101b40a0 1998 devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
34dec947 1999 lif->rxqcqs[i] = NULL;
d5eddde5 2000 }
0f3154e6 2001 }
f0790bcd
SN		 2002
2003 if (lif->hwstamp_txq) {
2004 ionic_qcq_free(lif, lif->hwstamp_txq);
2005 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2006 lif->hwstamp_txq = NULL;
2007 }
2008
2009 if (lif->hwstamp_rxq) {
2010 ionic_qcq_free(lif, lif->hwstamp_rxq);
2011 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2012 lif->hwstamp_rxq = NULL;
2013 }
0f3154e6
SN		 2014}
2015
2016static int ionic_txrx_alloc(struct ionic_lif *lif)
2017{
33c252e1
SN		 2018 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2019 unsigned int flags, i;
0f3154e6
SN		 2020 int err = 0;
2021
0ec9f666
SN		 2022 num_desc = lif->ntxq_descs;
2023 desc_sz = sizeof(struct ionic_txq_desc);
2024 comp_sz = sizeof(struct ionic_txq_comp);
2025
5b3f3f2a
SN		 2026 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2027 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2028 sizeof(struct ionic_txq_sg_desc_v1))
2029 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2030 else
2031 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2032
0f3154e6 2033 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
fe8c30b5
SN		 2034 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2035 flags |= IONIC_QCQ_F_INTR;
0f3154e6
SN		 2036 for (i = 0; i < lif->nxqs; i++) {
2037 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
0ec9f666 2038 num_desc, desc_sz, comp_sz, sg_desc_sz,
34dec947 2039 lif->kern_pid, &lif->txqcqs[i]);
0f3154e6
SN		 2040 if (err)
2041 goto err_out;
2042
04a83459 2043 if (flags & IONIC_QCQ_F_INTR) {
fe8c30b5 2044 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
34dec947 2045 lif->txqcqs[i]->intr.index,
fe8c30b5 2046 lif->tx_coalesce_hw);
04a83459
SN		 2047 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2048 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2049 }
fe8c30b5 2050
34dec947 2051 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
0f3154e6
SN		 2052 }
2053
08f2e4b2 2054 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
0ec9f666
SN		 2055
2056 num_desc = lif->nrxq_descs;
2057 desc_sz = sizeof(struct ionic_rxq_desc);
2058 comp_sz = sizeof(struct ionic_rxq_comp);
2059 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2060
2061 if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2062 comp_sz *= 2;
2063
0f3154e6
SN		 2064 for (i = 0; i < lif->nxqs; i++) {
2065 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
0ec9f666 2066 num_desc, desc_sz, comp_sz, sg_desc_sz,
34dec947 2067 lif->kern_pid, &lif->rxqcqs[i]);
0f3154e6
SN		 2068 if (err)
2069 goto err_out;
2070
0ec9f666
SN		 2071 lif->rxqcqs[i]->q.features = lif->rxq_features;
2072
8c15440b 2073 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
34dec947 2074 lif->rxqcqs[i]->intr.index,
780eded3 2075 lif->rx_coalesce_hw);
04a83459
SN		 2076 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2077 lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
fe8c30b5
SN		 2078
2079 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
34dec947
SN		 2080 ionic_link_qcq_interrupts(lif->rxqcqs[i],
2081 lif->txqcqs[i]);
fe8c30b5 2082
34dec947 2083 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
0f3154e6
SN		 2084 }
2085
2086 return 0;
2087
2088err_out:
2089 ionic_txrx_free(lif);
2090
2091 return err;
2092}
2093
2094static int ionic_txrx_init(struct ionic_lif *lif)
2095{
2096 unsigned int i;
2097 int err;
2098
2099 for (i = 0; i < lif->nxqs; i++) {
34dec947 2100 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
0f3154e6
SN		 2101 if (err)
2102 goto err_out;
2103
34dec947 2104 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
0f3154e6 2105 if (err) {
34dec947 2106 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
0f3154e6
SN		 2107 goto err_out;
2108 }
2109 }
2110
aa319881
SN		 2111 if (lif->netdev->features & NETIF_F_RXHASH)
2112 ionic_lif_rss_init(lif);
2113
7c8d008c 2114 ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
0f3154e6
SN		 2115
2116 return 0;
2117
2118err_out:
2119 while (i--) {
34dec947
SN		 2120 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2121 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
0f3154e6
SN		 2122 }
2123
2124 return err;
2125}
2126
2127static int ionic_txrx_enable(struct ionic_lif *lif)
2128{
ba6ab8ac 2129 int derr = 0;
0f3154e6
SN		 2130 int i, err;
2131
2132 for (i = 0; i < lif->nxqs; i++) {
7c737fc4
SN		 2133 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2134 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2135 err = -ENXIO;
2136 goto err_out;
2137 }
2138
34dec947
SN		 2139 ionic_rx_fill(&lif->rxqcqs[i]->q);
2140 err = ionic_qcq_enable(lif->rxqcqs[i]);
0f3154e6
SN		 2141 if (err)
2142 goto err_out;
2143
34dec947 2144 err = ionic_qcq_enable(lif->txqcqs[i]);
0f3154e6 2145 if (err) {
ba6ab8ac 2146 derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
0f3154e6
SN		 2147 goto err_out;
2148 }
2149 }
2150
a8771bfe
SN		 2151 if (lif->hwstamp_rxq) {
2152 ionic_rx_fill(&lif->hwstamp_rxq->q);
2153 err = ionic_qcq_enable(lif->hwstamp_rxq);
2154 if (err)
2155 goto err_out_hwstamp_rx;
2156 }
2157
2158 if (lif->hwstamp_txq) {
2159 err = ionic_qcq_enable(lif->hwstamp_txq);
2160 if (err)
2161 goto err_out_hwstamp_tx;
2162 }
2163
0f3154e6
SN		 2164 return 0;
2165
a8771bfe
SN		 2166err_out_hwstamp_tx:
2167 if (lif->hwstamp_rxq)
2168 derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2169err_out_hwstamp_rx:
2170 i = lif->nxqs;
0f3154e6
SN		 2171err_out:
2172 while (i--) {
ba6ab8ac
SN		 2173 derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2174 derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
0f3154e6
SN		 2175 }
2176
2177 return err;
2178}
2179
49d3b493
SN		 2180static int ionic_start_queues(struct ionic_lif *lif)
2181{
2182 int err;
2183
9e8eaf84
SN		 2184 if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2185 return -EIO;
2186
8c775344
SN		 2187 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2188 return -EBUSY;
2189
49d3b493
SN		 2190 if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2191 return 0;
2192
2193 err = ionic_txrx_enable(lif);
2194 if (err) {
2195 clear_bit(IONIC_LIF_F_UP, lif->state);
2196 return err;
2197 }
2198 netif_tx_wake_all_queues(lif->netdev);
2199
2200 return 0;
2201}
2202
d4881430 2203static int ionic_open(struct net_device *netdev)
beead698
SN		 2204{
2205 struct ionic_lif *lif = netdev_priv(netdev);
0f3154e6 2206 int err;
beead698 2207
9e8eaf84
SN		 2208 /* If recovering from a broken state, clear the bit and we'll try again */
2209 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2210 netdev_info(netdev, "clearing broken state\n");
2211
0f3154e6
SN		 2212 err = ionic_txrx_alloc(lif);
2213 if (err)
2214 return err;
2215
2216 err = ionic_txrx_init(lif);
2217 if (err)
25cc5a5f 2218 goto err_txrx_free;
beead698 2219
fa48494c
SN		 2220 err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2221 if (err)
2222 goto err_txrx_deinit;
2223
2224 err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2225 if (err)
2226 goto err_txrx_deinit;
2227
49d3b493
SN		 2228 /* don't start the queues until we have link */
2229 if (netif_carrier_ok(netdev)) {
2230 err = ionic_start_queues(lif);
2231 if (err)
2232 goto err_txrx_deinit;
2233 }
8d61aad4 2234
beead698 2235 return 0;
0f3154e6
SN		 2236
2237err_txrx_deinit:
2238 ionic_txrx_deinit(lif);
25cc5a5f 2239err_txrx_free:
0f3154e6
SN		 2240 ionic_txrx_free(lif);
2241 return err;
beead698
SN		 2242}
2243
49d3b493 2244static void ionic_stop_queues(struct ionic_lif *lif)
beead698 2245{
49d3b493
SN		 2246 if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2247 return;
beead698 2248
49d3b493 2249 netif_tx_disable(lif->netdev);
b59eabd2 2250 ionic_txrx_disable(lif);
49d3b493 2251}
beead698 2252
d4881430 2253static int ionic_stop(struct net_device *netdev)
49d3b493
SN		 2254{
2255 struct ionic_lif *lif = netdev_priv(netdev);
0f3154e6 2256
b59eabd2 2257 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
c672412f
SN		 2258 return 0;
2259
49d3b493 2260 ionic_stop_queues(lif);
0f3154e6
SN		 2261 ionic_txrx_deinit(lif);
2262 ionic_txrx_free(lif);
beead698 2263
49d3b493 2264 return 0;
beead698
SN		 2265}
2266
afeefec6
SN		 2267static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2268{
2269 struct ionic_lif *lif = netdev_priv(netdev);
2270
2271 switch (cmd) {
2272 case SIOCSHWTSTAMP:
2273 return ionic_lif_hwstamp_set(lif, ifr);
2274 case SIOCGHWTSTAMP:
2275 return ionic_lif_hwstamp_get(lif, ifr);
2276 default:
2277 return -EOPNOTSUPP;
2278 }
2279}
2280
fbb39807
SN		 2281static int ionic_get_vf_config(struct net_device *netdev,
2282 int vf, struct ifla_vf_info *ivf)
2283{
2284 struct ionic_lif *lif = netdev_priv(netdev);
2285 struct ionic *ionic = lif->ionic;
2286 int ret = 0;
2287
a836c352
SN		 2288 if (!netif_device_present(netdev))
2289 return -EBUSY;
2290
fbb39807
SN		 2291 down_read(&ionic->vf_op_lock);
2292
2293 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2294 ret = -EINVAL;
2295 } else {
2296 ivf->vf = vf;
d701ec32 2297 ivf->vlan = le16_to_cpu(ionic->vfs[vf].vlanid);
fbb39807
SN		 2298 ivf->qos = 0;
2299 ivf->spoofchk = ionic->vfs[vf].spoofchk;
2300 ivf->linkstate = ionic->vfs[vf].linkstate;
d701ec32 2301 ivf->max_tx_rate = le32_to_cpu(ionic->vfs[vf].maxrate);
fbb39807
SN		 2302 ivf->trusted = ionic->vfs[vf].trusted;
2303 ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2304 }
2305
2306 up_read(&ionic->vf_op_lock);
2307 return ret;
2308}
2309
2310static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2311 struct ifla_vf_stats *vf_stats)
2312{
2313 struct ionic_lif *lif = netdev_priv(netdev);
2314 struct ionic *ionic = lif->ionic;
2315 struct ionic_lif_stats *vs;
2316 int ret = 0;
2317
a836c352
SN		 2318 if (!netif_device_present(netdev))
2319 return -EBUSY;
2320
fbb39807
SN		 2321 down_read(&ionic->vf_op_lock);
2322
2323 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2324 ret = -EINVAL;
2325 } else {
2326 memset(vf_stats, 0, sizeof(*vf_stats));
2327 vs = &ionic->vfs[vf].stats;
2328
2329 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2330 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2331 vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
2332 vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
2333 vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
2334 vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
2335 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2336 le64_to_cpu(vs->rx_mcast_drop_packets) +
2337 le64_to_cpu(vs->rx_bcast_drop_packets);
2338 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2339 le64_to_cpu(vs->tx_mcast_drop_packets) +
2340 le64_to_cpu(vs->tx_bcast_drop_packets);
2341 }
2342
2343 up_read(&ionic->vf_op_lock);
2344 return ret;
2345}
2346
2347static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2348{
2349 struct ionic_lif *lif = netdev_priv(netdev);
2350 struct ionic *ionic = lif->ionic;
2351 int ret;
2352
2353 if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2354 return -EINVAL;
2355
a836c352
SN		 2356 if (!netif_device_present(netdev))
2357 return -EBUSY;
2358
e396ce5f 2359 down_write(&ionic->vf_op_lock);
fbb39807
SN		 2360
2361 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2362 ret = -EINVAL;
2363 } else {
2364 ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2365 if (!ret)
2366 ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2367 }
2368
e396ce5f 2369 up_write(&ionic->vf_op_lock);
fbb39807
SN		 2370 return ret;
2371}
2372
2373static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2374 u8 qos, __be16 proto)
2375{
2376 struct ionic_lif *lif = netdev_priv(netdev);
2377 struct ionic *ionic = lif->ionic;
2378 int ret;
2379
2380 /* until someday when we support qos */
2381 if (qos)
2382 return -EINVAL;
2383
2384 if (vlan > 4095)
2385 return -EINVAL;
2386
2387 if (proto != htons(ETH_P_8021Q))
2388 return -EPROTONOSUPPORT;
2389
a836c352
SN		 2390 if (!netif_device_present(netdev))
2391 return -EBUSY;
2392
e396ce5f 2393 down_write(&ionic->vf_op_lock);
fbb39807
SN		 2394
2395 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396 ret = -EINVAL;
2397 } else {
2398 ret = ionic_set_vf_config(ionic, vf,
2399 IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2400 if (!ret)
d701ec32 2401 ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
fbb39807
SN		 2402 }
2403
e396ce5f 2404 up_write(&ionic->vf_op_lock);
fbb39807
SN		 2405 return ret;
2406}
2407
2408static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2409 int tx_min, int tx_max)
2410{
2411 struct ionic_lif *lif = netdev_priv(netdev);
2412 struct ionic *ionic = lif->ionic;
2413 int ret;
2414
2415 /* setting the min just seems silly */
2416 if (tx_min)
2417 return -EINVAL;
2418
a836c352
SN		 2419 if (!netif_device_present(netdev))
2420 return -EBUSY;
2421
fbb39807
SN		 2422 down_write(&ionic->vf_op_lock);
2423
2424 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2425 ret = -EINVAL;
2426 } else {
2427 ret = ionic_set_vf_config(ionic, vf,
2428 IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2429 if (!ret)
d701ec32 2430 lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
fbb39807
SN		 2431 }
2432
2433 up_write(&ionic->vf_op_lock);
2434 return ret;
2435}
2436
2437static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2438{
2439 struct ionic_lif *lif = netdev_priv(netdev);
2440 struct ionic *ionic = lif->ionic;
2441 u8 data = set; /* convert to u8 for config */
2442 int ret;
2443
a836c352
SN		 2444 if (!netif_device_present(netdev))
2445 return -EBUSY;
2446
fbb39807
SN		 2447 down_write(&ionic->vf_op_lock);
2448
2449 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2450 ret = -EINVAL;
2451 } else {
2452 ret = ionic_set_vf_config(ionic, vf,
2453 IONIC_VF_ATTR_SPOOFCHK, &data);
2454 if (!ret)
2455 ionic->vfs[vf].spoofchk = data;
2456 }
2457
2458 up_write(&ionic->vf_op_lock);
2459 return ret;
2460}
2461
2462static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2463{
2464 struct ionic_lif *lif = netdev_priv(netdev);
2465 struct ionic *ionic = lif->ionic;
2466 u8 data = set; /* convert to u8 for config */
2467 int ret;
2468
a836c352
SN		 2469 if (!netif_device_present(netdev))
2470 return -EBUSY;
2471
fbb39807
SN		 2472 down_write(&ionic->vf_op_lock);
2473
2474 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2475 ret = -EINVAL;
2476 } else {
2477 ret = ionic_set_vf_config(ionic, vf,
2478 IONIC_VF_ATTR_TRUST, &data);
2479 if (!ret)
2480 ionic->vfs[vf].trusted = data;
2481 }
2482
2483 up_write(&ionic->vf_op_lock);
2484 return ret;
2485}
2486
2487static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2488{
2489 struct ionic_lif *lif = netdev_priv(netdev);
2490 struct ionic *ionic = lif->ionic;
2491 u8 data;
2492 int ret;
2493
2494 switch (set) {
2495 case IFLA_VF_LINK_STATE_ENABLE:
2496 data = IONIC_VF_LINK_STATUS_UP;
2497 break;
2498 case IFLA_VF_LINK_STATE_DISABLE:
2499 data = IONIC_VF_LINK_STATUS_DOWN;
2500 break;
2501 case IFLA_VF_LINK_STATE_AUTO:
2502 data = IONIC_VF_LINK_STATUS_AUTO;
2503 break;
2504 default:
2505 return -EINVAL;
2506 }
2507
a836c352
SN		 2508 if (!netif_device_present(netdev))
2509 return -EBUSY;
2510
fbb39807
SN		 2511 down_write(&ionic->vf_op_lock);
2512
2513 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2514 ret = -EINVAL;
2515 } else {
2516 ret = ionic_set_vf_config(ionic, vf,
2517 IONIC_VF_ATTR_LINKSTATE, &data);
2518 if (!ret)
2519 ionic->vfs[vf].linkstate = set;
2520 }
2521
2522 up_write(&ionic->vf_op_lock);
2523 return ret;
2524}
2525
beead698
SN		 2526static const struct net_device_ops ionic_netdev_ops = {
2527 .ndo_open = ionic_open,
2528 .ndo_stop = ionic_stop,
afeefec6 2529 .ndo_do_ioctl = ionic_do_ioctl,
0f3154e6 2530 .ndo_start_xmit = ionic_start_xmit,
8d61aad4 2531 .ndo_get_stats64 = ionic_get_stats64,
1800eee1 2532 .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
beead698
SN		 2533 .ndo_set_features = ionic_set_features,
2534 .ndo_set_mac_address = ionic_set_mac_address,
2535 .ndo_validate_addr = eth_validate_addr,
2536 .ndo_tx_timeout = ionic_tx_timeout,
2537 .ndo_change_mtu = ionic_change_mtu,
2538 .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
2539 .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
fbb39807
SN		 2540 .ndo_set_vf_vlan = ionic_set_vf_vlan,
2541 .ndo_set_vf_trust = ionic_set_vf_trust,
2542 .ndo_set_vf_mac = ionic_set_vf_mac,
2543 .ndo_set_vf_rate = ionic_set_vf_rate,
2544 .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
2545 .ndo_get_vf_config = ionic_get_vf_config,
2546 .ndo_set_vf_link_state = ionic_set_vf_link_state,
2547 .ndo_get_vf_stats = ionic_get_vf_stats,
beead698
SN		 2548};
2549
a34e25ab
SN		 2550static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2551{
2552 /* only swapping the queues, not the napi, flags, or other stuff */
57a3a98d 2553 swap(a->q.features, b->q.features);
a34e25ab 2554 swap(a->q.num_descs, b->q.num_descs);
0ec9f666 2555 swap(a->q.desc_size, b->q.desc_size);
a34e25ab
SN		 2556 swap(a->q.base, b->q.base);
2557 swap(a->q.base_pa, b->q.base_pa);
2558 swap(a->q.info, b->q.info);
2559 swap(a->q_base, b->q_base);
2560 swap(a->q_base_pa, b->q_base_pa);
2561 swap(a->q_size, b->q_size);
2562
0ec9f666 2563 swap(a->q.sg_desc_size, b->q.sg_desc_size);
a34e25ab
SN		 2564 swap(a->q.sg_base, b->q.sg_base);
2565 swap(a->q.sg_base_pa, b->q.sg_base_pa);
2566 swap(a->sg_base, b->sg_base);
2567 swap(a->sg_base_pa, b->sg_base_pa);
2568 swap(a->sg_size, b->sg_size);
2569
2570 swap(a->cq.num_descs, b->cq.num_descs);
0ec9f666 2571 swap(a->cq.desc_size, b->cq.desc_size);
a34e25ab
SN		 2572 swap(a->cq.base, b->cq.base);
2573 swap(a->cq.base_pa, b->cq.base_pa);
2574 swap(a->cq.info, b->cq.info);
2575 swap(a->cq_base, b->cq_base);
2576 swap(a->cq_base_pa, b->cq_base_pa);
2577 swap(a->cq_size, b->cq_size);
55eda6bb
SN		 2578
2579 ionic_debugfs_del_qcq(a);
2580 ionic_debugfs_add_qcq(a->q.lif, a);
a34e25ab
SN		 2581}
2582
2583int ionic_reconfigure_queues(struct ionic_lif *lif,
2584 struct ionic_queue_params *qparam)
2585{
33c252e1 2586 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
a34e25ab
SN		 2587 struct ionic_qcq **tx_qcqs = NULL;
2588 struct ionic_qcq **rx_qcqs = NULL;
33c252e1 2589 unsigned int flags, i;
a34e25ab 2590 int err = -ENOMEM;
a34e25ab
SN		 2591
2592 /* allocate temporary qcq arrays to hold new queue structs */
101b40a0
SN		 2593 if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2594 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
a34e25ab
SN		 2595 sizeof(struct ionic_qcq *), GFP_KERNEL);
2596 if (!tx_qcqs)
2597 goto err_out;
2598 }
0ec9f666
SN		 2599 if (qparam->nxqs != lif->nxqs ||
2600 qparam->nrxq_descs != lif->nrxq_descs ||
2601 qparam->rxq_features != lif->rxq_features) {
101b40a0 2602 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
a34e25ab
SN		 2603 sizeof(struct ionic_qcq *), GFP_KERNEL);
2604 if (!rx_qcqs)
2605 goto err_out;
2606 }
2607
101b40a0
SN		 2608 /* allocate new desc_info and rings, but leave the interrupt setup
2609 * until later so as to not mess with the still-running queues
2610 */
a34e25ab 2611 if (tx_qcqs) {
0ec9f666
SN		 2612 num_desc = qparam->ntxq_descs;
2613 desc_sz = sizeof(struct ionic_txq_desc);
2614 comp_sz = sizeof(struct ionic_txq_comp);
2615
2616 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2617 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2618 sizeof(struct ionic_txq_sg_desc_v1))
2619 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2620 else
2621 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2622
101b40a0 2623 for (i = 0; i < qparam->nxqs; i++) {
a34e25ab
SN		 2624 flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2625 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
0ec9f666 2626 num_desc, desc_sz, comp_sz, sg_desc_sz,
a34e25ab
SN		 2627 lif->kern_pid, &tx_qcqs[i]);
2628 if (err)
2629 goto err_out;
2630 }
2631 }
2632
2633 if (rx_qcqs) {
0ec9f666
SN		 2634 num_desc = qparam->nrxq_descs;
2635 desc_sz = sizeof(struct ionic_rxq_desc);
2636 comp_sz = sizeof(struct ionic_rxq_comp);
2637 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2638
2639 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2640 comp_sz *= 2;
2641
101b40a0 2642 for (i = 0; i < qparam->nxqs; i++) {
a34e25ab
SN		 2643 flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2644 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
0ec9f666 2645 num_desc, desc_sz, comp_sz, sg_desc_sz,
a34e25ab
SN		 2646 lif->kern_pid, &rx_qcqs[i]);
2647 if (err)
2648 goto err_out;
0ec9f666
SN		 2649
2650 rx_qcqs[i]->q.features = qparam->rxq_features;
a34e25ab
SN		 2651 }
2652 }
2653
2654 /* stop and clean the queues */
2655 ionic_stop_queues_reconfig(lif);
2656
101b40a0
SN		 2657 if (qparam->nxqs != lif->nxqs) {
2658 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2659 if (err)
2660 goto err_out_reinit_unlock;
2661 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2662 if (err) {
2663 netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2664 goto err_out_reinit_unlock;
2665 }
2666 }
2667
a34e25ab
SN		 2668 /* swap new desc_info and rings, keeping existing interrupt config */
2669 if (tx_qcqs) {
2670 lif->ntxq_descs = qparam->ntxq_descs;
101b40a0 2671 for (i = 0; i < qparam->nxqs; i++)
a34e25ab
SN		 2672 ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2673 }
2674
2675 if (rx_qcqs) {
2676 lif->nrxq_descs = qparam->nrxq_descs;
101b40a0 2677 for (i = 0; i < qparam->nxqs; i++)
a34e25ab
SN		 2678 ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2679 }
2680
101b40a0
SN		 2681 /* if we need to change the interrupt layout, this is the time */
2682 if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2683 qparam->nxqs != lif->nxqs) {
2684 if (qparam->intr_split) {
2685 set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2686 } else {
2687 clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2688 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2689 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2690 }
2691
2692 /* clear existing interrupt assignments */
2693 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2694 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2695 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2696 }
2697
2698 /* re-assign the interrupts */
2699 for (i = 0; i < qparam->nxqs; i++) {
2700 lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2701 err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2702 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2703 lif->rxqcqs[i]->intr.index,
2704 lif->rx_coalesce_hw);
2705
2706 if (qparam->intr_split) {
2707 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2708 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2709 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2710 lif->txqcqs[i]->intr.index,
2711 lif->tx_coalesce_hw);
04a83459
SN		 2712 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2713 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
101b40a0
SN		 2714 } else {
2715 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2716 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2717 }
2718 }
2719 }
2720
ed6d9b02
SN		 2721 /* now we can rework the debugfs mappings */
2722 if (tx_qcqs) {
2723 for (i = 0; i < qparam->nxqs; i++) {
2724 ionic_debugfs_del_qcq(lif->txqcqs[i]);
2725 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2726 }
2727 }
2728
2729 if (rx_qcqs) {
2730 for (i = 0; i < qparam->nxqs; i++) {
2731 ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2732 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2733 }
2734 }
2735
101b40a0 2736 swap(lif->nxqs, qparam->nxqs);
0ec9f666 2737 swap(lif->rxq_features, qparam->rxq_features);
101b40a0
SN		 2738
2739err_out_reinit_unlock:
25cc5a5f 2740 /* re-init the queues, but don't lose an error code */
101b40a0
SN		 2741 if (err)
2742 ionic_start_queues_reconfig(lif);
2743 else
2744 err = ionic_start_queues_reconfig(lif);
a34e25ab
SN		 2745
2746err_out:
2747 /* free old allocs without cleaning intr */
101b40a0 2748 for (i = 0; i < qparam->nxqs; i++) {
a34e25ab
SN		 2749 if (tx_qcqs && tx_qcqs[i]) {
2750 tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2751 ionic_qcq_free(lif, tx_qcqs[i]);
101b40a0 2752 devm_kfree(lif->ionic->dev, tx_qcqs[i]);
a34e25ab
SN		 2753 tx_qcqs[i] = NULL;
2754 }
2755 if (rx_qcqs && rx_qcqs[i]) {
2756 rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2757 ionic_qcq_free(lif, rx_qcqs[i]);
101b40a0 2758 devm_kfree(lif->ionic->dev, rx_qcqs[i]);
a34e25ab
SN		 2759 rx_qcqs[i] = NULL;
2760 }
2761 }
2762
2763 /* free q array */
2764 if (rx_qcqs) {
2765 devm_kfree(lif->ionic->dev, rx_qcqs);
2766 rx_qcqs = NULL;
2767 }
2768 if (tx_qcqs) {
2769 devm_kfree(lif->ionic->dev, tx_qcqs);
2770 tx_qcqs = NULL;
2771 }
2772
101b40a0
SN		 2773 /* clean the unused dma and info allocations when new set is smaller
2774 * than the full array, but leave the qcq shells in place
2775 */
2776 for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2777 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2778 ionic_qcq_free(lif, lif->txqcqs[i]);
2779
2780 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2781 ionic_qcq_free(lif, lif->rxqcqs[i]);
2782 }
2783
a34e25ab
SN		 2784 return err;
2785}
2786
30b87ab4 2787int ionic_lif_alloc(struct ionic *ionic)
1a58e196
SN		 2788{
2789 struct device *dev = ionic->dev;
4b03b273 2790 union ionic_lif_identity *lid;
1a58e196
SN		 2791 struct net_device *netdev;
2792 struct ionic_lif *lif;
aa319881 2793 int tbl_sz;
1a58e196
SN		 2794 int err;
2795
4b03b273
SN		 2796 lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2797 if (!lid)
30b87ab4 2798 return -ENOMEM;
4b03b273 2799
1a58e196
SN		 2800 netdev = alloc_etherdev_mqs(sizeof(*lif),
2801 ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2802 if (!netdev) {
2803 dev_err(dev, "Cannot allocate netdev, aborting\n");
4b1debbe
CIK		 2804 err = -ENOMEM;
2805 goto err_out_free_lid;
1a58e196
SN		 2806 }
2807
2808 SET_NETDEV_DEV(netdev, dev);
2809
2810 lif = netdev_priv(netdev);
2811 lif->netdev = netdev;
30b87ab4 2812 ionic->lif = lif;
beead698 2813 netdev->netdev_ops = &ionic_netdev_ops;
4d03e00a 2814 ionic_ethtool_set_ops(netdev);
beead698
SN		 2815
2816 netdev->watchdog_timeo = 2 * HZ;
aa47b540
SN		 2817 netif_carrier_off(netdev);
2818
4b03b273
SN		 2819 lif->identity = lid;
2820 lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
bb9f80f3
SN		 2821 err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2822 if (err) {
2823 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2824 lif->lif_type, err);
2825 goto err_out_free_netdev;
2826 }
eba87609
SN		 2827 lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2828 le32_to_cpu(lif->identity->eth.min_frame_size));
4b03b273
SN		 2829 lif->netdev->max_mtu =
2830 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
1a58e196
SN		 2831
2832 lif->neqs = ionic->neqs_per_lif;
2833 lif->nxqs = ionic->ntxqs_per_lif;
2834
2835 lif->ionic = ionic;
30b87ab4 2836 lif->index = 0;
0f3154e6
SN		 2837 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2838 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
1a58e196 2839
8c15440b 2840 /* Convert the default coalesce value to actual hw resolution */
780eded3 2841 lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
ff7ebed9 2842 lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
780eded3 2843 lif->rx_coalesce_usecs);
fe8c30b5
SN		 2844 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2845 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
04a83459
SN		 2846 set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2847 set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
8c15440b 2848
30b87ab4 2849 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
1a58e196 2850
1d062b7b
SN		 2851 spin_lock_init(&lif->adminq_lock);
2852
2a654540
SN		 2853 spin_lock_init(&lif->deferred.lock);
2854 INIT_LIST_HEAD(&lif->deferred.list);
2855 INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2856
1a58e196
SN		 2857 /* allocate lif info */
2858 lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2859 lif->info = dma_alloc_coherent(dev, lif->info_sz,
2860 &lif->info_pa, GFP_KERNEL);
2861 if (!lif->info) {
2862 dev_err(dev, "Failed to allocate lif info, aborting\n");
2863 err = -ENOMEM;
2864 goto err_out_free_netdev;
2865 }
2866
2a8c2c1a
SN		 2867 ionic_debugfs_add_lif(lif);
2868
30b87ab4
SN		 2869 /* allocate control queues and txrx queue arrays */
2870 ionic_lif_queue_identify(lif);
1d062b7b
SN		 2871 err = ionic_qcqs_alloc(lif);
2872 if (err)
2873 goto err_out_free_lif_info;
2874
aa319881
SN		 2875 /* allocate rss indirection table */
2876 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2877 lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2878 lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2879 &lif->rss_ind_tbl_pa,
2880 GFP_KERNEL);
2881
2882 if (!lif->rss_ind_tbl) {
73a63ee9 2883 err = -ENOMEM;
aa319881
SN		 2884 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2885 goto err_out_free_qcqs;
2886 }
ffac2027 2887 netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
aa319881 2888
f0790bcd
SN		 2889 ionic_lif_alloc_phc(lif);
2890
30b87ab4 2891 return 0;
1a58e196 2892
aa319881
SN		 2893err_out_free_qcqs:
2894 ionic_qcqs_free(lif);
1d062b7b
SN		 2895err_out_free_lif_info:
2896 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2897 lif->info = NULL;
2898 lif->info_pa = 0;
1a58e196
SN		 2899err_out_free_netdev:
2900 free_netdev(lif->netdev);
2901 lif = NULL;
4b1debbe 2902err_out_free_lid:
4b03b273 2903 kfree(lid);
1a58e196 2904
30b87ab4 2905 return err;
1a58e196
SN		 2906}
2907
2908static void ionic_lif_reset(struct ionic_lif *lif)
2909{
2910 struct ionic_dev *idev = &lif->ionic->idev;
2911
2912 mutex_lock(&lif->ionic->dev_cmd_lock);
2913 ionic_dev_cmd_lif_reset(idev, lif->index);
2914 ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2915 mutex_unlock(&lif->ionic->dev_cmd_lock);
2916}
2917
c672412f
SN		 2918static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2919{
2920 struct ionic *ionic = lif->ionic;
2921
2922 if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2923 return;
2924
2925 dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2926
2927 netif_device_detach(lif->netdev);
2928
2929 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2930 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
0925e9db 2931 mutex_lock(&lif->queue_lock);
c672412f 2932 ionic_stop_queues(lif);
0925e9db 2933 mutex_unlock(&lif->queue_lock);
c672412f
SN		 2934 }
2935
2936 if (netif_running(lif->netdev)) {
2937 ionic_txrx_deinit(lif);
2938 ionic_txrx_free(lif);
2939 }
30b87ab4 2940 ionic_lif_deinit(lif);
6bc977fa 2941 ionic_reset(ionic);
c672412f
SN		 2942 ionic_qcqs_free(lif);
2943
2944 dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2945}
2946
2947static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2948{
2949 struct ionic *ionic = lif->ionic;
2950 int err;
2951
2952 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2953 return;
2954
2955 dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2956
1d53aedc 2957 ionic_init_devinfo(ionic);
a21b5d49
SN		 2958 err = ionic_identify(ionic);
2959 if (err)
2960 goto err_out;
2961 err = ionic_port_identify(ionic);
2962 if (err)
2963 goto err_out;
2964 err = ionic_port_init(ionic);
2965 if (err)
2966 goto err_out;
c672412f
SN		 2967 err = ionic_qcqs_alloc(lif);
2968 if (err)
2969 goto err_out;
2970
30b87ab4 2971 err = ionic_lif_init(lif);
c672412f
SN		 2972 if (err)
2973 goto err_qcqs_free;
2974
2975 if (lif->registered)
2976 ionic_lif_set_netdev_info(lif);
2977
7e4d4759
SN		 2978 ionic_rx_filter_replay(lif);
2979
c672412f
SN		 2980 if (netif_running(lif->netdev)) {
2981 err = ionic_txrx_alloc(lif);
2982 if (err)
2983 goto err_lifs_deinit;
2984
2985 err = ionic_txrx_init(lif);
2986 if (err)
2987 goto err_txrx_free;
2988 }
2989
a8771bfe
SN		 2990 /* restore the hardware timestamping queues */
2991 ionic_lif_hwstamp_set(lif, NULL);
2992
c672412f 2993 clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
25cc5a5f 2994 ionic_link_status_check_request(lif, CAN_SLEEP);
c672412f
SN		 2995 netif_device_attach(lif->netdev);
2996 dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2997
2998 return;
2999
3000err_txrx_free:
3001 ionic_txrx_free(lif);
3002err_lifs_deinit:
30b87ab4 3003 ionic_lif_deinit(lif);
c672412f
SN		 3004err_qcqs_free:
3005 ionic_qcqs_free(lif);
3006err_out:
3007 dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3008}
3009
30b87ab4 3010void ionic_lif_free(struct ionic_lif *lif)
1a58e196
SN		 3011{
3012 struct device *dev = lif->ionic->dev;
3013
f0790bcd
SN		 3014 ionic_lif_free_phc(lif);
3015
aa319881
SN		 3016 /* free rss indirection table */
3017 dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3018 lif->rss_ind_tbl_pa);
3019 lif->rss_ind_tbl = NULL;
3020 lif->rss_ind_tbl_pa = 0;
3021
1d062b7b
SN		 3022 /* free queues */
3023 ionic_qcqs_free(lif);
c672412f
SN		 3024 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3025 ionic_lif_reset(lif);
1a58e196
SN		 3026
3027 /* free lif info */
4b03b273 3028 kfree(lif->identity);
1a58e196
SN		 3029 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3030 lif->info = NULL;
3031 lif->info_pa = 0;
3032
6461b446
SN		 3033 /* unmap doorbell page */
3034 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3035 lif->kern_dbpage = NULL;
3036 kfree(lif->dbid_inuse);
3037 lif->dbid_inuse = NULL;
3038
1a58e196
SN		 3039 /* free netdev & lif */
3040 ionic_debugfs_del_lif(lif);
1a58e196
SN		 3041 free_netdev(lif->netdev);
3042}
3043
30b87ab4 3044void ionic_lif_deinit(struct ionic_lif *lif)
1a58e196 3045{
c672412f 3046 if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
1a58e196
SN		 3047 return;
3048
c672412f
SN		 3049 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3050 cancel_work_sync(&lif->deferred.work);
3051 cancel_work_sync(&lif->tx_timeout_work);
7e4d4759 3052 ionic_rx_filters_deinit(lif);
bdff4666
SN		 3053 if (lif->netdev->features & NETIF_F_RXHASH)
3054 ionic_lif_rss_deinit(lif);
c672412f 3055 }
1a58e196 3056
1d062b7b 3057 napi_disable(&lif->adminqcq->napi);
77ceb68e 3058 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1d062b7b
SN		 3059 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3060
0925e9db 3061 mutex_destroy(&lif->queue_lock);
1a58e196
SN		 3062 ionic_lif_reset(lif);
3063}
3064
1d062b7b
SN		 3065static int ionic_lif_adminq_init(struct ionic_lif *lif)
3066{
3067 struct device *dev = lif->ionic->dev;
3068 struct ionic_q_init_comp comp;
3069 struct ionic_dev *idev;
3070 struct ionic_qcq *qcq;
3071 struct ionic_queue *q;
3072 int err;
3073
3074 idev = &lif->ionic->idev;
3075 qcq = lif->adminqcq;
3076 q = &qcq->q;
3077
3078 mutex_lock(&lif->ionic->dev_cmd_lock);
3079 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3080 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3081 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3082 mutex_unlock(&lif->ionic->dev_cmd_lock);
3083 if (err) {
3084 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3085 return err;
3086 }
3087
3088 q->hw_type = comp.hw_type;
3089 q->hw_index = le32_to_cpu(comp.hw_index);
3090 q->dbval = IONIC_DBELL_QID(q->hw_index);
3091
3092 dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3093 dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3094
3095 netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3096 NAPI_POLL_WEIGHT);
3097
1d062b7b
SN		 3098 napi_enable(&qcq->napi);
3099
3100 if (qcq->flags & IONIC_QCQ_F_INTR)
3101 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3102 IONIC_INTR_MASK_CLEAR);
3103
3104 qcq->flags |= IONIC_QCQ_F_INITED;
3105
1d062b7b
SN		 3106 return 0;
3107}
3108
77ceb68e
SN		 3109static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3110{
3111 struct ionic_qcq *qcq = lif->notifyqcq;
3112 struct device *dev = lif->ionic->dev;
3113 struct ionic_queue *q = &qcq->q;
3114 int err;
3115
3116 struct ionic_admin_ctx ctx = {
3117 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3118 .cmd.q_init = {
3119 .opcode = IONIC_CMD_Q_INIT,
3120 .lif_index = cpu_to_le16(lif->index),
3121 .type = q->type,
5b3f3f2a 3122 .ver = lif->qtype_info[q->type].version,
77ceb68e
SN		 3123 .index = cpu_to_le32(q->index),
3124 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3125 IONIC_QINIT_F_ENA),
3126 .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3127 .pid = cpu_to_le16(q->pid),
3128 .ring_size = ilog2(q->num_descs),
3129 .ring_base = cpu_to_le64(q->base_pa),
3130 }
3131 };
3132
3133 dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3134 dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3135 dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3136 dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3137
3138 err = ionic_adminq_post_wait(lif, &ctx);
3139 if (err)
3140 return err;
3141
c672412f 3142 lif->last_eid = 0;
77ceb68e
SN		 3143 q->hw_type = ctx.comp.q_init.hw_type;
3144 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3145 q->dbval = IONIC_DBELL_QID(q->hw_index);
3146
3147 dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3148 dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3149
3150 /* preset the callback info */
3151 q->info[0].cb_arg = lif;
3152
3153 qcq->flags |= IONIC_QCQ_F_INITED;
3154
77ceb68e
SN		 3155 return 0;
3156}
3157
2a654540
SN		 3158static int ionic_station_set(struct ionic_lif *lif)
3159{
3160 struct net_device *netdev = lif->netdev;
3161 struct ionic_admin_ctx ctx = {
3162 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3163 .cmd.lif_getattr = {
3164 .opcode = IONIC_CMD_LIF_GETATTR,
3165 .index = cpu_to_le16(lif->index),
3166 .attr = IONIC_LIF_ATTR_MAC,
3167 },
3168 };
3169 struct sockaddr addr;
3170 int err;
3171
3172 err = ionic_adminq_post_wait(lif, &ctx);
3173 if (err)
3174 return err;
216902ae
SN		 3175 netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3176 ctx.comp.lif_getattr.mac);
fbb39807
SN		 3177 if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3178 return 0;
3179
f20a4d40
SN		 3180 if (!is_zero_ether_addr(netdev->dev_addr)) {
3181 /* If the netdev mac is non-zero and doesn't match the default
3182 * device address, it was set by something earlier and we're
3183 * likely here again after a fw-upgrade reset. We need to be
3184 * sure the netdev mac is in our filter list.
3185 */
3186 if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3187 netdev->dev_addr))
7c8d008c 3188 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
f20a4d40
SN		 3189 } else {
3190 /* Update the netdev mac with the device's mac */
216902ae
SN		 3191 memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3192 addr.sa_family = AF_INET;
3193 err = eth_prepare_mac_addr_change(netdev, &addr);
3194 if (err) {
3195 netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3196 addr.sa_data, err);
3197 return 0;
3198 }
2a654540 3199
216902ae
SN		 3200 eth_commit_mac_addr_change(netdev, &addr);
3201 }
fbb39807 3202
2a654540
SN		 3203 netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3204 netdev->dev_addr);
7c8d008c 3205 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
2a654540
SN		 3206
3207 return 0;
3208}
3209
30b87ab4 3210int ionic_lif_init(struct ionic_lif *lif)
1a58e196
SN		 3211{
3212 struct ionic_dev *idev = &lif->ionic->idev;
6461b446 3213 struct device *dev = lif->ionic->dev;
1a58e196 3214 struct ionic_lif_init_comp comp;
6461b446 3215 int dbpage_num;
1a58e196
SN		 3216 int err;
3217
1a58e196
SN		 3218 mutex_lock(&lif->ionic->dev_cmd_lock);
3219 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3220 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3221 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3222 mutex_unlock(&lif->ionic->dev_cmd_lock);
3223 if (err)
3224 return err;
3225
3226 lif->hw_index = le16_to_cpu(comp.hw_index);
0925e9db 3227 mutex_init(&lif->queue_lock);
1a58e196 3228
6461b446
SN		 3229 /* now that we have the hw_index we can figure out our doorbell page */
3230 lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3231 if (!lif->dbid_count) {
3232 dev_err(dev, "No doorbell pages, aborting\n");
3233 return -EINVAL;
3234 }
3235
3236 lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3237 if (!lif->dbid_inuse) {
3238 dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3239 return -ENOMEM;
3240 }
3241
3242 /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3243 set_bit(0, lif->dbid_inuse);
3244 lif->kern_pid = 0;
3245
3246 dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3247 lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3248 if (!lif->kern_dbpage) {
3249 dev_err(dev, "Cannot map dbpage, aborting\n");
3250 err = -ENOMEM;
3251 goto err_out_free_dbid;
3252 }
3253
1d062b7b
SN		 3254 err = ionic_lif_adminq_init(lif);
3255 if (err)
3256 goto err_out_adminq_deinit;
3257
77ceb68e
SN		 3258 if (lif->ionic->nnqs_per_lif) {
3259 err = ionic_lif_notifyq_init(lif);
3260 if (err)
3261 goto err_out_notifyq_deinit;
3262 }
3263
beead698
SN		 3264 err = ionic_init_nic_features(lif);
3265 if (err)
3266 goto err_out_notifyq_deinit;
3267
7e4d4759
SN		 3268 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3269 err = ionic_rx_filters_init(lif);
3270 if (err)
3271 goto err_out_notifyq_deinit;
3272 }
c1e329eb 3273
2a654540
SN		 3274 err = ionic_station_set(lif);
3275 if (err)
3276 goto err_out_notifyq_deinit;
3277
0f3154e6
SN		 3278 lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3279
c6d3d73a 3280 set_bit(IONIC_LIF_F_INITED, lif->state);
1a58e196 3281
8c15440b
SN		 3282 INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3283
1a58e196 3284 return 0;
6461b446 3285
77ceb68e
SN		 3286err_out_notifyq_deinit:
3287 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1d062b7b
SN		 3288err_out_adminq_deinit:
3289 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3290 ionic_lif_reset(lif);
3291 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3292 lif->kern_dbpage = NULL;
6461b446
SN		 3293err_out_free_dbid:
3294 kfree(lif->dbid_inuse);
3295 lif->dbid_inuse = NULL;
3296
3297 return err;
1a58e196
SN		 3298}
3299
1a371ea1
SN		 3300static void ionic_lif_notify_work(struct work_struct *ws)
3301{
3302}
3303
3304static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3305{
3306 struct ionic_admin_ctx ctx = {
3307 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3308 .cmd.lif_setattr = {
3309 .opcode = IONIC_CMD_LIF_SETATTR,
3310 .index = cpu_to_le16(lif->index),
3311 .attr = IONIC_LIF_ATTR_NAME,
3312 },
3313 };
3314
3315 strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3316 sizeof(ctx.cmd.lif_setattr.name));
3317
3318 ionic_adminq_post_wait(lif, &ctx);
3319}
3320
3321static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3322{
3323 if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3324 return NULL;
3325
3326 return netdev_priv(netdev);
3327}
3328
3329static int ionic_lif_notify(struct notifier_block *nb,
3330 unsigned long event, void *info)
3331{
3332 struct net_device *ndev = netdev_notifier_info_to_dev(info);
3333 struct ionic *ionic = container_of(nb, struct ionic, nb);
3334 struct ionic_lif *lif = ionic_netdev_lif(ndev);
3335
3336 if (!lif || lif->ionic != ionic)
3337 return NOTIFY_DONE;
3338
3339 switch (event) {
3340 case NETDEV_CHANGENAME:
3341 ionic_lif_set_netdev_info(lif);
3342 break;
3343 }
3344
3345 return NOTIFY_DONE;
3346}
3347
30b87ab4 3348int ionic_lif_register(struct ionic_lif *lif)
beead698
SN		 3349{
3350 int err;
3351
afeefec6
SN		 3352 ionic_lif_register_phc(lif);
3353
30b87ab4 3354 INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
1a371ea1 3355
30b87ab4 3356 lif->ionic->nb.notifier_call = ionic_lif_notify;
1a371ea1 3357
30b87ab4 3358 err = register_netdevice_notifier(&lif->ionic->nb);
1a371ea1 3359 if (err)
30b87ab4 3360 lif->ionic->nb.notifier_call = NULL;
1a371ea1 3361
beead698 3362 /* only register LIF0 for now */
30b87ab4 3363 err = register_netdev(lif->netdev);
beead698 3364 if (err) {
30b87ab4 3365 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
afeefec6 3366 ionic_lif_unregister_phc(lif);
beead698
SN		 3367 return err;
3368 }
f6e428b2 3369
25cc5a5f 3370 ionic_link_status_check_request(lif, CAN_SLEEP);
30b87ab4
SN		 3371 lif->registered = true;
3372 ionic_lif_set_netdev_info(lif);
beead698
SN		 3373
3374 return 0;
3375}
3376
30b87ab4 3377void ionic_lif_unregister(struct ionic_lif *lif)
beead698 3378{
30b87ab4
SN		 3379 if (lif->ionic->nb.notifier_call) {
3380 unregister_netdevice_notifier(&lif->ionic->nb);
3381 cancel_work_sync(&lif->ionic->nb_work);
3382 lif->ionic->nb.notifier_call = NULL;
1a371ea1
SN		 3383 }
3384
30b87ab4
SN		 3385 if (lif->netdev->reg_state == NETREG_REGISTERED)
3386 unregister_netdev(lif->netdev);
f0790bcd 3387
afeefec6
SN		 3388 ionic_lif_unregister_phc(lif);
3389
30b87ab4 3390 lif->registered = false;
beead698
SN		 3391}
3392
5b3f3f2a
SN		 3393static void ionic_lif_queue_identify(struct ionic_lif *lif)
3394{
d701ec32 3395 union ionic_q_identity __iomem *q_ident;
5b3f3f2a 3396 struct ionic *ionic = lif->ionic;
5b3f3f2a
SN		 3397 struct ionic_dev *idev;
3398 int qtype;
3399 int err;
3400
3401 idev = &lif->ionic->idev;
d701ec32 3402 q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
5b3f3f2a
SN		 3403
3404 for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3405 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3406
3407 /* filter out the ones we know about */
3408 switch (qtype) {
3409 case IONIC_QTYPE_ADMINQ:
3410 case IONIC_QTYPE_NOTIFYQ:
3411 case IONIC_QTYPE_RXQ:
3412 case IONIC_QTYPE_TXQ:
3413 break;
3414 default:
3415 continue;
3416 }
3417
3418 memset(qti, 0, sizeof(*qti));
3419
3420 mutex_lock(&ionic->dev_cmd_lock);
3421 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3422 ionic_qtype_versions[qtype]);
3423 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3424 if (!err) {
d701ec32
SN		 3425 qti->version = readb(&q_ident->version);
3426 qti->supported = readb(&q_ident->supported);
3427 qti->features = readq(&q_ident->features);
3428 qti->desc_sz = readw(&q_ident->desc_sz);
3429 qti->comp_sz = readw(&q_ident->comp_sz);
3430 qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
3431 qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3432 qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
5b3f3f2a
SN		 3433 }
3434 mutex_unlock(&ionic->dev_cmd_lock);
3435
3436 if (err == -EINVAL) {
3437 dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3438 continue;
3439 } else if (err == -EIO) {
3440 dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3441 return;
3442 } else if (err) {
3443 dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3444 qtype, err);
3445 return;
3446 }
3447
3448 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3449 qtype, qti->version);
3450 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3451 qtype, qti->supported);
3452 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3453 qtype, qti->features);
3454 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3455 qtype, qti->desc_sz);
3456 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3457 qtype, qti->comp_sz);
3458 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3459 qtype, qti->sg_desc_sz);
3460 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3461 qtype, qti->max_sg_elems);
3462 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3463 qtype, qti->sg_desc_stride);
3464 }
3465}
3466
1a58e196
SN		 3467int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3468 union ionic_lif_identity *lid)
3469{
3470 struct ionic_dev *idev = &ionic->idev;
3471 size_t sz;
3472 int err;
3473
3474 sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3475
3476 mutex_lock(&ionic->dev_cmd_lock);
3477 ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3478 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3479 memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3480 mutex_unlock(&ionic->dev_cmd_lock);
3481 if (err)
3482 return (err);
3483
3484 dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3485 le64_to_cpu(lid->capabilities));
3486
3487 dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3488 le32_to_cpu(lid->eth.max_ucast_filters));
3489 dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3490 le32_to_cpu(lid->eth.max_mcast_filters));
3491 dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3492 le64_to_cpu(lid->eth.config.features));
3493 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3494 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3495 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3496 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3497 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3498 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3499 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3500 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3501 dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3502 dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3503 dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3504 le32_to_cpu(lid->eth.config.mtu));
3505
3506 return 0;
3507}
3508
30b87ab4 3509int ionic_lif_size(struct ionic *ionic)
1a58e196
SN		 3510{
3511 struct ionic_identity *ident = &ionic->ident;
3512 unsigned int nintrs, dev_nintrs;
3513 union ionic_lif_config *lc;
3514 unsigned int ntxqs_per_lif;
3515 unsigned int nrxqs_per_lif;
3516 unsigned int neqs_per_lif;
3517 unsigned int nnqs_per_lif;
3518 unsigned int nxqs, neqs;
3519 unsigned int min_intrs;
3520 int err;
3521
3522 lc = &ident->lif.eth.config;
3523 dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3524 neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3525 nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3526 ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3527 nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3528
f0790bcd
SN		 3529 /* reserve last queue id for hardware timestamping */
3530 if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3531 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3532 lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3533 } else {
3534 ntxqs_per_lif -= 1;
3535 nrxqs_per_lif -= 1;
3536 }
3537 }
3538
1a58e196
SN		 3539 nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3540 nxqs = min(nxqs, num_online_cpus());
3541 neqs = min(neqs_per_lif, num_online_cpus());
3542
3543try_again:
3544 /* interrupt usage:
3545 * 1 for master lif adminq/notifyq
3546 * 1 for each CPU for master lif TxRx queue pairs
3547 * whatever's left is for RDMA queues
3548 */
3549 nintrs = 1 + nxqs + neqs;
3550 min_intrs = 2; /* adminq + 1 TxRx queue pair */
3551
3552 if (nintrs > dev_nintrs)
3553 goto try_fewer;
3554
3555 err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3556 if (err < 0 && err != -ENOSPC) {
3557 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3558 return err;
3559 }
3560 if (err == -ENOSPC)
3561 goto try_fewer;
3562
3563 if (err != nintrs) {
3564 ionic_bus_free_irq_vectors(ionic);
3565 goto try_fewer;
3566 }
3567
3568 ionic->nnqs_per_lif = nnqs_per_lif;
3569 ionic->neqs_per_lif = neqs;
3570 ionic->ntxqs_per_lif = nxqs;
3571 ionic->nrxqs_per_lif = nxqs;
3572 ionic->nintrs = nintrs;
3573
3574 ionic_debugfs_add_sizes(ionic);
3575
3576 return 0;
3577
3578try_fewer:
3579 if (nnqs_per_lif > 1) {
3580 nnqs_per_lif >>= 1;
3581 goto try_again;
3582 }
3583 if (neqs > 1) {
3584 neqs >>= 1;
3585 goto try_again;
3586 }
3587 if (nxqs > 1) {
3588 nxqs >>= 1;
3589 goto try_again;
3590 }
3591 dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3592 return -ENOSPC;
3593}
Ubuntu Jammy Linux kernel mirror