1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14 #include <linux/crash_dump.h>
17 #include "ionic_bus.h"
18 #include "ionic_lif.h"
19 #include "ionic_txrx.h"
20 #include "ionic_ethtool.h"
21 #include "ionic_debugfs.h"
23 /* queuetype support level */
24 static const u8 ionic_qtype_versions
[IONIC_QTYPE_MAX
] = {
25 [IONIC_QTYPE_ADMINQ
] = 0, /* 0 = Base version with CQ support */
26 [IONIC_QTYPE_NOTIFYQ
] = 0, /* 0 = Base version */
27 [IONIC_QTYPE_RXQ
] = 0, /* 0 = Base version with CQ+SG support */
28 [IONIC_QTYPE_TXQ
] = 1, /* 0 = Base version with CQ+SG support
29 * 1 = ... with Tx SG version 1
33 static void ionic_link_status_check(struct ionic_lif
*lif
);
34 static void ionic_lif_handle_fw_down(struct ionic_lif
*lif
);
35 static void ionic_lif_handle_fw_up(struct ionic_lif
*lif
);
36 static void ionic_lif_set_netdev_info(struct ionic_lif
*lif
);
38 static void ionic_txrx_deinit(struct ionic_lif
*lif
);
39 static int ionic_txrx_init(struct ionic_lif
*lif
);
40 static int ionic_start_queues(struct ionic_lif
*lif
);
41 static void ionic_stop_queues(struct ionic_lif
*lif
);
42 static void ionic_lif_queue_identify(struct ionic_lif
*lif
);
44 static void ionic_dim_work(struct work_struct
*work
)
46 struct dim
*dim
= container_of(work
, struct dim
, work
);
47 struct dim_cq_moder cur_moder
;
48 struct ionic_qcq
*qcq
;
51 cur_moder
= net_dim_get_rx_moderation(dim
->mode
, dim
->profile_ix
);
52 qcq
= container_of(dim
, struct ionic_qcq
, dim
);
53 new_coal
= ionic_coal_usec_to_hw(qcq
->q
.lif
->ionic
, cur_moder
.usec
);
54 new_coal
= new_coal
? new_coal
: 1;
56 if (qcq
->intr
.dim_coal_hw
!= new_coal
) {
57 unsigned int qi
= qcq
->cq
.bound_q
->index
;
58 struct ionic_lif
*lif
= qcq
->q
.lif
;
60 qcq
->intr
.dim_coal_hw
= new_coal
;
62 ionic_intr_coal_init(lif
->ionic
->idev
.intr_ctrl
,
63 lif
->rxqcqs
[qi
]->intr
.index
,
64 qcq
->intr
.dim_coal_hw
);
67 dim
->state
= DIM_START_MEASURE
;
70 static void ionic_lif_deferred_work(struct work_struct
*work
)
72 struct ionic_lif
*lif
= container_of(work
, struct ionic_lif
, deferred
.work
);
73 struct ionic_deferred
*def
= &lif
->deferred
;
74 struct ionic_deferred_work
*w
= NULL
;
77 spin_lock_bh(&def
->lock
);
78 if (!list_empty(&def
->list
)) {
79 w
= list_first_entry(&def
->list
,
80 struct ionic_deferred_work
, list
);
83 spin_unlock_bh(&def
->lock
);
89 case IONIC_DW_TYPE_RX_MODE
:
90 ionic_lif_rx_mode(lif
);
92 case IONIC_DW_TYPE_LINK_STATUS
:
93 ionic_link_status_check(lif
);
95 case IONIC_DW_TYPE_LIF_RESET
:
97 ionic_lif_handle_fw_up(lif
);
99 ionic_lif_handle_fw_down(lif
);
101 /* Fire off another watchdog to see
102 * if the FW is already back rather than
103 * waiting another whole cycle
105 mod_timer(&lif
->ionic
->watchdog_timer
, jiffies
+ 1);
116 void ionic_lif_deferred_enqueue(struct ionic_deferred
*def
,
117 struct ionic_deferred_work
*work
)
119 spin_lock_bh(&def
->lock
);
120 list_add_tail(&work
->list
, &def
->list
);
121 spin_unlock_bh(&def
->lock
);
122 schedule_work(&def
->work
);
125 static void ionic_link_status_check(struct ionic_lif
*lif
)
127 struct net_device
*netdev
= lif
->netdev
;
131 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED
, lif
->state
))
134 /* Don't put carrier back up if we're in a broken state */
135 if (test_bit(IONIC_LIF_F_BROKEN
, lif
->state
)) {
136 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED
, lif
->state
);
140 link_status
= le16_to_cpu(lif
->info
->status
.link_status
);
141 link_up
= link_status
== IONIC_PORT_OPER_STATUS_UP
;
146 if (netdev
->flags
& IFF_UP
&& netif_running(netdev
)) {
147 mutex_lock(&lif
->queue_lock
);
148 err
= ionic_start_queues(lif
);
149 if (err
&& err
!= -EBUSY
) {
150 netdev_err(lif
->netdev
,
151 "Failed to start queues: %d\n", err
);
152 set_bit(IONIC_LIF_F_BROKEN
, lif
->state
);
153 netif_carrier_off(lif
->netdev
);
155 mutex_unlock(&lif
->queue_lock
);
158 if (!err
&& !netif_carrier_ok(netdev
)) {
159 ionic_port_identify(lif
->ionic
);
160 netdev_info(netdev
, "Link up - %d Gbps\n",
161 le32_to_cpu(lif
->info
->status
.link_speed
) / 1000);
162 netif_carrier_on(netdev
);
165 if (netif_carrier_ok(netdev
)) {
166 netdev_info(netdev
, "Link down\n");
167 netif_carrier_off(netdev
);
170 if (netdev
->flags
& IFF_UP
&& netif_running(netdev
)) {
171 mutex_lock(&lif
->queue_lock
);
172 ionic_stop_queues(lif
);
173 mutex_unlock(&lif
->queue_lock
);
177 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED
, lif
->state
);
180 void ionic_link_status_check_request(struct ionic_lif
*lif
, bool can_sleep
)
182 struct ionic_deferred_work
*work
;
184 /* we only need one request outstanding at a time */
185 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED
, lif
->state
))
189 work
= kzalloc(sizeof(*work
), GFP_ATOMIC
);
191 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED
, lif
->state
);
195 work
->type
= IONIC_DW_TYPE_LINK_STATUS
;
196 ionic_lif_deferred_enqueue(&lif
->deferred
, work
);
198 ionic_link_status_check(lif
);
202 static irqreturn_t
ionic_isr(int irq
, void *data
)
204 struct napi_struct
*napi
= data
;
206 napi_schedule_irqoff(napi
);
211 static int ionic_request_irq(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
213 struct ionic_intr_info
*intr
= &qcq
->intr
;
214 struct device
*dev
= lif
->ionic
->dev
;
215 struct ionic_queue
*q
= &qcq
->q
;
219 name
= lif
->netdev
->name
;
221 name
= dev_name(dev
);
223 snprintf(intr
->name
, sizeof(intr
->name
),
224 "%s-%s-%s", IONIC_DRV_NAME
, name
, q
->name
);
226 return devm_request_irq(dev
, intr
->vector
, ionic_isr
,
227 0, intr
->name
, &qcq
->napi
);
230 static int ionic_intr_alloc(struct ionic_lif
*lif
, struct ionic_intr_info
*intr
)
232 struct ionic
*ionic
= lif
->ionic
;
235 index
= find_first_zero_bit(ionic
->intrs
, ionic
->nintrs
);
236 if (index
== ionic
->nintrs
) {
237 netdev_warn(lif
->netdev
, "%s: no intr, index=%d nintrs=%d\n",
238 __func__
, index
, ionic
->nintrs
);
242 set_bit(index
, ionic
->intrs
);
243 ionic_intr_init(&ionic
->idev
, intr
, index
);
248 static void ionic_intr_free(struct ionic
*ionic
, int index
)
250 if (index
!= IONIC_INTR_INDEX_NOT_ASSIGNED
&& index
< ionic
->nintrs
)
251 clear_bit(index
, ionic
->intrs
);
254 static int ionic_qcq_enable(struct ionic_qcq
*qcq
)
256 struct ionic_queue
*q
= &qcq
->q
;
257 struct ionic_lif
*lif
= q
->lif
;
258 struct ionic_dev
*idev
;
261 struct ionic_admin_ctx ctx
= {
262 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
264 .opcode
= IONIC_CMD_Q_CONTROL
,
265 .lif_index
= cpu_to_le16(lif
->index
),
267 .index
= cpu_to_le32(q
->index
),
268 .oper
= IONIC_Q_ENABLE
,
272 idev
= &lif
->ionic
->idev
;
273 dev
= lif
->ionic
->dev
;
275 dev_dbg(dev
, "q_enable.index %d q_enable.qtype %d\n",
276 ctx
.cmd
.q_control
.index
, ctx
.cmd
.q_control
.type
);
278 if (qcq
->flags
& IONIC_QCQ_F_INTR
) {
279 irq_set_affinity_hint(qcq
->intr
.vector
,
280 &qcq
->intr
.affinity_mask
);
281 napi_enable(&qcq
->napi
);
282 ionic_intr_clean(idev
->intr_ctrl
, qcq
->intr
.index
);
283 ionic_intr_mask(idev
->intr_ctrl
, qcq
->intr
.index
,
284 IONIC_INTR_MASK_CLEAR
);
287 return ionic_adminq_post_wait(lif
, &ctx
);
290 static int ionic_qcq_disable(struct ionic_qcq
*qcq
, bool send_to_hw
)
292 struct ionic_queue
*q
;
293 struct ionic_lif
*lif
;
296 struct ionic_admin_ctx ctx
= {
297 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
299 .opcode
= IONIC_CMD_Q_CONTROL
,
300 .oper
= IONIC_Q_DISABLE
,
310 if (qcq
->flags
& IONIC_QCQ_F_INTR
) {
311 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
313 cancel_work_sync(&qcq
->dim
.work
);
314 ionic_intr_mask(idev
->intr_ctrl
, qcq
->intr
.index
,
315 IONIC_INTR_MASK_SET
);
316 synchronize_irq(qcq
->intr
.vector
);
317 irq_set_affinity_hint(qcq
->intr
.vector
, NULL
);
318 napi_disable(&qcq
->napi
);
322 ctx
.cmd
.q_control
.lif_index
= cpu_to_le16(lif
->index
);
323 ctx
.cmd
.q_control
.type
= q
->type
;
324 ctx
.cmd
.q_control
.index
= cpu_to_le32(q
->index
);
325 dev_dbg(lif
->ionic
->dev
, "q_disable.index %d q_disable.qtype %d\n",
326 ctx
.cmd
.q_control
.index
, ctx
.cmd
.q_control
.type
);
328 err
= ionic_adminq_post_wait(lif
, &ctx
);
334 static void ionic_lif_qcq_deinit(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
336 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
341 if (!(qcq
->flags
& IONIC_QCQ_F_INITED
))
344 if (qcq
->flags
& IONIC_QCQ_F_INTR
) {
345 ionic_intr_mask(idev
->intr_ctrl
, qcq
->intr
.index
,
346 IONIC_INTR_MASK_SET
);
347 netif_napi_del(&qcq
->napi
);
350 qcq
->flags
&= ~IONIC_QCQ_F_INITED
;
353 static void ionic_qcq_intr_free(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
355 if (!(qcq
->flags
& IONIC_QCQ_F_INTR
) || qcq
->intr
.vector
== 0)
358 irq_set_affinity_hint(qcq
->intr
.vector
, NULL
);
359 devm_free_irq(lif
->ionic
->dev
, qcq
->intr
.vector
, &qcq
->napi
);
360 qcq
->intr
.vector
= 0;
361 ionic_intr_free(lif
->ionic
, qcq
->intr
.index
);
362 qcq
->intr
.index
= IONIC_INTR_INDEX_NOT_ASSIGNED
;
365 static void ionic_qcq_free(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
367 struct device
*dev
= lif
->ionic
->dev
;
372 ionic_debugfs_del_qcq(qcq
);
375 dma_free_coherent(dev
, qcq
->q_size
, qcq
->q_base
, qcq
->q_base_pa
);
381 dma_free_coherent(dev
, qcq
->cq_size
, qcq
->cq_base
, qcq
->cq_base_pa
);
387 dma_free_coherent(dev
, qcq
->sg_size
, qcq
->sg_base
, qcq
->sg_base_pa
);
392 ionic_qcq_intr_free(lif
, qcq
);
395 devm_kfree(dev
, qcq
->cq
.info
);
399 devm_kfree(dev
, qcq
->q
.info
);
404 static void ionic_qcqs_free(struct ionic_lif
*lif
)
406 struct device
*dev
= lif
->ionic
->dev
;
407 struct ionic_qcq
*adminqcq
;
408 unsigned long irqflags
;
410 if (lif
->notifyqcq
) {
411 ionic_qcq_free(lif
, lif
->notifyqcq
);
412 devm_kfree(dev
, lif
->notifyqcq
);
413 lif
->notifyqcq
= NULL
;
417 spin_lock_irqsave(&lif
->adminq_lock
, irqflags
);
418 adminqcq
= READ_ONCE(lif
->adminqcq
);
419 lif
->adminqcq
= NULL
;
420 spin_unlock_irqrestore(&lif
->adminq_lock
, irqflags
);
422 ionic_qcq_free(lif
, adminqcq
);
423 devm_kfree(dev
, adminqcq
);
428 devm_kfree(dev
, lif
->rxqstats
);
429 lif
->rxqstats
= NULL
;
430 devm_kfree(dev
, lif
->rxqcqs
);
435 devm_kfree(dev
, lif
->txqstats
);
436 lif
->txqstats
= NULL
;
437 devm_kfree(dev
, lif
->txqcqs
);
442 static void ionic_link_qcq_interrupts(struct ionic_qcq
*src_qcq
,
443 struct ionic_qcq
*n_qcq
)
445 if (WARN_ON(n_qcq
->flags
& IONIC_QCQ_F_INTR
)) {
446 ionic_intr_free(n_qcq
->cq
.lif
->ionic
, n_qcq
->intr
.index
);
447 n_qcq
->flags
&= ~IONIC_QCQ_F_INTR
;
450 n_qcq
->intr
.vector
= src_qcq
->intr
.vector
;
451 n_qcq
->intr
.index
= src_qcq
->intr
.index
;
454 static int ionic_alloc_qcq_interrupt(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
458 if (!(qcq
->flags
& IONIC_QCQ_F_INTR
)) {
459 qcq
->intr
.index
= IONIC_INTR_INDEX_NOT_ASSIGNED
;
463 err
= ionic_intr_alloc(lif
, &qcq
->intr
);
465 netdev_warn(lif
->netdev
, "no intr for %s: %d\n",
470 err
= ionic_bus_get_irq(lif
->ionic
, qcq
->intr
.index
);
472 netdev_warn(lif
->netdev
, "no vector for %s: %d\n",
474 goto err_out_free_intr
;
476 qcq
->intr
.vector
= err
;
477 ionic_intr_mask_assert(lif
->ionic
->idev
.intr_ctrl
, qcq
->intr
.index
,
478 IONIC_INTR_MASK_SET
);
480 err
= ionic_request_irq(lif
, qcq
);
482 netdev_warn(lif
->netdev
, "irq request failed %d\n", err
);
483 goto err_out_free_intr
;
486 /* try to get the irq on the local numa node first */
487 qcq
->intr
.cpu
= cpumask_local_spread(qcq
->intr
.index
,
488 dev_to_node(lif
->ionic
->dev
));
489 if (qcq
->intr
.cpu
!= -1)
490 cpumask_set_cpu(qcq
->intr
.cpu
, &qcq
->intr
.affinity_mask
);
492 netdev_dbg(lif
->netdev
, "%s: Interrupt index %d\n", qcq
->q
.name
, qcq
->intr
.index
);
496 ionic_intr_free(lif
->ionic
, qcq
->intr
.index
);
501 static int ionic_qcq_alloc(struct ionic_lif
*lif
, unsigned int type
,
503 const char *name
, unsigned int flags
,
504 unsigned int num_descs
, unsigned int desc_size
,
505 unsigned int cq_desc_size
,
506 unsigned int sg_desc_size
,
507 unsigned int pid
, struct ionic_qcq
**qcq
)
509 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
510 struct device
*dev
= lif
->ionic
->dev
;
511 void *q_base
, *cq_base
, *sg_base
;
512 dma_addr_t cq_base_pa
= 0;
513 dma_addr_t sg_base_pa
= 0;
514 dma_addr_t q_base_pa
= 0;
515 struct ionic_qcq
*new;
520 new = devm_kzalloc(dev
, sizeof(*new), GFP_KERNEL
);
522 netdev_err(lif
->netdev
, "Cannot allocate queue structure\n");
530 new->q
.info
= devm_kcalloc(dev
, num_descs
, sizeof(*new->q
.info
),
533 netdev_err(lif
->netdev
, "Cannot allocate queue info\n");
535 goto err_out_free_qcq
;
539 new->q
.max_sg_elems
= lif
->qtype_info
[type
].max_sg_elems
;
541 err
= ionic_q_init(lif
, idev
, &new->q
, index
, name
, num_descs
,
542 desc_size
, sg_desc_size
, pid
);
544 netdev_err(lif
->netdev
, "Cannot initialize queue\n");
545 goto err_out_free_q_info
;
548 err
= ionic_alloc_qcq_interrupt(lif
, new);
552 new->cq
.info
= devm_kcalloc(dev
, num_descs
, sizeof(*new->cq
.info
),
555 netdev_err(lif
->netdev
, "Cannot allocate completion queue info\n");
557 goto err_out_free_irq
;
560 err
= ionic_cq_init(lif
, &new->cq
, &new->intr
, num_descs
, cq_desc_size
);
562 netdev_err(lif
->netdev
, "Cannot initialize completion queue\n");
563 goto err_out_free_cq_info
;
566 if (flags
& IONIC_QCQ_F_NOTIFYQ
) {
569 /* q & cq need to be contiguous in case of notifyq */
570 q_size
= ALIGN(num_descs
* desc_size
, PAGE_SIZE
);
571 cq_size
= ALIGN(num_descs
* cq_desc_size
, PAGE_SIZE
);
573 new->q_size
= PAGE_SIZE
+ q_size
+ cq_size
;
574 new->q_base
= dma_alloc_coherent(dev
, new->q_size
,
575 &new->q_base_pa
, GFP_KERNEL
);
577 netdev_err(lif
->netdev
, "Cannot allocate qcq DMA memory\n");
579 goto err_out_free_cq_info
;
581 q_base
= PTR_ALIGN(new->q_base
, PAGE_SIZE
);
582 q_base_pa
= ALIGN(new->q_base_pa
, PAGE_SIZE
);
583 ionic_q_map(&new->q
, q_base
, q_base_pa
);
585 cq_base
= PTR_ALIGN(q_base
+ q_size
, PAGE_SIZE
);
586 cq_base_pa
= ALIGN(new->q_base_pa
+ q_size
, PAGE_SIZE
);
587 ionic_cq_map(&new->cq
, cq_base
, cq_base_pa
);
588 ionic_cq_bind(&new->cq
, &new->q
);
590 new->q_size
= PAGE_SIZE
+ (num_descs
* desc_size
);
591 new->q_base
= dma_alloc_coherent(dev
, new->q_size
, &new->q_base_pa
,
594 netdev_err(lif
->netdev
, "Cannot allocate queue DMA memory\n");
596 goto err_out_free_cq_info
;
598 q_base
= PTR_ALIGN(new->q_base
, PAGE_SIZE
);
599 q_base_pa
= ALIGN(new->q_base_pa
, PAGE_SIZE
);
600 ionic_q_map(&new->q
, q_base
, q_base_pa
);
602 new->cq_size
= PAGE_SIZE
+ (num_descs
* cq_desc_size
);
603 new->cq_base
= dma_alloc_coherent(dev
, new->cq_size
, &new->cq_base_pa
,
606 netdev_err(lif
->netdev
, "Cannot allocate cq DMA memory\n");
610 cq_base
= PTR_ALIGN(new->cq_base
, PAGE_SIZE
);
611 cq_base_pa
= ALIGN(new->cq_base_pa
, PAGE_SIZE
);
612 ionic_cq_map(&new->cq
, cq_base
, cq_base_pa
);
613 ionic_cq_bind(&new->cq
, &new->q
);
616 if (flags
& IONIC_QCQ_F_SG
) {
617 new->sg_size
= PAGE_SIZE
+ (num_descs
* sg_desc_size
);
618 new->sg_base
= dma_alloc_coherent(dev
, new->sg_size
, &new->sg_base_pa
,
621 netdev_err(lif
->netdev
, "Cannot allocate sg DMA memory\n");
623 goto err_out_free_cq
;
625 sg_base
= PTR_ALIGN(new->sg_base
, PAGE_SIZE
);
626 sg_base_pa
= ALIGN(new->sg_base_pa
, PAGE_SIZE
);
627 ionic_q_sg_map(&new->q
, sg_base
, sg_base_pa
);
630 INIT_WORK(&new->dim
.work
, ionic_dim_work
);
631 new->dim
.mode
= DIM_CQ_PERIOD_MODE_START_FROM_EQE
;
638 dma_free_coherent(dev
, new->cq_size
, new->cq_base
, new->cq_base_pa
);
640 dma_free_coherent(dev
, new->q_size
, new->q_base
, new->q_base_pa
);
641 err_out_free_cq_info
:
642 devm_kfree(dev
, new->cq
.info
);
644 if (flags
& IONIC_QCQ_F_INTR
) {
645 devm_free_irq(dev
, new->intr
.vector
, &new->napi
);
646 ionic_intr_free(lif
->ionic
, new->intr
.index
);
649 devm_kfree(dev
, new->q
.info
);
651 devm_kfree(dev
, new);
653 dev_err(dev
, "qcq alloc of %s%d failed %d\n", name
, index
, err
);
657 static int ionic_qcqs_alloc(struct ionic_lif
*lif
)
659 struct device
*dev
= lif
->ionic
->dev
;
663 flags
= IONIC_QCQ_F_INTR
;
664 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_ADMINQ
, 0, "admin", flags
,
666 sizeof(struct ionic_admin_cmd
),
667 sizeof(struct ionic_admin_comp
),
668 0, lif
->kern_pid
, &lif
->adminqcq
);
671 ionic_debugfs_add_qcq(lif
, lif
->adminqcq
);
673 if (lif
->ionic
->nnqs_per_lif
) {
674 flags
= IONIC_QCQ_F_NOTIFYQ
;
675 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_NOTIFYQ
, 0, "notifyq",
676 flags
, IONIC_NOTIFYQ_LENGTH
,
677 sizeof(struct ionic_notifyq_cmd
),
678 sizeof(union ionic_notifyq_comp
),
679 0, lif
->kern_pid
, &lif
->notifyqcq
);
682 ionic_debugfs_add_qcq(lif
, lif
->notifyqcq
);
684 /* Let the notifyq ride on the adminq interrupt */
685 ionic_link_qcq_interrupts(lif
->adminqcq
, lif
->notifyqcq
);
689 lif
->txqcqs
= devm_kcalloc(dev
, lif
->ionic
->ntxqs_per_lif
,
690 sizeof(*lif
->txqcqs
), GFP_KERNEL
);
693 lif
->rxqcqs
= devm_kcalloc(dev
, lif
->ionic
->nrxqs_per_lif
,
694 sizeof(*lif
->rxqcqs
), GFP_KERNEL
);
698 lif
->txqstats
= devm_kcalloc(dev
, lif
->ionic
->ntxqs_per_lif
+ 1,
699 sizeof(*lif
->txqstats
), GFP_KERNEL
);
702 lif
->rxqstats
= devm_kcalloc(dev
, lif
->ionic
->nrxqs_per_lif
+ 1,
703 sizeof(*lif
->rxqstats
), GFP_KERNEL
);
710 ionic_qcqs_free(lif
);
714 static void ionic_qcq_sanitize(struct ionic_qcq
*qcq
)
718 qcq
->cq
.tail_idx
= 0;
719 qcq
->cq
.done_color
= 1;
720 memset(qcq
->q_base
, 0, qcq
->q_size
);
721 memset(qcq
->cq_base
, 0, qcq
->cq_size
);
722 memset(qcq
->sg_base
, 0, qcq
->sg_size
);
725 static int ionic_lif_txq_init(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
727 struct device
*dev
= lif
->ionic
->dev
;
728 struct ionic_queue
*q
= &qcq
->q
;
729 struct ionic_cq
*cq
= &qcq
->cq
;
730 struct ionic_admin_ctx ctx
= {
731 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
733 .opcode
= IONIC_CMD_Q_INIT
,
734 .lif_index
= cpu_to_le16(lif
->index
),
736 .ver
= lif
->qtype_info
[q
->type
].version
,
737 .index
= cpu_to_le32(q
->index
),
738 .flags
= cpu_to_le16(IONIC_QINIT_F_IRQ
|
740 .pid
= cpu_to_le16(q
->pid
),
741 .ring_size
= ilog2(q
->num_descs
),
742 .ring_base
= cpu_to_le64(q
->base_pa
),
743 .cq_ring_base
= cpu_to_le64(cq
->base_pa
),
744 .sg_ring_base
= cpu_to_le64(q
->sg_base_pa
),
745 .features
= cpu_to_le64(q
->features
),
748 unsigned int intr_index
;
751 intr_index
= qcq
->intr
.index
;
753 ctx
.cmd
.q_init
.intr_index
= cpu_to_le16(intr_index
);
755 dev_dbg(dev
, "txq_init.pid %d\n", ctx
.cmd
.q_init
.pid
);
756 dev_dbg(dev
, "txq_init.index %d\n", ctx
.cmd
.q_init
.index
);
757 dev_dbg(dev
, "txq_init.ring_base 0x%llx\n", ctx
.cmd
.q_init
.ring_base
);
758 dev_dbg(dev
, "txq_init.ring_size %d\n", ctx
.cmd
.q_init
.ring_size
);
759 dev_dbg(dev
, "txq_init.flags 0x%x\n", ctx
.cmd
.q_init
.flags
);
760 dev_dbg(dev
, "txq_init.ver %d\n", ctx
.cmd
.q_init
.ver
);
761 dev_dbg(dev
, "txq_init.intr_index %d\n", ctx
.cmd
.q_init
.intr_index
);
763 ionic_qcq_sanitize(qcq
);
765 err
= ionic_adminq_post_wait(lif
, &ctx
);
769 q
->hw_type
= ctx
.comp
.q_init
.hw_type
;
770 q
->hw_index
= le32_to_cpu(ctx
.comp
.q_init
.hw_index
);
771 q
->dbval
= IONIC_DBELL_QID(q
->hw_index
);
773 dev_dbg(dev
, "txq->hw_type %d\n", q
->hw_type
);
774 dev_dbg(dev
, "txq->hw_index %d\n", q
->hw_index
);
776 if (test_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
))
777 netif_napi_add(lif
->netdev
, &qcq
->napi
, ionic_tx_napi
,
780 qcq
->flags
|= IONIC_QCQ_F_INITED
;
785 static int ionic_lif_rxq_init(struct ionic_lif
*lif
, struct ionic_qcq
*qcq
)
787 struct device
*dev
= lif
->ionic
->dev
;
788 struct ionic_queue
*q
= &qcq
->q
;
789 struct ionic_cq
*cq
= &qcq
->cq
;
790 struct ionic_admin_ctx ctx
= {
791 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
793 .opcode
= IONIC_CMD_Q_INIT
,
794 .lif_index
= cpu_to_le16(lif
->index
),
796 .ver
= lif
->qtype_info
[q
->type
].version
,
797 .index
= cpu_to_le32(q
->index
),
798 .flags
= cpu_to_le16(IONIC_QINIT_F_IRQ
|
800 .intr_index
= cpu_to_le16(cq
->bound_intr
->index
),
801 .pid
= cpu_to_le16(q
->pid
),
802 .ring_size
= ilog2(q
->num_descs
),
803 .ring_base
= cpu_to_le64(q
->base_pa
),
804 .cq_ring_base
= cpu_to_le64(cq
->base_pa
),
805 .sg_ring_base
= cpu_to_le64(q
->sg_base_pa
),
806 .features
= cpu_to_le64(q
->features
),
811 dev_dbg(dev
, "rxq_init.pid %d\n", ctx
.cmd
.q_init
.pid
);
812 dev_dbg(dev
, "rxq_init.index %d\n", ctx
.cmd
.q_init
.index
);
813 dev_dbg(dev
, "rxq_init.ring_base 0x%llx\n", ctx
.cmd
.q_init
.ring_base
);
814 dev_dbg(dev
, "rxq_init.ring_size %d\n", ctx
.cmd
.q_init
.ring_size
);
815 dev_dbg(dev
, "rxq_init.flags 0x%x\n", ctx
.cmd
.q_init
.flags
);
816 dev_dbg(dev
, "rxq_init.ver %d\n", ctx
.cmd
.q_init
.ver
);
817 dev_dbg(dev
, "rxq_init.intr_index %d\n", ctx
.cmd
.q_init
.intr_index
);
819 ionic_qcq_sanitize(qcq
);
821 err
= ionic_adminq_post_wait(lif
, &ctx
);
825 q
->hw_type
= ctx
.comp
.q_init
.hw_type
;
826 q
->hw_index
= le32_to_cpu(ctx
.comp
.q_init
.hw_index
);
827 q
->dbval
= IONIC_DBELL_QID(q
->hw_index
);
829 dev_dbg(dev
, "rxq->hw_type %d\n", q
->hw_type
);
830 dev_dbg(dev
, "rxq->hw_index %d\n", q
->hw_index
);
832 if (test_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
))
833 netif_napi_add(lif
->netdev
, &qcq
->napi
, ionic_rx_napi
,
836 netif_napi_add(lif
->netdev
, &qcq
->napi
, ionic_txrx_napi
,
839 qcq
->flags
|= IONIC_QCQ_F_INITED
;
844 int ionic_lif_create_hwstamp_txq(struct ionic_lif
*lif
)
846 unsigned int num_desc
, desc_sz
, comp_sz
, sg_desc_sz
;
847 unsigned int txq_i
, flags
;
848 struct ionic_qcq
*txq
;
852 if (lif
->hwstamp_txq
)
855 features
= IONIC_Q_F_2X_CQ_DESC
| IONIC_TXQ_F_HWSTAMP
;
857 num_desc
= IONIC_MIN_TXRX_DESC
;
858 desc_sz
= sizeof(struct ionic_txq_desc
);
859 comp_sz
= 2 * sizeof(struct ionic_txq_comp
);
861 if (lif
->qtype_info
[IONIC_QTYPE_TXQ
].version
>= 1 &&
862 lif
->qtype_info
[IONIC_QTYPE_TXQ
].sg_desc_sz
== sizeof(struct ionic_txq_sg_desc_v1
))
863 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc_v1
);
865 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc
);
867 txq_i
= lif
->ionic
->ntxqs_per_lif
;
868 flags
= IONIC_QCQ_F_TX_STATS
| IONIC_QCQ_F_SG
;
870 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_TXQ
, txq_i
, "hwstamp_tx", flags
,
871 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
872 lif
->kern_pid
, &txq
);
876 txq
->q
.features
= features
;
878 ionic_link_qcq_interrupts(lif
->adminqcq
, txq
);
879 ionic_debugfs_add_qcq(lif
, txq
);
881 lif
->hwstamp_txq
= txq
;
883 if (netif_running(lif
->netdev
)) {
884 err
= ionic_lif_txq_init(lif
, txq
);
888 if (test_bit(IONIC_LIF_F_UP
, lif
->state
)) {
889 err
= ionic_qcq_enable(txq
);
898 ionic_lif_qcq_deinit(lif
, txq
);
900 lif
->hwstamp_txq
= NULL
;
901 ionic_debugfs_del_qcq(txq
);
902 ionic_qcq_free(lif
, txq
);
903 devm_kfree(lif
->ionic
->dev
, txq
);
908 int ionic_lif_create_hwstamp_rxq(struct ionic_lif
*lif
)
910 unsigned int num_desc
, desc_sz
, comp_sz
, sg_desc_sz
;
911 unsigned int rxq_i
, flags
;
912 struct ionic_qcq
*rxq
;
916 if (lif
->hwstamp_rxq
)
919 features
= IONIC_Q_F_2X_CQ_DESC
| IONIC_RXQ_F_HWSTAMP
;
921 num_desc
= IONIC_MIN_TXRX_DESC
;
922 desc_sz
= sizeof(struct ionic_rxq_desc
);
923 comp_sz
= 2 * sizeof(struct ionic_rxq_comp
);
924 sg_desc_sz
= sizeof(struct ionic_rxq_sg_desc
);
926 rxq_i
= lif
->ionic
->nrxqs_per_lif
;
927 flags
= IONIC_QCQ_F_RX_STATS
| IONIC_QCQ_F_SG
;
929 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_RXQ
, rxq_i
, "hwstamp_rx", flags
,
930 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
931 lif
->kern_pid
, &rxq
);
935 rxq
->q
.features
= features
;
937 ionic_link_qcq_interrupts(lif
->adminqcq
, rxq
);
938 ionic_debugfs_add_qcq(lif
, rxq
);
940 lif
->hwstamp_rxq
= rxq
;
942 if (netif_running(lif
->netdev
)) {
943 err
= ionic_lif_rxq_init(lif
, rxq
);
947 if (test_bit(IONIC_LIF_F_UP
, lif
->state
)) {
948 ionic_rx_fill(&rxq
->q
);
949 err
= ionic_qcq_enable(rxq
);
958 ionic_lif_qcq_deinit(lif
, rxq
);
960 lif
->hwstamp_rxq
= NULL
;
961 ionic_debugfs_del_qcq(rxq
);
962 ionic_qcq_free(lif
, rxq
);
963 devm_kfree(lif
->ionic
->dev
, rxq
);
968 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif
*lif
, bool rx_all
)
970 struct ionic_queue_params qparam
;
972 ionic_init_queue_params(lif
, &qparam
);
975 qparam
.rxq_features
= IONIC_Q_F_2X_CQ_DESC
| IONIC_RXQ_F_HWSTAMP
;
977 qparam
.rxq_features
= 0;
979 /* if we're not running, just set the values and return */
980 if (!netif_running(lif
->netdev
)) {
981 lif
->rxq_features
= qparam
.rxq_features
;
985 return ionic_reconfigure_queues(lif
, &qparam
);
988 int ionic_lif_set_hwstamp_txmode(struct ionic_lif
*lif
, u16 txstamp_mode
)
990 struct ionic_admin_ctx ctx
= {
991 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
993 .opcode
= IONIC_CMD_LIF_SETATTR
,
994 .index
= cpu_to_le16(lif
->index
),
995 .attr
= IONIC_LIF_ATTR_TXSTAMP
,
996 .txstamp_mode
= cpu_to_le16(txstamp_mode
),
1000 return ionic_adminq_post_wait(lif
, &ctx
);
1003 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif
*lif
)
1005 struct ionic_admin_ctx ctx
= {
1006 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1007 .cmd
.rx_filter_del
= {
1008 .opcode
= IONIC_CMD_RX_FILTER_DEL
,
1009 .lif_index
= cpu_to_le16(lif
->index
),
1012 struct ionic_rx_filter
*f
;
1016 spin_lock_bh(&lif
->rx_filters
.lock
);
1018 f
= ionic_rx_filter_rxsteer(lif
);
1020 spin_unlock_bh(&lif
->rx_filters
.lock
);
1024 filter_id
= f
->filter_id
;
1025 ionic_rx_filter_free(lif
, f
);
1027 spin_unlock_bh(&lif
->rx_filters
.lock
);
1029 netdev_dbg(lif
->netdev
, "rx_filter del RXSTEER (id %d)\n", filter_id
);
1031 ctx
.cmd
.rx_filter_del
.filter_id
= cpu_to_le32(filter_id
);
1033 err
= ionic_adminq_post_wait(lif
, &ctx
);
1034 if (err
&& err
!= -EEXIST
)
1035 netdev_dbg(lif
->netdev
, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id
);
1038 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif
*lif
, u64 pkt_class
)
1040 struct ionic_admin_ctx ctx
= {
1041 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1042 .cmd
.rx_filter_add
= {
1043 .opcode
= IONIC_CMD_RX_FILTER_ADD
,
1044 .lif_index
= cpu_to_le16(lif
->index
),
1045 .match
= cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS
),
1046 .pkt_class
= cpu_to_le64(pkt_class
),
1053 if (!lif
->hwstamp_rxq
)
1056 qtype
= lif
->hwstamp_rxq
->q
.type
;
1057 ctx
.cmd
.rx_filter_add
.qtype
= qtype
;
1059 qid
= lif
->hwstamp_rxq
->q
.index
;
1060 ctx
.cmd
.rx_filter_add
.qid
= cpu_to_le32(qid
);
1062 netdev_dbg(lif
->netdev
, "rx_filter add RXSTEER\n");
1063 err
= ionic_adminq_post_wait(lif
, &ctx
);
1064 if (err
&& err
!= -EEXIST
)
1067 spin_lock_bh(&lif
->rx_filters
.lock
);
1068 err
= ionic_rx_filter_save(lif
, 0, qid
, 0, &ctx
, IONIC_FILTER_STATE_SYNCED
);
1069 spin_unlock_bh(&lif
->rx_filters
.lock
);
1074 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif
*lif
, u64 pkt_class
)
1076 ionic_lif_del_hwstamp_rxfilt(lif
);
1081 return ionic_lif_add_hwstamp_rxfilt(lif
, pkt_class
);
1084 static bool ionic_notifyq_service(struct ionic_cq
*cq
,
1085 struct ionic_cq_info
*cq_info
)
1087 union ionic_notifyq_comp
*comp
= cq_info
->cq_desc
;
1088 struct ionic_deferred_work
*work
;
1089 struct net_device
*netdev
;
1090 struct ionic_queue
*q
;
1091 struct ionic_lif
*lif
;
1095 lif
= q
->info
[0].cb_arg
;
1096 netdev
= lif
->netdev
;
1097 eid
= le64_to_cpu(comp
->event
.eid
);
1099 /* Have we run out of new completions to process? */
1100 if ((s64
)(eid
- lif
->last_eid
) <= 0)
1103 lif
->last_eid
= eid
;
1105 dev_dbg(lif
->ionic
->dev
, "notifyq event:\n");
1106 dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET
, 16, 1,
1107 comp
, sizeof(*comp
), true);
1109 switch (le16_to_cpu(comp
->event
.ecode
)) {
1110 case IONIC_EVENT_LINK_CHANGE
:
1111 ionic_link_status_check_request(lif
, CAN_NOT_SLEEP
);
1113 case IONIC_EVENT_RESET
:
1114 work
= kzalloc(sizeof(*work
), GFP_ATOMIC
);
1116 netdev_err(lif
->netdev
, "Reset event dropped\n");
1118 work
->type
= IONIC_DW_TYPE_LIF_RESET
;
1119 ionic_lif_deferred_enqueue(&lif
->deferred
, work
);
1123 netdev_warn(netdev
, "Notifyq event ecode=%d eid=%lld\n",
1124 comp
->event
.ecode
, eid
);
1131 static bool ionic_adminq_service(struct ionic_cq
*cq
,
1132 struct ionic_cq_info
*cq_info
)
1134 struct ionic_admin_comp
*comp
= cq_info
->cq_desc
;
1136 if (!color_match(comp
->color
, cq
->done_color
))
1139 ionic_q_service(cq
->bound_q
, cq_info
, le16_to_cpu(comp
->comp_index
));
1144 static int ionic_adminq_napi(struct napi_struct
*napi
, int budget
)
1146 struct ionic_intr_info
*intr
= napi_to_cq(napi
)->bound_intr
;
1147 struct ionic_lif
*lif
= napi_to_cq(napi
)->lif
;
1148 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
1149 unsigned long irqflags
;
1150 unsigned int flags
= 0;
1158 if (lif
->notifyqcq
&& lif
->notifyqcq
->flags
& IONIC_QCQ_F_INITED
)
1159 n_work
= ionic_cq_service(&lif
->notifyqcq
->cq
, budget
,
1160 ionic_notifyq_service
, NULL
, NULL
);
1162 spin_lock_irqsave(&lif
->adminq_lock
, irqflags
);
1163 if (lif
->adminqcq
&& lif
->adminqcq
->flags
& IONIC_QCQ_F_INITED
)
1164 a_work
= ionic_cq_service(&lif
->adminqcq
->cq
, budget
,
1165 ionic_adminq_service
, NULL
, NULL
);
1166 spin_unlock_irqrestore(&lif
->adminq_lock
, irqflags
);
1168 if (lif
->hwstamp_rxq
)
1169 rx_work
= ionic_cq_service(&lif
->hwstamp_rxq
->cq
, budget
,
1170 ionic_rx_service
, NULL
, NULL
);
1172 if (lif
->hwstamp_txq
)
1173 tx_work
= ionic_cq_service(&lif
->hwstamp_txq
->cq
, budget
,
1174 ionic_tx_service
, NULL
, NULL
);
1176 work_done
= max(max(n_work
, a_work
), max(rx_work
, tx_work
));
1177 if (work_done
< budget
&& napi_complete_done(napi
, work_done
)) {
1178 flags
|= IONIC_INTR_CRED_UNMASK
;
1179 intr
->rearm_count
++;
1182 if (work_done
|| flags
) {
1183 flags
|= IONIC_INTR_CRED_RESET_COALESCE
;
1184 credits
= n_work
+ a_work
+ rx_work
+ tx_work
;
1185 ionic_intr_credits(idev
->intr_ctrl
, intr
->index
, credits
, flags
);
1191 void ionic_get_stats64(struct net_device
*netdev
,
1192 struct rtnl_link_stats64
*ns
)
1194 struct ionic_lif
*lif
= netdev_priv(netdev
);
1195 struct ionic_lif_stats
*ls
;
1197 memset(ns
, 0, sizeof(*ns
));
1198 ls
= &lif
->info
->stats
;
1200 ns
->rx_packets
= le64_to_cpu(ls
->rx_ucast_packets
) +
1201 le64_to_cpu(ls
->rx_mcast_packets
) +
1202 le64_to_cpu(ls
->rx_bcast_packets
);
1204 ns
->tx_packets
= le64_to_cpu(ls
->tx_ucast_packets
) +
1205 le64_to_cpu(ls
->tx_mcast_packets
) +
1206 le64_to_cpu(ls
->tx_bcast_packets
);
1208 ns
->rx_bytes
= le64_to_cpu(ls
->rx_ucast_bytes
) +
1209 le64_to_cpu(ls
->rx_mcast_bytes
) +
1210 le64_to_cpu(ls
->rx_bcast_bytes
);
1212 ns
->tx_bytes
= le64_to_cpu(ls
->tx_ucast_bytes
) +
1213 le64_to_cpu(ls
->tx_mcast_bytes
) +
1214 le64_to_cpu(ls
->tx_bcast_bytes
);
1216 ns
->rx_dropped
= le64_to_cpu(ls
->rx_ucast_drop_packets
) +
1217 le64_to_cpu(ls
->rx_mcast_drop_packets
) +
1218 le64_to_cpu(ls
->rx_bcast_drop_packets
);
1220 ns
->tx_dropped
= le64_to_cpu(ls
->tx_ucast_drop_packets
) +
1221 le64_to_cpu(ls
->tx_mcast_drop_packets
) +
1222 le64_to_cpu(ls
->tx_bcast_drop_packets
);
1224 ns
->multicast
= le64_to_cpu(ls
->rx_mcast_packets
);
1226 ns
->rx_over_errors
= le64_to_cpu(ls
->rx_queue_empty
);
1228 ns
->rx_missed_errors
= le64_to_cpu(ls
->rx_dma_error
) +
1229 le64_to_cpu(ls
->rx_queue_disabled
) +
1230 le64_to_cpu(ls
->rx_desc_fetch_error
) +
1231 le64_to_cpu(ls
->rx_desc_data_error
);
1233 ns
->tx_aborted_errors
= le64_to_cpu(ls
->tx_dma_error
) +
1234 le64_to_cpu(ls
->tx_queue_disabled
) +
1235 le64_to_cpu(ls
->tx_desc_fetch_error
) +
1236 le64_to_cpu(ls
->tx_desc_data_error
);
1238 ns
->rx_errors
= ns
->rx_over_errors
+
1239 ns
->rx_missed_errors
;
1241 ns
->tx_errors
= ns
->tx_aborted_errors
;
1244 int ionic_lif_addr_add(struct ionic_lif
*lif
, const u8
*addr
)
1246 struct ionic_admin_ctx ctx
= {
1247 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1248 .cmd
.rx_filter_add
= {
1249 .opcode
= IONIC_CMD_RX_FILTER_ADD
,
1250 .lif_index
= cpu_to_le16(lif
->index
),
1251 .match
= cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC
),
1254 int nfilters
= le32_to_cpu(lif
->identity
->eth
.max_ucast_filters
);
1255 bool mc
= is_multicast_ether_addr(addr
);
1256 struct ionic_rx_filter
*f
;
1259 memcpy(ctx
.cmd
.rx_filter_add
.mac
.addr
, addr
, ETH_ALEN
);
1261 spin_lock_bh(&lif
->rx_filters
.lock
);
1262 f
= ionic_rx_filter_by_addr(lif
, addr
);
1264 /* don't bother if we already have it and it is sync'd */
1265 if (f
->state
== IONIC_FILTER_STATE_SYNCED
) {
1266 spin_unlock_bh(&lif
->rx_filters
.lock
);
1270 /* mark preemptively as sync'd to block any parallel attempts */
1271 f
->state
= IONIC_FILTER_STATE_SYNCED
;
1273 /* save as SYNCED to catch any DEL requests while processing */
1274 err
= ionic_rx_filter_save(lif
, 0, IONIC_RXQ_INDEX_ANY
, 0, &ctx
,
1275 IONIC_FILTER_STATE_SYNCED
);
1277 spin_unlock_bh(&lif
->rx_filters
.lock
);
1281 netdev_dbg(lif
->netdev
, "rx_filter add ADDR %pM\n", addr
);
1283 /* Don't bother with the write to FW if we know there's no room,
1284 * we can try again on the next sync attempt.
1286 if ((lif
->nucast
+ lif
->nmcast
) >= nfilters
)
1289 err
= ionic_adminq_post_wait(lif
, &ctx
);
1291 spin_lock_bh(&lif
->rx_filters
.lock
);
1292 if (err
&& err
!= -EEXIST
) {
1293 /* set the state back to NEW so we can try again later */
1294 f
= ionic_rx_filter_by_addr(lif
, addr
);
1295 if (f
&& f
->state
== IONIC_FILTER_STATE_SYNCED
) {
1296 f
->state
= IONIC_FILTER_STATE_NEW
;
1297 set_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED
, lif
->state
);
1300 spin_unlock_bh(&lif
->rx_filters
.lock
);
1313 f
= ionic_rx_filter_by_addr(lif
, addr
);
1314 if (f
&& f
->state
== IONIC_FILTER_STATE_OLD
) {
1315 /* Someone requested a delete while we were adding
1316 * so update the filter info with the results from the add
1317 * and the data will be there for the delete on the next
1320 err
= ionic_rx_filter_save(lif
, 0, IONIC_RXQ_INDEX_ANY
, 0, &ctx
,
1321 IONIC_FILTER_STATE_OLD
);
1323 err
= ionic_rx_filter_save(lif
, 0, IONIC_RXQ_INDEX_ANY
, 0, &ctx
,
1324 IONIC_FILTER_STATE_SYNCED
);
1327 spin_unlock_bh(&lif
->rx_filters
.lock
);
1332 int ionic_lif_addr_del(struct ionic_lif
*lif
, const u8
*addr
)
1334 struct ionic_admin_ctx ctx
= {
1335 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1336 .cmd
.rx_filter_del
= {
1337 .opcode
= IONIC_CMD_RX_FILTER_DEL
,
1338 .lif_index
= cpu_to_le16(lif
->index
),
1341 struct ionic_rx_filter
*f
;
1345 spin_lock_bh(&lif
->rx_filters
.lock
);
1346 f
= ionic_rx_filter_by_addr(lif
, addr
);
1348 spin_unlock_bh(&lif
->rx_filters
.lock
);
1352 netdev_dbg(lif
->netdev
, "rx_filter del ADDR %pM (id %d)\n",
1353 addr
, f
->filter_id
);
1356 ctx
.cmd
.rx_filter_del
.filter_id
= cpu_to_le32(f
->filter_id
);
1357 ionic_rx_filter_free(lif
, f
);
1359 if (is_multicast_ether_addr(addr
) && lif
->nmcast
)
1361 else if (!is_multicast_ether_addr(addr
) && lif
->nucast
)
1364 spin_unlock_bh(&lif
->rx_filters
.lock
);
1366 if (state
!= IONIC_FILTER_STATE_NEW
) {
1367 err
= ionic_adminq_post_wait(lif
, &ctx
);
1368 if (err
&& err
!= -EEXIST
)
1375 static int ionic_addr_add(struct net_device
*netdev
, const u8
*addr
)
1377 return ionic_lif_list_addr(netdev_priv(netdev
), addr
, ADD_ADDR
);
1380 static int ionic_addr_del(struct net_device
*netdev
, const u8
*addr
)
1382 /* Don't delete our own address from the uc list */
1383 if (ether_addr_equal(addr
, netdev
->dev_addr
))
1386 return ionic_lif_list_addr(netdev_priv(netdev
), addr
, DEL_ADDR
);
1389 void ionic_lif_rx_mode(struct ionic_lif
*lif
)
1391 struct net_device
*netdev
= lif
->netdev
;
1392 unsigned int nfilters
;
1393 unsigned int nd_flags
;
1397 #define REMAIN(__x) (sizeof(buf) - (__x))
1399 mutex_lock(&lif
->config_lock
);
1401 /* grab the flags once for local use */
1402 nd_flags
= netdev
->flags
;
1404 rx_mode
= IONIC_RX_MODE_F_UNICAST
;
1405 rx_mode
|= (nd_flags
& IFF_MULTICAST
) ? IONIC_RX_MODE_F_MULTICAST
: 0;
1406 rx_mode
|= (nd_flags
& IFF_BROADCAST
) ? IONIC_RX_MODE_F_BROADCAST
: 0;
1407 rx_mode
|= (nd_flags
& IFF_PROMISC
) ? IONIC_RX_MODE_F_PROMISC
: 0;
1408 rx_mode
|= (nd_flags
& IFF_ALLMULTI
) ? IONIC_RX_MODE_F_ALLMULTI
: 0;
1410 /* sync the mac filters */
1411 ionic_rx_filter_sync(lif
);
1413 /* check for overflow state
1414 * if so, we track that we overflowed and enable NIC PROMISC
1415 * else if the overflow is set and not needed
1416 * we remove our overflow flag and check the netdev flags
1417 * to see if we can disable NIC PROMISC
1419 nfilters
= le32_to_cpu(lif
->identity
->eth
.max_ucast_filters
);
1420 if ((lif
->nucast
+ lif
->nmcast
) >= nfilters
) {
1421 rx_mode
|= IONIC_RX_MODE_F_PROMISC
;
1422 rx_mode
|= IONIC_RX_MODE_F_ALLMULTI
;
1423 lif
->uc_overflow
= true;
1424 lif
->mc_overflow
= true;
1425 } else if (lif
->uc_overflow
) {
1426 lif
->uc_overflow
= false;
1427 lif
->mc_overflow
= false;
1428 if (!(nd_flags
& IFF_PROMISC
))
1429 rx_mode
&= ~IONIC_RX_MODE_F_PROMISC
;
1430 if (!(nd_flags
& IFF_ALLMULTI
))
1431 rx_mode
&= ~IONIC_RX_MODE_F_ALLMULTI
;
1434 i
= scnprintf(buf
, sizeof(buf
), "rx_mode 0x%04x -> 0x%04x:",
1435 lif
->rx_mode
, rx_mode
);
1436 if (rx_mode
& IONIC_RX_MODE_F_UNICAST
)
1437 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_UNICAST");
1438 if (rx_mode
& IONIC_RX_MODE_F_MULTICAST
)
1439 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_MULTICAST");
1440 if (rx_mode
& IONIC_RX_MODE_F_BROADCAST
)
1441 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_BROADCAST");
1442 if (rx_mode
& IONIC_RX_MODE_F_PROMISC
)
1443 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_PROMISC");
1444 if (rx_mode
& IONIC_RX_MODE_F_ALLMULTI
)
1445 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_ALLMULTI");
1446 if (rx_mode
& IONIC_RX_MODE_F_RDMA_SNIFFER
)
1447 i
+= scnprintf(&buf
[i
], REMAIN(i
), " RX_MODE_F_RDMA_SNIFFER");
1448 netdev_dbg(netdev
, "lif%d %s\n", lif
->index
, buf
);
1450 if (lif
->rx_mode
!= rx_mode
) {
1451 struct ionic_admin_ctx ctx
= {
1452 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1453 .cmd
.rx_mode_set
= {
1454 .opcode
= IONIC_CMD_RX_MODE_SET
,
1455 .lif_index
= cpu_to_le16(lif
->index
),
1460 ctx
.cmd
.rx_mode_set
.rx_mode
= cpu_to_le16(rx_mode
);
1461 err
= ionic_adminq_post_wait(lif
, &ctx
);
1463 netdev_warn(netdev
, "set rx_mode 0x%04x failed: %d\n",
1466 lif
->rx_mode
= rx_mode
;
1469 mutex_unlock(&lif
->config_lock
);
1472 static void ionic_ndo_set_rx_mode(struct net_device
*netdev
)
1474 struct ionic_lif
*lif
= netdev_priv(netdev
);
1475 struct ionic_deferred_work
*work
;
1477 /* Sync the kernel filter list with the driver filter list */
1478 __dev_uc_sync(netdev
, ionic_addr_add
, ionic_addr_del
);
1479 __dev_mc_sync(netdev
, ionic_addr_add
, ionic_addr_del
);
1481 /* Shove off the rest of the rxmode work to the work task
1482 * which will include syncing the filters to the firmware.
1484 work
= kzalloc(sizeof(*work
), GFP_ATOMIC
);
1486 netdev_err(lif
->netdev
, "rxmode change dropped\n");
1489 work
->type
= IONIC_DW_TYPE_RX_MODE
;
1490 netdev_dbg(lif
->netdev
, "deferred: rx_mode\n");
1491 ionic_lif_deferred_enqueue(&lif
->deferred
, work
);
1494 static __le64
ionic_netdev_features_to_nic(netdev_features_t features
)
1498 if (features
& NETIF_F_HW_VLAN_CTAG_TX
)
1499 wanted
|= IONIC_ETH_HW_VLAN_TX_TAG
;
1500 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
1501 wanted
|= IONIC_ETH_HW_VLAN_RX_STRIP
;
1502 if (features
& NETIF_F_HW_VLAN_CTAG_FILTER
)
1503 wanted
|= IONIC_ETH_HW_VLAN_RX_FILTER
;
1504 if (features
& NETIF_F_RXHASH
)
1505 wanted
|= IONIC_ETH_HW_RX_HASH
;
1506 if (features
& NETIF_F_RXCSUM
)
1507 wanted
|= IONIC_ETH_HW_RX_CSUM
;
1508 if (features
& NETIF_F_SG
)
1509 wanted
|= IONIC_ETH_HW_TX_SG
;
1510 if (features
& NETIF_F_HW_CSUM
)
1511 wanted
|= IONIC_ETH_HW_TX_CSUM
;
1512 if (features
& NETIF_F_TSO
)
1513 wanted
|= IONIC_ETH_HW_TSO
;
1514 if (features
& NETIF_F_TSO6
)
1515 wanted
|= IONIC_ETH_HW_TSO_IPV6
;
1516 if (features
& NETIF_F_TSO_ECN
)
1517 wanted
|= IONIC_ETH_HW_TSO_ECN
;
1518 if (features
& NETIF_F_GSO_GRE
)
1519 wanted
|= IONIC_ETH_HW_TSO_GRE
;
1520 if (features
& NETIF_F_GSO_GRE_CSUM
)
1521 wanted
|= IONIC_ETH_HW_TSO_GRE_CSUM
;
1522 if (features
& NETIF_F_GSO_IPXIP4
)
1523 wanted
|= IONIC_ETH_HW_TSO_IPXIP4
;
1524 if (features
& NETIF_F_GSO_IPXIP6
)
1525 wanted
|= IONIC_ETH_HW_TSO_IPXIP6
;
1526 if (features
& NETIF_F_GSO_UDP_TUNNEL
)
1527 wanted
|= IONIC_ETH_HW_TSO_UDP
;
1528 if (features
& NETIF_F_GSO_UDP_TUNNEL_CSUM
)
1529 wanted
|= IONIC_ETH_HW_TSO_UDP_CSUM
;
1531 return cpu_to_le64(wanted
);
1534 static int ionic_set_nic_features(struct ionic_lif
*lif
,
1535 netdev_features_t features
)
1537 struct device
*dev
= lif
->ionic
->dev
;
1538 struct ionic_admin_ctx ctx
= {
1539 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1540 .cmd
.lif_setattr
= {
1541 .opcode
= IONIC_CMD_LIF_SETATTR
,
1542 .index
= cpu_to_le16(lif
->index
),
1543 .attr
= IONIC_LIF_ATTR_FEATURES
,
1546 u64 vlan_flags
= IONIC_ETH_HW_VLAN_TX_TAG
|
1547 IONIC_ETH_HW_VLAN_RX_STRIP
|
1548 IONIC_ETH_HW_VLAN_RX_FILTER
;
1549 u64 old_hw_features
;
1552 ctx
.cmd
.lif_setattr
.features
= ionic_netdev_features_to_nic(features
);
1555 ctx
.cmd
.lif_setattr
.features
|= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP
);
1557 err
= ionic_adminq_post_wait(lif
, &ctx
);
1561 old_hw_features
= lif
->hw_features
;
1562 lif
->hw_features
= le64_to_cpu(ctx
.cmd
.lif_setattr
.features
&
1563 ctx
.comp
.lif_setattr
.features
);
1565 if ((old_hw_features
^ lif
->hw_features
) & IONIC_ETH_HW_RX_HASH
)
1566 ionic_lif_rss_config(lif
, lif
->rss_types
, NULL
, NULL
);
1568 if ((vlan_flags
& features
) &&
1569 !(vlan_flags
& le64_to_cpu(ctx
.comp
.lif_setattr
.features
)))
1570 dev_info_once(lif
->ionic
->dev
, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1572 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_TX_TAG
)
1573 dev_dbg(dev
, "feature ETH_HW_VLAN_TX_TAG\n");
1574 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_RX_STRIP
)
1575 dev_dbg(dev
, "feature ETH_HW_VLAN_RX_STRIP\n");
1576 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_RX_FILTER
)
1577 dev_dbg(dev
, "feature ETH_HW_VLAN_RX_FILTER\n");
1578 if (lif
->hw_features
& IONIC_ETH_HW_RX_HASH
)
1579 dev_dbg(dev
, "feature ETH_HW_RX_HASH\n");
1580 if (lif
->hw_features
& IONIC_ETH_HW_TX_SG
)
1581 dev_dbg(dev
, "feature ETH_HW_TX_SG\n");
1582 if (lif
->hw_features
& IONIC_ETH_HW_TX_CSUM
)
1583 dev_dbg(dev
, "feature ETH_HW_TX_CSUM\n");
1584 if (lif
->hw_features
& IONIC_ETH_HW_RX_CSUM
)
1585 dev_dbg(dev
, "feature ETH_HW_RX_CSUM\n");
1586 if (lif
->hw_features
& IONIC_ETH_HW_TSO
)
1587 dev_dbg(dev
, "feature ETH_HW_TSO\n");
1588 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPV6
)
1589 dev_dbg(dev
, "feature ETH_HW_TSO_IPV6\n");
1590 if (lif
->hw_features
& IONIC_ETH_HW_TSO_ECN
)
1591 dev_dbg(dev
, "feature ETH_HW_TSO_ECN\n");
1592 if (lif
->hw_features
& IONIC_ETH_HW_TSO_GRE
)
1593 dev_dbg(dev
, "feature ETH_HW_TSO_GRE\n");
1594 if (lif
->hw_features
& IONIC_ETH_HW_TSO_GRE_CSUM
)
1595 dev_dbg(dev
, "feature ETH_HW_TSO_GRE_CSUM\n");
1596 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPXIP4
)
1597 dev_dbg(dev
, "feature ETH_HW_TSO_IPXIP4\n");
1598 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPXIP6
)
1599 dev_dbg(dev
, "feature ETH_HW_TSO_IPXIP6\n");
1600 if (lif
->hw_features
& IONIC_ETH_HW_TSO_UDP
)
1601 dev_dbg(dev
, "feature ETH_HW_TSO_UDP\n");
1602 if (lif
->hw_features
& IONIC_ETH_HW_TSO_UDP_CSUM
)
1603 dev_dbg(dev
, "feature ETH_HW_TSO_UDP_CSUM\n");
1604 if (lif
->hw_features
& IONIC_ETH_HW_TIMESTAMP
)
1605 dev_dbg(dev
, "feature ETH_HW_TIMESTAMP\n");
1610 static int ionic_init_nic_features(struct ionic_lif
*lif
)
1612 struct net_device
*netdev
= lif
->netdev
;
1613 netdev_features_t features
;
1616 /* set up what we expect to support by default */
1617 features
= NETIF_F_HW_VLAN_CTAG_TX
|
1618 NETIF_F_HW_VLAN_CTAG_RX
|
1619 NETIF_F_HW_VLAN_CTAG_FILTER
|
1628 features
|= NETIF_F_RXHASH
;
1630 err
= ionic_set_nic_features(lif
, features
);
1634 /* tell the netdev what we actually can support */
1635 netdev
->features
|= NETIF_F_HIGHDMA
;
1637 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_TX_TAG
)
1638 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_TX
;
1639 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_RX_STRIP
)
1640 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_RX
;
1641 if (lif
->hw_features
& IONIC_ETH_HW_VLAN_RX_FILTER
)
1642 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_FILTER
;
1643 if (lif
->hw_features
& IONIC_ETH_HW_RX_HASH
)
1644 netdev
->hw_features
|= NETIF_F_RXHASH
;
1645 if (lif
->hw_features
& IONIC_ETH_HW_TX_SG
)
1646 netdev
->hw_features
|= NETIF_F_SG
;
1648 if (lif
->hw_features
& IONIC_ETH_HW_TX_CSUM
)
1649 netdev
->hw_enc_features
|= NETIF_F_HW_CSUM
;
1650 if (lif
->hw_features
& IONIC_ETH_HW_RX_CSUM
)
1651 netdev
->hw_enc_features
|= NETIF_F_RXCSUM
;
1652 if (lif
->hw_features
& IONIC_ETH_HW_TSO
)
1653 netdev
->hw_enc_features
|= NETIF_F_TSO
;
1654 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPV6
)
1655 netdev
->hw_enc_features
|= NETIF_F_TSO6
;
1656 if (lif
->hw_features
& IONIC_ETH_HW_TSO_ECN
)
1657 netdev
->hw_enc_features
|= NETIF_F_TSO_ECN
;
1658 if (lif
->hw_features
& IONIC_ETH_HW_TSO_GRE
)
1659 netdev
->hw_enc_features
|= NETIF_F_GSO_GRE
;
1660 if (lif
->hw_features
& IONIC_ETH_HW_TSO_GRE_CSUM
)
1661 netdev
->hw_enc_features
|= NETIF_F_GSO_GRE_CSUM
;
1662 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPXIP4
)
1663 netdev
->hw_enc_features
|= NETIF_F_GSO_IPXIP4
;
1664 if (lif
->hw_features
& IONIC_ETH_HW_TSO_IPXIP6
)
1665 netdev
->hw_enc_features
|= NETIF_F_GSO_IPXIP6
;
1666 if (lif
->hw_features
& IONIC_ETH_HW_TSO_UDP
)
1667 netdev
->hw_enc_features
|= NETIF_F_GSO_UDP_TUNNEL
;
1668 if (lif
->hw_features
& IONIC_ETH_HW_TSO_UDP_CSUM
)
1669 netdev
->hw_enc_features
|= NETIF_F_GSO_UDP_TUNNEL_CSUM
;
1671 netdev
->hw_features
|= netdev
->hw_enc_features
;
1672 netdev
->features
|= netdev
->hw_features
;
1673 netdev
->vlan_features
|= netdev
->features
& ~NETIF_F_VLAN_FEATURES
;
1675 netdev
->priv_flags
|= IFF_UNICAST_FLT
|
1676 IFF_LIVE_ADDR_CHANGE
;
1681 static int ionic_set_features(struct net_device
*netdev
,
1682 netdev_features_t features
)
1684 struct ionic_lif
*lif
= netdev_priv(netdev
);
1687 netdev_dbg(netdev
, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1688 __func__
, (u64
)lif
->netdev
->features
, (u64
)features
);
1690 err
= ionic_set_nic_features(lif
, features
);
1695 static int ionic_set_mac_address(struct net_device
*netdev
, void *sa
)
1697 struct sockaddr
*addr
= sa
;
1701 mac
= (u8
*)addr
->sa_data
;
1702 if (ether_addr_equal(netdev
->dev_addr
, mac
))
1705 err
= eth_prepare_mac_addr_change(netdev
, addr
);
1709 if (!is_zero_ether_addr(netdev
->dev_addr
)) {
1710 netdev_info(netdev
, "deleting mac addr %pM\n",
1712 ionic_lif_addr_del(netdev_priv(netdev
), netdev
->dev_addr
);
1715 eth_commit_mac_addr_change(netdev
, addr
);
1716 netdev_info(netdev
, "updating mac addr %pM\n", mac
);
1718 return ionic_lif_addr_add(netdev_priv(netdev
), mac
);
1721 static void ionic_stop_queues_reconfig(struct ionic_lif
*lif
)
1723 /* Stop and clean the queues before reconfiguration */
1724 netif_device_detach(lif
->netdev
);
1725 ionic_stop_queues(lif
);
1726 ionic_txrx_deinit(lif
);
1729 static int ionic_start_queues_reconfig(struct ionic_lif
*lif
)
1733 /* Re-init the queues after reconfiguration */
1735 /* The only way txrx_init can fail here is if communication
1736 * with FW is suddenly broken. There's not much we can do
1737 * at this point - error messages have already been printed,
1738 * so we can continue on and the user can eventually do a
1739 * DOWN and UP to try to reset and clear the issue.
1741 err
= ionic_txrx_init(lif
);
1742 ionic_link_status_check_request(lif
, CAN_NOT_SLEEP
);
1743 netif_device_attach(lif
->netdev
);
1748 static int ionic_change_mtu(struct net_device
*netdev
, int new_mtu
)
1750 struct ionic_lif
*lif
= netdev_priv(netdev
);
1751 struct ionic_admin_ctx ctx
= {
1752 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1753 .cmd
.lif_setattr
= {
1754 .opcode
= IONIC_CMD_LIF_SETATTR
,
1755 .index
= cpu_to_le16(lif
->index
),
1756 .attr
= IONIC_LIF_ATTR_MTU
,
1757 .mtu
= cpu_to_le32(new_mtu
),
1762 err
= ionic_adminq_post_wait(lif
, &ctx
);
1766 /* if we're not running, nothing more to do */
1767 if (!netif_running(netdev
)) {
1768 netdev
->mtu
= new_mtu
;
1772 mutex_lock(&lif
->queue_lock
);
1773 ionic_stop_queues_reconfig(lif
);
1774 netdev
->mtu
= new_mtu
;
1775 err
= ionic_start_queues_reconfig(lif
);
1776 mutex_unlock(&lif
->queue_lock
);
1781 static void ionic_tx_timeout_work(struct work_struct
*ws
)
1783 struct ionic_lif
*lif
= container_of(ws
, struct ionic_lif
, tx_timeout_work
);
1785 if (test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
1788 /* if we were stopped before this scheduled job was launched,
1789 * don't bother the queues as they are already stopped.
1791 if (!netif_running(lif
->netdev
))
1794 mutex_lock(&lif
->queue_lock
);
1795 ionic_stop_queues_reconfig(lif
);
1796 ionic_start_queues_reconfig(lif
);
1797 mutex_unlock(&lif
->queue_lock
);
1800 static void ionic_tx_timeout(struct net_device
*netdev
, unsigned int txqueue
)
1802 struct ionic_lif
*lif
= netdev_priv(netdev
);
1804 netdev_info(lif
->netdev
, "Tx Timeout triggered - txq %d\n", txqueue
);
1805 schedule_work(&lif
->tx_timeout_work
);
1808 static int ionic_vlan_rx_add_vid(struct net_device
*netdev
, __be16 proto
,
1811 struct ionic_lif
*lif
= netdev_priv(netdev
);
1812 struct ionic_admin_ctx ctx
= {
1813 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1814 .cmd
.rx_filter_add
= {
1815 .opcode
= IONIC_CMD_RX_FILTER_ADD
,
1816 .lif_index
= cpu_to_le16(lif
->index
),
1817 .match
= cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN
),
1818 .vlan
.vlan
= cpu_to_le16(vid
),
1823 netdev_dbg(netdev
, "rx_filter add VLAN %d\n", vid
);
1824 err
= ionic_adminq_post_wait(lif
, &ctx
);
1828 spin_lock_bh(&lif
->rx_filters
.lock
);
1829 err
= ionic_rx_filter_save(lif
, 0, IONIC_RXQ_INDEX_ANY
, 0, &ctx
,
1830 IONIC_FILTER_STATE_SYNCED
);
1831 spin_unlock_bh(&lif
->rx_filters
.lock
);
1836 static int ionic_vlan_rx_kill_vid(struct net_device
*netdev
, __be16 proto
,
1839 struct ionic_lif
*lif
= netdev_priv(netdev
);
1840 struct ionic_admin_ctx ctx
= {
1841 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1842 .cmd
.rx_filter_del
= {
1843 .opcode
= IONIC_CMD_RX_FILTER_DEL
,
1844 .lif_index
= cpu_to_le16(lif
->index
),
1847 struct ionic_rx_filter
*f
;
1849 spin_lock_bh(&lif
->rx_filters
.lock
);
1851 f
= ionic_rx_filter_by_vlan(lif
, vid
);
1853 spin_unlock_bh(&lif
->rx_filters
.lock
);
1857 netdev_dbg(netdev
, "rx_filter del VLAN %d (id %d)\n",
1860 ctx
.cmd
.rx_filter_del
.filter_id
= cpu_to_le32(f
->filter_id
);
1861 ionic_rx_filter_free(lif
, f
);
1862 spin_unlock_bh(&lif
->rx_filters
.lock
);
1864 return ionic_adminq_post_wait(lif
, &ctx
);
1867 int ionic_lif_rss_config(struct ionic_lif
*lif
, const u16 types
,
1868 const u8
*key
, const u32
*indir
)
1870 struct ionic_admin_ctx ctx
= {
1871 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1872 .cmd
.lif_setattr
= {
1873 .opcode
= IONIC_CMD_LIF_SETATTR
,
1874 .attr
= IONIC_LIF_ATTR_RSS
,
1875 .rss
.addr
= cpu_to_le64(lif
->rss_ind_tbl_pa
),
1878 unsigned int i
, tbl_sz
;
1880 if (lif
->hw_features
& IONIC_ETH_HW_RX_HASH
) {
1881 lif
->rss_types
= types
;
1882 ctx
.cmd
.lif_setattr
.rss
.types
= cpu_to_le16(types
);
1886 memcpy(lif
->rss_hash_key
, key
, IONIC_RSS_HASH_KEY_SIZE
);
1889 tbl_sz
= le16_to_cpu(lif
->ionic
->ident
.lif
.eth
.rss_ind_tbl_sz
);
1890 for (i
= 0; i
< tbl_sz
; i
++)
1891 lif
->rss_ind_tbl
[i
] = indir
[i
];
1894 memcpy(ctx
.cmd
.lif_setattr
.rss
.key
, lif
->rss_hash_key
,
1895 IONIC_RSS_HASH_KEY_SIZE
);
1897 return ionic_adminq_post_wait(lif
, &ctx
);
1900 static int ionic_lif_rss_init(struct ionic_lif
*lif
)
1902 unsigned int tbl_sz
;
1905 lif
->rss_types
= IONIC_RSS_TYPE_IPV4
|
1906 IONIC_RSS_TYPE_IPV4_TCP
|
1907 IONIC_RSS_TYPE_IPV4_UDP
|
1908 IONIC_RSS_TYPE_IPV6
|
1909 IONIC_RSS_TYPE_IPV6_TCP
|
1910 IONIC_RSS_TYPE_IPV6_UDP
;
1912 /* Fill indirection table with 'default' values */
1913 tbl_sz
= le16_to_cpu(lif
->ionic
->ident
.lif
.eth
.rss_ind_tbl_sz
);
1914 for (i
= 0; i
< tbl_sz
; i
++)
1915 lif
->rss_ind_tbl
[i
] = ethtool_rxfh_indir_default(i
, lif
->nxqs
);
1917 return ionic_lif_rss_config(lif
, lif
->rss_types
, NULL
, NULL
);
1920 static void ionic_lif_rss_deinit(struct ionic_lif
*lif
)
1924 tbl_sz
= le16_to_cpu(lif
->ionic
->ident
.lif
.eth
.rss_ind_tbl_sz
);
1925 memset(lif
->rss_ind_tbl
, 0, tbl_sz
);
1926 memset(lif
->rss_hash_key
, 0, IONIC_RSS_HASH_KEY_SIZE
);
1928 ionic_lif_rss_config(lif
, 0x0, NULL
, NULL
);
1931 static void ionic_lif_quiesce(struct ionic_lif
*lif
)
1933 struct ionic_admin_ctx ctx
= {
1934 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
1935 .cmd
.lif_setattr
= {
1936 .opcode
= IONIC_CMD_LIF_SETATTR
,
1937 .index
= cpu_to_le16(lif
->index
),
1938 .attr
= IONIC_LIF_ATTR_STATE
,
1939 .state
= IONIC_LIF_QUIESCE
,
1944 err
= ionic_adminq_post_wait(lif
, &ctx
);
1946 netdev_err(lif
->netdev
, "lif quiesce failed %d\n", err
);
1949 static void ionic_txrx_disable(struct ionic_lif
*lif
)
1955 for (i
= 0; i
< lif
->nxqs
; i
++)
1956 err
= ionic_qcq_disable(lif
->txqcqs
[i
], (err
!= -ETIMEDOUT
));
1959 if (lif
->hwstamp_txq
)
1960 err
= ionic_qcq_disable(lif
->hwstamp_txq
, (err
!= -ETIMEDOUT
));
1963 for (i
= 0; i
< lif
->nxqs
; i
++)
1964 err
= ionic_qcq_disable(lif
->rxqcqs
[i
], (err
!= -ETIMEDOUT
));
1967 if (lif
->hwstamp_rxq
)
1968 err
= ionic_qcq_disable(lif
->hwstamp_rxq
, (err
!= -ETIMEDOUT
));
1970 ionic_lif_quiesce(lif
);
1973 static void ionic_txrx_deinit(struct ionic_lif
*lif
)
1978 for (i
= 0; i
< lif
->nxqs
&& lif
->txqcqs
[i
]; i
++) {
1979 ionic_lif_qcq_deinit(lif
, lif
->txqcqs
[i
]);
1980 ionic_tx_flush(&lif
->txqcqs
[i
]->cq
);
1981 ionic_tx_empty(&lif
->txqcqs
[i
]->q
);
1986 for (i
= 0; i
< lif
->nxqs
&& lif
->rxqcqs
[i
]; i
++) {
1987 ionic_lif_qcq_deinit(lif
, lif
->rxqcqs
[i
]);
1988 ionic_rx_empty(&lif
->rxqcqs
[i
]->q
);
1993 if (lif
->hwstamp_txq
) {
1994 ionic_lif_qcq_deinit(lif
, lif
->hwstamp_txq
);
1995 ionic_tx_flush(&lif
->hwstamp_txq
->cq
);
1996 ionic_tx_empty(&lif
->hwstamp_txq
->q
);
1999 if (lif
->hwstamp_rxq
) {
2000 ionic_lif_qcq_deinit(lif
, lif
->hwstamp_rxq
);
2001 ionic_rx_empty(&lif
->hwstamp_rxq
->q
);
2005 static void ionic_txrx_free(struct ionic_lif
*lif
)
2010 for (i
= 0; i
< lif
->ionic
->ntxqs_per_lif
&& lif
->txqcqs
[i
]; i
++) {
2011 ionic_qcq_free(lif
, lif
->txqcqs
[i
]);
2012 devm_kfree(lif
->ionic
->dev
, lif
->txqcqs
[i
]);
2013 lif
->txqcqs
[i
] = NULL
;
2018 for (i
= 0; i
< lif
->ionic
->nrxqs_per_lif
&& lif
->rxqcqs
[i
]; i
++) {
2019 ionic_qcq_free(lif
, lif
->rxqcqs
[i
]);
2020 devm_kfree(lif
->ionic
->dev
, lif
->rxqcqs
[i
]);
2021 lif
->rxqcqs
[i
] = NULL
;
2025 if (lif
->hwstamp_txq
) {
2026 ionic_qcq_free(lif
, lif
->hwstamp_txq
);
2027 devm_kfree(lif
->ionic
->dev
, lif
->hwstamp_txq
);
2028 lif
->hwstamp_txq
= NULL
;
2031 if (lif
->hwstamp_rxq
) {
2032 ionic_qcq_free(lif
, lif
->hwstamp_rxq
);
2033 devm_kfree(lif
->ionic
->dev
, lif
->hwstamp_rxq
);
2034 lif
->hwstamp_rxq
= NULL
;
2038 static int ionic_txrx_alloc(struct ionic_lif
*lif
)
2040 unsigned int comp_sz
, desc_sz
, num_desc
, sg_desc_sz
;
2041 unsigned int flags
, i
;
2044 num_desc
= lif
->ntxq_descs
;
2045 desc_sz
= sizeof(struct ionic_txq_desc
);
2046 comp_sz
= sizeof(struct ionic_txq_comp
);
2048 if (lif
->qtype_info
[IONIC_QTYPE_TXQ
].version
>= 1 &&
2049 lif
->qtype_info
[IONIC_QTYPE_TXQ
].sg_desc_sz
==
2050 sizeof(struct ionic_txq_sg_desc_v1
))
2051 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc_v1
);
2053 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc
);
2055 flags
= IONIC_QCQ_F_TX_STATS
| IONIC_QCQ_F_SG
;
2056 if (test_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
))
2057 flags
|= IONIC_QCQ_F_INTR
;
2058 for (i
= 0; i
< lif
->nxqs
; i
++) {
2059 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_TXQ
, i
, "tx", flags
,
2060 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
2061 lif
->kern_pid
, &lif
->txqcqs
[i
]);
2065 if (flags
& IONIC_QCQ_F_INTR
) {
2066 ionic_intr_coal_init(lif
->ionic
->idev
.intr_ctrl
,
2067 lif
->txqcqs
[i
]->intr
.index
,
2068 lif
->tx_coalesce_hw
);
2069 if (test_bit(IONIC_LIF_F_TX_DIM_INTR
, lif
->state
))
2070 lif
->txqcqs
[i
]->intr
.dim_coal_hw
= lif
->tx_coalesce_hw
;
2073 ionic_debugfs_add_qcq(lif
, lif
->txqcqs
[i
]);
2076 flags
= IONIC_QCQ_F_RX_STATS
| IONIC_QCQ_F_SG
| IONIC_QCQ_F_INTR
;
2078 num_desc
= lif
->nrxq_descs
;
2079 desc_sz
= sizeof(struct ionic_rxq_desc
);
2080 comp_sz
= sizeof(struct ionic_rxq_comp
);
2081 sg_desc_sz
= sizeof(struct ionic_rxq_sg_desc
);
2083 if (lif
->rxq_features
& IONIC_Q_F_2X_CQ_DESC
)
2086 for (i
= 0; i
< lif
->nxqs
; i
++) {
2087 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_RXQ
, i
, "rx", flags
,
2088 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
2089 lif
->kern_pid
, &lif
->rxqcqs
[i
]);
2093 lif
->rxqcqs
[i
]->q
.features
= lif
->rxq_features
;
2095 ionic_intr_coal_init(lif
->ionic
->idev
.intr_ctrl
,
2096 lif
->rxqcqs
[i
]->intr
.index
,
2097 lif
->rx_coalesce_hw
);
2098 if (test_bit(IONIC_LIF_F_RX_DIM_INTR
, lif
->state
))
2099 lif
->rxqcqs
[i
]->intr
.dim_coal_hw
= lif
->rx_coalesce_hw
;
2101 if (!test_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
))
2102 ionic_link_qcq_interrupts(lif
->rxqcqs
[i
],
2105 ionic_debugfs_add_qcq(lif
, lif
->rxqcqs
[i
]);
2111 ionic_txrx_free(lif
);
2116 static int ionic_txrx_init(struct ionic_lif
*lif
)
2121 for (i
= 0; i
< lif
->nxqs
; i
++) {
2122 err
= ionic_lif_txq_init(lif
, lif
->txqcqs
[i
]);
2126 err
= ionic_lif_rxq_init(lif
, lif
->rxqcqs
[i
]);
2128 ionic_lif_qcq_deinit(lif
, lif
->txqcqs
[i
]);
2133 if (lif
->netdev
->features
& NETIF_F_RXHASH
)
2134 ionic_lif_rss_init(lif
);
2136 ionic_lif_rx_mode(lif
);
2142 ionic_lif_qcq_deinit(lif
, lif
->txqcqs
[i
]);
2143 ionic_lif_qcq_deinit(lif
, lif
->rxqcqs
[i
]);
2149 static int ionic_txrx_enable(struct ionic_lif
*lif
)
2154 for (i
= 0; i
< lif
->nxqs
; i
++) {
2155 if (!(lif
->rxqcqs
[i
] && lif
->txqcqs
[i
])) {
2156 dev_err(lif
->ionic
->dev
, "%s: bad qcq %d\n", __func__
, i
);
2161 ionic_rx_fill(&lif
->rxqcqs
[i
]->q
);
2162 err
= ionic_qcq_enable(lif
->rxqcqs
[i
]);
2166 err
= ionic_qcq_enable(lif
->txqcqs
[i
]);
2168 derr
= ionic_qcq_disable(lif
->rxqcqs
[i
], (err
!= -ETIMEDOUT
));
2173 if (lif
->hwstamp_rxq
) {
2174 ionic_rx_fill(&lif
->hwstamp_rxq
->q
);
2175 err
= ionic_qcq_enable(lif
->hwstamp_rxq
);
2177 goto err_out_hwstamp_rx
;
2180 if (lif
->hwstamp_txq
) {
2181 err
= ionic_qcq_enable(lif
->hwstamp_txq
);
2183 goto err_out_hwstamp_tx
;
2189 if (lif
->hwstamp_rxq
)
2190 derr
= ionic_qcq_disable(lif
->hwstamp_rxq
, (derr
!= -ETIMEDOUT
));
2195 derr
= ionic_qcq_disable(lif
->txqcqs
[i
], (derr
!= -ETIMEDOUT
));
2196 derr
= ionic_qcq_disable(lif
->rxqcqs
[i
], (derr
!= -ETIMEDOUT
));
2202 static int ionic_start_queues(struct ionic_lif
*lif
)
2206 if (test_bit(IONIC_LIF_F_BROKEN
, lif
->state
))
2209 if (test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
2212 if (test_and_set_bit(IONIC_LIF_F_UP
, lif
->state
))
2215 err
= ionic_txrx_enable(lif
);
2217 clear_bit(IONIC_LIF_F_UP
, lif
->state
);
2220 netif_tx_wake_all_queues(lif
->netdev
);
2225 static int ionic_open(struct net_device
*netdev
)
2227 struct ionic_lif
*lif
= netdev_priv(netdev
);
2230 /* If recovering from a broken state, clear the bit and we'll try again */
2231 if (test_and_clear_bit(IONIC_LIF_F_BROKEN
, lif
->state
))
2232 netdev_info(netdev
, "clearing broken state\n");
2234 mutex_lock(&lif
->queue_lock
);
2236 err
= ionic_txrx_alloc(lif
);
2240 err
= ionic_txrx_init(lif
);
2244 err
= netif_set_real_num_tx_queues(netdev
, lif
->nxqs
);
2246 goto err_txrx_deinit
;
2248 err
= netif_set_real_num_rx_queues(netdev
, lif
->nxqs
);
2250 goto err_txrx_deinit
;
2252 /* don't start the queues until we have link */
2253 if (netif_carrier_ok(netdev
)) {
2254 err
= ionic_start_queues(lif
);
2256 goto err_txrx_deinit
;
2259 /* If hardware timestamping is enabled, but the queues were freed by
2260 * ionic_stop, those need to be reallocated and initialized, too.
2262 ionic_lif_hwstamp_recreate_queues(lif
);
2264 mutex_unlock(&lif
->queue_lock
);
2269 ionic_txrx_deinit(lif
);
2271 ionic_txrx_free(lif
);
2273 mutex_unlock(&lif
->queue_lock
);
2277 static void ionic_stop_queues(struct ionic_lif
*lif
)
2279 if (!test_and_clear_bit(IONIC_LIF_F_UP
, lif
->state
))
2282 netif_tx_disable(lif
->netdev
);
2283 ionic_txrx_disable(lif
);
2286 static int ionic_stop(struct net_device
*netdev
)
2288 struct ionic_lif
*lif
= netdev_priv(netdev
);
2290 if (test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
2293 mutex_lock(&lif
->queue_lock
);
2294 ionic_stop_queues(lif
);
2295 ionic_txrx_deinit(lif
);
2296 ionic_txrx_free(lif
);
2297 mutex_unlock(&lif
->queue_lock
);
2302 static int ionic_eth_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
2304 struct ionic_lif
*lif
= netdev_priv(netdev
);
2308 return ionic_lif_hwstamp_set(lif
, ifr
);
2310 return ionic_lif_hwstamp_get(lif
, ifr
);
2316 static int ionic_get_vf_config(struct net_device
*netdev
,
2317 int vf
, struct ifla_vf_info
*ivf
)
2319 struct ionic_lif
*lif
= netdev_priv(netdev
);
2320 struct ionic
*ionic
= lif
->ionic
;
2323 if (!netif_device_present(netdev
))
2326 down_read(&ionic
->vf_op_lock
);
2328 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2332 ivf
->vlan
= le16_to_cpu(ionic
->vfs
[vf
].vlanid
);
2334 ivf
->spoofchk
= ionic
->vfs
[vf
].spoofchk
;
2335 ivf
->linkstate
= ionic
->vfs
[vf
].linkstate
;
2336 ivf
->max_tx_rate
= le32_to_cpu(ionic
->vfs
[vf
].maxrate
);
2337 ivf
->trusted
= ionic
->vfs
[vf
].trusted
;
2338 ether_addr_copy(ivf
->mac
, ionic
->vfs
[vf
].macaddr
);
2341 up_read(&ionic
->vf_op_lock
);
2345 static int ionic_get_vf_stats(struct net_device
*netdev
, int vf
,
2346 struct ifla_vf_stats
*vf_stats
)
2348 struct ionic_lif
*lif
= netdev_priv(netdev
);
2349 struct ionic
*ionic
= lif
->ionic
;
2350 struct ionic_lif_stats
*vs
;
2353 if (!netif_device_present(netdev
))
2356 down_read(&ionic
->vf_op_lock
);
2358 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2361 memset(vf_stats
, 0, sizeof(*vf_stats
));
2362 vs
= &ionic
->vfs
[vf
].stats
;
2364 vf_stats
->rx_packets
= le64_to_cpu(vs
->rx_ucast_packets
);
2365 vf_stats
->tx_packets
= le64_to_cpu(vs
->tx_ucast_packets
);
2366 vf_stats
->rx_bytes
= le64_to_cpu(vs
->rx_ucast_bytes
);
2367 vf_stats
->tx_bytes
= le64_to_cpu(vs
->tx_ucast_bytes
);
2368 vf_stats
->broadcast
= le64_to_cpu(vs
->rx_bcast_packets
);
2369 vf_stats
->multicast
= le64_to_cpu(vs
->rx_mcast_packets
);
2370 vf_stats
->rx_dropped
= le64_to_cpu(vs
->rx_ucast_drop_packets
) +
2371 le64_to_cpu(vs
->rx_mcast_drop_packets
) +
2372 le64_to_cpu(vs
->rx_bcast_drop_packets
);
2373 vf_stats
->tx_dropped
= le64_to_cpu(vs
->tx_ucast_drop_packets
) +
2374 le64_to_cpu(vs
->tx_mcast_drop_packets
) +
2375 le64_to_cpu(vs
->tx_bcast_drop_packets
);
2378 up_read(&ionic
->vf_op_lock
);
2382 static int ionic_set_vf_mac(struct net_device
*netdev
, int vf
, u8
*mac
)
2384 struct ionic_lif
*lif
= netdev_priv(netdev
);
2385 struct ionic
*ionic
= lif
->ionic
;
2388 if (!(is_zero_ether_addr(mac
) || is_valid_ether_addr(mac
)))
2391 if (!netif_device_present(netdev
))
2394 down_write(&ionic
->vf_op_lock
);
2396 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2399 ret
= ionic_set_vf_config(ionic
, vf
, IONIC_VF_ATTR_MAC
, mac
);
2401 ether_addr_copy(ionic
->vfs
[vf
].macaddr
, mac
);
2404 up_write(&ionic
->vf_op_lock
);
2408 static int ionic_set_vf_vlan(struct net_device
*netdev
, int vf
, u16 vlan
,
2409 u8 qos
, __be16 proto
)
2411 struct ionic_lif
*lif
= netdev_priv(netdev
);
2412 struct ionic
*ionic
= lif
->ionic
;
2415 /* until someday when we support qos */
2422 if (proto
!= htons(ETH_P_8021Q
))
2423 return -EPROTONOSUPPORT
;
2425 if (!netif_device_present(netdev
))
2428 down_write(&ionic
->vf_op_lock
);
2430 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2433 ret
= ionic_set_vf_config(ionic
, vf
,
2434 IONIC_VF_ATTR_VLAN
, (u8
*)&vlan
);
2436 ionic
->vfs
[vf
].vlanid
= cpu_to_le16(vlan
);
2439 up_write(&ionic
->vf_op_lock
);
2443 static int ionic_set_vf_rate(struct net_device
*netdev
, int vf
,
2444 int tx_min
, int tx_max
)
2446 struct ionic_lif
*lif
= netdev_priv(netdev
);
2447 struct ionic
*ionic
= lif
->ionic
;
2450 /* setting the min just seems silly */
2454 if (!netif_device_present(netdev
))
2457 down_write(&ionic
->vf_op_lock
);
2459 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2462 ret
= ionic_set_vf_config(ionic
, vf
,
2463 IONIC_VF_ATTR_RATE
, (u8
*)&tx_max
);
2465 lif
->ionic
->vfs
[vf
].maxrate
= cpu_to_le32(tx_max
);
2468 up_write(&ionic
->vf_op_lock
);
2472 static int ionic_set_vf_spoofchk(struct net_device
*netdev
, int vf
, bool set
)
2474 struct ionic_lif
*lif
= netdev_priv(netdev
);
2475 struct ionic
*ionic
= lif
->ionic
;
2476 u8 data
= set
; /* convert to u8 for config */
2479 if (!netif_device_present(netdev
))
2482 down_write(&ionic
->vf_op_lock
);
2484 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2487 ret
= ionic_set_vf_config(ionic
, vf
,
2488 IONIC_VF_ATTR_SPOOFCHK
, &data
);
2490 ionic
->vfs
[vf
].spoofchk
= data
;
2493 up_write(&ionic
->vf_op_lock
);
2497 static int ionic_set_vf_trust(struct net_device
*netdev
, int vf
, bool set
)
2499 struct ionic_lif
*lif
= netdev_priv(netdev
);
2500 struct ionic
*ionic
= lif
->ionic
;
2501 u8 data
= set
; /* convert to u8 for config */
2504 if (!netif_device_present(netdev
))
2507 down_write(&ionic
->vf_op_lock
);
2509 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2512 ret
= ionic_set_vf_config(ionic
, vf
,
2513 IONIC_VF_ATTR_TRUST
, &data
);
2515 ionic
->vfs
[vf
].trusted
= data
;
2518 up_write(&ionic
->vf_op_lock
);
2522 static int ionic_set_vf_link_state(struct net_device
*netdev
, int vf
, int set
)
2524 struct ionic_lif
*lif
= netdev_priv(netdev
);
2525 struct ionic
*ionic
= lif
->ionic
;
2530 case IFLA_VF_LINK_STATE_ENABLE
:
2531 data
= IONIC_VF_LINK_STATUS_UP
;
2533 case IFLA_VF_LINK_STATE_DISABLE
:
2534 data
= IONIC_VF_LINK_STATUS_DOWN
;
2536 case IFLA_VF_LINK_STATE_AUTO
:
2537 data
= IONIC_VF_LINK_STATUS_AUTO
;
2543 if (!netif_device_present(netdev
))
2546 down_write(&ionic
->vf_op_lock
);
2548 if (vf
>= pci_num_vf(ionic
->pdev
) || !ionic
->vfs
) {
2551 ret
= ionic_set_vf_config(ionic
, vf
,
2552 IONIC_VF_ATTR_LINKSTATE
, &data
);
2554 ionic
->vfs
[vf
].linkstate
= set
;
2557 up_write(&ionic
->vf_op_lock
);
2561 static const struct net_device_ops ionic_netdev_ops
= {
2562 .ndo_open
= ionic_open
,
2563 .ndo_stop
= ionic_stop
,
2564 .ndo_eth_ioctl
= ionic_eth_ioctl
,
2565 .ndo_start_xmit
= ionic_start_xmit
,
2566 .ndo_get_stats64
= ionic_get_stats64
,
2567 .ndo_set_rx_mode
= ionic_ndo_set_rx_mode
,
2568 .ndo_set_features
= ionic_set_features
,
2569 .ndo_set_mac_address
= ionic_set_mac_address
,
2570 .ndo_validate_addr
= eth_validate_addr
,
2571 .ndo_tx_timeout
= ionic_tx_timeout
,
2572 .ndo_change_mtu
= ionic_change_mtu
,
2573 .ndo_vlan_rx_add_vid
= ionic_vlan_rx_add_vid
,
2574 .ndo_vlan_rx_kill_vid
= ionic_vlan_rx_kill_vid
,
2575 .ndo_set_vf_vlan
= ionic_set_vf_vlan
,
2576 .ndo_set_vf_trust
= ionic_set_vf_trust
,
2577 .ndo_set_vf_mac
= ionic_set_vf_mac
,
2578 .ndo_set_vf_rate
= ionic_set_vf_rate
,
2579 .ndo_set_vf_spoofchk
= ionic_set_vf_spoofchk
,
2580 .ndo_get_vf_config
= ionic_get_vf_config
,
2581 .ndo_set_vf_link_state
= ionic_set_vf_link_state
,
2582 .ndo_get_vf_stats
= ionic_get_vf_stats
,
2585 static void ionic_swap_queues(struct ionic_qcq
*a
, struct ionic_qcq
*b
)
2587 /* only swapping the queues, not the napi, flags, or other stuff */
2588 swap(a
->q
.features
, b
->q
.features
);
2589 swap(a
->q
.num_descs
, b
->q
.num_descs
);
2590 swap(a
->q
.desc_size
, b
->q
.desc_size
);
2591 swap(a
->q
.base
, b
->q
.base
);
2592 swap(a
->q
.base_pa
, b
->q
.base_pa
);
2593 swap(a
->q
.info
, b
->q
.info
);
2594 swap(a
->q_base
, b
->q_base
);
2595 swap(a
->q_base_pa
, b
->q_base_pa
);
2596 swap(a
->q_size
, b
->q_size
);
2598 swap(a
->q
.sg_desc_size
, b
->q
.sg_desc_size
);
2599 swap(a
->q
.sg_base
, b
->q
.sg_base
);
2600 swap(a
->q
.sg_base_pa
, b
->q
.sg_base_pa
);
2601 swap(a
->sg_base
, b
->sg_base
);
2602 swap(a
->sg_base_pa
, b
->sg_base_pa
);
2603 swap(a
->sg_size
, b
->sg_size
);
2605 swap(a
->cq
.num_descs
, b
->cq
.num_descs
);
2606 swap(a
->cq
.desc_size
, b
->cq
.desc_size
);
2607 swap(a
->cq
.base
, b
->cq
.base
);
2608 swap(a
->cq
.base_pa
, b
->cq
.base_pa
);
2609 swap(a
->cq
.info
, b
->cq
.info
);
2610 swap(a
->cq_base
, b
->cq_base
);
2611 swap(a
->cq_base_pa
, b
->cq_base_pa
);
2612 swap(a
->cq_size
, b
->cq_size
);
2614 ionic_debugfs_del_qcq(a
);
2615 ionic_debugfs_add_qcq(a
->q
.lif
, a
);
2618 int ionic_reconfigure_queues(struct ionic_lif
*lif
,
2619 struct ionic_queue_params
*qparam
)
2621 unsigned int comp_sz
, desc_sz
, num_desc
, sg_desc_sz
;
2622 struct ionic_qcq
**tx_qcqs
= NULL
;
2623 struct ionic_qcq
**rx_qcqs
= NULL
;
2624 unsigned int flags
, i
;
2627 /* allocate temporary qcq arrays to hold new queue structs */
2628 if (qparam
->nxqs
!= lif
->nxqs
|| qparam
->ntxq_descs
!= lif
->ntxq_descs
) {
2629 tx_qcqs
= devm_kcalloc(lif
->ionic
->dev
, lif
->ionic
->ntxqs_per_lif
,
2630 sizeof(struct ionic_qcq
*), GFP_KERNEL
);
2636 if (qparam
->nxqs
!= lif
->nxqs
||
2637 qparam
->nrxq_descs
!= lif
->nrxq_descs
||
2638 qparam
->rxq_features
!= lif
->rxq_features
) {
2639 rx_qcqs
= devm_kcalloc(lif
->ionic
->dev
, lif
->ionic
->nrxqs_per_lif
,
2640 sizeof(struct ionic_qcq
*), GFP_KERNEL
);
2647 /* allocate new desc_info and rings, but leave the interrupt setup
2648 * until later so as to not mess with the still-running queues
2651 num_desc
= qparam
->ntxq_descs
;
2652 desc_sz
= sizeof(struct ionic_txq_desc
);
2653 comp_sz
= sizeof(struct ionic_txq_comp
);
2655 if (lif
->qtype_info
[IONIC_QTYPE_TXQ
].version
>= 1 &&
2656 lif
->qtype_info
[IONIC_QTYPE_TXQ
].sg_desc_sz
==
2657 sizeof(struct ionic_txq_sg_desc_v1
))
2658 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc_v1
);
2660 sg_desc_sz
= sizeof(struct ionic_txq_sg_desc
);
2662 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2663 flags
= lif
->txqcqs
[i
]->flags
& ~IONIC_QCQ_F_INTR
;
2664 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_TXQ
, i
, "tx", flags
,
2665 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
2666 lif
->kern_pid
, &tx_qcqs
[i
]);
2673 num_desc
= qparam
->nrxq_descs
;
2674 desc_sz
= sizeof(struct ionic_rxq_desc
);
2675 comp_sz
= sizeof(struct ionic_rxq_comp
);
2676 sg_desc_sz
= sizeof(struct ionic_rxq_sg_desc
);
2678 if (qparam
->rxq_features
& IONIC_Q_F_2X_CQ_DESC
)
2681 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2682 flags
= lif
->rxqcqs
[i
]->flags
& ~IONIC_QCQ_F_INTR
;
2683 err
= ionic_qcq_alloc(lif
, IONIC_QTYPE_RXQ
, i
, "rx", flags
,
2684 num_desc
, desc_sz
, comp_sz
, sg_desc_sz
,
2685 lif
->kern_pid
, &rx_qcqs
[i
]);
2689 rx_qcqs
[i
]->q
.features
= qparam
->rxq_features
;
2693 /* stop and clean the queues */
2694 ionic_stop_queues_reconfig(lif
);
2696 if (qparam
->nxqs
!= lif
->nxqs
) {
2697 err
= netif_set_real_num_tx_queues(lif
->netdev
, qparam
->nxqs
);
2699 goto err_out_reinit_unlock
;
2700 err
= netif_set_real_num_rx_queues(lif
->netdev
, qparam
->nxqs
);
2702 netif_set_real_num_tx_queues(lif
->netdev
, lif
->nxqs
);
2703 goto err_out_reinit_unlock
;
2707 /* swap new desc_info and rings, keeping existing interrupt config */
2709 lif
->ntxq_descs
= qparam
->ntxq_descs
;
2710 for (i
= 0; i
< qparam
->nxqs
; i
++)
2711 ionic_swap_queues(lif
->txqcqs
[i
], tx_qcqs
[i
]);
2715 lif
->nrxq_descs
= qparam
->nrxq_descs
;
2716 for (i
= 0; i
< qparam
->nxqs
; i
++)
2717 ionic_swap_queues(lif
->rxqcqs
[i
], rx_qcqs
[i
]);
2720 /* if we need to change the interrupt layout, this is the time */
2721 if (qparam
->intr_split
!= test_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
) ||
2722 qparam
->nxqs
!= lif
->nxqs
) {
2723 if (qparam
->intr_split
) {
2724 set_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
);
2726 clear_bit(IONIC_LIF_F_SPLIT_INTR
, lif
->state
);
2727 lif
->tx_coalesce_usecs
= lif
->rx_coalesce_usecs
;
2728 lif
->tx_coalesce_hw
= lif
->rx_coalesce_hw
;
2731 /* clear existing interrupt assignments */
2732 for (i
= 0; i
< lif
->ionic
->ntxqs_per_lif
; i
++) {
2733 ionic_qcq_intr_free(lif
, lif
->txqcqs
[i
]);
2734 ionic_qcq_intr_free(lif
, lif
->rxqcqs
[i
]);
2737 /* re-assign the interrupts */
2738 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2739 lif
->rxqcqs
[i
]->flags
|= IONIC_QCQ_F_INTR
;
2740 err
= ionic_alloc_qcq_interrupt(lif
, lif
->rxqcqs
[i
]);
2741 ionic_intr_coal_init(lif
->ionic
->idev
.intr_ctrl
,
2742 lif
->rxqcqs
[i
]->intr
.index
,
2743 lif
->rx_coalesce_hw
);
2745 if (qparam
->intr_split
) {
2746 lif
->txqcqs
[i
]->flags
|= IONIC_QCQ_F_INTR
;
2747 err
= ionic_alloc_qcq_interrupt(lif
, lif
->txqcqs
[i
]);
2748 ionic_intr_coal_init(lif
->ionic
->idev
.intr_ctrl
,
2749 lif
->txqcqs
[i
]->intr
.index
,
2750 lif
->tx_coalesce_hw
);
2751 if (test_bit(IONIC_LIF_F_TX_DIM_INTR
, lif
->state
))
2752 lif
->txqcqs
[i
]->intr
.dim_coal_hw
= lif
->tx_coalesce_hw
;
2754 lif
->txqcqs
[i
]->flags
&= ~IONIC_QCQ_F_INTR
;
2755 ionic_link_qcq_interrupts(lif
->rxqcqs
[i
], lif
->txqcqs
[i
]);
2760 /* now we can rework the debugfs mappings */
2762 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2763 ionic_debugfs_del_qcq(lif
->txqcqs
[i
]);
2764 ionic_debugfs_add_qcq(lif
, lif
->txqcqs
[i
]);
2769 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2770 ionic_debugfs_del_qcq(lif
->rxqcqs
[i
]);
2771 ionic_debugfs_add_qcq(lif
, lif
->rxqcqs
[i
]);
2775 swap(lif
->nxqs
, qparam
->nxqs
);
2776 swap(lif
->rxq_features
, qparam
->rxq_features
);
2778 err_out_reinit_unlock
:
2779 /* re-init the queues, but don't lose an error code */
2781 ionic_start_queues_reconfig(lif
);
2783 err
= ionic_start_queues_reconfig(lif
);
2786 /* free old allocs without cleaning intr */
2787 for (i
= 0; i
< qparam
->nxqs
; i
++) {
2788 if (tx_qcqs
&& tx_qcqs
[i
]) {
2789 tx_qcqs
[i
]->flags
&= ~IONIC_QCQ_F_INTR
;
2790 ionic_qcq_free(lif
, tx_qcqs
[i
]);
2791 devm_kfree(lif
->ionic
->dev
, tx_qcqs
[i
]);
2794 if (rx_qcqs
&& rx_qcqs
[i
]) {
2795 rx_qcqs
[i
]->flags
&= ~IONIC_QCQ_F_INTR
;
2796 ionic_qcq_free(lif
, rx_qcqs
[i
]);
2797 devm_kfree(lif
->ionic
->dev
, rx_qcqs
[i
]);
2804 devm_kfree(lif
->ionic
->dev
, rx_qcqs
);
2808 devm_kfree(lif
->ionic
->dev
, tx_qcqs
);
2812 /* clean the unused dma and info allocations when new set is smaller
2813 * than the full array, but leave the qcq shells in place
2815 for (i
= lif
->nxqs
; i
< lif
->ionic
->ntxqs_per_lif
; i
++) {
2816 lif
->txqcqs
[i
]->flags
&= ~IONIC_QCQ_F_INTR
;
2817 ionic_qcq_free(lif
, lif
->txqcqs
[i
]);
2819 lif
->rxqcqs
[i
]->flags
&= ~IONIC_QCQ_F_INTR
;
2820 ionic_qcq_free(lif
, lif
->rxqcqs
[i
]);
2824 netdev_info(lif
->netdev
, "%s: failed %d\n", __func__
, err
);
2829 int ionic_lif_alloc(struct ionic
*ionic
)
2831 struct device
*dev
= ionic
->dev
;
2832 union ionic_lif_identity
*lid
;
2833 struct net_device
*netdev
;
2834 struct ionic_lif
*lif
;
2838 lid
= kzalloc(sizeof(*lid
), GFP_KERNEL
);
2842 netdev
= alloc_etherdev_mqs(sizeof(*lif
),
2843 ionic
->ntxqs_per_lif
, ionic
->ntxqs_per_lif
);
2845 dev_err(dev
, "Cannot allocate netdev, aborting\n");
2847 goto err_out_free_lid
;
2850 SET_NETDEV_DEV(netdev
, dev
);
2852 lif
= netdev_priv(netdev
);
2853 lif
->netdev
= netdev
;
2855 netdev
->netdev_ops
= &ionic_netdev_ops
;
2856 ionic_ethtool_set_ops(netdev
);
2858 netdev
->watchdog_timeo
= 2 * HZ
;
2859 netif_carrier_off(netdev
);
2861 lif
->identity
= lid
;
2862 lif
->lif_type
= IONIC_LIF_TYPE_CLASSIC
;
2863 err
= ionic_lif_identify(ionic
, lif
->lif_type
, lif
->identity
);
2865 dev_err(ionic
->dev
, "Cannot identify type %d: %d\n",
2866 lif
->lif_type
, err
);
2867 goto err_out_free_netdev
;
2869 lif
->netdev
->min_mtu
= max_t(unsigned int, ETH_MIN_MTU
,
2870 le32_to_cpu(lif
->identity
->eth
.min_frame_size
));
2871 lif
->netdev
->max_mtu
=
2872 le32_to_cpu(lif
->identity
->eth
.max_frame_size
) - ETH_HLEN
- VLAN_HLEN
;
2874 lif
->neqs
= ionic
->neqs_per_lif
;
2875 lif
->nxqs
= ionic
->ntxqs_per_lif
;
2880 if (is_kdump_kernel()) {
2881 lif
->ntxq_descs
= IONIC_MIN_TXRX_DESC
;
2882 lif
->nrxq_descs
= IONIC_MIN_TXRX_DESC
;
2884 lif
->ntxq_descs
= IONIC_DEF_TXRX_DESC
;
2885 lif
->nrxq_descs
= IONIC_DEF_TXRX_DESC
;
2888 /* Convert the default coalesce value to actual hw resolution */
2889 lif
->rx_coalesce_usecs
= IONIC_ITR_COAL_USEC_DEFAULT
;
2890 lif
->rx_coalesce_hw
= ionic_coal_usec_to_hw(lif
->ionic
,
2891 lif
->rx_coalesce_usecs
);
2892 lif
->tx_coalesce_usecs
= lif
->rx_coalesce_usecs
;
2893 lif
->tx_coalesce_hw
= lif
->rx_coalesce_hw
;
2894 set_bit(IONIC_LIF_F_RX_DIM_INTR
, lif
->state
);
2895 set_bit(IONIC_LIF_F_TX_DIM_INTR
, lif
->state
);
2897 snprintf(lif
->name
, sizeof(lif
->name
), "lif%u", lif
->index
);
2899 spin_lock_init(&lif
->adminq_lock
);
2901 spin_lock_init(&lif
->deferred
.lock
);
2902 INIT_LIST_HEAD(&lif
->deferred
.list
);
2903 INIT_WORK(&lif
->deferred
.work
, ionic_lif_deferred_work
);
2905 /* allocate lif info */
2906 lif
->info_sz
= ALIGN(sizeof(*lif
->info
), PAGE_SIZE
);
2907 lif
->info
= dma_alloc_coherent(dev
, lif
->info_sz
,
2908 &lif
->info_pa
, GFP_KERNEL
);
2910 dev_err(dev
, "Failed to allocate lif info, aborting\n");
2912 goto err_out_free_netdev
;
2915 ionic_debugfs_add_lif(lif
);
2917 /* allocate control queues and txrx queue arrays */
2918 ionic_lif_queue_identify(lif
);
2919 err
= ionic_qcqs_alloc(lif
);
2921 goto err_out_free_lif_info
;
2923 /* allocate rss indirection table */
2924 tbl_sz
= le16_to_cpu(lif
->ionic
->ident
.lif
.eth
.rss_ind_tbl_sz
);
2925 lif
->rss_ind_tbl_sz
= sizeof(*lif
->rss_ind_tbl
) * tbl_sz
;
2926 lif
->rss_ind_tbl
= dma_alloc_coherent(dev
, lif
->rss_ind_tbl_sz
,
2927 &lif
->rss_ind_tbl_pa
,
2930 if (!lif
->rss_ind_tbl
) {
2932 dev_err(dev
, "Failed to allocate rss indirection table, aborting\n");
2933 goto err_out_free_qcqs
;
2935 netdev_rss_key_fill(lif
->rss_hash_key
, IONIC_RSS_HASH_KEY_SIZE
);
2937 ionic_lif_alloc_phc(lif
);
2942 ionic_qcqs_free(lif
);
2943 err_out_free_lif_info
:
2944 dma_free_coherent(dev
, lif
->info_sz
, lif
->info
, lif
->info_pa
);
2947 err_out_free_netdev
:
2948 free_netdev(lif
->netdev
);
2956 static void ionic_lif_reset(struct ionic_lif
*lif
)
2958 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
2960 mutex_lock(&lif
->ionic
->dev_cmd_lock
);
2961 ionic_dev_cmd_lif_reset(idev
, lif
->index
);
2962 ionic_dev_cmd_wait(lif
->ionic
, DEVCMD_TIMEOUT
);
2963 mutex_unlock(&lif
->ionic
->dev_cmd_lock
);
2966 static void ionic_lif_handle_fw_down(struct ionic_lif
*lif
)
2968 struct ionic
*ionic
= lif
->ionic
;
2970 if (test_and_set_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
2973 dev_info(ionic
->dev
, "FW Down: Stopping LIFs\n");
2975 netif_device_detach(lif
->netdev
);
2977 if (test_bit(IONIC_LIF_F_UP
, lif
->state
)) {
2978 dev_info(ionic
->dev
, "Surprise FW stop, stopping queues\n");
2979 mutex_lock(&lif
->queue_lock
);
2980 ionic_stop_queues(lif
);
2981 mutex_unlock(&lif
->queue_lock
);
2984 if (netif_running(lif
->netdev
)) {
2985 ionic_txrx_deinit(lif
);
2986 ionic_txrx_free(lif
);
2988 ionic_lif_deinit(lif
);
2990 ionic_qcqs_free(lif
);
2992 dev_info(ionic
->dev
, "FW Down: LIFs stopped\n");
2995 static void ionic_lif_handle_fw_up(struct ionic_lif
*lif
)
2997 struct ionic
*ionic
= lif
->ionic
;
3000 if (!test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
3003 dev_info(ionic
->dev
, "FW Up: restarting LIFs\n");
3005 ionic_init_devinfo(ionic
);
3006 err
= ionic_identify(ionic
);
3009 err
= ionic_port_identify(ionic
);
3012 err
= ionic_port_init(ionic
);
3015 err
= ionic_qcqs_alloc(lif
);
3019 err
= ionic_lif_init(lif
);
3023 if (lif
->registered
)
3024 ionic_lif_set_netdev_info(lif
);
3026 ionic_rx_filter_replay(lif
);
3028 if (netif_running(lif
->netdev
)) {
3029 err
= ionic_txrx_alloc(lif
);
3031 goto err_lifs_deinit
;
3033 err
= ionic_txrx_init(lif
);
3038 clear_bit(IONIC_LIF_F_FW_RESET
, lif
->state
);
3039 ionic_link_status_check_request(lif
, CAN_SLEEP
);
3040 netif_device_attach(lif
->netdev
);
3041 dev_info(ionic
->dev
, "FW Up: LIFs restarted\n");
3043 /* restore the hardware timestamping queues */
3044 ionic_lif_hwstamp_replay(lif
);
3049 ionic_txrx_free(lif
);
3051 ionic_lif_deinit(lif
);
3053 ionic_qcqs_free(lif
);
3055 dev_err(ionic
->dev
, "FW Up: LIFs restart failed - err %d\n", err
);
3058 void ionic_lif_free(struct ionic_lif
*lif
)
3060 struct device
*dev
= lif
->ionic
->dev
;
3062 ionic_lif_free_phc(lif
);
3064 /* free rss indirection table */
3065 dma_free_coherent(dev
, lif
->rss_ind_tbl_sz
, lif
->rss_ind_tbl
,
3066 lif
->rss_ind_tbl_pa
);
3067 lif
->rss_ind_tbl
= NULL
;
3068 lif
->rss_ind_tbl_pa
= 0;
3071 ionic_qcqs_free(lif
);
3072 if (!test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
))
3073 ionic_lif_reset(lif
);
3076 kfree(lif
->identity
);
3077 dma_free_coherent(dev
, lif
->info_sz
, lif
->info
, lif
->info_pa
);
3081 /* unmap doorbell page */
3082 ionic_bus_unmap_dbpage(lif
->ionic
, lif
->kern_dbpage
);
3083 lif
->kern_dbpage
= NULL
;
3084 kfree(lif
->dbid_inuse
);
3085 lif
->dbid_inuse
= NULL
;
3087 /* free netdev & lif */
3088 ionic_debugfs_del_lif(lif
);
3089 free_netdev(lif
->netdev
);
3092 void ionic_lif_deinit(struct ionic_lif
*lif
)
3094 if (!test_and_clear_bit(IONIC_LIF_F_INITED
, lif
->state
))
3097 if (!test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
)) {
3098 cancel_work_sync(&lif
->deferred
.work
);
3099 cancel_work_sync(&lif
->tx_timeout_work
);
3100 ionic_rx_filters_deinit(lif
);
3101 if (lif
->netdev
->features
& NETIF_F_RXHASH
)
3102 ionic_lif_rss_deinit(lif
);
3105 napi_disable(&lif
->adminqcq
->napi
);
3106 ionic_lif_qcq_deinit(lif
, lif
->notifyqcq
);
3107 ionic_lif_qcq_deinit(lif
, lif
->adminqcq
);
3109 mutex_destroy(&lif
->config_lock
);
3110 mutex_destroy(&lif
->queue_lock
);
3111 ionic_lif_reset(lif
);
3114 static int ionic_lif_adminq_init(struct ionic_lif
*lif
)
3116 struct device
*dev
= lif
->ionic
->dev
;
3117 struct ionic_q_init_comp comp
;
3118 struct ionic_dev
*idev
;
3119 struct ionic_qcq
*qcq
;
3120 struct ionic_queue
*q
;
3123 idev
= &lif
->ionic
->idev
;
3124 qcq
= lif
->adminqcq
;
3127 mutex_lock(&lif
->ionic
->dev_cmd_lock
);
3128 ionic_dev_cmd_adminq_init(idev
, qcq
, lif
->index
, qcq
->intr
.index
);
3129 err
= ionic_dev_cmd_wait(lif
->ionic
, DEVCMD_TIMEOUT
);
3130 ionic_dev_cmd_comp(idev
, (union ionic_dev_cmd_comp
*)&comp
);
3131 mutex_unlock(&lif
->ionic
->dev_cmd_lock
);
3133 netdev_err(lif
->netdev
, "adminq init failed %d\n", err
);
3137 q
->hw_type
= comp
.hw_type
;
3138 q
->hw_index
= le32_to_cpu(comp
.hw_index
);
3139 q
->dbval
= IONIC_DBELL_QID(q
->hw_index
);
3141 dev_dbg(dev
, "adminq->hw_type %d\n", q
->hw_type
);
3142 dev_dbg(dev
, "adminq->hw_index %d\n", q
->hw_index
);
3144 netif_napi_add(lif
->netdev
, &qcq
->napi
, ionic_adminq_napi
,
3147 napi_enable(&qcq
->napi
);
3149 if (qcq
->flags
& IONIC_QCQ_F_INTR
)
3150 ionic_intr_mask(idev
->intr_ctrl
, qcq
->intr
.index
,
3151 IONIC_INTR_MASK_CLEAR
);
3153 qcq
->flags
|= IONIC_QCQ_F_INITED
;
3158 static int ionic_lif_notifyq_init(struct ionic_lif
*lif
)
3160 struct ionic_qcq
*qcq
= lif
->notifyqcq
;
3161 struct device
*dev
= lif
->ionic
->dev
;
3162 struct ionic_queue
*q
= &qcq
->q
;
3165 struct ionic_admin_ctx ctx
= {
3166 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
3168 .opcode
= IONIC_CMD_Q_INIT
,
3169 .lif_index
= cpu_to_le16(lif
->index
),
3171 .ver
= lif
->qtype_info
[q
->type
].version
,
3172 .index
= cpu_to_le32(q
->index
),
3173 .flags
= cpu_to_le16(IONIC_QINIT_F_IRQ
|
3175 .intr_index
= cpu_to_le16(lif
->adminqcq
->intr
.index
),
3176 .pid
= cpu_to_le16(q
->pid
),
3177 .ring_size
= ilog2(q
->num_descs
),
3178 .ring_base
= cpu_to_le64(q
->base_pa
),
3182 dev_dbg(dev
, "notifyq_init.pid %d\n", ctx
.cmd
.q_init
.pid
);
3183 dev_dbg(dev
, "notifyq_init.index %d\n", ctx
.cmd
.q_init
.index
);
3184 dev_dbg(dev
, "notifyq_init.ring_base 0x%llx\n", ctx
.cmd
.q_init
.ring_base
);
3185 dev_dbg(dev
, "notifyq_init.ring_size %d\n", ctx
.cmd
.q_init
.ring_size
);
3187 err
= ionic_adminq_post_wait(lif
, &ctx
);
3192 q
->hw_type
= ctx
.comp
.q_init
.hw_type
;
3193 q
->hw_index
= le32_to_cpu(ctx
.comp
.q_init
.hw_index
);
3194 q
->dbval
= IONIC_DBELL_QID(q
->hw_index
);
3196 dev_dbg(dev
, "notifyq->hw_type %d\n", q
->hw_type
);
3197 dev_dbg(dev
, "notifyq->hw_index %d\n", q
->hw_index
);
3199 /* preset the callback info */
3200 q
->info
[0].cb_arg
= lif
;
3202 qcq
->flags
|= IONIC_QCQ_F_INITED
;
3207 static int ionic_station_set(struct ionic_lif
*lif
)
3209 struct net_device
*netdev
= lif
->netdev
;
3210 struct ionic_admin_ctx ctx
= {
3211 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
3212 .cmd
.lif_getattr
= {
3213 .opcode
= IONIC_CMD_LIF_GETATTR
,
3214 .index
= cpu_to_le16(lif
->index
),
3215 .attr
= IONIC_LIF_ATTR_MAC
,
3218 struct sockaddr addr
;
3221 err
= ionic_adminq_post_wait(lif
, &ctx
);
3224 netdev_dbg(lif
->netdev
, "found initial MAC addr %pM\n",
3225 ctx
.comp
.lif_getattr
.mac
);
3226 if (is_zero_ether_addr(ctx
.comp
.lif_getattr
.mac
))
3229 if (!is_zero_ether_addr(netdev
->dev_addr
)) {
3230 /* If the netdev mac is non-zero and doesn't match the default
3231 * device address, it was set by something earlier and we're
3232 * likely here again after a fw-upgrade reset. We need to be
3233 * sure the netdev mac is in our filter list.
3235 if (!ether_addr_equal(ctx
.comp
.lif_getattr
.mac
,
3237 ionic_lif_addr_add(lif
, netdev
->dev_addr
);
3239 /* Update the netdev mac with the device's mac */
3240 memcpy(addr
.sa_data
, ctx
.comp
.lif_getattr
.mac
, netdev
->addr_len
);
3241 addr
.sa_family
= AF_INET
;
3242 err
= eth_prepare_mac_addr_change(netdev
, &addr
);
3244 netdev_warn(lif
->netdev
, "ignoring bad MAC addr from NIC %pM - err %d\n",
3249 eth_commit_mac_addr_change(netdev
, &addr
);
3252 netdev_dbg(lif
->netdev
, "adding station MAC addr %pM\n",
3254 ionic_lif_addr_add(lif
, netdev
->dev_addr
);
3259 int ionic_lif_init(struct ionic_lif
*lif
)
3261 struct ionic_dev
*idev
= &lif
->ionic
->idev
;
3262 struct device
*dev
= lif
->ionic
->dev
;
3263 struct ionic_lif_init_comp comp
;
3267 mutex_lock(&lif
->ionic
->dev_cmd_lock
);
3268 ionic_dev_cmd_lif_init(idev
, lif
->index
, lif
->info_pa
);
3269 err
= ionic_dev_cmd_wait(lif
->ionic
, DEVCMD_TIMEOUT
);
3270 ionic_dev_cmd_comp(idev
, (union ionic_dev_cmd_comp
*)&comp
);
3271 mutex_unlock(&lif
->ionic
->dev_cmd_lock
);
3275 lif
->hw_index
= le16_to_cpu(comp
.hw_index
);
3276 mutex_init(&lif
->queue_lock
);
3277 mutex_init(&lif
->config_lock
);
3279 /* now that we have the hw_index we can figure out our doorbell page */
3280 lif
->dbid_count
= le32_to_cpu(lif
->ionic
->ident
.dev
.ndbpgs_per_lif
);
3281 if (!lif
->dbid_count
) {
3282 dev_err(dev
, "No doorbell pages, aborting\n");
3286 lif
->dbid_inuse
= bitmap_zalloc(lif
->dbid_count
, GFP_KERNEL
);
3287 if (!lif
->dbid_inuse
) {
3288 dev_err(dev
, "Failed alloc doorbell id bitmap, aborting\n");
3292 /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3293 set_bit(0, lif
->dbid_inuse
);
3296 dbpage_num
= ionic_db_page_num(lif
, lif
->kern_pid
);
3297 lif
->kern_dbpage
= ionic_bus_map_dbpage(lif
->ionic
, dbpage_num
);
3298 if (!lif
->kern_dbpage
) {
3299 dev_err(dev
, "Cannot map dbpage, aborting\n");
3301 goto err_out_free_dbid
;
3304 err
= ionic_lif_adminq_init(lif
);
3306 goto err_out_adminq_deinit
;
3308 if (lif
->ionic
->nnqs_per_lif
) {
3309 err
= ionic_lif_notifyq_init(lif
);
3311 goto err_out_notifyq_deinit
;
3314 err
= ionic_init_nic_features(lif
);
3316 goto err_out_notifyq_deinit
;
3318 if (!test_bit(IONIC_LIF_F_FW_RESET
, lif
->state
)) {
3319 err
= ionic_rx_filters_init(lif
);
3321 goto err_out_notifyq_deinit
;
3324 err
= ionic_station_set(lif
);
3326 goto err_out_notifyq_deinit
;
3328 lif
->rx_copybreak
= IONIC_RX_COPYBREAK_DEFAULT
;
3330 set_bit(IONIC_LIF_F_INITED
, lif
->state
);
3332 INIT_WORK(&lif
->tx_timeout_work
, ionic_tx_timeout_work
);
3336 err_out_notifyq_deinit
:
3337 ionic_lif_qcq_deinit(lif
, lif
->notifyqcq
);
3338 err_out_adminq_deinit
:
3339 ionic_lif_qcq_deinit(lif
, lif
->adminqcq
);
3340 ionic_lif_reset(lif
);
3341 ionic_bus_unmap_dbpage(lif
->ionic
, lif
->kern_dbpage
);
3342 lif
->kern_dbpage
= NULL
;
3344 kfree(lif
->dbid_inuse
);
3345 lif
->dbid_inuse
= NULL
;
3350 static void ionic_lif_notify_work(struct work_struct
*ws
)
3354 static void ionic_lif_set_netdev_info(struct ionic_lif
*lif
)
3356 struct ionic_admin_ctx ctx
= {
3357 .work
= COMPLETION_INITIALIZER_ONSTACK(ctx
.work
),
3358 .cmd
.lif_setattr
= {
3359 .opcode
= IONIC_CMD_LIF_SETATTR
,
3360 .index
= cpu_to_le16(lif
->index
),
3361 .attr
= IONIC_LIF_ATTR_NAME
,
3365 strlcpy(ctx
.cmd
.lif_setattr
.name
, lif
->netdev
->name
,
3366 sizeof(ctx
.cmd
.lif_setattr
.name
));
3368 ionic_adminq_post_wait(lif
, &ctx
);
3371 static struct ionic_lif
*ionic_netdev_lif(struct net_device
*netdev
)
3373 if (!netdev
|| netdev
->netdev_ops
->ndo_start_xmit
!= ionic_start_xmit
)
3376 return netdev_priv(netdev
);
3379 static int ionic_lif_notify(struct notifier_block
*nb
,
3380 unsigned long event
, void *info
)
3382 struct net_device
*ndev
= netdev_notifier_info_to_dev(info
);
3383 struct ionic
*ionic
= container_of(nb
, struct ionic
, nb
);
3384 struct ionic_lif
*lif
= ionic_netdev_lif(ndev
);
3386 if (!lif
|| lif
->ionic
!= ionic
)
3390 case NETDEV_CHANGENAME
:
3391 ionic_lif_set_netdev_info(lif
);
3398 int ionic_lif_register(struct ionic_lif
*lif
)
3402 ionic_lif_register_phc(lif
);
3404 INIT_WORK(&lif
->ionic
->nb_work
, ionic_lif_notify_work
);
3406 lif
->ionic
->nb
.notifier_call
= ionic_lif_notify
;
3408 err
= register_netdevice_notifier(&lif
->ionic
->nb
);
3410 lif
->ionic
->nb
.notifier_call
= NULL
;
3412 /* only register LIF0 for now */
3413 err
= register_netdev(lif
->netdev
);
3415 dev_err(lif
->ionic
->dev
, "Cannot register net device, aborting\n");
3416 ionic_lif_unregister_phc(lif
);
3420 ionic_link_status_check_request(lif
, CAN_SLEEP
);
3421 lif
->registered
= true;
3422 ionic_lif_set_netdev_info(lif
);
3427 void ionic_lif_unregister(struct ionic_lif
*lif
)
3429 if (lif
->ionic
->nb
.notifier_call
) {
3430 unregister_netdevice_notifier(&lif
->ionic
->nb
);
3431 cancel_work_sync(&lif
->ionic
->nb_work
);
3432 lif
->ionic
->nb
.notifier_call
= NULL
;
3435 if (lif
->netdev
->reg_state
== NETREG_REGISTERED
)
3436 unregister_netdev(lif
->netdev
);
3438 ionic_lif_unregister_phc(lif
);
3440 lif
->registered
= false;
3443 static void ionic_lif_queue_identify(struct ionic_lif
*lif
)
3445 union ionic_q_identity __iomem
*q_ident
;
3446 struct ionic
*ionic
= lif
->ionic
;
3447 struct ionic_dev
*idev
;
3451 idev
= &lif
->ionic
->idev
;
3452 q_ident
= (union ionic_q_identity __iomem
*)&idev
->dev_cmd_regs
->data
;
3454 for (qtype
= 0; qtype
< ARRAY_SIZE(ionic_qtype_versions
); qtype
++) {
3455 struct ionic_qtype_info
*qti
= &lif
->qtype_info
[qtype
];
3457 /* filter out the ones we know about */
3459 case IONIC_QTYPE_ADMINQ
:
3460 case IONIC_QTYPE_NOTIFYQ
:
3461 case IONIC_QTYPE_RXQ
:
3462 case IONIC_QTYPE_TXQ
:
3468 memset(qti
, 0, sizeof(*qti
));
3470 mutex_lock(&ionic
->dev_cmd_lock
);
3471 ionic_dev_cmd_queue_identify(idev
, lif
->lif_type
, qtype
,
3472 ionic_qtype_versions
[qtype
]);
3473 err
= ionic_dev_cmd_wait(ionic
, DEVCMD_TIMEOUT
);
3475 qti
->version
= readb(&q_ident
->version
);
3476 qti
->supported
= readb(&q_ident
->supported
);
3477 qti
->features
= readq(&q_ident
->features
);
3478 qti
->desc_sz
= readw(&q_ident
->desc_sz
);
3479 qti
->comp_sz
= readw(&q_ident
->comp_sz
);
3480 qti
->sg_desc_sz
= readw(&q_ident
->sg_desc_sz
);
3481 qti
->max_sg_elems
= readw(&q_ident
->max_sg_elems
);
3482 qti
->sg_desc_stride
= readw(&q_ident
->sg_desc_stride
);
3484 mutex_unlock(&ionic
->dev_cmd_lock
);
3486 if (err
== -EINVAL
) {
3487 dev_err(ionic
->dev
, "qtype %d not supported\n", qtype
);
3489 } else if (err
== -EIO
) {
3490 dev_err(ionic
->dev
, "q_ident failed, not supported on older FW\n");
3493 dev_err(ionic
->dev
, "q_ident failed, qtype %d: %d\n",
3498 dev_dbg(ionic
->dev
, " qtype[%d].version = %d\n",
3499 qtype
, qti
->version
);
3500 dev_dbg(ionic
->dev
, " qtype[%d].supported = 0x%02x\n",
3501 qtype
, qti
->supported
);
3502 dev_dbg(ionic
->dev
, " qtype[%d].features = 0x%04llx\n",
3503 qtype
, qti
->features
);
3504 dev_dbg(ionic
->dev
, " qtype[%d].desc_sz = %d\n",
3505 qtype
, qti
->desc_sz
);
3506 dev_dbg(ionic
->dev
, " qtype[%d].comp_sz = %d\n",
3507 qtype
, qti
->comp_sz
);
3508 dev_dbg(ionic
->dev
, " qtype[%d].sg_desc_sz = %d\n",
3509 qtype
, qti
->sg_desc_sz
);
3510 dev_dbg(ionic
->dev
, " qtype[%d].max_sg_elems = %d\n",
3511 qtype
, qti
->max_sg_elems
);
3512 dev_dbg(ionic
->dev
, " qtype[%d].sg_desc_stride = %d\n",
3513 qtype
, qti
->sg_desc_stride
);
3517 int ionic_lif_identify(struct ionic
*ionic
, u8 lif_type
,
3518 union ionic_lif_identity
*lid
)
3520 struct ionic_dev
*idev
= &ionic
->idev
;
3524 sz
= min(sizeof(*lid
), sizeof(idev
->dev_cmd_regs
->data
));
3526 mutex_lock(&ionic
->dev_cmd_lock
);
3527 ionic_dev_cmd_lif_identify(idev
, lif_type
, IONIC_IDENTITY_VERSION_1
);
3528 err
= ionic_dev_cmd_wait(ionic
, DEVCMD_TIMEOUT
);
3529 memcpy_fromio(lid
, &idev
->dev_cmd_regs
->data
, sz
);
3530 mutex_unlock(&ionic
->dev_cmd_lock
);
3534 dev_dbg(ionic
->dev
, "capabilities 0x%llx\n",
3535 le64_to_cpu(lid
->capabilities
));
3537 dev_dbg(ionic
->dev
, "eth.max_ucast_filters %d\n",
3538 le32_to_cpu(lid
->eth
.max_ucast_filters
));
3539 dev_dbg(ionic
->dev
, "eth.max_mcast_filters %d\n",
3540 le32_to_cpu(lid
->eth
.max_mcast_filters
));
3541 dev_dbg(ionic
->dev
, "eth.features 0x%llx\n",
3542 le64_to_cpu(lid
->eth
.config
.features
));
3543 dev_dbg(ionic
->dev
, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3544 le32_to_cpu(lid
->eth
.config
.queue_count
[IONIC_QTYPE_ADMINQ
]));
3545 dev_dbg(ionic
->dev
, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3546 le32_to_cpu(lid
->eth
.config
.queue_count
[IONIC_QTYPE_NOTIFYQ
]));
3547 dev_dbg(ionic
->dev
, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3548 le32_to_cpu(lid
->eth
.config
.queue_count
[IONIC_QTYPE_RXQ
]));
3549 dev_dbg(ionic
->dev
, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3550 le32_to_cpu(lid
->eth
.config
.queue_count
[IONIC_QTYPE_TXQ
]));
3551 dev_dbg(ionic
->dev
, "eth.config.name %s\n", lid
->eth
.config
.name
);
3552 dev_dbg(ionic
->dev
, "eth.config.mac %pM\n", lid
->eth
.config
.mac
);
3553 dev_dbg(ionic
->dev
, "eth.config.mtu %d\n",
3554 le32_to_cpu(lid
->eth
.config
.mtu
));
3559 int ionic_lif_size(struct ionic
*ionic
)
3561 struct ionic_identity
*ident
= &ionic
->ident
;
3562 unsigned int nintrs
, dev_nintrs
;
3563 union ionic_lif_config
*lc
;
3564 unsigned int ntxqs_per_lif
;
3565 unsigned int nrxqs_per_lif
;
3566 unsigned int neqs_per_lif
;
3567 unsigned int nnqs_per_lif
;
3568 unsigned int nxqs
, neqs
;
3569 unsigned int min_intrs
;
3572 /* retrieve basic values from FW */
3573 lc
= &ident
->lif
.eth
.config
;
3574 dev_nintrs
= le32_to_cpu(ident
->dev
.nintrs
);
3575 neqs_per_lif
= le32_to_cpu(ident
->lif
.rdma
.eq_qtype
.qid_count
);
3576 nnqs_per_lif
= le32_to_cpu(lc
->queue_count
[IONIC_QTYPE_NOTIFYQ
]);
3577 ntxqs_per_lif
= le32_to_cpu(lc
->queue_count
[IONIC_QTYPE_TXQ
]);
3578 nrxqs_per_lif
= le32_to_cpu(lc
->queue_count
[IONIC_QTYPE_RXQ
]);
3580 /* limit values to play nice with kdump */
3581 if (is_kdump_kernel()) {
3589 /* reserve last queue id for hardware timestamping */
3590 if (lc
->features
& cpu_to_le64(IONIC_ETH_HW_TIMESTAMP
)) {
3591 if (ntxqs_per_lif
<= 1 || nrxqs_per_lif
<= 1) {
3592 lc
->features
&= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP
);
3599 nxqs
= min(ntxqs_per_lif
, nrxqs_per_lif
);
3600 nxqs
= min(nxqs
, num_online_cpus());
3601 neqs
= min(neqs_per_lif
, num_online_cpus());
3605 * 1 for master lif adminq/notifyq
3606 * 1 for each CPU for master lif TxRx queue pairs
3607 * whatever's left is for RDMA queues
3609 nintrs
= 1 + nxqs
+ neqs
;
3610 min_intrs
= 2; /* adminq + 1 TxRx queue pair */
3612 if (nintrs
> dev_nintrs
)
3615 err
= ionic_bus_alloc_irq_vectors(ionic
, nintrs
);
3616 if (err
< 0 && err
!= -ENOSPC
) {
3617 dev_err(ionic
->dev
, "Can't get intrs from OS: %d\n", err
);
3623 if (err
!= nintrs
) {
3624 ionic_bus_free_irq_vectors(ionic
);
3628 ionic
->nnqs_per_lif
= nnqs_per_lif
;
3629 ionic
->neqs_per_lif
= neqs
;
3630 ionic
->ntxqs_per_lif
= nxqs
;
3631 ionic
->nrxqs_per_lif
= nxqs
;
3632 ionic
->nintrs
= nintrs
;
3634 ionic_debugfs_add_sizes(ionic
);
3639 if (nnqs_per_lif
> 1) {
3651 dev_err(ionic
->dev
, "Can't get minimum %d intrs from OS\n", min_intrs
);