]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/net/ethernet/cavium/thunder/nicvf_main.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branches 'asoc/topic/sgtl5000', 'asoc/topic/simple', 'asoc...
[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / cavium / thunder / nicvf_main.c
1 /*
2 * Copyright (C) 2015 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9 #include <linux/module.h>
10 #include <linux/interrupt.h>
11 #include <linux/pci.h>
12 #include <linux/netdevice.h>
13 #include <linux/if_vlan.h>
14 #include <linux/etherdevice.h>
15 #include <linux/ethtool.h>
16 #include <linux/log2.h>
17 #include <linux/prefetch.h>
18 #include <linux/irq.h>
19
20 #include "nic_reg.h"
21 #include "nic.h"
22 #include "nicvf_queues.h"
23 #include "thunder_bgx.h"
24
25 #define DRV_NAME "thunder-nicvf"
26 #define DRV_VERSION "1.0"
27
28 /* Supported devices */
29 static const struct pci_device_id nicvf_id_table[] = {
30 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
31 PCI_DEVICE_ID_THUNDER_NIC_VF,
32 PCI_VENDOR_ID_CAVIUM, 0xA134) },
33 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
34 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
35 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
36 { 0, } /* end of table */
37 };
38
39 MODULE_AUTHOR("Sunil Goutham");
40 MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
41 MODULE_LICENSE("GPL v2");
42 MODULE_VERSION(DRV_VERSION);
43 MODULE_DEVICE_TABLE(pci, nicvf_id_table);
44
45 static int debug = 0x00;
46 module_param(debug, int, 0644);
47 MODULE_PARM_DESC(debug, "Debug message level bitmap");
48
49 static int cpi_alg = CPI_ALG_NONE;
50 module_param(cpi_alg, int, S_IRUGO);
51 MODULE_PARM_DESC(cpi_alg,
52 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
53
54 static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
55 {
56 if (nic->sqs_mode)
57 return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
58 else
59 return qidx;
60 }
61
62 static inline void nicvf_set_rx_frame_cnt(struct nicvf *nic,
63 struct sk_buff *skb)
64 {
65 if (skb->len <= 64)
66 nic->drv_stats.rx_frames_64++;
67 else if (skb->len <= 127)
68 nic->drv_stats.rx_frames_127++;
69 else if (skb->len <= 255)
70 nic->drv_stats.rx_frames_255++;
71 else if (skb->len <= 511)
72 nic->drv_stats.rx_frames_511++;
73 else if (skb->len <= 1023)
74 nic->drv_stats.rx_frames_1023++;
75 else if (skb->len <= 1518)
76 nic->drv_stats.rx_frames_1518++;
77 else
78 nic->drv_stats.rx_frames_jumbo++;
79 }
80
81 /* The Cavium ThunderX network controller can *only* be found in SoCs
82 * containing the ThunderX ARM64 CPU implementation. All accesses to the device
83 * registers on this platform are implicitly strongly ordered with respect
84 * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
85 * with no memory barriers in this driver. The readq()/writeq() functions add
86 * explicit ordering operation which in this case are redundant, and only
87 * add overhead.
88 */
89
90 /* Register read/write APIs */
91 void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
92 {
93 writeq_relaxed(val, nic->reg_base + offset);
94 }
95
96 u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
97 {
98 return readq_relaxed(nic->reg_base + offset);
99 }
100
101 void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
102 u64 qidx, u64 val)
103 {
104 void __iomem *addr = nic->reg_base + offset;
105
106 writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
107 }
108
109 u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
110 {
111 void __iomem *addr = nic->reg_base + offset;
112
113 return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
114 }
115
116 /* VF -> PF mailbox communication */
117 static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
118 {
119 u64 *msg = (u64 *)mbx;
120
121 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
122 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
123 }
124
125 int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
126 {
127 int timeout = NIC_MBOX_MSG_TIMEOUT;
128 int sleep = 10;
129
130 nic->pf_acked = false;
131 nic->pf_nacked = false;
132
133 nicvf_write_to_mbx(nic, mbx);
134
135 /* Wait for previous message to be acked, timeout 2sec */
136 while (!nic->pf_acked) {
137 if (nic->pf_nacked)
138 return -EINVAL;
139 msleep(sleep);
140 if (nic->pf_acked)
141 break;
142 timeout -= sleep;
143 if (!timeout) {
144 netdev_err(nic->netdev,
145 "PF didn't ack to mbox msg %d from VF%d\n",
146 (mbx->msg.msg & 0xFF), nic->vf_id);
147 return -EBUSY;
148 }
149 }
150 return 0;
151 }
152
153 /* Checks if VF is able to comminicate with PF
154 * and also gets the VNIC number this VF is associated to.
155 */
156 static int nicvf_check_pf_ready(struct nicvf *nic)
157 {
158 union nic_mbx mbx = {};
159
160 mbx.msg.msg = NIC_MBOX_MSG_READY;
161 if (nicvf_send_msg_to_pf(nic, &mbx)) {
162 netdev_err(nic->netdev,
163 "PF didn't respond to READY msg\n");
164 return 0;
165 }
166
167 return 1;
168 }
169
170 static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
171 {
172 if (bgx->rx)
173 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
174 else
175 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
176 }
177
178 static void nicvf_handle_mbx_intr(struct nicvf *nic)
179 {
180 union nic_mbx mbx = {};
181 u64 *mbx_data;
182 u64 mbx_addr;
183 int i;
184
185 mbx_addr = NIC_VF_PF_MAILBOX_0_1;
186 mbx_data = (u64 *)&mbx;
187
188 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
189 *mbx_data = nicvf_reg_read(nic, mbx_addr);
190 mbx_data++;
191 mbx_addr += sizeof(u64);
192 }
193
194 netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
195 switch (mbx.msg.msg) {
196 case NIC_MBOX_MSG_READY:
197 nic->pf_acked = true;
198 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
199 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
200 nic->node = mbx.nic_cfg.node_id;
201 if (!nic->set_mac_pending)
202 ether_addr_copy(nic->netdev->dev_addr,
203 mbx.nic_cfg.mac_addr);
204 nic->sqs_mode = mbx.nic_cfg.sqs_mode;
205 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
206 nic->link_up = false;
207 nic->duplex = 0;
208 nic->speed = 0;
209 break;
210 case NIC_MBOX_MSG_ACK:
211 nic->pf_acked = true;
212 break;
213 case NIC_MBOX_MSG_NACK:
214 nic->pf_nacked = true;
215 break;
216 case NIC_MBOX_MSG_RSS_SIZE:
217 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
218 nic->pf_acked = true;
219 break;
220 case NIC_MBOX_MSG_BGX_STATS:
221 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
222 nic->pf_acked = true;
223 break;
224 case NIC_MBOX_MSG_BGX_LINK_CHANGE:
225 nic->pf_acked = true;
226 nic->link_up = mbx.link_status.link_up;
227 nic->duplex = mbx.link_status.duplex;
228 nic->speed = mbx.link_status.speed;
229 if (nic->link_up) {
230 netdev_info(nic->netdev, "%s: Link is Up %d Mbps %s\n",
231 nic->netdev->name, nic->speed,
232 nic->duplex == DUPLEX_FULL ?
233 "Full duplex" : "Half duplex");
234 netif_carrier_on(nic->netdev);
235 netif_tx_start_all_queues(nic->netdev);
236 } else {
237 netdev_info(nic->netdev, "%s: Link is Down\n",
238 nic->netdev->name);
239 netif_carrier_off(nic->netdev);
240 netif_tx_stop_all_queues(nic->netdev);
241 }
242 break;
243 case NIC_MBOX_MSG_ALLOC_SQS:
244 nic->sqs_count = mbx.sqs_alloc.qs_count;
245 nic->pf_acked = true;
246 break;
247 case NIC_MBOX_MSG_SNICVF_PTR:
248 /* Primary VF: make note of secondary VF's pointer
249 * to be used while packet transmission.
250 */
251 nic->snicvf[mbx.nicvf.sqs_id] =
252 (struct nicvf *)mbx.nicvf.nicvf;
253 nic->pf_acked = true;
254 break;
255 case NIC_MBOX_MSG_PNICVF_PTR:
256 /* Secondary VF/Qset: make note of primary VF's pointer
257 * to be used while packet reception, to handover packet
258 * to primary VF's netdev.
259 */
260 nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
261 nic->pf_acked = true;
262 break;
263 default:
264 netdev_err(nic->netdev,
265 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
266 break;
267 }
268 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
269 }
270
271 static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
272 {
273 union nic_mbx mbx = {};
274
275 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
276 mbx.mac.vf_id = nic->vf_id;
277 ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
278
279 return nicvf_send_msg_to_pf(nic, &mbx);
280 }
281
282 static void nicvf_config_cpi(struct nicvf *nic)
283 {
284 union nic_mbx mbx = {};
285
286 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
287 mbx.cpi_cfg.vf_id = nic->vf_id;
288 mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
289 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
290
291 nicvf_send_msg_to_pf(nic, &mbx);
292 }
293
294 static void nicvf_get_rss_size(struct nicvf *nic)
295 {
296 union nic_mbx mbx = {};
297
298 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
299 mbx.rss_size.vf_id = nic->vf_id;
300 nicvf_send_msg_to_pf(nic, &mbx);
301 }
302
303 void nicvf_config_rss(struct nicvf *nic)
304 {
305 union nic_mbx mbx = {};
306 struct nicvf_rss_info *rss = &nic->rss_info;
307 int ind_tbl_len = rss->rss_size;
308 int i, nextq = 0;
309
310 mbx.rss_cfg.vf_id = nic->vf_id;
311 mbx.rss_cfg.hash_bits = rss->hash_bits;
312 while (ind_tbl_len) {
313 mbx.rss_cfg.tbl_offset = nextq;
314 mbx.rss_cfg.tbl_len = min(ind_tbl_len,
315 RSS_IND_TBL_LEN_PER_MBX_MSG);
316 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
317 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
318
319 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
320 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
321
322 nicvf_send_msg_to_pf(nic, &mbx);
323
324 ind_tbl_len -= mbx.rss_cfg.tbl_len;
325 }
326 }
327
328 void nicvf_set_rss_key(struct nicvf *nic)
329 {
330 struct nicvf_rss_info *rss = &nic->rss_info;
331 u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
332 int idx;
333
334 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
335 nicvf_reg_write(nic, key_addr, rss->key[idx]);
336 key_addr += sizeof(u64);
337 }
338 }
339
340 static int nicvf_rss_init(struct nicvf *nic)
341 {
342 struct nicvf_rss_info *rss = &nic->rss_info;
343 int idx;
344
345 nicvf_get_rss_size(nic);
346
347 if (cpi_alg != CPI_ALG_NONE) {
348 rss->enable = false;
349 rss->hash_bits = 0;
350 return 0;
351 }
352
353 rss->enable = true;
354
355 /* Using the HW reset value for now */
356 rss->key[0] = 0xFEED0BADFEED0BADULL;
357 rss->key[1] = 0xFEED0BADFEED0BADULL;
358 rss->key[2] = 0xFEED0BADFEED0BADULL;
359 rss->key[3] = 0xFEED0BADFEED0BADULL;
360 rss->key[4] = 0xFEED0BADFEED0BADULL;
361
362 nicvf_set_rss_key(nic);
363
364 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
365 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
366
367 rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size));
368
369 for (idx = 0; idx < rss->rss_size; idx++)
370 rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
371 nic->rx_queues);
372 nicvf_config_rss(nic);
373 return 1;
374 }
375
376 /* Request PF to allocate additional Qsets */
377 static void nicvf_request_sqs(struct nicvf *nic)
378 {
379 union nic_mbx mbx = {};
380 int sqs;
381 int sqs_count = nic->sqs_count;
382 int rx_queues = 0, tx_queues = 0;
383
384 /* Only primary VF should request */
385 if (nic->sqs_mode || !nic->sqs_count)
386 return;
387
388 mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
389 mbx.sqs_alloc.vf_id = nic->vf_id;
390 mbx.sqs_alloc.qs_count = nic->sqs_count;
391 if (nicvf_send_msg_to_pf(nic, &mbx)) {
392 /* No response from PF */
393 nic->sqs_count = 0;
394 return;
395 }
396
397 /* Return if no Secondary Qsets available */
398 if (!nic->sqs_count)
399 return;
400
401 if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
402 rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
403 if (nic->tx_queues > MAX_SND_QUEUES_PER_QS)
404 tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS;
405
406 /* Set no of Rx/Tx queues in each of the SQsets */
407 for (sqs = 0; sqs < nic->sqs_count; sqs++) {
408 mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
409 mbx.nicvf.vf_id = nic->vf_id;
410 mbx.nicvf.sqs_id = sqs;
411 nicvf_send_msg_to_pf(nic, &mbx);
412
413 nic->snicvf[sqs]->sqs_id = sqs;
414 if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
415 nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
416 rx_queues -= MAX_RCV_QUEUES_PER_QS;
417 } else {
418 nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
419 rx_queues = 0;
420 }
421
422 if (tx_queues > MAX_SND_QUEUES_PER_QS) {
423 nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
424 tx_queues -= MAX_SND_QUEUES_PER_QS;
425 } else {
426 nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
427 tx_queues = 0;
428 }
429
430 nic->snicvf[sqs]->qs->cq_cnt =
431 max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
432
433 /* Initialize secondary Qset's queues and its interrupts */
434 nicvf_open(nic->snicvf[sqs]->netdev);
435 }
436
437 /* Update stack with actual Rx/Tx queue count allocated */
438 if (sqs_count != nic->sqs_count)
439 nicvf_set_real_num_queues(nic->netdev,
440 nic->tx_queues, nic->rx_queues);
441 }
442
443 /* Send this Qset's nicvf pointer to PF.
444 * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
445 * so that packets received by these Qsets can use primary VF's netdev
446 */
447 static void nicvf_send_vf_struct(struct nicvf *nic)
448 {
449 union nic_mbx mbx = {};
450
451 mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
452 mbx.nicvf.sqs_mode = nic->sqs_mode;
453 mbx.nicvf.nicvf = (u64)nic;
454 nicvf_send_msg_to_pf(nic, &mbx);
455 }
456
457 static void nicvf_get_primary_vf_struct(struct nicvf *nic)
458 {
459 union nic_mbx mbx = {};
460
461 mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
462 nicvf_send_msg_to_pf(nic, &mbx);
463 }
464
465 int nicvf_set_real_num_queues(struct net_device *netdev,
466 int tx_queues, int rx_queues)
467 {
468 int err = 0;
469
470 err = netif_set_real_num_tx_queues(netdev, tx_queues);
471 if (err) {
472 netdev_err(netdev,
473 "Failed to set no of Tx queues: %d\n", tx_queues);
474 return err;
475 }
476
477 err = netif_set_real_num_rx_queues(netdev, rx_queues);
478 if (err)
479 netdev_err(netdev,
480 "Failed to set no of Rx queues: %d\n", rx_queues);
481 return err;
482 }
483
484 static int nicvf_init_resources(struct nicvf *nic)
485 {
486 int err;
487 union nic_mbx mbx = {};
488
489 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
490
491 /* Enable Qset */
492 nicvf_qset_config(nic, true);
493
494 /* Initialize queues and HW for data transfer */
495 err = nicvf_config_data_transfer(nic, true);
496 if (err) {
497 netdev_err(nic->netdev,
498 "Failed to alloc/config VF's QSet resources\n");
499 return err;
500 }
501
502 /* Send VF config done msg to PF */
503 nicvf_write_to_mbx(nic, &mbx);
504
505 return 0;
506 }
507
508 static void nicvf_snd_pkt_handler(struct net_device *netdev,
509 struct cmp_queue *cq,
510 struct cqe_send_t *cqe_tx, int cqe_type)
511 {
512 struct sk_buff *skb = NULL;
513 struct nicvf *nic = netdev_priv(netdev);
514 struct snd_queue *sq;
515 struct sq_hdr_subdesc *hdr;
516 struct sq_hdr_subdesc *tso_sqe;
517
518 sq = &nic->qs->sq[cqe_tx->sq_idx];
519
520 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
521 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
522 return;
523
524 netdev_dbg(nic->netdev,
525 "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
526 __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
527 cqe_tx->sqe_ptr, hdr->subdesc_cnt);
528
529 nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
530 skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
531 if (skb) {
532 /* Check for dummy descriptor used for HW TSO offload on 88xx */
533 if (hdr->dont_send) {
534 /* Get actual TSO descriptors and free them */
535 tso_sqe =
536 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
537 nicvf_put_sq_desc(sq, tso_sqe->subdesc_cnt + 1);
538 }
539 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
540 prefetch(skb);
541 dev_consume_skb_any(skb);
542 sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
543 } else {
544 /* In case of SW TSO on 88xx, only last segment will have
545 * a SKB attached, so just free SQEs here.
546 */
547 if (!nic->hw_tso)
548 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
549 }
550 }
551
552 static inline void nicvf_set_rxhash(struct net_device *netdev,
553 struct cqe_rx_t *cqe_rx,
554 struct sk_buff *skb)
555 {
556 u8 hash_type;
557 u32 hash;
558
559 if (!(netdev->features & NETIF_F_RXHASH))
560 return;
561
562 switch (cqe_rx->rss_alg) {
563 case RSS_ALG_TCP_IP:
564 case RSS_ALG_UDP_IP:
565 hash_type = PKT_HASH_TYPE_L4;
566 hash = cqe_rx->rss_tag;
567 break;
568 case RSS_ALG_IP:
569 hash_type = PKT_HASH_TYPE_L3;
570 hash = cqe_rx->rss_tag;
571 break;
572 default:
573 hash_type = PKT_HASH_TYPE_NONE;
574 hash = 0;
575 }
576
577 skb_set_hash(skb, hash, hash_type);
578 }
579
580 static void nicvf_rcv_pkt_handler(struct net_device *netdev,
581 struct napi_struct *napi,
582 struct cqe_rx_t *cqe_rx)
583 {
584 struct sk_buff *skb;
585 struct nicvf *nic = netdev_priv(netdev);
586 int err = 0;
587 int rq_idx;
588
589 rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
590
591 if (nic->sqs_mode) {
592 /* Use primary VF's 'nicvf' struct */
593 nic = nic->pnicvf;
594 netdev = nic->netdev;
595 }
596
597 /* Check for errors */
598 err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
599 if (err && !cqe_rx->rb_cnt)
600 return;
601
602 skb = nicvf_get_rcv_skb(nic, cqe_rx);
603 if (!skb) {
604 netdev_dbg(nic->netdev, "Packet not received\n");
605 return;
606 }
607
608 if (netif_msg_pktdata(nic)) {
609 netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
610 skb, skb->len);
611 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
612 skb->data, skb->len, true);
613 }
614
615 /* If error packet, drop it here */
616 if (err) {
617 dev_kfree_skb_any(skb);
618 return;
619 }
620
621 nicvf_set_rx_frame_cnt(nic, skb);
622
623 nicvf_set_rxhash(netdev, cqe_rx, skb);
624
625 skb_record_rx_queue(skb, rq_idx);
626 if (netdev->hw_features & NETIF_F_RXCSUM) {
627 /* HW by default verifies TCP/UDP/SCTP checksums */
628 skb->ip_summed = CHECKSUM_UNNECESSARY;
629 } else {
630 skb_checksum_none_assert(skb);
631 }
632
633 skb->protocol = eth_type_trans(skb, netdev);
634
635 /* Check for stripped VLAN */
636 if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
637 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
638 ntohs((__force __be16)cqe_rx->vlan_tci));
639
640 if (napi && (netdev->features & NETIF_F_GRO))
641 napi_gro_receive(napi, skb);
642 else
643 netif_receive_skb(skb);
644 }
645
646 static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
647 struct napi_struct *napi, int budget)
648 {
649 int processed_cqe, work_done = 0, tx_done = 0;
650 int cqe_count, cqe_head;
651 struct nicvf *nic = netdev_priv(netdev);
652 struct queue_set *qs = nic->qs;
653 struct cmp_queue *cq = &qs->cq[cq_idx];
654 struct cqe_rx_t *cq_desc;
655 struct netdev_queue *txq;
656
657 spin_lock_bh(&cq->lock);
658 loop:
659 processed_cqe = 0;
660 /* Get no of valid CQ entries to process */
661 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
662 cqe_count &= CQ_CQE_COUNT;
663 if (!cqe_count)
664 goto done;
665
666 /* Get head of the valid CQ entries */
667 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
668 cqe_head &= 0xFFFF;
669
670 netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
671 __func__, cq_idx, cqe_count, cqe_head);
672 while (processed_cqe < cqe_count) {
673 /* Get the CQ descriptor */
674 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
675 cqe_head++;
676 cqe_head &= (cq->dmem.q_len - 1);
677 /* Initiate prefetch for next descriptor */
678 prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
679
680 if ((work_done >= budget) && napi &&
681 (cq_desc->cqe_type != CQE_TYPE_SEND)) {
682 break;
683 }
684
685 netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
686 cq_idx, cq_desc->cqe_type);
687 switch (cq_desc->cqe_type) {
688 case CQE_TYPE_RX:
689 nicvf_rcv_pkt_handler(netdev, napi, cq_desc);
690 work_done++;
691 break;
692 case CQE_TYPE_SEND:
693 nicvf_snd_pkt_handler(netdev, cq,
694 (void *)cq_desc, CQE_TYPE_SEND);
695 tx_done++;
696 break;
697 case CQE_TYPE_INVALID:
698 case CQE_TYPE_RX_SPLIT:
699 case CQE_TYPE_RX_TCP:
700 case CQE_TYPE_SEND_PTP:
701 /* Ignore for now */
702 break;
703 }
704 processed_cqe++;
705 }
706 netdev_dbg(nic->netdev,
707 "%s CQ%d processed_cqe %d work_done %d budget %d\n",
708 __func__, cq_idx, processed_cqe, work_done, budget);
709
710 /* Ring doorbell to inform H/W to reuse processed CQEs */
711 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
712 cq_idx, processed_cqe);
713
714 if ((work_done < budget) && napi)
715 goto loop;
716
717 done:
718 /* Wakeup TXQ if its stopped earlier due to SQ full */
719 if (tx_done) {
720 netdev = nic->pnicvf->netdev;
721 txq = netdev_get_tx_queue(netdev,
722 nicvf_netdev_qidx(nic, cq_idx));
723 nic = nic->pnicvf;
724 if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
725 netif_tx_start_queue(txq);
726 nic->drv_stats.txq_wake++;
727 if (netif_msg_tx_err(nic))
728 netdev_warn(netdev,
729 "%s: Transmit queue wakeup SQ%d\n",
730 netdev->name, cq_idx);
731 }
732 }
733
734 spin_unlock_bh(&cq->lock);
735 return work_done;
736 }
737
738 static int nicvf_poll(struct napi_struct *napi, int budget)
739 {
740 u64 cq_head;
741 int work_done = 0;
742 struct net_device *netdev = napi->dev;
743 struct nicvf *nic = netdev_priv(netdev);
744 struct nicvf_cq_poll *cq;
745
746 cq = container_of(napi, struct nicvf_cq_poll, napi);
747 work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
748
749 if (work_done < budget) {
750 /* Slow packet rate, exit polling */
751 napi_complete(napi);
752 /* Re-enable interrupts */
753 cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
754 cq->cq_idx);
755 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
756 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
757 cq->cq_idx, cq_head);
758 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
759 }
760 return work_done;
761 }
762
763 /* Qset error interrupt handler
764 *
765 * As of now only CQ errors are handled
766 */
767 static void nicvf_handle_qs_err(unsigned long data)
768 {
769 struct nicvf *nic = (struct nicvf *)data;
770 struct queue_set *qs = nic->qs;
771 int qidx;
772 u64 status;
773
774 netif_tx_disable(nic->netdev);
775
776 /* Check if it is CQ err */
777 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
778 status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
779 qidx);
780 if (!(status & CQ_ERR_MASK))
781 continue;
782 /* Process already queued CQEs and reconfig CQ */
783 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
784 nicvf_sq_disable(nic, qidx);
785 nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
786 nicvf_cmp_queue_config(nic, qs, qidx, true);
787 nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
788 nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
789
790 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
791 }
792
793 netif_tx_start_all_queues(nic->netdev);
794 /* Re-enable Qset error interrupt */
795 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
796 }
797
798 static void nicvf_dump_intr_status(struct nicvf *nic)
799 {
800 if (netif_msg_intr(nic))
801 netdev_info(nic->netdev, "%s: interrupt status 0x%llx\n",
802 nic->netdev->name, nicvf_reg_read(nic, NIC_VF_INT));
803 }
804
805 static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
806 {
807 struct nicvf *nic = (struct nicvf *)nicvf_irq;
808 u64 intr;
809
810 nicvf_dump_intr_status(nic);
811
812 intr = nicvf_reg_read(nic, NIC_VF_INT);
813 /* Check for spurious interrupt */
814 if (!(intr & NICVF_INTR_MBOX_MASK))
815 return IRQ_HANDLED;
816
817 nicvf_handle_mbx_intr(nic);
818
819 return IRQ_HANDLED;
820 }
821
822 static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
823 {
824 struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
825 struct nicvf *nic = cq_poll->nicvf;
826 int qidx = cq_poll->cq_idx;
827
828 nicvf_dump_intr_status(nic);
829
830 /* Disable interrupts */
831 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
832
833 /* Schedule NAPI */
834 napi_schedule_irqoff(&cq_poll->napi);
835
836 /* Clear interrupt */
837 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
838
839 return IRQ_HANDLED;
840 }
841
842 static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
843 {
844 struct nicvf *nic = (struct nicvf *)nicvf_irq;
845 u8 qidx;
846
847
848 nicvf_dump_intr_status(nic);
849
850 /* Disable RBDR interrupt and schedule softirq */
851 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
852 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
853 continue;
854 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
855 tasklet_hi_schedule(&nic->rbdr_task);
856 /* Clear interrupt */
857 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
858 }
859
860 return IRQ_HANDLED;
861 }
862
863 static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
864 {
865 struct nicvf *nic = (struct nicvf *)nicvf_irq;
866
867 nicvf_dump_intr_status(nic);
868
869 /* Disable Qset err interrupt and schedule softirq */
870 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
871 tasklet_hi_schedule(&nic->qs_err_task);
872 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
873
874 return IRQ_HANDLED;
875 }
876
877 static int nicvf_enable_msix(struct nicvf *nic)
878 {
879 int ret, vec;
880
881 nic->num_vec = NIC_VF_MSIX_VECTORS;
882
883 for (vec = 0; vec < nic->num_vec; vec++)
884 nic->msix_entries[vec].entry = vec;
885
886 ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
887 if (ret) {
888 netdev_err(nic->netdev,
889 "Req for #%d msix vectors failed\n", nic->num_vec);
890 return 0;
891 }
892 nic->msix_enabled = 1;
893 return 1;
894 }
895
896 static void nicvf_disable_msix(struct nicvf *nic)
897 {
898 if (nic->msix_enabled) {
899 pci_disable_msix(nic->pdev);
900 nic->msix_enabled = 0;
901 nic->num_vec = 0;
902 }
903 }
904
905 static void nicvf_set_irq_affinity(struct nicvf *nic)
906 {
907 int vec, cpu;
908 int irqnum;
909
910 for (vec = 0; vec < nic->num_vec; vec++) {
911 if (!nic->irq_allocated[vec])
912 continue;
913
914 if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL))
915 return;
916 /* CQ interrupts */
917 if (vec < NICVF_INTR_ID_SQ)
918 /* Leave CPU0 for RBDR and other interrupts */
919 cpu = nicvf_netdev_qidx(nic, vec) + 1;
920 else
921 cpu = 0;
922
923 cpumask_set_cpu(cpumask_local_spread(cpu, nic->node),
924 nic->affinity_mask[vec]);
925 irqnum = nic->msix_entries[vec].vector;
926 irq_set_affinity_hint(irqnum, nic->affinity_mask[vec]);
927 }
928 }
929
930 static int nicvf_register_interrupts(struct nicvf *nic)
931 {
932 int irq, ret = 0;
933 int vector;
934
935 for_each_cq_irq(irq)
936 sprintf(nic->irq_name[irq], "NICVF%d CQ%d",
937 nic->vf_id, irq);
938
939 for_each_sq_irq(irq)
940 sprintf(nic->irq_name[irq], "NICVF%d SQ%d",
941 nic->vf_id, irq - NICVF_INTR_ID_SQ);
942
943 for_each_rbdr_irq(irq)
944 sprintf(nic->irq_name[irq], "NICVF%d RBDR%d",
945 nic->vf_id, irq - NICVF_INTR_ID_RBDR);
946
947 /* Register CQ interrupts */
948 for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
949 vector = nic->msix_entries[irq].vector;
950 ret = request_irq(vector, nicvf_intr_handler,
951 0, nic->irq_name[irq], nic->napi[irq]);
952 if (ret)
953 goto err;
954 nic->irq_allocated[irq] = true;
955 }
956
957 /* Register RBDR interrupt */
958 for (irq = NICVF_INTR_ID_RBDR;
959 irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
960 vector = nic->msix_entries[irq].vector;
961 ret = request_irq(vector, nicvf_rbdr_intr_handler,
962 0, nic->irq_name[irq], nic);
963 if (ret)
964 goto err;
965 nic->irq_allocated[irq] = true;
966 }
967
968 /* Register QS error interrupt */
969 sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR],
970 "NICVF%d Qset error", nic->vf_id);
971 irq = NICVF_INTR_ID_QS_ERR;
972 ret = request_irq(nic->msix_entries[irq].vector,
973 nicvf_qs_err_intr_handler,
974 0, nic->irq_name[irq], nic);
975 if (ret)
976 goto err;
977
978 nic->irq_allocated[irq] = true;
979
980 /* Set IRQ affinities */
981 nicvf_set_irq_affinity(nic);
982
983 err:
984 if (ret)
985 netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
986
987 return ret;
988 }
989
990 static void nicvf_unregister_interrupts(struct nicvf *nic)
991 {
992 int irq;
993
994 /* Free registered interrupts */
995 for (irq = 0; irq < nic->num_vec; irq++) {
996 if (!nic->irq_allocated[irq])
997 continue;
998
999 irq_set_affinity_hint(nic->msix_entries[irq].vector, NULL);
1000 free_cpumask_var(nic->affinity_mask[irq]);
1001
1002 if (irq < NICVF_INTR_ID_SQ)
1003 free_irq(nic->msix_entries[irq].vector, nic->napi[irq]);
1004 else
1005 free_irq(nic->msix_entries[irq].vector, nic);
1006
1007 nic->irq_allocated[irq] = false;
1008 }
1009
1010 /* Disable MSI-X */
1011 nicvf_disable_msix(nic);
1012 }
1013
1014 /* Initialize MSIX vectors and register MISC interrupt.
1015 * Send READY message to PF to check if its alive
1016 */
1017 static int nicvf_register_misc_interrupt(struct nicvf *nic)
1018 {
1019 int ret = 0;
1020 int irq = NICVF_INTR_ID_MISC;
1021
1022 /* Return if mailbox interrupt is already registered */
1023 if (nic->msix_enabled)
1024 return 0;
1025
1026 /* Enable MSI-X */
1027 if (!nicvf_enable_msix(nic))
1028 return 1;
1029
1030 sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
1031 /* Register Misc interrupt */
1032 ret = request_irq(nic->msix_entries[irq].vector,
1033 nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
1034
1035 if (ret)
1036 return ret;
1037 nic->irq_allocated[irq] = true;
1038
1039 /* Enable mailbox interrupt */
1040 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
1041
1042 /* Check if VF is able to communicate with PF */
1043 if (!nicvf_check_pf_ready(nic)) {
1044 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1045 nicvf_unregister_interrupts(nic);
1046 return 1;
1047 }
1048
1049 return 0;
1050 }
1051
1052 static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
1053 {
1054 struct nicvf *nic = netdev_priv(netdev);
1055 int qid = skb_get_queue_mapping(skb);
1056 struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
1057
1058 /* Check for minimum packet length */
1059 if (skb->len <= ETH_HLEN) {
1060 dev_kfree_skb(skb);
1061 return NETDEV_TX_OK;
1062 }
1063
1064 if (!netif_tx_queue_stopped(txq) && !nicvf_sq_append_skb(nic, skb)) {
1065 netif_tx_stop_queue(txq);
1066 nic->drv_stats.txq_stop++;
1067 if (netif_msg_tx_err(nic))
1068 netdev_warn(netdev,
1069 "%s: Transmit ring full, stopping SQ%d\n",
1070 netdev->name, qid);
1071 return NETDEV_TX_BUSY;
1072 }
1073
1074 return NETDEV_TX_OK;
1075 }
1076
1077 static inline void nicvf_free_cq_poll(struct nicvf *nic)
1078 {
1079 struct nicvf_cq_poll *cq_poll;
1080 int qidx;
1081
1082 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1083 cq_poll = nic->napi[qidx];
1084 if (!cq_poll)
1085 continue;
1086 nic->napi[qidx] = NULL;
1087 kfree(cq_poll);
1088 }
1089 }
1090
1091 int nicvf_stop(struct net_device *netdev)
1092 {
1093 int irq, qidx;
1094 struct nicvf *nic = netdev_priv(netdev);
1095 struct queue_set *qs = nic->qs;
1096 struct nicvf_cq_poll *cq_poll = NULL;
1097 union nic_mbx mbx = {};
1098
1099 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
1100 nicvf_send_msg_to_pf(nic, &mbx);
1101
1102 netif_carrier_off(netdev);
1103 netif_tx_stop_all_queues(nic->netdev);
1104 nic->link_up = false;
1105
1106 /* Teardown secondary qsets first */
1107 if (!nic->sqs_mode) {
1108 for (qidx = 0; qidx < nic->sqs_count; qidx++) {
1109 if (!nic->snicvf[qidx])
1110 continue;
1111 nicvf_stop(nic->snicvf[qidx]->netdev);
1112 nic->snicvf[qidx] = NULL;
1113 }
1114 }
1115
1116 /* Disable RBDR & QS error interrupts */
1117 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1118 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1119 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1120 }
1121 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1122 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1123
1124 /* Wait for pending IRQ handlers to finish */
1125 for (irq = 0; irq < nic->num_vec; irq++)
1126 synchronize_irq(nic->msix_entries[irq].vector);
1127
1128 tasklet_kill(&nic->rbdr_task);
1129 tasklet_kill(&nic->qs_err_task);
1130 if (nic->rb_work_scheduled)
1131 cancel_delayed_work_sync(&nic->rbdr_work);
1132
1133 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1134 cq_poll = nic->napi[qidx];
1135 if (!cq_poll)
1136 continue;
1137 napi_synchronize(&cq_poll->napi);
1138 /* CQ intr is enabled while napi_complete,
1139 * so disable it now
1140 */
1141 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1142 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1143 napi_disable(&cq_poll->napi);
1144 netif_napi_del(&cq_poll->napi);
1145 }
1146
1147 netif_tx_disable(netdev);
1148
1149 /* Free resources */
1150 nicvf_config_data_transfer(nic, false);
1151
1152 /* Disable HW Qset */
1153 nicvf_qset_config(nic, false);
1154
1155 /* disable mailbox interrupt */
1156 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1157
1158 nicvf_unregister_interrupts(nic);
1159
1160 nicvf_free_cq_poll(nic);
1161
1162 /* Clear multiqset info */
1163 nic->pnicvf = nic;
1164
1165 return 0;
1166 }
1167
1168 int nicvf_open(struct net_device *netdev)
1169 {
1170 int err, qidx;
1171 struct nicvf *nic = netdev_priv(netdev);
1172 struct queue_set *qs = nic->qs;
1173 struct nicvf_cq_poll *cq_poll = NULL;
1174
1175 nic->mtu = netdev->mtu;
1176
1177 netif_carrier_off(netdev);
1178
1179 err = nicvf_register_misc_interrupt(nic);
1180 if (err)
1181 return err;
1182
1183 /* Register NAPI handler for processing CQEs */
1184 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1185 cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
1186 if (!cq_poll) {
1187 err = -ENOMEM;
1188 goto napi_del;
1189 }
1190 cq_poll->cq_idx = qidx;
1191 cq_poll->nicvf = nic;
1192 netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
1193 NAPI_POLL_WEIGHT);
1194 napi_enable(&cq_poll->napi);
1195 nic->napi[qidx] = cq_poll;
1196 }
1197
1198 /* Check if we got MAC address from PF or else generate a radom MAC */
1199 if (is_zero_ether_addr(netdev->dev_addr)) {
1200 eth_hw_addr_random(netdev);
1201 nicvf_hw_set_mac_addr(nic, netdev);
1202 }
1203
1204 if (nic->set_mac_pending) {
1205 nic->set_mac_pending = false;
1206 nicvf_hw_set_mac_addr(nic, netdev);
1207 }
1208
1209 /* Init tasklet for handling Qset err interrupt */
1210 tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err,
1211 (unsigned long)nic);
1212
1213 /* Init RBDR tasklet which will refill RBDR */
1214 tasklet_init(&nic->rbdr_task, nicvf_rbdr_task,
1215 (unsigned long)nic);
1216 INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
1217
1218 /* Configure CPI alorithm */
1219 nic->cpi_alg = cpi_alg;
1220 if (!nic->sqs_mode)
1221 nicvf_config_cpi(nic);
1222
1223 nicvf_request_sqs(nic);
1224 if (nic->sqs_mode)
1225 nicvf_get_primary_vf_struct(nic);
1226
1227 /* Configure receive side scaling */
1228 if (!nic->sqs_mode)
1229 nicvf_rss_init(nic);
1230
1231 err = nicvf_register_interrupts(nic);
1232 if (err)
1233 goto cleanup;
1234
1235 /* Initialize the queues */
1236 err = nicvf_init_resources(nic);
1237 if (err)
1238 goto cleanup;
1239
1240 /* Make sure queue initialization is written */
1241 wmb();
1242
1243 nicvf_reg_write(nic, NIC_VF_INT, -1);
1244 /* Enable Qset err interrupt */
1245 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
1246
1247 /* Enable completion queue interrupt */
1248 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1249 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
1250
1251 /* Enable RBDR threshold interrupt */
1252 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1253 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
1254
1255 nic->drv_stats.txq_stop = 0;
1256 nic->drv_stats.txq_wake = 0;
1257
1258 return 0;
1259 cleanup:
1260 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1261 nicvf_unregister_interrupts(nic);
1262 tasklet_kill(&nic->qs_err_task);
1263 tasklet_kill(&nic->rbdr_task);
1264 napi_del:
1265 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1266 cq_poll = nic->napi[qidx];
1267 if (!cq_poll)
1268 continue;
1269 napi_disable(&cq_poll->napi);
1270 netif_napi_del(&cq_poll->napi);
1271 }
1272 nicvf_free_cq_poll(nic);
1273 return err;
1274 }
1275
1276 static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
1277 {
1278 union nic_mbx mbx = {};
1279
1280 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
1281 mbx.frs.max_frs = mtu;
1282 mbx.frs.vf_id = nic->vf_id;
1283
1284 return nicvf_send_msg_to_pf(nic, &mbx);
1285 }
1286
1287 static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
1288 {
1289 struct nicvf *nic = netdev_priv(netdev);
1290
1291 if (new_mtu > NIC_HW_MAX_FRS)
1292 return -EINVAL;
1293
1294 if (new_mtu < NIC_HW_MIN_FRS)
1295 return -EINVAL;
1296
1297 if (nicvf_update_hw_max_frs(nic, new_mtu))
1298 return -EINVAL;
1299 netdev->mtu = new_mtu;
1300 nic->mtu = new_mtu;
1301
1302 return 0;
1303 }
1304
1305 static int nicvf_set_mac_address(struct net_device *netdev, void *p)
1306 {
1307 struct sockaddr *addr = p;
1308 struct nicvf *nic = netdev_priv(netdev);
1309
1310 if (!is_valid_ether_addr(addr->sa_data))
1311 return -EADDRNOTAVAIL;
1312
1313 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1314
1315 if (nic->msix_enabled) {
1316 if (nicvf_hw_set_mac_addr(nic, netdev))
1317 return -EBUSY;
1318 } else {
1319 nic->set_mac_pending = true;
1320 }
1321
1322 return 0;
1323 }
1324
1325 void nicvf_update_lmac_stats(struct nicvf *nic)
1326 {
1327 int stat = 0;
1328 union nic_mbx mbx = {};
1329
1330 if (!netif_running(nic->netdev))
1331 return;
1332
1333 mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
1334 mbx.bgx_stats.vf_id = nic->vf_id;
1335 /* Rx stats */
1336 mbx.bgx_stats.rx = 1;
1337 while (stat < BGX_RX_STATS_COUNT) {
1338 mbx.bgx_stats.idx = stat;
1339 if (nicvf_send_msg_to_pf(nic, &mbx))
1340 return;
1341 stat++;
1342 }
1343
1344 stat = 0;
1345
1346 /* Tx stats */
1347 mbx.bgx_stats.rx = 0;
1348 while (stat < BGX_TX_STATS_COUNT) {
1349 mbx.bgx_stats.idx = stat;
1350 if (nicvf_send_msg_to_pf(nic, &mbx))
1351 return;
1352 stat++;
1353 }
1354 }
1355
1356 void nicvf_update_stats(struct nicvf *nic)
1357 {
1358 int qidx;
1359 struct nicvf_hw_stats *stats = &nic->hw_stats;
1360 struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
1361 struct queue_set *qs = nic->qs;
1362
1363 #define GET_RX_STATS(reg) \
1364 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
1365 #define GET_TX_STATS(reg) \
1366 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
1367
1368 stats->rx_bytes = GET_RX_STATS(RX_OCTS);
1369 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
1370 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
1371 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
1372 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
1373 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
1374 stats->rx_drop_red = GET_RX_STATS(RX_RED);
1375 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
1376 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
1377 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
1378 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
1379 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
1380 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
1381 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
1382
1383 stats->tx_bytes_ok = GET_TX_STATS(TX_OCTS);
1384 stats->tx_ucast_frames_ok = GET_TX_STATS(TX_UCAST);
1385 stats->tx_bcast_frames_ok = GET_TX_STATS(TX_BCAST);
1386 stats->tx_mcast_frames_ok = GET_TX_STATS(TX_MCAST);
1387 stats->tx_drops = GET_TX_STATS(TX_DROP);
1388
1389 drv_stats->tx_frames_ok = stats->tx_ucast_frames_ok +
1390 stats->tx_bcast_frames_ok +
1391 stats->tx_mcast_frames_ok;
1392 drv_stats->rx_frames_ok = stats->rx_ucast_frames +
1393 stats->rx_bcast_frames +
1394 stats->rx_mcast_frames;
1395 drv_stats->rx_drops = stats->rx_drop_red +
1396 stats->rx_drop_overrun;
1397 drv_stats->tx_drops = stats->tx_drops;
1398
1399 /* Update RQ and SQ stats */
1400 for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1401 nicvf_update_rq_stats(nic, qidx);
1402 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1403 nicvf_update_sq_stats(nic, qidx);
1404 }
1405
1406 static struct rtnl_link_stats64 *nicvf_get_stats64(struct net_device *netdev,
1407 struct rtnl_link_stats64 *stats)
1408 {
1409 struct nicvf *nic = netdev_priv(netdev);
1410 struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
1411 struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
1412
1413 nicvf_update_stats(nic);
1414
1415 stats->rx_bytes = hw_stats->rx_bytes;
1416 stats->rx_packets = drv_stats->rx_frames_ok;
1417 stats->rx_dropped = drv_stats->rx_drops;
1418 stats->multicast = hw_stats->rx_mcast_frames;
1419
1420 stats->tx_bytes = hw_stats->tx_bytes_ok;
1421 stats->tx_packets = drv_stats->tx_frames_ok;
1422 stats->tx_dropped = drv_stats->tx_drops;
1423
1424 return stats;
1425 }
1426
1427 static void nicvf_tx_timeout(struct net_device *dev)
1428 {
1429 struct nicvf *nic = netdev_priv(dev);
1430
1431 if (netif_msg_tx_err(nic))
1432 netdev_warn(dev, "%s: Transmit timed out, resetting\n",
1433 dev->name);
1434
1435 nic->drv_stats.tx_timeout++;
1436 schedule_work(&nic->reset_task);
1437 }
1438
1439 static void nicvf_reset_task(struct work_struct *work)
1440 {
1441 struct nicvf *nic;
1442
1443 nic = container_of(work, struct nicvf, reset_task);
1444
1445 if (!netif_running(nic->netdev))
1446 return;
1447
1448 nicvf_stop(nic->netdev);
1449 nicvf_open(nic->netdev);
1450 netif_trans_update(nic->netdev);
1451 }
1452
1453 static int nicvf_config_loopback(struct nicvf *nic,
1454 netdev_features_t features)
1455 {
1456 union nic_mbx mbx = {};
1457
1458 mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
1459 mbx.lbk.vf_id = nic->vf_id;
1460 mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
1461
1462 return nicvf_send_msg_to_pf(nic, &mbx);
1463 }
1464
1465 static netdev_features_t nicvf_fix_features(struct net_device *netdev,
1466 netdev_features_t features)
1467 {
1468 struct nicvf *nic = netdev_priv(netdev);
1469
1470 if ((features & NETIF_F_LOOPBACK) &&
1471 netif_running(netdev) && !nic->loopback_supported)
1472 features &= ~NETIF_F_LOOPBACK;
1473
1474 return features;
1475 }
1476
1477 static int nicvf_set_features(struct net_device *netdev,
1478 netdev_features_t features)
1479 {
1480 struct nicvf *nic = netdev_priv(netdev);
1481 netdev_features_t changed = features ^ netdev->features;
1482
1483 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1484 nicvf_config_vlan_stripping(nic, features);
1485
1486 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
1487 return nicvf_config_loopback(nic, features);
1488
1489 return 0;
1490 }
1491
1492 static const struct net_device_ops nicvf_netdev_ops = {
1493 .ndo_open = nicvf_open,
1494 .ndo_stop = nicvf_stop,
1495 .ndo_start_xmit = nicvf_xmit,
1496 .ndo_change_mtu = nicvf_change_mtu,
1497 .ndo_set_mac_address = nicvf_set_mac_address,
1498 .ndo_get_stats64 = nicvf_get_stats64,
1499 .ndo_tx_timeout = nicvf_tx_timeout,
1500 .ndo_fix_features = nicvf_fix_features,
1501 .ndo_set_features = nicvf_set_features,
1502 };
1503
1504 static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1505 {
1506 struct device *dev = &pdev->dev;
1507 struct net_device *netdev;
1508 struct nicvf *nic;
1509 int err, qcount;
1510 u16 sdevid;
1511
1512 err = pci_enable_device(pdev);
1513 if (err) {
1514 dev_err(dev, "Failed to enable PCI device\n");
1515 return err;
1516 }
1517
1518 err = pci_request_regions(pdev, DRV_NAME);
1519 if (err) {
1520 dev_err(dev, "PCI request regions failed 0x%x\n", err);
1521 goto err_disable_device;
1522 }
1523
1524 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
1525 if (err) {
1526 dev_err(dev, "Unable to get usable DMA configuration\n");
1527 goto err_release_regions;
1528 }
1529
1530 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
1531 if (err) {
1532 dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n");
1533 goto err_release_regions;
1534 }
1535
1536 qcount = MAX_CMP_QUEUES_PER_QS;
1537
1538 /* Restrict multiqset support only for host bound VFs */
1539 if (pdev->is_virtfn) {
1540 /* Set max number of queues per VF */
1541 qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS);
1542 qcount = min(qcount,
1543 (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
1544 }
1545
1546 netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
1547 if (!netdev) {
1548 err = -ENOMEM;
1549 goto err_release_regions;
1550 }
1551
1552 pci_set_drvdata(pdev, netdev);
1553
1554 SET_NETDEV_DEV(netdev, &pdev->dev);
1555
1556 nic = netdev_priv(netdev);
1557 nic->netdev = netdev;
1558 nic->pdev = pdev;
1559 nic->pnicvf = nic;
1560 nic->max_queues = qcount;
1561
1562 /* MAP VF's configuration registers */
1563 nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
1564 if (!nic->reg_base) {
1565 dev_err(dev, "Cannot map config register space, aborting\n");
1566 err = -ENOMEM;
1567 goto err_free_netdev;
1568 }
1569
1570 err = nicvf_set_qset_resources(nic);
1571 if (err)
1572 goto err_free_netdev;
1573
1574 /* Check if PF is alive and get MAC address for this VF */
1575 err = nicvf_register_misc_interrupt(nic);
1576 if (err)
1577 goto err_free_netdev;
1578
1579 nicvf_send_vf_struct(nic);
1580
1581 if (!pass1_silicon(nic->pdev))
1582 nic->hw_tso = true;
1583
1584 pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
1585 if (sdevid == 0xA134)
1586 nic->t88 = true;
1587
1588 /* Check if this VF is in QS only mode */
1589 if (nic->sqs_mode)
1590 return 0;
1591
1592 err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
1593 if (err)
1594 goto err_unregister_interrupts;
1595
1596 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG |
1597 NETIF_F_TSO | NETIF_F_GRO |
1598 NETIF_F_HW_VLAN_CTAG_RX);
1599
1600 netdev->hw_features |= NETIF_F_RXHASH;
1601
1602 netdev->features |= netdev->hw_features;
1603 netdev->hw_features |= NETIF_F_LOOPBACK;
1604
1605 netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
1606
1607 netdev->netdev_ops = &nicvf_netdev_ops;
1608 netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
1609
1610 INIT_WORK(&nic->reset_task, nicvf_reset_task);
1611
1612 err = register_netdev(netdev);
1613 if (err) {
1614 dev_err(dev, "Failed to register netdevice\n");
1615 goto err_unregister_interrupts;
1616 }
1617
1618 nic->msg_enable = debug;
1619
1620 nicvf_set_ethtool_ops(netdev);
1621
1622 return 0;
1623
1624 err_unregister_interrupts:
1625 nicvf_unregister_interrupts(nic);
1626 err_free_netdev:
1627 pci_set_drvdata(pdev, NULL);
1628 free_netdev(netdev);
1629 err_release_regions:
1630 pci_release_regions(pdev);
1631 err_disable_device:
1632 pci_disable_device(pdev);
1633 return err;
1634 }
1635
1636 static void nicvf_remove(struct pci_dev *pdev)
1637 {
1638 struct net_device *netdev = pci_get_drvdata(pdev);
1639 struct nicvf *nic;
1640 struct net_device *pnetdev;
1641
1642 if (!netdev)
1643 return;
1644
1645 nic = netdev_priv(netdev);
1646 pnetdev = nic->pnicvf->netdev;
1647
1648 /* Check if this Qset is assigned to different VF.
1649 * If yes, clean primary and all secondary Qsets.
1650 */
1651 if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
1652 unregister_netdev(pnetdev);
1653 nicvf_unregister_interrupts(nic);
1654 pci_set_drvdata(pdev, NULL);
1655 free_netdev(netdev);
1656 pci_release_regions(pdev);
1657 pci_disable_device(pdev);
1658 }
1659
1660 static void nicvf_shutdown(struct pci_dev *pdev)
1661 {
1662 nicvf_remove(pdev);
1663 }
1664
1665 static struct pci_driver nicvf_driver = {
1666 .name = DRV_NAME,
1667 .id_table = nicvf_id_table,
1668 .probe = nicvf_probe,
1669 .remove = nicvf_remove,
1670 .shutdown = nicvf_shutdown,
1671 };
1672
1673 static int __init nicvf_init_module(void)
1674 {
1675 pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
1676
1677 return pci_register_driver(&nicvf_driver);
1678 }
1679
1680 static void __exit nicvf_cleanup_module(void)
1681 {
1682 pci_unregister_driver(&nicvf_driver);
1683 }
1684
1685 module_init(nicvf_init_module);
1686 module_exit(nicvf_cleanup_module);
ubuntu-bionic-kernel mirror