]>
Commit | Line | Data |
---|---|---|
25763b3c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
4863dea3 SG |
2 | /* |
3 | * Copyright (C) 2015 Cavium, Inc. | |
4863dea3 SG |
4 | */ |
5 | ||
6 | #include <linux/module.h> | |
7 | #include <linux/interrupt.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/netdevice.h> | |
aa2e259b | 10 | #include <linux/if_vlan.h> |
4863dea3 SG |
11 | #include <linux/etherdevice.h> |
12 | #include <linux/ethtool.h> | |
13 | #include <linux/log2.h> | |
14 | #include <linux/prefetch.h> | |
15 | #include <linux/irq.h> | |
83abb7d7 | 16 | #include <linux/iommu.h> |
05c773f5 | 17 | #include <linux/bpf.h> |
c56d91ce | 18 | #include <linux/bpf_trace.h> |
05c773f5 | 19 | #include <linux/filter.h> |
4a875509 | 20 | #include <linux/net_tstamp.h> |
37c3347e | 21 | #include <linux/workqueue.h> |
4863dea3 SG |
22 | |
23 | #include "nic_reg.h" | |
24 | #include "nic.h" | |
25 | #include "nicvf_queues.h" | |
26 | #include "thunder_bgx.h" | |
4a875509 | 27 | #include "../common/cavium_ptp.h" |
4863dea3 | 28 | |
6b9e6547 | 29 | #define DRV_NAME "nicvf" |
4863dea3 SG |
30 | #define DRV_VERSION "1.0" |
31 | ||
5ee15c10 MC |
32 | /* NOTE: Packets bigger than 1530 are split across multiple pages and XDP needs |
33 | * the buffer to be contiguous. Allow XDP to be set up only if we don't exceed | |
34 | * this value, keeping headroom for the 14 byte Ethernet header and two | |
35 | * VLAN tags (for QinQ) | |
36 | */ | |
37 | #define MAX_XDP_MTU (1530 - ETH_HLEN - VLAN_HLEN * 2) | |
38 | ||
4863dea3 SG |
39 | /* Supported devices */ |
40 | static const struct pci_device_id nicvf_id_table[] = { | |
41 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
42 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
f7ff0ae8 SG |
43 | PCI_VENDOR_ID_CAVIUM, |
44 | PCI_SUBSYS_DEVID_88XX_NIC_VF) }, | |
4863dea3 SG |
45 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, |
46 | PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF, | |
f7ff0ae8 SG |
47 | PCI_VENDOR_ID_CAVIUM, |
48 | PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) }, | |
49 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
50 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
51 | PCI_VENDOR_ID_CAVIUM, | |
52 | PCI_SUBSYS_DEVID_81XX_NIC_VF) }, | |
53 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
54 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
55 | PCI_VENDOR_ID_CAVIUM, | |
56 | PCI_SUBSYS_DEVID_83XX_NIC_VF) }, | |
4863dea3 SG |
57 | { 0, } /* end of table */ |
58 | }; | |
59 | ||
60 | MODULE_AUTHOR("Sunil Goutham"); | |
61 | MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver"); | |
62 | MODULE_LICENSE("GPL v2"); | |
63 | MODULE_VERSION(DRV_VERSION); | |
64 | MODULE_DEVICE_TABLE(pci, nicvf_id_table); | |
65 | ||
66 | static int debug = 0x00; | |
67 | module_param(debug, int, 0644); | |
68 | MODULE_PARM_DESC(debug, "Debug message level bitmap"); | |
69 | ||
70 | static int cpi_alg = CPI_ALG_NONE; | |
d3757ba4 | 71 | module_param(cpi_alg, int, 0444); |
4863dea3 SG |
72 | MODULE_PARM_DESC(cpi_alg, |
73 | "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)"); | |
74 | ||
92dc8769 SG |
75 | static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx) |
76 | { | |
77 | if (nic->sqs_mode) | |
78 | return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS); | |
79 | else | |
80 | return qidx; | |
81 | } | |
82 | ||
4863dea3 SG |
83 | /* The Cavium ThunderX network controller can *only* be found in SoCs |
84 | * containing the ThunderX ARM64 CPU implementation. All accesses to the device | |
85 | * registers on this platform are implicitly strongly ordered with respect | |
86 | * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use | |
87 | * with no memory barriers in this driver. The readq()/writeq() functions add | |
88 | * explicit ordering operation which in this case are redundant, and only | |
89 | * add overhead. | |
90 | */ | |
91 | ||
92 | /* Register read/write APIs */ | |
93 | void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val) | |
94 | { | |
95 | writeq_relaxed(val, nic->reg_base + offset); | |
96 | } | |
97 | ||
98 | u64 nicvf_reg_read(struct nicvf *nic, u64 offset) | |
99 | { | |
100 | return readq_relaxed(nic->reg_base + offset); | |
101 | } | |
102 | ||
103 | void nicvf_queue_reg_write(struct nicvf *nic, u64 offset, | |
104 | u64 qidx, u64 val) | |
105 | { | |
106 | void __iomem *addr = nic->reg_base + offset; | |
107 | ||
108 | writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT)); | |
109 | } | |
110 | ||
111 | u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx) | |
112 | { | |
113 | void __iomem *addr = nic->reg_base + offset; | |
114 | ||
115 | return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT)); | |
116 | } | |
117 | ||
118 | /* VF -> PF mailbox communication */ | |
2cd2a196 AM |
119 | static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) |
120 | { | |
121 | u64 *msg = (u64 *)mbx; | |
122 | ||
123 | nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); | |
124 | nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); | |
125 | } | |
126 | ||
4863dea3 SG |
127 | int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) |
128 | { | |
129 | int timeout = NIC_MBOX_MSG_TIMEOUT; | |
130 | int sleep = 10; | |
609ea65c VL |
131 | int ret = 0; |
132 | ||
133 | mutex_lock(&nic->rx_mode_mtx); | |
4863dea3 SG |
134 | |
135 | nic->pf_acked = false; | |
136 | nic->pf_nacked = false; | |
137 | ||
2cd2a196 | 138 | nicvf_write_to_mbx(nic, mbx); |
4863dea3 SG |
139 | |
140 | /* Wait for previous message to be acked, timeout 2sec */ | |
141 | while (!nic->pf_acked) { | |
ecae29cb RB |
142 | if (nic->pf_nacked) { |
143 | netdev_err(nic->netdev, | |
144 | "PF NACK to mbox msg 0x%02x from VF%d\n", | |
145 | (mbx->msg.msg & 0xFF), nic->vf_id); | |
609ea65c VL |
146 | ret = -EINVAL; |
147 | break; | |
ecae29cb | 148 | } |
4863dea3 SG |
149 | msleep(sleep); |
150 | if (nic->pf_acked) | |
151 | break; | |
152 | timeout -= sleep; | |
153 | if (!timeout) { | |
154 | netdev_err(nic->netdev, | |
ecae29cb | 155 | "PF didn't ACK to mbox msg 0x%02x from VF%d\n", |
4863dea3 | 156 | (mbx->msg.msg & 0xFF), nic->vf_id); |
609ea65c VL |
157 | ret = -EBUSY; |
158 | break; | |
4863dea3 SG |
159 | } |
160 | } | |
609ea65c VL |
161 | mutex_unlock(&nic->rx_mode_mtx); |
162 | return ret; | |
4863dea3 SG |
163 | } |
164 | ||
165 | /* Checks if VF is able to comminicate with PF | |
166 | * and also gets the VNIC number this VF is associated to. | |
167 | */ | |
168 | static int nicvf_check_pf_ready(struct nicvf *nic) | |
169 | { | |
2cd2a196 AM |
170 | union nic_mbx mbx = {}; |
171 | ||
172 | mbx.msg.msg = NIC_MBOX_MSG_READY; | |
6051cba7 SG |
173 | if (nicvf_send_msg_to_pf(nic, &mbx)) { |
174 | netdev_err(nic->netdev, | |
175 | "PF didn't respond to READY msg\n"); | |
176 | return 0; | |
4863dea3 | 177 | } |
6051cba7 | 178 | |
4863dea3 SG |
179 | return 1; |
180 | } | |
181 | ||
0dd563b9 VL |
182 | static void nicvf_send_cfg_done(struct nicvf *nic) |
183 | { | |
184 | union nic_mbx mbx = {}; | |
185 | ||
186 | mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; | |
187 | if (nicvf_send_msg_to_pf(nic, &mbx)) { | |
188 | netdev_err(nic->netdev, | |
189 | "PF didn't respond to CFG DONE msg\n"); | |
190 | } | |
191 | } | |
192 | ||
fd7ec062 AM |
193 | static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) |
194 | { | |
195 | if (bgx->rx) | |
196 | nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; | |
197 | else | |
198 | nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; | |
199 | } | |
200 | ||
4863dea3 SG |
201 | static void nicvf_handle_mbx_intr(struct nicvf *nic) |
202 | { | |
203 | union nic_mbx mbx = {}; | |
204 | u64 *mbx_data; | |
205 | u64 mbx_addr; | |
206 | int i; | |
207 | ||
208 | mbx_addr = NIC_VF_PF_MAILBOX_0_1; | |
209 | mbx_data = (u64 *)&mbx; | |
210 | ||
211 | for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { | |
212 | *mbx_data = nicvf_reg_read(nic, mbx_addr); | |
213 | mbx_data++; | |
214 | mbx_addr += sizeof(u64); | |
215 | } | |
216 | ||
217 | netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg); | |
218 | switch (mbx.msg.msg) { | |
219 | case NIC_MBOX_MSG_READY: | |
6051cba7 | 220 | nic->pf_acked = true; |
4863dea3 SG |
221 | nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; |
222 | nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; | |
223 | nic->node = mbx.nic_cfg.node_id; | |
bd049a90 PF |
224 | if (!nic->set_mac_pending) |
225 | ether_addr_copy(nic->netdev->dev_addr, | |
226 | mbx.nic_cfg.mac_addr); | |
92dc8769 | 227 | nic->sqs_mode = mbx.nic_cfg.sqs_mode; |
d77a2384 | 228 | nic->loopback_supported = mbx.nic_cfg.loopback_supported; |
4863dea3 SG |
229 | nic->link_up = false; |
230 | nic->duplex = 0; | |
231 | nic->speed = 0; | |
232 | break; | |
233 | case NIC_MBOX_MSG_ACK: | |
234 | nic->pf_acked = true; | |
235 | break; | |
236 | case NIC_MBOX_MSG_NACK: | |
237 | nic->pf_nacked = true; | |
238 | break; | |
239 | case NIC_MBOX_MSG_RSS_SIZE: | |
240 | nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; | |
241 | nic->pf_acked = true; | |
242 | break; | |
243 | case NIC_MBOX_MSG_BGX_STATS: | |
244 | nicvf_read_bgx_stats(nic, &mbx.bgx_stats); | |
245 | nic->pf_acked = true; | |
4863dea3 SG |
246 | break; |
247 | case NIC_MBOX_MSG_BGX_LINK_CHANGE: | |
248 | nic->pf_acked = true; | |
2c632ad8 VL |
249 | if (nic->link_up != mbx.link_status.link_up) { |
250 | nic->link_up = mbx.link_status.link_up; | |
251 | nic->duplex = mbx.link_status.duplex; | |
252 | nic->speed = mbx.link_status.speed; | |
253 | nic->mac_type = mbx.link_status.mac_type; | |
254 | if (nic->link_up) { | |
255 | netdev_info(nic->netdev, | |
256 | "Link is Up %d Mbps %s duplex\n", | |
257 | nic->speed, | |
258 | nic->duplex == DUPLEX_FULL ? | |
259 | "Full" : "Half"); | |
260 | netif_carrier_on(nic->netdev); | |
261 | netif_tx_start_all_queues(nic->netdev); | |
262 | } else { | |
263 | netdev_info(nic->netdev, "Link is Down\n"); | |
264 | netif_carrier_off(nic->netdev); | |
265 | netif_tx_stop_all_queues(nic->netdev); | |
266 | } | |
4863dea3 SG |
267 | } |
268 | break; | |
92dc8769 SG |
269 | case NIC_MBOX_MSG_ALLOC_SQS: |
270 | nic->sqs_count = mbx.sqs_alloc.qs_count; | |
271 | nic->pf_acked = true; | |
272 | break; | |
273 | case NIC_MBOX_MSG_SNICVF_PTR: | |
274 | /* Primary VF: make note of secondary VF's pointer | |
275 | * to be used while packet transmission. | |
276 | */ | |
277 | nic->snicvf[mbx.nicvf.sqs_id] = | |
278 | (struct nicvf *)mbx.nicvf.nicvf; | |
279 | nic->pf_acked = true; | |
280 | break; | |
281 | case NIC_MBOX_MSG_PNICVF_PTR: | |
282 | /* Secondary VF/Qset: make note of primary VF's pointer | |
283 | * to be used while packet reception, to handover packet | |
284 | * to primary VF's netdev. | |
285 | */ | |
286 | nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf; | |
287 | nic->pf_acked = true; | |
288 | break; | |
430da208 SG |
289 | case NIC_MBOX_MSG_PFC: |
290 | nic->pfc.autoneg = mbx.pfc.autoneg; | |
291 | nic->pfc.fc_rx = mbx.pfc.fc_rx; | |
292 | nic->pfc.fc_tx = mbx.pfc.fc_tx; | |
293 | nic->pf_acked = true; | |
294 | break; | |
4863dea3 SG |
295 | default: |
296 | netdev_err(nic->netdev, | |
297 | "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); | |
298 | break; | |
299 | } | |
300 | nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); | |
301 | } | |
302 | ||
303 | static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev) | |
304 | { | |
305 | union nic_mbx mbx = {}; | |
4863dea3 SG |
306 | |
307 | mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; | |
308 | mbx.mac.vf_id = nic->vf_id; | |
e610cb32 | 309 | ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr); |
4863dea3 SG |
310 | |
311 | return nicvf_send_msg_to_pf(nic, &mbx); | |
312 | } | |
313 | ||
fd7ec062 | 314 | static void nicvf_config_cpi(struct nicvf *nic) |
4863dea3 SG |
315 | { |
316 | union nic_mbx mbx = {}; | |
317 | ||
318 | mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; | |
319 | mbx.cpi_cfg.vf_id = nic->vf_id; | |
320 | mbx.cpi_cfg.cpi_alg = nic->cpi_alg; | |
321 | mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; | |
322 | ||
323 | nicvf_send_msg_to_pf(nic, &mbx); | |
324 | } | |
325 | ||
fd7ec062 | 326 | static void nicvf_get_rss_size(struct nicvf *nic) |
4863dea3 SG |
327 | { |
328 | union nic_mbx mbx = {}; | |
329 | ||
330 | mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; | |
331 | mbx.rss_size.vf_id = nic->vf_id; | |
332 | nicvf_send_msg_to_pf(nic, &mbx); | |
333 | } | |
334 | ||
335 | void nicvf_config_rss(struct nicvf *nic) | |
336 | { | |
337 | union nic_mbx mbx = {}; | |
338 | struct nicvf_rss_info *rss = &nic->rss_info; | |
339 | int ind_tbl_len = rss->rss_size; | |
340 | int i, nextq = 0; | |
341 | ||
342 | mbx.rss_cfg.vf_id = nic->vf_id; | |
343 | mbx.rss_cfg.hash_bits = rss->hash_bits; | |
344 | while (ind_tbl_len) { | |
345 | mbx.rss_cfg.tbl_offset = nextq; | |
346 | mbx.rss_cfg.tbl_len = min(ind_tbl_len, | |
347 | RSS_IND_TBL_LEN_PER_MBX_MSG); | |
348 | mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? | |
349 | NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; | |
350 | ||
351 | for (i = 0; i < mbx.rss_cfg.tbl_len; i++) | |
352 | mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; | |
353 | ||
354 | nicvf_send_msg_to_pf(nic, &mbx); | |
355 | ||
356 | ind_tbl_len -= mbx.rss_cfg.tbl_len; | |
357 | } | |
358 | } | |
359 | ||
360 | void nicvf_set_rss_key(struct nicvf *nic) | |
361 | { | |
362 | struct nicvf_rss_info *rss = &nic->rss_info; | |
363 | u64 key_addr = NIC_VNIC_RSS_KEY_0_4; | |
364 | int idx; | |
365 | ||
366 | for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { | |
367 | nicvf_reg_write(nic, key_addr, rss->key[idx]); | |
368 | key_addr += sizeof(u64); | |
369 | } | |
370 | } | |
371 | ||
372 | static int nicvf_rss_init(struct nicvf *nic) | |
373 | { | |
374 | struct nicvf_rss_info *rss = &nic->rss_info; | |
375 | int idx; | |
376 | ||
377 | nicvf_get_rss_size(nic); | |
378 | ||
38bb5d4f | 379 | if (cpi_alg != CPI_ALG_NONE) { |
4863dea3 SG |
380 | rss->enable = false; |
381 | rss->hash_bits = 0; | |
382 | return 0; | |
383 | } | |
384 | ||
385 | rss->enable = true; | |
386 | ||
0052c92f | 387 | netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64)); |
4863dea3 SG |
388 | nicvf_set_rss_key(nic); |
389 | ||
390 | rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; | |
391 | nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); | |
392 | ||
393 | rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size)); | |
394 | ||
395 | for (idx = 0; idx < rss->rss_size; idx++) | |
396 | rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx, | |
92dc8769 | 397 | nic->rx_queues); |
4863dea3 SG |
398 | nicvf_config_rss(nic); |
399 | return 1; | |
400 | } | |
401 | ||
92dc8769 SG |
402 | /* Request PF to allocate additional Qsets */ |
403 | static void nicvf_request_sqs(struct nicvf *nic) | |
404 | { | |
405 | union nic_mbx mbx = {}; | |
406 | int sqs; | |
407 | int sqs_count = nic->sqs_count; | |
408 | int rx_queues = 0, tx_queues = 0; | |
409 | ||
410 | /* Only primary VF should request */ | |
411 | if (nic->sqs_mode || !nic->sqs_count) | |
412 | return; | |
413 | ||
414 | mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; | |
415 | mbx.sqs_alloc.vf_id = nic->vf_id; | |
416 | mbx.sqs_alloc.qs_count = nic->sqs_count; | |
417 | if (nicvf_send_msg_to_pf(nic, &mbx)) { | |
418 | /* No response from PF */ | |
419 | nic->sqs_count = 0; | |
420 | return; | |
421 | } | |
422 | ||
423 | /* Return if no Secondary Qsets available */ | |
424 | if (!nic->sqs_count) | |
425 | return; | |
426 | ||
427 | if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS) | |
428 | rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS; | |
05c773f5 SG |
429 | |
430 | tx_queues = nic->tx_queues + nic->xdp_tx_queues; | |
431 | if (tx_queues > MAX_SND_QUEUES_PER_QS) | |
432 | tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS; | |
92dc8769 SG |
433 | |
434 | /* Set no of Rx/Tx queues in each of the SQsets */ | |
435 | for (sqs = 0; sqs < nic->sqs_count; sqs++) { | |
436 | mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; | |
437 | mbx.nicvf.vf_id = nic->vf_id; | |
438 | mbx.nicvf.sqs_id = sqs; | |
439 | nicvf_send_msg_to_pf(nic, &mbx); | |
440 | ||
441 | nic->snicvf[sqs]->sqs_id = sqs; | |
442 | if (rx_queues > MAX_RCV_QUEUES_PER_QS) { | |
443 | nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS; | |
444 | rx_queues -= MAX_RCV_QUEUES_PER_QS; | |
445 | } else { | |
446 | nic->snicvf[sqs]->qs->rq_cnt = rx_queues; | |
447 | rx_queues = 0; | |
448 | } | |
449 | ||
450 | if (tx_queues > MAX_SND_QUEUES_PER_QS) { | |
451 | nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS; | |
452 | tx_queues -= MAX_SND_QUEUES_PER_QS; | |
453 | } else { | |
454 | nic->snicvf[sqs]->qs->sq_cnt = tx_queues; | |
455 | tx_queues = 0; | |
456 | } | |
457 | ||
458 | nic->snicvf[sqs]->qs->cq_cnt = | |
459 | max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt); | |
460 | ||
461 | /* Initialize secondary Qset's queues and its interrupts */ | |
462 | nicvf_open(nic->snicvf[sqs]->netdev); | |
463 | } | |
464 | ||
465 | /* Update stack with actual Rx/Tx queue count allocated */ | |
466 | if (sqs_count != nic->sqs_count) | |
467 | nicvf_set_real_num_queues(nic->netdev, | |
468 | nic->tx_queues, nic->rx_queues); | |
469 | } | |
470 | ||
471 | /* Send this Qset's nicvf pointer to PF. | |
472 | * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs | |
473 | * so that packets received by these Qsets can use primary VF's netdev | |
474 | */ | |
475 | static void nicvf_send_vf_struct(struct nicvf *nic) | |
476 | { | |
477 | union nic_mbx mbx = {}; | |
478 | ||
479 | mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR; | |
480 | mbx.nicvf.sqs_mode = nic->sqs_mode; | |
481 | mbx.nicvf.nicvf = (u64)nic; | |
482 | nicvf_send_msg_to_pf(nic, &mbx); | |
483 | } | |
484 | ||
485 | static void nicvf_get_primary_vf_struct(struct nicvf *nic) | |
486 | { | |
487 | union nic_mbx mbx = {}; | |
488 | ||
489 | mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; | |
490 | nicvf_send_msg_to_pf(nic, &mbx); | |
491 | } | |
492 | ||
4863dea3 SG |
493 | int nicvf_set_real_num_queues(struct net_device *netdev, |
494 | int tx_queues, int rx_queues) | |
495 | { | |
496 | int err = 0; | |
497 | ||
498 | err = netif_set_real_num_tx_queues(netdev, tx_queues); | |
499 | if (err) { | |
500 | netdev_err(netdev, | |
501 | "Failed to set no of Tx queues: %d\n", tx_queues); | |
502 | return err; | |
503 | } | |
504 | ||
505 | err = netif_set_real_num_rx_queues(netdev, rx_queues); | |
506 | if (err) | |
507 | netdev_err(netdev, | |
508 | "Failed to set no of Rx queues: %d\n", rx_queues); | |
509 | return err; | |
510 | } | |
511 | ||
512 | static int nicvf_init_resources(struct nicvf *nic) | |
513 | { | |
514 | int err; | |
4863dea3 SG |
515 | |
516 | /* Enable Qset */ | |
517 | nicvf_qset_config(nic, true); | |
518 | ||
519 | /* Initialize queues and HW for data transfer */ | |
520 | err = nicvf_config_data_transfer(nic, true); | |
521 | if (err) { | |
522 | netdev_err(nic->netdev, | |
523 | "Failed to alloc/config VF's QSet resources\n"); | |
524 | return err; | |
525 | } | |
526 | ||
4863dea3 SG |
527 | return 0; |
528 | } | |
529 | ||
16f2bccd | 530 | static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog, |
e3d06ff9 | 531 | struct cqe_rx_t *cqe_rx, struct snd_queue *sq, |
27e95e36 | 532 | struct rcv_queue *rq, struct sk_buff **skb) |
05c773f5 SG |
533 | { |
534 | struct xdp_buff xdp; | |
c56d91ce | 535 | struct page *page; |
05c773f5 | 536 | u32 action; |
e6dbe939 | 537 | u16 len, offset = 0; |
05c773f5 | 538 | u64 dma_addr, cpu_addr; |
e3d06ff9 | 539 | void *orig_data; |
05c773f5 SG |
540 | |
541 | /* Retrieve packet buffer's DMA address and length */ | |
542 | len = *((u16 *)((void *)cqe_rx + (3 * sizeof(u64)))); | |
543 | dma_addr = *((u64 *)((void *)cqe_rx + (7 * sizeof(u64)))); | |
544 | ||
545 | cpu_addr = nicvf_iova_to_phys(nic, dma_addr); | |
546 | if (!cpu_addr) | |
547 | return false; | |
e3d06ff9 SG |
548 | cpu_addr = (u64)phys_to_virt(cpu_addr); |
549 | page = virt_to_page((void *)cpu_addr); | |
05c773f5 | 550 | |
e6dbe939 | 551 | xdp.data_hard_start = page_address(page); |
e3d06ff9 | 552 | xdp.data = (void *)cpu_addr; |
de8f3a83 | 553 | xdp_set_data_meta_invalid(&xdp); |
05c773f5 | 554 | xdp.data_end = xdp.data + len; |
27e95e36 | 555 | xdp.rxq = &rq->xdp_rxq; |
e3d06ff9 | 556 | orig_data = xdp.data; |
05c773f5 SG |
557 | |
558 | rcu_read_lock(); | |
559 | action = bpf_prog_run_xdp(prog, &xdp); | |
560 | rcu_read_unlock(); | |
561 | ||
a48ce00f | 562 | len = xdp.data_end - xdp.data; |
e3d06ff9 SG |
563 | /* Check if XDP program has changed headers */ |
564 | if (orig_data != xdp.data) { | |
e3d06ff9 SG |
565 | offset = orig_data - xdp.data; |
566 | dma_addr -= offset; | |
567 | } | |
568 | ||
05c773f5 SG |
569 | switch (action) { |
570 | case XDP_PASS: | |
e6dbe939 JDB |
571 | /* Check if it's a recycled page, if not |
572 | * unmap the DMA mapping. | |
573 | * | |
574 | * Recycled page holds an extra reference. | |
575 | */ | |
576 | if (page_ref_count(page) == 1) { | |
577 | dma_addr &= PAGE_MASK; | |
578 | dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, | |
579 | RCV_FRAG_LEN + XDP_PACKET_HEADROOM, | |
580 | DMA_FROM_DEVICE, | |
581 | DMA_ATTR_SKIP_CPU_SYNC); | |
582 | } | |
e3d06ff9 SG |
583 | |
584 | /* Build SKB and pass on packet to network stack */ | |
585 | *skb = build_skb(xdp.data, | |
586 | RCV_FRAG_LEN - cqe_rx->align_pad + offset); | |
587 | if (!*skb) | |
588 | put_page(page); | |
589 | else | |
590 | skb_put(*skb, len); | |
05c773f5 | 591 | return false; |
16f2bccd SG |
592 | case XDP_TX: |
593 | nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len); | |
594 | return true; | |
05c773f5 SG |
595 | default: |
596 | bpf_warn_invalid_xdp_action(action); | |
1a2ace56 | 597 | /* fall through */ |
c56d91ce SG |
598 | case XDP_ABORTED: |
599 | trace_xdp_exception(nic->netdev, prog, action); | |
1a2ace56 | 600 | /* fall through */ |
c56d91ce | 601 | case XDP_DROP: |
e6dbe939 JDB |
602 | /* Check if it's a recycled page, if not |
603 | * unmap the DMA mapping. | |
604 | * | |
605 | * Recycled page holds an extra reference. | |
606 | */ | |
607 | if (page_ref_count(page) == 1) { | |
608 | dma_addr &= PAGE_MASK; | |
609 | dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, | |
610 | RCV_FRAG_LEN + XDP_PACKET_HEADROOM, | |
611 | DMA_FROM_DEVICE, | |
612 | DMA_ATTR_SKIP_CPU_SYNC); | |
613 | } | |
c56d91ce SG |
614 | put_page(page); |
615 | return true; | |
05c773f5 SG |
616 | } |
617 | return false; | |
618 | } | |
619 | ||
4a875509 SG |
620 | static void nicvf_snd_ptp_handler(struct net_device *netdev, |
621 | struct cqe_send_t *cqe_tx) | |
622 | { | |
623 | struct nicvf *nic = netdev_priv(netdev); | |
624 | struct skb_shared_hwtstamps ts; | |
625 | u64 ns; | |
626 | ||
627 | nic = nic->pnicvf; | |
628 | ||
629 | /* Sync for 'ptp_skb' */ | |
630 | smp_rmb(); | |
631 | ||
632 | /* New timestamp request can be queued now */ | |
633 | atomic_set(&nic->tx_ptp_skbs, 0); | |
634 | ||
635 | /* Check for timestamp requested skb */ | |
636 | if (!nic->ptp_skb) | |
637 | return; | |
638 | ||
639 | /* Check if timestamping is timedout, which is set to 10us */ | |
640 | if (cqe_tx->send_status == CQ_TX_ERROP_TSTMP_TIMEOUT || | |
641 | cqe_tx->send_status == CQ_TX_ERROP_TSTMP_CONFLICT) | |
642 | goto no_tstamp; | |
643 | ||
644 | /* Get the timestamp */ | |
645 | memset(&ts, 0, sizeof(ts)); | |
646 | ns = cavium_ptp_tstamp2time(nic->ptp_clock, cqe_tx->ptp_timestamp); | |
647 | ts.hwtstamp = ns_to_ktime(ns); | |
648 | skb_tstamp_tx(nic->ptp_skb, &ts); | |
649 | ||
650 | no_tstamp: | |
651 | /* Free the original skb */ | |
652 | dev_kfree_skb_any(nic->ptp_skb); | |
653 | nic->ptp_skb = NULL; | |
654 | /* Sync 'ptp_skb' */ | |
655 | smp_wmb(); | |
656 | } | |
657 | ||
4863dea3 | 658 | static void nicvf_snd_pkt_handler(struct net_device *netdev, |
c43548d2 | 659 | struct cqe_send_t *cqe_tx, |
0dada88b | 660 | int budget, int *subdesc_cnt, |
2c204c2b | 661 | unsigned int *tx_pkts, unsigned int *tx_bytes) |
4863dea3 SG |
662 | { |
663 | struct sk_buff *skb = NULL; | |
16f2bccd | 664 | struct page *page; |
4863dea3 SG |
665 | struct nicvf *nic = netdev_priv(netdev); |
666 | struct snd_queue *sq; | |
667 | struct sq_hdr_subdesc *hdr; | |
7ceb8a13 | 668 | struct sq_hdr_subdesc *tso_sqe; |
4863dea3 SG |
669 | |
670 | sq = &nic->qs->sq[cqe_tx->sq_idx]; | |
671 | ||
672 | hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr); | |
673 | if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) | |
674 | return; | |
675 | ||
0dada88b SG |
676 | /* Check for errors */ |
677 | if (cqe_tx->send_status) | |
678 | nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx); | |
4863dea3 | 679 | |
16f2bccd SG |
680 | /* Is this a XDP designated Tx queue */ |
681 | if (sq->is_xdp) { | |
682 | page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr]; | |
683 | /* Check if it's recycled page or else unmap DMA mapping */ | |
684 | if (page && (page_ref_count(page) == 1)) | |
685 | nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr, | |
686 | hdr->subdesc_cnt); | |
687 | ||
688 | /* Release page reference for recycling */ | |
689 | if (page) | |
690 | put_page(page); | |
691 | sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL; | |
692 | *subdesc_cnt += hdr->subdesc_cnt + 1; | |
693 | return; | |
694 | } | |
695 | ||
4863dea3 | 696 | skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr]; |
4863dea3 | 697 | if (skb) { |
7ceb8a13 SG |
698 | /* Check for dummy descriptor used for HW TSO offload on 88xx */ |
699 | if (hdr->dont_send) { | |
700 | /* Get actual TSO descriptors and free them */ | |
701 | tso_sqe = | |
702 | (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2); | |
83abb7d7 SG |
703 | nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2, |
704 | tso_sqe->subdesc_cnt); | |
0dada88b | 705 | *subdesc_cnt += tso_sqe->subdesc_cnt + 1; |
83abb7d7 SG |
706 | } else { |
707 | nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr, | |
708 | hdr->subdesc_cnt); | |
7ceb8a13 | 709 | } |
0dada88b | 710 | *subdesc_cnt += hdr->subdesc_cnt + 1; |
4863dea3 | 711 | prefetch(skb); |
2c204c2b SG |
712 | (*tx_pkts)++; |
713 | *tx_bytes += skb->len; | |
4a875509 SG |
714 | /* If timestamp is requested for this skb, don't free it */ |
715 | if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS && | |
716 | !nic->pnicvf->ptp_skb) | |
717 | nic->pnicvf->ptp_skb = skb; | |
718 | else | |
719 | napi_consume_skb(skb, budget); | |
143ceb0b | 720 | sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL; |
40fb5f8a | 721 | } else { |
7ceb8a13 SG |
722 | /* In case of SW TSO on 88xx, only last segment will have |
723 | * a SKB attached, so just free SQEs here. | |
40fb5f8a SG |
724 | */ |
725 | if (!nic->hw_tso) | |
0dada88b | 726 | *subdesc_cnt += hdr->subdesc_cnt + 1; |
4863dea3 SG |
727 | } |
728 | } | |
729 | ||
38bb5d4f SG |
730 | static inline void nicvf_set_rxhash(struct net_device *netdev, |
731 | struct cqe_rx_t *cqe_rx, | |
732 | struct sk_buff *skb) | |
733 | { | |
734 | u8 hash_type; | |
735 | u32 hash; | |
736 | ||
737 | if (!(netdev->features & NETIF_F_RXHASH)) | |
738 | return; | |
739 | ||
740 | switch (cqe_rx->rss_alg) { | |
741 | case RSS_ALG_TCP_IP: | |
742 | case RSS_ALG_UDP_IP: | |
743 | hash_type = PKT_HASH_TYPE_L4; | |
744 | hash = cqe_rx->rss_tag; | |
745 | break; | |
746 | case RSS_ALG_IP: | |
747 | hash_type = PKT_HASH_TYPE_L3; | |
748 | hash = cqe_rx->rss_tag; | |
749 | break; | |
750 | default: | |
751 | hash_type = PKT_HASH_TYPE_NONE; | |
752 | hash = 0; | |
753 | } | |
754 | ||
755 | skb_set_hash(skb, hash, hash_type); | |
756 | } | |
757 | ||
4a875509 SG |
758 | static inline void nicvf_set_rxtstamp(struct nicvf *nic, struct sk_buff *skb) |
759 | { | |
760 | u64 ns; | |
761 | ||
762 | if (!nic->ptp_clock || !nic->hw_rx_tstamp) | |
763 | return; | |
764 | ||
765 | /* The first 8 bytes is the timestamp */ | |
766 | ns = cavium_ptp_tstamp2time(nic->ptp_clock, | |
767 | be64_to_cpu(*(__be64 *)skb->data)); | |
768 | skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); | |
769 | ||
770 | __skb_pull(skb, 8); | |
771 | } | |
772 | ||
4863dea3 SG |
773 | static void nicvf_rcv_pkt_handler(struct net_device *netdev, |
774 | struct napi_struct *napi, | |
27e95e36 JDB |
775 | struct cqe_rx_t *cqe_rx, |
776 | struct snd_queue *sq, struct rcv_queue *rq) | |
4863dea3 | 777 | { |
e3d06ff9 | 778 | struct sk_buff *skb = NULL; |
4863dea3 | 779 | struct nicvf *nic = netdev_priv(netdev); |
83abb7d7 | 780 | struct nicvf *snic = nic; |
4863dea3 | 781 | int err = 0; |
92dc8769 SG |
782 | int rq_idx; |
783 | ||
784 | rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx); | |
785 | ||
786 | if (nic->sqs_mode) { | |
787 | /* Use primary VF's 'nicvf' struct */ | |
788 | nic = nic->pnicvf; | |
789 | netdev = nic->netdev; | |
790 | } | |
4863dea3 SG |
791 | |
792 | /* Check for errors */ | |
0dada88b SG |
793 | if (cqe_rx->err_level || cqe_rx->err_opcode) { |
794 | err = nicvf_check_cqe_rx_errs(nic, cqe_rx); | |
795 | if (err && !cqe_rx->rb_cnt) | |
796 | return; | |
797 | } | |
4863dea3 | 798 | |
05c773f5 | 799 | /* For XDP, ignore pkts spanning multiple pages */ |
e3d06ff9 SG |
800 | if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) { |
801 | /* Packet consumed by XDP */ | |
27e95e36 | 802 | if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, rq, &skb)) |
05c773f5 | 803 | return; |
e3d06ff9 SG |
804 | } else { |
805 | skb = nicvf_get_rcv_skb(snic, cqe_rx, | |
806 | nic->xdp_prog ? true : false); | |
807 | } | |
05c773f5 | 808 | |
e3d06ff9 | 809 | if (!skb) |
4863dea3 | 810 | return; |
4863dea3 SG |
811 | |
812 | if (netif_msg_pktdata(nic)) { | |
bf24e136 | 813 | netdev_info(nic->netdev, "skb 0x%p, len=%d\n", skb, skb->len); |
4863dea3 SG |
814 | print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, |
815 | skb->data, skb->len, true); | |
816 | } | |
817 | ||
a2dc5ded SG |
818 | /* If error packet, drop it here */ |
819 | if (err) { | |
820 | dev_kfree_skb_any(skb); | |
821 | return; | |
822 | } | |
823 | ||
4a875509 | 824 | nicvf_set_rxtstamp(nic, skb); |
38bb5d4f SG |
825 | nicvf_set_rxhash(netdev, cqe_rx, skb); |
826 | ||
92dc8769 | 827 | skb_record_rx_queue(skb, rq_idx); |
4863dea3 SG |
828 | if (netdev->hw_features & NETIF_F_RXCSUM) { |
829 | /* HW by default verifies TCP/UDP/SCTP checksums */ | |
830 | skb->ip_summed = CHECKSUM_UNNECESSARY; | |
831 | } else { | |
832 | skb_checksum_none_assert(skb); | |
833 | } | |
834 | ||
835 | skb->protocol = eth_type_trans(skb, netdev); | |
836 | ||
aa2e259b SG |
837 | /* Check for stripped VLAN */ |
838 | if (cqe_rx->vlan_found && cqe_rx->vlan_stripped) | |
839 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), | |
840 | ntohs((__force __be16)cqe_rx->vlan_tci)); | |
841 | ||
4863dea3 SG |
842 | if (napi && (netdev->features & NETIF_F_GRO)) |
843 | napi_gro_receive(napi, skb); | |
844 | else | |
845 | netif_receive_skb(skb); | |
846 | } | |
847 | ||
848 | static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx, | |
849 | struct napi_struct *napi, int budget) | |
850 | { | |
74840b83 | 851 | int processed_cqe, work_done = 0, tx_done = 0; |
4863dea3 | 852 | int cqe_count, cqe_head; |
0dada88b | 853 | int subdesc_cnt = 0; |
4863dea3 SG |
854 | struct nicvf *nic = netdev_priv(netdev); |
855 | struct queue_set *qs = nic->qs; | |
856 | struct cmp_queue *cq = &qs->cq[cq_idx]; | |
857 | struct cqe_rx_t *cq_desc; | |
74840b83 | 858 | struct netdev_queue *txq; |
16f2bccd | 859 | struct snd_queue *sq = &qs->sq[cq_idx]; |
27e95e36 | 860 | struct rcv_queue *rq = &qs->rq[cq_idx]; |
16f2bccd | 861 | unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx; |
4863dea3 SG |
862 | |
863 | spin_lock_bh(&cq->lock); | |
864 | loop: | |
865 | processed_cqe = 0; | |
866 | /* Get no of valid CQ entries to process */ | |
867 | cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); | |
868 | cqe_count &= CQ_CQE_COUNT; | |
869 | if (!cqe_count) | |
870 | goto done; | |
871 | ||
872 | /* Get head of the valid CQ entries */ | |
873 | cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; | |
874 | cqe_head &= 0xFFFF; | |
875 | ||
4863dea3 SG |
876 | while (processed_cqe < cqe_count) { |
877 | /* Get the CQ descriptor */ | |
878 | cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); | |
879 | cqe_head++; | |
880 | cqe_head &= (cq->dmem.q_len - 1); | |
881 | /* Initiate prefetch for next descriptor */ | |
882 | prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head)); | |
883 | ||
884 | if ((work_done >= budget) && napi && | |
885 | (cq_desc->cqe_type != CQE_TYPE_SEND)) { | |
886 | break; | |
887 | } | |
888 | ||
4863dea3 SG |
889 | switch (cq_desc->cqe_type) { |
890 | case CQE_TYPE_RX: | |
27e95e36 | 891 | nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq, rq); |
4863dea3 SG |
892 | work_done++; |
893 | break; | |
894 | case CQE_TYPE_SEND: | |
0dada88b SG |
895 | nicvf_snd_pkt_handler(netdev, (void *)cq_desc, |
896 | budget, &subdesc_cnt, | |
897 | &tx_pkts, &tx_bytes); | |
74840b83 | 898 | tx_done++; |
4863dea3 | 899 | break; |
4a875509 SG |
900 | case CQE_TYPE_SEND_PTP: |
901 | nicvf_snd_ptp_handler(netdev, (void *)cq_desc); | |
902 | break; | |
4863dea3 SG |
903 | case CQE_TYPE_INVALID: |
904 | case CQE_TYPE_RX_SPLIT: | |
905 | case CQE_TYPE_RX_TCP: | |
4863dea3 SG |
906 | /* Ignore for now */ |
907 | break; | |
908 | } | |
909 | processed_cqe++; | |
910 | } | |
4863dea3 SG |
911 | |
912 | /* Ring doorbell to inform H/W to reuse processed CQEs */ | |
913 | nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, | |
914 | cq_idx, processed_cqe); | |
915 | ||
916 | if ((work_done < budget) && napi) | |
917 | goto loop; | |
918 | ||
919 | done: | |
0dada88b SG |
920 | /* Update SQ's descriptor free count */ |
921 | if (subdesc_cnt) | |
922 | nicvf_put_sq_desc(sq, subdesc_cnt); | |
923 | ||
16f2bccd SG |
924 | txq_idx = nicvf_netdev_qidx(nic, cq_idx); |
925 | /* Handle XDP TX queues */ | |
926 | if (nic->pnicvf->xdp_prog) { | |
927 | if (txq_idx < nic->pnicvf->xdp_tx_queues) { | |
928 | nicvf_xdp_sq_doorbell(nic, sq, cq_idx); | |
929 | goto out; | |
930 | } | |
931 | nic = nic->pnicvf; | |
932 | txq_idx -= nic->pnicvf->xdp_tx_queues; | |
933 | } | |
934 | ||
0dada88b | 935 | /* Wakeup TXQ if its stopped earlier due to SQ full */ |
bd3ad7d3 SG |
936 | if (tx_done || |
937 | (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) { | |
92dc8769 | 938 | netdev = nic->pnicvf->netdev; |
16f2bccd | 939 | txq = netdev_get_tx_queue(netdev, txq_idx); |
2c204c2b SG |
940 | if (tx_pkts) |
941 | netdev_tx_completed_queue(txq, tx_pkts, tx_bytes); | |
942 | ||
bd3ad7d3 SG |
943 | /* To read updated queue and carrier status */ |
944 | smp_mb(); | |
92dc8769 | 945 | if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) { |
bd3ad7d3 SG |
946 | netif_tx_wake_queue(txq); |
947 | nic = nic->pnicvf; | |
964cb69b | 948 | this_cpu_inc(nic->drv_stats->txq_wake); |
bf24e136 JP |
949 | netif_warn(nic, tx_err, netdev, |
950 | "Transmit queue wakeup SQ%d\n", txq_idx); | |
74840b83 SG |
951 | } |
952 | } | |
953 | ||
16f2bccd | 954 | out: |
4863dea3 SG |
955 | spin_unlock_bh(&cq->lock); |
956 | return work_done; | |
957 | } | |
958 | ||
959 | static int nicvf_poll(struct napi_struct *napi, int budget) | |
960 | { | |
961 | u64 cq_head; | |
962 | int work_done = 0; | |
963 | struct net_device *netdev = napi->dev; | |
964 | struct nicvf *nic = netdev_priv(netdev); | |
965 | struct nicvf_cq_poll *cq; | |
4863dea3 SG |
966 | |
967 | cq = container_of(napi, struct nicvf_cq_poll, napi); | |
968 | work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget); | |
969 | ||
4863dea3 SG |
970 | if (work_done < budget) { |
971 | /* Slow packet rate, exit polling */ | |
6ad20165 | 972 | napi_complete_done(napi, work_done); |
4863dea3 SG |
973 | /* Re-enable interrupts */ |
974 | cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, | |
975 | cq->cq_idx); | |
976 | nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx); | |
977 | nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD, | |
978 | cq->cq_idx, cq_head); | |
979 | nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx); | |
980 | } | |
981 | return work_done; | |
982 | } | |
983 | ||
984 | /* Qset error interrupt handler | |
985 | * | |
986 | * As of now only CQ errors are handled | |
987 | */ | |
fd7ec062 | 988 | static void nicvf_handle_qs_err(unsigned long data) |
4863dea3 SG |
989 | { |
990 | struct nicvf *nic = (struct nicvf *)data; | |
991 | struct queue_set *qs = nic->qs; | |
992 | int qidx; | |
993 | u64 status; | |
994 | ||
995 | netif_tx_disable(nic->netdev); | |
996 | ||
997 | /* Check if it is CQ err */ | |
998 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
999 | status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, | |
1000 | qidx); | |
1001 | if (!(status & CQ_ERR_MASK)) | |
1002 | continue; | |
1003 | /* Process already queued CQEs and reconfig CQ */ | |
1004 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1005 | nicvf_sq_disable(nic, qidx); | |
1006 | nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0); | |
1007 | nicvf_cmp_queue_config(nic, qs, qidx, true); | |
1008 | nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx); | |
1009 | nicvf_sq_enable(nic, &qs->sq[qidx], qidx); | |
1010 | ||
1011 | nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); | |
1012 | } | |
1013 | ||
1014 | netif_tx_start_all_queues(nic->netdev); | |
1015 | /* Re-enable Qset error interrupt */ | |
1016 | nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1017 | } | |
1018 | ||
39ad6eea SG |
1019 | static void nicvf_dump_intr_status(struct nicvf *nic) |
1020 | { | |
bf24e136 JP |
1021 | netif_info(nic, intr, nic->netdev, "interrupt status 0x%llx\n", |
1022 | nicvf_reg_read(nic, NIC_VF_INT)); | |
39ad6eea SG |
1023 | } |
1024 | ||
4863dea3 SG |
1025 | static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq) |
1026 | { | |
1027 | struct nicvf *nic = (struct nicvf *)nicvf_irq; | |
1028 | u64 intr; | |
1029 | ||
39ad6eea SG |
1030 | nicvf_dump_intr_status(nic); |
1031 | ||
4863dea3 SG |
1032 | intr = nicvf_reg_read(nic, NIC_VF_INT); |
1033 | /* Check for spurious interrupt */ | |
1034 | if (!(intr & NICVF_INTR_MBOX_MASK)) | |
1035 | return IRQ_HANDLED; | |
1036 | ||
1037 | nicvf_handle_mbx_intr(nic); | |
1038 | ||
1039 | return IRQ_HANDLED; | |
1040 | } | |
1041 | ||
39ad6eea SG |
1042 | static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq) |
1043 | { | |
1044 | struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq; | |
1045 | struct nicvf *nic = cq_poll->nicvf; | |
1046 | int qidx = cq_poll->cq_idx; | |
1047 | ||
1048 | nicvf_dump_intr_status(nic); | |
1049 | ||
1050 | /* Disable interrupts */ | |
1051 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1052 | ||
1053 | /* Schedule NAPI */ | |
ef0a4d86 | 1054 | napi_schedule_irqoff(&cq_poll->napi); |
39ad6eea SG |
1055 | |
1056 | /* Clear interrupt */ | |
1057 | nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); | |
1058 | ||
1059 | return IRQ_HANDLED; | |
1060 | } | |
1061 | ||
1062 | static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq) | |
4863dea3 | 1063 | { |
4863dea3 | 1064 | struct nicvf *nic = (struct nicvf *)nicvf_irq; |
39ad6eea | 1065 | u8 qidx; |
4863dea3 | 1066 | |
4863dea3 | 1067 | |
39ad6eea SG |
1068 | nicvf_dump_intr_status(nic); |
1069 | ||
1070 | /* Disable RBDR interrupt and schedule softirq */ | |
1071 | for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { | |
1072 | if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) | |
4863dea3 | 1073 | continue; |
39ad6eea SG |
1074 | nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); |
1075 | tasklet_hi_schedule(&nic->rbdr_task); | |
1076 | /* Clear interrupt */ | |
1077 | nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); | |
1078 | } | |
4863dea3 | 1079 | |
39ad6eea SG |
1080 | return IRQ_HANDLED; |
1081 | } | |
4863dea3 | 1082 | |
39ad6eea SG |
1083 | static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq) |
1084 | { | |
1085 | struct nicvf *nic = (struct nicvf *)nicvf_irq; | |
4863dea3 | 1086 | |
39ad6eea SG |
1087 | nicvf_dump_intr_status(nic); |
1088 | ||
1089 | /* Disable Qset err interrupt and schedule softirq */ | |
1090 | nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1091 | tasklet_hi_schedule(&nic->qs_err_task); | |
1092 | nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); | |
4863dea3 | 1093 | |
4863dea3 SG |
1094 | return IRQ_HANDLED; |
1095 | } | |
1096 | ||
fb4b7d98 SG |
1097 | static void nicvf_set_irq_affinity(struct nicvf *nic) |
1098 | { | |
1099 | int vec, cpu; | |
fb4b7d98 SG |
1100 | |
1101 | for (vec = 0; vec < nic->num_vec; vec++) { | |
1102 | if (!nic->irq_allocated[vec]) | |
1103 | continue; | |
1104 | ||
1105 | if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL)) | |
1106 | return; | |
1107 | /* CQ interrupts */ | |
1108 | if (vec < NICVF_INTR_ID_SQ) | |
1109 | /* Leave CPU0 for RBDR and other interrupts */ | |
1110 | cpu = nicvf_netdev_qidx(nic, vec) + 1; | |
1111 | else | |
1112 | cpu = 0; | |
1113 | ||
1114 | cpumask_set_cpu(cpumask_local_spread(cpu, nic->node), | |
1115 | nic->affinity_mask[vec]); | |
ce211b17 TS |
1116 | irq_set_affinity_hint(pci_irq_vector(nic->pdev, vec), |
1117 | nic->affinity_mask[vec]); | |
fb4b7d98 SG |
1118 | } |
1119 | } | |
1120 | ||
4863dea3 SG |
1121 | static int nicvf_register_interrupts(struct nicvf *nic) |
1122 | { | |
39ad6eea | 1123 | int irq, ret = 0; |
4863dea3 SG |
1124 | |
1125 | for_each_cq_irq(irq) | |
e4126213 SG |
1126 | sprintf(nic->irq_name[irq], "%s-rxtx-%d", |
1127 | nic->pnicvf->netdev->name, | |
1128 | nicvf_netdev_qidx(nic, irq)); | |
4863dea3 SG |
1129 | |
1130 | for_each_sq_irq(irq) | |
e4126213 SG |
1131 | sprintf(nic->irq_name[irq], "%s-sq-%d", |
1132 | nic->pnicvf->netdev->name, | |
1133 | nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ)); | |
4863dea3 SG |
1134 | |
1135 | for_each_rbdr_irq(irq) | |
e4126213 SG |
1136 | sprintf(nic->irq_name[irq], "%s-rbdr-%d", |
1137 | nic->pnicvf->netdev->name, | |
1138 | nic->sqs_mode ? (nic->sqs_id + 1) : 0); | |
4863dea3 | 1139 | |
39ad6eea SG |
1140 | /* Register CQ interrupts */ |
1141 | for (irq = 0; irq < nic->qs->cq_cnt; irq++) { | |
ce211b17 TS |
1142 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
1143 | nicvf_intr_handler, | |
39ad6eea | 1144 | 0, nic->irq_name[irq], nic->napi[irq]); |
4863dea3 | 1145 | if (ret) |
39ad6eea | 1146 | goto err; |
4863dea3 SG |
1147 | nic->irq_allocated[irq] = true; |
1148 | } | |
1149 | ||
39ad6eea SG |
1150 | /* Register RBDR interrupt */ |
1151 | for (irq = NICVF_INTR_ID_RBDR; | |
1152 | irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) { | |
ce211b17 TS |
1153 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
1154 | nicvf_rbdr_intr_handler, | |
4863dea3 SG |
1155 | 0, nic->irq_name[irq], nic); |
1156 | if (ret) | |
39ad6eea | 1157 | goto err; |
4863dea3 SG |
1158 | nic->irq_allocated[irq] = true; |
1159 | } | |
1160 | ||
39ad6eea | 1161 | /* Register QS error interrupt */ |
e4126213 SG |
1162 | sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d", |
1163 | nic->pnicvf->netdev->name, | |
1164 | nic->sqs_mode ? (nic->sqs_id + 1) : 0); | |
39ad6eea | 1165 | irq = NICVF_INTR_ID_QS_ERR; |
ce211b17 | 1166 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
39ad6eea SG |
1167 | nicvf_qs_err_intr_handler, |
1168 | 0, nic->irq_name[irq], nic); | |
fb4b7d98 SG |
1169 | if (ret) |
1170 | goto err; | |
1171 | ||
1172 | nic->irq_allocated[irq] = true; | |
1173 | ||
1174 | /* Set IRQ affinities */ | |
1175 | nicvf_set_irq_affinity(nic); | |
4863dea3 | 1176 | |
39ad6eea SG |
1177 | err: |
1178 | if (ret) | |
1179 | netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq); | |
4863dea3 | 1180 | |
39ad6eea | 1181 | return ret; |
4863dea3 SG |
1182 | } |
1183 | ||
1184 | static void nicvf_unregister_interrupts(struct nicvf *nic) | |
1185 | { | |
ce211b17 | 1186 | struct pci_dev *pdev = nic->pdev; |
4863dea3 SG |
1187 | int irq; |
1188 | ||
1189 | /* Free registered interrupts */ | |
1190 | for (irq = 0; irq < nic->num_vec; irq++) { | |
39ad6eea SG |
1191 | if (!nic->irq_allocated[irq]) |
1192 | continue; | |
1193 | ||
ce211b17 | 1194 | irq_set_affinity_hint(pci_irq_vector(pdev, irq), NULL); |
fb4b7d98 SG |
1195 | free_cpumask_var(nic->affinity_mask[irq]); |
1196 | ||
39ad6eea | 1197 | if (irq < NICVF_INTR_ID_SQ) |
ce211b17 | 1198 | free_irq(pci_irq_vector(pdev, irq), nic->napi[irq]); |
39ad6eea | 1199 | else |
ce211b17 | 1200 | free_irq(pci_irq_vector(pdev, irq), nic); |
39ad6eea | 1201 | |
4863dea3 SG |
1202 | nic->irq_allocated[irq] = false; |
1203 | } | |
1204 | ||
1205 | /* Disable MSI-X */ | |
ce211b17 TS |
1206 | pci_free_irq_vectors(pdev); |
1207 | nic->num_vec = 0; | |
4863dea3 SG |
1208 | } |
1209 | ||
1210 | /* Initialize MSIX vectors and register MISC interrupt. | |
1211 | * Send READY message to PF to check if its alive | |
1212 | */ | |
1213 | static int nicvf_register_misc_interrupt(struct nicvf *nic) | |
1214 | { | |
1215 | int ret = 0; | |
1216 | int irq = NICVF_INTR_ID_MISC; | |
1217 | ||
1218 | /* Return if mailbox interrupt is already registered */ | |
ce211b17 | 1219 | if (nic->pdev->msix_enabled) |
4863dea3 SG |
1220 | return 0; |
1221 | ||
1222 | /* Enable MSI-X */ | |
ce211b17 TS |
1223 | nic->num_vec = pci_msix_vec_count(nic->pdev); |
1224 | ret = pci_alloc_irq_vectors(nic->pdev, nic->num_vec, nic->num_vec, | |
1225 | PCI_IRQ_MSIX); | |
1226 | if (ret < 0) { | |
1227 | netdev_err(nic->netdev, | |
1228 | "Req for #%d msix vectors failed\n", nic->num_vec); | |
4863dea3 | 1229 | return 1; |
ce211b17 | 1230 | } |
4863dea3 SG |
1231 | |
1232 | sprintf(nic->irq_name[irq], "%s Mbox", "NICVF"); | |
1233 | /* Register Misc interrupt */ | |
ce211b17 | 1234 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
4863dea3 SG |
1235 | nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic); |
1236 | ||
1237 | if (ret) | |
1238 | return ret; | |
1239 | nic->irq_allocated[irq] = true; | |
1240 | ||
1241 | /* Enable mailbox interrupt */ | |
1242 | nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); | |
1243 | ||
1244 | /* Check if VF is able to communicate with PF */ | |
1245 | if (!nicvf_check_pf_ready(nic)) { | |
1246 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1247 | nicvf_unregister_interrupts(nic); | |
1248 | return 1; | |
1249 | } | |
1250 | ||
1251 | return 0; | |
1252 | } | |
1253 | ||
1254 | static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev) | |
1255 | { | |
1256 | struct nicvf *nic = netdev_priv(netdev); | |
1257 | int qid = skb_get_queue_mapping(skb); | |
1258 | struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid); | |
bd3ad7d3 SG |
1259 | struct nicvf *snic; |
1260 | struct snd_queue *sq; | |
1261 | int tmp; | |
4863dea3 SG |
1262 | |
1263 | /* Check for minimum packet length */ | |
1264 | if (skb->len <= ETH_HLEN) { | |
1265 | dev_kfree_skb(skb); | |
1266 | return NETDEV_TX_OK; | |
1267 | } | |
1268 | ||
16f2bccd SG |
1269 | /* In XDP case, initial HW tx queues are used for XDP, |
1270 | * but stack's queue mapping starts at '0', so skip the | |
1271 | * Tx queues attached to Rx queues for XDP. | |
1272 | */ | |
1273 | if (nic->xdp_prog) | |
1274 | qid += nic->xdp_tx_queues; | |
1275 | ||
bd3ad7d3 SG |
1276 | snic = nic; |
1277 | /* Get secondary Qset's SQ structure */ | |
1278 | if (qid >= MAX_SND_QUEUES_PER_QS) { | |
1279 | tmp = qid / MAX_SND_QUEUES_PER_QS; | |
1280 | snic = (struct nicvf *)nic->snicvf[tmp - 1]; | |
1281 | if (!snic) { | |
1282 | netdev_warn(nic->netdev, | |
1283 | "Secondary Qset#%d's ptr not initialized\n", | |
1284 | tmp - 1); | |
1285 | dev_kfree_skb(skb); | |
1286 | return NETDEV_TX_OK; | |
1287 | } | |
1288 | qid = qid % MAX_SND_QUEUES_PER_QS; | |
1289 | } | |
1290 | ||
1291 | sq = &snic->qs->sq[qid]; | |
1292 | if (!netif_tx_queue_stopped(txq) && | |
1293 | !nicvf_sq_append_skb(snic, sq, skb, qid)) { | |
4863dea3 | 1294 | netif_tx_stop_queue(txq); |
bd3ad7d3 SG |
1295 | |
1296 | /* Barrier, so that stop_queue visible to other cpus */ | |
1297 | smp_mb(); | |
1298 | ||
1299 | /* Check again, incase another cpu freed descriptors */ | |
1300 | if (atomic_read(&sq->free_cnt) > MIN_SQ_DESC_PER_PKT_XMIT) { | |
1301 | netif_tx_wake_queue(txq); | |
1302 | } else { | |
1303 | this_cpu_inc(nic->drv_stats->txq_stop); | |
bf24e136 JP |
1304 | netif_warn(nic, tx_err, netdev, |
1305 | "Transmit ring full, stopping SQ%d\n", qid); | |
bd3ad7d3 | 1306 | } |
4863dea3 SG |
1307 | return NETDEV_TX_BUSY; |
1308 | } | |
1309 | ||
1310 | return NETDEV_TX_OK; | |
1311 | } | |
1312 | ||
39ad6eea SG |
1313 | static inline void nicvf_free_cq_poll(struct nicvf *nic) |
1314 | { | |
1315 | struct nicvf_cq_poll *cq_poll; | |
1316 | int qidx; | |
1317 | ||
1318 | for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { | |
1319 | cq_poll = nic->napi[qidx]; | |
1320 | if (!cq_poll) | |
1321 | continue; | |
1322 | nic->napi[qidx] = NULL; | |
1323 | kfree(cq_poll); | |
1324 | } | |
1325 | } | |
1326 | ||
4863dea3 SG |
1327 | int nicvf_stop(struct net_device *netdev) |
1328 | { | |
1329 | int irq, qidx; | |
1330 | struct nicvf *nic = netdev_priv(netdev); | |
1331 | struct queue_set *qs = nic->qs; | |
1332 | struct nicvf_cq_poll *cq_poll = NULL; | |
1333 | union nic_mbx mbx = {}; | |
1334 | ||
2ecbe4f4 | 1335 | /* wait till all queued set_rx_mode tasks completes */ |
2ec1ed2a LB |
1336 | if (nic->nicvf_rx_mode_wq) { |
1337 | cancel_delayed_work_sync(&nic->link_change_work); | |
1338 | drain_workqueue(nic->nicvf_rx_mode_wq); | |
1339 | } | |
2ecbe4f4 | 1340 | |
4863dea3 SG |
1341 | mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; |
1342 | nicvf_send_msg_to_pf(nic, &mbx); | |
1343 | ||
1344 | netif_carrier_off(netdev); | |
92dc8769 | 1345 | netif_tx_stop_all_queues(nic->netdev); |
0b72a9a1 | 1346 | nic->link_up = false; |
92dc8769 SG |
1347 | |
1348 | /* Teardown secondary qsets first */ | |
1349 | if (!nic->sqs_mode) { | |
1350 | for (qidx = 0; qidx < nic->sqs_count; qidx++) { | |
1351 | if (!nic->snicvf[qidx]) | |
1352 | continue; | |
1353 | nicvf_stop(nic->snicvf[qidx]->netdev); | |
1354 | nic->snicvf[qidx] = NULL; | |
1355 | } | |
1356 | } | |
4863dea3 SG |
1357 | |
1358 | /* Disable RBDR & QS error interrupts */ | |
1359 | for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { | |
1360 | nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); | |
1361 | nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); | |
1362 | } | |
1363 | nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1364 | nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1365 | ||
1366 | /* Wait for pending IRQ handlers to finish */ | |
1367 | for (irq = 0; irq < nic->num_vec; irq++) | |
ce211b17 | 1368 | synchronize_irq(pci_irq_vector(nic->pdev, irq)); |
4863dea3 SG |
1369 | |
1370 | tasklet_kill(&nic->rbdr_task); | |
1371 | tasklet_kill(&nic->qs_err_task); | |
1372 | if (nic->rb_work_scheduled) | |
1373 | cancel_delayed_work_sync(&nic->rbdr_work); | |
1374 | ||
1375 | for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { | |
1376 | cq_poll = nic->napi[qidx]; | |
1377 | if (!cq_poll) | |
1378 | continue; | |
4863dea3 SG |
1379 | napi_synchronize(&cq_poll->napi); |
1380 | /* CQ intr is enabled while napi_complete, | |
1381 | * so disable it now | |
1382 | */ | |
1383 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1384 | nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); | |
1385 | napi_disable(&cq_poll->napi); | |
1386 | netif_napi_del(&cq_poll->napi); | |
4863dea3 SG |
1387 | } |
1388 | ||
b49087dd SG |
1389 | netif_tx_disable(netdev); |
1390 | ||
2c204c2b SG |
1391 | for (qidx = 0; qidx < netdev->num_tx_queues; qidx++) |
1392 | netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx)); | |
1393 | ||
4863dea3 SG |
1394 | /* Free resources */ |
1395 | nicvf_config_data_transfer(nic, false); | |
1396 | ||
1397 | /* Disable HW Qset */ | |
1398 | nicvf_qset_config(nic, false); | |
1399 | ||
1400 | /* disable mailbox interrupt */ | |
1401 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1402 | ||
1403 | nicvf_unregister_interrupts(nic); | |
1404 | ||
39ad6eea SG |
1405 | nicvf_free_cq_poll(nic); |
1406 | ||
4a875509 SG |
1407 | /* Free any pending SKB saved to receive timestamp */ |
1408 | if (nic->ptp_skb) { | |
1409 | dev_kfree_skb_any(nic->ptp_skb); | |
1410 | nic->ptp_skb = NULL; | |
1411 | } | |
1412 | ||
92dc8769 SG |
1413 | /* Clear multiqset info */ |
1414 | nic->pnicvf = nic; | |
92dc8769 | 1415 | |
4863dea3 SG |
1416 | return 0; |
1417 | } | |
1418 | ||
4a875509 SG |
1419 | static int nicvf_config_hw_rx_tstamp(struct nicvf *nic, bool enable) |
1420 | { | |
1421 | union nic_mbx mbx = {}; | |
1422 | ||
1423 | mbx.ptp.msg = NIC_MBOX_MSG_PTP_CFG; | |
1424 | mbx.ptp.enable = enable; | |
1425 | ||
1426 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1427 | } | |
1428 | ||
712c3185 SG |
1429 | static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) |
1430 | { | |
1431 | union nic_mbx mbx = {}; | |
1432 | ||
1433 | mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; | |
1434 | mbx.frs.max_frs = mtu; | |
1435 | mbx.frs.vf_id = nic->vf_id; | |
1436 | ||
1437 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1438 | } | |
1439 | ||
2c632ad8 VL |
1440 | static void nicvf_link_status_check_task(struct work_struct *work_arg) |
1441 | { | |
1442 | struct nicvf *nic = container_of(work_arg, | |
1443 | struct nicvf, | |
1444 | link_change_work.work); | |
1445 | union nic_mbx mbx = {}; | |
1446 | mbx.msg.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE; | |
1447 | nicvf_send_msg_to_pf(nic, &mbx); | |
1448 | queue_delayed_work(nic->nicvf_rx_mode_wq, | |
1449 | &nic->link_change_work, 2 * HZ); | |
1450 | } | |
1451 | ||
4863dea3 SG |
1452 | int nicvf_open(struct net_device *netdev) |
1453 | { | |
964cb69b | 1454 | int cpu, err, qidx; |
4863dea3 SG |
1455 | struct nicvf *nic = netdev_priv(netdev); |
1456 | struct queue_set *qs = nic->qs; | |
1457 | struct nicvf_cq_poll *cq_poll = NULL; | |
4863dea3 | 1458 | |
2ecbe4f4 | 1459 | /* wait till all queued set_rx_mode tasks completes if any */ |
2ec1ed2a LB |
1460 | if (nic->nicvf_rx_mode_wq) |
1461 | drain_workqueue(nic->nicvf_rx_mode_wq); | |
2ecbe4f4 | 1462 | |
4863dea3 SG |
1463 | netif_carrier_off(netdev); |
1464 | ||
1465 | err = nicvf_register_misc_interrupt(nic); | |
1466 | if (err) | |
1467 | return err; | |
1468 | ||
1469 | /* Register NAPI handler for processing CQEs */ | |
1470 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
1471 | cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL); | |
1472 | if (!cq_poll) { | |
1473 | err = -ENOMEM; | |
1474 | goto napi_del; | |
1475 | } | |
1476 | cq_poll->cq_idx = qidx; | |
39ad6eea | 1477 | cq_poll->nicvf = nic; |
4863dea3 SG |
1478 | netif_napi_add(netdev, &cq_poll->napi, nicvf_poll, |
1479 | NAPI_POLL_WEIGHT); | |
1480 | napi_enable(&cq_poll->napi); | |
1481 | nic->napi[qidx] = cq_poll; | |
1482 | } | |
1483 | ||
1484 | /* Check if we got MAC address from PF or else generate a radom MAC */ | |
a3a8ce4c | 1485 | if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) { |
4863dea3 SG |
1486 | eth_hw_addr_random(netdev); |
1487 | nicvf_hw_set_mac_addr(nic, netdev); | |
1488 | } | |
1489 | ||
bd049a90 PF |
1490 | if (nic->set_mac_pending) { |
1491 | nic->set_mac_pending = false; | |
1492 | nicvf_hw_set_mac_addr(nic, netdev); | |
1493 | } | |
1494 | ||
4863dea3 SG |
1495 | /* Init tasklet for handling Qset err interrupt */ |
1496 | tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err, | |
1497 | (unsigned long)nic); | |
1498 | ||
1499 | /* Init RBDR tasklet which will refill RBDR */ | |
1500 | tasklet_init(&nic->rbdr_task, nicvf_rbdr_task, | |
1501 | (unsigned long)nic); | |
1502 | INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work); | |
1503 | ||
1504 | /* Configure CPI alorithm */ | |
1505 | nic->cpi_alg = cpi_alg; | |
92dc8769 SG |
1506 | if (!nic->sqs_mode) |
1507 | nicvf_config_cpi(nic); | |
1508 | ||
1509 | nicvf_request_sqs(nic); | |
1510 | if (nic->sqs_mode) | |
1511 | nicvf_get_primary_vf_struct(nic); | |
4863dea3 | 1512 | |
4a875509 SG |
1513 | /* Configure PTP timestamp */ |
1514 | if (nic->ptp_clock) | |
1515 | nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp); | |
1516 | atomic_set(&nic->tx_ptp_skbs, 0); | |
1517 | nic->ptp_skb = NULL; | |
1518 | ||
712c3185 SG |
1519 | /* Configure receive side scaling and MTU */ |
1520 | if (!nic->sqs_mode) { | |
92dc8769 | 1521 | nicvf_rss_init(nic); |
60dce04b WY |
1522 | err = nicvf_update_hw_max_frs(nic, netdev->mtu); |
1523 | if (err) | |
712c3185 | 1524 | goto cleanup; |
964cb69b SG |
1525 | |
1526 | /* Clear percpu stats */ | |
1527 | for_each_possible_cpu(cpu) | |
1528 | memset(per_cpu_ptr(nic->drv_stats, cpu), 0, | |
1529 | sizeof(struct nicvf_drv_stats)); | |
712c3185 | 1530 | } |
4863dea3 SG |
1531 | |
1532 | err = nicvf_register_interrupts(nic); | |
1533 | if (err) | |
1534 | goto cleanup; | |
1535 | ||
1536 | /* Initialize the queues */ | |
1537 | err = nicvf_init_resources(nic); | |
1538 | if (err) | |
1539 | goto cleanup; | |
1540 | ||
1541 | /* Make sure queue initialization is written */ | |
1542 | wmb(); | |
1543 | ||
1544 | nicvf_reg_write(nic, NIC_VF_INT, -1); | |
1545 | /* Enable Qset err interrupt */ | |
1546 | nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1547 | ||
1548 | /* Enable completion queue interrupt */ | |
1549 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) | |
1550 | nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); | |
1551 | ||
1552 | /* Enable RBDR threshold interrupt */ | |
1553 | for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) | |
1554 | nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); | |
1555 | ||
c94acf80 | 1556 | /* Send VF config done msg to PF */ |
0dd563b9 | 1557 | nicvf_send_cfg_done(nic); |
74840b83 | 1558 | |
2ec1ed2a LB |
1559 | if (nic->nicvf_rx_mode_wq) { |
1560 | INIT_DELAYED_WORK(&nic->link_change_work, | |
1561 | nicvf_link_status_check_task); | |
1562 | queue_delayed_work(nic->nicvf_rx_mode_wq, | |
1563 | &nic->link_change_work, 0); | |
1564 | } | |
2c632ad8 | 1565 | |
4863dea3 SG |
1566 | return 0; |
1567 | cleanup: | |
1568 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1569 | nicvf_unregister_interrupts(nic); | |
39ad6eea SG |
1570 | tasklet_kill(&nic->qs_err_task); |
1571 | tasklet_kill(&nic->rbdr_task); | |
4863dea3 SG |
1572 | napi_del: |
1573 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
1574 | cq_poll = nic->napi[qidx]; | |
1575 | if (!cq_poll) | |
1576 | continue; | |
1577 | napi_disable(&cq_poll->napi); | |
1578 | netif_napi_del(&cq_poll->napi); | |
4863dea3 | 1579 | } |
39ad6eea | 1580 | nicvf_free_cq_poll(nic); |
4863dea3 SG |
1581 | return err; |
1582 | } | |
1583 | ||
4863dea3 SG |
1584 | static int nicvf_change_mtu(struct net_device *netdev, int new_mtu) |
1585 | { | |
1586 | struct nicvf *nic = netdev_priv(netdev); | |
f9aa9dc7 | 1587 | int orig_mtu = netdev->mtu; |
4863dea3 | 1588 | |
1f227d16 MC |
1589 | /* For now just support only the usual MTU sized frames, |
1590 | * plus some headroom for VLAN, QinQ. | |
1591 | */ | |
1592 | if (nic->xdp_prog && new_mtu > MAX_XDP_MTU) { | |
1593 | netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n", | |
1594 | netdev->mtu); | |
1595 | return -EINVAL; | |
1596 | } | |
1597 | ||
4863dea3 | 1598 | netdev->mtu = new_mtu; |
712c3185 SG |
1599 | |
1600 | if (!netif_running(netdev)) | |
1601 | return 0; | |
1602 | ||
f9aa9dc7 DM |
1603 | if (nicvf_update_hw_max_frs(nic, new_mtu)) { |
1604 | netdev->mtu = orig_mtu; | |
4863dea3 | 1605 | return -EINVAL; |
f9aa9dc7 | 1606 | } |
4863dea3 SG |
1607 | |
1608 | return 0; | |
1609 | } | |
1610 | ||
1611 | static int nicvf_set_mac_address(struct net_device *netdev, void *p) | |
1612 | { | |
1613 | struct sockaddr *addr = p; | |
1614 | struct nicvf *nic = netdev_priv(netdev); | |
1615 | ||
1616 | if (!is_valid_ether_addr(addr->sa_data)) | |
1617 | return -EADDRNOTAVAIL; | |
1618 | ||
1619 | memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); | |
1620 | ||
ce211b17 | 1621 | if (nic->pdev->msix_enabled) { |
4863dea3 SG |
1622 | if (nicvf_hw_set_mac_addr(nic, netdev)) |
1623 | return -EBUSY; | |
bd049a90 PF |
1624 | } else { |
1625 | nic->set_mac_pending = true; | |
1626 | } | |
4863dea3 SG |
1627 | |
1628 | return 0; | |
1629 | } | |
1630 | ||
4863dea3 SG |
1631 | void nicvf_update_lmac_stats(struct nicvf *nic) |
1632 | { | |
1633 | int stat = 0; | |
1634 | union nic_mbx mbx = {}; | |
4863dea3 SG |
1635 | |
1636 | if (!netif_running(nic->netdev)) | |
1637 | return; | |
1638 | ||
1639 | mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS; | |
1640 | mbx.bgx_stats.vf_id = nic->vf_id; | |
1641 | /* Rx stats */ | |
1642 | mbx.bgx_stats.rx = 1; | |
1643 | while (stat < BGX_RX_STATS_COUNT) { | |
4863dea3 | 1644 | mbx.bgx_stats.idx = stat; |
6051cba7 SG |
1645 | if (nicvf_send_msg_to_pf(nic, &mbx)) |
1646 | return; | |
4863dea3 SG |
1647 | stat++; |
1648 | } | |
1649 | ||
1650 | stat = 0; | |
1651 | ||
1652 | /* Tx stats */ | |
1653 | mbx.bgx_stats.rx = 0; | |
1654 | while (stat < BGX_TX_STATS_COUNT) { | |
4863dea3 | 1655 | mbx.bgx_stats.idx = stat; |
6051cba7 SG |
1656 | if (nicvf_send_msg_to_pf(nic, &mbx)) |
1657 | return; | |
4863dea3 SG |
1658 | stat++; |
1659 | } | |
1660 | } | |
1661 | ||
1662 | void nicvf_update_stats(struct nicvf *nic) | |
1663 | { | |
964cb69b SG |
1664 | int qidx, cpu; |
1665 | u64 tmp_stats = 0; | |
a2dc5ded | 1666 | struct nicvf_hw_stats *stats = &nic->hw_stats; |
964cb69b | 1667 | struct nicvf_drv_stats *drv_stats; |
4863dea3 SG |
1668 | struct queue_set *qs = nic->qs; |
1669 | ||
1670 | #define GET_RX_STATS(reg) \ | |
1671 | nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3)) | |
1672 | #define GET_TX_STATS(reg) \ | |
1673 | nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3)) | |
1674 | ||
a2dc5ded SG |
1675 | stats->rx_bytes = GET_RX_STATS(RX_OCTS); |
1676 | stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); | |
1677 | stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); | |
1678 | stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); | |
4863dea3 SG |
1679 | stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); |
1680 | stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); | |
1681 | stats->rx_drop_red = GET_RX_STATS(RX_RED); | |
a2dc5ded | 1682 | stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); |
4863dea3 | 1683 | stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); |
a2dc5ded | 1684 | stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); |
4863dea3 SG |
1685 | stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); |
1686 | stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); | |
1687 | stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); | |
1688 | stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); | |
1689 | ||
964cb69b SG |
1690 | stats->tx_bytes = GET_TX_STATS(TX_OCTS); |
1691 | stats->tx_ucast_frames = GET_TX_STATS(TX_UCAST); | |
1692 | stats->tx_bcast_frames = GET_TX_STATS(TX_BCAST); | |
1693 | stats->tx_mcast_frames = GET_TX_STATS(TX_MCAST); | |
4863dea3 SG |
1694 | stats->tx_drops = GET_TX_STATS(TX_DROP); |
1695 | ||
964cb69b SG |
1696 | /* On T88 pass 2.0, the dummy SQE added for TSO notification |
1697 | * via CQE has 'dont_send' set. Hence HW drops the pkt pointed | |
1698 | * pointed by dummy SQE and results in tx_drops counter being | |
1699 | * incremented. Subtracting it from tx_tso counter will give | |
1700 | * exact tx_drops counter. | |
1701 | */ | |
1702 | if (nic->t88 && nic->hw_tso) { | |
1703 | for_each_possible_cpu(cpu) { | |
1704 | drv_stats = per_cpu_ptr(nic->drv_stats, cpu); | |
1705 | tmp_stats += drv_stats->tx_tso; | |
1706 | } | |
1707 | stats->tx_drops = tmp_stats - stats->tx_drops; | |
1708 | } | |
1709 | stats->tx_frames = stats->tx_ucast_frames + | |
1710 | stats->tx_bcast_frames + | |
1711 | stats->tx_mcast_frames; | |
1712 | stats->rx_frames = stats->rx_ucast_frames + | |
1713 | stats->rx_bcast_frames + | |
1714 | stats->rx_mcast_frames; | |
1715 | stats->rx_drops = stats->rx_drop_red + | |
1716 | stats->rx_drop_overrun; | |
4863dea3 SG |
1717 | |
1718 | /* Update RQ and SQ stats */ | |
1719 | for (qidx = 0; qidx < qs->rq_cnt; qidx++) | |
1720 | nicvf_update_rq_stats(nic, qidx); | |
1721 | for (qidx = 0; qidx < qs->sq_cnt; qidx++) | |
1722 | nicvf_update_sq_stats(nic, qidx); | |
1723 | } | |
1724 | ||
bc1f4470 | 1725 | static void nicvf_get_stats64(struct net_device *netdev, |
1726 | struct rtnl_link_stats64 *stats) | |
4863dea3 SG |
1727 | { |
1728 | struct nicvf *nic = netdev_priv(netdev); | |
a2dc5ded | 1729 | struct nicvf_hw_stats *hw_stats = &nic->hw_stats; |
4863dea3 SG |
1730 | |
1731 | nicvf_update_stats(nic); | |
1732 | ||
a2dc5ded | 1733 | stats->rx_bytes = hw_stats->rx_bytes; |
964cb69b SG |
1734 | stats->rx_packets = hw_stats->rx_frames; |
1735 | stats->rx_dropped = hw_stats->rx_drops; | |
a2dc5ded | 1736 | stats->multicast = hw_stats->rx_mcast_frames; |
4863dea3 | 1737 | |
964cb69b SG |
1738 | stats->tx_bytes = hw_stats->tx_bytes; |
1739 | stats->tx_packets = hw_stats->tx_frames; | |
1740 | stats->tx_dropped = hw_stats->tx_drops; | |
4863dea3 | 1741 | |
4863dea3 SG |
1742 | } |
1743 | ||
1744 | static void nicvf_tx_timeout(struct net_device *dev) | |
1745 | { | |
1746 | struct nicvf *nic = netdev_priv(dev); | |
1747 | ||
bf24e136 | 1748 | netif_warn(nic, tx_err, dev, "Transmit timed out, resetting\n"); |
4863dea3 | 1749 | |
964cb69b | 1750 | this_cpu_inc(nic->drv_stats->tx_timeout); |
4863dea3 SG |
1751 | schedule_work(&nic->reset_task); |
1752 | } | |
1753 | ||
1754 | static void nicvf_reset_task(struct work_struct *work) | |
1755 | { | |
1756 | struct nicvf *nic; | |
1757 | ||
1758 | nic = container_of(work, struct nicvf, reset_task); | |
1759 | ||
1760 | if (!netif_running(nic->netdev)) | |
1761 | return; | |
1762 | ||
1763 | nicvf_stop(nic->netdev); | |
1764 | nicvf_open(nic->netdev); | |
860e9538 | 1765 | netif_trans_update(nic->netdev); |
4863dea3 SG |
1766 | } |
1767 | ||
d77a2384 SG |
1768 | static int nicvf_config_loopback(struct nicvf *nic, |
1769 | netdev_features_t features) | |
1770 | { | |
1771 | union nic_mbx mbx = {}; | |
1772 | ||
1773 | mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK; | |
1774 | mbx.lbk.vf_id = nic->vf_id; | |
1775 | mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0; | |
1776 | ||
1777 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1778 | } | |
1779 | ||
1780 | static netdev_features_t nicvf_fix_features(struct net_device *netdev, | |
1781 | netdev_features_t features) | |
1782 | { | |
1783 | struct nicvf *nic = netdev_priv(netdev); | |
1784 | ||
1785 | if ((features & NETIF_F_LOOPBACK) && | |
1786 | netif_running(netdev) && !nic->loopback_supported) | |
1787 | features &= ~NETIF_F_LOOPBACK; | |
1788 | ||
1789 | return features; | |
1790 | } | |
1791 | ||
aa2e259b SG |
1792 | static int nicvf_set_features(struct net_device *netdev, |
1793 | netdev_features_t features) | |
1794 | { | |
1795 | struct nicvf *nic = netdev_priv(netdev); | |
1796 | netdev_features_t changed = features ^ netdev->features; | |
1797 | ||
1798 | if (changed & NETIF_F_HW_VLAN_CTAG_RX) | |
1799 | nicvf_config_vlan_stripping(nic, features); | |
1800 | ||
d77a2384 SG |
1801 | if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev)) |
1802 | return nicvf_config_loopback(nic, features); | |
1803 | ||
aa2e259b SG |
1804 | return 0; |
1805 | } | |
1806 | ||
05c773f5 SG |
1807 | static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached) |
1808 | { | |
1809 | u8 cq_count, txq_count; | |
1810 | ||
1811 | /* Set XDP Tx queue count same as Rx queue count */ | |
1812 | if (!bpf_attached) | |
1813 | nic->xdp_tx_queues = 0; | |
1814 | else | |
1815 | nic->xdp_tx_queues = nic->rx_queues; | |
1816 | ||
1817 | /* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets | |
1818 | * needs to be allocated, check how many. | |
1819 | */ | |
1820 | txq_count = nic->xdp_tx_queues + nic->tx_queues; | |
1821 | cq_count = max(nic->rx_queues, txq_count); | |
1822 | if (cq_count > MAX_CMP_QUEUES_PER_QS) { | |
1823 | nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS); | |
1824 | nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1; | |
1825 | } else { | |
1826 | nic->sqs_count = 0; | |
1827 | } | |
1828 | ||
1829 | /* Set primary Qset's resources */ | |
1830 | nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS); | |
1831 | nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS); | |
1832 | nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt); | |
1833 | ||
1834 | /* Update stack */ | |
1835 | nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues); | |
1836 | } | |
1837 | ||
1838 | static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog) | |
1839 | { | |
1840 | struct net_device *dev = nic->netdev; | |
1841 | bool if_up = netif_running(nic->netdev); | |
1842 | struct bpf_prog *old_prog; | |
1843 | bool bpf_attached = false; | |
6d0f60b0 | 1844 | int ret = 0; |
05c773f5 | 1845 | |
5ee15c10 MC |
1846 | /* For now just support only the usual MTU sized frames, |
1847 | * plus some headroom for VLAN, QinQ. | |
1848 | */ | |
1849 | if (prog && dev->mtu > MAX_XDP_MTU) { | |
05c773f5 SG |
1850 | netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n", |
1851 | dev->mtu); | |
1852 | return -EOPNOTSUPP; | |
1853 | } | |
1854 | ||
05c773f5 SG |
1855 | /* ALL SQs attached to CQs i.e same as RQs, are treated as |
1856 | * XDP Tx queues and more Tx queues are allocated for | |
1857 | * network stack to send pkts out. | |
1858 | * | |
1859 | * No of Tx queues are either same as Rx queues or whatever | |
1860 | * is left in max no of queues possible. | |
1861 | */ | |
1862 | if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) { | |
1863 | netdev_warn(dev, | |
1864 | "Failed to attach BPF prog, RXQs + TXQs > Max %d\n", | |
1865 | nic->max_queues); | |
1866 | return -ENOMEM; | |
1867 | } | |
1868 | ||
1869 | if (if_up) | |
1870 | nicvf_stop(nic->netdev); | |
1871 | ||
1872 | old_prog = xchg(&nic->xdp_prog, prog); | |
1873 | /* Detach old prog, if any */ | |
1874 | if (old_prog) | |
1875 | bpf_prog_put(old_prog); | |
1876 | ||
1877 | if (nic->xdp_prog) { | |
1878 | /* Attach BPF program */ | |
1879 | nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1); | |
6d0f60b0 | 1880 | if (!IS_ERR(nic->xdp_prog)) { |
05c773f5 | 1881 | bpf_attached = true; |
6d0f60b0 LB |
1882 | } else { |
1883 | ret = PTR_ERR(nic->xdp_prog); | |
1884 | nic->xdp_prog = NULL; | |
1885 | } | |
05c773f5 SG |
1886 | } |
1887 | ||
1888 | /* Calculate Tx queues needed for XDP and network stack */ | |
1889 | nicvf_set_xdp_queues(nic, bpf_attached); | |
1890 | ||
1891 | if (if_up) { | |
1892 | /* Reinitialize interface, clean slate */ | |
1893 | nicvf_open(nic->netdev); | |
1894 | netif_trans_update(nic->netdev); | |
1895 | } | |
1896 | ||
6d0f60b0 | 1897 | return ret; |
05c773f5 SG |
1898 | } |
1899 | ||
f4e63525 | 1900 | static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp) |
05c773f5 SG |
1901 | { |
1902 | struct nicvf *nic = netdev_priv(netdev); | |
1903 | ||
1904 | /* To avoid checks while retrieving buffer address from CQE_RX, | |
1905 | * do not support XDP for T88 pass1.x silicons which are anyway | |
1906 | * not in use widely. | |
1907 | */ | |
1908 | if (pass1_silicon(nic->pdev)) | |
1909 | return -EOPNOTSUPP; | |
1910 | ||
1911 | switch (xdp->command) { | |
1912 | case XDP_SETUP_PROG: | |
1913 | return nicvf_xdp_setup(nic, xdp->prog); | |
1914 | case XDP_QUERY_PROG: | |
1efde2b6 | 1915 | xdp->prog_id = nic->xdp_prog ? nic->xdp_prog->aux->id : 0; |
05c773f5 SG |
1916 | return 0; |
1917 | default: | |
1918 | return -EINVAL; | |
1919 | } | |
1920 | } | |
1921 | ||
4a875509 SG |
1922 | static int nicvf_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) |
1923 | { | |
1924 | struct hwtstamp_config config; | |
1925 | struct nicvf *nic = netdev_priv(netdev); | |
1926 | ||
1927 | if (!nic->ptp_clock) | |
1928 | return -ENODEV; | |
1929 | ||
1930 | if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) | |
1931 | return -EFAULT; | |
1932 | ||
1933 | /* reserved for future extensions */ | |
1934 | if (config.flags) | |
1935 | return -EINVAL; | |
1936 | ||
1937 | switch (config.tx_type) { | |
1938 | case HWTSTAMP_TX_OFF: | |
1939 | case HWTSTAMP_TX_ON: | |
1940 | break; | |
1941 | default: | |
1942 | return -ERANGE; | |
1943 | } | |
1944 | ||
1945 | switch (config.rx_filter) { | |
1946 | case HWTSTAMP_FILTER_NONE: | |
1947 | nic->hw_rx_tstamp = false; | |
1948 | break; | |
1949 | case HWTSTAMP_FILTER_ALL: | |
1950 | case HWTSTAMP_FILTER_SOME: | |
1951 | case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: | |
1952 | case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: | |
1953 | case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: | |
1954 | case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: | |
1955 | case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: | |
1956 | case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: | |
1957 | case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: | |
1958 | case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: | |
1959 | case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: | |
1960 | case HWTSTAMP_FILTER_PTP_V2_EVENT: | |
1961 | case HWTSTAMP_FILTER_PTP_V2_SYNC: | |
1962 | case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: | |
1963 | nic->hw_rx_tstamp = true; | |
1964 | config.rx_filter = HWTSTAMP_FILTER_ALL; | |
1965 | break; | |
1966 | default: | |
1967 | return -ERANGE; | |
1968 | } | |
1969 | ||
1970 | if (netif_running(netdev)) | |
1971 | nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp); | |
1972 | ||
1973 | if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) | |
1974 | return -EFAULT; | |
1975 | ||
1976 | return 0; | |
1977 | } | |
1978 | ||
1979 | static int nicvf_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) | |
1980 | { | |
1981 | switch (cmd) { | |
1982 | case SIOCSHWTSTAMP: | |
1983 | return nicvf_config_hwtstamp(netdev, req); | |
1984 | default: | |
1985 | return -EOPNOTSUPP; | |
1986 | } | |
1987 | } | |
1988 | ||
469998c8 VL |
1989 | static void __nicvf_set_rx_mode_task(u8 mode, struct xcast_addr_list *mc_addrs, |
1990 | struct nicvf *nic) | |
37c3347e | 1991 | { |
37c3347e | 1992 | union nic_mbx mbx = {}; |
9b5c4dfb | 1993 | int idx; |
37c3347e | 1994 | |
37c3347e VL |
1995 | /* From the inside of VM code flow we have only 128 bits memory |
1996 | * available to send message to host's PF, so send all mc addrs | |
1997 | * one by one, starting from flush command in case if kernel | |
1998 | * requests to configure specific MAC filtering | |
1999 | */ | |
2000 | ||
2001 | /* flush DMAC filters and reset RX mode */ | |
2002 | mbx.xcast.msg = NIC_MBOX_MSG_RESET_XCAST; | |
7db730d9 VL |
2003 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
2004 | goto free_mc; | |
37c3347e | 2005 | |
469998c8 | 2006 | if (mode & BGX_XCAST_MCAST_FILTER) { |
37c3347e VL |
2007 | /* once enabling filtering, we need to signal to PF to add |
2008 | * its' own LMAC to the filter to accept packets for it. | |
2009 | */ | |
2010 | mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST; | |
53544396 | 2011 | mbx.xcast.mac = 0; |
7db730d9 VL |
2012 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
2013 | goto free_mc; | |
37c3347e VL |
2014 | } |
2015 | ||
2016 | /* check if we have any specific MACs to be added to PF DMAC filter */ | |
469998c8 | 2017 | if (mc_addrs) { |
37c3347e | 2018 | /* now go through kernel list of MACs and add them one by one */ |
469998c8 | 2019 | for (idx = 0; idx < mc_addrs->count; idx++) { |
37c3347e | 2020 | mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST; |
53544396 | 2021 | mbx.xcast.mac = mc_addrs->mc[idx]; |
7db730d9 VL |
2022 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
2023 | goto free_mc; | |
37c3347e | 2024 | } |
37c3347e VL |
2025 | } |
2026 | ||
2027 | /* and finally set rx mode for PF accordingly */ | |
2028 | mbx.xcast.msg = NIC_MBOX_MSG_SET_XCAST; | |
53544396 | 2029 | mbx.xcast.mode = mode; |
37c3347e VL |
2030 | |
2031 | nicvf_send_msg_to_pf(nic, &mbx); | |
7db730d9 VL |
2032 | free_mc: |
2033 | kfree(mc_addrs); | |
37c3347e VL |
2034 | } |
2035 | ||
469998c8 VL |
2036 | static void nicvf_set_rx_mode_task(struct work_struct *work_arg) |
2037 | { | |
2038 | struct nicvf_work *vf_work = container_of(work_arg, struct nicvf_work, | |
2ecbe4f4 | 2039 | work); |
469998c8 VL |
2040 | struct nicvf *nic = container_of(vf_work, struct nicvf, rx_mode_work); |
2041 | u8 mode; | |
2042 | struct xcast_addr_list *mc; | |
2043 | ||
2044 | if (!vf_work) | |
2045 | return; | |
2046 | ||
2047 | /* Save message data locally to prevent them from | |
2048 | * being overwritten by next ndo_set_rx_mode call(). | |
2049 | */ | |
2050 | spin_lock(&nic->rx_mode_wq_lock); | |
2051 | mode = vf_work->mode; | |
2052 | mc = vf_work->mc; | |
2053 | vf_work->mc = NULL; | |
2054 | spin_unlock(&nic->rx_mode_wq_lock); | |
2055 | ||
2056 | __nicvf_set_rx_mode_task(mode, mc, nic); | |
2057 | } | |
2058 | ||
37c3347e VL |
2059 | static void nicvf_set_rx_mode(struct net_device *netdev) |
2060 | { | |
2061 | struct nicvf *nic = netdev_priv(netdev); | |
2062 | struct netdev_hw_addr *ha; | |
2063 | struct xcast_addr_list *mc_list = NULL; | |
2064 | u8 mode = 0; | |
2065 | ||
2066 | if (netdev->flags & IFF_PROMISC) { | |
2067 | mode = BGX_XCAST_BCAST_ACCEPT | BGX_XCAST_MCAST_ACCEPT; | |
2068 | } else { | |
2069 | if (netdev->flags & IFF_BROADCAST) | |
2070 | mode |= BGX_XCAST_BCAST_ACCEPT; | |
2071 | ||
2072 | if (netdev->flags & IFF_ALLMULTI) { | |
2073 | mode |= BGX_XCAST_MCAST_ACCEPT; | |
2074 | } else if (netdev->flags & IFF_MULTICAST) { | |
2075 | mode |= BGX_XCAST_MCAST_FILTER; | |
2076 | /* here we need to copy mc addrs */ | |
2077 | if (netdev_mc_count(netdev)) { | |
9b5c4dfb VL |
2078 | mc_list = kmalloc(offsetof(typeof(*mc_list), |
2079 | mc[netdev_mc_count(netdev)]), | |
2080 | GFP_ATOMIC); | |
2081 | if (unlikely(!mc_list)) | |
2082 | return; | |
2083 | mc_list->count = 0; | |
37c3347e | 2084 | netdev_hw_addr_list_for_each(ha, &netdev->mc) { |
9b5c4dfb | 2085 | mc_list->mc[mc_list->count] = |
37c3347e | 2086 | ether_addr_to_u64(ha->addr); |
37c3347e VL |
2087 | mc_list->count++; |
2088 | } | |
2089 | } | |
2090 | } | |
2091 | } | |
469998c8 VL |
2092 | spin_lock(&nic->rx_mode_wq_lock); |
2093 | kfree(nic->rx_mode_work.mc); | |
37c3347e VL |
2094 | nic->rx_mode_work.mc = mc_list; |
2095 | nic->rx_mode_work.mode = mode; | |
2ecbe4f4 | 2096 | queue_work(nic->nicvf_rx_mode_wq, &nic->rx_mode_work.work); |
469998c8 | 2097 | spin_unlock(&nic->rx_mode_wq_lock); |
37c3347e VL |
2098 | } |
2099 | ||
4863dea3 SG |
2100 | static const struct net_device_ops nicvf_netdev_ops = { |
2101 | .ndo_open = nicvf_open, | |
2102 | .ndo_stop = nicvf_stop, | |
2103 | .ndo_start_xmit = nicvf_xmit, | |
2104 | .ndo_change_mtu = nicvf_change_mtu, | |
2105 | .ndo_set_mac_address = nicvf_set_mac_address, | |
2106 | .ndo_get_stats64 = nicvf_get_stats64, | |
2107 | .ndo_tx_timeout = nicvf_tx_timeout, | |
d77a2384 | 2108 | .ndo_fix_features = nicvf_fix_features, |
aa2e259b | 2109 | .ndo_set_features = nicvf_set_features, |
f4e63525 | 2110 | .ndo_bpf = nicvf_xdp, |
4a875509 | 2111 | .ndo_do_ioctl = nicvf_ioctl, |
37c3347e | 2112 | .ndo_set_rx_mode = nicvf_set_rx_mode, |
4863dea3 SG |
2113 | }; |
2114 | ||
2115 | static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) | |
2116 | { | |
2117 | struct device *dev = &pdev->dev; | |
2118 | struct net_device *netdev; | |
2119 | struct nicvf *nic; | |
92dc8769 | 2120 | int err, qcount; |
7ceb8a13 | 2121 | u16 sdevid; |
4a875509 SG |
2122 | struct cavium_ptp *ptp_clock; |
2123 | ||
2124 | ptp_clock = cavium_ptp_get(); | |
2125 | if (IS_ERR(ptp_clock)) { | |
2126 | if (PTR_ERR(ptp_clock) == -ENODEV) | |
2127 | /* In virtualized environment we proceed without ptp */ | |
2128 | ptp_clock = NULL; | |
2129 | else | |
2130 | return PTR_ERR(ptp_clock); | |
2131 | } | |
4863dea3 SG |
2132 | |
2133 | err = pci_enable_device(pdev); | |
2134 | if (err) { | |
2135 | dev_err(dev, "Failed to enable PCI device\n"); | |
2136 | return err; | |
2137 | } | |
2138 | ||
2139 | err = pci_request_regions(pdev, DRV_NAME); | |
2140 | if (err) { | |
2141 | dev_err(dev, "PCI request regions failed 0x%x\n", err); | |
2142 | goto err_disable_device; | |
2143 | } | |
2144 | ||
2145 | err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); | |
2146 | if (err) { | |
2147 | dev_err(dev, "Unable to get usable DMA configuration\n"); | |
2148 | goto err_release_regions; | |
2149 | } | |
2150 | ||
2151 | err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); | |
2152 | if (err) { | |
2153 | dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n"); | |
2154 | goto err_release_regions; | |
2155 | } | |
2156 | ||
3a397ebe | 2157 | qcount = netif_get_num_default_rss_queues(); |
92dc8769 SG |
2158 | |
2159 | /* Restrict multiqset support only for host bound VFs */ | |
2160 | if (pdev->is_virtfn) { | |
2161 | /* Set max number of queues per VF */ | |
3a397ebe SG |
2162 | qcount = min_t(int, num_online_cpus(), |
2163 | (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS); | |
92dc8769 SG |
2164 | } |
2165 | ||
2166 | netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount); | |
4863dea3 SG |
2167 | if (!netdev) { |
2168 | err = -ENOMEM; | |
2169 | goto err_release_regions; | |
2170 | } | |
2171 | ||
2172 | pci_set_drvdata(pdev, netdev); | |
2173 | ||
2174 | SET_NETDEV_DEV(netdev, &pdev->dev); | |
2175 | ||
2176 | nic = netdev_priv(netdev); | |
2177 | nic->netdev = netdev; | |
2178 | nic->pdev = pdev; | |
92dc8769 SG |
2179 | nic->pnicvf = nic; |
2180 | nic->max_queues = qcount; | |
87de0838 SG |
2181 | /* If no of CPUs are too low, there won't be any queues left |
2182 | * for XDP_TX, hence double it. | |
2183 | */ | |
2184 | if (!nic->t88) | |
2185 | nic->max_queues *= 2; | |
4a875509 | 2186 | nic->ptp_clock = ptp_clock; |
4863dea3 SG |
2187 | |
2188 | /* MAP VF's configuration registers */ | |
2189 | nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); | |
2190 | if (!nic->reg_base) { | |
2191 | dev_err(dev, "Cannot map config register space, aborting\n"); | |
2192 | err = -ENOMEM; | |
2193 | goto err_free_netdev; | |
2194 | } | |
2195 | ||
964cb69b SG |
2196 | nic->drv_stats = netdev_alloc_pcpu_stats(struct nicvf_drv_stats); |
2197 | if (!nic->drv_stats) { | |
2198 | err = -ENOMEM; | |
2199 | goto err_free_netdev; | |
2200 | } | |
2201 | ||
4863dea3 SG |
2202 | err = nicvf_set_qset_resources(nic); |
2203 | if (err) | |
2204 | goto err_free_netdev; | |
2205 | ||
4863dea3 SG |
2206 | /* Check if PF is alive and get MAC address for this VF */ |
2207 | err = nicvf_register_misc_interrupt(nic); | |
2208 | if (err) | |
2209 | goto err_free_netdev; | |
2210 | ||
92dc8769 SG |
2211 | nicvf_send_vf_struct(nic); |
2212 | ||
8d210d54 SG |
2213 | if (!pass1_silicon(nic->pdev)) |
2214 | nic->hw_tso = true; | |
2215 | ||
83abb7d7 SG |
2216 | /* Get iommu domain for iova to physical addr conversion */ |
2217 | nic->iommu_domain = iommu_get_domain_for_dev(dev); | |
2218 | ||
7ceb8a13 SG |
2219 | pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid); |
2220 | if (sdevid == 0xA134) | |
2221 | nic->t88 = true; | |
2222 | ||
92dc8769 SG |
2223 | /* Check if this VF is in QS only mode */ |
2224 | if (nic->sqs_mode) | |
2225 | return 0; | |
2226 | ||
2227 | err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues); | |
2228 | if (err) | |
2229 | goto err_unregister_interrupts; | |
2230 | ||
3a9024f5 TS |
2231 | netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_SG | |
2232 | NETIF_F_TSO | NETIF_F_GRO | NETIF_F_TSO6 | | |
2233 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | | |
92dc8769 SG |
2234 | NETIF_F_HW_VLAN_CTAG_RX); |
2235 | ||
2236 | netdev->hw_features |= NETIF_F_RXHASH; | |
aa2e259b SG |
2237 | |
2238 | netdev->features |= netdev->hw_features; | |
d77a2384 | 2239 | netdev->hw_features |= NETIF_F_LOOPBACK; |
38bb5d4f | 2240 | |
3a9024f5 TS |
2241 | netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | |
2242 | NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6; | |
4863dea3 SG |
2243 | |
2244 | netdev->netdev_ops = &nicvf_netdev_ops; | |
3d7a8aaa | 2245 | netdev->watchdog_timeo = NICVF_TX_TIMEOUT; |
4863dea3 | 2246 | |
109cc165 JW |
2247 | /* MTU range: 64 - 9200 */ |
2248 | netdev->min_mtu = NIC_HW_MIN_FRS; | |
2249 | netdev->max_mtu = NIC_HW_MAX_FRS; | |
2250 | ||
4863dea3 SG |
2251 | INIT_WORK(&nic->reset_task, nicvf_reset_task); |
2252 | ||
2ecbe4f4 VL |
2253 | nic->nicvf_rx_mode_wq = alloc_ordered_workqueue("nicvf_rx_mode_wq_VF%d", |
2254 | WQ_MEM_RECLAIM, | |
2255 | nic->vf_id); | |
0b31d98d KL |
2256 | if (!nic->nicvf_rx_mode_wq) { |
2257 | err = -ENOMEM; | |
2258 | dev_err(dev, "Failed to allocate work queue\n"); | |
2259 | goto err_unregister_interrupts; | |
2260 | } | |
2261 | ||
2ecbe4f4 | 2262 | INIT_WORK(&nic->rx_mode_work.work, nicvf_set_rx_mode_task); |
469998c8 | 2263 | spin_lock_init(&nic->rx_mode_wq_lock); |
609ea65c | 2264 | mutex_init(&nic->rx_mode_mtx); |
37c3347e | 2265 | |
4863dea3 SG |
2266 | err = register_netdev(netdev); |
2267 | if (err) { | |
2268 | dev_err(dev, "Failed to register netdevice\n"); | |
2269 | goto err_unregister_interrupts; | |
2270 | } | |
2271 | ||
2272 | nic->msg_enable = debug; | |
2273 | ||
2274 | nicvf_set_ethtool_ops(netdev); | |
2275 | ||
2276 | return 0; | |
2277 | ||
2278 | err_unregister_interrupts: | |
2279 | nicvf_unregister_interrupts(nic); | |
2280 | err_free_netdev: | |
2281 | pci_set_drvdata(pdev, NULL); | |
964cb69b SG |
2282 | if (nic->drv_stats) |
2283 | free_percpu(nic->drv_stats); | |
4863dea3 SG |
2284 | free_netdev(netdev); |
2285 | err_release_regions: | |
2286 | pci_release_regions(pdev); | |
2287 | err_disable_device: | |
2288 | pci_disable_device(pdev); | |
2289 | return err; | |
2290 | } | |
2291 | ||
2292 | static void nicvf_remove(struct pci_dev *pdev) | |
2293 | { | |
2294 | struct net_device *netdev = pci_get_drvdata(pdev); | |
7750130d PF |
2295 | struct nicvf *nic; |
2296 | struct net_device *pnetdev; | |
2297 | ||
2298 | if (!netdev) | |
2299 | return; | |
2300 | ||
2301 | nic = netdev_priv(netdev); | |
2302 | pnetdev = nic->pnicvf->netdev; | |
4863dea3 | 2303 | |
92dc8769 SG |
2304 | /* Check if this Qset is assigned to different VF. |
2305 | * If yes, clean primary and all secondary Qsets. | |
2306 | */ | |
2307 | if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED)) | |
2308 | unregister_netdev(pnetdev); | |
2ecbe4f4 VL |
2309 | if (nic->nicvf_rx_mode_wq) { |
2310 | destroy_workqueue(nic->nicvf_rx_mode_wq); | |
2311 | nic->nicvf_rx_mode_wq = NULL; | |
2312 | } | |
4863dea3 SG |
2313 | nicvf_unregister_interrupts(nic); |
2314 | pci_set_drvdata(pdev, NULL); | |
964cb69b SG |
2315 | if (nic->drv_stats) |
2316 | free_percpu(nic->drv_stats); | |
4a875509 | 2317 | cavium_ptp_put(nic->ptp_clock); |
4863dea3 SG |
2318 | free_netdev(netdev); |
2319 | pci_release_regions(pdev); | |
2320 | pci_disable_device(pdev); | |
2321 | } | |
2322 | ||
4adf4351 SG |
2323 | static void nicvf_shutdown(struct pci_dev *pdev) |
2324 | { | |
2325 | nicvf_remove(pdev); | |
2326 | } | |
2327 | ||
4863dea3 SG |
2328 | static struct pci_driver nicvf_driver = { |
2329 | .name = DRV_NAME, | |
2330 | .id_table = nicvf_id_table, | |
2331 | .probe = nicvf_probe, | |
2332 | .remove = nicvf_remove, | |
4adf4351 | 2333 | .shutdown = nicvf_shutdown, |
4863dea3 SG |
2334 | }; |
2335 | ||
2336 | static int __init nicvf_init_module(void) | |
2337 | { | |
2338 | pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); | |
4863dea3 SG |
2339 | return pci_register_driver(&nicvf_driver); |
2340 | } | |
2341 | ||
2342 | static void __exit nicvf_cleanup_module(void) | |
2343 | { | |
2344 | pci_unregister_driver(&nicvf_driver); | |
2345 | } | |
2346 | ||
2347 | module_init(nicvf_init_module); | |
2348 | module_exit(nicvf_cleanup_module); |