]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/net/ethernet/emulex/benet/be_main.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
x86/ioapic: Restore IO-APIC irq_chip retrigger callback
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / emulex / benet / be_main.c
1 /*
2 * Copyright (C) 2005 - 2016 Broadcom
3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
10 * Contact Information:
11 * linux-drivers@emulex.com
12 *
13 * Emulex
14 * 3333 Susan Street
15 * Costa Mesa, CA 92626
16 */
17
18 #include <linux/prefetch.h>
19 #include <linux/module.h>
20 #include "be.h"
21 #include "be_cmds.h"
22 #include <asm/div64.h>
23 #include <linux/aer.h>
24 #include <linux/if_bridge.h>
25 #include <net/busy_poll.h>
26 #include <net/vxlan.h>
27
28 MODULE_VERSION(DRV_VER);
29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
30 MODULE_AUTHOR("Emulex Corporation");
31 MODULE_LICENSE("GPL");
32
33 /* num_vfs module param is obsolete.
34 * Use sysfs method to enable/disable VFs.
35 */
36 static unsigned int num_vfs;
37 module_param(num_vfs, uint, S_IRUGO);
38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
39
40 static ushort rx_frag_size = 2048;
41 module_param(rx_frag_size, ushort, S_IRUGO);
42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
43
44 /* Per-module error detection/recovery workq shared across all functions.
45 * Each function schedules its own work request on this shared workq.
46 */
47 static struct workqueue_struct *be_err_recovery_workq;
48
49 static const struct pci_device_id be_dev_ids[] = {
50 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
51 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
52 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
53 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
54 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
55 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
56 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
57 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
58 { 0 }
59 };
60 MODULE_DEVICE_TABLE(pci, be_dev_ids);
61
62 /* Workqueue used by all functions for defering cmd calls to the adapter */
63 static struct workqueue_struct *be_wq;
64
65 /* UE Status Low CSR */
66 static const char * const ue_status_low_desc[] = {
67 "CEV",
68 "CTX",
69 "DBUF",
70 "ERX",
71 "Host",
72 "MPU",
73 "NDMA",
74 "PTC ",
75 "RDMA ",
76 "RXF ",
77 "RXIPS ",
78 "RXULP0 ",
79 "RXULP1 ",
80 "RXULP2 ",
81 "TIM ",
82 "TPOST ",
83 "TPRE ",
84 "TXIPS ",
85 "TXULP0 ",
86 "TXULP1 ",
87 "UC ",
88 "WDMA ",
89 "TXULP2 ",
90 "HOST1 ",
91 "P0_OB_LINK ",
92 "P1_OB_LINK ",
93 "HOST_GPIO ",
94 "MBOX ",
95 "ERX2 ",
96 "SPARE ",
97 "JTAG ",
98 "MPU_INTPEND "
99 };
100
101 /* UE Status High CSR */
102 static const char * const ue_status_hi_desc[] = {
103 "LPCMEMHOST",
104 "MGMT_MAC",
105 "PCS0ONLINE",
106 "MPU_IRAM",
107 "PCS1ONLINE",
108 "PCTL0",
109 "PCTL1",
110 "PMEM",
111 "RR",
112 "TXPB",
113 "RXPP",
114 "XAUI",
115 "TXP",
116 "ARM",
117 "IPC",
118 "HOST2",
119 "HOST3",
120 "HOST4",
121 "HOST5",
122 "HOST6",
123 "HOST7",
124 "ECRC",
125 "Poison TLP",
126 "NETC",
127 "PERIPH",
128 "LLTXULP",
129 "D2P",
130 "RCON",
131 "LDMA",
132 "LLTXP",
133 "LLTXPB",
134 "Unknown"
135 };
136
137 #define BE_VF_IF_EN_FLAGS (BE_IF_FLAGS_UNTAGGED | \
138 BE_IF_FLAGS_BROADCAST | \
139 BE_IF_FLAGS_MULTICAST | \
140 BE_IF_FLAGS_PASS_L3L4_ERRORS)
141
142 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
143 {
144 struct be_dma_mem *mem = &q->dma_mem;
145
146 if (mem->va) {
147 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
148 mem->dma);
149 mem->va = NULL;
150 }
151 }
152
153 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
154 u16 len, u16 entry_size)
155 {
156 struct be_dma_mem *mem = &q->dma_mem;
157
158 memset(q, 0, sizeof(*q));
159 q->len = len;
160 q->entry_size = entry_size;
161 mem->size = len * entry_size;
162 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
163 GFP_KERNEL);
164 if (!mem->va)
165 return -ENOMEM;
166 return 0;
167 }
168
169 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
170 {
171 u32 reg, enabled;
172
173 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
174 &reg);
175 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
176
177 if (!enabled && enable)
178 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
179 else if (enabled && !enable)
180 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
181 else
182 return;
183
184 pci_write_config_dword(adapter->pdev,
185 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
186 }
187
188 static void be_intr_set(struct be_adapter *adapter, bool enable)
189 {
190 int status = 0;
191
192 /* On lancer interrupts can't be controlled via this register */
193 if (lancer_chip(adapter))
194 return;
195
196 if (be_check_error(adapter, BE_ERROR_EEH))
197 return;
198
199 status = be_cmd_intr_set(adapter, enable);
200 if (status)
201 be_reg_intr_set(adapter, enable);
202 }
203
204 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
205 {
206 u32 val = 0;
207
208 if (be_check_error(adapter, BE_ERROR_HW))
209 return;
210
211 val |= qid & DB_RQ_RING_ID_MASK;
212 val |= posted << DB_RQ_NUM_POSTED_SHIFT;
213
214 wmb();
215 iowrite32(val, adapter->db + DB_RQ_OFFSET);
216 }
217
218 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
219 u16 posted)
220 {
221 u32 val = 0;
222
223 if (be_check_error(adapter, BE_ERROR_HW))
224 return;
225
226 val |= txo->q.id & DB_TXULP_RING_ID_MASK;
227 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
228
229 wmb();
230 iowrite32(val, adapter->db + txo->db_offset);
231 }
232
233 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
234 bool arm, bool clear_int, u16 num_popped,
235 u32 eq_delay_mult_enc)
236 {
237 u32 val = 0;
238
239 val |= qid & DB_EQ_RING_ID_MASK;
240 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
241
242 if (be_check_error(adapter, BE_ERROR_HW))
243 return;
244
245 if (arm)
246 val |= 1 << DB_EQ_REARM_SHIFT;
247 if (clear_int)
248 val |= 1 << DB_EQ_CLR_SHIFT;
249 val |= 1 << DB_EQ_EVNT_SHIFT;
250 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
251 val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT;
252 iowrite32(val, adapter->db + DB_EQ_OFFSET);
253 }
254
255 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
256 {
257 u32 val = 0;
258
259 val |= qid & DB_CQ_RING_ID_MASK;
260 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
261 DB_CQ_RING_ID_EXT_MASK_SHIFT);
262
263 if (be_check_error(adapter, BE_ERROR_HW))
264 return;
265
266 if (arm)
267 val |= 1 << DB_CQ_REARM_SHIFT;
268 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
269 iowrite32(val, adapter->db + DB_CQ_OFFSET);
270 }
271
272 static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac)
273 {
274 int i;
275
276 /* Check if mac has already been added as part of uc-list */
277 for (i = 0; i < adapter->uc_macs; i++) {
278 if (ether_addr_equal(adapter->uc_list[i].mac, mac)) {
279 /* mac already added, skip addition */
280 adapter->pmac_id[0] = adapter->pmac_id[i + 1];
281 return 0;
282 }
283 }
284
285 return be_cmd_pmac_add(adapter, mac, adapter->if_handle,
286 &adapter->pmac_id[0], 0);
287 }
288
289 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id)
290 {
291 int i;
292
293 /* Skip deletion if the programmed mac is
294 * being used in uc-list
295 */
296 for (i = 0; i < adapter->uc_macs; i++) {
297 if (adapter->pmac_id[i + 1] == pmac_id)
298 return;
299 }
300 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
301 }
302
303 static int be_mac_addr_set(struct net_device *netdev, void *p)
304 {
305 struct be_adapter *adapter = netdev_priv(netdev);
306 struct device *dev = &adapter->pdev->dev;
307 struct sockaddr *addr = p;
308 int status;
309 u8 mac[ETH_ALEN];
310 u32 old_pmac_id = adapter->pmac_id[0];
311
312 if (!is_valid_ether_addr(addr->sa_data))
313 return -EADDRNOTAVAIL;
314
315 /* Proceed further only if, User provided MAC is different
316 * from active MAC
317 */
318 if (ether_addr_equal(addr->sa_data, adapter->dev_mac))
319 return 0;
320
321 /* if device is not running, copy MAC to netdev->dev_addr */
322 if (!netif_running(netdev))
323 goto done;
324
325 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
326 * privilege or if PF did not provision the new MAC address.
327 * On BE3, this cmd will always fail if the VF doesn't have the
328 * FILTMGMT privilege. This failure is OK, only if the PF programmed
329 * the MAC for the VF.
330 */
331 mutex_lock(&adapter->rx_filter_lock);
332 status = be_dev_mac_add(adapter, (u8 *)addr->sa_data);
333 if (!status) {
334
335 /* Delete the old programmed MAC. This call may fail if the
336 * old MAC was already deleted by the PF driver.
337 */
338 if (adapter->pmac_id[0] != old_pmac_id)
339 be_dev_mac_del(adapter, old_pmac_id);
340 }
341
342 mutex_unlock(&adapter->rx_filter_lock);
343 /* Decide if the new MAC is successfully activated only after
344 * querying the FW
345 */
346 status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac,
347 adapter->if_handle, true, 0);
348 if (status)
349 goto err;
350
351 /* The MAC change did not happen, either due to lack of privilege
352 * or PF didn't pre-provision.
353 */
354 if (!ether_addr_equal(addr->sa_data, mac)) {
355 status = -EPERM;
356 goto err;
357 }
358 done:
359 ether_addr_copy(adapter->dev_mac, addr->sa_data);
360 ether_addr_copy(netdev->dev_addr, addr->sa_data);
361 dev_info(dev, "MAC address changed to %pM\n", addr->sa_data);
362 return 0;
363 err:
364 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
365 return status;
366 }
367
368 /* BE2 supports only v0 cmd */
369 static void *hw_stats_from_cmd(struct be_adapter *adapter)
370 {
371 if (BE2_chip(adapter)) {
372 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
373
374 return &cmd->hw_stats;
375 } else if (BE3_chip(adapter)) {
376 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
377
378 return &cmd->hw_stats;
379 } else {
380 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
381
382 return &cmd->hw_stats;
383 }
384 }
385
386 /* BE2 supports only v0 cmd */
387 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
388 {
389 if (BE2_chip(adapter)) {
390 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
391
392 return &hw_stats->erx;
393 } else if (BE3_chip(adapter)) {
394 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
395
396 return &hw_stats->erx;
397 } else {
398 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
399
400 return &hw_stats->erx;
401 }
402 }
403
404 static void populate_be_v0_stats(struct be_adapter *adapter)
405 {
406 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
407 struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
408 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
409 struct be_port_rxf_stats_v0 *port_stats =
410 &rxf_stats->port[adapter->port_num];
411 struct be_drv_stats *drvs = &adapter->drv_stats;
412
413 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
414 drvs->rx_pause_frames = port_stats->rx_pause_frames;
415 drvs->rx_crc_errors = port_stats->rx_crc_errors;
416 drvs->rx_control_frames = port_stats->rx_control_frames;
417 drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
418 drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
419 drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
420 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
421 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
422 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
423 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
424 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
425 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
426 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
427 drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
428 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
429 drvs->rx_dropped_header_too_small =
430 port_stats->rx_dropped_header_too_small;
431 drvs->rx_address_filtered =
432 port_stats->rx_address_filtered +
433 port_stats->rx_vlan_filtered;
434 drvs->rx_alignment_symbol_errors =
435 port_stats->rx_alignment_symbol_errors;
436
437 drvs->tx_pauseframes = port_stats->tx_pauseframes;
438 drvs->tx_controlframes = port_stats->tx_controlframes;
439
440 if (adapter->port_num)
441 drvs->jabber_events = rxf_stats->port1_jabber_events;
442 else
443 drvs->jabber_events = rxf_stats->port0_jabber_events;
444 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
445 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
446 drvs->forwarded_packets = rxf_stats->forwarded_packets;
447 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
448 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
449 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
450 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
451 }
452
453 static void populate_be_v1_stats(struct be_adapter *adapter)
454 {
455 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
456 struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
457 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
458 struct be_port_rxf_stats_v1 *port_stats =
459 &rxf_stats->port[adapter->port_num];
460 struct be_drv_stats *drvs = &adapter->drv_stats;
461
462 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
463 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
464 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
465 drvs->rx_pause_frames = port_stats->rx_pause_frames;
466 drvs->rx_crc_errors = port_stats->rx_crc_errors;
467 drvs->rx_control_frames = port_stats->rx_control_frames;
468 drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
469 drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
470 drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
471 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
472 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
473 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
474 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
475 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
476 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
477 drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
478 drvs->rx_dropped_header_too_small =
479 port_stats->rx_dropped_header_too_small;
480 drvs->rx_input_fifo_overflow_drop =
481 port_stats->rx_input_fifo_overflow_drop;
482 drvs->rx_address_filtered = port_stats->rx_address_filtered;
483 drvs->rx_alignment_symbol_errors =
484 port_stats->rx_alignment_symbol_errors;
485 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
486 drvs->tx_pauseframes = port_stats->tx_pauseframes;
487 drvs->tx_controlframes = port_stats->tx_controlframes;
488 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
489 drvs->jabber_events = port_stats->jabber_events;
490 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
491 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
492 drvs->forwarded_packets = rxf_stats->forwarded_packets;
493 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
494 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
495 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
496 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
497 }
498
499 static void populate_be_v2_stats(struct be_adapter *adapter)
500 {
501 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
502 struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
503 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
504 struct be_port_rxf_stats_v2 *port_stats =
505 &rxf_stats->port[adapter->port_num];
506 struct be_drv_stats *drvs = &adapter->drv_stats;
507
508 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
509 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
510 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
511 drvs->rx_pause_frames = port_stats->rx_pause_frames;
512 drvs->rx_crc_errors = port_stats->rx_crc_errors;
513 drvs->rx_control_frames = port_stats->rx_control_frames;
514 drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
515 drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
516 drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
517 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
518 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
519 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
520 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
521 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
522 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
523 drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
524 drvs->rx_dropped_header_too_small =
525 port_stats->rx_dropped_header_too_small;
526 drvs->rx_input_fifo_overflow_drop =
527 port_stats->rx_input_fifo_overflow_drop;
528 drvs->rx_address_filtered = port_stats->rx_address_filtered;
529 drvs->rx_alignment_symbol_errors =
530 port_stats->rx_alignment_symbol_errors;
531 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
532 drvs->tx_pauseframes = port_stats->tx_pauseframes;
533 drvs->tx_controlframes = port_stats->tx_controlframes;
534 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
535 drvs->jabber_events = port_stats->jabber_events;
536 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
537 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
538 drvs->forwarded_packets = rxf_stats->forwarded_packets;
539 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
540 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
541 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
542 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
543 if (be_roce_supported(adapter)) {
544 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
545 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
546 drvs->rx_roce_frames = port_stats->roce_frames_received;
547 drvs->roce_drops_crc = port_stats->roce_drops_crc;
548 drvs->roce_drops_payload_len =
549 port_stats->roce_drops_payload_len;
550 }
551 }
552
553 static void populate_lancer_stats(struct be_adapter *adapter)
554 {
555 struct be_drv_stats *drvs = &adapter->drv_stats;
556 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
557
558 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
559 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
560 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
561 drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
562 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
563 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
564 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
565 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
566 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
567 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
568 drvs->rx_dropped_tcp_length =
569 pport_stats->rx_dropped_invalid_tcp_length;
570 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
571 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
572 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
573 drvs->rx_dropped_header_too_small =
574 pport_stats->rx_dropped_header_too_small;
575 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
576 drvs->rx_address_filtered =
577 pport_stats->rx_address_filtered +
578 pport_stats->rx_vlan_filtered;
579 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
580 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
581 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
582 drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
583 drvs->jabber_events = pport_stats->rx_jabbers;
584 drvs->forwarded_packets = pport_stats->num_forwards_lo;
585 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
586 drvs->rx_drops_too_many_frags =
587 pport_stats->rx_drops_too_many_frags_lo;
588 }
589
590 static void accumulate_16bit_val(u32 *acc, u16 val)
591 {
592 #define lo(x) (x & 0xFFFF)
593 #define hi(x) (x & 0xFFFF0000)
594 bool wrapped = val < lo(*acc);
595 u32 newacc = hi(*acc) + val;
596
597 if (wrapped)
598 newacc += 65536;
599 ACCESS_ONCE(*acc) = newacc;
600 }
601
602 static void populate_erx_stats(struct be_adapter *adapter,
603 struct be_rx_obj *rxo, u32 erx_stat)
604 {
605 if (!BEx_chip(adapter))
606 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
607 else
608 /* below erx HW counter can actually wrap around after
609 * 65535. Driver accumulates a 32-bit value
610 */
611 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
612 (u16)erx_stat);
613 }
614
615 void be_parse_stats(struct be_adapter *adapter)
616 {
617 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
618 struct be_rx_obj *rxo;
619 int i;
620 u32 erx_stat;
621
622 if (lancer_chip(adapter)) {
623 populate_lancer_stats(adapter);
624 } else {
625 if (BE2_chip(adapter))
626 populate_be_v0_stats(adapter);
627 else if (BE3_chip(adapter))
628 /* for BE3 */
629 populate_be_v1_stats(adapter);
630 else
631 populate_be_v2_stats(adapter);
632
633 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
634 for_all_rx_queues(adapter, rxo, i) {
635 erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
636 populate_erx_stats(adapter, rxo, erx_stat);
637 }
638 }
639 }
640
641 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
642 struct rtnl_link_stats64 *stats)
643 {
644 struct be_adapter *adapter = netdev_priv(netdev);
645 struct be_drv_stats *drvs = &adapter->drv_stats;
646 struct be_rx_obj *rxo;
647 struct be_tx_obj *txo;
648 u64 pkts, bytes;
649 unsigned int start;
650 int i;
651
652 for_all_rx_queues(adapter, rxo, i) {
653 const struct be_rx_stats *rx_stats = rx_stats(rxo);
654
655 do {
656 start = u64_stats_fetch_begin_irq(&rx_stats->sync);
657 pkts = rx_stats(rxo)->rx_pkts;
658 bytes = rx_stats(rxo)->rx_bytes;
659 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
660 stats->rx_packets += pkts;
661 stats->rx_bytes += bytes;
662 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
663 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
664 rx_stats(rxo)->rx_drops_no_frags;
665 }
666
667 for_all_tx_queues(adapter, txo, i) {
668 const struct be_tx_stats *tx_stats = tx_stats(txo);
669
670 do {
671 start = u64_stats_fetch_begin_irq(&tx_stats->sync);
672 pkts = tx_stats(txo)->tx_pkts;
673 bytes = tx_stats(txo)->tx_bytes;
674 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
675 stats->tx_packets += pkts;
676 stats->tx_bytes += bytes;
677 }
678
679 /* bad pkts received */
680 stats->rx_errors = drvs->rx_crc_errors +
681 drvs->rx_alignment_symbol_errors +
682 drvs->rx_in_range_errors +
683 drvs->rx_out_range_errors +
684 drvs->rx_frame_too_long +
685 drvs->rx_dropped_too_small +
686 drvs->rx_dropped_too_short +
687 drvs->rx_dropped_header_too_small +
688 drvs->rx_dropped_tcp_length +
689 drvs->rx_dropped_runt;
690
691 /* detailed rx errors */
692 stats->rx_length_errors = drvs->rx_in_range_errors +
693 drvs->rx_out_range_errors +
694 drvs->rx_frame_too_long;
695
696 stats->rx_crc_errors = drvs->rx_crc_errors;
697
698 /* frame alignment errors */
699 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
700
701 /* receiver fifo overrun */
702 /* drops_no_pbuf is no per i/f, it's per BE card */
703 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
704 drvs->rx_input_fifo_overflow_drop +
705 drvs->rx_drops_no_pbuf;
706 return stats;
707 }
708
709 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
710 {
711 struct net_device *netdev = adapter->netdev;
712
713 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
714 netif_carrier_off(netdev);
715 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
716 }
717
718 if (link_status)
719 netif_carrier_on(netdev);
720 else
721 netif_carrier_off(netdev);
722
723 netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down");
724 }
725
726 static int be_gso_hdr_len(struct sk_buff *skb)
727 {
728 if (skb->encapsulation)
729 return skb_inner_transport_offset(skb) +
730 inner_tcp_hdrlen(skb);
731 return skb_transport_offset(skb) + tcp_hdrlen(skb);
732 }
733
734 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
735 {
736 struct be_tx_stats *stats = tx_stats(txo);
737 u32 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1;
738 /* Account for headers which get duplicated in TSO pkt */
739 u32 dup_hdr_len = tx_pkts > 1 ? be_gso_hdr_len(skb) * (tx_pkts - 1) : 0;
740
741 u64_stats_update_begin(&stats->sync);
742 stats->tx_reqs++;
743 stats->tx_bytes += skb->len + dup_hdr_len;
744 stats->tx_pkts += tx_pkts;
745 if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL)
746 stats->tx_vxlan_offload_pkts += tx_pkts;
747 u64_stats_update_end(&stats->sync);
748 }
749
750 /* Returns number of WRBs needed for the skb */
751 static u32 skb_wrb_cnt(struct sk_buff *skb)
752 {
753 /* +1 for the header wrb */
754 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
755 }
756
757 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
758 {
759 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
760 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
761 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
762 wrb->rsvd0 = 0;
763 }
764
765 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
766 * to avoid the swap and shift/mask operations in wrb_fill().
767 */
768 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
769 {
770 wrb->frag_pa_hi = 0;
771 wrb->frag_pa_lo = 0;
772 wrb->frag_len = 0;
773 wrb->rsvd0 = 0;
774 }
775
776 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
777 struct sk_buff *skb)
778 {
779 u8 vlan_prio;
780 u16 vlan_tag;
781
782 vlan_tag = skb_vlan_tag_get(skb);
783 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
784 /* If vlan priority provided by OS is NOT in available bmap */
785 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
786 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
787 adapter->recommended_prio_bits;
788
789 return vlan_tag;
790 }
791
792 /* Used only for IP tunnel packets */
793 static u16 skb_inner_ip_proto(struct sk_buff *skb)
794 {
795 return (inner_ip_hdr(skb)->version == 4) ?
796 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
797 }
798
799 static u16 skb_ip_proto(struct sk_buff *skb)
800 {
801 return (ip_hdr(skb)->version == 4) ?
802 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
803 }
804
805 static inline bool be_is_txq_full(struct be_tx_obj *txo)
806 {
807 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;
808 }
809
810 static inline bool be_can_txq_wake(struct be_tx_obj *txo)
811 {
812 return atomic_read(&txo->q.used) < txo->q.len / 2;
813 }
814
815 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)
816 {
817 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;
818 }
819
820 static void be_get_wrb_params_from_skb(struct be_adapter *adapter,
821 struct sk_buff *skb,
822 struct be_wrb_params *wrb_params)
823 {
824 u16 proto;
825
826 if (skb_is_gso(skb)) {
827 BE_WRB_F_SET(wrb_params->features, LSO, 1);
828 wrb_params->lso_mss = skb_shinfo(skb)->gso_size;
829 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
830 BE_WRB_F_SET(wrb_params->features, LSO6, 1);
831 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
832 if (skb->encapsulation) {
833 BE_WRB_F_SET(wrb_params->features, IPCS, 1);
834 proto = skb_inner_ip_proto(skb);
835 } else {
836 proto = skb_ip_proto(skb);
837 }
838 if (proto == IPPROTO_TCP)
839 BE_WRB_F_SET(wrb_params->features, TCPCS, 1);
840 else if (proto == IPPROTO_UDP)
841 BE_WRB_F_SET(wrb_params->features, UDPCS, 1);
842 }
843
844 if (skb_vlan_tag_present(skb)) {
845 BE_WRB_F_SET(wrb_params->features, VLAN, 1);
846 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb);
847 }
848
849 BE_WRB_F_SET(wrb_params->features, CRC, 1);
850 }
851
852 static void wrb_fill_hdr(struct be_adapter *adapter,
853 struct be_eth_hdr_wrb *hdr,
854 struct be_wrb_params *wrb_params,
855 struct sk_buff *skb)
856 {
857 memset(hdr, 0, sizeof(*hdr));
858
859 SET_TX_WRB_HDR_BITS(crc, hdr,
860 BE_WRB_F_GET(wrb_params->features, CRC));
861 SET_TX_WRB_HDR_BITS(ipcs, hdr,
862 BE_WRB_F_GET(wrb_params->features, IPCS));
863 SET_TX_WRB_HDR_BITS(tcpcs, hdr,
864 BE_WRB_F_GET(wrb_params->features, TCPCS));
865 SET_TX_WRB_HDR_BITS(udpcs, hdr,
866 BE_WRB_F_GET(wrb_params->features, UDPCS));
867
868 SET_TX_WRB_HDR_BITS(lso, hdr,
869 BE_WRB_F_GET(wrb_params->features, LSO));
870 SET_TX_WRB_HDR_BITS(lso6, hdr,
871 BE_WRB_F_GET(wrb_params->features, LSO6));
872 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss);
873
874 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this
875 * hack is not needed, the evt bit is set while ringing DB.
876 */
877 SET_TX_WRB_HDR_BITS(event, hdr,
878 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW));
879 SET_TX_WRB_HDR_BITS(vlan, hdr,
880 BE_WRB_F_GET(wrb_params->features, VLAN));
881 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag);
882
883 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb));
884 SET_TX_WRB_HDR_BITS(len, hdr, skb->len);
885 SET_TX_WRB_HDR_BITS(mgmt, hdr,
886 BE_WRB_F_GET(wrb_params->features, OS2BMC));
887 }
888
889 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
890 bool unmap_single)
891 {
892 dma_addr_t dma;
893 u32 frag_len = le32_to_cpu(wrb->frag_len);
894
895
896 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
897 (u64)le32_to_cpu(wrb->frag_pa_lo);
898 if (frag_len) {
899 if (unmap_single)
900 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
901 else
902 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
903 }
904 }
905
906 /* Grab a WRB header for xmit */
907 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo)
908 {
909 u32 head = txo->q.head;
910
911 queue_head_inc(&txo->q);
912 return head;
913 }
914
915 /* Set up the WRB header for xmit */
916 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter,
917 struct be_tx_obj *txo,
918 struct be_wrb_params *wrb_params,
919 struct sk_buff *skb, u16 head)
920 {
921 u32 num_frags = skb_wrb_cnt(skb);
922 struct be_queue_info *txq = &txo->q;
923 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head);
924
925 wrb_fill_hdr(adapter, hdr, wrb_params, skb);
926 be_dws_cpu_to_le(hdr, sizeof(*hdr));
927
928 BUG_ON(txo->sent_skb_list[head]);
929 txo->sent_skb_list[head] = skb;
930 txo->last_req_hdr = head;
931 atomic_add(num_frags, &txq->used);
932 txo->last_req_wrb_cnt = num_frags;
933 txo->pend_wrb_cnt += num_frags;
934 }
935
936 /* Setup a WRB fragment (buffer descriptor) for xmit */
937 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,
938 int len)
939 {
940 struct be_eth_wrb *wrb;
941 struct be_queue_info *txq = &txo->q;
942
943 wrb = queue_head_node(txq);
944 wrb_fill(wrb, busaddr, len);
945 queue_head_inc(txq);
946 }
947
948 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine
949 * was invoked. The producer index is restored to the previous packet and the
950 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors.
951 */
952 static void be_xmit_restore(struct be_adapter *adapter,
953 struct be_tx_obj *txo, u32 head, bool map_single,
954 u32 copied)
955 {
956 struct device *dev;
957 struct be_eth_wrb *wrb;
958 struct be_queue_info *txq = &txo->q;
959
960 dev = &adapter->pdev->dev;
961 txq->head = head;
962
963 /* skip the first wrb (hdr); it's not mapped */
964 queue_head_inc(txq);
965 while (copied) {
966 wrb = queue_head_node(txq);
967 unmap_tx_frag(dev, wrb, map_single);
968 map_single = false;
969 copied -= le32_to_cpu(wrb->frag_len);
970 queue_head_inc(txq);
971 }
972
973 txq->head = head;
974 }
975
976 /* Enqueue the given packet for transmit. This routine allocates WRBs for the
977 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number
978 * of WRBs used up by the packet.
979 */
980 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
981 struct sk_buff *skb,
982 struct be_wrb_params *wrb_params)
983 {
984 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
985 struct device *dev = &adapter->pdev->dev;
986 struct be_queue_info *txq = &txo->q;
987 bool map_single = false;
988 u32 head = txq->head;
989 dma_addr_t busaddr;
990 int len;
991
992 head = be_tx_get_wrb_hdr(txo);
993
994 if (skb->len > skb->data_len) {
995 len = skb_headlen(skb);
996
997 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
998 if (dma_mapping_error(dev, busaddr))
999 goto dma_err;
1000 map_single = true;
1001 be_tx_setup_wrb_frag(txo, busaddr, len);
1002 copied += len;
1003 }
1004
1005 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1006 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
1007 len = skb_frag_size(frag);
1008
1009 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
1010 if (dma_mapping_error(dev, busaddr))
1011 goto dma_err;
1012 be_tx_setup_wrb_frag(txo, busaddr, len);
1013 copied += len;
1014 }
1015
1016 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);
1017
1018 be_tx_stats_update(txo, skb);
1019 return wrb_cnt;
1020
1021 dma_err:
1022 adapter->drv_stats.dma_map_errors++;
1023 be_xmit_restore(adapter, txo, head, map_single, copied);
1024 return 0;
1025 }
1026
1027 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
1028 {
1029 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
1030 }
1031
1032 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
1033 struct sk_buff *skb,
1034 struct be_wrb_params
1035 *wrb_params)
1036 {
1037 u16 vlan_tag = 0;
1038
1039 skb = skb_share_check(skb, GFP_ATOMIC);
1040 if (unlikely(!skb))
1041 return skb;
1042
1043 if (skb_vlan_tag_present(skb))
1044 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
1045
1046 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
1047 if (!vlan_tag)
1048 vlan_tag = adapter->pvid;
1049 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
1050 * skip VLAN insertion
1051 */
1052 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1053 }
1054
1055 if (vlan_tag) {
1056 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
1057 vlan_tag);
1058 if (unlikely(!skb))
1059 return skb;
1060 skb->vlan_tci = 0;
1061 }
1062
1063 /* Insert the outer VLAN, if any */
1064 if (adapter->qnq_vid) {
1065 vlan_tag = adapter->qnq_vid;
1066 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
1067 vlan_tag);
1068 if (unlikely(!skb))
1069 return skb;
1070 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1071 }
1072
1073 return skb;
1074 }
1075
1076 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
1077 {
1078 struct ethhdr *eh = (struct ethhdr *)skb->data;
1079 u16 offset = ETH_HLEN;
1080
1081 if (eh->h_proto == htons(ETH_P_IPV6)) {
1082 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
1083
1084 offset += sizeof(struct ipv6hdr);
1085 if (ip6h->nexthdr != NEXTHDR_TCP &&
1086 ip6h->nexthdr != NEXTHDR_UDP) {
1087 struct ipv6_opt_hdr *ehdr =
1088 (struct ipv6_opt_hdr *)(skb->data + offset);
1089
1090 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
1091 if (ehdr->hdrlen == 0xff)
1092 return true;
1093 }
1094 }
1095 return false;
1096 }
1097
1098 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
1099 {
1100 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
1101 }
1102
1103 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
1104 {
1105 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
1106 }
1107
1108 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
1109 struct sk_buff *skb,
1110 struct be_wrb_params
1111 *wrb_params)
1112 {
1113 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1114 unsigned int eth_hdr_len;
1115 struct iphdr *ip;
1116
1117 /* For padded packets, BE HW modifies tot_len field in IP header
1118 * incorrecly when VLAN tag is inserted by HW.
1119 * For padded packets, Lancer computes incorrect checksum.
1120 */
1121 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
1122 VLAN_ETH_HLEN : ETH_HLEN;
1123 if (skb->len <= 60 &&
1124 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
1125 is_ipv4_pkt(skb)) {
1126 ip = (struct iphdr *)ip_hdr(skb);
1127 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
1128 }
1129
1130 /* If vlan tag is already inlined in the packet, skip HW VLAN
1131 * tagging in pvid-tagging mode
1132 */
1133 if (be_pvid_tagging_enabled(adapter) &&
1134 veh->h_vlan_proto == htons(ETH_P_8021Q))
1135 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1136
1137 /* HW has a bug wherein it will calculate CSUM for VLAN
1138 * pkts even though it is disabled.
1139 * Manually insert VLAN in pkt.
1140 */
1141 if (skb->ip_summed != CHECKSUM_PARTIAL &&
1142 skb_vlan_tag_present(skb)) {
1143 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1144 if (unlikely(!skb))
1145 goto err;
1146 }
1147
1148 /* HW may lockup when VLAN HW tagging is requested on
1149 * certain ipv6 packets. Drop such pkts if the HW workaround to
1150 * skip HW tagging is not enabled by FW.
1151 */
1152 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
1153 (adapter->pvid || adapter->qnq_vid) &&
1154 !qnq_async_evt_rcvd(adapter)))
1155 goto tx_drop;
1156
1157 /* Manual VLAN tag insertion to prevent:
1158 * ASIC lockup when the ASIC inserts VLAN tag into
1159 * certain ipv6 packets. Insert VLAN tags in driver,
1160 * and set event, completion, vlan bits accordingly
1161 * in the Tx WRB.
1162 */
1163 if (be_ipv6_tx_stall_chk(adapter, skb) &&
1164 be_vlan_tag_tx_chk(adapter, skb)) {
1165 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1166 if (unlikely(!skb))
1167 goto err;
1168 }
1169
1170 return skb;
1171 tx_drop:
1172 dev_kfree_skb_any(skb);
1173 err:
1174 return NULL;
1175 }
1176
1177 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1178 struct sk_buff *skb,
1179 struct be_wrb_params *wrb_params)
1180 {
1181 int err;
1182
1183 /* Lancer, SH and BE3 in SRIOV mode have a bug wherein
1184 * packets that are 32b or less may cause a transmit stall
1185 * on that port. The workaround is to pad such packets
1186 * (len <= 32 bytes) to a minimum length of 36b.
1187 */
1188 if (skb->len <= 32) {
1189 if (skb_put_padto(skb, 36))
1190 return NULL;
1191 }
1192
1193 if (BEx_chip(adapter) || lancer_chip(adapter)) {
1194 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params);
1195 if (!skb)
1196 return NULL;
1197 }
1198
1199 /* The stack can send us skbs with length greater than
1200 * what the HW can handle. Trim the extra bytes.
1201 */
1202 WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE);
1203 err = pskb_trim(skb, BE_MAX_GSO_SIZE);
1204 WARN_ON(err);
1205
1206 return skb;
1207 }
1208
1209 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1210 {
1211 struct be_queue_info *txq = &txo->q;
1212 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1213
1214 /* Mark the last request eventable if it hasn't been marked already */
1215 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1216 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1217
1218 /* compose a dummy wrb if there are odd set of wrbs to notify */
1219 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1220 wrb_fill_dummy(queue_head_node(txq));
1221 queue_head_inc(txq);
1222 atomic_inc(&txq->used);
1223 txo->pend_wrb_cnt++;
1224 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1225 TX_HDR_WRB_NUM_SHIFT);
1226 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1227 TX_HDR_WRB_NUM_SHIFT);
1228 }
1229 be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1230 txo->pend_wrb_cnt = 0;
1231 }
1232
1233 /* OS2BMC related */
1234
1235 #define DHCP_CLIENT_PORT 68
1236 #define DHCP_SERVER_PORT 67
1237 #define NET_BIOS_PORT1 137
1238 #define NET_BIOS_PORT2 138
1239 #define DHCPV6_RAS_PORT 547
1240
1241 #define is_mc_allowed_on_bmc(adapter, eh) \
1242 (!is_multicast_filt_enabled(adapter) && \
1243 is_multicast_ether_addr(eh->h_dest) && \
1244 !is_broadcast_ether_addr(eh->h_dest))
1245
1246 #define is_bc_allowed_on_bmc(adapter, eh) \
1247 (!is_broadcast_filt_enabled(adapter) && \
1248 is_broadcast_ether_addr(eh->h_dest))
1249
1250 #define is_arp_allowed_on_bmc(adapter, skb) \
1251 (is_arp(skb) && is_arp_filt_enabled(adapter))
1252
1253 #define is_broadcast_packet(eh, adapter) \
1254 (is_multicast_ether_addr(eh->h_dest) && \
1255 !compare_ether_addr(eh->h_dest, adapter->netdev->broadcast))
1256
1257 #define is_arp(skb) (skb->protocol == htons(ETH_P_ARP))
1258
1259 #define is_arp_filt_enabled(adapter) \
1260 (adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP))
1261
1262 #define is_dhcp_client_filt_enabled(adapter) \
1263 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT)
1264
1265 #define is_dhcp_srvr_filt_enabled(adapter) \
1266 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER)
1267
1268 #define is_nbios_filt_enabled(adapter) \
1269 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS)
1270
1271 #define is_ipv6_na_filt_enabled(adapter) \
1272 (adapter->bmc_filt_mask & \
1273 BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER)
1274
1275 #define is_ipv6_ra_filt_enabled(adapter) \
1276 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA)
1277
1278 #define is_ipv6_ras_filt_enabled(adapter) \
1279 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS)
1280
1281 #define is_broadcast_filt_enabled(adapter) \
1282 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST)
1283
1284 #define is_multicast_filt_enabled(adapter) \
1285 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST)
1286
1287 static bool be_send_pkt_to_bmc(struct be_adapter *adapter,
1288 struct sk_buff **skb)
1289 {
1290 struct ethhdr *eh = (struct ethhdr *)(*skb)->data;
1291 bool os2bmc = false;
1292
1293 if (!be_is_os2bmc_enabled(adapter))
1294 goto done;
1295
1296 if (!is_multicast_ether_addr(eh->h_dest))
1297 goto done;
1298
1299 if (is_mc_allowed_on_bmc(adapter, eh) ||
1300 is_bc_allowed_on_bmc(adapter, eh) ||
1301 is_arp_allowed_on_bmc(adapter, (*skb))) {
1302 os2bmc = true;
1303 goto done;
1304 }
1305
1306 if ((*skb)->protocol == htons(ETH_P_IPV6)) {
1307 struct ipv6hdr *hdr = ipv6_hdr((*skb));
1308 u8 nexthdr = hdr->nexthdr;
1309
1310 if (nexthdr == IPPROTO_ICMPV6) {
1311 struct icmp6hdr *icmp6 = icmp6_hdr((*skb));
1312
1313 switch (icmp6->icmp6_type) {
1314 case NDISC_ROUTER_ADVERTISEMENT:
1315 os2bmc = is_ipv6_ra_filt_enabled(adapter);
1316 goto done;
1317 case NDISC_NEIGHBOUR_ADVERTISEMENT:
1318 os2bmc = is_ipv6_na_filt_enabled(adapter);
1319 goto done;
1320 default:
1321 break;
1322 }
1323 }
1324 }
1325
1326 if (is_udp_pkt((*skb))) {
1327 struct udphdr *udp = udp_hdr((*skb));
1328
1329 switch (ntohs(udp->dest)) {
1330 case DHCP_CLIENT_PORT:
1331 os2bmc = is_dhcp_client_filt_enabled(adapter);
1332 goto done;
1333 case DHCP_SERVER_PORT:
1334 os2bmc = is_dhcp_srvr_filt_enabled(adapter);
1335 goto done;
1336 case NET_BIOS_PORT1:
1337 case NET_BIOS_PORT2:
1338 os2bmc = is_nbios_filt_enabled(adapter);
1339 goto done;
1340 case DHCPV6_RAS_PORT:
1341 os2bmc = is_ipv6_ras_filt_enabled(adapter);
1342 goto done;
1343 default:
1344 break;
1345 }
1346 }
1347 done:
1348 /* For packets over a vlan, which are destined
1349 * to BMC, asic expects the vlan to be inline in the packet.
1350 */
1351 if (os2bmc)
1352 *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL);
1353
1354 return os2bmc;
1355 }
1356
1357 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1358 {
1359 struct be_adapter *adapter = netdev_priv(netdev);
1360 u16 q_idx = skb_get_queue_mapping(skb);
1361 struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1362 struct be_wrb_params wrb_params = { 0 };
1363 bool flush = !skb->xmit_more;
1364 u16 wrb_cnt;
1365
1366 skb = be_xmit_workarounds(adapter, skb, &wrb_params);
1367 if (unlikely(!skb))
1368 goto drop;
1369
1370 be_get_wrb_params_from_skb(adapter, skb, &wrb_params);
1371
1372 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1373 if (unlikely(!wrb_cnt)) {
1374 dev_kfree_skb_any(skb);
1375 goto drop;
1376 }
1377
1378 /* if os2bmc is enabled and if the pkt is destined to bmc,
1379 * enqueue the pkt a 2nd time with mgmt bit set.
1380 */
1381 if (be_send_pkt_to_bmc(adapter, &skb)) {
1382 BE_WRB_F_SET(wrb_params.features, OS2BMC, 1);
1383 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1384 if (unlikely(!wrb_cnt))
1385 goto drop;
1386 else
1387 skb_get(skb);
1388 }
1389
1390 if (be_is_txq_full(txo)) {
1391 netif_stop_subqueue(netdev, q_idx);
1392 tx_stats(txo)->tx_stops++;
1393 }
1394
1395 if (flush || __netif_subqueue_stopped(netdev, q_idx))
1396 be_xmit_flush(adapter, txo);
1397
1398 return NETDEV_TX_OK;
1399 drop:
1400 tx_stats(txo)->tx_drv_drops++;
1401 /* Flush the already enqueued tx requests */
1402 if (flush && txo->pend_wrb_cnt)
1403 be_xmit_flush(adapter, txo);
1404
1405 return NETDEV_TX_OK;
1406 }
1407
1408 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1409 {
1410 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1411 BE_IF_FLAGS_ALL_PROMISCUOUS;
1412 }
1413
1414 static int be_set_vlan_promisc(struct be_adapter *adapter)
1415 {
1416 struct device *dev = &adapter->pdev->dev;
1417 int status;
1418
1419 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1420 return 0;
1421
1422 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1423 if (!status) {
1424 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1425 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1426 } else {
1427 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1428 }
1429 return status;
1430 }
1431
1432 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1433 {
1434 struct device *dev = &adapter->pdev->dev;
1435 int status;
1436
1437 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1438 if (!status) {
1439 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1440 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1441 }
1442 return status;
1443 }
1444
1445 /*
1446 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1447 * If the user configures more, place BE in vlan promiscuous mode.
1448 */
1449 static int be_vid_config(struct be_adapter *adapter)
1450 {
1451 struct device *dev = &adapter->pdev->dev;
1452 u16 vids[BE_NUM_VLANS_SUPPORTED];
1453 u16 num = 0, i = 0;
1454 int status = 0;
1455
1456 /* No need to change the VLAN state if the I/F is in promiscuous */
1457 if (adapter->netdev->flags & IFF_PROMISC)
1458 return 0;
1459
1460 if (adapter->vlans_added > be_max_vlans(adapter))
1461 return be_set_vlan_promisc(adapter);
1462
1463 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1464 status = be_clear_vlan_promisc(adapter);
1465 if (status)
1466 return status;
1467 }
1468 /* Construct VLAN Table to give to HW */
1469 for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1470 vids[num++] = cpu_to_le16(i);
1471
1472 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1473 if (status) {
1474 dev_err(dev, "Setting HW VLAN filtering failed\n");
1475 /* Set to VLAN promisc mode as setting VLAN filter failed */
1476 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS ||
1477 addl_status(status) ==
1478 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1479 return be_set_vlan_promisc(adapter);
1480 }
1481 return status;
1482 }
1483
1484 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1485 {
1486 struct be_adapter *adapter = netdev_priv(netdev);
1487 int status = 0;
1488
1489 mutex_lock(&adapter->rx_filter_lock);
1490
1491 /* Packets with VID 0 are always received by Lancer by default */
1492 if (lancer_chip(adapter) && vid == 0)
1493 goto done;
1494
1495 if (test_bit(vid, adapter->vids))
1496 goto done;
1497
1498 set_bit(vid, adapter->vids);
1499 adapter->vlans_added++;
1500
1501 status = be_vid_config(adapter);
1502 done:
1503 mutex_unlock(&adapter->rx_filter_lock);
1504 return status;
1505 }
1506
1507 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1508 {
1509 struct be_adapter *adapter = netdev_priv(netdev);
1510 int status = 0;
1511
1512 mutex_lock(&adapter->rx_filter_lock);
1513
1514 /* Packets with VID 0 are always received by Lancer by default */
1515 if (lancer_chip(adapter) && vid == 0)
1516 goto done;
1517
1518 if (!test_bit(vid, adapter->vids))
1519 goto done;
1520
1521 clear_bit(vid, adapter->vids);
1522 adapter->vlans_added--;
1523
1524 status = be_vid_config(adapter);
1525 done:
1526 mutex_unlock(&adapter->rx_filter_lock);
1527 return status;
1528 }
1529
1530 static void be_set_all_promisc(struct be_adapter *adapter)
1531 {
1532 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1533 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1534 }
1535
1536 static void be_set_mc_promisc(struct be_adapter *adapter)
1537 {
1538 int status;
1539
1540 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1541 return;
1542
1543 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1544 if (!status)
1545 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1546 }
1547
1548 static void be_set_uc_promisc(struct be_adapter *adapter)
1549 {
1550 int status;
1551
1552 if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)
1553 return;
1554
1555 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON);
1556 if (!status)
1557 adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS;
1558 }
1559
1560 static void be_clear_uc_promisc(struct be_adapter *adapter)
1561 {
1562 int status;
1563
1564 if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS))
1565 return;
1566
1567 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF);
1568 if (!status)
1569 adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS;
1570 }
1571
1572 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync().
1573 * We use a single callback function for both sync and unsync. We really don't
1574 * add/remove addresses through this callback. But, we use it to detect changes
1575 * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode().
1576 */
1577 static int be_uc_list_update(struct net_device *netdev,
1578 const unsigned char *addr)
1579 {
1580 struct be_adapter *adapter = netdev_priv(netdev);
1581
1582 adapter->update_uc_list = true;
1583 return 0;
1584 }
1585
1586 static int be_mc_list_update(struct net_device *netdev,
1587 const unsigned char *addr)
1588 {
1589 struct be_adapter *adapter = netdev_priv(netdev);
1590
1591 adapter->update_mc_list = true;
1592 return 0;
1593 }
1594
1595 static void be_set_mc_list(struct be_adapter *adapter)
1596 {
1597 struct net_device *netdev = adapter->netdev;
1598 struct netdev_hw_addr *ha;
1599 bool mc_promisc = false;
1600 int status;
1601
1602 netif_addr_lock_bh(netdev);
1603 __dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update);
1604
1605 if (netdev->flags & IFF_PROMISC) {
1606 adapter->update_mc_list = false;
1607 } else if (netdev->flags & IFF_ALLMULTI ||
1608 netdev_mc_count(netdev) > be_max_mc(adapter)) {
1609 /* Enable multicast promisc if num configured exceeds
1610 * what we support
1611 */
1612 mc_promisc = true;
1613 adapter->update_mc_list = false;
1614 } else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) {
1615 /* Update mc-list unconditionally if the iface was previously
1616 * in mc-promisc mode and now is out of that mode.
1617 */
1618 adapter->update_mc_list = true;
1619 }
1620
1621 if (adapter->update_mc_list) {
1622 int i = 0;
1623
1624 /* cache the mc-list in adapter */
1625 netdev_for_each_mc_addr(ha, netdev) {
1626 ether_addr_copy(adapter->mc_list[i].mac, ha->addr);
1627 i++;
1628 }
1629 adapter->mc_count = netdev_mc_count(netdev);
1630 }
1631 netif_addr_unlock_bh(netdev);
1632
1633 if (mc_promisc) {
1634 be_set_mc_promisc(adapter);
1635 } else if (adapter->update_mc_list) {
1636 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1637 if (!status)
1638 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1639 else
1640 be_set_mc_promisc(adapter);
1641
1642 adapter->update_mc_list = false;
1643 }
1644 }
1645
1646 static void be_clear_mc_list(struct be_adapter *adapter)
1647 {
1648 struct net_device *netdev = adapter->netdev;
1649
1650 __dev_mc_unsync(netdev, NULL);
1651 be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF);
1652 adapter->mc_count = 0;
1653 }
1654
1655 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx)
1656 {
1657 if (ether_addr_equal(adapter->uc_list[uc_idx].mac, adapter->dev_mac)) {
1658 adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0];
1659 return 0;
1660 }
1661
1662 return be_cmd_pmac_add(adapter, adapter->uc_list[uc_idx].mac,
1663 adapter->if_handle,
1664 &adapter->pmac_id[uc_idx + 1], 0);
1665 }
1666
1667 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id)
1668 {
1669 if (pmac_id == adapter->pmac_id[0])
1670 return;
1671
1672 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
1673 }
1674
1675 static void be_set_uc_list(struct be_adapter *adapter)
1676 {
1677 struct net_device *netdev = adapter->netdev;
1678 struct netdev_hw_addr *ha;
1679 bool uc_promisc = false;
1680 int curr_uc_macs = 0, i;
1681
1682 netif_addr_lock_bh(netdev);
1683 __dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update);
1684
1685 if (netdev->flags & IFF_PROMISC) {
1686 adapter->update_uc_list = false;
1687 } else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) {
1688 uc_promisc = true;
1689 adapter->update_uc_list = false;
1690 } else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) {
1691 /* Update uc-list unconditionally if the iface was previously
1692 * in uc-promisc mode and now is out of that mode.
1693 */
1694 adapter->update_uc_list = true;
1695 }
1696
1697 if (adapter->update_uc_list) {
1698 /* cache the uc-list in adapter array */
1699 i = 0;
1700 netdev_for_each_uc_addr(ha, netdev) {
1701 ether_addr_copy(adapter->uc_list[i].mac, ha->addr);
1702 i++;
1703 }
1704 curr_uc_macs = netdev_uc_count(netdev);
1705 }
1706 netif_addr_unlock_bh(netdev);
1707
1708 if (uc_promisc) {
1709 be_set_uc_promisc(adapter);
1710 } else if (adapter->update_uc_list) {
1711 be_clear_uc_promisc(adapter);
1712
1713 for (i = 0; i < adapter->uc_macs; i++)
1714 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
1715
1716 for (i = 0; i < curr_uc_macs; i++)
1717 be_uc_mac_add(adapter, i);
1718 adapter->uc_macs = curr_uc_macs;
1719 adapter->update_uc_list = false;
1720 }
1721 }
1722
1723 static void be_clear_uc_list(struct be_adapter *adapter)
1724 {
1725 struct net_device *netdev = adapter->netdev;
1726 int i;
1727
1728 __dev_uc_unsync(netdev, NULL);
1729 for (i = 0; i < adapter->uc_macs; i++)
1730 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
1731
1732 adapter->uc_macs = 0;
1733 }
1734
1735 static void __be_set_rx_mode(struct be_adapter *adapter)
1736 {
1737 struct net_device *netdev = adapter->netdev;
1738
1739 mutex_lock(&adapter->rx_filter_lock);
1740
1741 if (netdev->flags & IFF_PROMISC) {
1742 if (!be_in_all_promisc(adapter))
1743 be_set_all_promisc(adapter);
1744 } else if (be_in_all_promisc(adapter)) {
1745 /* We need to re-program the vlan-list or clear
1746 * vlan-promisc mode (if needed) when the interface
1747 * comes out of promisc mode.
1748 */
1749 be_vid_config(adapter);
1750 }
1751
1752 be_set_uc_list(adapter);
1753 be_set_mc_list(adapter);
1754
1755 mutex_unlock(&adapter->rx_filter_lock);
1756 }
1757
1758 static void be_work_set_rx_mode(struct work_struct *work)
1759 {
1760 struct be_cmd_work *cmd_work =
1761 container_of(work, struct be_cmd_work, work);
1762
1763 __be_set_rx_mode(cmd_work->adapter);
1764 kfree(cmd_work);
1765 }
1766
1767 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1768 {
1769 struct be_adapter *adapter = netdev_priv(netdev);
1770 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1771 int status;
1772
1773 if (!sriov_enabled(adapter))
1774 return -EPERM;
1775
1776 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1777 return -EINVAL;
1778
1779 /* Proceed further only if user provided MAC is different
1780 * from active MAC
1781 */
1782 if (ether_addr_equal(mac, vf_cfg->mac_addr))
1783 return 0;
1784
1785 if (BEx_chip(adapter)) {
1786 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1787 vf + 1);
1788
1789 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1790 &vf_cfg->pmac_id, vf + 1);
1791 } else {
1792 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1793 vf + 1);
1794 }
1795
1796 if (status) {
1797 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1798 mac, vf, status);
1799 return be_cmd_status(status);
1800 }
1801
1802 ether_addr_copy(vf_cfg->mac_addr, mac);
1803
1804 return 0;
1805 }
1806
1807 static int be_get_vf_config(struct net_device *netdev, int vf,
1808 struct ifla_vf_info *vi)
1809 {
1810 struct be_adapter *adapter = netdev_priv(netdev);
1811 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1812
1813 if (!sriov_enabled(adapter))
1814 return -EPERM;
1815
1816 if (vf >= adapter->num_vfs)
1817 return -EINVAL;
1818
1819 vi->vf = vf;
1820 vi->max_tx_rate = vf_cfg->tx_rate;
1821 vi->min_tx_rate = 0;
1822 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1823 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1824 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1825 vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1826 vi->spoofchk = adapter->vf_cfg[vf].spoofchk;
1827
1828 return 0;
1829 }
1830
1831 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1832 {
1833 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1834 u16 vids[BE_NUM_VLANS_SUPPORTED];
1835 int vf_if_id = vf_cfg->if_handle;
1836 int status;
1837
1838 /* Enable Transparent VLAN Tagging */
1839 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0);
1840 if (status)
1841 return status;
1842
1843 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1844 vids[0] = 0;
1845 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1846 if (!status)
1847 dev_info(&adapter->pdev->dev,
1848 "Cleared guest VLANs on VF%d", vf);
1849
1850 /* After TVT is enabled, disallow VFs to program VLAN filters */
1851 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1852 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1853 ~BE_PRIV_FILTMGMT, vf + 1);
1854 if (!status)
1855 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1856 }
1857 return 0;
1858 }
1859
1860 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1861 {
1862 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1863 struct device *dev = &adapter->pdev->dev;
1864 int status;
1865
1866 /* Reset Transparent VLAN Tagging. */
1867 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1868 vf_cfg->if_handle, 0, 0);
1869 if (status)
1870 return status;
1871
1872 /* Allow VFs to program VLAN filtering */
1873 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1874 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1875 BE_PRIV_FILTMGMT, vf + 1);
1876 if (!status) {
1877 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1878 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1879 }
1880 }
1881
1882 dev_info(dev,
1883 "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1884 return 0;
1885 }
1886
1887 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
1888 __be16 vlan_proto)
1889 {
1890 struct be_adapter *adapter = netdev_priv(netdev);
1891 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1892 int status;
1893
1894 if (!sriov_enabled(adapter))
1895 return -EPERM;
1896
1897 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1898 return -EINVAL;
1899
1900 if (vlan_proto != htons(ETH_P_8021Q))
1901 return -EPROTONOSUPPORT;
1902
1903 if (vlan || qos) {
1904 vlan |= qos << VLAN_PRIO_SHIFT;
1905 status = be_set_vf_tvt(adapter, vf, vlan);
1906 } else {
1907 status = be_clear_vf_tvt(adapter, vf);
1908 }
1909
1910 if (status) {
1911 dev_err(&adapter->pdev->dev,
1912 "VLAN %d config on VF %d failed : %#x\n", vlan, vf,
1913 status);
1914 return be_cmd_status(status);
1915 }
1916
1917 vf_cfg->vlan_tag = vlan;
1918 return 0;
1919 }
1920
1921 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1922 int min_tx_rate, int max_tx_rate)
1923 {
1924 struct be_adapter *adapter = netdev_priv(netdev);
1925 struct device *dev = &adapter->pdev->dev;
1926 int percent_rate, status = 0;
1927 u16 link_speed = 0;
1928 u8 link_status;
1929
1930 if (!sriov_enabled(adapter))
1931 return -EPERM;
1932
1933 if (vf >= adapter->num_vfs)
1934 return -EINVAL;
1935
1936 if (min_tx_rate)
1937 return -EINVAL;
1938
1939 if (!max_tx_rate)
1940 goto config_qos;
1941
1942 status = be_cmd_link_status_query(adapter, &link_speed,
1943 &link_status, 0);
1944 if (status)
1945 goto err;
1946
1947 if (!link_status) {
1948 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1949 status = -ENETDOWN;
1950 goto err;
1951 }
1952
1953 if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1954 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1955 link_speed);
1956 status = -EINVAL;
1957 goto err;
1958 }
1959
1960 /* On Skyhawk the QOS setting must be done only as a % value */
1961 percent_rate = link_speed / 100;
1962 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1963 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1964 percent_rate);
1965 status = -EINVAL;
1966 goto err;
1967 }
1968
1969 config_qos:
1970 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1971 if (status)
1972 goto err;
1973
1974 adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1975 return 0;
1976
1977 err:
1978 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1979 max_tx_rate, vf);
1980 return be_cmd_status(status);
1981 }
1982
1983 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1984 int link_state)
1985 {
1986 struct be_adapter *adapter = netdev_priv(netdev);
1987 int status;
1988
1989 if (!sriov_enabled(adapter))
1990 return -EPERM;
1991
1992 if (vf >= adapter->num_vfs)
1993 return -EINVAL;
1994
1995 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1996 if (status) {
1997 dev_err(&adapter->pdev->dev,
1998 "Link state change on VF %d failed: %#x\n", vf, status);
1999 return be_cmd_status(status);
2000 }
2001
2002 adapter->vf_cfg[vf].plink_tracking = link_state;
2003
2004 return 0;
2005 }
2006
2007 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
2008 {
2009 struct be_adapter *adapter = netdev_priv(netdev);
2010 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
2011 u8 spoofchk;
2012 int status;
2013
2014 if (!sriov_enabled(adapter))
2015 return -EPERM;
2016
2017 if (vf >= adapter->num_vfs)
2018 return -EINVAL;
2019
2020 if (BEx_chip(adapter))
2021 return -EOPNOTSUPP;
2022
2023 if (enable == vf_cfg->spoofchk)
2024 return 0;
2025
2026 spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK;
2027
2028 status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle,
2029 0, spoofchk);
2030 if (status) {
2031 dev_err(&adapter->pdev->dev,
2032 "Spoofchk change on VF %d failed: %#x\n", vf, status);
2033 return be_cmd_status(status);
2034 }
2035
2036 vf_cfg->spoofchk = enable;
2037 return 0;
2038 }
2039
2040 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
2041 ulong now)
2042 {
2043 aic->rx_pkts_prev = rx_pkts;
2044 aic->tx_reqs_prev = tx_pkts;
2045 aic->jiffies = now;
2046 }
2047
2048 static int be_get_new_eqd(struct be_eq_obj *eqo)
2049 {
2050 struct be_adapter *adapter = eqo->adapter;
2051 int eqd, start;
2052 struct be_aic_obj *aic;
2053 struct be_rx_obj *rxo;
2054 struct be_tx_obj *txo;
2055 u64 rx_pkts = 0, tx_pkts = 0;
2056 ulong now;
2057 u32 pps, delta;
2058 int i;
2059
2060 aic = &adapter->aic_obj[eqo->idx];
2061 if (!aic->enable) {
2062 if (aic->jiffies)
2063 aic->jiffies = 0;
2064 eqd = aic->et_eqd;
2065 return eqd;
2066 }
2067
2068 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2069 do {
2070 start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
2071 rx_pkts += rxo->stats.rx_pkts;
2072 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
2073 }
2074
2075 for_all_tx_queues_on_eq(adapter, eqo, txo, i) {
2076 do {
2077 start = u64_stats_fetch_begin_irq(&txo->stats.sync);
2078 tx_pkts += txo->stats.tx_reqs;
2079 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
2080 }
2081
2082 /* Skip, if wrapped around or first calculation */
2083 now = jiffies;
2084 if (!aic->jiffies || time_before(now, aic->jiffies) ||
2085 rx_pkts < aic->rx_pkts_prev ||
2086 tx_pkts < aic->tx_reqs_prev) {
2087 be_aic_update(aic, rx_pkts, tx_pkts, now);
2088 return aic->prev_eqd;
2089 }
2090
2091 delta = jiffies_to_msecs(now - aic->jiffies);
2092 if (delta == 0)
2093 return aic->prev_eqd;
2094
2095 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
2096 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
2097 eqd = (pps / 15000) << 2;
2098
2099 if (eqd < 8)
2100 eqd = 0;
2101 eqd = min_t(u32, eqd, aic->max_eqd);
2102 eqd = max_t(u32, eqd, aic->min_eqd);
2103
2104 be_aic_update(aic, rx_pkts, tx_pkts, now);
2105
2106 return eqd;
2107 }
2108
2109 /* For Skyhawk-R only */
2110 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo)
2111 {
2112 struct be_adapter *adapter = eqo->adapter;
2113 struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx];
2114 ulong now = jiffies;
2115 int eqd;
2116 u32 mult_enc;
2117
2118 if (!aic->enable)
2119 return 0;
2120
2121 if (jiffies_to_msecs(now - aic->jiffies) < 1)
2122 eqd = aic->prev_eqd;
2123 else
2124 eqd = be_get_new_eqd(eqo);
2125
2126 if (eqd > 100)
2127 mult_enc = R2I_DLY_ENC_1;
2128 else if (eqd > 60)
2129 mult_enc = R2I_DLY_ENC_2;
2130 else if (eqd > 20)
2131 mult_enc = R2I_DLY_ENC_3;
2132 else
2133 mult_enc = R2I_DLY_ENC_0;
2134
2135 aic->prev_eqd = eqd;
2136
2137 return mult_enc;
2138 }
2139
2140 void be_eqd_update(struct be_adapter *adapter, bool force_update)
2141 {
2142 struct be_set_eqd set_eqd[MAX_EVT_QS];
2143 struct be_aic_obj *aic;
2144 struct be_eq_obj *eqo;
2145 int i, num = 0, eqd;
2146
2147 for_all_evt_queues(adapter, eqo, i) {
2148 aic = &adapter->aic_obj[eqo->idx];
2149 eqd = be_get_new_eqd(eqo);
2150 if (force_update || eqd != aic->prev_eqd) {
2151 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2152 set_eqd[num].eq_id = eqo->q.id;
2153 aic->prev_eqd = eqd;
2154 num++;
2155 }
2156 }
2157
2158 if (num)
2159 be_cmd_modify_eqd(adapter, set_eqd, num);
2160 }
2161
2162 static void be_rx_stats_update(struct be_rx_obj *rxo,
2163 struct be_rx_compl_info *rxcp)
2164 {
2165 struct be_rx_stats *stats = rx_stats(rxo);
2166
2167 u64_stats_update_begin(&stats->sync);
2168 stats->rx_compl++;
2169 stats->rx_bytes += rxcp->pkt_size;
2170 stats->rx_pkts++;
2171 if (rxcp->tunneled)
2172 stats->rx_vxlan_offload_pkts++;
2173 if (rxcp->pkt_type == BE_MULTICAST_PACKET)
2174 stats->rx_mcast_pkts++;
2175 if (rxcp->err)
2176 stats->rx_compl_err++;
2177 u64_stats_update_end(&stats->sync);
2178 }
2179
2180 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
2181 {
2182 /* L4 checksum is not reliable for non TCP/UDP packets.
2183 * Also ignore ipcksm for ipv6 pkts
2184 */
2185 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
2186 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
2187 }
2188
2189 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
2190 {
2191 struct be_adapter *adapter = rxo->adapter;
2192 struct be_rx_page_info *rx_page_info;
2193 struct be_queue_info *rxq = &rxo->q;
2194 u32 frag_idx = rxq->tail;
2195
2196 rx_page_info = &rxo->page_info_tbl[frag_idx];
2197 BUG_ON(!rx_page_info->page);
2198
2199 if (rx_page_info->last_frag) {
2200 dma_unmap_page(&adapter->pdev->dev,
2201 dma_unmap_addr(rx_page_info, bus),
2202 adapter->big_page_size, DMA_FROM_DEVICE);
2203 rx_page_info->last_frag = false;
2204 } else {
2205 dma_sync_single_for_cpu(&adapter->pdev->dev,
2206 dma_unmap_addr(rx_page_info, bus),
2207 rx_frag_size, DMA_FROM_DEVICE);
2208 }
2209
2210 queue_tail_inc(rxq);
2211 atomic_dec(&rxq->used);
2212 return rx_page_info;
2213 }
2214
2215 /* Throwaway the data in the Rx completion */
2216 static void be_rx_compl_discard(struct be_rx_obj *rxo,
2217 struct be_rx_compl_info *rxcp)
2218 {
2219 struct be_rx_page_info *page_info;
2220 u16 i, num_rcvd = rxcp->num_rcvd;
2221
2222 for (i = 0; i < num_rcvd; i++) {
2223 page_info = get_rx_page_info(rxo);
2224 put_page(page_info->page);
2225 memset(page_info, 0, sizeof(*page_info));
2226 }
2227 }
2228
2229 /*
2230 * skb_fill_rx_data forms a complete skb for an ether frame
2231 * indicated by rxcp.
2232 */
2233 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
2234 struct be_rx_compl_info *rxcp)
2235 {
2236 struct be_rx_page_info *page_info;
2237 u16 i, j;
2238 u16 hdr_len, curr_frag_len, remaining;
2239 u8 *start;
2240
2241 page_info = get_rx_page_info(rxo);
2242 start = page_address(page_info->page) + page_info->page_offset;
2243 prefetch(start);
2244
2245 /* Copy data in the first descriptor of this completion */
2246 curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
2247
2248 skb->len = curr_frag_len;
2249 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
2250 memcpy(skb->data, start, curr_frag_len);
2251 /* Complete packet has now been moved to data */
2252 put_page(page_info->page);
2253 skb->data_len = 0;
2254 skb->tail += curr_frag_len;
2255 } else {
2256 hdr_len = ETH_HLEN;
2257 memcpy(skb->data, start, hdr_len);
2258 skb_shinfo(skb)->nr_frags = 1;
2259 skb_frag_set_page(skb, 0, page_info->page);
2260 skb_shinfo(skb)->frags[0].page_offset =
2261 page_info->page_offset + hdr_len;
2262 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
2263 curr_frag_len - hdr_len);
2264 skb->data_len = curr_frag_len - hdr_len;
2265 skb->truesize += rx_frag_size;
2266 skb->tail += hdr_len;
2267 }
2268 page_info->page = NULL;
2269
2270 if (rxcp->pkt_size <= rx_frag_size) {
2271 BUG_ON(rxcp->num_rcvd != 1);
2272 return;
2273 }
2274
2275 /* More frags present for this completion */
2276 remaining = rxcp->pkt_size - curr_frag_len;
2277 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
2278 page_info = get_rx_page_info(rxo);
2279 curr_frag_len = min(remaining, rx_frag_size);
2280
2281 /* Coalesce all frags from the same physical page in one slot */
2282 if (page_info->page_offset == 0) {
2283 /* Fresh page */
2284 j++;
2285 skb_frag_set_page(skb, j, page_info->page);
2286 skb_shinfo(skb)->frags[j].page_offset =
2287 page_info->page_offset;
2288 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
2289 skb_shinfo(skb)->nr_frags++;
2290 } else {
2291 put_page(page_info->page);
2292 }
2293
2294 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
2295 skb->len += curr_frag_len;
2296 skb->data_len += curr_frag_len;
2297 skb->truesize += rx_frag_size;
2298 remaining -= curr_frag_len;
2299 page_info->page = NULL;
2300 }
2301 BUG_ON(j > MAX_SKB_FRAGS);
2302 }
2303
2304 /* Process the RX completion indicated by rxcp when GRO is disabled */
2305 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
2306 struct be_rx_compl_info *rxcp)
2307 {
2308 struct be_adapter *adapter = rxo->adapter;
2309 struct net_device *netdev = adapter->netdev;
2310 struct sk_buff *skb;
2311
2312 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
2313 if (unlikely(!skb)) {
2314 rx_stats(rxo)->rx_drops_no_skbs++;
2315 be_rx_compl_discard(rxo, rxcp);
2316 return;
2317 }
2318
2319 skb_fill_rx_data(rxo, skb, rxcp);
2320
2321 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
2322 skb->ip_summed = CHECKSUM_UNNECESSARY;
2323 else
2324 skb_checksum_none_assert(skb);
2325
2326 skb->protocol = eth_type_trans(skb, netdev);
2327 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
2328 if (netdev->features & NETIF_F_RXHASH)
2329 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
2330
2331 skb->csum_level = rxcp->tunneled;
2332 skb_mark_napi_id(skb, napi);
2333
2334 if (rxcp->vlanf)
2335 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
2336
2337 netif_receive_skb(skb);
2338 }
2339
2340 /* Process the RX completion indicated by rxcp when GRO is enabled */
2341 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
2342 struct napi_struct *napi,
2343 struct be_rx_compl_info *rxcp)
2344 {
2345 struct be_adapter *adapter = rxo->adapter;
2346 struct be_rx_page_info *page_info;
2347 struct sk_buff *skb = NULL;
2348 u16 remaining, curr_frag_len;
2349 u16 i, j;
2350
2351 skb = napi_get_frags(napi);
2352 if (!skb) {
2353 be_rx_compl_discard(rxo, rxcp);
2354 return;
2355 }
2356
2357 remaining = rxcp->pkt_size;
2358 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
2359 page_info = get_rx_page_info(rxo);
2360
2361 curr_frag_len = min(remaining, rx_frag_size);
2362
2363 /* Coalesce all frags from the same physical page in one slot */
2364 if (i == 0 || page_info->page_offset == 0) {
2365 /* First frag or Fresh page */
2366 j++;
2367 skb_frag_set_page(skb, j, page_info->page);
2368 skb_shinfo(skb)->frags[j].page_offset =
2369 page_info->page_offset;
2370 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
2371 } else {
2372 put_page(page_info->page);
2373 }
2374 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
2375 skb->truesize += rx_frag_size;
2376 remaining -= curr_frag_len;
2377 memset(page_info, 0, sizeof(*page_info));
2378 }
2379 BUG_ON(j > MAX_SKB_FRAGS);
2380
2381 skb_shinfo(skb)->nr_frags = j + 1;
2382 skb->len = rxcp->pkt_size;
2383 skb->data_len = rxcp->pkt_size;
2384 skb->ip_summed = CHECKSUM_UNNECESSARY;
2385 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
2386 if (adapter->netdev->features & NETIF_F_RXHASH)
2387 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
2388
2389 skb->csum_level = rxcp->tunneled;
2390
2391 if (rxcp->vlanf)
2392 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
2393
2394 napi_gro_frags(napi);
2395 }
2396
2397 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
2398 struct be_rx_compl_info *rxcp)
2399 {
2400 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
2401 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
2402 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
2403 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
2404 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
2405 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
2406 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
2407 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
2408 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
2409 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
2410 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
2411 if (rxcp->vlanf) {
2412 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
2413 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
2414 }
2415 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
2416 rxcp->tunneled =
2417 GET_RX_COMPL_V1_BITS(tunneled, compl);
2418 }
2419
2420 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
2421 struct be_rx_compl_info *rxcp)
2422 {
2423 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
2424 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
2425 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
2426 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
2427 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
2428 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
2429 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
2430 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
2431 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
2432 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
2433 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
2434 if (rxcp->vlanf) {
2435 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
2436 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
2437 }
2438 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
2439 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
2440 }
2441
2442 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
2443 {
2444 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
2445 struct be_rx_compl_info *rxcp = &rxo->rxcp;
2446 struct be_adapter *adapter = rxo->adapter;
2447
2448 /* For checking the valid bit it is Ok to use either definition as the
2449 * valid bit is at the same position in both v0 and v1 Rx compl */
2450 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
2451 return NULL;
2452
2453 rmb();
2454 be_dws_le_to_cpu(compl, sizeof(*compl));
2455
2456 if (adapter->be3_native)
2457 be_parse_rx_compl_v1(compl, rxcp);
2458 else
2459 be_parse_rx_compl_v0(compl, rxcp);
2460
2461 if (rxcp->ip_frag)
2462 rxcp->l4_csum = 0;
2463
2464 if (rxcp->vlanf) {
2465 /* In QNQ modes, if qnq bit is not set, then the packet was
2466 * tagged only with the transparent outer vlan-tag and must
2467 * not be treated as a vlan packet by host
2468 */
2469 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
2470 rxcp->vlanf = 0;
2471
2472 if (!lancer_chip(adapter))
2473 rxcp->vlan_tag = swab16(rxcp->vlan_tag);
2474
2475 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
2476 !test_bit(rxcp->vlan_tag, adapter->vids))
2477 rxcp->vlanf = 0;
2478 }
2479
2480 /* As the compl has been parsed, reset it; we wont touch it again */
2481 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
2482
2483 queue_tail_inc(&rxo->cq);
2484 return rxcp;
2485 }
2486
2487 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
2488 {
2489 u32 order = get_order(size);
2490
2491 if (order > 0)
2492 gfp |= __GFP_COMP;
2493 return alloc_pages(gfp, order);
2494 }
2495
2496 /*
2497 * Allocate a page, split it to fragments of size rx_frag_size and post as
2498 * receive buffers to BE
2499 */
2500 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
2501 {
2502 struct be_adapter *adapter = rxo->adapter;
2503 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
2504 struct be_queue_info *rxq = &rxo->q;
2505 struct page *pagep = NULL;
2506 struct device *dev = &adapter->pdev->dev;
2507 struct be_eth_rx_d *rxd;
2508 u64 page_dmaaddr = 0, frag_dmaaddr;
2509 u32 posted, page_offset = 0, notify = 0;
2510
2511 page_info = &rxo->page_info_tbl[rxq->head];
2512 for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
2513 if (!pagep) {
2514 pagep = be_alloc_pages(adapter->big_page_size, gfp);
2515 if (unlikely(!pagep)) {
2516 rx_stats(rxo)->rx_post_fail++;
2517 break;
2518 }
2519 page_dmaaddr = dma_map_page(dev, pagep, 0,
2520 adapter->big_page_size,
2521 DMA_FROM_DEVICE);
2522 if (dma_mapping_error(dev, page_dmaaddr)) {
2523 put_page(pagep);
2524 pagep = NULL;
2525 adapter->drv_stats.dma_map_errors++;
2526 break;
2527 }
2528 page_offset = 0;
2529 } else {
2530 get_page(pagep);
2531 page_offset += rx_frag_size;
2532 }
2533 page_info->page_offset = page_offset;
2534 page_info->page = pagep;
2535
2536 rxd = queue_head_node(rxq);
2537 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2538 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2539 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2540
2541 /* Any space left in the current big page for another frag? */
2542 if ((page_offset + rx_frag_size + rx_frag_size) >
2543 adapter->big_page_size) {
2544 pagep = NULL;
2545 page_info->last_frag = true;
2546 dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2547 } else {
2548 dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2549 }
2550
2551 prev_page_info = page_info;
2552 queue_head_inc(rxq);
2553 page_info = &rxo->page_info_tbl[rxq->head];
2554 }
2555
2556 /* Mark the last frag of a page when we break out of the above loop
2557 * with no more slots available in the RXQ
2558 */
2559 if (pagep) {
2560 prev_page_info->last_frag = true;
2561 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2562 }
2563
2564 if (posted) {
2565 atomic_add(posted, &rxq->used);
2566 if (rxo->rx_post_starved)
2567 rxo->rx_post_starved = false;
2568 do {
2569 notify = min(MAX_NUM_POST_ERX_DB, posted);
2570 be_rxq_notify(adapter, rxq->id, notify);
2571 posted -= notify;
2572 } while (posted);
2573 } else if (atomic_read(&rxq->used) == 0) {
2574 /* Let be_worker replenish when memory is available */
2575 rxo->rx_post_starved = true;
2576 }
2577 }
2578
2579 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)
2580 {
2581 struct be_queue_info *tx_cq = &txo->cq;
2582 struct be_tx_compl_info *txcp = &txo->txcp;
2583 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq);
2584
2585 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2586 return NULL;
2587
2588 /* Ensure load ordering of valid bit dword and other dwords below */
2589 rmb();
2590 be_dws_le_to_cpu(compl, sizeof(*compl));
2591
2592 txcp->status = GET_TX_COMPL_BITS(status, compl);
2593 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl);
2594
2595 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2596 queue_tail_inc(tx_cq);
2597 return txcp;
2598 }
2599
2600 static u16 be_tx_compl_process(struct be_adapter *adapter,
2601 struct be_tx_obj *txo, u16 last_index)
2602 {
2603 struct sk_buff **sent_skbs = txo->sent_skb_list;
2604 struct be_queue_info *txq = &txo->q;
2605 struct sk_buff *skb = NULL;
2606 bool unmap_skb_hdr = false;
2607 struct be_eth_wrb *wrb;
2608 u16 num_wrbs = 0;
2609 u32 frag_index;
2610
2611 do {
2612 if (sent_skbs[txq->tail]) {
2613 /* Free skb from prev req */
2614 if (skb)
2615 dev_consume_skb_any(skb);
2616 skb = sent_skbs[txq->tail];
2617 sent_skbs[txq->tail] = NULL;
2618 queue_tail_inc(txq); /* skip hdr wrb */
2619 num_wrbs++;
2620 unmap_skb_hdr = true;
2621 }
2622 wrb = queue_tail_node(txq);
2623 frag_index = txq->tail;
2624 unmap_tx_frag(&adapter->pdev->dev, wrb,
2625 (unmap_skb_hdr && skb_headlen(skb)));
2626 unmap_skb_hdr = false;
2627 queue_tail_inc(txq);
2628 num_wrbs++;
2629 } while (frag_index != last_index);
2630 dev_consume_skb_any(skb);
2631
2632 return num_wrbs;
2633 }
2634
2635 /* Return the number of events in the event queue */
2636 static inline int events_get(struct be_eq_obj *eqo)
2637 {
2638 struct be_eq_entry *eqe;
2639 int num = 0;
2640
2641 do {
2642 eqe = queue_tail_node(&eqo->q);
2643 if (eqe->evt == 0)
2644 break;
2645
2646 rmb();
2647 eqe->evt = 0;
2648 num++;
2649 queue_tail_inc(&eqo->q);
2650 } while (true);
2651
2652 return num;
2653 }
2654
2655 /* Leaves the EQ is disarmed state */
2656 static void be_eq_clean(struct be_eq_obj *eqo)
2657 {
2658 int num = events_get(eqo);
2659
2660 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0);
2661 }
2662
2663 /* Free posted rx buffers that were not used */
2664 static void be_rxq_clean(struct be_rx_obj *rxo)
2665 {
2666 struct be_queue_info *rxq = &rxo->q;
2667 struct be_rx_page_info *page_info;
2668
2669 while (atomic_read(&rxq->used) > 0) {
2670 page_info = get_rx_page_info(rxo);
2671 put_page(page_info->page);
2672 memset(page_info, 0, sizeof(*page_info));
2673 }
2674 BUG_ON(atomic_read(&rxq->used));
2675 rxq->tail = 0;
2676 rxq->head = 0;
2677 }
2678
2679 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2680 {
2681 struct be_queue_info *rx_cq = &rxo->cq;
2682 struct be_rx_compl_info *rxcp;
2683 struct be_adapter *adapter = rxo->adapter;
2684 int flush_wait = 0;
2685
2686 /* Consume pending rx completions.
2687 * Wait for the flush completion (identified by zero num_rcvd)
2688 * to arrive. Notify CQ even when there are no more CQ entries
2689 * for HW to flush partially coalesced CQ entries.
2690 * In Lancer, there is no need to wait for flush compl.
2691 */
2692 for (;;) {
2693 rxcp = be_rx_compl_get(rxo);
2694 if (!rxcp) {
2695 if (lancer_chip(adapter))
2696 break;
2697
2698 if (flush_wait++ > 50 ||
2699 be_check_error(adapter,
2700 BE_ERROR_HW)) {
2701 dev_warn(&adapter->pdev->dev,
2702 "did not receive flush compl\n");
2703 break;
2704 }
2705 be_cq_notify(adapter, rx_cq->id, true, 0);
2706 mdelay(1);
2707 } else {
2708 be_rx_compl_discard(rxo, rxcp);
2709 be_cq_notify(adapter, rx_cq->id, false, 1);
2710 if (rxcp->num_rcvd == 0)
2711 break;
2712 }
2713 }
2714
2715 /* After cleanup, leave the CQ in unarmed state */
2716 be_cq_notify(adapter, rx_cq->id, false, 0);
2717 }
2718
2719 static void be_tx_compl_clean(struct be_adapter *adapter)
2720 {
2721 struct device *dev = &adapter->pdev->dev;
2722 u16 cmpl = 0, timeo = 0, num_wrbs = 0;
2723 struct be_tx_compl_info *txcp;
2724 struct be_queue_info *txq;
2725 u32 end_idx, notified_idx;
2726 struct be_tx_obj *txo;
2727 int i, pending_txqs;
2728
2729 /* Stop polling for compls when HW has been silent for 10ms */
2730 do {
2731 pending_txqs = adapter->num_tx_qs;
2732
2733 for_all_tx_queues(adapter, txo, i) {
2734 cmpl = 0;
2735 num_wrbs = 0;
2736 txq = &txo->q;
2737 while ((txcp = be_tx_compl_get(txo))) {
2738 num_wrbs +=
2739 be_tx_compl_process(adapter, txo,
2740 txcp->end_index);
2741 cmpl++;
2742 }
2743 if (cmpl) {
2744 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2745 atomic_sub(num_wrbs, &txq->used);
2746 timeo = 0;
2747 }
2748 if (!be_is_tx_compl_pending(txo))
2749 pending_txqs--;
2750 }
2751
2752 if (pending_txqs == 0 || ++timeo > 10 ||
2753 be_check_error(adapter, BE_ERROR_HW))
2754 break;
2755
2756 mdelay(1);
2757 } while (true);
2758
2759 /* Free enqueued TX that was never notified to HW */
2760 for_all_tx_queues(adapter, txo, i) {
2761 txq = &txo->q;
2762
2763 if (atomic_read(&txq->used)) {
2764 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2765 i, atomic_read(&txq->used));
2766 notified_idx = txq->tail;
2767 end_idx = txq->tail;
2768 index_adv(&end_idx, atomic_read(&txq->used) - 1,
2769 txq->len);
2770 /* Use the tx-compl process logic to handle requests
2771 * that were not sent to the HW.
2772 */
2773 num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2774 atomic_sub(num_wrbs, &txq->used);
2775 BUG_ON(atomic_read(&txq->used));
2776 txo->pend_wrb_cnt = 0;
2777 /* Since hw was never notified of these requests,
2778 * reset TXQ indices
2779 */
2780 txq->head = notified_idx;
2781 txq->tail = notified_idx;
2782 }
2783 }
2784 }
2785
2786 static void be_evt_queues_destroy(struct be_adapter *adapter)
2787 {
2788 struct be_eq_obj *eqo;
2789 int i;
2790
2791 for_all_evt_queues(adapter, eqo, i) {
2792 if (eqo->q.created) {
2793 be_eq_clean(eqo);
2794 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2795 netif_napi_del(&eqo->napi);
2796 free_cpumask_var(eqo->affinity_mask);
2797 }
2798 be_queue_free(adapter, &eqo->q);
2799 }
2800 }
2801
2802 static int be_evt_queues_create(struct be_adapter *adapter)
2803 {
2804 struct be_queue_info *eq;
2805 struct be_eq_obj *eqo;
2806 struct be_aic_obj *aic;
2807 int i, rc;
2808
2809 /* need enough EQs to service both RX and TX queues */
2810 adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2811 max(adapter->cfg_num_rx_irqs,
2812 adapter->cfg_num_tx_irqs));
2813
2814 for_all_evt_queues(adapter, eqo, i) {
2815 int numa_node = dev_to_node(&adapter->pdev->dev);
2816
2817 aic = &adapter->aic_obj[i];
2818 eqo->adapter = adapter;
2819 eqo->idx = i;
2820 aic->max_eqd = BE_MAX_EQD;
2821 aic->enable = true;
2822
2823 eq = &eqo->q;
2824 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2825 sizeof(struct be_eq_entry));
2826 if (rc)
2827 return rc;
2828
2829 rc = be_cmd_eq_create(adapter, eqo);
2830 if (rc)
2831 return rc;
2832
2833 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL))
2834 return -ENOMEM;
2835 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
2836 eqo->affinity_mask);
2837 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2838 BE_NAPI_WEIGHT);
2839 }
2840 return 0;
2841 }
2842
2843 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2844 {
2845 struct be_queue_info *q;
2846
2847 q = &adapter->mcc_obj.q;
2848 if (q->created)
2849 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2850 be_queue_free(adapter, q);
2851
2852 q = &adapter->mcc_obj.cq;
2853 if (q->created)
2854 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2855 be_queue_free(adapter, q);
2856 }
2857
2858 /* Must be called only after TX qs are created as MCC shares TX EQ */
2859 static int be_mcc_queues_create(struct be_adapter *adapter)
2860 {
2861 struct be_queue_info *q, *cq;
2862
2863 cq = &adapter->mcc_obj.cq;
2864 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2865 sizeof(struct be_mcc_compl)))
2866 goto err;
2867
2868 /* Use the default EQ for MCC completions */
2869 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2870 goto mcc_cq_free;
2871
2872 q = &adapter->mcc_obj.q;
2873 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2874 goto mcc_cq_destroy;
2875
2876 if (be_cmd_mccq_create(adapter, q, cq))
2877 goto mcc_q_free;
2878
2879 return 0;
2880
2881 mcc_q_free:
2882 be_queue_free(adapter, q);
2883 mcc_cq_destroy:
2884 be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2885 mcc_cq_free:
2886 be_queue_free(adapter, cq);
2887 err:
2888 return -1;
2889 }
2890
2891 static void be_tx_queues_destroy(struct be_adapter *adapter)
2892 {
2893 struct be_queue_info *q;
2894 struct be_tx_obj *txo;
2895 u8 i;
2896
2897 for_all_tx_queues(adapter, txo, i) {
2898 q = &txo->q;
2899 if (q->created)
2900 be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2901 be_queue_free(adapter, q);
2902
2903 q = &txo->cq;
2904 if (q->created)
2905 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2906 be_queue_free(adapter, q);
2907 }
2908 }
2909
2910 static int be_tx_qs_create(struct be_adapter *adapter)
2911 {
2912 struct be_queue_info *cq;
2913 struct be_tx_obj *txo;
2914 struct be_eq_obj *eqo;
2915 int status, i;
2916
2917 adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs);
2918
2919 for_all_tx_queues(adapter, txo, i) {
2920 cq = &txo->cq;
2921 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2922 sizeof(struct be_eth_tx_compl));
2923 if (status)
2924 return status;
2925
2926 u64_stats_init(&txo->stats.sync);
2927 u64_stats_init(&txo->stats.sync_compl);
2928
2929 /* If num_evt_qs is less than num_tx_qs, then more than
2930 * one txq share an eq
2931 */
2932 eqo = &adapter->eq_obj[i % adapter->num_evt_qs];
2933 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3);
2934 if (status)
2935 return status;
2936
2937 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2938 sizeof(struct be_eth_wrb));
2939 if (status)
2940 return status;
2941
2942 status = be_cmd_txq_create(adapter, txo);
2943 if (status)
2944 return status;
2945
2946 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask,
2947 eqo->idx);
2948 }
2949
2950 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2951 adapter->num_tx_qs);
2952 return 0;
2953 }
2954
2955 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2956 {
2957 struct be_queue_info *q;
2958 struct be_rx_obj *rxo;
2959 int i;
2960
2961 for_all_rx_queues(adapter, rxo, i) {
2962 q = &rxo->cq;
2963 if (q->created)
2964 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2965 be_queue_free(adapter, q);
2966 }
2967 }
2968
2969 static int be_rx_cqs_create(struct be_adapter *adapter)
2970 {
2971 struct be_queue_info *eq, *cq;
2972 struct be_rx_obj *rxo;
2973 int rc, i;
2974
2975 adapter->num_rss_qs =
2976 min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs);
2977
2978 /* We'll use RSS only if atleast 2 RSS rings are supported. */
2979 if (adapter->num_rss_qs < 2)
2980 adapter->num_rss_qs = 0;
2981
2982 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq;
2983
2984 /* When the interface is not capable of RSS rings (and there is no
2985 * need to create a default RXQ) we'll still need one RXQ
2986 */
2987 if (adapter->num_rx_qs == 0)
2988 adapter->num_rx_qs = 1;
2989
2990 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2991 for_all_rx_queues(adapter, rxo, i) {
2992 rxo->adapter = adapter;
2993 cq = &rxo->cq;
2994 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2995 sizeof(struct be_eth_rx_compl));
2996 if (rc)
2997 return rc;
2998
2999 u64_stats_init(&rxo->stats.sync);
3000 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
3001 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
3002 if (rc)
3003 return rc;
3004 }
3005
3006 dev_info(&adapter->pdev->dev,
3007 "created %d RX queue(s)\n", adapter->num_rx_qs);
3008 return 0;
3009 }
3010
3011 static irqreturn_t be_intx(int irq, void *dev)
3012 {
3013 struct be_eq_obj *eqo = dev;
3014 struct be_adapter *adapter = eqo->adapter;
3015 int num_evts = 0;
3016
3017 /* IRQ is not expected when NAPI is scheduled as the EQ
3018 * will not be armed.
3019 * But, this can happen on Lancer INTx where it takes
3020 * a while to de-assert INTx or in BE2 where occasionaly
3021 * an interrupt may be raised even when EQ is unarmed.
3022 * If NAPI is already scheduled, then counting & notifying
3023 * events will orphan them.
3024 */
3025 if (napi_schedule_prep(&eqo->napi)) {
3026 num_evts = events_get(eqo);
3027 __napi_schedule(&eqo->napi);
3028 if (num_evts)
3029 eqo->spurious_intr = 0;
3030 }
3031 be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0);
3032
3033 /* Return IRQ_HANDLED only for the the first spurious intr
3034 * after a valid intr to stop the kernel from branding
3035 * this irq as a bad one!
3036 */
3037 if (num_evts || eqo->spurious_intr++ == 0)
3038 return IRQ_HANDLED;
3039 else
3040 return IRQ_NONE;
3041 }
3042
3043 static irqreturn_t be_msix(int irq, void *dev)
3044 {
3045 struct be_eq_obj *eqo = dev;
3046
3047 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
3048 napi_schedule(&eqo->napi);
3049 return IRQ_HANDLED;
3050 }
3051
3052 static inline bool do_gro(struct be_rx_compl_info *rxcp)
3053 {
3054 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
3055 }
3056
3057 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
3058 int budget, int polling)
3059 {
3060 struct be_adapter *adapter = rxo->adapter;
3061 struct be_queue_info *rx_cq = &rxo->cq;
3062 struct be_rx_compl_info *rxcp;
3063 u32 work_done;
3064 u32 frags_consumed = 0;
3065
3066 for (work_done = 0; work_done < budget; work_done++) {
3067 rxcp = be_rx_compl_get(rxo);
3068 if (!rxcp)
3069 break;
3070
3071 /* Is it a flush compl that has no data */
3072 if (unlikely(rxcp->num_rcvd == 0))
3073 goto loop_continue;
3074
3075 /* Discard compl with partial DMA Lancer B0 */
3076 if (unlikely(!rxcp->pkt_size)) {
3077 be_rx_compl_discard(rxo, rxcp);
3078 goto loop_continue;
3079 }
3080
3081 /* On BE drop pkts that arrive due to imperfect filtering in
3082 * promiscuous mode on some skews
3083 */
3084 if (unlikely(rxcp->port != adapter->port_num &&
3085 !lancer_chip(adapter))) {
3086 be_rx_compl_discard(rxo, rxcp);
3087 goto loop_continue;
3088 }
3089
3090 /* Don't do gro when we're busy_polling */
3091 if (do_gro(rxcp) && polling != BUSY_POLLING)
3092 be_rx_compl_process_gro(rxo, napi, rxcp);
3093 else
3094 be_rx_compl_process(rxo, napi, rxcp);
3095
3096 loop_continue:
3097 frags_consumed += rxcp->num_rcvd;
3098 be_rx_stats_update(rxo, rxcp);
3099 }
3100
3101 if (work_done) {
3102 be_cq_notify(adapter, rx_cq->id, true, work_done);
3103
3104 /* When an rx-obj gets into post_starved state, just
3105 * let be_worker do the posting.
3106 */
3107 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
3108 !rxo->rx_post_starved)
3109 be_post_rx_frags(rxo, GFP_ATOMIC,
3110 max_t(u32, MAX_RX_POST,
3111 frags_consumed));
3112 }
3113
3114 return work_done;
3115 }
3116
3117 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
3118 {
3119 switch (status) {
3120 case BE_TX_COMP_HDR_PARSE_ERR:
3121 tx_stats(txo)->tx_hdr_parse_err++;
3122 break;
3123 case BE_TX_COMP_NDMA_ERR:
3124 tx_stats(txo)->tx_dma_err++;
3125 break;
3126 case BE_TX_COMP_ACL_ERR:
3127 tx_stats(txo)->tx_spoof_check_err++;
3128 break;
3129 }
3130 }
3131
3132 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
3133 {
3134 switch (status) {
3135 case LANCER_TX_COMP_LSO_ERR:
3136 tx_stats(txo)->tx_tso_err++;
3137 break;
3138 case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
3139 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
3140 tx_stats(txo)->tx_spoof_check_err++;
3141 break;
3142 case LANCER_TX_COMP_QINQ_ERR:
3143 tx_stats(txo)->tx_qinq_err++;
3144 break;
3145 case LANCER_TX_COMP_PARITY_ERR:
3146 tx_stats(txo)->tx_internal_parity_err++;
3147 break;
3148 case LANCER_TX_COMP_DMA_ERR:
3149 tx_stats(txo)->tx_dma_err++;
3150 break;
3151 }
3152 }
3153
3154 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
3155 int idx)
3156 {
3157 int num_wrbs = 0, work_done = 0;
3158 struct be_tx_compl_info *txcp;
3159
3160 while ((txcp = be_tx_compl_get(txo))) {
3161 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);
3162 work_done++;
3163
3164 if (txcp->status) {
3165 if (lancer_chip(adapter))
3166 lancer_update_tx_err(txo, txcp->status);
3167 else
3168 be_update_tx_err(txo, txcp->status);
3169 }
3170 }
3171
3172 if (work_done) {
3173 be_cq_notify(adapter, txo->cq.id, true, work_done);
3174 atomic_sub(num_wrbs, &txo->q.used);
3175
3176 /* As Tx wrbs have been freed up, wake up netdev queue
3177 * if it was stopped due to lack of tx wrbs. */
3178 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
3179 be_can_txq_wake(txo)) {
3180 netif_wake_subqueue(adapter->netdev, idx);
3181 }
3182
3183 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
3184 tx_stats(txo)->tx_compl += work_done;
3185 u64_stats_update_end(&tx_stats(txo)->sync_compl);
3186 }
3187 }
3188
3189 #ifdef CONFIG_NET_RX_BUSY_POLL
3190 static inline bool be_lock_napi(struct be_eq_obj *eqo)
3191 {
3192 bool status = true;
3193
3194 spin_lock(&eqo->lock); /* BH is already disabled */
3195 if (eqo->state & BE_EQ_LOCKED) {
3196 WARN_ON(eqo->state & BE_EQ_NAPI);
3197 eqo->state |= BE_EQ_NAPI_YIELD;
3198 status = false;
3199 } else {
3200 eqo->state = BE_EQ_NAPI;
3201 }
3202 spin_unlock(&eqo->lock);
3203 return status;
3204 }
3205
3206 static inline void be_unlock_napi(struct be_eq_obj *eqo)
3207 {
3208 spin_lock(&eqo->lock); /* BH is already disabled */
3209
3210 WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
3211 eqo->state = BE_EQ_IDLE;
3212
3213 spin_unlock(&eqo->lock);
3214 }
3215
3216 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
3217 {
3218 bool status = true;
3219
3220 spin_lock_bh(&eqo->lock);
3221 if (eqo->state & BE_EQ_LOCKED) {
3222 eqo->state |= BE_EQ_POLL_YIELD;
3223 status = false;
3224 } else {
3225 eqo->state |= BE_EQ_POLL;
3226 }
3227 spin_unlock_bh(&eqo->lock);
3228 return status;
3229 }
3230
3231 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
3232 {
3233 spin_lock_bh(&eqo->lock);
3234
3235 WARN_ON(eqo->state & (BE_EQ_NAPI));
3236 eqo->state = BE_EQ_IDLE;
3237
3238 spin_unlock_bh(&eqo->lock);
3239 }
3240
3241 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
3242 {
3243 spin_lock_init(&eqo->lock);
3244 eqo->state = BE_EQ_IDLE;
3245 }
3246
3247 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
3248 {
3249 local_bh_disable();
3250
3251 /* It's enough to just acquire napi lock on the eqo to stop
3252 * be_busy_poll() from processing any queueus.
3253 */
3254 while (!be_lock_napi(eqo))
3255 mdelay(1);
3256
3257 local_bh_enable();
3258 }
3259
3260 #else /* CONFIG_NET_RX_BUSY_POLL */
3261
3262 static inline bool be_lock_napi(struct be_eq_obj *eqo)
3263 {
3264 return true;
3265 }
3266
3267 static inline void be_unlock_napi(struct be_eq_obj *eqo)
3268 {
3269 }
3270
3271 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
3272 {
3273 return false;
3274 }
3275
3276 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
3277 {
3278 }
3279
3280 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
3281 {
3282 }
3283
3284 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
3285 {
3286 }
3287 #endif /* CONFIG_NET_RX_BUSY_POLL */
3288
3289 int be_poll(struct napi_struct *napi, int budget)
3290 {
3291 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
3292 struct be_adapter *adapter = eqo->adapter;
3293 int max_work = 0, work, i, num_evts;
3294 struct be_rx_obj *rxo;
3295 struct be_tx_obj *txo;
3296 u32 mult_enc = 0;
3297
3298 num_evts = events_get(eqo);
3299
3300 for_all_tx_queues_on_eq(adapter, eqo, txo, i)
3301 be_process_tx(adapter, txo, i);
3302
3303 if (be_lock_napi(eqo)) {
3304 /* This loop will iterate twice for EQ0 in which
3305 * completions of the last RXQ (default one) are also processed
3306 * For other EQs the loop iterates only once
3307 */
3308 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
3309 work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
3310 max_work = max(work, max_work);
3311 }
3312 be_unlock_napi(eqo);
3313 } else {
3314 max_work = budget;
3315 }
3316
3317 if (is_mcc_eqo(eqo))
3318 be_process_mcc(adapter);
3319
3320 if (max_work < budget) {
3321 napi_complete(napi);
3322
3323 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt
3324 * delay via a delay multiplier encoding value
3325 */
3326 if (skyhawk_chip(adapter))
3327 mult_enc = be_get_eq_delay_mult_enc(eqo);
3328
3329 be_eq_notify(adapter, eqo->q.id, true, false, num_evts,
3330 mult_enc);
3331 } else {
3332 /* As we'll continue in polling mode, count and clear events */
3333 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0);
3334 }
3335 return max_work;
3336 }
3337
3338 #ifdef CONFIG_NET_RX_BUSY_POLL
3339 static int be_busy_poll(struct napi_struct *napi)
3340 {
3341 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
3342 struct be_adapter *adapter = eqo->adapter;
3343 struct be_rx_obj *rxo;
3344 int i, work = 0;
3345
3346 if (!be_lock_busy_poll(eqo))
3347 return LL_FLUSH_BUSY;
3348
3349 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
3350 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
3351 if (work)
3352 break;
3353 }
3354
3355 be_unlock_busy_poll(eqo);
3356 return work;
3357 }
3358 #endif
3359
3360 void be_detect_error(struct be_adapter *adapter)
3361 {
3362 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
3363 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
3364 u32 i;
3365 struct device *dev = &adapter->pdev->dev;
3366
3367 if (be_check_error(adapter, BE_ERROR_HW))
3368 return;
3369
3370 if (lancer_chip(adapter)) {
3371 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
3372 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
3373 be_set_error(adapter, BE_ERROR_UE);
3374 sliport_err1 = ioread32(adapter->db +
3375 SLIPORT_ERROR1_OFFSET);
3376 sliport_err2 = ioread32(adapter->db +
3377 SLIPORT_ERROR2_OFFSET);
3378 /* Do not log error messages if its a FW reset */
3379 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
3380 sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
3381 dev_info(dev, "Firmware update in progress\n");
3382 } else {
3383 dev_err(dev, "Error detected in the card\n");
3384 dev_err(dev, "ERR: sliport status 0x%x\n",
3385 sliport_status);
3386 dev_err(dev, "ERR: sliport error1 0x%x\n",
3387 sliport_err1);
3388 dev_err(dev, "ERR: sliport error2 0x%x\n",
3389 sliport_err2);
3390 }
3391 }
3392 } else {
3393 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW);
3394 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH);
3395 ue_lo_mask = ioread32(adapter->pcicfg +
3396 PCICFG_UE_STATUS_LOW_MASK);
3397 ue_hi_mask = ioread32(adapter->pcicfg +
3398 PCICFG_UE_STATUS_HI_MASK);
3399
3400 ue_lo = (ue_lo & ~ue_lo_mask);
3401 ue_hi = (ue_hi & ~ue_hi_mask);
3402
3403 /* On certain platforms BE hardware can indicate spurious UEs.
3404 * Allow HW to stop working completely in case of a real UE.
3405 * Hence not setting the hw_error for UE detection.
3406 */
3407
3408 if (ue_lo || ue_hi) {
3409 dev_err(dev, "Error detected in the adapter");
3410 if (skyhawk_chip(adapter))
3411 be_set_error(adapter, BE_ERROR_UE);
3412
3413 for (i = 0; ue_lo; ue_lo >>= 1, i++) {
3414 if (ue_lo & 1)
3415 dev_err(dev, "UE: %s bit set\n",
3416 ue_status_low_desc[i]);
3417 }
3418 for (i = 0; ue_hi; ue_hi >>= 1, i++) {
3419 if (ue_hi & 1)
3420 dev_err(dev, "UE: %s bit set\n",
3421 ue_status_hi_desc[i]);
3422 }
3423 }
3424 }
3425 }
3426
3427 static void be_msix_disable(struct be_adapter *adapter)
3428 {
3429 if (msix_enabled(adapter)) {
3430 pci_disable_msix(adapter->pdev);
3431 adapter->num_msix_vec = 0;
3432 adapter->num_msix_roce_vec = 0;
3433 }
3434 }
3435
3436 static int be_msix_enable(struct be_adapter *adapter)
3437 {
3438 unsigned int i, max_roce_eqs;
3439 struct device *dev = &adapter->pdev->dev;
3440 int num_vec;
3441
3442 /* If RoCE is supported, program the max number of vectors that
3443 * could be used for NIC and RoCE, else, just program the number
3444 * we'll use initially.
3445 */
3446 if (be_roce_supported(adapter)) {
3447 max_roce_eqs =
3448 be_max_func_eqs(adapter) - be_max_nic_eqs(adapter);
3449 max_roce_eqs = min(max_roce_eqs, num_online_cpus());
3450 num_vec = be_max_any_irqs(adapter) + max_roce_eqs;
3451 } else {
3452 num_vec = max(adapter->cfg_num_rx_irqs,
3453 adapter->cfg_num_tx_irqs);
3454 }
3455
3456 for (i = 0; i < num_vec; i++)
3457 adapter->msix_entries[i].entry = i;
3458
3459 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
3460 MIN_MSIX_VECTORS, num_vec);
3461 if (num_vec < 0)
3462 goto fail;
3463
3464 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
3465 adapter->num_msix_roce_vec = num_vec / 2;
3466 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
3467 adapter->num_msix_roce_vec);
3468 }
3469
3470 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
3471
3472 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
3473 adapter->num_msix_vec);
3474 return 0;
3475
3476 fail:
3477 dev_warn(dev, "MSIx enable failed\n");
3478
3479 /* INTx is not supported in VFs, so fail probe if enable_msix fails */
3480 if (be_virtfn(adapter))
3481 return num_vec;
3482 return 0;
3483 }
3484
3485 static inline int be_msix_vec_get(struct be_adapter *adapter,
3486 struct be_eq_obj *eqo)
3487 {
3488 return adapter->msix_entries[eqo->msix_idx].vector;
3489 }
3490
3491 static int be_msix_register(struct be_adapter *adapter)
3492 {
3493 struct net_device *netdev = adapter->netdev;
3494 struct be_eq_obj *eqo;
3495 int status, i, vec;
3496
3497 for_all_evt_queues(adapter, eqo, i) {
3498 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
3499 vec = be_msix_vec_get(adapter, eqo);
3500 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
3501 if (status)
3502 goto err_msix;
3503
3504 irq_set_affinity_hint(vec, eqo->affinity_mask);
3505 }
3506
3507 return 0;
3508 err_msix:
3509 for (i--; i >= 0; i--) {
3510 eqo = &adapter->eq_obj[i];
3511 free_irq(be_msix_vec_get(adapter, eqo), eqo);
3512 }
3513 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
3514 status);
3515 be_msix_disable(adapter);
3516 return status;
3517 }
3518
3519 static int be_irq_register(struct be_adapter *adapter)
3520 {
3521 struct net_device *netdev = adapter->netdev;
3522 int status;
3523
3524 if (msix_enabled(adapter)) {
3525 status = be_msix_register(adapter);
3526 if (status == 0)
3527 goto done;
3528 /* INTx is not supported for VF */
3529 if (be_virtfn(adapter))
3530 return status;
3531 }
3532
3533 /* INTx: only the first EQ is used */
3534 netdev->irq = adapter->pdev->irq;
3535 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
3536 &adapter->eq_obj[0]);
3537 if (status) {
3538 dev_err(&adapter->pdev->dev,
3539 "INTx request IRQ failed - err %d\n", status);
3540 return status;
3541 }
3542 done:
3543 adapter->isr_registered = true;
3544 return 0;
3545 }
3546
3547 static void be_irq_unregister(struct be_adapter *adapter)
3548 {
3549 struct net_device *netdev = adapter->netdev;
3550 struct be_eq_obj *eqo;
3551 int i, vec;
3552
3553 if (!adapter->isr_registered)
3554 return;
3555
3556 /* INTx */
3557 if (!msix_enabled(adapter)) {
3558 free_irq(netdev->irq, &adapter->eq_obj[0]);
3559 goto done;
3560 }
3561
3562 /* MSIx */
3563 for_all_evt_queues(adapter, eqo, i) {
3564 vec = be_msix_vec_get(adapter, eqo);
3565 irq_set_affinity_hint(vec, NULL);
3566 free_irq(vec, eqo);
3567 }
3568
3569 done:
3570 adapter->isr_registered = false;
3571 }
3572
3573 static void be_rx_qs_destroy(struct be_adapter *adapter)
3574 {
3575 struct rss_info *rss = &adapter->rss_info;
3576 struct be_queue_info *q;
3577 struct be_rx_obj *rxo;
3578 int i;
3579
3580 for_all_rx_queues(adapter, rxo, i) {
3581 q = &rxo->q;
3582 if (q->created) {
3583 /* If RXQs are destroyed while in an "out of buffer"
3584 * state, there is a possibility of an HW stall on
3585 * Lancer. So, post 64 buffers to each queue to relieve
3586 * the "out of buffer" condition.
3587 * Make sure there's space in the RXQ before posting.
3588 */
3589 if (lancer_chip(adapter)) {
3590 be_rx_cq_clean(rxo);
3591 if (atomic_read(&q->used) == 0)
3592 be_post_rx_frags(rxo, GFP_KERNEL,
3593 MAX_RX_POST);
3594 }
3595
3596 be_cmd_rxq_destroy(adapter, q);
3597 be_rx_cq_clean(rxo);
3598 be_rxq_clean(rxo);
3599 }
3600 be_queue_free(adapter, q);
3601 }
3602
3603 if (rss->rss_flags) {
3604 rss->rss_flags = RSS_ENABLE_NONE;
3605 be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3606 128, rss->rss_hkey);
3607 }
3608 }
3609
3610 static void be_disable_if_filters(struct be_adapter *adapter)
3611 {
3612 be_dev_mac_del(adapter, adapter->pmac_id[0]);
3613 be_clear_uc_list(adapter);
3614 be_clear_mc_list(adapter);
3615
3616 /* The IFACE flags are enabled in the open path and cleared
3617 * in the close path. When a VF gets detached from the host and
3618 * assigned to a VM the following happens:
3619 * - VF's IFACE flags get cleared in the detach path
3620 * - IFACE create is issued by the VF in the attach path
3621 * Due to a bug in the BE3/Skyhawk-R FW
3622 * (Lancer FW doesn't have the bug), the IFACE capability flags
3623 * specified along with the IFACE create cmd issued by a VF are not
3624 * honoured by FW. As a consequence, if a *new* driver
3625 * (that enables/disables IFACE flags in open/close)
3626 * is loaded in the host and an *old* driver is * used by a VM/VF,
3627 * the IFACE gets created *without* the needed flags.
3628 * To avoid this, disable RX-filter flags only for Lancer.
3629 */
3630 if (lancer_chip(adapter)) {
3631 be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF);
3632 adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS;
3633 }
3634 }
3635
3636 static int be_close(struct net_device *netdev)
3637 {
3638 struct be_adapter *adapter = netdev_priv(netdev);
3639 struct be_eq_obj *eqo;
3640 int i;
3641
3642 /* This protection is needed as be_close() may be called even when the
3643 * adapter is in cleared state (after eeh perm failure)
3644 */
3645 if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3646 return 0;
3647
3648 /* Before attempting cleanup ensure all the pending cmds in the
3649 * config_wq have finished execution
3650 */
3651 flush_workqueue(be_wq);
3652
3653 be_disable_if_filters(adapter);
3654
3655 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3656 for_all_evt_queues(adapter, eqo, i) {
3657 napi_disable(&eqo->napi);
3658 be_disable_busy_poll(eqo);
3659 }
3660 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3661 }
3662
3663 be_async_mcc_disable(adapter);
3664
3665 /* Wait for all pending tx completions to arrive so that
3666 * all tx skbs are freed.
3667 */
3668 netif_tx_disable(netdev);
3669 be_tx_compl_clean(adapter);
3670
3671 be_rx_qs_destroy(adapter);
3672
3673 for_all_evt_queues(adapter, eqo, i) {
3674 if (msix_enabled(adapter))
3675 synchronize_irq(be_msix_vec_get(adapter, eqo));
3676 else
3677 synchronize_irq(netdev->irq);
3678 be_eq_clean(eqo);
3679 }
3680
3681 be_irq_unregister(adapter);
3682
3683 return 0;
3684 }
3685
3686 static int be_rx_qs_create(struct be_adapter *adapter)
3687 {
3688 struct rss_info *rss = &adapter->rss_info;
3689 u8 rss_key[RSS_HASH_KEY_LEN];
3690 struct be_rx_obj *rxo;
3691 int rc, i, j;
3692
3693 for_all_rx_queues(adapter, rxo, i) {
3694 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3695 sizeof(struct be_eth_rx_d));
3696 if (rc)
3697 return rc;
3698 }
3699
3700 if (adapter->need_def_rxq || !adapter->num_rss_qs) {
3701 rxo = default_rxo(adapter);
3702 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3703 rx_frag_size, adapter->if_handle,
3704 false, &rxo->rss_id);
3705 if (rc)
3706 return rc;
3707 }
3708
3709 for_all_rss_queues(adapter, rxo, i) {
3710 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3711 rx_frag_size, adapter->if_handle,
3712 true, &rxo->rss_id);
3713 if (rc)
3714 return rc;
3715 }
3716
3717 if (be_multi_rxq(adapter)) {
3718 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) {
3719 for_all_rss_queues(adapter, rxo, i) {
3720 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3721 break;
3722 rss->rsstable[j + i] = rxo->rss_id;
3723 rss->rss_queue[j + i] = i;
3724 }
3725 }
3726 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3727 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3728
3729 if (!BEx_chip(adapter))
3730 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3731 RSS_ENABLE_UDP_IPV6;
3732
3733 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3734 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3735 RSS_INDIR_TABLE_LEN, rss_key);
3736 if (rc) {
3737 rss->rss_flags = RSS_ENABLE_NONE;
3738 return rc;
3739 }
3740
3741 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3742 } else {
3743 /* Disable RSS, if only default RX Q is created */
3744 rss->rss_flags = RSS_ENABLE_NONE;
3745 }
3746
3747
3748 /* Post 1 less than RXQ-len to avoid head being equal to tail,
3749 * which is a queue empty condition
3750 */
3751 for_all_rx_queues(adapter, rxo, i)
3752 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1);
3753
3754 return 0;
3755 }
3756
3757 static int be_enable_if_filters(struct be_adapter *adapter)
3758 {
3759 int status;
3760
3761 status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON);
3762 if (status)
3763 return status;
3764
3765 /* For BE3 VFs, the PF programs the initial MAC address */
3766 if (!(BEx_chip(adapter) && be_virtfn(adapter))) {
3767 status = be_dev_mac_add(adapter, adapter->netdev->dev_addr);
3768 if (status)
3769 return status;
3770 ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr);
3771 }
3772
3773 if (adapter->vlans_added)
3774 be_vid_config(adapter);
3775
3776 __be_set_rx_mode(adapter);
3777
3778 return 0;
3779 }
3780
3781 static int be_open(struct net_device *netdev)
3782 {
3783 struct be_adapter *adapter = netdev_priv(netdev);
3784 struct be_eq_obj *eqo;
3785 struct be_rx_obj *rxo;
3786 struct be_tx_obj *txo;
3787 u8 link_status;
3788 int status, i;
3789
3790 status = be_rx_qs_create(adapter);
3791 if (status)
3792 goto err;
3793
3794 status = be_enable_if_filters(adapter);
3795 if (status)
3796 goto err;
3797
3798 status = be_irq_register(adapter);
3799 if (status)
3800 goto err;
3801
3802 for_all_rx_queues(adapter, rxo, i)
3803 be_cq_notify(adapter, rxo->cq.id, true, 0);
3804
3805 for_all_tx_queues(adapter, txo, i)
3806 be_cq_notify(adapter, txo->cq.id, true, 0);
3807
3808 be_async_mcc_enable(adapter);
3809
3810 for_all_evt_queues(adapter, eqo, i) {
3811 napi_enable(&eqo->napi);
3812 be_enable_busy_poll(eqo);
3813 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0);
3814 }
3815 adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3816
3817 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3818 if (!status)
3819 be_link_status_update(adapter, link_status);
3820
3821 netif_tx_start_all_queues(netdev);
3822 if (skyhawk_chip(adapter))
3823 udp_tunnel_get_rx_info(netdev);
3824
3825 return 0;
3826 err:
3827 be_close(adapter->netdev);
3828 return -EIO;
3829 }
3830
3831 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3832 {
3833 u32 addr;
3834
3835 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3836
3837 mac[5] = (u8)(addr & 0xFF);
3838 mac[4] = (u8)((addr >> 8) & 0xFF);
3839 mac[3] = (u8)((addr >> 16) & 0xFF);
3840 /* Use the OUI from the current MAC address */
3841 memcpy(mac, adapter->netdev->dev_addr, 3);
3842 }
3843
3844 /*
3845 * Generate a seed MAC address from the PF MAC Address using jhash.
3846 * MAC Address for VFs are assigned incrementally starting from the seed.
3847 * These addresses are programmed in the ASIC by the PF and the VF driver
3848 * queries for the MAC address during its probe.
3849 */
3850 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3851 {
3852 u32 vf;
3853 int status = 0;
3854 u8 mac[ETH_ALEN];
3855 struct be_vf_cfg *vf_cfg;
3856
3857 be_vf_eth_addr_generate(adapter, mac);
3858
3859 for_all_vfs(adapter, vf_cfg, vf) {
3860 if (BEx_chip(adapter))
3861 status = be_cmd_pmac_add(adapter, mac,
3862 vf_cfg->if_handle,
3863 &vf_cfg->pmac_id, vf + 1);
3864 else
3865 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3866 vf + 1);
3867
3868 if (status)
3869 dev_err(&adapter->pdev->dev,
3870 "Mac address assignment failed for VF %d\n",
3871 vf);
3872 else
3873 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3874
3875 mac[5] += 1;
3876 }
3877 return status;
3878 }
3879
3880 static int be_vfs_mac_query(struct be_adapter *adapter)
3881 {
3882 int status, vf;
3883 u8 mac[ETH_ALEN];
3884 struct be_vf_cfg *vf_cfg;
3885
3886 for_all_vfs(adapter, vf_cfg, vf) {
3887 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3888 mac, vf_cfg->if_handle,
3889 false, vf+1);
3890 if (status)
3891 return status;
3892 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3893 }
3894 return 0;
3895 }
3896
3897 static void be_vf_clear(struct be_adapter *adapter)
3898 {
3899 struct be_vf_cfg *vf_cfg;
3900 u32 vf;
3901
3902 if (pci_vfs_assigned(adapter->pdev)) {
3903 dev_warn(&adapter->pdev->dev,
3904 "VFs are assigned to VMs: not disabling VFs\n");
3905 goto done;
3906 }
3907
3908 pci_disable_sriov(adapter->pdev);
3909
3910 for_all_vfs(adapter, vf_cfg, vf) {
3911 if (BEx_chip(adapter))
3912 be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3913 vf_cfg->pmac_id, vf + 1);
3914 else
3915 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3916 vf + 1);
3917
3918 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3919 }
3920
3921 if (BE3_chip(adapter))
3922 be_cmd_set_hsw_config(adapter, 0, 0,
3923 adapter->if_handle,
3924 PORT_FWD_TYPE_PASSTHRU, 0);
3925 done:
3926 kfree(adapter->vf_cfg);
3927 adapter->num_vfs = 0;
3928 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3929 }
3930
3931 static void be_clear_queues(struct be_adapter *adapter)
3932 {
3933 be_mcc_queues_destroy(adapter);
3934 be_rx_cqs_destroy(adapter);
3935 be_tx_queues_destroy(adapter);
3936 be_evt_queues_destroy(adapter);
3937 }
3938
3939 static void be_cancel_worker(struct be_adapter *adapter)
3940 {
3941 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3942 cancel_delayed_work_sync(&adapter->work);
3943 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3944 }
3945 }
3946
3947 static void be_cancel_err_detection(struct be_adapter *adapter)
3948 {
3949 struct be_error_recovery *err_rec = &adapter->error_recovery;
3950
3951 if (!be_err_recovery_workq)
3952 return;
3953
3954 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) {
3955 cancel_delayed_work_sync(&err_rec->err_detection_work);
3956 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED;
3957 }
3958 }
3959
3960 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3961 {
3962 struct net_device *netdev = adapter->netdev;
3963
3964 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3965 be_cmd_manage_iface(adapter, adapter->if_handle,
3966 OP_CONVERT_TUNNEL_TO_NORMAL);
3967
3968 if (adapter->vxlan_port)
3969 be_cmd_set_vxlan_port(adapter, 0);
3970
3971 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3972 adapter->vxlan_port = 0;
3973
3974 netdev->hw_enc_features = 0;
3975 netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3976 netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3977 }
3978
3979 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs,
3980 struct be_resources *vft_res)
3981 {
3982 struct be_resources res = adapter->pool_res;
3983 u32 vf_if_cap_flags = res.vf_if_cap_flags;
3984 struct be_resources res_mod = {0};
3985 u16 num_vf_qs = 1;
3986
3987 /* Distribute the queue resources among the PF and it's VFs */
3988 if (num_vfs) {
3989 /* Divide the rx queues evenly among the VFs and the PF, capped
3990 * at VF-EQ-count. Any remainder queues belong to the PF.
3991 */
3992 num_vf_qs = min(SH_VF_MAX_NIC_EQS,
3993 res.max_rss_qs / (num_vfs + 1));
3994
3995 /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES
3996 * RSS Tables per port. Provide RSS on VFs, only if number of
3997 * VFs requested is less than it's PF Pool's RSS Tables limit.
3998 */
3999 if (num_vfs >= be_max_pf_pool_rss_tables(adapter))
4000 num_vf_qs = 1;
4001 }
4002
4003 /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd,
4004 * which are modifiable using SET_PROFILE_CONFIG cmd.
4005 */
4006 be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE,
4007 RESOURCE_MODIFIABLE, 0);
4008
4009 /* If RSS IFACE capability flags are modifiable for a VF, set the
4010 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if
4011 * more than 1 RSSQ is available for a VF.
4012 * Otherwise, provision only 1 queue pair for VF.
4013 */
4014 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) {
4015 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
4016 if (num_vf_qs > 1) {
4017 vf_if_cap_flags |= BE_IF_FLAGS_RSS;
4018 if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)
4019 vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS;
4020 } else {
4021 vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS |
4022 BE_IF_FLAGS_DEFQ_RSS);
4023 }
4024 } else {
4025 num_vf_qs = 1;
4026 }
4027
4028 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
4029 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
4030 vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
4031 }
4032
4033 vft_res->vf_if_cap_flags = vf_if_cap_flags;
4034 vft_res->max_rx_qs = num_vf_qs;
4035 vft_res->max_rss_qs = num_vf_qs;
4036 vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1);
4037 vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1);
4038
4039 /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally
4040 * among the PF and it's VFs, if the fields are changeable
4041 */
4042 if (res_mod.max_uc_mac == FIELD_MODIFIABLE)
4043 vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1);
4044
4045 if (res_mod.max_vlans == FIELD_MODIFIABLE)
4046 vft_res->max_vlans = res.max_vlans / (num_vfs + 1);
4047
4048 if (res_mod.max_iface_count == FIELD_MODIFIABLE)
4049 vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1);
4050
4051 if (res_mod.max_mcc_count == FIELD_MODIFIABLE)
4052 vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1);
4053 }
4054
4055 static void be_if_destroy(struct be_adapter *adapter)
4056 {
4057 be_cmd_if_destroy(adapter, adapter->if_handle, 0);
4058
4059 kfree(adapter->pmac_id);
4060 adapter->pmac_id = NULL;
4061
4062 kfree(adapter->mc_list);
4063 adapter->mc_list = NULL;
4064
4065 kfree(adapter->uc_list);
4066 adapter->uc_list = NULL;
4067 }
4068
4069 static int be_clear(struct be_adapter *adapter)
4070 {
4071 struct pci_dev *pdev = adapter->pdev;
4072 struct be_resources vft_res = {0};
4073
4074 be_cancel_worker(adapter);
4075
4076 flush_workqueue(be_wq);
4077
4078 if (sriov_enabled(adapter))
4079 be_vf_clear(adapter);
4080
4081 /* Re-configure FW to distribute resources evenly across max-supported
4082 * number of VFs, only when VFs are not already enabled.
4083 */
4084 if (skyhawk_chip(adapter) && be_physfn(adapter) &&
4085 !pci_vfs_assigned(pdev)) {
4086 be_calculate_vf_res(adapter,
4087 pci_sriov_get_totalvfs(pdev),
4088 &vft_res);
4089 be_cmd_set_sriov_config(adapter, adapter->pool_res,
4090 pci_sriov_get_totalvfs(pdev),
4091 &vft_res);
4092 }
4093
4094 be_disable_vxlan_offloads(adapter);
4095
4096 be_if_destroy(adapter);
4097
4098 be_clear_queues(adapter);
4099
4100 be_msix_disable(adapter);
4101 adapter->flags &= ~BE_FLAGS_SETUP_DONE;
4102 return 0;
4103 }
4104
4105 static int be_vfs_if_create(struct be_adapter *adapter)
4106 {
4107 struct be_resources res = {0};
4108 u32 cap_flags, en_flags, vf;
4109 struct be_vf_cfg *vf_cfg;
4110 int status;
4111
4112 /* If a FW profile exists, then cap_flags are updated */
4113 cap_flags = BE_VF_IF_EN_FLAGS;
4114
4115 for_all_vfs(adapter, vf_cfg, vf) {
4116 if (!BE3_chip(adapter)) {
4117 status = be_cmd_get_profile_config(adapter, &res, NULL,
4118 ACTIVE_PROFILE_TYPE,
4119 RESOURCE_LIMITS,
4120 vf + 1);
4121 if (!status) {
4122 cap_flags = res.if_cap_flags;
4123 /* Prevent VFs from enabling VLAN promiscuous
4124 * mode
4125 */
4126 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
4127 }
4128 }
4129
4130 /* PF should enable IF flags during proxy if_create call */
4131 en_flags = cap_flags & BE_VF_IF_EN_FLAGS;
4132 status = be_cmd_if_create(adapter, cap_flags, en_flags,
4133 &vf_cfg->if_handle, vf + 1);
4134 if (status)
4135 return status;
4136 }
4137
4138 return 0;
4139 }
4140
4141 static int be_vf_setup_init(struct be_adapter *adapter)
4142 {
4143 struct be_vf_cfg *vf_cfg;
4144 int vf;
4145
4146 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
4147 GFP_KERNEL);
4148 if (!adapter->vf_cfg)
4149 return -ENOMEM;
4150
4151 for_all_vfs(adapter, vf_cfg, vf) {
4152 vf_cfg->if_handle = -1;
4153 vf_cfg->pmac_id = -1;
4154 }
4155 return 0;
4156 }
4157
4158 static int be_vf_setup(struct be_adapter *adapter)
4159 {
4160 struct device *dev = &adapter->pdev->dev;
4161 struct be_vf_cfg *vf_cfg;
4162 int status, old_vfs, vf;
4163 bool spoofchk;
4164
4165 old_vfs = pci_num_vf(adapter->pdev);
4166
4167 status = be_vf_setup_init(adapter);
4168 if (status)
4169 goto err;
4170
4171 if (old_vfs) {
4172 for_all_vfs(adapter, vf_cfg, vf) {
4173 status = be_cmd_get_if_id(adapter, vf_cfg, vf);
4174 if (status)
4175 goto err;
4176 }
4177
4178 status = be_vfs_mac_query(adapter);
4179 if (status)
4180 goto err;
4181 } else {
4182 status = be_vfs_if_create(adapter);
4183 if (status)
4184 goto err;
4185
4186 status = be_vf_eth_addr_config(adapter);
4187 if (status)
4188 goto err;
4189 }
4190
4191 for_all_vfs(adapter, vf_cfg, vf) {
4192 /* Allow VFs to programs MAC/VLAN filters */
4193 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
4194 vf + 1);
4195 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
4196 status = be_cmd_set_fn_privileges(adapter,
4197 vf_cfg->privileges |
4198 BE_PRIV_FILTMGMT,
4199 vf + 1);
4200 if (!status) {
4201 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
4202 dev_info(dev, "VF%d has FILTMGMT privilege\n",
4203 vf);
4204 }
4205 }
4206
4207 /* Allow full available bandwidth */
4208 if (!old_vfs)
4209 be_cmd_config_qos(adapter, 0, 0, vf + 1);
4210
4211 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1,
4212 vf_cfg->if_handle, NULL,
4213 &spoofchk);
4214 if (!status)
4215 vf_cfg->spoofchk = spoofchk;
4216
4217 if (!old_vfs) {
4218 be_cmd_enable_vf(adapter, vf + 1);
4219 be_cmd_set_logical_link_config(adapter,
4220 IFLA_VF_LINK_STATE_AUTO,
4221 vf+1);
4222 }
4223 }
4224
4225 if (!old_vfs) {
4226 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
4227 if (status) {
4228 dev_err(dev, "SRIOV enable failed\n");
4229 adapter->num_vfs = 0;
4230 goto err;
4231 }
4232 }
4233
4234 if (BE3_chip(adapter)) {
4235 /* On BE3, enable VEB only when SRIOV is enabled */
4236 status = be_cmd_set_hsw_config(adapter, 0, 0,
4237 adapter->if_handle,
4238 PORT_FWD_TYPE_VEB, 0);
4239 if (status)
4240 goto err;
4241 }
4242
4243 adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
4244 return 0;
4245 err:
4246 dev_err(dev, "VF setup failed\n");
4247 be_vf_clear(adapter);
4248 return status;
4249 }
4250
4251 /* Converting function_mode bits on BE3 to SH mc_type enums */
4252
4253 static u8 be_convert_mc_type(u32 function_mode)
4254 {
4255 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
4256 return vNIC1;
4257 else if (function_mode & QNQ_MODE)
4258 return FLEX10;
4259 else if (function_mode & VNIC_MODE)
4260 return vNIC2;
4261 else if (function_mode & UMC_ENABLED)
4262 return UMC;
4263 else
4264 return MC_NONE;
4265 }
4266
4267 /* On BE2/BE3 FW does not suggest the supported limits */
4268 static void BEx_get_resources(struct be_adapter *adapter,
4269 struct be_resources *res)
4270 {
4271 bool use_sriov = adapter->num_vfs ? 1 : 0;
4272
4273 if (be_physfn(adapter))
4274 res->max_uc_mac = BE_UC_PMAC_COUNT;
4275 else
4276 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
4277
4278 adapter->mc_type = be_convert_mc_type(adapter->function_mode);
4279
4280 if (be_is_mc(adapter)) {
4281 /* Assuming that there are 4 channels per port,
4282 * when multi-channel is enabled
4283 */
4284 if (be_is_qnq_mode(adapter))
4285 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
4286 else
4287 /* In a non-qnq multichannel mode, the pvid
4288 * takes up one vlan entry
4289 */
4290 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
4291 } else {
4292 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
4293 }
4294
4295 res->max_mcast_mac = BE_MAX_MC;
4296
4297 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
4298 * 2) Create multiple TX rings on a BE3-R multi-channel interface
4299 * *only* if it is RSS-capable.
4300 */
4301 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) ||
4302 be_virtfn(adapter) ||
4303 (be_is_mc(adapter) &&
4304 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
4305 res->max_tx_qs = 1;
4306 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
4307 struct be_resources super_nic_res = {0};
4308
4309 /* On a SuperNIC profile, the driver needs to use the
4310 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
4311 */
4312 be_cmd_get_profile_config(adapter, &super_nic_res, NULL,
4313 ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS,
4314 0);
4315 /* Some old versions of BE3 FW don't report max_tx_qs value */
4316 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
4317 } else {
4318 res->max_tx_qs = BE3_MAX_TX_QS;
4319 }
4320
4321 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
4322 !use_sriov && be_physfn(adapter))
4323 res->max_rss_qs = (adapter->be3_native) ?
4324 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
4325 res->max_rx_qs = res->max_rss_qs + 1;
4326
4327 if (be_physfn(adapter))
4328 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
4329 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
4330 else
4331 res->max_evt_qs = 1;
4332
4333 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
4334 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS;
4335 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
4336 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
4337 }
4338
4339 static void be_setup_init(struct be_adapter *adapter)
4340 {
4341 adapter->vlan_prio_bmap = 0xff;
4342 adapter->phy.link_speed = -1;
4343 adapter->if_handle = -1;
4344 adapter->be3_native = false;
4345 adapter->if_flags = 0;
4346 adapter->phy_state = BE_UNKNOWN_PHY_STATE;
4347 if (be_physfn(adapter))
4348 adapter->cmd_privileges = MAX_PRIVILEGES;
4349 else
4350 adapter->cmd_privileges = MIN_PRIVILEGES;
4351 }
4352
4353 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port.
4354 * However, this HW limitation is not exposed to the host via any SLI cmd.
4355 * As a result, in the case of SRIOV and in particular multi-partition configs
4356 * the driver needs to calcuate a proportional share of RSS Tables per PF-pool
4357 * for distribution between the VFs. This self-imposed limit will determine the
4358 * no: of VFs for which RSS can be enabled.
4359 */
4360 static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter)
4361 {
4362 struct be_port_resources port_res = {0};
4363 u8 rss_tables_on_port;
4364 u16 max_vfs = be_max_vfs(adapter);
4365
4366 be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE,
4367 RESOURCE_LIMITS, 0);
4368
4369 rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs;
4370
4371 /* Each PF Pool's RSS Tables limit =
4372 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port
4373 */
4374 adapter->pool_res.max_rss_tables =
4375 max_vfs * rss_tables_on_port / port_res.max_vfs;
4376 }
4377
4378 static int be_get_sriov_config(struct be_adapter *adapter)
4379 {
4380 struct be_resources res = {0};
4381 int max_vfs, old_vfs;
4382
4383 be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE,
4384 RESOURCE_LIMITS, 0);
4385
4386 /* Some old versions of BE3 FW don't report max_vfs value */
4387 if (BE3_chip(adapter) && !res.max_vfs) {
4388 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
4389 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
4390 }
4391
4392 adapter->pool_res = res;
4393
4394 /* If during previous unload of the driver, the VFs were not disabled,
4395 * then we cannot rely on the PF POOL limits for the TotalVFs value.
4396 * Instead use the TotalVFs value stored in the pci-dev struct.
4397 */
4398 old_vfs = pci_num_vf(adapter->pdev);
4399 if (old_vfs) {
4400 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n",
4401 old_vfs);
4402
4403 adapter->pool_res.max_vfs =
4404 pci_sriov_get_totalvfs(adapter->pdev);
4405 adapter->num_vfs = old_vfs;
4406 }
4407
4408 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
4409 be_calculate_pf_pool_rss_tables(adapter);
4410 dev_info(&adapter->pdev->dev,
4411 "RSS can be enabled for all VFs if num_vfs <= %d\n",
4412 be_max_pf_pool_rss_tables(adapter));
4413 }
4414 return 0;
4415 }
4416
4417 static void be_alloc_sriov_res(struct be_adapter *adapter)
4418 {
4419 int old_vfs = pci_num_vf(adapter->pdev);
4420 struct be_resources vft_res = {0};
4421 int status;
4422
4423 be_get_sriov_config(adapter);
4424
4425 if (!old_vfs)
4426 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
4427
4428 /* When the HW is in SRIOV capable configuration, the PF-pool
4429 * resources are given to PF during driver load, if there are no
4430 * old VFs. This facility is not available in BE3 FW.
4431 * Also, this is done by FW in Lancer chip.
4432 */
4433 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
4434 be_calculate_vf_res(adapter, 0, &vft_res);
4435 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0,
4436 &vft_res);
4437 if (status)
4438 dev_err(&adapter->pdev->dev,
4439 "Failed to optimize SRIOV resources\n");
4440 }
4441 }
4442
4443 static int be_get_resources(struct be_adapter *adapter)
4444 {
4445 struct device *dev = &adapter->pdev->dev;
4446 struct be_resources res = {0};
4447 int status;
4448
4449 /* For Lancer, SH etc read per-function resource limits from FW.
4450 * GET_FUNC_CONFIG returns per function guaranteed limits.
4451 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
4452 */
4453 if (BEx_chip(adapter)) {
4454 BEx_get_resources(adapter, &res);
4455 } else {
4456 status = be_cmd_get_func_config(adapter, &res);
4457 if (status)
4458 return status;
4459
4460 /* If a deafault RXQ must be created, we'll use up one RSSQ*/
4461 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs &&
4462 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS))
4463 res.max_rss_qs -= 1;
4464 }
4465
4466 /* If RoCE is supported stash away half the EQs for RoCE */
4467 res.max_nic_evt_qs = be_roce_supported(adapter) ?
4468 res.max_evt_qs / 2 : res.max_evt_qs;
4469 adapter->res = res;
4470
4471 /* If FW supports RSS default queue, then skip creating non-RSS
4472 * queue for non-IP traffic.
4473 */
4474 adapter->need_def_rxq = (be_if_cap_flags(adapter) &
4475 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1;
4476
4477 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
4478 be_max_txqs(adapter), be_max_rxqs(adapter),
4479 be_max_rss(adapter), be_max_nic_eqs(adapter),
4480 be_max_vfs(adapter));
4481 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
4482 be_max_uc(adapter), be_max_mc(adapter),
4483 be_max_vlans(adapter));
4484
4485 /* Ensure RX and TX queues are created in pairs at init time */
4486 adapter->cfg_num_rx_irqs =
4487 min_t(u16, netif_get_num_default_rss_queues(),
4488 be_max_qp_irqs(adapter));
4489 adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs;
4490 return 0;
4491 }
4492
4493 static int be_get_config(struct be_adapter *adapter)
4494 {
4495 int status, level;
4496 u16 profile_id;
4497
4498 status = be_cmd_get_cntl_attributes(adapter);
4499 if (status)
4500 return status;
4501
4502 status = be_cmd_query_fw_cfg(adapter);
4503 if (status)
4504 return status;
4505
4506 if (!lancer_chip(adapter) && be_physfn(adapter))
4507 be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len);
4508
4509 if (BEx_chip(adapter)) {
4510 level = be_cmd_get_fw_log_level(adapter);
4511 adapter->msg_enable =
4512 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4513 }
4514
4515 be_cmd_get_acpi_wol_cap(adapter);
4516 pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en);
4517 pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en);
4518
4519 be_cmd_query_port_name(adapter);
4520
4521 if (be_physfn(adapter)) {
4522 status = be_cmd_get_active_profile(adapter, &profile_id);
4523 if (!status)
4524 dev_info(&adapter->pdev->dev,
4525 "Using profile 0x%x\n", profile_id);
4526 }
4527
4528 return 0;
4529 }
4530
4531 static int be_mac_setup(struct be_adapter *adapter)
4532 {
4533 u8 mac[ETH_ALEN];
4534 int status;
4535
4536 if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
4537 status = be_cmd_get_perm_mac(adapter, mac);
4538 if (status)
4539 return status;
4540
4541 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
4542 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
4543 }
4544
4545 return 0;
4546 }
4547
4548 static void be_schedule_worker(struct be_adapter *adapter)
4549 {
4550 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
4551 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
4552 }
4553
4554 static void be_destroy_err_recovery_workq(void)
4555 {
4556 if (!be_err_recovery_workq)
4557 return;
4558
4559 flush_workqueue(be_err_recovery_workq);
4560 destroy_workqueue(be_err_recovery_workq);
4561 be_err_recovery_workq = NULL;
4562 }
4563
4564 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay)
4565 {
4566 struct be_error_recovery *err_rec = &adapter->error_recovery;
4567
4568 if (!be_err_recovery_workq)
4569 return;
4570
4571 queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work,
4572 msecs_to_jiffies(delay));
4573 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED;
4574 }
4575
4576 static int be_setup_queues(struct be_adapter *adapter)
4577 {
4578 struct net_device *netdev = adapter->netdev;
4579 int status;
4580
4581 status = be_evt_queues_create(adapter);
4582 if (status)
4583 goto err;
4584
4585 status = be_tx_qs_create(adapter);
4586 if (status)
4587 goto err;
4588
4589 status = be_rx_cqs_create(adapter);
4590 if (status)
4591 goto err;
4592
4593 status = be_mcc_queues_create(adapter);
4594 if (status)
4595 goto err;
4596
4597 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
4598 if (status)
4599 goto err;
4600
4601 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
4602 if (status)
4603 goto err;
4604
4605 return 0;
4606 err:
4607 dev_err(&adapter->pdev->dev, "queue_setup failed\n");
4608 return status;
4609 }
4610
4611 static int be_if_create(struct be_adapter *adapter)
4612 {
4613 u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS;
4614 u32 cap_flags = be_if_cap_flags(adapter);
4615 int status;
4616
4617 /* alloc required memory for other filtering fields */
4618 adapter->pmac_id = kcalloc(be_max_uc(adapter),
4619 sizeof(*adapter->pmac_id), GFP_KERNEL);
4620 if (!adapter->pmac_id)
4621 return -ENOMEM;
4622
4623 adapter->mc_list = kcalloc(be_max_mc(adapter),
4624 sizeof(*adapter->mc_list), GFP_KERNEL);
4625 if (!adapter->mc_list)
4626 return -ENOMEM;
4627
4628 adapter->uc_list = kcalloc(be_max_uc(adapter),
4629 sizeof(*adapter->uc_list), GFP_KERNEL);
4630 if (!adapter->uc_list)
4631 return -ENOMEM;
4632
4633 if (adapter->cfg_num_rx_irqs == 1)
4634 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS);
4635
4636 en_flags &= cap_flags;
4637 /* will enable all the needed filter flags in be_open() */
4638 status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
4639 &adapter->if_handle, 0);
4640
4641 if (status)
4642 return status;
4643
4644 return 0;
4645 }
4646
4647 int be_update_queues(struct be_adapter *adapter)
4648 {
4649 struct net_device *netdev = adapter->netdev;
4650 int status;
4651
4652 if (netif_running(netdev))
4653 be_close(netdev);
4654
4655 be_cancel_worker(adapter);
4656
4657 /* If any vectors have been shared with RoCE we cannot re-program
4658 * the MSIx table.
4659 */
4660 if (!adapter->num_msix_roce_vec)
4661 be_msix_disable(adapter);
4662
4663 be_clear_queues(adapter);
4664 status = be_cmd_if_destroy(adapter, adapter->if_handle, 0);
4665 if (status)
4666 return status;
4667
4668 if (!msix_enabled(adapter)) {
4669 status = be_msix_enable(adapter);
4670 if (status)
4671 return status;
4672 }
4673
4674 status = be_if_create(adapter);
4675 if (status)
4676 return status;
4677
4678 status = be_setup_queues(adapter);
4679 if (status)
4680 return status;
4681
4682 be_schedule_worker(adapter);
4683
4684 if (netif_running(netdev))
4685 status = be_open(netdev);
4686
4687 return status;
4688 }
4689
4690 static inline int fw_major_num(const char *fw_ver)
4691 {
4692 int fw_major = 0, i;
4693
4694 i = sscanf(fw_ver, "%d.", &fw_major);
4695 if (i != 1)
4696 return 0;
4697
4698 return fw_major;
4699 }
4700
4701 /* If it is error recovery, FLR the PF
4702 * Else if any VFs are already enabled don't FLR the PF
4703 */
4704 static bool be_reset_required(struct be_adapter *adapter)
4705 {
4706 if (be_error_recovering(adapter))
4707 return true;
4708 else
4709 return pci_num_vf(adapter->pdev) == 0;
4710 }
4711
4712 /* Wait for the FW to be ready and perform the required initialization */
4713 static int be_func_init(struct be_adapter *adapter)
4714 {
4715 int status;
4716
4717 status = be_fw_wait_ready(adapter);
4718 if (status)
4719 return status;
4720
4721 /* FW is now ready; clear errors to allow cmds/doorbell */
4722 be_clear_error(adapter, BE_CLEAR_ALL);
4723
4724 if (be_reset_required(adapter)) {
4725 status = be_cmd_reset_function(adapter);
4726 if (status)
4727 return status;
4728
4729 /* Wait for interrupts to quiesce after an FLR */
4730 msleep(100);
4731 }
4732
4733 /* Tell FW we're ready to fire cmds */
4734 status = be_cmd_fw_init(adapter);
4735 if (status)
4736 return status;
4737
4738 /* Allow interrupts for other ULPs running on NIC function */
4739 be_intr_set(adapter, true);
4740
4741 return 0;
4742 }
4743
4744 static int be_setup(struct be_adapter *adapter)
4745 {
4746 struct device *dev = &adapter->pdev->dev;
4747 int status;
4748
4749 status = be_func_init(adapter);
4750 if (status)
4751 return status;
4752
4753 be_setup_init(adapter);
4754
4755 if (!lancer_chip(adapter))
4756 be_cmd_req_native_mode(adapter);
4757
4758 /* invoke this cmd first to get pf_num and vf_num which are needed
4759 * for issuing profile related cmds
4760 */
4761 if (!BEx_chip(adapter)) {
4762 status = be_cmd_get_func_config(adapter, NULL);
4763 if (status)
4764 return status;
4765 }
4766
4767 status = be_get_config(adapter);
4768 if (status)
4769 goto err;
4770
4771 if (!BE2_chip(adapter) && be_physfn(adapter))
4772 be_alloc_sriov_res(adapter);
4773
4774 status = be_get_resources(adapter);
4775 if (status)
4776 goto err;
4777
4778 status = be_msix_enable(adapter);
4779 if (status)
4780 goto err;
4781
4782 /* will enable all the needed filter flags in be_open() */
4783 status = be_if_create(adapter);
4784 if (status)
4785 goto err;
4786
4787 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
4788 rtnl_lock();
4789 status = be_setup_queues(adapter);
4790 rtnl_unlock();
4791 if (status)
4792 goto err;
4793
4794 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
4795
4796 status = be_mac_setup(adapter);
4797 if (status)
4798 goto err;
4799
4800 be_cmd_get_fw_ver(adapter);
4801 dev_info(dev, "FW version is %s\n", adapter->fw_ver);
4802
4803 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
4804 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
4805 adapter->fw_ver);
4806 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
4807 }
4808
4809 status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
4810 adapter->rx_fc);
4811 if (status)
4812 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
4813 &adapter->rx_fc);
4814
4815 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
4816 adapter->tx_fc, adapter->rx_fc);
4817
4818 if (be_physfn(adapter))
4819 be_cmd_set_logical_link_config(adapter,
4820 IFLA_VF_LINK_STATE_AUTO, 0);
4821
4822 /* BE3 EVB echoes broadcast/multicast packets back to PF's vport
4823 * confusing a linux bridge or OVS that it might be connected to.
4824 * Set the EVB to PASSTHRU mode which effectively disables the EVB
4825 * when SRIOV is not enabled.
4826 */
4827 if (BE3_chip(adapter))
4828 be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle,
4829 PORT_FWD_TYPE_PASSTHRU, 0);
4830
4831 if (adapter->num_vfs)
4832 be_vf_setup(adapter);
4833
4834 status = be_cmd_get_phy_info(adapter);
4835 if (!status && be_pause_supported(adapter))
4836 adapter->phy.fc_autoneg = 1;
4837
4838 if (be_physfn(adapter) && !lancer_chip(adapter))
4839 be_cmd_set_features(adapter);
4840
4841 be_schedule_worker(adapter);
4842 adapter->flags |= BE_FLAGS_SETUP_DONE;
4843 return 0;
4844 err:
4845 be_clear(adapter);
4846 return status;
4847 }
4848
4849 #ifdef CONFIG_NET_POLL_CONTROLLER
4850 static void be_netpoll(struct net_device *netdev)
4851 {
4852 struct be_adapter *adapter = netdev_priv(netdev);
4853 struct be_eq_obj *eqo;
4854 int i;
4855
4856 for_all_evt_queues(adapter, eqo, i) {
4857 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
4858 napi_schedule(&eqo->napi);
4859 }
4860 }
4861 #endif
4862
4863 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4864 {
4865 const struct firmware *fw;
4866 int status;
4867
4868 if (!netif_running(adapter->netdev)) {
4869 dev_err(&adapter->pdev->dev,
4870 "Firmware load not allowed (interface is down)\n");
4871 return -ENETDOWN;
4872 }
4873
4874 status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4875 if (status)
4876 goto fw_exit;
4877
4878 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4879
4880 if (lancer_chip(adapter))
4881 status = lancer_fw_download(adapter, fw);
4882 else
4883 status = be_fw_download(adapter, fw);
4884
4885 if (!status)
4886 be_cmd_get_fw_ver(adapter);
4887
4888 fw_exit:
4889 release_firmware(fw);
4890 return status;
4891 }
4892
4893 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4894 u16 flags)
4895 {
4896 struct be_adapter *adapter = netdev_priv(dev);
4897 struct nlattr *attr, *br_spec;
4898 int rem;
4899 int status = 0;
4900 u16 mode = 0;
4901
4902 if (!sriov_enabled(adapter))
4903 return -EOPNOTSUPP;
4904
4905 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4906 if (!br_spec)
4907 return -EINVAL;
4908
4909 nla_for_each_nested(attr, br_spec, rem) {
4910 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4911 continue;
4912
4913 if (nla_len(attr) < sizeof(mode))
4914 return -EINVAL;
4915
4916 mode = nla_get_u16(attr);
4917 if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA)
4918 return -EOPNOTSUPP;
4919
4920 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4921 return -EINVAL;
4922
4923 status = be_cmd_set_hsw_config(adapter, 0, 0,
4924 adapter->if_handle,
4925 mode == BRIDGE_MODE_VEPA ?
4926 PORT_FWD_TYPE_VEPA :
4927 PORT_FWD_TYPE_VEB, 0);
4928 if (status)
4929 goto err;
4930
4931 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4932 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4933
4934 return status;
4935 }
4936 err:
4937 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4938 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4939
4940 return status;
4941 }
4942
4943 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4944 struct net_device *dev, u32 filter_mask,
4945 int nlflags)
4946 {
4947 struct be_adapter *adapter = netdev_priv(dev);
4948 int status = 0;
4949 u8 hsw_mode;
4950
4951 /* BE and Lancer chips support VEB mode only */
4952 if (BEx_chip(adapter) || lancer_chip(adapter)) {
4953 /* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */
4954 if (!pci_sriov_get_totalvfs(adapter->pdev))
4955 return 0;
4956 hsw_mode = PORT_FWD_TYPE_VEB;
4957 } else {
4958 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4959 adapter->if_handle, &hsw_mode,
4960 NULL);
4961 if (status)
4962 return 0;
4963
4964 if (hsw_mode == PORT_FWD_TYPE_PASSTHRU)
4965 return 0;
4966 }
4967
4968 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4969 hsw_mode == PORT_FWD_TYPE_VEPA ?
4970 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4971 0, 0, nlflags, filter_mask, NULL);
4972 }
4973
4974 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter,
4975 void (*func)(struct work_struct *))
4976 {
4977 struct be_cmd_work *work;
4978
4979 work = kzalloc(sizeof(*work), GFP_ATOMIC);
4980 if (!work) {
4981 dev_err(&adapter->pdev->dev,
4982 "be_work memory allocation failed\n");
4983 return NULL;
4984 }
4985
4986 INIT_WORK(&work->work, func);
4987 work->adapter = adapter;
4988 return work;
4989 }
4990
4991 /* VxLAN offload Notes:
4992 *
4993 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4994 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4995 * is expected to work across all types of IP tunnels once exported. Skyhawk
4996 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4997 * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4998 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4999 * those other tunnels are unexported on the fly through ndo_features_check().
5000 *
5001 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
5002 * adds more than one port, disable offloads and don't re-enable them again
5003 * until after all the tunnels are removed.
5004 */
5005 static void be_work_add_vxlan_port(struct work_struct *work)
5006 {
5007 struct be_cmd_work *cmd_work =
5008 container_of(work, struct be_cmd_work, work);
5009 struct be_adapter *adapter = cmd_work->adapter;
5010 struct net_device *netdev = adapter->netdev;
5011 struct device *dev = &adapter->pdev->dev;
5012 __be16 port = cmd_work->info.vxlan_port;
5013 int status;
5014
5015 if (adapter->vxlan_port == port && adapter->vxlan_port_count) {
5016 adapter->vxlan_port_aliases++;
5017 goto done;
5018 }
5019
5020 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
5021 dev_info(dev,
5022 "Only one UDP port supported for VxLAN offloads\n");
5023 dev_info(dev, "Disabling VxLAN offloads\n");
5024 adapter->vxlan_port_count++;
5025 goto err;
5026 }
5027
5028 if (adapter->vxlan_port_count++ >= 1)
5029 goto done;
5030
5031 status = be_cmd_manage_iface(adapter, adapter->if_handle,
5032 OP_CONVERT_NORMAL_TO_TUNNEL);
5033 if (status) {
5034 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
5035 goto err;
5036 }
5037
5038 status = be_cmd_set_vxlan_port(adapter, port);
5039 if (status) {
5040 dev_warn(dev, "Failed to add VxLAN port\n");
5041 goto err;
5042 }
5043 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
5044 adapter->vxlan_port = port;
5045
5046 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
5047 NETIF_F_TSO | NETIF_F_TSO6 |
5048 NETIF_F_GSO_UDP_TUNNEL;
5049 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
5050 netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
5051
5052 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
5053 be16_to_cpu(port));
5054 goto done;
5055 err:
5056 be_disable_vxlan_offloads(adapter);
5057 done:
5058 kfree(cmd_work);
5059 }
5060
5061 static void be_work_del_vxlan_port(struct work_struct *work)
5062 {
5063 struct be_cmd_work *cmd_work =
5064 container_of(work, struct be_cmd_work, work);
5065 struct be_adapter *adapter = cmd_work->adapter;
5066 __be16 port = cmd_work->info.vxlan_port;
5067
5068 if (adapter->vxlan_port != port)
5069 goto done;
5070
5071 if (adapter->vxlan_port_aliases) {
5072 adapter->vxlan_port_aliases--;
5073 goto out;
5074 }
5075
5076 be_disable_vxlan_offloads(adapter);
5077
5078 dev_info(&adapter->pdev->dev,
5079 "Disabled VxLAN offloads for UDP port %d\n",
5080 be16_to_cpu(port));
5081 done:
5082 adapter->vxlan_port_count--;
5083 out:
5084 kfree(cmd_work);
5085 }
5086
5087 static void be_cfg_vxlan_port(struct net_device *netdev,
5088 struct udp_tunnel_info *ti,
5089 void (*func)(struct work_struct *))
5090 {
5091 struct be_adapter *adapter = netdev_priv(netdev);
5092 struct be_cmd_work *cmd_work;
5093
5094 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
5095 return;
5096
5097 if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
5098 return;
5099
5100 cmd_work = be_alloc_work(adapter, func);
5101 if (cmd_work) {
5102 cmd_work->info.vxlan_port = ti->port;
5103 queue_work(be_wq, &cmd_work->work);
5104 }
5105 }
5106
5107 static void be_del_vxlan_port(struct net_device *netdev,
5108 struct udp_tunnel_info *ti)
5109 {
5110 be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port);
5111 }
5112
5113 static void be_add_vxlan_port(struct net_device *netdev,
5114 struct udp_tunnel_info *ti)
5115 {
5116 be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port);
5117 }
5118
5119 static netdev_features_t be_features_check(struct sk_buff *skb,
5120 struct net_device *dev,
5121 netdev_features_t features)
5122 {
5123 struct be_adapter *adapter = netdev_priv(dev);
5124 u8 l4_hdr = 0;
5125
5126 /* The code below restricts offload features for some tunneled packets.
5127 * Offload features for normal (non tunnel) packets are unchanged.
5128 */
5129 if (!skb->encapsulation ||
5130 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
5131 return features;
5132
5133 /* It's an encapsulated packet and VxLAN offloads are enabled. We
5134 * should disable tunnel offload features if it's not a VxLAN packet,
5135 * as tunnel offloads have been enabled only for VxLAN. This is done to
5136 * allow other tunneled traffic like GRE work fine while VxLAN
5137 * offloads are configured in Skyhawk-R.
5138 */
5139 switch (vlan_get_protocol(skb)) {
5140 case htons(ETH_P_IP):
5141 l4_hdr = ip_hdr(skb)->protocol;
5142 break;
5143 case htons(ETH_P_IPV6):
5144 l4_hdr = ipv6_hdr(skb)->nexthdr;
5145 break;
5146 default:
5147 return features;
5148 }
5149
5150 if (l4_hdr != IPPROTO_UDP ||
5151 skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
5152 skb->inner_protocol != htons(ETH_P_TEB) ||
5153 skb_inner_mac_header(skb) - skb_transport_header(skb) !=
5154 sizeof(struct udphdr) + sizeof(struct vxlanhdr) ||
5155 !adapter->vxlan_port ||
5156 udp_hdr(skb)->dest != adapter->vxlan_port)
5157 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
5158
5159 return features;
5160 }
5161
5162 static int be_get_phys_port_id(struct net_device *dev,
5163 struct netdev_phys_item_id *ppid)
5164 {
5165 int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1;
5166 struct be_adapter *adapter = netdev_priv(dev);
5167 u8 *id;
5168
5169 if (MAX_PHYS_ITEM_ID_LEN < id_len)
5170 return -ENOSPC;
5171
5172 ppid->id[0] = adapter->hba_port_num + 1;
5173 id = &ppid->id[1];
5174 for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0;
5175 i--, id += CNTL_SERIAL_NUM_WORD_SZ)
5176 memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ);
5177
5178 ppid->id_len = id_len;
5179
5180 return 0;
5181 }
5182
5183 static void be_set_rx_mode(struct net_device *dev)
5184 {
5185 struct be_adapter *adapter = netdev_priv(dev);
5186 struct be_cmd_work *work;
5187
5188 work = be_alloc_work(adapter, be_work_set_rx_mode);
5189 if (work)
5190 queue_work(be_wq, &work->work);
5191 }
5192
5193 static const struct net_device_ops be_netdev_ops = {
5194 .ndo_open = be_open,
5195 .ndo_stop = be_close,
5196 .ndo_start_xmit = be_xmit,
5197 .ndo_set_rx_mode = be_set_rx_mode,
5198 .ndo_set_mac_address = be_mac_addr_set,
5199 .ndo_get_stats64 = be_get_stats64,
5200 .ndo_validate_addr = eth_validate_addr,
5201 .ndo_vlan_rx_add_vid = be_vlan_add_vid,
5202 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid,
5203 .ndo_set_vf_mac = be_set_vf_mac,
5204 .ndo_set_vf_vlan = be_set_vf_vlan,
5205 .ndo_set_vf_rate = be_set_vf_tx_rate,
5206 .ndo_get_vf_config = be_get_vf_config,
5207 .ndo_set_vf_link_state = be_set_vf_link_state,
5208 .ndo_set_vf_spoofchk = be_set_vf_spoofchk,
5209 #ifdef CONFIG_NET_POLL_CONTROLLER
5210 .ndo_poll_controller = be_netpoll,
5211 #endif
5212 .ndo_bridge_setlink = be_ndo_bridge_setlink,
5213 .ndo_bridge_getlink = be_ndo_bridge_getlink,
5214 #ifdef CONFIG_NET_RX_BUSY_POLL
5215 .ndo_busy_poll = be_busy_poll,
5216 #endif
5217 .ndo_udp_tunnel_add = be_add_vxlan_port,
5218 .ndo_udp_tunnel_del = be_del_vxlan_port,
5219 .ndo_features_check = be_features_check,
5220 .ndo_get_phys_port_id = be_get_phys_port_id,
5221 };
5222
5223 static void be_netdev_init(struct net_device *netdev)
5224 {
5225 struct be_adapter *adapter = netdev_priv(netdev);
5226
5227 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5228 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
5229 NETIF_F_HW_VLAN_CTAG_TX;
5230 if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS))
5231 netdev->hw_features |= NETIF_F_RXHASH;
5232
5233 netdev->features |= netdev->hw_features |
5234 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
5235
5236 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5237 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
5238
5239 netdev->priv_flags |= IFF_UNICAST_FLT;
5240
5241 netdev->flags |= IFF_MULTICAST;
5242
5243 netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN);
5244
5245 netdev->netdev_ops = &be_netdev_ops;
5246
5247 netdev->ethtool_ops = &be_ethtool_ops;
5248
5249 /* MTU range: 256 - 9000 */
5250 netdev->min_mtu = BE_MIN_MTU;
5251 netdev->max_mtu = BE_MAX_MTU;
5252 }
5253
5254 static void be_cleanup(struct be_adapter *adapter)
5255 {
5256 struct net_device *netdev = adapter->netdev;
5257
5258 rtnl_lock();
5259 netif_device_detach(netdev);
5260 if (netif_running(netdev))
5261 be_close(netdev);
5262 rtnl_unlock();
5263
5264 be_clear(adapter);
5265 }
5266
5267 static int be_resume(struct be_adapter *adapter)
5268 {
5269 struct net_device *netdev = adapter->netdev;
5270 int status;
5271
5272 status = be_setup(adapter);
5273 if (status)
5274 return status;
5275
5276 rtnl_lock();
5277 if (netif_running(netdev))
5278 status = be_open(netdev);
5279 rtnl_unlock();
5280
5281 if (status)
5282 return status;
5283
5284 netif_device_attach(netdev);
5285
5286 return 0;
5287 }
5288
5289 static void be_soft_reset(struct be_adapter *adapter)
5290 {
5291 u32 val;
5292
5293 dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n");
5294 val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
5295 val |= SLIPORT_SOFTRESET_SR_MASK;
5296 iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
5297 }
5298
5299 static bool be_err_is_recoverable(struct be_adapter *adapter)
5300 {
5301 struct be_error_recovery *err_rec = &adapter->error_recovery;
5302 unsigned long initial_idle_time =
5303 msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME);
5304 unsigned long recovery_interval =
5305 msecs_to_jiffies(ERR_RECOVERY_INTERVAL);
5306 u16 ue_err_code;
5307 u32 val;
5308
5309 val = be_POST_stage_get(adapter);
5310 if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR)
5311 return false;
5312 ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK;
5313 if (ue_err_code == 0)
5314 return false;
5315
5316 dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n",
5317 ue_err_code);
5318
5319 if (jiffies - err_rec->probe_time <= initial_idle_time) {
5320 dev_err(&adapter->pdev->dev,
5321 "Cannot recover within %lu sec from driver load\n",
5322 jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC);
5323 return false;
5324 }
5325
5326 if (err_rec->last_recovery_time &&
5327 (jiffies - err_rec->last_recovery_time <= recovery_interval)) {
5328 dev_err(&adapter->pdev->dev,
5329 "Cannot recover within %lu sec from last recovery\n",
5330 jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC);
5331 return false;
5332 }
5333
5334 if (ue_err_code == err_rec->last_err_code) {
5335 dev_err(&adapter->pdev->dev,
5336 "Cannot recover from a consecutive TPE error\n");
5337 return false;
5338 }
5339
5340 err_rec->last_recovery_time = jiffies;
5341 err_rec->last_err_code = ue_err_code;
5342 return true;
5343 }
5344
5345 static int be_tpe_recover(struct be_adapter *adapter)
5346 {
5347 struct be_error_recovery *err_rec = &adapter->error_recovery;
5348 int status = -EAGAIN;
5349 u32 val;
5350
5351 switch (err_rec->recovery_state) {
5352 case ERR_RECOVERY_ST_NONE:
5353 err_rec->recovery_state = ERR_RECOVERY_ST_DETECT;
5354 err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION;
5355 break;
5356
5357 case ERR_RECOVERY_ST_DETECT:
5358 val = be_POST_stage_get(adapter);
5359 if ((val & POST_STAGE_RECOVERABLE_ERR) !=
5360 POST_STAGE_RECOVERABLE_ERR) {
5361 dev_err(&adapter->pdev->dev,
5362 "Unrecoverable HW error detected: 0x%x\n", val);
5363 status = -EINVAL;
5364 err_rec->resched_delay = 0;
5365 break;
5366 }
5367
5368 dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n");
5369
5370 /* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR
5371 * milliseconds before it checks for final error status in
5372 * SLIPORT_SEMAPHORE to determine if recovery criteria is met.
5373 * If it does, then PF0 initiates a Soft Reset.
5374 */
5375 if (adapter->pf_num == 0) {
5376 err_rec->recovery_state = ERR_RECOVERY_ST_RESET;
5377 err_rec->resched_delay = err_rec->ue_to_reset_time -
5378 ERR_RECOVERY_UE_DETECT_DURATION;
5379 break;
5380 }
5381
5382 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
5383 err_rec->resched_delay = err_rec->ue_to_poll_time -
5384 ERR_RECOVERY_UE_DETECT_DURATION;
5385 break;
5386
5387 case ERR_RECOVERY_ST_RESET:
5388 if (!be_err_is_recoverable(adapter)) {
5389 dev_err(&adapter->pdev->dev,
5390 "Failed to meet recovery criteria\n");
5391 status = -EIO;
5392 err_rec->resched_delay = 0;
5393 break;
5394 }
5395 be_soft_reset(adapter);
5396 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
5397 err_rec->resched_delay = err_rec->ue_to_poll_time -
5398 err_rec->ue_to_reset_time;
5399 break;
5400
5401 case ERR_RECOVERY_ST_PRE_POLL:
5402 err_rec->recovery_state = ERR_RECOVERY_ST_REINIT;
5403 err_rec->resched_delay = 0;
5404 status = 0; /* done */
5405 break;
5406
5407 default:
5408 status = -EINVAL;
5409 err_rec->resched_delay = 0;
5410 break;
5411 }
5412
5413 return status;
5414 }
5415
5416 static int be_err_recover(struct be_adapter *adapter)
5417 {
5418 int status;
5419
5420 if (!lancer_chip(adapter)) {
5421 if (!adapter->error_recovery.recovery_supported ||
5422 adapter->priv_flags & BE_DISABLE_TPE_RECOVERY)
5423 return -EIO;
5424 status = be_tpe_recover(adapter);
5425 if (status)
5426 goto err;
5427 }
5428
5429 /* Wait for adapter to reach quiescent state before
5430 * destroying queues
5431 */
5432 status = be_fw_wait_ready(adapter);
5433 if (status)
5434 goto err;
5435
5436 adapter->flags |= BE_FLAGS_TRY_RECOVERY;
5437
5438 be_cleanup(adapter);
5439
5440 status = be_resume(adapter);
5441 if (status)
5442 goto err;
5443
5444 adapter->flags &= ~BE_FLAGS_TRY_RECOVERY;
5445
5446 err:
5447 return status;
5448 }
5449
5450 static void be_err_detection_task(struct work_struct *work)
5451 {
5452 struct be_error_recovery *err_rec =
5453 container_of(work, struct be_error_recovery,
5454 err_detection_work.work);
5455 struct be_adapter *adapter =
5456 container_of(err_rec, struct be_adapter,
5457 error_recovery);
5458 u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY;
5459 struct device *dev = &adapter->pdev->dev;
5460 int recovery_status;
5461
5462 be_detect_error(adapter);
5463 if (!be_check_error(adapter, BE_ERROR_HW))
5464 goto reschedule_task;
5465
5466 recovery_status = be_err_recover(adapter);
5467 if (!recovery_status) {
5468 err_rec->recovery_retries = 0;
5469 err_rec->recovery_state = ERR_RECOVERY_ST_NONE;
5470 dev_info(dev, "Adapter recovery successful\n");
5471 goto reschedule_task;
5472 } else if (!lancer_chip(adapter) && err_rec->resched_delay) {
5473 /* BEx/SH recovery state machine */
5474 if (adapter->pf_num == 0 &&
5475 err_rec->recovery_state > ERR_RECOVERY_ST_DETECT)
5476 dev_err(&adapter->pdev->dev,
5477 "Adapter recovery in progress\n");
5478 resched_delay = err_rec->resched_delay;
5479 goto reschedule_task;
5480 } else if (lancer_chip(adapter) && be_virtfn(adapter)) {
5481 /* For VFs, check if PF have allocated resources
5482 * every second.
5483 */
5484 dev_err(dev, "Re-trying adapter recovery\n");
5485 goto reschedule_task;
5486 } else if (lancer_chip(adapter) && err_rec->recovery_retries++ <
5487 ERR_RECOVERY_MAX_RETRY_COUNT) {
5488 /* In case of another error during recovery, it takes 30 sec
5489 * for adapter to come out of error. Retry error recovery after
5490 * this time interval.
5491 */
5492 dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n");
5493 resched_delay = ERR_RECOVERY_RETRY_DELAY;
5494 goto reschedule_task;
5495 } else {
5496 dev_err(dev, "Adapter recovery failed\n");
5497 dev_err(dev, "Please reboot server to recover\n");
5498 }
5499
5500 return;
5501
5502 reschedule_task:
5503 be_schedule_err_detection(adapter, resched_delay);
5504 }
5505
5506 static void be_log_sfp_info(struct be_adapter *adapter)
5507 {
5508 int status;
5509
5510 status = be_cmd_query_sfp_info(adapter);
5511 if (!status) {
5512 dev_err(&adapter->pdev->dev,
5513 "Port %c: %s Vendor: %s part no: %s",
5514 adapter->port_name,
5515 be_misconfig_evt_port_state[adapter->phy_state],
5516 adapter->phy.vendor_name,
5517 adapter->phy.vendor_pn);
5518 }
5519 adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED;
5520 }
5521
5522 static void be_worker(struct work_struct *work)
5523 {
5524 struct be_adapter *adapter =
5525 container_of(work, struct be_adapter, work.work);
5526 struct be_rx_obj *rxo;
5527 int i;
5528
5529 if (be_physfn(adapter) &&
5530 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5531 be_cmd_get_die_temperature(adapter);
5532
5533 /* when interrupts are not yet enabled, just reap any pending
5534 * mcc completions
5535 */
5536 if (!netif_running(adapter->netdev)) {
5537 local_bh_disable();
5538 be_process_mcc(adapter);
5539 local_bh_enable();
5540 goto reschedule;
5541 }
5542
5543 if (!adapter->stats_cmd_sent) {
5544 if (lancer_chip(adapter))
5545 lancer_cmd_get_pport_stats(adapter,
5546 &adapter->stats_cmd);
5547 else
5548 be_cmd_get_stats(adapter, &adapter->stats_cmd);
5549 }
5550
5551 for_all_rx_queues(adapter, rxo, i) {
5552 /* Replenish RX-queues starved due to memory
5553 * allocation failures.
5554 */
5555 if (rxo->rx_post_starved)
5556 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5557 }
5558
5559 /* EQ-delay update for Skyhawk is done while notifying EQ */
5560 if (!skyhawk_chip(adapter))
5561 be_eqd_update(adapter, false);
5562
5563 if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED)
5564 be_log_sfp_info(adapter);
5565
5566 reschedule:
5567 adapter->work_counter++;
5568 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
5569 }
5570
5571 static void be_unmap_pci_bars(struct be_adapter *adapter)
5572 {
5573 if (adapter->csr)
5574 pci_iounmap(adapter->pdev, adapter->csr);
5575 if (adapter->db)
5576 pci_iounmap(adapter->pdev, adapter->db);
5577 if (adapter->pcicfg && adapter->pcicfg_mapped)
5578 pci_iounmap(adapter->pdev, adapter->pcicfg);
5579 }
5580
5581 static int db_bar(struct be_adapter *adapter)
5582 {
5583 if (lancer_chip(adapter) || be_virtfn(adapter))
5584 return 0;
5585 else
5586 return 4;
5587 }
5588
5589 static int be_roce_map_pci_bars(struct be_adapter *adapter)
5590 {
5591 if (skyhawk_chip(adapter)) {
5592 adapter->roce_db.size = 4096;
5593 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
5594 db_bar(adapter));
5595 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
5596 db_bar(adapter));
5597 }
5598 return 0;
5599 }
5600
5601 static int be_map_pci_bars(struct be_adapter *adapter)
5602 {
5603 struct pci_dev *pdev = adapter->pdev;
5604 u8 __iomem *addr;
5605 u32 sli_intf;
5606
5607 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
5608 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
5609 SLI_INTF_FAMILY_SHIFT;
5610 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
5611
5612 if (BEx_chip(adapter) && be_physfn(adapter)) {
5613 adapter->csr = pci_iomap(pdev, 2, 0);
5614 if (!adapter->csr)
5615 return -ENOMEM;
5616 }
5617
5618 addr = pci_iomap(pdev, db_bar(adapter), 0);
5619 if (!addr)
5620 goto pci_map_err;
5621 adapter->db = addr;
5622
5623 if (skyhawk_chip(adapter) || BEx_chip(adapter)) {
5624 if (be_physfn(adapter)) {
5625 /* PCICFG is the 2nd BAR in BE2 */
5626 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0);
5627 if (!addr)
5628 goto pci_map_err;
5629 adapter->pcicfg = addr;
5630 adapter->pcicfg_mapped = true;
5631 } else {
5632 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
5633 adapter->pcicfg_mapped = false;
5634 }
5635 }
5636
5637 be_roce_map_pci_bars(adapter);
5638 return 0;
5639
5640 pci_map_err:
5641 dev_err(&pdev->dev, "Error in mapping PCI BARs\n");
5642 be_unmap_pci_bars(adapter);
5643 return -ENOMEM;
5644 }
5645
5646 static void be_drv_cleanup(struct be_adapter *adapter)
5647 {
5648 struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
5649 struct device *dev = &adapter->pdev->dev;
5650
5651 if (mem->va)
5652 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5653
5654 mem = &adapter->rx_filter;
5655 if (mem->va)
5656 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5657
5658 mem = &adapter->stats_cmd;
5659 if (mem->va)
5660 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5661 }
5662
5663 /* Allocate and initialize various fields in be_adapter struct */
5664 static int be_drv_init(struct be_adapter *adapter)
5665 {
5666 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
5667 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
5668 struct be_dma_mem *rx_filter = &adapter->rx_filter;
5669 struct be_dma_mem *stats_cmd = &adapter->stats_cmd;
5670 struct device *dev = &adapter->pdev->dev;
5671 int status = 0;
5672
5673 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
5674 mbox_mem_alloc->va = dma_zalloc_coherent(dev, mbox_mem_alloc->size,
5675 &mbox_mem_alloc->dma,
5676 GFP_KERNEL);
5677 if (!mbox_mem_alloc->va)
5678 return -ENOMEM;
5679
5680 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
5681 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
5682 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
5683
5684 rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
5685 rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size,
5686 &rx_filter->dma, GFP_KERNEL);
5687 if (!rx_filter->va) {
5688 status = -ENOMEM;
5689 goto free_mbox;
5690 }
5691
5692 if (lancer_chip(adapter))
5693 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5694 else if (BE2_chip(adapter))
5695 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5696 else if (BE3_chip(adapter))
5697 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5698 else
5699 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5700 stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size,
5701 &stats_cmd->dma, GFP_KERNEL);
5702 if (!stats_cmd->va) {
5703 status = -ENOMEM;
5704 goto free_rx_filter;
5705 }
5706
5707 mutex_init(&adapter->mbox_lock);
5708 mutex_init(&adapter->mcc_lock);
5709 mutex_init(&adapter->rx_filter_lock);
5710 spin_lock_init(&adapter->mcc_cq_lock);
5711 init_completion(&adapter->et_cmd_compl);
5712
5713 pci_save_state(adapter->pdev);
5714
5715 INIT_DELAYED_WORK(&adapter->work, be_worker);
5716
5717 adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE;
5718 adapter->error_recovery.resched_delay = 0;
5719 INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work,
5720 be_err_detection_task);
5721
5722 adapter->rx_fc = true;
5723 adapter->tx_fc = true;
5724
5725 /* Must be a power of 2 or else MODULO will BUG_ON */
5726 adapter->be_get_temp_freq = 64;
5727
5728 return 0;
5729
5730 free_rx_filter:
5731 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma);
5732 free_mbox:
5733 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va,
5734 mbox_mem_alloc->dma);
5735 return status;
5736 }
5737
5738 static void be_remove(struct pci_dev *pdev)
5739 {
5740 struct be_adapter *adapter = pci_get_drvdata(pdev);
5741
5742 if (!adapter)
5743 return;
5744
5745 be_roce_dev_remove(adapter);
5746 be_intr_set(adapter, false);
5747
5748 be_cancel_err_detection(adapter);
5749
5750 unregister_netdev(adapter->netdev);
5751
5752 be_clear(adapter);
5753
5754 if (!pci_vfs_assigned(adapter->pdev))
5755 be_cmd_reset_function(adapter);
5756
5757 /* tell fw we're done with firing cmds */
5758 be_cmd_fw_clean(adapter);
5759
5760 be_unmap_pci_bars(adapter);
5761 be_drv_cleanup(adapter);
5762
5763 pci_disable_pcie_error_reporting(pdev);
5764
5765 pci_release_regions(pdev);
5766 pci_disable_device(pdev);
5767
5768 free_netdev(adapter->netdev);
5769 }
5770
5771 static ssize_t be_hwmon_show_temp(struct device *dev,
5772 struct device_attribute *dev_attr,
5773 char *buf)
5774 {
5775 struct be_adapter *adapter = dev_get_drvdata(dev);
5776
5777 /* Unit: millidegree Celsius */
5778 if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP)
5779 return -EIO;
5780 else
5781 return sprintf(buf, "%u\n",
5782 adapter->hwmon_info.be_on_die_temp * 1000);
5783 }
5784
5785 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO,
5786 be_hwmon_show_temp, NULL, 1);
5787
5788 static struct attribute *be_hwmon_attrs[] = {
5789 &sensor_dev_attr_temp1_input.dev_attr.attr,
5790 NULL
5791 };
5792
5793 ATTRIBUTE_GROUPS(be_hwmon);
5794
5795 static char *mc_name(struct be_adapter *adapter)
5796 {
5797 char *str = ""; /* default */
5798
5799 switch (adapter->mc_type) {
5800 case UMC:
5801 str = "UMC";
5802 break;
5803 case FLEX10:
5804 str = "FLEX10";
5805 break;
5806 case vNIC1:
5807 str = "vNIC-1";
5808 break;
5809 case nPAR:
5810 str = "nPAR";
5811 break;
5812 case UFP:
5813 str = "UFP";
5814 break;
5815 case vNIC2:
5816 str = "vNIC-2";
5817 break;
5818 default:
5819 str = "";
5820 }
5821
5822 return str;
5823 }
5824
5825 static inline char *func_name(struct be_adapter *adapter)
5826 {
5827 return be_physfn(adapter) ? "PF" : "VF";
5828 }
5829
5830 static inline char *nic_name(struct pci_dev *pdev)
5831 {
5832 switch (pdev->device) {
5833 case OC_DEVICE_ID1:
5834 return OC_NAME;
5835 case OC_DEVICE_ID2:
5836 return OC_NAME_BE;
5837 case OC_DEVICE_ID3:
5838 case OC_DEVICE_ID4:
5839 return OC_NAME_LANCER;
5840 case BE_DEVICE_ID2:
5841 return BE3_NAME;
5842 case OC_DEVICE_ID5:
5843 case OC_DEVICE_ID6:
5844 return OC_NAME_SH;
5845 default:
5846 return BE_NAME;
5847 }
5848 }
5849
5850 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5851 {
5852 struct be_adapter *adapter;
5853 struct net_device *netdev;
5854 int status = 0;
5855
5856 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5857
5858 status = pci_enable_device(pdev);
5859 if (status)
5860 goto do_none;
5861
5862 status = pci_request_regions(pdev, DRV_NAME);
5863 if (status)
5864 goto disable_dev;
5865 pci_set_master(pdev);
5866
5867 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5868 if (!netdev) {
5869 status = -ENOMEM;
5870 goto rel_reg;
5871 }
5872 adapter = netdev_priv(netdev);
5873 adapter->pdev = pdev;
5874 pci_set_drvdata(pdev, adapter);
5875 adapter->netdev = netdev;
5876 SET_NETDEV_DEV(netdev, &pdev->dev);
5877
5878 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5879 if (!status) {
5880 netdev->features |= NETIF_F_HIGHDMA;
5881 } else {
5882 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5883 if (status) {
5884 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5885 goto free_netdev;
5886 }
5887 }
5888
5889 status = pci_enable_pcie_error_reporting(pdev);
5890 if (!status)
5891 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5892
5893 status = be_map_pci_bars(adapter);
5894 if (status)
5895 goto free_netdev;
5896
5897 status = be_drv_init(adapter);
5898 if (status)
5899 goto unmap_bars;
5900
5901 status = be_setup(adapter);
5902 if (status)
5903 goto drv_cleanup;
5904
5905 be_netdev_init(netdev);
5906 status = register_netdev(netdev);
5907 if (status != 0)
5908 goto unsetup;
5909
5910 be_roce_dev_add(adapter);
5911
5912 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
5913 adapter->error_recovery.probe_time = jiffies;
5914
5915 /* On Die temperature not supported for VF. */
5916 if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) {
5917 adapter->hwmon_info.hwmon_dev =
5918 devm_hwmon_device_register_with_groups(&pdev->dev,
5919 DRV_NAME,
5920 adapter,
5921 be_hwmon_groups);
5922 adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP;
5923 }
5924
5925 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5926 func_name(adapter), mc_name(adapter), adapter->port_name);
5927
5928 return 0;
5929
5930 unsetup:
5931 be_clear(adapter);
5932 drv_cleanup:
5933 be_drv_cleanup(adapter);
5934 unmap_bars:
5935 be_unmap_pci_bars(adapter);
5936 free_netdev:
5937 free_netdev(netdev);
5938 rel_reg:
5939 pci_release_regions(pdev);
5940 disable_dev:
5941 pci_disable_device(pdev);
5942 do_none:
5943 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5944 return status;
5945 }
5946
5947 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5948 {
5949 struct be_adapter *adapter = pci_get_drvdata(pdev);
5950
5951 be_intr_set(adapter, false);
5952 be_cancel_err_detection(adapter);
5953
5954 be_cleanup(adapter);
5955
5956 pci_save_state(pdev);
5957 pci_disable_device(pdev);
5958 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5959 return 0;
5960 }
5961
5962 static int be_pci_resume(struct pci_dev *pdev)
5963 {
5964 struct be_adapter *adapter = pci_get_drvdata(pdev);
5965 int status = 0;
5966
5967 status = pci_enable_device(pdev);
5968 if (status)
5969 return status;
5970
5971 pci_restore_state(pdev);
5972
5973 status = be_resume(adapter);
5974 if (status)
5975 return status;
5976
5977 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
5978
5979 return 0;
5980 }
5981
5982 /*
5983 * An FLR will stop BE from DMAing any data.
5984 */
5985 static void be_shutdown(struct pci_dev *pdev)
5986 {
5987 struct be_adapter *adapter = pci_get_drvdata(pdev);
5988
5989 if (!adapter)
5990 return;
5991
5992 be_roce_dev_shutdown(adapter);
5993 cancel_delayed_work_sync(&adapter->work);
5994 be_cancel_err_detection(adapter);
5995
5996 netif_device_detach(adapter->netdev);
5997
5998 be_cmd_reset_function(adapter);
5999
6000 pci_disable_device(pdev);
6001 }
6002
6003 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
6004 pci_channel_state_t state)
6005 {
6006 struct be_adapter *adapter = pci_get_drvdata(pdev);
6007
6008 dev_err(&adapter->pdev->dev, "EEH error detected\n");
6009
6010 be_roce_dev_remove(adapter);
6011
6012 if (!be_check_error(adapter, BE_ERROR_EEH)) {
6013 be_set_error(adapter, BE_ERROR_EEH);
6014
6015 be_cancel_err_detection(adapter);
6016
6017 be_cleanup(adapter);
6018 }
6019
6020 if (state == pci_channel_io_perm_failure)
6021 return PCI_ERS_RESULT_DISCONNECT;
6022
6023 pci_disable_device(pdev);
6024
6025 /* The error could cause the FW to trigger a flash debug dump.
6026 * Resetting the card while flash dump is in progress
6027 * can cause it not to recover; wait for it to finish.
6028 * Wait only for first function as it is needed only once per
6029 * adapter.
6030 */
6031 if (pdev->devfn == 0)
6032 ssleep(30);
6033
6034 return PCI_ERS_RESULT_NEED_RESET;
6035 }
6036
6037 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
6038 {
6039 struct be_adapter *adapter = pci_get_drvdata(pdev);
6040 int status;
6041
6042 dev_info(&adapter->pdev->dev, "EEH reset\n");
6043
6044 status = pci_enable_device(pdev);
6045 if (status)
6046 return PCI_ERS_RESULT_DISCONNECT;
6047
6048 pci_set_master(pdev);
6049 pci_restore_state(pdev);
6050
6051 /* Check if card is ok and fw is ready */
6052 dev_info(&adapter->pdev->dev,
6053 "Waiting for FW to be ready after EEH reset\n");
6054 status = be_fw_wait_ready(adapter);
6055 if (status)
6056 return PCI_ERS_RESULT_DISCONNECT;
6057
6058 pci_cleanup_aer_uncorrect_error_status(pdev);
6059 be_clear_error(adapter, BE_CLEAR_ALL);
6060 return PCI_ERS_RESULT_RECOVERED;
6061 }
6062
6063 static void be_eeh_resume(struct pci_dev *pdev)
6064 {
6065 int status = 0;
6066 struct be_adapter *adapter = pci_get_drvdata(pdev);
6067
6068 dev_info(&adapter->pdev->dev, "EEH resume\n");
6069
6070 pci_save_state(pdev);
6071
6072 status = be_resume(adapter);
6073 if (status)
6074 goto err;
6075
6076 be_roce_dev_add(adapter);
6077
6078 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
6079 return;
6080 err:
6081 dev_err(&adapter->pdev->dev, "EEH resume failed\n");
6082 }
6083
6084 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
6085 {
6086 struct be_adapter *adapter = pci_get_drvdata(pdev);
6087 struct be_resources vft_res = {0};
6088 int status;
6089
6090 if (!num_vfs)
6091 be_vf_clear(adapter);
6092
6093 adapter->num_vfs = num_vfs;
6094
6095 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) {
6096 dev_warn(&pdev->dev,
6097 "Cannot disable VFs while they are assigned\n");
6098 return -EBUSY;
6099 }
6100
6101 /* When the HW is in SRIOV capable configuration, the PF-pool resources
6102 * are equally distributed across the max-number of VFs. The user may
6103 * request only a subset of the max-vfs to be enabled.
6104 * Based on num_vfs, redistribute the resources across num_vfs so that
6105 * each VF will have access to more number of resources.
6106 * This facility is not available in BE3 FW.
6107 * Also, this is done by FW in Lancer chip.
6108 */
6109 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) {
6110 be_calculate_vf_res(adapter, adapter->num_vfs,
6111 &vft_res);
6112 status = be_cmd_set_sriov_config(adapter, adapter->pool_res,
6113 adapter->num_vfs, &vft_res);
6114 if (status)
6115 dev_err(&pdev->dev,
6116 "Failed to optimize SR-IOV resources\n");
6117 }
6118
6119 status = be_get_resources(adapter);
6120 if (status)
6121 return be_cmd_status(status);
6122
6123 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
6124 rtnl_lock();
6125 status = be_update_queues(adapter);
6126 rtnl_unlock();
6127 if (status)
6128 return be_cmd_status(status);
6129
6130 if (adapter->num_vfs)
6131 status = be_vf_setup(adapter);
6132
6133 if (!status)
6134 return adapter->num_vfs;
6135
6136 return 0;
6137 }
6138
6139 static const struct pci_error_handlers be_eeh_handlers = {
6140 .error_detected = be_eeh_err_detected,
6141 .slot_reset = be_eeh_reset,
6142 .resume = be_eeh_resume,
6143 };
6144
6145 static struct pci_driver be_driver = {
6146 .name = DRV_NAME,
6147 .id_table = be_dev_ids,
6148 .probe = be_probe,
6149 .remove = be_remove,
6150 .suspend = be_suspend,
6151 .resume = be_pci_resume,
6152 .shutdown = be_shutdown,
6153 .sriov_configure = be_pci_sriov_configure,
6154 .err_handler = &be_eeh_handlers
6155 };
6156
6157 static int __init be_init_module(void)
6158 {
6159 int status;
6160
6161 if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
6162 rx_frag_size != 2048) {
6163 printk(KERN_WARNING DRV_NAME
6164 " : Module param rx_frag_size must be 2048/4096/8192."
6165 " Using 2048\n");
6166 rx_frag_size = 2048;
6167 }
6168
6169 if (num_vfs > 0) {
6170 pr_info(DRV_NAME " : Module param num_vfs is obsolete.");
6171 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n");
6172 }
6173
6174 be_wq = create_singlethread_workqueue("be_wq");
6175 if (!be_wq) {
6176 pr_warn(DRV_NAME "workqueue creation failed\n");
6177 return -1;
6178 }
6179
6180 be_err_recovery_workq =
6181 create_singlethread_workqueue("be_err_recover");
6182 if (!be_err_recovery_workq)
6183 pr_warn(DRV_NAME "Could not create error recovery workqueue\n");
6184
6185 status = pci_register_driver(&be_driver);
6186 if (status) {
6187 destroy_workqueue(be_wq);
6188 be_destroy_err_recovery_workq();
6189 }
6190 return status;
6191 }
6192 module_init(be_init_module);
6193
6194 static void __exit be_exit_module(void)
6195 {
6196 pci_unregister_driver(&be_driver);
6197
6198 be_destroy_err_recovery_workq();
6199
6200 if (be_wq)
6201 destroy_workqueue(be_wq);
6202 }
6203 module_exit(be_exit_module);
ubuntu-zesty-kernel mirror