]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/net/ethernet/broadcom/bnxt/bnxt.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnxt_en: Fix VF resource checking.
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
CommitLineData
c0c050c5
MC		 1/* Broadcom NetXtreme-C/E network driver.
2 *
11f15ed3 3 * Copyright (c) 2014-2016 Broadcom Corporation
bac9a7e0 4 * Copyright (c) 2016-2017 Broadcom Limited
c0c050c5
MC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11#include <linux/module.h>
12
13#include <linux/stringify.h>
14#include <linux/kernel.h>
15#include <linux/timer.h>
16#include <linux/errno.h>
17#include <linux/ioport.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20#include <linux/interrupt.h>
21#include <linux/pci.h>
22#include <linux/netdevice.h>
23#include <linux/etherdevice.h>
24#include <linux/skbuff.h>
25#include <linux/dma-mapping.h>
26#include <linux/bitops.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
30#include <asm/byteorder.h>
31#include <asm/page.h>
32#include <linux/time.h>
33#include <linux/mii.h>
34#include <linux/if.h>
35#include <linux/if_vlan.h>
32e8239c 36#include <linux/if_bridge.h>
5ac67d8b 37#include <linux/rtc.h>
c6d30e83 38#include <linux/bpf.h>
c0c050c5
MC		 39#include <net/ip.h>
40#include <net/tcp.h>
41#include <net/udp.h>
42#include <net/checksum.h>
43#include <net/ip6_checksum.h>
ad51b8e9 44#include <net/udp_tunnel.h>
c0c050c5
MC		 45#include <linux/workqueue.h>
46#include <linux/prefetch.h>
47#include <linux/cache.h>
48#include <linux/log2.h>
49#include <linux/aer.h>
50#include <linux/bitmap.h>
51#include <linux/cpu_rmap.h>
56f0fd80 52#include <linux/cpumask.h>
2ae7408f 53#include <net/pkt_cls.h>
c0c050c5
MC		 54
55#include "bnxt_hsi.h"
56#include "bnxt.h"
a588e458 57#include "bnxt_ulp.h"
c0c050c5
MC		 58#include "bnxt_sriov.h"
59#include "bnxt_ethtool.h"
7df4ae9f 60#include "bnxt_dcb.h"
c6d30e83 61#include "bnxt_xdp.h"
4ab0c6a8 62#include "bnxt_vfr.h"
2ae7408f 63#include "bnxt_tc.h"
c0c050c5
MC		 64
65#define BNXT_TX_TIMEOUT (5 * HZ)
66
67static const char version[] =
68 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
69
70MODULE_LICENSE("GPL");
71MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
72MODULE_VERSION(DRV_MODULE_VERSION);
73
74#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
75#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
76#define BNXT_RX_COPY_THRESH 256
77
4419dbe6 78#define BNXT_TX_PUSH_THRESH 164
c0c050c5
MC		 79
80enum board_idx {
fbc9a523 81 BCM57301,
c0c050c5
MC		 82 BCM57302,
83 BCM57304,
1f681688 84 BCM57417_NPAR,
fa853dda 85 BCM58700,
b24eb6ae
MC		 86 BCM57311,
87 BCM57312,
fbc9a523 88 BCM57402,
c0c050c5
MC		 89 BCM57404,
90 BCM57406,
1f681688
MC		 91 BCM57402_NPAR,
92 BCM57407,
b24eb6ae
MC		 93 BCM57412,
94 BCM57414,
95 BCM57416,
96 BCM57417,
1f681688 97 BCM57412_NPAR,
5049e33b 98 BCM57314,
1f681688
MC		 99 BCM57417_SFP,
100 BCM57416_SFP,
101 BCM57404_NPAR,
102 BCM57406_NPAR,
103 BCM57407_SFP,
adbc8305 104 BCM57407_NPAR,
1f681688
MC		 105 BCM57414_NPAR,
106 BCM57416_NPAR,
32b40798
DK		 107 BCM57452,
108 BCM57454,
4a58139b
RJ		 109 BCM58802,
110 BCM58808,
adbc8305
MC		 111 NETXTREME_E_VF,
112 NETXTREME_C_VF,
c0c050c5
MC		 113};
114
115/* indexed by enum above */
116static const struct {
117 char *name;
118} board_info[] = {
27573a7d
SB		 119 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
120 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
121 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
122 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
123 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
124 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
125 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
126 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
127 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
128 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
129 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
130 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
131 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
132 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
133 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
134 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
135 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
136 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
137 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
138 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
139 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
140 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
141 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
142 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
143 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
144 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
145 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
146 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
147 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
148 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
149 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
150 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
c0c050c5
MC		 151};
152
153static const struct pci_device_id bnxt_pci_tbl[] = {
4a58139b 154 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
adbc8305 155 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
fbc9a523 156 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
c0c050c5
MC		 157 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
158 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
1f681688 159 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
fa853dda 160 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
b24eb6ae
MC		 161 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
162 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
fbc9a523 163 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
c0c050c5
MC		 164 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
165 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
1f681688
MC		 166 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
167 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
b24eb6ae
MC		 168 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
169 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
170 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
171 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
1f681688 172 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
5049e33b 173 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
1f681688
MC		 174 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
175 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
176 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
177 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
178 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
adbc8305
MC		 179 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
180 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
1f681688 181 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
adbc8305 182 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
1f681688 183 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
adbc8305 184 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
4a58139b 185 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
32b40798 186 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
4a58139b 187 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
c0c050c5 188#ifdef CONFIG_BNXT_SRIOV
c7ef35eb
DK		 189 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
190 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
adbc8305
MC		 191 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
192 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
193 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
194 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
195 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
196 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
c0c050c5
MC		 197#endif
198 { 0 }
199};
200
201MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
202
203static const u16 bnxt_vf_req_snif[] = {
204 HWRM_FUNC_CFG,
205 HWRM_PORT_PHY_QCFG,
206 HWRM_CFA_L2_FILTER_ALLOC,
207};
208
25be8623 209static const u16 bnxt_async_events_arr[] = {
87c374de
MC		 210 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
211 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
212 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
213 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
214 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
25be8623
MC		 215};
216
c213eae8
MC		 217static struct workqueue_struct *bnxt_pf_wq;
218
c0c050c5
MC		 219static bool bnxt_vf_pciid(enum board_idx idx)
220{
adbc8305 221 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF);
c0c050c5
MC		 222}
223
224#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
225#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
226#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
227
228#define BNXT_CP_DB_REARM(db, raw_cons) \
229 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
230
231#define BNXT_CP_DB(db, raw_cons) \
232 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
233
234#define BNXT_CP_DB_IRQ_DIS(db) \
235 writel(DB_CP_IRQ_DIS_FLAGS, db)
236
38413406 237const u16 bnxt_lhint_arr[] = {
c0c050c5
MC		 238 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
239 TX_BD_FLAGS_LHINT_512_TO_1023,
240 TX_BD_FLAGS_LHINT_1024_TO_2047,
241 TX_BD_FLAGS_LHINT_1024_TO_2047,
242 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
243 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
244 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
245 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
246 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
247 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
248 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
249 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
250 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
251 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
252 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
253 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
254 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
255 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
256 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
257};
258
ee5c7fb3
SP		 259static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
260{
261 struct metadata_dst *md_dst = skb_metadata_dst(skb);
262
263 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
264 return 0;
265
266 return md_dst->u.port_info.port_id;
267}
268
c0c050c5
MC		 269static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
270{
271 struct bnxt *bp = netdev_priv(dev);
272 struct tx_bd *txbd;
273 struct tx_bd_ext *txbd1;
274 struct netdev_queue *txq;
275 int i;
276 dma_addr_t mapping;
277 unsigned int length, pad = 0;
278 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
279 u16 prod, last_frag;
280 struct pci_dev *pdev = bp->pdev;
c0c050c5
MC		 281 struct bnxt_tx_ring_info *txr;
282 struct bnxt_sw_tx_bd *tx_buf;
283
284 i = skb_get_queue_mapping(skb);
285 if (unlikely(i >= bp->tx_nr_rings)) {
286 dev_kfree_skb_any(skb);
287 return NETDEV_TX_OK;
288 }
289
c0c050c5 290 txq = netdev_get_tx_queue(dev, i);
a960dec9 291 txr = &bp->tx_ring[bp->tx_ring_map[i]];
c0c050c5
MC		 292 prod = txr->tx_prod;
293
294 free_size = bnxt_tx_avail(bp, txr);
295 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
296 netif_tx_stop_queue(txq);
297 return NETDEV_TX_BUSY;
298 }
299
300 length = skb->len;
301 len = skb_headlen(skb);
302 last_frag = skb_shinfo(skb)->nr_frags;
303
304 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
305
306 txbd->tx_bd_opaque = prod;
307
308 tx_buf = &txr->tx_buf_ring[prod];
309 tx_buf->skb = skb;
310 tx_buf->nr_frags = last_frag;
311
312 vlan_tag_flags = 0;
ee5c7fb3 313 cfa_action = bnxt_xmit_get_cfa_action(skb);
c0c050c5
MC		 314 if (skb_vlan_tag_present(skb)) {
315 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
316 skb_vlan_tag_get(skb);
317 /* Currently supports 8021Q, 8021AD vlan offloads
318 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
319 */
320 if (skb->vlan_proto == htons(ETH_P_8021Q))
321 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
322 }
323
324 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
4419dbe6
MC		 325 struct tx_push_buffer *tx_push_buf = txr->tx_push;
326 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
327 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
328 void *pdata = tx_push_buf->data;
329 u64 *end;
330 int j, push_len;
c0c050c5
MC		 331
332 /* Set COAL_NOW to be ready quickly for the next push */
333 tx_push->tx_bd_len_flags_type =
334 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
335 TX_BD_TYPE_LONG_TX_BD |
336 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
337 TX_BD_FLAGS_COAL_NOW |
338 TX_BD_FLAGS_PACKET_END |
339 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
340
341 if (skb->ip_summed == CHECKSUM_PARTIAL)
342 tx_push1->tx_bd_hsize_lflags =
343 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
344 else
345 tx_push1->tx_bd_hsize_lflags = 0;
346
347 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 348 tx_push1->tx_bd_cfa_action =
349 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5 350
fbb0fa8b
MC		 351 end = pdata + length;
352 end = PTR_ALIGN(end, 8) - 1;
4419dbe6
MC		 353 *end = 0;
354
c0c050c5
MC		 355 skb_copy_from_linear_data(skb, pdata, len);
356 pdata += len;
357 for (j = 0; j < last_frag; j++) {
358 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
359 void *fptr;
360
361 fptr = skb_frag_address_safe(frag);
362 if (!fptr)
363 goto normal_tx;
364
365 memcpy(pdata, fptr, skb_frag_size(frag));
366 pdata += skb_frag_size(frag);
367 }
368
4419dbe6
MC		 369 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
370 txbd->tx_bd_haddr = txr->data_mapping;
c0c050c5
MC		 371 prod = NEXT_TX(prod);
372 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
373 memcpy(txbd, tx_push1, sizeof(*txbd));
374 prod = NEXT_TX(prod);
4419dbe6 375 tx_push->doorbell =
c0c050c5
MC		 376 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
377 txr->tx_prod = prod;
378
b9a8460a 379 tx_buf->is_push = 1;
c0c050c5 380 netdev_tx_sent_queue(txq, skb->len);
b9a8460a 381 wmb(); /* Sync is_push and byte queue before pushing data */
c0c050c5 382
4419dbe6
MC		 383 push_len = (length + sizeof(*tx_push) + 7) / 8;
384 if (push_len > 16) {
385 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
9d13744b
MC		 386 __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
387 (push_len - 16) << 1);
4419dbe6
MC		 388 } else {
389 __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
390 push_len);
391 }
c0c050c5 392
c0c050c5
MC		 393 goto tx_done;
394 }
395
396normal_tx:
397 if (length < BNXT_MIN_PKT_SIZE) {
398 pad = BNXT_MIN_PKT_SIZE - length;
399 if (skb_pad(skb, pad)) {
400 /* SKB already freed. */
401 tx_buf->skb = NULL;
402 return NETDEV_TX_OK;
403 }
404 length = BNXT_MIN_PKT_SIZE;
405 }
406
407 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
408
409 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
410 dev_kfree_skb_any(skb);
411 tx_buf->skb = NULL;
412 return NETDEV_TX_OK;
413 }
414
415 dma_unmap_addr_set(tx_buf, mapping, mapping);
416 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
417 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
418
419 txbd->tx_bd_haddr = cpu_to_le64(mapping);
420
421 prod = NEXT_TX(prod);
422 txbd1 = (struct tx_bd_ext *)
423 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
424
425 txbd1->tx_bd_hsize_lflags = 0;
426 if (skb_is_gso(skb)) {
427 u32 hdr_len;
428
429 if (skb->encapsulation)
430 hdr_len = skb_inner_network_offset(skb) +
431 skb_inner_network_header_len(skb) +
432 inner_tcp_hdrlen(skb);
433 else
434 hdr_len = skb_transport_offset(skb) +
435 tcp_hdrlen(skb);
436
437 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
438 TX_BD_FLAGS_T_IPID |
439 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
440 length = skb_shinfo(skb)->gso_size;
441 txbd1->tx_bd_mss = cpu_to_le32(length);
442 length += hdr_len;
443 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
444 txbd1->tx_bd_hsize_lflags =
445 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
446 txbd1->tx_bd_mss = 0;
447 }
448
449 length >>= 9;
450 flags |= bnxt_lhint_arr[length];
451 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
452
453 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 454 txbd1->tx_bd_cfa_action =
455 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5
MC		 456 for (i = 0; i < last_frag; i++) {
457 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
458
459 prod = NEXT_TX(prod);
460 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
461
462 len = skb_frag_size(frag);
463 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
464 DMA_TO_DEVICE);
465
466 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
467 goto tx_dma_error;
468
469 tx_buf = &txr->tx_buf_ring[prod];
470 dma_unmap_addr_set(tx_buf, mapping, mapping);
471
472 txbd->tx_bd_haddr = cpu_to_le64(mapping);
473
474 flags = len << TX_BD_LEN_SHIFT;
475 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
476 }
477
478 flags &= ~TX_BD_LEN;
479 txbd->tx_bd_len_flags_type =
480 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
481 TX_BD_FLAGS_PACKET_END);
482
483 netdev_tx_sent_queue(txq, skb->len);
484
485 /* Sync BD data before updating doorbell */
486 wmb();
487
488 prod = NEXT_TX(prod);
489 txr->tx_prod = prod;
490
ffe40645 491 if (!skb->xmit_more || netif_xmit_stopped(txq))
4d172f21 492 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
c0c050c5
MC		 493
494tx_done:
495
496 mmiowb();
497
498 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
4d172f21
MC		 499 if (skb->xmit_more && !tx_buf->is_push)
500 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
501
c0c050c5
MC		 502 netif_tx_stop_queue(txq);
503
504 /* netif_tx_stop_queue() must be done before checking
505 * tx index in bnxt_tx_avail() below, because in
506 * bnxt_tx_int(), we update tx index before checking for
507 * netif_tx_queue_stopped().
508 */
509 smp_mb();
510 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
511 netif_tx_wake_queue(txq);
512 }
513 return NETDEV_TX_OK;
514
515tx_dma_error:
516 last_frag = i;
517
518 /* start back at beginning and unmap skb */
519 prod = txr->tx_prod;
520 tx_buf = &txr->tx_buf_ring[prod];
521 tx_buf->skb = NULL;
522 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
523 skb_headlen(skb), PCI_DMA_TODEVICE);
524 prod = NEXT_TX(prod);
525
526 /* unmap remaining mapped pages */
527 for (i = 0; i < last_frag; i++) {
528 prod = NEXT_TX(prod);
529 tx_buf = &txr->tx_buf_ring[prod];
530 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
531 skb_frag_size(&skb_shinfo(skb)->frags[i]),
532 PCI_DMA_TODEVICE);
533 }
534
535 dev_kfree_skb_any(skb);
536 return NETDEV_TX_OK;
537}
538
539static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
540{
b6ab4b01 541 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
a960dec9 542 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
c0c050c5
MC		 543 u16 cons = txr->tx_cons;
544 struct pci_dev *pdev = bp->pdev;
545 int i;
546 unsigned int tx_bytes = 0;
547
548 for (i = 0; i < nr_pkts; i++) {
549 struct bnxt_sw_tx_bd *tx_buf;
550 struct sk_buff *skb;
551 int j, last;
552
553 tx_buf = &txr->tx_buf_ring[cons];
554 cons = NEXT_TX(cons);
555 skb = tx_buf->skb;
556 tx_buf->skb = NULL;
557
558 if (tx_buf->is_push) {
559 tx_buf->is_push = 0;
560 goto next_tx_int;
561 }
562
563 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
564 skb_headlen(skb), PCI_DMA_TODEVICE);
565 last = tx_buf->nr_frags;
566
567 for (j = 0; j < last; j++) {
568 cons = NEXT_TX(cons);
569 tx_buf = &txr->tx_buf_ring[cons];
570 dma_unmap_page(
571 &pdev->dev,
572 dma_unmap_addr(tx_buf, mapping),
573 skb_frag_size(&skb_shinfo(skb)->frags[j]),
574 PCI_DMA_TODEVICE);
575 }
576
577next_tx_int:
578 cons = NEXT_TX(cons);
579
580 tx_bytes += skb->len;
581 dev_kfree_skb_any(skb);
582 }
583
584 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
585 txr->tx_cons = cons;
586
587 /* Need to make the tx_cons update visible to bnxt_start_xmit()
588 * before checking for netif_tx_queue_stopped(). Without the
589 * memory barrier, there is a small possibility that bnxt_start_xmit()
590 * will miss it and cause the queue to be stopped forever.
591 */
592 smp_mb();
593
594 if (unlikely(netif_tx_queue_stopped(txq)) &&
595 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
596 __netif_tx_lock(txq, smp_processor_id());
597 if (netif_tx_queue_stopped(txq) &&
598 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
599 txr->dev_state != BNXT_DEV_STATE_CLOSING)
600 netif_tx_wake_queue(txq);
601 __netif_tx_unlock(txq);
602 }
603}
604
c61fb99c
MC		 605static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
606 gfp_t gfp)
607{
608 struct device *dev = &bp->pdev->dev;
609 struct page *page;
610
611 page = alloc_page(gfp);
612 if (!page)
613 return NULL;
614
c519fe9a
SN		 615 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
616 DMA_ATTR_WEAK_ORDERING);
c61fb99c
MC		 617 if (dma_mapping_error(dev, *mapping)) {
618 __free_page(page);
619 return NULL;
620 }
621 *mapping += bp->rx_dma_offset;
622 return page;
623}
624
c0c050c5
MC		 625static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
626 gfp_t gfp)
627{
628 u8 *data;
629 struct pci_dev *pdev = bp->pdev;
630
631 data = kmalloc(bp->rx_buf_size, gfp);
632 if (!data)
633 return NULL;
634
c519fe9a
SN		 635 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
636 bp->rx_buf_use_size, bp->rx_dir,
637 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 638
639 if (dma_mapping_error(&pdev->dev, *mapping)) {
640 kfree(data);
641 data = NULL;
642 }
643 return data;
644}
645
38413406
MC		 646int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
647 u16 prod, gfp_t gfp)
c0c050c5
MC		 648{
649 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
650 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
c0c050c5
MC		 651 dma_addr_t mapping;
652
c61fb99c
MC		 653 if (BNXT_RX_PAGE_MODE(bp)) {
654 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
c0c050c5 655
c61fb99c
MC		 656 if (!page)
657 return -ENOMEM;
658
659 rx_buf->data = page;
660 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
661 } else {
662 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
663
664 if (!data)
665 return -ENOMEM;
666
667 rx_buf->data = data;
668 rx_buf->data_ptr = data + bp->rx_offset;
669 }
11cd119d 670 rx_buf->mapping = mapping;
c0c050c5
MC		 671
672 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
c0c050c5
MC		 673 return 0;
674}
675
c6d30e83 676void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
c0c050c5
MC		 677{
678 u16 prod = rxr->rx_prod;
679 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
680 struct rx_bd *cons_bd, *prod_bd;
681
682 prod_rx_buf = &rxr->rx_buf_ring[prod];
683 cons_rx_buf = &rxr->rx_buf_ring[cons];
684
685 prod_rx_buf->data = data;
6bb19474 686 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 687
11cd119d 688 prod_rx_buf->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 689
690 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
691 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
692
693 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
694}
695
696static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
697{
698 u16 next, max = rxr->rx_agg_bmap_size;
699
700 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
701 if (next >= max)
702 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
703 return next;
704}
705
706static inline int bnxt_alloc_rx_page(struct bnxt *bp,
707 struct bnxt_rx_ring_info *rxr,
708 u16 prod, gfp_t gfp)
709{
710 struct rx_bd *rxbd =
711 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
712 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
713 struct pci_dev *pdev = bp->pdev;
714 struct page *page;
715 dma_addr_t mapping;
716 u16 sw_prod = rxr->rx_sw_agg_prod;
89d0a06c 717 unsigned int offset = 0;
c0c050c5 718
89d0a06c
MC		 719 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
720 page = rxr->rx_page;
721 if (!page) {
722 page = alloc_page(gfp);
723 if (!page)
724 return -ENOMEM;
725 rxr->rx_page = page;
726 rxr->rx_page_offset = 0;
727 }
728 offset = rxr->rx_page_offset;
729 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
730 if (rxr->rx_page_offset == PAGE_SIZE)
731 rxr->rx_page = NULL;
732 else
733 get_page(page);
734 } else {
735 page = alloc_page(gfp);
736 if (!page)
737 return -ENOMEM;
738 }
c0c050c5 739
c519fe9a
SN		 740 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
741 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
742 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 743 if (dma_mapping_error(&pdev->dev, mapping)) {
744 __free_page(page);
745 return -EIO;
746 }
747
748 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
749 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
750
751 __set_bit(sw_prod, rxr->rx_agg_bmap);
752 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
753 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
754
755 rx_agg_buf->page = page;
89d0a06c 756 rx_agg_buf->offset = offset;
c0c050c5
MC		 757 rx_agg_buf->mapping = mapping;
758 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
759 rxbd->rx_bd_opaque = sw_prod;
760 return 0;
761}
762
763static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
764 u32 agg_bufs)
765{
766 struct bnxt *bp = bnapi->bp;
767 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 768 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 769 u16 prod = rxr->rx_agg_prod;
770 u16 sw_prod = rxr->rx_sw_agg_prod;
771 u32 i;
772
773 for (i = 0; i < agg_bufs; i++) {
774 u16 cons;
775 struct rx_agg_cmp *agg;
776 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
777 struct rx_bd *prod_bd;
778 struct page *page;
779
780 agg = (struct rx_agg_cmp *)
781 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
782 cons = agg->rx_agg_cmp_opaque;
783 __clear_bit(cons, rxr->rx_agg_bmap);
784
785 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
786 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
787
788 __set_bit(sw_prod, rxr->rx_agg_bmap);
789 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
790 cons_rx_buf = &rxr->rx_agg_ring[cons];
791
792 /* It is possible for sw_prod to be equal to cons, so
793 * set cons_rx_buf->page to NULL first.
794 */
795 page = cons_rx_buf->page;
796 cons_rx_buf->page = NULL;
797 prod_rx_buf->page = page;
89d0a06c 798 prod_rx_buf->offset = cons_rx_buf->offset;
c0c050c5
MC		 799
800 prod_rx_buf->mapping = cons_rx_buf->mapping;
801
802 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
803
804 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
805 prod_bd->rx_bd_opaque = sw_prod;
806
807 prod = NEXT_RX_AGG(prod);
808 sw_prod = NEXT_RX_AGG(sw_prod);
809 cp_cons = NEXT_CMP(cp_cons);
810 }
811 rxr->rx_agg_prod = prod;
812 rxr->rx_sw_agg_prod = sw_prod;
813}
814
c61fb99c
MC		 815static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
816 struct bnxt_rx_ring_info *rxr,
817 u16 cons, void *data, u8 *data_ptr,
818 dma_addr_t dma_addr,
819 unsigned int offset_and_len)
820{
821 unsigned int payload = offset_and_len >> 16;
822 unsigned int len = offset_and_len & 0xffff;
823 struct skb_frag_struct *frag;
824 struct page *page = data;
825 u16 prod = rxr->rx_prod;
826 struct sk_buff *skb;
827 int off, err;
828
829 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
830 if (unlikely(err)) {
831 bnxt_reuse_rx_data(rxr, cons, data);
832 return NULL;
833 }
834 dma_addr -= bp->rx_dma_offset;
c519fe9a
SN		 835 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
836 DMA_ATTR_WEAK_ORDERING);
c61fb99c
MC		 837
838 if (unlikely(!payload))
839 payload = eth_get_headlen(data_ptr, len);
840
841 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
842 if (!skb) {
843 __free_page(page);
844 return NULL;
845 }
846
847 off = (void *)data_ptr - page_address(page);
848 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
849 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
850 payload + NET_IP_ALIGN);
851
852 frag = &skb_shinfo(skb)->frags[0];
853 skb_frag_size_sub(frag, payload);
854 frag->page_offset += payload;
855 skb->data_len -= payload;
856 skb->tail += payload;
857
858 return skb;
859}
860
c0c050c5
MC		 861static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
862 struct bnxt_rx_ring_info *rxr, u16 cons,
6bb19474
MC		 863 void *data, u8 *data_ptr,
864 dma_addr_t dma_addr,
865 unsigned int offset_and_len)
c0c050c5 866{
6bb19474 867 u16 prod = rxr->rx_prod;
c0c050c5 868 struct sk_buff *skb;
6bb19474 869 int err;
c0c050c5
MC		 870
871 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
872 if (unlikely(err)) {
873 bnxt_reuse_rx_data(rxr, cons, data);
874 return NULL;
875 }
876
877 skb = build_skb(data, 0);
c519fe9a
SN		 878 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
879 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 880 if (!skb) {
881 kfree(data);
882 return NULL;
883 }
884
b3dba77c 885 skb_reserve(skb, bp->rx_offset);
6bb19474 886 skb_put(skb, offset_and_len & 0xffff);
c0c050c5
MC		 887 return skb;
888}
889
890static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
891 struct sk_buff *skb, u16 cp_cons,
892 u32 agg_bufs)
893{
894 struct pci_dev *pdev = bp->pdev;
895 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 896 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 897 u16 prod = rxr->rx_agg_prod;
898 u32 i;
899
900 for (i = 0; i < agg_bufs; i++) {
901 u16 cons, frag_len;
902 struct rx_agg_cmp *agg;
903 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
904 struct page *page;
905 dma_addr_t mapping;
906
907 agg = (struct rx_agg_cmp *)
908 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
909 cons = agg->rx_agg_cmp_opaque;
910 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
911 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
912
913 cons_rx_buf = &rxr->rx_agg_ring[cons];
89d0a06c
MC		 914 skb_fill_page_desc(skb, i, cons_rx_buf->page,
915 cons_rx_buf->offset, frag_len);
c0c050c5
MC		 916 __clear_bit(cons, rxr->rx_agg_bmap);
917
918 /* It is possible for bnxt_alloc_rx_page() to allocate
919 * a sw_prod index that equals the cons index, so we
920 * need to clear the cons entry now.
921 */
11cd119d 922 mapping = cons_rx_buf->mapping;
c0c050c5
MC		 923 page = cons_rx_buf->page;
924 cons_rx_buf->page = NULL;
925
926 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
927 struct skb_shared_info *shinfo;
928 unsigned int nr_frags;
929
930 shinfo = skb_shinfo(skb);
931 nr_frags = --shinfo->nr_frags;
932 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
933
934 dev_kfree_skb(skb);
935
936 cons_rx_buf->page = page;
937
938 /* Update prod since possibly some pages have been
939 * allocated already.
940 */
941 rxr->rx_agg_prod = prod;
942 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
943 return NULL;
944 }
945
c519fe9a
SN		 946 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
947 PCI_DMA_FROMDEVICE,
948 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 949
950 skb->data_len += frag_len;
951 skb->len += frag_len;
952 skb->truesize += PAGE_SIZE;
953
954 prod = NEXT_RX_AGG(prod);
955 cp_cons = NEXT_CMP(cp_cons);
956 }
957 rxr->rx_agg_prod = prod;
958 return skb;
959}
960
961static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
962 u8 agg_bufs, u32 *raw_cons)
963{
964 u16 last;
965 struct rx_agg_cmp *agg;
966
967 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
968 last = RING_CMP(*raw_cons);
969 agg = (struct rx_agg_cmp *)
970 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
971 return RX_AGG_CMP_VALID(agg, *raw_cons);
972}
973
974static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
975 unsigned int len,
976 dma_addr_t mapping)
977{
978 struct bnxt *bp = bnapi->bp;
979 struct pci_dev *pdev = bp->pdev;
980 struct sk_buff *skb;
981
982 skb = napi_alloc_skb(&bnapi->napi, len);
983 if (!skb)
984 return NULL;
985
745fc05c
MC		 986 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
987 bp->rx_dir);
c0c050c5 988
6bb19474
MC		 989 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
990 len + NET_IP_ALIGN);
c0c050c5 991
745fc05c
MC		 992 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
993 bp->rx_dir);
c0c050c5
MC		 994
995 skb_put(skb, len);
996 return skb;
997}
998
fa7e2812
MC		 999static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
1000 u32 *raw_cons, void *cmp)
1001{
1002 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1003 struct rx_cmp *rxcmp = cmp;
1004 u32 tmp_raw_cons = *raw_cons;
1005 u8 cmp_type, agg_bufs = 0;
1006
1007 cmp_type = RX_CMP_TYPE(rxcmp);
1008
1009 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1010 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1011 RX_CMP_AGG_BUFS) >>
1012 RX_CMP_AGG_BUFS_SHIFT;
1013 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1014 struct rx_tpa_end_cmp *tpa_end = cmp;
1015
1016 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1017 RX_TPA_END_CMP_AGG_BUFS) >>
1018 RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1019 }
1020
1021 if (agg_bufs) {
1022 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1023 return -EBUSY;
1024 }
1025 *raw_cons = tmp_raw_cons;
1026 return 0;
1027}
1028
c213eae8
MC		 1029static void bnxt_queue_sp_work(struct bnxt *bp)
1030{
1031 if (BNXT_PF(bp))
1032 queue_work(bnxt_pf_wq, &bp->sp_task);
1033 else
1034 schedule_work(&bp->sp_task);
1035}
1036
1037static void bnxt_cancel_sp_work(struct bnxt *bp)
1038{
1039 if (BNXT_PF(bp))
1040 flush_workqueue(bnxt_pf_wq);
1041 else
1042 cancel_work_sync(&bp->sp_task);
1043}
1044
fa7e2812
MC		 1045static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1046{
1047 if (!rxr->bnapi->in_reset) {
1048 rxr->bnapi->in_reset = true;
1049 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 1050 bnxt_queue_sp_work(bp);
fa7e2812
MC		 1051 }
1052 rxr->rx_next_cons = 0xffff;
1053}
1054
c0c050c5
MC		 1055static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1056 struct rx_tpa_start_cmp *tpa_start,
1057 struct rx_tpa_start_cmp_ext *tpa_start1)
1058{
1059 u8 agg_id = TPA_START_AGG_ID(tpa_start);
1060 u16 cons, prod;
1061 struct bnxt_tpa_info *tpa_info;
1062 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1063 struct rx_bd *prod_bd;
1064 dma_addr_t mapping;
1065
1066 cons = tpa_start->rx_tpa_start_cmp_opaque;
1067 prod = rxr->rx_prod;
1068 cons_rx_buf = &rxr->rx_buf_ring[cons];
1069 prod_rx_buf = &rxr->rx_buf_ring[prod];
1070 tpa_info = &rxr->rx_tpa[agg_id];
1071
fa7e2812
MC		 1072 if (unlikely(cons != rxr->rx_next_cons)) {
1073 bnxt_sched_reset(bp, rxr);
1074 return;
1075 }
ee5c7fb3
SP		 1076 /* Store cfa_code in tpa_info to use in tpa_end
1077 * completion processing.
1078 */
1079 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
c0c050c5 1080 prod_rx_buf->data = tpa_info->data;
6bb19474 1081 prod_rx_buf->data_ptr = tpa_info->data_ptr;
c0c050c5
MC		 1082
1083 mapping = tpa_info->mapping;
11cd119d 1084 prod_rx_buf->mapping = mapping;
c0c050c5
MC		 1085
1086 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1087
1088 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1089
1090 tpa_info->data = cons_rx_buf->data;
6bb19474 1091 tpa_info->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 1092 cons_rx_buf->data = NULL;
11cd119d 1093 tpa_info->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 1094
1095 tpa_info->len =
1096 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1097 RX_TPA_START_CMP_LEN_SHIFT;
1098 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1099 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1100
1101 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1102 tpa_info->gso_type = SKB_GSO_TCPV4;
1103 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1104 if (hash_type == 3)
1105 tpa_info->gso_type = SKB_GSO_TCPV6;
1106 tpa_info->rss_hash =
1107 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1108 } else {
1109 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1110 tpa_info->gso_type = 0;
1111 if (netif_msg_rx_err(bp))
1112 netdev_warn(bp->dev, "TPA packet without valid hash\n");
1113 }
1114 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1115 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
94758f8d 1116 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
c0c050c5
MC		 1117
1118 rxr->rx_prod = NEXT_RX(prod);
1119 cons = NEXT_RX(cons);
376a5b86 1120 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 1121 cons_rx_buf = &rxr->rx_buf_ring[cons];
1122
1123 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1124 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1125 cons_rx_buf->data = NULL;
1126}
1127
1128static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
1129 u16 cp_cons, u32 agg_bufs)
1130{
1131 if (agg_bufs)
1132 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1133}
1134
94758f8d
MC		 1135static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1136 int payload_off, int tcp_ts,
1137 struct sk_buff *skb)
1138{
1139#ifdef CONFIG_INET
1140 struct tcphdr *th;
1141 int len, nw_off;
1142 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1143 u32 hdr_info = tpa_info->hdr_info;
1144 bool loopback = false;
1145
1146 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1147 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1148 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1149
1150 /* If the packet is an internal loopback packet, the offsets will
1151 * have an extra 4 bytes.
1152 */
1153 if (inner_mac_off == 4) {
1154 loopback = true;
1155 } else if (inner_mac_off > 4) {
1156 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1157 ETH_HLEN - 2));
1158
1159 /* We only support inner iPv4/ipv6. If we don't see the
1160 * correct protocol ID, it must be a loopback packet where
1161 * the offsets are off by 4.
1162 */
09a7636a 1163 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
94758f8d
MC		 1164 loopback = true;
1165 }
1166 if (loopback) {
1167 /* internal loopback packet, subtract all offsets by 4 */
1168 inner_ip_off -= 4;
1169 inner_mac_off -= 4;
1170 outer_ip_off -= 4;
1171 }
1172
1173 nw_off = inner_ip_off - ETH_HLEN;
1174 skb_set_network_header(skb, nw_off);
1175 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1176 struct ipv6hdr *iph = ipv6_hdr(skb);
1177
1178 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1179 len = skb->len - skb_transport_offset(skb);
1180 th = tcp_hdr(skb);
1181 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1182 } else {
1183 struct iphdr *iph = ip_hdr(skb);
1184
1185 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1186 len = skb->len - skb_transport_offset(skb);
1187 th = tcp_hdr(skb);
1188 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1189 }
1190
1191 if (inner_mac_off) { /* tunnel */
1192 struct udphdr *uh = NULL;
1193 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1194 ETH_HLEN - 2));
1195
1196 if (proto == htons(ETH_P_IP)) {
1197 struct iphdr *iph = (struct iphdr *)skb->data;
1198
1199 if (iph->protocol == IPPROTO_UDP)
1200 uh = (struct udphdr *)(iph + 1);
1201 } else {
1202 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1203
1204 if (iph->nexthdr == IPPROTO_UDP)
1205 uh = (struct udphdr *)(iph + 1);
1206 }
1207 if (uh) {
1208 if (uh->check)
1209 skb_shinfo(skb)->gso_type |=
1210 SKB_GSO_UDP_TUNNEL_CSUM;
1211 else
1212 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1213 }
1214 }
1215#endif
1216 return skb;
1217}
1218
c0c050c5
MC		 1219#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1220#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1221
309369c9
MC		 1222static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1223 int payload_off, int tcp_ts,
c0c050c5
MC		 1224 struct sk_buff *skb)
1225{
d1611c3a 1226#ifdef CONFIG_INET
c0c050c5 1227 struct tcphdr *th;
719ca811 1228 int len, nw_off, tcp_opt_len = 0;
27e24189 1229
309369c9 1230 if (tcp_ts)
c0c050c5
MC		 1231 tcp_opt_len = 12;
1232
c0c050c5
MC		 1233 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1234 struct iphdr *iph;
1235
1236 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1237 ETH_HLEN;
1238 skb_set_network_header(skb, nw_off);
1239 iph = ip_hdr(skb);
1240 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1241 len = skb->len - skb_transport_offset(skb);
1242 th = tcp_hdr(skb);
1243 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1244 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1245 struct ipv6hdr *iph;
1246
1247 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1248 ETH_HLEN;
1249 skb_set_network_header(skb, nw_off);
1250 iph = ipv6_hdr(skb);
1251 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1252 len = skb->len - skb_transport_offset(skb);
1253 th = tcp_hdr(skb);
1254 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1255 } else {
1256 dev_kfree_skb_any(skb);
1257 return NULL;
1258 }
c0c050c5
MC		 1259
1260 if (nw_off) { /* tunnel */
1261 struct udphdr *uh = NULL;
1262
1263 if (skb->protocol == htons(ETH_P_IP)) {
1264 struct iphdr *iph = (struct iphdr *)skb->data;
1265
1266 if (iph->protocol == IPPROTO_UDP)
1267 uh = (struct udphdr *)(iph + 1);
1268 } else {
1269 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1270
1271 if (iph->nexthdr == IPPROTO_UDP)
1272 uh = (struct udphdr *)(iph + 1);
1273 }
1274 if (uh) {
1275 if (uh->check)
1276 skb_shinfo(skb)->gso_type |=
1277 SKB_GSO_UDP_TUNNEL_CSUM;
1278 else
1279 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1280 }
1281 }
1282#endif
1283 return skb;
1284}
1285
309369c9
MC		 1286static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1287 struct bnxt_tpa_info *tpa_info,
1288 struct rx_tpa_end_cmp *tpa_end,
1289 struct rx_tpa_end_cmp_ext *tpa_end1,
1290 struct sk_buff *skb)
1291{
1292#ifdef CONFIG_INET
1293 int payload_off;
1294 u16 segs;
1295
1296 segs = TPA_END_TPA_SEGS(tpa_end);
1297 if (segs == 1)
1298 return skb;
1299
1300 NAPI_GRO_CB(skb)->count = segs;
1301 skb_shinfo(skb)->gso_size =
1302 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1303 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1304 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1305 RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1306 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1307 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
5910906c
MC		 1308 if (likely(skb))
1309 tcp_gro_complete(skb);
309369c9
MC		 1310#endif
1311 return skb;
1312}
1313
ee5c7fb3
SP		 1314/* Given the cfa_code of a received packet determine which
1315 * netdev (vf-rep or PF) the packet is destined to.
1316 */
1317static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1318{
1319 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1320
1321 /* if vf-rep dev is NULL, the must belongs to the PF */
1322 return dev ? dev : bp->dev;
1323}
1324
c0c050c5
MC		 1325static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1326 struct bnxt_napi *bnapi,
1327 u32 *raw_cons,
1328 struct rx_tpa_end_cmp *tpa_end,
1329 struct rx_tpa_end_cmp_ext *tpa_end1,
4e5dbbda 1330 u8 *event)
c0c050c5
MC		 1331{
1332 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 1333 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 1334 u8 agg_id = TPA_END_AGG_ID(tpa_end);
6bb19474 1335 u8 *data_ptr, agg_bufs;
c0c050c5
MC		 1336 u16 cp_cons = RING_CMP(*raw_cons);
1337 unsigned int len;
1338 struct bnxt_tpa_info *tpa_info;
1339 dma_addr_t mapping;
1340 struct sk_buff *skb;
6bb19474 1341 void *data;
c0c050c5 1342
fa7e2812
MC		 1343 if (unlikely(bnapi->in_reset)) {
1344 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1345
1346 if (rc < 0)
1347 return ERR_PTR(-EBUSY);
1348 return NULL;
1349 }
1350
c0c050c5
MC		 1351 tpa_info = &rxr->rx_tpa[agg_id];
1352 data = tpa_info->data;
6bb19474
MC		 1353 data_ptr = tpa_info->data_ptr;
1354 prefetch(data_ptr);
c0c050c5
MC		 1355 len = tpa_info->len;
1356 mapping = tpa_info->mapping;
1357
1358 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1359 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1360
1361 if (agg_bufs) {
1362 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1363 return ERR_PTR(-EBUSY);
1364
4e5dbbda 1365 *event |= BNXT_AGG_EVENT;
c0c050c5
MC		 1366 cp_cons = NEXT_CMP(cp_cons);
1367 }
1368
69c149e2 1369 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
c0c050c5 1370 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
69c149e2
MC		 1371 if (agg_bufs > MAX_SKB_FRAGS)
1372 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1373 agg_bufs, (int)MAX_SKB_FRAGS);
c0c050c5
MC		 1374 return NULL;
1375 }
1376
1377 if (len <= bp->rx_copy_thresh) {
6bb19474 1378 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
c0c050c5
MC		 1379 if (!skb) {
1380 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1381 return NULL;
1382 }
1383 } else {
1384 u8 *new_data;
1385 dma_addr_t new_mapping;
1386
1387 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1388 if (!new_data) {
1389 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1390 return NULL;
1391 }
1392
1393 tpa_info->data = new_data;
b3dba77c 1394 tpa_info->data_ptr = new_data + bp->rx_offset;
c0c050c5
MC		 1395 tpa_info->mapping = new_mapping;
1396
1397 skb = build_skb(data, 0);
c519fe9a
SN		 1398 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1399 bp->rx_buf_use_size, bp->rx_dir,
1400 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 1401
1402 if (!skb) {
1403 kfree(data);
1404 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1405 return NULL;
1406 }
b3dba77c 1407 skb_reserve(skb, bp->rx_offset);
c0c050c5
MC		 1408 skb_put(skb, len);
1409 }
1410
1411 if (agg_bufs) {
1412 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1413 if (!skb) {
1414 /* Page reuse already handled by bnxt_rx_pages(). */
1415 return NULL;
1416 }
1417 }
ee5c7fb3
SP		 1418
1419 skb->protocol =
1420 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
c0c050c5
MC		 1421
1422 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1423 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1424
8852ddb4
MC		 1425 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1426 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5
MC		 1427 u16 vlan_proto = tpa_info->metadata >>
1428 RX_CMP_FLAGS2_METADATA_TPID_SFT;
8852ddb4 1429 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
c0c050c5 1430
8852ddb4 1431 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1432 }
1433
1434 skb_checksum_none_assert(skb);
1435 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1436 skb->ip_summed = CHECKSUM_UNNECESSARY;
1437 skb->csum_level =
1438 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1439 }
1440
1441 if (TPA_END_GRO(tpa_end))
309369c9 1442 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
c0c050c5
MC		 1443
1444 return skb;
1445}
1446
ee5c7fb3
SP		 1447static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1448 struct sk_buff *skb)
1449{
1450 if (skb->dev != bp->dev) {
1451 /* this packet belongs to a vf-rep */
1452 bnxt_vf_rep_rx(bp, skb);
1453 return;
1454 }
1455 skb_record_rx_queue(skb, bnapi->index);
1456 napi_gro_receive(&bnapi->napi, skb);
1457}
1458
c0c050c5
MC		 1459/* returns the following:
1460 * 1 - 1 packet successfully received
1461 * 0 - successful TPA_START, packet not completed yet
1462 * -EBUSY - completion ring does not have all the agg buffers yet
1463 * -ENOMEM - packet aborted due to out of memory
1464 * -EIO - packet aborted due to hw error indicated in BD
1465 */
1466static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
4e5dbbda 1467 u8 *event)
c0c050c5
MC		 1468{
1469 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 1470 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1471 struct net_device *dev = bp->dev;
1472 struct rx_cmp *rxcmp;
1473 struct rx_cmp_ext *rxcmp1;
1474 u32 tmp_raw_cons = *raw_cons;
ee5c7fb3 1475 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
c0c050c5
MC		 1476 struct bnxt_sw_rx_bd *rx_buf;
1477 unsigned int len;
6bb19474 1478 u8 *data_ptr, agg_bufs, cmp_type;
c0c050c5
MC		 1479 dma_addr_t dma_addr;
1480 struct sk_buff *skb;
6bb19474 1481 void *data;
c0c050c5 1482 int rc = 0;
c61fb99c 1483 u32 misc;
c0c050c5
MC		 1484
1485 rxcmp = (struct rx_cmp *)
1486 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1487
1488 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1489 cp_cons = RING_CMP(tmp_raw_cons);
1490 rxcmp1 = (struct rx_cmp_ext *)
1491 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1492
1493 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1494 return -EBUSY;
1495
1496 cmp_type = RX_CMP_TYPE(rxcmp);
1497
1498 prod = rxr->rx_prod;
1499
1500 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1501 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1502 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1503
4e5dbbda 1504 *event |= BNXT_RX_EVENT;
c0c050c5
MC		 1505 goto next_rx_no_prod;
1506
1507 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1508 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1509 (struct rx_tpa_end_cmp *)rxcmp,
4e5dbbda 1510 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
c0c050c5
MC		 1511
1512 if (unlikely(IS_ERR(skb)))
1513 return -EBUSY;
1514
1515 rc = -ENOMEM;
1516 if (likely(skb)) {
ee5c7fb3 1517 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1518 rc = 1;
1519 }
4e5dbbda 1520 *event |= BNXT_RX_EVENT;
c0c050c5
MC		 1521 goto next_rx_no_prod;
1522 }
1523
1524 cons = rxcmp->rx_cmp_opaque;
1525 rx_buf = &rxr->rx_buf_ring[cons];
1526 data = rx_buf->data;
6bb19474 1527 data_ptr = rx_buf->data_ptr;
fa7e2812
MC		 1528 if (unlikely(cons != rxr->rx_next_cons)) {
1529 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1530
1531 bnxt_sched_reset(bp, rxr);
1532 return rc1;
1533 }
6bb19474 1534 prefetch(data_ptr);
c0c050c5 1535
c61fb99c
MC		 1536 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1537 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
c0c050c5
MC		 1538
1539 if (agg_bufs) {
1540 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1541 return -EBUSY;
1542
1543 cp_cons = NEXT_CMP(cp_cons);
4e5dbbda 1544 *event |= BNXT_AGG_EVENT;
c0c050c5 1545 }
4e5dbbda 1546 *event |= BNXT_RX_EVENT;
c0c050c5
MC		 1547
1548 rx_buf->data = NULL;
1549 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1550 bnxt_reuse_rx_data(rxr, cons, data);
1551 if (agg_bufs)
1552 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1553
1554 rc = -EIO;
1555 goto next_rx;
1556 }
1557
1558 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
11cd119d 1559 dma_addr = rx_buf->mapping;
c0c050c5 1560
c6d30e83
MC		 1561 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1562 rc = 1;
1563 goto next_rx;
1564 }
1565
c0c050c5 1566 if (len <= bp->rx_copy_thresh) {
6bb19474 1567 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
c0c050c5
MC		 1568 bnxt_reuse_rx_data(rxr, cons, data);
1569 if (!skb) {
1570 rc = -ENOMEM;
1571 goto next_rx;
1572 }
1573 } else {
c61fb99c
MC		 1574 u32 payload;
1575
c6d30e83
MC		 1576 if (rx_buf->data_ptr == data_ptr)
1577 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1578 else
1579 payload = 0;
6bb19474 1580 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
c61fb99c 1581 payload | len);
c0c050c5
MC		 1582 if (!skb) {
1583 rc = -ENOMEM;
1584 goto next_rx;
1585 }
1586 }
1587
1588 if (agg_bufs) {
1589 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1590 if (!skb) {
1591 rc = -ENOMEM;
1592 goto next_rx;
1593 }
1594 }
1595
1596 if (RX_CMP_HASH_VALID(rxcmp)) {
1597 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1598 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1599
1600 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1601 if (hash_type != 1 && hash_type != 3)
1602 type = PKT_HASH_TYPE_L3;
1603 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1604 }
1605
ee5c7fb3
SP		 1606 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1607 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
c0c050c5 1608
8852ddb4
MC		 1609 if ((rxcmp1->rx_cmp_flags2 &
1610 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1611 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5 1612 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
8852ddb4 1613 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
c0c050c5
MC		 1614 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1615
8852ddb4 1616 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1617 }
1618
1619 skb_checksum_none_assert(skb);
1620 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1621 if (dev->features & NETIF_F_RXCSUM) {
1622 skb->ip_summed = CHECKSUM_UNNECESSARY;
1623 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1624 }
1625 } else {
665e350d
SB		 1626 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1627 if (dev->features & NETIF_F_RXCSUM)
1628 cpr->rx_l4_csum_errors++;
1629 }
c0c050c5
MC		 1630 }
1631
ee5c7fb3 1632 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1633 rc = 1;
1634
1635next_rx:
1636 rxr->rx_prod = NEXT_RX(prod);
376a5b86 1637 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 1638
1639next_rx_no_prod:
1640 *raw_cons = tmp_raw_cons;
1641
1642 return rc;
1643}
1644
2270bc5d
MC		 1645/* In netpoll mode, if we are using a combined completion ring, we need to
1646 * discard the rx packets and recycle the buffers.
1647 */
1648static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi,
1649 u32 *raw_cons, u8 *event)
1650{
1651 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1652 u32 tmp_raw_cons = *raw_cons;
1653 struct rx_cmp_ext *rxcmp1;
1654 struct rx_cmp *rxcmp;
1655 u16 cp_cons;
1656 u8 cmp_type;
1657
1658 cp_cons = RING_CMP(tmp_raw_cons);
1659 rxcmp = (struct rx_cmp *)
1660 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1661
1662 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1663 cp_cons = RING_CMP(tmp_raw_cons);
1664 rxcmp1 = (struct rx_cmp_ext *)
1665 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1666
1667 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1668 return -EBUSY;
1669
1670 cmp_type = RX_CMP_TYPE(rxcmp);
1671 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1672 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1673 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1674 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1675 struct rx_tpa_end_cmp_ext *tpa_end1;
1676
1677 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1678 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1679 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1680 }
1681 return bnxt_rx_pkt(bp, bnapi, raw_cons, event);
1682}
1683
4bb13abf 1684#define BNXT_GET_EVENT_PORT(data) \
87c374de
MC		 1685 ((data) & \
1686 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
4bb13abf 1687
c0c050c5
MC		 1688static int bnxt_async_event_process(struct bnxt *bp,
1689 struct hwrm_async_event_cmpl *cmpl)
1690{
1691 u16 event_id = le16_to_cpu(cmpl->event_id);
1692
1693 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1694 switch (event_id) {
87c374de 1695 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
8cbde117
MC		 1696 u32 data1 = le32_to_cpu(cmpl->event_data1);
1697 struct bnxt_link_info *link_info = &bp->link_info;
1698
1699 if (BNXT_VF(bp))
1700 goto async_event_process_exit;
1701 if (data1 & 0x20000) {
1702 u16 fw_speed = link_info->force_link_speed;
1703 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1704
1705 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1706 speed);
1707 }
286ef9d6 1708 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
8cbde117
MC		 1709 /* fall thru */
1710 }
87c374de 1711 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
c0c050c5 1712 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
19241368 1713 break;
87c374de 1714 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
19241368 1715 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
c0c050c5 1716 break;
87c374de 1717 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
4bb13abf
MC		 1718 u32 data1 = le32_to_cpu(cmpl->event_data1);
1719 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1720
1721 if (BNXT_VF(bp))
1722 break;
1723
1724 if (bp->pf.port_id != port_id)
1725 break;
1726
4bb13abf
MC		 1727 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1728 break;
1729 }
87c374de 1730 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
fc0f1929
MC		 1731 if (BNXT_PF(bp))
1732 goto async_event_process_exit;
1733 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1734 break;
c0c050c5 1735 default:
19241368 1736 goto async_event_process_exit;
c0c050c5 1737 }
c213eae8 1738 bnxt_queue_sp_work(bp);
19241368 1739async_event_process_exit:
a588e458 1740 bnxt_ulp_async_events(bp, cmpl);
c0c050c5
MC		 1741 return 0;
1742}
1743
1744static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1745{
1746 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1747 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1748 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1749 (struct hwrm_fwd_req_cmpl *)txcmp;
1750
1751 switch (cmpl_type) {
1752 case CMPL_BASE_TYPE_HWRM_DONE:
1753 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1754 if (seq_id == bp->hwrm_intr_seq_id)
1755 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1756 else
1757 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1758 break;
1759
1760 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1761 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1762
1763 if ((vf_id < bp->pf.first_vf_id) ||
1764 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1765 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1766 vf_id);
1767 return -EINVAL;
1768 }
1769
1770 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1771 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
c213eae8 1772 bnxt_queue_sp_work(bp);
c0c050c5
MC		 1773 break;
1774
1775 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1776 bnxt_async_event_process(bp,
1777 (struct hwrm_async_event_cmpl *)txcmp);
1778
1779 default:
1780 break;
1781 }
1782
1783 return 0;
1784}
1785
1786static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1787{
1788 struct bnxt_napi *bnapi = dev_instance;
1789 struct bnxt *bp = bnapi->bp;
1790 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1791 u32 cons = RING_CMP(cpr->cp_raw_cons);
1792
1793 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1794 napi_schedule(&bnapi->napi);
1795 return IRQ_HANDLED;
1796}
1797
1798static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1799{
1800 u32 raw_cons = cpr->cp_raw_cons;
1801 u16 cons = RING_CMP(raw_cons);
1802 struct tx_cmp *txcmp;
1803
1804 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1805
1806 return TX_CMP_VALID(txcmp, raw_cons);
1807}
1808
c0c050c5
MC		 1809static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1810{
1811 struct bnxt_napi *bnapi = dev_instance;
1812 struct bnxt *bp = bnapi->bp;
1813 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1814 u32 cons = RING_CMP(cpr->cp_raw_cons);
1815 u32 int_status;
1816
1817 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1818
1819 if (!bnxt_has_work(bp, cpr)) {
11809490 1820 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
c0c050c5
MC		 1821 /* return if erroneous interrupt */
1822 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1823 return IRQ_NONE;
1824 }
1825
1826 /* disable ring IRQ */
1827 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1828
1829 /* Return here if interrupt is shared and is disabled. */
1830 if (unlikely(atomic_read(&bp->intr_sem) != 0))
1831 return IRQ_HANDLED;
1832
1833 napi_schedule(&bnapi->napi);
1834 return IRQ_HANDLED;
1835}
1836
1837static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1838{
1839 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1840 u32 raw_cons = cpr->cp_raw_cons;
1841 u32 cons;
1842 int tx_pkts = 0;
1843 int rx_pkts = 0;
4e5dbbda 1844 u8 event = 0;
c0c050c5
MC		 1845 struct tx_cmp *txcmp;
1846
1847 while (1) {
1848 int rc;
1849
1850 cons = RING_CMP(raw_cons);
1851 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1852
1853 if (!TX_CMP_VALID(txcmp, raw_cons))
1854 break;
1855
67a95e20
MC		 1856 /* The valid test of the entry must be done first before
1857 * reading any further.
1858 */
b67daab0 1859 dma_rmb();
c0c050c5
MC		 1860 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1861 tx_pkts++;
1862 /* return full budget so NAPI will complete. */
1863 if (unlikely(tx_pkts > bp->tx_wake_thresh))
1864 rx_pkts = budget;
1865 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2270bc5d
MC		 1866 if (likely(budget))
1867 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1868 else
1869 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons,
1870 &event);
c0c050c5
MC		 1871 if (likely(rc >= 0))
1872 rx_pkts += rc;
903649e7
MC		 1873 /* Increment rx_pkts when rc is -ENOMEM to count towards
1874 * the NAPI budget. Otherwise, we may potentially loop
1875 * here forever if we consistently cannot allocate
1876 * buffers.
1877 */
1878 else if (rc == -ENOMEM)
1879 rx_pkts++;
c0c050c5
MC		 1880 else if (rc == -EBUSY) /* partial completion */
1881 break;
c0c050c5
MC		 1882 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1883 CMPL_BASE_TYPE_HWRM_DONE) ||
1884 (TX_CMP_TYPE(txcmp) ==
1885 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1886 (TX_CMP_TYPE(txcmp) ==
1887 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1888 bnxt_hwrm_handler(bp, txcmp);
1889 }
1890 raw_cons = NEXT_RAW_CMP(raw_cons);
1891
1892 if (rx_pkts == budget)
1893 break;
1894 }
1895
38413406
MC		 1896 if (event & BNXT_TX_EVENT) {
1897 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1898 void __iomem *db = txr->tx_doorbell;
1899 u16 prod = txr->tx_prod;
1900
1901 /* Sync BD data before updating doorbell */
1902 wmb();
1903
434c975a 1904 bnxt_db_write(bp, db, DB_KEY_TX | prod);
38413406
MC		 1905 }
1906
c0c050c5
MC		 1907 cpr->cp_raw_cons = raw_cons;
1908 /* ACK completion ring before freeing tx ring and producing new
1909 * buffers in rx/agg rings to prevent overflowing the completion
1910 * ring.
1911 */
1912 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1913
1914 if (tx_pkts)
fa3e93e8 1915 bnapi->tx_int(bp, bnapi, tx_pkts);
c0c050c5 1916
4e5dbbda 1917 if (event & BNXT_RX_EVENT) {
b6ab4b01 1918 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 1919
434c975a
MC		 1920 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1921 if (event & BNXT_AGG_EVENT)
1922 bnxt_db_write(bp, rxr->rx_agg_doorbell,
1923 DB_KEY_RX | rxr->rx_agg_prod);
c0c050c5
MC		 1924 }
1925 return rx_pkts;
1926}
1927
10bbdaf5
PS		 1928static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
1929{
1930 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1931 struct bnxt *bp = bnapi->bp;
1932 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1933 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1934 struct tx_cmp *txcmp;
1935 struct rx_cmp_ext *rxcmp1;
1936 u32 cp_cons, tmp_raw_cons;
1937 u32 raw_cons = cpr->cp_raw_cons;
1938 u32 rx_pkts = 0;
4e5dbbda 1939 u8 event = 0;
10bbdaf5
PS		 1940
1941 while (1) {
1942 int rc;
1943
1944 cp_cons = RING_CMP(raw_cons);
1945 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1946
1947 if (!TX_CMP_VALID(txcmp, raw_cons))
1948 break;
1949
1950 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1951 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
1952 cp_cons = RING_CMP(tmp_raw_cons);
1953 rxcmp1 = (struct rx_cmp_ext *)
1954 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1955
1956 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1957 break;
1958
1959 /* force an error to recycle the buffer */
1960 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1961 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1962
4e5dbbda 1963 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
10bbdaf5
PS		 1964 if (likely(rc == -EIO))
1965 rx_pkts++;
1966 else if (rc == -EBUSY) /* partial completion */
1967 break;
1968 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
1969 CMPL_BASE_TYPE_HWRM_DONE)) {
1970 bnxt_hwrm_handler(bp, txcmp);
1971 } else {
1972 netdev_err(bp->dev,
1973 "Invalid completion received on special ring\n");
1974 }
1975 raw_cons = NEXT_RAW_CMP(raw_cons);
1976
1977 if (rx_pkts == budget)
1978 break;
1979 }
1980
1981 cpr->cp_raw_cons = raw_cons;
1982 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
434c975a 1983 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
10bbdaf5 1984
434c975a
MC		 1985 if (event & BNXT_AGG_EVENT)
1986 bnxt_db_write(bp, rxr->rx_agg_doorbell,
1987 DB_KEY_RX | rxr->rx_agg_prod);
10bbdaf5
PS		 1988
1989 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
6ad20165 1990 napi_complete_done(napi, rx_pkts);
10bbdaf5
PS		 1991 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1992 }
1993 return rx_pkts;
1994}
1995
c0c050c5
MC		 1996static int bnxt_poll(struct napi_struct *napi, int budget)
1997{
1998 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1999 struct bnxt *bp = bnapi->bp;
2000 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2001 int work_done = 0;
2002
c0c050c5
MC		 2003 while (1) {
2004 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
2005
2006 if (work_done >= budget)
2007 break;
2008
2009 if (!bnxt_has_work(bp, cpr)) {
e7b95691
MC		 2010 if (napi_complete_done(napi, work_done))
2011 BNXT_CP_DB_REARM(cpr->cp_doorbell,
2012 cpr->cp_raw_cons);
c0c050c5
MC		 2013 break;
2014 }
2015 }
2016 mmiowb();
c0c050c5
MC		 2017 return work_done;
2018}
2019
c0c050c5
MC		 2020static void bnxt_free_tx_skbs(struct bnxt *bp)
2021{
2022 int i, max_idx;
2023 struct pci_dev *pdev = bp->pdev;
2024
b6ab4b01 2025 if (!bp->tx_ring)
c0c050c5
MC		 2026 return;
2027
2028 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2029 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2030 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2031 int j;
2032
c0c050c5
MC		 2033 for (j = 0; j < max_idx;) {
2034 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2035 struct sk_buff *skb = tx_buf->skb;
2036 int k, last;
2037
2038 if (!skb) {
2039 j++;
2040 continue;
2041 }
2042
2043 tx_buf->skb = NULL;
2044
2045 if (tx_buf->is_push) {
2046 dev_kfree_skb(skb);
2047 j += 2;
2048 continue;
2049 }
2050
2051 dma_unmap_single(&pdev->dev,
2052 dma_unmap_addr(tx_buf, mapping),
2053 skb_headlen(skb),
2054 PCI_DMA_TODEVICE);
2055
2056 last = tx_buf->nr_frags;
2057 j += 2;
d612a579
MC		 2058 for (k = 0; k < last; k++, j++) {
2059 int ring_idx = j & bp->tx_ring_mask;
c0c050c5
MC		 2060 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2061
d612a579 2062 tx_buf = &txr->tx_buf_ring[ring_idx];
c0c050c5
MC		 2063 dma_unmap_page(
2064 &pdev->dev,
2065 dma_unmap_addr(tx_buf, mapping),
2066 skb_frag_size(frag), PCI_DMA_TODEVICE);
2067 }
2068 dev_kfree_skb(skb);
2069 }
2070 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2071 }
2072}
2073
2074static void bnxt_free_rx_skbs(struct bnxt *bp)
2075{
2076 int i, max_idx, max_agg_idx;
2077 struct pci_dev *pdev = bp->pdev;
2078
b6ab4b01 2079 if (!bp->rx_ring)
c0c050c5
MC		 2080 return;
2081
2082 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2083 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2084 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2085 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2086 int j;
2087
c0c050c5
MC		 2088 if (rxr->rx_tpa) {
2089 for (j = 0; j < MAX_TPA; j++) {
2090 struct bnxt_tpa_info *tpa_info =
2091 &rxr->rx_tpa[j];
2092 u8 *data = tpa_info->data;
2093
2094 if (!data)
2095 continue;
2096
c519fe9a
SN		 2097 dma_unmap_single_attrs(&pdev->dev,
2098 tpa_info->mapping,
2099 bp->rx_buf_use_size,
2100 bp->rx_dir,
2101 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2102
2103 tpa_info->data = NULL;
2104
2105 kfree(data);
2106 }
2107 }
2108
2109 for (j = 0; j < max_idx; j++) {
2110 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
3ed3a83e 2111 dma_addr_t mapping = rx_buf->mapping;
6bb19474 2112 void *data = rx_buf->data;
c0c050c5
MC		 2113
2114 if (!data)
2115 continue;
2116
c0c050c5
MC		 2117 rx_buf->data = NULL;
2118
3ed3a83e
MC		 2119 if (BNXT_RX_PAGE_MODE(bp)) {
2120 mapping -= bp->rx_dma_offset;
c519fe9a
SN		 2121 dma_unmap_page_attrs(&pdev->dev, mapping,
2122 PAGE_SIZE, bp->rx_dir,
2123 DMA_ATTR_WEAK_ORDERING);
c61fb99c 2124 __free_page(data);
3ed3a83e 2125 } else {
c519fe9a
SN		 2126 dma_unmap_single_attrs(&pdev->dev, mapping,
2127 bp->rx_buf_use_size,
2128 bp->rx_dir,
2129 DMA_ATTR_WEAK_ORDERING);
c61fb99c 2130 kfree(data);
3ed3a83e 2131 }
c0c050c5
MC		 2132 }
2133
2134 for (j = 0; j < max_agg_idx; j++) {
2135 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2136 &rxr->rx_agg_ring[j];
2137 struct page *page = rx_agg_buf->page;
2138
2139 if (!page)
2140 continue;
2141
c519fe9a
SN		 2142 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2143 BNXT_RX_PAGE_SIZE,
2144 PCI_DMA_FROMDEVICE,
2145 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2146
2147 rx_agg_buf->page = NULL;
2148 __clear_bit(j, rxr->rx_agg_bmap);
2149
2150 __free_page(page);
2151 }
89d0a06c
MC		 2152 if (rxr->rx_page) {
2153 __free_page(rxr->rx_page);
2154 rxr->rx_page = NULL;
2155 }
c0c050c5
MC		 2156 }
2157}
2158
2159static void bnxt_free_skbs(struct bnxt *bp)
2160{
2161 bnxt_free_tx_skbs(bp);
2162 bnxt_free_rx_skbs(bp);
2163}
2164
2165static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2166{
2167 struct pci_dev *pdev = bp->pdev;
2168 int i;
2169
2170 for (i = 0; i < ring->nr_pages; i++) {
2171 if (!ring->pg_arr[i])
2172 continue;
2173
2174 dma_free_coherent(&pdev->dev, ring->page_size,
2175 ring->pg_arr[i], ring->dma_arr[i]);
2176
2177 ring->pg_arr[i] = NULL;
2178 }
2179 if (ring->pg_tbl) {
2180 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
2181 ring->pg_tbl, ring->pg_tbl_map);
2182 ring->pg_tbl = NULL;
2183 }
2184 if (ring->vmem_size && *ring->vmem) {
2185 vfree(*ring->vmem);
2186 *ring->vmem = NULL;
2187 }
2188}
2189
2190static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2191{
2192 int i;
2193 struct pci_dev *pdev = bp->pdev;
2194
2195 if (ring->nr_pages > 1) {
2196 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
2197 ring->nr_pages * 8,
2198 &ring->pg_tbl_map,
2199 GFP_KERNEL);
2200 if (!ring->pg_tbl)
2201 return -ENOMEM;
2202 }
2203
2204 for (i = 0; i < ring->nr_pages; i++) {
2205 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2206 ring->page_size,
2207 &ring->dma_arr[i],
2208 GFP_KERNEL);
2209 if (!ring->pg_arr[i])
2210 return -ENOMEM;
2211
2212 if (ring->nr_pages > 1)
2213 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
2214 }
2215
2216 if (ring->vmem_size) {
2217 *ring->vmem = vzalloc(ring->vmem_size);
2218 if (!(*ring->vmem))
2219 return -ENOMEM;
2220 }
2221 return 0;
2222}
2223
2224static void bnxt_free_rx_rings(struct bnxt *bp)
2225{
2226 int i;
2227
b6ab4b01 2228 if (!bp->rx_ring)
c0c050c5
MC		 2229 return;
2230
2231 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2232 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2233 struct bnxt_ring_struct *ring;
2234
c6d30e83
MC		 2235 if (rxr->xdp_prog)
2236 bpf_prog_put(rxr->xdp_prog);
2237
c0c050c5
MC		 2238 kfree(rxr->rx_tpa);
2239 rxr->rx_tpa = NULL;
2240
2241 kfree(rxr->rx_agg_bmap);
2242 rxr->rx_agg_bmap = NULL;
2243
2244 ring = &rxr->rx_ring_struct;
2245 bnxt_free_ring(bp, ring);
2246
2247 ring = &rxr->rx_agg_ring_struct;
2248 bnxt_free_ring(bp, ring);
2249 }
2250}
2251
2252static int bnxt_alloc_rx_rings(struct bnxt *bp)
2253{
2254 int i, rc, agg_rings = 0, tpa_rings = 0;
2255
b6ab4b01
MC		 2256 if (!bp->rx_ring)
2257 return -ENOMEM;
2258
c0c050c5
MC		 2259 if (bp->flags & BNXT_FLAG_AGG_RINGS)
2260 agg_rings = 1;
2261
2262 if (bp->flags & BNXT_FLAG_TPA)
2263 tpa_rings = 1;
2264
2265 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2266 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2267 struct bnxt_ring_struct *ring;
2268
c0c050c5
MC		 2269 ring = &rxr->rx_ring_struct;
2270
2271 rc = bnxt_alloc_ring(bp, ring);
2272 if (rc)
2273 return rc;
2274
2275 if (agg_rings) {
2276 u16 mem_size;
2277
2278 ring = &rxr->rx_agg_ring_struct;
2279 rc = bnxt_alloc_ring(bp, ring);
2280 if (rc)
2281 return rc;
2282
2283 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2284 mem_size = rxr->rx_agg_bmap_size / 8;
2285 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2286 if (!rxr->rx_agg_bmap)
2287 return -ENOMEM;
2288
2289 if (tpa_rings) {
2290 rxr->rx_tpa = kcalloc(MAX_TPA,
2291 sizeof(struct bnxt_tpa_info),
2292 GFP_KERNEL);
2293 if (!rxr->rx_tpa)
2294 return -ENOMEM;
2295 }
2296 }
2297 }
2298 return 0;
2299}
2300
2301static void bnxt_free_tx_rings(struct bnxt *bp)
2302{
2303 int i;
2304 struct pci_dev *pdev = bp->pdev;
2305
b6ab4b01 2306 if (!bp->tx_ring)
c0c050c5
MC		 2307 return;
2308
2309 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2310 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2311 struct bnxt_ring_struct *ring;
2312
c0c050c5
MC		 2313 if (txr->tx_push) {
2314 dma_free_coherent(&pdev->dev, bp->tx_push_size,
2315 txr->tx_push, txr->tx_push_mapping);
2316 txr->tx_push = NULL;
2317 }
2318
2319 ring = &txr->tx_ring_struct;
2320
2321 bnxt_free_ring(bp, ring);
2322 }
2323}
2324
2325static int bnxt_alloc_tx_rings(struct bnxt *bp)
2326{
2327 int i, j, rc;
2328 struct pci_dev *pdev = bp->pdev;
2329
2330 bp->tx_push_size = 0;
2331 if (bp->tx_push_thresh) {
2332 int push_size;
2333
2334 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2335 bp->tx_push_thresh);
2336
4419dbe6 2337 if (push_size > 256) {
c0c050c5
MC		 2338 push_size = 0;
2339 bp->tx_push_thresh = 0;
2340 }
2341
2342 bp->tx_push_size = push_size;
2343 }
2344
2345 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2346 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2347 struct bnxt_ring_struct *ring;
2348
c0c050c5
MC		 2349 ring = &txr->tx_ring_struct;
2350
2351 rc = bnxt_alloc_ring(bp, ring);
2352 if (rc)
2353 return rc;
2354
2355 if (bp->tx_push_size) {
c0c050c5
MC		 2356 dma_addr_t mapping;
2357
2358 /* One pre-allocated DMA buffer to backup
2359 * TX push operation
2360 */
2361 txr->tx_push = dma_alloc_coherent(&pdev->dev,
2362 bp->tx_push_size,
2363 &txr->tx_push_mapping,
2364 GFP_KERNEL);
2365
2366 if (!txr->tx_push)
2367 return -ENOMEM;
2368
c0c050c5
MC		 2369 mapping = txr->tx_push_mapping +
2370 sizeof(struct tx_push_bd);
4419dbe6 2371 txr->data_mapping = cpu_to_le64(mapping);
c0c050c5 2372
4419dbe6 2373 memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
c0c050c5
MC		 2374 }
2375 ring->queue_id = bp->q_info[j].queue_id;
5f449249
MC		 2376 if (i < bp->tx_nr_rings_xdp)
2377 continue;
c0c050c5
MC		 2378 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2379 j++;
2380 }
2381 return 0;
2382}
2383
2384static void bnxt_free_cp_rings(struct bnxt *bp)
2385{
2386 int i;
2387
2388 if (!bp->bnapi)
2389 return;
2390
2391 for (i = 0; i < bp->cp_nr_rings; i++) {
2392 struct bnxt_napi *bnapi = bp->bnapi[i];
2393 struct bnxt_cp_ring_info *cpr;
2394 struct bnxt_ring_struct *ring;
2395
2396 if (!bnapi)
2397 continue;
2398
2399 cpr = &bnapi->cp_ring;
2400 ring = &cpr->cp_ring_struct;
2401
2402 bnxt_free_ring(bp, ring);
2403 }
2404}
2405
2406static int bnxt_alloc_cp_rings(struct bnxt *bp)
2407{
2408 int i, rc;
2409
2410 for (i = 0; i < bp->cp_nr_rings; i++) {
2411 struct bnxt_napi *bnapi = bp->bnapi[i];
2412 struct bnxt_cp_ring_info *cpr;
2413 struct bnxt_ring_struct *ring;
2414
2415 if (!bnapi)
2416 continue;
2417
2418 cpr = &bnapi->cp_ring;
2419 ring = &cpr->cp_ring_struct;
2420
2421 rc = bnxt_alloc_ring(bp, ring);
2422 if (rc)
2423 return rc;
2424 }
2425 return 0;
2426}
2427
2428static void bnxt_init_ring_struct(struct bnxt *bp)
2429{
2430 int i;
2431
2432 for (i = 0; i < bp->cp_nr_rings; i++) {
2433 struct bnxt_napi *bnapi = bp->bnapi[i];
2434 struct bnxt_cp_ring_info *cpr;
2435 struct bnxt_rx_ring_info *rxr;
2436 struct bnxt_tx_ring_info *txr;
2437 struct bnxt_ring_struct *ring;
2438
2439 if (!bnapi)
2440 continue;
2441
2442 cpr = &bnapi->cp_ring;
2443 ring = &cpr->cp_ring_struct;
2444 ring->nr_pages = bp->cp_nr_pages;
2445 ring->page_size = HW_CMPD_RING_SIZE;
2446 ring->pg_arr = (void **)cpr->cp_desc_ring;
2447 ring->dma_arr = cpr->cp_desc_mapping;
2448 ring->vmem_size = 0;
2449
b6ab4b01 2450 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 2451 if (!rxr)
2452 goto skip_rx;
2453
c0c050c5
MC		 2454 ring = &rxr->rx_ring_struct;
2455 ring->nr_pages = bp->rx_nr_pages;
2456 ring->page_size = HW_RXBD_RING_SIZE;
2457 ring->pg_arr = (void **)rxr->rx_desc_ring;
2458 ring->dma_arr = rxr->rx_desc_mapping;
2459 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2460 ring->vmem = (void **)&rxr->rx_buf_ring;
2461
2462 ring = &rxr->rx_agg_ring_struct;
2463 ring->nr_pages = bp->rx_agg_nr_pages;
2464 ring->page_size = HW_RXBD_RING_SIZE;
2465 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2466 ring->dma_arr = rxr->rx_agg_desc_mapping;
2467 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2468 ring->vmem = (void **)&rxr->rx_agg_ring;
2469
3b2b7d9d 2470skip_rx:
b6ab4b01 2471 txr = bnapi->tx_ring;
3b2b7d9d
MC		 2472 if (!txr)
2473 continue;
2474
c0c050c5
MC		 2475 ring = &txr->tx_ring_struct;
2476 ring->nr_pages = bp->tx_nr_pages;
2477 ring->page_size = HW_RXBD_RING_SIZE;
2478 ring->pg_arr = (void **)txr->tx_desc_ring;
2479 ring->dma_arr = txr->tx_desc_mapping;
2480 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2481 ring->vmem = (void **)&txr->tx_buf_ring;
2482 }
2483}
2484
2485static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2486{
2487 int i;
2488 u32 prod;
2489 struct rx_bd **rx_buf_ring;
2490
2491 rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2492 for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2493 int j;
2494 struct rx_bd *rxbd;
2495
2496 rxbd = rx_buf_ring[i];
2497 if (!rxbd)
2498 continue;
2499
2500 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2501 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2502 rxbd->rx_bd_opaque = prod;
2503 }
2504 }
2505}
2506
2507static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2508{
2509 struct net_device *dev = bp->dev;
c0c050c5
MC		 2510 struct bnxt_rx_ring_info *rxr;
2511 struct bnxt_ring_struct *ring;
2512 u32 prod, type;
2513 int i;
2514
c0c050c5
MC		 2515 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2516 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2517
2518 if (NET_IP_ALIGN == 2)
2519 type |= RX_BD_FLAGS_SOP;
2520
b6ab4b01 2521 rxr = &bp->rx_ring[ring_nr];
c0c050c5
MC		 2522 ring = &rxr->rx_ring_struct;
2523 bnxt_init_rxbd_pages(ring, type);
2524
c6d30e83
MC		 2525 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2526 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2527 if (IS_ERR(rxr->xdp_prog)) {
2528 int rc = PTR_ERR(rxr->xdp_prog);
2529
2530 rxr->xdp_prog = NULL;
2531 return rc;
2532 }
2533 }
c0c050c5
MC		 2534 prod = rxr->rx_prod;
2535 for (i = 0; i < bp->rx_ring_size; i++) {
2536 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2537 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2538 ring_nr, i, bp->rx_ring_size);
2539 break;
2540 }
2541 prod = NEXT_RX(prod);
2542 }
2543 rxr->rx_prod = prod;
2544 ring->fw_ring_id = INVALID_HW_RING_ID;
2545
edd0c2cc
MC		 2546 ring = &rxr->rx_agg_ring_struct;
2547 ring->fw_ring_id = INVALID_HW_RING_ID;
2548
c0c050c5
MC		 2549 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2550 return 0;
2551
2839f28b 2552 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
c0c050c5
MC		 2553 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2554
2555 bnxt_init_rxbd_pages(ring, type);
2556
2557 prod = rxr->rx_agg_prod;
2558 for (i = 0; i < bp->rx_agg_ring_size; i++) {
2559 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2560 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2561 ring_nr, i, bp->rx_ring_size);
2562 break;
2563 }
2564 prod = NEXT_RX_AGG(prod);
2565 }
2566 rxr->rx_agg_prod = prod;
c0c050c5
MC		 2567
2568 if (bp->flags & BNXT_FLAG_TPA) {
2569 if (rxr->rx_tpa) {
2570 u8 *data;
2571 dma_addr_t mapping;
2572
2573 for (i = 0; i < MAX_TPA; i++) {
2574 data = __bnxt_alloc_rx_data(bp, &mapping,
2575 GFP_KERNEL);
2576 if (!data)
2577 return -ENOMEM;
2578
2579 rxr->rx_tpa[i].data = data;
b3dba77c 2580 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
c0c050c5
MC		 2581 rxr->rx_tpa[i].mapping = mapping;
2582 }
2583 } else {
2584 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2585 return -ENOMEM;
2586 }
2587 }
2588
2589 return 0;
2590}
2591
2247925f
SP		 2592static void bnxt_init_cp_rings(struct bnxt *bp)
2593{
2594 int i;
2595
2596 for (i = 0; i < bp->cp_nr_rings; i++) {
2597 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2598 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2599
2600 ring->fw_ring_id = INVALID_HW_RING_ID;
2601 }
2602}
2603
c0c050c5
MC		 2604static int bnxt_init_rx_rings(struct bnxt *bp)
2605{
2606 int i, rc = 0;
2607
c61fb99c 2608 if (BNXT_RX_PAGE_MODE(bp)) {
c6d30e83
MC		 2609 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2610 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
c61fb99c
MC		 2611 } else {
2612 bp->rx_offset = BNXT_RX_OFFSET;
2613 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2614 }
b3dba77c 2615
c0c050c5
MC		 2616 for (i = 0; i < bp->rx_nr_rings; i++) {
2617 rc = bnxt_init_one_rx_ring(bp, i);
2618 if (rc)
2619 break;
2620 }
2621
2622 return rc;
2623}
2624
2625static int bnxt_init_tx_rings(struct bnxt *bp)
2626{
2627 u16 i;
2628
2629 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2630 MAX_SKB_FRAGS + 1);
2631
2632 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2633 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2634 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2635
2636 ring->fw_ring_id = INVALID_HW_RING_ID;
2637 }
2638
2639 return 0;
2640}
2641
2642static void bnxt_free_ring_grps(struct bnxt *bp)
2643{
2644 kfree(bp->grp_info);
2645 bp->grp_info = NULL;
2646}
2647
2648static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2649{
2650 int i;
2651
2652 if (irq_re_init) {
2653 bp->grp_info = kcalloc(bp->cp_nr_rings,
2654 sizeof(struct bnxt_ring_grp_info),
2655 GFP_KERNEL);
2656 if (!bp->grp_info)
2657 return -ENOMEM;
2658 }
2659 for (i = 0; i < bp->cp_nr_rings; i++) {
2660 if (irq_re_init)
2661 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2662 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2663 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2664 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2665 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2666 }
2667 return 0;
2668}
2669
2670static void bnxt_free_vnics(struct bnxt *bp)
2671{
2672 kfree(bp->vnic_info);
2673 bp->vnic_info = NULL;
2674 bp->nr_vnics = 0;
2675}
2676
2677static int bnxt_alloc_vnics(struct bnxt *bp)
2678{
2679 int num_vnics = 1;
2680
2681#ifdef CONFIG_RFS_ACCEL
2682 if (bp->flags & BNXT_FLAG_RFS)
2683 num_vnics += bp->rx_nr_rings;
2684#endif
2685
dc52c6c7
PS		 2686 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
2687 num_vnics++;
2688
c0c050c5
MC		 2689 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2690 GFP_KERNEL);
2691 if (!bp->vnic_info)
2692 return -ENOMEM;
2693
2694 bp->nr_vnics = num_vnics;
2695 return 0;
2696}
2697
2698static void bnxt_init_vnics(struct bnxt *bp)
2699{
2700 int i;
2701
2702 for (i = 0; i < bp->nr_vnics; i++) {
2703 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2704
2705 vnic->fw_vnic_id = INVALID_HW_RING_ID;
94ce9caa
PS		 2706 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
2707 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
c0c050c5
MC		 2708 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2709
2710 if (bp->vnic_info[i].rss_hash_key) {
2711 if (i == 0)
2712 prandom_bytes(vnic->rss_hash_key,
2713 HW_HASH_KEY_SIZE);
2714 else
2715 memcpy(vnic->rss_hash_key,
2716 bp->vnic_info[0].rss_hash_key,
2717 HW_HASH_KEY_SIZE);
2718 }
2719 }
2720}
2721
2722static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2723{
2724 int pages;
2725
2726 pages = ring_size / desc_per_pg;
2727
2728 if (!pages)
2729 return 1;
2730
2731 pages++;
2732
2733 while (pages & (pages - 1))
2734 pages++;
2735
2736 return pages;
2737}
2738
c6d30e83 2739void bnxt_set_tpa_flags(struct bnxt *bp)
c0c050c5
MC		 2740{
2741 bp->flags &= ~BNXT_FLAG_TPA;
341138c3
MC		 2742 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
2743 return;
c0c050c5
MC		 2744 if (bp->dev->features & NETIF_F_LRO)
2745 bp->flags |= BNXT_FLAG_LRO;
94758f8d 2746 if (bp->dev->features & NETIF_F_GRO)
c0c050c5
MC		 2747 bp->flags |= BNXT_FLAG_GRO;
2748}
2749
2750/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2751 * be set on entry.
2752 */
2753void bnxt_set_ring_params(struct bnxt *bp)
2754{
2755 u32 ring_size, rx_size, rx_space;
2756 u32 agg_factor = 0, agg_ring_size = 0;
2757
2758 /* 8 for CRC and VLAN */
2759 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2760
2761 rx_space = rx_size + NET_SKB_PAD +
2762 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2763
2764 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2765 ring_size = bp->rx_ring_size;
2766 bp->rx_agg_ring_size = 0;
2767 bp->rx_agg_nr_pages = 0;
2768
2769 if (bp->flags & BNXT_FLAG_TPA)
2839f28b 2770 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
c0c050c5
MC		 2771
2772 bp->flags &= ~BNXT_FLAG_JUMBO;
bdbd1eb5 2773 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
c0c050c5
MC		 2774 u32 jumbo_factor;
2775
2776 bp->flags |= BNXT_FLAG_JUMBO;
2777 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2778 if (jumbo_factor > agg_factor)
2779 agg_factor = jumbo_factor;
2780 }
2781 agg_ring_size = ring_size * agg_factor;
2782
2783 if (agg_ring_size) {
2784 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2785 RX_DESC_CNT);
2786 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2787 u32 tmp = agg_ring_size;
2788
2789 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2790 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2791 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2792 tmp, agg_ring_size);
2793 }
2794 bp->rx_agg_ring_size = agg_ring_size;
2795 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2796 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2797 rx_space = rx_size + NET_SKB_PAD +
2798 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2799 }
2800
2801 bp->rx_buf_use_size = rx_size;
2802 bp->rx_buf_size = rx_space;
2803
2804 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2805 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2806
2807 ring_size = bp->tx_ring_size;
2808 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2809 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2810
2811 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2812 bp->cp_ring_size = ring_size;
2813
2814 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2815 if (bp->cp_nr_pages > MAX_CP_PAGES) {
2816 bp->cp_nr_pages = MAX_CP_PAGES;
2817 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2818 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2819 ring_size, bp->cp_ring_size);
2820 }
2821 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2822 bp->cp_ring_mask = bp->cp_bit - 1;
2823}
2824
c61fb99c 2825int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
6bb19474 2826{
c61fb99c
MC		 2827 if (page_mode) {
2828 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
2829 return -EOPNOTSUPP;
2830 bp->dev->max_mtu = BNXT_MAX_PAGE_MODE_MTU;
2831 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
2832 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
2833 bp->dev->hw_features &= ~NETIF_F_LRO;
2834 bp->dev->features &= ~NETIF_F_LRO;
2835 bp->rx_dir = DMA_BIDIRECTIONAL;
2836 bp->rx_skb_func = bnxt_rx_page_skb;
2837 } else {
2838 bp->dev->max_mtu = BNXT_MAX_MTU;
2839 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
2840 bp->rx_dir = DMA_FROM_DEVICE;
2841 bp->rx_skb_func = bnxt_rx_skb;
2842 }
6bb19474
MC		 2843 return 0;
2844}
2845
c0c050c5
MC		 2846static void bnxt_free_vnic_attributes(struct bnxt *bp)
2847{
2848 int i;
2849 struct bnxt_vnic_info *vnic;
2850 struct pci_dev *pdev = bp->pdev;
2851
2852 if (!bp->vnic_info)
2853 return;
2854
2855 for (i = 0; i < bp->nr_vnics; i++) {
2856 vnic = &bp->vnic_info[i];
2857
2858 kfree(vnic->fw_grp_ids);
2859 vnic->fw_grp_ids = NULL;
2860
2861 kfree(vnic->uc_list);
2862 vnic->uc_list = NULL;
2863
2864 if (vnic->mc_list) {
2865 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2866 vnic->mc_list, vnic->mc_list_mapping);
2867 vnic->mc_list = NULL;
2868 }
2869
2870 if (vnic->rss_table) {
2871 dma_free_coherent(&pdev->dev, PAGE_SIZE,
2872 vnic->rss_table,
2873 vnic->rss_table_dma_addr);
2874 vnic->rss_table = NULL;
2875 }
2876
2877 vnic->rss_hash_key = NULL;
2878 vnic->flags = 0;
2879 }
2880}
2881
2882static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2883{
2884 int i, rc = 0, size;
2885 struct bnxt_vnic_info *vnic;
2886 struct pci_dev *pdev = bp->pdev;
2887 int max_rings;
2888
2889 for (i = 0; i < bp->nr_vnics; i++) {
2890 vnic = &bp->vnic_info[i];
2891
2892 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2893 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2894
2895 if (mem_size > 0) {
2896 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2897 if (!vnic->uc_list) {
2898 rc = -ENOMEM;
2899 goto out;
2900 }
2901 }
2902 }
2903
2904 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2905 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2906 vnic->mc_list =
2907 dma_alloc_coherent(&pdev->dev,
2908 vnic->mc_list_size,
2909 &vnic->mc_list_mapping,
2910 GFP_KERNEL);
2911 if (!vnic->mc_list) {
2912 rc = -ENOMEM;
2913 goto out;
2914 }
2915 }
2916
2917 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2918 max_rings = bp->rx_nr_rings;
2919 else
2920 max_rings = 1;
2921
2922 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2923 if (!vnic->fw_grp_ids) {
2924 rc = -ENOMEM;
2925 goto out;
2926 }
2927
ae10ae74
MC		 2928 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
2929 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
2930 continue;
2931
c0c050c5
MC		 2932 /* Allocate rss table and hash key */
2933 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2934 &vnic->rss_table_dma_addr,
2935 GFP_KERNEL);
2936 if (!vnic->rss_table) {
2937 rc = -ENOMEM;
2938 goto out;
2939 }
2940
2941 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2942
2943 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2944 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2945 }
2946 return 0;
2947
2948out:
2949 return rc;
2950}
2951
2952static void bnxt_free_hwrm_resources(struct bnxt *bp)
2953{
2954 struct pci_dev *pdev = bp->pdev;
2955
2956 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2957 bp->hwrm_cmd_resp_dma_addr);
2958
2959 bp->hwrm_cmd_resp_addr = NULL;
2960 if (bp->hwrm_dbg_resp_addr) {
2961 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2962 bp->hwrm_dbg_resp_addr,
2963 bp->hwrm_dbg_resp_dma_addr);
2964
2965 bp->hwrm_dbg_resp_addr = NULL;
2966 }
2967}
2968
2969static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2970{
2971 struct pci_dev *pdev = bp->pdev;
2972
2973 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2974 &bp->hwrm_cmd_resp_dma_addr,
2975 GFP_KERNEL);
2976 if (!bp->hwrm_cmd_resp_addr)
2977 return -ENOMEM;
2978 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2979 HWRM_DBG_REG_BUF_SIZE,
2980 &bp->hwrm_dbg_resp_dma_addr,
2981 GFP_KERNEL);
2982 if (!bp->hwrm_dbg_resp_addr)
2983 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2984
2985 return 0;
2986}
2987
e605db80
DK		 2988static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
2989{
2990 if (bp->hwrm_short_cmd_req_addr) {
2991 struct pci_dev *pdev = bp->pdev;
2992
2993 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
2994 bp->hwrm_short_cmd_req_addr,
2995 bp->hwrm_short_cmd_req_dma_addr);
2996 bp->hwrm_short_cmd_req_addr = NULL;
2997 }
2998}
2999
3000static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3001{
3002 struct pci_dev *pdev = bp->pdev;
3003
3004 bp->hwrm_short_cmd_req_addr =
3005 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3006 &bp->hwrm_short_cmd_req_dma_addr,
3007 GFP_KERNEL);
3008 if (!bp->hwrm_short_cmd_req_addr)
3009 return -ENOMEM;
3010
3011 return 0;
3012}
3013
c0c050c5
MC		 3014static void bnxt_free_stats(struct bnxt *bp)
3015{
3016 u32 size, i;
3017 struct pci_dev *pdev = bp->pdev;
3018
3bdf56c4
MC		 3019 if (bp->hw_rx_port_stats) {
3020 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3021 bp->hw_rx_port_stats,
3022 bp->hw_rx_port_stats_map);
3023 bp->hw_rx_port_stats = NULL;
3024 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3025 }
3026
c0c050c5
MC		 3027 if (!bp->bnapi)
3028 return;
3029
3030 size = sizeof(struct ctx_hw_stats);
3031
3032 for (i = 0; i < bp->cp_nr_rings; i++) {
3033 struct bnxt_napi *bnapi = bp->bnapi[i];
3034 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3035
3036 if (cpr->hw_stats) {
3037 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3038 cpr->hw_stats_map);
3039 cpr->hw_stats = NULL;
3040 }
3041 }
3042}
3043
3044static int bnxt_alloc_stats(struct bnxt *bp)
3045{
3046 u32 size, i;
3047 struct pci_dev *pdev = bp->pdev;
3048
3049 size = sizeof(struct ctx_hw_stats);
3050
3051 for (i = 0; i < bp->cp_nr_rings; i++) {
3052 struct bnxt_napi *bnapi = bp->bnapi[i];
3053 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3054
3055 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3056 &cpr->hw_stats_map,
3057 GFP_KERNEL);
3058 if (!cpr->hw_stats)
3059 return -ENOMEM;
3060
3061 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3062 }
3bdf56c4 3063
3e8060fa 3064 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3bdf56c4
MC		 3065 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3066 sizeof(struct tx_port_stats) + 1024;
3067
3068 bp->hw_rx_port_stats =
3069 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3070 &bp->hw_rx_port_stats_map,
3071 GFP_KERNEL);
3072 if (!bp->hw_rx_port_stats)
3073 return -ENOMEM;
3074
3075 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3076 512;
3077 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3078 sizeof(struct rx_port_stats) + 512;
3079 bp->flags |= BNXT_FLAG_PORT_STATS;
3080 }
c0c050c5
MC		 3081 return 0;
3082}
3083
3084static void bnxt_clear_ring_indices(struct bnxt *bp)
3085{
3086 int i;
3087
3088 if (!bp->bnapi)
3089 return;
3090
3091 for (i = 0; i < bp->cp_nr_rings; i++) {
3092 struct bnxt_napi *bnapi = bp->bnapi[i];
3093 struct bnxt_cp_ring_info *cpr;
3094 struct bnxt_rx_ring_info *rxr;
3095 struct bnxt_tx_ring_info *txr;
3096
3097 if (!bnapi)
3098 continue;
3099
3100 cpr = &bnapi->cp_ring;
3101 cpr->cp_raw_cons = 0;
3102
b6ab4b01 3103 txr = bnapi->tx_ring;
3b2b7d9d
MC		 3104 if (txr) {
3105 txr->tx_prod = 0;
3106 txr->tx_cons = 0;
3107 }
c0c050c5 3108
b6ab4b01 3109 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 3110 if (rxr) {
3111 rxr->rx_prod = 0;
3112 rxr->rx_agg_prod = 0;
3113 rxr->rx_sw_agg_prod = 0;
376a5b86 3114 rxr->rx_next_cons = 0;
3b2b7d9d 3115 }
c0c050c5
MC		 3116 }
3117}
3118
3119static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3120{
3121#ifdef CONFIG_RFS_ACCEL
3122 int i;
3123
3124 /* Under rtnl_lock and all our NAPIs have been disabled. It's
3125 * safe to delete the hash table.
3126 */
3127 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3128 struct hlist_head *head;
3129 struct hlist_node *tmp;
3130 struct bnxt_ntuple_filter *fltr;
3131
3132 head = &bp->ntp_fltr_hash_tbl[i];
3133 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3134 hlist_del(&fltr->hash);
3135 kfree(fltr);
3136 }
3137 }
3138 if (irq_reinit) {
3139 kfree(bp->ntp_fltr_bmap);
3140 bp->ntp_fltr_bmap = NULL;
3141 }
3142 bp->ntp_fltr_count = 0;
3143#endif
3144}
3145
3146static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3147{
3148#ifdef CONFIG_RFS_ACCEL
3149 int i, rc = 0;
3150
3151 if (!(bp->flags & BNXT_FLAG_RFS))
3152 return 0;
3153
3154 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3155 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3156
3157 bp->ntp_fltr_count = 0;
ac45bd93
DC		 3158 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3159 sizeof(long),
c0c050c5
MC		 3160 GFP_KERNEL);
3161
3162 if (!bp->ntp_fltr_bmap)
3163 rc = -ENOMEM;
3164
3165 return rc;
3166#else
3167 return 0;
3168#endif
3169}
3170
3171static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3172{
3173 bnxt_free_vnic_attributes(bp);
3174 bnxt_free_tx_rings(bp);
3175 bnxt_free_rx_rings(bp);
3176 bnxt_free_cp_rings(bp);
3177 bnxt_free_ntp_fltrs(bp, irq_re_init);
3178 if (irq_re_init) {
3179 bnxt_free_stats(bp);
3180 bnxt_free_ring_grps(bp);
3181 bnxt_free_vnics(bp);
a960dec9
MC		 3182 kfree(bp->tx_ring_map);
3183 bp->tx_ring_map = NULL;
b6ab4b01
MC		 3184 kfree(bp->tx_ring);
3185 bp->tx_ring = NULL;
3186 kfree(bp->rx_ring);
3187 bp->rx_ring = NULL;
c0c050c5
MC		 3188 kfree(bp->bnapi);
3189 bp->bnapi = NULL;
3190 } else {
3191 bnxt_clear_ring_indices(bp);
3192 }
3193}
3194
3195static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3196{
01657bcd 3197 int i, j, rc, size, arr_size;
c0c050c5
MC		 3198 void *bnapi;
3199
3200 if (irq_re_init) {
3201 /* Allocate bnapi mem pointer array and mem block for
3202 * all queues
3203 */
3204 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3205 bp->cp_nr_rings);
3206 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3207 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3208 if (!bnapi)
3209 return -ENOMEM;
3210
3211 bp->bnapi = bnapi;
3212 bnapi += arr_size;
3213 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3214 bp->bnapi[i] = bnapi;
3215 bp->bnapi[i]->index = i;
3216 bp->bnapi[i]->bp = bp;
3217 }
3218
b6ab4b01
MC		 3219 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3220 sizeof(struct bnxt_rx_ring_info),
3221 GFP_KERNEL);
3222 if (!bp->rx_ring)
3223 return -ENOMEM;
3224
3225 for (i = 0; i < bp->rx_nr_rings; i++) {
3226 bp->rx_ring[i].bnapi = bp->bnapi[i];
3227 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3228 }
3229
3230 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3231 sizeof(struct bnxt_tx_ring_info),
3232 GFP_KERNEL);
3233 if (!bp->tx_ring)
3234 return -ENOMEM;
3235
a960dec9
MC		 3236 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3237 GFP_KERNEL);
3238
3239 if (!bp->tx_ring_map)
3240 return -ENOMEM;
3241
01657bcd
MC		 3242 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3243 j = 0;
3244 else
3245 j = bp->rx_nr_rings;
3246
3247 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3248 bp->tx_ring[i].bnapi = bp->bnapi[j];
3249 bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
5f449249 3250 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
38413406 3251 if (i >= bp->tx_nr_rings_xdp) {
5f449249
MC		 3252 bp->tx_ring[i].txq_index = i -
3253 bp->tx_nr_rings_xdp;
38413406
MC		 3254 bp->bnapi[j]->tx_int = bnxt_tx_int;
3255 } else {
fa3e93e8 3256 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
38413406
MC		 3257 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3258 }
b6ab4b01
MC		 3259 }
3260
c0c050c5
MC		 3261 rc = bnxt_alloc_stats(bp);
3262 if (rc)
3263 goto alloc_mem_err;
3264
3265 rc = bnxt_alloc_ntp_fltrs(bp);
3266 if (rc)
3267 goto alloc_mem_err;
3268
3269 rc = bnxt_alloc_vnics(bp);
3270 if (rc)
3271 goto alloc_mem_err;
3272 }
3273
3274 bnxt_init_ring_struct(bp);
3275
3276 rc = bnxt_alloc_rx_rings(bp);
3277 if (rc)
3278 goto alloc_mem_err;
3279
3280 rc = bnxt_alloc_tx_rings(bp);
3281 if (rc)
3282 goto alloc_mem_err;
3283
3284 rc = bnxt_alloc_cp_rings(bp);
3285 if (rc)
3286 goto alloc_mem_err;
3287
3288 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3289 BNXT_VNIC_UCAST_FLAG;
3290 rc = bnxt_alloc_vnic_attributes(bp);
3291 if (rc)
3292 goto alloc_mem_err;
3293 return 0;
3294
3295alloc_mem_err:
3296 bnxt_free_mem(bp, true);
3297 return rc;
3298}
3299
9d8bc097
MC		 3300static void bnxt_disable_int(struct bnxt *bp)
3301{
3302 int i;
3303
3304 if (!bp->bnapi)
3305 return;
3306
3307 for (i = 0; i < bp->cp_nr_rings; i++) {
3308 struct bnxt_napi *bnapi = bp->bnapi[i];
3309 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
daf1f1e7 3310 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
9d8bc097 3311
daf1f1e7
MC		 3312 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3313 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
9d8bc097
MC		 3314 }
3315}
3316
3317static void bnxt_disable_int_sync(struct bnxt *bp)
3318{
3319 int i;
3320
3321 atomic_inc(&bp->intr_sem);
3322
3323 bnxt_disable_int(bp);
3324 for (i = 0; i < bp->cp_nr_rings; i++)
3325 synchronize_irq(bp->irq_tbl[i].vector);
3326}
3327
3328static void bnxt_enable_int(struct bnxt *bp)
3329{
3330 int i;
3331
3332 atomic_set(&bp->intr_sem, 0);
3333 for (i = 0; i < bp->cp_nr_rings; i++) {
3334 struct bnxt_napi *bnapi = bp->bnapi[i];
3335 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3336
3337 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
3338 }
3339}
3340
c0c050c5
MC		 3341void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3342 u16 cmpl_ring, u16 target_id)
3343{
a8643e16 3344 struct input *req = request;
c0c050c5 3345
a8643e16
MC		 3346 req->req_type = cpu_to_le16(req_type);
3347 req->cmpl_ring = cpu_to_le16(cmpl_ring);
3348 req->target_id = cpu_to_le16(target_id);
c0c050c5
MC		 3349 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3350}
3351
fbfbc485
MC		 3352static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3353 int timeout, bool silent)
c0c050c5 3354{
a11fa2be 3355 int i, intr_process, rc, tmo_count;
a8643e16 3356 struct input *req = msg;
c0c050c5
MC		 3357 u32 *data = msg;
3358 __le32 *resp_len, *valid;
3359 u16 cp_ring_id, len = 0;
3360 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
e605db80 3361 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
c0c050c5 3362
a8643e16 3363 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
c0c050c5 3364 memset(resp, 0, PAGE_SIZE);
a8643e16 3365 cp_ring_id = le16_to_cpu(req->cmpl_ring);
c0c050c5
MC		 3366 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3367
e605db80
DK		 3368 if (bp->flags & BNXT_FLAG_SHORT_CMD) {
3369 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3370 struct hwrm_short_input short_input = {0};
3371
3372 memcpy(short_cmd_req, req, msg_len);
3373 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN -
3374 msg_len);
3375
3376 short_input.req_type = req->req_type;
3377 short_input.signature =
3378 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3379 short_input.size = cpu_to_le16(msg_len);
3380 short_input.req_addr =
3381 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3382
3383 data = (u32 *)&short_input;
3384 msg_len = sizeof(short_input);
3385
3386 /* Sync memory write before updating doorbell */
3387 wmb();
3388
3389 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3390 }
3391
c0c050c5
MC		 3392 /* Write request msg to hwrm channel */
3393 __iowrite32_copy(bp->bar0, data, msg_len / 4);
3394
e605db80 3395 for (i = msg_len; i < max_req_len; i += 4)
d79979a1
MC		 3396 writel(0, bp->bar0 + i);
3397
c0c050c5
MC		 3398 /* currently supports only one outstanding message */
3399 if (intr_process)
a8643e16 3400 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
c0c050c5
MC		 3401
3402 /* Ring channel doorbell */
3403 writel(1, bp->bar0 + 0x100);
3404
ff4fe81d
MC		 3405 if (!timeout)
3406 timeout = DFLT_HWRM_CMD_TIMEOUT;
3407
c0c050c5 3408 i = 0;
a11fa2be 3409 tmo_count = timeout * 40;
c0c050c5
MC		 3410 if (intr_process) {
3411 /* Wait until hwrm response cmpl interrupt is processed */
3412 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
a11fa2be
MC		 3413 i++ < tmo_count) {
3414 usleep_range(25, 40);
c0c050c5
MC		 3415 }
3416
3417 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
3418 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
a8643e16 3419 le16_to_cpu(req->req_type));
c0c050c5
MC		 3420 return -1;
3421 }
3422 } else {
3423 /* Check if response len is updated */
3424 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
a11fa2be 3425 for (i = 0; i < tmo_count; i++) {
c0c050c5
MC		 3426 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3427 HWRM_RESP_LEN_SFT;
3428 if (len)
3429 break;
a11fa2be 3430 usleep_range(25, 40);
c0c050c5
MC		 3431 }
3432
a11fa2be 3433 if (i >= tmo_count) {
c0c050c5 3434 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
a8643e16 3435 timeout, le16_to_cpu(req->req_type),
8578d6c1 3436 le16_to_cpu(req->seq_id), len);
c0c050c5
MC		 3437 return -1;
3438 }
3439
3440 /* Last word of resp contains valid bit */
3441 valid = bp->hwrm_cmd_resp_addr + len - 4;
a11fa2be 3442 for (i = 0; i < 5; i++) {
c0c050c5
MC		 3443 if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
3444 break;
a11fa2be 3445 udelay(1);
c0c050c5
MC		 3446 }
3447
a11fa2be 3448 if (i >= 5) {
c0c050c5 3449 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
a8643e16
MC		 3450 timeout, le16_to_cpu(req->req_type),
3451 le16_to_cpu(req->seq_id), len, *valid);
c0c050c5
MC		 3452 return -1;
3453 }
3454 }
3455
3456 rc = le16_to_cpu(resp->error_code);
fbfbc485 3457 if (rc && !silent)
c0c050c5
MC		 3458 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3459 le16_to_cpu(resp->req_type),
3460 le16_to_cpu(resp->seq_id), rc);
fbfbc485
MC		 3461 return rc;
3462}
3463
3464int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3465{
3466 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
c0c050c5
MC		 3467}
3468
3469int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3470{
3471 int rc;
3472
3473 mutex_lock(&bp->hwrm_cmd_lock);
3474 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3475 mutex_unlock(&bp->hwrm_cmd_lock);
3476 return rc;
3477}
3478
90e20921
MC		 3479int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3480 int timeout)
3481{
3482 int rc;
3483
3484 mutex_lock(&bp->hwrm_cmd_lock);
3485 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3486 mutex_unlock(&bp->hwrm_cmd_lock);
3487 return rc;
3488}
3489
a1653b13
MC		 3490int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3491 int bmap_size)
c0c050c5
MC		 3492{
3493 struct hwrm_func_drv_rgtr_input req = {0};
25be8623
MC		 3494 DECLARE_BITMAP(async_events_bmap, 256);
3495 u32 *events = (u32 *)async_events_bmap;
a1653b13 3496 int i;
c0c050c5
MC		 3497
3498 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3499
3500 req.enables =
a1653b13 3501 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
c0c050c5 3502
25be8623
MC		 3503 memset(async_events_bmap, 0, sizeof(async_events_bmap));
3504 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
3505 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
3506
a1653b13
MC		 3507 if (bmap && bmap_size) {
3508 for (i = 0; i < bmap_size; i++) {
3509 if (test_bit(i, bmap))
3510 __set_bit(i, async_events_bmap);
3511 }
3512 }
3513
25be8623
MC		 3514 for (i = 0; i < 8; i++)
3515 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
3516
a1653b13
MC		 3517 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3518}
3519
3520static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
3521{
3522 struct hwrm_func_drv_rgtr_input req = {0};
3523
3524 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3525
3526 req.enables =
3527 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
3528 FUNC_DRV_RGTR_REQ_ENABLES_VER);
3529
11f15ed3 3530 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
c0c050c5
MC		 3531 req.ver_maj = DRV_VER_MAJ;
3532 req.ver_min = DRV_VER_MIN;
3533 req.ver_upd = DRV_VER_UPD;
3534
3535 if (BNXT_PF(bp)) {
9b0436c3 3536 u32 data[8];
a1653b13 3537 int i;
c0c050c5 3538
9b0436c3
MC		 3539 memset(data, 0, sizeof(data));
3540 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
3541 u16 cmd = bnxt_vf_req_snif[i];
3542 unsigned int bit, idx;
3543
3544 idx = cmd / 32;
3545 bit = cmd % 32;
3546 data[idx] |= 1 << bit;
3547 }
c0c050c5 3548
de68f5de
MC		 3549 for (i = 0; i < 8; i++)
3550 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
3551
c0c050c5
MC		 3552 req.enables |=
3553 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
3554 }
3555
3556 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3557}
3558
be58a0da
JH		 3559static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
3560{
3561 struct hwrm_func_drv_unrgtr_input req = {0};
3562
3563 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
3564 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3565}
3566
c0c050c5
MC		 3567static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
3568{
3569 u32 rc = 0;
3570 struct hwrm_tunnel_dst_port_free_input req = {0};
3571
3572 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
3573 req.tunnel_type = tunnel_type;
3574
3575 switch (tunnel_type) {
3576 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
3577 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
3578 break;
3579 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
3580 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
3581 break;
3582 default:
3583 break;
3584 }
3585
3586 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3587 if (rc)
3588 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
3589 rc);
3590 return rc;
3591}
3592
3593static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
3594 u8 tunnel_type)
3595{
3596 u32 rc = 0;
3597 struct hwrm_tunnel_dst_port_alloc_input req = {0};
3598 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3599
3600 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
3601
3602 req.tunnel_type = tunnel_type;
3603 req.tunnel_dst_port_val = port;
3604
3605 mutex_lock(&bp->hwrm_cmd_lock);
3606 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3607 if (rc) {
3608 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
3609 rc);
3610 goto err_out;
3611 }
3612
57aac71b
CJ		 3613 switch (tunnel_type) {
3614 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
c0c050c5 3615 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 3616 break;
3617 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
c0c050c5 3618 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 3619 break;
3620 default:
3621 break;
3622 }
3623
c0c050c5
MC		 3624err_out:
3625 mutex_unlock(&bp->hwrm_cmd_lock);
3626 return rc;
3627}
3628
3629static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3630{
3631 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3632 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3633
3634 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
c193554e 3635 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
c0c050c5
MC		 3636
3637 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3638 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3639 req.mask = cpu_to_le32(vnic->rx_mask);
3640 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3641}
3642
3643#ifdef CONFIG_RFS_ACCEL
3644static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3645 struct bnxt_ntuple_filter *fltr)
3646{
3647 struct hwrm_cfa_ntuple_filter_free_input req = {0};
3648
3649 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3650 req.ntuple_filter_id = fltr->filter_id;
3651 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3652}
3653
3654#define BNXT_NTP_FLTR_FLAGS \
3655 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
3656 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
3657 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
3658 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
3659 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
3660 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
3661 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
3662 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
3663 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
3664 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
3665 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
3666 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
3667 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
c193554e 3668 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
c0c050c5 3669
61aad724
MC		 3670#define BNXT_NTP_TUNNEL_FLTR_FLAG \
3671 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
3672
c0c050c5
MC		 3673static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3674 struct bnxt_ntuple_filter *fltr)
3675{
3676 int rc = 0;
3677 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3678 struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3679 bp->hwrm_cmd_resp_addr;
3680 struct flow_keys *keys = &fltr->fkeys;
3681 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3682
3683 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
a54c4d74 3684 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
c0c050c5
MC		 3685
3686 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3687
3688 req.ethertype = htons(ETH_P_IP);
3689 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
c193554e 3690 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
c0c050c5
MC		 3691 req.ip_protocol = keys->basic.ip_proto;
3692
dda0e746
MC		 3693 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
3694 int i;
3695
3696 req.ethertype = htons(ETH_P_IPV6);
3697 req.ip_addr_type =
3698 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
3699 *(struct in6_addr *)&req.src_ipaddr[0] =
3700 keys->addrs.v6addrs.src;
3701 *(struct in6_addr *)&req.dst_ipaddr[0] =
3702 keys->addrs.v6addrs.dst;
3703 for (i = 0; i < 4; i++) {
3704 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3705 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3706 }
3707 } else {
3708 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3709 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3710 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3711 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3712 }
61aad724
MC		 3713 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
3714 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
3715 req.tunnel_type =
3716 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
3717 }
c0c050c5
MC		 3718
3719 req.src_port = keys->ports.src;
3720 req.src_port_mask = cpu_to_be16(0xffff);
3721 req.dst_port = keys->ports.dst;
3722 req.dst_port_mask = cpu_to_be16(0xffff);
3723
c193554e 3724 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
c0c050c5
MC		 3725 mutex_lock(&bp->hwrm_cmd_lock);
3726 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3727 if (!rc)
3728 fltr->filter_id = resp->ntuple_filter_id;
3729 mutex_unlock(&bp->hwrm_cmd_lock);
3730 return rc;
3731}
3732#endif
3733
3734static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3735 u8 *mac_addr)
3736{
3737 u32 rc = 0;
3738 struct hwrm_cfa_l2_filter_alloc_input req = {0};
3739 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3740
3741 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
dc52c6c7
PS		 3742 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
3743 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
3744 req.flags |=
3745 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
c193554e 3746 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
c0c050c5
MC		 3747 req.enables =
3748 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
c193554e 3749 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
c0c050c5
MC		 3750 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3751 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3752 req.l2_addr_mask[0] = 0xff;
3753 req.l2_addr_mask[1] = 0xff;
3754 req.l2_addr_mask[2] = 0xff;
3755 req.l2_addr_mask[3] = 0xff;
3756 req.l2_addr_mask[4] = 0xff;
3757 req.l2_addr_mask[5] = 0xff;
3758
3759 mutex_lock(&bp->hwrm_cmd_lock);
3760 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3761 if (!rc)
3762 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3763 resp->l2_filter_id;
3764 mutex_unlock(&bp->hwrm_cmd_lock);
3765 return rc;
3766}
3767
3768static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3769{
3770 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3771 int rc = 0;
3772
3773 /* Any associated ntuple filters will also be cleared by firmware. */
3774 mutex_lock(&bp->hwrm_cmd_lock);
3775 for (i = 0; i < num_of_vnics; i++) {
3776 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3777
3778 for (j = 0; j < vnic->uc_filter_count; j++) {
3779 struct hwrm_cfa_l2_filter_free_input req = {0};
3780
3781 bnxt_hwrm_cmd_hdr_init(bp, &req,
3782 HWRM_CFA_L2_FILTER_FREE, -1, -1);
3783
3784 req.l2_filter_id = vnic->fw_l2_filter_id[j];
3785
3786 rc = _hwrm_send_message(bp, &req, sizeof(req),
3787 HWRM_CMD_TIMEOUT);
3788 }
3789 vnic->uc_filter_count = 0;
3790 }
3791 mutex_unlock(&bp->hwrm_cmd_lock);
3792
3793 return rc;
3794}
3795
3796static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3797{
3798 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3799 struct hwrm_vnic_tpa_cfg_input req = {0};
3800
3801 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3802
3803 if (tpa_flags) {
3804 u16 mss = bp->dev->mtu - 40;
3805 u32 nsegs, n, segs = 0, flags;
3806
3807 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3808 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3809 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3810 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3811 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3812 if (tpa_flags & BNXT_FLAG_GRO)
3813 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3814
3815 req.flags = cpu_to_le32(flags);
3816
3817 req.enables =
3818 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
c193554e
MC		 3819 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3820 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
c0c050c5
MC		 3821
3822 /* Number of segs are log2 units, and first packet is not
3823 * included as part of this units.
3824 */
2839f28b
MC		 3825 if (mss <= BNXT_RX_PAGE_SIZE) {
3826 n = BNXT_RX_PAGE_SIZE / mss;
c0c050c5
MC		 3827 nsegs = (MAX_SKB_FRAGS - 1) * n;
3828 } else {
2839f28b
MC		 3829 n = mss / BNXT_RX_PAGE_SIZE;
3830 if (mss & (BNXT_RX_PAGE_SIZE - 1))
c0c050c5
MC		 3831 n++;
3832 nsegs = (MAX_SKB_FRAGS - n) / n;
3833 }
3834
3835 segs = ilog2(nsegs);
3836 req.max_agg_segs = cpu_to_le16(segs);
3837 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
c193554e
MC		 3838
3839 req.min_agg_len = cpu_to_le32(512);
c0c050c5
MC		 3840 }
3841 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3842
3843 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3844}
3845
3846static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3847{
3848 u32 i, j, max_rings;
3849 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3850 struct hwrm_vnic_rss_cfg_input req = {0};
3851
94ce9caa 3852 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
c0c050c5
MC		 3853 return 0;
3854
3855 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3856 if (set_rss) {
87da7f79 3857 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
dc52c6c7
PS		 3858 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
3859 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3860 max_rings = bp->rx_nr_rings - 1;
3861 else
3862 max_rings = bp->rx_nr_rings;
3863 } else {
c0c050c5 3864 max_rings = 1;
dc52c6c7 3865 }
c0c050c5
MC		 3866
3867 /* Fill the RSS indirection table with ring group ids */
3868 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3869 if (j == max_rings)
3870 j = 0;
3871 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3872 }
3873
3874 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3875 req.hash_key_tbl_addr =
3876 cpu_to_le64(vnic->rss_hash_key_dma_addr);
3877 }
94ce9caa 3878 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
c0c050c5
MC		 3879 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3880}
3881
3882static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3883{
3884 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3885 struct hwrm_vnic_plcmodes_cfg_input req = {0};
3886
3887 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3888 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3889 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3890 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3891 req.enables =
3892 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3893 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3894 /* thresholds not implemented in firmware yet */
3895 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3896 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3897 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3898 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3899}
3900
94ce9caa
PS		 3901static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
3902 u16 ctx_idx)
c0c050c5
MC		 3903{
3904 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3905
3906 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3907 req.rss_cos_lb_ctx_id =
94ce9caa 3908 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
c0c050c5
MC		 3909
3910 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
94ce9caa 3911 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
c0c050c5
MC		 3912}
3913
3914static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3915{
94ce9caa 3916 int i, j;
c0c050c5
MC		 3917
3918 for (i = 0; i < bp->nr_vnics; i++) {
3919 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3920
94ce9caa
PS		 3921 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
3922 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
3923 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
3924 }
c0c050c5
MC		 3925 }
3926 bp->rsscos_nr_ctxs = 0;
3927}
3928
94ce9caa 3929static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
c0c050c5
MC		 3930{
3931 int rc;
3932 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3933 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3934 bp->hwrm_cmd_resp_addr;
3935
3936 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3937 -1);
3938
3939 mutex_lock(&bp->hwrm_cmd_lock);
3940 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3941 if (!rc)
94ce9caa 3942 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
c0c050c5
MC		 3943 le16_to_cpu(resp->rss_cos_lb_ctx_id);
3944 mutex_unlock(&bp->hwrm_cmd_lock);
3945
3946 return rc;
3947}
3948
a588e458 3949int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
c0c050c5 3950{
b81a90d3 3951 unsigned int ring = 0, grp_idx;
c0c050c5
MC		 3952 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3953 struct hwrm_vnic_cfg_input req = {0};
cf6645f8 3954 u16 def_vlan = 0;
c0c050c5
MC		 3955
3956 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
dc52c6c7
PS		 3957
3958 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
c0c050c5 3959 /* Only RSS support for now TBD: COS & LB */
dc52c6c7
PS		 3960 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
3961 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3962 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3963 VNIC_CFG_REQ_ENABLES_MRU);
ae10ae74
MC		 3964 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
3965 req.rss_rule =
3966 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
3967 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3968 VNIC_CFG_REQ_ENABLES_MRU);
3969 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
dc52c6c7
PS		 3970 } else {
3971 req.rss_rule = cpu_to_le16(0xffff);
3972 }
94ce9caa 3973
dc52c6c7
PS		 3974 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
3975 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
94ce9caa
PS		 3976 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
3977 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
3978 } else {
3979 req.cos_rule = cpu_to_le16(0xffff);
3980 }
3981
c0c050c5 3982 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
b81a90d3 3983 ring = 0;
c0c050c5 3984 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
b81a90d3 3985 ring = vnic_id - 1;
76595193
PS		 3986 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
3987 ring = bp->rx_nr_rings - 1;
c0c050c5 3988
b81a90d3 3989 grp_idx = bp->rx_ring[ring].bnapi->index;
c0c050c5
MC		 3990 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3991 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3992
3993 req.lb_rule = cpu_to_le16(0xffff);
3994 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3995 VLAN_HLEN);
3996
cf6645f8
MC		 3997#ifdef CONFIG_BNXT_SRIOV
3998 if (BNXT_VF(bp))
3999 def_vlan = bp->vf.vlan;
4000#endif
4001 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
c0c050c5 4002 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
a588e458
MC		 4003 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4004 req.flags |=
4005 cpu_to_le32(VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE);
c0c050c5
MC		 4006
4007 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4008}
4009
4010static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4011{
4012 u32 rc = 0;
4013
4014 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4015 struct hwrm_vnic_free_input req = {0};
4016
4017 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4018 req.vnic_id =
4019 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4020
4021 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4022 if (rc)
4023 return rc;
4024 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4025 }
4026 return rc;
4027}
4028
4029static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4030{
4031 u16 i;
4032
4033 for (i = 0; i < bp->nr_vnics; i++)
4034 bnxt_hwrm_vnic_free_one(bp, i);
4035}
4036
b81a90d3
MC		 4037static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4038 unsigned int start_rx_ring_idx,
4039 unsigned int nr_rings)
c0c050c5 4040{
b81a90d3
MC		 4041 int rc = 0;
4042 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
c0c050c5
MC		 4043 struct hwrm_vnic_alloc_input req = {0};
4044 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4045
4046 /* map ring groups to this vnic */
b81a90d3
MC		 4047 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4048 grp_idx = bp->rx_ring[i].bnapi->index;
4049 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
c0c050c5 4050 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
b81a90d3 4051 j, nr_rings);
c0c050c5
MC		 4052 break;
4053 }
4054 bp->vnic_info[vnic_id].fw_grp_ids[j] =
b81a90d3 4055 bp->grp_info[grp_idx].fw_grp_id;
c0c050c5
MC		 4056 }
4057
94ce9caa
PS		 4058 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
4059 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
c0c050c5
MC		 4060 if (vnic_id == 0)
4061 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4062
4063 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4064
4065 mutex_lock(&bp->hwrm_cmd_lock);
4066 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4067 if (!rc)
4068 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
4069 mutex_unlock(&bp->hwrm_cmd_lock);
4070 return rc;
4071}
4072
8fdefd63
MC		 4073static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4074{
4075 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4076 struct hwrm_vnic_qcaps_input req = {0};
4077 int rc;
4078
4079 if (bp->hwrm_spec_code < 0x10600)
4080 return 0;
4081
4082 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4083 mutex_lock(&bp->hwrm_cmd_lock);
4084 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4085 if (!rc) {
4086 if (resp->flags &
4087 cpu_to_le32(VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4088 bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4089 }
4090 mutex_unlock(&bp->hwrm_cmd_lock);
4091 return rc;
4092}
4093
c0c050c5
MC		 4094static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4095{
4096 u16 i;
4097 u32 rc = 0;
4098
4099 mutex_lock(&bp->hwrm_cmd_lock);
4100 for (i = 0; i < bp->rx_nr_rings; i++) {
4101 struct hwrm_ring_grp_alloc_input req = {0};
4102 struct hwrm_ring_grp_alloc_output *resp =
4103 bp->hwrm_cmd_resp_addr;
b81a90d3 4104 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
c0c050c5
MC		 4105
4106 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4107
b81a90d3
MC		 4108 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4109 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4110 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4111 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
c0c050c5
MC		 4112
4113 rc = _hwrm_send_message(bp, &req, sizeof(req),
4114 HWRM_CMD_TIMEOUT);
4115 if (rc)
4116 break;
4117
b81a90d3
MC		 4118 bp->grp_info[grp_idx].fw_grp_id =
4119 le32_to_cpu(resp->ring_group_id);
c0c050c5
MC		 4120 }
4121 mutex_unlock(&bp->hwrm_cmd_lock);
4122 return rc;
4123}
4124
4125static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4126{
4127 u16 i;
4128 u32 rc = 0;
4129 struct hwrm_ring_grp_free_input req = {0};
4130
4131 if (!bp->grp_info)
4132 return 0;
4133
4134 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4135
4136 mutex_lock(&bp->hwrm_cmd_lock);
4137 for (i = 0; i < bp->cp_nr_rings; i++) {
4138 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4139 continue;
4140 req.ring_group_id =
4141 cpu_to_le32(bp->grp_info[i].fw_grp_id);
4142
4143 rc = _hwrm_send_message(bp, &req, sizeof(req),
4144 HWRM_CMD_TIMEOUT);
4145 if (rc)
4146 break;
4147 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4148 }
4149 mutex_unlock(&bp->hwrm_cmd_lock);
4150 return rc;
4151}
4152
4153static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4154 struct bnxt_ring_struct *ring,
4155 u32 ring_type, u32 map_index,
4156 u32 stats_ctx_id)
4157{
4158 int rc = 0, err = 0;
4159 struct hwrm_ring_alloc_input req = {0};
4160 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4161 u16 ring_id;
4162
4163 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4164
4165 req.enables = 0;
4166 if (ring->nr_pages > 1) {
4167 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
4168 /* Page size is in log2 units */
4169 req.page_size = BNXT_PAGE_SHIFT;
4170 req.page_tbl_depth = 1;
4171 } else {
4172 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]);
4173 }
4174 req.fbo = 0;
4175 /* Association of ring index with doorbell index and MSIX number */
4176 req.logical_id = cpu_to_le16(map_index);
4177
4178 switch (ring_type) {
4179 case HWRM_RING_ALLOC_TX:
4180 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4181 /* Association of transmit ring with completion ring */
4182 req.cmpl_ring_id =
4183 cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
4184 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4185 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
4186 req.queue_id = cpu_to_le16(ring->queue_id);
4187 break;
4188 case HWRM_RING_ALLOC_RX:
4189 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4190 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4191 break;
4192 case HWRM_RING_ALLOC_AGG:
4193 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4194 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4195 break;
4196 case HWRM_RING_ALLOC_CMPL:
bac9a7e0 4197 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
c0c050c5
MC		 4198 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4199 if (bp->flags & BNXT_FLAG_USING_MSIX)
4200 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4201 break;
4202 default:
4203 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4204 ring_type);
4205 return -1;
4206 }
4207
4208 mutex_lock(&bp->hwrm_cmd_lock);
4209 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4210 err = le16_to_cpu(resp->error_code);
4211 ring_id = le16_to_cpu(resp->ring_id);
4212 mutex_unlock(&bp->hwrm_cmd_lock);
4213
4214 if (rc || err) {
4215 switch (ring_type) {
bac9a7e0 4216 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
c0c050c5
MC		 4217 netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
4218 rc, err);
4219 return -1;
4220
4221 case RING_FREE_REQ_RING_TYPE_RX:
4222 netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
4223 rc, err);
4224 return -1;
4225
4226 case RING_FREE_REQ_RING_TYPE_TX:
4227 netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
4228 rc, err);
4229 return -1;
4230
4231 default:
4232 netdev_err(bp->dev, "Invalid ring\n");
4233 return -1;
4234 }
4235 }
4236 ring->fw_ring_id = ring_id;
4237 return rc;
4238}
4239
486b5c22
MC		 4240static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4241{
4242 int rc;
4243
4244 if (BNXT_PF(bp)) {
4245 struct hwrm_func_cfg_input req = {0};
4246
4247 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4248 req.fid = cpu_to_le16(0xffff);
4249 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4250 req.async_event_cr = cpu_to_le16(idx);
4251 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4252 } else {
4253 struct hwrm_func_vf_cfg_input req = {0};
4254
4255 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4256 req.enables =
4257 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4258 req.async_event_cr = cpu_to_le16(idx);
4259 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4260 }
4261 return rc;
4262}
4263
c0c050c5
MC		 4264static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4265{
4266 int i, rc = 0;
4267
edd0c2cc
MC		 4268 for (i = 0; i < bp->cp_nr_rings; i++) {
4269 struct bnxt_napi *bnapi = bp->bnapi[i];
4270 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4271 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
c0c050c5 4272
33e52d88 4273 cpr->cp_doorbell = bp->bar1 + i * 0x80;
edd0c2cc
MC		 4274 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
4275 INVALID_STATS_CTX_ID);
4276 if (rc)
4277 goto err_out;
edd0c2cc
MC		 4278 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4279 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
486b5c22
MC		 4280
4281 if (!i) {
4282 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4283 if (rc)
4284 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4285 }
c0c050c5
MC		 4286 }
4287
edd0c2cc 4288 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 4289 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 4290 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
b81a90d3
MC		 4291 u32 map_idx = txr->bnapi->index;
4292 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
c0c050c5 4293
b81a90d3
MC		 4294 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
4295 map_idx, fw_stats_ctx);
edd0c2cc
MC		 4296 if (rc)
4297 goto err_out;
b81a90d3 4298 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
c0c050c5
MC		 4299 }
4300
edd0c2cc 4301 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 4302 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 4303 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3 4304 u32 map_idx = rxr->bnapi->index;
c0c050c5 4305
b81a90d3
MC		 4306 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
4307 map_idx, INVALID_STATS_CTX_ID);
edd0c2cc
MC		 4308 if (rc)
4309 goto err_out;
b81a90d3 4310 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
edd0c2cc 4311 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
b81a90d3 4312 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 4313 }
4314
4315 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4316 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 4317 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 4318 struct bnxt_ring_struct *ring =
4319 &rxr->rx_agg_ring_struct;
b81a90d3
MC		 4320 u32 grp_idx = rxr->bnapi->index;
4321 u32 map_idx = grp_idx + bp->rx_nr_rings;
c0c050c5
MC		 4322
4323 rc = hwrm_ring_alloc_send_msg(bp, ring,
4324 HWRM_RING_ALLOC_AGG,
b81a90d3 4325 map_idx,
c0c050c5
MC		 4326 INVALID_STATS_CTX_ID);
4327 if (rc)
4328 goto err_out;
4329
b81a90d3 4330 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
c0c050c5
MC		 4331 writel(DB_KEY_RX | rxr->rx_agg_prod,
4332 rxr->rx_agg_doorbell);
b81a90d3 4333 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 4334 }
4335 }
4336err_out:
4337 return rc;
4338}
4339
4340static int hwrm_ring_free_send_msg(struct bnxt *bp,
4341 struct bnxt_ring_struct *ring,
4342 u32 ring_type, int cmpl_ring_id)
4343{
4344 int rc;
4345 struct hwrm_ring_free_input req = {0};
4346 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
4347 u16 error_code;
4348
74608fc9 4349 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
c0c050c5
MC		 4350 req.ring_type = ring_type;
4351 req.ring_id = cpu_to_le16(ring->fw_ring_id);
4352
4353 mutex_lock(&bp->hwrm_cmd_lock);
4354 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4355 error_code = le16_to_cpu(resp->error_code);
4356 mutex_unlock(&bp->hwrm_cmd_lock);
4357
4358 if (rc || error_code) {
4359 switch (ring_type) {
bac9a7e0 4360 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
c0c050c5
MC		 4361 netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
4362 rc);
4363 return rc;
4364 case RING_FREE_REQ_RING_TYPE_RX:
4365 netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
4366 rc);
4367 return rc;
4368 case RING_FREE_REQ_RING_TYPE_TX:
4369 netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
4370 rc);
4371 return rc;
4372 default:
4373 netdev_err(bp->dev, "Invalid ring\n");
4374 return -1;
4375 }
4376 }
4377 return 0;
4378}
4379
edd0c2cc 4380static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
c0c050c5 4381{
edd0c2cc 4382 int i;
c0c050c5
MC		 4383
4384 if (!bp->bnapi)
edd0c2cc 4385 return;
c0c050c5 4386
edd0c2cc 4387 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 4388 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 4389 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
b81a90d3
MC		 4390 u32 grp_idx = txr->bnapi->index;
4391 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 4392
4393 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4394 hwrm_ring_free_send_msg(bp, ring,
4395 RING_FREE_REQ_RING_TYPE_TX,
4396 close_path ? cmpl_ring_id :
4397 INVALID_HW_RING_ID);
4398 ring->fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 4399 }
4400 }
4401
edd0c2cc 4402 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 4403 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 4404 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3
MC		 4405 u32 grp_idx = rxr->bnapi->index;
4406 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 4407
4408 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4409 hwrm_ring_free_send_msg(bp, ring,
4410 RING_FREE_REQ_RING_TYPE_RX,
4411 close_path ? cmpl_ring_id :
4412 INVALID_HW_RING_ID);
4413 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 4414 bp->grp_info[grp_idx].rx_fw_ring_id =
4415 INVALID_HW_RING_ID;
c0c050c5
MC		 4416 }
4417 }
4418
edd0c2cc 4419 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 4420 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 4421 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
b81a90d3
MC		 4422 u32 grp_idx = rxr->bnapi->index;
4423 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 4424
4425 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4426 hwrm_ring_free_send_msg(bp, ring,
4427 RING_FREE_REQ_RING_TYPE_RX,
4428 close_path ? cmpl_ring_id :
4429 INVALID_HW_RING_ID);
4430 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 4431 bp->grp_info[grp_idx].agg_fw_ring_id =
4432 INVALID_HW_RING_ID;
c0c050c5
MC		 4433 }
4434 }
4435
9d8bc097
MC		 4436 /* The completion rings are about to be freed. After that the
4437 * IRQ doorbell will not work anymore. So we need to disable
4438 * IRQ here.
4439 */
4440 bnxt_disable_int_sync(bp);
4441
edd0c2cc
MC		 4442 for (i = 0; i < bp->cp_nr_rings; i++) {
4443 struct bnxt_napi *bnapi = bp->bnapi[i];
4444 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4445 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4446
4447 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4448 hwrm_ring_free_send_msg(bp, ring,
bac9a7e0 4449 RING_FREE_REQ_RING_TYPE_L2_CMPL,
edd0c2cc
MC		 4450 INVALID_HW_RING_ID);
4451 ring->fw_ring_id = INVALID_HW_RING_ID;
4452 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 4453 }
4454 }
c0c050c5
MC		 4455}
4456
391be5c2
MC		 4457/* Caller must hold bp->hwrm_cmd_lock */
4458int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
4459{
4460 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4461 struct hwrm_func_qcfg_input req = {0};
4462 int rc;
4463
4464 if (bp->hwrm_spec_code < 0x10601)
4465 return 0;
4466
4467 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4468 req.fid = cpu_to_le16(fid);
4469 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4470 if (!rc)
4471 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
4472
4473 return rc;
4474}
4475
d1e7925e 4476static int bnxt_hwrm_reserve_tx_rings(struct bnxt *bp, int *tx_rings)
391be5c2
MC		 4477{
4478 struct hwrm_func_cfg_input req = {0};
4479 int rc;
4480
4481 if (bp->hwrm_spec_code < 0x10601)
4482 return 0;
4483
4484 if (BNXT_VF(bp))
4485 return 0;
4486
4487 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4488 req.fid = cpu_to_le16(0xffff);
4489 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4490 req.num_tx_rings = cpu_to_le16(*tx_rings);
4491 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4492 if (rc)
4493 return rc;
4494
4495 mutex_lock(&bp->hwrm_cmd_lock);
4496 rc = __bnxt_hwrm_get_tx_rings(bp, 0xffff, tx_rings);
4497 mutex_unlock(&bp->hwrm_cmd_lock);
98fdbe73
MC		 4498 if (!rc)
4499 bp->tx_reserved_rings = *tx_rings;
391be5c2
MC		 4500 return rc;
4501}
4502
98fdbe73
MC		 4503static int bnxt_hwrm_check_tx_rings(struct bnxt *bp, int tx_rings)
4504{
4505 struct hwrm_func_cfg_input req = {0};
4506 int rc;
4507
4508 if (bp->hwrm_spec_code < 0x10801)
4509 return 0;
4510
4511 if (BNXT_VF(bp))
4512 return 0;
4513
4514 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4515 req.fid = cpu_to_le16(0xffff);
4516 req.flags = cpu_to_le32(FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST);
4517 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4518 req.num_tx_rings = cpu_to_le16(tx_rings);
4519 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4520 if (rc)
4521 return -ENOMEM;
4522 return 0;
4523}
4524
bb053f52
MC		 4525static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
4526 u32 buf_tmrs, u16 flags,
4527 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
4528{
4529 req->flags = cpu_to_le16(flags);
4530 req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
4531 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
4532 req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
4533 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
4534 /* Minimum time between 2 interrupts set to buf_tmr x 2 */
4535 req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
4536 req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
4537 req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
4538}
4539
c0c050c5
MC		 4540int bnxt_hwrm_set_coal(struct bnxt *bp)
4541{
4542 int i, rc = 0;
dfc9c94a
MC		 4543 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
4544 req_tx = {0}, *req;
c0c050c5
MC		 4545 u16 max_buf, max_buf_irq;
4546 u16 buf_tmr, buf_tmr_irq;
4547 u32 flags;
4548
dfc9c94a
MC		 4549 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
4550 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4551 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
4552 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
c0c050c5 4553
dfb5b894
MC		 4554 /* Each rx completion (2 records) should be DMAed immediately.
4555 * DMA 1/4 of the completion buffers at a time.
4556 */
4557 max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
c0c050c5
MC		 4558 /* max_buf must not be zero */
4559 max_buf = clamp_t(u16, max_buf, 1, 63);
dfb5b894
MC		 4560 max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
4561 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
4562 /* buf timer set to 1/4 of interrupt timer */
4563 buf_tmr = max_t(u16, buf_tmr / 4, 1);
4564 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
4565 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
c0c050c5
MC		 4566
4567 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4568
4569 /* RING_IDLE generates more IRQs for lower latency. Enable it only
4570 * if coal_ticks is less than 25 us.
4571 */
dfb5b894 4572 if (bp->rx_coal_ticks < 25)
c0c050c5
MC		 4573 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
4574
bb053f52 4575 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
dfc9c94a
MC		 4576 buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
4577
4578 /* max_buf must not be zero */
4579 max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
4580 max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
4581 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
4582 /* buf timer set to 1/4 of interrupt timer */
4583 buf_tmr = max_t(u16, buf_tmr / 4, 1);
4584 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
4585 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
4586
4587 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4588 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
4589 buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
c0c050c5
MC		 4590
4591 mutex_lock(&bp->hwrm_cmd_lock);
4592 for (i = 0; i < bp->cp_nr_rings; i++) {
dfc9c94a 4593 struct bnxt_napi *bnapi = bp->bnapi[i];
c0c050c5 4594
dfc9c94a
MC		 4595 req = &req_rx;
4596 if (!bnapi->rx_ring)
4597 req = &req_tx;
4598 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
4599
4600 rc = _hwrm_send_message(bp, req, sizeof(*req),
c0c050c5
MC		 4601 HWRM_CMD_TIMEOUT);
4602 if (rc)
4603 break;
4604 }
4605 mutex_unlock(&bp->hwrm_cmd_lock);
4606 return rc;
4607}
4608
4609static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
4610{
4611 int rc = 0, i;
4612 struct hwrm_stat_ctx_free_input req = {0};
4613
4614 if (!bp->bnapi)
4615 return 0;
4616
3e8060fa
PS		 4617 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4618 return 0;
4619
c0c050c5
MC		 4620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
4621
4622 mutex_lock(&bp->hwrm_cmd_lock);
4623 for (i = 0; i < bp->cp_nr_rings; i++) {
4624 struct bnxt_napi *bnapi = bp->bnapi[i];
4625 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4626
4627 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
4628 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
4629
4630 rc = _hwrm_send_message(bp, &req, sizeof(req),
4631 HWRM_CMD_TIMEOUT);
4632 if (rc)
4633 break;
4634
4635 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4636 }
4637 }
4638 mutex_unlock(&bp->hwrm_cmd_lock);
4639 return rc;
4640}
4641
4642static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
4643{
4644 int rc = 0, i;
4645 struct hwrm_stat_ctx_alloc_input req = {0};
4646 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4647
3e8060fa
PS		 4648 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4649 return 0;
4650
c0c050c5
MC		 4651 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
4652
51f30785 4653 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
c0c050c5
MC		 4654
4655 mutex_lock(&bp->hwrm_cmd_lock);
4656 for (i = 0; i < bp->cp_nr_rings; i++) {
4657 struct bnxt_napi *bnapi = bp->bnapi[i];
4658 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4659
4660 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
4661
4662 rc = _hwrm_send_message(bp, &req, sizeof(req),
4663 HWRM_CMD_TIMEOUT);
4664 if (rc)
4665 break;
4666
4667 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
4668
4669 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
4670 }
4671 mutex_unlock(&bp->hwrm_cmd_lock);
89aa8445 4672 return rc;
c0c050c5
MC		 4673}
4674
cf6645f8
MC		 4675static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
4676{
4677 struct hwrm_func_qcfg_input req = {0};
567b2abe 4678 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9315edca 4679 u16 flags;
cf6645f8
MC		 4680 int rc;
4681
4682 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4683 req.fid = cpu_to_le16(0xffff);
4684 mutex_lock(&bp->hwrm_cmd_lock);
4685 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4686 if (rc)
4687 goto func_qcfg_exit;
4688
4689#ifdef CONFIG_BNXT_SRIOV
4690 if (BNXT_VF(bp)) {
cf6645f8
MC		 4691 struct bnxt_vf_info *vf = &bp->vf;
4692
4693 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
4694 }
4695#endif
9315edca
MC		 4696 flags = le16_to_cpu(resp->flags);
4697 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
4698 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
4699 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
4700 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
4701 bp->flags |= BNXT_FLAG_FW_DCBX_AGENT;
4702 }
4703 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
4704 bp->flags |= BNXT_FLAG_MULTI_HOST;
bc39f885 4705
567b2abe
SB		 4706 switch (resp->port_partition_type) {
4707 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
4708 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
4709 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
4710 bp->port_partition_type = resp->port_partition_type;
4711 break;
4712 }
32e8239c
MC		 4713 if (bp->hwrm_spec_code < 0x10707 ||
4714 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
4715 bp->br_mode = BRIDGE_MODE_VEB;
4716 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
4717 bp->br_mode = BRIDGE_MODE_VEPA;
4718 else
4719 bp->br_mode = BRIDGE_MODE_UNDEF;
cf6645f8
MC		 4720
4721func_qcfg_exit:
4722 mutex_unlock(&bp->hwrm_cmd_lock);
4723 return rc;
4724}
4725
7b08f661 4726static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
c0c050c5
MC		 4727{
4728 int rc = 0;
4729 struct hwrm_func_qcaps_input req = {0};
4730 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4731
4732 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
4733 req.fid = cpu_to_le16(0xffff);
4734
4735 mutex_lock(&bp->hwrm_cmd_lock);
4736 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4737 if (rc)
4738 goto hwrm_func_qcaps_exit;
4739
e4060d30
MC		 4740 if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED))
4741 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
4742 if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED))
4743 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
4744
7cc5a20e
MC		 4745 bp->tx_push_thresh = 0;
4746 if (resp->flags &
4747 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
4748 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
4749
c0c050c5
MC		 4750 if (BNXT_PF(bp)) {
4751 struct bnxt_pf_info *pf = &bp->pf;
4752
4753 pf->fw_fid = le16_to_cpu(resp->fid);
4754 pf->port_id = le16_to_cpu(resp->port_id);
87027db1 4755 bp->dev->dev_port = pf->port_id;
11f15ed3 4756 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
c0c050c5
MC		 4757 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4758 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4759 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
c0c050c5 4760 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
b72d4a68
MC		 4761 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4762 if (!pf->max_hw_ring_grps)
4763 pf->max_hw_ring_grps = pf->max_tx_rings;
c0c050c5
MC		 4764 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4765 pf->max_vnics = le16_to_cpu(resp->max_vnics);
4766 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
4767 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
4768 pf->max_vfs = le16_to_cpu(resp->max_vfs);
4769 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
4770 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
4771 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
4772 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
4773 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
4774 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
c1ef146a
MC		 4775 if (resp->flags &
4776 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED))
4777 bp->flags |= BNXT_FLAG_WOL_CAP;
c0c050c5 4778 } else {
379a80a1 4779#ifdef CONFIG_BNXT_SRIOV
c0c050c5
MC		 4780 struct bnxt_vf_info *vf = &bp->vf;
4781
4782 vf->fw_fid = le16_to_cpu(resp->fid);
c0c050c5
MC		 4783
4784 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4785 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4786 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
4787 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
b72d4a68
MC		 4788 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4789 if (!vf->max_hw_ring_grps)
4790 vf->max_hw_ring_grps = vf->max_tx_rings;
c0c050c5
MC		 4791 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4792 vf->max_vnics = le16_to_cpu(resp->max_vnics);
4793 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7cc5a20e
MC		 4794
4795 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
379a80a1 4796#endif
c0c050c5
MC		 4797 }
4798
c0c050c5
MC		 4799hwrm_func_qcaps_exit:
4800 mutex_unlock(&bp->hwrm_cmd_lock);
4801 return rc;
4802}
4803
4804static int bnxt_hwrm_func_reset(struct bnxt *bp)
4805{
4806 struct hwrm_func_reset_input req = {0};
4807
4808 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
4809 req.enables = 0;
4810
4811 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
4812}
4813
4814static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
4815{
4816 int rc = 0;
4817 struct hwrm_queue_qportcfg_input req = {0};
4818 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
4819 u8 i, *qptr;
4820
4821 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
4822
4823 mutex_lock(&bp->hwrm_cmd_lock);
4824 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4825 if (rc)
4826 goto qportcfg_exit;
4827
4828 if (!resp->max_configurable_queues) {
4829 rc = -EINVAL;
4830 goto qportcfg_exit;
4831 }
4832 bp->max_tc = resp->max_configurable_queues;
87c374de 4833 bp->max_lltc = resp->max_configurable_lossless_queues;
c0c050c5
MC		 4834 if (bp->max_tc > BNXT_MAX_QUEUE)
4835 bp->max_tc = BNXT_MAX_QUEUE;
4836
441cabbb
MC		 4837 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
4838 bp->max_tc = 1;
4839
87c374de
MC		 4840 if (bp->max_lltc > bp->max_tc)
4841 bp->max_lltc = bp->max_tc;
4842
c0c050c5
MC		 4843 qptr = &resp->queue_id0;
4844 for (i = 0; i < bp->max_tc; i++) {
4845 bp->q_info[i].queue_id = *qptr++;
4846 bp->q_info[i].queue_profile = *qptr++;
4847 }
4848
4849qportcfg_exit:
4850 mutex_unlock(&bp->hwrm_cmd_lock);
4851 return rc;
4852}
4853
4854static int bnxt_hwrm_ver_get(struct bnxt *bp)
4855{
4856 int rc;
4857 struct hwrm_ver_get_input req = {0};
4858 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
e605db80 4859 u32 dev_caps_cfg;
c0c050c5 4860
e6ef2699 4861 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
c0c050c5
MC		 4862 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
4863 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
4864 req.hwrm_intf_min = HWRM_VERSION_MINOR;
4865 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
4866 mutex_lock(&bp->hwrm_cmd_lock);
4867 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4868 if (rc)
4869 goto hwrm_ver_get_exit;
4870
4871 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
4872
11f15ed3
MC		 4873 bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
4874 resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
c193554e
MC		 4875 if (resp->hwrm_intf_maj < 1) {
4876 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
c0c050c5 4877 resp->hwrm_intf_maj, resp->hwrm_intf_min,
c193554e
MC		 4878 resp->hwrm_intf_upd);
4879 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
c0c050c5 4880 }
3ebf6f0a 4881 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
c0c050c5
MC		 4882 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4883 resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
4884
ff4fe81d
MC		 4885 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4886 if (!bp->hwrm_cmd_timeout)
4887 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4888
e6ef2699
MC		 4889 if (resp->hwrm_intf_maj >= 1)
4890 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4891
659c805c 4892 bp->chip_num = le16_to_cpu(resp->chip_num);
3e8060fa
PS		 4893 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
4894 !resp->chip_metal)
4895 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
659c805c 4896
e605db80
DK		 4897 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
4898 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
4899 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
4900 bp->flags |= BNXT_FLAG_SHORT_CMD;
4901
c0c050c5
MC		 4902hwrm_ver_get_exit:
4903 mutex_unlock(&bp->hwrm_cmd_lock);
4904 return rc;
4905}
4906
5ac67d8b
RS		 4907int bnxt_hwrm_fw_set_time(struct bnxt *bp)
4908{
878786d9 4909#if IS_ENABLED(CONFIG_RTC_LIB)
5ac67d8b
RS		 4910 struct hwrm_fw_set_time_input req = {0};
4911 struct rtc_time tm;
4912 struct timeval tv;
4913
4914 if (bp->hwrm_spec_code < 0x10400)
4915 return -EOPNOTSUPP;
4916
4917 do_gettimeofday(&tv);
4918 rtc_time_to_tm(tv.tv_sec, &tm);
4919 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
4920 req.year = cpu_to_le16(1900 + tm.tm_year);
4921 req.month = 1 + tm.tm_mon;
4922 req.day = tm.tm_mday;
4923 req.hour = tm.tm_hour;
4924 req.minute = tm.tm_min;
4925 req.second = tm.tm_sec;
4926 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
878786d9
RS		 4927#else
4928 return -EOPNOTSUPP;
4929#endif
5ac67d8b
RS		 4930}
4931
3bdf56c4
MC		 4932static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4933{
4934 int rc;
4935 struct bnxt_pf_info *pf = &bp->pf;
4936 struct hwrm_port_qstats_input req = {0};
4937
4938 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4939 return 0;
4940
4941 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4942 req.port_id = cpu_to_le16(pf->port_id);
4943 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4944 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4945 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4946 return rc;
4947}
4948
c0c050c5
MC		 4949static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4950{
4951 if (bp->vxlan_port_cnt) {
4952 bnxt_hwrm_tunnel_dst_port_free(
4953 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4954 }
4955 bp->vxlan_port_cnt = 0;
4956 if (bp->nge_port_cnt) {
4957 bnxt_hwrm_tunnel_dst_port_free(
4958 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4959 }
4960 bp->nge_port_cnt = 0;
4961}
4962
4963static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4964{
4965 int rc, i;
4966 u32 tpa_flags = 0;
4967
4968 if (set_tpa)
4969 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4970 for (i = 0; i < bp->nr_vnics; i++) {
4971 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4972 if (rc) {
4973 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
23e12c89 4974 i, rc);
c0c050c5
MC		 4975 return rc;
4976 }
4977 }
4978 return 0;
4979}
4980
4981static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4982{
4983 int i;
4984
4985 for (i = 0; i < bp->nr_vnics; i++)
4986 bnxt_hwrm_vnic_set_rss(bp, i, false);
4987}
4988
4989static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4990 bool irq_re_init)
4991{
4992 if (bp->vnic_info) {
4993 bnxt_hwrm_clear_vnic_filter(bp);
4994 /* clear all RSS setting before free vnic ctx */
4995 bnxt_hwrm_clear_vnic_rss(bp);
4996 bnxt_hwrm_vnic_ctx_free(bp);
4997 /* before free the vnic, undo the vnic tpa settings */
4998 if (bp->flags & BNXT_FLAG_TPA)
4999 bnxt_set_tpa(bp, false);
5000 bnxt_hwrm_vnic_free(bp);
5001 }
5002 bnxt_hwrm_ring_free(bp, close_path);
5003 bnxt_hwrm_ring_grp_free(bp);
5004 if (irq_re_init) {
5005 bnxt_hwrm_stat_ctx_free(bp);
5006 bnxt_hwrm_free_tunnel_ports(bp);
5007 }
5008}
5009
39d8ba2e
MC		 5010static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
5011{
5012 struct hwrm_func_cfg_input req = {0};
5013 int rc;
5014
5015 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5016 req.fid = cpu_to_le16(0xffff);
5017 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
5018 if (br_mode == BRIDGE_MODE_VEB)
5019 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
5020 else if (br_mode == BRIDGE_MODE_VEPA)
5021 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
5022 else
5023 return -EINVAL;
5024 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5025 if (rc)
5026 rc = -EIO;
5027 return rc;
5028}
5029
c0c050c5
MC		 5030static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
5031{
ae10ae74 5032 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
c0c050c5
MC		 5033 int rc;
5034
ae10ae74
MC		 5035 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
5036 goto skip_rss_ctx;
5037
c0c050c5 5038 /* allocate context for vnic */
94ce9caa 5039 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
c0c050c5
MC		 5040 if (rc) {
5041 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5042 vnic_id, rc);
5043 goto vnic_setup_err;
5044 }
5045 bp->rsscos_nr_ctxs++;
5046
94ce9caa
PS		 5047 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5048 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
5049 if (rc) {
5050 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
5051 vnic_id, rc);
5052 goto vnic_setup_err;
5053 }
5054 bp->rsscos_nr_ctxs++;
5055 }
5056
ae10ae74 5057skip_rss_ctx:
c0c050c5
MC		 5058 /* configure default vnic, ring grp */
5059 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
5060 if (rc) {
5061 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
5062 vnic_id, rc);
5063 goto vnic_setup_err;
5064 }
5065
5066 /* Enable RSS hashing on vnic */
5067 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
5068 if (rc) {
5069 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
5070 vnic_id, rc);
5071 goto vnic_setup_err;
5072 }
5073
5074 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5075 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
5076 if (rc) {
5077 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
5078 vnic_id, rc);
5079 }
5080 }
5081
5082vnic_setup_err:
5083 return rc;
5084}
5085
5086static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
5087{
5088#ifdef CONFIG_RFS_ACCEL
5089 int i, rc = 0;
5090
5091 for (i = 0; i < bp->rx_nr_rings; i++) {
ae10ae74 5092 struct bnxt_vnic_info *vnic;
c0c050c5
MC		 5093 u16 vnic_id = i + 1;
5094 u16 ring_id = i;
5095
5096 if (vnic_id >= bp->nr_vnics)
5097 break;
5098
ae10ae74
MC		 5099 vnic = &bp->vnic_info[vnic_id];
5100 vnic->flags |= BNXT_VNIC_RFS_FLAG;
5101 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
5102 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
b81a90d3 5103 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
c0c050c5
MC		 5104 if (rc) {
5105 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5106 vnic_id, rc);
5107 break;
5108 }
5109 rc = bnxt_setup_vnic(bp, vnic_id);
5110 if (rc)
5111 break;
5112 }
5113 return rc;
5114#else
5115 return 0;
5116#endif
5117}
5118
17c71ac3
MC		 5119/* Allow PF and VF with default VLAN to be in promiscuous mode */
5120static bool bnxt_promisc_ok(struct bnxt *bp)
5121{
5122#ifdef CONFIG_BNXT_SRIOV
5123 if (BNXT_VF(bp) && !bp->vf.vlan)
5124 return false;
5125#endif
5126 return true;
5127}
5128
dc52c6c7
PS		 5129static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
5130{
5131 unsigned int rc = 0;
5132
5133 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
5134 if (rc) {
5135 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5136 rc);
5137 return rc;
5138 }
5139
5140 rc = bnxt_hwrm_vnic_cfg(bp, 1);
5141 if (rc) {
5142 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5143 rc);
5144 return rc;
5145 }
5146 return rc;
5147}
5148
b664f008 5149static int bnxt_cfg_rx_mode(struct bnxt *);
7d2837dd 5150static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
b664f008 5151
c0c050c5
MC		 5152static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
5153{
7d2837dd 5154 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
c0c050c5 5155 int rc = 0;
76595193 5156 unsigned int rx_nr_rings = bp->rx_nr_rings;
c0c050c5
MC		 5157
5158 if (irq_re_init) {
5159 rc = bnxt_hwrm_stat_ctx_alloc(bp);
5160 if (rc) {
5161 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
5162 rc);
5163 goto err_out;
5164 }
98fdbe73
MC		 5165 if (bp->tx_reserved_rings != bp->tx_nr_rings) {
5166 int tx = bp->tx_nr_rings;
5167
5168 if (bnxt_hwrm_reserve_tx_rings(bp, &tx) ||
5169 tx < bp->tx_nr_rings) {
5170 rc = -ENOMEM;
5171 goto err_out;
5172 }
5173 }
c0c050c5
MC		 5174 }
5175
5176 rc = bnxt_hwrm_ring_alloc(bp);
5177 if (rc) {
5178 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
5179 goto err_out;
5180 }
5181
5182 rc = bnxt_hwrm_ring_grp_alloc(bp);
5183 if (rc) {
5184 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
5185 goto err_out;
5186 }
5187
76595193
PS		 5188 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5189 rx_nr_rings--;
5190
c0c050c5 5191 /* default vnic 0 */
76595193 5192 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
c0c050c5
MC		 5193 if (rc) {
5194 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
5195 goto err_out;
5196 }
5197
5198 rc = bnxt_setup_vnic(bp, 0);
5199 if (rc)
5200 goto err_out;
5201
5202 if (bp->flags & BNXT_FLAG_RFS) {
5203 rc = bnxt_alloc_rfs_vnics(bp);
5204 if (rc)
5205 goto err_out;
5206 }
5207
5208 if (bp->flags & BNXT_FLAG_TPA) {
5209 rc = bnxt_set_tpa(bp, true);
5210 if (rc)
5211 goto err_out;
5212 }
5213
5214 if (BNXT_VF(bp))
5215 bnxt_update_vf_mac(bp);
5216
5217 /* Filter for default vnic 0 */
5218 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
5219 if (rc) {
5220 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
5221 goto err_out;
5222 }
7d2837dd 5223 vnic->uc_filter_count = 1;
c0c050c5 5224
7d2837dd 5225 vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5 5226
17c71ac3 5227 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7d2837dd
MC		 5228 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5229
5230 if (bp->dev->flags & IFF_ALLMULTI) {
5231 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5232 vnic->mc_list_count = 0;
5233 } else {
5234 u32 mask = 0;
5235
5236 bnxt_mc_list_updated(bp, &mask);
5237 vnic->rx_mask |= mask;
5238 }
c0c050c5 5239
b664f008
MC		 5240 rc = bnxt_cfg_rx_mode(bp);
5241 if (rc)
c0c050c5 5242 goto err_out;
c0c050c5
MC		 5243
5244 rc = bnxt_hwrm_set_coal(bp);
5245 if (rc)
5246 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
dc52c6c7
PS		 5247 rc);
5248
5249 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5250 rc = bnxt_setup_nitroa0_vnic(bp);
5251 if (rc)
5252 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
5253 rc);
5254 }
c0c050c5 5255
cf6645f8
MC		 5256 if (BNXT_VF(bp)) {
5257 bnxt_hwrm_func_qcfg(bp);
5258 netdev_update_features(bp->dev);
5259 }
5260
c0c050c5
MC		 5261 return 0;
5262
5263err_out:
5264 bnxt_hwrm_resource_free(bp, 0, true);
5265
5266 return rc;
5267}
5268
5269static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
5270{
5271 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
5272 return 0;
5273}
5274
5275static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
5276{
2247925f 5277 bnxt_init_cp_rings(bp);
c0c050c5
MC		 5278 bnxt_init_rx_rings(bp);
5279 bnxt_init_tx_rings(bp);
5280 bnxt_init_ring_grps(bp, irq_re_init);
5281 bnxt_init_vnics(bp);
5282
5283 return bnxt_init_chip(bp, irq_re_init);
5284}
5285
c0c050c5
MC		 5286static int bnxt_set_real_num_queues(struct bnxt *bp)
5287{
5288 int rc;
5289 struct net_device *dev = bp->dev;
5290
5f449249
MC		 5291 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
5292 bp->tx_nr_rings_xdp);
c0c050c5
MC		 5293 if (rc)
5294 return rc;
5295
5296 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
5297 if (rc)
5298 return rc;
5299
5300#ifdef CONFIG_RFS_ACCEL
45019a18 5301 if (bp->flags & BNXT_FLAG_RFS)
c0c050c5 5302 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
c0c050c5
MC		 5303#endif
5304
5305 return rc;
5306}
5307
6e6c5a57
MC		 5308static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5309 bool shared)
5310{
5311 int _rx = *rx, _tx = *tx;
5312
5313 if (shared) {
5314 *rx = min_t(int, _rx, max);
5315 *tx = min_t(int, _tx, max);
5316 } else {
5317 if (max < 2)
5318 return -ENOMEM;
5319
5320 while (_rx + _tx > max) {
5321 if (_rx > _tx && _rx > 1)
5322 _rx--;
5323 else if (_tx > 1)
5324 _tx--;
5325 }
5326 *rx = _rx;
5327 *tx = _tx;
5328 }
5329 return 0;
5330}
5331
7809592d
MC		 5332static void bnxt_setup_msix(struct bnxt *bp)
5333{
5334 const int len = sizeof(bp->irq_tbl[0].name);
5335 struct net_device *dev = bp->dev;
5336 int tcs, i;
5337
5338 tcs = netdev_get_num_tc(dev);
5339 if (tcs > 1) {
d1e7925e 5340 int i, off, count;
7809592d 5341
d1e7925e
MC		 5342 for (i = 0; i < tcs; i++) {
5343 count = bp->tx_nr_rings_per_tc;
5344 off = i * count;
5345 netdev_set_tc_queue(dev, i, count, off);
7809592d
MC		 5346 }
5347 }
5348
5349 for (i = 0; i < bp->cp_nr_rings; i++) {
5350 char *attr;
5351
5352 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5353 attr = "TxRx";
5354 else if (i < bp->rx_nr_rings)
5355 attr = "rx";
5356 else
5357 attr = "tx";
5358
5359 snprintf(bp->irq_tbl[i].name, len, "%s-%s-%d", dev->name, attr,
5360 i);
5361 bp->irq_tbl[i].handler = bnxt_msix;
5362 }
5363}
5364
5365static void bnxt_setup_inta(struct bnxt *bp)
5366{
5367 const int len = sizeof(bp->irq_tbl[0].name);
5368
5369 if (netdev_get_num_tc(bp->dev))
5370 netdev_reset_tc(bp->dev);
5371
5372 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
5373 0);
5374 bp->irq_tbl[0].handler = bnxt_inta;
5375}
5376
5377static int bnxt_setup_int_mode(struct bnxt *bp)
5378{
5379 int rc;
5380
5381 if (bp->flags & BNXT_FLAG_USING_MSIX)
5382 bnxt_setup_msix(bp);
5383 else
5384 bnxt_setup_inta(bp);
5385
5386 rc = bnxt_set_real_num_queues(bp);
5387 return rc;
5388}
5389
b7429954 5390#ifdef CONFIG_RFS_ACCEL
8079e8f1
MC		 5391static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
5392{
5393#if defined(CONFIG_BNXT_SRIOV)
5394 if (BNXT_VF(bp))
5395 return bp->vf.max_rsscos_ctxs;
5396#endif
5397 return bp->pf.max_rsscos_ctxs;
5398}
5399
5400static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
5401{
5402#if defined(CONFIG_BNXT_SRIOV)
5403 if (BNXT_VF(bp))
5404 return bp->vf.max_vnics;
5405#endif
5406 return bp->pf.max_vnics;
5407}
b7429954 5408#endif
8079e8f1 5409
e4060d30
MC		 5410unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
5411{
5412#if defined(CONFIG_BNXT_SRIOV)
5413 if (BNXT_VF(bp))
5414 return bp->vf.max_stat_ctxs;
5415#endif
5416 return bp->pf.max_stat_ctxs;
5417}
5418
a588e458
MC		 5419void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
5420{
5421#if defined(CONFIG_BNXT_SRIOV)
5422 if (BNXT_VF(bp))
5423 bp->vf.max_stat_ctxs = max;
5424 else
5425#endif
5426 bp->pf.max_stat_ctxs = max;
5427}
5428
e4060d30
MC		 5429unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
5430{
5431#if defined(CONFIG_BNXT_SRIOV)
5432 if (BNXT_VF(bp))
5433 return bp->vf.max_cp_rings;
5434#endif
5435 return bp->pf.max_cp_rings;
5436}
5437
a588e458
MC		 5438void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max)
5439{
5440#if defined(CONFIG_BNXT_SRIOV)
5441 if (BNXT_VF(bp))
5442 bp->vf.max_cp_rings = max;
5443 else
5444#endif
5445 bp->pf.max_cp_rings = max;
5446}
5447
7809592d
MC		 5448static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
5449{
5450#if defined(CONFIG_BNXT_SRIOV)
5451 if (BNXT_VF(bp))
68a946bb
MC		 5452 return min_t(unsigned int, bp->vf.max_irqs,
5453 bp->vf.max_cp_rings);
7809592d 5454#endif
68a946bb 5455 return min_t(unsigned int, bp->pf.max_irqs, bp->pf.max_cp_rings);
7809592d
MC		 5456}
5457
33c2657e
MC		 5458void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
5459{
5460#if defined(CONFIG_BNXT_SRIOV)
5461 if (BNXT_VF(bp))
5462 bp->vf.max_irqs = max_irqs;
5463 else
5464#endif
5465 bp->pf.max_irqs = max_irqs;
5466}
5467
7809592d 5468static int bnxt_init_msix(struct bnxt *bp)
c0c050c5 5469{
01657bcd 5470 int i, total_vecs, rc = 0, min = 1;
7809592d 5471 struct msix_entry *msix_ent;
c0c050c5 5472
7809592d 5473 total_vecs = bnxt_get_max_func_irqs(bp);
c0c050c5
MC		 5474 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
5475 if (!msix_ent)
5476 return -ENOMEM;
5477
5478 for (i = 0; i < total_vecs; i++) {
5479 msix_ent[i].entry = i;
5480 msix_ent[i].vector = 0;
5481 }
5482
01657bcd
MC		 5483 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
5484 min = 2;
5485
5486 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
c0c050c5
MC		 5487 if (total_vecs < 0) {
5488 rc = -ENODEV;
5489 goto msix_setup_exit;
5490 }
5491
5492 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
5493 if (bp->irq_tbl) {
7809592d
MC		 5494 for (i = 0; i < total_vecs; i++)
5495 bp->irq_tbl[i].vector = msix_ent[i].vector;
c0c050c5 5496
7809592d 5497 bp->total_irqs = total_vecs;
c0c050c5 5498 /* Trim rings based upon num of vectors allocated */
6e6c5a57 5499 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
01657bcd 5500 total_vecs, min == 1);
6e6c5a57
MC		 5501 if (rc)
5502 goto msix_setup_exit;
5503
c0c050c5 5504 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
7809592d
MC		 5505 bp->cp_nr_rings = (min == 1) ?
5506 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
5507 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5 5508
c0c050c5
MC		 5509 } else {
5510 rc = -ENOMEM;
5511 goto msix_setup_exit;
5512 }
5513 bp->flags |= BNXT_FLAG_USING_MSIX;
5514 kfree(msix_ent);
5515 return 0;
5516
5517msix_setup_exit:
7809592d
MC		 5518 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
5519 kfree(bp->irq_tbl);
5520 bp->irq_tbl = NULL;
c0c050c5
MC		 5521 pci_disable_msix(bp->pdev);
5522 kfree(msix_ent);
5523 return rc;
5524}
5525
7809592d 5526static int bnxt_init_inta(struct bnxt *bp)
c0c050c5 5527{
c0c050c5 5528 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7809592d
MC		 5529 if (!bp->irq_tbl)
5530 return -ENOMEM;
5531
5532 bp->total_irqs = 1;
c0c050c5
MC		 5533 bp->rx_nr_rings = 1;
5534 bp->tx_nr_rings = 1;
5535 bp->cp_nr_rings = 1;
5536 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
01657bcd 5537 bp->flags |= BNXT_FLAG_SHARED_RINGS;
c0c050c5 5538 bp->irq_tbl[0].vector = bp->pdev->irq;
7809592d 5539 return 0;
c0c050c5
MC		 5540}
5541
7809592d 5542static int bnxt_init_int_mode(struct bnxt *bp)
c0c050c5
MC		 5543{
5544 int rc = 0;
5545
5546 if (bp->flags & BNXT_FLAG_MSIX_CAP)
7809592d 5547 rc = bnxt_init_msix(bp);
c0c050c5 5548
1fa72e29 5549 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
c0c050c5 5550 /* fallback to INTA */
7809592d 5551 rc = bnxt_init_inta(bp);
c0c050c5
MC		 5552 }
5553 return rc;
5554}
5555
7809592d
MC		 5556static void bnxt_clear_int_mode(struct bnxt *bp)
5557{
5558 if (bp->flags & BNXT_FLAG_USING_MSIX)
5559 pci_disable_msix(bp->pdev);
5560
5561 kfree(bp->irq_tbl);
5562 bp->irq_tbl = NULL;
5563 bp->flags &= ~BNXT_FLAG_USING_MSIX;
5564}
5565
c0c050c5
MC		 5566static void bnxt_free_irq(struct bnxt *bp)
5567{
5568 struct bnxt_irq *irq;
5569 int i;
5570
5571#ifdef CONFIG_RFS_ACCEL
5572 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
5573 bp->dev->rx_cpu_rmap = NULL;
5574#endif
5575 if (!bp->irq_tbl)
5576 return;
5577
5578 for (i = 0; i < bp->cp_nr_rings; i++) {
5579 irq = &bp->irq_tbl[i];
56f0fd80
VV		 5580 if (irq->requested) {
5581 if (irq->have_cpumask) {
5582 irq_set_affinity_hint(irq->vector, NULL);
5583 free_cpumask_var(irq->cpu_mask);
5584 irq->have_cpumask = 0;
5585 }
c0c050c5 5586 free_irq(irq->vector, bp->bnapi[i]);
56f0fd80
VV		 5587 }
5588
c0c050c5
MC		 5589 irq->requested = 0;
5590 }
c0c050c5
MC		 5591}
5592
5593static int bnxt_request_irq(struct bnxt *bp)
5594{
b81a90d3 5595 int i, j, rc = 0;
c0c050c5
MC		 5596 unsigned long flags = 0;
5597#ifdef CONFIG_RFS_ACCEL
5598 struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
5599#endif
5600
5601 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
5602 flags = IRQF_SHARED;
5603
b81a90d3 5604 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
c0c050c5
MC		 5605 struct bnxt_irq *irq = &bp->irq_tbl[i];
5606#ifdef CONFIG_RFS_ACCEL
b81a90d3 5607 if (rmap && bp->bnapi[i]->rx_ring) {
c0c050c5
MC		 5608 rc = irq_cpu_rmap_add(rmap, irq->vector);
5609 if (rc)
5610 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
b81a90d3
MC		 5611 j);
5612 j++;
c0c050c5
MC		 5613 }
5614#endif
5615 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
5616 bp->bnapi[i]);
5617 if (rc)
5618 break;
5619
5620 irq->requested = 1;
56f0fd80
VV		 5621
5622 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
5623 int numa_node = dev_to_node(&bp->pdev->dev);
5624
5625 irq->have_cpumask = 1;
5626 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
5627 irq->cpu_mask);
5628 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
5629 if (rc) {
5630 netdev_warn(bp->dev,
5631 "Set affinity failed, IRQ = %d\n",
5632 irq->vector);
5633 break;
5634 }
5635 }
c0c050c5
MC		 5636 }
5637 return rc;
5638}
5639
5640static void bnxt_del_napi(struct bnxt *bp)
5641{
5642 int i;
5643
5644 if (!bp->bnapi)
5645 return;
5646
5647 for (i = 0; i < bp->cp_nr_rings; i++) {
5648 struct bnxt_napi *bnapi = bp->bnapi[i];
5649
5650 napi_hash_del(&bnapi->napi);
5651 netif_napi_del(&bnapi->napi);
5652 }
e5f6f564
ED		 5653 /* We called napi_hash_del() before netif_napi_del(), we need
5654 * to respect an RCU grace period before freeing napi structures.
5655 */
5656 synchronize_net();
c0c050c5
MC		 5657}
5658
5659static void bnxt_init_napi(struct bnxt *bp)
5660{
5661 int i;
10bbdaf5 5662 unsigned int cp_nr_rings = bp->cp_nr_rings;
c0c050c5
MC		 5663 struct bnxt_napi *bnapi;
5664
5665 if (bp->flags & BNXT_FLAG_USING_MSIX) {
10bbdaf5
PS		 5666 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5667 cp_nr_rings--;
5668 for (i = 0; i < cp_nr_rings; i++) {
c0c050c5
MC		 5669 bnapi = bp->bnapi[i];
5670 netif_napi_add(bp->dev, &bnapi->napi,
5671 bnxt_poll, 64);
c0c050c5 5672 }
10bbdaf5
PS		 5673 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5674 bnapi = bp->bnapi[cp_nr_rings];
5675 netif_napi_add(bp->dev, &bnapi->napi,
5676 bnxt_poll_nitroa0, 64);
10bbdaf5 5677 }
c0c050c5
MC		 5678 } else {
5679 bnapi = bp->bnapi[0];
5680 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
c0c050c5
MC		 5681 }
5682}
5683
5684static void bnxt_disable_napi(struct bnxt *bp)
5685{
5686 int i;
5687
5688 if (!bp->bnapi)
5689 return;
5690
b356a2e7 5691 for (i = 0; i < bp->cp_nr_rings; i++)
c0c050c5 5692 napi_disable(&bp->bnapi[i]->napi);
c0c050c5
MC		 5693}
5694
5695static void bnxt_enable_napi(struct bnxt *bp)
5696{
5697 int i;
5698
5699 for (i = 0; i < bp->cp_nr_rings; i++) {
fa7e2812 5700 bp->bnapi[i]->in_reset = false;
c0c050c5
MC		 5701 napi_enable(&bp->bnapi[i]->napi);
5702 }
5703}
5704
7df4ae9f 5705void bnxt_tx_disable(struct bnxt *bp)
c0c050c5
MC		 5706{
5707 int i;
c0c050c5 5708 struct bnxt_tx_ring_info *txr;
c0c050c5 5709
b6ab4b01 5710 if (bp->tx_ring) {
c0c050c5 5711 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5712 txr = &bp->tx_ring[i];
c0c050c5 5713 txr->dev_state = BNXT_DEV_STATE_CLOSING;
c0c050c5
MC		 5714 }
5715 }
5716 /* Stop all TX queues */
5717 netif_tx_disable(bp->dev);
5718 netif_carrier_off(bp->dev);
5719}
5720
7df4ae9f 5721void bnxt_tx_enable(struct bnxt *bp)
c0c050c5
MC		 5722{
5723 int i;
c0c050c5 5724 struct bnxt_tx_ring_info *txr;
c0c050c5
MC		 5725
5726 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5727 txr = &bp->tx_ring[i];
c0c050c5
MC		 5728 txr->dev_state = 0;
5729 }
5730 netif_tx_wake_all_queues(bp->dev);
5731 if (bp->link_info.link_up)
5732 netif_carrier_on(bp->dev);
5733}
5734
5735static void bnxt_report_link(struct bnxt *bp)
5736{
5737 if (bp->link_info.link_up) {
5738 const char *duplex;
5739 const char *flow_ctrl;
38a21b34
DK		 5740 u32 speed;
5741 u16 fec;
c0c050c5
MC		 5742
5743 netif_carrier_on(bp->dev);
5744 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
5745 duplex = "full";
5746 else
5747 duplex = "half";
5748 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
5749 flow_ctrl = "ON - receive & transmit";
5750 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
5751 flow_ctrl = "ON - transmit";
5752 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
5753 flow_ctrl = "ON - receive";
5754 else
5755 flow_ctrl = "none";
5756 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
38a21b34 5757 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
c0c050c5 5758 speed, duplex, flow_ctrl);
170ce013
MC		 5759 if (bp->flags & BNXT_FLAG_EEE_CAP)
5760 netdev_info(bp->dev, "EEE is %s\n",
5761 bp->eee.eee_active ? "active" :
5762 "not active");
e70c752f
MC		 5763 fec = bp->link_info.fec_cfg;
5764 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
5765 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
5766 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
5767 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
5768 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
c0c050c5
MC		 5769 } else {
5770 netif_carrier_off(bp->dev);
5771 netdev_err(bp->dev, "NIC Link is Down\n");
5772 }
5773}
5774
170ce013
MC		 5775static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
5776{
5777 int rc = 0;
5778 struct hwrm_port_phy_qcaps_input req = {0};
5779 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
93ed8117 5780 struct bnxt_link_info *link_info = &bp->link_info;
170ce013
MC		 5781
5782 if (bp->hwrm_spec_code < 0x10201)
5783 return 0;
5784
5785 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
5786
5787 mutex_lock(&bp->hwrm_cmd_lock);
5788 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5789 if (rc)
5790 goto hwrm_phy_qcaps_exit;
5791
acb20054 5792 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
170ce013
MC		 5793 struct ethtool_eee *eee = &bp->eee;
5794 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
5795
5796 bp->flags |= BNXT_FLAG_EEE_CAP;
5797 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5798 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
5799 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
5800 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
5801 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
5802 }
520ad89a
MC		 5803 if (resp->supported_speeds_auto_mode)
5804 link_info->support_auto_speeds =
5805 le16_to_cpu(resp->supported_speeds_auto_mode);
170ce013 5806
d5430d31
MC		 5807 bp->port_count = resp->port_cnt;
5808
170ce013
MC		 5809hwrm_phy_qcaps_exit:
5810 mutex_unlock(&bp->hwrm_cmd_lock);
5811 return rc;
5812}
5813
c0c050c5
MC		 5814static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
5815{
5816 int rc = 0;
5817 struct bnxt_link_info *link_info = &bp->link_info;
5818 struct hwrm_port_phy_qcfg_input req = {0};
5819 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5820 u8 link_up = link_info->link_up;
286ef9d6 5821 u16 diff;
c0c050c5
MC		 5822
5823 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
5824
5825 mutex_lock(&bp->hwrm_cmd_lock);
5826 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5827 if (rc) {
5828 mutex_unlock(&bp->hwrm_cmd_lock);
5829 return rc;
5830 }
5831
5832 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
5833 link_info->phy_link_status = resp->link;
acb20054
MC		 5834 link_info->duplex = resp->duplex_cfg;
5835 if (bp->hwrm_spec_code >= 0x10800)
5836 link_info->duplex = resp->duplex_state;
c0c050c5
MC		 5837 link_info->pause = resp->pause;
5838 link_info->auto_mode = resp->auto_mode;
5839 link_info->auto_pause_setting = resp->auto_pause;
3277360e 5840 link_info->lp_pause = resp->link_partner_adv_pause;
c0c050c5 5841 link_info->force_pause_setting = resp->force_pause;
acb20054 5842 link_info->duplex_setting = resp->duplex_cfg;
c0c050c5
MC		 5843 if (link_info->phy_link_status == BNXT_LINK_LINK)
5844 link_info->link_speed = le16_to_cpu(resp->link_speed);
5845 else
5846 link_info->link_speed = 0;
5847 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
c0c050c5
MC		 5848 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
5849 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
3277360e
MC		 5850 link_info->lp_auto_link_speeds =
5851 le16_to_cpu(resp->link_partner_adv_speeds);
c0c050c5
MC		 5852 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
5853 link_info->phy_ver[0] = resp->phy_maj;
5854 link_info->phy_ver[1] = resp->phy_min;
5855 link_info->phy_ver[2] = resp->phy_bld;
5856 link_info->media_type = resp->media_type;
03efbec0 5857 link_info->phy_type = resp->phy_type;
11f15ed3 5858 link_info->transceiver = resp->xcvr_pkg_type;
170ce013
MC		 5859 link_info->phy_addr = resp->eee_config_phy_addr &
5860 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
42ee18fe 5861 link_info->module_status = resp->module_status;
170ce013
MC		 5862
5863 if (bp->flags & BNXT_FLAG_EEE_CAP) {
5864 struct ethtool_eee *eee = &bp->eee;
5865 u16 fw_speeds;
5866
5867 eee->eee_active = 0;
5868 if (resp->eee_config_phy_addr &
5869 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
5870 eee->eee_active = 1;
5871 fw_speeds = le16_to_cpu(
5872 resp->link_partner_adv_eee_link_speed_mask);
5873 eee->lp_advertised =
5874 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5875 }
5876
5877 /* Pull initial EEE config */
5878 if (!chng_link_state) {
5879 if (resp->eee_config_phy_addr &
5880 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
5881 eee->eee_enabled = 1;
c0c050c5 5882
170ce013
MC		 5883 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
5884 eee->advertised =
5885 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5886
5887 if (resp->eee_config_phy_addr &
5888 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
5889 __le32 tmr;
5890
5891 eee->tx_lpi_enabled = 1;
5892 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
5893 eee->tx_lpi_timer = le32_to_cpu(tmr) &
5894 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
5895 }
5896 }
5897 }
e70c752f
MC		 5898
5899 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
5900 if (bp->hwrm_spec_code >= 0x10504)
5901 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
5902
c0c050c5
MC		 5903 /* TODO: need to add more logic to report VF link */
5904 if (chng_link_state) {
5905 if (link_info->phy_link_status == BNXT_LINK_LINK)
5906 link_info->link_up = 1;
5907 else
5908 link_info->link_up = 0;
5909 if (link_up != link_info->link_up)
5910 bnxt_report_link(bp);
5911 } else {
5912 /* alwasy link down if not require to update link state */
5913 link_info->link_up = 0;
5914 }
5915 mutex_unlock(&bp->hwrm_cmd_lock);
286ef9d6
MC		 5916
5917 diff = link_info->support_auto_speeds ^ link_info->advertising;
5918 if ((link_info->support_auto_speeds | diff) !=
5919 link_info->support_auto_speeds) {
5920 /* An advertised speed is no longer supported, so we need to
0eaa24b9
MC		 5921 * update the advertisement settings. Caller holds RTNL
5922 * so we can modify link settings.
286ef9d6 5923 */
286ef9d6 5924 link_info->advertising = link_info->support_auto_speeds;
0eaa24b9 5925 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
286ef9d6 5926 bnxt_hwrm_set_link_setting(bp, true, false);
286ef9d6 5927 }
c0c050c5
MC		 5928 return 0;
5929}
5930
10289bec
MC		 5931static void bnxt_get_port_module_status(struct bnxt *bp)
5932{
5933 struct bnxt_link_info *link_info = &bp->link_info;
5934 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
5935 u8 module_status;
5936
5937 if (bnxt_update_link(bp, true))
5938 return;
5939
5940 module_status = link_info->module_status;
5941 switch (module_status) {
5942 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
5943 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
5944 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
5945 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
5946 bp->pf.port_id);
5947 if (bp->hwrm_spec_code >= 0x10201) {
5948 netdev_warn(bp->dev, "Module part number %s\n",
5949 resp->phy_vendor_partnumber);
5950 }
5951 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
5952 netdev_warn(bp->dev, "TX is disabled\n");
5953 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
5954 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
5955 }
5956}
5957
c0c050c5
MC		 5958static void
5959bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
5960{
5961 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
c9ee9516
MC		 5962 if (bp->hwrm_spec_code >= 0x10201)
5963 req->auto_pause =
5964 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
c0c050c5
MC		 5965 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5966 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
5967 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
49b5c7a1 5968 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
c0c050c5
MC		 5969 req->enables |=
5970 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5971 } else {
5972 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5973 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
5974 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
5975 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
5976 req->enables |=
5977 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
c9ee9516
MC		 5978 if (bp->hwrm_spec_code >= 0x10201) {
5979 req->auto_pause = req->force_pause;
5980 req->enables |= cpu_to_le32(
5981 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5982 }
c0c050c5
MC		 5983 }
5984}
5985
5986static void bnxt_hwrm_set_link_common(struct bnxt *bp,
5987 struct hwrm_port_phy_cfg_input *req)
5988{
5989 u8 autoneg = bp->link_info.autoneg;
5990 u16 fw_link_speed = bp->link_info.req_link_speed;
68515a18 5991 u16 advertising = bp->link_info.advertising;
c0c050c5
MC		 5992
5993 if (autoneg & BNXT_AUTONEG_SPEED) {
5994 req->auto_mode |=
11f15ed3 5995 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
c0c050c5
MC		 5996
5997 req->enables |= cpu_to_le32(
5998 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
5999 req->auto_link_speed_mask = cpu_to_le16(advertising);
6000
6001 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
6002 req->flags |=
6003 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
6004 } else {
6005 req->force_link_speed = cpu_to_le16(fw_link_speed);
6006 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
6007 }
6008
c0c050c5
MC		 6009 /* tell chimp that the setting takes effect immediately */
6010 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
6011}
6012
6013int bnxt_hwrm_set_pause(struct bnxt *bp)
6014{
6015 struct hwrm_port_phy_cfg_input req = {0};
6016 int rc;
6017
6018 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6019 bnxt_hwrm_set_pause_common(bp, &req);
6020
6021 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
6022 bp->link_info.force_link_chng)
6023 bnxt_hwrm_set_link_common(bp, &req);
6024
6025 mutex_lock(&bp->hwrm_cmd_lock);
6026 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6027 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
6028 /* since changing of pause setting doesn't trigger any link
6029 * change event, the driver needs to update the current pause
6030 * result upon successfully return of the phy_cfg command
6031 */
6032 bp->link_info.pause =
6033 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
6034 bp->link_info.auto_pause_setting = 0;
6035 if (!bp->link_info.force_link_chng)
6036 bnxt_report_link(bp);
6037 }
6038 bp->link_info.force_link_chng = false;
6039 mutex_unlock(&bp->hwrm_cmd_lock);
6040 return rc;
6041}
6042
939f7f0c
MC		 6043static void bnxt_hwrm_set_eee(struct bnxt *bp,
6044 struct hwrm_port_phy_cfg_input *req)
6045{
6046 struct ethtool_eee *eee = &bp->eee;
6047
6048 if (eee->eee_enabled) {
6049 u16 eee_speeds;
6050 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
6051
6052 if (eee->tx_lpi_enabled)
6053 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
6054 else
6055 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
6056
6057 req->flags |= cpu_to_le32(flags);
6058 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
6059 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
6060 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
6061 } else {
6062 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
6063 }
6064}
6065
6066int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
c0c050c5
MC		 6067{
6068 struct hwrm_port_phy_cfg_input req = {0};
6069
6070 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6071 if (set_pause)
6072 bnxt_hwrm_set_pause_common(bp, &req);
6073
6074 bnxt_hwrm_set_link_common(bp, &req);
939f7f0c
MC		 6075
6076 if (set_eee)
6077 bnxt_hwrm_set_eee(bp, &req);
c0c050c5
MC		 6078 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6079}
6080
33f7d55f
MC		 6081static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
6082{
6083 struct hwrm_port_phy_cfg_input req = {0};
6084
567b2abe 6085 if (!BNXT_SINGLE_PF(bp))
33f7d55f
MC		 6086 return 0;
6087
6088 if (pci_num_vf(bp->pdev))
6089 return 0;
6090
6091 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
16d663a6 6092 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
33f7d55f
MC		 6093 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6094}
6095
5ad2cbee
MC		 6096static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
6097{
6098 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6099 struct hwrm_port_led_qcaps_input req = {0};
6100 struct bnxt_pf_info *pf = &bp->pf;
6101 int rc;
6102
6103 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
6104 return 0;
6105
6106 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
6107 req.port_id = cpu_to_le16(pf->port_id);
6108 mutex_lock(&bp->hwrm_cmd_lock);
6109 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6110 if (rc) {
6111 mutex_unlock(&bp->hwrm_cmd_lock);
6112 return rc;
6113 }
6114 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
6115 int i;
6116
6117 bp->num_leds = resp->num_leds;
6118 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
6119 bp->num_leds);
6120 for (i = 0; i < bp->num_leds; i++) {
6121 struct bnxt_led_info *led = &bp->leds[i];
6122 __le16 caps = led->led_state_caps;
6123
6124 if (!led->led_group_id ||
6125 !BNXT_LED_ALT_BLINK_CAP(caps)) {
6126 bp->num_leds = 0;
6127 break;
6128 }
6129 }
6130 }
6131 mutex_unlock(&bp->hwrm_cmd_lock);
6132 return 0;
6133}
6134
5282db6c
MC		 6135int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
6136{
6137 struct hwrm_wol_filter_alloc_input req = {0};
6138 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6139 int rc;
6140
6141 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
6142 req.port_id = cpu_to_le16(bp->pf.port_id);
6143 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
6144 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
6145 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
6146 mutex_lock(&bp->hwrm_cmd_lock);
6147 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6148 if (!rc)
6149 bp->wol_filter_id = resp->wol_filter_id;
6150 mutex_unlock(&bp->hwrm_cmd_lock);
6151 return rc;
6152}
6153
6154int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
6155{
6156 struct hwrm_wol_filter_free_input req = {0};
6157 int rc;
6158
6159 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
6160 req.port_id = cpu_to_le16(bp->pf.port_id);
6161 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
6162 req.wol_filter_id = bp->wol_filter_id;
6163 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6164 return rc;
6165}
6166
c1ef146a
MC		 6167static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
6168{
6169 struct hwrm_wol_filter_qcfg_input req = {0};
6170 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6171 u16 next_handle = 0;
6172 int rc;
6173
6174 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
6175 req.port_id = cpu_to_le16(bp->pf.port_id);
6176 req.handle = cpu_to_le16(handle);
6177 mutex_lock(&bp->hwrm_cmd_lock);
6178 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6179 if (!rc) {
6180 next_handle = le16_to_cpu(resp->next_handle);
6181 if (next_handle != 0) {
6182 if (resp->wol_type ==
6183 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
6184 bp->wol = 1;
6185 bp->wol_filter_id = resp->wol_filter_id;
6186 }
6187 }
6188 }
6189 mutex_unlock(&bp->hwrm_cmd_lock);
6190 return next_handle;
6191}
6192
6193static void bnxt_get_wol_settings(struct bnxt *bp)
6194{
6195 u16 handle = 0;
6196
6197 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
6198 return;
6199
6200 do {
6201 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
6202 } while (handle && handle != 0xffff);
6203}
6204
939f7f0c
MC		 6205static bool bnxt_eee_config_ok(struct bnxt *bp)
6206{
6207 struct ethtool_eee *eee = &bp->eee;
6208 struct bnxt_link_info *link_info = &bp->link_info;
6209
6210 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
6211 return true;
6212
6213 if (eee->eee_enabled) {
6214 u32 advertising =
6215 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
6216
6217 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6218 eee->eee_enabled = 0;
6219 return false;
6220 }
6221 if (eee->advertised & ~advertising) {
6222 eee->advertised = advertising & eee->supported;
6223 return false;
6224 }
6225 }
6226 return true;
6227}
6228
c0c050c5
MC		 6229static int bnxt_update_phy_setting(struct bnxt *bp)
6230{
6231 int rc;
6232 bool update_link = false;
6233 bool update_pause = false;
939f7f0c 6234 bool update_eee = false;
c0c050c5
MC		 6235 struct bnxt_link_info *link_info = &bp->link_info;
6236
6237 rc = bnxt_update_link(bp, true);
6238 if (rc) {
6239 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
6240 rc);
6241 return rc;
6242 }
33dac24a
MC		 6243 if (!BNXT_SINGLE_PF(bp))
6244 return 0;
6245
c0c050c5 6246 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
c9ee9516
MC		 6247 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
6248 link_info->req_flow_ctrl)
c0c050c5
MC		 6249 update_pause = true;
6250 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6251 link_info->force_pause_setting != link_info->req_flow_ctrl)
6252 update_pause = true;
c0c050c5
MC		 6253 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6254 if (BNXT_AUTO_MODE(link_info->auto_mode))
6255 update_link = true;
6256 if (link_info->req_link_speed != link_info->force_link_speed)
6257 update_link = true;
de73018f
MC		 6258 if (link_info->req_duplex != link_info->duplex_setting)
6259 update_link = true;
c0c050c5
MC		 6260 } else {
6261 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
6262 update_link = true;
6263 if (link_info->advertising != link_info->auto_link_speeds)
6264 update_link = true;
c0c050c5
MC		 6265 }
6266
16d663a6
MC		 6267 /* The last close may have shutdown the link, so need to call
6268 * PHY_CFG to bring it back up.
6269 */
6270 if (!netif_carrier_ok(bp->dev))
6271 update_link = true;
6272
939f7f0c
MC		 6273 if (!bnxt_eee_config_ok(bp))
6274 update_eee = true;
6275
c0c050c5 6276 if (update_link)
939f7f0c 6277 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
c0c050c5
MC		 6278 else if (update_pause)
6279 rc = bnxt_hwrm_set_pause(bp);
6280 if (rc) {
6281 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
6282 rc);
6283 return rc;
6284 }
6285
6286 return rc;
6287}
6288
11809490
JH		 6289/* Common routine to pre-map certain register block to different GRC window.
6290 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
6291 * in PF and 3 windows in VF that can be customized to map in different
6292 * register blocks.
6293 */
6294static void bnxt_preset_reg_win(struct bnxt *bp)
6295{
6296 if (BNXT_PF(bp)) {
6297 /* CAG registers map to GRC window #4 */
6298 writel(BNXT_CAG_REG_BASE,
6299 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
6300 }
6301}
6302
c0c050c5
MC		 6303static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6304{
6305 int rc = 0;
6306
11809490 6307 bnxt_preset_reg_win(bp);
c0c050c5
MC		 6308 netif_carrier_off(bp->dev);
6309 if (irq_re_init) {
6310 rc = bnxt_setup_int_mode(bp);
6311 if (rc) {
6312 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
6313 rc);
6314 return rc;
6315 }
6316 }
6317 if ((bp->flags & BNXT_FLAG_RFS) &&
6318 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
6319 /* disable RFS if falling back to INTA */
6320 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
6321 bp->flags &= ~BNXT_FLAG_RFS;
6322 }
6323
6324 rc = bnxt_alloc_mem(bp, irq_re_init);
6325 if (rc) {
6326 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6327 goto open_err_free_mem;
6328 }
6329
6330 if (irq_re_init) {
6331 bnxt_init_napi(bp);
6332 rc = bnxt_request_irq(bp);
6333 if (rc) {
6334 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
6335 goto open_err;
6336 }
6337 }
6338
6339 bnxt_enable_napi(bp);
6340
6341 rc = bnxt_init_nic(bp, irq_re_init);
6342 if (rc) {
6343 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6344 goto open_err;
6345 }
6346
6347 if (link_re_init) {
e2dc9b6e 6348 mutex_lock(&bp->link_lock);
c0c050c5 6349 rc = bnxt_update_phy_setting(bp);
e2dc9b6e 6350 mutex_unlock(&bp->link_lock);
c0c050c5 6351 if (rc)
ba41d46f 6352 netdev_warn(bp->dev, "failed to update phy settings\n");
c0c050c5
MC		 6353 }
6354
7cdd5fc3 6355 if (irq_re_init)
ad51b8e9 6356 udp_tunnel_get_rx_info(bp->dev);
c0c050c5 6357
caefe526 6358 set_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 6359 bnxt_enable_int(bp);
6360 /* Enable TX queues */
6361 bnxt_tx_enable(bp);
6362 mod_timer(&bp->timer, jiffies + bp->current_interval);
10289bec
MC		 6363 /* Poll link status and check for SFP+ module status */
6364 bnxt_get_port_module_status(bp);
c0c050c5 6365
ee5c7fb3
SP		 6366 /* VF-reps may need to be re-opened after the PF is re-opened */
6367 if (BNXT_PF(bp))
6368 bnxt_vf_reps_open(bp);
c0c050c5
MC		 6369 return 0;
6370
6371open_err:
6372 bnxt_disable_napi(bp);
6373 bnxt_del_napi(bp);
6374
6375open_err_free_mem:
6376 bnxt_free_skbs(bp);
6377 bnxt_free_irq(bp);
6378 bnxt_free_mem(bp, true);
6379 return rc;
6380}
6381
6382/* rtnl_lock held */
6383int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6384{
6385 int rc = 0;
6386
6387 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
6388 if (rc) {
6389 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
6390 dev_close(bp->dev);
6391 }
6392 return rc;
6393}
6394
f7dc1ea6
MC		 6395/* rtnl_lock held, open the NIC half way by allocating all resources, but
6396 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
6397 * self tests.
6398 */
6399int bnxt_half_open_nic(struct bnxt *bp)
6400{
6401 int rc = 0;
6402
6403 rc = bnxt_alloc_mem(bp, false);
6404 if (rc) {
6405 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6406 goto half_open_err;
6407 }
6408 rc = bnxt_init_nic(bp, false);
6409 if (rc) {
6410 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6411 goto half_open_err;
6412 }
6413 return 0;
6414
6415half_open_err:
6416 bnxt_free_skbs(bp);
6417 bnxt_free_mem(bp, false);
6418 dev_close(bp->dev);
6419 return rc;
6420}
6421
6422/* rtnl_lock held, this call can only be made after a previous successful
6423 * call to bnxt_half_open_nic().
6424 */
6425void bnxt_half_close_nic(struct bnxt *bp)
6426{
6427 bnxt_hwrm_resource_free(bp, false, false);
6428 bnxt_free_skbs(bp);
6429 bnxt_free_mem(bp, false);
6430}
6431
c0c050c5
MC		 6432static int bnxt_open(struct net_device *dev)
6433{
6434 struct bnxt *bp = netdev_priv(dev);
c0c050c5 6435
c0c050c5
MC		 6436 return __bnxt_open_nic(bp, true, true);
6437}
6438
f9b76ebd
MC		 6439static bool bnxt_drv_busy(struct bnxt *bp)
6440{
6441 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
6442 test_bit(BNXT_STATE_READ_STATS, &bp->state));
6443}
6444
c0c050c5
MC		 6445int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6446{
6447 int rc = 0;
6448
6449#ifdef CONFIG_BNXT_SRIOV
6450 if (bp->sriov_cfg) {
6451 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
6452 !bp->sriov_cfg,
6453 BNXT_SRIOV_CFG_WAIT_TMO);
6454 if (rc)
6455 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
6456 }
ee5c7fb3
SP		 6457
6458 /* Close the VF-reps before closing PF */
6459 if (BNXT_PF(bp))
6460 bnxt_vf_reps_close(bp);
c0c050c5
MC		 6461#endif
6462 /* Change device state to avoid TX queue wake up's */
6463 bnxt_tx_disable(bp);
6464
caefe526 6465 clear_bit(BNXT_STATE_OPEN, &bp->state);
4cebdcec 6466 smp_mb__after_atomic();
f9b76ebd 6467 while (bnxt_drv_busy(bp))
4cebdcec 6468 msleep(20);
c0c050c5 6469
9d8bc097 6470 /* Flush rings and and disable interrupts */
c0c050c5
MC		 6471 bnxt_shutdown_nic(bp, irq_re_init);
6472
6473 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
6474
6475 bnxt_disable_napi(bp);
c0c050c5
MC		 6476 del_timer_sync(&bp->timer);
6477 bnxt_free_skbs(bp);
6478
6479 if (irq_re_init) {
6480 bnxt_free_irq(bp);
6481 bnxt_del_napi(bp);
6482 }
6483 bnxt_free_mem(bp, irq_re_init);
6484 return rc;
6485}
6486
6487static int bnxt_close(struct net_device *dev)
6488{
6489 struct bnxt *bp = netdev_priv(dev);
6490
6491 bnxt_close_nic(bp, true, true);
33f7d55f 6492 bnxt_hwrm_shutdown_link(bp);
c0c050c5
MC		 6493 return 0;
6494}
6495
6496/* rtnl_lock held */
6497static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
6498{
6499 switch (cmd) {
6500 case SIOCGMIIPHY:
6501 /* fallthru */
6502 case SIOCGMIIREG: {
6503 if (!netif_running(dev))
6504 return -EAGAIN;
6505
6506 return 0;
6507 }
6508
6509 case SIOCSMIIREG:
6510 if (!netif_running(dev))
6511 return -EAGAIN;
6512
6513 return 0;
6514
6515 default:
6516 /* do nothing */
6517 break;
6518 }
6519 return -EOPNOTSUPP;
6520}
6521
bc1f4470 6522static void
c0c050c5
MC		 6523bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
6524{
6525 u32 i;
6526 struct bnxt *bp = netdev_priv(dev);
6527
f9b76ebd
MC		 6528 set_bit(BNXT_STATE_READ_STATS, &bp->state);
6529 /* Make sure bnxt_close_nic() sees that we are reading stats before
6530 * we check the BNXT_STATE_OPEN flag.
6531 */
6532 smp_mb__after_atomic();
6533 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
6534 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
bc1f4470 6535 return;
f9b76ebd 6536 }
c0c050c5
MC		 6537
6538 /* TODO check if we need to synchronize with bnxt_close path */
6539 for (i = 0; i < bp->cp_nr_rings; i++) {
6540 struct bnxt_napi *bnapi = bp->bnapi[i];
6541 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6542 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
6543
6544 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
6545 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
6546 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
6547
6548 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
6549 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
6550 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
6551
6552 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
6553 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
6554 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
6555
6556 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
6557 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
6558 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
6559
6560 stats->rx_missed_errors +=
6561 le64_to_cpu(hw_stats->rx_discard_pkts);
6562
6563 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
6564
c0c050c5
MC		 6565 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
6566 }
6567
9947f83f
MC		 6568 if (bp->flags & BNXT_FLAG_PORT_STATS) {
6569 struct rx_port_stats *rx = bp->hw_rx_port_stats;
6570 struct tx_port_stats *tx = bp->hw_tx_port_stats;
6571
6572 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
6573 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
6574 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
6575 le64_to_cpu(rx->rx_ovrsz_frames) +
6576 le64_to_cpu(rx->rx_runt_frames);
6577 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
6578 le64_to_cpu(rx->rx_jbr_frames);
6579 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
6580 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
6581 stats->tx_errors = le64_to_cpu(tx->tx_err);
6582 }
f9b76ebd 6583 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
c0c050c5
MC		 6584}
6585
6586static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
6587{
6588 struct net_device *dev = bp->dev;
6589 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6590 struct netdev_hw_addr *ha;
6591 u8 *haddr;
6592 int mc_count = 0;
6593 bool update = false;
6594 int off = 0;
6595
6596 netdev_for_each_mc_addr(ha, dev) {
6597 if (mc_count >= BNXT_MAX_MC_ADDRS) {
6598 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6599 vnic->mc_list_count = 0;
6600 return false;
6601 }
6602 haddr = ha->addr;
6603 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
6604 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
6605 update = true;
6606 }
6607 off += ETH_ALEN;
6608 mc_count++;
6609 }
6610 if (mc_count)
6611 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
6612
6613 if (mc_count != vnic->mc_list_count) {
6614 vnic->mc_list_count = mc_count;
6615 update = true;
6616 }
6617 return update;
6618}
6619
6620static bool bnxt_uc_list_updated(struct bnxt *bp)
6621{
6622 struct net_device *dev = bp->dev;
6623 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6624 struct netdev_hw_addr *ha;
6625 int off = 0;
6626
6627 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
6628 return true;
6629
6630 netdev_for_each_uc_addr(ha, dev) {
6631 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
6632 return true;
6633
6634 off += ETH_ALEN;
6635 }
6636 return false;
6637}
6638
6639static void bnxt_set_rx_mode(struct net_device *dev)
6640{
6641 struct bnxt *bp = netdev_priv(dev);
6642 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6643 u32 mask = vnic->rx_mask;
6644 bool mc_update = false;
6645 bool uc_update;
6646
6647 if (!netif_running(dev))
6648 return;
6649
6650 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
6651 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
6652 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
6653
17c71ac3 6654 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
c0c050c5
MC		 6655 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6656
6657 uc_update = bnxt_uc_list_updated(bp);
6658
6659 if (dev->flags & IFF_ALLMULTI) {
6660 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6661 vnic->mc_list_count = 0;
6662 } else {
6663 mc_update = bnxt_mc_list_updated(bp, &mask);
6664 }
6665
6666 if (mask != vnic->rx_mask || uc_update || mc_update) {
6667 vnic->rx_mask = mask;
6668
6669 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
c213eae8 6670 bnxt_queue_sp_work(bp);
c0c050c5
MC		 6671 }
6672}
6673
b664f008 6674static int bnxt_cfg_rx_mode(struct bnxt *bp)
c0c050c5
MC		 6675{
6676 struct net_device *dev = bp->dev;
6677 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6678 struct netdev_hw_addr *ha;
6679 int i, off = 0, rc;
6680 bool uc_update;
6681
6682 netif_addr_lock_bh(dev);
6683 uc_update = bnxt_uc_list_updated(bp);
6684 netif_addr_unlock_bh(dev);
6685
6686 if (!uc_update)
6687 goto skip_uc;
6688
6689 mutex_lock(&bp->hwrm_cmd_lock);
6690 for (i = 1; i < vnic->uc_filter_count; i++) {
6691 struct hwrm_cfa_l2_filter_free_input req = {0};
6692
6693 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
6694 -1);
6695
6696 req.l2_filter_id = vnic->fw_l2_filter_id[i];
6697
6698 rc = _hwrm_send_message(bp, &req, sizeof(req),
6699 HWRM_CMD_TIMEOUT);
6700 }
6701 mutex_unlock(&bp->hwrm_cmd_lock);
6702
6703 vnic->uc_filter_count = 1;
6704
6705 netif_addr_lock_bh(dev);
6706 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
6707 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6708 } else {
6709 netdev_for_each_uc_addr(ha, dev) {
6710 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
6711 off += ETH_ALEN;
6712 vnic->uc_filter_count++;
6713 }
6714 }
6715 netif_addr_unlock_bh(dev);
6716
6717 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
6718 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
6719 if (rc) {
6720 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
6721 rc);
6722 vnic->uc_filter_count = i;
b664f008 6723 return rc;
c0c050c5
MC		 6724 }
6725 }
6726
6727skip_uc:
6728 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
6729 if (rc)
6730 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
6731 rc);
b664f008
MC		 6732
6733 return rc;
c0c050c5
MC		 6734}
6735
8079e8f1
MC		 6736/* If the chip and firmware supports RFS */
6737static bool bnxt_rfs_supported(struct bnxt *bp)
6738{
6739 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
6740 return true;
ae10ae74
MC		 6741 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6742 return true;
8079e8f1
MC		 6743 return false;
6744}
6745
6746/* If runtime conditions support RFS */
2bcfa6f6
MC		 6747static bool bnxt_rfs_capable(struct bnxt *bp)
6748{
6749#ifdef CONFIG_RFS_ACCEL
8079e8f1 6750 int vnics, max_vnics, max_rss_ctxs;
2bcfa6f6 6751
964fd480 6752 if (!(bp->flags & BNXT_FLAG_MSIX_CAP))
2bcfa6f6
MC		 6753 return false;
6754
6755 vnics = 1 + bp->rx_nr_rings;
8079e8f1
MC		 6756 max_vnics = bnxt_get_max_func_vnics(bp);
6757 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
ae10ae74
MC		 6758
6759 /* RSS contexts not a limiting factor */
6760 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6761 max_rss_ctxs = max_vnics;
8079e8f1 6762 if (vnics > max_vnics || vnics > max_rss_ctxs) {
a2304909
VV		 6763 netdev_warn(bp->dev,
6764 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
8079e8f1 6765 min(max_rss_ctxs - 1, max_vnics - 1));
2bcfa6f6 6766 return false;
a2304909 6767 }
2bcfa6f6
MC		 6768
6769 return true;
6770#else
6771 return false;
6772#endif
6773}
6774
c0c050c5
MC		 6775static netdev_features_t bnxt_fix_features(struct net_device *dev,
6776 netdev_features_t features)
6777{
2bcfa6f6
MC		 6778 struct bnxt *bp = netdev_priv(dev);
6779
a2304909 6780 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
2bcfa6f6 6781 features &= ~NETIF_F_NTUPLE;
5a9f6b23
MC		 6782
6783 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
6784 * turned on or off together.
6785 */
6786 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
6787 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
6788 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
6789 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6790 NETIF_F_HW_VLAN_STAG_RX);
6791 else
6792 features |= NETIF_F_HW_VLAN_CTAG_RX |
6793 NETIF_F_HW_VLAN_STAG_RX;
6794 }
cf6645f8
MC		 6795#ifdef CONFIG_BNXT_SRIOV
6796 if (BNXT_VF(bp)) {
6797 if (bp->vf.vlan) {
6798 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6799 NETIF_F_HW_VLAN_STAG_RX);
6800 }
6801 }
6802#endif
c0c050c5
MC		 6803 return features;
6804}
6805
6806static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
6807{
6808 struct bnxt *bp = netdev_priv(dev);
6809 u32 flags = bp->flags;
6810 u32 changes;
6811 int rc = 0;
6812 bool re_init = false;
6813 bool update_tpa = false;
6814
6815 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
3e8060fa 6816 if ((features & NETIF_F_GRO) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
c0c050c5
MC		 6817 flags |= BNXT_FLAG_GRO;
6818 if (features & NETIF_F_LRO)
6819 flags |= BNXT_FLAG_LRO;
6820
bdbd1eb5
MC		 6821 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
6822 flags &= ~BNXT_FLAG_TPA;
6823
c0c050c5
MC		 6824 if (features & NETIF_F_HW_VLAN_CTAG_RX)
6825 flags |= BNXT_FLAG_STRIP_VLAN;
6826
6827 if (features & NETIF_F_NTUPLE)
6828 flags |= BNXT_FLAG_RFS;
6829
6830 changes = flags ^ bp->flags;
6831 if (changes & BNXT_FLAG_TPA) {
6832 update_tpa = true;
6833 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
6834 (flags & BNXT_FLAG_TPA) == 0)
6835 re_init = true;
6836 }
6837
6838 if (changes & ~BNXT_FLAG_TPA)
6839 re_init = true;
6840
6841 if (flags != bp->flags) {
6842 u32 old_flags = bp->flags;
6843
6844 bp->flags = flags;
6845
2bcfa6f6 6846 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
c0c050c5
MC		 6847 if (update_tpa)
6848 bnxt_set_ring_params(bp);
6849 return rc;
6850 }
6851
6852 if (re_init) {
6853 bnxt_close_nic(bp, false, false);
6854 if (update_tpa)
6855 bnxt_set_ring_params(bp);
6856
6857 return bnxt_open_nic(bp, false, false);
6858 }
6859 if (update_tpa) {
6860 rc = bnxt_set_tpa(bp,
6861 (flags & BNXT_FLAG_TPA) ?
6862 true : false);
6863 if (rc)
6864 bp->flags = old_flags;
6865 }
6866 }
6867 return rc;
6868}
6869
9f554590
MC		 6870static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
6871{
b6ab4b01 6872 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9f554590
MC		 6873 int i = bnapi->index;
6874
3b2b7d9d
MC		 6875 if (!txr)
6876 return;
6877
9f554590
MC		 6878 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
6879 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
6880 txr->tx_cons);
6881}
6882
6883static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
6884{
b6ab4b01 6885 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9f554590
MC		 6886 int i = bnapi->index;
6887
3b2b7d9d
MC		 6888 if (!rxr)
6889 return;
6890
9f554590
MC		 6891 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
6892 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
6893 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
6894 rxr->rx_sw_agg_prod);
6895}
6896
6897static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
6898{
6899 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6900 int i = bnapi->index;
6901
6902 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
6903 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
6904}
6905
c0c050c5
MC		 6906static void bnxt_dbg_dump_states(struct bnxt *bp)
6907{
6908 int i;
6909 struct bnxt_napi *bnapi;
c0c050c5
MC		 6910
6911 for (i = 0; i < bp->cp_nr_rings; i++) {
6912 bnapi = bp->bnapi[i];
c0c050c5 6913 if (netif_msg_drv(bp)) {
9f554590
MC		 6914 bnxt_dump_tx_sw_state(bnapi);
6915 bnxt_dump_rx_sw_state(bnapi);
6916 bnxt_dump_cp_sw_state(bnapi);
c0c050c5
MC		 6917 }
6918 }
6919}
6920
6988bd92 6921static void bnxt_reset_task(struct bnxt *bp, bool silent)
c0c050c5 6922{
6988bd92
MC		 6923 if (!silent)
6924 bnxt_dbg_dump_states(bp);
028de140 6925 if (netif_running(bp->dev)) {
b386cd36
MC		 6926 int rc;
6927
6928 if (!silent)
6929 bnxt_ulp_stop(bp);
028de140 6930 bnxt_close_nic(bp, false, false);
b386cd36
MC		 6931 rc = bnxt_open_nic(bp, false, false);
6932 if (!silent && !rc)
6933 bnxt_ulp_start(bp);
028de140 6934 }
c0c050c5
MC		 6935}
6936
6937static void bnxt_tx_timeout(struct net_device *dev)
6938{
6939 struct bnxt *bp = netdev_priv(dev);
6940
6941 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
6942 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 6943 bnxt_queue_sp_work(bp);
c0c050c5
MC		 6944}
6945
6946#ifdef CONFIG_NET_POLL_CONTROLLER
6947static void bnxt_poll_controller(struct net_device *dev)
6948{
6949 struct bnxt *bp = netdev_priv(dev);
6950 int i;
6951
2270bc5d
MC		 6952 /* Only process tx rings/combined rings in netpoll mode. */
6953 for (i = 0; i < bp->tx_nr_rings; i++) {
6954 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5 6955
2270bc5d 6956 napi_schedule(&txr->bnapi->napi);
c0c050c5
MC		 6957 }
6958}
6959#endif
6960
6961static void bnxt_timer(unsigned long data)
6962{
6963 struct bnxt *bp = (struct bnxt *)data;
6964 struct net_device *dev = bp->dev;
6965
6966 if (!netif_running(dev))
6967 return;
6968
6969 if (atomic_read(&bp->intr_sem) != 0)
6970 goto bnxt_restart_timer;
6971
adcc331e
MC		 6972 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
6973 bp->stats_coal_ticks) {
3bdf56c4 6974 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
c213eae8 6975 bnxt_queue_sp_work(bp);
3bdf56c4 6976 }
c0c050c5
MC		 6977bnxt_restart_timer:
6978 mod_timer(&bp->timer, jiffies + bp->current_interval);
6979}
6980
a551ee94 6981static void bnxt_rtnl_lock_sp(struct bnxt *bp)
6988bd92 6982{
a551ee94
MC		 6983 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
6984 * set. If the device is being closed, bnxt_close() may be holding
6988bd92
MC		 6985 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
6986 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
6987 */
6988 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
6989 rtnl_lock();
a551ee94
MC		 6990}
6991
6992static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
6993{
6988bd92
MC		 6994 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
6995 rtnl_unlock();
6996}
6997
a551ee94
MC		 6998/* Only called from bnxt_sp_task() */
6999static void bnxt_reset(struct bnxt *bp, bool silent)
7000{
7001 bnxt_rtnl_lock_sp(bp);
7002 if (test_bit(BNXT_STATE_OPEN, &bp->state))
7003 bnxt_reset_task(bp, silent);
7004 bnxt_rtnl_unlock_sp(bp);
7005}
7006
c0c050c5
MC		 7007static void bnxt_cfg_ntp_filters(struct bnxt *);
7008
7009static void bnxt_sp_task(struct work_struct *work)
7010{
7011 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
c0c050c5 7012
4cebdcec
MC		 7013 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7014 smp_mb__after_atomic();
7015 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7016 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5 7017 return;
4cebdcec 7018 }
c0c050c5
MC		 7019
7020 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
7021 bnxt_cfg_rx_mode(bp);
7022
7023 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
7024 bnxt_cfg_ntp_filters(bp);
c0c050c5
MC		 7025 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
7026 bnxt_hwrm_exec_fwd_req(bp);
7027 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7028 bnxt_hwrm_tunnel_dst_port_alloc(
7029 bp, bp->vxlan_port,
7030 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7031 }
7032 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7033 bnxt_hwrm_tunnel_dst_port_free(
7034 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7035 }
7cdd5fc3
AD		 7036 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7037 bnxt_hwrm_tunnel_dst_port_alloc(
7038 bp, bp->nge_port,
7039 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7040 }
7041 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7042 bnxt_hwrm_tunnel_dst_port_free(
7043 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7044 }
3bdf56c4
MC		 7045 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
7046 bnxt_hwrm_port_qstats(bp);
7047
0eaa24b9 7048 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
e2dc9b6e 7049 int rc;
0eaa24b9 7050
e2dc9b6e 7051 mutex_lock(&bp->link_lock);
0eaa24b9
MC		 7052 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
7053 &bp->sp_event))
7054 bnxt_hwrm_phy_qcaps(bp);
7055
e2dc9b6e
MC		 7056 rc = bnxt_update_link(bp, true);
7057 mutex_unlock(&bp->link_lock);
0eaa24b9
MC		 7058 if (rc)
7059 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
7060 rc);
7061 }
90c694bb 7062 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
e2dc9b6e
MC		 7063 mutex_lock(&bp->link_lock);
7064 bnxt_get_port_module_status(bp);
7065 mutex_unlock(&bp->link_lock);
90c694bb 7066 }
e2dc9b6e
MC		 7067 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
7068 * must be the last functions to be called before exiting.
7069 */
6988bd92
MC		 7070 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
7071 bnxt_reset(bp, false);
4cebdcec 7072
fc0f1929
MC		 7073 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
7074 bnxt_reset(bp, true);
7075
4cebdcec
MC		 7076 smp_mb__before_atomic();
7077 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5
MC		 7078}
7079
d1e7925e 7080/* Under rtnl_lock */
98fdbe73
MC		 7081int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
7082 int tx_xdp)
d1e7925e
MC		 7083{
7084 int max_rx, max_tx, tx_sets = 1;
7085 int tx_rings_needed;
d1e7925e
MC		 7086 int rc;
7087
d1e7925e
MC		 7088 if (tcs)
7089 tx_sets = tcs;
7090
7091 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
7092 if (rc)
7093 return rc;
7094
7095 if (max_rx < rx)
7096 return -ENOMEM;
7097
5f449249 7098 tx_rings_needed = tx * tx_sets + tx_xdp;
d1e7925e
MC		 7099 if (max_tx < tx_rings_needed)
7100 return -ENOMEM;
7101
98fdbe73 7102 return bnxt_hwrm_check_tx_rings(bp, tx_rings_needed);
d1e7925e
MC		 7103}
7104
17086399
SP		 7105static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
7106{
7107 if (bp->bar2) {
7108 pci_iounmap(pdev, bp->bar2);
7109 bp->bar2 = NULL;
7110 }
7111
7112 if (bp->bar1) {
7113 pci_iounmap(pdev, bp->bar1);
7114 bp->bar1 = NULL;
7115 }
7116
7117 if (bp->bar0) {
7118 pci_iounmap(pdev, bp->bar0);
7119 bp->bar0 = NULL;
7120 }
7121}
7122
7123static void bnxt_cleanup_pci(struct bnxt *bp)
7124{
7125 bnxt_unmap_bars(bp, bp->pdev);
7126 pci_release_regions(bp->pdev);
7127 pci_disable_device(bp->pdev);
7128}
7129
c0c050c5
MC		 7130static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
7131{
7132 int rc;
7133 struct bnxt *bp = netdev_priv(dev);
7134
7135 SET_NETDEV_DEV(dev, &pdev->dev);
7136
7137 /* enable device (incl. PCI PM wakeup), and bus-mastering */
7138 rc = pci_enable_device(pdev);
7139 if (rc) {
7140 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
7141 goto init_err;
7142 }
7143
7144 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
7145 dev_err(&pdev->dev,
7146 "Cannot find PCI device base address, aborting\n");
7147 rc = -ENODEV;
7148 goto init_err_disable;
7149 }
7150
7151 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
7152 if (rc) {
7153 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
7154 goto init_err_disable;
7155 }
7156
7157 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
7158 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
7159 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
7160 goto init_err_disable;
7161 }
7162
7163 pci_set_master(pdev);
7164
7165 bp->dev = dev;
7166 bp->pdev = pdev;
7167
7168 bp->bar0 = pci_ioremap_bar(pdev, 0);
7169 if (!bp->bar0) {
7170 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
7171 rc = -ENOMEM;
7172 goto init_err_release;
7173 }
7174
7175 bp->bar1 = pci_ioremap_bar(pdev, 2);
7176 if (!bp->bar1) {
7177 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
7178 rc = -ENOMEM;
7179 goto init_err_release;
7180 }
7181
7182 bp->bar2 = pci_ioremap_bar(pdev, 4);
7183 if (!bp->bar2) {
7184 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
7185 rc = -ENOMEM;
7186 goto init_err_release;
7187 }
7188
6316ea6d
SB		 7189 pci_enable_pcie_error_reporting(pdev);
7190
c0c050c5
MC		 7191 INIT_WORK(&bp->sp_task, bnxt_sp_task);
7192
7193 spin_lock_init(&bp->ntp_fltr_lock);
7194
7195 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
7196 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
7197
dfb5b894 7198 /* tick values in micro seconds */
dfc9c94a
MC		 7199 bp->rx_coal_ticks = 12;
7200 bp->rx_coal_bufs = 30;
dfb5b894
MC		 7201 bp->rx_coal_ticks_irq = 1;
7202 bp->rx_coal_bufs_irq = 2;
c0c050c5 7203
dfc9c94a
MC		 7204 bp->tx_coal_ticks = 25;
7205 bp->tx_coal_bufs = 30;
7206 bp->tx_coal_ticks_irq = 2;
7207 bp->tx_coal_bufs_irq = 2;
7208
51f30785
MC		 7209 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
7210
c0c050c5
MC		 7211 init_timer(&bp->timer);
7212 bp->timer.data = (unsigned long)bp;
7213 bp->timer.function = bnxt_timer;
7214 bp->current_interval = BNXT_TIMER_INTERVAL;
7215
caefe526 7216 clear_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 7217 return 0;
7218
7219init_err_release:
17086399 7220 bnxt_unmap_bars(bp, pdev);
c0c050c5
MC		 7221 pci_release_regions(pdev);
7222
7223init_err_disable:
7224 pci_disable_device(pdev);
7225
7226init_err:
7227 return rc;
7228}
7229
7230/* rtnl_lock held */
7231static int bnxt_change_mac_addr(struct net_device *dev, void *p)
7232{
7233 struct sockaddr *addr = p;
1fc2cfd0
JH		 7234 struct bnxt *bp = netdev_priv(dev);
7235 int rc = 0;
c0c050c5
MC		 7236
7237 if (!is_valid_ether_addr(addr->sa_data))
7238 return -EADDRNOTAVAIL;
7239
84c33dd3
MC		 7240 rc = bnxt_approve_mac(bp, addr->sa_data);
7241 if (rc)
7242 return rc;
bdd4347b 7243
1fc2cfd0
JH		 7244 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
7245 return 0;
7246
c0c050c5 7247 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1fc2cfd0
JH		 7248 if (netif_running(dev)) {
7249 bnxt_close_nic(bp, false, false);
7250 rc = bnxt_open_nic(bp, false, false);
7251 }
c0c050c5 7252
1fc2cfd0 7253 return rc;
c0c050c5
MC		 7254}
7255
7256/* rtnl_lock held */
7257static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
7258{
7259 struct bnxt *bp = netdev_priv(dev);
7260
c0c050c5
MC		 7261 if (netif_running(dev))
7262 bnxt_close_nic(bp, false, false);
7263
7264 dev->mtu = new_mtu;
7265 bnxt_set_ring_params(bp);
7266
7267 if (netif_running(dev))
7268 return bnxt_open_nic(bp, false, false);
7269
7270 return 0;
7271}
7272
c5e3deb8 7273int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
c0c050c5
MC		 7274{
7275 struct bnxt *bp = netdev_priv(dev);
3ffb6a39 7276 bool sh = false;
d1e7925e 7277 int rc;
16e5cc64 7278
c0c050c5 7279 if (tc > bp->max_tc) {
b451c8b6 7280 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
c0c050c5
MC		 7281 tc, bp->max_tc);
7282 return -EINVAL;
7283 }
7284
7285 if (netdev_get_num_tc(dev) == tc)
7286 return 0;
7287
3ffb6a39
MC		 7288 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7289 sh = true;
7290
98fdbe73
MC		 7291 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
7292 sh, tc, bp->tx_nr_rings_xdp);
d1e7925e
MC		 7293 if (rc)
7294 return rc;
c0c050c5
MC		 7295
7296 /* Needs to close the device and do hw resource re-allocations */
7297 if (netif_running(bp->dev))
7298 bnxt_close_nic(bp, true, false);
7299
7300 if (tc) {
7301 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
7302 netdev_set_num_tc(dev, tc);
7303 } else {
7304 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7305 netdev_reset_tc(dev);
7306 }
87e9b377 7307 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
3ffb6a39
MC		 7308 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7309 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5
MC		 7310 bp->num_stat_ctxs = bp->cp_nr_rings;
7311
7312 if (netif_running(bp->dev))
7313 return bnxt_open_nic(bp, true, false);
7314
7315 return 0;
7316}
7317
2ae7408f
SP		 7318static int bnxt_setup_flower(struct net_device *dev,
7319 struct tc_cls_flower_offload *cls_flower)
c5e3deb8 7320{
2ae7408f 7321 struct bnxt *bp = netdev_priv(dev);
de4784ca 7322
2ae7408f 7323 if (BNXT_VF(bp))
38cf0426 7324 return -EOPNOTSUPP;
c5e3deb8 7325
2ae7408f
SP		 7326 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, cls_flower);
7327}
7328
7329static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
7330 void *type_data)
7331{
7332 switch (type) {
7333 case TC_SETUP_CLSFLOWER:
7334 return bnxt_setup_flower(dev, type_data);
7335 case TC_SETUP_MQPRIO: {
7336 struct tc_mqprio_qopt *mqprio = type_data;
7337
7338 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
56f36acd 7339
2ae7408f
SP		 7340 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
7341 }
7342 default:
7343 return -EOPNOTSUPP;
7344 }
c5e3deb8
MC		 7345}
7346
c0c050c5
MC		 7347#ifdef CONFIG_RFS_ACCEL
7348static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
7349 struct bnxt_ntuple_filter *f2)
7350{
7351 struct flow_keys *keys1 = &f1->fkeys;
7352 struct flow_keys *keys2 = &f2->fkeys;
7353
7354 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
7355 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
7356 keys1->ports.ports == keys2->ports.ports &&
7357 keys1->basic.ip_proto == keys2->basic.ip_proto &&
7358 keys1->basic.n_proto == keys2->basic.n_proto &&
61aad724 7359 keys1->control.flags == keys2->control.flags &&
a54c4d74
MC		 7360 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
7361 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
c0c050c5
MC		 7362 return true;
7363
7364 return false;
7365}
7366
7367static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
7368 u16 rxq_index, u32 flow_id)
7369{
7370 struct bnxt *bp = netdev_priv(dev);
7371 struct bnxt_ntuple_filter *fltr, *new_fltr;
7372 struct flow_keys *fkeys;
7373 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
a54c4d74 7374 int rc = 0, idx, bit_id, l2_idx = 0;
c0c050c5
MC		 7375 struct hlist_head *head;
7376
a54c4d74
MC		 7377 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
7378 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7379 int off = 0, j;
7380
7381 netif_addr_lock_bh(dev);
7382 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
7383 if (ether_addr_equal(eth->h_dest,
7384 vnic->uc_list + off)) {
7385 l2_idx = j + 1;
7386 break;
7387 }
7388 }
7389 netif_addr_unlock_bh(dev);
7390 if (!l2_idx)
7391 return -EINVAL;
7392 }
c0c050c5
MC		 7393 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
7394 if (!new_fltr)
7395 return -ENOMEM;
7396
7397 fkeys = &new_fltr->fkeys;
7398 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
7399 rc = -EPROTONOSUPPORT;
7400 goto err_free;
7401 }
7402
dda0e746
MC		 7403 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
7404 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
c0c050c5
MC		 7405 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
7406 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
7407 rc = -EPROTONOSUPPORT;
7408 goto err_free;
7409 }
dda0e746
MC		 7410 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
7411 bp->hwrm_spec_code < 0x10601) {
7412 rc = -EPROTONOSUPPORT;
7413 goto err_free;
7414 }
61aad724
MC		 7415 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
7416 bp->hwrm_spec_code < 0x10601) {
7417 rc = -EPROTONOSUPPORT;
7418 goto err_free;
7419 }
c0c050c5 7420
a54c4d74 7421 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
c0c050c5
MC		 7422 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
7423
7424 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
7425 head = &bp->ntp_fltr_hash_tbl[idx];
7426 rcu_read_lock();
7427 hlist_for_each_entry_rcu(fltr, head, hash) {
7428 if (bnxt_fltr_match(fltr, new_fltr)) {
7429 rcu_read_unlock();
7430 rc = 0;
7431 goto err_free;
7432 }
7433 }
7434 rcu_read_unlock();
7435
7436 spin_lock_bh(&bp->ntp_fltr_lock);
84e86b98
MC		 7437 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
7438 BNXT_NTP_FLTR_MAX_FLTR, 0);
7439 if (bit_id < 0) {
c0c050c5
MC		 7440 spin_unlock_bh(&bp->ntp_fltr_lock);
7441 rc = -ENOMEM;
7442 goto err_free;
7443 }
7444
84e86b98 7445 new_fltr->sw_id = (u16)bit_id;
c0c050c5 7446 new_fltr->flow_id = flow_id;
a54c4d74 7447 new_fltr->l2_fltr_idx = l2_idx;
c0c050c5
MC		 7448 new_fltr->rxq = rxq_index;
7449 hlist_add_head_rcu(&new_fltr->hash, head);
7450 bp->ntp_fltr_count++;
7451 spin_unlock_bh(&bp->ntp_fltr_lock);
7452
7453 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
c213eae8 7454 bnxt_queue_sp_work(bp);
c0c050c5
MC		 7455
7456 return new_fltr->sw_id;
7457
7458err_free:
7459 kfree(new_fltr);
7460 return rc;
7461}
7462
7463static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7464{
7465 int i;
7466
7467 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
7468 struct hlist_head *head;
7469 struct hlist_node *tmp;
7470 struct bnxt_ntuple_filter *fltr;
7471 int rc;
7472
7473 head = &bp->ntp_fltr_hash_tbl[i];
7474 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
7475 bool del = false;
7476
7477 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
7478 if (rps_may_expire_flow(bp->dev, fltr->rxq,
7479 fltr->flow_id,
7480 fltr->sw_id)) {
7481 bnxt_hwrm_cfa_ntuple_filter_free(bp,
7482 fltr);
7483 del = true;
7484 }
7485 } else {
7486 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
7487 fltr);
7488 if (rc)
7489 del = true;
7490 else
7491 set_bit(BNXT_FLTR_VALID, &fltr->state);
7492 }
7493
7494 if (del) {
7495 spin_lock_bh(&bp->ntp_fltr_lock);
7496 hlist_del_rcu(&fltr->hash);
7497 bp->ntp_fltr_count--;
7498 spin_unlock_bh(&bp->ntp_fltr_lock);
7499 synchronize_rcu();
7500 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
7501 kfree(fltr);
7502 }
7503 }
7504 }
19241368
JH		 7505 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
7506 netdev_info(bp->dev, "Receive PF driver unload event!");
c0c050c5
MC		 7507}
7508
7509#else
7510
7511static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7512{
7513}
7514
7515#endif /* CONFIG_RFS_ACCEL */
7516
ad51b8e9
AD		 7517static void bnxt_udp_tunnel_add(struct net_device *dev,
7518 struct udp_tunnel_info *ti)
c0c050c5
MC		 7519{
7520 struct bnxt *bp = netdev_priv(dev);
7521
ad51b8e9 7522 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 7523 return;
7524
ad51b8e9 7525 if (!netif_running(dev))
c0c050c5
MC		 7526 return;
7527
ad51b8e9
AD		 7528 switch (ti->type) {
7529 case UDP_TUNNEL_TYPE_VXLAN:
7530 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
7531 return;
c0c050c5 7532
ad51b8e9
AD		 7533 bp->vxlan_port_cnt++;
7534 if (bp->vxlan_port_cnt == 1) {
7535 bp->vxlan_port = ti->port;
7536 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
c213eae8 7537 bnxt_queue_sp_work(bp);
ad51b8e9
AD		 7538 }
7539 break;
7cdd5fc3
AD		 7540 case UDP_TUNNEL_TYPE_GENEVE:
7541 if (bp->nge_port_cnt && bp->nge_port != ti->port)
7542 return;
7543
7544 bp->nge_port_cnt++;
7545 if (bp->nge_port_cnt == 1) {
7546 bp->nge_port = ti->port;
7547 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
7548 }
7549 break;
ad51b8e9
AD		 7550 default:
7551 return;
c0c050c5 7552 }
ad51b8e9 7553
c213eae8 7554 bnxt_queue_sp_work(bp);
c0c050c5
MC		 7555}
7556
ad51b8e9
AD		 7557static void bnxt_udp_tunnel_del(struct net_device *dev,
7558 struct udp_tunnel_info *ti)
c0c050c5
MC		 7559{
7560 struct bnxt *bp = netdev_priv(dev);
7561
ad51b8e9 7562 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 7563 return;
7564
ad51b8e9 7565 if (!netif_running(dev))
c0c050c5
MC		 7566 return;
7567
ad51b8e9
AD		 7568 switch (ti->type) {
7569 case UDP_TUNNEL_TYPE_VXLAN:
7570 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
7571 return;
c0c050c5
MC		 7572 bp->vxlan_port_cnt--;
7573
ad51b8e9
AD		 7574 if (bp->vxlan_port_cnt != 0)
7575 return;
7576
7577 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
7578 break;
7cdd5fc3
AD		 7579 case UDP_TUNNEL_TYPE_GENEVE:
7580 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
7581 return;
7582 bp->nge_port_cnt--;
7583
7584 if (bp->nge_port_cnt != 0)
7585 return;
7586
7587 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
7588 break;
ad51b8e9
AD		 7589 default:
7590 return;
c0c050c5 7591 }
ad51b8e9 7592
c213eae8 7593 bnxt_queue_sp_work(bp);
c0c050c5
MC		 7594}
7595
39d8ba2e
MC		 7596static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7597 struct net_device *dev, u32 filter_mask,
7598 int nlflags)
7599{
7600 struct bnxt *bp = netdev_priv(dev);
7601
7602 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
7603 nlflags, filter_mask, NULL);
7604}
7605
7606static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7607 u16 flags)
7608{
7609 struct bnxt *bp = netdev_priv(dev);
7610 struct nlattr *attr, *br_spec;
7611 int rem, rc = 0;
7612
7613 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
7614 return -EOPNOTSUPP;
7615
7616 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7617 if (!br_spec)
7618 return -EINVAL;
7619
7620 nla_for_each_nested(attr, br_spec, rem) {
7621 u16 mode;
7622
7623 if (nla_type(attr) != IFLA_BRIDGE_MODE)
7624 continue;
7625
7626 if (nla_len(attr) < sizeof(mode))
7627 return -EINVAL;
7628
7629 mode = nla_get_u16(attr);
7630 if (mode == bp->br_mode)
7631 break;
7632
7633 rc = bnxt_hwrm_set_br_mode(bp, mode);
7634 if (!rc)
7635 bp->br_mode = mode;
7636 break;
7637 }
7638 return rc;
7639}
7640
c124a62f
SP		 7641static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
7642 size_t len)
7643{
7644 struct bnxt *bp = netdev_priv(dev);
7645 int rc;
7646
7647 /* The PF and it's VF-reps only support the switchdev framework */
7648 if (!BNXT_PF(bp))
7649 return -EOPNOTSUPP;
7650
53f70b8b 7651 rc = snprintf(buf, len, "p%d", bp->pf.port_id);
c124a62f
SP		 7652
7653 if (rc >= len)
7654 return -EOPNOTSUPP;
7655 return 0;
7656}
7657
7658int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
7659{
7660 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
7661 return -EOPNOTSUPP;
7662
7663 /* The PF and it's VF-reps only support the switchdev framework */
7664 if (!BNXT_PF(bp))
7665 return -EOPNOTSUPP;
7666
7667 switch (attr->id) {
7668 case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
7669 /* In SRIOV each PF-pool (PF + child VFs) serves as a
7670 * switching domain, the PF's perm mac-addr can be used
7671 * as the unique parent-id
7672 */
7673 attr->u.ppid.id_len = ETH_ALEN;
7674 ether_addr_copy(attr->u.ppid.id, bp->pf.mac_addr);
7675 break;
7676 default:
7677 return -EOPNOTSUPP;
7678 }
7679 return 0;
7680}
7681
7682static int bnxt_swdev_port_attr_get(struct net_device *dev,
7683 struct switchdev_attr *attr)
7684{
7685 return bnxt_port_attr_get(netdev_priv(dev), attr);
7686}
7687
7688static const struct switchdev_ops bnxt_switchdev_ops = {
7689 .switchdev_port_attr_get = bnxt_swdev_port_attr_get
7690};
7691
c0c050c5
MC		 7692static const struct net_device_ops bnxt_netdev_ops = {
7693 .ndo_open = bnxt_open,
7694 .ndo_start_xmit = bnxt_start_xmit,
7695 .ndo_stop = bnxt_close,
7696 .ndo_get_stats64 = bnxt_get_stats64,
7697 .ndo_set_rx_mode = bnxt_set_rx_mode,
7698 .ndo_do_ioctl = bnxt_ioctl,
7699 .ndo_validate_addr = eth_validate_addr,
7700 .ndo_set_mac_address = bnxt_change_mac_addr,
7701 .ndo_change_mtu = bnxt_change_mtu,
7702 .ndo_fix_features = bnxt_fix_features,
7703 .ndo_set_features = bnxt_set_features,
7704 .ndo_tx_timeout = bnxt_tx_timeout,
7705#ifdef CONFIG_BNXT_SRIOV
7706 .ndo_get_vf_config = bnxt_get_vf_config,
7707 .ndo_set_vf_mac = bnxt_set_vf_mac,
7708 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
7709 .ndo_set_vf_rate = bnxt_set_vf_bw,
7710 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
7711 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
7712#endif
7713#ifdef CONFIG_NET_POLL_CONTROLLER
7714 .ndo_poll_controller = bnxt_poll_controller,
7715#endif
7716 .ndo_setup_tc = bnxt_setup_tc,
7717#ifdef CONFIG_RFS_ACCEL
7718 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
7719#endif
ad51b8e9
AD		 7720 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add,
7721 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del,
c6d30e83 7722 .ndo_xdp = bnxt_xdp,
39d8ba2e
MC		 7723 .ndo_bridge_getlink = bnxt_bridge_getlink,
7724 .ndo_bridge_setlink = bnxt_bridge_setlink,
c124a62f 7725 .ndo_get_phys_port_name = bnxt_get_phys_port_name
c0c050c5
MC		 7726};
7727
7728static void bnxt_remove_one(struct pci_dev *pdev)
7729{
7730 struct net_device *dev = pci_get_drvdata(pdev);
7731 struct bnxt *bp = netdev_priv(dev);
7732
4ab0c6a8 7733 if (BNXT_PF(bp)) {
c0c050c5 7734 bnxt_sriov_disable(bp);
4ab0c6a8
SP		 7735 bnxt_dl_unregister(bp);
7736 }
c0c050c5 7737
6316ea6d 7738 pci_disable_pcie_error_reporting(pdev);
c0c050c5 7739 unregister_netdev(dev);
2ae7408f 7740 bnxt_shutdown_tc(bp);
c213eae8 7741 bnxt_cancel_sp_work(bp);
c0c050c5
MC		 7742 bp->sp_event = 0;
7743
7809592d 7744 bnxt_clear_int_mode(bp);
be58a0da 7745 bnxt_hwrm_func_drv_unrgtr(bp);
c0c050c5 7746 bnxt_free_hwrm_resources(bp);
e605db80 7747 bnxt_free_hwrm_short_cmd_req(bp);
eb513658 7748 bnxt_ethtool_free(bp);
7df4ae9f 7749 bnxt_dcb_free(bp);
a588e458
MC		 7750 kfree(bp->edev);
7751 bp->edev = NULL;
c6d30e83
MC		 7752 if (bp->xdp_prog)
7753 bpf_prog_put(bp->xdp_prog);
17086399 7754 bnxt_cleanup_pci(bp);
c0c050c5 7755 free_netdev(dev);
c0c050c5
MC		 7756}
7757
7758static int bnxt_probe_phy(struct bnxt *bp)
7759{
7760 int rc = 0;
7761 struct bnxt_link_info *link_info = &bp->link_info;
c0c050c5 7762
170ce013
MC		 7763 rc = bnxt_hwrm_phy_qcaps(bp);
7764 if (rc) {
7765 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
7766 rc);
7767 return rc;
7768 }
e2dc9b6e 7769 mutex_init(&bp->link_lock);
170ce013 7770
c0c050c5
MC		 7771 rc = bnxt_update_link(bp, false);
7772 if (rc) {
7773 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
7774 rc);
7775 return rc;
7776 }
7777
93ed8117
MC		 7778 /* Older firmware does not have supported_auto_speeds, so assume
7779 * that all supported speeds can be autonegotiated.
7780 */
7781 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
7782 link_info->support_auto_speeds = link_info->support_speeds;
7783
c0c050c5 7784 /*initialize the ethool setting copy with NVM settings */
0d8abf02 7785 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
c9ee9516
MC		 7786 link_info->autoneg = BNXT_AUTONEG_SPEED;
7787 if (bp->hwrm_spec_code >= 0x10201) {
7788 if (link_info->auto_pause_setting &
7789 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
7790 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7791 } else {
7792 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7793 }
0d8abf02 7794 link_info->advertising = link_info->auto_link_speeds;
0d8abf02
MC		 7795 } else {
7796 link_info->req_link_speed = link_info->force_link_speed;
7797 link_info->req_duplex = link_info->duplex_setting;
c0c050c5 7798 }
c9ee9516
MC		 7799 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
7800 link_info->req_flow_ctrl =
7801 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
7802 else
7803 link_info->req_flow_ctrl = link_info->force_pause_setting;
c0c050c5
MC		 7804 return rc;
7805}
7806
7807static int bnxt_get_max_irq(struct pci_dev *pdev)
7808{
7809 u16 ctrl;
7810
7811 if (!pdev->msix_cap)
7812 return 1;
7813
7814 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
7815 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
7816}
7817
6e6c5a57
MC		 7818static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7819 int *max_cp)
c0c050c5 7820{
6e6c5a57 7821 int max_ring_grps = 0;
c0c050c5 7822
379a80a1 7823#ifdef CONFIG_BNXT_SRIOV
415b6f19 7824 if (!BNXT_PF(bp)) {
c0c050c5
MC		 7825 *max_tx = bp->vf.max_tx_rings;
7826 *max_rx = bp->vf.max_rx_rings;
6e6c5a57
MC		 7827 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
7828 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
b72d4a68 7829 max_ring_grps = bp->vf.max_hw_ring_grps;
415b6f19 7830 } else
379a80a1 7831#endif
415b6f19
AB		 7832 {
7833 *max_tx = bp->pf.max_tx_rings;
7834 *max_rx = bp->pf.max_rx_rings;
7835 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
7836 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
7837 max_ring_grps = bp->pf.max_hw_ring_grps;
c0c050c5 7838 }
76595193
PS		 7839 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
7840 *max_cp -= 1;
7841 *max_rx -= 2;
7842 }
c0c050c5
MC		 7843 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7844 *max_rx >>= 1;
b72d4a68 7845 *max_rx = min_t(int, *max_rx, max_ring_grps);
6e6c5a57
MC		 7846}
7847
7848int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
7849{
7850 int rx, tx, cp;
7851
7852 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
7853 if (!rx || !tx || !cp)
7854 return -ENOMEM;
7855
7856 *max_rx = rx;
7857 *max_tx = tx;
7858 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
7859}
7860
e4060d30
MC		 7861static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7862 bool shared)
7863{
7864 int rc;
7865
7866 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
bdbd1eb5
MC		 7867 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
7868 /* Not enough rings, try disabling agg rings. */
7869 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7870 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
7871 if (rc)
7872 return rc;
7873 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7874 bp->dev->hw_features &= ~NETIF_F_LRO;
7875 bp->dev->features &= ~NETIF_F_LRO;
7876 bnxt_set_ring_params(bp);
7877 }
e4060d30
MC		 7878
7879 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
7880 int max_cp, max_stat, max_irq;
7881
7882 /* Reserve minimum resources for RoCE */
7883 max_cp = bnxt_get_max_func_cp_rings(bp);
7884 max_stat = bnxt_get_max_func_stat_ctxs(bp);
7885 max_irq = bnxt_get_max_func_irqs(bp);
7886 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
7887 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
7888 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
7889 return 0;
7890
7891 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
7892 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
7893 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
7894 max_cp = min_t(int, max_cp, max_irq);
7895 max_cp = min_t(int, max_cp, max_stat);
7896 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
7897 if (rc)
7898 rc = 0;
7899 }
7900 return rc;
7901}
7902
702c221c 7903static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
6e6c5a57
MC		 7904{
7905 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6e6c5a57
MC		 7906
7907 if (sh)
7908 bp->flags |= BNXT_FLAG_SHARED_RINGS;
7909 dflt_rings = netif_get_num_default_rss_queues();
d5430d31
MC		 7910 /* Reduce default rings to reduce memory usage on multi-port cards */
7911 if (bp->port_count > 1)
7912 dflt_rings = min_t(int, dflt_rings, 4);
e4060d30 7913 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6e6c5a57
MC		 7914 if (rc)
7915 return rc;
7916 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
7917 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
391be5c2
MC		 7918
7919 rc = bnxt_hwrm_reserve_tx_rings(bp, &bp->tx_nr_rings_per_tc);
7920 if (rc)
7921 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
7922
6e6c5a57
MC		 7923 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7924 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7925 bp->tx_nr_rings + bp->rx_nr_rings;
7926 bp->num_stat_ctxs = bp->cp_nr_rings;
76595193
PS		 7927 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7928 bp->rx_nr_rings++;
7929 bp->cp_nr_rings++;
7930 }
6e6c5a57 7931 return rc;
c0c050c5
MC		 7932}
7933
7b08f661
MC		 7934void bnxt_restore_pf_fw_resources(struct bnxt *bp)
7935{
7936 ASSERT_RTNL();
7937 bnxt_hwrm_func_qcaps(bp);
a588e458 7938 bnxt_subtract_ulp_resources(bp, BNXT_ROCE_ULP);
7b08f661
MC		 7939}
7940
a22a6ac2
MC		 7941static int bnxt_init_mac_addr(struct bnxt *bp)
7942{
7943 int rc = 0;
7944
7945 if (BNXT_PF(bp)) {
7946 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
7947 } else {
7948#ifdef CONFIG_BNXT_SRIOV
7949 struct bnxt_vf_info *vf = &bp->vf;
7950
7951 if (is_valid_ether_addr(vf->mac_addr)) {
7952 /* overwrite netdev dev_adr with admin VF MAC */
7953 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
7954 } else {
7955 eth_hw_addr_random(bp->dev);
7956 rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
7957 }
7958#endif
7959 }
7960 return rc;
7961}
7962
90c4f788
AK		 7963static void bnxt_parse_log_pcie_link(struct bnxt *bp)
7964{
7965 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
7966 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
7967
7ab0760f 7968 if (pcie_get_minimum_link(pci_physfn(bp->pdev), &speed, &width) ||
90c4f788
AK		 7969 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
7970 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
7971 else
7972 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
7973 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
7974 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
7975 speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
7976 "Unknown", width);
7977}
7978
c0c050c5
MC		 7979static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
7980{
7981 static int version_printed;
7982 struct net_device *dev;
7983 struct bnxt *bp;
6e6c5a57 7984 int rc, max_irqs;
c0c050c5 7985
4e00338a 7986 if (pci_is_bridge(pdev))
fa853dda
PS		 7987 return -ENODEV;
7988
c0c050c5
MC		 7989 if (version_printed++ == 0)
7990 pr_info("%s", version);
7991
7992 max_irqs = bnxt_get_max_irq(pdev);
7993 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
7994 if (!dev)
7995 return -ENOMEM;
7996
7997 bp = netdev_priv(dev);
7998
7999 if (bnxt_vf_pciid(ent->driver_data))
8000 bp->flags |= BNXT_FLAG_VF;
8001
2bcfa6f6 8002 if (pdev->msix_cap)
c0c050c5 8003 bp->flags |= BNXT_FLAG_MSIX_CAP;
c0c050c5
MC		 8004
8005 rc = bnxt_init_board(pdev, dev);
8006 if (rc < 0)
8007 goto init_err_free;
8008
8009 dev->netdev_ops = &bnxt_netdev_ops;
8010 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
8011 dev->ethtool_ops = &bnxt_ethtool_ops;
bc88055a 8012 SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
c0c050c5
MC		 8013 pci_set_drvdata(pdev, dev);
8014
3e8060fa
PS		 8015 rc = bnxt_alloc_hwrm_resources(bp);
8016 if (rc)
17086399 8017 goto init_err_pci_clean;
3e8060fa
PS		 8018
8019 mutex_init(&bp->hwrm_cmd_lock);
8020 rc = bnxt_hwrm_ver_get(bp);
8021 if (rc)
17086399 8022 goto init_err_pci_clean;
3e8060fa 8023
e605db80
DK		 8024 if (bp->flags & BNXT_FLAG_SHORT_CMD) {
8025 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
8026 if (rc)
8027 goto init_err_pci_clean;
8028 }
8029
3c2217a6
MC		 8030 rc = bnxt_hwrm_func_reset(bp);
8031 if (rc)
8032 goto init_err_pci_clean;
8033
5ac67d8b
RS		 8034 bnxt_hwrm_fw_set_time(bp);
8035
c0c050c5
MC		 8036 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8037 NETIF_F_TSO | NETIF_F_TSO6 |
8038 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
7e13318d 8039 NETIF_F_GSO_IPXIP4 |
152971ee
AD		 8040 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8041 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
3e8060fa
PS		 8042 NETIF_F_RXCSUM | NETIF_F_GRO;
8043
8044 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
8045 dev->hw_features |= NETIF_F_LRO;
c0c050c5 8046
c0c050c5
MC		 8047 dev->hw_enc_features =
8048 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8049 NETIF_F_TSO | NETIF_F_TSO6 |
8050 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
152971ee 8051 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
7e13318d 8052 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
152971ee
AD		 8053 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
8054 NETIF_F_GSO_GRE_CSUM;
c0c050c5
MC		 8055 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
8056 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
8057 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
8058 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
8059 dev->priv_flags |= IFF_UNICAST_FLT;
8060
e1c6dcca
JW		 8061 /* MTU range: 60 - 9500 */
8062 dev->min_mtu = ETH_ZLEN;
c61fb99c 8063 dev->max_mtu = BNXT_MAX_MTU;
e1c6dcca 8064
c0c050c5
MC		 8065#ifdef CONFIG_BNXT_SRIOV
8066 init_waitqueue_head(&bp->sriov_cfg_wait);
4ab0c6a8 8067 mutex_init(&bp->sriov_lock);
c0c050c5 8068#endif
309369c9 8069 bp->gro_func = bnxt_gro_func_5730x;
3284f9e1 8070 if (BNXT_CHIP_P4_PLUS(bp))
94758f8d 8071 bp->gro_func = bnxt_gro_func_5731x;
434c975a
MC		 8072 else
8073 bp->flags |= BNXT_FLAG_DOUBLE_DB;
309369c9 8074
c0c050c5
MC		 8075 rc = bnxt_hwrm_func_drv_rgtr(bp);
8076 if (rc)
17086399 8077 goto init_err_pci_clean;
c0c050c5 8078
a1653b13
MC		 8079 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
8080 if (rc)
17086399 8081 goto init_err_pci_clean;
a1653b13 8082
a588e458
MC		 8083 bp->ulp_probe = bnxt_ulp_probe;
8084
c0c050c5
MC		 8085 /* Get the MAX capabilities for this function */
8086 rc = bnxt_hwrm_func_qcaps(bp);
8087 if (rc) {
8088 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
8089 rc);
8090 rc = -1;
17086399 8091 goto init_err_pci_clean;
c0c050c5 8092 }
a22a6ac2
MC		 8093 rc = bnxt_init_mac_addr(bp);
8094 if (rc) {
8095 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
8096 rc = -EADDRNOTAVAIL;
8097 goto init_err_pci_clean;
8098 }
c0c050c5
MC		 8099 rc = bnxt_hwrm_queue_qportcfg(bp);
8100 if (rc) {
8101 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
8102 rc);
8103 rc = -1;
17086399 8104 goto init_err_pci_clean;
c0c050c5
MC		 8105 }
8106
567b2abe 8107 bnxt_hwrm_func_qcfg(bp);
5ad2cbee 8108 bnxt_hwrm_port_led_qcaps(bp);
eb513658 8109 bnxt_ethtool_init(bp);
87fe6032 8110 bnxt_dcb_init(bp);
567b2abe 8111
d5430d31
MC		 8112 rc = bnxt_probe_phy(bp);
8113 if (rc)
8114 goto init_err_pci_clean;
8115
c61fb99c 8116 bnxt_set_rx_skb_mode(bp, false);
c0c050c5
MC		 8117 bnxt_set_tpa_flags(bp);
8118 bnxt_set_ring_params(bp);
33c2657e 8119 bnxt_set_max_func_irqs(bp, max_irqs);
702c221c 8120 rc = bnxt_set_dflt_rings(bp, true);
bdbd1eb5
MC		 8121 if (rc) {
8122 netdev_err(bp->dev, "Not enough rings available.\n");
8123 rc = -ENOMEM;
17086399 8124 goto init_err_pci_clean;
bdbd1eb5 8125 }
c0c050c5 8126
87da7f79
MC		 8127 /* Default RSS hash cfg. */
8128 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
8129 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
8130 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
8131 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
3284f9e1 8132 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
87da7f79
MC		 8133 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
8134 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
8135 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
8136 }
8137
8fdefd63 8138 bnxt_hwrm_vnic_qcaps(bp);
8079e8f1 8139 if (bnxt_rfs_supported(bp)) {
2bcfa6f6
MC		 8140 dev->hw_features |= NETIF_F_NTUPLE;
8141 if (bnxt_rfs_capable(bp)) {
8142 bp->flags |= BNXT_FLAG_RFS;
8143 dev->features |= NETIF_F_NTUPLE;
8144 }
8145 }
8146
c0c050c5
MC		 8147 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
8148 bp->flags |= BNXT_FLAG_STRIP_VLAN;
8149
7809592d 8150 rc = bnxt_init_int_mode(bp);
c0c050c5 8151 if (rc)
17086399 8152 goto init_err_pci_clean;
c0c050c5 8153
c1ef146a 8154 bnxt_get_wol_settings(bp);
d196ece7
MC		 8155 if (bp->flags & BNXT_FLAG_WOL_CAP)
8156 device_set_wakeup_enable(&pdev->dev, bp->wol);
8157 else
8158 device_set_wakeup_capable(&pdev->dev, false);
c1ef146a 8159
c213eae8
MC		 8160 if (BNXT_PF(bp)) {
8161 if (!bnxt_pf_wq) {
8162 bnxt_pf_wq =
8163 create_singlethread_workqueue("bnxt_pf_wq");
8164 if (!bnxt_pf_wq) {
8165 dev_err(&pdev->dev, "Unable to create workqueue.\n");
8166 goto init_err_pci_clean;
8167 }
8168 }
2ae7408f 8169 bnxt_init_tc(bp);
c213eae8 8170 }
2ae7408f 8171
7809592d
MC		 8172 rc = register_netdev(dev);
8173 if (rc)
2ae7408f 8174 goto init_err_cleanup_tc;
7809592d 8175
4ab0c6a8
SP		 8176 if (BNXT_PF(bp))
8177 bnxt_dl_register(bp);
8178
c0c050c5
MC		 8179 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
8180 board_info[ent->driver_data].name,
8181 (long)pci_resource_start(pdev, 0), dev->dev_addr);
8182
90c4f788
AK		 8183 bnxt_parse_log_pcie_link(bp);
8184
c0c050c5
MC		 8185 return 0;
8186
2ae7408f
SP		 8187init_err_cleanup_tc:
8188 bnxt_shutdown_tc(bp);
7809592d
MC		 8189 bnxt_clear_int_mode(bp);
8190
17086399
SP		 8191init_err_pci_clean:
8192 bnxt_cleanup_pci(bp);
c0c050c5
MC		 8193
8194init_err_free:
8195 free_netdev(dev);
8196 return rc;
8197}
8198
d196ece7
MC		 8199static void bnxt_shutdown(struct pci_dev *pdev)
8200{
8201 struct net_device *dev = pci_get_drvdata(pdev);
8202 struct bnxt *bp;
8203
8204 if (!dev)
8205 return;
8206
8207 rtnl_lock();
8208 bp = netdev_priv(dev);
8209 if (!bp)
8210 goto shutdown_exit;
8211
8212 if (netif_running(dev))
8213 dev_close(dev);
8214
8215 if (system_state == SYSTEM_POWER_OFF) {
0efd2fc6 8216 bnxt_ulp_shutdown(bp);
d196ece7
MC		 8217 bnxt_clear_int_mode(bp);
8218 pci_wake_from_d3(pdev, bp->wol);
8219 pci_set_power_state(pdev, PCI_D3hot);
8220 }
8221
8222shutdown_exit:
8223 rtnl_unlock();
8224}
8225
f65a2044
MC		 8226#ifdef CONFIG_PM_SLEEP
8227static int bnxt_suspend(struct device *device)
8228{
8229 struct pci_dev *pdev = to_pci_dev(device);
8230 struct net_device *dev = pci_get_drvdata(pdev);
8231 struct bnxt *bp = netdev_priv(dev);
8232 int rc = 0;
8233
8234 rtnl_lock();
8235 if (netif_running(dev)) {
8236 netif_device_detach(dev);
8237 rc = bnxt_close(dev);
8238 }
8239 bnxt_hwrm_func_drv_unrgtr(bp);
8240 rtnl_unlock();
8241 return rc;
8242}
8243
8244static int bnxt_resume(struct device *device)
8245{
8246 struct pci_dev *pdev = to_pci_dev(device);
8247 struct net_device *dev = pci_get_drvdata(pdev);
8248 struct bnxt *bp = netdev_priv(dev);
8249 int rc = 0;
8250
8251 rtnl_lock();
8252 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
8253 rc = -ENODEV;
8254 goto resume_exit;
8255 }
8256 rc = bnxt_hwrm_func_reset(bp);
8257 if (rc) {
8258 rc = -EBUSY;
8259 goto resume_exit;
8260 }
8261 bnxt_get_wol_settings(bp);
8262 if (netif_running(dev)) {
8263 rc = bnxt_open(dev);
8264 if (!rc)
8265 netif_device_attach(dev);
8266 }
8267
8268resume_exit:
8269 rtnl_unlock();
8270 return rc;
8271}
8272
8273static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
8274#define BNXT_PM_OPS (&bnxt_pm_ops)
8275
8276#else
8277
8278#define BNXT_PM_OPS NULL
8279
8280#endif /* CONFIG_PM_SLEEP */
8281
6316ea6d
SB		 8282/**
8283 * bnxt_io_error_detected - called when PCI error is detected
8284 * @pdev: Pointer to PCI device
8285 * @state: The current pci connection state
8286 *
8287 * This function is called after a PCI bus error affecting
8288 * this device has been detected.
8289 */
8290static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
8291 pci_channel_state_t state)
8292{
8293 struct net_device *netdev = pci_get_drvdata(pdev);
a588e458 8294 struct bnxt *bp = netdev_priv(netdev);
6316ea6d
SB		 8295
8296 netdev_info(netdev, "PCI I/O error detected\n");
8297
8298 rtnl_lock();
8299 netif_device_detach(netdev);
8300
a588e458
MC		 8301 bnxt_ulp_stop(bp);
8302
6316ea6d
SB		 8303 if (state == pci_channel_io_perm_failure) {
8304 rtnl_unlock();
8305 return PCI_ERS_RESULT_DISCONNECT;
8306 }
8307
8308 if (netif_running(netdev))
8309 bnxt_close(netdev);
8310
8311 pci_disable_device(pdev);
8312 rtnl_unlock();
8313
8314 /* Request a slot slot reset. */
8315 return PCI_ERS_RESULT_NEED_RESET;
8316}
8317
8318/**
8319 * bnxt_io_slot_reset - called after the pci bus has been reset.
8320 * @pdev: Pointer to PCI device
8321 *
8322 * Restart the card from scratch, as if from a cold-boot.
8323 * At this point, the card has exprienced a hard reset,
8324 * followed by fixups by BIOS, and has its config space
8325 * set up identically to what it was at cold boot.
8326 */
8327static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
8328{
8329 struct net_device *netdev = pci_get_drvdata(pdev);
8330 struct bnxt *bp = netdev_priv(netdev);
8331 int err = 0;
8332 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
8333
8334 netdev_info(bp->dev, "PCI Slot Reset\n");
8335
8336 rtnl_lock();
8337
8338 if (pci_enable_device(pdev)) {
8339 dev_err(&pdev->dev,
8340 "Cannot re-enable PCI device after reset.\n");
8341 } else {
8342 pci_set_master(pdev);
8343
aa8ed021
MC		 8344 err = bnxt_hwrm_func_reset(bp);
8345 if (!err && netif_running(netdev))
6316ea6d
SB		 8346 err = bnxt_open(netdev);
8347
a588e458 8348 if (!err) {
6316ea6d 8349 result = PCI_ERS_RESULT_RECOVERED;
a588e458
MC		 8350 bnxt_ulp_start(bp);
8351 }
6316ea6d
SB		 8352 }
8353
8354 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
8355 dev_close(netdev);
8356
8357 rtnl_unlock();
8358
8359 err = pci_cleanup_aer_uncorrect_error_status(pdev);
8360 if (err) {
8361 dev_err(&pdev->dev,
8362 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8363 err); /* non-fatal, continue */
8364 }
8365
8366 return PCI_ERS_RESULT_RECOVERED;
8367}
8368
8369/**
8370 * bnxt_io_resume - called when traffic can start flowing again.
8371 * @pdev: Pointer to PCI device
8372 *
8373 * This callback is called when the error recovery driver tells
8374 * us that its OK to resume normal operation.
8375 */
8376static void bnxt_io_resume(struct pci_dev *pdev)
8377{
8378 struct net_device *netdev = pci_get_drvdata(pdev);
8379
8380 rtnl_lock();
8381
8382 netif_device_attach(netdev);
8383
8384 rtnl_unlock();
8385}
8386
8387static const struct pci_error_handlers bnxt_err_handler = {
8388 .error_detected = bnxt_io_error_detected,
8389 .slot_reset = bnxt_io_slot_reset,
8390 .resume = bnxt_io_resume
8391};
8392
c0c050c5
MC		 8393static struct pci_driver bnxt_pci_driver = {
8394 .name = DRV_MODULE_NAME,
8395 .id_table = bnxt_pci_tbl,
8396 .probe = bnxt_init_one,
8397 .remove = bnxt_remove_one,
d196ece7 8398 .shutdown = bnxt_shutdown,
f65a2044 8399 .driver.pm = BNXT_PM_OPS,
6316ea6d 8400 .err_handler = &bnxt_err_handler,
c0c050c5
MC		 8401#if defined(CONFIG_BNXT_SRIOV)
8402 .sriov_configure = bnxt_sriov_configure,
8403#endif
8404};
8405
c213eae8
MC		 8406static int __init bnxt_init(void)
8407{
8408 return pci_register_driver(&bnxt_pci_driver);
8409}
8410
8411static void __exit bnxt_exit(void)
8412{
8413 pci_unregister_driver(&bnxt_pci_driver);
8414 if (bnxt_pf_wq)
8415 destroy_workqueue(bnxt_pf_wq);
8416}
8417
8418module_init(bnxt_init);
8419module_exit(bnxt_exit);
Ubuntu Jammy Linux kernel mirror