]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/net/ethernet/broadcom/bnxt/bnxt.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnxt_en: Add fast path logic for TPA on 57500 chips.
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
CommitLineData
c0c050c5
MC		 1/* Broadcom NetXtreme-C/E network driver.
2 *
11f15ed3 3 * Copyright (c) 2014-2016 Broadcom Corporation
c6cc32a2 4 * Copyright (c) 2016-2019 Broadcom Limited
c0c050c5
MC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11#include <linux/module.h>
12
13#include <linux/stringify.h>
14#include <linux/kernel.h>
15#include <linux/timer.h>
16#include <linux/errno.h>
17#include <linux/ioport.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20#include <linux/interrupt.h>
21#include <linux/pci.h>
22#include <linux/netdevice.h>
23#include <linux/etherdevice.h>
24#include <linux/skbuff.h>
25#include <linux/dma-mapping.h>
26#include <linux/bitops.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
30#include <asm/byteorder.h>
31#include <asm/page.h>
32#include <linux/time.h>
33#include <linux/mii.h>
0ca12be9 34#include <linux/mdio.h>
c0c050c5
MC		 35#include <linux/if.h>
36#include <linux/if_vlan.h>
32e8239c 37#include <linux/if_bridge.h>
5ac67d8b 38#include <linux/rtc.h>
c6d30e83 39#include <linux/bpf.h>
c0c050c5
MC		 40#include <net/ip.h>
41#include <net/tcp.h>
42#include <net/udp.h>
43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
ad51b8e9 45#include <net/udp_tunnel.h>
c0c050c5
MC		 46#include <linux/workqueue.h>
47#include <linux/prefetch.h>
48#include <linux/cache.h>
49#include <linux/log2.h>
50#include <linux/aer.h>
51#include <linux/bitmap.h>
52#include <linux/cpu_rmap.h>
56f0fd80 53#include <linux/cpumask.h>
2ae7408f 54#include <net/pkt_cls.h>
cde49a42
VV		 55#include <linux/hwmon.h>
56#include <linux/hwmon-sysfs.h>
322b87ca 57#include <net/page_pool.h>
c0c050c5
MC		 58
59#include "bnxt_hsi.h"
60#include "bnxt.h"
a588e458 61#include "bnxt_ulp.h"
c0c050c5
MC		 62#include "bnxt_sriov.h"
63#include "bnxt_ethtool.h"
7df4ae9f 64#include "bnxt_dcb.h"
c6d30e83 65#include "bnxt_xdp.h"
4ab0c6a8 66#include "bnxt_vfr.h"
2ae7408f 67#include "bnxt_tc.h"
3c467bf3 68#include "bnxt_devlink.h"
cabfb09d 69#include "bnxt_debugfs.h"
c0c050c5
MC		 70
71#define BNXT_TX_TIMEOUT (5 * HZ)
72
73static const char version[] =
74 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
75
76MODULE_LICENSE("GPL");
77MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
78MODULE_VERSION(DRV_MODULE_VERSION);
79
80#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
81#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
82#define BNXT_RX_COPY_THRESH 256
83
4419dbe6 84#define BNXT_TX_PUSH_THRESH 164
c0c050c5
MC		 85
86enum board_idx {
fbc9a523 87 BCM57301,
c0c050c5
MC		 88 BCM57302,
89 BCM57304,
1f681688 90 BCM57417_NPAR,
fa853dda 91 BCM58700,
b24eb6ae
MC		 92 BCM57311,
93 BCM57312,
fbc9a523 94 BCM57402,
c0c050c5
MC		 95 BCM57404,
96 BCM57406,
1f681688
MC		 97 BCM57402_NPAR,
98 BCM57407,
b24eb6ae
MC		 99 BCM57412,
100 BCM57414,
101 BCM57416,
102 BCM57417,
1f681688 103 BCM57412_NPAR,
5049e33b 104 BCM57314,
1f681688
MC		 105 BCM57417_SFP,
106 BCM57416_SFP,
107 BCM57404_NPAR,
108 BCM57406_NPAR,
109 BCM57407_SFP,
adbc8305 110 BCM57407_NPAR,
1f681688
MC		 111 BCM57414_NPAR,
112 BCM57416_NPAR,
32b40798
DK		 113 BCM57452,
114 BCM57454,
92abef36 115 BCM5745x_NPAR,
1ab968d2 116 BCM57508,
c6cc32a2 117 BCM57504,
51fec80d 118 BCM57502,
4a58139b 119 BCM58802,
8ed693b7 120 BCM58804,
4a58139b 121 BCM58808,
adbc8305
MC		 122 NETXTREME_E_VF,
123 NETXTREME_C_VF,
618784e3 124 NETXTREME_S_VF,
b16b6891 125 NETXTREME_E_P5_VF,
c0c050c5
MC		 126};
127
128/* indexed by enum above */
129static const struct {
130 char *name;
131} board_info[] = {
27573a7d
SB		 132 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
133 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
134 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
135 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
136 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
137 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
138 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
139 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
140 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
141 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
142 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
143 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
144 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
145 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
146 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
147 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
148 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
149 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
150 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
151 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
152 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
153 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
154 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
155 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
156 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
157 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
158 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
159 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
92abef36 160 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
1ab968d2 161 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
c6cc32a2 162 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
51fec80d 163 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
27573a7d 164 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
8ed693b7 165 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
27573a7d
SB		 166 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
167 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
168 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
618784e3 169 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
b16b6891 170 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
c0c050c5
MC		 171};
172
173static const struct pci_device_id bnxt_pci_tbl[] = {
92abef36
VV		 174 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
175 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
4a58139b 176 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
adbc8305 177 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
fbc9a523 178 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
c0c050c5
MC		 179 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
180 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
1f681688 181 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
fa853dda 182 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
b24eb6ae
MC		 183 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
184 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
fbc9a523 185 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
c0c050c5
MC		 186 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
187 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
1f681688
MC		 188 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
189 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
b24eb6ae
MC		 190 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
191 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
192 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
193 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
1f681688 194 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
5049e33b 195 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
1f681688
MC		 196 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
197 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
198 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
199 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
200 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
adbc8305
MC		 201 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
202 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
1f681688 203 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
adbc8305 204 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
1f681688 205 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
adbc8305 206 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
4a58139b 207 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
32b40798 208 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
1ab968d2 209 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
c6cc32a2 210 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
51fec80d 211 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
4a58139b 212 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
8ed693b7 213 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
c0c050c5 214#ifdef CONFIG_BNXT_SRIOV
c7ef35eb
DK		 215 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
216 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
adbc8305
MC		 217 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
218 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
219 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
220 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
221 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
222 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
51fec80d 223 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
b16b6891 224 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
618784e3 225 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
c0c050c5
MC		 226#endif
227 { 0 }
228};
229
230MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
231
232static const u16 bnxt_vf_req_snif[] = {
233 HWRM_FUNC_CFG,
91cdda40 234 HWRM_FUNC_VF_CFG,
c0c050c5
MC		 235 HWRM_PORT_PHY_QCFG,
236 HWRM_CFA_L2_FILTER_ALLOC,
237};
238
25be8623 239static const u16 bnxt_async_events_arr[] = {
87c374de
MC		 240 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
241 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
242 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
243 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
244 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
25be8623
MC		 245};
246
c213eae8
MC		 247static struct workqueue_struct *bnxt_pf_wq;
248
c0c050c5
MC		 249static bool bnxt_vf_pciid(enum board_idx idx)
250{
618784e3 251 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
b16b6891 252 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
c0c050c5
MC		 253}
254
255#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
256#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
257#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
258
c0c050c5
MC		 259#define BNXT_CP_DB_IRQ_DIS(db) \
260 writel(DB_CP_IRQ_DIS_FLAGS, db)
261
697197e5
MC		 262#define BNXT_DB_CQ(db, idx) \
263 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
264
265#define BNXT_DB_NQ_P5(db, idx) \
266 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
267
268#define BNXT_DB_CQ_ARM(db, idx) \
269 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
270
271#define BNXT_DB_NQ_ARM_P5(db, idx) \
272 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
273
274static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
275{
276 if (bp->flags & BNXT_FLAG_CHIP_P5)
277 BNXT_DB_NQ_P5(db, idx);
278 else
279 BNXT_DB_CQ(db, idx);
280}
281
282static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
283{
284 if (bp->flags & BNXT_FLAG_CHIP_P5)
285 BNXT_DB_NQ_ARM_P5(db, idx);
286 else
287 BNXT_DB_CQ_ARM(db, idx);
288}
289
290static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
291{
292 if (bp->flags & BNXT_FLAG_CHIP_P5)
293 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
294 db->doorbell);
295 else
296 BNXT_DB_CQ(db, idx);
297}
298
38413406 299const u16 bnxt_lhint_arr[] = {
c0c050c5
MC		 300 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
301 TX_BD_FLAGS_LHINT_512_TO_1023,
302 TX_BD_FLAGS_LHINT_1024_TO_2047,
303 TX_BD_FLAGS_LHINT_1024_TO_2047,
304 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
305 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
306 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
307 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
308 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
309 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
310 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
311 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
312 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
313 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
314 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
315 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
316 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
317 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
318 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
319};
320
ee5c7fb3
SP		 321static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
322{
323 struct metadata_dst *md_dst = skb_metadata_dst(skb);
324
325 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
326 return 0;
327
328 return md_dst->u.port_info.port_id;
329}
330
c0c050c5
MC		 331static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
332{
333 struct bnxt *bp = netdev_priv(dev);
334 struct tx_bd *txbd;
335 struct tx_bd_ext *txbd1;
336 struct netdev_queue *txq;
337 int i;
338 dma_addr_t mapping;
339 unsigned int length, pad = 0;
340 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
341 u16 prod, last_frag;
342 struct pci_dev *pdev = bp->pdev;
c0c050c5
MC		 343 struct bnxt_tx_ring_info *txr;
344 struct bnxt_sw_tx_bd *tx_buf;
345
346 i = skb_get_queue_mapping(skb);
347 if (unlikely(i >= bp->tx_nr_rings)) {
348 dev_kfree_skb_any(skb);
349 return NETDEV_TX_OK;
350 }
351
c0c050c5 352 txq = netdev_get_tx_queue(dev, i);
a960dec9 353 txr = &bp->tx_ring[bp->tx_ring_map[i]];
c0c050c5
MC		 354 prod = txr->tx_prod;
355
356 free_size = bnxt_tx_avail(bp, txr);
357 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
358 netif_tx_stop_queue(txq);
359 return NETDEV_TX_BUSY;
360 }
361
362 length = skb->len;
363 len = skb_headlen(skb);
364 last_frag = skb_shinfo(skb)->nr_frags;
365
366 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
367
368 txbd->tx_bd_opaque = prod;
369
370 tx_buf = &txr->tx_buf_ring[prod];
371 tx_buf->skb = skb;
372 tx_buf->nr_frags = last_frag;
373
374 vlan_tag_flags = 0;
ee5c7fb3 375 cfa_action = bnxt_xmit_get_cfa_action(skb);
c0c050c5
MC		 376 if (skb_vlan_tag_present(skb)) {
377 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
378 skb_vlan_tag_get(skb);
379 /* Currently supports 8021Q, 8021AD vlan offloads
380 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
381 */
382 if (skb->vlan_proto == htons(ETH_P_8021Q))
383 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
384 }
385
386 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
4419dbe6
MC		 387 struct tx_push_buffer *tx_push_buf = txr->tx_push;
388 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
389 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
697197e5 390 void __iomem *db = txr->tx_db.doorbell;
4419dbe6
MC		 391 void *pdata = tx_push_buf->data;
392 u64 *end;
393 int j, push_len;
c0c050c5
MC		 394
395 /* Set COAL_NOW to be ready quickly for the next push */
396 tx_push->tx_bd_len_flags_type =
397 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
398 TX_BD_TYPE_LONG_TX_BD |
399 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
400 TX_BD_FLAGS_COAL_NOW |
401 TX_BD_FLAGS_PACKET_END |
402 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
403
404 if (skb->ip_summed == CHECKSUM_PARTIAL)
405 tx_push1->tx_bd_hsize_lflags =
406 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
407 else
408 tx_push1->tx_bd_hsize_lflags = 0;
409
410 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 411 tx_push1->tx_bd_cfa_action =
412 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5 413
fbb0fa8b
MC		 414 end = pdata + length;
415 end = PTR_ALIGN(end, 8) - 1;
4419dbe6
MC		 416 *end = 0;
417
c0c050c5
MC		 418 skb_copy_from_linear_data(skb, pdata, len);
419 pdata += len;
420 for (j = 0; j < last_frag; j++) {
421 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
422 void *fptr;
423
424 fptr = skb_frag_address_safe(frag);
425 if (!fptr)
426 goto normal_tx;
427
428 memcpy(pdata, fptr, skb_frag_size(frag));
429 pdata += skb_frag_size(frag);
430 }
431
4419dbe6
MC		 432 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
433 txbd->tx_bd_haddr = txr->data_mapping;
c0c050c5
MC		 434 prod = NEXT_TX(prod);
435 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
436 memcpy(txbd, tx_push1, sizeof(*txbd));
437 prod = NEXT_TX(prod);
4419dbe6 438 tx_push->doorbell =
c0c050c5
MC		 439 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
440 txr->tx_prod = prod;
441
b9a8460a 442 tx_buf->is_push = 1;
c0c050c5 443 netdev_tx_sent_queue(txq, skb->len);
b9a8460a 444 wmb(); /* Sync is_push and byte queue before pushing data */
c0c050c5 445
4419dbe6
MC		 446 push_len = (length + sizeof(*tx_push) + 7) / 8;
447 if (push_len > 16) {
697197e5
MC		 448 __iowrite64_copy(db, tx_push_buf, 16);
449 __iowrite32_copy(db + 4, tx_push_buf + 1,
9d13744b 450 (push_len - 16) << 1);
4419dbe6 451 } else {
697197e5 452 __iowrite64_copy(db, tx_push_buf, push_len);
4419dbe6 453 }
c0c050c5 454
c0c050c5
MC		 455 goto tx_done;
456 }
457
458normal_tx:
459 if (length < BNXT_MIN_PKT_SIZE) {
460 pad = BNXT_MIN_PKT_SIZE - length;
461 if (skb_pad(skb, pad)) {
462 /* SKB already freed. */
463 tx_buf->skb = NULL;
464 return NETDEV_TX_OK;
465 }
466 length = BNXT_MIN_PKT_SIZE;
467 }
468
469 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
470
471 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
472 dev_kfree_skb_any(skb);
473 tx_buf->skb = NULL;
474 return NETDEV_TX_OK;
475 }
476
477 dma_unmap_addr_set(tx_buf, mapping, mapping);
478 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
479 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
480
481 txbd->tx_bd_haddr = cpu_to_le64(mapping);
482
483 prod = NEXT_TX(prod);
484 txbd1 = (struct tx_bd_ext *)
485 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
486
487 txbd1->tx_bd_hsize_lflags = 0;
488 if (skb_is_gso(skb)) {
489 u32 hdr_len;
490
491 if (skb->encapsulation)
492 hdr_len = skb_inner_network_offset(skb) +
493 skb_inner_network_header_len(skb) +
494 inner_tcp_hdrlen(skb);
495 else
496 hdr_len = skb_transport_offset(skb) +
497 tcp_hdrlen(skb);
498
499 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
500 TX_BD_FLAGS_T_IPID |
501 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
502 length = skb_shinfo(skb)->gso_size;
503 txbd1->tx_bd_mss = cpu_to_le32(length);
504 length += hdr_len;
505 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
506 txbd1->tx_bd_hsize_lflags =
507 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
508 txbd1->tx_bd_mss = 0;
509 }
510
511 length >>= 9;
2b3c6885
MC		 512 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
513 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
514 skb->len);
515 i = 0;
516 goto tx_dma_error;
517 }
c0c050c5
MC		 518 flags |= bnxt_lhint_arr[length];
519 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
520
521 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 522 txbd1->tx_bd_cfa_action =
523 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5
MC		 524 for (i = 0; i < last_frag; i++) {
525 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
526
527 prod = NEXT_TX(prod);
528 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
529
530 len = skb_frag_size(frag);
531 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
532 DMA_TO_DEVICE);
533
534 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
535 goto tx_dma_error;
536
537 tx_buf = &txr->tx_buf_ring[prod];
538 dma_unmap_addr_set(tx_buf, mapping, mapping);
539
540 txbd->tx_bd_haddr = cpu_to_le64(mapping);
541
542 flags = len << TX_BD_LEN_SHIFT;
543 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
544 }
545
546 flags &= ~TX_BD_LEN;
547 txbd->tx_bd_len_flags_type =
548 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
549 TX_BD_FLAGS_PACKET_END);
550
551 netdev_tx_sent_queue(txq, skb->len);
552
553 /* Sync BD data before updating doorbell */
554 wmb();
555
556 prod = NEXT_TX(prod);
557 txr->tx_prod = prod;
558
6b16f9ee 559 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
697197e5 560 bnxt_db_write(bp, &txr->tx_db, prod);
c0c050c5
MC		 561
562tx_done:
563
c0c050c5 564 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
6b16f9ee 565 if (netdev_xmit_more() && !tx_buf->is_push)
697197e5 566 bnxt_db_write(bp, &txr->tx_db, prod);
4d172f21 567
c0c050c5
MC		 568 netif_tx_stop_queue(txq);
569
570 /* netif_tx_stop_queue() must be done before checking
571 * tx index in bnxt_tx_avail() below, because in
572 * bnxt_tx_int(), we update tx index before checking for
573 * netif_tx_queue_stopped().
574 */
575 smp_mb();
576 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
577 netif_tx_wake_queue(txq);
578 }
579 return NETDEV_TX_OK;
580
581tx_dma_error:
582 last_frag = i;
583
584 /* start back at beginning and unmap skb */
585 prod = txr->tx_prod;
586 tx_buf = &txr->tx_buf_ring[prod];
587 tx_buf->skb = NULL;
588 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
589 skb_headlen(skb), PCI_DMA_TODEVICE);
590 prod = NEXT_TX(prod);
591
592 /* unmap remaining mapped pages */
593 for (i = 0; i < last_frag; i++) {
594 prod = NEXT_TX(prod);
595 tx_buf = &txr->tx_buf_ring[prod];
596 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
597 skb_frag_size(&skb_shinfo(skb)->frags[i]),
598 PCI_DMA_TODEVICE);
599 }
600
601 dev_kfree_skb_any(skb);
602 return NETDEV_TX_OK;
603}
604
605static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
606{
b6ab4b01 607 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
a960dec9 608 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
c0c050c5
MC		 609 u16 cons = txr->tx_cons;
610 struct pci_dev *pdev = bp->pdev;
611 int i;
612 unsigned int tx_bytes = 0;
613
614 for (i = 0; i < nr_pkts; i++) {
615 struct bnxt_sw_tx_bd *tx_buf;
616 struct sk_buff *skb;
617 int j, last;
618
619 tx_buf = &txr->tx_buf_ring[cons];
620 cons = NEXT_TX(cons);
621 skb = tx_buf->skb;
622 tx_buf->skb = NULL;
623
624 if (tx_buf->is_push) {
625 tx_buf->is_push = 0;
626 goto next_tx_int;
627 }
628
629 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
630 skb_headlen(skb), PCI_DMA_TODEVICE);
631 last = tx_buf->nr_frags;
632
633 for (j = 0; j < last; j++) {
634 cons = NEXT_TX(cons);
635 tx_buf = &txr->tx_buf_ring[cons];
636 dma_unmap_page(
637 &pdev->dev,
638 dma_unmap_addr(tx_buf, mapping),
639 skb_frag_size(&skb_shinfo(skb)->frags[j]),
640 PCI_DMA_TODEVICE);
641 }
642
643next_tx_int:
644 cons = NEXT_TX(cons);
645
646 tx_bytes += skb->len;
647 dev_kfree_skb_any(skb);
648 }
649
650 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
651 txr->tx_cons = cons;
652
653 /* Need to make the tx_cons update visible to bnxt_start_xmit()
654 * before checking for netif_tx_queue_stopped(). Without the
655 * memory barrier, there is a small possibility that bnxt_start_xmit()
656 * will miss it and cause the queue to be stopped forever.
657 */
658 smp_mb();
659
660 if (unlikely(netif_tx_queue_stopped(txq)) &&
661 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
662 __netif_tx_lock(txq, smp_processor_id());
663 if (netif_tx_queue_stopped(txq) &&
664 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
665 txr->dev_state != BNXT_DEV_STATE_CLOSING)
666 netif_tx_wake_queue(txq);
667 __netif_tx_unlock(txq);
668 }
669}
670
c61fb99c 671static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
322b87ca 672 struct bnxt_rx_ring_info *rxr,
c61fb99c
MC		 673 gfp_t gfp)
674{
675 struct device *dev = &bp->pdev->dev;
676 struct page *page;
677
322b87ca 678 page = page_pool_dev_alloc_pages(rxr->page_pool);
c61fb99c
MC		 679 if (!page)
680 return NULL;
681
c519fe9a
SN		 682 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
683 DMA_ATTR_WEAK_ORDERING);
c61fb99c 684 if (dma_mapping_error(dev, *mapping)) {
322b87ca 685 page_pool_recycle_direct(rxr->page_pool, page);
c61fb99c
MC		 686 return NULL;
687 }
688 *mapping += bp->rx_dma_offset;
689 return page;
690}
691
c0c050c5
MC		 692static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
693 gfp_t gfp)
694{
695 u8 *data;
696 struct pci_dev *pdev = bp->pdev;
697
698 data = kmalloc(bp->rx_buf_size, gfp);
699 if (!data)
700 return NULL;
701
c519fe9a
SN		 702 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
703 bp->rx_buf_use_size, bp->rx_dir,
704 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 705
706 if (dma_mapping_error(&pdev->dev, *mapping)) {
707 kfree(data);
708 data = NULL;
709 }
710 return data;
711}
712
38413406
MC		 713int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
714 u16 prod, gfp_t gfp)
c0c050c5
MC		 715{
716 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
717 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
c0c050c5
MC		 718 dma_addr_t mapping;
719
c61fb99c 720 if (BNXT_RX_PAGE_MODE(bp)) {
322b87ca
AG		 721 struct page *page =
722 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
c0c050c5 723
c61fb99c
MC		 724 if (!page)
725 return -ENOMEM;
726
727 rx_buf->data = page;
728 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
729 } else {
730 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
731
732 if (!data)
733 return -ENOMEM;
734
735 rx_buf->data = data;
736 rx_buf->data_ptr = data + bp->rx_offset;
737 }
11cd119d 738 rx_buf->mapping = mapping;
c0c050c5
MC		 739
740 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
c0c050c5
MC		 741 return 0;
742}
743
c6d30e83 744void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
c0c050c5
MC		 745{
746 u16 prod = rxr->rx_prod;
747 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
748 struct rx_bd *cons_bd, *prod_bd;
749
750 prod_rx_buf = &rxr->rx_buf_ring[prod];
751 cons_rx_buf = &rxr->rx_buf_ring[cons];
752
753 prod_rx_buf->data = data;
6bb19474 754 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 755
11cd119d 756 prod_rx_buf->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 757
758 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
759 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
760
761 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
762}
763
764static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
765{
766 u16 next, max = rxr->rx_agg_bmap_size;
767
768 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
769 if (next >= max)
770 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
771 return next;
772}
773
774static inline int bnxt_alloc_rx_page(struct bnxt *bp,
775 struct bnxt_rx_ring_info *rxr,
776 u16 prod, gfp_t gfp)
777{
778 struct rx_bd *rxbd =
779 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
780 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
781 struct pci_dev *pdev = bp->pdev;
782 struct page *page;
783 dma_addr_t mapping;
784 u16 sw_prod = rxr->rx_sw_agg_prod;
89d0a06c 785 unsigned int offset = 0;
c0c050c5 786
89d0a06c
MC		 787 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
788 page = rxr->rx_page;
789 if (!page) {
790 page = alloc_page(gfp);
791 if (!page)
792 return -ENOMEM;
793 rxr->rx_page = page;
794 rxr->rx_page_offset = 0;
795 }
796 offset = rxr->rx_page_offset;
797 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
798 if (rxr->rx_page_offset == PAGE_SIZE)
799 rxr->rx_page = NULL;
800 else
801 get_page(page);
802 } else {
803 page = alloc_page(gfp);
804 if (!page)
805 return -ENOMEM;
806 }
c0c050c5 807
c519fe9a
SN		 808 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
809 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
810 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 811 if (dma_mapping_error(&pdev->dev, mapping)) {
812 __free_page(page);
813 return -EIO;
814 }
815
816 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
817 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
818
819 __set_bit(sw_prod, rxr->rx_agg_bmap);
820 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
821 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
822
823 rx_agg_buf->page = page;
89d0a06c 824 rx_agg_buf->offset = offset;
c0c050c5
MC		 825 rx_agg_buf->mapping = mapping;
826 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
827 rxbd->rx_bd_opaque = sw_prod;
828 return 0;
829}
830
4a228a3a
MC		 831static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
832 struct bnxt_cp_ring_info *cpr,
833 u16 cp_cons, u16 curr)
834{
835 struct rx_agg_cmp *agg;
836
837 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
838 agg = (struct rx_agg_cmp *)
839 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
840 return agg;
841}
842
bfcd8d79
MC		 843static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
844 struct bnxt_rx_ring_info *rxr,
845 u16 agg_id, u16 curr)
846{
847 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
848
849 return &tpa_info->agg_arr[curr];
850}
851
4a228a3a
MC		 852static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
853 u16 start, u32 agg_bufs, bool tpa)
c0c050c5 854{
e44758b7 855 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5 856 struct bnxt *bp = bnapi->bp;
b6ab4b01 857 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 858 u16 prod = rxr->rx_agg_prod;
859 u16 sw_prod = rxr->rx_sw_agg_prod;
bfcd8d79 860 bool p5_tpa = false;
c0c050c5
MC		 861 u32 i;
862
bfcd8d79
MC		 863 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
864 p5_tpa = true;
865
c0c050c5
MC		 866 for (i = 0; i < agg_bufs; i++) {
867 u16 cons;
868 struct rx_agg_cmp *agg;
869 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
870 struct rx_bd *prod_bd;
871 struct page *page;
872
bfcd8d79
MC		 873 if (p5_tpa)
874 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
875 else
876 agg = bnxt_get_agg(bp, cpr, idx, start + i);
c0c050c5
MC		 877 cons = agg->rx_agg_cmp_opaque;
878 __clear_bit(cons, rxr->rx_agg_bmap);
879
880 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
881 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
882
883 __set_bit(sw_prod, rxr->rx_agg_bmap);
884 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
885 cons_rx_buf = &rxr->rx_agg_ring[cons];
886
887 /* It is possible for sw_prod to be equal to cons, so
888 * set cons_rx_buf->page to NULL first.
889 */
890 page = cons_rx_buf->page;
891 cons_rx_buf->page = NULL;
892 prod_rx_buf->page = page;
89d0a06c 893 prod_rx_buf->offset = cons_rx_buf->offset;
c0c050c5
MC		 894
895 prod_rx_buf->mapping = cons_rx_buf->mapping;
896
897 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
898
899 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
900 prod_bd->rx_bd_opaque = sw_prod;
901
902 prod = NEXT_RX_AGG(prod);
903 sw_prod = NEXT_RX_AGG(sw_prod);
c0c050c5
MC		 904 }
905 rxr->rx_agg_prod = prod;
906 rxr->rx_sw_agg_prod = sw_prod;
907}
908
c61fb99c
MC		 909static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
910 struct bnxt_rx_ring_info *rxr,
911 u16 cons, void *data, u8 *data_ptr,
912 dma_addr_t dma_addr,
913 unsigned int offset_and_len)
914{
915 unsigned int payload = offset_and_len >> 16;
916 unsigned int len = offset_and_len & 0xffff;
d7840976 917 skb_frag_t *frag;
c61fb99c
MC		 918 struct page *page = data;
919 u16 prod = rxr->rx_prod;
920 struct sk_buff *skb;
921 int off, err;
922
923 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
924 if (unlikely(err)) {
925 bnxt_reuse_rx_data(rxr, cons, data);
926 return NULL;
927 }
928 dma_addr -= bp->rx_dma_offset;
c519fe9a
SN		 929 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
930 DMA_ATTR_WEAK_ORDERING);
c61fb99c
MC		 931
932 if (unlikely(!payload))
c43f1255 933 payload = eth_get_headlen(bp->dev, data_ptr, len);
c61fb99c
MC		 934
935 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
936 if (!skb) {
937 __free_page(page);
938 return NULL;
939 }
940
941 off = (void *)data_ptr - page_address(page);
942 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
943 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
944 payload + NET_IP_ALIGN);
945
946 frag = &skb_shinfo(skb)->frags[0];
947 skb_frag_size_sub(frag, payload);
948 frag->page_offset += payload;
949 skb->data_len -= payload;
950 skb->tail += payload;
951
952 return skb;
953}
954
c0c050c5
MC		 955static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
956 struct bnxt_rx_ring_info *rxr, u16 cons,
6bb19474
MC		 957 void *data, u8 *data_ptr,
958 dma_addr_t dma_addr,
959 unsigned int offset_and_len)
c0c050c5 960{
6bb19474 961 u16 prod = rxr->rx_prod;
c0c050c5 962 struct sk_buff *skb;
6bb19474 963 int err;
c0c050c5
MC		 964
965 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
966 if (unlikely(err)) {
967 bnxt_reuse_rx_data(rxr, cons, data);
968 return NULL;
969 }
970
971 skb = build_skb(data, 0);
c519fe9a
SN		 972 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
973 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 974 if (!skb) {
975 kfree(data);
976 return NULL;
977 }
978
b3dba77c 979 skb_reserve(skb, bp->rx_offset);
6bb19474 980 skb_put(skb, offset_and_len & 0xffff);
c0c050c5
MC		 981 return skb;
982}
983
e44758b7
MC		 984static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
985 struct bnxt_cp_ring_info *cpr,
4a228a3a
MC		 986 struct sk_buff *skb, u16 idx,
987 u32 agg_bufs, bool tpa)
c0c050c5 988{
e44758b7 989 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5 990 struct pci_dev *pdev = bp->pdev;
b6ab4b01 991 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 992 u16 prod = rxr->rx_agg_prod;
bfcd8d79 993 bool p5_tpa = false;
c0c050c5
MC		 994 u32 i;
995
bfcd8d79
MC		 996 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
997 p5_tpa = true;
998
c0c050c5
MC		 999 for (i = 0; i < agg_bufs; i++) {
1000 u16 cons, frag_len;
1001 struct rx_agg_cmp *agg;
1002 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1003 struct page *page;
1004 dma_addr_t mapping;
1005
bfcd8d79
MC		 1006 if (p5_tpa)
1007 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1008 else
1009 agg = bnxt_get_agg(bp, cpr, idx, i);
c0c050c5
MC		 1010 cons = agg->rx_agg_cmp_opaque;
1011 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1012 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1013
1014 cons_rx_buf = &rxr->rx_agg_ring[cons];
89d0a06c
MC		 1015 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1016 cons_rx_buf->offset, frag_len);
c0c050c5
MC		 1017 __clear_bit(cons, rxr->rx_agg_bmap);
1018
1019 /* It is possible for bnxt_alloc_rx_page() to allocate
1020 * a sw_prod index that equals the cons index, so we
1021 * need to clear the cons entry now.
1022 */
11cd119d 1023 mapping = cons_rx_buf->mapping;
c0c050c5
MC		 1024 page = cons_rx_buf->page;
1025 cons_rx_buf->page = NULL;
1026
1027 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1028 struct skb_shared_info *shinfo;
1029 unsigned int nr_frags;
1030
1031 shinfo = skb_shinfo(skb);
1032 nr_frags = --shinfo->nr_frags;
1033 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1034
1035 dev_kfree_skb(skb);
1036
1037 cons_rx_buf->page = page;
1038
1039 /* Update prod since possibly some pages have been
1040 * allocated already.
1041 */
1042 rxr->rx_agg_prod = prod;
4a228a3a 1043 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
c0c050c5
MC		 1044 return NULL;
1045 }
1046
c519fe9a
SN		 1047 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1048 PCI_DMA_FROMDEVICE,
1049 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 1050
1051 skb->data_len += frag_len;
1052 skb->len += frag_len;
1053 skb->truesize += PAGE_SIZE;
1054
1055 prod = NEXT_RX_AGG(prod);
c0c050c5
MC		 1056 }
1057 rxr->rx_agg_prod = prod;
1058 return skb;
1059}
1060
1061static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1062 u8 agg_bufs, u32 *raw_cons)
1063{
1064 u16 last;
1065 struct rx_agg_cmp *agg;
1066
1067 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1068 last = RING_CMP(*raw_cons);
1069 agg = (struct rx_agg_cmp *)
1070 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1071 return RX_AGG_CMP_VALID(agg, *raw_cons);
1072}
1073
1074static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1075 unsigned int len,
1076 dma_addr_t mapping)
1077{
1078 struct bnxt *bp = bnapi->bp;
1079 struct pci_dev *pdev = bp->pdev;
1080 struct sk_buff *skb;
1081
1082 skb = napi_alloc_skb(&bnapi->napi, len);
1083 if (!skb)
1084 return NULL;
1085
745fc05c
MC		 1086 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1087 bp->rx_dir);
c0c050c5 1088
6bb19474
MC		 1089 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1090 len + NET_IP_ALIGN);
c0c050c5 1091
745fc05c
MC		 1092 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1093 bp->rx_dir);
c0c050c5
MC		 1094
1095 skb_put(skb, len);
1096 return skb;
1097}
1098
e44758b7 1099static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
fa7e2812
MC		 1100 u32 *raw_cons, void *cmp)
1101{
fa7e2812
MC		 1102 struct rx_cmp *rxcmp = cmp;
1103 u32 tmp_raw_cons = *raw_cons;
1104 u8 cmp_type, agg_bufs = 0;
1105
1106 cmp_type = RX_CMP_TYPE(rxcmp);
1107
1108 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1109 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1110 RX_CMP_AGG_BUFS) >>
1111 RX_CMP_AGG_BUFS_SHIFT;
1112 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1113 struct rx_tpa_end_cmp *tpa_end = cmp;
1114
bfcd8d79
MC		 1115 if (bp->flags & BNXT_FLAG_CHIP_P5)
1116 return 0;
1117
4a228a3a 1118 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
fa7e2812
MC		 1119 }
1120
1121 if (agg_bufs) {
1122 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1123 return -EBUSY;
1124 }
1125 *raw_cons = tmp_raw_cons;
1126 return 0;
1127}
1128
c213eae8
MC		 1129static void bnxt_queue_sp_work(struct bnxt *bp)
1130{
1131 if (BNXT_PF(bp))
1132 queue_work(bnxt_pf_wq, &bp->sp_task);
1133 else
1134 schedule_work(&bp->sp_task);
1135}
1136
1137static void bnxt_cancel_sp_work(struct bnxt *bp)
1138{
1139 if (BNXT_PF(bp))
1140 flush_workqueue(bnxt_pf_wq);
1141 else
1142 cancel_work_sync(&bp->sp_task);
1143}
1144
fa7e2812
MC		 1145static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1146{
1147 if (!rxr->bnapi->in_reset) {
1148 rxr->bnapi->in_reset = true;
1149 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 1150 bnxt_queue_sp_work(bp);
fa7e2812
MC		 1151 }
1152 rxr->rx_next_cons = 0xffff;
1153}
1154
c0c050c5
MC		 1155static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1156 struct rx_tpa_start_cmp *tpa_start,
1157 struct rx_tpa_start_cmp_ext *tpa_start1)
1158{
c0c050c5 1159 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
bfcd8d79
MC		 1160 struct bnxt_tpa_info *tpa_info;
1161 u16 cons, prod, agg_id;
c0c050c5
MC		 1162 struct rx_bd *prod_bd;
1163 dma_addr_t mapping;
1164
bfcd8d79
MC		 1165 if (bp->flags & BNXT_FLAG_CHIP_P5)
1166 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1167 else
1168 agg_id = TPA_START_AGG_ID(tpa_start);
c0c050c5
MC		 1169 cons = tpa_start->rx_tpa_start_cmp_opaque;
1170 prod = rxr->rx_prod;
1171 cons_rx_buf = &rxr->rx_buf_ring[cons];
1172 prod_rx_buf = &rxr->rx_buf_ring[prod];
1173 tpa_info = &rxr->rx_tpa[agg_id];
1174
bfcd8d79
MC		 1175 if (unlikely(cons != rxr->rx_next_cons ||
1176 TPA_START_ERROR(tpa_start))) {
1177 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1178 cons, rxr->rx_next_cons,
1179 TPA_START_ERROR_CODE(tpa_start1));
fa7e2812
MC		 1180 bnxt_sched_reset(bp, rxr);
1181 return;
1182 }
ee5c7fb3
SP		 1183 /* Store cfa_code in tpa_info to use in tpa_end
1184 * completion processing.
1185 */
1186 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
c0c050c5 1187 prod_rx_buf->data = tpa_info->data;
6bb19474 1188 prod_rx_buf->data_ptr = tpa_info->data_ptr;
c0c050c5
MC		 1189
1190 mapping = tpa_info->mapping;
11cd119d 1191 prod_rx_buf->mapping = mapping;
c0c050c5
MC		 1192
1193 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1194
1195 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1196
1197 tpa_info->data = cons_rx_buf->data;
6bb19474 1198 tpa_info->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 1199 cons_rx_buf->data = NULL;
11cd119d 1200 tpa_info->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 1201
1202 tpa_info->len =
1203 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1204 RX_TPA_START_CMP_LEN_SHIFT;
1205 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1206 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1207
1208 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1209 tpa_info->gso_type = SKB_GSO_TCPV4;
1210 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
50f011b6 1211 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
c0c050c5
MC		 1212 tpa_info->gso_type = SKB_GSO_TCPV6;
1213 tpa_info->rss_hash =
1214 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1215 } else {
1216 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1217 tpa_info->gso_type = 0;
1218 if (netif_msg_rx_err(bp))
1219 netdev_warn(bp->dev, "TPA packet without valid hash\n");
1220 }
1221 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1222 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
94758f8d 1223 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
bfcd8d79 1224 tpa_info->agg_count = 0;
c0c050c5
MC		 1225
1226 rxr->rx_prod = NEXT_RX(prod);
1227 cons = NEXT_RX(cons);
376a5b86 1228 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 1229 cons_rx_buf = &rxr->rx_buf_ring[cons];
1230
1231 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1232 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1233 cons_rx_buf->data = NULL;
1234}
1235
4a228a3a 1236static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
c0c050c5
MC		 1237{
1238 if (agg_bufs)
4a228a3a 1239 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
c0c050c5
MC		 1240}
1241
bee5a188
MC		 1242#ifdef CONFIG_INET
1243static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1244{
1245 struct udphdr *uh = NULL;
1246
1247 if (ip_proto == htons(ETH_P_IP)) {
1248 struct iphdr *iph = (struct iphdr *)skb->data;
1249
1250 if (iph->protocol == IPPROTO_UDP)
1251 uh = (struct udphdr *)(iph + 1);
1252 } else {
1253 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1254
1255 if (iph->nexthdr == IPPROTO_UDP)
1256 uh = (struct udphdr *)(iph + 1);
1257 }
1258 if (uh) {
1259 if (uh->check)
1260 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1261 else
1262 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1263 }
1264}
1265#endif
1266
94758f8d
MC		 1267static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1268 int payload_off, int tcp_ts,
1269 struct sk_buff *skb)
1270{
1271#ifdef CONFIG_INET
1272 struct tcphdr *th;
1273 int len, nw_off;
1274 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1275 u32 hdr_info = tpa_info->hdr_info;
1276 bool loopback = false;
1277
1278 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1279 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1280 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1281
1282 /* If the packet is an internal loopback packet, the offsets will
1283 * have an extra 4 bytes.
1284 */
1285 if (inner_mac_off == 4) {
1286 loopback = true;
1287 } else if (inner_mac_off > 4) {
1288 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1289 ETH_HLEN - 2));
1290
1291 /* We only support inner iPv4/ipv6. If we don't see the
1292 * correct protocol ID, it must be a loopback packet where
1293 * the offsets are off by 4.
1294 */
09a7636a 1295 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
94758f8d
MC		 1296 loopback = true;
1297 }
1298 if (loopback) {
1299 /* internal loopback packet, subtract all offsets by 4 */
1300 inner_ip_off -= 4;
1301 inner_mac_off -= 4;
1302 outer_ip_off -= 4;
1303 }
1304
1305 nw_off = inner_ip_off - ETH_HLEN;
1306 skb_set_network_header(skb, nw_off);
1307 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1308 struct ipv6hdr *iph = ipv6_hdr(skb);
1309
1310 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1311 len = skb->len - skb_transport_offset(skb);
1312 th = tcp_hdr(skb);
1313 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1314 } else {
1315 struct iphdr *iph = ip_hdr(skb);
1316
1317 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1318 len = skb->len - skb_transport_offset(skb);
1319 th = tcp_hdr(skb);
1320 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1321 }
1322
1323 if (inner_mac_off) { /* tunnel */
94758f8d
MC		 1324 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1325 ETH_HLEN - 2));
1326
bee5a188 1327 bnxt_gro_tunnel(skb, proto);
94758f8d
MC		 1328 }
1329#endif
1330 return skb;
1331}
1332
c0c050c5
MC		 1333#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1334#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1335
309369c9
MC		 1336static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1337 int payload_off, int tcp_ts,
c0c050c5
MC		 1338 struct sk_buff *skb)
1339{
d1611c3a 1340#ifdef CONFIG_INET
c0c050c5 1341 struct tcphdr *th;
719ca811 1342 int len, nw_off, tcp_opt_len = 0;
27e24189 1343
309369c9 1344 if (tcp_ts)
c0c050c5
MC		 1345 tcp_opt_len = 12;
1346
c0c050c5
MC		 1347 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1348 struct iphdr *iph;
1349
1350 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1351 ETH_HLEN;
1352 skb_set_network_header(skb, nw_off);
1353 iph = ip_hdr(skb);
1354 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1355 len = skb->len - skb_transport_offset(skb);
1356 th = tcp_hdr(skb);
1357 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1358 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1359 struct ipv6hdr *iph;
1360
1361 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1362 ETH_HLEN;
1363 skb_set_network_header(skb, nw_off);
1364 iph = ipv6_hdr(skb);
1365 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1366 len = skb->len - skb_transport_offset(skb);
1367 th = tcp_hdr(skb);
1368 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1369 } else {
1370 dev_kfree_skb_any(skb);
1371 return NULL;
1372 }
c0c050c5 1373
bee5a188
MC		 1374 if (nw_off) /* tunnel */
1375 bnxt_gro_tunnel(skb, skb->protocol);
c0c050c5
MC		 1376#endif
1377 return skb;
1378}
1379
309369c9
MC		 1380static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1381 struct bnxt_tpa_info *tpa_info,
1382 struct rx_tpa_end_cmp *tpa_end,
1383 struct rx_tpa_end_cmp_ext *tpa_end1,
1384 struct sk_buff *skb)
1385{
1386#ifdef CONFIG_INET
1387 int payload_off;
1388 u16 segs;
1389
1390 segs = TPA_END_TPA_SEGS(tpa_end);
1391 if (segs == 1)
1392 return skb;
1393
1394 NAPI_GRO_CB(skb)->count = segs;
1395 skb_shinfo(skb)->gso_size =
1396 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1397 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
bfcd8d79
MC		 1398 if (bp->flags & BNXT_FLAG_CHIP_P5)
1399 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1400 else
1401 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
309369c9 1402 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
5910906c
MC		 1403 if (likely(skb))
1404 tcp_gro_complete(skb);
309369c9
MC		 1405#endif
1406 return skb;
1407}
1408
ee5c7fb3
SP		 1409/* Given the cfa_code of a received packet determine which
1410 * netdev (vf-rep or PF) the packet is destined to.
1411 */
1412static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1413{
1414 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1415
1416 /* if vf-rep dev is NULL, the must belongs to the PF */
1417 return dev ? dev : bp->dev;
1418}
1419
c0c050c5 1420static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
e44758b7 1421 struct bnxt_cp_ring_info *cpr,
c0c050c5
MC		 1422 u32 *raw_cons,
1423 struct rx_tpa_end_cmp *tpa_end,
1424 struct rx_tpa_end_cmp_ext *tpa_end1,
4e5dbbda 1425 u8 *event)
c0c050c5 1426{
e44758b7 1427 struct bnxt_napi *bnapi = cpr->bnapi;
b6ab4b01 1428 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
6bb19474 1429 u8 *data_ptr, agg_bufs;
c0c050c5
MC		 1430 unsigned int len;
1431 struct bnxt_tpa_info *tpa_info;
1432 dma_addr_t mapping;
1433 struct sk_buff *skb;
bfcd8d79 1434 u16 idx = 0, agg_id;
6bb19474 1435 void *data;
bfcd8d79 1436 bool gro;
c0c050c5 1437
fa7e2812 1438 if (unlikely(bnapi->in_reset)) {
e44758b7 1439 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
fa7e2812
MC		 1440
1441 if (rc < 0)
1442 return ERR_PTR(-EBUSY);
1443 return NULL;
1444 }
1445
bfcd8d79
MC		 1446 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1447 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1448 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1449 tpa_info = &rxr->rx_tpa[agg_id];
1450 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1451 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1452 agg_bufs, tpa_info->agg_count);
1453 agg_bufs = tpa_info->agg_count;
1454 }
1455 tpa_info->agg_count = 0;
1456 *event |= BNXT_AGG_EVENT;
1457 idx = agg_id;
1458 gro = !!(bp->flags & BNXT_FLAG_GRO);
1459 } else {
1460 agg_id = TPA_END_AGG_ID(tpa_end);
1461 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1462 tpa_info = &rxr->rx_tpa[agg_id];
1463 idx = RING_CMP(*raw_cons);
1464 if (agg_bufs) {
1465 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1466 return ERR_PTR(-EBUSY);
1467
1468 *event |= BNXT_AGG_EVENT;
1469 idx = NEXT_CMP(idx);
1470 }
1471 gro = !!TPA_END_GRO(tpa_end);
1472 }
c0c050c5 1473 data = tpa_info->data;
6bb19474
MC		 1474 data_ptr = tpa_info->data_ptr;
1475 prefetch(data_ptr);
c0c050c5
MC		 1476 len = tpa_info->len;
1477 mapping = tpa_info->mapping;
1478
69c149e2 1479 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
4a228a3a 1480 bnxt_abort_tpa(cpr, idx, agg_bufs);
69c149e2
MC		 1481 if (agg_bufs > MAX_SKB_FRAGS)
1482 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1483 agg_bufs, (int)MAX_SKB_FRAGS);
c0c050c5
MC		 1484 return NULL;
1485 }
1486
1487 if (len <= bp->rx_copy_thresh) {
6bb19474 1488 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
c0c050c5 1489 if (!skb) {
4a228a3a 1490 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1491 return NULL;
1492 }
1493 } else {
1494 u8 *new_data;
1495 dma_addr_t new_mapping;
1496
1497 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1498 if (!new_data) {
4a228a3a 1499 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1500 return NULL;
1501 }
1502
1503 tpa_info->data = new_data;
b3dba77c 1504 tpa_info->data_ptr = new_data + bp->rx_offset;
c0c050c5
MC		 1505 tpa_info->mapping = new_mapping;
1506
1507 skb = build_skb(data, 0);
c519fe9a
SN		 1508 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1509 bp->rx_buf_use_size, bp->rx_dir,
1510 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 1511
1512 if (!skb) {
1513 kfree(data);
4a228a3a 1514 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1515 return NULL;
1516 }
b3dba77c 1517 skb_reserve(skb, bp->rx_offset);
c0c050c5
MC		 1518 skb_put(skb, len);
1519 }
1520
1521 if (agg_bufs) {
4a228a3a 1522 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
c0c050c5
MC		 1523 if (!skb) {
1524 /* Page reuse already handled by bnxt_rx_pages(). */
1525 return NULL;
1526 }
1527 }
ee5c7fb3
SP		 1528
1529 skb->protocol =
1530 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
c0c050c5
MC		 1531
1532 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1533 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1534
8852ddb4
MC		 1535 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1536 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5
MC		 1537 u16 vlan_proto = tpa_info->metadata >>
1538 RX_CMP_FLAGS2_METADATA_TPID_SFT;
ed7bc602 1539 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
c0c050c5 1540
8852ddb4 1541 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1542 }
1543
1544 skb_checksum_none_assert(skb);
1545 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1546 skb->ip_summed = CHECKSUM_UNNECESSARY;
1547 skb->csum_level =
1548 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1549 }
1550
bfcd8d79 1551 if (gro)
309369c9 1552 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
c0c050c5
MC		 1553
1554 return skb;
1555}
1556
8fe88ce7
MC		 1557static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1558 struct rx_agg_cmp *rx_agg)
1559{
1560 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1561 struct bnxt_tpa_info *tpa_info;
1562
1563 tpa_info = &rxr->rx_tpa[agg_id];
1564 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1565 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1566}
1567
ee5c7fb3
SP		 1568static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1569 struct sk_buff *skb)
1570{
1571 if (skb->dev != bp->dev) {
1572 /* this packet belongs to a vf-rep */
1573 bnxt_vf_rep_rx(bp, skb);
1574 return;
1575 }
1576 skb_record_rx_queue(skb, bnapi->index);
1577 napi_gro_receive(&bnapi->napi, skb);
1578}
1579
c0c050c5
MC		 1580/* returns the following:
1581 * 1 - 1 packet successfully received
1582 * 0 - successful TPA_START, packet not completed yet
1583 * -EBUSY - completion ring does not have all the agg buffers yet
1584 * -ENOMEM - packet aborted due to out of memory
1585 * -EIO - packet aborted due to hw error indicated in BD
1586 */
e44758b7
MC		 1587static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1588 u32 *raw_cons, u8 *event)
c0c050c5 1589{
e44758b7 1590 struct bnxt_napi *bnapi = cpr->bnapi;
b6ab4b01 1591 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1592 struct net_device *dev = bp->dev;
1593 struct rx_cmp *rxcmp;
1594 struct rx_cmp_ext *rxcmp1;
1595 u32 tmp_raw_cons = *raw_cons;
ee5c7fb3 1596 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
c0c050c5
MC		 1597 struct bnxt_sw_rx_bd *rx_buf;
1598 unsigned int len;
6bb19474 1599 u8 *data_ptr, agg_bufs, cmp_type;
c0c050c5
MC		 1600 dma_addr_t dma_addr;
1601 struct sk_buff *skb;
6bb19474 1602 void *data;
c0c050c5 1603 int rc = 0;
c61fb99c 1604 u32 misc;
c0c050c5
MC		 1605
1606 rxcmp = (struct rx_cmp *)
1607 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1608
8fe88ce7
MC		 1609 cmp_type = RX_CMP_TYPE(rxcmp);
1610
1611 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1612 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1613 goto next_rx_no_prod_no_len;
1614 }
1615
c0c050c5
MC		 1616 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1617 cp_cons = RING_CMP(tmp_raw_cons);
1618 rxcmp1 = (struct rx_cmp_ext *)
1619 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1620
1621 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1622 return -EBUSY;
1623
c0c050c5
MC		 1624 prod = rxr->rx_prod;
1625
1626 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1627 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1628 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1629
4e5dbbda 1630 *event |= BNXT_RX_EVENT;
e7e70fa6 1631 goto next_rx_no_prod_no_len;
c0c050c5
MC		 1632
1633 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
e44758b7 1634 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
c0c050c5 1635 (struct rx_tpa_end_cmp *)rxcmp,
4e5dbbda 1636 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
c0c050c5 1637
1fac4b2f 1638 if (IS_ERR(skb))
c0c050c5
MC		 1639 return -EBUSY;
1640
1641 rc = -ENOMEM;
1642 if (likely(skb)) {
ee5c7fb3 1643 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1644 rc = 1;
1645 }
4e5dbbda 1646 *event |= BNXT_RX_EVENT;
e7e70fa6 1647 goto next_rx_no_prod_no_len;
c0c050c5
MC		 1648 }
1649
1650 cons = rxcmp->rx_cmp_opaque;
fa7e2812 1651 if (unlikely(cons != rxr->rx_next_cons)) {
e44758b7 1652 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
fa7e2812 1653
a1b0e4e6
MC		 1654 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1655 cons, rxr->rx_next_cons);
fa7e2812
MC		 1656 bnxt_sched_reset(bp, rxr);
1657 return rc1;
1658 }
a1b0e4e6
MC		 1659 rx_buf = &rxr->rx_buf_ring[cons];
1660 data = rx_buf->data;
1661 data_ptr = rx_buf->data_ptr;
6bb19474 1662 prefetch(data_ptr);
c0c050c5 1663
c61fb99c
MC		 1664 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1665 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
c0c050c5
MC		 1666
1667 if (agg_bufs) {
1668 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1669 return -EBUSY;
1670
1671 cp_cons = NEXT_CMP(cp_cons);
4e5dbbda 1672 *event |= BNXT_AGG_EVENT;
c0c050c5 1673 }
4e5dbbda 1674 *event |= BNXT_RX_EVENT;
c0c050c5
MC		 1675
1676 rx_buf->data = NULL;
1677 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
8e44e96c
MC		 1678 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1679
c0c050c5
MC		 1680 bnxt_reuse_rx_data(rxr, cons, data);
1681 if (agg_bufs)
4a228a3a
MC		 1682 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1683 false);
c0c050c5
MC		 1684
1685 rc = -EIO;
8e44e96c
MC		 1686 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1687 netdev_warn(bp->dev, "RX buffer error %x\n", rx_err);
1688 bnxt_sched_reset(bp, rxr);
1689 }
0b397b17 1690 goto next_rx_no_len;
c0c050c5
MC		 1691 }
1692
1693 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
11cd119d 1694 dma_addr = rx_buf->mapping;
c0c050c5 1695
c6d30e83
MC		 1696 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1697 rc = 1;
1698 goto next_rx;
1699 }
1700
c0c050c5 1701 if (len <= bp->rx_copy_thresh) {
6bb19474 1702 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
c0c050c5
MC		 1703 bnxt_reuse_rx_data(rxr, cons, data);
1704 if (!skb) {
296d5b54 1705 if (agg_bufs)
4a228a3a
MC		 1706 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1707 agg_bufs, false);
c0c050c5
MC		 1708 rc = -ENOMEM;
1709 goto next_rx;
1710 }
1711 } else {
c61fb99c
MC		 1712 u32 payload;
1713
c6d30e83
MC		 1714 if (rx_buf->data_ptr == data_ptr)
1715 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1716 else
1717 payload = 0;
6bb19474 1718 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
c61fb99c 1719 payload | len);
c0c050c5
MC		 1720 if (!skb) {
1721 rc = -ENOMEM;
1722 goto next_rx;
1723 }
1724 }
1725
1726 if (agg_bufs) {
4a228a3a 1727 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
c0c050c5
MC		 1728 if (!skb) {
1729 rc = -ENOMEM;
1730 goto next_rx;
1731 }
1732 }
1733
1734 if (RX_CMP_HASH_VALID(rxcmp)) {
1735 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1736 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1737
1738 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1739 if (hash_type != 1 && hash_type != 3)
1740 type = PKT_HASH_TYPE_L3;
1741 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1742 }
1743
ee5c7fb3
SP		 1744 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1745 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
c0c050c5 1746
8852ddb4
MC		 1747 if ((rxcmp1->rx_cmp_flags2 &
1748 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1749 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5 1750 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
ed7bc602 1751 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
c0c050c5
MC		 1752 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1753
8852ddb4 1754 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1755 }
1756
1757 skb_checksum_none_assert(skb);
1758 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1759 if (dev->features & NETIF_F_RXCSUM) {
1760 skb->ip_summed = CHECKSUM_UNNECESSARY;
1761 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1762 }
1763 } else {
665e350d
SB		 1764 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1765 if (dev->features & NETIF_F_RXCSUM)
d1981929 1766 bnapi->cp_ring.rx_l4_csum_errors++;
665e350d 1767 }
c0c050c5
MC		 1768 }
1769
ee5c7fb3 1770 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1771 rc = 1;
1772
1773next_rx:
6a8788f2
AG		 1774 cpr->rx_packets += 1;
1775 cpr->rx_bytes += len;
e7e70fa6 1776
0b397b17
MC		 1777next_rx_no_len:
1778 rxr->rx_prod = NEXT_RX(prod);
1779 rxr->rx_next_cons = NEXT_RX(cons);
1780
e7e70fa6 1781next_rx_no_prod_no_len:
c0c050c5
MC		 1782 *raw_cons = tmp_raw_cons;
1783
1784 return rc;
1785}
1786
2270bc5d
MC		 1787/* In netpoll mode, if we are using a combined completion ring, we need to
1788 * discard the rx packets and recycle the buffers.
1789 */
e44758b7
MC		 1790static int bnxt_force_rx_discard(struct bnxt *bp,
1791 struct bnxt_cp_ring_info *cpr,
2270bc5d
MC		 1792 u32 *raw_cons, u8 *event)
1793{
2270bc5d
MC		 1794 u32 tmp_raw_cons = *raw_cons;
1795 struct rx_cmp_ext *rxcmp1;
1796 struct rx_cmp *rxcmp;
1797 u16 cp_cons;
1798 u8 cmp_type;
1799
1800 cp_cons = RING_CMP(tmp_raw_cons);
1801 rxcmp = (struct rx_cmp *)
1802 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1803
1804 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1805 cp_cons = RING_CMP(tmp_raw_cons);
1806 rxcmp1 = (struct rx_cmp_ext *)
1807 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1808
1809 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1810 return -EBUSY;
1811
1812 cmp_type = RX_CMP_TYPE(rxcmp);
1813 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1814 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1815 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1816 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1817 struct rx_tpa_end_cmp_ext *tpa_end1;
1818
1819 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1820 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1821 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1822 }
e44758b7 1823 return bnxt_rx_pkt(bp, cpr, raw_cons, event);
2270bc5d
MC		 1824}
1825
4bb13abf 1826#define BNXT_GET_EVENT_PORT(data) \
87c374de
MC		 1827 ((data) & \
1828 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
4bb13abf 1829
c0c050c5
MC		 1830static int bnxt_async_event_process(struct bnxt *bp,
1831 struct hwrm_async_event_cmpl *cmpl)
1832{
1833 u16 event_id = le16_to_cpu(cmpl->event_id);
1834
1835 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1836 switch (event_id) {
87c374de 1837 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
8cbde117
MC		 1838 u32 data1 = le32_to_cpu(cmpl->event_data1);
1839 struct bnxt_link_info *link_info = &bp->link_info;
1840
1841 if (BNXT_VF(bp))
1842 goto async_event_process_exit;
a8168b6c
MC		 1843
1844 /* print unsupported speed warning in forced speed mode only */
1845 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1846 (data1 & 0x20000)) {
8cbde117
MC		 1847 u16 fw_speed = link_info->force_link_speed;
1848 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1849
a8168b6c
MC		 1850 if (speed != SPEED_UNKNOWN)
1851 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1852 speed);
8cbde117 1853 }
286ef9d6 1854 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
8cbde117 1855 }
bc171e87 1856 /* fall through */
87c374de 1857 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
c0c050c5 1858 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
19241368 1859 break;
87c374de 1860 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
19241368 1861 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
c0c050c5 1862 break;
87c374de 1863 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
4bb13abf
MC		 1864 u32 data1 = le32_to_cpu(cmpl->event_data1);
1865 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1866
1867 if (BNXT_VF(bp))
1868 break;
1869
1870 if (bp->pf.port_id != port_id)
1871 break;
1872
4bb13abf
MC		 1873 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1874 break;
1875 }
87c374de 1876 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
fc0f1929
MC		 1877 if (BNXT_PF(bp))
1878 goto async_event_process_exit;
1879 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1880 break;
c0c050c5 1881 default:
19241368 1882 goto async_event_process_exit;
c0c050c5 1883 }
c213eae8 1884 bnxt_queue_sp_work(bp);
19241368 1885async_event_process_exit:
a588e458 1886 bnxt_ulp_async_events(bp, cmpl);
c0c050c5
MC		 1887 return 0;
1888}
1889
1890static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1891{
1892 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1893 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1894 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1895 (struct hwrm_fwd_req_cmpl *)txcmp;
1896
1897 switch (cmpl_type) {
1898 case CMPL_BASE_TYPE_HWRM_DONE:
1899 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1900 if (seq_id == bp->hwrm_intr_seq_id)
fc718bb2 1901 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
c0c050c5
MC		 1902 else
1903 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1904 break;
1905
1906 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1907 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1908
1909 if ((vf_id < bp->pf.first_vf_id) ||
1910 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1911 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1912 vf_id);
1913 return -EINVAL;
1914 }
1915
1916 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1917 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
c213eae8 1918 bnxt_queue_sp_work(bp);
c0c050c5
MC		 1919 break;
1920
1921 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1922 bnxt_async_event_process(bp,
1923 (struct hwrm_async_event_cmpl *)txcmp);
1924
1925 default:
1926 break;
1927 }
1928
1929 return 0;
1930}
1931
1932static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1933{
1934 struct bnxt_napi *bnapi = dev_instance;
1935 struct bnxt *bp = bnapi->bp;
1936 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1937 u32 cons = RING_CMP(cpr->cp_raw_cons);
1938
6a8788f2 1939 cpr->event_ctr++;
c0c050c5
MC		 1940 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1941 napi_schedule(&bnapi->napi);
1942 return IRQ_HANDLED;
1943}
1944
1945static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1946{
1947 u32 raw_cons = cpr->cp_raw_cons;
1948 u16 cons = RING_CMP(raw_cons);
1949 struct tx_cmp *txcmp;
1950
1951 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1952
1953 return TX_CMP_VALID(txcmp, raw_cons);
1954}
1955
c0c050c5
MC		 1956static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1957{
1958 struct bnxt_napi *bnapi = dev_instance;
1959 struct bnxt *bp = bnapi->bp;
1960 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1961 u32 cons = RING_CMP(cpr->cp_raw_cons);
1962 u32 int_status;
1963
1964 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1965
1966 if (!bnxt_has_work(bp, cpr)) {
11809490 1967 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
c0c050c5
MC		 1968 /* return if erroneous interrupt */
1969 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1970 return IRQ_NONE;
1971 }
1972
1973 /* disable ring IRQ */
697197e5 1974 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
c0c050c5
MC		 1975
1976 /* Return here if interrupt is shared and is disabled. */
1977 if (unlikely(atomic_read(&bp->intr_sem) != 0))
1978 return IRQ_HANDLED;
1979
1980 napi_schedule(&bnapi->napi);
1981 return IRQ_HANDLED;
1982}
1983
3675b92f
MC		 1984static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1985 int budget)
c0c050c5 1986{
e44758b7 1987 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5
MC		 1988 u32 raw_cons = cpr->cp_raw_cons;
1989 u32 cons;
1990 int tx_pkts = 0;
1991 int rx_pkts = 0;
4e5dbbda 1992 u8 event = 0;
c0c050c5
MC		 1993 struct tx_cmp *txcmp;
1994
0fcec985 1995 cpr->has_more_work = 0;
c0c050c5
MC		 1996 while (1) {
1997 int rc;
1998
1999 cons = RING_CMP(raw_cons);
2000 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2001
2002 if (!TX_CMP_VALID(txcmp, raw_cons))
2003 break;
2004
67a95e20
MC		 2005 /* The valid test of the entry must be done first before
2006 * reading any further.
2007 */
b67daab0 2008 dma_rmb();
3675b92f 2009 cpr->had_work_done = 1;
c0c050c5
MC		 2010 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2011 tx_pkts++;
2012 /* return full budget so NAPI will complete. */
73f21c65 2013 if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
c0c050c5 2014 rx_pkts = budget;
73f21c65 2015 raw_cons = NEXT_RAW_CMP(raw_cons);
0fcec985
MC		 2016 if (budget)
2017 cpr->has_more_work = 1;
73f21c65
MC		 2018 break;
2019 }
c0c050c5 2020 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2270bc5d 2021 if (likely(budget))
e44758b7 2022 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2270bc5d 2023 else
e44758b7 2024 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2270bc5d 2025 &event);
c0c050c5
MC		 2026 if (likely(rc >= 0))
2027 rx_pkts += rc;
903649e7
MC		 2028 /* Increment rx_pkts when rc is -ENOMEM to count towards
2029 * the NAPI budget. Otherwise, we may potentially loop
2030 * here forever if we consistently cannot allocate
2031 * buffers.
2032 */
2edbdb31 2033 else if (rc == -ENOMEM && budget)
903649e7 2034 rx_pkts++;
c0c050c5
MC		 2035 else if (rc == -EBUSY) /* partial completion */
2036 break;
c0c050c5
MC		 2037 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2038 CMPL_BASE_TYPE_HWRM_DONE) ||
2039 (TX_CMP_TYPE(txcmp) ==
2040 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2041 (TX_CMP_TYPE(txcmp) ==
2042 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2043 bnxt_hwrm_handler(bp, txcmp);
2044 }
2045 raw_cons = NEXT_RAW_CMP(raw_cons);
2046
0fcec985
MC		 2047 if (rx_pkts && rx_pkts == budget) {
2048 cpr->has_more_work = 1;
c0c050c5 2049 break;
0fcec985 2050 }
c0c050c5
MC		 2051 }
2052
f18c2b77
AG		 2053 if (event & BNXT_REDIRECT_EVENT)
2054 xdp_do_flush_map();
2055
38413406
MC		 2056 if (event & BNXT_TX_EVENT) {
2057 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
38413406
MC		 2058 u16 prod = txr->tx_prod;
2059
2060 /* Sync BD data before updating doorbell */
2061 wmb();
2062
697197e5 2063 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
38413406
MC		 2064 }
2065
c0c050c5 2066 cpr->cp_raw_cons = raw_cons;
3675b92f
MC		 2067 bnapi->tx_pkts += tx_pkts;
2068 bnapi->events |= event;
2069 return rx_pkts;
2070}
c0c050c5 2071
3675b92f
MC		 2072static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2073{
2074 if (bnapi->tx_pkts) {
2075 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2076 bnapi->tx_pkts = 0;
2077 }
c0c050c5 2078
3675b92f 2079 if (bnapi->events & BNXT_RX_EVENT) {
b6ab4b01 2080 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 2081
697197e5 2082 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
3675b92f 2083 if (bnapi->events & BNXT_AGG_EVENT)
697197e5 2084 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
c0c050c5 2085 }
3675b92f
MC		 2086 bnapi->events = 0;
2087}
2088
2089static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2090 int budget)
2091{
2092 struct bnxt_napi *bnapi = cpr->bnapi;
2093 int rx_pkts;
2094
2095 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2096
2097 /* ACK completion ring before freeing tx ring and producing new
2098 * buffers in rx/agg rings to prevent overflowing the completion
2099 * ring.
2100 */
2101 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2102
2103 __bnxt_poll_work_done(bp, bnapi);
c0c050c5
MC		 2104 return rx_pkts;
2105}
2106
10bbdaf5
PS		 2107static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2108{
2109 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2110 struct bnxt *bp = bnapi->bp;
2111 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2112 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2113 struct tx_cmp *txcmp;
2114 struct rx_cmp_ext *rxcmp1;
2115 u32 cp_cons, tmp_raw_cons;
2116 u32 raw_cons = cpr->cp_raw_cons;
2117 u32 rx_pkts = 0;
4e5dbbda 2118 u8 event = 0;
10bbdaf5
PS		 2119
2120 while (1) {
2121 int rc;
2122
2123 cp_cons = RING_CMP(raw_cons);
2124 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2125
2126 if (!TX_CMP_VALID(txcmp, raw_cons))
2127 break;
2128
2129 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2130 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2131 cp_cons = RING_CMP(tmp_raw_cons);
2132 rxcmp1 = (struct rx_cmp_ext *)
2133 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2134
2135 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2136 break;
2137
2138 /* force an error to recycle the buffer */
2139 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2140 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2141
e44758b7 2142 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2edbdb31 2143 if (likely(rc == -EIO) && budget)
10bbdaf5
PS		 2144 rx_pkts++;
2145 else if (rc == -EBUSY) /* partial completion */
2146 break;
2147 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2148 CMPL_BASE_TYPE_HWRM_DONE)) {
2149 bnxt_hwrm_handler(bp, txcmp);
2150 } else {
2151 netdev_err(bp->dev,
2152 "Invalid completion received on special ring\n");
2153 }
2154 raw_cons = NEXT_RAW_CMP(raw_cons);
2155
2156 if (rx_pkts == budget)
2157 break;
2158 }
2159
2160 cpr->cp_raw_cons = raw_cons;
697197e5
MC		 2161 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2162 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10bbdaf5 2163
434c975a 2164 if (event & BNXT_AGG_EVENT)
697197e5 2165 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10bbdaf5
PS		 2166
2167 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
6ad20165 2168 napi_complete_done(napi, rx_pkts);
697197e5 2169 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
10bbdaf5
PS		 2170 }
2171 return rx_pkts;
2172}
2173
c0c050c5
MC		 2174static int bnxt_poll(struct napi_struct *napi, int budget)
2175{
2176 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2177 struct bnxt *bp = bnapi->bp;
2178 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2179 int work_done = 0;
2180
c0c050c5 2181 while (1) {
e44758b7 2182 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
c0c050c5 2183
73f21c65
MC		 2184 if (work_done >= budget) {
2185 if (!budget)
697197e5 2186 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
c0c050c5 2187 break;
73f21c65 2188 }
c0c050c5
MC		 2189
2190 if (!bnxt_has_work(bp, cpr)) {
e7b95691 2191 if (napi_complete_done(napi, work_done))
697197e5 2192 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
c0c050c5
MC		 2193 break;
2194 }
2195 }
6a8788f2 2196 if (bp->flags & BNXT_FLAG_DIM) {
8960b389 2197 struct dim_sample dim_sample;
6a8788f2 2198
8960b389
TG		 2199 dim_update_sample(cpr->event_ctr,
2200 cpr->rx_packets,
2201 cpr->rx_bytes,
2202 &dim_sample);
6a8788f2
AG		 2203 net_dim(&cpr->dim, dim_sample);
2204 }
c0c050c5
MC		 2205 return work_done;
2206}
2207
0fcec985
MC		 2208static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2209{
2210 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2211 int i, work_done = 0;
2212
2213 for (i = 0; i < 2; i++) {
2214 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2215
2216 if (cpr2) {
2217 work_done += __bnxt_poll_work(bp, cpr2,
2218 budget - work_done);
2219 cpr->has_more_work |= cpr2->has_more_work;
2220 }
2221 }
2222 return work_done;
2223}
2224
2225static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2226 u64 dbr_type, bool all)
2227{
2228 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2229 int i;
2230
2231 for (i = 0; i < 2; i++) {
2232 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2233 struct bnxt_db_info *db;
2234
2235 if (cpr2 && (all || cpr2->had_work_done)) {
2236 db = &cpr2->cp_db;
2237 writeq(db->db_key64 | dbr_type |
2238 RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2239 cpr2->had_work_done = 0;
2240 }
2241 }
2242 __bnxt_poll_work_done(bp, bnapi);
2243}
2244
2245static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2246{
2247 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2248 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2249 u32 raw_cons = cpr->cp_raw_cons;
2250 struct bnxt *bp = bnapi->bp;
2251 struct nqe_cn *nqcmp;
2252 int work_done = 0;
2253 u32 cons;
2254
2255 if (cpr->has_more_work) {
2256 cpr->has_more_work = 0;
2257 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2258 if (cpr->has_more_work) {
2259 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2260 return work_done;
2261 }
2262 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2263 if (napi_complete_done(napi, work_done))
2264 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2265 return work_done;
2266 }
2267 while (1) {
2268 cons = RING_CMP(raw_cons);
2269 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2270
2271 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2272 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2273 false);
2274 cpr->cp_raw_cons = raw_cons;
2275 if (napi_complete_done(napi, work_done))
2276 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2277 cpr->cp_raw_cons);
2278 return work_done;
2279 }
2280
2281 /* The valid test of the entry must be done first before
2282 * reading any further.
2283 */
2284 dma_rmb();
2285
2286 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2287 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2288 struct bnxt_cp_ring_info *cpr2;
2289
2290 cpr2 = cpr->cp_ring_arr[idx];
2291 work_done += __bnxt_poll_work(bp, cpr2,
2292 budget - work_done);
2293 cpr->has_more_work = cpr2->has_more_work;
2294 } else {
2295 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2296 }
2297 raw_cons = NEXT_RAW_CMP(raw_cons);
2298 if (cpr->has_more_work)
2299 break;
2300 }
2301 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2302 cpr->cp_raw_cons = raw_cons;
2303 return work_done;
2304}
2305
c0c050c5
MC		 2306static void bnxt_free_tx_skbs(struct bnxt *bp)
2307{
2308 int i, max_idx;
2309 struct pci_dev *pdev = bp->pdev;
2310
b6ab4b01 2311 if (!bp->tx_ring)
c0c050c5
MC		 2312 return;
2313
2314 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2315 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2316 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2317 int j;
2318
c0c050c5
MC		 2319 for (j = 0; j < max_idx;) {
2320 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
f18c2b77 2321 struct sk_buff *skb;
c0c050c5
MC		 2322 int k, last;
2323
f18c2b77
AG		 2324 if (i < bp->tx_nr_rings_xdp &&
2325 tx_buf->action == XDP_REDIRECT) {
2326 dma_unmap_single(&pdev->dev,
2327 dma_unmap_addr(tx_buf, mapping),
2328 dma_unmap_len(tx_buf, len),
2329 PCI_DMA_TODEVICE);
2330 xdp_return_frame(tx_buf->xdpf);
2331 tx_buf->action = 0;
2332 tx_buf->xdpf = NULL;
2333 j++;
2334 continue;
2335 }
2336
2337 skb = tx_buf->skb;
c0c050c5
MC		 2338 if (!skb) {
2339 j++;
2340 continue;
2341 }
2342
2343 tx_buf->skb = NULL;
2344
2345 if (tx_buf->is_push) {
2346 dev_kfree_skb(skb);
2347 j += 2;
2348 continue;
2349 }
2350
2351 dma_unmap_single(&pdev->dev,
2352 dma_unmap_addr(tx_buf, mapping),
2353 skb_headlen(skb),
2354 PCI_DMA_TODEVICE);
2355
2356 last = tx_buf->nr_frags;
2357 j += 2;
d612a579
MC		 2358 for (k = 0; k < last; k++, j++) {
2359 int ring_idx = j & bp->tx_ring_mask;
c0c050c5
MC		 2360 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2361
d612a579 2362 tx_buf = &txr->tx_buf_ring[ring_idx];
c0c050c5
MC		 2363 dma_unmap_page(
2364 &pdev->dev,
2365 dma_unmap_addr(tx_buf, mapping),
2366 skb_frag_size(frag), PCI_DMA_TODEVICE);
2367 }
2368 dev_kfree_skb(skb);
2369 }
2370 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2371 }
2372}
2373
2374static void bnxt_free_rx_skbs(struct bnxt *bp)
2375{
2376 int i, max_idx, max_agg_idx;
2377 struct pci_dev *pdev = bp->pdev;
2378
b6ab4b01 2379 if (!bp->rx_ring)
c0c050c5
MC		 2380 return;
2381
2382 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2383 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2384 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2385 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2386 int j;
2387
c0c050c5 2388 if (rxr->rx_tpa) {
79632e9b 2389 for (j = 0; j < bp->max_tpa; j++) {
c0c050c5
MC		 2390 struct bnxt_tpa_info *tpa_info =
2391 &rxr->rx_tpa[j];
2392 u8 *data = tpa_info->data;
2393
2394 if (!data)
2395 continue;
2396
c519fe9a
SN		 2397 dma_unmap_single_attrs(&pdev->dev,
2398 tpa_info->mapping,
2399 bp->rx_buf_use_size,
2400 bp->rx_dir,
2401 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2402
2403 tpa_info->data = NULL;
2404
2405 kfree(data);
2406 }
2407 }
2408
2409 for (j = 0; j < max_idx; j++) {
2410 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
3ed3a83e 2411 dma_addr_t mapping = rx_buf->mapping;
6bb19474 2412 void *data = rx_buf->data;
c0c050c5
MC		 2413
2414 if (!data)
2415 continue;
2416
c0c050c5
MC		 2417 rx_buf->data = NULL;
2418
3ed3a83e
MC		 2419 if (BNXT_RX_PAGE_MODE(bp)) {
2420 mapping -= bp->rx_dma_offset;
c519fe9a
SN		 2421 dma_unmap_page_attrs(&pdev->dev, mapping,
2422 PAGE_SIZE, bp->rx_dir,
2423 DMA_ATTR_WEAK_ORDERING);
322b87ca 2424 page_pool_recycle_direct(rxr->page_pool, data);
3ed3a83e 2425 } else {
c519fe9a
SN		 2426 dma_unmap_single_attrs(&pdev->dev, mapping,
2427 bp->rx_buf_use_size,
2428 bp->rx_dir,
2429 DMA_ATTR_WEAK_ORDERING);
c61fb99c 2430 kfree(data);
3ed3a83e 2431 }
c0c050c5
MC		 2432 }
2433
2434 for (j = 0; j < max_agg_idx; j++) {
2435 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2436 &rxr->rx_agg_ring[j];
2437 struct page *page = rx_agg_buf->page;
2438
2439 if (!page)
2440 continue;
2441
c519fe9a
SN		 2442 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2443 BNXT_RX_PAGE_SIZE,
2444 PCI_DMA_FROMDEVICE,
2445 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2446
2447 rx_agg_buf->page = NULL;
2448 __clear_bit(j, rxr->rx_agg_bmap);
2449
2450 __free_page(page);
2451 }
89d0a06c
MC		 2452 if (rxr->rx_page) {
2453 __free_page(rxr->rx_page);
2454 rxr->rx_page = NULL;
2455 }
c0c050c5
MC		 2456 }
2457}
2458
2459static void bnxt_free_skbs(struct bnxt *bp)
2460{
2461 bnxt_free_tx_skbs(bp);
2462 bnxt_free_rx_skbs(bp);
2463}
2464
6fe19886 2465static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
c0c050c5
MC		 2466{
2467 struct pci_dev *pdev = bp->pdev;
2468 int i;
2469
6fe19886
MC		 2470 for (i = 0; i < rmem->nr_pages; i++) {
2471 if (!rmem->pg_arr[i])
c0c050c5
MC		 2472 continue;
2473
6fe19886
MC		 2474 dma_free_coherent(&pdev->dev, rmem->page_size,
2475 rmem->pg_arr[i], rmem->dma_arr[i]);
c0c050c5 2476
6fe19886 2477 rmem->pg_arr[i] = NULL;
c0c050c5 2478 }
6fe19886 2479 if (rmem->pg_tbl) {
4f49b2b8
MC		 2480 size_t pg_tbl_size = rmem->nr_pages * 8;
2481
2482 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2483 pg_tbl_size = rmem->page_size;
2484 dma_free_coherent(&pdev->dev, pg_tbl_size,
6fe19886
MC		 2485 rmem->pg_tbl, rmem->pg_tbl_map);
2486 rmem->pg_tbl = NULL;
c0c050c5 2487 }
6fe19886
MC		 2488 if (rmem->vmem_size && *rmem->vmem) {
2489 vfree(*rmem->vmem);
2490 *rmem->vmem = NULL;
c0c050c5
MC		 2491 }
2492}
2493
6fe19886 2494static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
c0c050c5 2495{
c0c050c5 2496 struct pci_dev *pdev = bp->pdev;
66cca20a 2497 u64 valid_bit = 0;
6fe19886 2498 int i;
c0c050c5 2499
66cca20a
MC		 2500 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2501 valid_bit = PTU_PTE_VALID;
4f49b2b8
MC		 2502 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2503 size_t pg_tbl_size = rmem->nr_pages * 8;
2504
2505 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2506 pg_tbl_size = rmem->page_size;
2507 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
6fe19886 2508 &rmem->pg_tbl_map,
c0c050c5 2509 GFP_KERNEL);
6fe19886 2510 if (!rmem->pg_tbl)
c0c050c5
MC		 2511 return -ENOMEM;
2512 }
2513
6fe19886 2514 for (i = 0; i < rmem->nr_pages; i++) {
66cca20a
MC		 2515 u64 extra_bits = valid_bit;
2516
6fe19886
MC		 2517 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2518 rmem->page_size,
2519 &rmem->dma_arr[i],
c0c050c5 2520 GFP_KERNEL);
6fe19886 2521 if (!rmem->pg_arr[i])
c0c050c5
MC		 2522 return -ENOMEM;
2523
4f49b2b8 2524 if (rmem->nr_pages > 1 || rmem->depth > 0) {
66cca20a
MC		 2525 if (i == rmem->nr_pages - 2 &&
2526 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2527 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2528 else if (i == rmem->nr_pages - 1 &&
2529 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2530 extra_bits |= PTU_PTE_LAST;
2531 rmem->pg_tbl[i] =
2532 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2533 }
c0c050c5
MC		 2534 }
2535
6fe19886
MC		 2536 if (rmem->vmem_size) {
2537 *rmem->vmem = vzalloc(rmem->vmem_size);
2538 if (!(*rmem->vmem))
c0c050c5
MC		 2539 return -ENOMEM;
2540 }
2541 return 0;
2542}
2543
4a228a3a
MC		 2544static void bnxt_free_tpa_info(struct bnxt *bp)
2545{
2546 int i;
2547
2548 for (i = 0; i < bp->rx_nr_rings; i++) {
2549 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2550
79632e9b
MC		 2551 if (rxr->rx_tpa) {
2552 kfree(rxr->rx_tpa[0].agg_arr);
2553 rxr->rx_tpa[0].agg_arr = NULL;
2554 }
4a228a3a
MC		 2555 kfree(rxr->rx_tpa);
2556 rxr->rx_tpa = NULL;
2557 }
2558}
2559
2560static int bnxt_alloc_tpa_info(struct bnxt *bp)
2561{
79632e9b
MC		 2562 int i, j, total_aggs = 0;
2563
2564 bp->max_tpa = MAX_TPA;
2565 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2566 if (!bp->max_tpa_v2)
2567 return 0;
2568 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2569 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2570 }
4a228a3a
MC		 2571
2572 for (i = 0; i < bp->rx_nr_rings; i++) {
2573 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
79632e9b 2574 struct rx_agg_cmp *agg;
4a228a3a 2575
79632e9b 2576 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
4a228a3a
MC		 2577 GFP_KERNEL);
2578 if (!rxr->rx_tpa)
2579 return -ENOMEM;
79632e9b
MC		 2580
2581 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2582 continue;
2583 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2584 rxr->rx_tpa[0].agg_arr = agg;
2585 if (!agg)
2586 return -ENOMEM;
2587 for (j = 1; j < bp->max_tpa; j++)
2588 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
4a228a3a
MC		 2589 }
2590 return 0;
2591}
2592
c0c050c5
MC		 2593static void bnxt_free_rx_rings(struct bnxt *bp)
2594{
2595 int i;
2596
b6ab4b01 2597 if (!bp->rx_ring)
c0c050c5
MC		 2598 return;
2599
4a228a3a 2600 bnxt_free_tpa_info(bp);
c0c050c5 2601 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2602 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2603 struct bnxt_ring_struct *ring;
2604
c6d30e83
MC		 2605 if (rxr->xdp_prog)
2606 bpf_prog_put(rxr->xdp_prog);
2607
96a8604f
JDB		 2608 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2609 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2610
12479f62 2611 page_pool_destroy(rxr->page_pool);
322b87ca
AG		 2612 rxr->page_pool = NULL;
2613
c0c050c5
MC		 2614 kfree(rxr->rx_agg_bmap);
2615 rxr->rx_agg_bmap = NULL;
2616
2617 ring = &rxr->rx_ring_struct;
6fe19886 2618 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2619
2620 ring = &rxr->rx_agg_ring_struct;
6fe19886 2621 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2622 }
2623}
2624
322b87ca
AG		 2625static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2626 struct bnxt_rx_ring_info *rxr)
2627{
2628 struct page_pool_params pp = { 0 };
2629
2630 pp.pool_size = bp->rx_ring_size;
2631 pp.nid = dev_to_node(&bp->pdev->dev);
2632 pp.dev = &bp->pdev->dev;
2633 pp.dma_dir = DMA_BIDIRECTIONAL;
2634
2635 rxr->page_pool = page_pool_create(&pp);
2636 if (IS_ERR(rxr->page_pool)) {
2637 int err = PTR_ERR(rxr->page_pool);
2638
2639 rxr->page_pool = NULL;
2640 return err;
2641 }
2642 return 0;
2643}
2644
c0c050c5
MC		 2645static int bnxt_alloc_rx_rings(struct bnxt *bp)
2646{
4a228a3a 2647 int i, rc = 0, agg_rings = 0;
c0c050c5 2648
b6ab4b01
MC		 2649 if (!bp->rx_ring)
2650 return -ENOMEM;
2651
c0c050c5
MC		 2652 if (bp->flags & BNXT_FLAG_AGG_RINGS)
2653 agg_rings = 1;
2654
c0c050c5 2655 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2656 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2657 struct bnxt_ring_struct *ring;
2658
c0c050c5
MC		 2659 ring = &rxr->rx_ring_struct;
2660
322b87ca
AG		 2661 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2662 if (rc)
2663 return rc;
2664
96a8604f 2665 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
12479f62 2666 if (rc < 0)
96a8604f
JDB		 2667 return rc;
2668
f18c2b77 2669 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
322b87ca
AG		 2670 MEM_TYPE_PAGE_POOL,
2671 rxr->page_pool);
f18c2b77
AG		 2672 if (rc) {
2673 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2674 return rc;
2675 }
2676
6fe19886 2677 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2678 if (rc)
2679 return rc;
2680
2c61d211 2681 ring->grp_idx = i;
c0c050c5
MC		 2682 if (agg_rings) {
2683 u16 mem_size;
2684
2685 ring = &rxr->rx_agg_ring_struct;
6fe19886 2686 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2687 if (rc)
2688 return rc;
2689
9899bb59 2690 ring->grp_idx = i;
c0c050c5
MC		 2691 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2692 mem_size = rxr->rx_agg_bmap_size / 8;
2693 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2694 if (!rxr->rx_agg_bmap)
2695 return -ENOMEM;
c0c050c5
MC		 2696 }
2697 }
4a228a3a
MC		 2698 if (bp->flags & BNXT_FLAG_TPA)
2699 rc = bnxt_alloc_tpa_info(bp);
2700 return rc;
c0c050c5
MC		 2701}
2702
2703static void bnxt_free_tx_rings(struct bnxt *bp)
2704{
2705 int i;
2706 struct pci_dev *pdev = bp->pdev;
2707
b6ab4b01 2708 if (!bp->tx_ring)
c0c050c5
MC		 2709 return;
2710
2711 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2712 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2713 struct bnxt_ring_struct *ring;
2714
c0c050c5
MC		 2715 if (txr->tx_push) {
2716 dma_free_coherent(&pdev->dev, bp->tx_push_size,
2717 txr->tx_push, txr->tx_push_mapping);
2718 txr->tx_push = NULL;
2719 }
2720
2721 ring = &txr->tx_ring_struct;
2722
6fe19886 2723 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2724 }
2725}
2726
2727static int bnxt_alloc_tx_rings(struct bnxt *bp)
2728{
2729 int i, j, rc;
2730 struct pci_dev *pdev = bp->pdev;
2731
2732 bp->tx_push_size = 0;
2733 if (bp->tx_push_thresh) {
2734 int push_size;
2735
2736 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2737 bp->tx_push_thresh);
2738
4419dbe6 2739 if (push_size > 256) {
c0c050c5
MC		 2740 push_size = 0;
2741 bp->tx_push_thresh = 0;
2742 }
2743
2744 bp->tx_push_size = push_size;
2745 }
2746
2747 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2748 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5 2749 struct bnxt_ring_struct *ring;
2e8ef77e 2750 u8 qidx;
c0c050c5 2751
c0c050c5
MC		 2752 ring = &txr->tx_ring_struct;
2753
6fe19886 2754 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2755 if (rc)
2756 return rc;
2757
9899bb59 2758 ring->grp_idx = txr->bnapi->index;
c0c050c5 2759 if (bp->tx_push_size) {
c0c050c5
MC		 2760 dma_addr_t mapping;
2761
2762 /* One pre-allocated DMA buffer to backup
2763 * TX push operation
2764 */
2765 txr->tx_push = dma_alloc_coherent(&pdev->dev,
2766 bp->tx_push_size,
2767 &txr->tx_push_mapping,
2768 GFP_KERNEL);
2769
2770 if (!txr->tx_push)
2771 return -ENOMEM;
2772
c0c050c5
MC		 2773 mapping = txr->tx_push_mapping +
2774 sizeof(struct tx_push_bd);
4419dbe6 2775 txr->data_mapping = cpu_to_le64(mapping);
c0c050c5 2776 }
2e8ef77e
MC		 2777 qidx = bp->tc_to_qidx[j];
2778 ring->queue_id = bp->q_info[qidx].queue_id;
5f449249
MC		 2779 if (i < bp->tx_nr_rings_xdp)
2780 continue;
c0c050c5
MC		 2781 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2782 j++;
2783 }
2784 return 0;
2785}
2786
2787static void bnxt_free_cp_rings(struct bnxt *bp)
2788{
2789 int i;
2790
2791 if (!bp->bnapi)
2792 return;
2793
2794 for (i = 0; i < bp->cp_nr_rings; i++) {
2795 struct bnxt_napi *bnapi = bp->bnapi[i];
2796 struct bnxt_cp_ring_info *cpr;
2797 struct bnxt_ring_struct *ring;
50e3ab78 2798 int j;
c0c050c5
MC		 2799
2800 if (!bnapi)
2801 continue;
2802
2803 cpr = &bnapi->cp_ring;
2804 ring = &cpr->cp_ring_struct;
2805
6fe19886 2806 bnxt_free_ring(bp, &ring->ring_mem);
50e3ab78
MC		 2807
2808 for (j = 0; j < 2; j++) {
2809 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2810
2811 if (cpr2) {
2812 ring = &cpr2->cp_ring_struct;
2813 bnxt_free_ring(bp, &ring->ring_mem);
2814 kfree(cpr2);
2815 cpr->cp_ring_arr[j] = NULL;
2816 }
2817 }
c0c050c5
MC		 2818 }
2819}
2820
50e3ab78
MC		 2821static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2822{
2823 struct bnxt_ring_mem_info *rmem;
2824 struct bnxt_ring_struct *ring;
2825 struct bnxt_cp_ring_info *cpr;
2826 int rc;
2827
2828 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
2829 if (!cpr)
2830 return NULL;
2831
2832 ring = &cpr->cp_ring_struct;
2833 rmem = &ring->ring_mem;
2834 rmem->nr_pages = bp->cp_nr_pages;
2835 rmem->page_size = HW_CMPD_RING_SIZE;
2836 rmem->pg_arr = (void **)cpr->cp_desc_ring;
2837 rmem->dma_arr = cpr->cp_desc_mapping;
2838 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
2839 rc = bnxt_alloc_ring(bp, rmem);
2840 if (rc) {
2841 bnxt_free_ring(bp, rmem);
2842 kfree(cpr);
2843 cpr = NULL;
2844 }
2845 return cpr;
2846}
2847
c0c050c5
MC		 2848static int bnxt_alloc_cp_rings(struct bnxt *bp)
2849{
50e3ab78 2850 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
e5811b8c 2851 int i, rc, ulp_base_vec, ulp_msix;
c0c050c5 2852
e5811b8c
MC		 2853 ulp_msix = bnxt_get_ulp_msix_num(bp);
2854 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
c0c050c5
MC		 2855 for (i = 0; i < bp->cp_nr_rings; i++) {
2856 struct bnxt_napi *bnapi = bp->bnapi[i];
2857 struct bnxt_cp_ring_info *cpr;
2858 struct bnxt_ring_struct *ring;
2859
2860 if (!bnapi)
2861 continue;
2862
2863 cpr = &bnapi->cp_ring;
50e3ab78 2864 cpr->bnapi = bnapi;
c0c050c5
MC		 2865 ring = &cpr->cp_ring_struct;
2866
6fe19886 2867 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2868 if (rc)
2869 return rc;
e5811b8c
MC		 2870
2871 if (ulp_msix && i >= ulp_base_vec)
2872 ring->map_idx = i + ulp_msix;
2873 else
2874 ring->map_idx = i;
50e3ab78
MC		 2875
2876 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2877 continue;
2878
2879 if (i < bp->rx_nr_rings) {
2880 struct bnxt_cp_ring_info *cpr2 =
2881 bnxt_alloc_cp_sub_ring(bp);
2882
2883 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
2884 if (!cpr2)
2885 return -ENOMEM;
2886 cpr2->bnapi = bnapi;
2887 }
2888 if ((sh && i < bp->tx_nr_rings) ||
2889 (!sh && i >= bp->rx_nr_rings)) {
2890 struct bnxt_cp_ring_info *cpr2 =
2891 bnxt_alloc_cp_sub_ring(bp);
2892
2893 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
2894 if (!cpr2)
2895 return -ENOMEM;
2896 cpr2->bnapi = bnapi;
2897 }
c0c050c5
MC		 2898 }
2899 return 0;
2900}
2901
2902static void bnxt_init_ring_struct(struct bnxt *bp)
2903{
2904 int i;
2905
2906 for (i = 0; i < bp->cp_nr_rings; i++) {
2907 struct bnxt_napi *bnapi = bp->bnapi[i];
6fe19886 2908 struct bnxt_ring_mem_info *rmem;
c0c050c5
MC		 2909 struct bnxt_cp_ring_info *cpr;
2910 struct bnxt_rx_ring_info *rxr;
2911 struct bnxt_tx_ring_info *txr;
2912 struct bnxt_ring_struct *ring;
2913
2914 if (!bnapi)
2915 continue;
2916
2917 cpr = &bnapi->cp_ring;
2918 ring = &cpr->cp_ring_struct;
6fe19886
MC		 2919 rmem = &ring->ring_mem;
2920 rmem->nr_pages = bp->cp_nr_pages;
2921 rmem->page_size = HW_CMPD_RING_SIZE;
2922 rmem->pg_arr = (void **)cpr->cp_desc_ring;
2923 rmem->dma_arr = cpr->cp_desc_mapping;
2924 rmem->vmem_size = 0;
c0c050c5 2925
b6ab4b01 2926 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 2927 if (!rxr)
2928 goto skip_rx;
2929
c0c050c5 2930 ring = &rxr->rx_ring_struct;
6fe19886
MC		 2931 rmem = &ring->ring_mem;
2932 rmem->nr_pages = bp->rx_nr_pages;
2933 rmem->page_size = HW_RXBD_RING_SIZE;
2934 rmem->pg_arr = (void **)rxr->rx_desc_ring;
2935 rmem->dma_arr = rxr->rx_desc_mapping;
2936 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2937 rmem->vmem = (void **)&rxr->rx_buf_ring;
c0c050c5
MC		 2938
2939 ring = &rxr->rx_agg_ring_struct;
6fe19886
MC		 2940 rmem = &ring->ring_mem;
2941 rmem->nr_pages = bp->rx_agg_nr_pages;
2942 rmem->page_size = HW_RXBD_RING_SIZE;
2943 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
2944 rmem->dma_arr = rxr->rx_agg_desc_mapping;
2945 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2946 rmem->vmem = (void **)&rxr->rx_agg_ring;
c0c050c5 2947
3b2b7d9d 2948skip_rx:
b6ab4b01 2949 txr = bnapi->tx_ring;
3b2b7d9d
MC		 2950 if (!txr)
2951 continue;
2952
c0c050c5 2953 ring = &txr->tx_ring_struct;
6fe19886
MC		 2954 rmem = &ring->ring_mem;
2955 rmem->nr_pages = bp->tx_nr_pages;
2956 rmem->page_size = HW_RXBD_RING_SIZE;
2957 rmem->pg_arr = (void **)txr->tx_desc_ring;
2958 rmem->dma_arr = txr->tx_desc_mapping;
2959 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2960 rmem->vmem = (void **)&txr->tx_buf_ring;
c0c050c5
MC		 2961 }
2962}
2963
2964static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2965{
2966 int i;
2967 u32 prod;
2968 struct rx_bd **rx_buf_ring;
2969
6fe19886
MC		 2970 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
2971 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
c0c050c5
MC		 2972 int j;
2973 struct rx_bd *rxbd;
2974
2975 rxbd = rx_buf_ring[i];
2976 if (!rxbd)
2977 continue;
2978
2979 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2980 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2981 rxbd->rx_bd_opaque = prod;
2982 }
2983 }
2984}
2985
2986static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2987{
2988 struct net_device *dev = bp->dev;
c0c050c5
MC		 2989 struct bnxt_rx_ring_info *rxr;
2990 struct bnxt_ring_struct *ring;
2991 u32 prod, type;
2992 int i;
2993
c0c050c5
MC		 2994 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2995 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2996
2997 if (NET_IP_ALIGN == 2)
2998 type |= RX_BD_FLAGS_SOP;
2999
b6ab4b01 3000 rxr = &bp->rx_ring[ring_nr];
c0c050c5
MC		 3001 ring = &rxr->rx_ring_struct;
3002 bnxt_init_rxbd_pages(ring, type);
3003
c6d30e83
MC		 3004 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3005 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
3006 if (IS_ERR(rxr->xdp_prog)) {
3007 int rc = PTR_ERR(rxr->xdp_prog);
3008
3009 rxr->xdp_prog = NULL;
3010 return rc;
3011 }
3012 }
c0c050c5
MC		 3013 prod = rxr->rx_prod;
3014 for (i = 0; i < bp->rx_ring_size; i++) {
3015 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3016 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3017 ring_nr, i, bp->rx_ring_size);
3018 break;
3019 }
3020 prod = NEXT_RX(prod);
3021 }
3022 rxr->rx_prod = prod;
3023 ring->fw_ring_id = INVALID_HW_RING_ID;
3024
edd0c2cc
MC		 3025 ring = &rxr->rx_agg_ring_struct;
3026 ring->fw_ring_id = INVALID_HW_RING_ID;
3027
c0c050c5
MC		 3028 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3029 return 0;
3030
2839f28b 3031 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
c0c050c5
MC		 3032 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3033
3034 bnxt_init_rxbd_pages(ring, type);
3035
3036 prod = rxr->rx_agg_prod;
3037 for (i = 0; i < bp->rx_agg_ring_size; i++) {
3038 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3039 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3040 ring_nr, i, bp->rx_ring_size);
3041 break;
3042 }
3043 prod = NEXT_RX_AGG(prod);
3044 }
3045 rxr->rx_agg_prod = prod;
c0c050c5
MC		 3046
3047 if (bp->flags & BNXT_FLAG_TPA) {
3048 if (rxr->rx_tpa) {
3049 u8 *data;
3050 dma_addr_t mapping;
3051
79632e9b 3052 for (i = 0; i < bp->max_tpa; i++) {
c0c050c5
MC		 3053 data = __bnxt_alloc_rx_data(bp, &mapping,
3054 GFP_KERNEL);
3055 if (!data)
3056 return -ENOMEM;
3057
3058 rxr->rx_tpa[i].data = data;
b3dba77c 3059 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
c0c050c5
MC		 3060 rxr->rx_tpa[i].mapping = mapping;
3061 }
3062 } else {
3063 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3064 return -ENOMEM;
3065 }
3066 }
3067
3068 return 0;
3069}
3070
2247925f
SP		 3071static void bnxt_init_cp_rings(struct bnxt *bp)
3072{
3e08b184 3073 int i, j;
2247925f
SP		 3074
3075 for (i = 0; i < bp->cp_nr_rings; i++) {
3076 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3077 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3078
3079 ring->fw_ring_id = INVALID_HW_RING_ID;
6a8788f2
AG		 3080 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3081 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3e08b184
MC		 3082 for (j = 0; j < 2; j++) {
3083 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3084
3085 if (!cpr2)
3086 continue;
3087
3088 ring = &cpr2->cp_ring_struct;
3089 ring->fw_ring_id = INVALID_HW_RING_ID;
3090 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3091 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3092 }
2247925f
SP		 3093 }
3094}
3095
c0c050c5
MC		 3096static int bnxt_init_rx_rings(struct bnxt *bp)
3097{
3098 int i, rc = 0;
3099
c61fb99c 3100 if (BNXT_RX_PAGE_MODE(bp)) {
c6d30e83
MC		 3101 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3102 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
c61fb99c
MC		 3103 } else {
3104 bp->rx_offset = BNXT_RX_OFFSET;
3105 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3106 }
b3dba77c 3107
c0c050c5
MC		 3108 for (i = 0; i < bp->rx_nr_rings; i++) {
3109 rc = bnxt_init_one_rx_ring(bp, i);
3110 if (rc)
3111 break;
3112 }
3113
3114 return rc;
3115}
3116
3117static int bnxt_init_tx_rings(struct bnxt *bp)
3118{
3119 u16 i;
3120
3121 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3122 MAX_SKB_FRAGS + 1);
3123
3124 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 3125 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 3126 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3127
3128 ring->fw_ring_id = INVALID_HW_RING_ID;
3129 }
3130
3131 return 0;
3132}
3133
3134static void bnxt_free_ring_grps(struct bnxt *bp)
3135{
3136 kfree(bp->grp_info);
3137 bp->grp_info = NULL;
3138}
3139
3140static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3141{
3142 int i;
3143
3144 if (irq_re_init) {
3145 bp->grp_info = kcalloc(bp->cp_nr_rings,
3146 sizeof(struct bnxt_ring_grp_info),
3147 GFP_KERNEL);
3148 if (!bp->grp_info)
3149 return -ENOMEM;
3150 }
3151 for (i = 0; i < bp->cp_nr_rings; i++) {
3152 if (irq_re_init)
3153 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3154 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3155 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3156 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3157 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3158 }
3159 return 0;
3160}
3161
3162static void bnxt_free_vnics(struct bnxt *bp)
3163{
3164 kfree(bp->vnic_info);
3165 bp->vnic_info = NULL;
3166 bp->nr_vnics = 0;
3167}
3168
3169static int bnxt_alloc_vnics(struct bnxt *bp)
3170{
3171 int num_vnics = 1;
3172
3173#ifdef CONFIG_RFS_ACCEL
9b3d15e6 3174 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
c0c050c5
MC		 3175 num_vnics += bp->rx_nr_rings;
3176#endif
3177
dc52c6c7
PS		 3178 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3179 num_vnics++;
3180
c0c050c5
MC		 3181 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3182 GFP_KERNEL);
3183 if (!bp->vnic_info)
3184 return -ENOMEM;
3185
3186 bp->nr_vnics = num_vnics;
3187 return 0;
3188}
3189
3190static void bnxt_init_vnics(struct bnxt *bp)
3191{
3192 int i;
3193
3194 for (i = 0; i < bp->nr_vnics; i++) {
3195 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
44c6f72a 3196 int j;
c0c050c5
MC		 3197
3198 vnic->fw_vnic_id = INVALID_HW_RING_ID;
44c6f72a
MC		 3199 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3200 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3201
c0c050c5
MC		 3202 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3203
3204 if (bp->vnic_info[i].rss_hash_key) {
3205 if (i == 0)
3206 prandom_bytes(vnic->rss_hash_key,
3207 HW_HASH_KEY_SIZE);
3208 else
3209 memcpy(vnic->rss_hash_key,
3210 bp->vnic_info[0].rss_hash_key,
3211 HW_HASH_KEY_SIZE);
3212 }
3213 }
3214}
3215
3216static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3217{
3218 int pages;
3219
3220 pages = ring_size / desc_per_pg;
3221
3222 if (!pages)
3223 return 1;
3224
3225 pages++;
3226
3227 while (pages & (pages - 1))
3228 pages++;
3229
3230 return pages;
3231}
3232
c6d30e83 3233void bnxt_set_tpa_flags(struct bnxt *bp)
c0c050c5
MC		 3234{
3235 bp->flags &= ~BNXT_FLAG_TPA;
341138c3
MC		 3236 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3237 return;
c0c050c5
MC		 3238 if (bp->dev->features & NETIF_F_LRO)
3239 bp->flags |= BNXT_FLAG_LRO;
1054aee8 3240 else if (bp->dev->features & NETIF_F_GRO_HW)
c0c050c5
MC		 3241 bp->flags |= BNXT_FLAG_GRO;
3242}
3243
3244/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3245 * be set on entry.
3246 */
3247void bnxt_set_ring_params(struct bnxt *bp)
3248{
3249 u32 ring_size, rx_size, rx_space;
3250 u32 agg_factor = 0, agg_ring_size = 0;
3251
3252 /* 8 for CRC and VLAN */
3253 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3254
3255 rx_space = rx_size + NET_SKB_PAD +
3256 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3257
3258 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3259 ring_size = bp->rx_ring_size;
3260 bp->rx_agg_ring_size = 0;
3261 bp->rx_agg_nr_pages = 0;
3262
3263 if (bp->flags & BNXT_FLAG_TPA)
2839f28b 3264 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
c0c050c5
MC		 3265
3266 bp->flags &= ~BNXT_FLAG_JUMBO;
bdbd1eb5 3267 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
c0c050c5
MC		 3268 u32 jumbo_factor;
3269
3270 bp->flags |= BNXT_FLAG_JUMBO;
3271 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3272 if (jumbo_factor > agg_factor)
3273 agg_factor = jumbo_factor;
3274 }
3275 agg_ring_size = ring_size * agg_factor;
3276
3277 if (agg_ring_size) {
3278 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3279 RX_DESC_CNT);
3280 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3281 u32 tmp = agg_ring_size;
3282
3283 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3284 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3285 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3286 tmp, agg_ring_size);
3287 }
3288 bp->rx_agg_ring_size = agg_ring_size;
3289 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3290 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3291 rx_space = rx_size + NET_SKB_PAD +
3292 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3293 }
3294
3295 bp->rx_buf_use_size = rx_size;
3296 bp->rx_buf_size = rx_space;
3297
3298 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3299 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3300
3301 ring_size = bp->tx_ring_size;
3302 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3303 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3304
3305 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3306 bp->cp_ring_size = ring_size;
3307
3308 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3309 if (bp->cp_nr_pages > MAX_CP_PAGES) {
3310 bp->cp_nr_pages = MAX_CP_PAGES;
3311 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3312 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3313 ring_size, bp->cp_ring_size);
3314 }
3315 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3316 bp->cp_ring_mask = bp->cp_bit - 1;
3317}
3318
96a8604f
JDB		 3319/* Changing allocation mode of RX rings.
3320 * TODO: Update when extending xdp_rxq_info to support allocation modes.
3321 */
c61fb99c 3322int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
6bb19474 3323{
c61fb99c
MC		 3324 if (page_mode) {
3325 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3326 return -EOPNOTSUPP;
7eb9bb3a
MC		 3327 bp->dev->max_mtu =
3328 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
c61fb99c
MC		 3329 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3330 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
c61fb99c
MC		 3331 bp->rx_dir = DMA_BIDIRECTIONAL;
3332 bp->rx_skb_func = bnxt_rx_page_skb;
1054aee8
MC		 3333 /* Disable LRO or GRO_HW */
3334 netdev_update_features(bp->dev);
c61fb99c 3335 } else {
7eb9bb3a 3336 bp->dev->max_mtu = bp->max_mtu;
c61fb99c
MC		 3337 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3338 bp->rx_dir = DMA_FROM_DEVICE;
3339 bp->rx_skb_func = bnxt_rx_skb;
3340 }
6bb19474
MC		 3341 return 0;
3342}
3343
c0c050c5
MC		 3344static void bnxt_free_vnic_attributes(struct bnxt *bp)
3345{
3346 int i;
3347 struct bnxt_vnic_info *vnic;
3348 struct pci_dev *pdev = bp->pdev;
3349
3350 if (!bp->vnic_info)
3351 return;
3352
3353 for (i = 0; i < bp->nr_vnics; i++) {
3354 vnic = &bp->vnic_info[i];
3355
3356 kfree(vnic->fw_grp_ids);
3357 vnic->fw_grp_ids = NULL;
3358
3359 kfree(vnic->uc_list);
3360 vnic->uc_list = NULL;
3361
3362 if (vnic->mc_list) {
3363 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3364 vnic->mc_list, vnic->mc_list_mapping);
3365 vnic->mc_list = NULL;
3366 }
3367
3368 if (vnic->rss_table) {
3369 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3370 vnic->rss_table,
3371 vnic->rss_table_dma_addr);
3372 vnic->rss_table = NULL;
3373 }
3374
3375 vnic->rss_hash_key = NULL;
3376 vnic->flags = 0;
3377 }
3378}
3379
3380static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3381{
3382 int i, rc = 0, size;
3383 struct bnxt_vnic_info *vnic;
3384 struct pci_dev *pdev = bp->pdev;
3385 int max_rings;
3386
3387 for (i = 0; i < bp->nr_vnics; i++) {
3388 vnic = &bp->vnic_info[i];
3389
3390 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3391 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3392
3393 if (mem_size > 0) {
3394 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3395 if (!vnic->uc_list) {
3396 rc = -ENOMEM;
3397 goto out;
3398 }
3399 }
3400 }
3401
3402 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3403 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3404 vnic->mc_list =
3405 dma_alloc_coherent(&pdev->dev,
3406 vnic->mc_list_size,
3407 &vnic->mc_list_mapping,
3408 GFP_KERNEL);
3409 if (!vnic->mc_list) {
3410 rc = -ENOMEM;
3411 goto out;
3412 }
3413 }
3414
44c6f72a
MC		 3415 if (bp->flags & BNXT_FLAG_CHIP_P5)
3416 goto vnic_skip_grps;
3417
c0c050c5
MC		 3418 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3419 max_rings = bp->rx_nr_rings;
3420 else
3421 max_rings = 1;
3422
3423 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3424 if (!vnic->fw_grp_ids) {
3425 rc = -ENOMEM;
3426 goto out;
3427 }
44c6f72a 3428vnic_skip_grps:
ae10ae74
MC		 3429 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3430 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3431 continue;
3432
c0c050c5
MC		 3433 /* Allocate rss table and hash key */
3434 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3435 &vnic->rss_table_dma_addr,
3436 GFP_KERNEL);
3437 if (!vnic->rss_table) {
3438 rc = -ENOMEM;
3439 goto out;
3440 }
3441
3442 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3443
3444 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3445 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3446 }
3447 return 0;
3448
3449out:
3450 return rc;
3451}
3452
3453static void bnxt_free_hwrm_resources(struct bnxt *bp)
3454{
3455 struct pci_dev *pdev = bp->pdev;
3456
a2bf74f4
VD		 3457 if (bp->hwrm_cmd_resp_addr) {
3458 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3459 bp->hwrm_cmd_resp_dma_addr);
3460 bp->hwrm_cmd_resp_addr = NULL;
3461 }
760b6d33
VD		 3462
3463 if (bp->hwrm_cmd_kong_resp_addr) {
3464 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3465 bp->hwrm_cmd_kong_resp_addr,
3466 bp->hwrm_cmd_kong_resp_dma_addr);
3467 bp->hwrm_cmd_kong_resp_addr = NULL;
3468 }
3469}
3470
3471static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3472{
3473 struct pci_dev *pdev = bp->pdev;
3474
3475 bp->hwrm_cmd_kong_resp_addr =
3476 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3477 &bp->hwrm_cmd_kong_resp_dma_addr,
3478 GFP_KERNEL);
3479 if (!bp->hwrm_cmd_kong_resp_addr)
3480 return -ENOMEM;
3481
3482 return 0;
c0c050c5
MC		 3483}
3484
3485static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3486{
3487 struct pci_dev *pdev = bp->pdev;
3488
3489 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3490 &bp->hwrm_cmd_resp_dma_addr,
3491 GFP_KERNEL);
3492 if (!bp->hwrm_cmd_resp_addr)
3493 return -ENOMEM;
c0c050c5
MC		 3494
3495 return 0;
3496}
3497
e605db80
DK		 3498static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3499{
3500 if (bp->hwrm_short_cmd_req_addr) {
3501 struct pci_dev *pdev = bp->pdev;
3502
1dfddc41 3503 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
e605db80
DK		 3504 bp->hwrm_short_cmd_req_addr,
3505 bp->hwrm_short_cmd_req_dma_addr);
3506 bp->hwrm_short_cmd_req_addr = NULL;
3507 }
3508}
3509
3510static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3511{
3512 struct pci_dev *pdev = bp->pdev;
3513
3514 bp->hwrm_short_cmd_req_addr =
1dfddc41 3515 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
e605db80
DK		 3516 &bp->hwrm_short_cmd_req_dma_addr,
3517 GFP_KERNEL);
3518 if (!bp->hwrm_short_cmd_req_addr)
3519 return -ENOMEM;
3520
3521 return 0;
3522}
3523
fd3ab1c7 3524static void bnxt_free_port_stats(struct bnxt *bp)
c0c050c5 3525{
c0c050c5
MC		 3526 struct pci_dev *pdev = bp->pdev;
3527
00db3cba
VV		 3528 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3529 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3530
3bdf56c4
MC		 3531 if (bp->hw_rx_port_stats) {
3532 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3533 bp->hw_rx_port_stats,
3534 bp->hw_rx_port_stats_map);
3535 bp->hw_rx_port_stats = NULL;
00db3cba
VV		 3536 }
3537
36e53349
MC		 3538 if (bp->hw_tx_port_stats_ext) {
3539 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3540 bp->hw_tx_port_stats_ext,
3541 bp->hw_tx_port_stats_ext_map);
3542 bp->hw_tx_port_stats_ext = NULL;
3543 }
3544
00db3cba
VV		 3545 if (bp->hw_rx_port_stats_ext) {
3546 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3547 bp->hw_rx_port_stats_ext,
3548 bp->hw_rx_port_stats_ext_map);
3549 bp->hw_rx_port_stats_ext = NULL;
3bdf56c4 3550 }
55e4398d
VV		 3551
3552 if (bp->hw_pcie_stats) {
3553 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3554 bp->hw_pcie_stats, bp->hw_pcie_stats_map);
3555 bp->hw_pcie_stats = NULL;
3556 }
fd3ab1c7
MC		 3557}
3558
3559static void bnxt_free_ring_stats(struct bnxt *bp)
3560{
3561 struct pci_dev *pdev = bp->pdev;
3562 int size, i;
3bdf56c4 3563
c0c050c5
MC		 3564 if (!bp->bnapi)
3565 return;
3566
3567 size = sizeof(struct ctx_hw_stats);
3568
3569 for (i = 0; i < bp->cp_nr_rings; i++) {
3570 struct bnxt_napi *bnapi = bp->bnapi[i];
3571 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3572
3573 if (cpr->hw_stats) {
3574 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3575 cpr->hw_stats_map);
3576 cpr->hw_stats = NULL;
3577 }
3578 }
3579}
3580
3581static int bnxt_alloc_stats(struct bnxt *bp)
3582{
3583 u32 size, i;
3584 struct pci_dev *pdev = bp->pdev;
3585
3586 size = sizeof(struct ctx_hw_stats);
3587
3588 for (i = 0; i < bp->cp_nr_rings; i++) {
3589 struct bnxt_napi *bnapi = bp->bnapi[i];
3590 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3591
3592 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3593 &cpr->hw_stats_map,
3594 GFP_KERNEL);
3595 if (!cpr->hw_stats)
3596 return -ENOMEM;
3597
3598 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3599 }
3bdf56c4 3600
a220eabc
VV		 3601 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3602 return 0;
fd3ab1c7 3603
a220eabc
VV		 3604 if (bp->hw_rx_port_stats)
3605 goto alloc_ext_stats;
3bdf56c4 3606
a220eabc
VV		 3607 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3608 sizeof(struct tx_port_stats) + 1024;
3bdf56c4 3609
a220eabc
VV		 3610 bp->hw_rx_port_stats =
3611 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3612 &bp->hw_rx_port_stats_map,
3613 GFP_KERNEL);
3614 if (!bp->hw_rx_port_stats)
3615 return -ENOMEM;
3bdf56c4 3616
a220eabc
VV		 3617 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512;
3618 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3619 sizeof(struct rx_port_stats) + 512;
3620 bp->flags |= BNXT_FLAG_PORT_STATS;
00db3cba 3621
fd3ab1c7 3622alloc_ext_stats:
a220eabc
VV		 3623 /* Display extended statistics only if FW supports it */
3624 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
6154532f 3625 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
00db3cba
VV		 3626 return 0;
3627
a220eabc
VV		 3628 if (bp->hw_rx_port_stats_ext)
3629 goto alloc_tx_ext_stats;
fd3ab1c7 3630
a220eabc
VV		 3631 bp->hw_rx_port_stats_ext =
3632 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3633 &bp->hw_rx_port_stats_ext_map, GFP_KERNEL);
3634 if (!bp->hw_rx_port_stats_ext)
3635 return 0;
00db3cba 3636
fd3ab1c7 3637alloc_tx_ext_stats:
a220eabc 3638 if (bp->hw_tx_port_stats_ext)
55e4398d 3639 goto alloc_pcie_stats;
fd3ab1c7 3640
6154532f
VV		 3641 if (bp->hwrm_spec_code >= 0x10902 ||
3642 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
a220eabc
VV		 3643 bp->hw_tx_port_stats_ext =
3644 dma_alloc_coherent(&pdev->dev,
3645 sizeof(struct tx_port_stats_ext),
3646 &bp->hw_tx_port_stats_ext_map,
3647 GFP_KERNEL);
3bdf56c4 3648 }
a220eabc 3649 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
55e4398d
VV		 3650
3651alloc_pcie_stats:
3652 if (bp->hw_pcie_stats ||
3653 !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED))
3654 return 0;
3655
3656 bp->hw_pcie_stats =
3657 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3658 &bp->hw_pcie_stats_map, GFP_KERNEL);
3659 if (!bp->hw_pcie_stats)
3660 return 0;
3661
3662 bp->flags |= BNXT_FLAG_PCIE_STATS;
c0c050c5
MC		 3663 return 0;
3664}
3665
3666static void bnxt_clear_ring_indices(struct bnxt *bp)
3667{
3668 int i;
3669
3670 if (!bp->bnapi)
3671 return;
3672
3673 for (i = 0; i < bp->cp_nr_rings; i++) {
3674 struct bnxt_napi *bnapi = bp->bnapi[i];
3675 struct bnxt_cp_ring_info *cpr;
3676 struct bnxt_rx_ring_info *rxr;
3677 struct bnxt_tx_ring_info *txr;
3678
3679 if (!bnapi)
3680 continue;
3681
3682 cpr = &bnapi->cp_ring;
3683 cpr->cp_raw_cons = 0;
3684
b6ab4b01 3685 txr = bnapi->tx_ring;
3b2b7d9d
MC		 3686 if (txr) {
3687 txr->tx_prod = 0;
3688 txr->tx_cons = 0;
3689 }
c0c050c5 3690
b6ab4b01 3691 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 3692 if (rxr) {
3693 rxr->rx_prod = 0;
3694 rxr->rx_agg_prod = 0;
3695 rxr->rx_sw_agg_prod = 0;
376a5b86 3696 rxr->rx_next_cons = 0;
3b2b7d9d 3697 }
c0c050c5
MC		 3698 }
3699}
3700
3701static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3702{
3703#ifdef CONFIG_RFS_ACCEL
3704 int i;
3705
3706 /* Under rtnl_lock and all our NAPIs have been disabled. It's
3707 * safe to delete the hash table.
3708 */
3709 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3710 struct hlist_head *head;
3711 struct hlist_node *tmp;
3712 struct bnxt_ntuple_filter *fltr;
3713
3714 head = &bp->ntp_fltr_hash_tbl[i];
3715 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3716 hlist_del(&fltr->hash);
3717 kfree(fltr);
3718 }
3719 }
3720 if (irq_reinit) {
3721 kfree(bp->ntp_fltr_bmap);
3722 bp->ntp_fltr_bmap = NULL;
3723 }
3724 bp->ntp_fltr_count = 0;
3725#endif
3726}
3727
3728static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3729{
3730#ifdef CONFIG_RFS_ACCEL
3731 int i, rc = 0;
3732
3733 if (!(bp->flags & BNXT_FLAG_RFS))
3734 return 0;
3735
3736 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3737 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3738
3739 bp->ntp_fltr_count = 0;
ac45bd93
DC		 3740 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3741 sizeof(long),
c0c050c5
MC		 3742 GFP_KERNEL);
3743
3744 if (!bp->ntp_fltr_bmap)
3745 rc = -ENOMEM;
3746
3747 return rc;
3748#else
3749 return 0;
3750#endif
3751}
3752
3753static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3754{
3755 bnxt_free_vnic_attributes(bp);
3756 bnxt_free_tx_rings(bp);
3757 bnxt_free_rx_rings(bp);
3758 bnxt_free_cp_rings(bp);
3759 bnxt_free_ntp_fltrs(bp, irq_re_init);
3760 if (irq_re_init) {
fd3ab1c7 3761 bnxt_free_ring_stats(bp);
c0c050c5
MC		 3762 bnxt_free_ring_grps(bp);
3763 bnxt_free_vnics(bp);
a960dec9
MC		 3764 kfree(bp->tx_ring_map);
3765 bp->tx_ring_map = NULL;
b6ab4b01
MC		 3766 kfree(bp->tx_ring);
3767 bp->tx_ring = NULL;
3768 kfree(bp->rx_ring);
3769 bp->rx_ring = NULL;
c0c050c5
MC		 3770 kfree(bp->bnapi);
3771 bp->bnapi = NULL;
3772 } else {
3773 bnxt_clear_ring_indices(bp);
3774 }
3775}
3776
3777static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3778{
01657bcd 3779 int i, j, rc, size, arr_size;
c0c050c5
MC		 3780 void *bnapi;
3781
3782 if (irq_re_init) {
3783 /* Allocate bnapi mem pointer array and mem block for
3784 * all queues
3785 */
3786 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3787 bp->cp_nr_rings);
3788 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3789 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3790 if (!bnapi)
3791 return -ENOMEM;
3792
3793 bp->bnapi = bnapi;
3794 bnapi += arr_size;
3795 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3796 bp->bnapi[i] = bnapi;
3797 bp->bnapi[i]->index = i;
3798 bp->bnapi[i]->bp = bp;
e38287b7
MC		 3799 if (bp->flags & BNXT_FLAG_CHIP_P5) {
3800 struct bnxt_cp_ring_info *cpr =
3801 &bp->bnapi[i]->cp_ring;
3802
3803 cpr->cp_ring_struct.ring_mem.flags =
3804 BNXT_RMEM_RING_PTE_FLAG;
3805 }
c0c050c5
MC		 3806 }
3807
b6ab4b01
MC		 3808 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3809 sizeof(struct bnxt_rx_ring_info),
3810 GFP_KERNEL);
3811 if (!bp->rx_ring)
3812 return -ENOMEM;
3813
3814 for (i = 0; i < bp->rx_nr_rings; i++) {
e38287b7
MC		 3815 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3816
3817 if (bp->flags & BNXT_FLAG_CHIP_P5) {
3818 rxr->rx_ring_struct.ring_mem.flags =
3819 BNXT_RMEM_RING_PTE_FLAG;
3820 rxr->rx_agg_ring_struct.ring_mem.flags =
3821 BNXT_RMEM_RING_PTE_FLAG;
3822 }
3823 rxr->bnapi = bp->bnapi[i];
b6ab4b01
MC		 3824 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3825 }
3826
3827 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3828 sizeof(struct bnxt_tx_ring_info),
3829 GFP_KERNEL);
3830 if (!bp->tx_ring)
3831 return -ENOMEM;
3832
a960dec9
MC		 3833 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3834 GFP_KERNEL);
3835
3836 if (!bp->tx_ring_map)
3837 return -ENOMEM;
3838
01657bcd
MC		 3839 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3840 j = 0;
3841 else
3842 j = bp->rx_nr_rings;
3843
3844 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
e38287b7
MC		 3845 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3846
3847 if (bp->flags & BNXT_FLAG_CHIP_P5)
3848 txr->tx_ring_struct.ring_mem.flags =
3849 BNXT_RMEM_RING_PTE_FLAG;
3850 txr->bnapi = bp->bnapi[j];
3851 bp->bnapi[j]->tx_ring = txr;
5f449249 3852 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
38413406 3853 if (i >= bp->tx_nr_rings_xdp) {
e38287b7 3854 txr->txq_index = i - bp->tx_nr_rings_xdp;
38413406
MC		 3855 bp->bnapi[j]->tx_int = bnxt_tx_int;
3856 } else {
fa3e93e8 3857 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
38413406
MC		 3858 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3859 }
b6ab4b01
MC		 3860 }
3861
c0c050c5
MC		 3862 rc = bnxt_alloc_stats(bp);
3863 if (rc)
3864 goto alloc_mem_err;
3865
3866 rc = bnxt_alloc_ntp_fltrs(bp);
3867 if (rc)
3868 goto alloc_mem_err;
3869
3870 rc = bnxt_alloc_vnics(bp);
3871 if (rc)
3872 goto alloc_mem_err;
3873 }
3874
3875 bnxt_init_ring_struct(bp);
3876
3877 rc = bnxt_alloc_rx_rings(bp);
3878 if (rc)
3879 goto alloc_mem_err;
3880
3881 rc = bnxt_alloc_tx_rings(bp);
3882 if (rc)
3883 goto alloc_mem_err;
3884
3885 rc = bnxt_alloc_cp_rings(bp);
3886 if (rc)
3887 goto alloc_mem_err;
3888
3889 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3890 BNXT_VNIC_UCAST_FLAG;
3891 rc = bnxt_alloc_vnic_attributes(bp);
3892 if (rc)
3893 goto alloc_mem_err;
3894 return 0;
3895
3896alloc_mem_err:
3897 bnxt_free_mem(bp, true);
3898 return rc;
3899}
3900
9d8bc097
MC		 3901static void bnxt_disable_int(struct bnxt *bp)
3902{
3903 int i;
3904
3905 if (!bp->bnapi)
3906 return;
3907
3908 for (i = 0; i < bp->cp_nr_rings; i++) {
3909 struct bnxt_napi *bnapi = bp->bnapi[i];
3910 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
daf1f1e7 3911 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
9d8bc097 3912
daf1f1e7 3913 if (ring->fw_ring_id != INVALID_HW_RING_ID)
697197e5 3914 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
9d8bc097
MC		 3915 }
3916}
3917
e5811b8c
MC		 3918static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
3919{
3920 struct bnxt_napi *bnapi = bp->bnapi[n];
3921 struct bnxt_cp_ring_info *cpr;
3922
3923 cpr = &bnapi->cp_ring;
3924 return cpr->cp_ring_struct.map_idx;
3925}
3926
9d8bc097
MC		 3927static void bnxt_disable_int_sync(struct bnxt *bp)
3928{
3929 int i;
3930
3931 atomic_inc(&bp->intr_sem);
3932
3933 bnxt_disable_int(bp);
e5811b8c
MC		 3934 for (i = 0; i < bp->cp_nr_rings; i++) {
3935 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
3936
3937 synchronize_irq(bp->irq_tbl[map_idx].vector);
3938 }
9d8bc097
MC		 3939}
3940
3941static void bnxt_enable_int(struct bnxt *bp)
3942{
3943 int i;
3944
3945 atomic_set(&bp->intr_sem, 0);
3946 for (i = 0; i < bp->cp_nr_rings; i++) {
3947 struct bnxt_napi *bnapi = bp->bnapi[i];
3948 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3949
697197e5 3950 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
9d8bc097
MC		 3951 }
3952}
3953
c0c050c5
MC		 3954void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3955 u16 cmpl_ring, u16 target_id)
3956{
a8643e16 3957 struct input *req = request;
c0c050c5 3958
a8643e16
MC		 3959 req->req_type = cpu_to_le16(req_type);
3960 req->cmpl_ring = cpu_to_le16(cmpl_ring);
3961 req->target_id = cpu_to_le16(target_id);
760b6d33
VD		 3962 if (bnxt_kong_hwrm_message(bp, req))
3963 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
3964 else
3965 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
c0c050c5
MC		 3966}
3967
fbfbc485
MC		 3968static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3969 int timeout, bool silent)
c0c050c5 3970{
a11fa2be 3971 int i, intr_process, rc, tmo_count;
a8643e16 3972 struct input *req = msg;
c0c050c5 3973 u32 *data = msg;
845adfe4
MC		 3974 __le32 *resp_len;
3975 u8 *valid;
c0c050c5
MC		 3976 u16 cp_ring_id, len = 0;
3977 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
e605db80 3978 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
ebd5818c 3979 struct hwrm_short_input short_input = {0};
2e9ee398 3980 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
89455017 3981 u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
2e9ee398 3982 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
760b6d33 3983 u16 dst = BNXT_HWRM_CHNL_CHIMP;
c0c050c5 3984
1dfddc41
MC		 3985 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
3986 if (msg_len > bp->hwrm_max_ext_req_len ||
3987 !bp->hwrm_short_cmd_req_addr)
3988 return -EINVAL;
3989 }
3990
760b6d33
VD		 3991 if (bnxt_hwrm_kong_chnl(bp, req)) {
3992 dst = BNXT_HWRM_CHNL_KONG;
3993 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
3994 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
3995 resp = bp->hwrm_cmd_kong_resp_addr;
3996 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
3997 }
3998
3999 memset(resp, 0, PAGE_SIZE);
4000 cp_ring_id = le16_to_cpu(req->cmpl_ring);
4001 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4002
4003 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4004 /* currently supports only one outstanding message */
4005 if (intr_process)
4006 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4007
1dfddc41
MC		 4008 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4009 msg_len > BNXT_HWRM_MAX_REQ_LEN) {
e605db80 4010 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
1dfddc41
MC		 4011 u16 max_msg_len;
4012
4013 /* Set boundary for maximum extended request length for short
4014 * cmd format. If passed up from device use the max supported
4015 * internal req length.
4016 */
4017 max_msg_len = bp->hwrm_max_ext_req_len;
e605db80
DK		 4018
4019 memcpy(short_cmd_req, req, msg_len);
1dfddc41
MC		 4020 if (msg_len < max_msg_len)
4021 memset(short_cmd_req + msg_len, 0,
4022 max_msg_len - msg_len);
e605db80
DK		 4023
4024 short_input.req_type = req->req_type;
4025 short_input.signature =
4026 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4027 short_input.size = cpu_to_le16(msg_len);
4028 short_input.req_addr =
4029 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4030
4031 data = (u32 *)&short_input;
4032 msg_len = sizeof(short_input);
4033
4034 /* Sync memory write before updating doorbell */
4035 wmb();
4036
4037 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4038 }
4039
c0c050c5 4040 /* Write request msg to hwrm channel */
2e9ee398 4041 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
c0c050c5 4042
e605db80 4043 for (i = msg_len; i < max_req_len; i += 4)
2e9ee398 4044 writel(0, bp->bar0 + bar_offset + i);
d79979a1 4045
c0c050c5 4046 /* Ring channel doorbell */
2e9ee398 4047 writel(1, bp->bar0 + doorbell_offset);
c0c050c5 4048
ff4fe81d
MC		 4049 if (!timeout)
4050 timeout = DFLT_HWRM_CMD_TIMEOUT;
9751e8e7
AG		 4051 /* convert timeout to usec */
4052 timeout *= 1000;
ff4fe81d 4053
c0c050c5 4054 i = 0;
9751e8e7
AG		 4055 /* Short timeout for the first few iterations:
4056 * number of loops = number of loops for short timeout +
4057 * number of loops for standard timeout.
4058 */
4059 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4060 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4061 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
89455017
VD		 4062 resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
4063
c0c050c5 4064 if (intr_process) {
fc718bb2
VD		 4065 u16 seq_id = bp->hwrm_intr_seq_id;
4066
c0c050c5 4067 /* Wait until hwrm response cmpl interrupt is processed */
fc718bb2 4068 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
a11fa2be 4069 i++ < tmo_count) {
9751e8e7
AG		 4070 /* on first few passes, just barely sleep */
4071 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4072 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4073 HWRM_SHORT_MAX_TIMEOUT);
4074 else
4075 usleep_range(HWRM_MIN_TIMEOUT,
4076 HWRM_MAX_TIMEOUT);
c0c050c5
MC		 4077 }
4078
fc718bb2 4079 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
c0c050c5 4080 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
a8643e16 4081 le16_to_cpu(req->req_type));
c0c050c5
MC		 4082 return -1;
4083 }
845adfe4
MC		 4084 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4085 HWRM_RESP_LEN_SFT;
89455017 4086 valid = resp_addr + len - 1;
c0c050c5 4087 } else {
cc559c1a
MC		 4088 int j;
4089
c0c050c5 4090 /* Check if response len is updated */
a11fa2be 4091 for (i = 0; i < tmo_count; i++) {
c0c050c5
MC		 4092 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4093 HWRM_RESP_LEN_SFT;
4094 if (len)
4095 break;
9751e8e7 4096 /* on first few passes, just barely sleep */
67681d02 4097 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
9751e8e7
AG		 4098 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4099 HWRM_SHORT_MAX_TIMEOUT);
4100 else
4101 usleep_range(HWRM_MIN_TIMEOUT,
4102 HWRM_MAX_TIMEOUT);
c0c050c5
MC		 4103 }
4104
a11fa2be 4105 if (i >= tmo_count) {
c0c050c5 4106 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
cc559c1a
MC		 4107 HWRM_TOTAL_TIMEOUT(i),
4108 le16_to_cpu(req->req_type),
8578d6c1 4109 le16_to_cpu(req->seq_id), len);
c0c050c5
MC		 4110 return -1;
4111 }
4112
845adfe4 4113 /* Last byte of resp contains valid bit */
89455017 4114 valid = resp_addr + len - 1;
cc559c1a 4115 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
845adfe4
MC		 4116 /* make sure we read from updated DMA memory */
4117 dma_rmb();
4118 if (*valid)
c0c050c5 4119 break;
0000b81a 4120 usleep_range(1, 5);
c0c050c5
MC		 4121 }
4122
cc559c1a 4123 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
c0c050c5 4124 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
cc559c1a
MC		 4125 HWRM_TOTAL_TIMEOUT(i),
4126 le16_to_cpu(req->req_type),
a8643e16 4127 le16_to_cpu(req->seq_id), len, *valid);
c0c050c5
MC		 4128 return -1;
4129 }
4130 }
4131
845adfe4
MC		 4132 /* Zero valid bit for compatibility. Valid bit in an older spec
4133 * may become a new field in a newer spec. We must make sure that
4134 * a new field not implemented by old spec will read zero.
4135 */
4136 *valid = 0;
c0c050c5 4137 rc = le16_to_cpu(resp->error_code);
fbfbc485 4138 if (rc && !silent)
c0c050c5
MC		 4139 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4140 le16_to_cpu(resp->req_type),
4141 le16_to_cpu(resp->seq_id), rc);
fbfbc485
MC		 4142 return rc;
4143}
4144
4145int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4146{
4147 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
c0c050c5
MC		 4148}
4149
cc72f3b1
MC		 4150int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4151 int timeout)
4152{
4153 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4154}
4155
c0c050c5
MC		 4156int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4157{
4158 int rc;
4159
4160 mutex_lock(&bp->hwrm_cmd_lock);
4161 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4162 mutex_unlock(&bp->hwrm_cmd_lock);
4163 return rc;
4164}
4165
90e20921
MC		 4166int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4167 int timeout)
4168{
4169 int rc;
4170
4171 mutex_lock(&bp->hwrm_cmd_lock);
4172 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4173 mutex_unlock(&bp->hwrm_cmd_lock);
4174 return rc;
4175}
4176
a1653b13
MC		 4177int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
4178 int bmap_size)
c0c050c5
MC		 4179{
4180 struct hwrm_func_drv_rgtr_input req = {0};
25be8623
MC		 4181 DECLARE_BITMAP(async_events_bmap, 256);
4182 u32 *events = (u32 *)async_events_bmap;
a1653b13 4183 int i;
c0c050c5
MC		 4184
4185 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4186
4187 req.enables =
a1653b13 4188 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
c0c050c5 4189
25be8623
MC		 4190 memset(async_events_bmap, 0, sizeof(async_events_bmap));
4191 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
4192 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4193
a1653b13
MC		 4194 if (bmap && bmap_size) {
4195 for (i = 0; i < bmap_size; i++) {
4196 if (test_bit(i, bmap))
4197 __set_bit(i, async_events_bmap);
4198 }
4199 }
4200
25be8623
MC		 4201 for (i = 0; i < 8; i++)
4202 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4203
a1653b13
MC		 4204 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4205}
4206
4207static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
4208{
25e1acd6 4209 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
a1653b13 4210 struct hwrm_func_drv_rgtr_input req = {0};
25e1acd6 4211 int rc;
a1653b13
MC		 4212
4213 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4214
4215 req.enables =
4216 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4217 FUNC_DRV_RGTR_REQ_ENABLES_VER);
4218
11f15ed3 4219 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
d4f52de0
MC		 4220 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
4221 req.ver_maj_8b = DRV_VER_MAJ;
4222 req.ver_min_8b = DRV_VER_MIN;
4223 req.ver_upd_8b = DRV_VER_UPD;
4224 req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4225 req.ver_min = cpu_to_le16(DRV_VER_MIN);
4226 req.ver_upd = cpu_to_le16(DRV_VER_UPD);
c0c050c5
MC		 4227
4228 if (BNXT_PF(bp)) {
9b0436c3 4229 u32 data[8];
a1653b13 4230 int i;
c0c050c5 4231
9b0436c3
MC		 4232 memset(data, 0, sizeof(data));
4233 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4234 u16 cmd = bnxt_vf_req_snif[i];
4235 unsigned int bit, idx;
4236
4237 idx = cmd / 32;
4238 bit = cmd % 32;
4239 data[idx] |= 1 << bit;
4240 }
c0c050c5 4241
de68f5de
MC		 4242 for (i = 0; i < 8; i++)
4243 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4244
c0c050c5
MC		 4245 req.enables |=
4246 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4247 }
4248
abd43a13
VD		 4249 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4250 req.flags |= cpu_to_le32(
4251 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4252
25e1acd6
MC		 4253 mutex_lock(&bp->hwrm_cmd_lock);
4254 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4255 if (rc)
4256 rc = -EIO;
4257 else if (resp->flags &
4258 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4259 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4260 mutex_unlock(&bp->hwrm_cmd_lock);
4261 return rc;
c0c050c5
MC		 4262}
4263
be58a0da
JH		 4264static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4265{
4266 struct hwrm_func_drv_unrgtr_input req = {0};
4267
4268 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4269 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4270}
4271
c0c050c5
MC		 4272static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4273{
4274 u32 rc = 0;
4275 struct hwrm_tunnel_dst_port_free_input req = {0};
4276
4277 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4278 req.tunnel_type = tunnel_type;
4279
4280 switch (tunnel_type) {
4281 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4282 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4283 break;
4284 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4285 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4286 break;
4287 default:
4288 break;
4289 }
4290
4291 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4292 if (rc)
4293 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4294 rc);
4295 return rc;
4296}
4297
4298static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4299 u8 tunnel_type)
4300{
4301 u32 rc = 0;
4302 struct hwrm_tunnel_dst_port_alloc_input req = {0};
4303 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4304
4305 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4306
4307 req.tunnel_type = tunnel_type;
4308 req.tunnel_dst_port_val = port;
4309
4310 mutex_lock(&bp->hwrm_cmd_lock);
4311 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4312 if (rc) {
4313 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4314 rc);
4315 goto err_out;
4316 }
4317
57aac71b
CJ		 4318 switch (tunnel_type) {
4319 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
c0c050c5 4320 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 4321 break;
4322 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
c0c050c5 4323 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 4324 break;
4325 default:
4326 break;
4327 }
4328
c0c050c5
MC		 4329err_out:
4330 mutex_unlock(&bp->hwrm_cmd_lock);
4331 return rc;
4332}
4333
4334static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4335{
4336 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4337 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4338
4339 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
c193554e 4340 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
c0c050c5
MC		 4341
4342 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4343 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4344 req.mask = cpu_to_le32(vnic->rx_mask);
4345 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4346}
4347
4348#ifdef CONFIG_RFS_ACCEL
4349static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4350 struct bnxt_ntuple_filter *fltr)
4351{
4352 struct hwrm_cfa_ntuple_filter_free_input req = {0};
4353
4354 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4355 req.ntuple_filter_id = fltr->filter_id;
4356 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4357}
4358
4359#define BNXT_NTP_FLTR_FLAGS \
4360 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
4361 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
4362 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
4363 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
4364 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
4365 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
4366 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
4367 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
4368 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
4369 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
4370 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
4371 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
4372 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
c193554e 4373 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
c0c050c5 4374
61aad724
MC		 4375#define BNXT_NTP_TUNNEL_FLTR_FLAG \
4376 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4377
c0c050c5
MC		 4378static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4379 struct bnxt_ntuple_filter *fltr)
4380{
c0c050c5 4381 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
5c209fc8 4382 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
c0c050c5 4383 struct flow_keys *keys = &fltr->fkeys;
ac33906c
MC		 4384 struct bnxt_vnic_info *vnic;
4385 u32 dst_ena = 0;
5c209fc8 4386 int rc = 0;
c0c050c5
MC		 4387
4388 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
a54c4d74 4389 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
c0c050c5 4390
ac33906c
MC		 4391 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX) {
4392 dst_ena = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
4393 req.rfs_ring_tbl_idx = cpu_to_le16(fltr->rxq);
4394 vnic = &bp->vnic_info[0];
4395 } else {
4396 vnic = &bp->vnic_info[fltr->rxq + 1];
4397 }
4398 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4399 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS | dst_ena);
c0c050c5
MC		 4400
4401 req.ethertype = htons(ETH_P_IP);
4402 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
c193554e 4403 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
c0c050c5
MC		 4404 req.ip_protocol = keys->basic.ip_proto;
4405
dda0e746
MC		 4406 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4407 int i;
4408
4409 req.ethertype = htons(ETH_P_IPV6);
4410 req.ip_addr_type =
4411 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4412 *(struct in6_addr *)&req.src_ipaddr[0] =
4413 keys->addrs.v6addrs.src;
4414 *(struct in6_addr *)&req.dst_ipaddr[0] =
4415 keys->addrs.v6addrs.dst;
4416 for (i = 0; i < 4; i++) {
4417 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4418 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4419 }
4420 } else {
4421 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4422 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4423 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4424 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4425 }
61aad724
MC		 4426 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4427 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4428 req.tunnel_type =
4429 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4430 }
c0c050c5
MC		 4431
4432 req.src_port = keys->ports.src;
4433 req.src_port_mask = cpu_to_be16(0xffff);
4434 req.dst_port = keys->ports.dst;
4435 req.dst_port_mask = cpu_to_be16(0xffff);
4436
c0c050c5
MC		 4437 mutex_lock(&bp->hwrm_cmd_lock);
4438 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5c209fc8
VD		 4439 if (!rc) {
4440 resp = bnxt_get_hwrm_resp_addr(bp, &req);
c0c050c5 4441 fltr->filter_id = resp->ntuple_filter_id;
5c209fc8 4442 }
c0c050c5
MC		 4443 mutex_unlock(&bp->hwrm_cmd_lock);
4444 return rc;
4445}
4446#endif
4447
4448static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4449 u8 *mac_addr)
4450{
4451 u32 rc = 0;
4452 struct hwrm_cfa_l2_filter_alloc_input req = {0};
4453 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4454
4455 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
dc52c6c7
PS		 4456 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4457 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4458 req.flags |=
4459 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
c193554e 4460 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
c0c050c5
MC		 4461 req.enables =
4462 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
c193554e 4463 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
c0c050c5
MC		 4464 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4465 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4466 req.l2_addr_mask[0] = 0xff;
4467 req.l2_addr_mask[1] = 0xff;
4468 req.l2_addr_mask[2] = 0xff;
4469 req.l2_addr_mask[3] = 0xff;
4470 req.l2_addr_mask[4] = 0xff;
4471 req.l2_addr_mask[5] = 0xff;
4472
4473 mutex_lock(&bp->hwrm_cmd_lock);
4474 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4475 if (!rc)
4476 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4477 resp->l2_filter_id;
4478 mutex_unlock(&bp->hwrm_cmd_lock);
4479 return rc;
4480}
4481
4482static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4483{
4484 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4485 int rc = 0;
4486
4487 /* Any associated ntuple filters will also be cleared by firmware. */
4488 mutex_lock(&bp->hwrm_cmd_lock);
4489 for (i = 0; i < num_of_vnics; i++) {
4490 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4491
4492 for (j = 0; j < vnic->uc_filter_count; j++) {
4493 struct hwrm_cfa_l2_filter_free_input req = {0};
4494
4495 bnxt_hwrm_cmd_hdr_init(bp, &req,
4496 HWRM_CFA_L2_FILTER_FREE, -1, -1);
4497
4498 req.l2_filter_id = vnic->fw_l2_filter_id[j];
4499
4500 rc = _hwrm_send_message(bp, &req, sizeof(req),
4501 HWRM_CMD_TIMEOUT);
4502 }
4503 vnic->uc_filter_count = 0;
4504 }
4505 mutex_unlock(&bp->hwrm_cmd_lock);
4506
4507 return rc;
4508}
4509
4510static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4511{
4512 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
79632e9b 4513 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
c0c050c5
MC		 4514 struct hwrm_vnic_tpa_cfg_input req = {0};
4515
3c4fe80b
MC		 4516 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4517 return 0;
4518
c0c050c5
MC		 4519 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4520
4521 if (tpa_flags) {
4522 u16 mss = bp->dev->mtu - 40;
4523 u32 nsegs, n, segs = 0, flags;
4524
4525 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4526 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4527 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4528 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4529 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4530 if (tpa_flags & BNXT_FLAG_GRO)
4531 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4532
4533 req.flags = cpu_to_le32(flags);
4534
4535 req.enables =
4536 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
c193554e
MC		 4537 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4538 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
c0c050c5
MC		 4539
4540 /* Number of segs are log2 units, and first packet is not
4541 * included as part of this units.
4542 */
2839f28b
MC		 4543 if (mss <= BNXT_RX_PAGE_SIZE) {
4544 n = BNXT_RX_PAGE_SIZE / mss;
c0c050c5
MC		 4545 nsegs = (MAX_SKB_FRAGS - 1) * n;
4546 } else {
2839f28b
MC		 4547 n = mss / BNXT_RX_PAGE_SIZE;
4548 if (mss & (BNXT_RX_PAGE_SIZE - 1))
c0c050c5
MC		 4549 n++;
4550 nsegs = (MAX_SKB_FRAGS - n) / n;
4551 }
4552
79632e9b
MC		 4553 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4554 segs = MAX_TPA_SEGS_P5;
4555 max_aggs = bp->max_tpa;
4556 } else {
4557 segs = ilog2(nsegs);
4558 }
c0c050c5 4559 req.max_agg_segs = cpu_to_le16(segs);
79632e9b 4560 req.max_aggs = cpu_to_le16(max_aggs);
c193554e
MC		 4561
4562 req.min_agg_len = cpu_to_le32(512);
c0c050c5
MC		 4563 }
4564 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4565
4566 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4567}
4568
2c61d211
MC		 4569static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4570{
4571 struct bnxt_ring_grp_info *grp_info;
4572
4573 grp_info = &bp->grp_info[ring->grp_idx];
4574 return grp_info->cp_fw_ring_id;
4575}
4576
4577static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4578{
4579 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4580 struct bnxt_napi *bnapi = rxr->bnapi;
4581 struct bnxt_cp_ring_info *cpr;
4582
4583 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4584 return cpr->cp_ring_struct.fw_ring_id;
4585 } else {
4586 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4587 }
4588}
4589
4590static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4591{
4592 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4593 struct bnxt_napi *bnapi = txr->bnapi;
4594 struct bnxt_cp_ring_info *cpr;
4595
4596 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4597 return cpr->cp_ring_struct.fw_ring_id;
4598 } else {
4599 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4600 }
4601}
4602
c0c050c5
MC		 4603static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4604{
4605 u32 i, j, max_rings;
4606 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4607 struct hwrm_vnic_rss_cfg_input req = {0};
4608
7b3af4f7
MC		 4609 if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4610 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
c0c050c5
MC		 4611 return 0;
4612
4613 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4614 if (set_rss) {
87da7f79 4615 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
50f011b6 4616 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
dc52c6c7
PS		 4617 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4618 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4619 max_rings = bp->rx_nr_rings - 1;
4620 else
4621 max_rings = bp->rx_nr_rings;
4622 } else {
c0c050c5 4623 max_rings = 1;
dc52c6c7 4624 }
c0c050c5
MC		 4625
4626 /* Fill the RSS indirection table with ring group ids */
4627 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4628 if (j == max_rings)
4629 j = 0;
4630 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4631 }
4632
4633 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4634 req.hash_key_tbl_addr =
4635 cpu_to_le64(vnic->rss_hash_key_dma_addr);
4636 }
94ce9caa 4637 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
c0c050c5
MC		 4638 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4639}
4640
7b3af4f7
MC		 4641static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4642{
4643 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4644 u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4645 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4646 struct hwrm_vnic_rss_cfg_input req = {0};
4647
4648 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4649 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4650 if (!set_rss) {
4651 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4652 return 0;
4653 }
4654 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4655 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4656 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4657 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4658 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4659 for (i = 0, k = 0; i < nr_ctxs; i++) {
4660 __le16 *ring_tbl = vnic->rss_table;
4661 int rc;
4662
4663 req.ring_table_pair_index = i;
4664 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4665 for (j = 0; j < 64; j++) {
4666 u16 ring_id;
4667
4668 ring_id = rxr->rx_ring_struct.fw_ring_id;
4669 *ring_tbl++ = cpu_to_le16(ring_id);
4670 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4671 *ring_tbl++ = cpu_to_le16(ring_id);
4672 rxr++;
4673 k++;
4674 if (k == max_rings) {
4675 k = 0;
4676 rxr = &bp->rx_ring[0];
4677 }
4678 }
4679 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4680 if (rc)
4681 return -EIO;
4682 }
4683 return 0;
4684}
4685
c0c050c5
MC		 4686static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4687{
4688 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4689 struct hwrm_vnic_plcmodes_cfg_input req = {0};
4690
4691 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4692 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4693 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4694 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4695 req.enables =
4696 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4697 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4698 /* thresholds not implemented in firmware yet */
4699 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4700 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4701 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4702 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4703}
4704
94ce9caa
PS		 4705static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4706 u16 ctx_idx)
c0c050c5
MC		 4707{
4708 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4709
4710 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4711 req.rss_cos_lb_ctx_id =
94ce9caa 4712 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
c0c050c5
MC		 4713
4714 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
94ce9caa 4715 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
c0c050c5
MC		 4716}
4717
4718static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4719{
94ce9caa 4720 int i, j;
c0c050c5
MC		 4721
4722 for (i = 0; i < bp->nr_vnics; i++) {
4723 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4724
94ce9caa
PS		 4725 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4726 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4727 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4728 }
c0c050c5
MC		 4729 }
4730 bp->rsscos_nr_ctxs = 0;
4731}
4732
94ce9caa 4733static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
c0c050c5
MC		 4734{
4735 int rc;
4736 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4737 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4738 bp->hwrm_cmd_resp_addr;
4739
4740 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4741 -1);
4742
4743 mutex_lock(&bp->hwrm_cmd_lock);
4744 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4745 if (!rc)
94ce9caa 4746 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
c0c050c5
MC		 4747 le16_to_cpu(resp->rss_cos_lb_ctx_id);
4748 mutex_unlock(&bp->hwrm_cmd_lock);
4749
4750 return rc;
4751}
4752
abe93ad2
MC		 4753static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4754{
4755 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4756 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4757 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4758}
4759
a588e458 4760int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
c0c050c5 4761{
b81a90d3 4762 unsigned int ring = 0, grp_idx;
c0c050c5
MC		 4763 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4764 struct hwrm_vnic_cfg_input req = {0};
cf6645f8 4765 u16 def_vlan = 0;
c0c050c5
MC		 4766
4767 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
dc52c6c7 4768
7b3af4f7
MC		 4769 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4770 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4771
4772 req.default_rx_ring_id =
4773 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
4774 req.default_cmpl_ring_id =
4775 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
4776 req.enables =
4777 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
4778 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
4779 goto vnic_mru;
4780 }
dc52c6c7 4781 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
c0c050c5 4782 /* Only RSS support for now TBD: COS & LB */
dc52c6c7
PS		 4783 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4784 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4785 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4786 VNIC_CFG_REQ_ENABLES_MRU);
ae10ae74
MC		 4787 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4788 req.rss_rule =
4789 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4790 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4791 VNIC_CFG_REQ_ENABLES_MRU);
4792 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
dc52c6c7
PS		 4793 } else {
4794 req.rss_rule = cpu_to_le16(0xffff);
4795 }
94ce9caa 4796
dc52c6c7
PS		 4797 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4798 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
94ce9caa
PS		 4799 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4800 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4801 } else {
4802 req.cos_rule = cpu_to_le16(0xffff);
4803 }
4804
c0c050c5 4805 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
b81a90d3 4806 ring = 0;
c0c050c5 4807 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
b81a90d3 4808 ring = vnic_id - 1;
76595193
PS		 4809 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4810 ring = bp->rx_nr_rings - 1;
c0c050c5 4811
b81a90d3 4812 grp_idx = bp->rx_ring[ring].bnapi->index;
c0c050c5 4813 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
c0c050c5 4814 req.lb_rule = cpu_to_le16(0xffff);
7b3af4f7 4815vnic_mru:
c0c050c5
MC		 4816 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4817 VLAN_HLEN);
4818
7b3af4f7 4819 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
cf6645f8
MC		 4820#ifdef CONFIG_BNXT_SRIOV
4821 if (BNXT_VF(bp))
4822 def_vlan = bp->vf.vlan;
4823#endif
4824 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
c0c050c5 4825 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
a588e458 4826 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
abe93ad2 4827 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
c0c050c5
MC		 4828
4829 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4830}
4831
4832static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4833{
4834 u32 rc = 0;
4835
4836 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4837 struct hwrm_vnic_free_input req = {0};
4838
4839 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4840 req.vnic_id =
4841 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4842
4843 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4844 if (rc)
4845 return rc;
4846 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4847 }
4848 return rc;
4849}
4850
4851static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4852{
4853 u16 i;
4854
4855 for (i = 0; i < bp->nr_vnics; i++)
4856 bnxt_hwrm_vnic_free_one(bp, i);
4857}
4858
b81a90d3
MC		 4859static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4860 unsigned int start_rx_ring_idx,
4861 unsigned int nr_rings)
c0c050c5 4862{
b81a90d3
MC		 4863 int rc = 0;
4864 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
c0c050c5
MC		 4865 struct hwrm_vnic_alloc_input req = {0};
4866 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
44c6f72a
MC		 4867 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4868
4869 if (bp->flags & BNXT_FLAG_CHIP_P5)
4870 goto vnic_no_ring_grps;
c0c050c5
MC		 4871
4872 /* map ring groups to this vnic */
b81a90d3
MC		 4873 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4874 grp_idx = bp->rx_ring[i].bnapi->index;
4875 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
c0c050c5 4876 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
b81a90d3 4877 j, nr_rings);
c0c050c5
MC		 4878 break;
4879 }
44c6f72a 4880 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
c0c050c5
MC		 4881 }
4882
44c6f72a
MC		 4883vnic_no_ring_grps:
4884 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
4885 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
c0c050c5
MC		 4886 if (vnic_id == 0)
4887 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4888
4889 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4890
4891 mutex_lock(&bp->hwrm_cmd_lock);
4892 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4893 if (!rc)
44c6f72a 4894 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
c0c050c5
MC		 4895 mutex_unlock(&bp->hwrm_cmd_lock);
4896 return rc;
4897}
4898
8fdefd63
MC		 4899static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4900{
4901 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4902 struct hwrm_vnic_qcaps_input req = {0};
4903 int rc;
4904
4905 if (bp->hwrm_spec_code < 0x10600)
4906 return 0;
4907
4908 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4909 mutex_lock(&bp->hwrm_cmd_lock);
4910 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4911 if (!rc) {
abe93ad2
MC		 4912 u32 flags = le32_to_cpu(resp->flags);
4913
41e8d798
MC		 4914 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
4915 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
8fdefd63 4916 bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
abe93ad2
MC		 4917 if (flags &
4918 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
4919 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
79632e9b 4920 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
8fdefd63
MC		 4921 }
4922 mutex_unlock(&bp->hwrm_cmd_lock);
4923 return rc;
4924}
4925
c0c050c5
MC		 4926static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4927{
4928 u16 i;
4929 u32 rc = 0;
4930
44c6f72a
MC		 4931 if (bp->flags & BNXT_FLAG_CHIP_P5)
4932 return 0;
4933
c0c050c5
MC		 4934 mutex_lock(&bp->hwrm_cmd_lock);
4935 for (i = 0; i < bp->rx_nr_rings; i++) {
4936 struct hwrm_ring_grp_alloc_input req = {0};
4937 struct hwrm_ring_grp_alloc_output *resp =
4938 bp->hwrm_cmd_resp_addr;
b81a90d3 4939 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
c0c050c5
MC		 4940
4941 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4942
b81a90d3
MC		 4943 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4944 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4945 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4946 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
c0c050c5
MC		 4947
4948 rc = _hwrm_send_message(bp, &req, sizeof(req),
4949 HWRM_CMD_TIMEOUT);
4950 if (rc)
4951 break;
4952
b81a90d3
MC		 4953 bp->grp_info[grp_idx].fw_grp_id =
4954 le32_to_cpu(resp->ring_group_id);
c0c050c5
MC		 4955 }
4956 mutex_unlock(&bp->hwrm_cmd_lock);
4957 return rc;
4958}
4959
4960static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4961{
4962 u16 i;
4963 u32 rc = 0;
4964 struct hwrm_ring_grp_free_input req = {0};
4965
44c6f72a 4966 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
c0c050c5
MC		 4967 return 0;
4968
4969 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4970
4971 mutex_lock(&bp->hwrm_cmd_lock);
4972 for (i = 0; i < bp->cp_nr_rings; i++) {
4973 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4974 continue;
4975 req.ring_group_id =
4976 cpu_to_le32(bp->grp_info[i].fw_grp_id);
4977
4978 rc = _hwrm_send_message(bp, &req, sizeof(req),
4979 HWRM_CMD_TIMEOUT);
4980 if (rc)
4981 break;
4982 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4983 }
4984 mutex_unlock(&bp->hwrm_cmd_lock);
4985 return rc;
4986}
4987
4988static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4989 struct bnxt_ring_struct *ring,
9899bb59 4990 u32 ring_type, u32 map_index)
c0c050c5
MC		 4991{
4992 int rc = 0, err = 0;
4993 struct hwrm_ring_alloc_input req = {0};
4994 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6fe19886 4995 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
9899bb59 4996 struct bnxt_ring_grp_info *grp_info;
c0c050c5
MC		 4997 u16 ring_id;
4998
4999 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5000
5001 req.enables = 0;
6fe19886
MC		 5002 if (rmem->nr_pages > 1) {
5003 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
c0c050c5
MC		 5004 /* Page size is in log2 units */
5005 req.page_size = BNXT_PAGE_SHIFT;
5006 req.page_tbl_depth = 1;
5007 } else {
6fe19886 5008 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
c0c050c5
MC		 5009 }
5010 req.fbo = 0;
5011 /* Association of ring index with doorbell index and MSIX number */
5012 req.logical_id = cpu_to_le16(map_index);
5013
5014 switch (ring_type) {
2c61d211
MC		 5015 case HWRM_RING_ALLOC_TX: {
5016 struct bnxt_tx_ring_info *txr;
5017
5018 txr = container_of(ring, struct bnxt_tx_ring_info,
5019 tx_ring_struct);
c0c050c5
MC		 5020 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5021 /* Association of transmit ring with completion ring */
9899bb59 5022 grp_info = &bp->grp_info[ring->grp_idx];
2c61d211 5023 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
c0c050c5 5024 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
9899bb59 5025 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
c0c050c5
MC		 5026 req.queue_id = cpu_to_le16(ring->queue_id);
5027 break;
2c61d211 5028 }
c0c050c5
MC		 5029 case HWRM_RING_ALLOC_RX:
5030 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5031 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
23aefdd7
MC		 5032 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5033 u16 flags = 0;
5034
5035 /* Association of rx ring with stats context */
5036 grp_info = &bp->grp_info[ring->grp_idx];
5037 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5038 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5039 req.enables |= cpu_to_le32(
5040 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5041 if (NET_IP_ALIGN == 2)
5042 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5043 req.flags = cpu_to_le16(flags);
5044 }
c0c050c5
MC		 5045 break;
5046 case HWRM_RING_ALLOC_AGG:
23aefdd7
MC		 5047 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5048 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5049 /* Association of agg ring with rx ring */
5050 grp_info = &bp->grp_info[ring->grp_idx];
5051 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5052 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5053 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5054 req.enables |= cpu_to_le32(
5055 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5056 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5057 } else {
5058 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5059 }
c0c050c5
MC		 5060 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5061 break;
5062 case HWRM_RING_ALLOC_CMPL:
bac9a7e0 5063 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
c0c050c5 5064 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
23aefdd7
MC		 5065 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5066 /* Association of cp ring with nq */
5067 grp_info = &bp->grp_info[map_index];
5068 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5069 req.cq_handle = cpu_to_le64(ring->handle);
5070 req.enables |= cpu_to_le32(
5071 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5072 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5073 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5074 }
5075 break;
5076 case HWRM_RING_ALLOC_NQ:
5077 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5078 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
c0c050c5
MC		 5079 if (bp->flags & BNXT_FLAG_USING_MSIX)
5080 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5081 break;
5082 default:
5083 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5084 ring_type);
5085 return -1;
5086 }
5087
5088 mutex_lock(&bp->hwrm_cmd_lock);
5089 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5090 err = le16_to_cpu(resp->error_code);
5091 ring_id = le16_to_cpu(resp->ring_id);
5092 mutex_unlock(&bp->hwrm_cmd_lock);
5093
5094 if (rc || err) {
2727c888
MC		 5095 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5096 ring_type, rc, err);
5097 return -EIO;
c0c050c5
MC		 5098 }
5099 ring->fw_ring_id = ring_id;
5100 return rc;
5101}
5102
486b5c22
MC		 5103static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5104{
5105 int rc;
5106
5107 if (BNXT_PF(bp)) {
5108 struct hwrm_func_cfg_input req = {0};
5109
5110 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5111 req.fid = cpu_to_le16(0xffff);
5112 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5113 req.async_event_cr = cpu_to_le16(idx);
5114 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5115 } else {
5116 struct hwrm_func_vf_cfg_input req = {0};
5117
5118 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5119 req.enables =
5120 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5121 req.async_event_cr = cpu_to_le16(idx);
5122 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5123 }
5124 return rc;
5125}
5126
697197e5
MC		 5127static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5128 u32 map_idx, u32 xid)
5129{
5130 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5131 if (BNXT_PF(bp))
5132 db->doorbell = bp->bar1 + 0x10000;
5133 else
5134 db->doorbell = bp->bar1 + 0x4000;
5135 switch (ring_type) {
5136 case HWRM_RING_ALLOC_TX:
5137 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5138 break;
5139 case HWRM_RING_ALLOC_RX:
5140 case HWRM_RING_ALLOC_AGG:
5141 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5142 break;
5143 case HWRM_RING_ALLOC_CMPL:
5144 db->db_key64 = DBR_PATH_L2;
5145 break;
5146 case HWRM_RING_ALLOC_NQ:
5147 db->db_key64 = DBR_PATH_L2;
5148 break;
5149 }
5150 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5151 } else {
5152 db->doorbell = bp->bar1 + map_idx * 0x80;
5153 switch (ring_type) {
5154 case HWRM_RING_ALLOC_TX:
5155 db->db_key32 = DB_KEY_TX;
5156 break;
5157 case HWRM_RING_ALLOC_RX:
5158 case HWRM_RING_ALLOC_AGG:
5159 db->db_key32 = DB_KEY_RX;
5160 break;
5161 case HWRM_RING_ALLOC_CMPL:
5162 db->db_key32 = DB_KEY_CP;
5163 break;
5164 }
5165 }
5166}
5167
c0c050c5
MC		 5168static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5169{
5170 int i, rc = 0;
697197e5 5171 u32 type;
c0c050c5 5172
23aefdd7
MC		 5173 if (bp->flags & BNXT_FLAG_CHIP_P5)
5174 type = HWRM_RING_ALLOC_NQ;
5175 else
5176 type = HWRM_RING_ALLOC_CMPL;
edd0c2cc
MC		 5177 for (i = 0; i < bp->cp_nr_rings; i++) {
5178 struct bnxt_napi *bnapi = bp->bnapi[i];
5179 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5180 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
9899bb59 5181 u32 map_idx = ring->map_idx;
5e66e35a 5182 unsigned int vector;
c0c050c5 5183
5e66e35a
MC		 5184 vector = bp->irq_tbl[map_idx].vector;
5185 disable_irq_nosync(vector);
697197e5 5186 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5e66e35a
MC		 5187 if (rc) {
5188 enable_irq(vector);
edd0c2cc 5189 goto err_out;
5e66e35a 5190 }
697197e5
MC		 5191 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5192 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5e66e35a 5193 enable_irq(vector);
edd0c2cc 5194 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
486b5c22
MC		 5195
5196 if (!i) {
5197 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5198 if (rc)
5199 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5200 }
c0c050c5
MC		 5201 }
5202
697197e5 5203 type = HWRM_RING_ALLOC_TX;
edd0c2cc 5204 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5205 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3e08b184
MC		 5206 struct bnxt_ring_struct *ring;
5207 u32 map_idx;
c0c050c5 5208
3e08b184
MC		 5209 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5210 struct bnxt_napi *bnapi = txr->bnapi;
5211 struct bnxt_cp_ring_info *cpr, *cpr2;
5212 u32 type2 = HWRM_RING_ALLOC_CMPL;
5213
5214 cpr = &bnapi->cp_ring;
5215 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5216 ring = &cpr2->cp_ring_struct;
5217 ring->handle = BNXT_TX_HDL;
5218 map_idx = bnapi->index;
5219 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5220 if (rc)
5221 goto err_out;
5222 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5223 ring->fw_ring_id);
5224 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5225 }
5226 ring = &txr->tx_ring_struct;
5227 map_idx = i;
697197e5 5228 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
edd0c2cc
MC		 5229 if (rc)
5230 goto err_out;
697197e5 5231 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
c0c050c5
MC		 5232 }
5233
697197e5 5234 type = HWRM_RING_ALLOC_RX;
edd0c2cc 5235 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5236 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5237 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3e08b184
MC		 5238 struct bnxt_napi *bnapi = rxr->bnapi;
5239 u32 map_idx = bnapi->index;
c0c050c5 5240
697197e5 5241 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
edd0c2cc
MC		 5242 if (rc)
5243 goto err_out;
697197e5
MC		 5244 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5245 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
b81a90d3 5246 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3e08b184
MC		 5247 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5248 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5249 u32 type2 = HWRM_RING_ALLOC_CMPL;
5250 struct bnxt_cp_ring_info *cpr2;
5251
5252 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5253 ring = &cpr2->cp_ring_struct;
5254 ring->handle = BNXT_RX_HDL;
5255 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5256 if (rc)
5257 goto err_out;
5258 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5259 ring->fw_ring_id);
5260 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5261 }
c0c050c5
MC		 5262 }
5263
5264 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
697197e5 5265 type = HWRM_RING_ALLOC_AGG;
c0c050c5 5266 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5267 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 5268 struct bnxt_ring_struct *ring =
5269 &rxr->rx_agg_ring_struct;
9899bb59 5270 u32 grp_idx = ring->grp_idx;
b81a90d3 5271 u32 map_idx = grp_idx + bp->rx_nr_rings;
c0c050c5 5272
697197e5 5273 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
c0c050c5
MC		 5274 if (rc)
5275 goto err_out;
5276
697197e5
MC		 5277 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5278 ring->fw_ring_id);
5279 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
b81a90d3 5280 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 5281 }
5282 }
5283err_out:
5284 return rc;
5285}
5286
5287static int hwrm_ring_free_send_msg(struct bnxt *bp,
5288 struct bnxt_ring_struct *ring,
5289 u32 ring_type, int cmpl_ring_id)
5290{
5291 int rc;
5292 struct hwrm_ring_free_input req = {0};
5293 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5294 u16 error_code;
5295
74608fc9 5296 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
c0c050c5
MC		 5297 req.ring_type = ring_type;
5298 req.ring_id = cpu_to_le16(ring->fw_ring_id);
5299
5300 mutex_lock(&bp->hwrm_cmd_lock);
5301 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5302 error_code = le16_to_cpu(resp->error_code);
5303 mutex_unlock(&bp->hwrm_cmd_lock);
5304
5305 if (rc || error_code) {
2727c888
MC		 5306 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5307 ring_type, rc, error_code);
5308 return -EIO;
c0c050c5
MC		 5309 }
5310 return 0;
5311}
5312
edd0c2cc 5313static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
c0c050c5 5314{
23aefdd7 5315 u32 type;
edd0c2cc 5316 int i;
c0c050c5
MC		 5317
5318 if (!bp->bnapi)
edd0c2cc 5319 return;
c0c050c5 5320
edd0c2cc 5321 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5322 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 5323 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
edd0c2cc
MC		 5324
5325 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5326 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5327
edd0c2cc
MC		 5328 hwrm_ring_free_send_msg(bp, ring,
5329 RING_FREE_REQ_RING_TYPE_TX,
5330 close_path ? cmpl_ring_id :
5331 INVALID_HW_RING_ID);
5332 ring->fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 5333 }
5334 }
5335
edd0c2cc 5336 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5337 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5338 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3 5339 u32 grp_idx = rxr->bnapi->index;
edd0c2cc
MC		 5340
5341 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5342 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5343
edd0c2cc
MC		 5344 hwrm_ring_free_send_msg(bp, ring,
5345 RING_FREE_REQ_RING_TYPE_RX,
5346 close_path ? cmpl_ring_id :
5347 INVALID_HW_RING_ID);
5348 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 5349 bp->grp_info[grp_idx].rx_fw_ring_id =
5350 INVALID_HW_RING_ID;
c0c050c5
MC		 5351 }
5352 }
5353
23aefdd7
MC		 5354 if (bp->flags & BNXT_FLAG_CHIP_P5)
5355 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5356 else
5357 type = RING_FREE_REQ_RING_TYPE_RX;
edd0c2cc 5358 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5359 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5360 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
b81a90d3 5361 u32 grp_idx = rxr->bnapi->index;
edd0c2cc
MC		 5362
5363 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5364 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5365
23aefdd7 5366 hwrm_ring_free_send_msg(bp, ring, type,
edd0c2cc
MC		 5367 close_path ? cmpl_ring_id :
5368 INVALID_HW_RING_ID);
5369 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 5370 bp->grp_info[grp_idx].agg_fw_ring_id =
5371 INVALID_HW_RING_ID;
c0c050c5
MC		 5372 }
5373 }
5374
9d8bc097
MC		 5375 /* The completion rings are about to be freed. After that the
5376 * IRQ doorbell will not work anymore. So we need to disable
5377 * IRQ here.
5378 */
5379 bnxt_disable_int_sync(bp);
5380
23aefdd7
MC		 5381 if (bp->flags & BNXT_FLAG_CHIP_P5)
5382 type = RING_FREE_REQ_RING_TYPE_NQ;
5383 else
5384 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
edd0c2cc
MC		 5385 for (i = 0; i < bp->cp_nr_rings; i++) {
5386 struct bnxt_napi *bnapi = bp->bnapi[i];
5387 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3e08b184
MC		 5388 struct bnxt_ring_struct *ring;
5389 int j;
edd0c2cc 5390
3e08b184
MC		 5391 for (j = 0; j < 2; j++) {
5392 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5393
5394 if (cpr2) {
5395 ring = &cpr2->cp_ring_struct;
5396 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5397 continue;
5398 hwrm_ring_free_send_msg(bp, ring,
5399 RING_FREE_REQ_RING_TYPE_L2_CMPL,
5400 INVALID_HW_RING_ID);
5401 ring->fw_ring_id = INVALID_HW_RING_ID;
5402 }
5403 }
5404 ring = &cpr->cp_ring_struct;
edd0c2cc 5405 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
23aefdd7 5406 hwrm_ring_free_send_msg(bp, ring, type,
edd0c2cc
MC		 5407 INVALID_HW_RING_ID);
5408 ring->fw_ring_id = INVALID_HW_RING_ID;
5409 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 5410 }
5411 }
c0c050c5
MC		 5412}
5413
41e8d798
MC		 5414static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5415 bool shared);
5416
674f50a5
MC		 5417static int bnxt_hwrm_get_rings(struct bnxt *bp)
5418{
5419 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5420 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5421 struct hwrm_func_qcfg_input req = {0};
5422 int rc;
5423
5424 if (bp->hwrm_spec_code < 0x10601)
5425 return 0;
5426
5427 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5428 req.fid = cpu_to_le16(0xffff);
5429 mutex_lock(&bp->hwrm_cmd_lock);
5430 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5431 if (rc) {
5432 mutex_unlock(&bp->hwrm_cmd_lock);
5433 return -EIO;
5434 }
5435
5436 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
f1ca94de 5437 if (BNXT_NEW_RM(bp)) {
674f50a5
MC		 5438 u16 cp, stats;
5439
5440 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5441 hw_resc->resv_hw_ring_grps =
5442 le32_to_cpu(resp->alloc_hw_ring_grps);
5443 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5444 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5445 stats = le16_to_cpu(resp->alloc_stat_ctx);
75720e63 5446 hw_resc->resv_irqs = cp;
41e8d798
MC		 5447 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5448 int rx = hw_resc->resv_rx_rings;
5449 int tx = hw_resc->resv_tx_rings;
5450
5451 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5452 rx >>= 1;
5453 if (cp < (rx + tx)) {
5454 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5455 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5456 rx <<= 1;
5457 hw_resc->resv_rx_rings = rx;
5458 hw_resc->resv_tx_rings = tx;
5459 }
75720e63 5460 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
41e8d798
MC		 5461 hw_resc->resv_hw_ring_grps = rx;
5462 }
674f50a5 5463 hw_resc->resv_cp_rings = cp;
780baad4 5464 hw_resc->resv_stat_ctxs = stats;
674f50a5
MC		 5465 }
5466 mutex_unlock(&bp->hwrm_cmd_lock);
5467 return 0;
5468}
5469
391be5c2
MC		 5470/* Caller must hold bp->hwrm_cmd_lock */
5471int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5472{
5473 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5474 struct hwrm_func_qcfg_input req = {0};
5475 int rc;
5476
5477 if (bp->hwrm_spec_code < 0x10601)
5478 return 0;
5479
5480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5481 req.fid = cpu_to_le16(fid);
5482 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5483 if (!rc)
5484 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5485
5486 return rc;
5487}
5488
41e8d798
MC		 5489static bool bnxt_rfs_supported(struct bnxt *bp);
5490
4ed50ef4
MC		 5491static void
5492__bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5493 int tx_rings, int rx_rings, int ring_grps,
780baad4 5494 int cp_rings, int stats, int vnics)
391be5c2 5495{
674f50a5 5496 u32 enables = 0;
391be5c2 5497
4ed50ef4
MC		 5498 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5499 req->fid = cpu_to_le16(0xffff);
674f50a5 5500 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
4ed50ef4 5501 req->num_tx_rings = cpu_to_le16(tx_rings);
f1ca94de 5502 if (BNXT_NEW_RM(bp)) {
674f50a5 5503 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
3f93cd3f 5504 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
41e8d798
MC		 5505 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5506 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5507 enables |= tx_rings + ring_grps ?
3f93cd3f 5508 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5509 enables |= rx_rings ?
5510 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5511 } else {
5512 enables |= cp_rings ?
3f93cd3f 5513 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5514 enables |= ring_grps ?
5515 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5516 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5517 }
dbe80d44 5518 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
674f50a5 5519
4ed50ef4 5520 req->num_rx_rings = cpu_to_le16(rx_rings);
41e8d798
MC		 5521 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5522 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5523 req->num_msix = cpu_to_le16(cp_rings);
5524 req->num_rsscos_ctxs =
5525 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5526 } else {
5527 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5528 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5529 req->num_rsscos_ctxs = cpu_to_le16(1);
5530 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5531 bnxt_rfs_supported(bp))
5532 req->num_rsscos_ctxs =
5533 cpu_to_le16(ring_grps + 1);
5534 }
780baad4 5535 req->num_stat_ctxs = cpu_to_le16(stats);
4ed50ef4 5536 req->num_vnics = cpu_to_le16(vnics);
674f50a5 5537 }
4ed50ef4
MC		 5538 req->enables = cpu_to_le32(enables);
5539}
5540
5541static void
5542__bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5543 struct hwrm_func_vf_cfg_input *req, int tx_rings,
5544 int rx_rings, int ring_grps, int cp_rings,
780baad4 5545 int stats, int vnics)
4ed50ef4
MC		 5546{
5547 u32 enables = 0;
5548
5549 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5550 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
41e8d798
MC		 5551 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5552 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
3f93cd3f 5553 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
41e8d798
MC		 5554 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5555 enables |= tx_rings + ring_grps ?
3f93cd3f 5556 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5557 } else {
5558 enables |= cp_rings ?
3f93cd3f 5559 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5560 enables |= ring_grps ?
5561 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5562 }
4ed50ef4 5563 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
41e8d798 5564 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
4ed50ef4 5565
41e8d798 5566 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
4ed50ef4
MC		 5567 req->num_tx_rings = cpu_to_le16(tx_rings);
5568 req->num_rx_rings = cpu_to_le16(rx_rings);
41e8d798
MC		 5569 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5570 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5571 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5572 } else {
5573 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5574 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5575 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5576 }
780baad4 5577 req->num_stat_ctxs = cpu_to_le16(stats);
4ed50ef4
MC		 5578 req->num_vnics = cpu_to_le16(vnics);
5579
5580 req->enables = cpu_to_le32(enables);
5581}
5582
5583static int
5584bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4 5585 int ring_grps, int cp_rings, int stats, int vnics)
4ed50ef4
MC		 5586{
5587 struct hwrm_func_cfg_input req = {0};
5588 int rc;
5589
5590 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5591 cp_rings, stats, vnics);
4ed50ef4 5592 if (!req.enables)
391be5c2
MC		 5593 return 0;
5594
674f50a5
MC		 5595 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5596 if (rc)
5597 return -ENOMEM;
5598
5599 if (bp->hwrm_spec_code < 0x10601)
5600 bp->hw_resc.resv_tx_rings = tx_rings;
5601
5602 rc = bnxt_hwrm_get_rings(bp);
5603 return rc;
5604}
5605
5606static int
5607bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4 5608 int ring_grps, int cp_rings, int stats, int vnics)
674f50a5
MC		 5609{
5610 struct hwrm_func_vf_cfg_input req = {0};
674f50a5
MC		 5611 int rc;
5612
f1ca94de 5613 if (!BNXT_NEW_RM(bp)) {
674f50a5 5614 bp->hw_resc.resv_tx_rings = tx_rings;
391be5c2 5615 return 0;
674f50a5 5616 }
391be5c2 5617
4ed50ef4 5618 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5619 cp_rings, stats, vnics);
391be5c2 5620 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
674f50a5
MC		 5621 if (rc)
5622 return -ENOMEM;
5623
5624 rc = bnxt_hwrm_get_rings(bp);
5625 return rc;
5626}
5627
5628static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
780baad4 5629 int cp, int stat, int vnic)
674f50a5
MC		 5630{
5631 if (BNXT_PF(bp))
780baad4
VV		 5632 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5633 vnic);
674f50a5 5634 else
780baad4
VV		 5635 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5636 vnic);
674f50a5
MC		 5637}
5638
b16b6891 5639int bnxt_nq_rings_in_use(struct bnxt *bp)
08654eb2
MC		 5640{
5641 int cp = bp->cp_nr_rings;
5642 int ulp_msix, ulp_base;
5643
5644 ulp_msix = bnxt_get_ulp_msix_num(bp);
5645 if (ulp_msix) {
5646 ulp_base = bnxt_get_ulp_msix_base(bp);
5647 cp += ulp_msix;
5648 if ((ulp_base + ulp_msix) > cp)
5649 cp = ulp_base + ulp_msix;
5650 }
5651 return cp;
5652}
5653
c0b8cda0
MC		 5654static int bnxt_cp_rings_in_use(struct bnxt *bp)
5655{
5656 int cp;
5657
5658 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5659 return bnxt_nq_rings_in_use(bp);
5660
5661 cp = bp->tx_nr_rings + bp->rx_nr_rings;
5662 return cp;
5663}
5664
780baad4
VV		 5665static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5666{
d77b1ad8
MC		 5667 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
5668 int cp = bp->cp_nr_rings;
5669
5670 if (!ulp_stat)
5671 return cp;
5672
5673 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
5674 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
5675
5676 return cp + ulp_stat;
780baad4
VV		 5677}
5678
4e41dc5d
MC		 5679static bool bnxt_need_reserve_rings(struct bnxt *bp)
5680{
5681 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
fbcfc8e4 5682 int cp = bnxt_cp_rings_in_use(bp);
c0b8cda0 5683 int nq = bnxt_nq_rings_in_use(bp);
780baad4 5684 int rx = bp->rx_nr_rings, stat;
4e41dc5d
MC		 5685 int vnic = 1, grp = rx;
5686
5687 if (bp->hwrm_spec_code < 0x10601)
5688 return false;
5689
5690 if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5691 return true;
5692
41e8d798 5693 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
4e41dc5d
MC		 5694 vnic = rx + 1;
5695 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5696 rx <<= 1;
780baad4 5697 stat = bnxt_get_func_stat_ctxs(bp);
f1ca94de 5698 if (BNXT_NEW_RM(bp) &&
4e41dc5d 5699 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
01989c6b 5700 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
41e8d798
MC		 5701 (hw_resc->resv_hw_ring_grps != grp &&
5702 !(bp->flags & BNXT_FLAG_CHIP_P5))))
4e41dc5d 5703 return true;
01989c6b
MC		 5704 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
5705 hw_resc->resv_irqs != nq)
5706 return true;
4e41dc5d
MC		 5707 return false;
5708}
5709
674f50a5
MC		 5710static int __bnxt_reserve_rings(struct bnxt *bp)
5711{
5712 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
c0b8cda0 5713 int cp = bnxt_nq_rings_in_use(bp);
674f50a5
MC		 5714 int tx = bp->tx_nr_rings;
5715 int rx = bp->rx_nr_rings;
674f50a5 5716 int grp, rx_rings, rc;
780baad4 5717 int vnic = 1, stat;
674f50a5 5718 bool sh = false;
674f50a5 5719
4e41dc5d 5720 if (!bnxt_need_reserve_rings(bp))
674f50a5
MC		 5721 return 0;
5722
5723 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5724 sh = true;
41e8d798 5725 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
674f50a5
MC		 5726 vnic = rx + 1;
5727 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5728 rx <<= 1;
674f50a5 5729 grp = bp->rx_nr_rings;
780baad4 5730 stat = bnxt_get_func_stat_ctxs(bp);
674f50a5 5731
780baad4 5732 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
391be5c2
MC		 5733 if (rc)
5734 return rc;
5735
674f50a5 5736 tx = hw_resc->resv_tx_rings;
f1ca94de 5737 if (BNXT_NEW_RM(bp)) {
674f50a5 5738 rx = hw_resc->resv_rx_rings;
c0b8cda0 5739 cp = hw_resc->resv_irqs;
674f50a5
MC		 5740 grp = hw_resc->resv_hw_ring_grps;
5741 vnic = hw_resc->resv_vnics;
780baad4 5742 stat = hw_resc->resv_stat_ctxs;
674f50a5
MC		 5743 }
5744
5745 rx_rings = rx;
5746 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5747 if (rx >= 2) {
5748 rx_rings = rx >> 1;
5749 } else {
5750 if (netif_running(bp->dev))
5751 return -ENOMEM;
5752
5753 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5754 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5755 bp->dev->hw_features &= ~NETIF_F_LRO;
5756 bp->dev->features &= ~NETIF_F_LRO;
5757 bnxt_set_ring_params(bp);
5758 }
5759 }
5760 rx_rings = min_t(int, rx_rings, grp);
780baad4
VV		 5761 cp = min_t(int, cp, bp->cp_nr_rings);
5762 if (stat > bnxt_get_ulp_stat_ctxs(bp))
5763 stat -= bnxt_get_ulp_stat_ctxs(bp);
5764 cp = min_t(int, cp, stat);
674f50a5
MC		 5765 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
5766 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5767 rx = rx_rings << 1;
5768 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
5769 bp->tx_nr_rings = tx;
5770 bp->rx_nr_rings = rx_rings;
5771 bp->cp_nr_rings = cp;
5772
780baad4 5773 if (!tx || !rx || !cp || !grp || !vnic || !stat)
674f50a5
MC		 5774 return -ENOMEM;
5775
391be5c2
MC		 5776 return rc;
5777}
5778
8f23d638 5779static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5780 int ring_grps, int cp_rings, int stats,
5781 int vnics)
98fdbe73 5782{
8f23d638 5783 struct hwrm_func_vf_cfg_input req = {0};
6fc2ffdf 5784 u32 flags;
98fdbe73
MC		 5785 int rc;
5786
f1ca94de 5787 if (!BNXT_NEW_RM(bp))
98fdbe73
MC		 5788 return 0;
5789
6fc2ffdf 5790 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5791 cp_rings, stats, vnics);
8f23d638
MC		 5792 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
5793 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5794 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
8f23d638 5795 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
41e8d798
MC		 5796 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
5797 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5798 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5799 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
8f23d638
MC		 5800
5801 req.flags = cpu_to_le32(flags);
8f23d638
MC		 5802 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5803 if (rc)
5804 return -ENOMEM;
5805 return 0;
5806}
5807
5808static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5809 int ring_grps, int cp_rings, int stats,
5810 int vnics)
8f23d638
MC		 5811{
5812 struct hwrm_func_cfg_input req = {0};
6fc2ffdf 5813 u32 flags;
8f23d638 5814 int rc;
98fdbe73 5815
6fc2ffdf 5816 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5817 cp_rings, stats, vnics);
8f23d638 5818 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
41e8d798 5819 if (BNXT_NEW_RM(bp)) {
8f23d638
MC		 5820 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5821 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
8f23d638
MC		 5822 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5823 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
41e8d798 5824 if (bp->flags & BNXT_FLAG_CHIP_P5)
0b815023
MC		 5825 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
5826 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
41e8d798
MC		 5827 else
5828 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5829 }
6fc2ffdf 5830
8f23d638 5831 req.flags = cpu_to_le32(flags);
98fdbe73
MC		 5832 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5833 if (rc)
5834 return -ENOMEM;
5835 return 0;
5836}
5837
8f23d638 5838static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5839 int ring_grps, int cp_rings, int stats,
5840 int vnics)
8f23d638
MC		 5841{
5842 if (bp->hwrm_spec_code < 0x10801)
5843 return 0;
5844
5845 if (BNXT_PF(bp))
5846 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
780baad4
VV		 5847 ring_grps, cp_rings, stats,
5848 vnics);
8f23d638
MC		 5849
5850 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
780baad4 5851 cp_rings, stats, vnics);
8f23d638
MC		 5852}
5853
74706afa
MC		 5854static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
5855{
5856 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5857 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5858 struct hwrm_ring_aggint_qcaps_input req = {0};
5859 int rc;
5860
5861 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
5862 coal_cap->num_cmpl_dma_aggr_max = 63;
5863 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
5864 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
5865 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
5866 coal_cap->int_lat_tmr_min_max = 65535;
5867 coal_cap->int_lat_tmr_max_max = 65535;
5868 coal_cap->num_cmpl_aggr_int_max = 65535;
5869 coal_cap->timer_units = 80;
5870
5871 if (bp->hwrm_spec_code < 0x10902)
5872 return;
5873
5874 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
5875 mutex_lock(&bp->hwrm_cmd_lock);
5876 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5877 if (!rc) {
5878 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
58590c8d 5879 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
74706afa
MC		 5880 coal_cap->num_cmpl_dma_aggr_max =
5881 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
5882 coal_cap->num_cmpl_dma_aggr_during_int_max =
5883 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
5884 coal_cap->cmpl_aggr_dma_tmr_max =
5885 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
5886 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
5887 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
5888 coal_cap->int_lat_tmr_min_max =
5889 le16_to_cpu(resp->int_lat_tmr_min_max);
5890 coal_cap->int_lat_tmr_max_max =
5891 le16_to_cpu(resp->int_lat_tmr_max_max);
5892 coal_cap->num_cmpl_aggr_int_max =
5893 le16_to_cpu(resp->num_cmpl_aggr_int_max);
5894 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
5895 }
5896 mutex_unlock(&bp->hwrm_cmd_lock);
5897}
5898
5899static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
5900{
5901 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5902
5903 return usec * 1000 / coal_cap->timer_units;
5904}
5905
5906static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
5907 struct bnxt_coal *hw_coal,
bb053f52
MC		 5908 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
5909{
74706afa
MC		 5910 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5911 u32 cmpl_params = coal_cap->cmpl_params;
5912 u16 val, tmr, max, flags = 0;
f8503969
MC		 5913
5914 max = hw_coal->bufs_per_record * 128;
5915 if (hw_coal->budget)
5916 max = hw_coal->bufs_per_record * hw_coal->budget;
74706afa 5917 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
f8503969
MC		 5918
5919 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
5920 req->num_cmpl_aggr_int = cpu_to_le16(val);
b153cbc5 5921
74706afa 5922 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
f8503969
MC		 5923 req->num_cmpl_dma_aggr = cpu_to_le16(val);
5924
74706afa
MC		 5925 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
5926 coal_cap->num_cmpl_dma_aggr_during_int_max);
f8503969
MC		 5927 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
5928
74706afa
MC		 5929 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
5930 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
f8503969
MC		 5931 req->int_lat_tmr_max = cpu_to_le16(tmr);
5932
5933 /* min timer set to 1/2 of interrupt timer */
74706afa
MC		 5934 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
5935 val = tmr / 2;
5936 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
5937 req->int_lat_tmr_min = cpu_to_le16(val);
5938 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5939 }
f8503969
MC		 5940
5941 /* buf timer set to 1/4 of interrupt timer */
74706afa 5942 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
f8503969
MC		 5943 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
5944
74706afa
MC		 5945 if (cmpl_params &
5946 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
5947 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
5948 val = clamp_t(u16, tmr, 1,
5949 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
5950 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
5951 req->enables |=
5952 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
5953 }
f8503969 5954
74706afa
MC		 5955 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
5956 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
5957 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
5958 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
f8503969 5959 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
bb053f52 5960 req->flags = cpu_to_le16(flags);
74706afa 5961 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
bb053f52
MC		 5962}
5963
58590c8d
MC		 5964/* Caller holds bp->hwrm_cmd_lock */
5965static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
5966 struct bnxt_coal *hw_coal)
5967{
5968 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
5969 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5970 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5971 u32 nq_params = coal_cap->nq_params;
5972 u16 tmr;
5973
5974 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
5975 return 0;
5976
5977 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
5978 -1, -1);
5979 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
5980 req.flags =
5981 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
5982
5983 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
5984 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
5985 req.int_lat_tmr_min = cpu_to_le16(tmr);
5986 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5987 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5988}
5989
6a8788f2
AG		 5990int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
5991{
5992 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
5993 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5994 struct bnxt_coal coal;
6a8788f2
AG		 5995
5996 /* Tick values in micro seconds.
5997 * 1 coal_buf x bufs_per_record = 1 completion record.
5998 */
5999 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6000
6001 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6002 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6003
6004 if (!bnapi->rx_ring)
6005 return -ENODEV;
6006
6007 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6008 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6009
74706afa 6010 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6a8788f2 6011
2c61d211 6012 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6a8788f2
AG		 6013
6014 return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6015 HWRM_CMD_TIMEOUT);
6016}
6017
c0c050c5
MC		 6018int bnxt_hwrm_set_coal(struct bnxt *bp)
6019{
6020 int i, rc = 0;
dfc9c94a
MC		 6021 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6022 req_tx = {0}, *req;
c0c050c5 6023
dfc9c94a
MC		 6024 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6025 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6026 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6027 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
c0c050c5 6028
74706afa
MC		 6029 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6030 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
c0c050c5
MC		 6031
6032 mutex_lock(&bp->hwrm_cmd_lock);
6033 for (i = 0; i < bp->cp_nr_rings; i++) {
dfc9c94a 6034 struct bnxt_napi *bnapi = bp->bnapi[i];
58590c8d 6035 struct bnxt_coal *hw_coal;
2c61d211 6036 u16 ring_id;
c0c050c5 6037
dfc9c94a 6038 req = &req_rx;
2c61d211
MC		 6039 if (!bnapi->rx_ring) {
6040 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
dfc9c94a 6041 req = &req_tx;
2c61d211
MC		 6042 } else {
6043 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6044 }
6045 req->ring_id = cpu_to_le16(ring_id);
dfc9c94a
MC		 6046
6047 rc = _hwrm_send_message(bp, req, sizeof(*req),
c0c050c5
MC		 6048 HWRM_CMD_TIMEOUT);
6049 if (rc)
6050 break;
58590c8d
MC		 6051
6052 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6053 continue;
6054
6055 if (bnapi->rx_ring && bnapi->tx_ring) {
6056 req = &req_tx;
6057 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6058 req->ring_id = cpu_to_le16(ring_id);
6059 rc = _hwrm_send_message(bp, req, sizeof(*req),
6060 HWRM_CMD_TIMEOUT);
6061 if (rc)
6062 break;
6063 }
6064 if (bnapi->rx_ring)
6065 hw_coal = &bp->rx_coal;
6066 else
6067 hw_coal = &bp->tx_coal;
6068 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
c0c050c5
MC		 6069 }
6070 mutex_unlock(&bp->hwrm_cmd_lock);
6071 return rc;
6072}
6073
6074static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6075{
6076 int rc = 0, i;
6077 struct hwrm_stat_ctx_free_input req = {0};
6078
6079 if (!bp->bnapi)
6080 return 0;
6081
3e8060fa
PS		 6082 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6083 return 0;
6084
c0c050c5
MC		 6085 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6086
6087 mutex_lock(&bp->hwrm_cmd_lock);
6088 for (i = 0; i < bp->cp_nr_rings; i++) {
6089 struct bnxt_napi *bnapi = bp->bnapi[i];
6090 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6091
6092 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6093 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6094
6095 rc = _hwrm_send_message(bp, &req, sizeof(req),
6096 HWRM_CMD_TIMEOUT);
6097 if (rc)
6098 break;
6099
6100 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6101 }
6102 }
6103 mutex_unlock(&bp->hwrm_cmd_lock);
6104 return rc;
6105}
6106
6107static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6108{
6109 int rc = 0, i;
6110 struct hwrm_stat_ctx_alloc_input req = {0};
6111 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6112
3e8060fa
PS		 6113 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6114 return 0;
6115
c0c050c5
MC		 6116 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6117
51f30785 6118 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
c0c050c5
MC		 6119
6120 mutex_lock(&bp->hwrm_cmd_lock);
6121 for (i = 0; i < bp->cp_nr_rings; i++) {
6122 struct bnxt_napi *bnapi = bp->bnapi[i];
6123 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6124
6125 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
6126
6127 rc = _hwrm_send_message(bp, &req, sizeof(req),
6128 HWRM_CMD_TIMEOUT);
6129 if (rc)
6130 break;
6131
6132 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6133
6134 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6135 }
6136 mutex_unlock(&bp->hwrm_cmd_lock);
89aa8445 6137 return rc;
c0c050c5
MC		 6138}
6139
cf6645f8
MC		 6140static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6141{
6142 struct hwrm_func_qcfg_input req = {0};
567b2abe 6143 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9315edca 6144 u16 flags;
cf6645f8
MC		 6145 int rc;
6146
6147 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6148 req.fid = cpu_to_le16(0xffff);
6149 mutex_lock(&bp->hwrm_cmd_lock);
6150 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6151 if (rc)
6152 goto func_qcfg_exit;
6153
6154#ifdef CONFIG_BNXT_SRIOV
6155 if (BNXT_VF(bp)) {
cf6645f8
MC		 6156 struct bnxt_vf_info *vf = &bp->vf;
6157
6158 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6159 }
6160#endif
9315edca
MC		 6161 flags = le16_to_cpu(resp->flags);
6162 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6163 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
97381a18 6164 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
9315edca 6165 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
97381a18 6166 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
9315edca
MC		 6167 }
6168 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6169 bp->flags |= BNXT_FLAG_MULTI_HOST;
bc39f885 6170
567b2abe
SB		 6171 switch (resp->port_partition_type) {
6172 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6173 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6174 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6175 bp->port_partition_type = resp->port_partition_type;
6176 break;
6177 }
32e8239c
MC		 6178 if (bp->hwrm_spec_code < 0x10707 ||
6179 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6180 bp->br_mode = BRIDGE_MODE_VEB;
6181 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6182 bp->br_mode = BRIDGE_MODE_VEPA;
6183 else
6184 bp->br_mode = BRIDGE_MODE_UNDEF;
cf6645f8 6185
7eb9bb3a
MC		 6186 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6187 if (!bp->max_mtu)
6188 bp->max_mtu = BNXT_MAX_MTU;
6189
cf6645f8
MC		 6190func_qcfg_exit:
6191 mutex_unlock(&bp->hwrm_cmd_lock);
6192 return rc;
6193}
6194
98f04cf0
MC		 6195static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6196{
6197 struct hwrm_func_backing_store_qcaps_input req = {0};
6198 struct hwrm_func_backing_store_qcaps_output *resp =
6199 bp->hwrm_cmd_resp_addr;
6200 int rc;
6201
6202 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6203 return 0;
6204
6205 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6206 mutex_lock(&bp->hwrm_cmd_lock);
6207 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6208 if (!rc) {
6209 struct bnxt_ctx_pg_info *ctx_pg;
6210 struct bnxt_ctx_mem_info *ctx;
6211 int i;
6212
6213 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6214 if (!ctx) {
6215 rc = -ENOMEM;
6216 goto ctx_err;
6217 }
6218 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6219 if (!ctx_pg) {
6220 kfree(ctx);
6221 rc = -ENOMEM;
6222 goto ctx_err;
6223 }
6224 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6225 ctx->tqm_mem[i] = ctx_pg;
6226
6227 bp->ctx = ctx;
6228 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6229 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6230 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6231 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6232 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6233 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6234 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6235 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6236 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6237 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6238 ctx->vnic_max_vnic_entries =
6239 le16_to_cpu(resp->vnic_max_vnic_entries);
6240 ctx->vnic_max_ring_table_entries =
6241 le16_to_cpu(resp->vnic_max_ring_table_entries);
6242 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6243 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6244 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6245 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6246 ctx->tqm_min_entries_per_ring =
6247 le32_to_cpu(resp->tqm_min_entries_per_ring);
6248 ctx->tqm_max_entries_per_ring =
6249 le32_to_cpu(resp->tqm_max_entries_per_ring);
6250 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6251 if (!ctx->tqm_entries_multiple)
6252 ctx->tqm_entries_multiple = 1;
6253 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6254 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
53579e37
DS		 6255 ctx->mrav_num_entries_units =
6256 le16_to_cpu(resp->mrav_num_entries_units);
98f04cf0
MC		 6257 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6258 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6259 } else {
6260 rc = 0;
6261 }
6262ctx_err:
6263 mutex_unlock(&bp->hwrm_cmd_lock);
6264 return rc;
6265}
6266
1b9394e5
MC		 6267static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6268 __le64 *pg_dir)
6269{
6270 u8 pg_size = 0;
6271
6272 if (BNXT_PAGE_SHIFT == 13)
6273 pg_size = 1 << 4;
6274 else if (BNXT_PAGE_SIZE == 16)
6275 pg_size = 2 << 4;
6276
6277 *pg_attr = pg_size;
08fe9d18
MC		 6278 if (rmem->depth >= 1) {
6279 if (rmem->depth == 2)
6280 *pg_attr |= 2;
6281 else
6282 *pg_attr |= 1;
1b9394e5
MC		 6283 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6284 } else {
6285 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6286 }
6287}
6288
6289#define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
6290 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
6291 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
6292 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
6293 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
6294 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6295
6296static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6297{
6298 struct hwrm_func_backing_store_cfg_input req = {0};
6299 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6300 struct bnxt_ctx_pg_info *ctx_pg;
6301 __le32 *num_entries;
6302 __le64 *pg_dir;
53579e37 6303 u32 flags = 0;
1b9394e5
MC		 6304 u8 *pg_attr;
6305 int i, rc;
6306 u32 ena;
6307
6308 if (!ctx)
6309 return 0;
6310
6311 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6312 req.enables = cpu_to_le32(enables);
6313
6314 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6315 ctx_pg = &ctx->qp_mem;
6316 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6317 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6318 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6319 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6320 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6321 &req.qpc_pg_size_qpc_lvl,
6322 &req.qpc_page_dir);
6323 }
6324 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6325 ctx_pg = &ctx->srq_mem;
6326 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6327 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6328 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6329 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6330 &req.srq_pg_size_srq_lvl,
6331 &req.srq_page_dir);
6332 }
6333 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6334 ctx_pg = &ctx->cq_mem;
6335 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6336 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6337 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6338 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6339 &req.cq_page_dir);
6340 }
6341 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6342 ctx_pg = &ctx->vnic_mem;
6343 req.vnic_num_vnic_entries =
6344 cpu_to_le16(ctx->vnic_max_vnic_entries);
6345 req.vnic_num_ring_table_entries =
6346 cpu_to_le16(ctx->vnic_max_ring_table_entries);
6347 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6348 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6349 &req.vnic_pg_size_vnic_lvl,
6350 &req.vnic_page_dir);
6351 }
6352 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6353 ctx_pg = &ctx->stat_mem;
6354 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6355 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6356 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6357 &req.stat_pg_size_stat_lvl,
6358 &req.stat_page_dir);
6359 }
cf6daed0
MC		 6360 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6361 ctx_pg = &ctx->mrav_mem;
6362 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
53579e37
DS		 6363 if (ctx->mrav_num_entries_units)
6364 flags |=
6365 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
cf6daed0
MC		 6366 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6367 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6368 &req.mrav_pg_size_mrav_lvl,
6369 &req.mrav_page_dir);
6370 }
6371 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6372 ctx_pg = &ctx->tim_mem;
6373 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6374 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6375 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6376 &req.tim_pg_size_tim_lvl,
6377 &req.tim_page_dir);
6378 }
1b9394e5
MC		 6379 for (i = 0, num_entries = &req.tqm_sp_num_entries,
6380 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6381 pg_dir = &req.tqm_sp_page_dir,
6382 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6383 i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6384 if (!(enables & ena))
6385 continue;
6386
6387 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6388 ctx_pg = ctx->tqm_mem[i];
6389 *num_entries = cpu_to_le32(ctx_pg->entries);
6390 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6391 }
53579e37 6392 req.flags = cpu_to_le32(flags);
1b9394e5
MC		 6393 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6394 if (rc)
6395 rc = -EIO;
6396 return rc;
6397}
6398
98f04cf0 6399static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
08fe9d18 6400 struct bnxt_ctx_pg_info *ctx_pg)
98f04cf0
MC		 6401{
6402 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6403
98f04cf0
MC		 6404 rmem->page_size = BNXT_PAGE_SIZE;
6405 rmem->pg_arr = ctx_pg->ctx_pg_arr;
6406 rmem->dma_arr = ctx_pg->ctx_dma_arr;
1b9394e5 6407 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
08fe9d18
MC		 6408 if (rmem->depth >= 1)
6409 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
98f04cf0
MC		 6410 return bnxt_alloc_ring(bp, rmem);
6411}
6412
08fe9d18
MC		 6413static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6414 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6415 u8 depth)
6416{
6417 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6418 int rc;
6419
6420 if (!mem_size)
6421 return 0;
6422
6423 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6424 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6425 ctx_pg->nr_pages = 0;
6426 return -EINVAL;
6427 }
6428 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6429 int nr_tbls, i;
6430
6431 rmem->depth = 2;
6432 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6433 GFP_KERNEL);
6434 if (!ctx_pg->ctx_pg_tbl)
6435 return -ENOMEM;
6436 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6437 rmem->nr_pages = nr_tbls;
6438 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6439 if (rc)
6440 return rc;
6441 for (i = 0; i < nr_tbls; i++) {
6442 struct bnxt_ctx_pg_info *pg_tbl;
6443
6444 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6445 if (!pg_tbl)
6446 return -ENOMEM;
6447 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6448 rmem = &pg_tbl->ring_mem;
6449 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6450 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6451 rmem->depth = 1;
6452 rmem->nr_pages = MAX_CTX_PAGES;
6ef982de
MC		 6453 if (i == (nr_tbls - 1)) {
6454 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6455
6456 if (rem)
6457 rmem->nr_pages = rem;
6458 }
08fe9d18
MC		 6459 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6460 if (rc)
6461 break;
6462 }
6463 } else {
6464 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6465 if (rmem->nr_pages > 1 || depth)
6466 rmem->depth = 1;
6467 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6468 }
6469 return rc;
6470}
6471
6472static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6473 struct bnxt_ctx_pg_info *ctx_pg)
6474{
6475 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6476
6477 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6478 ctx_pg->ctx_pg_tbl) {
6479 int i, nr_tbls = rmem->nr_pages;
6480
6481 for (i = 0; i < nr_tbls; i++) {
6482 struct bnxt_ctx_pg_info *pg_tbl;
6483 struct bnxt_ring_mem_info *rmem2;
6484
6485 pg_tbl = ctx_pg->ctx_pg_tbl[i];
6486 if (!pg_tbl)
6487 continue;
6488 rmem2 = &pg_tbl->ring_mem;
6489 bnxt_free_ring(bp, rmem2);
6490 ctx_pg->ctx_pg_arr[i] = NULL;
6491 kfree(pg_tbl);
6492 ctx_pg->ctx_pg_tbl[i] = NULL;
6493 }
6494 kfree(ctx_pg->ctx_pg_tbl);
6495 ctx_pg->ctx_pg_tbl = NULL;
6496 }
6497 bnxt_free_ring(bp, rmem);
6498 ctx_pg->nr_pages = 0;
6499}
6500
98f04cf0
MC		 6501static void bnxt_free_ctx_mem(struct bnxt *bp)
6502{
6503 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6504 int i;
6505
6506 if (!ctx)
6507 return;
6508
6509 if (ctx->tqm_mem[0]) {
6510 for (i = 0; i < bp->max_q + 1; i++)
08fe9d18 6511 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
98f04cf0
MC		 6512 kfree(ctx->tqm_mem[0]);
6513 ctx->tqm_mem[0] = NULL;
6514 }
6515
cf6daed0
MC		 6516 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6517 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
08fe9d18
MC		 6518 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6519 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6520 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6521 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6522 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
98f04cf0
MC		 6523 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6524}
6525
6526static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6527{
6528 struct bnxt_ctx_pg_info *ctx_pg;
6529 struct bnxt_ctx_mem_info *ctx;
1b9394e5 6530 u32 mem_size, ena, entries;
53579e37 6531 u32 num_mr, num_ah;
cf6daed0
MC		 6532 u32 extra_srqs = 0;
6533 u32 extra_qps = 0;
6534 u8 pg_lvl = 1;
98f04cf0
MC		 6535 int i, rc;
6536
6537 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6538 if (rc) {
6539 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6540 rc);
6541 return rc;
6542 }
6543 ctx = bp->ctx;
6544 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6545 return 0;
6546
d629522e 6547 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
cf6daed0
MC		 6548 pg_lvl = 2;
6549 extra_qps = 65536;
6550 extra_srqs = 8192;
6551 }
6552
98f04cf0 6553 ctx_pg = &ctx->qp_mem;
cf6daed0
MC		 6554 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6555 extra_qps;
98f04cf0 6556 mem_size = ctx->qp_entry_size * ctx_pg->entries;
cf6daed0 6557 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6558 if (rc)
6559 return rc;
6560
6561 ctx_pg = &ctx->srq_mem;
cf6daed0 6562 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
98f04cf0 6563 mem_size = ctx->srq_entry_size * ctx_pg->entries;
cf6daed0 6564 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6565 if (rc)
6566 return rc;
6567
6568 ctx_pg = &ctx->cq_mem;
cf6daed0 6569 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
98f04cf0 6570 mem_size = ctx->cq_entry_size * ctx_pg->entries;
cf6daed0 6571 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6572 if (rc)
6573 return rc;
6574
6575 ctx_pg = &ctx->vnic_mem;
6576 ctx_pg->entries = ctx->vnic_max_vnic_entries +
6577 ctx->vnic_max_ring_table_entries;
6578 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
08fe9d18 6579 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6580 if (rc)
6581 return rc;
6582
6583 ctx_pg = &ctx->stat_mem;
6584 ctx_pg->entries = ctx->stat_max_entries;
6585 mem_size = ctx->stat_entry_size * ctx_pg->entries;
08fe9d18 6586 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6587 if (rc)
6588 return rc;
6589
cf6daed0
MC		 6590 ena = 0;
6591 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6592 goto skip_rdma;
6593
6594 ctx_pg = &ctx->mrav_mem;
53579e37
DS		 6595 /* 128K extra is needed to accommodate static AH context
6596 * allocation by f/w.
6597 */
6598 num_mr = 1024 * 256;
6599 num_ah = 1024 * 128;
6600 ctx_pg->entries = num_mr + num_ah;
cf6daed0
MC		 6601 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6602 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2);
6603 if (rc)
6604 return rc;
6605 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
53579e37
DS		 6606 if (ctx->mrav_num_entries_units)
6607 ctx_pg->entries =
6608 ((num_mr / ctx->mrav_num_entries_units) << 16) |
6609 (num_ah / ctx->mrav_num_entries_units);
cf6daed0
MC		 6610
6611 ctx_pg = &ctx->tim_mem;
6612 ctx_pg->entries = ctx->qp_mem.entries;
6613 mem_size = ctx->tim_entry_size * ctx_pg->entries;
6614 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6615 if (rc)
6616 return rc;
6617 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6618
6619skip_rdma:
6620 entries = ctx->qp_max_l2_entries + extra_qps;
98f04cf0
MC		 6621 entries = roundup(entries, ctx->tqm_entries_multiple);
6622 entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6623 ctx->tqm_max_entries_per_ring);
cf6daed0 6624 for (i = 0; i < bp->max_q + 1; i++) {
98f04cf0
MC		 6625 ctx_pg = ctx->tqm_mem[i];
6626 ctx_pg->entries = entries;
6627 mem_size = ctx->tqm_entry_size * entries;
08fe9d18 6628 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6629 if (rc)
6630 return rc;
1b9394e5 6631 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
98f04cf0 6632 }
1b9394e5
MC		 6633 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6634 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6635 if (rc)
6636 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6637 rc);
6638 else
6639 ctx->flags |= BNXT_CTX_FLAG_INITED;
6640
98f04cf0
MC		 6641 return 0;
6642}
6643
db4723b3 6644int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
be0dd9c4
MC		 6645{
6646 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6647 struct hwrm_func_resource_qcaps_input req = {0};
6648 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6649 int rc;
6650
6651 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6652 req.fid = cpu_to_le16(0xffff);
6653
6654 mutex_lock(&bp->hwrm_cmd_lock);
351cbde9
JT		 6655 rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6656 HWRM_CMD_TIMEOUT);
be0dd9c4
MC		 6657 if (rc) {
6658 rc = -EIO;
6659 goto hwrm_func_resc_qcaps_exit;
6660 }
6661
db4723b3
MC		 6662 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6663 if (!all)
6664 goto hwrm_func_resc_qcaps_exit;
6665
be0dd9c4
MC		 6666 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6667 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6668 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6669 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6670 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6671 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6672 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6673 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6674 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6675 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6676 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6677 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6678 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6679 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6680 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6681 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6682
9c1fabdf
MC		 6683 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6684 u16 max_msix = le16_to_cpu(resp->max_msix);
6685
f7588cd8 6686 hw_resc->max_nqs = max_msix;
9c1fabdf
MC		 6687 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6688 }
6689
4673d664
MC		 6690 if (BNXT_PF(bp)) {
6691 struct bnxt_pf_info *pf = &bp->pf;
6692
6693 pf->vf_resv_strategy =
6694 le16_to_cpu(resp->vf_reservation_strategy);
bf82736d 6695 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
4673d664
MC		 6696 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6697 }
be0dd9c4
MC		 6698hwrm_func_resc_qcaps_exit:
6699 mutex_unlock(&bp->hwrm_cmd_lock);
6700 return rc;
6701}
6702
6703static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
c0c050c5
MC		 6704{
6705 int rc = 0;
6706 struct hwrm_func_qcaps_input req = {0};
6707 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6a4f2947
MC		 6708 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6709 u32 flags;
c0c050c5
MC		 6710
6711 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6712 req.fid = cpu_to_le16(0xffff);
6713
6714 mutex_lock(&bp->hwrm_cmd_lock);
6715 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6716 if (rc)
6717 goto hwrm_func_qcaps_exit;
6718
6a4f2947
MC		 6719 flags = le32_to_cpu(resp->flags);
6720 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
e4060d30 6721 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6a4f2947 6722 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
e4060d30 6723 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
55e4398d
VV		 6724 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
6725 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
6154532f
VV		 6726 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
6727 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
e4060d30 6728
7cc5a20e 6729 bp->tx_push_thresh = 0;
6a4f2947 6730 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
7cc5a20e
MC		 6731 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6732
6a4f2947
MC		 6733 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6734 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6735 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6736 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6737 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6738 if (!hw_resc->max_hw_ring_grps)
6739 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6740 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6741 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6742 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6743
c0c050c5
MC		 6744 if (BNXT_PF(bp)) {
6745 struct bnxt_pf_info *pf = &bp->pf;
6746
6747 pf->fw_fid = le16_to_cpu(resp->fid);
6748 pf->port_id = le16_to_cpu(resp->port_id);
87027db1 6749 bp->dev->dev_port = pf->port_id;
11f15ed3 6750 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
c0c050c5
MC		 6751 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6752 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6753 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6754 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6755 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6756 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6757 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6758 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
6a4f2947 6759 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
c1ef146a 6760 bp->flags |= BNXT_FLAG_WOL_CAP;
c0c050c5 6761 } else {
379a80a1 6762#ifdef CONFIG_BNXT_SRIOV
c0c050c5
MC		 6763 struct bnxt_vf_info *vf = &bp->vf;
6764
6765 vf->fw_fid = le16_to_cpu(resp->fid);
7cc5a20e 6766 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
379a80a1 6767#endif
c0c050c5
MC		 6768 }
6769
c0c050c5
MC		 6770hwrm_func_qcaps_exit:
6771 mutex_unlock(&bp->hwrm_cmd_lock);
6772 return rc;
6773}
6774
804fba4e
MC		 6775static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
6776
be0dd9c4
MC		 6777static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
6778{
6779 int rc;
6780
6781 rc = __bnxt_hwrm_func_qcaps(bp);
6782 if (rc)
6783 return rc;
804fba4e
MC		 6784 rc = bnxt_hwrm_queue_qportcfg(bp);
6785 if (rc) {
6786 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
6787 return rc;
6788 }
be0dd9c4 6789 if (bp->hwrm_spec_code >= 0x10803) {
98f04cf0
MC		 6790 rc = bnxt_alloc_ctx_mem(bp);
6791 if (rc)
6792 return rc;
db4723b3 6793 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
be0dd9c4 6794 if (!rc)
97381a18 6795 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
be0dd9c4
MC		 6796 }
6797 return 0;
6798}
6799
e969ae5b
MC		 6800static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
6801{
6802 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
6803 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
6804 int rc = 0;
6805 u32 flags;
6806
6807 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
6808 return 0;
6809
6810 resp = bp->hwrm_cmd_resp_addr;
6811 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
6812
6813 mutex_lock(&bp->hwrm_cmd_lock);
6814 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6815 if (rc)
6816 goto hwrm_cfa_adv_qcaps_exit;
6817
6818 flags = le32_to_cpu(resp->flags);
6819 if (flags &
6820 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED)
6821 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX;
6822
6823hwrm_cfa_adv_qcaps_exit:
6824 mutex_unlock(&bp->hwrm_cmd_lock);
6825 return rc;
6826}
6827
c0c050c5
MC		 6828static int bnxt_hwrm_func_reset(struct bnxt *bp)
6829{
6830 struct hwrm_func_reset_input req = {0};
6831
6832 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
6833 req.enables = 0;
6834
6835 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
6836}
6837
6838static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
6839{
6840 int rc = 0;
6841 struct hwrm_queue_qportcfg_input req = {0};
6842 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
aabfc016
MC		 6843 u8 i, j, *qptr;
6844 bool no_rdma;
c0c050c5
MC		 6845
6846 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
6847
6848 mutex_lock(&bp->hwrm_cmd_lock);
6849 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6850 if (rc)
6851 goto qportcfg_exit;
6852
6853 if (!resp->max_configurable_queues) {
6854 rc = -EINVAL;
6855 goto qportcfg_exit;
6856 }
6857 bp->max_tc = resp->max_configurable_queues;
87c374de 6858 bp->max_lltc = resp->max_configurable_lossless_queues;
c0c050c5
MC		 6859 if (bp->max_tc > BNXT_MAX_QUEUE)
6860 bp->max_tc = BNXT_MAX_QUEUE;
6861
aabfc016
MC		 6862 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
6863 qptr = &resp->queue_id0;
6864 for (i = 0, j = 0; i < bp->max_tc; i++) {
98f04cf0
MC		 6865 bp->q_info[j].queue_id = *qptr;
6866 bp->q_ids[i] = *qptr++;
aabfc016
MC		 6867 bp->q_info[j].queue_profile = *qptr++;
6868 bp->tc_to_qidx[j] = j;
6869 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
6870 (no_rdma && BNXT_PF(bp)))
6871 j++;
6872 }
98f04cf0 6873 bp->max_q = bp->max_tc;
aabfc016
MC		 6874 bp->max_tc = max_t(u8, j, 1);
6875
441cabbb
MC		 6876 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
6877 bp->max_tc = 1;
6878
87c374de
MC		 6879 if (bp->max_lltc > bp->max_tc)
6880 bp->max_lltc = bp->max_tc;
6881
c0c050c5
MC		 6882qportcfg_exit:
6883 mutex_unlock(&bp->hwrm_cmd_lock);
6884 return rc;
6885}
6886
6887static int bnxt_hwrm_ver_get(struct bnxt *bp)
6888{
6889 int rc;
6890 struct hwrm_ver_get_input req = {0};
6891 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
e605db80 6892 u32 dev_caps_cfg;
c0c050c5 6893
e6ef2699 6894 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
c0c050c5
MC		 6895 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
6896 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
6897 req.hwrm_intf_min = HWRM_VERSION_MINOR;
6898 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
6899 mutex_lock(&bp->hwrm_cmd_lock);
6900 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6901 if (rc)
6902 goto hwrm_ver_get_exit;
6903
6904 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
6905
894aa69a
MC		 6906 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
6907 resp->hwrm_intf_min_8b << 8 |
6908 resp->hwrm_intf_upd_8b;
6909 if (resp->hwrm_intf_maj_8b < 1) {
c193554e 6910 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
894aa69a
MC		 6911 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
6912 resp->hwrm_intf_upd_8b);
c193554e 6913 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
c0c050c5 6914 }
431aa1eb 6915 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
894aa69a
MC		 6916 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
6917 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
c0c050c5 6918
691aa620
VV		 6919 if (strlen(resp->active_pkg_name)) {
6920 int fw_ver_len = strlen(bp->fw_ver_str);
6921
6922 snprintf(bp->fw_ver_str + fw_ver_len,
6923 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
6924 resp->active_pkg_name);
6925 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
6926 }
6927
ff4fe81d
MC		 6928 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
6929 if (!bp->hwrm_cmd_timeout)
6930 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
6931
1dfddc41 6932 if (resp->hwrm_intf_maj_8b >= 1) {
e6ef2699 6933 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
1dfddc41
MC		 6934 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
6935 }
6936 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
6937 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
e6ef2699 6938
659c805c 6939 bp->chip_num = le16_to_cpu(resp->chip_num);
3e8060fa
PS		 6940 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
6941 !resp->chip_metal)
6942 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
659c805c 6943
e605db80
DK		 6944 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
6945 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
6946 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
97381a18 6947 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
e605db80 6948
760b6d33
VD		 6949 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
6950 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
6951
abd43a13
VD		 6952 if (dev_caps_cfg &
6953 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
6954 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
6955
2a516444
MC		 6956 if (dev_caps_cfg &
6957 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
6958 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
6959
e969ae5b
MC		 6960 if (dev_caps_cfg &
6961 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
6962 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
6963
c0c050c5
MC		 6964hwrm_ver_get_exit:
6965 mutex_unlock(&bp->hwrm_cmd_lock);
6966 return rc;
6967}
6968
5ac67d8b
RS		 6969int bnxt_hwrm_fw_set_time(struct bnxt *bp)
6970{
6971 struct hwrm_fw_set_time_input req = {0};
7dfaa7bc
AB		 6972 struct tm tm;
6973 time64_t now = ktime_get_real_seconds();
5ac67d8b 6974
ca2c39e2
MC		 6975 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
6976 bp->hwrm_spec_code < 0x10400)
5ac67d8b
RS		 6977 return -EOPNOTSUPP;
6978
7dfaa7bc 6979 time64_to_tm(now, 0, &tm);
5ac67d8b
RS		 6980 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
6981 req.year = cpu_to_le16(1900 + tm.tm_year);
6982 req.month = 1 + tm.tm_mon;
6983 req.day = tm.tm_mday;
6984 req.hour = tm.tm_hour;
6985 req.minute = tm.tm_min;
6986 req.second = tm.tm_sec;
6987 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6988}
6989
3bdf56c4
MC		 6990static int bnxt_hwrm_port_qstats(struct bnxt *bp)
6991{
6992 int rc;
6993 struct bnxt_pf_info *pf = &bp->pf;
6994 struct hwrm_port_qstats_input req = {0};
6995
6996 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
6997 return 0;
6998
6999 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7000 req.port_id = cpu_to_le16(pf->port_id);
7001 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
7002 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
7003 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7004 return rc;
7005}
7006
00db3cba
VV		 7007static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
7008{
36e53349 7009 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
e37fed79 7010 struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
00db3cba
VV		 7011 struct hwrm_port_qstats_ext_input req = {0};
7012 struct bnxt_pf_info *pf = &bp->pf;
ad361adf 7013 u32 tx_stat_size;
36e53349 7014 int rc;
00db3cba
VV		 7015
7016 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7017 return 0;
7018
7019 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7020 req.port_id = cpu_to_le16(pf->port_id);
7021 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7022 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
ad361adf
MC		 7023 tx_stat_size = bp->hw_tx_port_stats_ext ?
7024 sizeof(*bp->hw_tx_port_stats_ext) : 0;
7025 req.tx_stat_size = cpu_to_le16(tx_stat_size);
36e53349
MC		 7026 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
7027 mutex_lock(&bp->hwrm_cmd_lock);
7028 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7029 if (!rc) {
7030 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
ad361adf
MC		 7031 bp->fw_tx_stats_ext_size = tx_stat_size ?
7032 le16_to_cpu(resp->tx_stat_size) / 8 : 0;
36e53349
MC		 7033 } else {
7034 bp->fw_rx_stats_ext_size = 0;
7035 bp->fw_tx_stats_ext_size = 0;
7036 }
e37fed79
MC		 7037 if (bp->fw_tx_stats_ext_size <=
7038 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7039 mutex_unlock(&bp->hwrm_cmd_lock);
7040 bp->pri2cos_valid = 0;
7041 return rc;
7042 }
7043
7044 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7045 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7046
7047 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7048 if (!rc) {
7049 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7050 u8 *pri2cos;
7051 int i, j;
7052
7053 resp2 = bp->hwrm_cmd_resp_addr;
7054 pri2cos = &resp2->pri0_cos_queue_id;
7055 for (i = 0; i < 8; i++) {
7056 u8 queue_id = pri2cos[i];
7057
7058 for (j = 0; j < bp->max_q; j++) {
7059 if (bp->q_ids[j] == queue_id)
7060 bp->pri2cos[i] = j;
7061 }
7062 }
7063 bp->pri2cos_valid = 1;
7064 }
36e53349
MC		 7065 mutex_unlock(&bp->hwrm_cmd_lock);
7066 return rc;
00db3cba
VV		 7067}
7068
55e4398d
VV		 7069static int bnxt_hwrm_pcie_qstats(struct bnxt *bp)
7070{
7071 struct hwrm_pcie_qstats_input req = {0};
7072
7073 if (!(bp->flags & BNXT_FLAG_PCIE_STATS))
7074 return 0;
7075
7076 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1);
7077 req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats));
7078 req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map);
7079 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7080}
7081
c0c050c5
MC		 7082static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7083{
7084 if (bp->vxlan_port_cnt) {
7085 bnxt_hwrm_tunnel_dst_port_free(
7086 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7087 }
7088 bp->vxlan_port_cnt = 0;
7089 if (bp->nge_port_cnt) {
7090 bnxt_hwrm_tunnel_dst_port_free(
7091 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7092 }
7093 bp->nge_port_cnt = 0;
7094}
7095
7096static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7097{
7098 int rc, i;
7099 u32 tpa_flags = 0;
7100
7101 if (set_tpa)
7102 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7103 for (i = 0; i < bp->nr_vnics; i++) {
7104 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7105 if (rc) {
7106 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
23e12c89 7107 i, rc);
c0c050c5
MC		 7108 return rc;
7109 }
7110 }
7111 return 0;
7112}
7113
7114static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7115{
7116 int i;
7117
7118 for (i = 0; i < bp->nr_vnics; i++)
7119 bnxt_hwrm_vnic_set_rss(bp, i, false);
7120}
7121
7122static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7123 bool irq_re_init)
7124{
7125 if (bp->vnic_info) {
7126 bnxt_hwrm_clear_vnic_filter(bp);
7127 /* clear all RSS setting before free vnic ctx */
7128 bnxt_hwrm_clear_vnic_rss(bp);
7129 bnxt_hwrm_vnic_ctx_free(bp);
7130 /* before free the vnic, undo the vnic tpa settings */
7131 if (bp->flags & BNXT_FLAG_TPA)
7132 bnxt_set_tpa(bp, false);
7133 bnxt_hwrm_vnic_free(bp);
7134 }
7135 bnxt_hwrm_ring_free(bp, close_path);
7136 bnxt_hwrm_ring_grp_free(bp);
7137 if (irq_re_init) {
7138 bnxt_hwrm_stat_ctx_free(bp);
7139 bnxt_hwrm_free_tunnel_ports(bp);
7140 }
7141}
7142
39d8ba2e
MC		 7143static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7144{
7145 struct hwrm_func_cfg_input req = {0};
7146 int rc;
7147
7148 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7149 req.fid = cpu_to_le16(0xffff);
7150 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7151 if (br_mode == BRIDGE_MODE_VEB)
7152 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7153 else if (br_mode == BRIDGE_MODE_VEPA)
7154 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7155 else
7156 return -EINVAL;
7157 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7158 if (rc)
7159 rc = -EIO;
7160 return rc;
7161}
7162
c3480a60
MC		 7163static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7164{
7165 struct hwrm_func_cfg_input req = {0};
7166 int rc;
7167
7168 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7169 return 0;
7170
7171 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7172 req.fid = cpu_to_le16(0xffff);
7173 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
d4f52de0 7174 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
c3480a60 7175 if (size == 128)
d4f52de0 7176 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
c3480a60
MC		 7177
7178 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7179 if (rc)
7180 rc = -EIO;
7181 return rc;
7182}
7183
7b3af4f7 7184static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
c0c050c5 7185{
ae10ae74 7186 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
c0c050c5
MC		 7187 int rc;
7188
ae10ae74
MC		 7189 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7190 goto skip_rss_ctx;
7191
c0c050c5 7192 /* allocate context for vnic */
94ce9caa 7193 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
c0c050c5
MC		 7194 if (rc) {
7195 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7196 vnic_id, rc);
7197 goto vnic_setup_err;
7198 }
7199 bp->rsscos_nr_ctxs++;
7200
94ce9caa
PS		 7201 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7202 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7203 if (rc) {
7204 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7205 vnic_id, rc);
7206 goto vnic_setup_err;
7207 }
7208 bp->rsscos_nr_ctxs++;
7209 }
7210
ae10ae74 7211skip_rss_ctx:
c0c050c5
MC		 7212 /* configure default vnic, ring grp */
7213 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7214 if (rc) {
7215 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7216 vnic_id, rc);
7217 goto vnic_setup_err;
7218 }
7219
7220 /* Enable RSS hashing on vnic */
7221 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7222 if (rc) {
7223 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7224 vnic_id, rc);
7225 goto vnic_setup_err;
7226 }
7227
7228 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7229 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7230 if (rc) {
7231 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7232 vnic_id, rc);
7233 }
7234 }
7235
7236vnic_setup_err:
7237 return rc;
7238}
7239
7b3af4f7
MC		 7240static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7241{
7242 int rc, i, nr_ctxs;
7243
7244 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
7245 for (i = 0; i < nr_ctxs; i++) {
7246 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7247 if (rc) {
7248 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7249 vnic_id, i, rc);
7250 break;
7251 }
7252 bp->rsscos_nr_ctxs++;
7253 }
7254 if (i < nr_ctxs)
7255 return -ENOMEM;
7256
7257 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7258 if (rc) {
7259 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7260 vnic_id, rc);
7261 return rc;
7262 }
7263 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7264 if (rc) {
7265 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7266 vnic_id, rc);
7267 return rc;
7268 }
7269 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7270 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7271 if (rc) {
7272 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7273 vnic_id, rc);
7274 }
7275 }
7276 return rc;
7277}
7278
7279static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7280{
7281 if (bp->flags & BNXT_FLAG_CHIP_P5)
7282 return __bnxt_setup_vnic_p5(bp, vnic_id);
7283 else
7284 return __bnxt_setup_vnic(bp, vnic_id);
7285}
7286
c0c050c5
MC		 7287static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
7288{
7289#ifdef CONFIG_RFS_ACCEL
7290 int i, rc = 0;
7291
9b3d15e6
MC		 7292 if (bp->flags & BNXT_FLAG_CHIP_P5)
7293 return 0;
7294
c0c050c5 7295 for (i = 0; i < bp->rx_nr_rings; i++) {
ae10ae74 7296 struct bnxt_vnic_info *vnic;
c0c050c5
MC		 7297 u16 vnic_id = i + 1;
7298 u16 ring_id = i;
7299
7300 if (vnic_id >= bp->nr_vnics)
7301 break;
7302
ae10ae74
MC		 7303 vnic = &bp->vnic_info[vnic_id];
7304 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7305 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7306 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
b81a90d3 7307 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
c0c050c5
MC		 7308 if (rc) {
7309 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7310 vnic_id, rc);
7311 break;
7312 }
7313 rc = bnxt_setup_vnic(bp, vnic_id);
7314 if (rc)
7315 break;
7316 }
7317 return rc;
7318#else
7319 return 0;
7320#endif
7321}
7322
17c71ac3
MC		 7323/* Allow PF and VF with default VLAN to be in promiscuous mode */
7324static bool bnxt_promisc_ok(struct bnxt *bp)
7325{
7326#ifdef CONFIG_BNXT_SRIOV
7327 if (BNXT_VF(bp) && !bp->vf.vlan)
7328 return false;
7329#endif
7330 return true;
7331}
7332
dc52c6c7
PS		 7333static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7334{
7335 unsigned int rc = 0;
7336
7337 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7338 if (rc) {
7339 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7340 rc);
7341 return rc;
7342 }
7343
7344 rc = bnxt_hwrm_vnic_cfg(bp, 1);
7345 if (rc) {
7346 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7347 rc);
7348 return rc;
7349 }
7350 return rc;
7351}
7352
b664f008 7353static int bnxt_cfg_rx_mode(struct bnxt *);
7d2837dd 7354static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
b664f008 7355
c0c050c5
MC		 7356static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7357{
7d2837dd 7358 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
c0c050c5 7359 int rc = 0;
76595193 7360 unsigned int rx_nr_rings = bp->rx_nr_rings;
c0c050c5
MC		 7361
7362 if (irq_re_init) {
7363 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7364 if (rc) {
7365 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7366 rc);
7367 goto err_out;
7368 }
7369 }
7370
7371 rc = bnxt_hwrm_ring_alloc(bp);
7372 if (rc) {
7373 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7374 goto err_out;
7375 }
7376
7377 rc = bnxt_hwrm_ring_grp_alloc(bp);
7378 if (rc) {
7379 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7380 goto err_out;
7381 }
7382
76595193
PS		 7383 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7384 rx_nr_rings--;
7385
c0c050c5 7386 /* default vnic 0 */
76595193 7387 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
c0c050c5
MC		 7388 if (rc) {
7389 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7390 goto err_out;
7391 }
7392
7393 rc = bnxt_setup_vnic(bp, 0);
7394 if (rc)
7395 goto err_out;
7396
7397 if (bp->flags & BNXT_FLAG_RFS) {
7398 rc = bnxt_alloc_rfs_vnics(bp);
7399 if (rc)
7400 goto err_out;
7401 }
7402
7403 if (bp->flags & BNXT_FLAG_TPA) {
7404 rc = bnxt_set_tpa(bp, true);
7405 if (rc)
7406 goto err_out;
7407 }
7408
7409 if (BNXT_VF(bp))
7410 bnxt_update_vf_mac(bp);
7411
7412 /* Filter for default vnic 0 */
7413 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7414 if (rc) {
7415 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7416 goto err_out;
7417 }
7d2837dd 7418 vnic->uc_filter_count = 1;
c0c050c5 7419
30e33848
MC		 7420 vnic->rx_mask = 0;
7421 if (bp->dev->flags & IFF_BROADCAST)
7422 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5 7423
17c71ac3 7424 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7d2837dd
MC		 7425 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7426
7427 if (bp->dev->flags & IFF_ALLMULTI) {
7428 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7429 vnic->mc_list_count = 0;
7430 } else {
7431 u32 mask = 0;
7432
7433 bnxt_mc_list_updated(bp, &mask);
7434 vnic->rx_mask |= mask;
7435 }
c0c050c5 7436
b664f008
MC		 7437 rc = bnxt_cfg_rx_mode(bp);
7438 if (rc)
c0c050c5 7439 goto err_out;
c0c050c5
MC		 7440
7441 rc = bnxt_hwrm_set_coal(bp);
7442 if (rc)
7443 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
dc52c6c7
PS		 7444 rc);
7445
7446 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7447 rc = bnxt_setup_nitroa0_vnic(bp);
7448 if (rc)
7449 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7450 rc);
7451 }
c0c050c5 7452
cf6645f8
MC		 7453 if (BNXT_VF(bp)) {
7454 bnxt_hwrm_func_qcfg(bp);
7455 netdev_update_features(bp->dev);
7456 }
7457
c0c050c5
MC		 7458 return 0;
7459
7460err_out:
7461 bnxt_hwrm_resource_free(bp, 0, true);
7462
7463 return rc;
7464}
7465
7466static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7467{
7468 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7469 return 0;
7470}
7471
7472static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7473{
2247925f 7474 bnxt_init_cp_rings(bp);
c0c050c5
MC		 7475 bnxt_init_rx_rings(bp);
7476 bnxt_init_tx_rings(bp);
7477 bnxt_init_ring_grps(bp, irq_re_init);
7478 bnxt_init_vnics(bp);
7479
7480 return bnxt_init_chip(bp, irq_re_init);
7481}
7482
c0c050c5
MC		 7483static int bnxt_set_real_num_queues(struct bnxt *bp)
7484{
7485 int rc;
7486 struct net_device *dev = bp->dev;
7487
5f449249
MC		 7488 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7489 bp->tx_nr_rings_xdp);
c0c050c5
MC		 7490 if (rc)
7491 return rc;
7492
7493 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7494 if (rc)
7495 return rc;
7496
7497#ifdef CONFIG_RFS_ACCEL
45019a18 7498 if (bp->flags & BNXT_FLAG_RFS)
c0c050c5 7499 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
c0c050c5
MC		 7500#endif
7501
7502 return rc;
7503}
7504
6e6c5a57
MC		 7505static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7506 bool shared)
7507{
7508 int _rx = *rx, _tx = *tx;
7509
7510 if (shared) {
7511 *rx = min_t(int, _rx, max);
7512 *tx = min_t(int, _tx, max);
7513 } else {
7514 if (max < 2)
7515 return -ENOMEM;
7516
7517 while (_rx + _tx > max) {
7518 if (_rx > _tx && _rx > 1)
7519 _rx--;
7520 else if (_tx > 1)
7521 _tx--;
7522 }
7523 *rx = _rx;
7524 *tx = _tx;
7525 }
7526 return 0;
7527}
7528
7809592d
MC		 7529static void bnxt_setup_msix(struct bnxt *bp)
7530{
7531 const int len = sizeof(bp->irq_tbl[0].name);
7532 struct net_device *dev = bp->dev;
7533 int tcs, i;
7534
7535 tcs = netdev_get_num_tc(dev);
7536 if (tcs > 1) {
d1e7925e 7537 int i, off, count;
7809592d 7538
d1e7925e
MC		 7539 for (i = 0; i < tcs; i++) {
7540 count = bp->tx_nr_rings_per_tc;
7541 off = i * count;
7542 netdev_set_tc_queue(dev, i, count, off);
7809592d
MC		 7543 }
7544 }
7545
7546 for (i = 0; i < bp->cp_nr_rings; i++) {
e5811b8c 7547 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7809592d
MC		 7548 char *attr;
7549
7550 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7551 attr = "TxRx";
7552 else if (i < bp->rx_nr_rings)
7553 attr = "rx";
7554 else
7555 attr = "tx";
7556
e5811b8c
MC		 7557 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7558 attr, i);
7559 bp->irq_tbl[map_idx].handler = bnxt_msix;
7809592d
MC		 7560 }
7561}
7562
7563static void bnxt_setup_inta(struct bnxt *bp)
7564{
7565 const int len = sizeof(bp->irq_tbl[0].name);
7566
7567 if (netdev_get_num_tc(bp->dev))
7568 netdev_reset_tc(bp->dev);
7569
7570 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7571 0);
7572 bp->irq_tbl[0].handler = bnxt_inta;
7573}
7574
7575static int bnxt_setup_int_mode(struct bnxt *bp)
7576{
7577 int rc;
7578
7579 if (bp->flags & BNXT_FLAG_USING_MSIX)
7580 bnxt_setup_msix(bp);
7581 else
7582 bnxt_setup_inta(bp);
7583
7584 rc = bnxt_set_real_num_queues(bp);
7585 return rc;
7586}
7587
b7429954 7588#ifdef CONFIG_RFS_ACCEL
8079e8f1
MC		 7589static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7590{
6a4f2947 7591 return bp->hw_resc.max_rsscos_ctxs;
8079e8f1
MC		 7592}
7593
7594static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7595{
6a4f2947 7596 return bp->hw_resc.max_vnics;
8079e8f1 7597}
b7429954 7598#endif
8079e8f1 7599
e4060d30
MC		 7600unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7601{
6a4f2947 7602 return bp->hw_resc.max_stat_ctxs;
e4060d30
MC		 7603}
7604
7605unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7606{
6a4f2947 7607 return bp->hw_resc.max_cp_rings;
e4060d30
MC		 7608}
7609
e916b081 7610static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
a588e458 7611{
c0b8cda0
MC		 7612 unsigned int cp = bp->hw_resc.max_cp_rings;
7613
7614 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7615 cp -= bnxt_get_ulp_msix_num(bp);
7616
7617 return cp;
a588e458
MC		 7618}
7619
ad95c27b 7620static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7809592d 7621{
6a4f2947
MC		 7622 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7623
f7588cd8
MC		 7624 if (bp->flags & BNXT_FLAG_CHIP_P5)
7625 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7626
6a4f2947 7627 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7809592d
MC		 7628}
7629
30f52947 7630static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
33c2657e 7631{
6a4f2947 7632 bp->hw_resc.max_irqs = max_irqs;
33c2657e
MC		 7633}
7634
e916b081
MC		 7635unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7636{
7637 unsigned int cp;
7638
7639 cp = bnxt_get_max_func_cp_rings_for_en(bp);
7640 if (bp->flags & BNXT_FLAG_CHIP_P5)
7641 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
7642 else
7643 return cp - bp->cp_nr_rings;
7644}
7645
c027c6b4
VV		 7646unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
7647{
d77b1ad8 7648 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
c027c6b4
VV		 7649}
7650
fbcfc8e4
MC		 7651int bnxt_get_avail_msix(struct bnxt *bp, int num)
7652{
7653 int max_cp = bnxt_get_max_func_cp_rings(bp);
7654 int max_irq = bnxt_get_max_func_irqs(bp);
7655 int total_req = bp->cp_nr_rings + num;
7656 int max_idx, avail_msix;
7657
75720e63
MC		 7658 max_idx = bp->total_irqs;
7659 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7660 max_idx = min_t(int, bp->total_irqs, max_cp);
fbcfc8e4 7661 avail_msix = max_idx - bp->cp_nr_rings;
f1ca94de 7662 if (!BNXT_NEW_RM(bp) || avail_msix >= num)
fbcfc8e4
MC		 7663 return avail_msix;
7664
7665 if (max_irq < total_req) {
7666 num = max_irq - bp->cp_nr_rings;
7667 if (num <= 0)
7668 return 0;
7669 }
7670 return num;
7671}
7672
08654eb2
MC		 7673static int bnxt_get_num_msix(struct bnxt *bp)
7674{
f1ca94de 7675 if (!BNXT_NEW_RM(bp))
08654eb2
MC		 7676 return bnxt_get_max_func_irqs(bp);
7677
c0b8cda0 7678 return bnxt_nq_rings_in_use(bp);
08654eb2
MC		 7679}
7680
7809592d 7681static int bnxt_init_msix(struct bnxt *bp)
c0c050c5 7682{
fbcfc8e4 7683 int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
7809592d 7684 struct msix_entry *msix_ent;
c0c050c5 7685
08654eb2
MC		 7686 total_vecs = bnxt_get_num_msix(bp);
7687 max = bnxt_get_max_func_irqs(bp);
7688 if (total_vecs > max)
7689 total_vecs = max;
7690
2773dfb2
MC		 7691 if (!total_vecs)
7692 return 0;
7693
c0c050c5
MC		 7694 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
7695 if (!msix_ent)
7696 return -ENOMEM;
7697
7698 for (i = 0; i < total_vecs; i++) {
7699 msix_ent[i].entry = i;
7700 msix_ent[i].vector = 0;
7701 }
7702
01657bcd
MC		 7703 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
7704 min = 2;
7705
7706 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
fbcfc8e4
MC		 7707 ulp_msix = bnxt_get_ulp_msix_num(bp);
7708 if (total_vecs < 0 || total_vecs < ulp_msix) {
c0c050c5
MC		 7709 rc = -ENODEV;
7710 goto msix_setup_exit;
7711 }
7712
7713 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
7714 if (bp->irq_tbl) {
7809592d
MC		 7715 for (i = 0; i < total_vecs; i++)
7716 bp->irq_tbl[i].vector = msix_ent[i].vector;
c0c050c5 7717
7809592d 7718 bp->total_irqs = total_vecs;
c0c050c5 7719 /* Trim rings based upon num of vectors allocated */
6e6c5a57 7720 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
fbcfc8e4 7721 total_vecs - ulp_msix, min == 1);
6e6c5a57
MC		 7722 if (rc)
7723 goto msix_setup_exit;
7724
7809592d
MC		 7725 bp->cp_nr_rings = (min == 1) ?
7726 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7727 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5 7728
c0c050c5
MC		 7729 } else {
7730 rc = -ENOMEM;
7731 goto msix_setup_exit;
7732 }
7733 bp->flags |= BNXT_FLAG_USING_MSIX;
7734 kfree(msix_ent);
7735 return 0;
7736
7737msix_setup_exit:
7809592d
MC		 7738 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
7739 kfree(bp->irq_tbl);
7740 bp->irq_tbl = NULL;
c0c050c5
MC		 7741 pci_disable_msix(bp->pdev);
7742 kfree(msix_ent);
7743 return rc;
7744}
7745
7809592d 7746static int bnxt_init_inta(struct bnxt *bp)
c0c050c5 7747{
c0c050c5 7748 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7809592d
MC		 7749 if (!bp->irq_tbl)
7750 return -ENOMEM;
7751
7752 bp->total_irqs = 1;
c0c050c5
MC		 7753 bp->rx_nr_rings = 1;
7754 bp->tx_nr_rings = 1;
7755 bp->cp_nr_rings = 1;
01657bcd 7756 bp->flags |= BNXT_FLAG_SHARED_RINGS;
c0c050c5 7757 bp->irq_tbl[0].vector = bp->pdev->irq;
7809592d 7758 return 0;
c0c050c5
MC		 7759}
7760
7809592d 7761static int bnxt_init_int_mode(struct bnxt *bp)
c0c050c5
MC		 7762{
7763 int rc = 0;
7764
7765 if (bp->flags & BNXT_FLAG_MSIX_CAP)
7809592d 7766 rc = bnxt_init_msix(bp);
c0c050c5 7767
1fa72e29 7768 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
c0c050c5 7769 /* fallback to INTA */
7809592d 7770 rc = bnxt_init_inta(bp);
c0c050c5
MC		 7771 }
7772 return rc;
7773}
7774
7809592d
MC		 7775static void bnxt_clear_int_mode(struct bnxt *bp)
7776{
7777 if (bp->flags & BNXT_FLAG_USING_MSIX)
7778 pci_disable_msix(bp->pdev);
7779
7780 kfree(bp->irq_tbl);
7781 bp->irq_tbl = NULL;
7782 bp->flags &= ~BNXT_FLAG_USING_MSIX;
7783}
7784
1b3f0b75 7785int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
674f50a5 7786{
674f50a5 7787 int tcs = netdev_get_num_tc(bp->dev);
1b3f0b75 7788 bool irq_cleared = false;
674f50a5
MC		 7789 int rc;
7790
7791 if (!bnxt_need_reserve_rings(bp))
7792 return 0;
7793
1b3f0b75
MC		 7794 if (irq_re_init && BNXT_NEW_RM(bp) &&
7795 bnxt_get_num_msix(bp) != bp->total_irqs) {
ec86f14e 7796 bnxt_ulp_irq_stop(bp);
674f50a5 7797 bnxt_clear_int_mode(bp);
1b3f0b75 7798 irq_cleared = true;
36d65be9
MC		 7799 }
7800 rc = __bnxt_reserve_rings(bp);
1b3f0b75 7801 if (irq_cleared) {
36d65be9
MC		 7802 if (!rc)
7803 rc = bnxt_init_int_mode(bp);
ec86f14e 7804 bnxt_ulp_irq_restart(bp, rc);
36d65be9
MC		 7805 }
7806 if (rc) {
7807 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
7808 return rc;
674f50a5
MC		 7809 }
7810 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
7811 netdev_err(bp->dev, "tx ring reservation failure\n");
7812 netdev_reset_tc(bp->dev);
7813 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
7814 return -ENOMEM;
7815 }
674f50a5
MC		 7816 return 0;
7817}
7818
c0c050c5
MC		 7819static void bnxt_free_irq(struct bnxt *bp)
7820{
7821 struct bnxt_irq *irq;
7822 int i;
7823
7824#ifdef CONFIG_RFS_ACCEL
7825 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
7826 bp->dev->rx_cpu_rmap = NULL;
7827#endif
cb98526b 7828 if (!bp->irq_tbl || !bp->bnapi)
c0c050c5
MC		 7829 return;
7830
7831 for (i = 0; i < bp->cp_nr_rings; i++) {
e5811b8c
MC		 7832 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7833
7834 irq = &bp->irq_tbl[map_idx];
56f0fd80
VV		 7835 if (irq->requested) {
7836 if (irq->have_cpumask) {
7837 irq_set_affinity_hint(irq->vector, NULL);
7838 free_cpumask_var(irq->cpu_mask);
7839 irq->have_cpumask = 0;
7840 }
c0c050c5 7841 free_irq(irq->vector, bp->bnapi[i]);
56f0fd80
VV		 7842 }
7843
c0c050c5
MC		 7844 irq->requested = 0;
7845 }
c0c050c5
MC		 7846}
7847
7848static int bnxt_request_irq(struct bnxt *bp)
7849{
b81a90d3 7850 int i, j, rc = 0;
c0c050c5
MC		 7851 unsigned long flags = 0;
7852#ifdef CONFIG_RFS_ACCEL
e5811b8c 7853 struct cpu_rmap *rmap;
c0c050c5
MC		 7854#endif
7855
e5811b8c
MC		 7856 rc = bnxt_setup_int_mode(bp);
7857 if (rc) {
7858 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
7859 rc);
7860 return rc;
7861 }
7862#ifdef CONFIG_RFS_ACCEL
7863 rmap = bp->dev->rx_cpu_rmap;
7864#endif
c0c050c5
MC		 7865 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
7866 flags = IRQF_SHARED;
7867
b81a90d3 7868 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
e5811b8c
MC		 7869 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7870 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
7871
c0c050c5 7872#ifdef CONFIG_RFS_ACCEL
b81a90d3 7873 if (rmap && bp->bnapi[i]->rx_ring) {
c0c050c5
MC		 7874 rc = irq_cpu_rmap_add(rmap, irq->vector);
7875 if (rc)
7876 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
b81a90d3
MC		 7877 j);
7878 j++;
c0c050c5
MC		 7879 }
7880#endif
7881 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
7882 bp->bnapi[i]);
7883 if (rc)
7884 break;
7885
7886 irq->requested = 1;
56f0fd80
VV		 7887
7888 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
7889 int numa_node = dev_to_node(&bp->pdev->dev);
7890
7891 irq->have_cpumask = 1;
7892 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
7893 irq->cpu_mask);
7894 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
7895 if (rc) {
7896 netdev_warn(bp->dev,
7897 "Set affinity failed, IRQ = %d\n",
7898 irq->vector);
7899 break;
7900 }
7901 }
c0c050c5
MC		 7902 }
7903 return rc;
7904}
7905
7906static void bnxt_del_napi(struct bnxt *bp)
7907{
7908 int i;
7909
7910 if (!bp->bnapi)
7911 return;
7912
7913 for (i = 0; i < bp->cp_nr_rings; i++) {
7914 struct bnxt_napi *bnapi = bp->bnapi[i];
7915
7916 napi_hash_del(&bnapi->napi);
7917 netif_napi_del(&bnapi->napi);
7918 }
e5f6f564
ED		 7919 /* We called napi_hash_del() before netif_napi_del(), we need
7920 * to respect an RCU grace period before freeing napi structures.
7921 */
7922 synchronize_net();
c0c050c5
MC		 7923}
7924
7925static void bnxt_init_napi(struct bnxt *bp)
7926{
7927 int i;
10bbdaf5 7928 unsigned int cp_nr_rings = bp->cp_nr_rings;
c0c050c5
MC		 7929 struct bnxt_napi *bnapi;
7930
7931 if (bp->flags & BNXT_FLAG_USING_MSIX) {
0fcec985
MC		 7932 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
7933
7934 if (bp->flags & BNXT_FLAG_CHIP_P5)
7935 poll_fn = bnxt_poll_p5;
7936 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10bbdaf5
PS		 7937 cp_nr_rings--;
7938 for (i = 0; i < cp_nr_rings; i++) {
c0c050c5 7939 bnapi = bp->bnapi[i];
0fcec985 7940 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
c0c050c5 7941 }
10bbdaf5
PS		 7942 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7943 bnapi = bp->bnapi[cp_nr_rings];
7944 netif_napi_add(bp->dev, &bnapi->napi,
7945 bnxt_poll_nitroa0, 64);
10bbdaf5 7946 }
c0c050c5
MC		 7947 } else {
7948 bnapi = bp->bnapi[0];
7949 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
c0c050c5
MC		 7950 }
7951}
7952
7953static void bnxt_disable_napi(struct bnxt *bp)
7954{
7955 int i;
7956
7957 if (!bp->bnapi)
7958 return;
7959
0bc0b97f
AG		 7960 for (i = 0; i < bp->cp_nr_rings; i++) {
7961 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
7962
7963 if (bp->bnapi[i]->rx_ring)
7964 cancel_work_sync(&cpr->dim.work);
7965
c0c050c5 7966 napi_disable(&bp->bnapi[i]->napi);
0bc0b97f 7967 }
c0c050c5
MC		 7968}
7969
7970static void bnxt_enable_napi(struct bnxt *bp)
7971{
7972 int i;
7973
7974 for (i = 0; i < bp->cp_nr_rings; i++) {
6a8788f2 7975 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
fa7e2812 7976 bp->bnapi[i]->in_reset = false;
6a8788f2
AG		 7977
7978 if (bp->bnapi[i]->rx_ring) {
7979 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
c002bd52 7980 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6a8788f2 7981 }
c0c050c5
MC		 7982 napi_enable(&bp->bnapi[i]->napi);
7983 }
7984}
7985
7df4ae9f 7986void bnxt_tx_disable(struct bnxt *bp)
c0c050c5
MC		 7987{
7988 int i;
c0c050c5 7989 struct bnxt_tx_ring_info *txr;
c0c050c5 7990
b6ab4b01 7991 if (bp->tx_ring) {
c0c050c5 7992 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 7993 txr = &bp->tx_ring[i];
c0c050c5 7994 txr->dev_state = BNXT_DEV_STATE_CLOSING;
c0c050c5
MC		 7995 }
7996 }
7997 /* Stop all TX queues */
7998 netif_tx_disable(bp->dev);
7999 netif_carrier_off(bp->dev);
8000}
8001
7df4ae9f 8002void bnxt_tx_enable(struct bnxt *bp)
c0c050c5
MC		 8003{
8004 int i;
c0c050c5 8005 struct bnxt_tx_ring_info *txr;
c0c050c5
MC		 8006
8007 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 8008 txr = &bp->tx_ring[i];
c0c050c5
MC		 8009 txr->dev_state = 0;
8010 }
8011 netif_tx_wake_all_queues(bp->dev);
8012 if (bp->link_info.link_up)
8013 netif_carrier_on(bp->dev);
8014}
8015
8016static void bnxt_report_link(struct bnxt *bp)
8017{
8018 if (bp->link_info.link_up) {
8019 const char *duplex;
8020 const char *flow_ctrl;
38a21b34
DK		 8021 u32 speed;
8022 u16 fec;
c0c050c5
MC		 8023
8024 netif_carrier_on(bp->dev);
8025 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8026 duplex = "full";
8027 else
8028 duplex = "half";
8029 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8030 flow_ctrl = "ON - receive & transmit";
8031 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8032 flow_ctrl = "ON - transmit";
8033 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8034 flow_ctrl = "ON - receive";
8035 else
8036 flow_ctrl = "none";
8037 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
38a21b34 8038 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
c0c050c5 8039 speed, duplex, flow_ctrl);
170ce013
MC		 8040 if (bp->flags & BNXT_FLAG_EEE_CAP)
8041 netdev_info(bp->dev, "EEE is %s\n",
8042 bp->eee.eee_active ? "active" :
8043 "not active");
e70c752f
MC		 8044 fec = bp->link_info.fec_cfg;
8045 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8046 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8047 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8048 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8049 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
c0c050c5
MC		 8050 } else {
8051 netif_carrier_off(bp->dev);
8052 netdev_err(bp->dev, "NIC Link is Down\n");
8053 }
8054}
8055
170ce013
MC		 8056static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8057{
8058 int rc = 0;
8059 struct hwrm_port_phy_qcaps_input req = {0};
8060 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
93ed8117 8061 struct bnxt_link_info *link_info = &bp->link_info;
170ce013
MC		 8062
8063 if (bp->hwrm_spec_code < 0x10201)
8064 return 0;
8065
8066 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8067
8068 mutex_lock(&bp->hwrm_cmd_lock);
8069 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8070 if (rc)
8071 goto hwrm_phy_qcaps_exit;
8072
acb20054 8073 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
170ce013
MC		 8074 struct ethtool_eee *eee = &bp->eee;
8075 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8076
8077 bp->flags |= BNXT_FLAG_EEE_CAP;
8078 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8079 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8080 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8081 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8082 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8083 }
55fd0cf3
MC		 8084 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8085 if (bp->test_info)
8086 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8087 }
520ad89a
MC		 8088 if (resp->supported_speeds_auto_mode)
8089 link_info->support_auto_speeds =
8090 le16_to_cpu(resp->supported_speeds_auto_mode);
170ce013 8091
d5430d31
MC		 8092 bp->port_count = resp->port_cnt;
8093
170ce013
MC		 8094hwrm_phy_qcaps_exit:
8095 mutex_unlock(&bp->hwrm_cmd_lock);
8096 return rc;
8097}
8098
c0c050c5
MC		 8099static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8100{
8101 int rc = 0;
8102 struct bnxt_link_info *link_info = &bp->link_info;
8103 struct hwrm_port_phy_qcfg_input req = {0};
8104 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8105 u8 link_up = link_info->link_up;
286ef9d6 8106 u16 diff;
c0c050c5
MC		 8107
8108 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8109
8110 mutex_lock(&bp->hwrm_cmd_lock);
8111 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8112 if (rc) {
8113 mutex_unlock(&bp->hwrm_cmd_lock);
8114 return rc;
8115 }
8116
8117 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8118 link_info->phy_link_status = resp->link;
acb20054
MC		 8119 link_info->duplex = resp->duplex_cfg;
8120 if (bp->hwrm_spec_code >= 0x10800)
8121 link_info->duplex = resp->duplex_state;
c0c050c5
MC		 8122 link_info->pause = resp->pause;
8123 link_info->auto_mode = resp->auto_mode;
8124 link_info->auto_pause_setting = resp->auto_pause;
3277360e 8125 link_info->lp_pause = resp->link_partner_adv_pause;
c0c050c5 8126 link_info->force_pause_setting = resp->force_pause;
acb20054 8127 link_info->duplex_setting = resp->duplex_cfg;
c0c050c5
MC		 8128 if (link_info->phy_link_status == BNXT_LINK_LINK)
8129 link_info->link_speed = le16_to_cpu(resp->link_speed);
8130 else
8131 link_info->link_speed = 0;
8132 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
c0c050c5
MC		 8133 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8134 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
3277360e
MC		 8135 link_info->lp_auto_link_speeds =
8136 le16_to_cpu(resp->link_partner_adv_speeds);
c0c050c5
MC		 8137 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8138 link_info->phy_ver[0] = resp->phy_maj;
8139 link_info->phy_ver[1] = resp->phy_min;
8140 link_info->phy_ver[2] = resp->phy_bld;
8141 link_info->media_type = resp->media_type;
03efbec0 8142 link_info->phy_type = resp->phy_type;
11f15ed3 8143 link_info->transceiver = resp->xcvr_pkg_type;
170ce013
MC		 8144 link_info->phy_addr = resp->eee_config_phy_addr &
8145 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
42ee18fe 8146 link_info->module_status = resp->module_status;
170ce013
MC		 8147
8148 if (bp->flags & BNXT_FLAG_EEE_CAP) {
8149 struct ethtool_eee *eee = &bp->eee;
8150 u16 fw_speeds;
8151
8152 eee->eee_active = 0;
8153 if (resp->eee_config_phy_addr &
8154 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8155 eee->eee_active = 1;
8156 fw_speeds = le16_to_cpu(
8157 resp->link_partner_adv_eee_link_speed_mask);
8158 eee->lp_advertised =
8159 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8160 }
8161
8162 /* Pull initial EEE config */
8163 if (!chng_link_state) {
8164 if (resp->eee_config_phy_addr &
8165 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8166 eee->eee_enabled = 1;
c0c050c5 8167
170ce013
MC		 8168 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8169 eee->advertised =
8170 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8171
8172 if (resp->eee_config_phy_addr &
8173 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8174 __le32 tmr;
8175
8176 eee->tx_lpi_enabled = 1;
8177 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8178 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8179 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8180 }
8181 }
8182 }
e70c752f
MC		 8183
8184 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8185 if (bp->hwrm_spec_code >= 0x10504)
8186 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8187
c0c050c5
MC		 8188 /* TODO: need to add more logic to report VF link */
8189 if (chng_link_state) {
8190 if (link_info->phy_link_status == BNXT_LINK_LINK)
8191 link_info->link_up = 1;
8192 else
8193 link_info->link_up = 0;
8194 if (link_up != link_info->link_up)
8195 bnxt_report_link(bp);
8196 } else {
8197 /* alwasy link down if not require to update link state */
8198 link_info->link_up = 0;
8199 }
8200 mutex_unlock(&bp->hwrm_cmd_lock);
286ef9d6 8201
dac04907
MC		 8202 if (!BNXT_SINGLE_PF(bp))
8203 return 0;
8204
286ef9d6
MC		 8205 diff = link_info->support_auto_speeds ^ link_info->advertising;
8206 if ((link_info->support_auto_speeds | diff) !=
8207 link_info->support_auto_speeds) {
8208 /* An advertised speed is no longer supported, so we need to
0eaa24b9
MC		 8209 * update the advertisement settings. Caller holds RTNL
8210 * so we can modify link settings.
286ef9d6 8211 */
286ef9d6 8212 link_info->advertising = link_info->support_auto_speeds;
0eaa24b9 8213 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
286ef9d6 8214 bnxt_hwrm_set_link_setting(bp, true, false);
286ef9d6 8215 }
c0c050c5
MC		 8216 return 0;
8217}
8218
10289bec
MC		 8219static void bnxt_get_port_module_status(struct bnxt *bp)
8220{
8221 struct bnxt_link_info *link_info = &bp->link_info;
8222 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8223 u8 module_status;
8224
8225 if (bnxt_update_link(bp, true))
8226 return;
8227
8228 module_status = link_info->module_status;
8229 switch (module_status) {
8230 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8231 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8232 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8233 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8234 bp->pf.port_id);
8235 if (bp->hwrm_spec_code >= 0x10201) {
8236 netdev_warn(bp->dev, "Module part number %s\n",
8237 resp->phy_vendor_partnumber);
8238 }
8239 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8240 netdev_warn(bp->dev, "TX is disabled\n");
8241 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8242 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8243 }
8244}
8245
c0c050c5
MC		 8246static void
8247bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8248{
8249 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
c9ee9516
MC		 8250 if (bp->hwrm_spec_code >= 0x10201)
8251 req->auto_pause =
8252 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
c0c050c5
MC		 8253 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8254 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8255 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
49b5c7a1 8256 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
c0c050c5
MC		 8257 req->enables |=
8258 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8259 } else {
8260 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8261 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8262 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8263 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
8264 req->enables |=
8265 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
c9ee9516
MC		 8266 if (bp->hwrm_spec_code >= 0x10201) {
8267 req->auto_pause = req->force_pause;
8268 req->enables |= cpu_to_le32(
8269 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8270 }
c0c050c5
MC		 8271 }
8272}
8273
8274static void bnxt_hwrm_set_link_common(struct bnxt *bp,
8275 struct hwrm_port_phy_cfg_input *req)
8276{
8277 u8 autoneg = bp->link_info.autoneg;
8278 u16 fw_link_speed = bp->link_info.req_link_speed;
68515a18 8279 u16 advertising = bp->link_info.advertising;
c0c050c5
MC		 8280
8281 if (autoneg & BNXT_AUTONEG_SPEED) {
8282 req->auto_mode |=
11f15ed3 8283 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
c0c050c5
MC		 8284
8285 req->enables |= cpu_to_le32(
8286 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
8287 req->auto_link_speed_mask = cpu_to_le16(advertising);
8288
8289 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
8290 req->flags |=
8291 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
8292 } else {
8293 req->force_link_speed = cpu_to_le16(fw_link_speed);
8294 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
8295 }
8296
c0c050c5
MC		 8297 /* tell chimp that the setting takes effect immediately */
8298 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8299}
8300
8301int bnxt_hwrm_set_pause(struct bnxt *bp)
8302{
8303 struct hwrm_port_phy_cfg_input req = {0};
8304 int rc;
8305
8306 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8307 bnxt_hwrm_set_pause_common(bp, &req);
8308
8309 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8310 bp->link_info.force_link_chng)
8311 bnxt_hwrm_set_link_common(bp, &req);
8312
8313 mutex_lock(&bp->hwrm_cmd_lock);
8314 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8315 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8316 /* since changing of pause setting doesn't trigger any link
8317 * change event, the driver needs to update the current pause
8318 * result upon successfully return of the phy_cfg command
8319 */
8320 bp->link_info.pause =
8321 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8322 bp->link_info.auto_pause_setting = 0;
8323 if (!bp->link_info.force_link_chng)
8324 bnxt_report_link(bp);
8325 }
8326 bp->link_info.force_link_chng = false;
8327 mutex_unlock(&bp->hwrm_cmd_lock);
8328 return rc;
8329}
8330
939f7f0c
MC		 8331static void bnxt_hwrm_set_eee(struct bnxt *bp,
8332 struct hwrm_port_phy_cfg_input *req)
8333{
8334 struct ethtool_eee *eee = &bp->eee;
8335
8336 if (eee->eee_enabled) {
8337 u16 eee_speeds;
8338 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8339
8340 if (eee->tx_lpi_enabled)
8341 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8342 else
8343 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8344
8345 req->flags |= cpu_to_le32(flags);
8346 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8347 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8348 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8349 } else {
8350 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8351 }
8352}
8353
8354int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
c0c050c5
MC		 8355{
8356 struct hwrm_port_phy_cfg_input req = {0};
8357
8358 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8359 if (set_pause)
8360 bnxt_hwrm_set_pause_common(bp, &req);
8361
8362 bnxt_hwrm_set_link_common(bp, &req);
939f7f0c
MC		 8363
8364 if (set_eee)
8365 bnxt_hwrm_set_eee(bp, &req);
c0c050c5
MC		 8366 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8367}
8368
33f7d55f
MC		 8369static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8370{
8371 struct hwrm_port_phy_cfg_input req = {0};
8372
567b2abe 8373 if (!BNXT_SINGLE_PF(bp))
33f7d55f
MC		 8374 return 0;
8375
8376 if (pci_num_vf(bp->pdev))
8377 return 0;
8378
8379 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
16d663a6 8380 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
33f7d55f
MC		 8381 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8382}
8383
25e1acd6
MC		 8384static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8385{
8386 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8387 struct hwrm_func_drv_if_change_input req = {0};
8388 bool resc_reinit = false;
8389 int rc;
8390
8391 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8392 return 0;
8393
8394 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8395 if (up)
8396 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8397 mutex_lock(&bp->hwrm_cmd_lock);
8398 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8399 if (!rc && (resp->flags &
8400 cpu_to_le32(FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)))
8401 resc_reinit = true;
8402 mutex_unlock(&bp->hwrm_cmd_lock);
8403
8404 if (up && resc_reinit && BNXT_NEW_RM(bp)) {
8405 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8406
8407 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8408 hw_resc->resv_cp_rings = 0;
780baad4 8409 hw_resc->resv_stat_ctxs = 0;
75720e63 8410 hw_resc->resv_irqs = 0;
25e1acd6
MC		 8411 hw_resc->resv_tx_rings = 0;
8412 hw_resc->resv_rx_rings = 0;
8413 hw_resc->resv_hw_ring_grps = 0;
8414 hw_resc->resv_vnics = 0;
6b95c3e9
MC		 8415 bp->tx_nr_rings = 0;
8416 bp->rx_nr_rings = 0;
25e1acd6
MC		 8417 }
8418 return rc;
8419}
8420
5ad2cbee
MC		 8421static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8422{
8423 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8424 struct hwrm_port_led_qcaps_input req = {0};
8425 struct bnxt_pf_info *pf = &bp->pf;
8426 int rc;
8427
8428 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8429 return 0;
8430
8431 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8432 req.port_id = cpu_to_le16(pf->port_id);
8433 mutex_lock(&bp->hwrm_cmd_lock);
8434 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8435 if (rc) {
8436 mutex_unlock(&bp->hwrm_cmd_lock);
8437 return rc;
8438 }
8439 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8440 int i;
8441
8442 bp->num_leds = resp->num_leds;
8443 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8444 bp->num_leds);
8445 for (i = 0; i < bp->num_leds; i++) {
8446 struct bnxt_led_info *led = &bp->leds[i];
8447 __le16 caps = led->led_state_caps;
8448
8449 if (!led->led_group_id ||
8450 !BNXT_LED_ALT_BLINK_CAP(caps)) {
8451 bp->num_leds = 0;
8452 break;
8453 }
8454 }
8455 }
8456 mutex_unlock(&bp->hwrm_cmd_lock);
8457 return 0;
8458}
8459
5282db6c
MC		 8460int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8461{
8462 struct hwrm_wol_filter_alloc_input req = {0};
8463 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8464 int rc;
8465
8466 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8467 req.port_id = cpu_to_le16(bp->pf.port_id);
8468 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8469 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8470 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8471 mutex_lock(&bp->hwrm_cmd_lock);
8472 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8473 if (!rc)
8474 bp->wol_filter_id = resp->wol_filter_id;
8475 mutex_unlock(&bp->hwrm_cmd_lock);
8476 return rc;
8477}
8478
8479int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8480{
8481 struct hwrm_wol_filter_free_input req = {0};
8482 int rc;
8483
8484 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8485 req.port_id = cpu_to_le16(bp->pf.port_id);
8486 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8487 req.wol_filter_id = bp->wol_filter_id;
8488 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8489 return rc;
8490}
8491
c1ef146a
MC		 8492static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8493{
8494 struct hwrm_wol_filter_qcfg_input req = {0};
8495 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8496 u16 next_handle = 0;
8497 int rc;
8498
8499 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8500 req.port_id = cpu_to_le16(bp->pf.port_id);
8501 req.handle = cpu_to_le16(handle);
8502 mutex_lock(&bp->hwrm_cmd_lock);
8503 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8504 if (!rc) {
8505 next_handle = le16_to_cpu(resp->next_handle);
8506 if (next_handle != 0) {
8507 if (resp->wol_type ==
8508 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8509 bp->wol = 1;
8510 bp->wol_filter_id = resp->wol_filter_id;
8511 }
8512 }
8513 }
8514 mutex_unlock(&bp->hwrm_cmd_lock);
8515 return next_handle;
8516}
8517
8518static void bnxt_get_wol_settings(struct bnxt *bp)
8519{
8520 u16 handle = 0;
8521
8522 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8523 return;
8524
8525 do {
8526 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8527 } while (handle && handle != 0xffff);
8528}
8529
cde49a42
VV		 8530#ifdef CONFIG_BNXT_HWMON
8531static ssize_t bnxt_show_temp(struct device *dev,
8532 struct device_attribute *devattr, char *buf)
8533{
8534 struct hwrm_temp_monitor_query_input req = {0};
8535 struct hwrm_temp_monitor_query_output *resp;
8536 struct bnxt *bp = dev_get_drvdata(dev);
8537 u32 temp = 0;
8538
8539 resp = bp->hwrm_cmd_resp_addr;
8540 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8541 mutex_lock(&bp->hwrm_cmd_lock);
8542 if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8543 temp = resp->temp * 1000; /* display millidegree */
8544 mutex_unlock(&bp->hwrm_cmd_lock);
8545
8546 return sprintf(buf, "%u\n", temp);
8547}
8548static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8549
8550static struct attribute *bnxt_attrs[] = {
8551 &sensor_dev_attr_temp1_input.dev_attr.attr,
8552 NULL
8553};
8554ATTRIBUTE_GROUPS(bnxt);
8555
8556static void bnxt_hwmon_close(struct bnxt *bp)
8557{
8558 if (bp->hwmon_dev) {
8559 hwmon_device_unregister(bp->hwmon_dev);
8560 bp->hwmon_dev = NULL;
8561 }
8562}
8563
8564static void bnxt_hwmon_open(struct bnxt *bp)
8565{
8566 struct pci_dev *pdev = bp->pdev;
8567
8568 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8569 DRV_MODULE_NAME, bp,
8570 bnxt_groups);
8571 if (IS_ERR(bp->hwmon_dev)) {
8572 bp->hwmon_dev = NULL;
8573 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8574 }
8575}
8576#else
8577static void bnxt_hwmon_close(struct bnxt *bp)
8578{
8579}
8580
8581static void bnxt_hwmon_open(struct bnxt *bp)
8582{
8583}
8584#endif
8585
939f7f0c
MC		 8586static bool bnxt_eee_config_ok(struct bnxt *bp)
8587{
8588 struct ethtool_eee *eee = &bp->eee;
8589 struct bnxt_link_info *link_info = &bp->link_info;
8590
8591 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8592 return true;
8593
8594 if (eee->eee_enabled) {
8595 u32 advertising =
8596 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8597
8598 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8599 eee->eee_enabled = 0;
8600 return false;
8601 }
8602 if (eee->advertised & ~advertising) {
8603 eee->advertised = advertising & eee->supported;
8604 return false;
8605 }
8606 }
8607 return true;
8608}
8609
c0c050c5
MC		 8610static int bnxt_update_phy_setting(struct bnxt *bp)
8611{
8612 int rc;
8613 bool update_link = false;
8614 bool update_pause = false;
939f7f0c 8615 bool update_eee = false;
c0c050c5
MC		 8616 struct bnxt_link_info *link_info = &bp->link_info;
8617
8618 rc = bnxt_update_link(bp, true);
8619 if (rc) {
8620 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
8621 rc);
8622 return rc;
8623 }
33dac24a
MC		 8624 if (!BNXT_SINGLE_PF(bp))
8625 return 0;
8626
c0c050c5 8627 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
c9ee9516
MC		 8628 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
8629 link_info->req_flow_ctrl)
c0c050c5
MC		 8630 update_pause = true;
8631 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8632 link_info->force_pause_setting != link_info->req_flow_ctrl)
8633 update_pause = true;
c0c050c5
MC		 8634 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8635 if (BNXT_AUTO_MODE(link_info->auto_mode))
8636 update_link = true;
8637 if (link_info->req_link_speed != link_info->force_link_speed)
8638 update_link = true;
de73018f
MC		 8639 if (link_info->req_duplex != link_info->duplex_setting)
8640 update_link = true;
c0c050c5
MC		 8641 } else {
8642 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
8643 update_link = true;
8644 if (link_info->advertising != link_info->auto_link_speeds)
8645 update_link = true;
c0c050c5
MC		 8646 }
8647
16d663a6
MC		 8648 /* The last close may have shutdown the link, so need to call
8649 * PHY_CFG to bring it back up.
8650 */
8651 if (!netif_carrier_ok(bp->dev))
8652 update_link = true;
8653
939f7f0c
MC		 8654 if (!bnxt_eee_config_ok(bp))
8655 update_eee = true;
8656
c0c050c5 8657 if (update_link)
939f7f0c 8658 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
c0c050c5
MC		 8659 else if (update_pause)
8660 rc = bnxt_hwrm_set_pause(bp);
8661 if (rc) {
8662 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
8663 rc);
8664 return rc;
8665 }
8666
8667 return rc;
8668}
8669
11809490
JH		 8670/* Common routine to pre-map certain register block to different GRC window.
8671 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
8672 * in PF and 3 windows in VF that can be customized to map in different
8673 * register blocks.
8674 */
8675static void bnxt_preset_reg_win(struct bnxt *bp)
8676{
8677 if (BNXT_PF(bp)) {
8678 /* CAG registers map to GRC window #4 */
8679 writel(BNXT_CAG_REG_BASE,
8680 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
8681 }
8682}
8683
47558acd
MC		 8684static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
8685
c0c050c5
MC		 8686static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8687{
8688 int rc = 0;
8689
11809490 8690 bnxt_preset_reg_win(bp);
c0c050c5
MC		 8691 netif_carrier_off(bp->dev);
8692 if (irq_re_init) {
47558acd
MC		 8693 /* Reserve rings now if none were reserved at driver probe. */
8694 rc = bnxt_init_dflt_ring_mode(bp);
8695 if (rc) {
8696 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
8697 return rc;
8698 }
c0c050c5 8699 }
1b3f0b75 8700 rc = bnxt_reserve_rings(bp, irq_re_init);
41e8d798
MC		 8701 if (rc)
8702 return rc;
c0c050c5
MC		 8703 if ((bp->flags & BNXT_FLAG_RFS) &&
8704 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
8705 /* disable RFS if falling back to INTA */
8706 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
8707 bp->flags &= ~BNXT_FLAG_RFS;
8708 }
8709
8710 rc = bnxt_alloc_mem(bp, irq_re_init);
8711 if (rc) {
8712 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8713 goto open_err_free_mem;
8714 }
8715
8716 if (irq_re_init) {
8717 bnxt_init_napi(bp);
8718 rc = bnxt_request_irq(bp);
8719 if (rc) {
8720 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
c58387ab 8721 goto open_err_irq;
c0c050c5
MC		 8722 }
8723 }
8724
8725 bnxt_enable_napi(bp);
cabfb09d 8726 bnxt_debug_dev_init(bp);
c0c050c5
MC		 8727
8728 rc = bnxt_init_nic(bp, irq_re_init);
8729 if (rc) {
8730 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8731 goto open_err;
8732 }
8733
8734 if (link_re_init) {
e2dc9b6e 8735 mutex_lock(&bp->link_lock);
c0c050c5 8736 rc = bnxt_update_phy_setting(bp);
e2dc9b6e 8737 mutex_unlock(&bp->link_lock);
a1ef4a79 8738 if (rc) {
ba41d46f 8739 netdev_warn(bp->dev, "failed to update phy settings\n");
a1ef4a79
MC		 8740 if (BNXT_SINGLE_PF(bp)) {
8741 bp->link_info.phy_retry = true;
8742 bp->link_info.phy_retry_expires =
8743 jiffies + 5 * HZ;
8744 }
8745 }
c0c050c5
MC		 8746 }
8747
7cdd5fc3 8748 if (irq_re_init)
ad51b8e9 8749 udp_tunnel_get_rx_info(bp->dev);
c0c050c5 8750
caefe526 8751 set_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 8752 bnxt_enable_int(bp);
8753 /* Enable TX queues */
8754 bnxt_tx_enable(bp);
8755 mod_timer(&bp->timer, jiffies + bp->current_interval);
10289bec
MC		 8756 /* Poll link status and check for SFP+ module status */
8757 bnxt_get_port_module_status(bp);
c0c050c5 8758
ee5c7fb3
SP		 8759 /* VF-reps may need to be re-opened after the PF is re-opened */
8760 if (BNXT_PF(bp))
8761 bnxt_vf_reps_open(bp);
c0c050c5
MC		 8762 return 0;
8763
8764open_err:
cabfb09d 8765 bnxt_debug_dev_exit(bp);
c0c050c5 8766 bnxt_disable_napi(bp);
c58387ab
VG		 8767
8768open_err_irq:
c0c050c5
MC		 8769 bnxt_del_napi(bp);
8770
8771open_err_free_mem:
8772 bnxt_free_skbs(bp);
8773 bnxt_free_irq(bp);
8774 bnxt_free_mem(bp, true);
8775 return rc;
8776}
8777
8778/* rtnl_lock held */
8779int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8780{
8781 int rc = 0;
8782
8783 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
8784 if (rc) {
8785 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
8786 dev_close(bp->dev);
8787 }
8788 return rc;
8789}
8790
f7dc1ea6
MC		 8791/* rtnl_lock held, open the NIC half way by allocating all resources, but
8792 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
8793 * self tests.
8794 */
8795int bnxt_half_open_nic(struct bnxt *bp)
8796{
8797 int rc = 0;
8798
8799 rc = bnxt_alloc_mem(bp, false);
8800 if (rc) {
8801 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8802 goto half_open_err;
8803 }
8804 rc = bnxt_init_nic(bp, false);
8805 if (rc) {
8806 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8807 goto half_open_err;
8808 }
8809 return 0;
8810
8811half_open_err:
8812 bnxt_free_skbs(bp);
8813 bnxt_free_mem(bp, false);
8814 dev_close(bp->dev);
8815 return rc;
8816}
8817
8818/* rtnl_lock held, this call can only be made after a previous successful
8819 * call to bnxt_half_open_nic().
8820 */
8821void bnxt_half_close_nic(struct bnxt *bp)
8822{
8823 bnxt_hwrm_resource_free(bp, false, false);
8824 bnxt_free_skbs(bp);
8825 bnxt_free_mem(bp, false);
8826}
8827
c0c050c5
MC		 8828static int bnxt_open(struct net_device *dev)
8829{
8830 struct bnxt *bp = netdev_priv(dev);
25e1acd6 8831 int rc;
c0c050c5 8832
25e1acd6
MC		 8833 bnxt_hwrm_if_change(bp, true);
8834 rc = __bnxt_open_nic(bp, true, true);
8835 if (rc)
8836 bnxt_hwrm_if_change(bp, false);
cde49a42
VV		 8837
8838 bnxt_hwmon_open(bp);
8839
25e1acd6 8840 return rc;
c0c050c5
MC		 8841}
8842
f9b76ebd
MC		 8843static bool bnxt_drv_busy(struct bnxt *bp)
8844{
8845 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
8846 test_bit(BNXT_STATE_READ_STATS, &bp->state));
8847}
8848
b8875ca3
MC		 8849static void bnxt_get_ring_stats(struct bnxt *bp,
8850 struct rtnl_link_stats64 *stats);
8851
86e953db
MC		 8852static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
8853 bool link_re_init)
c0c050c5 8854{
ee5c7fb3
SP		 8855 /* Close the VF-reps before closing PF */
8856 if (BNXT_PF(bp))
8857 bnxt_vf_reps_close(bp);
86e953db 8858
c0c050c5
MC		 8859 /* Change device state to avoid TX queue wake up's */
8860 bnxt_tx_disable(bp);
8861
caefe526 8862 clear_bit(BNXT_STATE_OPEN, &bp->state);
4cebdcec 8863 smp_mb__after_atomic();
f9b76ebd 8864 while (bnxt_drv_busy(bp))
4cebdcec 8865 msleep(20);
c0c050c5 8866
9d8bc097 8867 /* Flush rings and and disable interrupts */
c0c050c5
MC		 8868 bnxt_shutdown_nic(bp, irq_re_init);
8869
8870 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
8871
cabfb09d 8872 bnxt_debug_dev_exit(bp);
c0c050c5 8873 bnxt_disable_napi(bp);
c0c050c5
MC		 8874 del_timer_sync(&bp->timer);
8875 bnxt_free_skbs(bp);
8876
b8875ca3
MC		 8877 /* Save ring stats before shutdown */
8878 if (bp->bnapi)
8879 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
c0c050c5
MC		 8880 if (irq_re_init) {
8881 bnxt_free_irq(bp);
8882 bnxt_del_napi(bp);
8883 }
8884 bnxt_free_mem(bp, irq_re_init);
86e953db
MC		 8885}
8886
8887int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8888{
8889 int rc = 0;
8890
8891#ifdef CONFIG_BNXT_SRIOV
8892 if (bp->sriov_cfg) {
8893 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
8894 !bp->sriov_cfg,
8895 BNXT_SRIOV_CFG_WAIT_TMO);
8896 if (rc)
8897 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
8898 }
8899#endif
8900 __bnxt_close_nic(bp, irq_re_init, link_re_init);
c0c050c5
MC		 8901 return rc;
8902}
8903
8904static int bnxt_close(struct net_device *dev)
8905{
8906 struct bnxt *bp = netdev_priv(dev);
8907
cde49a42 8908 bnxt_hwmon_close(bp);
c0c050c5 8909 bnxt_close_nic(bp, true, true);
33f7d55f 8910 bnxt_hwrm_shutdown_link(bp);
25e1acd6 8911 bnxt_hwrm_if_change(bp, false);
c0c050c5
MC		 8912 return 0;
8913}
8914
0ca12be9
VV		 8915static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
8916 u16 *val)
8917{
8918 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
8919 struct hwrm_port_phy_mdio_read_input req = {0};
8920 int rc;
8921
8922 if (bp->hwrm_spec_code < 0x10a00)
8923 return -EOPNOTSUPP;
8924
8925 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
8926 req.port_id = cpu_to_le16(bp->pf.port_id);
8927 req.phy_addr = phy_addr;
8928 req.reg_addr = cpu_to_le16(reg & 0x1f);
2730214d 8929 if (mdio_phy_id_is_c45(phy_addr)) {
0ca12be9
VV		 8930 req.cl45_mdio = 1;
8931 req.phy_addr = mdio_phy_id_prtad(phy_addr);
8932 req.dev_addr = mdio_phy_id_devad(phy_addr);
8933 req.reg_addr = cpu_to_le16(reg);
8934 }
8935
8936 mutex_lock(&bp->hwrm_cmd_lock);
8937 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8938 if (!rc)
8939 *val = le16_to_cpu(resp->reg_data);
8940 mutex_unlock(&bp->hwrm_cmd_lock);
8941 return rc;
8942}
8943
8944static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
8945 u16 val)
8946{
8947 struct hwrm_port_phy_mdio_write_input req = {0};
8948
8949 if (bp->hwrm_spec_code < 0x10a00)
8950 return -EOPNOTSUPP;
8951
8952 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
8953 req.port_id = cpu_to_le16(bp->pf.port_id);
8954 req.phy_addr = phy_addr;
8955 req.reg_addr = cpu_to_le16(reg & 0x1f);
2730214d 8956 if (mdio_phy_id_is_c45(phy_addr)) {
0ca12be9
VV		 8957 req.cl45_mdio = 1;
8958 req.phy_addr = mdio_phy_id_prtad(phy_addr);
8959 req.dev_addr = mdio_phy_id_devad(phy_addr);
8960 req.reg_addr = cpu_to_le16(reg);
8961 }
8962 req.reg_data = cpu_to_le16(val);
8963
8964 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8965}
8966
c0c050c5
MC		 8967/* rtnl_lock held */
8968static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
8969{
0ca12be9
VV		 8970 struct mii_ioctl_data *mdio = if_mii(ifr);
8971 struct bnxt *bp = netdev_priv(dev);
8972 int rc;
8973
c0c050c5
MC		 8974 switch (cmd) {
8975 case SIOCGMIIPHY:
0ca12be9
VV		 8976 mdio->phy_id = bp->link_info.phy_addr;
8977
c0c050c5
MC		 8978 /* fallthru */
8979 case SIOCGMIIREG: {
0ca12be9
VV		 8980 u16 mii_regval = 0;
8981
c0c050c5
MC		 8982 if (!netif_running(dev))
8983 return -EAGAIN;
8984
0ca12be9
VV		 8985 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
8986 &mii_regval);
8987 mdio->val_out = mii_regval;
8988 return rc;
c0c050c5
MC		 8989 }
8990
8991 case SIOCSMIIREG:
8992 if (!netif_running(dev))
8993 return -EAGAIN;
8994
0ca12be9
VV		 8995 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
8996 mdio->val_in);
c0c050c5
MC		 8997
8998 default:
8999 /* do nothing */
9000 break;
9001 }
9002 return -EOPNOTSUPP;
9003}
9004
b8875ca3
MC		 9005static void bnxt_get_ring_stats(struct bnxt *bp,
9006 struct rtnl_link_stats64 *stats)
c0c050c5 9007{
b8875ca3 9008 int i;
c0c050c5 9009
c0c050c5 9010
c0c050c5
MC		 9011 for (i = 0; i < bp->cp_nr_rings; i++) {
9012 struct bnxt_napi *bnapi = bp->bnapi[i];
9013 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9014 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
9015
9016 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
9017 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
9018 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
9019
9020 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
9021 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
9022 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
9023
9024 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
9025 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
9026 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
9027
9028 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
9029 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
9030 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
9031
9032 stats->rx_missed_errors +=
9033 le64_to_cpu(hw_stats->rx_discard_pkts);
9034
9035 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
9036
c0c050c5
MC		 9037 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
9038 }
b8875ca3
MC		 9039}
9040
9041static void bnxt_add_prev_stats(struct bnxt *bp,
9042 struct rtnl_link_stats64 *stats)
9043{
9044 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9045
9046 stats->rx_packets += prev_stats->rx_packets;
9047 stats->tx_packets += prev_stats->tx_packets;
9048 stats->rx_bytes += prev_stats->rx_bytes;
9049 stats->tx_bytes += prev_stats->tx_bytes;
9050 stats->rx_missed_errors += prev_stats->rx_missed_errors;
9051 stats->multicast += prev_stats->multicast;
9052 stats->tx_dropped += prev_stats->tx_dropped;
9053}
9054
9055static void
9056bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9057{
9058 struct bnxt *bp = netdev_priv(dev);
9059
9060 set_bit(BNXT_STATE_READ_STATS, &bp->state);
9061 /* Make sure bnxt_close_nic() sees that we are reading stats before
9062 * we check the BNXT_STATE_OPEN flag.
9063 */
9064 smp_mb__after_atomic();
9065 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9066 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9067 *stats = bp->net_stats_prev;
9068 return;
9069 }
9070
9071 bnxt_get_ring_stats(bp, stats);
9072 bnxt_add_prev_stats(bp, stats);
c0c050c5 9073
9947f83f
MC		 9074 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9075 struct rx_port_stats *rx = bp->hw_rx_port_stats;
9076 struct tx_port_stats *tx = bp->hw_tx_port_stats;
9077
9078 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
9079 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
9080 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
9081 le64_to_cpu(rx->rx_ovrsz_frames) +
9082 le64_to_cpu(rx->rx_runt_frames);
9083 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
9084 le64_to_cpu(rx->rx_jbr_frames);
9085 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
9086 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
9087 stats->tx_errors = le64_to_cpu(tx->tx_err);
9088 }
f9b76ebd 9089 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
c0c050c5
MC		 9090}
9091
9092static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9093{
9094 struct net_device *dev = bp->dev;
9095 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9096 struct netdev_hw_addr *ha;
9097 u8 *haddr;
9098 int mc_count = 0;
9099 bool update = false;
9100 int off = 0;
9101
9102 netdev_for_each_mc_addr(ha, dev) {
9103 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9104 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9105 vnic->mc_list_count = 0;
9106 return false;
9107 }
9108 haddr = ha->addr;
9109 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9110 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9111 update = true;
9112 }
9113 off += ETH_ALEN;
9114 mc_count++;
9115 }
9116 if (mc_count)
9117 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9118
9119 if (mc_count != vnic->mc_list_count) {
9120 vnic->mc_list_count = mc_count;
9121 update = true;
9122 }
9123 return update;
9124}
9125
9126static bool bnxt_uc_list_updated(struct bnxt *bp)
9127{
9128 struct net_device *dev = bp->dev;
9129 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9130 struct netdev_hw_addr *ha;
9131 int off = 0;
9132
9133 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9134 return true;
9135
9136 netdev_for_each_uc_addr(ha, dev) {
9137 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9138 return true;
9139
9140 off += ETH_ALEN;
9141 }
9142 return false;
9143}
9144
9145static void bnxt_set_rx_mode(struct net_device *dev)
9146{
9147 struct bnxt *bp = netdev_priv(dev);
9148 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9149 u32 mask = vnic->rx_mask;
9150 bool mc_update = false;
9151 bool uc_update;
9152
9153 if (!netif_running(dev))
9154 return;
9155
9156 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9157 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
30e33848
MC		 9158 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9159 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
c0c050c5 9160
17c71ac3 9161 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
c0c050c5
MC		 9162 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9163
9164 uc_update = bnxt_uc_list_updated(bp);
9165
30e33848
MC		 9166 if (dev->flags & IFF_BROADCAST)
9167 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5
MC		 9168 if (dev->flags & IFF_ALLMULTI) {
9169 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9170 vnic->mc_list_count = 0;
9171 } else {
9172 mc_update = bnxt_mc_list_updated(bp, &mask);
9173 }
9174
9175 if (mask != vnic->rx_mask || uc_update || mc_update) {
9176 vnic->rx_mask = mask;
9177
9178 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
c213eae8 9179 bnxt_queue_sp_work(bp);
c0c050c5
MC		 9180 }
9181}
9182
b664f008 9183static int bnxt_cfg_rx_mode(struct bnxt *bp)
c0c050c5
MC		 9184{
9185 struct net_device *dev = bp->dev;
9186 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9187 struct netdev_hw_addr *ha;
9188 int i, off = 0, rc;
9189 bool uc_update;
9190
9191 netif_addr_lock_bh(dev);
9192 uc_update = bnxt_uc_list_updated(bp);
9193 netif_addr_unlock_bh(dev);
9194
9195 if (!uc_update)
9196 goto skip_uc;
9197
9198 mutex_lock(&bp->hwrm_cmd_lock);
9199 for (i = 1; i < vnic->uc_filter_count; i++) {
9200 struct hwrm_cfa_l2_filter_free_input req = {0};
9201
9202 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
9203 -1);
9204
9205 req.l2_filter_id = vnic->fw_l2_filter_id[i];
9206
9207 rc = _hwrm_send_message(bp, &req, sizeof(req),
9208 HWRM_CMD_TIMEOUT);
9209 }
9210 mutex_unlock(&bp->hwrm_cmd_lock);
9211
9212 vnic->uc_filter_count = 1;
9213
9214 netif_addr_lock_bh(dev);
9215 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
9216 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9217 } else {
9218 netdev_for_each_uc_addr(ha, dev) {
9219 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
9220 off += ETH_ALEN;
9221 vnic->uc_filter_count++;
9222 }
9223 }
9224 netif_addr_unlock_bh(dev);
9225
9226 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
9227 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
9228 if (rc) {
9229 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
9230 rc);
9231 vnic->uc_filter_count = i;
b664f008 9232 return rc;
c0c050c5
MC		 9233 }
9234 }
9235
9236skip_uc:
9237 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
b4e30e8e
MC		 9238 if (rc && vnic->mc_list_count) {
9239 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
9240 rc);
9241 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9242 vnic->mc_list_count = 0;
9243 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9244 }
c0c050c5 9245 if (rc)
b4e30e8e 9246 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
c0c050c5 9247 rc);
b664f008
MC		 9248
9249 return rc;
c0c050c5
MC		 9250}
9251
2773dfb2
MC		 9252static bool bnxt_can_reserve_rings(struct bnxt *bp)
9253{
9254#ifdef CONFIG_BNXT_SRIOV
f1ca94de 9255 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
2773dfb2
MC		 9256 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9257
9258 /* No minimum rings were provisioned by the PF. Don't
9259 * reserve rings by default when device is down.
9260 */
9261 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
9262 return true;
9263
9264 if (!netif_running(bp->dev))
9265 return false;
9266 }
9267#endif
9268 return true;
9269}
9270
8079e8f1
MC		 9271/* If the chip and firmware supports RFS */
9272static bool bnxt_rfs_supported(struct bnxt *bp)
9273{
e969ae5b
MC		 9274 if (bp->flags & BNXT_FLAG_CHIP_P5) {
9275 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX)
9276 return true;
41e8d798 9277 return false;
e969ae5b 9278 }
8079e8f1
MC		 9279 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
9280 return true;
ae10ae74
MC		 9281 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9282 return true;
8079e8f1
MC		 9283 return false;
9284}
9285
9286/* If runtime conditions support RFS */
2bcfa6f6
MC		 9287static bool bnxt_rfs_capable(struct bnxt *bp)
9288{
9289#ifdef CONFIG_RFS_ACCEL
8079e8f1 9290 int vnics, max_vnics, max_rss_ctxs;
2bcfa6f6 9291
41e8d798 9292 if (bp->flags & BNXT_FLAG_CHIP_P5)
ac33906c 9293 return bnxt_rfs_supported(bp);
2773dfb2 9294 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
2bcfa6f6
MC		 9295 return false;
9296
9297 vnics = 1 + bp->rx_nr_rings;
8079e8f1
MC		 9298 max_vnics = bnxt_get_max_func_vnics(bp);
9299 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
ae10ae74
MC		 9300
9301 /* RSS contexts not a limiting factor */
9302 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9303 max_rss_ctxs = max_vnics;
8079e8f1 9304 if (vnics > max_vnics || vnics > max_rss_ctxs) {
6a1eef5b
MC		 9305 if (bp->rx_nr_rings > 1)
9306 netdev_warn(bp->dev,
9307 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
9308 min(max_rss_ctxs - 1, max_vnics - 1));
2bcfa6f6 9309 return false;
a2304909 9310 }
2bcfa6f6 9311
f1ca94de 9312 if (!BNXT_NEW_RM(bp))
6a1eef5b
MC		 9313 return true;
9314
9315 if (vnics == bp->hw_resc.resv_vnics)
9316 return true;
9317
780baad4 9318 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
6a1eef5b
MC		 9319 if (vnics <= bp->hw_resc.resv_vnics)
9320 return true;
9321
9322 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
780baad4 9323 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
6a1eef5b 9324 return false;
2bcfa6f6
MC		 9325#else
9326 return false;
9327#endif
9328}
9329
c0c050c5
MC		 9330static netdev_features_t bnxt_fix_features(struct net_device *dev,
9331 netdev_features_t features)
9332{
2bcfa6f6
MC		 9333 struct bnxt *bp = netdev_priv(dev);
9334
a2304909 9335 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
2bcfa6f6 9336 features &= ~NETIF_F_NTUPLE;
5a9f6b23 9337
1054aee8
MC		 9338 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9339 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9340
9341 if (!(features & NETIF_F_GRO))
9342 features &= ~NETIF_F_GRO_HW;
9343
9344 if (features & NETIF_F_GRO_HW)
9345 features &= ~NETIF_F_LRO;
9346
5a9f6b23
MC		 9347 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
9348 * turned on or off together.
9349 */
9350 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
9351 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
9352 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
9353 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9354 NETIF_F_HW_VLAN_STAG_RX);
9355 else
9356 features |= NETIF_F_HW_VLAN_CTAG_RX |
9357 NETIF_F_HW_VLAN_STAG_RX;
9358 }
cf6645f8
MC		 9359#ifdef CONFIG_BNXT_SRIOV
9360 if (BNXT_VF(bp)) {
9361 if (bp->vf.vlan) {
9362 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9363 NETIF_F_HW_VLAN_STAG_RX);
9364 }
9365 }
9366#endif
c0c050c5
MC		 9367 return features;
9368}
9369
9370static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
9371{
9372 struct bnxt *bp = netdev_priv(dev);
9373 u32 flags = bp->flags;
9374 u32 changes;
9375 int rc = 0;
9376 bool re_init = false;
9377 bool update_tpa = false;
9378
9379 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
1054aee8 9380 if (features & NETIF_F_GRO_HW)
c0c050c5 9381 flags |= BNXT_FLAG_GRO;
1054aee8 9382 else if (features & NETIF_F_LRO)
c0c050c5
MC		 9383 flags |= BNXT_FLAG_LRO;
9384
bdbd1eb5
MC		 9385 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9386 flags &= ~BNXT_FLAG_TPA;
9387
c0c050c5
MC		 9388 if (features & NETIF_F_HW_VLAN_CTAG_RX)
9389 flags |= BNXT_FLAG_STRIP_VLAN;
9390
9391 if (features & NETIF_F_NTUPLE)
9392 flags |= BNXT_FLAG_RFS;
9393
9394 changes = flags ^ bp->flags;
9395 if (changes & BNXT_FLAG_TPA) {
9396 update_tpa = true;
9397 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
f45b7b78
MC		 9398 (flags & BNXT_FLAG_TPA) == 0 ||
9399 (bp->flags & BNXT_FLAG_CHIP_P5))
c0c050c5
MC		 9400 re_init = true;
9401 }
9402
9403 if (changes & ~BNXT_FLAG_TPA)
9404 re_init = true;
9405
9406 if (flags != bp->flags) {
9407 u32 old_flags = bp->flags;
9408
2bcfa6f6 9409 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
f45b7b78 9410 bp->flags = flags;
c0c050c5
MC		 9411 if (update_tpa)
9412 bnxt_set_ring_params(bp);
9413 return rc;
9414 }
9415
9416 if (re_init) {
9417 bnxt_close_nic(bp, false, false);
f45b7b78 9418 bp->flags = flags;
c0c050c5
MC		 9419 if (update_tpa)
9420 bnxt_set_ring_params(bp);
9421
9422 return bnxt_open_nic(bp, false, false);
9423 }
9424 if (update_tpa) {
f45b7b78 9425 bp->flags = flags;
c0c050c5
MC		 9426 rc = bnxt_set_tpa(bp,
9427 (flags & BNXT_FLAG_TPA) ?
9428 true : false);
9429 if (rc)
9430 bp->flags = old_flags;
9431 }
9432 }
9433 return rc;
9434}
9435
ffd77621
MC		 9436static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9437 u32 ring_id, u32 *prod, u32 *cons)
9438{
9439 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9440 struct hwrm_dbg_ring_info_get_input req = {0};
9441 int rc;
9442
9443 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9444 req.ring_type = ring_type;
9445 req.fw_ring_id = cpu_to_le32(ring_id);
9446 mutex_lock(&bp->hwrm_cmd_lock);
9447 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9448 if (!rc) {
9449 *prod = le32_to_cpu(resp->producer_index);
9450 *cons = le32_to_cpu(resp->consumer_index);
9451 }
9452 mutex_unlock(&bp->hwrm_cmd_lock);
9453 return rc;
9454}
9455
9f554590
MC		 9456static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9457{
b6ab4b01 9458 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9f554590
MC		 9459 int i = bnapi->index;
9460
3b2b7d9d
MC		 9461 if (!txr)
9462 return;
9463
9f554590
MC		 9464 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9465 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9466 txr->tx_cons);
9467}
9468
9469static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9470{
b6ab4b01 9471 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9f554590
MC		 9472 int i = bnapi->index;
9473
3b2b7d9d
MC		 9474 if (!rxr)
9475 return;
9476
9f554590
MC		 9477 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9478 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9479 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9480 rxr->rx_sw_agg_prod);
9481}
9482
9483static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9484{
9485 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9486 int i = bnapi->index;
9487
9488 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9489 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9490}
9491
c0c050c5
MC		 9492static void bnxt_dbg_dump_states(struct bnxt *bp)
9493{
9494 int i;
9495 struct bnxt_napi *bnapi;
c0c050c5
MC		 9496
9497 for (i = 0; i < bp->cp_nr_rings; i++) {
9498 bnapi = bp->bnapi[i];
c0c050c5 9499 if (netif_msg_drv(bp)) {
9f554590
MC		 9500 bnxt_dump_tx_sw_state(bnapi);
9501 bnxt_dump_rx_sw_state(bnapi);
9502 bnxt_dump_cp_sw_state(bnapi);
c0c050c5
MC		 9503 }
9504 }
9505}
9506
6988bd92 9507static void bnxt_reset_task(struct bnxt *bp, bool silent)
c0c050c5 9508{
6988bd92
MC		 9509 if (!silent)
9510 bnxt_dbg_dump_states(bp);
028de140 9511 if (netif_running(bp->dev)) {
b386cd36
MC		 9512 int rc;
9513
9514 if (!silent)
9515 bnxt_ulp_stop(bp);
028de140 9516 bnxt_close_nic(bp, false, false);
b386cd36
MC		 9517 rc = bnxt_open_nic(bp, false, false);
9518 if (!silent && !rc)
9519 bnxt_ulp_start(bp);
028de140 9520 }
c0c050c5
MC		 9521}
9522
9523static void bnxt_tx_timeout(struct net_device *dev)
9524{
9525 struct bnxt *bp = netdev_priv(dev);
9526
9527 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
9528 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 9529 bnxt_queue_sp_work(bp);
c0c050c5
MC		 9530}
9531
e99e88a9 9532static void bnxt_timer(struct timer_list *t)
c0c050c5 9533{
e99e88a9 9534 struct bnxt *bp = from_timer(bp, t, timer);
c0c050c5
MC		 9535 struct net_device *dev = bp->dev;
9536
9537 if (!netif_running(dev))
9538 return;
9539
9540 if (atomic_read(&bp->intr_sem) != 0)
9541 goto bnxt_restart_timer;
9542
adcc331e
MC		 9543 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
9544 bp->stats_coal_ticks) {
3bdf56c4 9545 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
c213eae8 9546 bnxt_queue_sp_work(bp);
3bdf56c4 9547 }
5a84acbe
SP		 9548
9549 if (bnxt_tc_flower_enabled(bp)) {
9550 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
9551 bnxt_queue_sp_work(bp);
9552 }
a1ef4a79
MC		 9553
9554 if (bp->link_info.phy_retry) {
9555 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
9556 bp->link_info.phy_retry = 0;
9557 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
9558 } else {
9559 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
9560 bnxt_queue_sp_work(bp);
9561 }
9562 }
ffd77621
MC		 9563
9564 if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
9565 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
9566 bnxt_queue_sp_work(bp);
9567 }
c0c050c5
MC		 9568bnxt_restart_timer:
9569 mod_timer(&bp->timer, jiffies + bp->current_interval);
9570}
9571
a551ee94 9572static void bnxt_rtnl_lock_sp(struct bnxt *bp)
6988bd92 9573{
a551ee94
MC		 9574 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
9575 * set. If the device is being closed, bnxt_close() may be holding
6988bd92
MC		 9576 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
9577 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
9578 */
9579 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9580 rtnl_lock();
a551ee94
MC		 9581}
9582
9583static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
9584{
6988bd92
MC		 9585 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9586 rtnl_unlock();
9587}
9588
a551ee94
MC		 9589/* Only called from bnxt_sp_task() */
9590static void bnxt_reset(struct bnxt *bp, bool silent)
9591{
9592 bnxt_rtnl_lock_sp(bp);
9593 if (test_bit(BNXT_STATE_OPEN, &bp->state))
9594 bnxt_reset_task(bp, silent);
9595 bnxt_rtnl_unlock_sp(bp);
9596}
9597
ffd77621
MC		 9598static void bnxt_chk_missed_irq(struct bnxt *bp)
9599{
9600 int i;
9601
9602 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9603 return;
9604
9605 for (i = 0; i < bp->cp_nr_rings; i++) {
9606 struct bnxt_napi *bnapi = bp->bnapi[i];
9607 struct bnxt_cp_ring_info *cpr;
9608 u32 fw_ring_id;
9609 int j;
9610
9611 if (!bnapi)
9612 continue;
9613
9614 cpr = &bnapi->cp_ring;
9615 for (j = 0; j < 2; j++) {
9616 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
9617 u32 val[2];
9618
9619 if (!cpr2 || cpr2->has_more_work ||
9620 !bnxt_has_work(bp, cpr2))
9621 continue;
9622
9623 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
9624 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
9625 continue;
9626 }
9627 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
9628 bnxt_dbg_hwrm_ring_info_get(bp,
9629 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
9630 fw_ring_id, &val[0], &val[1]);
83eb5c5c 9631 cpr->missed_irqs++;
ffd77621
MC		 9632 }
9633 }
9634}
9635
c0c050c5
MC		 9636static void bnxt_cfg_ntp_filters(struct bnxt *);
9637
9638static void bnxt_sp_task(struct work_struct *work)
9639{
9640 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
c0c050c5 9641
4cebdcec
MC		 9642 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9643 smp_mb__after_atomic();
9644 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9645 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5 9646 return;
4cebdcec 9647 }
c0c050c5
MC		 9648
9649 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
9650 bnxt_cfg_rx_mode(bp);
9651
9652 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
9653 bnxt_cfg_ntp_filters(bp);
c0c050c5
MC		 9654 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
9655 bnxt_hwrm_exec_fwd_req(bp);
9656 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9657 bnxt_hwrm_tunnel_dst_port_alloc(
9658 bp, bp->vxlan_port,
9659 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9660 }
9661 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9662 bnxt_hwrm_tunnel_dst_port_free(
9663 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9664 }
7cdd5fc3
AD		 9665 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9666 bnxt_hwrm_tunnel_dst_port_alloc(
9667 bp, bp->nge_port,
9668 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9669 }
9670 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9671 bnxt_hwrm_tunnel_dst_port_free(
9672 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9673 }
00db3cba 9674 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
3bdf56c4 9675 bnxt_hwrm_port_qstats(bp);
00db3cba 9676 bnxt_hwrm_port_qstats_ext(bp);
55e4398d 9677 bnxt_hwrm_pcie_qstats(bp);
00db3cba 9678 }
3bdf56c4 9679
0eaa24b9 9680 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
e2dc9b6e 9681 int rc;
0eaa24b9 9682
e2dc9b6e 9683 mutex_lock(&bp->link_lock);
0eaa24b9
MC		 9684 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
9685 &bp->sp_event))
9686 bnxt_hwrm_phy_qcaps(bp);
9687
e2dc9b6e
MC		 9688 rc = bnxt_update_link(bp, true);
9689 mutex_unlock(&bp->link_lock);
0eaa24b9
MC		 9690 if (rc)
9691 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
9692 rc);
9693 }
a1ef4a79
MC		 9694 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
9695 int rc;
9696
9697 mutex_lock(&bp->link_lock);
9698 rc = bnxt_update_phy_setting(bp);
9699 mutex_unlock(&bp->link_lock);
9700 if (rc) {
9701 netdev_warn(bp->dev, "update phy settings retry failed\n");
9702 } else {
9703 bp->link_info.phy_retry = false;
9704 netdev_info(bp->dev, "update phy settings retry succeeded\n");
9705 }
9706 }
90c694bb 9707 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
e2dc9b6e
MC		 9708 mutex_lock(&bp->link_lock);
9709 bnxt_get_port_module_status(bp);
9710 mutex_unlock(&bp->link_lock);
90c694bb 9711 }
5a84acbe
SP		 9712
9713 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
9714 bnxt_tc_flow_stats_work(bp);
9715
ffd77621
MC		 9716 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
9717 bnxt_chk_missed_irq(bp);
9718
e2dc9b6e
MC		 9719 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
9720 * must be the last functions to be called before exiting.
9721 */
6988bd92
MC		 9722 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
9723 bnxt_reset(bp, false);
4cebdcec 9724
fc0f1929
MC		 9725 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
9726 bnxt_reset(bp, true);
9727
4cebdcec
MC		 9728 smp_mb__before_atomic();
9729 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5
MC		 9730}
9731
d1e7925e 9732/* Under rtnl_lock */
98fdbe73
MC		 9733int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
9734 int tx_xdp)
d1e7925e
MC		 9735{
9736 int max_rx, max_tx, tx_sets = 1;
780baad4 9737 int tx_rings_needed, stats;
8f23d638 9738 int rx_rings = rx;
6fc2ffdf 9739 int cp, vnics, rc;
d1e7925e 9740
d1e7925e
MC		 9741 if (tcs)
9742 tx_sets = tcs;
9743
9744 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
9745 if (rc)
9746 return rc;
9747
9748 if (max_rx < rx)
9749 return -ENOMEM;
9750
5f449249 9751 tx_rings_needed = tx * tx_sets + tx_xdp;
d1e7925e
MC		 9752 if (max_tx < tx_rings_needed)
9753 return -ENOMEM;
9754
6fc2ffdf 9755 vnics = 1;
9b3d15e6 9756 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
6fc2ffdf
EW		 9757 vnics += rx_rings;
9758
8f23d638
MC		 9759 if (bp->flags & BNXT_FLAG_AGG_RINGS)
9760 rx_rings <<= 1;
9761 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
780baad4
VV		 9762 stats = cp;
9763 if (BNXT_NEW_RM(bp)) {
11c3ec7b 9764 cp += bnxt_get_ulp_msix_num(bp);
780baad4
VV		 9765 stats += bnxt_get_ulp_stat_ctxs(bp);
9766 }
6fc2ffdf 9767 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
780baad4 9768 stats, vnics);
d1e7925e
MC		 9769}
9770
17086399
SP		 9771static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
9772{
9773 if (bp->bar2) {
9774 pci_iounmap(pdev, bp->bar2);
9775 bp->bar2 = NULL;
9776 }
9777
9778 if (bp->bar1) {
9779 pci_iounmap(pdev, bp->bar1);
9780 bp->bar1 = NULL;
9781 }
9782
9783 if (bp->bar0) {
9784 pci_iounmap(pdev, bp->bar0);
9785 bp->bar0 = NULL;
9786 }
9787}
9788
9789static void bnxt_cleanup_pci(struct bnxt *bp)
9790{
9791 bnxt_unmap_bars(bp, bp->pdev);
9792 pci_release_regions(bp->pdev);
9793 pci_disable_device(bp->pdev);
9794}
9795
18775aa8
MC		 9796static void bnxt_init_dflt_coal(struct bnxt *bp)
9797{
9798 struct bnxt_coal *coal;
9799
9800 /* Tick values in micro seconds.
9801 * 1 coal_buf x bufs_per_record = 1 completion record.
9802 */
9803 coal = &bp->rx_coal;
0c2ff8d7 9804 coal->coal_ticks = 10;
18775aa8
MC		 9805 coal->coal_bufs = 30;
9806 coal->coal_ticks_irq = 1;
9807 coal->coal_bufs_irq = 2;
05abe4dd 9808 coal->idle_thresh = 50;
18775aa8
MC		 9809 coal->bufs_per_record = 2;
9810 coal->budget = 64; /* NAPI budget */
9811
9812 coal = &bp->tx_coal;
9813 coal->coal_ticks = 28;
9814 coal->coal_bufs = 30;
9815 coal->coal_ticks_irq = 2;
9816 coal->coal_bufs_irq = 2;
9817 coal->bufs_per_record = 1;
9818
9819 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
9820}
9821
c0c050c5
MC		 9822static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
9823{
9824 int rc;
9825 struct bnxt *bp = netdev_priv(dev);
9826
9827 SET_NETDEV_DEV(dev, &pdev->dev);
9828
9829 /* enable device (incl. PCI PM wakeup), and bus-mastering */
9830 rc = pci_enable_device(pdev);
9831 if (rc) {
9832 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
9833 goto init_err;
9834 }
9835
9836 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
9837 dev_err(&pdev->dev,
9838 "Cannot find PCI device base address, aborting\n");
9839 rc = -ENODEV;
9840 goto init_err_disable;
9841 }
9842
9843 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
9844 if (rc) {
9845 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
9846 goto init_err_disable;
9847 }
9848
9849 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
9850 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
9851 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
9852 goto init_err_disable;
9853 }
9854
9855 pci_set_master(pdev);
9856
9857 bp->dev = dev;
9858 bp->pdev = pdev;
9859
9860 bp->bar0 = pci_ioremap_bar(pdev, 0);
9861 if (!bp->bar0) {
9862 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
9863 rc = -ENOMEM;
9864 goto init_err_release;
9865 }
9866
9867 bp->bar1 = pci_ioremap_bar(pdev, 2);
9868 if (!bp->bar1) {
9869 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
9870 rc = -ENOMEM;
9871 goto init_err_release;
9872 }
9873
9874 bp->bar2 = pci_ioremap_bar(pdev, 4);
9875 if (!bp->bar2) {
9876 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
9877 rc = -ENOMEM;
9878 goto init_err_release;
9879 }
9880
6316ea6d
SB		 9881 pci_enable_pcie_error_reporting(pdev);
9882
c0c050c5
MC		 9883 INIT_WORK(&bp->sp_task, bnxt_sp_task);
9884
9885 spin_lock_init(&bp->ntp_fltr_lock);
697197e5
MC		 9886#if BITS_PER_LONG == 32
9887 spin_lock_init(&bp->db_lock);
9888#endif
c0c050c5
MC		 9889
9890 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
9891 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
9892
18775aa8 9893 bnxt_init_dflt_coal(bp);
51f30785 9894
e99e88a9 9895 timer_setup(&bp->timer, bnxt_timer, 0);
c0c050c5
MC		 9896 bp->current_interval = BNXT_TIMER_INTERVAL;
9897
caefe526 9898 clear_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 9899 return 0;
9900
9901init_err_release:
17086399 9902 bnxt_unmap_bars(bp, pdev);
c0c050c5
MC		 9903 pci_release_regions(pdev);
9904
9905init_err_disable:
9906 pci_disable_device(pdev);
9907
9908init_err:
9909 return rc;
9910}
9911
9912/* rtnl_lock held */
9913static int bnxt_change_mac_addr(struct net_device *dev, void *p)
9914{
9915 struct sockaddr *addr = p;
1fc2cfd0
JH		 9916 struct bnxt *bp = netdev_priv(dev);
9917 int rc = 0;
c0c050c5
MC		 9918
9919 if (!is_valid_ether_addr(addr->sa_data))
9920 return -EADDRNOTAVAIL;
9921
c1a7bdff
MC		 9922 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
9923 return 0;
9924
28ea334b 9925 rc = bnxt_approve_mac(bp, addr->sa_data, true);
84c33dd3
MC		 9926 if (rc)
9927 return rc;
bdd4347b 9928
c0c050c5 9929 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1fc2cfd0
JH		 9930 if (netif_running(dev)) {
9931 bnxt_close_nic(bp, false, false);
9932 rc = bnxt_open_nic(bp, false, false);
9933 }
c0c050c5 9934
1fc2cfd0 9935 return rc;
c0c050c5
MC		 9936}
9937
9938/* rtnl_lock held */
9939static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
9940{
9941 struct bnxt *bp = netdev_priv(dev);
9942
c0c050c5
MC		 9943 if (netif_running(dev))
9944 bnxt_close_nic(bp, false, false);
9945
9946 dev->mtu = new_mtu;
9947 bnxt_set_ring_params(bp);
9948
9949 if (netif_running(dev))
9950 return bnxt_open_nic(bp, false, false);
9951
9952 return 0;
9953}
9954
c5e3deb8 9955int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
c0c050c5
MC		 9956{
9957 struct bnxt *bp = netdev_priv(dev);
3ffb6a39 9958 bool sh = false;
d1e7925e 9959 int rc;
16e5cc64 9960
c0c050c5 9961 if (tc > bp->max_tc) {
b451c8b6 9962 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
c0c050c5
MC		 9963 tc, bp->max_tc);
9964 return -EINVAL;
9965 }
9966
9967 if (netdev_get_num_tc(dev) == tc)
9968 return 0;
9969
3ffb6a39
MC		 9970 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
9971 sh = true;
9972
98fdbe73
MC		 9973 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
9974 sh, tc, bp->tx_nr_rings_xdp);
d1e7925e
MC		 9975 if (rc)
9976 return rc;
c0c050c5
MC		 9977
9978 /* Needs to close the device and do hw resource re-allocations */
9979 if (netif_running(bp->dev))
9980 bnxt_close_nic(bp, true, false);
9981
9982 if (tc) {
9983 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
9984 netdev_set_num_tc(dev, tc);
9985 } else {
9986 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
9987 netdev_reset_tc(dev);
9988 }
87e9b377 9989 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
3ffb6a39
MC		 9990 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9991 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5
MC		 9992
9993 if (netif_running(bp->dev))
9994 return bnxt_open_nic(bp, true, false);
9995
9996 return 0;
9997}
9998
9e0fd15d
JP		 9999static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10000 void *cb_priv)
c5e3deb8 10001{
9e0fd15d 10002 struct bnxt *bp = cb_priv;
de4784ca 10003
312324f1
JK		 10004 if (!bnxt_tc_flower_enabled(bp) ||
10005 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
38cf0426 10006 return -EOPNOTSUPP;
c5e3deb8 10007
9e0fd15d
JP		 10008 switch (type) {
10009 case TC_SETUP_CLSFLOWER:
10010 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
10011 default:
10012 return -EOPNOTSUPP;
10013 }
10014}
10015
955bcb6e
PNA		 10016static LIST_HEAD(bnxt_block_cb_list);
10017
2ae7408f
SP		 10018static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
10019 void *type_data)
10020{
4e95bc26
PNA		 10021 struct bnxt *bp = netdev_priv(dev);
10022
2ae7408f 10023 switch (type) {
9e0fd15d 10024 case TC_SETUP_BLOCK:
955bcb6e
PNA		 10025 return flow_block_cb_setup_simple(type_data,
10026 &bnxt_block_cb_list,
4e95bc26
PNA		 10027 bnxt_setup_tc_block_cb,
10028 bp, bp, true);
575ed7d3 10029 case TC_SETUP_QDISC_MQPRIO: {
2ae7408f
SP		 10030 struct tc_mqprio_qopt *mqprio = type_data;
10031
10032 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
56f36acd 10033
2ae7408f
SP		 10034 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
10035 }
10036 default:
10037 return -EOPNOTSUPP;
10038 }
c5e3deb8
MC		 10039}
10040
c0c050c5
MC		 10041#ifdef CONFIG_RFS_ACCEL
10042static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
10043 struct bnxt_ntuple_filter *f2)
10044{
10045 struct flow_keys *keys1 = &f1->fkeys;
10046 struct flow_keys *keys2 = &f2->fkeys;
10047
10048 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
10049 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
10050 keys1->ports.ports == keys2->ports.ports &&
10051 keys1->basic.ip_proto == keys2->basic.ip_proto &&
10052 keys1->basic.n_proto == keys2->basic.n_proto &&
61aad724 10053 keys1->control.flags == keys2->control.flags &&
a54c4d74
MC		 10054 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
10055 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
c0c050c5
MC		 10056 return true;
10057
10058 return false;
10059}
10060
10061static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
10062 u16 rxq_index, u32 flow_id)
10063{
10064 struct bnxt *bp = netdev_priv(dev);
10065 struct bnxt_ntuple_filter *fltr, *new_fltr;
10066 struct flow_keys *fkeys;
10067 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
a54c4d74 10068 int rc = 0, idx, bit_id, l2_idx = 0;
c0c050c5
MC		 10069 struct hlist_head *head;
10070
a54c4d74
MC		 10071 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
10072 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10073 int off = 0, j;
10074
10075 netif_addr_lock_bh(dev);
10076 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
10077 if (ether_addr_equal(eth->h_dest,
10078 vnic->uc_list + off)) {
10079 l2_idx = j + 1;
10080 break;
10081 }
10082 }
10083 netif_addr_unlock_bh(dev);
10084 if (!l2_idx)
10085 return -EINVAL;
10086 }
c0c050c5
MC		 10087 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
10088 if (!new_fltr)
10089 return -ENOMEM;
10090
10091 fkeys = &new_fltr->fkeys;
10092 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
10093 rc = -EPROTONOSUPPORT;
10094 goto err_free;
10095 }
10096
dda0e746
MC		 10097 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
10098 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
c0c050c5
MC		 10099 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
10100 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
10101 rc = -EPROTONOSUPPORT;
10102 goto err_free;
10103 }
dda0e746
MC		 10104 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
10105 bp->hwrm_spec_code < 0x10601) {
10106 rc = -EPROTONOSUPPORT;
10107 goto err_free;
10108 }
61aad724
MC		 10109 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
10110 bp->hwrm_spec_code < 0x10601) {
10111 rc = -EPROTONOSUPPORT;
10112 goto err_free;
10113 }
c0c050c5 10114
a54c4d74 10115 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
c0c050c5
MC		 10116 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
10117
10118 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
10119 head = &bp->ntp_fltr_hash_tbl[idx];
10120 rcu_read_lock();
10121 hlist_for_each_entry_rcu(fltr, head, hash) {
10122 if (bnxt_fltr_match(fltr, new_fltr)) {
10123 rcu_read_unlock();
10124 rc = 0;
10125 goto err_free;
10126 }
10127 }
10128 rcu_read_unlock();
10129
10130 spin_lock_bh(&bp->ntp_fltr_lock);
84e86b98
MC		 10131 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
10132 BNXT_NTP_FLTR_MAX_FLTR, 0);
10133 if (bit_id < 0) {
c0c050c5
MC		 10134 spin_unlock_bh(&bp->ntp_fltr_lock);
10135 rc = -ENOMEM;
10136 goto err_free;
10137 }
10138
84e86b98 10139 new_fltr->sw_id = (u16)bit_id;
c0c050c5 10140 new_fltr->flow_id = flow_id;
a54c4d74 10141 new_fltr->l2_fltr_idx = l2_idx;
c0c050c5
MC		 10142 new_fltr->rxq = rxq_index;
10143 hlist_add_head_rcu(&new_fltr->hash, head);
10144 bp->ntp_fltr_count++;
10145 spin_unlock_bh(&bp->ntp_fltr_lock);
10146
10147 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
c213eae8 10148 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10149
10150 return new_fltr->sw_id;
10151
10152err_free:
10153 kfree(new_fltr);
10154 return rc;
10155}
10156
10157static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10158{
10159 int i;
10160
10161 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
10162 struct hlist_head *head;
10163 struct hlist_node *tmp;
10164 struct bnxt_ntuple_filter *fltr;
10165 int rc;
10166
10167 head = &bp->ntp_fltr_hash_tbl[i];
10168 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
10169 bool del = false;
10170
10171 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
10172 if (rps_may_expire_flow(bp->dev, fltr->rxq,
10173 fltr->flow_id,
10174 fltr->sw_id)) {
10175 bnxt_hwrm_cfa_ntuple_filter_free(bp,
10176 fltr);
10177 del = true;
10178 }
10179 } else {
10180 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
10181 fltr);
10182 if (rc)
10183 del = true;
10184 else
10185 set_bit(BNXT_FLTR_VALID, &fltr->state);
10186 }
10187
10188 if (del) {
10189 spin_lock_bh(&bp->ntp_fltr_lock);
10190 hlist_del_rcu(&fltr->hash);
10191 bp->ntp_fltr_count--;
10192 spin_unlock_bh(&bp->ntp_fltr_lock);
10193 synchronize_rcu();
10194 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
10195 kfree(fltr);
10196 }
10197 }
10198 }
19241368
JH		 10199 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
10200 netdev_info(bp->dev, "Receive PF driver unload event!");
c0c050c5
MC		 10201}
10202
10203#else
10204
10205static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10206{
10207}
10208
10209#endif /* CONFIG_RFS_ACCEL */
10210
ad51b8e9
AD		 10211static void bnxt_udp_tunnel_add(struct net_device *dev,
10212 struct udp_tunnel_info *ti)
c0c050c5
MC		 10213{
10214 struct bnxt *bp = netdev_priv(dev);
10215
ad51b8e9 10216 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 10217 return;
10218
ad51b8e9 10219 if (!netif_running(dev))
c0c050c5
MC		 10220 return;
10221
ad51b8e9
AD		 10222 switch (ti->type) {
10223 case UDP_TUNNEL_TYPE_VXLAN:
10224 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
10225 return;
c0c050c5 10226
ad51b8e9
AD		 10227 bp->vxlan_port_cnt++;
10228 if (bp->vxlan_port_cnt == 1) {
10229 bp->vxlan_port = ti->port;
10230 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
c213eae8 10231 bnxt_queue_sp_work(bp);
ad51b8e9
AD		 10232 }
10233 break;
7cdd5fc3
AD		 10234 case UDP_TUNNEL_TYPE_GENEVE:
10235 if (bp->nge_port_cnt && bp->nge_port != ti->port)
10236 return;
10237
10238 bp->nge_port_cnt++;
10239 if (bp->nge_port_cnt == 1) {
10240 bp->nge_port = ti->port;
10241 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
10242 }
10243 break;
ad51b8e9
AD		 10244 default:
10245 return;
c0c050c5 10246 }
ad51b8e9 10247
c213eae8 10248 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10249}
10250
ad51b8e9
AD		 10251static void bnxt_udp_tunnel_del(struct net_device *dev,
10252 struct udp_tunnel_info *ti)
c0c050c5
MC		 10253{
10254 struct bnxt *bp = netdev_priv(dev);
10255
ad51b8e9 10256 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 10257 return;
10258
ad51b8e9 10259 if (!netif_running(dev))
c0c050c5
MC		 10260 return;
10261
ad51b8e9
AD		 10262 switch (ti->type) {
10263 case UDP_TUNNEL_TYPE_VXLAN:
10264 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
10265 return;
c0c050c5
MC		 10266 bp->vxlan_port_cnt--;
10267
ad51b8e9
AD		 10268 if (bp->vxlan_port_cnt != 0)
10269 return;
10270
10271 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
10272 break;
7cdd5fc3
AD		 10273 case UDP_TUNNEL_TYPE_GENEVE:
10274 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
10275 return;
10276 bp->nge_port_cnt--;
10277
10278 if (bp->nge_port_cnt != 0)
10279 return;
10280
10281 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
10282 break;
ad51b8e9
AD		 10283 default:
10284 return;
c0c050c5 10285 }
ad51b8e9 10286
c213eae8 10287 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10288}
10289
39d8ba2e
MC		 10290static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
10291 struct net_device *dev, u32 filter_mask,
10292 int nlflags)
10293{
10294 struct bnxt *bp = netdev_priv(dev);
10295
10296 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
10297 nlflags, filter_mask, NULL);
10298}
10299
10300static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
2fd527b7 10301 u16 flags, struct netlink_ext_ack *extack)
39d8ba2e
MC		 10302{
10303 struct bnxt *bp = netdev_priv(dev);
10304 struct nlattr *attr, *br_spec;
10305 int rem, rc = 0;
10306
10307 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
10308 return -EOPNOTSUPP;
10309
10310 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
10311 if (!br_spec)
10312 return -EINVAL;
10313
10314 nla_for_each_nested(attr, br_spec, rem) {
10315 u16 mode;
10316
10317 if (nla_type(attr) != IFLA_BRIDGE_MODE)
10318 continue;
10319
10320 if (nla_len(attr) < sizeof(mode))
10321 return -EINVAL;
10322
10323 mode = nla_get_u16(attr);
10324 if (mode == bp->br_mode)
10325 break;
10326
10327 rc = bnxt_hwrm_set_br_mode(bp, mode);
10328 if (!rc)
10329 bp->br_mode = mode;
10330 break;
10331 }
10332 return rc;
10333}
10334
52d5254a
FF		 10335int bnxt_get_port_parent_id(struct net_device *dev,
10336 struct netdev_phys_item_id *ppid)
c124a62f 10337{
52d5254a
FF		 10338 struct bnxt *bp = netdev_priv(dev);
10339
c124a62f
SP		 10340 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
10341 return -EOPNOTSUPP;
10342
10343 /* The PF and it's VF-reps only support the switchdev framework */
10344 if (!BNXT_PF(bp))
10345 return -EOPNOTSUPP;
10346
52d5254a
FF		 10347 ppid->id_len = sizeof(bp->switch_id);
10348 memcpy(ppid->id, bp->switch_id, ppid->id_len);
c124a62f 10349
52d5254a 10350 return 0;
c124a62f
SP		 10351}
10352
c9c49a65
JP		 10353static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
10354{
10355 struct bnxt *bp = netdev_priv(dev);
10356
10357 return &bp->dl_port;
10358}
10359
c0c050c5
MC		 10360static const struct net_device_ops bnxt_netdev_ops = {
10361 .ndo_open = bnxt_open,
10362 .ndo_start_xmit = bnxt_start_xmit,
10363 .ndo_stop = bnxt_close,
10364 .ndo_get_stats64 = bnxt_get_stats64,
10365 .ndo_set_rx_mode = bnxt_set_rx_mode,
10366 .ndo_do_ioctl = bnxt_ioctl,
10367 .ndo_validate_addr = eth_validate_addr,
10368 .ndo_set_mac_address = bnxt_change_mac_addr,
10369 .ndo_change_mtu = bnxt_change_mtu,
10370 .ndo_fix_features = bnxt_fix_features,
10371 .ndo_set_features = bnxt_set_features,
10372 .ndo_tx_timeout = bnxt_tx_timeout,
10373#ifdef CONFIG_BNXT_SRIOV
10374 .ndo_get_vf_config = bnxt_get_vf_config,
10375 .ndo_set_vf_mac = bnxt_set_vf_mac,
10376 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
10377 .ndo_set_vf_rate = bnxt_set_vf_bw,
10378 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
10379 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
746df139 10380 .ndo_set_vf_trust = bnxt_set_vf_trust,
c0c050c5
MC		 10381#endif
10382 .ndo_setup_tc = bnxt_setup_tc,
10383#ifdef CONFIG_RFS_ACCEL
10384 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
10385#endif
ad51b8e9
AD		 10386 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add,
10387 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del,
f4e63525 10388 .ndo_bpf = bnxt_xdp,
f18c2b77 10389 .ndo_xdp_xmit = bnxt_xdp_xmit,
39d8ba2e
MC		 10390 .ndo_bridge_getlink = bnxt_bridge_getlink,
10391 .ndo_bridge_setlink = bnxt_bridge_setlink,
c9c49a65 10392 .ndo_get_devlink_port = bnxt_get_devlink_port,
c0c050c5
MC		 10393};
10394
10395static void bnxt_remove_one(struct pci_dev *pdev)
10396{
10397 struct net_device *dev = pci_get_drvdata(pdev);
10398 struct bnxt *bp = netdev_priv(dev);
10399
4ab0c6a8 10400 if (BNXT_PF(bp)) {
c0c050c5 10401 bnxt_sriov_disable(bp);
4ab0c6a8
SP		 10402 bnxt_dl_unregister(bp);
10403 }
c0c050c5 10404
6316ea6d 10405 pci_disable_pcie_error_reporting(pdev);
c0c050c5 10406 unregister_netdev(dev);
2ae7408f 10407 bnxt_shutdown_tc(bp);
c213eae8 10408 bnxt_cancel_sp_work(bp);
c0c050c5
MC		 10409 bp->sp_event = 0;
10410
7809592d 10411 bnxt_clear_int_mode(bp);
be58a0da 10412 bnxt_hwrm_func_drv_unrgtr(bp);
c0c050c5 10413 bnxt_free_hwrm_resources(bp);
e605db80 10414 bnxt_free_hwrm_short_cmd_req(bp);
eb513658 10415 bnxt_ethtool_free(bp);
7df4ae9f 10416 bnxt_dcb_free(bp);
a588e458
MC		 10417 kfree(bp->edev);
10418 bp->edev = NULL;
c20dc142 10419 bnxt_cleanup_pci(bp);
98f04cf0
MC		 10420 bnxt_free_ctx_mem(bp);
10421 kfree(bp->ctx);
10422 bp->ctx = NULL;
fd3ab1c7 10423 bnxt_free_port_stats(bp);
c0c050c5 10424 free_netdev(dev);
c0c050c5
MC		 10425}
10426
10427static int bnxt_probe_phy(struct bnxt *bp)
10428{
10429 int rc = 0;
10430 struct bnxt_link_info *link_info = &bp->link_info;
c0c050c5 10431
170ce013
MC		 10432 rc = bnxt_hwrm_phy_qcaps(bp);
10433 if (rc) {
10434 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
10435 rc);
10436 return rc;
10437 }
e2dc9b6e 10438 mutex_init(&bp->link_lock);
170ce013 10439
c0c050c5
MC		 10440 rc = bnxt_update_link(bp, false);
10441 if (rc) {
10442 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
10443 rc);
10444 return rc;
10445 }
10446
93ed8117
MC		 10447 /* Older firmware does not have supported_auto_speeds, so assume
10448 * that all supported speeds can be autonegotiated.
10449 */
10450 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
10451 link_info->support_auto_speeds = link_info->support_speeds;
10452
c0c050c5 10453 /*initialize the ethool setting copy with NVM settings */
0d8abf02 10454 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
c9ee9516
MC		 10455 link_info->autoneg = BNXT_AUTONEG_SPEED;
10456 if (bp->hwrm_spec_code >= 0x10201) {
10457 if (link_info->auto_pause_setting &
10458 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10459 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10460 } else {
10461 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10462 }
0d8abf02 10463 link_info->advertising = link_info->auto_link_speeds;
0d8abf02
MC		 10464 } else {
10465 link_info->req_link_speed = link_info->force_link_speed;
10466 link_info->req_duplex = link_info->duplex_setting;
c0c050c5 10467 }
c9ee9516
MC		 10468 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10469 link_info->req_flow_ctrl =
10470 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10471 else
10472 link_info->req_flow_ctrl = link_info->force_pause_setting;
c0c050c5
MC		 10473 return rc;
10474}
10475
10476static int bnxt_get_max_irq(struct pci_dev *pdev)
10477{
10478 u16 ctrl;
10479
10480 if (!pdev->msix_cap)
10481 return 1;
10482
10483 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
10484 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
10485}
10486
6e6c5a57
MC		 10487static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10488 int *max_cp)
c0c050c5 10489{
6a4f2947 10490 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
e30fbc33 10491 int max_ring_grps = 0, max_irq;
c0c050c5 10492
6a4f2947
MC		 10493 *max_tx = hw_resc->max_tx_rings;
10494 *max_rx = hw_resc->max_rx_rings;
e30fbc33
MC		 10495 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
10496 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
10497 bnxt_get_ulp_msix_num(bp),
c027c6b4 10498 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
e30fbc33
MC		 10499 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10500 *max_cp = min_t(int, *max_cp, max_irq);
6a4f2947 10501 max_ring_grps = hw_resc->max_hw_ring_grps;
76595193
PS		 10502 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
10503 *max_cp -= 1;
10504 *max_rx -= 2;
10505 }
c0c050c5
MC		 10506 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10507 *max_rx >>= 1;
e30fbc33
MC		 10508 if (bp->flags & BNXT_FLAG_CHIP_P5) {
10509 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
10510 /* On P5 chips, max_cp output param should be available NQs */
10511 *max_cp = max_irq;
10512 }
b72d4a68 10513 *max_rx = min_t(int, *max_rx, max_ring_grps);
6e6c5a57
MC		 10514}
10515
10516int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
10517{
10518 int rx, tx, cp;
10519
10520 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
78f058a4
MC		 10521 *max_rx = rx;
10522 *max_tx = tx;
6e6c5a57
MC		 10523 if (!rx || !tx || !cp)
10524 return -ENOMEM;
10525
6e6c5a57
MC		 10526 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
10527}
10528
e4060d30
MC		 10529static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10530 bool shared)
10531{
10532 int rc;
10533
10534 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
bdbd1eb5
MC		 10535 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
10536 /* Not enough rings, try disabling agg rings. */
10537 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
10538 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
07f4fde5
MC		 10539 if (rc) {
10540 /* set BNXT_FLAG_AGG_RINGS back for consistency */
10541 bp->flags |= BNXT_FLAG_AGG_RINGS;
bdbd1eb5 10542 return rc;
07f4fde5 10543 }
bdbd1eb5 10544 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
1054aee8
MC		 10545 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10546 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
bdbd1eb5
MC		 10547 bnxt_set_ring_params(bp);
10548 }
e4060d30
MC		 10549
10550 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
10551 int max_cp, max_stat, max_irq;
10552
10553 /* Reserve minimum resources for RoCE */
10554 max_cp = bnxt_get_max_func_cp_rings(bp);
10555 max_stat = bnxt_get_max_func_stat_ctxs(bp);
10556 max_irq = bnxt_get_max_func_irqs(bp);
10557 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
10558 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
10559 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
10560 return 0;
10561
10562 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
10563 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
10564 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
10565 max_cp = min_t(int, max_cp, max_irq);
10566 max_cp = min_t(int, max_cp, max_stat);
10567 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
10568 if (rc)
10569 rc = 0;
10570 }
10571 return rc;
10572}
10573
58ea801a
MC		 10574/* In initial default shared ring setting, each shared ring must have a
10575 * RX/TX ring pair.
10576 */
10577static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
10578{
10579 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
10580 bp->rx_nr_rings = bp->cp_nr_rings;
10581 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
10582 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10583}
10584
702c221c 10585static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
6e6c5a57
MC		 10586{
10587 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6e6c5a57 10588
2773dfb2
MC		 10589 if (!bnxt_can_reserve_rings(bp))
10590 return 0;
10591
6e6c5a57
MC		 10592 if (sh)
10593 bp->flags |= BNXT_FLAG_SHARED_RINGS;
d629522e 10594 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
1d3ef13d
MC		 10595 /* Reduce default rings on multi-port cards so that total default
10596 * rings do not exceed CPU count.
10597 */
10598 if (bp->port_count > 1) {
10599 int max_rings =
10600 max_t(int, num_online_cpus() / bp->port_count, 1);
10601
10602 dflt_rings = min_t(int, dflt_rings, max_rings);
10603 }
e4060d30 10604 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6e6c5a57
MC		 10605 if (rc)
10606 return rc;
10607 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
10608 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
58ea801a
MC		 10609 if (sh)
10610 bnxt_trim_dflt_sh_rings(bp);
10611 else
10612 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
10613 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
391be5c2 10614
674f50a5 10615 rc = __bnxt_reserve_rings(bp);
391be5c2
MC		 10616 if (rc)
10617 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
58ea801a
MC		 10618 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10619 if (sh)
10620 bnxt_trim_dflt_sh_rings(bp);
391be5c2 10621
674f50a5
MC		 10622 /* Rings may have been trimmed, re-reserve the trimmed rings. */
10623 if (bnxt_need_reserve_rings(bp)) {
10624 rc = __bnxt_reserve_rings(bp);
10625 if (rc)
10626 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
10627 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10628 }
76595193
PS		 10629 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10630 bp->rx_nr_rings++;
10631 bp->cp_nr_rings++;
10632 }
6e6c5a57 10633 return rc;
c0c050c5
MC		 10634}
10635
47558acd
MC		 10636static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
10637{
10638 int rc;
10639
10640 if (bp->tx_nr_rings)
10641 return 0;
10642
6b95c3e9
MC		 10643 bnxt_ulp_irq_stop(bp);
10644 bnxt_clear_int_mode(bp);
47558acd
MC		 10645 rc = bnxt_set_dflt_rings(bp, true);
10646 if (rc) {
10647 netdev_err(bp->dev, "Not enough rings available.\n");
6b95c3e9 10648 goto init_dflt_ring_err;
47558acd
MC		 10649 }
10650 rc = bnxt_init_int_mode(bp);
10651 if (rc)
6b95c3e9
MC		 10652 goto init_dflt_ring_err;
10653
47558acd
MC		 10654 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10655 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
10656 bp->flags |= BNXT_FLAG_RFS;
10657 bp->dev->features |= NETIF_F_NTUPLE;
10658 }
6b95c3e9
MC		 10659init_dflt_ring_err:
10660 bnxt_ulp_irq_restart(bp, rc);
10661 return rc;
47558acd
MC		 10662}
10663
80fcaf46 10664int bnxt_restore_pf_fw_resources(struct bnxt *bp)
7b08f661 10665{
80fcaf46
MC		 10666 int rc;
10667
7b08f661
MC		 10668 ASSERT_RTNL();
10669 bnxt_hwrm_func_qcaps(bp);
1a037782
VD		 10670
10671 if (netif_running(bp->dev))
10672 __bnxt_close_nic(bp, true, false);
10673
ec86f14e 10674 bnxt_ulp_irq_stop(bp);
80fcaf46
MC		 10675 bnxt_clear_int_mode(bp);
10676 rc = bnxt_init_int_mode(bp);
ec86f14e 10677 bnxt_ulp_irq_restart(bp, rc);
1a037782
VD		 10678
10679 if (netif_running(bp->dev)) {
10680 if (rc)
10681 dev_close(bp->dev);
10682 else
10683 rc = bnxt_open_nic(bp, true, false);
10684 }
10685
80fcaf46 10686 return rc;
7b08f661
MC		 10687}
10688
a22a6ac2
MC		 10689static int bnxt_init_mac_addr(struct bnxt *bp)
10690{
10691 int rc = 0;
10692
10693 if (BNXT_PF(bp)) {
10694 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
10695 } else {
10696#ifdef CONFIG_BNXT_SRIOV
10697 struct bnxt_vf_info *vf = &bp->vf;
28ea334b 10698 bool strict_approval = true;
a22a6ac2
MC		 10699
10700 if (is_valid_ether_addr(vf->mac_addr)) {
91cdda40 10701 /* overwrite netdev dev_addr with admin VF MAC */
a22a6ac2 10702 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
28ea334b
MC		 10703 /* Older PF driver or firmware may not approve this
10704 * correctly.
10705 */
10706 strict_approval = false;
a22a6ac2
MC		 10707 } else {
10708 eth_hw_addr_random(bp->dev);
a22a6ac2 10709 }
28ea334b 10710 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
a22a6ac2
MC		 10711#endif
10712 }
10713 return rc;
10714}
10715
03213a99
JP		 10716static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
10717{
10718 struct pci_dev *pdev = bp->pdev;
10719 int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
10720 u32 dw;
10721
10722 if (!pos) {
10723 netdev_info(bp->dev, "Unable do read adapter's DSN");
10724 return -EOPNOTSUPP;
10725 }
10726
10727 /* DSN (two dw) is at an offset of 4 from the cap pos */
10728 pos += 4;
10729 pci_read_config_dword(pdev, pos, &dw);
10730 put_unaligned_le32(dw, &dsn[0]);
10731 pci_read_config_dword(pdev, pos + 4, &dw);
10732 put_unaligned_le32(dw, &dsn[4]);
10733 return 0;
10734}
10735
c0c050c5
MC		 10736static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
10737{
10738 static int version_printed;
10739 struct net_device *dev;
10740 struct bnxt *bp;
6e6c5a57 10741 int rc, max_irqs;
c0c050c5 10742
4e00338a 10743 if (pci_is_bridge(pdev))
fa853dda
PS		 10744 return -ENODEV;
10745
c0c050c5
MC		 10746 if (version_printed++ == 0)
10747 pr_info("%s", version);
10748
10749 max_irqs = bnxt_get_max_irq(pdev);
10750 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
10751 if (!dev)
10752 return -ENOMEM;
10753
10754 bp = netdev_priv(dev);
9c1fabdf 10755 bnxt_set_max_func_irqs(bp, max_irqs);
c0c050c5
MC		 10756
10757 if (bnxt_vf_pciid(ent->driver_data))
10758 bp->flags |= BNXT_FLAG_VF;
10759
2bcfa6f6 10760 if (pdev->msix_cap)
c0c050c5 10761 bp->flags |= BNXT_FLAG_MSIX_CAP;
c0c050c5
MC		 10762
10763 rc = bnxt_init_board(pdev, dev);
10764 if (rc < 0)
10765 goto init_err_free;
10766
10767 dev->netdev_ops = &bnxt_netdev_ops;
10768 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
10769 dev->ethtool_ops = &bnxt_ethtool_ops;
c0c050c5
MC		 10770 pci_set_drvdata(pdev, dev);
10771
3e8060fa
PS		 10772 rc = bnxt_alloc_hwrm_resources(bp);
10773 if (rc)
17086399 10774 goto init_err_pci_clean;
3e8060fa
PS		 10775
10776 mutex_init(&bp->hwrm_cmd_lock);
10777 rc = bnxt_hwrm_ver_get(bp);
10778 if (rc)
17086399 10779 goto init_err_pci_clean;
3e8060fa 10780
760b6d33
VD		 10781 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10782 rc = bnxt_alloc_kong_hwrm_resources(bp);
10783 if (rc)
10784 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10785 }
10786
1dfddc41
MC		 10787 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10788 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
e605db80
DK		 10789 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10790 if (rc)
10791 goto init_err_pci_clean;
10792 }
10793
e38287b7
MC		 10794 if (BNXT_CHIP_P5(bp))
10795 bp->flags |= BNXT_FLAG_CHIP_P5;
10796
3c2217a6
MC		 10797 rc = bnxt_hwrm_func_reset(bp);
10798 if (rc)
10799 goto init_err_pci_clean;
10800
5ac67d8b
RS		 10801 bnxt_hwrm_fw_set_time(bp);
10802
c0c050c5
MC		 10803 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10804 NETIF_F_TSO | NETIF_F_TSO6 |
10805 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
7e13318d 10806 NETIF_F_GSO_IPXIP4 |
152971ee
AD		 10807 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10808 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
3e8060fa
PS		 10809 NETIF_F_RXCSUM | NETIF_F_GRO;
10810
e38287b7 10811 if (BNXT_SUPPORTS_TPA(bp))
3e8060fa 10812 dev->hw_features |= NETIF_F_LRO;
c0c050c5 10813
c0c050c5
MC		 10814 dev->hw_enc_features =
10815 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10816 NETIF_F_TSO | NETIF_F_TSO6 |
10817 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
152971ee 10818 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
7e13318d 10819 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
152971ee
AD		 10820 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
10821 NETIF_F_GSO_GRE_CSUM;
c0c050c5
MC		 10822 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
10823 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
10824 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
e38287b7 10825 if (BNXT_SUPPORTS_TPA(bp))
1054aee8 10826 dev->hw_features |= NETIF_F_GRO_HW;
c0c050c5 10827 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
1054aee8
MC		 10828 if (dev->features & NETIF_F_GRO_HW)
10829 dev->features &= ~NETIF_F_LRO;
c0c050c5
MC		 10830 dev->priv_flags |= IFF_UNICAST_FLT;
10831
10832#ifdef CONFIG_BNXT_SRIOV
10833 init_waitqueue_head(&bp->sriov_cfg_wait);
4ab0c6a8 10834 mutex_init(&bp->sriov_lock);
c0c050c5 10835#endif
e38287b7
MC		 10836 if (BNXT_SUPPORTS_TPA(bp)) {
10837 bp->gro_func = bnxt_gro_func_5730x;
bfcd8d79 10838 if (BNXT_CHIP_P4_PLUS(bp))
e38287b7
MC		 10839 bp->gro_func = bnxt_gro_func_5731x;
10840 }
10841 if (!BNXT_CHIP_P4_PLUS(bp))
434c975a 10842 bp->flags |= BNXT_FLAG_DOUBLE_DB;
309369c9 10843
c0c050c5
MC		 10844 rc = bnxt_hwrm_func_drv_rgtr(bp);
10845 if (rc)
17086399 10846 goto init_err_pci_clean;
c0c050c5 10847
a1653b13
MC		 10848 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
10849 if (rc)
17086399 10850 goto init_err_pci_clean;
a1653b13 10851
a588e458
MC		 10852 bp->ulp_probe = bnxt_ulp_probe;
10853
98f04cf0
MC		 10854 rc = bnxt_hwrm_queue_qportcfg(bp);
10855 if (rc) {
10856 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
10857 rc);
10858 rc = -1;
10859 goto init_err_pci_clean;
10860 }
c0c050c5
MC		 10861 /* Get the MAX capabilities for this function */
10862 rc = bnxt_hwrm_func_qcaps(bp);
10863 if (rc) {
10864 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10865 rc);
10866 rc = -1;
17086399 10867 goto init_err_pci_clean;
c0c050c5 10868 }
e969ae5b
MC		 10869
10870 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10871 if (rc)
10872 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10873 rc);
10874
a22a6ac2
MC		 10875 rc = bnxt_init_mac_addr(bp);
10876 if (rc) {
10877 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
10878 rc = -EADDRNOTAVAIL;
10879 goto init_err_pci_clean;
10880 }
c0c050c5 10881
2e9217d1
VV		 10882 if (BNXT_PF(bp)) {
10883 /* Read the adapter's DSN to use as the eswitch switch_id */
10884 rc = bnxt_pcie_dsn_get(bp, bp->switch_id);
10885 if (rc)
10886 goto init_err_pci_clean;
10887 }
567b2abe 10888 bnxt_hwrm_func_qcfg(bp);
6ba99038 10889 bnxt_hwrm_vnic_qcaps(bp);
5ad2cbee 10890 bnxt_hwrm_port_led_qcaps(bp);
eb513658 10891 bnxt_ethtool_init(bp);
87fe6032 10892 bnxt_dcb_init(bp);
567b2abe 10893
7eb9bb3a
MC		 10894 /* MTU range: 60 - FW defined max */
10895 dev->min_mtu = ETH_ZLEN;
10896 dev->max_mtu = bp->max_mtu;
10897
d5430d31
MC		 10898 rc = bnxt_probe_phy(bp);
10899 if (rc)
10900 goto init_err_pci_clean;
10901
c61fb99c 10902 bnxt_set_rx_skb_mode(bp, false);
c0c050c5
MC		 10903 bnxt_set_tpa_flags(bp);
10904 bnxt_set_ring_params(bp);
702c221c 10905 rc = bnxt_set_dflt_rings(bp, true);
bdbd1eb5
MC		 10906 if (rc) {
10907 netdev_err(bp->dev, "Not enough rings available.\n");
10908 rc = -ENOMEM;
17086399 10909 goto init_err_pci_clean;
bdbd1eb5 10910 }
c0c050c5 10911
87da7f79
MC		 10912 /* Default RSS hash cfg. */
10913 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10914 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10915 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10916 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
e38287b7 10917 if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
87da7f79
MC		 10918 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10919 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10920 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10921 }
10922
8079e8f1 10923 if (bnxt_rfs_supported(bp)) {
2bcfa6f6
MC		 10924 dev->hw_features |= NETIF_F_NTUPLE;
10925 if (bnxt_rfs_capable(bp)) {
10926 bp->flags |= BNXT_FLAG_RFS;
10927 dev->features |= NETIF_F_NTUPLE;
10928 }
10929 }
10930
c0c050c5
MC		 10931 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
10932 bp->flags |= BNXT_FLAG_STRIP_VLAN;
10933
7809592d 10934 rc = bnxt_init_int_mode(bp);
c0c050c5 10935 if (rc)
17086399 10936 goto init_err_pci_clean;
c0c050c5 10937
832aed16
MC		 10938 /* No TC has been set yet and rings may have been trimmed due to
10939 * limited MSIX, so we re-initialize the TX rings per TC.
10940 */
10941 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10942
c1ef146a 10943 bnxt_get_wol_settings(bp);
d196ece7
MC		 10944 if (bp->flags & BNXT_FLAG_WOL_CAP)
10945 device_set_wakeup_enable(&pdev->dev, bp->wol);
10946 else
10947 device_set_wakeup_capable(&pdev->dev, false);
c1ef146a 10948
c3480a60
MC		 10949 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10950
74706afa
MC		 10951 bnxt_hwrm_coal_params_qcaps(bp);
10952
c213eae8
MC		 10953 if (BNXT_PF(bp)) {
10954 if (!bnxt_pf_wq) {
10955 bnxt_pf_wq =
10956 create_singlethread_workqueue("bnxt_pf_wq");
10957 if (!bnxt_pf_wq) {
10958 dev_err(&pdev->dev, "Unable to create workqueue.\n");
10959 goto init_err_pci_clean;
10960 }
10961 }
2ae7408f 10962 bnxt_init_tc(bp);
c213eae8 10963 }
2ae7408f 10964
7809592d
MC		 10965 rc = register_netdev(dev);
10966 if (rc)
2ae7408f 10967 goto init_err_cleanup_tc;
7809592d 10968
4ab0c6a8
SP		 10969 if (BNXT_PF(bp))
10970 bnxt_dl_register(bp);
10971
c0c050c5
MC		 10972 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
10973 board_info[ent->driver_data].name,
10974 (long)pci_resource_start(pdev, 0), dev->dev_addr);
af125b75 10975 pcie_print_link_status(pdev);
90c4f788 10976
c0c050c5
MC		 10977 return 0;
10978
2ae7408f
SP		 10979init_err_cleanup_tc:
10980 bnxt_shutdown_tc(bp);
7809592d
MC		 10981 bnxt_clear_int_mode(bp);
10982
17086399 10983init_err_pci_clean:
f9099d61 10984 bnxt_free_hwrm_short_cmd_req(bp);
a2bf74f4 10985 bnxt_free_hwrm_resources(bp);
98f04cf0
MC		 10986 bnxt_free_ctx_mem(bp);
10987 kfree(bp->ctx);
10988 bp->ctx = NULL;
17086399 10989 bnxt_cleanup_pci(bp);
c0c050c5
MC		 10990
10991init_err_free:
10992 free_netdev(dev);
10993 return rc;
10994}
10995
d196ece7
MC		 10996static void bnxt_shutdown(struct pci_dev *pdev)
10997{
10998 struct net_device *dev = pci_get_drvdata(pdev);
10999 struct bnxt *bp;
11000
11001 if (!dev)
11002 return;
11003
11004 rtnl_lock();
11005 bp = netdev_priv(dev);
11006 if (!bp)
11007 goto shutdown_exit;
11008
11009 if (netif_running(dev))
11010 dev_close(dev);
11011
a7f3f939
RJ		 11012 bnxt_ulp_shutdown(bp);
11013
d196ece7
MC		 11014 if (system_state == SYSTEM_POWER_OFF) {
11015 bnxt_clear_int_mode(bp);
c20dc142 11016 pci_disable_device(pdev);
d196ece7
MC		 11017 pci_wake_from_d3(pdev, bp->wol);
11018 pci_set_power_state(pdev, PCI_D3hot);
11019 }
11020
11021shutdown_exit:
11022 rtnl_unlock();
11023}
11024
f65a2044
MC		 11025#ifdef CONFIG_PM_SLEEP
11026static int bnxt_suspend(struct device *device)
11027{
f521eaa9 11028 struct net_device *dev = dev_get_drvdata(device);
f65a2044
MC		 11029 struct bnxt *bp = netdev_priv(dev);
11030 int rc = 0;
11031
11032 rtnl_lock();
11033 if (netif_running(dev)) {
11034 netif_device_detach(dev);
11035 rc = bnxt_close(dev);
11036 }
11037 bnxt_hwrm_func_drv_unrgtr(bp);
11038 rtnl_unlock();
11039 return rc;
11040}
11041
11042static int bnxt_resume(struct device *device)
11043{
f521eaa9 11044 struct net_device *dev = dev_get_drvdata(device);
f65a2044
MC		 11045 struct bnxt *bp = netdev_priv(dev);
11046 int rc = 0;
11047
11048 rtnl_lock();
11049 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
11050 rc = -ENODEV;
11051 goto resume_exit;
11052 }
11053 rc = bnxt_hwrm_func_reset(bp);
11054 if (rc) {
11055 rc = -EBUSY;
11056 goto resume_exit;
11057 }
11058 bnxt_get_wol_settings(bp);
11059 if (netif_running(dev)) {
11060 rc = bnxt_open(dev);
11061 if (!rc)
11062 netif_device_attach(dev);
11063 }
11064
11065resume_exit:
11066 rtnl_unlock();
11067 return rc;
11068}
11069
11070static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
11071#define BNXT_PM_OPS (&bnxt_pm_ops)
11072
11073#else
11074
11075#define BNXT_PM_OPS NULL
11076
11077#endif /* CONFIG_PM_SLEEP */
11078
6316ea6d
SB		 11079/**
11080 * bnxt_io_error_detected - called when PCI error is detected
11081 * @pdev: Pointer to PCI device
11082 * @state: The current pci connection state
11083 *
11084 * This function is called after a PCI bus error affecting
11085 * this device has been detected.
11086 */
11087static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
11088 pci_channel_state_t state)
11089{
11090 struct net_device *netdev = pci_get_drvdata(pdev);
a588e458 11091 struct bnxt *bp = netdev_priv(netdev);
6316ea6d
SB		 11092
11093 netdev_info(netdev, "PCI I/O error detected\n");
11094
11095 rtnl_lock();
11096 netif_device_detach(netdev);
11097
a588e458
MC		 11098 bnxt_ulp_stop(bp);
11099
6316ea6d
SB		 11100 if (state == pci_channel_io_perm_failure) {
11101 rtnl_unlock();
11102 return PCI_ERS_RESULT_DISCONNECT;
11103 }
11104
11105 if (netif_running(netdev))
11106 bnxt_close(netdev);
11107
11108 pci_disable_device(pdev);
11109 rtnl_unlock();
11110
11111 /* Request a slot slot reset. */
11112 return PCI_ERS_RESULT_NEED_RESET;
11113}
11114
11115/**
11116 * bnxt_io_slot_reset - called after the pci bus has been reset.
11117 * @pdev: Pointer to PCI device
11118 *
11119 * Restart the card from scratch, as if from a cold-boot.
11120 * At this point, the card has exprienced a hard reset,
11121 * followed by fixups by BIOS, and has its config space
11122 * set up identically to what it was at cold boot.
11123 */
11124static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
11125{
11126 struct net_device *netdev = pci_get_drvdata(pdev);
11127 struct bnxt *bp = netdev_priv(netdev);
11128 int err = 0;
11129 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
11130
11131 netdev_info(bp->dev, "PCI Slot Reset\n");
11132
11133 rtnl_lock();
11134
11135 if (pci_enable_device(pdev)) {
11136 dev_err(&pdev->dev,
11137 "Cannot re-enable PCI device after reset.\n");
11138 } else {
11139 pci_set_master(pdev);
11140
aa8ed021
MC		 11141 err = bnxt_hwrm_func_reset(bp);
11142 if (!err && netif_running(netdev))
6316ea6d
SB		 11143 err = bnxt_open(netdev);
11144
a588e458 11145 if (!err) {
6316ea6d 11146 result = PCI_ERS_RESULT_RECOVERED;
a588e458
MC		 11147 bnxt_ulp_start(bp);
11148 }
6316ea6d
SB		 11149 }
11150
11151 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
11152 dev_close(netdev);
11153
11154 rtnl_unlock();
11155
6316ea6d
SB		 11156 return PCI_ERS_RESULT_RECOVERED;
11157}
11158
11159/**
11160 * bnxt_io_resume - called when traffic can start flowing again.
11161 * @pdev: Pointer to PCI device
11162 *
11163 * This callback is called when the error recovery driver tells
11164 * us that its OK to resume normal operation.
11165 */
11166static void bnxt_io_resume(struct pci_dev *pdev)
11167{
11168 struct net_device *netdev = pci_get_drvdata(pdev);
11169
11170 rtnl_lock();
11171
11172 netif_device_attach(netdev);
11173
11174 rtnl_unlock();
11175}
11176
11177static const struct pci_error_handlers bnxt_err_handler = {
11178 .error_detected = bnxt_io_error_detected,
11179 .slot_reset = bnxt_io_slot_reset,
11180 .resume = bnxt_io_resume
11181};
11182
c0c050c5
MC		 11183static struct pci_driver bnxt_pci_driver = {
11184 .name = DRV_MODULE_NAME,
11185 .id_table = bnxt_pci_tbl,
11186 .probe = bnxt_init_one,
11187 .remove = bnxt_remove_one,
d196ece7 11188 .shutdown = bnxt_shutdown,
f65a2044 11189 .driver.pm = BNXT_PM_OPS,
6316ea6d 11190 .err_handler = &bnxt_err_handler,
c0c050c5
MC		 11191#if defined(CONFIG_BNXT_SRIOV)
11192 .sriov_configure = bnxt_sriov_configure,
11193#endif
11194};
11195
c213eae8
MC		 11196static int __init bnxt_init(void)
11197{
cabfb09d 11198 bnxt_debug_init();
c213eae8
MC		 11199 return pci_register_driver(&bnxt_pci_driver);
11200}
11201
11202static void __exit bnxt_exit(void)
11203{
11204 pci_unregister_driver(&bnxt_pci_driver);
11205 if (bnxt_pf_wq)
11206 destroy_workqueue(bnxt_pf_wq);
cabfb09d 11207 bnxt_debug_exit();
c213eae8
MC		 11208}
11209
11210module_init(bnxt_init);
11211module_exit(bnxt_exit);
Ubuntu Jammy Linux kernel mirror