]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/net/ethernet/broadcom/bnxt/bnxt.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnxt_en: Pre-map the firmware health monitoring registers.
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
CommitLineData
c0c050c5
MC		 1/* Broadcom NetXtreme-C/E network driver.
2 *
11f15ed3 3 * Copyright (c) 2014-2016 Broadcom Corporation
c6cc32a2 4 * Copyright (c) 2016-2019 Broadcom Limited
c0c050c5
MC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11#include <linux/module.h>
12
13#include <linux/stringify.h>
14#include <linux/kernel.h>
15#include <linux/timer.h>
16#include <linux/errno.h>
17#include <linux/ioport.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20#include <linux/interrupt.h>
21#include <linux/pci.h>
22#include <linux/netdevice.h>
23#include <linux/etherdevice.h>
24#include <linux/skbuff.h>
25#include <linux/dma-mapping.h>
26#include <linux/bitops.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
30#include <asm/byteorder.h>
31#include <asm/page.h>
32#include <linux/time.h>
33#include <linux/mii.h>
0ca12be9 34#include <linux/mdio.h>
c0c050c5
MC		 35#include <linux/if.h>
36#include <linux/if_vlan.h>
32e8239c 37#include <linux/if_bridge.h>
5ac67d8b 38#include <linux/rtc.h>
c6d30e83 39#include <linux/bpf.h>
c0c050c5
MC		 40#include <net/ip.h>
41#include <net/tcp.h>
42#include <net/udp.h>
43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
ad51b8e9 45#include <net/udp_tunnel.h>
c0c050c5
MC		 46#include <linux/workqueue.h>
47#include <linux/prefetch.h>
48#include <linux/cache.h>
49#include <linux/log2.h>
50#include <linux/aer.h>
51#include <linux/bitmap.h>
52#include <linux/cpu_rmap.h>
56f0fd80 53#include <linux/cpumask.h>
2ae7408f 54#include <net/pkt_cls.h>
cde49a42
VV		 55#include <linux/hwmon.h>
56#include <linux/hwmon-sysfs.h>
322b87ca 57#include <net/page_pool.h>
c0c050c5
MC		 58
59#include "bnxt_hsi.h"
60#include "bnxt.h"
a588e458 61#include "bnxt_ulp.h"
c0c050c5
MC		 62#include "bnxt_sriov.h"
63#include "bnxt_ethtool.h"
7df4ae9f 64#include "bnxt_dcb.h"
c6d30e83 65#include "bnxt_xdp.h"
4ab0c6a8 66#include "bnxt_vfr.h"
2ae7408f 67#include "bnxt_tc.h"
3c467bf3 68#include "bnxt_devlink.h"
cabfb09d 69#include "bnxt_debugfs.h"
c0c050c5
MC		 70
71#define BNXT_TX_TIMEOUT (5 * HZ)
72
73static const char version[] =
74 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
75
76MODULE_LICENSE("GPL");
77MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
78MODULE_VERSION(DRV_MODULE_VERSION);
79
80#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
81#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
82#define BNXT_RX_COPY_THRESH 256
83
4419dbe6 84#define BNXT_TX_PUSH_THRESH 164
c0c050c5
MC		 85
86enum board_idx {
fbc9a523 87 BCM57301,
c0c050c5
MC		 88 BCM57302,
89 BCM57304,
1f681688 90 BCM57417_NPAR,
fa853dda 91 BCM58700,
b24eb6ae
MC		 92 BCM57311,
93 BCM57312,
fbc9a523 94 BCM57402,
c0c050c5
MC		 95 BCM57404,
96 BCM57406,
1f681688
MC		 97 BCM57402_NPAR,
98 BCM57407,
b24eb6ae
MC		 99 BCM57412,
100 BCM57414,
101 BCM57416,
102 BCM57417,
1f681688 103 BCM57412_NPAR,
5049e33b 104 BCM57314,
1f681688
MC		 105 BCM57417_SFP,
106 BCM57416_SFP,
107 BCM57404_NPAR,
108 BCM57406_NPAR,
109 BCM57407_SFP,
adbc8305 110 BCM57407_NPAR,
1f681688
MC		 111 BCM57414_NPAR,
112 BCM57416_NPAR,
32b40798
DK		 113 BCM57452,
114 BCM57454,
92abef36 115 BCM5745x_NPAR,
1ab968d2 116 BCM57508,
c6cc32a2 117 BCM57504,
51fec80d 118 BCM57502,
49c98421
MC		 119 BCM57508_NPAR,
120 BCM57504_NPAR,
121 BCM57502_NPAR,
4a58139b 122 BCM58802,
8ed693b7 123 BCM58804,
4a58139b 124 BCM58808,
adbc8305
MC		 125 NETXTREME_E_VF,
126 NETXTREME_C_VF,
618784e3 127 NETXTREME_S_VF,
b16b6891 128 NETXTREME_E_P5_VF,
c0c050c5
MC		 129};
130
131/* indexed by enum above */
132static const struct {
133 char *name;
134} board_info[] = {
27573a7d
SB		 135 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
136 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
137 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
138 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
139 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
140 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
141 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
142 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
143 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
144 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
145 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
146 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
147 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
148 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
149 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
150 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
151 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
152 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
153 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
154 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
155 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
156 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
157 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
158 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
159 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
160 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
161 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
162 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
92abef36 163 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
1ab968d2 164 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
c6cc32a2 165 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
51fec80d 166 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
49c98421
MC		 167 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
168 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
169 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
27573a7d 170 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
8ed693b7 171 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
27573a7d
SB		 172 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
173 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
174 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
618784e3 175 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
b16b6891 176 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
c0c050c5
MC		 177};
178
179static const struct pci_device_id bnxt_pci_tbl[] = {
92abef36
VV		 180 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
181 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
4a58139b 182 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
adbc8305 183 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
fbc9a523 184 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
c0c050c5
MC		 185 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
186 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
1f681688 187 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
fa853dda 188 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
b24eb6ae
MC		 189 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
190 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
fbc9a523 191 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
c0c050c5
MC		 192 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
193 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
1f681688
MC		 194 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
195 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
b24eb6ae
MC		 196 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
197 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
198 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
199 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
1f681688 200 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
5049e33b 201 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
1f681688
MC		 202 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
203 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
204 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
205 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
206 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
adbc8305
MC		 207 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
208 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
1f681688 209 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
adbc8305 210 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
1f681688 211 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
adbc8305 212 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
4a58139b 213 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
32b40798 214 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
1ab968d2 215 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
c6cc32a2 216 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
51fec80d 217 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
49c98421
MC		 218 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
219 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
220 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
221 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
222 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
223 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
4a58139b 224 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
8ed693b7 225 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
c0c050c5 226#ifdef CONFIG_BNXT_SRIOV
c7ef35eb
DK		 227 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
228 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
adbc8305
MC		 229 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
230 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
231 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
232 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
233 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
234 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
51fec80d 235 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
b16b6891 236 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
618784e3 237 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
c0c050c5
MC		 238#endif
239 { 0 }
240};
241
242MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
243
244static const u16 bnxt_vf_req_snif[] = {
245 HWRM_FUNC_CFG,
91cdda40 246 HWRM_FUNC_VF_CFG,
c0c050c5
MC		 247 HWRM_PORT_PHY_QCFG,
248 HWRM_CFA_L2_FILTER_ALLOC,
249};
250
25be8623 251static const u16 bnxt_async_events_arr[] = {
87c374de
MC		 252 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
253 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
254 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
255 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
256 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
25be8623
MC		 257};
258
c213eae8
MC		 259static struct workqueue_struct *bnxt_pf_wq;
260
c0c050c5
MC		 261static bool bnxt_vf_pciid(enum board_idx idx)
262{
618784e3 263 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
b16b6891 264 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
c0c050c5
MC		 265}
266
267#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
268#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
269#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
270
c0c050c5
MC		 271#define BNXT_CP_DB_IRQ_DIS(db) \
272 writel(DB_CP_IRQ_DIS_FLAGS, db)
273
697197e5
MC		 274#define BNXT_DB_CQ(db, idx) \
275 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
276
277#define BNXT_DB_NQ_P5(db, idx) \
278 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
279
280#define BNXT_DB_CQ_ARM(db, idx) \
281 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
282
283#define BNXT_DB_NQ_ARM_P5(db, idx) \
284 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
285
286static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
287{
288 if (bp->flags & BNXT_FLAG_CHIP_P5)
289 BNXT_DB_NQ_P5(db, idx);
290 else
291 BNXT_DB_CQ(db, idx);
292}
293
294static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
295{
296 if (bp->flags & BNXT_FLAG_CHIP_P5)
297 BNXT_DB_NQ_ARM_P5(db, idx);
298 else
299 BNXT_DB_CQ_ARM(db, idx);
300}
301
302static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
303{
304 if (bp->flags & BNXT_FLAG_CHIP_P5)
305 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
306 db->doorbell);
307 else
308 BNXT_DB_CQ(db, idx);
309}
310
38413406 311const u16 bnxt_lhint_arr[] = {
c0c050c5
MC		 312 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
313 TX_BD_FLAGS_LHINT_512_TO_1023,
314 TX_BD_FLAGS_LHINT_1024_TO_2047,
315 TX_BD_FLAGS_LHINT_1024_TO_2047,
316 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
317 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
318 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
319 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
320 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
321 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
322 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
323 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
324 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
325 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
326 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
327 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
328 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
329 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
330 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
331};
332
ee5c7fb3
SP		 333static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
334{
335 struct metadata_dst *md_dst = skb_metadata_dst(skb);
336
337 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
338 return 0;
339
340 return md_dst->u.port_info.port_id;
341}
342
c0c050c5
MC		 343static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
344{
345 struct bnxt *bp = netdev_priv(dev);
346 struct tx_bd *txbd;
347 struct tx_bd_ext *txbd1;
348 struct netdev_queue *txq;
349 int i;
350 dma_addr_t mapping;
351 unsigned int length, pad = 0;
352 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
353 u16 prod, last_frag;
354 struct pci_dev *pdev = bp->pdev;
c0c050c5
MC		 355 struct bnxt_tx_ring_info *txr;
356 struct bnxt_sw_tx_bd *tx_buf;
357
358 i = skb_get_queue_mapping(skb);
359 if (unlikely(i >= bp->tx_nr_rings)) {
360 dev_kfree_skb_any(skb);
361 return NETDEV_TX_OK;
362 }
363
c0c050c5 364 txq = netdev_get_tx_queue(dev, i);
a960dec9 365 txr = &bp->tx_ring[bp->tx_ring_map[i]];
c0c050c5
MC		 366 prod = txr->tx_prod;
367
368 free_size = bnxt_tx_avail(bp, txr);
369 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
370 netif_tx_stop_queue(txq);
371 return NETDEV_TX_BUSY;
372 }
373
374 length = skb->len;
375 len = skb_headlen(skb);
376 last_frag = skb_shinfo(skb)->nr_frags;
377
378 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
379
380 txbd->tx_bd_opaque = prod;
381
382 tx_buf = &txr->tx_buf_ring[prod];
383 tx_buf->skb = skb;
384 tx_buf->nr_frags = last_frag;
385
386 vlan_tag_flags = 0;
ee5c7fb3 387 cfa_action = bnxt_xmit_get_cfa_action(skb);
c0c050c5
MC		 388 if (skb_vlan_tag_present(skb)) {
389 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
390 skb_vlan_tag_get(skb);
391 /* Currently supports 8021Q, 8021AD vlan offloads
392 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
393 */
394 if (skb->vlan_proto == htons(ETH_P_8021Q))
395 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
396 }
397
398 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
4419dbe6
MC		 399 struct tx_push_buffer *tx_push_buf = txr->tx_push;
400 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
401 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
697197e5 402 void __iomem *db = txr->tx_db.doorbell;
4419dbe6
MC		 403 void *pdata = tx_push_buf->data;
404 u64 *end;
405 int j, push_len;
c0c050c5
MC		 406
407 /* Set COAL_NOW to be ready quickly for the next push */
408 tx_push->tx_bd_len_flags_type =
409 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
410 TX_BD_TYPE_LONG_TX_BD |
411 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
412 TX_BD_FLAGS_COAL_NOW |
413 TX_BD_FLAGS_PACKET_END |
414 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
415
416 if (skb->ip_summed == CHECKSUM_PARTIAL)
417 tx_push1->tx_bd_hsize_lflags =
418 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
419 else
420 tx_push1->tx_bd_hsize_lflags = 0;
421
422 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 423 tx_push1->tx_bd_cfa_action =
424 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5 425
fbb0fa8b
MC		 426 end = pdata + length;
427 end = PTR_ALIGN(end, 8) - 1;
4419dbe6
MC		 428 *end = 0;
429
c0c050c5
MC		 430 skb_copy_from_linear_data(skb, pdata, len);
431 pdata += len;
432 for (j = 0; j < last_frag; j++) {
433 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
434 void *fptr;
435
436 fptr = skb_frag_address_safe(frag);
437 if (!fptr)
438 goto normal_tx;
439
440 memcpy(pdata, fptr, skb_frag_size(frag));
441 pdata += skb_frag_size(frag);
442 }
443
4419dbe6
MC		 444 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
445 txbd->tx_bd_haddr = txr->data_mapping;
c0c050c5
MC		 446 prod = NEXT_TX(prod);
447 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
448 memcpy(txbd, tx_push1, sizeof(*txbd));
449 prod = NEXT_TX(prod);
4419dbe6 450 tx_push->doorbell =
c0c050c5
MC		 451 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
452 txr->tx_prod = prod;
453
b9a8460a 454 tx_buf->is_push = 1;
c0c050c5 455 netdev_tx_sent_queue(txq, skb->len);
b9a8460a 456 wmb(); /* Sync is_push and byte queue before pushing data */
c0c050c5 457
4419dbe6
MC		 458 push_len = (length + sizeof(*tx_push) + 7) / 8;
459 if (push_len > 16) {
697197e5
MC		 460 __iowrite64_copy(db, tx_push_buf, 16);
461 __iowrite32_copy(db + 4, tx_push_buf + 1,
9d13744b 462 (push_len - 16) << 1);
4419dbe6 463 } else {
697197e5 464 __iowrite64_copy(db, tx_push_buf, push_len);
4419dbe6 465 }
c0c050c5 466
c0c050c5
MC		 467 goto tx_done;
468 }
469
470normal_tx:
471 if (length < BNXT_MIN_PKT_SIZE) {
472 pad = BNXT_MIN_PKT_SIZE - length;
473 if (skb_pad(skb, pad)) {
474 /* SKB already freed. */
475 tx_buf->skb = NULL;
476 return NETDEV_TX_OK;
477 }
478 length = BNXT_MIN_PKT_SIZE;
479 }
480
481 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
482
483 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
484 dev_kfree_skb_any(skb);
485 tx_buf->skb = NULL;
486 return NETDEV_TX_OK;
487 }
488
489 dma_unmap_addr_set(tx_buf, mapping, mapping);
490 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
491 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
492
493 txbd->tx_bd_haddr = cpu_to_le64(mapping);
494
495 prod = NEXT_TX(prod);
496 txbd1 = (struct tx_bd_ext *)
497 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
498
499 txbd1->tx_bd_hsize_lflags = 0;
500 if (skb_is_gso(skb)) {
501 u32 hdr_len;
502
503 if (skb->encapsulation)
504 hdr_len = skb_inner_network_offset(skb) +
505 skb_inner_network_header_len(skb) +
506 inner_tcp_hdrlen(skb);
507 else
508 hdr_len = skb_transport_offset(skb) +
509 tcp_hdrlen(skb);
510
511 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
512 TX_BD_FLAGS_T_IPID |
513 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
514 length = skb_shinfo(skb)->gso_size;
515 txbd1->tx_bd_mss = cpu_to_le32(length);
516 length += hdr_len;
517 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
518 txbd1->tx_bd_hsize_lflags =
519 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
520 txbd1->tx_bd_mss = 0;
521 }
522
523 length >>= 9;
2b3c6885
MC		 524 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
525 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
526 skb->len);
527 i = 0;
528 goto tx_dma_error;
529 }
c0c050c5
MC		 530 flags |= bnxt_lhint_arr[length];
531 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
532
533 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
ee5c7fb3
SP		 534 txbd1->tx_bd_cfa_action =
535 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
c0c050c5
MC		 536 for (i = 0; i < last_frag; i++) {
537 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
538
539 prod = NEXT_TX(prod);
540 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
541
542 len = skb_frag_size(frag);
543 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
544 DMA_TO_DEVICE);
545
546 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
547 goto tx_dma_error;
548
549 tx_buf = &txr->tx_buf_ring[prod];
550 dma_unmap_addr_set(tx_buf, mapping, mapping);
551
552 txbd->tx_bd_haddr = cpu_to_le64(mapping);
553
554 flags = len << TX_BD_LEN_SHIFT;
555 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
556 }
557
558 flags &= ~TX_BD_LEN;
559 txbd->tx_bd_len_flags_type =
560 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
561 TX_BD_FLAGS_PACKET_END);
562
563 netdev_tx_sent_queue(txq, skb->len);
564
565 /* Sync BD data before updating doorbell */
566 wmb();
567
568 prod = NEXT_TX(prod);
569 txr->tx_prod = prod;
570
6b16f9ee 571 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
697197e5 572 bnxt_db_write(bp, &txr->tx_db, prod);
c0c050c5
MC		 573
574tx_done:
575
c0c050c5 576 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
6b16f9ee 577 if (netdev_xmit_more() && !tx_buf->is_push)
697197e5 578 bnxt_db_write(bp, &txr->tx_db, prod);
4d172f21 579
c0c050c5
MC		 580 netif_tx_stop_queue(txq);
581
582 /* netif_tx_stop_queue() must be done before checking
583 * tx index in bnxt_tx_avail() below, because in
584 * bnxt_tx_int(), we update tx index before checking for
585 * netif_tx_queue_stopped().
586 */
587 smp_mb();
588 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
589 netif_tx_wake_queue(txq);
590 }
591 return NETDEV_TX_OK;
592
593tx_dma_error:
594 last_frag = i;
595
596 /* start back at beginning and unmap skb */
597 prod = txr->tx_prod;
598 tx_buf = &txr->tx_buf_ring[prod];
599 tx_buf->skb = NULL;
600 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
601 skb_headlen(skb), PCI_DMA_TODEVICE);
602 prod = NEXT_TX(prod);
603
604 /* unmap remaining mapped pages */
605 for (i = 0; i < last_frag; i++) {
606 prod = NEXT_TX(prod);
607 tx_buf = &txr->tx_buf_ring[prod];
608 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
609 skb_frag_size(&skb_shinfo(skb)->frags[i]),
610 PCI_DMA_TODEVICE);
611 }
612
613 dev_kfree_skb_any(skb);
614 return NETDEV_TX_OK;
615}
616
617static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
618{
b6ab4b01 619 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
a960dec9 620 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
c0c050c5
MC		 621 u16 cons = txr->tx_cons;
622 struct pci_dev *pdev = bp->pdev;
623 int i;
624 unsigned int tx_bytes = 0;
625
626 for (i = 0; i < nr_pkts; i++) {
627 struct bnxt_sw_tx_bd *tx_buf;
628 struct sk_buff *skb;
629 int j, last;
630
631 tx_buf = &txr->tx_buf_ring[cons];
632 cons = NEXT_TX(cons);
633 skb = tx_buf->skb;
634 tx_buf->skb = NULL;
635
636 if (tx_buf->is_push) {
637 tx_buf->is_push = 0;
638 goto next_tx_int;
639 }
640
641 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
642 skb_headlen(skb), PCI_DMA_TODEVICE);
643 last = tx_buf->nr_frags;
644
645 for (j = 0; j < last; j++) {
646 cons = NEXT_TX(cons);
647 tx_buf = &txr->tx_buf_ring[cons];
648 dma_unmap_page(
649 &pdev->dev,
650 dma_unmap_addr(tx_buf, mapping),
651 skb_frag_size(&skb_shinfo(skb)->frags[j]),
652 PCI_DMA_TODEVICE);
653 }
654
655next_tx_int:
656 cons = NEXT_TX(cons);
657
658 tx_bytes += skb->len;
659 dev_kfree_skb_any(skb);
660 }
661
662 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
663 txr->tx_cons = cons;
664
665 /* Need to make the tx_cons update visible to bnxt_start_xmit()
666 * before checking for netif_tx_queue_stopped(). Without the
667 * memory barrier, there is a small possibility that bnxt_start_xmit()
668 * will miss it and cause the queue to be stopped forever.
669 */
670 smp_mb();
671
672 if (unlikely(netif_tx_queue_stopped(txq)) &&
673 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
674 __netif_tx_lock(txq, smp_processor_id());
675 if (netif_tx_queue_stopped(txq) &&
676 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
677 txr->dev_state != BNXT_DEV_STATE_CLOSING)
678 netif_tx_wake_queue(txq);
679 __netif_tx_unlock(txq);
680 }
681}
682
c61fb99c 683static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
322b87ca 684 struct bnxt_rx_ring_info *rxr,
c61fb99c
MC		 685 gfp_t gfp)
686{
687 struct device *dev = &bp->pdev->dev;
688 struct page *page;
689
322b87ca 690 page = page_pool_dev_alloc_pages(rxr->page_pool);
c61fb99c
MC		 691 if (!page)
692 return NULL;
693
c519fe9a
SN		 694 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
695 DMA_ATTR_WEAK_ORDERING);
c61fb99c 696 if (dma_mapping_error(dev, *mapping)) {
322b87ca 697 page_pool_recycle_direct(rxr->page_pool, page);
c61fb99c
MC		 698 return NULL;
699 }
700 *mapping += bp->rx_dma_offset;
701 return page;
702}
703
c0c050c5
MC		 704static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
705 gfp_t gfp)
706{
707 u8 *data;
708 struct pci_dev *pdev = bp->pdev;
709
710 data = kmalloc(bp->rx_buf_size, gfp);
711 if (!data)
712 return NULL;
713
c519fe9a
SN		 714 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
715 bp->rx_buf_use_size, bp->rx_dir,
716 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 717
718 if (dma_mapping_error(&pdev->dev, *mapping)) {
719 kfree(data);
720 data = NULL;
721 }
722 return data;
723}
724
38413406
MC		 725int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
726 u16 prod, gfp_t gfp)
c0c050c5
MC		 727{
728 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
729 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
c0c050c5
MC		 730 dma_addr_t mapping;
731
c61fb99c 732 if (BNXT_RX_PAGE_MODE(bp)) {
322b87ca
AG		 733 struct page *page =
734 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
c0c050c5 735
c61fb99c
MC		 736 if (!page)
737 return -ENOMEM;
738
739 rx_buf->data = page;
740 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
741 } else {
742 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
743
744 if (!data)
745 return -ENOMEM;
746
747 rx_buf->data = data;
748 rx_buf->data_ptr = data + bp->rx_offset;
749 }
11cd119d 750 rx_buf->mapping = mapping;
c0c050c5
MC		 751
752 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
c0c050c5
MC		 753 return 0;
754}
755
c6d30e83 756void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
c0c050c5
MC		 757{
758 u16 prod = rxr->rx_prod;
759 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
760 struct rx_bd *cons_bd, *prod_bd;
761
762 prod_rx_buf = &rxr->rx_buf_ring[prod];
763 cons_rx_buf = &rxr->rx_buf_ring[cons];
764
765 prod_rx_buf->data = data;
6bb19474 766 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 767
11cd119d 768 prod_rx_buf->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 769
770 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
771 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
772
773 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
774}
775
776static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
777{
778 u16 next, max = rxr->rx_agg_bmap_size;
779
780 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
781 if (next >= max)
782 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
783 return next;
784}
785
786static inline int bnxt_alloc_rx_page(struct bnxt *bp,
787 struct bnxt_rx_ring_info *rxr,
788 u16 prod, gfp_t gfp)
789{
790 struct rx_bd *rxbd =
791 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
792 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
793 struct pci_dev *pdev = bp->pdev;
794 struct page *page;
795 dma_addr_t mapping;
796 u16 sw_prod = rxr->rx_sw_agg_prod;
89d0a06c 797 unsigned int offset = 0;
c0c050c5 798
89d0a06c
MC		 799 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
800 page = rxr->rx_page;
801 if (!page) {
802 page = alloc_page(gfp);
803 if (!page)
804 return -ENOMEM;
805 rxr->rx_page = page;
806 rxr->rx_page_offset = 0;
807 }
808 offset = rxr->rx_page_offset;
809 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
810 if (rxr->rx_page_offset == PAGE_SIZE)
811 rxr->rx_page = NULL;
812 else
813 get_page(page);
814 } else {
815 page = alloc_page(gfp);
816 if (!page)
817 return -ENOMEM;
818 }
c0c050c5 819
c519fe9a
SN		 820 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
821 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
822 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 823 if (dma_mapping_error(&pdev->dev, mapping)) {
824 __free_page(page);
825 return -EIO;
826 }
827
828 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
829 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
830
831 __set_bit(sw_prod, rxr->rx_agg_bmap);
832 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
833 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
834
835 rx_agg_buf->page = page;
89d0a06c 836 rx_agg_buf->offset = offset;
c0c050c5
MC		 837 rx_agg_buf->mapping = mapping;
838 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
839 rxbd->rx_bd_opaque = sw_prod;
840 return 0;
841}
842
4a228a3a
MC		 843static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
844 struct bnxt_cp_ring_info *cpr,
845 u16 cp_cons, u16 curr)
846{
847 struct rx_agg_cmp *agg;
848
849 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
850 agg = (struct rx_agg_cmp *)
851 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
852 return agg;
853}
854
bfcd8d79
MC		 855static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
856 struct bnxt_rx_ring_info *rxr,
857 u16 agg_id, u16 curr)
858{
859 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
860
861 return &tpa_info->agg_arr[curr];
862}
863
4a228a3a
MC		 864static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
865 u16 start, u32 agg_bufs, bool tpa)
c0c050c5 866{
e44758b7 867 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5 868 struct bnxt *bp = bnapi->bp;
b6ab4b01 869 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 870 u16 prod = rxr->rx_agg_prod;
871 u16 sw_prod = rxr->rx_sw_agg_prod;
bfcd8d79 872 bool p5_tpa = false;
c0c050c5
MC		 873 u32 i;
874
bfcd8d79
MC		 875 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
876 p5_tpa = true;
877
c0c050c5
MC		 878 for (i = 0; i < agg_bufs; i++) {
879 u16 cons;
880 struct rx_agg_cmp *agg;
881 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
882 struct rx_bd *prod_bd;
883 struct page *page;
884
bfcd8d79
MC		 885 if (p5_tpa)
886 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
887 else
888 agg = bnxt_get_agg(bp, cpr, idx, start + i);
c0c050c5
MC		 889 cons = agg->rx_agg_cmp_opaque;
890 __clear_bit(cons, rxr->rx_agg_bmap);
891
892 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
893 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
894
895 __set_bit(sw_prod, rxr->rx_agg_bmap);
896 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
897 cons_rx_buf = &rxr->rx_agg_ring[cons];
898
899 /* It is possible for sw_prod to be equal to cons, so
900 * set cons_rx_buf->page to NULL first.
901 */
902 page = cons_rx_buf->page;
903 cons_rx_buf->page = NULL;
904 prod_rx_buf->page = page;
89d0a06c 905 prod_rx_buf->offset = cons_rx_buf->offset;
c0c050c5
MC		 906
907 prod_rx_buf->mapping = cons_rx_buf->mapping;
908
909 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
910
911 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
912 prod_bd->rx_bd_opaque = sw_prod;
913
914 prod = NEXT_RX_AGG(prod);
915 sw_prod = NEXT_RX_AGG(sw_prod);
c0c050c5
MC		 916 }
917 rxr->rx_agg_prod = prod;
918 rxr->rx_sw_agg_prod = sw_prod;
919}
920
c61fb99c
MC		 921static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
922 struct bnxt_rx_ring_info *rxr,
923 u16 cons, void *data, u8 *data_ptr,
924 dma_addr_t dma_addr,
925 unsigned int offset_and_len)
926{
927 unsigned int payload = offset_and_len >> 16;
928 unsigned int len = offset_and_len & 0xffff;
d7840976 929 skb_frag_t *frag;
c61fb99c
MC		 930 struct page *page = data;
931 u16 prod = rxr->rx_prod;
932 struct sk_buff *skb;
933 int off, err;
934
935 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
936 if (unlikely(err)) {
937 bnxt_reuse_rx_data(rxr, cons, data);
938 return NULL;
939 }
940 dma_addr -= bp->rx_dma_offset;
c519fe9a
SN		 941 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
942 DMA_ATTR_WEAK_ORDERING);
c61fb99c
MC		 943
944 if (unlikely(!payload))
c43f1255 945 payload = eth_get_headlen(bp->dev, data_ptr, len);
c61fb99c
MC		 946
947 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
948 if (!skb) {
949 __free_page(page);
950 return NULL;
951 }
952
953 off = (void *)data_ptr - page_address(page);
954 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
955 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
956 payload + NET_IP_ALIGN);
957
958 frag = &skb_shinfo(skb)->frags[0];
959 skb_frag_size_sub(frag, payload);
b54c9d5b 960 skb_frag_off_add(frag, payload);
c61fb99c
MC		 961 skb->data_len -= payload;
962 skb->tail += payload;
963
964 return skb;
965}
966
c0c050c5
MC		 967static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
968 struct bnxt_rx_ring_info *rxr, u16 cons,
6bb19474
MC		 969 void *data, u8 *data_ptr,
970 dma_addr_t dma_addr,
971 unsigned int offset_and_len)
c0c050c5 972{
6bb19474 973 u16 prod = rxr->rx_prod;
c0c050c5 974 struct sk_buff *skb;
6bb19474 975 int err;
c0c050c5
MC		 976
977 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
978 if (unlikely(err)) {
979 bnxt_reuse_rx_data(rxr, cons, data);
980 return NULL;
981 }
982
983 skb = build_skb(data, 0);
c519fe9a
SN		 984 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
985 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 986 if (!skb) {
987 kfree(data);
988 return NULL;
989 }
990
b3dba77c 991 skb_reserve(skb, bp->rx_offset);
6bb19474 992 skb_put(skb, offset_and_len & 0xffff);
c0c050c5
MC		 993 return skb;
994}
995
e44758b7
MC		 996static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
997 struct bnxt_cp_ring_info *cpr,
4a228a3a
MC		 998 struct sk_buff *skb, u16 idx,
999 u32 agg_bufs, bool tpa)
c0c050c5 1000{
e44758b7 1001 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5 1002 struct pci_dev *pdev = bp->pdev;
b6ab4b01 1003 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 1004 u16 prod = rxr->rx_agg_prod;
bfcd8d79 1005 bool p5_tpa = false;
c0c050c5
MC		 1006 u32 i;
1007
bfcd8d79
MC		 1008 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1009 p5_tpa = true;
1010
c0c050c5
MC		 1011 for (i = 0; i < agg_bufs; i++) {
1012 u16 cons, frag_len;
1013 struct rx_agg_cmp *agg;
1014 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1015 struct page *page;
1016 dma_addr_t mapping;
1017
bfcd8d79
MC		 1018 if (p5_tpa)
1019 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1020 else
1021 agg = bnxt_get_agg(bp, cpr, idx, i);
c0c050c5
MC		 1022 cons = agg->rx_agg_cmp_opaque;
1023 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1024 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1025
1026 cons_rx_buf = &rxr->rx_agg_ring[cons];
89d0a06c
MC		 1027 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1028 cons_rx_buf->offset, frag_len);
c0c050c5
MC		 1029 __clear_bit(cons, rxr->rx_agg_bmap);
1030
1031 /* It is possible for bnxt_alloc_rx_page() to allocate
1032 * a sw_prod index that equals the cons index, so we
1033 * need to clear the cons entry now.
1034 */
11cd119d 1035 mapping = cons_rx_buf->mapping;
c0c050c5
MC		 1036 page = cons_rx_buf->page;
1037 cons_rx_buf->page = NULL;
1038
1039 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1040 struct skb_shared_info *shinfo;
1041 unsigned int nr_frags;
1042
1043 shinfo = skb_shinfo(skb);
1044 nr_frags = --shinfo->nr_frags;
1045 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1046
1047 dev_kfree_skb(skb);
1048
1049 cons_rx_buf->page = page;
1050
1051 /* Update prod since possibly some pages have been
1052 * allocated already.
1053 */
1054 rxr->rx_agg_prod = prod;
4a228a3a 1055 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
c0c050c5
MC		 1056 return NULL;
1057 }
1058
c519fe9a
SN		 1059 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1060 PCI_DMA_FROMDEVICE,
1061 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 1062
1063 skb->data_len += frag_len;
1064 skb->len += frag_len;
1065 skb->truesize += PAGE_SIZE;
1066
1067 prod = NEXT_RX_AGG(prod);
c0c050c5
MC		 1068 }
1069 rxr->rx_agg_prod = prod;
1070 return skb;
1071}
1072
1073static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1074 u8 agg_bufs, u32 *raw_cons)
1075{
1076 u16 last;
1077 struct rx_agg_cmp *agg;
1078
1079 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1080 last = RING_CMP(*raw_cons);
1081 agg = (struct rx_agg_cmp *)
1082 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1083 return RX_AGG_CMP_VALID(agg, *raw_cons);
1084}
1085
1086static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1087 unsigned int len,
1088 dma_addr_t mapping)
1089{
1090 struct bnxt *bp = bnapi->bp;
1091 struct pci_dev *pdev = bp->pdev;
1092 struct sk_buff *skb;
1093
1094 skb = napi_alloc_skb(&bnapi->napi, len);
1095 if (!skb)
1096 return NULL;
1097
745fc05c
MC		 1098 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1099 bp->rx_dir);
c0c050c5 1100
6bb19474
MC		 1101 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1102 len + NET_IP_ALIGN);
c0c050c5 1103
745fc05c
MC		 1104 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1105 bp->rx_dir);
c0c050c5
MC		 1106
1107 skb_put(skb, len);
1108 return skb;
1109}
1110
e44758b7 1111static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
fa7e2812
MC		 1112 u32 *raw_cons, void *cmp)
1113{
fa7e2812
MC		 1114 struct rx_cmp *rxcmp = cmp;
1115 u32 tmp_raw_cons = *raw_cons;
1116 u8 cmp_type, agg_bufs = 0;
1117
1118 cmp_type = RX_CMP_TYPE(rxcmp);
1119
1120 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1121 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1122 RX_CMP_AGG_BUFS) >>
1123 RX_CMP_AGG_BUFS_SHIFT;
1124 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1125 struct rx_tpa_end_cmp *tpa_end = cmp;
1126
bfcd8d79
MC		 1127 if (bp->flags & BNXT_FLAG_CHIP_P5)
1128 return 0;
1129
4a228a3a 1130 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
fa7e2812
MC		 1131 }
1132
1133 if (agg_bufs) {
1134 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1135 return -EBUSY;
1136 }
1137 *raw_cons = tmp_raw_cons;
1138 return 0;
1139}
1140
c213eae8
MC		 1141static void bnxt_queue_sp_work(struct bnxt *bp)
1142{
1143 if (BNXT_PF(bp))
1144 queue_work(bnxt_pf_wq, &bp->sp_task);
1145 else
1146 schedule_work(&bp->sp_task);
1147}
1148
1149static void bnxt_cancel_sp_work(struct bnxt *bp)
1150{
1151 if (BNXT_PF(bp))
1152 flush_workqueue(bnxt_pf_wq);
1153 else
1154 cancel_work_sync(&bp->sp_task);
1155}
1156
fa7e2812
MC		 1157static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1158{
1159 if (!rxr->bnapi->in_reset) {
1160 rxr->bnapi->in_reset = true;
1161 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 1162 bnxt_queue_sp_work(bp);
fa7e2812
MC		 1163 }
1164 rxr->rx_next_cons = 0xffff;
1165}
1166
ec4d8e7c
MC		 1167static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1168{
1169 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1170 u16 idx = agg_id & MAX_TPA_P5_MASK;
1171
1172 if (test_bit(idx, map->agg_idx_bmap))
1173 idx = find_first_zero_bit(map->agg_idx_bmap,
1174 BNXT_AGG_IDX_BMAP_SIZE);
1175 __set_bit(idx, map->agg_idx_bmap);
1176 map->agg_id_tbl[agg_id] = idx;
1177 return idx;
1178}
1179
1180static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1181{
1182 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1183
1184 __clear_bit(idx, map->agg_idx_bmap);
1185}
1186
1187static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1188{
1189 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1190
1191 return map->agg_id_tbl[agg_id];
1192}
1193
c0c050c5
MC		 1194static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1195 struct rx_tpa_start_cmp *tpa_start,
1196 struct rx_tpa_start_cmp_ext *tpa_start1)
1197{
c0c050c5 1198 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
bfcd8d79
MC		 1199 struct bnxt_tpa_info *tpa_info;
1200 u16 cons, prod, agg_id;
c0c050c5
MC		 1201 struct rx_bd *prod_bd;
1202 dma_addr_t mapping;
1203
ec4d8e7c 1204 if (bp->flags & BNXT_FLAG_CHIP_P5) {
bfcd8d79 1205 agg_id = TPA_START_AGG_ID_P5(tpa_start);
ec4d8e7c
MC		 1206 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1207 } else {
bfcd8d79 1208 agg_id = TPA_START_AGG_ID(tpa_start);
ec4d8e7c 1209 }
c0c050c5
MC		 1210 cons = tpa_start->rx_tpa_start_cmp_opaque;
1211 prod = rxr->rx_prod;
1212 cons_rx_buf = &rxr->rx_buf_ring[cons];
1213 prod_rx_buf = &rxr->rx_buf_ring[prod];
1214 tpa_info = &rxr->rx_tpa[agg_id];
1215
bfcd8d79
MC		 1216 if (unlikely(cons != rxr->rx_next_cons ||
1217 TPA_START_ERROR(tpa_start))) {
1218 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1219 cons, rxr->rx_next_cons,
1220 TPA_START_ERROR_CODE(tpa_start1));
fa7e2812
MC		 1221 bnxt_sched_reset(bp, rxr);
1222 return;
1223 }
ee5c7fb3
SP		 1224 /* Store cfa_code in tpa_info to use in tpa_end
1225 * completion processing.
1226 */
1227 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
c0c050c5 1228 prod_rx_buf->data = tpa_info->data;
6bb19474 1229 prod_rx_buf->data_ptr = tpa_info->data_ptr;
c0c050c5
MC		 1230
1231 mapping = tpa_info->mapping;
11cd119d 1232 prod_rx_buf->mapping = mapping;
c0c050c5
MC		 1233
1234 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1235
1236 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1237
1238 tpa_info->data = cons_rx_buf->data;
6bb19474 1239 tpa_info->data_ptr = cons_rx_buf->data_ptr;
c0c050c5 1240 cons_rx_buf->data = NULL;
11cd119d 1241 tpa_info->mapping = cons_rx_buf->mapping;
c0c050c5
MC		 1242
1243 tpa_info->len =
1244 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1245 RX_TPA_START_CMP_LEN_SHIFT;
1246 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1247 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1248
1249 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1250 tpa_info->gso_type = SKB_GSO_TCPV4;
1251 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
50f011b6 1252 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
c0c050c5
MC		 1253 tpa_info->gso_type = SKB_GSO_TCPV6;
1254 tpa_info->rss_hash =
1255 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1256 } else {
1257 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1258 tpa_info->gso_type = 0;
1259 if (netif_msg_rx_err(bp))
1260 netdev_warn(bp->dev, "TPA packet without valid hash\n");
1261 }
1262 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1263 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
94758f8d 1264 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
bfcd8d79 1265 tpa_info->agg_count = 0;
c0c050c5
MC		 1266
1267 rxr->rx_prod = NEXT_RX(prod);
1268 cons = NEXT_RX(cons);
376a5b86 1269 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 1270 cons_rx_buf = &rxr->rx_buf_ring[cons];
1271
1272 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1273 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1274 cons_rx_buf->data = NULL;
1275}
1276
4a228a3a 1277static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
c0c050c5
MC		 1278{
1279 if (agg_bufs)
4a228a3a 1280 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
c0c050c5
MC		 1281}
1282
bee5a188
MC		 1283#ifdef CONFIG_INET
1284static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1285{
1286 struct udphdr *uh = NULL;
1287
1288 if (ip_proto == htons(ETH_P_IP)) {
1289 struct iphdr *iph = (struct iphdr *)skb->data;
1290
1291 if (iph->protocol == IPPROTO_UDP)
1292 uh = (struct udphdr *)(iph + 1);
1293 } else {
1294 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1295
1296 if (iph->nexthdr == IPPROTO_UDP)
1297 uh = (struct udphdr *)(iph + 1);
1298 }
1299 if (uh) {
1300 if (uh->check)
1301 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1302 else
1303 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1304 }
1305}
1306#endif
1307
94758f8d
MC		 1308static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1309 int payload_off, int tcp_ts,
1310 struct sk_buff *skb)
1311{
1312#ifdef CONFIG_INET
1313 struct tcphdr *th;
1314 int len, nw_off;
1315 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1316 u32 hdr_info = tpa_info->hdr_info;
1317 bool loopback = false;
1318
1319 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1320 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1321 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1322
1323 /* If the packet is an internal loopback packet, the offsets will
1324 * have an extra 4 bytes.
1325 */
1326 if (inner_mac_off == 4) {
1327 loopback = true;
1328 } else if (inner_mac_off > 4) {
1329 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1330 ETH_HLEN - 2));
1331
1332 /* We only support inner iPv4/ipv6. If we don't see the
1333 * correct protocol ID, it must be a loopback packet where
1334 * the offsets are off by 4.
1335 */
09a7636a 1336 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
94758f8d
MC		 1337 loopback = true;
1338 }
1339 if (loopback) {
1340 /* internal loopback packet, subtract all offsets by 4 */
1341 inner_ip_off -= 4;
1342 inner_mac_off -= 4;
1343 outer_ip_off -= 4;
1344 }
1345
1346 nw_off = inner_ip_off - ETH_HLEN;
1347 skb_set_network_header(skb, nw_off);
1348 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1349 struct ipv6hdr *iph = ipv6_hdr(skb);
1350
1351 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1352 len = skb->len - skb_transport_offset(skb);
1353 th = tcp_hdr(skb);
1354 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1355 } else {
1356 struct iphdr *iph = ip_hdr(skb);
1357
1358 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1359 len = skb->len - skb_transport_offset(skb);
1360 th = tcp_hdr(skb);
1361 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1362 }
1363
1364 if (inner_mac_off) { /* tunnel */
94758f8d
MC		 1365 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1366 ETH_HLEN - 2));
1367
bee5a188 1368 bnxt_gro_tunnel(skb, proto);
94758f8d
MC		 1369 }
1370#endif
1371 return skb;
1372}
1373
67912c36
MC		 1374static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1375 int payload_off, int tcp_ts,
1376 struct sk_buff *skb)
1377{
1378#ifdef CONFIG_INET
1379 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1380 u32 hdr_info = tpa_info->hdr_info;
1381 int iphdr_len, nw_off;
1382
1383 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1384 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1385 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1386
1387 nw_off = inner_ip_off - ETH_HLEN;
1388 skb_set_network_header(skb, nw_off);
1389 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1390 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1391 skb_set_transport_header(skb, nw_off + iphdr_len);
1392
1393 if (inner_mac_off) { /* tunnel */
1394 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1395 ETH_HLEN - 2));
1396
1397 bnxt_gro_tunnel(skb, proto);
1398 }
1399#endif
1400 return skb;
1401}
1402
c0c050c5
MC		 1403#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1404#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1405
309369c9
MC		 1406static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1407 int payload_off, int tcp_ts,
c0c050c5
MC		 1408 struct sk_buff *skb)
1409{
d1611c3a 1410#ifdef CONFIG_INET
c0c050c5 1411 struct tcphdr *th;
719ca811 1412 int len, nw_off, tcp_opt_len = 0;
27e24189 1413
309369c9 1414 if (tcp_ts)
c0c050c5
MC		 1415 tcp_opt_len = 12;
1416
c0c050c5
MC		 1417 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1418 struct iphdr *iph;
1419
1420 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1421 ETH_HLEN;
1422 skb_set_network_header(skb, nw_off);
1423 iph = ip_hdr(skb);
1424 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1425 len = skb->len - skb_transport_offset(skb);
1426 th = tcp_hdr(skb);
1427 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1428 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1429 struct ipv6hdr *iph;
1430
1431 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1432 ETH_HLEN;
1433 skb_set_network_header(skb, nw_off);
1434 iph = ipv6_hdr(skb);
1435 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1436 len = skb->len - skb_transport_offset(skb);
1437 th = tcp_hdr(skb);
1438 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1439 } else {
1440 dev_kfree_skb_any(skb);
1441 return NULL;
1442 }
c0c050c5 1443
bee5a188
MC		 1444 if (nw_off) /* tunnel */
1445 bnxt_gro_tunnel(skb, skb->protocol);
c0c050c5
MC		 1446#endif
1447 return skb;
1448}
1449
309369c9
MC		 1450static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1451 struct bnxt_tpa_info *tpa_info,
1452 struct rx_tpa_end_cmp *tpa_end,
1453 struct rx_tpa_end_cmp_ext *tpa_end1,
1454 struct sk_buff *skb)
1455{
1456#ifdef CONFIG_INET
1457 int payload_off;
1458 u16 segs;
1459
1460 segs = TPA_END_TPA_SEGS(tpa_end);
1461 if (segs == 1)
1462 return skb;
1463
1464 NAPI_GRO_CB(skb)->count = segs;
1465 skb_shinfo(skb)->gso_size =
1466 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1467 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
bfcd8d79
MC		 1468 if (bp->flags & BNXT_FLAG_CHIP_P5)
1469 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1470 else
1471 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
309369c9 1472 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
5910906c
MC		 1473 if (likely(skb))
1474 tcp_gro_complete(skb);
309369c9
MC		 1475#endif
1476 return skb;
1477}
1478
ee5c7fb3
SP		 1479/* Given the cfa_code of a received packet determine which
1480 * netdev (vf-rep or PF) the packet is destined to.
1481 */
1482static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1483{
1484 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1485
1486 /* if vf-rep dev is NULL, the must belongs to the PF */
1487 return dev ? dev : bp->dev;
1488}
1489
c0c050c5 1490static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
e44758b7 1491 struct bnxt_cp_ring_info *cpr,
c0c050c5
MC		 1492 u32 *raw_cons,
1493 struct rx_tpa_end_cmp *tpa_end,
1494 struct rx_tpa_end_cmp_ext *tpa_end1,
4e5dbbda 1495 u8 *event)
c0c050c5 1496{
e44758b7 1497 struct bnxt_napi *bnapi = cpr->bnapi;
b6ab4b01 1498 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
6bb19474 1499 u8 *data_ptr, agg_bufs;
c0c050c5
MC		 1500 unsigned int len;
1501 struct bnxt_tpa_info *tpa_info;
1502 dma_addr_t mapping;
1503 struct sk_buff *skb;
bfcd8d79 1504 u16 idx = 0, agg_id;
6bb19474 1505 void *data;
bfcd8d79 1506 bool gro;
c0c050c5 1507
fa7e2812 1508 if (unlikely(bnapi->in_reset)) {
e44758b7 1509 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
fa7e2812
MC		 1510
1511 if (rc < 0)
1512 return ERR_PTR(-EBUSY);
1513 return NULL;
1514 }
1515
bfcd8d79
MC		 1516 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1517 agg_id = TPA_END_AGG_ID_P5(tpa_end);
ec4d8e7c 1518 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
bfcd8d79
MC		 1519 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1520 tpa_info = &rxr->rx_tpa[agg_id];
1521 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1522 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1523 agg_bufs, tpa_info->agg_count);
1524 agg_bufs = tpa_info->agg_count;
1525 }
1526 tpa_info->agg_count = 0;
1527 *event |= BNXT_AGG_EVENT;
ec4d8e7c 1528 bnxt_free_agg_idx(rxr, agg_id);
bfcd8d79
MC		 1529 idx = agg_id;
1530 gro = !!(bp->flags & BNXT_FLAG_GRO);
1531 } else {
1532 agg_id = TPA_END_AGG_ID(tpa_end);
1533 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1534 tpa_info = &rxr->rx_tpa[agg_id];
1535 idx = RING_CMP(*raw_cons);
1536 if (agg_bufs) {
1537 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1538 return ERR_PTR(-EBUSY);
1539
1540 *event |= BNXT_AGG_EVENT;
1541 idx = NEXT_CMP(idx);
1542 }
1543 gro = !!TPA_END_GRO(tpa_end);
1544 }
c0c050c5 1545 data = tpa_info->data;
6bb19474
MC		 1546 data_ptr = tpa_info->data_ptr;
1547 prefetch(data_ptr);
c0c050c5
MC		 1548 len = tpa_info->len;
1549 mapping = tpa_info->mapping;
1550
69c149e2 1551 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
4a228a3a 1552 bnxt_abort_tpa(cpr, idx, agg_bufs);
69c149e2
MC		 1553 if (agg_bufs > MAX_SKB_FRAGS)
1554 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1555 agg_bufs, (int)MAX_SKB_FRAGS);
c0c050c5
MC		 1556 return NULL;
1557 }
1558
1559 if (len <= bp->rx_copy_thresh) {
6bb19474 1560 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
c0c050c5 1561 if (!skb) {
4a228a3a 1562 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1563 return NULL;
1564 }
1565 } else {
1566 u8 *new_data;
1567 dma_addr_t new_mapping;
1568
1569 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1570 if (!new_data) {
4a228a3a 1571 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1572 return NULL;
1573 }
1574
1575 tpa_info->data = new_data;
b3dba77c 1576 tpa_info->data_ptr = new_data + bp->rx_offset;
c0c050c5
MC		 1577 tpa_info->mapping = new_mapping;
1578
1579 skb = build_skb(data, 0);
c519fe9a
SN		 1580 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1581 bp->rx_buf_use_size, bp->rx_dir,
1582 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 1583
1584 if (!skb) {
1585 kfree(data);
4a228a3a 1586 bnxt_abort_tpa(cpr, idx, agg_bufs);
c0c050c5
MC		 1587 return NULL;
1588 }
b3dba77c 1589 skb_reserve(skb, bp->rx_offset);
c0c050c5
MC		 1590 skb_put(skb, len);
1591 }
1592
1593 if (agg_bufs) {
4a228a3a 1594 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
c0c050c5
MC		 1595 if (!skb) {
1596 /* Page reuse already handled by bnxt_rx_pages(). */
1597 return NULL;
1598 }
1599 }
ee5c7fb3
SP		 1600
1601 skb->protocol =
1602 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
c0c050c5
MC		 1603
1604 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1605 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1606
8852ddb4
MC		 1607 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1608 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5
MC		 1609 u16 vlan_proto = tpa_info->metadata >>
1610 RX_CMP_FLAGS2_METADATA_TPID_SFT;
ed7bc602 1611 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
c0c050c5 1612
8852ddb4 1613 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1614 }
1615
1616 skb_checksum_none_assert(skb);
1617 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1618 skb->ip_summed = CHECKSUM_UNNECESSARY;
1619 skb->csum_level =
1620 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1621 }
1622
bfcd8d79 1623 if (gro)
309369c9 1624 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
c0c050c5
MC		 1625
1626 return skb;
1627}
1628
8fe88ce7
MC		 1629static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1630 struct rx_agg_cmp *rx_agg)
1631{
1632 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1633 struct bnxt_tpa_info *tpa_info;
1634
ec4d8e7c 1635 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
8fe88ce7
MC		 1636 tpa_info = &rxr->rx_tpa[agg_id];
1637 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1638 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1639}
1640
ee5c7fb3
SP		 1641static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1642 struct sk_buff *skb)
1643{
1644 if (skb->dev != bp->dev) {
1645 /* this packet belongs to a vf-rep */
1646 bnxt_vf_rep_rx(bp, skb);
1647 return;
1648 }
1649 skb_record_rx_queue(skb, bnapi->index);
1650 napi_gro_receive(&bnapi->napi, skb);
1651}
1652
c0c050c5
MC		 1653/* returns the following:
1654 * 1 - 1 packet successfully received
1655 * 0 - successful TPA_START, packet not completed yet
1656 * -EBUSY - completion ring does not have all the agg buffers yet
1657 * -ENOMEM - packet aborted due to out of memory
1658 * -EIO - packet aborted due to hw error indicated in BD
1659 */
e44758b7
MC		 1660static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1661 u32 *raw_cons, u8 *event)
c0c050c5 1662{
e44758b7 1663 struct bnxt_napi *bnapi = cpr->bnapi;
b6ab4b01 1664 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1665 struct net_device *dev = bp->dev;
1666 struct rx_cmp *rxcmp;
1667 struct rx_cmp_ext *rxcmp1;
1668 u32 tmp_raw_cons = *raw_cons;
ee5c7fb3 1669 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
c0c050c5
MC		 1670 struct bnxt_sw_rx_bd *rx_buf;
1671 unsigned int len;
6bb19474 1672 u8 *data_ptr, agg_bufs, cmp_type;
c0c050c5
MC		 1673 dma_addr_t dma_addr;
1674 struct sk_buff *skb;
6bb19474 1675 void *data;
c0c050c5 1676 int rc = 0;
c61fb99c 1677 u32 misc;
c0c050c5
MC		 1678
1679 rxcmp = (struct rx_cmp *)
1680 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1681
8fe88ce7
MC		 1682 cmp_type = RX_CMP_TYPE(rxcmp);
1683
1684 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1685 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1686 goto next_rx_no_prod_no_len;
1687 }
1688
c0c050c5
MC		 1689 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1690 cp_cons = RING_CMP(tmp_raw_cons);
1691 rxcmp1 = (struct rx_cmp_ext *)
1692 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1693
1694 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1695 return -EBUSY;
1696
c0c050c5
MC		 1697 prod = rxr->rx_prod;
1698
1699 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1700 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1701 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1702
4e5dbbda 1703 *event |= BNXT_RX_EVENT;
e7e70fa6 1704 goto next_rx_no_prod_no_len;
c0c050c5
MC		 1705
1706 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
e44758b7 1707 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
c0c050c5 1708 (struct rx_tpa_end_cmp *)rxcmp,
4e5dbbda 1709 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
c0c050c5 1710
1fac4b2f 1711 if (IS_ERR(skb))
c0c050c5
MC		 1712 return -EBUSY;
1713
1714 rc = -ENOMEM;
1715 if (likely(skb)) {
ee5c7fb3 1716 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1717 rc = 1;
1718 }
4e5dbbda 1719 *event |= BNXT_RX_EVENT;
e7e70fa6 1720 goto next_rx_no_prod_no_len;
c0c050c5
MC		 1721 }
1722
1723 cons = rxcmp->rx_cmp_opaque;
fa7e2812 1724 if (unlikely(cons != rxr->rx_next_cons)) {
e44758b7 1725 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
fa7e2812 1726
a1b0e4e6
MC		 1727 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1728 cons, rxr->rx_next_cons);
fa7e2812
MC		 1729 bnxt_sched_reset(bp, rxr);
1730 return rc1;
1731 }
a1b0e4e6
MC		 1732 rx_buf = &rxr->rx_buf_ring[cons];
1733 data = rx_buf->data;
1734 data_ptr = rx_buf->data_ptr;
6bb19474 1735 prefetch(data_ptr);
c0c050c5 1736
c61fb99c
MC		 1737 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1738 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
c0c050c5
MC		 1739
1740 if (agg_bufs) {
1741 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1742 return -EBUSY;
1743
1744 cp_cons = NEXT_CMP(cp_cons);
4e5dbbda 1745 *event |= BNXT_AGG_EVENT;
c0c050c5 1746 }
4e5dbbda 1747 *event |= BNXT_RX_EVENT;
c0c050c5
MC		 1748
1749 rx_buf->data = NULL;
1750 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
8e44e96c
MC		 1751 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1752
c0c050c5
MC		 1753 bnxt_reuse_rx_data(rxr, cons, data);
1754 if (agg_bufs)
4a228a3a
MC		 1755 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1756 false);
c0c050c5
MC		 1757
1758 rc = -EIO;
8e44e96c
MC		 1759 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1760 netdev_warn(bp->dev, "RX buffer error %x\n", rx_err);
1761 bnxt_sched_reset(bp, rxr);
1762 }
0b397b17 1763 goto next_rx_no_len;
c0c050c5
MC		 1764 }
1765
1766 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
11cd119d 1767 dma_addr = rx_buf->mapping;
c0c050c5 1768
c6d30e83
MC		 1769 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1770 rc = 1;
1771 goto next_rx;
1772 }
1773
c0c050c5 1774 if (len <= bp->rx_copy_thresh) {
6bb19474 1775 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
c0c050c5
MC		 1776 bnxt_reuse_rx_data(rxr, cons, data);
1777 if (!skb) {
296d5b54 1778 if (agg_bufs)
4a228a3a
MC		 1779 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1780 agg_bufs, false);
c0c050c5
MC		 1781 rc = -ENOMEM;
1782 goto next_rx;
1783 }
1784 } else {
c61fb99c
MC		 1785 u32 payload;
1786
c6d30e83
MC		 1787 if (rx_buf->data_ptr == data_ptr)
1788 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1789 else
1790 payload = 0;
6bb19474 1791 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
c61fb99c 1792 payload | len);
c0c050c5
MC		 1793 if (!skb) {
1794 rc = -ENOMEM;
1795 goto next_rx;
1796 }
1797 }
1798
1799 if (agg_bufs) {
4a228a3a 1800 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
c0c050c5
MC		 1801 if (!skb) {
1802 rc = -ENOMEM;
1803 goto next_rx;
1804 }
1805 }
1806
1807 if (RX_CMP_HASH_VALID(rxcmp)) {
1808 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1809 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1810
1811 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1812 if (hash_type != 1 && hash_type != 3)
1813 type = PKT_HASH_TYPE_L3;
1814 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1815 }
1816
ee5c7fb3
SP		 1817 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1818 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
c0c050c5 1819
8852ddb4
MC		 1820 if ((rxcmp1->rx_cmp_flags2 &
1821 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1822 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
c0c050c5 1823 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
ed7bc602 1824 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
c0c050c5
MC		 1825 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1826
8852ddb4 1827 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
c0c050c5
MC		 1828 }
1829
1830 skb_checksum_none_assert(skb);
1831 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1832 if (dev->features & NETIF_F_RXCSUM) {
1833 skb->ip_summed = CHECKSUM_UNNECESSARY;
1834 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1835 }
1836 } else {
665e350d
SB		 1837 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1838 if (dev->features & NETIF_F_RXCSUM)
d1981929 1839 bnapi->cp_ring.rx_l4_csum_errors++;
665e350d 1840 }
c0c050c5
MC		 1841 }
1842
ee5c7fb3 1843 bnxt_deliver_skb(bp, bnapi, skb);
c0c050c5
MC		 1844 rc = 1;
1845
1846next_rx:
6a8788f2
AG		 1847 cpr->rx_packets += 1;
1848 cpr->rx_bytes += len;
e7e70fa6 1849
0b397b17
MC		 1850next_rx_no_len:
1851 rxr->rx_prod = NEXT_RX(prod);
1852 rxr->rx_next_cons = NEXT_RX(cons);
1853
e7e70fa6 1854next_rx_no_prod_no_len:
c0c050c5
MC		 1855 *raw_cons = tmp_raw_cons;
1856
1857 return rc;
1858}
1859
2270bc5d
MC		 1860/* In netpoll mode, if we are using a combined completion ring, we need to
1861 * discard the rx packets and recycle the buffers.
1862 */
e44758b7
MC		 1863static int bnxt_force_rx_discard(struct bnxt *bp,
1864 struct bnxt_cp_ring_info *cpr,
2270bc5d
MC		 1865 u32 *raw_cons, u8 *event)
1866{
2270bc5d
MC		 1867 u32 tmp_raw_cons = *raw_cons;
1868 struct rx_cmp_ext *rxcmp1;
1869 struct rx_cmp *rxcmp;
1870 u16 cp_cons;
1871 u8 cmp_type;
1872
1873 cp_cons = RING_CMP(tmp_raw_cons);
1874 rxcmp = (struct rx_cmp *)
1875 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1876
1877 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1878 cp_cons = RING_CMP(tmp_raw_cons);
1879 rxcmp1 = (struct rx_cmp_ext *)
1880 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1881
1882 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1883 return -EBUSY;
1884
1885 cmp_type = RX_CMP_TYPE(rxcmp);
1886 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1887 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1888 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1889 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1890 struct rx_tpa_end_cmp_ext *tpa_end1;
1891
1892 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1893 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1894 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1895 }
e44758b7 1896 return bnxt_rx_pkt(bp, cpr, raw_cons, event);
2270bc5d
MC		 1897}
1898
4bb13abf 1899#define BNXT_GET_EVENT_PORT(data) \
87c374de
MC		 1900 ((data) & \
1901 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
4bb13abf 1902
c0c050c5
MC		 1903static int bnxt_async_event_process(struct bnxt *bp,
1904 struct hwrm_async_event_cmpl *cmpl)
1905{
1906 u16 event_id = le16_to_cpu(cmpl->event_id);
1907
1908 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1909 switch (event_id) {
87c374de 1910 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
8cbde117
MC		 1911 u32 data1 = le32_to_cpu(cmpl->event_data1);
1912 struct bnxt_link_info *link_info = &bp->link_info;
1913
1914 if (BNXT_VF(bp))
1915 goto async_event_process_exit;
a8168b6c
MC		 1916
1917 /* print unsupported speed warning in forced speed mode only */
1918 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1919 (data1 & 0x20000)) {
8cbde117
MC		 1920 u16 fw_speed = link_info->force_link_speed;
1921 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1922
a8168b6c
MC		 1923 if (speed != SPEED_UNKNOWN)
1924 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1925 speed);
8cbde117 1926 }
286ef9d6 1927 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
8cbde117 1928 }
bc171e87 1929 /* fall through */
87c374de 1930 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
c0c050c5 1931 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
19241368 1932 break;
87c374de 1933 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
19241368 1934 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
c0c050c5 1935 break;
87c374de 1936 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
4bb13abf
MC		 1937 u32 data1 = le32_to_cpu(cmpl->event_data1);
1938 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1939
1940 if (BNXT_VF(bp))
1941 break;
1942
1943 if (bp->pf.port_id != port_id)
1944 break;
1945
4bb13abf
MC		 1946 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1947 break;
1948 }
87c374de 1949 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
fc0f1929
MC		 1950 if (BNXT_PF(bp))
1951 goto async_event_process_exit;
1952 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1953 break;
c0c050c5 1954 default:
19241368 1955 goto async_event_process_exit;
c0c050c5 1956 }
c213eae8 1957 bnxt_queue_sp_work(bp);
19241368 1958async_event_process_exit:
a588e458 1959 bnxt_ulp_async_events(bp, cmpl);
c0c050c5
MC		 1960 return 0;
1961}
1962
1963static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1964{
1965 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1966 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1967 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1968 (struct hwrm_fwd_req_cmpl *)txcmp;
1969
1970 switch (cmpl_type) {
1971 case CMPL_BASE_TYPE_HWRM_DONE:
1972 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1973 if (seq_id == bp->hwrm_intr_seq_id)
fc718bb2 1974 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
c0c050c5
MC		 1975 else
1976 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1977 break;
1978
1979 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1980 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1981
1982 if ((vf_id < bp->pf.first_vf_id) ||
1983 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1984 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1985 vf_id);
1986 return -EINVAL;
1987 }
1988
1989 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1990 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
c213eae8 1991 bnxt_queue_sp_work(bp);
c0c050c5
MC		 1992 break;
1993
1994 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1995 bnxt_async_event_process(bp,
1996 (struct hwrm_async_event_cmpl *)txcmp);
1997
1998 default:
1999 break;
2000 }
2001
2002 return 0;
2003}
2004
2005static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2006{
2007 struct bnxt_napi *bnapi = dev_instance;
2008 struct bnxt *bp = bnapi->bp;
2009 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2010 u32 cons = RING_CMP(cpr->cp_raw_cons);
2011
6a8788f2 2012 cpr->event_ctr++;
c0c050c5
MC		 2013 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2014 napi_schedule(&bnapi->napi);
2015 return IRQ_HANDLED;
2016}
2017
2018static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2019{
2020 u32 raw_cons = cpr->cp_raw_cons;
2021 u16 cons = RING_CMP(raw_cons);
2022 struct tx_cmp *txcmp;
2023
2024 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2025
2026 return TX_CMP_VALID(txcmp, raw_cons);
2027}
2028
c0c050c5
MC		 2029static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2030{
2031 struct bnxt_napi *bnapi = dev_instance;
2032 struct bnxt *bp = bnapi->bp;
2033 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2034 u32 cons = RING_CMP(cpr->cp_raw_cons);
2035 u32 int_status;
2036
2037 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2038
2039 if (!bnxt_has_work(bp, cpr)) {
11809490 2040 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
c0c050c5
MC		 2041 /* return if erroneous interrupt */
2042 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2043 return IRQ_NONE;
2044 }
2045
2046 /* disable ring IRQ */
697197e5 2047 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
c0c050c5
MC		 2048
2049 /* Return here if interrupt is shared and is disabled. */
2050 if (unlikely(atomic_read(&bp->intr_sem) != 0))
2051 return IRQ_HANDLED;
2052
2053 napi_schedule(&bnapi->napi);
2054 return IRQ_HANDLED;
2055}
2056
3675b92f
MC		 2057static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2058 int budget)
c0c050c5 2059{
e44758b7 2060 struct bnxt_napi *bnapi = cpr->bnapi;
c0c050c5
MC		 2061 u32 raw_cons = cpr->cp_raw_cons;
2062 u32 cons;
2063 int tx_pkts = 0;
2064 int rx_pkts = 0;
4e5dbbda 2065 u8 event = 0;
c0c050c5
MC		 2066 struct tx_cmp *txcmp;
2067
0fcec985 2068 cpr->has_more_work = 0;
c0c050c5
MC		 2069 while (1) {
2070 int rc;
2071
2072 cons = RING_CMP(raw_cons);
2073 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2074
2075 if (!TX_CMP_VALID(txcmp, raw_cons))
2076 break;
2077
67a95e20
MC		 2078 /* The valid test of the entry must be done first before
2079 * reading any further.
2080 */
b67daab0 2081 dma_rmb();
3675b92f 2082 cpr->had_work_done = 1;
c0c050c5
MC		 2083 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2084 tx_pkts++;
2085 /* return full budget so NAPI will complete. */
73f21c65 2086 if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
c0c050c5 2087 rx_pkts = budget;
73f21c65 2088 raw_cons = NEXT_RAW_CMP(raw_cons);
0fcec985
MC		 2089 if (budget)
2090 cpr->has_more_work = 1;
73f21c65
MC		 2091 break;
2092 }
c0c050c5 2093 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2270bc5d 2094 if (likely(budget))
e44758b7 2095 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2270bc5d 2096 else
e44758b7 2097 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2270bc5d 2098 &event);
c0c050c5
MC		 2099 if (likely(rc >= 0))
2100 rx_pkts += rc;
903649e7
MC		 2101 /* Increment rx_pkts when rc is -ENOMEM to count towards
2102 * the NAPI budget. Otherwise, we may potentially loop
2103 * here forever if we consistently cannot allocate
2104 * buffers.
2105 */
2edbdb31 2106 else if (rc == -ENOMEM && budget)
903649e7 2107 rx_pkts++;
c0c050c5
MC		 2108 else if (rc == -EBUSY) /* partial completion */
2109 break;
c0c050c5
MC		 2110 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2111 CMPL_BASE_TYPE_HWRM_DONE) ||
2112 (TX_CMP_TYPE(txcmp) ==
2113 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2114 (TX_CMP_TYPE(txcmp) ==
2115 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2116 bnxt_hwrm_handler(bp, txcmp);
2117 }
2118 raw_cons = NEXT_RAW_CMP(raw_cons);
2119
0fcec985
MC		 2120 if (rx_pkts && rx_pkts == budget) {
2121 cpr->has_more_work = 1;
c0c050c5 2122 break;
0fcec985 2123 }
c0c050c5
MC		 2124 }
2125
f18c2b77
AG		 2126 if (event & BNXT_REDIRECT_EVENT)
2127 xdp_do_flush_map();
2128
38413406
MC		 2129 if (event & BNXT_TX_EVENT) {
2130 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
38413406
MC		 2131 u16 prod = txr->tx_prod;
2132
2133 /* Sync BD data before updating doorbell */
2134 wmb();
2135
697197e5 2136 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
38413406
MC		 2137 }
2138
c0c050c5 2139 cpr->cp_raw_cons = raw_cons;
3675b92f
MC		 2140 bnapi->tx_pkts += tx_pkts;
2141 bnapi->events |= event;
2142 return rx_pkts;
2143}
c0c050c5 2144
3675b92f
MC		 2145static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2146{
2147 if (bnapi->tx_pkts) {
2148 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2149 bnapi->tx_pkts = 0;
2150 }
c0c050c5 2151
3675b92f 2152 if (bnapi->events & BNXT_RX_EVENT) {
b6ab4b01 2153 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5 2154
3675b92f 2155 if (bnapi->events & BNXT_AGG_EVENT)
697197e5 2156 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
e8f267b0 2157 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
c0c050c5 2158 }
3675b92f
MC		 2159 bnapi->events = 0;
2160}
2161
2162static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2163 int budget)
2164{
2165 struct bnxt_napi *bnapi = cpr->bnapi;
2166 int rx_pkts;
2167
2168 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2169
2170 /* ACK completion ring before freeing tx ring and producing new
2171 * buffers in rx/agg rings to prevent overflowing the completion
2172 * ring.
2173 */
2174 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2175
2176 __bnxt_poll_work_done(bp, bnapi);
c0c050c5
MC		 2177 return rx_pkts;
2178}
2179
10bbdaf5
PS		 2180static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2181{
2182 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2183 struct bnxt *bp = bnapi->bp;
2184 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2185 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2186 struct tx_cmp *txcmp;
2187 struct rx_cmp_ext *rxcmp1;
2188 u32 cp_cons, tmp_raw_cons;
2189 u32 raw_cons = cpr->cp_raw_cons;
2190 u32 rx_pkts = 0;
4e5dbbda 2191 u8 event = 0;
10bbdaf5
PS		 2192
2193 while (1) {
2194 int rc;
2195
2196 cp_cons = RING_CMP(raw_cons);
2197 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2198
2199 if (!TX_CMP_VALID(txcmp, raw_cons))
2200 break;
2201
2202 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2203 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2204 cp_cons = RING_CMP(tmp_raw_cons);
2205 rxcmp1 = (struct rx_cmp_ext *)
2206 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2207
2208 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2209 break;
2210
2211 /* force an error to recycle the buffer */
2212 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2213 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2214
e44758b7 2215 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2edbdb31 2216 if (likely(rc == -EIO) && budget)
10bbdaf5
PS		 2217 rx_pkts++;
2218 else if (rc == -EBUSY) /* partial completion */
2219 break;
2220 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2221 CMPL_BASE_TYPE_HWRM_DONE)) {
2222 bnxt_hwrm_handler(bp, txcmp);
2223 } else {
2224 netdev_err(bp->dev,
2225 "Invalid completion received on special ring\n");
2226 }
2227 raw_cons = NEXT_RAW_CMP(raw_cons);
2228
2229 if (rx_pkts == budget)
2230 break;
2231 }
2232
2233 cpr->cp_raw_cons = raw_cons;
697197e5
MC		 2234 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2235 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10bbdaf5 2236
434c975a 2237 if (event & BNXT_AGG_EVENT)
697197e5 2238 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10bbdaf5
PS		 2239
2240 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
6ad20165 2241 napi_complete_done(napi, rx_pkts);
697197e5 2242 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
10bbdaf5
PS		 2243 }
2244 return rx_pkts;
2245}
2246
c0c050c5
MC		 2247static int bnxt_poll(struct napi_struct *napi, int budget)
2248{
2249 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2250 struct bnxt *bp = bnapi->bp;
2251 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2252 int work_done = 0;
2253
c0c050c5 2254 while (1) {
e44758b7 2255 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
c0c050c5 2256
73f21c65
MC		 2257 if (work_done >= budget) {
2258 if (!budget)
697197e5 2259 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
c0c050c5 2260 break;
73f21c65 2261 }
c0c050c5
MC		 2262
2263 if (!bnxt_has_work(bp, cpr)) {
e7b95691 2264 if (napi_complete_done(napi, work_done))
697197e5 2265 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
c0c050c5
MC		 2266 break;
2267 }
2268 }
6a8788f2 2269 if (bp->flags & BNXT_FLAG_DIM) {
f06d0ca4 2270 struct dim_sample dim_sample = {};
6a8788f2 2271
8960b389
TG		 2272 dim_update_sample(cpr->event_ctr,
2273 cpr->rx_packets,
2274 cpr->rx_bytes,
2275 &dim_sample);
6a8788f2
AG		 2276 net_dim(&cpr->dim, dim_sample);
2277 }
c0c050c5
MC		 2278 return work_done;
2279}
2280
0fcec985
MC		 2281static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2282{
2283 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2284 int i, work_done = 0;
2285
2286 for (i = 0; i < 2; i++) {
2287 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2288
2289 if (cpr2) {
2290 work_done += __bnxt_poll_work(bp, cpr2,
2291 budget - work_done);
2292 cpr->has_more_work |= cpr2->has_more_work;
2293 }
2294 }
2295 return work_done;
2296}
2297
2298static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2299 u64 dbr_type, bool all)
2300{
2301 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2302 int i;
2303
2304 for (i = 0; i < 2; i++) {
2305 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2306 struct bnxt_db_info *db;
2307
2308 if (cpr2 && (all || cpr2->had_work_done)) {
2309 db = &cpr2->cp_db;
2310 writeq(db->db_key64 | dbr_type |
2311 RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2312 cpr2->had_work_done = 0;
2313 }
2314 }
2315 __bnxt_poll_work_done(bp, bnapi);
2316}
2317
2318static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2319{
2320 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2321 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2322 u32 raw_cons = cpr->cp_raw_cons;
2323 struct bnxt *bp = bnapi->bp;
2324 struct nqe_cn *nqcmp;
2325 int work_done = 0;
2326 u32 cons;
2327
2328 if (cpr->has_more_work) {
2329 cpr->has_more_work = 0;
2330 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2331 if (cpr->has_more_work) {
2332 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2333 return work_done;
2334 }
2335 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2336 if (napi_complete_done(napi, work_done))
2337 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2338 return work_done;
2339 }
2340 while (1) {
2341 cons = RING_CMP(raw_cons);
2342 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2343
2344 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2345 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2346 false);
2347 cpr->cp_raw_cons = raw_cons;
2348 if (napi_complete_done(napi, work_done))
2349 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2350 cpr->cp_raw_cons);
2351 return work_done;
2352 }
2353
2354 /* The valid test of the entry must be done first before
2355 * reading any further.
2356 */
2357 dma_rmb();
2358
2359 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2360 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2361 struct bnxt_cp_ring_info *cpr2;
2362
2363 cpr2 = cpr->cp_ring_arr[idx];
2364 work_done += __bnxt_poll_work(bp, cpr2,
2365 budget - work_done);
2366 cpr->has_more_work = cpr2->has_more_work;
2367 } else {
2368 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2369 }
2370 raw_cons = NEXT_RAW_CMP(raw_cons);
2371 if (cpr->has_more_work)
2372 break;
2373 }
2374 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2375 cpr->cp_raw_cons = raw_cons;
2376 return work_done;
2377}
2378
c0c050c5
MC		 2379static void bnxt_free_tx_skbs(struct bnxt *bp)
2380{
2381 int i, max_idx;
2382 struct pci_dev *pdev = bp->pdev;
2383
b6ab4b01 2384 if (!bp->tx_ring)
c0c050c5
MC		 2385 return;
2386
2387 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2388 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2389 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2390 int j;
2391
c0c050c5
MC		 2392 for (j = 0; j < max_idx;) {
2393 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
f18c2b77 2394 struct sk_buff *skb;
c0c050c5
MC		 2395 int k, last;
2396
f18c2b77
AG		 2397 if (i < bp->tx_nr_rings_xdp &&
2398 tx_buf->action == XDP_REDIRECT) {
2399 dma_unmap_single(&pdev->dev,
2400 dma_unmap_addr(tx_buf, mapping),
2401 dma_unmap_len(tx_buf, len),
2402 PCI_DMA_TODEVICE);
2403 xdp_return_frame(tx_buf->xdpf);
2404 tx_buf->action = 0;
2405 tx_buf->xdpf = NULL;
2406 j++;
2407 continue;
2408 }
2409
2410 skb = tx_buf->skb;
c0c050c5
MC		 2411 if (!skb) {
2412 j++;
2413 continue;
2414 }
2415
2416 tx_buf->skb = NULL;
2417
2418 if (tx_buf->is_push) {
2419 dev_kfree_skb(skb);
2420 j += 2;
2421 continue;
2422 }
2423
2424 dma_unmap_single(&pdev->dev,
2425 dma_unmap_addr(tx_buf, mapping),
2426 skb_headlen(skb),
2427 PCI_DMA_TODEVICE);
2428
2429 last = tx_buf->nr_frags;
2430 j += 2;
d612a579
MC		 2431 for (k = 0; k < last; k++, j++) {
2432 int ring_idx = j & bp->tx_ring_mask;
c0c050c5
MC		 2433 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2434
d612a579 2435 tx_buf = &txr->tx_buf_ring[ring_idx];
c0c050c5
MC		 2436 dma_unmap_page(
2437 &pdev->dev,
2438 dma_unmap_addr(tx_buf, mapping),
2439 skb_frag_size(frag), PCI_DMA_TODEVICE);
2440 }
2441 dev_kfree_skb(skb);
2442 }
2443 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2444 }
2445}
2446
2447static void bnxt_free_rx_skbs(struct bnxt *bp)
2448{
2449 int i, max_idx, max_agg_idx;
2450 struct pci_dev *pdev = bp->pdev;
2451
b6ab4b01 2452 if (!bp->rx_ring)
c0c050c5
MC		 2453 return;
2454
2455 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2456 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2457 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2458 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
ec4d8e7c 2459 struct bnxt_tpa_idx_map *map;
c0c050c5
MC		 2460 int j;
2461
c0c050c5 2462 if (rxr->rx_tpa) {
79632e9b 2463 for (j = 0; j < bp->max_tpa; j++) {
c0c050c5
MC		 2464 struct bnxt_tpa_info *tpa_info =
2465 &rxr->rx_tpa[j];
2466 u8 *data = tpa_info->data;
2467
2468 if (!data)
2469 continue;
2470
c519fe9a
SN		 2471 dma_unmap_single_attrs(&pdev->dev,
2472 tpa_info->mapping,
2473 bp->rx_buf_use_size,
2474 bp->rx_dir,
2475 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2476
2477 tpa_info->data = NULL;
2478
2479 kfree(data);
2480 }
2481 }
2482
2483 for (j = 0; j < max_idx; j++) {
2484 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
3ed3a83e 2485 dma_addr_t mapping = rx_buf->mapping;
6bb19474 2486 void *data = rx_buf->data;
c0c050c5
MC		 2487
2488 if (!data)
2489 continue;
2490
c0c050c5
MC		 2491 rx_buf->data = NULL;
2492
3ed3a83e
MC		 2493 if (BNXT_RX_PAGE_MODE(bp)) {
2494 mapping -= bp->rx_dma_offset;
c519fe9a
SN		 2495 dma_unmap_page_attrs(&pdev->dev, mapping,
2496 PAGE_SIZE, bp->rx_dir,
2497 DMA_ATTR_WEAK_ORDERING);
322b87ca 2498 page_pool_recycle_direct(rxr->page_pool, data);
3ed3a83e 2499 } else {
c519fe9a
SN		 2500 dma_unmap_single_attrs(&pdev->dev, mapping,
2501 bp->rx_buf_use_size,
2502 bp->rx_dir,
2503 DMA_ATTR_WEAK_ORDERING);
c61fb99c 2504 kfree(data);
3ed3a83e 2505 }
c0c050c5
MC		 2506 }
2507
2508 for (j = 0; j < max_agg_idx; j++) {
2509 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2510 &rxr->rx_agg_ring[j];
2511 struct page *page = rx_agg_buf->page;
2512
2513 if (!page)
2514 continue;
2515
c519fe9a
SN		 2516 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2517 BNXT_RX_PAGE_SIZE,
2518 PCI_DMA_FROMDEVICE,
2519 DMA_ATTR_WEAK_ORDERING);
c0c050c5
MC		 2520
2521 rx_agg_buf->page = NULL;
2522 __clear_bit(j, rxr->rx_agg_bmap);
2523
2524 __free_page(page);
2525 }
89d0a06c
MC		 2526 if (rxr->rx_page) {
2527 __free_page(rxr->rx_page);
2528 rxr->rx_page = NULL;
2529 }
ec4d8e7c
MC		 2530 map = rxr->rx_tpa_idx_map;
2531 if (map)
2532 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
c0c050c5
MC		 2533 }
2534}
2535
2536static void bnxt_free_skbs(struct bnxt *bp)
2537{
2538 bnxt_free_tx_skbs(bp);
2539 bnxt_free_rx_skbs(bp);
2540}
2541
6fe19886 2542static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
c0c050c5
MC		 2543{
2544 struct pci_dev *pdev = bp->pdev;
2545 int i;
2546
6fe19886
MC		 2547 for (i = 0; i < rmem->nr_pages; i++) {
2548 if (!rmem->pg_arr[i])
c0c050c5
MC		 2549 continue;
2550
6fe19886
MC		 2551 dma_free_coherent(&pdev->dev, rmem->page_size,
2552 rmem->pg_arr[i], rmem->dma_arr[i]);
c0c050c5 2553
6fe19886 2554 rmem->pg_arr[i] = NULL;
c0c050c5 2555 }
6fe19886 2556 if (rmem->pg_tbl) {
4f49b2b8
MC		 2557 size_t pg_tbl_size = rmem->nr_pages * 8;
2558
2559 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2560 pg_tbl_size = rmem->page_size;
2561 dma_free_coherent(&pdev->dev, pg_tbl_size,
6fe19886
MC		 2562 rmem->pg_tbl, rmem->pg_tbl_map);
2563 rmem->pg_tbl = NULL;
c0c050c5 2564 }
6fe19886
MC		 2565 if (rmem->vmem_size && *rmem->vmem) {
2566 vfree(*rmem->vmem);
2567 *rmem->vmem = NULL;
c0c050c5
MC		 2568 }
2569}
2570
6fe19886 2571static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
c0c050c5 2572{
c0c050c5 2573 struct pci_dev *pdev = bp->pdev;
66cca20a 2574 u64 valid_bit = 0;
6fe19886 2575 int i;
c0c050c5 2576
66cca20a
MC		 2577 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2578 valid_bit = PTU_PTE_VALID;
4f49b2b8
MC		 2579 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2580 size_t pg_tbl_size = rmem->nr_pages * 8;
2581
2582 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2583 pg_tbl_size = rmem->page_size;
2584 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
6fe19886 2585 &rmem->pg_tbl_map,
c0c050c5 2586 GFP_KERNEL);
6fe19886 2587 if (!rmem->pg_tbl)
c0c050c5
MC		 2588 return -ENOMEM;
2589 }
2590
6fe19886 2591 for (i = 0; i < rmem->nr_pages; i++) {
66cca20a
MC		 2592 u64 extra_bits = valid_bit;
2593
6fe19886
MC		 2594 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2595 rmem->page_size,
2596 &rmem->dma_arr[i],
c0c050c5 2597 GFP_KERNEL);
6fe19886 2598 if (!rmem->pg_arr[i])
c0c050c5
MC		 2599 return -ENOMEM;
2600
4f49b2b8 2601 if (rmem->nr_pages > 1 || rmem->depth > 0) {
66cca20a
MC		 2602 if (i == rmem->nr_pages - 2 &&
2603 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2604 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2605 else if (i == rmem->nr_pages - 1 &&
2606 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2607 extra_bits |= PTU_PTE_LAST;
2608 rmem->pg_tbl[i] =
2609 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2610 }
c0c050c5
MC		 2611 }
2612
6fe19886
MC		 2613 if (rmem->vmem_size) {
2614 *rmem->vmem = vzalloc(rmem->vmem_size);
2615 if (!(*rmem->vmem))
c0c050c5
MC		 2616 return -ENOMEM;
2617 }
2618 return 0;
2619}
2620
4a228a3a
MC		 2621static void bnxt_free_tpa_info(struct bnxt *bp)
2622{
2623 int i;
2624
2625 for (i = 0; i < bp->rx_nr_rings; i++) {
2626 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2627
ec4d8e7c
MC		 2628 kfree(rxr->rx_tpa_idx_map);
2629 rxr->rx_tpa_idx_map = NULL;
79632e9b
MC		 2630 if (rxr->rx_tpa) {
2631 kfree(rxr->rx_tpa[0].agg_arr);
2632 rxr->rx_tpa[0].agg_arr = NULL;
2633 }
4a228a3a
MC		 2634 kfree(rxr->rx_tpa);
2635 rxr->rx_tpa = NULL;
2636 }
2637}
2638
2639static int bnxt_alloc_tpa_info(struct bnxt *bp)
2640{
79632e9b
MC		 2641 int i, j, total_aggs = 0;
2642
2643 bp->max_tpa = MAX_TPA;
2644 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2645 if (!bp->max_tpa_v2)
2646 return 0;
2647 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2648 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2649 }
4a228a3a
MC		 2650
2651 for (i = 0; i < bp->rx_nr_rings; i++) {
2652 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
79632e9b 2653 struct rx_agg_cmp *agg;
4a228a3a 2654
79632e9b 2655 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
4a228a3a
MC		 2656 GFP_KERNEL);
2657 if (!rxr->rx_tpa)
2658 return -ENOMEM;
79632e9b
MC		 2659
2660 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2661 continue;
2662 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2663 rxr->rx_tpa[0].agg_arr = agg;
2664 if (!agg)
2665 return -ENOMEM;
2666 for (j = 1; j < bp->max_tpa; j++)
2667 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
ec4d8e7c
MC		 2668 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2669 GFP_KERNEL);
2670 if (!rxr->rx_tpa_idx_map)
2671 return -ENOMEM;
4a228a3a
MC		 2672 }
2673 return 0;
2674}
2675
c0c050c5
MC		 2676static void bnxt_free_rx_rings(struct bnxt *bp)
2677{
2678 int i;
2679
b6ab4b01 2680 if (!bp->rx_ring)
c0c050c5
MC		 2681 return;
2682
4a228a3a 2683 bnxt_free_tpa_info(bp);
c0c050c5 2684 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2685 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2686 struct bnxt_ring_struct *ring;
2687
c6d30e83
MC		 2688 if (rxr->xdp_prog)
2689 bpf_prog_put(rxr->xdp_prog);
2690
96a8604f
JDB		 2691 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2692 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2693
12479f62 2694 page_pool_destroy(rxr->page_pool);
322b87ca
AG		 2695 rxr->page_pool = NULL;
2696
c0c050c5
MC		 2697 kfree(rxr->rx_agg_bmap);
2698 rxr->rx_agg_bmap = NULL;
2699
2700 ring = &rxr->rx_ring_struct;
6fe19886 2701 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2702
2703 ring = &rxr->rx_agg_ring_struct;
6fe19886 2704 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2705 }
2706}
2707
322b87ca
AG		 2708static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2709 struct bnxt_rx_ring_info *rxr)
2710{
2711 struct page_pool_params pp = { 0 };
2712
2713 pp.pool_size = bp->rx_ring_size;
2714 pp.nid = dev_to_node(&bp->pdev->dev);
2715 pp.dev = &bp->pdev->dev;
2716 pp.dma_dir = DMA_BIDIRECTIONAL;
2717
2718 rxr->page_pool = page_pool_create(&pp);
2719 if (IS_ERR(rxr->page_pool)) {
2720 int err = PTR_ERR(rxr->page_pool);
2721
2722 rxr->page_pool = NULL;
2723 return err;
2724 }
2725 return 0;
2726}
2727
c0c050c5
MC		 2728static int bnxt_alloc_rx_rings(struct bnxt *bp)
2729{
4a228a3a 2730 int i, rc = 0, agg_rings = 0;
c0c050c5 2731
b6ab4b01
MC		 2732 if (!bp->rx_ring)
2733 return -ENOMEM;
2734
c0c050c5
MC		 2735 if (bp->flags & BNXT_FLAG_AGG_RINGS)
2736 agg_rings = 1;
2737
c0c050c5 2738 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 2739 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 2740 struct bnxt_ring_struct *ring;
2741
c0c050c5
MC		 2742 ring = &rxr->rx_ring_struct;
2743
322b87ca
AG		 2744 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2745 if (rc)
2746 return rc;
2747
96a8604f 2748 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
12479f62 2749 if (rc < 0)
96a8604f
JDB		 2750 return rc;
2751
f18c2b77 2752 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
322b87ca
AG		 2753 MEM_TYPE_PAGE_POOL,
2754 rxr->page_pool);
f18c2b77
AG		 2755 if (rc) {
2756 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2757 return rc;
2758 }
2759
6fe19886 2760 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2761 if (rc)
2762 return rc;
2763
2c61d211 2764 ring->grp_idx = i;
c0c050c5
MC		 2765 if (agg_rings) {
2766 u16 mem_size;
2767
2768 ring = &rxr->rx_agg_ring_struct;
6fe19886 2769 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2770 if (rc)
2771 return rc;
2772
9899bb59 2773 ring->grp_idx = i;
c0c050c5
MC		 2774 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2775 mem_size = rxr->rx_agg_bmap_size / 8;
2776 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2777 if (!rxr->rx_agg_bmap)
2778 return -ENOMEM;
c0c050c5
MC		 2779 }
2780 }
4a228a3a
MC		 2781 if (bp->flags & BNXT_FLAG_TPA)
2782 rc = bnxt_alloc_tpa_info(bp);
2783 return rc;
c0c050c5
MC		 2784}
2785
2786static void bnxt_free_tx_rings(struct bnxt *bp)
2787{
2788 int i;
2789 struct pci_dev *pdev = bp->pdev;
2790
b6ab4b01 2791 if (!bp->tx_ring)
c0c050c5
MC		 2792 return;
2793
2794 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2795 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2796 struct bnxt_ring_struct *ring;
2797
c0c050c5
MC		 2798 if (txr->tx_push) {
2799 dma_free_coherent(&pdev->dev, bp->tx_push_size,
2800 txr->tx_push, txr->tx_push_mapping);
2801 txr->tx_push = NULL;
2802 }
2803
2804 ring = &txr->tx_ring_struct;
2805
6fe19886 2806 bnxt_free_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2807 }
2808}
2809
2810static int bnxt_alloc_tx_rings(struct bnxt *bp)
2811{
2812 int i, j, rc;
2813 struct pci_dev *pdev = bp->pdev;
2814
2815 bp->tx_push_size = 0;
2816 if (bp->tx_push_thresh) {
2817 int push_size;
2818
2819 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2820 bp->tx_push_thresh);
2821
4419dbe6 2822 if (push_size > 256) {
c0c050c5
MC		 2823 push_size = 0;
2824 bp->tx_push_thresh = 0;
2825 }
2826
2827 bp->tx_push_size = push_size;
2828 }
2829
2830 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2831 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5 2832 struct bnxt_ring_struct *ring;
2e8ef77e 2833 u8 qidx;
c0c050c5 2834
c0c050c5
MC		 2835 ring = &txr->tx_ring_struct;
2836
6fe19886 2837 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2838 if (rc)
2839 return rc;
2840
9899bb59 2841 ring->grp_idx = txr->bnapi->index;
c0c050c5 2842 if (bp->tx_push_size) {
c0c050c5
MC		 2843 dma_addr_t mapping;
2844
2845 /* One pre-allocated DMA buffer to backup
2846 * TX push operation
2847 */
2848 txr->tx_push = dma_alloc_coherent(&pdev->dev,
2849 bp->tx_push_size,
2850 &txr->tx_push_mapping,
2851 GFP_KERNEL);
2852
2853 if (!txr->tx_push)
2854 return -ENOMEM;
2855
c0c050c5
MC		 2856 mapping = txr->tx_push_mapping +
2857 sizeof(struct tx_push_bd);
4419dbe6 2858 txr->data_mapping = cpu_to_le64(mapping);
c0c050c5 2859 }
2e8ef77e
MC		 2860 qidx = bp->tc_to_qidx[j];
2861 ring->queue_id = bp->q_info[qidx].queue_id;
5f449249
MC		 2862 if (i < bp->tx_nr_rings_xdp)
2863 continue;
c0c050c5
MC		 2864 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2865 j++;
2866 }
2867 return 0;
2868}
2869
2870static void bnxt_free_cp_rings(struct bnxt *bp)
2871{
2872 int i;
2873
2874 if (!bp->bnapi)
2875 return;
2876
2877 for (i = 0; i < bp->cp_nr_rings; i++) {
2878 struct bnxt_napi *bnapi = bp->bnapi[i];
2879 struct bnxt_cp_ring_info *cpr;
2880 struct bnxt_ring_struct *ring;
50e3ab78 2881 int j;
c0c050c5
MC		 2882
2883 if (!bnapi)
2884 continue;
2885
2886 cpr = &bnapi->cp_ring;
2887 ring = &cpr->cp_ring_struct;
2888
6fe19886 2889 bnxt_free_ring(bp, &ring->ring_mem);
50e3ab78
MC		 2890
2891 for (j = 0; j < 2; j++) {
2892 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2893
2894 if (cpr2) {
2895 ring = &cpr2->cp_ring_struct;
2896 bnxt_free_ring(bp, &ring->ring_mem);
2897 kfree(cpr2);
2898 cpr->cp_ring_arr[j] = NULL;
2899 }
2900 }
c0c050c5
MC		 2901 }
2902}
2903
50e3ab78
MC		 2904static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2905{
2906 struct bnxt_ring_mem_info *rmem;
2907 struct bnxt_ring_struct *ring;
2908 struct bnxt_cp_ring_info *cpr;
2909 int rc;
2910
2911 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
2912 if (!cpr)
2913 return NULL;
2914
2915 ring = &cpr->cp_ring_struct;
2916 rmem = &ring->ring_mem;
2917 rmem->nr_pages = bp->cp_nr_pages;
2918 rmem->page_size = HW_CMPD_RING_SIZE;
2919 rmem->pg_arr = (void **)cpr->cp_desc_ring;
2920 rmem->dma_arr = cpr->cp_desc_mapping;
2921 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
2922 rc = bnxt_alloc_ring(bp, rmem);
2923 if (rc) {
2924 bnxt_free_ring(bp, rmem);
2925 kfree(cpr);
2926 cpr = NULL;
2927 }
2928 return cpr;
2929}
2930
c0c050c5
MC		 2931static int bnxt_alloc_cp_rings(struct bnxt *bp)
2932{
50e3ab78 2933 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
e5811b8c 2934 int i, rc, ulp_base_vec, ulp_msix;
c0c050c5 2935
e5811b8c
MC		 2936 ulp_msix = bnxt_get_ulp_msix_num(bp);
2937 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
c0c050c5
MC		 2938 for (i = 0; i < bp->cp_nr_rings; i++) {
2939 struct bnxt_napi *bnapi = bp->bnapi[i];
2940 struct bnxt_cp_ring_info *cpr;
2941 struct bnxt_ring_struct *ring;
2942
2943 if (!bnapi)
2944 continue;
2945
2946 cpr = &bnapi->cp_ring;
50e3ab78 2947 cpr->bnapi = bnapi;
c0c050c5
MC		 2948 ring = &cpr->cp_ring_struct;
2949
6fe19886 2950 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
c0c050c5
MC		 2951 if (rc)
2952 return rc;
e5811b8c
MC		 2953
2954 if (ulp_msix && i >= ulp_base_vec)
2955 ring->map_idx = i + ulp_msix;
2956 else
2957 ring->map_idx = i;
50e3ab78
MC		 2958
2959 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2960 continue;
2961
2962 if (i < bp->rx_nr_rings) {
2963 struct bnxt_cp_ring_info *cpr2 =
2964 bnxt_alloc_cp_sub_ring(bp);
2965
2966 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
2967 if (!cpr2)
2968 return -ENOMEM;
2969 cpr2->bnapi = bnapi;
2970 }
2971 if ((sh && i < bp->tx_nr_rings) ||
2972 (!sh && i >= bp->rx_nr_rings)) {
2973 struct bnxt_cp_ring_info *cpr2 =
2974 bnxt_alloc_cp_sub_ring(bp);
2975
2976 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
2977 if (!cpr2)
2978 return -ENOMEM;
2979 cpr2->bnapi = bnapi;
2980 }
c0c050c5
MC		 2981 }
2982 return 0;
2983}
2984
2985static void bnxt_init_ring_struct(struct bnxt *bp)
2986{
2987 int i;
2988
2989 for (i = 0; i < bp->cp_nr_rings; i++) {
2990 struct bnxt_napi *bnapi = bp->bnapi[i];
6fe19886 2991 struct bnxt_ring_mem_info *rmem;
c0c050c5
MC		 2992 struct bnxt_cp_ring_info *cpr;
2993 struct bnxt_rx_ring_info *rxr;
2994 struct bnxt_tx_ring_info *txr;
2995 struct bnxt_ring_struct *ring;
2996
2997 if (!bnapi)
2998 continue;
2999
3000 cpr = &bnapi->cp_ring;
3001 ring = &cpr->cp_ring_struct;
6fe19886
MC		 3002 rmem = &ring->ring_mem;
3003 rmem->nr_pages = bp->cp_nr_pages;
3004 rmem->page_size = HW_CMPD_RING_SIZE;
3005 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3006 rmem->dma_arr = cpr->cp_desc_mapping;
3007 rmem->vmem_size = 0;
c0c050c5 3008
b6ab4b01 3009 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 3010 if (!rxr)
3011 goto skip_rx;
3012
c0c050c5 3013 ring = &rxr->rx_ring_struct;
6fe19886
MC		 3014 rmem = &ring->ring_mem;
3015 rmem->nr_pages = bp->rx_nr_pages;
3016 rmem->page_size = HW_RXBD_RING_SIZE;
3017 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3018 rmem->dma_arr = rxr->rx_desc_mapping;
3019 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3020 rmem->vmem = (void **)&rxr->rx_buf_ring;
c0c050c5
MC		 3021
3022 ring = &rxr->rx_agg_ring_struct;
6fe19886
MC		 3023 rmem = &ring->ring_mem;
3024 rmem->nr_pages = bp->rx_agg_nr_pages;
3025 rmem->page_size = HW_RXBD_RING_SIZE;
3026 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3027 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3028 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3029 rmem->vmem = (void **)&rxr->rx_agg_ring;
c0c050c5 3030
3b2b7d9d 3031skip_rx:
b6ab4b01 3032 txr = bnapi->tx_ring;
3b2b7d9d
MC		 3033 if (!txr)
3034 continue;
3035
c0c050c5 3036 ring = &txr->tx_ring_struct;
6fe19886
MC		 3037 rmem = &ring->ring_mem;
3038 rmem->nr_pages = bp->tx_nr_pages;
3039 rmem->page_size = HW_RXBD_RING_SIZE;
3040 rmem->pg_arr = (void **)txr->tx_desc_ring;
3041 rmem->dma_arr = txr->tx_desc_mapping;
3042 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3043 rmem->vmem = (void **)&txr->tx_buf_ring;
c0c050c5
MC		 3044 }
3045}
3046
3047static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3048{
3049 int i;
3050 u32 prod;
3051 struct rx_bd **rx_buf_ring;
3052
6fe19886
MC		 3053 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3054 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
c0c050c5
MC		 3055 int j;
3056 struct rx_bd *rxbd;
3057
3058 rxbd = rx_buf_ring[i];
3059 if (!rxbd)
3060 continue;
3061
3062 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3063 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3064 rxbd->rx_bd_opaque = prod;
3065 }
3066 }
3067}
3068
3069static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3070{
3071 struct net_device *dev = bp->dev;
c0c050c5
MC		 3072 struct bnxt_rx_ring_info *rxr;
3073 struct bnxt_ring_struct *ring;
3074 u32 prod, type;
3075 int i;
3076
c0c050c5
MC		 3077 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3078 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3079
3080 if (NET_IP_ALIGN == 2)
3081 type |= RX_BD_FLAGS_SOP;
3082
b6ab4b01 3083 rxr = &bp->rx_ring[ring_nr];
c0c050c5
MC		 3084 ring = &rxr->rx_ring_struct;
3085 bnxt_init_rxbd_pages(ring, type);
3086
c6d30e83
MC		 3087 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3088 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
3089 if (IS_ERR(rxr->xdp_prog)) {
3090 int rc = PTR_ERR(rxr->xdp_prog);
3091
3092 rxr->xdp_prog = NULL;
3093 return rc;
3094 }
3095 }
c0c050c5
MC		 3096 prod = rxr->rx_prod;
3097 for (i = 0; i < bp->rx_ring_size; i++) {
3098 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3099 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3100 ring_nr, i, bp->rx_ring_size);
3101 break;
3102 }
3103 prod = NEXT_RX(prod);
3104 }
3105 rxr->rx_prod = prod;
3106 ring->fw_ring_id = INVALID_HW_RING_ID;
3107
edd0c2cc
MC		 3108 ring = &rxr->rx_agg_ring_struct;
3109 ring->fw_ring_id = INVALID_HW_RING_ID;
3110
c0c050c5
MC		 3111 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3112 return 0;
3113
2839f28b 3114 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
c0c050c5
MC		 3115 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3116
3117 bnxt_init_rxbd_pages(ring, type);
3118
3119 prod = rxr->rx_agg_prod;
3120 for (i = 0; i < bp->rx_agg_ring_size; i++) {
3121 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3122 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3123 ring_nr, i, bp->rx_ring_size);
3124 break;
3125 }
3126 prod = NEXT_RX_AGG(prod);
3127 }
3128 rxr->rx_agg_prod = prod;
c0c050c5
MC		 3129
3130 if (bp->flags & BNXT_FLAG_TPA) {
3131 if (rxr->rx_tpa) {
3132 u8 *data;
3133 dma_addr_t mapping;
3134
79632e9b 3135 for (i = 0; i < bp->max_tpa; i++) {
c0c050c5
MC		 3136 data = __bnxt_alloc_rx_data(bp, &mapping,
3137 GFP_KERNEL);
3138 if (!data)
3139 return -ENOMEM;
3140
3141 rxr->rx_tpa[i].data = data;
b3dba77c 3142 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
c0c050c5
MC		 3143 rxr->rx_tpa[i].mapping = mapping;
3144 }
3145 } else {
3146 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3147 return -ENOMEM;
3148 }
3149 }
3150
3151 return 0;
3152}
3153
2247925f
SP		 3154static void bnxt_init_cp_rings(struct bnxt *bp)
3155{
3e08b184 3156 int i, j;
2247925f
SP		 3157
3158 for (i = 0; i < bp->cp_nr_rings; i++) {
3159 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3160 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3161
3162 ring->fw_ring_id = INVALID_HW_RING_ID;
6a8788f2
AG		 3163 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3164 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3e08b184
MC		 3165 for (j = 0; j < 2; j++) {
3166 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3167
3168 if (!cpr2)
3169 continue;
3170
3171 ring = &cpr2->cp_ring_struct;
3172 ring->fw_ring_id = INVALID_HW_RING_ID;
3173 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3174 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3175 }
2247925f
SP		 3176 }
3177}
3178
c0c050c5
MC		 3179static int bnxt_init_rx_rings(struct bnxt *bp)
3180{
3181 int i, rc = 0;
3182
c61fb99c 3183 if (BNXT_RX_PAGE_MODE(bp)) {
c6d30e83
MC		 3184 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3185 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
c61fb99c
MC		 3186 } else {
3187 bp->rx_offset = BNXT_RX_OFFSET;
3188 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3189 }
b3dba77c 3190
c0c050c5
MC		 3191 for (i = 0; i < bp->rx_nr_rings; i++) {
3192 rc = bnxt_init_one_rx_ring(bp, i);
3193 if (rc)
3194 break;
3195 }
3196
3197 return rc;
3198}
3199
3200static int bnxt_init_tx_rings(struct bnxt *bp)
3201{
3202 u16 i;
3203
3204 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3205 MAX_SKB_FRAGS + 1);
3206
3207 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 3208 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 3209 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3210
3211 ring->fw_ring_id = INVALID_HW_RING_ID;
3212 }
3213
3214 return 0;
3215}
3216
3217static void bnxt_free_ring_grps(struct bnxt *bp)
3218{
3219 kfree(bp->grp_info);
3220 bp->grp_info = NULL;
3221}
3222
3223static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3224{
3225 int i;
3226
3227 if (irq_re_init) {
3228 bp->grp_info = kcalloc(bp->cp_nr_rings,
3229 sizeof(struct bnxt_ring_grp_info),
3230 GFP_KERNEL);
3231 if (!bp->grp_info)
3232 return -ENOMEM;
3233 }
3234 for (i = 0; i < bp->cp_nr_rings; i++) {
3235 if (irq_re_init)
3236 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3237 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3238 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3239 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3240 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3241 }
3242 return 0;
3243}
3244
3245static void bnxt_free_vnics(struct bnxt *bp)
3246{
3247 kfree(bp->vnic_info);
3248 bp->vnic_info = NULL;
3249 bp->nr_vnics = 0;
3250}
3251
3252static int bnxt_alloc_vnics(struct bnxt *bp)
3253{
3254 int num_vnics = 1;
3255
3256#ifdef CONFIG_RFS_ACCEL
9b3d15e6 3257 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
c0c050c5
MC		 3258 num_vnics += bp->rx_nr_rings;
3259#endif
3260
dc52c6c7
PS		 3261 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3262 num_vnics++;
3263
c0c050c5
MC		 3264 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3265 GFP_KERNEL);
3266 if (!bp->vnic_info)
3267 return -ENOMEM;
3268
3269 bp->nr_vnics = num_vnics;
3270 return 0;
3271}
3272
3273static void bnxt_init_vnics(struct bnxt *bp)
3274{
3275 int i;
3276
3277 for (i = 0; i < bp->nr_vnics; i++) {
3278 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
44c6f72a 3279 int j;
c0c050c5
MC		 3280
3281 vnic->fw_vnic_id = INVALID_HW_RING_ID;
44c6f72a
MC		 3282 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3283 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3284
c0c050c5
MC		 3285 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3286
3287 if (bp->vnic_info[i].rss_hash_key) {
3288 if (i == 0)
3289 prandom_bytes(vnic->rss_hash_key,
3290 HW_HASH_KEY_SIZE);
3291 else
3292 memcpy(vnic->rss_hash_key,
3293 bp->vnic_info[0].rss_hash_key,
3294 HW_HASH_KEY_SIZE);
3295 }
3296 }
3297}
3298
3299static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3300{
3301 int pages;
3302
3303 pages = ring_size / desc_per_pg;
3304
3305 if (!pages)
3306 return 1;
3307
3308 pages++;
3309
3310 while (pages & (pages - 1))
3311 pages++;
3312
3313 return pages;
3314}
3315
c6d30e83 3316void bnxt_set_tpa_flags(struct bnxt *bp)
c0c050c5
MC		 3317{
3318 bp->flags &= ~BNXT_FLAG_TPA;
341138c3
MC		 3319 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3320 return;
c0c050c5
MC		 3321 if (bp->dev->features & NETIF_F_LRO)
3322 bp->flags |= BNXT_FLAG_LRO;
1054aee8 3323 else if (bp->dev->features & NETIF_F_GRO_HW)
c0c050c5
MC		 3324 bp->flags |= BNXT_FLAG_GRO;
3325}
3326
3327/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3328 * be set on entry.
3329 */
3330void bnxt_set_ring_params(struct bnxt *bp)
3331{
3332 u32 ring_size, rx_size, rx_space;
3333 u32 agg_factor = 0, agg_ring_size = 0;
3334
3335 /* 8 for CRC and VLAN */
3336 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3337
3338 rx_space = rx_size + NET_SKB_PAD +
3339 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3340
3341 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3342 ring_size = bp->rx_ring_size;
3343 bp->rx_agg_ring_size = 0;
3344 bp->rx_agg_nr_pages = 0;
3345
3346 if (bp->flags & BNXT_FLAG_TPA)
2839f28b 3347 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
c0c050c5
MC		 3348
3349 bp->flags &= ~BNXT_FLAG_JUMBO;
bdbd1eb5 3350 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
c0c050c5
MC		 3351 u32 jumbo_factor;
3352
3353 bp->flags |= BNXT_FLAG_JUMBO;
3354 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3355 if (jumbo_factor > agg_factor)
3356 agg_factor = jumbo_factor;
3357 }
3358 agg_ring_size = ring_size * agg_factor;
3359
3360 if (agg_ring_size) {
3361 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3362 RX_DESC_CNT);
3363 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3364 u32 tmp = agg_ring_size;
3365
3366 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3367 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3368 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3369 tmp, agg_ring_size);
3370 }
3371 bp->rx_agg_ring_size = agg_ring_size;
3372 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3373 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3374 rx_space = rx_size + NET_SKB_PAD +
3375 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3376 }
3377
3378 bp->rx_buf_use_size = rx_size;
3379 bp->rx_buf_size = rx_space;
3380
3381 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3382 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3383
3384 ring_size = bp->tx_ring_size;
3385 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3386 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3387
3388 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3389 bp->cp_ring_size = ring_size;
3390
3391 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3392 if (bp->cp_nr_pages > MAX_CP_PAGES) {
3393 bp->cp_nr_pages = MAX_CP_PAGES;
3394 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3395 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3396 ring_size, bp->cp_ring_size);
3397 }
3398 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3399 bp->cp_ring_mask = bp->cp_bit - 1;
3400}
3401
96a8604f
JDB		 3402/* Changing allocation mode of RX rings.
3403 * TODO: Update when extending xdp_rxq_info to support allocation modes.
3404 */
c61fb99c 3405int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
6bb19474 3406{
c61fb99c
MC		 3407 if (page_mode) {
3408 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3409 return -EOPNOTSUPP;
7eb9bb3a
MC		 3410 bp->dev->max_mtu =
3411 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
c61fb99c
MC		 3412 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3413 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
c61fb99c
MC		 3414 bp->rx_dir = DMA_BIDIRECTIONAL;
3415 bp->rx_skb_func = bnxt_rx_page_skb;
1054aee8
MC		 3416 /* Disable LRO or GRO_HW */
3417 netdev_update_features(bp->dev);
c61fb99c 3418 } else {
7eb9bb3a 3419 bp->dev->max_mtu = bp->max_mtu;
c61fb99c
MC		 3420 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3421 bp->rx_dir = DMA_FROM_DEVICE;
3422 bp->rx_skb_func = bnxt_rx_skb;
3423 }
6bb19474
MC		 3424 return 0;
3425}
3426
c0c050c5
MC		 3427static void bnxt_free_vnic_attributes(struct bnxt *bp)
3428{
3429 int i;
3430 struct bnxt_vnic_info *vnic;
3431 struct pci_dev *pdev = bp->pdev;
3432
3433 if (!bp->vnic_info)
3434 return;
3435
3436 for (i = 0; i < bp->nr_vnics; i++) {
3437 vnic = &bp->vnic_info[i];
3438
3439 kfree(vnic->fw_grp_ids);
3440 vnic->fw_grp_ids = NULL;
3441
3442 kfree(vnic->uc_list);
3443 vnic->uc_list = NULL;
3444
3445 if (vnic->mc_list) {
3446 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3447 vnic->mc_list, vnic->mc_list_mapping);
3448 vnic->mc_list = NULL;
3449 }
3450
3451 if (vnic->rss_table) {
3452 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3453 vnic->rss_table,
3454 vnic->rss_table_dma_addr);
3455 vnic->rss_table = NULL;
3456 }
3457
3458 vnic->rss_hash_key = NULL;
3459 vnic->flags = 0;
3460 }
3461}
3462
3463static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3464{
3465 int i, rc = 0, size;
3466 struct bnxt_vnic_info *vnic;
3467 struct pci_dev *pdev = bp->pdev;
3468 int max_rings;
3469
3470 for (i = 0; i < bp->nr_vnics; i++) {
3471 vnic = &bp->vnic_info[i];
3472
3473 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3474 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3475
3476 if (mem_size > 0) {
3477 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3478 if (!vnic->uc_list) {
3479 rc = -ENOMEM;
3480 goto out;
3481 }
3482 }
3483 }
3484
3485 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3486 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3487 vnic->mc_list =
3488 dma_alloc_coherent(&pdev->dev,
3489 vnic->mc_list_size,
3490 &vnic->mc_list_mapping,
3491 GFP_KERNEL);
3492 if (!vnic->mc_list) {
3493 rc = -ENOMEM;
3494 goto out;
3495 }
3496 }
3497
44c6f72a
MC		 3498 if (bp->flags & BNXT_FLAG_CHIP_P5)
3499 goto vnic_skip_grps;
3500
c0c050c5
MC		 3501 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3502 max_rings = bp->rx_nr_rings;
3503 else
3504 max_rings = 1;
3505
3506 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3507 if (!vnic->fw_grp_ids) {
3508 rc = -ENOMEM;
3509 goto out;
3510 }
44c6f72a 3511vnic_skip_grps:
ae10ae74
MC		 3512 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3513 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3514 continue;
3515
c0c050c5
MC		 3516 /* Allocate rss table and hash key */
3517 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3518 &vnic->rss_table_dma_addr,
3519 GFP_KERNEL);
3520 if (!vnic->rss_table) {
3521 rc = -ENOMEM;
3522 goto out;
3523 }
3524
3525 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3526
3527 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3528 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3529 }
3530 return 0;
3531
3532out:
3533 return rc;
3534}
3535
3536static void bnxt_free_hwrm_resources(struct bnxt *bp)
3537{
3538 struct pci_dev *pdev = bp->pdev;
3539
a2bf74f4
VD		 3540 if (bp->hwrm_cmd_resp_addr) {
3541 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3542 bp->hwrm_cmd_resp_dma_addr);
3543 bp->hwrm_cmd_resp_addr = NULL;
3544 }
760b6d33
VD		 3545
3546 if (bp->hwrm_cmd_kong_resp_addr) {
3547 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3548 bp->hwrm_cmd_kong_resp_addr,
3549 bp->hwrm_cmd_kong_resp_dma_addr);
3550 bp->hwrm_cmd_kong_resp_addr = NULL;
3551 }
3552}
3553
3554static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3555{
3556 struct pci_dev *pdev = bp->pdev;
3557
ba642ab7
MC		 3558 if (bp->hwrm_cmd_kong_resp_addr)
3559 return 0;
3560
760b6d33
VD		 3561 bp->hwrm_cmd_kong_resp_addr =
3562 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3563 &bp->hwrm_cmd_kong_resp_dma_addr,
3564 GFP_KERNEL);
3565 if (!bp->hwrm_cmd_kong_resp_addr)
3566 return -ENOMEM;
3567
3568 return 0;
c0c050c5
MC		 3569}
3570
3571static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3572{
3573 struct pci_dev *pdev = bp->pdev;
3574
3575 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3576 &bp->hwrm_cmd_resp_dma_addr,
3577 GFP_KERNEL);
3578 if (!bp->hwrm_cmd_resp_addr)
3579 return -ENOMEM;
c0c050c5
MC		 3580
3581 return 0;
3582}
3583
e605db80
DK		 3584static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3585{
3586 if (bp->hwrm_short_cmd_req_addr) {
3587 struct pci_dev *pdev = bp->pdev;
3588
1dfddc41 3589 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
e605db80
DK		 3590 bp->hwrm_short_cmd_req_addr,
3591 bp->hwrm_short_cmd_req_dma_addr);
3592 bp->hwrm_short_cmd_req_addr = NULL;
3593 }
3594}
3595
3596static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3597{
3598 struct pci_dev *pdev = bp->pdev;
3599
ba642ab7
MC		 3600 if (bp->hwrm_short_cmd_req_addr)
3601 return 0;
3602
e605db80 3603 bp->hwrm_short_cmd_req_addr =
1dfddc41 3604 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
e605db80
DK		 3605 &bp->hwrm_short_cmd_req_dma_addr,
3606 GFP_KERNEL);
3607 if (!bp->hwrm_short_cmd_req_addr)
3608 return -ENOMEM;
3609
3610 return 0;
3611}
3612
fd3ab1c7 3613static void bnxt_free_port_stats(struct bnxt *bp)
c0c050c5 3614{
c0c050c5
MC		 3615 struct pci_dev *pdev = bp->pdev;
3616
00db3cba
VV		 3617 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3618 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3619
3bdf56c4
MC		 3620 if (bp->hw_rx_port_stats) {
3621 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3622 bp->hw_rx_port_stats,
3623 bp->hw_rx_port_stats_map);
3624 bp->hw_rx_port_stats = NULL;
00db3cba
VV		 3625 }
3626
36e53349
MC		 3627 if (bp->hw_tx_port_stats_ext) {
3628 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3629 bp->hw_tx_port_stats_ext,
3630 bp->hw_tx_port_stats_ext_map);
3631 bp->hw_tx_port_stats_ext = NULL;
3632 }
3633
00db3cba
VV		 3634 if (bp->hw_rx_port_stats_ext) {
3635 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3636 bp->hw_rx_port_stats_ext,
3637 bp->hw_rx_port_stats_ext_map);
3638 bp->hw_rx_port_stats_ext = NULL;
3bdf56c4 3639 }
55e4398d
VV		 3640
3641 if (bp->hw_pcie_stats) {
3642 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3643 bp->hw_pcie_stats, bp->hw_pcie_stats_map);
3644 bp->hw_pcie_stats = NULL;
3645 }
fd3ab1c7
MC		 3646}
3647
3648static void bnxt_free_ring_stats(struct bnxt *bp)
3649{
3650 struct pci_dev *pdev = bp->pdev;
3651 int size, i;
3bdf56c4 3652
c0c050c5
MC		 3653 if (!bp->bnapi)
3654 return;
3655
4e748506 3656 size = bp->hw_ring_stats_size;
c0c050c5
MC		 3657
3658 for (i = 0; i < bp->cp_nr_rings; i++) {
3659 struct bnxt_napi *bnapi = bp->bnapi[i];
3660 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3661
3662 if (cpr->hw_stats) {
3663 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3664 cpr->hw_stats_map);
3665 cpr->hw_stats = NULL;
3666 }
3667 }
3668}
3669
3670static int bnxt_alloc_stats(struct bnxt *bp)
3671{
3672 u32 size, i;
3673 struct pci_dev *pdev = bp->pdev;
3674
4e748506 3675 size = bp->hw_ring_stats_size;
c0c050c5
MC		 3676
3677 for (i = 0; i < bp->cp_nr_rings; i++) {
3678 struct bnxt_napi *bnapi = bp->bnapi[i];
3679 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3680
3681 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3682 &cpr->hw_stats_map,
3683 GFP_KERNEL);
3684 if (!cpr->hw_stats)
3685 return -ENOMEM;
3686
3687 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3688 }
3bdf56c4 3689
a220eabc
VV		 3690 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3691 return 0;
fd3ab1c7 3692
a220eabc
VV		 3693 if (bp->hw_rx_port_stats)
3694 goto alloc_ext_stats;
3bdf56c4 3695
a220eabc
VV		 3696 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3697 sizeof(struct tx_port_stats) + 1024;
3bdf56c4 3698
a220eabc
VV		 3699 bp->hw_rx_port_stats =
3700 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3701 &bp->hw_rx_port_stats_map,
3702 GFP_KERNEL);
3703 if (!bp->hw_rx_port_stats)
3704 return -ENOMEM;
3bdf56c4 3705
a220eabc
VV		 3706 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512;
3707 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3708 sizeof(struct rx_port_stats) + 512;
3709 bp->flags |= BNXT_FLAG_PORT_STATS;
00db3cba 3710
fd3ab1c7 3711alloc_ext_stats:
a220eabc
VV		 3712 /* Display extended statistics only if FW supports it */
3713 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
6154532f 3714 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
00db3cba
VV		 3715 return 0;
3716
a220eabc
VV		 3717 if (bp->hw_rx_port_stats_ext)
3718 goto alloc_tx_ext_stats;
fd3ab1c7 3719
a220eabc
VV		 3720 bp->hw_rx_port_stats_ext =
3721 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3722 &bp->hw_rx_port_stats_ext_map, GFP_KERNEL);
3723 if (!bp->hw_rx_port_stats_ext)
3724 return 0;
00db3cba 3725
fd3ab1c7 3726alloc_tx_ext_stats:
a220eabc 3727 if (bp->hw_tx_port_stats_ext)
55e4398d 3728 goto alloc_pcie_stats;
fd3ab1c7 3729
6154532f
VV		 3730 if (bp->hwrm_spec_code >= 0x10902 ||
3731 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
a220eabc
VV		 3732 bp->hw_tx_port_stats_ext =
3733 dma_alloc_coherent(&pdev->dev,
3734 sizeof(struct tx_port_stats_ext),
3735 &bp->hw_tx_port_stats_ext_map,
3736 GFP_KERNEL);
3bdf56c4 3737 }
a220eabc 3738 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
55e4398d
VV		 3739
3740alloc_pcie_stats:
3741 if (bp->hw_pcie_stats ||
3742 !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED))
3743 return 0;
3744
3745 bp->hw_pcie_stats =
3746 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3747 &bp->hw_pcie_stats_map, GFP_KERNEL);
3748 if (!bp->hw_pcie_stats)
3749 return 0;
3750
3751 bp->flags |= BNXT_FLAG_PCIE_STATS;
c0c050c5
MC		 3752 return 0;
3753}
3754
3755static void bnxt_clear_ring_indices(struct bnxt *bp)
3756{
3757 int i;
3758
3759 if (!bp->bnapi)
3760 return;
3761
3762 for (i = 0; i < bp->cp_nr_rings; i++) {
3763 struct bnxt_napi *bnapi = bp->bnapi[i];
3764 struct bnxt_cp_ring_info *cpr;
3765 struct bnxt_rx_ring_info *rxr;
3766 struct bnxt_tx_ring_info *txr;
3767
3768 if (!bnapi)
3769 continue;
3770
3771 cpr = &bnapi->cp_ring;
3772 cpr->cp_raw_cons = 0;
3773
b6ab4b01 3774 txr = bnapi->tx_ring;
3b2b7d9d
MC		 3775 if (txr) {
3776 txr->tx_prod = 0;
3777 txr->tx_cons = 0;
3778 }
c0c050c5 3779
b6ab4b01 3780 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 3781 if (rxr) {
3782 rxr->rx_prod = 0;
3783 rxr->rx_agg_prod = 0;
3784 rxr->rx_sw_agg_prod = 0;
376a5b86 3785 rxr->rx_next_cons = 0;
3b2b7d9d 3786 }
c0c050c5
MC		 3787 }
3788}
3789
3790static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3791{
3792#ifdef CONFIG_RFS_ACCEL
3793 int i;
3794
3795 /* Under rtnl_lock and all our NAPIs have been disabled. It's
3796 * safe to delete the hash table.
3797 */
3798 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3799 struct hlist_head *head;
3800 struct hlist_node *tmp;
3801 struct bnxt_ntuple_filter *fltr;
3802
3803 head = &bp->ntp_fltr_hash_tbl[i];
3804 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3805 hlist_del(&fltr->hash);
3806 kfree(fltr);
3807 }
3808 }
3809 if (irq_reinit) {
3810 kfree(bp->ntp_fltr_bmap);
3811 bp->ntp_fltr_bmap = NULL;
3812 }
3813 bp->ntp_fltr_count = 0;
3814#endif
3815}
3816
3817static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3818{
3819#ifdef CONFIG_RFS_ACCEL
3820 int i, rc = 0;
3821
3822 if (!(bp->flags & BNXT_FLAG_RFS))
3823 return 0;
3824
3825 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3826 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3827
3828 bp->ntp_fltr_count = 0;
ac45bd93
DC		 3829 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3830 sizeof(long),
c0c050c5
MC		 3831 GFP_KERNEL);
3832
3833 if (!bp->ntp_fltr_bmap)
3834 rc = -ENOMEM;
3835
3836 return rc;
3837#else
3838 return 0;
3839#endif
3840}
3841
3842static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3843{
3844 bnxt_free_vnic_attributes(bp);
3845 bnxt_free_tx_rings(bp);
3846 bnxt_free_rx_rings(bp);
3847 bnxt_free_cp_rings(bp);
3848 bnxt_free_ntp_fltrs(bp, irq_re_init);
3849 if (irq_re_init) {
fd3ab1c7 3850 bnxt_free_ring_stats(bp);
c0c050c5
MC		 3851 bnxt_free_ring_grps(bp);
3852 bnxt_free_vnics(bp);
a960dec9
MC		 3853 kfree(bp->tx_ring_map);
3854 bp->tx_ring_map = NULL;
b6ab4b01
MC		 3855 kfree(bp->tx_ring);
3856 bp->tx_ring = NULL;
3857 kfree(bp->rx_ring);
3858 bp->rx_ring = NULL;
c0c050c5
MC		 3859 kfree(bp->bnapi);
3860 bp->bnapi = NULL;
3861 } else {
3862 bnxt_clear_ring_indices(bp);
3863 }
3864}
3865
3866static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3867{
01657bcd 3868 int i, j, rc, size, arr_size;
c0c050c5
MC		 3869 void *bnapi;
3870
3871 if (irq_re_init) {
3872 /* Allocate bnapi mem pointer array and mem block for
3873 * all queues
3874 */
3875 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3876 bp->cp_nr_rings);
3877 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3878 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3879 if (!bnapi)
3880 return -ENOMEM;
3881
3882 bp->bnapi = bnapi;
3883 bnapi += arr_size;
3884 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3885 bp->bnapi[i] = bnapi;
3886 bp->bnapi[i]->index = i;
3887 bp->bnapi[i]->bp = bp;
e38287b7
MC		 3888 if (bp->flags & BNXT_FLAG_CHIP_P5) {
3889 struct bnxt_cp_ring_info *cpr =
3890 &bp->bnapi[i]->cp_ring;
3891
3892 cpr->cp_ring_struct.ring_mem.flags =
3893 BNXT_RMEM_RING_PTE_FLAG;
3894 }
c0c050c5
MC		 3895 }
3896
b6ab4b01
MC		 3897 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3898 sizeof(struct bnxt_rx_ring_info),
3899 GFP_KERNEL);
3900 if (!bp->rx_ring)
3901 return -ENOMEM;
3902
3903 for (i = 0; i < bp->rx_nr_rings; i++) {
e38287b7
MC		 3904 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3905
3906 if (bp->flags & BNXT_FLAG_CHIP_P5) {
3907 rxr->rx_ring_struct.ring_mem.flags =
3908 BNXT_RMEM_RING_PTE_FLAG;
3909 rxr->rx_agg_ring_struct.ring_mem.flags =
3910 BNXT_RMEM_RING_PTE_FLAG;
3911 }
3912 rxr->bnapi = bp->bnapi[i];
b6ab4b01
MC		 3913 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3914 }
3915
3916 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3917 sizeof(struct bnxt_tx_ring_info),
3918 GFP_KERNEL);
3919 if (!bp->tx_ring)
3920 return -ENOMEM;
3921
a960dec9
MC		 3922 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3923 GFP_KERNEL);
3924
3925 if (!bp->tx_ring_map)
3926 return -ENOMEM;
3927
01657bcd
MC		 3928 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3929 j = 0;
3930 else
3931 j = bp->rx_nr_rings;
3932
3933 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
e38287b7
MC		 3934 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3935
3936 if (bp->flags & BNXT_FLAG_CHIP_P5)
3937 txr->tx_ring_struct.ring_mem.flags =
3938 BNXT_RMEM_RING_PTE_FLAG;
3939 txr->bnapi = bp->bnapi[j];
3940 bp->bnapi[j]->tx_ring = txr;
5f449249 3941 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
38413406 3942 if (i >= bp->tx_nr_rings_xdp) {
e38287b7 3943 txr->txq_index = i - bp->tx_nr_rings_xdp;
38413406
MC		 3944 bp->bnapi[j]->tx_int = bnxt_tx_int;
3945 } else {
fa3e93e8 3946 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
38413406
MC		 3947 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3948 }
b6ab4b01
MC		 3949 }
3950
c0c050c5
MC		 3951 rc = bnxt_alloc_stats(bp);
3952 if (rc)
3953 goto alloc_mem_err;
3954
3955 rc = bnxt_alloc_ntp_fltrs(bp);
3956 if (rc)
3957 goto alloc_mem_err;
3958
3959 rc = bnxt_alloc_vnics(bp);
3960 if (rc)
3961 goto alloc_mem_err;
3962 }
3963
3964 bnxt_init_ring_struct(bp);
3965
3966 rc = bnxt_alloc_rx_rings(bp);
3967 if (rc)
3968 goto alloc_mem_err;
3969
3970 rc = bnxt_alloc_tx_rings(bp);
3971 if (rc)
3972 goto alloc_mem_err;
3973
3974 rc = bnxt_alloc_cp_rings(bp);
3975 if (rc)
3976 goto alloc_mem_err;
3977
3978 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3979 BNXT_VNIC_UCAST_FLAG;
3980 rc = bnxt_alloc_vnic_attributes(bp);
3981 if (rc)
3982 goto alloc_mem_err;
3983 return 0;
3984
3985alloc_mem_err:
3986 bnxt_free_mem(bp, true);
3987 return rc;
3988}
3989
9d8bc097
MC		 3990static void bnxt_disable_int(struct bnxt *bp)
3991{
3992 int i;
3993
3994 if (!bp->bnapi)
3995 return;
3996
3997 for (i = 0; i < bp->cp_nr_rings; i++) {
3998 struct bnxt_napi *bnapi = bp->bnapi[i];
3999 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
daf1f1e7 4000 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
9d8bc097 4001
daf1f1e7 4002 if (ring->fw_ring_id != INVALID_HW_RING_ID)
697197e5 4003 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
9d8bc097
MC		 4004 }
4005}
4006
e5811b8c
MC		 4007static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4008{
4009 struct bnxt_napi *bnapi = bp->bnapi[n];
4010 struct bnxt_cp_ring_info *cpr;
4011
4012 cpr = &bnapi->cp_ring;
4013 return cpr->cp_ring_struct.map_idx;
4014}
4015
9d8bc097
MC		 4016static void bnxt_disable_int_sync(struct bnxt *bp)
4017{
4018 int i;
4019
4020 atomic_inc(&bp->intr_sem);
4021
4022 bnxt_disable_int(bp);
e5811b8c
MC		 4023 for (i = 0; i < bp->cp_nr_rings; i++) {
4024 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4025
4026 synchronize_irq(bp->irq_tbl[map_idx].vector);
4027 }
9d8bc097
MC		 4028}
4029
4030static void bnxt_enable_int(struct bnxt *bp)
4031{
4032 int i;
4033
4034 atomic_set(&bp->intr_sem, 0);
4035 for (i = 0; i < bp->cp_nr_rings; i++) {
4036 struct bnxt_napi *bnapi = bp->bnapi[i];
4037 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4038
697197e5 4039 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
9d8bc097
MC		 4040 }
4041}
4042
c0c050c5
MC		 4043void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4044 u16 cmpl_ring, u16 target_id)
4045{
a8643e16 4046 struct input *req = request;
c0c050c5 4047
a8643e16
MC		 4048 req->req_type = cpu_to_le16(req_type);
4049 req->cmpl_ring = cpu_to_le16(cmpl_ring);
4050 req->target_id = cpu_to_le16(target_id);
760b6d33
VD		 4051 if (bnxt_kong_hwrm_message(bp, req))
4052 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4053 else
4054 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
c0c050c5
MC		 4055}
4056
d4f1420d
MC		 4057static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4058{
4059 switch (hwrm_err) {
4060 case HWRM_ERR_CODE_SUCCESS:
4061 return 0;
4062 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4063 return -EACCES;
4064 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4065 return -ENOSPC;
4066 case HWRM_ERR_CODE_INVALID_PARAMS:
4067 case HWRM_ERR_CODE_INVALID_FLAGS:
4068 case HWRM_ERR_CODE_INVALID_ENABLES:
4069 case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4070 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4071 return -EINVAL;
4072 case HWRM_ERR_CODE_NO_BUFFER:
4073 return -ENOMEM;
4074 case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4075 return -EAGAIN;
4076 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4077 return -EOPNOTSUPP;
4078 default:
4079 return -EIO;
4080 }
4081}
4082
fbfbc485
MC		 4083static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4084 int timeout, bool silent)
c0c050c5 4085{
a11fa2be 4086 int i, intr_process, rc, tmo_count;
a8643e16 4087 struct input *req = msg;
c0c050c5 4088 u32 *data = msg;
845adfe4
MC		 4089 __le32 *resp_len;
4090 u8 *valid;
c0c050c5
MC		 4091 u16 cp_ring_id, len = 0;
4092 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
e605db80 4093 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
ebd5818c 4094 struct hwrm_short_input short_input = {0};
2e9ee398 4095 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
89455017 4096 u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
2e9ee398 4097 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
760b6d33 4098 u16 dst = BNXT_HWRM_CHNL_CHIMP;
c0c050c5 4099
1dfddc41
MC		 4100 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4101 if (msg_len > bp->hwrm_max_ext_req_len ||
4102 !bp->hwrm_short_cmd_req_addr)
4103 return -EINVAL;
4104 }
4105
760b6d33
VD		 4106 if (bnxt_hwrm_kong_chnl(bp, req)) {
4107 dst = BNXT_HWRM_CHNL_KONG;
4108 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4109 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4110 resp = bp->hwrm_cmd_kong_resp_addr;
4111 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
4112 }
4113
4114 memset(resp, 0, PAGE_SIZE);
4115 cp_ring_id = le16_to_cpu(req->cmpl_ring);
4116 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4117
4118 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4119 /* currently supports only one outstanding message */
4120 if (intr_process)
4121 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4122
1dfddc41
MC		 4123 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4124 msg_len > BNXT_HWRM_MAX_REQ_LEN) {
e605db80 4125 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
1dfddc41
MC		 4126 u16 max_msg_len;
4127
4128 /* Set boundary for maximum extended request length for short
4129 * cmd format. If passed up from device use the max supported
4130 * internal req length.
4131 */
4132 max_msg_len = bp->hwrm_max_ext_req_len;
e605db80
DK		 4133
4134 memcpy(short_cmd_req, req, msg_len);
1dfddc41
MC		 4135 if (msg_len < max_msg_len)
4136 memset(short_cmd_req + msg_len, 0,
4137 max_msg_len - msg_len);
e605db80
DK		 4138
4139 short_input.req_type = req->req_type;
4140 short_input.signature =
4141 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4142 short_input.size = cpu_to_le16(msg_len);
4143 short_input.req_addr =
4144 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4145
4146 data = (u32 *)&short_input;
4147 msg_len = sizeof(short_input);
4148
4149 /* Sync memory write before updating doorbell */
4150 wmb();
4151
4152 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4153 }
4154
c0c050c5 4155 /* Write request msg to hwrm channel */
2e9ee398 4156 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
c0c050c5 4157
e605db80 4158 for (i = msg_len; i < max_req_len; i += 4)
2e9ee398 4159 writel(0, bp->bar0 + bar_offset + i);
d79979a1 4160
c0c050c5 4161 /* Ring channel doorbell */
2e9ee398 4162 writel(1, bp->bar0 + doorbell_offset);
c0c050c5 4163
5bedb529
MC		 4164 if (!pci_is_enabled(bp->pdev))
4165 return 0;
4166
ff4fe81d
MC		 4167 if (!timeout)
4168 timeout = DFLT_HWRM_CMD_TIMEOUT;
9751e8e7
AG		 4169 /* convert timeout to usec */
4170 timeout *= 1000;
ff4fe81d 4171
c0c050c5 4172 i = 0;
9751e8e7
AG		 4173 /* Short timeout for the first few iterations:
4174 * number of loops = number of loops for short timeout +
4175 * number of loops for standard timeout.
4176 */
4177 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4178 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4179 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
89455017
VD		 4180 resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
4181
c0c050c5 4182 if (intr_process) {
fc718bb2
VD		 4183 u16 seq_id = bp->hwrm_intr_seq_id;
4184
c0c050c5 4185 /* Wait until hwrm response cmpl interrupt is processed */
fc718bb2 4186 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
a11fa2be 4187 i++ < tmo_count) {
9751e8e7
AG		 4188 /* on first few passes, just barely sleep */
4189 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4190 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4191 HWRM_SHORT_MAX_TIMEOUT);
4192 else
4193 usleep_range(HWRM_MIN_TIMEOUT,
4194 HWRM_MAX_TIMEOUT);
c0c050c5
MC		 4195 }
4196
fc718bb2 4197 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
5bedb529
MC		 4198 if (!silent)
4199 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4200 le16_to_cpu(req->req_type));
a935cb7e 4201 return -EBUSY;
c0c050c5 4202 }
845adfe4
MC		 4203 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4204 HWRM_RESP_LEN_SFT;
89455017 4205 valid = resp_addr + len - 1;
c0c050c5 4206 } else {
cc559c1a
MC		 4207 int j;
4208
c0c050c5 4209 /* Check if response len is updated */
a11fa2be 4210 for (i = 0; i < tmo_count; i++) {
c0c050c5
MC		 4211 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4212 HWRM_RESP_LEN_SFT;
4213 if (len)
4214 break;
9751e8e7 4215 /* on first few passes, just barely sleep */
67681d02 4216 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
9751e8e7
AG		 4217 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4218 HWRM_SHORT_MAX_TIMEOUT);
4219 else
4220 usleep_range(HWRM_MIN_TIMEOUT,
4221 HWRM_MAX_TIMEOUT);
c0c050c5
MC		 4222 }
4223
a11fa2be 4224 if (i >= tmo_count) {
5bedb529
MC		 4225 if (!silent)
4226 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4227 HWRM_TOTAL_TIMEOUT(i),
4228 le16_to_cpu(req->req_type),
4229 le16_to_cpu(req->seq_id), len);
a935cb7e 4230 return -EBUSY;
c0c050c5
MC		 4231 }
4232
845adfe4 4233 /* Last byte of resp contains valid bit */
89455017 4234 valid = resp_addr + len - 1;
cc559c1a 4235 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
845adfe4
MC		 4236 /* make sure we read from updated DMA memory */
4237 dma_rmb();
4238 if (*valid)
c0c050c5 4239 break;
0000b81a 4240 usleep_range(1, 5);
c0c050c5
MC		 4241 }
4242
cc559c1a 4243 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
5bedb529
MC		 4244 if (!silent)
4245 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4246 HWRM_TOTAL_TIMEOUT(i),
4247 le16_to_cpu(req->req_type),
4248 le16_to_cpu(req->seq_id), len,
4249 *valid);
a935cb7e 4250 return -EBUSY;
c0c050c5
MC		 4251 }
4252 }
4253
845adfe4
MC		 4254 /* Zero valid bit for compatibility. Valid bit in an older spec
4255 * may become a new field in a newer spec. We must make sure that
4256 * a new field not implemented by old spec will read zero.
4257 */
4258 *valid = 0;
c0c050c5 4259 rc = le16_to_cpu(resp->error_code);
fbfbc485 4260 if (rc && !silent)
c0c050c5
MC		 4261 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4262 le16_to_cpu(resp->req_type),
4263 le16_to_cpu(resp->seq_id), rc);
d4f1420d 4264 return bnxt_hwrm_to_stderr(rc);
fbfbc485
MC		 4265}
4266
4267int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4268{
4269 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
c0c050c5
MC		 4270}
4271
cc72f3b1
MC		 4272int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4273 int timeout)
4274{
4275 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4276}
4277
c0c050c5
MC		 4278int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4279{
4280 int rc;
4281
4282 mutex_lock(&bp->hwrm_cmd_lock);
4283 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4284 mutex_unlock(&bp->hwrm_cmd_lock);
4285 return rc;
4286}
4287
90e20921
MC		 4288int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4289 int timeout)
4290{
4291 int rc;
4292
4293 mutex_lock(&bp->hwrm_cmd_lock);
4294 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4295 mutex_unlock(&bp->hwrm_cmd_lock);
4296 return rc;
4297}
4298
a1653b13
MC		 4299int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
4300 int bmap_size)
c0c050c5
MC		 4301{
4302 struct hwrm_func_drv_rgtr_input req = {0};
25be8623
MC		 4303 DECLARE_BITMAP(async_events_bmap, 256);
4304 u32 *events = (u32 *)async_events_bmap;
a1653b13 4305 int i;
c0c050c5
MC		 4306
4307 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4308
4309 req.enables =
a1653b13 4310 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
c0c050c5 4311
25be8623
MC		 4312 memset(async_events_bmap, 0, sizeof(async_events_bmap));
4313 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
4314 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4315
a1653b13
MC		 4316 if (bmap && bmap_size) {
4317 for (i = 0; i < bmap_size; i++) {
4318 if (test_bit(i, bmap))
4319 __set_bit(i, async_events_bmap);
4320 }
4321 }
4322
25be8623
MC		 4323 for (i = 0; i < 8; i++)
4324 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4325
a1653b13
MC		 4326 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4327}
4328
4329static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
4330{
25e1acd6 4331 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
a1653b13 4332 struct hwrm_func_drv_rgtr_input req = {0};
25e1acd6 4333 int rc;
a1653b13
MC		 4334
4335 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4336
4337 req.enables =
4338 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4339 FUNC_DRV_RGTR_REQ_ENABLES_VER);
4340
11f15ed3 4341 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
d4f52de0
MC		 4342 req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
4343 req.ver_maj_8b = DRV_VER_MAJ;
4344 req.ver_min_8b = DRV_VER_MIN;
4345 req.ver_upd_8b = DRV_VER_UPD;
4346 req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4347 req.ver_min = cpu_to_le16(DRV_VER_MIN);
4348 req.ver_upd = cpu_to_le16(DRV_VER_UPD);
c0c050c5
MC		 4349
4350 if (BNXT_PF(bp)) {
9b0436c3 4351 u32 data[8];
a1653b13 4352 int i;
c0c050c5 4353
9b0436c3
MC		 4354 memset(data, 0, sizeof(data));
4355 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4356 u16 cmd = bnxt_vf_req_snif[i];
4357 unsigned int bit, idx;
4358
4359 idx = cmd / 32;
4360 bit = cmd % 32;
4361 data[idx] |= 1 << bit;
4362 }
c0c050c5 4363
de68f5de
MC		 4364 for (i = 0; i < 8; i++)
4365 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4366
c0c050c5
MC		 4367 req.enables |=
4368 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4369 }
4370
abd43a13
VD		 4371 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4372 req.flags |= cpu_to_le32(
4373 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4374
25e1acd6
MC		 4375 mutex_lock(&bp->hwrm_cmd_lock);
4376 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
d4f1420d
MC		 4377 if (!rc && (resp->flags &
4378 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)))
25e1acd6
MC		 4379 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4380 mutex_unlock(&bp->hwrm_cmd_lock);
4381 return rc;
c0c050c5
MC		 4382}
4383
be58a0da
JH		 4384static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4385{
4386 struct hwrm_func_drv_unrgtr_input req = {0};
4387
4388 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4389 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4390}
4391
c0c050c5
MC		 4392static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4393{
4394 u32 rc = 0;
4395 struct hwrm_tunnel_dst_port_free_input req = {0};
4396
4397 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4398 req.tunnel_type = tunnel_type;
4399
4400 switch (tunnel_type) {
4401 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4402 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4403 break;
4404 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4405 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4406 break;
4407 default:
4408 break;
4409 }
4410
4411 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4412 if (rc)
4413 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4414 rc);
4415 return rc;
4416}
4417
4418static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4419 u8 tunnel_type)
4420{
4421 u32 rc = 0;
4422 struct hwrm_tunnel_dst_port_alloc_input req = {0};
4423 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4424
4425 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4426
4427 req.tunnel_type = tunnel_type;
4428 req.tunnel_dst_port_val = port;
4429
4430 mutex_lock(&bp->hwrm_cmd_lock);
4431 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4432 if (rc) {
4433 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4434 rc);
4435 goto err_out;
4436 }
4437
57aac71b
CJ		 4438 switch (tunnel_type) {
4439 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
c0c050c5 4440 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 4441 break;
4442 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
c0c050c5 4443 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
57aac71b
CJ		 4444 break;
4445 default:
4446 break;
4447 }
4448
c0c050c5
MC		 4449err_out:
4450 mutex_unlock(&bp->hwrm_cmd_lock);
4451 return rc;
4452}
4453
4454static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4455{
4456 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4457 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4458
4459 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
c193554e 4460 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
c0c050c5
MC		 4461
4462 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4463 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4464 req.mask = cpu_to_le32(vnic->rx_mask);
4465 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4466}
4467
4468#ifdef CONFIG_RFS_ACCEL
4469static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4470 struct bnxt_ntuple_filter *fltr)
4471{
4472 struct hwrm_cfa_ntuple_filter_free_input req = {0};
4473
4474 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4475 req.ntuple_filter_id = fltr->filter_id;
4476 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4477}
4478
4479#define BNXT_NTP_FLTR_FLAGS \
4480 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
4481 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
4482 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
4483 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
4484 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
4485 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
4486 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
4487 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
4488 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
4489 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
4490 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
4491 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
4492 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
c193554e 4493 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
c0c050c5 4494
61aad724
MC		 4495#define BNXT_NTP_TUNNEL_FLTR_FLAG \
4496 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4497
c0c050c5
MC		 4498static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4499 struct bnxt_ntuple_filter *fltr)
4500{
c0c050c5 4501 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
5c209fc8 4502 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
c0c050c5 4503 struct flow_keys *keys = &fltr->fkeys;
ac33906c
MC		 4504 struct bnxt_vnic_info *vnic;
4505 u32 dst_ena = 0;
5c209fc8 4506 int rc = 0;
c0c050c5
MC		 4507
4508 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
a54c4d74 4509 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
c0c050c5 4510
ac33906c
MC		 4511 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX) {
4512 dst_ena = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
4513 req.rfs_ring_tbl_idx = cpu_to_le16(fltr->rxq);
4514 vnic = &bp->vnic_info[0];
4515 } else {
4516 vnic = &bp->vnic_info[fltr->rxq + 1];
4517 }
4518 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4519 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS | dst_ena);
c0c050c5
MC		 4520
4521 req.ethertype = htons(ETH_P_IP);
4522 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
c193554e 4523 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
c0c050c5
MC		 4524 req.ip_protocol = keys->basic.ip_proto;
4525
dda0e746
MC		 4526 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4527 int i;
4528
4529 req.ethertype = htons(ETH_P_IPV6);
4530 req.ip_addr_type =
4531 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4532 *(struct in6_addr *)&req.src_ipaddr[0] =
4533 keys->addrs.v6addrs.src;
4534 *(struct in6_addr *)&req.dst_ipaddr[0] =
4535 keys->addrs.v6addrs.dst;
4536 for (i = 0; i < 4; i++) {
4537 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4538 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4539 }
4540 } else {
4541 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4542 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4543 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4544 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4545 }
61aad724
MC		 4546 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4547 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4548 req.tunnel_type =
4549 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4550 }
c0c050c5
MC		 4551
4552 req.src_port = keys->ports.src;
4553 req.src_port_mask = cpu_to_be16(0xffff);
4554 req.dst_port = keys->ports.dst;
4555 req.dst_port_mask = cpu_to_be16(0xffff);
4556
c0c050c5
MC		 4557 mutex_lock(&bp->hwrm_cmd_lock);
4558 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5c209fc8
VD		 4559 if (!rc) {
4560 resp = bnxt_get_hwrm_resp_addr(bp, &req);
c0c050c5 4561 fltr->filter_id = resp->ntuple_filter_id;
5c209fc8 4562 }
c0c050c5
MC		 4563 mutex_unlock(&bp->hwrm_cmd_lock);
4564 return rc;
4565}
4566#endif
4567
4568static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4569 u8 *mac_addr)
4570{
4571 u32 rc = 0;
4572 struct hwrm_cfa_l2_filter_alloc_input req = {0};
4573 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4574
4575 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
dc52c6c7
PS		 4576 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4577 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4578 req.flags |=
4579 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
c193554e 4580 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
c0c050c5
MC		 4581 req.enables =
4582 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
c193554e 4583 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
c0c050c5
MC		 4584 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4585 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4586 req.l2_addr_mask[0] = 0xff;
4587 req.l2_addr_mask[1] = 0xff;
4588 req.l2_addr_mask[2] = 0xff;
4589 req.l2_addr_mask[3] = 0xff;
4590 req.l2_addr_mask[4] = 0xff;
4591 req.l2_addr_mask[5] = 0xff;
4592
4593 mutex_lock(&bp->hwrm_cmd_lock);
4594 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4595 if (!rc)
4596 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4597 resp->l2_filter_id;
4598 mutex_unlock(&bp->hwrm_cmd_lock);
4599 return rc;
4600}
4601
4602static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4603{
4604 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4605 int rc = 0;
4606
4607 /* Any associated ntuple filters will also be cleared by firmware. */
4608 mutex_lock(&bp->hwrm_cmd_lock);
4609 for (i = 0; i < num_of_vnics; i++) {
4610 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4611
4612 for (j = 0; j < vnic->uc_filter_count; j++) {
4613 struct hwrm_cfa_l2_filter_free_input req = {0};
4614
4615 bnxt_hwrm_cmd_hdr_init(bp, &req,
4616 HWRM_CFA_L2_FILTER_FREE, -1, -1);
4617
4618 req.l2_filter_id = vnic->fw_l2_filter_id[j];
4619
4620 rc = _hwrm_send_message(bp, &req, sizeof(req),
4621 HWRM_CMD_TIMEOUT);
4622 }
4623 vnic->uc_filter_count = 0;
4624 }
4625 mutex_unlock(&bp->hwrm_cmd_lock);
4626
4627 return rc;
4628}
4629
4630static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4631{
4632 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
79632e9b 4633 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
c0c050c5
MC		 4634 struct hwrm_vnic_tpa_cfg_input req = {0};
4635
3c4fe80b
MC		 4636 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4637 return 0;
4638
c0c050c5
MC		 4639 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4640
4641 if (tpa_flags) {
4642 u16 mss = bp->dev->mtu - 40;
4643 u32 nsegs, n, segs = 0, flags;
4644
4645 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4646 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4647 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4648 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4649 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4650 if (tpa_flags & BNXT_FLAG_GRO)
4651 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4652
4653 req.flags = cpu_to_le32(flags);
4654
4655 req.enables =
4656 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
c193554e
MC		 4657 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4658 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
c0c050c5
MC		 4659
4660 /* Number of segs are log2 units, and first packet is not
4661 * included as part of this units.
4662 */
2839f28b
MC		 4663 if (mss <= BNXT_RX_PAGE_SIZE) {
4664 n = BNXT_RX_PAGE_SIZE / mss;
c0c050c5
MC		 4665 nsegs = (MAX_SKB_FRAGS - 1) * n;
4666 } else {
2839f28b
MC		 4667 n = mss / BNXT_RX_PAGE_SIZE;
4668 if (mss & (BNXT_RX_PAGE_SIZE - 1))
c0c050c5
MC		 4669 n++;
4670 nsegs = (MAX_SKB_FRAGS - n) / n;
4671 }
4672
79632e9b
MC		 4673 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4674 segs = MAX_TPA_SEGS_P5;
4675 max_aggs = bp->max_tpa;
4676 } else {
4677 segs = ilog2(nsegs);
4678 }
c0c050c5 4679 req.max_agg_segs = cpu_to_le16(segs);
79632e9b 4680 req.max_aggs = cpu_to_le16(max_aggs);
c193554e
MC		 4681
4682 req.min_agg_len = cpu_to_le32(512);
c0c050c5
MC		 4683 }
4684 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4685
4686 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4687}
4688
2c61d211
MC		 4689static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4690{
4691 struct bnxt_ring_grp_info *grp_info;
4692
4693 grp_info = &bp->grp_info[ring->grp_idx];
4694 return grp_info->cp_fw_ring_id;
4695}
4696
4697static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4698{
4699 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4700 struct bnxt_napi *bnapi = rxr->bnapi;
4701 struct bnxt_cp_ring_info *cpr;
4702
4703 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4704 return cpr->cp_ring_struct.fw_ring_id;
4705 } else {
4706 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4707 }
4708}
4709
4710static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4711{
4712 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4713 struct bnxt_napi *bnapi = txr->bnapi;
4714 struct bnxt_cp_ring_info *cpr;
4715
4716 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4717 return cpr->cp_ring_struct.fw_ring_id;
4718 } else {
4719 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4720 }
4721}
4722
c0c050c5
MC		 4723static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4724{
4725 u32 i, j, max_rings;
4726 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4727 struct hwrm_vnic_rss_cfg_input req = {0};
4728
7b3af4f7
MC		 4729 if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4730 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
c0c050c5
MC		 4731 return 0;
4732
4733 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4734 if (set_rss) {
87da7f79 4735 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
50f011b6 4736 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
dc52c6c7
PS		 4737 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4738 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4739 max_rings = bp->rx_nr_rings - 1;
4740 else
4741 max_rings = bp->rx_nr_rings;
4742 } else {
c0c050c5 4743 max_rings = 1;
dc52c6c7 4744 }
c0c050c5
MC		 4745
4746 /* Fill the RSS indirection table with ring group ids */
4747 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4748 if (j == max_rings)
4749 j = 0;
4750 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4751 }
4752
4753 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4754 req.hash_key_tbl_addr =
4755 cpu_to_le64(vnic->rss_hash_key_dma_addr);
4756 }
94ce9caa 4757 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
c0c050c5
MC		 4758 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4759}
4760
7b3af4f7
MC		 4761static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4762{
4763 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4764 u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4765 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4766 struct hwrm_vnic_rss_cfg_input req = {0};
4767
4768 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4769 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4770 if (!set_rss) {
4771 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4772 return 0;
4773 }
4774 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4775 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4776 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4777 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4778 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4779 for (i = 0, k = 0; i < nr_ctxs; i++) {
4780 __le16 *ring_tbl = vnic->rss_table;
4781 int rc;
4782
4783 req.ring_table_pair_index = i;
4784 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4785 for (j = 0; j < 64; j++) {
4786 u16 ring_id;
4787
4788 ring_id = rxr->rx_ring_struct.fw_ring_id;
4789 *ring_tbl++ = cpu_to_le16(ring_id);
4790 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4791 *ring_tbl++ = cpu_to_le16(ring_id);
4792 rxr++;
4793 k++;
4794 if (k == max_rings) {
4795 k = 0;
4796 rxr = &bp->rx_ring[0];
4797 }
4798 }
4799 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4800 if (rc)
d4f1420d 4801 return rc;
7b3af4f7
MC		 4802 }
4803 return 0;
4804}
4805
c0c050c5
MC		 4806static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4807{
4808 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4809 struct hwrm_vnic_plcmodes_cfg_input req = {0};
4810
4811 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4812 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4813 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4814 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4815 req.enables =
4816 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4817 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4818 /* thresholds not implemented in firmware yet */
4819 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4820 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4821 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4822 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4823}
4824
94ce9caa
PS		 4825static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4826 u16 ctx_idx)
c0c050c5
MC		 4827{
4828 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4829
4830 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4831 req.rss_cos_lb_ctx_id =
94ce9caa 4832 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
c0c050c5
MC		 4833
4834 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
94ce9caa 4835 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
c0c050c5
MC		 4836}
4837
4838static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4839{
94ce9caa 4840 int i, j;
c0c050c5
MC		 4841
4842 for (i = 0; i < bp->nr_vnics; i++) {
4843 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4844
94ce9caa
PS		 4845 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4846 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4847 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4848 }
c0c050c5
MC		 4849 }
4850 bp->rsscos_nr_ctxs = 0;
4851}
4852
94ce9caa 4853static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
c0c050c5
MC		 4854{
4855 int rc;
4856 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4857 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4858 bp->hwrm_cmd_resp_addr;
4859
4860 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4861 -1);
4862
4863 mutex_lock(&bp->hwrm_cmd_lock);
4864 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4865 if (!rc)
94ce9caa 4866 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
c0c050c5
MC		 4867 le16_to_cpu(resp->rss_cos_lb_ctx_id);
4868 mutex_unlock(&bp->hwrm_cmd_lock);
4869
4870 return rc;
4871}
4872
abe93ad2
MC		 4873static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4874{
4875 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4876 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4877 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4878}
4879
a588e458 4880int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
c0c050c5 4881{
b81a90d3 4882 unsigned int ring = 0, grp_idx;
c0c050c5
MC		 4883 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4884 struct hwrm_vnic_cfg_input req = {0};
cf6645f8 4885 u16 def_vlan = 0;
c0c050c5
MC		 4886
4887 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
dc52c6c7 4888
7b3af4f7
MC		 4889 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4890 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4891
4892 req.default_rx_ring_id =
4893 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
4894 req.default_cmpl_ring_id =
4895 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
4896 req.enables =
4897 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
4898 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
4899 goto vnic_mru;
4900 }
dc52c6c7 4901 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
c0c050c5 4902 /* Only RSS support for now TBD: COS & LB */
dc52c6c7
PS		 4903 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4904 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4905 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4906 VNIC_CFG_REQ_ENABLES_MRU);
ae10ae74
MC		 4907 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4908 req.rss_rule =
4909 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4910 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4911 VNIC_CFG_REQ_ENABLES_MRU);
4912 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
dc52c6c7
PS		 4913 } else {
4914 req.rss_rule = cpu_to_le16(0xffff);
4915 }
94ce9caa 4916
dc52c6c7
PS		 4917 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4918 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
94ce9caa
PS		 4919 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4920 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4921 } else {
4922 req.cos_rule = cpu_to_le16(0xffff);
4923 }
4924
c0c050c5 4925 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
b81a90d3 4926 ring = 0;
c0c050c5 4927 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
b81a90d3 4928 ring = vnic_id - 1;
76595193
PS		 4929 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4930 ring = bp->rx_nr_rings - 1;
c0c050c5 4931
b81a90d3 4932 grp_idx = bp->rx_ring[ring].bnapi->index;
c0c050c5 4933 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
c0c050c5 4934 req.lb_rule = cpu_to_le16(0xffff);
7b3af4f7 4935vnic_mru:
c0c050c5
MC		 4936 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4937 VLAN_HLEN);
4938
7b3af4f7 4939 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
cf6645f8
MC		 4940#ifdef CONFIG_BNXT_SRIOV
4941 if (BNXT_VF(bp))
4942 def_vlan = bp->vf.vlan;
4943#endif
4944 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
c0c050c5 4945 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
a588e458 4946 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
abe93ad2 4947 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
c0c050c5
MC		 4948
4949 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4950}
4951
4952static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4953{
4954 u32 rc = 0;
4955
4956 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4957 struct hwrm_vnic_free_input req = {0};
4958
4959 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4960 req.vnic_id =
4961 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4962
4963 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4964 if (rc)
4965 return rc;
4966 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4967 }
4968 return rc;
4969}
4970
4971static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4972{
4973 u16 i;
4974
4975 for (i = 0; i < bp->nr_vnics; i++)
4976 bnxt_hwrm_vnic_free_one(bp, i);
4977}
4978
b81a90d3
MC		 4979static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4980 unsigned int start_rx_ring_idx,
4981 unsigned int nr_rings)
c0c050c5 4982{
b81a90d3
MC		 4983 int rc = 0;
4984 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
c0c050c5
MC		 4985 struct hwrm_vnic_alloc_input req = {0};
4986 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
44c6f72a
MC		 4987 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4988
4989 if (bp->flags & BNXT_FLAG_CHIP_P5)
4990 goto vnic_no_ring_grps;
c0c050c5
MC		 4991
4992 /* map ring groups to this vnic */
b81a90d3
MC		 4993 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4994 grp_idx = bp->rx_ring[i].bnapi->index;
4995 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
c0c050c5 4996 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
b81a90d3 4997 j, nr_rings);
c0c050c5
MC		 4998 break;
4999 }
44c6f72a 5000 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
c0c050c5
MC		 5001 }
5002
44c6f72a
MC		 5003vnic_no_ring_grps:
5004 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5005 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
c0c050c5
MC		 5006 if (vnic_id == 0)
5007 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5008
5009 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5010
5011 mutex_lock(&bp->hwrm_cmd_lock);
5012 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5013 if (!rc)
44c6f72a 5014 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
c0c050c5
MC		 5015 mutex_unlock(&bp->hwrm_cmd_lock);
5016 return rc;
5017}
5018
8fdefd63
MC		 5019static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5020{
5021 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5022 struct hwrm_vnic_qcaps_input req = {0};
5023 int rc;
5024
fbbdbc64 5025 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
ba642ab7 5026 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
8fdefd63
MC		 5027 if (bp->hwrm_spec_code < 0x10600)
5028 return 0;
5029
5030 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5031 mutex_lock(&bp->hwrm_cmd_lock);
5032 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5033 if (!rc) {
abe93ad2
MC		 5034 u32 flags = le32_to_cpu(resp->flags);
5035
41e8d798
MC		 5036 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5037 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
8fdefd63 5038 bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
abe93ad2
MC		 5039 if (flags &
5040 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5041 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
79632e9b 5042 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
4e748506
MC		 5043 if (bp->max_tpa_v2)
5044 bp->hw_ring_stats_size =
5045 sizeof(struct ctx_hw_stats_ext);
8fdefd63
MC		 5046 }
5047 mutex_unlock(&bp->hwrm_cmd_lock);
5048 return rc;
5049}
5050
c0c050c5
MC		 5051static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5052{
5053 u16 i;
5054 u32 rc = 0;
5055
44c6f72a
MC		 5056 if (bp->flags & BNXT_FLAG_CHIP_P5)
5057 return 0;
5058
c0c050c5
MC		 5059 mutex_lock(&bp->hwrm_cmd_lock);
5060 for (i = 0; i < bp->rx_nr_rings; i++) {
5061 struct hwrm_ring_grp_alloc_input req = {0};
5062 struct hwrm_ring_grp_alloc_output *resp =
5063 bp->hwrm_cmd_resp_addr;
b81a90d3 5064 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
c0c050c5
MC		 5065
5066 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5067
b81a90d3
MC		 5068 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5069 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5070 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5071 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
c0c050c5
MC		 5072
5073 rc = _hwrm_send_message(bp, &req, sizeof(req),
5074 HWRM_CMD_TIMEOUT);
5075 if (rc)
5076 break;
5077
b81a90d3
MC		 5078 bp->grp_info[grp_idx].fw_grp_id =
5079 le32_to_cpu(resp->ring_group_id);
c0c050c5
MC		 5080 }
5081 mutex_unlock(&bp->hwrm_cmd_lock);
5082 return rc;
5083}
5084
5085static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5086{
5087 u16 i;
5088 u32 rc = 0;
5089 struct hwrm_ring_grp_free_input req = {0};
5090
44c6f72a 5091 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
c0c050c5
MC		 5092 return 0;
5093
5094 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5095
5096 mutex_lock(&bp->hwrm_cmd_lock);
5097 for (i = 0; i < bp->cp_nr_rings; i++) {
5098 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5099 continue;
5100 req.ring_group_id =
5101 cpu_to_le32(bp->grp_info[i].fw_grp_id);
5102
5103 rc = _hwrm_send_message(bp, &req, sizeof(req),
5104 HWRM_CMD_TIMEOUT);
5105 if (rc)
5106 break;
5107 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5108 }
5109 mutex_unlock(&bp->hwrm_cmd_lock);
5110 return rc;
5111}
5112
5113static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5114 struct bnxt_ring_struct *ring,
9899bb59 5115 u32 ring_type, u32 map_index)
c0c050c5
MC		 5116{
5117 int rc = 0, err = 0;
5118 struct hwrm_ring_alloc_input req = {0};
5119 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6fe19886 5120 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
9899bb59 5121 struct bnxt_ring_grp_info *grp_info;
c0c050c5
MC		 5122 u16 ring_id;
5123
5124 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5125
5126 req.enables = 0;
6fe19886
MC		 5127 if (rmem->nr_pages > 1) {
5128 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
c0c050c5
MC		 5129 /* Page size is in log2 units */
5130 req.page_size = BNXT_PAGE_SHIFT;
5131 req.page_tbl_depth = 1;
5132 } else {
6fe19886 5133 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
c0c050c5
MC		 5134 }
5135 req.fbo = 0;
5136 /* Association of ring index with doorbell index and MSIX number */
5137 req.logical_id = cpu_to_le16(map_index);
5138
5139 switch (ring_type) {
2c61d211
MC		 5140 case HWRM_RING_ALLOC_TX: {
5141 struct bnxt_tx_ring_info *txr;
5142
5143 txr = container_of(ring, struct bnxt_tx_ring_info,
5144 tx_ring_struct);
c0c050c5
MC		 5145 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5146 /* Association of transmit ring with completion ring */
9899bb59 5147 grp_info = &bp->grp_info[ring->grp_idx];
2c61d211 5148 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
c0c050c5 5149 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
9899bb59 5150 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
c0c050c5
MC		 5151 req.queue_id = cpu_to_le16(ring->queue_id);
5152 break;
2c61d211 5153 }
c0c050c5
MC		 5154 case HWRM_RING_ALLOC_RX:
5155 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5156 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
23aefdd7
MC		 5157 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5158 u16 flags = 0;
5159
5160 /* Association of rx ring with stats context */
5161 grp_info = &bp->grp_info[ring->grp_idx];
5162 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5163 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5164 req.enables |= cpu_to_le32(
5165 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5166 if (NET_IP_ALIGN == 2)
5167 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5168 req.flags = cpu_to_le16(flags);
5169 }
c0c050c5
MC		 5170 break;
5171 case HWRM_RING_ALLOC_AGG:
23aefdd7
MC		 5172 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5173 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5174 /* Association of agg ring with rx ring */
5175 grp_info = &bp->grp_info[ring->grp_idx];
5176 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5177 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5178 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5179 req.enables |= cpu_to_le32(
5180 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5181 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5182 } else {
5183 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5184 }
c0c050c5
MC		 5185 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5186 break;
5187 case HWRM_RING_ALLOC_CMPL:
bac9a7e0 5188 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
c0c050c5 5189 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
23aefdd7
MC		 5190 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5191 /* Association of cp ring with nq */
5192 grp_info = &bp->grp_info[map_index];
5193 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5194 req.cq_handle = cpu_to_le64(ring->handle);
5195 req.enables |= cpu_to_le32(
5196 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5197 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5198 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5199 }
5200 break;
5201 case HWRM_RING_ALLOC_NQ:
5202 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5203 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
c0c050c5
MC		 5204 if (bp->flags & BNXT_FLAG_USING_MSIX)
5205 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5206 break;
5207 default:
5208 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5209 ring_type);
5210 return -1;
5211 }
5212
5213 mutex_lock(&bp->hwrm_cmd_lock);
5214 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5215 err = le16_to_cpu(resp->error_code);
5216 ring_id = le16_to_cpu(resp->ring_id);
5217 mutex_unlock(&bp->hwrm_cmd_lock);
5218
5219 if (rc || err) {
2727c888
MC		 5220 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5221 ring_type, rc, err);
5222 return -EIO;
c0c050c5
MC		 5223 }
5224 ring->fw_ring_id = ring_id;
5225 return rc;
5226}
5227
486b5c22
MC		 5228static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5229{
5230 int rc;
5231
5232 if (BNXT_PF(bp)) {
5233 struct hwrm_func_cfg_input req = {0};
5234
5235 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5236 req.fid = cpu_to_le16(0xffff);
5237 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5238 req.async_event_cr = cpu_to_le16(idx);
5239 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5240 } else {
5241 struct hwrm_func_vf_cfg_input req = {0};
5242
5243 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5244 req.enables =
5245 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5246 req.async_event_cr = cpu_to_le16(idx);
5247 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5248 }
5249 return rc;
5250}
5251
697197e5
MC		 5252static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5253 u32 map_idx, u32 xid)
5254{
5255 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5256 if (BNXT_PF(bp))
5257 db->doorbell = bp->bar1 + 0x10000;
5258 else
5259 db->doorbell = bp->bar1 + 0x4000;
5260 switch (ring_type) {
5261 case HWRM_RING_ALLOC_TX:
5262 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5263 break;
5264 case HWRM_RING_ALLOC_RX:
5265 case HWRM_RING_ALLOC_AGG:
5266 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5267 break;
5268 case HWRM_RING_ALLOC_CMPL:
5269 db->db_key64 = DBR_PATH_L2;
5270 break;
5271 case HWRM_RING_ALLOC_NQ:
5272 db->db_key64 = DBR_PATH_L2;
5273 break;
5274 }
5275 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5276 } else {
5277 db->doorbell = bp->bar1 + map_idx * 0x80;
5278 switch (ring_type) {
5279 case HWRM_RING_ALLOC_TX:
5280 db->db_key32 = DB_KEY_TX;
5281 break;
5282 case HWRM_RING_ALLOC_RX:
5283 case HWRM_RING_ALLOC_AGG:
5284 db->db_key32 = DB_KEY_RX;
5285 break;
5286 case HWRM_RING_ALLOC_CMPL:
5287 db->db_key32 = DB_KEY_CP;
5288 break;
5289 }
5290 }
5291}
5292
c0c050c5
MC		 5293static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5294{
e8f267b0 5295 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
c0c050c5 5296 int i, rc = 0;
697197e5 5297 u32 type;
c0c050c5 5298
23aefdd7
MC		 5299 if (bp->flags & BNXT_FLAG_CHIP_P5)
5300 type = HWRM_RING_ALLOC_NQ;
5301 else
5302 type = HWRM_RING_ALLOC_CMPL;
edd0c2cc
MC		 5303 for (i = 0; i < bp->cp_nr_rings; i++) {
5304 struct bnxt_napi *bnapi = bp->bnapi[i];
5305 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5306 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
9899bb59 5307 u32 map_idx = ring->map_idx;
5e66e35a 5308 unsigned int vector;
c0c050c5 5309
5e66e35a
MC		 5310 vector = bp->irq_tbl[map_idx].vector;
5311 disable_irq_nosync(vector);
697197e5 5312 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5e66e35a
MC		 5313 if (rc) {
5314 enable_irq(vector);
edd0c2cc 5315 goto err_out;
5e66e35a 5316 }
697197e5
MC		 5317 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5318 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5e66e35a 5319 enable_irq(vector);
edd0c2cc 5320 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
486b5c22
MC		 5321
5322 if (!i) {
5323 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5324 if (rc)
5325 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5326 }
c0c050c5
MC		 5327 }
5328
697197e5 5329 type = HWRM_RING_ALLOC_TX;
edd0c2cc 5330 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5331 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3e08b184
MC		 5332 struct bnxt_ring_struct *ring;
5333 u32 map_idx;
c0c050c5 5334
3e08b184
MC		 5335 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5336 struct bnxt_napi *bnapi = txr->bnapi;
5337 struct bnxt_cp_ring_info *cpr, *cpr2;
5338 u32 type2 = HWRM_RING_ALLOC_CMPL;
5339
5340 cpr = &bnapi->cp_ring;
5341 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5342 ring = &cpr2->cp_ring_struct;
5343 ring->handle = BNXT_TX_HDL;
5344 map_idx = bnapi->index;
5345 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5346 if (rc)
5347 goto err_out;
5348 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5349 ring->fw_ring_id);
5350 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5351 }
5352 ring = &txr->tx_ring_struct;
5353 map_idx = i;
697197e5 5354 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
edd0c2cc
MC		 5355 if (rc)
5356 goto err_out;
697197e5 5357 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
c0c050c5
MC		 5358 }
5359
697197e5 5360 type = HWRM_RING_ALLOC_RX;
edd0c2cc 5361 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5362 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5363 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3e08b184
MC		 5364 struct bnxt_napi *bnapi = rxr->bnapi;
5365 u32 map_idx = bnapi->index;
c0c050c5 5366
697197e5 5367 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
edd0c2cc
MC		 5368 if (rc)
5369 goto err_out;
697197e5 5370 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
e8f267b0
MC		 5371 /* If we have agg rings, post agg buffers first. */
5372 if (!agg_rings)
5373 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
b81a90d3 5374 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3e08b184
MC		 5375 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5376 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5377 u32 type2 = HWRM_RING_ALLOC_CMPL;
5378 struct bnxt_cp_ring_info *cpr2;
5379
5380 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5381 ring = &cpr2->cp_ring_struct;
5382 ring->handle = BNXT_RX_HDL;
5383 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5384 if (rc)
5385 goto err_out;
5386 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5387 ring->fw_ring_id);
5388 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5389 }
c0c050c5
MC		 5390 }
5391
e8f267b0 5392 if (agg_rings) {
697197e5 5393 type = HWRM_RING_ALLOC_AGG;
c0c050c5 5394 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5395 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 5396 struct bnxt_ring_struct *ring =
5397 &rxr->rx_agg_ring_struct;
9899bb59 5398 u32 grp_idx = ring->grp_idx;
b81a90d3 5399 u32 map_idx = grp_idx + bp->rx_nr_rings;
c0c050c5 5400
697197e5 5401 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
c0c050c5
MC		 5402 if (rc)
5403 goto err_out;
5404
697197e5
MC		 5405 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5406 ring->fw_ring_id);
5407 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
e8f267b0 5408 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
b81a90d3 5409 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 5410 }
5411 }
5412err_out:
5413 return rc;
5414}
5415
5416static int hwrm_ring_free_send_msg(struct bnxt *bp,
5417 struct bnxt_ring_struct *ring,
5418 u32 ring_type, int cmpl_ring_id)
5419{
5420 int rc;
5421 struct hwrm_ring_free_input req = {0};
5422 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5423 u16 error_code;
5424
74608fc9 5425 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
c0c050c5
MC		 5426 req.ring_type = ring_type;
5427 req.ring_id = cpu_to_le16(ring->fw_ring_id);
5428
5429 mutex_lock(&bp->hwrm_cmd_lock);
5430 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5431 error_code = le16_to_cpu(resp->error_code);
5432 mutex_unlock(&bp->hwrm_cmd_lock);
5433
5434 if (rc || error_code) {
2727c888
MC		 5435 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5436 ring_type, rc, error_code);
5437 return -EIO;
c0c050c5
MC		 5438 }
5439 return 0;
5440}
5441
edd0c2cc 5442static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
c0c050c5 5443{
23aefdd7 5444 u32 type;
edd0c2cc 5445 int i;
c0c050c5
MC		 5446
5447 if (!bp->bnapi)
edd0c2cc 5448 return;
c0c050c5 5449
edd0c2cc 5450 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 5451 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 5452 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
edd0c2cc
MC		 5453
5454 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5455 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5456
edd0c2cc
MC		 5457 hwrm_ring_free_send_msg(bp, ring,
5458 RING_FREE_REQ_RING_TYPE_TX,
5459 close_path ? cmpl_ring_id :
5460 INVALID_HW_RING_ID);
5461 ring->fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 5462 }
5463 }
5464
edd0c2cc 5465 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5466 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5467 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3 5468 u32 grp_idx = rxr->bnapi->index;
edd0c2cc
MC		 5469
5470 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5471 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5472
edd0c2cc
MC		 5473 hwrm_ring_free_send_msg(bp, ring,
5474 RING_FREE_REQ_RING_TYPE_RX,
5475 close_path ? cmpl_ring_id :
5476 INVALID_HW_RING_ID);
5477 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 5478 bp->grp_info[grp_idx].rx_fw_ring_id =
5479 INVALID_HW_RING_ID;
c0c050c5
MC		 5480 }
5481 }
5482
23aefdd7
MC		 5483 if (bp->flags & BNXT_FLAG_CHIP_P5)
5484 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5485 else
5486 type = RING_FREE_REQ_RING_TYPE_RX;
edd0c2cc 5487 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 5488 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 5489 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
b81a90d3 5490 u32 grp_idx = rxr->bnapi->index;
edd0c2cc
MC		 5491
5492 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1f83391b
MC		 5493 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5494
23aefdd7 5495 hwrm_ring_free_send_msg(bp, ring, type,
edd0c2cc
MC		 5496 close_path ? cmpl_ring_id :
5497 INVALID_HW_RING_ID);
5498 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 5499 bp->grp_info[grp_idx].agg_fw_ring_id =
5500 INVALID_HW_RING_ID;
c0c050c5
MC		 5501 }
5502 }
5503
9d8bc097
MC		 5504 /* The completion rings are about to be freed. After that the
5505 * IRQ doorbell will not work anymore. So we need to disable
5506 * IRQ here.
5507 */
5508 bnxt_disable_int_sync(bp);
5509
23aefdd7
MC		 5510 if (bp->flags & BNXT_FLAG_CHIP_P5)
5511 type = RING_FREE_REQ_RING_TYPE_NQ;
5512 else
5513 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
edd0c2cc
MC		 5514 for (i = 0; i < bp->cp_nr_rings; i++) {
5515 struct bnxt_napi *bnapi = bp->bnapi[i];
5516 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3e08b184
MC		 5517 struct bnxt_ring_struct *ring;
5518 int j;
edd0c2cc 5519
3e08b184
MC		 5520 for (j = 0; j < 2; j++) {
5521 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5522
5523 if (cpr2) {
5524 ring = &cpr2->cp_ring_struct;
5525 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5526 continue;
5527 hwrm_ring_free_send_msg(bp, ring,
5528 RING_FREE_REQ_RING_TYPE_L2_CMPL,
5529 INVALID_HW_RING_ID);
5530 ring->fw_ring_id = INVALID_HW_RING_ID;
5531 }
5532 }
5533 ring = &cpr->cp_ring_struct;
edd0c2cc 5534 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
23aefdd7 5535 hwrm_ring_free_send_msg(bp, ring, type,
edd0c2cc
MC		 5536 INVALID_HW_RING_ID);
5537 ring->fw_ring_id = INVALID_HW_RING_ID;
5538 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 5539 }
5540 }
c0c050c5
MC		 5541}
5542
41e8d798
MC		 5543static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5544 bool shared);
5545
674f50a5
MC		 5546static int bnxt_hwrm_get_rings(struct bnxt *bp)
5547{
5548 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5549 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5550 struct hwrm_func_qcfg_input req = {0};
5551 int rc;
5552
5553 if (bp->hwrm_spec_code < 0x10601)
5554 return 0;
5555
5556 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5557 req.fid = cpu_to_le16(0xffff);
5558 mutex_lock(&bp->hwrm_cmd_lock);
5559 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5560 if (rc) {
5561 mutex_unlock(&bp->hwrm_cmd_lock);
d4f1420d 5562 return rc;
674f50a5
MC		 5563 }
5564
5565 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
f1ca94de 5566 if (BNXT_NEW_RM(bp)) {
674f50a5
MC		 5567 u16 cp, stats;
5568
5569 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5570 hw_resc->resv_hw_ring_grps =
5571 le32_to_cpu(resp->alloc_hw_ring_grps);
5572 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5573 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5574 stats = le16_to_cpu(resp->alloc_stat_ctx);
75720e63 5575 hw_resc->resv_irqs = cp;
41e8d798
MC		 5576 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5577 int rx = hw_resc->resv_rx_rings;
5578 int tx = hw_resc->resv_tx_rings;
5579
5580 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5581 rx >>= 1;
5582 if (cp < (rx + tx)) {
5583 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5584 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5585 rx <<= 1;
5586 hw_resc->resv_rx_rings = rx;
5587 hw_resc->resv_tx_rings = tx;
5588 }
75720e63 5589 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
41e8d798
MC		 5590 hw_resc->resv_hw_ring_grps = rx;
5591 }
674f50a5 5592 hw_resc->resv_cp_rings = cp;
780baad4 5593 hw_resc->resv_stat_ctxs = stats;
674f50a5
MC		 5594 }
5595 mutex_unlock(&bp->hwrm_cmd_lock);
5596 return 0;
5597}
5598
391be5c2
MC		 5599/* Caller must hold bp->hwrm_cmd_lock */
5600int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5601{
5602 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5603 struct hwrm_func_qcfg_input req = {0};
5604 int rc;
5605
5606 if (bp->hwrm_spec_code < 0x10601)
5607 return 0;
5608
5609 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5610 req.fid = cpu_to_le16(fid);
5611 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5612 if (!rc)
5613 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5614
5615 return rc;
5616}
5617
41e8d798
MC		 5618static bool bnxt_rfs_supported(struct bnxt *bp);
5619
4ed50ef4
MC		 5620static void
5621__bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5622 int tx_rings, int rx_rings, int ring_grps,
780baad4 5623 int cp_rings, int stats, int vnics)
391be5c2 5624{
674f50a5 5625 u32 enables = 0;
391be5c2 5626
4ed50ef4
MC		 5627 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5628 req->fid = cpu_to_le16(0xffff);
674f50a5 5629 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
4ed50ef4 5630 req->num_tx_rings = cpu_to_le16(tx_rings);
f1ca94de 5631 if (BNXT_NEW_RM(bp)) {
674f50a5 5632 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
3f93cd3f 5633 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
41e8d798
MC		 5634 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5635 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5636 enables |= tx_rings + ring_grps ?
3f93cd3f 5637 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5638 enables |= rx_rings ?
5639 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5640 } else {
5641 enables |= cp_rings ?
3f93cd3f 5642 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5643 enables |= ring_grps ?
5644 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5645 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5646 }
dbe80d44 5647 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
674f50a5 5648
4ed50ef4 5649 req->num_rx_rings = cpu_to_le16(rx_rings);
41e8d798
MC		 5650 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5651 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5652 req->num_msix = cpu_to_le16(cp_rings);
5653 req->num_rsscos_ctxs =
5654 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5655 } else {
5656 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5657 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5658 req->num_rsscos_ctxs = cpu_to_le16(1);
5659 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5660 bnxt_rfs_supported(bp))
5661 req->num_rsscos_ctxs =
5662 cpu_to_le16(ring_grps + 1);
5663 }
780baad4 5664 req->num_stat_ctxs = cpu_to_le16(stats);
4ed50ef4 5665 req->num_vnics = cpu_to_le16(vnics);
674f50a5 5666 }
4ed50ef4
MC		 5667 req->enables = cpu_to_le32(enables);
5668}
5669
5670static void
5671__bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5672 struct hwrm_func_vf_cfg_input *req, int tx_rings,
5673 int rx_rings, int ring_grps, int cp_rings,
780baad4 5674 int stats, int vnics)
4ed50ef4
MC		 5675{
5676 u32 enables = 0;
5677
5678 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5679 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
41e8d798
MC		 5680 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5681 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
3f93cd3f 5682 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
41e8d798
MC		 5683 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5684 enables |= tx_rings + ring_grps ?
3f93cd3f 5685 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5686 } else {
5687 enables |= cp_rings ?
3f93cd3f 5688 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
41e8d798
MC		 5689 enables |= ring_grps ?
5690 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5691 }
4ed50ef4 5692 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
41e8d798 5693 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
4ed50ef4 5694
41e8d798 5695 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
4ed50ef4
MC		 5696 req->num_tx_rings = cpu_to_le16(tx_rings);
5697 req->num_rx_rings = cpu_to_le16(rx_rings);
41e8d798
MC		 5698 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5699 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5700 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5701 } else {
5702 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5703 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5704 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5705 }
780baad4 5706 req->num_stat_ctxs = cpu_to_le16(stats);
4ed50ef4
MC		 5707 req->num_vnics = cpu_to_le16(vnics);
5708
5709 req->enables = cpu_to_le32(enables);
5710}
5711
5712static int
5713bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4 5714 int ring_grps, int cp_rings, int stats, int vnics)
4ed50ef4
MC		 5715{
5716 struct hwrm_func_cfg_input req = {0};
5717 int rc;
5718
5719 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5720 cp_rings, stats, vnics);
4ed50ef4 5721 if (!req.enables)
391be5c2
MC		 5722 return 0;
5723
674f50a5
MC		 5724 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5725 if (rc)
d4f1420d 5726 return rc;
674f50a5
MC		 5727
5728 if (bp->hwrm_spec_code < 0x10601)
5729 bp->hw_resc.resv_tx_rings = tx_rings;
5730
5731 rc = bnxt_hwrm_get_rings(bp);
5732 return rc;
5733}
5734
5735static int
5736bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4 5737 int ring_grps, int cp_rings, int stats, int vnics)
674f50a5
MC		 5738{
5739 struct hwrm_func_vf_cfg_input req = {0};
674f50a5
MC		 5740 int rc;
5741
f1ca94de 5742 if (!BNXT_NEW_RM(bp)) {
674f50a5 5743 bp->hw_resc.resv_tx_rings = tx_rings;
391be5c2 5744 return 0;
674f50a5 5745 }
391be5c2 5746
4ed50ef4 5747 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5748 cp_rings, stats, vnics);
391be5c2 5749 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
674f50a5 5750 if (rc)
d4f1420d 5751 return rc;
674f50a5
MC		 5752
5753 rc = bnxt_hwrm_get_rings(bp);
5754 return rc;
5755}
5756
5757static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
780baad4 5758 int cp, int stat, int vnic)
674f50a5
MC		 5759{
5760 if (BNXT_PF(bp))
780baad4
VV		 5761 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5762 vnic);
674f50a5 5763 else
780baad4
VV		 5764 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5765 vnic);
674f50a5
MC		 5766}
5767
b16b6891 5768int bnxt_nq_rings_in_use(struct bnxt *bp)
08654eb2
MC		 5769{
5770 int cp = bp->cp_nr_rings;
5771 int ulp_msix, ulp_base;
5772
5773 ulp_msix = bnxt_get_ulp_msix_num(bp);
5774 if (ulp_msix) {
5775 ulp_base = bnxt_get_ulp_msix_base(bp);
5776 cp += ulp_msix;
5777 if ((ulp_base + ulp_msix) > cp)
5778 cp = ulp_base + ulp_msix;
5779 }
5780 return cp;
5781}
5782
c0b8cda0
MC		 5783static int bnxt_cp_rings_in_use(struct bnxt *bp)
5784{
5785 int cp;
5786
5787 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5788 return bnxt_nq_rings_in_use(bp);
5789
5790 cp = bp->tx_nr_rings + bp->rx_nr_rings;
5791 return cp;
5792}
5793
780baad4
VV		 5794static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5795{
d77b1ad8
MC		 5796 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
5797 int cp = bp->cp_nr_rings;
5798
5799 if (!ulp_stat)
5800 return cp;
5801
5802 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
5803 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
5804
5805 return cp + ulp_stat;
780baad4
VV		 5806}
5807
4e41dc5d
MC		 5808static bool bnxt_need_reserve_rings(struct bnxt *bp)
5809{
5810 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
fbcfc8e4 5811 int cp = bnxt_cp_rings_in_use(bp);
c0b8cda0 5812 int nq = bnxt_nq_rings_in_use(bp);
780baad4 5813 int rx = bp->rx_nr_rings, stat;
4e41dc5d
MC		 5814 int vnic = 1, grp = rx;
5815
5816 if (bp->hwrm_spec_code < 0x10601)
5817 return false;
5818
5819 if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5820 return true;
5821
41e8d798 5822 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
4e41dc5d
MC		 5823 vnic = rx + 1;
5824 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5825 rx <<= 1;
780baad4 5826 stat = bnxt_get_func_stat_ctxs(bp);
f1ca94de 5827 if (BNXT_NEW_RM(bp) &&
4e41dc5d 5828 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
01989c6b 5829 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
41e8d798
MC		 5830 (hw_resc->resv_hw_ring_grps != grp &&
5831 !(bp->flags & BNXT_FLAG_CHIP_P5))))
4e41dc5d 5832 return true;
01989c6b
MC		 5833 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
5834 hw_resc->resv_irqs != nq)
5835 return true;
4e41dc5d
MC		 5836 return false;
5837}
5838
674f50a5
MC		 5839static int __bnxt_reserve_rings(struct bnxt *bp)
5840{
5841 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
c0b8cda0 5842 int cp = bnxt_nq_rings_in_use(bp);
674f50a5
MC		 5843 int tx = bp->tx_nr_rings;
5844 int rx = bp->rx_nr_rings;
674f50a5 5845 int grp, rx_rings, rc;
780baad4 5846 int vnic = 1, stat;
674f50a5 5847 bool sh = false;
674f50a5 5848
4e41dc5d 5849 if (!bnxt_need_reserve_rings(bp))
674f50a5
MC		 5850 return 0;
5851
5852 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5853 sh = true;
41e8d798 5854 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
674f50a5
MC		 5855 vnic = rx + 1;
5856 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5857 rx <<= 1;
674f50a5 5858 grp = bp->rx_nr_rings;
780baad4 5859 stat = bnxt_get_func_stat_ctxs(bp);
674f50a5 5860
780baad4 5861 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
391be5c2
MC		 5862 if (rc)
5863 return rc;
5864
674f50a5 5865 tx = hw_resc->resv_tx_rings;
f1ca94de 5866 if (BNXT_NEW_RM(bp)) {
674f50a5 5867 rx = hw_resc->resv_rx_rings;
c0b8cda0 5868 cp = hw_resc->resv_irqs;
674f50a5
MC		 5869 grp = hw_resc->resv_hw_ring_grps;
5870 vnic = hw_resc->resv_vnics;
780baad4 5871 stat = hw_resc->resv_stat_ctxs;
674f50a5
MC		 5872 }
5873
5874 rx_rings = rx;
5875 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5876 if (rx >= 2) {
5877 rx_rings = rx >> 1;
5878 } else {
5879 if (netif_running(bp->dev))
5880 return -ENOMEM;
5881
5882 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5883 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5884 bp->dev->hw_features &= ~NETIF_F_LRO;
5885 bp->dev->features &= ~NETIF_F_LRO;
5886 bnxt_set_ring_params(bp);
5887 }
5888 }
5889 rx_rings = min_t(int, rx_rings, grp);
780baad4
VV		 5890 cp = min_t(int, cp, bp->cp_nr_rings);
5891 if (stat > bnxt_get_ulp_stat_ctxs(bp))
5892 stat -= bnxt_get_ulp_stat_ctxs(bp);
5893 cp = min_t(int, cp, stat);
674f50a5
MC		 5894 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
5895 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5896 rx = rx_rings << 1;
5897 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
5898 bp->tx_nr_rings = tx;
5899 bp->rx_nr_rings = rx_rings;
5900 bp->cp_nr_rings = cp;
5901
780baad4 5902 if (!tx || !rx || !cp || !grp || !vnic || !stat)
674f50a5
MC		 5903 return -ENOMEM;
5904
391be5c2
MC		 5905 return rc;
5906}
5907
8f23d638 5908static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5909 int ring_grps, int cp_rings, int stats,
5910 int vnics)
98fdbe73 5911{
8f23d638 5912 struct hwrm_func_vf_cfg_input req = {0};
6fc2ffdf 5913 u32 flags;
98fdbe73
MC		 5914 int rc;
5915
f1ca94de 5916 if (!BNXT_NEW_RM(bp))
98fdbe73
MC		 5917 return 0;
5918
6fc2ffdf 5919 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5920 cp_rings, stats, vnics);
8f23d638
MC		 5921 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
5922 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5923 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
8f23d638 5924 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
41e8d798
MC		 5925 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
5926 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5927 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5928 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
8f23d638
MC		 5929
5930 req.flags = cpu_to_le32(flags);
8f23d638 5931 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
d4f1420d 5932 return rc;
8f23d638
MC		 5933}
5934
5935static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5936 int ring_grps, int cp_rings, int stats,
5937 int vnics)
8f23d638
MC		 5938{
5939 struct hwrm_func_cfg_input req = {0};
6fc2ffdf 5940 u32 flags;
8f23d638 5941 int rc;
98fdbe73 5942
6fc2ffdf 5943 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
780baad4 5944 cp_rings, stats, vnics);
8f23d638 5945 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
41e8d798 5946 if (BNXT_NEW_RM(bp)) {
8f23d638
MC		 5947 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5948 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
8f23d638
MC		 5949 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5950 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
41e8d798 5951 if (bp->flags & BNXT_FLAG_CHIP_P5)
0b815023
MC		 5952 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
5953 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
41e8d798
MC		 5954 else
5955 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5956 }
6fc2ffdf 5957
8f23d638 5958 req.flags = cpu_to_le32(flags);
98fdbe73 5959 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
d4f1420d 5960 return rc;
98fdbe73
MC		 5961}
5962
8f23d638 5963static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
780baad4
VV		 5964 int ring_grps, int cp_rings, int stats,
5965 int vnics)
8f23d638
MC		 5966{
5967 if (bp->hwrm_spec_code < 0x10801)
5968 return 0;
5969
5970 if (BNXT_PF(bp))
5971 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
780baad4
VV		 5972 ring_grps, cp_rings, stats,
5973 vnics);
8f23d638
MC		 5974
5975 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
780baad4 5976 cp_rings, stats, vnics);
8f23d638
MC		 5977}
5978
74706afa
MC		 5979static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
5980{
5981 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5982 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5983 struct hwrm_ring_aggint_qcaps_input req = {0};
5984 int rc;
5985
5986 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
5987 coal_cap->num_cmpl_dma_aggr_max = 63;
5988 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
5989 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
5990 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
5991 coal_cap->int_lat_tmr_min_max = 65535;
5992 coal_cap->int_lat_tmr_max_max = 65535;
5993 coal_cap->num_cmpl_aggr_int_max = 65535;
5994 coal_cap->timer_units = 80;
5995
5996 if (bp->hwrm_spec_code < 0x10902)
5997 return;
5998
5999 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6000 mutex_lock(&bp->hwrm_cmd_lock);
6001 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6002 if (!rc) {
6003 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
58590c8d 6004 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
74706afa
MC		 6005 coal_cap->num_cmpl_dma_aggr_max =
6006 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6007 coal_cap->num_cmpl_dma_aggr_during_int_max =
6008 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6009 coal_cap->cmpl_aggr_dma_tmr_max =
6010 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6011 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6012 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6013 coal_cap->int_lat_tmr_min_max =
6014 le16_to_cpu(resp->int_lat_tmr_min_max);
6015 coal_cap->int_lat_tmr_max_max =
6016 le16_to_cpu(resp->int_lat_tmr_max_max);
6017 coal_cap->num_cmpl_aggr_int_max =
6018 le16_to_cpu(resp->num_cmpl_aggr_int_max);
6019 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6020 }
6021 mutex_unlock(&bp->hwrm_cmd_lock);
6022}
6023
6024static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6025{
6026 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6027
6028 return usec * 1000 / coal_cap->timer_units;
6029}
6030
6031static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6032 struct bnxt_coal *hw_coal,
bb053f52
MC		 6033 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6034{
74706afa
MC		 6035 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6036 u32 cmpl_params = coal_cap->cmpl_params;
6037 u16 val, tmr, max, flags = 0;
f8503969
MC		 6038
6039 max = hw_coal->bufs_per_record * 128;
6040 if (hw_coal->budget)
6041 max = hw_coal->bufs_per_record * hw_coal->budget;
74706afa 6042 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
f8503969
MC		 6043
6044 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6045 req->num_cmpl_aggr_int = cpu_to_le16(val);
b153cbc5 6046
74706afa 6047 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
f8503969
MC		 6048 req->num_cmpl_dma_aggr = cpu_to_le16(val);
6049
74706afa
MC		 6050 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6051 coal_cap->num_cmpl_dma_aggr_during_int_max);
f8503969
MC		 6052 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6053
74706afa
MC		 6054 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6055 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
f8503969
MC		 6056 req->int_lat_tmr_max = cpu_to_le16(tmr);
6057
6058 /* min timer set to 1/2 of interrupt timer */
74706afa
MC		 6059 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6060 val = tmr / 2;
6061 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6062 req->int_lat_tmr_min = cpu_to_le16(val);
6063 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6064 }
f8503969
MC		 6065
6066 /* buf timer set to 1/4 of interrupt timer */
74706afa 6067 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
f8503969
MC		 6068 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6069
74706afa
MC		 6070 if (cmpl_params &
6071 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6072 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6073 val = clamp_t(u16, tmr, 1,
6074 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6075 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
6076 req->enables |=
6077 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6078 }
f8503969 6079
74706afa
MC		 6080 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6081 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6082 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6083 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
f8503969 6084 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
bb053f52 6085 req->flags = cpu_to_le16(flags);
74706afa 6086 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
bb053f52
MC		 6087}
6088
58590c8d
MC		 6089/* Caller holds bp->hwrm_cmd_lock */
6090static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6091 struct bnxt_coal *hw_coal)
6092{
6093 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6094 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6095 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6096 u32 nq_params = coal_cap->nq_params;
6097 u16 tmr;
6098
6099 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6100 return 0;
6101
6102 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6103 -1, -1);
6104 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6105 req.flags =
6106 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6107
6108 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6109 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6110 req.int_lat_tmr_min = cpu_to_le16(tmr);
6111 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6112 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6113}
6114
6a8788f2
AG		 6115int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6116{
6117 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6118 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6119 struct bnxt_coal coal;
6a8788f2
AG		 6120
6121 /* Tick values in micro seconds.
6122 * 1 coal_buf x bufs_per_record = 1 completion record.
6123 */
6124 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6125
6126 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6127 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6128
6129 if (!bnapi->rx_ring)
6130 return -ENODEV;
6131
6132 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6133 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6134
74706afa 6135 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6a8788f2 6136
2c61d211 6137 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6a8788f2
AG		 6138
6139 return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6140 HWRM_CMD_TIMEOUT);
6141}
6142
c0c050c5
MC		 6143int bnxt_hwrm_set_coal(struct bnxt *bp)
6144{
6145 int i, rc = 0;
dfc9c94a
MC		 6146 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6147 req_tx = {0}, *req;
c0c050c5 6148
dfc9c94a
MC		 6149 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6150 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6151 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6152 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
c0c050c5 6153
74706afa
MC		 6154 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6155 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
c0c050c5
MC		 6156
6157 mutex_lock(&bp->hwrm_cmd_lock);
6158 for (i = 0; i < bp->cp_nr_rings; i++) {
dfc9c94a 6159 struct bnxt_napi *bnapi = bp->bnapi[i];
58590c8d 6160 struct bnxt_coal *hw_coal;
2c61d211 6161 u16 ring_id;
c0c050c5 6162
dfc9c94a 6163 req = &req_rx;
2c61d211
MC		 6164 if (!bnapi->rx_ring) {
6165 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
dfc9c94a 6166 req = &req_tx;
2c61d211
MC		 6167 } else {
6168 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6169 }
6170 req->ring_id = cpu_to_le16(ring_id);
dfc9c94a
MC		 6171
6172 rc = _hwrm_send_message(bp, req, sizeof(*req),
c0c050c5
MC		 6173 HWRM_CMD_TIMEOUT);
6174 if (rc)
6175 break;
58590c8d
MC		 6176
6177 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6178 continue;
6179
6180 if (bnapi->rx_ring && bnapi->tx_ring) {
6181 req = &req_tx;
6182 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6183 req->ring_id = cpu_to_le16(ring_id);
6184 rc = _hwrm_send_message(bp, req, sizeof(*req),
6185 HWRM_CMD_TIMEOUT);
6186 if (rc)
6187 break;
6188 }
6189 if (bnapi->rx_ring)
6190 hw_coal = &bp->rx_coal;
6191 else
6192 hw_coal = &bp->tx_coal;
6193 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
c0c050c5
MC		 6194 }
6195 mutex_unlock(&bp->hwrm_cmd_lock);
6196 return rc;
6197}
6198
6199static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6200{
6201 int rc = 0, i;
6202 struct hwrm_stat_ctx_free_input req = {0};
6203
6204 if (!bp->bnapi)
6205 return 0;
6206
3e8060fa
PS		 6207 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6208 return 0;
6209
c0c050c5
MC		 6210 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6211
6212 mutex_lock(&bp->hwrm_cmd_lock);
6213 for (i = 0; i < bp->cp_nr_rings; i++) {
6214 struct bnxt_napi *bnapi = bp->bnapi[i];
6215 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6216
6217 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6218 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6219
6220 rc = _hwrm_send_message(bp, &req, sizeof(req),
6221 HWRM_CMD_TIMEOUT);
6222 if (rc)
6223 break;
6224
6225 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6226 }
6227 }
6228 mutex_unlock(&bp->hwrm_cmd_lock);
6229 return rc;
6230}
6231
6232static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6233{
6234 int rc = 0, i;
6235 struct hwrm_stat_ctx_alloc_input req = {0};
6236 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6237
3e8060fa
PS		 6238 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6239 return 0;
6240
c0c050c5
MC		 6241 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6242
4e748506 6243 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
51f30785 6244 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
c0c050c5
MC		 6245
6246 mutex_lock(&bp->hwrm_cmd_lock);
6247 for (i = 0; i < bp->cp_nr_rings; i++) {
6248 struct bnxt_napi *bnapi = bp->bnapi[i];
6249 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6250
6251 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
6252
6253 rc = _hwrm_send_message(bp, &req, sizeof(req),
6254 HWRM_CMD_TIMEOUT);
6255 if (rc)
6256 break;
6257
6258 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6259
6260 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6261 }
6262 mutex_unlock(&bp->hwrm_cmd_lock);
89aa8445 6263 return rc;
c0c050c5
MC		 6264}
6265
cf6645f8
MC		 6266static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6267{
6268 struct hwrm_func_qcfg_input req = {0};
567b2abe 6269 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9315edca 6270 u16 flags;
cf6645f8
MC		 6271 int rc;
6272
6273 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6274 req.fid = cpu_to_le16(0xffff);
6275 mutex_lock(&bp->hwrm_cmd_lock);
6276 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6277 if (rc)
6278 goto func_qcfg_exit;
6279
6280#ifdef CONFIG_BNXT_SRIOV
6281 if (BNXT_VF(bp)) {
cf6645f8
MC		 6282 struct bnxt_vf_info *vf = &bp->vf;
6283
6284 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6285 }
6286#endif
9315edca
MC		 6287 flags = le16_to_cpu(resp->flags);
6288 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6289 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
97381a18 6290 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
9315edca 6291 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
97381a18 6292 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
9315edca
MC		 6293 }
6294 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6295 bp->flags |= BNXT_FLAG_MULTI_HOST;
bc39f885 6296
567b2abe
SB		 6297 switch (resp->port_partition_type) {
6298 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6299 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6300 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6301 bp->port_partition_type = resp->port_partition_type;
6302 break;
6303 }
32e8239c
MC		 6304 if (bp->hwrm_spec_code < 0x10707 ||
6305 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6306 bp->br_mode = BRIDGE_MODE_VEB;
6307 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6308 bp->br_mode = BRIDGE_MODE_VEPA;
6309 else
6310 bp->br_mode = BRIDGE_MODE_UNDEF;
cf6645f8 6311
7eb9bb3a
MC		 6312 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6313 if (!bp->max_mtu)
6314 bp->max_mtu = BNXT_MAX_MTU;
6315
cf6645f8
MC		 6316func_qcfg_exit:
6317 mutex_unlock(&bp->hwrm_cmd_lock);
6318 return rc;
6319}
6320
98f04cf0
MC		 6321static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6322{
6323 struct hwrm_func_backing_store_qcaps_input req = {0};
6324 struct hwrm_func_backing_store_qcaps_output *resp =
6325 bp->hwrm_cmd_resp_addr;
6326 int rc;
6327
6328 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6329 return 0;
6330
6331 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6332 mutex_lock(&bp->hwrm_cmd_lock);
6333 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6334 if (!rc) {
6335 struct bnxt_ctx_pg_info *ctx_pg;
6336 struct bnxt_ctx_mem_info *ctx;
6337 int i;
6338
6339 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6340 if (!ctx) {
6341 rc = -ENOMEM;
6342 goto ctx_err;
6343 }
6344 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6345 if (!ctx_pg) {
6346 kfree(ctx);
6347 rc = -ENOMEM;
6348 goto ctx_err;
6349 }
6350 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6351 ctx->tqm_mem[i] = ctx_pg;
6352
6353 bp->ctx = ctx;
6354 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6355 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6356 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6357 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6358 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6359 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6360 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6361 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6362 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6363 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6364 ctx->vnic_max_vnic_entries =
6365 le16_to_cpu(resp->vnic_max_vnic_entries);
6366 ctx->vnic_max_ring_table_entries =
6367 le16_to_cpu(resp->vnic_max_ring_table_entries);
6368 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6369 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6370 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6371 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6372 ctx->tqm_min_entries_per_ring =
6373 le32_to_cpu(resp->tqm_min_entries_per_ring);
6374 ctx->tqm_max_entries_per_ring =
6375 le32_to_cpu(resp->tqm_max_entries_per_ring);
6376 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6377 if (!ctx->tqm_entries_multiple)
6378 ctx->tqm_entries_multiple = 1;
6379 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6380 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
53579e37
DS		 6381 ctx->mrav_num_entries_units =
6382 le16_to_cpu(resp->mrav_num_entries_units);
98f04cf0
MC		 6383 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6384 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6385 } else {
6386 rc = 0;
6387 }
6388ctx_err:
6389 mutex_unlock(&bp->hwrm_cmd_lock);
6390 return rc;
6391}
6392
1b9394e5
MC		 6393static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6394 __le64 *pg_dir)
6395{
6396 u8 pg_size = 0;
6397
6398 if (BNXT_PAGE_SHIFT == 13)
6399 pg_size = 1 << 4;
6400 else if (BNXT_PAGE_SIZE == 16)
6401 pg_size = 2 << 4;
6402
6403 *pg_attr = pg_size;
08fe9d18
MC		 6404 if (rmem->depth >= 1) {
6405 if (rmem->depth == 2)
6406 *pg_attr |= 2;
6407 else
6408 *pg_attr |= 1;
1b9394e5
MC		 6409 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6410 } else {
6411 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6412 }
6413}
6414
6415#define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
6416 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
6417 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
6418 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
6419 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
6420 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6421
6422static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6423{
6424 struct hwrm_func_backing_store_cfg_input req = {0};
6425 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6426 struct bnxt_ctx_pg_info *ctx_pg;
6427 __le32 *num_entries;
6428 __le64 *pg_dir;
53579e37 6429 u32 flags = 0;
1b9394e5
MC		 6430 u8 *pg_attr;
6431 int i, rc;
6432 u32 ena;
6433
6434 if (!ctx)
6435 return 0;
6436
6437 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6438 req.enables = cpu_to_le32(enables);
6439
6440 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6441 ctx_pg = &ctx->qp_mem;
6442 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6443 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6444 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6445 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6446 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6447 &req.qpc_pg_size_qpc_lvl,
6448 &req.qpc_page_dir);
6449 }
6450 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6451 ctx_pg = &ctx->srq_mem;
6452 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6453 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6454 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6455 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6456 &req.srq_pg_size_srq_lvl,
6457 &req.srq_page_dir);
6458 }
6459 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6460 ctx_pg = &ctx->cq_mem;
6461 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6462 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6463 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6464 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6465 &req.cq_page_dir);
6466 }
6467 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6468 ctx_pg = &ctx->vnic_mem;
6469 req.vnic_num_vnic_entries =
6470 cpu_to_le16(ctx->vnic_max_vnic_entries);
6471 req.vnic_num_ring_table_entries =
6472 cpu_to_le16(ctx->vnic_max_ring_table_entries);
6473 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6474 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6475 &req.vnic_pg_size_vnic_lvl,
6476 &req.vnic_page_dir);
6477 }
6478 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6479 ctx_pg = &ctx->stat_mem;
6480 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6481 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6482 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6483 &req.stat_pg_size_stat_lvl,
6484 &req.stat_page_dir);
6485 }
cf6daed0
MC		 6486 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6487 ctx_pg = &ctx->mrav_mem;
6488 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
53579e37
DS		 6489 if (ctx->mrav_num_entries_units)
6490 flags |=
6491 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
cf6daed0
MC		 6492 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6493 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6494 &req.mrav_pg_size_mrav_lvl,
6495 &req.mrav_page_dir);
6496 }
6497 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6498 ctx_pg = &ctx->tim_mem;
6499 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6500 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6501 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6502 &req.tim_pg_size_tim_lvl,
6503 &req.tim_page_dir);
6504 }
1b9394e5
MC		 6505 for (i = 0, num_entries = &req.tqm_sp_num_entries,
6506 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6507 pg_dir = &req.tqm_sp_page_dir,
6508 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6509 i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6510 if (!(enables & ena))
6511 continue;
6512
6513 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6514 ctx_pg = ctx->tqm_mem[i];
6515 *num_entries = cpu_to_le32(ctx_pg->entries);
6516 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6517 }
53579e37 6518 req.flags = cpu_to_le32(flags);
1b9394e5 6519 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
1b9394e5
MC		 6520 return rc;
6521}
6522
98f04cf0 6523static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
08fe9d18 6524 struct bnxt_ctx_pg_info *ctx_pg)
98f04cf0
MC		 6525{
6526 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6527
98f04cf0
MC		 6528 rmem->page_size = BNXT_PAGE_SIZE;
6529 rmem->pg_arr = ctx_pg->ctx_pg_arr;
6530 rmem->dma_arr = ctx_pg->ctx_dma_arr;
1b9394e5 6531 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
08fe9d18
MC		 6532 if (rmem->depth >= 1)
6533 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
98f04cf0
MC		 6534 return bnxt_alloc_ring(bp, rmem);
6535}
6536
08fe9d18
MC		 6537static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6538 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6539 u8 depth)
6540{
6541 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6542 int rc;
6543
6544 if (!mem_size)
6545 return 0;
6546
6547 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6548 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6549 ctx_pg->nr_pages = 0;
6550 return -EINVAL;
6551 }
6552 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6553 int nr_tbls, i;
6554
6555 rmem->depth = 2;
6556 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6557 GFP_KERNEL);
6558 if (!ctx_pg->ctx_pg_tbl)
6559 return -ENOMEM;
6560 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6561 rmem->nr_pages = nr_tbls;
6562 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6563 if (rc)
6564 return rc;
6565 for (i = 0; i < nr_tbls; i++) {
6566 struct bnxt_ctx_pg_info *pg_tbl;
6567
6568 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6569 if (!pg_tbl)
6570 return -ENOMEM;
6571 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6572 rmem = &pg_tbl->ring_mem;
6573 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6574 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6575 rmem->depth = 1;
6576 rmem->nr_pages = MAX_CTX_PAGES;
6ef982de
MC		 6577 if (i == (nr_tbls - 1)) {
6578 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6579
6580 if (rem)
6581 rmem->nr_pages = rem;
6582 }
08fe9d18
MC		 6583 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6584 if (rc)
6585 break;
6586 }
6587 } else {
6588 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6589 if (rmem->nr_pages > 1 || depth)
6590 rmem->depth = 1;
6591 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6592 }
6593 return rc;
6594}
6595
6596static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6597 struct bnxt_ctx_pg_info *ctx_pg)
6598{
6599 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6600
6601 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6602 ctx_pg->ctx_pg_tbl) {
6603 int i, nr_tbls = rmem->nr_pages;
6604
6605 for (i = 0; i < nr_tbls; i++) {
6606 struct bnxt_ctx_pg_info *pg_tbl;
6607 struct bnxt_ring_mem_info *rmem2;
6608
6609 pg_tbl = ctx_pg->ctx_pg_tbl[i];
6610 if (!pg_tbl)
6611 continue;
6612 rmem2 = &pg_tbl->ring_mem;
6613 bnxt_free_ring(bp, rmem2);
6614 ctx_pg->ctx_pg_arr[i] = NULL;
6615 kfree(pg_tbl);
6616 ctx_pg->ctx_pg_tbl[i] = NULL;
6617 }
6618 kfree(ctx_pg->ctx_pg_tbl);
6619 ctx_pg->ctx_pg_tbl = NULL;
6620 }
6621 bnxt_free_ring(bp, rmem);
6622 ctx_pg->nr_pages = 0;
6623}
6624
98f04cf0
MC		 6625static void bnxt_free_ctx_mem(struct bnxt *bp)
6626{
6627 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6628 int i;
6629
6630 if (!ctx)
6631 return;
6632
6633 if (ctx->tqm_mem[0]) {
6634 for (i = 0; i < bp->max_q + 1; i++)
08fe9d18 6635 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
98f04cf0
MC		 6636 kfree(ctx->tqm_mem[0]);
6637 ctx->tqm_mem[0] = NULL;
6638 }
6639
cf6daed0
MC		 6640 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6641 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
08fe9d18
MC		 6642 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6643 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6644 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6645 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6646 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
98f04cf0
MC		 6647 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6648}
6649
6650static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6651{
6652 struct bnxt_ctx_pg_info *ctx_pg;
6653 struct bnxt_ctx_mem_info *ctx;
1b9394e5 6654 u32 mem_size, ena, entries;
53579e37 6655 u32 num_mr, num_ah;
cf6daed0
MC		 6656 u32 extra_srqs = 0;
6657 u32 extra_qps = 0;
6658 u8 pg_lvl = 1;
98f04cf0
MC		 6659 int i, rc;
6660
6661 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6662 if (rc) {
6663 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6664 rc);
6665 return rc;
6666 }
6667 ctx = bp->ctx;
6668 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6669 return 0;
6670
d629522e 6671 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
cf6daed0
MC		 6672 pg_lvl = 2;
6673 extra_qps = 65536;
6674 extra_srqs = 8192;
6675 }
6676
98f04cf0 6677 ctx_pg = &ctx->qp_mem;
cf6daed0
MC		 6678 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6679 extra_qps;
98f04cf0 6680 mem_size = ctx->qp_entry_size * ctx_pg->entries;
cf6daed0 6681 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6682 if (rc)
6683 return rc;
6684
6685 ctx_pg = &ctx->srq_mem;
cf6daed0 6686 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
98f04cf0 6687 mem_size = ctx->srq_entry_size * ctx_pg->entries;
cf6daed0 6688 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6689 if (rc)
6690 return rc;
6691
6692 ctx_pg = &ctx->cq_mem;
cf6daed0 6693 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
98f04cf0 6694 mem_size = ctx->cq_entry_size * ctx_pg->entries;
cf6daed0 6695 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
98f04cf0
MC		 6696 if (rc)
6697 return rc;
6698
6699 ctx_pg = &ctx->vnic_mem;
6700 ctx_pg->entries = ctx->vnic_max_vnic_entries +
6701 ctx->vnic_max_ring_table_entries;
6702 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
08fe9d18 6703 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6704 if (rc)
6705 return rc;
6706
6707 ctx_pg = &ctx->stat_mem;
6708 ctx_pg->entries = ctx->stat_max_entries;
6709 mem_size = ctx->stat_entry_size * ctx_pg->entries;
08fe9d18 6710 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6711 if (rc)
6712 return rc;
6713
cf6daed0
MC		 6714 ena = 0;
6715 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6716 goto skip_rdma;
6717
6718 ctx_pg = &ctx->mrav_mem;
53579e37
DS		 6719 /* 128K extra is needed to accommodate static AH context
6720 * allocation by f/w.
6721 */
6722 num_mr = 1024 * 256;
6723 num_ah = 1024 * 128;
6724 ctx_pg->entries = num_mr + num_ah;
cf6daed0
MC		 6725 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6726 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2);
6727 if (rc)
6728 return rc;
6729 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
53579e37
DS		 6730 if (ctx->mrav_num_entries_units)
6731 ctx_pg->entries =
6732 ((num_mr / ctx->mrav_num_entries_units) << 16) |
6733 (num_ah / ctx->mrav_num_entries_units);
cf6daed0
MC		 6734
6735 ctx_pg = &ctx->tim_mem;
6736 ctx_pg->entries = ctx->qp_mem.entries;
6737 mem_size = ctx->tim_entry_size * ctx_pg->entries;
6738 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6739 if (rc)
6740 return rc;
6741 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6742
6743skip_rdma:
6744 entries = ctx->qp_max_l2_entries + extra_qps;
98f04cf0
MC		 6745 entries = roundup(entries, ctx->tqm_entries_multiple);
6746 entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6747 ctx->tqm_max_entries_per_ring);
cf6daed0 6748 for (i = 0; i < bp->max_q + 1; i++) {
98f04cf0
MC		 6749 ctx_pg = ctx->tqm_mem[i];
6750 ctx_pg->entries = entries;
6751 mem_size = ctx->tqm_entry_size * entries;
08fe9d18 6752 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
98f04cf0
MC		 6753 if (rc)
6754 return rc;
1b9394e5 6755 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
98f04cf0 6756 }
1b9394e5
MC		 6757 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6758 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6759 if (rc)
6760 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6761 rc);
6762 else
6763 ctx->flags |= BNXT_CTX_FLAG_INITED;
6764
98f04cf0
MC		 6765 return 0;
6766}
6767
db4723b3 6768int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
be0dd9c4
MC		 6769{
6770 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6771 struct hwrm_func_resource_qcaps_input req = {0};
6772 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6773 int rc;
6774
6775 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6776 req.fid = cpu_to_le16(0xffff);
6777
6778 mutex_lock(&bp->hwrm_cmd_lock);
351cbde9
JT		 6779 rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6780 HWRM_CMD_TIMEOUT);
d4f1420d 6781 if (rc)
be0dd9c4 6782 goto hwrm_func_resc_qcaps_exit;
be0dd9c4 6783
db4723b3
MC		 6784 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6785 if (!all)
6786 goto hwrm_func_resc_qcaps_exit;
6787
be0dd9c4
MC		 6788 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6789 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6790 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6791 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6792 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6793 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6794 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6795 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6796 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6797 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6798 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6799 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6800 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6801 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6802 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6803 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6804
9c1fabdf
MC		 6805 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6806 u16 max_msix = le16_to_cpu(resp->max_msix);
6807
f7588cd8 6808 hw_resc->max_nqs = max_msix;
9c1fabdf
MC		 6809 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6810 }
6811
4673d664
MC		 6812 if (BNXT_PF(bp)) {
6813 struct bnxt_pf_info *pf = &bp->pf;
6814
6815 pf->vf_resv_strategy =
6816 le16_to_cpu(resp->vf_reservation_strategy);
bf82736d 6817 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
4673d664
MC		 6818 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6819 }
be0dd9c4
MC		 6820hwrm_func_resc_qcaps_exit:
6821 mutex_unlock(&bp->hwrm_cmd_lock);
6822 return rc;
6823}
6824
6825static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
c0c050c5
MC		 6826{
6827 int rc = 0;
6828 struct hwrm_func_qcaps_input req = {0};
6829 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6a4f2947
MC		 6830 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6831 u32 flags;
c0c050c5
MC		 6832
6833 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6834 req.fid = cpu_to_le16(0xffff);
6835
6836 mutex_lock(&bp->hwrm_cmd_lock);
6837 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6838 if (rc)
6839 goto hwrm_func_qcaps_exit;
6840
6a4f2947
MC		 6841 flags = le32_to_cpu(resp->flags);
6842 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
e4060d30 6843 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6a4f2947 6844 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
e4060d30 6845 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
55e4398d
VV		 6846 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
6847 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
6154532f
VV		 6848 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
6849 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
07f83d72
MC		 6850 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
6851 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
e4060d30 6852
7cc5a20e 6853 bp->tx_push_thresh = 0;
6a4f2947 6854 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
7cc5a20e
MC		 6855 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6856
6a4f2947
MC		 6857 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6858 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6859 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6860 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6861 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6862 if (!hw_resc->max_hw_ring_grps)
6863 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6864 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6865 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6866 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6867
c0c050c5
MC		 6868 if (BNXT_PF(bp)) {
6869 struct bnxt_pf_info *pf = &bp->pf;
6870
6871 pf->fw_fid = le16_to_cpu(resp->fid);
6872 pf->port_id = le16_to_cpu(resp->port_id);
87027db1 6873 bp->dev->dev_port = pf->port_id;
11f15ed3 6874 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
c0c050c5
MC		 6875 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6876 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6877 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6878 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6879 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6880 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6881 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6882 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
ba642ab7 6883 bp->flags &= ~BNXT_FLAG_WOL_CAP;
6a4f2947 6884 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
c1ef146a 6885 bp->flags |= BNXT_FLAG_WOL_CAP;
c0c050c5 6886 } else {
379a80a1 6887#ifdef CONFIG_BNXT_SRIOV
c0c050c5
MC		 6888 struct bnxt_vf_info *vf = &bp->vf;
6889
6890 vf->fw_fid = le16_to_cpu(resp->fid);
7cc5a20e 6891 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
379a80a1 6892#endif
c0c050c5
MC		 6893 }
6894
c0c050c5
MC		 6895hwrm_func_qcaps_exit:
6896 mutex_unlock(&bp->hwrm_cmd_lock);
6897 return rc;
6898}
6899
804fba4e
MC		 6900static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
6901
be0dd9c4
MC		 6902static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
6903{
6904 int rc;
6905
6906 rc = __bnxt_hwrm_func_qcaps(bp);
6907 if (rc)
6908 return rc;
804fba4e
MC		 6909 rc = bnxt_hwrm_queue_qportcfg(bp);
6910 if (rc) {
6911 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
6912 return rc;
6913 }
be0dd9c4 6914 if (bp->hwrm_spec_code >= 0x10803) {
98f04cf0
MC		 6915 rc = bnxt_alloc_ctx_mem(bp);
6916 if (rc)
6917 return rc;
db4723b3 6918 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
be0dd9c4 6919 if (!rc)
97381a18 6920 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
be0dd9c4
MC		 6921 }
6922 return 0;
6923}
6924
e969ae5b
MC		 6925static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
6926{
6927 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
6928 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
6929 int rc = 0;
6930 u32 flags;
6931
6932 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
6933 return 0;
6934
6935 resp = bp->hwrm_cmd_resp_addr;
6936 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
6937
6938 mutex_lock(&bp->hwrm_cmd_lock);
6939 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6940 if (rc)
6941 goto hwrm_cfa_adv_qcaps_exit;
6942
6943 flags = le32_to_cpu(resp->flags);
6944 if (flags &
6945 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED)
6946 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX;
6947
6948hwrm_cfa_adv_qcaps_exit:
6949 mutex_unlock(&bp->hwrm_cmd_lock);
6950 return rc;
6951}
6952
9ffbd677
MC		 6953static int bnxt_map_fw_health_regs(struct bnxt *bp)
6954{
6955 struct bnxt_fw_health *fw_health = bp->fw_health;
6956 u32 reg_base = 0xffffffff;
6957 int i;
6958
6959 /* Only pre-map the monitoring GRC registers using window 3 */
6960 for (i = 0; i < 4; i++) {
6961 u32 reg = fw_health->regs[i];
6962
6963 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
6964 continue;
6965 if (reg_base == 0xffffffff)
6966 reg_base = reg & BNXT_GRC_BASE_MASK;
6967 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
6968 return -ERANGE;
6969 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
6970 (reg & BNXT_GRC_OFFSET_MASK);
6971 }
6972 if (reg_base == 0xffffffff)
6973 return 0;
6974
6975 writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
6976 BNXT_FW_HEALTH_WIN_MAP_OFF);
6977 return 0;
6978}
6979
07f83d72
MC		 6980static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
6981{
6982 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6983 struct bnxt_fw_health *fw_health = bp->fw_health;
6984 struct hwrm_error_recovery_qcfg_input req = {0};
6985 int rc, i;
6986
6987 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
6988 return 0;
6989
6990 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
6991 mutex_lock(&bp->hwrm_cmd_lock);
6992 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6993 if (rc)
6994 goto err_recovery_out;
6995 if (!fw_health) {
6996 fw_health = kzalloc(sizeof(*fw_health), GFP_KERNEL);
6997 bp->fw_health = fw_health;
6998 if (!fw_health) {
6999 rc = -ENOMEM;
7000 goto err_recovery_out;
7001 }
7002 }
7003 fw_health->flags = le32_to_cpu(resp->flags);
7004 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7005 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7006 rc = -EINVAL;
7007 goto err_recovery_out;
7008 }
7009 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7010 fw_health->master_func_wait_dsecs =
7011 le32_to_cpu(resp->master_func_wait_period);
7012 fw_health->normal_func_wait_dsecs =
7013 le32_to_cpu(resp->normal_func_wait_period);
7014 fw_health->post_reset_wait_dsecs =
7015 le32_to_cpu(resp->master_func_wait_period_after_reset);
7016 fw_health->post_reset_max_wait_dsecs =
7017 le32_to_cpu(resp->max_bailout_time_after_reset);
7018 fw_health->regs[BNXT_FW_HEALTH_REG] =
7019 le32_to_cpu(resp->fw_health_status_reg);
7020 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7021 le32_to_cpu(resp->fw_heartbeat_reg);
7022 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7023 le32_to_cpu(resp->fw_reset_cnt_reg);
7024 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7025 le32_to_cpu(resp->reset_inprogress_reg);
7026 fw_health->fw_reset_inprog_reg_mask =
7027 le32_to_cpu(resp->reset_inprogress_reg_mask);
7028 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7029 if (fw_health->fw_reset_seq_cnt >= 16) {
7030 rc = -EINVAL;
7031 goto err_recovery_out;
7032 }
7033 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7034 fw_health->fw_reset_seq_regs[i] =
7035 le32_to_cpu(resp->reset_reg[i]);
7036 fw_health->fw_reset_seq_vals[i] =
7037 le32_to_cpu(resp->reset_reg_val[i]);
7038 fw_health->fw_reset_seq_delay_msec[i] =
7039 resp->delay_after_reset[i];
7040 }
7041err_recovery_out:
7042 mutex_unlock(&bp->hwrm_cmd_lock);
9ffbd677
MC		 7043 if (!rc)
7044 rc = bnxt_map_fw_health_regs(bp);
07f83d72
MC		 7045 if (rc)
7046 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7047 return rc;
7048}
7049
c0c050c5
MC		 7050static int bnxt_hwrm_func_reset(struct bnxt *bp)
7051{
7052 struct hwrm_func_reset_input req = {0};
7053
7054 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7055 req.enables = 0;
7056
7057 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7058}
7059
7060static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7061{
7062 int rc = 0;
7063 struct hwrm_queue_qportcfg_input req = {0};
7064 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
aabfc016
MC		 7065 u8 i, j, *qptr;
7066 bool no_rdma;
c0c050c5
MC		 7067
7068 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7069
7070 mutex_lock(&bp->hwrm_cmd_lock);
7071 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7072 if (rc)
7073 goto qportcfg_exit;
7074
7075 if (!resp->max_configurable_queues) {
7076 rc = -EINVAL;
7077 goto qportcfg_exit;
7078 }
7079 bp->max_tc = resp->max_configurable_queues;
87c374de 7080 bp->max_lltc = resp->max_configurable_lossless_queues;
c0c050c5
MC		 7081 if (bp->max_tc > BNXT_MAX_QUEUE)
7082 bp->max_tc = BNXT_MAX_QUEUE;
7083
aabfc016
MC		 7084 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7085 qptr = &resp->queue_id0;
7086 for (i = 0, j = 0; i < bp->max_tc; i++) {
98f04cf0
MC		 7087 bp->q_info[j].queue_id = *qptr;
7088 bp->q_ids[i] = *qptr++;
aabfc016
MC		 7089 bp->q_info[j].queue_profile = *qptr++;
7090 bp->tc_to_qidx[j] = j;
7091 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7092 (no_rdma && BNXT_PF(bp)))
7093 j++;
7094 }
98f04cf0 7095 bp->max_q = bp->max_tc;
aabfc016
MC		 7096 bp->max_tc = max_t(u8, j, 1);
7097
441cabbb
MC		 7098 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7099 bp->max_tc = 1;
7100
87c374de
MC		 7101 if (bp->max_lltc > bp->max_tc)
7102 bp->max_lltc = bp->max_tc;
7103
c0c050c5
MC		 7104qportcfg_exit:
7105 mutex_unlock(&bp->hwrm_cmd_lock);
7106 return rc;
7107}
7108
ba642ab7 7109static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
c0c050c5 7110{
c0c050c5 7111 struct hwrm_ver_get_input req = {0};
ba642ab7 7112 int rc;
c0c050c5
MC		 7113
7114 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7115 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7116 req.hwrm_intf_min = HWRM_VERSION_MINOR;
7117 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
ba642ab7
MC		 7118
7119 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7120 silent);
7121 return rc;
7122}
7123
7124static int bnxt_hwrm_ver_get(struct bnxt *bp)
7125{
7126 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7127 u32 dev_caps_cfg;
7128 int rc;
7129
7130 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
c0c050c5 7131 mutex_lock(&bp->hwrm_cmd_lock);
ba642ab7 7132 rc = __bnxt_hwrm_ver_get(bp, false);
c0c050c5
MC		 7133 if (rc)
7134 goto hwrm_ver_get_exit;
7135
7136 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7137
894aa69a
MC		 7138 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7139 resp->hwrm_intf_min_8b << 8 |
7140 resp->hwrm_intf_upd_8b;
7141 if (resp->hwrm_intf_maj_8b < 1) {
c193554e 7142 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
894aa69a
MC		 7143 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7144 resp->hwrm_intf_upd_8b);
c193554e 7145 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
c0c050c5 7146 }
431aa1eb 7147 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
894aa69a
MC		 7148 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
7149 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
c0c050c5 7150
691aa620
VV		 7151 if (strlen(resp->active_pkg_name)) {
7152 int fw_ver_len = strlen(bp->fw_ver_str);
7153
7154 snprintf(bp->fw_ver_str + fw_ver_len,
7155 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7156 resp->active_pkg_name);
7157 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7158 }
7159
ff4fe81d
MC		 7160 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7161 if (!bp->hwrm_cmd_timeout)
7162 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7163
1dfddc41 7164 if (resp->hwrm_intf_maj_8b >= 1) {
e6ef2699 7165 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
1dfddc41
MC		 7166 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7167 }
7168 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7169 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
e6ef2699 7170
659c805c 7171 bp->chip_num = le16_to_cpu(resp->chip_num);
3e8060fa
PS		 7172 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7173 !resp->chip_metal)
7174 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
659c805c 7175
e605db80
DK		 7176 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7177 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7178 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
97381a18 7179 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
e605db80 7180
760b6d33
VD		 7181 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7182 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7183
abd43a13
VD		 7184 if (dev_caps_cfg &
7185 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7186 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7187
2a516444
MC		 7188 if (dev_caps_cfg &
7189 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7190 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7191
e969ae5b
MC		 7192 if (dev_caps_cfg &
7193 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7194 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7195
c0c050c5
MC		 7196hwrm_ver_get_exit:
7197 mutex_unlock(&bp->hwrm_cmd_lock);
7198 return rc;
7199}
7200
5ac67d8b
RS		 7201int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7202{
7203 struct hwrm_fw_set_time_input req = {0};
7dfaa7bc
AB		 7204 struct tm tm;
7205 time64_t now = ktime_get_real_seconds();
5ac67d8b 7206
ca2c39e2
MC		 7207 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7208 bp->hwrm_spec_code < 0x10400)
5ac67d8b
RS		 7209 return -EOPNOTSUPP;
7210
7dfaa7bc 7211 time64_to_tm(now, 0, &tm);
5ac67d8b
RS		 7212 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7213 req.year = cpu_to_le16(1900 + tm.tm_year);
7214 req.month = 1 + tm.tm_mon;
7215 req.day = tm.tm_mday;
7216 req.hour = tm.tm_hour;
7217 req.minute = tm.tm_min;
7218 req.second = tm.tm_sec;
7219 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7220}
7221
3bdf56c4
MC		 7222static int bnxt_hwrm_port_qstats(struct bnxt *bp)
7223{
7224 int rc;
7225 struct bnxt_pf_info *pf = &bp->pf;
7226 struct hwrm_port_qstats_input req = {0};
7227
7228 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7229 return 0;
7230
7231 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7232 req.port_id = cpu_to_le16(pf->port_id);
7233 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
7234 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
7235 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7236 return rc;
7237}
7238
00db3cba
VV		 7239static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
7240{
36e53349 7241 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
e37fed79 7242 struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
00db3cba
VV		 7243 struct hwrm_port_qstats_ext_input req = {0};
7244 struct bnxt_pf_info *pf = &bp->pf;
ad361adf 7245 u32 tx_stat_size;
36e53349 7246 int rc;
00db3cba
VV		 7247
7248 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7249 return 0;
7250
7251 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7252 req.port_id = cpu_to_le16(pf->port_id);
7253 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7254 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
ad361adf
MC		 7255 tx_stat_size = bp->hw_tx_port_stats_ext ?
7256 sizeof(*bp->hw_tx_port_stats_ext) : 0;
7257 req.tx_stat_size = cpu_to_le16(tx_stat_size);
36e53349
MC		 7258 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
7259 mutex_lock(&bp->hwrm_cmd_lock);
7260 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7261 if (!rc) {
7262 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
ad361adf
MC		 7263 bp->fw_tx_stats_ext_size = tx_stat_size ?
7264 le16_to_cpu(resp->tx_stat_size) / 8 : 0;
36e53349
MC		 7265 } else {
7266 bp->fw_rx_stats_ext_size = 0;
7267 bp->fw_tx_stats_ext_size = 0;
7268 }
e37fed79
MC		 7269 if (bp->fw_tx_stats_ext_size <=
7270 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7271 mutex_unlock(&bp->hwrm_cmd_lock);
7272 bp->pri2cos_valid = 0;
7273 return rc;
7274 }
7275
7276 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7277 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7278
7279 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7280 if (!rc) {
7281 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7282 u8 *pri2cos;
7283 int i, j;
7284
7285 resp2 = bp->hwrm_cmd_resp_addr;
7286 pri2cos = &resp2->pri0_cos_queue_id;
7287 for (i = 0; i < 8; i++) {
7288 u8 queue_id = pri2cos[i];
7289
7290 for (j = 0; j < bp->max_q; j++) {
7291 if (bp->q_ids[j] == queue_id)
7292 bp->pri2cos[i] = j;
7293 }
7294 }
7295 bp->pri2cos_valid = 1;
7296 }
36e53349
MC		 7297 mutex_unlock(&bp->hwrm_cmd_lock);
7298 return rc;
00db3cba
VV		 7299}
7300
55e4398d
VV		 7301static int bnxt_hwrm_pcie_qstats(struct bnxt *bp)
7302{
7303 struct hwrm_pcie_qstats_input req = {0};
7304
7305 if (!(bp->flags & BNXT_FLAG_PCIE_STATS))
7306 return 0;
7307
7308 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1);
7309 req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats));
7310 req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map);
7311 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7312}
7313
c0c050c5
MC		 7314static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7315{
7316 if (bp->vxlan_port_cnt) {
7317 bnxt_hwrm_tunnel_dst_port_free(
7318 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7319 }
7320 bp->vxlan_port_cnt = 0;
7321 if (bp->nge_port_cnt) {
7322 bnxt_hwrm_tunnel_dst_port_free(
7323 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7324 }
7325 bp->nge_port_cnt = 0;
7326}
7327
7328static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7329{
7330 int rc, i;
7331 u32 tpa_flags = 0;
7332
7333 if (set_tpa)
7334 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7335 for (i = 0; i < bp->nr_vnics; i++) {
7336 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7337 if (rc) {
7338 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
23e12c89 7339 i, rc);
c0c050c5
MC		 7340 return rc;
7341 }
7342 }
7343 return 0;
7344}
7345
7346static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7347{
7348 int i;
7349
7350 for (i = 0; i < bp->nr_vnics; i++)
7351 bnxt_hwrm_vnic_set_rss(bp, i, false);
7352}
7353
a46ecb11 7354static void bnxt_clear_vnic(struct bnxt *bp)
c0c050c5 7355{
a46ecb11
MC		 7356 if (!bp->vnic_info)
7357 return;
7358
7359 bnxt_hwrm_clear_vnic_filter(bp);
7360 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
c0c050c5
MC		 7361 /* clear all RSS setting before free vnic ctx */
7362 bnxt_hwrm_clear_vnic_rss(bp);
7363 bnxt_hwrm_vnic_ctx_free(bp);
c0c050c5 7364 }
a46ecb11
MC		 7365 /* before free the vnic, undo the vnic tpa settings */
7366 if (bp->flags & BNXT_FLAG_TPA)
7367 bnxt_set_tpa(bp, false);
7368 bnxt_hwrm_vnic_free(bp);
7369 if (bp->flags & BNXT_FLAG_CHIP_P5)
7370 bnxt_hwrm_vnic_ctx_free(bp);
7371}
7372
7373static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7374 bool irq_re_init)
7375{
7376 bnxt_clear_vnic(bp);
c0c050c5
MC		 7377 bnxt_hwrm_ring_free(bp, close_path);
7378 bnxt_hwrm_ring_grp_free(bp);
7379 if (irq_re_init) {
7380 bnxt_hwrm_stat_ctx_free(bp);
7381 bnxt_hwrm_free_tunnel_ports(bp);
7382 }
7383}
7384
39d8ba2e
MC		 7385static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7386{
7387 struct hwrm_func_cfg_input req = {0};
7388 int rc;
7389
7390 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7391 req.fid = cpu_to_le16(0xffff);
7392 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7393 if (br_mode == BRIDGE_MODE_VEB)
7394 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7395 else if (br_mode == BRIDGE_MODE_VEPA)
7396 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7397 else
7398 return -EINVAL;
7399 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
39d8ba2e
MC		 7400 return rc;
7401}
7402
c3480a60
MC		 7403static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7404{
7405 struct hwrm_func_cfg_input req = {0};
7406 int rc;
7407
7408 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7409 return 0;
7410
7411 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7412 req.fid = cpu_to_le16(0xffff);
7413 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
d4f52de0 7414 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
c3480a60 7415 if (size == 128)
d4f52de0 7416 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
c3480a60
MC		 7417
7418 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
c3480a60
MC		 7419 return rc;
7420}
7421
7b3af4f7 7422static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
c0c050c5 7423{
ae10ae74 7424 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
c0c050c5
MC		 7425 int rc;
7426
ae10ae74
MC		 7427 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7428 goto skip_rss_ctx;
7429
c0c050c5 7430 /* allocate context for vnic */
94ce9caa 7431 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
c0c050c5
MC		 7432 if (rc) {
7433 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7434 vnic_id, rc);
7435 goto vnic_setup_err;
7436 }
7437 bp->rsscos_nr_ctxs++;
7438
94ce9caa
PS		 7439 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7440 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7441 if (rc) {
7442 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7443 vnic_id, rc);
7444 goto vnic_setup_err;
7445 }
7446 bp->rsscos_nr_ctxs++;
7447 }
7448
ae10ae74 7449skip_rss_ctx:
c0c050c5
MC		 7450 /* configure default vnic, ring grp */
7451 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7452 if (rc) {
7453 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7454 vnic_id, rc);
7455 goto vnic_setup_err;
7456 }
7457
7458 /* Enable RSS hashing on vnic */
7459 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7460 if (rc) {
7461 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7462 vnic_id, rc);
7463 goto vnic_setup_err;
7464 }
7465
7466 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7467 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7468 if (rc) {
7469 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7470 vnic_id, rc);
7471 }
7472 }
7473
7474vnic_setup_err:
7475 return rc;
7476}
7477
7b3af4f7
MC		 7478static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7479{
7480 int rc, i, nr_ctxs;
7481
7482 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
7483 for (i = 0; i < nr_ctxs; i++) {
7484 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7485 if (rc) {
7486 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7487 vnic_id, i, rc);
7488 break;
7489 }
7490 bp->rsscos_nr_ctxs++;
7491 }
7492 if (i < nr_ctxs)
7493 return -ENOMEM;
7494
7495 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7496 if (rc) {
7497 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7498 vnic_id, rc);
7499 return rc;
7500 }
7501 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7502 if (rc) {
7503 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7504 vnic_id, rc);
7505 return rc;
7506 }
7507 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7508 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7509 if (rc) {
7510 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7511 vnic_id, rc);
7512 }
7513 }
7514 return rc;
7515}
7516
7517static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7518{
7519 if (bp->flags & BNXT_FLAG_CHIP_P5)
7520 return __bnxt_setup_vnic_p5(bp, vnic_id);
7521 else
7522 return __bnxt_setup_vnic(bp, vnic_id);
7523}
7524
c0c050c5
MC		 7525static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
7526{
7527#ifdef CONFIG_RFS_ACCEL
7528 int i, rc = 0;
7529
9b3d15e6
MC		 7530 if (bp->flags & BNXT_FLAG_CHIP_P5)
7531 return 0;
7532
c0c050c5 7533 for (i = 0; i < bp->rx_nr_rings; i++) {
ae10ae74 7534 struct bnxt_vnic_info *vnic;
c0c050c5
MC		 7535 u16 vnic_id = i + 1;
7536 u16 ring_id = i;
7537
7538 if (vnic_id >= bp->nr_vnics)
7539 break;
7540
ae10ae74
MC		 7541 vnic = &bp->vnic_info[vnic_id];
7542 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7543 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7544 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
b81a90d3 7545 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
c0c050c5
MC		 7546 if (rc) {
7547 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7548 vnic_id, rc);
7549 break;
7550 }
7551 rc = bnxt_setup_vnic(bp, vnic_id);
7552 if (rc)
7553 break;
7554 }
7555 return rc;
7556#else
7557 return 0;
7558#endif
7559}
7560
17c71ac3
MC		 7561/* Allow PF and VF with default VLAN to be in promiscuous mode */
7562static bool bnxt_promisc_ok(struct bnxt *bp)
7563{
7564#ifdef CONFIG_BNXT_SRIOV
7565 if (BNXT_VF(bp) && !bp->vf.vlan)
7566 return false;
7567#endif
7568 return true;
7569}
7570
dc52c6c7
PS		 7571static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7572{
7573 unsigned int rc = 0;
7574
7575 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7576 if (rc) {
7577 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7578 rc);
7579 return rc;
7580 }
7581
7582 rc = bnxt_hwrm_vnic_cfg(bp, 1);
7583 if (rc) {
7584 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7585 rc);
7586 return rc;
7587 }
7588 return rc;
7589}
7590
b664f008 7591static int bnxt_cfg_rx_mode(struct bnxt *);
7d2837dd 7592static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
b664f008 7593
c0c050c5
MC		 7594static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7595{
7d2837dd 7596 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
c0c050c5 7597 int rc = 0;
76595193 7598 unsigned int rx_nr_rings = bp->rx_nr_rings;
c0c050c5
MC		 7599
7600 if (irq_re_init) {
7601 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7602 if (rc) {
7603 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7604 rc);
7605 goto err_out;
7606 }
7607 }
7608
7609 rc = bnxt_hwrm_ring_alloc(bp);
7610 if (rc) {
7611 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7612 goto err_out;
7613 }
7614
7615 rc = bnxt_hwrm_ring_grp_alloc(bp);
7616 if (rc) {
7617 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7618 goto err_out;
7619 }
7620
76595193
PS		 7621 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7622 rx_nr_rings--;
7623
c0c050c5 7624 /* default vnic 0 */
76595193 7625 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
c0c050c5
MC		 7626 if (rc) {
7627 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7628 goto err_out;
7629 }
7630
7631 rc = bnxt_setup_vnic(bp, 0);
7632 if (rc)
7633 goto err_out;
7634
7635 if (bp->flags & BNXT_FLAG_RFS) {
7636 rc = bnxt_alloc_rfs_vnics(bp);
7637 if (rc)
7638 goto err_out;
7639 }
7640
7641 if (bp->flags & BNXT_FLAG_TPA) {
7642 rc = bnxt_set_tpa(bp, true);
7643 if (rc)
7644 goto err_out;
7645 }
7646
7647 if (BNXT_VF(bp))
7648 bnxt_update_vf_mac(bp);
7649
7650 /* Filter for default vnic 0 */
7651 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7652 if (rc) {
7653 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7654 goto err_out;
7655 }
7d2837dd 7656 vnic->uc_filter_count = 1;
c0c050c5 7657
30e33848
MC		 7658 vnic->rx_mask = 0;
7659 if (bp->dev->flags & IFF_BROADCAST)
7660 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5 7661
17c71ac3 7662 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7d2837dd
MC		 7663 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7664
7665 if (bp->dev->flags & IFF_ALLMULTI) {
7666 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7667 vnic->mc_list_count = 0;
7668 } else {
7669 u32 mask = 0;
7670
7671 bnxt_mc_list_updated(bp, &mask);
7672 vnic->rx_mask |= mask;
7673 }
c0c050c5 7674
b664f008
MC		 7675 rc = bnxt_cfg_rx_mode(bp);
7676 if (rc)
c0c050c5 7677 goto err_out;
c0c050c5
MC		 7678
7679 rc = bnxt_hwrm_set_coal(bp);
7680 if (rc)
7681 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
dc52c6c7
PS		 7682 rc);
7683
7684 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7685 rc = bnxt_setup_nitroa0_vnic(bp);
7686 if (rc)
7687 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7688 rc);
7689 }
c0c050c5 7690
cf6645f8
MC		 7691 if (BNXT_VF(bp)) {
7692 bnxt_hwrm_func_qcfg(bp);
7693 netdev_update_features(bp->dev);
7694 }
7695
c0c050c5
MC		 7696 return 0;
7697
7698err_out:
7699 bnxt_hwrm_resource_free(bp, 0, true);
7700
7701 return rc;
7702}
7703
7704static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7705{
7706 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7707 return 0;
7708}
7709
7710static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7711{
2247925f 7712 bnxt_init_cp_rings(bp);
c0c050c5
MC		 7713 bnxt_init_rx_rings(bp);
7714 bnxt_init_tx_rings(bp);
7715 bnxt_init_ring_grps(bp, irq_re_init);
7716 bnxt_init_vnics(bp);
7717
7718 return bnxt_init_chip(bp, irq_re_init);
7719}
7720
c0c050c5
MC		 7721static int bnxt_set_real_num_queues(struct bnxt *bp)
7722{
7723 int rc;
7724 struct net_device *dev = bp->dev;
7725
5f449249
MC		 7726 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7727 bp->tx_nr_rings_xdp);
c0c050c5
MC		 7728 if (rc)
7729 return rc;
7730
7731 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7732 if (rc)
7733 return rc;
7734
7735#ifdef CONFIG_RFS_ACCEL
45019a18 7736 if (bp->flags & BNXT_FLAG_RFS)
c0c050c5 7737 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
c0c050c5
MC		 7738#endif
7739
7740 return rc;
7741}
7742
6e6c5a57
MC		 7743static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7744 bool shared)
7745{
7746 int _rx = *rx, _tx = *tx;
7747
7748 if (shared) {
7749 *rx = min_t(int, _rx, max);
7750 *tx = min_t(int, _tx, max);
7751 } else {
7752 if (max < 2)
7753 return -ENOMEM;
7754
7755 while (_rx + _tx > max) {
7756 if (_rx > _tx && _rx > 1)
7757 _rx--;
7758 else if (_tx > 1)
7759 _tx--;
7760 }
7761 *rx = _rx;
7762 *tx = _tx;
7763 }
7764 return 0;
7765}
7766
7809592d
MC		 7767static void bnxt_setup_msix(struct bnxt *bp)
7768{
7769 const int len = sizeof(bp->irq_tbl[0].name);
7770 struct net_device *dev = bp->dev;
7771 int tcs, i;
7772
7773 tcs = netdev_get_num_tc(dev);
7774 if (tcs > 1) {
d1e7925e 7775 int i, off, count;
7809592d 7776
d1e7925e
MC		 7777 for (i = 0; i < tcs; i++) {
7778 count = bp->tx_nr_rings_per_tc;
7779 off = i * count;
7780 netdev_set_tc_queue(dev, i, count, off);
7809592d
MC		 7781 }
7782 }
7783
7784 for (i = 0; i < bp->cp_nr_rings; i++) {
e5811b8c 7785 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7809592d
MC		 7786 char *attr;
7787
7788 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7789 attr = "TxRx";
7790 else if (i < bp->rx_nr_rings)
7791 attr = "rx";
7792 else
7793 attr = "tx";
7794
e5811b8c
MC		 7795 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7796 attr, i);
7797 bp->irq_tbl[map_idx].handler = bnxt_msix;
7809592d
MC		 7798 }
7799}
7800
7801static void bnxt_setup_inta(struct bnxt *bp)
7802{
7803 const int len = sizeof(bp->irq_tbl[0].name);
7804
7805 if (netdev_get_num_tc(bp->dev))
7806 netdev_reset_tc(bp->dev);
7807
7808 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7809 0);
7810 bp->irq_tbl[0].handler = bnxt_inta;
7811}
7812
7813static int bnxt_setup_int_mode(struct bnxt *bp)
7814{
7815 int rc;
7816
7817 if (bp->flags & BNXT_FLAG_USING_MSIX)
7818 bnxt_setup_msix(bp);
7819 else
7820 bnxt_setup_inta(bp);
7821
7822 rc = bnxt_set_real_num_queues(bp);
7823 return rc;
7824}
7825
b7429954 7826#ifdef CONFIG_RFS_ACCEL
8079e8f1
MC		 7827static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7828{
6a4f2947 7829 return bp->hw_resc.max_rsscos_ctxs;
8079e8f1
MC		 7830}
7831
7832static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7833{
6a4f2947 7834 return bp->hw_resc.max_vnics;
8079e8f1 7835}
b7429954 7836#endif
8079e8f1 7837
e4060d30
MC		 7838unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7839{
6a4f2947 7840 return bp->hw_resc.max_stat_ctxs;
e4060d30
MC		 7841}
7842
7843unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7844{
6a4f2947 7845 return bp->hw_resc.max_cp_rings;
e4060d30
MC		 7846}
7847
e916b081 7848static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
a588e458 7849{
c0b8cda0
MC		 7850 unsigned int cp = bp->hw_resc.max_cp_rings;
7851
7852 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7853 cp -= bnxt_get_ulp_msix_num(bp);
7854
7855 return cp;
a588e458
MC		 7856}
7857
ad95c27b 7858static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7809592d 7859{
6a4f2947
MC		 7860 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7861
f7588cd8
MC		 7862 if (bp->flags & BNXT_FLAG_CHIP_P5)
7863 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7864
6a4f2947 7865 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7809592d
MC		 7866}
7867
30f52947 7868static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
33c2657e 7869{
6a4f2947 7870 bp->hw_resc.max_irqs = max_irqs;
33c2657e
MC		 7871}
7872
e916b081
MC		 7873unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7874{
7875 unsigned int cp;
7876
7877 cp = bnxt_get_max_func_cp_rings_for_en(bp);
7878 if (bp->flags & BNXT_FLAG_CHIP_P5)
7879 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
7880 else
7881 return cp - bp->cp_nr_rings;
7882}
7883
c027c6b4
VV		 7884unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
7885{
d77b1ad8 7886 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
c027c6b4
VV		 7887}
7888
fbcfc8e4
MC		 7889int bnxt_get_avail_msix(struct bnxt *bp, int num)
7890{
7891 int max_cp = bnxt_get_max_func_cp_rings(bp);
7892 int max_irq = bnxt_get_max_func_irqs(bp);
7893 int total_req = bp->cp_nr_rings + num;
7894 int max_idx, avail_msix;
7895
75720e63
MC		 7896 max_idx = bp->total_irqs;
7897 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7898 max_idx = min_t(int, bp->total_irqs, max_cp);
fbcfc8e4 7899 avail_msix = max_idx - bp->cp_nr_rings;
f1ca94de 7900 if (!BNXT_NEW_RM(bp) || avail_msix >= num)
fbcfc8e4
MC		 7901 return avail_msix;
7902
7903 if (max_irq < total_req) {
7904 num = max_irq - bp->cp_nr_rings;
7905 if (num <= 0)
7906 return 0;
7907 }
7908 return num;
7909}
7910
08654eb2
MC		 7911static int bnxt_get_num_msix(struct bnxt *bp)
7912{
f1ca94de 7913 if (!BNXT_NEW_RM(bp))
08654eb2
MC		 7914 return bnxt_get_max_func_irqs(bp);
7915
c0b8cda0 7916 return bnxt_nq_rings_in_use(bp);
08654eb2
MC		 7917}
7918
7809592d 7919static int bnxt_init_msix(struct bnxt *bp)
c0c050c5 7920{
fbcfc8e4 7921 int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
7809592d 7922 struct msix_entry *msix_ent;
c0c050c5 7923
08654eb2
MC		 7924 total_vecs = bnxt_get_num_msix(bp);
7925 max = bnxt_get_max_func_irqs(bp);
7926 if (total_vecs > max)
7927 total_vecs = max;
7928
2773dfb2
MC		 7929 if (!total_vecs)
7930 return 0;
7931
c0c050c5
MC		 7932 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
7933 if (!msix_ent)
7934 return -ENOMEM;
7935
7936 for (i = 0; i < total_vecs; i++) {
7937 msix_ent[i].entry = i;
7938 msix_ent[i].vector = 0;
7939 }
7940
01657bcd
MC		 7941 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
7942 min = 2;
7943
7944 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
fbcfc8e4
MC		 7945 ulp_msix = bnxt_get_ulp_msix_num(bp);
7946 if (total_vecs < 0 || total_vecs < ulp_msix) {
c0c050c5
MC		 7947 rc = -ENODEV;
7948 goto msix_setup_exit;
7949 }
7950
7951 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
7952 if (bp->irq_tbl) {
7809592d
MC		 7953 for (i = 0; i < total_vecs; i++)
7954 bp->irq_tbl[i].vector = msix_ent[i].vector;
c0c050c5 7955
7809592d 7956 bp->total_irqs = total_vecs;
c0c050c5 7957 /* Trim rings based upon num of vectors allocated */
6e6c5a57 7958 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
fbcfc8e4 7959 total_vecs - ulp_msix, min == 1);
6e6c5a57
MC		 7960 if (rc)
7961 goto msix_setup_exit;
7962
7809592d
MC		 7963 bp->cp_nr_rings = (min == 1) ?
7964 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7965 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5 7966
c0c050c5
MC		 7967 } else {
7968 rc = -ENOMEM;
7969 goto msix_setup_exit;
7970 }
7971 bp->flags |= BNXT_FLAG_USING_MSIX;
7972 kfree(msix_ent);
7973 return 0;
7974
7975msix_setup_exit:
7809592d
MC		 7976 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
7977 kfree(bp->irq_tbl);
7978 bp->irq_tbl = NULL;
c0c050c5
MC		 7979 pci_disable_msix(bp->pdev);
7980 kfree(msix_ent);
7981 return rc;
7982}
7983
7809592d 7984static int bnxt_init_inta(struct bnxt *bp)
c0c050c5 7985{
c0c050c5 7986 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7809592d
MC		 7987 if (!bp->irq_tbl)
7988 return -ENOMEM;
7989
7990 bp->total_irqs = 1;
c0c050c5
MC		 7991 bp->rx_nr_rings = 1;
7992 bp->tx_nr_rings = 1;
7993 bp->cp_nr_rings = 1;
01657bcd 7994 bp->flags |= BNXT_FLAG_SHARED_RINGS;
c0c050c5 7995 bp->irq_tbl[0].vector = bp->pdev->irq;
7809592d 7996 return 0;
c0c050c5
MC		 7997}
7998
7809592d 7999static int bnxt_init_int_mode(struct bnxt *bp)
c0c050c5
MC		 8000{
8001 int rc = 0;
8002
8003 if (bp->flags & BNXT_FLAG_MSIX_CAP)
7809592d 8004 rc = bnxt_init_msix(bp);
c0c050c5 8005
1fa72e29 8006 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
c0c050c5 8007 /* fallback to INTA */
7809592d 8008 rc = bnxt_init_inta(bp);
c0c050c5
MC		 8009 }
8010 return rc;
8011}
8012
7809592d
MC		 8013static void bnxt_clear_int_mode(struct bnxt *bp)
8014{
8015 if (bp->flags & BNXT_FLAG_USING_MSIX)
8016 pci_disable_msix(bp->pdev);
8017
8018 kfree(bp->irq_tbl);
8019 bp->irq_tbl = NULL;
8020 bp->flags &= ~BNXT_FLAG_USING_MSIX;
8021}
8022
1b3f0b75 8023int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
674f50a5 8024{
674f50a5 8025 int tcs = netdev_get_num_tc(bp->dev);
1b3f0b75 8026 bool irq_cleared = false;
674f50a5
MC		 8027 int rc;
8028
8029 if (!bnxt_need_reserve_rings(bp))
8030 return 0;
8031
1b3f0b75
MC		 8032 if (irq_re_init && BNXT_NEW_RM(bp) &&
8033 bnxt_get_num_msix(bp) != bp->total_irqs) {
ec86f14e 8034 bnxt_ulp_irq_stop(bp);
674f50a5 8035 bnxt_clear_int_mode(bp);
1b3f0b75 8036 irq_cleared = true;
36d65be9
MC		 8037 }
8038 rc = __bnxt_reserve_rings(bp);
1b3f0b75 8039 if (irq_cleared) {
36d65be9
MC		 8040 if (!rc)
8041 rc = bnxt_init_int_mode(bp);
ec86f14e 8042 bnxt_ulp_irq_restart(bp, rc);
36d65be9
MC		 8043 }
8044 if (rc) {
8045 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8046 return rc;
674f50a5
MC		 8047 }
8048 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8049 netdev_err(bp->dev, "tx ring reservation failure\n");
8050 netdev_reset_tc(bp->dev);
8051 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8052 return -ENOMEM;
8053 }
674f50a5
MC		 8054 return 0;
8055}
8056
c0c050c5
MC		 8057static void bnxt_free_irq(struct bnxt *bp)
8058{
8059 struct bnxt_irq *irq;
8060 int i;
8061
8062#ifdef CONFIG_RFS_ACCEL
8063 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8064 bp->dev->rx_cpu_rmap = NULL;
8065#endif
cb98526b 8066 if (!bp->irq_tbl || !bp->bnapi)
c0c050c5
MC		 8067 return;
8068
8069 for (i = 0; i < bp->cp_nr_rings; i++) {
e5811b8c
MC		 8070 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8071
8072 irq = &bp->irq_tbl[map_idx];
56f0fd80
VV		 8073 if (irq->requested) {
8074 if (irq->have_cpumask) {
8075 irq_set_affinity_hint(irq->vector, NULL);
8076 free_cpumask_var(irq->cpu_mask);
8077 irq->have_cpumask = 0;
8078 }
c0c050c5 8079 free_irq(irq->vector, bp->bnapi[i]);
56f0fd80
VV		 8080 }
8081
c0c050c5
MC		 8082 irq->requested = 0;
8083 }
c0c050c5
MC		 8084}
8085
8086static int bnxt_request_irq(struct bnxt *bp)
8087{
b81a90d3 8088 int i, j, rc = 0;
c0c050c5
MC		 8089 unsigned long flags = 0;
8090#ifdef CONFIG_RFS_ACCEL
e5811b8c 8091 struct cpu_rmap *rmap;
c0c050c5
MC		 8092#endif
8093
e5811b8c
MC		 8094 rc = bnxt_setup_int_mode(bp);
8095 if (rc) {
8096 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8097 rc);
8098 return rc;
8099 }
8100#ifdef CONFIG_RFS_ACCEL
8101 rmap = bp->dev->rx_cpu_rmap;
8102#endif
c0c050c5
MC		 8103 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8104 flags = IRQF_SHARED;
8105
b81a90d3 8106 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
e5811b8c
MC		 8107 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8108 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8109
c0c050c5 8110#ifdef CONFIG_RFS_ACCEL
b81a90d3 8111 if (rmap && bp->bnapi[i]->rx_ring) {
c0c050c5
MC		 8112 rc = irq_cpu_rmap_add(rmap, irq->vector);
8113 if (rc)
8114 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
b81a90d3
MC		 8115 j);
8116 j++;
c0c050c5
MC		 8117 }
8118#endif
8119 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8120 bp->bnapi[i]);
8121 if (rc)
8122 break;
8123
8124 irq->requested = 1;
56f0fd80
VV		 8125
8126 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8127 int numa_node = dev_to_node(&bp->pdev->dev);
8128
8129 irq->have_cpumask = 1;
8130 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8131 irq->cpu_mask);
8132 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8133 if (rc) {
8134 netdev_warn(bp->dev,
8135 "Set affinity failed, IRQ = %d\n",
8136 irq->vector);
8137 break;
8138 }
8139 }
c0c050c5
MC		 8140 }
8141 return rc;
8142}
8143
8144static void bnxt_del_napi(struct bnxt *bp)
8145{
8146 int i;
8147
8148 if (!bp->bnapi)
8149 return;
8150
8151 for (i = 0; i < bp->cp_nr_rings; i++) {
8152 struct bnxt_napi *bnapi = bp->bnapi[i];
8153
8154 napi_hash_del(&bnapi->napi);
8155 netif_napi_del(&bnapi->napi);
8156 }
e5f6f564
ED		 8157 /* We called napi_hash_del() before netif_napi_del(), we need
8158 * to respect an RCU grace period before freeing napi structures.
8159 */
8160 synchronize_net();
c0c050c5
MC		 8161}
8162
8163static void bnxt_init_napi(struct bnxt *bp)
8164{
8165 int i;
10bbdaf5 8166 unsigned int cp_nr_rings = bp->cp_nr_rings;
c0c050c5
MC		 8167 struct bnxt_napi *bnapi;
8168
8169 if (bp->flags & BNXT_FLAG_USING_MSIX) {
0fcec985
MC		 8170 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8171
8172 if (bp->flags & BNXT_FLAG_CHIP_P5)
8173 poll_fn = bnxt_poll_p5;
8174 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10bbdaf5
PS		 8175 cp_nr_rings--;
8176 for (i = 0; i < cp_nr_rings; i++) {
c0c050c5 8177 bnapi = bp->bnapi[i];
0fcec985 8178 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
c0c050c5 8179 }
10bbdaf5
PS		 8180 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8181 bnapi = bp->bnapi[cp_nr_rings];
8182 netif_napi_add(bp->dev, &bnapi->napi,
8183 bnxt_poll_nitroa0, 64);
10bbdaf5 8184 }
c0c050c5
MC		 8185 } else {
8186 bnapi = bp->bnapi[0];
8187 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
c0c050c5
MC		 8188 }
8189}
8190
8191static void bnxt_disable_napi(struct bnxt *bp)
8192{
8193 int i;
8194
8195 if (!bp->bnapi)
8196 return;
8197
0bc0b97f
AG		 8198 for (i = 0; i < bp->cp_nr_rings; i++) {
8199 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8200
8201 if (bp->bnapi[i]->rx_ring)
8202 cancel_work_sync(&cpr->dim.work);
8203
c0c050c5 8204 napi_disable(&bp->bnapi[i]->napi);
0bc0b97f 8205 }
c0c050c5
MC		 8206}
8207
8208static void bnxt_enable_napi(struct bnxt *bp)
8209{
8210 int i;
8211
8212 for (i = 0; i < bp->cp_nr_rings; i++) {
6a8788f2 8213 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
fa7e2812 8214 bp->bnapi[i]->in_reset = false;
6a8788f2
AG		 8215
8216 if (bp->bnapi[i]->rx_ring) {
8217 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
c002bd52 8218 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6a8788f2 8219 }
c0c050c5
MC		 8220 napi_enable(&bp->bnapi[i]->napi);
8221 }
8222}
8223
7df4ae9f 8224void bnxt_tx_disable(struct bnxt *bp)
c0c050c5
MC		 8225{
8226 int i;
c0c050c5 8227 struct bnxt_tx_ring_info *txr;
c0c050c5 8228
b6ab4b01 8229 if (bp->tx_ring) {
c0c050c5 8230 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 8231 txr = &bp->tx_ring[i];
c0c050c5 8232 txr->dev_state = BNXT_DEV_STATE_CLOSING;
c0c050c5
MC		 8233 }
8234 }
8235 /* Stop all TX queues */
8236 netif_tx_disable(bp->dev);
8237 netif_carrier_off(bp->dev);
8238}
8239
7df4ae9f 8240void bnxt_tx_enable(struct bnxt *bp)
c0c050c5
MC		 8241{
8242 int i;
c0c050c5 8243 struct bnxt_tx_ring_info *txr;
c0c050c5
MC		 8244
8245 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 8246 txr = &bp->tx_ring[i];
c0c050c5
MC		 8247 txr->dev_state = 0;
8248 }
8249 netif_tx_wake_all_queues(bp->dev);
8250 if (bp->link_info.link_up)
8251 netif_carrier_on(bp->dev);
8252}
8253
8254static void bnxt_report_link(struct bnxt *bp)
8255{
8256 if (bp->link_info.link_up) {
8257 const char *duplex;
8258 const char *flow_ctrl;
38a21b34
DK		 8259 u32 speed;
8260 u16 fec;
c0c050c5
MC		 8261
8262 netif_carrier_on(bp->dev);
8263 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8264 duplex = "full";
8265 else
8266 duplex = "half";
8267 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8268 flow_ctrl = "ON - receive & transmit";
8269 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8270 flow_ctrl = "ON - transmit";
8271 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8272 flow_ctrl = "ON - receive";
8273 else
8274 flow_ctrl = "none";
8275 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
38a21b34 8276 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
c0c050c5 8277 speed, duplex, flow_ctrl);
170ce013
MC		 8278 if (bp->flags & BNXT_FLAG_EEE_CAP)
8279 netdev_info(bp->dev, "EEE is %s\n",
8280 bp->eee.eee_active ? "active" :
8281 "not active");
e70c752f
MC		 8282 fec = bp->link_info.fec_cfg;
8283 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8284 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8285 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8286 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8287 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
c0c050c5
MC		 8288 } else {
8289 netif_carrier_off(bp->dev);
8290 netdev_err(bp->dev, "NIC Link is Down\n");
8291 }
8292}
8293
170ce013
MC		 8294static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8295{
8296 int rc = 0;
8297 struct hwrm_port_phy_qcaps_input req = {0};
8298 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
93ed8117 8299 struct bnxt_link_info *link_info = &bp->link_info;
170ce013 8300
ba642ab7
MC		 8301 bp->flags &= ~BNXT_FLAG_EEE_CAP;
8302 if (bp->test_info)
8303 bp->test_info->flags &= ~BNXT_TEST_FL_EXT_LPBK;
170ce013
MC		 8304 if (bp->hwrm_spec_code < 0x10201)
8305 return 0;
8306
8307 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8308
8309 mutex_lock(&bp->hwrm_cmd_lock);
8310 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8311 if (rc)
8312 goto hwrm_phy_qcaps_exit;
8313
acb20054 8314 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
170ce013
MC		 8315 struct ethtool_eee *eee = &bp->eee;
8316 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8317
8318 bp->flags |= BNXT_FLAG_EEE_CAP;
8319 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8320 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8321 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8322 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8323 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8324 }
55fd0cf3
MC		 8325 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8326 if (bp->test_info)
8327 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8328 }
520ad89a
MC		 8329 if (resp->supported_speeds_auto_mode)
8330 link_info->support_auto_speeds =
8331 le16_to_cpu(resp->supported_speeds_auto_mode);
170ce013 8332
d5430d31
MC		 8333 bp->port_count = resp->port_cnt;
8334
170ce013
MC		 8335hwrm_phy_qcaps_exit:
8336 mutex_unlock(&bp->hwrm_cmd_lock);
8337 return rc;
8338}
8339
c0c050c5
MC		 8340static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8341{
8342 int rc = 0;
8343 struct bnxt_link_info *link_info = &bp->link_info;
8344 struct hwrm_port_phy_qcfg_input req = {0};
8345 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8346 u8 link_up = link_info->link_up;
286ef9d6 8347 u16 diff;
c0c050c5
MC		 8348
8349 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8350
8351 mutex_lock(&bp->hwrm_cmd_lock);
8352 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8353 if (rc) {
8354 mutex_unlock(&bp->hwrm_cmd_lock);
8355 return rc;
8356 }
8357
8358 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8359 link_info->phy_link_status = resp->link;
acb20054
MC		 8360 link_info->duplex = resp->duplex_cfg;
8361 if (bp->hwrm_spec_code >= 0x10800)
8362 link_info->duplex = resp->duplex_state;
c0c050c5
MC		 8363 link_info->pause = resp->pause;
8364 link_info->auto_mode = resp->auto_mode;
8365 link_info->auto_pause_setting = resp->auto_pause;
3277360e 8366 link_info->lp_pause = resp->link_partner_adv_pause;
c0c050c5 8367 link_info->force_pause_setting = resp->force_pause;
acb20054 8368 link_info->duplex_setting = resp->duplex_cfg;
c0c050c5
MC		 8369 if (link_info->phy_link_status == BNXT_LINK_LINK)
8370 link_info->link_speed = le16_to_cpu(resp->link_speed);
8371 else
8372 link_info->link_speed = 0;
8373 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
c0c050c5
MC		 8374 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8375 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
3277360e
MC		 8376 link_info->lp_auto_link_speeds =
8377 le16_to_cpu(resp->link_partner_adv_speeds);
c0c050c5
MC		 8378 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8379 link_info->phy_ver[0] = resp->phy_maj;
8380 link_info->phy_ver[1] = resp->phy_min;
8381 link_info->phy_ver[2] = resp->phy_bld;
8382 link_info->media_type = resp->media_type;
03efbec0 8383 link_info->phy_type = resp->phy_type;
11f15ed3 8384 link_info->transceiver = resp->xcvr_pkg_type;
170ce013
MC		 8385 link_info->phy_addr = resp->eee_config_phy_addr &
8386 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
42ee18fe 8387 link_info->module_status = resp->module_status;
170ce013
MC		 8388
8389 if (bp->flags & BNXT_FLAG_EEE_CAP) {
8390 struct ethtool_eee *eee = &bp->eee;
8391 u16 fw_speeds;
8392
8393 eee->eee_active = 0;
8394 if (resp->eee_config_phy_addr &
8395 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8396 eee->eee_active = 1;
8397 fw_speeds = le16_to_cpu(
8398 resp->link_partner_adv_eee_link_speed_mask);
8399 eee->lp_advertised =
8400 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8401 }
8402
8403 /* Pull initial EEE config */
8404 if (!chng_link_state) {
8405 if (resp->eee_config_phy_addr &
8406 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8407 eee->eee_enabled = 1;
c0c050c5 8408
170ce013
MC		 8409 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8410 eee->advertised =
8411 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8412
8413 if (resp->eee_config_phy_addr &
8414 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8415 __le32 tmr;
8416
8417 eee->tx_lpi_enabled = 1;
8418 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8419 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8420 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8421 }
8422 }
8423 }
e70c752f
MC		 8424
8425 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8426 if (bp->hwrm_spec_code >= 0x10504)
8427 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8428
c0c050c5
MC		 8429 /* TODO: need to add more logic to report VF link */
8430 if (chng_link_state) {
8431 if (link_info->phy_link_status == BNXT_LINK_LINK)
8432 link_info->link_up = 1;
8433 else
8434 link_info->link_up = 0;
8435 if (link_up != link_info->link_up)
8436 bnxt_report_link(bp);
8437 } else {
8438 /* alwasy link down if not require to update link state */
8439 link_info->link_up = 0;
8440 }
8441 mutex_unlock(&bp->hwrm_cmd_lock);
286ef9d6 8442
dac04907
MC		 8443 if (!BNXT_SINGLE_PF(bp))
8444 return 0;
8445
286ef9d6
MC		 8446 diff = link_info->support_auto_speeds ^ link_info->advertising;
8447 if ((link_info->support_auto_speeds | diff) !=
8448 link_info->support_auto_speeds) {
8449 /* An advertised speed is no longer supported, so we need to
0eaa24b9
MC		 8450 * update the advertisement settings. Caller holds RTNL
8451 * so we can modify link settings.
286ef9d6 8452 */
286ef9d6 8453 link_info->advertising = link_info->support_auto_speeds;
0eaa24b9 8454 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
286ef9d6 8455 bnxt_hwrm_set_link_setting(bp, true, false);
286ef9d6 8456 }
c0c050c5
MC		 8457 return 0;
8458}
8459
10289bec
MC		 8460static void bnxt_get_port_module_status(struct bnxt *bp)
8461{
8462 struct bnxt_link_info *link_info = &bp->link_info;
8463 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8464 u8 module_status;
8465
8466 if (bnxt_update_link(bp, true))
8467 return;
8468
8469 module_status = link_info->module_status;
8470 switch (module_status) {
8471 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8472 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8473 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8474 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8475 bp->pf.port_id);
8476 if (bp->hwrm_spec_code >= 0x10201) {
8477 netdev_warn(bp->dev, "Module part number %s\n",
8478 resp->phy_vendor_partnumber);
8479 }
8480 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8481 netdev_warn(bp->dev, "TX is disabled\n");
8482 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8483 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8484 }
8485}
8486
c0c050c5
MC		 8487static void
8488bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8489{
8490 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
c9ee9516
MC		 8491 if (bp->hwrm_spec_code >= 0x10201)
8492 req->auto_pause =
8493 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
c0c050c5
MC		 8494 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8495 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8496 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
49b5c7a1 8497 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
c0c050c5
MC		 8498 req->enables |=
8499 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8500 } else {
8501 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8502 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8503 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8504 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
8505 req->enables |=
8506 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
c9ee9516
MC		 8507 if (bp->hwrm_spec_code >= 0x10201) {
8508 req->auto_pause = req->force_pause;
8509 req->enables |= cpu_to_le32(
8510 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8511 }
c0c050c5
MC		 8512 }
8513}
8514
8515static void bnxt_hwrm_set_link_common(struct bnxt *bp,
8516 struct hwrm_port_phy_cfg_input *req)
8517{
8518 u8 autoneg = bp->link_info.autoneg;
8519 u16 fw_link_speed = bp->link_info.req_link_speed;
68515a18 8520 u16 advertising = bp->link_info.advertising;
c0c050c5
MC		 8521
8522 if (autoneg & BNXT_AUTONEG_SPEED) {
8523 req->auto_mode |=
11f15ed3 8524 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
c0c050c5
MC		 8525
8526 req->enables |= cpu_to_le32(
8527 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
8528 req->auto_link_speed_mask = cpu_to_le16(advertising);
8529
8530 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
8531 req->flags |=
8532 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
8533 } else {
8534 req->force_link_speed = cpu_to_le16(fw_link_speed);
8535 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
8536 }
8537
c0c050c5
MC		 8538 /* tell chimp that the setting takes effect immediately */
8539 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8540}
8541
8542int bnxt_hwrm_set_pause(struct bnxt *bp)
8543{
8544 struct hwrm_port_phy_cfg_input req = {0};
8545 int rc;
8546
8547 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8548 bnxt_hwrm_set_pause_common(bp, &req);
8549
8550 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8551 bp->link_info.force_link_chng)
8552 bnxt_hwrm_set_link_common(bp, &req);
8553
8554 mutex_lock(&bp->hwrm_cmd_lock);
8555 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8556 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8557 /* since changing of pause setting doesn't trigger any link
8558 * change event, the driver needs to update the current pause
8559 * result upon successfully return of the phy_cfg command
8560 */
8561 bp->link_info.pause =
8562 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8563 bp->link_info.auto_pause_setting = 0;
8564 if (!bp->link_info.force_link_chng)
8565 bnxt_report_link(bp);
8566 }
8567 bp->link_info.force_link_chng = false;
8568 mutex_unlock(&bp->hwrm_cmd_lock);
8569 return rc;
8570}
8571
939f7f0c
MC		 8572static void bnxt_hwrm_set_eee(struct bnxt *bp,
8573 struct hwrm_port_phy_cfg_input *req)
8574{
8575 struct ethtool_eee *eee = &bp->eee;
8576
8577 if (eee->eee_enabled) {
8578 u16 eee_speeds;
8579 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8580
8581 if (eee->tx_lpi_enabled)
8582 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8583 else
8584 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8585
8586 req->flags |= cpu_to_le32(flags);
8587 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8588 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8589 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8590 } else {
8591 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8592 }
8593}
8594
8595int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
c0c050c5
MC		 8596{
8597 struct hwrm_port_phy_cfg_input req = {0};
8598
8599 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8600 if (set_pause)
8601 bnxt_hwrm_set_pause_common(bp, &req);
8602
8603 bnxt_hwrm_set_link_common(bp, &req);
939f7f0c
MC		 8604
8605 if (set_eee)
8606 bnxt_hwrm_set_eee(bp, &req);
c0c050c5
MC		 8607 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8608}
8609
33f7d55f
MC		 8610static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8611{
8612 struct hwrm_port_phy_cfg_input req = {0};
8613
567b2abe 8614 if (!BNXT_SINGLE_PF(bp))
33f7d55f
MC		 8615 return 0;
8616
8617 if (pci_num_vf(bp->pdev))
8618 return 0;
8619
8620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
16d663a6 8621 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
33f7d55f
MC		 8622 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8623}
8624
ec5d31e3
MC		 8625static int bnxt_fw_init_one(struct bnxt *bp);
8626
25e1acd6
MC		 8627static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8628{
8629 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8630 struct hwrm_func_drv_if_change_input req = {0};
ec5d31e3
MC		 8631 bool resc_reinit = false, fw_reset = false;
8632 u32 flags = 0;
25e1acd6
MC		 8633 int rc;
8634
8635 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8636 return 0;
8637
8638 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8639 if (up)
8640 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8641 mutex_lock(&bp->hwrm_cmd_lock);
8642 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
ec5d31e3
MC		 8643 if (!rc)
8644 flags = le32_to_cpu(resp->flags);
25e1acd6 8645 mutex_unlock(&bp->hwrm_cmd_lock);
ec5d31e3
MC		 8646 if (rc)
8647 return rc;
25e1acd6 8648
ec5d31e3
MC		 8649 if (!up)
8650 return 0;
25e1acd6 8651
ec5d31e3
MC		 8652 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
8653 resc_reinit = true;
8654 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
8655 fw_reset = true;
8656
8657 if (resc_reinit || fw_reset) {
8658 if (fw_reset) {
8659 rc = bnxt_fw_init_one(bp);
8660 if (rc) {
8661 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
8662 return rc;
8663 }
8664 bnxt_clear_int_mode(bp);
8665 rc = bnxt_init_int_mode(bp);
8666 if (rc) {
8667 netdev_err(bp->dev, "init int mode failed\n");
8668 return rc;
8669 }
8670 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
8671 }
8672 if (BNXT_NEW_RM(bp)) {
8673 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8674
8675 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8676 hw_resc->resv_cp_rings = 0;
8677 hw_resc->resv_stat_ctxs = 0;
8678 hw_resc->resv_irqs = 0;
8679 hw_resc->resv_tx_rings = 0;
8680 hw_resc->resv_rx_rings = 0;
8681 hw_resc->resv_hw_ring_grps = 0;
8682 hw_resc->resv_vnics = 0;
8683 if (!fw_reset) {
8684 bp->tx_nr_rings = 0;
8685 bp->rx_nr_rings = 0;
8686 }
8687 }
25e1acd6 8688 }
ec5d31e3 8689 return 0;
25e1acd6
MC		 8690}
8691
5ad2cbee
MC		 8692static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8693{
8694 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8695 struct hwrm_port_led_qcaps_input req = {0};
8696 struct bnxt_pf_info *pf = &bp->pf;
8697 int rc;
8698
ba642ab7 8699 bp->num_leds = 0;
5ad2cbee
MC		 8700 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8701 return 0;
8702
8703 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8704 req.port_id = cpu_to_le16(pf->port_id);
8705 mutex_lock(&bp->hwrm_cmd_lock);
8706 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8707 if (rc) {
8708 mutex_unlock(&bp->hwrm_cmd_lock);
8709 return rc;
8710 }
8711 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8712 int i;
8713
8714 bp->num_leds = resp->num_leds;
8715 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8716 bp->num_leds);
8717 for (i = 0; i < bp->num_leds; i++) {
8718 struct bnxt_led_info *led = &bp->leds[i];
8719 __le16 caps = led->led_state_caps;
8720
8721 if (!led->led_group_id ||
8722 !BNXT_LED_ALT_BLINK_CAP(caps)) {
8723 bp->num_leds = 0;
8724 break;
8725 }
8726 }
8727 }
8728 mutex_unlock(&bp->hwrm_cmd_lock);
8729 return 0;
8730}
8731
5282db6c
MC		 8732int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8733{
8734 struct hwrm_wol_filter_alloc_input req = {0};
8735 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8736 int rc;
8737
8738 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8739 req.port_id = cpu_to_le16(bp->pf.port_id);
8740 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8741 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8742 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8743 mutex_lock(&bp->hwrm_cmd_lock);
8744 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8745 if (!rc)
8746 bp->wol_filter_id = resp->wol_filter_id;
8747 mutex_unlock(&bp->hwrm_cmd_lock);
8748 return rc;
8749}
8750
8751int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8752{
8753 struct hwrm_wol_filter_free_input req = {0};
8754 int rc;
8755
8756 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8757 req.port_id = cpu_to_le16(bp->pf.port_id);
8758 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8759 req.wol_filter_id = bp->wol_filter_id;
8760 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8761 return rc;
8762}
8763
c1ef146a
MC		 8764static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8765{
8766 struct hwrm_wol_filter_qcfg_input req = {0};
8767 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8768 u16 next_handle = 0;
8769 int rc;
8770
8771 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8772 req.port_id = cpu_to_le16(bp->pf.port_id);
8773 req.handle = cpu_to_le16(handle);
8774 mutex_lock(&bp->hwrm_cmd_lock);
8775 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8776 if (!rc) {
8777 next_handle = le16_to_cpu(resp->next_handle);
8778 if (next_handle != 0) {
8779 if (resp->wol_type ==
8780 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8781 bp->wol = 1;
8782 bp->wol_filter_id = resp->wol_filter_id;
8783 }
8784 }
8785 }
8786 mutex_unlock(&bp->hwrm_cmd_lock);
8787 return next_handle;
8788}
8789
8790static void bnxt_get_wol_settings(struct bnxt *bp)
8791{
8792 u16 handle = 0;
8793
ba642ab7 8794 bp->wol = 0;
c1ef146a
MC		 8795 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8796 return;
8797
8798 do {
8799 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8800 } while (handle && handle != 0xffff);
8801}
8802
cde49a42
VV		 8803#ifdef CONFIG_BNXT_HWMON
8804static ssize_t bnxt_show_temp(struct device *dev,
8805 struct device_attribute *devattr, char *buf)
8806{
8807 struct hwrm_temp_monitor_query_input req = {0};
8808 struct hwrm_temp_monitor_query_output *resp;
8809 struct bnxt *bp = dev_get_drvdata(dev);
8810 u32 temp = 0;
8811
8812 resp = bp->hwrm_cmd_resp_addr;
8813 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8814 mutex_lock(&bp->hwrm_cmd_lock);
8815 if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8816 temp = resp->temp * 1000; /* display millidegree */
8817 mutex_unlock(&bp->hwrm_cmd_lock);
8818
8819 return sprintf(buf, "%u\n", temp);
8820}
8821static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8822
8823static struct attribute *bnxt_attrs[] = {
8824 &sensor_dev_attr_temp1_input.dev_attr.attr,
8825 NULL
8826};
8827ATTRIBUTE_GROUPS(bnxt);
8828
8829static void bnxt_hwmon_close(struct bnxt *bp)
8830{
8831 if (bp->hwmon_dev) {
8832 hwmon_device_unregister(bp->hwmon_dev);
8833 bp->hwmon_dev = NULL;
8834 }
8835}
8836
8837static void bnxt_hwmon_open(struct bnxt *bp)
8838{
8839 struct pci_dev *pdev = bp->pdev;
8840
ba642ab7
MC		 8841 if (bp->hwmon_dev)
8842 return;
8843
cde49a42
VV		 8844 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8845 DRV_MODULE_NAME, bp,
8846 bnxt_groups);
8847 if (IS_ERR(bp->hwmon_dev)) {
8848 bp->hwmon_dev = NULL;
8849 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8850 }
8851}
8852#else
8853static void bnxt_hwmon_close(struct bnxt *bp)
8854{
8855}
8856
8857static void bnxt_hwmon_open(struct bnxt *bp)
8858{
8859}
8860#endif
8861
939f7f0c
MC		 8862static bool bnxt_eee_config_ok(struct bnxt *bp)
8863{
8864 struct ethtool_eee *eee = &bp->eee;
8865 struct bnxt_link_info *link_info = &bp->link_info;
8866
8867 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8868 return true;
8869
8870 if (eee->eee_enabled) {
8871 u32 advertising =
8872 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8873
8874 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8875 eee->eee_enabled = 0;
8876 return false;
8877 }
8878 if (eee->advertised & ~advertising) {
8879 eee->advertised = advertising & eee->supported;
8880 return false;
8881 }
8882 }
8883 return true;
8884}
8885
c0c050c5
MC		 8886static int bnxt_update_phy_setting(struct bnxt *bp)
8887{
8888 int rc;
8889 bool update_link = false;
8890 bool update_pause = false;
939f7f0c 8891 bool update_eee = false;
c0c050c5
MC		 8892 struct bnxt_link_info *link_info = &bp->link_info;
8893
8894 rc = bnxt_update_link(bp, true);
8895 if (rc) {
8896 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
8897 rc);
8898 return rc;
8899 }
33dac24a
MC		 8900 if (!BNXT_SINGLE_PF(bp))
8901 return 0;
8902
c0c050c5 8903 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
c9ee9516
MC		 8904 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
8905 link_info->req_flow_ctrl)
c0c050c5
MC		 8906 update_pause = true;
8907 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8908 link_info->force_pause_setting != link_info->req_flow_ctrl)
8909 update_pause = true;
c0c050c5
MC		 8910 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8911 if (BNXT_AUTO_MODE(link_info->auto_mode))
8912 update_link = true;
8913 if (link_info->req_link_speed != link_info->force_link_speed)
8914 update_link = true;
de73018f
MC		 8915 if (link_info->req_duplex != link_info->duplex_setting)
8916 update_link = true;
c0c050c5
MC		 8917 } else {
8918 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
8919 update_link = true;
8920 if (link_info->advertising != link_info->auto_link_speeds)
8921 update_link = true;
c0c050c5
MC		 8922 }
8923
16d663a6
MC		 8924 /* The last close may have shutdown the link, so need to call
8925 * PHY_CFG to bring it back up.
8926 */
8927 if (!netif_carrier_ok(bp->dev))
8928 update_link = true;
8929
939f7f0c
MC		 8930 if (!bnxt_eee_config_ok(bp))
8931 update_eee = true;
8932
c0c050c5 8933 if (update_link)
939f7f0c 8934 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
c0c050c5
MC		 8935 else if (update_pause)
8936 rc = bnxt_hwrm_set_pause(bp);
8937 if (rc) {
8938 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
8939 rc);
8940 return rc;
8941 }
8942
8943 return rc;
8944}
8945
11809490
JH		 8946/* Common routine to pre-map certain register block to different GRC window.
8947 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
8948 * in PF and 3 windows in VF that can be customized to map in different
8949 * register blocks.
8950 */
8951static void bnxt_preset_reg_win(struct bnxt *bp)
8952{
8953 if (BNXT_PF(bp)) {
8954 /* CAG registers map to GRC window #4 */
8955 writel(BNXT_CAG_REG_BASE,
8956 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
8957 }
8958}
8959
47558acd
MC		 8960static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
8961
c0c050c5
MC		 8962static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8963{
8964 int rc = 0;
8965
11809490 8966 bnxt_preset_reg_win(bp);
c0c050c5
MC		 8967 netif_carrier_off(bp->dev);
8968 if (irq_re_init) {
47558acd
MC		 8969 /* Reserve rings now if none were reserved at driver probe. */
8970 rc = bnxt_init_dflt_ring_mode(bp);
8971 if (rc) {
8972 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
8973 return rc;
8974 }
c0c050c5 8975 }
1b3f0b75 8976 rc = bnxt_reserve_rings(bp, irq_re_init);
41e8d798
MC		 8977 if (rc)
8978 return rc;
c0c050c5
MC		 8979 if ((bp->flags & BNXT_FLAG_RFS) &&
8980 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
8981 /* disable RFS if falling back to INTA */
8982 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
8983 bp->flags &= ~BNXT_FLAG_RFS;
8984 }
8985
8986 rc = bnxt_alloc_mem(bp, irq_re_init);
8987 if (rc) {
8988 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8989 goto open_err_free_mem;
8990 }
8991
8992 if (irq_re_init) {
8993 bnxt_init_napi(bp);
8994 rc = bnxt_request_irq(bp);
8995 if (rc) {
8996 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
c58387ab 8997 goto open_err_irq;
c0c050c5
MC		 8998 }
8999 }
9000
9001 bnxt_enable_napi(bp);
cabfb09d 9002 bnxt_debug_dev_init(bp);
c0c050c5
MC		 9003
9004 rc = bnxt_init_nic(bp, irq_re_init);
9005 if (rc) {
9006 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9007 goto open_err;
9008 }
9009
9010 if (link_re_init) {
e2dc9b6e 9011 mutex_lock(&bp->link_lock);
c0c050c5 9012 rc = bnxt_update_phy_setting(bp);
e2dc9b6e 9013 mutex_unlock(&bp->link_lock);
a1ef4a79 9014 if (rc) {
ba41d46f 9015 netdev_warn(bp->dev, "failed to update phy settings\n");
a1ef4a79
MC		 9016 if (BNXT_SINGLE_PF(bp)) {
9017 bp->link_info.phy_retry = true;
9018 bp->link_info.phy_retry_expires =
9019 jiffies + 5 * HZ;
9020 }
9021 }
c0c050c5
MC		 9022 }
9023
7cdd5fc3 9024 if (irq_re_init)
ad51b8e9 9025 udp_tunnel_get_rx_info(bp->dev);
c0c050c5 9026
caefe526 9027 set_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 9028 bnxt_enable_int(bp);
9029 /* Enable TX queues */
9030 bnxt_tx_enable(bp);
9031 mod_timer(&bp->timer, jiffies + bp->current_interval);
10289bec
MC		 9032 /* Poll link status and check for SFP+ module status */
9033 bnxt_get_port_module_status(bp);
c0c050c5 9034
ee5c7fb3
SP		 9035 /* VF-reps may need to be re-opened after the PF is re-opened */
9036 if (BNXT_PF(bp))
9037 bnxt_vf_reps_open(bp);
c0c050c5
MC		 9038 return 0;
9039
9040open_err:
cabfb09d 9041 bnxt_debug_dev_exit(bp);
c0c050c5 9042 bnxt_disable_napi(bp);
c58387ab
VG		 9043
9044open_err_irq:
c0c050c5
MC		 9045 bnxt_del_napi(bp);
9046
9047open_err_free_mem:
9048 bnxt_free_skbs(bp);
9049 bnxt_free_irq(bp);
9050 bnxt_free_mem(bp, true);
9051 return rc;
9052}
9053
9054/* rtnl_lock held */
9055int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9056{
9057 int rc = 0;
9058
9059 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9060 if (rc) {
9061 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9062 dev_close(bp->dev);
9063 }
9064 return rc;
9065}
9066
f7dc1ea6
MC		 9067/* rtnl_lock held, open the NIC half way by allocating all resources, but
9068 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
9069 * self tests.
9070 */
9071int bnxt_half_open_nic(struct bnxt *bp)
9072{
9073 int rc = 0;
9074
9075 rc = bnxt_alloc_mem(bp, false);
9076 if (rc) {
9077 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9078 goto half_open_err;
9079 }
9080 rc = bnxt_init_nic(bp, false);
9081 if (rc) {
9082 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9083 goto half_open_err;
9084 }
9085 return 0;
9086
9087half_open_err:
9088 bnxt_free_skbs(bp);
9089 bnxt_free_mem(bp, false);
9090 dev_close(bp->dev);
9091 return rc;
9092}
9093
9094/* rtnl_lock held, this call can only be made after a previous successful
9095 * call to bnxt_half_open_nic().
9096 */
9097void bnxt_half_close_nic(struct bnxt *bp)
9098{
9099 bnxt_hwrm_resource_free(bp, false, false);
9100 bnxt_free_skbs(bp);
9101 bnxt_free_mem(bp, false);
9102}
9103
c0c050c5
MC		 9104static int bnxt_open(struct net_device *dev)
9105{
9106 struct bnxt *bp = netdev_priv(dev);
25e1acd6 9107 int rc;
c0c050c5 9108
ec5d31e3
MC		 9109 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9110 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9111 return -ENODEV;
9112 }
9113
9114 rc = bnxt_hwrm_if_change(bp, true);
25e1acd6 9115 if (rc)
ec5d31e3
MC		 9116 return rc;
9117 rc = __bnxt_open_nic(bp, true, true);
9118 if (rc) {
25e1acd6 9119 bnxt_hwrm_if_change(bp, false);
ec5d31e3
MC		 9120 } else {
9121 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state) &&
9122 BNXT_PF(bp)) {
9123 struct bnxt_pf_info *pf = &bp->pf;
9124 int n = pf->active_vfs;
cde49a42 9125
ec5d31e3
MC		 9126 if (n)
9127 bnxt_cfg_hw_sriov(bp, &n);
9128 }
9129 bnxt_hwmon_open(bp);
9130 }
cde49a42 9131
25e1acd6 9132 return rc;
c0c050c5
MC		 9133}
9134
f9b76ebd
MC		 9135static bool bnxt_drv_busy(struct bnxt *bp)
9136{
9137 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9138 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9139}
9140
b8875ca3
MC		 9141static void bnxt_get_ring_stats(struct bnxt *bp,
9142 struct rtnl_link_stats64 *stats);
9143
86e953db
MC		 9144static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9145 bool link_re_init)
c0c050c5 9146{
ee5c7fb3
SP		 9147 /* Close the VF-reps before closing PF */
9148 if (BNXT_PF(bp))
9149 bnxt_vf_reps_close(bp);
86e953db 9150
c0c050c5
MC		 9151 /* Change device state to avoid TX queue wake up's */
9152 bnxt_tx_disable(bp);
9153
caefe526 9154 clear_bit(BNXT_STATE_OPEN, &bp->state);
4cebdcec 9155 smp_mb__after_atomic();
f9b76ebd 9156 while (bnxt_drv_busy(bp))
4cebdcec 9157 msleep(20);
c0c050c5 9158
9d8bc097 9159 /* Flush rings and and disable interrupts */
c0c050c5
MC		 9160 bnxt_shutdown_nic(bp, irq_re_init);
9161
9162 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9163
cabfb09d 9164 bnxt_debug_dev_exit(bp);
c0c050c5 9165 bnxt_disable_napi(bp);
c0c050c5
MC		 9166 del_timer_sync(&bp->timer);
9167 bnxt_free_skbs(bp);
9168
b8875ca3
MC		 9169 /* Save ring stats before shutdown */
9170 if (bp->bnapi)
9171 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
c0c050c5
MC		 9172 if (irq_re_init) {
9173 bnxt_free_irq(bp);
9174 bnxt_del_napi(bp);
9175 }
9176 bnxt_free_mem(bp, irq_re_init);
86e953db
MC		 9177}
9178
9179int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9180{
9181 int rc = 0;
9182
9183#ifdef CONFIG_BNXT_SRIOV
9184 if (bp->sriov_cfg) {
9185 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9186 !bp->sriov_cfg,
9187 BNXT_SRIOV_CFG_WAIT_TMO);
9188 if (rc)
9189 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9190 }
9191#endif
9192 __bnxt_close_nic(bp, irq_re_init, link_re_init);
c0c050c5
MC		 9193 return rc;
9194}
9195
9196static int bnxt_close(struct net_device *dev)
9197{
9198 struct bnxt *bp = netdev_priv(dev);
9199
cde49a42 9200 bnxt_hwmon_close(bp);
c0c050c5 9201 bnxt_close_nic(bp, true, true);
33f7d55f 9202 bnxt_hwrm_shutdown_link(bp);
25e1acd6 9203 bnxt_hwrm_if_change(bp, false);
c0c050c5
MC		 9204 return 0;
9205}
9206
0ca12be9
VV		 9207static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9208 u16 *val)
9209{
9210 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9211 struct hwrm_port_phy_mdio_read_input req = {0};
9212 int rc;
9213
9214 if (bp->hwrm_spec_code < 0x10a00)
9215 return -EOPNOTSUPP;
9216
9217 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9218 req.port_id = cpu_to_le16(bp->pf.port_id);
9219 req.phy_addr = phy_addr;
9220 req.reg_addr = cpu_to_le16(reg & 0x1f);
2730214d 9221 if (mdio_phy_id_is_c45(phy_addr)) {
0ca12be9
VV		 9222 req.cl45_mdio = 1;
9223 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9224 req.dev_addr = mdio_phy_id_devad(phy_addr);
9225 req.reg_addr = cpu_to_le16(reg);
9226 }
9227
9228 mutex_lock(&bp->hwrm_cmd_lock);
9229 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9230 if (!rc)
9231 *val = le16_to_cpu(resp->reg_data);
9232 mutex_unlock(&bp->hwrm_cmd_lock);
9233 return rc;
9234}
9235
9236static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9237 u16 val)
9238{
9239 struct hwrm_port_phy_mdio_write_input req = {0};
9240
9241 if (bp->hwrm_spec_code < 0x10a00)
9242 return -EOPNOTSUPP;
9243
9244 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9245 req.port_id = cpu_to_le16(bp->pf.port_id);
9246 req.phy_addr = phy_addr;
9247 req.reg_addr = cpu_to_le16(reg & 0x1f);
2730214d 9248 if (mdio_phy_id_is_c45(phy_addr)) {
0ca12be9
VV		 9249 req.cl45_mdio = 1;
9250 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9251 req.dev_addr = mdio_phy_id_devad(phy_addr);
9252 req.reg_addr = cpu_to_le16(reg);
9253 }
9254 req.reg_data = cpu_to_le16(val);
9255
9256 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9257}
9258
c0c050c5
MC		 9259/* rtnl_lock held */
9260static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9261{
0ca12be9
VV		 9262 struct mii_ioctl_data *mdio = if_mii(ifr);
9263 struct bnxt *bp = netdev_priv(dev);
9264 int rc;
9265
c0c050c5
MC		 9266 switch (cmd) {
9267 case SIOCGMIIPHY:
0ca12be9
VV		 9268 mdio->phy_id = bp->link_info.phy_addr;
9269
c0c050c5
MC		 9270 /* fallthru */
9271 case SIOCGMIIREG: {
0ca12be9
VV		 9272 u16 mii_regval = 0;
9273
c0c050c5
MC		 9274 if (!netif_running(dev))
9275 return -EAGAIN;
9276
0ca12be9
VV		 9277 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9278 &mii_regval);
9279 mdio->val_out = mii_regval;
9280 return rc;
c0c050c5
MC		 9281 }
9282
9283 case SIOCSMIIREG:
9284 if (!netif_running(dev))
9285 return -EAGAIN;
9286
0ca12be9
VV		 9287 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9288 mdio->val_in);
c0c050c5
MC		 9289
9290 default:
9291 /* do nothing */
9292 break;
9293 }
9294 return -EOPNOTSUPP;
9295}
9296
b8875ca3
MC		 9297static void bnxt_get_ring_stats(struct bnxt *bp,
9298 struct rtnl_link_stats64 *stats)
c0c050c5 9299{
b8875ca3 9300 int i;
c0c050c5 9301
c0c050c5 9302
c0c050c5
MC		 9303 for (i = 0; i < bp->cp_nr_rings; i++) {
9304 struct bnxt_napi *bnapi = bp->bnapi[i];
9305 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9306 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
9307
9308 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
9309 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
9310 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
9311
9312 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
9313 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
9314 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
9315
9316 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
9317 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
9318 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
9319
9320 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
9321 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
9322 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
9323
9324 stats->rx_missed_errors +=
9325 le64_to_cpu(hw_stats->rx_discard_pkts);
9326
9327 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
9328
c0c050c5
MC		 9329 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
9330 }
b8875ca3
MC		 9331}
9332
9333static void bnxt_add_prev_stats(struct bnxt *bp,
9334 struct rtnl_link_stats64 *stats)
9335{
9336 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9337
9338 stats->rx_packets += prev_stats->rx_packets;
9339 stats->tx_packets += prev_stats->tx_packets;
9340 stats->rx_bytes += prev_stats->rx_bytes;
9341 stats->tx_bytes += prev_stats->tx_bytes;
9342 stats->rx_missed_errors += prev_stats->rx_missed_errors;
9343 stats->multicast += prev_stats->multicast;
9344 stats->tx_dropped += prev_stats->tx_dropped;
9345}
9346
9347static void
9348bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9349{
9350 struct bnxt *bp = netdev_priv(dev);
9351
9352 set_bit(BNXT_STATE_READ_STATS, &bp->state);
9353 /* Make sure bnxt_close_nic() sees that we are reading stats before
9354 * we check the BNXT_STATE_OPEN flag.
9355 */
9356 smp_mb__after_atomic();
9357 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9358 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9359 *stats = bp->net_stats_prev;
9360 return;
9361 }
9362
9363 bnxt_get_ring_stats(bp, stats);
9364 bnxt_add_prev_stats(bp, stats);
c0c050c5 9365
9947f83f
MC		 9366 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9367 struct rx_port_stats *rx = bp->hw_rx_port_stats;
9368 struct tx_port_stats *tx = bp->hw_tx_port_stats;
9369
9370 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
9371 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
9372 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
9373 le64_to_cpu(rx->rx_ovrsz_frames) +
9374 le64_to_cpu(rx->rx_runt_frames);
9375 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
9376 le64_to_cpu(rx->rx_jbr_frames);
9377 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
9378 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
9379 stats->tx_errors = le64_to_cpu(tx->tx_err);
9380 }
f9b76ebd 9381 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
c0c050c5
MC		 9382}
9383
9384static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9385{
9386 struct net_device *dev = bp->dev;
9387 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9388 struct netdev_hw_addr *ha;
9389 u8 *haddr;
9390 int mc_count = 0;
9391 bool update = false;
9392 int off = 0;
9393
9394 netdev_for_each_mc_addr(ha, dev) {
9395 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9396 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9397 vnic->mc_list_count = 0;
9398 return false;
9399 }
9400 haddr = ha->addr;
9401 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9402 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9403 update = true;
9404 }
9405 off += ETH_ALEN;
9406 mc_count++;
9407 }
9408 if (mc_count)
9409 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9410
9411 if (mc_count != vnic->mc_list_count) {
9412 vnic->mc_list_count = mc_count;
9413 update = true;
9414 }
9415 return update;
9416}
9417
9418static bool bnxt_uc_list_updated(struct bnxt *bp)
9419{
9420 struct net_device *dev = bp->dev;
9421 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9422 struct netdev_hw_addr *ha;
9423 int off = 0;
9424
9425 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9426 return true;
9427
9428 netdev_for_each_uc_addr(ha, dev) {
9429 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9430 return true;
9431
9432 off += ETH_ALEN;
9433 }
9434 return false;
9435}
9436
9437static void bnxt_set_rx_mode(struct net_device *dev)
9438{
9439 struct bnxt *bp = netdev_priv(dev);
9440 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9441 u32 mask = vnic->rx_mask;
9442 bool mc_update = false;
9443 bool uc_update;
9444
9445 if (!netif_running(dev))
9446 return;
9447
9448 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9449 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
30e33848
MC		 9450 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9451 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
c0c050c5 9452
17c71ac3 9453 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
c0c050c5
MC		 9454 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9455
9456 uc_update = bnxt_uc_list_updated(bp);
9457
30e33848
MC		 9458 if (dev->flags & IFF_BROADCAST)
9459 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5
MC		 9460 if (dev->flags & IFF_ALLMULTI) {
9461 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9462 vnic->mc_list_count = 0;
9463 } else {
9464 mc_update = bnxt_mc_list_updated(bp, &mask);
9465 }
9466
9467 if (mask != vnic->rx_mask || uc_update || mc_update) {
9468 vnic->rx_mask = mask;
9469
9470 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
c213eae8 9471 bnxt_queue_sp_work(bp);
c0c050c5
MC		 9472 }
9473}
9474
b664f008 9475static int bnxt_cfg_rx_mode(struct bnxt *bp)
c0c050c5
MC		 9476{
9477 struct net_device *dev = bp->dev;
9478 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9479 struct netdev_hw_addr *ha;
9480 int i, off = 0, rc;
9481 bool uc_update;
9482
9483 netif_addr_lock_bh(dev);
9484 uc_update = bnxt_uc_list_updated(bp);
9485 netif_addr_unlock_bh(dev);
9486
9487 if (!uc_update)
9488 goto skip_uc;
9489
9490 mutex_lock(&bp->hwrm_cmd_lock);
9491 for (i = 1; i < vnic->uc_filter_count; i++) {
9492 struct hwrm_cfa_l2_filter_free_input req = {0};
9493
9494 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
9495 -1);
9496
9497 req.l2_filter_id = vnic->fw_l2_filter_id[i];
9498
9499 rc = _hwrm_send_message(bp, &req, sizeof(req),
9500 HWRM_CMD_TIMEOUT);
9501 }
9502 mutex_unlock(&bp->hwrm_cmd_lock);
9503
9504 vnic->uc_filter_count = 1;
9505
9506 netif_addr_lock_bh(dev);
9507 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
9508 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9509 } else {
9510 netdev_for_each_uc_addr(ha, dev) {
9511 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
9512 off += ETH_ALEN;
9513 vnic->uc_filter_count++;
9514 }
9515 }
9516 netif_addr_unlock_bh(dev);
9517
9518 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
9519 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
9520 if (rc) {
9521 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
9522 rc);
9523 vnic->uc_filter_count = i;
b664f008 9524 return rc;
c0c050c5
MC		 9525 }
9526 }
9527
9528skip_uc:
9529 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
b4e30e8e
MC		 9530 if (rc && vnic->mc_list_count) {
9531 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
9532 rc);
9533 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9534 vnic->mc_list_count = 0;
9535 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9536 }
c0c050c5 9537 if (rc)
b4e30e8e 9538 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
c0c050c5 9539 rc);
b664f008
MC		 9540
9541 return rc;
c0c050c5
MC		 9542}
9543
2773dfb2
MC		 9544static bool bnxt_can_reserve_rings(struct bnxt *bp)
9545{
9546#ifdef CONFIG_BNXT_SRIOV
f1ca94de 9547 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
2773dfb2
MC		 9548 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9549
9550 /* No minimum rings were provisioned by the PF. Don't
9551 * reserve rings by default when device is down.
9552 */
9553 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
9554 return true;
9555
9556 if (!netif_running(bp->dev))
9557 return false;
9558 }
9559#endif
9560 return true;
9561}
9562
8079e8f1
MC		 9563/* If the chip and firmware supports RFS */
9564static bool bnxt_rfs_supported(struct bnxt *bp)
9565{
e969ae5b
MC		 9566 if (bp->flags & BNXT_FLAG_CHIP_P5) {
9567 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX)
9568 return true;
41e8d798 9569 return false;
e969ae5b 9570 }
8079e8f1
MC		 9571 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
9572 return true;
ae10ae74
MC		 9573 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9574 return true;
8079e8f1
MC		 9575 return false;
9576}
9577
9578/* If runtime conditions support RFS */
2bcfa6f6
MC		 9579static bool bnxt_rfs_capable(struct bnxt *bp)
9580{
9581#ifdef CONFIG_RFS_ACCEL
8079e8f1 9582 int vnics, max_vnics, max_rss_ctxs;
2bcfa6f6 9583
41e8d798 9584 if (bp->flags & BNXT_FLAG_CHIP_P5)
ac33906c 9585 return bnxt_rfs_supported(bp);
2773dfb2 9586 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
2bcfa6f6
MC		 9587 return false;
9588
9589 vnics = 1 + bp->rx_nr_rings;
8079e8f1
MC		 9590 max_vnics = bnxt_get_max_func_vnics(bp);
9591 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
ae10ae74
MC		 9592
9593 /* RSS contexts not a limiting factor */
9594 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9595 max_rss_ctxs = max_vnics;
8079e8f1 9596 if (vnics > max_vnics || vnics > max_rss_ctxs) {
6a1eef5b
MC		 9597 if (bp->rx_nr_rings > 1)
9598 netdev_warn(bp->dev,
9599 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
9600 min(max_rss_ctxs - 1, max_vnics - 1));
2bcfa6f6 9601 return false;
a2304909 9602 }
2bcfa6f6 9603
f1ca94de 9604 if (!BNXT_NEW_RM(bp))
6a1eef5b
MC		 9605 return true;
9606
9607 if (vnics == bp->hw_resc.resv_vnics)
9608 return true;
9609
780baad4 9610 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
6a1eef5b
MC		 9611 if (vnics <= bp->hw_resc.resv_vnics)
9612 return true;
9613
9614 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
780baad4 9615 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
6a1eef5b 9616 return false;
2bcfa6f6
MC		 9617#else
9618 return false;
9619#endif
9620}
9621
c0c050c5
MC		 9622static netdev_features_t bnxt_fix_features(struct net_device *dev,
9623 netdev_features_t features)
9624{
2bcfa6f6
MC		 9625 struct bnxt *bp = netdev_priv(dev);
9626
a2304909 9627 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
2bcfa6f6 9628 features &= ~NETIF_F_NTUPLE;
5a9f6b23 9629
1054aee8
MC		 9630 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9631 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9632
9633 if (!(features & NETIF_F_GRO))
9634 features &= ~NETIF_F_GRO_HW;
9635
9636 if (features & NETIF_F_GRO_HW)
9637 features &= ~NETIF_F_LRO;
9638
5a9f6b23
MC		 9639 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
9640 * turned on or off together.
9641 */
9642 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
9643 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
9644 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
9645 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9646 NETIF_F_HW_VLAN_STAG_RX);
9647 else
9648 features |= NETIF_F_HW_VLAN_CTAG_RX |
9649 NETIF_F_HW_VLAN_STAG_RX;
9650 }
cf6645f8
MC		 9651#ifdef CONFIG_BNXT_SRIOV
9652 if (BNXT_VF(bp)) {
9653 if (bp->vf.vlan) {
9654 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9655 NETIF_F_HW_VLAN_STAG_RX);
9656 }
9657 }
9658#endif
c0c050c5
MC		 9659 return features;
9660}
9661
9662static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
9663{
9664 struct bnxt *bp = netdev_priv(dev);
9665 u32 flags = bp->flags;
9666 u32 changes;
9667 int rc = 0;
9668 bool re_init = false;
9669 bool update_tpa = false;
9670
9671 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
1054aee8 9672 if (features & NETIF_F_GRO_HW)
c0c050c5 9673 flags |= BNXT_FLAG_GRO;
1054aee8 9674 else if (features & NETIF_F_LRO)
c0c050c5
MC		 9675 flags |= BNXT_FLAG_LRO;
9676
bdbd1eb5
MC		 9677 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9678 flags &= ~BNXT_FLAG_TPA;
9679
c0c050c5
MC		 9680 if (features & NETIF_F_HW_VLAN_CTAG_RX)
9681 flags |= BNXT_FLAG_STRIP_VLAN;
9682
9683 if (features & NETIF_F_NTUPLE)
9684 flags |= BNXT_FLAG_RFS;
9685
9686 changes = flags ^ bp->flags;
9687 if (changes & BNXT_FLAG_TPA) {
9688 update_tpa = true;
9689 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
f45b7b78
MC		 9690 (flags & BNXT_FLAG_TPA) == 0 ||
9691 (bp->flags & BNXT_FLAG_CHIP_P5))
c0c050c5
MC		 9692 re_init = true;
9693 }
9694
9695 if (changes & ~BNXT_FLAG_TPA)
9696 re_init = true;
9697
9698 if (flags != bp->flags) {
9699 u32 old_flags = bp->flags;
9700
2bcfa6f6 9701 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
f45b7b78 9702 bp->flags = flags;
c0c050c5
MC		 9703 if (update_tpa)
9704 bnxt_set_ring_params(bp);
9705 return rc;
9706 }
9707
9708 if (re_init) {
9709 bnxt_close_nic(bp, false, false);
f45b7b78 9710 bp->flags = flags;
c0c050c5
MC		 9711 if (update_tpa)
9712 bnxt_set_ring_params(bp);
9713
9714 return bnxt_open_nic(bp, false, false);
9715 }
9716 if (update_tpa) {
f45b7b78 9717 bp->flags = flags;
c0c050c5
MC		 9718 rc = bnxt_set_tpa(bp,
9719 (flags & BNXT_FLAG_TPA) ?
9720 true : false);
9721 if (rc)
9722 bp->flags = old_flags;
9723 }
9724 }
9725 return rc;
9726}
9727
ffd77621
MC		 9728static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9729 u32 ring_id, u32 *prod, u32 *cons)
9730{
9731 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9732 struct hwrm_dbg_ring_info_get_input req = {0};
9733 int rc;
9734
9735 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9736 req.ring_type = ring_type;
9737 req.fw_ring_id = cpu_to_le32(ring_id);
9738 mutex_lock(&bp->hwrm_cmd_lock);
9739 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9740 if (!rc) {
9741 *prod = le32_to_cpu(resp->producer_index);
9742 *cons = le32_to_cpu(resp->consumer_index);
9743 }
9744 mutex_unlock(&bp->hwrm_cmd_lock);
9745 return rc;
9746}
9747
9f554590
MC		 9748static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9749{
b6ab4b01 9750 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9f554590
MC		 9751 int i = bnapi->index;
9752
3b2b7d9d
MC		 9753 if (!txr)
9754 return;
9755
9f554590
MC		 9756 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9757 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9758 txr->tx_cons);
9759}
9760
9761static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9762{
b6ab4b01 9763 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9f554590
MC		 9764 int i = bnapi->index;
9765
3b2b7d9d
MC		 9766 if (!rxr)
9767 return;
9768
9f554590
MC		 9769 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9770 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9771 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9772 rxr->rx_sw_agg_prod);
9773}
9774
9775static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9776{
9777 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9778 int i = bnapi->index;
9779
9780 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9781 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9782}
9783
c0c050c5
MC		 9784static void bnxt_dbg_dump_states(struct bnxt *bp)
9785{
9786 int i;
9787 struct bnxt_napi *bnapi;
c0c050c5
MC		 9788
9789 for (i = 0; i < bp->cp_nr_rings; i++) {
9790 bnapi = bp->bnapi[i];
c0c050c5 9791 if (netif_msg_drv(bp)) {
9f554590
MC		 9792 bnxt_dump_tx_sw_state(bnapi);
9793 bnxt_dump_rx_sw_state(bnapi);
9794 bnxt_dump_cp_sw_state(bnapi);
c0c050c5
MC		 9795 }
9796 }
9797}
9798
6988bd92 9799static void bnxt_reset_task(struct bnxt *bp, bool silent)
c0c050c5 9800{
6988bd92
MC		 9801 if (!silent)
9802 bnxt_dbg_dump_states(bp);
028de140 9803 if (netif_running(bp->dev)) {
b386cd36
MC		 9804 int rc;
9805
9806 if (!silent)
9807 bnxt_ulp_stop(bp);
028de140 9808 bnxt_close_nic(bp, false, false);
b386cd36
MC		 9809 rc = bnxt_open_nic(bp, false, false);
9810 if (!silent && !rc)
9811 bnxt_ulp_start(bp);
028de140 9812 }
c0c050c5
MC		 9813}
9814
9815static void bnxt_tx_timeout(struct net_device *dev)
9816{
9817 struct bnxt *bp = netdev_priv(dev);
9818
9819 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
9820 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
c213eae8 9821 bnxt_queue_sp_work(bp);
c0c050c5
MC		 9822}
9823
e99e88a9 9824static void bnxt_timer(struct timer_list *t)
c0c050c5 9825{
e99e88a9 9826 struct bnxt *bp = from_timer(bp, t, timer);
c0c050c5
MC		 9827 struct net_device *dev = bp->dev;
9828
9829 if (!netif_running(dev))
9830 return;
9831
9832 if (atomic_read(&bp->intr_sem) != 0)
9833 goto bnxt_restart_timer;
9834
adcc331e
MC		 9835 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
9836 bp->stats_coal_ticks) {
3bdf56c4 9837 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
c213eae8 9838 bnxt_queue_sp_work(bp);
3bdf56c4 9839 }
5a84acbe
SP		 9840
9841 if (bnxt_tc_flower_enabled(bp)) {
9842 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
9843 bnxt_queue_sp_work(bp);
9844 }
a1ef4a79
MC		 9845
9846 if (bp->link_info.phy_retry) {
9847 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
9848 bp->link_info.phy_retry = 0;
9849 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
9850 } else {
9851 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
9852 bnxt_queue_sp_work(bp);
9853 }
9854 }
ffd77621
MC		 9855
9856 if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
9857 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
9858 bnxt_queue_sp_work(bp);
9859 }
c0c050c5
MC		 9860bnxt_restart_timer:
9861 mod_timer(&bp->timer, jiffies + bp->current_interval);
9862}
9863
a551ee94 9864static void bnxt_rtnl_lock_sp(struct bnxt *bp)
6988bd92 9865{
a551ee94
MC		 9866 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
9867 * set. If the device is being closed, bnxt_close() may be holding
6988bd92
MC		 9868 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
9869 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
9870 */
9871 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9872 rtnl_lock();
a551ee94
MC		 9873}
9874
9875static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
9876{
6988bd92
MC		 9877 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9878 rtnl_unlock();
9879}
9880
a551ee94
MC		 9881/* Only called from bnxt_sp_task() */
9882static void bnxt_reset(struct bnxt *bp, bool silent)
9883{
9884 bnxt_rtnl_lock_sp(bp);
9885 if (test_bit(BNXT_STATE_OPEN, &bp->state))
9886 bnxt_reset_task(bp, silent);
9887 bnxt_rtnl_unlock_sp(bp);
9888}
9889
ffd77621
MC		 9890static void bnxt_chk_missed_irq(struct bnxt *bp)
9891{
9892 int i;
9893
9894 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9895 return;
9896
9897 for (i = 0; i < bp->cp_nr_rings; i++) {
9898 struct bnxt_napi *bnapi = bp->bnapi[i];
9899 struct bnxt_cp_ring_info *cpr;
9900 u32 fw_ring_id;
9901 int j;
9902
9903 if (!bnapi)
9904 continue;
9905
9906 cpr = &bnapi->cp_ring;
9907 for (j = 0; j < 2; j++) {
9908 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
9909 u32 val[2];
9910
9911 if (!cpr2 || cpr2->has_more_work ||
9912 !bnxt_has_work(bp, cpr2))
9913 continue;
9914
9915 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
9916 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
9917 continue;
9918 }
9919 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
9920 bnxt_dbg_hwrm_ring_info_get(bp,
9921 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
9922 fw_ring_id, &val[0], &val[1]);
83eb5c5c 9923 cpr->missed_irqs++;
ffd77621
MC		 9924 }
9925 }
9926}
9927
c0c050c5
MC		 9928static void bnxt_cfg_ntp_filters(struct bnxt *);
9929
9930static void bnxt_sp_task(struct work_struct *work)
9931{
9932 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
c0c050c5 9933
4cebdcec
MC		 9934 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9935 smp_mb__after_atomic();
9936 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9937 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5 9938 return;
4cebdcec 9939 }
c0c050c5
MC		 9940
9941 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
9942 bnxt_cfg_rx_mode(bp);
9943
9944 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
9945 bnxt_cfg_ntp_filters(bp);
c0c050c5
MC		 9946 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
9947 bnxt_hwrm_exec_fwd_req(bp);
9948 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9949 bnxt_hwrm_tunnel_dst_port_alloc(
9950 bp, bp->vxlan_port,
9951 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9952 }
9953 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9954 bnxt_hwrm_tunnel_dst_port_free(
9955 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9956 }
7cdd5fc3
AD		 9957 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9958 bnxt_hwrm_tunnel_dst_port_alloc(
9959 bp, bp->nge_port,
9960 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9961 }
9962 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9963 bnxt_hwrm_tunnel_dst_port_free(
9964 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9965 }
00db3cba 9966 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
3bdf56c4 9967 bnxt_hwrm_port_qstats(bp);
00db3cba 9968 bnxt_hwrm_port_qstats_ext(bp);
55e4398d 9969 bnxt_hwrm_pcie_qstats(bp);
00db3cba 9970 }
3bdf56c4 9971
0eaa24b9 9972 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
e2dc9b6e 9973 int rc;
0eaa24b9 9974
e2dc9b6e 9975 mutex_lock(&bp->link_lock);
0eaa24b9
MC		 9976 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
9977 &bp->sp_event))
9978 bnxt_hwrm_phy_qcaps(bp);
9979
e2dc9b6e
MC		 9980 rc = bnxt_update_link(bp, true);
9981 mutex_unlock(&bp->link_lock);
0eaa24b9
MC		 9982 if (rc)
9983 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
9984 rc);
9985 }
a1ef4a79
MC		 9986 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
9987 int rc;
9988
9989 mutex_lock(&bp->link_lock);
9990 rc = bnxt_update_phy_setting(bp);
9991 mutex_unlock(&bp->link_lock);
9992 if (rc) {
9993 netdev_warn(bp->dev, "update phy settings retry failed\n");
9994 } else {
9995 bp->link_info.phy_retry = false;
9996 netdev_info(bp->dev, "update phy settings retry succeeded\n");
9997 }
9998 }
90c694bb 9999 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
e2dc9b6e
MC		 10000 mutex_lock(&bp->link_lock);
10001 bnxt_get_port_module_status(bp);
10002 mutex_unlock(&bp->link_lock);
90c694bb 10003 }
5a84acbe
SP		 10004
10005 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10006 bnxt_tc_flow_stats_work(bp);
10007
ffd77621
MC		 10008 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10009 bnxt_chk_missed_irq(bp);
10010
e2dc9b6e
MC		 10011 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
10012 * must be the last functions to be called before exiting.
10013 */
6988bd92
MC		 10014 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10015 bnxt_reset(bp, false);
4cebdcec 10016
fc0f1929
MC		 10017 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10018 bnxt_reset(bp, true);
10019
4cebdcec
MC		 10020 smp_mb__before_atomic();
10021 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5
MC		 10022}
10023
d1e7925e 10024/* Under rtnl_lock */
98fdbe73
MC		 10025int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10026 int tx_xdp)
d1e7925e
MC		 10027{
10028 int max_rx, max_tx, tx_sets = 1;
780baad4 10029 int tx_rings_needed, stats;
8f23d638 10030 int rx_rings = rx;
6fc2ffdf 10031 int cp, vnics, rc;
d1e7925e 10032
d1e7925e
MC		 10033 if (tcs)
10034 tx_sets = tcs;
10035
10036 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10037 if (rc)
10038 return rc;
10039
10040 if (max_rx < rx)
10041 return -ENOMEM;
10042
5f449249 10043 tx_rings_needed = tx * tx_sets + tx_xdp;
d1e7925e
MC		 10044 if (max_tx < tx_rings_needed)
10045 return -ENOMEM;
10046
6fc2ffdf 10047 vnics = 1;
9b3d15e6 10048 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
6fc2ffdf
EW		 10049 vnics += rx_rings;
10050
8f23d638
MC		 10051 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10052 rx_rings <<= 1;
10053 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
780baad4
VV		 10054 stats = cp;
10055 if (BNXT_NEW_RM(bp)) {
11c3ec7b 10056 cp += bnxt_get_ulp_msix_num(bp);
780baad4
VV		 10057 stats += bnxt_get_ulp_stat_ctxs(bp);
10058 }
6fc2ffdf 10059 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
780baad4 10060 stats, vnics);
d1e7925e
MC		 10061}
10062
17086399
SP		 10063static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10064{
10065 if (bp->bar2) {
10066 pci_iounmap(pdev, bp->bar2);
10067 bp->bar2 = NULL;
10068 }
10069
10070 if (bp->bar1) {
10071 pci_iounmap(pdev, bp->bar1);
10072 bp->bar1 = NULL;
10073 }
10074
10075 if (bp->bar0) {
10076 pci_iounmap(pdev, bp->bar0);
10077 bp->bar0 = NULL;
10078 }
10079}
10080
10081static void bnxt_cleanup_pci(struct bnxt *bp)
10082{
10083 bnxt_unmap_bars(bp, bp->pdev);
10084 pci_release_regions(bp->pdev);
10085 pci_disable_device(bp->pdev);
10086}
10087
18775aa8
MC		 10088static void bnxt_init_dflt_coal(struct bnxt *bp)
10089{
10090 struct bnxt_coal *coal;
10091
10092 /* Tick values in micro seconds.
10093 * 1 coal_buf x bufs_per_record = 1 completion record.
10094 */
10095 coal = &bp->rx_coal;
0c2ff8d7 10096 coal->coal_ticks = 10;
18775aa8
MC		 10097 coal->coal_bufs = 30;
10098 coal->coal_ticks_irq = 1;
10099 coal->coal_bufs_irq = 2;
05abe4dd 10100 coal->idle_thresh = 50;
18775aa8
MC		 10101 coal->bufs_per_record = 2;
10102 coal->budget = 64; /* NAPI budget */
10103
10104 coal = &bp->tx_coal;
10105 coal->coal_ticks = 28;
10106 coal->coal_bufs = 30;
10107 coal->coal_ticks_irq = 2;
10108 coal->coal_bufs_irq = 2;
10109 coal->bufs_per_record = 1;
10110
10111 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10112}
10113
7c380918
MC		 10114static int bnxt_fw_init_one_p1(struct bnxt *bp)
10115{
10116 int rc;
10117
10118 bp->fw_cap = 0;
10119 rc = bnxt_hwrm_ver_get(bp);
10120 if (rc)
10121 return rc;
10122
10123 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10124 rc = bnxt_alloc_kong_hwrm_resources(bp);
10125 if (rc)
10126 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10127 }
10128
10129 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10130 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10131 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10132 if (rc)
10133 return rc;
10134 }
10135 rc = bnxt_hwrm_func_reset(bp);
10136 if (rc)
10137 return -ENODEV;
10138
10139 bnxt_hwrm_fw_set_time(bp);
10140 return 0;
10141}
10142
10143static int bnxt_fw_init_one_p2(struct bnxt *bp)
10144{
10145 int rc;
10146
10147 /* Get the MAX capabilities for this function */
10148 rc = bnxt_hwrm_func_qcaps(bp);
10149 if (rc) {
10150 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10151 rc);
10152 return -ENODEV;
10153 }
10154
10155 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10156 if (rc)
10157 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10158 rc);
10159
07f83d72
MC		 10160 rc = bnxt_hwrm_error_recovery_qcfg(bp);
10161 if (rc)
10162 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
10163 rc);
10164
7c380918
MC		 10165 rc = bnxt_hwrm_func_drv_rgtr(bp);
10166 if (rc)
10167 return -ENODEV;
10168
10169 rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
10170 if (rc)
10171 return -ENODEV;
10172
10173 bnxt_hwrm_func_qcfg(bp);
10174 bnxt_hwrm_vnic_qcaps(bp);
10175 bnxt_hwrm_port_led_qcaps(bp);
10176 bnxt_ethtool_init(bp);
10177 bnxt_dcb_init(bp);
10178 return 0;
10179}
10180
ba642ab7
MC		 10181static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
10182{
10183 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
10184 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10185 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10186 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10187 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10188 if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
10189 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10190 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10191 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10192 }
10193}
10194
10195static void bnxt_set_dflt_rfs(struct bnxt *bp)
10196{
10197 struct net_device *dev = bp->dev;
10198
10199 dev->hw_features &= ~NETIF_F_NTUPLE;
10200 dev->features &= ~NETIF_F_NTUPLE;
10201 bp->flags &= ~BNXT_FLAG_RFS;
10202 if (bnxt_rfs_supported(bp)) {
10203 dev->hw_features |= NETIF_F_NTUPLE;
10204 if (bnxt_rfs_capable(bp)) {
10205 bp->flags |= BNXT_FLAG_RFS;
10206 dev->features |= NETIF_F_NTUPLE;
10207 }
10208 }
10209}
10210
10211static void bnxt_fw_init_one_p3(struct bnxt *bp)
10212{
10213 struct pci_dev *pdev = bp->pdev;
10214
10215 bnxt_set_dflt_rss_hash_type(bp);
10216 bnxt_set_dflt_rfs(bp);
10217
10218 bnxt_get_wol_settings(bp);
10219 if (bp->flags & BNXT_FLAG_WOL_CAP)
10220 device_set_wakeup_enable(&pdev->dev, bp->wol);
10221 else
10222 device_set_wakeup_capable(&pdev->dev, false);
10223
10224 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10225 bnxt_hwrm_coal_params_qcaps(bp);
10226}
10227
ec5d31e3
MC		 10228static int bnxt_fw_init_one(struct bnxt *bp)
10229{
10230 int rc;
10231
10232 rc = bnxt_fw_init_one_p1(bp);
10233 if (rc) {
10234 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
10235 return rc;
10236 }
10237 rc = bnxt_fw_init_one_p2(bp);
10238 if (rc) {
10239 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
10240 return rc;
10241 }
10242 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
10243 if (rc)
10244 return rc;
10245 bnxt_fw_init_one_p3(bp);
10246 return 0;
10247}
10248
c0c050c5
MC		 10249static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
10250{
10251 int rc;
10252 struct bnxt *bp = netdev_priv(dev);
10253
10254 SET_NETDEV_DEV(dev, &pdev->dev);
10255
10256 /* enable device (incl. PCI PM wakeup), and bus-mastering */
10257 rc = pci_enable_device(pdev);
10258 if (rc) {
10259 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
10260 goto init_err;
10261 }
10262
10263 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
10264 dev_err(&pdev->dev,
10265 "Cannot find PCI device base address, aborting\n");
10266 rc = -ENODEV;
10267 goto init_err_disable;
10268 }
10269
10270 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
10271 if (rc) {
10272 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
10273 goto init_err_disable;
10274 }
10275
10276 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
10277 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
10278 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
10279 goto init_err_disable;
10280 }
10281
10282 pci_set_master(pdev);
10283
10284 bp->dev = dev;
10285 bp->pdev = pdev;
10286
10287 bp->bar0 = pci_ioremap_bar(pdev, 0);
10288 if (!bp->bar0) {
10289 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
10290 rc = -ENOMEM;
10291 goto init_err_release;
10292 }
10293
10294 bp->bar1 = pci_ioremap_bar(pdev, 2);
10295 if (!bp->bar1) {
10296 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
10297 rc = -ENOMEM;
10298 goto init_err_release;
10299 }
10300
10301 bp->bar2 = pci_ioremap_bar(pdev, 4);
10302 if (!bp->bar2) {
10303 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
10304 rc = -ENOMEM;
10305 goto init_err_release;
10306 }
10307
6316ea6d
SB		 10308 pci_enable_pcie_error_reporting(pdev);
10309
c0c050c5
MC		 10310 INIT_WORK(&bp->sp_task, bnxt_sp_task);
10311
10312 spin_lock_init(&bp->ntp_fltr_lock);
697197e5
MC		 10313#if BITS_PER_LONG == 32
10314 spin_lock_init(&bp->db_lock);
10315#endif
c0c050c5
MC		 10316
10317 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
10318 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
10319
18775aa8 10320 bnxt_init_dflt_coal(bp);
51f30785 10321
e99e88a9 10322 timer_setup(&bp->timer, bnxt_timer, 0);
c0c050c5
MC		 10323 bp->current_interval = BNXT_TIMER_INTERVAL;
10324
caefe526 10325 clear_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 10326 return 0;
10327
10328init_err_release:
17086399 10329 bnxt_unmap_bars(bp, pdev);
c0c050c5
MC		 10330 pci_release_regions(pdev);
10331
10332init_err_disable:
10333 pci_disable_device(pdev);
10334
10335init_err:
10336 return rc;
10337}
10338
10339/* rtnl_lock held */
10340static int bnxt_change_mac_addr(struct net_device *dev, void *p)
10341{
10342 struct sockaddr *addr = p;
1fc2cfd0
JH		 10343 struct bnxt *bp = netdev_priv(dev);
10344 int rc = 0;
c0c050c5
MC		 10345
10346 if (!is_valid_ether_addr(addr->sa_data))
10347 return -EADDRNOTAVAIL;
10348
c1a7bdff
MC		 10349 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
10350 return 0;
10351
28ea334b 10352 rc = bnxt_approve_mac(bp, addr->sa_data, true);
84c33dd3
MC		 10353 if (rc)
10354 return rc;
bdd4347b 10355
c0c050c5 10356 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1fc2cfd0
JH		 10357 if (netif_running(dev)) {
10358 bnxt_close_nic(bp, false, false);
10359 rc = bnxt_open_nic(bp, false, false);
10360 }
c0c050c5 10361
1fc2cfd0 10362 return rc;
c0c050c5
MC		 10363}
10364
10365/* rtnl_lock held */
10366static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
10367{
10368 struct bnxt *bp = netdev_priv(dev);
10369
c0c050c5
MC		 10370 if (netif_running(dev))
10371 bnxt_close_nic(bp, false, false);
10372
10373 dev->mtu = new_mtu;
10374 bnxt_set_ring_params(bp);
10375
10376 if (netif_running(dev))
10377 return bnxt_open_nic(bp, false, false);
10378
10379 return 0;
10380}
10381
c5e3deb8 10382int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
c0c050c5
MC		 10383{
10384 struct bnxt *bp = netdev_priv(dev);
3ffb6a39 10385 bool sh = false;
d1e7925e 10386 int rc;
16e5cc64 10387
c0c050c5 10388 if (tc > bp->max_tc) {
b451c8b6 10389 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
c0c050c5
MC		 10390 tc, bp->max_tc);
10391 return -EINVAL;
10392 }
10393
10394 if (netdev_get_num_tc(dev) == tc)
10395 return 0;
10396
3ffb6a39
MC		 10397 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10398 sh = true;
10399
98fdbe73
MC		 10400 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
10401 sh, tc, bp->tx_nr_rings_xdp);
d1e7925e
MC		 10402 if (rc)
10403 return rc;
c0c050c5
MC		 10404
10405 /* Needs to close the device and do hw resource re-allocations */
10406 if (netif_running(bp->dev))
10407 bnxt_close_nic(bp, true, false);
10408
10409 if (tc) {
10410 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
10411 netdev_set_num_tc(dev, tc);
10412 } else {
10413 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10414 netdev_reset_tc(dev);
10415 }
87e9b377 10416 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
3ffb6a39
MC		 10417 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
10418 bp->tx_nr_rings + bp->rx_nr_rings;
c0c050c5
MC		 10419
10420 if (netif_running(bp->dev))
10421 return bnxt_open_nic(bp, true, false);
10422
10423 return 0;
10424}
10425
9e0fd15d
JP		 10426static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10427 void *cb_priv)
c5e3deb8 10428{
9e0fd15d 10429 struct bnxt *bp = cb_priv;
de4784ca 10430
312324f1
JK		 10431 if (!bnxt_tc_flower_enabled(bp) ||
10432 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
38cf0426 10433 return -EOPNOTSUPP;
c5e3deb8 10434
9e0fd15d
JP		 10435 switch (type) {
10436 case TC_SETUP_CLSFLOWER:
10437 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
10438 default:
10439 return -EOPNOTSUPP;
10440 }
10441}
10442
955bcb6e
PNA		 10443static LIST_HEAD(bnxt_block_cb_list);
10444
2ae7408f
SP		 10445static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
10446 void *type_data)
10447{
4e95bc26
PNA		 10448 struct bnxt *bp = netdev_priv(dev);
10449
2ae7408f 10450 switch (type) {
9e0fd15d 10451 case TC_SETUP_BLOCK:
955bcb6e
PNA		 10452 return flow_block_cb_setup_simple(type_data,
10453 &bnxt_block_cb_list,
4e95bc26
PNA		 10454 bnxt_setup_tc_block_cb,
10455 bp, bp, true);
575ed7d3 10456 case TC_SETUP_QDISC_MQPRIO: {
2ae7408f
SP		 10457 struct tc_mqprio_qopt *mqprio = type_data;
10458
10459 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
56f36acd 10460
2ae7408f
SP		 10461 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
10462 }
10463 default:
10464 return -EOPNOTSUPP;
10465 }
c5e3deb8
MC		 10466}
10467
c0c050c5
MC		 10468#ifdef CONFIG_RFS_ACCEL
10469static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
10470 struct bnxt_ntuple_filter *f2)
10471{
10472 struct flow_keys *keys1 = &f1->fkeys;
10473 struct flow_keys *keys2 = &f2->fkeys;
10474
10475 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
10476 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
10477 keys1->ports.ports == keys2->ports.ports &&
10478 keys1->basic.ip_proto == keys2->basic.ip_proto &&
10479 keys1->basic.n_proto == keys2->basic.n_proto &&
61aad724 10480 keys1->control.flags == keys2->control.flags &&
a54c4d74
MC		 10481 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
10482 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
c0c050c5
MC		 10483 return true;
10484
10485 return false;
10486}
10487
10488static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
10489 u16 rxq_index, u32 flow_id)
10490{
10491 struct bnxt *bp = netdev_priv(dev);
10492 struct bnxt_ntuple_filter *fltr, *new_fltr;
10493 struct flow_keys *fkeys;
10494 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
a54c4d74 10495 int rc = 0, idx, bit_id, l2_idx = 0;
c0c050c5
MC		 10496 struct hlist_head *head;
10497
a54c4d74
MC		 10498 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
10499 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10500 int off = 0, j;
10501
10502 netif_addr_lock_bh(dev);
10503 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
10504 if (ether_addr_equal(eth->h_dest,
10505 vnic->uc_list + off)) {
10506 l2_idx = j + 1;
10507 break;
10508 }
10509 }
10510 netif_addr_unlock_bh(dev);
10511 if (!l2_idx)
10512 return -EINVAL;
10513 }
c0c050c5
MC		 10514 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
10515 if (!new_fltr)
10516 return -ENOMEM;
10517
10518 fkeys = &new_fltr->fkeys;
10519 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
10520 rc = -EPROTONOSUPPORT;
10521 goto err_free;
10522 }
10523
dda0e746
MC		 10524 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
10525 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
c0c050c5
MC		 10526 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
10527 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
10528 rc = -EPROTONOSUPPORT;
10529 goto err_free;
10530 }
dda0e746
MC		 10531 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
10532 bp->hwrm_spec_code < 0x10601) {
10533 rc = -EPROTONOSUPPORT;
10534 goto err_free;
10535 }
61aad724
MC		 10536 if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
10537 bp->hwrm_spec_code < 0x10601) {
10538 rc = -EPROTONOSUPPORT;
10539 goto err_free;
10540 }
c0c050c5 10541
a54c4d74 10542 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
c0c050c5
MC		 10543 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
10544
10545 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
10546 head = &bp->ntp_fltr_hash_tbl[idx];
10547 rcu_read_lock();
10548 hlist_for_each_entry_rcu(fltr, head, hash) {
10549 if (bnxt_fltr_match(fltr, new_fltr)) {
10550 rcu_read_unlock();
10551 rc = 0;
10552 goto err_free;
10553 }
10554 }
10555 rcu_read_unlock();
10556
10557 spin_lock_bh(&bp->ntp_fltr_lock);
84e86b98
MC		 10558 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
10559 BNXT_NTP_FLTR_MAX_FLTR, 0);
10560 if (bit_id < 0) {
c0c050c5
MC		 10561 spin_unlock_bh(&bp->ntp_fltr_lock);
10562 rc = -ENOMEM;
10563 goto err_free;
10564 }
10565
84e86b98 10566 new_fltr->sw_id = (u16)bit_id;
c0c050c5 10567 new_fltr->flow_id = flow_id;
a54c4d74 10568 new_fltr->l2_fltr_idx = l2_idx;
c0c050c5
MC		 10569 new_fltr->rxq = rxq_index;
10570 hlist_add_head_rcu(&new_fltr->hash, head);
10571 bp->ntp_fltr_count++;
10572 spin_unlock_bh(&bp->ntp_fltr_lock);
10573
10574 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
c213eae8 10575 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10576
10577 return new_fltr->sw_id;
10578
10579err_free:
10580 kfree(new_fltr);
10581 return rc;
10582}
10583
10584static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10585{
10586 int i;
10587
10588 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
10589 struct hlist_head *head;
10590 struct hlist_node *tmp;
10591 struct bnxt_ntuple_filter *fltr;
10592 int rc;
10593
10594 head = &bp->ntp_fltr_hash_tbl[i];
10595 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
10596 bool del = false;
10597
10598 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
10599 if (rps_may_expire_flow(bp->dev, fltr->rxq,
10600 fltr->flow_id,
10601 fltr->sw_id)) {
10602 bnxt_hwrm_cfa_ntuple_filter_free(bp,
10603 fltr);
10604 del = true;
10605 }
10606 } else {
10607 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
10608 fltr);
10609 if (rc)
10610 del = true;
10611 else
10612 set_bit(BNXT_FLTR_VALID, &fltr->state);
10613 }
10614
10615 if (del) {
10616 spin_lock_bh(&bp->ntp_fltr_lock);
10617 hlist_del_rcu(&fltr->hash);
10618 bp->ntp_fltr_count--;
10619 spin_unlock_bh(&bp->ntp_fltr_lock);
10620 synchronize_rcu();
10621 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
10622 kfree(fltr);
10623 }
10624 }
10625 }
19241368
JH		 10626 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
10627 netdev_info(bp->dev, "Receive PF driver unload event!");
c0c050c5
MC		 10628}
10629
10630#else
10631
10632static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10633{
10634}
10635
10636#endif /* CONFIG_RFS_ACCEL */
10637
ad51b8e9
AD		 10638static void bnxt_udp_tunnel_add(struct net_device *dev,
10639 struct udp_tunnel_info *ti)
c0c050c5
MC		 10640{
10641 struct bnxt *bp = netdev_priv(dev);
10642
ad51b8e9 10643 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 10644 return;
10645
ad51b8e9 10646 if (!netif_running(dev))
c0c050c5
MC		 10647 return;
10648
ad51b8e9
AD		 10649 switch (ti->type) {
10650 case UDP_TUNNEL_TYPE_VXLAN:
10651 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
10652 return;
c0c050c5 10653
ad51b8e9
AD		 10654 bp->vxlan_port_cnt++;
10655 if (bp->vxlan_port_cnt == 1) {
10656 bp->vxlan_port = ti->port;
10657 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
c213eae8 10658 bnxt_queue_sp_work(bp);
ad51b8e9
AD		 10659 }
10660 break;
7cdd5fc3
AD		 10661 case UDP_TUNNEL_TYPE_GENEVE:
10662 if (bp->nge_port_cnt && bp->nge_port != ti->port)
10663 return;
10664
10665 bp->nge_port_cnt++;
10666 if (bp->nge_port_cnt == 1) {
10667 bp->nge_port = ti->port;
10668 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
10669 }
10670 break;
ad51b8e9
AD		 10671 default:
10672 return;
c0c050c5 10673 }
ad51b8e9 10674
c213eae8 10675 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10676}
10677
ad51b8e9
AD		 10678static void bnxt_udp_tunnel_del(struct net_device *dev,
10679 struct udp_tunnel_info *ti)
c0c050c5
MC		 10680{
10681 struct bnxt *bp = netdev_priv(dev);
10682
ad51b8e9 10683 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
c0c050c5
MC		 10684 return;
10685
ad51b8e9 10686 if (!netif_running(dev))
c0c050c5
MC		 10687 return;
10688
ad51b8e9
AD		 10689 switch (ti->type) {
10690 case UDP_TUNNEL_TYPE_VXLAN:
10691 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
10692 return;
c0c050c5
MC		 10693 bp->vxlan_port_cnt--;
10694
ad51b8e9
AD		 10695 if (bp->vxlan_port_cnt != 0)
10696 return;
10697
10698 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
10699 break;
7cdd5fc3
AD		 10700 case UDP_TUNNEL_TYPE_GENEVE:
10701 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
10702 return;
10703 bp->nge_port_cnt--;
10704
10705 if (bp->nge_port_cnt != 0)
10706 return;
10707
10708 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
10709 break;
ad51b8e9
AD		 10710 default:
10711 return;
c0c050c5 10712 }
ad51b8e9 10713
c213eae8 10714 bnxt_queue_sp_work(bp);
c0c050c5
MC		 10715}
10716
39d8ba2e
MC		 10717static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
10718 struct net_device *dev, u32 filter_mask,
10719 int nlflags)
10720{
10721 struct bnxt *bp = netdev_priv(dev);
10722
10723 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
10724 nlflags, filter_mask, NULL);
10725}
10726
10727static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
2fd527b7 10728 u16 flags, struct netlink_ext_ack *extack)
39d8ba2e
MC		 10729{
10730 struct bnxt *bp = netdev_priv(dev);
10731 struct nlattr *attr, *br_spec;
10732 int rem, rc = 0;
10733
10734 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
10735 return -EOPNOTSUPP;
10736
10737 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
10738 if (!br_spec)
10739 return -EINVAL;
10740
10741 nla_for_each_nested(attr, br_spec, rem) {
10742 u16 mode;
10743
10744 if (nla_type(attr) != IFLA_BRIDGE_MODE)
10745 continue;
10746
10747 if (nla_len(attr) < sizeof(mode))
10748 return -EINVAL;
10749
10750 mode = nla_get_u16(attr);
10751 if (mode == bp->br_mode)
10752 break;
10753
10754 rc = bnxt_hwrm_set_br_mode(bp, mode);
10755 if (!rc)
10756 bp->br_mode = mode;
10757 break;
10758 }
10759 return rc;
10760}
10761
52d5254a
FF		 10762int bnxt_get_port_parent_id(struct net_device *dev,
10763 struct netdev_phys_item_id *ppid)
c124a62f 10764{
52d5254a
FF		 10765 struct bnxt *bp = netdev_priv(dev);
10766
c124a62f
SP		 10767 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
10768 return -EOPNOTSUPP;
10769
10770 /* The PF and it's VF-reps only support the switchdev framework */
10771 if (!BNXT_PF(bp))
10772 return -EOPNOTSUPP;
10773
52d5254a
FF		 10774 ppid->id_len = sizeof(bp->switch_id);
10775 memcpy(ppid->id, bp->switch_id, ppid->id_len);
c124a62f 10776
52d5254a 10777 return 0;
c124a62f
SP		 10778}
10779
c9c49a65
JP		 10780static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
10781{
10782 struct bnxt *bp = netdev_priv(dev);
10783
10784 return &bp->dl_port;
10785}
10786
c0c050c5
MC		 10787static const struct net_device_ops bnxt_netdev_ops = {
10788 .ndo_open = bnxt_open,
10789 .ndo_start_xmit = bnxt_start_xmit,
10790 .ndo_stop = bnxt_close,
10791 .ndo_get_stats64 = bnxt_get_stats64,
10792 .ndo_set_rx_mode = bnxt_set_rx_mode,
10793 .ndo_do_ioctl = bnxt_ioctl,
10794 .ndo_validate_addr = eth_validate_addr,
10795 .ndo_set_mac_address = bnxt_change_mac_addr,
10796 .ndo_change_mtu = bnxt_change_mtu,
10797 .ndo_fix_features = bnxt_fix_features,
10798 .ndo_set_features = bnxt_set_features,
10799 .ndo_tx_timeout = bnxt_tx_timeout,
10800#ifdef CONFIG_BNXT_SRIOV
10801 .ndo_get_vf_config = bnxt_get_vf_config,
10802 .ndo_set_vf_mac = bnxt_set_vf_mac,
10803 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
10804 .ndo_set_vf_rate = bnxt_set_vf_bw,
10805 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
10806 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
746df139 10807 .ndo_set_vf_trust = bnxt_set_vf_trust,
c0c050c5
MC		 10808#endif
10809 .ndo_setup_tc = bnxt_setup_tc,
10810#ifdef CONFIG_RFS_ACCEL
10811 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
10812#endif
ad51b8e9
AD		 10813 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add,
10814 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del,
f4e63525 10815 .ndo_bpf = bnxt_xdp,
f18c2b77 10816 .ndo_xdp_xmit = bnxt_xdp_xmit,
39d8ba2e
MC		 10817 .ndo_bridge_getlink = bnxt_bridge_getlink,
10818 .ndo_bridge_setlink = bnxt_bridge_setlink,
c9c49a65 10819 .ndo_get_devlink_port = bnxt_get_devlink_port,
c0c050c5
MC		 10820};
10821
10822static void bnxt_remove_one(struct pci_dev *pdev)
10823{
10824 struct net_device *dev = pci_get_drvdata(pdev);
10825 struct bnxt *bp = netdev_priv(dev);
10826
4ab0c6a8 10827 if (BNXT_PF(bp)) {
c0c050c5 10828 bnxt_sriov_disable(bp);
4ab0c6a8
SP		 10829 bnxt_dl_unregister(bp);
10830 }
c0c050c5 10831
6316ea6d 10832 pci_disable_pcie_error_reporting(pdev);
c0c050c5 10833 unregister_netdev(dev);
2ae7408f 10834 bnxt_shutdown_tc(bp);
c213eae8 10835 bnxt_cancel_sp_work(bp);
c0c050c5
MC		 10836 bp->sp_event = 0;
10837
7809592d 10838 bnxt_clear_int_mode(bp);
be58a0da 10839 bnxt_hwrm_func_drv_unrgtr(bp);
c0c050c5 10840 bnxt_free_hwrm_resources(bp);
e605db80 10841 bnxt_free_hwrm_short_cmd_req(bp);
eb513658 10842 bnxt_ethtool_free(bp);
7df4ae9f 10843 bnxt_dcb_free(bp);
a588e458
MC		 10844 kfree(bp->edev);
10845 bp->edev = NULL;
c20dc142 10846 bnxt_cleanup_pci(bp);
98f04cf0
MC		 10847 bnxt_free_ctx_mem(bp);
10848 kfree(bp->ctx);
10849 bp->ctx = NULL;
fd3ab1c7 10850 bnxt_free_port_stats(bp);
c0c050c5 10851 free_netdev(dev);
c0c050c5
MC		 10852}
10853
ba642ab7 10854static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
c0c050c5
MC		 10855{
10856 int rc = 0;
10857 struct bnxt_link_info *link_info = &bp->link_info;
c0c050c5 10858
170ce013
MC		 10859 rc = bnxt_hwrm_phy_qcaps(bp);
10860 if (rc) {
10861 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
10862 rc);
10863 return rc;
10864 }
c0c050c5
MC		 10865 rc = bnxt_update_link(bp, false);
10866 if (rc) {
10867 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
10868 rc);
10869 return rc;
10870 }
10871
93ed8117
MC		 10872 /* Older firmware does not have supported_auto_speeds, so assume
10873 * that all supported speeds can be autonegotiated.
10874 */
10875 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
10876 link_info->support_auto_speeds = link_info->support_speeds;
10877
ba642ab7
MC		 10878 if (!fw_dflt)
10879 return 0;
10880
c0c050c5 10881 /*initialize the ethool setting copy with NVM settings */
0d8abf02 10882 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
c9ee9516
MC		 10883 link_info->autoneg = BNXT_AUTONEG_SPEED;
10884 if (bp->hwrm_spec_code >= 0x10201) {
10885 if (link_info->auto_pause_setting &
10886 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10887 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10888 } else {
10889 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10890 }
0d8abf02 10891 link_info->advertising = link_info->auto_link_speeds;
0d8abf02
MC		 10892 } else {
10893 link_info->req_link_speed = link_info->force_link_speed;
10894 link_info->req_duplex = link_info->duplex_setting;
c0c050c5 10895 }
c9ee9516
MC		 10896 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10897 link_info->req_flow_ctrl =
10898 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10899 else
10900 link_info->req_flow_ctrl = link_info->force_pause_setting;
ba642ab7 10901 return 0;
c0c050c5
MC		 10902}
10903
10904static int bnxt_get_max_irq(struct pci_dev *pdev)
10905{
10906 u16 ctrl;
10907
10908 if (!pdev->msix_cap)
10909 return 1;
10910
10911 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
10912 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
10913}
10914
6e6c5a57
MC		 10915static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10916 int *max_cp)
c0c050c5 10917{
6a4f2947 10918 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
e30fbc33 10919 int max_ring_grps = 0, max_irq;
c0c050c5 10920
6a4f2947
MC		 10921 *max_tx = hw_resc->max_tx_rings;
10922 *max_rx = hw_resc->max_rx_rings;
e30fbc33
MC		 10923 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
10924 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
10925 bnxt_get_ulp_msix_num(bp),
c027c6b4 10926 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
e30fbc33
MC		 10927 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10928 *max_cp = min_t(int, *max_cp, max_irq);
6a4f2947 10929 max_ring_grps = hw_resc->max_hw_ring_grps;
76595193
PS		 10930 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
10931 *max_cp -= 1;
10932 *max_rx -= 2;
10933 }
c0c050c5
MC		 10934 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10935 *max_rx >>= 1;
e30fbc33
MC		 10936 if (bp->flags & BNXT_FLAG_CHIP_P5) {
10937 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
10938 /* On P5 chips, max_cp output param should be available NQs */
10939 *max_cp = max_irq;
10940 }
b72d4a68 10941 *max_rx = min_t(int, *max_rx, max_ring_grps);
6e6c5a57
MC		 10942}
10943
10944int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
10945{
10946 int rx, tx, cp;
10947
10948 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
78f058a4
MC		 10949 *max_rx = rx;
10950 *max_tx = tx;
6e6c5a57
MC		 10951 if (!rx || !tx || !cp)
10952 return -ENOMEM;
10953
6e6c5a57
MC		 10954 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
10955}
10956
e4060d30
MC		 10957static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10958 bool shared)
10959{
10960 int rc;
10961
10962 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
bdbd1eb5
MC		 10963 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
10964 /* Not enough rings, try disabling agg rings. */
10965 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
10966 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
07f4fde5
MC		 10967 if (rc) {
10968 /* set BNXT_FLAG_AGG_RINGS back for consistency */
10969 bp->flags |= BNXT_FLAG_AGG_RINGS;
bdbd1eb5 10970 return rc;
07f4fde5 10971 }
bdbd1eb5 10972 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
1054aee8
MC		 10973 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10974 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
bdbd1eb5
MC		 10975 bnxt_set_ring_params(bp);
10976 }
e4060d30
MC		 10977
10978 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
10979 int max_cp, max_stat, max_irq;
10980
10981 /* Reserve minimum resources for RoCE */
10982 max_cp = bnxt_get_max_func_cp_rings(bp);
10983 max_stat = bnxt_get_max_func_stat_ctxs(bp);
10984 max_irq = bnxt_get_max_func_irqs(bp);
10985 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
10986 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
10987 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
10988 return 0;
10989
10990 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
10991 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
10992 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
10993 max_cp = min_t(int, max_cp, max_irq);
10994 max_cp = min_t(int, max_cp, max_stat);
10995 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
10996 if (rc)
10997 rc = 0;
10998 }
10999 return rc;
11000}
11001
58ea801a
MC		 11002/* In initial default shared ring setting, each shared ring must have a
11003 * RX/TX ring pair.
11004 */
11005static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
11006{
11007 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
11008 bp->rx_nr_rings = bp->cp_nr_rings;
11009 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
11010 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11011}
11012
702c221c 11013static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
6e6c5a57
MC		 11014{
11015 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6e6c5a57 11016
2773dfb2
MC		 11017 if (!bnxt_can_reserve_rings(bp))
11018 return 0;
11019
6e6c5a57
MC		 11020 if (sh)
11021 bp->flags |= BNXT_FLAG_SHARED_RINGS;
d629522e 11022 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
1d3ef13d
MC		 11023 /* Reduce default rings on multi-port cards so that total default
11024 * rings do not exceed CPU count.
11025 */
11026 if (bp->port_count > 1) {
11027 int max_rings =
11028 max_t(int, num_online_cpus() / bp->port_count, 1);
11029
11030 dflt_rings = min_t(int, dflt_rings, max_rings);
11031 }
e4060d30 11032 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6e6c5a57
MC		 11033 if (rc)
11034 return rc;
11035 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
11036 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
58ea801a
MC		 11037 if (sh)
11038 bnxt_trim_dflt_sh_rings(bp);
11039 else
11040 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
11041 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
391be5c2 11042
674f50a5 11043 rc = __bnxt_reserve_rings(bp);
391be5c2
MC		 11044 if (rc)
11045 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
58ea801a
MC		 11046 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11047 if (sh)
11048 bnxt_trim_dflt_sh_rings(bp);
391be5c2 11049
674f50a5
MC		 11050 /* Rings may have been trimmed, re-reserve the trimmed rings. */
11051 if (bnxt_need_reserve_rings(bp)) {
11052 rc = __bnxt_reserve_rings(bp);
11053 if (rc)
11054 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
11055 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11056 }
76595193
PS		 11057 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11058 bp->rx_nr_rings++;
11059 bp->cp_nr_rings++;
11060 }
6e6c5a57 11061 return rc;
c0c050c5
MC		 11062}
11063
47558acd
MC		 11064static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
11065{
11066 int rc;
11067
11068 if (bp->tx_nr_rings)
11069 return 0;
11070
6b95c3e9
MC		 11071 bnxt_ulp_irq_stop(bp);
11072 bnxt_clear_int_mode(bp);
47558acd
MC		 11073 rc = bnxt_set_dflt_rings(bp, true);
11074 if (rc) {
11075 netdev_err(bp->dev, "Not enough rings available.\n");
6b95c3e9 11076 goto init_dflt_ring_err;
47558acd
MC		 11077 }
11078 rc = bnxt_init_int_mode(bp);
11079 if (rc)
6b95c3e9
MC		 11080 goto init_dflt_ring_err;
11081
47558acd
MC		 11082 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11083 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
11084 bp->flags |= BNXT_FLAG_RFS;
11085 bp->dev->features |= NETIF_F_NTUPLE;
11086 }
6b95c3e9
MC		 11087init_dflt_ring_err:
11088 bnxt_ulp_irq_restart(bp, rc);
11089 return rc;
47558acd
MC		 11090}
11091
80fcaf46 11092int bnxt_restore_pf_fw_resources(struct bnxt *bp)
7b08f661 11093{
80fcaf46
MC		 11094 int rc;
11095
7b08f661
MC		 11096 ASSERT_RTNL();
11097 bnxt_hwrm_func_qcaps(bp);
1a037782
VD		 11098
11099 if (netif_running(bp->dev))
11100 __bnxt_close_nic(bp, true, false);
11101
ec86f14e 11102 bnxt_ulp_irq_stop(bp);
80fcaf46
MC		 11103 bnxt_clear_int_mode(bp);
11104 rc = bnxt_init_int_mode(bp);
ec86f14e 11105 bnxt_ulp_irq_restart(bp, rc);
1a037782
VD		 11106
11107 if (netif_running(bp->dev)) {
11108 if (rc)
11109 dev_close(bp->dev);
11110 else
11111 rc = bnxt_open_nic(bp, true, false);
11112 }
11113
80fcaf46 11114 return rc;
7b08f661
MC		 11115}
11116
a22a6ac2
MC		 11117static int bnxt_init_mac_addr(struct bnxt *bp)
11118{
11119 int rc = 0;
11120
11121 if (BNXT_PF(bp)) {
11122 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
11123 } else {
11124#ifdef CONFIG_BNXT_SRIOV
11125 struct bnxt_vf_info *vf = &bp->vf;
28ea334b 11126 bool strict_approval = true;
a22a6ac2
MC		 11127
11128 if (is_valid_ether_addr(vf->mac_addr)) {
91cdda40 11129 /* overwrite netdev dev_addr with admin VF MAC */
a22a6ac2 11130 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
28ea334b
MC		 11131 /* Older PF driver or firmware may not approve this
11132 * correctly.
11133 */
11134 strict_approval = false;
a22a6ac2
MC		 11135 } else {
11136 eth_hw_addr_random(bp->dev);
a22a6ac2 11137 }
28ea334b 11138 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
a22a6ac2
MC		 11139#endif
11140 }
11141 return rc;
11142}
11143
03213a99
JP		 11144static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
11145{
11146 struct pci_dev *pdev = bp->pdev;
11147 int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
11148 u32 dw;
11149
11150 if (!pos) {
11151 netdev_info(bp->dev, "Unable do read adapter's DSN");
11152 return -EOPNOTSUPP;
11153 }
11154
11155 /* DSN (two dw) is at an offset of 4 from the cap pos */
11156 pos += 4;
11157 pci_read_config_dword(pdev, pos, &dw);
11158 put_unaligned_le32(dw, &dsn[0]);
11159 pci_read_config_dword(pdev, pos + 4, &dw);
11160 put_unaligned_le32(dw, &dsn[4]);
11161 return 0;
11162}
11163
c0c050c5
MC		 11164static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
11165{
11166 static int version_printed;
11167 struct net_device *dev;
11168 struct bnxt *bp;
6e6c5a57 11169 int rc, max_irqs;
c0c050c5 11170
4e00338a 11171 if (pci_is_bridge(pdev))
fa853dda
PS		 11172 return -ENODEV;
11173
c0c050c5
MC		 11174 if (version_printed++ == 0)
11175 pr_info("%s", version);
11176
11177 max_irqs = bnxt_get_max_irq(pdev);
11178 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
11179 if (!dev)
11180 return -ENOMEM;
11181
11182 bp = netdev_priv(dev);
9c1fabdf 11183 bnxt_set_max_func_irqs(bp, max_irqs);
c0c050c5
MC		 11184
11185 if (bnxt_vf_pciid(ent->driver_data))
11186 bp->flags |= BNXT_FLAG_VF;
11187
2bcfa6f6 11188 if (pdev->msix_cap)
c0c050c5 11189 bp->flags |= BNXT_FLAG_MSIX_CAP;
c0c050c5
MC		 11190
11191 rc = bnxt_init_board(pdev, dev);
11192 if (rc < 0)
11193 goto init_err_free;
11194
11195 dev->netdev_ops = &bnxt_netdev_ops;
11196 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
11197 dev->ethtool_ops = &bnxt_ethtool_ops;
c0c050c5
MC		 11198 pci_set_drvdata(pdev, dev);
11199
3e8060fa
PS		 11200 rc = bnxt_alloc_hwrm_resources(bp);
11201 if (rc)
17086399 11202 goto init_err_pci_clean;
3e8060fa
PS		 11203
11204 mutex_init(&bp->hwrm_cmd_lock);
ba642ab7 11205 mutex_init(&bp->link_lock);
7c380918
MC		 11206
11207 rc = bnxt_fw_init_one_p1(bp);
3e8060fa 11208 if (rc)
17086399 11209 goto init_err_pci_clean;
3e8060fa 11210
e38287b7
MC		 11211 if (BNXT_CHIP_P5(bp))
11212 bp->flags |= BNXT_FLAG_CHIP_P5;
11213
7c380918 11214 rc = bnxt_fw_init_one_p2(bp);
3c2217a6
MC		 11215 if (rc)
11216 goto init_err_pci_clean;
11217
c0c050c5
MC		 11218 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11219 NETIF_F_TSO | NETIF_F_TSO6 |
11220 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
7e13318d 11221 NETIF_F_GSO_IPXIP4 |
152971ee
AD		 11222 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
11223 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
3e8060fa
PS		 11224 NETIF_F_RXCSUM | NETIF_F_GRO;
11225
e38287b7 11226 if (BNXT_SUPPORTS_TPA(bp))
3e8060fa 11227 dev->hw_features |= NETIF_F_LRO;
c0c050c5 11228
c0c050c5
MC		 11229 dev->hw_enc_features =
11230 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11231 NETIF_F_TSO | NETIF_F_TSO6 |
11232 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
152971ee 11233 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
7e13318d 11234 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
152971ee
AD		 11235 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
11236 NETIF_F_GSO_GRE_CSUM;
c0c050c5
MC		 11237 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
11238 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
11239 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
e38287b7 11240 if (BNXT_SUPPORTS_TPA(bp))
1054aee8 11241 dev->hw_features |= NETIF_F_GRO_HW;
c0c050c5 11242 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
1054aee8
MC		 11243 if (dev->features & NETIF_F_GRO_HW)
11244 dev->features &= ~NETIF_F_LRO;
c0c050c5
MC		 11245 dev->priv_flags |= IFF_UNICAST_FLT;
11246
11247#ifdef CONFIG_BNXT_SRIOV
11248 init_waitqueue_head(&bp->sriov_cfg_wait);
4ab0c6a8 11249 mutex_init(&bp->sriov_lock);
c0c050c5 11250#endif
e38287b7
MC		 11251 if (BNXT_SUPPORTS_TPA(bp)) {
11252 bp->gro_func = bnxt_gro_func_5730x;
67912c36 11253 if (BNXT_CHIP_P4(bp))
e38287b7 11254 bp->gro_func = bnxt_gro_func_5731x;
67912c36
MC		 11255 else if (BNXT_CHIP_P5(bp))
11256 bp->gro_func = bnxt_gro_func_5750x;
e38287b7
MC		 11257 }
11258 if (!BNXT_CHIP_P4_PLUS(bp))
434c975a 11259 bp->flags |= BNXT_FLAG_DOUBLE_DB;
309369c9 11260
a588e458
MC		 11261 bp->ulp_probe = bnxt_ulp_probe;
11262
a22a6ac2
MC		 11263 rc = bnxt_init_mac_addr(bp);
11264 if (rc) {
11265 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
11266 rc = -EADDRNOTAVAIL;
11267 goto init_err_pci_clean;
11268 }
c0c050c5 11269
2e9217d1
VV		 11270 if (BNXT_PF(bp)) {
11271 /* Read the adapter's DSN to use as the eswitch switch_id */
11272 rc = bnxt_pcie_dsn_get(bp, bp->switch_id);
11273 if (rc)
11274 goto init_err_pci_clean;
11275 }
567b2abe 11276
7eb9bb3a
MC		 11277 /* MTU range: 60 - FW defined max */
11278 dev->min_mtu = ETH_ZLEN;
11279 dev->max_mtu = bp->max_mtu;
11280
ba642ab7 11281 rc = bnxt_probe_phy(bp, true);
d5430d31
MC		 11282 if (rc)
11283 goto init_err_pci_clean;
11284
c61fb99c 11285 bnxt_set_rx_skb_mode(bp, false);
c0c050c5
MC		 11286 bnxt_set_tpa_flags(bp);
11287 bnxt_set_ring_params(bp);
702c221c 11288 rc = bnxt_set_dflt_rings(bp, true);
bdbd1eb5
MC		 11289 if (rc) {
11290 netdev_err(bp->dev, "Not enough rings available.\n");
11291 rc = -ENOMEM;
17086399 11292 goto init_err_pci_clean;
bdbd1eb5 11293 }
c0c050c5 11294
ba642ab7 11295 bnxt_fw_init_one_p3(bp);
2bcfa6f6 11296
c0c050c5
MC		 11297 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
11298 bp->flags |= BNXT_FLAG_STRIP_VLAN;
11299
7809592d 11300 rc = bnxt_init_int_mode(bp);
c0c050c5 11301 if (rc)
17086399 11302 goto init_err_pci_clean;
c0c050c5 11303
832aed16
MC		 11304 /* No TC has been set yet and rings may have been trimmed due to
11305 * limited MSIX, so we re-initialize the TX rings per TC.
11306 */
11307 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11308
c213eae8
MC		 11309 if (BNXT_PF(bp)) {
11310 if (!bnxt_pf_wq) {
11311 bnxt_pf_wq =
11312 create_singlethread_workqueue("bnxt_pf_wq");
11313 if (!bnxt_pf_wq) {
11314 dev_err(&pdev->dev, "Unable to create workqueue.\n");
11315 goto init_err_pci_clean;
11316 }
11317 }
2ae7408f 11318 bnxt_init_tc(bp);
c213eae8 11319 }
2ae7408f 11320
7809592d
MC		 11321 rc = register_netdev(dev);
11322 if (rc)
2ae7408f 11323 goto init_err_cleanup_tc;
7809592d 11324
4ab0c6a8
SP		 11325 if (BNXT_PF(bp))
11326 bnxt_dl_register(bp);
11327
c0c050c5
MC		 11328 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
11329 board_info[ent->driver_data].name,
11330 (long)pci_resource_start(pdev, 0), dev->dev_addr);
af125b75 11331 pcie_print_link_status(pdev);
90c4f788 11332
c0c050c5
MC		 11333 return 0;
11334
2ae7408f
SP		 11335init_err_cleanup_tc:
11336 bnxt_shutdown_tc(bp);
7809592d
MC		 11337 bnxt_clear_int_mode(bp);
11338
17086399 11339init_err_pci_clean:
f9099d61 11340 bnxt_free_hwrm_short_cmd_req(bp);
a2bf74f4 11341 bnxt_free_hwrm_resources(bp);
98f04cf0
MC		 11342 bnxt_free_ctx_mem(bp);
11343 kfree(bp->ctx);
11344 bp->ctx = NULL;
07f83d72
MC		 11345 kfree(bp->fw_health);
11346 bp->fw_health = NULL;
17086399 11347 bnxt_cleanup_pci(bp);
c0c050c5
MC		 11348
11349init_err_free:
11350 free_netdev(dev);
11351 return rc;
11352}
11353
d196ece7
MC		 11354static void bnxt_shutdown(struct pci_dev *pdev)
11355{
11356 struct net_device *dev = pci_get_drvdata(pdev);
11357 struct bnxt *bp;
11358
11359 if (!dev)
11360 return;
11361
11362 rtnl_lock();
11363 bp = netdev_priv(dev);
11364 if (!bp)
11365 goto shutdown_exit;
11366
11367 if (netif_running(dev))
11368 dev_close(dev);
11369
a7f3f939
RJ		 11370 bnxt_ulp_shutdown(bp);
11371
d196ece7
MC		 11372 if (system_state == SYSTEM_POWER_OFF) {
11373 bnxt_clear_int_mode(bp);
c20dc142 11374 pci_disable_device(pdev);
d196ece7
MC		 11375 pci_wake_from_d3(pdev, bp->wol);
11376 pci_set_power_state(pdev, PCI_D3hot);
11377 }
11378
11379shutdown_exit:
11380 rtnl_unlock();
11381}
11382
f65a2044
MC		 11383#ifdef CONFIG_PM_SLEEP
11384static int bnxt_suspend(struct device *device)
11385{
f521eaa9 11386 struct net_device *dev = dev_get_drvdata(device);
f65a2044
MC		 11387 struct bnxt *bp = netdev_priv(dev);
11388 int rc = 0;
11389
11390 rtnl_lock();
11391 if (netif_running(dev)) {
11392 netif_device_detach(dev);
11393 rc = bnxt_close(dev);
11394 }
11395 bnxt_hwrm_func_drv_unrgtr(bp);
11396 rtnl_unlock();
11397 return rc;
11398}
11399
11400static int bnxt_resume(struct device *device)
11401{
f521eaa9 11402 struct net_device *dev = dev_get_drvdata(device);
f65a2044
MC		 11403 struct bnxt *bp = netdev_priv(dev);
11404 int rc = 0;
11405
11406 rtnl_lock();
11407 if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
11408 rc = -ENODEV;
11409 goto resume_exit;
11410 }
11411 rc = bnxt_hwrm_func_reset(bp);
11412 if (rc) {
11413 rc = -EBUSY;
11414 goto resume_exit;
11415 }
11416 bnxt_get_wol_settings(bp);
11417 if (netif_running(dev)) {
11418 rc = bnxt_open(dev);
11419 if (!rc)
11420 netif_device_attach(dev);
11421 }
11422
11423resume_exit:
11424 rtnl_unlock();
11425 return rc;
11426}
11427
11428static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
11429#define BNXT_PM_OPS (&bnxt_pm_ops)
11430
11431#else
11432
11433#define BNXT_PM_OPS NULL
11434
11435#endif /* CONFIG_PM_SLEEP */
11436
6316ea6d
SB		 11437/**
11438 * bnxt_io_error_detected - called when PCI error is detected
11439 * @pdev: Pointer to PCI device
11440 * @state: The current pci connection state
11441 *
11442 * This function is called after a PCI bus error affecting
11443 * this device has been detected.
11444 */
11445static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
11446 pci_channel_state_t state)
11447{
11448 struct net_device *netdev = pci_get_drvdata(pdev);
a588e458 11449 struct bnxt *bp = netdev_priv(netdev);
6316ea6d
SB		 11450
11451 netdev_info(netdev, "PCI I/O error detected\n");
11452
11453 rtnl_lock();
11454 netif_device_detach(netdev);
11455
a588e458
MC		 11456 bnxt_ulp_stop(bp);
11457
6316ea6d
SB		 11458 if (state == pci_channel_io_perm_failure) {
11459 rtnl_unlock();
11460 return PCI_ERS_RESULT_DISCONNECT;
11461 }
11462
11463 if (netif_running(netdev))
11464 bnxt_close(netdev);
11465
11466 pci_disable_device(pdev);
11467 rtnl_unlock();
11468
11469 /* Request a slot slot reset. */
11470 return PCI_ERS_RESULT_NEED_RESET;
11471}
11472
11473/**
11474 * bnxt_io_slot_reset - called after the pci bus has been reset.
11475 * @pdev: Pointer to PCI device
11476 *
11477 * Restart the card from scratch, as if from a cold-boot.
11478 * At this point, the card has exprienced a hard reset,
11479 * followed by fixups by BIOS, and has its config space
11480 * set up identically to what it was at cold boot.
11481 */
11482static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
11483{
11484 struct net_device *netdev = pci_get_drvdata(pdev);
11485 struct bnxt *bp = netdev_priv(netdev);
11486 int err = 0;
11487 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
11488
11489 netdev_info(bp->dev, "PCI Slot Reset\n");
11490
11491 rtnl_lock();
11492
11493 if (pci_enable_device(pdev)) {
11494 dev_err(&pdev->dev,
11495 "Cannot re-enable PCI device after reset.\n");
11496 } else {
11497 pci_set_master(pdev);
11498
aa8ed021
MC		 11499 err = bnxt_hwrm_func_reset(bp);
11500 if (!err && netif_running(netdev))
6316ea6d
SB		 11501 err = bnxt_open(netdev);
11502
a588e458 11503 if (!err) {
6316ea6d 11504 result = PCI_ERS_RESULT_RECOVERED;
a588e458
MC		 11505 bnxt_ulp_start(bp);
11506 }
6316ea6d
SB		 11507 }
11508
11509 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
11510 dev_close(netdev);
11511
11512 rtnl_unlock();
11513
6316ea6d
SB		 11514 return PCI_ERS_RESULT_RECOVERED;
11515}
11516
11517/**
11518 * bnxt_io_resume - called when traffic can start flowing again.
11519 * @pdev: Pointer to PCI device
11520 *
11521 * This callback is called when the error recovery driver tells
11522 * us that its OK to resume normal operation.
11523 */
11524static void bnxt_io_resume(struct pci_dev *pdev)
11525{
11526 struct net_device *netdev = pci_get_drvdata(pdev);
11527
11528 rtnl_lock();
11529
11530 netif_device_attach(netdev);
11531
11532 rtnl_unlock();
11533}
11534
11535static const struct pci_error_handlers bnxt_err_handler = {
11536 .error_detected = bnxt_io_error_detected,
11537 .slot_reset = bnxt_io_slot_reset,
11538 .resume = bnxt_io_resume
11539};
11540
c0c050c5
MC		 11541static struct pci_driver bnxt_pci_driver = {
11542 .name = DRV_MODULE_NAME,
11543 .id_table = bnxt_pci_tbl,
11544 .probe = bnxt_init_one,
11545 .remove = bnxt_remove_one,
d196ece7 11546 .shutdown = bnxt_shutdown,
f65a2044 11547 .driver.pm = BNXT_PM_OPS,
6316ea6d 11548 .err_handler = &bnxt_err_handler,
c0c050c5
MC		 11549#if defined(CONFIG_BNXT_SRIOV)
11550 .sriov_configure = bnxt_sriov_configure,
11551#endif
11552};
11553
c213eae8
MC		 11554static int __init bnxt_init(void)
11555{
cabfb09d 11556 bnxt_debug_init();
c213eae8
MC		 11557 return pci_register_driver(&bnxt_pci_driver);
11558}
11559
11560static void __exit bnxt_exit(void)
11561{
11562 pci_unregister_driver(&bnxt_pci_driver);
11563 if (bnxt_pf_wq)
11564 destroy_workqueue(bnxt_pf_wq);
cabfb09d 11565 bnxt_debug_exit();
c213eae8
MC		 11566}
11567
11568module_init(bnxt_init);
11569module_exit(bnxt_exit);
Ubuntu Jammy Linux kernel mirror