]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/net/ethernet/broadcom/bnxt/bnxt.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
bnxt_en: avoid link reset if speed is not changed
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 * Copyright (c) 2016-2019 Broadcom Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/ip.h>
41 #include <net/tcp.h>
42 #include <net/udp.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <net/udp_tunnel.h>
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool.h>
58
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_ulp.h"
62 #include "bnxt_sriov.h"
63 #include "bnxt_ethtool.h"
64 #include "bnxt_dcb.h"
65 #include "bnxt_xdp.h"
66 #include "bnxt_vfr.h"
67 #include "bnxt_tc.h"
68 #include "bnxt_devlink.h"
69 #include "bnxt_debugfs.h"
70
71 #define BNXT_TX_TIMEOUT (5 * HZ)
72
73 MODULE_LICENSE("GPL");
74 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
75
76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
78 #define BNXT_RX_COPY_THRESH 256
79
80 #define BNXT_TX_PUSH_THRESH 164
81
82 enum board_idx {
83 BCM57301,
84 BCM57302,
85 BCM57304,
86 BCM57417_NPAR,
87 BCM58700,
88 BCM57311,
89 BCM57312,
90 BCM57402,
91 BCM57404,
92 BCM57406,
93 BCM57402_NPAR,
94 BCM57407,
95 BCM57412,
96 BCM57414,
97 BCM57416,
98 BCM57417,
99 BCM57412_NPAR,
100 BCM57314,
101 BCM57417_SFP,
102 BCM57416_SFP,
103 BCM57404_NPAR,
104 BCM57406_NPAR,
105 BCM57407_SFP,
106 BCM57407_NPAR,
107 BCM57414_NPAR,
108 BCM57416_NPAR,
109 BCM57452,
110 BCM57454,
111 BCM5745x_NPAR,
112 BCM57508,
113 BCM57504,
114 BCM57502,
115 BCM57508_NPAR,
116 BCM57504_NPAR,
117 BCM57502_NPAR,
118 BCM58802,
119 BCM58804,
120 BCM58808,
121 NETXTREME_E_VF,
122 NETXTREME_C_VF,
123 NETXTREME_S_VF,
124 NETXTREME_E_P5_VF,
125 };
126
127 /* indexed by enum above */
128 static const struct {
129 char *name;
130 } board_info[] = {
131 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
132 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
133 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
134 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
135 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
136 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
137 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
138 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
139 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
140 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
141 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
142 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
143 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
144 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
145 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
146 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
147 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
148 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
149 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
150 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
151 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
152 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
153 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
154 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
155 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
156 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
157 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
158 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
159 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
160 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
161 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
162 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
163 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
164 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
165 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
166 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
167 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
168 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
169 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
170 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
171 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
172 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
173 };
174
175 static const struct pci_device_id bnxt_pci_tbl[] = {
176 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
177 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
178 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
179 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
180 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
181 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
182 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
183 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
184 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
185 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
186 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
187 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
188 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
189 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
190 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
191 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
192 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
193 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
194 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
195 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
196 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
197 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
198 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
199 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
200 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
201 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
202 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
203 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
204 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
205 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
206 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
207 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
208 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
209 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
210 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
211 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
212 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
213 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
214 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
215 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
216 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
217 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
218 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
219 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
220 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
221 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
222 #ifdef CONFIG_BNXT_SRIOV
223 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
224 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
225 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
226 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
227 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
228 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
229 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
230 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
231 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
232 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
233 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
234 #endif
235 { 0 }
236 };
237
238 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
239
240 static const u16 bnxt_vf_req_snif[] = {
241 HWRM_FUNC_CFG,
242 HWRM_FUNC_VF_CFG,
243 HWRM_PORT_PHY_QCFG,
244 HWRM_CFA_L2_FILTER_ALLOC,
245 };
246
247 static const u16 bnxt_async_events_arr[] = {
248 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
249 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
250 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
251 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
252 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
253 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
254 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
255 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
256 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
257 };
258
259 static struct workqueue_struct *bnxt_pf_wq;
260
261 static bool bnxt_vf_pciid(enum board_idx idx)
262 {
263 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
264 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
265 }
266
267 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
268 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
269 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
270
271 #define BNXT_CP_DB_IRQ_DIS(db) \
272 writel(DB_CP_IRQ_DIS_FLAGS, db)
273
274 #define BNXT_DB_CQ(db, idx) \
275 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
276
277 #define BNXT_DB_NQ_P5(db, idx) \
278 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
279
280 #define BNXT_DB_CQ_ARM(db, idx) \
281 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
282
283 #define BNXT_DB_NQ_ARM_P5(db, idx) \
284 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
285
286 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
287 {
288 if (bp->flags & BNXT_FLAG_CHIP_P5)
289 BNXT_DB_NQ_P5(db, idx);
290 else
291 BNXT_DB_CQ(db, idx);
292 }
293
294 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
295 {
296 if (bp->flags & BNXT_FLAG_CHIP_P5)
297 BNXT_DB_NQ_ARM_P5(db, idx);
298 else
299 BNXT_DB_CQ_ARM(db, idx);
300 }
301
302 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
303 {
304 if (bp->flags & BNXT_FLAG_CHIP_P5)
305 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
306 db->doorbell);
307 else
308 BNXT_DB_CQ(db, idx);
309 }
310
311 const u16 bnxt_lhint_arr[] = {
312 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
313 TX_BD_FLAGS_LHINT_512_TO_1023,
314 TX_BD_FLAGS_LHINT_1024_TO_2047,
315 TX_BD_FLAGS_LHINT_1024_TO_2047,
316 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
317 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
318 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
319 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
320 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
321 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
322 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
323 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
324 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
325 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
326 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
327 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
328 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
329 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
330 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
331 };
332
333 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
334 {
335 struct metadata_dst *md_dst = skb_metadata_dst(skb);
336
337 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
338 return 0;
339
340 return md_dst->u.port_info.port_id;
341 }
342
343 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
344 {
345 struct bnxt *bp = netdev_priv(dev);
346 struct tx_bd *txbd;
347 struct tx_bd_ext *txbd1;
348 struct netdev_queue *txq;
349 int i;
350 dma_addr_t mapping;
351 unsigned int length, pad = 0;
352 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
353 u16 prod, last_frag;
354 struct pci_dev *pdev = bp->pdev;
355 struct bnxt_tx_ring_info *txr;
356 struct bnxt_sw_tx_bd *tx_buf;
357
358 i = skb_get_queue_mapping(skb);
359 if (unlikely(i >= bp->tx_nr_rings)) {
360 dev_kfree_skb_any(skb);
361 return NETDEV_TX_OK;
362 }
363
364 txq = netdev_get_tx_queue(dev, i);
365 txr = &bp->tx_ring[bp->tx_ring_map[i]];
366 prod = txr->tx_prod;
367
368 free_size = bnxt_tx_avail(bp, txr);
369 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
370 netif_tx_stop_queue(txq);
371 return NETDEV_TX_BUSY;
372 }
373
374 length = skb->len;
375 len = skb_headlen(skb);
376 last_frag = skb_shinfo(skb)->nr_frags;
377
378 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
379
380 txbd->tx_bd_opaque = prod;
381
382 tx_buf = &txr->tx_buf_ring[prod];
383 tx_buf->skb = skb;
384 tx_buf->nr_frags = last_frag;
385
386 vlan_tag_flags = 0;
387 cfa_action = bnxt_xmit_get_cfa_action(skb);
388 if (skb_vlan_tag_present(skb)) {
389 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
390 skb_vlan_tag_get(skb);
391 /* Currently supports 8021Q, 8021AD vlan offloads
392 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
393 */
394 if (skb->vlan_proto == htons(ETH_P_8021Q))
395 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
396 }
397
398 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
399 struct tx_push_buffer *tx_push_buf = txr->tx_push;
400 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
401 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
402 void __iomem *db = txr->tx_db.doorbell;
403 void *pdata = tx_push_buf->data;
404 u64 *end;
405 int j, push_len;
406
407 /* Set COAL_NOW to be ready quickly for the next push */
408 tx_push->tx_bd_len_flags_type =
409 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
410 TX_BD_TYPE_LONG_TX_BD |
411 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
412 TX_BD_FLAGS_COAL_NOW |
413 TX_BD_FLAGS_PACKET_END |
414 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
415
416 if (skb->ip_summed == CHECKSUM_PARTIAL)
417 tx_push1->tx_bd_hsize_lflags =
418 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
419 else
420 tx_push1->tx_bd_hsize_lflags = 0;
421
422 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
423 tx_push1->tx_bd_cfa_action =
424 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
425
426 end = pdata + length;
427 end = PTR_ALIGN(end, 8) - 1;
428 *end = 0;
429
430 skb_copy_from_linear_data(skb, pdata, len);
431 pdata += len;
432 for (j = 0; j < last_frag; j++) {
433 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
434 void *fptr;
435
436 fptr = skb_frag_address_safe(frag);
437 if (!fptr)
438 goto normal_tx;
439
440 memcpy(pdata, fptr, skb_frag_size(frag));
441 pdata += skb_frag_size(frag);
442 }
443
444 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
445 txbd->tx_bd_haddr = txr->data_mapping;
446 prod = NEXT_TX(prod);
447 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
448 memcpy(txbd, tx_push1, sizeof(*txbd));
449 prod = NEXT_TX(prod);
450 tx_push->doorbell =
451 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
452 txr->tx_prod = prod;
453
454 tx_buf->is_push = 1;
455 netdev_tx_sent_queue(txq, skb->len);
456 wmb(); /* Sync is_push and byte queue before pushing data */
457
458 push_len = (length + sizeof(*tx_push) + 7) / 8;
459 if (push_len > 16) {
460 __iowrite64_copy(db, tx_push_buf, 16);
461 __iowrite32_copy(db + 4, tx_push_buf + 1,
462 (push_len - 16) << 1);
463 } else {
464 __iowrite64_copy(db, tx_push_buf, push_len);
465 }
466
467 goto tx_done;
468 }
469
470 normal_tx:
471 if (length < BNXT_MIN_PKT_SIZE) {
472 pad = BNXT_MIN_PKT_SIZE - length;
473 if (skb_pad(skb, pad)) {
474 /* SKB already freed. */
475 tx_buf->skb = NULL;
476 return NETDEV_TX_OK;
477 }
478 length = BNXT_MIN_PKT_SIZE;
479 }
480
481 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
482
483 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
484 dev_kfree_skb_any(skb);
485 tx_buf->skb = NULL;
486 return NETDEV_TX_OK;
487 }
488
489 dma_unmap_addr_set(tx_buf, mapping, mapping);
490 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
491 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
492
493 txbd->tx_bd_haddr = cpu_to_le64(mapping);
494
495 prod = NEXT_TX(prod);
496 txbd1 = (struct tx_bd_ext *)
497 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
498
499 txbd1->tx_bd_hsize_lflags = 0;
500 if (skb_is_gso(skb)) {
501 u32 hdr_len;
502
503 if (skb->encapsulation)
504 hdr_len = skb_inner_network_offset(skb) +
505 skb_inner_network_header_len(skb) +
506 inner_tcp_hdrlen(skb);
507 else
508 hdr_len = skb_transport_offset(skb) +
509 tcp_hdrlen(skb);
510
511 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
512 TX_BD_FLAGS_T_IPID |
513 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
514 length = skb_shinfo(skb)->gso_size;
515 txbd1->tx_bd_mss = cpu_to_le32(length);
516 length += hdr_len;
517 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
518 txbd1->tx_bd_hsize_lflags =
519 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
520 txbd1->tx_bd_mss = 0;
521 }
522
523 length >>= 9;
524 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
525 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
526 skb->len);
527 i = 0;
528 goto tx_dma_error;
529 }
530 flags |= bnxt_lhint_arr[length];
531 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
532
533 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
534 txbd1->tx_bd_cfa_action =
535 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
536 for (i = 0; i < last_frag; i++) {
537 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
538
539 prod = NEXT_TX(prod);
540 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
541
542 len = skb_frag_size(frag);
543 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
544 DMA_TO_DEVICE);
545
546 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
547 goto tx_dma_error;
548
549 tx_buf = &txr->tx_buf_ring[prod];
550 dma_unmap_addr_set(tx_buf, mapping, mapping);
551
552 txbd->tx_bd_haddr = cpu_to_le64(mapping);
553
554 flags = len << TX_BD_LEN_SHIFT;
555 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
556 }
557
558 flags &= ~TX_BD_LEN;
559 txbd->tx_bd_len_flags_type =
560 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
561 TX_BD_FLAGS_PACKET_END);
562
563 netdev_tx_sent_queue(txq, skb->len);
564
565 /* Sync BD data before updating doorbell */
566 wmb();
567
568 prod = NEXT_TX(prod);
569 txr->tx_prod = prod;
570
571 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
572 bnxt_db_write(bp, &txr->tx_db, prod);
573
574 tx_done:
575
576 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
577 if (netdev_xmit_more() && !tx_buf->is_push)
578 bnxt_db_write(bp, &txr->tx_db, prod);
579
580 netif_tx_stop_queue(txq);
581
582 /* netif_tx_stop_queue() must be done before checking
583 * tx index in bnxt_tx_avail() below, because in
584 * bnxt_tx_int(), we update tx index before checking for
585 * netif_tx_queue_stopped().
586 */
587 smp_mb();
588 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
589 netif_tx_wake_queue(txq);
590 }
591 return NETDEV_TX_OK;
592
593 tx_dma_error:
594 last_frag = i;
595
596 /* start back at beginning and unmap skb */
597 prod = txr->tx_prod;
598 tx_buf = &txr->tx_buf_ring[prod];
599 tx_buf->skb = NULL;
600 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
601 skb_headlen(skb), PCI_DMA_TODEVICE);
602 prod = NEXT_TX(prod);
603
604 /* unmap remaining mapped pages */
605 for (i = 0; i < last_frag; i++) {
606 prod = NEXT_TX(prod);
607 tx_buf = &txr->tx_buf_ring[prod];
608 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
609 skb_frag_size(&skb_shinfo(skb)->frags[i]),
610 PCI_DMA_TODEVICE);
611 }
612
613 dev_kfree_skb_any(skb);
614 return NETDEV_TX_OK;
615 }
616
617 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
618 {
619 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
620 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
621 u16 cons = txr->tx_cons;
622 struct pci_dev *pdev = bp->pdev;
623 int i;
624 unsigned int tx_bytes = 0;
625
626 for (i = 0; i < nr_pkts; i++) {
627 struct bnxt_sw_tx_bd *tx_buf;
628 struct sk_buff *skb;
629 int j, last;
630
631 tx_buf = &txr->tx_buf_ring[cons];
632 cons = NEXT_TX(cons);
633 skb = tx_buf->skb;
634 tx_buf->skb = NULL;
635
636 if (tx_buf->is_push) {
637 tx_buf->is_push = 0;
638 goto next_tx_int;
639 }
640
641 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
642 skb_headlen(skb), PCI_DMA_TODEVICE);
643 last = tx_buf->nr_frags;
644
645 for (j = 0; j < last; j++) {
646 cons = NEXT_TX(cons);
647 tx_buf = &txr->tx_buf_ring[cons];
648 dma_unmap_page(
649 &pdev->dev,
650 dma_unmap_addr(tx_buf, mapping),
651 skb_frag_size(&skb_shinfo(skb)->frags[j]),
652 PCI_DMA_TODEVICE);
653 }
654
655 next_tx_int:
656 cons = NEXT_TX(cons);
657
658 tx_bytes += skb->len;
659 dev_kfree_skb_any(skb);
660 }
661
662 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
663 txr->tx_cons = cons;
664
665 /* Need to make the tx_cons update visible to bnxt_start_xmit()
666 * before checking for netif_tx_queue_stopped(). Without the
667 * memory barrier, there is a small possibility that bnxt_start_xmit()
668 * will miss it and cause the queue to be stopped forever.
669 */
670 smp_mb();
671
672 if (unlikely(netif_tx_queue_stopped(txq)) &&
673 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
674 __netif_tx_lock(txq, smp_processor_id());
675 if (netif_tx_queue_stopped(txq) &&
676 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
677 txr->dev_state != BNXT_DEV_STATE_CLOSING)
678 netif_tx_wake_queue(txq);
679 __netif_tx_unlock(txq);
680 }
681 }
682
683 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
684 struct bnxt_rx_ring_info *rxr,
685 gfp_t gfp)
686 {
687 struct device *dev = &bp->pdev->dev;
688 struct page *page;
689
690 page = page_pool_dev_alloc_pages(rxr->page_pool);
691 if (!page)
692 return NULL;
693
694 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
695 DMA_ATTR_WEAK_ORDERING);
696 if (dma_mapping_error(dev, *mapping)) {
697 page_pool_recycle_direct(rxr->page_pool, page);
698 return NULL;
699 }
700 *mapping += bp->rx_dma_offset;
701 return page;
702 }
703
704 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
705 gfp_t gfp)
706 {
707 u8 *data;
708 struct pci_dev *pdev = bp->pdev;
709
710 data = kmalloc(bp->rx_buf_size, gfp);
711 if (!data)
712 return NULL;
713
714 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
715 bp->rx_buf_use_size, bp->rx_dir,
716 DMA_ATTR_WEAK_ORDERING);
717
718 if (dma_mapping_error(&pdev->dev, *mapping)) {
719 kfree(data);
720 data = NULL;
721 }
722 return data;
723 }
724
725 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
726 u16 prod, gfp_t gfp)
727 {
728 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
729 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
730 dma_addr_t mapping;
731
732 if (BNXT_RX_PAGE_MODE(bp)) {
733 struct page *page =
734 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
735
736 if (!page)
737 return -ENOMEM;
738
739 rx_buf->data = page;
740 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
741 } else {
742 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
743
744 if (!data)
745 return -ENOMEM;
746
747 rx_buf->data = data;
748 rx_buf->data_ptr = data + bp->rx_offset;
749 }
750 rx_buf->mapping = mapping;
751
752 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
753 return 0;
754 }
755
756 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
757 {
758 u16 prod = rxr->rx_prod;
759 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
760 struct rx_bd *cons_bd, *prod_bd;
761
762 prod_rx_buf = &rxr->rx_buf_ring[prod];
763 cons_rx_buf = &rxr->rx_buf_ring[cons];
764
765 prod_rx_buf->data = data;
766 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
767
768 prod_rx_buf->mapping = cons_rx_buf->mapping;
769
770 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
771 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
772
773 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
774 }
775
776 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
777 {
778 u16 next, max = rxr->rx_agg_bmap_size;
779
780 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
781 if (next >= max)
782 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
783 return next;
784 }
785
786 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
787 struct bnxt_rx_ring_info *rxr,
788 u16 prod, gfp_t gfp)
789 {
790 struct rx_bd *rxbd =
791 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
792 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
793 struct pci_dev *pdev = bp->pdev;
794 struct page *page;
795 dma_addr_t mapping;
796 u16 sw_prod = rxr->rx_sw_agg_prod;
797 unsigned int offset = 0;
798
799 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
800 page = rxr->rx_page;
801 if (!page) {
802 page = alloc_page(gfp);
803 if (!page)
804 return -ENOMEM;
805 rxr->rx_page = page;
806 rxr->rx_page_offset = 0;
807 }
808 offset = rxr->rx_page_offset;
809 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
810 if (rxr->rx_page_offset == PAGE_SIZE)
811 rxr->rx_page = NULL;
812 else
813 get_page(page);
814 } else {
815 page = alloc_page(gfp);
816 if (!page)
817 return -ENOMEM;
818 }
819
820 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
821 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
822 DMA_ATTR_WEAK_ORDERING);
823 if (dma_mapping_error(&pdev->dev, mapping)) {
824 __free_page(page);
825 return -EIO;
826 }
827
828 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
829 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
830
831 __set_bit(sw_prod, rxr->rx_agg_bmap);
832 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
833 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
834
835 rx_agg_buf->page = page;
836 rx_agg_buf->offset = offset;
837 rx_agg_buf->mapping = mapping;
838 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
839 rxbd->rx_bd_opaque = sw_prod;
840 return 0;
841 }
842
843 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
844 struct bnxt_cp_ring_info *cpr,
845 u16 cp_cons, u16 curr)
846 {
847 struct rx_agg_cmp *agg;
848
849 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
850 agg = (struct rx_agg_cmp *)
851 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
852 return agg;
853 }
854
855 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
856 struct bnxt_rx_ring_info *rxr,
857 u16 agg_id, u16 curr)
858 {
859 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
860
861 return &tpa_info->agg_arr[curr];
862 }
863
864 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
865 u16 start, u32 agg_bufs, bool tpa)
866 {
867 struct bnxt_napi *bnapi = cpr->bnapi;
868 struct bnxt *bp = bnapi->bp;
869 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
870 u16 prod = rxr->rx_agg_prod;
871 u16 sw_prod = rxr->rx_sw_agg_prod;
872 bool p5_tpa = false;
873 u32 i;
874
875 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
876 p5_tpa = true;
877
878 for (i = 0; i < agg_bufs; i++) {
879 u16 cons;
880 struct rx_agg_cmp *agg;
881 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
882 struct rx_bd *prod_bd;
883 struct page *page;
884
885 if (p5_tpa)
886 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
887 else
888 agg = bnxt_get_agg(bp, cpr, idx, start + i);
889 cons = agg->rx_agg_cmp_opaque;
890 __clear_bit(cons, rxr->rx_agg_bmap);
891
892 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
893 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
894
895 __set_bit(sw_prod, rxr->rx_agg_bmap);
896 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
897 cons_rx_buf = &rxr->rx_agg_ring[cons];
898
899 /* It is possible for sw_prod to be equal to cons, so
900 * set cons_rx_buf->page to NULL first.
901 */
902 page = cons_rx_buf->page;
903 cons_rx_buf->page = NULL;
904 prod_rx_buf->page = page;
905 prod_rx_buf->offset = cons_rx_buf->offset;
906
907 prod_rx_buf->mapping = cons_rx_buf->mapping;
908
909 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
910
911 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
912 prod_bd->rx_bd_opaque = sw_prod;
913
914 prod = NEXT_RX_AGG(prod);
915 sw_prod = NEXT_RX_AGG(sw_prod);
916 }
917 rxr->rx_agg_prod = prod;
918 rxr->rx_sw_agg_prod = sw_prod;
919 }
920
921 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
922 struct bnxt_rx_ring_info *rxr,
923 u16 cons, void *data, u8 *data_ptr,
924 dma_addr_t dma_addr,
925 unsigned int offset_and_len)
926 {
927 unsigned int payload = offset_and_len >> 16;
928 unsigned int len = offset_and_len & 0xffff;
929 skb_frag_t *frag;
930 struct page *page = data;
931 u16 prod = rxr->rx_prod;
932 struct sk_buff *skb;
933 int off, err;
934
935 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
936 if (unlikely(err)) {
937 bnxt_reuse_rx_data(rxr, cons, data);
938 return NULL;
939 }
940 dma_addr -= bp->rx_dma_offset;
941 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
942 DMA_ATTR_WEAK_ORDERING);
943 page_pool_release_page(rxr->page_pool, page);
944
945 if (unlikely(!payload))
946 payload = eth_get_headlen(bp->dev, data_ptr, len);
947
948 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
949 if (!skb) {
950 __free_page(page);
951 return NULL;
952 }
953
954 off = (void *)data_ptr - page_address(page);
955 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
956 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
957 payload + NET_IP_ALIGN);
958
959 frag = &skb_shinfo(skb)->frags[0];
960 skb_frag_size_sub(frag, payload);
961 skb_frag_off_add(frag, payload);
962 skb->data_len -= payload;
963 skb->tail += payload;
964
965 return skb;
966 }
967
968 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
969 struct bnxt_rx_ring_info *rxr, u16 cons,
970 void *data, u8 *data_ptr,
971 dma_addr_t dma_addr,
972 unsigned int offset_and_len)
973 {
974 u16 prod = rxr->rx_prod;
975 struct sk_buff *skb;
976 int err;
977
978 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
979 if (unlikely(err)) {
980 bnxt_reuse_rx_data(rxr, cons, data);
981 return NULL;
982 }
983
984 skb = build_skb(data, 0);
985 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
986 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
987 if (!skb) {
988 kfree(data);
989 return NULL;
990 }
991
992 skb_reserve(skb, bp->rx_offset);
993 skb_put(skb, offset_and_len & 0xffff);
994 return skb;
995 }
996
997 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
998 struct bnxt_cp_ring_info *cpr,
999 struct sk_buff *skb, u16 idx,
1000 u32 agg_bufs, bool tpa)
1001 {
1002 struct bnxt_napi *bnapi = cpr->bnapi;
1003 struct pci_dev *pdev = bp->pdev;
1004 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1005 u16 prod = rxr->rx_agg_prod;
1006 bool p5_tpa = false;
1007 u32 i;
1008
1009 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1010 p5_tpa = true;
1011
1012 for (i = 0; i < agg_bufs; i++) {
1013 u16 cons, frag_len;
1014 struct rx_agg_cmp *agg;
1015 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1016 struct page *page;
1017 dma_addr_t mapping;
1018
1019 if (p5_tpa)
1020 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1021 else
1022 agg = bnxt_get_agg(bp, cpr, idx, i);
1023 cons = agg->rx_agg_cmp_opaque;
1024 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1025 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1026
1027 cons_rx_buf = &rxr->rx_agg_ring[cons];
1028 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1029 cons_rx_buf->offset, frag_len);
1030 __clear_bit(cons, rxr->rx_agg_bmap);
1031
1032 /* It is possible for bnxt_alloc_rx_page() to allocate
1033 * a sw_prod index that equals the cons index, so we
1034 * need to clear the cons entry now.
1035 */
1036 mapping = cons_rx_buf->mapping;
1037 page = cons_rx_buf->page;
1038 cons_rx_buf->page = NULL;
1039
1040 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1041 struct skb_shared_info *shinfo;
1042 unsigned int nr_frags;
1043
1044 shinfo = skb_shinfo(skb);
1045 nr_frags = --shinfo->nr_frags;
1046 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1047
1048 dev_kfree_skb(skb);
1049
1050 cons_rx_buf->page = page;
1051
1052 /* Update prod since possibly some pages have been
1053 * allocated already.
1054 */
1055 rxr->rx_agg_prod = prod;
1056 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1057 return NULL;
1058 }
1059
1060 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1061 PCI_DMA_FROMDEVICE,
1062 DMA_ATTR_WEAK_ORDERING);
1063
1064 skb->data_len += frag_len;
1065 skb->len += frag_len;
1066 skb->truesize += PAGE_SIZE;
1067
1068 prod = NEXT_RX_AGG(prod);
1069 }
1070 rxr->rx_agg_prod = prod;
1071 return skb;
1072 }
1073
1074 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1075 u8 agg_bufs, u32 *raw_cons)
1076 {
1077 u16 last;
1078 struct rx_agg_cmp *agg;
1079
1080 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1081 last = RING_CMP(*raw_cons);
1082 agg = (struct rx_agg_cmp *)
1083 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1084 return RX_AGG_CMP_VALID(agg, *raw_cons);
1085 }
1086
1087 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1088 unsigned int len,
1089 dma_addr_t mapping)
1090 {
1091 struct bnxt *bp = bnapi->bp;
1092 struct pci_dev *pdev = bp->pdev;
1093 struct sk_buff *skb;
1094
1095 skb = napi_alloc_skb(&bnapi->napi, len);
1096 if (!skb)
1097 return NULL;
1098
1099 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1100 bp->rx_dir);
1101
1102 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1103 len + NET_IP_ALIGN);
1104
1105 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1106 bp->rx_dir);
1107
1108 skb_put(skb, len);
1109 return skb;
1110 }
1111
1112 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1113 u32 *raw_cons, void *cmp)
1114 {
1115 struct rx_cmp *rxcmp = cmp;
1116 u32 tmp_raw_cons = *raw_cons;
1117 u8 cmp_type, agg_bufs = 0;
1118
1119 cmp_type = RX_CMP_TYPE(rxcmp);
1120
1121 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1122 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1123 RX_CMP_AGG_BUFS) >>
1124 RX_CMP_AGG_BUFS_SHIFT;
1125 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1126 struct rx_tpa_end_cmp *tpa_end = cmp;
1127
1128 if (bp->flags & BNXT_FLAG_CHIP_P5)
1129 return 0;
1130
1131 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1132 }
1133
1134 if (agg_bufs) {
1135 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1136 return -EBUSY;
1137 }
1138 *raw_cons = tmp_raw_cons;
1139 return 0;
1140 }
1141
1142 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1143 {
1144 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1145 return;
1146
1147 if (BNXT_PF(bp))
1148 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1149 else
1150 schedule_delayed_work(&bp->fw_reset_task, delay);
1151 }
1152
1153 static void bnxt_queue_sp_work(struct bnxt *bp)
1154 {
1155 if (BNXT_PF(bp))
1156 queue_work(bnxt_pf_wq, &bp->sp_task);
1157 else
1158 schedule_work(&bp->sp_task);
1159 }
1160
1161 static void bnxt_cancel_sp_work(struct bnxt *bp)
1162 {
1163 if (BNXT_PF(bp)) {
1164 flush_workqueue(bnxt_pf_wq);
1165 } else {
1166 cancel_work_sync(&bp->sp_task);
1167 cancel_delayed_work_sync(&bp->fw_reset_task);
1168 }
1169 }
1170
1171 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1172 {
1173 if (!rxr->bnapi->in_reset) {
1174 rxr->bnapi->in_reset = true;
1175 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1176 bnxt_queue_sp_work(bp);
1177 }
1178 rxr->rx_next_cons = 0xffff;
1179 }
1180
1181 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1182 {
1183 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1184 u16 idx = agg_id & MAX_TPA_P5_MASK;
1185
1186 if (test_bit(idx, map->agg_idx_bmap))
1187 idx = find_first_zero_bit(map->agg_idx_bmap,
1188 BNXT_AGG_IDX_BMAP_SIZE);
1189 __set_bit(idx, map->agg_idx_bmap);
1190 map->agg_id_tbl[agg_id] = idx;
1191 return idx;
1192 }
1193
1194 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1195 {
1196 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1197
1198 __clear_bit(idx, map->agg_idx_bmap);
1199 }
1200
1201 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1202 {
1203 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1204
1205 return map->agg_id_tbl[agg_id];
1206 }
1207
1208 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1209 struct rx_tpa_start_cmp *tpa_start,
1210 struct rx_tpa_start_cmp_ext *tpa_start1)
1211 {
1212 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1213 struct bnxt_tpa_info *tpa_info;
1214 u16 cons, prod, agg_id;
1215 struct rx_bd *prod_bd;
1216 dma_addr_t mapping;
1217
1218 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1219 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1220 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1221 } else {
1222 agg_id = TPA_START_AGG_ID(tpa_start);
1223 }
1224 cons = tpa_start->rx_tpa_start_cmp_opaque;
1225 prod = rxr->rx_prod;
1226 cons_rx_buf = &rxr->rx_buf_ring[cons];
1227 prod_rx_buf = &rxr->rx_buf_ring[prod];
1228 tpa_info = &rxr->rx_tpa[agg_id];
1229
1230 if (unlikely(cons != rxr->rx_next_cons ||
1231 TPA_START_ERROR(tpa_start))) {
1232 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1233 cons, rxr->rx_next_cons,
1234 TPA_START_ERROR_CODE(tpa_start1));
1235 bnxt_sched_reset(bp, rxr);
1236 return;
1237 }
1238 /* Store cfa_code in tpa_info to use in tpa_end
1239 * completion processing.
1240 */
1241 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1242 prod_rx_buf->data = tpa_info->data;
1243 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1244
1245 mapping = tpa_info->mapping;
1246 prod_rx_buf->mapping = mapping;
1247
1248 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1249
1250 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1251
1252 tpa_info->data = cons_rx_buf->data;
1253 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1254 cons_rx_buf->data = NULL;
1255 tpa_info->mapping = cons_rx_buf->mapping;
1256
1257 tpa_info->len =
1258 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1259 RX_TPA_START_CMP_LEN_SHIFT;
1260 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1261 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1262
1263 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1264 tpa_info->gso_type = SKB_GSO_TCPV4;
1265 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1266 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1267 tpa_info->gso_type = SKB_GSO_TCPV6;
1268 tpa_info->rss_hash =
1269 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1270 } else {
1271 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1272 tpa_info->gso_type = 0;
1273 if (netif_msg_rx_err(bp))
1274 netdev_warn(bp->dev, "TPA packet without valid hash\n");
1275 }
1276 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1277 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1278 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1279 tpa_info->agg_count = 0;
1280
1281 rxr->rx_prod = NEXT_RX(prod);
1282 cons = NEXT_RX(cons);
1283 rxr->rx_next_cons = NEXT_RX(cons);
1284 cons_rx_buf = &rxr->rx_buf_ring[cons];
1285
1286 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1287 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1288 cons_rx_buf->data = NULL;
1289 }
1290
1291 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1292 {
1293 if (agg_bufs)
1294 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1295 }
1296
1297 #ifdef CONFIG_INET
1298 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1299 {
1300 struct udphdr *uh = NULL;
1301
1302 if (ip_proto == htons(ETH_P_IP)) {
1303 struct iphdr *iph = (struct iphdr *)skb->data;
1304
1305 if (iph->protocol == IPPROTO_UDP)
1306 uh = (struct udphdr *)(iph + 1);
1307 } else {
1308 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1309
1310 if (iph->nexthdr == IPPROTO_UDP)
1311 uh = (struct udphdr *)(iph + 1);
1312 }
1313 if (uh) {
1314 if (uh->check)
1315 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1316 else
1317 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1318 }
1319 }
1320 #endif
1321
1322 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1323 int payload_off, int tcp_ts,
1324 struct sk_buff *skb)
1325 {
1326 #ifdef CONFIG_INET
1327 struct tcphdr *th;
1328 int len, nw_off;
1329 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1330 u32 hdr_info = tpa_info->hdr_info;
1331 bool loopback = false;
1332
1333 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1334 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1335 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1336
1337 /* If the packet is an internal loopback packet, the offsets will
1338 * have an extra 4 bytes.
1339 */
1340 if (inner_mac_off == 4) {
1341 loopback = true;
1342 } else if (inner_mac_off > 4) {
1343 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1344 ETH_HLEN - 2));
1345
1346 /* We only support inner iPv4/ipv6. If we don't see the
1347 * correct protocol ID, it must be a loopback packet where
1348 * the offsets are off by 4.
1349 */
1350 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1351 loopback = true;
1352 }
1353 if (loopback) {
1354 /* internal loopback packet, subtract all offsets by 4 */
1355 inner_ip_off -= 4;
1356 inner_mac_off -= 4;
1357 outer_ip_off -= 4;
1358 }
1359
1360 nw_off = inner_ip_off - ETH_HLEN;
1361 skb_set_network_header(skb, nw_off);
1362 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1363 struct ipv6hdr *iph = ipv6_hdr(skb);
1364
1365 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1366 len = skb->len - skb_transport_offset(skb);
1367 th = tcp_hdr(skb);
1368 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1369 } else {
1370 struct iphdr *iph = ip_hdr(skb);
1371
1372 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1373 len = skb->len - skb_transport_offset(skb);
1374 th = tcp_hdr(skb);
1375 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1376 }
1377
1378 if (inner_mac_off) { /* tunnel */
1379 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1380 ETH_HLEN - 2));
1381
1382 bnxt_gro_tunnel(skb, proto);
1383 }
1384 #endif
1385 return skb;
1386 }
1387
1388 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1389 int payload_off, int tcp_ts,
1390 struct sk_buff *skb)
1391 {
1392 #ifdef CONFIG_INET
1393 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1394 u32 hdr_info = tpa_info->hdr_info;
1395 int iphdr_len, nw_off;
1396
1397 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1398 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1399 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1400
1401 nw_off = inner_ip_off - ETH_HLEN;
1402 skb_set_network_header(skb, nw_off);
1403 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1404 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1405 skb_set_transport_header(skb, nw_off + iphdr_len);
1406
1407 if (inner_mac_off) { /* tunnel */
1408 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1409 ETH_HLEN - 2));
1410
1411 bnxt_gro_tunnel(skb, proto);
1412 }
1413 #endif
1414 return skb;
1415 }
1416
1417 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1418 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1419
1420 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1421 int payload_off, int tcp_ts,
1422 struct sk_buff *skb)
1423 {
1424 #ifdef CONFIG_INET
1425 struct tcphdr *th;
1426 int len, nw_off, tcp_opt_len = 0;
1427
1428 if (tcp_ts)
1429 tcp_opt_len = 12;
1430
1431 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1432 struct iphdr *iph;
1433
1434 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1435 ETH_HLEN;
1436 skb_set_network_header(skb, nw_off);
1437 iph = ip_hdr(skb);
1438 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1439 len = skb->len - skb_transport_offset(skb);
1440 th = tcp_hdr(skb);
1441 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1442 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1443 struct ipv6hdr *iph;
1444
1445 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1446 ETH_HLEN;
1447 skb_set_network_header(skb, nw_off);
1448 iph = ipv6_hdr(skb);
1449 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1450 len = skb->len - skb_transport_offset(skb);
1451 th = tcp_hdr(skb);
1452 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1453 } else {
1454 dev_kfree_skb_any(skb);
1455 return NULL;
1456 }
1457
1458 if (nw_off) /* tunnel */
1459 bnxt_gro_tunnel(skb, skb->protocol);
1460 #endif
1461 return skb;
1462 }
1463
1464 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1465 struct bnxt_tpa_info *tpa_info,
1466 struct rx_tpa_end_cmp *tpa_end,
1467 struct rx_tpa_end_cmp_ext *tpa_end1,
1468 struct sk_buff *skb)
1469 {
1470 #ifdef CONFIG_INET
1471 int payload_off;
1472 u16 segs;
1473
1474 segs = TPA_END_TPA_SEGS(tpa_end);
1475 if (segs == 1)
1476 return skb;
1477
1478 NAPI_GRO_CB(skb)->count = segs;
1479 skb_shinfo(skb)->gso_size =
1480 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1481 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1482 if (bp->flags & BNXT_FLAG_CHIP_P5)
1483 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1484 else
1485 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1486 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1487 if (likely(skb))
1488 tcp_gro_complete(skb);
1489 #endif
1490 return skb;
1491 }
1492
1493 /* Given the cfa_code of a received packet determine which
1494 * netdev (vf-rep or PF) the packet is destined to.
1495 */
1496 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1497 {
1498 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1499
1500 /* if vf-rep dev is NULL, the must belongs to the PF */
1501 return dev ? dev : bp->dev;
1502 }
1503
1504 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1505 struct bnxt_cp_ring_info *cpr,
1506 u32 *raw_cons,
1507 struct rx_tpa_end_cmp *tpa_end,
1508 struct rx_tpa_end_cmp_ext *tpa_end1,
1509 u8 *event)
1510 {
1511 struct bnxt_napi *bnapi = cpr->bnapi;
1512 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1513 u8 *data_ptr, agg_bufs;
1514 unsigned int len;
1515 struct bnxt_tpa_info *tpa_info;
1516 dma_addr_t mapping;
1517 struct sk_buff *skb;
1518 u16 idx = 0, agg_id;
1519 void *data;
1520 bool gro;
1521
1522 if (unlikely(bnapi->in_reset)) {
1523 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1524
1525 if (rc < 0)
1526 return ERR_PTR(-EBUSY);
1527 return NULL;
1528 }
1529
1530 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1531 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1532 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1533 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1534 tpa_info = &rxr->rx_tpa[agg_id];
1535 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1536 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1537 agg_bufs, tpa_info->agg_count);
1538 agg_bufs = tpa_info->agg_count;
1539 }
1540 tpa_info->agg_count = 0;
1541 *event |= BNXT_AGG_EVENT;
1542 bnxt_free_agg_idx(rxr, agg_id);
1543 idx = agg_id;
1544 gro = !!(bp->flags & BNXT_FLAG_GRO);
1545 } else {
1546 agg_id = TPA_END_AGG_ID(tpa_end);
1547 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1548 tpa_info = &rxr->rx_tpa[agg_id];
1549 idx = RING_CMP(*raw_cons);
1550 if (agg_bufs) {
1551 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1552 return ERR_PTR(-EBUSY);
1553
1554 *event |= BNXT_AGG_EVENT;
1555 idx = NEXT_CMP(idx);
1556 }
1557 gro = !!TPA_END_GRO(tpa_end);
1558 }
1559 data = tpa_info->data;
1560 data_ptr = tpa_info->data_ptr;
1561 prefetch(data_ptr);
1562 len = tpa_info->len;
1563 mapping = tpa_info->mapping;
1564
1565 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1566 bnxt_abort_tpa(cpr, idx, agg_bufs);
1567 if (agg_bufs > MAX_SKB_FRAGS)
1568 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1569 agg_bufs, (int)MAX_SKB_FRAGS);
1570 return NULL;
1571 }
1572
1573 if (len <= bp->rx_copy_thresh) {
1574 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1575 if (!skb) {
1576 bnxt_abort_tpa(cpr, idx, agg_bufs);
1577 return NULL;
1578 }
1579 } else {
1580 u8 *new_data;
1581 dma_addr_t new_mapping;
1582
1583 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1584 if (!new_data) {
1585 bnxt_abort_tpa(cpr, idx, agg_bufs);
1586 return NULL;
1587 }
1588
1589 tpa_info->data = new_data;
1590 tpa_info->data_ptr = new_data + bp->rx_offset;
1591 tpa_info->mapping = new_mapping;
1592
1593 skb = build_skb(data, 0);
1594 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1595 bp->rx_buf_use_size, bp->rx_dir,
1596 DMA_ATTR_WEAK_ORDERING);
1597
1598 if (!skb) {
1599 kfree(data);
1600 bnxt_abort_tpa(cpr, idx, agg_bufs);
1601 return NULL;
1602 }
1603 skb_reserve(skb, bp->rx_offset);
1604 skb_put(skb, len);
1605 }
1606
1607 if (agg_bufs) {
1608 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1609 if (!skb) {
1610 /* Page reuse already handled by bnxt_rx_pages(). */
1611 return NULL;
1612 }
1613 }
1614
1615 skb->protocol =
1616 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1617
1618 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1619 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1620
1621 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1622 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1623 u16 vlan_proto = tpa_info->metadata >>
1624 RX_CMP_FLAGS2_METADATA_TPID_SFT;
1625 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1626
1627 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1628 }
1629
1630 skb_checksum_none_assert(skb);
1631 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1632 skb->ip_summed = CHECKSUM_UNNECESSARY;
1633 skb->csum_level =
1634 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1635 }
1636
1637 if (gro)
1638 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1639
1640 return skb;
1641 }
1642
1643 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1644 struct rx_agg_cmp *rx_agg)
1645 {
1646 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1647 struct bnxt_tpa_info *tpa_info;
1648
1649 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1650 tpa_info = &rxr->rx_tpa[agg_id];
1651 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1652 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1653 }
1654
1655 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1656 struct sk_buff *skb)
1657 {
1658 if (skb->dev != bp->dev) {
1659 /* this packet belongs to a vf-rep */
1660 bnxt_vf_rep_rx(bp, skb);
1661 return;
1662 }
1663 skb_record_rx_queue(skb, bnapi->index);
1664 napi_gro_receive(&bnapi->napi, skb);
1665 }
1666
1667 /* returns the following:
1668 * 1 - 1 packet successfully received
1669 * 0 - successful TPA_START, packet not completed yet
1670 * -EBUSY - completion ring does not have all the agg buffers yet
1671 * -ENOMEM - packet aborted due to out of memory
1672 * -EIO - packet aborted due to hw error indicated in BD
1673 */
1674 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1675 u32 *raw_cons, u8 *event)
1676 {
1677 struct bnxt_napi *bnapi = cpr->bnapi;
1678 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1679 struct net_device *dev = bp->dev;
1680 struct rx_cmp *rxcmp;
1681 struct rx_cmp_ext *rxcmp1;
1682 u32 tmp_raw_cons = *raw_cons;
1683 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1684 struct bnxt_sw_rx_bd *rx_buf;
1685 unsigned int len;
1686 u8 *data_ptr, agg_bufs, cmp_type;
1687 dma_addr_t dma_addr;
1688 struct sk_buff *skb;
1689 void *data;
1690 int rc = 0;
1691 u32 misc;
1692
1693 rxcmp = (struct rx_cmp *)
1694 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1695
1696 cmp_type = RX_CMP_TYPE(rxcmp);
1697
1698 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1699 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1700 goto next_rx_no_prod_no_len;
1701 }
1702
1703 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1704 cp_cons = RING_CMP(tmp_raw_cons);
1705 rxcmp1 = (struct rx_cmp_ext *)
1706 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1707
1708 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1709 return -EBUSY;
1710
1711 prod = rxr->rx_prod;
1712
1713 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1714 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1715 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1716
1717 *event |= BNXT_RX_EVENT;
1718 goto next_rx_no_prod_no_len;
1719
1720 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1721 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1722 (struct rx_tpa_end_cmp *)rxcmp,
1723 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1724
1725 if (IS_ERR(skb))
1726 return -EBUSY;
1727
1728 rc = -ENOMEM;
1729 if (likely(skb)) {
1730 bnxt_deliver_skb(bp, bnapi, skb);
1731 rc = 1;
1732 }
1733 *event |= BNXT_RX_EVENT;
1734 goto next_rx_no_prod_no_len;
1735 }
1736
1737 cons = rxcmp->rx_cmp_opaque;
1738 if (unlikely(cons != rxr->rx_next_cons)) {
1739 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1740
1741 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1742 cons, rxr->rx_next_cons);
1743 bnxt_sched_reset(bp, rxr);
1744 return rc1;
1745 }
1746 rx_buf = &rxr->rx_buf_ring[cons];
1747 data = rx_buf->data;
1748 data_ptr = rx_buf->data_ptr;
1749 prefetch(data_ptr);
1750
1751 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1752 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1753
1754 if (agg_bufs) {
1755 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1756 return -EBUSY;
1757
1758 cp_cons = NEXT_CMP(cp_cons);
1759 *event |= BNXT_AGG_EVENT;
1760 }
1761 *event |= BNXT_RX_EVENT;
1762
1763 rx_buf->data = NULL;
1764 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1765 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1766
1767 bnxt_reuse_rx_data(rxr, cons, data);
1768 if (agg_bufs)
1769 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1770 false);
1771
1772 rc = -EIO;
1773 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1774 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1775 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
1776 netdev_warn(bp->dev, "RX buffer error %x\n",
1777 rx_err);
1778 bnxt_sched_reset(bp, rxr);
1779 }
1780 }
1781 goto next_rx_no_len;
1782 }
1783
1784 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1785 dma_addr = rx_buf->mapping;
1786
1787 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1788 rc = 1;
1789 goto next_rx;
1790 }
1791
1792 if (len <= bp->rx_copy_thresh) {
1793 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1794 bnxt_reuse_rx_data(rxr, cons, data);
1795 if (!skb) {
1796 if (agg_bufs)
1797 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1798 agg_bufs, false);
1799 rc = -ENOMEM;
1800 goto next_rx;
1801 }
1802 } else {
1803 u32 payload;
1804
1805 if (rx_buf->data_ptr == data_ptr)
1806 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1807 else
1808 payload = 0;
1809 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1810 payload | len);
1811 if (!skb) {
1812 rc = -ENOMEM;
1813 goto next_rx;
1814 }
1815 }
1816
1817 if (agg_bufs) {
1818 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1819 if (!skb) {
1820 rc = -ENOMEM;
1821 goto next_rx;
1822 }
1823 }
1824
1825 if (RX_CMP_HASH_VALID(rxcmp)) {
1826 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1827 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1828
1829 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1830 if (hash_type != 1 && hash_type != 3)
1831 type = PKT_HASH_TYPE_L3;
1832 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1833 }
1834
1835 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1836 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1837
1838 if ((rxcmp1->rx_cmp_flags2 &
1839 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1840 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1841 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1842 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1843 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1844
1845 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1846 }
1847
1848 skb_checksum_none_assert(skb);
1849 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1850 if (dev->features & NETIF_F_RXCSUM) {
1851 skb->ip_summed = CHECKSUM_UNNECESSARY;
1852 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1853 }
1854 } else {
1855 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1856 if (dev->features & NETIF_F_RXCSUM)
1857 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1858 }
1859 }
1860
1861 bnxt_deliver_skb(bp, bnapi, skb);
1862 rc = 1;
1863
1864 next_rx:
1865 cpr->rx_packets += 1;
1866 cpr->rx_bytes += len;
1867
1868 next_rx_no_len:
1869 rxr->rx_prod = NEXT_RX(prod);
1870 rxr->rx_next_cons = NEXT_RX(cons);
1871
1872 next_rx_no_prod_no_len:
1873 *raw_cons = tmp_raw_cons;
1874
1875 return rc;
1876 }
1877
1878 /* In netpoll mode, if we are using a combined completion ring, we need to
1879 * discard the rx packets and recycle the buffers.
1880 */
1881 static int bnxt_force_rx_discard(struct bnxt *bp,
1882 struct bnxt_cp_ring_info *cpr,
1883 u32 *raw_cons, u8 *event)
1884 {
1885 u32 tmp_raw_cons = *raw_cons;
1886 struct rx_cmp_ext *rxcmp1;
1887 struct rx_cmp *rxcmp;
1888 u16 cp_cons;
1889 u8 cmp_type;
1890
1891 cp_cons = RING_CMP(tmp_raw_cons);
1892 rxcmp = (struct rx_cmp *)
1893 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1894
1895 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1896 cp_cons = RING_CMP(tmp_raw_cons);
1897 rxcmp1 = (struct rx_cmp_ext *)
1898 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1899
1900 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1901 return -EBUSY;
1902
1903 cmp_type = RX_CMP_TYPE(rxcmp);
1904 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1905 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1906 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1907 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1908 struct rx_tpa_end_cmp_ext *tpa_end1;
1909
1910 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1911 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1912 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1913 }
1914 return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1915 }
1916
1917 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1918 {
1919 struct bnxt_fw_health *fw_health = bp->fw_health;
1920 u32 reg = fw_health->regs[reg_idx];
1921 u32 reg_type, reg_off, val = 0;
1922
1923 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1924 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1925 switch (reg_type) {
1926 case BNXT_FW_HEALTH_REG_TYPE_CFG:
1927 pci_read_config_dword(bp->pdev, reg_off, &val);
1928 break;
1929 case BNXT_FW_HEALTH_REG_TYPE_GRC:
1930 reg_off = fw_health->mapped_regs[reg_idx];
1931 fallthrough;
1932 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1933 val = readl(bp->bar0 + reg_off);
1934 break;
1935 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1936 val = readl(bp->bar1 + reg_off);
1937 break;
1938 }
1939 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1940 val &= fw_health->fw_reset_inprog_reg_mask;
1941 return val;
1942 }
1943
1944 #define BNXT_GET_EVENT_PORT(data) \
1945 ((data) & \
1946 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1947
1948 static int bnxt_async_event_process(struct bnxt *bp,
1949 struct hwrm_async_event_cmpl *cmpl)
1950 {
1951 u16 event_id = le16_to_cpu(cmpl->event_id);
1952
1953 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1954 switch (event_id) {
1955 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1956 u32 data1 = le32_to_cpu(cmpl->event_data1);
1957 struct bnxt_link_info *link_info = &bp->link_info;
1958
1959 if (BNXT_VF(bp))
1960 goto async_event_process_exit;
1961
1962 /* print unsupported speed warning in forced speed mode only */
1963 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1964 (data1 & 0x20000)) {
1965 u16 fw_speed = link_info->force_link_speed;
1966 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1967
1968 if (speed != SPEED_UNKNOWN)
1969 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1970 speed);
1971 }
1972 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1973 }
1974 fallthrough;
1975 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1976 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1977 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
1978 fallthrough;
1979 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1980 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1981 break;
1982 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1983 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1984 break;
1985 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1986 u32 data1 = le32_to_cpu(cmpl->event_data1);
1987 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1988
1989 if (BNXT_VF(bp))
1990 break;
1991
1992 if (bp->pf.port_id != port_id)
1993 break;
1994
1995 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1996 break;
1997 }
1998 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1999 if (BNXT_PF(bp))
2000 goto async_event_process_exit;
2001 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2002 break;
2003 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2004 u32 data1 = le32_to_cpu(cmpl->event_data1);
2005
2006 if (!bp->fw_health)
2007 goto async_event_process_exit;
2008
2009 bp->fw_reset_timestamp = jiffies;
2010 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2011 if (!bp->fw_reset_min_dsecs)
2012 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2013 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2014 if (!bp->fw_reset_max_dsecs)
2015 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2016 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2017 netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2018 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2019 } else {
2020 netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2021 bp->fw_reset_max_dsecs * 100);
2022 }
2023 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2024 break;
2025 }
2026 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2027 struct bnxt_fw_health *fw_health = bp->fw_health;
2028 u32 data1 = le32_to_cpu(cmpl->event_data1);
2029
2030 if (!fw_health)
2031 goto async_event_process_exit;
2032
2033 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2034 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2035 if (!fw_health->enabled)
2036 break;
2037
2038 if (netif_msg_drv(bp))
2039 netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2040 fw_health->enabled, fw_health->master,
2041 bnxt_fw_health_readl(bp,
2042 BNXT_FW_RESET_CNT_REG),
2043 bnxt_fw_health_readl(bp,
2044 BNXT_FW_HEALTH_REG));
2045 fw_health->tmr_multiplier =
2046 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2047 bp->current_interval * 10);
2048 fw_health->tmr_counter = fw_health->tmr_multiplier;
2049 fw_health->last_fw_heartbeat =
2050 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2051 fw_health->last_fw_reset_cnt =
2052 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2053 goto async_event_process_exit;
2054 }
2055 default:
2056 goto async_event_process_exit;
2057 }
2058 bnxt_queue_sp_work(bp);
2059 async_event_process_exit:
2060 bnxt_ulp_async_events(bp, cmpl);
2061 return 0;
2062 }
2063
2064 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2065 {
2066 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2067 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2068 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2069 (struct hwrm_fwd_req_cmpl *)txcmp;
2070
2071 switch (cmpl_type) {
2072 case CMPL_BASE_TYPE_HWRM_DONE:
2073 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2074 if (seq_id == bp->hwrm_intr_seq_id)
2075 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2076 else
2077 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2078 break;
2079
2080 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2081 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2082
2083 if ((vf_id < bp->pf.first_vf_id) ||
2084 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2085 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2086 vf_id);
2087 return -EINVAL;
2088 }
2089
2090 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2091 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2092 bnxt_queue_sp_work(bp);
2093 break;
2094
2095 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2096 bnxt_async_event_process(bp,
2097 (struct hwrm_async_event_cmpl *)txcmp);
2098
2099 default:
2100 break;
2101 }
2102
2103 return 0;
2104 }
2105
2106 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2107 {
2108 struct bnxt_napi *bnapi = dev_instance;
2109 struct bnxt *bp = bnapi->bp;
2110 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2111 u32 cons = RING_CMP(cpr->cp_raw_cons);
2112
2113 cpr->event_ctr++;
2114 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2115 napi_schedule(&bnapi->napi);
2116 return IRQ_HANDLED;
2117 }
2118
2119 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2120 {
2121 u32 raw_cons = cpr->cp_raw_cons;
2122 u16 cons = RING_CMP(raw_cons);
2123 struct tx_cmp *txcmp;
2124
2125 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2126
2127 return TX_CMP_VALID(txcmp, raw_cons);
2128 }
2129
2130 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2131 {
2132 struct bnxt_napi *bnapi = dev_instance;
2133 struct bnxt *bp = bnapi->bp;
2134 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2135 u32 cons = RING_CMP(cpr->cp_raw_cons);
2136 u32 int_status;
2137
2138 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2139
2140 if (!bnxt_has_work(bp, cpr)) {
2141 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2142 /* return if erroneous interrupt */
2143 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2144 return IRQ_NONE;
2145 }
2146
2147 /* disable ring IRQ */
2148 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2149
2150 /* Return here if interrupt is shared and is disabled. */
2151 if (unlikely(atomic_read(&bp->intr_sem) != 0))
2152 return IRQ_HANDLED;
2153
2154 napi_schedule(&bnapi->napi);
2155 return IRQ_HANDLED;
2156 }
2157
2158 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2159 int budget)
2160 {
2161 struct bnxt_napi *bnapi = cpr->bnapi;
2162 u32 raw_cons = cpr->cp_raw_cons;
2163 u32 cons;
2164 int tx_pkts = 0;
2165 int rx_pkts = 0;
2166 u8 event = 0;
2167 struct tx_cmp *txcmp;
2168
2169 cpr->has_more_work = 0;
2170 cpr->had_work_done = 1;
2171 while (1) {
2172 int rc;
2173
2174 cons = RING_CMP(raw_cons);
2175 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2176
2177 if (!TX_CMP_VALID(txcmp, raw_cons))
2178 break;
2179
2180 /* The valid test of the entry must be done first before
2181 * reading any further.
2182 */
2183 dma_rmb();
2184 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2185 tx_pkts++;
2186 /* return full budget so NAPI will complete. */
2187 if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2188 rx_pkts = budget;
2189 raw_cons = NEXT_RAW_CMP(raw_cons);
2190 if (budget)
2191 cpr->has_more_work = 1;
2192 break;
2193 }
2194 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2195 if (likely(budget))
2196 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2197 else
2198 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2199 &event);
2200 if (likely(rc >= 0))
2201 rx_pkts += rc;
2202 /* Increment rx_pkts when rc is -ENOMEM to count towards
2203 * the NAPI budget. Otherwise, we may potentially loop
2204 * here forever if we consistently cannot allocate
2205 * buffers.
2206 */
2207 else if (rc == -ENOMEM && budget)
2208 rx_pkts++;
2209 else if (rc == -EBUSY) /* partial completion */
2210 break;
2211 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2212 CMPL_BASE_TYPE_HWRM_DONE) ||
2213 (TX_CMP_TYPE(txcmp) ==
2214 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2215 (TX_CMP_TYPE(txcmp) ==
2216 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2217 bnxt_hwrm_handler(bp, txcmp);
2218 }
2219 raw_cons = NEXT_RAW_CMP(raw_cons);
2220
2221 if (rx_pkts && rx_pkts == budget) {
2222 cpr->has_more_work = 1;
2223 break;
2224 }
2225 }
2226
2227 if (event & BNXT_REDIRECT_EVENT)
2228 xdp_do_flush_map();
2229
2230 if (event & BNXT_TX_EVENT) {
2231 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2232 u16 prod = txr->tx_prod;
2233
2234 /* Sync BD data before updating doorbell */
2235 wmb();
2236
2237 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2238 }
2239
2240 cpr->cp_raw_cons = raw_cons;
2241 bnapi->tx_pkts += tx_pkts;
2242 bnapi->events |= event;
2243 return rx_pkts;
2244 }
2245
2246 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2247 {
2248 if (bnapi->tx_pkts) {
2249 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2250 bnapi->tx_pkts = 0;
2251 }
2252
2253 if (bnapi->events & BNXT_RX_EVENT) {
2254 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2255
2256 if (bnapi->events & BNXT_AGG_EVENT)
2257 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2258 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2259 }
2260 bnapi->events = 0;
2261 }
2262
2263 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2264 int budget)
2265 {
2266 struct bnxt_napi *bnapi = cpr->bnapi;
2267 int rx_pkts;
2268
2269 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2270
2271 /* ACK completion ring before freeing tx ring and producing new
2272 * buffers in rx/agg rings to prevent overflowing the completion
2273 * ring.
2274 */
2275 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2276
2277 __bnxt_poll_work_done(bp, bnapi);
2278 return rx_pkts;
2279 }
2280
2281 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2282 {
2283 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2284 struct bnxt *bp = bnapi->bp;
2285 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2286 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2287 struct tx_cmp *txcmp;
2288 struct rx_cmp_ext *rxcmp1;
2289 u32 cp_cons, tmp_raw_cons;
2290 u32 raw_cons = cpr->cp_raw_cons;
2291 u32 rx_pkts = 0;
2292 u8 event = 0;
2293
2294 while (1) {
2295 int rc;
2296
2297 cp_cons = RING_CMP(raw_cons);
2298 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2299
2300 if (!TX_CMP_VALID(txcmp, raw_cons))
2301 break;
2302
2303 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2304 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2305 cp_cons = RING_CMP(tmp_raw_cons);
2306 rxcmp1 = (struct rx_cmp_ext *)
2307 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2308
2309 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2310 break;
2311
2312 /* force an error to recycle the buffer */
2313 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2314 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2315
2316 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2317 if (likely(rc == -EIO) && budget)
2318 rx_pkts++;
2319 else if (rc == -EBUSY) /* partial completion */
2320 break;
2321 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2322 CMPL_BASE_TYPE_HWRM_DONE)) {
2323 bnxt_hwrm_handler(bp, txcmp);
2324 } else {
2325 netdev_err(bp->dev,
2326 "Invalid completion received on special ring\n");
2327 }
2328 raw_cons = NEXT_RAW_CMP(raw_cons);
2329
2330 if (rx_pkts == budget)
2331 break;
2332 }
2333
2334 cpr->cp_raw_cons = raw_cons;
2335 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2336 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2337
2338 if (event & BNXT_AGG_EVENT)
2339 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2340
2341 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2342 napi_complete_done(napi, rx_pkts);
2343 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2344 }
2345 return rx_pkts;
2346 }
2347
2348 static int bnxt_poll(struct napi_struct *napi, int budget)
2349 {
2350 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2351 struct bnxt *bp = bnapi->bp;
2352 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2353 int work_done = 0;
2354
2355 while (1) {
2356 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2357
2358 if (work_done >= budget) {
2359 if (!budget)
2360 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2361 break;
2362 }
2363
2364 if (!bnxt_has_work(bp, cpr)) {
2365 if (napi_complete_done(napi, work_done))
2366 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2367 break;
2368 }
2369 }
2370 if (bp->flags & BNXT_FLAG_DIM) {
2371 struct dim_sample dim_sample = {};
2372
2373 dim_update_sample(cpr->event_ctr,
2374 cpr->rx_packets,
2375 cpr->rx_bytes,
2376 &dim_sample);
2377 net_dim(&cpr->dim, dim_sample);
2378 }
2379 return work_done;
2380 }
2381
2382 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2383 {
2384 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2385 int i, work_done = 0;
2386
2387 for (i = 0; i < 2; i++) {
2388 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2389
2390 if (cpr2) {
2391 work_done += __bnxt_poll_work(bp, cpr2,
2392 budget - work_done);
2393 cpr->has_more_work |= cpr2->has_more_work;
2394 }
2395 }
2396 return work_done;
2397 }
2398
2399 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2400 u64 dbr_type)
2401 {
2402 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2403 int i;
2404
2405 for (i = 0; i < 2; i++) {
2406 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2407 struct bnxt_db_info *db;
2408
2409 if (cpr2 && cpr2->had_work_done) {
2410 db = &cpr2->cp_db;
2411 writeq(db->db_key64 | dbr_type |
2412 RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2413 cpr2->had_work_done = 0;
2414 }
2415 }
2416 __bnxt_poll_work_done(bp, bnapi);
2417 }
2418
2419 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2420 {
2421 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2422 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2423 u32 raw_cons = cpr->cp_raw_cons;
2424 struct bnxt *bp = bnapi->bp;
2425 struct nqe_cn *nqcmp;
2426 int work_done = 0;
2427 u32 cons;
2428
2429 if (cpr->has_more_work) {
2430 cpr->has_more_work = 0;
2431 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2432 }
2433 while (1) {
2434 cons = RING_CMP(raw_cons);
2435 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2436
2437 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2438 if (cpr->has_more_work)
2439 break;
2440
2441 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2442 cpr->cp_raw_cons = raw_cons;
2443 if (napi_complete_done(napi, work_done))
2444 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2445 cpr->cp_raw_cons);
2446 return work_done;
2447 }
2448
2449 /* The valid test of the entry must be done first before
2450 * reading any further.
2451 */
2452 dma_rmb();
2453
2454 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2455 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2456 struct bnxt_cp_ring_info *cpr2;
2457
2458 cpr2 = cpr->cp_ring_arr[idx];
2459 work_done += __bnxt_poll_work(bp, cpr2,
2460 budget - work_done);
2461 cpr->has_more_work |= cpr2->has_more_work;
2462 } else {
2463 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2464 }
2465 raw_cons = NEXT_RAW_CMP(raw_cons);
2466 }
2467 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2468 if (raw_cons != cpr->cp_raw_cons) {
2469 cpr->cp_raw_cons = raw_cons;
2470 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2471 }
2472 return work_done;
2473 }
2474
2475 static void bnxt_free_tx_skbs(struct bnxt *bp)
2476 {
2477 int i, max_idx;
2478 struct pci_dev *pdev = bp->pdev;
2479
2480 if (!bp->tx_ring)
2481 return;
2482
2483 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2484 for (i = 0; i < bp->tx_nr_rings; i++) {
2485 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2486 int j;
2487
2488 for (j = 0; j < max_idx;) {
2489 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2490 struct sk_buff *skb;
2491 int k, last;
2492
2493 if (i < bp->tx_nr_rings_xdp &&
2494 tx_buf->action == XDP_REDIRECT) {
2495 dma_unmap_single(&pdev->dev,
2496 dma_unmap_addr(tx_buf, mapping),
2497 dma_unmap_len(tx_buf, len),
2498 PCI_DMA_TODEVICE);
2499 xdp_return_frame(tx_buf->xdpf);
2500 tx_buf->action = 0;
2501 tx_buf->xdpf = NULL;
2502 j++;
2503 continue;
2504 }
2505
2506 skb = tx_buf->skb;
2507 if (!skb) {
2508 j++;
2509 continue;
2510 }
2511
2512 tx_buf->skb = NULL;
2513
2514 if (tx_buf->is_push) {
2515 dev_kfree_skb(skb);
2516 j += 2;
2517 continue;
2518 }
2519
2520 dma_unmap_single(&pdev->dev,
2521 dma_unmap_addr(tx_buf, mapping),
2522 skb_headlen(skb),
2523 PCI_DMA_TODEVICE);
2524
2525 last = tx_buf->nr_frags;
2526 j += 2;
2527 for (k = 0; k < last; k++, j++) {
2528 int ring_idx = j & bp->tx_ring_mask;
2529 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2530
2531 tx_buf = &txr->tx_buf_ring[ring_idx];
2532 dma_unmap_page(
2533 &pdev->dev,
2534 dma_unmap_addr(tx_buf, mapping),
2535 skb_frag_size(frag), PCI_DMA_TODEVICE);
2536 }
2537 dev_kfree_skb(skb);
2538 }
2539 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2540 }
2541 }
2542
2543 static void bnxt_free_rx_skbs(struct bnxt *bp)
2544 {
2545 int i, max_idx, max_agg_idx;
2546 struct pci_dev *pdev = bp->pdev;
2547
2548 if (!bp->rx_ring)
2549 return;
2550
2551 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2552 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2553 for (i = 0; i < bp->rx_nr_rings; i++) {
2554 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2555 struct bnxt_tpa_idx_map *map;
2556 int j;
2557
2558 if (rxr->rx_tpa) {
2559 for (j = 0; j < bp->max_tpa; j++) {
2560 struct bnxt_tpa_info *tpa_info =
2561 &rxr->rx_tpa[j];
2562 u8 *data = tpa_info->data;
2563
2564 if (!data)
2565 continue;
2566
2567 dma_unmap_single_attrs(&pdev->dev,
2568 tpa_info->mapping,
2569 bp->rx_buf_use_size,
2570 bp->rx_dir,
2571 DMA_ATTR_WEAK_ORDERING);
2572
2573 tpa_info->data = NULL;
2574
2575 kfree(data);
2576 }
2577 }
2578
2579 for (j = 0; j < max_idx; j++) {
2580 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2581 dma_addr_t mapping = rx_buf->mapping;
2582 void *data = rx_buf->data;
2583
2584 if (!data)
2585 continue;
2586
2587 rx_buf->data = NULL;
2588
2589 if (BNXT_RX_PAGE_MODE(bp)) {
2590 mapping -= bp->rx_dma_offset;
2591 dma_unmap_page_attrs(&pdev->dev, mapping,
2592 PAGE_SIZE, bp->rx_dir,
2593 DMA_ATTR_WEAK_ORDERING);
2594 page_pool_recycle_direct(rxr->page_pool, data);
2595 } else {
2596 dma_unmap_single_attrs(&pdev->dev, mapping,
2597 bp->rx_buf_use_size,
2598 bp->rx_dir,
2599 DMA_ATTR_WEAK_ORDERING);
2600 kfree(data);
2601 }
2602 }
2603
2604 for (j = 0; j < max_agg_idx; j++) {
2605 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2606 &rxr->rx_agg_ring[j];
2607 struct page *page = rx_agg_buf->page;
2608
2609 if (!page)
2610 continue;
2611
2612 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2613 BNXT_RX_PAGE_SIZE,
2614 PCI_DMA_FROMDEVICE,
2615 DMA_ATTR_WEAK_ORDERING);
2616
2617 rx_agg_buf->page = NULL;
2618 __clear_bit(j, rxr->rx_agg_bmap);
2619
2620 __free_page(page);
2621 }
2622 if (rxr->rx_page) {
2623 __free_page(rxr->rx_page);
2624 rxr->rx_page = NULL;
2625 }
2626 map = rxr->rx_tpa_idx_map;
2627 if (map)
2628 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2629 }
2630 }
2631
2632 static void bnxt_free_skbs(struct bnxt *bp)
2633 {
2634 bnxt_free_tx_skbs(bp);
2635 bnxt_free_rx_skbs(bp);
2636 }
2637
2638 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2639 {
2640 struct pci_dev *pdev = bp->pdev;
2641 int i;
2642
2643 for (i = 0; i < rmem->nr_pages; i++) {
2644 if (!rmem->pg_arr[i])
2645 continue;
2646
2647 dma_free_coherent(&pdev->dev, rmem->page_size,
2648 rmem->pg_arr[i], rmem->dma_arr[i]);
2649
2650 rmem->pg_arr[i] = NULL;
2651 }
2652 if (rmem->pg_tbl) {
2653 size_t pg_tbl_size = rmem->nr_pages * 8;
2654
2655 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2656 pg_tbl_size = rmem->page_size;
2657 dma_free_coherent(&pdev->dev, pg_tbl_size,
2658 rmem->pg_tbl, rmem->pg_tbl_map);
2659 rmem->pg_tbl = NULL;
2660 }
2661 if (rmem->vmem_size && *rmem->vmem) {
2662 vfree(*rmem->vmem);
2663 *rmem->vmem = NULL;
2664 }
2665 }
2666
2667 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2668 {
2669 struct pci_dev *pdev = bp->pdev;
2670 u64 valid_bit = 0;
2671 int i;
2672
2673 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2674 valid_bit = PTU_PTE_VALID;
2675 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2676 size_t pg_tbl_size = rmem->nr_pages * 8;
2677
2678 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2679 pg_tbl_size = rmem->page_size;
2680 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2681 &rmem->pg_tbl_map,
2682 GFP_KERNEL);
2683 if (!rmem->pg_tbl)
2684 return -ENOMEM;
2685 }
2686
2687 for (i = 0; i < rmem->nr_pages; i++) {
2688 u64 extra_bits = valid_bit;
2689
2690 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2691 rmem->page_size,
2692 &rmem->dma_arr[i],
2693 GFP_KERNEL);
2694 if (!rmem->pg_arr[i])
2695 return -ENOMEM;
2696
2697 if (rmem->init_val)
2698 memset(rmem->pg_arr[i], rmem->init_val,
2699 rmem->page_size);
2700 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2701 if (i == rmem->nr_pages - 2 &&
2702 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2703 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2704 else if (i == rmem->nr_pages - 1 &&
2705 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2706 extra_bits |= PTU_PTE_LAST;
2707 rmem->pg_tbl[i] =
2708 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2709 }
2710 }
2711
2712 if (rmem->vmem_size) {
2713 *rmem->vmem = vzalloc(rmem->vmem_size);
2714 if (!(*rmem->vmem))
2715 return -ENOMEM;
2716 }
2717 return 0;
2718 }
2719
2720 static void bnxt_free_tpa_info(struct bnxt *bp)
2721 {
2722 int i;
2723
2724 for (i = 0; i < bp->rx_nr_rings; i++) {
2725 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2726
2727 kfree(rxr->rx_tpa_idx_map);
2728 rxr->rx_tpa_idx_map = NULL;
2729 if (rxr->rx_tpa) {
2730 kfree(rxr->rx_tpa[0].agg_arr);
2731 rxr->rx_tpa[0].agg_arr = NULL;
2732 }
2733 kfree(rxr->rx_tpa);
2734 rxr->rx_tpa = NULL;
2735 }
2736 }
2737
2738 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2739 {
2740 int i, j, total_aggs = 0;
2741
2742 bp->max_tpa = MAX_TPA;
2743 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2744 if (!bp->max_tpa_v2)
2745 return 0;
2746 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2747 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2748 }
2749
2750 for (i = 0; i < bp->rx_nr_rings; i++) {
2751 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2752 struct rx_agg_cmp *agg;
2753
2754 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2755 GFP_KERNEL);
2756 if (!rxr->rx_tpa)
2757 return -ENOMEM;
2758
2759 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2760 continue;
2761 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2762 rxr->rx_tpa[0].agg_arr = agg;
2763 if (!agg)
2764 return -ENOMEM;
2765 for (j = 1; j < bp->max_tpa; j++)
2766 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2767 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2768 GFP_KERNEL);
2769 if (!rxr->rx_tpa_idx_map)
2770 return -ENOMEM;
2771 }
2772 return 0;
2773 }
2774
2775 static void bnxt_free_rx_rings(struct bnxt *bp)
2776 {
2777 int i;
2778
2779 if (!bp->rx_ring)
2780 return;
2781
2782 bnxt_free_tpa_info(bp);
2783 for (i = 0; i < bp->rx_nr_rings; i++) {
2784 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2785 struct bnxt_ring_struct *ring;
2786
2787 if (rxr->xdp_prog)
2788 bpf_prog_put(rxr->xdp_prog);
2789
2790 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2791 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2792
2793 page_pool_destroy(rxr->page_pool);
2794 rxr->page_pool = NULL;
2795
2796 kfree(rxr->rx_agg_bmap);
2797 rxr->rx_agg_bmap = NULL;
2798
2799 ring = &rxr->rx_ring_struct;
2800 bnxt_free_ring(bp, &ring->ring_mem);
2801
2802 ring = &rxr->rx_agg_ring_struct;
2803 bnxt_free_ring(bp, &ring->ring_mem);
2804 }
2805 }
2806
2807 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2808 struct bnxt_rx_ring_info *rxr)
2809 {
2810 struct page_pool_params pp = { 0 };
2811
2812 pp.pool_size = bp->rx_ring_size;
2813 pp.nid = dev_to_node(&bp->pdev->dev);
2814 pp.dev = &bp->pdev->dev;
2815 pp.dma_dir = DMA_BIDIRECTIONAL;
2816
2817 rxr->page_pool = page_pool_create(&pp);
2818 if (IS_ERR(rxr->page_pool)) {
2819 int err = PTR_ERR(rxr->page_pool);
2820
2821 rxr->page_pool = NULL;
2822 return err;
2823 }
2824 return 0;
2825 }
2826
2827 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2828 {
2829 int i, rc = 0, agg_rings = 0;
2830
2831 if (!bp->rx_ring)
2832 return -ENOMEM;
2833
2834 if (bp->flags & BNXT_FLAG_AGG_RINGS)
2835 agg_rings = 1;
2836
2837 for (i = 0; i < bp->rx_nr_rings; i++) {
2838 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2839 struct bnxt_ring_struct *ring;
2840
2841 ring = &rxr->rx_ring_struct;
2842
2843 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2844 if (rc)
2845 return rc;
2846
2847 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2848 if (rc < 0)
2849 return rc;
2850
2851 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2852 MEM_TYPE_PAGE_POOL,
2853 rxr->page_pool);
2854 if (rc) {
2855 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2856 return rc;
2857 }
2858
2859 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2860 if (rc)
2861 return rc;
2862
2863 ring->grp_idx = i;
2864 if (agg_rings) {
2865 u16 mem_size;
2866
2867 ring = &rxr->rx_agg_ring_struct;
2868 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2869 if (rc)
2870 return rc;
2871
2872 ring->grp_idx = i;
2873 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2874 mem_size = rxr->rx_agg_bmap_size / 8;
2875 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2876 if (!rxr->rx_agg_bmap)
2877 return -ENOMEM;
2878 }
2879 }
2880 if (bp->flags & BNXT_FLAG_TPA)
2881 rc = bnxt_alloc_tpa_info(bp);
2882 return rc;
2883 }
2884
2885 static void bnxt_free_tx_rings(struct bnxt *bp)
2886 {
2887 int i;
2888 struct pci_dev *pdev = bp->pdev;
2889
2890 if (!bp->tx_ring)
2891 return;
2892
2893 for (i = 0; i < bp->tx_nr_rings; i++) {
2894 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2895 struct bnxt_ring_struct *ring;
2896
2897 if (txr->tx_push) {
2898 dma_free_coherent(&pdev->dev, bp->tx_push_size,
2899 txr->tx_push, txr->tx_push_mapping);
2900 txr->tx_push = NULL;
2901 }
2902
2903 ring = &txr->tx_ring_struct;
2904
2905 bnxt_free_ring(bp, &ring->ring_mem);
2906 }
2907 }
2908
2909 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2910 {
2911 int i, j, rc;
2912 struct pci_dev *pdev = bp->pdev;
2913
2914 bp->tx_push_size = 0;
2915 if (bp->tx_push_thresh) {
2916 int push_size;
2917
2918 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2919 bp->tx_push_thresh);
2920
2921 if (push_size > 256) {
2922 push_size = 0;
2923 bp->tx_push_thresh = 0;
2924 }
2925
2926 bp->tx_push_size = push_size;
2927 }
2928
2929 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2930 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2931 struct bnxt_ring_struct *ring;
2932 u8 qidx;
2933
2934 ring = &txr->tx_ring_struct;
2935
2936 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2937 if (rc)
2938 return rc;
2939
2940 ring->grp_idx = txr->bnapi->index;
2941 if (bp->tx_push_size) {
2942 dma_addr_t mapping;
2943
2944 /* One pre-allocated DMA buffer to backup
2945 * TX push operation
2946 */
2947 txr->tx_push = dma_alloc_coherent(&pdev->dev,
2948 bp->tx_push_size,
2949 &txr->tx_push_mapping,
2950 GFP_KERNEL);
2951
2952 if (!txr->tx_push)
2953 return -ENOMEM;
2954
2955 mapping = txr->tx_push_mapping +
2956 sizeof(struct tx_push_bd);
2957 txr->data_mapping = cpu_to_le64(mapping);
2958 }
2959 qidx = bp->tc_to_qidx[j];
2960 ring->queue_id = bp->q_info[qidx].queue_id;
2961 if (i < bp->tx_nr_rings_xdp)
2962 continue;
2963 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2964 j++;
2965 }
2966 return 0;
2967 }
2968
2969 static void bnxt_free_cp_rings(struct bnxt *bp)
2970 {
2971 int i;
2972
2973 if (!bp->bnapi)
2974 return;
2975
2976 for (i = 0; i < bp->cp_nr_rings; i++) {
2977 struct bnxt_napi *bnapi = bp->bnapi[i];
2978 struct bnxt_cp_ring_info *cpr;
2979 struct bnxt_ring_struct *ring;
2980 int j;
2981
2982 if (!bnapi)
2983 continue;
2984
2985 cpr = &bnapi->cp_ring;
2986 ring = &cpr->cp_ring_struct;
2987
2988 bnxt_free_ring(bp, &ring->ring_mem);
2989
2990 for (j = 0; j < 2; j++) {
2991 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2992
2993 if (cpr2) {
2994 ring = &cpr2->cp_ring_struct;
2995 bnxt_free_ring(bp, &ring->ring_mem);
2996 kfree(cpr2);
2997 cpr->cp_ring_arr[j] = NULL;
2998 }
2999 }
3000 }
3001 }
3002
3003 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3004 {
3005 struct bnxt_ring_mem_info *rmem;
3006 struct bnxt_ring_struct *ring;
3007 struct bnxt_cp_ring_info *cpr;
3008 int rc;
3009
3010 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3011 if (!cpr)
3012 return NULL;
3013
3014 ring = &cpr->cp_ring_struct;
3015 rmem = &ring->ring_mem;
3016 rmem->nr_pages = bp->cp_nr_pages;
3017 rmem->page_size = HW_CMPD_RING_SIZE;
3018 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3019 rmem->dma_arr = cpr->cp_desc_mapping;
3020 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3021 rc = bnxt_alloc_ring(bp, rmem);
3022 if (rc) {
3023 bnxt_free_ring(bp, rmem);
3024 kfree(cpr);
3025 cpr = NULL;
3026 }
3027 return cpr;
3028 }
3029
3030 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3031 {
3032 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3033 int i, rc, ulp_base_vec, ulp_msix;
3034
3035 ulp_msix = bnxt_get_ulp_msix_num(bp);
3036 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3037 for (i = 0; i < bp->cp_nr_rings; i++) {
3038 struct bnxt_napi *bnapi = bp->bnapi[i];
3039 struct bnxt_cp_ring_info *cpr;
3040 struct bnxt_ring_struct *ring;
3041
3042 if (!bnapi)
3043 continue;
3044
3045 cpr = &bnapi->cp_ring;
3046 cpr->bnapi = bnapi;
3047 ring = &cpr->cp_ring_struct;
3048
3049 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3050 if (rc)
3051 return rc;
3052
3053 if (ulp_msix && i >= ulp_base_vec)
3054 ring->map_idx = i + ulp_msix;
3055 else
3056 ring->map_idx = i;
3057
3058 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3059 continue;
3060
3061 if (i < bp->rx_nr_rings) {
3062 struct bnxt_cp_ring_info *cpr2 =
3063 bnxt_alloc_cp_sub_ring(bp);
3064
3065 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3066 if (!cpr2)
3067 return -ENOMEM;
3068 cpr2->bnapi = bnapi;
3069 }
3070 if ((sh && i < bp->tx_nr_rings) ||
3071 (!sh && i >= bp->rx_nr_rings)) {
3072 struct bnxt_cp_ring_info *cpr2 =
3073 bnxt_alloc_cp_sub_ring(bp);
3074
3075 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3076 if (!cpr2)
3077 return -ENOMEM;
3078 cpr2->bnapi = bnapi;
3079 }
3080 }
3081 return 0;
3082 }
3083
3084 static void bnxt_init_ring_struct(struct bnxt *bp)
3085 {
3086 int i;
3087
3088 for (i = 0; i < bp->cp_nr_rings; i++) {
3089 struct bnxt_napi *bnapi = bp->bnapi[i];
3090 struct bnxt_ring_mem_info *rmem;
3091 struct bnxt_cp_ring_info *cpr;
3092 struct bnxt_rx_ring_info *rxr;
3093 struct bnxt_tx_ring_info *txr;
3094 struct bnxt_ring_struct *ring;
3095
3096 if (!bnapi)
3097 continue;
3098
3099 cpr = &bnapi->cp_ring;
3100 ring = &cpr->cp_ring_struct;
3101 rmem = &ring->ring_mem;
3102 rmem->nr_pages = bp->cp_nr_pages;
3103 rmem->page_size = HW_CMPD_RING_SIZE;
3104 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3105 rmem->dma_arr = cpr->cp_desc_mapping;
3106 rmem->vmem_size = 0;
3107
3108 rxr = bnapi->rx_ring;
3109 if (!rxr)
3110 goto skip_rx;
3111
3112 ring = &rxr->rx_ring_struct;
3113 rmem = &ring->ring_mem;
3114 rmem->nr_pages = bp->rx_nr_pages;
3115 rmem->page_size = HW_RXBD_RING_SIZE;
3116 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3117 rmem->dma_arr = rxr->rx_desc_mapping;
3118 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3119 rmem->vmem = (void **)&rxr->rx_buf_ring;
3120
3121 ring = &rxr->rx_agg_ring_struct;
3122 rmem = &ring->ring_mem;
3123 rmem->nr_pages = bp->rx_agg_nr_pages;
3124 rmem->page_size = HW_RXBD_RING_SIZE;
3125 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3126 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3127 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3128 rmem->vmem = (void **)&rxr->rx_agg_ring;
3129
3130 skip_rx:
3131 txr = bnapi->tx_ring;
3132 if (!txr)
3133 continue;
3134
3135 ring = &txr->tx_ring_struct;
3136 rmem = &ring->ring_mem;
3137 rmem->nr_pages = bp->tx_nr_pages;
3138 rmem->page_size = HW_RXBD_RING_SIZE;
3139 rmem->pg_arr = (void **)txr->tx_desc_ring;
3140 rmem->dma_arr = txr->tx_desc_mapping;
3141 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3142 rmem->vmem = (void **)&txr->tx_buf_ring;
3143 }
3144 }
3145
3146 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3147 {
3148 int i;
3149 u32 prod;
3150 struct rx_bd **rx_buf_ring;
3151
3152 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3153 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3154 int j;
3155 struct rx_bd *rxbd;
3156
3157 rxbd = rx_buf_ring[i];
3158 if (!rxbd)
3159 continue;
3160
3161 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3162 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3163 rxbd->rx_bd_opaque = prod;
3164 }
3165 }
3166 }
3167
3168 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3169 {
3170 struct net_device *dev = bp->dev;
3171 struct bnxt_rx_ring_info *rxr;
3172 struct bnxt_ring_struct *ring;
3173 u32 prod, type;
3174 int i;
3175
3176 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3177 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3178
3179 if (NET_IP_ALIGN == 2)
3180 type |= RX_BD_FLAGS_SOP;
3181
3182 rxr = &bp->rx_ring[ring_nr];
3183 ring = &rxr->rx_ring_struct;
3184 bnxt_init_rxbd_pages(ring, type);
3185
3186 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3187 bpf_prog_add(bp->xdp_prog, 1);
3188 rxr->xdp_prog = bp->xdp_prog;
3189 }
3190 prod = rxr->rx_prod;
3191 for (i = 0; i < bp->rx_ring_size; i++) {
3192 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3193 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3194 ring_nr, i, bp->rx_ring_size);
3195 break;
3196 }
3197 prod = NEXT_RX(prod);
3198 }
3199 rxr->rx_prod = prod;
3200 ring->fw_ring_id = INVALID_HW_RING_ID;
3201
3202 ring = &rxr->rx_agg_ring_struct;
3203 ring->fw_ring_id = INVALID_HW_RING_ID;
3204
3205 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3206 return 0;
3207
3208 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3209 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3210
3211 bnxt_init_rxbd_pages(ring, type);
3212
3213 prod = rxr->rx_agg_prod;
3214 for (i = 0; i < bp->rx_agg_ring_size; i++) {
3215 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3216 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3217 ring_nr, i, bp->rx_ring_size);
3218 break;
3219 }
3220 prod = NEXT_RX_AGG(prod);
3221 }
3222 rxr->rx_agg_prod = prod;
3223
3224 if (bp->flags & BNXT_FLAG_TPA) {
3225 if (rxr->rx_tpa) {
3226 u8 *data;
3227 dma_addr_t mapping;
3228
3229 for (i = 0; i < bp->max_tpa; i++) {
3230 data = __bnxt_alloc_rx_data(bp, &mapping,
3231 GFP_KERNEL);
3232 if (!data)
3233 return -ENOMEM;
3234
3235 rxr->rx_tpa[i].data = data;
3236 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3237 rxr->rx_tpa[i].mapping = mapping;
3238 }
3239 } else {
3240 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3241 return -ENOMEM;
3242 }
3243 }
3244
3245 return 0;
3246 }
3247
3248 static void bnxt_init_cp_rings(struct bnxt *bp)
3249 {
3250 int i, j;
3251
3252 for (i = 0; i < bp->cp_nr_rings; i++) {
3253 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3254 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3255
3256 ring->fw_ring_id = INVALID_HW_RING_ID;
3257 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3258 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3259 for (j = 0; j < 2; j++) {
3260 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3261
3262 if (!cpr2)
3263 continue;
3264
3265 ring = &cpr2->cp_ring_struct;
3266 ring->fw_ring_id = INVALID_HW_RING_ID;
3267 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3268 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3269 }
3270 }
3271 }
3272
3273 static int bnxt_init_rx_rings(struct bnxt *bp)
3274 {
3275 int i, rc = 0;
3276
3277 if (BNXT_RX_PAGE_MODE(bp)) {
3278 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3279 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3280 } else {
3281 bp->rx_offset = BNXT_RX_OFFSET;
3282 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3283 }
3284
3285 for (i = 0; i < bp->rx_nr_rings; i++) {
3286 rc = bnxt_init_one_rx_ring(bp, i);
3287 if (rc)
3288 break;
3289 }
3290
3291 return rc;
3292 }
3293
3294 static int bnxt_init_tx_rings(struct bnxt *bp)
3295 {
3296 u16 i;
3297
3298 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3299 MAX_SKB_FRAGS + 1);
3300
3301 for (i = 0; i < bp->tx_nr_rings; i++) {
3302 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3303 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3304
3305 ring->fw_ring_id = INVALID_HW_RING_ID;
3306 }
3307
3308 return 0;
3309 }
3310
3311 static void bnxt_free_ring_grps(struct bnxt *bp)
3312 {
3313 kfree(bp->grp_info);
3314 bp->grp_info = NULL;
3315 }
3316
3317 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3318 {
3319 int i;
3320
3321 if (irq_re_init) {
3322 bp->grp_info = kcalloc(bp->cp_nr_rings,
3323 sizeof(struct bnxt_ring_grp_info),
3324 GFP_KERNEL);
3325 if (!bp->grp_info)
3326 return -ENOMEM;
3327 }
3328 for (i = 0; i < bp->cp_nr_rings; i++) {
3329 if (irq_re_init)
3330 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3331 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3332 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3333 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3334 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3335 }
3336 return 0;
3337 }
3338
3339 static void bnxt_free_vnics(struct bnxt *bp)
3340 {
3341 kfree(bp->vnic_info);
3342 bp->vnic_info = NULL;
3343 bp->nr_vnics = 0;
3344 }
3345
3346 static int bnxt_alloc_vnics(struct bnxt *bp)
3347 {
3348 int num_vnics = 1;
3349
3350 #ifdef CONFIG_RFS_ACCEL
3351 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3352 num_vnics += bp->rx_nr_rings;
3353 #endif
3354
3355 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3356 num_vnics++;
3357
3358 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3359 GFP_KERNEL);
3360 if (!bp->vnic_info)
3361 return -ENOMEM;
3362
3363 bp->nr_vnics = num_vnics;
3364 return 0;
3365 }
3366
3367 static void bnxt_init_vnics(struct bnxt *bp)
3368 {
3369 int i;
3370
3371 for (i = 0; i < bp->nr_vnics; i++) {
3372 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3373 int j;
3374
3375 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3376 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3377 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3378
3379 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3380
3381 if (bp->vnic_info[i].rss_hash_key) {
3382 if (i == 0)
3383 prandom_bytes(vnic->rss_hash_key,
3384 HW_HASH_KEY_SIZE);
3385 else
3386 memcpy(vnic->rss_hash_key,
3387 bp->vnic_info[0].rss_hash_key,
3388 HW_HASH_KEY_SIZE);
3389 }
3390 }
3391 }
3392
3393 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3394 {
3395 int pages;
3396
3397 pages = ring_size / desc_per_pg;
3398
3399 if (!pages)
3400 return 1;
3401
3402 pages++;
3403
3404 while (pages & (pages - 1))
3405 pages++;
3406
3407 return pages;
3408 }
3409
3410 void bnxt_set_tpa_flags(struct bnxt *bp)
3411 {
3412 bp->flags &= ~BNXT_FLAG_TPA;
3413 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3414 return;
3415 if (bp->dev->features & NETIF_F_LRO)
3416 bp->flags |= BNXT_FLAG_LRO;
3417 else if (bp->dev->features & NETIF_F_GRO_HW)
3418 bp->flags |= BNXT_FLAG_GRO;
3419 }
3420
3421 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3422 * be set on entry.
3423 */
3424 void bnxt_set_ring_params(struct bnxt *bp)
3425 {
3426 u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3427 u32 agg_factor = 0, agg_ring_size = 0;
3428
3429 /* 8 for CRC and VLAN */
3430 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3431
3432 rx_space = rx_size + NET_SKB_PAD +
3433 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3434
3435 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3436 ring_size = bp->rx_ring_size;
3437 bp->rx_agg_ring_size = 0;
3438 bp->rx_agg_nr_pages = 0;
3439
3440 if (bp->flags & BNXT_FLAG_TPA)
3441 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3442
3443 bp->flags &= ~BNXT_FLAG_JUMBO;
3444 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3445 u32 jumbo_factor;
3446
3447 bp->flags |= BNXT_FLAG_JUMBO;
3448 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3449 if (jumbo_factor > agg_factor)
3450 agg_factor = jumbo_factor;
3451 }
3452 agg_ring_size = ring_size * agg_factor;
3453
3454 if (agg_ring_size) {
3455 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3456 RX_DESC_CNT);
3457 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3458 u32 tmp = agg_ring_size;
3459
3460 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3461 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3462 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3463 tmp, agg_ring_size);
3464 }
3465 bp->rx_agg_ring_size = agg_ring_size;
3466 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3467 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3468 rx_space = rx_size + NET_SKB_PAD +
3469 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3470 }
3471
3472 bp->rx_buf_use_size = rx_size;
3473 bp->rx_buf_size = rx_space;
3474
3475 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3476 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3477
3478 ring_size = bp->tx_ring_size;
3479 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3480 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3481
3482 max_rx_cmpl = bp->rx_ring_size;
3483 /* MAX TPA needs to be added because TPA_START completions are
3484 * immediately recycled, so the TPA completions are not bound by
3485 * the RX ring size.
3486 */
3487 if (bp->flags & BNXT_FLAG_TPA)
3488 max_rx_cmpl += bp->max_tpa;
3489 /* RX and TPA completions are 32-byte, all others are 16-byte */
3490 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3491 bp->cp_ring_size = ring_size;
3492
3493 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3494 if (bp->cp_nr_pages > MAX_CP_PAGES) {
3495 bp->cp_nr_pages = MAX_CP_PAGES;
3496 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3497 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3498 ring_size, bp->cp_ring_size);
3499 }
3500 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3501 bp->cp_ring_mask = bp->cp_bit - 1;
3502 }
3503
3504 /* Changing allocation mode of RX rings.
3505 * TODO: Update when extending xdp_rxq_info to support allocation modes.
3506 */
3507 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3508 {
3509 if (page_mode) {
3510 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3511 return -EOPNOTSUPP;
3512 bp->dev->max_mtu =
3513 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3514 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3515 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3516 bp->rx_dir = DMA_BIDIRECTIONAL;
3517 bp->rx_skb_func = bnxt_rx_page_skb;
3518 /* Disable LRO or GRO_HW */
3519 netdev_update_features(bp->dev);
3520 } else {
3521 bp->dev->max_mtu = bp->max_mtu;
3522 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3523 bp->rx_dir = DMA_FROM_DEVICE;
3524 bp->rx_skb_func = bnxt_rx_skb;
3525 }
3526 return 0;
3527 }
3528
3529 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3530 {
3531 int i;
3532 struct bnxt_vnic_info *vnic;
3533 struct pci_dev *pdev = bp->pdev;
3534
3535 if (!bp->vnic_info)
3536 return;
3537
3538 for (i = 0; i < bp->nr_vnics; i++) {
3539 vnic = &bp->vnic_info[i];
3540
3541 kfree(vnic->fw_grp_ids);
3542 vnic->fw_grp_ids = NULL;
3543
3544 kfree(vnic->uc_list);
3545 vnic->uc_list = NULL;
3546
3547 if (vnic->mc_list) {
3548 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3549 vnic->mc_list, vnic->mc_list_mapping);
3550 vnic->mc_list = NULL;
3551 }
3552
3553 if (vnic->rss_table) {
3554 dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3555 vnic->rss_table,
3556 vnic->rss_table_dma_addr);
3557 vnic->rss_table = NULL;
3558 }
3559
3560 vnic->rss_hash_key = NULL;
3561 vnic->flags = 0;
3562 }
3563 }
3564
3565 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3566 {
3567 int i, rc = 0, size;
3568 struct bnxt_vnic_info *vnic;
3569 struct pci_dev *pdev = bp->pdev;
3570 int max_rings;
3571
3572 for (i = 0; i < bp->nr_vnics; i++) {
3573 vnic = &bp->vnic_info[i];
3574
3575 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3576 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3577
3578 if (mem_size > 0) {
3579 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3580 if (!vnic->uc_list) {
3581 rc = -ENOMEM;
3582 goto out;
3583 }
3584 }
3585 }
3586
3587 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3588 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3589 vnic->mc_list =
3590 dma_alloc_coherent(&pdev->dev,
3591 vnic->mc_list_size,
3592 &vnic->mc_list_mapping,
3593 GFP_KERNEL);
3594 if (!vnic->mc_list) {
3595 rc = -ENOMEM;
3596 goto out;
3597 }
3598 }
3599
3600 if (bp->flags & BNXT_FLAG_CHIP_P5)
3601 goto vnic_skip_grps;
3602
3603 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3604 max_rings = bp->rx_nr_rings;
3605 else
3606 max_rings = 1;
3607
3608 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3609 if (!vnic->fw_grp_ids) {
3610 rc = -ENOMEM;
3611 goto out;
3612 }
3613 vnic_skip_grps:
3614 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3615 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3616 continue;
3617
3618 /* Allocate rss table and hash key */
3619 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3620 if (bp->flags & BNXT_FLAG_CHIP_P5)
3621 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3622
3623 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3624 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3625 vnic->rss_table_size,
3626 &vnic->rss_table_dma_addr,
3627 GFP_KERNEL);
3628 if (!vnic->rss_table) {
3629 rc = -ENOMEM;
3630 goto out;
3631 }
3632
3633 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3634 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3635 }
3636 return 0;
3637
3638 out:
3639 return rc;
3640 }
3641
3642 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3643 {
3644 struct pci_dev *pdev = bp->pdev;
3645
3646 if (bp->hwrm_cmd_resp_addr) {
3647 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3648 bp->hwrm_cmd_resp_dma_addr);
3649 bp->hwrm_cmd_resp_addr = NULL;
3650 }
3651
3652 if (bp->hwrm_cmd_kong_resp_addr) {
3653 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3654 bp->hwrm_cmd_kong_resp_addr,
3655 bp->hwrm_cmd_kong_resp_dma_addr);
3656 bp->hwrm_cmd_kong_resp_addr = NULL;
3657 }
3658 }
3659
3660 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3661 {
3662 struct pci_dev *pdev = bp->pdev;
3663
3664 if (bp->hwrm_cmd_kong_resp_addr)
3665 return 0;
3666
3667 bp->hwrm_cmd_kong_resp_addr =
3668 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3669 &bp->hwrm_cmd_kong_resp_dma_addr,
3670 GFP_KERNEL);
3671 if (!bp->hwrm_cmd_kong_resp_addr)
3672 return -ENOMEM;
3673
3674 return 0;
3675 }
3676
3677 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3678 {
3679 struct pci_dev *pdev = bp->pdev;
3680
3681 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3682 &bp->hwrm_cmd_resp_dma_addr,
3683 GFP_KERNEL);
3684 if (!bp->hwrm_cmd_resp_addr)
3685 return -ENOMEM;
3686
3687 return 0;
3688 }
3689
3690 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3691 {
3692 if (bp->hwrm_short_cmd_req_addr) {
3693 struct pci_dev *pdev = bp->pdev;
3694
3695 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3696 bp->hwrm_short_cmd_req_addr,
3697 bp->hwrm_short_cmd_req_dma_addr);
3698 bp->hwrm_short_cmd_req_addr = NULL;
3699 }
3700 }
3701
3702 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3703 {
3704 struct pci_dev *pdev = bp->pdev;
3705
3706 if (bp->hwrm_short_cmd_req_addr)
3707 return 0;
3708
3709 bp->hwrm_short_cmd_req_addr =
3710 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3711 &bp->hwrm_short_cmd_req_dma_addr,
3712 GFP_KERNEL);
3713 if (!bp->hwrm_short_cmd_req_addr)
3714 return -ENOMEM;
3715
3716 return 0;
3717 }
3718
3719 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3720 {
3721 kfree(stats->hw_masks);
3722 stats->hw_masks = NULL;
3723 kfree(stats->sw_stats);
3724 stats->sw_stats = NULL;
3725 if (stats->hw_stats) {
3726 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3727 stats->hw_stats_map);
3728 stats->hw_stats = NULL;
3729 }
3730 }
3731
3732 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3733 bool alloc_masks)
3734 {
3735 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3736 &stats->hw_stats_map, GFP_KERNEL);
3737 if (!stats->hw_stats)
3738 return -ENOMEM;
3739
3740 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3741 if (!stats->sw_stats)
3742 goto stats_mem_err;
3743
3744 if (alloc_masks) {
3745 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3746 if (!stats->hw_masks)
3747 goto stats_mem_err;
3748 }
3749 return 0;
3750
3751 stats_mem_err:
3752 bnxt_free_stats_mem(bp, stats);
3753 return -ENOMEM;
3754 }
3755
3756 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3757 {
3758 int i;
3759
3760 for (i = 0; i < count; i++)
3761 mask_arr[i] = mask;
3762 }
3763
3764 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3765 {
3766 int i;
3767
3768 for (i = 0; i < count; i++)
3769 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3770 }
3771
3772 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3773 struct bnxt_stats_mem *stats)
3774 {
3775 struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3776 struct hwrm_func_qstats_ext_input req = {0};
3777 __le64 *hw_masks;
3778 int rc;
3779
3780 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3781 !(bp->flags & BNXT_FLAG_CHIP_P5))
3782 return -EOPNOTSUPP;
3783
3784 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3785 req.fid = cpu_to_le16(0xffff);
3786 req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3787 mutex_lock(&bp->hwrm_cmd_lock);
3788 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3789 if (rc)
3790 goto qstat_exit;
3791
3792 hw_masks = &resp->rx_ucast_pkts;
3793 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3794
3795 qstat_exit:
3796 mutex_unlock(&bp->hwrm_cmd_lock);
3797 return rc;
3798 }
3799
3800 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3801 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3802
3803 static void bnxt_init_stats(struct bnxt *bp)
3804 {
3805 struct bnxt_napi *bnapi = bp->bnapi[0];
3806 struct bnxt_cp_ring_info *cpr;
3807 struct bnxt_stats_mem *stats;
3808 __le64 *rx_stats, *tx_stats;
3809 int rc, rx_count, tx_count;
3810 u64 *rx_masks, *tx_masks;
3811 u64 mask;
3812 u8 flags;
3813
3814 cpr = &bnapi->cp_ring;
3815 stats = &cpr->stats;
3816 rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3817 if (rc) {
3818 if (bp->flags & BNXT_FLAG_CHIP_P5)
3819 mask = (1ULL << 48) - 1;
3820 else
3821 mask = -1ULL;
3822 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3823 }
3824 if (bp->flags & BNXT_FLAG_PORT_STATS) {
3825 stats = &bp->port_stats;
3826 rx_stats = stats->hw_stats;
3827 rx_masks = stats->hw_masks;
3828 rx_count = sizeof(struct rx_port_stats) / 8;
3829 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3830 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3831 tx_count = sizeof(struct tx_port_stats) / 8;
3832
3833 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3834 rc = bnxt_hwrm_port_qstats(bp, flags);
3835 if (rc) {
3836 mask = (1ULL << 40) - 1;
3837
3838 bnxt_fill_masks(rx_masks, mask, rx_count);
3839 bnxt_fill_masks(tx_masks, mask, tx_count);
3840 } else {
3841 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3842 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3843 bnxt_hwrm_port_qstats(bp, 0);
3844 }
3845 }
3846 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3847 stats = &bp->rx_port_stats_ext;
3848 rx_stats = stats->hw_stats;
3849 rx_masks = stats->hw_masks;
3850 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3851 stats = &bp->tx_port_stats_ext;
3852 tx_stats = stats->hw_stats;
3853 tx_masks = stats->hw_masks;
3854 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3855
3856 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3857 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3858 if (rc) {
3859 mask = (1ULL << 40) - 1;
3860
3861 bnxt_fill_masks(rx_masks, mask, rx_count);
3862 if (tx_stats)
3863 bnxt_fill_masks(tx_masks, mask, tx_count);
3864 } else {
3865 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3866 if (tx_stats)
3867 bnxt_copy_hw_masks(tx_masks, tx_stats,
3868 tx_count);
3869 bnxt_hwrm_port_qstats_ext(bp, 0);
3870 }
3871 }
3872 }
3873
3874 static void bnxt_free_port_stats(struct bnxt *bp)
3875 {
3876 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3877 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3878
3879 bnxt_free_stats_mem(bp, &bp->port_stats);
3880 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3881 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3882 }
3883
3884 static void bnxt_free_ring_stats(struct bnxt *bp)
3885 {
3886 int i;
3887
3888 if (!bp->bnapi)
3889 return;
3890
3891 for (i = 0; i < bp->cp_nr_rings; i++) {
3892 struct bnxt_napi *bnapi = bp->bnapi[i];
3893 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3894
3895 bnxt_free_stats_mem(bp, &cpr->stats);
3896 }
3897 }
3898
3899 static int bnxt_alloc_stats(struct bnxt *bp)
3900 {
3901 u32 size, i;
3902 int rc;
3903
3904 size = bp->hw_ring_stats_size;
3905
3906 for (i = 0; i < bp->cp_nr_rings; i++) {
3907 struct bnxt_napi *bnapi = bp->bnapi[i];
3908 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3909
3910 cpr->stats.len = size;
3911 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3912 if (rc)
3913 return rc;
3914
3915 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3916 }
3917
3918 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3919 return 0;
3920
3921 if (bp->port_stats.hw_stats)
3922 goto alloc_ext_stats;
3923
3924 bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3925 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3926 if (rc)
3927 return rc;
3928
3929 bp->flags |= BNXT_FLAG_PORT_STATS;
3930
3931 alloc_ext_stats:
3932 /* Display extended statistics only if FW supports it */
3933 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3934 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3935 return 0;
3936
3937 if (bp->rx_port_stats_ext.hw_stats)
3938 goto alloc_tx_ext_stats;
3939
3940 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3941 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3942 /* Extended stats are optional */
3943 if (rc)
3944 return 0;
3945
3946 alloc_tx_ext_stats:
3947 if (bp->tx_port_stats_ext.hw_stats)
3948 return 0;
3949
3950 if (bp->hwrm_spec_code >= 0x10902 ||
3951 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3952 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3953 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3954 /* Extended stats are optional */
3955 if (rc)
3956 return 0;
3957 }
3958 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3959 return 0;
3960 }
3961
3962 static void bnxt_clear_ring_indices(struct bnxt *bp)
3963 {
3964 int i;
3965
3966 if (!bp->bnapi)
3967 return;
3968
3969 for (i = 0; i < bp->cp_nr_rings; i++) {
3970 struct bnxt_napi *bnapi = bp->bnapi[i];
3971 struct bnxt_cp_ring_info *cpr;
3972 struct bnxt_rx_ring_info *rxr;
3973 struct bnxt_tx_ring_info *txr;
3974
3975 if (!bnapi)
3976 continue;
3977
3978 cpr = &bnapi->cp_ring;
3979 cpr->cp_raw_cons = 0;
3980
3981 txr = bnapi->tx_ring;
3982 if (txr) {
3983 txr->tx_prod = 0;
3984 txr->tx_cons = 0;
3985 }
3986
3987 rxr = bnapi->rx_ring;
3988 if (rxr) {
3989 rxr->rx_prod = 0;
3990 rxr->rx_agg_prod = 0;
3991 rxr->rx_sw_agg_prod = 0;
3992 rxr->rx_next_cons = 0;
3993 }
3994 }
3995 }
3996
3997 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3998 {
3999 #ifdef CONFIG_RFS_ACCEL
4000 int i;
4001
4002 /* Under rtnl_lock and all our NAPIs have been disabled. It's
4003 * safe to delete the hash table.
4004 */
4005 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4006 struct hlist_head *head;
4007 struct hlist_node *tmp;
4008 struct bnxt_ntuple_filter *fltr;
4009
4010 head = &bp->ntp_fltr_hash_tbl[i];
4011 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4012 hlist_del(&fltr->hash);
4013 kfree(fltr);
4014 }
4015 }
4016 if (irq_reinit) {
4017 kfree(bp->ntp_fltr_bmap);
4018 bp->ntp_fltr_bmap = NULL;
4019 }
4020 bp->ntp_fltr_count = 0;
4021 #endif
4022 }
4023
4024 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4025 {
4026 #ifdef CONFIG_RFS_ACCEL
4027 int i, rc = 0;
4028
4029 if (!(bp->flags & BNXT_FLAG_RFS))
4030 return 0;
4031
4032 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4033 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4034
4035 bp->ntp_fltr_count = 0;
4036 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4037 sizeof(long),
4038 GFP_KERNEL);
4039
4040 if (!bp->ntp_fltr_bmap)
4041 rc = -ENOMEM;
4042
4043 return rc;
4044 #else
4045 return 0;
4046 #endif
4047 }
4048
4049 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4050 {
4051 bnxt_free_vnic_attributes(bp);
4052 bnxt_free_tx_rings(bp);
4053 bnxt_free_rx_rings(bp);
4054 bnxt_free_cp_rings(bp);
4055 bnxt_free_ntp_fltrs(bp, irq_re_init);
4056 if (irq_re_init) {
4057 bnxt_free_ring_stats(bp);
4058 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET))
4059 bnxt_free_port_stats(bp);
4060 bnxt_free_ring_grps(bp);
4061 bnxt_free_vnics(bp);
4062 kfree(bp->tx_ring_map);
4063 bp->tx_ring_map = NULL;
4064 kfree(bp->tx_ring);
4065 bp->tx_ring = NULL;
4066 kfree(bp->rx_ring);
4067 bp->rx_ring = NULL;
4068 kfree(bp->bnapi);
4069 bp->bnapi = NULL;
4070 } else {
4071 bnxt_clear_ring_indices(bp);
4072 }
4073 }
4074
4075 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4076 {
4077 int i, j, rc, size, arr_size;
4078 void *bnapi;
4079
4080 if (irq_re_init) {
4081 /* Allocate bnapi mem pointer array and mem block for
4082 * all queues
4083 */
4084 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4085 bp->cp_nr_rings);
4086 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4087 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4088 if (!bnapi)
4089 return -ENOMEM;
4090
4091 bp->bnapi = bnapi;
4092 bnapi += arr_size;
4093 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4094 bp->bnapi[i] = bnapi;
4095 bp->bnapi[i]->index = i;
4096 bp->bnapi[i]->bp = bp;
4097 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4098 struct bnxt_cp_ring_info *cpr =
4099 &bp->bnapi[i]->cp_ring;
4100
4101 cpr->cp_ring_struct.ring_mem.flags =
4102 BNXT_RMEM_RING_PTE_FLAG;
4103 }
4104 }
4105
4106 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4107 sizeof(struct bnxt_rx_ring_info),
4108 GFP_KERNEL);
4109 if (!bp->rx_ring)
4110 return -ENOMEM;
4111
4112 for (i = 0; i < bp->rx_nr_rings; i++) {
4113 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4114
4115 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4116 rxr->rx_ring_struct.ring_mem.flags =
4117 BNXT_RMEM_RING_PTE_FLAG;
4118 rxr->rx_agg_ring_struct.ring_mem.flags =
4119 BNXT_RMEM_RING_PTE_FLAG;
4120 }
4121 rxr->bnapi = bp->bnapi[i];
4122 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4123 }
4124
4125 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4126 sizeof(struct bnxt_tx_ring_info),
4127 GFP_KERNEL);
4128 if (!bp->tx_ring)
4129 return -ENOMEM;
4130
4131 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4132 GFP_KERNEL);
4133
4134 if (!bp->tx_ring_map)
4135 return -ENOMEM;
4136
4137 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4138 j = 0;
4139 else
4140 j = bp->rx_nr_rings;
4141
4142 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4143 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4144
4145 if (bp->flags & BNXT_FLAG_CHIP_P5)
4146 txr->tx_ring_struct.ring_mem.flags =
4147 BNXT_RMEM_RING_PTE_FLAG;
4148 txr->bnapi = bp->bnapi[j];
4149 bp->bnapi[j]->tx_ring = txr;
4150 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4151 if (i >= bp->tx_nr_rings_xdp) {
4152 txr->txq_index = i - bp->tx_nr_rings_xdp;
4153 bp->bnapi[j]->tx_int = bnxt_tx_int;
4154 } else {
4155 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4156 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4157 }
4158 }
4159
4160 rc = bnxt_alloc_stats(bp);
4161 if (rc)
4162 goto alloc_mem_err;
4163 bnxt_init_stats(bp);
4164
4165 rc = bnxt_alloc_ntp_fltrs(bp);
4166 if (rc)
4167 goto alloc_mem_err;
4168
4169 rc = bnxt_alloc_vnics(bp);
4170 if (rc)
4171 goto alloc_mem_err;
4172 }
4173
4174 bnxt_init_ring_struct(bp);
4175
4176 rc = bnxt_alloc_rx_rings(bp);
4177 if (rc)
4178 goto alloc_mem_err;
4179
4180 rc = bnxt_alloc_tx_rings(bp);
4181 if (rc)
4182 goto alloc_mem_err;
4183
4184 rc = bnxt_alloc_cp_rings(bp);
4185 if (rc)
4186 goto alloc_mem_err;
4187
4188 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4189 BNXT_VNIC_UCAST_FLAG;
4190 rc = bnxt_alloc_vnic_attributes(bp);
4191 if (rc)
4192 goto alloc_mem_err;
4193 return 0;
4194
4195 alloc_mem_err:
4196 bnxt_free_mem(bp, true);
4197 return rc;
4198 }
4199
4200 static void bnxt_disable_int(struct bnxt *bp)
4201 {
4202 int i;
4203
4204 if (!bp->bnapi)
4205 return;
4206
4207 for (i = 0; i < bp->cp_nr_rings; i++) {
4208 struct bnxt_napi *bnapi = bp->bnapi[i];
4209 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4210 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4211
4212 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4213 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4214 }
4215 }
4216
4217 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4218 {
4219 struct bnxt_napi *bnapi = bp->bnapi[n];
4220 struct bnxt_cp_ring_info *cpr;
4221
4222 cpr = &bnapi->cp_ring;
4223 return cpr->cp_ring_struct.map_idx;
4224 }
4225
4226 static void bnxt_disable_int_sync(struct bnxt *bp)
4227 {
4228 int i;
4229
4230 atomic_inc(&bp->intr_sem);
4231
4232 bnxt_disable_int(bp);
4233 for (i = 0; i < bp->cp_nr_rings; i++) {
4234 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4235
4236 synchronize_irq(bp->irq_tbl[map_idx].vector);
4237 }
4238 }
4239
4240 static void bnxt_enable_int(struct bnxt *bp)
4241 {
4242 int i;
4243
4244 atomic_set(&bp->intr_sem, 0);
4245 for (i = 0; i < bp->cp_nr_rings; i++) {
4246 struct bnxt_napi *bnapi = bp->bnapi[i];
4247 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4248
4249 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4250 }
4251 }
4252
4253 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4254 u16 cmpl_ring, u16 target_id)
4255 {
4256 struct input *req = request;
4257
4258 req->req_type = cpu_to_le16(req_type);
4259 req->cmpl_ring = cpu_to_le16(cmpl_ring);
4260 req->target_id = cpu_to_le16(target_id);
4261 if (bnxt_kong_hwrm_message(bp, req))
4262 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4263 else
4264 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4265 }
4266
4267 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4268 {
4269 switch (hwrm_err) {
4270 case HWRM_ERR_CODE_SUCCESS:
4271 return 0;
4272 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4273 return -EACCES;
4274 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4275 return -ENOSPC;
4276 case HWRM_ERR_CODE_INVALID_PARAMS:
4277 case HWRM_ERR_CODE_INVALID_FLAGS:
4278 case HWRM_ERR_CODE_INVALID_ENABLES:
4279 case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4280 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4281 return -EINVAL;
4282 case HWRM_ERR_CODE_NO_BUFFER:
4283 return -ENOMEM;
4284 case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4285 case HWRM_ERR_CODE_BUSY:
4286 return -EAGAIN;
4287 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4288 return -EOPNOTSUPP;
4289 default:
4290 return -EIO;
4291 }
4292 }
4293
4294 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4295 int timeout, bool silent)
4296 {
4297 int i, intr_process, rc, tmo_count;
4298 struct input *req = msg;
4299 u32 *data = msg;
4300 u8 *valid;
4301 u16 cp_ring_id, len = 0;
4302 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4303 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4304 struct hwrm_short_input short_input = {0};
4305 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4306 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4307 u16 dst = BNXT_HWRM_CHNL_CHIMP;
4308
4309 if (BNXT_NO_FW_ACCESS(bp))
4310 return -EBUSY;
4311
4312 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4313 if (msg_len > bp->hwrm_max_ext_req_len ||
4314 !bp->hwrm_short_cmd_req_addr)
4315 return -EINVAL;
4316 }
4317
4318 if (bnxt_hwrm_kong_chnl(bp, req)) {
4319 dst = BNXT_HWRM_CHNL_KONG;
4320 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4321 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4322 resp = bp->hwrm_cmd_kong_resp_addr;
4323 }
4324
4325 memset(resp, 0, PAGE_SIZE);
4326 cp_ring_id = le16_to_cpu(req->cmpl_ring);
4327 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4328
4329 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4330 /* currently supports only one outstanding message */
4331 if (intr_process)
4332 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4333
4334 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4335 msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4336 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4337 u16 max_msg_len;
4338
4339 /* Set boundary for maximum extended request length for short
4340 * cmd format. If passed up from device use the max supported
4341 * internal req length.
4342 */
4343 max_msg_len = bp->hwrm_max_ext_req_len;
4344
4345 memcpy(short_cmd_req, req, msg_len);
4346 if (msg_len < max_msg_len)
4347 memset(short_cmd_req + msg_len, 0,
4348 max_msg_len - msg_len);
4349
4350 short_input.req_type = req->req_type;
4351 short_input.signature =
4352 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4353 short_input.size = cpu_to_le16(msg_len);
4354 short_input.req_addr =
4355 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4356
4357 data = (u32 *)&short_input;
4358 msg_len = sizeof(short_input);
4359
4360 /* Sync memory write before updating doorbell */
4361 wmb();
4362
4363 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4364 }
4365
4366 /* Write request msg to hwrm channel */
4367 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4368
4369 for (i = msg_len; i < max_req_len; i += 4)
4370 writel(0, bp->bar0 + bar_offset + i);
4371
4372 /* Ring channel doorbell */
4373 writel(1, bp->bar0 + doorbell_offset);
4374
4375 if (!pci_is_enabled(bp->pdev))
4376 return 0;
4377
4378 if (!timeout)
4379 timeout = DFLT_HWRM_CMD_TIMEOUT;
4380 /* convert timeout to usec */
4381 timeout *= 1000;
4382
4383 i = 0;
4384 /* Short timeout for the first few iterations:
4385 * number of loops = number of loops for short timeout +
4386 * number of loops for standard timeout.
4387 */
4388 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4389 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4390 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4391
4392 if (intr_process) {
4393 u16 seq_id = bp->hwrm_intr_seq_id;
4394
4395 /* Wait until hwrm response cmpl interrupt is processed */
4396 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4397 i++ < tmo_count) {
4398 /* Abort the wait for completion if the FW health
4399 * check has failed.
4400 */
4401 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4402 return -EBUSY;
4403 /* on first few passes, just barely sleep */
4404 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4405 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4406 HWRM_SHORT_MAX_TIMEOUT);
4407 else
4408 usleep_range(HWRM_MIN_TIMEOUT,
4409 HWRM_MAX_TIMEOUT);
4410 }
4411
4412 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4413 if (!silent)
4414 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4415 le16_to_cpu(req->req_type));
4416 return -EBUSY;
4417 }
4418 len = le16_to_cpu(resp->resp_len);
4419 valid = ((u8 *)resp) + len - 1;
4420 } else {
4421 int j;
4422
4423 /* Check if response len is updated */
4424 for (i = 0; i < tmo_count; i++) {
4425 /* Abort the wait for completion if the FW health
4426 * check has failed.
4427 */
4428 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4429 return -EBUSY;
4430 len = le16_to_cpu(resp->resp_len);
4431 if (len)
4432 break;
4433 /* on first few passes, just barely sleep */
4434 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4435 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4436 HWRM_SHORT_MAX_TIMEOUT);
4437 else
4438 usleep_range(HWRM_MIN_TIMEOUT,
4439 HWRM_MAX_TIMEOUT);
4440 }
4441
4442 if (i >= tmo_count) {
4443 if (!silent)
4444 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4445 HWRM_TOTAL_TIMEOUT(i),
4446 le16_to_cpu(req->req_type),
4447 le16_to_cpu(req->seq_id), len);
4448 return -EBUSY;
4449 }
4450
4451 /* Last byte of resp contains valid bit */
4452 valid = ((u8 *)resp) + len - 1;
4453 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4454 /* make sure we read from updated DMA memory */
4455 dma_rmb();
4456 if (*valid)
4457 break;
4458 usleep_range(1, 5);
4459 }
4460
4461 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4462 if (!silent)
4463 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4464 HWRM_TOTAL_TIMEOUT(i),
4465 le16_to_cpu(req->req_type),
4466 le16_to_cpu(req->seq_id), len,
4467 *valid);
4468 return -EBUSY;
4469 }
4470 }
4471
4472 /* Zero valid bit for compatibility. Valid bit in an older spec
4473 * may become a new field in a newer spec. We must make sure that
4474 * a new field not implemented by old spec will read zero.
4475 */
4476 *valid = 0;
4477 rc = le16_to_cpu(resp->error_code);
4478 if (rc && !silent)
4479 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4480 le16_to_cpu(resp->req_type),
4481 le16_to_cpu(resp->seq_id), rc);
4482 return bnxt_hwrm_to_stderr(rc);
4483 }
4484
4485 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4486 {
4487 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4488 }
4489
4490 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4491 int timeout)
4492 {
4493 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4494 }
4495
4496 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4497 {
4498 int rc;
4499
4500 mutex_lock(&bp->hwrm_cmd_lock);
4501 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4502 mutex_unlock(&bp->hwrm_cmd_lock);
4503 return rc;
4504 }
4505
4506 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4507 int timeout)
4508 {
4509 int rc;
4510
4511 mutex_lock(&bp->hwrm_cmd_lock);
4512 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4513 mutex_unlock(&bp->hwrm_cmd_lock);
4514 return rc;
4515 }
4516
4517 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4518 bool async_only)
4519 {
4520 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4521 struct hwrm_func_drv_rgtr_input req = {0};
4522 DECLARE_BITMAP(async_events_bmap, 256);
4523 u32 *events = (u32 *)async_events_bmap;
4524 u32 flags;
4525 int rc, i;
4526
4527 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4528
4529 req.enables =
4530 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4531 FUNC_DRV_RGTR_REQ_ENABLES_VER |
4532 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4533
4534 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4535 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4536 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4537 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4538 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4539 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4540 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4541 req.flags = cpu_to_le32(flags);
4542 req.ver_maj_8b = DRV_VER_MAJ;
4543 req.ver_min_8b = DRV_VER_MIN;
4544 req.ver_upd_8b = DRV_VER_UPD;
4545 req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4546 req.ver_min = cpu_to_le16(DRV_VER_MIN);
4547 req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4548
4549 if (BNXT_PF(bp)) {
4550 u32 data[8];
4551 int i;
4552
4553 memset(data, 0, sizeof(data));
4554 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4555 u16 cmd = bnxt_vf_req_snif[i];
4556 unsigned int bit, idx;
4557
4558 idx = cmd / 32;
4559 bit = cmd % 32;
4560 data[idx] |= 1 << bit;
4561 }
4562
4563 for (i = 0; i < 8; i++)
4564 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4565
4566 req.enables |=
4567 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4568 }
4569
4570 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4571 req.flags |= cpu_to_le32(
4572 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4573
4574 memset(async_events_bmap, 0, sizeof(async_events_bmap));
4575 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4576 u16 event_id = bnxt_async_events_arr[i];
4577
4578 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4579 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4580 continue;
4581 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4582 }
4583 if (bmap && bmap_size) {
4584 for (i = 0; i < bmap_size; i++) {
4585 if (test_bit(i, bmap))
4586 __set_bit(i, async_events_bmap);
4587 }
4588 }
4589 for (i = 0; i < 8; i++)
4590 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4591
4592 if (async_only)
4593 req.enables =
4594 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4595
4596 mutex_lock(&bp->hwrm_cmd_lock);
4597 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4598 if (!rc) {
4599 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4600 if (resp->flags &
4601 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4602 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4603 }
4604 mutex_unlock(&bp->hwrm_cmd_lock);
4605 return rc;
4606 }
4607
4608 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4609 {
4610 struct hwrm_func_drv_unrgtr_input req = {0};
4611
4612 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4613 return 0;
4614
4615 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4616 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4617 }
4618
4619 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4620 {
4621 u32 rc = 0;
4622 struct hwrm_tunnel_dst_port_free_input req = {0};
4623
4624 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4625 req.tunnel_type = tunnel_type;
4626
4627 switch (tunnel_type) {
4628 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4629 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4630 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4631 break;
4632 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4633 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4634 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4635 break;
4636 default:
4637 break;
4638 }
4639
4640 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4641 if (rc)
4642 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4643 rc);
4644 return rc;
4645 }
4646
4647 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4648 u8 tunnel_type)
4649 {
4650 u32 rc = 0;
4651 struct hwrm_tunnel_dst_port_alloc_input req = {0};
4652 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4653
4654 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4655
4656 req.tunnel_type = tunnel_type;
4657 req.tunnel_dst_port_val = port;
4658
4659 mutex_lock(&bp->hwrm_cmd_lock);
4660 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4661 if (rc) {
4662 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4663 rc);
4664 goto err_out;
4665 }
4666
4667 switch (tunnel_type) {
4668 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4669 bp->vxlan_fw_dst_port_id =
4670 le16_to_cpu(resp->tunnel_dst_port_id);
4671 break;
4672 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4673 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4674 break;
4675 default:
4676 break;
4677 }
4678
4679 err_out:
4680 mutex_unlock(&bp->hwrm_cmd_lock);
4681 return rc;
4682 }
4683
4684 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4685 {
4686 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4687 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4688
4689 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4690 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4691
4692 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4693 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4694 req.mask = cpu_to_le32(vnic->rx_mask);
4695 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4696 }
4697
4698 #ifdef CONFIG_RFS_ACCEL
4699 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4700 struct bnxt_ntuple_filter *fltr)
4701 {
4702 struct hwrm_cfa_ntuple_filter_free_input req = {0};
4703
4704 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4705 req.ntuple_filter_id = fltr->filter_id;
4706 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4707 }
4708
4709 #define BNXT_NTP_FLTR_FLAGS \
4710 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
4711 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
4712 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
4713 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
4714 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
4715 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
4716 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
4717 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
4718 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
4719 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
4720 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
4721 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
4722 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
4723 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4724
4725 #define BNXT_NTP_TUNNEL_FLTR_FLAG \
4726 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4727
4728 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4729 struct bnxt_ntuple_filter *fltr)
4730 {
4731 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4732 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4733 struct flow_keys *keys = &fltr->fkeys;
4734 struct bnxt_vnic_info *vnic;
4735 u32 flags = 0;
4736 int rc = 0;
4737
4738 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4739 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4740
4741 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4742 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4743 req.dst_id = cpu_to_le16(fltr->rxq);
4744 } else {
4745 vnic = &bp->vnic_info[fltr->rxq + 1];
4746 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4747 }
4748 req.flags = cpu_to_le32(flags);
4749 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4750
4751 req.ethertype = htons(ETH_P_IP);
4752 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4753 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4754 req.ip_protocol = keys->basic.ip_proto;
4755
4756 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4757 int i;
4758
4759 req.ethertype = htons(ETH_P_IPV6);
4760 req.ip_addr_type =
4761 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4762 *(struct in6_addr *)&req.src_ipaddr[0] =
4763 keys->addrs.v6addrs.src;
4764 *(struct in6_addr *)&req.dst_ipaddr[0] =
4765 keys->addrs.v6addrs.dst;
4766 for (i = 0; i < 4; i++) {
4767 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4768 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4769 }
4770 } else {
4771 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4772 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4773 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4774 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4775 }
4776 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4777 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4778 req.tunnel_type =
4779 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4780 }
4781
4782 req.src_port = keys->ports.src;
4783 req.src_port_mask = cpu_to_be16(0xffff);
4784 req.dst_port = keys->ports.dst;
4785 req.dst_port_mask = cpu_to_be16(0xffff);
4786
4787 mutex_lock(&bp->hwrm_cmd_lock);
4788 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4789 if (!rc) {
4790 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4791 fltr->filter_id = resp->ntuple_filter_id;
4792 }
4793 mutex_unlock(&bp->hwrm_cmd_lock);
4794 return rc;
4795 }
4796 #endif
4797
4798 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4799 u8 *mac_addr)
4800 {
4801 u32 rc = 0;
4802 struct hwrm_cfa_l2_filter_alloc_input req = {0};
4803 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4804
4805 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4806 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4807 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4808 req.flags |=
4809 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4810 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4811 req.enables =
4812 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4813 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4814 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4815 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4816 req.l2_addr_mask[0] = 0xff;
4817 req.l2_addr_mask[1] = 0xff;
4818 req.l2_addr_mask[2] = 0xff;
4819 req.l2_addr_mask[3] = 0xff;
4820 req.l2_addr_mask[4] = 0xff;
4821 req.l2_addr_mask[5] = 0xff;
4822
4823 mutex_lock(&bp->hwrm_cmd_lock);
4824 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4825 if (!rc)
4826 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4827 resp->l2_filter_id;
4828 mutex_unlock(&bp->hwrm_cmd_lock);
4829 return rc;
4830 }
4831
4832 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4833 {
4834 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4835 int rc = 0;
4836
4837 /* Any associated ntuple filters will also be cleared by firmware. */
4838 mutex_lock(&bp->hwrm_cmd_lock);
4839 for (i = 0; i < num_of_vnics; i++) {
4840 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4841
4842 for (j = 0; j < vnic->uc_filter_count; j++) {
4843 struct hwrm_cfa_l2_filter_free_input req = {0};
4844
4845 bnxt_hwrm_cmd_hdr_init(bp, &req,
4846 HWRM_CFA_L2_FILTER_FREE, -1, -1);
4847
4848 req.l2_filter_id = vnic->fw_l2_filter_id[j];
4849
4850 rc = _hwrm_send_message(bp, &req, sizeof(req),
4851 HWRM_CMD_TIMEOUT);
4852 }
4853 vnic->uc_filter_count = 0;
4854 }
4855 mutex_unlock(&bp->hwrm_cmd_lock);
4856
4857 return rc;
4858 }
4859
4860 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4861 {
4862 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4863 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4864 struct hwrm_vnic_tpa_cfg_input req = {0};
4865
4866 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4867 return 0;
4868
4869 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4870
4871 if (tpa_flags) {
4872 u16 mss = bp->dev->mtu - 40;
4873 u32 nsegs, n, segs = 0, flags;
4874
4875 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4876 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4877 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4878 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4879 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4880 if (tpa_flags & BNXT_FLAG_GRO)
4881 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4882
4883 req.flags = cpu_to_le32(flags);
4884
4885 req.enables =
4886 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4887 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4888 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4889
4890 /* Number of segs are log2 units, and first packet is not
4891 * included as part of this units.
4892 */
4893 if (mss <= BNXT_RX_PAGE_SIZE) {
4894 n = BNXT_RX_PAGE_SIZE / mss;
4895 nsegs = (MAX_SKB_FRAGS - 1) * n;
4896 } else {
4897 n = mss / BNXT_RX_PAGE_SIZE;
4898 if (mss & (BNXT_RX_PAGE_SIZE - 1))
4899 n++;
4900 nsegs = (MAX_SKB_FRAGS - n) / n;
4901 }
4902
4903 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4904 segs = MAX_TPA_SEGS_P5;
4905 max_aggs = bp->max_tpa;
4906 } else {
4907 segs = ilog2(nsegs);
4908 }
4909 req.max_agg_segs = cpu_to_le16(segs);
4910 req.max_aggs = cpu_to_le16(max_aggs);
4911
4912 req.min_agg_len = cpu_to_le32(512);
4913 }
4914 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4915
4916 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4917 }
4918
4919 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4920 {
4921 struct bnxt_ring_grp_info *grp_info;
4922
4923 grp_info = &bp->grp_info[ring->grp_idx];
4924 return grp_info->cp_fw_ring_id;
4925 }
4926
4927 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4928 {
4929 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4930 struct bnxt_napi *bnapi = rxr->bnapi;
4931 struct bnxt_cp_ring_info *cpr;
4932
4933 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4934 return cpr->cp_ring_struct.fw_ring_id;
4935 } else {
4936 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4937 }
4938 }
4939
4940 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4941 {
4942 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4943 struct bnxt_napi *bnapi = txr->bnapi;
4944 struct bnxt_cp_ring_info *cpr;
4945
4946 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4947 return cpr->cp_ring_struct.fw_ring_id;
4948 } else {
4949 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4950 }
4951 }
4952
4953 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
4954 {
4955 int entries;
4956
4957 if (bp->flags & BNXT_FLAG_CHIP_P5)
4958 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
4959 else
4960 entries = HW_HASH_INDEX_SIZE;
4961
4962 bp->rss_indir_tbl_entries = entries;
4963 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
4964 GFP_KERNEL);
4965 if (!bp->rss_indir_tbl)
4966 return -ENOMEM;
4967 return 0;
4968 }
4969
4970 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
4971 {
4972 u16 max_rings, max_entries, pad, i;
4973
4974 if (!bp->rx_nr_rings)
4975 return;
4976
4977 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4978 max_rings = bp->rx_nr_rings - 1;
4979 else
4980 max_rings = bp->rx_nr_rings;
4981
4982 max_entries = bnxt_get_rxfh_indir_size(bp->dev);
4983
4984 for (i = 0; i < max_entries; i++)
4985 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
4986
4987 pad = bp->rss_indir_tbl_entries - max_entries;
4988 if (pad)
4989 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
4990 }
4991
4992 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
4993 {
4994 u16 i, tbl_size, max_ring = 0;
4995
4996 if (!bp->rss_indir_tbl)
4997 return 0;
4998
4999 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5000 for (i = 0; i < tbl_size; i++)
5001 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5002 return max_ring;
5003 }
5004
5005 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5006 {
5007 if (bp->flags & BNXT_FLAG_CHIP_P5)
5008 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5009 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5010 return 2;
5011 return 1;
5012 }
5013
5014 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5015 {
5016 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5017 u16 i, j;
5018
5019 /* Fill the RSS indirection table with ring group ids */
5020 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5021 if (!no_rss)
5022 j = bp->rss_indir_tbl[i];
5023 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5024 }
5025 }
5026
5027 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5028 struct bnxt_vnic_info *vnic)
5029 {
5030 __le16 *ring_tbl = vnic->rss_table;
5031 struct bnxt_rx_ring_info *rxr;
5032 u16 tbl_size, i;
5033
5034 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5035
5036 for (i = 0; i < tbl_size; i++) {
5037 u16 ring_id, j;
5038
5039 j = bp->rss_indir_tbl[i];
5040 rxr = &bp->rx_ring[j];
5041
5042 ring_id = rxr->rx_ring_struct.fw_ring_id;
5043 *ring_tbl++ = cpu_to_le16(ring_id);
5044 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5045 *ring_tbl++ = cpu_to_le16(ring_id);
5046 }
5047 }
5048
5049 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5050 {
5051 if (bp->flags & BNXT_FLAG_CHIP_P5)
5052 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5053 else
5054 __bnxt_fill_hw_rss_tbl(bp, vnic);
5055 }
5056
5057 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5058 {
5059 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5060 struct hwrm_vnic_rss_cfg_input req = {0};
5061
5062 if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5063 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5064 return 0;
5065
5066 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5067 if (set_rss) {
5068 bnxt_fill_hw_rss_tbl(bp, vnic);
5069 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5070 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5071 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5072 req.hash_key_tbl_addr =
5073 cpu_to_le64(vnic->rss_hash_key_dma_addr);
5074 }
5075 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5076 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5077 }
5078
5079 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5080 {
5081 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5082 struct hwrm_vnic_rss_cfg_input req = {0};
5083 dma_addr_t ring_tbl_map;
5084 u32 i, nr_ctxs;
5085
5086 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5087 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5088 if (!set_rss) {
5089 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5090 return 0;
5091 }
5092 bnxt_fill_hw_rss_tbl(bp, vnic);
5093 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5094 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5095 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5096 ring_tbl_map = vnic->rss_table_dma_addr;
5097 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5098 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5099 int rc;
5100
5101 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5102 req.ring_table_pair_index = i;
5103 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5104 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5105 if (rc)
5106 return rc;
5107 }
5108 return 0;
5109 }
5110
5111 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5112 {
5113 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5114 struct hwrm_vnic_plcmodes_cfg_input req = {0};
5115
5116 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5117 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5118 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5119 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5120 req.enables =
5121 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5122 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5123 /* thresholds not implemented in firmware yet */
5124 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5125 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5126 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5127 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5128 }
5129
5130 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5131 u16 ctx_idx)
5132 {
5133 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5134
5135 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5136 req.rss_cos_lb_ctx_id =
5137 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5138
5139 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5140 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5141 }
5142
5143 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5144 {
5145 int i, j;
5146
5147 for (i = 0; i < bp->nr_vnics; i++) {
5148 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5149
5150 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5151 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5152 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5153 }
5154 }
5155 bp->rsscos_nr_ctxs = 0;
5156 }
5157
5158 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5159 {
5160 int rc;
5161 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5162 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5163 bp->hwrm_cmd_resp_addr;
5164
5165 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5166 -1);
5167
5168 mutex_lock(&bp->hwrm_cmd_lock);
5169 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5170 if (!rc)
5171 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5172 le16_to_cpu(resp->rss_cos_lb_ctx_id);
5173 mutex_unlock(&bp->hwrm_cmd_lock);
5174
5175 return rc;
5176 }
5177
5178 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5179 {
5180 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5181 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5182 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5183 }
5184
5185 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5186 {
5187 unsigned int ring = 0, grp_idx;
5188 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5189 struct hwrm_vnic_cfg_input req = {0};
5190 u16 def_vlan = 0;
5191
5192 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5193
5194 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5195 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5196
5197 req.default_rx_ring_id =
5198 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5199 req.default_cmpl_ring_id =
5200 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5201 req.enables =
5202 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5203 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5204 goto vnic_mru;
5205 }
5206 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5207 /* Only RSS support for now TBD: COS & LB */
5208 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5209 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5210 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5211 VNIC_CFG_REQ_ENABLES_MRU);
5212 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5213 req.rss_rule =
5214 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5215 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5216 VNIC_CFG_REQ_ENABLES_MRU);
5217 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5218 } else {
5219 req.rss_rule = cpu_to_le16(0xffff);
5220 }
5221
5222 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5223 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5224 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5225 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5226 } else {
5227 req.cos_rule = cpu_to_le16(0xffff);
5228 }
5229
5230 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5231 ring = 0;
5232 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5233 ring = vnic_id - 1;
5234 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5235 ring = bp->rx_nr_rings - 1;
5236
5237 grp_idx = bp->rx_ring[ring].bnapi->index;
5238 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5239 req.lb_rule = cpu_to_le16(0xffff);
5240 vnic_mru:
5241 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5242
5243 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5244 #ifdef CONFIG_BNXT_SRIOV
5245 if (BNXT_VF(bp))
5246 def_vlan = bp->vf.vlan;
5247 #endif
5248 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5249 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5250 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5251 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5252
5253 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5254 }
5255
5256 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5257 {
5258 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5259 struct hwrm_vnic_free_input req = {0};
5260
5261 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5262 req.vnic_id =
5263 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5264
5265 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5266 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5267 }
5268 }
5269
5270 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5271 {
5272 u16 i;
5273
5274 for (i = 0; i < bp->nr_vnics; i++)
5275 bnxt_hwrm_vnic_free_one(bp, i);
5276 }
5277
5278 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5279 unsigned int start_rx_ring_idx,
5280 unsigned int nr_rings)
5281 {
5282 int rc = 0;
5283 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5284 struct hwrm_vnic_alloc_input req = {0};
5285 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5286 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5287
5288 if (bp->flags & BNXT_FLAG_CHIP_P5)
5289 goto vnic_no_ring_grps;
5290
5291 /* map ring groups to this vnic */
5292 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5293 grp_idx = bp->rx_ring[i].bnapi->index;
5294 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5295 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5296 j, nr_rings);
5297 break;
5298 }
5299 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5300 }
5301
5302 vnic_no_ring_grps:
5303 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5304 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5305 if (vnic_id == 0)
5306 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5307
5308 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5309
5310 mutex_lock(&bp->hwrm_cmd_lock);
5311 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5312 if (!rc)
5313 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5314 mutex_unlock(&bp->hwrm_cmd_lock);
5315 return rc;
5316 }
5317
5318 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5319 {
5320 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5321 struct hwrm_vnic_qcaps_input req = {0};
5322 int rc;
5323
5324 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5325 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5326 if (bp->hwrm_spec_code < 0x10600)
5327 return 0;
5328
5329 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5330 mutex_lock(&bp->hwrm_cmd_lock);
5331 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5332 if (!rc) {
5333 u32 flags = le32_to_cpu(resp->flags);
5334
5335 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5336 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5337 bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5338 if (flags &
5339 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5340 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5341
5342 /* Older P5 fw before EXT_HW_STATS support did not set
5343 * VLAN_STRIP_CAP properly.
5344 */
5345 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5346 (BNXT_CHIP_P5_THOR(bp) &&
5347 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5348 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5349 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5350 if (bp->max_tpa_v2) {
5351 if (BNXT_CHIP_P5_THOR(bp))
5352 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5353 else
5354 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5355 }
5356 }
5357 mutex_unlock(&bp->hwrm_cmd_lock);
5358 return rc;
5359 }
5360
5361 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5362 {
5363 u16 i;
5364 u32 rc = 0;
5365
5366 if (bp->flags & BNXT_FLAG_CHIP_P5)
5367 return 0;
5368
5369 mutex_lock(&bp->hwrm_cmd_lock);
5370 for (i = 0; i < bp->rx_nr_rings; i++) {
5371 struct hwrm_ring_grp_alloc_input req = {0};
5372 struct hwrm_ring_grp_alloc_output *resp =
5373 bp->hwrm_cmd_resp_addr;
5374 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5375
5376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5377
5378 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5379 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5380 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5381 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5382
5383 rc = _hwrm_send_message(bp, &req, sizeof(req),
5384 HWRM_CMD_TIMEOUT);
5385 if (rc)
5386 break;
5387
5388 bp->grp_info[grp_idx].fw_grp_id =
5389 le32_to_cpu(resp->ring_group_id);
5390 }
5391 mutex_unlock(&bp->hwrm_cmd_lock);
5392 return rc;
5393 }
5394
5395 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5396 {
5397 u16 i;
5398 struct hwrm_ring_grp_free_input req = {0};
5399
5400 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5401 return;
5402
5403 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5404
5405 mutex_lock(&bp->hwrm_cmd_lock);
5406 for (i = 0; i < bp->cp_nr_rings; i++) {
5407 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5408 continue;
5409 req.ring_group_id =
5410 cpu_to_le32(bp->grp_info[i].fw_grp_id);
5411
5412 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5413 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5414 }
5415 mutex_unlock(&bp->hwrm_cmd_lock);
5416 }
5417
5418 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5419 struct bnxt_ring_struct *ring,
5420 u32 ring_type, u32 map_index)
5421 {
5422 int rc = 0, err = 0;
5423 struct hwrm_ring_alloc_input req = {0};
5424 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5425 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5426 struct bnxt_ring_grp_info *grp_info;
5427 u16 ring_id;
5428
5429 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5430
5431 req.enables = 0;
5432 if (rmem->nr_pages > 1) {
5433 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5434 /* Page size is in log2 units */
5435 req.page_size = BNXT_PAGE_SHIFT;
5436 req.page_tbl_depth = 1;
5437 } else {
5438 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
5439 }
5440 req.fbo = 0;
5441 /* Association of ring index with doorbell index and MSIX number */
5442 req.logical_id = cpu_to_le16(map_index);
5443
5444 switch (ring_type) {
5445 case HWRM_RING_ALLOC_TX: {
5446 struct bnxt_tx_ring_info *txr;
5447
5448 txr = container_of(ring, struct bnxt_tx_ring_info,
5449 tx_ring_struct);
5450 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5451 /* Association of transmit ring with completion ring */
5452 grp_info = &bp->grp_info[ring->grp_idx];
5453 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5454 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5455 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5456 req.queue_id = cpu_to_le16(ring->queue_id);
5457 break;
5458 }
5459 case HWRM_RING_ALLOC_RX:
5460 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5461 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5462 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5463 u16 flags = 0;
5464
5465 /* Association of rx ring with stats context */
5466 grp_info = &bp->grp_info[ring->grp_idx];
5467 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5468 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5469 req.enables |= cpu_to_le32(
5470 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5471 if (NET_IP_ALIGN == 2)
5472 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5473 req.flags = cpu_to_le16(flags);
5474 }
5475 break;
5476 case HWRM_RING_ALLOC_AGG:
5477 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5478 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5479 /* Association of agg ring with rx ring */
5480 grp_info = &bp->grp_info[ring->grp_idx];
5481 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5482 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5483 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5484 req.enables |= cpu_to_le32(
5485 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5486 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5487 } else {
5488 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5489 }
5490 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5491 break;
5492 case HWRM_RING_ALLOC_CMPL:
5493 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5494 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5495 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5496 /* Association of cp ring with nq */
5497 grp_info = &bp->grp_info[map_index];
5498 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5499 req.cq_handle = cpu_to_le64(ring->handle);
5500 req.enables |= cpu_to_le32(
5501 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5502 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5503 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5504 }
5505 break;
5506 case HWRM_RING_ALLOC_NQ:
5507 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5508 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5509 if (bp->flags & BNXT_FLAG_USING_MSIX)
5510 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5511 break;
5512 default:
5513 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5514 ring_type);
5515 return -1;
5516 }
5517
5518 mutex_lock(&bp->hwrm_cmd_lock);
5519 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5520 err = le16_to_cpu(resp->error_code);
5521 ring_id = le16_to_cpu(resp->ring_id);
5522 mutex_unlock(&bp->hwrm_cmd_lock);
5523
5524 if (rc || err) {
5525 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5526 ring_type, rc, err);
5527 return -EIO;
5528 }
5529 ring->fw_ring_id = ring_id;
5530 return rc;
5531 }
5532
5533 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5534 {
5535 int rc;
5536
5537 if (BNXT_PF(bp)) {
5538 struct hwrm_func_cfg_input req = {0};
5539
5540 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5541 req.fid = cpu_to_le16(0xffff);
5542 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5543 req.async_event_cr = cpu_to_le16(idx);
5544 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5545 } else {
5546 struct hwrm_func_vf_cfg_input req = {0};
5547
5548 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5549 req.enables =
5550 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5551 req.async_event_cr = cpu_to_le16(idx);
5552 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5553 }
5554 return rc;
5555 }
5556
5557 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5558 u32 map_idx, u32 xid)
5559 {
5560 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5561 if (BNXT_PF(bp))
5562 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5563 else
5564 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5565 switch (ring_type) {
5566 case HWRM_RING_ALLOC_TX:
5567 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5568 break;
5569 case HWRM_RING_ALLOC_RX:
5570 case HWRM_RING_ALLOC_AGG:
5571 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5572 break;
5573 case HWRM_RING_ALLOC_CMPL:
5574 db->db_key64 = DBR_PATH_L2;
5575 break;
5576 case HWRM_RING_ALLOC_NQ:
5577 db->db_key64 = DBR_PATH_L2;
5578 break;
5579 }
5580 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5581 } else {
5582 db->doorbell = bp->bar1 + map_idx * 0x80;
5583 switch (ring_type) {
5584 case HWRM_RING_ALLOC_TX:
5585 db->db_key32 = DB_KEY_TX;
5586 break;
5587 case HWRM_RING_ALLOC_RX:
5588 case HWRM_RING_ALLOC_AGG:
5589 db->db_key32 = DB_KEY_RX;
5590 break;
5591 case HWRM_RING_ALLOC_CMPL:
5592 db->db_key32 = DB_KEY_CP;
5593 break;
5594 }
5595 }
5596 }
5597
5598 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5599 {
5600 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5601 int i, rc = 0;
5602 u32 type;
5603
5604 if (bp->flags & BNXT_FLAG_CHIP_P5)
5605 type = HWRM_RING_ALLOC_NQ;
5606 else
5607 type = HWRM_RING_ALLOC_CMPL;
5608 for (i = 0; i < bp->cp_nr_rings; i++) {
5609 struct bnxt_napi *bnapi = bp->bnapi[i];
5610 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5611 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5612 u32 map_idx = ring->map_idx;
5613 unsigned int vector;
5614
5615 vector = bp->irq_tbl[map_idx].vector;
5616 disable_irq_nosync(vector);
5617 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5618 if (rc) {
5619 enable_irq(vector);
5620 goto err_out;
5621 }
5622 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5623 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5624 enable_irq(vector);
5625 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5626
5627 if (!i) {
5628 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5629 if (rc)
5630 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5631 }
5632 }
5633
5634 type = HWRM_RING_ALLOC_TX;
5635 for (i = 0; i < bp->tx_nr_rings; i++) {
5636 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5637 struct bnxt_ring_struct *ring;
5638 u32 map_idx;
5639
5640 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5641 struct bnxt_napi *bnapi = txr->bnapi;
5642 struct bnxt_cp_ring_info *cpr, *cpr2;
5643 u32 type2 = HWRM_RING_ALLOC_CMPL;
5644
5645 cpr = &bnapi->cp_ring;
5646 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5647 ring = &cpr2->cp_ring_struct;
5648 ring->handle = BNXT_TX_HDL;
5649 map_idx = bnapi->index;
5650 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5651 if (rc)
5652 goto err_out;
5653 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5654 ring->fw_ring_id);
5655 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5656 }
5657 ring = &txr->tx_ring_struct;
5658 map_idx = i;
5659 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5660 if (rc)
5661 goto err_out;
5662 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5663 }
5664
5665 type = HWRM_RING_ALLOC_RX;
5666 for (i = 0; i < bp->rx_nr_rings; i++) {
5667 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5668 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5669 struct bnxt_napi *bnapi = rxr->bnapi;
5670 u32 map_idx = bnapi->index;
5671
5672 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5673 if (rc)
5674 goto err_out;
5675 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5676 /* If we have agg rings, post agg buffers first. */
5677 if (!agg_rings)
5678 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5679 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5680 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5681 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5682 u32 type2 = HWRM_RING_ALLOC_CMPL;
5683 struct bnxt_cp_ring_info *cpr2;
5684
5685 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5686 ring = &cpr2->cp_ring_struct;
5687 ring->handle = BNXT_RX_HDL;
5688 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5689 if (rc)
5690 goto err_out;
5691 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5692 ring->fw_ring_id);
5693 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5694 }
5695 }
5696
5697 if (agg_rings) {
5698 type = HWRM_RING_ALLOC_AGG;
5699 for (i = 0; i < bp->rx_nr_rings; i++) {
5700 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5701 struct bnxt_ring_struct *ring =
5702 &rxr->rx_agg_ring_struct;
5703 u32 grp_idx = ring->grp_idx;
5704 u32 map_idx = grp_idx + bp->rx_nr_rings;
5705
5706 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5707 if (rc)
5708 goto err_out;
5709
5710 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5711 ring->fw_ring_id);
5712 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5713 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5714 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5715 }
5716 }
5717 err_out:
5718 return rc;
5719 }
5720
5721 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5722 struct bnxt_ring_struct *ring,
5723 u32 ring_type, int cmpl_ring_id)
5724 {
5725 int rc;
5726 struct hwrm_ring_free_input req = {0};
5727 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5728 u16 error_code;
5729
5730 if (BNXT_NO_FW_ACCESS(bp))
5731 return 0;
5732
5733 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5734 req.ring_type = ring_type;
5735 req.ring_id = cpu_to_le16(ring->fw_ring_id);
5736
5737 mutex_lock(&bp->hwrm_cmd_lock);
5738 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5739 error_code = le16_to_cpu(resp->error_code);
5740 mutex_unlock(&bp->hwrm_cmd_lock);
5741
5742 if (rc || error_code) {
5743 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5744 ring_type, rc, error_code);
5745 return -EIO;
5746 }
5747 return 0;
5748 }
5749
5750 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5751 {
5752 u32 type;
5753 int i;
5754
5755 if (!bp->bnapi)
5756 return;
5757
5758 for (i = 0; i < bp->tx_nr_rings; i++) {
5759 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5760 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5761
5762 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5763 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5764
5765 hwrm_ring_free_send_msg(bp, ring,
5766 RING_FREE_REQ_RING_TYPE_TX,
5767 close_path ? cmpl_ring_id :
5768 INVALID_HW_RING_ID);
5769 ring->fw_ring_id = INVALID_HW_RING_ID;
5770 }
5771 }
5772
5773 for (i = 0; i < bp->rx_nr_rings; i++) {
5774 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5775 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5776 u32 grp_idx = rxr->bnapi->index;
5777
5778 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5779 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5780
5781 hwrm_ring_free_send_msg(bp, ring,
5782 RING_FREE_REQ_RING_TYPE_RX,
5783 close_path ? cmpl_ring_id :
5784 INVALID_HW_RING_ID);
5785 ring->fw_ring_id = INVALID_HW_RING_ID;
5786 bp->grp_info[grp_idx].rx_fw_ring_id =
5787 INVALID_HW_RING_ID;
5788 }
5789 }
5790
5791 if (bp->flags & BNXT_FLAG_CHIP_P5)
5792 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5793 else
5794 type = RING_FREE_REQ_RING_TYPE_RX;
5795 for (i = 0; i < bp->rx_nr_rings; i++) {
5796 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5797 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5798 u32 grp_idx = rxr->bnapi->index;
5799
5800 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5801 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5802
5803 hwrm_ring_free_send_msg(bp, ring, type,
5804 close_path ? cmpl_ring_id :
5805 INVALID_HW_RING_ID);
5806 ring->fw_ring_id = INVALID_HW_RING_ID;
5807 bp->grp_info[grp_idx].agg_fw_ring_id =
5808 INVALID_HW_RING_ID;
5809 }
5810 }
5811
5812 /* The completion rings are about to be freed. After that the
5813 * IRQ doorbell will not work anymore. So we need to disable
5814 * IRQ here.
5815 */
5816 bnxt_disable_int_sync(bp);
5817
5818 if (bp->flags & BNXT_FLAG_CHIP_P5)
5819 type = RING_FREE_REQ_RING_TYPE_NQ;
5820 else
5821 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5822 for (i = 0; i < bp->cp_nr_rings; i++) {
5823 struct bnxt_napi *bnapi = bp->bnapi[i];
5824 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5825 struct bnxt_ring_struct *ring;
5826 int j;
5827
5828 for (j = 0; j < 2; j++) {
5829 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5830
5831 if (cpr2) {
5832 ring = &cpr2->cp_ring_struct;
5833 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5834 continue;
5835 hwrm_ring_free_send_msg(bp, ring,
5836 RING_FREE_REQ_RING_TYPE_L2_CMPL,
5837 INVALID_HW_RING_ID);
5838 ring->fw_ring_id = INVALID_HW_RING_ID;
5839 }
5840 }
5841 ring = &cpr->cp_ring_struct;
5842 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5843 hwrm_ring_free_send_msg(bp, ring, type,
5844 INVALID_HW_RING_ID);
5845 ring->fw_ring_id = INVALID_HW_RING_ID;
5846 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5847 }
5848 }
5849 }
5850
5851 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5852 bool shared);
5853
5854 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5855 {
5856 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5857 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5858 struct hwrm_func_qcfg_input req = {0};
5859 int rc;
5860
5861 if (bp->hwrm_spec_code < 0x10601)
5862 return 0;
5863
5864 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5865 req.fid = cpu_to_le16(0xffff);
5866 mutex_lock(&bp->hwrm_cmd_lock);
5867 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5868 if (rc) {
5869 mutex_unlock(&bp->hwrm_cmd_lock);
5870 return rc;
5871 }
5872
5873 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5874 if (BNXT_NEW_RM(bp)) {
5875 u16 cp, stats;
5876
5877 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5878 hw_resc->resv_hw_ring_grps =
5879 le32_to_cpu(resp->alloc_hw_ring_grps);
5880 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5881 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5882 stats = le16_to_cpu(resp->alloc_stat_ctx);
5883 hw_resc->resv_irqs = cp;
5884 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5885 int rx = hw_resc->resv_rx_rings;
5886 int tx = hw_resc->resv_tx_rings;
5887
5888 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5889 rx >>= 1;
5890 if (cp < (rx + tx)) {
5891 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5892 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5893 rx <<= 1;
5894 hw_resc->resv_rx_rings = rx;
5895 hw_resc->resv_tx_rings = tx;
5896 }
5897 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5898 hw_resc->resv_hw_ring_grps = rx;
5899 }
5900 hw_resc->resv_cp_rings = cp;
5901 hw_resc->resv_stat_ctxs = stats;
5902 }
5903 mutex_unlock(&bp->hwrm_cmd_lock);
5904 return 0;
5905 }
5906
5907 /* Caller must hold bp->hwrm_cmd_lock */
5908 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5909 {
5910 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5911 struct hwrm_func_qcfg_input req = {0};
5912 int rc;
5913
5914 if (bp->hwrm_spec_code < 0x10601)
5915 return 0;
5916
5917 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5918 req.fid = cpu_to_le16(fid);
5919 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5920 if (!rc)
5921 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5922
5923 return rc;
5924 }
5925
5926 static bool bnxt_rfs_supported(struct bnxt *bp);
5927
5928 static void
5929 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5930 int tx_rings, int rx_rings, int ring_grps,
5931 int cp_rings, int stats, int vnics)
5932 {
5933 u32 enables = 0;
5934
5935 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5936 req->fid = cpu_to_le16(0xffff);
5937 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5938 req->num_tx_rings = cpu_to_le16(tx_rings);
5939 if (BNXT_NEW_RM(bp)) {
5940 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5941 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5942 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5943 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5944 enables |= tx_rings + ring_grps ?
5945 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5946 enables |= rx_rings ?
5947 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5948 } else {
5949 enables |= cp_rings ?
5950 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5951 enables |= ring_grps ?
5952 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5953 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5954 }
5955 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5956
5957 req->num_rx_rings = cpu_to_le16(rx_rings);
5958 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5959 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5960 req->num_msix = cpu_to_le16(cp_rings);
5961 req->num_rsscos_ctxs =
5962 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5963 } else {
5964 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5965 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5966 req->num_rsscos_ctxs = cpu_to_le16(1);
5967 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5968 bnxt_rfs_supported(bp))
5969 req->num_rsscos_ctxs =
5970 cpu_to_le16(ring_grps + 1);
5971 }
5972 req->num_stat_ctxs = cpu_to_le16(stats);
5973 req->num_vnics = cpu_to_le16(vnics);
5974 }
5975 req->enables = cpu_to_le32(enables);
5976 }
5977
5978 static void
5979 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5980 struct hwrm_func_vf_cfg_input *req, int tx_rings,
5981 int rx_rings, int ring_grps, int cp_rings,
5982 int stats, int vnics)
5983 {
5984 u32 enables = 0;
5985
5986 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5987 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5988 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5989 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5990 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5991 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5992 enables |= tx_rings + ring_grps ?
5993 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5994 } else {
5995 enables |= cp_rings ?
5996 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5997 enables |= ring_grps ?
5998 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5999 }
6000 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6001 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6002
6003 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6004 req->num_tx_rings = cpu_to_le16(tx_rings);
6005 req->num_rx_rings = cpu_to_le16(rx_rings);
6006 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6007 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6008 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6009 } else {
6010 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6011 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6012 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6013 }
6014 req->num_stat_ctxs = cpu_to_le16(stats);
6015 req->num_vnics = cpu_to_le16(vnics);
6016
6017 req->enables = cpu_to_le32(enables);
6018 }
6019
6020 static int
6021 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6022 int ring_grps, int cp_rings, int stats, int vnics)
6023 {
6024 struct hwrm_func_cfg_input req = {0};
6025 int rc;
6026
6027 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6028 cp_rings, stats, vnics);
6029 if (!req.enables)
6030 return 0;
6031
6032 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6033 if (rc)
6034 return rc;
6035
6036 if (bp->hwrm_spec_code < 0x10601)
6037 bp->hw_resc.resv_tx_rings = tx_rings;
6038
6039 return bnxt_hwrm_get_rings(bp);
6040 }
6041
6042 static int
6043 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6044 int ring_grps, int cp_rings, int stats, int vnics)
6045 {
6046 struct hwrm_func_vf_cfg_input req = {0};
6047 int rc;
6048
6049 if (!BNXT_NEW_RM(bp)) {
6050 bp->hw_resc.resv_tx_rings = tx_rings;
6051 return 0;
6052 }
6053
6054 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6055 cp_rings, stats, vnics);
6056 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6057 if (rc)
6058 return rc;
6059
6060 return bnxt_hwrm_get_rings(bp);
6061 }
6062
6063 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6064 int cp, int stat, int vnic)
6065 {
6066 if (BNXT_PF(bp))
6067 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6068 vnic);
6069 else
6070 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6071 vnic);
6072 }
6073
6074 int bnxt_nq_rings_in_use(struct bnxt *bp)
6075 {
6076 int cp = bp->cp_nr_rings;
6077 int ulp_msix, ulp_base;
6078
6079 ulp_msix = bnxt_get_ulp_msix_num(bp);
6080 if (ulp_msix) {
6081 ulp_base = bnxt_get_ulp_msix_base(bp);
6082 cp += ulp_msix;
6083 if ((ulp_base + ulp_msix) > cp)
6084 cp = ulp_base + ulp_msix;
6085 }
6086 return cp;
6087 }
6088
6089 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6090 {
6091 int cp;
6092
6093 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6094 return bnxt_nq_rings_in_use(bp);
6095
6096 cp = bp->tx_nr_rings + bp->rx_nr_rings;
6097 return cp;
6098 }
6099
6100 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6101 {
6102 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6103 int cp = bp->cp_nr_rings;
6104
6105 if (!ulp_stat)
6106 return cp;
6107
6108 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6109 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6110
6111 return cp + ulp_stat;
6112 }
6113
6114 /* Check if a default RSS map needs to be setup. This function is only
6115 * used on older firmware that does not require reserving RX rings.
6116 */
6117 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6118 {
6119 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6120
6121 /* The RSS map is valid for RX rings set to resv_rx_rings */
6122 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6123 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6124 if (!netif_is_rxfh_configured(bp->dev))
6125 bnxt_set_dflt_rss_indir_tbl(bp);
6126 }
6127 }
6128
6129 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6130 {
6131 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6132 int cp = bnxt_cp_rings_in_use(bp);
6133 int nq = bnxt_nq_rings_in_use(bp);
6134 int rx = bp->rx_nr_rings, stat;
6135 int vnic = 1, grp = rx;
6136
6137 if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6138 bp->hwrm_spec_code >= 0x10601)
6139 return true;
6140
6141 /* Old firmware does not need RX ring reservations but we still
6142 * need to setup a default RSS map when needed. With new firmware
6143 * we go through RX ring reservations first and then set up the
6144 * RSS map for the successfully reserved RX rings when needed.
6145 */
6146 if (!BNXT_NEW_RM(bp)) {
6147 bnxt_check_rss_tbl_no_rmgr(bp);
6148 return false;
6149 }
6150 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6151 vnic = rx + 1;
6152 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6153 rx <<= 1;
6154 stat = bnxt_get_func_stat_ctxs(bp);
6155 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6156 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6157 (hw_resc->resv_hw_ring_grps != grp &&
6158 !(bp->flags & BNXT_FLAG_CHIP_P5)))
6159 return true;
6160 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6161 hw_resc->resv_irqs != nq)
6162 return true;
6163 return false;
6164 }
6165
6166 static int __bnxt_reserve_rings(struct bnxt *bp)
6167 {
6168 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6169 int cp = bnxt_nq_rings_in_use(bp);
6170 int tx = bp->tx_nr_rings;
6171 int rx = bp->rx_nr_rings;
6172 int grp, rx_rings, rc;
6173 int vnic = 1, stat;
6174 bool sh = false;
6175
6176 if (!bnxt_need_reserve_rings(bp))
6177 return 0;
6178
6179 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6180 sh = true;
6181 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6182 vnic = rx + 1;
6183 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6184 rx <<= 1;
6185 grp = bp->rx_nr_rings;
6186 stat = bnxt_get_func_stat_ctxs(bp);
6187
6188 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6189 if (rc)
6190 return rc;
6191
6192 tx = hw_resc->resv_tx_rings;
6193 if (BNXT_NEW_RM(bp)) {
6194 rx = hw_resc->resv_rx_rings;
6195 cp = hw_resc->resv_irqs;
6196 grp = hw_resc->resv_hw_ring_grps;
6197 vnic = hw_resc->resv_vnics;
6198 stat = hw_resc->resv_stat_ctxs;
6199 }
6200
6201 rx_rings = rx;
6202 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6203 if (rx >= 2) {
6204 rx_rings = rx >> 1;
6205 } else {
6206 if (netif_running(bp->dev))
6207 return -ENOMEM;
6208
6209 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6210 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6211 bp->dev->hw_features &= ~NETIF_F_LRO;
6212 bp->dev->features &= ~NETIF_F_LRO;
6213 bnxt_set_ring_params(bp);
6214 }
6215 }
6216 rx_rings = min_t(int, rx_rings, grp);
6217 cp = min_t(int, cp, bp->cp_nr_rings);
6218 if (stat > bnxt_get_ulp_stat_ctxs(bp))
6219 stat -= bnxt_get_ulp_stat_ctxs(bp);
6220 cp = min_t(int, cp, stat);
6221 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6222 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6223 rx = rx_rings << 1;
6224 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6225 bp->tx_nr_rings = tx;
6226
6227 /* If we cannot reserve all the RX rings, reset the RSS map only
6228 * if absolutely necessary
6229 */
6230 if (rx_rings != bp->rx_nr_rings) {
6231 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6232 rx_rings, bp->rx_nr_rings);
6233 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6234 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6235 bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6236 bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6237 netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6238 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6239 }
6240 }
6241 bp->rx_nr_rings = rx_rings;
6242 bp->cp_nr_rings = cp;
6243
6244 if (!tx || !rx || !cp || !grp || !vnic || !stat)
6245 return -ENOMEM;
6246
6247 if (!netif_is_rxfh_configured(bp->dev))
6248 bnxt_set_dflt_rss_indir_tbl(bp);
6249
6250 return rc;
6251 }
6252
6253 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6254 int ring_grps, int cp_rings, int stats,
6255 int vnics)
6256 {
6257 struct hwrm_func_vf_cfg_input req = {0};
6258 u32 flags;
6259
6260 if (!BNXT_NEW_RM(bp))
6261 return 0;
6262
6263 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6264 cp_rings, stats, vnics);
6265 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6266 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6267 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6268 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6269 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6270 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6271 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6272 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6273
6274 req.flags = cpu_to_le32(flags);
6275 return hwrm_send_message_silent(bp, &req, sizeof(req),
6276 HWRM_CMD_TIMEOUT);
6277 }
6278
6279 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6280 int ring_grps, int cp_rings, int stats,
6281 int vnics)
6282 {
6283 struct hwrm_func_cfg_input req = {0};
6284 u32 flags;
6285
6286 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6287 cp_rings, stats, vnics);
6288 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6289 if (BNXT_NEW_RM(bp)) {
6290 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6291 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6292 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6293 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6294 if (bp->flags & BNXT_FLAG_CHIP_P5)
6295 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6296 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6297 else
6298 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6299 }
6300
6301 req.flags = cpu_to_le32(flags);
6302 return hwrm_send_message_silent(bp, &req, sizeof(req),
6303 HWRM_CMD_TIMEOUT);
6304 }
6305
6306 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6307 int ring_grps, int cp_rings, int stats,
6308 int vnics)
6309 {
6310 if (bp->hwrm_spec_code < 0x10801)
6311 return 0;
6312
6313 if (BNXT_PF(bp))
6314 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6315 ring_grps, cp_rings, stats,
6316 vnics);
6317
6318 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6319 cp_rings, stats, vnics);
6320 }
6321
6322 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6323 {
6324 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6325 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6326 struct hwrm_ring_aggint_qcaps_input req = {0};
6327 int rc;
6328
6329 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6330 coal_cap->num_cmpl_dma_aggr_max = 63;
6331 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6332 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6333 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6334 coal_cap->int_lat_tmr_min_max = 65535;
6335 coal_cap->int_lat_tmr_max_max = 65535;
6336 coal_cap->num_cmpl_aggr_int_max = 65535;
6337 coal_cap->timer_units = 80;
6338
6339 if (bp->hwrm_spec_code < 0x10902)
6340 return;
6341
6342 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6343 mutex_lock(&bp->hwrm_cmd_lock);
6344 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6345 if (!rc) {
6346 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6347 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6348 coal_cap->num_cmpl_dma_aggr_max =
6349 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6350 coal_cap->num_cmpl_dma_aggr_during_int_max =
6351 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6352 coal_cap->cmpl_aggr_dma_tmr_max =
6353 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6354 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6355 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6356 coal_cap->int_lat_tmr_min_max =
6357 le16_to_cpu(resp->int_lat_tmr_min_max);
6358 coal_cap->int_lat_tmr_max_max =
6359 le16_to_cpu(resp->int_lat_tmr_max_max);
6360 coal_cap->num_cmpl_aggr_int_max =
6361 le16_to_cpu(resp->num_cmpl_aggr_int_max);
6362 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6363 }
6364 mutex_unlock(&bp->hwrm_cmd_lock);
6365 }
6366
6367 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6368 {
6369 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6370
6371 return usec * 1000 / coal_cap->timer_units;
6372 }
6373
6374 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6375 struct bnxt_coal *hw_coal,
6376 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6377 {
6378 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6379 u32 cmpl_params = coal_cap->cmpl_params;
6380 u16 val, tmr, max, flags = 0;
6381
6382 max = hw_coal->bufs_per_record * 128;
6383 if (hw_coal->budget)
6384 max = hw_coal->bufs_per_record * hw_coal->budget;
6385 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6386
6387 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6388 req->num_cmpl_aggr_int = cpu_to_le16(val);
6389
6390 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6391 req->num_cmpl_dma_aggr = cpu_to_le16(val);
6392
6393 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6394 coal_cap->num_cmpl_dma_aggr_during_int_max);
6395 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6396
6397 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6398 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6399 req->int_lat_tmr_max = cpu_to_le16(tmr);
6400
6401 /* min timer set to 1/2 of interrupt timer */
6402 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6403 val = tmr / 2;
6404 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6405 req->int_lat_tmr_min = cpu_to_le16(val);
6406 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6407 }
6408
6409 /* buf timer set to 1/4 of interrupt timer */
6410 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6411 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6412
6413 if (cmpl_params &
6414 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6415 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6416 val = clamp_t(u16, tmr, 1,
6417 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6418 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6419 req->enables |=
6420 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6421 }
6422
6423 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6424 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6425 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6426 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6427 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6428 req->flags = cpu_to_le16(flags);
6429 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6430 }
6431
6432 /* Caller holds bp->hwrm_cmd_lock */
6433 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6434 struct bnxt_coal *hw_coal)
6435 {
6436 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6437 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6438 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6439 u32 nq_params = coal_cap->nq_params;
6440 u16 tmr;
6441
6442 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6443 return 0;
6444
6445 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6446 -1, -1);
6447 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6448 req.flags =
6449 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6450
6451 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6452 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6453 req.int_lat_tmr_min = cpu_to_le16(tmr);
6454 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6455 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6456 }
6457
6458 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6459 {
6460 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6461 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6462 struct bnxt_coal coal;
6463
6464 /* Tick values in micro seconds.
6465 * 1 coal_buf x bufs_per_record = 1 completion record.
6466 */
6467 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6468
6469 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6470 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6471
6472 if (!bnapi->rx_ring)
6473 return -ENODEV;
6474
6475 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6476 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6477
6478 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6479
6480 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6481
6482 return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6483 HWRM_CMD_TIMEOUT);
6484 }
6485
6486 int bnxt_hwrm_set_coal(struct bnxt *bp)
6487 {
6488 int i, rc = 0;
6489 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6490 req_tx = {0}, *req;
6491
6492 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6493 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6494 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6495 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6496
6497 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6498 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6499
6500 mutex_lock(&bp->hwrm_cmd_lock);
6501 for (i = 0; i < bp->cp_nr_rings; i++) {
6502 struct bnxt_napi *bnapi = bp->bnapi[i];
6503 struct bnxt_coal *hw_coal;
6504 u16 ring_id;
6505
6506 req = &req_rx;
6507 if (!bnapi->rx_ring) {
6508 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6509 req = &req_tx;
6510 } else {
6511 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6512 }
6513 req->ring_id = cpu_to_le16(ring_id);
6514
6515 rc = _hwrm_send_message(bp, req, sizeof(*req),
6516 HWRM_CMD_TIMEOUT);
6517 if (rc)
6518 break;
6519
6520 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6521 continue;
6522
6523 if (bnapi->rx_ring && bnapi->tx_ring) {
6524 req = &req_tx;
6525 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6526 req->ring_id = cpu_to_le16(ring_id);
6527 rc = _hwrm_send_message(bp, req, sizeof(*req),
6528 HWRM_CMD_TIMEOUT);
6529 if (rc)
6530 break;
6531 }
6532 if (bnapi->rx_ring)
6533 hw_coal = &bp->rx_coal;
6534 else
6535 hw_coal = &bp->tx_coal;
6536 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6537 }
6538 mutex_unlock(&bp->hwrm_cmd_lock);
6539 return rc;
6540 }
6541
6542 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6543 {
6544 struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6545 struct hwrm_stat_ctx_free_input req = {0};
6546 int i;
6547
6548 if (!bp->bnapi)
6549 return;
6550
6551 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6552 return;
6553
6554 bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6555 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6556
6557 mutex_lock(&bp->hwrm_cmd_lock);
6558 for (i = 0; i < bp->cp_nr_rings; i++) {
6559 struct bnxt_napi *bnapi = bp->bnapi[i];
6560 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6561
6562 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6563 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6564 if (BNXT_FW_MAJ(bp) <= 20) {
6565 req0.stat_ctx_id = req.stat_ctx_id;
6566 _hwrm_send_message(bp, &req0, sizeof(req0),
6567 HWRM_CMD_TIMEOUT);
6568 }
6569 _hwrm_send_message(bp, &req, sizeof(req),
6570 HWRM_CMD_TIMEOUT);
6571
6572 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6573 }
6574 }
6575 mutex_unlock(&bp->hwrm_cmd_lock);
6576 }
6577
6578 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6579 {
6580 int rc = 0, i;
6581 struct hwrm_stat_ctx_alloc_input req = {0};
6582 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6583
6584 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6585 return 0;
6586
6587 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6588
6589 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6590 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6591
6592 mutex_lock(&bp->hwrm_cmd_lock);
6593 for (i = 0; i < bp->cp_nr_rings; i++) {
6594 struct bnxt_napi *bnapi = bp->bnapi[i];
6595 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6596
6597 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6598
6599 rc = _hwrm_send_message(bp, &req, sizeof(req),
6600 HWRM_CMD_TIMEOUT);
6601 if (rc)
6602 break;
6603
6604 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6605
6606 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6607 }
6608 mutex_unlock(&bp->hwrm_cmd_lock);
6609 return rc;
6610 }
6611
6612 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6613 {
6614 struct hwrm_func_qcfg_input req = {0};
6615 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6616 u32 min_db_offset = 0;
6617 u16 flags;
6618 int rc;
6619
6620 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6621 req.fid = cpu_to_le16(0xffff);
6622 mutex_lock(&bp->hwrm_cmd_lock);
6623 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6624 if (rc)
6625 goto func_qcfg_exit;
6626
6627 #ifdef CONFIG_BNXT_SRIOV
6628 if (BNXT_VF(bp)) {
6629 struct bnxt_vf_info *vf = &bp->vf;
6630
6631 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6632 } else {
6633 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6634 }
6635 #endif
6636 flags = le16_to_cpu(resp->flags);
6637 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6638 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6639 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6640 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6641 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6642 }
6643 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6644 bp->flags |= BNXT_FLAG_MULTI_HOST;
6645
6646 switch (resp->port_partition_type) {
6647 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6648 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6649 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6650 bp->port_partition_type = resp->port_partition_type;
6651 break;
6652 }
6653 if (bp->hwrm_spec_code < 0x10707 ||
6654 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6655 bp->br_mode = BRIDGE_MODE_VEB;
6656 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6657 bp->br_mode = BRIDGE_MODE_VEPA;
6658 else
6659 bp->br_mode = BRIDGE_MODE_UNDEF;
6660
6661 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6662 if (!bp->max_mtu)
6663 bp->max_mtu = BNXT_MAX_MTU;
6664
6665 if (bp->db_size)
6666 goto func_qcfg_exit;
6667
6668 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6669 if (BNXT_PF(bp))
6670 min_db_offset = DB_PF_OFFSET_P5;
6671 else
6672 min_db_offset = DB_VF_OFFSET_P5;
6673 }
6674 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6675 1024);
6676 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6677 bp->db_size <= min_db_offset)
6678 bp->db_size = pci_resource_len(bp->pdev, 2);
6679
6680 func_qcfg_exit:
6681 mutex_unlock(&bp->hwrm_cmd_lock);
6682 return rc;
6683 }
6684
6685 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6686 {
6687 struct hwrm_func_backing_store_qcaps_input req = {0};
6688 struct hwrm_func_backing_store_qcaps_output *resp =
6689 bp->hwrm_cmd_resp_addr;
6690 int rc;
6691
6692 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6693 return 0;
6694
6695 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6696 mutex_lock(&bp->hwrm_cmd_lock);
6697 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6698 if (!rc) {
6699 struct bnxt_ctx_pg_info *ctx_pg;
6700 struct bnxt_ctx_mem_info *ctx;
6701 int i, tqm_rings;
6702
6703 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6704 if (!ctx) {
6705 rc = -ENOMEM;
6706 goto ctx_err;
6707 }
6708 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6709 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6710 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6711 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6712 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6713 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6714 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6715 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6716 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6717 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6718 ctx->vnic_max_vnic_entries =
6719 le16_to_cpu(resp->vnic_max_vnic_entries);
6720 ctx->vnic_max_ring_table_entries =
6721 le16_to_cpu(resp->vnic_max_ring_table_entries);
6722 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6723 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6724 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6725 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6726 ctx->tqm_min_entries_per_ring =
6727 le32_to_cpu(resp->tqm_min_entries_per_ring);
6728 ctx->tqm_max_entries_per_ring =
6729 le32_to_cpu(resp->tqm_max_entries_per_ring);
6730 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6731 if (!ctx->tqm_entries_multiple)
6732 ctx->tqm_entries_multiple = 1;
6733 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6734 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6735 ctx->mrav_num_entries_units =
6736 le16_to_cpu(resp->mrav_num_entries_units);
6737 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6738 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6739 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6740 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6741 if (!ctx->tqm_fp_rings_count)
6742 ctx->tqm_fp_rings_count = bp->max_q;
6743
6744 tqm_rings = ctx->tqm_fp_rings_count + 1;
6745 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6746 if (!ctx_pg) {
6747 kfree(ctx);
6748 rc = -ENOMEM;
6749 goto ctx_err;
6750 }
6751 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6752 ctx->tqm_mem[i] = ctx_pg;
6753 bp->ctx = ctx;
6754 } else {
6755 rc = 0;
6756 }
6757 ctx_err:
6758 mutex_unlock(&bp->hwrm_cmd_lock);
6759 return rc;
6760 }
6761
6762 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6763 __le64 *pg_dir)
6764 {
6765 u8 pg_size = 0;
6766
6767 if (BNXT_PAGE_SHIFT == 13)
6768 pg_size = 1 << 4;
6769 else if (BNXT_PAGE_SIZE == 16)
6770 pg_size = 2 << 4;
6771
6772 *pg_attr = pg_size;
6773 if (rmem->depth >= 1) {
6774 if (rmem->depth == 2)
6775 *pg_attr |= 2;
6776 else
6777 *pg_attr |= 1;
6778 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6779 } else {
6780 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6781 }
6782 }
6783
6784 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
6785 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
6786 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
6787 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
6788 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
6789 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6790
6791 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6792 {
6793 struct hwrm_func_backing_store_cfg_input req = {0};
6794 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6795 struct bnxt_ctx_pg_info *ctx_pg;
6796 __le32 *num_entries;
6797 __le64 *pg_dir;
6798 u32 flags = 0;
6799 u8 *pg_attr;
6800 u32 ena;
6801 int i;
6802
6803 if (!ctx)
6804 return 0;
6805
6806 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6807 req.enables = cpu_to_le32(enables);
6808
6809 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6810 ctx_pg = &ctx->qp_mem;
6811 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6812 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6813 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6814 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6815 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6816 &req.qpc_pg_size_qpc_lvl,
6817 &req.qpc_page_dir);
6818 }
6819 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6820 ctx_pg = &ctx->srq_mem;
6821 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6822 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6823 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6824 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6825 &req.srq_pg_size_srq_lvl,
6826 &req.srq_page_dir);
6827 }
6828 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6829 ctx_pg = &ctx->cq_mem;
6830 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6831 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6832 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6833 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6834 &req.cq_page_dir);
6835 }
6836 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6837 ctx_pg = &ctx->vnic_mem;
6838 req.vnic_num_vnic_entries =
6839 cpu_to_le16(ctx->vnic_max_vnic_entries);
6840 req.vnic_num_ring_table_entries =
6841 cpu_to_le16(ctx->vnic_max_ring_table_entries);
6842 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6843 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6844 &req.vnic_pg_size_vnic_lvl,
6845 &req.vnic_page_dir);
6846 }
6847 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6848 ctx_pg = &ctx->stat_mem;
6849 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6850 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6851 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6852 &req.stat_pg_size_stat_lvl,
6853 &req.stat_page_dir);
6854 }
6855 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6856 ctx_pg = &ctx->mrav_mem;
6857 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6858 if (ctx->mrav_num_entries_units)
6859 flags |=
6860 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6861 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6862 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6863 &req.mrav_pg_size_mrav_lvl,
6864 &req.mrav_page_dir);
6865 }
6866 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6867 ctx_pg = &ctx->tim_mem;
6868 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6869 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6870 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6871 &req.tim_pg_size_tim_lvl,
6872 &req.tim_page_dir);
6873 }
6874 for (i = 0, num_entries = &req.tqm_sp_num_entries,
6875 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6876 pg_dir = &req.tqm_sp_page_dir,
6877 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6878 i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6879 if (!(enables & ena))
6880 continue;
6881
6882 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6883 ctx_pg = ctx->tqm_mem[i];
6884 *num_entries = cpu_to_le32(ctx_pg->entries);
6885 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6886 }
6887 req.flags = cpu_to_le32(flags);
6888 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6889 }
6890
6891 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6892 struct bnxt_ctx_pg_info *ctx_pg)
6893 {
6894 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6895
6896 rmem->page_size = BNXT_PAGE_SIZE;
6897 rmem->pg_arr = ctx_pg->ctx_pg_arr;
6898 rmem->dma_arr = ctx_pg->ctx_dma_arr;
6899 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6900 if (rmem->depth >= 1)
6901 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6902 return bnxt_alloc_ring(bp, rmem);
6903 }
6904
6905 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6906 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6907 u8 depth, bool use_init_val)
6908 {
6909 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6910 int rc;
6911
6912 if (!mem_size)
6913 return -EINVAL;
6914
6915 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6916 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6917 ctx_pg->nr_pages = 0;
6918 return -EINVAL;
6919 }
6920 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6921 int nr_tbls, i;
6922
6923 rmem->depth = 2;
6924 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6925 GFP_KERNEL);
6926 if (!ctx_pg->ctx_pg_tbl)
6927 return -ENOMEM;
6928 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6929 rmem->nr_pages = nr_tbls;
6930 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6931 if (rc)
6932 return rc;
6933 for (i = 0; i < nr_tbls; i++) {
6934 struct bnxt_ctx_pg_info *pg_tbl;
6935
6936 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6937 if (!pg_tbl)
6938 return -ENOMEM;
6939 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6940 rmem = &pg_tbl->ring_mem;
6941 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6942 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6943 rmem->depth = 1;
6944 rmem->nr_pages = MAX_CTX_PAGES;
6945 if (use_init_val)
6946 rmem->init_val = bp->ctx->ctx_kind_initializer;
6947 if (i == (nr_tbls - 1)) {
6948 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6949
6950 if (rem)
6951 rmem->nr_pages = rem;
6952 }
6953 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6954 if (rc)
6955 break;
6956 }
6957 } else {
6958 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6959 if (rmem->nr_pages > 1 || depth)
6960 rmem->depth = 1;
6961 if (use_init_val)
6962 rmem->init_val = bp->ctx->ctx_kind_initializer;
6963 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6964 }
6965 return rc;
6966 }
6967
6968 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6969 struct bnxt_ctx_pg_info *ctx_pg)
6970 {
6971 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6972
6973 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6974 ctx_pg->ctx_pg_tbl) {
6975 int i, nr_tbls = rmem->nr_pages;
6976
6977 for (i = 0; i < nr_tbls; i++) {
6978 struct bnxt_ctx_pg_info *pg_tbl;
6979 struct bnxt_ring_mem_info *rmem2;
6980
6981 pg_tbl = ctx_pg->ctx_pg_tbl[i];
6982 if (!pg_tbl)
6983 continue;
6984 rmem2 = &pg_tbl->ring_mem;
6985 bnxt_free_ring(bp, rmem2);
6986 ctx_pg->ctx_pg_arr[i] = NULL;
6987 kfree(pg_tbl);
6988 ctx_pg->ctx_pg_tbl[i] = NULL;
6989 }
6990 kfree(ctx_pg->ctx_pg_tbl);
6991 ctx_pg->ctx_pg_tbl = NULL;
6992 }
6993 bnxt_free_ring(bp, rmem);
6994 ctx_pg->nr_pages = 0;
6995 }
6996
6997 static void bnxt_free_ctx_mem(struct bnxt *bp)
6998 {
6999 struct bnxt_ctx_mem_info *ctx = bp->ctx;
7000 int i;
7001
7002 if (!ctx)
7003 return;
7004
7005 if (ctx->tqm_mem[0]) {
7006 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7007 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7008 kfree(ctx->tqm_mem[0]);
7009 ctx->tqm_mem[0] = NULL;
7010 }
7011
7012 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7013 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7014 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7015 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7016 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7017 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7018 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7019 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7020 }
7021
7022 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7023 {
7024 struct bnxt_ctx_pg_info *ctx_pg;
7025 struct bnxt_ctx_mem_info *ctx;
7026 u32 mem_size, ena, entries;
7027 u32 entries_sp, min;
7028 u32 num_mr, num_ah;
7029 u32 extra_srqs = 0;
7030 u32 extra_qps = 0;
7031 u8 pg_lvl = 1;
7032 int i, rc;
7033
7034 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7035 if (rc) {
7036 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7037 rc);
7038 return rc;
7039 }
7040 ctx = bp->ctx;
7041 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7042 return 0;
7043
7044 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7045 pg_lvl = 2;
7046 extra_qps = 65536;
7047 extra_srqs = 8192;
7048 }
7049
7050 ctx_pg = &ctx->qp_mem;
7051 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7052 extra_qps;
7053 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7054 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7055 if (rc)
7056 return rc;
7057
7058 ctx_pg = &ctx->srq_mem;
7059 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7060 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7061 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7062 if (rc)
7063 return rc;
7064
7065 ctx_pg = &ctx->cq_mem;
7066 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7067 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7068 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7069 if (rc)
7070 return rc;
7071
7072 ctx_pg = &ctx->vnic_mem;
7073 ctx_pg->entries = ctx->vnic_max_vnic_entries +
7074 ctx->vnic_max_ring_table_entries;
7075 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7076 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7077 if (rc)
7078 return rc;
7079
7080 ctx_pg = &ctx->stat_mem;
7081 ctx_pg->entries = ctx->stat_max_entries;
7082 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7083 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7084 if (rc)
7085 return rc;
7086
7087 ena = 0;
7088 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7089 goto skip_rdma;
7090
7091 ctx_pg = &ctx->mrav_mem;
7092 /* 128K extra is needed to accommodate static AH context
7093 * allocation by f/w.
7094 */
7095 num_mr = 1024 * 256;
7096 num_ah = 1024 * 128;
7097 ctx_pg->entries = num_mr + num_ah;
7098 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7099 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7100 if (rc)
7101 return rc;
7102 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7103 if (ctx->mrav_num_entries_units)
7104 ctx_pg->entries =
7105 ((num_mr / ctx->mrav_num_entries_units) << 16) |
7106 (num_ah / ctx->mrav_num_entries_units);
7107
7108 ctx_pg = &ctx->tim_mem;
7109 ctx_pg->entries = ctx->qp_mem.entries;
7110 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7111 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7112 if (rc)
7113 return rc;
7114 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7115
7116 skip_rdma:
7117 min = ctx->tqm_min_entries_per_ring;
7118 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7119 2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7120 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7121 entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7122 entries = roundup(entries, ctx->tqm_entries_multiple);
7123 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7124 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7125 ctx_pg = ctx->tqm_mem[i];
7126 ctx_pg->entries = i ? entries : entries_sp;
7127 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7128 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7129 if (rc)
7130 return rc;
7131 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7132 }
7133 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7134 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7135 if (rc) {
7136 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7137 rc);
7138 return rc;
7139 }
7140 ctx->flags |= BNXT_CTX_FLAG_INITED;
7141 return 0;
7142 }
7143
7144 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7145 {
7146 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7147 struct hwrm_func_resource_qcaps_input req = {0};
7148 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7149 int rc;
7150
7151 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7152 req.fid = cpu_to_le16(0xffff);
7153
7154 mutex_lock(&bp->hwrm_cmd_lock);
7155 rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7156 HWRM_CMD_TIMEOUT);
7157 if (rc)
7158 goto hwrm_func_resc_qcaps_exit;
7159
7160 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7161 if (!all)
7162 goto hwrm_func_resc_qcaps_exit;
7163
7164 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7165 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7166 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7167 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7168 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7169 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7170 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7171 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7172 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7173 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7174 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7175 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7176 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7177 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7178 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7179 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7180
7181 if (bp->flags & BNXT_FLAG_CHIP_P5) {
7182 u16 max_msix = le16_to_cpu(resp->max_msix);
7183
7184 hw_resc->max_nqs = max_msix;
7185 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7186 }
7187
7188 if (BNXT_PF(bp)) {
7189 struct bnxt_pf_info *pf = &bp->pf;
7190
7191 pf->vf_resv_strategy =
7192 le16_to_cpu(resp->vf_reservation_strategy);
7193 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7194 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7195 }
7196 hwrm_func_resc_qcaps_exit:
7197 mutex_unlock(&bp->hwrm_cmd_lock);
7198 return rc;
7199 }
7200
7201 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7202 {
7203 int rc = 0;
7204 struct hwrm_func_qcaps_input req = {0};
7205 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7206 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7207 u32 flags, flags_ext;
7208
7209 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7210 req.fid = cpu_to_le16(0xffff);
7211
7212 mutex_lock(&bp->hwrm_cmd_lock);
7213 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7214 if (rc)
7215 goto hwrm_func_qcaps_exit;
7216
7217 flags = le32_to_cpu(resp->flags);
7218 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7219 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7220 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7221 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7222 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7223 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7224 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7225 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7226 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7227 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7228 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7229 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7230 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7231 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7232 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7233 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7234
7235 flags_ext = le32_to_cpu(resp->flags_ext);
7236 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7237 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7238
7239 bp->tx_push_thresh = 0;
7240 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7241 BNXT_FW_MAJ(bp) > 217)
7242 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7243
7244 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7245 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7246 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7247 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7248 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7249 if (!hw_resc->max_hw_ring_grps)
7250 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7251 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7252 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7253 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7254
7255 if (BNXT_PF(bp)) {
7256 struct bnxt_pf_info *pf = &bp->pf;
7257
7258 pf->fw_fid = le16_to_cpu(resp->fid);
7259 pf->port_id = le16_to_cpu(resp->port_id);
7260 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7261 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7262 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7263 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7264 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7265 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7266 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7267 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7268 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7269 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7270 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7271 bp->flags |= BNXT_FLAG_WOL_CAP;
7272 } else {
7273 #ifdef CONFIG_BNXT_SRIOV
7274 struct bnxt_vf_info *vf = &bp->vf;
7275
7276 vf->fw_fid = le16_to_cpu(resp->fid);
7277 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7278 #endif
7279 }
7280
7281 hwrm_func_qcaps_exit:
7282 mutex_unlock(&bp->hwrm_cmd_lock);
7283 return rc;
7284 }
7285
7286 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7287
7288 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7289 {
7290 int rc;
7291
7292 rc = __bnxt_hwrm_func_qcaps(bp);
7293 if (rc)
7294 return rc;
7295 rc = bnxt_hwrm_queue_qportcfg(bp);
7296 if (rc) {
7297 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7298 return rc;
7299 }
7300 if (bp->hwrm_spec_code >= 0x10803) {
7301 rc = bnxt_alloc_ctx_mem(bp);
7302 if (rc)
7303 return rc;
7304 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7305 if (!rc)
7306 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7307 }
7308 return 0;
7309 }
7310
7311 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7312 {
7313 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7314 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7315 int rc = 0;
7316 u32 flags;
7317
7318 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7319 return 0;
7320
7321 resp = bp->hwrm_cmd_resp_addr;
7322 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7323
7324 mutex_lock(&bp->hwrm_cmd_lock);
7325 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7326 if (rc)
7327 goto hwrm_cfa_adv_qcaps_exit;
7328
7329 flags = le32_to_cpu(resp->flags);
7330 if (flags &
7331 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7332 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7333
7334 hwrm_cfa_adv_qcaps_exit:
7335 mutex_unlock(&bp->hwrm_cmd_lock);
7336 return rc;
7337 }
7338
7339 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7340 {
7341 struct bnxt_fw_health *fw_health = bp->fw_health;
7342 u32 reg_base = 0xffffffff;
7343 int i;
7344
7345 /* Only pre-map the monitoring GRC registers using window 3 */
7346 for (i = 0; i < 4; i++) {
7347 u32 reg = fw_health->regs[i];
7348
7349 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7350 continue;
7351 if (reg_base == 0xffffffff)
7352 reg_base = reg & BNXT_GRC_BASE_MASK;
7353 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7354 return -ERANGE;
7355 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
7356 (reg & BNXT_GRC_OFFSET_MASK);
7357 }
7358 if (reg_base == 0xffffffff)
7359 return 0;
7360
7361 writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7362 BNXT_FW_HEALTH_WIN_MAP_OFF);
7363 return 0;
7364 }
7365
7366 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7367 {
7368 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7369 struct bnxt_fw_health *fw_health = bp->fw_health;
7370 struct hwrm_error_recovery_qcfg_input req = {0};
7371 int rc, i;
7372
7373 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7374 return 0;
7375
7376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7377 mutex_lock(&bp->hwrm_cmd_lock);
7378 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7379 if (rc)
7380 goto err_recovery_out;
7381 fw_health->flags = le32_to_cpu(resp->flags);
7382 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7383 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7384 rc = -EINVAL;
7385 goto err_recovery_out;
7386 }
7387 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7388 fw_health->master_func_wait_dsecs =
7389 le32_to_cpu(resp->master_func_wait_period);
7390 fw_health->normal_func_wait_dsecs =
7391 le32_to_cpu(resp->normal_func_wait_period);
7392 fw_health->post_reset_wait_dsecs =
7393 le32_to_cpu(resp->master_func_wait_period_after_reset);
7394 fw_health->post_reset_max_wait_dsecs =
7395 le32_to_cpu(resp->max_bailout_time_after_reset);
7396 fw_health->regs[BNXT_FW_HEALTH_REG] =
7397 le32_to_cpu(resp->fw_health_status_reg);
7398 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7399 le32_to_cpu(resp->fw_heartbeat_reg);
7400 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7401 le32_to_cpu(resp->fw_reset_cnt_reg);
7402 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7403 le32_to_cpu(resp->reset_inprogress_reg);
7404 fw_health->fw_reset_inprog_reg_mask =
7405 le32_to_cpu(resp->reset_inprogress_reg_mask);
7406 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7407 if (fw_health->fw_reset_seq_cnt >= 16) {
7408 rc = -EINVAL;
7409 goto err_recovery_out;
7410 }
7411 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7412 fw_health->fw_reset_seq_regs[i] =
7413 le32_to_cpu(resp->reset_reg[i]);
7414 fw_health->fw_reset_seq_vals[i] =
7415 le32_to_cpu(resp->reset_reg_val[i]);
7416 fw_health->fw_reset_seq_delay_msec[i] =
7417 resp->delay_after_reset[i];
7418 }
7419 err_recovery_out:
7420 mutex_unlock(&bp->hwrm_cmd_lock);
7421 if (!rc)
7422 rc = bnxt_map_fw_health_regs(bp);
7423 if (rc)
7424 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7425 return rc;
7426 }
7427
7428 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7429 {
7430 struct hwrm_func_reset_input req = {0};
7431
7432 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7433 req.enables = 0;
7434
7435 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7436 }
7437
7438 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7439 {
7440 int rc = 0;
7441 struct hwrm_queue_qportcfg_input req = {0};
7442 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7443 u8 i, j, *qptr;
7444 bool no_rdma;
7445
7446 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7447
7448 mutex_lock(&bp->hwrm_cmd_lock);
7449 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7450 if (rc)
7451 goto qportcfg_exit;
7452
7453 if (!resp->max_configurable_queues) {
7454 rc = -EINVAL;
7455 goto qportcfg_exit;
7456 }
7457 bp->max_tc = resp->max_configurable_queues;
7458 bp->max_lltc = resp->max_configurable_lossless_queues;
7459 if (bp->max_tc > BNXT_MAX_QUEUE)
7460 bp->max_tc = BNXT_MAX_QUEUE;
7461
7462 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7463 qptr = &resp->queue_id0;
7464 for (i = 0, j = 0; i < bp->max_tc; i++) {
7465 bp->q_info[j].queue_id = *qptr;
7466 bp->q_ids[i] = *qptr++;
7467 bp->q_info[j].queue_profile = *qptr++;
7468 bp->tc_to_qidx[j] = j;
7469 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7470 (no_rdma && BNXT_PF(bp)))
7471 j++;
7472 }
7473 bp->max_q = bp->max_tc;
7474 bp->max_tc = max_t(u8, j, 1);
7475
7476 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7477 bp->max_tc = 1;
7478
7479 if (bp->max_lltc > bp->max_tc)
7480 bp->max_lltc = bp->max_tc;
7481
7482 qportcfg_exit:
7483 mutex_unlock(&bp->hwrm_cmd_lock);
7484 return rc;
7485 }
7486
7487 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7488 {
7489 struct hwrm_ver_get_input req = {0};
7490 int rc;
7491
7492 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7493 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7494 req.hwrm_intf_min = HWRM_VERSION_MINOR;
7495 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7496
7497 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7498 silent);
7499 return rc;
7500 }
7501
7502 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7503 {
7504 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7505 u16 fw_maj, fw_min, fw_bld, fw_rsv;
7506 u32 dev_caps_cfg, hwrm_ver;
7507 int rc, len;
7508
7509 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7510 mutex_lock(&bp->hwrm_cmd_lock);
7511 rc = __bnxt_hwrm_ver_get(bp, false);
7512 if (rc)
7513 goto hwrm_ver_get_exit;
7514
7515 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7516
7517 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7518 resp->hwrm_intf_min_8b << 8 |
7519 resp->hwrm_intf_upd_8b;
7520 if (resp->hwrm_intf_maj_8b < 1) {
7521 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7522 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7523 resp->hwrm_intf_upd_8b);
7524 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7525 }
7526
7527 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7528 HWRM_VERSION_UPDATE;
7529
7530 if (bp->hwrm_spec_code > hwrm_ver)
7531 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7532 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7533 HWRM_VERSION_UPDATE);
7534 else
7535 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7536 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7537 resp->hwrm_intf_upd_8b);
7538
7539 fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7540 if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7541 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7542 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7543 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7544 len = FW_VER_STR_LEN;
7545 } else {
7546 fw_maj = resp->hwrm_fw_maj_8b;
7547 fw_min = resp->hwrm_fw_min_8b;
7548 fw_bld = resp->hwrm_fw_bld_8b;
7549 fw_rsv = resp->hwrm_fw_rsvd_8b;
7550 len = BC_HWRM_STR_LEN;
7551 }
7552 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7553 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7554 fw_rsv);
7555
7556 if (strlen(resp->active_pkg_name)) {
7557 int fw_ver_len = strlen(bp->fw_ver_str);
7558
7559 snprintf(bp->fw_ver_str + fw_ver_len,
7560 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7561 resp->active_pkg_name);
7562 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7563 }
7564
7565 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7566 if (!bp->hwrm_cmd_timeout)
7567 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7568
7569 if (resp->hwrm_intf_maj_8b >= 1) {
7570 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7571 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7572 }
7573 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7574 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7575
7576 bp->chip_num = le16_to_cpu(resp->chip_num);
7577 bp->chip_rev = resp->chip_rev;
7578 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7579 !resp->chip_metal)
7580 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7581
7582 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7583 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7584 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7585 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7586
7587 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7588 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7589
7590 if (dev_caps_cfg &
7591 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7592 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7593
7594 if (dev_caps_cfg &
7595 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7596 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7597
7598 if (dev_caps_cfg &
7599 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7600 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7601
7602 hwrm_ver_get_exit:
7603 mutex_unlock(&bp->hwrm_cmd_lock);
7604 return rc;
7605 }
7606
7607 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7608 {
7609 struct hwrm_fw_set_time_input req = {0};
7610 struct tm tm;
7611 time64_t now = ktime_get_real_seconds();
7612
7613 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7614 bp->hwrm_spec_code < 0x10400)
7615 return -EOPNOTSUPP;
7616
7617 time64_to_tm(now, 0, &tm);
7618 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7619 req.year = cpu_to_le16(1900 + tm.tm_year);
7620 req.month = 1 + tm.tm_mon;
7621 req.day = tm.tm_mday;
7622 req.hour = tm.tm_hour;
7623 req.minute = tm.tm_min;
7624 req.second = tm.tm_sec;
7625 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7626 }
7627
7628 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7629 {
7630 u64 sw_tmp;
7631
7632 sw_tmp = (*sw & ~mask) | hw;
7633 if (hw < (*sw & mask))
7634 sw_tmp += mask + 1;
7635 WRITE_ONCE(*sw, sw_tmp);
7636 }
7637
7638 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7639 int count, bool ignore_zero)
7640 {
7641 int i;
7642
7643 for (i = 0; i < count; i++) {
7644 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7645
7646 if (ignore_zero && !hw)
7647 continue;
7648
7649 if (masks[i] == -1ULL)
7650 sw_stats[i] = hw;
7651 else
7652 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7653 }
7654 }
7655
7656 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7657 {
7658 if (!stats->hw_stats)
7659 return;
7660
7661 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7662 stats->hw_masks, stats->len / 8, false);
7663 }
7664
7665 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7666 {
7667 struct bnxt_stats_mem *ring0_stats;
7668 bool ignore_zero = false;
7669 int i;
7670
7671 /* Chip bug. Counter intermittently becomes 0. */
7672 if (bp->flags & BNXT_FLAG_CHIP_P5)
7673 ignore_zero = true;
7674
7675 for (i = 0; i < bp->cp_nr_rings; i++) {
7676 struct bnxt_napi *bnapi = bp->bnapi[i];
7677 struct bnxt_cp_ring_info *cpr;
7678 struct bnxt_stats_mem *stats;
7679
7680 cpr = &bnapi->cp_ring;
7681 stats = &cpr->stats;
7682 if (!i)
7683 ring0_stats = stats;
7684 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7685 ring0_stats->hw_masks,
7686 ring0_stats->len / 8, ignore_zero);
7687 }
7688 if (bp->flags & BNXT_FLAG_PORT_STATS) {
7689 struct bnxt_stats_mem *stats = &bp->port_stats;
7690 __le64 *hw_stats = stats->hw_stats;
7691 u64 *sw_stats = stats->sw_stats;
7692 u64 *masks = stats->hw_masks;
7693 int cnt;
7694
7695 cnt = sizeof(struct rx_port_stats) / 8;
7696 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7697
7698 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7699 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7700 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7701 cnt = sizeof(struct tx_port_stats) / 8;
7702 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7703 }
7704 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7705 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7706 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7707 }
7708 }
7709
7710 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7711 {
7712 struct bnxt_pf_info *pf = &bp->pf;
7713 struct hwrm_port_qstats_input req = {0};
7714
7715 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7716 return 0;
7717
7718 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7719 return -EOPNOTSUPP;
7720
7721 req.flags = flags;
7722 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7723 req.port_id = cpu_to_le16(pf->port_id);
7724 req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7725 BNXT_TX_PORT_STATS_BYTE_OFFSET);
7726 req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7727 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7728 }
7729
7730 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7731 {
7732 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7733 struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7734 struct hwrm_port_qstats_ext_input req = {0};
7735 struct bnxt_pf_info *pf = &bp->pf;
7736 u32 tx_stat_size;
7737 int rc;
7738
7739 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7740 return 0;
7741
7742 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7743 return -EOPNOTSUPP;
7744
7745 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7746 req.flags = flags;
7747 req.port_id = cpu_to_le16(pf->port_id);
7748 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7749 req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7750 tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7751 sizeof(struct tx_port_stats_ext) : 0;
7752 req.tx_stat_size = cpu_to_le16(tx_stat_size);
7753 req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7754 mutex_lock(&bp->hwrm_cmd_lock);
7755 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7756 if (!rc) {
7757 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7758 bp->fw_tx_stats_ext_size = tx_stat_size ?
7759 le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7760 } else {
7761 bp->fw_rx_stats_ext_size = 0;
7762 bp->fw_tx_stats_ext_size = 0;
7763 }
7764 if (flags)
7765 goto qstats_done;
7766
7767 if (bp->fw_tx_stats_ext_size <=
7768 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7769 mutex_unlock(&bp->hwrm_cmd_lock);
7770 bp->pri2cos_valid = 0;
7771 return rc;
7772 }
7773
7774 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7775 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7776
7777 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7778 if (!rc) {
7779 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7780 u8 *pri2cos;
7781 int i, j;
7782
7783 resp2 = bp->hwrm_cmd_resp_addr;
7784 pri2cos = &resp2->pri0_cos_queue_id;
7785 for (i = 0; i < 8; i++) {
7786 u8 queue_id = pri2cos[i];
7787 u8 queue_idx;
7788
7789 /* Per port queue IDs start from 0, 10, 20, etc */
7790 queue_idx = queue_id % 10;
7791 if (queue_idx > BNXT_MAX_QUEUE) {
7792 bp->pri2cos_valid = false;
7793 goto qstats_done;
7794 }
7795 for (j = 0; j < bp->max_q; j++) {
7796 if (bp->q_ids[j] == queue_id)
7797 bp->pri2cos_idx[i] = queue_idx;
7798 }
7799 }
7800 bp->pri2cos_valid = 1;
7801 }
7802 qstats_done:
7803 mutex_unlock(&bp->hwrm_cmd_lock);
7804 return rc;
7805 }
7806
7807 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7808 {
7809 if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7810 bnxt_hwrm_tunnel_dst_port_free(
7811 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7812 if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7813 bnxt_hwrm_tunnel_dst_port_free(
7814 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7815 }
7816
7817 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7818 {
7819 int rc, i;
7820 u32 tpa_flags = 0;
7821
7822 if (set_tpa)
7823 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7824 else if (BNXT_NO_FW_ACCESS(bp))
7825 return 0;
7826 for (i = 0; i < bp->nr_vnics; i++) {
7827 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7828 if (rc) {
7829 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7830 i, rc);
7831 return rc;
7832 }
7833 }
7834 return 0;
7835 }
7836
7837 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7838 {
7839 int i;
7840
7841 for (i = 0; i < bp->nr_vnics; i++)
7842 bnxt_hwrm_vnic_set_rss(bp, i, false);
7843 }
7844
7845 static void bnxt_clear_vnic(struct bnxt *bp)
7846 {
7847 if (!bp->vnic_info)
7848 return;
7849
7850 bnxt_hwrm_clear_vnic_filter(bp);
7851 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7852 /* clear all RSS setting before free vnic ctx */
7853 bnxt_hwrm_clear_vnic_rss(bp);
7854 bnxt_hwrm_vnic_ctx_free(bp);
7855 }
7856 /* before free the vnic, undo the vnic tpa settings */
7857 if (bp->flags & BNXT_FLAG_TPA)
7858 bnxt_set_tpa(bp, false);
7859 bnxt_hwrm_vnic_free(bp);
7860 if (bp->flags & BNXT_FLAG_CHIP_P5)
7861 bnxt_hwrm_vnic_ctx_free(bp);
7862 }
7863
7864 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7865 bool irq_re_init)
7866 {
7867 bnxt_clear_vnic(bp);
7868 bnxt_hwrm_ring_free(bp, close_path);
7869 bnxt_hwrm_ring_grp_free(bp);
7870 if (irq_re_init) {
7871 bnxt_hwrm_stat_ctx_free(bp);
7872 bnxt_hwrm_free_tunnel_ports(bp);
7873 }
7874 }
7875
7876 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7877 {
7878 struct hwrm_func_cfg_input req = {0};
7879
7880 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7881 req.fid = cpu_to_le16(0xffff);
7882 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7883 if (br_mode == BRIDGE_MODE_VEB)
7884 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7885 else if (br_mode == BRIDGE_MODE_VEPA)
7886 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7887 else
7888 return -EINVAL;
7889 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7890 }
7891
7892 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7893 {
7894 struct hwrm_func_cfg_input req = {0};
7895
7896 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7897 return 0;
7898
7899 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7900 req.fid = cpu_to_le16(0xffff);
7901 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7902 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7903 if (size == 128)
7904 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7905
7906 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7907 }
7908
7909 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7910 {
7911 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7912 int rc;
7913
7914 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7915 goto skip_rss_ctx;
7916
7917 /* allocate context for vnic */
7918 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7919 if (rc) {
7920 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7921 vnic_id, rc);
7922 goto vnic_setup_err;
7923 }
7924 bp->rsscos_nr_ctxs++;
7925
7926 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7927 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7928 if (rc) {
7929 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7930 vnic_id, rc);
7931 goto vnic_setup_err;
7932 }
7933 bp->rsscos_nr_ctxs++;
7934 }
7935
7936 skip_rss_ctx:
7937 /* configure default vnic, ring grp */
7938 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7939 if (rc) {
7940 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7941 vnic_id, rc);
7942 goto vnic_setup_err;
7943 }
7944
7945 /* Enable RSS hashing on vnic */
7946 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7947 if (rc) {
7948 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7949 vnic_id, rc);
7950 goto vnic_setup_err;
7951 }
7952
7953 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7954 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7955 if (rc) {
7956 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7957 vnic_id, rc);
7958 }
7959 }
7960
7961 vnic_setup_err:
7962 return rc;
7963 }
7964
7965 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7966 {
7967 int rc, i, nr_ctxs;
7968
7969 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
7970 for (i = 0; i < nr_ctxs; i++) {
7971 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7972 if (rc) {
7973 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7974 vnic_id, i, rc);
7975 break;
7976 }
7977 bp->rsscos_nr_ctxs++;
7978 }
7979 if (i < nr_ctxs)
7980 return -ENOMEM;
7981
7982 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7983 if (rc) {
7984 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7985 vnic_id, rc);
7986 return rc;
7987 }
7988 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7989 if (rc) {
7990 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7991 vnic_id, rc);
7992 return rc;
7993 }
7994 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7995 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7996 if (rc) {
7997 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7998 vnic_id, rc);
7999 }
8000 }
8001 return rc;
8002 }
8003
8004 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8005 {
8006 if (bp->flags & BNXT_FLAG_CHIP_P5)
8007 return __bnxt_setup_vnic_p5(bp, vnic_id);
8008 else
8009 return __bnxt_setup_vnic(bp, vnic_id);
8010 }
8011
8012 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8013 {
8014 #ifdef CONFIG_RFS_ACCEL
8015 int i, rc = 0;
8016
8017 if (bp->flags & BNXT_FLAG_CHIP_P5)
8018 return 0;
8019
8020 for (i = 0; i < bp->rx_nr_rings; i++) {
8021 struct bnxt_vnic_info *vnic;
8022 u16 vnic_id = i + 1;
8023 u16 ring_id = i;
8024
8025 if (vnic_id >= bp->nr_vnics)
8026 break;
8027
8028 vnic = &bp->vnic_info[vnic_id];
8029 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8030 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8031 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8032 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8033 if (rc) {
8034 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8035 vnic_id, rc);
8036 break;
8037 }
8038 rc = bnxt_setup_vnic(bp, vnic_id);
8039 if (rc)
8040 break;
8041 }
8042 return rc;
8043 #else
8044 return 0;
8045 #endif
8046 }
8047
8048 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8049 static bool bnxt_promisc_ok(struct bnxt *bp)
8050 {
8051 #ifdef CONFIG_BNXT_SRIOV
8052 if (BNXT_VF(bp) && !bp->vf.vlan)
8053 return false;
8054 #endif
8055 return true;
8056 }
8057
8058 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8059 {
8060 unsigned int rc = 0;
8061
8062 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8063 if (rc) {
8064 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8065 rc);
8066 return rc;
8067 }
8068
8069 rc = bnxt_hwrm_vnic_cfg(bp, 1);
8070 if (rc) {
8071 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8072 rc);
8073 return rc;
8074 }
8075 return rc;
8076 }
8077
8078 static int bnxt_cfg_rx_mode(struct bnxt *);
8079 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8080
8081 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8082 {
8083 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8084 int rc = 0;
8085 unsigned int rx_nr_rings = bp->rx_nr_rings;
8086
8087 if (irq_re_init) {
8088 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8089 if (rc) {
8090 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8091 rc);
8092 goto err_out;
8093 }
8094 }
8095
8096 rc = bnxt_hwrm_ring_alloc(bp);
8097 if (rc) {
8098 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8099 goto err_out;
8100 }
8101
8102 rc = bnxt_hwrm_ring_grp_alloc(bp);
8103 if (rc) {
8104 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8105 goto err_out;
8106 }
8107
8108 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8109 rx_nr_rings--;
8110
8111 /* default vnic 0 */
8112 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8113 if (rc) {
8114 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8115 goto err_out;
8116 }
8117
8118 rc = bnxt_setup_vnic(bp, 0);
8119 if (rc)
8120 goto err_out;
8121
8122 if (bp->flags & BNXT_FLAG_RFS) {
8123 rc = bnxt_alloc_rfs_vnics(bp);
8124 if (rc)
8125 goto err_out;
8126 }
8127
8128 if (bp->flags & BNXT_FLAG_TPA) {
8129 rc = bnxt_set_tpa(bp, true);
8130 if (rc)
8131 goto err_out;
8132 }
8133
8134 if (BNXT_VF(bp))
8135 bnxt_update_vf_mac(bp);
8136
8137 /* Filter for default vnic 0 */
8138 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8139 if (rc) {
8140 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8141 goto err_out;
8142 }
8143 vnic->uc_filter_count = 1;
8144
8145 vnic->rx_mask = 0;
8146 if (bp->dev->flags & IFF_BROADCAST)
8147 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8148
8149 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8150 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8151
8152 if (bp->dev->flags & IFF_ALLMULTI) {
8153 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8154 vnic->mc_list_count = 0;
8155 } else {
8156 u32 mask = 0;
8157
8158 bnxt_mc_list_updated(bp, &mask);
8159 vnic->rx_mask |= mask;
8160 }
8161
8162 rc = bnxt_cfg_rx_mode(bp);
8163 if (rc)
8164 goto err_out;
8165
8166 rc = bnxt_hwrm_set_coal(bp);
8167 if (rc)
8168 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8169 rc);
8170
8171 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8172 rc = bnxt_setup_nitroa0_vnic(bp);
8173 if (rc)
8174 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8175 rc);
8176 }
8177
8178 if (BNXT_VF(bp)) {
8179 bnxt_hwrm_func_qcfg(bp);
8180 netdev_update_features(bp->dev);
8181 }
8182
8183 return 0;
8184
8185 err_out:
8186 bnxt_hwrm_resource_free(bp, 0, true);
8187
8188 return rc;
8189 }
8190
8191 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8192 {
8193 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8194 return 0;
8195 }
8196
8197 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8198 {
8199 bnxt_init_cp_rings(bp);
8200 bnxt_init_rx_rings(bp);
8201 bnxt_init_tx_rings(bp);
8202 bnxt_init_ring_grps(bp, irq_re_init);
8203 bnxt_init_vnics(bp);
8204
8205 return bnxt_init_chip(bp, irq_re_init);
8206 }
8207
8208 static int bnxt_set_real_num_queues(struct bnxt *bp)
8209 {
8210 int rc;
8211 struct net_device *dev = bp->dev;
8212
8213 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8214 bp->tx_nr_rings_xdp);
8215 if (rc)
8216 return rc;
8217
8218 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8219 if (rc)
8220 return rc;
8221
8222 #ifdef CONFIG_RFS_ACCEL
8223 if (bp->flags & BNXT_FLAG_RFS)
8224 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8225 #endif
8226
8227 return rc;
8228 }
8229
8230 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8231 bool shared)
8232 {
8233 int _rx = *rx, _tx = *tx;
8234
8235 if (shared) {
8236 *rx = min_t(int, _rx, max);
8237 *tx = min_t(int, _tx, max);
8238 } else {
8239 if (max < 2)
8240 return -ENOMEM;
8241
8242 while (_rx + _tx > max) {
8243 if (_rx > _tx && _rx > 1)
8244 _rx--;
8245 else if (_tx > 1)
8246 _tx--;
8247 }
8248 *rx = _rx;
8249 *tx = _tx;
8250 }
8251 return 0;
8252 }
8253
8254 static void bnxt_setup_msix(struct bnxt *bp)
8255 {
8256 const int len = sizeof(bp->irq_tbl[0].name);
8257 struct net_device *dev = bp->dev;
8258 int tcs, i;
8259
8260 tcs = netdev_get_num_tc(dev);
8261 if (tcs) {
8262 int i, off, count;
8263
8264 for (i = 0; i < tcs; i++) {
8265 count = bp->tx_nr_rings_per_tc;
8266 off = i * count;
8267 netdev_set_tc_queue(dev, i, count, off);
8268 }
8269 }
8270
8271 for (i = 0; i < bp->cp_nr_rings; i++) {
8272 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8273 char *attr;
8274
8275 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8276 attr = "TxRx";
8277 else if (i < bp->rx_nr_rings)
8278 attr = "rx";
8279 else
8280 attr = "tx";
8281
8282 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8283 attr, i);
8284 bp->irq_tbl[map_idx].handler = bnxt_msix;
8285 }
8286 }
8287
8288 static void bnxt_setup_inta(struct bnxt *bp)
8289 {
8290 const int len = sizeof(bp->irq_tbl[0].name);
8291
8292 if (netdev_get_num_tc(bp->dev))
8293 netdev_reset_tc(bp->dev);
8294
8295 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8296 0);
8297 bp->irq_tbl[0].handler = bnxt_inta;
8298 }
8299
8300 static int bnxt_setup_int_mode(struct bnxt *bp)
8301 {
8302 int rc;
8303
8304 if (bp->flags & BNXT_FLAG_USING_MSIX)
8305 bnxt_setup_msix(bp);
8306 else
8307 bnxt_setup_inta(bp);
8308
8309 rc = bnxt_set_real_num_queues(bp);
8310 return rc;
8311 }
8312
8313 #ifdef CONFIG_RFS_ACCEL
8314 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8315 {
8316 return bp->hw_resc.max_rsscos_ctxs;
8317 }
8318
8319 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8320 {
8321 return bp->hw_resc.max_vnics;
8322 }
8323 #endif
8324
8325 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8326 {
8327 return bp->hw_resc.max_stat_ctxs;
8328 }
8329
8330 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8331 {
8332 return bp->hw_resc.max_cp_rings;
8333 }
8334
8335 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8336 {
8337 unsigned int cp = bp->hw_resc.max_cp_rings;
8338
8339 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8340 cp -= bnxt_get_ulp_msix_num(bp);
8341
8342 return cp;
8343 }
8344
8345 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8346 {
8347 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8348
8349 if (bp->flags & BNXT_FLAG_CHIP_P5)
8350 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8351
8352 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8353 }
8354
8355 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8356 {
8357 bp->hw_resc.max_irqs = max_irqs;
8358 }
8359
8360 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8361 {
8362 unsigned int cp;
8363
8364 cp = bnxt_get_max_func_cp_rings_for_en(bp);
8365 if (bp->flags & BNXT_FLAG_CHIP_P5)
8366 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8367 else
8368 return cp - bp->cp_nr_rings;
8369 }
8370
8371 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8372 {
8373 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8374 }
8375
8376 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8377 {
8378 int max_cp = bnxt_get_max_func_cp_rings(bp);
8379 int max_irq = bnxt_get_max_func_irqs(bp);
8380 int total_req = bp->cp_nr_rings + num;
8381 int max_idx, avail_msix;
8382
8383 max_idx = bp->total_irqs;
8384 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8385 max_idx = min_t(int, bp->total_irqs, max_cp);
8386 avail_msix = max_idx - bp->cp_nr_rings;
8387 if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8388 return avail_msix;
8389
8390 if (max_irq < total_req) {
8391 num = max_irq - bp->cp_nr_rings;
8392 if (num <= 0)
8393 return 0;
8394 }
8395 return num;
8396 }
8397
8398 static int bnxt_get_num_msix(struct bnxt *bp)
8399 {
8400 if (!BNXT_NEW_RM(bp))
8401 return bnxt_get_max_func_irqs(bp);
8402
8403 return bnxt_nq_rings_in_use(bp);
8404 }
8405
8406 static int bnxt_init_msix(struct bnxt *bp)
8407 {
8408 int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8409 struct msix_entry *msix_ent;
8410
8411 total_vecs = bnxt_get_num_msix(bp);
8412 max = bnxt_get_max_func_irqs(bp);
8413 if (total_vecs > max)
8414 total_vecs = max;
8415
8416 if (!total_vecs)
8417 return 0;
8418
8419 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8420 if (!msix_ent)
8421 return -ENOMEM;
8422
8423 for (i = 0; i < total_vecs; i++) {
8424 msix_ent[i].entry = i;
8425 msix_ent[i].vector = 0;
8426 }
8427
8428 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8429 min = 2;
8430
8431 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8432 ulp_msix = bnxt_get_ulp_msix_num(bp);
8433 if (total_vecs < 0 || total_vecs < ulp_msix) {
8434 rc = -ENODEV;
8435 goto msix_setup_exit;
8436 }
8437
8438 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8439 if (bp->irq_tbl) {
8440 for (i = 0; i < total_vecs; i++)
8441 bp->irq_tbl[i].vector = msix_ent[i].vector;
8442
8443 bp->total_irqs = total_vecs;
8444 /* Trim rings based upon num of vectors allocated */
8445 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8446 total_vecs - ulp_msix, min == 1);
8447 if (rc)
8448 goto msix_setup_exit;
8449
8450 bp->cp_nr_rings = (min == 1) ?
8451 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8452 bp->tx_nr_rings + bp->rx_nr_rings;
8453
8454 } else {
8455 rc = -ENOMEM;
8456 goto msix_setup_exit;
8457 }
8458 bp->flags |= BNXT_FLAG_USING_MSIX;
8459 kfree(msix_ent);
8460 return 0;
8461
8462 msix_setup_exit:
8463 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8464 kfree(bp->irq_tbl);
8465 bp->irq_tbl = NULL;
8466 pci_disable_msix(bp->pdev);
8467 kfree(msix_ent);
8468 return rc;
8469 }
8470
8471 static int bnxt_init_inta(struct bnxt *bp)
8472 {
8473 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8474 if (!bp->irq_tbl)
8475 return -ENOMEM;
8476
8477 bp->total_irqs = 1;
8478 bp->rx_nr_rings = 1;
8479 bp->tx_nr_rings = 1;
8480 bp->cp_nr_rings = 1;
8481 bp->flags |= BNXT_FLAG_SHARED_RINGS;
8482 bp->irq_tbl[0].vector = bp->pdev->irq;
8483 return 0;
8484 }
8485
8486 static int bnxt_init_int_mode(struct bnxt *bp)
8487 {
8488 int rc = 0;
8489
8490 if (bp->flags & BNXT_FLAG_MSIX_CAP)
8491 rc = bnxt_init_msix(bp);
8492
8493 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8494 /* fallback to INTA */
8495 rc = bnxt_init_inta(bp);
8496 }
8497 return rc;
8498 }
8499
8500 static void bnxt_clear_int_mode(struct bnxt *bp)
8501 {
8502 if (bp->flags & BNXT_FLAG_USING_MSIX)
8503 pci_disable_msix(bp->pdev);
8504
8505 kfree(bp->irq_tbl);
8506 bp->irq_tbl = NULL;
8507 bp->flags &= ~BNXT_FLAG_USING_MSIX;
8508 }
8509
8510 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8511 {
8512 int tcs = netdev_get_num_tc(bp->dev);
8513 bool irq_cleared = false;
8514 int rc;
8515
8516 if (!bnxt_need_reserve_rings(bp))
8517 return 0;
8518
8519 if (irq_re_init && BNXT_NEW_RM(bp) &&
8520 bnxt_get_num_msix(bp) != bp->total_irqs) {
8521 bnxt_ulp_irq_stop(bp);
8522 bnxt_clear_int_mode(bp);
8523 irq_cleared = true;
8524 }
8525 rc = __bnxt_reserve_rings(bp);
8526 if (irq_cleared) {
8527 if (!rc)
8528 rc = bnxt_init_int_mode(bp);
8529 bnxt_ulp_irq_restart(bp, rc);
8530 }
8531 if (rc) {
8532 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8533 return rc;
8534 }
8535 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8536 netdev_err(bp->dev, "tx ring reservation failure\n");
8537 netdev_reset_tc(bp->dev);
8538 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8539 return -ENOMEM;
8540 }
8541 return 0;
8542 }
8543
8544 static void bnxt_free_irq(struct bnxt *bp)
8545 {
8546 struct bnxt_irq *irq;
8547 int i;
8548
8549 #ifdef CONFIG_RFS_ACCEL
8550 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8551 bp->dev->rx_cpu_rmap = NULL;
8552 #endif
8553 if (!bp->irq_tbl || !bp->bnapi)
8554 return;
8555
8556 for (i = 0; i < bp->cp_nr_rings; i++) {
8557 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8558
8559 irq = &bp->irq_tbl[map_idx];
8560 if (irq->requested) {
8561 if (irq->have_cpumask) {
8562 irq_set_affinity_hint(irq->vector, NULL);
8563 free_cpumask_var(irq->cpu_mask);
8564 irq->have_cpumask = 0;
8565 }
8566 free_irq(irq->vector, bp->bnapi[i]);
8567 }
8568
8569 irq->requested = 0;
8570 }
8571 }
8572
8573 static int bnxt_request_irq(struct bnxt *bp)
8574 {
8575 int i, j, rc = 0;
8576 unsigned long flags = 0;
8577 #ifdef CONFIG_RFS_ACCEL
8578 struct cpu_rmap *rmap;
8579 #endif
8580
8581 rc = bnxt_setup_int_mode(bp);
8582 if (rc) {
8583 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8584 rc);
8585 return rc;
8586 }
8587 #ifdef CONFIG_RFS_ACCEL
8588 rmap = bp->dev->rx_cpu_rmap;
8589 #endif
8590 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8591 flags = IRQF_SHARED;
8592
8593 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8594 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8595 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8596
8597 #ifdef CONFIG_RFS_ACCEL
8598 if (rmap && bp->bnapi[i]->rx_ring) {
8599 rc = irq_cpu_rmap_add(rmap, irq->vector);
8600 if (rc)
8601 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8602 j);
8603 j++;
8604 }
8605 #endif
8606 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8607 bp->bnapi[i]);
8608 if (rc)
8609 break;
8610
8611 irq->requested = 1;
8612
8613 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8614 int numa_node = dev_to_node(&bp->pdev->dev);
8615
8616 irq->have_cpumask = 1;
8617 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8618 irq->cpu_mask);
8619 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8620 if (rc) {
8621 netdev_warn(bp->dev,
8622 "Set affinity failed, IRQ = %d\n",
8623 irq->vector);
8624 break;
8625 }
8626 }
8627 }
8628 return rc;
8629 }
8630
8631 static void bnxt_del_napi(struct bnxt *bp)
8632 {
8633 int i;
8634
8635 if (!bp->bnapi)
8636 return;
8637
8638 for (i = 0; i < bp->cp_nr_rings; i++) {
8639 struct bnxt_napi *bnapi = bp->bnapi[i];
8640
8641 __netif_napi_del(&bnapi->napi);
8642 }
8643 /* We called __netif_napi_del(), we need
8644 * to respect an RCU grace period before freeing napi structures.
8645 */
8646 synchronize_net();
8647 }
8648
8649 static void bnxt_init_napi(struct bnxt *bp)
8650 {
8651 int i;
8652 unsigned int cp_nr_rings = bp->cp_nr_rings;
8653 struct bnxt_napi *bnapi;
8654
8655 if (bp->flags & BNXT_FLAG_USING_MSIX) {
8656 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8657
8658 if (bp->flags & BNXT_FLAG_CHIP_P5)
8659 poll_fn = bnxt_poll_p5;
8660 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8661 cp_nr_rings--;
8662 for (i = 0; i < cp_nr_rings; i++) {
8663 bnapi = bp->bnapi[i];
8664 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8665 }
8666 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8667 bnapi = bp->bnapi[cp_nr_rings];
8668 netif_napi_add(bp->dev, &bnapi->napi,
8669 bnxt_poll_nitroa0, 64);
8670 }
8671 } else {
8672 bnapi = bp->bnapi[0];
8673 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8674 }
8675 }
8676
8677 static void bnxt_disable_napi(struct bnxt *bp)
8678 {
8679 int i;
8680
8681 if (!bp->bnapi)
8682 return;
8683
8684 for (i = 0; i < bp->cp_nr_rings; i++) {
8685 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8686
8687 if (bp->bnapi[i]->rx_ring)
8688 cancel_work_sync(&cpr->dim.work);
8689
8690 napi_disable(&bp->bnapi[i]->napi);
8691 }
8692 }
8693
8694 static void bnxt_enable_napi(struct bnxt *bp)
8695 {
8696 int i;
8697
8698 for (i = 0; i < bp->cp_nr_rings; i++) {
8699 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8700 bp->bnapi[i]->in_reset = false;
8701
8702 if (bp->bnapi[i]->rx_ring) {
8703 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8704 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8705 }
8706 napi_enable(&bp->bnapi[i]->napi);
8707 }
8708 }
8709
8710 void bnxt_tx_disable(struct bnxt *bp)
8711 {
8712 int i;
8713 struct bnxt_tx_ring_info *txr;
8714
8715 if (bp->tx_ring) {
8716 for (i = 0; i < bp->tx_nr_rings; i++) {
8717 txr = &bp->tx_ring[i];
8718 txr->dev_state = BNXT_DEV_STATE_CLOSING;
8719 }
8720 }
8721 /* Stop all TX queues */
8722 netif_tx_disable(bp->dev);
8723 netif_carrier_off(bp->dev);
8724 }
8725
8726 void bnxt_tx_enable(struct bnxt *bp)
8727 {
8728 int i;
8729 struct bnxt_tx_ring_info *txr;
8730
8731 for (i = 0; i < bp->tx_nr_rings; i++) {
8732 txr = &bp->tx_ring[i];
8733 txr->dev_state = 0;
8734 }
8735 netif_tx_wake_all_queues(bp->dev);
8736 if (bp->link_info.link_up)
8737 netif_carrier_on(bp->dev);
8738 }
8739
8740 static void bnxt_report_link(struct bnxt *bp)
8741 {
8742 if (bp->link_info.link_up) {
8743 const char *duplex;
8744 const char *flow_ctrl;
8745 u32 speed;
8746 u16 fec;
8747
8748 netif_carrier_on(bp->dev);
8749 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8750 duplex = "full";
8751 else
8752 duplex = "half";
8753 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8754 flow_ctrl = "ON - receive & transmit";
8755 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8756 flow_ctrl = "ON - transmit";
8757 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8758 flow_ctrl = "ON - receive";
8759 else
8760 flow_ctrl = "none";
8761 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8762 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8763 speed, duplex, flow_ctrl);
8764 if (bp->flags & BNXT_FLAG_EEE_CAP)
8765 netdev_info(bp->dev, "EEE is %s\n",
8766 bp->eee.eee_active ? "active" :
8767 "not active");
8768 fec = bp->link_info.fec_cfg;
8769 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8770 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8771 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8772 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8773 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8774 } else {
8775 netif_carrier_off(bp->dev);
8776 netdev_err(bp->dev, "NIC Link is Down\n");
8777 }
8778 }
8779
8780 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
8781 {
8782 if (!resp->supported_speeds_auto_mode &&
8783 !resp->supported_speeds_force_mode &&
8784 !resp->supported_pam4_speeds_auto_mode &&
8785 !resp->supported_pam4_speeds_force_mode)
8786 return true;
8787 return false;
8788 }
8789
8790 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8791 {
8792 int rc = 0;
8793 struct hwrm_port_phy_qcaps_input req = {0};
8794 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8795 struct bnxt_link_info *link_info = &bp->link_info;
8796
8797 bp->flags &= ~BNXT_FLAG_EEE_CAP;
8798 if (bp->test_info)
8799 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8800 BNXT_TEST_FL_AN_PHY_LPBK);
8801 if (bp->hwrm_spec_code < 0x10201)
8802 return 0;
8803
8804 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8805
8806 mutex_lock(&bp->hwrm_cmd_lock);
8807 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8808 if (rc)
8809 goto hwrm_phy_qcaps_exit;
8810
8811 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8812 struct ethtool_eee *eee = &bp->eee;
8813 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8814
8815 bp->flags |= BNXT_FLAG_EEE_CAP;
8816 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8817 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8818 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8819 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8820 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8821 }
8822 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8823 if (bp->test_info)
8824 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8825 }
8826 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8827 if (bp->test_info)
8828 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8829 }
8830 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8831 if (BNXT_PF(bp))
8832 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8833 }
8834 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8835 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8836
8837 if (bp->hwrm_spec_code >= 0x10a01) {
8838 if (bnxt_phy_qcaps_no_speed(resp)) {
8839 link_info->phy_state = BNXT_PHY_STATE_DISABLED;
8840 netdev_warn(bp->dev, "Ethernet link disabled\n");
8841 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
8842 link_info->phy_state = BNXT_PHY_STATE_ENABLED;
8843 netdev_info(bp->dev, "Ethernet link enabled\n");
8844 /* Phy re-enabled, reprobe the speeds */
8845 link_info->support_auto_speeds = 0;
8846 link_info->support_pam4_auto_speeds = 0;
8847 }
8848 }
8849 if (resp->supported_speeds_auto_mode)
8850 link_info->support_auto_speeds =
8851 le16_to_cpu(resp->supported_speeds_auto_mode);
8852 if (resp->supported_pam4_speeds_auto_mode)
8853 link_info->support_pam4_auto_speeds =
8854 le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
8855
8856 bp->port_count = resp->port_cnt;
8857
8858 hwrm_phy_qcaps_exit:
8859 mutex_unlock(&bp->hwrm_cmd_lock);
8860 return rc;
8861 }
8862
8863 static bool bnxt_support_dropped(u16 advertising, u16 supported)
8864 {
8865 u16 diff = advertising ^ supported;
8866
8867 return ((supported | diff) != supported);
8868 }
8869
8870 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8871 {
8872 int rc = 0;
8873 struct bnxt_link_info *link_info = &bp->link_info;
8874 struct hwrm_port_phy_qcfg_input req = {0};
8875 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8876 u8 link_up = link_info->link_up;
8877 bool support_changed = false;
8878
8879 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8880
8881 mutex_lock(&bp->hwrm_cmd_lock);
8882 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8883 if (rc) {
8884 mutex_unlock(&bp->hwrm_cmd_lock);
8885 return rc;
8886 }
8887
8888 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8889 link_info->phy_link_status = resp->link;
8890 link_info->duplex = resp->duplex_cfg;
8891 if (bp->hwrm_spec_code >= 0x10800)
8892 link_info->duplex = resp->duplex_state;
8893 link_info->pause = resp->pause;
8894 link_info->auto_mode = resp->auto_mode;
8895 link_info->auto_pause_setting = resp->auto_pause;
8896 link_info->lp_pause = resp->link_partner_adv_pause;
8897 link_info->force_pause_setting = resp->force_pause;
8898 link_info->duplex_setting = resp->duplex_cfg;
8899 if (link_info->phy_link_status == BNXT_LINK_LINK)
8900 link_info->link_speed = le16_to_cpu(resp->link_speed);
8901 else
8902 link_info->link_speed = 0;
8903 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8904 link_info->force_pam4_link_speed =
8905 le16_to_cpu(resp->force_pam4_link_speed);
8906 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8907 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
8908 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8909 link_info->auto_pam4_link_speeds =
8910 le16_to_cpu(resp->auto_pam4_link_speed_mask);
8911 link_info->lp_auto_link_speeds =
8912 le16_to_cpu(resp->link_partner_adv_speeds);
8913 link_info->lp_auto_pam4_link_speeds =
8914 resp->link_partner_pam4_adv_speeds;
8915 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8916 link_info->phy_ver[0] = resp->phy_maj;
8917 link_info->phy_ver[1] = resp->phy_min;
8918 link_info->phy_ver[2] = resp->phy_bld;
8919 link_info->media_type = resp->media_type;
8920 link_info->phy_type = resp->phy_type;
8921 link_info->transceiver = resp->xcvr_pkg_type;
8922 link_info->phy_addr = resp->eee_config_phy_addr &
8923 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8924 link_info->module_status = resp->module_status;
8925
8926 if (bp->flags & BNXT_FLAG_EEE_CAP) {
8927 struct ethtool_eee *eee = &bp->eee;
8928 u16 fw_speeds;
8929
8930 eee->eee_active = 0;
8931 if (resp->eee_config_phy_addr &
8932 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8933 eee->eee_active = 1;
8934 fw_speeds = le16_to_cpu(
8935 resp->link_partner_adv_eee_link_speed_mask);
8936 eee->lp_advertised =
8937 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8938 }
8939
8940 /* Pull initial EEE config */
8941 if (!chng_link_state) {
8942 if (resp->eee_config_phy_addr &
8943 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8944 eee->eee_enabled = 1;
8945
8946 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8947 eee->advertised =
8948 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8949
8950 if (resp->eee_config_phy_addr &
8951 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8952 __le32 tmr;
8953
8954 eee->tx_lpi_enabled = 1;
8955 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8956 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8957 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8958 }
8959 }
8960 }
8961
8962 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8963 if (bp->hwrm_spec_code >= 0x10504)
8964 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8965
8966 /* TODO: need to add more logic to report VF link */
8967 if (chng_link_state) {
8968 if (link_info->phy_link_status == BNXT_LINK_LINK)
8969 link_info->link_up = 1;
8970 else
8971 link_info->link_up = 0;
8972 if (link_up != link_info->link_up)
8973 bnxt_report_link(bp);
8974 } else {
8975 /* alwasy link down if not require to update link state */
8976 link_info->link_up = 0;
8977 }
8978 mutex_unlock(&bp->hwrm_cmd_lock);
8979
8980 if (!BNXT_PHY_CFG_ABLE(bp))
8981 return 0;
8982
8983 /* Check if any advertised speeds are no longer supported. The caller
8984 * holds the link_lock mutex, so we can modify link_info settings.
8985 */
8986 if (bnxt_support_dropped(link_info->advertising,
8987 link_info->support_auto_speeds)) {
8988 link_info->advertising = link_info->support_auto_speeds;
8989 support_changed = true;
8990 }
8991 if (bnxt_support_dropped(link_info->advertising_pam4,
8992 link_info->support_pam4_auto_speeds)) {
8993 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
8994 support_changed = true;
8995 }
8996 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
8997 bnxt_hwrm_set_link_setting(bp, true, false);
8998 return 0;
8999 }
9000
9001 static void bnxt_get_port_module_status(struct bnxt *bp)
9002 {
9003 struct bnxt_link_info *link_info = &bp->link_info;
9004 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9005 u8 module_status;
9006
9007 if (bnxt_update_link(bp, true))
9008 return;
9009
9010 module_status = link_info->module_status;
9011 switch (module_status) {
9012 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9013 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9014 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9015 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9016 bp->pf.port_id);
9017 if (bp->hwrm_spec_code >= 0x10201) {
9018 netdev_warn(bp->dev, "Module part number %s\n",
9019 resp->phy_vendor_partnumber);
9020 }
9021 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9022 netdev_warn(bp->dev, "TX is disabled\n");
9023 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9024 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9025 }
9026 }
9027
9028 static void
9029 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9030 {
9031 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9032 if (bp->hwrm_spec_code >= 0x10201)
9033 req->auto_pause =
9034 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9035 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9036 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9037 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9038 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9039 req->enables |=
9040 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9041 } else {
9042 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9043 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9044 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9045 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9046 req->enables |=
9047 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9048 if (bp->hwrm_spec_code >= 0x10201) {
9049 req->auto_pause = req->force_pause;
9050 req->enables |= cpu_to_le32(
9051 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9052 }
9053 }
9054 }
9055
9056 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9057 {
9058 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9059 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9060 if (bp->link_info.advertising) {
9061 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9062 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9063 }
9064 if (bp->link_info.advertising_pam4) {
9065 req->enables |=
9066 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9067 req->auto_link_pam4_speed_mask =
9068 cpu_to_le16(bp->link_info.advertising_pam4);
9069 }
9070 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9071 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9072 } else {
9073 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9074 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9075 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9076 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9077 } else {
9078 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9079 }
9080 }
9081
9082 /* tell chimp that the setting takes effect immediately */
9083 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9084 }
9085
9086 int bnxt_hwrm_set_pause(struct bnxt *bp)
9087 {
9088 struct hwrm_port_phy_cfg_input req = {0};
9089 int rc;
9090
9091 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9092 bnxt_hwrm_set_pause_common(bp, &req);
9093
9094 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9095 bp->link_info.force_link_chng)
9096 bnxt_hwrm_set_link_common(bp, &req);
9097
9098 mutex_lock(&bp->hwrm_cmd_lock);
9099 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9100 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9101 /* since changing of pause setting doesn't trigger any link
9102 * change event, the driver needs to update the current pause
9103 * result upon successfully return of the phy_cfg command
9104 */
9105 bp->link_info.pause =
9106 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9107 bp->link_info.auto_pause_setting = 0;
9108 if (!bp->link_info.force_link_chng)
9109 bnxt_report_link(bp);
9110 }
9111 bp->link_info.force_link_chng = false;
9112 mutex_unlock(&bp->hwrm_cmd_lock);
9113 return rc;
9114 }
9115
9116 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9117 struct hwrm_port_phy_cfg_input *req)
9118 {
9119 struct ethtool_eee *eee = &bp->eee;
9120
9121 if (eee->eee_enabled) {
9122 u16 eee_speeds;
9123 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9124
9125 if (eee->tx_lpi_enabled)
9126 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9127 else
9128 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9129
9130 req->flags |= cpu_to_le32(flags);
9131 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9132 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9133 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9134 } else {
9135 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9136 }
9137 }
9138
9139 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9140 {
9141 struct hwrm_port_phy_cfg_input req = {0};
9142
9143 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9144 if (set_pause)
9145 bnxt_hwrm_set_pause_common(bp, &req);
9146
9147 bnxt_hwrm_set_link_common(bp, &req);
9148
9149 if (set_eee)
9150 bnxt_hwrm_set_eee(bp, &req);
9151 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9152 }
9153
9154 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9155 {
9156 struct hwrm_port_phy_cfg_input req = {0};
9157
9158 if (!BNXT_SINGLE_PF(bp))
9159 return 0;
9160
9161 if (pci_num_vf(bp->pdev))
9162 return 0;
9163
9164 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9165 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9166 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9167 }
9168
9169 static int bnxt_fw_init_one(struct bnxt *bp);
9170
9171 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9172 {
9173 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9174 struct hwrm_func_drv_if_change_input req = {0};
9175 bool resc_reinit = false, fw_reset = false;
9176 u32 flags = 0;
9177 int rc;
9178
9179 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9180 return 0;
9181
9182 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9183 if (up)
9184 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9185 mutex_lock(&bp->hwrm_cmd_lock);
9186 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9187 if (!rc)
9188 flags = le32_to_cpu(resp->flags);
9189 mutex_unlock(&bp->hwrm_cmd_lock);
9190 if (rc)
9191 return rc;
9192
9193 if (!up)
9194 return 0;
9195
9196 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9197 resc_reinit = true;
9198 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9199 fw_reset = true;
9200
9201 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9202 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9203 return -ENODEV;
9204 }
9205 if (resc_reinit || fw_reset) {
9206 if (fw_reset) {
9207 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9208 bnxt_ulp_stop(bp);
9209 bnxt_free_ctx_mem(bp);
9210 kfree(bp->ctx);
9211 bp->ctx = NULL;
9212 bnxt_dcb_free(bp);
9213 rc = bnxt_fw_init_one(bp);
9214 if (rc) {
9215 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9216 return rc;
9217 }
9218 bnxt_clear_int_mode(bp);
9219 rc = bnxt_init_int_mode(bp);
9220 if (rc) {
9221 netdev_err(bp->dev, "init int mode failed\n");
9222 return rc;
9223 }
9224 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9225 }
9226 if (BNXT_NEW_RM(bp)) {
9227 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9228
9229 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9230 hw_resc->resv_cp_rings = 0;
9231 hw_resc->resv_stat_ctxs = 0;
9232 hw_resc->resv_irqs = 0;
9233 hw_resc->resv_tx_rings = 0;
9234 hw_resc->resv_rx_rings = 0;
9235 hw_resc->resv_hw_ring_grps = 0;
9236 hw_resc->resv_vnics = 0;
9237 if (!fw_reset) {
9238 bp->tx_nr_rings = 0;
9239 bp->rx_nr_rings = 0;
9240 }
9241 }
9242 }
9243 return 0;
9244 }
9245
9246 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9247 {
9248 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9249 struct hwrm_port_led_qcaps_input req = {0};
9250 struct bnxt_pf_info *pf = &bp->pf;
9251 int rc;
9252
9253 bp->num_leds = 0;
9254 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9255 return 0;
9256
9257 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9258 req.port_id = cpu_to_le16(pf->port_id);
9259 mutex_lock(&bp->hwrm_cmd_lock);
9260 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9261 if (rc) {
9262 mutex_unlock(&bp->hwrm_cmd_lock);
9263 return rc;
9264 }
9265 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9266 int i;
9267
9268 bp->num_leds = resp->num_leds;
9269 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9270 bp->num_leds);
9271 for (i = 0; i < bp->num_leds; i++) {
9272 struct bnxt_led_info *led = &bp->leds[i];
9273 __le16 caps = led->led_state_caps;
9274
9275 if (!led->led_group_id ||
9276 !BNXT_LED_ALT_BLINK_CAP(caps)) {
9277 bp->num_leds = 0;
9278 break;
9279 }
9280 }
9281 }
9282 mutex_unlock(&bp->hwrm_cmd_lock);
9283 return 0;
9284 }
9285
9286 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9287 {
9288 struct hwrm_wol_filter_alloc_input req = {0};
9289 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9290 int rc;
9291
9292 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9293 req.port_id = cpu_to_le16(bp->pf.port_id);
9294 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9295 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9296 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9297 mutex_lock(&bp->hwrm_cmd_lock);
9298 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9299 if (!rc)
9300 bp->wol_filter_id = resp->wol_filter_id;
9301 mutex_unlock(&bp->hwrm_cmd_lock);
9302 return rc;
9303 }
9304
9305 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9306 {
9307 struct hwrm_wol_filter_free_input req = {0};
9308
9309 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9310 req.port_id = cpu_to_le16(bp->pf.port_id);
9311 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9312 req.wol_filter_id = bp->wol_filter_id;
9313 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9314 }
9315
9316 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9317 {
9318 struct hwrm_wol_filter_qcfg_input req = {0};
9319 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9320 u16 next_handle = 0;
9321 int rc;
9322
9323 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9324 req.port_id = cpu_to_le16(bp->pf.port_id);
9325 req.handle = cpu_to_le16(handle);
9326 mutex_lock(&bp->hwrm_cmd_lock);
9327 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9328 if (!rc) {
9329 next_handle = le16_to_cpu(resp->next_handle);
9330 if (next_handle != 0) {
9331 if (resp->wol_type ==
9332 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9333 bp->wol = 1;
9334 bp->wol_filter_id = resp->wol_filter_id;
9335 }
9336 }
9337 }
9338 mutex_unlock(&bp->hwrm_cmd_lock);
9339 return next_handle;
9340 }
9341
9342 static void bnxt_get_wol_settings(struct bnxt *bp)
9343 {
9344 u16 handle = 0;
9345
9346 bp->wol = 0;
9347 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9348 return;
9349
9350 do {
9351 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9352 } while (handle && handle != 0xffff);
9353 }
9354
9355 #ifdef CONFIG_BNXT_HWMON
9356 static ssize_t bnxt_show_temp(struct device *dev,
9357 struct device_attribute *devattr, char *buf)
9358 {
9359 struct hwrm_temp_monitor_query_input req = {0};
9360 struct hwrm_temp_monitor_query_output *resp;
9361 struct bnxt *bp = dev_get_drvdata(dev);
9362 u32 len = 0;
9363 int rc;
9364
9365 resp = bp->hwrm_cmd_resp_addr;
9366 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9367 mutex_lock(&bp->hwrm_cmd_lock);
9368 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9369 if (!rc)
9370 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9371 mutex_unlock(&bp->hwrm_cmd_lock);
9372 return rc ?: len;
9373 }
9374 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9375
9376 static struct attribute *bnxt_attrs[] = {
9377 &sensor_dev_attr_temp1_input.dev_attr.attr,
9378 NULL
9379 };
9380 ATTRIBUTE_GROUPS(bnxt);
9381
9382 static void bnxt_hwmon_close(struct bnxt *bp)
9383 {
9384 if (bp->hwmon_dev) {
9385 hwmon_device_unregister(bp->hwmon_dev);
9386 bp->hwmon_dev = NULL;
9387 }
9388 }
9389
9390 static void bnxt_hwmon_open(struct bnxt *bp)
9391 {
9392 struct hwrm_temp_monitor_query_input req = {0};
9393 struct pci_dev *pdev = bp->pdev;
9394 int rc;
9395
9396 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9397 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9398 if (rc == -EACCES || rc == -EOPNOTSUPP) {
9399 bnxt_hwmon_close(bp);
9400 return;
9401 }
9402
9403 if (bp->hwmon_dev)
9404 return;
9405
9406 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9407 DRV_MODULE_NAME, bp,
9408 bnxt_groups);
9409 if (IS_ERR(bp->hwmon_dev)) {
9410 bp->hwmon_dev = NULL;
9411 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9412 }
9413 }
9414 #else
9415 static void bnxt_hwmon_close(struct bnxt *bp)
9416 {
9417 }
9418
9419 static void bnxt_hwmon_open(struct bnxt *bp)
9420 {
9421 }
9422 #endif
9423
9424 static bool bnxt_eee_config_ok(struct bnxt *bp)
9425 {
9426 struct ethtool_eee *eee = &bp->eee;
9427 struct bnxt_link_info *link_info = &bp->link_info;
9428
9429 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9430 return true;
9431
9432 if (eee->eee_enabled) {
9433 u32 advertising =
9434 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9435
9436 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9437 eee->eee_enabled = 0;
9438 return false;
9439 }
9440 if (eee->advertised & ~advertising) {
9441 eee->advertised = advertising & eee->supported;
9442 return false;
9443 }
9444 }
9445 return true;
9446 }
9447
9448 static int bnxt_update_phy_setting(struct bnxt *bp)
9449 {
9450 int rc;
9451 bool update_link = false;
9452 bool update_pause = false;
9453 bool update_eee = false;
9454 struct bnxt_link_info *link_info = &bp->link_info;
9455
9456 rc = bnxt_update_link(bp, true);
9457 if (rc) {
9458 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9459 rc);
9460 return rc;
9461 }
9462 if (!BNXT_SINGLE_PF(bp))
9463 return 0;
9464
9465 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9466 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9467 link_info->req_flow_ctrl)
9468 update_pause = true;
9469 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9470 link_info->force_pause_setting != link_info->req_flow_ctrl)
9471 update_pause = true;
9472 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9473 if (BNXT_AUTO_MODE(link_info->auto_mode))
9474 update_link = true;
9475 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9476 link_info->req_link_speed != link_info->force_link_speed)
9477 update_link = true;
9478 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9479 link_info->req_link_speed != link_info->force_pam4_link_speed)
9480 update_link = true;
9481 if (link_info->req_duplex != link_info->duplex_setting)
9482 update_link = true;
9483 } else {
9484 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9485 update_link = true;
9486 if (link_info->advertising != link_info->auto_link_speeds ||
9487 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9488 update_link = true;
9489 }
9490
9491 /* The last close may have shutdown the link, so need to call
9492 * PHY_CFG to bring it back up.
9493 */
9494 if (!bp->link_info.link_up)
9495 update_link = true;
9496
9497 if (!bnxt_eee_config_ok(bp))
9498 update_eee = true;
9499
9500 if (update_link)
9501 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9502 else if (update_pause)
9503 rc = bnxt_hwrm_set_pause(bp);
9504 if (rc) {
9505 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9506 rc);
9507 return rc;
9508 }
9509
9510 return rc;
9511 }
9512
9513 /* Common routine to pre-map certain register block to different GRC window.
9514 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9515 * in PF and 3 windows in VF that can be customized to map in different
9516 * register blocks.
9517 */
9518 static void bnxt_preset_reg_win(struct bnxt *bp)
9519 {
9520 if (BNXT_PF(bp)) {
9521 /* CAG registers map to GRC window #4 */
9522 writel(BNXT_CAG_REG_BASE,
9523 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9524 }
9525 }
9526
9527 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9528
9529 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9530 {
9531 int rc = 0;
9532
9533 bnxt_preset_reg_win(bp);
9534 netif_carrier_off(bp->dev);
9535 if (irq_re_init) {
9536 /* Reserve rings now if none were reserved at driver probe. */
9537 rc = bnxt_init_dflt_ring_mode(bp);
9538 if (rc) {
9539 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9540 return rc;
9541 }
9542 }
9543 rc = bnxt_reserve_rings(bp, irq_re_init);
9544 if (rc)
9545 return rc;
9546 if ((bp->flags & BNXT_FLAG_RFS) &&
9547 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9548 /* disable RFS if falling back to INTA */
9549 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9550 bp->flags &= ~BNXT_FLAG_RFS;
9551 }
9552
9553 rc = bnxt_alloc_mem(bp, irq_re_init);
9554 if (rc) {
9555 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9556 goto open_err_free_mem;
9557 }
9558
9559 if (irq_re_init) {
9560 bnxt_init_napi(bp);
9561 rc = bnxt_request_irq(bp);
9562 if (rc) {
9563 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9564 goto open_err_irq;
9565 }
9566 }
9567
9568 rc = bnxt_init_nic(bp, irq_re_init);
9569 if (rc) {
9570 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9571 goto open_err_irq;
9572 }
9573
9574 bnxt_enable_napi(bp);
9575 bnxt_debug_dev_init(bp);
9576
9577 if (link_re_init) {
9578 mutex_lock(&bp->link_lock);
9579 rc = bnxt_update_phy_setting(bp);
9580 mutex_unlock(&bp->link_lock);
9581 if (rc) {
9582 netdev_warn(bp->dev, "failed to update phy settings\n");
9583 if (BNXT_SINGLE_PF(bp)) {
9584 bp->link_info.phy_retry = true;
9585 bp->link_info.phy_retry_expires =
9586 jiffies + 5 * HZ;
9587 }
9588 }
9589 }
9590
9591 if (irq_re_init)
9592 udp_tunnel_nic_reset_ntf(bp->dev);
9593
9594 set_bit(BNXT_STATE_OPEN, &bp->state);
9595 bnxt_enable_int(bp);
9596 /* Enable TX queues */
9597 bnxt_tx_enable(bp);
9598 mod_timer(&bp->timer, jiffies + bp->current_interval);
9599 /* Poll link status and check for SFP+ module status */
9600 bnxt_get_port_module_status(bp);
9601
9602 /* VF-reps may need to be re-opened after the PF is re-opened */
9603 if (BNXT_PF(bp))
9604 bnxt_vf_reps_open(bp);
9605 return 0;
9606
9607 open_err_irq:
9608 bnxt_del_napi(bp);
9609
9610 open_err_free_mem:
9611 bnxt_free_skbs(bp);
9612 bnxt_free_irq(bp);
9613 bnxt_free_mem(bp, true);
9614 return rc;
9615 }
9616
9617 /* rtnl_lock held */
9618 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9619 {
9620 int rc = 0;
9621
9622 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9623 if (rc) {
9624 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9625 dev_close(bp->dev);
9626 }
9627 return rc;
9628 }
9629
9630 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9631 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
9632 * self tests.
9633 */
9634 int bnxt_half_open_nic(struct bnxt *bp)
9635 {
9636 int rc = 0;
9637
9638 rc = bnxt_alloc_mem(bp, false);
9639 if (rc) {
9640 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9641 goto half_open_err;
9642 }
9643 rc = bnxt_init_nic(bp, false);
9644 if (rc) {
9645 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9646 goto half_open_err;
9647 }
9648 return 0;
9649
9650 half_open_err:
9651 bnxt_free_skbs(bp);
9652 bnxt_free_mem(bp, false);
9653 dev_close(bp->dev);
9654 return rc;
9655 }
9656
9657 /* rtnl_lock held, this call can only be made after a previous successful
9658 * call to bnxt_half_open_nic().
9659 */
9660 void bnxt_half_close_nic(struct bnxt *bp)
9661 {
9662 bnxt_hwrm_resource_free(bp, false, false);
9663 bnxt_free_skbs(bp);
9664 bnxt_free_mem(bp, false);
9665 }
9666
9667 static void bnxt_reenable_sriov(struct bnxt *bp)
9668 {
9669 if (BNXT_PF(bp)) {
9670 struct bnxt_pf_info *pf = &bp->pf;
9671 int n = pf->active_vfs;
9672
9673 if (n)
9674 bnxt_cfg_hw_sriov(bp, &n, true);
9675 }
9676 }
9677
9678 static int bnxt_open(struct net_device *dev)
9679 {
9680 struct bnxt *bp = netdev_priv(dev);
9681 int rc;
9682
9683 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9684 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9685 return -ENODEV;
9686 }
9687
9688 rc = bnxt_hwrm_if_change(bp, true);
9689 if (rc)
9690 return rc;
9691 rc = __bnxt_open_nic(bp, true, true);
9692 if (rc) {
9693 bnxt_hwrm_if_change(bp, false);
9694 } else {
9695 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9696 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9697 bnxt_ulp_start(bp, 0);
9698 bnxt_reenable_sriov(bp);
9699 }
9700 }
9701 bnxt_hwmon_open(bp);
9702 }
9703
9704 return rc;
9705 }
9706
9707 static bool bnxt_drv_busy(struct bnxt *bp)
9708 {
9709 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9710 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9711 }
9712
9713 static void bnxt_get_ring_stats(struct bnxt *bp,
9714 struct rtnl_link_stats64 *stats);
9715
9716 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9717 bool link_re_init)
9718 {
9719 /* Close the VF-reps before closing PF */
9720 if (BNXT_PF(bp))
9721 bnxt_vf_reps_close(bp);
9722
9723 /* Change device state to avoid TX queue wake up's */
9724 bnxt_tx_disable(bp);
9725
9726 clear_bit(BNXT_STATE_OPEN, &bp->state);
9727 smp_mb__after_atomic();
9728 while (bnxt_drv_busy(bp))
9729 msleep(20);
9730
9731 /* Flush rings and and disable interrupts */
9732 bnxt_shutdown_nic(bp, irq_re_init);
9733
9734 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9735
9736 bnxt_debug_dev_exit(bp);
9737 bnxt_disable_napi(bp);
9738 del_timer_sync(&bp->timer);
9739 bnxt_free_skbs(bp);
9740
9741 /* Save ring stats before shutdown */
9742 if (bp->bnapi && irq_re_init)
9743 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9744 if (irq_re_init) {
9745 bnxt_free_irq(bp);
9746 bnxt_del_napi(bp);
9747 }
9748 bnxt_free_mem(bp, irq_re_init);
9749 }
9750
9751 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9752 {
9753 int rc = 0;
9754
9755 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9756 /* If we get here, it means firmware reset is in progress
9757 * while we are trying to close. We can safely proceed with
9758 * the close because we are holding rtnl_lock(). Some firmware
9759 * messages may fail as we proceed to close. We set the
9760 * ABORT_ERR flag here so that the FW reset thread will later
9761 * abort when it gets the rtnl_lock() and sees the flag.
9762 */
9763 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9764 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9765 }
9766
9767 #ifdef CONFIG_BNXT_SRIOV
9768 if (bp->sriov_cfg) {
9769 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9770 !bp->sriov_cfg,
9771 BNXT_SRIOV_CFG_WAIT_TMO);
9772 if (rc)
9773 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9774 }
9775 #endif
9776 __bnxt_close_nic(bp, irq_re_init, link_re_init);
9777 return rc;
9778 }
9779
9780 static int bnxt_close(struct net_device *dev)
9781 {
9782 struct bnxt *bp = netdev_priv(dev);
9783
9784 bnxt_hwmon_close(bp);
9785 bnxt_close_nic(bp, true, true);
9786 bnxt_hwrm_shutdown_link(bp);
9787 bnxt_hwrm_if_change(bp, false);
9788 return 0;
9789 }
9790
9791 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9792 u16 *val)
9793 {
9794 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9795 struct hwrm_port_phy_mdio_read_input req = {0};
9796 int rc;
9797
9798 if (bp->hwrm_spec_code < 0x10a00)
9799 return -EOPNOTSUPP;
9800
9801 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9802 req.port_id = cpu_to_le16(bp->pf.port_id);
9803 req.phy_addr = phy_addr;
9804 req.reg_addr = cpu_to_le16(reg & 0x1f);
9805 if (mdio_phy_id_is_c45(phy_addr)) {
9806 req.cl45_mdio = 1;
9807 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9808 req.dev_addr = mdio_phy_id_devad(phy_addr);
9809 req.reg_addr = cpu_to_le16(reg);
9810 }
9811
9812 mutex_lock(&bp->hwrm_cmd_lock);
9813 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9814 if (!rc)
9815 *val = le16_to_cpu(resp->reg_data);
9816 mutex_unlock(&bp->hwrm_cmd_lock);
9817 return rc;
9818 }
9819
9820 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9821 u16 val)
9822 {
9823 struct hwrm_port_phy_mdio_write_input req = {0};
9824
9825 if (bp->hwrm_spec_code < 0x10a00)
9826 return -EOPNOTSUPP;
9827
9828 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9829 req.port_id = cpu_to_le16(bp->pf.port_id);
9830 req.phy_addr = phy_addr;
9831 req.reg_addr = cpu_to_le16(reg & 0x1f);
9832 if (mdio_phy_id_is_c45(phy_addr)) {
9833 req.cl45_mdio = 1;
9834 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9835 req.dev_addr = mdio_phy_id_devad(phy_addr);
9836 req.reg_addr = cpu_to_le16(reg);
9837 }
9838 req.reg_data = cpu_to_le16(val);
9839
9840 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9841 }
9842
9843 /* rtnl_lock held */
9844 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9845 {
9846 struct mii_ioctl_data *mdio = if_mii(ifr);
9847 struct bnxt *bp = netdev_priv(dev);
9848 int rc;
9849
9850 switch (cmd) {
9851 case SIOCGMIIPHY:
9852 mdio->phy_id = bp->link_info.phy_addr;
9853
9854 fallthrough;
9855 case SIOCGMIIREG: {
9856 u16 mii_regval = 0;
9857
9858 if (!netif_running(dev))
9859 return -EAGAIN;
9860
9861 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9862 &mii_regval);
9863 mdio->val_out = mii_regval;
9864 return rc;
9865 }
9866
9867 case SIOCSMIIREG:
9868 if (!netif_running(dev))
9869 return -EAGAIN;
9870
9871 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9872 mdio->val_in);
9873
9874 default:
9875 /* do nothing */
9876 break;
9877 }
9878 return -EOPNOTSUPP;
9879 }
9880
9881 static void bnxt_get_ring_stats(struct bnxt *bp,
9882 struct rtnl_link_stats64 *stats)
9883 {
9884 int i;
9885
9886 for (i = 0; i < bp->cp_nr_rings; i++) {
9887 struct bnxt_napi *bnapi = bp->bnapi[i];
9888 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9889 u64 *sw = cpr->stats.sw_stats;
9890
9891 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
9892 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9893 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
9894
9895 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
9896 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
9897 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
9898
9899 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
9900 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
9901 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
9902
9903 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
9904 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
9905 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
9906
9907 stats->rx_missed_errors +=
9908 BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
9909
9910 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9911
9912 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
9913 }
9914 }
9915
9916 static void bnxt_add_prev_stats(struct bnxt *bp,
9917 struct rtnl_link_stats64 *stats)
9918 {
9919 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9920
9921 stats->rx_packets += prev_stats->rx_packets;
9922 stats->tx_packets += prev_stats->tx_packets;
9923 stats->rx_bytes += prev_stats->rx_bytes;
9924 stats->tx_bytes += prev_stats->tx_bytes;
9925 stats->rx_missed_errors += prev_stats->rx_missed_errors;
9926 stats->multicast += prev_stats->multicast;
9927 stats->tx_dropped += prev_stats->tx_dropped;
9928 }
9929
9930 static void
9931 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9932 {
9933 struct bnxt *bp = netdev_priv(dev);
9934
9935 set_bit(BNXT_STATE_READ_STATS, &bp->state);
9936 /* Make sure bnxt_close_nic() sees that we are reading stats before
9937 * we check the BNXT_STATE_OPEN flag.
9938 */
9939 smp_mb__after_atomic();
9940 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9941 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9942 *stats = bp->net_stats_prev;
9943 return;
9944 }
9945
9946 bnxt_get_ring_stats(bp, stats);
9947 bnxt_add_prev_stats(bp, stats);
9948
9949 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9950 u64 *rx = bp->port_stats.sw_stats;
9951 u64 *tx = bp->port_stats.sw_stats +
9952 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9953
9954 stats->rx_crc_errors =
9955 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
9956 stats->rx_frame_errors =
9957 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
9958 stats->rx_length_errors =
9959 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
9960 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
9961 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
9962 stats->rx_errors =
9963 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
9964 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
9965 stats->collisions =
9966 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
9967 stats->tx_fifo_errors =
9968 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
9969 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
9970 }
9971 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9972 }
9973
9974 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9975 {
9976 struct net_device *dev = bp->dev;
9977 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9978 struct netdev_hw_addr *ha;
9979 u8 *haddr;
9980 int mc_count = 0;
9981 bool update = false;
9982 int off = 0;
9983
9984 netdev_for_each_mc_addr(ha, dev) {
9985 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9986 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9987 vnic->mc_list_count = 0;
9988 return false;
9989 }
9990 haddr = ha->addr;
9991 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9992 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9993 update = true;
9994 }
9995 off += ETH_ALEN;
9996 mc_count++;
9997 }
9998 if (mc_count)
9999 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10000
10001 if (mc_count != vnic->mc_list_count) {
10002 vnic->mc_list_count = mc_count;
10003 update = true;
10004 }
10005 return update;
10006 }
10007
10008 static bool bnxt_uc_list_updated(struct bnxt *bp)
10009 {
10010 struct net_device *dev = bp->dev;
10011 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10012 struct netdev_hw_addr *ha;
10013 int off = 0;
10014
10015 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10016 return true;
10017
10018 netdev_for_each_uc_addr(ha, dev) {
10019 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10020 return true;
10021
10022 off += ETH_ALEN;
10023 }
10024 return false;
10025 }
10026
10027 static void bnxt_set_rx_mode(struct net_device *dev)
10028 {
10029 struct bnxt *bp = netdev_priv(dev);
10030 struct bnxt_vnic_info *vnic;
10031 bool mc_update = false;
10032 bool uc_update;
10033 u32 mask;
10034
10035 if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10036 return;
10037
10038 vnic = &bp->vnic_info[0];
10039 mask = vnic->rx_mask;
10040 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10041 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10042 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10043 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10044
10045 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10046 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10047
10048 uc_update = bnxt_uc_list_updated(bp);
10049
10050 if (dev->flags & IFF_BROADCAST)
10051 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10052 if (dev->flags & IFF_ALLMULTI) {
10053 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10054 vnic->mc_list_count = 0;
10055 } else {
10056 mc_update = bnxt_mc_list_updated(bp, &mask);
10057 }
10058
10059 if (mask != vnic->rx_mask || uc_update || mc_update) {
10060 vnic->rx_mask = mask;
10061
10062 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10063 bnxt_queue_sp_work(bp);
10064 }
10065 }
10066
10067 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10068 {
10069 struct net_device *dev = bp->dev;
10070 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10071 struct netdev_hw_addr *ha;
10072 int i, off = 0, rc;
10073 bool uc_update;
10074
10075 netif_addr_lock_bh(dev);
10076 uc_update = bnxt_uc_list_updated(bp);
10077 netif_addr_unlock_bh(dev);
10078
10079 if (!uc_update)
10080 goto skip_uc;
10081
10082 mutex_lock(&bp->hwrm_cmd_lock);
10083 for (i = 1; i < vnic->uc_filter_count; i++) {
10084 struct hwrm_cfa_l2_filter_free_input req = {0};
10085
10086 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10087 -1);
10088
10089 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10090
10091 rc = _hwrm_send_message(bp, &req, sizeof(req),
10092 HWRM_CMD_TIMEOUT);
10093 }
10094 mutex_unlock(&bp->hwrm_cmd_lock);
10095
10096 vnic->uc_filter_count = 1;
10097
10098 netif_addr_lock_bh(dev);
10099 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10100 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10101 } else {
10102 netdev_for_each_uc_addr(ha, dev) {
10103 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10104 off += ETH_ALEN;
10105 vnic->uc_filter_count++;
10106 }
10107 }
10108 netif_addr_unlock_bh(dev);
10109
10110 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10111 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10112 if (rc) {
10113 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10114 rc);
10115 vnic->uc_filter_count = i;
10116 return rc;
10117 }
10118 }
10119
10120 skip_uc:
10121 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10122 if (rc && vnic->mc_list_count) {
10123 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10124 rc);
10125 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10126 vnic->mc_list_count = 0;
10127 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10128 }
10129 if (rc)
10130 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10131 rc);
10132
10133 return rc;
10134 }
10135
10136 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10137 {
10138 #ifdef CONFIG_BNXT_SRIOV
10139 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10140 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10141
10142 /* No minimum rings were provisioned by the PF. Don't
10143 * reserve rings by default when device is down.
10144 */
10145 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10146 return true;
10147
10148 if (!netif_running(bp->dev))
10149 return false;
10150 }
10151 #endif
10152 return true;
10153 }
10154
10155 /* If the chip and firmware supports RFS */
10156 static bool bnxt_rfs_supported(struct bnxt *bp)
10157 {
10158 if (bp->flags & BNXT_FLAG_CHIP_P5) {
10159 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10160 return true;
10161 return false;
10162 }
10163 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10164 return true;
10165 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10166 return true;
10167 return false;
10168 }
10169
10170 /* If runtime conditions support RFS */
10171 static bool bnxt_rfs_capable(struct bnxt *bp)
10172 {
10173 #ifdef CONFIG_RFS_ACCEL
10174 int vnics, max_vnics, max_rss_ctxs;
10175
10176 if (bp->flags & BNXT_FLAG_CHIP_P5)
10177 return bnxt_rfs_supported(bp);
10178 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10179 return false;
10180
10181 vnics = 1 + bp->rx_nr_rings;
10182 max_vnics = bnxt_get_max_func_vnics(bp);
10183 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10184
10185 /* RSS contexts not a limiting factor */
10186 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10187 max_rss_ctxs = max_vnics;
10188 if (vnics > max_vnics || vnics > max_rss_ctxs) {
10189 if (bp->rx_nr_rings > 1)
10190 netdev_warn(bp->dev,
10191 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10192 min(max_rss_ctxs - 1, max_vnics - 1));
10193 return false;
10194 }
10195
10196 if (!BNXT_NEW_RM(bp))
10197 return true;
10198
10199 if (vnics == bp->hw_resc.resv_vnics)
10200 return true;
10201
10202 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10203 if (vnics <= bp->hw_resc.resv_vnics)
10204 return true;
10205
10206 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10207 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10208 return false;
10209 #else
10210 return false;
10211 #endif
10212 }
10213
10214 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10215 netdev_features_t features)
10216 {
10217 struct bnxt *bp = netdev_priv(dev);
10218 netdev_features_t vlan_features;
10219
10220 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10221 features &= ~NETIF_F_NTUPLE;
10222
10223 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10224 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10225
10226 if (!(features & NETIF_F_GRO))
10227 features &= ~NETIF_F_GRO_HW;
10228
10229 if (features & NETIF_F_GRO_HW)
10230 features &= ~NETIF_F_LRO;
10231
10232 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10233 * turned on or off together.
10234 */
10235 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10236 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10237 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10238 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10239 else if (vlan_features)
10240 features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10241 }
10242 #ifdef CONFIG_BNXT_SRIOV
10243 if (BNXT_VF(bp) && bp->vf.vlan)
10244 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10245 #endif
10246 return features;
10247 }
10248
10249 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10250 {
10251 struct bnxt *bp = netdev_priv(dev);
10252 u32 flags = bp->flags;
10253 u32 changes;
10254 int rc = 0;
10255 bool re_init = false;
10256 bool update_tpa = false;
10257
10258 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10259 if (features & NETIF_F_GRO_HW)
10260 flags |= BNXT_FLAG_GRO;
10261 else if (features & NETIF_F_LRO)
10262 flags |= BNXT_FLAG_LRO;
10263
10264 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10265 flags &= ~BNXT_FLAG_TPA;
10266
10267 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10268 flags |= BNXT_FLAG_STRIP_VLAN;
10269
10270 if (features & NETIF_F_NTUPLE)
10271 flags |= BNXT_FLAG_RFS;
10272
10273 changes = flags ^ bp->flags;
10274 if (changes & BNXT_FLAG_TPA) {
10275 update_tpa = true;
10276 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10277 (flags & BNXT_FLAG_TPA) == 0 ||
10278 (bp->flags & BNXT_FLAG_CHIP_P5))
10279 re_init = true;
10280 }
10281
10282 if (changes & ~BNXT_FLAG_TPA)
10283 re_init = true;
10284
10285 if (flags != bp->flags) {
10286 u32 old_flags = bp->flags;
10287
10288 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10289 bp->flags = flags;
10290 if (update_tpa)
10291 bnxt_set_ring_params(bp);
10292 return rc;
10293 }
10294
10295 if (re_init) {
10296 bnxt_close_nic(bp, false, false);
10297 bp->flags = flags;
10298 if (update_tpa)
10299 bnxt_set_ring_params(bp);
10300
10301 return bnxt_open_nic(bp, false, false);
10302 }
10303 if (update_tpa) {
10304 bp->flags = flags;
10305 rc = bnxt_set_tpa(bp,
10306 (flags & BNXT_FLAG_TPA) ?
10307 true : false);
10308 if (rc)
10309 bp->flags = old_flags;
10310 }
10311 }
10312 return rc;
10313 }
10314
10315 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10316 u32 *reg_buf)
10317 {
10318 struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10319 struct hwrm_dbg_read_direct_input req = {0};
10320 __le32 *dbg_reg_buf;
10321 dma_addr_t mapping;
10322 int rc, i;
10323
10324 dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10325 &mapping, GFP_KERNEL);
10326 if (!dbg_reg_buf)
10327 return -ENOMEM;
10328 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10329 req.host_dest_addr = cpu_to_le64(mapping);
10330 req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10331 req.read_len32 = cpu_to_le32(num_words);
10332 mutex_lock(&bp->hwrm_cmd_lock);
10333 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10334 if (rc || resp->error_code) {
10335 rc = -EIO;
10336 goto dbg_rd_reg_exit;
10337 }
10338 for (i = 0; i < num_words; i++)
10339 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10340
10341 dbg_rd_reg_exit:
10342 mutex_unlock(&bp->hwrm_cmd_lock);
10343 dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10344 return rc;
10345 }
10346
10347 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10348 u32 ring_id, u32 *prod, u32 *cons)
10349 {
10350 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10351 struct hwrm_dbg_ring_info_get_input req = {0};
10352 int rc;
10353
10354 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10355 req.ring_type = ring_type;
10356 req.fw_ring_id = cpu_to_le32(ring_id);
10357 mutex_lock(&bp->hwrm_cmd_lock);
10358 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10359 if (!rc) {
10360 *prod = le32_to_cpu(resp->producer_index);
10361 *cons = le32_to_cpu(resp->consumer_index);
10362 }
10363 mutex_unlock(&bp->hwrm_cmd_lock);
10364 return rc;
10365 }
10366
10367 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10368 {
10369 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10370 int i = bnapi->index;
10371
10372 if (!txr)
10373 return;
10374
10375 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10376 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10377 txr->tx_cons);
10378 }
10379
10380 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10381 {
10382 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10383 int i = bnapi->index;
10384
10385 if (!rxr)
10386 return;
10387
10388 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10389 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10390 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10391 rxr->rx_sw_agg_prod);
10392 }
10393
10394 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10395 {
10396 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10397 int i = bnapi->index;
10398
10399 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10400 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10401 }
10402
10403 static void bnxt_dbg_dump_states(struct bnxt *bp)
10404 {
10405 int i;
10406 struct bnxt_napi *bnapi;
10407
10408 for (i = 0; i < bp->cp_nr_rings; i++) {
10409 bnapi = bp->bnapi[i];
10410 if (netif_msg_drv(bp)) {
10411 bnxt_dump_tx_sw_state(bnapi);
10412 bnxt_dump_rx_sw_state(bnapi);
10413 bnxt_dump_cp_sw_state(bnapi);
10414 }
10415 }
10416 }
10417
10418 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10419 {
10420 if (!silent)
10421 bnxt_dbg_dump_states(bp);
10422 if (netif_running(bp->dev)) {
10423 int rc;
10424
10425 if (silent) {
10426 bnxt_close_nic(bp, false, false);
10427 bnxt_open_nic(bp, false, false);
10428 } else {
10429 bnxt_ulp_stop(bp);
10430 bnxt_close_nic(bp, true, false);
10431 rc = bnxt_open_nic(bp, true, false);
10432 bnxt_ulp_start(bp, rc);
10433 }
10434 }
10435 }
10436
10437 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10438 {
10439 struct bnxt *bp = netdev_priv(dev);
10440
10441 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
10442 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10443 bnxt_queue_sp_work(bp);
10444 }
10445
10446 static void bnxt_fw_health_check(struct bnxt *bp)
10447 {
10448 struct bnxt_fw_health *fw_health = bp->fw_health;
10449 u32 val;
10450
10451 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10452 return;
10453
10454 if (fw_health->tmr_counter) {
10455 fw_health->tmr_counter--;
10456 return;
10457 }
10458
10459 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10460 if (val == fw_health->last_fw_heartbeat)
10461 goto fw_reset;
10462
10463 fw_health->last_fw_heartbeat = val;
10464
10465 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10466 if (val != fw_health->last_fw_reset_cnt)
10467 goto fw_reset;
10468
10469 fw_health->tmr_counter = fw_health->tmr_multiplier;
10470 return;
10471
10472 fw_reset:
10473 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10474 bnxt_queue_sp_work(bp);
10475 }
10476
10477 static void bnxt_timer(struct timer_list *t)
10478 {
10479 struct bnxt *bp = from_timer(bp, t, timer);
10480 struct net_device *dev = bp->dev;
10481
10482 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10483 return;
10484
10485 if (atomic_read(&bp->intr_sem) != 0)
10486 goto bnxt_restart_timer;
10487
10488 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10489 bnxt_fw_health_check(bp);
10490
10491 if (bp->link_info.link_up && bp->stats_coal_ticks) {
10492 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10493 bnxt_queue_sp_work(bp);
10494 }
10495
10496 if (bnxt_tc_flower_enabled(bp)) {
10497 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10498 bnxt_queue_sp_work(bp);
10499 }
10500
10501 #ifdef CONFIG_RFS_ACCEL
10502 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10503 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10504 bnxt_queue_sp_work(bp);
10505 }
10506 #endif /*CONFIG_RFS_ACCEL*/
10507
10508 if (bp->link_info.phy_retry) {
10509 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10510 bp->link_info.phy_retry = false;
10511 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10512 } else {
10513 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10514 bnxt_queue_sp_work(bp);
10515 }
10516 }
10517
10518 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10519 netif_carrier_ok(dev)) {
10520 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10521 bnxt_queue_sp_work(bp);
10522 }
10523 bnxt_restart_timer:
10524 mod_timer(&bp->timer, jiffies + bp->current_interval);
10525 }
10526
10527 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10528 {
10529 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10530 * set. If the device is being closed, bnxt_close() may be holding
10531 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
10532 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10533 */
10534 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10535 rtnl_lock();
10536 }
10537
10538 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10539 {
10540 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10541 rtnl_unlock();
10542 }
10543
10544 /* Only called from bnxt_sp_task() */
10545 static void bnxt_reset(struct bnxt *bp, bool silent)
10546 {
10547 bnxt_rtnl_lock_sp(bp);
10548 if (test_bit(BNXT_STATE_OPEN, &bp->state))
10549 bnxt_reset_task(bp, silent);
10550 bnxt_rtnl_unlock_sp(bp);
10551 }
10552
10553 static void bnxt_fw_reset_close(struct bnxt *bp)
10554 {
10555 bnxt_ulp_stop(bp);
10556 /* When firmware is fatal state, disable PCI device to prevent
10557 * any potential bad DMAs before freeing kernel memory.
10558 */
10559 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10560 pci_disable_device(bp->pdev);
10561 __bnxt_close_nic(bp, true, false);
10562 bnxt_clear_int_mode(bp);
10563 bnxt_hwrm_func_drv_unrgtr(bp);
10564 if (pci_is_enabled(bp->pdev))
10565 pci_disable_device(bp->pdev);
10566 bnxt_free_ctx_mem(bp);
10567 kfree(bp->ctx);
10568 bp->ctx = NULL;
10569 }
10570
10571 static bool is_bnxt_fw_ok(struct bnxt *bp)
10572 {
10573 struct bnxt_fw_health *fw_health = bp->fw_health;
10574 bool no_heartbeat = false, has_reset = false;
10575 u32 val;
10576
10577 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10578 if (val == fw_health->last_fw_heartbeat)
10579 no_heartbeat = true;
10580
10581 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10582 if (val != fw_health->last_fw_reset_cnt)
10583 has_reset = true;
10584
10585 if (!no_heartbeat && has_reset)
10586 return true;
10587
10588 return false;
10589 }
10590
10591 /* rtnl_lock is acquired before calling this function */
10592 static void bnxt_force_fw_reset(struct bnxt *bp)
10593 {
10594 struct bnxt_fw_health *fw_health = bp->fw_health;
10595 u32 wait_dsecs;
10596
10597 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10598 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10599 return;
10600
10601 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10602 bnxt_fw_reset_close(bp);
10603 wait_dsecs = fw_health->master_func_wait_dsecs;
10604 if (fw_health->master) {
10605 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10606 wait_dsecs = 0;
10607 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10608 } else {
10609 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10610 wait_dsecs = fw_health->normal_func_wait_dsecs;
10611 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10612 }
10613
10614 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10615 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10616 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10617 }
10618
10619 void bnxt_fw_exception(struct bnxt *bp)
10620 {
10621 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10622 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10623 bnxt_rtnl_lock_sp(bp);
10624 bnxt_force_fw_reset(bp);
10625 bnxt_rtnl_unlock_sp(bp);
10626 }
10627
10628 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10629 * < 0 on error.
10630 */
10631 static int bnxt_get_registered_vfs(struct bnxt *bp)
10632 {
10633 #ifdef CONFIG_BNXT_SRIOV
10634 int rc;
10635
10636 if (!BNXT_PF(bp))
10637 return 0;
10638
10639 rc = bnxt_hwrm_func_qcfg(bp);
10640 if (rc) {
10641 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10642 return rc;
10643 }
10644 if (bp->pf.registered_vfs)
10645 return bp->pf.registered_vfs;
10646 if (bp->sriov_cfg)
10647 return 1;
10648 #endif
10649 return 0;
10650 }
10651
10652 void bnxt_fw_reset(struct bnxt *bp)
10653 {
10654 bnxt_rtnl_lock_sp(bp);
10655 if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10656 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10657 int n = 0, tmo;
10658
10659 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10660 if (bp->pf.active_vfs &&
10661 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10662 n = bnxt_get_registered_vfs(bp);
10663 if (n < 0) {
10664 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10665 n);
10666 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10667 dev_close(bp->dev);
10668 goto fw_reset_exit;
10669 } else if (n > 0) {
10670 u16 vf_tmo_dsecs = n * 10;
10671
10672 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10673 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10674 bp->fw_reset_state =
10675 BNXT_FW_RESET_STATE_POLL_VF;
10676 bnxt_queue_fw_reset_work(bp, HZ / 10);
10677 goto fw_reset_exit;
10678 }
10679 bnxt_fw_reset_close(bp);
10680 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10681 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10682 tmo = HZ / 10;
10683 } else {
10684 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10685 tmo = bp->fw_reset_min_dsecs * HZ / 10;
10686 }
10687 bnxt_queue_fw_reset_work(bp, tmo);
10688 }
10689 fw_reset_exit:
10690 bnxt_rtnl_unlock_sp(bp);
10691 }
10692
10693 static void bnxt_chk_missed_irq(struct bnxt *bp)
10694 {
10695 int i;
10696
10697 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10698 return;
10699
10700 for (i = 0; i < bp->cp_nr_rings; i++) {
10701 struct bnxt_napi *bnapi = bp->bnapi[i];
10702 struct bnxt_cp_ring_info *cpr;
10703 u32 fw_ring_id;
10704 int j;
10705
10706 if (!bnapi)
10707 continue;
10708
10709 cpr = &bnapi->cp_ring;
10710 for (j = 0; j < 2; j++) {
10711 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10712 u32 val[2];
10713
10714 if (!cpr2 || cpr2->has_more_work ||
10715 !bnxt_has_work(bp, cpr2))
10716 continue;
10717
10718 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10719 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10720 continue;
10721 }
10722 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10723 bnxt_dbg_hwrm_ring_info_get(bp,
10724 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10725 fw_ring_id, &val[0], &val[1]);
10726 cpr->sw_stats.cmn.missed_irqs++;
10727 }
10728 }
10729 }
10730
10731 static void bnxt_cfg_ntp_filters(struct bnxt *);
10732
10733 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10734 {
10735 struct bnxt_link_info *link_info = &bp->link_info;
10736
10737 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10738 link_info->autoneg = BNXT_AUTONEG_SPEED;
10739 if (bp->hwrm_spec_code >= 0x10201) {
10740 if (link_info->auto_pause_setting &
10741 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10742 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10743 } else {
10744 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10745 }
10746 link_info->advertising = link_info->auto_link_speeds;
10747 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
10748 } else {
10749 link_info->req_link_speed = link_info->force_link_speed;
10750 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
10751 if (link_info->force_pam4_link_speed) {
10752 link_info->req_link_speed =
10753 link_info->force_pam4_link_speed;
10754 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
10755 }
10756 link_info->req_duplex = link_info->duplex_setting;
10757 }
10758 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10759 link_info->req_flow_ctrl =
10760 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10761 else
10762 link_info->req_flow_ctrl = link_info->force_pause_setting;
10763 }
10764
10765 static void bnxt_sp_task(struct work_struct *work)
10766 {
10767 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10768
10769 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10770 smp_mb__after_atomic();
10771 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10772 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10773 return;
10774 }
10775
10776 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
10777 bnxt_cfg_rx_mode(bp);
10778
10779 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
10780 bnxt_cfg_ntp_filters(bp);
10781 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
10782 bnxt_hwrm_exec_fwd_req(bp);
10783 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
10784 bnxt_hwrm_port_qstats(bp, 0);
10785 bnxt_hwrm_port_qstats_ext(bp, 0);
10786 bnxt_accumulate_all_stats(bp);
10787 }
10788
10789 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
10790 int rc;
10791
10792 mutex_lock(&bp->link_lock);
10793 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
10794 &bp->sp_event))
10795 bnxt_hwrm_phy_qcaps(bp);
10796
10797 rc = bnxt_update_link(bp, true);
10798 if (rc)
10799 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
10800 rc);
10801
10802 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
10803 &bp->sp_event))
10804 bnxt_init_ethtool_link_settings(bp);
10805 mutex_unlock(&bp->link_lock);
10806 }
10807 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
10808 int rc;
10809
10810 mutex_lock(&bp->link_lock);
10811 rc = bnxt_update_phy_setting(bp);
10812 mutex_unlock(&bp->link_lock);
10813 if (rc) {
10814 netdev_warn(bp->dev, "update phy settings retry failed\n");
10815 } else {
10816 bp->link_info.phy_retry = false;
10817 netdev_info(bp->dev, "update phy settings retry succeeded\n");
10818 }
10819 }
10820 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
10821 mutex_lock(&bp->link_lock);
10822 bnxt_get_port_module_status(bp);
10823 mutex_unlock(&bp->link_lock);
10824 }
10825
10826 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10827 bnxt_tc_flow_stats_work(bp);
10828
10829 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10830 bnxt_chk_missed_irq(bp);
10831
10832 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
10833 * must be the last functions to be called before exiting.
10834 */
10835 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10836 bnxt_reset(bp, false);
10837
10838 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10839 bnxt_reset(bp, true);
10840
10841 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
10842 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
10843
10844 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
10845 if (!is_bnxt_fw_ok(bp))
10846 bnxt_devlink_health_report(bp,
10847 BNXT_FW_EXCEPTION_SP_EVENT);
10848 }
10849
10850 smp_mb__before_atomic();
10851 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10852 }
10853
10854 /* Under rtnl_lock */
10855 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10856 int tx_xdp)
10857 {
10858 int max_rx, max_tx, tx_sets = 1;
10859 int tx_rings_needed, stats;
10860 int rx_rings = rx;
10861 int cp, vnics, rc;
10862
10863 if (tcs)
10864 tx_sets = tcs;
10865
10866 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10867 if (rc)
10868 return rc;
10869
10870 if (max_rx < rx)
10871 return -ENOMEM;
10872
10873 tx_rings_needed = tx * tx_sets + tx_xdp;
10874 if (max_tx < tx_rings_needed)
10875 return -ENOMEM;
10876
10877 vnics = 1;
10878 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
10879 vnics += rx_rings;
10880
10881 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10882 rx_rings <<= 1;
10883 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
10884 stats = cp;
10885 if (BNXT_NEW_RM(bp)) {
10886 cp += bnxt_get_ulp_msix_num(bp);
10887 stats += bnxt_get_ulp_stat_ctxs(bp);
10888 }
10889 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
10890 stats, vnics);
10891 }
10892
10893 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10894 {
10895 if (bp->bar2) {
10896 pci_iounmap(pdev, bp->bar2);
10897 bp->bar2 = NULL;
10898 }
10899
10900 if (bp->bar1) {
10901 pci_iounmap(pdev, bp->bar1);
10902 bp->bar1 = NULL;
10903 }
10904
10905 if (bp->bar0) {
10906 pci_iounmap(pdev, bp->bar0);
10907 bp->bar0 = NULL;
10908 }
10909 }
10910
10911 static void bnxt_cleanup_pci(struct bnxt *bp)
10912 {
10913 bnxt_unmap_bars(bp, bp->pdev);
10914 pci_release_regions(bp->pdev);
10915 if (pci_is_enabled(bp->pdev))
10916 pci_disable_device(bp->pdev);
10917 }
10918
10919 static void bnxt_init_dflt_coal(struct bnxt *bp)
10920 {
10921 struct bnxt_coal *coal;
10922
10923 /* Tick values in micro seconds.
10924 * 1 coal_buf x bufs_per_record = 1 completion record.
10925 */
10926 coal = &bp->rx_coal;
10927 coal->coal_ticks = 10;
10928 coal->coal_bufs = 30;
10929 coal->coal_ticks_irq = 1;
10930 coal->coal_bufs_irq = 2;
10931 coal->idle_thresh = 50;
10932 coal->bufs_per_record = 2;
10933 coal->budget = 64; /* NAPI budget */
10934
10935 coal = &bp->tx_coal;
10936 coal->coal_ticks = 28;
10937 coal->coal_bufs = 30;
10938 coal->coal_ticks_irq = 2;
10939 coal->coal_bufs_irq = 2;
10940 coal->bufs_per_record = 1;
10941
10942 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10943 }
10944
10945 static void bnxt_alloc_fw_health(struct bnxt *bp)
10946 {
10947 if (bp->fw_health)
10948 return;
10949
10950 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
10951 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
10952 return;
10953
10954 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
10955 if (!bp->fw_health) {
10956 netdev_warn(bp->dev, "Failed to allocate fw_health\n");
10957 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
10958 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
10959 }
10960 }
10961
10962 static int bnxt_fw_init_one_p1(struct bnxt *bp)
10963 {
10964 int rc;
10965
10966 bp->fw_cap = 0;
10967 rc = bnxt_hwrm_ver_get(bp);
10968 if (rc)
10969 return rc;
10970
10971 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10972 rc = bnxt_alloc_kong_hwrm_resources(bp);
10973 if (rc)
10974 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10975 }
10976
10977 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10978 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10979 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10980 if (rc)
10981 return rc;
10982 }
10983 rc = bnxt_hwrm_func_reset(bp);
10984 if (rc)
10985 return -ENODEV;
10986
10987 bnxt_hwrm_fw_set_time(bp);
10988 return 0;
10989 }
10990
10991 static int bnxt_fw_init_one_p2(struct bnxt *bp)
10992 {
10993 int rc;
10994
10995 /* Get the MAX capabilities for this function */
10996 rc = bnxt_hwrm_func_qcaps(bp);
10997 if (rc) {
10998 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10999 rc);
11000 return -ENODEV;
11001 }
11002
11003 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11004 if (rc)
11005 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11006 rc);
11007
11008 bnxt_alloc_fw_health(bp);
11009 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11010 if (rc)
11011 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11012 rc);
11013
11014 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11015 if (rc)
11016 return -ENODEV;
11017
11018 bnxt_hwrm_func_qcfg(bp);
11019 bnxt_hwrm_vnic_qcaps(bp);
11020 bnxt_hwrm_port_led_qcaps(bp);
11021 bnxt_ethtool_init(bp);
11022 bnxt_dcb_init(bp);
11023 return 0;
11024 }
11025
11026 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11027 {
11028 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11029 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11030 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11031 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11032 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11033 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11034 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11035 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11036 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11037 }
11038 }
11039
11040 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11041 {
11042 struct net_device *dev = bp->dev;
11043
11044 dev->hw_features &= ~NETIF_F_NTUPLE;
11045 dev->features &= ~NETIF_F_NTUPLE;
11046 bp->flags &= ~BNXT_FLAG_RFS;
11047 if (bnxt_rfs_supported(bp)) {
11048 dev->hw_features |= NETIF_F_NTUPLE;
11049 if (bnxt_rfs_capable(bp)) {
11050 bp->flags |= BNXT_FLAG_RFS;
11051 dev->features |= NETIF_F_NTUPLE;
11052 }
11053 }
11054 }
11055
11056 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11057 {
11058 struct pci_dev *pdev = bp->pdev;
11059
11060 bnxt_set_dflt_rss_hash_type(bp);
11061 bnxt_set_dflt_rfs(bp);
11062
11063 bnxt_get_wol_settings(bp);
11064 if (bp->flags & BNXT_FLAG_WOL_CAP)
11065 device_set_wakeup_enable(&pdev->dev, bp->wol);
11066 else
11067 device_set_wakeup_capable(&pdev->dev, false);
11068
11069 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11070 bnxt_hwrm_coal_params_qcaps(bp);
11071 }
11072
11073 static int bnxt_fw_init_one(struct bnxt *bp)
11074 {
11075 int rc;
11076
11077 rc = bnxt_fw_init_one_p1(bp);
11078 if (rc) {
11079 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11080 return rc;
11081 }
11082 rc = bnxt_fw_init_one_p2(bp);
11083 if (rc) {
11084 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11085 return rc;
11086 }
11087 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11088 if (rc)
11089 return rc;
11090
11091 /* In case fw capabilities have changed, destroy the unneeded
11092 * reporters and create newly capable ones.
11093 */
11094 bnxt_dl_fw_reporters_destroy(bp, false);
11095 bnxt_dl_fw_reporters_create(bp);
11096 bnxt_fw_init_one_p3(bp);
11097 return 0;
11098 }
11099
11100 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11101 {
11102 struct bnxt_fw_health *fw_health = bp->fw_health;
11103 u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11104 u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11105 u32 reg_type, reg_off, delay_msecs;
11106
11107 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11108 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11109 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11110 switch (reg_type) {
11111 case BNXT_FW_HEALTH_REG_TYPE_CFG:
11112 pci_write_config_dword(bp->pdev, reg_off, val);
11113 break;
11114 case BNXT_FW_HEALTH_REG_TYPE_GRC:
11115 writel(reg_off & BNXT_GRC_BASE_MASK,
11116 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11117 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11118 fallthrough;
11119 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11120 writel(val, bp->bar0 + reg_off);
11121 break;
11122 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11123 writel(val, bp->bar1 + reg_off);
11124 break;
11125 }
11126 if (delay_msecs) {
11127 pci_read_config_dword(bp->pdev, 0, &val);
11128 msleep(delay_msecs);
11129 }
11130 }
11131
11132 static void bnxt_reset_all(struct bnxt *bp)
11133 {
11134 struct bnxt_fw_health *fw_health = bp->fw_health;
11135 int i, rc;
11136
11137 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11138 #ifdef CONFIG_TEE_BNXT_FW
11139 rc = tee_bnxt_fw_load();
11140 if (rc)
11141 netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc);
11142 bp->fw_reset_timestamp = jiffies;
11143 #endif
11144 return;
11145 }
11146
11147 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11148 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11149 bnxt_fw_reset_writel(bp, i);
11150 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11151 struct hwrm_fw_reset_input req = {0};
11152
11153 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11154 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11155 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11156 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11157 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11158 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11159 if (rc)
11160 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11161 }
11162 bp->fw_reset_timestamp = jiffies;
11163 }
11164
11165 static void bnxt_fw_reset_task(struct work_struct *work)
11166 {
11167 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11168 int rc;
11169
11170 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11171 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11172 return;
11173 }
11174
11175 switch (bp->fw_reset_state) {
11176 case BNXT_FW_RESET_STATE_POLL_VF: {
11177 int n = bnxt_get_registered_vfs(bp);
11178 int tmo;
11179
11180 if (n < 0) {
11181 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11182 n, jiffies_to_msecs(jiffies -
11183 bp->fw_reset_timestamp));
11184 goto fw_reset_abort;
11185 } else if (n > 0) {
11186 if (time_after(jiffies, bp->fw_reset_timestamp +
11187 (bp->fw_reset_max_dsecs * HZ / 10))) {
11188 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11189 bp->fw_reset_state = 0;
11190 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11191 n);
11192 return;
11193 }
11194 bnxt_queue_fw_reset_work(bp, HZ / 10);
11195 return;
11196 }
11197 bp->fw_reset_timestamp = jiffies;
11198 rtnl_lock();
11199 bnxt_fw_reset_close(bp);
11200 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11201 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11202 tmo = HZ / 10;
11203 } else {
11204 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11205 tmo = bp->fw_reset_min_dsecs * HZ / 10;
11206 }
11207 rtnl_unlock();
11208 bnxt_queue_fw_reset_work(bp, tmo);
11209 return;
11210 }
11211 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11212 u32 val;
11213
11214 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11215 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11216 !time_after(jiffies, bp->fw_reset_timestamp +
11217 (bp->fw_reset_max_dsecs * HZ / 10))) {
11218 bnxt_queue_fw_reset_work(bp, HZ / 5);
11219 return;
11220 }
11221
11222 if (!bp->fw_health->master) {
11223 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11224
11225 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11226 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11227 return;
11228 }
11229 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11230 }
11231 fallthrough;
11232 case BNXT_FW_RESET_STATE_RESET_FW:
11233 bnxt_reset_all(bp);
11234 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11235 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11236 return;
11237 case BNXT_FW_RESET_STATE_ENABLE_DEV:
11238 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11239 u32 val;
11240
11241 val = bnxt_fw_health_readl(bp,
11242 BNXT_FW_RESET_INPROG_REG);
11243 if (val)
11244 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11245 val);
11246 }
11247 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11248 if (pci_enable_device(bp->pdev)) {
11249 netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11250 goto fw_reset_abort;
11251 }
11252 pci_set_master(bp->pdev);
11253 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11254 fallthrough;
11255 case BNXT_FW_RESET_STATE_POLL_FW:
11256 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11257 rc = __bnxt_hwrm_ver_get(bp, true);
11258 if (rc) {
11259 if (time_after(jiffies, bp->fw_reset_timestamp +
11260 (bp->fw_reset_max_dsecs * HZ / 10))) {
11261 netdev_err(bp->dev, "Firmware reset aborted\n");
11262 goto fw_reset_abort;
11263 }
11264 bnxt_queue_fw_reset_work(bp, HZ / 5);
11265 return;
11266 }
11267 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11268 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11269 fallthrough;
11270 case BNXT_FW_RESET_STATE_OPENING:
11271 while (!rtnl_trylock()) {
11272 bnxt_queue_fw_reset_work(bp, HZ / 10);
11273 return;
11274 }
11275 rc = bnxt_open(bp->dev);
11276 if (rc) {
11277 netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11278 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11279 dev_close(bp->dev);
11280 }
11281
11282 bp->fw_reset_state = 0;
11283 /* Make sure fw_reset_state is 0 before clearing the flag */
11284 smp_mb__before_atomic();
11285 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11286 bnxt_ulp_start(bp, rc);
11287 if (!rc)
11288 bnxt_reenable_sriov(bp);
11289 bnxt_dl_health_recovery_done(bp);
11290 bnxt_dl_health_status_update(bp, true);
11291 rtnl_unlock();
11292 break;
11293 }
11294 return;
11295
11296 fw_reset_abort:
11297 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11298 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11299 bnxt_dl_health_status_update(bp, false);
11300 bp->fw_reset_state = 0;
11301 rtnl_lock();
11302 dev_close(bp->dev);
11303 rtnl_unlock();
11304 }
11305
11306 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11307 {
11308 int rc;
11309 struct bnxt *bp = netdev_priv(dev);
11310
11311 SET_NETDEV_DEV(dev, &pdev->dev);
11312
11313 /* enable device (incl. PCI PM wakeup), and bus-mastering */
11314 rc = pci_enable_device(pdev);
11315 if (rc) {
11316 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11317 goto init_err;
11318 }
11319
11320 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11321 dev_err(&pdev->dev,
11322 "Cannot find PCI device base address, aborting\n");
11323 rc = -ENODEV;
11324 goto init_err_disable;
11325 }
11326
11327 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11328 if (rc) {
11329 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11330 goto init_err_disable;
11331 }
11332
11333 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11334 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11335 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11336 goto init_err_disable;
11337 }
11338
11339 pci_set_master(pdev);
11340
11341 bp->dev = dev;
11342 bp->pdev = pdev;
11343
11344 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11345 * determines the BAR size.
11346 */
11347 bp->bar0 = pci_ioremap_bar(pdev, 0);
11348 if (!bp->bar0) {
11349 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11350 rc = -ENOMEM;
11351 goto init_err_release;
11352 }
11353
11354 bp->bar2 = pci_ioremap_bar(pdev, 4);
11355 if (!bp->bar2) {
11356 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11357 rc = -ENOMEM;
11358 goto init_err_release;
11359 }
11360
11361 pci_enable_pcie_error_reporting(pdev);
11362
11363 INIT_WORK(&bp->sp_task, bnxt_sp_task);
11364 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11365
11366 spin_lock_init(&bp->ntp_fltr_lock);
11367 #if BITS_PER_LONG == 32
11368 spin_lock_init(&bp->db_lock);
11369 #endif
11370
11371 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11372 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11373
11374 bnxt_init_dflt_coal(bp);
11375
11376 timer_setup(&bp->timer, bnxt_timer, 0);
11377 bp->current_interval = BNXT_TIMER_INTERVAL;
11378
11379 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11380 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11381
11382 clear_bit(BNXT_STATE_OPEN, &bp->state);
11383 return 0;
11384
11385 init_err_release:
11386 bnxt_unmap_bars(bp, pdev);
11387 pci_release_regions(pdev);
11388
11389 init_err_disable:
11390 pci_disable_device(pdev);
11391
11392 init_err:
11393 return rc;
11394 }
11395
11396 /* rtnl_lock held */
11397 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11398 {
11399 struct sockaddr *addr = p;
11400 struct bnxt *bp = netdev_priv(dev);
11401 int rc = 0;
11402
11403 if (!is_valid_ether_addr(addr->sa_data))
11404 return -EADDRNOTAVAIL;
11405
11406 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11407 return 0;
11408
11409 rc = bnxt_approve_mac(bp, addr->sa_data, true);
11410 if (rc)
11411 return rc;
11412
11413 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11414 if (netif_running(dev)) {
11415 bnxt_close_nic(bp, false, false);
11416 rc = bnxt_open_nic(bp, false, false);
11417 }
11418
11419 return rc;
11420 }
11421
11422 /* rtnl_lock held */
11423 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11424 {
11425 struct bnxt *bp = netdev_priv(dev);
11426
11427 if (netif_running(dev))
11428 bnxt_close_nic(bp, true, false);
11429
11430 dev->mtu = new_mtu;
11431 bnxt_set_ring_params(bp);
11432
11433 if (netif_running(dev))
11434 return bnxt_open_nic(bp, true, false);
11435
11436 return 0;
11437 }
11438
11439 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11440 {
11441 struct bnxt *bp = netdev_priv(dev);
11442 bool sh = false;
11443 int rc;
11444
11445 if (tc > bp->max_tc) {
11446 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11447 tc, bp->max_tc);
11448 return -EINVAL;
11449 }
11450
11451 if (netdev_get_num_tc(dev) == tc)
11452 return 0;
11453
11454 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11455 sh = true;
11456
11457 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11458 sh, tc, bp->tx_nr_rings_xdp);
11459 if (rc)
11460 return rc;
11461
11462 /* Needs to close the device and do hw resource re-allocations */
11463 if (netif_running(bp->dev))
11464 bnxt_close_nic(bp, true, false);
11465
11466 if (tc) {
11467 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11468 netdev_set_num_tc(dev, tc);
11469 } else {
11470 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11471 netdev_reset_tc(dev);
11472 }
11473 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11474 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11475 bp->tx_nr_rings + bp->rx_nr_rings;
11476
11477 if (netif_running(bp->dev))
11478 return bnxt_open_nic(bp, true, false);
11479
11480 return 0;
11481 }
11482
11483 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11484 void *cb_priv)
11485 {
11486 struct bnxt *bp = cb_priv;
11487
11488 if (!bnxt_tc_flower_enabled(bp) ||
11489 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11490 return -EOPNOTSUPP;
11491
11492 switch (type) {
11493 case TC_SETUP_CLSFLOWER:
11494 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11495 default:
11496 return -EOPNOTSUPP;
11497 }
11498 }
11499
11500 LIST_HEAD(bnxt_block_cb_list);
11501
11502 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11503 void *type_data)
11504 {
11505 struct bnxt *bp = netdev_priv(dev);
11506
11507 switch (type) {
11508 case TC_SETUP_BLOCK:
11509 return flow_block_cb_setup_simple(type_data,
11510 &bnxt_block_cb_list,
11511 bnxt_setup_tc_block_cb,
11512 bp, bp, true);
11513 case TC_SETUP_QDISC_MQPRIO: {
11514 struct tc_mqprio_qopt *mqprio = type_data;
11515
11516 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11517
11518 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11519 }
11520 default:
11521 return -EOPNOTSUPP;
11522 }
11523 }
11524
11525 #ifdef CONFIG_RFS_ACCEL
11526 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11527 struct bnxt_ntuple_filter *f2)
11528 {
11529 struct flow_keys *keys1 = &f1->fkeys;
11530 struct flow_keys *keys2 = &f2->fkeys;
11531
11532 if (keys1->basic.n_proto != keys2->basic.n_proto ||
11533 keys1->basic.ip_proto != keys2->basic.ip_proto)
11534 return false;
11535
11536 if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11537 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11538 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11539 return false;
11540 } else {
11541 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11542 sizeof(keys1->addrs.v6addrs.src)) ||
11543 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11544 sizeof(keys1->addrs.v6addrs.dst)))
11545 return false;
11546 }
11547
11548 if (keys1->ports.ports == keys2->ports.ports &&
11549 keys1->control.flags == keys2->control.flags &&
11550 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11551 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11552 return true;
11553
11554 return false;
11555 }
11556
11557 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11558 u16 rxq_index, u32 flow_id)
11559 {
11560 struct bnxt *bp = netdev_priv(dev);
11561 struct bnxt_ntuple_filter *fltr, *new_fltr;
11562 struct flow_keys *fkeys;
11563 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11564 int rc = 0, idx, bit_id, l2_idx = 0;
11565 struct hlist_head *head;
11566 u32 flags;
11567
11568 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11569 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11570 int off = 0, j;
11571
11572 netif_addr_lock_bh(dev);
11573 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11574 if (ether_addr_equal(eth->h_dest,
11575 vnic->uc_list + off)) {
11576 l2_idx = j + 1;
11577 break;
11578 }
11579 }
11580 netif_addr_unlock_bh(dev);
11581 if (!l2_idx)
11582 return -EINVAL;
11583 }
11584 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11585 if (!new_fltr)
11586 return -ENOMEM;
11587
11588 fkeys = &new_fltr->fkeys;
11589 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11590 rc = -EPROTONOSUPPORT;
11591 goto err_free;
11592 }
11593
11594 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11595 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11596 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11597 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11598 rc = -EPROTONOSUPPORT;
11599 goto err_free;
11600 }
11601 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11602 bp->hwrm_spec_code < 0x10601) {
11603 rc = -EPROTONOSUPPORT;
11604 goto err_free;
11605 }
11606 flags = fkeys->control.flags;
11607 if (((flags & FLOW_DIS_ENCAPSULATION) &&
11608 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11609 rc = -EPROTONOSUPPORT;
11610 goto err_free;
11611 }
11612
11613 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11614 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11615
11616 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11617 head = &bp->ntp_fltr_hash_tbl[idx];
11618 rcu_read_lock();
11619 hlist_for_each_entry_rcu(fltr, head, hash) {
11620 if (bnxt_fltr_match(fltr, new_fltr)) {
11621 rcu_read_unlock();
11622 rc = 0;
11623 goto err_free;
11624 }
11625 }
11626 rcu_read_unlock();
11627
11628 spin_lock_bh(&bp->ntp_fltr_lock);
11629 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11630 BNXT_NTP_FLTR_MAX_FLTR, 0);
11631 if (bit_id < 0) {
11632 spin_unlock_bh(&bp->ntp_fltr_lock);
11633 rc = -ENOMEM;
11634 goto err_free;
11635 }
11636
11637 new_fltr->sw_id = (u16)bit_id;
11638 new_fltr->flow_id = flow_id;
11639 new_fltr->l2_fltr_idx = l2_idx;
11640 new_fltr->rxq = rxq_index;
11641 hlist_add_head_rcu(&new_fltr->hash, head);
11642 bp->ntp_fltr_count++;
11643 spin_unlock_bh(&bp->ntp_fltr_lock);
11644
11645 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11646 bnxt_queue_sp_work(bp);
11647
11648 return new_fltr->sw_id;
11649
11650 err_free:
11651 kfree(new_fltr);
11652 return rc;
11653 }
11654
11655 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11656 {
11657 int i;
11658
11659 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11660 struct hlist_head *head;
11661 struct hlist_node *tmp;
11662 struct bnxt_ntuple_filter *fltr;
11663 int rc;
11664
11665 head = &bp->ntp_fltr_hash_tbl[i];
11666 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11667 bool del = false;
11668
11669 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11670 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11671 fltr->flow_id,
11672 fltr->sw_id)) {
11673 bnxt_hwrm_cfa_ntuple_filter_free(bp,
11674 fltr);
11675 del = true;
11676 }
11677 } else {
11678 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11679 fltr);
11680 if (rc)
11681 del = true;
11682 else
11683 set_bit(BNXT_FLTR_VALID, &fltr->state);
11684 }
11685
11686 if (del) {
11687 spin_lock_bh(&bp->ntp_fltr_lock);
11688 hlist_del_rcu(&fltr->hash);
11689 bp->ntp_fltr_count--;
11690 spin_unlock_bh(&bp->ntp_fltr_lock);
11691 synchronize_rcu();
11692 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11693 kfree(fltr);
11694 }
11695 }
11696 }
11697 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11698 netdev_info(bp->dev, "Receive PF driver unload event!\n");
11699 }
11700
11701 #else
11702
11703 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11704 {
11705 }
11706
11707 #endif /* CONFIG_RFS_ACCEL */
11708
11709 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11710 {
11711 struct bnxt *bp = netdev_priv(netdev);
11712 struct udp_tunnel_info ti;
11713 unsigned int cmd;
11714
11715 udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11716 if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11717 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11718 else
11719 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11720
11721 if (ti.port)
11722 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11723
11724 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11725 }
11726
11727 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11728 .sync_table = bnxt_udp_tunnel_sync,
11729 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11730 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11731 .tables = {
11732 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
11733 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11734 },
11735 };
11736
11737 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11738 struct net_device *dev, u32 filter_mask,
11739 int nlflags)
11740 {
11741 struct bnxt *bp = netdev_priv(dev);
11742
11743 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11744 nlflags, filter_mask, NULL);
11745 }
11746
11747 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
11748 u16 flags, struct netlink_ext_ack *extack)
11749 {
11750 struct bnxt *bp = netdev_priv(dev);
11751 struct nlattr *attr, *br_spec;
11752 int rem, rc = 0;
11753
11754 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
11755 return -EOPNOTSUPP;
11756
11757 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11758 if (!br_spec)
11759 return -EINVAL;
11760
11761 nla_for_each_nested(attr, br_spec, rem) {
11762 u16 mode;
11763
11764 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11765 continue;
11766
11767 if (nla_len(attr) < sizeof(mode))
11768 return -EINVAL;
11769
11770 mode = nla_get_u16(attr);
11771 if (mode == bp->br_mode)
11772 break;
11773
11774 rc = bnxt_hwrm_set_br_mode(bp, mode);
11775 if (!rc)
11776 bp->br_mode = mode;
11777 break;
11778 }
11779 return rc;
11780 }
11781
11782 int bnxt_get_port_parent_id(struct net_device *dev,
11783 struct netdev_phys_item_id *ppid)
11784 {
11785 struct bnxt *bp = netdev_priv(dev);
11786
11787 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
11788 return -EOPNOTSUPP;
11789
11790 /* The PF and it's VF-reps only support the switchdev framework */
11791 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
11792 return -EOPNOTSUPP;
11793
11794 ppid->id_len = sizeof(bp->dsn);
11795 memcpy(ppid->id, bp->dsn, ppid->id_len);
11796
11797 return 0;
11798 }
11799
11800 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
11801 {
11802 struct bnxt *bp = netdev_priv(dev);
11803
11804 return &bp->dl_port;
11805 }
11806
11807 static const struct net_device_ops bnxt_netdev_ops = {
11808 .ndo_open = bnxt_open,
11809 .ndo_start_xmit = bnxt_start_xmit,
11810 .ndo_stop = bnxt_close,
11811 .ndo_get_stats64 = bnxt_get_stats64,
11812 .ndo_set_rx_mode = bnxt_set_rx_mode,
11813 .ndo_do_ioctl = bnxt_ioctl,
11814 .ndo_validate_addr = eth_validate_addr,
11815 .ndo_set_mac_address = bnxt_change_mac_addr,
11816 .ndo_change_mtu = bnxt_change_mtu,
11817 .ndo_fix_features = bnxt_fix_features,
11818 .ndo_set_features = bnxt_set_features,
11819 .ndo_tx_timeout = bnxt_tx_timeout,
11820 #ifdef CONFIG_BNXT_SRIOV
11821 .ndo_get_vf_config = bnxt_get_vf_config,
11822 .ndo_set_vf_mac = bnxt_set_vf_mac,
11823 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
11824 .ndo_set_vf_rate = bnxt_set_vf_bw,
11825 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
11826 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
11827 .ndo_set_vf_trust = bnxt_set_vf_trust,
11828 #endif
11829 .ndo_setup_tc = bnxt_setup_tc,
11830 #ifdef CONFIG_RFS_ACCEL
11831 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
11832 #endif
11833 .ndo_udp_tunnel_add = udp_tunnel_nic_add_port,
11834 .ndo_udp_tunnel_del = udp_tunnel_nic_del_port,
11835 .ndo_bpf = bnxt_xdp,
11836 .ndo_xdp_xmit = bnxt_xdp_xmit,
11837 .ndo_bridge_getlink = bnxt_bridge_getlink,
11838 .ndo_bridge_setlink = bnxt_bridge_setlink,
11839 .ndo_get_devlink_port = bnxt_get_devlink_port,
11840 };
11841
11842 static void bnxt_remove_one(struct pci_dev *pdev)
11843 {
11844 struct net_device *dev = pci_get_drvdata(pdev);
11845 struct bnxt *bp = netdev_priv(dev);
11846
11847 if (BNXT_PF(bp))
11848 bnxt_sriov_disable(bp);
11849
11850 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11851 bnxt_cancel_sp_work(bp);
11852 bp->sp_event = 0;
11853
11854 bnxt_dl_fw_reporters_destroy(bp, true);
11855 if (BNXT_PF(bp))
11856 devlink_port_type_clear(&bp->dl_port);
11857 pci_disable_pcie_error_reporting(pdev);
11858 unregister_netdev(dev);
11859 bnxt_dl_unregister(bp);
11860 bnxt_shutdown_tc(bp);
11861
11862 bnxt_clear_int_mode(bp);
11863 bnxt_hwrm_func_drv_unrgtr(bp);
11864 bnxt_free_hwrm_resources(bp);
11865 bnxt_free_hwrm_short_cmd_req(bp);
11866 bnxt_ethtool_free(bp);
11867 bnxt_dcb_free(bp);
11868 kfree(bp->edev);
11869 bp->edev = NULL;
11870 kfree(bp->fw_health);
11871 bp->fw_health = NULL;
11872 bnxt_cleanup_pci(bp);
11873 bnxt_free_ctx_mem(bp);
11874 kfree(bp->ctx);
11875 bp->ctx = NULL;
11876 kfree(bp->rss_indir_tbl);
11877 bp->rss_indir_tbl = NULL;
11878 bnxt_free_port_stats(bp);
11879 free_netdev(dev);
11880 }
11881
11882 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
11883 {
11884 int rc = 0;
11885 struct bnxt_link_info *link_info = &bp->link_info;
11886
11887 rc = bnxt_hwrm_phy_qcaps(bp);
11888 if (rc) {
11889 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
11890 rc);
11891 return rc;
11892 }
11893 if (!fw_dflt)
11894 return 0;
11895
11896 rc = bnxt_update_link(bp, false);
11897 if (rc) {
11898 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
11899 rc);
11900 return rc;
11901 }
11902
11903 /* Older firmware does not have supported_auto_speeds, so assume
11904 * that all supported speeds can be autonegotiated.
11905 */
11906 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
11907 link_info->support_auto_speeds = link_info->support_speeds;
11908
11909 bnxt_init_ethtool_link_settings(bp);
11910 return 0;
11911 }
11912
11913 static int bnxt_get_max_irq(struct pci_dev *pdev)
11914 {
11915 u16 ctrl;
11916
11917 if (!pdev->msix_cap)
11918 return 1;
11919
11920 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
11921 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
11922 }
11923
11924 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11925 int *max_cp)
11926 {
11927 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11928 int max_ring_grps = 0, max_irq;
11929
11930 *max_tx = hw_resc->max_tx_rings;
11931 *max_rx = hw_resc->max_rx_rings;
11932 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
11933 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
11934 bnxt_get_ulp_msix_num(bp),
11935 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
11936 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11937 *max_cp = min_t(int, *max_cp, max_irq);
11938 max_ring_grps = hw_resc->max_hw_ring_grps;
11939 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
11940 *max_cp -= 1;
11941 *max_rx -= 2;
11942 }
11943 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11944 *max_rx >>= 1;
11945 if (bp->flags & BNXT_FLAG_CHIP_P5) {
11946 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
11947 /* On P5 chips, max_cp output param should be available NQs */
11948 *max_cp = max_irq;
11949 }
11950 *max_rx = min_t(int, *max_rx, max_ring_grps);
11951 }
11952
11953 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
11954 {
11955 int rx, tx, cp;
11956
11957 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
11958 *max_rx = rx;
11959 *max_tx = tx;
11960 if (!rx || !tx || !cp)
11961 return -ENOMEM;
11962
11963 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
11964 }
11965
11966 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11967 bool shared)
11968 {
11969 int rc;
11970
11971 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11972 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
11973 /* Not enough rings, try disabling agg rings. */
11974 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
11975 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11976 if (rc) {
11977 /* set BNXT_FLAG_AGG_RINGS back for consistency */
11978 bp->flags |= BNXT_FLAG_AGG_RINGS;
11979 return rc;
11980 }
11981 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
11982 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11983 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11984 bnxt_set_ring_params(bp);
11985 }
11986
11987 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
11988 int max_cp, max_stat, max_irq;
11989
11990 /* Reserve minimum resources for RoCE */
11991 max_cp = bnxt_get_max_func_cp_rings(bp);
11992 max_stat = bnxt_get_max_func_stat_ctxs(bp);
11993 max_irq = bnxt_get_max_func_irqs(bp);
11994 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
11995 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
11996 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
11997 return 0;
11998
11999 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12000 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12001 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12002 max_cp = min_t(int, max_cp, max_irq);
12003 max_cp = min_t(int, max_cp, max_stat);
12004 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12005 if (rc)
12006 rc = 0;
12007 }
12008 return rc;
12009 }
12010
12011 /* In initial default shared ring setting, each shared ring must have a
12012 * RX/TX ring pair.
12013 */
12014 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12015 {
12016 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12017 bp->rx_nr_rings = bp->cp_nr_rings;
12018 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12019 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12020 }
12021
12022 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12023 {
12024 int dflt_rings, max_rx_rings, max_tx_rings, rc;
12025
12026 if (!bnxt_can_reserve_rings(bp))
12027 return 0;
12028
12029 if (sh)
12030 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12031 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12032 /* Reduce default rings on multi-port cards so that total default
12033 * rings do not exceed CPU count.
12034 */
12035 if (bp->port_count > 1) {
12036 int max_rings =
12037 max_t(int, num_online_cpus() / bp->port_count, 1);
12038
12039 dflt_rings = min_t(int, dflt_rings, max_rings);
12040 }
12041 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12042 if (rc)
12043 return rc;
12044 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12045 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12046 if (sh)
12047 bnxt_trim_dflt_sh_rings(bp);
12048 else
12049 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12050 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12051
12052 rc = __bnxt_reserve_rings(bp);
12053 if (rc)
12054 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12055 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12056 if (sh)
12057 bnxt_trim_dflt_sh_rings(bp);
12058
12059 /* Rings may have been trimmed, re-reserve the trimmed rings. */
12060 if (bnxt_need_reserve_rings(bp)) {
12061 rc = __bnxt_reserve_rings(bp);
12062 if (rc)
12063 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12064 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12065 }
12066 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12067 bp->rx_nr_rings++;
12068 bp->cp_nr_rings++;
12069 }
12070 if (rc) {
12071 bp->tx_nr_rings = 0;
12072 bp->rx_nr_rings = 0;
12073 }
12074 return rc;
12075 }
12076
12077 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12078 {
12079 int rc;
12080
12081 if (bp->tx_nr_rings)
12082 return 0;
12083
12084 bnxt_ulp_irq_stop(bp);
12085 bnxt_clear_int_mode(bp);
12086 rc = bnxt_set_dflt_rings(bp, true);
12087 if (rc) {
12088 netdev_err(bp->dev, "Not enough rings available.\n");
12089 goto init_dflt_ring_err;
12090 }
12091 rc = bnxt_init_int_mode(bp);
12092 if (rc)
12093 goto init_dflt_ring_err;
12094
12095 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12096 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12097 bp->flags |= BNXT_FLAG_RFS;
12098 bp->dev->features |= NETIF_F_NTUPLE;
12099 }
12100 init_dflt_ring_err:
12101 bnxt_ulp_irq_restart(bp, rc);
12102 return rc;
12103 }
12104
12105 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12106 {
12107 int rc;
12108
12109 ASSERT_RTNL();
12110 bnxt_hwrm_func_qcaps(bp);
12111
12112 if (netif_running(bp->dev))
12113 __bnxt_close_nic(bp, true, false);
12114
12115 bnxt_ulp_irq_stop(bp);
12116 bnxt_clear_int_mode(bp);
12117 rc = bnxt_init_int_mode(bp);
12118 bnxt_ulp_irq_restart(bp, rc);
12119
12120 if (netif_running(bp->dev)) {
12121 if (rc)
12122 dev_close(bp->dev);
12123 else
12124 rc = bnxt_open_nic(bp, true, false);
12125 }
12126
12127 return rc;
12128 }
12129
12130 static int bnxt_init_mac_addr(struct bnxt *bp)
12131 {
12132 int rc = 0;
12133
12134 if (BNXT_PF(bp)) {
12135 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12136 } else {
12137 #ifdef CONFIG_BNXT_SRIOV
12138 struct bnxt_vf_info *vf = &bp->vf;
12139 bool strict_approval = true;
12140
12141 if (is_valid_ether_addr(vf->mac_addr)) {
12142 /* overwrite netdev dev_addr with admin VF MAC */
12143 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12144 /* Older PF driver or firmware may not approve this
12145 * correctly.
12146 */
12147 strict_approval = false;
12148 } else {
12149 eth_hw_addr_random(bp->dev);
12150 }
12151 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12152 #endif
12153 }
12154 return rc;
12155 }
12156
12157 #define BNXT_VPD_LEN 512
12158 static void bnxt_vpd_read_info(struct bnxt *bp)
12159 {
12160 struct pci_dev *pdev = bp->pdev;
12161 int i, len, pos, ro_size, size;
12162 ssize_t vpd_size;
12163 u8 *vpd_data;
12164
12165 vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12166 if (!vpd_data)
12167 return;
12168
12169 vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12170 if (vpd_size <= 0) {
12171 netdev_err(bp->dev, "Unable to read VPD\n");
12172 goto exit;
12173 }
12174
12175 i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12176 if (i < 0) {
12177 netdev_err(bp->dev, "VPD READ-Only not found\n");
12178 goto exit;
12179 }
12180
12181 ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12182 i += PCI_VPD_LRDT_TAG_SIZE;
12183 if (i + ro_size > vpd_size)
12184 goto exit;
12185
12186 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12187 PCI_VPD_RO_KEYWORD_PARTNO);
12188 if (pos < 0)
12189 goto read_sn;
12190
12191 len = pci_vpd_info_field_size(&vpd_data[pos]);
12192 pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12193 if (len + pos > vpd_size)
12194 goto read_sn;
12195
12196 size = min(len, BNXT_VPD_FLD_LEN - 1);
12197 memcpy(bp->board_partno, &vpd_data[pos], size);
12198
12199 read_sn:
12200 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12201 PCI_VPD_RO_KEYWORD_SERIALNO);
12202 if (pos < 0)
12203 goto exit;
12204
12205 len = pci_vpd_info_field_size(&vpd_data[pos]);
12206 pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12207 if (len + pos > vpd_size)
12208 goto exit;
12209
12210 size = min(len, BNXT_VPD_FLD_LEN - 1);
12211 memcpy(bp->board_serialno, &vpd_data[pos], size);
12212 exit:
12213 kfree(vpd_data);
12214 }
12215
12216 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12217 {
12218 struct pci_dev *pdev = bp->pdev;
12219 u64 qword;
12220
12221 qword = pci_get_dsn(pdev);
12222 if (!qword) {
12223 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12224 return -EOPNOTSUPP;
12225 }
12226
12227 put_unaligned_le64(qword, dsn);
12228
12229 bp->flags |= BNXT_FLAG_DSN_VALID;
12230 return 0;
12231 }
12232
12233 static int bnxt_map_db_bar(struct bnxt *bp)
12234 {
12235 if (!bp->db_size)
12236 return -ENODEV;
12237 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12238 if (!bp->bar1)
12239 return -ENOMEM;
12240 return 0;
12241 }
12242
12243 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12244 {
12245 struct net_device *dev;
12246 struct bnxt *bp;
12247 int rc, max_irqs;
12248
12249 if (pci_is_bridge(pdev))
12250 return -ENODEV;
12251
12252 /* Clear any pending DMA transactions from crash kernel
12253 * while loading driver in capture kernel.
12254 */
12255 if (is_kdump_kernel()) {
12256 pci_clear_master(pdev);
12257 pcie_flr(pdev);
12258 }
12259
12260 max_irqs = bnxt_get_max_irq(pdev);
12261 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12262 if (!dev)
12263 return -ENOMEM;
12264
12265 bp = netdev_priv(dev);
12266 bnxt_set_max_func_irqs(bp, max_irqs);
12267
12268 if (bnxt_vf_pciid(ent->driver_data))
12269 bp->flags |= BNXT_FLAG_VF;
12270
12271 if (pdev->msix_cap)
12272 bp->flags |= BNXT_FLAG_MSIX_CAP;
12273
12274 rc = bnxt_init_board(pdev, dev);
12275 if (rc < 0)
12276 goto init_err_free;
12277
12278 dev->netdev_ops = &bnxt_netdev_ops;
12279 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12280 dev->ethtool_ops = &bnxt_ethtool_ops;
12281 pci_set_drvdata(pdev, dev);
12282
12283 if (BNXT_PF(bp))
12284 bnxt_vpd_read_info(bp);
12285
12286 rc = bnxt_alloc_hwrm_resources(bp);
12287 if (rc)
12288 goto init_err_pci_clean;
12289
12290 mutex_init(&bp->hwrm_cmd_lock);
12291 mutex_init(&bp->link_lock);
12292
12293 rc = bnxt_fw_init_one_p1(bp);
12294 if (rc)
12295 goto init_err_pci_clean;
12296
12297 if (BNXT_CHIP_P5(bp)) {
12298 bp->flags |= BNXT_FLAG_CHIP_P5;
12299 if (BNXT_CHIP_SR2(bp))
12300 bp->flags |= BNXT_FLAG_CHIP_SR2;
12301 }
12302
12303 rc = bnxt_alloc_rss_indir_tbl(bp);
12304 if (rc)
12305 goto init_err_pci_clean;
12306
12307 rc = bnxt_fw_init_one_p2(bp);
12308 if (rc)
12309 goto init_err_pci_clean;
12310
12311 rc = bnxt_map_db_bar(bp);
12312 if (rc) {
12313 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12314 rc);
12315 goto init_err_pci_clean;
12316 }
12317
12318 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12319 NETIF_F_TSO | NETIF_F_TSO6 |
12320 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12321 NETIF_F_GSO_IPXIP4 |
12322 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12323 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12324 NETIF_F_RXCSUM | NETIF_F_GRO;
12325
12326 if (BNXT_SUPPORTS_TPA(bp))
12327 dev->hw_features |= NETIF_F_LRO;
12328
12329 dev->hw_enc_features =
12330 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12331 NETIF_F_TSO | NETIF_F_TSO6 |
12332 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12333 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12334 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12335 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12336
12337 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12338 NETIF_F_GSO_GRE_CSUM;
12339 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12340 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12341 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12342 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12343 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12344 if (BNXT_SUPPORTS_TPA(bp))
12345 dev->hw_features |= NETIF_F_GRO_HW;
12346 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12347 if (dev->features & NETIF_F_GRO_HW)
12348 dev->features &= ~NETIF_F_LRO;
12349 dev->priv_flags |= IFF_UNICAST_FLT;
12350
12351 #ifdef CONFIG_BNXT_SRIOV
12352 init_waitqueue_head(&bp->sriov_cfg_wait);
12353 mutex_init(&bp->sriov_lock);
12354 #endif
12355 if (BNXT_SUPPORTS_TPA(bp)) {
12356 bp->gro_func = bnxt_gro_func_5730x;
12357 if (BNXT_CHIP_P4(bp))
12358 bp->gro_func = bnxt_gro_func_5731x;
12359 else if (BNXT_CHIP_P5(bp))
12360 bp->gro_func = bnxt_gro_func_5750x;
12361 }
12362 if (!BNXT_CHIP_P4_PLUS(bp))
12363 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12364
12365 bp->ulp_probe = bnxt_ulp_probe;
12366
12367 rc = bnxt_init_mac_addr(bp);
12368 if (rc) {
12369 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12370 rc = -EADDRNOTAVAIL;
12371 goto init_err_pci_clean;
12372 }
12373
12374 if (BNXT_PF(bp)) {
12375 /* Read the adapter's DSN to use as the eswitch switch_id */
12376 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12377 }
12378
12379 /* MTU range: 60 - FW defined max */
12380 dev->min_mtu = ETH_ZLEN;
12381 dev->max_mtu = bp->max_mtu;
12382
12383 rc = bnxt_probe_phy(bp, true);
12384 if (rc)
12385 goto init_err_pci_clean;
12386
12387 bnxt_set_rx_skb_mode(bp, false);
12388 bnxt_set_tpa_flags(bp);
12389 bnxt_set_ring_params(bp);
12390 rc = bnxt_set_dflt_rings(bp, true);
12391 if (rc) {
12392 netdev_err(bp->dev, "Not enough rings available.\n");
12393 rc = -ENOMEM;
12394 goto init_err_pci_clean;
12395 }
12396
12397 bnxt_fw_init_one_p3(bp);
12398
12399 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12400 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12401
12402 rc = bnxt_init_int_mode(bp);
12403 if (rc)
12404 goto init_err_pci_clean;
12405
12406 /* No TC has been set yet and rings may have been trimmed due to
12407 * limited MSIX, so we re-initialize the TX rings per TC.
12408 */
12409 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12410
12411 if (BNXT_PF(bp)) {
12412 if (!bnxt_pf_wq) {
12413 bnxt_pf_wq =
12414 create_singlethread_workqueue("bnxt_pf_wq");
12415 if (!bnxt_pf_wq) {
12416 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12417 goto init_err_pci_clean;
12418 }
12419 }
12420 rc = bnxt_init_tc(bp);
12421 if (rc)
12422 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12423 rc);
12424 }
12425
12426 bnxt_dl_register(bp);
12427
12428 rc = register_netdev(dev);
12429 if (rc)
12430 goto init_err_cleanup;
12431
12432 if (BNXT_PF(bp))
12433 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12434 bnxt_dl_fw_reporters_create(bp);
12435
12436 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12437 board_info[ent->driver_data].name,
12438 (long)pci_resource_start(pdev, 0), dev->dev_addr);
12439 pcie_print_link_status(pdev);
12440
12441 pci_save_state(pdev);
12442 return 0;
12443
12444 init_err_cleanup:
12445 bnxt_dl_unregister(bp);
12446 bnxt_shutdown_tc(bp);
12447 bnxt_clear_int_mode(bp);
12448
12449 init_err_pci_clean:
12450 bnxt_hwrm_func_drv_unrgtr(bp);
12451 bnxt_free_hwrm_short_cmd_req(bp);
12452 bnxt_free_hwrm_resources(bp);
12453 kfree(bp->fw_health);
12454 bp->fw_health = NULL;
12455 bnxt_cleanup_pci(bp);
12456 bnxt_free_ctx_mem(bp);
12457 kfree(bp->ctx);
12458 bp->ctx = NULL;
12459 kfree(bp->rss_indir_tbl);
12460 bp->rss_indir_tbl = NULL;
12461
12462 init_err_free:
12463 free_netdev(dev);
12464 return rc;
12465 }
12466
12467 static void bnxt_shutdown(struct pci_dev *pdev)
12468 {
12469 struct net_device *dev = pci_get_drvdata(pdev);
12470 struct bnxt *bp;
12471
12472 if (!dev)
12473 return;
12474
12475 rtnl_lock();
12476 bp = netdev_priv(dev);
12477 if (!bp)
12478 goto shutdown_exit;
12479
12480 if (netif_running(dev))
12481 dev_close(dev);
12482
12483 bnxt_ulp_shutdown(bp);
12484 bnxt_clear_int_mode(bp);
12485 pci_disable_device(pdev);
12486
12487 if (system_state == SYSTEM_POWER_OFF) {
12488 pci_wake_from_d3(pdev, bp->wol);
12489 pci_set_power_state(pdev, PCI_D3hot);
12490 }
12491
12492 shutdown_exit:
12493 rtnl_unlock();
12494 }
12495
12496 #ifdef CONFIG_PM_SLEEP
12497 static int bnxt_suspend(struct device *device)
12498 {
12499 struct net_device *dev = dev_get_drvdata(device);
12500 struct bnxt *bp = netdev_priv(dev);
12501 int rc = 0;
12502
12503 rtnl_lock();
12504 bnxt_ulp_stop(bp);
12505 if (netif_running(dev)) {
12506 netif_device_detach(dev);
12507 rc = bnxt_close(dev);
12508 }
12509 bnxt_hwrm_func_drv_unrgtr(bp);
12510 pci_disable_device(bp->pdev);
12511 bnxt_free_ctx_mem(bp);
12512 kfree(bp->ctx);
12513 bp->ctx = NULL;
12514 rtnl_unlock();
12515 return rc;
12516 }
12517
12518 static int bnxt_resume(struct device *device)
12519 {
12520 struct net_device *dev = dev_get_drvdata(device);
12521 struct bnxt *bp = netdev_priv(dev);
12522 int rc = 0;
12523
12524 rtnl_lock();
12525 rc = pci_enable_device(bp->pdev);
12526 if (rc) {
12527 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12528 rc);
12529 goto resume_exit;
12530 }
12531 pci_set_master(bp->pdev);
12532 if (bnxt_hwrm_ver_get(bp)) {
12533 rc = -ENODEV;
12534 goto resume_exit;
12535 }
12536 rc = bnxt_hwrm_func_reset(bp);
12537 if (rc) {
12538 rc = -EBUSY;
12539 goto resume_exit;
12540 }
12541
12542 rc = bnxt_hwrm_func_qcaps(bp);
12543 if (rc)
12544 goto resume_exit;
12545
12546 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12547 rc = -ENODEV;
12548 goto resume_exit;
12549 }
12550
12551 bnxt_get_wol_settings(bp);
12552 if (netif_running(dev)) {
12553 rc = bnxt_open(dev);
12554 if (!rc)
12555 netif_device_attach(dev);
12556 }
12557
12558 resume_exit:
12559 bnxt_ulp_start(bp, rc);
12560 if (!rc)
12561 bnxt_reenable_sriov(bp);
12562 rtnl_unlock();
12563 return rc;
12564 }
12565
12566 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12567 #define BNXT_PM_OPS (&bnxt_pm_ops)
12568
12569 #else
12570
12571 #define BNXT_PM_OPS NULL
12572
12573 #endif /* CONFIG_PM_SLEEP */
12574
12575 /**
12576 * bnxt_io_error_detected - called when PCI error is detected
12577 * @pdev: Pointer to PCI device
12578 * @state: The current pci connection state
12579 *
12580 * This function is called after a PCI bus error affecting
12581 * this device has been detected.
12582 */
12583 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12584 pci_channel_state_t state)
12585 {
12586 struct net_device *netdev = pci_get_drvdata(pdev);
12587 struct bnxt *bp = netdev_priv(netdev);
12588
12589 netdev_info(netdev, "PCI I/O error detected\n");
12590
12591 rtnl_lock();
12592 netif_device_detach(netdev);
12593
12594 bnxt_ulp_stop(bp);
12595
12596 if (state == pci_channel_io_perm_failure) {
12597 rtnl_unlock();
12598 return PCI_ERS_RESULT_DISCONNECT;
12599 }
12600
12601 if (netif_running(netdev))
12602 bnxt_close(netdev);
12603
12604 pci_disable_device(pdev);
12605 bnxt_free_ctx_mem(bp);
12606 kfree(bp->ctx);
12607 bp->ctx = NULL;
12608 rtnl_unlock();
12609
12610 /* Request a slot slot reset. */
12611 return PCI_ERS_RESULT_NEED_RESET;
12612 }
12613
12614 /**
12615 * bnxt_io_slot_reset - called after the pci bus has been reset.
12616 * @pdev: Pointer to PCI device
12617 *
12618 * Restart the card from scratch, as if from a cold-boot.
12619 * At this point, the card has exprienced a hard reset,
12620 * followed by fixups by BIOS, and has its config space
12621 * set up identically to what it was at cold boot.
12622 */
12623 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12624 {
12625 struct net_device *netdev = pci_get_drvdata(pdev);
12626 struct bnxt *bp = netdev_priv(netdev);
12627 int err = 0;
12628 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12629
12630 netdev_info(bp->dev, "PCI Slot Reset\n");
12631
12632 rtnl_lock();
12633
12634 if (pci_enable_device(pdev)) {
12635 dev_err(&pdev->dev,
12636 "Cannot re-enable PCI device after reset.\n");
12637 } else {
12638 pci_set_master(pdev);
12639 pci_restore_state(pdev);
12640 pci_save_state(pdev);
12641
12642 err = bnxt_hwrm_func_reset(bp);
12643 if (!err) {
12644 err = bnxt_hwrm_func_qcaps(bp);
12645 if (!err && netif_running(netdev))
12646 err = bnxt_open(netdev);
12647 }
12648 bnxt_ulp_start(bp, err);
12649 if (!err) {
12650 bnxt_reenable_sriov(bp);
12651 result = PCI_ERS_RESULT_RECOVERED;
12652 }
12653 }
12654
12655 if (result != PCI_ERS_RESULT_RECOVERED) {
12656 if (netif_running(netdev))
12657 dev_close(netdev);
12658 pci_disable_device(pdev);
12659 }
12660
12661 rtnl_unlock();
12662
12663 return result;
12664 }
12665
12666 /**
12667 * bnxt_io_resume - called when traffic can start flowing again.
12668 * @pdev: Pointer to PCI device
12669 *
12670 * This callback is called when the error recovery driver tells
12671 * us that its OK to resume normal operation.
12672 */
12673 static void bnxt_io_resume(struct pci_dev *pdev)
12674 {
12675 struct net_device *netdev = pci_get_drvdata(pdev);
12676
12677 rtnl_lock();
12678
12679 netif_device_attach(netdev);
12680
12681 rtnl_unlock();
12682 }
12683
12684 static const struct pci_error_handlers bnxt_err_handler = {
12685 .error_detected = bnxt_io_error_detected,
12686 .slot_reset = bnxt_io_slot_reset,
12687 .resume = bnxt_io_resume
12688 };
12689
12690 static struct pci_driver bnxt_pci_driver = {
12691 .name = DRV_MODULE_NAME,
12692 .id_table = bnxt_pci_tbl,
12693 .probe = bnxt_init_one,
12694 .remove = bnxt_remove_one,
12695 .shutdown = bnxt_shutdown,
12696 .driver.pm = BNXT_PM_OPS,
12697 .err_handler = &bnxt_err_handler,
12698 #if defined(CONFIG_BNXT_SRIOV)
12699 .sriov_configure = bnxt_sriov_configure,
12700 #endif
12701 };
12702
12703 static int __init bnxt_init(void)
12704 {
12705 bnxt_debug_init();
12706 return pci_register_driver(&bnxt_pci_driver);
12707 }
12708
12709 static void __exit bnxt_exit(void)
12710 {
12711 pci_unregister_driver(&bnxt_pci_driver);
12712 if (bnxt_pf_wq)
12713 destroy_workqueue(bnxt_pf_wq);
12714 bnxt_debug_exit();
12715 }
12716
12717 module_init(bnxt_init);
12718 module_exit(bnxt_exit);
Ubuntu Jammy Linux kernel mirror