]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
octeontx2-af: Fix LBK backpressure id count
[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / marvell / octeontx2 / af / rvu_nix.c
1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Admin Function driver
3 *
4 * Copyright (C) 2018 Marvell.
5 *
6 */
7
8 #include <linux/module.h>
9 #include <linux/pci.h>
10
11 #include "rvu_struct.h"
12 #include "rvu_reg.h"
13 #include "rvu.h"
14 #include "npc.h"
15 #include "cgx.h"
16 #include "lmac_common.h"
17
18 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc);
19 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
20 int type, int chan_id);
21 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc,
22 int type, bool add);
23 static int nix_setup_ipolicers(struct rvu *rvu,
24 struct nix_hw *nix_hw, int blkaddr);
25 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw);
26 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req,
27 struct nix_hw *nix_hw, u16 pcifunc);
28 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc);
29 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw,
30 u32 leaf_prof);
31
32 enum mc_tbl_sz {
33 MC_TBL_SZ_256,
34 MC_TBL_SZ_512,
35 MC_TBL_SZ_1K,
36 MC_TBL_SZ_2K,
37 MC_TBL_SZ_4K,
38 MC_TBL_SZ_8K,
39 MC_TBL_SZ_16K,
40 MC_TBL_SZ_32K,
41 MC_TBL_SZ_64K,
42 };
43
44 enum mc_buf_cnt {
45 MC_BUF_CNT_8,
46 MC_BUF_CNT_16,
47 MC_BUF_CNT_32,
48 MC_BUF_CNT_64,
49 MC_BUF_CNT_128,
50 MC_BUF_CNT_256,
51 MC_BUF_CNT_512,
52 MC_BUF_CNT_1024,
53 MC_BUF_CNT_2048,
54 };
55
56 enum nix_makr_fmt_indexes {
57 NIX_MARK_CFG_IP_DSCP_RED,
58 NIX_MARK_CFG_IP_DSCP_YELLOW,
59 NIX_MARK_CFG_IP_DSCP_YELLOW_RED,
60 NIX_MARK_CFG_IP_ECN_RED,
61 NIX_MARK_CFG_IP_ECN_YELLOW,
62 NIX_MARK_CFG_IP_ECN_YELLOW_RED,
63 NIX_MARK_CFG_VLAN_DEI_RED,
64 NIX_MARK_CFG_VLAN_DEI_YELLOW,
65 NIX_MARK_CFG_VLAN_DEI_YELLOW_RED,
66 NIX_MARK_CFG_MAX,
67 };
68
69 /* For now considering MC resources needed for broadcast
70 * pkt replication only. i.e 256 HWVFs + 12 PFs.
71 */
72 #define MC_TBL_SIZE MC_TBL_SZ_512
73 #define MC_BUF_CNT MC_BUF_CNT_128
74
75 struct mce {
76 struct hlist_node node;
77 u16 pcifunc;
78 };
79
80 int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr)
81 {
82 int i = 0;
83
84 /*If blkaddr is 0, return the first nix block address*/
85 if (blkaddr == 0)
86 return rvu->nix_blkaddr[blkaddr];
87
88 while (i + 1 < MAX_NIX_BLKS) {
89 if (rvu->nix_blkaddr[i] == blkaddr)
90 return rvu->nix_blkaddr[i + 1];
91 i++;
92 }
93
94 return 0;
95 }
96
97 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc)
98 {
99 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
100 int blkaddr;
101
102 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
103 if (!pfvf->nixlf || blkaddr < 0)
104 return false;
105 return true;
106 }
107
108 int rvu_get_nixlf_count(struct rvu *rvu)
109 {
110 int blkaddr = 0, max = 0;
111 struct rvu_block *block;
112
113 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
114 while (blkaddr) {
115 block = &rvu->hw->block[blkaddr];
116 max += block->lf.max;
117 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
118 }
119 return max;
120 }
121
122 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr)
123 {
124 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
125 struct rvu_hwinfo *hw = rvu->hw;
126 int blkaddr;
127
128 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
129 if (!pfvf->nixlf || blkaddr < 0)
130 return NIX_AF_ERR_AF_LF_INVALID;
131
132 *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
133 if (*nixlf < 0)
134 return NIX_AF_ERR_AF_LF_INVALID;
135
136 if (nix_blkaddr)
137 *nix_blkaddr = blkaddr;
138
139 return 0;
140 }
141
142 int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc,
143 struct nix_hw **nix_hw, int *blkaddr)
144 {
145 struct rvu_pfvf *pfvf;
146
147 pfvf = rvu_get_pfvf(rvu, pcifunc);
148 *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
149 if (!pfvf->nixlf || *blkaddr < 0)
150 return NIX_AF_ERR_AF_LF_INVALID;
151
152 *nix_hw = get_nix_hw(rvu->hw, *blkaddr);
153 if (!*nix_hw)
154 return NIX_AF_ERR_INVALID_NIXBLK;
155 return 0;
156 }
157
158 static void nix_mce_list_init(struct nix_mce_list *list, int max)
159 {
160 INIT_HLIST_HEAD(&list->head);
161 list->count = 0;
162 list->max = max;
163 }
164
165 static u16 nix_alloc_mce_list(struct nix_mcast *mcast, int count)
166 {
167 int idx;
168
169 if (!mcast)
170 return 0;
171
172 idx = mcast->next_free_mce;
173 mcast->next_free_mce += count;
174 return idx;
175 }
176
177 struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr)
178 {
179 int nix_blkaddr = 0, i = 0;
180 struct rvu *rvu = hw->rvu;
181
182 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
183 while (nix_blkaddr) {
184 if (blkaddr == nix_blkaddr && hw->nix)
185 return &hw->nix[i];
186 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
187 i++;
188 }
189 return NULL;
190 }
191
192 u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu)
193 {
194 dwrr_mtu &= 0x1FULL;
195
196 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes.
197 * Value of 4 is reserved for MTU value of 9728 bytes.
198 * Value of 5 is reserved for MTU value of 10240 bytes.
199 */
200 switch (dwrr_mtu) {
201 case 4:
202 return 9728;
203 case 5:
204 return 10240;
205 default:
206 return BIT_ULL(dwrr_mtu);
207 }
208
209 return 0;
210 }
211
212 u32 convert_bytes_to_dwrr_mtu(u32 bytes)
213 {
214 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes.
215 * Value of 4 is reserved for MTU value of 9728 bytes.
216 * Value of 5 is reserved for MTU value of 10240 bytes.
217 */
218 if (bytes > BIT_ULL(16))
219 return 0;
220
221 switch (bytes) {
222 case 9728:
223 return 4;
224 case 10240:
225 return 5;
226 default:
227 return ilog2(bytes);
228 }
229
230 return 0;
231 }
232
233 static void nix_rx_sync(struct rvu *rvu, int blkaddr)
234 {
235 int err;
236
237 /* Sync all in flight RX packets to LLC/DRAM */
238 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
239 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
240 if (err)
241 dev_err(rvu->dev, "SYNC1: NIX RX software sync failed\n");
242
243 /* SW_SYNC ensures all existing transactions are finished and pkts
244 * are written to LLC/DRAM, queues should be teared down after
245 * successful SW_SYNC. Due to a HW errata, in some rare scenarios
246 * an existing transaction might end after SW_SYNC operation. To
247 * ensure operation is fully done, do the SW_SYNC twice.
248 */
249 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
250 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
251 if (err)
252 dev_err(rvu->dev, "SYNC2: NIX RX software sync failed\n");
253 }
254
255 static bool is_valid_txschq(struct rvu *rvu, int blkaddr,
256 int lvl, u16 pcifunc, u16 schq)
257 {
258 struct rvu_hwinfo *hw = rvu->hw;
259 struct nix_txsch *txsch;
260 struct nix_hw *nix_hw;
261 u16 map_func;
262
263 nix_hw = get_nix_hw(rvu->hw, blkaddr);
264 if (!nix_hw)
265 return false;
266
267 txsch = &nix_hw->txsch[lvl];
268 /* Check out of bounds */
269 if (schq >= txsch->schq.max)
270 return false;
271
272 mutex_lock(&rvu->rsrc_lock);
273 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
274 mutex_unlock(&rvu->rsrc_lock);
275
276 /* TLs aggegating traffic are shared across PF and VFs */
277 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
278 if (rvu_get_pf(map_func) != rvu_get_pf(pcifunc))
279 return false;
280 else
281 return true;
282 }
283
284 if (map_func != pcifunc)
285 return false;
286
287 return true;
288 }
289
290 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf,
291 struct nix_lf_alloc_rsp *rsp, bool loop)
292 {
293 struct rvu_pfvf *parent_pf, *pfvf = rvu_get_pfvf(rvu, pcifunc);
294 u16 req_chan_base, req_chan_end, req_chan_cnt;
295 struct rvu_hwinfo *hw = rvu->hw;
296 struct sdp_node_info *sdp_info;
297 int pkind, pf, vf, lbkid, vfid;
298 struct mac_ops *mac_ops;
299 u8 cgx_id, lmac_id;
300 bool from_vf;
301 int err;
302
303 pf = rvu_get_pf(pcifunc);
304 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK &&
305 type != NIX_INTF_TYPE_SDP)
306 return 0;
307
308 switch (type) {
309 case NIX_INTF_TYPE_CGX:
310 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf];
311 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
312
313 pkind = rvu_npc_get_pkind(rvu, pf);
314 if (pkind < 0) {
315 dev_err(rvu->dev,
316 "PF_Func 0x%x: Invalid pkind\n", pcifunc);
317 return -EINVAL;
318 }
319 pfvf->rx_chan_base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0);
320 pfvf->tx_chan_base = pfvf->rx_chan_base;
321 pfvf->rx_chan_cnt = 1;
322 pfvf->tx_chan_cnt = 1;
323 rsp->tx_link = cgx_id * hw->lmac_per_cgx + lmac_id;
324
325 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind);
326 rvu_npc_set_pkind(rvu, pkind, pfvf);
327
328 mac_ops = get_mac_ops(rvu_cgx_pdata(cgx_id, rvu));
329
330 /* By default we enable pause frames */
331 if ((pcifunc & RVU_PFVF_FUNC_MASK) == 0)
332 mac_ops->mac_enadis_pause_frm(rvu_cgx_pdata(cgx_id,
333 rvu),
334 lmac_id, true, true);
335 break;
336 case NIX_INTF_TYPE_LBK:
337 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
338
339 /* If NIX1 block is present on the silicon then NIXes are
340 * assigned alternatively for lbk interfaces. NIX0 should
341 * send packets on lbk link 1 channels and NIX1 should send
342 * on lbk link 0 channels for the communication between
343 * NIX0 and NIX1.
344 */
345 lbkid = 0;
346 if (rvu->hw->lbk_links > 1)
347 lbkid = vf & 0x1 ? 0 : 1;
348
349 /* By default NIX0 is configured to send packet on lbk link 1
350 * (which corresponds to LBK1), same packet will receive on
351 * NIX1 over lbk link 0. If NIX1 sends packet on lbk link 0
352 * (which corresponds to LBK2) packet will receive on NIX0 lbk
353 * link 1.
354 * But if lbk links for NIX0 and NIX1 are negated, i.e NIX0
355 * transmits and receives on lbk link 0, whick corresponds
356 * to LBK1 block, back to back connectivity between NIX and
357 * LBK can be achieved (which is similar to 96xx)
358 *
359 * RX TX
360 * NIX0 lbk link 1 (LBK2) 1 (LBK1)
361 * NIX0 lbk link 0 (LBK0) 0 (LBK0)
362 * NIX1 lbk link 0 (LBK1) 0 (LBK2)
363 * NIX1 lbk link 1 (LBK3) 1 (LBK3)
364 */
365 if (loop)
366 lbkid = !lbkid;
367
368 /* Note that AF's VFs work in pairs and talk over consecutive
369 * loopback channels.Therefore if odd number of AF VFs are
370 * enabled then the last VF remains with no pair.
371 */
372 pfvf->rx_chan_base = rvu_nix_chan_lbk(rvu, lbkid, vf);
373 pfvf->tx_chan_base = vf & 0x1 ?
374 rvu_nix_chan_lbk(rvu, lbkid, vf - 1) :
375 rvu_nix_chan_lbk(rvu, lbkid, vf + 1);
376 pfvf->rx_chan_cnt = 1;
377 pfvf->tx_chan_cnt = 1;
378 rsp->tx_link = hw->cgx_links + lbkid;
379 pfvf->lbkid = lbkid;
380 rvu_npc_set_pkind(rvu, NPC_RX_LBK_PKIND, pfvf);
381 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
382 pfvf->rx_chan_base,
383 pfvf->rx_chan_cnt);
384
385 break;
386 case NIX_INTF_TYPE_SDP:
387 from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK);
388 parent_pf = &rvu->pf[rvu_get_pf(pcifunc)];
389 sdp_info = parent_pf->sdp_info;
390 if (!sdp_info) {
391 dev_err(rvu->dev, "Invalid sdp_info pointer\n");
392 return -EINVAL;
393 }
394 if (from_vf) {
395 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn +
396 sdp_info->num_pf_rings;
397 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
398 for (vfid = 0; vfid < vf; vfid++)
399 req_chan_base += sdp_info->vf_rings[vfid];
400 req_chan_cnt = sdp_info->vf_rings[vf];
401 req_chan_end = req_chan_base + req_chan_cnt - 1;
402 if (req_chan_base < rvu_nix_chan_sdp(rvu, 0) ||
403 req_chan_end > rvu_nix_chan_sdp(rvu, 255)) {
404 dev_err(rvu->dev,
405 "PF_Func 0x%x: Invalid channel base and count\n",
406 pcifunc);
407 return -EINVAL;
408 }
409 } else {
410 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn;
411 req_chan_cnt = sdp_info->num_pf_rings;
412 }
413
414 pfvf->rx_chan_base = req_chan_base;
415 pfvf->rx_chan_cnt = req_chan_cnt;
416 pfvf->tx_chan_base = pfvf->rx_chan_base;
417 pfvf->tx_chan_cnt = pfvf->rx_chan_cnt;
418
419 rsp->tx_link = hw->cgx_links + hw->lbk_links;
420 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
421 pfvf->rx_chan_base,
422 pfvf->rx_chan_cnt);
423 break;
424 }
425
426 /* Add a UCAST forwarding rule in MCAM with this NIXLF attached
427 * RVU PF/VF's MAC address.
428 */
429 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
430 pfvf->rx_chan_base, pfvf->mac_addr);
431
432 /* Add this PF_FUNC to bcast pkt replication list */
433 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true);
434 if (err) {
435 dev_err(rvu->dev,
436 "Bcast list, failed to enable PF_FUNC 0x%x\n",
437 pcifunc);
438 return err;
439 }
440 /* Install MCAM rule matching Ethernet broadcast mac address */
441 rvu_npc_install_bcast_match_entry(rvu, pcifunc,
442 nixlf, pfvf->rx_chan_base);
443
444 pfvf->maxlen = NIC_HW_MIN_FRS;
445 pfvf->minlen = NIC_HW_MIN_FRS;
446
447 return 0;
448 }
449
450 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf)
451 {
452 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
453 int err;
454
455 pfvf->maxlen = 0;
456 pfvf->minlen = 0;
457
458 /* Remove this PF_FUNC from bcast pkt replication list */
459 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false);
460 if (err) {
461 dev_err(rvu->dev,
462 "Bcast list, failed to disable PF_FUNC 0x%x\n",
463 pcifunc);
464 }
465
466 /* Free and disable any MCAM entries used by this NIX LF */
467 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
468
469 /* Disable DMAC filters used */
470 rvu_cgx_disable_dmac_entries(rvu, pcifunc);
471 }
472
473 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
474 struct nix_bp_cfg_req *req,
475 struct msg_rsp *rsp)
476 {
477 u16 pcifunc = req->hdr.pcifunc;
478 struct rvu_pfvf *pfvf;
479 int blkaddr, pf, type;
480 u16 chan_base, chan;
481 u64 cfg;
482
483 pf = rvu_get_pf(pcifunc);
484 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
485 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
486 return 0;
487
488 pfvf = rvu_get_pfvf(rvu, pcifunc);
489 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
490
491 chan_base = pfvf->rx_chan_base + req->chan_base;
492 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
493 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
494 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
495 cfg & ~BIT_ULL(16));
496 }
497 return 0;
498 }
499
500 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
501 int type, int chan_id)
502 {
503 int bpid, blkaddr, lmac_chan_cnt, sdp_chan_cnt;
504 u16 cgx_bpid_cnt, lbk_bpid_cnt, sdp_bpid_cnt;
505 struct rvu_hwinfo *hw = rvu->hw;
506 struct rvu_pfvf *pfvf;
507 u8 cgx_id, lmac_id;
508 u64 cfg;
509
510 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
511 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
512 lmac_chan_cnt = cfg & 0xFF;
513
514 cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt;
515 lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF);
516
517 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
518 sdp_chan_cnt = cfg & 0xFFF;
519 sdp_bpid_cnt = hw->sdp_links * sdp_chan_cnt;
520
521 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
522
523 /* Backpressure IDs range division
524 * CGX channles are mapped to (0 - 191) BPIDs
525 * LBK channles are mapped to (192 - 255) BPIDs
526 * SDP channles are mapped to (256 - 511) BPIDs
527 *
528 * Lmac channles and bpids mapped as follows
529 * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15)
530 * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) ....
531 * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) ....
532 */
533 switch (type) {
534 case NIX_INTF_TYPE_CGX:
535 if ((req->chan_base + req->chan_cnt) > 15)
536 return -EINVAL;
537 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
538 /* Assign bpid based on cgx, lmac and chan id */
539 bpid = (cgx_id * hw->lmac_per_cgx * lmac_chan_cnt) +
540 (lmac_id * lmac_chan_cnt) + req->chan_base;
541
542 if (req->bpid_per_chan)
543 bpid += chan_id;
544 if (bpid > cgx_bpid_cnt)
545 return -EINVAL;
546 break;
547
548 case NIX_INTF_TYPE_LBK:
549 if ((req->chan_base + req->chan_cnt) > 63)
550 return -EINVAL;
551 bpid = cgx_bpid_cnt + req->chan_base;
552 if (req->bpid_per_chan)
553 bpid += chan_id;
554 if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt))
555 return -EINVAL;
556 break;
557 case NIX_INTF_TYPE_SDP:
558 if ((req->chan_base + req->chan_cnt) > 255)
559 return -EINVAL;
560
561 bpid = sdp_bpid_cnt + req->chan_base;
562 if (req->bpid_per_chan)
563 bpid += chan_id;
564
565 if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt + sdp_bpid_cnt))
566 return -EINVAL;
567 break;
568 default:
569 return -EINVAL;
570 }
571 return bpid;
572 }
573
574 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
575 struct nix_bp_cfg_req *req,
576 struct nix_bp_cfg_rsp *rsp)
577 {
578 int blkaddr, pf, type, chan_id = 0;
579 u16 pcifunc = req->hdr.pcifunc;
580 struct rvu_pfvf *pfvf;
581 u16 chan_base, chan;
582 s16 bpid, bpid_base;
583 u64 cfg;
584
585 pf = rvu_get_pf(pcifunc);
586 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
587 if (is_sdp_pfvf(pcifunc))
588 type = NIX_INTF_TYPE_SDP;
589
590 /* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */
591 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK &&
592 type != NIX_INTF_TYPE_SDP)
593 return 0;
594
595 pfvf = rvu_get_pfvf(rvu, pcifunc);
596 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
597
598 bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id);
599 chan_base = pfvf->rx_chan_base + req->chan_base;
600 bpid = bpid_base;
601
602 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
603 if (bpid < 0) {
604 dev_warn(rvu->dev, "Fail to enable backpressure\n");
605 return -EINVAL;
606 }
607
608 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
609 cfg &= ~GENMASK_ULL(8, 0);
610 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
611 cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16));
612 chan_id++;
613 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id);
614 }
615
616 for (chan = 0; chan < req->chan_cnt; chan++) {
617 /* Map channel and bpid assign to it */
618 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 |
619 (bpid_base & 0x3FF);
620 if (req->bpid_per_chan)
621 bpid_base++;
622 }
623 rsp->chan_cnt = req->chan_cnt;
624
625 return 0;
626 }
627
628 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr,
629 u64 format, bool v4, u64 *fidx)
630 {
631 struct nix_lso_format field = {0};
632
633 /* IP's Length field */
634 field.layer = NIX_TXLAYER_OL3;
635 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */
636 field.offset = v4 ? 2 : 4;
637 field.sizem1 = 1; /* i.e 2 bytes */
638 field.alg = NIX_LSOALG_ADD_PAYLEN;
639 rvu_write64(rvu, blkaddr,
640 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
641 *(u64 *)&field);
642
643 /* No ID field in IPv6 header */
644 if (!v4)
645 return;
646
647 /* IP's ID field */
648 field.layer = NIX_TXLAYER_OL3;
649 field.offset = 4;
650 field.sizem1 = 1; /* i.e 2 bytes */
651 field.alg = NIX_LSOALG_ADD_SEGNUM;
652 rvu_write64(rvu, blkaddr,
653 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
654 *(u64 *)&field);
655 }
656
657 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr,
658 u64 format, u64 *fidx)
659 {
660 struct nix_lso_format field = {0};
661
662 /* TCP's sequence number field */
663 field.layer = NIX_TXLAYER_OL4;
664 field.offset = 4;
665 field.sizem1 = 3; /* i.e 4 bytes */
666 field.alg = NIX_LSOALG_ADD_OFFSET;
667 rvu_write64(rvu, blkaddr,
668 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
669 *(u64 *)&field);
670
671 /* TCP's flags field */
672 field.layer = NIX_TXLAYER_OL4;
673 field.offset = 12;
674 field.sizem1 = 1; /* 2 bytes */
675 field.alg = NIX_LSOALG_TCP_FLAGS;
676 rvu_write64(rvu, blkaddr,
677 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
678 *(u64 *)&field);
679 }
680
681 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
682 {
683 u64 cfg, idx, fidx = 0;
684
685 /* Get max HW supported format indices */
686 cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF;
687 nix_hw->lso.total = cfg;
688
689 /* Enable LSO */
690 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG);
691 /* For TSO, set first and middle segment flags to
692 * mask out PSH, RST & FIN flags in TCP packet
693 */
694 cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16));
695 cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16);
696 rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63));
697
698 /* Setup default static LSO formats
699 *
700 * Configure format fields for TCPv4 segmentation offload
701 */
702 idx = NIX_LSO_FORMAT_IDX_TSOV4;
703 nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx);
704 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
705
706 /* Set rest of the fields to NOP */
707 for (; fidx < 8; fidx++) {
708 rvu_write64(rvu, blkaddr,
709 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
710 }
711 nix_hw->lso.in_use++;
712
713 /* Configure format fields for TCPv6 segmentation offload */
714 idx = NIX_LSO_FORMAT_IDX_TSOV6;
715 fidx = 0;
716 nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx);
717 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
718
719 /* Set rest of the fields to NOP */
720 for (; fidx < 8; fidx++) {
721 rvu_write64(rvu, blkaddr,
722 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
723 }
724 nix_hw->lso.in_use++;
725 }
726
727 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
728 {
729 kfree(pfvf->rq_bmap);
730 kfree(pfvf->sq_bmap);
731 kfree(pfvf->cq_bmap);
732 if (pfvf->rq_ctx)
733 qmem_free(rvu->dev, pfvf->rq_ctx);
734 if (pfvf->sq_ctx)
735 qmem_free(rvu->dev, pfvf->sq_ctx);
736 if (pfvf->cq_ctx)
737 qmem_free(rvu->dev, pfvf->cq_ctx);
738 if (pfvf->rss_ctx)
739 qmem_free(rvu->dev, pfvf->rss_ctx);
740 if (pfvf->nix_qints_ctx)
741 qmem_free(rvu->dev, pfvf->nix_qints_ctx);
742 if (pfvf->cq_ints_ctx)
743 qmem_free(rvu->dev, pfvf->cq_ints_ctx);
744
745 pfvf->rq_bmap = NULL;
746 pfvf->cq_bmap = NULL;
747 pfvf->sq_bmap = NULL;
748 pfvf->rq_ctx = NULL;
749 pfvf->sq_ctx = NULL;
750 pfvf->cq_ctx = NULL;
751 pfvf->rss_ctx = NULL;
752 pfvf->nix_qints_ctx = NULL;
753 pfvf->cq_ints_ctx = NULL;
754 }
755
756 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr,
757 struct rvu_pfvf *pfvf, int nixlf,
758 int rss_sz, int rss_grps, int hwctx_size,
759 u64 way_mask, bool tag_lsb_as_adder)
760 {
761 int err, grp, num_indices;
762 u64 val;
763
764 /* RSS is not requested for this NIXLF */
765 if (!rss_sz)
766 return 0;
767 num_indices = rss_sz * rss_grps;
768
769 /* Alloc NIX RSS HW context memory and config the base */
770 err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size);
771 if (err)
772 return err;
773
774 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf),
775 (u64)pfvf->rss_ctx->iova);
776
777 /* Config full RSS table size, enable RSS and caching */
778 val = BIT_ULL(36) | BIT_ULL(4) | way_mask << 20 |
779 ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE);
780
781 if (tag_lsb_as_adder)
782 val |= BIT_ULL(5);
783
784 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf), val);
785 /* Config RSS group offset and sizes */
786 for (grp = 0; grp < rss_grps; grp++)
787 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp),
788 ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp));
789 return 0;
790 }
791
792 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
793 struct nix_aq_inst_s *inst)
794 {
795 struct admin_queue *aq = block->aq;
796 struct nix_aq_res_s *result;
797 int timeout = 1000;
798 u64 reg, head;
799
800 result = (struct nix_aq_res_s *)aq->res->base;
801
802 /* Get current head pointer where to append this instruction */
803 reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS);
804 head = (reg >> 4) & AQ_PTR_MASK;
805
806 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
807 (void *)inst, aq->inst->entry_sz);
808 memset(result, 0, sizeof(*result));
809 /* sync into memory */
810 wmb();
811
812 /* Ring the doorbell and wait for result */
813 rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1);
814 while (result->compcode == NIX_AQ_COMP_NOTDONE) {
815 cpu_relax();
816 udelay(1);
817 timeout--;
818 if (!timeout)
819 return -EBUSY;
820 }
821
822 if (result->compcode != NIX_AQ_COMP_GOOD)
823 /* TODO: Replace this with some error code */
824 return -EBUSY;
825
826 return 0;
827 }
828
829 static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw,
830 struct nix_aq_enq_req *req,
831 struct nix_aq_enq_rsp *rsp)
832 {
833 struct rvu_hwinfo *hw = rvu->hw;
834 u16 pcifunc = req->hdr.pcifunc;
835 int nixlf, blkaddr, rc = 0;
836 struct nix_aq_inst_s inst;
837 struct rvu_block *block;
838 struct admin_queue *aq;
839 struct rvu_pfvf *pfvf;
840 void *ctx, *mask;
841 bool ena;
842 u64 cfg;
843
844 blkaddr = nix_hw->blkaddr;
845 block = &hw->block[blkaddr];
846 aq = block->aq;
847 if (!aq) {
848 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__);
849 return NIX_AF_ERR_AQ_ENQUEUE;
850 }
851
852 pfvf = rvu_get_pfvf(rvu, pcifunc);
853 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
854
855 /* Skip NIXLF check for broadcast MCE entry and bandwidth profile
856 * operations done by AF itself.
857 */
858 if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) ||
859 (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) {
860 if (!pfvf->nixlf || nixlf < 0)
861 return NIX_AF_ERR_AF_LF_INVALID;
862 }
863
864 switch (req->ctype) {
865 case NIX_AQ_CTYPE_RQ:
866 /* Check if index exceeds max no of queues */
867 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize)
868 rc = NIX_AF_ERR_AQ_ENQUEUE;
869 break;
870 case NIX_AQ_CTYPE_SQ:
871 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize)
872 rc = NIX_AF_ERR_AQ_ENQUEUE;
873 break;
874 case NIX_AQ_CTYPE_CQ:
875 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize)
876 rc = NIX_AF_ERR_AQ_ENQUEUE;
877 break;
878 case NIX_AQ_CTYPE_RSS:
879 /* Check if RSS is enabled and qidx is within range */
880 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf));
881 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx ||
882 (req->qidx >= (256UL << (cfg & 0xF))))
883 rc = NIX_AF_ERR_AQ_ENQUEUE;
884 break;
885 case NIX_AQ_CTYPE_MCE:
886 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG);
887
888 /* Check if index exceeds MCE list length */
889 if (!nix_hw->mcast.mce_ctx ||
890 (req->qidx >= (256UL << (cfg & 0xF))))
891 rc = NIX_AF_ERR_AQ_ENQUEUE;
892
893 /* Adding multicast lists for requests from PF/VFs is not
894 * yet supported, so ignore this.
895 */
896 if (rsp)
897 rc = NIX_AF_ERR_AQ_ENQUEUE;
898 break;
899 case NIX_AQ_CTYPE_BANDPROF:
900 if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req,
901 nix_hw, pcifunc))
902 rc = NIX_AF_ERR_INVALID_BANDPROF;
903 break;
904 default:
905 rc = NIX_AF_ERR_AQ_ENQUEUE;
906 }
907
908 if (rc)
909 return rc;
910
911 /* Check if SQ pointed SMQ belongs to this PF/VF or not */
912 if (req->ctype == NIX_AQ_CTYPE_SQ &&
913 ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) ||
914 (req->op == NIX_AQ_INSTOP_WRITE &&
915 req->sq_mask.ena && req->sq_mask.smq && req->sq.ena))) {
916 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ,
917 pcifunc, req->sq.smq))
918 return NIX_AF_ERR_AQ_ENQUEUE;
919 }
920
921 memset(&inst, 0, sizeof(struct nix_aq_inst_s));
922 inst.lf = nixlf;
923 inst.cindex = req->qidx;
924 inst.ctype = req->ctype;
925 inst.op = req->op;
926 /* Currently we are not supporting enqueuing multiple instructions,
927 * so always choose first entry in result memory.
928 */
929 inst.res_addr = (u64)aq->res->iova;
930
931 /* Hardware uses same aq->res->base for updating result of
932 * previous instruction hence wait here till it is done.
933 */
934 spin_lock(&aq->lock);
935
936 /* Clean result + context memory */
937 memset(aq->res->base, 0, aq->res->entry_sz);
938 /* Context needs to be written at RES_ADDR + 128 */
939 ctx = aq->res->base + 128;
940 /* Mask needs to be written at RES_ADDR + 256 */
941 mask = aq->res->base + 256;
942
943 switch (req->op) {
944 case NIX_AQ_INSTOP_WRITE:
945 if (req->ctype == NIX_AQ_CTYPE_RQ)
946 memcpy(mask, &req->rq_mask,
947 sizeof(struct nix_rq_ctx_s));
948 else if (req->ctype == NIX_AQ_CTYPE_SQ)
949 memcpy(mask, &req->sq_mask,
950 sizeof(struct nix_sq_ctx_s));
951 else if (req->ctype == NIX_AQ_CTYPE_CQ)
952 memcpy(mask, &req->cq_mask,
953 sizeof(struct nix_cq_ctx_s));
954 else if (req->ctype == NIX_AQ_CTYPE_RSS)
955 memcpy(mask, &req->rss_mask,
956 sizeof(struct nix_rsse_s));
957 else if (req->ctype == NIX_AQ_CTYPE_MCE)
958 memcpy(mask, &req->mce_mask,
959 sizeof(struct nix_rx_mce_s));
960 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
961 memcpy(mask, &req->prof_mask,
962 sizeof(struct nix_bandprof_s));
963 fallthrough;
964 case NIX_AQ_INSTOP_INIT:
965 if (req->ctype == NIX_AQ_CTYPE_RQ)
966 memcpy(ctx, &req->rq, sizeof(struct nix_rq_ctx_s));
967 else if (req->ctype == NIX_AQ_CTYPE_SQ)
968 memcpy(ctx, &req->sq, sizeof(struct nix_sq_ctx_s));
969 else if (req->ctype == NIX_AQ_CTYPE_CQ)
970 memcpy(ctx, &req->cq, sizeof(struct nix_cq_ctx_s));
971 else if (req->ctype == NIX_AQ_CTYPE_RSS)
972 memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s));
973 else if (req->ctype == NIX_AQ_CTYPE_MCE)
974 memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s));
975 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
976 memcpy(ctx, &req->prof, sizeof(struct nix_bandprof_s));
977 break;
978 case NIX_AQ_INSTOP_NOP:
979 case NIX_AQ_INSTOP_READ:
980 case NIX_AQ_INSTOP_LOCK:
981 case NIX_AQ_INSTOP_UNLOCK:
982 break;
983 default:
984 rc = NIX_AF_ERR_AQ_ENQUEUE;
985 spin_unlock(&aq->lock);
986 return rc;
987 }
988
989 /* Submit the instruction to AQ */
990 rc = nix_aq_enqueue_wait(rvu, block, &inst);
991 if (rc) {
992 spin_unlock(&aq->lock);
993 return rc;
994 }
995
996 /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */
997 if (req->op == NIX_AQ_INSTOP_INIT) {
998 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena)
999 __set_bit(req->qidx, pfvf->rq_bmap);
1000 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena)
1001 __set_bit(req->qidx, pfvf->sq_bmap);
1002 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena)
1003 __set_bit(req->qidx, pfvf->cq_bmap);
1004 }
1005
1006 if (req->op == NIX_AQ_INSTOP_WRITE) {
1007 if (req->ctype == NIX_AQ_CTYPE_RQ) {
1008 ena = (req->rq.ena & req->rq_mask.ena) |
1009 (test_bit(req->qidx, pfvf->rq_bmap) &
1010 ~req->rq_mask.ena);
1011 if (ena)
1012 __set_bit(req->qidx, pfvf->rq_bmap);
1013 else
1014 __clear_bit(req->qidx, pfvf->rq_bmap);
1015 }
1016 if (req->ctype == NIX_AQ_CTYPE_SQ) {
1017 ena = (req->rq.ena & req->sq_mask.ena) |
1018 (test_bit(req->qidx, pfvf->sq_bmap) &
1019 ~req->sq_mask.ena);
1020 if (ena)
1021 __set_bit(req->qidx, pfvf->sq_bmap);
1022 else
1023 __clear_bit(req->qidx, pfvf->sq_bmap);
1024 }
1025 if (req->ctype == NIX_AQ_CTYPE_CQ) {
1026 ena = (req->rq.ena & req->cq_mask.ena) |
1027 (test_bit(req->qidx, pfvf->cq_bmap) &
1028 ~req->cq_mask.ena);
1029 if (ena)
1030 __set_bit(req->qidx, pfvf->cq_bmap);
1031 else
1032 __clear_bit(req->qidx, pfvf->cq_bmap);
1033 }
1034 }
1035
1036 if (rsp) {
1037 /* Copy read context into mailbox */
1038 if (req->op == NIX_AQ_INSTOP_READ) {
1039 if (req->ctype == NIX_AQ_CTYPE_RQ)
1040 memcpy(&rsp->rq, ctx,
1041 sizeof(struct nix_rq_ctx_s));
1042 else if (req->ctype == NIX_AQ_CTYPE_SQ)
1043 memcpy(&rsp->sq, ctx,
1044 sizeof(struct nix_sq_ctx_s));
1045 else if (req->ctype == NIX_AQ_CTYPE_CQ)
1046 memcpy(&rsp->cq, ctx,
1047 sizeof(struct nix_cq_ctx_s));
1048 else if (req->ctype == NIX_AQ_CTYPE_RSS)
1049 memcpy(&rsp->rss, ctx,
1050 sizeof(struct nix_rsse_s));
1051 else if (req->ctype == NIX_AQ_CTYPE_MCE)
1052 memcpy(&rsp->mce, ctx,
1053 sizeof(struct nix_rx_mce_s));
1054 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
1055 memcpy(&rsp->prof, ctx,
1056 sizeof(struct nix_bandprof_s));
1057 }
1058 }
1059
1060 spin_unlock(&aq->lock);
1061 return 0;
1062 }
1063
1064 static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
1065 struct nix_aq_enq_rsp *rsp)
1066 {
1067 struct nix_hw *nix_hw;
1068 int blkaddr;
1069
1070 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
1071 if (blkaddr < 0)
1072 return NIX_AF_ERR_AF_LF_INVALID;
1073
1074 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1075 if (!nix_hw)
1076 return NIX_AF_ERR_INVALID_NIXBLK;
1077
1078 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp);
1079 }
1080
1081 static const char *nix_get_ctx_name(int ctype)
1082 {
1083 switch (ctype) {
1084 case NIX_AQ_CTYPE_CQ:
1085 return "CQ";
1086 case NIX_AQ_CTYPE_SQ:
1087 return "SQ";
1088 case NIX_AQ_CTYPE_RQ:
1089 return "RQ";
1090 case NIX_AQ_CTYPE_RSS:
1091 return "RSS";
1092 }
1093 return "";
1094 }
1095
1096 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
1097 {
1098 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
1099 struct nix_aq_enq_req aq_req;
1100 unsigned long *bmap;
1101 int qidx, q_cnt = 0;
1102 int err = 0, rc;
1103
1104 if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx)
1105 return NIX_AF_ERR_AQ_ENQUEUE;
1106
1107 memset(&aq_req, 0, sizeof(struct nix_aq_enq_req));
1108 aq_req.hdr.pcifunc = req->hdr.pcifunc;
1109
1110 if (req->ctype == NIX_AQ_CTYPE_CQ) {
1111 aq_req.cq.ena = 0;
1112 aq_req.cq_mask.ena = 1;
1113 aq_req.cq.bp_ena = 0;
1114 aq_req.cq_mask.bp_ena = 1;
1115 q_cnt = pfvf->cq_ctx->qsize;
1116 bmap = pfvf->cq_bmap;
1117 }
1118 if (req->ctype == NIX_AQ_CTYPE_SQ) {
1119 aq_req.sq.ena = 0;
1120 aq_req.sq_mask.ena = 1;
1121 q_cnt = pfvf->sq_ctx->qsize;
1122 bmap = pfvf->sq_bmap;
1123 }
1124 if (req->ctype == NIX_AQ_CTYPE_RQ) {
1125 aq_req.rq.ena = 0;
1126 aq_req.rq_mask.ena = 1;
1127 q_cnt = pfvf->rq_ctx->qsize;
1128 bmap = pfvf->rq_bmap;
1129 }
1130
1131 aq_req.ctype = req->ctype;
1132 aq_req.op = NIX_AQ_INSTOP_WRITE;
1133
1134 for (qidx = 0; qidx < q_cnt; qidx++) {
1135 if (!test_bit(qidx, bmap))
1136 continue;
1137 aq_req.qidx = qidx;
1138 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL);
1139 if (rc) {
1140 err = rc;
1141 dev_err(rvu->dev, "Failed to disable %s:%d context\n",
1142 nix_get_ctx_name(req->ctype), qidx);
1143 }
1144 }
1145
1146 return err;
1147 }
1148
1149 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
1150 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req)
1151 {
1152 struct nix_aq_enq_req lock_ctx_req;
1153 int err;
1154
1155 if (req->op != NIX_AQ_INSTOP_INIT)
1156 return 0;
1157
1158 if (req->ctype == NIX_AQ_CTYPE_MCE ||
1159 req->ctype == NIX_AQ_CTYPE_DYNO)
1160 return 0;
1161
1162 memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req));
1163 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
1164 lock_ctx_req.ctype = req->ctype;
1165 lock_ctx_req.op = NIX_AQ_INSTOP_LOCK;
1166 lock_ctx_req.qidx = req->qidx;
1167 err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL);
1168 if (err)
1169 dev_err(rvu->dev,
1170 "PFUNC 0x%x: Failed to lock NIX %s:%d context\n",
1171 req->hdr.pcifunc,
1172 nix_get_ctx_name(req->ctype), req->qidx);
1173 return err;
1174 }
1175
1176 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
1177 struct nix_aq_enq_req *req,
1178 struct nix_aq_enq_rsp *rsp)
1179 {
1180 int err;
1181
1182 err = rvu_nix_aq_enq_inst(rvu, req, rsp);
1183 if (!err)
1184 err = nix_lf_hwctx_lockdown(rvu, req);
1185 return err;
1186 }
1187 #else
1188
1189 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
1190 struct nix_aq_enq_req *req,
1191 struct nix_aq_enq_rsp *rsp)
1192 {
1193 return rvu_nix_aq_enq_inst(rvu, req, rsp);
1194 }
1195 #endif
1196 /* CN10K mbox handler */
1197 int rvu_mbox_handler_nix_cn10k_aq_enq(struct rvu *rvu,
1198 struct nix_cn10k_aq_enq_req *req,
1199 struct nix_cn10k_aq_enq_rsp *rsp)
1200 {
1201 return rvu_nix_aq_enq_inst(rvu, (struct nix_aq_enq_req *)req,
1202 (struct nix_aq_enq_rsp *)rsp);
1203 }
1204
1205 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu,
1206 struct hwctx_disable_req *req,
1207 struct msg_rsp *rsp)
1208 {
1209 return nix_lf_hwctx_disable(rvu, req);
1210 }
1211
1212 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu,
1213 struct nix_lf_alloc_req *req,
1214 struct nix_lf_alloc_rsp *rsp)
1215 {
1216 int nixlf, qints, hwctx_size, intf, err, rc = 0;
1217 struct rvu_hwinfo *hw = rvu->hw;
1218 u16 pcifunc = req->hdr.pcifunc;
1219 struct rvu_block *block;
1220 struct rvu_pfvf *pfvf;
1221 u64 cfg, ctx_cfg;
1222 int blkaddr;
1223
1224 if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt)
1225 return NIX_AF_ERR_PARAM;
1226
1227 if (req->way_mask)
1228 req->way_mask &= 0xFFFF;
1229
1230 pfvf = rvu_get_pfvf(rvu, pcifunc);
1231 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1232 if (!pfvf->nixlf || blkaddr < 0)
1233 return NIX_AF_ERR_AF_LF_INVALID;
1234
1235 block = &hw->block[blkaddr];
1236 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1237 if (nixlf < 0)
1238 return NIX_AF_ERR_AF_LF_INVALID;
1239
1240 /* Check if requested 'NIXLF <=> NPALF' mapping is valid */
1241 if (req->npa_func) {
1242 /* If default, use 'this' NIXLF's PFFUNC */
1243 if (req->npa_func == RVU_DEFAULT_PF_FUNC)
1244 req->npa_func = pcifunc;
1245 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA))
1246 return NIX_AF_INVAL_NPA_PF_FUNC;
1247 }
1248
1249 /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */
1250 if (req->sso_func) {
1251 /* If default, use 'this' NIXLF's PFFUNC */
1252 if (req->sso_func == RVU_DEFAULT_PF_FUNC)
1253 req->sso_func = pcifunc;
1254 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO))
1255 return NIX_AF_INVAL_SSO_PF_FUNC;
1256 }
1257
1258 /* If RSS is being enabled, check if requested config is valid.
1259 * RSS table size should be power of two, otherwise
1260 * RSS_GRP::OFFSET + adder might go beyond that group or
1261 * won't be able to use entire table.
1262 */
1263 if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE ||
1264 !is_power_of_2(req->rss_sz)))
1265 return NIX_AF_ERR_RSS_SIZE_INVALID;
1266
1267 if (req->rss_sz &&
1268 (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS))
1269 return NIX_AF_ERR_RSS_GRPS_INVALID;
1270
1271 /* Reset this NIX LF */
1272 err = rvu_lf_reset(rvu, block, nixlf);
1273 if (err) {
1274 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1275 block->addr - BLKADDR_NIX0, nixlf);
1276 return NIX_AF_ERR_LF_RESET;
1277 }
1278
1279 ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3);
1280
1281 /* Alloc NIX RQ HW context memory and config the base */
1282 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
1283 err = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size);
1284 if (err)
1285 goto free_mem;
1286
1287 pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL);
1288 if (!pfvf->rq_bmap)
1289 goto free_mem;
1290
1291 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf),
1292 (u64)pfvf->rq_ctx->iova);
1293
1294 /* Set caching and queue count in HW */
1295 cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20;
1296 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg);
1297
1298 /* Alloc NIX SQ HW context memory and config the base */
1299 hwctx_size = 1UL << (ctx_cfg & 0xF);
1300 err = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size);
1301 if (err)
1302 goto free_mem;
1303
1304 pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL);
1305 if (!pfvf->sq_bmap)
1306 goto free_mem;
1307
1308 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf),
1309 (u64)pfvf->sq_ctx->iova);
1310
1311 cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20;
1312 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg);
1313
1314 /* Alloc NIX CQ HW context memory and config the base */
1315 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
1316 err = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size);
1317 if (err)
1318 goto free_mem;
1319
1320 pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL);
1321 if (!pfvf->cq_bmap)
1322 goto free_mem;
1323
1324 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf),
1325 (u64)pfvf->cq_ctx->iova);
1326
1327 cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20;
1328 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg);
1329
1330 /* Initialize receive side scaling (RSS) */
1331 hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF);
1332 err = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz,
1333 req->rss_grps, hwctx_size, req->way_mask,
1334 !!(req->flags & NIX_LF_RSS_TAG_LSB_AS_ADDER));
1335 if (err)
1336 goto free_mem;
1337
1338 /* Alloc memory for CQINT's HW contexts */
1339 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1340 qints = (cfg >> 24) & 0xFFF;
1341 hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF);
1342 err = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size);
1343 if (err)
1344 goto free_mem;
1345
1346 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf),
1347 (u64)pfvf->cq_ints_ctx->iova);
1348
1349 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf),
1350 BIT_ULL(36) | req->way_mask << 20);
1351
1352 /* Alloc memory for QINT's HW contexts */
1353 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1354 qints = (cfg >> 12) & 0xFFF;
1355 hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF);
1356 err = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size);
1357 if (err)
1358 goto free_mem;
1359
1360 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf),
1361 (u64)pfvf->nix_qints_ctx->iova);
1362 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf),
1363 BIT_ULL(36) | req->way_mask << 20);
1364
1365 /* Setup VLANX TPID's.
1366 * Use VLAN1 for 802.1Q
1367 * and VLAN0 for 802.1AD.
1368 */
1369 cfg = (0x8100ULL << 16) | 0x88A8ULL;
1370 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
1371
1372 /* Enable LMTST for this NIX LF */
1373 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0));
1374
1375 /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */
1376 if (req->npa_func)
1377 cfg = req->npa_func;
1378 if (req->sso_func)
1379 cfg |= (u64)req->sso_func << 16;
1380
1381 cfg |= (u64)req->xqe_sz << 33;
1382 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg);
1383
1384 /* Config Rx pkt length, csum checks and apad enable / disable */
1385 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg);
1386
1387 /* Configure pkind for TX parse config */
1388 cfg = NPC_TX_DEF_PKIND;
1389 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg);
1390
1391 intf = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
1392 if (is_sdp_pfvf(pcifunc))
1393 intf = NIX_INTF_TYPE_SDP;
1394
1395 err = nix_interface_init(rvu, pcifunc, intf, nixlf, rsp,
1396 !!(req->flags & NIX_LF_LBK_BLK_SEL));
1397 if (err)
1398 goto free_mem;
1399
1400 /* Disable NPC entries as NIXLF's contexts are not initialized yet */
1401 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
1402
1403 /* Configure RX VTAG Type 7 (strip) for vf vlan */
1404 rvu_write64(rvu, blkaddr,
1405 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7),
1406 VTAGSIZE_T4 | VTAG_STRIP);
1407
1408 goto exit;
1409
1410 free_mem:
1411 nix_ctx_free(rvu, pfvf);
1412 rc = -ENOMEM;
1413
1414 exit:
1415 /* Set macaddr of this PF/VF */
1416 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
1417
1418 /* set SQB size info */
1419 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST);
1420 rsp->sqb_size = (cfg >> 34) & 0xFFFF;
1421 rsp->rx_chan_base = pfvf->rx_chan_base;
1422 rsp->tx_chan_base = pfvf->tx_chan_base;
1423 rsp->rx_chan_cnt = pfvf->rx_chan_cnt;
1424 rsp->tx_chan_cnt = pfvf->tx_chan_cnt;
1425 rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4;
1426 rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6;
1427 /* Get HW supported stat count */
1428 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
1429 rsp->lf_rx_stats = ((cfg >> 32) & 0xFF);
1430 rsp->lf_tx_stats = ((cfg >> 24) & 0xFF);
1431 /* Get count of CQ IRQs and error IRQs supported per LF */
1432 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1433 rsp->qints = ((cfg >> 12) & 0xFFF);
1434 rsp->cints = ((cfg >> 24) & 0xFFF);
1435 rsp->cgx_links = hw->cgx_links;
1436 rsp->lbk_links = hw->lbk_links;
1437 rsp->sdp_links = hw->sdp_links;
1438
1439 return rc;
1440 }
1441
1442 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req,
1443 struct msg_rsp *rsp)
1444 {
1445 struct rvu_hwinfo *hw = rvu->hw;
1446 u16 pcifunc = req->hdr.pcifunc;
1447 struct rvu_block *block;
1448 int blkaddr, nixlf, err;
1449 struct rvu_pfvf *pfvf;
1450
1451 pfvf = rvu_get_pfvf(rvu, pcifunc);
1452 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1453 if (!pfvf->nixlf || blkaddr < 0)
1454 return NIX_AF_ERR_AF_LF_INVALID;
1455
1456 block = &hw->block[blkaddr];
1457 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1458 if (nixlf < 0)
1459 return NIX_AF_ERR_AF_LF_INVALID;
1460
1461 if (req->flags & NIX_LF_DISABLE_FLOWS)
1462 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
1463 else
1464 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
1465
1466 /* Free any tx vtag def entries used by this NIX LF */
1467 if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG))
1468 nix_free_tx_vtag_entries(rvu, pcifunc);
1469
1470 nix_interface_deinit(rvu, pcifunc, nixlf);
1471
1472 /* Reset this NIX LF */
1473 err = rvu_lf_reset(rvu, block, nixlf);
1474 if (err) {
1475 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1476 block->addr - BLKADDR_NIX0, nixlf);
1477 return NIX_AF_ERR_LF_RESET;
1478 }
1479
1480 nix_ctx_free(rvu, pfvf);
1481
1482 return 0;
1483 }
1484
1485 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu,
1486 struct nix_mark_format_cfg *req,
1487 struct nix_mark_format_cfg_rsp *rsp)
1488 {
1489 u16 pcifunc = req->hdr.pcifunc;
1490 struct nix_hw *nix_hw;
1491 struct rvu_pfvf *pfvf;
1492 int blkaddr, rc;
1493 u32 cfg;
1494
1495 pfvf = rvu_get_pfvf(rvu, pcifunc);
1496 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1497 if (!pfvf->nixlf || blkaddr < 0)
1498 return NIX_AF_ERR_AF_LF_INVALID;
1499
1500 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1501 if (!nix_hw)
1502 return NIX_AF_ERR_INVALID_NIXBLK;
1503
1504 cfg = (((u32)req->offset & 0x7) << 16) |
1505 (((u32)req->y_mask & 0xF) << 12) |
1506 (((u32)req->y_val & 0xF) << 8) |
1507 (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF);
1508
1509 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg);
1510 if (rc < 0) {
1511 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)",
1512 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
1513 return NIX_AF_ERR_MARK_CFG_FAIL;
1514 }
1515
1516 rsp->mark_format_idx = rc;
1517 return 0;
1518 }
1519
1520 /* Handle shaper update specially for few revisions */
1521 static bool
1522 handle_txschq_shaper_update(struct rvu *rvu, int blkaddr, int nixlf,
1523 int lvl, u64 reg, u64 regval)
1524 {
1525 u64 regbase, oldval, sw_xoff = 0;
1526 u64 dbgval, md_debug0 = 0;
1527 unsigned long poll_tmo;
1528 bool rate_reg = 0;
1529 u32 schq;
1530
1531 regbase = reg & 0xFFFF;
1532 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
1533
1534 /* Check for rate register */
1535 switch (lvl) {
1536 case NIX_TXSCH_LVL_TL1:
1537 md_debug0 = NIX_AF_TL1X_MD_DEBUG0(schq);
1538 sw_xoff = NIX_AF_TL1X_SW_XOFF(schq);
1539
1540 rate_reg = !!(regbase == NIX_AF_TL1X_CIR(0));
1541 break;
1542 case NIX_TXSCH_LVL_TL2:
1543 md_debug0 = NIX_AF_TL2X_MD_DEBUG0(schq);
1544 sw_xoff = NIX_AF_TL2X_SW_XOFF(schq);
1545
1546 rate_reg = (regbase == NIX_AF_TL2X_CIR(0) ||
1547 regbase == NIX_AF_TL2X_PIR(0));
1548 break;
1549 case NIX_TXSCH_LVL_TL3:
1550 md_debug0 = NIX_AF_TL3X_MD_DEBUG0(schq);
1551 sw_xoff = NIX_AF_TL3X_SW_XOFF(schq);
1552
1553 rate_reg = (regbase == NIX_AF_TL3X_CIR(0) ||
1554 regbase == NIX_AF_TL3X_PIR(0));
1555 break;
1556 case NIX_TXSCH_LVL_TL4:
1557 md_debug0 = NIX_AF_TL4X_MD_DEBUG0(schq);
1558 sw_xoff = NIX_AF_TL4X_SW_XOFF(schq);
1559
1560 rate_reg = (regbase == NIX_AF_TL4X_CIR(0) ||
1561 regbase == NIX_AF_TL4X_PIR(0));
1562 break;
1563 case NIX_TXSCH_LVL_MDQ:
1564 sw_xoff = NIX_AF_MDQX_SW_XOFF(schq);
1565 rate_reg = (regbase == NIX_AF_MDQX_CIR(0) ||
1566 regbase == NIX_AF_MDQX_PIR(0));
1567 break;
1568 }
1569
1570 if (!rate_reg)
1571 return false;
1572
1573 /* Nothing special to do when state is not toggled */
1574 oldval = rvu_read64(rvu, blkaddr, reg);
1575 if ((oldval & 0x1) == (regval & 0x1)) {
1576 rvu_write64(rvu, blkaddr, reg, regval);
1577 return true;
1578 }
1579
1580 /* PIR/CIR disable */
1581 if (!(regval & 0x1)) {
1582 rvu_write64(rvu, blkaddr, sw_xoff, 1);
1583 rvu_write64(rvu, blkaddr, reg, 0);
1584 udelay(4);
1585 rvu_write64(rvu, blkaddr, sw_xoff, 0);
1586 return true;
1587 }
1588
1589 /* PIR/CIR enable */
1590 rvu_write64(rvu, blkaddr, sw_xoff, 1);
1591 if (md_debug0) {
1592 poll_tmo = jiffies + usecs_to_jiffies(10000);
1593 /* Wait until VLD(bit32) == 1 or C_CON(bit48) == 0 */
1594 do {
1595 if (time_after(jiffies, poll_tmo)) {
1596 dev_err(rvu->dev,
1597 "NIXLF%d: TLX%u(lvl %u) CIR/PIR enable failed\n",
1598 nixlf, schq, lvl);
1599 goto exit;
1600 }
1601 usleep_range(1, 5);
1602 dbgval = rvu_read64(rvu, blkaddr, md_debug0);
1603 } while (!(dbgval & BIT_ULL(32)) && (dbgval & BIT_ULL(48)));
1604 }
1605 rvu_write64(rvu, blkaddr, reg, regval);
1606 exit:
1607 rvu_write64(rvu, blkaddr, sw_xoff, 0);
1608 return true;
1609 }
1610
1611 /* Disable shaping of pkts by a scheduler queue
1612 * at a given scheduler level.
1613 */
1614 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr,
1615 int nixlf, int lvl, int schq)
1616 {
1617 struct rvu_hwinfo *hw = rvu->hw;
1618 u64 cir_reg = 0, pir_reg = 0;
1619 u64 cfg;
1620
1621 switch (lvl) {
1622 case NIX_TXSCH_LVL_TL1:
1623 cir_reg = NIX_AF_TL1X_CIR(schq);
1624 pir_reg = 0; /* PIR not available at TL1 */
1625 break;
1626 case NIX_TXSCH_LVL_TL2:
1627 cir_reg = NIX_AF_TL2X_CIR(schq);
1628 pir_reg = NIX_AF_TL2X_PIR(schq);
1629 break;
1630 case NIX_TXSCH_LVL_TL3:
1631 cir_reg = NIX_AF_TL3X_CIR(schq);
1632 pir_reg = NIX_AF_TL3X_PIR(schq);
1633 break;
1634 case NIX_TXSCH_LVL_TL4:
1635 cir_reg = NIX_AF_TL4X_CIR(schq);
1636 pir_reg = NIX_AF_TL4X_PIR(schq);
1637 break;
1638 case NIX_TXSCH_LVL_MDQ:
1639 cir_reg = NIX_AF_MDQX_CIR(schq);
1640 pir_reg = NIX_AF_MDQX_PIR(schq);
1641 break;
1642 }
1643
1644 /* Shaper state toggle needs wait/poll */
1645 if (hw->cap.nix_shaper_toggle_wait) {
1646 if (cir_reg)
1647 handle_txschq_shaper_update(rvu, blkaddr, nixlf,
1648 lvl, cir_reg, 0);
1649 if (pir_reg)
1650 handle_txschq_shaper_update(rvu, blkaddr, nixlf,
1651 lvl, pir_reg, 0);
1652 return;
1653 }
1654
1655 if (!cir_reg)
1656 return;
1657 cfg = rvu_read64(rvu, blkaddr, cir_reg);
1658 rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0));
1659
1660 if (!pir_reg)
1661 return;
1662 cfg = rvu_read64(rvu, blkaddr, pir_reg);
1663 rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0));
1664 }
1665
1666 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr,
1667 int lvl, int schq)
1668 {
1669 struct rvu_hwinfo *hw = rvu->hw;
1670 int link_level;
1671 int link;
1672
1673 if (lvl >= hw->cap.nix_tx_aggr_lvl)
1674 return;
1675
1676 /* Reset TL4's SDP link config */
1677 if (lvl == NIX_TXSCH_LVL_TL4)
1678 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00);
1679
1680 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
1681 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
1682 if (lvl != link_level)
1683 return;
1684
1685 /* Reset TL2's CGX or LBK link config */
1686 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++)
1687 rvu_write64(rvu, blkaddr,
1688 NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00);
1689 }
1690
1691 static void nix_clear_tx_xoff(struct rvu *rvu, int blkaddr,
1692 int lvl, int schq)
1693 {
1694 struct rvu_hwinfo *hw = rvu->hw;
1695 u64 reg;
1696
1697 /* Skip this if shaping is not supported */
1698 if (!hw->cap.nix_shaping)
1699 return;
1700
1701 /* Clear level specific SW_XOFF */
1702 switch (lvl) {
1703 case NIX_TXSCH_LVL_TL1:
1704 reg = NIX_AF_TL1X_SW_XOFF(schq);
1705 break;
1706 case NIX_TXSCH_LVL_TL2:
1707 reg = NIX_AF_TL2X_SW_XOFF(schq);
1708 break;
1709 case NIX_TXSCH_LVL_TL3:
1710 reg = NIX_AF_TL3X_SW_XOFF(schq);
1711 break;
1712 case NIX_TXSCH_LVL_TL4:
1713 reg = NIX_AF_TL4X_SW_XOFF(schq);
1714 break;
1715 case NIX_TXSCH_LVL_MDQ:
1716 reg = NIX_AF_MDQX_SW_XOFF(schq);
1717 break;
1718 default:
1719 return;
1720 }
1721
1722 rvu_write64(rvu, blkaddr, reg, 0x0);
1723 }
1724
1725 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc)
1726 {
1727 struct rvu_hwinfo *hw = rvu->hw;
1728 int pf = rvu_get_pf(pcifunc);
1729 u8 cgx_id = 0, lmac_id = 0;
1730
1731 if (is_afvf(pcifunc)) {/* LBK links */
1732 return hw->cgx_links;
1733 } else if (is_pf_cgxmapped(rvu, pf)) {
1734 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
1735 return (cgx_id * hw->lmac_per_cgx) + lmac_id;
1736 }
1737
1738 /* SDP link */
1739 return hw->cgx_links + hw->lbk_links;
1740 }
1741
1742 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc,
1743 int link, int *start, int *end)
1744 {
1745 struct rvu_hwinfo *hw = rvu->hw;
1746 int pf = rvu_get_pf(pcifunc);
1747
1748 if (is_afvf(pcifunc)) { /* LBK links */
1749 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
1750 *end = *start + hw->cap.nix_txsch_per_lbk_lmac;
1751 } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */
1752 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
1753 *end = *start + hw->cap.nix_txsch_per_cgx_lmac;
1754 } else { /* SDP link */
1755 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) +
1756 (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links);
1757 *end = *start + hw->cap.nix_txsch_per_sdp_lmac;
1758 }
1759 }
1760
1761 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc,
1762 struct nix_hw *nix_hw,
1763 struct nix_txsch_alloc_req *req)
1764 {
1765 struct rvu_hwinfo *hw = rvu->hw;
1766 int schq, req_schq, free_cnt;
1767 struct nix_txsch *txsch;
1768 int link, start, end;
1769
1770 txsch = &nix_hw->txsch[lvl];
1771 req_schq = req->schq_contig[lvl] + req->schq[lvl];
1772
1773 if (!req_schq)
1774 return 0;
1775
1776 link = nix_get_tx_link(rvu, pcifunc);
1777
1778 /* For traffic aggregating scheduler level, one queue is enough */
1779 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1780 if (req_schq != 1)
1781 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1782 return 0;
1783 }
1784
1785 /* Get free SCHQ count and check if request can be accomodated */
1786 if (hw->cap.nix_fixed_txschq_mapping) {
1787 nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
1788 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK);
1789 if (end <= txsch->schq.max && schq < end &&
1790 !test_bit(schq, txsch->schq.bmap))
1791 free_cnt = 1;
1792 else
1793 free_cnt = 0;
1794 } else {
1795 free_cnt = rvu_rsrc_free_count(&txsch->schq);
1796 }
1797
1798 if (free_cnt < req_schq || req_schq > MAX_TXSCHQ_PER_FUNC)
1799 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1800
1801 /* If contiguous queues are needed, check for availability */
1802 if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] &&
1803 !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl]))
1804 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1805
1806 return 0;
1807 }
1808
1809 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch,
1810 struct nix_txsch_alloc_rsp *rsp,
1811 int lvl, int start, int end)
1812 {
1813 struct rvu_hwinfo *hw = rvu->hw;
1814 u16 pcifunc = rsp->hdr.pcifunc;
1815 int idx, schq;
1816
1817 /* For traffic aggregating levels, queue alloc is based
1818 * on transmit link to which PF_FUNC is mapped to.
1819 */
1820 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1821 /* A single TL queue is allocated */
1822 if (rsp->schq_contig[lvl]) {
1823 rsp->schq_contig[lvl] = 1;
1824 rsp->schq_contig_list[lvl][0] = start;
1825 }
1826
1827 /* Both contig and non-contig reqs doesn't make sense here */
1828 if (rsp->schq_contig[lvl])
1829 rsp->schq[lvl] = 0;
1830
1831 if (rsp->schq[lvl]) {
1832 rsp->schq[lvl] = 1;
1833 rsp->schq_list[lvl][0] = start;
1834 }
1835 return;
1836 }
1837
1838 /* Adjust the queue request count if HW supports
1839 * only one queue per level configuration.
1840 */
1841 if (hw->cap.nix_fixed_txschq_mapping) {
1842 idx = pcifunc & RVU_PFVF_FUNC_MASK;
1843 schq = start + idx;
1844 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) {
1845 rsp->schq_contig[lvl] = 0;
1846 rsp->schq[lvl] = 0;
1847 return;
1848 }
1849
1850 if (rsp->schq_contig[lvl]) {
1851 rsp->schq_contig[lvl] = 1;
1852 set_bit(schq, txsch->schq.bmap);
1853 rsp->schq_contig_list[lvl][0] = schq;
1854 rsp->schq[lvl] = 0;
1855 } else if (rsp->schq[lvl]) {
1856 rsp->schq[lvl] = 1;
1857 set_bit(schq, txsch->schq.bmap);
1858 rsp->schq_list[lvl][0] = schq;
1859 }
1860 return;
1861 }
1862
1863 /* Allocate contiguous queue indices requesty first */
1864 if (rsp->schq_contig[lvl]) {
1865 schq = bitmap_find_next_zero_area(txsch->schq.bmap,
1866 txsch->schq.max, start,
1867 rsp->schq_contig[lvl], 0);
1868 if (schq >= end)
1869 rsp->schq_contig[lvl] = 0;
1870 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) {
1871 set_bit(schq, txsch->schq.bmap);
1872 rsp->schq_contig_list[lvl][idx] = schq;
1873 schq++;
1874 }
1875 }
1876
1877 /* Allocate non-contiguous queue indices */
1878 if (rsp->schq[lvl]) {
1879 idx = 0;
1880 for (schq = start; schq < end; schq++) {
1881 if (!test_bit(schq, txsch->schq.bmap)) {
1882 set_bit(schq, txsch->schq.bmap);
1883 rsp->schq_list[lvl][idx++] = schq;
1884 }
1885 if (idx == rsp->schq[lvl])
1886 break;
1887 }
1888 /* Update how many were allocated */
1889 rsp->schq[lvl] = idx;
1890 }
1891 }
1892
1893 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu,
1894 struct nix_txsch_alloc_req *req,
1895 struct nix_txsch_alloc_rsp *rsp)
1896 {
1897 struct rvu_hwinfo *hw = rvu->hw;
1898 u16 pcifunc = req->hdr.pcifunc;
1899 int link, blkaddr, rc = 0;
1900 int lvl, idx, start, end;
1901 struct nix_txsch *txsch;
1902 struct nix_hw *nix_hw;
1903 u32 *pfvf_map;
1904 int nixlf;
1905 u16 schq;
1906
1907 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
1908 if (rc)
1909 return rc;
1910
1911 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1912 if (!nix_hw)
1913 return NIX_AF_ERR_INVALID_NIXBLK;
1914
1915 mutex_lock(&rvu->rsrc_lock);
1916
1917 /* Check if request is valid as per HW capabilities
1918 * and can be accomodated.
1919 */
1920 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1921 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req);
1922 if (rc)
1923 goto err;
1924 }
1925
1926 /* Allocate requested Tx scheduler queues */
1927 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1928 txsch = &nix_hw->txsch[lvl];
1929 pfvf_map = txsch->pfvf_map;
1930
1931 if (!req->schq[lvl] && !req->schq_contig[lvl])
1932 continue;
1933
1934 rsp->schq[lvl] = req->schq[lvl];
1935 rsp->schq_contig[lvl] = req->schq_contig[lvl];
1936
1937 link = nix_get_tx_link(rvu, pcifunc);
1938
1939 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1940 start = link;
1941 end = link;
1942 } else if (hw->cap.nix_fixed_txschq_mapping) {
1943 nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
1944 } else {
1945 start = 0;
1946 end = txsch->schq.max;
1947 }
1948
1949 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end);
1950
1951 /* Reset queue config */
1952 for (idx = 0; idx < req->schq_contig[lvl]; idx++) {
1953 schq = rsp->schq_contig_list[lvl][idx];
1954 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
1955 NIX_TXSCHQ_CFG_DONE))
1956 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
1957 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
1958 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
1959 }
1960
1961 for (idx = 0; idx < req->schq[lvl]; idx++) {
1962 schq = rsp->schq_list[lvl][idx];
1963 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
1964 NIX_TXSCHQ_CFG_DONE))
1965 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
1966 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
1967 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
1968 }
1969 }
1970
1971 rsp->aggr_level = hw->cap.nix_tx_aggr_lvl;
1972 rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
1973 rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr,
1974 NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
1975 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
1976 goto exit;
1977 err:
1978 rc = NIX_AF_ERR_TLX_ALLOC_FAIL;
1979 exit:
1980 mutex_unlock(&rvu->rsrc_lock);
1981 return rc;
1982 }
1983
1984 static int nix_smq_flush(struct rvu *rvu, int blkaddr,
1985 int smq, u16 pcifunc, int nixlf)
1986 {
1987 int pf = rvu_get_pf(pcifunc);
1988 u8 cgx_id = 0, lmac_id = 0;
1989 int err, restore_tx_en = 0;
1990 u64 cfg;
1991
1992 /* enable cgx tx if disabled */
1993 if (is_pf_cgxmapped(rvu, pf)) {
1994 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
1995 restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
1996 lmac_id, true);
1997 }
1998
1999 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
2000 /* Do SMQ flush and set enqueue xoff */
2001 cfg |= BIT_ULL(50) | BIT_ULL(49);
2002 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg);
2003
2004 /* Disable backpressure from physical link,
2005 * otherwise SMQ flush may stall.
2006 */
2007 rvu_cgx_enadis_rx_bp(rvu, pf, false);
2008
2009 /* Wait for flush to complete */
2010 err = rvu_poll_reg(rvu, blkaddr,
2011 NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true);
2012 if (err)
2013 dev_err(rvu->dev,
2014 "NIXLF%d: SMQ%d flush failed\n", nixlf, smq);
2015
2016 rvu_cgx_enadis_rx_bp(rvu, pf, true);
2017 /* restore cgx tx state */
2018 if (restore_tx_en)
2019 cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
2020 return err;
2021 }
2022
2023 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc)
2024 {
2025 int blkaddr, nixlf, lvl, schq, err;
2026 struct rvu_hwinfo *hw = rvu->hw;
2027 struct nix_txsch *txsch;
2028 struct nix_hw *nix_hw;
2029 u16 map_func;
2030
2031 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2032 if (blkaddr < 0)
2033 return NIX_AF_ERR_AF_LF_INVALID;
2034
2035 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2036 if (!nix_hw)
2037 return NIX_AF_ERR_INVALID_NIXBLK;
2038
2039 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
2040 if (nixlf < 0)
2041 return NIX_AF_ERR_AF_LF_INVALID;
2042
2043 /* Disable TL2/3 queue links and all XOFF's before SMQ flush*/
2044 mutex_lock(&rvu->rsrc_lock);
2045 for (lvl = NIX_TXSCH_LVL_MDQ; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2046 txsch = &nix_hw->txsch[lvl];
2047
2048 if (lvl >= hw->cap.nix_tx_aggr_lvl)
2049 continue;
2050
2051 for (schq = 0; schq < txsch->schq.max; schq++) {
2052 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2053 continue;
2054 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
2055 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq);
2056 }
2057 }
2058 nix_clear_tx_xoff(rvu, blkaddr, NIX_TXSCH_LVL_TL1,
2059 nix_get_tx_link(rvu, pcifunc));
2060
2061 /* On PF cleanup, clear cfg done flag as
2062 * PF would have changed default config.
2063 */
2064 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
2065 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL1];
2066 schq = nix_get_tx_link(rvu, pcifunc);
2067 /* Do not clear pcifunc in txsch->pfvf_map[schq] because
2068 * VF might be using this TL1 queue
2069 */
2070 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
2071 txsch->pfvf_map[schq] = TXSCH_SET_FLAG(map_func, 0x0);
2072 }
2073
2074 /* Flush SMQs */
2075 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
2076 for (schq = 0; schq < txsch->schq.max; schq++) {
2077 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2078 continue;
2079 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
2080 }
2081
2082 /* Now free scheduler queues to free pool */
2083 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2084 /* TLs above aggregation level are shared across all PF
2085 * and it's VFs, hence skip freeing them.
2086 */
2087 if (lvl >= hw->cap.nix_tx_aggr_lvl)
2088 continue;
2089
2090 txsch = &nix_hw->txsch[lvl];
2091 for (schq = 0; schq < txsch->schq.max; schq++) {
2092 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2093 continue;
2094 rvu_free_rsrc(&txsch->schq, schq);
2095 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
2096 }
2097 }
2098 mutex_unlock(&rvu->rsrc_lock);
2099
2100 /* Sync cached info for this LF in NDC-TX to LLC/DRAM */
2101 rvu_write64(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12) | nixlf);
2102 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12), true);
2103 if (err)
2104 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf);
2105
2106 return 0;
2107 }
2108
2109 static int nix_txschq_free_one(struct rvu *rvu,
2110 struct nix_txsch_free_req *req)
2111 {
2112 struct rvu_hwinfo *hw = rvu->hw;
2113 u16 pcifunc = req->hdr.pcifunc;
2114 int lvl, schq, nixlf, blkaddr;
2115 struct nix_txsch *txsch;
2116 struct nix_hw *nix_hw;
2117 u32 *pfvf_map;
2118 int rc;
2119
2120 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2121 if (blkaddr < 0)
2122 return NIX_AF_ERR_AF_LF_INVALID;
2123
2124 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2125 if (!nix_hw)
2126 return NIX_AF_ERR_INVALID_NIXBLK;
2127
2128 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
2129 if (nixlf < 0)
2130 return NIX_AF_ERR_AF_LF_INVALID;
2131
2132 lvl = req->schq_lvl;
2133 schq = req->schq;
2134 txsch = &nix_hw->txsch[lvl];
2135
2136 if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max)
2137 return 0;
2138
2139 pfvf_map = txsch->pfvf_map;
2140 mutex_lock(&rvu->rsrc_lock);
2141
2142 if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) {
2143 rc = NIX_AF_ERR_TLX_INVALID;
2144 goto err;
2145 }
2146
2147 /* Clear SW_XOFF of this resource only.
2148 * For SMQ level, all path XOFF's
2149 * need to be made clear by user
2150 */
2151 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq);
2152
2153 /* Flush if it is a SMQ. Onus of disabling
2154 * TL2/3 queue links before SMQ flush is on user
2155 */
2156 if (lvl == NIX_TXSCH_LVL_SMQ &&
2157 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf)) {
2158 rc = NIX_AF_SMQ_FLUSH_FAILED;
2159 goto err;
2160 }
2161
2162 /* Free the resource */
2163 rvu_free_rsrc(&txsch->schq, schq);
2164 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
2165 mutex_unlock(&rvu->rsrc_lock);
2166 return 0;
2167 err:
2168 mutex_unlock(&rvu->rsrc_lock);
2169 return rc;
2170 }
2171
2172 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu,
2173 struct nix_txsch_free_req *req,
2174 struct msg_rsp *rsp)
2175 {
2176 if (req->flags & TXSCHQ_FREE_ALL)
2177 return nix_txschq_free(rvu, req->hdr.pcifunc);
2178 else
2179 return nix_txschq_free_one(rvu, req);
2180 }
2181
2182 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr,
2183 int lvl, u64 reg, u64 regval)
2184 {
2185 u64 regbase = reg & 0xFFFF;
2186 u16 schq, parent;
2187
2188 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg))
2189 return false;
2190
2191 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2192 /* Check if this schq belongs to this PF/VF or not */
2193 if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq))
2194 return false;
2195
2196 parent = (regval >> 16) & 0x1FF;
2197 /* Validate MDQ's TL4 parent */
2198 if (regbase == NIX_AF_MDQX_PARENT(0) &&
2199 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent))
2200 return false;
2201
2202 /* Validate TL4's TL3 parent */
2203 if (regbase == NIX_AF_TL4X_PARENT(0) &&
2204 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent))
2205 return false;
2206
2207 /* Validate TL3's TL2 parent */
2208 if (regbase == NIX_AF_TL3X_PARENT(0) &&
2209 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent))
2210 return false;
2211
2212 /* Validate TL2's TL1 parent */
2213 if (regbase == NIX_AF_TL2X_PARENT(0) &&
2214 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent))
2215 return false;
2216
2217 return true;
2218 }
2219
2220 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg)
2221 {
2222 u64 regbase;
2223
2224 if (hw->cap.nix_shaping)
2225 return true;
2226
2227 /* If shaping and coloring is not supported, then
2228 * *_CIR and *_PIR registers should not be configured.
2229 */
2230 regbase = reg & 0xFFFF;
2231
2232 switch (lvl) {
2233 case NIX_TXSCH_LVL_TL1:
2234 if (regbase == NIX_AF_TL1X_CIR(0))
2235 return false;
2236 break;
2237 case NIX_TXSCH_LVL_TL2:
2238 if (regbase == NIX_AF_TL2X_CIR(0) ||
2239 regbase == NIX_AF_TL2X_PIR(0))
2240 return false;
2241 break;
2242 case NIX_TXSCH_LVL_TL3:
2243 if (regbase == NIX_AF_TL3X_CIR(0) ||
2244 regbase == NIX_AF_TL3X_PIR(0))
2245 return false;
2246 break;
2247 case NIX_TXSCH_LVL_TL4:
2248 if (regbase == NIX_AF_TL4X_CIR(0) ||
2249 regbase == NIX_AF_TL4X_PIR(0))
2250 return false;
2251 break;
2252 case NIX_TXSCH_LVL_MDQ:
2253 if (regbase == NIX_AF_MDQX_CIR(0) ||
2254 regbase == NIX_AF_MDQX_PIR(0))
2255 return false;
2256 break;
2257 }
2258 return true;
2259 }
2260
2261 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw,
2262 u16 pcifunc, int blkaddr)
2263 {
2264 u32 *pfvf_map;
2265 int schq;
2266
2267 schq = nix_get_tx_link(rvu, pcifunc);
2268 pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map;
2269 /* Skip if PF has already done the config */
2270 if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE)
2271 return;
2272 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq),
2273 (TXSCH_TL1_DFLT_RR_PRIO << 1));
2274
2275 /* On OcteonTx2 the config was in bytes and newer silcons
2276 * it's changed to weight.
2277 */
2278 if (!rvu->hw->cap.nix_common_dwrr_mtu)
2279 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
2280 TXSCH_TL1_DFLT_RR_QTM);
2281 else
2282 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
2283 CN10K_MAX_DWRR_WEIGHT);
2284
2285 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00);
2286 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE);
2287 }
2288
2289 /* Register offset - [15:0]
2290 * Scheduler Queue number - [25:16]
2291 */
2292 #define NIX_TX_SCHQ_MASK GENMASK_ULL(25, 0)
2293
2294 static int nix_txschq_cfg_read(struct rvu *rvu, struct nix_hw *nix_hw,
2295 int blkaddr, struct nix_txschq_config *req,
2296 struct nix_txschq_config *rsp)
2297 {
2298 u16 pcifunc = req->hdr.pcifunc;
2299 int idx, schq;
2300 u64 reg;
2301
2302 for (idx = 0; idx < req->num_regs; idx++) {
2303 reg = req->reg[idx];
2304 reg &= NIX_TX_SCHQ_MASK;
2305 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2306 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, req->lvl, reg) ||
2307 !is_valid_txschq(rvu, blkaddr, req->lvl, pcifunc, schq))
2308 return NIX_AF_INVAL_TXSCHQ_CFG;
2309 rsp->regval[idx] = rvu_read64(rvu, blkaddr, reg);
2310 }
2311 rsp->lvl = req->lvl;
2312 rsp->num_regs = req->num_regs;
2313 return 0;
2314 }
2315
2316 static void rvu_nix_tx_tl2_cfg(struct rvu *rvu, int blkaddr,
2317 u16 pcifunc, struct nix_txsch *txsch)
2318 {
2319 struct rvu_hwinfo *hw = rvu->hw;
2320 int lbk_link_start, lbk_links;
2321 u8 pf = rvu_get_pf(pcifunc);
2322 int schq;
2323
2324 if (!is_pf_cgxmapped(rvu, pf))
2325 return;
2326
2327 lbk_link_start = hw->cgx_links;
2328
2329 for (schq = 0; schq < txsch->schq.max; schq++) {
2330 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2331 continue;
2332 /* Enable all LBK links with channel 63 by default so that
2333 * packets can be sent to LBK with a NPC TX MCAM rule
2334 */
2335 lbk_links = hw->lbk_links;
2336 while (lbk_links--)
2337 rvu_write64(rvu, blkaddr,
2338 NIX_AF_TL3_TL2X_LINKX_CFG(schq,
2339 lbk_link_start +
2340 lbk_links),
2341 BIT_ULL(12) | RVU_SWITCH_LBK_CHAN);
2342 }
2343 }
2344
2345 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu,
2346 struct nix_txschq_config *req,
2347 struct nix_txschq_config *rsp)
2348 {
2349 u64 reg, val, regval, schq_regbase, val_mask;
2350 struct rvu_hwinfo *hw = rvu->hw;
2351 u16 pcifunc = req->hdr.pcifunc;
2352 struct nix_txsch *txsch;
2353 struct nix_hw *nix_hw;
2354 int blkaddr, idx, err;
2355 int nixlf, schq;
2356 u32 *pfvf_map;
2357
2358 if (req->lvl >= NIX_TXSCH_LVL_CNT ||
2359 req->num_regs > MAX_REGS_PER_MBOX_MSG)
2360 return NIX_AF_INVAL_TXSCHQ_CFG;
2361
2362 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2363 if (err)
2364 return err;
2365
2366 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2367 if (!nix_hw)
2368 return NIX_AF_ERR_INVALID_NIXBLK;
2369
2370 if (req->read)
2371 return nix_txschq_cfg_read(rvu, nix_hw, blkaddr, req, rsp);
2372
2373 txsch = &nix_hw->txsch[req->lvl];
2374 pfvf_map = txsch->pfvf_map;
2375
2376 if (req->lvl >= hw->cap.nix_tx_aggr_lvl &&
2377 pcifunc & RVU_PFVF_FUNC_MASK) {
2378 mutex_lock(&rvu->rsrc_lock);
2379 if (req->lvl == NIX_TXSCH_LVL_TL1)
2380 nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr);
2381 mutex_unlock(&rvu->rsrc_lock);
2382 return 0;
2383 }
2384
2385 for (idx = 0; idx < req->num_regs; idx++) {
2386 reg = req->reg[idx];
2387 reg &= NIX_TX_SCHQ_MASK;
2388 regval = req->regval[idx];
2389 schq_regbase = reg & 0xFFFF;
2390 val_mask = req->regval_mask[idx];
2391
2392 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr,
2393 txsch->lvl, reg, regval))
2394 return NIX_AF_INVAL_TXSCHQ_CFG;
2395
2396 /* Check if shaping and coloring is supported */
2397 if (!is_txschq_shaping_valid(hw, req->lvl, reg))
2398 continue;
2399
2400 val = rvu_read64(rvu, blkaddr, reg);
2401 regval = (val & val_mask) | (regval & ~val_mask);
2402
2403 /* Handle shaping state toggle specially */
2404 if (hw->cap.nix_shaper_toggle_wait &&
2405 handle_txschq_shaper_update(rvu, blkaddr, nixlf,
2406 req->lvl, reg, regval))
2407 continue;
2408
2409 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */
2410 if (schq_regbase == NIX_AF_SMQX_CFG(0)) {
2411 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],
2412 pcifunc, 0);
2413 regval &= ~(0x7FULL << 24);
2414 regval |= ((u64)nixlf << 24);
2415 }
2416
2417 /* Clear 'BP_ENA' config, if it's not allowed */
2418 if (!hw->cap.nix_tx_link_bp) {
2419 if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) ||
2420 (schq_regbase & 0xFF00) ==
2421 NIX_AF_TL3_TL2X_LINKX_CFG(0, 0))
2422 regval &= ~BIT_ULL(13);
2423 }
2424
2425 /* Mark config as done for TL1 by PF */
2426 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) &&
2427 schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) {
2428 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2429 mutex_lock(&rvu->rsrc_lock);
2430 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq],
2431 NIX_TXSCHQ_CFG_DONE);
2432 mutex_unlock(&rvu->rsrc_lock);
2433 }
2434
2435 /* SMQ flush is special hence split register writes such
2436 * that flush first and write rest of the bits later.
2437 */
2438 if (schq_regbase == NIX_AF_SMQX_CFG(0) &&
2439 (regval & BIT_ULL(49))) {
2440 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2441 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
2442 regval &= ~BIT_ULL(49);
2443 }
2444 rvu_write64(rvu, blkaddr, reg, regval);
2445 }
2446
2447 rvu_nix_tx_tl2_cfg(rvu, blkaddr, pcifunc,
2448 &nix_hw->txsch[NIX_TXSCH_LVL_TL2]);
2449 return 0;
2450 }
2451
2452 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr,
2453 struct nix_vtag_config *req)
2454 {
2455 u64 regval = req->vtag_size;
2456
2457 if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 ||
2458 req->vtag_size > VTAGSIZE_T8)
2459 return -EINVAL;
2460
2461 /* RX VTAG Type 7 reserved for vf vlan */
2462 if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7)
2463 return NIX_AF_ERR_RX_VTAG_INUSE;
2464
2465 if (req->rx.capture_vtag)
2466 regval |= BIT_ULL(5);
2467 if (req->rx.strip_vtag)
2468 regval |= BIT_ULL(4);
2469
2470 rvu_write64(rvu, blkaddr,
2471 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval);
2472 return 0;
2473 }
2474
2475 static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr,
2476 u16 pcifunc, int index)
2477 {
2478 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2479 struct nix_txvlan *vlan;
2480
2481 if (!nix_hw)
2482 return NIX_AF_ERR_INVALID_NIXBLK;
2483
2484 vlan = &nix_hw->txvlan;
2485 if (vlan->entry2pfvf_map[index] != pcifunc)
2486 return NIX_AF_ERR_PARAM;
2487
2488 rvu_write64(rvu, blkaddr,
2489 NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull);
2490 rvu_write64(rvu, blkaddr,
2491 NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull);
2492
2493 vlan->entry2pfvf_map[index] = 0;
2494 rvu_free_rsrc(&vlan->rsrc, index);
2495
2496 return 0;
2497 }
2498
2499 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc)
2500 {
2501 struct nix_txvlan *vlan;
2502 struct nix_hw *nix_hw;
2503 int index, blkaddr;
2504
2505 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2506 if (blkaddr < 0)
2507 return;
2508
2509 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2510 if (!nix_hw)
2511 return;
2512
2513 vlan = &nix_hw->txvlan;
2514
2515 mutex_lock(&vlan->rsrc_lock);
2516 /* Scan all the entries and free the ones mapped to 'pcifunc' */
2517 for (index = 0; index < vlan->rsrc.max; index++) {
2518 if (vlan->entry2pfvf_map[index] == pcifunc)
2519 nix_tx_vtag_free(rvu, blkaddr, pcifunc, index);
2520 }
2521 mutex_unlock(&vlan->rsrc_lock);
2522 }
2523
2524 static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr,
2525 u64 vtag, u8 size)
2526 {
2527 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2528 struct nix_txvlan *vlan;
2529 u64 regval;
2530 int index;
2531
2532 if (!nix_hw)
2533 return NIX_AF_ERR_INVALID_NIXBLK;
2534
2535 vlan = &nix_hw->txvlan;
2536
2537 mutex_lock(&vlan->rsrc_lock);
2538
2539 index = rvu_alloc_rsrc(&vlan->rsrc);
2540 if (index < 0) {
2541 mutex_unlock(&vlan->rsrc_lock);
2542 return index;
2543 }
2544
2545 mutex_unlock(&vlan->rsrc_lock);
2546
2547 regval = size ? vtag : vtag << 32;
2548
2549 rvu_write64(rvu, blkaddr,
2550 NIX_AF_TX_VTAG_DEFX_DATA(index), regval);
2551 rvu_write64(rvu, blkaddr,
2552 NIX_AF_TX_VTAG_DEFX_CTL(index), size);
2553
2554 return index;
2555 }
2556
2557 static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr,
2558 struct nix_vtag_config *req)
2559 {
2560 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2561 u16 pcifunc = req->hdr.pcifunc;
2562 int idx0 = req->tx.vtag0_idx;
2563 int idx1 = req->tx.vtag1_idx;
2564 struct nix_txvlan *vlan;
2565 int err = 0;
2566
2567 if (!nix_hw)
2568 return NIX_AF_ERR_INVALID_NIXBLK;
2569
2570 vlan = &nix_hw->txvlan;
2571 if (req->tx.free_vtag0 && req->tx.free_vtag1)
2572 if (vlan->entry2pfvf_map[idx0] != pcifunc ||
2573 vlan->entry2pfvf_map[idx1] != pcifunc)
2574 return NIX_AF_ERR_PARAM;
2575
2576 mutex_lock(&vlan->rsrc_lock);
2577
2578 if (req->tx.free_vtag0) {
2579 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0);
2580 if (err)
2581 goto exit;
2582 }
2583
2584 if (req->tx.free_vtag1)
2585 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1);
2586
2587 exit:
2588 mutex_unlock(&vlan->rsrc_lock);
2589 return err;
2590 }
2591
2592 static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr,
2593 struct nix_vtag_config *req,
2594 struct nix_vtag_config_rsp *rsp)
2595 {
2596 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2597 struct nix_txvlan *vlan;
2598 u16 pcifunc = req->hdr.pcifunc;
2599
2600 if (!nix_hw)
2601 return NIX_AF_ERR_INVALID_NIXBLK;
2602
2603 vlan = &nix_hw->txvlan;
2604 if (req->tx.cfg_vtag0) {
2605 rsp->vtag0_idx =
2606 nix_tx_vtag_alloc(rvu, blkaddr,
2607 req->tx.vtag0, req->vtag_size);
2608
2609 if (rsp->vtag0_idx < 0)
2610 return NIX_AF_ERR_TX_VTAG_NOSPC;
2611
2612 vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc;
2613 }
2614
2615 if (req->tx.cfg_vtag1) {
2616 rsp->vtag1_idx =
2617 nix_tx_vtag_alloc(rvu, blkaddr,
2618 req->tx.vtag1, req->vtag_size);
2619
2620 if (rsp->vtag1_idx < 0)
2621 goto err_free;
2622
2623 vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc;
2624 }
2625
2626 return 0;
2627
2628 err_free:
2629 if (req->tx.cfg_vtag0)
2630 nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx);
2631
2632 return NIX_AF_ERR_TX_VTAG_NOSPC;
2633 }
2634
2635 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu,
2636 struct nix_vtag_config *req,
2637 struct nix_vtag_config_rsp *rsp)
2638 {
2639 u16 pcifunc = req->hdr.pcifunc;
2640 int blkaddr, nixlf, err;
2641
2642 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2643 if (err)
2644 return err;
2645
2646 if (req->cfg_type) {
2647 /* rx vtag configuration */
2648 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req);
2649 if (err)
2650 return NIX_AF_ERR_PARAM;
2651 } else {
2652 /* tx vtag configuration */
2653 if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) &&
2654 (req->tx.free_vtag0 || req->tx.free_vtag1))
2655 return NIX_AF_ERR_PARAM;
2656
2657 if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1)
2658 return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp);
2659
2660 if (req->tx.free_vtag0 || req->tx.free_vtag1)
2661 return nix_tx_vtag_decfg(rvu, blkaddr, req);
2662 }
2663
2664 return 0;
2665 }
2666
2667 static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw,
2668 int mce, u8 op, u16 pcifunc, int next, bool eol)
2669 {
2670 struct nix_aq_enq_req aq_req;
2671 int err;
2672
2673 aq_req.hdr.pcifunc = 0;
2674 aq_req.ctype = NIX_AQ_CTYPE_MCE;
2675 aq_req.op = op;
2676 aq_req.qidx = mce;
2677
2678 /* Use RSS with RSS index 0 */
2679 aq_req.mce.op = 1;
2680 aq_req.mce.index = 0;
2681 aq_req.mce.eol = eol;
2682 aq_req.mce.pf_func = pcifunc;
2683 aq_req.mce.next = next;
2684
2685 /* All fields valid */
2686 *(u64 *)(&aq_req.mce_mask) = ~0ULL;
2687
2688 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL);
2689 if (err) {
2690 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n",
2691 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
2692 return err;
2693 }
2694 return 0;
2695 }
2696
2697 static int nix_update_mce_list_entry(struct nix_mce_list *mce_list,
2698 u16 pcifunc, bool add)
2699 {
2700 struct mce *mce, *tail = NULL;
2701 bool delete = false;
2702
2703 /* Scan through the current list */
2704 hlist_for_each_entry(mce, &mce_list->head, node) {
2705 /* If already exists, then delete */
2706 if (mce->pcifunc == pcifunc && !add) {
2707 delete = true;
2708 break;
2709 } else if (mce->pcifunc == pcifunc && add) {
2710 /* entry already exists */
2711 return 0;
2712 }
2713 tail = mce;
2714 }
2715
2716 if (delete) {
2717 hlist_del(&mce->node);
2718 kfree(mce);
2719 mce_list->count--;
2720 return 0;
2721 }
2722
2723 if (!add)
2724 return 0;
2725
2726 /* Add a new one to the list, at the tail */
2727 mce = kzalloc(sizeof(*mce), GFP_KERNEL);
2728 if (!mce)
2729 return -ENOMEM;
2730 mce->pcifunc = pcifunc;
2731 if (!tail)
2732 hlist_add_head(&mce->node, &mce_list->head);
2733 else
2734 hlist_add_behind(&mce->node, &tail->node);
2735 mce_list->count++;
2736 return 0;
2737 }
2738
2739 int nix_update_mce_list(struct rvu *rvu, u16 pcifunc,
2740 struct nix_mce_list *mce_list,
2741 int mce_idx, int mcam_index, bool add)
2742 {
2743 int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr;
2744 struct npc_mcam *mcam = &rvu->hw->mcam;
2745 struct nix_mcast *mcast;
2746 struct nix_hw *nix_hw;
2747 struct mce *mce;
2748
2749 if (!mce_list)
2750 return -EINVAL;
2751
2752 /* Get this PF/VF func's MCE index */
2753 idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK);
2754
2755 if (idx > (mce_idx + mce_list->max)) {
2756 dev_err(rvu->dev,
2757 "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n",
2758 __func__, idx, mce_list->max,
2759 pcifunc >> RVU_PFVF_PF_SHIFT);
2760 return -EINVAL;
2761 }
2762
2763 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
2764 if (err)
2765 return err;
2766
2767 mcast = &nix_hw->mcast;
2768 mutex_lock(&mcast->mce_lock);
2769
2770 err = nix_update_mce_list_entry(mce_list, pcifunc, add);
2771 if (err)
2772 goto end;
2773
2774 /* Disable MCAM entry in NPC */
2775 if (!mce_list->count) {
2776 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2777 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false);
2778 goto end;
2779 }
2780
2781 /* Dump the updated list to HW */
2782 idx = mce_idx;
2783 last_idx = idx + mce_list->count - 1;
2784 hlist_for_each_entry(mce, &mce_list->head, node) {
2785 if (idx > last_idx)
2786 break;
2787
2788 next_idx = idx + 1;
2789 /* EOL should be set in last MCE */
2790 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE,
2791 mce->pcifunc, next_idx,
2792 (next_idx > last_idx) ? true : false);
2793 if (err)
2794 goto end;
2795 idx++;
2796 }
2797
2798 end:
2799 mutex_unlock(&mcast->mce_lock);
2800 return err;
2801 }
2802
2803 void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type,
2804 struct nix_mce_list **mce_list, int *mce_idx)
2805 {
2806 struct rvu_hwinfo *hw = rvu->hw;
2807 struct rvu_pfvf *pfvf;
2808
2809 if (!hw->cap.nix_rx_multicast ||
2810 !is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc & ~RVU_PFVF_FUNC_MASK))) {
2811 *mce_list = NULL;
2812 *mce_idx = 0;
2813 return;
2814 }
2815
2816 /* Get this PF/VF func's MCE index */
2817 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
2818
2819 if (type == NIXLF_BCAST_ENTRY) {
2820 *mce_list = &pfvf->bcast_mce_list;
2821 *mce_idx = pfvf->bcast_mce_idx;
2822 } else if (type == NIXLF_ALLMULTI_ENTRY) {
2823 *mce_list = &pfvf->mcast_mce_list;
2824 *mce_idx = pfvf->mcast_mce_idx;
2825 } else if (type == NIXLF_PROMISC_ENTRY) {
2826 *mce_list = &pfvf->promisc_mce_list;
2827 *mce_idx = pfvf->promisc_mce_idx;
2828 } else {
2829 *mce_list = NULL;
2830 *mce_idx = 0;
2831 }
2832 }
2833
2834 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc,
2835 int type, bool add)
2836 {
2837 int err = 0, nixlf, blkaddr, mcam_index, mce_idx;
2838 struct npc_mcam *mcam = &rvu->hw->mcam;
2839 struct rvu_hwinfo *hw = rvu->hw;
2840 struct nix_mce_list *mce_list;
2841 int pf;
2842
2843 /* skip multicast pkt replication for AF's VFs & SDP links */
2844 if (is_afvf(pcifunc) || is_sdp_pfvf(pcifunc))
2845 return 0;
2846
2847 if (!hw->cap.nix_rx_multicast)
2848 return 0;
2849
2850 pf = rvu_get_pf(pcifunc);
2851 if (!is_pf_cgxmapped(rvu, pf))
2852 return 0;
2853
2854 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2855 if (blkaddr < 0)
2856 return -EINVAL;
2857
2858 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
2859 if (nixlf < 0)
2860 return -EINVAL;
2861
2862 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx);
2863
2864 mcam_index = npc_get_nixlf_mcam_index(mcam,
2865 pcifunc & ~RVU_PFVF_FUNC_MASK,
2866 nixlf, type);
2867 err = nix_update_mce_list(rvu, pcifunc, mce_list,
2868 mce_idx, mcam_index, add);
2869 return err;
2870 }
2871
2872 static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw)
2873 {
2874 struct nix_mcast *mcast = &nix_hw->mcast;
2875 int err, pf, numvfs, idx;
2876 struct rvu_pfvf *pfvf;
2877 u16 pcifunc;
2878 u64 cfg;
2879
2880 /* Skip PF0 (i.e AF) */
2881 for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) {
2882 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2883 /* If PF is not enabled, nothing to do */
2884 if (!((cfg >> 20) & 0x01))
2885 continue;
2886 /* Get numVFs attached to this PF */
2887 numvfs = (cfg >> 12) & 0xFF;
2888
2889 pfvf = &rvu->pf[pf];
2890
2891 /* This NIX0/1 block mapped to PF ? */
2892 if (pfvf->nix_blkaddr != nix_hw->blkaddr)
2893 continue;
2894
2895 /* save start idx of broadcast mce list */
2896 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1);
2897 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1);
2898
2899 /* save start idx of multicast mce list */
2900 pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1);
2901 nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1);
2902
2903 /* save the start idx of promisc mce list */
2904 pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1);
2905 nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1);
2906
2907 for (idx = 0; idx < (numvfs + 1); idx++) {
2908 /* idx-0 is for PF, followed by VFs */
2909 pcifunc = (pf << RVU_PFVF_PF_SHIFT);
2910 pcifunc |= idx;
2911 /* Add dummy entries now, so that we don't have to check
2912 * for whether AQ_OP should be INIT/WRITE later on.
2913 * Will be updated when a NIXLF is attached/detached to
2914 * these PF/VFs.
2915 */
2916 err = nix_blk_setup_mce(rvu, nix_hw,
2917 pfvf->bcast_mce_idx + idx,
2918 NIX_AQ_INSTOP_INIT,
2919 pcifunc, 0, true);
2920 if (err)
2921 return err;
2922
2923 /* add dummy entries to multicast mce list */
2924 err = nix_blk_setup_mce(rvu, nix_hw,
2925 pfvf->mcast_mce_idx + idx,
2926 NIX_AQ_INSTOP_INIT,
2927 pcifunc, 0, true);
2928 if (err)
2929 return err;
2930
2931 /* add dummy entries to promisc mce list */
2932 err = nix_blk_setup_mce(rvu, nix_hw,
2933 pfvf->promisc_mce_idx + idx,
2934 NIX_AQ_INSTOP_INIT,
2935 pcifunc, 0, true);
2936 if (err)
2937 return err;
2938 }
2939 }
2940 return 0;
2941 }
2942
2943 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
2944 {
2945 struct nix_mcast *mcast = &nix_hw->mcast;
2946 struct rvu_hwinfo *hw = rvu->hw;
2947 int err, size;
2948
2949 size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F;
2950 size = (1ULL << size);
2951
2952 /* Alloc memory for multicast/mirror replication entries */
2953 err = qmem_alloc(rvu->dev, &mcast->mce_ctx,
2954 (256UL << MC_TBL_SIZE), size);
2955 if (err)
2956 return -ENOMEM;
2957
2958 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE,
2959 (u64)mcast->mce_ctx->iova);
2960
2961 /* Set max list length equal to max no of VFs per PF + PF itself */
2962 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG,
2963 BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE);
2964
2965 /* Alloc memory for multicast replication buffers */
2966 size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF;
2967 err = qmem_alloc(rvu->dev, &mcast->mcast_buf,
2968 (8UL << MC_BUF_CNT), size);
2969 if (err)
2970 return -ENOMEM;
2971
2972 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE,
2973 (u64)mcast->mcast_buf->iova);
2974
2975 /* Alloc pkind for NIX internal RX multicast/mirror replay */
2976 mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc);
2977
2978 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG,
2979 BIT_ULL(63) | (mcast->replay_pkind << 24) |
2980 BIT_ULL(20) | MC_BUF_CNT);
2981
2982 mutex_init(&mcast->mce_lock);
2983
2984 return nix_setup_mce_tables(rvu, nix_hw);
2985 }
2986
2987 static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw)
2988 {
2989 struct nix_txvlan *vlan = &nix_hw->txvlan;
2990 int err;
2991
2992 /* Allocate resource bimap for tx vtag def registers*/
2993 vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX;
2994 err = rvu_alloc_bitmap(&vlan->rsrc);
2995 if (err)
2996 return -ENOMEM;
2997
2998 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */
2999 vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max,
3000 sizeof(u16), GFP_KERNEL);
3001 if (!vlan->entry2pfvf_map)
3002 goto free_mem;
3003
3004 mutex_init(&vlan->rsrc_lock);
3005 return 0;
3006
3007 free_mem:
3008 kfree(vlan->rsrc.bmap);
3009 return -ENOMEM;
3010 }
3011
3012 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
3013 {
3014 struct nix_txsch *txsch;
3015 int err, lvl, schq;
3016 u64 cfg, reg;
3017
3018 /* Get scheduler queue count of each type and alloc
3019 * bitmap for each for alloc/free/attach operations.
3020 */
3021 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
3022 txsch = &nix_hw->txsch[lvl];
3023 txsch->lvl = lvl;
3024 switch (lvl) {
3025 case NIX_TXSCH_LVL_SMQ:
3026 reg = NIX_AF_MDQ_CONST;
3027 break;
3028 case NIX_TXSCH_LVL_TL4:
3029 reg = NIX_AF_TL4_CONST;
3030 break;
3031 case NIX_TXSCH_LVL_TL3:
3032 reg = NIX_AF_TL3_CONST;
3033 break;
3034 case NIX_TXSCH_LVL_TL2:
3035 reg = NIX_AF_TL2_CONST;
3036 break;
3037 case NIX_TXSCH_LVL_TL1:
3038 reg = NIX_AF_TL1_CONST;
3039 break;
3040 }
3041 cfg = rvu_read64(rvu, blkaddr, reg);
3042 txsch->schq.max = cfg & 0xFFFF;
3043 err = rvu_alloc_bitmap(&txsch->schq);
3044 if (err)
3045 return err;
3046
3047 /* Allocate memory for scheduler queues to
3048 * PF/VF pcifunc mapping info.
3049 */
3050 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max,
3051 sizeof(u32), GFP_KERNEL);
3052 if (!txsch->pfvf_map)
3053 return -ENOMEM;
3054 for (schq = 0; schq < txsch->schq.max; schq++)
3055 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
3056 }
3057
3058 /* Setup a default value of 8192 as DWRR MTU */
3059 if (rvu->hw->cap.nix_common_dwrr_mtu) {
3060 rvu_write64(rvu, blkaddr, NIX_AF_DWRR_RPM_MTU,
3061 convert_bytes_to_dwrr_mtu(8192));
3062 rvu_write64(rvu, blkaddr, NIX_AF_DWRR_SDP_MTU,
3063 convert_bytes_to_dwrr_mtu(8192));
3064 }
3065
3066 return 0;
3067 }
3068
3069 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw,
3070 int blkaddr, u32 cfg)
3071 {
3072 int fmt_idx;
3073
3074 for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) {
3075 if (nix_hw->mark_format.cfg[fmt_idx] == cfg)
3076 return fmt_idx;
3077 }
3078 if (fmt_idx >= nix_hw->mark_format.total)
3079 return -ERANGE;
3080
3081 rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg);
3082 nix_hw->mark_format.cfg[fmt_idx] = cfg;
3083 nix_hw->mark_format.in_use++;
3084 return fmt_idx;
3085 }
3086
3087 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw,
3088 int blkaddr)
3089 {
3090 u64 cfgs[] = {
3091 [NIX_MARK_CFG_IP_DSCP_RED] = 0x10003,
3092 [NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200,
3093 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203,
3094 [NIX_MARK_CFG_IP_ECN_RED] = 0x6000c,
3095 [NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00,
3096 [NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c,
3097 [NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008,
3098 [NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800,
3099 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808,
3100 };
3101 int i, rc;
3102 u64 total;
3103
3104 total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8;
3105 nix_hw->mark_format.total = (u8)total;
3106 nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32),
3107 GFP_KERNEL);
3108 if (!nix_hw->mark_format.cfg)
3109 return -ENOMEM;
3110 for (i = 0; i < NIX_MARK_CFG_MAX; i++) {
3111 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]);
3112 if (rc < 0)
3113 dev_err(rvu->dev, "Err %d in setup mark format %d\n",
3114 i, rc);
3115 }
3116
3117 return 0;
3118 }
3119
3120 static void rvu_get_lbk_link_max_frs(struct rvu *rvu, u16 *max_mtu)
3121 {
3122 /* CN10K supports LBK FIFO size 72 KB */
3123 if (rvu->hw->lbk_bufsize == 0x12000)
3124 *max_mtu = CN10K_LBK_LINK_MAX_FRS;
3125 else
3126 *max_mtu = NIC_HW_MAX_FRS;
3127 }
3128
3129 static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu)
3130 {
3131 /* RPM supports FIFO len 128 KB */
3132 if (rvu_cgx_get_fifolen(rvu) == 0x20000)
3133 *max_mtu = CN10K_LMAC_LINK_MAX_FRS;
3134 else
3135 *max_mtu = NIC_HW_MAX_FRS;
3136 }
3137
3138 int rvu_mbox_handler_nix_get_hw_info(struct rvu *rvu, struct msg_req *req,
3139 struct nix_hw_info *rsp)
3140 {
3141 u16 pcifunc = req->hdr.pcifunc;
3142 u64 dwrr_mtu;
3143 int blkaddr;
3144
3145 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3146 if (blkaddr < 0)
3147 return NIX_AF_ERR_AF_LF_INVALID;
3148
3149 if (is_afvf(pcifunc))
3150 rvu_get_lbk_link_max_frs(rvu, &rsp->max_mtu);
3151 else
3152 rvu_get_lmac_link_max_frs(rvu, &rsp->max_mtu);
3153
3154 rsp->min_mtu = NIC_HW_MIN_FRS;
3155
3156 if (!rvu->hw->cap.nix_common_dwrr_mtu) {
3157 /* Return '1' on OTx2 */
3158 rsp->rpm_dwrr_mtu = 1;
3159 rsp->sdp_dwrr_mtu = 1;
3160 return 0;
3161 }
3162
3163 dwrr_mtu = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_DWRR_RPM_MTU);
3164 rsp->rpm_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu);
3165
3166 dwrr_mtu = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_DWRR_SDP_MTU);
3167 rsp->sdp_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu);
3168
3169 return 0;
3170 }
3171
3172 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req,
3173 struct msg_rsp *rsp)
3174 {
3175 u16 pcifunc = req->hdr.pcifunc;
3176 int i, nixlf, blkaddr, err;
3177 u64 stats;
3178
3179 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3180 if (err)
3181 return err;
3182
3183 /* Get stats count supported by HW */
3184 stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
3185
3186 /* Reset tx stats */
3187 for (i = 0; i < ((stats >> 24) & 0xFF); i++)
3188 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0);
3189
3190 /* Reset rx stats */
3191 for (i = 0; i < ((stats >> 32) & 0xFF); i++)
3192 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0);
3193
3194 return 0;
3195 }
3196
3197 /* Returns the ALG index to be set into NPC_RX_ACTION */
3198 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg)
3199 {
3200 int i;
3201
3202 /* Scan over exiting algo entries to find a match */
3203 for (i = 0; i < nix_hw->flowkey.in_use; i++)
3204 if (nix_hw->flowkey.flowkey[i] == flow_cfg)
3205 return i;
3206
3207 return -ERANGE;
3208 }
3209
3210 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg)
3211 {
3212 int idx, nr_field, key_off, field_marker, keyoff_marker;
3213 int max_key_off, max_bit_pos, group_member;
3214 struct nix_rx_flowkey_alg *field;
3215 struct nix_rx_flowkey_alg tmp;
3216 u32 key_type, valid_key;
3217 int l4_key_offset = 0;
3218
3219 if (!alg)
3220 return -EINVAL;
3221
3222 #define FIELDS_PER_ALG 5
3223 #define MAX_KEY_OFF 40
3224 /* Clear all fields */
3225 memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG);
3226
3227 /* Each of the 32 possible flow key algorithm definitions should
3228 * fall into above incremental config (except ALG0). Otherwise a
3229 * single NPC MCAM entry is not sufficient for supporting RSS.
3230 *
3231 * If a different definition or combination needed then NPC MCAM
3232 * has to be programmed to filter such pkts and it's action should
3233 * point to this definition to calculate flowtag or hash.
3234 *
3235 * The `for loop` goes over _all_ protocol field and the following
3236 * variables depicts the state machine forward progress logic.
3237 *
3238 * keyoff_marker - Enabled when hash byte length needs to be accounted
3239 * in field->key_offset update.
3240 * field_marker - Enabled when a new field needs to be selected.
3241 * group_member - Enabled when protocol is part of a group.
3242 */
3243
3244 keyoff_marker = 0; max_key_off = 0; group_member = 0;
3245 nr_field = 0; key_off = 0; field_marker = 1;
3246 field = &tmp; max_bit_pos = fls(flow_cfg);
3247 for (idx = 0;
3248 idx < max_bit_pos && nr_field < FIELDS_PER_ALG &&
3249 key_off < MAX_KEY_OFF; idx++) {
3250 key_type = BIT(idx);
3251 valid_key = flow_cfg & key_type;
3252 /* Found a field marker, reset the field values */
3253 if (field_marker)
3254 memset(&tmp, 0, sizeof(tmp));
3255
3256 field_marker = true;
3257 keyoff_marker = true;
3258 switch (key_type) {
3259 case NIX_FLOW_KEY_TYPE_PORT:
3260 field->sel_chan = true;
3261 /* This should be set to 1, when SEL_CHAN is set */
3262 field->bytesm1 = 1;
3263 break;
3264 case NIX_FLOW_KEY_TYPE_IPV4_PROTO:
3265 field->lid = NPC_LID_LC;
3266 field->hdr_offset = 9; /* offset */
3267 field->bytesm1 = 0; /* 1 byte */
3268 field->ltype_match = NPC_LT_LC_IP;
3269 field->ltype_mask = 0xF;
3270 break;
3271 case NIX_FLOW_KEY_TYPE_IPV4:
3272 case NIX_FLOW_KEY_TYPE_INNR_IPV4:
3273 field->lid = NPC_LID_LC;
3274 field->ltype_match = NPC_LT_LC_IP;
3275 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) {
3276 field->lid = NPC_LID_LG;
3277 field->ltype_match = NPC_LT_LG_TU_IP;
3278 }
3279 field->hdr_offset = 12; /* SIP offset */
3280 field->bytesm1 = 7; /* SIP + DIP, 8 bytes */
3281 field->ltype_mask = 0xF; /* Match only IPv4 */
3282 keyoff_marker = false;
3283 break;
3284 case NIX_FLOW_KEY_TYPE_IPV6:
3285 case NIX_FLOW_KEY_TYPE_INNR_IPV6:
3286 field->lid = NPC_LID_LC;
3287 field->ltype_match = NPC_LT_LC_IP6;
3288 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) {
3289 field->lid = NPC_LID_LG;
3290 field->ltype_match = NPC_LT_LG_TU_IP6;
3291 }
3292 field->hdr_offset = 8; /* SIP offset */
3293 field->bytesm1 = 31; /* SIP + DIP, 32 bytes */
3294 field->ltype_mask = 0xF; /* Match only IPv6 */
3295 break;
3296 case NIX_FLOW_KEY_TYPE_TCP:
3297 case NIX_FLOW_KEY_TYPE_UDP:
3298 case NIX_FLOW_KEY_TYPE_SCTP:
3299 case NIX_FLOW_KEY_TYPE_INNR_TCP:
3300 case NIX_FLOW_KEY_TYPE_INNR_UDP:
3301 case NIX_FLOW_KEY_TYPE_INNR_SCTP:
3302 field->lid = NPC_LID_LD;
3303 if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP ||
3304 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP ||
3305 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP)
3306 field->lid = NPC_LID_LH;
3307 field->bytesm1 = 3; /* Sport + Dport, 4 bytes */
3308
3309 /* Enum values for NPC_LID_LD and NPC_LID_LG are same,
3310 * so no need to change the ltype_match, just change
3311 * the lid for inner protocols
3312 */
3313 BUILD_BUG_ON((int)NPC_LT_LD_TCP !=
3314 (int)NPC_LT_LH_TU_TCP);
3315 BUILD_BUG_ON((int)NPC_LT_LD_UDP !=
3316 (int)NPC_LT_LH_TU_UDP);
3317 BUILD_BUG_ON((int)NPC_LT_LD_SCTP !=
3318 (int)NPC_LT_LH_TU_SCTP);
3319
3320 if ((key_type == NIX_FLOW_KEY_TYPE_TCP ||
3321 key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) &&
3322 valid_key) {
3323 field->ltype_match |= NPC_LT_LD_TCP;
3324 group_member = true;
3325 } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP ||
3326 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) &&
3327 valid_key) {
3328 field->ltype_match |= NPC_LT_LD_UDP;
3329 group_member = true;
3330 } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP ||
3331 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) &&
3332 valid_key) {
3333 field->ltype_match |= NPC_LT_LD_SCTP;
3334 group_member = true;
3335 }
3336 field->ltype_mask = ~field->ltype_match;
3337 if (key_type == NIX_FLOW_KEY_TYPE_SCTP ||
3338 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) {
3339 /* Handle the case where any of the group item
3340 * is enabled in the group but not the final one
3341 */
3342 if (group_member) {
3343 valid_key = true;
3344 group_member = false;
3345 }
3346 } else {
3347 field_marker = false;
3348 keyoff_marker = false;
3349 }
3350
3351 /* TCP/UDP/SCTP and ESP/AH falls at same offset so
3352 * remember the TCP key offset of 40 byte hash key.
3353 */
3354 if (key_type == NIX_FLOW_KEY_TYPE_TCP)
3355 l4_key_offset = key_off;
3356 break;
3357 case NIX_FLOW_KEY_TYPE_NVGRE:
3358 field->lid = NPC_LID_LD;
3359 field->hdr_offset = 4; /* VSID offset */
3360 field->bytesm1 = 2;
3361 field->ltype_match = NPC_LT_LD_NVGRE;
3362 field->ltype_mask = 0xF;
3363 break;
3364 case NIX_FLOW_KEY_TYPE_VXLAN:
3365 case NIX_FLOW_KEY_TYPE_GENEVE:
3366 field->lid = NPC_LID_LE;
3367 field->bytesm1 = 2;
3368 field->hdr_offset = 4;
3369 field->ltype_mask = 0xF;
3370 field_marker = false;
3371 keyoff_marker = false;
3372
3373 if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) {
3374 field->ltype_match |= NPC_LT_LE_VXLAN;
3375 group_member = true;
3376 }
3377
3378 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) {
3379 field->ltype_match |= NPC_LT_LE_GENEVE;
3380 group_member = true;
3381 }
3382
3383 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) {
3384 if (group_member) {
3385 field->ltype_mask = ~field->ltype_match;
3386 field_marker = true;
3387 keyoff_marker = true;
3388 valid_key = true;
3389 group_member = false;
3390 }
3391 }
3392 break;
3393 case NIX_FLOW_KEY_TYPE_ETH_DMAC:
3394 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC:
3395 field->lid = NPC_LID_LA;
3396 field->ltype_match = NPC_LT_LA_ETHER;
3397 if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) {
3398 field->lid = NPC_LID_LF;
3399 field->ltype_match = NPC_LT_LF_TU_ETHER;
3400 }
3401 field->hdr_offset = 0;
3402 field->bytesm1 = 5; /* DMAC 6 Byte */
3403 field->ltype_mask = 0xF;
3404 break;
3405 case NIX_FLOW_KEY_TYPE_IPV6_EXT:
3406 field->lid = NPC_LID_LC;
3407 field->hdr_offset = 40; /* IPV6 hdr */
3408 field->bytesm1 = 0; /* 1 Byte ext hdr*/
3409 field->ltype_match = NPC_LT_LC_IP6_EXT;
3410 field->ltype_mask = 0xF;
3411 break;
3412 case NIX_FLOW_KEY_TYPE_GTPU:
3413 field->lid = NPC_LID_LE;
3414 field->hdr_offset = 4;
3415 field->bytesm1 = 3; /* 4 bytes TID*/
3416 field->ltype_match = NPC_LT_LE_GTPU;
3417 field->ltype_mask = 0xF;
3418 break;
3419 case NIX_FLOW_KEY_TYPE_VLAN:
3420 field->lid = NPC_LID_LB;
3421 field->hdr_offset = 2; /* Skip TPID (2-bytes) */
3422 field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */
3423 field->ltype_match = NPC_LT_LB_CTAG;
3424 field->ltype_mask = 0xF;
3425 field->fn_mask = 1; /* Mask out the first nibble */
3426 break;
3427 case NIX_FLOW_KEY_TYPE_AH:
3428 case NIX_FLOW_KEY_TYPE_ESP:
3429 field->hdr_offset = 0;
3430 field->bytesm1 = 7; /* SPI + sequence number */
3431 field->ltype_mask = 0xF;
3432 field->lid = NPC_LID_LE;
3433 field->ltype_match = NPC_LT_LE_ESP;
3434 if (key_type == NIX_FLOW_KEY_TYPE_AH) {
3435 field->lid = NPC_LID_LD;
3436 field->ltype_match = NPC_LT_LD_AH;
3437 field->hdr_offset = 4;
3438 keyoff_marker = false;
3439 }
3440 break;
3441 }
3442 field->ena = 1;
3443
3444 /* Found a valid flow key type */
3445 if (valid_key) {
3446 /* Use the key offset of TCP/UDP/SCTP fields
3447 * for ESP/AH fields.
3448 */
3449 if (key_type == NIX_FLOW_KEY_TYPE_ESP ||
3450 key_type == NIX_FLOW_KEY_TYPE_AH)
3451 key_off = l4_key_offset;
3452 field->key_offset = key_off;
3453 memcpy(&alg[nr_field], field, sizeof(*field));
3454 max_key_off = max(max_key_off, field->bytesm1 + 1);
3455
3456 /* Found a field marker, get the next field */
3457 if (field_marker)
3458 nr_field++;
3459 }
3460
3461 /* Found a keyoff marker, update the new key_off */
3462 if (keyoff_marker) {
3463 key_off += max_key_off;
3464 max_key_off = 0;
3465 }
3466 }
3467 /* Processed all the flow key types */
3468 if (idx == max_bit_pos && key_off <= MAX_KEY_OFF)
3469 return 0;
3470 else
3471 return NIX_AF_ERR_RSS_NOSPC_FIELD;
3472 }
3473
3474 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg)
3475 {
3476 u64 field[FIELDS_PER_ALG];
3477 struct nix_hw *hw;
3478 int fid, rc;
3479
3480 hw = get_nix_hw(rvu->hw, blkaddr);
3481 if (!hw)
3482 return NIX_AF_ERR_INVALID_NIXBLK;
3483
3484 /* No room to add new flow hash algoritham */
3485 if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX)
3486 return NIX_AF_ERR_RSS_NOSPC_ALGO;
3487
3488 /* Generate algo fields for the given flow_cfg */
3489 rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg);
3490 if (rc)
3491 return rc;
3492
3493 /* Update ALGX_FIELDX register with generated fields */
3494 for (fid = 0; fid < FIELDS_PER_ALG; fid++)
3495 rvu_write64(rvu, blkaddr,
3496 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use,
3497 fid), field[fid]);
3498
3499 /* Store the flow_cfg for futher lookup */
3500 rc = hw->flowkey.in_use;
3501 hw->flowkey.flowkey[rc] = flow_cfg;
3502 hw->flowkey.in_use++;
3503
3504 return rc;
3505 }
3506
3507 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu,
3508 struct nix_rss_flowkey_cfg *req,
3509 struct nix_rss_flowkey_cfg_rsp *rsp)
3510 {
3511 u16 pcifunc = req->hdr.pcifunc;
3512 int alg_idx, nixlf, blkaddr;
3513 struct nix_hw *nix_hw;
3514 int err;
3515
3516 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3517 if (err)
3518 return err;
3519
3520 nix_hw = get_nix_hw(rvu->hw, blkaddr);
3521 if (!nix_hw)
3522 return NIX_AF_ERR_INVALID_NIXBLK;
3523
3524 alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg);
3525 /* Failed to get algo index from the exiting list, reserve new */
3526 if (alg_idx < 0) {
3527 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr,
3528 req->flowkey_cfg);
3529 if (alg_idx < 0)
3530 return alg_idx;
3531 }
3532 rsp->alg_idx = alg_idx;
3533 rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group,
3534 alg_idx, req->mcam_index);
3535 return 0;
3536 }
3537
3538 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr)
3539 {
3540 u32 flowkey_cfg, minkey_cfg;
3541 int alg, fid, rc;
3542
3543 /* Disable all flow key algx fieldx */
3544 for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) {
3545 for (fid = 0; fid < FIELDS_PER_ALG; fid++)
3546 rvu_write64(rvu, blkaddr,
3547 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid),
3548 0);
3549 }
3550
3551 /* IPv4/IPv6 SIP/DIPs */
3552 flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6;
3553 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3554 if (rc < 0)
3555 return rc;
3556
3557 /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
3558 minkey_cfg = flowkey_cfg;
3559 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP;
3560 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3561 if (rc < 0)
3562 return rc;
3563
3564 /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
3565 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP;
3566 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3567 if (rc < 0)
3568 return rc;
3569
3570 /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
3571 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP;
3572 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3573 if (rc < 0)
3574 return rc;
3575
3576 /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */
3577 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
3578 NIX_FLOW_KEY_TYPE_UDP;
3579 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3580 if (rc < 0)
3581 return rc;
3582
3583 /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
3584 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
3585 NIX_FLOW_KEY_TYPE_SCTP;
3586 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3587 if (rc < 0)
3588 return rc;
3589
3590 /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
3591 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP |
3592 NIX_FLOW_KEY_TYPE_SCTP;
3593 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3594 if (rc < 0)
3595 return rc;
3596
3597 /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
3598 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
3599 NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP;
3600 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
3601 if (rc < 0)
3602 return rc;
3603
3604 return 0;
3605 }
3606
3607 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu,
3608 struct nix_set_mac_addr *req,
3609 struct msg_rsp *rsp)
3610 {
3611 bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK;
3612 u16 pcifunc = req->hdr.pcifunc;
3613 int blkaddr, nixlf, err;
3614 struct rvu_pfvf *pfvf;
3615
3616 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3617 if (err)
3618 return err;
3619
3620 pfvf = rvu_get_pfvf(rvu, pcifunc);
3621
3622 /* untrusted VF can't overwrite admin(PF) changes */
3623 if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) &&
3624 (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) {
3625 dev_warn(rvu->dev,
3626 "MAC address set by admin(PF) cannot be overwritten by untrusted VF");
3627 return -EPERM;
3628 }
3629
3630 ether_addr_copy(pfvf->mac_addr, req->mac_addr);
3631
3632 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
3633 pfvf->rx_chan_base, req->mac_addr);
3634
3635 if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf)
3636 ether_addr_copy(pfvf->default_mac, req->mac_addr);
3637
3638 rvu_switch_update_rules(rvu, pcifunc);
3639
3640 return 0;
3641 }
3642
3643 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu,
3644 struct msg_req *req,
3645 struct nix_get_mac_addr_rsp *rsp)
3646 {
3647 u16 pcifunc = req->hdr.pcifunc;
3648 struct rvu_pfvf *pfvf;
3649
3650 if (!is_nixlf_attached(rvu, pcifunc))
3651 return NIX_AF_ERR_AF_LF_INVALID;
3652
3653 pfvf = rvu_get_pfvf(rvu, pcifunc);
3654
3655 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
3656
3657 return 0;
3658 }
3659
3660 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req,
3661 struct msg_rsp *rsp)
3662 {
3663 bool allmulti, promisc, nix_rx_multicast;
3664 u16 pcifunc = req->hdr.pcifunc;
3665 struct rvu_pfvf *pfvf;
3666 int nixlf, err;
3667
3668 pfvf = rvu_get_pfvf(rvu, pcifunc);
3669 promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false;
3670 allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false;
3671 pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false;
3672
3673 nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list;
3674
3675 if (is_vf(pcifunc) && !nix_rx_multicast &&
3676 (promisc || allmulti)) {
3677 dev_warn_ratelimited(rvu->dev,
3678 "VF promisc/multicast not supported\n");
3679 return 0;
3680 }
3681
3682 /* untrusted VF can't configure promisc/allmulti */
3683 if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) &&
3684 (promisc || allmulti))
3685 return 0;
3686
3687 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
3688 if (err)
3689 return err;
3690
3691 if (nix_rx_multicast) {
3692 /* add/del this PF_FUNC to/from mcast pkt replication list */
3693 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY,
3694 allmulti);
3695 if (err) {
3696 dev_err(rvu->dev,
3697 "Failed to update pcifunc 0x%x to multicast list\n",
3698 pcifunc);
3699 return err;
3700 }
3701
3702 /* add/del this PF_FUNC to/from promisc pkt replication list */
3703 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY,
3704 promisc);
3705 if (err) {
3706 dev_err(rvu->dev,
3707 "Failed to update pcifunc 0x%x to promisc list\n",
3708 pcifunc);
3709 return err;
3710 }
3711 }
3712
3713 /* install/uninstall allmulti entry */
3714 if (allmulti) {
3715 rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf,
3716 pfvf->rx_chan_base);
3717 } else {
3718 if (!nix_rx_multicast)
3719 rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false);
3720 }
3721
3722 /* install/uninstall promisc entry */
3723 if (promisc) {
3724 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
3725 pfvf->rx_chan_base,
3726 pfvf->rx_chan_cnt);
3727 } else {
3728 if (!nix_rx_multicast)
3729 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false);
3730 }
3731
3732 return 0;
3733 }
3734
3735 static void nix_find_link_frs(struct rvu *rvu,
3736 struct nix_frs_cfg *req, u16 pcifunc)
3737 {
3738 int pf = rvu_get_pf(pcifunc);
3739 struct rvu_pfvf *pfvf;
3740 int maxlen, minlen;
3741 int numvfs, hwvf;
3742 int vf;
3743
3744 /* Update with requester's min/max lengths */
3745 pfvf = rvu_get_pfvf(rvu, pcifunc);
3746 pfvf->maxlen = req->maxlen;
3747 if (req->update_minlen)
3748 pfvf->minlen = req->minlen;
3749
3750 maxlen = req->maxlen;
3751 minlen = req->update_minlen ? req->minlen : 0;
3752
3753 /* Get this PF's numVFs and starting hwvf */
3754 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
3755
3756 /* For each VF, compare requested max/minlen */
3757 for (vf = 0; vf < numvfs; vf++) {
3758 pfvf = &rvu->hwvf[hwvf + vf];
3759 if (pfvf->maxlen > maxlen)
3760 maxlen = pfvf->maxlen;
3761 if (req->update_minlen &&
3762 pfvf->minlen && pfvf->minlen < minlen)
3763 minlen = pfvf->minlen;
3764 }
3765
3766 /* Compare requested max/minlen with PF's max/minlen */
3767 pfvf = &rvu->pf[pf];
3768 if (pfvf->maxlen > maxlen)
3769 maxlen = pfvf->maxlen;
3770 if (req->update_minlen &&
3771 pfvf->minlen && pfvf->minlen < minlen)
3772 minlen = pfvf->minlen;
3773
3774 /* Update the request with max/min PF's and it's VF's max/min */
3775 req->maxlen = maxlen;
3776 if (req->update_minlen)
3777 req->minlen = minlen;
3778 }
3779
3780 static int
3781 nix_config_link_credits(struct rvu *rvu, int blkaddr, int link,
3782 u16 pcifunc, u64 tx_credits)
3783 {
3784 struct rvu_hwinfo *hw = rvu->hw;
3785 int pf = rvu_get_pf(pcifunc);
3786 u8 cgx_id = 0, lmac_id = 0;
3787 unsigned long poll_tmo;
3788 bool restore_tx_en = 0;
3789 struct nix_hw *nix_hw;
3790 u64 cfg, sw_xoff = 0;
3791 u32 schq = 0;
3792 u32 credits;
3793 int rc;
3794
3795 nix_hw = get_nix_hw(rvu->hw, blkaddr);
3796 if (!nix_hw)
3797 return NIX_AF_ERR_INVALID_NIXBLK;
3798
3799 if (tx_credits == nix_hw->tx_credits[link])
3800 return 0;
3801
3802 /* Enable cgx tx if disabled for credits to be back */
3803 if (is_pf_cgxmapped(rvu, pf)) {
3804 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
3805 restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
3806 lmac_id, true);
3807 }
3808
3809 mutex_lock(&rvu->rsrc_lock);
3810 /* Disable new traffic to link */
3811 if (hw->cap.nix_shaping) {
3812 schq = nix_get_tx_link(rvu, pcifunc);
3813 sw_xoff = rvu_read64(rvu, blkaddr, NIX_AF_TL1X_SW_XOFF(schq));
3814 rvu_write64(rvu, blkaddr,
3815 NIX_AF_TL1X_SW_XOFF(schq), BIT_ULL(0));
3816 }
3817
3818 rc = -EBUSY;
3819 poll_tmo = jiffies + usecs_to_jiffies(10000);
3820 /* Wait for credits to return */
3821 do {
3822 if (time_after(jiffies, poll_tmo))
3823 goto exit;
3824 usleep_range(100, 200);
3825
3826 cfg = rvu_read64(rvu, blkaddr,
3827 NIX_AF_TX_LINKX_NORM_CREDIT(link));
3828 credits = (cfg >> 12) & 0xFFFFFULL;
3829 } while (credits != nix_hw->tx_credits[link]);
3830
3831 cfg &= ~(0xFFFFFULL << 12);
3832 cfg |= (tx_credits << 12);
3833 rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg);
3834 rc = 0;
3835
3836 nix_hw->tx_credits[link] = tx_credits;
3837
3838 exit:
3839 /* Enable traffic back */
3840 if (hw->cap.nix_shaping && !sw_xoff)
3841 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SW_XOFF(schq), 0);
3842
3843 /* Restore state of cgx tx */
3844 if (restore_tx_en)
3845 cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
3846
3847 mutex_unlock(&rvu->rsrc_lock);
3848 return rc;
3849 }
3850
3851 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req,
3852 struct msg_rsp *rsp)
3853 {
3854 struct rvu_hwinfo *hw = rvu->hw;
3855 u16 pcifunc = req->hdr.pcifunc;
3856 int pf = rvu_get_pf(pcifunc);
3857 int blkaddr, schq, link = -1;
3858 struct nix_txsch *txsch;
3859 u64 cfg, lmac_fifo_len;
3860 struct nix_hw *nix_hw;
3861 struct rvu_pfvf *pfvf;
3862 u8 cgx = 0, lmac = 0;
3863 u16 max_mtu;
3864
3865 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3866 if (blkaddr < 0)
3867 return NIX_AF_ERR_AF_LF_INVALID;
3868
3869 nix_hw = get_nix_hw(rvu->hw, blkaddr);
3870 if (!nix_hw)
3871 return NIX_AF_ERR_INVALID_NIXBLK;
3872
3873 if (is_afvf(pcifunc))
3874 rvu_get_lbk_link_max_frs(rvu, &max_mtu);
3875 else
3876 rvu_get_lmac_link_max_frs(rvu, &max_mtu);
3877
3878 if (!req->sdp_link && req->maxlen > max_mtu)
3879 return NIX_AF_ERR_FRS_INVALID;
3880
3881 if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS)
3882 return NIX_AF_ERR_FRS_INVALID;
3883
3884 /* Check if requester wants to update SMQ's */
3885 if (!req->update_smq)
3886 goto rx_frscfg;
3887
3888 /* Update min/maxlen in each of the SMQ attached to this PF/VF */
3889 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
3890 mutex_lock(&rvu->rsrc_lock);
3891 for (schq = 0; schq < txsch->schq.max; schq++) {
3892 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
3893 continue;
3894 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq));
3895 cfg = (cfg & ~(0xFFFFULL << 8)) | ((u64)req->maxlen << 8);
3896 if (req->update_minlen)
3897 cfg = (cfg & ~0x7FULL) | ((u64)req->minlen & 0x7F);
3898 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq), cfg);
3899 }
3900 mutex_unlock(&rvu->rsrc_lock);
3901
3902 rx_frscfg:
3903 /* Check if config is for SDP link */
3904 if (req->sdp_link) {
3905 if (!hw->sdp_links)
3906 return NIX_AF_ERR_RX_LINK_INVALID;
3907 link = hw->cgx_links + hw->lbk_links;
3908 goto linkcfg;
3909 }
3910
3911 /* Check if the request is from CGX mapped RVU PF */
3912 if (is_pf_cgxmapped(rvu, pf)) {
3913 /* Get CGX and LMAC to which this PF is mapped and find link */
3914 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac);
3915 link = (cgx * hw->lmac_per_cgx) + lmac;
3916 } else if (pf == 0) {
3917 /* For VFs of PF0 ingress is LBK port, so config LBK link */
3918 pfvf = rvu_get_pfvf(rvu, pcifunc);
3919 link = hw->cgx_links + pfvf->lbkid;
3920 }
3921
3922 if (link < 0)
3923 return NIX_AF_ERR_RX_LINK_INVALID;
3924
3925 nix_find_link_frs(rvu, req, pcifunc);
3926
3927 linkcfg:
3928 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link));
3929 cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16);
3930 if (req->update_minlen)
3931 cfg = (cfg & ~0xFFFFULL) | req->minlen;
3932 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg);
3933
3934 if (req->sdp_link || pf == 0)
3935 return 0;
3936
3937 /* Update transmit credits for CGX links */
3938 lmac_fifo_len =
3939 rvu_cgx_get_fifolen(rvu) /
3940 cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
3941 return nix_config_link_credits(rvu, blkaddr, link, pcifunc,
3942 (lmac_fifo_len - req->maxlen) / 16);
3943 }
3944
3945 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req,
3946 struct msg_rsp *rsp)
3947 {
3948 int nixlf, blkaddr, err;
3949 u64 cfg;
3950
3951 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr);
3952 if (err)
3953 return err;
3954
3955 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf));
3956 /* Set the interface configuration */
3957 if (req->len_verify & BIT(0))
3958 cfg |= BIT_ULL(41);
3959 else
3960 cfg &= ~BIT_ULL(41);
3961
3962 if (req->len_verify & BIT(1))
3963 cfg |= BIT_ULL(40);
3964 else
3965 cfg &= ~BIT_ULL(40);
3966
3967 if (req->csum_verify & BIT(0))
3968 cfg |= BIT_ULL(37);
3969 else
3970 cfg &= ~BIT_ULL(37);
3971
3972 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg);
3973
3974 return 0;
3975 }
3976
3977 static u64 rvu_get_lbk_link_credits(struct rvu *rvu, u16 lbk_max_frs)
3978 {
3979 /* CN10k supports 72KB FIFO size and max packet size of 64k */
3980 if (rvu->hw->lbk_bufsize == 0x12000)
3981 return (rvu->hw->lbk_bufsize - lbk_max_frs) / 16;
3982
3983 return 1600; /* 16 * max LBK datarate = 16 * 100Gbps */
3984 }
3985
3986 static void nix_link_config(struct rvu *rvu, int blkaddr,
3987 struct nix_hw *nix_hw)
3988 {
3989 struct rvu_hwinfo *hw = rvu->hw;
3990 int cgx, lmac_cnt, slink, link;
3991 u16 lbk_max_frs, lmac_max_frs;
3992 u64 tx_credits, cfg;
3993
3994 rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs);
3995 rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs);
3996
3997 /* Set default min/max packet lengths allowed on NIX Rx links.
3998 *
3999 * With HW reset minlen value of 60byte, HW will treat ARP pkts
4000 * as undersize and report them to SW as error pkts, hence
4001 * setting it to 40 bytes.
4002 */
4003 for (link = 0; link < hw->cgx_links; link++) {
4004 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4005 ((u64)lmac_max_frs << 16) | NIC_HW_MIN_FRS);
4006 }
4007
4008 for (link = hw->cgx_links; link < hw->lbk_links; link++) {
4009 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4010 ((u64)lbk_max_frs << 16) | NIC_HW_MIN_FRS);
4011 }
4012 if (hw->sdp_links) {
4013 link = hw->cgx_links + hw->lbk_links;
4014 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4015 SDP_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
4016 }
4017
4018 /* Set credits for Tx links assuming max packet length allowed.
4019 * This will be reconfigured based on MTU set for PF/VF.
4020 */
4021 for (cgx = 0; cgx < hw->cgx; cgx++) {
4022 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
4023 /* Skip when cgx is not available or lmac cnt is zero */
4024 if (lmac_cnt <= 0)
4025 continue;
4026 tx_credits = ((rvu_cgx_get_fifolen(rvu) / lmac_cnt) -
4027 lmac_max_frs) / 16;
4028 /* Enable credits and set credit pkt count to max allowed */
4029 cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
4030 slink = cgx * hw->lmac_per_cgx;
4031 for (link = slink; link < (slink + lmac_cnt); link++) {
4032 nix_hw->tx_credits[link] = tx_credits;
4033 rvu_write64(rvu, blkaddr,
4034 NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg);
4035 }
4036 }
4037
4038 /* Set Tx credits for LBK link */
4039 slink = hw->cgx_links;
4040 for (link = slink; link < (slink + hw->lbk_links); link++) {
4041 tx_credits = rvu_get_lbk_link_credits(rvu, lbk_max_frs);
4042 nix_hw->tx_credits[link] = tx_credits;
4043 /* Enable credits and set credit pkt count to max allowed */
4044 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
4045 rvu_write64(rvu, blkaddr,
4046 NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits);
4047 }
4048 }
4049
4050 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr)
4051 {
4052 int idx, err;
4053 u64 status;
4054
4055 /* Start X2P bus calibration */
4056 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4057 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9));
4058 /* Wait for calibration to complete */
4059 err = rvu_poll_reg(rvu, blkaddr,
4060 NIX_AF_STATUS, BIT_ULL(10), false);
4061 if (err) {
4062 dev_err(rvu->dev, "NIX X2P bus calibration failed\n");
4063 return err;
4064 }
4065
4066 status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS);
4067 /* Check if CGX devices are ready */
4068 for (idx = 0; idx < rvu->cgx_cnt_max; idx++) {
4069 /* Skip when cgx port is not available */
4070 if (!rvu_cgx_pdata(idx, rvu) ||
4071 (status & (BIT_ULL(16 + idx))))
4072 continue;
4073 dev_err(rvu->dev,
4074 "CGX%d didn't respond to NIX X2P calibration\n", idx);
4075 err = -EBUSY;
4076 }
4077
4078 /* Check if LBK is ready */
4079 if (!(status & BIT_ULL(19))) {
4080 dev_err(rvu->dev,
4081 "LBK didn't respond to NIX X2P calibration\n");
4082 err = -EBUSY;
4083 }
4084
4085 /* Clear 'calibrate_x2p' bit */
4086 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4087 rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9));
4088 if (err || (status & 0x3FFULL))
4089 dev_err(rvu->dev,
4090 "NIX X2P calibration failed, status 0x%llx\n", status);
4091 if (err)
4092 return err;
4093 return 0;
4094 }
4095
4096 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block)
4097 {
4098 u64 cfg;
4099 int err;
4100
4101 /* Set admin queue endianness */
4102 cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG);
4103 #ifdef __BIG_ENDIAN
4104 cfg |= BIT_ULL(8);
4105 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
4106 #else
4107 cfg &= ~BIT_ULL(8);
4108 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
4109 #endif
4110
4111 /* Do not bypass NDC cache */
4112 cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG);
4113 cfg &= ~0x3FFEULL;
4114 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
4115 /* Disable caching of SQB aka SQEs */
4116 cfg |= 0x04ULL;
4117 #endif
4118 rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg);
4119
4120 /* Result structure can be followed by RQ/SQ/CQ context at
4121 * RES + 128bytes and a write mask at RES + 256 bytes, depending on
4122 * operation type. Alloc sufficient result memory for all operations.
4123 */
4124 err = rvu_aq_alloc(rvu, &block->aq,
4125 Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s),
4126 ALIGN(sizeof(struct nix_aq_res_s), 128) + 256);
4127 if (err)
4128 return err;
4129
4130 rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE);
4131 rvu_write64(rvu, block->addr,
4132 NIX_AF_AQ_BASE, (u64)block->aq->inst->iova);
4133 return 0;
4134 }
4135
4136 static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr)
4137 {
4138 struct rvu_hwinfo *hw = rvu->hw;
4139 u64 hw_const;
4140
4141 hw_const = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
4142
4143 /* On OcteonTx2 DWRR quantum is directly configured into each of
4144 * the transmit scheduler queues. And PF/VF drivers were free to
4145 * config any value upto 2^24.
4146 * On CN10K, HW is modified, the quantum configuration at scheduler
4147 * queues is in terms of weight. And SW needs to setup a base DWRR MTU
4148 * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do
4149 * 'DWRR MTU * weight' to get the quantum.
4150 *
4151 * Check if HW uses a common MTU for all DWRR quantum configs.
4152 * On OcteonTx2 this register field is '0'.
4153 */
4154 if (((hw_const >> 56) & 0x10) == 0x10)
4155 hw->cap.nix_common_dwrr_mtu = true;
4156 }
4157
4158 static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw)
4159 {
4160 const struct npc_lt_def_cfg *ltdefs;
4161 struct rvu_hwinfo *hw = rvu->hw;
4162 int blkaddr = nix_hw->blkaddr;
4163 struct rvu_block *block;
4164 int err;
4165 u64 cfg;
4166
4167 block = &hw->block[blkaddr];
4168
4169 if (is_rvu_96xx_B0(rvu)) {
4170 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt
4171 * internal state when conditional clocks are turned off.
4172 * Hence enable them.
4173 */
4174 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4175 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL);
4176
4177 /* Set chan/link to backpressure TL3 instead of TL2 */
4178 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01);
4179
4180 /* Disable SQ manager's sticky mode operation (set TM6 = 0)
4181 * This sticky mode is known to cause SQ stalls when multiple
4182 * SQs are mapped to same SMQ and transmitting pkts at a time.
4183 */
4184 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS);
4185 cfg &= ~BIT_ULL(15);
4186 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg);
4187 }
4188
4189 ltdefs = rvu->kpu.lt_def;
4190 /* Calibrate X2P bus to check if CGX/LBK links are fine */
4191 err = nix_calibrate_x2p(rvu, blkaddr);
4192 if (err)
4193 return err;
4194
4195 /* Setup capabilities of the NIX block */
4196 rvu_nix_setup_capabilities(rvu, blkaddr);
4197
4198 /* Initialize admin queue */
4199 err = nix_aq_init(rvu, block);
4200 if (err)
4201 return err;
4202
4203 /* Restore CINT timer delay to HW reset values */
4204 rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL);
4205
4206 /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */
4207 rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, 0x1ULL);
4208
4209 if (is_block_implemented(hw, blkaddr)) {
4210 err = nix_setup_txschq(rvu, nix_hw, blkaddr);
4211 if (err)
4212 return err;
4213
4214 err = nix_setup_ipolicers(rvu, nix_hw, blkaddr);
4215 if (err)
4216 return err;
4217
4218 err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr);
4219 if (err)
4220 return err;
4221
4222 err = nix_setup_mcast(rvu, nix_hw, blkaddr);
4223 if (err)
4224 return err;
4225
4226 err = nix_setup_txvlan(rvu, nix_hw);
4227 if (err)
4228 return err;
4229
4230 /* Configure segmentation offload formats */
4231 nix_setup_lso(rvu, nix_hw, blkaddr);
4232
4233 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info.
4234 * This helps HW protocol checker to identify headers
4235 * and validate length and checksums.
4236 */
4237 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2,
4238 (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) |
4239 ltdefs->rx_ol2.ltype_mask);
4240 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4,
4241 (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) |
4242 ltdefs->rx_oip4.ltype_mask);
4243 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4,
4244 (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) |
4245 ltdefs->rx_iip4.ltype_mask);
4246 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6,
4247 (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) |
4248 ltdefs->rx_oip6.ltype_mask);
4249 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6,
4250 (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) |
4251 ltdefs->rx_iip6.ltype_mask);
4252 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP,
4253 (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) |
4254 ltdefs->rx_otcp.ltype_mask);
4255 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP,
4256 (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) |
4257 ltdefs->rx_itcp.ltype_mask);
4258 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP,
4259 (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) |
4260 ltdefs->rx_oudp.ltype_mask);
4261 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP,
4262 (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) |
4263 ltdefs->rx_iudp.ltype_mask);
4264 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP,
4265 (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) |
4266 ltdefs->rx_osctp.ltype_mask);
4267 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP,
4268 (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) |
4269 ltdefs->rx_isctp.ltype_mask);
4270
4271 if (!is_rvu_otx2(rvu)) {
4272 /* Enable APAD calculation for other protocols
4273 * matching APAD0 and APAD1 lt def registers.
4274 */
4275 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0,
4276 (ltdefs->rx_apad0.valid << 11) |
4277 (ltdefs->rx_apad0.lid << 8) |
4278 (ltdefs->rx_apad0.ltype_match << 4) |
4279 ltdefs->rx_apad0.ltype_mask);
4280 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1,
4281 (ltdefs->rx_apad1.valid << 11) |
4282 (ltdefs->rx_apad1.lid << 8) |
4283 (ltdefs->rx_apad1.ltype_match << 4) |
4284 ltdefs->rx_apad1.ltype_mask);
4285
4286 /* Receive ethertype defination register defines layer
4287 * information in NPC_RESULT_S to identify the Ethertype
4288 * location in L2 header. Used for Ethertype overwriting
4289 * in inline IPsec flow.
4290 */
4291 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0),
4292 (ltdefs->rx_et[0].offset << 12) |
4293 (ltdefs->rx_et[0].valid << 11) |
4294 (ltdefs->rx_et[0].lid << 8) |
4295 (ltdefs->rx_et[0].ltype_match << 4) |
4296 ltdefs->rx_et[0].ltype_mask);
4297 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1),
4298 (ltdefs->rx_et[1].offset << 12) |
4299 (ltdefs->rx_et[1].valid << 11) |
4300 (ltdefs->rx_et[1].lid << 8) |
4301 (ltdefs->rx_et[1].ltype_match << 4) |
4302 ltdefs->rx_et[1].ltype_mask);
4303 }
4304
4305 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr);
4306 if (err)
4307 return err;
4308
4309 nix_hw->tx_credits = kcalloc(hw->cgx_links + hw->lbk_links,
4310 sizeof(u64), GFP_KERNEL);
4311 if (!nix_hw->tx_credits)
4312 return -ENOMEM;
4313
4314 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */
4315 nix_link_config(rvu, blkaddr, nix_hw);
4316
4317 /* Enable Channel backpressure */
4318 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0));
4319 }
4320 return 0;
4321 }
4322
4323 int rvu_nix_init(struct rvu *rvu)
4324 {
4325 struct rvu_hwinfo *hw = rvu->hw;
4326 struct nix_hw *nix_hw;
4327 int blkaddr = 0, err;
4328 int i = 0;
4329
4330 hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw),
4331 GFP_KERNEL);
4332 if (!hw->nix)
4333 return -ENOMEM;
4334
4335 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
4336 while (blkaddr) {
4337 nix_hw = &hw->nix[i];
4338 nix_hw->rvu = rvu;
4339 nix_hw->blkaddr = blkaddr;
4340 err = rvu_nix_block_init(rvu, nix_hw);
4341 if (err)
4342 return err;
4343 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
4344 i++;
4345 }
4346
4347 return 0;
4348 }
4349
4350 static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr,
4351 struct rvu_block *block)
4352 {
4353 struct nix_txsch *txsch;
4354 struct nix_mcast *mcast;
4355 struct nix_txvlan *vlan;
4356 struct nix_hw *nix_hw;
4357 int lvl;
4358
4359 rvu_aq_free(rvu, block->aq);
4360
4361 if (is_block_implemented(rvu->hw, blkaddr)) {
4362 nix_hw = get_nix_hw(rvu->hw, blkaddr);
4363 if (!nix_hw)
4364 return;
4365
4366 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
4367 txsch = &nix_hw->txsch[lvl];
4368 kfree(txsch->schq.bmap);
4369 }
4370
4371 kfree(nix_hw->tx_credits);
4372
4373 nix_ipolicer_freemem(rvu, nix_hw);
4374
4375 vlan = &nix_hw->txvlan;
4376 kfree(vlan->rsrc.bmap);
4377 mutex_destroy(&vlan->rsrc_lock);
4378
4379 mcast = &nix_hw->mcast;
4380 qmem_free(rvu->dev, mcast->mce_ctx);
4381 qmem_free(rvu->dev, mcast->mcast_buf);
4382 mutex_destroy(&mcast->mce_lock);
4383 }
4384 }
4385
4386 void rvu_nix_freemem(struct rvu *rvu)
4387 {
4388 struct rvu_hwinfo *hw = rvu->hw;
4389 struct rvu_block *block;
4390 int blkaddr = 0;
4391
4392 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
4393 while (blkaddr) {
4394 block = &hw->block[blkaddr];
4395 rvu_nix_block_freemem(rvu, blkaddr, block);
4396 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
4397 }
4398 }
4399
4400 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req,
4401 struct msg_rsp *rsp)
4402 {
4403 u16 pcifunc = req->hdr.pcifunc;
4404 struct rvu_pfvf *pfvf;
4405 int nixlf, err;
4406
4407 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
4408 if (err)
4409 return err;
4410
4411 rvu_npc_enable_default_entries(rvu, pcifunc, nixlf);
4412
4413 npc_mcam_enable_flows(rvu, pcifunc);
4414
4415 pfvf = rvu_get_pfvf(rvu, pcifunc);
4416 set_bit(NIXLF_INITIALIZED, &pfvf->flags);
4417
4418 rvu_switch_update_rules(rvu, pcifunc);
4419
4420 return rvu_cgx_start_stop_io(rvu, pcifunc, true);
4421 }
4422
4423 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req,
4424 struct msg_rsp *rsp)
4425 {
4426 u16 pcifunc = req->hdr.pcifunc;
4427 struct rvu_pfvf *pfvf;
4428 int nixlf, err;
4429
4430 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
4431 if (err)
4432 return err;
4433
4434 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
4435
4436 pfvf = rvu_get_pfvf(rvu, pcifunc);
4437 clear_bit(NIXLF_INITIALIZED, &pfvf->flags);
4438
4439 return rvu_cgx_start_stop_io(rvu, pcifunc, false);
4440 }
4441
4442 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
4443 {
4444 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
4445 struct hwctx_disable_req ctx_req;
4446 int err;
4447
4448 ctx_req.hdr.pcifunc = pcifunc;
4449
4450 /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */
4451 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
4452 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
4453 nix_interface_deinit(rvu, pcifunc, nixlf);
4454 nix_rx_sync(rvu, blkaddr);
4455 nix_txschq_free(rvu, pcifunc);
4456
4457 clear_bit(NIXLF_INITIALIZED, &pfvf->flags);
4458
4459 rvu_cgx_start_stop_io(rvu, pcifunc, false);
4460
4461 if (pfvf->sq_ctx) {
4462 ctx_req.ctype = NIX_AQ_CTYPE_SQ;
4463 err = nix_lf_hwctx_disable(rvu, &ctx_req);
4464 if (err)
4465 dev_err(rvu->dev, "SQ ctx disable failed\n");
4466 }
4467
4468 if (pfvf->rq_ctx) {
4469 ctx_req.ctype = NIX_AQ_CTYPE_RQ;
4470 err = nix_lf_hwctx_disable(rvu, &ctx_req);
4471 if (err)
4472 dev_err(rvu->dev, "RQ ctx disable failed\n");
4473 }
4474
4475 if (pfvf->cq_ctx) {
4476 ctx_req.ctype = NIX_AQ_CTYPE_CQ;
4477 err = nix_lf_hwctx_disable(rvu, &ctx_req);
4478 if (err)
4479 dev_err(rvu->dev, "CQ ctx disable failed\n");
4480 }
4481
4482 nix_ctx_free(rvu, pfvf);
4483
4484 nix_free_all_bandprof(rvu, pcifunc);
4485 }
4486
4487 #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32)
4488
4489 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
4490 {
4491 struct rvu_hwinfo *hw = rvu->hw;
4492 struct rvu_block *block;
4493 int blkaddr, pf;
4494 int nixlf;
4495 u64 cfg;
4496
4497 pf = rvu_get_pf(pcifunc);
4498 if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_PTP))
4499 return 0;
4500
4501 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
4502 if (blkaddr < 0)
4503 return NIX_AF_ERR_AF_LF_INVALID;
4504
4505 block = &hw->block[blkaddr];
4506 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
4507 if (nixlf < 0)
4508 return NIX_AF_ERR_AF_LF_INVALID;
4509
4510 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf));
4511
4512 if (enable)
4513 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN;
4514 else
4515 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN;
4516
4517 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
4518
4519 return 0;
4520 }
4521
4522 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req,
4523 struct msg_rsp *rsp)
4524 {
4525 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true);
4526 }
4527
4528 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req,
4529 struct msg_rsp *rsp)
4530 {
4531 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false);
4532 }
4533
4534 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu,
4535 struct nix_lso_format_cfg *req,
4536 struct nix_lso_format_cfg_rsp *rsp)
4537 {
4538 u16 pcifunc = req->hdr.pcifunc;
4539 struct nix_hw *nix_hw;
4540 struct rvu_pfvf *pfvf;
4541 int blkaddr, idx, f;
4542 u64 reg;
4543
4544 pfvf = rvu_get_pfvf(rvu, pcifunc);
4545 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
4546 if (!pfvf->nixlf || blkaddr < 0)
4547 return NIX_AF_ERR_AF_LF_INVALID;
4548
4549 nix_hw = get_nix_hw(rvu->hw, blkaddr);
4550 if (!nix_hw)
4551 return NIX_AF_ERR_INVALID_NIXBLK;
4552
4553 /* Find existing matching LSO format, if any */
4554 for (idx = 0; idx < nix_hw->lso.in_use; idx++) {
4555 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) {
4556 reg = rvu_read64(rvu, blkaddr,
4557 NIX_AF_LSO_FORMATX_FIELDX(idx, f));
4558 if (req->fields[f] != (reg & req->field_mask))
4559 break;
4560 }
4561
4562 if (f == NIX_LSO_FIELD_MAX)
4563 break;
4564 }
4565
4566 if (idx < nix_hw->lso.in_use) {
4567 /* Match found */
4568 rsp->lso_format_idx = idx;
4569 return 0;
4570 }
4571
4572 if (nix_hw->lso.in_use == nix_hw->lso.total)
4573 return NIX_AF_ERR_LSO_CFG_FAIL;
4574
4575 rsp->lso_format_idx = nix_hw->lso.in_use++;
4576
4577 for (f = 0; f < NIX_LSO_FIELD_MAX; f++)
4578 rvu_write64(rvu, blkaddr,
4579 NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f),
4580 req->fields[f]);
4581
4582 return 0;
4583 }
4584
4585 void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc)
4586 {
4587 bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK);
4588
4589 /* overwrite vf mac address with default_mac */
4590 if (from_vf)
4591 ether_addr_copy(pfvf->mac_addr, pfvf->default_mac);
4592 }
4593
4594 /* NIX ingress policers or bandwidth profiles APIs */
4595 static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr)
4596 {
4597 struct npc_lt_def_cfg defs, *ltdefs;
4598
4599 ltdefs = &defs;
4600 memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg));
4601
4602 /* Extract PCP and DEI fields from outer VLAN from byte offset
4603 * 2 from the start of LB_PTR (ie TAG).
4604 * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN
4605 * fields are considered when 'Tunnel enable' is set in profile.
4606 */
4607 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI,
4608 (2UL << 12) | (ltdefs->ovlan.lid << 8) |
4609 (ltdefs->ovlan.ltype_match << 4) |
4610 ltdefs->ovlan.ltype_mask);
4611 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI,
4612 (2UL << 12) | (ltdefs->ivlan.lid << 8) |
4613 (ltdefs->ivlan.ltype_match << 4) |
4614 ltdefs->ivlan.ltype_mask);
4615
4616 /* DSCP field in outer and tunneled IPv4 packets */
4617 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP,
4618 (1UL << 12) | (ltdefs->rx_oip4.lid << 8) |
4619 (ltdefs->rx_oip4.ltype_match << 4) |
4620 ltdefs->rx_oip4.ltype_mask);
4621 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP,
4622 (1UL << 12) | (ltdefs->rx_iip4.lid << 8) |
4623 (ltdefs->rx_iip4.ltype_match << 4) |
4624 ltdefs->rx_iip4.ltype_mask);
4625
4626 /* DSCP field (traffic class) in outer and tunneled IPv6 packets */
4627 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP,
4628 (1UL << 11) | (ltdefs->rx_oip6.lid << 8) |
4629 (ltdefs->rx_oip6.ltype_match << 4) |
4630 ltdefs->rx_oip6.ltype_mask);
4631 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP,
4632 (1UL << 11) | (ltdefs->rx_iip6.lid << 8) |
4633 (ltdefs->rx_iip6.ltype_match << 4) |
4634 ltdefs->rx_iip6.ltype_mask);
4635 }
4636
4637 static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw,
4638 int layer, int prof_idx)
4639 {
4640 struct nix_cn10k_aq_enq_req aq_req;
4641 int rc;
4642
4643 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
4644
4645 aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14);
4646 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF;
4647 aq_req.op = NIX_AQ_INSTOP_INIT;
4648
4649 /* Context is all zeros, submit to AQ */
4650 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
4651 (struct nix_aq_enq_req *)&aq_req, NULL);
4652 if (rc)
4653 dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n",
4654 layer, prof_idx);
4655 return rc;
4656 }
4657
4658 static int nix_setup_ipolicers(struct rvu *rvu,
4659 struct nix_hw *nix_hw, int blkaddr)
4660 {
4661 struct rvu_hwinfo *hw = rvu->hw;
4662 struct nix_ipolicer *ipolicer;
4663 int err, layer, prof_idx;
4664 u64 cfg;
4665
4666 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
4667 if (!(cfg & BIT_ULL(61))) {
4668 hw->cap.ipolicer = false;
4669 return 0;
4670 }
4671
4672 hw->cap.ipolicer = true;
4673 nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS,
4674 sizeof(*ipolicer), GFP_KERNEL);
4675 if (!nix_hw->ipolicer)
4676 return -ENOMEM;
4677
4678 cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST);
4679
4680 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
4681 ipolicer = &nix_hw->ipolicer[layer];
4682 switch (layer) {
4683 case BAND_PROF_LEAF_LAYER:
4684 ipolicer->band_prof.max = cfg & 0XFFFF;
4685 break;
4686 case BAND_PROF_MID_LAYER:
4687 ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF;
4688 break;
4689 case BAND_PROF_TOP_LAYER:
4690 ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF;
4691 break;
4692 }
4693
4694 if (!ipolicer->band_prof.max)
4695 continue;
4696
4697 err = rvu_alloc_bitmap(&ipolicer->band_prof);
4698 if (err)
4699 return err;
4700
4701 ipolicer->pfvf_map = devm_kcalloc(rvu->dev,
4702 ipolicer->band_prof.max,
4703 sizeof(u16), GFP_KERNEL);
4704 if (!ipolicer->pfvf_map)
4705 return -ENOMEM;
4706
4707 ipolicer->match_id = devm_kcalloc(rvu->dev,
4708 ipolicer->band_prof.max,
4709 sizeof(u16), GFP_KERNEL);
4710 if (!ipolicer->match_id)
4711 return -ENOMEM;
4712
4713 for (prof_idx = 0;
4714 prof_idx < ipolicer->band_prof.max; prof_idx++) {
4715 /* Set AF as current owner for INIT ops to succeed */
4716 ipolicer->pfvf_map[prof_idx] = 0x00;
4717
4718 /* There is no enable bit in the profile context,
4719 * so no context disable. So let's INIT them here
4720 * so that PF/VF later on have to just do WRITE to
4721 * setup policer rates and config.
4722 */
4723 err = nix_init_policer_context(rvu, nix_hw,
4724 layer, prof_idx);
4725 if (err)
4726 return err;
4727 }
4728
4729 /* Allocate memory for maintaining ref_counts for MID level
4730 * profiles, this will be needed for leaf layer profiles'
4731 * aggregation.
4732 */
4733 if (layer != BAND_PROF_MID_LAYER)
4734 continue;
4735
4736 ipolicer->ref_count = devm_kcalloc(rvu->dev,
4737 ipolicer->band_prof.max,
4738 sizeof(u16), GFP_KERNEL);
4739 }
4740
4741 /* Set policer timeunit to 2us ie (19 + 1) * 100 nsec = 2us */
4742 rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19);
4743
4744 nix_config_rx_pkt_policer_precolor(rvu, blkaddr);
4745
4746 return 0;
4747 }
4748
4749 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw)
4750 {
4751 struct nix_ipolicer *ipolicer;
4752 int layer;
4753
4754 if (!rvu->hw->cap.ipolicer)
4755 return;
4756
4757 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
4758 ipolicer = &nix_hw->ipolicer[layer];
4759
4760 if (!ipolicer->band_prof.max)
4761 continue;
4762
4763 kfree(ipolicer->band_prof.bmap);
4764 }
4765 }
4766
4767 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req,
4768 struct nix_hw *nix_hw, u16 pcifunc)
4769 {
4770 struct nix_ipolicer *ipolicer;
4771 int layer, hi_layer, prof_idx;
4772
4773 /* Bits [15:14] in profile index represent layer */
4774 layer = (req->qidx >> 14) & 0x03;
4775 prof_idx = req->qidx & 0x3FFF;
4776
4777 ipolicer = &nix_hw->ipolicer[layer];
4778 if (prof_idx >= ipolicer->band_prof.max)
4779 return -EINVAL;
4780
4781 /* Check if the profile is allocated to the requesting PCIFUNC or not
4782 * with the exception of AF. AF is allowed to read and update contexts.
4783 */
4784 if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc)
4785 return -EINVAL;
4786
4787 /* If this profile is linked to higher layer profile then check
4788 * if that profile is also allocated to the requesting PCIFUNC
4789 * or not.
4790 */
4791 if (!req->prof.hl_en)
4792 return 0;
4793
4794 /* Leaf layer profile can link only to mid layer and
4795 * mid layer to top layer.
4796 */
4797 if (layer == BAND_PROF_LEAF_LAYER)
4798 hi_layer = BAND_PROF_MID_LAYER;
4799 else if (layer == BAND_PROF_MID_LAYER)
4800 hi_layer = BAND_PROF_TOP_LAYER;
4801 else
4802 return -EINVAL;
4803
4804 ipolicer = &nix_hw->ipolicer[hi_layer];
4805 prof_idx = req->prof.band_prof_id;
4806 if (prof_idx >= ipolicer->band_prof.max ||
4807 ipolicer->pfvf_map[prof_idx] != pcifunc)
4808 return -EINVAL;
4809
4810 return 0;
4811 }
4812
4813 int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu,
4814 struct nix_bandprof_alloc_req *req,
4815 struct nix_bandprof_alloc_rsp *rsp)
4816 {
4817 int blkaddr, layer, prof, idx, err;
4818 u16 pcifunc = req->hdr.pcifunc;
4819 struct nix_ipolicer *ipolicer;
4820 struct nix_hw *nix_hw;
4821
4822 if (!rvu->hw->cap.ipolicer)
4823 return NIX_AF_ERR_IPOLICER_NOTSUPP;
4824
4825 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
4826 if (err)
4827 return err;
4828
4829 mutex_lock(&rvu->rsrc_lock);
4830 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
4831 if (layer == BAND_PROF_INVAL_LAYER)
4832 continue;
4833 if (!req->prof_count[layer])
4834 continue;
4835
4836 ipolicer = &nix_hw->ipolicer[layer];
4837 for (idx = 0; idx < req->prof_count[layer]; idx++) {
4838 /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */
4839 if (idx == MAX_BANDPROF_PER_PFFUNC)
4840 break;
4841
4842 prof = rvu_alloc_rsrc(&ipolicer->band_prof);
4843 if (prof < 0)
4844 break;
4845 rsp->prof_count[layer]++;
4846 rsp->prof_idx[layer][idx] = prof;
4847 ipolicer->pfvf_map[prof] = pcifunc;
4848 }
4849 }
4850 mutex_unlock(&rvu->rsrc_lock);
4851 return 0;
4852 }
4853
4854 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc)
4855 {
4856 int blkaddr, layer, prof_idx, err;
4857 struct nix_ipolicer *ipolicer;
4858 struct nix_hw *nix_hw;
4859
4860 if (!rvu->hw->cap.ipolicer)
4861 return NIX_AF_ERR_IPOLICER_NOTSUPP;
4862
4863 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
4864 if (err)
4865 return err;
4866
4867 mutex_lock(&rvu->rsrc_lock);
4868 /* Free all the profiles allocated to the PCIFUNC */
4869 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
4870 if (layer == BAND_PROF_INVAL_LAYER)
4871 continue;
4872 ipolicer = &nix_hw->ipolicer[layer];
4873
4874 for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) {
4875 if (ipolicer->pfvf_map[prof_idx] != pcifunc)
4876 continue;
4877
4878 /* Clear ratelimit aggregation, if any */
4879 if (layer == BAND_PROF_LEAF_LAYER &&
4880 ipolicer->match_id[prof_idx])
4881 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx);
4882
4883 ipolicer->pfvf_map[prof_idx] = 0x00;
4884 ipolicer->match_id[prof_idx] = 0;
4885 rvu_free_rsrc(&ipolicer->band_prof, prof_idx);
4886 }
4887 }
4888 mutex_unlock(&rvu->rsrc_lock);
4889 return 0;
4890 }
4891
4892 int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu,
4893 struct nix_bandprof_free_req *req,
4894 struct msg_rsp *rsp)
4895 {
4896 int blkaddr, layer, prof_idx, idx, err;
4897 u16 pcifunc = req->hdr.pcifunc;
4898 struct nix_ipolicer *ipolicer;
4899 struct nix_hw *nix_hw;
4900
4901 if (req->free_all)
4902 return nix_free_all_bandprof(rvu, pcifunc);
4903
4904 if (!rvu->hw->cap.ipolicer)
4905 return NIX_AF_ERR_IPOLICER_NOTSUPP;
4906
4907 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
4908 if (err)
4909 return err;
4910
4911 mutex_lock(&rvu->rsrc_lock);
4912 /* Free the requested profile indices */
4913 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
4914 if (layer == BAND_PROF_INVAL_LAYER)
4915 continue;
4916 if (!req->prof_count[layer])
4917 continue;
4918
4919 ipolicer = &nix_hw->ipolicer[layer];
4920 for (idx = 0; idx < req->prof_count[layer]; idx++) {
4921 prof_idx = req->prof_idx[layer][idx];
4922 if (prof_idx >= ipolicer->band_prof.max ||
4923 ipolicer->pfvf_map[prof_idx] != pcifunc)
4924 continue;
4925
4926 /* Clear ratelimit aggregation, if any */
4927 if (layer == BAND_PROF_LEAF_LAYER &&
4928 ipolicer->match_id[prof_idx])
4929 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx);
4930
4931 ipolicer->pfvf_map[prof_idx] = 0x00;
4932 ipolicer->match_id[prof_idx] = 0;
4933 rvu_free_rsrc(&ipolicer->band_prof, prof_idx);
4934 if (idx == MAX_BANDPROF_PER_PFFUNC)
4935 break;
4936 }
4937 }
4938 mutex_unlock(&rvu->rsrc_lock);
4939 return 0;
4940 }
4941
4942 int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw,
4943 struct nix_cn10k_aq_enq_req *aq_req,
4944 struct nix_cn10k_aq_enq_rsp *aq_rsp,
4945 u16 pcifunc, u8 ctype, u32 qidx)
4946 {
4947 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
4948 aq_req->hdr.pcifunc = pcifunc;
4949 aq_req->ctype = ctype;
4950 aq_req->op = NIX_AQ_INSTOP_READ;
4951 aq_req->qidx = qidx;
4952
4953 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
4954 (struct nix_aq_enq_req *)aq_req,
4955 (struct nix_aq_enq_rsp *)aq_rsp);
4956 }
4957
4958 static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu,
4959 struct nix_hw *nix_hw,
4960 struct nix_cn10k_aq_enq_req *aq_req,
4961 struct nix_cn10k_aq_enq_rsp *aq_rsp,
4962 u32 leaf_prof, u16 mid_prof)
4963 {
4964 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
4965 aq_req->hdr.pcifunc = 0x00;
4966 aq_req->ctype = NIX_AQ_CTYPE_BANDPROF;
4967 aq_req->op = NIX_AQ_INSTOP_WRITE;
4968 aq_req->qidx = leaf_prof;
4969
4970 aq_req->prof.band_prof_id = mid_prof;
4971 aq_req->prof_mask.band_prof_id = GENMASK(6, 0);
4972 aq_req->prof.hl_en = 1;
4973 aq_req->prof_mask.hl_en = 1;
4974
4975 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
4976 (struct nix_aq_enq_req *)aq_req,
4977 (struct nix_aq_enq_rsp *)aq_rsp);
4978 }
4979
4980 int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc,
4981 u16 rq_idx, u16 match_id)
4982 {
4983 int leaf_prof, mid_prof, leaf_match;
4984 struct nix_cn10k_aq_enq_req aq_req;
4985 struct nix_cn10k_aq_enq_rsp aq_rsp;
4986 struct nix_ipolicer *ipolicer;
4987 struct nix_hw *nix_hw;
4988 int blkaddr, idx, rc;
4989
4990 if (!rvu->hw->cap.ipolicer)
4991 return 0;
4992
4993 rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
4994 if (rc)
4995 return rc;
4996
4997 /* Fetch the RQ's context to see if policing is enabled */
4998 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc,
4999 NIX_AQ_CTYPE_RQ, rq_idx);
5000 if (rc) {
5001 dev_err(rvu->dev,
5002 "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n",
5003 __func__, rq_idx, pcifunc);
5004 return rc;
5005 }
5006
5007 if (!aq_rsp.rq.policer_ena)
5008 return 0;
5009
5010 /* Get the bandwidth profile ID mapped to this RQ */
5011 leaf_prof = aq_rsp.rq.band_prof_id;
5012
5013 ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER];
5014 ipolicer->match_id[leaf_prof] = match_id;
5015
5016 /* Check if any other leaf profile is marked with same match_id */
5017 for (idx = 0; idx < ipolicer->band_prof.max; idx++) {
5018 if (idx == leaf_prof)
5019 continue;
5020 if (ipolicer->match_id[idx] != match_id)
5021 continue;
5022
5023 leaf_match = idx;
5024 break;
5025 }
5026
5027 if (idx == ipolicer->band_prof.max)
5028 return 0;
5029
5030 /* Fetch the matching profile's context to check if it's already
5031 * mapped to a mid level profile.
5032 */
5033 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
5034 NIX_AQ_CTYPE_BANDPROF, leaf_match);
5035 if (rc) {
5036 dev_err(rvu->dev,
5037 "%s: Failed to fetch context of leaf profile %d\n",
5038 __func__, leaf_match);
5039 return rc;
5040 }
5041
5042 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER];
5043 if (aq_rsp.prof.hl_en) {
5044 /* Get Mid layer prof index and map leaf_prof index
5045 * also such that flows that are being steered
5046 * to different RQs and marked with same match_id
5047 * are rate limited in a aggregate fashion
5048 */
5049 mid_prof = aq_rsp.prof.band_prof_id;
5050 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
5051 &aq_req, &aq_rsp,
5052 leaf_prof, mid_prof);
5053 if (rc) {
5054 dev_err(rvu->dev,
5055 "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
5056 __func__, leaf_prof, mid_prof);
5057 goto exit;
5058 }
5059
5060 mutex_lock(&rvu->rsrc_lock);
5061 ipolicer->ref_count[mid_prof]++;
5062 mutex_unlock(&rvu->rsrc_lock);
5063 goto exit;
5064 }
5065
5066 /* Allocate a mid layer profile and
5067 * map both 'leaf_prof' and 'leaf_match' profiles to it.
5068 */
5069 mutex_lock(&rvu->rsrc_lock);
5070 mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof);
5071 if (mid_prof < 0) {
5072 dev_err(rvu->dev,
5073 "%s: Unable to allocate mid layer profile\n", __func__);
5074 mutex_unlock(&rvu->rsrc_lock);
5075 goto exit;
5076 }
5077 mutex_unlock(&rvu->rsrc_lock);
5078 ipolicer->pfvf_map[mid_prof] = 0x00;
5079 ipolicer->ref_count[mid_prof] = 0;
5080
5081 /* Initialize mid layer profile same as 'leaf_prof' */
5082 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
5083 NIX_AQ_CTYPE_BANDPROF, leaf_prof);
5084 if (rc) {
5085 dev_err(rvu->dev,
5086 "%s: Failed to fetch context of leaf profile %d\n",
5087 __func__, leaf_prof);
5088 goto exit;
5089 }
5090
5091 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
5092 aq_req.hdr.pcifunc = 0x00;
5093 aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14);
5094 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF;
5095 aq_req.op = NIX_AQ_INSTOP_WRITE;
5096 memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s));
5097 /* Clear higher layer enable bit in the mid profile, just in case */
5098 aq_req.prof.hl_en = 0;
5099 aq_req.prof_mask.hl_en = 1;
5100
5101 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
5102 (struct nix_aq_enq_req *)&aq_req, NULL);
5103 if (rc) {
5104 dev_err(rvu->dev,
5105 "%s: Failed to INIT context of mid layer profile %d\n",
5106 __func__, mid_prof);
5107 goto exit;
5108 }
5109
5110 /* Map both leaf profiles to this mid layer profile */
5111 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
5112 &aq_req, &aq_rsp,
5113 leaf_prof, mid_prof);
5114 if (rc) {
5115 dev_err(rvu->dev,
5116 "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
5117 __func__, leaf_prof, mid_prof);
5118 goto exit;
5119 }
5120
5121 mutex_lock(&rvu->rsrc_lock);
5122 ipolicer->ref_count[mid_prof]++;
5123 mutex_unlock(&rvu->rsrc_lock);
5124
5125 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
5126 &aq_req, &aq_rsp,
5127 leaf_match, mid_prof);
5128 if (rc) {
5129 dev_err(rvu->dev,
5130 "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
5131 __func__, leaf_match, mid_prof);
5132 ipolicer->ref_count[mid_prof]--;
5133 goto exit;
5134 }
5135
5136 mutex_lock(&rvu->rsrc_lock);
5137 ipolicer->ref_count[mid_prof]++;
5138 mutex_unlock(&rvu->rsrc_lock);
5139
5140 exit:
5141 return rc;
5142 }
5143
5144 /* Called with mutex rsrc_lock */
5145 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw,
5146 u32 leaf_prof)
5147 {
5148 struct nix_cn10k_aq_enq_req aq_req;
5149 struct nix_cn10k_aq_enq_rsp aq_rsp;
5150 struct nix_ipolicer *ipolicer;
5151 u16 mid_prof;
5152 int rc;
5153
5154 mutex_unlock(&rvu->rsrc_lock);
5155
5156 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
5157 NIX_AQ_CTYPE_BANDPROF, leaf_prof);
5158
5159 mutex_lock(&rvu->rsrc_lock);
5160 if (rc) {
5161 dev_err(rvu->dev,
5162 "%s: Failed to fetch context of leaf profile %d\n",
5163 __func__, leaf_prof);
5164 return;
5165 }
5166
5167 if (!aq_rsp.prof.hl_en)
5168 return;
5169
5170 mid_prof = aq_rsp.prof.band_prof_id;
5171 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER];
5172 ipolicer->ref_count[mid_prof]--;
5173 /* If ref_count is zero, free mid layer profile */
5174 if (!ipolicer->ref_count[mid_prof]) {
5175 ipolicer->pfvf_map[mid_prof] = 0x00;
5176 rvu_free_rsrc(&ipolicer->band_prof, mid_prof);
5177 }
5178 }
5179
5180 int rvu_mbox_handler_nix_bandprof_get_hwinfo(struct rvu *rvu, struct msg_req *req,
5181 struct nix_bandprof_get_hwinfo_rsp *rsp)
5182 {
5183 struct nix_ipolicer *ipolicer;
5184 int blkaddr, layer, err;
5185 struct nix_hw *nix_hw;
5186 u64 tu;
5187
5188 if (!rvu->hw->cap.ipolicer)
5189 return NIX_AF_ERR_IPOLICER_NOTSUPP;
5190
5191 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
5192 if (err)
5193 return err;
5194
5195 /* Return number of bandwidth profiles free at each layer */
5196 mutex_lock(&rvu->rsrc_lock);
5197 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5198 if (layer == BAND_PROF_INVAL_LAYER)
5199 continue;
5200
5201 ipolicer = &nix_hw->ipolicer[layer];
5202 rsp->prof_count[layer] = rvu_rsrc_free_count(&ipolicer->band_prof);
5203 }
5204 mutex_unlock(&rvu->rsrc_lock);
5205
5206 /* Set the policer timeunit in nanosec */
5207 tu = rvu_read64(rvu, blkaddr, NIX_AF_PL_TS) & GENMASK_ULL(9, 0);
5208 rsp->policer_timeunit = (tu + 1) * 100;
5209
5210 return 0;
5211 }
Ubuntu Jammy Linux kernel mirror