1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2014-2018 Broadcom
8 #include <rte_malloc.h>
11 #include "bnxt_filter.h"
12 #include "bnxt_hwrm.h"
13 #include "bnxt_ring.h"
16 #include "bnxt_vnic.h"
17 #include "hsi_struct_def_dpdk.h"
23 void bnxt_free_rxq_stats(struct bnxt_rx_queue
*rxq
)
25 if (rxq
&& rxq
->cp_ring
&& rxq
->cp_ring
->hw_stats
)
26 rxq
->cp_ring
->hw_stats
= NULL
;
29 int bnxt_mq_rx_configure(struct bnxt
*bp
)
31 struct rte_eth_conf
*dev_conf
= &bp
->eth_dev
->data
->dev_conf
;
32 const struct rte_eth_vmdq_rx_conf
*conf
=
33 &dev_conf
->rx_adv_conf
.vmdq_rx_conf
;
34 unsigned int i
, j
, nb_q_per_grp
= 1, ring_idx
= 0;
35 int start_grp_id
, end_grp_id
= 1, rc
= 0;
36 struct bnxt_vnic_info
*vnic
;
37 struct bnxt_filter_info
*filter
;
38 enum rte_eth_nb_pools pools
= 1, max_pools
= 0;
39 struct bnxt_rx_queue
*rxq
;
43 /* Single queue mode */
44 if (bp
->rx_cp_nr_rings
< 2) {
45 vnic
= &bp
->vnic_info
[0];
47 PMD_DRV_LOG(ERR
, "VNIC alloc failed\n");
51 vnic
->flags
|= BNXT_VNIC_INFO_BCAST
;
54 rxq
= bp
->eth_dev
->data
->rx_queues
[0];
57 vnic
->func_default
= true;
58 vnic
->start_grp_id
= 0;
59 vnic
->end_grp_id
= vnic
->start_grp_id
;
60 filter
= bnxt_alloc_filter(bp
);
62 PMD_DRV_LOG(ERR
, "L2 filter alloc failed\n");
66 filter
->mac_index
= 0;
67 filter
->flags
|= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST
;
68 STAILQ_INSERT_TAIL(&vnic
->filter
, filter
, next
);
72 /* Multi-queue mode */
73 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_VMDQ_DCB_RSS
) {
74 /* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */
76 switch (dev_conf
->rxmode
.mq_mode
) {
77 case ETH_MQ_RX_VMDQ_RSS
:
78 case ETH_MQ_RX_VMDQ_ONLY
:
79 case ETH_MQ_RX_VMDQ_DCB_RSS
:
82 pools
= conf
->nb_queue_pools
;
83 /* For each pool, allocate MACVLAN CFA rule & VNIC */
84 max_pools
= RTE_MIN(bp
->max_vnics
,
85 RTE_MIN(bp
->max_l2_ctx
,
86 RTE_MIN(bp
->max_rsscos_ctx
,
89 "pools = %u max_pools = %u\n",
91 if (pools
> max_pools
)
95 pools
= bp
->rx_cosq_cnt
? bp
->rx_cosq_cnt
: 1;
98 PMD_DRV_LOG(ERR
, "Unsupported mq_mod %d\n",
99 dev_conf
->rxmode
.mq_mode
);
103 } else if (!dev_conf
->rxmode
.mq_mode
) {
104 pools
= bp
->rx_cosq_cnt
? bp
->rx_cosq_cnt
: pools
;
107 pools
= RTE_MIN(pools
, bp
->rx_cp_nr_rings
);
108 nb_q_per_grp
= bp
->rx_cp_nr_rings
/ pools
;
109 bp
->rx_num_qs_per_vnic
= nb_q_per_grp
;
110 PMD_DRV_LOG(DEBUG
, "pools = %u nb_q_per_grp = %u\n",
111 pools
, nb_q_per_grp
);
113 end_grp_id
= nb_q_per_grp
;
115 for (i
= 0; i
< pools
; i
++) {
116 vnic
= &bp
->vnic_info
[i
];
118 PMD_DRV_LOG(ERR
, "VNIC alloc failed\n");
122 vnic
->flags
|= BNXT_VNIC_INFO_BCAST
;
125 for (j
= 0; j
< nb_q_per_grp
; j
++, ring_idx
++) {
126 rxq
= bp
->eth_dev
->data
->rx_queues
[ring_idx
];
129 "rxq[%d] = %p vnic[%d] = %p\n",
130 ring_idx
, rxq
, i
, vnic
);
133 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_VMDQ_DCB
) {
134 bp
->eth_dev
->data
->promiscuous
= 1;
135 vnic
->flags
|= BNXT_VNIC_INFO_PROMISC
;
137 vnic
->func_default
= true;
139 vnic
->start_grp_id
= start_grp_id
;
140 vnic
->end_grp_id
= end_grp_id
;
143 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_VMDQ_DCB
||
144 !(dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_RSS
))
145 vnic
->rss_dflt_cr
= true;
146 goto skip_filter_allocation
;
148 filter
= bnxt_alloc_filter(bp
);
150 PMD_DRV_LOG(ERR
, "L2 filter alloc failed\n");
154 filter
->mac_index
= 0;
155 filter
->flags
|= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST
;
157 * TODO: Configure & associate CFA rule for
158 * each VNIC for each VMDq with MACVLAN, MACVLAN+TC
160 STAILQ_INSERT_TAIL(&vnic
->filter
, filter
, next
);
162 skip_filter_allocation
:
163 start_grp_id
= end_grp_id
;
164 end_grp_id
+= nb_q_per_grp
;
168 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_RSS_FLAG
) {
169 struct rte_eth_rss_conf
*rss
= &dev_conf
->rx_adv_conf
.rss_conf
;
171 if (bp
->flags
& BNXT_FLAG_UPDATE_HASH
)
172 bp
->flags
&= ~BNXT_FLAG_UPDATE_HASH
;
174 for (i
= 0; i
< bp
->nr_vnics
; i
++) {
175 vnic
= &bp
->vnic_info
[i
];
177 bnxt_rte_to_hwrm_hash_types(rss
->rss_hf
);
180 * Use the supplied key if the key length is
181 * acceptable and the rss_key is not NULL
184 rss
->rss_key_len
<= HW_HASH_KEY_SIZE
)
185 memcpy(vnic
->rss_hash_key
,
186 rss
->rss_key
, rss
->rss_key_len
);
193 /* Free allocated vnic/filters */
198 void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue
*rxq
)
200 struct bnxt_sw_rx_bd
*sw_ring
;
201 struct bnxt_tpa_info
*tpa_info
;
207 rte_spinlock_lock(&rxq
->lock
);
209 sw_ring
= rxq
->rx_ring
->rx_buf_ring
;
212 i
< rxq
->rx_ring
->rx_ring_struct
->ring_size
; i
++) {
213 if (sw_ring
[i
].mbuf
) {
214 rte_pktmbuf_free_seg(sw_ring
[i
].mbuf
);
215 sw_ring
[i
].mbuf
= NULL
;
219 /* Free up mbufs in Agg ring */
220 sw_ring
= rxq
->rx_ring
->ag_buf_ring
;
223 i
< rxq
->rx_ring
->ag_ring_struct
->ring_size
; i
++) {
224 if (sw_ring
[i
].mbuf
) {
225 rte_pktmbuf_free_seg(sw_ring
[i
].mbuf
);
226 sw_ring
[i
].mbuf
= NULL
;
231 /* Free up mbufs in TPA */
232 tpa_info
= rxq
->rx_ring
->tpa_info
;
234 int max_aggs
= BNXT_TPA_MAX_AGGS(rxq
->bp
);
236 for (i
= 0; i
< max_aggs
; i
++) {
237 if (tpa_info
[i
].mbuf
) {
238 rte_pktmbuf_free_seg(tpa_info
[i
].mbuf
);
239 tpa_info
[i
].mbuf
= NULL
;
244 rte_spinlock_unlock(&rxq
->lock
);
247 void bnxt_free_rx_mbufs(struct bnxt
*bp
)
249 struct bnxt_rx_queue
*rxq
;
252 for (i
= 0; i
< (int)bp
->rx_nr_rings
; i
++) {
253 rxq
= bp
->rx_queues
[i
];
254 bnxt_rx_queue_release_mbufs(rxq
);
258 void bnxt_rx_queue_release_op(void *rx_queue
)
260 struct bnxt_rx_queue
*rxq
= (struct bnxt_rx_queue
*)rx_queue
;
263 if (is_bnxt_in_error(rxq
->bp
))
266 bnxt_rx_queue_release_mbufs(rxq
);
268 /* Free RX ring hardware descriptors */
269 bnxt_free_ring(rxq
->rx_ring
->rx_ring_struct
);
270 /* Free RX Agg ring hardware descriptors */
271 bnxt_free_ring(rxq
->rx_ring
->ag_ring_struct
);
273 /* Free RX completion ring hardware descriptors */
274 bnxt_free_ring(rxq
->cp_ring
->cp_ring_struct
);
276 bnxt_free_rxq_stats(rxq
);
277 rte_memzone_free(rxq
->mz
);
284 int bnxt_rx_queue_setup_op(struct rte_eth_dev
*eth_dev
,
287 unsigned int socket_id
,
288 const struct rte_eth_rxconf
*rx_conf
,
289 struct rte_mempool
*mp
)
291 struct bnxt
*bp
= eth_dev
->data
->dev_private
;
292 uint64_t rx_offloads
= eth_dev
->data
->dev_conf
.rxmode
.offloads
;
293 struct bnxt_rx_queue
*rxq
;
297 rc
= is_bnxt_in_error(bp
);
301 if (queue_idx
>= BNXT_MAX_RINGS(bp
)) {
303 "Cannot create Rx ring %d. Only %d rings available\n",
304 queue_idx
, bp
->max_rx_rings
);
308 if (!nb_desc
|| nb_desc
> MAX_RX_DESC_CNT
) {
309 PMD_DRV_LOG(ERR
, "nb_desc %d is invalid\n", nb_desc
);
314 if (eth_dev
->data
->rx_queues
) {
315 rxq
= eth_dev
->data
->rx_queues
[queue_idx
];
317 bnxt_rx_queue_release_op(rxq
);
319 rxq
= rte_zmalloc_socket("bnxt_rx_queue", sizeof(struct bnxt_rx_queue
),
320 RTE_CACHE_LINE_SIZE
, socket_id
);
322 PMD_DRV_LOG(ERR
, "bnxt_rx_queue allocation failed!\n");
328 rxq
->nb_rx_desc
= nb_desc
;
329 rxq
->rx_free_thresh
= rx_conf
->rx_free_thresh
;
331 PMD_DRV_LOG(DEBUG
, "RX Buf MTU %d\n", eth_dev
->data
->mtu
);
333 rc
= bnxt_init_rx_ring_struct(rxq
, socket_id
);
337 PMD_DRV_LOG(DEBUG
, "RX Buf size is %d\n", rxq
->rx_buf_size
);
338 rxq
->queue_id
= queue_idx
;
339 rxq
->port_id
= eth_dev
->data
->port_id
;
340 if (rx_offloads
& DEV_RX_OFFLOAD_KEEP_CRC
)
341 rxq
->crc_len
= RTE_ETHER_CRC_LEN
;
345 eth_dev
->data
->rx_queues
[queue_idx
] = rxq
;
346 /* Allocate RX ring hardware descriptors */
347 if (bnxt_alloc_rings(bp
, queue_idx
, NULL
, rxq
, rxq
->cp_ring
, NULL
,
350 "ring_dma_zone_reserve for rx_ring failed!\n");
351 bnxt_rx_queue_release_op(rxq
);
355 rte_atomic64_init(&rxq
->rx_mbuf_alloc_fail
);
357 /* rxq 0 must not be stopped when used as async CPR */
358 if (!BNXT_NUM_ASYNC_CPR(bp
) && queue_idx
== 0)
359 rxq
->rx_deferred_start
= false;
361 rxq
->rx_deferred_start
= rx_conf
->rx_deferred_start
;
363 if (rxq
->rx_deferred_start
) {
364 queue_state
= RTE_ETH_QUEUE_STATE_STOPPED
;
365 rxq
->rx_started
= false;
367 queue_state
= RTE_ETH_QUEUE_STATE_STARTED
;
368 rxq
->rx_started
= true;
370 eth_dev
->data
->rx_queue_state
[queue_idx
] = queue_state
;
371 rte_spinlock_init(&rxq
->lock
);
373 /* Configure mtu if it is different from what was configured before */
375 bnxt_mtu_set_op(eth_dev
, eth_dev
->data
->mtu
);
382 bnxt_rx_queue_intr_enable_op(struct rte_eth_dev
*eth_dev
, uint16_t queue_id
)
384 struct bnxt
*bp
= eth_dev
->data
->dev_private
;
385 struct bnxt_rx_queue
*rxq
;
386 struct bnxt_cp_ring_info
*cpr
;
389 rc
= is_bnxt_in_error(bp
);
393 if (eth_dev
->data
->rx_queues
) {
394 rxq
= eth_dev
->data
->rx_queues
[queue_id
];
399 B_CP_DB_REARM(cpr
, cpr
->cp_raw_cons
);
405 bnxt_rx_queue_intr_disable_op(struct rte_eth_dev
*eth_dev
, uint16_t queue_id
)
407 struct bnxt
*bp
= eth_dev
->data
->dev_private
;
408 struct bnxt_rx_queue
*rxq
;
409 struct bnxt_cp_ring_info
*cpr
;
412 rc
= is_bnxt_in_error(bp
);
416 if (eth_dev
->data
->rx_queues
) {
417 rxq
= eth_dev
->data
->rx_queues
[queue_id
];
427 int bnxt_rx_queue_start(struct rte_eth_dev
*dev
, uint16_t rx_queue_id
)
429 struct bnxt
*bp
= dev
->data
->dev_private
;
430 struct rte_eth_conf
*dev_conf
= &bp
->eth_dev
->data
->dev_conf
;
431 struct bnxt_rx_queue
*rxq
= bp
->rx_queues
[rx_queue_id
];
432 struct bnxt_vnic_info
*vnic
= NULL
;
435 rc
= is_bnxt_in_error(bp
);
440 PMD_DRV_LOG(ERR
, "Invalid Rx queue %d\n", rx_queue_id
);
444 /* Set the queue state to started here.
445 * We check the status of the queue while posting buffer.
446 * If queue is it started, we do not post buffers for Rx.
448 rxq
->rx_started
= true;
449 dev
->data
->rx_queue_state
[rx_queue_id
] = RTE_ETH_QUEUE_STATE_STARTED
;
451 bnxt_free_hwrm_rx_ring(bp
, rx_queue_id
);
452 rc
= bnxt_alloc_hwrm_rx_ring(bp
, rx_queue_id
);
456 if (BNXT_CHIP_THOR(bp
)) {
457 /* Reconfigure default receive ring and MRU. */
458 bnxt_hwrm_vnic_cfg(bp
, rxq
->vnic
);
460 PMD_DRV_LOG(INFO
, "Rx queue started %d\n", rx_queue_id
);
462 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_RSS_FLAG
) {
465 if (BNXT_HAS_RING_GRPS(bp
)) {
466 if (vnic
->fw_grp_ids
[rx_queue_id
] != INVALID_HW_RING_ID
)
469 vnic
->fw_grp_ids
[rx_queue_id
] =
470 bp
->grp_info
[rx_queue_id
].fw_grp_id
;
472 "vnic = %p fw_grp_id = %d\n",
473 vnic
, bp
->grp_info
[rx_queue_id
].fw_grp_id
);
476 PMD_DRV_LOG(DEBUG
, "Rx Queue Count %d\n", vnic
->rx_queue_cnt
);
477 rc
= bnxt_vnic_rss_configure(bp
, vnic
);
481 dev
->data
->rx_queue_state
[rx_queue_id
] =
482 RTE_ETH_QUEUE_STATE_STOPPED
;
483 rxq
->rx_started
= false;
487 "queue %d, rx_deferred_start %d, state %d!\n",
488 rx_queue_id
, rxq
->rx_deferred_start
,
489 bp
->eth_dev
->data
->rx_queue_state
[rx_queue_id
]);
494 int bnxt_rx_queue_stop(struct rte_eth_dev
*dev
, uint16_t rx_queue_id
)
496 struct bnxt
*bp
= dev
->data
->dev_private
;
497 struct rte_eth_conf
*dev_conf
= &bp
->eth_dev
->data
->dev_conf
;
498 struct bnxt_vnic_info
*vnic
= NULL
;
499 struct bnxt_rx_queue
*rxq
= NULL
;
500 int active_queue_cnt
= 0;
503 rc
= is_bnxt_in_error(bp
);
507 /* For the stingray platform and other platforms needing tighter
508 * control of resource utilization, Rx CQ 0 also works as
509 * Default CQ for async notifications
511 if (!BNXT_NUM_ASYNC_CPR(bp
) && !rx_queue_id
) {
512 PMD_DRV_LOG(ERR
, "Cannot stop Rx queue id %d\n", rx_queue_id
);
516 rxq
= bp
->rx_queues
[rx_queue_id
];
518 PMD_DRV_LOG(ERR
, "Invalid Rx queue %d\n", rx_queue_id
);
524 PMD_DRV_LOG(ERR
, "VNIC not initialized for RxQ %d\n",
529 dev
->data
->rx_queue_state
[rx_queue_id
] = RTE_ETH_QUEUE_STATE_STOPPED
;
530 rxq
->rx_started
= false;
531 PMD_DRV_LOG(DEBUG
, "Rx queue stopped\n");
533 if (dev_conf
->rxmode
.mq_mode
& ETH_MQ_RX_RSS_FLAG
) {
534 if (BNXT_HAS_RING_GRPS(bp
))
535 vnic
->fw_grp_ids
[rx_queue_id
] = INVALID_HW_RING_ID
;
537 PMD_DRV_LOG(DEBUG
, "Rx Queue Count %d\n", vnic
->rx_queue_cnt
);
538 rc
= bnxt_vnic_rss_configure(bp
, vnic
);
541 if (BNXT_CHIP_THOR(bp
)) {
542 /* Compute current number of active receive queues. */
543 for (i
= vnic
->start_grp_id
; i
< vnic
->end_grp_id
; i
++)
544 if (bp
->rx_queues
[i
]->rx_started
)
548 * For Thor, we need to ensure that the VNIC default receive
549 * ring corresponds to an active receive queue. When no queue
550 * is active, we need to temporarily set the MRU to zero so
551 * that packets are dropped early in the receive pipeline in
552 * order to prevent the VNIC default receive ring from being
555 if (active_queue_cnt
== 0) {
556 uint16_t saved_mru
= vnic
->mru
;
559 /* Reconfigure default receive ring and MRU. */
560 bnxt_hwrm_vnic_cfg(bp
, vnic
);
561 vnic
->mru
= saved_mru
;
563 /* Reconfigure default receive ring. */
564 bnxt_hwrm_vnic_cfg(bp
, vnic
);
569 bnxt_rx_queue_release_mbufs(rxq
);