4 * Copyright(c) Broadcom Limited.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Broadcom Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_byteorder.h>
38 #include <rte_malloc.h>
39 #include <rte_memory.h>
43 #include "bnxt_ring.h"
46 #include "hsi_struct_def_dpdk.h"
52 static inline struct rte_mbuf
*__bnxt_alloc_rx_data(struct rte_mempool
*mb
)
54 struct rte_mbuf
*data
;
56 data
= rte_mbuf_raw_alloc(mb
);
61 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue
*rxq
,
62 struct bnxt_rx_ring_info
*rxr
,
65 struct rx_prod_pkt_bd
*rxbd
= &rxr
->rx_desc_ring
[prod
];
66 struct bnxt_sw_rx_bd
*rx_buf
= &rxr
->rx_buf_ring
[prod
];
67 struct rte_mbuf
*data
;
69 data
= __bnxt_alloc_rx_data(rxq
->mb_pool
);
75 rxbd
->addr
= rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(rx_buf
->mbuf
));
80 static void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info
*rxr
, uint16_t cons
,
81 struct rte_mbuf
*mbuf
)
83 uint16_t prod
= rxr
->rx_prod
;
84 struct bnxt_sw_rx_bd
*prod_rx_buf
;
85 struct rx_prod_pkt_bd
*prod_bd
, *cons_bd
;
87 prod_rx_buf
= &rxr
->rx_buf_ring
[prod
];
89 prod_rx_buf
->mbuf
= mbuf
;
91 prod_bd
= &rxr
->rx_desc_ring
[prod
];
92 cons_bd
= &rxr
->rx_desc_ring
[cons
];
94 prod_bd
->addr
= cons_bd
->addr
;
97 static uint16_t bnxt_rx_pkt(struct rte_mbuf
**rx_pkt
,
98 struct bnxt_rx_queue
*rxq
, uint32_t *raw_cons
)
100 struct bnxt_cp_ring_info
*cpr
= rxq
->cp_ring
;
101 struct bnxt_rx_ring_info
*rxr
= rxq
->rx_ring
;
102 struct rx_pkt_cmpl
*rxcmp
;
103 struct rx_pkt_cmpl_hi
*rxcmp1
;
104 uint32_t tmp_raw_cons
= *raw_cons
;
105 uint16_t cons
, prod
, cp_cons
=
106 RING_CMP(cpr
->cp_ring_struct
, tmp_raw_cons
);
107 struct bnxt_sw_rx_bd
*rx_buf
;
108 struct rte_mbuf
*mbuf
;
111 rxcmp
= (struct rx_pkt_cmpl
*)
112 &cpr
->cp_desc_ring
[cp_cons
];
114 tmp_raw_cons
= NEXT_RAW_CMP(tmp_raw_cons
);
115 cp_cons
= RING_CMP(cpr
->cp_ring_struct
, tmp_raw_cons
);
116 rxcmp1
= (struct rx_pkt_cmpl_hi
*)&cpr
->cp_desc_ring
[cp_cons
];
118 if (!CMP_VALID(rxcmp1
, tmp_raw_cons
, cpr
->cp_ring_struct
))
123 /* EW - GRO deferred to phase 3 */
124 cons
= rxcmp
->opaque
;
125 rx_buf
= &rxr
->rx_buf_ring
[cons
];
131 mbuf
->pkt_len
= rxcmp
->len
;
132 mbuf
->data_len
= mbuf
->pkt_len
;
133 mbuf
->port
= rxq
->port_id
;
135 if (rxcmp
->flags_type
& RX_PKT_CMPL_FLAGS_RSS_VALID
) {
136 mbuf
->hash
.rss
= rxcmp
->rss_hash
;
137 mbuf
->ol_flags
|= PKT_RX_RSS_HASH
;
139 mbuf
->hash
.fdir
.id
= rxcmp1
->cfa_code
;
140 mbuf
->ol_flags
|= PKT_RX_FDIR
| PKT_RX_FDIR_ID
;
142 if (rxcmp1
->flags2
& RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN
) {
143 mbuf
->vlan_tci
= rxcmp1
->metadata
&
144 (RX_PKT_CMPL_METADATA_VID_MASK
|
145 RX_PKT_CMPL_METADATA_DE
|
146 RX_PKT_CMPL_METADATA_PRI_MASK
);
147 mbuf
->ol_flags
|= PKT_RX_VLAN_PKT
;
151 if (rxcmp1
->errors_v2
& RX_CMP_L2_ERRORS
) {
152 /* Re-install the mbuf back to the rx ring */
153 bnxt_reuse_rx_mbuf(rxr
, cons
, mbuf
);
159 * TODO: Redesign this....
160 * If the allocation fails, the packet does not get received.
161 * Simply returning this will result in slowly falling behind
162 * on the producer ring buffers.
163 * Instead, "filling up" the producer just before ringing the
164 * doorbell could be a better solution since it will let the
165 * producer ring starve until memory is available again pushing
166 * the drops into hardware and getting them out of the driver
167 * allowing recovery to a full producer ring.
169 * This could also help with cache usage by preventing per-packet
170 * calls in favour of a tight loop with the same function being called
173 if (bnxt_alloc_rx_data(rxq
, rxr
, prod
)) {
174 RTE_LOG(ERR
, PMD
, "mbuf alloc failed with prod=0x%x\n", prod
);
180 * All MBUFs are allocated with the same size under DPDK,
181 * no optimization for rx_copy_thresh
184 /* AGG buf operation is deferred */
186 /* EW - VLAN reception. Must compare against the ol_flags */
190 rxr
->rx_prod
= RING_NEXT(rxr
->rx_ring_struct
, prod
);
192 *raw_cons
= tmp_raw_cons
;
197 uint16_t bnxt_recv_pkts(void *rx_queue
, struct rte_mbuf
**rx_pkts
,
200 struct bnxt_rx_queue
*rxq
= rx_queue
;
201 struct bnxt_cp_ring_info
*cpr
= rxq
->cp_ring
;
202 struct bnxt_rx_ring_info
*rxr
= rxq
->rx_ring
;
203 uint32_t raw_cons
= cpr
->cp_raw_cons
;
206 bool rx_event
= false;
207 struct rx_pkt_cmpl
*rxcmp
;
209 /* Handle RX burst request */
213 cons
= RING_CMP(cpr
->cp_ring_struct
, raw_cons
);
214 rte_prefetch0(&cpr
->cp_desc_ring
[cons
]);
215 rxcmp
= (struct rx_pkt_cmpl
*)&cpr
->cp_desc_ring
[cons
];
217 if (!CMP_VALID(rxcmp
, raw_cons
, cpr
->cp_ring_struct
))
220 /* TODO: Avoid magic numbers... */
221 if ((CMP_TYPE(rxcmp
) & 0x30) == 0x10) {
222 rc
= bnxt_rx_pkt(&rx_pkts
[nb_rx_pkts
], rxq
, &raw_cons
);
225 else if (rc
== -EBUSY
) /* partial completion */
229 raw_cons
= NEXT_RAW_CMP(raw_cons
);
230 if (nb_rx_pkts
== nb_pkts
)
233 if (raw_cons
== cpr
->cp_raw_cons
) {
235 * For PMD, there is no need to keep on pushing to REARM
236 * the doorbell if there are no new completions
240 cpr
->cp_raw_cons
= raw_cons
;
242 B_CP_DIS_DB(cpr
, cpr
->cp_raw_cons
);
244 B_RX_DB(rxr
->rx_doorbell
, rxr
->rx_prod
);
248 void bnxt_free_rx_rings(struct bnxt
*bp
)
252 for (i
= 0; i
< (int)bp
->rx_nr_rings
; i
++) {
253 struct bnxt_rx_queue
*rxq
= bp
->rx_queues
[i
];
258 bnxt_free_ring(rxq
->rx_ring
->rx_ring_struct
);
259 rte_free(rxq
->rx_ring
->rx_ring_struct
);
260 rte_free(rxq
->rx_ring
);
262 bnxt_free_ring(rxq
->cp_ring
->cp_ring_struct
);
263 rte_free(rxq
->cp_ring
->cp_ring_struct
);
264 rte_free(rxq
->cp_ring
);
267 bp
->rx_queues
[i
] = NULL
;
271 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue
*rxq
, unsigned int socket_id
)
273 struct bnxt
*bp
= rxq
->bp
;
274 struct bnxt_cp_ring_info
*cpr
;
275 struct bnxt_rx_ring_info
*rxr
;
276 struct bnxt_ring
*ring
;
278 rxq
->rx_buf_use_size
= bp
->eth_dev
->data
->mtu
+
279 ETHER_HDR_LEN
+ ETHER_CRC_LEN
+
281 rxq
->rx_buf_size
= rxq
->rx_buf_use_size
+ sizeof(struct rte_mbuf
);
283 rxr
= rte_zmalloc_socket("bnxt_rx_ring",
284 sizeof(struct bnxt_rx_ring_info
),
285 RTE_CACHE_LINE_SIZE
, socket_id
);
290 ring
= rte_zmalloc_socket("bnxt_rx_ring_struct",
291 sizeof(struct bnxt_ring
),
292 RTE_CACHE_LINE_SIZE
, socket_id
);
295 rxr
->rx_ring_struct
= ring
;
296 ring
->ring_size
= rte_align32pow2(rxq
->nb_rx_desc
);
297 ring
->ring_mask
= ring
->ring_size
- 1;
298 ring
->bd
= (void *)rxr
->rx_desc_ring
;
299 ring
->bd_dma
= rxr
->rx_desc_mapping
;
300 ring
->vmem_size
= ring
->ring_size
* sizeof(struct bnxt_sw_rx_bd
);
301 ring
->vmem
= (void **)&rxr
->rx_buf_ring
;
303 cpr
= rte_zmalloc_socket("bnxt_rx_ring",
304 sizeof(struct bnxt_cp_ring_info
),
305 RTE_CACHE_LINE_SIZE
, socket_id
);
310 ring
= rte_zmalloc_socket("bnxt_rx_ring_struct",
311 sizeof(struct bnxt_ring
),
312 RTE_CACHE_LINE_SIZE
, socket_id
);
315 cpr
->cp_ring_struct
= ring
;
316 ring
->ring_size
= rxr
->rx_ring_struct
->ring_size
* 2;
317 ring
->ring_mask
= ring
->ring_size
- 1;
318 ring
->bd
= (void *)cpr
->cp_desc_ring
;
319 ring
->bd_dma
= cpr
->cp_desc_mapping
;
326 static void bnxt_init_rxbds(struct bnxt_ring
*ring
, uint32_t type
,
330 struct rx_prod_pkt_bd
*rx_bd_ring
= (struct rx_prod_pkt_bd
*)ring
->bd
;
334 for (j
= 0; j
< ring
->ring_size
; j
++) {
335 rx_bd_ring
[j
].flags_type
= type
;
336 rx_bd_ring
[j
].len
= len
;
337 rx_bd_ring
[j
].opaque
= j
;
341 int bnxt_init_one_rx_ring(struct bnxt_rx_queue
*rxq
)
343 struct bnxt_rx_ring_info
*rxr
;
344 struct bnxt_ring
*ring
;
348 type
= RX_PROD_PKT_BD_TYPE_RX_PROD_PKT
| RX_PROD_PKT_BD_FLAGS_EOP_PAD
;
351 ring
= rxr
->rx_ring_struct
;
352 bnxt_init_rxbds(ring
, type
, rxq
->rx_buf_use_size
);
355 for (i
= 0; i
< ring
->ring_size
; i
++) {
356 if (bnxt_alloc_rx_data(rxq
, rxr
, prod
) != 0) {
357 RTE_LOG(WARNING
, PMD
,
358 "init'ed rx ring %d with %d/%d mbufs only\n",
359 rxq
->queue_id
, i
, ring
->ring_size
);
363 prod
= RING_NEXT(rxr
->rx_ring_struct
, prod
);