]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright 2015 6WIND S.A. | |
3 | * Copyright 2015 Mellanox Technologies, Ltd | |
7c673cae FG |
4 | */ |
5 | ||
6 | #ifndef RTE_PMD_MLX5_RXTX_H_ | |
7 | #define RTE_PMD_MLX5_RXTX_H_ | |
8 | ||
9 | #include <stddef.h> | |
10 | #include <stdint.h> | |
9f95a23c | 11 | #include <sys/queue.h> |
7c673cae FG |
12 | |
13 | /* Verbs header. */ | |
14 | /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ | |
15 | #ifdef PEDANTIC | |
16 | #pragma GCC diagnostic ignored "-Wpedantic" | |
17 | #endif | |
18 | #include <infiniband/verbs.h> | |
9f95a23c | 19 | #include <infiniband/mlx5dv.h> |
7c673cae FG |
20 | #ifdef PEDANTIC |
21 | #pragma GCC diagnostic error "-Wpedantic" | |
22 | #endif | |
23 | ||
7c673cae FG |
24 | #include <rte_mbuf.h> |
25 | #include <rte_mempool.h> | |
26 | #include <rte_common.h> | |
9f95a23c TL |
27 | #include <rte_hexdump.h> |
28 | #include <rte_atomic.h> | |
29 | #include <rte_spinlock.h> | |
30 | #include <rte_io.h> | |
31 | #include <rte_bus_pci.h> | |
7c673cae FG |
32 | |
33 | #include "mlx5_utils.h" | |
34 | #include "mlx5.h" | |
9f95a23c | 35 | #include "mlx5_mr.h" |
7c673cae FG |
36 | #include "mlx5_autoconf.h" |
37 | #include "mlx5_defs.h" | |
38 | #include "mlx5_prm.h" | |
39 | ||
9f95a23c TL |
40 | /* Support tunnel matching. */ |
41 | #define MLX5_FLOW_TUNNEL 5 | |
42 | ||
7c673cae | 43 | struct mlx5_rxq_stats { |
7c673cae FG |
44 | #ifdef MLX5_PMD_SOFT_COUNTERS |
45 | uint64_t ipackets; /**< Total of successfully received packets. */ | |
46 | uint64_t ibytes; /**< Total of successfully received bytes. */ | |
47 | #endif | |
48 | uint64_t idropped; /**< Total of packets dropped when RX ring full. */ | |
49 | uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */ | |
50 | }; | |
51 | ||
52 | struct mlx5_txq_stats { | |
7c673cae FG |
53 | #ifdef MLX5_PMD_SOFT_COUNTERS |
54 | uint64_t opackets; /**< Total of successfully sent packets. */ | |
55 | uint64_t obytes; /**< Total of successfully sent bytes. */ | |
56 | #endif | |
9f95a23c | 57 | uint64_t oerrors; /**< Total number of failed transmitted packets. */ |
7c673cae FG |
58 | }; |
59 | ||
9f95a23c | 60 | struct mlx5_priv; |
7c673cae FG |
61 | |
62 | /* Compressed CQE context. */ | |
63 | struct rxq_zip { | |
64 | uint16_t ai; /* Array index. */ | |
65 | uint16_t ca; /* Current array index. */ | |
66 | uint16_t na; /* Next array index. */ | |
67 | uint16_t cq_ci; /* The next CQE. */ | |
68 | uint32_t cqe_cnt; /* Number of CQEs. */ | |
69 | }; | |
70 | ||
9f95a23c TL |
71 | /* Multi-Packet RQ buffer header. */ |
72 | struct mlx5_mprq_buf { | |
73 | struct rte_mempool *mp; | |
74 | rte_atomic16_t refcnt; /* Atomically accessed refcnt. */ | |
75 | uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */ | |
76 | } __rte_cache_aligned; | |
77 | ||
78 | /* Get pointer to the first stride. */ | |
79 | #define mlx5_mprq_buf_addr(ptr) ((ptr) + 1) | |
80 | ||
7c673cae | 81 | /* RX queue descriptor. */ |
9f95a23c | 82 | struct mlx5_rxq_data { |
7c673cae | 83 | unsigned int csum:1; /* Enable checksum offloading. */ |
9f95a23c | 84 | unsigned int hw_timestamp:1; /* Enable HW timestamp. */ |
7c673cae FG |
85 | unsigned int vlan_strip:1; /* Enable VLAN stripping. */ |
86 | unsigned int crc_present:1; /* CRC must be subtracted. */ | |
87 | unsigned int sges_n:2; /* Log 2 of SGEs (max buffers per packet). */ | |
88 | unsigned int cqe_n:4; /* Log 2 of CQ elements. */ | |
89 | unsigned int elts_n:4; /* Log 2 of Mbufs. */ | |
7c673cae | 90 | unsigned int rss_hash:1; /* RSS hash result is enabled. */ |
11fdf7f2 | 91 | unsigned int mark:1; /* Marked flow available on the queue. */ |
9f95a23c TL |
92 | unsigned int strd_num_n:5; /* Log 2 of the number of stride. */ |
93 | unsigned int strd_sz_n:4; /* Log 2 of stride size. */ | |
94 | unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */ | |
95 | unsigned int :6; /* Remaining bits. */ | |
7c673cae FG |
96 | volatile uint32_t *rq_db; |
97 | volatile uint32_t *cq_db; | |
9f95a23c TL |
98 | uint16_t port_id; |
99 | uint32_t rq_ci; | |
100 | uint16_t consumed_strd; /* Number of consumed strides in WQE. */ | |
101 | uint32_t rq_pi; | |
102 | uint32_t cq_ci; | |
103 | uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */ | |
104 | struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ | |
105 | uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */ | |
106 | volatile void *wqes; | |
7c673cae FG |
107 | volatile struct mlx5_cqe(*cqes)[]; |
108 | struct rxq_zip zip; /* Compressed context. */ | |
9f95a23c TL |
109 | RTE_STD_C11 |
110 | union { | |
111 | struct rte_mbuf *(*elts)[]; | |
112 | struct mlx5_mprq_buf *(*mprq_bufs)[]; | |
113 | }; | |
7c673cae | 114 | struct rte_mempool *mp; |
9f95a23c TL |
115 | struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */ |
116 | struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */ | |
117 | uint16_t idx; /* Queue index. */ | |
7c673cae | 118 | struct mlx5_rxq_stats stats; |
9f95a23c TL |
119 | uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */ |
120 | struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ | |
121 | void *cq_uar; /* CQ user access region. */ | |
122 | uint32_t cqn; /* CQ number. */ | |
123 | uint8_t cq_arm_sn; /* CQ arm seq number. */ | |
124 | #ifndef RTE_ARCH_64 | |
125 | rte_spinlock_t *uar_lock_cq; | |
126 | /* CQ (UAR) access lock required for 32bit implementations */ | |
127 | #endif | |
128 | uint32_t tunnel; /* Tunnel information. */ | |
7c673cae FG |
129 | } __rte_cache_aligned; |
130 | ||
9f95a23c TL |
131 | /* Verbs Rx queue elements. */ |
132 | struct mlx5_rxq_ibv { | |
133 | LIST_ENTRY(mlx5_rxq_ibv) next; /* Pointer to the next element. */ | |
134 | rte_atomic32_t refcnt; /* Reference counter. */ | |
135 | struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */ | |
7c673cae | 136 | struct ibv_cq *cq; /* Completion Queue. */ |
9f95a23c | 137 | struct ibv_wq *wq; /* Work Queue. */ |
11fdf7f2 | 138 | struct ibv_comp_channel *channel; |
7c673cae FG |
139 | }; |
140 | ||
9f95a23c TL |
141 | /* RX queue control descriptor. */ |
142 | struct mlx5_rxq_ctrl { | |
143 | struct mlx5_rxq_data rxq; /* Data path structure. */ | |
144 | LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ | |
145 | rte_atomic32_t refcnt; /* Reference counter. */ | |
146 | struct mlx5_rxq_ibv *ibv; /* Verbs elements. */ | |
147 | struct mlx5_priv *priv; /* Back pointer to private data. */ | |
148 | unsigned int socket; /* CPU socket ID for allocations. */ | |
149 | unsigned int irq:1; /* Whether IRQ is enabled. */ | |
150 | uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */ | |
151 | uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */ | |
7c673cae FG |
152 | }; |
153 | ||
9f95a23c TL |
154 | /* Indirection table. */ |
155 | struct mlx5_ind_table_ibv { | |
156 | LIST_ENTRY(mlx5_ind_table_ibv) next; /* Pointer to the next element. */ | |
157 | rte_atomic32_t refcnt; /* Reference counter. */ | |
158 | struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ | |
159 | uint32_t queues_n; /**< Number of queues in the list. */ | |
160 | uint16_t queues[]; /**< Queue list. */ | |
7c673cae FG |
161 | }; |
162 | ||
9f95a23c TL |
163 | /* Hash Rx queue. */ |
164 | struct mlx5_hrxq { | |
165 | LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */ | |
166 | rte_atomic32_t refcnt; /* Reference counter. */ | |
167 | struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */ | |
168 | struct ibv_qp *qp; /* Verbs queue pair. */ | |
169 | #ifdef HAVE_IBV_FLOW_DV_SUPPORT | |
170 | void *action; /* DV QP action pointer. */ | |
171 | #endif | |
172 | uint64_t hash_fields; /* Verbs Hash fields. */ | |
173 | uint32_t rss_key_len; /* Hash key length in bytes. */ | |
174 | uint8_t rss_key[]; /* Hash key. */ | |
7c673cae FG |
175 | }; |
176 | ||
177 | /* TX queue descriptor. */ | |
9f95a23c TL |
178 | __extension__ |
179 | struct mlx5_txq_data { | |
180 | uint16_t elts_head; /* Current counter in (*elts)[]. */ | |
181 | uint16_t elts_tail; /* Counter of first element awaiting completion. */ | |
7c673cae | 182 | uint16_t elts_comp; /* Counter since last completion request. */ |
11fdf7f2 | 183 | uint16_t mpw_comp; /* WQ index since last completion request. */ |
7c673cae | 184 | uint16_t cq_ci; /* Consumer index for completion queue. */ |
9f95a23c | 185 | #ifndef NDEBUG |
11fdf7f2 | 186 | uint16_t cq_pi; /* Producer index for completion queue. */ |
9f95a23c | 187 | #endif |
7c673cae | 188 | uint16_t wqe_ci; /* Consumer index for work queue. */ |
11fdf7f2 | 189 | uint16_t wqe_pi; /* Producer index for work queue. */ |
7c673cae FG |
190 | uint16_t elts_n:4; /* (*elts)[] length (in log2). */ |
191 | uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ | |
192 | uint16_t wqe_n:4; /* Number of of WQ elements (in log2). */ | |
11fdf7f2 TL |
193 | uint16_t tso_en:1; /* When set hardware TSO is enabled. */ |
194 | uint16_t tunnel_en:1; | |
195 | /* When set TX offload for tunneled packets are supported. */ | |
9f95a23c | 196 | uint16_t swp_en:1; /* Whether SW parser is enabled. */ |
11fdf7f2 | 197 | uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ |
7c673cae | 198 | uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ |
11fdf7f2 | 199 | uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ |
7c673cae | 200 | uint32_t qp_num_8s; /* QP number shifted by 8. */ |
9f95a23c TL |
201 | uint64_t offloads; /* Offloads for Tx Queue. */ |
202 | struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ | |
7c673cae | 203 | volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ |
11fdf7f2 | 204 | volatile void *wqes; /* Work queue (use volatile to write into). */ |
7c673cae FG |
205 | volatile uint32_t *qp_db; /* Work queue doorbell. */ |
206 | volatile uint32_t *cq_db; /* Completion queue doorbell. */ | |
7c673cae | 207 | struct rte_mbuf *(*elts)[]; /* TX elements. */ |
9f95a23c TL |
208 | uint16_t port_id; /* Port ID of device. */ |
209 | uint16_t idx; /* Queue index. */ | |
7c673cae | 210 | struct mlx5_txq_stats stats; /* TX queue counters. */ |
9f95a23c TL |
211 | #ifndef RTE_ARCH_64 |
212 | rte_spinlock_t *uar_lock; | |
213 | /* UAR access lock required for 32bit implementations */ | |
214 | #endif | |
7c673cae FG |
215 | } __rte_cache_aligned; |
216 | ||
9f95a23c TL |
217 | /* Verbs Rx queue elements. */ |
218 | struct mlx5_txq_ibv { | |
219 | LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */ | |
220 | rte_atomic32_t refcnt; /* Reference counter. */ | |
221 | struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */ | |
7c673cae FG |
222 | struct ibv_cq *cq; /* Completion Queue. */ |
223 | struct ibv_qp *qp; /* Queue Pair. */ | |
9f95a23c TL |
224 | }; |
225 | ||
226 | /* TX queue control descriptor. */ | |
227 | struct mlx5_txq_ctrl { | |
228 | struct mlx5_txq_data txq; /* Data path structure. */ | |
229 | LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */ | |
230 | rte_atomic32_t refcnt; /* Reference counter. */ | |
7c673cae | 231 | unsigned int socket; /* CPU socket ID for allocations. */ |
9f95a23c TL |
232 | unsigned int max_inline_data; /* Max inline data. */ |
233 | unsigned int max_tso_header; /* Max TSO header size. */ | |
234 | struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ | |
235 | struct mlx5_priv *priv; /* Back pointer to private data. */ | |
236 | off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ | |
237 | void *bf_reg; /* BlueFlame register from Verbs. */ | |
7c673cae FG |
238 | }; |
239 | ||
9f95a23c TL |
240 | #define MLX5_TX_BFREG(txq) \ |
241 | (MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx]) | |
7c673cae | 242 | |
9f95a23c | 243 | /* mlx5_rxq.c */ |
7c673cae FG |
244 | |
245 | extern uint8_t rss_hash_default_key[]; | |
9f95a23c TL |
246 | |
247 | int mlx5_check_mprq_support(struct rte_eth_dev *dev); | |
248 | int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq); | |
249 | int mlx5_mprq_enabled(struct rte_eth_dev *dev); | |
250 | int mlx5_mprq_free_mp(struct rte_eth_dev *dev); | |
251 | int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev); | |
252 | void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl); | |
253 | int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, | |
254 | unsigned int socket, const struct rte_eth_rxconf *conf, | |
255 | struct rte_mempool *mp); | |
256 | void mlx5_rx_queue_release(void *dpdk_rxq); | |
257 | int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); | |
258 | void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); | |
259 | int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); | |
260 | int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); | |
261 | struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); | |
262 | struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); | |
263 | int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv); | |
264 | int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv); | |
265 | struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev); | |
266 | void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev); | |
267 | int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev); | |
268 | struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, | |
269 | uint16_t desc, unsigned int socket, | |
270 | const struct rte_eth_rxconf *conf, | |
271 | struct rte_mempool *mp); | |
272 | struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); | |
273 | int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); | |
274 | int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx); | |
275 | int mlx5_rxq_verify(struct rte_eth_dev *dev); | |
276 | int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); | |
277 | int rxq_alloc_mprq_buf(struct mlx5_rxq_ctrl *rxq_ctrl); | |
278 | struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, | |
279 | const uint16_t *queues, | |
280 | uint32_t queues_n); | |
281 | struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, | |
282 | const uint16_t *queues, | |
283 | uint32_t queues_n); | |
284 | int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, | |
285 | struct mlx5_ind_table_ibv *ind_tbl); | |
286 | int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev); | |
287 | struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev); | |
288 | void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev); | |
289 | struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, | |
290 | const uint8_t *rss_key, uint32_t rss_key_len, | |
291 | uint64_t hash_fields, | |
292 | const uint16_t *queues, uint32_t queues_n, | |
293 | int tunnel __rte_unused); | |
294 | struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, | |
295 | const uint8_t *rss_key, uint32_t rss_key_len, | |
296 | uint64_t hash_fields, | |
297 | const uint16_t *queues, uint32_t queues_n); | |
298 | int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq); | |
299 | int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev); | |
300 | struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev); | |
301 | void mlx5_hrxq_drop_release(struct rte_eth_dev *dev); | |
302 | uint64_t mlx5_get_rx_port_offloads(void); | |
303 | uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev); | |
7c673cae FG |
304 | |
305 | /* mlx5_txq.c */ | |
306 | ||
9f95a23c TL |
307 | int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, |
308 | unsigned int socket, const struct rte_eth_txconf *conf); | |
309 | void mlx5_tx_queue_release(void *dpdk_txq); | |
310 | int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd); | |
311 | struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); | |
312 | struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); | |
313 | int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv); | |
314 | int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv); | |
315 | int mlx5_txq_ibv_verify(struct rte_eth_dev *dev); | |
316 | struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, | |
317 | uint16_t desc, unsigned int socket, | |
318 | const struct rte_eth_txconf *conf); | |
319 | struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); | |
320 | int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); | |
321 | int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); | |
322 | int mlx5_txq_verify(struct rte_eth_dev *dev); | |
323 | void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl); | |
324 | uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev); | |
7c673cae FG |
325 | |
326 | /* mlx5_rxtx.c */ | |
327 | ||
9f95a23c TL |
328 | extern uint32_t mlx5_ptype_table[]; |
329 | extern uint8_t mlx5_cksum_table[]; | |
330 | extern uint8_t mlx5_swp_types_table[]; | |
331 | ||
332 | void mlx5_set_ptype_table(void); | |
333 | void mlx5_set_cksum_table(void); | |
334 | void mlx5_set_swp_types_table(void); | |
335 | uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, | |
336 | uint16_t pkts_n); | |
337 | uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, | |
338 | uint16_t pkts_n); | |
339 | uint16_t mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, | |
340 | uint16_t pkts_n); | |
341 | uint16_t mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, | |
342 | uint16_t pkts_n); | |
343 | uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); | |
344 | void mlx5_mprq_buf_free_cb(void *addr, void *opaque); | |
345 | void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); | |
346 | uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, | |
347 | uint16_t pkts_n); | |
348 | uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, | |
349 | uint16_t pkts_n); | |
350 | uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, | |
351 | uint16_t pkts_n); | |
352 | int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); | |
353 | int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); | |
354 | uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); | |
355 | ||
356 | /* Vectorized version of mlx5_rxtx.c */ | |
357 | int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev); | |
358 | int mlx5_check_vec_tx_support(struct rte_eth_dev *dev); | |
359 | int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); | |
360 | int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); | |
361 | uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, | |
362 | uint16_t pkts_n); | |
363 | uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, | |
364 | uint16_t pkts_n); | |
365 | uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, | |
366 | uint16_t pkts_n); | |
7c673cae FG |
367 | |
368 | /* mlx5_mr.c */ | |
369 | ||
9f95a23c TL |
370 | void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl); |
371 | uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr); | |
372 | uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb); | |
373 | uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr, | |
374 | struct rte_mempool *mp); | |
375 | int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova, | |
376 | size_t len); | |
377 | int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova, | |
378 | size_t len); | |
379 | ||
380 | /** | |
381 | * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and | |
382 | * 64bit architectures. | |
383 | * | |
384 | * @param val | |
385 | * value to write in CPU endian format. | |
386 | * @param addr | |
387 | * Address to write to. | |
388 | * @param lock | |
389 | * Address of the lock to use for that UAR access. | |
390 | */ | |
391 | static __rte_always_inline void | |
392 | __mlx5_uar_write64_relaxed(uint64_t val, void *addr, | |
393 | rte_spinlock_t *lock __rte_unused) | |
394 | { | |
395 | #ifdef RTE_ARCH_64 | |
396 | *(uint64_t *)addr = val; | |
397 | #else /* !RTE_ARCH_64 */ | |
398 | rte_spinlock_lock(lock); | |
399 | *(uint32_t *)addr = val; | |
400 | rte_io_wmb(); | |
401 | *((uint32_t *)addr + 1) = val >> 32; | |
402 | rte_spinlock_unlock(lock); | |
403 | #endif | |
404 | } | |
405 | ||
406 | /** | |
407 | * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and | |
408 | * 64bit architectures while guaranteeing the order of execution with the | |
409 | * code being executed. | |
410 | * | |
411 | * @param val | |
412 | * value to write in CPU endian format. | |
413 | * @param addr | |
414 | * Address to write to. | |
415 | * @param lock | |
416 | * Address of the lock to use for that UAR access. | |
417 | */ | |
418 | static __rte_always_inline void | |
419 | __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock) | |
420 | { | |
421 | rte_io_wmb(); | |
422 | __mlx5_uar_write64_relaxed(val, addr, lock); | |
423 | } | |
424 | ||
425 | /* Assist macros, used instead of directly calling the functions they wrap. */ | |
426 | #ifdef RTE_ARCH_64 | |
427 | #define mlx5_uar_write64_relaxed(val, dst, lock) \ | |
428 | __mlx5_uar_write64_relaxed(val, dst, NULL) | |
429 | #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL) | |
430 | #else | |
431 | #define mlx5_uar_write64_relaxed(val, dst, lock) \ | |
432 | __mlx5_uar_write64_relaxed(val, dst, lock) | |
433 | #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock) | |
434 | #endif | |
435 | ||
436 | #ifndef NDEBUG | |
437 | /** | |
438 | * Verify or set magic value in CQE. | |
439 | * | |
440 | * @param cqe | |
441 | * Pointer to CQE. | |
442 | * | |
443 | * @return | |
444 | * 0 the first time. | |
445 | */ | |
446 | static inline int | |
447 | check_cqe_seen(volatile struct mlx5_cqe *cqe) | |
448 | { | |
449 | static const uint8_t magic[] = "seen"; | |
450 | volatile uint8_t (*buf)[sizeof(cqe->rsvd1)] = &cqe->rsvd1; | |
451 | int ret = 1; | |
452 | unsigned int i; | |
453 | ||
454 | for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i) | |
455 | if (!ret || (*buf)[i] != magic[i]) { | |
456 | ret = 0; | |
457 | (*buf)[i] = magic[i]; | |
458 | } | |
459 | return ret; | |
460 | } | |
461 | #endif /* NDEBUG */ | |
462 | ||
463 | /** | |
464 | * Check whether CQE is valid. | |
465 | * | |
466 | * @param cqe | |
467 | * Pointer to CQE. | |
468 | * @param cqes_n | |
469 | * Size of completion queue. | |
470 | * @param ci | |
471 | * Consumer index. | |
472 | * | |
473 | * @return | |
474 | * 0 on success, 1 on failure. | |
475 | */ | |
476 | static __rte_always_inline int | |
477 | check_cqe(volatile struct mlx5_cqe *cqe, | |
478 | unsigned int cqes_n, const uint16_t ci) | |
479 | { | |
480 | uint16_t idx = ci & cqes_n; | |
481 | uint8_t op_own = cqe->op_own; | |
482 | uint8_t op_owner = MLX5_CQE_OWNER(op_own); | |
483 | uint8_t op_code = MLX5_CQE_OPCODE(op_own); | |
484 | ||
485 | if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) | |
486 | return 1; /* No CQE. */ | |
487 | #ifndef NDEBUG | |
488 | if ((op_code == MLX5_CQE_RESP_ERR) || | |
489 | (op_code == MLX5_CQE_REQ_ERR)) { | |
490 | volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe; | |
491 | uint8_t syndrome = err_cqe->syndrome; | |
492 | ||
493 | if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) || | |
494 | (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) | |
495 | return 0; | |
496 | if (!check_cqe_seen(cqe)) { | |
497 | DRV_LOG(ERR, | |
498 | "unexpected CQE error %u (0x%02x) syndrome" | |
499 | " 0x%02x", | |
500 | op_code, op_code, syndrome); | |
501 | rte_hexdump(stderr, "MLX5 Error CQE:", | |
502 | (const void *)((uintptr_t)err_cqe), | |
503 | sizeof(*cqe)); | |
504 | } | |
505 | return 1; | |
506 | } else if ((op_code != MLX5_CQE_RESP_SEND) && | |
507 | (op_code != MLX5_CQE_REQ)) { | |
508 | if (!check_cqe_seen(cqe)) { | |
509 | DRV_LOG(ERR, "unexpected CQE opcode %u (0x%02x)", | |
510 | op_code, op_code); | |
511 | rte_hexdump(stderr, "MLX5 CQE:", | |
512 | (const void *)((uintptr_t)cqe), | |
513 | sizeof(*cqe)); | |
514 | } | |
515 | return 1; | |
516 | } | |
517 | #endif /* NDEBUG */ | |
518 | return 0; | |
519 | } | |
520 | ||
521 | /** | |
522 | * Return the address of the WQE. | |
523 | * | |
524 | * @param txq | |
525 | * Pointer to TX queue structure. | |
526 | * @param wqe_ci | |
527 | * WQE consumer index. | |
528 | * | |
529 | * @return | |
530 | * WQE address. | |
531 | */ | |
532 | static inline uintptr_t * | |
533 | tx_mlx5_wqe(struct mlx5_txq_data *txq, uint16_t ci) | |
534 | { | |
535 | ci &= ((1 << txq->wqe_n) - 1); | |
536 | return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE); | |
537 | } | |
538 | ||
539 | /** | |
540 | * Manage TX completions. | |
541 | * | |
542 | * When sending a burst, mlx5_tx_burst() posts several WRs. | |
543 | * | |
544 | * @param txq | |
545 | * Pointer to TX queue structure. | |
546 | */ | |
547 | static __rte_always_inline void | |
548 | mlx5_tx_complete(struct mlx5_txq_data *txq) | |
549 | { | |
550 | const uint16_t elts_n = 1 << txq->elts_n; | |
551 | const uint16_t elts_m = elts_n - 1; | |
552 | const unsigned int cqe_n = 1 << txq->cqe_n; | |
553 | const unsigned int cqe_cnt = cqe_n - 1; | |
554 | uint16_t elts_free = txq->elts_tail; | |
555 | uint16_t elts_tail; | |
556 | uint16_t cq_ci = txq->cq_ci; | |
557 | volatile struct mlx5_cqe *cqe = NULL; | |
558 | volatile struct mlx5_wqe_ctrl *ctrl; | |
559 | struct rte_mbuf *m, *free[elts_n]; | |
560 | struct rte_mempool *pool = NULL; | |
561 | unsigned int blk_n = 0; | |
562 | ||
563 | cqe = &(*txq->cqes)[cq_ci & cqe_cnt]; | |
564 | if (unlikely(check_cqe(cqe, cqe_n, cq_ci))) | |
565 | return; | |
566 | #ifndef NDEBUG | |
567 | if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || | |
568 | (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { | |
569 | if (!check_cqe_seen(cqe)) { | |
570 | DRV_LOG(ERR, "unexpected error CQE, Tx stopped"); | |
571 | rte_hexdump(stderr, "MLX5 TXQ:", | |
572 | (const void *)((uintptr_t)txq->wqes), | |
573 | ((1 << txq->wqe_n) * | |
574 | MLX5_WQE_SIZE)); | |
575 | } | |
576 | return; | |
577 | } | |
578 | #endif /* NDEBUG */ | |
579 | ++cq_ci; | |
580 | rte_cio_rmb(); | |
581 | txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter); | |
582 | ctrl = (volatile struct mlx5_wqe_ctrl *) | |
583 | tx_mlx5_wqe(txq, txq->wqe_pi); | |
584 | elts_tail = ctrl->ctrl3; | |
585 | assert((elts_tail & elts_m) < (1 << txq->wqe_n)); | |
586 | /* Free buffers. */ | |
587 | while (elts_free != elts_tail) { | |
588 | m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]); | |
589 | if (likely(m != NULL)) { | |
590 | if (likely(m->pool == pool)) { | |
591 | free[blk_n++] = m; | |
592 | } else { | |
593 | if (likely(pool != NULL)) | |
594 | rte_mempool_put_bulk(pool, | |
595 | (void *)free, | |
596 | blk_n); | |
597 | free[0] = m; | |
598 | pool = m->pool; | |
599 | blk_n = 1; | |
600 | } | |
601 | } | |
602 | } | |
603 | if (blk_n) | |
604 | rte_mempool_put_bulk(pool, (void *)free, blk_n); | |
605 | #ifndef NDEBUG | |
606 | elts_free = txq->elts_tail; | |
607 | /* Poisoning. */ | |
608 | while (elts_free != elts_tail) { | |
609 | memset(&(*txq->elts)[elts_free & elts_m], | |
610 | 0x66, | |
611 | sizeof((*txq->elts)[elts_free & elts_m])); | |
612 | ++elts_free; | |
613 | } | |
614 | #endif | |
615 | txq->cq_ci = cq_ci; | |
616 | txq->elts_tail = elts_tail; | |
617 | /* Update the consumer index. */ | |
618 | rte_compiler_barrier(); | |
619 | *txq->cq_db = rte_cpu_to_be_32(cq_ci); | |
620 | } | |
621 | ||
622 | /** | |
623 | * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the | |
624 | * cloned mbuf is allocated is returned instead. | |
625 | * | |
626 | * @param buf | |
627 | * Pointer to mbuf. | |
628 | * | |
629 | * @return | |
630 | * Memory pool where data is located for given mbuf. | |
631 | */ | |
632 | static inline struct rte_mempool * | |
633 | mlx5_mb2mp(struct rte_mbuf *buf) | |
634 | { | |
635 | if (unlikely(RTE_MBUF_CLONED(buf))) | |
636 | return rte_mbuf_from_indirect(buf)->pool; | |
637 | return buf->pool; | |
638 | } | |
639 | ||
640 | /** | |
641 | * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx | |
642 | * as mempool is pre-configured and static. | |
643 | * | |
644 | * @param rxq | |
645 | * Pointer to Rx queue structure. | |
646 | * @param addr | |
647 | * Address to search. | |
648 | * | |
649 | * @return | |
650 | * Searched LKey on success, UINT32_MAX on no match. | |
651 | */ | |
652 | static __rte_always_inline uint32_t | |
653 | mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) | |
654 | { | |
655 | struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; | |
656 | uint32_t lkey; | |
657 | ||
658 | /* Linear search on MR cache array. */ | |
659 | lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru, | |
660 | MLX5_MR_CACHE_N, addr); | |
661 | if (likely(lkey != UINT32_MAX)) | |
662 | return lkey; | |
663 | /* Take slower bottom-half (Binary Search) on miss. */ | |
664 | return mlx5_rx_addr2mr_bh(rxq, addr); | |
665 | } | |
666 | ||
667 | #define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr)) | |
668 | ||
669 | /** | |
670 | * Query LKey from a packet buffer for Tx. If not found, add the mempool. | |
671 | * | |
672 | * @param txq | |
673 | * Pointer to Tx queue structure. | |
674 | * @param addr | |
675 | * Address to search. | |
676 | * | |
677 | * @return | |
678 | * Searched LKey on success, UINT32_MAX on no match. | |
679 | */ | |
680 | static __rte_always_inline uint32_t | |
681 | mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) | |
682 | { | |
683 | struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; | |
684 | uintptr_t addr = (uintptr_t)mb->buf_addr; | |
685 | uint32_t lkey; | |
686 | ||
687 | /* Check generation bit to see if there's any change on existing MRs. */ | |
688 | if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen)) | |
689 | mlx5_mr_flush_local_cache(mr_ctrl); | |
690 | /* Linear search on MR cache array. */ | |
691 | lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru, | |
692 | MLX5_MR_CACHE_N, addr); | |
693 | if (likely(lkey != UINT32_MAX)) | |
694 | return lkey; | |
695 | /* Take slower bottom-half on miss. */ | |
696 | return mlx5_tx_mb2mr_bh(txq, mb); | |
697 | } | |
698 | ||
699 | /** | |
700 | * Ring TX queue doorbell and flush the update if requested. | |
701 | * | |
702 | * @param txq | |
703 | * Pointer to TX queue structure. | |
704 | * @param wqe | |
705 | * Pointer to the last WQE posted in the NIC. | |
706 | * @param cond | |
707 | * Request for write memory barrier after BlueFlame update. | |
708 | */ | |
709 | static __rte_always_inline void | |
710 | mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe, | |
711 | int cond) | |
712 | { | |
713 | uint64_t *dst = MLX5_TX_BFREG(txq); | |
714 | volatile uint64_t *src = ((volatile uint64_t *)wqe); | |
715 | ||
716 | rte_cio_wmb(); | |
717 | *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci); | |
718 | /* Ensure ordering between DB record and BF copy. */ | |
719 | rte_wmb(); | |
720 | mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock); | |
721 | if (cond) | |
722 | rte_wmb(); | |
723 | } | |
724 | ||
725 | /** | |
726 | * Ring TX queue doorbell and flush the update by write memory barrier. | |
727 | * | |
728 | * @param txq | |
729 | * Pointer to TX queue structure. | |
730 | * @param wqe | |
731 | * Pointer to the last WQE posted in the NIC. | |
732 | */ | |
733 | static __rte_always_inline void | |
734 | mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe) | |
735 | { | |
736 | mlx5_tx_dbrec_cond_wmb(txq, wqe, 1); | |
737 | } | |
738 | ||
739 | /** | |
740 | * Convert mbuf to Verb SWP. | |
741 | * | |
742 | * @param txq_data | |
743 | * Pointer to the Tx queue. | |
744 | * @param buf | |
745 | * Pointer to the mbuf. | |
746 | * @param offsets | |
747 | * Pointer to the SWP header offsets. | |
748 | * @param swp_types | |
749 | * Pointer to the SWP header types. | |
750 | */ | |
751 | static __rte_always_inline void | |
752 | txq_mbuf_to_swp(struct mlx5_txq_data *txq, struct rte_mbuf *buf, | |
753 | uint8_t *offsets, uint8_t *swp_types) | |
754 | { | |
755 | const uint64_t vlan = buf->ol_flags & PKT_TX_VLAN_PKT; | |
756 | const uint64_t tunnel = buf->ol_flags & PKT_TX_TUNNEL_MASK; | |
757 | const uint64_t tso = buf->ol_flags & PKT_TX_TCP_SEG; | |
758 | const uint64_t csum_flags = buf->ol_flags & PKT_TX_L4_MASK; | |
759 | const uint64_t inner_ip = | |
760 | buf->ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6); | |
761 | const uint64_t ol_flags_mask = PKT_TX_L4_MASK | PKT_TX_IPV6 | | |
762 | PKT_TX_OUTER_IPV6; | |
763 | uint16_t idx; | |
764 | uint16_t off; | |
765 | ||
766 | if (likely(!txq->swp_en || (tunnel != PKT_TX_TUNNEL_UDP && | |
767 | tunnel != PKT_TX_TUNNEL_IP))) | |
768 | return; | |
769 | /* | |
770 | * The index should have: | |
771 | * bit[0:1] = PKT_TX_L4_MASK | |
772 | * bit[4] = PKT_TX_IPV6 | |
773 | * bit[8] = PKT_TX_OUTER_IPV6 | |
774 | * bit[9] = PKT_TX_OUTER_UDP | |
775 | */ | |
776 | idx = (buf->ol_flags & ol_flags_mask) >> 52; | |
777 | if (tunnel == PKT_TX_TUNNEL_UDP) | |
778 | idx |= 1 << 9; | |
779 | *swp_types = mlx5_swp_types_table[idx]; | |
780 | /* | |
781 | * Set offsets for SW parser. Since ConnectX-5, SW parser just | |
782 | * complements HW parser. SW parser starts to engage only if HW parser | |
783 | * can't reach a header. For the older devices, HW parser will not kick | |
784 | * in if any of SWP offsets is set. Therefore, all of the L3 offsets | |
785 | * should be set regardless of HW offload. | |
786 | */ | |
787 | off = buf->outer_l2_len + (vlan ? sizeof(struct vlan_hdr) : 0); | |
788 | offsets[1] = off >> 1; /* Outer L3 offset. */ | |
789 | off += buf->outer_l3_len; | |
790 | if (tunnel == PKT_TX_TUNNEL_UDP) | |
791 | offsets[0] = off >> 1; /* Outer L4 offset. */ | |
792 | if (inner_ip) { | |
793 | off += buf->l2_len; | |
794 | offsets[3] = off >> 1; /* Inner L3 offset. */ | |
795 | if (csum_flags == PKT_TX_TCP_CKSUM || tso || | |
796 | csum_flags == PKT_TX_UDP_CKSUM) { | |
797 | off += buf->l3_len; | |
798 | offsets[2] = off >> 1; /* Inner L4 offset. */ | |
799 | } | |
800 | } | |
801 | } | |
802 | ||
803 | /** | |
804 | * Convert the Checksum offloads to Verbs. | |
805 | * | |
806 | * @param buf | |
807 | * Pointer to the mbuf. | |
808 | * | |
809 | * @return | |
810 | * Converted checksum flags. | |
811 | */ | |
812 | static __rte_always_inline uint8_t | |
813 | txq_ol_cksum_to_cs(struct rte_mbuf *buf) | |
814 | { | |
815 | uint32_t idx; | |
816 | uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK); | |
817 | const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK | | |
818 | PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM; | |
819 | ||
820 | /* | |
821 | * The index should have: | |
822 | * bit[0] = PKT_TX_TCP_SEG | |
823 | * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM | |
824 | * bit[4] = PKT_TX_IP_CKSUM | |
825 | * bit[8] = PKT_TX_OUTER_IP_CKSUM | |
826 | * bit[9] = tunnel | |
827 | */ | |
828 | idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9); | |
829 | return mlx5_cksum_table[idx]; | |
830 | } | |
831 | ||
832 | /** | |
833 | * Count the number of contiguous single segment packets. | |
834 | * | |
835 | * @param pkts | |
836 | * Pointer to array of packets. | |
837 | * @param pkts_n | |
838 | * Number of packets. | |
839 | * | |
840 | * @return | |
841 | * Number of contiguous single segment packets. | |
842 | */ | |
843 | static __rte_always_inline unsigned int | |
844 | txq_count_contig_single_seg(struct rte_mbuf **pkts, uint16_t pkts_n) | |
845 | { | |
846 | unsigned int pos; | |
847 | ||
848 | if (!pkts_n) | |
849 | return 0; | |
850 | /* Count the number of contiguous single segment packets. */ | |
851 | for (pos = 0; pos < pkts_n; ++pos) | |
852 | if (NB_SEGS(pkts[pos]) > 1) | |
853 | break; | |
854 | return pos; | |
855 | } | |
856 | ||
857 | /** | |
858 | * Count the number of contiguous multi-segment packets. | |
859 | * | |
860 | * @param pkts | |
861 | * Pointer to array of packets. | |
862 | * @param pkts_n | |
863 | * Number of packets. | |
864 | * | |
865 | * @return | |
866 | * Number of contiguous multi-segment packets. | |
867 | */ | |
868 | static __rte_always_inline unsigned int | |
869 | txq_count_contig_multi_seg(struct rte_mbuf **pkts, uint16_t pkts_n) | |
870 | { | |
871 | unsigned int pos; | |
872 | ||
873 | if (!pkts_n) | |
874 | return 0; | |
875 | /* Count the number of contiguous multi-segment packets. */ | |
876 | for (pos = 0; pos < pkts_n; ++pos) | |
877 | if (NB_SEGS(pkts[pos]) == 1) | |
878 | break; | |
879 | return pos; | |
880 | } | |
7c673cae FG |
881 | |
882 | #endif /* RTE_PMD_MLX5_RXTX_H_ */ |