]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright 2015 6WIND S.A. | |
3 | * Copyright 2015 Mellanox Technologies, Ltd | |
4 | */ | |
5 | ||
6 | #include <stddef.h> | |
7 | #include <assert.h> | |
8 | #include <errno.h> | |
9 | #include <string.h> | |
10 | #include <stdint.h> | |
11 | #include <unistd.h> | |
12 | #include <sys/mman.h> | |
13 | ||
14 | /* Verbs header. */ | |
15 | /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ | |
16 | #ifdef PEDANTIC | |
17 | #pragma GCC diagnostic ignored "-Wpedantic" | |
18 | #endif | |
19 | #include <infiniband/verbs.h> | |
20 | #ifdef PEDANTIC | |
21 | #pragma GCC diagnostic error "-Wpedantic" | |
22 | #endif | |
23 | ||
24 | #include <rte_mbuf.h> | |
25 | #include <rte_malloc.h> | |
26 | #include <rte_ethdev_driver.h> | |
27 | #include <rte_common.h> | |
28 | ||
29 | #include "mlx5_utils.h" | |
30 | #include "mlx5_defs.h" | |
31 | #include "mlx5.h" | |
32 | #include "mlx5_rxtx.h" | |
33 | #include "mlx5_autoconf.h" | |
34 | #include "mlx5_glue.h" | |
35 | ||
36 | /** | |
37 | * Allocate TX queue elements. | |
38 | * | |
39 | * @param txq_ctrl | |
40 | * Pointer to TX queue structure. | |
41 | */ | |
42 | void | |
43 | txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl) | |
44 | { | |
45 | const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n; | |
46 | unsigned int i; | |
47 | ||
48 | for (i = 0; (i != elts_n); ++i) | |
49 | (*txq_ctrl->txq.elts)[i] = NULL; | |
50 | DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs", | |
9f95a23c | 51 | PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx, elts_n); |
11fdf7f2 TL |
52 | txq_ctrl->txq.elts_head = 0; |
53 | txq_ctrl->txq.elts_tail = 0; | |
54 | txq_ctrl->txq.elts_comp = 0; | |
55 | } | |
56 | ||
57 | /** | |
58 | * Free TX queue elements. | |
59 | * | |
60 | * @param txq_ctrl | |
61 | * Pointer to TX queue structure. | |
62 | */ | |
63 | static void | |
64 | txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) | |
65 | { | |
66 | const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n; | |
67 | const uint16_t elts_m = elts_n - 1; | |
68 | uint16_t elts_head = txq_ctrl->txq.elts_head; | |
69 | uint16_t elts_tail = txq_ctrl->txq.elts_tail; | |
70 | struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; | |
71 | ||
72 | DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs", | |
9f95a23c | 73 | PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx); |
11fdf7f2 TL |
74 | txq_ctrl->txq.elts_head = 0; |
75 | txq_ctrl->txq.elts_tail = 0; | |
76 | txq_ctrl->txq.elts_comp = 0; | |
77 | ||
78 | while (elts_tail != elts_head) { | |
79 | struct rte_mbuf *elt = (*elts)[elts_tail & elts_m]; | |
80 | ||
81 | assert(elt != NULL); | |
82 | rte_pktmbuf_free_seg(elt); | |
83 | #ifndef NDEBUG | |
84 | /* Poisoning. */ | |
85 | memset(&(*elts)[elts_tail & elts_m], | |
86 | 0x77, | |
87 | sizeof((*elts)[elts_tail & elts_m])); | |
88 | #endif | |
89 | ++elts_tail; | |
90 | } | |
91 | } | |
92 | ||
93 | /** | |
94 | * Returns the per-port supported offloads. | |
95 | * | |
96 | * @param dev | |
97 | * Pointer to Ethernet device. | |
98 | * | |
99 | * @return | |
100 | * Supported Tx offloads. | |
101 | */ | |
102 | uint64_t | |
103 | mlx5_get_tx_port_offloads(struct rte_eth_dev *dev) | |
104 | { | |
9f95a23c | 105 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
106 | uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS | |
107 | DEV_TX_OFFLOAD_VLAN_INSERT); | |
108 | struct mlx5_dev_config *config = &priv->config; | |
109 | ||
110 | if (config->hw_csum) | |
111 | offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM | | |
112 | DEV_TX_OFFLOAD_UDP_CKSUM | | |
113 | DEV_TX_OFFLOAD_TCP_CKSUM); | |
114 | if (config->tso) | |
115 | offloads |= DEV_TX_OFFLOAD_TCP_TSO; | |
116 | if (config->swp) { | |
117 | if (config->hw_csum) | |
118 | offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; | |
119 | if (config->tso) | |
120 | offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO | | |
121 | DEV_TX_OFFLOAD_UDP_TNL_TSO); | |
122 | } | |
11fdf7f2 TL |
123 | if (config->tunnel_en) { |
124 | if (config->hw_csum) | |
125 | offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; | |
126 | if (config->tso) | |
127 | offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | | |
128 | DEV_TX_OFFLOAD_GRE_TNL_TSO); | |
129 | } | |
9f95a23c TL |
130 | #ifdef HAVE_IBV_FLOW_DV_SUPPORT |
131 | if (config->dv_flow_en) | |
132 | offloads |= DEV_TX_OFFLOAD_MATCH_METADATA; | |
133 | #endif | |
11fdf7f2 TL |
134 | return offloads; |
135 | } | |
136 | ||
137 | /** | |
138 | * DPDK callback to configure a TX queue. | |
139 | * | |
140 | * @param dev | |
141 | * Pointer to Ethernet device structure. | |
142 | * @param idx | |
143 | * TX queue index. | |
144 | * @param desc | |
145 | * Number of descriptors to configure in queue. | |
146 | * @param socket | |
147 | * NUMA socket on which memory must be allocated. | |
148 | * @param[in] conf | |
149 | * Thresholds parameters. | |
150 | * | |
151 | * @return | |
152 | * 0 on success, a negative errno value otherwise and rte_errno is set. | |
153 | */ | |
154 | int | |
155 | mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, | |
156 | unsigned int socket, const struct rte_eth_txconf *conf) | |
157 | { | |
9f95a23c | 158 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
159 | struct mlx5_txq_data *txq = (*priv->txqs)[idx]; |
160 | struct mlx5_txq_ctrl *txq_ctrl = | |
161 | container_of(txq, struct mlx5_txq_ctrl, txq); | |
162 | ||
163 | if (desc <= MLX5_TX_COMP_THRESH) { | |
164 | DRV_LOG(WARNING, | |
165 | "port %u number of descriptors requested for Tx queue" | |
166 | " %u must be higher than MLX5_TX_COMP_THRESH, using %u" | |
167 | " instead of %u", | |
168 | dev->data->port_id, idx, MLX5_TX_COMP_THRESH + 1, desc); | |
169 | desc = MLX5_TX_COMP_THRESH + 1; | |
170 | } | |
171 | if (!rte_is_power_of_2(desc)) { | |
172 | desc = 1 << log2above(desc); | |
173 | DRV_LOG(WARNING, | |
174 | "port %u increased number of descriptors in Tx queue" | |
175 | " %u to the next power of two (%d)", | |
176 | dev->data->port_id, idx, desc); | |
177 | } | |
178 | DRV_LOG(DEBUG, "port %u configuring queue %u for %u descriptors", | |
179 | dev->data->port_id, idx, desc); | |
180 | if (idx >= priv->txqs_n) { | |
181 | DRV_LOG(ERR, "port %u Tx queue index out of range (%u >= %u)", | |
182 | dev->data->port_id, idx, priv->txqs_n); | |
183 | rte_errno = EOVERFLOW; | |
184 | return -rte_errno; | |
185 | } | |
186 | if (!mlx5_txq_releasable(dev, idx)) { | |
187 | rte_errno = EBUSY; | |
188 | DRV_LOG(ERR, "port %u unable to release queue index %u", | |
189 | dev->data->port_id, idx); | |
190 | return -rte_errno; | |
191 | } | |
192 | mlx5_txq_release(dev, idx); | |
193 | txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf); | |
194 | if (!txq_ctrl) { | |
195 | DRV_LOG(ERR, "port %u unable to allocate queue index %u", | |
196 | dev->data->port_id, idx); | |
197 | return -rte_errno; | |
198 | } | |
199 | DRV_LOG(DEBUG, "port %u adding Tx queue %u to list", | |
200 | dev->data->port_id, idx); | |
201 | (*priv->txqs)[idx] = &txq_ctrl->txq; | |
202 | return 0; | |
203 | } | |
204 | ||
205 | /** | |
206 | * DPDK callback to release a TX queue. | |
207 | * | |
208 | * @param dpdk_txq | |
209 | * Generic TX queue pointer. | |
210 | */ | |
211 | void | |
212 | mlx5_tx_queue_release(void *dpdk_txq) | |
213 | { | |
214 | struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; | |
215 | struct mlx5_txq_ctrl *txq_ctrl; | |
9f95a23c | 216 | struct mlx5_priv *priv; |
11fdf7f2 TL |
217 | unsigned int i; |
218 | ||
219 | if (txq == NULL) | |
220 | return; | |
221 | txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); | |
222 | priv = txq_ctrl->priv; | |
223 | for (i = 0; (i != priv->txqs_n); ++i) | |
224 | if ((*priv->txqs)[i] == txq) { | |
225 | mlx5_txq_release(ETH_DEV(priv), i); | |
226 | DRV_LOG(DEBUG, "port %u removing Tx queue %u from list", | |
9f95a23c | 227 | PORT_ID(priv), txq->idx); |
11fdf7f2 TL |
228 | break; |
229 | } | |
230 | } | |
231 | ||
9f95a23c TL |
232 | /** |
233 | * Initialize Tx UAR registers for primary process. | |
234 | * | |
235 | * @param txq_ctrl | |
236 | * Pointer to Tx queue control structure. | |
237 | */ | |
238 | static void | |
239 | txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl) | |
240 | { | |
241 | struct mlx5_priv *priv = txq_ctrl->priv; | |
242 | struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); | |
243 | #ifndef RTE_ARCH_64 | |
244 | unsigned int lock_idx; | |
245 | const size_t page_size = sysconf(_SC_PAGESIZE); | |
246 | #endif | |
247 | ||
248 | assert(rte_eal_process_type() == RTE_PROC_PRIMARY); | |
249 | assert(ppriv); | |
250 | ppriv->uar_table[txq_ctrl->txq.idx] = txq_ctrl->bf_reg; | |
251 | #ifndef RTE_ARCH_64 | |
252 | /* Assign an UAR lock according to UAR page number */ | |
253 | lock_idx = (txq_ctrl->uar_mmap_offset / page_size) & | |
254 | MLX5_UAR_PAGE_NUM_MASK; | |
255 | txq_ctrl->txq.uar_lock = &priv->uar_lock[lock_idx]; | |
256 | #endif | |
257 | } | |
11fdf7f2 TL |
258 | |
259 | /** | |
9f95a23c | 260 | * Remap UAR register of a Tx queue for secondary process. |
11fdf7f2 | 261 | * |
9f95a23c TL |
262 | * Remapped address is stored at the table in the process private structure of |
263 | * the device, indexed by queue index. | |
264 | * | |
265 | * @param txq_ctrl | |
266 | * Pointer to Tx queue control structure. | |
11fdf7f2 TL |
267 | * @param fd |
268 | * Verbs file descriptor to map UAR pages. | |
269 | * | |
270 | * @return | |
271 | * 0 on success, a negative errno value otherwise and rte_errno is set. | |
272 | */ | |
9f95a23c TL |
273 | static int |
274 | txq_uar_init_secondary(struct mlx5_txq_ctrl *txq_ctrl, int fd) | |
11fdf7f2 | 275 | { |
9f95a23c TL |
276 | struct mlx5_priv *priv = txq_ctrl->priv; |
277 | struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); | |
278 | struct mlx5_txq_data *txq = &txq_ctrl->txq; | |
279 | void *addr; | |
11fdf7f2 | 280 | uintptr_t uar_va; |
9f95a23c TL |
281 | uintptr_t offset; |
282 | const size_t page_size = sysconf(_SC_PAGESIZE); | |
283 | ||
284 | assert(ppriv); | |
285 | /* | |
286 | * As rdma-core, UARs are mapped in size of OS page | |
287 | * size. Ref to libmlx5 function: mlx5_init_context() | |
288 | */ | |
289 | uar_va = (uintptr_t)txq_ctrl->bf_reg; | |
290 | offset = uar_va & (page_size - 1); /* Offset in page. */ | |
291 | addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, | |
292 | txq_ctrl->uar_mmap_offset); | |
293 | if (addr == MAP_FAILED) { | |
294 | DRV_LOG(ERR, | |
295 | "port %u mmap failed for BF reg of txq %u", | |
296 | txq->port_id, txq->idx); | |
297 | rte_errno = ENXIO; | |
298 | return -rte_errno; | |
299 | } | |
300 | addr = RTE_PTR_ADD(addr, offset); | |
301 | ppriv->uar_table[txq->idx] = addr; | |
302 | return 0; | |
303 | } | |
304 | ||
305 | /** | |
306 | * Unmap UAR register of a Tx queue for secondary process. | |
307 | * | |
308 | * @param txq_ctrl | |
309 | * Pointer to Tx queue control structure. | |
310 | */ | |
311 | static void | |
312 | txq_uar_uninit_secondary(struct mlx5_txq_ctrl *txq_ctrl) | |
313 | { | |
314 | struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(txq_ctrl->priv)); | |
315 | const size_t page_size = sysconf(_SC_PAGESIZE); | |
11fdf7f2 | 316 | void *addr; |
9f95a23c TL |
317 | |
318 | addr = ppriv->uar_table[txq_ctrl->txq.idx]; | |
319 | munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size); | |
320 | } | |
321 | ||
322 | /** | |
323 | * Initialize Tx UAR registers for secondary process. | |
324 | * | |
325 | * @param dev | |
326 | * Pointer to Ethernet device. | |
327 | * @param fd | |
328 | * Verbs file descriptor to map UAR pages. | |
329 | * | |
330 | * @return | |
331 | * 0 on success, a negative errno value otherwise and rte_errno is set. | |
332 | */ | |
333 | int | |
334 | mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd) | |
335 | { | |
336 | struct mlx5_priv *priv = dev->data->dev_private; | |
11fdf7f2 TL |
337 | struct mlx5_txq_data *txq; |
338 | struct mlx5_txq_ctrl *txq_ctrl; | |
9f95a23c TL |
339 | unsigned int i; |
340 | int ret; | |
11fdf7f2 | 341 | |
9f95a23c | 342 | assert(rte_eal_process_type() == RTE_PROC_SECONDARY); |
11fdf7f2 TL |
343 | for (i = 0; i != priv->txqs_n; ++i) { |
344 | if (!(*priv->txqs)[i]) | |
345 | continue; | |
346 | txq = (*priv->txqs)[i]; | |
347 | txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); | |
9f95a23c TL |
348 | assert(txq->idx == (uint16_t)i); |
349 | ret = txq_uar_init_secondary(txq_ctrl, fd); | |
350 | if (ret) | |
351 | goto error; | |
11fdf7f2 TL |
352 | } |
353 | return 0; | |
9f95a23c TL |
354 | error: |
355 | /* Rollback. */ | |
356 | do { | |
357 | if (!(*priv->txqs)[i]) | |
358 | continue; | |
359 | txq = (*priv->txqs)[i]; | |
360 | txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); | |
361 | txq_uar_uninit_secondary(txq_ctrl); | |
362 | } while (i--); | |
363 | return -rte_errno; | |
11fdf7f2 TL |
364 | } |
365 | ||
366 | /** | |
367 | * Check if the burst function is using eMPW. | |
368 | * | |
369 | * @param tx_pkt_burst | |
370 | * Tx burst function pointer. | |
371 | * | |
372 | * @return | |
373 | * 1 if the burst function is using eMPW, 0 otherwise. | |
374 | */ | |
375 | static int | |
376 | is_empw_burst_func(eth_tx_burst_t tx_pkt_burst) | |
377 | { | |
378 | if (tx_pkt_burst == mlx5_tx_burst_raw_vec || | |
379 | tx_pkt_burst == mlx5_tx_burst_vec || | |
380 | tx_pkt_burst == mlx5_tx_burst_empw) | |
381 | return 1; | |
382 | return 0; | |
383 | } | |
384 | ||
385 | /** | |
386 | * Create the Tx queue Verbs object. | |
387 | * | |
388 | * @param dev | |
389 | * Pointer to Ethernet device. | |
390 | * @param idx | |
9f95a23c | 391 | * Queue index in DPDK Tx queue array. |
11fdf7f2 TL |
392 | * |
393 | * @return | |
394 | * The Verbs object initialised, NULL otherwise and rte_errno is set. | |
395 | */ | |
396 | struct mlx5_txq_ibv * | |
397 | mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) | |
398 | { | |
9f95a23c | 399 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
400 | struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; |
401 | struct mlx5_txq_ctrl *txq_ctrl = | |
402 | container_of(txq_data, struct mlx5_txq_ctrl, txq); | |
403 | struct mlx5_txq_ibv tmpl; | |
404 | struct mlx5_txq_ibv *txq_ibv; | |
405 | union { | |
406 | struct ibv_qp_init_attr_ex init; | |
407 | struct ibv_cq_init_attr_ex cq; | |
408 | struct ibv_qp_attr mod; | |
409 | struct ibv_cq_ex cq_attr; | |
410 | } attr; | |
411 | unsigned int cqe_n; | |
412 | struct mlx5dv_qp qp = { .comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET }; | |
413 | struct mlx5dv_cq cq_info; | |
414 | struct mlx5dv_obj obj; | |
415 | const int desc = 1 << txq_data->elts_n; | |
416 | eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev); | |
417 | int ret = 0; | |
418 | ||
419 | assert(txq_data); | |
420 | priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_TX_QUEUE; | |
421 | priv->verbs_alloc_ctx.obj = txq_ctrl; | |
422 | if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { | |
423 | DRV_LOG(ERR, | |
424 | "port %u MLX5_ENABLE_CQE_COMPRESSION must never be set", | |
425 | dev->data->port_id); | |
426 | rte_errno = EINVAL; | |
427 | return NULL; | |
428 | } | |
429 | memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv)); | |
430 | attr.cq = (struct ibv_cq_init_attr_ex){ | |
431 | .comp_mask = 0, | |
432 | }; | |
433 | cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ? | |
434 | ((desc / MLX5_TX_COMP_THRESH) - 1) : 1; | |
435 | if (is_empw_burst_func(tx_pkt_burst)) | |
436 | cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; | |
9f95a23c | 437 | tmpl.cq = mlx5_glue->create_cq(priv->sh->ctx, cqe_n, NULL, NULL, 0); |
11fdf7f2 TL |
438 | if (tmpl.cq == NULL) { |
439 | DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure", | |
440 | dev->data->port_id, idx); | |
441 | rte_errno = errno; | |
442 | goto error; | |
443 | } | |
444 | attr.init = (struct ibv_qp_init_attr_ex){ | |
445 | /* CQ to be associated with the send queue. */ | |
446 | .send_cq = tmpl.cq, | |
447 | /* CQ to be associated with the receive queue. */ | |
448 | .recv_cq = tmpl.cq, | |
449 | .cap = { | |
450 | /* Max number of outstanding WRs. */ | |
451 | .max_send_wr = | |
9f95a23c | 452 | ((priv->sh->device_attr.orig_attr.max_qp_wr < |
11fdf7f2 | 453 | desc) ? |
9f95a23c | 454 | priv->sh->device_attr.orig_attr.max_qp_wr : |
11fdf7f2 TL |
455 | desc), |
456 | /* | |
457 | * Max number of scatter/gather elements in a WR, | |
458 | * must be 1 to prevent libmlx5 from trying to affect | |
459 | * too much memory. TX gather is not impacted by the | |
9f95a23c | 460 | * device_attr.max_sge limit and will still work |
11fdf7f2 TL |
461 | * properly. |
462 | */ | |
463 | .max_send_sge = 1, | |
464 | }, | |
465 | .qp_type = IBV_QPT_RAW_PACKET, | |
466 | /* | |
467 | * Do *NOT* enable this, completions events are managed per | |
468 | * Tx burst. | |
469 | */ | |
470 | .sq_sig_all = 0, | |
9f95a23c | 471 | .pd = priv->sh->pd, |
11fdf7f2 TL |
472 | .comp_mask = IBV_QP_INIT_ATTR_PD, |
473 | }; | |
474 | if (txq_data->max_inline) | |
475 | attr.init.cap.max_inline_data = txq_ctrl->max_inline_data; | |
476 | if (txq_data->tso_en) { | |
477 | attr.init.max_tso_header = txq_ctrl->max_tso_header; | |
478 | attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER; | |
479 | } | |
9f95a23c | 480 | tmpl.qp = mlx5_glue->create_qp_ex(priv->sh->ctx, &attr.init); |
11fdf7f2 TL |
481 | if (tmpl.qp == NULL) { |
482 | DRV_LOG(ERR, "port %u Tx queue %u QP creation failure", | |
483 | dev->data->port_id, idx); | |
484 | rte_errno = errno; | |
485 | goto error; | |
486 | } | |
487 | attr.mod = (struct ibv_qp_attr){ | |
488 | /* Move the QP to this state. */ | |
489 | .qp_state = IBV_QPS_INIT, | |
9f95a23c TL |
490 | /* IB device port number. */ |
491 | .port_num = (uint8_t)priv->ibv_port, | |
11fdf7f2 TL |
492 | }; |
493 | ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, | |
494 | (IBV_QP_STATE | IBV_QP_PORT)); | |
495 | if (ret) { | |
496 | DRV_LOG(ERR, | |
497 | "port %u Tx queue %u QP state to IBV_QPS_INIT failed", | |
498 | dev->data->port_id, idx); | |
499 | rte_errno = errno; | |
500 | goto error; | |
501 | } | |
502 | attr.mod = (struct ibv_qp_attr){ | |
503 | .qp_state = IBV_QPS_RTR | |
504 | }; | |
505 | ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); | |
506 | if (ret) { | |
507 | DRV_LOG(ERR, | |
508 | "port %u Tx queue %u QP state to IBV_QPS_RTR failed", | |
509 | dev->data->port_id, idx); | |
510 | rte_errno = errno; | |
511 | goto error; | |
512 | } | |
513 | attr.mod.qp_state = IBV_QPS_RTS; | |
514 | ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); | |
515 | if (ret) { | |
516 | DRV_LOG(ERR, | |
517 | "port %u Tx queue %u QP state to IBV_QPS_RTS failed", | |
518 | dev->data->port_id, idx); | |
519 | rte_errno = errno; | |
520 | goto error; | |
521 | } | |
522 | txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0, | |
523 | txq_ctrl->socket); | |
524 | if (!txq_ibv) { | |
525 | DRV_LOG(ERR, "port %u Tx queue %u cannot allocate memory", | |
526 | dev->data->port_id, idx); | |
527 | rte_errno = ENOMEM; | |
528 | goto error; | |
529 | } | |
530 | obj.cq.in = tmpl.cq; | |
531 | obj.cq.out = &cq_info; | |
532 | obj.qp.in = tmpl.qp; | |
533 | obj.qp.out = &qp; | |
534 | ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP); | |
535 | if (ret != 0) { | |
536 | rte_errno = errno; | |
537 | goto error; | |
538 | } | |
539 | if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { | |
540 | DRV_LOG(ERR, | |
541 | "port %u wrong MLX5_CQE_SIZE environment variable" | |
542 | " value: it should be set to %u", | |
543 | dev->data->port_id, RTE_CACHE_LINE_SIZE); | |
544 | rte_errno = EINVAL; | |
545 | goto error; | |
546 | } | |
547 | txq_data->cqe_n = log2above(cq_info.cqe_cnt); | |
548 | txq_data->qp_num_8s = tmpl.qp->qp_num << 8; | |
549 | txq_data->wqes = qp.sq.buf; | |
550 | txq_data->wqe_n = log2above(qp.sq.wqe_cnt); | |
551 | txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR]; | |
11fdf7f2 TL |
552 | txq_data->cq_db = cq_info.dbrec; |
553 | txq_data->cqes = | |
554 | (volatile struct mlx5_cqe (*)[]) | |
555 | (uintptr_t)cq_info.buf; | |
556 | txq_data->cq_ci = 0; | |
557 | #ifndef NDEBUG | |
558 | txq_data->cq_pi = 0; | |
559 | #endif | |
560 | txq_data->wqe_ci = 0; | |
561 | txq_data->wqe_pi = 0; | |
562 | txq_ibv->qp = tmpl.qp; | |
563 | txq_ibv->cq = tmpl.cq; | |
564 | rte_atomic32_inc(&txq_ibv->refcnt); | |
9f95a23c TL |
565 | txq_ctrl->bf_reg = qp.bf.reg; |
566 | txq_uar_init(txq_ctrl); | |
11fdf7f2 TL |
567 | if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) { |
568 | txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset; | |
569 | DRV_LOG(DEBUG, "port %u: uar_mmap_offset 0x%lx", | |
570 | dev->data->port_id, txq_ctrl->uar_mmap_offset); | |
571 | } else { | |
572 | DRV_LOG(ERR, | |
573 | "port %u failed to retrieve UAR info, invalid" | |
574 | " libmlx5.so", | |
575 | dev->data->port_id); | |
576 | rte_errno = EINVAL; | |
577 | goto error; | |
578 | } | |
579 | LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); | |
580 | txq_ibv->txq_ctrl = txq_ctrl; | |
581 | priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; | |
582 | return txq_ibv; | |
583 | error: | |
584 | ret = rte_errno; /* Save rte_errno before cleanup. */ | |
585 | if (tmpl.cq) | |
586 | claim_zero(mlx5_glue->destroy_cq(tmpl.cq)); | |
587 | if (tmpl.qp) | |
588 | claim_zero(mlx5_glue->destroy_qp(tmpl.qp)); | |
589 | priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; | |
590 | rte_errno = ret; /* Restore rte_errno. */ | |
591 | return NULL; | |
592 | } | |
593 | ||
594 | /** | |
595 | * Get an Tx queue Verbs object. | |
596 | * | |
597 | * @param dev | |
598 | * Pointer to Ethernet device. | |
599 | * @param idx | |
9f95a23c | 600 | * Queue index in DPDK Tx queue array. |
11fdf7f2 TL |
601 | * |
602 | * @return | |
603 | * The Verbs object if it exists. | |
604 | */ | |
605 | struct mlx5_txq_ibv * | |
606 | mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) | |
607 | { | |
9f95a23c | 608 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
609 | struct mlx5_txq_ctrl *txq_ctrl; |
610 | ||
611 | if (idx >= priv->txqs_n) | |
612 | return NULL; | |
613 | if (!(*priv->txqs)[idx]) | |
614 | return NULL; | |
615 | txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); | |
616 | if (txq_ctrl->ibv) | |
617 | rte_atomic32_inc(&txq_ctrl->ibv->refcnt); | |
618 | return txq_ctrl->ibv; | |
619 | } | |
620 | ||
621 | /** | |
622 | * Release an Tx verbs queue object. | |
623 | * | |
624 | * @param txq_ibv | |
625 | * Verbs Tx queue object. | |
626 | * | |
627 | * @return | |
628 | * 1 while a reference on it exists, 0 when freed. | |
629 | */ | |
630 | int | |
631 | mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv) | |
632 | { | |
633 | assert(txq_ibv); | |
634 | if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { | |
635 | claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp)); | |
636 | claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq)); | |
637 | LIST_REMOVE(txq_ibv, next); | |
638 | rte_free(txq_ibv); | |
639 | return 0; | |
640 | } | |
641 | return 1; | |
642 | } | |
643 | ||
644 | /** | |
645 | * Return true if a single reference exists on the object. | |
646 | * | |
647 | * @param txq_ibv | |
648 | * Verbs Tx queue object. | |
649 | */ | |
650 | int | |
651 | mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv) | |
652 | { | |
653 | assert(txq_ibv); | |
654 | return (rte_atomic32_read(&txq_ibv->refcnt) == 1); | |
655 | } | |
656 | ||
657 | /** | |
658 | * Verify the Verbs Tx queue list is empty | |
659 | * | |
660 | * @param dev | |
661 | * Pointer to Ethernet device. | |
662 | * | |
663 | * @return | |
664 | * The number of object not released. | |
665 | */ | |
666 | int | |
667 | mlx5_txq_ibv_verify(struct rte_eth_dev *dev) | |
668 | { | |
9f95a23c | 669 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
670 | int ret = 0; |
671 | struct mlx5_txq_ibv *txq_ibv; | |
672 | ||
673 | LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { | |
674 | DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced", | |
9f95a23c | 675 | dev->data->port_id, txq_ibv->txq_ctrl->txq.idx); |
11fdf7f2 TL |
676 | ++ret; |
677 | } | |
678 | return ret; | |
679 | } | |
680 | ||
9f95a23c TL |
681 | /** |
682 | * Calcuate the total number of WQEBB for Tx queue. | |
683 | * | |
684 | * Simplified version of calc_sq_size() in rdma-core. | |
685 | * | |
686 | * @param txq_ctrl | |
687 | * Pointer to Tx queue control structure. | |
688 | * | |
689 | * @return | |
690 | * The number of WQEBB. | |
691 | */ | |
692 | static int | |
693 | txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl) | |
694 | { | |
695 | unsigned int wqe_size; | |
696 | const unsigned int desc = 1 << txq_ctrl->txq.elts_n; | |
697 | ||
698 | wqe_size = MLX5_WQE_SIZE + txq_ctrl->max_inline_data; | |
699 | return rte_align32pow2(wqe_size * desc) / MLX5_WQE_SIZE; | |
700 | } | |
701 | ||
11fdf7f2 TL |
702 | /** |
703 | * Set Tx queue parameters from device configuration. | |
704 | * | |
705 | * @param txq_ctrl | |
706 | * Pointer to Tx queue control structure. | |
707 | */ | |
708 | static void | |
709 | txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) | |
710 | { | |
9f95a23c | 711 | struct mlx5_priv *priv = txq_ctrl->priv; |
11fdf7f2 TL |
712 | struct mlx5_dev_config *config = &priv->config; |
713 | const unsigned int max_tso_inline = | |
714 | ((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) / | |
715 | RTE_CACHE_LINE_SIZE); | |
716 | unsigned int txq_inline; | |
717 | unsigned int txqs_inline; | |
718 | unsigned int inline_max_packet_sz; | |
719 | eth_tx_burst_t tx_pkt_burst = | |
720 | mlx5_select_tx_function(ETH_DEV(priv)); | |
721 | int is_empw_func = is_empw_burst_func(tx_pkt_burst); | |
722 | int tso = !!(txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO | | |
723 | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | | |
724 | DEV_TX_OFFLOAD_GRE_TNL_TSO | | |
725 | DEV_TX_OFFLOAD_IP_TNL_TSO | | |
726 | DEV_TX_OFFLOAD_UDP_TNL_TSO)); | |
727 | ||
728 | txq_inline = (config->txq_inline == MLX5_ARG_UNSET) ? | |
729 | 0 : config->txq_inline; | |
730 | txqs_inline = (config->txqs_inline == MLX5_ARG_UNSET) ? | |
731 | 0 : config->txqs_inline; | |
732 | inline_max_packet_sz = | |
733 | (config->inline_max_packet_sz == MLX5_ARG_UNSET) ? | |
734 | 0 : config->inline_max_packet_sz; | |
735 | if (is_empw_func) { | |
736 | if (config->txq_inline == MLX5_ARG_UNSET) | |
737 | txq_inline = MLX5_WQE_SIZE_MAX - MLX5_WQE_SIZE; | |
738 | if (config->txqs_inline == MLX5_ARG_UNSET) | |
739 | txqs_inline = MLX5_EMPW_MIN_TXQS; | |
740 | if (config->inline_max_packet_sz == MLX5_ARG_UNSET) | |
741 | inline_max_packet_sz = MLX5_EMPW_MAX_INLINE_LEN; | |
742 | txq_ctrl->txq.mpw_hdr_dseg = config->mpw_hdr_dseg; | |
743 | txq_ctrl->txq.inline_max_packet_sz = inline_max_packet_sz; | |
744 | } | |
745 | if (txq_inline && priv->txqs_n >= txqs_inline) { | |
746 | unsigned int ds_cnt; | |
747 | ||
748 | txq_ctrl->txq.max_inline = | |
749 | ((txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / | |
750 | RTE_CACHE_LINE_SIZE); | |
751 | if (is_empw_func) { | |
752 | /* To minimize the size of data set, avoid requesting | |
753 | * too large WQ. | |
754 | */ | |
755 | txq_ctrl->max_inline_data = | |
756 | ((RTE_MIN(txq_inline, | |
757 | inline_max_packet_sz) + | |
758 | (RTE_CACHE_LINE_SIZE - 1)) / | |
759 | RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE; | |
760 | } else { | |
761 | txq_ctrl->max_inline_data = | |
762 | txq_ctrl->txq.max_inline * RTE_CACHE_LINE_SIZE; | |
763 | } | |
764 | /* | |
765 | * Check if the inline size is too large in a way which | |
766 | * can make the WQE DS to overflow. | |
767 | * Considering in calculation: | |
768 | * WQE CTRL (1 DS) | |
769 | * WQE ETH (1 DS) | |
770 | * Inline part (N DS) | |
771 | */ | |
772 | ds_cnt = 2 + (txq_ctrl->txq.max_inline / MLX5_WQE_DWORD_SIZE); | |
773 | if (ds_cnt > MLX5_DSEG_MAX) { | |
774 | unsigned int max_inline = (MLX5_DSEG_MAX - 2) * | |
775 | MLX5_WQE_DWORD_SIZE; | |
776 | ||
777 | max_inline = max_inline - (max_inline % | |
778 | RTE_CACHE_LINE_SIZE); | |
779 | DRV_LOG(WARNING, | |
780 | "port %u txq inline is too large (%d) setting" | |
781 | " it to the maximum possible: %d\n", | |
782 | PORT_ID(priv), txq_inline, max_inline); | |
783 | txq_ctrl->txq.max_inline = max_inline / | |
784 | RTE_CACHE_LINE_SIZE; | |
785 | } | |
786 | } | |
787 | if (tso) { | |
788 | txq_ctrl->max_tso_header = max_tso_inline * RTE_CACHE_LINE_SIZE; | |
789 | txq_ctrl->txq.max_inline = RTE_MAX(txq_ctrl->txq.max_inline, | |
790 | max_tso_inline); | |
791 | txq_ctrl->txq.tso_en = 1; | |
792 | } | |
793 | txq_ctrl->txq.tunnel_en = config->tunnel_en | config->swp; | |
794 | txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO | | |
795 | DEV_TX_OFFLOAD_UDP_TNL_TSO | | |
796 | DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) & | |
797 | txq_ctrl->txq.offloads) && config->swp; | |
798 | } | |
799 | ||
800 | /** | |
801 | * Create a DPDK Tx queue. | |
802 | * | |
803 | * @param dev | |
804 | * Pointer to Ethernet device. | |
805 | * @param idx | |
806 | * TX queue index. | |
807 | * @param desc | |
808 | * Number of descriptors to configure in queue. | |
809 | * @param socket | |
810 | * NUMA socket on which memory must be allocated. | |
811 | * @param[in] conf | |
812 | * Thresholds parameters. | |
813 | * | |
814 | * @return | |
815 | * A DPDK queue object on success, NULL otherwise and rte_errno is set. | |
816 | */ | |
817 | struct mlx5_txq_ctrl * | |
818 | mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, | |
819 | unsigned int socket, const struct rte_eth_txconf *conf) | |
820 | { | |
9f95a23c | 821 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
822 | struct mlx5_txq_ctrl *tmpl; |
823 | ||
824 | tmpl = rte_calloc_socket("TXQ", 1, | |
825 | sizeof(*tmpl) + | |
826 | desc * sizeof(struct rte_mbuf *), | |
827 | 0, socket); | |
828 | if (!tmpl) { | |
829 | rte_errno = ENOMEM; | |
830 | return NULL; | |
831 | } | |
832 | if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh, | |
833 | MLX5_MR_BTREE_CACHE_N, socket)) { | |
834 | /* rte_errno is already set. */ | |
835 | goto error; | |
836 | } | |
837 | /* Save pointer of global generation number to check memory event. */ | |
9f95a23c | 838 | tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->mr.dev_gen; |
11fdf7f2 TL |
839 | assert(desc > MLX5_TX_COMP_THRESH); |
840 | tmpl->txq.offloads = conf->offloads | | |
841 | dev->data->dev_conf.txmode.offloads; | |
842 | tmpl->priv = priv; | |
843 | tmpl->socket = socket; | |
844 | tmpl->txq.elts_n = log2above(desc); | |
9f95a23c TL |
845 | tmpl->txq.port_id = dev->data->port_id; |
846 | tmpl->txq.idx = idx; | |
11fdf7f2 | 847 | txq_set_params(tmpl); |
9f95a23c TL |
848 | if (txq_calc_wqebb_cnt(tmpl) > |
849 | priv->sh->device_attr.orig_attr.max_qp_wr) { | |
850 | DRV_LOG(ERR, | |
851 | "port %u Tx WQEBB count (%d) exceeds the limit (%d)," | |
852 | " try smaller queue size", | |
853 | dev->data->port_id, txq_calc_wqebb_cnt(tmpl), | |
854 | priv->sh->device_attr.orig_attr.max_qp_wr); | |
855 | rte_errno = ENOMEM; | |
856 | goto error; | |
857 | } | |
11fdf7f2 TL |
858 | tmpl->txq.elts = |
859 | (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); | |
11fdf7f2 TL |
860 | rte_atomic32_inc(&tmpl->refcnt); |
861 | LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); | |
862 | return tmpl; | |
863 | error: | |
864 | rte_free(tmpl); | |
865 | return NULL; | |
866 | } | |
867 | ||
868 | /** | |
869 | * Get a Tx queue. | |
870 | * | |
871 | * @param dev | |
872 | * Pointer to Ethernet device. | |
873 | * @param idx | |
874 | * TX queue index. | |
875 | * | |
876 | * @return | |
877 | * A pointer to the queue if it exists. | |
878 | */ | |
879 | struct mlx5_txq_ctrl * | |
880 | mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) | |
881 | { | |
9f95a23c | 882 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
883 | struct mlx5_txq_ctrl *ctrl = NULL; |
884 | ||
885 | if ((*priv->txqs)[idx]) { | |
886 | ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, | |
887 | txq); | |
888 | mlx5_txq_ibv_get(dev, idx); | |
889 | rte_atomic32_inc(&ctrl->refcnt); | |
890 | } | |
891 | return ctrl; | |
892 | } | |
893 | ||
894 | /** | |
895 | * Release a Tx queue. | |
896 | * | |
897 | * @param dev | |
898 | * Pointer to Ethernet device. | |
899 | * @param idx | |
900 | * TX queue index. | |
901 | * | |
902 | * @return | |
903 | * 1 while a reference on it exists, 0 when freed. | |
904 | */ | |
905 | int | |
906 | mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) | |
907 | { | |
9f95a23c | 908 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 | 909 | struct mlx5_txq_ctrl *txq; |
11fdf7f2 TL |
910 | |
911 | if (!(*priv->txqs)[idx]) | |
912 | return 0; | |
913 | txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); | |
914 | if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv)) | |
915 | txq->ibv = NULL; | |
11fdf7f2 TL |
916 | if (rte_atomic32_dec_and_test(&txq->refcnt)) { |
917 | txq_free_elts(txq); | |
918 | mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh); | |
919 | LIST_REMOVE(txq, next); | |
920 | rte_free(txq); | |
921 | (*priv->txqs)[idx] = NULL; | |
922 | return 0; | |
923 | } | |
924 | return 1; | |
925 | } | |
926 | ||
927 | /** | |
928 | * Verify if the queue can be released. | |
929 | * | |
930 | * @param dev | |
931 | * Pointer to Ethernet device. | |
932 | * @param idx | |
933 | * TX queue index. | |
934 | * | |
935 | * @return | |
936 | * 1 if the queue can be released. | |
937 | */ | |
938 | int | |
939 | mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) | |
940 | { | |
9f95a23c | 941 | struct mlx5_priv *priv = dev->data->dev_private; |
11fdf7f2 TL |
942 | struct mlx5_txq_ctrl *txq; |
943 | ||
944 | if (!(*priv->txqs)[idx]) | |
945 | return -1; | |
946 | txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); | |
947 | return (rte_atomic32_read(&txq->refcnt) == 1); | |
948 | } | |
949 | ||
950 | /** | |
951 | * Verify the Tx Queue list is empty | |
952 | * | |
953 | * @param dev | |
954 | * Pointer to Ethernet device. | |
955 | * | |
956 | * @return | |
957 | * The number of object not released. | |
958 | */ | |
959 | int | |
960 | mlx5_txq_verify(struct rte_eth_dev *dev) | |
961 | { | |
9f95a23c TL |
962 | struct mlx5_priv *priv = dev->data->dev_private; |
963 | struct mlx5_txq_ctrl *txq_ctrl; | |
11fdf7f2 TL |
964 | int ret = 0; |
965 | ||
9f95a23c | 966 | LIST_FOREACH(txq_ctrl, &priv->txqsctrl, next) { |
11fdf7f2 | 967 | DRV_LOG(DEBUG, "port %u Tx queue %u still referenced", |
9f95a23c | 968 | dev->data->port_id, txq_ctrl->txq.idx); |
11fdf7f2 TL |
969 | ++ret; |
970 | } | |
971 | return ret; | |
972 | } |