1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2015 6WIND S.A.
3 * Copyright 2015 Mellanox Technologies, Ltd
15 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
17 #pragma GCC diagnostic ignored "-Wpedantic"
19 #include <infiniband/verbs.h>
20 #include <infiniband/mlx5dv.h>
22 #pragma GCC diagnostic error "-Wpedantic"
26 #include <rte_malloc.h>
27 #include <rte_ethdev_driver.h>
28 #include <rte_common.h>
30 #include <mlx5_glue.h>
31 #include <mlx5_devx_cmds.h>
32 #include <mlx5_common.h>
33 #include <mlx5_common_mr.h>
35 #include "mlx5_defs.h"
36 #include "mlx5_utils.h"
38 #include "mlx5_rxtx.h"
39 #include "mlx5_autoconf.h"
42 * Allocate TX queue elements.
45 * Pointer to TX queue structure.
48 txq_alloc_elts(struct mlx5_txq_ctrl
*txq_ctrl
)
50 const unsigned int elts_n
= 1 << txq_ctrl
->txq
.elts_n
;
53 for (i
= 0; (i
!= elts_n
); ++i
)
54 txq_ctrl
->txq
.elts
[i
] = NULL
;
55 DRV_LOG(DEBUG
, "port %u Tx queue %u allocated and configured %u WRs",
56 PORT_ID(txq_ctrl
->priv
), txq_ctrl
->txq
.idx
, elts_n
);
57 txq_ctrl
->txq
.elts_head
= 0;
58 txq_ctrl
->txq
.elts_tail
= 0;
59 txq_ctrl
->txq
.elts_comp
= 0;
63 * Free TX queue elements.
66 * Pointer to TX queue structure.
69 txq_free_elts(struct mlx5_txq_ctrl
*txq_ctrl
)
71 const uint16_t elts_n
= 1 << txq_ctrl
->txq
.elts_n
;
72 const uint16_t elts_m
= elts_n
- 1;
73 uint16_t elts_head
= txq_ctrl
->txq
.elts_head
;
74 uint16_t elts_tail
= txq_ctrl
->txq
.elts_tail
;
75 struct rte_mbuf
*(*elts
)[elts_n
] = &txq_ctrl
->txq
.elts
;
77 DRV_LOG(DEBUG
, "port %u Tx queue %u freeing WRs",
78 PORT_ID(txq_ctrl
->priv
), txq_ctrl
->txq
.idx
);
79 txq_ctrl
->txq
.elts_head
= 0;
80 txq_ctrl
->txq
.elts_tail
= 0;
81 txq_ctrl
->txq
.elts_comp
= 0;
83 while (elts_tail
!= elts_head
) {
84 struct rte_mbuf
*elt
= (*elts
)[elts_tail
& elts_m
];
86 MLX5_ASSERT(elt
!= NULL
);
87 rte_pktmbuf_free_seg(elt
);
88 #ifdef RTE_LIBRTE_MLX5_DEBUG
90 memset(&(*elts
)[elts_tail
& elts_m
],
92 sizeof((*elts
)[elts_tail
& elts_m
]));
99 * Returns the per-port supported offloads.
102 * Pointer to Ethernet device.
105 * Supported Tx offloads.
108 mlx5_get_tx_port_offloads(struct rte_eth_dev
*dev
)
110 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
111 uint64_t offloads
= (DEV_TX_OFFLOAD_MULTI_SEGS
|
112 DEV_TX_OFFLOAD_VLAN_INSERT
);
113 struct mlx5_dev_config
*config
= &priv
->config
;
116 offloads
|= (DEV_TX_OFFLOAD_IPV4_CKSUM
|
117 DEV_TX_OFFLOAD_UDP_CKSUM
|
118 DEV_TX_OFFLOAD_TCP_CKSUM
);
120 offloads
|= DEV_TX_OFFLOAD_TCP_TSO
;
123 offloads
|= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
;
125 offloads
|= (DEV_TX_OFFLOAD_IP_TNL_TSO
|
126 DEV_TX_OFFLOAD_UDP_TNL_TSO
);
128 if (config
->tunnel_en
) {
130 offloads
|= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
;
132 offloads
|= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO
|
133 DEV_TX_OFFLOAD_GRE_TNL_TSO
|
134 DEV_TX_OFFLOAD_GENEVE_TNL_TSO
);
140 * Tx queue presetup checks.
143 * Pointer to Ethernet device structure.
147 * Number of descriptors to configure in queue.
150 * 0 on success, a negative errno value otherwise and rte_errno is set.
153 mlx5_tx_queue_pre_setup(struct rte_eth_dev
*dev
, uint16_t idx
, uint16_t desc
)
155 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
157 if (desc
<= MLX5_TX_COMP_THRESH
) {
159 "port %u number of descriptors requested for Tx queue"
160 " %u must be higher than MLX5_TX_COMP_THRESH, using %u"
162 dev
->data
->port_id
, idx
, MLX5_TX_COMP_THRESH
+ 1, desc
);
163 desc
= MLX5_TX_COMP_THRESH
+ 1;
165 if (!rte_is_power_of_2(desc
)) {
166 desc
= 1 << log2above(desc
);
168 "port %u increased number of descriptors in Tx queue"
169 " %u to the next power of two (%d)",
170 dev
->data
->port_id
, idx
, desc
);
172 DRV_LOG(DEBUG
, "port %u configuring queue %u for %u descriptors",
173 dev
->data
->port_id
, idx
, desc
);
174 if (idx
>= priv
->txqs_n
) {
175 DRV_LOG(ERR
, "port %u Tx queue index out of range (%u >= %u)",
176 dev
->data
->port_id
, idx
, priv
->txqs_n
);
177 rte_errno
= EOVERFLOW
;
180 if (!mlx5_txq_releasable(dev
, idx
)) {
182 DRV_LOG(ERR
, "port %u unable to release queue index %u",
183 dev
->data
->port_id
, idx
);
186 mlx5_txq_release(dev
, idx
);
190 * DPDK callback to configure a TX queue.
193 * Pointer to Ethernet device structure.
197 * Number of descriptors to configure in queue.
199 * NUMA socket on which memory must be allocated.
201 * Thresholds parameters.
204 * 0 on success, a negative errno value otherwise and rte_errno is set.
207 mlx5_tx_queue_setup(struct rte_eth_dev
*dev
, uint16_t idx
, uint16_t desc
,
208 unsigned int socket
, const struct rte_eth_txconf
*conf
)
210 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
211 struct mlx5_txq_data
*txq
= (*priv
->txqs
)[idx
];
212 struct mlx5_txq_ctrl
*txq_ctrl
=
213 container_of(txq
, struct mlx5_txq_ctrl
, txq
);
216 res
= mlx5_tx_queue_pre_setup(dev
, idx
, desc
);
219 txq_ctrl
= mlx5_txq_new(dev
, idx
, desc
, socket
, conf
);
221 DRV_LOG(ERR
, "port %u unable to allocate queue index %u",
222 dev
->data
->port_id
, idx
);
225 DRV_LOG(DEBUG
, "port %u adding Tx queue %u to list",
226 dev
->data
->port_id
, idx
);
227 (*priv
->txqs
)[idx
] = &txq_ctrl
->txq
;
232 * DPDK callback to configure a TX hairpin queue.
235 * Pointer to Ethernet device structure.
239 * Number of descriptors to configure in queue.
240 * @param[in] hairpin_conf
241 * The hairpin binding configuration.
244 * 0 on success, a negative errno value otherwise and rte_errno is set.
247 mlx5_tx_hairpin_queue_setup(struct rte_eth_dev
*dev
, uint16_t idx
,
249 const struct rte_eth_hairpin_conf
*hairpin_conf
)
251 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
252 struct mlx5_txq_data
*txq
= (*priv
->txqs
)[idx
];
253 struct mlx5_txq_ctrl
*txq_ctrl
=
254 container_of(txq
, struct mlx5_txq_ctrl
, txq
);
257 res
= mlx5_tx_queue_pre_setup(dev
, idx
, desc
);
260 if (hairpin_conf
->peer_count
!= 1 ||
261 hairpin_conf
->peers
[0].port
!= dev
->data
->port_id
||
262 hairpin_conf
->peers
[0].queue
>= priv
->rxqs_n
) {
263 DRV_LOG(ERR
, "port %u unable to setup hairpin queue index %u "
264 " invalid hairpind configuration", dev
->data
->port_id
,
269 txq_ctrl
= mlx5_txq_hairpin_new(dev
, idx
, desc
, hairpin_conf
);
271 DRV_LOG(ERR
, "port %u unable to allocate queue index %u",
272 dev
->data
->port_id
, idx
);
275 DRV_LOG(DEBUG
, "port %u adding Tx queue %u to list",
276 dev
->data
->port_id
, idx
);
277 (*priv
->txqs
)[idx
] = &txq_ctrl
->txq
;
282 * DPDK callback to release a TX queue.
285 * Generic TX queue pointer.
288 mlx5_tx_queue_release(void *dpdk_txq
)
290 struct mlx5_txq_data
*txq
= (struct mlx5_txq_data
*)dpdk_txq
;
291 struct mlx5_txq_ctrl
*txq_ctrl
;
292 struct mlx5_priv
*priv
;
297 txq_ctrl
= container_of(txq
, struct mlx5_txq_ctrl
, txq
);
298 priv
= txq_ctrl
->priv
;
299 for (i
= 0; (i
!= priv
->txqs_n
); ++i
)
300 if ((*priv
->txqs
)[i
] == txq
) {
301 DRV_LOG(DEBUG
, "port %u removing Tx queue %u from list",
302 PORT_ID(priv
), txq
->idx
);
303 mlx5_txq_release(ETH_DEV(priv
), i
);
309 * Configure the doorbell register non-cached attribute.
312 * Pointer to Tx queue control structure.
317 txq_uar_ncattr_init(struct mlx5_txq_ctrl
*txq_ctrl
, size_t page_size
)
319 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
322 txq_ctrl
->txq
.db_heu
= priv
->config
.dbnc
== MLX5_TXDB_HEURISTIC
;
323 txq_ctrl
->txq
.db_nc
= 0;
324 /* Check the doorbell register mapping type. */
325 cmd
= txq_ctrl
->uar_mmap_offset
/ page_size
;
326 cmd
>>= MLX5_UAR_MMAP_CMD_SHIFT
;
327 cmd
&= MLX5_UAR_MMAP_CMD_MASK
;
328 if (cmd
== MLX5_MMAP_GET_NC_PAGES_CMD
)
329 txq_ctrl
->txq
.db_nc
= 1;
333 * Initialize Tx UAR registers for primary process.
336 * Pointer to Tx queue control structure.
339 txq_uar_init(struct mlx5_txq_ctrl
*txq_ctrl
)
341 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
342 struct mlx5_proc_priv
*ppriv
= MLX5_PROC_PRIV(PORT_ID(priv
));
343 const size_t page_size
= sysconf(_SC_PAGESIZE
);
345 unsigned int lock_idx
;
348 if (txq_ctrl
->type
!= MLX5_TXQ_TYPE_STANDARD
)
350 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY
);
352 ppriv
->uar_table
[txq_ctrl
->txq
.idx
] = txq_ctrl
->bf_reg
;
353 txq_uar_ncattr_init(txq_ctrl
, page_size
);
355 /* Assign an UAR lock according to UAR page number */
356 lock_idx
= (txq_ctrl
->uar_mmap_offset
/ page_size
) &
357 MLX5_UAR_PAGE_NUM_MASK
;
358 txq_ctrl
->txq
.uar_lock
= &priv
->uar_lock
[lock_idx
];
363 * Remap UAR register of a Tx queue for secondary process.
365 * Remapped address is stored at the table in the process private structure of
366 * the device, indexed by queue index.
369 * Pointer to Tx queue control structure.
371 * Verbs file descriptor to map UAR pages.
374 * 0 on success, a negative errno value otherwise and rte_errno is set.
377 txq_uar_init_secondary(struct mlx5_txq_ctrl
*txq_ctrl
, int fd
)
379 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
380 struct mlx5_proc_priv
*ppriv
= MLX5_PROC_PRIV(PORT_ID(priv
));
381 struct mlx5_txq_data
*txq
= &txq_ctrl
->txq
;
385 const size_t page_size
= sysconf(_SC_PAGESIZE
);
387 if (txq_ctrl
->type
!= MLX5_TXQ_TYPE_STANDARD
)
391 * As rdma-core, UARs are mapped in size of OS page
392 * size. Ref to libmlx5 function: mlx5_init_context()
394 uar_va
= (uintptr_t)txq_ctrl
->bf_reg
;
395 offset
= uar_va
& (page_size
- 1); /* Offset in page. */
396 addr
= mmap(NULL
, page_size
, PROT_WRITE
, MAP_SHARED
, fd
,
397 txq_ctrl
->uar_mmap_offset
);
398 if (addr
== MAP_FAILED
) {
400 "port %u mmap failed for BF reg of txq %u",
401 txq
->port_id
, txq
->idx
);
405 addr
= RTE_PTR_ADD(addr
, offset
);
406 ppriv
->uar_table
[txq
->idx
] = addr
;
407 txq_uar_ncattr_init(txq_ctrl
, page_size
);
412 * Unmap UAR register of a Tx queue for secondary process.
415 * Pointer to Tx queue control structure.
418 txq_uar_uninit_secondary(struct mlx5_txq_ctrl
*txq_ctrl
)
420 struct mlx5_proc_priv
*ppriv
= MLX5_PROC_PRIV(PORT_ID(txq_ctrl
->priv
));
421 const size_t page_size
= sysconf(_SC_PAGESIZE
);
424 if (txq_ctrl
->type
!= MLX5_TXQ_TYPE_STANDARD
)
426 addr
= ppriv
->uar_table
[txq_ctrl
->txq
.idx
];
427 munmap(RTE_PTR_ALIGN_FLOOR(addr
, page_size
), page_size
);
431 * Initialize Tx UAR registers for secondary process.
434 * Pointer to Ethernet device.
436 * Verbs file descriptor to map UAR pages.
439 * 0 on success, a negative errno value otherwise and rte_errno is set.
442 mlx5_tx_uar_init_secondary(struct rte_eth_dev
*dev
, int fd
)
444 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
445 struct mlx5_txq_data
*txq
;
446 struct mlx5_txq_ctrl
*txq_ctrl
;
450 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY
);
451 for (i
= 0; i
!= priv
->txqs_n
; ++i
) {
452 if (!(*priv
->txqs
)[i
])
454 txq
= (*priv
->txqs
)[i
];
455 txq_ctrl
= container_of(txq
, struct mlx5_txq_ctrl
, txq
);
456 if (txq_ctrl
->type
!= MLX5_TXQ_TYPE_STANDARD
)
458 MLX5_ASSERT(txq
->idx
== (uint16_t)i
);
459 ret
= txq_uar_init_secondary(txq_ctrl
, fd
);
467 if (!(*priv
->txqs
)[i
])
469 txq
= (*priv
->txqs
)[i
];
470 txq_ctrl
= container_of(txq
, struct mlx5_txq_ctrl
, txq
);
471 txq_uar_uninit_secondary(txq_ctrl
);
477 * Create the Tx hairpin queue object.
480 * Pointer to Ethernet device.
482 * Queue index in DPDK Tx queue array
485 * The hairpin DevX object initialised, NULL otherwise and rte_errno is set.
487 static struct mlx5_txq_obj
*
488 mlx5_txq_obj_hairpin_new(struct rte_eth_dev
*dev
, uint16_t idx
)
490 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
491 struct mlx5_txq_data
*txq_data
= (*priv
->txqs
)[idx
];
492 struct mlx5_txq_ctrl
*txq_ctrl
=
493 container_of(txq_data
, struct mlx5_txq_ctrl
, txq
);
494 struct mlx5_devx_create_sq_attr attr
= { 0 };
495 struct mlx5_txq_obj
*tmpl
= NULL
;
497 uint32_t max_wq_data
;
499 MLX5_ASSERT(txq_data
);
500 MLX5_ASSERT(!txq_ctrl
->obj
);
501 tmpl
= rte_calloc_socket(__func__
, 1, sizeof(*tmpl
), 0,
505 "port %u Tx queue %u cannot allocate memory resources",
506 dev
->data
->port_id
, txq_data
->idx
);
510 tmpl
->type
= MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN
;
511 tmpl
->txq_ctrl
= txq_ctrl
;
514 max_wq_data
= priv
->config
.hca_attr
.log_max_hairpin_wq_data_sz
;
515 /* Jumbo frames > 9KB should be supported, and more packets. */
516 if (priv
->config
.log_hp_size
!= (uint32_t)MLX5_ARG_UNSET
) {
517 if (priv
->config
.log_hp_size
> max_wq_data
) {
518 DRV_LOG(ERR
, "total data size %u power of 2 is "
519 "too large for hairpin",
520 priv
->config
.log_hp_size
);
524 attr
.wq_attr
.log_hairpin_data_sz
= priv
->config
.log_hp_size
;
526 attr
.wq_attr
.log_hairpin_data_sz
=
527 (max_wq_data
< MLX5_HAIRPIN_JUMBO_LOG_SIZE
) ?
528 max_wq_data
: MLX5_HAIRPIN_JUMBO_LOG_SIZE
;
530 /* Set the packets number to the maximum value for performance. */
531 attr
.wq_attr
.log_hairpin_num_packets
=
532 attr
.wq_attr
.log_hairpin_data_sz
-
533 MLX5_HAIRPIN_QUEUE_STRIDE
;
534 attr
.tis_num
= priv
->sh
->tis
->id
;
535 tmpl
->sq
= mlx5_devx_cmd_create_sq(priv
->sh
->ctx
, &attr
);
538 "port %u tx hairpin queue %u can't create sq object",
539 dev
->data
->port_id
, idx
);
543 DRV_LOG(DEBUG
, "port %u sxq %u updated with %p", dev
->data
->port_id
,
545 rte_atomic32_inc(&tmpl
->refcnt
);
546 LIST_INSERT_HEAD(&priv
->txqsobj
, tmpl
, next
);
549 ret
= rte_errno
; /* Save rte_errno before cleanup. */
551 mlx5_devx_cmd_destroy(tmpl
->tis
);
553 mlx5_devx_cmd_destroy(tmpl
->sq
);
554 rte_errno
= ret
; /* Restore rte_errno. */
559 * Create the Tx queue Verbs object.
562 * Pointer to Ethernet device.
564 * Queue index in DPDK Tx queue array.
566 * Type of the Tx queue object to create.
569 * The Verbs object initialised, NULL otherwise and rte_errno is set.
571 struct mlx5_txq_obj
*
572 mlx5_txq_obj_new(struct rte_eth_dev
*dev
, uint16_t idx
,
573 enum mlx5_txq_obj_type type
)
575 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
576 struct mlx5_txq_data
*txq_data
= (*priv
->txqs
)[idx
];
577 struct mlx5_txq_ctrl
*txq_ctrl
=
578 container_of(txq_data
, struct mlx5_txq_ctrl
, txq
);
579 struct mlx5_txq_obj tmpl
;
580 struct mlx5_txq_obj
*txq_obj
= NULL
;
582 struct ibv_qp_init_attr_ex init
;
583 struct ibv_cq_init_attr_ex cq
;
584 struct ibv_qp_attr mod
;
587 struct mlx5dv_qp qp
= { .comp_mask
= MLX5DV_QP_MASK_UAR_MMAP_OFFSET
};
588 struct mlx5dv_cq cq_info
;
589 struct mlx5dv_obj obj
;
590 const int desc
= 1 << txq_data
->elts_n
;
593 if (type
== MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN
)
594 return mlx5_txq_obj_hairpin_new(dev
, idx
);
595 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
596 /* If using DevX, need additional mask to read tisn value. */
597 if (priv
->config
.devx
&& !priv
->sh
->tdn
)
598 qp
.comp_mask
|= MLX5DV_QP_MASK_RAW_QP_HANDLES
;
600 MLX5_ASSERT(txq_data
);
601 priv
->verbs_alloc_ctx
.type
= MLX5_VERBS_ALLOC_TYPE_TX_QUEUE
;
602 priv
->verbs_alloc_ctx
.obj
= txq_ctrl
;
603 if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
605 "port %u MLX5_ENABLE_CQE_COMPRESSION must never be set",
610 memset(&tmpl
, 0, sizeof(struct mlx5_txq_obj
));
611 attr
.cq
= (struct ibv_cq_init_attr_ex
){
614 cqe_n
= desc
/ MLX5_TX_COMP_THRESH
+
615 1 + MLX5_TX_COMP_THRESH_INLINE_DIV
;
616 tmpl
.cq
= mlx5_glue
->create_cq(priv
->sh
->ctx
, cqe_n
, NULL
, NULL
, 0);
617 if (tmpl
.cq
== NULL
) {
618 DRV_LOG(ERR
, "port %u Tx queue %u CQ creation failure",
619 dev
->data
->port_id
, idx
);
623 attr
.init
= (struct ibv_qp_init_attr_ex
){
624 /* CQ to be associated with the send queue. */
626 /* CQ to be associated with the receive queue. */
629 /* Max number of outstanding WRs. */
631 ((priv
->sh
->device_attr
.orig_attr
.max_qp_wr
<
633 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
:
636 * Max number of scatter/gather elements in a WR,
637 * must be 1 to prevent libmlx5 from trying to affect
638 * too much memory. TX gather is not impacted by the
639 * device_attr.max_sge limit and will still work
644 .qp_type
= IBV_QPT_RAW_PACKET
,
646 * Do *NOT* enable this, completions events are managed per
651 .comp_mask
= IBV_QP_INIT_ATTR_PD
,
653 if (txq_data
->inlen_send
)
654 attr
.init
.cap
.max_inline_data
= txq_ctrl
->max_inline_data
;
655 if (txq_data
->tso_en
) {
656 attr
.init
.max_tso_header
= txq_ctrl
->max_tso_header
;
657 attr
.init
.comp_mask
|= IBV_QP_INIT_ATTR_MAX_TSO_HEADER
;
659 tmpl
.qp
= mlx5_glue
->create_qp_ex(priv
->sh
->ctx
, &attr
.init
);
660 if (tmpl
.qp
== NULL
) {
661 DRV_LOG(ERR
, "port %u Tx queue %u QP creation failure",
662 dev
->data
->port_id
, idx
);
666 attr
.mod
= (struct ibv_qp_attr
){
667 /* Move the QP to this state. */
668 .qp_state
= IBV_QPS_INIT
,
669 /* IB device port number. */
670 .port_num
= (uint8_t)priv
->ibv_port
,
672 ret
= mlx5_glue
->modify_qp(tmpl
.qp
, &attr
.mod
,
673 (IBV_QP_STATE
| IBV_QP_PORT
));
676 "port %u Tx queue %u QP state to IBV_QPS_INIT failed",
677 dev
->data
->port_id
, idx
);
681 attr
.mod
= (struct ibv_qp_attr
){
682 .qp_state
= IBV_QPS_RTR
684 ret
= mlx5_glue
->modify_qp(tmpl
.qp
, &attr
.mod
, IBV_QP_STATE
);
687 "port %u Tx queue %u QP state to IBV_QPS_RTR failed",
688 dev
->data
->port_id
, idx
);
692 attr
.mod
.qp_state
= IBV_QPS_RTS
;
693 ret
= mlx5_glue
->modify_qp(tmpl
.qp
, &attr
.mod
, IBV_QP_STATE
);
696 "port %u Tx queue %u QP state to IBV_QPS_RTS failed",
697 dev
->data
->port_id
, idx
);
701 txq_obj
= rte_calloc_socket(__func__
, 1, sizeof(struct mlx5_txq_obj
), 0,
704 DRV_LOG(ERR
, "port %u Tx queue %u cannot allocate memory",
705 dev
->data
->port_id
, idx
);
710 obj
.cq
.out
= &cq_info
;
713 ret
= mlx5_glue
->dv_init_obj(&obj
, MLX5DV_OBJ_CQ
| MLX5DV_OBJ_QP
);
718 if (cq_info
.cqe_size
!= RTE_CACHE_LINE_SIZE
) {
720 "port %u wrong MLX5_CQE_SIZE environment variable"
721 " value: it should be set to %u",
722 dev
->data
->port_id
, RTE_CACHE_LINE_SIZE
);
726 txq_data
->cqe_n
= log2above(cq_info
.cqe_cnt
);
727 txq_data
->cqe_s
= 1 << txq_data
->cqe_n
;
728 txq_data
->cqe_m
= txq_data
->cqe_s
- 1;
729 txq_data
->qp_num_8s
= tmpl
.qp
->qp_num
<< 8;
730 txq_data
->wqes
= qp
.sq
.buf
;
731 txq_data
->wqe_n
= log2above(qp
.sq
.wqe_cnt
);
732 txq_data
->wqe_s
= 1 << txq_data
->wqe_n
;
733 txq_data
->wqe_m
= txq_data
->wqe_s
- 1;
734 txq_data
->wqes_end
= txq_data
->wqes
+ txq_data
->wqe_s
;
735 txq_data
->qp_db
= &qp
.dbrec
[MLX5_SND_DBR
];
736 txq_data
->cq_db
= cq_info
.dbrec
;
737 txq_data
->cqes
= (volatile struct mlx5_cqe
*)cq_info
.buf
;
740 txq_data
->wqe_ci
= 0;
741 txq_data
->wqe_pi
= 0;
742 txq_data
->wqe_comp
= 0;
743 txq_data
->wqe_thres
= txq_data
->wqe_s
/ MLX5_TX_COMP_THRESH_INLINE_DIV
;
744 txq_data
->fcqs
= rte_calloc_socket(__func__
,
746 sizeof(*txq_data
->fcqs
),
749 if (!txq_data
->fcqs
) {
750 DRV_LOG(ERR
, "port %u Tx queue %u cannot allocate memory (FCQ)",
751 dev
->data
->port_id
, idx
);
755 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
757 * If using DevX need to query and store TIS transport domain value.
758 * This is done once per port.
759 * Will use this value on Rx, when creating matching TIR.
761 if (priv
->config
.devx
&& !priv
->sh
->tdn
) {
762 ret
= mlx5_devx_cmd_qp_query_tis_td(tmpl
.qp
, qp
.tisn
,
765 DRV_LOG(ERR
, "Fail to query port %u Tx queue %u QP TIS "
766 "transport domain", dev
->data
->port_id
, idx
);
770 DRV_LOG(DEBUG
, "port %u Tx queue %u TIS number %d "
771 "transport domain %d", dev
->data
->port_id
,
772 idx
, qp
.tisn
, priv
->sh
->tdn
);
776 txq_obj
->qp
= tmpl
.qp
;
777 txq_obj
->cq
= tmpl
.cq
;
778 rte_atomic32_inc(&txq_obj
->refcnt
);
779 txq_ctrl
->bf_reg
= qp
.bf
.reg
;
780 if (qp
.comp_mask
& MLX5DV_QP_MASK_UAR_MMAP_OFFSET
) {
781 txq_ctrl
->uar_mmap_offset
= qp
.uar_mmap_offset
;
782 DRV_LOG(DEBUG
, "port %u: uar_mmap_offset 0x%"PRIx64
,
783 dev
->data
->port_id
, txq_ctrl
->uar_mmap_offset
);
786 "port %u failed to retrieve UAR info, invalid"
792 txq_uar_init(txq_ctrl
);
793 LIST_INSERT_HEAD(&priv
->txqsobj
, txq_obj
, next
);
794 txq_obj
->txq_ctrl
= txq_ctrl
;
795 priv
->verbs_alloc_ctx
.type
= MLX5_VERBS_ALLOC_TYPE_NONE
;
798 ret
= rte_errno
; /* Save rte_errno before cleanup. */
800 claim_zero(mlx5_glue
->destroy_cq(tmpl
.cq
));
802 claim_zero(mlx5_glue
->destroy_qp(tmpl
.qp
));
803 if (txq_data
&& txq_data
->fcqs
)
804 rte_free(txq_data
->fcqs
);
807 priv
->verbs_alloc_ctx
.type
= MLX5_VERBS_ALLOC_TYPE_NONE
;
808 rte_errno
= ret
; /* Restore rte_errno. */
813 * Get an Tx queue Verbs object.
816 * Pointer to Ethernet device.
818 * Queue index in DPDK Tx queue array.
821 * The Verbs object if it exists.
823 struct mlx5_txq_obj
*
824 mlx5_txq_obj_get(struct rte_eth_dev
*dev
, uint16_t idx
)
826 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
827 struct mlx5_txq_ctrl
*txq_ctrl
;
829 if (idx
>= priv
->txqs_n
)
831 if (!(*priv
->txqs
)[idx
])
833 txq_ctrl
= container_of((*priv
->txqs
)[idx
], struct mlx5_txq_ctrl
, txq
);
835 rte_atomic32_inc(&txq_ctrl
->obj
->refcnt
);
836 return txq_ctrl
->obj
;
840 * Release an Tx verbs queue object.
843 * Verbs Tx queue object.
846 * 1 while a reference on it exists, 0 when freed.
849 mlx5_txq_obj_release(struct mlx5_txq_obj
*txq_obj
)
851 MLX5_ASSERT(txq_obj
);
852 if (rte_atomic32_dec_and_test(&txq_obj
->refcnt
)) {
853 if (txq_obj
->type
== MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN
) {
855 claim_zero(mlx5_devx_cmd_destroy(txq_obj
->tis
));
857 claim_zero(mlx5_glue
->destroy_qp(txq_obj
->qp
));
858 claim_zero(mlx5_glue
->destroy_cq(txq_obj
->cq
));
859 if (txq_obj
->txq_ctrl
->txq
.fcqs
)
860 rte_free(txq_obj
->txq_ctrl
->txq
.fcqs
);
862 LIST_REMOVE(txq_obj
, next
);
870 * Verify the Verbs Tx queue list is empty
873 * Pointer to Ethernet device.
876 * The number of object not released.
879 mlx5_txq_obj_verify(struct rte_eth_dev
*dev
)
881 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
883 struct mlx5_txq_obj
*txq_obj
;
885 LIST_FOREACH(txq_obj
, &priv
->txqsobj
, next
) {
886 DRV_LOG(DEBUG
, "port %u Verbs Tx queue %u still referenced",
887 dev
->data
->port_id
, txq_obj
->txq_ctrl
->txq
.idx
);
894 * Calculate the total number of WQEBB for Tx queue.
896 * Simplified version of calc_sq_size() in rdma-core.
899 * Pointer to Tx queue control structure.
902 * The number of WQEBB.
905 txq_calc_wqebb_cnt(struct mlx5_txq_ctrl
*txq_ctrl
)
907 unsigned int wqe_size
;
908 const unsigned int desc
= 1 << txq_ctrl
->txq
.elts_n
;
910 wqe_size
= MLX5_WQE_CSEG_SIZE
+
913 MLX5_ESEG_MIN_INLINE_SIZE
+
914 txq_ctrl
->max_inline_data
;
915 return rte_align32pow2(wqe_size
* desc
) / MLX5_WQE_SIZE
;
919 * Calculate the maximal inline data size for Tx queue.
922 * Pointer to Tx queue control structure.
925 * The maximal inline data size.
928 txq_calc_inline_max(struct mlx5_txq_ctrl
*txq_ctrl
)
930 const unsigned int desc
= 1 << txq_ctrl
->txq
.elts_n
;
931 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
932 unsigned int wqe_size
;
934 wqe_size
= priv
->sh
->device_attr
.orig_attr
.max_qp_wr
/ desc
;
938 * This calculation is derived from tthe source of
939 * mlx5_calc_send_wqe() in rdma_core library.
941 wqe_size
= wqe_size
* MLX5_WQE_SIZE
-
946 MLX5_DSEG_MIN_INLINE_SIZE
;
951 * Set Tx queue parameters from device configuration.
954 * Pointer to Tx queue control structure.
957 txq_set_params(struct mlx5_txq_ctrl
*txq_ctrl
)
959 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
960 struct mlx5_dev_config
*config
= &priv
->config
;
961 unsigned int inlen_send
; /* Inline data for ordinary SEND.*/
962 unsigned int inlen_empw
; /* Inline data for enhanced MPW. */
963 unsigned int inlen_mode
; /* Minimal required Inline data. */
964 unsigned int txqs_inline
; /* Min Tx queues to enable inline. */
965 uint64_t dev_txoff
= priv
->dev_data
->dev_conf
.txmode
.offloads
;
966 bool tso
= txq_ctrl
->txq
.offloads
& (DEV_TX_OFFLOAD_TCP_TSO
|
967 DEV_TX_OFFLOAD_VXLAN_TNL_TSO
|
968 DEV_TX_OFFLOAD_GRE_TNL_TSO
|
969 DEV_TX_OFFLOAD_IP_TNL_TSO
|
970 DEV_TX_OFFLOAD_UDP_TNL_TSO
);
974 if (config
->txqs_inline
== MLX5_ARG_UNSET
)
976 #if defined(RTE_ARCH_ARM64)
977 (priv
->pci_dev
->id
.device_id
==
978 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF
) ?
979 MLX5_INLINE_MAX_TXQS_BLUEFIELD
:
981 MLX5_INLINE_MAX_TXQS
;
983 txqs_inline
= (unsigned int)config
->txqs_inline
;
984 inlen_send
= (config
->txq_inline_max
== MLX5_ARG_UNSET
) ?
985 MLX5_SEND_DEF_INLINE_LEN
:
986 (unsigned int)config
->txq_inline_max
;
987 inlen_empw
= (config
->txq_inline_mpw
== MLX5_ARG_UNSET
) ?
988 MLX5_EMPW_DEF_INLINE_LEN
:
989 (unsigned int)config
->txq_inline_mpw
;
990 inlen_mode
= (config
->txq_inline_min
== MLX5_ARG_UNSET
) ?
991 0 : (unsigned int)config
->txq_inline_min
;
992 if (config
->mps
!= MLX5_MPW_ENHANCED
&& config
->mps
!= MLX5_MPW
)
995 * If there is requested minimal amount of data to inline
996 * we MUST enable inlining. This is a case for ConnectX-4
997 * which usually requires L2 inlined for correct operating
998 * and ConnectX-4 Lx which requires L2-L4 inlined to
999 * support E-Switch Flows.
1002 if (inlen_mode
<= MLX5_ESEG_MIN_INLINE_SIZE
) {
1004 * Optimize minimal inlining for single
1005 * segment packets to fill one WQEBB
1008 temp
= MLX5_ESEG_MIN_INLINE_SIZE
;
1010 temp
= inlen_mode
- MLX5_ESEG_MIN_INLINE_SIZE
;
1011 temp
= RTE_ALIGN(temp
, MLX5_WSEG_SIZE
) +
1012 MLX5_ESEG_MIN_INLINE_SIZE
;
1013 temp
= RTE_MIN(temp
, MLX5_SEND_MAX_INLINE_LEN
);
1015 if (temp
!= inlen_mode
) {
1017 "port %u minimal required inline setting"
1018 " aligned from %u to %u",
1019 PORT_ID(priv
), inlen_mode
, temp
);
1024 * If port is configured to support VLAN insertion and device
1025 * does not support this feature by HW (for NICs before ConnectX-5
1026 * or in case of wqe_vlan_insert flag is not set) we must enable
1027 * data inline on all queues because it is supported by single
1030 txq_ctrl
->txq
.vlan_en
= config
->hw_vlan_insert
;
1031 vlan_inline
= (dev_txoff
& DEV_TX_OFFLOAD_VLAN_INSERT
) &&
1032 !config
->hw_vlan_insert
;
1034 * If there are few Tx queues it is prioritized
1035 * to save CPU cycles and disable data inlining at all.
1037 if (inlen_send
&& priv
->txqs_n
>= txqs_inline
) {
1039 * The data sent with ordinal MLX5_OPCODE_SEND
1040 * may be inlined in Ethernet Segment, align the
1041 * length accordingly to fit entire WQEBBs.
1043 temp
= RTE_MAX(inlen_send
,
1044 MLX5_ESEG_MIN_INLINE_SIZE
+ MLX5_WQE_DSEG_SIZE
);
1045 temp
-= MLX5_ESEG_MIN_INLINE_SIZE
+ MLX5_WQE_DSEG_SIZE
;
1046 temp
= RTE_ALIGN(temp
, MLX5_WQE_SIZE
);
1047 temp
+= MLX5_ESEG_MIN_INLINE_SIZE
+ MLX5_WQE_DSEG_SIZE
;
1048 temp
= RTE_MIN(temp
, MLX5_WQE_SIZE_MAX
+
1049 MLX5_ESEG_MIN_INLINE_SIZE
-
1050 MLX5_WQE_CSEG_SIZE
-
1051 MLX5_WQE_ESEG_SIZE
-
1052 MLX5_WQE_DSEG_SIZE
* 2);
1053 temp
= RTE_MIN(temp
, MLX5_SEND_MAX_INLINE_LEN
);
1054 temp
= RTE_MAX(temp
, inlen_mode
);
1055 if (temp
!= inlen_send
) {
1057 "port %u ordinary send inline setting"
1058 " aligned from %u to %u",
1059 PORT_ID(priv
), inlen_send
, temp
);
1063 * Not aligned to cache lines, but to WQEs.
1064 * First bytes of data (initial alignment)
1065 * is going to be copied explicitly at the
1066 * beginning of inlining buffer in Ethernet
1069 MLX5_ASSERT(inlen_send
>= MLX5_ESEG_MIN_INLINE_SIZE
);
1070 MLX5_ASSERT(inlen_send
<= MLX5_WQE_SIZE_MAX
+
1071 MLX5_ESEG_MIN_INLINE_SIZE
-
1072 MLX5_WQE_CSEG_SIZE
-
1073 MLX5_WQE_ESEG_SIZE
-
1074 MLX5_WQE_DSEG_SIZE
* 2);
1075 } else if (inlen_mode
) {
1077 * If minimal inlining is requested we must
1078 * enable inlining in general, despite the
1079 * number of configured queues. Ignore the
1080 * txq_inline_max devarg, this is not
1081 * full-featured inline.
1083 inlen_send
= inlen_mode
;
1085 } else if (vlan_inline
) {
1087 * Hardware does not report offload for
1088 * VLAN insertion, we must enable data inline
1089 * to implement feature by software.
1091 inlen_send
= MLX5_ESEG_MIN_INLINE_SIZE
;
1097 txq_ctrl
->txq
.inlen_send
= inlen_send
;
1098 txq_ctrl
->txq
.inlen_mode
= inlen_mode
;
1099 txq_ctrl
->txq
.inlen_empw
= 0;
1100 if (inlen_send
&& inlen_empw
&& priv
->txqs_n
>= txqs_inline
) {
1102 * The data sent with MLX5_OPCODE_ENHANCED_MPSW
1103 * may be inlined in Data Segment, align the
1104 * length accordingly to fit entire WQEBBs.
1106 temp
= RTE_MAX(inlen_empw
,
1107 MLX5_WQE_SIZE
+ MLX5_DSEG_MIN_INLINE_SIZE
);
1108 temp
-= MLX5_DSEG_MIN_INLINE_SIZE
;
1109 temp
= RTE_ALIGN(temp
, MLX5_WQE_SIZE
);
1110 temp
+= MLX5_DSEG_MIN_INLINE_SIZE
;
1111 temp
= RTE_MIN(temp
, MLX5_WQE_SIZE_MAX
+
1112 MLX5_DSEG_MIN_INLINE_SIZE
-
1113 MLX5_WQE_CSEG_SIZE
-
1114 MLX5_WQE_ESEG_SIZE
-
1115 MLX5_WQE_DSEG_SIZE
);
1116 temp
= RTE_MIN(temp
, MLX5_EMPW_MAX_INLINE_LEN
);
1117 if (temp
!= inlen_empw
) {
1119 "port %u enhanced empw inline setting"
1120 " aligned from %u to %u",
1121 PORT_ID(priv
), inlen_empw
, temp
);
1124 MLX5_ASSERT(inlen_empw
>= MLX5_ESEG_MIN_INLINE_SIZE
);
1125 MLX5_ASSERT(inlen_empw
<= MLX5_WQE_SIZE_MAX
+
1126 MLX5_DSEG_MIN_INLINE_SIZE
-
1127 MLX5_WQE_CSEG_SIZE
-
1128 MLX5_WQE_ESEG_SIZE
-
1129 MLX5_WQE_DSEG_SIZE
);
1130 txq_ctrl
->txq
.inlen_empw
= inlen_empw
;
1132 txq_ctrl
->max_inline_data
= RTE_MAX(inlen_send
, inlen_empw
);
1134 txq_ctrl
->max_tso_header
= MLX5_MAX_TSO_HEADER
;
1135 txq_ctrl
->max_inline_data
= RTE_MAX(txq_ctrl
->max_inline_data
,
1136 MLX5_MAX_TSO_HEADER
);
1137 txq_ctrl
->txq
.tso_en
= 1;
1139 txq_ctrl
->txq
.tunnel_en
= config
->tunnel_en
| config
->swp
;
1140 txq_ctrl
->txq
.swp_en
= ((DEV_TX_OFFLOAD_IP_TNL_TSO
|
1141 DEV_TX_OFFLOAD_UDP_TNL_TSO
|
1142 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
) &
1143 txq_ctrl
->txq
.offloads
) && config
->swp
;
1147 * Adjust Tx queue data inline parameters for large queue sizes.
1148 * The data inline feature requires multiple WQEs to fit the packets,
1149 * and if the large amount of Tx descriptors is requested by application
1150 * the total WQE amount may exceed the hardware capabilities. If the
1151 * default inline setting are used we can try to adjust these ones and
1152 * meet the hardware requirements and not exceed the queue size.
1155 * Pointer to Tx queue control structure.
1158 * Zero on success, otherwise the parameters can not be adjusted.
1161 txq_adjust_params(struct mlx5_txq_ctrl
*txq_ctrl
)
1163 struct mlx5_priv
*priv
= txq_ctrl
->priv
;
1164 struct mlx5_dev_config
*config
= &priv
->config
;
1165 unsigned int max_inline
;
1167 max_inline
= txq_calc_inline_max(txq_ctrl
);
1168 if (!txq_ctrl
->txq
.inlen_send
) {
1170 * Inline data feature is not engaged at all.
1171 * There is nothing to adjust.
1175 if (txq_ctrl
->max_inline_data
<= max_inline
) {
1177 * The requested inline data length does not
1178 * exceed queue capabilities.
1182 if (txq_ctrl
->txq
.inlen_mode
> max_inline
) {
1184 "minimal data inline requirements (%u) are not"
1185 " satisfied (%u) on port %u, try the smaller"
1186 " Tx queue size (%d)",
1187 txq_ctrl
->txq
.inlen_mode
, max_inline
,
1188 priv
->dev_data
->port_id
,
1189 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
);
1192 if (txq_ctrl
->txq
.inlen_send
> max_inline
&&
1193 config
->txq_inline_max
!= MLX5_ARG_UNSET
&&
1194 config
->txq_inline_max
> (int)max_inline
) {
1196 "txq_inline_max requirements (%u) are not"
1197 " satisfied (%u) on port %u, try the smaller"
1198 " Tx queue size (%d)",
1199 txq_ctrl
->txq
.inlen_send
, max_inline
,
1200 priv
->dev_data
->port_id
,
1201 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
);
1204 if (txq_ctrl
->txq
.inlen_empw
> max_inline
&&
1205 config
->txq_inline_mpw
!= MLX5_ARG_UNSET
&&
1206 config
->txq_inline_mpw
> (int)max_inline
) {
1208 "txq_inline_mpw requirements (%u) are not"
1209 " satisfied (%u) on port %u, try the smaller"
1210 " Tx queue size (%d)",
1211 txq_ctrl
->txq
.inlen_empw
, max_inline
,
1212 priv
->dev_data
->port_id
,
1213 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
);
1216 if (txq_ctrl
->txq
.tso_en
&& max_inline
< MLX5_MAX_TSO_HEADER
) {
1218 "tso header inline requirements (%u) are not"
1219 " satisfied (%u) on port %u, try the smaller"
1220 " Tx queue size (%d)",
1221 MLX5_MAX_TSO_HEADER
, max_inline
,
1222 priv
->dev_data
->port_id
,
1223 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
);
1226 if (txq_ctrl
->txq
.inlen_send
> max_inline
) {
1228 "adjust txq_inline_max (%u->%u)"
1229 " due to large Tx queue on port %u",
1230 txq_ctrl
->txq
.inlen_send
, max_inline
,
1231 priv
->dev_data
->port_id
);
1232 txq_ctrl
->txq
.inlen_send
= max_inline
;
1234 if (txq_ctrl
->txq
.inlen_empw
> max_inline
) {
1236 "adjust txq_inline_mpw (%u->%u)"
1237 "due to large Tx queue on port %u",
1238 txq_ctrl
->txq
.inlen_empw
, max_inline
,
1239 priv
->dev_data
->port_id
);
1240 txq_ctrl
->txq
.inlen_empw
= max_inline
;
1242 txq_ctrl
->max_inline_data
= RTE_MAX(txq_ctrl
->txq
.inlen_send
,
1243 txq_ctrl
->txq
.inlen_empw
);
1244 MLX5_ASSERT(txq_ctrl
->max_inline_data
<= max_inline
);
1245 MLX5_ASSERT(txq_ctrl
->txq
.inlen_mode
<= max_inline
);
1246 MLX5_ASSERT(txq_ctrl
->txq
.inlen_mode
<= txq_ctrl
->txq
.inlen_send
);
1247 MLX5_ASSERT(txq_ctrl
->txq
.inlen_mode
<= txq_ctrl
->txq
.inlen_empw
||
1248 !txq_ctrl
->txq
.inlen_empw
);
1256 * Create a DPDK Tx queue.
1259 * Pointer to Ethernet device.
1263 * Number of descriptors to configure in queue.
1265 * NUMA socket on which memory must be allocated.
1267 * Thresholds parameters.
1270 * A DPDK queue object on success, NULL otherwise and rte_errno is set.
1272 struct mlx5_txq_ctrl
*
1273 mlx5_txq_new(struct rte_eth_dev
*dev
, uint16_t idx
, uint16_t desc
,
1274 unsigned int socket
, const struct rte_eth_txconf
*conf
)
1276 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1277 struct mlx5_txq_ctrl
*tmpl
;
1279 tmpl
= rte_calloc_socket("TXQ", 1,
1281 desc
* sizeof(struct rte_mbuf
*),
1287 if (mlx5_mr_btree_init(&tmpl
->txq
.mr_ctrl
.cache_bh
,
1288 MLX5_MR_BTREE_CACHE_N
, socket
)) {
1289 /* rte_errno is already set. */
1292 /* Save pointer of global generation number to check memory event. */
1293 tmpl
->txq
.mr_ctrl
.dev_gen_ptr
= &priv
->sh
->share_cache
.dev_gen
;
1294 MLX5_ASSERT(desc
> MLX5_TX_COMP_THRESH
);
1295 tmpl
->txq
.offloads
= conf
->offloads
|
1296 dev
->data
->dev_conf
.txmode
.offloads
;
1298 tmpl
->socket
= socket
;
1299 tmpl
->txq
.elts_n
= log2above(desc
);
1300 tmpl
->txq
.elts_s
= desc
;
1301 tmpl
->txq
.elts_m
= desc
- 1;
1302 tmpl
->txq
.port_id
= dev
->data
->port_id
;
1303 tmpl
->txq
.idx
= idx
;
1304 txq_set_params(tmpl
);
1305 if (txq_adjust_params(tmpl
))
1307 if (txq_calc_wqebb_cnt(tmpl
) >
1308 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
) {
1310 "port %u Tx WQEBB count (%d) exceeds the limit (%d),"
1311 " try smaller queue size",
1312 dev
->data
->port_id
, txq_calc_wqebb_cnt(tmpl
),
1313 priv
->sh
->device_attr
.orig_attr
.max_qp_wr
);
1317 rte_atomic32_inc(&tmpl
->refcnt
);
1318 tmpl
->type
= MLX5_TXQ_TYPE_STANDARD
;
1319 LIST_INSERT_HEAD(&priv
->txqsctrl
, tmpl
, next
);
1327 * Create a DPDK Tx hairpin queue.
1330 * Pointer to Ethernet device.
1334 * Number of descriptors to configure in queue.
1335 * @param hairpin_conf
1336 * The hairpin configuration.
1339 * A DPDK queue object on success, NULL otherwise and rte_errno is set.
1341 struct mlx5_txq_ctrl
*
1342 mlx5_txq_hairpin_new(struct rte_eth_dev
*dev
, uint16_t idx
, uint16_t desc
,
1343 const struct rte_eth_hairpin_conf
*hairpin_conf
)
1345 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1346 struct mlx5_txq_ctrl
*tmpl
;
1348 tmpl
= rte_calloc_socket("TXQ", 1,
1349 sizeof(*tmpl
), 0, SOCKET_ID_ANY
);
1355 tmpl
->socket
= SOCKET_ID_ANY
;
1356 tmpl
->txq
.elts_n
= log2above(desc
);
1357 tmpl
->txq
.port_id
= dev
->data
->port_id
;
1358 tmpl
->txq
.idx
= idx
;
1359 tmpl
->hairpin_conf
= *hairpin_conf
;
1360 tmpl
->type
= MLX5_TXQ_TYPE_HAIRPIN
;
1361 rte_atomic32_inc(&tmpl
->refcnt
);
1362 LIST_INSERT_HEAD(&priv
->txqsctrl
, tmpl
, next
);
1370 * Pointer to Ethernet device.
1375 * A pointer to the queue if it exists.
1377 struct mlx5_txq_ctrl
*
1378 mlx5_txq_get(struct rte_eth_dev
*dev
, uint16_t idx
)
1380 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1381 struct mlx5_txq_ctrl
*ctrl
= NULL
;
1383 if ((*priv
->txqs
)[idx
]) {
1384 ctrl
= container_of((*priv
->txqs
)[idx
], struct mlx5_txq_ctrl
,
1386 mlx5_txq_obj_get(dev
, idx
);
1387 rte_atomic32_inc(&ctrl
->refcnt
);
1393 * Release a Tx queue.
1396 * Pointer to Ethernet device.
1401 * 1 while a reference on it exists, 0 when freed.
1404 mlx5_txq_release(struct rte_eth_dev
*dev
, uint16_t idx
)
1406 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1407 struct mlx5_txq_ctrl
*txq
;
1409 if (!(*priv
->txqs
)[idx
])
1411 txq
= container_of((*priv
->txqs
)[idx
], struct mlx5_txq_ctrl
, txq
);
1412 if (txq
->obj
&& !mlx5_txq_obj_release(txq
->obj
))
1414 if (rte_atomic32_dec_and_test(&txq
->refcnt
)) {
1416 mlx5_mr_btree_free(&txq
->txq
.mr_ctrl
.cache_bh
);
1417 LIST_REMOVE(txq
, next
);
1419 (*priv
->txqs
)[idx
] = NULL
;
1426 * Verify if the queue can be released.
1429 * Pointer to Ethernet device.
1434 * 1 if the queue can be released.
1437 mlx5_txq_releasable(struct rte_eth_dev
*dev
, uint16_t idx
)
1439 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1440 struct mlx5_txq_ctrl
*txq
;
1442 if (!(*priv
->txqs
)[idx
])
1444 txq
= container_of((*priv
->txqs
)[idx
], struct mlx5_txq_ctrl
, txq
);
1445 return (rte_atomic32_read(&txq
->refcnt
) == 1);
1449 * Verify the Tx Queue list is empty
1452 * Pointer to Ethernet device.
1455 * The number of object not released.
1458 mlx5_txq_verify(struct rte_eth_dev
*dev
)
1460 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1461 struct mlx5_txq_ctrl
*txq_ctrl
;
1464 LIST_FOREACH(txq_ctrl
, &priv
->txqsctrl
, next
) {
1465 DRV_LOG(DEBUG
, "port %u Tx queue %u still referenced",
1466 dev
->data
->port_id
, txq_ctrl
->txq
.idx
);