1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
8 * Tx queues configuration for mlx4 driver.
18 /* Verbs headers do not support -pedantic. */
20 #pragma GCC diagnostic ignored "-Wpedantic"
22 #include <infiniband/verbs.h>
24 #pragma GCC diagnostic error "-Wpedantic"
27 #include <rte_common.h>
28 #include <rte_errno.h>
29 #include <rte_ethdev_driver.h>
30 #include <rte_malloc.h>
32 #include <rte_mempool.h>
35 #include "mlx4_glue.h"
37 #include "mlx4_rxtx.h"
38 #include "mlx4_utils.h"
41 * Free Tx queue elements.
44 * Pointer to Tx queue structure.
47 mlx4_txq_free_elts(struct txq
*txq
)
49 unsigned int elts_head
= txq
->elts_head
;
50 unsigned int elts_tail
= txq
->elts_tail
;
51 struct txq_elt (*elts
)[txq
->elts_n
] = txq
->elts
;
52 unsigned int elts_m
= txq
->elts_n
- 1;
54 DEBUG("%p: freeing WRs", (void *)txq
);
55 while (elts_tail
!= elts_head
) {
56 struct txq_elt
*elt
= &(*elts
)[elts_tail
++ & elts_m
];
58 assert(elt
->buf
!= NULL
);
59 rte_pktmbuf_free(elt
->buf
);
63 txq
->elts_tail
= txq
->elts_head
;
67 * Retrieves information needed in order to directly access the Tx queue.
70 * Pointer to Tx queue structure.
72 * Pointer to device information for this Tx queue.
75 mlx4_txq_fill_dv_obj_info(struct txq
*txq
, struct mlx4dv_obj
*mlxdv
)
77 struct mlx4_sq
*sq
= &txq
->msq
;
78 struct mlx4_cq
*cq
= &txq
->mcq
;
79 struct mlx4dv_qp
*dqp
= mlxdv
->qp
.out
;
80 struct mlx4dv_cq
*dcq
= mlxdv
->cq
.out
;
82 /* Total length, including headroom and spare WQEs. */
83 sq
->size
= (uint32_t)dqp
->rq
.offset
- (uint32_t)dqp
->sq
.offset
;
84 sq
->buf
= (uint8_t *)dqp
->buf
.buf
+ dqp
->sq
.offset
;
85 sq
->eob
= sq
->buf
+ sq
->size
;
86 uint32_t headroom_size
= 2048 + (1 << dqp
->sq
.wqe_shift
);
87 /* Continuous headroom size bytes must always stay freed. */
88 sq
->remain_size
= sq
->size
- headroom_size
;
89 sq
->owner_opcode
= MLX4_OPCODE_SEND
| (0u << MLX4_SQ_OWNER_BIT
);
90 sq
->stamp
= rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL
|
91 (0u << MLX4_SQ_OWNER_BIT
));
93 sq
->doorbell_qpn
= dqp
->doorbell_qpn
;
94 cq
->buf
= dcq
->buf
.buf
;
95 cq
->cqe_cnt
= dcq
->cqe_cnt
;
96 cq
->set_ci_db
= dcq
->set_ci_db
;
97 cq
->cqe_64
= (dcq
->cqe_size
& 64) ? 1 : 0;
101 * Returns the per-port supported offloads.
104 * Pointer to private structure.
107 * Supported Tx offloads.
110 mlx4_get_tx_port_offloads(struct priv
*priv
)
112 uint64_t offloads
= DEV_TX_OFFLOAD_MULTI_SEGS
;
115 offloads
|= (DEV_TX_OFFLOAD_IPV4_CKSUM
|
116 DEV_TX_OFFLOAD_UDP_CKSUM
|
117 DEV_TX_OFFLOAD_TCP_CKSUM
);
120 offloads
|= DEV_TX_OFFLOAD_TCP_TSO
;
121 if (priv
->hw_csum_l2tun
) {
122 offloads
|= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
;
124 offloads
|= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO
|
125 DEV_TX_OFFLOAD_GRE_TNL_TSO
);
131 * DPDK callback to configure a Tx queue.
134 * Pointer to Ethernet device structure.
138 * Number of descriptors to configure in queue.
140 * NUMA socket on which memory must be allocated.
142 * Thresholds parameters.
145 * 0 on success, negative errno value otherwise and rte_errno is set.
148 mlx4_tx_queue_setup(struct rte_eth_dev
*dev
, uint16_t idx
, uint16_t desc
,
149 unsigned int socket
, const struct rte_eth_txconf
*conf
)
151 struct priv
*priv
= dev
->data
->dev_private
;
152 struct mlx4dv_obj mlxdv
;
153 struct mlx4dv_qp dv_qp
;
154 struct mlx4dv_cq dv_cq
;
155 struct txq_elt (*elts
)[rte_align32pow2(desc
)];
156 struct ibv_qp_init_attr qp_init_attr
;
159 struct mlx4_malloc_vec vec
[] = {
161 .align
= RTE_CACHE_LINE_SIZE
,
162 .size
= sizeof(*txq
),
163 .addr
= (void **)&txq
,
166 .align
= RTE_CACHE_LINE_SIZE
,
167 .size
= sizeof(*elts
),
168 .addr
= (void **)&elts
,
171 .align
= RTE_CACHE_LINE_SIZE
,
172 .size
= MLX4_MAX_WQE_SIZE
,
173 .addr
= (void **)&bounce_buf
,
179 offloads
= conf
->offloads
| dev
->data
->dev_conf
.txmode
.offloads
;
181 DEBUG("%p: configuring queue %u for %u descriptors",
182 (void *)dev
, idx
, desc
);
184 if (idx
>= dev
->data
->nb_tx_queues
) {
185 rte_errno
= EOVERFLOW
;
186 ERROR("%p: queue index out of range (%u >= %u)",
187 (void *)dev
, idx
, dev
->data
->nb_tx_queues
);
190 txq
= dev
->data
->tx_queues
[idx
];
193 DEBUG("%p: Tx queue %u already configured, release it first",
199 ERROR("%p: invalid number of Tx descriptors", (void *)dev
);
202 if (desc
!= RTE_DIM(*elts
)) {
203 desc
= RTE_DIM(*elts
);
204 WARN("%p: increased number of descriptors in Tx queue %u"
205 " to the next power of two (%u)",
206 (void *)dev
, idx
, desc
);
208 /* Allocate and initialize Tx queue. */
209 mlx4_zmallocv_socket("TXQ", vec
, RTE_DIM(vec
), socket
);
211 ERROR("%p: unable to allocate queue index %u",
226 * Request send completion every MLX4_PMD_TX_PER_COMP_REQ
227 * packets or at least 4 times per ring.
230 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ
, desc
/ 4),
232 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ
, desc
/ 4),
233 .csum
= priv
->hw_csum
&&
234 (offloads
& (DEV_TX_OFFLOAD_IPV4_CKSUM
|
235 DEV_TX_OFFLOAD_UDP_CKSUM
|
236 DEV_TX_OFFLOAD_TCP_CKSUM
)),
237 .csum_l2tun
= priv
->hw_csum_l2tun
&&
239 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
),
240 /* Enable Tx loopback for VF devices. */
242 .bounce_buf
= bounce_buf
,
244 txq
->cq
= mlx4_glue
->create_cq(priv
->ctx
, desc
, NULL
, NULL
, 0);
247 ERROR("%p: CQ creation failure: %s",
248 (void *)dev
, strerror(rte_errno
));
251 qp_init_attr
= (struct ibv_qp_init_attr
){
256 RTE_MIN(priv
->device_attr
.max_qp_wr
, desc
),
258 .max_inline_data
= MLX4_PMD_MAX_INLINE
,
260 .qp_type
= IBV_QPT_RAW_PACKET
,
261 /* No completion events must occur by default. */
264 txq
->qp
= mlx4_glue
->create_qp(priv
->pd
, &qp_init_attr
);
266 rte_errno
= errno
? errno
: EINVAL
;
267 ERROR("%p: QP creation failure: %s",
268 (void *)dev
, strerror(rte_errno
));
271 txq
->max_inline
= qp_init_attr
.cap
.max_inline_data
;
272 ret
= mlx4_glue
->modify_qp
274 &(struct ibv_qp_attr
){
275 .qp_state
= IBV_QPS_INIT
,
276 .port_num
= priv
->port
,
278 IBV_QP_STATE
| IBV_QP_PORT
);
281 ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
282 (void *)dev
, strerror(rte_errno
));
285 ret
= mlx4_glue
->modify_qp
287 &(struct ibv_qp_attr
){
288 .qp_state
= IBV_QPS_RTR
,
293 ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
294 (void *)dev
, strerror(rte_errno
));
297 ret
= mlx4_glue
->modify_qp
299 &(struct ibv_qp_attr
){
300 .qp_state
= IBV_QPS_RTS
,
305 ERROR("%p: QP state to IBV_QPS_RTS failed: %s",
306 (void *)dev
, strerror(rte_errno
));
309 /* Retrieve device queue information. */
310 mlxdv
.cq
.in
= txq
->cq
;
311 mlxdv
.cq
.out
= &dv_cq
;
312 mlxdv
.qp
.in
= txq
->qp
;
313 mlxdv
.qp
.out
= &dv_qp
;
314 ret
= mlx4_glue
->dv_init_obj(&mlxdv
, MLX4DV_OBJ_QP
| MLX4DV_OBJ_CQ
);
317 ERROR("%p: failed to obtain information needed for"
318 " accessing the device queues", (void *)dev
);
321 mlx4_txq_fill_dv_obj_info(txq
, &mlxdv
);
322 /* Save first wqe pointer in the first element. */
323 (&(*txq
->elts
)[0])->wqe
=
324 (volatile struct mlx4_wqe_ctrl_seg
*)txq
->msq
.buf
;
325 if (mlx4_mr_btree_init(&txq
->mr_ctrl
.cache_bh
,
326 MLX4_MR_BTREE_CACHE_N
, socket
)) {
327 /* rte_errno is already set. */
330 /* Save pointer of global generation number to check memory event. */
331 txq
->mr_ctrl
.dev_gen_ptr
= &priv
->mr
.dev_gen
;
332 DEBUG("%p: adding Tx queue %p to list", (void *)dev
, (void *)txq
);
333 dev
->data
->tx_queues
[idx
] = txq
;
336 dev
->data
->tx_queues
[idx
] = NULL
;
338 mlx4_tx_queue_release(txq
);
340 assert(rte_errno
> 0);
345 * DPDK callback to release a Tx queue.
348 * Generic Tx queue pointer.
351 mlx4_tx_queue_release(void *dpdk_txq
)
353 struct txq
*txq
= (struct txq
*)dpdk_txq
;
360 for (i
= 0; i
!= priv
->dev
->data
->nb_tx_queues
; ++i
)
361 if (priv
->dev
->data
->tx_queues
[i
] == txq
) {
362 DEBUG("%p: removing Tx queue %p from list",
363 (void *)priv
->dev
, (void *)txq
);
364 priv
->dev
->data
->tx_queues
[i
] = NULL
;
367 mlx4_txq_free_elts(txq
);
369 claim_zero(mlx4_glue
->destroy_qp(txq
->qp
));
371 claim_zero(mlx4_glue
->destroy_cq(txq
->cq
));
372 mlx4_mr_btree_free(&txq
->mr_ctrl
.cache_bh
);