4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
37 #pragma GCC diagnostic ignored "-Wpedantic"
39 #include <infiniband/verbs.h>
41 #pragma GCC diagnostic error "-Wpedantic"
44 /* DPDK headers don't like -pedantic. */
46 #pragma GCC diagnostic ignored "-Wpedantic"
48 #include <rte_mempool.h>
50 #pragma GCC diagnostic error "-Wpedantic"
54 #include "mlx5_rxtx.h"
56 struct mlx5_check_mempool_data
{
62 /* Called by mlx5_check_mempool() when iterating the memory chunks. */
64 mlx5_check_mempool_cb(struct rte_mempool
*mp
,
65 void *opaque
, struct rte_mempool_memhdr
*memhdr
,
68 struct mlx5_check_mempool_data
*data
= opaque
;
73 /* It already failed, skip the next chunks. */
76 /* It is the first chunk. */
77 if (data
->start
== NULL
&& data
->end
== NULL
) {
78 data
->start
= memhdr
->addr
;
79 data
->end
= data
->start
+ memhdr
->len
;
82 if (data
->end
== memhdr
->addr
) {
83 data
->end
+= memhdr
->len
;
86 if (data
->start
== (char *)memhdr
->addr
+ memhdr
->len
) {
87 data
->start
-= memhdr
->len
;
90 /* Error, mempool is not virtually contiguous. */
95 * Check if a mempool can be used: it must be virtually contiguous.
98 * Pointer to memory pool.
100 * Pointer to the start address of the mempool virtual memory area
102 * Pointer to the end address of the mempool virtual memory area
105 * 0 on success (mempool is virtually contiguous), -1 on error.
107 static int mlx5_check_mempool(struct rte_mempool
*mp
, uintptr_t *start
,
110 struct mlx5_check_mempool_data data
;
112 memset(&data
, 0, sizeof(data
));
113 rte_mempool_mem_iter(mp
, mlx5_check_mempool_cb
, &data
);
114 *start
= (uintptr_t)data
.start
;
115 *end
= (uintptr_t)data
.end
;
121 * Register mempool as a memory region.
124 * Pointer to protection domain.
126 * Pointer to memory pool.
129 * Memory region pointer, NULL in case of error.
132 mlx5_mp2mr(struct ibv_pd
*pd
, struct rte_mempool
*mp
)
134 const struct rte_memseg
*ms
= rte_eal_get_physmem_layout();
139 if (mlx5_check_mempool(mp
, &start
, &end
) != 0) {
140 ERROR("mempool %p: not virtually contiguous",
145 DEBUG("mempool %p area start=%p end=%p size=%zu",
146 (void *)mp
, (void *)start
, (void *)end
,
147 (size_t)(end
- start
));
148 /* Round start and end to page boundary if found in memory segments. */
149 for (i
= 0; (i
< RTE_MAX_MEMSEG
) && (ms
[i
].addr
!= NULL
); ++i
) {
150 uintptr_t addr
= (uintptr_t)ms
[i
].addr
;
151 size_t len
= ms
[i
].len
;
152 unsigned int align
= ms
[i
].hugepage_sz
;
154 if ((start
> addr
) && (start
< addr
+ len
))
155 start
= RTE_ALIGN_FLOOR(start
, align
);
156 if ((end
> addr
) && (end
< addr
+ len
))
157 end
= RTE_ALIGN_CEIL(end
, align
);
159 DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
160 (void *)mp
, (void *)start
, (void *)end
,
161 (size_t)(end
- start
));
162 return ibv_reg_mr(pd
,
165 IBV_ACCESS_LOCAL_WRITE
);
169 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
170 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
172 * This function should only be called by txq_mp2mr().
175 * Pointer to TX queue structure.
177 * Memory Pool for which a Memory Region lkey must be returned.
179 * Index of the next available entry.
182 * mr->lkey on success, (uint32_t)-1 on failure.
185 txq_mp2mr_reg(struct txq
*txq
, struct rte_mempool
*mp
, unsigned int idx
)
187 struct txq_ctrl
*txq_ctrl
= container_of(txq
, struct txq_ctrl
, txq
);
190 /* Add a new entry, register MR first. */
191 DEBUG("%p: discovered new memory pool \"%s\" (%p)",
192 (void *)txq_ctrl
, mp
->name
, (void *)mp
);
193 mr
= mlx5_mp2mr(txq_ctrl
->priv
->pd
, mp
);
194 if (unlikely(mr
== NULL
)) {
195 DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
199 if (unlikely(idx
== RTE_DIM(txq_ctrl
->txq
.mp2mr
))) {
200 /* Table is full, remove oldest entry. */
201 DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
204 claim_zero(ibv_dereg_mr(txq_ctrl
->txq
.mp2mr
[0].mr
));
205 memmove(&txq_ctrl
->txq
.mp2mr
[0], &txq_ctrl
->txq
.mp2mr
[1],
206 (sizeof(txq_ctrl
->txq
.mp2mr
) -
207 sizeof(txq_ctrl
->txq
.mp2mr
[0])));
209 /* Store the new entry. */
210 txq_ctrl
->txq
.mp2mr
[idx
].mp
= mp
;
211 txq_ctrl
->txq
.mp2mr
[idx
].mr
= mr
;
212 txq_ctrl
->txq
.mp2mr
[idx
].lkey
= htonl(mr
->lkey
);
213 DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32
,
214 (void *)txq_ctrl
, mp
->name
, (void *)mp
,
215 txq_ctrl
->txq
.mp2mr
[idx
].lkey
);
216 return txq_ctrl
->txq
.mp2mr
[idx
].lkey
;
219 struct txq_mp2mr_mbuf_check_data
{
224 * Callback function for rte_mempool_obj_iter() to check whether a given
225 * mempool object looks like a mbuf.
228 * The mempool pointer
230 * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
235 * Object index, unused.
238 txq_mp2mr_mbuf_check(struct rte_mempool
*mp
, void *arg
, void *obj
,
239 uint32_t index __rte_unused
)
241 struct txq_mp2mr_mbuf_check_data
*data
= arg
;
242 struct rte_mbuf
*buf
= obj
;
245 * Check whether mbuf structure fits element size and whether mempool
248 if (sizeof(*buf
) > mp
->elt_size
|| buf
->pool
!= mp
)
253 * Iterator function for rte_mempool_walk() to register existing mempools and
254 * fill the MP to MR cache of a TX queue.
257 * Memory Pool to register.
259 * Pointer to TX queue structure.
262 txq_mp2mr_iter(struct rte_mempool
*mp
, void *arg
)
264 struct txq_ctrl
*txq_ctrl
= arg
;
265 struct txq_mp2mr_mbuf_check_data data
= {
270 /* Register mempool only if the first element looks like a mbuf. */
271 if (rte_mempool_obj_iter(mp
, txq_mp2mr_mbuf_check
, &data
) == 0 ||
274 for (i
= 0; (i
!= RTE_DIM(txq_ctrl
->txq
.mp2mr
)); ++i
) {
275 if (unlikely(txq_ctrl
->txq
.mp2mr
[i
].mp
== NULL
)) {
276 /* Unknown MP, add a new MR for it. */
279 if (txq_ctrl
->txq
.mp2mr
[i
].mp
== mp
)
282 txq_mp2mr_reg(&txq_ctrl
->txq
, mp
, i
);