1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 6WIND S.A.
3 * Copyright 2018 Mellanox Technologies, Ltd
6 #ifndef RTE_PMD_MLX5_MR_H_
7 #define RTE_PMD_MLX5_MR_H_
11 #include <sys/queue.h>
14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
16 #pragma GCC diagnostic ignored "-Wpedantic"
18 #include <infiniband/verbs.h>
19 #include <infiniband/mlx5dv.h>
21 #pragma GCC diagnostic error "-Wpedantic"
24 #include <rte_eal_memconfig.h>
25 #include <rte_ethdev.h>
26 #include <rte_rwlock.h>
27 #include <rte_bitmap.h>
29 /* Memory Region object. */
31 LIST_ENTRY(mlx5_mr
) mr
; /**< Pointer to the prev/next entry. */
32 struct ibv_mr
*ibv_mr
; /* Verbs Memory Region. */
33 const struct rte_memseg_list
*msl
;
34 int ms_base_idx
; /* Start index of msl->memseg_arr[]. */
35 int ms_n
; /* Number of memsegs in use. */
36 uint32_t ms_bmp_n
; /* Number of bits in memsegs bit-mask. */
37 struct rte_bitmap
*ms_bmp
; /* Bit-mask of memsegs belonged to MR. */
40 /* Cache entry for Memory Region. */
41 struct mlx5_mr_cache
{
42 uintptr_t start
; /* Start address of MR. */
43 uintptr_t end
; /* End address of MR. */
44 uint32_t lkey
; /* rte_cpu_to_be_32(ibv_mr->lkey). */
47 /* MR Cache table for Binary search. */
48 struct mlx5_mr_btree
{
49 uint16_t len
; /* Number of entries. */
50 uint16_t size
; /* Total number of entries. */
51 int overflow
; /* Mark failure of table expansion. */
52 struct mlx5_mr_cache (*table
)[];
55 /* Per-queue MR control descriptor. */
57 uint32_t *dev_gen_ptr
; /* Generation number of device to poll. */
58 uint32_t cur_gen
; /* Generation number saved to flush caches. */
59 uint16_t mru
; /* Index of last hit entry in top-half cache. */
60 uint16_t head
; /* Index of the oldest entry in top-half cache. */
61 struct mlx5_mr_cache cache
[MLX5_MR_CACHE_N
]; /* Cache for top-half. */
62 struct mlx5_mr_btree cache_bh
; /* Cache for bottom-half. */
65 struct mlx5_ibv_shared
;
66 extern struct mlx5_dev_list mlx5_mem_event_cb_list
;
67 extern rte_rwlock_t mlx5_mem_event_rwlock
;
69 /* First entry must be NULL for comparison. */
70 #define mlx5_mr_btree_len(bt) ((bt)->len - 1)
72 int mlx5_mr_btree_init(struct mlx5_mr_btree
*bt
, int n
, int socket
);
73 void mlx5_mr_btree_free(struct mlx5_mr_btree
*bt
);
74 uint32_t mlx5_mr_create_primary(struct rte_eth_dev
*dev
,
75 struct mlx5_mr_cache
*entry
, uintptr_t addr
);
76 void mlx5_mr_mem_event_cb(enum rte_mem_event event_type
, const void *addr
,
77 size_t len
, void *arg
);
78 int mlx5_mr_update_mp(struct rte_eth_dev
*dev
, struct mlx5_mr_ctrl
*mr_ctrl
,
79 struct rte_mempool
*mp
);
80 void mlx5_mr_release(struct mlx5_ibv_shared
*sh
);
82 /* Debug purpose functions. */
83 void mlx5_mr_btree_dump(struct mlx5_mr_btree
*bt
);
84 void mlx5_mr_dump_dev(struct mlx5_ibv_shared
*sh
);
87 * Look up LKey from given lookup table by linear search. Firstly look up the
88 * last-hit entry. If miss, the entire array is searched. If found, update the
89 * last-hit index and return LKey.
92 * Pointer to lookup table.
93 * @param[in,out] cached_idx
94 * Pointer to last-hit index.
96 * Size of lookup table.
101 * Searched LKey on success, UINT32_MAX on no match.
103 static __rte_always_inline
uint32_t
104 mlx5_mr_lookup_cache(struct mlx5_mr_cache
*lkp_tbl
, uint16_t *cached_idx
,
105 uint16_t n
, uintptr_t addr
)
109 if (likely(addr
>= lkp_tbl
[*cached_idx
].start
&&
110 addr
< lkp_tbl
[*cached_idx
].end
))
111 return lkp_tbl
[*cached_idx
].lkey
;
112 for (idx
= 0; idx
< n
&& lkp_tbl
[idx
].start
!= 0; ++idx
) {
113 if (addr
>= lkp_tbl
[idx
].start
&&
114 addr
< lkp_tbl
[idx
].end
) {
117 return lkp_tbl
[idx
].lkey
;
123 #endif /* RTE_PMD_MLX5_MR_H_ */