1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 6WIND S.A.
3 * Copyright 2018 Mellanox Technologies, Ltd
6 #ifndef RTE_PMD_MLX4_MR_H_
7 #define RTE_PMD_MLX4_MR_H_
11 #include <sys/queue.h>
13 /* Verbs headers do not support -pedantic. */
15 #pragma GCC diagnostic ignored "-Wpedantic"
17 #include <infiniband/verbs.h>
19 #pragma GCC diagnostic error "-Wpedantic"
22 #include <rte_eal_memconfig.h>
23 #include <rte_ethdev.h>
24 #include <rte_rwlock.h>
25 #include <rte_bitmap.h>
27 /* Size of per-queue MR cache array for linear search. */
28 #define MLX4_MR_CACHE_N 8
30 /* Size of MR cache table for binary search. */
31 #define MLX4_MR_BTREE_CACHE_N 256
33 /* Memory Region object. */
35 LIST_ENTRY(mlx4_mr
) mr
; /**< Pointer to the prev/next entry. */
36 struct ibv_mr
*ibv_mr
; /* Verbs Memory Region. */
37 const struct rte_memseg_list
*msl
;
38 int ms_base_idx
; /* Start index of msl->memseg_arr[]. */
39 int ms_n
; /* Number of memsegs in use. */
40 uint32_t ms_bmp_n
; /* Number of bits in memsegs bit-mask. */
41 struct rte_bitmap
*ms_bmp
; /* Bit-mask of memsegs belonged to MR. */
44 /* Cache entry for Memory Region. */
45 struct mlx4_mr_cache
{
46 uintptr_t start
; /* Start address of MR. */
47 uintptr_t end
; /* End address of MR. */
48 uint32_t lkey
; /* rte_cpu_to_be_32(ibv_mr->lkey). */
51 /* MR Cache table for Binary search. */
52 struct mlx4_mr_btree
{
53 uint16_t len
; /* Number of entries. */
54 uint16_t size
; /* Total number of entries. */
55 int overflow
; /* Mark failure of table expansion. */
56 struct mlx4_mr_cache (*table
)[];
59 /* Per-queue MR control descriptor. */
61 uint32_t *dev_gen_ptr
; /* Generation number of device to poll. */
62 uint32_t cur_gen
; /* Generation number saved to flush caches. */
63 uint16_t mru
; /* Index of last hit entry in top-half cache. */
64 uint16_t head
; /* Index of the oldest entry in top-half cache. */
65 struct mlx4_mr_cache cache
[MLX4_MR_CACHE_N
]; /* Cache for top-half. */
66 struct mlx4_mr_btree cache_bh
; /* Cache for bottom-half. */
69 extern struct mlx4_dev_list mlx4_mem_event_cb_list
;
70 extern rte_rwlock_t mlx4_mem_event_rwlock
;
72 /* First entry must be NULL for comparison. */
73 #define mlx4_mr_btree_len(bt) ((bt)->len - 1)
75 int mlx4_mr_btree_init(struct mlx4_mr_btree
*bt
, int n
, int socket
);
76 void mlx4_mr_btree_free(struct mlx4_mr_btree
*bt
);
77 void mlx4_mr_btree_dump(struct mlx4_mr_btree
*bt
);
78 uint32_t mlx4_mr_create_primary(struct rte_eth_dev
*dev
,
79 struct mlx4_mr_cache
*entry
, uintptr_t addr
);
80 void mlx4_mr_mem_event_cb(enum rte_mem_event event_type
, const void *addr
,
81 size_t len
, void *arg
);
82 int mlx4_mr_update_mp(struct rte_eth_dev
*dev
, struct mlx4_mr_ctrl
*mr_ctrl
,
83 struct rte_mempool
*mp
);
84 void mlx4_mr_dump_dev(struct rte_eth_dev
*dev
);
85 void mlx4_mr_release(struct rte_eth_dev
*dev
);
88 * Look up LKey from given lookup table by linear search. Firstly look up the
89 * last-hit entry. If miss, the entire array is searched. If found, update the
90 * last-hit index and return LKey.
93 * Pointer to lookup table.
94 * @param[in,out] cached_idx
95 * Pointer to last-hit index.
97 * Size of lookup table.
102 * Searched LKey on success, UINT32_MAX on no match.
104 static __rte_always_inline
uint32_t
105 mlx4_mr_lookup_cache(struct mlx4_mr_cache
*lkp_tbl
, uint16_t *cached_idx
,
106 uint16_t n
, uintptr_t addr
)
110 if (likely(addr
>= lkp_tbl
[*cached_idx
].start
&&
111 addr
< lkp_tbl
[*cached_idx
].end
))
112 return lkp_tbl
[*cached_idx
].lkey
;
113 for (idx
= 0; idx
< n
&& lkp_tbl
[idx
].start
!= 0; ++idx
) {
114 if (addr
>= lkp_tbl
[idx
].start
&&
115 addr
< lkp_tbl
[idx
].end
) {
118 return lkp_tbl
[idx
].lkey
;
124 #endif /* RTE_PMD_MLX4_MR_H_ */