]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright 2018 6WIND S.A. | |
3 | * Copyright 2018 Mellanox Technologies, Ltd | |
4 | */ | |
5 | ||
6 | #ifndef RTE_PMD_MLX5_MR_H_ | |
7 | #define RTE_PMD_MLX5_MR_H_ | |
8 | ||
9 | #include <stddef.h> | |
10 | #include <stdint.h> | |
11 | #include <sys/queue.h> | |
12 | ||
13 | /* Verbs header. */ | |
14 | /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ | |
15 | #ifdef PEDANTIC | |
16 | #pragma GCC diagnostic ignored "-Wpedantic" | |
17 | #endif | |
18 | #include <infiniband/verbs.h> | |
19 | #include <infiniband/mlx5dv.h> | |
20 | #ifdef PEDANTIC | |
21 | #pragma GCC diagnostic error "-Wpedantic" | |
22 | #endif | |
23 | ||
24 | #include <rte_eal_memconfig.h> | |
25 | #include <rte_ethdev.h> | |
26 | #include <rte_rwlock.h> | |
27 | #include <rte_bitmap.h> | |
28 | ||
29 | /* Memory Region object. */ | |
30 | struct mlx5_mr { | |
31 | LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */ | |
32 | struct ibv_mr *ibv_mr; /* Verbs Memory Region. */ | |
33 | const struct rte_memseg_list *msl; | |
34 | int ms_base_idx; /* Start index of msl->memseg_arr[]. */ | |
35 | int ms_n; /* Number of memsegs in use. */ | |
36 | uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */ | |
37 | struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */ | |
38 | }; | |
39 | ||
40 | /* Cache entry for Memory Region. */ | |
41 | struct mlx5_mr_cache { | |
42 | uintptr_t start; /* Start address of MR. */ | |
43 | uintptr_t end; /* End address of MR. */ | |
44 | uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */ | |
45 | } __rte_packed; | |
46 | ||
47 | /* MR Cache table for Binary search. */ | |
48 | struct mlx5_mr_btree { | |
49 | uint16_t len; /* Number of entries. */ | |
50 | uint16_t size; /* Total number of entries. */ | |
51 | int overflow; /* Mark failure of table expansion. */ | |
52 | struct mlx5_mr_cache (*table)[]; | |
53 | } __rte_packed; | |
54 | ||
55 | /* Per-queue MR control descriptor. */ | |
56 | struct mlx5_mr_ctrl { | |
57 | uint32_t *dev_gen_ptr; /* Generation number of device to poll. */ | |
58 | uint32_t cur_gen; /* Generation number saved to flush caches. */ | |
59 | uint16_t mru; /* Index of last hit entry in top-half cache. */ | |
60 | uint16_t head; /* Index of the oldest entry in top-half cache. */ | |
61 | struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for top-half. */ | |
62 | struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */ | |
63 | } __rte_packed; | |
64 | ||
9f95a23c | 65 | struct mlx5_ibv_shared; |
11fdf7f2 TL |
66 | extern struct mlx5_dev_list mlx5_mem_event_cb_list; |
67 | extern rte_rwlock_t mlx5_mem_event_rwlock; | |
68 | ||
69 | /* First entry must be NULL for comparison. */ | |
70 | #define mlx5_mr_btree_len(bt) ((bt)->len - 1) | |
71 | ||
72 | int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket); | |
73 | void mlx5_mr_btree_free(struct mlx5_mr_btree *bt); | |
9f95a23c TL |
74 | uint32_t mlx5_mr_create_primary(struct rte_eth_dev *dev, |
75 | struct mlx5_mr_cache *entry, uintptr_t addr); | |
11fdf7f2 TL |
76 | void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr, |
77 | size_t len, void *arg); | |
78 | int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl, | |
79 | struct rte_mempool *mp); | |
9f95a23c | 80 | void mlx5_mr_release(struct mlx5_ibv_shared *sh); |
11fdf7f2 TL |
81 | |
82 | /* Debug purpose functions. */ | |
83 | void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt); | |
9f95a23c | 84 | void mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh); |
11fdf7f2 TL |
85 | |
86 | /** | |
87 | * Look up LKey from given lookup table by linear search. Firstly look up the | |
88 | * last-hit entry. If miss, the entire array is searched. If found, update the | |
89 | * last-hit index and return LKey. | |
90 | * | |
91 | * @param lkp_tbl | |
92 | * Pointer to lookup table. | |
93 | * @param[in,out] cached_idx | |
94 | * Pointer to last-hit index. | |
95 | * @param n | |
96 | * Size of lookup table. | |
97 | * @param addr | |
98 | * Search key. | |
99 | * | |
100 | * @return | |
101 | * Searched LKey on success, UINT32_MAX on no match. | |
102 | */ | |
103 | static __rte_always_inline uint32_t | |
104 | mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t *cached_idx, | |
105 | uint16_t n, uintptr_t addr) | |
106 | { | |
107 | uint16_t idx; | |
108 | ||
109 | if (likely(addr >= lkp_tbl[*cached_idx].start && | |
110 | addr < lkp_tbl[*cached_idx].end)) | |
111 | return lkp_tbl[*cached_idx].lkey; | |
112 | for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) { | |
113 | if (addr >= lkp_tbl[idx].start && | |
114 | addr < lkp_tbl[idx].end) { | |
115 | /* Found. */ | |
116 | *cached_idx = idx; | |
117 | return lkp_tbl[idx].lkey; | |
118 | } | |
119 | } | |
120 | return UINT32_MAX; | |
121 | } | |
122 | ||
123 | #endif /* RTE_PMD_MLX5_MR_H_ */ |