]>
Commit | Line | Data |
---|---|---|
ef6d4ccd YS |
1 | /* |
2 | * QEMU paravirtual RDMA - Resource Manager Implementation | |
3 | * | |
4 | * Copyright (C) 2018 Oracle | |
5 | * Copyright (C) 2018 Red Hat Inc | |
6 | * | |
7 | * Authors: | |
8 | * Yuval Shaia <yuval.shaia@oracle.com> | |
9 | * Marcel Apfelbaum <marcel@redhat.com> | |
10 | * | |
11 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
12 | * See the COPYING file in the top-level directory. | |
13 | * | |
14 | */ | |
15 | ||
0efc9511 MT |
16 | #include "qemu/osdep.h" |
17 | #include "qapi/error.h" | |
18 | #include "cpu.h" | |
f4b2c02a | 19 | #include "monitor/monitor.h" |
ef6d4ccd | 20 | |
4d71b38a | 21 | #include "trace.h" |
ef6d4ccd YS |
22 | #include "rdma_utils.h" |
23 | #include "rdma_backend.h" | |
24 | #include "rdma_rm.h" | |
25 | ||
ef6d4ccd YS |
26 | /* Page directory and page tables */ |
27 | #define PG_DIR_SZ { TARGET_PAGE_SIZE / sizeof(__u64) } | |
28 | #define PG_TBL_SZ { TARGET_PAGE_SIZE / sizeof(__u64) } | |
29 | ||
f4b2c02a YS |
30 | void rdma_dump_device_counters(Monitor *mon, RdmaDeviceResources *dev_res) |
31 | { | |
32 | monitor_printf(mon, "\ttx : %" PRId64 "\n", | |
33 | dev_res->stats.tx); | |
34 | monitor_printf(mon, "\ttx_len : %" PRId64 "\n", | |
35 | dev_res->stats.tx_len); | |
36 | monitor_printf(mon, "\ttx_err : %" PRId64 "\n", | |
37 | dev_res->stats.tx_err); | |
38 | monitor_printf(mon, "\trx_bufs : %" PRId64 "\n", | |
39 | dev_res->stats.rx_bufs); | |
e926c9f1 KH |
40 | monitor_printf(mon, "\trx_srq : %" PRId64 "\n", |
41 | dev_res->stats.rx_srq); | |
f4b2c02a YS |
42 | monitor_printf(mon, "\trx_bufs_len : %" PRId64 "\n", |
43 | dev_res->stats.rx_bufs_len); | |
44 | monitor_printf(mon, "\trx_bufs_err : %" PRId64 "\n", | |
45 | dev_res->stats.rx_bufs_err); | |
46 | monitor_printf(mon, "\tcomps : %" PRId64 "\n", | |
47 | dev_res->stats.completions); | |
48 | monitor_printf(mon, "\tmissing_comps : %" PRId32 "\n", | |
49 | dev_res->stats.missing_cqe); | |
50 | monitor_printf(mon, "\tpoll_cq (bk) : %" PRId64 "\n", | |
51 | dev_res->stats.poll_cq_from_bk); | |
52 | monitor_printf(mon, "\tpoll_cq_ppoll_to : %" PRId64 "\n", | |
53 | dev_res->stats.poll_cq_ppoll_to); | |
54 | monitor_printf(mon, "\tpoll_cq (fe) : %" PRId64 "\n", | |
55 | dev_res->stats.poll_cq_from_guest); | |
56 | monitor_printf(mon, "\tpoll_cq_empty : %" PRId64 "\n", | |
57 | dev_res->stats.poll_cq_from_guest_empty); | |
58 | monitor_printf(mon, "\tmad_tx : %" PRId64 "\n", | |
59 | dev_res->stats.mad_tx); | |
60 | monitor_printf(mon, "\tmad_tx_err : %" PRId64 "\n", | |
61 | dev_res->stats.mad_tx_err); | |
62 | monitor_printf(mon, "\tmad_rx : %" PRId64 "\n", | |
63 | dev_res->stats.mad_rx); | |
64 | monitor_printf(mon, "\tmad_rx_err : %" PRId64 "\n", | |
65 | dev_res->stats.mad_rx_err); | |
66 | monitor_printf(mon, "\tmad_rx_bufs : %" PRId64 "\n", | |
67 | dev_res->stats.mad_rx_bufs); | |
68 | monitor_printf(mon, "\tmad_rx_bufs_err : %" PRId64 "\n", | |
69 | dev_res->stats.mad_rx_bufs_err); | |
70 | monitor_printf(mon, "\tPDs : %" PRId32 "\n", | |
71 | dev_res->pd_tbl.used); | |
72 | monitor_printf(mon, "\tMRs : %" PRId32 "\n", | |
73 | dev_res->mr_tbl.used); | |
74 | monitor_printf(mon, "\tUCs : %" PRId32 "\n", | |
75 | dev_res->uc_tbl.used); | |
76 | monitor_printf(mon, "\tQPs : %" PRId32 "\n", | |
77 | dev_res->qp_tbl.used); | |
78 | monitor_printf(mon, "\tCQs : %" PRId32 "\n", | |
79 | dev_res->cq_tbl.used); | |
80 | monitor_printf(mon, "\tCEQ_CTXs : %" PRId32 "\n", | |
81 | dev_res->cqe_ctx_tbl.used); | |
82 | } | |
83 | ||
ef6d4ccd YS |
84 | static inline void res_tbl_init(const char *name, RdmaRmResTbl *tbl, |
85 | uint32_t tbl_sz, uint32_t res_sz) | |
86 | { | |
87 | tbl->tbl = g_malloc(tbl_sz * res_sz); | |
88 | ||
89 | strncpy(tbl->name, name, MAX_RM_TBL_NAME); | |
90 | tbl->name[MAX_RM_TBL_NAME - 1] = 0; | |
91 | ||
92 | tbl->bitmap = bitmap_new(tbl_sz); | |
93 | tbl->tbl_sz = tbl_sz; | |
94 | tbl->res_sz = res_sz; | |
c2dd117b | 95 | tbl->used = 0; |
ef6d4ccd YS |
96 | qemu_mutex_init(&tbl->lock); |
97 | } | |
98 | ||
99 | static inline void res_tbl_free(RdmaRmResTbl *tbl) | |
100 | { | |
a1aa88b7 YS |
101 | if (!tbl->bitmap) { |
102 | return; | |
103 | } | |
ef6d4ccd YS |
104 | qemu_mutex_destroy(&tbl->lock); |
105 | g_free(tbl->tbl); | |
9a3053d2 | 106 | g_free(tbl->bitmap); |
ef6d4ccd YS |
107 | } |
108 | ||
4d71b38a | 109 | static inline void *rdma_res_tbl_get(RdmaRmResTbl *tbl, uint32_t handle) |
ef6d4ccd | 110 | { |
4d71b38a | 111 | trace_rdma_res_tbl_get(tbl->name, handle); |
ef6d4ccd YS |
112 | |
113 | if ((handle < tbl->tbl_sz) && (test_bit(handle, tbl->bitmap))) { | |
114 | return tbl->tbl + handle * tbl->res_sz; | |
115 | } else { | |
4d71b38a | 116 | rdma_error_report("Table %s, invalid handle %d", tbl->name, handle); |
ef6d4ccd YS |
117 | return NULL; |
118 | } | |
119 | } | |
120 | ||
4d71b38a | 121 | static inline void *rdma_res_tbl_alloc(RdmaRmResTbl *tbl, uint32_t *handle) |
ef6d4ccd YS |
122 | { |
123 | qemu_mutex_lock(&tbl->lock); | |
124 | ||
125 | *handle = find_first_zero_bit(tbl->bitmap, tbl->tbl_sz); | |
126 | if (*handle > tbl->tbl_sz) { | |
4d71b38a YS |
127 | rdma_error_report("Table %s, failed to allocate, bitmap is full", |
128 | tbl->name); | |
ef6d4ccd YS |
129 | qemu_mutex_unlock(&tbl->lock); |
130 | return NULL; | |
131 | } | |
132 | ||
133 | set_bit(*handle, tbl->bitmap); | |
134 | ||
c2dd117b YS |
135 | tbl->used++; |
136 | ||
ef6d4ccd YS |
137 | qemu_mutex_unlock(&tbl->lock); |
138 | ||
139 | memset(tbl->tbl + *handle * tbl->res_sz, 0, tbl->res_sz); | |
140 | ||
4d71b38a | 141 | trace_rdma_res_tbl_alloc(tbl->name, *handle); |
ef6d4ccd YS |
142 | |
143 | return tbl->tbl + *handle * tbl->res_sz; | |
144 | } | |
145 | ||
4d71b38a | 146 | static inline void rdma_res_tbl_dealloc(RdmaRmResTbl *tbl, uint32_t handle) |
ef6d4ccd | 147 | { |
4d71b38a | 148 | trace_rdma_res_tbl_dealloc(tbl->name, handle); |
ef6d4ccd | 149 | |
08b689aa | 150 | QEMU_LOCK_GUARD(&tbl->lock); |
ef6d4ccd YS |
151 | |
152 | if (handle < tbl->tbl_sz) { | |
153 | clear_bit(handle, tbl->bitmap); | |
c2dd117b | 154 | tbl->used--; |
ef6d4ccd YS |
155 | } |
156 | ||
ef6d4ccd YS |
157 | } |
158 | ||
159 | int rdma_rm_alloc_pd(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
160 | uint32_t *pd_handle, uint32_t ctx_handle) | |
161 | { | |
162 | RdmaRmPD *pd; | |
163 | int ret = -ENOMEM; | |
164 | ||
4d71b38a | 165 | pd = rdma_res_tbl_alloc(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
166 | if (!pd) { |
167 | goto out; | |
168 | } | |
169 | ||
170 | ret = rdma_backend_create_pd(backend_dev, &pd->backend_pd); | |
171 | if (ret) { | |
172 | ret = -EIO; | |
173 | goto out_tbl_dealloc; | |
174 | } | |
175 | ||
176 | pd->ctx_handle = ctx_handle; | |
177 | ||
178 | return 0; | |
179 | ||
180 | out_tbl_dealloc: | |
4d71b38a | 181 | rdma_res_tbl_dealloc(&dev_res->pd_tbl, *pd_handle); |
ef6d4ccd YS |
182 | |
183 | out: | |
184 | return ret; | |
185 | } | |
186 | ||
187 | RdmaRmPD *rdma_rm_get_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle) | |
188 | { | |
4d71b38a | 189 | return rdma_res_tbl_get(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
190 | } |
191 | ||
192 | void rdma_rm_dealloc_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle) | |
193 | { | |
194 | RdmaRmPD *pd = rdma_rm_get_pd(dev_res, pd_handle); | |
195 | ||
196 | if (pd) { | |
197 | rdma_backend_destroy_pd(&pd->backend_pd); | |
4d71b38a | 198 | rdma_res_tbl_dealloc(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
199 | } |
200 | } | |
201 | ||
202 | int rdma_rm_alloc_mr(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
4d71b38a YS |
203 | uint64_t guest_start, uint64_t guest_length, |
204 | void *host_virt, int access_flags, uint32_t *mr_handle, | |
205 | uint32_t *lkey, uint32_t *rkey) | |
ef6d4ccd YS |
206 | { |
207 | RdmaRmMR *mr; | |
208 | int ret = 0; | |
209 | RdmaRmPD *pd; | |
ef6d4ccd YS |
210 | |
211 | pd = rdma_rm_get_pd(dev_res, pd_handle); | |
212 | if (!pd) { | |
ef6d4ccd YS |
213 | return -EINVAL; |
214 | } | |
215 | ||
4d71b38a | 216 | mr = rdma_res_tbl_alloc(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd | 217 | if (!mr) { |
ef6d4ccd YS |
218 | return -ENOMEM; |
219 | } | |
4d71b38a YS |
220 | trace_rdma_rm_alloc_mr(*mr_handle, host_virt, guest_start, guest_length, |
221 | access_flags); | |
4c408c9d YS |
222 | |
223 | if (host_virt) { | |
7f99daad | 224 | mr->virt = host_virt; |
7f99daad | 225 | mr->start = guest_start; |
4c408c9d | 226 | mr->length = guest_length; |
7d2ce4b0 | 227 | mr->virt += (mr->start & (TARGET_PAGE_SIZE - 1)); |
ef6d4ccd | 228 | |
68b89aee YS |
229 | ret = rdma_backend_create_mr(&mr->backend_mr, &pd->backend_pd, mr->virt, |
230 | mr->length, guest_start, access_flags); | |
4c408c9d | 231 | if (ret) { |
4c408c9d YS |
232 | ret = -EIO; |
233 | goto out_dealloc_mr; | |
234 | } | |
b196d4f1 YS |
235 | #ifdef LEGACY_RDMA_REG_MR |
236 | /* We keep mr_handle in lkey so send and recv get get mr ptr */ | |
237 | *lkey = *mr_handle; | |
238 | #else | |
239 | *lkey = rdma_backend_mr_lkey(&mr->backend_mr); | |
240 | #endif | |
ef6d4ccd YS |
241 | } |
242 | ||
4c408c9d | 243 | *rkey = -1; |
ef6d4ccd YS |
244 | |
245 | mr->pd_handle = pd_handle; | |
246 | ||
247 | return 0; | |
248 | ||
249 | out_dealloc_mr: | |
4d71b38a | 250 | rdma_res_tbl_dealloc(&dev_res->mr_tbl, *mr_handle); |
ef6d4ccd YS |
251 | |
252 | return ret; | |
253 | } | |
254 | ||
255 | RdmaRmMR *rdma_rm_get_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle) | |
256 | { | |
4d71b38a | 257 | return rdma_res_tbl_get(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd YS |
258 | } |
259 | ||
260 | void rdma_rm_dealloc_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle) | |
261 | { | |
262 | RdmaRmMR *mr = rdma_rm_get_mr(dev_res, mr_handle); | |
263 | ||
264 | if (mr) { | |
265 | rdma_backend_destroy_mr(&mr->backend_mr); | |
4d71b38a | 266 | trace_rdma_rm_dealloc_mr(mr_handle, mr->start); |
4c408c9d | 267 | if (mr->start) { |
7d2ce4b0 | 268 | mr->virt -= (mr->start & (TARGET_PAGE_SIZE - 1)); |
4c408c9d YS |
269 | munmap(mr->virt, mr->length); |
270 | } | |
4d71b38a | 271 | rdma_res_tbl_dealloc(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd YS |
272 | } |
273 | } | |
274 | ||
275 | int rdma_rm_alloc_uc(RdmaDeviceResources *dev_res, uint32_t pfn, | |
276 | uint32_t *uc_handle) | |
277 | { | |
278 | RdmaRmUC *uc; | |
279 | ||
280 | /* TODO: Need to make sure pfn is between bar start address and | |
281 | * bsd+RDMA_BAR2_UAR_SIZE | |
282 | if (pfn > RDMA_BAR2_UAR_SIZE) { | |
4d71b38a YS |
283 | rdma_error_report("pfn out of range (%d > %d)", pfn, |
284 | RDMA_BAR2_UAR_SIZE); | |
ef6d4ccd YS |
285 | return -ENOMEM; |
286 | } | |
287 | */ | |
288 | ||
4d71b38a | 289 | uc = rdma_res_tbl_alloc(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
290 | if (!uc) { |
291 | return -ENOMEM; | |
292 | } | |
293 | ||
294 | return 0; | |
295 | } | |
296 | ||
297 | RdmaRmUC *rdma_rm_get_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle) | |
298 | { | |
4d71b38a | 299 | return rdma_res_tbl_get(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
300 | } |
301 | ||
302 | void rdma_rm_dealloc_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle) | |
303 | { | |
304 | RdmaRmUC *uc = rdma_rm_get_uc(dev_res, uc_handle); | |
305 | ||
306 | if (uc) { | |
4d71b38a | 307 | rdma_res_tbl_dealloc(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
308 | } |
309 | } | |
310 | ||
311 | RdmaRmCQ *rdma_rm_get_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle) | |
312 | { | |
4d71b38a | 313 | return rdma_res_tbl_get(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
314 | } |
315 | ||
316 | int rdma_rm_alloc_cq(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
317 | uint32_t cqe, uint32_t *cq_handle, void *opaque) | |
318 | { | |
319 | int rc; | |
320 | RdmaRmCQ *cq; | |
321 | ||
4d71b38a | 322 | cq = rdma_res_tbl_alloc(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
323 | if (!cq) { |
324 | return -ENOMEM; | |
325 | } | |
326 | ||
327 | cq->opaque = opaque; | |
4082e533 | 328 | cq->notify = CNT_CLEAR; |
ef6d4ccd YS |
329 | |
330 | rc = rdma_backend_create_cq(backend_dev, &cq->backend_cq, cqe); | |
331 | if (rc) { | |
332 | rc = -EIO; | |
333 | goto out_dealloc_cq; | |
334 | } | |
335 | ||
336 | return 0; | |
337 | ||
338 | out_dealloc_cq: | |
339 | rdma_rm_dealloc_cq(dev_res, *cq_handle); | |
340 | ||
341 | return rc; | |
342 | } | |
343 | ||
344 | void rdma_rm_req_notify_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle, | |
345 | bool notify) | |
346 | { | |
347 | RdmaRmCQ *cq; | |
348 | ||
ef6d4ccd YS |
349 | cq = rdma_rm_get_cq(dev_res, cq_handle); |
350 | if (!cq) { | |
351 | return; | |
352 | } | |
353 | ||
4082e533 YS |
354 | if (cq->notify != CNT_SET) { |
355 | cq->notify = notify ? CNT_ARM : CNT_CLEAR; | |
356 | } | |
ef6d4ccd YS |
357 | } |
358 | ||
359 | void rdma_rm_dealloc_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle) | |
360 | { | |
361 | RdmaRmCQ *cq; | |
362 | ||
363 | cq = rdma_rm_get_cq(dev_res, cq_handle); | |
364 | if (!cq) { | |
365 | return; | |
366 | } | |
367 | ||
368 | rdma_backend_destroy_cq(&cq->backend_cq); | |
369 | ||
4d71b38a | 370 | rdma_res_tbl_dealloc(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
371 | } |
372 | ||
373 | RdmaRmQP *rdma_rm_get_qp(RdmaDeviceResources *dev_res, uint32_t qpn) | |
374 | { | |
375 | GBytes *key = g_bytes_new(&qpn, sizeof(qpn)); | |
376 | ||
377 | RdmaRmQP *qp = g_hash_table_lookup(dev_res->qp_hash, key); | |
378 | ||
379 | g_bytes_unref(key); | |
380 | ||
4d71b38a YS |
381 | if (!qp) { |
382 | rdma_error_report("Invalid QP handle %d", qpn); | |
383 | } | |
384 | ||
ef6d4ccd YS |
385 | return qp; |
386 | } | |
387 | ||
388 | int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
389 | uint8_t qp_type, uint32_t max_send_wr, | |
390 | uint32_t max_send_sge, uint32_t send_cq_handle, | |
391 | uint32_t max_recv_wr, uint32_t max_recv_sge, | |
8b42cfab KH |
392 | uint32_t recv_cq_handle, void *opaque, uint32_t *qpn, |
393 | uint8_t is_srq, uint32_t srq_handle) | |
ef6d4ccd YS |
394 | { |
395 | int rc; | |
396 | RdmaRmQP *qp; | |
397 | RdmaRmCQ *scq, *rcq; | |
398 | RdmaRmPD *pd; | |
8b42cfab | 399 | RdmaRmSRQ *srq = NULL; |
ef6d4ccd YS |
400 | uint32_t rm_qpn; |
401 | ||
ef6d4ccd YS |
402 | pd = rdma_rm_get_pd(dev_res, pd_handle); |
403 | if (!pd) { | |
ef6d4ccd YS |
404 | return -EINVAL; |
405 | } | |
406 | ||
407 | scq = rdma_rm_get_cq(dev_res, send_cq_handle); | |
408 | rcq = rdma_rm_get_cq(dev_res, recv_cq_handle); | |
409 | ||
410 | if (!scq || !rcq) { | |
4d71b38a YS |
411 | rdma_error_report("Invalid send_cqn or recv_cqn (%d, %d)", |
412 | send_cq_handle, recv_cq_handle); | |
ef6d4ccd YS |
413 | return -EINVAL; |
414 | } | |
415 | ||
8b42cfab KH |
416 | if (is_srq) { |
417 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
418 | if (!srq) { | |
419 | rdma_error_report("Invalid srqn %d", srq_handle); | |
420 | return -EINVAL; | |
421 | } | |
422 | ||
423 | srq->recv_cq_handle = recv_cq_handle; | |
424 | } | |
425 | ||
4082e533 YS |
426 | if (qp_type == IBV_QPT_GSI) { |
427 | scq->notify = CNT_SET; | |
428 | rcq->notify = CNT_SET; | |
429 | } | |
430 | ||
4d71b38a | 431 | qp = rdma_res_tbl_alloc(&dev_res->qp_tbl, &rm_qpn); |
ef6d4ccd YS |
432 | if (!qp) { |
433 | return -ENOMEM; | |
434 | } | |
ef6d4ccd YS |
435 | |
436 | qp->qpn = rm_qpn; | |
437 | qp->qp_state = IBV_QPS_RESET; | |
438 | qp->qp_type = qp_type; | |
439 | qp->send_cq_handle = send_cq_handle; | |
440 | qp->recv_cq_handle = recv_cq_handle; | |
441 | qp->opaque = opaque; | |
8b42cfab | 442 | qp->is_srq = is_srq; |
ef6d4ccd YS |
443 | |
444 | rc = rdma_backend_create_qp(&qp->backend_qp, qp_type, &pd->backend_pd, | |
8b42cfab KH |
445 | &scq->backend_cq, &rcq->backend_cq, |
446 | is_srq ? &srq->backend_srq : NULL, | |
447 | max_send_wr, max_recv_wr, max_send_sge, | |
448 | max_recv_sge); | |
449 | ||
ef6d4ccd YS |
450 | if (rc) { |
451 | rc = -EIO; | |
452 | goto out_dealloc_qp; | |
453 | } | |
454 | ||
455 | *qpn = rdma_backend_qpn(&qp->backend_qp); | |
4d71b38a | 456 | trace_rdma_rm_alloc_qp(rm_qpn, *qpn, qp_type); |
ef6d4ccd YS |
457 | g_hash_table_insert(dev_res->qp_hash, g_bytes_new(qpn, sizeof(*qpn)), qp); |
458 | ||
459 | return 0; | |
460 | ||
461 | out_dealloc_qp: | |
4d71b38a | 462 | rdma_res_tbl_dealloc(&dev_res->qp_tbl, qp->qpn); |
ef6d4ccd YS |
463 | |
464 | return rc; | |
465 | } | |
466 | ||
467 | int rdma_rm_modify_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
2b05705d | 468 | uint32_t qp_handle, uint32_t attr_mask, uint8_t sgid_idx, |
ef6d4ccd YS |
469 | union ibv_gid *dgid, uint32_t dqpn, |
470 | enum ibv_qp_state qp_state, uint32_t qkey, | |
471 | uint32_t rq_psn, uint32_t sq_psn) | |
472 | { | |
473 | RdmaRmQP *qp; | |
474 | int ret; | |
475 | ||
ef6d4ccd YS |
476 | qp = rdma_rm_get_qp(dev_res, qp_handle); |
477 | if (!qp) { | |
478 | return -EINVAL; | |
479 | } | |
480 | ||
ef6d4ccd | 481 | if (qp->qp_type == IBV_QPT_SMI) { |
4d71b38a | 482 | rdma_error_report("Got QP0 request"); |
ef6d4ccd YS |
483 | return -EPERM; |
484 | } else if (qp->qp_type == IBV_QPT_GSI) { | |
ef6d4ccd YS |
485 | return 0; |
486 | } | |
487 | ||
4d71b38a YS |
488 | trace_rdma_rm_modify_qp(qp_handle, attr_mask, qp_state, sgid_idx); |
489 | ||
ef6d4ccd YS |
490 | if (attr_mask & IBV_QP_STATE) { |
491 | qp->qp_state = qp_state; | |
ef6d4ccd YS |
492 | |
493 | if (qp->qp_state == IBV_QPS_INIT) { | |
494 | ret = rdma_backend_qp_state_init(backend_dev, &qp->backend_qp, | |
495 | qp->qp_type, qkey); | |
496 | if (ret) { | |
497 | return -EIO; | |
498 | } | |
499 | } | |
500 | ||
501 | if (qp->qp_state == IBV_QPS_RTR) { | |
2b05705d | 502 | /* Get backend gid index */ |
2b05705d YS |
503 | sgid_idx = rdma_rm_get_backend_gid_index(dev_res, backend_dev, |
504 | sgid_idx); | |
505 | if (sgid_idx <= 0) { /* TODO check also less than bk.max_sgid */ | |
4d71b38a YS |
506 | rdma_error_report("Failed to get bk sgid_idx for sgid_idx %d", |
507 | sgid_idx); | |
2b05705d YS |
508 | return -EIO; |
509 | } | |
510 | ||
ef6d4ccd | 511 | ret = rdma_backend_qp_state_rtr(backend_dev, &qp->backend_qp, |
2b05705d YS |
512 | qp->qp_type, sgid_idx, dgid, dqpn, |
513 | rq_psn, qkey, | |
514 | attr_mask & IBV_QP_QKEY); | |
ef6d4ccd YS |
515 | if (ret) { |
516 | return -EIO; | |
517 | } | |
518 | } | |
519 | ||
520 | if (qp->qp_state == IBV_QPS_RTS) { | |
521 | ret = rdma_backend_qp_state_rts(&qp->backend_qp, qp->qp_type, | |
522 | sq_psn, qkey, | |
523 | attr_mask & IBV_QP_QKEY); | |
524 | if (ret) { | |
525 | return -EIO; | |
526 | } | |
527 | } | |
528 | } | |
529 | ||
530 | return 0; | |
531 | } | |
532 | ||
c99f2174 YS |
533 | int rdma_rm_query_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
534 | uint32_t qp_handle, struct ibv_qp_attr *attr, | |
535 | int attr_mask, struct ibv_qp_init_attr *init_attr) | |
536 | { | |
537 | RdmaRmQP *qp; | |
538 | ||
c99f2174 YS |
539 | qp = rdma_rm_get_qp(dev_res, qp_handle); |
540 | if (!qp) { | |
541 | return -EINVAL; | |
542 | } | |
543 | ||
c99f2174 YS |
544 | return rdma_backend_query_qp(&qp->backend_qp, attr, attr_mask, init_attr); |
545 | } | |
546 | ||
ef6d4ccd YS |
547 | void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle) |
548 | { | |
549 | RdmaRmQP *qp; | |
550 | GBytes *key; | |
551 | ||
552 | key = g_bytes_new(&qp_handle, sizeof(qp_handle)); | |
553 | qp = g_hash_table_lookup(dev_res->qp_hash, key); | |
554 | g_hash_table_remove(dev_res->qp_hash, key); | |
555 | g_bytes_unref(key); | |
556 | ||
557 | if (!qp) { | |
558 | return; | |
559 | } | |
560 | ||
bf441451 | 561 | rdma_backend_destroy_qp(&qp->backend_qp, dev_res); |
ef6d4ccd | 562 | |
4d71b38a | 563 | rdma_res_tbl_dealloc(&dev_res->qp_tbl, qp->qpn); |
ef6d4ccd YS |
564 | } |
565 | ||
cdc84058 KH |
566 | RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle) |
567 | { | |
568 | return rdma_res_tbl_get(&dev_res->srq_tbl, srq_handle); | |
569 | } | |
570 | ||
571 | int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
572 | uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit, | |
573 | uint32_t *srq_handle, void *opaque) | |
574 | { | |
575 | RdmaRmSRQ *srq; | |
576 | RdmaRmPD *pd; | |
577 | int rc; | |
578 | ||
579 | pd = rdma_rm_get_pd(dev_res, pd_handle); | |
580 | if (!pd) { | |
581 | return -EINVAL; | |
582 | } | |
583 | ||
584 | srq = rdma_res_tbl_alloc(&dev_res->srq_tbl, srq_handle); | |
585 | if (!srq) { | |
586 | return -ENOMEM; | |
587 | } | |
588 | ||
589 | rc = rdma_backend_create_srq(&srq->backend_srq, &pd->backend_pd, | |
590 | max_wr, max_sge, srq_limit); | |
591 | if (rc) { | |
592 | rc = -EIO; | |
593 | goto out_dealloc_srq; | |
594 | } | |
595 | ||
596 | srq->opaque = opaque; | |
597 | ||
598 | return 0; | |
599 | ||
600 | out_dealloc_srq: | |
601 | rdma_res_tbl_dealloc(&dev_res->srq_tbl, *srq_handle); | |
602 | ||
603 | return rc; | |
604 | } | |
605 | ||
606 | int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle, | |
607 | struct ibv_srq_attr *srq_attr) | |
608 | { | |
609 | RdmaRmSRQ *srq; | |
610 | ||
611 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
612 | if (!srq) { | |
613 | return -EINVAL; | |
614 | } | |
615 | ||
616 | return rdma_backend_query_srq(&srq->backend_srq, srq_attr); | |
617 | } | |
618 | ||
619 | int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle, | |
620 | struct ibv_srq_attr *srq_attr, int srq_attr_mask) | |
621 | { | |
622 | RdmaRmSRQ *srq; | |
623 | ||
624 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
625 | if (!srq) { | |
626 | return -EINVAL; | |
627 | } | |
628 | ||
629 | if ((srq_attr_mask & IBV_SRQ_LIMIT) && | |
630 | (srq_attr->srq_limit == 0)) { | |
631 | return -EINVAL; | |
632 | } | |
633 | ||
634 | if ((srq_attr_mask & IBV_SRQ_MAX_WR) && | |
635 | (srq_attr->max_wr == 0)) { | |
636 | return -EINVAL; | |
637 | } | |
638 | ||
639 | return rdma_backend_modify_srq(&srq->backend_srq, srq_attr, | |
640 | srq_attr_mask); | |
641 | } | |
642 | ||
643 | void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle) | |
644 | { | |
645 | RdmaRmSRQ *srq; | |
646 | ||
647 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
648 | if (!srq) { | |
649 | return; | |
650 | } | |
651 | ||
652 | rdma_backend_destroy_srq(&srq->backend_srq, dev_res); | |
653 | rdma_res_tbl_dealloc(&dev_res->srq_tbl, srq_handle); | |
654 | } | |
655 | ||
ef6d4ccd YS |
656 | void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id) |
657 | { | |
658 | void **cqe_ctx; | |
659 | ||
4d71b38a | 660 | cqe_ctx = rdma_res_tbl_get(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
661 | if (!cqe_ctx) { |
662 | return NULL; | |
663 | } | |
664 | ||
ef6d4ccd YS |
665 | return *cqe_ctx; |
666 | } | |
667 | ||
668 | int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id, | |
669 | void *ctx) | |
670 | { | |
671 | void **cqe_ctx; | |
672 | ||
4d71b38a | 673 | cqe_ctx = rdma_res_tbl_alloc(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
674 | if (!cqe_ctx) { |
675 | return -ENOMEM; | |
676 | } | |
677 | ||
ef6d4ccd YS |
678 | *cqe_ctx = ctx; |
679 | ||
680 | return 0; | |
681 | } | |
682 | ||
683 | void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id) | |
684 | { | |
4d71b38a | 685 | rdma_res_tbl_dealloc(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
686 | } |
687 | ||
2b05705d YS |
688 | int rdma_rm_add_gid(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
689 | const char *ifname, union ibv_gid *gid, int gid_idx) | |
690 | { | |
691 | int rc; | |
692 | ||
693 | rc = rdma_backend_add_gid(backend_dev, ifname, gid); | |
694 | if (rc) { | |
2b05705d YS |
695 | return -EINVAL; |
696 | } | |
697 | ||
14c74f72 | 698 | memcpy(&dev_res->port.gid_tbl[gid_idx].gid, gid, sizeof(*gid)); |
2b05705d YS |
699 | |
700 | return 0; | |
701 | } | |
702 | ||
703 | int rdma_rm_del_gid(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
704 | const char *ifname, int gid_idx) | |
705 | { | |
706 | int rc; | |
707 | ||
305fd2ba YS |
708 | if (!dev_res->port.gid_tbl[gid_idx].gid.global.interface_id) { |
709 | return 0; | |
710 | } | |
711 | ||
2b05705d | 712 | rc = rdma_backend_del_gid(backend_dev, ifname, |
14c74f72 | 713 | &dev_res->port.gid_tbl[gid_idx].gid); |
2b05705d | 714 | if (rc) { |
2b05705d YS |
715 | return -EINVAL; |
716 | } | |
717 | ||
14c74f72 YS |
718 | memset(dev_res->port.gid_tbl[gid_idx].gid.raw, 0, |
719 | sizeof(dev_res->port.gid_tbl[gid_idx].gid)); | |
720 | dev_res->port.gid_tbl[gid_idx].backend_gid_index = -1; | |
2b05705d YS |
721 | |
722 | return 0; | |
723 | } | |
724 | ||
725 | int rdma_rm_get_backend_gid_index(RdmaDeviceResources *dev_res, | |
726 | RdmaBackendDev *backend_dev, int sgid_idx) | |
727 | { | |
a5fe209d | 728 | if (unlikely(sgid_idx < 0 || sgid_idx >= MAX_PORT_GIDS)) { |
4d71b38a | 729 | rdma_error_report("Got invalid sgid_idx %d", sgid_idx); |
2b05705d YS |
730 | return -EINVAL; |
731 | } | |
732 | ||
14c74f72 YS |
733 | if (unlikely(dev_res->port.gid_tbl[sgid_idx].backend_gid_index == -1)) { |
734 | dev_res->port.gid_tbl[sgid_idx].backend_gid_index = | |
2b05705d | 735 | rdma_backend_get_gid_index(backend_dev, |
14c74f72 | 736 | &dev_res->port.gid_tbl[sgid_idx].gid); |
2b05705d YS |
737 | } |
738 | ||
14c74f72 | 739 | return dev_res->port.gid_tbl[sgid_idx].backend_gid_index; |
2b05705d YS |
740 | } |
741 | ||
ef6d4ccd YS |
742 | static void destroy_qp_hash_key(gpointer data) |
743 | { | |
744 | g_bytes_unref(data); | |
745 | } | |
746 | ||
2b05705d YS |
747 | static void init_ports(RdmaDeviceResources *dev_res) |
748 | { | |
14c74f72 | 749 | int i; |
2b05705d | 750 | |
14c74f72 | 751 | memset(&dev_res->port, 0, sizeof(dev_res->port)); |
2b05705d | 752 | |
14c74f72 YS |
753 | dev_res->port.state = IBV_PORT_DOWN; |
754 | for (i = 0; i < MAX_PORT_GIDS; i++) { | |
755 | dev_res->port.gid_tbl[i].backend_gid_index = -1; | |
2b05705d YS |
756 | } |
757 | } | |
758 | ||
759 | static void fini_ports(RdmaDeviceResources *dev_res, | |
760 | RdmaBackendDev *backend_dev, const char *ifname) | |
761 | { | |
762 | int i; | |
763 | ||
14c74f72 | 764 | dev_res->port.state = IBV_PORT_DOWN; |
2b05705d YS |
765 | for (i = 0; i < MAX_PORT_GIDS; i++) { |
766 | rdma_rm_del_gid(dev_res, backend_dev, ifname, i); | |
767 | } | |
768 | } | |
769 | ||
4d71b38a | 770 | int rdma_rm_init(RdmaDeviceResources *dev_res, struct ibv_device_attr *dev_attr) |
ef6d4ccd YS |
771 | { |
772 | dev_res->qp_hash = g_hash_table_new_full(g_bytes_hash, g_bytes_equal, | |
773 | destroy_qp_hash_key, NULL); | |
774 | if (!dev_res->qp_hash) { | |
775 | return -ENOMEM; | |
776 | } | |
777 | ||
778 | res_tbl_init("PD", &dev_res->pd_tbl, dev_attr->max_pd, sizeof(RdmaRmPD)); | |
779 | res_tbl_init("CQ", &dev_res->cq_tbl, dev_attr->max_cq, sizeof(RdmaRmCQ)); | |
780 | res_tbl_init("MR", &dev_res->mr_tbl, dev_attr->max_mr, sizeof(RdmaRmMR)); | |
781 | res_tbl_init("QP", &dev_res->qp_tbl, dev_attr->max_qp, sizeof(RdmaRmQP)); | |
782 | res_tbl_init("CQE_CTX", &dev_res->cqe_ctx_tbl, dev_attr->max_qp * | |
783 | dev_attr->max_qp_wr, sizeof(void *)); | |
784 | res_tbl_init("UC", &dev_res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC)); | |
cdc84058 KH |
785 | res_tbl_init("SRQ", &dev_res->srq_tbl, dev_attr->max_srq, |
786 | sizeof(RdmaRmSRQ)); | |
ef6d4ccd | 787 | |
2b05705d YS |
788 | init_ports(dev_res); |
789 | ||
2cfa9530 YS |
790 | qemu_mutex_init(&dev_res->lock); |
791 | ||
c2dd117b | 792 | memset(&dev_res->stats, 0, sizeof(dev_res->stats)); |
d73415a3 | 793 | qatomic_set(&dev_res->stats.missing_cqe, 0); |
c2dd117b | 794 | |
ef6d4ccd YS |
795 | return 0; |
796 | } | |
797 | ||
2b05705d YS |
798 | void rdma_rm_fini(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
799 | const char *ifname) | |
ef6d4ccd | 800 | { |
2cfa9530 YS |
801 | qemu_mutex_destroy(&dev_res->lock); |
802 | ||
2b05705d YS |
803 | fini_ports(dev_res, backend_dev, ifname); |
804 | ||
cdc84058 | 805 | res_tbl_free(&dev_res->srq_tbl); |
ef6d4ccd YS |
806 | res_tbl_free(&dev_res->uc_tbl); |
807 | res_tbl_free(&dev_res->cqe_ctx_tbl); | |
808 | res_tbl_free(&dev_res->qp_tbl); | |
ef6d4ccd | 809 | res_tbl_free(&dev_res->mr_tbl); |
7131c4b0 | 810 | res_tbl_free(&dev_res->cq_tbl); |
ef6d4ccd | 811 | res_tbl_free(&dev_res->pd_tbl); |
7131c4b0 | 812 | |
a1aa88b7 YS |
813 | if (dev_res->qp_hash) { |
814 | g_hash_table_destroy(dev_res->qp_hash); | |
815 | } | |
ef6d4ccd | 816 | } |