]>
Commit | Line | Data |
---|---|---|
ef6d4ccd YS |
1 | /* |
2 | * QEMU paravirtual RDMA - Resource Manager Implementation | |
3 | * | |
4 | * Copyright (C) 2018 Oracle | |
5 | * Copyright (C) 2018 Red Hat Inc | |
6 | * | |
7 | * Authors: | |
8 | * Yuval Shaia <yuval.shaia@oracle.com> | |
9 | * Marcel Apfelbaum <marcel@redhat.com> | |
10 | * | |
11 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
12 | * See the COPYING file in the top-level directory. | |
13 | * | |
14 | */ | |
15 | ||
0efc9511 MT |
16 | #include "qemu/osdep.h" |
17 | #include "qapi/error.h" | |
18 | #include "cpu.h" | |
f4b2c02a | 19 | #include "monitor/monitor.h" |
ef6d4ccd | 20 | |
4d71b38a | 21 | #include "trace.h" |
ef6d4ccd YS |
22 | #include "rdma_utils.h" |
23 | #include "rdma_backend.h" | |
24 | #include "rdma_rm.h" | |
25 | ||
ef6d4ccd YS |
26 | /* Page directory and page tables */ |
27 | #define PG_DIR_SZ { TARGET_PAGE_SIZE / sizeof(__u64) } | |
28 | #define PG_TBL_SZ { TARGET_PAGE_SIZE / sizeof(__u64) } | |
29 | ||
f4b2c02a YS |
30 | void rdma_dump_device_counters(Monitor *mon, RdmaDeviceResources *dev_res) |
31 | { | |
32 | monitor_printf(mon, "\ttx : %" PRId64 "\n", | |
33 | dev_res->stats.tx); | |
34 | monitor_printf(mon, "\ttx_len : %" PRId64 "\n", | |
35 | dev_res->stats.tx_len); | |
36 | monitor_printf(mon, "\ttx_err : %" PRId64 "\n", | |
37 | dev_res->stats.tx_err); | |
38 | monitor_printf(mon, "\trx_bufs : %" PRId64 "\n", | |
39 | dev_res->stats.rx_bufs); | |
e926c9f1 KH |
40 | monitor_printf(mon, "\trx_srq : %" PRId64 "\n", |
41 | dev_res->stats.rx_srq); | |
f4b2c02a YS |
42 | monitor_printf(mon, "\trx_bufs_len : %" PRId64 "\n", |
43 | dev_res->stats.rx_bufs_len); | |
44 | monitor_printf(mon, "\trx_bufs_err : %" PRId64 "\n", | |
45 | dev_res->stats.rx_bufs_err); | |
46 | monitor_printf(mon, "\tcomps : %" PRId64 "\n", | |
47 | dev_res->stats.completions); | |
48 | monitor_printf(mon, "\tmissing_comps : %" PRId32 "\n", | |
49 | dev_res->stats.missing_cqe); | |
50 | monitor_printf(mon, "\tpoll_cq (bk) : %" PRId64 "\n", | |
51 | dev_res->stats.poll_cq_from_bk); | |
52 | monitor_printf(mon, "\tpoll_cq_ppoll_to : %" PRId64 "\n", | |
53 | dev_res->stats.poll_cq_ppoll_to); | |
54 | monitor_printf(mon, "\tpoll_cq (fe) : %" PRId64 "\n", | |
55 | dev_res->stats.poll_cq_from_guest); | |
56 | monitor_printf(mon, "\tpoll_cq_empty : %" PRId64 "\n", | |
57 | dev_res->stats.poll_cq_from_guest_empty); | |
58 | monitor_printf(mon, "\tmad_tx : %" PRId64 "\n", | |
59 | dev_res->stats.mad_tx); | |
60 | monitor_printf(mon, "\tmad_tx_err : %" PRId64 "\n", | |
61 | dev_res->stats.mad_tx_err); | |
62 | monitor_printf(mon, "\tmad_rx : %" PRId64 "\n", | |
63 | dev_res->stats.mad_rx); | |
64 | monitor_printf(mon, "\tmad_rx_err : %" PRId64 "\n", | |
65 | dev_res->stats.mad_rx_err); | |
66 | monitor_printf(mon, "\tmad_rx_bufs : %" PRId64 "\n", | |
67 | dev_res->stats.mad_rx_bufs); | |
68 | monitor_printf(mon, "\tmad_rx_bufs_err : %" PRId64 "\n", | |
69 | dev_res->stats.mad_rx_bufs_err); | |
70 | monitor_printf(mon, "\tPDs : %" PRId32 "\n", | |
71 | dev_res->pd_tbl.used); | |
72 | monitor_printf(mon, "\tMRs : %" PRId32 "\n", | |
73 | dev_res->mr_tbl.used); | |
74 | monitor_printf(mon, "\tUCs : %" PRId32 "\n", | |
75 | dev_res->uc_tbl.used); | |
76 | monitor_printf(mon, "\tQPs : %" PRId32 "\n", | |
77 | dev_res->qp_tbl.used); | |
78 | monitor_printf(mon, "\tCQs : %" PRId32 "\n", | |
79 | dev_res->cq_tbl.used); | |
80 | monitor_printf(mon, "\tCEQ_CTXs : %" PRId32 "\n", | |
81 | dev_res->cqe_ctx_tbl.used); | |
82 | } | |
83 | ||
ef6d4ccd YS |
84 | static inline void res_tbl_init(const char *name, RdmaRmResTbl *tbl, |
85 | uint32_t tbl_sz, uint32_t res_sz) | |
86 | { | |
87 | tbl->tbl = g_malloc(tbl_sz * res_sz); | |
88 | ||
89 | strncpy(tbl->name, name, MAX_RM_TBL_NAME); | |
90 | tbl->name[MAX_RM_TBL_NAME - 1] = 0; | |
91 | ||
92 | tbl->bitmap = bitmap_new(tbl_sz); | |
93 | tbl->tbl_sz = tbl_sz; | |
94 | tbl->res_sz = res_sz; | |
c2dd117b | 95 | tbl->used = 0; |
ef6d4ccd YS |
96 | qemu_mutex_init(&tbl->lock); |
97 | } | |
98 | ||
99 | static inline void res_tbl_free(RdmaRmResTbl *tbl) | |
100 | { | |
a1aa88b7 YS |
101 | if (!tbl->bitmap) { |
102 | return; | |
103 | } | |
ef6d4ccd YS |
104 | qemu_mutex_destroy(&tbl->lock); |
105 | g_free(tbl->tbl); | |
9a3053d2 | 106 | g_free(tbl->bitmap); |
ef6d4ccd YS |
107 | } |
108 | ||
4d71b38a | 109 | static inline void *rdma_res_tbl_get(RdmaRmResTbl *tbl, uint32_t handle) |
ef6d4ccd | 110 | { |
4d71b38a | 111 | trace_rdma_res_tbl_get(tbl->name, handle); |
ef6d4ccd YS |
112 | |
113 | if ((handle < tbl->tbl_sz) && (test_bit(handle, tbl->bitmap))) { | |
114 | return tbl->tbl + handle * tbl->res_sz; | |
115 | } else { | |
4d71b38a | 116 | rdma_error_report("Table %s, invalid handle %d", tbl->name, handle); |
ef6d4ccd YS |
117 | return NULL; |
118 | } | |
119 | } | |
120 | ||
4d71b38a | 121 | static inline void *rdma_res_tbl_alloc(RdmaRmResTbl *tbl, uint32_t *handle) |
ef6d4ccd YS |
122 | { |
123 | qemu_mutex_lock(&tbl->lock); | |
124 | ||
125 | *handle = find_first_zero_bit(tbl->bitmap, tbl->tbl_sz); | |
126 | if (*handle > tbl->tbl_sz) { | |
4d71b38a YS |
127 | rdma_error_report("Table %s, failed to allocate, bitmap is full", |
128 | tbl->name); | |
ef6d4ccd YS |
129 | qemu_mutex_unlock(&tbl->lock); |
130 | return NULL; | |
131 | } | |
132 | ||
133 | set_bit(*handle, tbl->bitmap); | |
134 | ||
c2dd117b YS |
135 | tbl->used++; |
136 | ||
ef6d4ccd YS |
137 | qemu_mutex_unlock(&tbl->lock); |
138 | ||
139 | memset(tbl->tbl + *handle * tbl->res_sz, 0, tbl->res_sz); | |
140 | ||
4d71b38a | 141 | trace_rdma_res_tbl_alloc(tbl->name, *handle); |
ef6d4ccd YS |
142 | |
143 | return tbl->tbl + *handle * tbl->res_sz; | |
144 | } | |
145 | ||
4d71b38a | 146 | static inline void rdma_res_tbl_dealloc(RdmaRmResTbl *tbl, uint32_t handle) |
ef6d4ccd | 147 | { |
4d71b38a | 148 | trace_rdma_res_tbl_dealloc(tbl->name, handle); |
ef6d4ccd YS |
149 | |
150 | qemu_mutex_lock(&tbl->lock); | |
151 | ||
152 | if (handle < tbl->tbl_sz) { | |
153 | clear_bit(handle, tbl->bitmap); | |
c2dd117b | 154 | tbl->used--; |
ef6d4ccd YS |
155 | } |
156 | ||
157 | qemu_mutex_unlock(&tbl->lock); | |
158 | } | |
159 | ||
160 | int rdma_rm_alloc_pd(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
161 | uint32_t *pd_handle, uint32_t ctx_handle) | |
162 | { | |
163 | RdmaRmPD *pd; | |
164 | int ret = -ENOMEM; | |
165 | ||
4d71b38a | 166 | pd = rdma_res_tbl_alloc(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
167 | if (!pd) { |
168 | goto out; | |
169 | } | |
170 | ||
171 | ret = rdma_backend_create_pd(backend_dev, &pd->backend_pd); | |
172 | if (ret) { | |
173 | ret = -EIO; | |
174 | goto out_tbl_dealloc; | |
175 | } | |
176 | ||
177 | pd->ctx_handle = ctx_handle; | |
178 | ||
179 | return 0; | |
180 | ||
181 | out_tbl_dealloc: | |
4d71b38a | 182 | rdma_res_tbl_dealloc(&dev_res->pd_tbl, *pd_handle); |
ef6d4ccd YS |
183 | |
184 | out: | |
185 | return ret; | |
186 | } | |
187 | ||
188 | RdmaRmPD *rdma_rm_get_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle) | |
189 | { | |
4d71b38a | 190 | return rdma_res_tbl_get(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
191 | } |
192 | ||
193 | void rdma_rm_dealloc_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle) | |
194 | { | |
195 | RdmaRmPD *pd = rdma_rm_get_pd(dev_res, pd_handle); | |
196 | ||
197 | if (pd) { | |
198 | rdma_backend_destroy_pd(&pd->backend_pd); | |
4d71b38a | 199 | rdma_res_tbl_dealloc(&dev_res->pd_tbl, pd_handle); |
ef6d4ccd YS |
200 | } |
201 | } | |
202 | ||
203 | int rdma_rm_alloc_mr(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
4d71b38a YS |
204 | uint64_t guest_start, uint64_t guest_length, |
205 | void *host_virt, int access_flags, uint32_t *mr_handle, | |
206 | uint32_t *lkey, uint32_t *rkey) | |
ef6d4ccd YS |
207 | { |
208 | RdmaRmMR *mr; | |
209 | int ret = 0; | |
210 | RdmaRmPD *pd; | |
ef6d4ccd YS |
211 | |
212 | pd = rdma_rm_get_pd(dev_res, pd_handle); | |
213 | if (!pd) { | |
ef6d4ccd YS |
214 | return -EINVAL; |
215 | } | |
216 | ||
4d71b38a | 217 | mr = rdma_res_tbl_alloc(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd | 218 | if (!mr) { |
ef6d4ccd YS |
219 | return -ENOMEM; |
220 | } | |
4d71b38a YS |
221 | trace_rdma_rm_alloc_mr(*mr_handle, host_virt, guest_start, guest_length, |
222 | access_flags); | |
4c408c9d YS |
223 | |
224 | if (host_virt) { | |
7f99daad | 225 | mr->virt = host_virt; |
7f99daad | 226 | mr->start = guest_start; |
4c408c9d | 227 | mr->length = guest_length; |
7d2ce4b0 | 228 | mr->virt += (mr->start & (TARGET_PAGE_SIZE - 1)); |
ef6d4ccd | 229 | |
4c408c9d YS |
230 | ret = rdma_backend_create_mr(&mr->backend_mr, &pd->backend_pd, mr->virt, |
231 | mr->length, access_flags); | |
232 | if (ret) { | |
4c408c9d YS |
233 | ret = -EIO; |
234 | goto out_dealloc_mr; | |
235 | } | |
ef6d4ccd YS |
236 | } |
237 | ||
4c408c9d YS |
238 | /* We keep mr_handle in lkey so send and recv get get mr ptr */ |
239 | *lkey = *mr_handle; | |
240 | *rkey = -1; | |
ef6d4ccd YS |
241 | |
242 | mr->pd_handle = pd_handle; | |
243 | ||
244 | return 0; | |
245 | ||
246 | out_dealloc_mr: | |
4d71b38a | 247 | rdma_res_tbl_dealloc(&dev_res->mr_tbl, *mr_handle); |
ef6d4ccd YS |
248 | |
249 | return ret; | |
250 | } | |
251 | ||
252 | RdmaRmMR *rdma_rm_get_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle) | |
253 | { | |
4d71b38a | 254 | return rdma_res_tbl_get(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd YS |
255 | } |
256 | ||
257 | void rdma_rm_dealloc_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle) | |
258 | { | |
259 | RdmaRmMR *mr = rdma_rm_get_mr(dev_res, mr_handle); | |
260 | ||
261 | if (mr) { | |
262 | rdma_backend_destroy_mr(&mr->backend_mr); | |
4d71b38a | 263 | trace_rdma_rm_dealloc_mr(mr_handle, mr->start); |
4c408c9d | 264 | if (mr->start) { |
7d2ce4b0 | 265 | mr->virt -= (mr->start & (TARGET_PAGE_SIZE - 1)); |
4c408c9d YS |
266 | munmap(mr->virt, mr->length); |
267 | } | |
4d71b38a | 268 | rdma_res_tbl_dealloc(&dev_res->mr_tbl, mr_handle); |
ef6d4ccd YS |
269 | } |
270 | } | |
271 | ||
272 | int rdma_rm_alloc_uc(RdmaDeviceResources *dev_res, uint32_t pfn, | |
273 | uint32_t *uc_handle) | |
274 | { | |
275 | RdmaRmUC *uc; | |
276 | ||
277 | /* TODO: Need to make sure pfn is between bar start address and | |
278 | * bsd+RDMA_BAR2_UAR_SIZE | |
279 | if (pfn > RDMA_BAR2_UAR_SIZE) { | |
4d71b38a YS |
280 | rdma_error_report("pfn out of range (%d > %d)", pfn, |
281 | RDMA_BAR2_UAR_SIZE); | |
ef6d4ccd YS |
282 | return -ENOMEM; |
283 | } | |
284 | */ | |
285 | ||
4d71b38a | 286 | uc = rdma_res_tbl_alloc(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
287 | if (!uc) { |
288 | return -ENOMEM; | |
289 | } | |
290 | ||
291 | return 0; | |
292 | } | |
293 | ||
294 | RdmaRmUC *rdma_rm_get_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle) | |
295 | { | |
4d71b38a | 296 | return rdma_res_tbl_get(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
297 | } |
298 | ||
299 | void rdma_rm_dealloc_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle) | |
300 | { | |
301 | RdmaRmUC *uc = rdma_rm_get_uc(dev_res, uc_handle); | |
302 | ||
303 | if (uc) { | |
4d71b38a | 304 | rdma_res_tbl_dealloc(&dev_res->uc_tbl, uc_handle); |
ef6d4ccd YS |
305 | } |
306 | } | |
307 | ||
308 | RdmaRmCQ *rdma_rm_get_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle) | |
309 | { | |
4d71b38a | 310 | return rdma_res_tbl_get(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
311 | } |
312 | ||
313 | int rdma_rm_alloc_cq(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
314 | uint32_t cqe, uint32_t *cq_handle, void *opaque) | |
315 | { | |
316 | int rc; | |
317 | RdmaRmCQ *cq; | |
318 | ||
4d71b38a | 319 | cq = rdma_res_tbl_alloc(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
320 | if (!cq) { |
321 | return -ENOMEM; | |
322 | } | |
323 | ||
324 | cq->opaque = opaque; | |
4082e533 | 325 | cq->notify = CNT_CLEAR; |
ef6d4ccd YS |
326 | |
327 | rc = rdma_backend_create_cq(backend_dev, &cq->backend_cq, cqe); | |
328 | if (rc) { | |
329 | rc = -EIO; | |
330 | goto out_dealloc_cq; | |
331 | } | |
332 | ||
333 | return 0; | |
334 | ||
335 | out_dealloc_cq: | |
336 | rdma_rm_dealloc_cq(dev_res, *cq_handle); | |
337 | ||
338 | return rc; | |
339 | } | |
340 | ||
341 | void rdma_rm_req_notify_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle, | |
342 | bool notify) | |
343 | { | |
344 | RdmaRmCQ *cq; | |
345 | ||
ef6d4ccd YS |
346 | cq = rdma_rm_get_cq(dev_res, cq_handle); |
347 | if (!cq) { | |
348 | return; | |
349 | } | |
350 | ||
4082e533 YS |
351 | if (cq->notify != CNT_SET) { |
352 | cq->notify = notify ? CNT_ARM : CNT_CLEAR; | |
353 | } | |
ef6d4ccd YS |
354 | } |
355 | ||
356 | void rdma_rm_dealloc_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle) | |
357 | { | |
358 | RdmaRmCQ *cq; | |
359 | ||
360 | cq = rdma_rm_get_cq(dev_res, cq_handle); | |
361 | if (!cq) { | |
362 | return; | |
363 | } | |
364 | ||
365 | rdma_backend_destroy_cq(&cq->backend_cq); | |
366 | ||
4d71b38a | 367 | rdma_res_tbl_dealloc(&dev_res->cq_tbl, cq_handle); |
ef6d4ccd YS |
368 | } |
369 | ||
370 | RdmaRmQP *rdma_rm_get_qp(RdmaDeviceResources *dev_res, uint32_t qpn) | |
371 | { | |
372 | GBytes *key = g_bytes_new(&qpn, sizeof(qpn)); | |
373 | ||
374 | RdmaRmQP *qp = g_hash_table_lookup(dev_res->qp_hash, key); | |
375 | ||
376 | g_bytes_unref(key); | |
377 | ||
4d71b38a YS |
378 | if (!qp) { |
379 | rdma_error_report("Invalid QP handle %d", qpn); | |
380 | } | |
381 | ||
ef6d4ccd YS |
382 | return qp; |
383 | } | |
384 | ||
385 | int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
386 | uint8_t qp_type, uint32_t max_send_wr, | |
387 | uint32_t max_send_sge, uint32_t send_cq_handle, | |
388 | uint32_t max_recv_wr, uint32_t max_recv_sge, | |
8b42cfab KH |
389 | uint32_t recv_cq_handle, void *opaque, uint32_t *qpn, |
390 | uint8_t is_srq, uint32_t srq_handle) | |
ef6d4ccd YS |
391 | { |
392 | int rc; | |
393 | RdmaRmQP *qp; | |
394 | RdmaRmCQ *scq, *rcq; | |
395 | RdmaRmPD *pd; | |
8b42cfab | 396 | RdmaRmSRQ *srq = NULL; |
ef6d4ccd YS |
397 | uint32_t rm_qpn; |
398 | ||
ef6d4ccd YS |
399 | pd = rdma_rm_get_pd(dev_res, pd_handle); |
400 | if (!pd) { | |
ef6d4ccd YS |
401 | return -EINVAL; |
402 | } | |
403 | ||
404 | scq = rdma_rm_get_cq(dev_res, send_cq_handle); | |
405 | rcq = rdma_rm_get_cq(dev_res, recv_cq_handle); | |
406 | ||
407 | if (!scq || !rcq) { | |
4d71b38a YS |
408 | rdma_error_report("Invalid send_cqn or recv_cqn (%d, %d)", |
409 | send_cq_handle, recv_cq_handle); | |
ef6d4ccd YS |
410 | return -EINVAL; |
411 | } | |
412 | ||
8b42cfab KH |
413 | if (is_srq) { |
414 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
415 | if (!srq) { | |
416 | rdma_error_report("Invalid srqn %d", srq_handle); | |
417 | return -EINVAL; | |
418 | } | |
419 | ||
420 | srq->recv_cq_handle = recv_cq_handle; | |
421 | } | |
422 | ||
4082e533 YS |
423 | if (qp_type == IBV_QPT_GSI) { |
424 | scq->notify = CNT_SET; | |
425 | rcq->notify = CNT_SET; | |
426 | } | |
427 | ||
4d71b38a | 428 | qp = rdma_res_tbl_alloc(&dev_res->qp_tbl, &rm_qpn); |
ef6d4ccd YS |
429 | if (!qp) { |
430 | return -ENOMEM; | |
431 | } | |
ef6d4ccd YS |
432 | |
433 | qp->qpn = rm_qpn; | |
434 | qp->qp_state = IBV_QPS_RESET; | |
435 | qp->qp_type = qp_type; | |
436 | qp->send_cq_handle = send_cq_handle; | |
437 | qp->recv_cq_handle = recv_cq_handle; | |
438 | qp->opaque = opaque; | |
8b42cfab | 439 | qp->is_srq = is_srq; |
ef6d4ccd YS |
440 | |
441 | rc = rdma_backend_create_qp(&qp->backend_qp, qp_type, &pd->backend_pd, | |
8b42cfab KH |
442 | &scq->backend_cq, &rcq->backend_cq, |
443 | is_srq ? &srq->backend_srq : NULL, | |
444 | max_send_wr, max_recv_wr, max_send_sge, | |
445 | max_recv_sge); | |
446 | ||
ef6d4ccd YS |
447 | if (rc) { |
448 | rc = -EIO; | |
449 | goto out_dealloc_qp; | |
450 | } | |
451 | ||
452 | *qpn = rdma_backend_qpn(&qp->backend_qp); | |
4d71b38a | 453 | trace_rdma_rm_alloc_qp(rm_qpn, *qpn, qp_type); |
ef6d4ccd YS |
454 | g_hash_table_insert(dev_res->qp_hash, g_bytes_new(qpn, sizeof(*qpn)), qp); |
455 | ||
456 | return 0; | |
457 | ||
458 | out_dealloc_qp: | |
4d71b38a | 459 | rdma_res_tbl_dealloc(&dev_res->qp_tbl, qp->qpn); |
ef6d4ccd YS |
460 | |
461 | return rc; | |
462 | } | |
463 | ||
464 | int rdma_rm_modify_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
2b05705d | 465 | uint32_t qp_handle, uint32_t attr_mask, uint8_t sgid_idx, |
ef6d4ccd YS |
466 | union ibv_gid *dgid, uint32_t dqpn, |
467 | enum ibv_qp_state qp_state, uint32_t qkey, | |
468 | uint32_t rq_psn, uint32_t sq_psn) | |
469 | { | |
470 | RdmaRmQP *qp; | |
471 | int ret; | |
472 | ||
ef6d4ccd YS |
473 | qp = rdma_rm_get_qp(dev_res, qp_handle); |
474 | if (!qp) { | |
475 | return -EINVAL; | |
476 | } | |
477 | ||
ef6d4ccd | 478 | if (qp->qp_type == IBV_QPT_SMI) { |
4d71b38a | 479 | rdma_error_report("Got QP0 request"); |
ef6d4ccd YS |
480 | return -EPERM; |
481 | } else if (qp->qp_type == IBV_QPT_GSI) { | |
ef6d4ccd YS |
482 | return 0; |
483 | } | |
484 | ||
4d71b38a YS |
485 | trace_rdma_rm_modify_qp(qp_handle, attr_mask, qp_state, sgid_idx); |
486 | ||
ef6d4ccd YS |
487 | if (attr_mask & IBV_QP_STATE) { |
488 | qp->qp_state = qp_state; | |
ef6d4ccd YS |
489 | |
490 | if (qp->qp_state == IBV_QPS_INIT) { | |
491 | ret = rdma_backend_qp_state_init(backend_dev, &qp->backend_qp, | |
492 | qp->qp_type, qkey); | |
493 | if (ret) { | |
494 | return -EIO; | |
495 | } | |
496 | } | |
497 | ||
498 | if (qp->qp_state == IBV_QPS_RTR) { | |
2b05705d | 499 | /* Get backend gid index */ |
2b05705d YS |
500 | sgid_idx = rdma_rm_get_backend_gid_index(dev_res, backend_dev, |
501 | sgid_idx); | |
502 | if (sgid_idx <= 0) { /* TODO check also less than bk.max_sgid */ | |
4d71b38a YS |
503 | rdma_error_report("Failed to get bk sgid_idx for sgid_idx %d", |
504 | sgid_idx); | |
2b05705d YS |
505 | return -EIO; |
506 | } | |
507 | ||
ef6d4ccd | 508 | ret = rdma_backend_qp_state_rtr(backend_dev, &qp->backend_qp, |
2b05705d YS |
509 | qp->qp_type, sgid_idx, dgid, dqpn, |
510 | rq_psn, qkey, | |
511 | attr_mask & IBV_QP_QKEY); | |
ef6d4ccd YS |
512 | if (ret) { |
513 | return -EIO; | |
514 | } | |
515 | } | |
516 | ||
517 | if (qp->qp_state == IBV_QPS_RTS) { | |
518 | ret = rdma_backend_qp_state_rts(&qp->backend_qp, qp->qp_type, | |
519 | sq_psn, qkey, | |
520 | attr_mask & IBV_QP_QKEY); | |
521 | if (ret) { | |
522 | return -EIO; | |
523 | } | |
524 | } | |
525 | } | |
526 | ||
527 | return 0; | |
528 | } | |
529 | ||
c99f2174 YS |
530 | int rdma_rm_query_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
531 | uint32_t qp_handle, struct ibv_qp_attr *attr, | |
532 | int attr_mask, struct ibv_qp_init_attr *init_attr) | |
533 | { | |
534 | RdmaRmQP *qp; | |
535 | ||
c99f2174 YS |
536 | qp = rdma_rm_get_qp(dev_res, qp_handle); |
537 | if (!qp) { | |
538 | return -EINVAL; | |
539 | } | |
540 | ||
c99f2174 YS |
541 | return rdma_backend_query_qp(&qp->backend_qp, attr, attr_mask, init_attr); |
542 | } | |
543 | ||
ef6d4ccd YS |
544 | void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle) |
545 | { | |
546 | RdmaRmQP *qp; | |
547 | GBytes *key; | |
548 | ||
549 | key = g_bytes_new(&qp_handle, sizeof(qp_handle)); | |
550 | qp = g_hash_table_lookup(dev_res->qp_hash, key); | |
551 | g_hash_table_remove(dev_res->qp_hash, key); | |
552 | g_bytes_unref(key); | |
553 | ||
554 | if (!qp) { | |
555 | return; | |
556 | } | |
557 | ||
bf441451 | 558 | rdma_backend_destroy_qp(&qp->backend_qp, dev_res); |
ef6d4ccd | 559 | |
4d71b38a | 560 | rdma_res_tbl_dealloc(&dev_res->qp_tbl, qp->qpn); |
ef6d4ccd YS |
561 | } |
562 | ||
cdc84058 KH |
563 | RdmaRmSRQ *rdma_rm_get_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle) |
564 | { | |
565 | return rdma_res_tbl_get(&dev_res->srq_tbl, srq_handle); | |
566 | } | |
567 | ||
568 | int rdma_rm_alloc_srq(RdmaDeviceResources *dev_res, uint32_t pd_handle, | |
569 | uint32_t max_wr, uint32_t max_sge, uint32_t srq_limit, | |
570 | uint32_t *srq_handle, void *opaque) | |
571 | { | |
572 | RdmaRmSRQ *srq; | |
573 | RdmaRmPD *pd; | |
574 | int rc; | |
575 | ||
576 | pd = rdma_rm_get_pd(dev_res, pd_handle); | |
577 | if (!pd) { | |
578 | return -EINVAL; | |
579 | } | |
580 | ||
581 | srq = rdma_res_tbl_alloc(&dev_res->srq_tbl, srq_handle); | |
582 | if (!srq) { | |
583 | return -ENOMEM; | |
584 | } | |
585 | ||
586 | rc = rdma_backend_create_srq(&srq->backend_srq, &pd->backend_pd, | |
587 | max_wr, max_sge, srq_limit); | |
588 | if (rc) { | |
589 | rc = -EIO; | |
590 | goto out_dealloc_srq; | |
591 | } | |
592 | ||
593 | srq->opaque = opaque; | |
594 | ||
595 | return 0; | |
596 | ||
597 | out_dealloc_srq: | |
598 | rdma_res_tbl_dealloc(&dev_res->srq_tbl, *srq_handle); | |
599 | ||
600 | return rc; | |
601 | } | |
602 | ||
603 | int rdma_rm_query_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle, | |
604 | struct ibv_srq_attr *srq_attr) | |
605 | { | |
606 | RdmaRmSRQ *srq; | |
607 | ||
608 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
609 | if (!srq) { | |
610 | return -EINVAL; | |
611 | } | |
612 | ||
613 | return rdma_backend_query_srq(&srq->backend_srq, srq_attr); | |
614 | } | |
615 | ||
616 | int rdma_rm_modify_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle, | |
617 | struct ibv_srq_attr *srq_attr, int srq_attr_mask) | |
618 | { | |
619 | RdmaRmSRQ *srq; | |
620 | ||
621 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
622 | if (!srq) { | |
623 | return -EINVAL; | |
624 | } | |
625 | ||
626 | if ((srq_attr_mask & IBV_SRQ_LIMIT) && | |
627 | (srq_attr->srq_limit == 0)) { | |
628 | return -EINVAL; | |
629 | } | |
630 | ||
631 | if ((srq_attr_mask & IBV_SRQ_MAX_WR) && | |
632 | (srq_attr->max_wr == 0)) { | |
633 | return -EINVAL; | |
634 | } | |
635 | ||
636 | return rdma_backend_modify_srq(&srq->backend_srq, srq_attr, | |
637 | srq_attr_mask); | |
638 | } | |
639 | ||
640 | void rdma_rm_dealloc_srq(RdmaDeviceResources *dev_res, uint32_t srq_handle) | |
641 | { | |
642 | RdmaRmSRQ *srq; | |
643 | ||
644 | srq = rdma_rm_get_srq(dev_res, srq_handle); | |
645 | if (!srq) { | |
646 | return; | |
647 | } | |
648 | ||
649 | rdma_backend_destroy_srq(&srq->backend_srq, dev_res); | |
650 | rdma_res_tbl_dealloc(&dev_res->srq_tbl, srq_handle); | |
651 | } | |
652 | ||
ef6d4ccd YS |
653 | void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id) |
654 | { | |
655 | void **cqe_ctx; | |
656 | ||
4d71b38a | 657 | cqe_ctx = rdma_res_tbl_get(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
658 | if (!cqe_ctx) { |
659 | return NULL; | |
660 | } | |
661 | ||
ef6d4ccd YS |
662 | return *cqe_ctx; |
663 | } | |
664 | ||
665 | int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id, | |
666 | void *ctx) | |
667 | { | |
668 | void **cqe_ctx; | |
669 | ||
4d71b38a | 670 | cqe_ctx = rdma_res_tbl_alloc(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
671 | if (!cqe_ctx) { |
672 | return -ENOMEM; | |
673 | } | |
674 | ||
ef6d4ccd YS |
675 | *cqe_ctx = ctx; |
676 | ||
677 | return 0; | |
678 | } | |
679 | ||
680 | void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id) | |
681 | { | |
4d71b38a | 682 | rdma_res_tbl_dealloc(&dev_res->cqe_ctx_tbl, cqe_ctx_id); |
ef6d4ccd YS |
683 | } |
684 | ||
2b05705d YS |
685 | int rdma_rm_add_gid(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
686 | const char *ifname, union ibv_gid *gid, int gid_idx) | |
687 | { | |
688 | int rc; | |
689 | ||
690 | rc = rdma_backend_add_gid(backend_dev, ifname, gid); | |
691 | if (rc) { | |
2b05705d YS |
692 | return -EINVAL; |
693 | } | |
694 | ||
14c74f72 | 695 | memcpy(&dev_res->port.gid_tbl[gid_idx].gid, gid, sizeof(*gid)); |
2b05705d YS |
696 | |
697 | return 0; | |
698 | } | |
699 | ||
700 | int rdma_rm_del_gid(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, | |
701 | const char *ifname, int gid_idx) | |
702 | { | |
703 | int rc; | |
704 | ||
305fd2ba YS |
705 | if (!dev_res->port.gid_tbl[gid_idx].gid.global.interface_id) { |
706 | return 0; | |
707 | } | |
708 | ||
2b05705d | 709 | rc = rdma_backend_del_gid(backend_dev, ifname, |
14c74f72 | 710 | &dev_res->port.gid_tbl[gid_idx].gid); |
2b05705d | 711 | if (rc) { |
2b05705d YS |
712 | return -EINVAL; |
713 | } | |
714 | ||
14c74f72 YS |
715 | memset(dev_res->port.gid_tbl[gid_idx].gid.raw, 0, |
716 | sizeof(dev_res->port.gid_tbl[gid_idx].gid)); | |
717 | dev_res->port.gid_tbl[gid_idx].backend_gid_index = -1; | |
2b05705d YS |
718 | |
719 | return 0; | |
720 | } | |
721 | ||
722 | int rdma_rm_get_backend_gid_index(RdmaDeviceResources *dev_res, | |
723 | RdmaBackendDev *backend_dev, int sgid_idx) | |
724 | { | |
a5fe209d | 725 | if (unlikely(sgid_idx < 0 || sgid_idx >= MAX_PORT_GIDS)) { |
4d71b38a | 726 | rdma_error_report("Got invalid sgid_idx %d", sgid_idx); |
2b05705d YS |
727 | return -EINVAL; |
728 | } | |
729 | ||
14c74f72 YS |
730 | if (unlikely(dev_res->port.gid_tbl[sgid_idx].backend_gid_index == -1)) { |
731 | dev_res->port.gid_tbl[sgid_idx].backend_gid_index = | |
2b05705d | 732 | rdma_backend_get_gid_index(backend_dev, |
14c74f72 | 733 | &dev_res->port.gid_tbl[sgid_idx].gid); |
2b05705d YS |
734 | } |
735 | ||
14c74f72 | 736 | return dev_res->port.gid_tbl[sgid_idx].backend_gid_index; |
2b05705d YS |
737 | } |
738 | ||
ef6d4ccd YS |
739 | static void destroy_qp_hash_key(gpointer data) |
740 | { | |
741 | g_bytes_unref(data); | |
742 | } | |
743 | ||
2b05705d YS |
744 | static void init_ports(RdmaDeviceResources *dev_res) |
745 | { | |
14c74f72 | 746 | int i; |
2b05705d | 747 | |
14c74f72 | 748 | memset(&dev_res->port, 0, sizeof(dev_res->port)); |
2b05705d | 749 | |
14c74f72 YS |
750 | dev_res->port.state = IBV_PORT_DOWN; |
751 | for (i = 0; i < MAX_PORT_GIDS; i++) { | |
752 | dev_res->port.gid_tbl[i].backend_gid_index = -1; | |
2b05705d YS |
753 | } |
754 | } | |
755 | ||
756 | static void fini_ports(RdmaDeviceResources *dev_res, | |
757 | RdmaBackendDev *backend_dev, const char *ifname) | |
758 | { | |
759 | int i; | |
760 | ||
14c74f72 | 761 | dev_res->port.state = IBV_PORT_DOWN; |
2b05705d YS |
762 | for (i = 0; i < MAX_PORT_GIDS; i++) { |
763 | rdma_rm_del_gid(dev_res, backend_dev, ifname, i); | |
764 | } | |
765 | } | |
766 | ||
4d71b38a | 767 | int rdma_rm_init(RdmaDeviceResources *dev_res, struct ibv_device_attr *dev_attr) |
ef6d4ccd YS |
768 | { |
769 | dev_res->qp_hash = g_hash_table_new_full(g_bytes_hash, g_bytes_equal, | |
770 | destroy_qp_hash_key, NULL); | |
771 | if (!dev_res->qp_hash) { | |
772 | return -ENOMEM; | |
773 | } | |
774 | ||
775 | res_tbl_init("PD", &dev_res->pd_tbl, dev_attr->max_pd, sizeof(RdmaRmPD)); | |
776 | res_tbl_init("CQ", &dev_res->cq_tbl, dev_attr->max_cq, sizeof(RdmaRmCQ)); | |
777 | res_tbl_init("MR", &dev_res->mr_tbl, dev_attr->max_mr, sizeof(RdmaRmMR)); | |
778 | res_tbl_init("QP", &dev_res->qp_tbl, dev_attr->max_qp, sizeof(RdmaRmQP)); | |
779 | res_tbl_init("CQE_CTX", &dev_res->cqe_ctx_tbl, dev_attr->max_qp * | |
780 | dev_attr->max_qp_wr, sizeof(void *)); | |
781 | res_tbl_init("UC", &dev_res->uc_tbl, MAX_UCS, sizeof(RdmaRmUC)); | |
cdc84058 KH |
782 | res_tbl_init("SRQ", &dev_res->srq_tbl, dev_attr->max_srq, |
783 | sizeof(RdmaRmSRQ)); | |
ef6d4ccd | 784 | |
2b05705d YS |
785 | init_ports(dev_res); |
786 | ||
2cfa9530 YS |
787 | qemu_mutex_init(&dev_res->lock); |
788 | ||
c2dd117b YS |
789 | memset(&dev_res->stats, 0, sizeof(dev_res->stats)); |
790 | atomic_set(&dev_res->stats.missing_cqe, 0); | |
791 | ||
ef6d4ccd YS |
792 | return 0; |
793 | } | |
794 | ||
2b05705d YS |
795 | void rdma_rm_fini(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev, |
796 | const char *ifname) | |
ef6d4ccd | 797 | { |
2cfa9530 YS |
798 | qemu_mutex_destroy(&dev_res->lock); |
799 | ||
2b05705d YS |
800 | fini_ports(dev_res, backend_dev, ifname); |
801 | ||
cdc84058 | 802 | res_tbl_free(&dev_res->srq_tbl); |
ef6d4ccd YS |
803 | res_tbl_free(&dev_res->uc_tbl); |
804 | res_tbl_free(&dev_res->cqe_ctx_tbl); | |
805 | res_tbl_free(&dev_res->qp_tbl); | |
ef6d4ccd | 806 | res_tbl_free(&dev_res->mr_tbl); |
7131c4b0 | 807 | res_tbl_free(&dev_res->cq_tbl); |
ef6d4ccd | 808 | res_tbl_free(&dev_res->pd_tbl); |
7131c4b0 | 809 | |
a1aa88b7 YS |
810 | if (dev_res->qp_hash) { |
811 | g_hash_table_destroy(dev_res->qp_hash); | |
812 | } | |
ef6d4ccd | 813 | } |