]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - drivers/infiniband/core/uverbs_cmd.c
projects / mirror_ubuntu-focal-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'sh-pfc-for-v5.1-tag2' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-focal-kernel.git] / drivers / infiniband / core / uverbs_cmd.c
1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #include <linux/file.h>
37 #include <linux/fs.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
40
41 #include <linux/uaccess.h>
42
43 #include <rdma/uverbs_types.h>
44 #include <rdma/uverbs_std_types.h>
45 #include "rdma_core.h"
46
47 #include "uverbs.h"
48 #include "core_priv.h"
49
50 /*
51 * Copy a response to userspace. If the provided 'resp' is larger than the
52 * user buffer it is silently truncated. If the user provided a larger buffer
53 * then the trailing portion is zero filled.
54 *
55 * These semantics are intended to support future extension of the output
56 * structures.
57 */
58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
59 size_t resp_len)
60 {
61 int ret;
62
63 if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
64 return uverbs_copy_to_struct_or_zero(
65 attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
66
67 if (copy_to_user(attrs->ucore.outbuf, resp,
68 min(attrs->ucore.outlen, resp_len)))
69 return -EFAULT;
70
71 if (resp_len < attrs->ucore.outlen) {
72 /*
73 * Zero fill any extra memory that user
74 * space might have provided.
75 */
76 ret = clear_user(attrs->ucore.outbuf + resp_len,
77 attrs->ucore.outlen - resp_len);
78 if (ret)
79 return -EFAULT;
80 }
81
82 return 0;
83 }
84
85 /*
86 * Copy a request from userspace. If the provided 'req' is larger than the
87 * user buffer then the user buffer is zero extended into the 'req'. If 'req'
88 * is smaller than the user buffer then the uncopied bytes in the user buffer
89 * must be zero.
90 */
91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
92 size_t req_len)
93 {
94 if (copy_from_user(req, attrs->ucore.inbuf,
95 min(attrs->ucore.inlen, req_len)))
96 return -EFAULT;
97
98 if (attrs->ucore.inlen < req_len) {
99 memset(req + attrs->ucore.inlen, 0,
100 req_len - attrs->ucore.inlen);
101 } else if (attrs->ucore.inlen > req_len) {
102 if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
103 attrs->ucore.inlen - req_len))
104 return -EOPNOTSUPP;
105 }
106 return 0;
107 }
108
109 /*
110 * Generate the value for the 'response_length' protocol used by write_ex.
111 * This is the number of bytes the kernel actually wrote. Userspace can use
112 * this to detect what structure members in the response the kernel
113 * understood.
114 */
115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
116 size_t resp_len)
117 {
118 return min_t(size_t, attrs->ucore.outlen, resp_len);
119 }
120
121 /*
122 * The iterator version of the request interface is for handlers that need to
123 * step over a flex array at the end of a command header.
124 */
125 struct uverbs_req_iter {
126 const void __user *cur;
127 const void __user *end;
128 };
129
130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
131 struct uverbs_req_iter *iter,
132 void *req,
133 size_t req_len)
134 {
135 if (attrs->ucore.inlen < req_len)
136 return -ENOSPC;
137
138 if (copy_from_user(req, attrs->ucore.inbuf, req_len))
139 return -EFAULT;
140
141 iter->cur = attrs->ucore.inbuf + req_len;
142 iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
143 return 0;
144 }
145
146 static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
147 size_t len)
148 {
149 if (iter->cur + len > iter->end)
150 return -ENOSPC;
151
152 if (copy_from_user(val, iter->cur, len))
153 return -EFAULT;
154
155 iter->cur += len;
156 return 0;
157 }
158
159 static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
160 size_t len)
161 {
162 const void __user *res = iter->cur;
163
164 if (iter->cur + len > iter->end)
165 return ERR_PTR(-ENOSPC);
166 iter->cur += len;
167 return res;
168 }
169
170 static int uverbs_request_finish(struct uverbs_req_iter *iter)
171 {
172 if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
173 return -EOPNOTSUPP;
174 return 0;
175 }
176
177 static struct ib_uverbs_completion_event_file *
178 _ib_uverbs_lookup_comp_file(s32 fd, const struct uverbs_attr_bundle *attrs)
179 {
180 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
181 fd, attrs);
182
183 if (IS_ERR(uobj))
184 return (void *)uobj;
185
186 uverbs_uobject_get(uobj);
187 uobj_put_read(uobj);
188
189 return container_of(uobj, struct ib_uverbs_completion_event_file,
190 uobj);
191 }
192 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \
193 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
194
195 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
196 {
197 struct ib_uverbs_file *file = attrs->ufile;
198 struct ib_uverbs_get_context cmd;
199 struct ib_uverbs_get_context_resp resp;
200 struct ib_ucontext *ucontext;
201 struct file *filp;
202 struct ib_rdmacg_object cg_obj;
203 struct ib_device *ib_dev;
204 int ret;
205
206 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
207 if (ret)
208 return ret;
209
210 mutex_lock(&file->ucontext_lock);
211 ib_dev = srcu_dereference(file->device->ib_dev,
212 &file->device->disassociate_srcu);
213 if (!ib_dev) {
214 ret = -EIO;
215 goto err;
216 }
217
218 if (file->ucontext) {
219 ret = -EINVAL;
220 goto err;
221 }
222
223 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
224 if (ret)
225 goto err;
226
227 ucontext = ib_dev->ops.alloc_ucontext(ib_dev, &attrs->driver_udata);
228 if (IS_ERR(ucontext)) {
229 ret = PTR_ERR(ucontext);
230 goto err_alloc;
231 }
232
233 ucontext->device = ib_dev;
234 ucontext->cg_obj = cg_obj;
235 /* ufile is required when some objects are released */
236 ucontext->ufile = file;
237
238 ucontext->closing = false;
239 ucontext->cleanup_retryable = false;
240
241 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
242 mutex_init(&ucontext->per_mm_list_lock);
243 INIT_LIST_HEAD(&ucontext->per_mm_list);
244 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
245 ucontext->invalidate_range = NULL;
246
247 #endif
248
249 resp.num_comp_vectors = file->device->num_comp_vectors;
250
251 ret = get_unused_fd_flags(O_CLOEXEC);
252 if (ret < 0)
253 goto err_free;
254 resp.async_fd = ret;
255
256 filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
257 if (IS_ERR(filp)) {
258 ret = PTR_ERR(filp);
259 goto err_fd;
260 }
261
262 ret = uverbs_response(attrs, &resp, sizeof(resp));
263 if (ret)
264 goto err_file;
265
266 fd_install(resp.async_fd, filp);
267
268 ucontext->res.type = RDMA_RESTRACK_CTX;
269 rdma_restrack_uadd(&ucontext->res);
270
271 /*
272 * Make sure that ib_uverbs_get_ucontext() sees the pointer update
273 * only after all writes to setup the ucontext have completed
274 */
275 smp_store_release(&file->ucontext, ucontext);
276
277 mutex_unlock(&file->ucontext_lock);
278
279 return 0;
280
281 err_file:
282 ib_uverbs_free_async_event_file(file);
283 fput(filp);
284
285 err_fd:
286 put_unused_fd(resp.async_fd);
287
288 err_free:
289 ib_dev->ops.dealloc_ucontext(ucontext);
290
291 err_alloc:
292 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
293
294 err:
295 mutex_unlock(&file->ucontext_lock);
296 return ret;
297 }
298
299 static void copy_query_dev_fields(struct ib_ucontext *ucontext,
300 struct ib_uverbs_query_device_resp *resp,
301 struct ib_device_attr *attr)
302 {
303 struct ib_device *ib_dev = ucontext->device;
304
305 resp->fw_ver = attr->fw_ver;
306 resp->node_guid = ib_dev->node_guid;
307 resp->sys_image_guid = attr->sys_image_guid;
308 resp->max_mr_size = attr->max_mr_size;
309 resp->page_size_cap = attr->page_size_cap;
310 resp->vendor_id = attr->vendor_id;
311 resp->vendor_part_id = attr->vendor_part_id;
312 resp->hw_ver = attr->hw_ver;
313 resp->max_qp = attr->max_qp;
314 resp->max_qp_wr = attr->max_qp_wr;
315 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags);
316 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge);
317 resp->max_sge_rd = attr->max_sge_rd;
318 resp->max_cq = attr->max_cq;
319 resp->max_cqe = attr->max_cqe;
320 resp->max_mr = attr->max_mr;
321 resp->max_pd = attr->max_pd;
322 resp->max_qp_rd_atom = attr->max_qp_rd_atom;
323 resp->max_ee_rd_atom = attr->max_ee_rd_atom;
324 resp->max_res_rd_atom = attr->max_res_rd_atom;
325 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom;
326 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom;
327 resp->atomic_cap = attr->atomic_cap;
328 resp->max_ee = attr->max_ee;
329 resp->max_rdd = attr->max_rdd;
330 resp->max_mw = attr->max_mw;
331 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp;
332 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp;
333 resp->max_mcast_grp = attr->max_mcast_grp;
334 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
335 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
336 resp->max_ah = attr->max_ah;
337 resp->max_fmr = attr->max_fmr;
338 resp->max_map_per_fmr = attr->max_map_per_fmr;
339 resp->max_srq = attr->max_srq;
340 resp->max_srq_wr = attr->max_srq_wr;
341 resp->max_srq_sge = attr->max_srq_sge;
342 resp->max_pkeys = attr->max_pkeys;
343 resp->local_ca_ack_delay = attr->local_ca_ack_delay;
344 resp->phys_port_cnt = ib_dev->phys_port_cnt;
345 }
346
347 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
348 {
349 struct ib_uverbs_query_device cmd;
350 struct ib_uverbs_query_device_resp resp;
351 struct ib_ucontext *ucontext;
352 int ret;
353
354 ucontext = ib_uverbs_get_ucontext(attrs);
355 if (IS_ERR(ucontext))
356 return PTR_ERR(ucontext);
357
358 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
359 if (ret)
360 return ret;
361
362 memset(&resp, 0, sizeof resp);
363 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
364
365 return uverbs_response(attrs, &resp, sizeof(resp));
366 }
367
368 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
369 {
370 struct ib_uverbs_query_port cmd;
371 struct ib_uverbs_query_port_resp resp;
372 struct ib_port_attr attr;
373 int ret;
374 struct ib_ucontext *ucontext;
375 struct ib_device *ib_dev;
376
377 ucontext = ib_uverbs_get_ucontext(attrs);
378 if (IS_ERR(ucontext))
379 return PTR_ERR(ucontext);
380 ib_dev = ucontext->device;
381
382 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
383 if (ret)
384 return ret;
385
386 ret = ib_query_port(ib_dev, cmd.port_num, &attr);
387 if (ret)
388 return ret;
389
390 memset(&resp, 0, sizeof resp);
391 copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
392
393 return uverbs_response(attrs, &resp, sizeof(resp));
394 }
395
396 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
397 {
398 struct ib_uverbs_alloc_pd cmd;
399 struct ib_uverbs_alloc_pd_resp resp;
400 struct ib_uobject *uobj;
401 struct ib_pd *pd;
402 int ret;
403 struct ib_device *ib_dev;
404
405 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
406 if (ret)
407 return ret;
408
409 uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
410 if (IS_ERR(uobj))
411 return PTR_ERR(uobj);
412
413 pd = ib_dev->ops.alloc_pd(ib_dev, uobj->context, &attrs->driver_udata);
414 if (IS_ERR(pd)) {
415 ret = PTR_ERR(pd);
416 goto err;
417 }
418
419 pd->device = ib_dev;
420 pd->uobject = uobj;
421 pd->__internal_mr = NULL;
422 atomic_set(&pd->usecnt, 0);
423
424 uobj->object = pd;
425 memset(&resp, 0, sizeof resp);
426 resp.pd_handle = uobj->id;
427 pd->res.type = RDMA_RESTRACK_PD;
428 rdma_restrack_uadd(&pd->res);
429
430 ret = uverbs_response(attrs, &resp, sizeof(resp));
431 if (ret)
432 goto err_copy;
433
434 return uobj_alloc_commit(uobj);
435
436 err_copy:
437 ib_dealloc_pd(pd);
438
439 err:
440 uobj_alloc_abort(uobj);
441 return ret;
442 }
443
444 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
445 {
446 struct ib_uverbs_dealloc_pd cmd;
447 int ret;
448
449 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
450 if (ret)
451 return ret;
452
453 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
454 }
455
456 struct xrcd_table_entry {
457 struct rb_node node;
458 struct ib_xrcd *xrcd;
459 struct inode *inode;
460 };
461
462 static int xrcd_table_insert(struct ib_uverbs_device *dev,
463 struct inode *inode,
464 struct ib_xrcd *xrcd)
465 {
466 struct xrcd_table_entry *entry, *scan;
467 struct rb_node **p = &dev->xrcd_tree.rb_node;
468 struct rb_node *parent = NULL;
469
470 entry = kmalloc(sizeof *entry, GFP_KERNEL);
471 if (!entry)
472 return -ENOMEM;
473
474 entry->xrcd = xrcd;
475 entry->inode = inode;
476
477 while (*p) {
478 parent = *p;
479 scan = rb_entry(parent, struct xrcd_table_entry, node);
480
481 if (inode < scan->inode) {
482 p = &(*p)->rb_left;
483 } else if (inode > scan->inode) {
484 p = &(*p)->rb_right;
485 } else {
486 kfree(entry);
487 return -EEXIST;
488 }
489 }
490
491 rb_link_node(&entry->node, parent, p);
492 rb_insert_color(&entry->node, &dev->xrcd_tree);
493 igrab(inode);
494 return 0;
495 }
496
497 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
498 struct inode *inode)
499 {
500 struct xrcd_table_entry *entry;
501 struct rb_node *p = dev->xrcd_tree.rb_node;
502
503 while (p) {
504 entry = rb_entry(p, struct xrcd_table_entry, node);
505
506 if (inode < entry->inode)
507 p = p->rb_left;
508 else if (inode > entry->inode)
509 p = p->rb_right;
510 else
511 return entry;
512 }
513
514 return NULL;
515 }
516
517 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
518 {
519 struct xrcd_table_entry *entry;
520
521 entry = xrcd_table_search(dev, inode);
522 if (!entry)
523 return NULL;
524
525 return entry->xrcd;
526 }
527
528 static void xrcd_table_delete(struct ib_uverbs_device *dev,
529 struct inode *inode)
530 {
531 struct xrcd_table_entry *entry;
532
533 entry = xrcd_table_search(dev, inode);
534 if (entry) {
535 iput(inode);
536 rb_erase(&entry->node, &dev->xrcd_tree);
537 kfree(entry);
538 }
539 }
540
541 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
542 {
543 struct ib_uverbs_device *ibudev = attrs->ufile->device;
544 struct ib_uverbs_open_xrcd cmd;
545 struct ib_uverbs_open_xrcd_resp resp;
546 struct ib_uxrcd_object *obj;
547 struct ib_xrcd *xrcd = NULL;
548 struct fd f = {NULL, 0};
549 struct inode *inode = NULL;
550 int ret = 0;
551 int new_xrcd = 0;
552 struct ib_device *ib_dev;
553
554 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
555 if (ret)
556 return ret;
557
558 mutex_lock(&ibudev->xrcd_tree_mutex);
559
560 if (cmd.fd != -1) {
561 /* search for file descriptor */
562 f = fdget(cmd.fd);
563 if (!f.file) {
564 ret = -EBADF;
565 goto err_tree_mutex_unlock;
566 }
567
568 inode = file_inode(f.file);
569 xrcd = find_xrcd(ibudev, inode);
570 if (!xrcd && !(cmd.oflags & O_CREAT)) {
571 /* no file descriptor. Need CREATE flag */
572 ret = -EAGAIN;
573 goto err_tree_mutex_unlock;
574 }
575
576 if (xrcd && cmd.oflags & O_EXCL) {
577 ret = -EINVAL;
578 goto err_tree_mutex_unlock;
579 }
580 }
581
582 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
583 &ib_dev);
584 if (IS_ERR(obj)) {
585 ret = PTR_ERR(obj);
586 goto err_tree_mutex_unlock;
587 }
588
589 if (!xrcd) {
590 xrcd = ib_dev->ops.alloc_xrcd(ib_dev, obj->uobject.context,
591 &attrs->driver_udata);
592 if (IS_ERR(xrcd)) {
593 ret = PTR_ERR(xrcd);
594 goto err;
595 }
596
597 xrcd->inode = inode;
598 xrcd->device = ib_dev;
599 atomic_set(&xrcd->usecnt, 0);
600 mutex_init(&xrcd->tgt_qp_mutex);
601 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
602 new_xrcd = 1;
603 }
604
605 atomic_set(&obj->refcnt, 0);
606 obj->uobject.object = xrcd;
607 memset(&resp, 0, sizeof resp);
608 resp.xrcd_handle = obj->uobject.id;
609
610 if (inode) {
611 if (new_xrcd) {
612 /* create new inode/xrcd table entry */
613 ret = xrcd_table_insert(ibudev, inode, xrcd);
614 if (ret)
615 goto err_dealloc_xrcd;
616 }
617 atomic_inc(&xrcd->usecnt);
618 }
619
620 ret = uverbs_response(attrs, &resp, sizeof(resp));
621 if (ret)
622 goto err_copy;
623
624 if (f.file)
625 fdput(f);
626
627 mutex_unlock(&ibudev->xrcd_tree_mutex);
628
629 return uobj_alloc_commit(&obj->uobject);
630
631 err_copy:
632 if (inode) {
633 if (new_xrcd)
634 xrcd_table_delete(ibudev, inode);
635 atomic_dec(&xrcd->usecnt);
636 }
637
638 err_dealloc_xrcd:
639 ib_dealloc_xrcd(xrcd);
640
641 err:
642 uobj_alloc_abort(&obj->uobject);
643
644 err_tree_mutex_unlock:
645 if (f.file)
646 fdput(f);
647
648 mutex_unlock(&ibudev->xrcd_tree_mutex);
649
650 return ret;
651 }
652
653 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
654 {
655 struct ib_uverbs_close_xrcd cmd;
656 int ret;
657
658 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
659 if (ret)
660 return ret;
661
662 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
663 }
664
665 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject,
666 struct ib_xrcd *xrcd,
667 enum rdma_remove_reason why)
668 {
669 struct inode *inode;
670 int ret;
671 struct ib_uverbs_device *dev = uobject->context->ufile->device;
672
673 inode = xrcd->inode;
674 if (inode && !atomic_dec_and_test(&xrcd->usecnt))
675 return 0;
676
677 ret = ib_dealloc_xrcd(xrcd);
678
679 if (ib_is_destroy_retryable(ret, why, uobject)) {
680 atomic_inc(&xrcd->usecnt);
681 return ret;
682 }
683
684 if (inode)
685 xrcd_table_delete(dev, inode);
686
687 return ret;
688 }
689
690 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
691 {
692 struct ib_uverbs_reg_mr cmd;
693 struct ib_uverbs_reg_mr_resp resp;
694 struct ib_uobject *uobj;
695 struct ib_pd *pd;
696 struct ib_mr *mr;
697 int ret;
698 struct ib_device *ib_dev;
699
700 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
701 if (ret)
702 return ret;
703
704 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
705 return -EINVAL;
706
707 ret = ib_check_mr_access(cmd.access_flags);
708 if (ret)
709 return ret;
710
711 uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
712 if (IS_ERR(uobj))
713 return PTR_ERR(uobj);
714
715 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
716 if (!pd) {
717 ret = -EINVAL;
718 goto err_free;
719 }
720
721 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
722 if (!(pd->device->attrs.device_cap_flags &
723 IB_DEVICE_ON_DEMAND_PAGING)) {
724 pr_debug("ODP support not available\n");
725 ret = -EINVAL;
726 goto err_put;
727 }
728 }
729
730 mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
731 cmd.access_flags,
732 &attrs->driver_udata);
733 if (IS_ERR(mr)) {
734 ret = PTR_ERR(mr);
735 goto err_put;
736 }
737
738 mr->device = pd->device;
739 mr->pd = pd;
740 mr->dm = NULL;
741 mr->uobject = uobj;
742 atomic_inc(&pd->usecnt);
743 mr->res.type = RDMA_RESTRACK_MR;
744 rdma_restrack_uadd(&mr->res);
745
746 uobj->object = mr;
747
748 memset(&resp, 0, sizeof resp);
749 resp.lkey = mr->lkey;
750 resp.rkey = mr->rkey;
751 resp.mr_handle = uobj->id;
752
753 ret = uverbs_response(attrs, &resp, sizeof(resp));
754 if (ret)
755 goto err_copy;
756
757 uobj_put_obj_read(pd);
758
759 return uobj_alloc_commit(uobj);
760
761 err_copy:
762 ib_dereg_mr(mr);
763
764 err_put:
765 uobj_put_obj_read(pd);
766
767 err_free:
768 uobj_alloc_abort(uobj);
769 return ret;
770 }
771
772 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
773 {
774 struct ib_uverbs_rereg_mr cmd;
775 struct ib_uverbs_rereg_mr_resp resp;
776 struct ib_pd *pd = NULL;
777 struct ib_mr *mr;
778 struct ib_pd *old_pd;
779 int ret;
780 struct ib_uobject *uobj;
781
782 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
783 if (ret)
784 return ret;
785
786 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
787 return -EINVAL;
788
789 if ((cmd.flags & IB_MR_REREG_TRANS) &&
790 (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
791 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
792 return -EINVAL;
793
794 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
795 if (IS_ERR(uobj))
796 return PTR_ERR(uobj);
797
798 mr = uobj->object;
799
800 if (mr->dm) {
801 ret = -EINVAL;
802 goto put_uobjs;
803 }
804
805 if (cmd.flags & IB_MR_REREG_ACCESS) {
806 ret = ib_check_mr_access(cmd.access_flags);
807 if (ret)
808 goto put_uobjs;
809 }
810
811 if (cmd.flags & IB_MR_REREG_PD) {
812 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
813 attrs);
814 if (!pd) {
815 ret = -EINVAL;
816 goto put_uobjs;
817 }
818 }
819
820 old_pd = mr->pd;
821 ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
822 cmd.length, cmd.hca_va,
823 cmd.access_flags, pd,
824 &attrs->driver_udata);
825 if (!ret) {
826 if (cmd.flags & IB_MR_REREG_PD) {
827 atomic_inc(&pd->usecnt);
828 mr->pd = pd;
829 atomic_dec(&old_pd->usecnt);
830 }
831 } else {
832 goto put_uobj_pd;
833 }
834
835 memset(&resp, 0, sizeof(resp));
836 resp.lkey = mr->lkey;
837 resp.rkey = mr->rkey;
838
839 ret = uverbs_response(attrs, &resp, sizeof(resp));
840
841 put_uobj_pd:
842 if (cmd.flags & IB_MR_REREG_PD)
843 uobj_put_obj_read(pd);
844
845 put_uobjs:
846 uobj_put_write(uobj);
847
848 return ret;
849 }
850
851 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
852 {
853 struct ib_uverbs_dereg_mr cmd;
854 int ret;
855
856 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
857 if (ret)
858 return ret;
859
860 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
861 }
862
863 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
864 {
865 struct ib_uverbs_alloc_mw cmd;
866 struct ib_uverbs_alloc_mw_resp resp;
867 struct ib_uobject *uobj;
868 struct ib_pd *pd;
869 struct ib_mw *mw;
870 int ret;
871 struct ib_device *ib_dev;
872
873 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
874 if (ret)
875 return ret;
876
877 uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
878 if (IS_ERR(uobj))
879 return PTR_ERR(uobj);
880
881 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
882 if (!pd) {
883 ret = -EINVAL;
884 goto err_free;
885 }
886
887 mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata);
888 if (IS_ERR(mw)) {
889 ret = PTR_ERR(mw);
890 goto err_put;
891 }
892
893 mw->device = pd->device;
894 mw->pd = pd;
895 mw->uobject = uobj;
896 atomic_inc(&pd->usecnt);
897
898 uobj->object = mw;
899
900 memset(&resp, 0, sizeof(resp));
901 resp.rkey = mw->rkey;
902 resp.mw_handle = uobj->id;
903
904 ret = uverbs_response(attrs, &resp, sizeof(resp));
905 if (ret)
906 goto err_copy;
907
908 uobj_put_obj_read(pd);
909 return uobj_alloc_commit(uobj);
910
911 err_copy:
912 uverbs_dealloc_mw(mw);
913 err_put:
914 uobj_put_obj_read(pd);
915 err_free:
916 uobj_alloc_abort(uobj);
917 return ret;
918 }
919
920 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
921 {
922 struct ib_uverbs_dealloc_mw cmd;
923 int ret;
924
925 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
926 if (ret)
927 return ret;
928
929 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
930 }
931
932 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
933 {
934 struct ib_uverbs_create_comp_channel cmd;
935 struct ib_uverbs_create_comp_channel_resp resp;
936 struct ib_uobject *uobj;
937 struct ib_uverbs_completion_event_file *ev_file;
938 struct ib_device *ib_dev;
939 int ret;
940
941 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
942 if (ret)
943 return ret;
944
945 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
946 if (IS_ERR(uobj))
947 return PTR_ERR(uobj);
948
949 resp.fd = uobj->id;
950
951 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
952 uobj);
953 ib_uverbs_init_event_queue(&ev_file->ev_queue);
954
955 ret = uverbs_response(attrs, &resp, sizeof(resp));
956 if (ret) {
957 uobj_alloc_abort(uobj);
958 return ret;
959 }
960
961 return uobj_alloc_commit(uobj);
962 }
963
964 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
965 struct ib_uverbs_ex_create_cq *cmd)
966 {
967 struct ib_ucq_object *obj;
968 struct ib_uverbs_completion_event_file *ev_file = NULL;
969 struct ib_cq *cq;
970 int ret;
971 struct ib_uverbs_ex_create_cq_resp resp;
972 struct ib_cq_init_attr attr = {};
973 struct ib_device *ib_dev;
974
975 if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
976 return ERR_PTR(-EINVAL);
977
978 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
979 &ib_dev);
980 if (IS_ERR(obj))
981 return obj;
982
983 if (cmd->comp_channel >= 0) {
984 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
985 if (IS_ERR(ev_file)) {
986 ret = PTR_ERR(ev_file);
987 goto err;
988 }
989 }
990
991 obj->uobject.user_handle = cmd->user_handle;
992 obj->comp_events_reported = 0;
993 obj->async_events_reported = 0;
994 INIT_LIST_HEAD(&obj->comp_list);
995 INIT_LIST_HEAD(&obj->async_list);
996
997 attr.cqe = cmd->cqe;
998 attr.comp_vector = cmd->comp_vector;
999 attr.flags = cmd->flags;
1000
1001 cq = ib_dev->ops.create_cq(ib_dev, &attr, obj->uobject.context,
1002 &attrs->driver_udata);
1003 if (IS_ERR(cq)) {
1004 ret = PTR_ERR(cq);
1005 goto err_file;
1006 }
1007
1008 cq->device = ib_dev;
1009 cq->uobject = &obj->uobject;
1010 cq->comp_handler = ib_uverbs_comp_handler;
1011 cq->event_handler = ib_uverbs_cq_event_handler;
1012 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
1013 atomic_set(&cq->usecnt, 0);
1014
1015 obj->uobject.object = cq;
1016 memset(&resp, 0, sizeof resp);
1017 resp.base.cq_handle = obj->uobject.id;
1018 resp.base.cqe = cq->cqe;
1019 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1020
1021 cq->res.type = RDMA_RESTRACK_CQ;
1022 rdma_restrack_uadd(&cq->res);
1023
1024 ret = uverbs_response(attrs, &resp, sizeof(resp));
1025 if (ret)
1026 goto err_cb;
1027
1028 ret = uobj_alloc_commit(&obj->uobject);
1029 if (ret)
1030 return ERR_PTR(ret);
1031 return obj;
1032
1033 err_cb:
1034 ib_destroy_cq(cq);
1035
1036 err_file:
1037 if (ev_file)
1038 ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
1039
1040 err:
1041 uobj_alloc_abort(&obj->uobject);
1042
1043 return ERR_PTR(ret);
1044 }
1045
1046 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1047 {
1048 struct ib_uverbs_create_cq cmd;
1049 struct ib_uverbs_ex_create_cq cmd_ex;
1050 struct ib_ucq_object *obj;
1051 int ret;
1052
1053 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1054 if (ret)
1055 return ret;
1056
1057 memset(&cmd_ex, 0, sizeof(cmd_ex));
1058 cmd_ex.user_handle = cmd.user_handle;
1059 cmd_ex.cqe = cmd.cqe;
1060 cmd_ex.comp_vector = cmd.comp_vector;
1061 cmd_ex.comp_channel = cmd.comp_channel;
1062
1063 obj = create_cq(attrs, &cmd_ex);
1064 return PTR_ERR_OR_ZERO(obj);
1065 }
1066
1067 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1068 {
1069 struct ib_uverbs_ex_create_cq cmd;
1070 struct ib_ucq_object *obj;
1071 int ret;
1072
1073 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1074 if (ret)
1075 return ret;
1076
1077 if (cmd.comp_mask)
1078 return -EINVAL;
1079
1080 if (cmd.reserved)
1081 return -EINVAL;
1082
1083 obj = create_cq(attrs, &cmd);
1084 return PTR_ERR_OR_ZERO(obj);
1085 }
1086
1087 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1088 {
1089 struct ib_uverbs_resize_cq cmd;
1090 struct ib_uverbs_resize_cq_resp resp = {};
1091 struct ib_cq *cq;
1092 int ret = -EINVAL;
1093
1094 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1095 if (ret)
1096 return ret;
1097
1098 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1099 if (!cq)
1100 return -EINVAL;
1101
1102 ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1103 if (ret)
1104 goto out;
1105
1106 resp.cqe = cq->cqe;
1107
1108 ret = uverbs_response(attrs, &resp, sizeof(resp));
1109 out:
1110 uobj_put_obj_read(cq);
1111
1112 return ret;
1113 }
1114
1115 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1116 struct ib_wc *wc)
1117 {
1118 struct ib_uverbs_wc tmp;
1119
1120 tmp.wr_id = wc->wr_id;
1121 tmp.status = wc->status;
1122 tmp.opcode = wc->opcode;
1123 tmp.vendor_err = wc->vendor_err;
1124 tmp.byte_len = wc->byte_len;
1125 tmp.ex.imm_data = wc->ex.imm_data;
1126 tmp.qp_num = wc->qp->qp_num;
1127 tmp.src_qp = wc->src_qp;
1128 tmp.wc_flags = wc->wc_flags;
1129 tmp.pkey_index = wc->pkey_index;
1130 if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1131 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid);
1132 else
1133 tmp.slid = ib_lid_cpu16(wc->slid);
1134 tmp.sl = wc->sl;
1135 tmp.dlid_path_bits = wc->dlid_path_bits;
1136 tmp.port_num = wc->port_num;
1137 tmp.reserved = 0;
1138
1139 if (copy_to_user(dest, &tmp, sizeof tmp))
1140 return -EFAULT;
1141
1142 return 0;
1143 }
1144
1145 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1146 {
1147 struct ib_uverbs_poll_cq cmd;
1148 struct ib_uverbs_poll_cq_resp resp;
1149 u8 __user *header_ptr;
1150 u8 __user *data_ptr;
1151 struct ib_cq *cq;
1152 struct ib_wc wc;
1153 int ret;
1154
1155 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1156 if (ret)
1157 return ret;
1158
1159 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1160 if (!cq)
1161 return -EINVAL;
1162
1163 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1164 header_ptr = attrs->ucore.outbuf;
1165 data_ptr = header_ptr + sizeof resp;
1166
1167 memset(&resp, 0, sizeof resp);
1168 while (resp.count < cmd.ne) {
1169 ret = ib_poll_cq(cq, 1, &wc);
1170 if (ret < 0)
1171 goto out_put;
1172 if (!ret)
1173 break;
1174
1175 ret = copy_wc_to_user(cq->device, data_ptr, &wc);
1176 if (ret)
1177 goto out_put;
1178
1179 data_ptr += sizeof(struct ib_uverbs_wc);
1180 ++resp.count;
1181 }
1182
1183 if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1184 ret = -EFAULT;
1185 goto out_put;
1186 }
1187
1188 if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
1189 ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1190
1191 ret = 0;
1192
1193 out_put:
1194 uobj_put_obj_read(cq);
1195 return ret;
1196 }
1197
1198 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1199 {
1200 struct ib_uverbs_req_notify_cq cmd;
1201 struct ib_cq *cq;
1202 int ret;
1203
1204 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1205 if (ret)
1206 return ret;
1207
1208 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1209 if (!cq)
1210 return -EINVAL;
1211
1212 ib_req_notify_cq(cq, cmd.solicited_only ?
1213 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1214
1215 uobj_put_obj_read(cq);
1216
1217 return 0;
1218 }
1219
1220 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1221 {
1222 struct ib_uverbs_destroy_cq cmd;
1223 struct ib_uverbs_destroy_cq_resp resp;
1224 struct ib_uobject *uobj;
1225 struct ib_ucq_object *obj;
1226 int ret;
1227
1228 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1229 if (ret)
1230 return ret;
1231
1232 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1233 if (IS_ERR(uobj))
1234 return PTR_ERR(uobj);
1235
1236 obj = container_of(uobj, struct ib_ucq_object, uobject);
1237 memset(&resp, 0, sizeof(resp));
1238 resp.comp_events_reported = obj->comp_events_reported;
1239 resp.async_events_reported = obj->async_events_reported;
1240
1241 uobj_put_destroy(uobj);
1242
1243 return uverbs_response(attrs, &resp, sizeof(resp));
1244 }
1245
1246 static int create_qp(struct uverbs_attr_bundle *attrs,
1247 struct ib_uverbs_ex_create_qp *cmd)
1248 {
1249 struct ib_uqp_object *obj;
1250 struct ib_device *device;
1251 struct ib_pd *pd = NULL;
1252 struct ib_xrcd *xrcd = NULL;
1253 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT);
1254 struct ib_cq *scq = NULL, *rcq = NULL;
1255 struct ib_srq *srq = NULL;
1256 struct ib_qp *qp;
1257 struct ib_qp_init_attr attr = {};
1258 struct ib_uverbs_ex_create_qp_resp resp;
1259 int ret;
1260 struct ib_rwq_ind_table *ind_tbl = NULL;
1261 bool has_sq = true;
1262 struct ib_device *ib_dev;
1263
1264 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1265 return -EPERM;
1266
1267 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1268 &ib_dev);
1269 if (IS_ERR(obj))
1270 return PTR_ERR(obj);
1271 obj->uxrcd = NULL;
1272 obj->uevent.uobject.user_handle = cmd->user_handle;
1273 mutex_init(&obj->mcast_lock);
1274
1275 if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1276 ind_tbl = uobj_get_obj_read(rwq_ind_table,
1277 UVERBS_OBJECT_RWQ_IND_TBL,
1278 cmd->rwq_ind_tbl_handle, attrs);
1279 if (!ind_tbl) {
1280 ret = -EINVAL;
1281 goto err_put;
1282 }
1283
1284 attr.rwq_ind_tbl = ind_tbl;
1285 }
1286
1287 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1288 ret = -EINVAL;
1289 goto err_put;
1290 }
1291
1292 if (ind_tbl && !cmd->max_send_wr)
1293 has_sq = false;
1294
1295 if (cmd->qp_type == IB_QPT_XRC_TGT) {
1296 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1297 attrs);
1298
1299 if (IS_ERR(xrcd_uobj)) {
1300 ret = -EINVAL;
1301 goto err_put;
1302 }
1303
1304 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1305 if (!xrcd) {
1306 ret = -EINVAL;
1307 goto err_put;
1308 }
1309 device = xrcd->device;
1310 } else {
1311 if (cmd->qp_type == IB_QPT_XRC_INI) {
1312 cmd->max_recv_wr = 0;
1313 cmd->max_recv_sge = 0;
1314 } else {
1315 if (cmd->is_srq) {
1316 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1317 cmd->srq_handle, attrs);
1318 if (!srq || srq->srq_type == IB_SRQT_XRC) {
1319 ret = -EINVAL;
1320 goto err_put;
1321 }
1322 }
1323
1324 if (!ind_tbl) {
1325 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1326 rcq = uobj_get_obj_read(
1327 cq, UVERBS_OBJECT_CQ,
1328 cmd->recv_cq_handle, attrs);
1329 if (!rcq) {
1330 ret = -EINVAL;
1331 goto err_put;
1332 }
1333 }
1334 }
1335 }
1336
1337 if (has_sq)
1338 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1339 cmd->send_cq_handle, attrs);
1340 if (!ind_tbl)
1341 rcq = rcq ?: scq;
1342 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1343 attrs);
1344 if (!pd || (!scq && has_sq)) {
1345 ret = -EINVAL;
1346 goto err_put;
1347 }
1348
1349 device = pd->device;
1350 }
1351
1352 attr.event_handler = ib_uverbs_qp_event_handler;
1353 attr.qp_context = attrs->ufile;
1354 attr.send_cq = scq;
1355 attr.recv_cq = rcq;
1356 attr.srq = srq;
1357 attr.xrcd = xrcd;
1358 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1359 IB_SIGNAL_REQ_WR;
1360 attr.qp_type = cmd->qp_type;
1361 attr.create_flags = 0;
1362
1363 attr.cap.max_send_wr = cmd->max_send_wr;
1364 attr.cap.max_recv_wr = cmd->max_recv_wr;
1365 attr.cap.max_send_sge = cmd->max_send_sge;
1366 attr.cap.max_recv_sge = cmd->max_recv_sge;
1367 attr.cap.max_inline_data = cmd->max_inline_data;
1368
1369 obj->uevent.events_reported = 0;
1370 INIT_LIST_HEAD(&obj->uevent.event_list);
1371 INIT_LIST_HEAD(&obj->mcast_list);
1372
1373 attr.create_flags = cmd->create_flags;
1374 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1375 IB_QP_CREATE_CROSS_CHANNEL |
1376 IB_QP_CREATE_MANAGED_SEND |
1377 IB_QP_CREATE_MANAGED_RECV |
1378 IB_QP_CREATE_SCATTER_FCS |
1379 IB_QP_CREATE_CVLAN_STRIPPING |
1380 IB_QP_CREATE_SOURCE_QPN |
1381 IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1382 ret = -EINVAL;
1383 goto err_put;
1384 }
1385
1386 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1387 if (!capable(CAP_NET_RAW)) {
1388 ret = -EPERM;
1389 goto err_put;
1390 }
1391
1392 attr.source_qpn = cmd->source_qpn;
1393 }
1394
1395 if (cmd->qp_type == IB_QPT_XRC_TGT)
1396 qp = ib_create_qp(pd, &attr);
1397 else
1398 qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
1399 &obj->uevent.uobject);
1400
1401 if (IS_ERR(qp)) {
1402 ret = PTR_ERR(qp);
1403 goto err_put;
1404 }
1405
1406 if (cmd->qp_type != IB_QPT_XRC_TGT) {
1407 ret = ib_create_qp_security(qp, device);
1408 if (ret)
1409 goto err_cb;
1410
1411 qp->real_qp = qp;
1412 qp->pd = pd;
1413 qp->send_cq = attr.send_cq;
1414 qp->recv_cq = attr.recv_cq;
1415 qp->srq = attr.srq;
1416 qp->rwq_ind_tbl = ind_tbl;
1417 qp->event_handler = attr.event_handler;
1418 qp->qp_context = attr.qp_context;
1419 qp->qp_type = attr.qp_type;
1420 atomic_set(&qp->usecnt, 0);
1421 atomic_inc(&pd->usecnt);
1422 qp->port = 0;
1423 if (attr.send_cq)
1424 atomic_inc(&attr.send_cq->usecnt);
1425 if (attr.recv_cq)
1426 atomic_inc(&attr.recv_cq->usecnt);
1427 if (attr.srq)
1428 atomic_inc(&attr.srq->usecnt);
1429 if (ind_tbl)
1430 atomic_inc(&ind_tbl->usecnt);
1431 } else {
1432 /* It is done in _ib_create_qp for other QP types */
1433 qp->uobject = &obj->uevent.uobject;
1434 }
1435
1436 obj->uevent.uobject.object = qp;
1437
1438 memset(&resp, 0, sizeof resp);
1439 resp.base.qpn = qp->qp_num;
1440 resp.base.qp_handle = obj->uevent.uobject.id;
1441 resp.base.max_recv_sge = attr.cap.max_recv_sge;
1442 resp.base.max_send_sge = attr.cap.max_send_sge;
1443 resp.base.max_recv_wr = attr.cap.max_recv_wr;
1444 resp.base.max_send_wr = attr.cap.max_send_wr;
1445 resp.base.max_inline_data = attr.cap.max_inline_data;
1446 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1447
1448 ret = uverbs_response(attrs, &resp, sizeof(resp));
1449 if (ret)
1450 goto err_cb;
1451
1452 if (xrcd) {
1453 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1454 uobject);
1455 atomic_inc(&obj->uxrcd->refcnt);
1456 uobj_put_read(xrcd_uobj);
1457 }
1458
1459 if (pd)
1460 uobj_put_obj_read(pd);
1461 if (scq)
1462 uobj_put_obj_read(scq);
1463 if (rcq && rcq != scq)
1464 uobj_put_obj_read(rcq);
1465 if (srq)
1466 uobj_put_obj_read(srq);
1467 if (ind_tbl)
1468 uobj_put_obj_read(ind_tbl);
1469
1470 return uobj_alloc_commit(&obj->uevent.uobject);
1471 err_cb:
1472 ib_destroy_qp(qp);
1473
1474 err_put:
1475 if (!IS_ERR(xrcd_uobj))
1476 uobj_put_read(xrcd_uobj);
1477 if (pd)
1478 uobj_put_obj_read(pd);
1479 if (scq)
1480 uobj_put_obj_read(scq);
1481 if (rcq && rcq != scq)
1482 uobj_put_obj_read(rcq);
1483 if (srq)
1484 uobj_put_obj_read(srq);
1485 if (ind_tbl)
1486 uobj_put_obj_read(ind_tbl);
1487
1488 uobj_alloc_abort(&obj->uevent.uobject);
1489 return ret;
1490 }
1491
1492 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1493 {
1494 struct ib_uverbs_create_qp cmd;
1495 struct ib_uverbs_ex_create_qp cmd_ex;
1496 int ret;
1497
1498 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1499 if (ret)
1500 return ret;
1501
1502 memset(&cmd_ex, 0, sizeof(cmd_ex));
1503 cmd_ex.user_handle = cmd.user_handle;
1504 cmd_ex.pd_handle = cmd.pd_handle;
1505 cmd_ex.send_cq_handle = cmd.send_cq_handle;
1506 cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1507 cmd_ex.srq_handle = cmd.srq_handle;
1508 cmd_ex.max_send_wr = cmd.max_send_wr;
1509 cmd_ex.max_recv_wr = cmd.max_recv_wr;
1510 cmd_ex.max_send_sge = cmd.max_send_sge;
1511 cmd_ex.max_recv_sge = cmd.max_recv_sge;
1512 cmd_ex.max_inline_data = cmd.max_inline_data;
1513 cmd_ex.sq_sig_all = cmd.sq_sig_all;
1514 cmd_ex.qp_type = cmd.qp_type;
1515 cmd_ex.is_srq = cmd.is_srq;
1516
1517 return create_qp(attrs, &cmd_ex);
1518 }
1519
1520 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1521 {
1522 struct ib_uverbs_ex_create_qp cmd;
1523 int ret;
1524
1525 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1526 if (ret)
1527 return ret;
1528
1529 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1530 return -EINVAL;
1531
1532 if (cmd.reserved)
1533 return -EINVAL;
1534
1535 return create_qp(attrs, &cmd);
1536 }
1537
1538 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1539 {
1540 struct ib_uverbs_open_qp cmd;
1541 struct ib_uverbs_create_qp_resp resp;
1542 struct ib_uqp_object *obj;
1543 struct ib_xrcd *xrcd;
1544 struct ib_uobject *uninitialized_var(xrcd_uobj);
1545 struct ib_qp *qp;
1546 struct ib_qp_open_attr attr;
1547 int ret;
1548 struct ib_device *ib_dev;
1549
1550 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1551 if (ret)
1552 return ret;
1553
1554 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1555 &ib_dev);
1556 if (IS_ERR(obj))
1557 return PTR_ERR(obj);
1558
1559 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1560 if (IS_ERR(xrcd_uobj)) {
1561 ret = -EINVAL;
1562 goto err_put;
1563 }
1564
1565 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1566 if (!xrcd) {
1567 ret = -EINVAL;
1568 goto err_xrcd;
1569 }
1570
1571 attr.event_handler = ib_uverbs_qp_event_handler;
1572 attr.qp_context = attrs->ufile;
1573 attr.qp_num = cmd.qpn;
1574 attr.qp_type = cmd.qp_type;
1575
1576 obj->uevent.events_reported = 0;
1577 INIT_LIST_HEAD(&obj->uevent.event_list);
1578 INIT_LIST_HEAD(&obj->mcast_list);
1579
1580 qp = ib_open_qp(xrcd, &attr);
1581 if (IS_ERR(qp)) {
1582 ret = PTR_ERR(qp);
1583 goto err_xrcd;
1584 }
1585
1586 obj->uevent.uobject.object = qp;
1587 obj->uevent.uobject.user_handle = cmd.user_handle;
1588
1589 memset(&resp, 0, sizeof resp);
1590 resp.qpn = qp->qp_num;
1591 resp.qp_handle = obj->uevent.uobject.id;
1592
1593 ret = uverbs_response(attrs, &resp, sizeof(resp));
1594 if (ret)
1595 goto err_destroy;
1596
1597 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1598 atomic_inc(&obj->uxrcd->refcnt);
1599 qp->uobject = &obj->uevent.uobject;
1600 uobj_put_read(xrcd_uobj);
1601
1602 return uobj_alloc_commit(&obj->uevent.uobject);
1603
1604 err_destroy:
1605 ib_destroy_qp(qp);
1606 err_xrcd:
1607 uobj_put_read(xrcd_uobj);
1608 err_put:
1609 uobj_alloc_abort(&obj->uevent.uobject);
1610 return ret;
1611 }
1612
1613 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1614 struct rdma_ah_attr *rdma_attr)
1615 {
1616 const struct ib_global_route *grh;
1617
1618 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr);
1619 uverb_attr->sl = rdma_ah_get_sl(rdma_attr);
1620 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr);
1621 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr);
1622 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) &
1623 IB_AH_GRH);
1624 if (uverb_attr->is_global) {
1625 grh = rdma_ah_read_grh(rdma_attr);
1626 memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1627 uverb_attr->flow_label = grh->flow_label;
1628 uverb_attr->sgid_index = grh->sgid_index;
1629 uverb_attr->hop_limit = grh->hop_limit;
1630 uverb_attr->traffic_class = grh->traffic_class;
1631 }
1632 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr);
1633 }
1634
1635 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1636 {
1637 struct ib_uverbs_query_qp cmd;
1638 struct ib_uverbs_query_qp_resp resp;
1639 struct ib_qp *qp;
1640 struct ib_qp_attr *attr;
1641 struct ib_qp_init_attr *init_attr;
1642 int ret;
1643
1644 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1645 if (ret)
1646 return ret;
1647
1648 attr = kmalloc(sizeof *attr, GFP_KERNEL);
1649 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1650 if (!attr || !init_attr) {
1651 ret = -ENOMEM;
1652 goto out;
1653 }
1654
1655 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1656 if (!qp) {
1657 ret = -EINVAL;
1658 goto out;
1659 }
1660
1661 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1662
1663 uobj_put_obj_read(qp);
1664
1665 if (ret)
1666 goto out;
1667
1668 memset(&resp, 0, sizeof resp);
1669
1670 resp.qp_state = attr->qp_state;
1671 resp.cur_qp_state = attr->cur_qp_state;
1672 resp.path_mtu = attr->path_mtu;
1673 resp.path_mig_state = attr->path_mig_state;
1674 resp.qkey = attr->qkey;
1675 resp.rq_psn = attr->rq_psn;
1676 resp.sq_psn = attr->sq_psn;
1677 resp.dest_qp_num = attr->dest_qp_num;
1678 resp.qp_access_flags = attr->qp_access_flags;
1679 resp.pkey_index = attr->pkey_index;
1680 resp.alt_pkey_index = attr->alt_pkey_index;
1681 resp.sq_draining = attr->sq_draining;
1682 resp.max_rd_atomic = attr->max_rd_atomic;
1683 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic;
1684 resp.min_rnr_timer = attr->min_rnr_timer;
1685 resp.port_num = attr->port_num;
1686 resp.timeout = attr->timeout;
1687 resp.retry_cnt = attr->retry_cnt;
1688 resp.rnr_retry = attr->rnr_retry;
1689 resp.alt_port_num = attr->alt_port_num;
1690 resp.alt_timeout = attr->alt_timeout;
1691
1692 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1693 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1694
1695 resp.max_send_wr = init_attr->cap.max_send_wr;
1696 resp.max_recv_wr = init_attr->cap.max_recv_wr;
1697 resp.max_send_sge = init_attr->cap.max_send_sge;
1698 resp.max_recv_sge = init_attr->cap.max_recv_sge;
1699 resp.max_inline_data = init_attr->cap.max_inline_data;
1700 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1701
1702 ret = uverbs_response(attrs, &resp, sizeof(resp));
1703
1704 out:
1705 kfree(attr);
1706 kfree(init_attr);
1707
1708 return ret;
1709 }
1710
1711 /* Remove ignored fields set in the attribute mask */
1712 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1713 {
1714 switch (qp_type) {
1715 case IB_QPT_XRC_INI:
1716 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1717 case IB_QPT_XRC_TGT:
1718 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1719 IB_QP_RNR_RETRY);
1720 default:
1721 return mask;
1722 }
1723 }
1724
1725 static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1726 struct rdma_ah_attr *rdma_attr,
1727 struct ib_uverbs_qp_dest *uverb_attr)
1728 {
1729 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1730 if (uverb_attr->is_global) {
1731 rdma_ah_set_grh(rdma_attr, NULL,
1732 uverb_attr->flow_label,
1733 uverb_attr->sgid_index,
1734 uverb_attr->hop_limit,
1735 uverb_attr->traffic_class);
1736 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1737 } else {
1738 rdma_ah_set_ah_flags(rdma_attr, 0);
1739 }
1740 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1741 rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1742 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1743 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1744 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1745 rdma_ah_set_make_grd(rdma_attr, false);
1746 }
1747
1748 static int modify_qp(struct uverbs_attr_bundle *attrs,
1749 struct ib_uverbs_ex_modify_qp *cmd)
1750 {
1751 struct ib_qp_attr *attr;
1752 struct ib_qp *qp;
1753 int ret;
1754
1755 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1756 if (!attr)
1757 return -ENOMEM;
1758
1759 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1760 attrs);
1761 if (!qp) {
1762 ret = -EINVAL;
1763 goto out;
1764 }
1765
1766 if ((cmd->base.attr_mask & IB_QP_PORT) &&
1767 !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1768 ret = -EINVAL;
1769 goto release_qp;
1770 }
1771
1772 if ((cmd->base.attr_mask & IB_QP_AV)) {
1773 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1774 ret = -EINVAL;
1775 goto release_qp;
1776 }
1777
1778 if (cmd->base.attr_mask & IB_QP_STATE &&
1779 cmd->base.qp_state == IB_QPS_RTR) {
1780 /* We are in INIT->RTR TRANSITION (if we are not,
1781 * this transition will be rejected in subsequent checks).
1782 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1783 * but the IB_QP_STATE flag is required.
1784 *
1785 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1786 * when IB_QP_AV is set, has required inclusion of a valid
1787 * port number in the primary AV. (AVs are created and handled
1788 * differently for infiniband and ethernet (RoCE) ports).
1789 *
1790 * Check the port number included in the primary AV against
1791 * the port number in the qp struct, which was set (and saved)
1792 * in the RST->INIT transition.
1793 */
1794 if (cmd->base.dest.port_num != qp->real_qp->port) {
1795 ret = -EINVAL;
1796 goto release_qp;
1797 }
1798 } else {
1799 /* We are in SQD->SQD. (If we are not, this transition will
1800 * be rejected later in the verbs layer checks).
1801 * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1802 * together in the SQD->SQD transition.
1803 *
1804 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1805 * verbs layer driver does not track primary port changes
1806 * resulting from path migration. Thus, in SQD, if the primary
1807 * AV is modified, the primary port should also be modified).
1808 *
1809 * Note that in this transition, the IB_QP_STATE flag
1810 * is not allowed.
1811 */
1812 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1813 == (IB_QP_AV | IB_QP_PORT)) &&
1814 cmd->base.port_num != cmd->base.dest.port_num) {
1815 ret = -EINVAL;
1816 goto release_qp;
1817 }
1818 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1819 == IB_QP_AV) {
1820 cmd->base.attr_mask |= IB_QP_PORT;
1821 cmd->base.port_num = cmd->base.dest.port_num;
1822 }
1823 }
1824 }
1825
1826 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1827 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
1828 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
1829 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
1830 ret = -EINVAL;
1831 goto release_qp;
1832 }
1833
1834 if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1835 cmd->base.cur_qp_state > IB_QPS_ERR) ||
1836 (cmd->base.attr_mask & IB_QP_STATE &&
1837 cmd->base.qp_state > IB_QPS_ERR)) {
1838 ret = -EINVAL;
1839 goto release_qp;
1840 }
1841
1842 if (cmd->base.attr_mask & IB_QP_STATE)
1843 attr->qp_state = cmd->base.qp_state;
1844 if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1845 attr->cur_qp_state = cmd->base.cur_qp_state;
1846 if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1847 attr->path_mtu = cmd->base.path_mtu;
1848 if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1849 attr->path_mig_state = cmd->base.path_mig_state;
1850 if (cmd->base.attr_mask & IB_QP_QKEY)
1851 attr->qkey = cmd->base.qkey;
1852 if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1853 attr->rq_psn = cmd->base.rq_psn;
1854 if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1855 attr->sq_psn = cmd->base.sq_psn;
1856 if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1857 attr->dest_qp_num = cmd->base.dest_qp_num;
1858 if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1859 attr->qp_access_flags = cmd->base.qp_access_flags;
1860 if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1861 attr->pkey_index = cmd->base.pkey_index;
1862 if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1863 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1864 if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1865 attr->max_rd_atomic = cmd->base.max_rd_atomic;
1866 if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1867 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1868 if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1869 attr->min_rnr_timer = cmd->base.min_rnr_timer;
1870 if (cmd->base.attr_mask & IB_QP_PORT)
1871 attr->port_num = cmd->base.port_num;
1872 if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1873 attr->timeout = cmd->base.timeout;
1874 if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1875 attr->retry_cnt = cmd->base.retry_cnt;
1876 if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1877 attr->rnr_retry = cmd->base.rnr_retry;
1878 if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1879 attr->alt_port_num = cmd->base.alt_port_num;
1880 attr->alt_timeout = cmd->base.alt_timeout;
1881 attr->alt_pkey_index = cmd->base.alt_pkey_index;
1882 }
1883 if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1884 attr->rate_limit = cmd->rate_limit;
1885
1886 if (cmd->base.attr_mask & IB_QP_AV)
1887 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
1888 &cmd->base.dest);
1889
1890 if (cmd->base.attr_mask & IB_QP_ALT_PATH)
1891 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
1892 &cmd->base.alt_dest);
1893
1894 ret = ib_modify_qp_with_udata(qp, attr,
1895 modify_qp_mask(qp->qp_type,
1896 cmd->base.attr_mask),
1897 &attrs->driver_udata);
1898
1899 release_qp:
1900 uobj_put_obj_read(qp);
1901 out:
1902 kfree(attr);
1903
1904 return ret;
1905 }
1906
1907 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
1908 {
1909 struct ib_uverbs_ex_modify_qp cmd;
1910 int ret;
1911
1912 ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
1913 if (ret)
1914 return ret;
1915
1916 if (cmd.base.attr_mask &
1917 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
1918 return -EOPNOTSUPP;
1919
1920 return modify_qp(attrs, &cmd);
1921 }
1922
1923 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
1924 {
1925 struct ib_uverbs_ex_modify_qp cmd;
1926 struct ib_uverbs_ex_modify_qp_resp resp = {
1927 .response_length = uverbs_response_length(attrs, sizeof(resp))
1928 };
1929 int ret;
1930
1931 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1932 if (ret)
1933 return ret;
1934
1935 /*
1936 * Last bit is reserved for extending the attr_mask by
1937 * using another field.
1938 */
1939 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
1940
1941 if (cmd.base.attr_mask &
1942 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
1943 return -EOPNOTSUPP;
1944
1945 ret = modify_qp(attrs, &cmd);
1946 if (ret)
1947 return ret;
1948
1949 return uverbs_response(attrs, &resp, sizeof(resp));
1950 }
1951
1952 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
1953 {
1954 struct ib_uverbs_destroy_qp cmd;
1955 struct ib_uverbs_destroy_qp_resp resp;
1956 struct ib_uobject *uobj;
1957 struct ib_uqp_object *obj;
1958 int ret;
1959
1960 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1961 if (ret)
1962 return ret;
1963
1964 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1965 if (IS_ERR(uobj))
1966 return PTR_ERR(uobj);
1967
1968 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
1969 memset(&resp, 0, sizeof(resp));
1970 resp.events_reported = obj->uevent.events_reported;
1971
1972 uobj_put_destroy(uobj);
1973
1974 return uverbs_response(attrs, &resp, sizeof(resp));
1975 }
1976
1977 static void *alloc_wr(size_t wr_size, __u32 num_sge)
1978 {
1979 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
1980 sizeof (struct ib_sge))
1981 return NULL;
1982
1983 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
1984 num_sge * sizeof (struct ib_sge), GFP_KERNEL);
1985 }
1986
1987 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
1988 {
1989 struct ib_uverbs_post_send cmd;
1990 struct ib_uverbs_post_send_resp resp;
1991 struct ib_uverbs_send_wr *user_wr;
1992 struct ib_send_wr *wr = NULL, *last, *next;
1993 const struct ib_send_wr *bad_wr;
1994 struct ib_qp *qp;
1995 int i, sg_ind;
1996 int is_ud;
1997 int ret, ret2;
1998 size_t next_size;
1999 const struct ib_sge __user *sgls;
2000 const void __user *wqes;
2001 struct uverbs_req_iter iter;
2002
2003 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2004 if (ret)
2005 return ret;
2006 wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
2007 if (IS_ERR(wqes))
2008 return PTR_ERR(wqes);
2009 sgls = uverbs_request_next_ptr(
2010 &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
2011 if (IS_ERR(sgls))
2012 return PTR_ERR(sgls);
2013 ret = uverbs_request_finish(&iter);
2014 if (ret)
2015 return ret;
2016
2017 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2018 if (!user_wr)
2019 return -ENOMEM;
2020
2021 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2022 if (!qp) {
2023 ret = -EINVAL;
2024 goto out;
2025 }
2026
2027 is_ud = qp->qp_type == IB_QPT_UD;
2028 sg_ind = 0;
2029 last = NULL;
2030 for (i = 0; i < cmd.wr_count; ++i) {
2031 if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2032 cmd.wqe_size)) {
2033 ret = -EFAULT;
2034 goto out_put;
2035 }
2036
2037 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2038 ret = -EINVAL;
2039 goto out_put;
2040 }
2041
2042 if (is_ud) {
2043 struct ib_ud_wr *ud;
2044
2045 if (user_wr->opcode != IB_WR_SEND &&
2046 user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2047 ret = -EINVAL;
2048 goto out_put;
2049 }
2050
2051 next_size = sizeof(*ud);
2052 ud = alloc_wr(next_size, user_wr->num_sge);
2053 if (!ud) {
2054 ret = -ENOMEM;
2055 goto out_put;
2056 }
2057
2058 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2059 user_wr->wr.ud.ah, attrs);
2060 if (!ud->ah) {
2061 kfree(ud);
2062 ret = -EINVAL;
2063 goto out_put;
2064 }
2065 ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2066 ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2067
2068 next = &ud->wr;
2069 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2070 user_wr->opcode == IB_WR_RDMA_WRITE ||
2071 user_wr->opcode == IB_WR_RDMA_READ) {
2072 struct ib_rdma_wr *rdma;
2073
2074 next_size = sizeof(*rdma);
2075 rdma = alloc_wr(next_size, user_wr->num_sge);
2076 if (!rdma) {
2077 ret = -ENOMEM;
2078 goto out_put;
2079 }
2080
2081 rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2082 rdma->rkey = user_wr->wr.rdma.rkey;
2083
2084 next = &rdma->wr;
2085 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2086 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2087 struct ib_atomic_wr *atomic;
2088
2089 next_size = sizeof(*atomic);
2090 atomic = alloc_wr(next_size, user_wr->num_sge);
2091 if (!atomic) {
2092 ret = -ENOMEM;
2093 goto out_put;
2094 }
2095
2096 atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2097 atomic->compare_add = user_wr->wr.atomic.compare_add;
2098 atomic->swap = user_wr->wr.atomic.swap;
2099 atomic->rkey = user_wr->wr.atomic.rkey;
2100
2101 next = &atomic->wr;
2102 } else if (user_wr->opcode == IB_WR_SEND ||
2103 user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2104 user_wr->opcode == IB_WR_SEND_WITH_INV) {
2105 next_size = sizeof(*next);
2106 next = alloc_wr(next_size, user_wr->num_sge);
2107 if (!next) {
2108 ret = -ENOMEM;
2109 goto out_put;
2110 }
2111 } else {
2112 ret = -EINVAL;
2113 goto out_put;
2114 }
2115
2116 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2117 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2118 next->ex.imm_data =
2119 (__be32 __force) user_wr->ex.imm_data;
2120 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2121 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2122 }
2123
2124 if (!last)
2125 wr = next;
2126 else
2127 last->next = next;
2128 last = next;
2129
2130 next->next = NULL;
2131 next->wr_id = user_wr->wr_id;
2132 next->num_sge = user_wr->num_sge;
2133 next->opcode = user_wr->opcode;
2134 next->send_flags = user_wr->send_flags;
2135
2136 if (next->num_sge) {
2137 next->sg_list = (void *) next +
2138 ALIGN(next_size, sizeof(struct ib_sge));
2139 if (copy_from_user(next->sg_list, sgls + sg_ind,
2140 next->num_sge *
2141 sizeof(struct ib_sge))) {
2142 ret = -EFAULT;
2143 goto out_put;
2144 }
2145 sg_ind += next->num_sge;
2146 } else
2147 next->sg_list = NULL;
2148 }
2149
2150 resp.bad_wr = 0;
2151 ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2152 if (ret)
2153 for (next = wr; next; next = next->next) {
2154 ++resp.bad_wr;
2155 if (next == bad_wr)
2156 break;
2157 }
2158
2159 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2160 if (ret2)
2161 ret = ret2;
2162
2163 out_put:
2164 uobj_put_obj_read(qp);
2165
2166 while (wr) {
2167 if (is_ud && ud_wr(wr)->ah)
2168 uobj_put_obj_read(ud_wr(wr)->ah);
2169 next = wr->next;
2170 kfree(wr);
2171 wr = next;
2172 }
2173
2174 out:
2175 kfree(user_wr);
2176
2177 return ret;
2178 }
2179
2180 static struct ib_recv_wr *
2181 ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2182 u32 wqe_size, u32 sge_count)
2183 {
2184 struct ib_uverbs_recv_wr *user_wr;
2185 struct ib_recv_wr *wr = NULL, *last, *next;
2186 int sg_ind;
2187 int i;
2188 int ret;
2189 const struct ib_sge __user *sgls;
2190 const void __user *wqes;
2191
2192 if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2193 return ERR_PTR(-EINVAL);
2194
2195 wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
2196 if (IS_ERR(wqes))
2197 return ERR_CAST(wqes);
2198 sgls = uverbs_request_next_ptr(
2199 iter, sge_count * sizeof(struct ib_uverbs_sge));
2200 if (IS_ERR(sgls))
2201 return ERR_CAST(sgls);
2202 ret = uverbs_request_finish(iter);
2203 if (ret)
2204 return ERR_PTR(ret);
2205
2206 user_wr = kmalloc(wqe_size, GFP_KERNEL);
2207 if (!user_wr)
2208 return ERR_PTR(-ENOMEM);
2209
2210 sg_ind = 0;
2211 last = NULL;
2212 for (i = 0; i < wr_count; ++i) {
2213 if (copy_from_user(user_wr, wqes + i * wqe_size,
2214 wqe_size)) {
2215 ret = -EFAULT;
2216 goto err;
2217 }
2218
2219 if (user_wr->num_sge + sg_ind > sge_count) {
2220 ret = -EINVAL;
2221 goto err;
2222 }
2223
2224 if (user_wr->num_sge >=
2225 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2226 sizeof (struct ib_sge)) {
2227 ret = -EINVAL;
2228 goto err;
2229 }
2230
2231 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2232 user_wr->num_sge * sizeof (struct ib_sge),
2233 GFP_KERNEL);
2234 if (!next) {
2235 ret = -ENOMEM;
2236 goto err;
2237 }
2238
2239 if (!last)
2240 wr = next;
2241 else
2242 last->next = next;
2243 last = next;
2244
2245 next->next = NULL;
2246 next->wr_id = user_wr->wr_id;
2247 next->num_sge = user_wr->num_sge;
2248
2249 if (next->num_sge) {
2250 next->sg_list = (void *) next +
2251 ALIGN(sizeof *next, sizeof (struct ib_sge));
2252 if (copy_from_user(next->sg_list, sgls + sg_ind,
2253 next->num_sge *
2254 sizeof(struct ib_sge))) {
2255 ret = -EFAULT;
2256 goto err;
2257 }
2258 sg_ind += next->num_sge;
2259 } else
2260 next->sg_list = NULL;
2261 }
2262
2263 kfree(user_wr);
2264 return wr;
2265
2266 err:
2267 kfree(user_wr);
2268
2269 while (wr) {
2270 next = wr->next;
2271 kfree(wr);
2272 wr = next;
2273 }
2274
2275 return ERR_PTR(ret);
2276 }
2277
2278 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2279 {
2280 struct ib_uverbs_post_recv cmd;
2281 struct ib_uverbs_post_recv_resp resp;
2282 struct ib_recv_wr *wr, *next;
2283 const struct ib_recv_wr *bad_wr;
2284 struct ib_qp *qp;
2285 int ret, ret2;
2286 struct uverbs_req_iter iter;
2287
2288 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2289 if (ret)
2290 return ret;
2291
2292 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2293 cmd.sge_count);
2294 if (IS_ERR(wr))
2295 return PTR_ERR(wr);
2296
2297 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2298 if (!qp) {
2299 ret = -EINVAL;
2300 goto out;
2301 }
2302
2303 resp.bad_wr = 0;
2304 ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2305
2306 uobj_put_obj_read(qp);
2307 if (ret) {
2308 for (next = wr; next; next = next->next) {
2309 ++resp.bad_wr;
2310 if (next == bad_wr)
2311 break;
2312 }
2313 }
2314
2315 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2316 if (ret2)
2317 ret = ret2;
2318 out:
2319 while (wr) {
2320 next = wr->next;
2321 kfree(wr);
2322 wr = next;
2323 }
2324
2325 return ret;
2326 }
2327
2328 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2329 {
2330 struct ib_uverbs_post_srq_recv cmd;
2331 struct ib_uverbs_post_srq_recv_resp resp;
2332 struct ib_recv_wr *wr, *next;
2333 const struct ib_recv_wr *bad_wr;
2334 struct ib_srq *srq;
2335 int ret, ret2;
2336 struct uverbs_req_iter iter;
2337
2338 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2339 if (ret)
2340 return ret;
2341
2342 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2343 cmd.sge_count);
2344 if (IS_ERR(wr))
2345 return PTR_ERR(wr);
2346
2347 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
2348 if (!srq) {
2349 ret = -EINVAL;
2350 goto out;
2351 }
2352
2353 resp.bad_wr = 0;
2354 ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2355
2356 uobj_put_obj_read(srq);
2357
2358 if (ret)
2359 for (next = wr; next; next = next->next) {
2360 ++resp.bad_wr;
2361 if (next == bad_wr)
2362 break;
2363 }
2364
2365 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2366 if (ret2)
2367 ret = ret2;
2368
2369 out:
2370 while (wr) {
2371 next = wr->next;
2372 kfree(wr);
2373 wr = next;
2374 }
2375
2376 return ret;
2377 }
2378
2379 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2380 {
2381 struct ib_uverbs_create_ah cmd;
2382 struct ib_uverbs_create_ah_resp resp;
2383 struct ib_uobject *uobj;
2384 struct ib_pd *pd;
2385 struct ib_ah *ah;
2386 struct rdma_ah_attr attr = {};
2387 int ret;
2388 struct ib_device *ib_dev;
2389
2390 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2391 if (ret)
2392 return ret;
2393
2394 uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2395 if (IS_ERR(uobj))
2396 return PTR_ERR(uobj);
2397
2398 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {
2399 ret = -EINVAL;
2400 goto err;
2401 }
2402
2403 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2404 if (!pd) {
2405 ret = -EINVAL;
2406 goto err;
2407 }
2408
2409 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2410 rdma_ah_set_make_grd(&attr, false);
2411 rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2412 rdma_ah_set_sl(&attr, cmd.attr.sl);
2413 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2414 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2415 rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2416
2417 if (cmd.attr.is_global) {
2418 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2419 cmd.attr.grh.sgid_index,
2420 cmd.attr.grh.hop_limit,
2421 cmd.attr.grh.traffic_class);
2422 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2423 } else {
2424 rdma_ah_set_ah_flags(&attr, 0);
2425 }
2426
2427 ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2428 if (IS_ERR(ah)) {
2429 ret = PTR_ERR(ah);
2430 goto err_put;
2431 }
2432
2433 ah->uobject = uobj;
2434 uobj->user_handle = cmd.user_handle;
2435 uobj->object = ah;
2436
2437 resp.ah_handle = uobj->id;
2438
2439 ret = uverbs_response(attrs, &resp, sizeof(resp));
2440 if (ret)
2441 goto err_copy;
2442
2443 uobj_put_obj_read(pd);
2444 return uobj_alloc_commit(uobj);
2445
2446 err_copy:
2447 rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
2448
2449 err_put:
2450 uobj_put_obj_read(pd);
2451
2452 err:
2453 uobj_alloc_abort(uobj);
2454 return ret;
2455 }
2456
2457 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2458 {
2459 struct ib_uverbs_destroy_ah cmd;
2460 int ret;
2461
2462 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2463 if (ret)
2464 return ret;
2465
2466 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2467 }
2468
2469 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2470 {
2471 struct ib_uverbs_attach_mcast cmd;
2472 struct ib_qp *qp;
2473 struct ib_uqp_object *obj;
2474 struct ib_uverbs_mcast_entry *mcast;
2475 int ret;
2476
2477 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2478 if (ret)
2479 return ret;
2480
2481 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2482 if (!qp)
2483 return -EINVAL;
2484
2485 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2486
2487 mutex_lock(&obj->mcast_lock);
2488 list_for_each_entry(mcast, &obj->mcast_list, list)
2489 if (cmd.mlid == mcast->lid &&
2490 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2491 ret = 0;
2492 goto out_put;
2493 }
2494
2495 mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2496 if (!mcast) {
2497 ret = -ENOMEM;
2498 goto out_put;
2499 }
2500
2501 mcast->lid = cmd.mlid;
2502 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2503
2504 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2505 if (!ret)
2506 list_add_tail(&mcast->list, &obj->mcast_list);
2507 else
2508 kfree(mcast);
2509
2510 out_put:
2511 mutex_unlock(&obj->mcast_lock);
2512 uobj_put_obj_read(qp);
2513
2514 return ret;
2515 }
2516
2517 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2518 {
2519 struct ib_uverbs_detach_mcast cmd;
2520 struct ib_uqp_object *obj;
2521 struct ib_qp *qp;
2522 struct ib_uverbs_mcast_entry *mcast;
2523 int ret = -EINVAL;
2524 bool found = false;
2525
2526 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2527 if (ret)
2528 return ret;
2529
2530 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2531 if (!qp)
2532 return -EINVAL;
2533
2534 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2535 mutex_lock(&obj->mcast_lock);
2536
2537 list_for_each_entry(mcast, &obj->mcast_list, list)
2538 if (cmd.mlid == mcast->lid &&
2539 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2540 list_del(&mcast->list);
2541 kfree(mcast);
2542 found = true;
2543 break;
2544 }
2545
2546 if (!found) {
2547 ret = -EINVAL;
2548 goto out_put;
2549 }
2550
2551 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2552
2553 out_put:
2554 mutex_unlock(&obj->mcast_lock);
2555 uobj_put_obj_read(qp);
2556 return ret;
2557 }
2558
2559 struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2560 {
2561 struct ib_uflow_resources *resources;
2562
2563 resources = kzalloc(sizeof(*resources), GFP_KERNEL);
2564
2565 if (!resources)
2566 return NULL;
2567
2568 if (!num_specs)
2569 goto out;
2570
2571 resources->counters =
2572 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);
2573 resources->collection =
2574 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);
2575
2576 if (!resources->counters || !resources->collection)
2577 goto err;
2578
2579 out:
2580 resources->max = num_specs;
2581 return resources;
2582
2583 err:
2584 kfree(resources->counters);
2585 kfree(resources);
2586
2587 return NULL;
2588 }
2589 EXPORT_SYMBOL(flow_resources_alloc);
2590
2591 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2592 {
2593 unsigned int i;
2594
2595 if (!uflow_res)
2596 return;
2597
2598 for (i = 0; i < uflow_res->collection_num; i++)
2599 atomic_dec(&uflow_res->collection[i]->usecnt);
2600
2601 for (i = 0; i < uflow_res->counters_num; i++)
2602 atomic_dec(&uflow_res->counters[i]->usecnt);
2603
2604 kfree(uflow_res->collection);
2605 kfree(uflow_res->counters);
2606 kfree(uflow_res);
2607 }
2608 EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2609
2610 void flow_resources_add(struct ib_uflow_resources *uflow_res,
2611 enum ib_flow_spec_type type,
2612 void *ibobj)
2613 {
2614 WARN_ON(uflow_res->num >= uflow_res->max);
2615
2616 switch (type) {
2617 case IB_FLOW_SPEC_ACTION_HANDLE:
2618 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
2619 uflow_res->collection[uflow_res->collection_num++] =
2620 (struct ib_flow_action *)ibobj;
2621 break;
2622 case IB_FLOW_SPEC_ACTION_COUNT:
2623 atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
2624 uflow_res->counters[uflow_res->counters_num++] =
2625 (struct ib_counters *)ibobj;
2626 break;
2627 default:
2628 WARN_ON(1);
2629 }
2630
2631 uflow_res->num++;
2632 }
2633 EXPORT_SYMBOL(flow_resources_add);
2634
2635 static int kern_spec_to_ib_spec_action(const struct uverbs_attr_bundle *attrs,
2636 struct ib_uverbs_flow_spec *kern_spec,
2637 union ib_flow_spec *ib_spec,
2638 struct ib_uflow_resources *uflow_res)
2639 {
2640 ib_spec->type = kern_spec->type;
2641 switch (ib_spec->type) {
2642 case IB_FLOW_SPEC_ACTION_TAG:
2643 if (kern_spec->flow_tag.size !=
2644 sizeof(struct ib_uverbs_flow_spec_action_tag))
2645 return -EINVAL;
2646
2647 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2648 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2649 break;
2650 case IB_FLOW_SPEC_ACTION_DROP:
2651 if (kern_spec->drop.size !=
2652 sizeof(struct ib_uverbs_flow_spec_action_drop))
2653 return -EINVAL;
2654
2655 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2656 break;
2657 case IB_FLOW_SPEC_ACTION_HANDLE:
2658 if (kern_spec->action.size !=
2659 sizeof(struct ib_uverbs_flow_spec_action_handle))
2660 return -EOPNOTSUPP;
2661 ib_spec->action.act = uobj_get_obj_read(flow_action,
2662 UVERBS_OBJECT_FLOW_ACTION,
2663 kern_spec->action.handle,
2664 attrs);
2665 if (!ib_spec->action.act)
2666 return -EINVAL;
2667 ib_spec->action.size =
2668 sizeof(struct ib_flow_spec_action_handle);
2669 flow_resources_add(uflow_res,
2670 IB_FLOW_SPEC_ACTION_HANDLE,
2671 ib_spec->action.act);
2672 uobj_put_obj_read(ib_spec->action.act);
2673 break;
2674 case IB_FLOW_SPEC_ACTION_COUNT:
2675 if (kern_spec->flow_count.size !=
2676 sizeof(struct ib_uverbs_flow_spec_action_count))
2677 return -EINVAL;
2678 ib_spec->flow_count.counters =
2679 uobj_get_obj_read(counters,
2680 UVERBS_OBJECT_COUNTERS,
2681 kern_spec->flow_count.handle,
2682 attrs);
2683 if (!ib_spec->flow_count.counters)
2684 return -EINVAL;
2685 ib_spec->flow_count.size =
2686 sizeof(struct ib_flow_spec_action_count);
2687 flow_resources_add(uflow_res,
2688 IB_FLOW_SPEC_ACTION_COUNT,
2689 ib_spec->flow_count.counters);
2690 uobj_put_obj_read(ib_spec->flow_count.counters);
2691 break;
2692 default:
2693 return -EINVAL;
2694 }
2695 return 0;
2696 }
2697
2698 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec)
2699 {
2700 /* Returns user space filter size, includes padding */
2701 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2702 }
2703
2704 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
2705 u16 ib_real_filter_sz)
2706 {
2707 /*
2708 * User space filter structures must be 64 bit aligned, otherwise this
2709 * may pass, but we won't handle additional new attributes.
2710 */
2711
2712 if (kern_filter_size > ib_real_filter_sz) {
2713 if (memchr_inv(kern_spec_filter +
2714 ib_real_filter_sz, 0,
2715 kern_filter_size - ib_real_filter_sz))
2716 return -EINVAL;
2717 return ib_real_filter_sz;
2718 }
2719 return kern_filter_size;
2720 }
2721
2722 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2723 const void *kern_spec_mask,
2724 const void *kern_spec_val,
2725 size_t kern_filter_sz,
2726 union ib_flow_spec *ib_spec)
2727 {
2728 ssize_t actual_filter_sz;
2729 ssize_t ib_filter_sz;
2730
2731 /* User flow spec size must be aligned to 4 bytes */
2732 if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2733 return -EINVAL;
2734
2735 ib_spec->type = type;
2736
2737 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2738 return -EINVAL;
2739
2740 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2741 case IB_FLOW_SPEC_ETH:
2742 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2743 actual_filter_sz = spec_filter_size(kern_spec_mask,
2744 kern_filter_sz,
2745 ib_filter_sz);
2746 if (actual_filter_sz <= 0)
2747 return -EINVAL;
2748 ib_spec->size = sizeof(struct ib_flow_spec_eth);
2749 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2750 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2751 break;
2752 case IB_FLOW_SPEC_IPV4:
2753 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2754 actual_filter_sz = spec_filter_size(kern_spec_mask,
2755 kern_filter_sz,
2756 ib_filter_sz);
2757 if (actual_filter_sz <= 0)
2758 return -EINVAL;
2759 ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2760 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2761 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2762 break;
2763 case IB_FLOW_SPEC_IPV6:
2764 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2765 actual_filter_sz = spec_filter_size(kern_spec_mask,
2766 kern_filter_sz,
2767 ib_filter_sz);
2768 if (actual_filter_sz <= 0)
2769 return -EINVAL;
2770 ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2771 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2772 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2773
2774 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2775 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2776 return -EINVAL;
2777 break;
2778 case IB_FLOW_SPEC_TCP:
2779 case IB_FLOW_SPEC_UDP:
2780 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2781 actual_filter_sz = spec_filter_size(kern_spec_mask,
2782 kern_filter_sz,
2783 ib_filter_sz);
2784 if (actual_filter_sz <= 0)
2785 return -EINVAL;
2786 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2787 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2788 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2789 break;
2790 case IB_FLOW_SPEC_VXLAN_TUNNEL:
2791 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2792 actual_filter_sz = spec_filter_size(kern_spec_mask,
2793 kern_filter_sz,
2794 ib_filter_sz);
2795 if (actual_filter_sz <= 0)
2796 return -EINVAL;
2797 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2798 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2799 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2800
2801 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2802 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2803 return -EINVAL;
2804 break;
2805 case IB_FLOW_SPEC_ESP:
2806 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz);
2807 actual_filter_sz = spec_filter_size(kern_spec_mask,
2808 kern_filter_sz,
2809 ib_filter_sz);
2810 if (actual_filter_sz <= 0)
2811 return -EINVAL;
2812 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2813 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2814 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2815 break;
2816 case IB_FLOW_SPEC_GRE:
2817 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);
2818 actual_filter_sz = spec_filter_size(kern_spec_mask,
2819 kern_filter_sz,
2820 ib_filter_sz);
2821 if (actual_filter_sz <= 0)
2822 return -EINVAL;
2823 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2824 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2825 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2826 break;
2827 case IB_FLOW_SPEC_MPLS:
2828 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);
2829 actual_filter_sz = spec_filter_size(kern_spec_mask,
2830 kern_filter_sz,
2831 ib_filter_sz);
2832 if (actual_filter_sz <= 0)
2833 return -EINVAL;
2834 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2835 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2836 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2837 break;
2838 default:
2839 return -EINVAL;
2840 }
2841 return 0;
2842 }
2843
2844 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2845 union ib_flow_spec *ib_spec)
2846 {
2847 ssize_t kern_filter_sz;
2848 void *kern_spec_mask;
2849 void *kern_spec_val;
2850
2851 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2852
2853 kern_spec_val = (void *)kern_spec +
2854 sizeof(struct ib_uverbs_flow_spec_hdr);
2855 kern_spec_mask = kern_spec_val + kern_filter_sz;
2856
2857 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,
2858 kern_spec_mask,
2859 kern_spec_val,
2860 kern_filter_sz, ib_spec);
2861 }
2862
2863 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
2864 struct ib_uverbs_flow_spec *kern_spec,
2865 union ib_flow_spec *ib_spec,
2866 struct ib_uflow_resources *uflow_res)
2867 {
2868 if (kern_spec->reserved)
2869 return -EINVAL;
2870
2871 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2872 return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
2873 uflow_res);
2874 else
2875 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2876 }
2877
2878 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
2879 {
2880 struct ib_uverbs_ex_create_wq cmd;
2881 struct ib_uverbs_ex_create_wq_resp resp = {};
2882 struct ib_uwq_object *obj;
2883 int err = 0;
2884 struct ib_cq *cq;
2885 struct ib_pd *pd;
2886 struct ib_wq *wq;
2887 struct ib_wq_init_attr wq_init_attr = {};
2888 struct ib_device *ib_dev;
2889
2890 err = uverbs_request(attrs, &cmd, sizeof(cmd));
2891 if (err)
2892 return err;
2893
2894 if (cmd.comp_mask)
2895 return -EOPNOTSUPP;
2896
2897 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
2898 &ib_dev);
2899 if (IS_ERR(obj))
2900 return PTR_ERR(obj);
2901
2902 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2903 if (!pd) {
2904 err = -EINVAL;
2905 goto err_uobj;
2906 }
2907
2908 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
2909 if (!cq) {
2910 err = -EINVAL;
2911 goto err_put_pd;
2912 }
2913
2914 wq_init_attr.cq = cq;
2915 wq_init_attr.max_sge = cmd.max_sge;
2916 wq_init_attr.max_wr = cmd.max_wr;
2917 wq_init_attr.wq_context = attrs->ufile;
2918 wq_init_attr.wq_type = cmd.wq_type;
2919 wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2920 wq_init_attr.create_flags = cmd.create_flags;
2921 obj->uevent.events_reported = 0;
2922 INIT_LIST_HEAD(&obj->uevent.event_list);
2923
2924 wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
2925 if (IS_ERR(wq)) {
2926 err = PTR_ERR(wq);
2927 goto err_put_cq;
2928 }
2929
2930 wq->uobject = &obj->uevent.uobject;
2931 obj->uevent.uobject.object = wq;
2932 wq->wq_type = wq_init_attr.wq_type;
2933 wq->cq = cq;
2934 wq->pd = pd;
2935 wq->device = pd->device;
2936 wq->wq_context = wq_init_attr.wq_context;
2937 atomic_set(&wq->usecnt, 0);
2938 atomic_inc(&pd->usecnt);
2939 atomic_inc(&cq->usecnt);
2940 wq->uobject = &obj->uevent.uobject;
2941 obj->uevent.uobject.object = wq;
2942
2943 memset(&resp, 0, sizeof(resp));
2944 resp.wq_handle = obj->uevent.uobject.id;
2945 resp.max_sge = wq_init_attr.max_sge;
2946 resp.max_wr = wq_init_attr.max_wr;
2947 resp.wqn = wq->wq_num;
2948 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2949 err = uverbs_response(attrs, &resp, sizeof(resp));
2950 if (err)
2951 goto err_copy;
2952
2953 uobj_put_obj_read(pd);
2954 uobj_put_obj_read(cq);
2955 return uobj_alloc_commit(&obj->uevent.uobject);
2956
2957 err_copy:
2958 ib_destroy_wq(wq);
2959 err_put_cq:
2960 uobj_put_obj_read(cq);
2961 err_put_pd:
2962 uobj_put_obj_read(pd);
2963 err_uobj:
2964 uobj_alloc_abort(&obj->uevent.uobject);
2965
2966 return err;
2967 }
2968
2969 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
2970 {
2971 struct ib_uverbs_ex_destroy_wq cmd;
2972 struct ib_uverbs_ex_destroy_wq_resp resp = {};
2973 struct ib_uobject *uobj;
2974 struct ib_uwq_object *obj;
2975 int ret;
2976
2977 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2978 if (ret)
2979 return ret;
2980
2981 if (cmd.comp_mask)
2982 return -EOPNOTSUPP;
2983
2984 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2985 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
2986 if (IS_ERR(uobj))
2987 return PTR_ERR(uobj);
2988
2989 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
2990 resp.events_reported = obj->uevent.events_reported;
2991
2992 uobj_put_destroy(uobj);
2993
2994 return uverbs_response(attrs, &resp, sizeof(resp));
2995 }
2996
2997 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
2998 {
2999 struct ib_uverbs_ex_modify_wq cmd;
3000 struct ib_wq *wq;
3001 struct ib_wq_attr wq_attr = {};
3002 int ret;
3003
3004 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3005 if (ret)
3006 return ret;
3007
3008 if (!cmd.attr_mask)
3009 return -EINVAL;
3010
3011 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3012 return -EINVAL;
3013
3014 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3015 if (!wq)
3016 return -EINVAL;
3017
3018 wq_attr.curr_wq_state = cmd.curr_wq_state;
3019 wq_attr.wq_state = cmd.wq_state;
3020 if (cmd.attr_mask & IB_WQ_FLAGS) {
3021 wq_attr.flags = cmd.flags;
3022 wq_attr.flags_mask = cmd.flags_mask;
3023 }
3024 ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3025 &attrs->driver_udata);
3026 uobj_put_obj_read(wq);
3027 return ret;
3028 }
3029
3030 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3031 {
3032 struct ib_uverbs_ex_create_rwq_ind_table cmd;
3033 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {};
3034 struct ib_uobject *uobj;
3035 int err;
3036 struct ib_rwq_ind_table_init_attr init_attr = {};
3037 struct ib_rwq_ind_table *rwq_ind_tbl;
3038 struct ib_wq **wqs = NULL;
3039 u32 *wqs_handles = NULL;
3040 struct ib_wq *wq = NULL;
3041 int i, j, num_read_wqs;
3042 u32 num_wq_handles;
3043 struct uverbs_req_iter iter;
3044 struct ib_device *ib_dev;
3045
3046 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3047 if (err)
3048 return err;
3049
3050 if (cmd.comp_mask)
3051 return -EOPNOTSUPP;
3052
3053 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3054 return -EINVAL;
3055
3056 num_wq_handles = 1 << cmd.log_ind_tbl_size;
3057 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3058 GFP_KERNEL);
3059 if (!wqs_handles)
3060 return -ENOMEM;
3061
3062 err = uverbs_request_next(&iter, wqs_handles,
3063 num_wq_handles * sizeof(__u32));
3064 if (err)
3065 goto err_free;
3066
3067 err = uverbs_request_finish(&iter);
3068 if (err)
3069 goto err_free;
3070
3071 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3072 if (!wqs) {
3073 err = -ENOMEM;
3074 goto err_free;
3075 }
3076
3077 for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3078 num_read_wqs++) {
3079 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3080 wqs_handles[num_read_wqs], attrs);
3081 if (!wq) {
3082 err = -EINVAL;
3083 goto put_wqs;
3084 }
3085
3086 wqs[num_read_wqs] = wq;
3087 }
3088
3089 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3090 if (IS_ERR(uobj)) {
3091 err = PTR_ERR(uobj);
3092 goto put_wqs;
3093 }
3094
3095 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3096 init_attr.ind_tbl = wqs;
3097
3098 rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr,
3099 &attrs->driver_udata);
3100
3101 if (IS_ERR(rwq_ind_tbl)) {
3102 err = PTR_ERR(rwq_ind_tbl);
3103 goto err_uobj;
3104 }
3105
3106 rwq_ind_tbl->ind_tbl = wqs;
3107 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3108 rwq_ind_tbl->uobject = uobj;
3109 uobj->object = rwq_ind_tbl;
3110 rwq_ind_tbl->device = ib_dev;
3111 atomic_set(&rwq_ind_tbl->usecnt, 0);
3112
3113 for (i = 0; i < num_wq_handles; i++)
3114 atomic_inc(&wqs[i]->usecnt);
3115
3116 resp.ind_tbl_handle = uobj->id;
3117 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3118 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3119
3120 err = uverbs_response(attrs, &resp, sizeof(resp));
3121 if (err)
3122 goto err_copy;
3123
3124 kfree(wqs_handles);
3125
3126 for (j = 0; j < num_read_wqs; j++)
3127 uobj_put_obj_read(wqs[j]);
3128
3129 return uobj_alloc_commit(uobj);
3130
3131 err_copy:
3132 ib_destroy_rwq_ind_table(rwq_ind_tbl);
3133 err_uobj:
3134 uobj_alloc_abort(uobj);
3135 put_wqs:
3136 for (j = 0; j < num_read_wqs; j++)
3137 uobj_put_obj_read(wqs[j]);
3138 err_free:
3139 kfree(wqs_handles);
3140 kfree(wqs);
3141 return err;
3142 }
3143
3144 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3145 {
3146 struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3147 int ret;
3148
3149 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3150 if (ret)
3151 return ret;
3152
3153 if (cmd.comp_mask)
3154 return -EOPNOTSUPP;
3155
3156 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3157 cmd.ind_tbl_handle, attrs);
3158 }
3159
3160 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3161 {
3162 struct ib_uverbs_create_flow cmd;
3163 struct ib_uverbs_create_flow_resp resp;
3164 struct ib_uobject *uobj;
3165 struct ib_flow *flow_id;
3166 struct ib_uverbs_flow_attr *kern_flow_attr;
3167 struct ib_flow_attr *flow_attr;
3168 struct ib_qp *qp;
3169 struct ib_uflow_resources *uflow_res;
3170 struct ib_uverbs_flow_spec_hdr *kern_spec;
3171 struct uverbs_req_iter iter;
3172 int err;
3173 void *ib_spec;
3174 int i;
3175 struct ib_device *ib_dev;
3176
3177 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3178 if (err)
3179 return err;
3180
3181 if (cmd.comp_mask)
3182 return -EINVAL;
3183
3184 if (!capable(CAP_NET_RAW))
3185 return -EPERM;
3186
3187 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3188 return -EINVAL;
3189
3190 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3191 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3192 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3193 return -EINVAL;
3194
3195 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3196 return -EINVAL;
3197
3198 if (cmd.flow_attr.size >
3199 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3200 return -EINVAL;
3201
3202 if (cmd.flow_attr.reserved[0] ||
3203 cmd.flow_attr.reserved[1])
3204 return -EINVAL;
3205
3206 if (cmd.flow_attr.num_of_specs) {
3207 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3208 GFP_KERNEL);
3209 if (!kern_flow_attr)
3210 return -ENOMEM;
3211
3212 *kern_flow_attr = cmd.flow_attr;
3213 err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
3214 cmd.flow_attr.size);
3215 if (err)
3216 goto err_free_attr;
3217 } else {
3218 kern_flow_attr = &cmd.flow_attr;
3219 }
3220
3221 err = uverbs_request_finish(&iter);
3222 if (err)
3223 goto err_free_attr;
3224
3225 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3226 if (IS_ERR(uobj)) {
3227 err = PTR_ERR(uobj);
3228 goto err_free_attr;
3229 }
3230
3231 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3232 if (!qp) {
3233 err = -EINVAL;
3234 goto err_uobj;
3235 }
3236
3237 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3238 err = -EINVAL;
3239 goto err_put;
3240 }
3241
3242 flow_attr = kzalloc(struct_size(flow_attr, flows,
3243 cmd.flow_attr.num_of_specs), GFP_KERNEL);
3244 if (!flow_attr) {
3245 err = -ENOMEM;
3246 goto err_put;
3247 }
3248 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3249 if (!uflow_res) {
3250 err = -ENOMEM;
3251 goto err_free_flow_attr;
3252 }
3253
3254 flow_attr->type = kern_flow_attr->type;
3255 flow_attr->priority = kern_flow_attr->priority;
3256 flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3257 flow_attr->port = kern_flow_attr->port;
3258 flow_attr->flags = kern_flow_attr->flags;
3259 flow_attr->size = sizeof(*flow_attr);
3260
3261 kern_spec = kern_flow_attr->flow_specs;
3262 ib_spec = flow_attr + 1;
3263 for (i = 0; i < flow_attr->num_of_specs &&
3264 cmd.flow_attr.size >= sizeof(*kern_spec) &&
3265 cmd.flow_attr.size >= kern_spec->size;
3266 i++) {
3267 err = kern_spec_to_ib_spec(
3268 attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3269 ib_spec, uflow_res);
3270 if (err)
3271 goto err_free;
3272
3273 flow_attr->size +=
3274 ((union ib_flow_spec *) ib_spec)->size;
3275 cmd.flow_attr.size -= kern_spec->size;
3276 kern_spec = ((void *)kern_spec) + kern_spec->size;
3277 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3278 }
3279 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3280 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3281 i, cmd.flow_attr.size);
3282 err = -EINVAL;
3283 goto err_free;
3284 }
3285
3286 flow_id = qp->device->ops.create_flow(
3287 qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata);
3288
3289 if (IS_ERR(flow_id)) {
3290 err = PTR_ERR(flow_id);
3291 goto err_free;
3292 }
3293
3294 ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3295
3296 memset(&resp, 0, sizeof(resp));
3297 resp.flow_handle = uobj->id;
3298
3299 err = uverbs_response(attrs, &resp, sizeof(resp));
3300 if (err)
3301 goto err_copy;
3302
3303 uobj_put_obj_read(qp);
3304 kfree(flow_attr);
3305 if (cmd.flow_attr.num_of_specs)
3306 kfree(kern_flow_attr);
3307 return uobj_alloc_commit(uobj);
3308 err_copy:
3309 if (!qp->device->ops.destroy_flow(flow_id))
3310 atomic_dec(&qp->usecnt);
3311 err_free:
3312 ib_uverbs_flow_resources_free(uflow_res);
3313 err_free_flow_attr:
3314 kfree(flow_attr);
3315 err_put:
3316 uobj_put_obj_read(qp);
3317 err_uobj:
3318 uobj_alloc_abort(uobj);
3319 err_free_attr:
3320 if (cmd.flow_attr.num_of_specs)
3321 kfree(kern_flow_attr);
3322 return err;
3323 }
3324
3325 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3326 {
3327 struct ib_uverbs_destroy_flow cmd;
3328 int ret;
3329
3330 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3331 if (ret)
3332 return ret;
3333
3334 if (cmd.comp_mask)
3335 return -EINVAL;
3336
3337 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3338 }
3339
3340 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3341 struct ib_uverbs_create_xsrq *cmd,
3342 struct ib_udata *udata)
3343 {
3344 struct ib_uverbs_create_srq_resp resp;
3345 struct ib_usrq_object *obj;
3346 struct ib_pd *pd;
3347 struct ib_srq *srq;
3348 struct ib_uobject *uninitialized_var(xrcd_uobj);
3349 struct ib_srq_init_attr attr;
3350 int ret;
3351 struct ib_device *ib_dev;
3352
3353 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3354 &ib_dev);
3355 if (IS_ERR(obj))
3356 return PTR_ERR(obj);
3357
3358 if (cmd->srq_type == IB_SRQT_TM)
3359 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3360
3361 if (cmd->srq_type == IB_SRQT_XRC) {
3362 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3363 attrs);
3364 if (IS_ERR(xrcd_uobj)) {
3365 ret = -EINVAL;
3366 goto err;
3367 }
3368
3369 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3370 if (!attr.ext.xrc.xrcd) {
3371 ret = -EINVAL;
3372 goto err_put_xrcd;
3373 }
3374
3375 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3376 atomic_inc(&obj->uxrcd->refcnt);
3377 }
3378
3379 if (ib_srq_has_cq(cmd->srq_type)) {
3380 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3381 cmd->cq_handle, attrs);
3382 if (!attr.ext.cq) {
3383 ret = -EINVAL;
3384 goto err_put_xrcd;
3385 }
3386 }
3387
3388 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3389 if (!pd) {
3390 ret = -EINVAL;
3391 goto err_put_cq;
3392 }
3393
3394 attr.event_handler = ib_uverbs_srq_event_handler;
3395 attr.srq_context = attrs->ufile;
3396 attr.srq_type = cmd->srq_type;
3397 attr.attr.max_wr = cmd->max_wr;
3398 attr.attr.max_sge = cmd->max_sge;
3399 attr.attr.srq_limit = cmd->srq_limit;
3400
3401 obj->uevent.events_reported = 0;
3402 INIT_LIST_HEAD(&obj->uevent.event_list);
3403
3404 srq = pd->device->ops.create_srq(pd, &attr, udata);
3405 if (IS_ERR(srq)) {
3406 ret = PTR_ERR(srq);
3407 goto err_put;
3408 }
3409
3410 srq->device = pd->device;
3411 srq->pd = pd;
3412 srq->srq_type = cmd->srq_type;
3413 srq->uobject = &obj->uevent.uobject;
3414 srq->event_handler = attr.event_handler;
3415 srq->srq_context = attr.srq_context;
3416
3417 if (ib_srq_has_cq(cmd->srq_type)) {
3418 srq->ext.cq = attr.ext.cq;
3419 atomic_inc(&attr.ext.cq->usecnt);
3420 }
3421
3422 if (cmd->srq_type == IB_SRQT_XRC) {
3423 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3424 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3425 }
3426
3427 atomic_inc(&pd->usecnt);
3428 atomic_set(&srq->usecnt, 0);
3429
3430 obj->uevent.uobject.object = srq;
3431 obj->uevent.uobject.user_handle = cmd->user_handle;
3432
3433 memset(&resp, 0, sizeof resp);
3434 resp.srq_handle = obj->uevent.uobject.id;
3435 resp.max_wr = attr.attr.max_wr;
3436 resp.max_sge = attr.attr.max_sge;
3437 if (cmd->srq_type == IB_SRQT_XRC)
3438 resp.srqn = srq->ext.xrc.srq_num;
3439
3440 ret = uverbs_response(attrs, &resp, sizeof(resp));
3441 if (ret)
3442 goto err_copy;
3443
3444 if (cmd->srq_type == IB_SRQT_XRC)
3445 uobj_put_read(xrcd_uobj);
3446
3447 if (ib_srq_has_cq(cmd->srq_type))
3448 uobj_put_obj_read(attr.ext.cq);
3449
3450 uobj_put_obj_read(pd);
3451 return uobj_alloc_commit(&obj->uevent.uobject);
3452
3453 err_copy:
3454 ib_destroy_srq(srq);
3455
3456 err_put:
3457 uobj_put_obj_read(pd);
3458
3459 err_put_cq:
3460 if (ib_srq_has_cq(cmd->srq_type))
3461 uobj_put_obj_read(attr.ext.cq);
3462
3463 err_put_xrcd:
3464 if (cmd->srq_type == IB_SRQT_XRC) {
3465 atomic_dec(&obj->uxrcd->refcnt);
3466 uobj_put_read(xrcd_uobj);
3467 }
3468
3469 err:
3470 uobj_alloc_abort(&obj->uevent.uobject);
3471 return ret;
3472 }
3473
3474 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3475 {
3476 struct ib_uverbs_create_srq cmd;
3477 struct ib_uverbs_create_xsrq xcmd;
3478 int ret;
3479
3480 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3481 if (ret)
3482 return ret;
3483
3484 memset(&xcmd, 0, sizeof(xcmd));
3485 xcmd.response = cmd.response;
3486 xcmd.user_handle = cmd.user_handle;
3487 xcmd.srq_type = IB_SRQT_BASIC;
3488 xcmd.pd_handle = cmd.pd_handle;
3489 xcmd.max_wr = cmd.max_wr;
3490 xcmd.max_sge = cmd.max_sge;
3491 xcmd.srq_limit = cmd.srq_limit;
3492
3493 return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3494 }
3495
3496 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3497 {
3498 struct ib_uverbs_create_xsrq cmd;
3499 int ret;
3500
3501 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3502 if (ret)
3503 return ret;
3504
3505 return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3506 }
3507
3508 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3509 {
3510 struct ib_uverbs_modify_srq cmd;
3511 struct ib_srq *srq;
3512 struct ib_srq_attr attr;
3513 int ret;
3514
3515 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3516 if (ret)
3517 return ret;
3518
3519 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3520 if (!srq)
3521 return -EINVAL;
3522
3523 attr.max_wr = cmd.max_wr;
3524 attr.srq_limit = cmd.srq_limit;
3525
3526 ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3527 &attrs->driver_udata);
3528
3529 uobj_put_obj_read(srq);
3530
3531 return ret;
3532 }
3533
3534 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3535 {
3536 struct ib_uverbs_query_srq cmd;
3537 struct ib_uverbs_query_srq_resp resp;
3538 struct ib_srq_attr attr;
3539 struct ib_srq *srq;
3540 int ret;
3541
3542 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3543 if (ret)
3544 return ret;
3545
3546 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3547 if (!srq)
3548 return -EINVAL;
3549
3550 ret = ib_query_srq(srq, &attr);
3551
3552 uobj_put_obj_read(srq);
3553
3554 if (ret)
3555 return ret;
3556
3557 memset(&resp, 0, sizeof resp);
3558
3559 resp.max_wr = attr.max_wr;
3560 resp.max_sge = attr.max_sge;
3561 resp.srq_limit = attr.srq_limit;
3562
3563 return uverbs_response(attrs, &resp, sizeof(resp));
3564 }
3565
3566 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3567 {
3568 struct ib_uverbs_destroy_srq cmd;
3569 struct ib_uverbs_destroy_srq_resp resp;
3570 struct ib_uobject *uobj;
3571 struct ib_uevent_object *obj;
3572 int ret;
3573
3574 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3575 if (ret)
3576 return ret;
3577
3578 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3579 if (IS_ERR(uobj))
3580 return PTR_ERR(uobj);
3581
3582 obj = container_of(uobj, struct ib_uevent_object, uobject);
3583 memset(&resp, 0, sizeof(resp));
3584 resp.events_reported = obj->events_reported;
3585
3586 uobj_put_destroy(uobj);
3587
3588 return uverbs_response(attrs, &resp, sizeof(resp));
3589 }
3590
3591 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3592 {
3593 struct ib_uverbs_ex_query_device_resp resp = {};
3594 struct ib_uverbs_ex_query_device cmd;
3595 struct ib_device_attr attr = {0};
3596 struct ib_ucontext *ucontext;
3597 struct ib_device *ib_dev;
3598 int err;
3599
3600 ucontext = ib_uverbs_get_ucontext(attrs);
3601 if (IS_ERR(ucontext))
3602 return PTR_ERR(ucontext);
3603 ib_dev = ucontext->device;
3604
3605 err = uverbs_request(attrs, &cmd, sizeof(cmd));
3606 if (err)
3607 return err;
3608
3609 if (cmd.comp_mask)
3610 return -EINVAL;
3611
3612 if (cmd.reserved)
3613 return -EINVAL;
3614
3615 err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
3616 if (err)
3617 return err;
3618
3619 copy_query_dev_fields(ucontext, &resp.base, &attr);
3620
3621 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3622 resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3623 resp.odp_caps.per_transport_caps.rc_odp_caps =
3624 attr.odp_caps.per_transport_caps.rc_odp_caps;
3625 resp.odp_caps.per_transport_caps.uc_odp_caps =
3626 attr.odp_caps.per_transport_caps.uc_odp_caps;
3627 resp.odp_caps.per_transport_caps.ud_odp_caps =
3628 attr.odp_caps.per_transport_caps.ud_odp_caps;
3629 #endif
3630
3631 resp.timestamp_mask = attr.timestamp_mask;
3632 resp.hca_core_clock = attr.hca_core_clock;
3633 resp.device_cap_flags_ex = attr.device_cap_flags;
3634 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3635 resp.rss_caps.max_rwq_indirection_tables =
3636 attr.rss_caps.max_rwq_indirection_tables;
3637 resp.rss_caps.max_rwq_indirection_table_size =
3638 attr.rss_caps.max_rwq_indirection_table_size;
3639 resp.max_wq_type_rq = attr.max_wq_type_rq;
3640 resp.raw_packet_caps = attr.raw_packet_caps;
3641 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size;
3642 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags;
3643 resp.tm_caps.max_ops = attr.tm_caps.max_ops;
3644 resp.tm_caps.max_sge = attr.tm_caps.max_sge;
3645 resp.tm_caps.flags = attr.tm_caps.flags;
3646 resp.cq_moderation_caps.max_cq_moderation_count =
3647 attr.cq_caps.max_cq_moderation_count;
3648 resp.cq_moderation_caps.max_cq_moderation_period =
3649 attr.cq_caps.max_cq_moderation_period;
3650 resp.max_dm_size = attr.max_dm_size;
3651 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3652
3653 return uverbs_response(attrs, &resp, sizeof(resp));
3654 }
3655
3656 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
3657 {
3658 struct ib_uverbs_ex_modify_cq cmd;
3659 struct ib_cq *cq;
3660 int ret;
3661
3662 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3663 if (ret)
3664 return ret;
3665
3666 if (!cmd.attr_mask || cmd.reserved)
3667 return -EINVAL;
3668
3669 if (cmd.attr_mask > IB_CQ_MODERATE)
3670 return -EOPNOTSUPP;
3671
3672 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3673 if (!cq)
3674 return -EINVAL;
3675
3676 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
3677
3678 uobj_put_obj_read(cq);
3679
3680 return ret;
3681 }
3682
3683 /*
3684 * Describe the input structs for write(). Some write methods have an input
3685 * only struct, most have an input and output. If the struct has an output then
3686 * the 'response' u64 must be the first field in the request structure.
3687 *
3688 * If udata is present then both the request and response structs have a
3689 * trailing driver_data flex array. In this case the size of the base struct
3690 * cannot be changed.
3691 */
3692 #define offsetof_after(_struct, _member) \
3693 (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
3694
3695 #define UAPI_DEF_WRITE_IO(req, resp) \
3696 .write.has_resp = 1 + \
3697 BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
3698 BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) != \
3699 sizeof(u64)), \
3700 .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3701
3702 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3703
3704 #define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
3705 UAPI_DEF_WRITE_IO(req, resp), \
3706 .write.has_udata = \
3707 1 + \
3708 BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3709 sizeof(req)) + \
3710 BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
3711 sizeof(resp))
3712
3713 #define UAPI_DEF_WRITE_UDATA_I(req) \
3714 UAPI_DEF_WRITE_I(req), \
3715 .write.has_udata = \
3716 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3717 sizeof(req))
3718
3719 /*
3720 * The _EX versions are for use with WRITE_EX and allow the last struct member
3721 * to be specified. Buffers that do not include that member will be rejected.
3722 */
3723 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
3724 .write.has_resp = 1, \
3725 .write.req_size = offsetof_after(req, req_last_member), \
3726 .write.resp_size = offsetof_after(resp, resp_last_member)
3727
3728 #define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
3729 .write.req_size = offsetof_after(req, req_last_member)
3730
3731 const struct uapi_definition uverbs_def_write_intf[] = {
3732 DECLARE_UVERBS_OBJECT(
3733 UVERBS_OBJECT_AH,
3734 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3735 ib_uverbs_create_ah,
3736 UAPI_DEF_WRITE_UDATA_IO(
3737 struct ib_uverbs_create_ah,
3738 struct ib_uverbs_create_ah_resp),
3739 UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
3740 DECLARE_UVERBS_WRITE(
3741 IB_USER_VERBS_CMD_DESTROY_AH,
3742 ib_uverbs_destroy_ah,
3743 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
3744 UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
3745
3746 DECLARE_UVERBS_OBJECT(
3747 UVERBS_OBJECT_COMP_CHANNEL,
3748 DECLARE_UVERBS_WRITE(
3749 IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3750 ib_uverbs_create_comp_channel,
3751 UAPI_DEF_WRITE_IO(
3752 struct ib_uverbs_create_comp_channel,
3753 struct ib_uverbs_create_comp_channel_resp))),
3754
3755 DECLARE_UVERBS_OBJECT(
3756 UVERBS_OBJECT_CQ,
3757 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
3758 ib_uverbs_create_cq,
3759 UAPI_DEF_WRITE_UDATA_IO(
3760 struct ib_uverbs_create_cq,
3761 struct ib_uverbs_create_cq_resp),
3762 UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3763 DECLARE_UVERBS_WRITE(
3764 IB_USER_VERBS_CMD_DESTROY_CQ,
3765 ib_uverbs_destroy_cq,
3766 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3767 struct ib_uverbs_destroy_cq_resp),
3768 UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3769 DECLARE_UVERBS_WRITE(
3770 IB_USER_VERBS_CMD_POLL_CQ,
3771 ib_uverbs_poll_cq,
3772 UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3773 struct ib_uverbs_poll_cq_resp),
3774 UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3775 DECLARE_UVERBS_WRITE(
3776 IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3777 ib_uverbs_req_notify_cq,
3778 UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3779 UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
3780 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
3781 ib_uverbs_resize_cq,
3782 UAPI_DEF_WRITE_UDATA_IO(
3783 struct ib_uverbs_resize_cq,
3784 struct ib_uverbs_resize_cq_resp),
3785 UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
3786 DECLARE_UVERBS_WRITE_EX(
3787 IB_USER_VERBS_EX_CMD_CREATE_CQ,
3788 ib_uverbs_ex_create_cq,
3789 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3790 reserved,
3791 struct ib_uverbs_ex_create_cq_resp,
3792 response_length),
3793 UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3794 DECLARE_UVERBS_WRITE_EX(
3795 IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3796 ib_uverbs_ex_modify_cq,
3797 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3798 UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
3799
3800 DECLARE_UVERBS_OBJECT(
3801 UVERBS_OBJECT_DEVICE,
3802 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
3803 ib_uverbs_get_context,
3804 UAPI_DEF_WRITE_UDATA_IO(
3805 struct ib_uverbs_get_context,
3806 struct ib_uverbs_get_context_resp)),
3807 DECLARE_UVERBS_WRITE(
3808 IB_USER_VERBS_CMD_QUERY_DEVICE,
3809 ib_uverbs_query_device,
3810 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3811 struct ib_uverbs_query_device_resp)),
3812 DECLARE_UVERBS_WRITE(
3813 IB_USER_VERBS_CMD_QUERY_PORT,
3814 ib_uverbs_query_port,
3815 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3816 struct ib_uverbs_query_port_resp),
3817 UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3818 DECLARE_UVERBS_WRITE_EX(
3819 IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3820 ib_uverbs_ex_query_device,
3821 UAPI_DEF_WRITE_IO_EX(
3822 struct ib_uverbs_ex_query_device,
3823 reserved,
3824 struct ib_uverbs_ex_query_device_resp,
3825 response_length),
3826 UAPI_DEF_METHOD_NEEDS_FN(query_device)),
3827 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3828 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
3829
3830 DECLARE_UVERBS_OBJECT(
3831 UVERBS_OBJECT_FLOW,
3832 DECLARE_UVERBS_WRITE_EX(
3833 IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3834 ib_uverbs_ex_create_flow,
3835 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3836 flow_attr,
3837 struct ib_uverbs_create_flow_resp,
3838 flow_handle),
3839 UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3840 DECLARE_UVERBS_WRITE_EX(
3841 IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3842 ib_uverbs_ex_destroy_flow,
3843 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3844 UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
3845
3846 DECLARE_UVERBS_OBJECT(
3847 UVERBS_OBJECT_MR,
3848 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3849 ib_uverbs_dereg_mr,
3850 UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
3851 UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
3852 DECLARE_UVERBS_WRITE(
3853 IB_USER_VERBS_CMD_REG_MR,
3854 ib_uverbs_reg_mr,
3855 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3856 struct ib_uverbs_reg_mr_resp),
3857 UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3858 DECLARE_UVERBS_WRITE(
3859 IB_USER_VERBS_CMD_REREG_MR,
3860 ib_uverbs_rereg_mr,
3861 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3862 struct ib_uverbs_rereg_mr_resp),
3863 UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
3864
3865 DECLARE_UVERBS_OBJECT(
3866 UVERBS_OBJECT_MW,
3867 DECLARE_UVERBS_WRITE(
3868 IB_USER_VERBS_CMD_ALLOC_MW,
3869 ib_uverbs_alloc_mw,
3870 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3871 struct ib_uverbs_alloc_mw_resp),
3872 UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3873 DECLARE_UVERBS_WRITE(
3874 IB_USER_VERBS_CMD_DEALLOC_MW,
3875 ib_uverbs_dealloc_mw,
3876 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3877 UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
3878
3879 DECLARE_UVERBS_OBJECT(
3880 UVERBS_OBJECT_PD,
3881 DECLARE_UVERBS_WRITE(
3882 IB_USER_VERBS_CMD_ALLOC_PD,
3883 ib_uverbs_alloc_pd,
3884 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3885 struct ib_uverbs_alloc_pd_resp),
3886 UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3887 DECLARE_UVERBS_WRITE(
3888 IB_USER_VERBS_CMD_DEALLOC_PD,
3889 ib_uverbs_dealloc_pd,
3890 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3891 UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
3892
3893 DECLARE_UVERBS_OBJECT(
3894 UVERBS_OBJECT_QP,
3895 DECLARE_UVERBS_WRITE(
3896 IB_USER_VERBS_CMD_ATTACH_MCAST,
3897 ib_uverbs_attach_mcast,
3898 UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3899 UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3900 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3901 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
3902 ib_uverbs_create_qp,
3903 UAPI_DEF_WRITE_UDATA_IO(
3904 struct ib_uverbs_create_qp,
3905 struct ib_uverbs_create_qp_resp),
3906 UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3907 DECLARE_UVERBS_WRITE(
3908 IB_USER_VERBS_CMD_DESTROY_QP,
3909 ib_uverbs_destroy_qp,
3910 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3911 struct ib_uverbs_destroy_qp_resp),
3912 UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3913 DECLARE_UVERBS_WRITE(
3914 IB_USER_VERBS_CMD_DETACH_MCAST,
3915 ib_uverbs_detach_mcast,
3916 UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3917 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3918 DECLARE_UVERBS_WRITE(
3919 IB_USER_VERBS_CMD_MODIFY_QP,
3920 ib_uverbs_modify_qp,
3921 UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3922 UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3923 DECLARE_UVERBS_WRITE(
3924 IB_USER_VERBS_CMD_POST_RECV,
3925 ib_uverbs_post_recv,
3926 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3927 struct ib_uverbs_post_recv_resp),
3928 UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3929 DECLARE_UVERBS_WRITE(
3930 IB_USER_VERBS_CMD_POST_SEND,
3931 ib_uverbs_post_send,
3932 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3933 struct ib_uverbs_post_send_resp),
3934 UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3935 DECLARE_UVERBS_WRITE(
3936 IB_USER_VERBS_CMD_QUERY_QP,
3937 ib_uverbs_query_qp,
3938 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3939 struct ib_uverbs_query_qp_resp),
3940 UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3941 DECLARE_UVERBS_WRITE_EX(
3942 IB_USER_VERBS_EX_CMD_CREATE_QP,
3943 ib_uverbs_ex_create_qp,
3944 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3945 comp_mask,
3946 struct ib_uverbs_ex_create_qp_resp,
3947 response_length),
3948 UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3949 DECLARE_UVERBS_WRITE_EX(
3950 IB_USER_VERBS_EX_CMD_MODIFY_QP,
3951 ib_uverbs_ex_modify_qp,
3952 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3953 base,
3954 struct ib_uverbs_ex_modify_qp_resp,
3955 response_length),
3956 UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
3957
3958 DECLARE_UVERBS_OBJECT(
3959 UVERBS_OBJECT_RWQ_IND_TBL,
3960 DECLARE_UVERBS_WRITE_EX(
3961 IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
3962 ib_uverbs_ex_create_rwq_ind_table,
3963 UAPI_DEF_WRITE_IO_EX(
3964 struct ib_uverbs_ex_create_rwq_ind_table,
3965 log_ind_tbl_size,
3966 struct ib_uverbs_ex_create_rwq_ind_table_resp,
3967 ind_tbl_num),
3968 UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
3969 DECLARE_UVERBS_WRITE_EX(
3970 IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
3971 ib_uverbs_ex_destroy_rwq_ind_table,
3972 UAPI_DEF_WRITE_I(
3973 struct ib_uverbs_ex_destroy_rwq_ind_table),
3974 UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
3975
3976 DECLARE_UVERBS_OBJECT(
3977 UVERBS_OBJECT_WQ,
3978 DECLARE_UVERBS_WRITE_EX(
3979 IB_USER_VERBS_EX_CMD_CREATE_WQ,
3980 ib_uverbs_ex_create_wq,
3981 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
3982 max_sge,
3983 struct ib_uverbs_ex_create_wq_resp,
3984 wqn),
3985 UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
3986 DECLARE_UVERBS_WRITE_EX(
3987 IB_USER_VERBS_EX_CMD_DESTROY_WQ,
3988 ib_uverbs_ex_destroy_wq,
3989 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
3990 wq_handle,
3991 struct ib_uverbs_ex_destroy_wq_resp,
3992 reserved),
3993 UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
3994 DECLARE_UVERBS_WRITE_EX(
3995 IB_USER_VERBS_EX_CMD_MODIFY_WQ,
3996 ib_uverbs_ex_modify_wq,
3997 UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
3998 curr_wq_state),
3999 UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
4000
4001 DECLARE_UVERBS_OBJECT(
4002 UVERBS_OBJECT_SRQ,
4003 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
4004 ib_uverbs_create_srq,
4005 UAPI_DEF_WRITE_UDATA_IO(
4006 struct ib_uverbs_create_srq,
4007 struct ib_uverbs_create_srq_resp),
4008 UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4009 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
4010 ib_uverbs_create_xsrq,
4011 UAPI_DEF_WRITE_UDATA_IO(
4012 struct ib_uverbs_create_xsrq,
4013 struct ib_uverbs_create_srq_resp),
4014 UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4015 DECLARE_UVERBS_WRITE(
4016 IB_USER_VERBS_CMD_DESTROY_SRQ,
4017 ib_uverbs_destroy_srq,
4018 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4019 struct ib_uverbs_destroy_srq_resp),
4020 UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4021 DECLARE_UVERBS_WRITE(
4022 IB_USER_VERBS_CMD_MODIFY_SRQ,
4023 ib_uverbs_modify_srq,
4024 UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4025 UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4026 DECLARE_UVERBS_WRITE(
4027 IB_USER_VERBS_CMD_POST_SRQ_RECV,
4028 ib_uverbs_post_srq_recv,
4029 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4030 struct ib_uverbs_post_srq_recv_resp),
4031 UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4032 DECLARE_UVERBS_WRITE(
4033 IB_USER_VERBS_CMD_QUERY_SRQ,
4034 ib_uverbs_query_srq,
4035 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4036 struct ib_uverbs_query_srq_resp),
4037 UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
4038
4039 DECLARE_UVERBS_OBJECT(
4040 UVERBS_OBJECT_XRCD,
4041 DECLARE_UVERBS_WRITE(
4042 IB_USER_VERBS_CMD_CLOSE_XRCD,
4043 ib_uverbs_close_xrcd,
4044 UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
4045 UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
4046 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
4047 ib_uverbs_open_qp,
4048 UAPI_DEF_WRITE_UDATA_IO(
4049 struct ib_uverbs_open_qp,
4050 struct ib_uverbs_create_qp_resp)),
4051 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4052 ib_uverbs_open_xrcd,
4053 UAPI_DEF_WRITE_UDATA_IO(
4054 struct ib_uverbs_open_xrcd,
4055 struct ib_uverbs_open_xrcd_resp),
4056 UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
4057
4058 {},
4059 };
Ubuntu Focal Linux kernel mirror