2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
41 #include <linux/uaccess.h>
43 #include <rdma/uverbs_types.h>
44 #include <rdma/uverbs_std_types.h>
45 #include "rdma_core.h"
48 #include "core_priv.h"
51 * Copy a response to userspace. If the provided 'resp' is larger than the
52 * user buffer it is silently truncated. If the user provided a larger buffer
53 * then the trailing portion is zero filled.
55 * These semantics are intended to support future extension of the output
58 static int uverbs_response(struct uverbs_attr_bundle
*attrs
, const void *resp
,
63 if (uverbs_attr_is_valid(attrs
, UVERBS_ATTR_CORE_OUT
))
64 return uverbs_copy_to_struct_or_zero(
65 attrs
, UVERBS_ATTR_CORE_OUT
, resp
, resp_len
);
67 if (copy_to_user(attrs
->ucore
.outbuf
, resp
,
68 min(attrs
->ucore
.outlen
, resp_len
)))
71 if (resp_len
< attrs
->ucore
.outlen
) {
73 * Zero fill any extra memory that user
74 * space might have provided.
76 ret
= clear_user(attrs
->ucore
.outbuf
+ resp_len
,
77 attrs
->ucore
.outlen
- resp_len
);
86 * Copy a request from userspace. If the provided 'req' is larger than the
87 * user buffer then the user buffer is zero extended into the 'req'. If 'req'
88 * is smaller than the user buffer then the uncopied bytes in the user buffer
91 static int uverbs_request(struct uverbs_attr_bundle
*attrs
, void *req
,
94 if (copy_from_user(req
, attrs
->ucore
.inbuf
,
95 min(attrs
->ucore
.inlen
, req_len
)))
98 if (attrs
->ucore
.inlen
< req_len
) {
99 memset(req
+ attrs
->ucore
.inlen
, 0,
100 req_len
- attrs
->ucore
.inlen
);
101 } else if (attrs
->ucore
.inlen
> req_len
) {
102 if (!ib_is_buffer_cleared(attrs
->ucore
.inbuf
+ req_len
,
103 attrs
->ucore
.inlen
- req_len
))
110 * Generate the value for the 'response_length' protocol used by write_ex.
111 * This is the number of bytes the kernel actually wrote. Userspace can use
112 * this to detect what structure members in the response the kernel
115 static u32
uverbs_response_length(struct uverbs_attr_bundle
*attrs
,
118 return min_t(size_t, attrs
->ucore
.outlen
, resp_len
);
122 * The iterator version of the request interface is for handlers that need to
123 * step over a flex array at the end of a command header.
125 struct uverbs_req_iter
{
126 const void __user
*cur
;
127 const void __user
*end
;
130 static int uverbs_request_start(struct uverbs_attr_bundle
*attrs
,
131 struct uverbs_req_iter
*iter
,
135 if (attrs
->ucore
.inlen
< req_len
)
138 if (copy_from_user(req
, attrs
->ucore
.inbuf
, req_len
))
141 iter
->cur
= attrs
->ucore
.inbuf
+ req_len
;
142 iter
->end
= attrs
->ucore
.inbuf
+ attrs
->ucore
.inlen
;
146 static int uverbs_request_next(struct uverbs_req_iter
*iter
, void *val
,
149 if (iter
->cur
+ len
> iter
->end
)
152 if (copy_from_user(val
, iter
->cur
, len
))
159 static const void __user
*uverbs_request_next_ptr(struct uverbs_req_iter
*iter
,
162 const void __user
*res
= iter
->cur
;
164 if (iter
->cur
+ len
> iter
->end
)
165 return (void __force __user
*)ERR_PTR(-ENOSPC
);
170 static int uverbs_request_finish(struct uverbs_req_iter
*iter
)
172 if (!ib_is_buffer_cleared(iter
->cur
, iter
->end
- iter
->cur
))
177 static struct ib_uverbs_completion_event_file
*
178 _ib_uverbs_lookup_comp_file(s32 fd
, struct uverbs_attr_bundle
*attrs
)
180 struct ib_uobject
*uobj
= ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL
,
186 uverbs_uobject_get(uobj
);
189 return container_of(uobj
, struct ib_uverbs_completion_event_file
,
192 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \
193 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
195 static int ib_uverbs_get_context(struct uverbs_attr_bundle
*attrs
)
197 struct ib_uverbs_file
*file
= attrs
->ufile
;
198 struct ib_uverbs_get_context cmd
;
199 struct ib_uverbs_get_context_resp resp
;
200 struct ib_ucontext
*ucontext
;
202 struct ib_rdmacg_object cg_obj
;
203 struct ib_device
*ib_dev
;
206 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
210 mutex_lock(&file
->ucontext_lock
);
211 ib_dev
= srcu_dereference(file
->device
->ib_dev
,
212 &file
->device
->disassociate_srcu
);
218 if (file
->ucontext
) {
223 ret
= ib_rdmacg_try_charge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
227 ucontext
= rdma_zalloc_drv_obj(ib_dev
, ib_ucontext
);
233 ucontext
->res
.type
= RDMA_RESTRACK_CTX
;
234 ucontext
->device
= ib_dev
;
235 ucontext
->cg_obj
= cg_obj
;
236 /* ufile is required when some objects are released */
237 ucontext
->ufile
= file
;
239 ucontext
->closing
= false;
240 ucontext
->cleanup_retryable
= false;
242 mutex_init(&ucontext
->per_mm_list_lock
);
243 INIT_LIST_HEAD(&ucontext
->per_mm_list
);
245 ret
= get_unused_fd_flags(O_CLOEXEC
);
250 filp
= ib_uverbs_alloc_async_event_file(file
, ib_dev
);
256 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
258 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
262 ret
= ib_dev
->ops
.alloc_ucontext(ucontext
, &attrs
->driver_udata
);
265 if (!(ib_dev
->attrs
.device_cap_flags
& IB_DEVICE_ON_DEMAND_PAGING
))
266 ucontext
->invalidate_range
= NULL
;
268 rdma_restrack_uadd(&ucontext
->res
);
270 fd_install(resp
.async_fd
, filp
);
273 * Make sure that ib_uverbs_get_ucontext() sees the pointer update
274 * only after all writes to setup the ucontext have completed
276 smp_store_release(&file
->ucontext
, ucontext
);
278 mutex_unlock(&file
->ucontext_lock
);
283 ib_uverbs_free_async_event_file(file
);
287 put_unused_fd(resp
.async_fd
);
293 ib_rdmacg_uncharge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
296 mutex_unlock(&file
->ucontext_lock
);
300 static void copy_query_dev_fields(struct ib_ucontext
*ucontext
,
301 struct ib_uverbs_query_device_resp
*resp
,
302 struct ib_device_attr
*attr
)
304 struct ib_device
*ib_dev
= ucontext
->device
;
306 resp
->fw_ver
= attr
->fw_ver
;
307 resp
->node_guid
= ib_dev
->node_guid
;
308 resp
->sys_image_guid
= attr
->sys_image_guid
;
309 resp
->max_mr_size
= attr
->max_mr_size
;
310 resp
->page_size_cap
= attr
->page_size_cap
;
311 resp
->vendor_id
= attr
->vendor_id
;
312 resp
->vendor_part_id
= attr
->vendor_part_id
;
313 resp
->hw_ver
= attr
->hw_ver
;
314 resp
->max_qp
= attr
->max_qp
;
315 resp
->max_qp_wr
= attr
->max_qp_wr
;
316 resp
->device_cap_flags
= lower_32_bits(attr
->device_cap_flags
);
317 resp
->max_sge
= min(attr
->max_send_sge
, attr
->max_recv_sge
);
318 resp
->max_sge_rd
= attr
->max_sge_rd
;
319 resp
->max_cq
= attr
->max_cq
;
320 resp
->max_cqe
= attr
->max_cqe
;
321 resp
->max_mr
= attr
->max_mr
;
322 resp
->max_pd
= attr
->max_pd
;
323 resp
->max_qp_rd_atom
= attr
->max_qp_rd_atom
;
324 resp
->max_ee_rd_atom
= attr
->max_ee_rd_atom
;
325 resp
->max_res_rd_atom
= attr
->max_res_rd_atom
;
326 resp
->max_qp_init_rd_atom
= attr
->max_qp_init_rd_atom
;
327 resp
->max_ee_init_rd_atom
= attr
->max_ee_init_rd_atom
;
328 resp
->atomic_cap
= attr
->atomic_cap
;
329 resp
->max_ee
= attr
->max_ee
;
330 resp
->max_rdd
= attr
->max_rdd
;
331 resp
->max_mw
= attr
->max_mw
;
332 resp
->max_raw_ipv6_qp
= attr
->max_raw_ipv6_qp
;
333 resp
->max_raw_ethy_qp
= attr
->max_raw_ethy_qp
;
334 resp
->max_mcast_grp
= attr
->max_mcast_grp
;
335 resp
->max_mcast_qp_attach
= attr
->max_mcast_qp_attach
;
336 resp
->max_total_mcast_qp_attach
= attr
->max_total_mcast_qp_attach
;
337 resp
->max_ah
= attr
->max_ah
;
338 resp
->max_fmr
= attr
->max_fmr
;
339 resp
->max_map_per_fmr
= attr
->max_map_per_fmr
;
340 resp
->max_srq
= attr
->max_srq
;
341 resp
->max_srq_wr
= attr
->max_srq_wr
;
342 resp
->max_srq_sge
= attr
->max_srq_sge
;
343 resp
->max_pkeys
= attr
->max_pkeys
;
344 resp
->local_ca_ack_delay
= attr
->local_ca_ack_delay
;
345 resp
->phys_port_cnt
= ib_dev
->phys_port_cnt
;
348 static int ib_uverbs_query_device(struct uverbs_attr_bundle
*attrs
)
350 struct ib_uverbs_query_device cmd
;
351 struct ib_uverbs_query_device_resp resp
;
352 struct ib_ucontext
*ucontext
;
355 ucontext
= ib_uverbs_get_ucontext(attrs
);
356 if (IS_ERR(ucontext
))
357 return PTR_ERR(ucontext
);
359 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
363 memset(&resp
, 0, sizeof resp
);
364 copy_query_dev_fields(ucontext
, &resp
, &ucontext
->device
->attrs
);
366 return uverbs_response(attrs
, &resp
, sizeof(resp
));
369 static int ib_uverbs_query_port(struct uverbs_attr_bundle
*attrs
)
371 struct ib_uverbs_query_port cmd
;
372 struct ib_uverbs_query_port_resp resp
;
373 struct ib_port_attr attr
;
375 struct ib_ucontext
*ucontext
;
376 struct ib_device
*ib_dev
;
378 ucontext
= ib_uverbs_get_ucontext(attrs
);
379 if (IS_ERR(ucontext
))
380 return PTR_ERR(ucontext
);
381 ib_dev
= ucontext
->device
;
383 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
387 ret
= ib_query_port(ib_dev
, cmd
.port_num
, &attr
);
391 memset(&resp
, 0, sizeof resp
);
392 copy_port_attr_to_resp(&attr
, &resp
, ib_dev
, cmd
.port_num
);
394 return uverbs_response(attrs
, &resp
, sizeof(resp
));
397 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle
*attrs
)
399 struct ib_uverbs_alloc_pd cmd
;
400 struct ib_uverbs_alloc_pd_resp resp
;
401 struct ib_uobject
*uobj
;
404 struct ib_device
*ib_dev
;
406 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
410 uobj
= uobj_alloc(UVERBS_OBJECT_PD
, attrs
, &ib_dev
);
412 return PTR_ERR(uobj
);
414 pd
= rdma_zalloc_drv_obj(ib_dev
, ib_pd
);
422 pd
->__internal_mr
= NULL
;
423 atomic_set(&pd
->usecnt
, 0);
424 pd
->res
.type
= RDMA_RESTRACK_PD
;
426 ret
= ib_dev
->ops
.alloc_pd(pd
, uobj
->context
, &attrs
->driver_udata
);
431 memset(&resp
, 0, sizeof resp
);
432 resp
.pd_handle
= uobj
->id
;
433 rdma_restrack_uadd(&pd
->res
);
435 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
439 return uobj_alloc_commit(uobj
, attrs
);
442 ib_dealloc_pd_user(pd
, &attrs
->driver_udata
);
447 uobj_alloc_abort(uobj
, attrs
);
451 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle
*attrs
)
453 struct ib_uverbs_dealloc_pd cmd
;
456 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
460 return uobj_perform_destroy(UVERBS_OBJECT_PD
, cmd
.pd_handle
, attrs
);
463 struct xrcd_table_entry
{
465 struct ib_xrcd
*xrcd
;
469 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
471 struct ib_xrcd
*xrcd
)
473 struct xrcd_table_entry
*entry
, *scan
;
474 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
475 struct rb_node
*parent
= NULL
;
477 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
482 entry
->inode
= inode
;
486 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
488 if (inode
< scan
->inode
) {
490 } else if (inode
> scan
->inode
) {
498 rb_link_node(&entry
->node
, parent
, p
);
499 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
504 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
507 struct xrcd_table_entry
*entry
;
508 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
511 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
513 if (inode
< entry
->inode
)
515 else if (inode
> entry
->inode
)
524 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
526 struct xrcd_table_entry
*entry
;
528 entry
= xrcd_table_search(dev
, inode
);
535 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
538 struct xrcd_table_entry
*entry
;
540 entry
= xrcd_table_search(dev
, inode
);
543 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
548 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle
*attrs
)
550 struct ib_uverbs_device
*ibudev
= attrs
->ufile
->device
;
551 struct ib_uverbs_open_xrcd cmd
;
552 struct ib_uverbs_open_xrcd_resp resp
;
553 struct ib_uxrcd_object
*obj
;
554 struct ib_xrcd
*xrcd
= NULL
;
555 struct fd f
= {NULL
, 0};
556 struct inode
*inode
= NULL
;
559 struct ib_device
*ib_dev
;
561 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
565 mutex_lock(&ibudev
->xrcd_tree_mutex
);
568 /* search for file descriptor */
572 goto err_tree_mutex_unlock
;
575 inode
= file_inode(f
.file
);
576 xrcd
= find_xrcd(ibudev
, inode
);
577 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
578 /* no file descriptor. Need CREATE flag */
580 goto err_tree_mutex_unlock
;
583 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
585 goto err_tree_mutex_unlock
;
589 obj
= (struct ib_uxrcd_object
*)uobj_alloc(UVERBS_OBJECT_XRCD
, attrs
,
593 goto err_tree_mutex_unlock
;
597 xrcd
= ib_dev
->ops
.alloc_xrcd(ib_dev
, obj
->uobject
.context
,
598 &attrs
->driver_udata
);
605 xrcd
->device
= ib_dev
;
606 atomic_set(&xrcd
->usecnt
, 0);
607 mutex_init(&xrcd
->tgt_qp_mutex
);
608 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
612 atomic_set(&obj
->refcnt
, 0);
613 obj
->uobject
.object
= xrcd
;
614 memset(&resp
, 0, sizeof resp
);
615 resp
.xrcd_handle
= obj
->uobject
.id
;
619 /* create new inode/xrcd table entry */
620 ret
= xrcd_table_insert(ibudev
, inode
, xrcd
);
622 goto err_dealloc_xrcd
;
624 atomic_inc(&xrcd
->usecnt
);
627 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
634 mutex_unlock(&ibudev
->xrcd_tree_mutex
);
636 return uobj_alloc_commit(&obj
->uobject
, attrs
);
641 xrcd_table_delete(ibudev
, inode
);
642 atomic_dec(&xrcd
->usecnt
);
646 ib_dealloc_xrcd(xrcd
, &attrs
->driver_udata
);
649 uobj_alloc_abort(&obj
->uobject
, attrs
);
651 err_tree_mutex_unlock
:
655 mutex_unlock(&ibudev
->xrcd_tree_mutex
);
660 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle
*attrs
)
662 struct ib_uverbs_close_xrcd cmd
;
665 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
669 return uobj_perform_destroy(UVERBS_OBJECT_XRCD
, cmd
.xrcd_handle
, attrs
);
672 int ib_uverbs_dealloc_xrcd(struct ib_uobject
*uobject
, struct ib_xrcd
*xrcd
,
673 enum rdma_remove_reason why
, struct ib_udata
*udata
)
677 struct ib_uverbs_device
*dev
= uobject
->context
->ufile
->device
;
680 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
683 ret
= ib_dealloc_xrcd(xrcd
, udata
);
685 if (ib_is_destroy_retryable(ret
, why
, uobject
)) {
686 atomic_inc(&xrcd
->usecnt
);
691 xrcd_table_delete(dev
, inode
);
696 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle
*attrs
)
698 struct ib_uverbs_reg_mr cmd
;
699 struct ib_uverbs_reg_mr_resp resp
;
700 struct ib_uobject
*uobj
;
704 struct ib_device
*ib_dev
;
706 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
710 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
713 ret
= ib_check_mr_access(cmd
.access_flags
);
717 uobj
= uobj_alloc(UVERBS_OBJECT_MR
, attrs
, &ib_dev
);
719 return PTR_ERR(uobj
);
721 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
.pd_handle
, attrs
);
727 if (cmd
.access_flags
& IB_ACCESS_ON_DEMAND
) {
728 if (!(pd
->device
->attrs
.device_cap_flags
&
729 IB_DEVICE_ON_DEMAND_PAGING
)) {
730 pr_debug("ODP support not available\n");
736 mr
= pd
->device
->ops
.reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
738 &attrs
->driver_udata
);
744 mr
->device
= pd
->device
;
748 atomic_inc(&pd
->usecnt
);
749 mr
->res
.type
= RDMA_RESTRACK_MR
;
750 rdma_restrack_uadd(&mr
->res
);
754 memset(&resp
, 0, sizeof resp
);
755 resp
.lkey
= mr
->lkey
;
756 resp
.rkey
= mr
->rkey
;
757 resp
.mr_handle
= uobj
->id
;
759 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
763 uobj_put_obj_read(pd
);
765 return uobj_alloc_commit(uobj
, attrs
);
768 ib_dereg_mr_user(mr
, &attrs
->driver_udata
);
771 uobj_put_obj_read(pd
);
774 uobj_alloc_abort(uobj
, attrs
);
778 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle
*attrs
)
780 struct ib_uverbs_rereg_mr cmd
;
781 struct ib_uverbs_rereg_mr_resp resp
;
782 struct ib_pd
*pd
= NULL
;
784 struct ib_pd
*old_pd
;
786 struct ib_uobject
*uobj
;
788 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
792 if (cmd
.flags
& ~IB_MR_REREG_SUPPORTED
|| !cmd
.flags
)
795 if ((cmd
.flags
& IB_MR_REREG_TRANS
) &&
796 (!cmd
.start
|| !cmd
.hca_va
|| 0 >= cmd
.length
||
797 (cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
)))
800 uobj
= uobj_get_write(UVERBS_OBJECT_MR
, cmd
.mr_handle
, attrs
);
802 return PTR_ERR(uobj
);
811 if (cmd
.flags
& IB_MR_REREG_ACCESS
) {
812 ret
= ib_check_mr_access(cmd
.access_flags
);
817 if (cmd
.flags
& IB_MR_REREG_PD
) {
818 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
.pd_handle
,
827 ret
= mr
->device
->ops
.rereg_user_mr(mr
, cmd
.flags
, cmd
.start
,
828 cmd
.length
, cmd
.hca_va
,
829 cmd
.access_flags
, pd
,
830 &attrs
->driver_udata
);
834 if (cmd
.flags
& IB_MR_REREG_PD
) {
835 atomic_inc(&pd
->usecnt
);
837 atomic_dec(&old_pd
->usecnt
);
840 memset(&resp
, 0, sizeof(resp
));
841 resp
.lkey
= mr
->lkey
;
842 resp
.rkey
= mr
->rkey
;
844 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
847 if (cmd
.flags
& IB_MR_REREG_PD
)
848 uobj_put_obj_read(pd
);
851 uobj_put_write(uobj
);
856 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle
*attrs
)
858 struct ib_uverbs_dereg_mr cmd
;
861 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
865 return uobj_perform_destroy(UVERBS_OBJECT_MR
, cmd
.mr_handle
, attrs
);
868 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle
*attrs
)
870 struct ib_uverbs_alloc_mw cmd
;
871 struct ib_uverbs_alloc_mw_resp resp
;
872 struct ib_uobject
*uobj
;
876 struct ib_device
*ib_dev
;
878 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
882 uobj
= uobj_alloc(UVERBS_OBJECT_MW
, attrs
, &ib_dev
);
884 return PTR_ERR(uobj
);
886 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
.pd_handle
, attrs
);
892 if (cmd
.mw_type
!= IB_MW_TYPE_1
&& cmd
.mw_type
!= IB_MW_TYPE_2
) {
897 mw
= pd
->device
->ops
.alloc_mw(pd
, cmd
.mw_type
, &attrs
->driver_udata
);
903 mw
->device
= pd
->device
;
906 atomic_inc(&pd
->usecnt
);
910 memset(&resp
, 0, sizeof(resp
));
911 resp
.rkey
= mw
->rkey
;
912 resp
.mw_handle
= uobj
->id
;
914 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
918 uobj_put_obj_read(pd
);
919 return uobj_alloc_commit(uobj
, attrs
);
922 uverbs_dealloc_mw(mw
);
924 uobj_put_obj_read(pd
);
926 uobj_alloc_abort(uobj
, attrs
);
930 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle
*attrs
)
932 struct ib_uverbs_dealloc_mw cmd
;
935 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
939 return uobj_perform_destroy(UVERBS_OBJECT_MW
, cmd
.mw_handle
, attrs
);
942 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle
*attrs
)
944 struct ib_uverbs_create_comp_channel cmd
;
945 struct ib_uverbs_create_comp_channel_resp resp
;
946 struct ib_uobject
*uobj
;
947 struct ib_uverbs_completion_event_file
*ev_file
;
948 struct ib_device
*ib_dev
;
951 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
955 uobj
= uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL
, attrs
, &ib_dev
);
957 return PTR_ERR(uobj
);
961 ev_file
= container_of(uobj
, struct ib_uverbs_completion_event_file
,
963 ib_uverbs_init_event_queue(&ev_file
->ev_queue
);
965 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
967 uobj_alloc_abort(uobj
, attrs
);
971 return uobj_alloc_commit(uobj
, attrs
);
974 static struct ib_ucq_object
*create_cq(struct uverbs_attr_bundle
*attrs
,
975 struct ib_uverbs_ex_create_cq
*cmd
)
977 struct ib_ucq_object
*obj
;
978 struct ib_uverbs_completion_event_file
*ev_file
= NULL
;
981 struct ib_uverbs_ex_create_cq_resp resp
;
982 struct ib_cq_init_attr attr
= {};
983 struct ib_device
*ib_dev
;
985 if (cmd
->comp_vector
>= attrs
->ufile
->device
->num_comp_vectors
)
986 return ERR_PTR(-EINVAL
);
988 obj
= (struct ib_ucq_object
*)uobj_alloc(UVERBS_OBJECT_CQ
, attrs
,
993 if (cmd
->comp_channel
>= 0) {
994 ev_file
= ib_uverbs_lookup_comp_file(cmd
->comp_channel
, attrs
);
995 if (IS_ERR(ev_file
)) {
996 ret
= PTR_ERR(ev_file
);
1001 obj
->uobject
.user_handle
= cmd
->user_handle
;
1002 obj
->comp_events_reported
= 0;
1003 obj
->async_events_reported
= 0;
1004 INIT_LIST_HEAD(&obj
->comp_list
);
1005 INIT_LIST_HEAD(&obj
->async_list
);
1007 attr
.cqe
= cmd
->cqe
;
1008 attr
.comp_vector
= cmd
->comp_vector
;
1009 attr
.flags
= cmd
->flags
;
1011 cq
= ib_dev
->ops
.create_cq(ib_dev
, &attr
, obj
->uobject
.context
,
1012 &attrs
->driver_udata
);
1018 cq
->device
= ib_dev
;
1019 cq
->uobject
= &obj
->uobject
;
1020 cq
->comp_handler
= ib_uverbs_comp_handler
;
1021 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1022 cq
->cq_context
= ev_file
? &ev_file
->ev_queue
: NULL
;
1023 atomic_set(&cq
->usecnt
, 0);
1025 obj
->uobject
.object
= cq
;
1026 memset(&resp
, 0, sizeof resp
);
1027 resp
.base
.cq_handle
= obj
->uobject
.id
;
1028 resp
.base
.cqe
= cq
->cqe
;
1029 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
1031 cq
->res
.type
= RDMA_RESTRACK_CQ
;
1032 rdma_restrack_uadd(&cq
->res
);
1034 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
1038 ret
= uobj_alloc_commit(&obj
->uobject
, attrs
);
1040 return ERR_PTR(ret
);
1048 ib_uverbs_release_ucq(attrs
->ufile
, ev_file
, obj
);
1051 uobj_alloc_abort(&obj
->uobject
, attrs
);
1053 return ERR_PTR(ret
);
1056 static int ib_uverbs_create_cq(struct uverbs_attr_bundle
*attrs
)
1058 struct ib_uverbs_create_cq cmd
;
1059 struct ib_uverbs_ex_create_cq cmd_ex
;
1060 struct ib_ucq_object
*obj
;
1063 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1067 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1068 cmd_ex
.user_handle
= cmd
.user_handle
;
1069 cmd_ex
.cqe
= cmd
.cqe
;
1070 cmd_ex
.comp_vector
= cmd
.comp_vector
;
1071 cmd_ex
.comp_channel
= cmd
.comp_channel
;
1073 obj
= create_cq(attrs
, &cmd_ex
);
1074 return PTR_ERR_OR_ZERO(obj
);
1077 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle
*attrs
)
1079 struct ib_uverbs_ex_create_cq cmd
;
1080 struct ib_ucq_object
*obj
;
1083 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1093 obj
= create_cq(attrs
, &cmd
);
1094 return PTR_ERR_OR_ZERO(obj
);
1097 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle
*attrs
)
1099 struct ib_uverbs_resize_cq cmd
;
1100 struct ib_uverbs_resize_cq_resp resp
= {};
1104 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1108 cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
1112 ret
= cq
->device
->ops
.resize_cq(cq
, cmd
.cqe
, &attrs
->driver_udata
);
1118 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
1120 uobj_put_obj_read(cq
);
1125 static int copy_wc_to_user(struct ib_device
*ib_dev
, void __user
*dest
,
1128 struct ib_uverbs_wc tmp
;
1130 tmp
.wr_id
= wc
->wr_id
;
1131 tmp
.status
= wc
->status
;
1132 tmp
.opcode
= wc
->opcode
;
1133 tmp
.vendor_err
= wc
->vendor_err
;
1134 tmp
.byte_len
= wc
->byte_len
;
1135 tmp
.ex
.imm_data
= wc
->ex
.imm_data
;
1136 tmp
.qp_num
= wc
->qp
->qp_num
;
1137 tmp
.src_qp
= wc
->src_qp
;
1138 tmp
.wc_flags
= wc
->wc_flags
;
1139 tmp
.pkey_index
= wc
->pkey_index
;
1140 if (rdma_cap_opa_ah(ib_dev
, wc
->port_num
))
1141 tmp
.slid
= OPA_TO_IB_UCAST_LID(wc
->slid
);
1143 tmp
.slid
= ib_lid_cpu16(wc
->slid
);
1145 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1146 tmp
.port_num
= wc
->port_num
;
1149 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1155 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle
*attrs
)
1157 struct ib_uverbs_poll_cq cmd
;
1158 struct ib_uverbs_poll_cq_resp resp
;
1159 u8 __user
*header_ptr
;
1160 u8 __user
*data_ptr
;
1165 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1169 cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
1173 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1174 header_ptr
= attrs
->ucore
.outbuf
;
1175 data_ptr
= header_ptr
+ sizeof resp
;
1177 memset(&resp
, 0, sizeof resp
);
1178 while (resp
.count
< cmd
.ne
) {
1179 ret
= ib_poll_cq(cq
, 1, &wc
);
1185 ret
= copy_wc_to_user(cq
->device
, data_ptr
, &wc
);
1189 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1193 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1199 if (uverbs_attr_is_valid(attrs
, UVERBS_ATTR_CORE_OUT
))
1200 ret
= uverbs_output_written(attrs
, UVERBS_ATTR_CORE_OUT
);
1203 uobj_put_obj_read(cq
);
1207 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle
*attrs
)
1209 struct ib_uverbs_req_notify_cq cmd
;
1213 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1217 cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
1221 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1222 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1224 uobj_put_obj_read(cq
);
1229 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle
*attrs
)
1231 struct ib_uverbs_destroy_cq cmd
;
1232 struct ib_uverbs_destroy_cq_resp resp
;
1233 struct ib_uobject
*uobj
;
1234 struct ib_ucq_object
*obj
;
1237 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1241 uobj
= uobj_get_destroy(UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
1243 return PTR_ERR(uobj
);
1245 obj
= container_of(uobj
, struct ib_ucq_object
, uobject
);
1246 memset(&resp
, 0, sizeof(resp
));
1247 resp
.comp_events_reported
= obj
->comp_events_reported
;
1248 resp
.async_events_reported
= obj
->async_events_reported
;
1250 uobj_put_destroy(uobj
);
1252 return uverbs_response(attrs
, &resp
, sizeof(resp
));
1255 static int create_qp(struct uverbs_attr_bundle
*attrs
,
1256 struct ib_uverbs_ex_create_qp
*cmd
)
1258 struct ib_uqp_object
*obj
;
1259 struct ib_device
*device
;
1260 struct ib_pd
*pd
= NULL
;
1261 struct ib_xrcd
*xrcd
= NULL
;
1262 struct ib_uobject
*xrcd_uobj
= ERR_PTR(-ENOENT
);
1263 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1264 struct ib_srq
*srq
= NULL
;
1266 struct ib_qp_init_attr attr
= {};
1267 struct ib_uverbs_ex_create_qp_resp resp
;
1269 struct ib_rwq_ind_table
*ind_tbl
= NULL
;
1271 struct ib_device
*ib_dev
;
1273 if (cmd
->qp_type
== IB_QPT_RAW_PACKET
&& !capable(CAP_NET_RAW
))
1276 obj
= (struct ib_uqp_object
*)uobj_alloc(UVERBS_OBJECT_QP
, attrs
,
1279 return PTR_ERR(obj
);
1281 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
1282 mutex_init(&obj
->mcast_lock
);
1284 if (cmd
->comp_mask
& IB_UVERBS_CREATE_QP_MASK_IND_TABLE
) {
1285 ind_tbl
= uobj_get_obj_read(rwq_ind_table
,
1286 UVERBS_OBJECT_RWQ_IND_TBL
,
1287 cmd
->rwq_ind_tbl_handle
, attrs
);
1293 attr
.rwq_ind_tbl
= ind_tbl
;
1296 if (ind_tbl
&& (cmd
->max_recv_wr
|| cmd
->max_recv_sge
|| cmd
->is_srq
)) {
1301 if (ind_tbl
&& !cmd
->max_send_wr
)
1304 if (cmd
->qp_type
== IB_QPT_XRC_TGT
) {
1305 xrcd_uobj
= uobj_get_read(UVERBS_OBJECT_XRCD
, cmd
->pd_handle
,
1308 if (IS_ERR(xrcd_uobj
)) {
1313 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1318 device
= xrcd
->device
;
1320 if (cmd
->qp_type
== IB_QPT_XRC_INI
) {
1321 cmd
->max_recv_wr
= 0;
1322 cmd
->max_recv_sge
= 0;
1325 srq
= uobj_get_obj_read(srq
, UVERBS_OBJECT_SRQ
,
1326 cmd
->srq_handle
, attrs
);
1327 if (!srq
|| srq
->srq_type
== IB_SRQT_XRC
) {
1334 if (cmd
->recv_cq_handle
!= cmd
->send_cq_handle
) {
1335 rcq
= uobj_get_obj_read(
1336 cq
, UVERBS_OBJECT_CQ
,
1337 cmd
->recv_cq_handle
, attrs
);
1347 scq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
,
1348 cmd
->send_cq_handle
, attrs
);
1351 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
->pd_handle
,
1353 if (!pd
|| (!scq
&& has_sq
)) {
1358 device
= pd
->device
;
1361 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1362 attr
.qp_context
= attrs
->ufile
;
1367 attr
.sq_sig_type
= cmd
->sq_sig_all
? IB_SIGNAL_ALL_WR
:
1369 attr
.qp_type
= cmd
->qp_type
;
1370 attr
.create_flags
= 0;
1372 attr
.cap
.max_send_wr
= cmd
->max_send_wr
;
1373 attr
.cap
.max_recv_wr
= cmd
->max_recv_wr
;
1374 attr
.cap
.max_send_sge
= cmd
->max_send_sge
;
1375 attr
.cap
.max_recv_sge
= cmd
->max_recv_sge
;
1376 attr
.cap
.max_inline_data
= cmd
->max_inline_data
;
1378 obj
->uevent
.events_reported
= 0;
1379 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1380 INIT_LIST_HEAD(&obj
->mcast_list
);
1382 attr
.create_flags
= cmd
->create_flags
;
1383 if (attr
.create_flags
& ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
|
1384 IB_QP_CREATE_CROSS_CHANNEL
|
1385 IB_QP_CREATE_MANAGED_SEND
|
1386 IB_QP_CREATE_MANAGED_RECV
|
1387 IB_QP_CREATE_SCATTER_FCS
|
1388 IB_QP_CREATE_CVLAN_STRIPPING
|
1389 IB_QP_CREATE_SOURCE_QPN
|
1390 IB_QP_CREATE_PCI_WRITE_END_PADDING
)) {
1395 if (attr
.create_flags
& IB_QP_CREATE_SOURCE_QPN
) {
1396 if (!capable(CAP_NET_RAW
)) {
1401 attr
.source_qpn
= cmd
->source_qpn
;
1404 if (cmd
->qp_type
== IB_QPT_XRC_TGT
)
1405 qp
= ib_create_qp(pd
, &attr
);
1407 qp
= _ib_create_qp(device
, pd
, &attr
, &attrs
->driver_udata
,
1408 &obj
->uevent
.uobject
);
1415 if (cmd
->qp_type
!= IB_QPT_XRC_TGT
) {
1416 ret
= ib_create_qp_security(qp
, device
);
1422 qp
->send_cq
= attr
.send_cq
;
1423 qp
->recv_cq
= attr
.recv_cq
;
1425 qp
->rwq_ind_tbl
= ind_tbl
;
1426 qp
->event_handler
= attr
.event_handler
;
1427 qp
->qp_context
= attr
.qp_context
;
1428 qp
->qp_type
= attr
.qp_type
;
1429 atomic_set(&qp
->usecnt
, 0);
1430 atomic_inc(&pd
->usecnt
);
1433 atomic_inc(&attr
.send_cq
->usecnt
);
1435 atomic_inc(&attr
.recv_cq
->usecnt
);
1437 atomic_inc(&attr
.srq
->usecnt
);
1439 atomic_inc(&ind_tbl
->usecnt
);
1441 /* It is done in _ib_create_qp for other QP types */
1442 qp
->uobject
= &obj
->uevent
.uobject
;
1445 obj
->uevent
.uobject
.object
= qp
;
1447 memset(&resp
, 0, sizeof resp
);
1448 resp
.base
.qpn
= qp
->qp_num
;
1449 resp
.base
.qp_handle
= obj
->uevent
.uobject
.id
;
1450 resp
.base
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1451 resp
.base
.max_send_sge
= attr
.cap
.max_send_sge
;
1452 resp
.base
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1453 resp
.base
.max_send_wr
= attr
.cap
.max_send_wr
;
1454 resp
.base
.max_inline_data
= attr
.cap
.max_inline_data
;
1455 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
1457 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
1462 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
,
1464 atomic_inc(&obj
->uxrcd
->refcnt
);
1465 uobj_put_read(xrcd_uobj
);
1469 uobj_put_obj_read(pd
);
1471 uobj_put_obj_read(scq
);
1472 if (rcq
&& rcq
!= scq
)
1473 uobj_put_obj_read(rcq
);
1475 uobj_put_obj_read(srq
);
1477 uobj_put_obj_read(ind_tbl
);
1479 return uobj_alloc_commit(&obj
->uevent
.uobject
, attrs
);
1484 if (!IS_ERR(xrcd_uobj
))
1485 uobj_put_read(xrcd_uobj
);
1487 uobj_put_obj_read(pd
);
1489 uobj_put_obj_read(scq
);
1490 if (rcq
&& rcq
!= scq
)
1491 uobj_put_obj_read(rcq
);
1493 uobj_put_obj_read(srq
);
1495 uobj_put_obj_read(ind_tbl
);
1497 uobj_alloc_abort(&obj
->uevent
.uobject
, attrs
);
1501 static int ib_uverbs_create_qp(struct uverbs_attr_bundle
*attrs
)
1503 struct ib_uverbs_create_qp cmd
;
1504 struct ib_uverbs_ex_create_qp cmd_ex
;
1507 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1511 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1512 cmd_ex
.user_handle
= cmd
.user_handle
;
1513 cmd_ex
.pd_handle
= cmd
.pd_handle
;
1514 cmd_ex
.send_cq_handle
= cmd
.send_cq_handle
;
1515 cmd_ex
.recv_cq_handle
= cmd
.recv_cq_handle
;
1516 cmd_ex
.srq_handle
= cmd
.srq_handle
;
1517 cmd_ex
.max_send_wr
= cmd
.max_send_wr
;
1518 cmd_ex
.max_recv_wr
= cmd
.max_recv_wr
;
1519 cmd_ex
.max_send_sge
= cmd
.max_send_sge
;
1520 cmd_ex
.max_recv_sge
= cmd
.max_recv_sge
;
1521 cmd_ex
.max_inline_data
= cmd
.max_inline_data
;
1522 cmd_ex
.sq_sig_all
= cmd
.sq_sig_all
;
1523 cmd_ex
.qp_type
= cmd
.qp_type
;
1524 cmd_ex
.is_srq
= cmd
.is_srq
;
1526 return create_qp(attrs
, &cmd_ex
);
1529 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle
*attrs
)
1531 struct ib_uverbs_ex_create_qp cmd
;
1534 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1538 if (cmd
.comp_mask
& ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK
)
1544 return create_qp(attrs
, &cmd
);
1547 static int ib_uverbs_open_qp(struct uverbs_attr_bundle
*attrs
)
1549 struct ib_uverbs_open_qp cmd
;
1550 struct ib_uverbs_create_qp_resp resp
;
1551 struct ib_uqp_object
*obj
;
1552 struct ib_xrcd
*xrcd
;
1553 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1555 struct ib_qp_open_attr attr
;
1557 struct ib_device
*ib_dev
;
1559 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1563 obj
= (struct ib_uqp_object
*)uobj_alloc(UVERBS_OBJECT_QP
, attrs
,
1566 return PTR_ERR(obj
);
1568 xrcd_uobj
= uobj_get_read(UVERBS_OBJECT_XRCD
, cmd
.pd_handle
, attrs
);
1569 if (IS_ERR(xrcd_uobj
)) {
1574 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1580 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1581 attr
.qp_context
= attrs
->ufile
;
1582 attr
.qp_num
= cmd
.qpn
;
1583 attr
.qp_type
= cmd
.qp_type
;
1585 obj
->uevent
.events_reported
= 0;
1586 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1587 INIT_LIST_HEAD(&obj
->mcast_list
);
1589 qp
= ib_open_qp(xrcd
, &attr
);
1595 obj
->uevent
.uobject
.object
= qp
;
1596 obj
->uevent
.uobject
.user_handle
= cmd
.user_handle
;
1598 memset(&resp
, 0, sizeof resp
);
1599 resp
.qpn
= qp
->qp_num
;
1600 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1602 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
1606 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
1607 atomic_inc(&obj
->uxrcd
->refcnt
);
1608 qp
->uobject
= &obj
->uevent
.uobject
;
1609 uobj_put_read(xrcd_uobj
);
1611 return uobj_alloc_commit(&obj
->uevent
.uobject
, attrs
);
1616 uobj_put_read(xrcd_uobj
);
1618 uobj_alloc_abort(&obj
->uevent
.uobject
, attrs
);
1622 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest
*uverb_attr
,
1623 struct rdma_ah_attr
*rdma_attr
)
1625 const struct ib_global_route
*grh
;
1627 uverb_attr
->dlid
= rdma_ah_get_dlid(rdma_attr
);
1628 uverb_attr
->sl
= rdma_ah_get_sl(rdma_attr
);
1629 uverb_attr
->src_path_bits
= rdma_ah_get_path_bits(rdma_attr
);
1630 uverb_attr
->static_rate
= rdma_ah_get_static_rate(rdma_attr
);
1631 uverb_attr
->is_global
= !!(rdma_ah_get_ah_flags(rdma_attr
) &
1633 if (uverb_attr
->is_global
) {
1634 grh
= rdma_ah_read_grh(rdma_attr
);
1635 memcpy(uverb_attr
->dgid
, grh
->dgid
.raw
, 16);
1636 uverb_attr
->flow_label
= grh
->flow_label
;
1637 uverb_attr
->sgid_index
= grh
->sgid_index
;
1638 uverb_attr
->hop_limit
= grh
->hop_limit
;
1639 uverb_attr
->traffic_class
= grh
->traffic_class
;
1641 uverb_attr
->port_num
= rdma_ah_get_port_num(rdma_attr
);
1644 static int ib_uverbs_query_qp(struct uverbs_attr_bundle
*attrs
)
1646 struct ib_uverbs_query_qp cmd
;
1647 struct ib_uverbs_query_qp_resp resp
;
1649 struct ib_qp_attr
*attr
;
1650 struct ib_qp_init_attr
*init_attr
;
1653 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1657 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1658 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
1659 if (!attr
|| !init_attr
) {
1664 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
1670 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
1672 uobj_put_obj_read(qp
);
1677 memset(&resp
, 0, sizeof resp
);
1679 resp
.qp_state
= attr
->qp_state
;
1680 resp
.cur_qp_state
= attr
->cur_qp_state
;
1681 resp
.path_mtu
= attr
->path_mtu
;
1682 resp
.path_mig_state
= attr
->path_mig_state
;
1683 resp
.qkey
= attr
->qkey
;
1684 resp
.rq_psn
= attr
->rq_psn
;
1685 resp
.sq_psn
= attr
->sq_psn
;
1686 resp
.dest_qp_num
= attr
->dest_qp_num
;
1687 resp
.qp_access_flags
= attr
->qp_access_flags
;
1688 resp
.pkey_index
= attr
->pkey_index
;
1689 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
1690 resp
.sq_draining
= attr
->sq_draining
;
1691 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
1692 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
1693 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
1694 resp
.port_num
= attr
->port_num
;
1695 resp
.timeout
= attr
->timeout
;
1696 resp
.retry_cnt
= attr
->retry_cnt
;
1697 resp
.rnr_retry
= attr
->rnr_retry
;
1698 resp
.alt_port_num
= attr
->alt_port_num
;
1699 resp
.alt_timeout
= attr
->alt_timeout
;
1701 copy_ah_attr_to_uverbs(&resp
.dest
, &attr
->ah_attr
);
1702 copy_ah_attr_to_uverbs(&resp
.alt_dest
, &attr
->alt_ah_attr
);
1704 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
1705 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
1706 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
1707 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
1708 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
1709 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
1711 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
1720 /* Remove ignored fields set in the attribute mask */
1721 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
1724 case IB_QPT_XRC_INI
:
1725 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
1726 case IB_QPT_XRC_TGT
:
1727 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
1734 static void copy_ah_attr_from_uverbs(struct ib_device
*dev
,
1735 struct rdma_ah_attr
*rdma_attr
,
1736 struct ib_uverbs_qp_dest
*uverb_attr
)
1738 rdma_attr
->type
= rdma_ah_find_type(dev
, uverb_attr
->port_num
);
1739 if (uverb_attr
->is_global
) {
1740 rdma_ah_set_grh(rdma_attr
, NULL
,
1741 uverb_attr
->flow_label
,
1742 uverb_attr
->sgid_index
,
1743 uverb_attr
->hop_limit
,
1744 uverb_attr
->traffic_class
);
1745 rdma_ah_set_dgid_raw(rdma_attr
, uverb_attr
->dgid
);
1747 rdma_ah_set_ah_flags(rdma_attr
, 0);
1749 rdma_ah_set_dlid(rdma_attr
, uverb_attr
->dlid
);
1750 rdma_ah_set_sl(rdma_attr
, uverb_attr
->sl
);
1751 rdma_ah_set_path_bits(rdma_attr
, uverb_attr
->src_path_bits
);
1752 rdma_ah_set_static_rate(rdma_attr
, uverb_attr
->static_rate
);
1753 rdma_ah_set_port_num(rdma_attr
, uverb_attr
->port_num
);
1754 rdma_ah_set_make_grd(rdma_attr
, false);
1757 static int modify_qp(struct uverbs_attr_bundle
*attrs
,
1758 struct ib_uverbs_ex_modify_qp
*cmd
)
1760 struct ib_qp_attr
*attr
;
1764 attr
= kzalloc(sizeof(*attr
), GFP_KERNEL
);
1768 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
->base
.qp_handle
,
1775 if ((cmd
->base
.attr_mask
& IB_QP_PORT
) &&
1776 !rdma_is_port_valid(qp
->device
, cmd
->base
.port_num
)) {
1781 if ((cmd
->base
.attr_mask
& IB_QP_AV
)) {
1782 if (!rdma_is_port_valid(qp
->device
, cmd
->base
.dest
.port_num
)) {
1787 if (cmd
->base
.attr_mask
& IB_QP_STATE
&&
1788 cmd
->base
.qp_state
== IB_QPS_RTR
) {
1789 /* We are in INIT->RTR TRANSITION (if we are not,
1790 * this transition will be rejected in subsequent checks).
1791 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1792 * but the IB_QP_STATE flag is required.
1794 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1795 * when IB_QP_AV is set, has required inclusion of a valid
1796 * port number in the primary AV. (AVs are created and handled
1797 * differently for infiniband and ethernet (RoCE) ports).
1799 * Check the port number included in the primary AV against
1800 * the port number in the qp struct, which was set (and saved)
1801 * in the RST->INIT transition.
1803 if (cmd
->base
.dest
.port_num
!= qp
->real_qp
->port
) {
1808 /* We are in SQD->SQD. (If we are not, this transition will
1809 * be rejected later in the verbs layer checks).
1810 * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1811 * together in the SQD->SQD transition.
1813 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1814 * verbs layer driver does not track primary port changes
1815 * resulting from path migration. Thus, in SQD, if the primary
1816 * AV is modified, the primary port should also be modified).
1818 * Note that in this transition, the IB_QP_STATE flag
1821 if (((cmd
->base
.attr_mask
& (IB_QP_AV
| IB_QP_PORT
))
1822 == (IB_QP_AV
| IB_QP_PORT
)) &&
1823 cmd
->base
.port_num
!= cmd
->base
.dest
.port_num
) {
1827 if ((cmd
->base
.attr_mask
& (IB_QP_AV
| IB_QP_PORT
))
1829 cmd
->base
.attr_mask
|= IB_QP_PORT
;
1830 cmd
->base
.port_num
= cmd
->base
.dest
.port_num
;
1835 if ((cmd
->base
.attr_mask
& IB_QP_ALT_PATH
) &&
1836 (!rdma_is_port_valid(qp
->device
, cmd
->base
.alt_port_num
) ||
1837 !rdma_is_port_valid(qp
->device
, cmd
->base
.alt_dest
.port_num
) ||
1838 cmd
->base
.alt_port_num
!= cmd
->base
.alt_dest
.port_num
)) {
1843 if ((cmd
->base
.attr_mask
& IB_QP_CUR_STATE
&&
1844 cmd
->base
.cur_qp_state
> IB_QPS_ERR
) ||
1845 (cmd
->base
.attr_mask
& IB_QP_STATE
&&
1846 cmd
->base
.qp_state
> IB_QPS_ERR
)) {
1851 if (cmd
->base
.attr_mask
& IB_QP_STATE
)
1852 attr
->qp_state
= cmd
->base
.qp_state
;
1853 if (cmd
->base
.attr_mask
& IB_QP_CUR_STATE
)
1854 attr
->cur_qp_state
= cmd
->base
.cur_qp_state
;
1855 if (cmd
->base
.attr_mask
& IB_QP_PATH_MTU
)
1856 attr
->path_mtu
= cmd
->base
.path_mtu
;
1857 if (cmd
->base
.attr_mask
& IB_QP_PATH_MIG_STATE
)
1858 attr
->path_mig_state
= cmd
->base
.path_mig_state
;
1859 if (cmd
->base
.attr_mask
& IB_QP_QKEY
)
1860 attr
->qkey
= cmd
->base
.qkey
;
1861 if (cmd
->base
.attr_mask
& IB_QP_RQ_PSN
)
1862 attr
->rq_psn
= cmd
->base
.rq_psn
;
1863 if (cmd
->base
.attr_mask
& IB_QP_SQ_PSN
)
1864 attr
->sq_psn
= cmd
->base
.sq_psn
;
1865 if (cmd
->base
.attr_mask
& IB_QP_DEST_QPN
)
1866 attr
->dest_qp_num
= cmd
->base
.dest_qp_num
;
1867 if (cmd
->base
.attr_mask
& IB_QP_ACCESS_FLAGS
)
1868 attr
->qp_access_flags
= cmd
->base
.qp_access_flags
;
1869 if (cmd
->base
.attr_mask
& IB_QP_PKEY_INDEX
)
1870 attr
->pkey_index
= cmd
->base
.pkey_index
;
1871 if (cmd
->base
.attr_mask
& IB_QP_EN_SQD_ASYNC_NOTIFY
)
1872 attr
->en_sqd_async_notify
= cmd
->base
.en_sqd_async_notify
;
1873 if (cmd
->base
.attr_mask
& IB_QP_MAX_QP_RD_ATOMIC
)
1874 attr
->max_rd_atomic
= cmd
->base
.max_rd_atomic
;
1875 if (cmd
->base
.attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
1876 attr
->max_dest_rd_atomic
= cmd
->base
.max_dest_rd_atomic
;
1877 if (cmd
->base
.attr_mask
& IB_QP_MIN_RNR_TIMER
)
1878 attr
->min_rnr_timer
= cmd
->base
.min_rnr_timer
;
1879 if (cmd
->base
.attr_mask
& IB_QP_PORT
)
1880 attr
->port_num
= cmd
->base
.port_num
;
1881 if (cmd
->base
.attr_mask
& IB_QP_TIMEOUT
)
1882 attr
->timeout
= cmd
->base
.timeout
;
1883 if (cmd
->base
.attr_mask
& IB_QP_RETRY_CNT
)
1884 attr
->retry_cnt
= cmd
->base
.retry_cnt
;
1885 if (cmd
->base
.attr_mask
& IB_QP_RNR_RETRY
)
1886 attr
->rnr_retry
= cmd
->base
.rnr_retry
;
1887 if (cmd
->base
.attr_mask
& IB_QP_ALT_PATH
) {
1888 attr
->alt_port_num
= cmd
->base
.alt_port_num
;
1889 attr
->alt_timeout
= cmd
->base
.alt_timeout
;
1890 attr
->alt_pkey_index
= cmd
->base
.alt_pkey_index
;
1892 if (cmd
->base
.attr_mask
& IB_QP_RATE_LIMIT
)
1893 attr
->rate_limit
= cmd
->rate_limit
;
1895 if (cmd
->base
.attr_mask
& IB_QP_AV
)
1896 copy_ah_attr_from_uverbs(qp
->device
, &attr
->ah_attr
,
1899 if (cmd
->base
.attr_mask
& IB_QP_ALT_PATH
)
1900 copy_ah_attr_from_uverbs(qp
->device
, &attr
->alt_ah_attr
,
1901 &cmd
->base
.alt_dest
);
1903 ret
= ib_modify_qp_with_udata(qp
, attr
,
1904 modify_qp_mask(qp
->qp_type
,
1905 cmd
->base
.attr_mask
),
1906 &attrs
->driver_udata
);
1909 uobj_put_obj_read(qp
);
1916 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle
*attrs
)
1918 struct ib_uverbs_ex_modify_qp cmd
;
1921 ret
= uverbs_request(attrs
, &cmd
.base
, sizeof(cmd
.base
));
1925 if (cmd
.base
.attr_mask
&
1926 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK
<< 1) - 1))
1929 return modify_qp(attrs
, &cmd
);
1932 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle
*attrs
)
1934 struct ib_uverbs_ex_modify_qp cmd
;
1935 struct ib_uverbs_ex_modify_qp_resp resp
= {
1936 .response_length
= uverbs_response_length(attrs
, sizeof(resp
))
1940 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1945 * Last bit is reserved for extending the attr_mask by
1946 * using another field.
1948 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK
== (1 << 31));
1950 if (cmd
.base
.attr_mask
&
1951 ~((IB_USER_LAST_QP_ATTR_MASK
<< 1) - 1))
1954 ret
= modify_qp(attrs
, &cmd
);
1958 return uverbs_response(attrs
, &resp
, sizeof(resp
));
1961 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle
*attrs
)
1963 struct ib_uverbs_destroy_qp cmd
;
1964 struct ib_uverbs_destroy_qp_resp resp
;
1965 struct ib_uobject
*uobj
;
1966 struct ib_uqp_object
*obj
;
1969 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
1973 uobj
= uobj_get_destroy(UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
1975 return PTR_ERR(uobj
);
1977 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
1978 memset(&resp
, 0, sizeof(resp
));
1979 resp
.events_reported
= obj
->uevent
.events_reported
;
1981 uobj_put_destroy(uobj
);
1983 return uverbs_response(attrs
, &resp
, sizeof(resp
));
1986 static void *alloc_wr(size_t wr_size
, __u32 num_sge
)
1988 if (num_sge
>= (U32_MAX
- ALIGN(wr_size
, sizeof (struct ib_sge
))) /
1989 sizeof (struct ib_sge
))
1992 return kmalloc(ALIGN(wr_size
, sizeof (struct ib_sge
)) +
1993 num_sge
* sizeof (struct ib_sge
), GFP_KERNEL
);
1996 static int ib_uverbs_post_send(struct uverbs_attr_bundle
*attrs
)
1998 struct ib_uverbs_post_send cmd
;
1999 struct ib_uverbs_post_send_resp resp
;
2000 struct ib_uverbs_send_wr
*user_wr
;
2001 struct ib_send_wr
*wr
= NULL
, *last
, *next
;
2002 const struct ib_send_wr
*bad_wr
;
2008 const struct ib_sge __user
*sgls
;
2009 const void __user
*wqes
;
2010 struct uverbs_req_iter iter
;
2012 ret
= uverbs_request_start(attrs
, &iter
, &cmd
, sizeof(cmd
));
2015 wqes
= uverbs_request_next_ptr(&iter
, cmd
.wqe_size
* cmd
.wr_count
);
2017 return PTR_ERR(wqes
);
2018 sgls
= uverbs_request_next_ptr(
2019 &iter
, cmd
.sge_count
* sizeof(struct ib_uverbs_sge
));
2021 return PTR_ERR(sgls
);
2022 ret
= uverbs_request_finish(&iter
);
2026 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
2030 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
2036 is_ud
= qp
->qp_type
== IB_QPT_UD
;
2039 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
2040 if (copy_from_user(user_wr
, wqes
+ i
* cmd
.wqe_size
,
2046 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
2052 struct ib_ud_wr
*ud
;
2054 if (user_wr
->opcode
!= IB_WR_SEND
&&
2055 user_wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
2060 next_size
= sizeof(*ud
);
2061 ud
= alloc_wr(next_size
, user_wr
->num_sge
);
2067 ud
->ah
= uobj_get_obj_read(ah
, UVERBS_OBJECT_AH
,
2068 user_wr
->wr
.ud
.ah
, attrs
);
2074 ud
->remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
2075 ud
->remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
2078 } else if (user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
||
2079 user_wr
->opcode
== IB_WR_RDMA_WRITE
||
2080 user_wr
->opcode
== IB_WR_RDMA_READ
) {
2081 struct ib_rdma_wr
*rdma
;
2083 next_size
= sizeof(*rdma
);
2084 rdma
= alloc_wr(next_size
, user_wr
->num_sge
);
2090 rdma
->remote_addr
= user_wr
->wr
.rdma
.remote_addr
;
2091 rdma
->rkey
= user_wr
->wr
.rdma
.rkey
;
2094 } else if (user_wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
||
2095 user_wr
->opcode
== IB_WR_ATOMIC_FETCH_AND_ADD
) {
2096 struct ib_atomic_wr
*atomic
;
2098 next_size
= sizeof(*atomic
);
2099 atomic
= alloc_wr(next_size
, user_wr
->num_sge
);
2105 atomic
->remote_addr
= user_wr
->wr
.atomic
.remote_addr
;
2106 atomic
->compare_add
= user_wr
->wr
.atomic
.compare_add
;
2107 atomic
->swap
= user_wr
->wr
.atomic
.swap
;
2108 atomic
->rkey
= user_wr
->wr
.atomic
.rkey
;
2111 } else if (user_wr
->opcode
== IB_WR_SEND
||
2112 user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2113 user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2114 next_size
= sizeof(*next
);
2115 next
= alloc_wr(next_size
, user_wr
->num_sge
);
2125 if (user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2126 user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) {
2128 (__be32 __force
) user_wr
->ex
.imm_data
;
2129 } else if (user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2130 next
->ex
.invalidate_rkey
= user_wr
->ex
.invalidate_rkey
;
2140 next
->wr_id
= user_wr
->wr_id
;
2141 next
->num_sge
= user_wr
->num_sge
;
2142 next
->opcode
= user_wr
->opcode
;
2143 next
->send_flags
= user_wr
->send_flags
;
2145 if (next
->num_sge
) {
2146 next
->sg_list
= (void *) next
+
2147 ALIGN(next_size
, sizeof(struct ib_sge
));
2148 if (copy_from_user(next
->sg_list
, sgls
+ sg_ind
,
2150 sizeof(struct ib_sge
))) {
2154 sg_ind
+= next
->num_sge
;
2156 next
->sg_list
= NULL
;
2160 ret
= qp
->device
->ops
.post_send(qp
->real_qp
, wr
, &bad_wr
);
2162 for (next
= wr
; next
; next
= next
->next
) {
2168 ret2
= uverbs_response(attrs
, &resp
, sizeof(resp
));
2173 uobj_put_obj_read(qp
);
2176 if (is_ud
&& ud_wr(wr
)->ah
)
2177 uobj_put_obj_read(ud_wr(wr
)->ah
);
2189 static struct ib_recv_wr
*
2190 ib_uverbs_unmarshall_recv(struct uverbs_req_iter
*iter
, u32 wr_count
,
2191 u32 wqe_size
, u32 sge_count
)
2193 struct ib_uverbs_recv_wr
*user_wr
;
2194 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2198 const struct ib_sge __user
*sgls
;
2199 const void __user
*wqes
;
2201 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2202 return ERR_PTR(-EINVAL
);
2204 wqes
= uverbs_request_next_ptr(iter
, wqe_size
* wr_count
);
2206 return ERR_CAST(wqes
);
2207 sgls
= uverbs_request_next_ptr(
2208 iter
, sge_count
* sizeof(struct ib_uverbs_sge
));
2210 return ERR_CAST(sgls
);
2211 ret
= uverbs_request_finish(iter
);
2213 return ERR_PTR(ret
);
2215 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2217 return ERR_PTR(-ENOMEM
);
2221 for (i
= 0; i
< wr_count
; ++i
) {
2222 if (copy_from_user(user_wr
, wqes
+ i
* wqe_size
,
2228 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2233 if (user_wr
->num_sge
>=
2234 (U32_MAX
- ALIGN(sizeof *next
, sizeof (struct ib_sge
))) /
2235 sizeof (struct ib_sge
)) {
2240 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2241 user_wr
->num_sge
* sizeof (struct ib_sge
),
2255 next
->wr_id
= user_wr
->wr_id
;
2256 next
->num_sge
= user_wr
->num_sge
;
2258 if (next
->num_sge
) {
2259 next
->sg_list
= (void *) next
+
2260 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2261 if (copy_from_user(next
->sg_list
, sgls
+ sg_ind
,
2263 sizeof(struct ib_sge
))) {
2267 sg_ind
+= next
->num_sge
;
2269 next
->sg_list
= NULL
;
2284 return ERR_PTR(ret
);
2287 static int ib_uverbs_post_recv(struct uverbs_attr_bundle
*attrs
)
2289 struct ib_uverbs_post_recv cmd
;
2290 struct ib_uverbs_post_recv_resp resp
;
2291 struct ib_recv_wr
*wr
, *next
;
2292 const struct ib_recv_wr
*bad_wr
;
2295 struct uverbs_req_iter iter
;
2297 ret
= uverbs_request_start(attrs
, &iter
, &cmd
, sizeof(cmd
));
2301 wr
= ib_uverbs_unmarshall_recv(&iter
, cmd
.wr_count
, cmd
.wqe_size
,
2306 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
2313 ret
= qp
->device
->ops
.post_recv(qp
->real_qp
, wr
, &bad_wr
);
2315 uobj_put_obj_read(qp
);
2317 for (next
= wr
; next
; next
= next
->next
) {
2324 ret2
= uverbs_response(attrs
, &resp
, sizeof(resp
));
2337 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle
*attrs
)
2339 struct ib_uverbs_post_srq_recv cmd
;
2340 struct ib_uverbs_post_srq_recv_resp resp
;
2341 struct ib_recv_wr
*wr
, *next
;
2342 const struct ib_recv_wr
*bad_wr
;
2345 struct uverbs_req_iter iter
;
2347 ret
= uverbs_request_start(attrs
, &iter
, &cmd
, sizeof(cmd
));
2351 wr
= ib_uverbs_unmarshall_recv(&iter
, cmd
.wr_count
, cmd
.wqe_size
,
2356 srq
= uobj_get_obj_read(srq
, UVERBS_OBJECT_SRQ
, cmd
.srq_handle
, attrs
);
2363 ret
= srq
->device
->ops
.post_srq_recv(srq
, wr
, &bad_wr
);
2365 uobj_put_obj_read(srq
);
2368 for (next
= wr
; next
; next
= next
->next
) {
2374 ret2
= uverbs_response(attrs
, &resp
, sizeof(resp
));
2388 static int ib_uverbs_create_ah(struct uverbs_attr_bundle
*attrs
)
2390 struct ib_uverbs_create_ah cmd
;
2391 struct ib_uverbs_create_ah_resp resp
;
2392 struct ib_uobject
*uobj
;
2395 struct rdma_ah_attr attr
= {};
2397 struct ib_device
*ib_dev
;
2399 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2403 uobj
= uobj_alloc(UVERBS_OBJECT_AH
, attrs
, &ib_dev
);
2405 return PTR_ERR(uobj
);
2407 if (!rdma_is_port_valid(ib_dev
, cmd
.attr
.port_num
)) {
2412 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
.pd_handle
, attrs
);
2418 attr
.type
= rdma_ah_find_type(ib_dev
, cmd
.attr
.port_num
);
2419 rdma_ah_set_make_grd(&attr
, false);
2420 rdma_ah_set_dlid(&attr
, cmd
.attr
.dlid
);
2421 rdma_ah_set_sl(&attr
, cmd
.attr
.sl
);
2422 rdma_ah_set_path_bits(&attr
, cmd
.attr
.src_path_bits
);
2423 rdma_ah_set_static_rate(&attr
, cmd
.attr
.static_rate
);
2424 rdma_ah_set_port_num(&attr
, cmd
.attr
.port_num
);
2426 if (cmd
.attr
.is_global
) {
2427 rdma_ah_set_grh(&attr
, NULL
, cmd
.attr
.grh
.flow_label
,
2428 cmd
.attr
.grh
.sgid_index
,
2429 cmd
.attr
.grh
.hop_limit
,
2430 cmd
.attr
.grh
.traffic_class
);
2431 rdma_ah_set_dgid_raw(&attr
, cmd
.attr
.grh
.dgid
);
2433 rdma_ah_set_ah_flags(&attr
, 0);
2436 ah
= rdma_create_user_ah(pd
, &attr
, &attrs
->driver_udata
);
2443 uobj
->user_handle
= cmd
.user_handle
;
2446 resp
.ah_handle
= uobj
->id
;
2448 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
2452 uobj_put_obj_read(pd
);
2453 return uobj_alloc_commit(uobj
, attrs
);
2456 rdma_destroy_ah(ah
, RDMA_DESTROY_AH_SLEEPABLE
);
2459 uobj_put_obj_read(pd
);
2462 uobj_alloc_abort(uobj
, attrs
);
2466 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle
*attrs
)
2468 struct ib_uverbs_destroy_ah cmd
;
2471 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2475 return uobj_perform_destroy(UVERBS_OBJECT_AH
, cmd
.ah_handle
, attrs
);
2478 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle
*attrs
)
2480 struct ib_uverbs_attach_mcast cmd
;
2482 struct ib_uqp_object
*obj
;
2483 struct ib_uverbs_mcast_entry
*mcast
;
2486 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2490 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
2494 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2496 mutex_lock(&obj
->mcast_lock
);
2497 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2498 if (cmd
.mlid
== mcast
->lid
&&
2499 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2504 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2510 mcast
->lid
= cmd
.mlid
;
2511 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2513 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2515 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2520 mutex_unlock(&obj
->mcast_lock
);
2521 uobj_put_obj_read(qp
);
2526 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle
*attrs
)
2528 struct ib_uverbs_detach_mcast cmd
;
2529 struct ib_uqp_object
*obj
;
2531 struct ib_uverbs_mcast_entry
*mcast
;
2535 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2539 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
2543 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2544 mutex_lock(&obj
->mcast_lock
);
2546 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2547 if (cmd
.mlid
== mcast
->lid
&&
2548 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2549 list_del(&mcast
->list
);
2560 ret
= ib_detach_mcast(qp
, (union ib_gid
*)cmd
.gid
, cmd
.mlid
);
2563 mutex_unlock(&obj
->mcast_lock
);
2564 uobj_put_obj_read(qp
);
2568 struct ib_uflow_resources
*flow_resources_alloc(size_t num_specs
)
2570 struct ib_uflow_resources
*resources
;
2572 resources
= kzalloc(sizeof(*resources
), GFP_KERNEL
);
2580 resources
->counters
=
2581 kcalloc(num_specs
, sizeof(*resources
->counters
), GFP_KERNEL
);
2582 resources
->collection
=
2583 kcalloc(num_specs
, sizeof(*resources
->collection
), GFP_KERNEL
);
2585 if (!resources
->counters
|| !resources
->collection
)
2589 resources
->max
= num_specs
;
2593 kfree(resources
->counters
);
2598 EXPORT_SYMBOL(flow_resources_alloc
);
2600 void ib_uverbs_flow_resources_free(struct ib_uflow_resources
*uflow_res
)
2607 for (i
= 0; i
< uflow_res
->collection_num
; i
++)
2608 atomic_dec(&uflow_res
->collection
[i
]->usecnt
);
2610 for (i
= 0; i
< uflow_res
->counters_num
; i
++)
2611 atomic_dec(&uflow_res
->counters
[i
]->usecnt
);
2613 kfree(uflow_res
->collection
);
2614 kfree(uflow_res
->counters
);
2617 EXPORT_SYMBOL(ib_uverbs_flow_resources_free
);
2619 void flow_resources_add(struct ib_uflow_resources
*uflow_res
,
2620 enum ib_flow_spec_type type
,
2623 WARN_ON(uflow_res
->num
>= uflow_res
->max
);
2626 case IB_FLOW_SPEC_ACTION_HANDLE
:
2627 atomic_inc(&((struct ib_flow_action
*)ibobj
)->usecnt
);
2628 uflow_res
->collection
[uflow_res
->collection_num
++] =
2629 (struct ib_flow_action
*)ibobj
;
2631 case IB_FLOW_SPEC_ACTION_COUNT
:
2632 atomic_inc(&((struct ib_counters
*)ibobj
)->usecnt
);
2633 uflow_res
->counters
[uflow_res
->counters_num
++] =
2634 (struct ib_counters
*)ibobj
;
2642 EXPORT_SYMBOL(flow_resources_add
);
2644 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle
*attrs
,
2645 struct ib_uverbs_flow_spec
*kern_spec
,
2646 union ib_flow_spec
*ib_spec
,
2647 struct ib_uflow_resources
*uflow_res
)
2649 ib_spec
->type
= kern_spec
->type
;
2650 switch (ib_spec
->type
) {
2651 case IB_FLOW_SPEC_ACTION_TAG
:
2652 if (kern_spec
->flow_tag
.size
!=
2653 sizeof(struct ib_uverbs_flow_spec_action_tag
))
2656 ib_spec
->flow_tag
.size
= sizeof(struct ib_flow_spec_action_tag
);
2657 ib_spec
->flow_tag
.tag_id
= kern_spec
->flow_tag
.tag_id
;
2659 case IB_FLOW_SPEC_ACTION_DROP
:
2660 if (kern_spec
->drop
.size
!=
2661 sizeof(struct ib_uverbs_flow_spec_action_drop
))
2664 ib_spec
->drop
.size
= sizeof(struct ib_flow_spec_action_drop
);
2666 case IB_FLOW_SPEC_ACTION_HANDLE
:
2667 if (kern_spec
->action
.size
!=
2668 sizeof(struct ib_uverbs_flow_spec_action_handle
))
2670 ib_spec
->action
.act
= uobj_get_obj_read(flow_action
,
2671 UVERBS_OBJECT_FLOW_ACTION
,
2672 kern_spec
->action
.handle
,
2674 if (!ib_spec
->action
.act
)
2676 ib_spec
->action
.size
=
2677 sizeof(struct ib_flow_spec_action_handle
);
2678 flow_resources_add(uflow_res
,
2679 IB_FLOW_SPEC_ACTION_HANDLE
,
2680 ib_spec
->action
.act
);
2681 uobj_put_obj_read(ib_spec
->action
.act
);
2683 case IB_FLOW_SPEC_ACTION_COUNT
:
2684 if (kern_spec
->flow_count
.size
!=
2685 sizeof(struct ib_uverbs_flow_spec_action_count
))
2687 ib_spec
->flow_count
.counters
=
2688 uobj_get_obj_read(counters
,
2689 UVERBS_OBJECT_COUNTERS
,
2690 kern_spec
->flow_count
.handle
,
2692 if (!ib_spec
->flow_count
.counters
)
2694 ib_spec
->flow_count
.size
=
2695 sizeof(struct ib_flow_spec_action_count
);
2696 flow_resources_add(uflow_res
,
2697 IB_FLOW_SPEC_ACTION_COUNT
,
2698 ib_spec
->flow_count
.counters
);
2699 uobj_put_obj_read(ib_spec
->flow_count
.counters
);
2707 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr
*spec
)
2709 /* Returns user space filter size, includes padding */
2710 return (spec
->size
- sizeof(struct ib_uverbs_flow_spec_hdr
)) / 2;
2713 static ssize_t
spec_filter_size(const void *kern_spec_filter
, u16 kern_filter_size
,
2714 u16 ib_real_filter_sz
)
2717 * User space filter structures must be 64 bit aligned, otherwise this
2718 * may pass, but we won't handle additional new attributes.
2721 if (kern_filter_size
> ib_real_filter_sz
) {
2722 if (memchr_inv(kern_spec_filter
+
2723 ib_real_filter_sz
, 0,
2724 kern_filter_size
- ib_real_filter_sz
))
2726 return ib_real_filter_sz
;
2728 return kern_filter_size
;
2731 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type
,
2732 const void *kern_spec_mask
,
2733 const void *kern_spec_val
,
2734 size_t kern_filter_sz
,
2735 union ib_flow_spec
*ib_spec
)
2737 ssize_t actual_filter_sz
;
2738 ssize_t ib_filter_sz
;
2740 /* User flow spec size must be aligned to 4 bytes */
2741 if (kern_filter_sz
!= ALIGN(kern_filter_sz
, 4))
2744 ib_spec
->type
= type
;
2746 if (ib_spec
->type
== (IB_FLOW_SPEC_INNER
| IB_FLOW_SPEC_VXLAN_TUNNEL
))
2749 switch (ib_spec
->type
& ~IB_FLOW_SPEC_INNER
) {
2750 case IB_FLOW_SPEC_ETH
:
2751 ib_filter_sz
= offsetof(struct ib_flow_eth_filter
, real_sz
);
2752 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2755 if (actual_filter_sz
<= 0)
2757 ib_spec
->size
= sizeof(struct ib_flow_spec_eth
);
2758 memcpy(&ib_spec
->eth
.val
, kern_spec_val
, actual_filter_sz
);
2759 memcpy(&ib_spec
->eth
.mask
, kern_spec_mask
, actual_filter_sz
);
2761 case IB_FLOW_SPEC_IPV4
:
2762 ib_filter_sz
= offsetof(struct ib_flow_ipv4_filter
, real_sz
);
2763 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2766 if (actual_filter_sz
<= 0)
2768 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv4
);
2769 memcpy(&ib_spec
->ipv4
.val
, kern_spec_val
, actual_filter_sz
);
2770 memcpy(&ib_spec
->ipv4
.mask
, kern_spec_mask
, actual_filter_sz
);
2772 case IB_FLOW_SPEC_IPV6
:
2773 ib_filter_sz
= offsetof(struct ib_flow_ipv6_filter
, real_sz
);
2774 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2777 if (actual_filter_sz
<= 0)
2779 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv6
);
2780 memcpy(&ib_spec
->ipv6
.val
, kern_spec_val
, actual_filter_sz
);
2781 memcpy(&ib_spec
->ipv6
.mask
, kern_spec_mask
, actual_filter_sz
);
2783 if ((ntohl(ib_spec
->ipv6
.mask
.flow_label
)) >= BIT(20) ||
2784 (ntohl(ib_spec
->ipv6
.val
.flow_label
)) >= BIT(20))
2787 case IB_FLOW_SPEC_TCP
:
2788 case IB_FLOW_SPEC_UDP
:
2789 ib_filter_sz
= offsetof(struct ib_flow_tcp_udp_filter
, real_sz
);
2790 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2793 if (actual_filter_sz
<= 0)
2795 ib_spec
->size
= sizeof(struct ib_flow_spec_tcp_udp
);
2796 memcpy(&ib_spec
->tcp_udp
.val
, kern_spec_val
, actual_filter_sz
);
2797 memcpy(&ib_spec
->tcp_udp
.mask
, kern_spec_mask
, actual_filter_sz
);
2799 case IB_FLOW_SPEC_VXLAN_TUNNEL
:
2800 ib_filter_sz
= offsetof(struct ib_flow_tunnel_filter
, real_sz
);
2801 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2804 if (actual_filter_sz
<= 0)
2806 ib_spec
->tunnel
.size
= sizeof(struct ib_flow_spec_tunnel
);
2807 memcpy(&ib_spec
->tunnel
.val
, kern_spec_val
, actual_filter_sz
);
2808 memcpy(&ib_spec
->tunnel
.mask
, kern_spec_mask
, actual_filter_sz
);
2810 if ((ntohl(ib_spec
->tunnel
.mask
.tunnel_id
)) >= BIT(24) ||
2811 (ntohl(ib_spec
->tunnel
.val
.tunnel_id
)) >= BIT(24))
2814 case IB_FLOW_SPEC_ESP
:
2815 ib_filter_sz
= offsetof(struct ib_flow_esp_filter
, real_sz
);
2816 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2819 if (actual_filter_sz
<= 0)
2821 ib_spec
->esp
.size
= sizeof(struct ib_flow_spec_esp
);
2822 memcpy(&ib_spec
->esp
.val
, kern_spec_val
, actual_filter_sz
);
2823 memcpy(&ib_spec
->esp
.mask
, kern_spec_mask
, actual_filter_sz
);
2825 case IB_FLOW_SPEC_GRE
:
2826 ib_filter_sz
= offsetof(struct ib_flow_gre_filter
, real_sz
);
2827 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2830 if (actual_filter_sz
<= 0)
2832 ib_spec
->gre
.size
= sizeof(struct ib_flow_spec_gre
);
2833 memcpy(&ib_spec
->gre
.val
, kern_spec_val
, actual_filter_sz
);
2834 memcpy(&ib_spec
->gre
.mask
, kern_spec_mask
, actual_filter_sz
);
2836 case IB_FLOW_SPEC_MPLS
:
2837 ib_filter_sz
= offsetof(struct ib_flow_mpls_filter
, real_sz
);
2838 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2841 if (actual_filter_sz
<= 0)
2843 ib_spec
->mpls
.size
= sizeof(struct ib_flow_spec_mpls
);
2844 memcpy(&ib_spec
->mpls
.val
, kern_spec_val
, actual_filter_sz
);
2845 memcpy(&ib_spec
->mpls
.mask
, kern_spec_mask
, actual_filter_sz
);
2853 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec
*kern_spec
,
2854 union ib_flow_spec
*ib_spec
)
2856 ssize_t kern_filter_sz
;
2857 void *kern_spec_mask
;
2858 void *kern_spec_val
;
2860 kern_filter_sz
= kern_spec_filter_sz(&kern_spec
->hdr
);
2862 kern_spec_val
= (void *)kern_spec
+
2863 sizeof(struct ib_uverbs_flow_spec_hdr
);
2864 kern_spec_mask
= kern_spec_val
+ kern_filter_sz
;
2866 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec
->type
,
2869 kern_filter_sz
, ib_spec
);
2872 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle
*attrs
,
2873 struct ib_uverbs_flow_spec
*kern_spec
,
2874 union ib_flow_spec
*ib_spec
,
2875 struct ib_uflow_resources
*uflow_res
)
2877 if (kern_spec
->reserved
)
2880 if (kern_spec
->type
>= IB_FLOW_SPEC_ACTION_TAG
)
2881 return kern_spec_to_ib_spec_action(attrs
, kern_spec
, ib_spec
,
2884 return kern_spec_to_ib_spec_filter(kern_spec
, ib_spec
);
2887 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle
*attrs
)
2889 struct ib_uverbs_ex_create_wq cmd
;
2890 struct ib_uverbs_ex_create_wq_resp resp
= {};
2891 struct ib_uwq_object
*obj
;
2896 struct ib_wq_init_attr wq_init_attr
= {};
2897 struct ib_device
*ib_dev
;
2899 err
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2906 obj
= (struct ib_uwq_object
*)uobj_alloc(UVERBS_OBJECT_WQ
, attrs
,
2909 return PTR_ERR(obj
);
2911 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
.pd_handle
, attrs
);
2917 cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
2923 wq_init_attr
.cq
= cq
;
2924 wq_init_attr
.max_sge
= cmd
.max_sge
;
2925 wq_init_attr
.max_wr
= cmd
.max_wr
;
2926 wq_init_attr
.wq_context
= attrs
->ufile
;
2927 wq_init_attr
.wq_type
= cmd
.wq_type
;
2928 wq_init_attr
.event_handler
= ib_uverbs_wq_event_handler
;
2929 wq_init_attr
.create_flags
= cmd
.create_flags
;
2930 obj
->uevent
.events_reported
= 0;
2931 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
2933 wq
= pd
->device
->ops
.create_wq(pd
, &wq_init_attr
, &attrs
->driver_udata
);
2939 wq
->uobject
= &obj
->uevent
.uobject
;
2940 obj
->uevent
.uobject
.object
= wq
;
2941 wq
->wq_type
= wq_init_attr
.wq_type
;
2944 wq
->device
= pd
->device
;
2945 wq
->wq_context
= wq_init_attr
.wq_context
;
2946 atomic_set(&wq
->usecnt
, 0);
2947 atomic_inc(&pd
->usecnt
);
2948 atomic_inc(&cq
->usecnt
);
2949 wq
->uobject
= &obj
->uevent
.uobject
;
2950 obj
->uevent
.uobject
.object
= wq
;
2952 memset(&resp
, 0, sizeof(resp
));
2953 resp
.wq_handle
= obj
->uevent
.uobject
.id
;
2954 resp
.max_sge
= wq_init_attr
.max_sge
;
2955 resp
.max_wr
= wq_init_attr
.max_wr
;
2956 resp
.wqn
= wq
->wq_num
;
2957 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
2958 err
= uverbs_response(attrs
, &resp
, sizeof(resp
));
2962 uobj_put_obj_read(pd
);
2963 uobj_put_obj_read(cq
);
2964 return uobj_alloc_commit(&obj
->uevent
.uobject
, attrs
);
2967 ib_destroy_wq(wq
, &attrs
->driver_udata
);
2969 uobj_put_obj_read(cq
);
2971 uobj_put_obj_read(pd
);
2973 uobj_alloc_abort(&obj
->uevent
.uobject
, attrs
);
2978 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle
*attrs
)
2980 struct ib_uverbs_ex_destroy_wq cmd
;
2981 struct ib_uverbs_ex_destroy_wq_resp resp
= {};
2982 struct ib_uobject
*uobj
;
2983 struct ib_uwq_object
*obj
;
2986 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
2993 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
2994 uobj
= uobj_get_destroy(UVERBS_OBJECT_WQ
, cmd
.wq_handle
, attrs
);
2996 return PTR_ERR(uobj
);
2998 obj
= container_of(uobj
, struct ib_uwq_object
, uevent
.uobject
);
2999 resp
.events_reported
= obj
->uevent
.events_reported
;
3001 uobj_put_destroy(uobj
);
3003 return uverbs_response(attrs
, &resp
, sizeof(resp
));
3006 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle
*attrs
)
3008 struct ib_uverbs_ex_modify_wq cmd
;
3010 struct ib_wq_attr wq_attr
= {};
3013 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3020 if (cmd
.attr_mask
> (IB_WQ_STATE
| IB_WQ_CUR_STATE
| IB_WQ_FLAGS
))
3023 wq
= uobj_get_obj_read(wq
, UVERBS_OBJECT_WQ
, cmd
.wq_handle
, attrs
);
3027 wq_attr
.curr_wq_state
= cmd
.curr_wq_state
;
3028 wq_attr
.wq_state
= cmd
.wq_state
;
3029 if (cmd
.attr_mask
& IB_WQ_FLAGS
) {
3030 wq_attr
.flags
= cmd
.flags
;
3031 wq_attr
.flags_mask
= cmd
.flags_mask
;
3033 ret
= wq
->device
->ops
.modify_wq(wq
, &wq_attr
, cmd
.attr_mask
,
3034 &attrs
->driver_udata
);
3035 uobj_put_obj_read(wq
);
3039 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle
*attrs
)
3041 struct ib_uverbs_ex_create_rwq_ind_table cmd
;
3042 struct ib_uverbs_ex_create_rwq_ind_table_resp resp
= {};
3043 struct ib_uobject
*uobj
;
3045 struct ib_rwq_ind_table_init_attr init_attr
= {};
3046 struct ib_rwq_ind_table
*rwq_ind_tbl
;
3047 struct ib_wq
**wqs
= NULL
;
3048 u32
*wqs_handles
= NULL
;
3049 struct ib_wq
*wq
= NULL
;
3050 int i
, j
, num_read_wqs
;
3052 struct uverbs_req_iter iter
;
3053 struct ib_device
*ib_dev
;
3055 err
= uverbs_request_start(attrs
, &iter
, &cmd
, sizeof(cmd
));
3062 if (cmd
.log_ind_tbl_size
> IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE
)
3065 num_wq_handles
= 1 << cmd
.log_ind_tbl_size
;
3066 wqs_handles
= kcalloc(num_wq_handles
, sizeof(*wqs_handles
),
3071 err
= uverbs_request_next(&iter
, wqs_handles
,
3072 num_wq_handles
* sizeof(__u32
));
3076 err
= uverbs_request_finish(&iter
);
3080 wqs
= kcalloc(num_wq_handles
, sizeof(*wqs
), GFP_KERNEL
);
3086 for (num_read_wqs
= 0; num_read_wqs
< num_wq_handles
;
3088 wq
= uobj_get_obj_read(wq
, UVERBS_OBJECT_WQ
,
3089 wqs_handles
[num_read_wqs
], attrs
);
3095 wqs
[num_read_wqs
] = wq
;
3098 uobj
= uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL
, attrs
, &ib_dev
);
3100 err
= PTR_ERR(uobj
);
3104 init_attr
.log_ind_tbl_size
= cmd
.log_ind_tbl_size
;
3105 init_attr
.ind_tbl
= wqs
;
3107 rwq_ind_tbl
= ib_dev
->ops
.create_rwq_ind_table(ib_dev
, &init_attr
,
3108 &attrs
->driver_udata
);
3110 if (IS_ERR(rwq_ind_tbl
)) {
3111 err
= PTR_ERR(rwq_ind_tbl
);
3115 rwq_ind_tbl
->ind_tbl
= wqs
;
3116 rwq_ind_tbl
->log_ind_tbl_size
= init_attr
.log_ind_tbl_size
;
3117 rwq_ind_tbl
->uobject
= uobj
;
3118 uobj
->object
= rwq_ind_tbl
;
3119 rwq_ind_tbl
->device
= ib_dev
;
3120 atomic_set(&rwq_ind_tbl
->usecnt
, 0);
3122 for (i
= 0; i
< num_wq_handles
; i
++)
3123 atomic_inc(&wqs
[i
]->usecnt
);
3125 resp
.ind_tbl_handle
= uobj
->id
;
3126 resp
.ind_tbl_num
= rwq_ind_tbl
->ind_tbl_num
;
3127 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
3129 err
= uverbs_response(attrs
, &resp
, sizeof(resp
));
3135 for (j
= 0; j
< num_read_wqs
; j
++)
3136 uobj_put_obj_read(wqs
[j
]);
3138 return uobj_alloc_commit(uobj
, attrs
);
3141 ib_destroy_rwq_ind_table(rwq_ind_tbl
);
3143 uobj_alloc_abort(uobj
, attrs
);
3145 for (j
= 0; j
< num_read_wqs
; j
++)
3146 uobj_put_obj_read(wqs
[j
]);
3153 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle
*attrs
)
3155 struct ib_uverbs_ex_destroy_rwq_ind_table cmd
;
3158 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3165 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL
,
3166 cmd
.ind_tbl_handle
, attrs
);
3169 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle
*attrs
)
3171 struct ib_uverbs_create_flow cmd
;
3172 struct ib_uverbs_create_flow_resp resp
;
3173 struct ib_uobject
*uobj
;
3174 struct ib_flow
*flow_id
;
3175 struct ib_uverbs_flow_attr
*kern_flow_attr
;
3176 struct ib_flow_attr
*flow_attr
;
3178 struct ib_uflow_resources
*uflow_res
;
3179 struct ib_uverbs_flow_spec_hdr
*kern_spec
;
3180 struct uverbs_req_iter iter
;
3184 struct ib_device
*ib_dev
;
3186 err
= uverbs_request_start(attrs
, &iter
, &cmd
, sizeof(cmd
));
3193 if (!capable(CAP_NET_RAW
))
3196 if (cmd
.flow_attr
.flags
>= IB_FLOW_ATTR_FLAGS_RESERVED
)
3199 if ((cmd
.flow_attr
.flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
) &&
3200 ((cmd
.flow_attr
.type
== IB_FLOW_ATTR_ALL_DEFAULT
) ||
3201 (cmd
.flow_attr
.type
== IB_FLOW_ATTR_MC_DEFAULT
)))
3204 if (cmd
.flow_attr
.num_of_specs
> IB_FLOW_SPEC_SUPPORT_LAYERS
)
3207 if (cmd
.flow_attr
.size
>
3208 (cmd
.flow_attr
.num_of_specs
* sizeof(struct ib_uverbs_flow_spec
)))
3211 if (cmd
.flow_attr
.reserved
[0] ||
3212 cmd
.flow_attr
.reserved
[1])
3215 if (cmd
.flow_attr
.num_of_specs
) {
3216 kern_flow_attr
= kmalloc(sizeof(*kern_flow_attr
) + cmd
.flow_attr
.size
,
3218 if (!kern_flow_attr
)
3221 *kern_flow_attr
= cmd
.flow_attr
;
3222 err
= uverbs_request_next(&iter
, &kern_flow_attr
->flow_specs
,
3223 cmd
.flow_attr
.size
);
3227 kern_flow_attr
= &cmd
.flow_attr
;
3230 err
= uverbs_request_finish(&iter
);
3234 uobj
= uobj_alloc(UVERBS_OBJECT_FLOW
, attrs
, &ib_dev
);
3236 err
= PTR_ERR(uobj
);
3240 qp
= uobj_get_obj_read(qp
, UVERBS_OBJECT_QP
, cmd
.qp_handle
, attrs
);
3246 if (qp
->qp_type
!= IB_QPT_UD
&& qp
->qp_type
!= IB_QPT_RAW_PACKET
) {
3251 flow_attr
= kzalloc(struct_size(flow_attr
, flows
,
3252 cmd
.flow_attr
.num_of_specs
), GFP_KERNEL
);
3257 uflow_res
= flow_resources_alloc(cmd
.flow_attr
.num_of_specs
);
3260 goto err_free_flow_attr
;
3263 flow_attr
->type
= kern_flow_attr
->type
;
3264 flow_attr
->priority
= kern_flow_attr
->priority
;
3265 flow_attr
->num_of_specs
= kern_flow_attr
->num_of_specs
;
3266 flow_attr
->port
= kern_flow_attr
->port
;
3267 flow_attr
->flags
= kern_flow_attr
->flags
;
3268 flow_attr
->size
= sizeof(*flow_attr
);
3270 kern_spec
= kern_flow_attr
->flow_specs
;
3271 ib_spec
= flow_attr
+ 1;
3272 for (i
= 0; i
< flow_attr
->num_of_specs
&&
3273 cmd
.flow_attr
.size
>= sizeof(*kern_spec
) &&
3274 cmd
.flow_attr
.size
>= kern_spec
->size
;
3276 err
= kern_spec_to_ib_spec(
3277 attrs
, (struct ib_uverbs_flow_spec
*)kern_spec
,
3278 ib_spec
, uflow_res
);
3283 ((union ib_flow_spec
*) ib_spec
)->size
;
3284 cmd
.flow_attr
.size
-= kern_spec
->size
;
3285 kern_spec
= ((void *)kern_spec
) + kern_spec
->size
;
3286 ib_spec
+= ((union ib_flow_spec
*) ib_spec
)->size
;
3288 if (cmd
.flow_attr
.size
|| (i
!= flow_attr
->num_of_specs
)) {
3289 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3290 i
, cmd
.flow_attr
.size
);
3295 flow_id
= qp
->device
->ops
.create_flow(
3296 qp
, flow_attr
, IB_FLOW_DOMAIN_USER
, &attrs
->driver_udata
);
3298 if (IS_ERR(flow_id
)) {
3299 err
= PTR_ERR(flow_id
);
3303 ib_set_flow(uobj
, flow_id
, qp
, qp
->device
, uflow_res
);
3305 memset(&resp
, 0, sizeof(resp
));
3306 resp
.flow_handle
= uobj
->id
;
3308 err
= uverbs_response(attrs
, &resp
, sizeof(resp
));
3312 uobj_put_obj_read(qp
);
3314 if (cmd
.flow_attr
.num_of_specs
)
3315 kfree(kern_flow_attr
);
3316 return uobj_alloc_commit(uobj
, attrs
);
3318 if (!qp
->device
->ops
.destroy_flow(flow_id
))
3319 atomic_dec(&qp
->usecnt
);
3321 ib_uverbs_flow_resources_free(uflow_res
);
3325 uobj_put_obj_read(qp
);
3327 uobj_alloc_abort(uobj
, attrs
);
3329 if (cmd
.flow_attr
.num_of_specs
)
3330 kfree(kern_flow_attr
);
3334 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle
*attrs
)
3336 struct ib_uverbs_destroy_flow cmd
;
3339 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3346 return uobj_perform_destroy(UVERBS_OBJECT_FLOW
, cmd
.flow_handle
, attrs
);
3349 static int __uverbs_create_xsrq(struct uverbs_attr_bundle
*attrs
,
3350 struct ib_uverbs_create_xsrq
*cmd
,
3351 struct ib_udata
*udata
)
3353 struct ib_uverbs_create_srq_resp resp
;
3354 struct ib_usrq_object
*obj
;
3357 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
3358 struct ib_srq_init_attr attr
;
3360 struct ib_device
*ib_dev
;
3362 obj
= (struct ib_usrq_object
*)uobj_alloc(UVERBS_OBJECT_SRQ
, attrs
,
3365 return PTR_ERR(obj
);
3367 if (cmd
->srq_type
== IB_SRQT_TM
)
3368 attr
.ext
.tag_matching
.max_num_tags
= cmd
->max_num_tags
;
3370 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3371 xrcd_uobj
= uobj_get_read(UVERBS_OBJECT_XRCD
, cmd
->xrcd_handle
,
3373 if (IS_ERR(xrcd_uobj
)) {
3378 attr
.ext
.xrc
.xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
3379 if (!attr
.ext
.xrc
.xrcd
) {
3384 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
3385 atomic_inc(&obj
->uxrcd
->refcnt
);
3388 if (ib_srq_has_cq(cmd
->srq_type
)) {
3389 attr
.ext
.cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
,
3390 cmd
->cq_handle
, attrs
);
3397 pd
= uobj_get_obj_read(pd
, UVERBS_OBJECT_PD
, cmd
->pd_handle
, attrs
);
3403 attr
.event_handler
= ib_uverbs_srq_event_handler
;
3404 attr
.srq_context
= attrs
->ufile
;
3405 attr
.srq_type
= cmd
->srq_type
;
3406 attr
.attr
.max_wr
= cmd
->max_wr
;
3407 attr
.attr
.max_sge
= cmd
->max_sge
;
3408 attr
.attr
.srq_limit
= cmd
->srq_limit
;
3410 obj
->uevent
.events_reported
= 0;
3411 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
3413 srq
= pd
->device
->ops
.create_srq(pd
, &attr
, udata
);
3419 srq
->device
= pd
->device
;
3421 srq
->srq_type
= cmd
->srq_type
;
3422 srq
->uobject
= &obj
->uevent
.uobject
;
3423 srq
->event_handler
= attr
.event_handler
;
3424 srq
->srq_context
= attr
.srq_context
;
3426 if (ib_srq_has_cq(cmd
->srq_type
)) {
3427 srq
->ext
.cq
= attr
.ext
.cq
;
3428 atomic_inc(&attr
.ext
.cq
->usecnt
);
3431 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3432 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
3433 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
3436 atomic_inc(&pd
->usecnt
);
3437 atomic_set(&srq
->usecnt
, 0);
3439 obj
->uevent
.uobject
.object
= srq
;
3440 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
3442 memset(&resp
, 0, sizeof resp
);
3443 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
3444 resp
.max_wr
= attr
.attr
.max_wr
;
3445 resp
.max_sge
= attr
.attr
.max_sge
;
3446 if (cmd
->srq_type
== IB_SRQT_XRC
)
3447 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
3449 ret
= uverbs_response(attrs
, &resp
, sizeof(resp
));
3453 if (cmd
->srq_type
== IB_SRQT_XRC
)
3454 uobj_put_read(xrcd_uobj
);
3456 if (ib_srq_has_cq(cmd
->srq_type
))
3457 uobj_put_obj_read(attr
.ext
.cq
);
3459 uobj_put_obj_read(pd
);
3460 return uobj_alloc_commit(&obj
->uevent
.uobject
, attrs
);
3463 ib_destroy_srq_user(srq
, &attrs
->driver_udata
);
3466 uobj_put_obj_read(pd
);
3469 if (ib_srq_has_cq(cmd
->srq_type
))
3470 uobj_put_obj_read(attr
.ext
.cq
);
3473 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3474 atomic_dec(&obj
->uxrcd
->refcnt
);
3475 uobj_put_read(xrcd_uobj
);
3479 uobj_alloc_abort(&obj
->uevent
.uobject
, attrs
);
3483 static int ib_uverbs_create_srq(struct uverbs_attr_bundle
*attrs
)
3485 struct ib_uverbs_create_srq cmd
;
3486 struct ib_uverbs_create_xsrq xcmd
;
3489 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3493 memset(&xcmd
, 0, sizeof(xcmd
));
3494 xcmd
.response
= cmd
.response
;
3495 xcmd
.user_handle
= cmd
.user_handle
;
3496 xcmd
.srq_type
= IB_SRQT_BASIC
;
3497 xcmd
.pd_handle
= cmd
.pd_handle
;
3498 xcmd
.max_wr
= cmd
.max_wr
;
3499 xcmd
.max_sge
= cmd
.max_sge
;
3500 xcmd
.srq_limit
= cmd
.srq_limit
;
3502 return __uverbs_create_xsrq(attrs
, &xcmd
, &attrs
->driver_udata
);
3505 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle
*attrs
)
3507 struct ib_uverbs_create_xsrq cmd
;
3510 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3514 return __uverbs_create_xsrq(attrs
, &cmd
, &attrs
->driver_udata
);
3517 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle
*attrs
)
3519 struct ib_uverbs_modify_srq cmd
;
3521 struct ib_srq_attr attr
;
3524 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3528 srq
= uobj_get_obj_read(srq
, UVERBS_OBJECT_SRQ
, cmd
.srq_handle
, attrs
);
3532 attr
.max_wr
= cmd
.max_wr
;
3533 attr
.srq_limit
= cmd
.srq_limit
;
3535 ret
= srq
->device
->ops
.modify_srq(srq
, &attr
, cmd
.attr_mask
,
3536 &attrs
->driver_udata
);
3538 uobj_put_obj_read(srq
);
3543 static int ib_uverbs_query_srq(struct uverbs_attr_bundle
*attrs
)
3545 struct ib_uverbs_query_srq cmd
;
3546 struct ib_uverbs_query_srq_resp resp
;
3547 struct ib_srq_attr attr
;
3551 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3555 srq
= uobj_get_obj_read(srq
, UVERBS_OBJECT_SRQ
, cmd
.srq_handle
, attrs
);
3559 ret
= ib_query_srq(srq
, &attr
);
3561 uobj_put_obj_read(srq
);
3566 memset(&resp
, 0, sizeof resp
);
3568 resp
.max_wr
= attr
.max_wr
;
3569 resp
.max_sge
= attr
.max_sge
;
3570 resp
.srq_limit
= attr
.srq_limit
;
3572 return uverbs_response(attrs
, &resp
, sizeof(resp
));
3575 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle
*attrs
)
3577 struct ib_uverbs_destroy_srq cmd
;
3578 struct ib_uverbs_destroy_srq_resp resp
;
3579 struct ib_uobject
*uobj
;
3580 struct ib_uevent_object
*obj
;
3583 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3587 uobj
= uobj_get_destroy(UVERBS_OBJECT_SRQ
, cmd
.srq_handle
, attrs
);
3589 return PTR_ERR(uobj
);
3591 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
3592 memset(&resp
, 0, sizeof(resp
));
3593 resp
.events_reported
= obj
->events_reported
;
3595 uobj_put_destroy(uobj
);
3597 return uverbs_response(attrs
, &resp
, sizeof(resp
));
3600 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle
*attrs
)
3602 struct ib_uverbs_ex_query_device_resp resp
= {};
3603 struct ib_uverbs_ex_query_device cmd
;
3604 struct ib_device_attr attr
= {0};
3605 struct ib_ucontext
*ucontext
;
3606 struct ib_device
*ib_dev
;
3609 ucontext
= ib_uverbs_get_ucontext(attrs
);
3610 if (IS_ERR(ucontext
))
3611 return PTR_ERR(ucontext
);
3612 ib_dev
= ucontext
->device
;
3614 err
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3624 err
= ib_dev
->ops
.query_device(ib_dev
, &attr
, &attrs
->driver_udata
);
3628 copy_query_dev_fields(ucontext
, &resp
.base
, &attr
);
3630 resp
.odp_caps
.general_caps
= attr
.odp_caps
.general_caps
;
3631 resp
.odp_caps
.per_transport_caps
.rc_odp_caps
=
3632 attr
.odp_caps
.per_transport_caps
.rc_odp_caps
;
3633 resp
.odp_caps
.per_transport_caps
.uc_odp_caps
=
3634 attr
.odp_caps
.per_transport_caps
.uc_odp_caps
;
3635 resp
.odp_caps
.per_transport_caps
.ud_odp_caps
=
3636 attr
.odp_caps
.per_transport_caps
.ud_odp_caps
;
3637 resp
.xrc_odp_caps
= attr
.odp_caps
.per_transport_caps
.xrc_odp_caps
;
3639 resp
.timestamp_mask
= attr
.timestamp_mask
;
3640 resp
.hca_core_clock
= attr
.hca_core_clock
;
3641 resp
.device_cap_flags_ex
= attr
.device_cap_flags
;
3642 resp
.rss_caps
.supported_qpts
= attr
.rss_caps
.supported_qpts
;
3643 resp
.rss_caps
.max_rwq_indirection_tables
=
3644 attr
.rss_caps
.max_rwq_indirection_tables
;
3645 resp
.rss_caps
.max_rwq_indirection_table_size
=
3646 attr
.rss_caps
.max_rwq_indirection_table_size
;
3647 resp
.max_wq_type_rq
= attr
.max_wq_type_rq
;
3648 resp
.raw_packet_caps
= attr
.raw_packet_caps
;
3649 resp
.tm_caps
.max_rndv_hdr_size
= attr
.tm_caps
.max_rndv_hdr_size
;
3650 resp
.tm_caps
.max_num_tags
= attr
.tm_caps
.max_num_tags
;
3651 resp
.tm_caps
.max_ops
= attr
.tm_caps
.max_ops
;
3652 resp
.tm_caps
.max_sge
= attr
.tm_caps
.max_sge
;
3653 resp
.tm_caps
.flags
= attr
.tm_caps
.flags
;
3654 resp
.cq_moderation_caps
.max_cq_moderation_count
=
3655 attr
.cq_caps
.max_cq_moderation_count
;
3656 resp
.cq_moderation_caps
.max_cq_moderation_period
=
3657 attr
.cq_caps
.max_cq_moderation_period
;
3658 resp
.max_dm_size
= attr
.max_dm_size
;
3659 resp
.response_length
= uverbs_response_length(attrs
, sizeof(resp
));
3661 return uverbs_response(attrs
, &resp
, sizeof(resp
));
3664 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle
*attrs
)
3666 struct ib_uverbs_ex_modify_cq cmd
;
3670 ret
= uverbs_request(attrs
, &cmd
, sizeof(cmd
));
3674 if (!cmd
.attr_mask
|| cmd
.reserved
)
3677 if (cmd
.attr_mask
> IB_CQ_MODERATE
)
3680 cq
= uobj_get_obj_read(cq
, UVERBS_OBJECT_CQ
, cmd
.cq_handle
, attrs
);
3684 ret
= rdma_set_cq_moderation(cq
, cmd
.attr
.cq_count
, cmd
.attr
.cq_period
);
3686 uobj_put_obj_read(cq
);
3692 * Describe the input structs for write(). Some write methods have an input
3693 * only struct, most have an input and output. If the struct has an output then
3694 * the 'response' u64 must be the first field in the request structure.
3696 * If udata is present then both the request and response structs have a
3697 * trailing driver_data flex array. In this case the size of the base struct
3698 * cannot be changed.
3700 #define offsetof_after(_struct, _member) \
3701 (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
3703 #define UAPI_DEF_WRITE_IO(req, resp) \
3704 .write.has_resp = 1 + \
3705 BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
3706 BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) != \
3708 .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3710 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3712 #define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
3713 UAPI_DEF_WRITE_IO(req, resp), \
3714 .write.has_udata = \
3716 BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3718 BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
3721 #define UAPI_DEF_WRITE_UDATA_I(req) \
3722 UAPI_DEF_WRITE_I(req), \
3723 .write.has_udata = \
3724 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3728 * The _EX versions are for use with WRITE_EX and allow the last struct member
3729 * to be specified. Buffers that do not include that member will be rejected.
3731 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
3732 .write.has_resp = 1, \
3733 .write.req_size = offsetof_after(req, req_last_member), \
3734 .write.resp_size = offsetof_after(resp, resp_last_member)
3736 #define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
3737 .write.req_size = offsetof_after(req, req_last_member)
3739 const struct uapi_definition uverbs_def_write_intf
[] = {
3740 DECLARE_UVERBS_OBJECT(
3742 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH
,
3743 ib_uverbs_create_ah
,
3744 UAPI_DEF_WRITE_UDATA_IO(
3745 struct ib_uverbs_create_ah
,
3746 struct ib_uverbs_create_ah_resp
),
3747 UAPI_DEF_METHOD_NEEDS_FN(create_ah
)),
3748 DECLARE_UVERBS_WRITE(
3749 IB_USER_VERBS_CMD_DESTROY_AH
,
3750 ib_uverbs_destroy_ah
,
3751 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah
),
3752 UAPI_DEF_METHOD_NEEDS_FN(destroy_ah
))),
3754 DECLARE_UVERBS_OBJECT(
3755 UVERBS_OBJECT_COMP_CHANNEL
,
3756 DECLARE_UVERBS_WRITE(
3757 IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
,
3758 ib_uverbs_create_comp_channel
,
3760 struct ib_uverbs_create_comp_channel
,
3761 struct ib_uverbs_create_comp_channel_resp
))),
3763 DECLARE_UVERBS_OBJECT(
3765 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ
,
3766 ib_uverbs_create_cq
,
3767 UAPI_DEF_WRITE_UDATA_IO(
3768 struct ib_uverbs_create_cq
,
3769 struct ib_uverbs_create_cq_resp
),
3770 UAPI_DEF_METHOD_NEEDS_FN(create_cq
)),
3771 DECLARE_UVERBS_WRITE(
3772 IB_USER_VERBS_CMD_DESTROY_CQ
,
3773 ib_uverbs_destroy_cq
,
3774 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq
,
3775 struct ib_uverbs_destroy_cq_resp
),
3776 UAPI_DEF_METHOD_NEEDS_FN(destroy_cq
)),
3777 DECLARE_UVERBS_WRITE(
3778 IB_USER_VERBS_CMD_POLL_CQ
,
3780 UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq
,
3781 struct ib_uverbs_poll_cq_resp
),
3782 UAPI_DEF_METHOD_NEEDS_FN(poll_cq
)),
3783 DECLARE_UVERBS_WRITE(
3784 IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
,
3785 ib_uverbs_req_notify_cq
,
3786 UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq
),
3787 UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq
)),
3788 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ
,
3789 ib_uverbs_resize_cq
,
3790 UAPI_DEF_WRITE_UDATA_IO(
3791 struct ib_uverbs_resize_cq
,
3792 struct ib_uverbs_resize_cq_resp
),
3793 UAPI_DEF_METHOD_NEEDS_FN(resize_cq
)),
3794 DECLARE_UVERBS_WRITE_EX(
3795 IB_USER_VERBS_EX_CMD_CREATE_CQ
,
3796 ib_uverbs_ex_create_cq
,
3797 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq
,
3799 struct ib_uverbs_ex_create_cq_resp
,
3801 UAPI_DEF_METHOD_NEEDS_FN(create_cq
)),
3802 DECLARE_UVERBS_WRITE_EX(
3803 IB_USER_VERBS_EX_CMD_MODIFY_CQ
,
3804 ib_uverbs_ex_modify_cq
,
3805 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq
),
3806 UAPI_DEF_METHOD_NEEDS_FN(create_cq
))),
3808 DECLARE_UVERBS_OBJECT(
3809 UVERBS_OBJECT_DEVICE
,
3810 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT
,
3811 ib_uverbs_get_context
,
3812 UAPI_DEF_WRITE_UDATA_IO(
3813 struct ib_uverbs_get_context
,
3814 struct ib_uverbs_get_context_resp
)),
3815 DECLARE_UVERBS_WRITE(
3816 IB_USER_VERBS_CMD_QUERY_DEVICE
,
3817 ib_uverbs_query_device
,
3818 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device
,
3819 struct ib_uverbs_query_device_resp
)),
3820 DECLARE_UVERBS_WRITE(
3821 IB_USER_VERBS_CMD_QUERY_PORT
,
3822 ib_uverbs_query_port
,
3823 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port
,
3824 struct ib_uverbs_query_port_resp
),
3825 UAPI_DEF_METHOD_NEEDS_FN(query_port
)),
3826 DECLARE_UVERBS_WRITE_EX(
3827 IB_USER_VERBS_EX_CMD_QUERY_DEVICE
,
3828 ib_uverbs_ex_query_device
,
3829 UAPI_DEF_WRITE_IO_EX(
3830 struct ib_uverbs_ex_query_device
,
3832 struct ib_uverbs_ex_query_device_resp
,
3834 UAPI_DEF_METHOD_NEEDS_FN(query_device
)),
3835 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext
),
3836 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext
)),
3838 DECLARE_UVERBS_OBJECT(
3840 DECLARE_UVERBS_WRITE_EX(
3841 IB_USER_VERBS_EX_CMD_CREATE_FLOW
,
3842 ib_uverbs_ex_create_flow
,
3843 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow
,
3845 struct ib_uverbs_create_flow_resp
,
3847 UAPI_DEF_METHOD_NEEDS_FN(create_flow
)),
3848 DECLARE_UVERBS_WRITE_EX(
3849 IB_USER_VERBS_EX_CMD_DESTROY_FLOW
,
3850 ib_uverbs_ex_destroy_flow
,
3851 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow
),
3852 UAPI_DEF_METHOD_NEEDS_FN(destroy_flow
))),
3854 DECLARE_UVERBS_OBJECT(
3856 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR
,
3858 UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr
),
3859 UAPI_DEF_METHOD_NEEDS_FN(dereg_mr
)),
3860 DECLARE_UVERBS_WRITE(
3861 IB_USER_VERBS_CMD_REG_MR
,
3863 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr
,
3864 struct ib_uverbs_reg_mr_resp
),
3865 UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr
)),
3866 DECLARE_UVERBS_WRITE(
3867 IB_USER_VERBS_CMD_REREG_MR
,
3869 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr
,
3870 struct ib_uverbs_rereg_mr_resp
),
3871 UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr
))),
3873 DECLARE_UVERBS_OBJECT(
3875 DECLARE_UVERBS_WRITE(
3876 IB_USER_VERBS_CMD_ALLOC_MW
,
3878 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw
,
3879 struct ib_uverbs_alloc_mw_resp
),
3880 UAPI_DEF_METHOD_NEEDS_FN(alloc_mw
)),
3881 DECLARE_UVERBS_WRITE(
3882 IB_USER_VERBS_CMD_DEALLOC_MW
,
3883 ib_uverbs_dealloc_mw
,
3884 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw
),
3885 UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw
))),
3887 DECLARE_UVERBS_OBJECT(
3889 DECLARE_UVERBS_WRITE(
3890 IB_USER_VERBS_CMD_ALLOC_PD
,
3892 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd
,
3893 struct ib_uverbs_alloc_pd_resp
),
3894 UAPI_DEF_METHOD_NEEDS_FN(alloc_pd
)),
3895 DECLARE_UVERBS_WRITE(
3896 IB_USER_VERBS_CMD_DEALLOC_PD
,
3897 ib_uverbs_dealloc_pd
,
3898 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd
),
3899 UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd
))),
3901 DECLARE_UVERBS_OBJECT(
3903 DECLARE_UVERBS_WRITE(
3904 IB_USER_VERBS_CMD_ATTACH_MCAST
,
3905 ib_uverbs_attach_mcast
,
3906 UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast
),
3907 UAPI_DEF_METHOD_NEEDS_FN(attach_mcast
),
3908 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast
)),
3909 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP
,
3910 ib_uverbs_create_qp
,
3911 UAPI_DEF_WRITE_UDATA_IO(
3912 struct ib_uverbs_create_qp
,
3913 struct ib_uverbs_create_qp_resp
),
3914 UAPI_DEF_METHOD_NEEDS_FN(create_qp
)),
3915 DECLARE_UVERBS_WRITE(
3916 IB_USER_VERBS_CMD_DESTROY_QP
,
3917 ib_uverbs_destroy_qp
,
3918 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp
,
3919 struct ib_uverbs_destroy_qp_resp
),
3920 UAPI_DEF_METHOD_NEEDS_FN(destroy_qp
)),
3921 DECLARE_UVERBS_WRITE(
3922 IB_USER_VERBS_CMD_DETACH_MCAST
,
3923 ib_uverbs_detach_mcast
,
3924 UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast
),
3925 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast
)),
3926 DECLARE_UVERBS_WRITE(
3927 IB_USER_VERBS_CMD_MODIFY_QP
,
3928 ib_uverbs_modify_qp
,
3929 UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp
),
3930 UAPI_DEF_METHOD_NEEDS_FN(modify_qp
)),
3931 DECLARE_UVERBS_WRITE(
3932 IB_USER_VERBS_CMD_POST_RECV
,
3933 ib_uverbs_post_recv
,
3934 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv
,
3935 struct ib_uverbs_post_recv_resp
),
3936 UAPI_DEF_METHOD_NEEDS_FN(post_recv
)),
3937 DECLARE_UVERBS_WRITE(
3938 IB_USER_VERBS_CMD_POST_SEND
,
3939 ib_uverbs_post_send
,
3940 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send
,
3941 struct ib_uverbs_post_send_resp
),
3942 UAPI_DEF_METHOD_NEEDS_FN(post_send
)),
3943 DECLARE_UVERBS_WRITE(
3944 IB_USER_VERBS_CMD_QUERY_QP
,
3946 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp
,
3947 struct ib_uverbs_query_qp_resp
),
3948 UAPI_DEF_METHOD_NEEDS_FN(query_qp
)),
3949 DECLARE_UVERBS_WRITE_EX(
3950 IB_USER_VERBS_EX_CMD_CREATE_QP
,
3951 ib_uverbs_ex_create_qp
,
3952 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp
,
3954 struct ib_uverbs_ex_create_qp_resp
,
3956 UAPI_DEF_METHOD_NEEDS_FN(create_qp
)),
3957 DECLARE_UVERBS_WRITE_EX(
3958 IB_USER_VERBS_EX_CMD_MODIFY_QP
,
3959 ib_uverbs_ex_modify_qp
,
3960 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp
,
3962 struct ib_uverbs_ex_modify_qp_resp
,
3964 UAPI_DEF_METHOD_NEEDS_FN(modify_qp
))),
3966 DECLARE_UVERBS_OBJECT(
3967 UVERBS_OBJECT_RWQ_IND_TBL
,
3968 DECLARE_UVERBS_WRITE_EX(
3969 IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL
,
3970 ib_uverbs_ex_create_rwq_ind_table
,
3971 UAPI_DEF_WRITE_IO_EX(
3972 struct ib_uverbs_ex_create_rwq_ind_table
,
3974 struct ib_uverbs_ex_create_rwq_ind_table_resp
,
3976 UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table
)),
3977 DECLARE_UVERBS_WRITE_EX(
3978 IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL
,
3979 ib_uverbs_ex_destroy_rwq_ind_table
,
3981 struct ib_uverbs_ex_destroy_rwq_ind_table
),
3982 UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table
))),
3984 DECLARE_UVERBS_OBJECT(
3986 DECLARE_UVERBS_WRITE_EX(
3987 IB_USER_VERBS_EX_CMD_CREATE_WQ
,
3988 ib_uverbs_ex_create_wq
,
3989 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq
,
3991 struct ib_uverbs_ex_create_wq_resp
,
3993 UAPI_DEF_METHOD_NEEDS_FN(create_wq
)),
3994 DECLARE_UVERBS_WRITE_EX(
3995 IB_USER_VERBS_EX_CMD_DESTROY_WQ
,
3996 ib_uverbs_ex_destroy_wq
,
3997 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq
,
3999 struct ib_uverbs_ex_destroy_wq_resp
,
4001 UAPI_DEF_METHOD_NEEDS_FN(destroy_wq
)),
4002 DECLARE_UVERBS_WRITE_EX(
4003 IB_USER_VERBS_EX_CMD_MODIFY_WQ
,
4004 ib_uverbs_ex_modify_wq
,
4005 UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq
,
4007 UAPI_DEF_METHOD_NEEDS_FN(modify_wq
))),
4009 DECLARE_UVERBS_OBJECT(
4011 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ
,
4012 ib_uverbs_create_srq
,
4013 UAPI_DEF_WRITE_UDATA_IO(
4014 struct ib_uverbs_create_srq
,
4015 struct ib_uverbs_create_srq_resp
),
4016 UAPI_DEF_METHOD_NEEDS_FN(create_srq
)),
4017 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ
,
4018 ib_uverbs_create_xsrq
,
4019 UAPI_DEF_WRITE_UDATA_IO(
4020 struct ib_uverbs_create_xsrq
,
4021 struct ib_uverbs_create_srq_resp
),
4022 UAPI_DEF_METHOD_NEEDS_FN(create_srq
)),
4023 DECLARE_UVERBS_WRITE(
4024 IB_USER_VERBS_CMD_DESTROY_SRQ
,
4025 ib_uverbs_destroy_srq
,
4026 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq
,
4027 struct ib_uverbs_destroy_srq_resp
),
4028 UAPI_DEF_METHOD_NEEDS_FN(destroy_srq
)),
4029 DECLARE_UVERBS_WRITE(
4030 IB_USER_VERBS_CMD_MODIFY_SRQ
,
4031 ib_uverbs_modify_srq
,
4032 UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq
),
4033 UAPI_DEF_METHOD_NEEDS_FN(modify_srq
)),
4034 DECLARE_UVERBS_WRITE(
4035 IB_USER_VERBS_CMD_POST_SRQ_RECV
,
4036 ib_uverbs_post_srq_recv
,
4037 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv
,
4038 struct ib_uverbs_post_srq_recv_resp
),
4039 UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv
)),
4040 DECLARE_UVERBS_WRITE(
4041 IB_USER_VERBS_CMD_QUERY_SRQ
,
4042 ib_uverbs_query_srq
,
4043 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq
,
4044 struct ib_uverbs_query_srq_resp
),
4045 UAPI_DEF_METHOD_NEEDS_FN(query_srq
))),
4047 DECLARE_UVERBS_OBJECT(
4049 DECLARE_UVERBS_WRITE(
4050 IB_USER_VERBS_CMD_CLOSE_XRCD
,
4051 ib_uverbs_close_xrcd
,
4052 UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd
),
4053 UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd
)),
4054 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP
,
4056 UAPI_DEF_WRITE_UDATA_IO(
4057 struct ib_uverbs_open_qp
,
4058 struct ib_uverbs_create_qp_resp
)),
4059 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD
,
4060 ib_uverbs_open_xrcd
,
4061 UAPI_DEF_WRITE_UDATA_IO(
4062 struct ib_uverbs_open_xrcd
,
4063 struct ib_uverbs_open_xrcd_resp
),
4064 UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd
))),