2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
41 #include <linux/uaccess.h>
43 #include <rdma/uverbs_types.h>
44 #include <rdma/uverbs_std_types.h>
45 #include "rdma_core.h"
48 #include "core_priv.h"
50 static struct ib_uverbs_completion_event_file
*
51 ib_uverbs_lookup_comp_file(int fd
, struct ib_ucontext
*context
)
53 struct ib_uobject
*uobj
= uobj_get_read(uobj_get_type(comp_channel
),
55 struct ib_uobject_file
*uobj_file
;
60 uverbs_uobject_get(uobj
);
63 uobj_file
= container_of(uobj
, struct ib_uobject_file
, uobj
);
64 return container_of(uobj_file
, struct ib_uverbs_completion_event_file
,
68 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
69 struct ib_device
*ib_dev
,
70 const char __user
*buf
,
71 int in_len
, int out_len
)
73 struct ib_uverbs_get_context cmd
;
74 struct ib_uverbs_get_context_resp resp
;
75 struct ib_udata udata
;
76 struct ib_ucontext
*ucontext
;
78 struct ib_rdmacg_object cg_obj
;
81 if (out_len
< sizeof resp
)
84 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
87 mutex_lock(&file
->mutex
);
94 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
95 (unsigned long) cmd
.response
+ sizeof resp
,
96 in_len
- sizeof cmd
, out_len
- sizeof resp
);
98 ret
= ib_rdmacg_try_charge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
102 ucontext
= ib_dev
->alloc_ucontext(ib_dev
, &udata
);
103 if (IS_ERR(ucontext
)) {
104 ret
= PTR_ERR(ucontext
);
108 ucontext
->device
= ib_dev
;
109 ucontext
->cg_obj
= cg_obj
;
110 /* ufile is required when some objects are released */
111 ucontext
->ufile
= file
;
112 uverbs_initialize_ucontext(ucontext
);
115 ucontext
->tgid
= get_task_pid(current
->group_leader
, PIDTYPE_PID
);
117 ucontext
->closing
= 0;
119 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
120 ucontext
->umem_tree
= RB_ROOT
;
121 init_rwsem(&ucontext
->umem_rwsem
);
122 ucontext
->odp_mrs_count
= 0;
123 INIT_LIST_HEAD(&ucontext
->no_private_counters
);
125 if (!(ib_dev
->attrs
.device_cap_flags
& IB_DEVICE_ON_DEMAND_PAGING
))
126 ucontext
->invalidate_range
= NULL
;
130 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
132 ret
= get_unused_fd_flags(O_CLOEXEC
);
137 filp
= ib_uverbs_alloc_async_event_file(file
, ib_dev
);
143 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
144 &resp
, sizeof resp
)) {
149 file
->ucontext
= ucontext
;
151 fd_install(resp
.async_fd
, filp
);
153 mutex_unlock(&file
->mutex
);
158 ib_uverbs_free_async_event_file(file
);
162 put_unused_fd(resp
.async_fd
);
165 put_pid(ucontext
->tgid
);
166 ib_dev
->dealloc_ucontext(ucontext
);
169 ib_rdmacg_uncharge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
172 mutex_unlock(&file
->mutex
);
176 static void copy_query_dev_fields(struct ib_uverbs_file
*file
,
177 struct ib_device
*ib_dev
,
178 struct ib_uverbs_query_device_resp
*resp
,
179 struct ib_device_attr
*attr
)
181 resp
->fw_ver
= attr
->fw_ver
;
182 resp
->node_guid
= ib_dev
->node_guid
;
183 resp
->sys_image_guid
= attr
->sys_image_guid
;
184 resp
->max_mr_size
= attr
->max_mr_size
;
185 resp
->page_size_cap
= attr
->page_size_cap
;
186 resp
->vendor_id
= attr
->vendor_id
;
187 resp
->vendor_part_id
= attr
->vendor_part_id
;
188 resp
->hw_ver
= attr
->hw_ver
;
189 resp
->max_qp
= attr
->max_qp
;
190 resp
->max_qp_wr
= attr
->max_qp_wr
;
191 resp
->device_cap_flags
= lower_32_bits(attr
->device_cap_flags
);
192 resp
->max_sge
= attr
->max_sge
;
193 resp
->max_sge_rd
= attr
->max_sge_rd
;
194 resp
->max_cq
= attr
->max_cq
;
195 resp
->max_cqe
= attr
->max_cqe
;
196 resp
->max_mr
= attr
->max_mr
;
197 resp
->max_pd
= attr
->max_pd
;
198 resp
->max_qp_rd_atom
= attr
->max_qp_rd_atom
;
199 resp
->max_ee_rd_atom
= attr
->max_ee_rd_atom
;
200 resp
->max_res_rd_atom
= attr
->max_res_rd_atom
;
201 resp
->max_qp_init_rd_atom
= attr
->max_qp_init_rd_atom
;
202 resp
->max_ee_init_rd_atom
= attr
->max_ee_init_rd_atom
;
203 resp
->atomic_cap
= attr
->atomic_cap
;
204 resp
->max_ee
= attr
->max_ee
;
205 resp
->max_rdd
= attr
->max_rdd
;
206 resp
->max_mw
= attr
->max_mw
;
207 resp
->max_raw_ipv6_qp
= attr
->max_raw_ipv6_qp
;
208 resp
->max_raw_ethy_qp
= attr
->max_raw_ethy_qp
;
209 resp
->max_mcast_grp
= attr
->max_mcast_grp
;
210 resp
->max_mcast_qp_attach
= attr
->max_mcast_qp_attach
;
211 resp
->max_total_mcast_qp_attach
= attr
->max_total_mcast_qp_attach
;
212 resp
->max_ah
= attr
->max_ah
;
213 resp
->max_fmr
= attr
->max_fmr
;
214 resp
->max_map_per_fmr
= attr
->max_map_per_fmr
;
215 resp
->max_srq
= attr
->max_srq
;
216 resp
->max_srq_wr
= attr
->max_srq_wr
;
217 resp
->max_srq_sge
= attr
->max_srq_sge
;
218 resp
->max_pkeys
= attr
->max_pkeys
;
219 resp
->local_ca_ack_delay
= attr
->local_ca_ack_delay
;
220 resp
->phys_port_cnt
= ib_dev
->phys_port_cnt
;
223 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
224 struct ib_device
*ib_dev
,
225 const char __user
*buf
,
226 int in_len
, int out_len
)
228 struct ib_uverbs_query_device cmd
;
229 struct ib_uverbs_query_device_resp resp
;
231 if (out_len
< sizeof resp
)
234 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
237 memset(&resp
, 0, sizeof resp
);
238 copy_query_dev_fields(file
, ib_dev
, &resp
, &ib_dev
->attrs
);
240 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
247 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
248 struct ib_device
*ib_dev
,
249 const char __user
*buf
,
250 int in_len
, int out_len
)
252 struct ib_uverbs_query_port cmd
;
253 struct ib_uverbs_query_port_resp resp
;
254 struct ib_port_attr attr
;
257 if (out_len
< sizeof resp
)
260 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
263 ret
= ib_query_port(ib_dev
, cmd
.port_num
, &attr
);
267 memset(&resp
, 0, sizeof resp
);
269 resp
.state
= attr
.state
;
270 resp
.max_mtu
= attr
.max_mtu
;
271 resp
.active_mtu
= attr
.active_mtu
;
272 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
273 resp
.port_cap_flags
= attr
.port_cap_flags
;
274 resp
.max_msg_sz
= attr
.max_msg_sz
;
275 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
276 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
277 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
279 resp
.sm_lid
= attr
.sm_lid
;
281 resp
.max_vl_num
= attr
.max_vl_num
;
282 resp
.sm_sl
= attr
.sm_sl
;
283 resp
.subnet_timeout
= attr
.subnet_timeout
;
284 resp
.init_type_reply
= attr
.init_type_reply
;
285 resp
.active_width
= attr
.active_width
;
286 resp
.active_speed
= attr
.active_speed
;
287 resp
.phys_state
= attr
.phys_state
;
288 resp
.link_layer
= rdma_port_get_link_layer(ib_dev
,
291 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
298 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
299 struct ib_device
*ib_dev
,
300 const char __user
*buf
,
301 int in_len
, int out_len
)
303 struct ib_uverbs_alloc_pd cmd
;
304 struct ib_uverbs_alloc_pd_resp resp
;
305 struct ib_udata udata
;
306 struct ib_uobject
*uobj
;
310 if (out_len
< sizeof resp
)
313 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
316 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
317 (unsigned long) cmd
.response
+ sizeof resp
,
318 in_len
- sizeof cmd
, out_len
- sizeof resp
);
320 uobj
= uobj_alloc(uobj_get_type(pd
), file
->ucontext
);
322 return PTR_ERR(uobj
);
324 pd
= ib_dev
->alloc_pd(ib_dev
, file
->ucontext
, &udata
);
332 pd
->__internal_mr
= NULL
;
333 atomic_set(&pd
->usecnt
, 0);
336 memset(&resp
, 0, sizeof resp
);
337 resp
.pd_handle
= uobj
->id
;
339 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
340 &resp
, sizeof resp
)) {
345 uobj_alloc_commit(uobj
);
353 uobj_alloc_abort(uobj
);
357 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
358 struct ib_device
*ib_dev
,
359 const char __user
*buf
,
360 int in_len
, int out_len
)
362 struct ib_uverbs_dealloc_pd cmd
;
363 struct ib_uobject
*uobj
;
366 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
369 uobj
= uobj_get_write(uobj_get_type(pd
), cmd
.pd_handle
,
372 return PTR_ERR(uobj
);
374 ret
= uobj_remove_commit(uobj
);
376 return ret
?: in_len
;
379 struct xrcd_table_entry
{
381 struct ib_xrcd
*xrcd
;
385 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
387 struct ib_xrcd
*xrcd
)
389 struct xrcd_table_entry
*entry
, *scan
;
390 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
391 struct rb_node
*parent
= NULL
;
393 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
398 entry
->inode
= inode
;
402 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
404 if (inode
< scan
->inode
) {
406 } else if (inode
> scan
->inode
) {
414 rb_link_node(&entry
->node
, parent
, p
);
415 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
420 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
423 struct xrcd_table_entry
*entry
;
424 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
427 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
429 if (inode
< entry
->inode
)
431 else if (inode
> entry
->inode
)
440 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
442 struct xrcd_table_entry
*entry
;
444 entry
= xrcd_table_search(dev
, inode
);
451 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
454 struct xrcd_table_entry
*entry
;
456 entry
= xrcd_table_search(dev
, inode
);
459 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
464 ssize_t
ib_uverbs_open_xrcd(struct ib_uverbs_file
*file
,
465 struct ib_device
*ib_dev
,
466 const char __user
*buf
, int in_len
,
469 struct ib_uverbs_open_xrcd cmd
;
470 struct ib_uverbs_open_xrcd_resp resp
;
471 struct ib_udata udata
;
472 struct ib_uxrcd_object
*obj
;
473 struct ib_xrcd
*xrcd
= NULL
;
474 struct fd f
= {NULL
, 0};
475 struct inode
*inode
= NULL
;
479 if (out_len
< sizeof resp
)
482 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
485 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
486 (unsigned long) cmd
.response
+ sizeof resp
,
487 in_len
- sizeof cmd
, out_len
- sizeof resp
);
489 mutex_lock(&file
->device
->xrcd_tree_mutex
);
492 /* search for file descriptor */
496 goto err_tree_mutex_unlock
;
499 inode
= file_inode(f
.file
);
500 xrcd
= find_xrcd(file
->device
, inode
);
501 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
502 /* no file descriptor. Need CREATE flag */
504 goto err_tree_mutex_unlock
;
507 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
509 goto err_tree_mutex_unlock
;
513 obj
= (struct ib_uxrcd_object
*)uobj_alloc(uobj_get_type(xrcd
),
517 goto err_tree_mutex_unlock
;
521 xrcd
= ib_dev
->alloc_xrcd(ib_dev
, file
->ucontext
, &udata
);
528 xrcd
->device
= ib_dev
;
529 atomic_set(&xrcd
->usecnt
, 0);
530 mutex_init(&xrcd
->tgt_qp_mutex
);
531 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
535 atomic_set(&obj
->refcnt
, 0);
536 obj
->uobject
.object
= xrcd
;
537 memset(&resp
, 0, sizeof resp
);
538 resp
.xrcd_handle
= obj
->uobject
.id
;
542 /* create new inode/xrcd table entry */
543 ret
= xrcd_table_insert(file
->device
, inode
, xrcd
);
545 goto err_dealloc_xrcd
;
547 atomic_inc(&xrcd
->usecnt
);
550 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
551 &resp
, sizeof resp
)) {
559 uobj_alloc_commit(&obj
->uobject
);
561 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
567 xrcd_table_delete(file
->device
, inode
);
568 atomic_dec(&xrcd
->usecnt
);
572 ib_dealloc_xrcd(xrcd
);
575 uobj_alloc_abort(&obj
->uobject
);
577 err_tree_mutex_unlock
:
581 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
586 ssize_t
ib_uverbs_close_xrcd(struct ib_uverbs_file
*file
,
587 struct ib_device
*ib_dev
,
588 const char __user
*buf
, int in_len
,
591 struct ib_uverbs_close_xrcd cmd
;
592 struct ib_uobject
*uobj
;
595 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
598 uobj
= uobj_get_write(uobj_get_type(xrcd
), cmd
.xrcd_handle
,
601 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
602 return PTR_ERR(uobj
);
605 ret
= uobj_remove_commit(uobj
);
606 return ret
?: in_len
;
609 int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device
*dev
,
610 struct ib_xrcd
*xrcd
,
611 enum rdma_remove_reason why
)
617 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
620 ret
= ib_dealloc_xrcd(xrcd
);
622 if (why
== RDMA_REMOVE_DESTROY
&& ret
)
623 atomic_inc(&xrcd
->usecnt
);
625 xrcd_table_delete(dev
, inode
);
630 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
631 struct ib_device
*ib_dev
,
632 const char __user
*buf
, int in_len
,
635 struct ib_uverbs_reg_mr cmd
;
636 struct ib_uverbs_reg_mr_resp resp
;
637 struct ib_udata udata
;
638 struct ib_uobject
*uobj
;
643 if (out_len
< sizeof resp
)
646 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
649 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
650 (unsigned long) cmd
.response
+ sizeof resp
,
651 in_len
- sizeof cmd
, out_len
- sizeof resp
);
653 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
656 ret
= ib_check_mr_access(cmd
.access_flags
);
660 uobj
= uobj_alloc(uobj_get_type(mr
), file
->ucontext
);
662 return PTR_ERR(uobj
);
664 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
670 if (cmd
.access_flags
& IB_ACCESS_ON_DEMAND
) {
671 if (!(pd
->device
->attrs
.device_cap_flags
&
672 IB_DEVICE_ON_DEMAND_PAGING
)) {
673 pr_debug("ODP support not available\n");
679 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
680 cmd
.access_flags
, &udata
);
686 mr
->device
= pd
->device
;
689 atomic_inc(&pd
->usecnt
);
693 memset(&resp
, 0, sizeof resp
);
694 resp
.lkey
= mr
->lkey
;
695 resp
.rkey
= mr
->rkey
;
696 resp
.mr_handle
= uobj
->id
;
698 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
699 &resp
, sizeof resp
)) {
704 uobj_put_obj_read(pd
);
706 uobj_alloc_commit(uobj
);
714 uobj_put_obj_read(pd
);
717 uobj_alloc_abort(uobj
);
721 ssize_t
ib_uverbs_rereg_mr(struct ib_uverbs_file
*file
,
722 struct ib_device
*ib_dev
,
723 const char __user
*buf
, int in_len
,
726 struct ib_uverbs_rereg_mr cmd
;
727 struct ib_uverbs_rereg_mr_resp resp
;
728 struct ib_udata udata
;
729 struct ib_pd
*pd
= NULL
;
731 struct ib_pd
*old_pd
;
733 struct ib_uobject
*uobj
;
735 if (out_len
< sizeof(resp
))
738 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
741 INIT_UDATA(&udata
, buf
+ sizeof(cmd
),
742 (unsigned long) cmd
.response
+ sizeof(resp
),
743 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
745 if (cmd
.flags
& ~IB_MR_REREG_SUPPORTED
|| !cmd
.flags
)
748 if ((cmd
.flags
& IB_MR_REREG_TRANS
) &&
749 (!cmd
.start
|| !cmd
.hca_va
|| 0 >= cmd
.length
||
750 (cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
)))
753 uobj
= uobj_get_write(uobj_get_type(mr
), cmd
.mr_handle
,
756 return PTR_ERR(uobj
);
760 if (cmd
.flags
& IB_MR_REREG_ACCESS
) {
761 ret
= ib_check_mr_access(cmd
.access_flags
);
766 if (cmd
.flags
& IB_MR_REREG_PD
) {
767 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
775 ret
= mr
->device
->rereg_user_mr(mr
, cmd
.flags
, cmd
.start
,
776 cmd
.length
, cmd
.hca_va
,
777 cmd
.access_flags
, pd
, &udata
);
779 if (cmd
.flags
& IB_MR_REREG_PD
) {
780 atomic_inc(&pd
->usecnt
);
782 atomic_dec(&old_pd
->usecnt
);
788 memset(&resp
, 0, sizeof(resp
));
789 resp
.lkey
= mr
->lkey
;
790 resp
.rkey
= mr
->rkey
;
792 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
793 &resp
, sizeof(resp
)))
799 if (cmd
.flags
& IB_MR_REREG_PD
)
800 uobj_put_obj_read(pd
);
803 uobj_put_write(uobj
);
808 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
809 struct ib_device
*ib_dev
,
810 const char __user
*buf
, int in_len
,
813 struct ib_uverbs_dereg_mr cmd
;
814 struct ib_uobject
*uobj
;
817 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
820 uobj
= uobj_get_write(uobj_get_type(mr
), cmd
.mr_handle
,
823 return PTR_ERR(uobj
);
825 ret
= uobj_remove_commit(uobj
);
827 return ret
?: in_len
;
830 ssize_t
ib_uverbs_alloc_mw(struct ib_uverbs_file
*file
,
831 struct ib_device
*ib_dev
,
832 const char __user
*buf
, int in_len
,
835 struct ib_uverbs_alloc_mw cmd
;
836 struct ib_uverbs_alloc_mw_resp resp
;
837 struct ib_uobject
*uobj
;
840 struct ib_udata udata
;
843 if (out_len
< sizeof(resp
))
846 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
849 uobj
= uobj_alloc(uobj_get_type(mw
), file
->ucontext
);
851 return PTR_ERR(uobj
);
853 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
859 INIT_UDATA(&udata
, buf
+ sizeof(cmd
),
860 (unsigned long)cmd
.response
+ sizeof(resp
),
861 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
862 out_len
- sizeof(resp
));
864 mw
= pd
->device
->alloc_mw(pd
, cmd
.mw_type
, &udata
);
870 mw
->device
= pd
->device
;
873 atomic_inc(&pd
->usecnt
);
877 memset(&resp
, 0, sizeof(resp
));
878 resp
.rkey
= mw
->rkey
;
879 resp
.mw_handle
= uobj
->id
;
881 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
882 &resp
, sizeof(resp
))) {
887 uobj_put_obj_read(pd
);
888 uobj_alloc_commit(uobj
);
893 uverbs_dealloc_mw(mw
);
895 uobj_put_obj_read(pd
);
897 uobj_alloc_abort(uobj
);
901 ssize_t
ib_uverbs_dealloc_mw(struct ib_uverbs_file
*file
,
902 struct ib_device
*ib_dev
,
903 const char __user
*buf
, int in_len
,
906 struct ib_uverbs_dealloc_mw cmd
;
907 struct ib_uobject
*uobj
;
910 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
913 uobj
= uobj_get_write(uobj_get_type(mw
), cmd
.mw_handle
,
916 return PTR_ERR(uobj
);
918 ret
= uobj_remove_commit(uobj
);
919 return ret
?: in_len
;
922 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
923 struct ib_device
*ib_dev
,
924 const char __user
*buf
, int in_len
,
927 struct ib_uverbs_create_comp_channel cmd
;
928 struct ib_uverbs_create_comp_channel_resp resp
;
929 struct ib_uobject
*uobj
;
930 struct ib_uverbs_completion_event_file
*ev_file
;
932 if (out_len
< sizeof resp
)
935 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
938 uobj
= uobj_alloc(uobj_get_type(comp_channel
), file
->ucontext
);
940 return PTR_ERR(uobj
);
944 ev_file
= container_of(uobj
, struct ib_uverbs_completion_event_file
,
946 ib_uverbs_init_event_queue(&ev_file
->ev_queue
);
948 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
949 &resp
, sizeof resp
)) {
950 uobj_alloc_abort(uobj
);
954 uobj_alloc_commit(uobj
);
958 static struct ib_ucq_object
*create_cq(struct ib_uverbs_file
*file
,
959 struct ib_device
*ib_dev
,
960 struct ib_udata
*ucore
,
961 struct ib_udata
*uhw
,
962 struct ib_uverbs_ex_create_cq
*cmd
,
964 int (*cb
)(struct ib_uverbs_file
*file
,
965 struct ib_ucq_object
*obj
,
966 struct ib_uverbs_ex_create_cq_resp
*resp
,
967 struct ib_udata
*udata
,
971 struct ib_ucq_object
*obj
;
972 struct ib_uverbs_completion_event_file
*ev_file
= NULL
;
975 struct ib_uverbs_ex_create_cq_resp resp
;
976 struct ib_cq_init_attr attr
= {};
978 if (cmd
->comp_vector
>= file
->device
->num_comp_vectors
)
979 return ERR_PTR(-EINVAL
);
981 obj
= (struct ib_ucq_object
*)uobj_alloc(uobj_get_type(cq
),
986 if (cmd
->comp_channel
>= 0) {
987 ev_file
= ib_uverbs_lookup_comp_file(cmd
->comp_channel
,
989 if (IS_ERR(ev_file
)) {
990 ret
= PTR_ERR(ev_file
);
995 obj
->uobject
.user_handle
= cmd
->user_handle
;
996 obj
->uverbs_file
= file
;
997 obj
->comp_events_reported
= 0;
998 obj
->async_events_reported
= 0;
999 INIT_LIST_HEAD(&obj
->comp_list
);
1000 INIT_LIST_HEAD(&obj
->async_list
);
1002 attr
.cqe
= cmd
->cqe
;
1003 attr
.comp_vector
= cmd
->comp_vector
;
1005 if (cmd_sz
> offsetof(typeof(*cmd
), flags
) + sizeof(cmd
->flags
))
1006 attr
.flags
= cmd
->flags
;
1008 cq
= ib_dev
->create_cq(ib_dev
, &attr
, file
->ucontext
, uhw
);
1014 cq
->device
= ib_dev
;
1015 cq
->uobject
= &obj
->uobject
;
1016 cq
->comp_handler
= ib_uverbs_comp_handler
;
1017 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1018 cq
->cq_context
= &ev_file
->ev_queue
;
1019 atomic_set(&cq
->usecnt
, 0);
1021 obj
->uobject
.object
= cq
;
1022 memset(&resp
, 0, sizeof resp
);
1023 resp
.base
.cq_handle
= obj
->uobject
.id
;
1024 resp
.base
.cqe
= cq
->cqe
;
1026 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1027 sizeof(resp
.response_length
);
1029 ret
= cb(file
, obj
, &resp
, ucore
, context
);
1033 uobj_alloc_commit(&obj
->uobject
);
1042 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1045 uobj_alloc_abort(&obj
->uobject
);
1047 return ERR_PTR(ret
);
1050 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file
*file
,
1051 struct ib_ucq_object
*obj
,
1052 struct ib_uverbs_ex_create_cq_resp
*resp
,
1053 struct ib_udata
*ucore
, void *context
)
1055 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1061 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
1062 struct ib_device
*ib_dev
,
1063 const char __user
*buf
, int in_len
,
1066 struct ib_uverbs_create_cq cmd
;
1067 struct ib_uverbs_ex_create_cq cmd_ex
;
1068 struct ib_uverbs_create_cq_resp resp
;
1069 struct ib_udata ucore
;
1070 struct ib_udata uhw
;
1071 struct ib_ucq_object
*obj
;
1073 if (out_len
< sizeof(resp
))
1076 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1079 INIT_UDATA(&ucore
, buf
, (unsigned long)cmd
.response
, sizeof(cmd
), sizeof(resp
));
1081 INIT_UDATA(&uhw
, buf
+ sizeof(cmd
),
1082 (unsigned long)cmd
.response
+ sizeof(resp
),
1083 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
1085 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1086 cmd_ex
.user_handle
= cmd
.user_handle
;
1087 cmd_ex
.cqe
= cmd
.cqe
;
1088 cmd_ex
.comp_vector
= cmd
.comp_vector
;
1089 cmd_ex
.comp_channel
= cmd
.comp_channel
;
1091 obj
= create_cq(file
, ib_dev
, &ucore
, &uhw
, &cmd_ex
,
1092 offsetof(typeof(cmd_ex
), comp_channel
) +
1093 sizeof(cmd
.comp_channel
), ib_uverbs_create_cq_cb
,
1097 return PTR_ERR(obj
);
1102 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file
*file
,
1103 struct ib_ucq_object
*obj
,
1104 struct ib_uverbs_ex_create_cq_resp
*resp
,
1105 struct ib_udata
*ucore
, void *context
)
1107 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1113 int ib_uverbs_ex_create_cq(struct ib_uverbs_file
*file
,
1114 struct ib_device
*ib_dev
,
1115 struct ib_udata
*ucore
,
1116 struct ib_udata
*uhw
)
1118 struct ib_uverbs_ex_create_cq_resp resp
;
1119 struct ib_uverbs_ex_create_cq cmd
;
1120 struct ib_ucq_object
*obj
;
1123 if (ucore
->inlen
< sizeof(cmd
))
1126 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
1136 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1137 sizeof(resp
.response_length
)))
1140 obj
= create_cq(file
, ib_dev
, ucore
, uhw
, &cmd
,
1141 min(ucore
->inlen
, sizeof(cmd
)),
1142 ib_uverbs_ex_create_cq_cb
, NULL
);
1145 return PTR_ERR(obj
);
1150 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
1151 struct ib_device
*ib_dev
,
1152 const char __user
*buf
, int in_len
,
1155 struct ib_uverbs_resize_cq cmd
;
1156 struct ib_uverbs_resize_cq_resp resp
;
1157 struct ib_udata udata
;
1161 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1164 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1165 (unsigned long) cmd
.response
+ sizeof resp
,
1166 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1168 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1172 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
1178 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1179 &resp
, sizeof resp
.cqe
))
1183 uobj_put_obj_read(cq
);
1185 return ret
? ret
: in_len
;
1188 static int copy_wc_to_user(void __user
*dest
, struct ib_wc
*wc
)
1190 struct ib_uverbs_wc tmp
;
1192 tmp
.wr_id
= wc
->wr_id
;
1193 tmp
.status
= wc
->status
;
1194 tmp
.opcode
= wc
->opcode
;
1195 tmp
.vendor_err
= wc
->vendor_err
;
1196 tmp
.byte_len
= wc
->byte_len
;
1197 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
1198 tmp
.qp_num
= wc
->qp
->qp_num
;
1199 tmp
.src_qp
= wc
->src_qp
;
1200 tmp
.wc_flags
= wc
->wc_flags
;
1201 tmp
.pkey_index
= wc
->pkey_index
;
1202 tmp
.slid
= wc
->slid
;
1204 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1205 tmp
.port_num
= wc
->port_num
;
1208 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1214 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
1215 struct ib_device
*ib_dev
,
1216 const char __user
*buf
, int in_len
,
1219 struct ib_uverbs_poll_cq cmd
;
1220 struct ib_uverbs_poll_cq_resp resp
;
1221 u8 __user
*header_ptr
;
1222 u8 __user
*data_ptr
;
1227 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1230 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1234 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1235 header_ptr
= (void __user
*)(unsigned long) cmd
.response
;
1236 data_ptr
= header_ptr
+ sizeof resp
;
1238 memset(&resp
, 0, sizeof resp
);
1239 while (resp
.count
< cmd
.ne
) {
1240 ret
= ib_poll_cq(cq
, 1, &wc
);
1246 ret
= copy_wc_to_user(data_ptr
, &wc
);
1250 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1254 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1262 uobj_put_obj_read(cq
);
1266 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
1267 struct ib_device
*ib_dev
,
1268 const char __user
*buf
, int in_len
,
1271 struct ib_uverbs_req_notify_cq cmd
;
1274 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1277 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1281 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1282 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1284 uobj_put_obj_read(cq
);
1289 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
1290 struct ib_device
*ib_dev
,
1291 const char __user
*buf
, int in_len
,
1294 struct ib_uverbs_destroy_cq cmd
;
1295 struct ib_uverbs_destroy_cq_resp resp
;
1296 struct ib_uobject
*uobj
;
1298 struct ib_ucq_object
*obj
;
1299 struct ib_uverbs_event_queue
*ev_queue
;
1302 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1305 uobj
= uobj_get_write(uobj_get_type(cq
), cmd
.cq_handle
,
1308 return PTR_ERR(uobj
);
1311 * Make sure we don't free the memory in remove_commit as we still
1312 * needs the uobject memory to create the response.
1314 uverbs_uobject_get(uobj
);
1316 ev_queue
= cq
->cq_context
;
1317 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1319 memset(&resp
, 0, sizeof(resp
));
1321 ret
= uobj_remove_commit(uobj
);
1323 uverbs_uobject_put(uobj
);
1327 resp
.comp_events_reported
= obj
->comp_events_reported
;
1328 resp
.async_events_reported
= obj
->async_events_reported
;
1330 uverbs_uobject_put(uobj
);
1331 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1332 &resp
, sizeof resp
))
1338 static int create_qp(struct ib_uverbs_file
*file
,
1339 struct ib_udata
*ucore
,
1340 struct ib_udata
*uhw
,
1341 struct ib_uverbs_ex_create_qp
*cmd
,
1343 int (*cb
)(struct ib_uverbs_file
*file
,
1344 struct ib_uverbs_ex_create_qp_resp
*resp
,
1345 struct ib_udata
*udata
),
1348 struct ib_uqp_object
*obj
;
1349 struct ib_device
*device
;
1350 struct ib_pd
*pd
= NULL
;
1351 struct ib_xrcd
*xrcd
= NULL
;
1352 struct ib_uobject
*xrcd_uobj
= ERR_PTR(-ENOENT
);
1353 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1354 struct ib_srq
*srq
= NULL
;
1357 struct ib_qp_init_attr attr
= {};
1358 struct ib_uverbs_ex_create_qp_resp resp
;
1360 struct ib_rwq_ind_table
*ind_tbl
= NULL
;
1363 if (cmd
->qp_type
== IB_QPT_RAW_PACKET
&& !capable(CAP_NET_RAW
))
1366 obj
= (struct ib_uqp_object
*)uobj_alloc(uobj_get_type(qp
),
1369 return PTR_ERR(obj
);
1371 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
1372 mutex_init(&obj
->mcast_lock
);
1374 if (cmd_sz
>= offsetof(typeof(*cmd
), rwq_ind_tbl_handle
) +
1375 sizeof(cmd
->rwq_ind_tbl_handle
) &&
1376 (cmd
->comp_mask
& IB_UVERBS_CREATE_QP_MASK_IND_TABLE
)) {
1377 ind_tbl
= uobj_get_obj_read(rwq_ind_table
,
1378 cmd
->rwq_ind_tbl_handle
,
1385 attr
.rwq_ind_tbl
= ind_tbl
;
1388 if ((cmd_sz
>= offsetof(typeof(*cmd
), reserved1
) +
1389 sizeof(cmd
->reserved1
)) && cmd
->reserved1
) {
1394 if (ind_tbl
&& (cmd
->max_recv_wr
|| cmd
->max_recv_sge
|| cmd
->is_srq
)) {
1399 if (ind_tbl
&& !cmd
->max_send_wr
)
1402 if (cmd
->qp_type
== IB_QPT_XRC_TGT
) {
1403 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
->pd_handle
,
1406 if (IS_ERR(xrcd_uobj
)) {
1411 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1416 device
= xrcd
->device
;
1418 if (cmd
->qp_type
== IB_QPT_XRC_INI
) {
1419 cmd
->max_recv_wr
= 0;
1420 cmd
->max_recv_sge
= 0;
1423 srq
= uobj_get_obj_read(srq
, cmd
->srq_handle
,
1425 if (!srq
|| srq
->srq_type
!= IB_SRQT_BASIC
) {
1432 if (cmd
->recv_cq_handle
!= cmd
->send_cq_handle
) {
1433 rcq
= uobj_get_obj_read(cq
, cmd
->recv_cq_handle
,
1444 scq
= uobj_get_obj_read(cq
, cmd
->send_cq_handle
,
1448 pd
= uobj_get_obj_read(pd
, cmd
->pd_handle
, file
->ucontext
);
1449 if (!pd
|| (!scq
&& has_sq
)) {
1454 device
= pd
->device
;
1457 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1458 attr
.qp_context
= file
;
1463 attr
.sq_sig_type
= cmd
->sq_sig_all
? IB_SIGNAL_ALL_WR
:
1465 attr
.qp_type
= cmd
->qp_type
;
1466 attr
.create_flags
= 0;
1468 attr
.cap
.max_send_wr
= cmd
->max_send_wr
;
1469 attr
.cap
.max_recv_wr
= cmd
->max_recv_wr
;
1470 attr
.cap
.max_send_sge
= cmd
->max_send_sge
;
1471 attr
.cap
.max_recv_sge
= cmd
->max_recv_sge
;
1472 attr
.cap
.max_inline_data
= cmd
->max_inline_data
;
1474 obj
->uevent
.events_reported
= 0;
1475 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1476 INIT_LIST_HEAD(&obj
->mcast_list
);
1478 if (cmd_sz
>= offsetof(typeof(*cmd
), create_flags
) +
1479 sizeof(cmd
->create_flags
))
1480 attr
.create_flags
= cmd
->create_flags
;
1482 if (attr
.create_flags
& ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
|
1483 IB_QP_CREATE_CROSS_CHANNEL
|
1484 IB_QP_CREATE_MANAGED_SEND
|
1485 IB_QP_CREATE_MANAGED_RECV
|
1486 IB_QP_CREATE_SCATTER_FCS
|
1487 IB_QP_CREATE_CVLAN_STRIPPING
)) {
1492 buf
= (void *)cmd
+ sizeof(*cmd
);
1493 if (cmd_sz
> sizeof(*cmd
))
1494 if (!(buf
[0] == 0 && !memcmp(buf
, buf
+ 1,
1495 cmd_sz
- sizeof(*cmd
) - 1))) {
1500 if (cmd
->qp_type
== IB_QPT_XRC_TGT
)
1501 qp
= ib_create_qp(pd
, &attr
);
1503 qp
= device
->create_qp(pd
, &attr
, uhw
);
1510 if (cmd
->qp_type
!= IB_QPT_XRC_TGT
) {
1511 ret
= ib_create_qp_security(qp
, device
);
1516 qp
->device
= device
;
1518 qp
->send_cq
= attr
.send_cq
;
1519 qp
->recv_cq
= attr
.recv_cq
;
1521 qp
->rwq_ind_tbl
= ind_tbl
;
1522 qp
->event_handler
= attr
.event_handler
;
1523 qp
->qp_context
= attr
.qp_context
;
1524 qp
->qp_type
= attr
.qp_type
;
1525 atomic_set(&qp
->usecnt
, 0);
1526 atomic_inc(&pd
->usecnt
);
1528 atomic_inc(&attr
.send_cq
->usecnt
);
1530 atomic_inc(&attr
.recv_cq
->usecnt
);
1532 atomic_inc(&attr
.srq
->usecnt
);
1534 atomic_inc(&ind_tbl
->usecnt
);
1536 qp
->uobject
= &obj
->uevent
.uobject
;
1538 obj
->uevent
.uobject
.object
= qp
;
1540 memset(&resp
, 0, sizeof resp
);
1541 resp
.base
.qpn
= qp
->qp_num
;
1542 resp
.base
.qp_handle
= obj
->uevent
.uobject
.id
;
1543 resp
.base
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1544 resp
.base
.max_send_sge
= attr
.cap
.max_send_sge
;
1545 resp
.base
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1546 resp
.base
.max_send_wr
= attr
.cap
.max_send_wr
;
1547 resp
.base
.max_inline_data
= attr
.cap
.max_inline_data
;
1549 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1550 sizeof(resp
.response_length
);
1552 ret
= cb(file
, &resp
, ucore
);
1557 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
,
1559 atomic_inc(&obj
->uxrcd
->refcnt
);
1560 uobj_put_read(xrcd_uobj
);
1564 uobj_put_obj_read(pd
);
1566 uobj_put_obj_read(scq
);
1567 if (rcq
&& rcq
!= scq
)
1568 uobj_put_obj_read(rcq
);
1570 uobj_put_obj_read(srq
);
1572 uobj_put_obj_read(ind_tbl
);
1574 uobj_alloc_commit(&obj
->uevent
.uobject
);
1581 if (!IS_ERR(xrcd_uobj
))
1582 uobj_put_read(xrcd_uobj
);
1584 uobj_put_obj_read(pd
);
1586 uobj_put_obj_read(scq
);
1587 if (rcq
&& rcq
!= scq
)
1588 uobj_put_obj_read(rcq
);
1590 uobj_put_obj_read(srq
);
1592 uobj_put_obj_read(ind_tbl
);
1594 uobj_alloc_abort(&obj
->uevent
.uobject
);
1598 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file
*file
,
1599 struct ib_uverbs_ex_create_qp_resp
*resp
,
1600 struct ib_udata
*ucore
)
1602 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1608 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1609 struct ib_device
*ib_dev
,
1610 const char __user
*buf
, int in_len
,
1613 struct ib_uverbs_create_qp cmd
;
1614 struct ib_uverbs_ex_create_qp cmd_ex
;
1615 struct ib_udata ucore
;
1616 struct ib_udata uhw
;
1617 ssize_t resp_size
= sizeof(struct ib_uverbs_create_qp_resp
);
1620 if (out_len
< resp_size
)
1623 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1626 INIT_UDATA(&ucore
, buf
, (unsigned long)cmd
.response
, sizeof(cmd
),
1628 INIT_UDATA(&uhw
, buf
+ sizeof(cmd
),
1629 (unsigned long)cmd
.response
+ resp_size
,
1630 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
1631 out_len
- resp_size
);
1633 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1634 cmd_ex
.user_handle
= cmd
.user_handle
;
1635 cmd_ex
.pd_handle
= cmd
.pd_handle
;
1636 cmd_ex
.send_cq_handle
= cmd
.send_cq_handle
;
1637 cmd_ex
.recv_cq_handle
= cmd
.recv_cq_handle
;
1638 cmd_ex
.srq_handle
= cmd
.srq_handle
;
1639 cmd_ex
.max_send_wr
= cmd
.max_send_wr
;
1640 cmd_ex
.max_recv_wr
= cmd
.max_recv_wr
;
1641 cmd_ex
.max_send_sge
= cmd
.max_send_sge
;
1642 cmd_ex
.max_recv_sge
= cmd
.max_recv_sge
;
1643 cmd_ex
.max_inline_data
= cmd
.max_inline_data
;
1644 cmd_ex
.sq_sig_all
= cmd
.sq_sig_all
;
1645 cmd_ex
.qp_type
= cmd
.qp_type
;
1646 cmd_ex
.is_srq
= cmd
.is_srq
;
1648 err
= create_qp(file
, &ucore
, &uhw
, &cmd_ex
,
1649 offsetof(typeof(cmd_ex
), is_srq
) +
1650 sizeof(cmd
.is_srq
), ib_uverbs_create_qp_cb
,
1659 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file
*file
,
1660 struct ib_uverbs_ex_create_qp_resp
*resp
,
1661 struct ib_udata
*ucore
)
1663 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1669 int ib_uverbs_ex_create_qp(struct ib_uverbs_file
*file
,
1670 struct ib_device
*ib_dev
,
1671 struct ib_udata
*ucore
,
1672 struct ib_udata
*uhw
)
1674 struct ib_uverbs_ex_create_qp_resp resp
;
1675 struct ib_uverbs_ex_create_qp cmd
= {0};
1678 if (ucore
->inlen
< (offsetof(typeof(cmd
), comp_mask
) +
1679 sizeof(cmd
.comp_mask
)))
1682 err
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
1686 if (cmd
.comp_mask
& ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK
)
1692 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1693 sizeof(resp
.response_length
)))
1696 err
= create_qp(file
, ucore
, uhw
, &cmd
,
1697 min(ucore
->inlen
, sizeof(cmd
)),
1698 ib_uverbs_ex_create_qp_cb
, NULL
);
1706 ssize_t
ib_uverbs_open_qp(struct ib_uverbs_file
*file
,
1707 struct ib_device
*ib_dev
,
1708 const char __user
*buf
, int in_len
, int out_len
)
1710 struct ib_uverbs_open_qp cmd
;
1711 struct ib_uverbs_create_qp_resp resp
;
1712 struct ib_udata udata
;
1713 struct ib_uqp_object
*obj
;
1714 struct ib_xrcd
*xrcd
;
1715 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1717 struct ib_qp_open_attr attr
;
1720 if (out_len
< sizeof resp
)
1723 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1726 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1727 (unsigned long) cmd
.response
+ sizeof resp
,
1728 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1730 obj
= (struct ib_uqp_object
*)uobj_alloc(uobj_get_type(qp
),
1733 return PTR_ERR(obj
);
1735 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
.pd_handle
,
1737 if (IS_ERR(xrcd_uobj
)) {
1742 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1748 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1749 attr
.qp_context
= file
;
1750 attr
.qp_num
= cmd
.qpn
;
1751 attr
.qp_type
= cmd
.qp_type
;
1753 obj
->uevent
.events_reported
= 0;
1754 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1755 INIT_LIST_HEAD(&obj
->mcast_list
);
1757 qp
= ib_open_qp(xrcd
, &attr
);
1763 obj
->uevent
.uobject
.object
= qp
;
1764 obj
->uevent
.uobject
.user_handle
= cmd
.user_handle
;
1766 memset(&resp
, 0, sizeof resp
);
1767 resp
.qpn
= qp
->qp_num
;
1768 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1770 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1771 &resp
, sizeof resp
)) {
1776 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
1777 atomic_inc(&obj
->uxrcd
->refcnt
);
1778 qp
->uobject
= &obj
->uevent
.uobject
;
1779 uobj_put_read(xrcd_uobj
);
1782 uobj_alloc_commit(&obj
->uevent
.uobject
);
1789 uobj_put_read(xrcd_uobj
);
1791 uobj_alloc_abort(&obj
->uevent
.uobject
);
1795 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
1796 struct ib_device
*ib_dev
,
1797 const char __user
*buf
, int in_len
,
1800 struct ib_uverbs_query_qp cmd
;
1801 struct ib_uverbs_query_qp_resp resp
;
1803 struct ib_qp_attr
*attr
;
1804 struct ib_qp_init_attr
*init_attr
;
1805 const struct ib_global_route
*grh
;
1808 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1811 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1812 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
1813 if (!attr
|| !init_attr
) {
1818 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
1824 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
1826 uobj_put_obj_read(qp
);
1831 memset(&resp
, 0, sizeof resp
);
1833 resp
.qp_state
= attr
->qp_state
;
1834 resp
.cur_qp_state
= attr
->cur_qp_state
;
1835 resp
.path_mtu
= attr
->path_mtu
;
1836 resp
.path_mig_state
= attr
->path_mig_state
;
1837 resp
.qkey
= attr
->qkey
;
1838 resp
.rq_psn
= attr
->rq_psn
;
1839 resp
.sq_psn
= attr
->sq_psn
;
1840 resp
.dest_qp_num
= attr
->dest_qp_num
;
1841 resp
.qp_access_flags
= attr
->qp_access_flags
;
1842 resp
.pkey_index
= attr
->pkey_index
;
1843 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
1844 resp
.sq_draining
= attr
->sq_draining
;
1845 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
1846 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
1847 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
1848 resp
.port_num
= attr
->port_num
;
1849 resp
.timeout
= attr
->timeout
;
1850 resp
.retry_cnt
= attr
->retry_cnt
;
1851 resp
.rnr_retry
= attr
->rnr_retry
;
1852 resp
.alt_port_num
= attr
->alt_port_num
;
1853 resp
.alt_timeout
= attr
->alt_timeout
;
1855 resp
.dest
.dlid
= rdma_ah_get_dlid(&attr
->ah_attr
);
1856 resp
.dest
.sl
= rdma_ah_get_sl(&attr
->ah_attr
);
1857 resp
.dest
.src_path_bits
= rdma_ah_get_path_bits(&attr
->ah_attr
);
1858 resp
.dest
.static_rate
= rdma_ah_get_static_rate(&attr
->ah_attr
);
1859 resp
.dest
.is_global
= !!(rdma_ah_get_ah_flags(&attr
->ah_attr
) &
1861 if (resp
.dest
.is_global
) {
1862 grh
= rdma_ah_read_grh(&attr
->ah_attr
);
1863 memcpy(resp
.dest
.dgid
, grh
->dgid
.raw
, 16);
1864 resp
.dest
.flow_label
= grh
->flow_label
;
1865 resp
.dest
.sgid_index
= grh
->sgid_index
;
1866 resp
.dest
.hop_limit
= grh
->hop_limit
;
1867 resp
.dest
.traffic_class
= grh
->traffic_class
;
1869 resp
.dest
.port_num
= rdma_ah_get_port_num(&attr
->ah_attr
);
1871 resp
.alt_dest
.dlid
= rdma_ah_get_dlid(&attr
->alt_ah_attr
);
1872 resp
.alt_dest
.sl
= rdma_ah_get_sl(&attr
->alt_ah_attr
);
1873 resp
.alt_dest
.src_path_bits
= rdma_ah_get_path_bits(&attr
->alt_ah_attr
);
1874 resp
.alt_dest
.static_rate
1875 = rdma_ah_get_static_rate(&attr
->alt_ah_attr
);
1876 resp
.alt_dest
.is_global
1877 = !!(rdma_ah_get_ah_flags(&attr
->alt_ah_attr
) &
1879 if (resp
.alt_dest
.is_global
) {
1880 grh
= rdma_ah_read_grh(&attr
->alt_ah_attr
);
1881 memcpy(resp
.alt_dest
.dgid
, grh
->dgid
.raw
, 16);
1882 resp
.alt_dest
.flow_label
= grh
->flow_label
;
1883 resp
.alt_dest
.sgid_index
= grh
->sgid_index
;
1884 resp
.alt_dest
.hop_limit
= grh
->hop_limit
;
1885 resp
.alt_dest
.traffic_class
= grh
->traffic_class
;
1887 resp
.alt_dest
.port_num
= rdma_ah_get_port_num(&attr
->alt_ah_attr
);
1889 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
1890 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
1891 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
1892 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
1893 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
1894 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
1896 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1897 &resp
, sizeof resp
))
1904 return ret
? ret
: in_len
;
1907 /* Remove ignored fields set in the attribute mask */
1908 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
1911 case IB_QPT_XRC_INI
:
1912 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
1913 case IB_QPT_XRC_TGT
:
1914 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
1921 static int modify_qp(struct ib_uverbs_file
*file
,
1922 struct ib_uverbs_ex_modify_qp
*cmd
, struct ib_udata
*udata
)
1924 struct ib_qp_attr
*attr
;
1928 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1932 qp
= uobj_get_obj_read(qp
, cmd
->base
.qp_handle
, file
->ucontext
);
1938 if (!rdma_is_port_valid(qp
->device
, cmd
->base
.port_num
)) {
1943 attr
->qp_state
= cmd
->base
.qp_state
;
1944 attr
->cur_qp_state
= cmd
->base
.cur_qp_state
;
1945 attr
->path_mtu
= cmd
->base
.path_mtu
;
1946 attr
->path_mig_state
= cmd
->base
.path_mig_state
;
1947 attr
->qkey
= cmd
->base
.qkey
;
1948 attr
->rq_psn
= cmd
->base
.rq_psn
;
1949 attr
->sq_psn
= cmd
->base
.sq_psn
;
1950 attr
->dest_qp_num
= cmd
->base
.dest_qp_num
;
1951 attr
->qp_access_flags
= cmd
->base
.qp_access_flags
;
1952 attr
->pkey_index
= cmd
->base
.pkey_index
;
1953 attr
->alt_pkey_index
= cmd
->base
.alt_pkey_index
;
1954 attr
->en_sqd_async_notify
= cmd
->base
.en_sqd_async_notify
;
1955 attr
->max_rd_atomic
= cmd
->base
.max_rd_atomic
;
1956 attr
->max_dest_rd_atomic
= cmd
->base
.max_dest_rd_atomic
;
1957 attr
->min_rnr_timer
= cmd
->base
.min_rnr_timer
;
1958 attr
->port_num
= cmd
->base
.port_num
;
1959 attr
->timeout
= cmd
->base
.timeout
;
1960 attr
->retry_cnt
= cmd
->base
.retry_cnt
;
1961 attr
->rnr_retry
= cmd
->base
.rnr_retry
;
1962 attr
->alt_port_num
= cmd
->base
.alt_port_num
;
1963 attr
->alt_timeout
= cmd
->base
.alt_timeout
;
1964 attr
->rate_limit
= cmd
->rate_limit
;
1966 attr
->ah_attr
.type
= rdma_ah_find_type(qp
->device
,
1967 cmd
->base
.dest
.port_num
);
1968 if (cmd
->base
.dest
.is_global
) {
1969 rdma_ah_set_grh(&attr
->ah_attr
, NULL
,
1970 cmd
->base
.dest
.flow_label
,
1971 cmd
->base
.dest
.sgid_index
,
1972 cmd
->base
.dest
.hop_limit
,
1973 cmd
->base
.dest
.traffic_class
);
1974 rdma_ah_set_dgid_raw(&attr
->ah_attr
, cmd
->base
.dest
.dgid
);
1976 rdma_ah_set_ah_flags(&attr
->ah_attr
, 0);
1978 rdma_ah_set_dlid(&attr
->ah_attr
, cmd
->base
.dest
.dlid
);
1979 rdma_ah_set_sl(&attr
->ah_attr
, cmd
->base
.dest
.sl
);
1980 rdma_ah_set_path_bits(&attr
->ah_attr
, cmd
->base
.dest
.src_path_bits
);
1981 rdma_ah_set_static_rate(&attr
->ah_attr
, cmd
->base
.dest
.static_rate
);
1982 rdma_ah_set_port_num(&attr
->ah_attr
,
1983 cmd
->base
.dest
.port_num
);
1985 attr
->alt_ah_attr
.type
= rdma_ah_find_type(qp
->device
,
1986 cmd
->base
.dest
.port_num
);
1987 if (cmd
->base
.alt_dest
.is_global
) {
1988 rdma_ah_set_grh(&attr
->alt_ah_attr
, NULL
,
1989 cmd
->base
.alt_dest
.flow_label
,
1990 cmd
->base
.alt_dest
.sgid_index
,
1991 cmd
->base
.alt_dest
.hop_limit
,
1992 cmd
->base
.alt_dest
.traffic_class
);
1993 rdma_ah_set_dgid_raw(&attr
->alt_ah_attr
,
1994 cmd
->base
.alt_dest
.dgid
);
1996 rdma_ah_set_ah_flags(&attr
->alt_ah_attr
, 0);
1999 rdma_ah_set_dlid(&attr
->alt_ah_attr
, cmd
->base
.alt_dest
.dlid
);
2000 rdma_ah_set_sl(&attr
->alt_ah_attr
, cmd
->base
.alt_dest
.sl
);
2001 rdma_ah_set_path_bits(&attr
->alt_ah_attr
,
2002 cmd
->base
.alt_dest
.src_path_bits
);
2003 rdma_ah_set_static_rate(&attr
->alt_ah_attr
,
2004 cmd
->base
.alt_dest
.static_rate
);
2005 rdma_ah_set_port_num(&attr
->alt_ah_attr
,
2006 cmd
->base
.alt_dest
.port_num
);
2008 if (qp
->real_qp
== qp
) {
2009 if (cmd
->base
.attr_mask
& IB_QP_AV
) {
2010 ret
= ib_resolve_eth_dmac(qp
->device
, &attr
->ah_attr
);
2014 ret
= ib_security_modify_qp(qp
,
2016 modify_qp_mask(qp
->qp_type
,
2017 cmd
->base
.attr_mask
),
2020 ret
= ib_security_modify_qp(qp
,
2022 modify_qp_mask(qp
->qp_type
,
2023 cmd
->base
.attr_mask
),
2028 uobj_put_obj_read(qp
);
2036 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
2037 struct ib_device
*ib_dev
,
2038 const char __user
*buf
, int in_len
,
2041 struct ib_uverbs_ex_modify_qp cmd
= {};
2042 struct ib_udata udata
;
2045 if (copy_from_user(&cmd
.base
, buf
, sizeof(cmd
.base
)))
2048 if (cmd
.base
.attr_mask
&
2049 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK
<< 1) - 1))
2052 INIT_UDATA(&udata
, buf
+ sizeof(cmd
.base
), NULL
,
2053 in_len
- sizeof(cmd
.base
), out_len
);
2055 ret
= modify_qp(file
, &cmd
, &udata
);
2062 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file
*file
,
2063 struct ib_device
*ib_dev
,
2064 struct ib_udata
*ucore
,
2065 struct ib_udata
*uhw
)
2067 struct ib_uverbs_ex_modify_qp cmd
= {};
2071 * Last bit is reserved for extending the attr_mask by
2072 * using another field.
2074 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK
== (1 << 31));
2076 if (ucore
->inlen
< sizeof(cmd
.base
))
2079 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
2083 if (cmd
.base
.attr_mask
&
2084 ~((IB_USER_LAST_QP_ATTR_MASK
<< 1) - 1))
2087 if (ucore
->inlen
> sizeof(cmd
)) {
2088 if (ib_is_udata_cleared(ucore
, sizeof(cmd
),
2089 ucore
->inlen
- sizeof(cmd
)))
2093 ret
= modify_qp(file
, &cmd
, uhw
);
2098 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
2099 struct ib_device
*ib_dev
,
2100 const char __user
*buf
, int in_len
,
2103 struct ib_uverbs_destroy_qp cmd
;
2104 struct ib_uverbs_destroy_qp_resp resp
;
2105 struct ib_uobject
*uobj
;
2107 struct ib_uqp_object
*obj
;
2110 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2113 memset(&resp
, 0, sizeof resp
);
2115 uobj
= uobj_get_write(uobj_get_type(qp
), cmd
.qp_handle
,
2118 return PTR_ERR(uobj
);
2121 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
2123 * Make sure we don't free the memory in remove_commit as we still
2124 * needs the uobject memory to create the response.
2126 uverbs_uobject_get(uobj
);
2128 ret
= uobj_remove_commit(uobj
);
2130 uverbs_uobject_put(uobj
);
2134 resp
.events_reported
= obj
->uevent
.events_reported
;
2135 uverbs_uobject_put(uobj
);
2137 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2138 &resp
, sizeof resp
))
2144 static void *alloc_wr(size_t wr_size
, __u32 num_sge
)
2146 if (num_sge
>= (U32_MAX
- ALIGN(wr_size
, sizeof (struct ib_sge
))) /
2147 sizeof (struct ib_sge
))
2150 return kmalloc(ALIGN(wr_size
, sizeof (struct ib_sge
)) +
2151 num_sge
* sizeof (struct ib_sge
), GFP_KERNEL
);
2154 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
2155 struct ib_device
*ib_dev
,
2156 const char __user
*buf
, int in_len
,
2159 struct ib_uverbs_post_send cmd
;
2160 struct ib_uverbs_post_send_resp resp
;
2161 struct ib_uverbs_send_wr
*user_wr
;
2162 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
2166 ssize_t ret
= -EINVAL
;
2169 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2172 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
2173 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
2176 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
2179 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
2183 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2187 is_ud
= qp
->qp_type
== IB_QPT_UD
;
2190 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
2191 if (copy_from_user(user_wr
,
2192 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
2198 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
2204 struct ib_ud_wr
*ud
;
2206 if (user_wr
->opcode
!= IB_WR_SEND
&&
2207 user_wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
2212 next_size
= sizeof(*ud
);
2213 ud
= alloc_wr(next_size
, user_wr
->num_sge
);
2219 ud
->ah
= uobj_get_obj_read(ah
, user_wr
->wr
.ud
.ah
,
2226 ud
->remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
2227 ud
->remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
2230 } else if (user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
||
2231 user_wr
->opcode
== IB_WR_RDMA_WRITE
||
2232 user_wr
->opcode
== IB_WR_RDMA_READ
) {
2233 struct ib_rdma_wr
*rdma
;
2235 next_size
= sizeof(*rdma
);
2236 rdma
= alloc_wr(next_size
, user_wr
->num_sge
);
2242 rdma
->remote_addr
= user_wr
->wr
.rdma
.remote_addr
;
2243 rdma
->rkey
= user_wr
->wr
.rdma
.rkey
;
2246 } else if (user_wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
||
2247 user_wr
->opcode
== IB_WR_ATOMIC_FETCH_AND_ADD
) {
2248 struct ib_atomic_wr
*atomic
;
2250 next_size
= sizeof(*atomic
);
2251 atomic
= alloc_wr(next_size
, user_wr
->num_sge
);
2257 atomic
->remote_addr
= user_wr
->wr
.atomic
.remote_addr
;
2258 atomic
->compare_add
= user_wr
->wr
.atomic
.compare_add
;
2259 atomic
->swap
= user_wr
->wr
.atomic
.swap
;
2260 atomic
->rkey
= user_wr
->wr
.atomic
.rkey
;
2263 } else if (user_wr
->opcode
== IB_WR_SEND
||
2264 user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2265 user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2266 next_size
= sizeof(*next
);
2267 next
= alloc_wr(next_size
, user_wr
->num_sge
);
2277 if (user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2278 user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) {
2280 (__be32 __force
) user_wr
->ex
.imm_data
;
2281 } else if (user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2282 next
->ex
.invalidate_rkey
= user_wr
->ex
.invalidate_rkey
;
2292 next
->wr_id
= user_wr
->wr_id
;
2293 next
->num_sge
= user_wr
->num_sge
;
2294 next
->opcode
= user_wr
->opcode
;
2295 next
->send_flags
= user_wr
->send_flags
;
2297 if (next
->num_sge
) {
2298 next
->sg_list
= (void *) next
+
2299 ALIGN(next_size
, sizeof(struct ib_sge
));
2300 if (copy_from_user(next
->sg_list
,
2302 cmd
.wr_count
* cmd
.wqe_size
+
2303 sg_ind
* sizeof (struct ib_sge
),
2304 next
->num_sge
* sizeof (struct ib_sge
))) {
2308 sg_ind
+= next
->num_sge
;
2310 next
->sg_list
= NULL
;
2314 ret
= qp
->device
->post_send(qp
->real_qp
, wr
, &bad_wr
);
2316 for (next
= wr
; next
; next
= next
->next
) {
2322 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2323 &resp
, sizeof resp
))
2327 uobj_put_obj_read(qp
);
2330 if (is_ud
&& ud_wr(wr
)->ah
)
2331 uobj_put_obj_read(ud_wr(wr
)->ah
);
2340 return ret
? ret
: in_len
;
2343 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
2349 struct ib_uverbs_recv_wr
*user_wr
;
2350 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2355 if (in_len
< wqe_size
* wr_count
+
2356 sge_count
* sizeof (struct ib_uverbs_sge
))
2357 return ERR_PTR(-EINVAL
);
2359 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2360 return ERR_PTR(-EINVAL
);
2362 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2364 return ERR_PTR(-ENOMEM
);
2368 for (i
= 0; i
< wr_count
; ++i
) {
2369 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
2375 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2380 if (user_wr
->num_sge
>=
2381 (U32_MAX
- ALIGN(sizeof *next
, sizeof (struct ib_sge
))) /
2382 sizeof (struct ib_sge
)) {
2387 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2388 user_wr
->num_sge
* sizeof (struct ib_sge
),
2402 next
->wr_id
= user_wr
->wr_id
;
2403 next
->num_sge
= user_wr
->num_sge
;
2405 if (next
->num_sge
) {
2406 next
->sg_list
= (void *) next
+
2407 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2408 if (copy_from_user(next
->sg_list
,
2409 buf
+ wr_count
* wqe_size
+
2410 sg_ind
* sizeof (struct ib_sge
),
2411 next
->num_sge
* sizeof (struct ib_sge
))) {
2415 sg_ind
+= next
->num_sge
;
2417 next
->sg_list
= NULL
;
2432 return ERR_PTR(ret
);
2435 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
2436 struct ib_device
*ib_dev
,
2437 const char __user
*buf
, int in_len
,
2440 struct ib_uverbs_post_recv cmd
;
2441 struct ib_uverbs_post_recv_resp resp
;
2442 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2444 ssize_t ret
= -EINVAL
;
2446 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2449 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2450 in_len
- sizeof cmd
, cmd
.wr_count
,
2451 cmd
.sge_count
, cmd
.wqe_size
);
2455 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2460 ret
= qp
->device
->post_recv(qp
->real_qp
, wr
, &bad_wr
);
2462 uobj_put_obj_read(qp
);
2464 for (next
= wr
; next
; next
= next
->next
) {
2471 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2472 &resp
, sizeof resp
))
2482 return ret
? ret
: in_len
;
2485 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
2486 struct ib_device
*ib_dev
,
2487 const char __user
*buf
, int in_len
,
2490 struct ib_uverbs_post_srq_recv cmd
;
2491 struct ib_uverbs_post_srq_recv_resp resp
;
2492 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2494 ssize_t ret
= -EINVAL
;
2496 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2499 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2500 in_len
- sizeof cmd
, cmd
.wr_count
,
2501 cmd
.sge_count
, cmd
.wqe_size
);
2505 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
2510 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
2512 uobj_put_obj_read(srq
);
2515 for (next
= wr
; next
; next
= next
->next
) {
2521 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2522 &resp
, sizeof resp
))
2532 return ret
? ret
: in_len
;
2535 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
2536 struct ib_device
*ib_dev
,
2537 const char __user
*buf
, int in_len
,
2540 struct ib_uverbs_create_ah cmd
;
2541 struct ib_uverbs_create_ah_resp resp
;
2542 struct ib_uobject
*uobj
;
2545 struct rdma_ah_attr attr
;
2547 struct ib_udata udata
;
2550 if (out_len
< sizeof resp
)
2553 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2556 if (!rdma_is_port_valid(ib_dev
, cmd
.attr
.port_num
))
2559 INIT_UDATA(&udata
, buf
+ sizeof(cmd
),
2560 (unsigned long)cmd
.response
+ sizeof(resp
),
2561 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
2563 uobj
= uobj_alloc(uobj_get_type(ah
), file
->ucontext
);
2565 return PTR_ERR(uobj
);
2567 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
2573 attr
.type
= rdma_ah_find_type(ib_dev
, cmd
.attr
.port_num
);
2574 rdma_ah_set_dlid(&attr
, cmd
.attr
.dlid
);
2575 rdma_ah_set_sl(&attr
, cmd
.attr
.sl
);
2576 rdma_ah_set_path_bits(&attr
, cmd
.attr
.src_path_bits
);
2577 rdma_ah_set_static_rate(&attr
, cmd
.attr
.static_rate
);
2578 rdma_ah_set_port_num(&attr
, cmd
.attr
.port_num
);
2580 if (cmd
.attr
.is_global
) {
2581 rdma_ah_set_grh(&attr
, NULL
, cmd
.attr
.grh
.flow_label
,
2582 cmd
.attr
.grh
.sgid_index
,
2583 cmd
.attr
.grh
.hop_limit
,
2584 cmd
.attr
.grh
.traffic_class
);
2585 rdma_ah_set_dgid_raw(&attr
, cmd
.attr
.grh
.dgid
);
2587 rdma_ah_set_ah_flags(&attr
, 0);
2589 dmac
= rdma_ah_retrieve_dmac(&attr
);
2591 memset(dmac
, 0, ETH_ALEN
);
2593 ah
= pd
->device
->create_ah(pd
, &attr
, &udata
);
2600 ah
->device
= pd
->device
;
2602 atomic_inc(&pd
->usecnt
);
2604 uobj
->user_handle
= cmd
.user_handle
;
2607 resp
.ah_handle
= uobj
->id
;
2609 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2610 &resp
, sizeof resp
)) {
2615 uobj_put_obj_read(pd
);
2616 uobj_alloc_commit(uobj
);
2621 rdma_destroy_ah(ah
);
2624 uobj_put_obj_read(pd
);
2627 uobj_alloc_abort(uobj
);
2631 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
2632 struct ib_device
*ib_dev
,
2633 const char __user
*buf
, int in_len
, int out_len
)
2635 struct ib_uverbs_destroy_ah cmd
;
2636 struct ib_uobject
*uobj
;
2639 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2642 uobj
= uobj_get_write(uobj_get_type(ah
), cmd
.ah_handle
,
2645 return PTR_ERR(uobj
);
2647 ret
= uobj_remove_commit(uobj
);
2648 return ret
?: in_len
;
2651 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
2652 struct ib_device
*ib_dev
,
2653 const char __user
*buf
, int in_len
,
2656 struct ib_uverbs_attach_mcast cmd
;
2658 struct ib_uqp_object
*obj
;
2659 struct ib_uverbs_mcast_entry
*mcast
;
2662 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2665 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2669 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2671 mutex_lock(&obj
->mcast_lock
);
2672 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2673 if (cmd
.mlid
== mcast
->lid
&&
2674 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2679 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2685 mcast
->lid
= cmd
.mlid
;
2686 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2688 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2690 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2695 mutex_unlock(&obj
->mcast_lock
);
2696 uobj_put_obj_read(qp
);
2698 return ret
? ret
: in_len
;
2701 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
2702 struct ib_device
*ib_dev
,
2703 const char __user
*buf
, int in_len
,
2706 struct ib_uverbs_detach_mcast cmd
;
2707 struct ib_uqp_object
*obj
;
2709 struct ib_uverbs_mcast_entry
*mcast
;
2713 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2716 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2720 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2721 mutex_lock(&obj
->mcast_lock
);
2723 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2724 if (cmd
.mlid
== mcast
->lid
&&
2725 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2726 list_del(&mcast
->list
);
2737 ret
= ib_detach_mcast(qp
, (union ib_gid
*)cmd
.gid
, cmd
.mlid
);
2740 mutex_unlock(&obj
->mcast_lock
);
2741 uobj_put_obj_read(qp
);
2742 return ret
? ret
: in_len
;
2745 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec
*kern_spec
,
2746 union ib_flow_spec
*ib_spec
)
2748 ib_spec
->type
= kern_spec
->type
;
2749 switch (ib_spec
->type
) {
2750 case IB_FLOW_SPEC_ACTION_TAG
:
2751 if (kern_spec
->flow_tag
.size
!=
2752 sizeof(struct ib_uverbs_flow_spec_action_tag
))
2755 ib_spec
->flow_tag
.size
= sizeof(struct ib_flow_spec_action_tag
);
2756 ib_spec
->flow_tag
.tag_id
= kern_spec
->flow_tag
.tag_id
;
2758 case IB_FLOW_SPEC_ACTION_DROP
:
2759 if (kern_spec
->drop
.size
!=
2760 sizeof(struct ib_uverbs_flow_spec_action_drop
))
2763 ib_spec
->drop
.size
= sizeof(struct ib_flow_spec_action_drop
);
2771 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr
*spec
)
2773 /* Returns user space filter size, includes padding */
2774 return (spec
->size
- sizeof(struct ib_uverbs_flow_spec_hdr
)) / 2;
2777 static ssize_t
spec_filter_size(void *kern_spec_filter
, u16 kern_filter_size
,
2778 u16 ib_real_filter_sz
)
2781 * User space filter structures must be 64 bit aligned, otherwise this
2782 * may pass, but we won't handle additional new attributes.
2785 if (kern_filter_size
> ib_real_filter_sz
) {
2786 if (memchr_inv(kern_spec_filter
+
2787 ib_real_filter_sz
, 0,
2788 kern_filter_size
- ib_real_filter_sz
))
2790 return ib_real_filter_sz
;
2792 return kern_filter_size
;
2795 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec
*kern_spec
,
2796 union ib_flow_spec
*ib_spec
)
2798 ssize_t actual_filter_sz
;
2799 ssize_t kern_filter_sz
;
2800 ssize_t ib_filter_sz
;
2801 void *kern_spec_mask
;
2802 void *kern_spec_val
;
2804 if (kern_spec
->reserved
)
2807 ib_spec
->type
= kern_spec
->type
;
2809 kern_filter_sz
= kern_spec_filter_sz(&kern_spec
->hdr
);
2810 /* User flow spec size must be aligned to 4 bytes */
2811 if (kern_filter_sz
!= ALIGN(kern_filter_sz
, 4))
2814 kern_spec_val
= (void *)kern_spec
+
2815 sizeof(struct ib_uverbs_flow_spec_hdr
);
2816 kern_spec_mask
= kern_spec_val
+ kern_filter_sz
;
2817 if (ib_spec
->type
== (IB_FLOW_SPEC_INNER
| IB_FLOW_SPEC_VXLAN_TUNNEL
))
2820 switch (ib_spec
->type
& ~IB_FLOW_SPEC_INNER
) {
2821 case IB_FLOW_SPEC_ETH
:
2822 ib_filter_sz
= offsetof(struct ib_flow_eth_filter
, real_sz
);
2823 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2826 if (actual_filter_sz
<= 0)
2828 ib_spec
->size
= sizeof(struct ib_flow_spec_eth
);
2829 memcpy(&ib_spec
->eth
.val
, kern_spec_val
, actual_filter_sz
);
2830 memcpy(&ib_spec
->eth
.mask
, kern_spec_mask
, actual_filter_sz
);
2832 case IB_FLOW_SPEC_IPV4
:
2833 ib_filter_sz
= offsetof(struct ib_flow_ipv4_filter
, real_sz
);
2834 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2837 if (actual_filter_sz
<= 0)
2839 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv4
);
2840 memcpy(&ib_spec
->ipv4
.val
, kern_spec_val
, actual_filter_sz
);
2841 memcpy(&ib_spec
->ipv4
.mask
, kern_spec_mask
, actual_filter_sz
);
2843 case IB_FLOW_SPEC_IPV6
:
2844 ib_filter_sz
= offsetof(struct ib_flow_ipv6_filter
, real_sz
);
2845 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2848 if (actual_filter_sz
<= 0)
2850 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv6
);
2851 memcpy(&ib_spec
->ipv6
.val
, kern_spec_val
, actual_filter_sz
);
2852 memcpy(&ib_spec
->ipv6
.mask
, kern_spec_mask
, actual_filter_sz
);
2854 if ((ntohl(ib_spec
->ipv6
.mask
.flow_label
)) >= BIT(20) ||
2855 (ntohl(ib_spec
->ipv6
.val
.flow_label
)) >= BIT(20))
2858 case IB_FLOW_SPEC_TCP
:
2859 case IB_FLOW_SPEC_UDP
:
2860 ib_filter_sz
= offsetof(struct ib_flow_tcp_udp_filter
, real_sz
);
2861 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2864 if (actual_filter_sz
<= 0)
2866 ib_spec
->size
= sizeof(struct ib_flow_spec_tcp_udp
);
2867 memcpy(&ib_spec
->tcp_udp
.val
, kern_spec_val
, actual_filter_sz
);
2868 memcpy(&ib_spec
->tcp_udp
.mask
, kern_spec_mask
, actual_filter_sz
);
2870 case IB_FLOW_SPEC_VXLAN_TUNNEL
:
2871 ib_filter_sz
= offsetof(struct ib_flow_tunnel_filter
, real_sz
);
2872 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2875 if (actual_filter_sz
<= 0)
2877 ib_spec
->tunnel
.size
= sizeof(struct ib_flow_spec_tunnel
);
2878 memcpy(&ib_spec
->tunnel
.val
, kern_spec_val
, actual_filter_sz
);
2879 memcpy(&ib_spec
->tunnel
.mask
, kern_spec_mask
, actual_filter_sz
);
2881 if ((ntohl(ib_spec
->tunnel
.mask
.tunnel_id
)) >= BIT(24) ||
2882 (ntohl(ib_spec
->tunnel
.val
.tunnel_id
)) >= BIT(24))
2891 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec
*kern_spec
,
2892 union ib_flow_spec
*ib_spec
)
2894 if (kern_spec
->reserved
)
2897 if (kern_spec
->type
>= IB_FLOW_SPEC_ACTION_TAG
)
2898 return kern_spec_to_ib_spec_action(kern_spec
, ib_spec
);
2900 return kern_spec_to_ib_spec_filter(kern_spec
, ib_spec
);
2903 int ib_uverbs_ex_create_wq(struct ib_uverbs_file
*file
,
2904 struct ib_device
*ib_dev
,
2905 struct ib_udata
*ucore
,
2906 struct ib_udata
*uhw
)
2908 struct ib_uverbs_ex_create_wq cmd
= {};
2909 struct ib_uverbs_ex_create_wq_resp resp
= {};
2910 struct ib_uwq_object
*obj
;
2915 struct ib_wq_init_attr wq_init_attr
= {};
2916 size_t required_cmd_sz
;
2917 size_t required_resp_len
;
2919 required_cmd_sz
= offsetof(typeof(cmd
), max_sge
) + sizeof(cmd
.max_sge
);
2920 required_resp_len
= offsetof(typeof(resp
), wqn
) + sizeof(resp
.wqn
);
2922 if (ucore
->inlen
< required_cmd_sz
)
2925 if (ucore
->outlen
< required_resp_len
)
2928 if (ucore
->inlen
> sizeof(cmd
) &&
2929 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
2930 ucore
->inlen
- sizeof(cmd
)))
2933 err
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
2940 obj
= (struct ib_uwq_object
*)uobj_alloc(uobj_get_type(wq
),
2943 return PTR_ERR(obj
);
2945 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
2951 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
2957 wq_init_attr
.cq
= cq
;
2958 wq_init_attr
.max_sge
= cmd
.max_sge
;
2959 wq_init_attr
.max_wr
= cmd
.max_wr
;
2960 wq_init_attr
.wq_context
= file
;
2961 wq_init_attr
.wq_type
= cmd
.wq_type
;
2962 wq_init_attr
.event_handler
= ib_uverbs_wq_event_handler
;
2963 if (ucore
->inlen
>= (offsetof(typeof(cmd
), create_flags
) +
2964 sizeof(cmd
.create_flags
)))
2965 wq_init_attr
.create_flags
= cmd
.create_flags
;
2966 obj
->uevent
.events_reported
= 0;
2967 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
2968 wq
= pd
->device
->create_wq(pd
, &wq_init_attr
, uhw
);
2974 wq
->uobject
= &obj
->uevent
.uobject
;
2975 obj
->uevent
.uobject
.object
= wq
;
2976 wq
->wq_type
= wq_init_attr
.wq_type
;
2979 wq
->device
= pd
->device
;
2980 wq
->wq_context
= wq_init_attr
.wq_context
;
2981 atomic_set(&wq
->usecnt
, 0);
2982 atomic_inc(&pd
->usecnt
);
2983 atomic_inc(&cq
->usecnt
);
2984 wq
->uobject
= &obj
->uevent
.uobject
;
2985 obj
->uevent
.uobject
.object
= wq
;
2987 memset(&resp
, 0, sizeof(resp
));
2988 resp
.wq_handle
= obj
->uevent
.uobject
.id
;
2989 resp
.max_sge
= wq_init_attr
.max_sge
;
2990 resp
.max_wr
= wq_init_attr
.max_wr
;
2991 resp
.wqn
= wq
->wq_num
;
2992 resp
.response_length
= required_resp_len
;
2993 err
= ib_copy_to_udata(ucore
,
2994 &resp
, resp
.response_length
);
2998 uobj_put_obj_read(pd
);
2999 uobj_put_obj_read(cq
);
3000 uobj_alloc_commit(&obj
->uevent
.uobject
);
3006 uobj_put_obj_read(cq
);
3008 uobj_put_obj_read(pd
);
3010 uobj_alloc_abort(&obj
->uevent
.uobject
);
3015 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file
*file
,
3016 struct ib_device
*ib_dev
,
3017 struct ib_udata
*ucore
,
3018 struct ib_udata
*uhw
)
3020 struct ib_uverbs_ex_destroy_wq cmd
= {};
3021 struct ib_uverbs_ex_destroy_wq_resp resp
= {};
3023 struct ib_uobject
*uobj
;
3024 struct ib_uwq_object
*obj
;
3025 size_t required_cmd_sz
;
3026 size_t required_resp_len
;
3029 required_cmd_sz
= offsetof(typeof(cmd
), wq_handle
) + sizeof(cmd
.wq_handle
);
3030 required_resp_len
= offsetof(typeof(resp
), reserved
) + sizeof(resp
.reserved
);
3032 if (ucore
->inlen
< required_cmd_sz
)
3035 if (ucore
->outlen
< required_resp_len
)
3038 if (ucore
->inlen
> sizeof(cmd
) &&
3039 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3040 ucore
->inlen
- sizeof(cmd
)))
3043 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3050 resp
.response_length
= required_resp_len
;
3051 uobj
= uobj_get_write(uobj_get_type(wq
), cmd
.wq_handle
,
3054 return PTR_ERR(uobj
);
3057 obj
= container_of(uobj
, struct ib_uwq_object
, uevent
.uobject
);
3059 * Make sure we don't free the memory in remove_commit as we still
3060 * needs the uobject memory to create the response.
3062 uverbs_uobject_get(uobj
);
3064 ret
= uobj_remove_commit(uobj
);
3065 resp
.events_reported
= obj
->uevent
.events_reported
;
3066 uverbs_uobject_put(uobj
);
3070 return ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);
3073 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file
*file
,
3074 struct ib_device
*ib_dev
,
3075 struct ib_udata
*ucore
,
3076 struct ib_udata
*uhw
)
3078 struct ib_uverbs_ex_modify_wq cmd
= {};
3080 struct ib_wq_attr wq_attr
= {};
3081 size_t required_cmd_sz
;
3084 required_cmd_sz
= offsetof(typeof(cmd
), curr_wq_state
) + sizeof(cmd
.curr_wq_state
);
3085 if (ucore
->inlen
< required_cmd_sz
)
3088 if (ucore
->inlen
> sizeof(cmd
) &&
3089 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3090 ucore
->inlen
- sizeof(cmd
)))
3093 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3100 if (cmd
.attr_mask
> (IB_WQ_STATE
| IB_WQ_CUR_STATE
| IB_WQ_FLAGS
))
3103 wq
= uobj_get_obj_read(wq
, cmd
.wq_handle
, file
->ucontext
);
3107 wq_attr
.curr_wq_state
= cmd
.curr_wq_state
;
3108 wq_attr
.wq_state
= cmd
.wq_state
;
3109 if (cmd
.attr_mask
& IB_WQ_FLAGS
) {
3110 wq_attr
.flags
= cmd
.flags
;
3111 wq_attr
.flags_mask
= cmd
.flags_mask
;
3113 ret
= wq
->device
->modify_wq(wq
, &wq_attr
, cmd
.attr_mask
, uhw
);
3114 uobj_put_obj_read(wq
);
3118 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file
*file
,
3119 struct ib_device
*ib_dev
,
3120 struct ib_udata
*ucore
,
3121 struct ib_udata
*uhw
)
3123 struct ib_uverbs_ex_create_rwq_ind_table cmd
= {};
3124 struct ib_uverbs_ex_create_rwq_ind_table_resp resp
= {};
3125 struct ib_uobject
*uobj
;
3127 struct ib_rwq_ind_table_init_attr init_attr
= {};
3128 struct ib_rwq_ind_table
*rwq_ind_tbl
;
3129 struct ib_wq
**wqs
= NULL
;
3130 u32
*wqs_handles
= NULL
;
3131 struct ib_wq
*wq
= NULL
;
3132 int i
, j
, num_read_wqs
;
3134 u32 expected_in_size
;
3135 size_t required_cmd_sz_header
;
3136 size_t required_resp_len
;
3138 required_cmd_sz_header
= offsetof(typeof(cmd
), log_ind_tbl_size
) + sizeof(cmd
.log_ind_tbl_size
);
3139 required_resp_len
= offsetof(typeof(resp
), ind_tbl_num
) + sizeof(resp
.ind_tbl_num
);
3141 if (ucore
->inlen
< required_cmd_sz_header
)
3144 if (ucore
->outlen
< required_resp_len
)
3147 err
= ib_copy_from_udata(&cmd
, ucore
, required_cmd_sz_header
);
3151 ucore
->inbuf
+= required_cmd_sz_header
;
3152 ucore
->inlen
-= required_cmd_sz_header
;
3157 if (cmd
.log_ind_tbl_size
> IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE
)
3160 num_wq_handles
= 1 << cmd
.log_ind_tbl_size
;
3161 expected_in_size
= num_wq_handles
* sizeof(__u32
);
3162 if (num_wq_handles
== 1)
3163 /* input size for wq handles is u64 aligned */
3164 expected_in_size
+= sizeof(__u32
);
3166 if (ucore
->inlen
< expected_in_size
)
3169 if (ucore
->inlen
> expected_in_size
&&
3170 !ib_is_udata_cleared(ucore
, expected_in_size
,
3171 ucore
->inlen
- expected_in_size
))
3174 wqs_handles
= kcalloc(num_wq_handles
, sizeof(*wqs_handles
),
3179 err
= ib_copy_from_udata(wqs_handles
, ucore
,
3180 num_wq_handles
* sizeof(__u32
));
3184 wqs
= kcalloc(num_wq_handles
, sizeof(*wqs
), GFP_KERNEL
);
3190 for (num_read_wqs
= 0; num_read_wqs
< num_wq_handles
;
3192 wq
= uobj_get_obj_read(wq
, wqs_handles
[num_read_wqs
],
3199 wqs
[num_read_wqs
] = wq
;
3202 uobj
= uobj_alloc(uobj_get_type(rwq_ind_table
), file
->ucontext
);
3204 err
= PTR_ERR(uobj
);
3208 init_attr
.log_ind_tbl_size
= cmd
.log_ind_tbl_size
;
3209 init_attr
.ind_tbl
= wqs
;
3210 rwq_ind_tbl
= ib_dev
->create_rwq_ind_table(ib_dev
, &init_attr
, uhw
);
3212 if (IS_ERR(rwq_ind_tbl
)) {
3213 err
= PTR_ERR(rwq_ind_tbl
);
3217 rwq_ind_tbl
->ind_tbl
= wqs
;
3218 rwq_ind_tbl
->log_ind_tbl_size
= init_attr
.log_ind_tbl_size
;
3219 rwq_ind_tbl
->uobject
= uobj
;
3220 uobj
->object
= rwq_ind_tbl
;
3221 rwq_ind_tbl
->device
= ib_dev
;
3222 atomic_set(&rwq_ind_tbl
->usecnt
, 0);
3224 for (i
= 0; i
< num_wq_handles
; i
++)
3225 atomic_inc(&wqs
[i
]->usecnt
);
3227 resp
.ind_tbl_handle
= uobj
->id
;
3228 resp
.ind_tbl_num
= rwq_ind_tbl
->ind_tbl_num
;
3229 resp
.response_length
= required_resp_len
;
3231 err
= ib_copy_to_udata(ucore
,
3232 &resp
, resp
.response_length
);
3238 for (j
= 0; j
< num_read_wqs
; j
++)
3239 uobj_put_obj_read(wqs
[j
]);
3241 uobj_alloc_commit(uobj
);
3245 ib_destroy_rwq_ind_table(rwq_ind_tbl
);
3247 uobj_alloc_abort(uobj
);
3249 for (j
= 0; j
< num_read_wqs
; j
++)
3250 uobj_put_obj_read(wqs
[j
]);
3257 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file
*file
,
3258 struct ib_device
*ib_dev
,
3259 struct ib_udata
*ucore
,
3260 struct ib_udata
*uhw
)
3262 struct ib_uverbs_ex_destroy_rwq_ind_table cmd
= {};
3263 struct ib_uobject
*uobj
;
3265 size_t required_cmd_sz
;
3267 required_cmd_sz
= offsetof(typeof(cmd
), ind_tbl_handle
) + sizeof(cmd
.ind_tbl_handle
);
3269 if (ucore
->inlen
< required_cmd_sz
)
3272 if (ucore
->inlen
> sizeof(cmd
) &&
3273 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3274 ucore
->inlen
- sizeof(cmd
)))
3277 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3284 uobj
= uobj_get_write(uobj_get_type(rwq_ind_table
), cmd
.ind_tbl_handle
,
3287 return PTR_ERR(uobj
);
3289 return uobj_remove_commit(uobj
);
3292 int ib_uverbs_ex_create_flow(struct ib_uverbs_file
*file
,
3293 struct ib_device
*ib_dev
,
3294 struct ib_udata
*ucore
,
3295 struct ib_udata
*uhw
)
3297 struct ib_uverbs_create_flow cmd
;
3298 struct ib_uverbs_create_flow_resp resp
;
3299 struct ib_uobject
*uobj
;
3300 struct ib_flow
*flow_id
;
3301 struct ib_uverbs_flow_attr
*kern_flow_attr
;
3302 struct ib_flow_attr
*flow_attr
;
3309 if (ucore
->inlen
< sizeof(cmd
))
3312 if (ucore
->outlen
< sizeof(resp
))
3315 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3319 ucore
->inbuf
+= sizeof(cmd
);
3320 ucore
->inlen
-= sizeof(cmd
);
3325 if (!capable(CAP_NET_RAW
))
3328 if (cmd
.flow_attr
.flags
>= IB_FLOW_ATTR_FLAGS_RESERVED
)
3331 if ((cmd
.flow_attr
.flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
) &&
3332 ((cmd
.flow_attr
.type
== IB_FLOW_ATTR_ALL_DEFAULT
) ||
3333 (cmd
.flow_attr
.type
== IB_FLOW_ATTR_MC_DEFAULT
)))
3336 if (cmd
.flow_attr
.num_of_specs
> IB_FLOW_SPEC_SUPPORT_LAYERS
)
3339 if (cmd
.flow_attr
.size
> ucore
->inlen
||
3340 cmd
.flow_attr
.size
>
3341 (cmd
.flow_attr
.num_of_specs
* sizeof(struct ib_uverbs_flow_spec
)))
3344 if (cmd
.flow_attr
.reserved
[0] ||
3345 cmd
.flow_attr
.reserved
[1])
3348 if (cmd
.flow_attr
.num_of_specs
) {
3349 kern_flow_attr
= kmalloc(sizeof(*kern_flow_attr
) + cmd
.flow_attr
.size
,
3351 if (!kern_flow_attr
)
3354 memcpy(kern_flow_attr
, &cmd
.flow_attr
, sizeof(*kern_flow_attr
));
3355 err
= ib_copy_from_udata(kern_flow_attr
+ 1, ucore
,
3356 cmd
.flow_attr
.size
);
3360 kern_flow_attr
= &cmd
.flow_attr
;
3363 uobj
= uobj_alloc(uobj_get_type(flow
), file
->ucontext
);
3365 err
= PTR_ERR(uobj
);
3369 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
3375 flow_attr
= kzalloc(sizeof(*flow_attr
) + cmd
.flow_attr
.num_of_specs
*
3376 sizeof(union ib_flow_spec
), GFP_KERNEL
);
3382 flow_attr
->type
= kern_flow_attr
->type
;
3383 flow_attr
->priority
= kern_flow_attr
->priority
;
3384 flow_attr
->num_of_specs
= kern_flow_attr
->num_of_specs
;
3385 flow_attr
->port
= kern_flow_attr
->port
;
3386 flow_attr
->flags
= kern_flow_attr
->flags
;
3387 flow_attr
->size
= sizeof(*flow_attr
);
3389 kern_spec
= kern_flow_attr
+ 1;
3390 ib_spec
= flow_attr
+ 1;
3391 for (i
= 0; i
< flow_attr
->num_of_specs
&&
3392 cmd
.flow_attr
.size
> offsetof(struct ib_uverbs_flow_spec
, reserved
) &&
3393 cmd
.flow_attr
.size
>=
3394 ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
; i
++) {
3395 err
= kern_spec_to_ib_spec(kern_spec
, ib_spec
);
3399 ((union ib_flow_spec
*) ib_spec
)->size
;
3400 cmd
.flow_attr
.size
-= ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
;
3401 kern_spec
+= ((struct ib_uverbs_flow_spec
*) kern_spec
)->size
;
3402 ib_spec
+= ((union ib_flow_spec
*) ib_spec
)->size
;
3404 if (cmd
.flow_attr
.size
|| (i
!= flow_attr
->num_of_specs
)) {
3405 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3406 i
, cmd
.flow_attr
.size
);
3410 flow_id
= ib_create_flow(qp
, flow_attr
, IB_FLOW_DOMAIN_USER
);
3411 if (IS_ERR(flow_id
)) {
3412 err
= PTR_ERR(flow_id
);
3415 flow_id
->uobject
= uobj
;
3416 uobj
->object
= flow_id
;
3418 memset(&resp
, 0, sizeof(resp
));
3419 resp
.flow_handle
= uobj
->id
;
3421 err
= ib_copy_to_udata(ucore
,
3422 &resp
, sizeof(resp
));
3426 uobj_put_obj_read(qp
);
3427 uobj_alloc_commit(uobj
);
3429 if (cmd
.flow_attr
.num_of_specs
)
3430 kfree(kern_flow_attr
);
3433 ib_destroy_flow(flow_id
);
3437 uobj_put_obj_read(qp
);
3439 uobj_alloc_abort(uobj
);
3441 if (cmd
.flow_attr
.num_of_specs
)
3442 kfree(kern_flow_attr
);
3446 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file
*file
,
3447 struct ib_device
*ib_dev
,
3448 struct ib_udata
*ucore
,
3449 struct ib_udata
*uhw
)
3451 struct ib_uverbs_destroy_flow cmd
;
3452 struct ib_uobject
*uobj
;
3455 if (ucore
->inlen
< sizeof(cmd
))
3458 ret
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3465 uobj
= uobj_get_write(uobj_get_type(flow
), cmd
.flow_handle
,
3468 return PTR_ERR(uobj
);
3470 ret
= uobj_remove_commit(uobj
);
3474 static int __uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3475 struct ib_device
*ib_dev
,
3476 struct ib_uverbs_create_xsrq
*cmd
,
3477 struct ib_udata
*udata
)
3479 struct ib_uverbs_create_srq_resp resp
;
3480 struct ib_usrq_object
*obj
;
3483 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
3484 struct ib_srq_init_attr attr
;
3487 obj
= (struct ib_usrq_object
*)uobj_alloc(uobj_get_type(srq
),
3490 return PTR_ERR(obj
);
3492 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3493 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
->xrcd_handle
,
3495 if (IS_ERR(xrcd_uobj
)) {
3500 attr
.ext
.xrc
.xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
3501 if (!attr
.ext
.xrc
.xrcd
) {
3506 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
3507 atomic_inc(&obj
->uxrcd
->refcnt
);
3509 attr
.ext
.xrc
.cq
= uobj_get_obj_read(cq
, cmd
->cq_handle
,
3511 if (!attr
.ext
.xrc
.cq
) {
3517 pd
= uobj_get_obj_read(pd
, cmd
->pd_handle
, file
->ucontext
);
3523 attr
.event_handler
= ib_uverbs_srq_event_handler
;
3524 attr
.srq_context
= file
;
3525 attr
.srq_type
= cmd
->srq_type
;
3526 attr
.attr
.max_wr
= cmd
->max_wr
;
3527 attr
.attr
.max_sge
= cmd
->max_sge
;
3528 attr
.attr
.srq_limit
= cmd
->srq_limit
;
3530 obj
->uevent
.events_reported
= 0;
3531 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
3533 srq
= pd
->device
->create_srq(pd
, &attr
, udata
);
3539 srq
->device
= pd
->device
;
3541 srq
->srq_type
= cmd
->srq_type
;
3542 srq
->uobject
= &obj
->uevent
.uobject
;
3543 srq
->event_handler
= attr
.event_handler
;
3544 srq
->srq_context
= attr
.srq_context
;
3546 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3547 srq
->ext
.xrc
.cq
= attr
.ext
.xrc
.cq
;
3548 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
3549 atomic_inc(&attr
.ext
.xrc
.cq
->usecnt
);
3550 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
3553 atomic_inc(&pd
->usecnt
);
3554 atomic_set(&srq
->usecnt
, 0);
3556 obj
->uevent
.uobject
.object
= srq
;
3557 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
3559 memset(&resp
, 0, sizeof resp
);
3560 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
3561 resp
.max_wr
= attr
.attr
.max_wr
;
3562 resp
.max_sge
= attr
.attr
.max_sge
;
3563 if (cmd
->srq_type
== IB_SRQT_XRC
)
3564 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
3566 if (copy_to_user((void __user
*) (unsigned long) cmd
->response
,
3567 &resp
, sizeof resp
)) {
3572 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3573 uobj_put_read(xrcd_uobj
);
3574 uobj_put_obj_read(attr
.ext
.xrc
.cq
);
3576 uobj_put_obj_read(pd
);
3577 uobj_alloc_commit(&obj
->uevent
.uobject
);
3582 ib_destroy_srq(srq
);
3585 uobj_put_obj_read(pd
);
3588 if (cmd
->srq_type
== IB_SRQT_XRC
)
3589 uobj_put_obj_read(attr
.ext
.xrc
.cq
);
3592 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3593 atomic_dec(&obj
->uxrcd
->refcnt
);
3594 uobj_put_read(xrcd_uobj
);
3598 uobj_alloc_abort(&obj
->uevent
.uobject
);
3602 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
3603 struct ib_device
*ib_dev
,
3604 const char __user
*buf
, int in_len
,
3607 struct ib_uverbs_create_srq cmd
;
3608 struct ib_uverbs_create_xsrq xcmd
;
3609 struct ib_uverbs_create_srq_resp resp
;
3610 struct ib_udata udata
;
3613 if (out_len
< sizeof resp
)
3616 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3619 xcmd
.response
= cmd
.response
;
3620 xcmd
.user_handle
= cmd
.user_handle
;
3621 xcmd
.srq_type
= IB_SRQT_BASIC
;
3622 xcmd
.pd_handle
= cmd
.pd_handle
;
3623 xcmd
.max_wr
= cmd
.max_wr
;
3624 xcmd
.max_sge
= cmd
.max_sge
;
3625 xcmd
.srq_limit
= cmd
.srq_limit
;
3627 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3628 (unsigned long) cmd
.response
+ sizeof resp
,
3629 in_len
- sizeof cmd
- sizeof(struct ib_uverbs_cmd_hdr
),
3630 out_len
- sizeof resp
);
3632 ret
= __uverbs_create_xsrq(file
, ib_dev
, &xcmd
, &udata
);
3639 ssize_t
ib_uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3640 struct ib_device
*ib_dev
,
3641 const char __user
*buf
, int in_len
, int out_len
)
3643 struct ib_uverbs_create_xsrq cmd
;
3644 struct ib_uverbs_create_srq_resp resp
;
3645 struct ib_udata udata
;
3648 if (out_len
< sizeof resp
)
3651 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3654 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3655 (unsigned long) cmd
.response
+ sizeof resp
,
3656 in_len
- sizeof cmd
- sizeof(struct ib_uverbs_cmd_hdr
),
3657 out_len
- sizeof resp
);
3659 ret
= __uverbs_create_xsrq(file
, ib_dev
, &cmd
, &udata
);
3666 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
3667 struct ib_device
*ib_dev
,
3668 const char __user
*buf
, int in_len
,
3671 struct ib_uverbs_modify_srq cmd
;
3672 struct ib_udata udata
;
3674 struct ib_srq_attr attr
;
3677 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3680 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
3683 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
3687 attr
.max_wr
= cmd
.max_wr
;
3688 attr
.srq_limit
= cmd
.srq_limit
;
3690 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
3692 uobj_put_obj_read(srq
);
3694 return ret
? ret
: in_len
;
3697 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
3698 struct ib_device
*ib_dev
,
3699 const char __user
*buf
,
3700 int in_len
, int out_len
)
3702 struct ib_uverbs_query_srq cmd
;
3703 struct ib_uverbs_query_srq_resp resp
;
3704 struct ib_srq_attr attr
;
3708 if (out_len
< sizeof resp
)
3711 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3714 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
3718 ret
= ib_query_srq(srq
, &attr
);
3720 uobj_put_obj_read(srq
);
3725 memset(&resp
, 0, sizeof resp
);
3727 resp
.max_wr
= attr
.max_wr
;
3728 resp
.max_sge
= attr
.max_sge
;
3729 resp
.srq_limit
= attr
.srq_limit
;
3731 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
3732 &resp
, sizeof resp
))
3738 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
3739 struct ib_device
*ib_dev
,
3740 const char __user
*buf
, int in_len
,
3743 struct ib_uverbs_destroy_srq cmd
;
3744 struct ib_uverbs_destroy_srq_resp resp
;
3745 struct ib_uobject
*uobj
;
3747 struct ib_uevent_object
*obj
;
3749 enum ib_srq_type srq_type
;
3751 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3754 uobj
= uobj_get_write(uobj_get_type(srq
), cmd
.srq_handle
,
3757 return PTR_ERR(uobj
);
3760 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
3761 srq_type
= srq
->srq_type
;
3763 * Make sure we don't free the memory in remove_commit as we still
3764 * needs the uobject memory to create the response.
3766 uverbs_uobject_get(uobj
);
3768 memset(&resp
, 0, sizeof(resp
));
3770 ret
= uobj_remove_commit(uobj
);
3772 uverbs_uobject_put(uobj
);
3775 resp
.events_reported
= obj
->events_reported
;
3776 uverbs_uobject_put(uobj
);
3777 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
3778 &resp
, sizeof(resp
)))
3784 int ib_uverbs_ex_query_device(struct ib_uverbs_file
*file
,
3785 struct ib_device
*ib_dev
,
3786 struct ib_udata
*ucore
,
3787 struct ib_udata
*uhw
)
3789 struct ib_uverbs_ex_query_device_resp resp
= { {0} };
3790 struct ib_uverbs_ex_query_device cmd
;
3791 struct ib_device_attr attr
= {0};
3794 if (ucore
->inlen
< sizeof(cmd
))
3797 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3807 resp
.response_length
= offsetof(typeof(resp
), odp_caps
);
3809 if (ucore
->outlen
< resp
.response_length
)
3812 err
= ib_dev
->query_device(ib_dev
, &attr
, uhw
);
3816 copy_query_dev_fields(file
, ib_dev
, &resp
.base
, &attr
);
3818 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.odp_caps
))
3821 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3822 resp
.odp_caps
.general_caps
= attr
.odp_caps
.general_caps
;
3823 resp
.odp_caps
.per_transport_caps
.rc_odp_caps
=
3824 attr
.odp_caps
.per_transport_caps
.rc_odp_caps
;
3825 resp
.odp_caps
.per_transport_caps
.uc_odp_caps
=
3826 attr
.odp_caps
.per_transport_caps
.uc_odp_caps
;
3827 resp
.odp_caps
.per_transport_caps
.ud_odp_caps
=
3828 attr
.odp_caps
.per_transport_caps
.ud_odp_caps
;
3830 resp
.response_length
+= sizeof(resp
.odp_caps
);
3832 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.timestamp_mask
))
3835 resp
.timestamp_mask
= attr
.timestamp_mask
;
3836 resp
.response_length
+= sizeof(resp
.timestamp_mask
);
3838 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.hca_core_clock
))
3841 resp
.hca_core_clock
= attr
.hca_core_clock
;
3842 resp
.response_length
+= sizeof(resp
.hca_core_clock
);
3844 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.device_cap_flags_ex
))
3847 resp
.device_cap_flags_ex
= attr
.device_cap_flags
;
3848 resp
.response_length
+= sizeof(resp
.device_cap_flags_ex
);
3850 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.rss_caps
))
3853 resp
.rss_caps
.supported_qpts
= attr
.rss_caps
.supported_qpts
;
3854 resp
.rss_caps
.max_rwq_indirection_tables
=
3855 attr
.rss_caps
.max_rwq_indirection_tables
;
3856 resp
.rss_caps
.max_rwq_indirection_table_size
=
3857 attr
.rss_caps
.max_rwq_indirection_table_size
;
3859 resp
.response_length
+= sizeof(resp
.rss_caps
);
3861 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.max_wq_type_rq
))
3864 resp
.max_wq_type_rq
= attr
.max_wq_type_rq
;
3865 resp
.response_length
+= sizeof(resp
.max_wq_type_rq
);
3867 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.raw_packet_caps
))
3870 resp
.raw_packet_caps
= attr
.raw_packet_caps
;
3871 resp
.response_length
+= sizeof(resp
.raw_packet_caps
);
3873 err
= ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);