2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
41 #include <linux/uaccess.h>
43 #include <rdma/uverbs_types.h>
44 #include <rdma/uverbs_std_types.h>
45 #include "rdma_core.h"
48 #include "core_priv.h"
50 static struct ib_uverbs_completion_event_file
*
51 ib_uverbs_lookup_comp_file(int fd
, struct ib_ucontext
*context
)
53 struct ib_uobject
*uobj
= uobj_get_read(uobj_get_type(comp_channel
),
55 struct ib_uobject_file
*uobj_file
;
60 uverbs_uobject_get(uobj
);
63 uobj_file
= container_of(uobj
, struct ib_uobject_file
, uobj
);
64 return container_of(uobj_file
, struct ib_uverbs_completion_event_file
,
68 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
69 struct ib_device
*ib_dev
,
70 const char __user
*buf
,
71 int in_len
, int out_len
)
73 struct ib_uverbs_get_context cmd
;
74 struct ib_uverbs_get_context_resp resp
;
75 struct ib_udata udata
;
76 struct ib_ucontext
*ucontext
;
78 struct ib_rdmacg_object cg_obj
;
81 if (out_len
< sizeof resp
)
84 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
87 mutex_lock(&file
->mutex
);
94 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
95 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
96 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
97 out_len
- sizeof(resp
));
99 ret
= ib_rdmacg_try_charge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
103 ucontext
= ib_dev
->alloc_ucontext(ib_dev
, &udata
);
104 if (IS_ERR(ucontext
)) {
105 ret
= PTR_ERR(ucontext
);
109 ucontext
->device
= ib_dev
;
110 ucontext
->cg_obj
= cg_obj
;
111 /* ufile is required when some objects are released */
112 ucontext
->ufile
= file
;
113 uverbs_initialize_ucontext(ucontext
);
116 ucontext
->tgid
= get_task_pid(current
->group_leader
, PIDTYPE_PID
);
118 ucontext
->closing
= 0;
120 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
121 ucontext
->umem_tree
= RB_ROOT_CACHED
;
122 init_rwsem(&ucontext
->umem_rwsem
);
123 ucontext
->odp_mrs_count
= 0;
124 INIT_LIST_HEAD(&ucontext
->no_private_counters
);
126 if (!(ib_dev
->attrs
.device_cap_flags
& IB_DEVICE_ON_DEMAND_PAGING
))
127 ucontext
->invalidate_range
= NULL
;
131 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
133 ret
= get_unused_fd_flags(O_CLOEXEC
);
138 filp
= ib_uverbs_alloc_async_event_file(file
, ib_dev
);
144 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
149 file
->ucontext
= ucontext
;
151 fd_install(resp
.async_fd
, filp
);
153 mutex_unlock(&file
->mutex
);
158 ib_uverbs_free_async_event_file(file
);
162 put_unused_fd(resp
.async_fd
);
165 put_pid(ucontext
->tgid
);
166 ib_dev
->dealloc_ucontext(ucontext
);
169 ib_rdmacg_uncharge(&cg_obj
, ib_dev
, RDMACG_RESOURCE_HCA_HANDLE
);
172 mutex_unlock(&file
->mutex
);
176 static void copy_query_dev_fields(struct ib_uverbs_file
*file
,
177 struct ib_device
*ib_dev
,
178 struct ib_uverbs_query_device_resp
*resp
,
179 struct ib_device_attr
*attr
)
181 resp
->fw_ver
= attr
->fw_ver
;
182 resp
->node_guid
= ib_dev
->node_guid
;
183 resp
->sys_image_guid
= attr
->sys_image_guid
;
184 resp
->max_mr_size
= attr
->max_mr_size
;
185 resp
->page_size_cap
= attr
->page_size_cap
;
186 resp
->vendor_id
= attr
->vendor_id
;
187 resp
->vendor_part_id
= attr
->vendor_part_id
;
188 resp
->hw_ver
= attr
->hw_ver
;
189 resp
->max_qp
= attr
->max_qp
;
190 resp
->max_qp_wr
= attr
->max_qp_wr
;
191 resp
->device_cap_flags
= lower_32_bits(attr
->device_cap_flags
);
192 resp
->max_sge
= attr
->max_sge
;
193 resp
->max_sge_rd
= attr
->max_sge_rd
;
194 resp
->max_cq
= attr
->max_cq
;
195 resp
->max_cqe
= attr
->max_cqe
;
196 resp
->max_mr
= attr
->max_mr
;
197 resp
->max_pd
= attr
->max_pd
;
198 resp
->max_qp_rd_atom
= attr
->max_qp_rd_atom
;
199 resp
->max_ee_rd_atom
= attr
->max_ee_rd_atom
;
200 resp
->max_res_rd_atom
= attr
->max_res_rd_atom
;
201 resp
->max_qp_init_rd_atom
= attr
->max_qp_init_rd_atom
;
202 resp
->max_ee_init_rd_atom
= attr
->max_ee_init_rd_atom
;
203 resp
->atomic_cap
= attr
->atomic_cap
;
204 resp
->max_ee
= attr
->max_ee
;
205 resp
->max_rdd
= attr
->max_rdd
;
206 resp
->max_mw
= attr
->max_mw
;
207 resp
->max_raw_ipv6_qp
= attr
->max_raw_ipv6_qp
;
208 resp
->max_raw_ethy_qp
= attr
->max_raw_ethy_qp
;
209 resp
->max_mcast_grp
= attr
->max_mcast_grp
;
210 resp
->max_mcast_qp_attach
= attr
->max_mcast_qp_attach
;
211 resp
->max_total_mcast_qp_attach
= attr
->max_total_mcast_qp_attach
;
212 resp
->max_ah
= attr
->max_ah
;
213 resp
->max_fmr
= attr
->max_fmr
;
214 resp
->max_map_per_fmr
= attr
->max_map_per_fmr
;
215 resp
->max_srq
= attr
->max_srq
;
216 resp
->max_srq_wr
= attr
->max_srq_wr
;
217 resp
->max_srq_sge
= attr
->max_srq_sge
;
218 resp
->max_pkeys
= attr
->max_pkeys
;
219 resp
->local_ca_ack_delay
= attr
->local_ca_ack_delay
;
220 resp
->phys_port_cnt
= ib_dev
->phys_port_cnt
;
223 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
224 struct ib_device
*ib_dev
,
225 const char __user
*buf
,
226 int in_len
, int out_len
)
228 struct ib_uverbs_query_device cmd
;
229 struct ib_uverbs_query_device_resp resp
;
231 if (out_len
< sizeof resp
)
234 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
237 memset(&resp
, 0, sizeof resp
);
238 copy_query_dev_fields(file
, ib_dev
, &resp
, &ib_dev
->attrs
);
240 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
246 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
247 struct ib_device
*ib_dev
,
248 const char __user
*buf
,
249 int in_len
, int out_len
)
251 struct ib_uverbs_query_port cmd
;
252 struct ib_uverbs_query_port_resp resp
;
253 struct ib_port_attr attr
;
256 if (out_len
< sizeof resp
)
259 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
262 ret
= ib_query_port(ib_dev
, cmd
.port_num
, &attr
);
266 memset(&resp
, 0, sizeof resp
);
268 resp
.state
= attr
.state
;
269 resp
.max_mtu
= attr
.max_mtu
;
270 resp
.active_mtu
= attr
.active_mtu
;
271 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
272 resp
.port_cap_flags
= attr
.port_cap_flags
;
273 resp
.max_msg_sz
= attr
.max_msg_sz
;
274 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
275 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
276 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
278 if (rdma_cap_opa_ah(ib_dev
, cmd
.port_num
)) {
279 resp
.lid
= OPA_TO_IB_UCAST_LID(attr
.lid
);
280 resp
.sm_lid
= OPA_TO_IB_UCAST_LID(attr
.sm_lid
);
282 resp
.lid
= ib_lid_cpu16(attr
.lid
);
283 resp
.sm_lid
= ib_lid_cpu16(attr
.sm_lid
);
286 resp
.max_vl_num
= attr
.max_vl_num
;
287 resp
.sm_sl
= attr
.sm_sl
;
288 resp
.subnet_timeout
= attr
.subnet_timeout
;
289 resp
.init_type_reply
= attr
.init_type_reply
;
290 resp
.active_width
= attr
.active_width
;
291 resp
.active_speed
= attr
.active_speed
;
292 resp
.phys_state
= attr
.phys_state
;
293 resp
.link_layer
= rdma_port_get_link_layer(ib_dev
,
296 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
302 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
303 struct ib_device
*ib_dev
,
304 const char __user
*buf
,
305 int in_len
, int out_len
)
307 struct ib_uverbs_alloc_pd cmd
;
308 struct ib_uverbs_alloc_pd_resp resp
;
309 struct ib_udata udata
;
310 struct ib_uobject
*uobj
;
314 if (out_len
< sizeof resp
)
317 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
320 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
321 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
322 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
323 out_len
- sizeof(resp
));
325 uobj
= uobj_alloc(uobj_get_type(pd
), file
->ucontext
);
327 return PTR_ERR(uobj
);
329 pd
= ib_dev
->alloc_pd(ib_dev
, file
->ucontext
, &udata
);
337 pd
->__internal_mr
= NULL
;
338 atomic_set(&pd
->usecnt
, 0);
341 memset(&resp
, 0, sizeof resp
);
342 resp
.pd_handle
= uobj
->id
;
344 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
349 uobj_alloc_commit(uobj
);
357 uobj_alloc_abort(uobj
);
361 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
362 struct ib_device
*ib_dev
,
363 const char __user
*buf
,
364 int in_len
, int out_len
)
366 struct ib_uverbs_dealloc_pd cmd
;
367 struct ib_uobject
*uobj
;
370 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
373 uobj
= uobj_get_write(uobj_get_type(pd
), cmd
.pd_handle
,
376 return PTR_ERR(uobj
);
378 ret
= uobj_remove_commit(uobj
);
380 return ret
?: in_len
;
383 struct xrcd_table_entry
{
385 struct ib_xrcd
*xrcd
;
389 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
391 struct ib_xrcd
*xrcd
)
393 struct xrcd_table_entry
*entry
, *scan
;
394 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
395 struct rb_node
*parent
= NULL
;
397 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
402 entry
->inode
= inode
;
406 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
408 if (inode
< scan
->inode
) {
410 } else if (inode
> scan
->inode
) {
418 rb_link_node(&entry
->node
, parent
, p
);
419 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
424 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
427 struct xrcd_table_entry
*entry
;
428 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
431 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
433 if (inode
< entry
->inode
)
435 else if (inode
> entry
->inode
)
444 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
446 struct xrcd_table_entry
*entry
;
448 entry
= xrcd_table_search(dev
, inode
);
455 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
458 struct xrcd_table_entry
*entry
;
460 entry
= xrcd_table_search(dev
, inode
);
463 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
468 ssize_t
ib_uverbs_open_xrcd(struct ib_uverbs_file
*file
,
469 struct ib_device
*ib_dev
,
470 const char __user
*buf
, int in_len
,
473 struct ib_uverbs_open_xrcd cmd
;
474 struct ib_uverbs_open_xrcd_resp resp
;
475 struct ib_udata udata
;
476 struct ib_uxrcd_object
*obj
;
477 struct ib_xrcd
*xrcd
= NULL
;
478 struct fd f
= {NULL
, 0};
479 struct inode
*inode
= NULL
;
483 if (out_len
< sizeof resp
)
486 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
489 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
490 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
491 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
492 out_len
- sizeof(resp
));
494 mutex_lock(&file
->device
->xrcd_tree_mutex
);
497 /* search for file descriptor */
501 goto err_tree_mutex_unlock
;
504 inode
= file_inode(f
.file
);
505 xrcd
= find_xrcd(file
->device
, inode
);
506 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
507 /* no file descriptor. Need CREATE flag */
509 goto err_tree_mutex_unlock
;
512 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
514 goto err_tree_mutex_unlock
;
518 obj
= (struct ib_uxrcd_object
*)uobj_alloc(uobj_get_type(xrcd
),
522 goto err_tree_mutex_unlock
;
526 xrcd
= ib_dev
->alloc_xrcd(ib_dev
, file
->ucontext
, &udata
);
533 xrcd
->device
= ib_dev
;
534 atomic_set(&xrcd
->usecnt
, 0);
535 mutex_init(&xrcd
->tgt_qp_mutex
);
536 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
540 atomic_set(&obj
->refcnt
, 0);
541 obj
->uobject
.object
= xrcd
;
542 memset(&resp
, 0, sizeof resp
);
543 resp
.xrcd_handle
= obj
->uobject
.id
;
547 /* create new inode/xrcd table entry */
548 ret
= xrcd_table_insert(file
->device
, inode
, xrcd
);
550 goto err_dealloc_xrcd
;
552 atomic_inc(&xrcd
->usecnt
);
555 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
563 uobj_alloc_commit(&obj
->uobject
);
565 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
571 xrcd_table_delete(file
->device
, inode
);
572 atomic_dec(&xrcd
->usecnt
);
576 ib_dealloc_xrcd(xrcd
);
579 uobj_alloc_abort(&obj
->uobject
);
581 err_tree_mutex_unlock
:
585 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
590 ssize_t
ib_uverbs_close_xrcd(struct ib_uverbs_file
*file
,
591 struct ib_device
*ib_dev
,
592 const char __user
*buf
, int in_len
,
595 struct ib_uverbs_close_xrcd cmd
;
596 struct ib_uobject
*uobj
;
599 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
602 uobj
= uobj_get_write(uobj_get_type(xrcd
), cmd
.xrcd_handle
,
605 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
606 return PTR_ERR(uobj
);
609 ret
= uobj_remove_commit(uobj
);
610 return ret
?: in_len
;
613 int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device
*dev
,
614 struct ib_xrcd
*xrcd
,
615 enum rdma_remove_reason why
)
621 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
624 ret
= ib_dealloc_xrcd(xrcd
);
626 if (why
== RDMA_REMOVE_DESTROY
&& ret
)
627 atomic_inc(&xrcd
->usecnt
);
629 xrcd_table_delete(dev
, inode
);
634 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
635 struct ib_device
*ib_dev
,
636 const char __user
*buf
, int in_len
,
639 struct ib_uverbs_reg_mr cmd
;
640 struct ib_uverbs_reg_mr_resp resp
;
641 struct ib_udata udata
;
642 struct ib_uobject
*uobj
;
647 if (out_len
< sizeof resp
)
650 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
653 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
654 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
655 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
656 out_len
- sizeof(resp
));
658 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
661 ret
= ib_check_mr_access(cmd
.access_flags
);
665 uobj
= uobj_alloc(uobj_get_type(mr
), file
->ucontext
);
667 return PTR_ERR(uobj
);
669 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
675 if (cmd
.access_flags
& IB_ACCESS_ON_DEMAND
) {
676 if (!(pd
->device
->attrs
.device_cap_flags
&
677 IB_DEVICE_ON_DEMAND_PAGING
)) {
678 pr_debug("ODP support not available\n");
684 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
685 cmd
.access_flags
, &udata
);
691 mr
->device
= pd
->device
;
694 atomic_inc(&pd
->usecnt
);
698 memset(&resp
, 0, sizeof resp
);
699 resp
.lkey
= mr
->lkey
;
700 resp
.rkey
= mr
->rkey
;
701 resp
.mr_handle
= uobj
->id
;
703 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
708 uobj_put_obj_read(pd
);
710 uobj_alloc_commit(uobj
);
718 uobj_put_obj_read(pd
);
721 uobj_alloc_abort(uobj
);
725 ssize_t
ib_uverbs_rereg_mr(struct ib_uverbs_file
*file
,
726 struct ib_device
*ib_dev
,
727 const char __user
*buf
, int in_len
,
730 struct ib_uverbs_rereg_mr cmd
;
731 struct ib_uverbs_rereg_mr_resp resp
;
732 struct ib_udata udata
;
733 struct ib_pd
*pd
= NULL
;
735 struct ib_pd
*old_pd
;
737 struct ib_uobject
*uobj
;
739 if (out_len
< sizeof(resp
))
742 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
745 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
746 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
747 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
748 out_len
- sizeof(resp
));
750 if (cmd
.flags
& ~IB_MR_REREG_SUPPORTED
|| !cmd
.flags
)
753 if ((cmd
.flags
& IB_MR_REREG_TRANS
) &&
754 (!cmd
.start
|| !cmd
.hca_va
|| 0 >= cmd
.length
||
755 (cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
)))
758 uobj
= uobj_get_write(uobj_get_type(mr
), cmd
.mr_handle
,
761 return PTR_ERR(uobj
);
765 if (cmd
.flags
& IB_MR_REREG_ACCESS
) {
766 ret
= ib_check_mr_access(cmd
.access_flags
);
771 if (cmd
.flags
& IB_MR_REREG_PD
) {
772 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
780 ret
= mr
->device
->rereg_user_mr(mr
, cmd
.flags
, cmd
.start
,
781 cmd
.length
, cmd
.hca_va
,
782 cmd
.access_flags
, pd
, &udata
);
784 if (cmd
.flags
& IB_MR_REREG_PD
) {
785 atomic_inc(&pd
->usecnt
);
787 atomic_dec(&old_pd
->usecnt
);
793 memset(&resp
, 0, sizeof(resp
));
794 resp
.lkey
= mr
->lkey
;
795 resp
.rkey
= mr
->rkey
;
797 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof(resp
)))
803 if (cmd
.flags
& IB_MR_REREG_PD
)
804 uobj_put_obj_read(pd
);
807 uobj_put_write(uobj
);
812 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
813 struct ib_device
*ib_dev
,
814 const char __user
*buf
, int in_len
,
817 struct ib_uverbs_dereg_mr cmd
;
818 struct ib_uobject
*uobj
;
821 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
824 uobj
= uobj_get_write(uobj_get_type(mr
), cmd
.mr_handle
,
827 return PTR_ERR(uobj
);
829 ret
= uobj_remove_commit(uobj
);
831 return ret
?: in_len
;
834 ssize_t
ib_uverbs_alloc_mw(struct ib_uverbs_file
*file
,
835 struct ib_device
*ib_dev
,
836 const char __user
*buf
, int in_len
,
839 struct ib_uverbs_alloc_mw cmd
;
840 struct ib_uverbs_alloc_mw_resp resp
;
841 struct ib_uobject
*uobj
;
844 struct ib_udata udata
;
847 if (out_len
< sizeof(resp
))
850 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
853 uobj
= uobj_alloc(uobj_get_type(mw
), file
->ucontext
);
855 return PTR_ERR(uobj
);
857 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
863 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
864 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
865 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
866 out_len
- sizeof(resp
));
868 mw
= pd
->device
->alloc_mw(pd
, cmd
.mw_type
, &udata
);
874 mw
->device
= pd
->device
;
877 atomic_inc(&pd
->usecnt
);
881 memset(&resp
, 0, sizeof(resp
));
882 resp
.rkey
= mw
->rkey
;
883 resp
.mw_handle
= uobj
->id
;
885 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof(resp
))) {
890 uobj_put_obj_read(pd
);
891 uobj_alloc_commit(uobj
);
896 uverbs_dealloc_mw(mw
);
898 uobj_put_obj_read(pd
);
900 uobj_alloc_abort(uobj
);
904 ssize_t
ib_uverbs_dealloc_mw(struct ib_uverbs_file
*file
,
905 struct ib_device
*ib_dev
,
906 const char __user
*buf
, int in_len
,
909 struct ib_uverbs_dealloc_mw cmd
;
910 struct ib_uobject
*uobj
;
913 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
916 uobj
= uobj_get_write(uobj_get_type(mw
), cmd
.mw_handle
,
919 return PTR_ERR(uobj
);
921 ret
= uobj_remove_commit(uobj
);
922 return ret
?: in_len
;
925 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
926 struct ib_device
*ib_dev
,
927 const char __user
*buf
, int in_len
,
930 struct ib_uverbs_create_comp_channel cmd
;
931 struct ib_uverbs_create_comp_channel_resp resp
;
932 struct ib_uobject
*uobj
;
933 struct ib_uverbs_completion_event_file
*ev_file
;
935 if (out_len
< sizeof resp
)
938 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
941 uobj
= uobj_alloc(uobj_get_type(comp_channel
), file
->ucontext
);
943 return PTR_ERR(uobj
);
947 ev_file
= container_of(uobj
, struct ib_uverbs_completion_event_file
,
949 ib_uverbs_init_event_queue(&ev_file
->ev_queue
);
951 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
952 uobj_alloc_abort(uobj
);
956 uobj_alloc_commit(uobj
);
960 static struct ib_ucq_object
*create_cq(struct ib_uverbs_file
*file
,
961 struct ib_device
*ib_dev
,
962 struct ib_udata
*ucore
,
963 struct ib_udata
*uhw
,
964 struct ib_uverbs_ex_create_cq
*cmd
,
966 int (*cb
)(struct ib_uverbs_file
*file
,
967 struct ib_ucq_object
*obj
,
968 struct ib_uverbs_ex_create_cq_resp
*resp
,
969 struct ib_udata
*udata
,
973 struct ib_ucq_object
*obj
;
974 struct ib_uverbs_completion_event_file
*ev_file
= NULL
;
977 struct ib_uverbs_ex_create_cq_resp resp
;
978 struct ib_cq_init_attr attr
= {};
980 if (cmd
->comp_vector
>= file
->device
->num_comp_vectors
)
981 return ERR_PTR(-EINVAL
);
983 obj
= (struct ib_ucq_object
*)uobj_alloc(uobj_get_type(cq
),
988 if (cmd
->comp_channel
>= 0) {
989 ev_file
= ib_uverbs_lookup_comp_file(cmd
->comp_channel
,
991 if (IS_ERR(ev_file
)) {
992 ret
= PTR_ERR(ev_file
);
997 obj
->uobject
.user_handle
= cmd
->user_handle
;
998 obj
->uverbs_file
= file
;
999 obj
->comp_events_reported
= 0;
1000 obj
->async_events_reported
= 0;
1001 INIT_LIST_HEAD(&obj
->comp_list
);
1002 INIT_LIST_HEAD(&obj
->async_list
);
1004 attr
.cqe
= cmd
->cqe
;
1005 attr
.comp_vector
= cmd
->comp_vector
;
1007 if (cmd_sz
> offsetof(typeof(*cmd
), flags
) + sizeof(cmd
->flags
))
1008 attr
.flags
= cmd
->flags
;
1010 cq
= ib_dev
->create_cq(ib_dev
, &attr
, file
->ucontext
, uhw
);
1016 cq
->device
= ib_dev
;
1017 cq
->uobject
= &obj
->uobject
;
1018 cq
->comp_handler
= ib_uverbs_comp_handler
;
1019 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1020 cq
->cq_context
= ev_file
? &ev_file
->ev_queue
: NULL
;
1021 atomic_set(&cq
->usecnt
, 0);
1023 obj
->uobject
.object
= cq
;
1024 memset(&resp
, 0, sizeof resp
);
1025 resp
.base
.cq_handle
= obj
->uobject
.id
;
1026 resp
.base
.cqe
= cq
->cqe
;
1028 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1029 sizeof(resp
.response_length
);
1031 ret
= cb(file
, obj
, &resp
, ucore
, context
);
1035 uobj_alloc_commit(&obj
->uobject
);
1044 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1047 uobj_alloc_abort(&obj
->uobject
);
1049 return ERR_PTR(ret
);
1052 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file
*file
,
1053 struct ib_ucq_object
*obj
,
1054 struct ib_uverbs_ex_create_cq_resp
*resp
,
1055 struct ib_udata
*ucore
, void *context
)
1057 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1063 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
1064 struct ib_device
*ib_dev
,
1065 const char __user
*buf
, int in_len
,
1068 struct ib_uverbs_create_cq cmd
;
1069 struct ib_uverbs_ex_create_cq cmd_ex
;
1070 struct ib_uverbs_create_cq_resp resp
;
1071 struct ib_udata ucore
;
1072 struct ib_udata uhw
;
1073 struct ib_ucq_object
*obj
;
1075 if (out_len
< sizeof(resp
))
1078 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1081 ib_uverbs_init_udata(&ucore
, buf
, u64_to_user_ptr(cmd
.response
),
1082 sizeof(cmd
), sizeof(resp
));
1084 ib_uverbs_init_udata(&uhw
, buf
+ sizeof(cmd
),
1085 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
1086 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
1087 out_len
- sizeof(resp
));
1089 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1090 cmd_ex
.user_handle
= cmd
.user_handle
;
1091 cmd_ex
.cqe
= cmd
.cqe
;
1092 cmd_ex
.comp_vector
= cmd
.comp_vector
;
1093 cmd_ex
.comp_channel
= cmd
.comp_channel
;
1095 obj
= create_cq(file
, ib_dev
, &ucore
, &uhw
, &cmd_ex
,
1096 offsetof(typeof(cmd_ex
), comp_channel
) +
1097 sizeof(cmd
.comp_channel
), ib_uverbs_create_cq_cb
,
1101 return PTR_ERR(obj
);
1106 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file
*file
,
1107 struct ib_ucq_object
*obj
,
1108 struct ib_uverbs_ex_create_cq_resp
*resp
,
1109 struct ib_udata
*ucore
, void *context
)
1111 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1117 int ib_uverbs_ex_create_cq(struct ib_uverbs_file
*file
,
1118 struct ib_device
*ib_dev
,
1119 struct ib_udata
*ucore
,
1120 struct ib_udata
*uhw
)
1122 struct ib_uverbs_ex_create_cq_resp resp
;
1123 struct ib_uverbs_ex_create_cq cmd
;
1124 struct ib_ucq_object
*obj
;
1127 if (ucore
->inlen
< sizeof(cmd
))
1130 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
1140 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1141 sizeof(resp
.response_length
)))
1144 obj
= create_cq(file
, ib_dev
, ucore
, uhw
, &cmd
,
1145 min(ucore
->inlen
, sizeof(cmd
)),
1146 ib_uverbs_ex_create_cq_cb
, NULL
);
1149 return PTR_ERR(obj
);
1154 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
1155 struct ib_device
*ib_dev
,
1156 const char __user
*buf
, int in_len
,
1159 struct ib_uverbs_resize_cq cmd
;
1160 struct ib_uverbs_resize_cq_resp resp
= {};
1161 struct ib_udata udata
;
1165 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1168 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
1169 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
1170 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
1171 out_len
- sizeof(resp
));
1173 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1177 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
1183 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
.cqe
))
1187 uobj_put_obj_read(cq
);
1189 return ret
? ret
: in_len
;
1192 static int copy_wc_to_user(struct ib_device
*ib_dev
, void __user
*dest
,
1195 struct ib_uverbs_wc tmp
;
1197 tmp
.wr_id
= wc
->wr_id
;
1198 tmp
.status
= wc
->status
;
1199 tmp
.opcode
= wc
->opcode
;
1200 tmp
.vendor_err
= wc
->vendor_err
;
1201 tmp
.byte_len
= wc
->byte_len
;
1202 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
1203 tmp
.qp_num
= wc
->qp
->qp_num
;
1204 tmp
.src_qp
= wc
->src_qp
;
1205 tmp
.wc_flags
= wc
->wc_flags
;
1206 tmp
.pkey_index
= wc
->pkey_index
;
1207 if (rdma_cap_opa_ah(ib_dev
, wc
->port_num
))
1208 tmp
.slid
= OPA_TO_IB_UCAST_LID(wc
->slid
);
1210 tmp
.slid
= ib_lid_cpu16(wc
->slid
);
1212 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1213 tmp
.port_num
= wc
->port_num
;
1216 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1222 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
1223 struct ib_device
*ib_dev
,
1224 const char __user
*buf
, int in_len
,
1227 struct ib_uverbs_poll_cq cmd
;
1228 struct ib_uverbs_poll_cq_resp resp
;
1229 u8 __user
*header_ptr
;
1230 u8 __user
*data_ptr
;
1235 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1238 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1242 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1243 header_ptr
= u64_to_user_ptr(cmd
.response
);
1244 data_ptr
= header_ptr
+ sizeof resp
;
1246 memset(&resp
, 0, sizeof resp
);
1247 while (resp
.count
< cmd
.ne
) {
1248 ret
= ib_poll_cq(cq
, 1, &wc
);
1254 ret
= copy_wc_to_user(ib_dev
, data_ptr
, &wc
);
1258 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1262 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1270 uobj_put_obj_read(cq
);
1274 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
1275 struct ib_device
*ib_dev
,
1276 const char __user
*buf
, int in_len
,
1279 struct ib_uverbs_req_notify_cq cmd
;
1282 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1285 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
1289 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1290 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1292 uobj_put_obj_read(cq
);
1297 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
1298 struct ib_device
*ib_dev
,
1299 const char __user
*buf
, int in_len
,
1302 struct ib_uverbs_destroy_cq cmd
;
1303 struct ib_uverbs_destroy_cq_resp resp
;
1304 struct ib_uobject
*uobj
;
1306 struct ib_ucq_object
*obj
;
1309 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1312 uobj
= uobj_get_write(uobj_get_type(cq
), cmd
.cq_handle
,
1315 return PTR_ERR(uobj
);
1318 * Make sure we don't free the memory in remove_commit as we still
1319 * needs the uobject memory to create the response.
1321 uverbs_uobject_get(uobj
);
1323 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1325 memset(&resp
, 0, sizeof(resp
));
1327 ret
= uobj_remove_commit(uobj
);
1329 uverbs_uobject_put(uobj
);
1333 resp
.comp_events_reported
= obj
->comp_events_reported
;
1334 resp
.async_events_reported
= obj
->async_events_reported
;
1336 uverbs_uobject_put(uobj
);
1337 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
1343 static int create_qp(struct ib_uverbs_file
*file
,
1344 struct ib_udata
*ucore
,
1345 struct ib_udata
*uhw
,
1346 struct ib_uverbs_ex_create_qp
*cmd
,
1348 int (*cb
)(struct ib_uverbs_file
*file
,
1349 struct ib_uverbs_ex_create_qp_resp
*resp
,
1350 struct ib_udata
*udata
),
1353 struct ib_uqp_object
*obj
;
1354 struct ib_device
*device
;
1355 struct ib_pd
*pd
= NULL
;
1356 struct ib_xrcd
*xrcd
= NULL
;
1357 struct ib_uobject
*xrcd_uobj
= ERR_PTR(-ENOENT
);
1358 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1359 struct ib_srq
*srq
= NULL
;
1362 struct ib_qp_init_attr attr
= {};
1363 struct ib_uverbs_ex_create_qp_resp resp
;
1365 struct ib_rwq_ind_table
*ind_tbl
= NULL
;
1368 if (cmd
->qp_type
== IB_QPT_RAW_PACKET
&& !capable(CAP_NET_RAW
))
1371 obj
= (struct ib_uqp_object
*)uobj_alloc(uobj_get_type(qp
),
1374 return PTR_ERR(obj
);
1376 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
1377 mutex_init(&obj
->mcast_lock
);
1379 if (cmd_sz
>= offsetof(typeof(*cmd
), rwq_ind_tbl_handle
) +
1380 sizeof(cmd
->rwq_ind_tbl_handle
) &&
1381 (cmd
->comp_mask
& IB_UVERBS_CREATE_QP_MASK_IND_TABLE
)) {
1382 ind_tbl
= uobj_get_obj_read(rwq_ind_table
,
1383 cmd
->rwq_ind_tbl_handle
,
1390 attr
.rwq_ind_tbl
= ind_tbl
;
1393 if (cmd_sz
> sizeof(*cmd
) &&
1394 !ib_is_udata_cleared(ucore
, sizeof(*cmd
),
1395 cmd_sz
- sizeof(*cmd
))) {
1400 if (ind_tbl
&& (cmd
->max_recv_wr
|| cmd
->max_recv_sge
|| cmd
->is_srq
)) {
1405 if (ind_tbl
&& !cmd
->max_send_wr
)
1408 if (cmd
->qp_type
== IB_QPT_XRC_TGT
) {
1409 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
->pd_handle
,
1412 if (IS_ERR(xrcd_uobj
)) {
1417 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1422 device
= xrcd
->device
;
1424 if (cmd
->qp_type
== IB_QPT_XRC_INI
) {
1425 cmd
->max_recv_wr
= 0;
1426 cmd
->max_recv_sge
= 0;
1429 srq
= uobj_get_obj_read(srq
, cmd
->srq_handle
,
1431 if (!srq
|| srq
->srq_type
== IB_SRQT_XRC
) {
1438 if (cmd
->recv_cq_handle
!= cmd
->send_cq_handle
) {
1439 rcq
= uobj_get_obj_read(cq
, cmd
->recv_cq_handle
,
1450 scq
= uobj_get_obj_read(cq
, cmd
->send_cq_handle
,
1454 pd
= uobj_get_obj_read(pd
, cmd
->pd_handle
, file
->ucontext
);
1455 if (!pd
|| (!scq
&& has_sq
)) {
1460 device
= pd
->device
;
1463 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1464 attr
.qp_context
= file
;
1469 attr
.sq_sig_type
= cmd
->sq_sig_all
? IB_SIGNAL_ALL_WR
:
1471 attr
.qp_type
= cmd
->qp_type
;
1472 attr
.create_flags
= 0;
1474 attr
.cap
.max_send_wr
= cmd
->max_send_wr
;
1475 attr
.cap
.max_recv_wr
= cmd
->max_recv_wr
;
1476 attr
.cap
.max_send_sge
= cmd
->max_send_sge
;
1477 attr
.cap
.max_recv_sge
= cmd
->max_recv_sge
;
1478 attr
.cap
.max_inline_data
= cmd
->max_inline_data
;
1480 obj
->uevent
.events_reported
= 0;
1481 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1482 INIT_LIST_HEAD(&obj
->mcast_list
);
1484 if (cmd_sz
>= offsetof(typeof(*cmd
), create_flags
) +
1485 sizeof(cmd
->create_flags
))
1486 attr
.create_flags
= cmd
->create_flags
;
1488 if (attr
.create_flags
& ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
|
1489 IB_QP_CREATE_CROSS_CHANNEL
|
1490 IB_QP_CREATE_MANAGED_SEND
|
1491 IB_QP_CREATE_MANAGED_RECV
|
1492 IB_QP_CREATE_SCATTER_FCS
|
1493 IB_QP_CREATE_CVLAN_STRIPPING
|
1494 IB_QP_CREATE_SOURCE_QPN
|
1495 IB_QP_CREATE_PCI_WRITE_END_PADDING
)) {
1500 if (attr
.create_flags
& IB_QP_CREATE_SOURCE_QPN
) {
1501 if (!capable(CAP_NET_RAW
)) {
1506 attr
.source_qpn
= cmd
->source_qpn
;
1509 buf
= (void *)cmd
+ sizeof(*cmd
);
1510 if (cmd_sz
> sizeof(*cmd
))
1511 if (!(buf
[0] == 0 && !memcmp(buf
, buf
+ 1,
1512 cmd_sz
- sizeof(*cmd
) - 1))) {
1517 if (cmd
->qp_type
== IB_QPT_XRC_TGT
)
1518 qp
= ib_create_qp(pd
, &attr
);
1520 qp
= device
->create_qp(pd
, &attr
, uhw
);
1527 if (cmd
->qp_type
!= IB_QPT_XRC_TGT
) {
1528 ret
= ib_create_qp_security(qp
, device
);
1533 qp
->device
= device
;
1535 qp
->send_cq
= attr
.send_cq
;
1536 qp
->recv_cq
= attr
.recv_cq
;
1538 qp
->rwq_ind_tbl
= ind_tbl
;
1539 qp
->event_handler
= attr
.event_handler
;
1540 qp
->qp_context
= attr
.qp_context
;
1541 qp
->qp_type
= attr
.qp_type
;
1542 atomic_set(&qp
->usecnt
, 0);
1543 atomic_inc(&pd
->usecnt
);
1546 atomic_inc(&attr
.send_cq
->usecnt
);
1548 atomic_inc(&attr
.recv_cq
->usecnt
);
1550 atomic_inc(&attr
.srq
->usecnt
);
1552 atomic_inc(&ind_tbl
->usecnt
);
1554 qp
->uobject
= &obj
->uevent
.uobject
;
1556 obj
->uevent
.uobject
.object
= qp
;
1558 memset(&resp
, 0, sizeof resp
);
1559 resp
.base
.qpn
= qp
->qp_num
;
1560 resp
.base
.qp_handle
= obj
->uevent
.uobject
.id
;
1561 resp
.base
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1562 resp
.base
.max_send_sge
= attr
.cap
.max_send_sge
;
1563 resp
.base
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1564 resp
.base
.max_send_wr
= attr
.cap
.max_send_wr
;
1565 resp
.base
.max_inline_data
= attr
.cap
.max_inline_data
;
1567 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1568 sizeof(resp
.response_length
);
1570 ret
= cb(file
, &resp
, ucore
);
1575 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
,
1577 atomic_inc(&obj
->uxrcd
->refcnt
);
1578 uobj_put_read(xrcd_uobj
);
1582 uobj_put_obj_read(pd
);
1584 uobj_put_obj_read(scq
);
1585 if (rcq
&& rcq
!= scq
)
1586 uobj_put_obj_read(rcq
);
1588 uobj_put_obj_read(srq
);
1590 uobj_put_obj_read(ind_tbl
);
1592 uobj_alloc_commit(&obj
->uevent
.uobject
);
1599 if (!IS_ERR(xrcd_uobj
))
1600 uobj_put_read(xrcd_uobj
);
1602 uobj_put_obj_read(pd
);
1604 uobj_put_obj_read(scq
);
1605 if (rcq
&& rcq
!= scq
)
1606 uobj_put_obj_read(rcq
);
1608 uobj_put_obj_read(srq
);
1610 uobj_put_obj_read(ind_tbl
);
1612 uobj_alloc_abort(&obj
->uevent
.uobject
);
1616 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file
*file
,
1617 struct ib_uverbs_ex_create_qp_resp
*resp
,
1618 struct ib_udata
*ucore
)
1620 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1626 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1627 struct ib_device
*ib_dev
,
1628 const char __user
*buf
, int in_len
,
1631 struct ib_uverbs_create_qp cmd
;
1632 struct ib_uverbs_ex_create_qp cmd_ex
;
1633 struct ib_udata ucore
;
1634 struct ib_udata uhw
;
1635 ssize_t resp_size
= sizeof(struct ib_uverbs_create_qp_resp
);
1638 if (out_len
< resp_size
)
1641 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1644 ib_uverbs_init_udata(&ucore
, buf
, u64_to_user_ptr(cmd
.response
),
1645 sizeof(cmd
), resp_size
);
1646 ib_uverbs_init_udata(&uhw
, buf
+ sizeof(cmd
),
1647 u64_to_user_ptr(cmd
.response
) + resp_size
,
1648 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
1649 out_len
- resp_size
);
1651 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1652 cmd_ex
.user_handle
= cmd
.user_handle
;
1653 cmd_ex
.pd_handle
= cmd
.pd_handle
;
1654 cmd_ex
.send_cq_handle
= cmd
.send_cq_handle
;
1655 cmd_ex
.recv_cq_handle
= cmd
.recv_cq_handle
;
1656 cmd_ex
.srq_handle
= cmd
.srq_handle
;
1657 cmd_ex
.max_send_wr
= cmd
.max_send_wr
;
1658 cmd_ex
.max_recv_wr
= cmd
.max_recv_wr
;
1659 cmd_ex
.max_send_sge
= cmd
.max_send_sge
;
1660 cmd_ex
.max_recv_sge
= cmd
.max_recv_sge
;
1661 cmd_ex
.max_inline_data
= cmd
.max_inline_data
;
1662 cmd_ex
.sq_sig_all
= cmd
.sq_sig_all
;
1663 cmd_ex
.qp_type
= cmd
.qp_type
;
1664 cmd_ex
.is_srq
= cmd
.is_srq
;
1666 err
= create_qp(file
, &ucore
, &uhw
, &cmd_ex
,
1667 offsetof(typeof(cmd_ex
), is_srq
) +
1668 sizeof(cmd
.is_srq
), ib_uverbs_create_qp_cb
,
1677 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file
*file
,
1678 struct ib_uverbs_ex_create_qp_resp
*resp
,
1679 struct ib_udata
*ucore
)
1681 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1687 int ib_uverbs_ex_create_qp(struct ib_uverbs_file
*file
,
1688 struct ib_device
*ib_dev
,
1689 struct ib_udata
*ucore
,
1690 struct ib_udata
*uhw
)
1692 struct ib_uverbs_ex_create_qp_resp resp
;
1693 struct ib_uverbs_ex_create_qp cmd
= {0};
1696 if (ucore
->inlen
< (offsetof(typeof(cmd
), comp_mask
) +
1697 sizeof(cmd
.comp_mask
)))
1700 err
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
1704 if (cmd
.comp_mask
& ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK
)
1710 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1711 sizeof(resp
.response_length
)))
1714 err
= create_qp(file
, ucore
, uhw
, &cmd
,
1715 min(ucore
->inlen
, sizeof(cmd
)),
1716 ib_uverbs_ex_create_qp_cb
, NULL
);
1724 ssize_t
ib_uverbs_open_qp(struct ib_uverbs_file
*file
,
1725 struct ib_device
*ib_dev
,
1726 const char __user
*buf
, int in_len
, int out_len
)
1728 struct ib_uverbs_open_qp cmd
;
1729 struct ib_uverbs_create_qp_resp resp
;
1730 struct ib_udata udata
;
1731 struct ib_uqp_object
*obj
;
1732 struct ib_xrcd
*xrcd
;
1733 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1735 struct ib_qp_open_attr attr
;
1738 if (out_len
< sizeof resp
)
1741 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1744 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
1745 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
1746 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
1747 out_len
- sizeof(resp
));
1749 obj
= (struct ib_uqp_object
*)uobj_alloc(uobj_get_type(qp
),
1752 return PTR_ERR(obj
);
1754 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
.pd_handle
,
1756 if (IS_ERR(xrcd_uobj
)) {
1761 xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
1767 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1768 attr
.qp_context
= file
;
1769 attr
.qp_num
= cmd
.qpn
;
1770 attr
.qp_type
= cmd
.qp_type
;
1772 obj
->uevent
.events_reported
= 0;
1773 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1774 INIT_LIST_HEAD(&obj
->mcast_list
);
1776 qp
= ib_open_qp(xrcd
, &attr
);
1782 obj
->uevent
.uobject
.object
= qp
;
1783 obj
->uevent
.uobject
.user_handle
= cmd
.user_handle
;
1785 memset(&resp
, 0, sizeof resp
);
1786 resp
.qpn
= qp
->qp_num
;
1787 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1789 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
1794 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
1795 atomic_inc(&obj
->uxrcd
->refcnt
);
1796 qp
->uobject
= &obj
->uevent
.uobject
;
1797 uobj_put_read(xrcd_uobj
);
1800 uobj_alloc_commit(&obj
->uevent
.uobject
);
1807 uobj_put_read(xrcd_uobj
);
1809 uobj_alloc_abort(&obj
->uevent
.uobject
);
1813 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest
*uverb_attr
,
1814 struct rdma_ah_attr
*rdma_attr
)
1816 const struct ib_global_route
*grh
;
1818 uverb_attr
->dlid
= rdma_ah_get_dlid(rdma_attr
);
1819 uverb_attr
->sl
= rdma_ah_get_sl(rdma_attr
);
1820 uverb_attr
->src_path_bits
= rdma_ah_get_path_bits(rdma_attr
);
1821 uverb_attr
->static_rate
= rdma_ah_get_static_rate(rdma_attr
);
1822 uverb_attr
->is_global
= !!(rdma_ah_get_ah_flags(rdma_attr
) &
1824 if (uverb_attr
->is_global
) {
1825 grh
= rdma_ah_read_grh(rdma_attr
);
1826 memcpy(uverb_attr
->dgid
, grh
->dgid
.raw
, 16);
1827 uverb_attr
->flow_label
= grh
->flow_label
;
1828 uverb_attr
->sgid_index
= grh
->sgid_index
;
1829 uverb_attr
->hop_limit
= grh
->hop_limit
;
1830 uverb_attr
->traffic_class
= grh
->traffic_class
;
1832 uverb_attr
->port_num
= rdma_ah_get_port_num(rdma_attr
);
1835 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
1836 struct ib_device
*ib_dev
,
1837 const char __user
*buf
, int in_len
,
1840 struct ib_uverbs_query_qp cmd
;
1841 struct ib_uverbs_query_qp_resp resp
;
1843 struct ib_qp_attr
*attr
;
1844 struct ib_qp_init_attr
*init_attr
;
1847 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1850 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1851 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
1852 if (!attr
|| !init_attr
) {
1857 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
1863 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
1865 uobj_put_obj_read(qp
);
1870 memset(&resp
, 0, sizeof resp
);
1872 resp
.qp_state
= attr
->qp_state
;
1873 resp
.cur_qp_state
= attr
->cur_qp_state
;
1874 resp
.path_mtu
= attr
->path_mtu
;
1875 resp
.path_mig_state
= attr
->path_mig_state
;
1876 resp
.qkey
= attr
->qkey
;
1877 resp
.rq_psn
= attr
->rq_psn
;
1878 resp
.sq_psn
= attr
->sq_psn
;
1879 resp
.dest_qp_num
= attr
->dest_qp_num
;
1880 resp
.qp_access_flags
= attr
->qp_access_flags
;
1881 resp
.pkey_index
= attr
->pkey_index
;
1882 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
1883 resp
.sq_draining
= attr
->sq_draining
;
1884 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
1885 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
1886 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
1887 resp
.port_num
= attr
->port_num
;
1888 resp
.timeout
= attr
->timeout
;
1889 resp
.retry_cnt
= attr
->retry_cnt
;
1890 resp
.rnr_retry
= attr
->rnr_retry
;
1891 resp
.alt_port_num
= attr
->alt_port_num
;
1892 resp
.alt_timeout
= attr
->alt_timeout
;
1894 copy_ah_attr_to_uverbs(&resp
.dest
, &attr
->ah_attr
);
1895 copy_ah_attr_to_uverbs(&resp
.alt_dest
, &attr
->alt_ah_attr
);
1897 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
1898 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
1899 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
1900 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
1901 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
1902 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
1904 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
1911 return ret
? ret
: in_len
;
1914 /* Remove ignored fields set in the attribute mask */
1915 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
1918 case IB_QPT_XRC_INI
:
1919 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
1920 case IB_QPT_XRC_TGT
:
1921 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
1928 static void copy_ah_attr_from_uverbs(struct ib_device
*dev
,
1929 struct rdma_ah_attr
*rdma_attr
,
1930 struct ib_uverbs_qp_dest
*uverb_attr
)
1932 rdma_attr
->type
= rdma_ah_find_type(dev
, uverb_attr
->port_num
);
1933 if (uverb_attr
->is_global
) {
1934 rdma_ah_set_grh(rdma_attr
, NULL
,
1935 uverb_attr
->flow_label
,
1936 uverb_attr
->sgid_index
,
1937 uverb_attr
->hop_limit
,
1938 uverb_attr
->traffic_class
);
1939 rdma_ah_set_dgid_raw(rdma_attr
, uverb_attr
->dgid
);
1941 rdma_ah_set_ah_flags(rdma_attr
, 0);
1943 rdma_ah_set_dlid(rdma_attr
, uverb_attr
->dlid
);
1944 rdma_ah_set_sl(rdma_attr
, uverb_attr
->sl
);
1945 rdma_ah_set_path_bits(rdma_attr
, uverb_attr
->src_path_bits
);
1946 rdma_ah_set_static_rate(rdma_attr
, uverb_attr
->static_rate
);
1947 rdma_ah_set_port_num(rdma_attr
, uverb_attr
->port_num
);
1948 rdma_ah_set_make_grd(rdma_attr
, false);
1951 static int modify_qp(struct ib_uverbs_file
*file
,
1952 struct ib_uverbs_ex_modify_qp
*cmd
, struct ib_udata
*udata
)
1954 struct ib_qp_attr
*attr
;
1958 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1962 qp
= uobj_get_obj_read(qp
, cmd
->base
.qp_handle
, file
->ucontext
);
1968 if ((cmd
->base
.attr_mask
& IB_QP_PORT
) &&
1969 !rdma_is_port_valid(qp
->device
, cmd
->base
.port_num
)) {
1974 attr
->qp_state
= cmd
->base
.qp_state
;
1975 attr
->cur_qp_state
= cmd
->base
.cur_qp_state
;
1976 attr
->path_mtu
= cmd
->base
.path_mtu
;
1977 attr
->path_mig_state
= cmd
->base
.path_mig_state
;
1978 attr
->qkey
= cmd
->base
.qkey
;
1979 attr
->rq_psn
= cmd
->base
.rq_psn
;
1980 attr
->sq_psn
= cmd
->base
.sq_psn
;
1981 attr
->dest_qp_num
= cmd
->base
.dest_qp_num
;
1982 attr
->qp_access_flags
= cmd
->base
.qp_access_flags
;
1983 attr
->pkey_index
= cmd
->base
.pkey_index
;
1984 attr
->alt_pkey_index
= cmd
->base
.alt_pkey_index
;
1985 attr
->en_sqd_async_notify
= cmd
->base
.en_sqd_async_notify
;
1986 attr
->max_rd_atomic
= cmd
->base
.max_rd_atomic
;
1987 attr
->max_dest_rd_atomic
= cmd
->base
.max_dest_rd_atomic
;
1988 attr
->min_rnr_timer
= cmd
->base
.min_rnr_timer
;
1989 attr
->port_num
= cmd
->base
.port_num
;
1990 attr
->timeout
= cmd
->base
.timeout
;
1991 attr
->retry_cnt
= cmd
->base
.retry_cnt
;
1992 attr
->rnr_retry
= cmd
->base
.rnr_retry
;
1993 attr
->alt_port_num
= cmd
->base
.alt_port_num
;
1994 attr
->alt_timeout
= cmd
->base
.alt_timeout
;
1995 attr
->rate_limit
= cmd
->rate_limit
;
1997 if (cmd
->base
.attr_mask
& IB_QP_AV
)
1998 copy_ah_attr_from_uverbs(qp
->device
, &attr
->ah_attr
,
2001 if (cmd
->base
.attr_mask
& IB_QP_ALT_PATH
)
2002 copy_ah_attr_from_uverbs(qp
->device
, &attr
->alt_ah_attr
,
2003 &cmd
->base
.alt_dest
);
2005 ret
= ib_modify_qp_with_udata(qp
, attr
,
2006 modify_qp_mask(qp
->qp_type
,
2007 cmd
->base
.attr_mask
),
2011 uobj_put_obj_read(qp
);
2018 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
2019 struct ib_device
*ib_dev
,
2020 const char __user
*buf
, int in_len
,
2023 struct ib_uverbs_ex_modify_qp cmd
= {};
2024 struct ib_udata udata
;
2027 if (copy_from_user(&cmd
.base
, buf
, sizeof(cmd
.base
)))
2030 if (cmd
.base
.attr_mask
&
2031 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK
<< 1) - 1))
2034 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
.base
), NULL
,
2035 in_len
- sizeof(cmd
.base
) - sizeof(struct ib_uverbs_cmd_hdr
),
2038 ret
= modify_qp(file
, &cmd
, &udata
);
2045 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file
*file
,
2046 struct ib_device
*ib_dev
,
2047 struct ib_udata
*ucore
,
2048 struct ib_udata
*uhw
)
2050 struct ib_uverbs_ex_modify_qp cmd
= {};
2054 * Last bit is reserved for extending the attr_mask by
2055 * using another field.
2057 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK
== (1 << 31));
2059 if (ucore
->inlen
< sizeof(cmd
.base
))
2062 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
2066 if (cmd
.base
.attr_mask
&
2067 ~((IB_USER_LAST_QP_ATTR_MASK
<< 1) - 1))
2070 if (ucore
->inlen
> sizeof(cmd
)) {
2071 if (ib_is_udata_cleared(ucore
, sizeof(cmd
),
2072 ucore
->inlen
- sizeof(cmd
)))
2076 ret
= modify_qp(file
, &cmd
, uhw
);
2081 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
2082 struct ib_device
*ib_dev
,
2083 const char __user
*buf
, int in_len
,
2086 struct ib_uverbs_destroy_qp cmd
;
2087 struct ib_uverbs_destroy_qp_resp resp
;
2088 struct ib_uobject
*uobj
;
2089 struct ib_uqp_object
*obj
;
2092 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2095 memset(&resp
, 0, sizeof resp
);
2097 uobj
= uobj_get_write(uobj_get_type(qp
), cmd
.qp_handle
,
2100 return PTR_ERR(uobj
);
2102 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
2104 * Make sure we don't free the memory in remove_commit as we still
2105 * needs the uobject memory to create the response.
2107 uverbs_uobject_get(uobj
);
2109 ret
= uobj_remove_commit(uobj
);
2111 uverbs_uobject_put(uobj
);
2115 resp
.events_reported
= obj
->uevent
.events_reported
;
2116 uverbs_uobject_put(uobj
);
2118 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
2124 static void *alloc_wr(size_t wr_size
, __u32 num_sge
)
2126 if (num_sge
>= (U32_MAX
- ALIGN(wr_size
, sizeof (struct ib_sge
))) /
2127 sizeof (struct ib_sge
))
2130 return kmalloc(ALIGN(wr_size
, sizeof (struct ib_sge
)) +
2131 num_sge
* sizeof (struct ib_sge
), GFP_KERNEL
);
2134 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
2135 struct ib_device
*ib_dev
,
2136 const char __user
*buf
, int in_len
,
2139 struct ib_uverbs_post_send cmd
;
2140 struct ib_uverbs_post_send_resp resp
;
2141 struct ib_uverbs_send_wr
*user_wr
;
2142 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
2146 ssize_t ret
= -EINVAL
;
2149 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2152 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
2153 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
2156 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
2159 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
2163 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2167 is_ud
= qp
->qp_type
== IB_QPT_UD
;
2170 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
2171 if (copy_from_user(user_wr
,
2172 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
2178 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
2184 struct ib_ud_wr
*ud
;
2186 if (user_wr
->opcode
!= IB_WR_SEND
&&
2187 user_wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
2192 next_size
= sizeof(*ud
);
2193 ud
= alloc_wr(next_size
, user_wr
->num_sge
);
2199 ud
->ah
= uobj_get_obj_read(ah
, user_wr
->wr
.ud
.ah
,
2206 ud
->remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
2207 ud
->remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
2210 } else if (user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
||
2211 user_wr
->opcode
== IB_WR_RDMA_WRITE
||
2212 user_wr
->opcode
== IB_WR_RDMA_READ
) {
2213 struct ib_rdma_wr
*rdma
;
2215 next_size
= sizeof(*rdma
);
2216 rdma
= alloc_wr(next_size
, user_wr
->num_sge
);
2222 rdma
->remote_addr
= user_wr
->wr
.rdma
.remote_addr
;
2223 rdma
->rkey
= user_wr
->wr
.rdma
.rkey
;
2226 } else if (user_wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
||
2227 user_wr
->opcode
== IB_WR_ATOMIC_FETCH_AND_ADD
) {
2228 struct ib_atomic_wr
*atomic
;
2230 next_size
= sizeof(*atomic
);
2231 atomic
= alloc_wr(next_size
, user_wr
->num_sge
);
2237 atomic
->remote_addr
= user_wr
->wr
.atomic
.remote_addr
;
2238 atomic
->compare_add
= user_wr
->wr
.atomic
.compare_add
;
2239 atomic
->swap
= user_wr
->wr
.atomic
.swap
;
2240 atomic
->rkey
= user_wr
->wr
.atomic
.rkey
;
2243 } else if (user_wr
->opcode
== IB_WR_SEND
||
2244 user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2245 user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2246 next_size
= sizeof(*next
);
2247 next
= alloc_wr(next_size
, user_wr
->num_sge
);
2257 if (user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2258 user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) {
2260 (__be32 __force
) user_wr
->ex
.imm_data
;
2261 } else if (user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2262 next
->ex
.invalidate_rkey
= user_wr
->ex
.invalidate_rkey
;
2272 next
->wr_id
= user_wr
->wr_id
;
2273 next
->num_sge
= user_wr
->num_sge
;
2274 next
->opcode
= user_wr
->opcode
;
2275 next
->send_flags
= user_wr
->send_flags
;
2277 if (next
->num_sge
) {
2278 next
->sg_list
= (void *) next
+
2279 ALIGN(next_size
, sizeof(struct ib_sge
));
2280 if (copy_from_user(next
->sg_list
,
2282 cmd
.wr_count
* cmd
.wqe_size
+
2283 sg_ind
* sizeof (struct ib_sge
),
2284 next
->num_sge
* sizeof (struct ib_sge
))) {
2288 sg_ind
+= next
->num_sge
;
2290 next
->sg_list
= NULL
;
2294 ret
= qp
->device
->post_send(qp
->real_qp
, wr
, &bad_wr
);
2296 for (next
= wr
; next
; next
= next
->next
) {
2302 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
2306 uobj_put_obj_read(qp
);
2309 if (is_ud
&& ud_wr(wr
)->ah
)
2310 uobj_put_obj_read(ud_wr(wr
)->ah
);
2319 return ret
? ret
: in_len
;
2322 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
2328 struct ib_uverbs_recv_wr
*user_wr
;
2329 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2334 if (in_len
< wqe_size
* wr_count
+
2335 sge_count
* sizeof (struct ib_uverbs_sge
))
2336 return ERR_PTR(-EINVAL
);
2338 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2339 return ERR_PTR(-EINVAL
);
2341 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2343 return ERR_PTR(-ENOMEM
);
2347 for (i
= 0; i
< wr_count
; ++i
) {
2348 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
2354 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2359 if (user_wr
->num_sge
>=
2360 (U32_MAX
- ALIGN(sizeof *next
, sizeof (struct ib_sge
))) /
2361 sizeof (struct ib_sge
)) {
2366 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2367 user_wr
->num_sge
* sizeof (struct ib_sge
),
2381 next
->wr_id
= user_wr
->wr_id
;
2382 next
->num_sge
= user_wr
->num_sge
;
2384 if (next
->num_sge
) {
2385 next
->sg_list
= (void *) next
+
2386 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2387 if (copy_from_user(next
->sg_list
,
2388 buf
+ wr_count
* wqe_size
+
2389 sg_ind
* sizeof (struct ib_sge
),
2390 next
->num_sge
* sizeof (struct ib_sge
))) {
2394 sg_ind
+= next
->num_sge
;
2396 next
->sg_list
= NULL
;
2411 return ERR_PTR(ret
);
2414 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
2415 struct ib_device
*ib_dev
,
2416 const char __user
*buf
, int in_len
,
2419 struct ib_uverbs_post_recv cmd
;
2420 struct ib_uverbs_post_recv_resp resp
;
2421 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2423 ssize_t ret
= -EINVAL
;
2425 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2428 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2429 in_len
- sizeof cmd
, cmd
.wr_count
,
2430 cmd
.sge_count
, cmd
.wqe_size
);
2434 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2439 ret
= qp
->device
->post_recv(qp
->real_qp
, wr
, &bad_wr
);
2441 uobj_put_obj_read(qp
);
2443 for (next
= wr
; next
; next
= next
->next
) {
2450 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
2460 return ret
? ret
: in_len
;
2463 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
2464 struct ib_device
*ib_dev
,
2465 const char __user
*buf
, int in_len
,
2468 struct ib_uverbs_post_srq_recv cmd
;
2469 struct ib_uverbs_post_srq_recv_resp resp
;
2470 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2472 ssize_t ret
= -EINVAL
;
2474 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2477 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2478 in_len
- sizeof cmd
, cmd
.wr_count
,
2479 cmd
.sge_count
, cmd
.wqe_size
);
2483 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
2488 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
2490 uobj_put_obj_read(srq
);
2493 for (next
= wr
; next
; next
= next
->next
) {
2499 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
2509 return ret
? ret
: in_len
;
2512 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
2513 struct ib_device
*ib_dev
,
2514 const char __user
*buf
, int in_len
,
2517 struct ib_uverbs_create_ah cmd
;
2518 struct ib_uverbs_create_ah_resp resp
;
2519 struct ib_uobject
*uobj
;
2522 struct rdma_ah_attr attr
;
2524 struct ib_udata udata
;
2526 if (out_len
< sizeof resp
)
2529 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2532 if (!rdma_is_port_valid(ib_dev
, cmd
.attr
.port_num
))
2535 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
2536 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
2537 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
2538 out_len
- sizeof(resp
));
2540 uobj
= uobj_alloc(uobj_get_type(ah
), file
->ucontext
);
2542 return PTR_ERR(uobj
);
2544 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
2550 attr
.type
= rdma_ah_find_type(ib_dev
, cmd
.attr
.port_num
);
2551 rdma_ah_set_make_grd(&attr
, false);
2552 rdma_ah_set_dlid(&attr
, cmd
.attr
.dlid
);
2553 rdma_ah_set_sl(&attr
, cmd
.attr
.sl
);
2554 rdma_ah_set_path_bits(&attr
, cmd
.attr
.src_path_bits
);
2555 rdma_ah_set_static_rate(&attr
, cmd
.attr
.static_rate
);
2556 rdma_ah_set_port_num(&attr
, cmd
.attr
.port_num
);
2558 if (cmd
.attr
.is_global
) {
2559 rdma_ah_set_grh(&attr
, NULL
, cmd
.attr
.grh
.flow_label
,
2560 cmd
.attr
.grh
.sgid_index
,
2561 cmd
.attr
.grh
.hop_limit
,
2562 cmd
.attr
.grh
.traffic_class
);
2563 rdma_ah_set_dgid_raw(&attr
, cmd
.attr
.grh
.dgid
);
2565 rdma_ah_set_ah_flags(&attr
, 0);
2568 ah
= rdma_create_user_ah(pd
, &attr
, &udata
);
2575 uobj
->user_handle
= cmd
.user_handle
;
2578 resp
.ah_handle
= uobj
->id
;
2580 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
)) {
2585 uobj_put_obj_read(pd
);
2586 uobj_alloc_commit(uobj
);
2591 rdma_destroy_ah(ah
);
2594 uobj_put_obj_read(pd
);
2597 uobj_alloc_abort(uobj
);
2601 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
2602 struct ib_device
*ib_dev
,
2603 const char __user
*buf
, int in_len
, int out_len
)
2605 struct ib_uverbs_destroy_ah cmd
;
2606 struct ib_uobject
*uobj
;
2609 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2612 uobj
= uobj_get_write(uobj_get_type(ah
), cmd
.ah_handle
,
2615 return PTR_ERR(uobj
);
2617 ret
= uobj_remove_commit(uobj
);
2618 return ret
?: in_len
;
2621 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
2622 struct ib_device
*ib_dev
,
2623 const char __user
*buf
, int in_len
,
2626 struct ib_uverbs_attach_mcast cmd
;
2628 struct ib_uqp_object
*obj
;
2629 struct ib_uverbs_mcast_entry
*mcast
;
2632 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2635 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2639 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2641 mutex_lock(&obj
->mcast_lock
);
2642 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2643 if (cmd
.mlid
== mcast
->lid
&&
2644 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2649 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2655 mcast
->lid
= cmd
.mlid
;
2656 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2658 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2660 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2665 mutex_unlock(&obj
->mcast_lock
);
2666 uobj_put_obj_read(qp
);
2668 return ret
? ret
: in_len
;
2671 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
2672 struct ib_device
*ib_dev
,
2673 const char __user
*buf
, int in_len
,
2676 struct ib_uverbs_detach_mcast cmd
;
2677 struct ib_uqp_object
*obj
;
2679 struct ib_uverbs_mcast_entry
*mcast
;
2683 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2686 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
2690 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2691 mutex_lock(&obj
->mcast_lock
);
2693 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2694 if (cmd
.mlid
== mcast
->lid
&&
2695 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2696 list_del(&mcast
->list
);
2707 ret
= ib_detach_mcast(qp
, (union ib_gid
*)cmd
.gid
, cmd
.mlid
);
2710 mutex_unlock(&obj
->mcast_lock
);
2711 uobj_put_obj_read(qp
);
2712 return ret
? ret
: in_len
;
2715 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec
*kern_spec
,
2716 union ib_flow_spec
*ib_spec
)
2718 ib_spec
->type
= kern_spec
->type
;
2719 switch (ib_spec
->type
) {
2720 case IB_FLOW_SPEC_ACTION_TAG
:
2721 if (kern_spec
->flow_tag
.size
!=
2722 sizeof(struct ib_uverbs_flow_spec_action_tag
))
2725 ib_spec
->flow_tag
.size
= sizeof(struct ib_flow_spec_action_tag
);
2726 ib_spec
->flow_tag
.tag_id
= kern_spec
->flow_tag
.tag_id
;
2728 case IB_FLOW_SPEC_ACTION_DROP
:
2729 if (kern_spec
->drop
.size
!=
2730 sizeof(struct ib_uverbs_flow_spec_action_drop
))
2733 ib_spec
->drop
.size
= sizeof(struct ib_flow_spec_action_drop
);
2741 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr
*spec
)
2743 /* Returns user space filter size, includes padding */
2744 return (spec
->size
- sizeof(struct ib_uverbs_flow_spec_hdr
)) / 2;
2747 static ssize_t
spec_filter_size(void *kern_spec_filter
, u16 kern_filter_size
,
2748 u16 ib_real_filter_sz
)
2751 * User space filter structures must be 64 bit aligned, otherwise this
2752 * may pass, but we won't handle additional new attributes.
2755 if (kern_filter_size
> ib_real_filter_sz
) {
2756 if (memchr_inv(kern_spec_filter
+
2757 ib_real_filter_sz
, 0,
2758 kern_filter_size
- ib_real_filter_sz
))
2760 return ib_real_filter_sz
;
2762 return kern_filter_size
;
2765 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec
*kern_spec
,
2766 union ib_flow_spec
*ib_spec
)
2768 ssize_t actual_filter_sz
;
2769 ssize_t kern_filter_sz
;
2770 ssize_t ib_filter_sz
;
2771 void *kern_spec_mask
;
2772 void *kern_spec_val
;
2774 if (kern_spec
->reserved
)
2777 ib_spec
->type
= kern_spec
->type
;
2779 kern_filter_sz
= kern_spec_filter_sz(&kern_spec
->hdr
);
2780 /* User flow spec size must be aligned to 4 bytes */
2781 if (kern_filter_sz
!= ALIGN(kern_filter_sz
, 4))
2784 kern_spec_val
= (void *)kern_spec
+
2785 sizeof(struct ib_uverbs_flow_spec_hdr
);
2786 kern_spec_mask
= kern_spec_val
+ kern_filter_sz
;
2787 if (ib_spec
->type
== (IB_FLOW_SPEC_INNER
| IB_FLOW_SPEC_VXLAN_TUNNEL
))
2790 switch (ib_spec
->type
& ~IB_FLOW_SPEC_INNER
) {
2791 case IB_FLOW_SPEC_ETH
:
2792 ib_filter_sz
= offsetof(struct ib_flow_eth_filter
, real_sz
);
2793 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2796 if (actual_filter_sz
<= 0)
2798 ib_spec
->size
= sizeof(struct ib_flow_spec_eth
);
2799 memcpy(&ib_spec
->eth
.val
, kern_spec_val
, actual_filter_sz
);
2800 memcpy(&ib_spec
->eth
.mask
, kern_spec_mask
, actual_filter_sz
);
2802 case IB_FLOW_SPEC_IPV4
:
2803 ib_filter_sz
= offsetof(struct ib_flow_ipv4_filter
, real_sz
);
2804 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2807 if (actual_filter_sz
<= 0)
2809 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv4
);
2810 memcpy(&ib_spec
->ipv4
.val
, kern_spec_val
, actual_filter_sz
);
2811 memcpy(&ib_spec
->ipv4
.mask
, kern_spec_mask
, actual_filter_sz
);
2813 case IB_FLOW_SPEC_IPV6
:
2814 ib_filter_sz
= offsetof(struct ib_flow_ipv6_filter
, real_sz
);
2815 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2818 if (actual_filter_sz
<= 0)
2820 ib_spec
->size
= sizeof(struct ib_flow_spec_ipv6
);
2821 memcpy(&ib_spec
->ipv6
.val
, kern_spec_val
, actual_filter_sz
);
2822 memcpy(&ib_spec
->ipv6
.mask
, kern_spec_mask
, actual_filter_sz
);
2824 if ((ntohl(ib_spec
->ipv6
.mask
.flow_label
)) >= BIT(20) ||
2825 (ntohl(ib_spec
->ipv6
.val
.flow_label
)) >= BIT(20))
2828 case IB_FLOW_SPEC_TCP
:
2829 case IB_FLOW_SPEC_UDP
:
2830 ib_filter_sz
= offsetof(struct ib_flow_tcp_udp_filter
, real_sz
);
2831 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2834 if (actual_filter_sz
<= 0)
2836 ib_spec
->size
= sizeof(struct ib_flow_spec_tcp_udp
);
2837 memcpy(&ib_spec
->tcp_udp
.val
, kern_spec_val
, actual_filter_sz
);
2838 memcpy(&ib_spec
->tcp_udp
.mask
, kern_spec_mask
, actual_filter_sz
);
2840 case IB_FLOW_SPEC_VXLAN_TUNNEL
:
2841 ib_filter_sz
= offsetof(struct ib_flow_tunnel_filter
, real_sz
);
2842 actual_filter_sz
= spec_filter_size(kern_spec_mask
,
2845 if (actual_filter_sz
<= 0)
2847 ib_spec
->tunnel
.size
= sizeof(struct ib_flow_spec_tunnel
);
2848 memcpy(&ib_spec
->tunnel
.val
, kern_spec_val
, actual_filter_sz
);
2849 memcpy(&ib_spec
->tunnel
.mask
, kern_spec_mask
, actual_filter_sz
);
2851 if ((ntohl(ib_spec
->tunnel
.mask
.tunnel_id
)) >= BIT(24) ||
2852 (ntohl(ib_spec
->tunnel
.val
.tunnel_id
)) >= BIT(24))
2861 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec
*kern_spec
,
2862 union ib_flow_spec
*ib_spec
)
2864 if (kern_spec
->reserved
)
2867 if (kern_spec
->type
>= IB_FLOW_SPEC_ACTION_TAG
)
2868 return kern_spec_to_ib_spec_action(kern_spec
, ib_spec
);
2870 return kern_spec_to_ib_spec_filter(kern_spec
, ib_spec
);
2873 int ib_uverbs_ex_create_wq(struct ib_uverbs_file
*file
,
2874 struct ib_device
*ib_dev
,
2875 struct ib_udata
*ucore
,
2876 struct ib_udata
*uhw
)
2878 struct ib_uverbs_ex_create_wq cmd
= {};
2879 struct ib_uverbs_ex_create_wq_resp resp
= {};
2880 struct ib_uwq_object
*obj
;
2885 struct ib_wq_init_attr wq_init_attr
= {};
2886 size_t required_cmd_sz
;
2887 size_t required_resp_len
;
2889 required_cmd_sz
= offsetof(typeof(cmd
), max_sge
) + sizeof(cmd
.max_sge
);
2890 required_resp_len
= offsetof(typeof(resp
), wqn
) + sizeof(resp
.wqn
);
2892 if (ucore
->inlen
< required_cmd_sz
)
2895 if (ucore
->outlen
< required_resp_len
)
2898 if (ucore
->inlen
> sizeof(cmd
) &&
2899 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
2900 ucore
->inlen
- sizeof(cmd
)))
2903 err
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
2910 obj
= (struct ib_uwq_object
*)uobj_alloc(uobj_get_type(wq
),
2913 return PTR_ERR(obj
);
2915 pd
= uobj_get_obj_read(pd
, cmd
.pd_handle
, file
->ucontext
);
2921 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
2927 wq_init_attr
.cq
= cq
;
2928 wq_init_attr
.max_sge
= cmd
.max_sge
;
2929 wq_init_attr
.max_wr
= cmd
.max_wr
;
2930 wq_init_attr
.wq_context
= file
;
2931 wq_init_attr
.wq_type
= cmd
.wq_type
;
2932 wq_init_attr
.event_handler
= ib_uverbs_wq_event_handler
;
2933 if (ucore
->inlen
>= (offsetof(typeof(cmd
), create_flags
) +
2934 sizeof(cmd
.create_flags
)))
2935 wq_init_attr
.create_flags
= cmd
.create_flags
;
2936 obj
->uevent
.events_reported
= 0;
2937 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
2938 wq
= pd
->device
->create_wq(pd
, &wq_init_attr
, uhw
);
2944 wq
->uobject
= &obj
->uevent
.uobject
;
2945 obj
->uevent
.uobject
.object
= wq
;
2946 wq
->wq_type
= wq_init_attr
.wq_type
;
2949 wq
->device
= pd
->device
;
2950 wq
->wq_context
= wq_init_attr
.wq_context
;
2951 atomic_set(&wq
->usecnt
, 0);
2952 atomic_inc(&pd
->usecnt
);
2953 atomic_inc(&cq
->usecnt
);
2954 wq
->uobject
= &obj
->uevent
.uobject
;
2955 obj
->uevent
.uobject
.object
= wq
;
2957 memset(&resp
, 0, sizeof(resp
));
2958 resp
.wq_handle
= obj
->uevent
.uobject
.id
;
2959 resp
.max_sge
= wq_init_attr
.max_sge
;
2960 resp
.max_wr
= wq_init_attr
.max_wr
;
2961 resp
.wqn
= wq
->wq_num
;
2962 resp
.response_length
= required_resp_len
;
2963 err
= ib_copy_to_udata(ucore
,
2964 &resp
, resp
.response_length
);
2968 uobj_put_obj_read(pd
);
2969 uobj_put_obj_read(cq
);
2970 uobj_alloc_commit(&obj
->uevent
.uobject
);
2976 uobj_put_obj_read(cq
);
2978 uobj_put_obj_read(pd
);
2980 uobj_alloc_abort(&obj
->uevent
.uobject
);
2985 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file
*file
,
2986 struct ib_device
*ib_dev
,
2987 struct ib_udata
*ucore
,
2988 struct ib_udata
*uhw
)
2990 struct ib_uverbs_ex_destroy_wq cmd
= {};
2991 struct ib_uverbs_ex_destroy_wq_resp resp
= {};
2992 struct ib_uobject
*uobj
;
2993 struct ib_uwq_object
*obj
;
2994 size_t required_cmd_sz
;
2995 size_t required_resp_len
;
2998 required_cmd_sz
= offsetof(typeof(cmd
), wq_handle
) + sizeof(cmd
.wq_handle
);
2999 required_resp_len
= offsetof(typeof(resp
), reserved
) + sizeof(resp
.reserved
);
3001 if (ucore
->inlen
< required_cmd_sz
)
3004 if (ucore
->outlen
< required_resp_len
)
3007 if (ucore
->inlen
> sizeof(cmd
) &&
3008 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3009 ucore
->inlen
- sizeof(cmd
)))
3012 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3019 resp
.response_length
= required_resp_len
;
3020 uobj
= uobj_get_write(uobj_get_type(wq
), cmd
.wq_handle
,
3023 return PTR_ERR(uobj
);
3025 obj
= container_of(uobj
, struct ib_uwq_object
, uevent
.uobject
);
3027 * Make sure we don't free the memory in remove_commit as we still
3028 * needs the uobject memory to create the response.
3030 uverbs_uobject_get(uobj
);
3032 ret
= uobj_remove_commit(uobj
);
3033 resp
.events_reported
= obj
->uevent
.events_reported
;
3034 uverbs_uobject_put(uobj
);
3038 return ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);
3041 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file
*file
,
3042 struct ib_device
*ib_dev
,
3043 struct ib_udata
*ucore
,
3044 struct ib_udata
*uhw
)
3046 struct ib_uverbs_ex_modify_wq cmd
= {};
3048 struct ib_wq_attr wq_attr
= {};
3049 size_t required_cmd_sz
;
3052 required_cmd_sz
= offsetof(typeof(cmd
), curr_wq_state
) + sizeof(cmd
.curr_wq_state
);
3053 if (ucore
->inlen
< required_cmd_sz
)
3056 if (ucore
->inlen
> sizeof(cmd
) &&
3057 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3058 ucore
->inlen
- sizeof(cmd
)))
3061 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3068 if (cmd
.attr_mask
> (IB_WQ_STATE
| IB_WQ_CUR_STATE
| IB_WQ_FLAGS
))
3071 wq
= uobj_get_obj_read(wq
, cmd
.wq_handle
, file
->ucontext
);
3075 wq_attr
.curr_wq_state
= cmd
.curr_wq_state
;
3076 wq_attr
.wq_state
= cmd
.wq_state
;
3077 if (cmd
.attr_mask
& IB_WQ_FLAGS
) {
3078 wq_attr
.flags
= cmd
.flags
;
3079 wq_attr
.flags_mask
= cmd
.flags_mask
;
3081 ret
= wq
->device
->modify_wq(wq
, &wq_attr
, cmd
.attr_mask
, uhw
);
3082 uobj_put_obj_read(wq
);
3086 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file
*file
,
3087 struct ib_device
*ib_dev
,
3088 struct ib_udata
*ucore
,
3089 struct ib_udata
*uhw
)
3091 struct ib_uverbs_ex_create_rwq_ind_table cmd
= {};
3092 struct ib_uverbs_ex_create_rwq_ind_table_resp resp
= {};
3093 struct ib_uobject
*uobj
;
3095 struct ib_rwq_ind_table_init_attr init_attr
= {};
3096 struct ib_rwq_ind_table
*rwq_ind_tbl
;
3097 struct ib_wq
**wqs
= NULL
;
3098 u32
*wqs_handles
= NULL
;
3099 struct ib_wq
*wq
= NULL
;
3100 int i
, j
, num_read_wqs
;
3102 u32 expected_in_size
;
3103 size_t required_cmd_sz_header
;
3104 size_t required_resp_len
;
3106 required_cmd_sz_header
= offsetof(typeof(cmd
), log_ind_tbl_size
) + sizeof(cmd
.log_ind_tbl_size
);
3107 required_resp_len
= offsetof(typeof(resp
), ind_tbl_num
) + sizeof(resp
.ind_tbl_num
);
3109 if (ucore
->inlen
< required_cmd_sz_header
)
3112 if (ucore
->outlen
< required_resp_len
)
3115 err
= ib_copy_from_udata(&cmd
, ucore
, required_cmd_sz_header
);
3119 ucore
->inbuf
+= required_cmd_sz_header
;
3120 ucore
->inlen
-= required_cmd_sz_header
;
3125 if (cmd
.log_ind_tbl_size
> IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE
)
3128 num_wq_handles
= 1 << cmd
.log_ind_tbl_size
;
3129 expected_in_size
= num_wq_handles
* sizeof(__u32
);
3130 if (num_wq_handles
== 1)
3131 /* input size for wq handles is u64 aligned */
3132 expected_in_size
+= sizeof(__u32
);
3134 if (ucore
->inlen
< expected_in_size
)
3137 if (ucore
->inlen
> expected_in_size
&&
3138 !ib_is_udata_cleared(ucore
, expected_in_size
,
3139 ucore
->inlen
- expected_in_size
))
3142 wqs_handles
= kcalloc(num_wq_handles
, sizeof(*wqs_handles
),
3147 err
= ib_copy_from_udata(wqs_handles
, ucore
,
3148 num_wq_handles
* sizeof(__u32
));
3152 wqs
= kcalloc(num_wq_handles
, sizeof(*wqs
), GFP_KERNEL
);
3158 for (num_read_wqs
= 0; num_read_wqs
< num_wq_handles
;
3160 wq
= uobj_get_obj_read(wq
, wqs_handles
[num_read_wqs
],
3167 wqs
[num_read_wqs
] = wq
;
3170 uobj
= uobj_alloc(uobj_get_type(rwq_ind_table
), file
->ucontext
);
3172 err
= PTR_ERR(uobj
);
3176 init_attr
.log_ind_tbl_size
= cmd
.log_ind_tbl_size
;
3177 init_attr
.ind_tbl
= wqs
;
3178 rwq_ind_tbl
= ib_dev
->create_rwq_ind_table(ib_dev
, &init_attr
, uhw
);
3180 if (IS_ERR(rwq_ind_tbl
)) {
3181 err
= PTR_ERR(rwq_ind_tbl
);
3185 rwq_ind_tbl
->ind_tbl
= wqs
;
3186 rwq_ind_tbl
->log_ind_tbl_size
= init_attr
.log_ind_tbl_size
;
3187 rwq_ind_tbl
->uobject
= uobj
;
3188 uobj
->object
= rwq_ind_tbl
;
3189 rwq_ind_tbl
->device
= ib_dev
;
3190 atomic_set(&rwq_ind_tbl
->usecnt
, 0);
3192 for (i
= 0; i
< num_wq_handles
; i
++)
3193 atomic_inc(&wqs
[i
]->usecnt
);
3195 resp
.ind_tbl_handle
= uobj
->id
;
3196 resp
.ind_tbl_num
= rwq_ind_tbl
->ind_tbl_num
;
3197 resp
.response_length
= required_resp_len
;
3199 err
= ib_copy_to_udata(ucore
,
3200 &resp
, resp
.response_length
);
3206 for (j
= 0; j
< num_read_wqs
; j
++)
3207 uobj_put_obj_read(wqs
[j
]);
3209 uobj_alloc_commit(uobj
);
3213 ib_destroy_rwq_ind_table(rwq_ind_tbl
);
3215 uobj_alloc_abort(uobj
);
3217 for (j
= 0; j
< num_read_wqs
; j
++)
3218 uobj_put_obj_read(wqs
[j
]);
3225 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file
*file
,
3226 struct ib_device
*ib_dev
,
3227 struct ib_udata
*ucore
,
3228 struct ib_udata
*uhw
)
3230 struct ib_uverbs_ex_destroy_rwq_ind_table cmd
= {};
3231 struct ib_uobject
*uobj
;
3233 size_t required_cmd_sz
;
3235 required_cmd_sz
= offsetof(typeof(cmd
), ind_tbl_handle
) + sizeof(cmd
.ind_tbl_handle
);
3237 if (ucore
->inlen
< required_cmd_sz
)
3240 if (ucore
->inlen
> sizeof(cmd
) &&
3241 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3242 ucore
->inlen
- sizeof(cmd
)))
3245 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3252 uobj
= uobj_get_write(uobj_get_type(rwq_ind_table
), cmd
.ind_tbl_handle
,
3255 return PTR_ERR(uobj
);
3257 return uobj_remove_commit(uobj
);
3260 int ib_uverbs_ex_create_flow(struct ib_uverbs_file
*file
,
3261 struct ib_device
*ib_dev
,
3262 struct ib_udata
*ucore
,
3263 struct ib_udata
*uhw
)
3265 struct ib_uverbs_create_flow cmd
;
3266 struct ib_uverbs_create_flow_resp resp
;
3267 struct ib_uobject
*uobj
;
3268 struct ib_flow
*flow_id
;
3269 struct ib_uverbs_flow_attr
*kern_flow_attr
;
3270 struct ib_flow_attr
*flow_attr
;
3277 if (ucore
->inlen
< sizeof(cmd
))
3280 if (ucore
->outlen
< sizeof(resp
))
3283 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3287 ucore
->inbuf
+= sizeof(cmd
);
3288 ucore
->inlen
-= sizeof(cmd
);
3293 if (!capable(CAP_NET_RAW
))
3296 if (cmd
.flow_attr
.flags
>= IB_FLOW_ATTR_FLAGS_RESERVED
)
3299 if ((cmd
.flow_attr
.flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
) &&
3300 ((cmd
.flow_attr
.type
== IB_FLOW_ATTR_ALL_DEFAULT
) ||
3301 (cmd
.flow_attr
.type
== IB_FLOW_ATTR_MC_DEFAULT
)))
3304 if (cmd
.flow_attr
.num_of_specs
> IB_FLOW_SPEC_SUPPORT_LAYERS
)
3307 if (cmd
.flow_attr
.size
> ucore
->inlen
||
3308 cmd
.flow_attr
.size
>
3309 (cmd
.flow_attr
.num_of_specs
* sizeof(struct ib_uverbs_flow_spec
)))
3312 if (cmd
.flow_attr
.reserved
[0] ||
3313 cmd
.flow_attr
.reserved
[1])
3316 if (cmd
.flow_attr
.num_of_specs
) {
3317 kern_flow_attr
= kmalloc(sizeof(*kern_flow_attr
) + cmd
.flow_attr
.size
,
3319 if (!kern_flow_attr
)
3322 memcpy(kern_flow_attr
, &cmd
.flow_attr
, sizeof(*kern_flow_attr
));
3323 err
= ib_copy_from_udata(kern_flow_attr
+ 1, ucore
,
3324 cmd
.flow_attr
.size
);
3328 kern_flow_attr
= &cmd
.flow_attr
;
3331 uobj
= uobj_alloc(uobj_get_type(flow
), file
->ucontext
);
3333 err
= PTR_ERR(uobj
);
3337 qp
= uobj_get_obj_read(qp
, cmd
.qp_handle
, file
->ucontext
);
3343 flow_attr
= kzalloc(sizeof(*flow_attr
) + cmd
.flow_attr
.num_of_specs
*
3344 sizeof(union ib_flow_spec
), GFP_KERNEL
);
3350 flow_attr
->type
= kern_flow_attr
->type
;
3351 flow_attr
->priority
= kern_flow_attr
->priority
;
3352 flow_attr
->num_of_specs
= kern_flow_attr
->num_of_specs
;
3353 flow_attr
->port
= kern_flow_attr
->port
;
3354 flow_attr
->flags
= kern_flow_attr
->flags
;
3355 flow_attr
->size
= sizeof(*flow_attr
);
3357 kern_spec
= kern_flow_attr
+ 1;
3358 ib_spec
= flow_attr
+ 1;
3359 for (i
= 0; i
< flow_attr
->num_of_specs
&&
3360 cmd
.flow_attr
.size
> offsetof(struct ib_uverbs_flow_spec
, reserved
) &&
3361 cmd
.flow_attr
.size
>=
3362 ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
; i
++) {
3363 err
= kern_spec_to_ib_spec(kern_spec
, ib_spec
);
3367 ((union ib_flow_spec
*) ib_spec
)->size
;
3368 cmd
.flow_attr
.size
-= ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
;
3369 kern_spec
+= ((struct ib_uverbs_flow_spec
*) kern_spec
)->size
;
3370 ib_spec
+= ((union ib_flow_spec
*) ib_spec
)->size
;
3372 if (cmd
.flow_attr
.size
|| (i
!= flow_attr
->num_of_specs
)) {
3373 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3374 i
, cmd
.flow_attr
.size
);
3378 flow_id
= ib_create_flow(qp
, flow_attr
, IB_FLOW_DOMAIN_USER
);
3379 if (IS_ERR(flow_id
)) {
3380 err
= PTR_ERR(flow_id
);
3383 flow_id
->uobject
= uobj
;
3384 uobj
->object
= flow_id
;
3386 memset(&resp
, 0, sizeof(resp
));
3387 resp
.flow_handle
= uobj
->id
;
3389 err
= ib_copy_to_udata(ucore
,
3390 &resp
, sizeof(resp
));
3394 uobj_put_obj_read(qp
);
3395 uobj_alloc_commit(uobj
);
3397 if (cmd
.flow_attr
.num_of_specs
)
3398 kfree(kern_flow_attr
);
3401 ib_destroy_flow(flow_id
);
3405 uobj_put_obj_read(qp
);
3407 uobj_alloc_abort(uobj
);
3409 if (cmd
.flow_attr
.num_of_specs
)
3410 kfree(kern_flow_attr
);
3414 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file
*file
,
3415 struct ib_device
*ib_dev
,
3416 struct ib_udata
*ucore
,
3417 struct ib_udata
*uhw
)
3419 struct ib_uverbs_destroy_flow cmd
;
3420 struct ib_uobject
*uobj
;
3423 if (ucore
->inlen
< sizeof(cmd
))
3426 ret
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3433 uobj
= uobj_get_write(uobj_get_type(flow
), cmd
.flow_handle
,
3436 return PTR_ERR(uobj
);
3438 ret
= uobj_remove_commit(uobj
);
3442 static int __uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3443 struct ib_device
*ib_dev
,
3444 struct ib_uverbs_create_xsrq
*cmd
,
3445 struct ib_udata
*udata
)
3447 struct ib_uverbs_create_srq_resp resp
;
3448 struct ib_usrq_object
*obj
;
3451 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
3452 struct ib_srq_init_attr attr
;
3455 obj
= (struct ib_usrq_object
*)uobj_alloc(uobj_get_type(srq
),
3458 return PTR_ERR(obj
);
3460 if (cmd
->srq_type
== IB_SRQT_TM
)
3461 attr
.ext
.tag_matching
.max_num_tags
= cmd
->max_num_tags
;
3463 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3464 xrcd_uobj
= uobj_get_read(uobj_get_type(xrcd
), cmd
->xrcd_handle
,
3466 if (IS_ERR(xrcd_uobj
)) {
3471 attr
.ext
.xrc
.xrcd
= (struct ib_xrcd
*)xrcd_uobj
->object
;
3472 if (!attr
.ext
.xrc
.xrcd
) {
3477 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
3478 atomic_inc(&obj
->uxrcd
->refcnt
);
3481 if (ib_srq_has_cq(cmd
->srq_type
)) {
3482 attr
.ext
.cq
= uobj_get_obj_read(cq
, cmd
->cq_handle
,
3490 pd
= uobj_get_obj_read(pd
, cmd
->pd_handle
, file
->ucontext
);
3496 attr
.event_handler
= ib_uverbs_srq_event_handler
;
3497 attr
.srq_context
= file
;
3498 attr
.srq_type
= cmd
->srq_type
;
3499 attr
.attr
.max_wr
= cmd
->max_wr
;
3500 attr
.attr
.max_sge
= cmd
->max_sge
;
3501 attr
.attr
.srq_limit
= cmd
->srq_limit
;
3503 obj
->uevent
.events_reported
= 0;
3504 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
3506 srq
= pd
->device
->create_srq(pd
, &attr
, udata
);
3512 srq
->device
= pd
->device
;
3514 srq
->srq_type
= cmd
->srq_type
;
3515 srq
->uobject
= &obj
->uevent
.uobject
;
3516 srq
->event_handler
= attr
.event_handler
;
3517 srq
->srq_context
= attr
.srq_context
;
3519 if (ib_srq_has_cq(cmd
->srq_type
)) {
3520 srq
->ext
.cq
= attr
.ext
.cq
;
3521 atomic_inc(&attr
.ext
.cq
->usecnt
);
3524 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3525 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
3526 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
3529 atomic_inc(&pd
->usecnt
);
3530 atomic_set(&srq
->usecnt
, 0);
3532 obj
->uevent
.uobject
.object
= srq
;
3533 obj
->uevent
.uobject
.user_handle
= cmd
->user_handle
;
3535 memset(&resp
, 0, sizeof resp
);
3536 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
3537 resp
.max_wr
= attr
.attr
.max_wr
;
3538 resp
.max_sge
= attr
.attr
.max_sge
;
3539 if (cmd
->srq_type
== IB_SRQT_XRC
)
3540 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
3542 if (copy_to_user((void __user
*) (unsigned long) cmd
->response
,
3543 &resp
, sizeof resp
)) {
3548 if (cmd
->srq_type
== IB_SRQT_XRC
)
3549 uobj_put_read(xrcd_uobj
);
3551 if (ib_srq_has_cq(cmd
->srq_type
))
3552 uobj_put_obj_read(attr
.ext
.cq
);
3554 uobj_put_obj_read(pd
);
3555 uobj_alloc_commit(&obj
->uevent
.uobject
);
3560 ib_destroy_srq(srq
);
3563 uobj_put_obj_read(pd
);
3566 if (ib_srq_has_cq(cmd
->srq_type
))
3567 uobj_put_obj_read(attr
.ext
.cq
);
3570 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3571 atomic_dec(&obj
->uxrcd
->refcnt
);
3572 uobj_put_read(xrcd_uobj
);
3576 uobj_alloc_abort(&obj
->uevent
.uobject
);
3580 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
3581 struct ib_device
*ib_dev
,
3582 const char __user
*buf
, int in_len
,
3585 struct ib_uverbs_create_srq cmd
;
3586 struct ib_uverbs_create_xsrq xcmd
;
3587 struct ib_uverbs_create_srq_resp resp
;
3588 struct ib_udata udata
;
3591 if (out_len
< sizeof resp
)
3594 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3597 memset(&xcmd
, 0, sizeof(xcmd
));
3598 xcmd
.response
= cmd
.response
;
3599 xcmd
.user_handle
= cmd
.user_handle
;
3600 xcmd
.srq_type
= IB_SRQT_BASIC
;
3601 xcmd
.pd_handle
= cmd
.pd_handle
;
3602 xcmd
.max_wr
= cmd
.max_wr
;
3603 xcmd
.max_sge
= cmd
.max_sge
;
3604 xcmd
.srq_limit
= cmd
.srq_limit
;
3606 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
3607 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
3608 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
3609 out_len
- sizeof(resp
));
3611 ret
= __uverbs_create_xsrq(file
, ib_dev
, &xcmd
, &udata
);
3618 ssize_t
ib_uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3619 struct ib_device
*ib_dev
,
3620 const char __user
*buf
, int in_len
, int out_len
)
3622 struct ib_uverbs_create_xsrq cmd
;
3623 struct ib_uverbs_create_srq_resp resp
;
3624 struct ib_udata udata
;
3627 if (out_len
< sizeof resp
)
3630 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3633 ib_uverbs_init_udata(&udata
, buf
+ sizeof(cmd
),
3634 u64_to_user_ptr(cmd
.response
) + sizeof(resp
),
3635 in_len
- sizeof(cmd
) - sizeof(struct ib_uverbs_cmd_hdr
),
3636 out_len
- sizeof(resp
));
3638 ret
= __uverbs_create_xsrq(file
, ib_dev
, &cmd
, &udata
);
3645 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
3646 struct ib_device
*ib_dev
,
3647 const char __user
*buf
, int in_len
,
3650 struct ib_uverbs_modify_srq cmd
;
3651 struct ib_udata udata
;
3653 struct ib_srq_attr attr
;
3656 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3659 ib_uverbs_init_udata(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
3662 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
3666 attr
.max_wr
= cmd
.max_wr
;
3667 attr
.srq_limit
= cmd
.srq_limit
;
3669 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
3671 uobj_put_obj_read(srq
);
3673 return ret
? ret
: in_len
;
3676 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
3677 struct ib_device
*ib_dev
,
3678 const char __user
*buf
,
3679 int in_len
, int out_len
)
3681 struct ib_uverbs_query_srq cmd
;
3682 struct ib_uverbs_query_srq_resp resp
;
3683 struct ib_srq_attr attr
;
3687 if (out_len
< sizeof resp
)
3690 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3693 srq
= uobj_get_obj_read(srq
, cmd
.srq_handle
, file
->ucontext
);
3697 ret
= ib_query_srq(srq
, &attr
);
3699 uobj_put_obj_read(srq
);
3704 memset(&resp
, 0, sizeof resp
);
3706 resp
.max_wr
= attr
.max_wr
;
3707 resp
.max_sge
= attr
.max_sge
;
3708 resp
.srq_limit
= attr
.srq_limit
;
3710 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof resp
))
3716 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
3717 struct ib_device
*ib_dev
,
3718 const char __user
*buf
, int in_len
,
3721 struct ib_uverbs_destroy_srq cmd
;
3722 struct ib_uverbs_destroy_srq_resp resp
;
3723 struct ib_uobject
*uobj
;
3724 struct ib_uevent_object
*obj
;
3727 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3730 uobj
= uobj_get_write(uobj_get_type(srq
), cmd
.srq_handle
,
3733 return PTR_ERR(uobj
);
3735 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
3737 * Make sure we don't free the memory in remove_commit as we still
3738 * needs the uobject memory to create the response.
3740 uverbs_uobject_get(uobj
);
3742 memset(&resp
, 0, sizeof(resp
));
3744 ret
= uobj_remove_commit(uobj
);
3746 uverbs_uobject_put(uobj
);
3749 resp
.events_reported
= obj
->events_reported
;
3750 uverbs_uobject_put(uobj
);
3751 if (copy_to_user(u64_to_user_ptr(cmd
.response
), &resp
, sizeof(resp
)))
3757 int ib_uverbs_ex_query_device(struct ib_uverbs_file
*file
,
3758 struct ib_device
*ib_dev
,
3759 struct ib_udata
*ucore
,
3760 struct ib_udata
*uhw
)
3762 struct ib_uverbs_ex_query_device_resp resp
= { {0} };
3763 struct ib_uverbs_ex_query_device cmd
;
3764 struct ib_device_attr attr
= {0};
3767 if (ucore
->inlen
< sizeof(cmd
))
3770 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3780 resp
.response_length
= offsetof(typeof(resp
), odp_caps
);
3782 if (ucore
->outlen
< resp
.response_length
)
3785 err
= ib_dev
->query_device(ib_dev
, &attr
, uhw
);
3789 copy_query_dev_fields(file
, ib_dev
, &resp
.base
, &attr
);
3791 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.odp_caps
))
3794 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3795 resp
.odp_caps
.general_caps
= attr
.odp_caps
.general_caps
;
3796 resp
.odp_caps
.per_transport_caps
.rc_odp_caps
=
3797 attr
.odp_caps
.per_transport_caps
.rc_odp_caps
;
3798 resp
.odp_caps
.per_transport_caps
.uc_odp_caps
=
3799 attr
.odp_caps
.per_transport_caps
.uc_odp_caps
;
3800 resp
.odp_caps
.per_transport_caps
.ud_odp_caps
=
3801 attr
.odp_caps
.per_transport_caps
.ud_odp_caps
;
3803 resp
.response_length
+= sizeof(resp
.odp_caps
);
3805 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.timestamp_mask
))
3808 resp
.timestamp_mask
= attr
.timestamp_mask
;
3809 resp
.response_length
+= sizeof(resp
.timestamp_mask
);
3811 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.hca_core_clock
))
3814 resp
.hca_core_clock
= attr
.hca_core_clock
;
3815 resp
.response_length
+= sizeof(resp
.hca_core_clock
);
3817 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.device_cap_flags_ex
))
3820 resp
.device_cap_flags_ex
= attr
.device_cap_flags
;
3821 resp
.response_length
+= sizeof(resp
.device_cap_flags_ex
);
3823 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.rss_caps
))
3826 resp
.rss_caps
.supported_qpts
= attr
.rss_caps
.supported_qpts
;
3827 resp
.rss_caps
.max_rwq_indirection_tables
=
3828 attr
.rss_caps
.max_rwq_indirection_tables
;
3829 resp
.rss_caps
.max_rwq_indirection_table_size
=
3830 attr
.rss_caps
.max_rwq_indirection_table_size
;
3832 resp
.response_length
+= sizeof(resp
.rss_caps
);
3834 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.max_wq_type_rq
))
3837 resp
.max_wq_type_rq
= attr
.max_wq_type_rq
;
3838 resp
.response_length
+= sizeof(resp
.max_wq_type_rq
);
3840 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.raw_packet_caps
))
3843 resp
.raw_packet_caps
= attr
.raw_packet_caps
;
3844 resp
.response_length
+= sizeof(resp
.raw_packet_caps
);
3846 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.tm_caps
))
3849 resp
.tm_caps
.max_rndv_hdr_size
= attr
.tm_caps
.max_rndv_hdr_size
;
3850 resp
.tm_caps
.max_num_tags
= attr
.tm_caps
.max_num_tags
;
3851 resp
.tm_caps
.max_ops
= attr
.tm_caps
.max_ops
;
3852 resp
.tm_caps
.max_sge
= attr
.tm_caps
.max_sge
;
3853 resp
.tm_caps
.flags
= attr
.tm_caps
.flags
;
3854 resp
.response_length
+= sizeof(resp
.tm_caps
);
3856 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.cq_moderation_caps
))
3859 resp
.cq_moderation_caps
.max_cq_moderation_count
=
3860 attr
.cq_caps
.max_cq_moderation_count
;
3861 resp
.cq_moderation_caps
.max_cq_moderation_period
=
3862 attr
.cq_caps
.max_cq_moderation_period
;
3863 resp
.response_length
+= sizeof(resp
.cq_moderation_caps
);
3865 err
= ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);
3869 int ib_uverbs_ex_modify_cq(struct ib_uverbs_file
*file
,
3870 struct ib_device
*ib_dev
,
3871 struct ib_udata
*ucore
,
3872 struct ib_udata
*uhw
)
3874 struct ib_uverbs_ex_modify_cq cmd
= {};
3876 size_t required_cmd_sz
;
3879 required_cmd_sz
= offsetof(typeof(cmd
), reserved
) +
3880 sizeof(cmd
.reserved
);
3881 if (ucore
->inlen
< required_cmd_sz
)
3885 if (ucore
->inlen
> sizeof(cmd
) &&
3886 !ib_is_udata_cleared(ucore
, sizeof(cmd
),
3887 ucore
->inlen
- sizeof(cmd
)))
3890 ret
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
3894 if (!cmd
.attr_mask
|| cmd
.reserved
)
3897 if (cmd
.attr_mask
> IB_CQ_MODERATE
)
3900 cq
= uobj_get_obj_read(cq
, cmd
.cq_handle
, file
->ucontext
);
3904 ret
= rdma_set_cq_moderation(cq
, cmd
.attr
.cq_count
, cmd
.attr
.cq_period
);
3906 uobj_put_obj_read(cq
);