2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
41 #include <asm/uaccess.h>
44 #include "core_priv.h"
46 struct uverbs_lock_class
{
47 struct lock_class_key key
;
51 static struct uverbs_lock_class pd_lock_class
= { .name
= "PD-uobj" };
52 static struct uverbs_lock_class mr_lock_class
= { .name
= "MR-uobj" };
53 static struct uverbs_lock_class mw_lock_class
= { .name
= "MW-uobj" };
54 static struct uverbs_lock_class cq_lock_class
= { .name
= "CQ-uobj" };
55 static struct uverbs_lock_class qp_lock_class
= { .name
= "QP-uobj" };
56 static struct uverbs_lock_class ah_lock_class
= { .name
= "AH-uobj" };
57 static struct uverbs_lock_class srq_lock_class
= { .name
= "SRQ-uobj" };
58 static struct uverbs_lock_class xrcd_lock_class
= { .name
= "XRCD-uobj" };
59 static struct uverbs_lock_class rule_lock_class
= { .name
= "RULE-uobj" };
62 * The ib_uobject locking scheme is as follows:
64 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
65 * needs to be held during all idr write operations. When an object is
66 * looked up, a reference must be taken on the object's kref before
67 * dropping this lock. For read operations, the rcu_read_lock()
68 * and rcu_write_lock() but similarly the kref reference is grabbed
69 * before the rcu_read_unlock().
71 * - Each object also has an rwsem. This rwsem must be held for
72 * reading while an operation that uses the object is performed.
73 * For example, while registering an MR, the associated PD's
74 * uobject.mutex must be held for reading. The rwsem must be held
75 * for writing while initializing or destroying an object.
77 * - In addition, each object has a "live" flag. If this flag is not
78 * set, then lookups of the object will fail even if it is found in
79 * the idr. This handles a reader that blocks and does not acquire
80 * the rwsem until after the object is destroyed. The destroy
81 * operation will set the live flag to 0 and then drop the rwsem;
82 * this will allow the reader to acquire the rwsem, see that the
83 * live flag is 0, and then drop the rwsem and its reference to
84 * object. The underlying storage will not be freed until the last
85 * reference to the object is dropped.
88 static void init_uobj(struct ib_uobject
*uobj
, u64 user_handle
,
89 struct ib_ucontext
*context
, struct uverbs_lock_class
*c
)
91 uobj
->user_handle
= user_handle
;
92 uobj
->context
= context
;
93 kref_init(&uobj
->ref
);
94 init_rwsem(&uobj
->mutex
);
95 lockdep_set_class_and_name(&uobj
->mutex
, &c
->key
, c
->name
);
99 static void release_uobj(struct kref
*kref
)
101 kfree_rcu(container_of(kref
, struct ib_uobject
, ref
), rcu
);
104 static void put_uobj(struct ib_uobject
*uobj
)
106 kref_put(&uobj
->ref
, release_uobj
);
109 static void put_uobj_read(struct ib_uobject
*uobj
)
111 up_read(&uobj
->mutex
);
115 static void put_uobj_write(struct ib_uobject
*uobj
)
117 up_write(&uobj
->mutex
);
121 static int idr_add_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
125 idr_preload(GFP_KERNEL
);
126 spin_lock(&ib_uverbs_idr_lock
);
128 ret
= idr_alloc(idr
, uobj
, 0, 0, GFP_NOWAIT
);
132 spin_unlock(&ib_uverbs_idr_lock
);
135 return ret
< 0 ? ret
: 0;
138 void idr_remove_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
140 spin_lock(&ib_uverbs_idr_lock
);
141 idr_remove(idr
, uobj
->id
);
142 spin_unlock(&ib_uverbs_idr_lock
);
145 static struct ib_uobject
*__idr_get_uobj(struct idr
*idr
, int id
,
146 struct ib_ucontext
*context
)
148 struct ib_uobject
*uobj
;
151 uobj
= idr_find(idr
, id
);
153 if (uobj
->context
== context
)
154 kref_get(&uobj
->ref
);
163 static struct ib_uobject
*idr_read_uobj(struct idr
*idr
, int id
,
164 struct ib_ucontext
*context
, int nested
)
166 struct ib_uobject
*uobj
;
168 uobj
= __idr_get_uobj(idr
, id
, context
);
173 down_read_nested(&uobj
->mutex
, SINGLE_DEPTH_NESTING
);
175 down_read(&uobj
->mutex
);
184 static struct ib_uobject
*idr_write_uobj(struct idr
*idr
, int id
,
185 struct ib_ucontext
*context
)
187 struct ib_uobject
*uobj
;
189 uobj
= __idr_get_uobj(idr
, id
, context
);
193 down_write(&uobj
->mutex
);
195 put_uobj_write(uobj
);
202 static void *idr_read_obj(struct idr
*idr
, int id
, struct ib_ucontext
*context
,
205 struct ib_uobject
*uobj
;
207 uobj
= idr_read_uobj(idr
, id
, context
, nested
);
208 return uobj
? uobj
->object
: NULL
;
211 static struct ib_pd
*idr_read_pd(int pd_handle
, struct ib_ucontext
*context
)
213 return idr_read_obj(&ib_uverbs_pd_idr
, pd_handle
, context
, 0);
216 static void put_pd_read(struct ib_pd
*pd
)
218 put_uobj_read(pd
->uobject
);
221 static struct ib_cq
*idr_read_cq(int cq_handle
, struct ib_ucontext
*context
, int nested
)
223 return idr_read_obj(&ib_uverbs_cq_idr
, cq_handle
, context
, nested
);
226 static void put_cq_read(struct ib_cq
*cq
)
228 put_uobj_read(cq
->uobject
);
231 static struct ib_ah
*idr_read_ah(int ah_handle
, struct ib_ucontext
*context
)
233 return idr_read_obj(&ib_uverbs_ah_idr
, ah_handle
, context
, 0);
236 static void put_ah_read(struct ib_ah
*ah
)
238 put_uobj_read(ah
->uobject
);
241 static struct ib_qp
*idr_read_qp(int qp_handle
, struct ib_ucontext
*context
)
243 return idr_read_obj(&ib_uverbs_qp_idr
, qp_handle
, context
, 0);
246 static struct ib_qp
*idr_write_qp(int qp_handle
, struct ib_ucontext
*context
)
248 struct ib_uobject
*uobj
;
250 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, qp_handle
, context
);
251 return uobj
? uobj
->object
: NULL
;
254 static void put_qp_read(struct ib_qp
*qp
)
256 put_uobj_read(qp
->uobject
);
259 static void put_qp_write(struct ib_qp
*qp
)
261 put_uobj_write(qp
->uobject
);
264 static struct ib_srq
*idr_read_srq(int srq_handle
, struct ib_ucontext
*context
)
266 return idr_read_obj(&ib_uverbs_srq_idr
, srq_handle
, context
, 0);
269 static void put_srq_read(struct ib_srq
*srq
)
271 put_uobj_read(srq
->uobject
);
274 static struct ib_xrcd
*idr_read_xrcd(int xrcd_handle
, struct ib_ucontext
*context
,
275 struct ib_uobject
**uobj
)
277 *uobj
= idr_read_uobj(&ib_uverbs_xrcd_idr
, xrcd_handle
, context
, 0);
278 return *uobj
? (*uobj
)->object
: NULL
;
281 static void put_xrcd_read(struct ib_uobject
*uobj
)
286 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
287 struct ib_device
*ib_dev
,
288 const char __user
*buf
,
289 int in_len
, int out_len
)
291 struct ib_uverbs_get_context cmd
;
292 struct ib_uverbs_get_context_resp resp
;
293 struct ib_udata udata
;
294 struct ib_ucontext
*ucontext
;
298 if (out_len
< sizeof resp
)
301 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
304 mutex_lock(&file
->mutex
);
306 if (file
->ucontext
) {
311 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
312 (unsigned long) cmd
.response
+ sizeof resp
,
313 in_len
- sizeof cmd
, out_len
- sizeof resp
);
315 ucontext
= ib_dev
->alloc_ucontext(ib_dev
, &udata
);
316 if (IS_ERR(ucontext
)) {
317 ret
= PTR_ERR(ucontext
);
321 ucontext
->device
= ib_dev
;
322 INIT_LIST_HEAD(&ucontext
->pd_list
);
323 INIT_LIST_HEAD(&ucontext
->mr_list
);
324 INIT_LIST_HEAD(&ucontext
->mw_list
);
325 INIT_LIST_HEAD(&ucontext
->cq_list
);
326 INIT_LIST_HEAD(&ucontext
->qp_list
);
327 INIT_LIST_HEAD(&ucontext
->srq_list
);
328 INIT_LIST_HEAD(&ucontext
->ah_list
);
329 INIT_LIST_HEAD(&ucontext
->xrcd_list
);
330 INIT_LIST_HEAD(&ucontext
->rule_list
);
332 ucontext
->tgid
= get_task_pid(current
->group_leader
, PIDTYPE_PID
);
334 ucontext
->closing
= 0;
336 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
337 ucontext
->umem_tree
= RB_ROOT
;
338 init_rwsem(&ucontext
->umem_rwsem
);
339 ucontext
->odp_mrs_count
= 0;
340 INIT_LIST_HEAD(&ucontext
->no_private_counters
);
342 if (!(ib_dev
->attrs
.device_cap_flags
& IB_DEVICE_ON_DEMAND_PAGING
))
343 ucontext
->invalidate_range
= NULL
;
347 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
349 ret
= get_unused_fd_flags(O_CLOEXEC
);
354 filp
= ib_uverbs_alloc_event_file(file
, ib_dev
, 1);
360 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
361 &resp
, sizeof resp
)) {
366 file
->ucontext
= ucontext
;
368 fd_install(resp
.async_fd
, filp
);
370 mutex_unlock(&file
->mutex
);
375 ib_uverbs_free_async_event_file(file
);
379 put_unused_fd(resp
.async_fd
);
382 put_pid(ucontext
->tgid
);
383 ib_dev
->dealloc_ucontext(ucontext
);
386 mutex_unlock(&file
->mutex
);
390 static void copy_query_dev_fields(struct ib_uverbs_file
*file
,
391 struct ib_device
*ib_dev
,
392 struct ib_uverbs_query_device_resp
*resp
,
393 struct ib_device_attr
*attr
)
395 resp
->fw_ver
= attr
->fw_ver
;
396 resp
->node_guid
= ib_dev
->node_guid
;
397 resp
->sys_image_guid
= attr
->sys_image_guid
;
398 resp
->max_mr_size
= attr
->max_mr_size
;
399 resp
->page_size_cap
= attr
->page_size_cap
;
400 resp
->vendor_id
= attr
->vendor_id
;
401 resp
->vendor_part_id
= attr
->vendor_part_id
;
402 resp
->hw_ver
= attr
->hw_ver
;
403 resp
->max_qp
= attr
->max_qp
;
404 resp
->max_qp_wr
= attr
->max_qp_wr
;
405 resp
->device_cap_flags
= attr
->device_cap_flags
;
406 resp
->max_sge
= attr
->max_sge
;
407 resp
->max_sge_rd
= attr
->max_sge_rd
;
408 resp
->max_cq
= attr
->max_cq
;
409 resp
->max_cqe
= attr
->max_cqe
;
410 resp
->max_mr
= attr
->max_mr
;
411 resp
->max_pd
= attr
->max_pd
;
412 resp
->max_qp_rd_atom
= attr
->max_qp_rd_atom
;
413 resp
->max_ee_rd_atom
= attr
->max_ee_rd_atom
;
414 resp
->max_res_rd_atom
= attr
->max_res_rd_atom
;
415 resp
->max_qp_init_rd_atom
= attr
->max_qp_init_rd_atom
;
416 resp
->max_ee_init_rd_atom
= attr
->max_ee_init_rd_atom
;
417 resp
->atomic_cap
= attr
->atomic_cap
;
418 resp
->max_ee
= attr
->max_ee
;
419 resp
->max_rdd
= attr
->max_rdd
;
420 resp
->max_mw
= attr
->max_mw
;
421 resp
->max_raw_ipv6_qp
= attr
->max_raw_ipv6_qp
;
422 resp
->max_raw_ethy_qp
= attr
->max_raw_ethy_qp
;
423 resp
->max_mcast_grp
= attr
->max_mcast_grp
;
424 resp
->max_mcast_qp_attach
= attr
->max_mcast_qp_attach
;
425 resp
->max_total_mcast_qp_attach
= attr
->max_total_mcast_qp_attach
;
426 resp
->max_ah
= attr
->max_ah
;
427 resp
->max_fmr
= attr
->max_fmr
;
428 resp
->max_map_per_fmr
= attr
->max_map_per_fmr
;
429 resp
->max_srq
= attr
->max_srq
;
430 resp
->max_srq_wr
= attr
->max_srq_wr
;
431 resp
->max_srq_sge
= attr
->max_srq_sge
;
432 resp
->max_pkeys
= attr
->max_pkeys
;
433 resp
->local_ca_ack_delay
= attr
->local_ca_ack_delay
;
434 resp
->phys_port_cnt
= ib_dev
->phys_port_cnt
;
437 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
438 struct ib_device
*ib_dev
,
439 const char __user
*buf
,
440 int in_len
, int out_len
)
442 struct ib_uverbs_query_device cmd
;
443 struct ib_uverbs_query_device_resp resp
;
445 if (out_len
< sizeof resp
)
448 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
451 memset(&resp
, 0, sizeof resp
);
452 copy_query_dev_fields(file
, ib_dev
, &resp
, &ib_dev
->attrs
);
454 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
461 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
462 struct ib_device
*ib_dev
,
463 const char __user
*buf
,
464 int in_len
, int out_len
)
466 struct ib_uverbs_query_port cmd
;
467 struct ib_uverbs_query_port_resp resp
;
468 struct ib_port_attr attr
;
471 if (out_len
< sizeof resp
)
474 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
477 ret
= ib_query_port(ib_dev
, cmd
.port_num
, &attr
);
481 memset(&resp
, 0, sizeof resp
);
483 resp
.state
= attr
.state
;
484 resp
.max_mtu
= attr
.max_mtu
;
485 resp
.active_mtu
= attr
.active_mtu
;
486 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
487 resp
.port_cap_flags
= attr
.port_cap_flags
;
488 resp
.max_msg_sz
= attr
.max_msg_sz
;
489 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
490 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
491 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
493 resp
.sm_lid
= attr
.sm_lid
;
495 resp
.max_vl_num
= attr
.max_vl_num
;
496 resp
.sm_sl
= attr
.sm_sl
;
497 resp
.subnet_timeout
= attr
.subnet_timeout
;
498 resp
.init_type_reply
= attr
.init_type_reply
;
499 resp
.active_width
= attr
.active_width
;
500 resp
.active_speed
= attr
.active_speed
;
501 resp
.phys_state
= attr
.phys_state
;
502 resp
.link_layer
= rdma_port_get_link_layer(ib_dev
,
505 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
512 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
513 struct ib_device
*ib_dev
,
514 const char __user
*buf
,
515 int in_len
, int out_len
)
517 struct ib_uverbs_alloc_pd cmd
;
518 struct ib_uverbs_alloc_pd_resp resp
;
519 struct ib_udata udata
;
520 struct ib_uobject
*uobj
;
524 if (out_len
< sizeof resp
)
527 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
530 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
531 (unsigned long) cmd
.response
+ sizeof resp
,
532 in_len
- sizeof cmd
, out_len
- sizeof resp
);
534 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
538 init_uobj(uobj
, 0, file
->ucontext
, &pd_lock_class
);
539 down_write(&uobj
->mutex
);
541 pd
= ib_dev
->alloc_pd(ib_dev
, file
->ucontext
, &udata
);
550 atomic_set(&pd
->usecnt
, 0);
553 ret
= idr_add_uobj(&ib_uverbs_pd_idr
, uobj
);
557 memset(&resp
, 0, sizeof resp
);
558 resp
.pd_handle
= uobj
->id
;
560 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
561 &resp
, sizeof resp
)) {
566 mutex_lock(&file
->mutex
);
567 list_add_tail(&uobj
->list
, &file
->ucontext
->pd_list
);
568 mutex_unlock(&file
->mutex
);
572 up_write(&uobj
->mutex
);
577 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
583 put_uobj_write(uobj
);
587 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
588 struct ib_device
*ib_dev
,
589 const char __user
*buf
,
590 int in_len
, int out_len
)
592 struct ib_uverbs_dealloc_pd cmd
;
593 struct ib_uobject
*uobj
;
597 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
600 uobj
= idr_write_uobj(&ib_uverbs_pd_idr
, cmd
.pd_handle
, file
->ucontext
);
605 if (atomic_read(&pd
->usecnt
)) {
610 ret
= pd
->device
->dealloc_pd(uobj
->object
);
611 WARN_ONCE(ret
, "Infiniband HW driver failed dealloc_pd");
616 put_uobj_write(uobj
);
618 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
620 mutex_lock(&file
->mutex
);
621 list_del(&uobj
->list
);
622 mutex_unlock(&file
->mutex
);
629 put_uobj_write(uobj
);
633 struct xrcd_table_entry
{
635 struct ib_xrcd
*xrcd
;
639 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
641 struct ib_xrcd
*xrcd
)
643 struct xrcd_table_entry
*entry
, *scan
;
644 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
645 struct rb_node
*parent
= NULL
;
647 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
652 entry
->inode
= inode
;
656 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
658 if (inode
< scan
->inode
) {
660 } else if (inode
> scan
->inode
) {
668 rb_link_node(&entry
->node
, parent
, p
);
669 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
674 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
677 struct xrcd_table_entry
*entry
;
678 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
681 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
683 if (inode
< entry
->inode
)
685 else if (inode
> entry
->inode
)
694 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
696 struct xrcd_table_entry
*entry
;
698 entry
= xrcd_table_search(dev
, inode
);
705 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
708 struct xrcd_table_entry
*entry
;
710 entry
= xrcd_table_search(dev
, inode
);
713 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
718 ssize_t
ib_uverbs_open_xrcd(struct ib_uverbs_file
*file
,
719 struct ib_device
*ib_dev
,
720 const char __user
*buf
, int in_len
,
723 struct ib_uverbs_open_xrcd cmd
;
724 struct ib_uverbs_open_xrcd_resp resp
;
725 struct ib_udata udata
;
726 struct ib_uxrcd_object
*obj
;
727 struct ib_xrcd
*xrcd
= NULL
;
728 struct fd f
= {NULL
, 0};
729 struct inode
*inode
= NULL
;
733 if (out_len
< sizeof resp
)
736 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
739 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
740 (unsigned long) cmd
.response
+ sizeof resp
,
741 in_len
- sizeof cmd
, out_len
- sizeof resp
);
743 mutex_lock(&file
->device
->xrcd_tree_mutex
);
746 /* search for file descriptor */
750 goto err_tree_mutex_unlock
;
753 inode
= file_inode(f
.file
);
754 xrcd
= find_xrcd(file
->device
, inode
);
755 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
756 /* no file descriptor. Need CREATE flag */
758 goto err_tree_mutex_unlock
;
761 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
763 goto err_tree_mutex_unlock
;
767 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
770 goto err_tree_mutex_unlock
;
773 init_uobj(&obj
->uobject
, 0, file
->ucontext
, &xrcd_lock_class
);
775 down_write(&obj
->uobject
.mutex
);
778 xrcd
= ib_dev
->alloc_xrcd(ib_dev
, file
->ucontext
, &udata
);
785 xrcd
->device
= ib_dev
;
786 atomic_set(&xrcd
->usecnt
, 0);
787 mutex_init(&xrcd
->tgt_qp_mutex
);
788 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
792 atomic_set(&obj
->refcnt
, 0);
793 obj
->uobject
.object
= xrcd
;
794 ret
= idr_add_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
798 memset(&resp
, 0, sizeof resp
);
799 resp
.xrcd_handle
= obj
->uobject
.id
;
803 /* create new inode/xrcd table entry */
804 ret
= xrcd_table_insert(file
->device
, inode
, xrcd
);
806 goto err_insert_xrcd
;
808 atomic_inc(&xrcd
->usecnt
);
811 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
812 &resp
, sizeof resp
)) {
820 mutex_lock(&file
->mutex
);
821 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->xrcd_list
);
822 mutex_unlock(&file
->mutex
);
824 obj
->uobject
.live
= 1;
825 up_write(&obj
->uobject
.mutex
);
827 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
833 xrcd_table_delete(file
->device
, inode
);
834 atomic_dec(&xrcd
->usecnt
);
838 idr_remove_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
841 ib_dealloc_xrcd(xrcd
);
844 put_uobj_write(&obj
->uobject
);
846 err_tree_mutex_unlock
:
850 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
855 ssize_t
ib_uverbs_close_xrcd(struct ib_uverbs_file
*file
,
856 struct ib_device
*ib_dev
,
857 const char __user
*buf
, int in_len
,
860 struct ib_uverbs_close_xrcd cmd
;
861 struct ib_uobject
*uobj
;
862 struct ib_xrcd
*xrcd
= NULL
;
863 struct inode
*inode
= NULL
;
864 struct ib_uxrcd_object
*obj
;
868 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
871 mutex_lock(&file
->device
->xrcd_tree_mutex
);
872 uobj
= idr_write_uobj(&ib_uverbs_xrcd_idr
, cmd
.xrcd_handle
, file
->ucontext
);
880 obj
= container_of(uobj
, struct ib_uxrcd_object
, uobject
);
881 if (atomic_read(&obj
->refcnt
)) {
882 put_uobj_write(uobj
);
887 if (!inode
|| atomic_dec_and_test(&xrcd
->usecnt
)) {
888 ret
= ib_dealloc_xrcd(uobj
->object
);
895 atomic_inc(&xrcd
->usecnt
);
897 put_uobj_write(uobj
);
903 xrcd_table_delete(file
->device
, inode
);
905 idr_remove_uobj(&ib_uverbs_xrcd_idr
, uobj
);
906 mutex_lock(&file
->mutex
);
907 list_del(&uobj
->list
);
908 mutex_unlock(&file
->mutex
);
914 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
918 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device
*dev
,
919 struct ib_xrcd
*xrcd
)
924 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
927 ib_dealloc_xrcd(xrcd
);
930 xrcd_table_delete(dev
, inode
);
933 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
934 struct ib_device
*ib_dev
,
935 const char __user
*buf
, int in_len
,
938 struct ib_uverbs_reg_mr cmd
;
939 struct ib_uverbs_reg_mr_resp resp
;
940 struct ib_udata udata
;
941 struct ib_uobject
*uobj
;
946 if (out_len
< sizeof resp
)
949 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
952 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
953 (unsigned long) cmd
.response
+ sizeof resp
,
954 in_len
- sizeof cmd
, out_len
- sizeof resp
);
956 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
959 ret
= ib_check_mr_access(cmd
.access_flags
);
963 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
967 init_uobj(uobj
, 0, file
->ucontext
, &mr_lock_class
);
968 down_write(&uobj
->mutex
);
970 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
976 if (cmd
.access_flags
& IB_ACCESS_ON_DEMAND
) {
977 if (!(pd
->device
->attrs
.device_cap_flags
&
978 IB_DEVICE_ON_DEMAND_PAGING
)) {
979 pr_debug("ODP support not available\n");
985 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
986 cmd
.access_flags
, &udata
);
992 mr
->device
= pd
->device
;
995 atomic_inc(&pd
->usecnt
);
998 ret
= idr_add_uobj(&ib_uverbs_mr_idr
, uobj
);
1002 memset(&resp
, 0, sizeof resp
);
1003 resp
.lkey
= mr
->lkey
;
1004 resp
.rkey
= mr
->rkey
;
1005 resp
.mr_handle
= uobj
->id
;
1007 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1008 &resp
, sizeof resp
)) {
1015 mutex_lock(&file
->mutex
);
1016 list_add_tail(&uobj
->list
, &file
->ucontext
->mr_list
);
1017 mutex_unlock(&file
->mutex
);
1021 up_write(&uobj
->mutex
);
1026 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
1035 put_uobj_write(uobj
);
1039 ssize_t
ib_uverbs_rereg_mr(struct ib_uverbs_file
*file
,
1040 struct ib_device
*ib_dev
,
1041 const char __user
*buf
, int in_len
,
1044 struct ib_uverbs_rereg_mr cmd
;
1045 struct ib_uverbs_rereg_mr_resp resp
;
1046 struct ib_udata udata
;
1047 struct ib_pd
*pd
= NULL
;
1049 struct ib_pd
*old_pd
;
1051 struct ib_uobject
*uobj
;
1053 if (out_len
< sizeof(resp
))
1056 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1059 INIT_UDATA(&udata
, buf
+ sizeof(cmd
),
1060 (unsigned long) cmd
.response
+ sizeof(resp
),
1061 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
1063 if (cmd
.flags
& ~IB_MR_REREG_SUPPORTED
|| !cmd
.flags
)
1066 if ((cmd
.flags
& IB_MR_REREG_TRANS
) &&
1067 (!cmd
.start
|| !cmd
.hca_va
|| 0 >= cmd
.length
||
1068 (cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
)))
1071 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
,
1079 if (cmd
.flags
& IB_MR_REREG_ACCESS
) {
1080 ret
= ib_check_mr_access(cmd
.access_flags
);
1085 if (cmd
.flags
& IB_MR_REREG_PD
) {
1086 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1094 ret
= mr
->device
->rereg_user_mr(mr
, cmd
.flags
, cmd
.start
,
1095 cmd
.length
, cmd
.hca_va
,
1096 cmd
.access_flags
, pd
, &udata
);
1098 if (cmd
.flags
& IB_MR_REREG_PD
) {
1099 atomic_inc(&pd
->usecnt
);
1101 atomic_dec(&old_pd
->usecnt
);
1107 memset(&resp
, 0, sizeof(resp
));
1108 resp
.lkey
= mr
->lkey
;
1109 resp
.rkey
= mr
->rkey
;
1111 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
1112 &resp
, sizeof(resp
)))
1118 if (cmd
.flags
& IB_MR_REREG_PD
)
1123 put_uobj_write(mr
->uobject
);
1128 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
1129 struct ib_device
*ib_dev
,
1130 const char __user
*buf
, int in_len
,
1133 struct ib_uverbs_dereg_mr cmd
;
1135 struct ib_uobject
*uobj
;
1138 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1141 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
, file
->ucontext
);
1147 ret
= ib_dereg_mr(mr
);
1151 put_uobj_write(uobj
);
1156 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
1158 mutex_lock(&file
->mutex
);
1159 list_del(&uobj
->list
);
1160 mutex_unlock(&file
->mutex
);
1167 ssize_t
ib_uverbs_alloc_mw(struct ib_uverbs_file
*file
,
1168 struct ib_device
*ib_dev
,
1169 const char __user
*buf
, int in_len
,
1172 struct ib_uverbs_alloc_mw cmd
;
1173 struct ib_uverbs_alloc_mw_resp resp
;
1174 struct ib_uobject
*uobj
;
1179 if (out_len
< sizeof(resp
))
1182 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1185 uobj
= kmalloc(sizeof(*uobj
), GFP_KERNEL
);
1189 init_uobj(uobj
, 0, file
->ucontext
, &mw_lock_class
);
1190 down_write(&uobj
->mutex
);
1192 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1198 mw
= pd
->device
->alloc_mw(pd
, cmd
.mw_type
);
1204 mw
->device
= pd
->device
;
1207 atomic_inc(&pd
->usecnt
);
1210 ret
= idr_add_uobj(&ib_uverbs_mw_idr
, uobj
);
1214 memset(&resp
, 0, sizeof(resp
));
1215 resp
.rkey
= mw
->rkey
;
1216 resp
.mw_handle
= uobj
->id
;
1218 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
1219 &resp
, sizeof(resp
))) {
1226 mutex_lock(&file
->mutex
);
1227 list_add_tail(&uobj
->list
, &file
->ucontext
->mw_list
);
1228 mutex_unlock(&file
->mutex
);
1232 up_write(&uobj
->mutex
);
1237 idr_remove_uobj(&ib_uverbs_mw_idr
, uobj
);
1240 uverbs_dealloc_mw(mw
);
1246 put_uobj_write(uobj
);
1250 ssize_t
ib_uverbs_dealloc_mw(struct ib_uverbs_file
*file
,
1251 struct ib_device
*ib_dev
,
1252 const char __user
*buf
, int in_len
,
1255 struct ib_uverbs_dealloc_mw cmd
;
1257 struct ib_uobject
*uobj
;
1260 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1263 uobj
= idr_write_uobj(&ib_uverbs_mw_idr
, cmd
.mw_handle
, file
->ucontext
);
1269 ret
= uverbs_dealloc_mw(mw
);
1273 put_uobj_write(uobj
);
1278 idr_remove_uobj(&ib_uverbs_mw_idr
, uobj
);
1280 mutex_lock(&file
->mutex
);
1281 list_del(&uobj
->list
);
1282 mutex_unlock(&file
->mutex
);
1289 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
1290 struct ib_device
*ib_dev
,
1291 const char __user
*buf
, int in_len
,
1294 struct ib_uverbs_create_comp_channel cmd
;
1295 struct ib_uverbs_create_comp_channel_resp resp
;
1299 if (out_len
< sizeof resp
)
1302 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1305 ret
= get_unused_fd_flags(O_CLOEXEC
);
1310 filp
= ib_uverbs_alloc_event_file(file
, ib_dev
, 0);
1312 put_unused_fd(resp
.fd
);
1313 return PTR_ERR(filp
);
1316 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1317 &resp
, sizeof resp
)) {
1318 put_unused_fd(resp
.fd
);
1323 fd_install(resp
.fd
, filp
);
1327 static struct ib_ucq_object
*create_cq(struct ib_uverbs_file
*file
,
1328 struct ib_device
*ib_dev
,
1329 struct ib_udata
*ucore
,
1330 struct ib_udata
*uhw
,
1331 struct ib_uverbs_ex_create_cq
*cmd
,
1333 int (*cb
)(struct ib_uverbs_file
*file
,
1334 struct ib_ucq_object
*obj
,
1335 struct ib_uverbs_ex_create_cq_resp
*resp
,
1336 struct ib_udata
*udata
,
1340 struct ib_ucq_object
*obj
;
1341 struct ib_uverbs_event_file
*ev_file
= NULL
;
1344 struct ib_uverbs_ex_create_cq_resp resp
;
1345 struct ib_cq_init_attr attr
= {};
1347 if (cmd
->comp_vector
>= file
->device
->num_comp_vectors
)
1348 return ERR_PTR(-EINVAL
);
1350 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1352 return ERR_PTR(-ENOMEM
);
1354 init_uobj(&obj
->uobject
, cmd
->user_handle
, file
->ucontext
, &cq_lock_class
);
1355 down_write(&obj
->uobject
.mutex
);
1357 if (cmd
->comp_channel
>= 0) {
1358 ev_file
= ib_uverbs_lookup_comp_file(cmd
->comp_channel
);
1365 obj
->uverbs_file
= file
;
1366 obj
->comp_events_reported
= 0;
1367 obj
->async_events_reported
= 0;
1368 INIT_LIST_HEAD(&obj
->comp_list
);
1369 INIT_LIST_HEAD(&obj
->async_list
);
1371 attr
.cqe
= cmd
->cqe
;
1372 attr
.comp_vector
= cmd
->comp_vector
;
1374 if (cmd_sz
> offsetof(typeof(*cmd
), flags
) + sizeof(cmd
->flags
))
1375 attr
.flags
= cmd
->flags
;
1377 cq
= ib_dev
->create_cq(ib_dev
, &attr
,
1378 file
->ucontext
, uhw
);
1384 cq
->device
= ib_dev
;
1385 cq
->uobject
= &obj
->uobject
;
1386 cq
->comp_handler
= ib_uverbs_comp_handler
;
1387 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1388 cq
->cq_context
= ev_file
;
1389 atomic_set(&cq
->usecnt
, 0);
1391 obj
->uobject
.object
= cq
;
1392 ret
= idr_add_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1396 memset(&resp
, 0, sizeof resp
);
1397 resp
.base
.cq_handle
= obj
->uobject
.id
;
1398 resp
.base
.cqe
= cq
->cqe
;
1400 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1401 sizeof(resp
.response_length
);
1403 ret
= cb(file
, obj
, &resp
, ucore
, context
);
1407 mutex_lock(&file
->mutex
);
1408 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->cq_list
);
1409 mutex_unlock(&file
->mutex
);
1411 obj
->uobject
.live
= 1;
1413 up_write(&obj
->uobject
.mutex
);
1418 idr_remove_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1425 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1428 put_uobj_write(&obj
->uobject
);
1430 return ERR_PTR(ret
);
1433 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file
*file
,
1434 struct ib_ucq_object
*obj
,
1435 struct ib_uverbs_ex_create_cq_resp
*resp
,
1436 struct ib_udata
*ucore
, void *context
)
1438 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1444 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
1445 struct ib_device
*ib_dev
,
1446 const char __user
*buf
, int in_len
,
1449 struct ib_uverbs_create_cq cmd
;
1450 struct ib_uverbs_ex_create_cq cmd_ex
;
1451 struct ib_uverbs_create_cq_resp resp
;
1452 struct ib_udata ucore
;
1453 struct ib_udata uhw
;
1454 struct ib_ucq_object
*obj
;
1456 if (out_len
< sizeof(resp
))
1459 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1462 INIT_UDATA(&ucore
, buf
, (unsigned long)cmd
.response
, sizeof(cmd
), sizeof(resp
));
1464 INIT_UDATA(&uhw
, buf
+ sizeof(cmd
),
1465 (unsigned long)cmd
.response
+ sizeof(resp
),
1466 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
1468 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1469 cmd_ex
.user_handle
= cmd
.user_handle
;
1470 cmd_ex
.cqe
= cmd
.cqe
;
1471 cmd_ex
.comp_vector
= cmd
.comp_vector
;
1472 cmd_ex
.comp_channel
= cmd
.comp_channel
;
1474 obj
= create_cq(file
, ib_dev
, &ucore
, &uhw
, &cmd_ex
,
1475 offsetof(typeof(cmd_ex
), comp_channel
) +
1476 sizeof(cmd
.comp_channel
), ib_uverbs_create_cq_cb
,
1480 return PTR_ERR(obj
);
1485 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file
*file
,
1486 struct ib_ucq_object
*obj
,
1487 struct ib_uverbs_ex_create_cq_resp
*resp
,
1488 struct ib_udata
*ucore
, void *context
)
1490 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1496 int ib_uverbs_ex_create_cq(struct ib_uverbs_file
*file
,
1497 struct ib_device
*ib_dev
,
1498 struct ib_udata
*ucore
,
1499 struct ib_udata
*uhw
)
1501 struct ib_uverbs_ex_create_cq_resp resp
;
1502 struct ib_uverbs_ex_create_cq cmd
;
1503 struct ib_ucq_object
*obj
;
1506 if (ucore
->inlen
< sizeof(cmd
))
1509 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
1519 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1520 sizeof(resp
.response_length
)))
1523 obj
= create_cq(file
, ib_dev
, ucore
, uhw
, &cmd
,
1524 min(ucore
->inlen
, sizeof(cmd
)),
1525 ib_uverbs_ex_create_cq_cb
, NULL
);
1528 return PTR_ERR(obj
);
1533 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
1534 struct ib_device
*ib_dev
,
1535 const char __user
*buf
, int in_len
,
1538 struct ib_uverbs_resize_cq cmd
;
1539 struct ib_uverbs_resize_cq_resp resp
;
1540 struct ib_udata udata
;
1544 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1547 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1548 (unsigned long) cmd
.response
+ sizeof resp
,
1549 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1551 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1555 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
1561 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1562 &resp
, sizeof resp
.cqe
))
1568 return ret
? ret
: in_len
;
1571 static int copy_wc_to_user(void __user
*dest
, struct ib_wc
*wc
)
1573 struct ib_uverbs_wc tmp
;
1575 tmp
.wr_id
= wc
->wr_id
;
1576 tmp
.status
= wc
->status
;
1577 tmp
.opcode
= wc
->opcode
;
1578 tmp
.vendor_err
= wc
->vendor_err
;
1579 tmp
.byte_len
= wc
->byte_len
;
1580 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
1581 tmp
.qp_num
= wc
->qp
->qp_num
;
1582 tmp
.src_qp
= wc
->src_qp
;
1583 tmp
.wc_flags
= wc
->wc_flags
;
1584 tmp
.pkey_index
= wc
->pkey_index
;
1585 tmp
.slid
= wc
->slid
;
1587 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1588 tmp
.port_num
= wc
->port_num
;
1591 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1597 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
1598 struct ib_device
*ib_dev
,
1599 const char __user
*buf
, int in_len
,
1602 struct ib_uverbs_poll_cq cmd
;
1603 struct ib_uverbs_poll_cq_resp resp
;
1604 u8 __user
*header_ptr
;
1605 u8 __user
*data_ptr
;
1610 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1613 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1617 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1618 header_ptr
= (void __user
*)(unsigned long) cmd
.response
;
1619 data_ptr
= header_ptr
+ sizeof resp
;
1621 memset(&resp
, 0, sizeof resp
);
1622 while (resp
.count
< cmd
.ne
) {
1623 ret
= ib_poll_cq(cq
, 1, &wc
);
1629 ret
= copy_wc_to_user(data_ptr
, &wc
);
1633 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1637 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1649 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
1650 struct ib_device
*ib_dev
,
1651 const char __user
*buf
, int in_len
,
1654 struct ib_uverbs_req_notify_cq cmd
;
1657 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1660 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1664 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1665 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1672 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
1673 struct ib_device
*ib_dev
,
1674 const char __user
*buf
, int in_len
,
1677 struct ib_uverbs_destroy_cq cmd
;
1678 struct ib_uverbs_destroy_cq_resp resp
;
1679 struct ib_uobject
*uobj
;
1681 struct ib_ucq_object
*obj
;
1682 struct ib_uverbs_event_file
*ev_file
;
1685 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1688 uobj
= idr_write_uobj(&ib_uverbs_cq_idr
, cmd
.cq_handle
, file
->ucontext
);
1692 ev_file
= cq
->cq_context
;
1693 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1695 ret
= ib_destroy_cq(cq
);
1699 put_uobj_write(uobj
);
1704 idr_remove_uobj(&ib_uverbs_cq_idr
, uobj
);
1706 mutex_lock(&file
->mutex
);
1707 list_del(&uobj
->list
);
1708 mutex_unlock(&file
->mutex
);
1710 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1712 memset(&resp
, 0, sizeof resp
);
1713 resp
.comp_events_reported
= obj
->comp_events_reported
;
1714 resp
.async_events_reported
= obj
->async_events_reported
;
1718 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1719 &resp
, sizeof resp
))
1725 static int create_qp(struct ib_uverbs_file
*file
,
1726 struct ib_udata
*ucore
,
1727 struct ib_udata
*uhw
,
1728 struct ib_uverbs_ex_create_qp
*cmd
,
1730 int (*cb
)(struct ib_uverbs_file
*file
,
1731 struct ib_uverbs_ex_create_qp_resp
*resp
,
1732 struct ib_udata
*udata
),
1735 struct ib_uqp_object
*obj
;
1736 struct ib_device
*device
;
1737 struct ib_pd
*pd
= NULL
;
1738 struct ib_xrcd
*xrcd
= NULL
;
1739 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1740 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1741 struct ib_srq
*srq
= NULL
;
1744 struct ib_qp_init_attr attr
;
1745 struct ib_uverbs_ex_create_qp_resp resp
;
1748 if (cmd
->qp_type
== IB_QPT_RAW_PACKET
&& !capable(CAP_NET_RAW
))
1751 obj
= kzalloc(sizeof *obj
, GFP_KERNEL
);
1755 init_uobj(&obj
->uevent
.uobject
, cmd
->user_handle
, file
->ucontext
,
1757 down_write(&obj
->uevent
.uobject
.mutex
);
1759 if (cmd
->qp_type
== IB_QPT_XRC_TGT
) {
1760 xrcd
= idr_read_xrcd(cmd
->pd_handle
, file
->ucontext
,
1766 device
= xrcd
->device
;
1768 if (cmd
->qp_type
== IB_QPT_XRC_INI
) {
1769 cmd
->max_recv_wr
= 0;
1770 cmd
->max_recv_sge
= 0;
1773 srq
= idr_read_srq(cmd
->srq_handle
,
1775 if (!srq
|| srq
->srq_type
!= IB_SRQT_BASIC
) {
1781 if (cmd
->recv_cq_handle
!= cmd
->send_cq_handle
) {
1782 rcq
= idr_read_cq(cmd
->recv_cq_handle
,
1791 scq
= idr_read_cq(cmd
->send_cq_handle
, file
->ucontext
, !!rcq
);
1793 pd
= idr_read_pd(cmd
->pd_handle
, file
->ucontext
);
1799 device
= pd
->device
;
1802 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1803 attr
.qp_context
= file
;
1808 attr
.sq_sig_type
= cmd
->sq_sig_all
? IB_SIGNAL_ALL_WR
:
1810 attr
.qp_type
= cmd
->qp_type
;
1811 attr
.create_flags
= 0;
1813 attr
.cap
.max_send_wr
= cmd
->max_send_wr
;
1814 attr
.cap
.max_recv_wr
= cmd
->max_recv_wr
;
1815 attr
.cap
.max_send_sge
= cmd
->max_send_sge
;
1816 attr
.cap
.max_recv_sge
= cmd
->max_recv_sge
;
1817 attr
.cap
.max_inline_data
= cmd
->max_inline_data
;
1819 obj
->uevent
.events_reported
= 0;
1820 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1821 INIT_LIST_HEAD(&obj
->mcast_list
);
1823 if (cmd_sz
>= offsetof(typeof(*cmd
), create_flags
) +
1824 sizeof(cmd
->create_flags
))
1825 attr
.create_flags
= cmd
->create_flags
;
1827 if (attr
.create_flags
& ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
|
1828 IB_QP_CREATE_CROSS_CHANNEL
|
1829 IB_QP_CREATE_MANAGED_SEND
|
1830 IB_QP_CREATE_MANAGED_RECV
)) {
1835 buf
= (void *)cmd
+ sizeof(*cmd
);
1836 if (cmd_sz
> sizeof(*cmd
))
1837 if (!(buf
[0] == 0 && !memcmp(buf
, buf
+ 1,
1838 cmd_sz
- sizeof(*cmd
) - 1))) {
1843 if (cmd
->qp_type
== IB_QPT_XRC_TGT
)
1844 qp
= ib_create_qp(pd
, &attr
);
1846 qp
= device
->create_qp(pd
, &attr
, uhw
);
1853 if (cmd
->qp_type
!= IB_QPT_XRC_TGT
) {
1855 qp
->device
= device
;
1857 qp
->send_cq
= attr
.send_cq
;
1858 qp
->recv_cq
= attr
.recv_cq
;
1860 qp
->event_handler
= attr
.event_handler
;
1861 qp
->qp_context
= attr
.qp_context
;
1862 qp
->qp_type
= attr
.qp_type
;
1863 atomic_set(&qp
->usecnt
, 0);
1864 atomic_inc(&pd
->usecnt
);
1865 atomic_inc(&attr
.send_cq
->usecnt
);
1867 atomic_inc(&attr
.recv_cq
->usecnt
);
1869 atomic_inc(&attr
.srq
->usecnt
);
1871 qp
->uobject
= &obj
->uevent
.uobject
;
1873 obj
->uevent
.uobject
.object
= qp
;
1874 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1878 memset(&resp
, 0, sizeof resp
);
1879 resp
.base
.qpn
= qp
->qp_num
;
1880 resp
.base
.qp_handle
= obj
->uevent
.uobject
.id
;
1881 resp
.base
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1882 resp
.base
.max_send_sge
= attr
.cap
.max_send_sge
;
1883 resp
.base
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1884 resp
.base
.max_send_wr
= attr
.cap
.max_send_wr
;
1885 resp
.base
.max_inline_data
= attr
.cap
.max_inline_data
;
1887 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1888 sizeof(resp
.response_length
);
1890 ret
= cb(file
, &resp
, ucore
);
1895 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
,
1897 atomic_inc(&obj
->uxrcd
->refcnt
);
1898 put_xrcd_read(xrcd_uobj
);
1905 if (rcq
&& rcq
!= scq
)
1910 mutex_lock(&file
->mutex
);
1911 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1912 mutex_unlock(&file
->mutex
);
1914 obj
->uevent
.uobject
.live
= 1;
1916 up_write(&obj
->uevent
.uobject
.mutex
);
1920 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1927 put_xrcd_read(xrcd_uobj
);
1932 if (rcq
&& rcq
!= scq
)
1937 put_uobj_write(&obj
->uevent
.uobject
);
1941 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file
*file
,
1942 struct ib_uverbs_ex_create_qp_resp
*resp
,
1943 struct ib_udata
*ucore
)
1945 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1951 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1952 struct ib_device
*ib_dev
,
1953 const char __user
*buf
, int in_len
,
1956 struct ib_uverbs_create_qp cmd
;
1957 struct ib_uverbs_ex_create_qp cmd_ex
;
1958 struct ib_udata ucore
;
1959 struct ib_udata uhw
;
1960 ssize_t resp_size
= sizeof(struct ib_uverbs_create_qp_resp
);
1963 if (out_len
< resp_size
)
1966 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1969 INIT_UDATA(&ucore
, buf
, (unsigned long)cmd
.response
, sizeof(cmd
),
1971 INIT_UDATA(&uhw
, buf
+ sizeof(cmd
),
1972 (unsigned long)cmd
.response
+ resp_size
,
1973 in_len
- sizeof(cmd
), out_len
- resp_size
);
1975 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1976 cmd_ex
.user_handle
= cmd
.user_handle
;
1977 cmd_ex
.pd_handle
= cmd
.pd_handle
;
1978 cmd_ex
.send_cq_handle
= cmd
.send_cq_handle
;
1979 cmd_ex
.recv_cq_handle
= cmd
.recv_cq_handle
;
1980 cmd_ex
.srq_handle
= cmd
.srq_handle
;
1981 cmd_ex
.max_send_wr
= cmd
.max_send_wr
;
1982 cmd_ex
.max_recv_wr
= cmd
.max_recv_wr
;
1983 cmd_ex
.max_send_sge
= cmd
.max_send_sge
;
1984 cmd_ex
.max_recv_sge
= cmd
.max_recv_sge
;
1985 cmd_ex
.max_inline_data
= cmd
.max_inline_data
;
1986 cmd_ex
.sq_sig_all
= cmd
.sq_sig_all
;
1987 cmd_ex
.qp_type
= cmd
.qp_type
;
1988 cmd_ex
.is_srq
= cmd
.is_srq
;
1990 err
= create_qp(file
, &ucore
, &uhw
, &cmd_ex
,
1991 offsetof(typeof(cmd_ex
), is_srq
) +
1992 sizeof(cmd
.is_srq
), ib_uverbs_create_qp_cb
,
2001 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file
*file
,
2002 struct ib_uverbs_ex_create_qp_resp
*resp
,
2003 struct ib_udata
*ucore
)
2005 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
2011 int ib_uverbs_ex_create_qp(struct ib_uverbs_file
*file
,
2012 struct ib_device
*ib_dev
,
2013 struct ib_udata
*ucore
,
2014 struct ib_udata
*uhw
)
2016 struct ib_uverbs_ex_create_qp_resp resp
;
2017 struct ib_uverbs_ex_create_qp cmd
= {0};
2020 if (ucore
->inlen
< (offsetof(typeof(cmd
), comp_mask
) +
2021 sizeof(cmd
.comp_mask
)))
2024 err
= ib_copy_from_udata(&cmd
, ucore
, min(sizeof(cmd
), ucore
->inlen
));
2034 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
2035 sizeof(resp
.response_length
)))
2038 err
= create_qp(file
, ucore
, uhw
, &cmd
,
2039 min(ucore
->inlen
, sizeof(cmd
)),
2040 ib_uverbs_ex_create_qp_cb
, NULL
);
2048 ssize_t
ib_uverbs_open_qp(struct ib_uverbs_file
*file
,
2049 struct ib_device
*ib_dev
,
2050 const char __user
*buf
, int in_len
, int out_len
)
2052 struct ib_uverbs_open_qp cmd
;
2053 struct ib_uverbs_create_qp_resp resp
;
2054 struct ib_udata udata
;
2055 struct ib_uqp_object
*obj
;
2056 struct ib_xrcd
*xrcd
;
2057 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
2059 struct ib_qp_open_attr attr
;
2062 if (out_len
< sizeof resp
)
2065 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2068 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
2069 (unsigned long) cmd
.response
+ sizeof resp
,
2070 in_len
- sizeof cmd
, out_len
- sizeof resp
);
2072 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
2076 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_class
);
2077 down_write(&obj
->uevent
.uobject
.mutex
);
2079 xrcd
= idr_read_xrcd(cmd
.pd_handle
, file
->ucontext
, &xrcd_uobj
);
2085 attr
.event_handler
= ib_uverbs_qp_event_handler
;
2086 attr
.qp_context
= file
;
2087 attr
.qp_num
= cmd
.qpn
;
2088 attr
.qp_type
= cmd
.qp_type
;
2090 obj
->uevent
.events_reported
= 0;
2091 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
2092 INIT_LIST_HEAD(&obj
->mcast_list
);
2094 qp
= ib_open_qp(xrcd
, &attr
);
2100 qp
->uobject
= &obj
->uevent
.uobject
;
2102 obj
->uevent
.uobject
.object
= qp
;
2103 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
2107 memset(&resp
, 0, sizeof resp
);
2108 resp
.qpn
= qp
->qp_num
;
2109 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
2111 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2112 &resp
, sizeof resp
)) {
2117 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
2118 atomic_inc(&obj
->uxrcd
->refcnt
);
2119 put_xrcd_read(xrcd_uobj
);
2121 mutex_lock(&file
->mutex
);
2122 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
2123 mutex_unlock(&file
->mutex
);
2125 obj
->uevent
.uobject
.live
= 1;
2127 up_write(&obj
->uevent
.uobject
.mutex
);
2132 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
2138 put_xrcd_read(xrcd_uobj
);
2139 put_uobj_write(&obj
->uevent
.uobject
);
2143 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
2144 struct ib_device
*ib_dev
,
2145 const char __user
*buf
, int in_len
,
2148 struct ib_uverbs_query_qp cmd
;
2149 struct ib_uverbs_query_qp_resp resp
;
2151 struct ib_qp_attr
*attr
;
2152 struct ib_qp_init_attr
*init_attr
;
2155 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2158 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
2159 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
2160 if (!attr
|| !init_attr
) {
2165 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2171 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
2178 memset(&resp
, 0, sizeof resp
);
2180 resp
.qp_state
= attr
->qp_state
;
2181 resp
.cur_qp_state
= attr
->cur_qp_state
;
2182 resp
.path_mtu
= attr
->path_mtu
;
2183 resp
.path_mig_state
= attr
->path_mig_state
;
2184 resp
.qkey
= attr
->qkey
;
2185 resp
.rq_psn
= attr
->rq_psn
;
2186 resp
.sq_psn
= attr
->sq_psn
;
2187 resp
.dest_qp_num
= attr
->dest_qp_num
;
2188 resp
.qp_access_flags
= attr
->qp_access_flags
;
2189 resp
.pkey_index
= attr
->pkey_index
;
2190 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
2191 resp
.sq_draining
= attr
->sq_draining
;
2192 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
2193 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
2194 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
2195 resp
.port_num
= attr
->port_num
;
2196 resp
.timeout
= attr
->timeout
;
2197 resp
.retry_cnt
= attr
->retry_cnt
;
2198 resp
.rnr_retry
= attr
->rnr_retry
;
2199 resp
.alt_port_num
= attr
->alt_port_num
;
2200 resp
.alt_timeout
= attr
->alt_timeout
;
2202 memcpy(resp
.dest
.dgid
, attr
->ah_attr
.grh
.dgid
.raw
, 16);
2203 resp
.dest
.flow_label
= attr
->ah_attr
.grh
.flow_label
;
2204 resp
.dest
.sgid_index
= attr
->ah_attr
.grh
.sgid_index
;
2205 resp
.dest
.hop_limit
= attr
->ah_attr
.grh
.hop_limit
;
2206 resp
.dest
.traffic_class
= attr
->ah_attr
.grh
.traffic_class
;
2207 resp
.dest
.dlid
= attr
->ah_attr
.dlid
;
2208 resp
.dest
.sl
= attr
->ah_attr
.sl
;
2209 resp
.dest
.src_path_bits
= attr
->ah_attr
.src_path_bits
;
2210 resp
.dest
.static_rate
= attr
->ah_attr
.static_rate
;
2211 resp
.dest
.is_global
= !!(attr
->ah_attr
.ah_flags
& IB_AH_GRH
);
2212 resp
.dest
.port_num
= attr
->ah_attr
.port_num
;
2214 memcpy(resp
.alt_dest
.dgid
, attr
->alt_ah_attr
.grh
.dgid
.raw
, 16);
2215 resp
.alt_dest
.flow_label
= attr
->alt_ah_attr
.grh
.flow_label
;
2216 resp
.alt_dest
.sgid_index
= attr
->alt_ah_attr
.grh
.sgid_index
;
2217 resp
.alt_dest
.hop_limit
= attr
->alt_ah_attr
.grh
.hop_limit
;
2218 resp
.alt_dest
.traffic_class
= attr
->alt_ah_attr
.grh
.traffic_class
;
2219 resp
.alt_dest
.dlid
= attr
->alt_ah_attr
.dlid
;
2220 resp
.alt_dest
.sl
= attr
->alt_ah_attr
.sl
;
2221 resp
.alt_dest
.src_path_bits
= attr
->alt_ah_attr
.src_path_bits
;
2222 resp
.alt_dest
.static_rate
= attr
->alt_ah_attr
.static_rate
;
2223 resp
.alt_dest
.is_global
= !!(attr
->alt_ah_attr
.ah_flags
& IB_AH_GRH
);
2224 resp
.alt_dest
.port_num
= attr
->alt_ah_attr
.port_num
;
2226 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
2227 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
2228 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
2229 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
2230 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
2231 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
2233 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2234 &resp
, sizeof resp
))
2241 return ret
? ret
: in_len
;
2244 /* Remove ignored fields set in the attribute mask */
2245 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
2248 case IB_QPT_XRC_INI
:
2249 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
2250 case IB_QPT_XRC_TGT
:
2251 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
2258 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
2259 struct ib_device
*ib_dev
,
2260 const char __user
*buf
, int in_len
,
2263 struct ib_uverbs_modify_qp cmd
;
2264 struct ib_udata udata
;
2266 struct ib_qp_attr
*attr
;
2269 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2272 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
2275 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
2279 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2285 attr
->qp_state
= cmd
.qp_state
;
2286 attr
->cur_qp_state
= cmd
.cur_qp_state
;
2287 attr
->path_mtu
= cmd
.path_mtu
;
2288 attr
->path_mig_state
= cmd
.path_mig_state
;
2289 attr
->qkey
= cmd
.qkey
;
2290 attr
->rq_psn
= cmd
.rq_psn
;
2291 attr
->sq_psn
= cmd
.sq_psn
;
2292 attr
->dest_qp_num
= cmd
.dest_qp_num
;
2293 attr
->qp_access_flags
= cmd
.qp_access_flags
;
2294 attr
->pkey_index
= cmd
.pkey_index
;
2295 attr
->alt_pkey_index
= cmd
.alt_pkey_index
;
2296 attr
->en_sqd_async_notify
= cmd
.en_sqd_async_notify
;
2297 attr
->max_rd_atomic
= cmd
.max_rd_atomic
;
2298 attr
->max_dest_rd_atomic
= cmd
.max_dest_rd_atomic
;
2299 attr
->min_rnr_timer
= cmd
.min_rnr_timer
;
2300 attr
->port_num
= cmd
.port_num
;
2301 attr
->timeout
= cmd
.timeout
;
2302 attr
->retry_cnt
= cmd
.retry_cnt
;
2303 attr
->rnr_retry
= cmd
.rnr_retry
;
2304 attr
->alt_port_num
= cmd
.alt_port_num
;
2305 attr
->alt_timeout
= cmd
.alt_timeout
;
2307 memcpy(attr
->ah_attr
.grh
.dgid
.raw
, cmd
.dest
.dgid
, 16);
2308 attr
->ah_attr
.grh
.flow_label
= cmd
.dest
.flow_label
;
2309 attr
->ah_attr
.grh
.sgid_index
= cmd
.dest
.sgid_index
;
2310 attr
->ah_attr
.grh
.hop_limit
= cmd
.dest
.hop_limit
;
2311 attr
->ah_attr
.grh
.traffic_class
= cmd
.dest
.traffic_class
;
2312 attr
->ah_attr
.dlid
= cmd
.dest
.dlid
;
2313 attr
->ah_attr
.sl
= cmd
.dest
.sl
;
2314 attr
->ah_attr
.src_path_bits
= cmd
.dest
.src_path_bits
;
2315 attr
->ah_attr
.static_rate
= cmd
.dest
.static_rate
;
2316 attr
->ah_attr
.ah_flags
= cmd
.dest
.is_global
? IB_AH_GRH
: 0;
2317 attr
->ah_attr
.port_num
= cmd
.dest
.port_num
;
2319 memcpy(attr
->alt_ah_attr
.grh
.dgid
.raw
, cmd
.alt_dest
.dgid
, 16);
2320 attr
->alt_ah_attr
.grh
.flow_label
= cmd
.alt_dest
.flow_label
;
2321 attr
->alt_ah_attr
.grh
.sgid_index
= cmd
.alt_dest
.sgid_index
;
2322 attr
->alt_ah_attr
.grh
.hop_limit
= cmd
.alt_dest
.hop_limit
;
2323 attr
->alt_ah_attr
.grh
.traffic_class
= cmd
.alt_dest
.traffic_class
;
2324 attr
->alt_ah_attr
.dlid
= cmd
.alt_dest
.dlid
;
2325 attr
->alt_ah_attr
.sl
= cmd
.alt_dest
.sl
;
2326 attr
->alt_ah_attr
.src_path_bits
= cmd
.alt_dest
.src_path_bits
;
2327 attr
->alt_ah_attr
.static_rate
= cmd
.alt_dest
.static_rate
;
2328 attr
->alt_ah_attr
.ah_flags
= cmd
.alt_dest
.is_global
? IB_AH_GRH
: 0;
2329 attr
->alt_ah_attr
.port_num
= cmd
.alt_dest
.port_num
;
2331 if (qp
->real_qp
== qp
) {
2332 ret
= ib_resolve_eth_dmac(qp
, attr
, &cmd
.attr_mask
);
2335 ret
= qp
->device
->modify_qp(qp
, attr
,
2336 modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
), &udata
);
2338 ret
= ib_modify_qp(qp
, attr
, modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
));
2355 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
2356 struct ib_device
*ib_dev
,
2357 const char __user
*buf
, int in_len
,
2360 struct ib_uverbs_destroy_qp cmd
;
2361 struct ib_uverbs_destroy_qp_resp resp
;
2362 struct ib_uobject
*uobj
;
2364 struct ib_uqp_object
*obj
;
2367 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2370 memset(&resp
, 0, sizeof resp
);
2372 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, cmd
.qp_handle
, file
->ucontext
);
2376 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
2378 if (!list_empty(&obj
->mcast_list
)) {
2379 put_uobj_write(uobj
);
2383 ret
= ib_destroy_qp(qp
);
2387 put_uobj_write(uobj
);
2393 atomic_dec(&obj
->uxrcd
->refcnt
);
2395 idr_remove_uobj(&ib_uverbs_qp_idr
, uobj
);
2397 mutex_lock(&file
->mutex
);
2398 list_del(&uobj
->list
);
2399 mutex_unlock(&file
->mutex
);
2401 ib_uverbs_release_uevent(file
, &obj
->uevent
);
2403 resp
.events_reported
= obj
->uevent
.events_reported
;
2407 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2408 &resp
, sizeof resp
))
2414 static void *alloc_wr(size_t wr_size
, __u32 num_sge
)
2416 return kmalloc(ALIGN(wr_size
, sizeof (struct ib_sge
)) +
2417 num_sge
* sizeof (struct ib_sge
), GFP_KERNEL
);
2420 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
2421 struct ib_device
*ib_dev
,
2422 const char __user
*buf
, int in_len
,
2425 struct ib_uverbs_post_send cmd
;
2426 struct ib_uverbs_post_send_resp resp
;
2427 struct ib_uverbs_send_wr
*user_wr
;
2428 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
2432 ssize_t ret
= -EINVAL
;
2435 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2438 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
2439 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
2442 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
2445 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
2449 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2453 is_ud
= qp
->qp_type
== IB_QPT_UD
;
2456 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
2457 if (copy_from_user(user_wr
,
2458 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
2464 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
2470 struct ib_ud_wr
*ud
;
2472 if (user_wr
->opcode
!= IB_WR_SEND
&&
2473 user_wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
2478 next_size
= sizeof(*ud
);
2479 ud
= alloc_wr(next_size
, user_wr
->num_sge
);
2485 ud
->ah
= idr_read_ah(user_wr
->wr
.ud
.ah
, file
->ucontext
);
2491 ud
->remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
2492 ud
->remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
2495 } else if (user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
||
2496 user_wr
->opcode
== IB_WR_RDMA_WRITE
||
2497 user_wr
->opcode
== IB_WR_RDMA_READ
) {
2498 struct ib_rdma_wr
*rdma
;
2500 next_size
= sizeof(*rdma
);
2501 rdma
= alloc_wr(next_size
, user_wr
->num_sge
);
2507 rdma
->remote_addr
= user_wr
->wr
.rdma
.remote_addr
;
2508 rdma
->rkey
= user_wr
->wr
.rdma
.rkey
;
2511 } else if (user_wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
||
2512 user_wr
->opcode
== IB_WR_ATOMIC_FETCH_AND_ADD
) {
2513 struct ib_atomic_wr
*atomic
;
2515 next_size
= sizeof(*atomic
);
2516 atomic
= alloc_wr(next_size
, user_wr
->num_sge
);
2522 atomic
->remote_addr
= user_wr
->wr
.atomic
.remote_addr
;
2523 atomic
->compare_add
= user_wr
->wr
.atomic
.compare_add
;
2524 atomic
->swap
= user_wr
->wr
.atomic
.swap
;
2525 atomic
->rkey
= user_wr
->wr
.atomic
.rkey
;
2528 } else if (user_wr
->opcode
== IB_WR_SEND
||
2529 user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2530 user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2531 next_size
= sizeof(*next
);
2532 next
= alloc_wr(next_size
, user_wr
->num_sge
);
2542 if (user_wr
->opcode
== IB_WR_SEND_WITH_IMM
||
2543 user_wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
) {
2545 (__be32 __force
) user_wr
->ex
.imm_data
;
2546 } else if (user_wr
->opcode
== IB_WR_SEND_WITH_INV
) {
2547 next
->ex
.invalidate_rkey
= user_wr
->ex
.invalidate_rkey
;
2557 next
->wr_id
= user_wr
->wr_id
;
2558 next
->num_sge
= user_wr
->num_sge
;
2559 next
->opcode
= user_wr
->opcode
;
2560 next
->send_flags
= user_wr
->send_flags
;
2562 if (next
->num_sge
) {
2563 next
->sg_list
= (void *) next
+
2564 ALIGN(next_size
, sizeof(struct ib_sge
));
2565 if (copy_from_user(next
->sg_list
,
2567 cmd
.wr_count
* cmd
.wqe_size
+
2568 sg_ind
* sizeof (struct ib_sge
),
2569 next
->num_sge
* sizeof (struct ib_sge
))) {
2573 sg_ind
+= next
->num_sge
;
2575 next
->sg_list
= NULL
;
2579 ret
= qp
->device
->post_send(qp
->real_qp
, wr
, &bad_wr
);
2581 for (next
= wr
; next
; next
= next
->next
) {
2587 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2588 &resp
, sizeof resp
))
2595 if (is_ud
&& ud_wr(wr
)->ah
)
2596 put_ah_read(ud_wr(wr
)->ah
);
2605 return ret
? ret
: in_len
;
2608 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
2614 struct ib_uverbs_recv_wr
*user_wr
;
2615 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2620 if (in_len
< wqe_size
* wr_count
+
2621 sge_count
* sizeof (struct ib_uverbs_sge
))
2622 return ERR_PTR(-EINVAL
);
2624 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2625 return ERR_PTR(-EINVAL
);
2627 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2629 return ERR_PTR(-ENOMEM
);
2633 for (i
= 0; i
< wr_count
; ++i
) {
2634 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
2640 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2645 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2646 user_wr
->num_sge
* sizeof (struct ib_sge
),
2660 next
->wr_id
= user_wr
->wr_id
;
2661 next
->num_sge
= user_wr
->num_sge
;
2663 if (next
->num_sge
) {
2664 next
->sg_list
= (void *) next
+
2665 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2666 if (copy_from_user(next
->sg_list
,
2667 buf
+ wr_count
* wqe_size
+
2668 sg_ind
* sizeof (struct ib_sge
),
2669 next
->num_sge
* sizeof (struct ib_sge
))) {
2673 sg_ind
+= next
->num_sge
;
2675 next
->sg_list
= NULL
;
2690 return ERR_PTR(ret
);
2693 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
2694 struct ib_device
*ib_dev
,
2695 const char __user
*buf
, int in_len
,
2698 struct ib_uverbs_post_recv cmd
;
2699 struct ib_uverbs_post_recv_resp resp
;
2700 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2702 ssize_t ret
= -EINVAL
;
2704 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2707 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2708 in_len
- sizeof cmd
, cmd
.wr_count
,
2709 cmd
.sge_count
, cmd
.wqe_size
);
2713 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2718 ret
= qp
->device
->post_recv(qp
->real_qp
, wr
, &bad_wr
);
2723 for (next
= wr
; next
; next
= next
->next
) {
2729 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2730 &resp
, sizeof resp
))
2740 return ret
? ret
: in_len
;
2743 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
2744 struct ib_device
*ib_dev
,
2745 const char __user
*buf
, int in_len
,
2748 struct ib_uverbs_post_srq_recv cmd
;
2749 struct ib_uverbs_post_srq_recv_resp resp
;
2750 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2752 ssize_t ret
= -EINVAL
;
2754 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2757 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2758 in_len
- sizeof cmd
, cmd
.wr_count
,
2759 cmd
.sge_count
, cmd
.wqe_size
);
2763 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2768 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
2773 for (next
= wr
; next
; next
= next
->next
) {
2779 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2780 &resp
, sizeof resp
))
2790 return ret
? ret
: in_len
;
2793 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
2794 struct ib_device
*ib_dev
,
2795 const char __user
*buf
, int in_len
,
2798 struct ib_uverbs_create_ah cmd
;
2799 struct ib_uverbs_create_ah_resp resp
;
2800 struct ib_uobject
*uobj
;
2803 struct ib_ah_attr attr
;
2806 if (out_len
< sizeof resp
)
2809 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2812 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
2816 init_uobj(uobj
, cmd
.user_handle
, file
->ucontext
, &ah_lock_class
);
2817 down_write(&uobj
->mutex
);
2819 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
2825 attr
.dlid
= cmd
.attr
.dlid
;
2826 attr
.sl
= cmd
.attr
.sl
;
2827 attr
.src_path_bits
= cmd
.attr
.src_path_bits
;
2828 attr
.static_rate
= cmd
.attr
.static_rate
;
2829 attr
.ah_flags
= cmd
.attr
.is_global
? IB_AH_GRH
: 0;
2830 attr
.port_num
= cmd
.attr
.port_num
;
2831 attr
.grh
.flow_label
= cmd
.attr
.grh
.flow_label
;
2832 attr
.grh
.sgid_index
= cmd
.attr
.grh
.sgid_index
;
2833 attr
.grh
.hop_limit
= cmd
.attr
.grh
.hop_limit
;
2834 attr
.grh
.traffic_class
= cmd
.attr
.grh
.traffic_class
;
2835 memset(&attr
.dmac
, 0, sizeof(attr
.dmac
));
2836 memcpy(attr
.grh
.dgid
.raw
, cmd
.attr
.grh
.dgid
, 16);
2838 ah
= ib_create_ah(pd
, &attr
);
2847 ret
= idr_add_uobj(&ib_uverbs_ah_idr
, uobj
);
2851 resp
.ah_handle
= uobj
->id
;
2853 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2854 &resp
, sizeof resp
)) {
2861 mutex_lock(&file
->mutex
);
2862 list_add_tail(&uobj
->list
, &file
->ucontext
->ah_list
);
2863 mutex_unlock(&file
->mutex
);
2867 up_write(&uobj
->mutex
);
2872 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2881 put_uobj_write(uobj
);
2885 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
2886 struct ib_device
*ib_dev
,
2887 const char __user
*buf
, int in_len
, int out_len
)
2889 struct ib_uverbs_destroy_ah cmd
;
2891 struct ib_uobject
*uobj
;
2894 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2897 uobj
= idr_write_uobj(&ib_uverbs_ah_idr
, cmd
.ah_handle
, file
->ucontext
);
2902 ret
= ib_destroy_ah(ah
);
2906 put_uobj_write(uobj
);
2911 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2913 mutex_lock(&file
->mutex
);
2914 list_del(&uobj
->list
);
2915 mutex_unlock(&file
->mutex
);
2922 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
2923 struct ib_device
*ib_dev
,
2924 const char __user
*buf
, int in_len
,
2927 struct ib_uverbs_attach_mcast cmd
;
2929 struct ib_uqp_object
*obj
;
2930 struct ib_uverbs_mcast_entry
*mcast
;
2933 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2936 qp
= idr_write_qp(cmd
.qp_handle
, file
->ucontext
);
2940 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2942 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2943 if (cmd
.mlid
== mcast
->lid
&&
2944 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2949 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2955 mcast
->lid
= cmd
.mlid
;
2956 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2958 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2960 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2967 return ret
? ret
: in_len
;
2970 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
2971 struct ib_device
*ib_dev
,
2972 const char __user
*buf
, int in_len
,
2975 struct ib_uverbs_detach_mcast cmd
;
2976 struct ib_uqp_object
*obj
;
2978 struct ib_uverbs_mcast_entry
*mcast
;
2981 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2984 qp
= idr_write_qp(cmd
.qp_handle
, file
->ucontext
);
2988 ret
= ib_detach_mcast(qp
, (union ib_gid
*) cmd
.gid
, cmd
.mlid
);
2992 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2994 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2995 if (cmd
.mlid
== mcast
->lid
&&
2996 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2997 list_del(&mcast
->list
);
3005 return ret
? ret
: in_len
;
3008 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec
*kern_spec
,
3009 union ib_flow_spec
*ib_spec
)
3011 if (kern_spec
->reserved
)
3014 ib_spec
->type
= kern_spec
->type
;
3016 switch (ib_spec
->type
) {
3017 case IB_FLOW_SPEC_ETH
:
3018 ib_spec
->eth
.size
= sizeof(struct ib_flow_spec_eth
);
3019 if (ib_spec
->eth
.size
!= kern_spec
->eth
.size
)
3021 memcpy(&ib_spec
->eth
.val
, &kern_spec
->eth
.val
,
3022 sizeof(struct ib_flow_eth_filter
));
3023 memcpy(&ib_spec
->eth
.mask
, &kern_spec
->eth
.mask
,
3024 sizeof(struct ib_flow_eth_filter
));
3026 case IB_FLOW_SPEC_IPV4
:
3027 ib_spec
->ipv4
.size
= sizeof(struct ib_flow_spec_ipv4
);
3028 if (ib_spec
->ipv4
.size
!= kern_spec
->ipv4
.size
)
3030 memcpy(&ib_spec
->ipv4
.val
, &kern_spec
->ipv4
.val
,
3031 sizeof(struct ib_flow_ipv4_filter
));
3032 memcpy(&ib_spec
->ipv4
.mask
, &kern_spec
->ipv4
.mask
,
3033 sizeof(struct ib_flow_ipv4_filter
));
3035 case IB_FLOW_SPEC_TCP
:
3036 case IB_FLOW_SPEC_UDP
:
3037 ib_spec
->tcp_udp
.size
= sizeof(struct ib_flow_spec_tcp_udp
);
3038 if (ib_spec
->tcp_udp
.size
!= kern_spec
->tcp_udp
.size
)
3040 memcpy(&ib_spec
->tcp_udp
.val
, &kern_spec
->tcp_udp
.val
,
3041 sizeof(struct ib_flow_tcp_udp_filter
));
3042 memcpy(&ib_spec
->tcp_udp
.mask
, &kern_spec
->tcp_udp
.mask
,
3043 sizeof(struct ib_flow_tcp_udp_filter
));
3051 int ib_uverbs_ex_create_flow(struct ib_uverbs_file
*file
,
3052 struct ib_device
*ib_dev
,
3053 struct ib_udata
*ucore
,
3054 struct ib_udata
*uhw
)
3056 struct ib_uverbs_create_flow cmd
;
3057 struct ib_uverbs_create_flow_resp resp
;
3058 struct ib_uobject
*uobj
;
3059 struct ib_flow
*flow_id
;
3060 struct ib_uverbs_flow_attr
*kern_flow_attr
;
3061 struct ib_flow_attr
*flow_attr
;
3068 if (ucore
->inlen
< sizeof(cmd
))
3071 if (ucore
->outlen
< sizeof(resp
))
3074 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3078 ucore
->inbuf
+= sizeof(cmd
);
3079 ucore
->inlen
-= sizeof(cmd
);
3084 if ((cmd
.flow_attr
.type
== IB_FLOW_ATTR_SNIFFER
&&
3085 !capable(CAP_NET_ADMIN
)) || !capable(CAP_NET_RAW
))
3088 if (cmd
.flow_attr
.num_of_specs
> IB_FLOW_SPEC_SUPPORT_LAYERS
)
3091 if (cmd
.flow_attr
.size
> ucore
->inlen
||
3092 cmd
.flow_attr
.size
>
3093 (cmd
.flow_attr
.num_of_specs
* sizeof(struct ib_uverbs_flow_spec
)))
3096 if (cmd
.flow_attr
.reserved
[0] ||
3097 cmd
.flow_attr
.reserved
[1])
3100 if (cmd
.flow_attr
.num_of_specs
) {
3101 kern_flow_attr
= kmalloc(sizeof(*kern_flow_attr
) + cmd
.flow_attr
.size
,
3103 if (!kern_flow_attr
)
3106 memcpy(kern_flow_attr
, &cmd
.flow_attr
, sizeof(*kern_flow_attr
));
3107 err
= ib_copy_from_udata(kern_flow_attr
+ 1, ucore
,
3108 cmd
.flow_attr
.size
);
3112 kern_flow_attr
= &cmd
.flow_attr
;
3115 uobj
= kmalloc(sizeof(*uobj
), GFP_KERNEL
);
3120 init_uobj(uobj
, 0, file
->ucontext
, &rule_lock_class
);
3121 down_write(&uobj
->mutex
);
3123 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
3129 flow_attr
= kmalloc(sizeof(*flow_attr
) + cmd
.flow_attr
.size
, GFP_KERNEL
);
3135 flow_attr
->type
= kern_flow_attr
->type
;
3136 flow_attr
->priority
= kern_flow_attr
->priority
;
3137 flow_attr
->num_of_specs
= kern_flow_attr
->num_of_specs
;
3138 flow_attr
->port
= kern_flow_attr
->port
;
3139 flow_attr
->flags
= kern_flow_attr
->flags
;
3140 flow_attr
->size
= sizeof(*flow_attr
);
3142 kern_spec
= kern_flow_attr
+ 1;
3143 ib_spec
= flow_attr
+ 1;
3144 for (i
= 0; i
< flow_attr
->num_of_specs
&&
3145 cmd
.flow_attr
.size
> offsetof(struct ib_uverbs_flow_spec
, reserved
) &&
3146 cmd
.flow_attr
.size
>=
3147 ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
; i
++) {
3148 err
= kern_spec_to_ib_spec(kern_spec
, ib_spec
);
3152 ((union ib_flow_spec
*) ib_spec
)->size
;
3153 cmd
.flow_attr
.size
-= ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
;
3154 kern_spec
+= ((struct ib_uverbs_flow_spec
*) kern_spec
)->size
;
3155 ib_spec
+= ((union ib_flow_spec
*) ib_spec
)->size
;
3157 if (cmd
.flow_attr
.size
|| (i
!= flow_attr
->num_of_specs
)) {
3158 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3159 i
, cmd
.flow_attr
.size
);
3163 flow_id
= ib_create_flow(qp
, flow_attr
, IB_FLOW_DOMAIN_USER
);
3164 if (IS_ERR(flow_id
)) {
3165 err
= PTR_ERR(flow_id
);
3169 flow_id
->uobject
= uobj
;
3170 uobj
->object
= flow_id
;
3172 err
= idr_add_uobj(&ib_uverbs_rule_idr
, uobj
);
3176 memset(&resp
, 0, sizeof(resp
));
3177 resp
.flow_handle
= uobj
->id
;
3179 err
= ib_copy_to_udata(ucore
,
3180 &resp
, sizeof(resp
));
3185 mutex_lock(&file
->mutex
);
3186 list_add_tail(&uobj
->list
, &file
->ucontext
->rule_list
);
3187 mutex_unlock(&file
->mutex
);
3191 up_write(&uobj
->mutex
);
3193 if (cmd
.flow_attr
.num_of_specs
)
3194 kfree(kern_flow_attr
);
3197 idr_remove_uobj(&ib_uverbs_rule_idr
, uobj
);
3199 ib_destroy_flow(flow_id
);
3205 put_uobj_write(uobj
);
3207 if (cmd
.flow_attr
.num_of_specs
)
3208 kfree(kern_flow_attr
);
3212 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file
*file
,
3213 struct ib_device
*ib_dev
,
3214 struct ib_udata
*ucore
,
3215 struct ib_udata
*uhw
)
3217 struct ib_uverbs_destroy_flow cmd
;
3218 struct ib_flow
*flow_id
;
3219 struct ib_uobject
*uobj
;
3222 if (ucore
->inlen
< sizeof(cmd
))
3225 ret
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3232 uobj
= idr_write_uobj(&ib_uverbs_rule_idr
, cmd
.flow_handle
,
3236 flow_id
= uobj
->object
;
3238 ret
= ib_destroy_flow(flow_id
);
3242 put_uobj_write(uobj
);
3244 idr_remove_uobj(&ib_uverbs_rule_idr
, uobj
);
3246 mutex_lock(&file
->mutex
);
3247 list_del(&uobj
->list
);
3248 mutex_unlock(&file
->mutex
);
3255 static int __uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3256 struct ib_device
*ib_dev
,
3257 struct ib_uverbs_create_xsrq
*cmd
,
3258 struct ib_udata
*udata
)
3260 struct ib_uverbs_create_srq_resp resp
;
3261 struct ib_usrq_object
*obj
;
3264 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
3265 struct ib_srq_init_attr attr
;
3268 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
3272 init_uobj(&obj
->uevent
.uobject
, cmd
->user_handle
, file
->ucontext
, &srq_lock_class
);
3273 down_write(&obj
->uevent
.uobject
.mutex
);
3275 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3276 attr
.ext
.xrc
.xrcd
= idr_read_xrcd(cmd
->xrcd_handle
, file
->ucontext
, &xrcd_uobj
);
3277 if (!attr
.ext
.xrc
.xrcd
) {
3282 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
3283 atomic_inc(&obj
->uxrcd
->refcnt
);
3285 attr
.ext
.xrc
.cq
= idr_read_cq(cmd
->cq_handle
, file
->ucontext
, 0);
3286 if (!attr
.ext
.xrc
.cq
) {
3292 pd
= idr_read_pd(cmd
->pd_handle
, file
->ucontext
);
3298 attr
.event_handler
= ib_uverbs_srq_event_handler
;
3299 attr
.srq_context
= file
;
3300 attr
.srq_type
= cmd
->srq_type
;
3301 attr
.attr
.max_wr
= cmd
->max_wr
;
3302 attr
.attr
.max_sge
= cmd
->max_sge
;
3303 attr
.attr
.srq_limit
= cmd
->srq_limit
;
3305 obj
->uevent
.events_reported
= 0;
3306 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
3308 srq
= pd
->device
->create_srq(pd
, &attr
, udata
);
3314 srq
->device
= pd
->device
;
3316 srq
->srq_type
= cmd
->srq_type
;
3317 srq
->uobject
= &obj
->uevent
.uobject
;
3318 srq
->event_handler
= attr
.event_handler
;
3319 srq
->srq_context
= attr
.srq_context
;
3321 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3322 srq
->ext
.xrc
.cq
= attr
.ext
.xrc
.cq
;
3323 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
3324 atomic_inc(&attr
.ext
.xrc
.cq
->usecnt
);
3325 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
3328 atomic_inc(&pd
->usecnt
);
3329 atomic_set(&srq
->usecnt
, 0);
3331 obj
->uevent
.uobject
.object
= srq
;
3332 ret
= idr_add_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
3336 memset(&resp
, 0, sizeof resp
);
3337 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
3338 resp
.max_wr
= attr
.attr
.max_wr
;
3339 resp
.max_sge
= attr
.attr
.max_sge
;
3340 if (cmd
->srq_type
== IB_SRQT_XRC
)
3341 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
3343 if (copy_to_user((void __user
*) (unsigned long) cmd
->response
,
3344 &resp
, sizeof resp
)) {
3349 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3350 put_uobj_read(xrcd_uobj
);
3351 put_cq_read(attr
.ext
.xrc
.cq
);
3355 mutex_lock(&file
->mutex
);
3356 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->srq_list
);
3357 mutex_unlock(&file
->mutex
);
3359 obj
->uevent
.uobject
.live
= 1;
3361 up_write(&obj
->uevent
.uobject
.mutex
);
3366 idr_remove_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
3369 ib_destroy_srq(srq
);
3375 if (cmd
->srq_type
== IB_SRQT_XRC
)
3376 put_cq_read(attr
.ext
.xrc
.cq
);
3379 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3380 atomic_dec(&obj
->uxrcd
->refcnt
);
3381 put_uobj_read(xrcd_uobj
);
3385 put_uobj_write(&obj
->uevent
.uobject
);
3389 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
3390 struct ib_device
*ib_dev
,
3391 const char __user
*buf
, int in_len
,
3394 struct ib_uverbs_create_srq cmd
;
3395 struct ib_uverbs_create_xsrq xcmd
;
3396 struct ib_uverbs_create_srq_resp resp
;
3397 struct ib_udata udata
;
3400 if (out_len
< sizeof resp
)
3403 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3406 xcmd
.response
= cmd
.response
;
3407 xcmd
.user_handle
= cmd
.user_handle
;
3408 xcmd
.srq_type
= IB_SRQT_BASIC
;
3409 xcmd
.pd_handle
= cmd
.pd_handle
;
3410 xcmd
.max_wr
= cmd
.max_wr
;
3411 xcmd
.max_sge
= cmd
.max_sge
;
3412 xcmd
.srq_limit
= cmd
.srq_limit
;
3414 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3415 (unsigned long) cmd
.response
+ sizeof resp
,
3416 in_len
- sizeof cmd
, out_len
- sizeof resp
);
3418 ret
= __uverbs_create_xsrq(file
, ib_dev
, &xcmd
, &udata
);
3425 ssize_t
ib_uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3426 struct ib_device
*ib_dev
,
3427 const char __user
*buf
, int in_len
, int out_len
)
3429 struct ib_uverbs_create_xsrq cmd
;
3430 struct ib_uverbs_create_srq_resp resp
;
3431 struct ib_udata udata
;
3434 if (out_len
< sizeof resp
)
3437 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3440 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3441 (unsigned long) cmd
.response
+ sizeof resp
,
3442 in_len
- sizeof cmd
, out_len
- sizeof resp
);
3444 ret
= __uverbs_create_xsrq(file
, ib_dev
, &cmd
, &udata
);
3451 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
3452 struct ib_device
*ib_dev
,
3453 const char __user
*buf
, int in_len
,
3456 struct ib_uverbs_modify_srq cmd
;
3457 struct ib_udata udata
;
3459 struct ib_srq_attr attr
;
3462 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3465 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
3468 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
3472 attr
.max_wr
= cmd
.max_wr
;
3473 attr
.srq_limit
= cmd
.srq_limit
;
3475 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
3479 return ret
? ret
: in_len
;
3482 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
3483 struct ib_device
*ib_dev
,
3484 const char __user
*buf
,
3485 int in_len
, int out_len
)
3487 struct ib_uverbs_query_srq cmd
;
3488 struct ib_uverbs_query_srq_resp resp
;
3489 struct ib_srq_attr attr
;
3493 if (out_len
< sizeof resp
)
3496 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3499 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
3503 ret
= ib_query_srq(srq
, &attr
);
3510 memset(&resp
, 0, sizeof resp
);
3512 resp
.max_wr
= attr
.max_wr
;
3513 resp
.max_sge
= attr
.max_sge
;
3514 resp
.srq_limit
= attr
.srq_limit
;
3516 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
3517 &resp
, sizeof resp
))
3523 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
3524 struct ib_device
*ib_dev
,
3525 const char __user
*buf
, int in_len
,
3528 struct ib_uverbs_destroy_srq cmd
;
3529 struct ib_uverbs_destroy_srq_resp resp
;
3530 struct ib_uobject
*uobj
;
3532 struct ib_uevent_object
*obj
;
3534 struct ib_usrq_object
*us
;
3535 enum ib_srq_type srq_type
;
3537 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3540 uobj
= idr_write_uobj(&ib_uverbs_srq_idr
, cmd
.srq_handle
, file
->ucontext
);
3544 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
3545 srq_type
= srq
->srq_type
;
3547 ret
= ib_destroy_srq(srq
);
3551 put_uobj_write(uobj
);
3556 if (srq_type
== IB_SRQT_XRC
) {
3557 us
= container_of(obj
, struct ib_usrq_object
, uevent
);
3558 atomic_dec(&us
->uxrcd
->refcnt
);
3561 idr_remove_uobj(&ib_uverbs_srq_idr
, uobj
);
3563 mutex_lock(&file
->mutex
);
3564 list_del(&uobj
->list
);
3565 mutex_unlock(&file
->mutex
);
3567 ib_uverbs_release_uevent(file
, obj
);
3569 memset(&resp
, 0, sizeof resp
);
3570 resp
.events_reported
= obj
->events_reported
;
3574 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
3575 &resp
, sizeof resp
))
3578 return ret
? ret
: in_len
;
3581 int ib_uverbs_ex_query_device(struct ib_uverbs_file
*file
,
3582 struct ib_device
*ib_dev
,
3583 struct ib_udata
*ucore
,
3584 struct ib_udata
*uhw
)
3586 struct ib_uverbs_ex_query_device_resp resp
;
3587 struct ib_uverbs_ex_query_device cmd
;
3588 struct ib_device_attr attr
;
3591 if (ucore
->inlen
< sizeof(cmd
))
3594 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3604 resp
.response_length
= offsetof(typeof(resp
), odp_caps
);
3606 if (ucore
->outlen
< resp
.response_length
)
3609 memset(&attr
, 0, sizeof(attr
));
3611 err
= ib_dev
->query_device(ib_dev
, &attr
, uhw
);
3615 copy_query_dev_fields(file
, ib_dev
, &resp
.base
, &attr
);
3618 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.odp_caps
))
3621 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3622 resp
.odp_caps
.general_caps
= attr
.odp_caps
.general_caps
;
3623 resp
.odp_caps
.per_transport_caps
.rc_odp_caps
=
3624 attr
.odp_caps
.per_transport_caps
.rc_odp_caps
;
3625 resp
.odp_caps
.per_transport_caps
.uc_odp_caps
=
3626 attr
.odp_caps
.per_transport_caps
.uc_odp_caps
;
3627 resp
.odp_caps
.per_transport_caps
.ud_odp_caps
=
3628 attr
.odp_caps
.per_transport_caps
.ud_odp_caps
;
3629 resp
.odp_caps
.reserved
= 0;
3631 memset(&resp
.odp_caps
, 0, sizeof(resp
.odp_caps
));
3633 resp
.response_length
+= sizeof(resp
.odp_caps
);
3635 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.timestamp_mask
))
3638 resp
.timestamp_mask
= attr
.timestamp_mask
;
3639 resp
.response_length
+= sizeof(resp
.timestamp_mask
);
3641 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.hca_core_clock
))
3644 resp
.hca_core_clock
= attr
.hca_core_clock
;
3645 resp
.response_length
+= sizeof(resp
.hca_core_clock
);
3648 err
= ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);