]>
Commit | Line | Data |
---|---|---|
bc38a6ab RD |
1 | /* |
2 | * Copyright (c) 2005 Topspin Communications. All rights reserved. | |
33b9b3ee | 3 | * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. |
2a1d9b7f RD |
4 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
5 | * Copyright (c) 2005 Voltaire, Inc. All rights reserved. | |
67cdb40c | 6 | * Copyright (c) 2005 PathScale, Inc. All rights reserved. |
bc38a6ab RD |
7 | * |
8 | * This software is available to you under a choice of one of two | |
9 | * licenses. You may choose to be licensed under the terms of the GNU | |
10 | * General Public License (GPL) Version 2, available from the file | |
11 | * COPYING in the main directory of this source tree, or the | |
12 | * OpenIB.org BSD license below: | |
13 | * | |
14 | * Redistribution and use in source and binary forms, with or | |
15 | * without modification, are permitted provided that the following | |
16 | * conditions are met: | |
17 | * | |
18 | * - Redistributions of source code must retain the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer. | |
21 | * | |
22 | * - Redistributions in binary form must reproduce the above | |
23 | * copyright notice, this list of conditions and the following | |
24 | * disclaimer in the documentation and/or other materials | |
25 | * provided with the distribution. | |
26 | * | |
27 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
28 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
29 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
30 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
31 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
32 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
33 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
34 | * SOFTWARE. | |
bc38a6ab RD |
35 | */ |
36 | ||
37 | #include <linux/module.h> | |
38 | #include <linux/init.h> | |
39 | #include <linux/device.h> | |
40 | #include <linux/err.h> | |
41 | #include <linux/fs.h> | |
42 | #include <linux/poll.h> | |
a99bbaf5 | 43 | #include <linux/sched.h> |
bc38a6ab | 44 | #include <linux/file.h> |
70a30e16 | 45 | #include <linux/cdev.h> |
a265e558 | 46 | #include <linux/anon_inodes.h> |
5a0e3ad6 | 47 | #include <linux/slab.h> |
bc38a6ab | 48 | |
7c0f6ba6 | 49 | #include <linux/uaccess.h> |
bc38a6ab | 50 | |
e6bd18f5 | 51 | #include <rdma/ib.h> |
52427112 | 52 | #include <rdma/uverbs_std_types.h> |
e6bd18f5 | 53 | |
bc38a6ab | 54 | #include "uverbs.h" |
43579b5f | 55 | #include "core_priv.h" |
fd3c7904 | 56 | #include "rdma_core.h" |
bc38a6ab RD |
57 | |
58 | MODULE_AUTHOR("Roland Dreier"); | |
59 | MODULE_DESCRIPTION("InfiniBand userspace verbs access"); | |
60 | MODULE_LICENSE("Dual BSD/GPL"); | |
61 | ||
bc38a6ab RD |
62 | enum { |
63 | IB_UVERBS_MAJOR = 231, | |
64 | IB_UVERBS_BASE_MINOR = 192, | |
65 | IB_UVERBS_MAX_DEVICES = 32 | |
66 | }; | |
67 | ||
68 | #define IB_UVERBS_BASE_DEV MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR) | |
69 | ||
70a30e16 RD |
70 | static struct class *uverbs_class; |
71 | ||
6276e08a | 72 | static DEFINE_SPINLOCK(map_lock); |
bc38a6ab RD |
73 | static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES); |
74 | ||
75 | static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file, | |
057aec0d | 76 | struct ib_device *ib_dev, |
bc38a6ab RD |
77 | const char __user *buf, int in_len, |
78 | int out_len) = { | |
9afed76d AC |
79 | [IB_USER_VERBS_CMD_GET_CONTEXT] = ib_uverbs_get_context, |
80 | [IB_USER_VERBS_CMD_QUERY_DEVICE] = ib_uverbs_query_device, | |
81 | [IB_USER_VERBS_CMD_QUERY_PORT] = ib_uverbs_query_port, | |
82 | [IB_USER_VERBS_CMD_ALLOC_PD] = ib_uverbs_alloc_pd, | |
83 | [IB_USER_VERBS_CMD_DEALLOC_PD] = ib_uverbs_dealloc_pd, | |
84 | [IB_USER_VERBS_CMD_REG_MR] = ib_uverbs_reg_mr, | |
7e6edb9b | 85 | [IB_USER_VERBS_CMD_REREG_MR] = ib_uverbs_rereg_mr, |
9afed76d | 86 | [IB_USER_VERBS_CMD_DEREG_MR] = ib_uverbs_dereg_mr, |
6b52a12b SM |
87 | [IB_USER_VERBS_CMD_ALLOC_MW] = ib_uverbs_alloc_mw, |
88 | [IB_USER_VERBS_CMD_DEALLOC_MW] = ib_uverbs_dealloc_mw, | |
6b73597e | 89 | [IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel, |
9afed76d AC |
90 | [IB_USER_VERBS_CMD_CREATE_CQ] = ib_uverbs_create_cq, |
91 | [IB_USER_VERBS_CMD_RESIZE_CQ] = ib_uverbs_resize_cq, | |
92 | [IB_USER_VERBS_CMD_POLL_CQ] = ib_uverbs_poll_cq, | |
93 | [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ] = ib_uverbs_req_notify_cq, | |
94 | [IB_USER_VERBS_CMD_DESTROY_CQ] = ib_uverbs_destroy_cq, | |
95 | [IB_USER_VERBS_CMD_CREATE_QP] = ib_uverbs_create_qp, | |
96 | [IB_USER_VERBS_CMD_QUERY_QP] = ib_uverbs_query_qp, | |
97 | [IB_USER_VERBS_CMD_MODIFY_QP] = ib_uverbs_modify_qp, | |
98 | [IB_USER_VERBS_CMD_DESTROY_QP] = ib_uverbs_destroy_qp, | |
99 | [IB_USER_VERBS_CMD_POST_SEND] = ib_uverbs_post_send, | |
100 | [IB_USER_VERBS_CMD_POST_RECV] = ib_uverbs_post_recv, | |
101 | [IB_USER_VERBS_CMD_POST_SRQ_RECV] = ib_uverbs_post_srq_recv, | |
102 | [IB_USER_VERBS_CMD_CREATE_AH] = ib_uverbs_create_ah, | |
103 | [IB_USER_VERBS_CMD_DESTROY_AH] = ib_uverbs_destroy_ah, | |
104 | [IB_USER_VERBS_CMD_ATTACH_MCAST] = ib_uverbs_attach_mcast, | |
105 | [IB_USER_VERBS_CMD_DETACH_MCAST] = ib_uverbs_detach_mcast, | |
106 | [IB_USER_VERBS_CMD_CREATE_SRQ] = ib_uverbs_create_srq, | |
107 | [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq, | |
108 | [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq, | |
109 | [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq, | |
53d0bd1e SH |
110 | [IB_USER_VERBS_CMD_OPEN_XRCD] = ib_uverbs_open_xrcd, |
111 | [IB_USER_VERBS_CMD_CLOSE_XRCD] = ib_uverbs_close_xrcd, | |
42849b26 | 112 | [IB_USER_VERBS_CMD_CREATE_XSRQ] = ib_uverbs_create_xsrq, |
436f2ad0 | 113 | [IB_USER_VERBS_CMD_OPEN_QP] = ib_uverbs_open_qp, |
f21519b2 YD |
114 | }; |
115 | ||
f21519b2 | 116 | static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file, |
057aec0d | 117 | struct ib_device *ib_dev, |
f21519b2 YD |
118 | struct ib_udata *ucore, |
119 | struct ib_udata *uhw) = { | |
120 | [IB_USER_VERBS_EX_CMD_CREATE_FLOW] = ib_uverbs_ex_create_flow, | |
5a77abf9 | 121 | [IB_USER_VERBS_EX_CMD_DESTROY_FLOW] = ib_uverbs_ex_destroy_flow, |
02d1aa7a | 122 | [IB_USER_VERBS_EX_CMD_QUERY_DEVICE] = ib_uverbs_ex_query_device, |
565197dd | 123 | [IB_USER_VERBS_EX_CMD_CREATE_CQ] = ib_uverbs_ex_create_cq, |
6d8a7497 | 124 | [IB_USER_VERBS_EX_CMD_CREATE_QP] = ib_uverbs_ex_create_qp, |
f213c052 YH |
125 | [IB_USER_VERBS_EX_CMD_CREATE_WQ] = ib_uverbs_ex_create_wq, |
126 | [IB_USER_VERBS_EX_CMD_MODIFY_WQ] = ib_uverbs_ex_modify_wq, | |
127 | [IB_USER_VERBS_EX_CMD_DESTROY_WQ] = ib_uverbs_ex_destroy_wq, | |
de019a94 YH |
128 | [IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL] = ib_uverbs_ex_create_rwq_ind_table, |
129 | [IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL] = ib_uverbs_ex_destroy_rwq_ind_table, | |
189aba99 | 130 | [IB_USER_VERBS_EX_CMD_MODIFY_QP] = ib_uverbs_ex_modify_qp, |
869ddcf8 | 131 | [IB_USER_VERBS_EX_CMD_MODIFY_CQ] = ib_uverbs_ex_modify_cq, |
bc38a6ab RD |
132 | }; |
133 | ||
bc38a6ab | 134 | static void ib_uverbs_add_one(struct ib_device *device); |
7c1eb45a | 135 | static void ib_uverbs_remove_one(struct ib_device *device, void *client_data); |
bc38a6ab | 136 | |
feb7c1e3 CH |
137 | int uverbs_dealloc_mw(struct ib_mw *mw) |
138 | { | |
139 | struct ib_pd *pd = mw->pd; | |
140 | int ret; | |
141 | ||
142 | ret = mw->device->dealloc_mw(mw); | |
143 | if (!ret) | |
144 | atomic_dec(&pd->usecnt); | |
145 | return ret; | |
146 | } | |
147 | ||
35d4a0b6 | 148 | static void ib_uverbs_release_dev(struct kobject *kobj) |
70a30e16 RD |
149 | { |
150 | struct ib_uverbs_device *dev = | |
35d4a0b6 | 151 | container_of(kobj, struct ib_uverbs_device, kobj); |
70a30e16 | 152 | |
036b1063 | 153 | cleanup_srcu_struct(&dev->disassociate_srcu); |
35d4a0b6 | 154 | kfree(dev); |
70a30e16 RD |
155 | } |
156 | ||
35d4a0b6 YH |
157 | static struct kobj_type ib_uverbs_dev_ktype = { |
158 | .release = ib_uverbs_release_dev, | |
159 | }; | |
160 | ||
1e7710f3 | 161 | static void ib_uverbs_release_async_event_file(struct kref *ref) |
04d29b0e | 162 | { |
1e7710f3 MB |
163 | struct ib_uverbs_async_event_file *file = |
164 | container_of(ref, struct ib_uverbs_async_event_file, ref); | |
04d29b0e RD |
165 | |
166 | kfree(file); | |
167 | } | |
168 | ||
70a30e16 | 169 | void ib_uverbs_release_ucq(struct ib_uverbs_file *file, |
1e7710f3 | 170 | struct ib_uverbs_completion_event_file *ev_file, |
70a30e16 RD |
171 | struct ib_ucq_object *uobj) |
172 | { | |
173 | struct ib_uverbs_event *evt, *tmp; | |
174 | ||
175 | if (ev_file) { | |
db1b5ddd | 176 | spin_lock_irq(&ev_file->ev_queue.lock); |
70a30e16 RD |
177 | list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) { |
178 | list_del(&evt->list); | |
179 | kfree(evt); | |
180 | } | |
db1b5ddd | 181 | spin_unlock_irq(&ev_file->ev_queue.lock); |
70a30e16 | 182 | |
1e7710f3 | 183 | uverbs_uobject_put(&ev_file->uobj_file.uobj); |
70a30e16 RD |
184 | } |
185 | ||
db1b5ddd | 186 | spin_lock_irq(&file->async_file->ev_queue.lock); |
70a30e16 RD |
187 | list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) { |
188 | list_del(&evt->list); | |
189 | kfree(evt); | |
190 | } | |
db1b5ddd | 191 | spin_unlock_irq(&file->async_file->ev_queue.lock); |
70a30e16 RD |
192 | } |
193 | ||
194 | void ib_uverbs_release_uevent(struct ib_uverbs_file *file, | |
195 | struct ib_uevent_object *uobj) | |
196 | { | |
197 | struct ib_uverbs_event *evt, *tmp; | |
198 | ||
db1b5ddd | 199 | spin_lock_irq(&file->async_file->ev_queue.lock); |
70a30e16 RD |
200 | list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { |
201 | list_del(&evt->list); | |
202 | kfree(evt); | |
203 | } | |
db1b5ddd | 204 | spin_unlock_irq(&file->async_file->ev_queue.lock); |
70a30e16 RD |
205 | } |
206 | ||
6be60aed MB |
207 | void ib_uverbs_detach_umcast(struct ib_qp *qp, |
208 | struct ib_uqp_object *uobj) | |
f4e40156 JM |
209 | { |
210 | struct ib_uverbs_mcast_entry *mcast, *tmp; | |
211 | ||
212 | list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) { | |
213 | ib_detach_mcast(qp, &mcast->gid, mcast->lid); | |
214 | list_del(&mcast->list); | |
215 | kfree(mcast); | |
216 | } | |
217 | } | |
218 | ||
70a30e16 | 219 | static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, |
fd3c7904 MB |
220 | struct ib_ucontext *context, |
221 | bool device_removed) | |
bc38a6ab | 222 | { |
f7c6a7b5 | 223 | context->closing = 1; |
fd3c7904 | 224 | uverbs_cleanup_ucontext(context, device_removed); |
8ada2c1c SR |
225 | put_pid(context->tgid); |
226 | ||
43579b5f PP |
227 | ib_rdmacg_uncharge(&context->cg_obj, context->device, |
228 | RDMACG_RESOURCE_HCA_HANDLE); | |
229 | ||
bc38a6ab RD |
230 | return context->device->dealloc_ucontext(context); |
231 | } | |
232 | ||
35d4a0b6 YH |
233 | static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) |
234 | { | |
235 | complete(&dev->comp); | |
236 | } | |
237 | ||
cf8966b3 | 238 | void ib_uverbs_release_file(struct kref *ref) |
bc38a6ab RD |
239 | { |
240 | struct ib_uverbs_file *file = | |
241 | container_of(ref, struct ib_uverbs_file, ref); | |
036b1063 YH |
242 | struct ib_device *ib_dev; |
243 | int srcu_key; | |
244 | ||
245 | srcu_key = srcu_read_lock(&file->device->disassociate_srcu); | |
246 | ib_dev = srcu_dereference(file->device->ib_dev, | |
247 | &file->device->disassociate_srcu); | |
248 | if (ib_dev && !ib_dev->disassociate_ucontext) | |
249 | module_put(ib_dev->owner); | |
250 | srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); | |
bc38a6ab | 251 | |
35d4a0b6 YH |
252 | if (atomic_dec_and_test(&file->device->refcount)) |
253 | ib_uverbs_comp_dev(file->device); | |
70a30e16 | 254 | |
12082bbd YH |
255 | if (file->async_file) |
256 | kref_put(&file->async_file->ref, | |
257 | ib_uverbs_release_async_event_file); | |
efdd6f53 | 258 | kobject_put(&file->device->kobj); |
bc38a6ab RD |
259 | kfree(file); |
260 | } | |
261 | ||
db1b5ddd | 262 | static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, |
1e7710f3 MB |
263 | struct file *filp, char __user *buf, |
264 | size_t count, loff_t *pos, | |
e0fcc611 | 265 | size_t eventsz) |
bc38a6ab | 266 | { |
63aaf647 | 267 | struct ib_uverbs_event *event; |
bc38a6ab RD |
268 | int ret = 0; |
269 | ||
db1b5ddd | 270 | spin_lock_irq(&ev_queue->lock); |
bc38a6ab | 271 | |
db1b5ddd MB |
272 | while (list_empty(&ev_queue->event_list)) { |
273 | spin_unlock_irq(&ev_queue->lock); | |
bc38a6ab RD |
274 | |
275 | if (filp->f_flags & O_NONBLOCK) | |
276 | return -EAGAIN; | |
277 | ||
db1b5ddd MB |
278 | if (wait_event_interruptible(ev_queue->poll_wait, |
279 | (!list_empty(&ev_queue->event_list) || | |
bd0b0a07 | 280 | ev_queue->is_closed))) |
bc38a6ab RD |
281 | return -ERESTARTSYS; |
282 | ||
bd0b0a07 JG |
283 | spin_lock_irq(&ev_queue->lock); |
284 | ||
036b1063 | 285 | /* If device was disassociated and no event exists set an error */ |
bd0b0a07 JG |
286 | if (list_empty(&ev_queue->event_list) && ev_queue->is_closed) { |
287 | spin_unlock_irq(&ev_queue->lock); | |
036b1063 | 288 | return -EIO; |
bd0b0a07 | 289 | } |
bc38a6ab RD |
290 | } |
291 | ||
db1b5ddd | 292 | event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list); |
63aaf647 | 293 | |
bc38a6ab RD |
294 | if (eventsz > count) { |
295 | ret = -EINVAL; | |
296 | event = NULL; | |
63aaf647 | 297 | } else { |
db1b5ddd | 298 | list_del(ev_queue->event_list.next); |
63aaf647 RD |
299 | if (event->counter) { |
300 | ++(*event->counter); | |
301 | list_del(&event->obj_list); | |
302 | } | |
303 | } | |
bc38a6ab | 304 | |
db1b5ddd | 305 | spin_unlock_irq(&ev_queue->lock); |
bc38a6ab RD |
306 | |
307 | if (event) { | |
308 | if (copy_to_user(buf, event, eventsz)) | |
309 | ret = -EFAULT; | |
310 | else | |
311 | ret = eventsz; | |
312 | } | |
313 | ||
314 | kfree(event); | |
315 | ||
316 | return ret; | |
317 | } | |
318 | ||
1e7710f3 MB |
319 | static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf, |
320 | size_t count, loff_t *pos) | |
321 | { | |
322 | struct ib_uverbs_async_event_file *file = filp->private_data; | |
323 | ||
bd0b0a07 | 324 | return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos, |
e0fcc611 | 325 | sizeof(struct ib_uverbs_async_event_desc)); |
1e7710f3 MB |
326 | } |
327 | ||
328 | static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf, | |
329 | size_t count, loff_t *pos) | |
330 | { | |
331 | struct ib_uverbs_completion_event_file *comp_ev_file = | |
332 | filp->private_data; | |
333 | ||
bd0b0a07 JG |
334 | return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count, |
335 | pos, | |
e0fcc611 | 336 | sizeof(struct ib_uverbs_comp_event_desc)); |
1e7710f3 MB |
337 | } |
338 | ||
db1b5ddd | 339 | static unsigned int ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue, |
1e7710f3 | 340 | struct file *filp, |
bc38a6ab RD |
341 | struct poll_table_struct *wait) |
342 | { | |
343 | unsigned int pollflags = 0; | |
bc38a6ab | 344 | |
db1b5ddd | 345 | poll_wait(filp, &ev_queue->poll_wait, wait); |
bc38a6ab | 346 | |
db1b5ddd MB |
347 | spin_lock_irq(&ev_queue->lock); |
348 | if (!list_empty(&ev_queue->event_list)) | |
bc38a6ab | 349 | pollflags = POLLIN | POLLRDNORM; |
db1b5ddd | 350 | spin_unlock_irq(&ev_queue->lock); |
bc38a6ab RD |
351 | |
352 | return pollflags; | |
353 | } | |
354 | ||
1e7710f3 MB |
355 | static unsigned int ib_uverbs_async_event_poll(struct file *filp, |
356 | struct poll_table_struct *wait) | |
357 | { | |
bd0b0a07 JG |
358 | struct ib_uverbs_async_event_file *file = filp->private_data; |
359 | ||
360 | return ib_uverbs_event_poll(&file->ev_queue, filp, wait); | |
1e7710f3 MB |
361 | } |
362 | ||
363 | static unsigned int ib_uverbs_comp_event_poll(struct file *filp, | |
364 | struct poll_table_struct *wait) | |
365 | { | |
366 | struct ib_uverbs_completion_event_file *comp_ev_file = | |
367 | filp->private_data; | |
368 | ||
db1b5ddd | 369 | return ib_uverbs_event_poll(&comp_ev_file->ev_queue, filp, wait); |
1e7710f3 MB |
370 | } |
371 | ||
372 | static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on) | |
abdf119b | 373 | { |
bd0b0a07 | 374 | struct ib_uverbs_async_event_file *file = filp->private_data; |
abdf119b | 375 | |
bd0b0a07 | 376 | return fasync_helper(fd, filp, on, &file->ev_queue.async_queue); |
abdf119b GN |
377 | } |
378 | ||
1e7710f3 | 379 | static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) |
bc38a6ab | 380 | { |
1e7710f3 MB |
381 | struct ib_uverbs_completion_event_file *comp_ev_file = |
382 | filp->private_data; | |
383 | ||
db1b5ddd | 384 | return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue); |
1e7710f3 MB |
385 | } |
386 | ||
387 | static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp) | |
388 | { | |
389 | struct ib_uverbs_async_event_file *file = filp->private_data; | |
390 | struct ib_uverbs_file *uverbs_file = file->uverbs_file; | |
6b73597e | 391 | struct ib_uverbs_event *entry, *tmp; |
036b1063 | 392 | int closed_already = 0; |
6b73597e | 393 | |
1e7710f3 | 394 | mutex_lock(&uverbs_file->device->lists_mutex); |
db1b5ddd MB |
395 | spin_lock_irq(&file->ev_queue.lock); |
396 | closed_already = file->ev_queue.is_closed; | |
397 | file->ev_queue.is_closed = 1; | |
398 | list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { | |
6b73597e RD |
399 | if (entry->counter) |
400 | list_del(&entry->obj_list); | |
401 | kfree(entry); | |
402 | } | |
db1b5ddd | 403 | spin_unlock_irq(&file->ev_queue.lock); |
036b1063 YH |
404 | if (!closed_already) { |
405 | list_del(&file->list); | |
1e7710f3 MB |
406 | ib_unregister_event_handler(&uverbs_file->event_handler); |
407 | } | |
408 | mutex_unlock(&uverbs_file->device->lists_mutex); | |
409 | ||
410 | kref_put(&uverbs_file->ref, ib_uverbs_release_file); | |
411 | kref_put(&file->ref, ib_uverbs_release_async_event_file); | |
412 | ||
413 | return 0; | |
414 | } | |
415 | ||
416 | static int ib_uverbs_comp_event_close(struct inode *inode, struct file *filp) | |
417 | { | |
418 | struct ib_uverbs_completion_event_file *file = filp->private_data; | |
419 | struct ib_uverbs_event *entry, *tmp; | |
420 | ||
db1b5ddd MB |
421 | spin_lock_irq(&file->ev_queue.lock); |
422 | list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { | |
1e7710f3 MB |
423 | if (entry->counter) |
424 | list_del(&entry->obj_list); | |
425 | kfree(entry); | |
036b1063 | 426 | } |
db74a0e1 | 427 | file->ev_queue.is_closed = 1; |
db1b5ddd | 428 | spin_unlock_irq(&file->ev_queue.lock); |
bc38a6ab | 429 | |
1e7710f3 | 430 | uverbs_close_fd(filp); |
bc38a6ab RD |
431 | |
432 | return 0; | |
433 | } | |
434 | ||
1e7710f3 | 435 | const struct file_operations uverbs_event_fops = { |
6b73597e | 436 | .owner = THIS_MODULE, |
1e7710f3 MB |
437 | .read = ib_uverbs_comp_event_read, |
438 | .poll = ib_uverbs_comp_event_poll, | |
439 | .release = ib_uverbs_comp_event_close, | |
440 | .fasync = ib_uverbs_comp_event_fasync, | |
441 | .llseek = no_llseek, | |
442 | }; | |
443 | ||
444 | static const struct file_operations uverbs_async_event_fops = { | |
445 | .owner = THIS_MODULE, | |
446 | .read = ib_uverbs_async_event_read, | |
447 | .poll = ib_uverbs_async_event_poll, | |
448 | .release = ib_uverbs_async_event_close, | |
449 | .fasync = ib_uverbs_async_event_fasync, | |
bc1db9af | 450 | .llseek = no_llseek, |
bc38a6ab RD |
451 | }; |
452 | ||
453 | void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) | |
454 | { | |
db1b5ddd | 455 | struct ib_uverbs_event_queue *ev_queue = cq_context; |
6b73597e RD |
456 | struct ib_ucq_object *uobj; |
457 | struct ib_uverbs_event *entry; | |
458 | unsigned long flags; | |
459 | ||
db1b5ddd | 460 | if (!ev_queue) |
6b73597e RD |
461 | return; |
462 | ||
db1b5ddd MB |
463 | spin_lock_irqsave(&ev_queue->lock, flags); |
464 | if (ev_queue->is_closed) { | |
465 | spin_unlock_irqrestore(&ev_queue->lock, flags); | |
6b73597e RD |
466 | return; |
467 | } | |
bc38a6ab RD |
468 | |
469 | entry = kmalloc(sizeof *entry, GFP_ATOMIC); | |
305a7e87 | 470 | if (!entry) { |
db1b5ddd | 471 | spin_unlock_irqrestore(&ev_queue->lock, flags); |
bc38a6ab | 472 | return; |
305a7e87 | 473 | } |
bc38a6ab | 474 | |
63aaf647 RD |
475 | uobj = container_of(cq->uobject, struct ib_ucq_object, uobject); |
476 | ||
477 | entry->desc.comp.cq_handle = cq->uobject->user_handle; | |
478 | entry->counter = &uobj->comp_events_reported; | |
bc38a6ab | 479 | |
db1b5ddd | 480 | list_add_tail(&entry->list, &ev_queue->event_list); |
63aaf647 | 481 | list_add_tail(&entry->obj_list, &uobj->comp_list); |
db1b5ddd | 482 | spin_unlock_irqrestore(&ev_queue->lock, flags); |
bc38a6ab | 483 | |
db1b5ddd MB |
484 | wake_up_interruptible(&ev_queue->poll_wait); |
485 | kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN); | |
bc38a6ab RD |
486 | } |
487 | ||
488 | static void ib_uverbs_async_handler(struct ib_uverbs_file *file, | |
63aaf647 RD |
489 | __u64 element, __u64 event, |
490 | struct list_head *obj_list, | |
491 | u32 *counter) | |
bc38a6ab | 492 | { |
63aaf647 | 493 | struct ib_uverbs_event *entry; |
bc38a6ab RD |
494 | unsigned long flags; |
495 | ||
db1b5ddd MB |
496 | spin_lock_irqsave(&file->async_file->ev_queue.lock, flags); |
497 | if (file->async_file->ev_queue.is_closed) { | |
498 | spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); | |
6b73597e RD |
499 | return; |
500 | } | |
501 | ||
bc38a6ab | 502 | entry = kmalloc(sizeof *entry, GFP_ATOMIC); |
305a7e87 | 503 | if (!entry) { |
db1b5ddd | 504 | spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); |
bc38a6ab | 505 | return; |
305a7e87 | 506 | } |
bc38a6ab | 507 | |
63aaf647 RD |
508 | entry->desc.async.element = element; |
509 | entry->desc.async.event_type = event; | |
377b5134 | 510 | entry->desc.async.reserved = 0; |
63aaf647 | 511 | entry->counter = counter; |
bc38a6ab | 512 | |
db1b5ddd | 513 | list_add_tail(&entry->list, &file->async_file->ev_queue.event_list); |
63aaf647 RD |
514 | if (obj_list) |
515 | list_add_tail(&entry->obj_list, obj_list); | |
db1b5ddd | 516 | spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); |
bc38a6ab | 517 | |
db1b5ddd MB |
518 | wake_up_interruptible(&file->async_file->ev_queue.poll_wait); |
519 | kill_fasync(&file->async_file->ev_queue.async_queue, SIGIO, POLL_IN); | |
bc38a6ab RD |
520 | } |
521 | ||
522 | void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) | |
523 | { | |
7162a3e0 RD |
524 | struct ib_ucq_object *uobj = container_of(event->element.cq->uobject, |
525 | struct ib_ucq_object, uobject); | |
63aaf647 | 526 | |
7162a3e0 | 527 | ib_uverbs_async_handler(uobj->uverbs_file, uobj->uobject.user_handle, |
63aaf647 RD |
528 | event->event, &uobj->async_list, |
529 | &uobj->async_events_reported); | |
bc38a6ab RD |
530 | } |
531 | ||
532 | void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) | |
533 | { | |
63aaf647 RD |
534 | struct ib_uevent_object *uobj; |
535 | ||
a040f95d | 536 | /* for XRC target qp's, check that qp is live */ |
fd3c7904 | 537 | if (!event->element.qp->uobject) |
a040f95d JM |
538 | return; |
539 | ||
63aaf647 RD |
540 | uobj = container_of(event->element.qp->uobject, |
541 | struct ib_uevent_object, uobject); | |
542 | ||
543 | ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, | |
544 | event->event, &uobj->event_list, | |
545 | &uobj->events_reported); | |
bc38a6ab RD |
546 | } |
547 | ||
f213c052 YH |
548 | void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr) |
549 | { | |
550 | struct ib_uevent_object *uobj = container_of(event->element.wq->uobject, | |
551 | struct ib_uevent_object, uobject); | |
552 | ||
553 | ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, | |
554 | event->event, &uobj->event_list, | |
555 | &uobj->events_reported); | |
556 | } | |
557 | ||
f520ba5a RD |
558 | void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) |
559 | { | |
63aaf647 RD |
560 | struct ib_uevent_object *uobj; |
561 | ||
562 | uobj = container_of(event->element.srq->uobject, | |
563 | struct ib_uevent_object, uobject); | |
564 | ||
565 | ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, | |
566 | event->event, &uobj->event_list, | |
567 | &uobj->events_reported); | |
f520ba5a RD |
568 | } |
569 | ||
6b73597e RD |
570 | void ib_uverbs_event_handler(struct ib_event_handler *handler, |
571 | struct ib_event *event) | |
bc38a6ab RD |
572 | { |
573 | struct ib_uverbs_file *file = | |
574 | container_of(handler, struct ib_uverbs_file, event_handler); | |
575 | ||
63aaf647 RD |
576 | ib_uverbs_async_handler(file, event->element.port_num, event->event, |
577 | NULL, NULL); | |
bc38a6ab RD |
578 | } |
579 | ||
03c40442 YH |
580 | void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file) |
581 | { | |
1e7710f3 | 582 | kref_put(&file->async_file->ref, ib_uverbs_release_async_event_file); |
03c40442 YH |
583 | file->async_file = NULL; |
584 | } | |
585 | ||
db1b5ddd | 586 | void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) |
bc38a6ab | 587 | { |
db1b5ddd MB |
588 | spin_lock_init(&ev_queue->lock); |
589 | INIT_LIST_HEAD(&ev_queue->event_list); | |
590 | init_waitqueue_head(&ev_queue->poll_wait); | |
591 | ev_queue->is_closed = 0; | |
592 | ev_queue->async_queue = NULL; | |
1e7710f3 MB |
593 | } |
594 | ||
595 | struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, | |
596 | struct ib_device *ib_dev) | |
597 | { | |
598 | struct ib_uverbs_async_event_file *ev_file; | |
bc38a6ab RD |
599 | struct file *filp; |
600 | ||
03c40442 | 601 | ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL); |
6b73597e RD |
602 | if (!ev_file) |
603 | return ERR_PTR(-ENOMEM); | |
604 | ||
db1b5ddd | 605 | ib_uverbs_init_event_queue(&ev_file->ev_queue); |
6b73597e | 606 | ev_file->uverbs_file = uverbs_file; |
03c40442 | 607 | kref_get(&ev_file->uverbs_file->ref); |
1e7710f3 MB |
608 | kref_init(&ev_file->ref); |
609 | filp = anon_inode_getfile("[infinibandevent]", &uverbs_async_event_fops, | |
a265e558 | 610 | ev_file, O_RDONLY); |
b1e4594b | 611 | if (IS_ERR(filp)) |
03c40442 YH |
612 | goto err_put_refs; |
613 | ||
036b1063 YH |
614 | mutex_lock(&uverbs_file->device->lists_mutex); |
615 | list_add_tail(&ev_file->list, | |
616 | &uverbs_file->device->uverbs_events_file_list); | |
617 | mutex_unlock(&uverbs_file->device->lists_mutex); | |
618 | ||
1e7710f3 MB |
619 | WARN_ON(uverbs_file->async_file); |
620 | uverbs_file->async_file = ev_file; | |
621 | kref_get(&uverbs_file->async_file->ref); | |
622 | INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler, | |
623 | ib_dev, | |
624 | ib_uverbs_event_handler); | |
dcc9881e | 625 | ib_register_event_handler(&uverbs_file->event_handler); |
1e7710f3 | 626 | /* At that point async file stuff was fully set */ |
03c40442 YH |
627 | |
628 | return filp; | |
629 | ||
03c40442 YH |
630 | err_put_refs: |
631 | kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file); | |
1e7710f3 | 632 | kref_put(&ev_file->ref, ib_uverbs_release_async_event_file); |
6b73597e | 633 | return filp; |
6b73597e RD |
634 | } |
635 | ||
2dbd5186 EC |
636 | static int verify_command_mask(struct ib_device *ib_dev, __u32 command) |
637 | { | |
638 | u64 mask; | |
639 | ||
640 | if (command <= IB_USER_VERBS_CMD_OPEN_QP) | |
641 | mask = ib_dev->uverbs_cmd_mask; | |
642 | else | |
643 | mask = ib_dev->uverbs_ex_cmd_mask; | |
644 | ||
645 | if (mask & ((u64)1 << command)) | |
646 | return 0; | |
647 | ||
648 | return -1; | |
649 | } | |
650 | ||
de98b2b9 LR |
651 | static bool verify_command_idx(u32 command, bool extended) |
652 | { | |
653 | if (extended) | |
654 | return command < ARRAY_SIZE(uverbs_ex_cmd_table); | |
655 | ||
656 | return command < ARRAY_SIZE(uverbs_cmd_table); | |
657 | } | |
658 | ||
bc38a6ab RD |
659 | static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, |
660 | size_t count, loff_t *pos) | |
661 | { | |
662 | struct ib_uverbs_file *file = filp->private_data; | |
036b1063 | 663 | struct ib_device *ib_dev; |
bc38a6ab | 664 | struct ib_uverbs_cmd_hdr hdr; |
de98b2b9 | 665 | bool extended_command; |
74a0b0a5 | 666 | __u32 command; |
f21519b2 | 667 | __u32 flags; |
036b1063 YH |
668 | int srcu_key; |
669 | ssize_t ret; | |
057aec0d | 670 | |
f73a1dbc LR |
671 | if (!ib_safe_file_access(filp)) { |
672 | pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n", | |
673 | task_tgid_vnr(current), current->comm); | |
e6bd18f5 | 674 | return -EACCES; |
f73a1dbc | 675 | } |
e6bd18f5 | 676 | |
bc38a6ab RD |
677 | if (count < sizeof hdr) |
678 | return -EINVAL; | |
679 | ||
680 | if (copy_from_user(&hdr, buf, sizeof hdr)) | |
681 | return -EFAULT; | |
682 | ||
036b1063 YH |
683 | srcu_key = srcu_read_lock(&file->device->disassociate_srcu); |
684 | ib_dev = srcu_dereference(file->device->ib_dev, | |
685 | &file->device->disassociate_srcu); | |
686 | if (!ib_dev) { | |
687 | ret = -EIO; | |
688 | goto out; | |
689 | } | |
690 | ||
74a0b0a5 EC |
691 | if (hdr.command & ~(__u32)(IB_USER_VERBS_CMD_FLAGS_MASK | |
692 | IB_USER_VERBS_CMD_COMMAND_MASK)) { | |
693 | ret = -EINVAL; | |
694 | goto out; | |
695 | } | |
696 | ||
697 | command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK; | |
de98b2b9 LR |
698 | flags = (hdr.command & |
699 | IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT; | |
700 | ||
701 | extended_command = flags & IB_USER_VERBS_CMD_FLAG_EXTENDED; | |
702 | if (!verify_command_idx(command, extended_command)) { | |
703 | ret = -EINVAL; | |
704 | goto out; | |
705 | } | |
706 | ||
2dbd5186 EC |
707 | if (verify_command_mask(ib_dev, command)) { |
708 | ret = -EOPNOTSUPP; | |
709 | goto out; | |
710 | } | |
74a0b0a5 | 711 | |
eaebc7d2 EC |
712 | if (!file->ucontext && |
713 | command != IB_USER_VERBS_CMD_GET_CONTEXT) { | |
714 | ret = -EINVAL; | |
715 | goto out; | |
716 | } | |
717 | ||
f21519b2 | 718 | if (!flags) { |
de98b2b9 | 719 | if (!uverbs_cmd_table[command]) { |
036b1063 YH |
720 | ret = -EINVAL; |
721 | goto out; | |
722 | } | |
400dbc96 | 723 | |
036b1063 YH |
724 | if (hdr.in_words * 4 != count) { |
725 | ret = -EINVAL; | |
726 | goto out; | |
727 | } | |
400dbc96 | 728 | |
036b1063 | 729 | ret = uverbs_cmd_table[command](file, ib_dev, |
f21519b2 YD |
730 | buf + sizeof(hdr), |
731 | hdr.in_words * 4, | |
732 | hdr.out_words * 4); | |
733 | ||
f21519b2 | 734 | } else if (flags == IB_USER_VERBS_CMD_FLAG_EXTENDED) { |
f21519b2 YD |
735 | struct ib_uverbs_ex_cmd_hdr ex_hdr; |
736 | struct ib_udata ucore; | |
737 | struct ib_udata uhw; | |
f21519b2 YD |
738 | size_t written_count = count; |
739 | ||
de98b2b9 | 740 | if (!uverbs_ex_cmd_table[command]) { |
036b1063 YH |
741 | ret = -ENOSYS; |
742 | goto out; | |
743 | } | |
f21519b2 | 744 | |
036b1063 YH |
745 | if (!file->ucontext) { |
746 | ret = -EINVAL; | |
747 | goto out; | |
748 | } | |
f21519b2 | 749 | |
036b1063 YH |
750 | if (count < (sizeof(hdr) + sizeof(ex_hdr))) { |
751 | ret = -EINVAL; | |
752 | goto out; | |
753 | } | |
f21519b2 | 754 | |
036b1063 YH |
755 | if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr))) { |
756 | ret = -EFAULT; | |
757 | goto out; | |
758 | } | |
f21519b2 YD |
759 | |
760 | count -= sizeof(hdr) + sizeof(ex_hdr); | |
761 | buf += sizeof(hdr) + sizeof(ex_hdr); | |
762 | ||
036b1063 YH |
763 | if ((hdr.in_words + ex_hdr.provider_in_words) * 8 != count) { |
764 | ret = -EINVAL; | |
765 | goto out; | |
766 | } | |
f21519b2 | 767 | |
036b1063 YH |
768 | if (ex_hdr.cmd_hdr_reserved) { |
769 | ret = -EINVAL; | |
770 | goto out; | |
771 | } | |
7efb1b19 | 772 | |
f21519b2 | 773 | if (ex_hdr.response) { |
036b1063 YH |
774 | if (!hdr.out_words && !ex_hdr.provider_out_words) { |
775 | ret = -EINVAL; | |
776 | goto out; | |
777 | } | |
6cc3df84 YD |
778 | |
779 | if (!access_ok(VERIFY_WRITE, | |
12f72772 | 780 | u64_to_user_ptr(ex_hdr.response), |
036b1063 YH |
781 | (hdr.out_words + ex_hdr.provider_out_words) * 8)) { |
782 | ret = -EFAULT; | |
783 | goto out; | |
784 | } | |
f21519b2 | 785 | } else { |
036b1063 YH |
786 | if (hdr.out_words || ex_hdr.provider_out_words) { |
787 | ret = -EINVAL; | |
788 | goto out; | |
789 | } | |
f21519b2 YD |
790 | } |
791 | ||
12f72772 AB |
792 | ib_uverbs_init_udata_buf_or_null(&ucore, buf, |
793 | u64_to_user_ptr(ex_hdr.response), | |
794 | hdr.in_words * 8, hdr.out_words * 8); | |
a96e4e2f | 795 | |
12f72772 AB |
796 | ib_uverbs_init_udata_buf_or_null(&uhw, |
797 | buf + ucore.inlen, | |
798 | u64_to_user_ptr(ex_hdr.response) + ucore.outlen, | |
799 | ex_hdr.provider_in_words * 8, | |
800 | ex_hdr.provider_out_words * 8); | |
f21519b2 | 801 | |
12f72772 | 802 | ret = uverbs_ex_cmd_table[command](file, ib_dev, &ucore, &uhw); |
036b1063 YH |
803 | if (!ret) |
804 | ret = written_count; | |
805 | } else { | |
806 | ret = -ENOSYS; | |
400dbc96 | 807 | } |
f21519b2 | 808 | |
036b1063 YH |
809 | out: |
810 | srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); | |
811 | return ret; | |
bc38a6ab RD |
812 | } |
813 | ||
814 | static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma) | |
815 | { | |
816 | struct ib_uverbs_file *file = filp->private_data; | |
036b1063 YH |
817 | struct ib_device *ib_dev; |
818 | int ret = 0; | |
819 | int srcu_key; | |
bc38a6ab | 820 | |
036b1063 YH |
821 | srcu_key = srcu_read_lock(&file->device->disassociate_srcu); |
822 | ib_dev = srcu_dereference(file->device->ib_dev, | |
823 | &file->device->disassociate_srcu); | |
824 | if (!ib_dev) { | |
825 | ret = -EIO; | |
826 | goto out; | |
827 | } | |
828 | ||
829 | if (!file->ucontext) | |
830 | ret = -ENODEV; | |
bc38a6ab | 831 | else |
036b1063 YH |
832 | ret = ib_dev->mmap(file->ucontext, vma); |
833 | out: | |
834 | srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); | |
835 | return ret; | |
bc38a6ab RD |
836 | } |
837 | ||
5b2d281a RD |
838 | /* |
839 | * ib_uverbs_open() does not need the BKL: | |
840 | * | |
2a72f212 | 841 | * - the ib_uverbs_device structures are properly reference counted and |
5b2d281a RD |
842 | * everything else is purely local to the file being created, so |
843 | * races against other open calls are not a problem; | |
844 | * - there is no ioctl method to race against; | |
2a72f212 AC |
845 | * - the open method will either immediately run -ENXIO, or all |
846 | * required initialization will be done. | |
5b2d281a | 847 | */ |
bc38a6ab RD |
848 | static int ib_uverbs_open(struct inode *inode, struct file *filp) |
849 | { | |
70a30e16 | 850 | struct ib_uverbs_device *dev; |
bc38a6ab | 851 | struct ib_uverbs_file *file; |
036b1063 | 852 | struct ib_device *ib_dev; |
70a30e16 | 853 | int ret; |
036b1063 YH |
854 | int module_dependent; |
855 | int srcu_key; | |
bc38a6ab | 856 | |
2a72f212 | 857 | dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev); |
35d4a0b6 | 858 | if (!atomic_inc_not_zero(&dev->refcount)) |
70a30e16 RD |
859 | return -ENXIO; |
860 | ||
036b1063 YH |
861 | srcu_key = srcu_read_lock(&dev->disassociate_srcu); |
862 | mutex_lock(&dev->lists_mutex); | |
863 | ib_dev = srcu_dereference(dev->ib_dev, | |
864 | &dev->disassociate_srcu); | |
865 | if (!ib_dev) { | |
866 | ret = -EIO; | |
70a30e16 RD |
867 | goto err; |
868 | } | |
bc38a6ab | 869 | |
036b1063 YH |
870 | /* In case IB device supports disassociate ucontext, there is no hard |
871 | * dependency between uverbs device and its low level device. | |
872 | */ | |
873 | module_dependent = !(ib_dev->disassociate_ucontext); | |
874 | ||
875 | if (module_dependent) { | |
876 | if (!try_module_get(ib_dev->owner)) { | |
877 | ret = -ENODEV; | |
878 | goto err; | |
879 | } | |
880 | } | |
881 | ||
882 | file = kzalloc(sizeof(*file), GFP_KERNEL); | |
63c47c28 | 883 | if (!file) { |
70a30e16 | 884 | ret = -ENOMEM; |
036b1063 YH |
885 | if (module_dependent) |
886 | goto err_module; | |
887 | ||
888 | goto err; | |
63c47c28 | 889 | } |
bc38a6ab | 890 | |
70a30e16 | 891 | file->device = dev; |
771addf6 MB |
892 | spin_lock_init(&file->idr_lock); |
893 | idr_init(&file->idr); | |
70a30e16 RD |
894 | file->ucontext = NULL; |
895 | file->async_file = NULL; | |
bc38a6ab | 896 | kref_init(&file->ref); |
95ed644f | 897 | mutex_init(&file->mutex); |
d1e09f30 | 898 | mutex_init(&file->cleanup_mutex); |
bc38a6ab | 899 | |
bc38a6ab | 900 | filp->private_data = file; |
35d4a0b6 | 901 | kobject_get(&dev->kobj); |
036b1063 YH |
902 | list_add_tail(&file->list, &dev->uverbs_file_list); |
903 | mutex_unlock(&dev->lists_mutex); | |
904 | srcu_read_unlock(&dev->disassociate_srcu, srcu_key); | |
bc38a6ab | 905 | |
bc1db9af | 906 | return nonseekable_open(inode, filp); |
70a30e16 RD |
907 | |
908 | err_module: | |
036b1063 | 909 | module_put(ib_dev->owner); |
70a30e16 RD |
910 | |
911 | err: | |
036b1063 YH |
912 | mutex_unlock(&dev->lists_mutex); |
913 | srcu_read_unlock(&dev->disassociate_srcu, srcu_key); | |
35d4a0b6 YH |
914 | if (atomic_dec_and_test(&dev->refcount)) |
915 | ib_uverbs_comp_dev(dev); | |
916 | ||
70a30e16 | 917 | return ret; |
bc38a6ab RD |
918 | } |
919 | ||
920 | static int ib_uverbs_close(struct inode *inode, struct file *filp) | |
921 | { | |
922 | struct ib_uverbs_file *file = filp->private_data; | |
d1e09f30 JG |
923 | |
924 | mutex_lock(&file->cleanup_mutex); | |
925 | if (file->ucontext) { | |
fd3c7904 | 926 | ib_uverbs_cleanup_ucontext(file, file->ucontext, false); |
d1e09f30 JG |
927 | file->ucontext = NULL; |
928 | } | |
929 | mutex_unlock(&file->cleanup_mutex); | |
771addf6 | 930 | idr_destroy(&file->idr); |
036b1063 YH |
931 | |
932 | mutex_lock(&file->device->lists_mutex); | |
036b1063 YH |
933 | if (!file->is_closed) { |
934 | list_del(&file->list); | |
935 | file->is_closed = 1; | |
936 | } | |
937 | mutex_unlock(&file->device->lists_mutex); | |
70a30e16 | 938 | |
bc38a6ab RD |
939 | kref_put(&file->ref, ib_uverbs_release_file); |
940 | ||
941 | return 0; | |
942 | } | |
943 | ||
2b8693c0 | 944 | static const struct file_operations uverbs_fops = { |
9afed76d AC |
945 | .owner = THIS_MODULE, |
946 | .write = ib_uverbs_write, | |
947 | .open = ib_uverbs_open, | |
bc1db9af RD |
948 | .release = ib_uverbs_close, |
949 | .llseek = no_llseek, | |
8eb19e8e MB |
950 | #if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS) |
951 | .unlocked_ioctl = ib_uverbs_ioctl, | |
952 | #endif | |
bc38a6ab RD |
953 | }; |
954 | ||
2b8693c0 | 955 | static const struct file_operations uverbs_mmap_fops = { |
9afed76d AC |
956 | .owner = THIS_MODULE, |
957 | .write = ib_uverbs_write, | |
bc38a6ab | 958 | .mmap = ib_uverbs_mmap, |
9afed76d | 959 | .open = ib_uverbs_open, |
bc1db9af RD |
960 | .release = ib_uverbs_close, |
961 | .llseek = no_llseek, | |
8eb19e8e MB |
962 | #if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS) |
963 | .unlocked_ioctl = ib_uverbs_ioctl, | |
964 | #endif | |
bc38a6ab RD |
965 | }; |
966 | ||
967 | static struct ib_client uverbs_client = { | |
968 | .name = "uverbs", | |
969 | .add = ib_uverbs_add_one, | |
970 | .remove = ib_uverbs_remove_one | |
971 | }; | |
972 | ||
f4e91eb4 TJ |
973 | static ssize_t show_ibdev(struct device *device, struct device_attribute *attr, |
974 | char *buf) | |
bc38a6ab | 975 | { |
036b1063 YH |
976 | int ret = -ENODEV; |
977 | int srcu_key; | |
f4e91eb4 | 978 | struct ib_uverbs_device *dev = dev_get_drvdata(device); |
036b1063 | 979 | struct ib_device *ib_dev; |
70a30e16 RD |
980 | |
981 | if (!dev) | |
982 | return -ENODEV; | |
bc38a6ab | 983 | |
036b1063 YH |
984 | srcu_key = srcu_read_lock(&dev->disassociate_srcu); |
985 | ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu); | |
986 | if (ib_dev) | |
987 | ret = sprintf(buf, "%s\n", ib_dev->name); | |
988 | srcu_read_unlock(&dev->disassociate_srcu, srcu_key); | |
989 | ||
990 | return ret; | |
bc38a6ab | 991 | } |
f4e91eb4 | 992 | static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); |
bc38a6ab | 993 | |
f4e91eb4 TJ |
994 | static ssize_t show_dev_abi_version(struct device *device, |
995 | struct device_attribute *attr, char *buf) | |
274c0891 | 996 | { |
f4e91eb4 | 997 | struct ib_uverbs_device *dev = dev_get_drvdata(device); |
036b1063 YH |
998 | int ret = -ENODEV; |
999 | int srcu_key; | |
1000 | struct ib_device *ib_dev; | |
70a30e16 RD |
1001 | |
1002 | if (!dev) | |
1003 | return -ENODEV; | |
036b1063 YH |
1004 | srcu_key = srcu_read_lock(&dev->disassociate_srcu); |
1005 | ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu); | |
1006 | if (ib_dev) | |
1007 | ret = sprintf(buf, "%d\n", ib_dev->uverbs_abi_ver); | |
1008 | srcu_read_unlock(&dev->disassociate_srcu, srcu_key); | |
274c0891 | 1009 | |
036b1063 | 1010 | return ret; |
274c0891 | 1011 | } |
f4e91eb4 | 1012 | static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL); |
274c0891 | 1013 | |
0933e2d9 AK |
1014 | static CLASS_ATTR_STRING(abi_version, S_IRUGO, |
1015 | __stringify(IB_USER_VERBS_ABI_VERSION)); | |
bc38a6ab | 1016 | |
6d6a0e71 AC |
1017 | static dev_t overflow_maj; |
1018 | static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES); | |
1019 | ||
1020 | /* | |
1021 | * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by | |
1022 | * requesting a new major number and doubling the number of max devices we | |
1023 | * support. It's stupid, but simple. | |
1024 | */ | |
1025 | static int find_overflow_devnum(void) | |
1026 | { | |
1027 | int ret; | |
1028 | ||
1029 | if (!overflow_maj) { | |
1030 | ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES, | |
1031 | "infiniband_verbs"); | |
1032 | if (ret) { | |
aba25a3e | 1033 | pr_err("user_verbs: couldn't register dynamic device number\n"); |
6d6a0e71 AC |
1034 | return ret; |
1035 | } | |
1036 | } | |
1037 | ||
1038 | ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES); | |
1039 | if (ret >= IB_UVERBS_MAX_DEVICES) | |
1040 | return -1; | |
1041 | ||
1042 | return ret; | |
1043 | } | |
1044 | ||
bc38a6ab RD |
1045 | static void ib_uverbs_add_one(struct ib_device *device) |
1046 | { | |
38707980 | 1047 | int devnum; |
ddbd6883 | 1048 | dev_t base; |
bc38a6ab | 1049 | struct ib_uverbs_device *uverbs_dev; |
036b1063 | 1050 | int ret; |
bc38a6ab RD |
1051 | |
1052 | if (!device->alloc_ucontext) | |
1053 | return; | |
1054 | ||
de6eb66b | 1055 | uverbs_dev = kzalloc(sizeof *uverbs_dev, GFP_KERNEL); |
bc38a6ab RD |
1056 | if (!uverbs_dev) |
1057 | return; | |
1058 | ||
036b1063 YH |
1059 | ret = init_srcu_struct(&uverbs_dev->disassociate_srcu); |
1060 | if (ret) { | |
1061 | kfree(uverbs_dev); | |
1062 | return; | |
1063 | } | |
1064 | ||
35d4a0b6 | 1065 | atomic_set(&uverbs_dev->refcount, 1); |
fd60ae40 | 1066 | init_completion(&uverbs_dev->comp); |
53d0bd1e SH |
1067 | uverbs_dev->xrcd_tree = RB_ROOT; |
1068 | mutex_init(&uverbs_dev->xrcd_tree_mutex); | |
35d4a0b6 | 1069 | kobject_init(&uverbs_dev->kobj, &ib_uverbs_dev_ktype); |
036b1063 YH |
1070 | mutex_init(&uverbs_dev->lists_mutex); |
1071 | INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list); | |
1072 | INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list); | |
70a30e16 | 1073 | |
bc38a6ab | 1074 | spin_lock(&map_lock); |
38707980 AC |
1075 | devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES); |
1076 | if (devnum >= IB_UVERBS_MAX_DEVICES) { | |
bc38a6ab | 1077 | spin_unlock(&map_lock); |
6d6a0e71 AC |
1078 | devnum = find_overflow_devnum(); |
1079 | if (devnum < 0) | |
1080 | goto err; | |
1081 | ||
1082 | spin_lock(&map_lock); | |
1083 | uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES; | |
1084 | base = devnum + overflow_maj; | |
1085 | set_bit(devnum, overflow_map); | |
1086 | } else { | |
1087 | uverbs_dev->devnum = devnum; | |
1088 | base = devnum + IB_UVERBS_BASE_DEV; | |
1089 | set_bit(devnum, dev_map); | |
bc38a6ab | 1090 | } |
bc38a6ab RD |
1091 | spin_unlock(&map_lock); |
1092 | ||
036b1063 | 1093 | rcu_assign_pointer(uverbs_dev->ib_dev, device); |
f4fd0b22 | 1094 | uverbs_dev->num_comp_vectors = device->num_comp_vectors; |
bc38a6ab | 1095 | |
055422dd AC |
1096 | cdev_init(&uverbs_dev->cdev, NULL); |
1097 | uverbs_dev->cdev.owner = THIS_MODULE; | |
1098 | uverbs_dev->cdev.ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops; | |
98508715 | 1099 | cdev_set_parent(&uverbs_dev->cdev, &uverbs_dev->kobj); |
055422dd | 1100 | kobject_set_name(&uverbs_dev->cdev.kobj, "uverbs%d", uverbs_dev->devnum); |
ddbd6883 | 1101 | if (cdev_add(&uverbs_dev->cdev, base, 1)) |
70a30e16 | 1102 | goto err_cdev; |
bc38a6ab | 1103 | |
1e35a088 | 1104 | uverbs_dev->dev = device_create(uverbs_class, device->dev.parent, |
055422dd | 1105 | uverbs_dev->cdev.dev, uverbs_dev, |
91bd418f | 1106 | "uverbs%d", uverbs_dev->devnum); |
f4e91eb4 | 1107 | if (IS_ERR(uverbs_dev->dev)) |
bc38a6ab RD |
1108 | goto err_cdev; |
1109 | ||
f4e91eb4 | 1110 | if (device_create_file(uverbs_dev->dev, &dev_attr_ibdev)) |
bc38a6ab | 1111 | goto err_class; |
f4e91eb4 | 1112 | if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version)) |
274c0891 | 1113 | goto err_class; |
bc38a6ab | 1114 | |
52427112 MB |
1115 | if (!device->specs_root) { |
1116 | const struct uverbs_object_tree_def *default_root[] = { | |
1117 | uverbs_default_get_objects()}; | |
1118 | ||
1119 | uverbs_dev->specs_root = uverbs_alloc_spec_tree(1, | |
1120 | default_root); | |
1121 | if (IS_ERR(uverbs_dev->specs_root)) | |
1122 | goto err_class; | |
1123 | ||
1124 | device->specs_root = uverbs_dev->specs_root; | |
1125 | } | |
1126 | ||
bc38a6ab RD |
1127 | ib_set_client_data(device, &uverbs_client, uverbs_dev); |
1128 | ||
1129 | return; | |
1130 | ||
1131 | err_class: | |
055422dd | 1132 | device_destroy(uverbs_class, uverbs_dev->cdev.dev); |
bc38a6ab RD |
1133 | |
1134 | err_cdev: | |
055422dd | 1135 | cdev_del(&uverbs_dev->cdev); |
6d6a0e71 AC |
1136 | if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES) |
1137 | clear_bit(devnum, dev_map); | |
1138 | else | |
1139 | clear_bit(devnum, overflow_map); | |
bc38a6ab RD |
1140 | |
1141 | err: | |
35d4a0b6 YH |
1142 | if (atomic_dec_and_test(&uverbs_dev->refcount)) |
1143 | ib_uverbs_comp_dev(uverbs_dev); | |
fd60ae40 | 1144 | wait_for_completion(&uverbs_dev->comp); |
35d4a0b6 | 1145 | kobject_put(&uverbs_dev->kobj); |
bc38a6ab RD |
1146 | return; |
1147 | } | |
1148 | ||
036b1063 YH |
1149 | static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, |
1150 | struct ib_device *ib_dev) | |
1151 | { | |
1152 | struct ib_uverbs_file *file; | |
1e7710f3 | 1153 | struct ib_uverbs_async_event_file *event_file; |
036b1063 YH |
1154 | struct ib_event event; |
1155 | ||
1156 | /* Pending running commands to terminate */ | |
1157 | synchronize_srcu(&uverbs_dev->disassociate_srcu); | |
1158 | event.event = IB_EVENT_DEVICE_FATAL; | |
1159 | event.element.port_num = 0; | |
1160 | event.device = ib_dev; | |
1161 | ||
1162 | mutex_lock(&uverbs_dev->lists_mutex); | |
1163 | while (!list_empty(&uverbs_dev->uverbs_file_list)) { | |
1164 | struct ib_ucontext *ucontext; | |
036b1063 YH |
1165 | file = list_first_entry(&uverbs_dev->uverbs_file_list, |
1166 | struct ib_uverbs_file, list); | |
1167 | file->is_closed = 1; | |
036b1063 | 1168 | list_del(&file->list); |
036b1063 YH |
1169 | kref_get(&file->ref); |
1170 | mutex_unlock(&uverbs_dev->lists_mutex); | |
d1e09f30 | 1171 | |
d1e09f30 JG |
1172 | |
1173 | mutex_lock(&file->cleanup_mutex); | |
1174 | ucontext = file->ucontext; | |
1175 | file->ucontext = NULL; | |
1176 | mutex_unlock(&file->cleanup_mutex); | |
1177 | ||
1178 | /* At this point ib_uverbs_close cannot be running | |
1179 | * ib_uverbs_cleanup_ucontext | |
1180 | */ | |
036b1063 | 1181 | if (ucontext) { |
d1e09f30 JG |
1182 | /* We must release the mutex before going ahead and |
1183 | * calling disassociate_ucontext. disassociate_ucontext | |
1184 | * might end up indirectly calling uverbs_close, | |
1185 | * for example due to freeing the resources | |
1186 | * (e.g mmput). | |
1187 | */ | |
870201f9 | 1188 | ib_uverbs_event_handler(&file->event_handler, &event); |
036b1063 | 1189 | ib_dev->disassociate_ucontext(ucontext); |
cf8966b3 | 1190 | mutex_lock(&file->cleanup_mutex); |
fd3c7904 | 1191 | ib_uverbs_cleanup_ucontext(file, ucontext, true); |
cf8966b3 | 1192 | mutex_unlock(&file->cleanup_mutex); |
036b1063 YH |
1193 | } |
1194 | ||
1195 | mutex_lock(&uverbs_dev->lists_mutex); | |
1196 | kref_put(&file->ref, ib_uverbs_release_file); | |
1197 | } | |
1198 | ||
1199 | while (!list_empty(&uverbs_dev->uverbs_events_file_list)) { | |
1200 | event_file = list_first_entry(&uverbs_dev-> | |
1201 | uverbs_events_file_list, | |
1e7710f3 | 1202 | struct ib_uverbs_async_event_file, |
036b1063 | 1203 | list); |
db1b5ddd MB |
1204 | spin_lock_irq(&event_file->ev_queue.lock); |
1205 | event_file->ev_queue.is_closed = 1; | |
1206 | spin_unlock_irq(&event_file->ev_queue.lock); | |
036b1063 YH |
1207 | |
1208 | list_del(&event_file->list); | |
1e7710f3 MB |
1209 | ib_unregister_event_handler( |
1210 | &event_file->uverbs_file->event_handler); | |
1211 | event_file->uverbs_file->event_handler.device = | |
1212 | NULL; | |
036b1063 | 1213 | |
db1b5ddd MB |
1214 | wake_up_interruptible(&event_file->ev_queue.poll_wait); |
1215 | kill_fasync(&event_file->ev_queue.async_queue, SIGIO, POLL_IN); | |
036b1063 YH |
1216 | } |
1217 | mutex_unlock(&uverbs_dev->lists_mutex); | |
1218 | } | |
1219 | ||
7c1eb45a | 1220 | static void ib_uverbs_remove_one(struct ib_device *device, void *client_data) |
bc38a6ab | 1221 | { |
7c1eb45a | 1222 | struct ib_uverbs_device *uverbs_dev = client_data; |
036b1063 | 1223 | int wait_clients = 1; |
bc38a6ab RD |
1224 | |
1225 | if (!uverbs_dev) | |
1226 | return; | |
1227 | ||
f4e91eb4 | 1228 | dev_set_drvdata(uverbs_dev->dev, NULL); |
055422dd AC |
1229 | device_destroy(uverbs_class, uverbs_dev->cdev.dev); |
1230 | cdev_del(&uverbs_dev->cdev); | |
70a30e16 | 1231 | |
6d6a0e71 AC |
1232 | if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES) |
1233 | clear_bit(uverbs_dev->devnum, dev_map); | |
1234 | else | |
1235 | clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map); | |
fd60ae40 | 1236 | |
036b1063 YH |
1237 | if (device->disassociate_ucontext) { |
1238 | /* We disassociate HW resources and immediately return. | |
1239 | * Userspace will see a EIO errno for all future access. | |
1240 | * Upon returning, ib_device may be freed internally and is not | |
1241 | * valid any more. | |
1242 | * uverbs_device is still available until all clients close | |
1243 | * their files, then the uverbs device ref count will be zero | |
1244 | * and its resources will be freed. | |
1245 | * Note: At this point no more files can be opened since the | |
1246 | * cdev was deleted, however active clients can still issue | |
1247 | * commands and close their open files. | |
1248 | */ | |
1249 | rcu_assign_pointer(uverbs_dev->ib_dev, NULL); | |
1250 | ib_uverbs_free_hw_resources(uverbs_dev, device); | |
1251 | wait_clients = 0; | |
1252 | } | |
1253 | ||
35d4a0b6 YH |
1254 | if (atomic_dec_and_test(&uverbs_dev->refcount)) |
1255 | ib_uverbs_comp_dev(uverbs_dev); | |
036b1063 YH |
1256 | if (wait_clients) |
1257 | wait_for_completion(&uverbs_dev->comp); | |
52427112 MB |
1258 | if (uverbs_dev->specs_root) { |
1259 | uverbs_free_spec_tree(uverbs_dev->specs_root); | |
1260 | device->specs_root = NULL; | |
1261 | } | |
1262 | ||
35d4a0b6 | 1263 | kobject_put(&uverbs_dev->kobj); |
bc38a6ab RD |
1264 | } |
1265 | ||
2c9ede55 | 1266 | static char *uverbs_devnode(struct device *dev, umode_t *mode) |
71c29bd5 | 1267 | { |
b2bc4782 GR |
1268 | if (mode) |
1269 | *mode = 0666; | |
71c29bd5 RD |
1270 | return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); |
1271 | } | |
1272 | ||
bc38a6ab RD |
1273 | static int __init ib_uverbs_init(void) |
1274 | { | |
1275 | int ret; | |
1276 | ||
bc38a6ab RD |
1277 | ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES, |
1278 | "infiniband_verbs"); | |
1279 | if (ret) { | |
aba25a3e | 1280 | pr_err("user_verbs: couldn't register device number\n"); |
bc38a6ab RD |
1281 | goto out; |
1282 | } | |
1283 | ||
70a30e16 RD |
1284 | uverbs_class = class_create(THIS_MODULE, "infiniband_verbs"); |
1285 | if (IS_ERR(uverbs_class)) { | |
1286 | ret = PTR_ERR(uverbs_class); | |
aba25a3e | 1287 | pr_err("user_verbs: couldn't create class infiniband_verbs\n"); |
bc38a6ab RD |
1288 | goto out_chrdev; |
1289 | } | |
1290 | ||
71c29bd5 RD |
1291 | uverbs_class->devnode = uverbs_devnode; |
1292 | ||
0933e2d9 | 1293 | ret = class_create_file(uverbs_class, &class_attr_abi_version.attr); |
bc38a6ab | 1294 | if (ret) { |
aba25a3e | 1295 | pr_err("user_verbs: couldn't create abi_version attribute\n"); |
bc38a6ab RD |
1296 | goto out_class; |
1297 | } | |
1298 | ||
bc38a6ab RD |
1299 | ret = ib_register_client(&uverbs_client); |
1300 | if (ret) { | |
aba25a3e | 1301 | pr_err("user_verbs: couldn't register client\n"); |
a265e558 | 1302 | goto out_class; |
bc38a6ab RD |
1303 | } |
1304 | ||
1305 | return 0; | |
1306 | ||
bc38a6ab | 1307 | out_class: |
70a30e16 | 1308 | class_destroy(uverbs_class); |
bc38a6ab RD |
1309 | |
1310 | out_chrdev: | |
1311 | unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES); | |
1312 | ||
1313 | out: | |
1314 | return ret; | |
1315 | } | |
1316 | ||
1317 | static void __exit ib_uverbs_cleanup(void) | |
1318 | { | |
1319 | ib_unregister_client(&uverbs_client); | |
70a30e16 | 1320 | class_destroy(uverbs_class); |
bc38a6ab | 1321 | unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES); |
6d6a0e71 AC |
1322 | if (overflow_maj) |
1323 | unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES); | |
bc38a6ab RD |
1324 | } |
1325 | ||
1326 | module_init(ib_uverbs_init); | |
1327 | module_exit(ib_uverbs_cleanup); |