]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - drivers/infiniband/hw/mlx5/devx.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
RDMA/mlx5: Delete create QP flags obfuscation
[mirror_ubuntu-hirsute-kernel.git] / drivers / infiniband / hw / mlx5 / devx.c
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/mlx5_user_ioctl_cmds.h>
11 #include <rdma/mlx5_user_ioctl_verbs.h>
12 #include <rdma/ib_umem.h>
13 #include <rdma/uverbs_std_types.h>
14 #include <linux/mlx5/driver.h>
15 #include <linux/mlx5/fs.h>
16 #include "mlx5_ib.h"
17 #include "qp.h"
18 #include <linux/xarray.h>
19
20 #define UVERBS_MODULE_NAME mlx5_ib
21 #include <rdma/uverbs_named_ioctl.h>
22
23 static void dispatch_event_fd(struct list_head *fd_list, const void *data);
24
25 enum devx_obj_flags {
26 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
27 DEVX_OBJ_FLAGS_DCT = 1 << 1,
28 DEVX_OBJ_FLAGS_CQ = 1 << 2,
29 };
30
31 struct devx_async_data {
32 struct mlx5_ib_dev *mdev;
33 struct list_head list;
34 struct devx_async_cmd_event_file *ev_file;
35 struct mlx5_async_work cb_work;
36 u16 cmd_out_len;
37 /* must be last field in this structure */
38 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
39 };
40
41 struct devx_async_event_data {
42 struct list_head list; /* headed in ev_file->event_list */
43 struct mlx5_ib_uapi_devx_async_event_hdr hdr;
44 };
45
46 /* first level XA value data structure */
47 struct devx_event {
48 struct xarray object_ids; /* second XA level, Key = object id */
49 struct list_head unaffiliated_list;
50 };
51
52 /* second level XA value data structure */
53 struct devx_obj_event {
54 struct rcu_head rcu;
55 struct list_head obj_sub_list;
56 };
57
58 struct devx_event_subscription {
59 struct list_head file_list; /* headed in ev_file->
60 * subscribed_events_list
61 */
62 struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
63 * devx_obj_event->obj_sub_list
64 */
65 struct list_head obj_list; /* headed in devx_object */
66 struct list_head event_list; /* headed in ev_file->event_list or in
67 * temp list via subscription
68 */
69
70 u8 is_cleaned:1;
71 u32 xa_key_level1;
72 u32 xa_key_level2;
73 struct rcu_head rcu;
74 u64 cookie;
75 struct devx_async_event_file *ev_file;
76 struct eventfd_ctx *eventfd;
77 };
78
79 struct devx_async_event_file {
80 struct ib_uobject uobj;
81 /* Head of events that are subscribed to this FD */
82 struct list_head subscribed_events_list;
83 spinlock_t lock;
84 wait_queue_head_t poll_wait;
85 struct list_head event_list;
86 struct mlx5_ib_dev *dev;
87 u8 omit_data:1;
88 u8 is_overflow_err:1;
89 u8 is_destroyed:1;
90 };
91
92 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
93 struct devx_obj {
94 struct mlx5_ib_dev *ib_dev;
95 u64 obj_id;
96 u32 dinlen; /* destroy inbox length */
97 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
98 u32 flags;
99 union {
100 struct mlx5_ib_devx_mr devx_mr;
101 struct mlx5_core_dct core_dct;
102 struct mlx5_core_cq core_cq;
103 u32 flow_counter_bulk_size;
104 };
105 struct list_head event_sub; /* holds devx_event_subscription entries */
106 };
107
108 struct devx_umem {
109 struct mlx5_core_dev *mdev;
110 struct ib_umem *umem;
111 u32 page_offset;
112 int page_shift;
113 int ncont;
114 u32 dinlen;
115 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
116 };
117
118 struct devx_umem_reg_cmd {
119 void *in;
120 u32 inlen;
121 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
122 };
123
124 static struct mlx5_ib_ucontext *
125 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
126 {
127 return to_mucontext(ib_uverbs_get_ucontext(attrs));
128 }
129
130 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
131 {
132 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
133 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
134 void *uctx;
135 int err;
136 u16 uid;
137 u32 cap = 0;
138
139 /* 0 means not supported */
140 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
141 return -EINVAL;
142
143 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
144 if (is_user && capable(CAP_NET_RAW) &&
145 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
146 cap |= MLX5_UCTX_CAP_RAW_TX;
147 if (is_user && capable(CAP_SYS_RAWIO) &&
148 (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
149 MLX5_UCTX_CAP_INTERNAL_DEV_RES))
150 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
151
152 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
153 MLX5_SET(uctx, uctx, cap, cap);
154
155 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
156 if (err)
157 return err;
158
159 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
160 return uid;
161 }
162
163 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
164 {
165 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
166 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
167
168 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
169 MLX5_SET(destroy_uctx_in, in, uid, uid);
170
171 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
172 }
173
174 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
175 {
176 struct devx_obj *devx_obj = obj;
177 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
178
179 switch (opcode) {
180 case MLX5_CMD_OP_DESTROY_TIR:
181 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
182 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
183 obj_id);
184 return true;
185
186 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
187 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
188 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
189 table_id);
190 return true;
191 default:
192 return false;
193 }
194 }
195
196 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
197 {
198 struct devx_obj *devx_obj = obj;
199 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
200
201 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
202
203 if (offset && offset >= devx_obj->flow_counter_bulk_size)
204 return false;
205
206 *counter_id = MLX5_GET(dealloc_flow_counter_in,
207 devx_obj->dinbox,
208 flow_counter_id);
209 *counter_id += offset;
210 return true;
211 }
212
213 return false;
214 }
215
216 static bool is_legacy_unaffiliated_event_num(u16 event_num)
217 {
218 switch (event_num) {
219 case MLX5_EVENT_TYPE_PORT_CHANGE:
220 return true;
221 default:
222 return false;
223 }
224 }
225
226 static bool is_legacy_obj_event_num(u16 event_num)
227 {
228 switch (event_num) {
229 case MLX5_EVENT_TYPE_PATH_MIG:
230 case MLX5_EVENT_TYPE_COMM_EST:
231 case MLX5_EVENT_TYPE_SQ_DRAINED:
232 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
233 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
234 case MLX5_EVENT_TYPE_CQ_ERROR:
235 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
236 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
237 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
238 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
239 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
240 case MLX5_EVENT_TYPE_DCT_DRAINED:
241 case MLX5_EVENT_TYPE_COMP:
242 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
243 case MLX5_EVENT_TYPE_XRQ_ERROR:
244 return true;
245 default:
246 return false;
247 }
248 }
249
250 static u16 get_legacy_obj_type(u16 opcode)
251 {
252 switch (opcode) {
253 case MLX5_CMD_OP_CREATE_RQ:
254 return MLX5_EVENT_QUEUE_TYPE_RQ;
255 case MLX5_CMD_OP_CREATE_QP:
256 return MLX5_EVENT_QUEUE_TYPE_QP;
257 case MLX5_CMD_OP_CREATE_SQ:
258 return MLX5_EVENT_QUEUE_TYPE_SQ;
259 case MLX5_CMD_OP_CREATE_DCT:
260 return MLX5_EVENT_QUEUE_TYPE_DCT;
261 default:
262 return 0;
263 }
264 }
265
266 static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
267 {
268 u16 opcode;
269
270 opcode = (obj->obj_id >> 32) & 0xffff;
271
272 if (is_legacy_obj_event_num(event_num))
273 return get_legacy_obj_type(opcode);
274
275 switch (opcode) {
276 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
277 return (obj->obj_id >> 48);
278 case MLX5_CMD_OP_CREATE_RQ:
279 return MLX5_OBJ_TYPE_RQ;
280 case MLX5_CMD_OP_CREATE_QP:
281 return MLX5_OBJ_TYPE_QP;
282 case MLX5_CMD_OP_CREATE_SQ:
283 return MLX5_OBJ_TYPE_SQ;
284 case MLX5_CMD_OP_CREATE_DCT:
285 return MLX5_OBJ_TYPE_DCT;
286 case MLX5_CMD_OP_CREATE_TIR:
287 return MLX5_OBJ_TYPE_TIR;
288 case MLX5_CMD_OP_CREATE_TIS:
289 return MLX5_OBJ_TYPE_TIS;
290 case MLX5_CMD_OP_CREATE_PSV:
291 return MLX5_OBJ_TYPE_PSV;
292 case MLX5_OBJ_TYPE_MKEY:
293 return MLX5_OBJ_TYPE_MKEY;
294 case MLX5_CMD_OP_CREATE_RMP:
295 return MLX5_OBJ_TYPE_RMP;
296 case MLX5_CMD_OP_CREATE_XRC_SRQ:
297 return MLX5_OBJ_TYPE_XRC_SRQ;
298 case MLX5_CMD_OP_CREATE_XRQ:
299 return MLX5_OBJ_TYPE_XRQ;
300 case MLX5_CMD_OP_CREATE_RQT:
301 return MLX5_OBJ_TYPE_RQT;
302 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
303 return MLX5_OBJ_TYPE_FLOW_COUNTER;
304 case MLX5_CMD_OP_CREATE_CQ:
305 return MLX5_OBJ_TYPE_CQ;
306 default:
307 return 0;
308 }
309 }
310
311 static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
312 {
313 switch (event_type) {
314 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
315 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
316 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
317 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
318 case MLX5_EVENT_TYPE_PATH_MIG:
319 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
320 case MLX5_EVENT_TYPE_COMM_EST:
321 case MLX5_EVENT_TYPE_SQ_DRAINED:
322 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
323 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
324 return eqe->data.qp_srq.type;
325 case MLX5_EVENT_TYPE_CQ_ERROR:
326 case MLX5_EVENT_TYPE_XRQ_ERROR:
327 return 0;
328 case MLX5_EVENT_TYPE_DCT_DRAINED:
329 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
330 return MLX5_EVENT_QUEUE_TYPE_DCT;
331 default:
332 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
333 }
334 }
335
336 static u32 get_dec_obj_id(u64 obj_id)
337 {
338 return (obj_id & 0xffffffff);
339 }
340
341 /*
342 * As the obj_id in the firmware is not globally unique the object type
343 * must be considered upon checking for a valid object id.
344 * For that the opcode of the creator command is encoded as part of the obj_id.
345 */
346 static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
347 {
348 return ((u64)opcode << 32) | obj_id;
349 }
350
351 static u64 devx_get_obj_id(const void *in)
352 {
353 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
354 u64 obj_id;
355
356 switch (opcode) {
357 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
358 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
359 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
360 MLX5_GET(general_obj_in_cmd_hdr, in,
361 obj_type) << 16,
362 MLX5_GET(general_obj_in_cmd_hdr, in,
363 obj_id));
364 break;
365 case MLX5_CMD_OP_QUERY_MKEY:
366 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
367 MLX5_GET(query_mkey_in, in,
368 mkey_index));
369 break;
370 case MLX5_CMD_OP_QUERY_CQ:
371 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
372 MLX5_GET(query_cq_in, in, cqn));
373 break;
374 case MLX5_CMD_OP_MODIFY_CQ:
375 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
376 MLX5_GET(modify_cq_in, in, cqn));
377 break;
378 case MLX5_CMD_OP_QUERY_SQ:
379 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
380 MLX5_GET(query_sq_in, in, sqn));
381 break;
382 case MLX5_CMD_OP_MODIFY_SQ:
383 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
384 MLX5_GET(modify_sq_in, in, sqn));
385 break;
386 case MLX5_CMD_OP_QUERY_RQ:
387 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
388 MLX5_GET(query_rq_in, in, rqn));
389 break;
390 case MLX5_CMD_OP_MODIFY_RQ:
391 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
392 MLX5_GET(modify_rq_in, in, rqn));
393 break;
394 case MLX5_CMD_OP_QUERY_RMP:
395 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
396 MLX5_GET(query_rmp_in, in, rmpn));
397 break;
398 case MLX5_CMD_OP_MODIFY_RMP:
399 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
400 MLX5_GET(modify_rmp_in, in, rmpn));
401 break;
402 case MLX5_CMD_OP_QUERY_RQT:
403 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
404 MLX5_GET(query_rqt_in, in, rqtn));
405 break;
406 case MLX5_CMD_OP_MODIFY_RQT:
407 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
408 MLX5_GET(modify_rqt_in, in, rqtn));
409 break;
410 case MLX5_CMD_OP_QUERY_TIR:
411 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
412 MLX5_GET(query_tir_in, in, tirn));
413 break;
414 case MLX5_CMD_OP_MODIFY_TIR:
415 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
416 MLX5_GET(modify_tir_in, in, tirn));
417 break;
418 case MLX5_CMD_OP_QUERY_TIS:
419 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
420 MLX5_GET(query_tis_in, in, tisn));
421 break;
422 case MLX5_CMD_OP_MODIFY_TIS:
423 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
424 MLX5_GET(modify_tis_in, in, tisn));
425 break;
426 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
427 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
428 MLX5_GET(query_flow_table_in, in,
429 table_id));
430 break;
431 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
432 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
433 MLX5_GET(modify_flow_table_in, in,
434 table_id));
435 break;
436 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
437 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
438 MLX5_GET(query_flow_group_in, in,
439 group_id));
440 break;
441 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
442 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
443 MLX5_GET(query_fte_in, in,
444 flow_index));
445 break;
446 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
447 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
448 MLX5_GET(set_fte_in, in, flow_index));
449 break;
450 case MLX5_CMD_OP_QUERY_Q_COUNTER:
451 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
452 MLX5_GET(query_q_counter_in, in,
453 counter_set_id));
454 break;
455 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
456 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
457 MLX5_GET(query_flow_counter_in, in,
458 flow_counter_id));
459 break;
460 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
461 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
462 MLX5_GET(general_obj_in_cmd_hdr, in,
463 obj_id));
464 break;
465 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
466 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
467 MLX5_GET(query_scheduling_element_in,
468 in, scheduling_element_id));
469 break;
470 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
471 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
472 MLX5_GET(modify_scheduling_element_in,
473 in, scheduling_element_id));
474 break;
475 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
476 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
477 MLX5_GET(add_vxlan_udp_dport_in, in,
478 vxlan_udp_port));
479 break;
480 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
481 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
482 MLX5_GET(query_l2_table_entry_in, in,
483 table_index));
484 break;
485 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
486 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
487 MLX5_GET(set_l2_table_entry_in, in,
488 table_index));
489 break;
490 case MLX5_CMD_OP_QUERY_QP:
491 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
492 MLX5_GET(query_qp_in, in, qpn));
493 break;
494 case MLX5_CMD_OP_RST2INIT_QP:
495 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
496 MLX5_GET(rst2init_qp_in, in, qpn));
497 break;
498 case MLX5_CMD_OP_INIT2RTR_QP:
499 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
500 MLX5_GET(init2rtr_qp_in, in, qpn));
501 break;
502 case MLX5_CMD_OP_RTR2RTS_QP:
503 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
504 MLX5_GET(rtr2rts_qp_in, in, qpn));
505 break;
506 case MLX5_CMD_OP_RTS2RTS_QP:
507 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
508 MLX5_GET(rts2rts_qp_in, in, qpn));
509 break;
510 case MLX5_CMD_OP_SQERR2RTS_QP:
511 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
512 MLX5_GET(sqerr2rts_qp_in, in, qpn));
513 break;
514 case MLX5_CMD_OP_2ERR_QP:
515 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
516 MLX5_GET(qp_2err_in, in, qpn));
517 break;
518 case MLX5_CMD_OP_2RST_QP:
519 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
520 MLX5_GET(qp_2rst_in, in, qpn));
521 break;
522 case MLX5_CMD_OP_QUERY_DCT:
523 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
524 MLX5_GET(query_dct_in, in, dctn));
525 break;
526 case MLX5_CMD_OP_QUERY_XRQ:
527 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
528 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
529 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
530 MLX5_GET(query_xrq_in, in, xrqn));
531 break;
532 case MLX5_CMD_OP_QUERY_XRC_SRQ:
533 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
534 MLX5_GET(query_xrc_srq_in, in,
535 xrc_srqn));
536 break;
537 case MLX5_CMD_OP_ARM_XRC_SRQ:
538 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
539 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
540 break;
541 case MLX5_CMD_OP_QUERY_SRQ:
542 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
543 MLX5_GET(query_srq_in, in, srqn));
544 break;
545 case MLX5_CMD_OP_ARM_RQ:
546 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
547 MLX5_GET(arm_rq_in, in, srq_number));
548 break;
549 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
550 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
551 MLX5_GET(drain_dct_in, in, dctn));
552 break;
553 case MLX5_CMD_OP_ARM_XRQ:
554 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
555 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
556 case MLX5_CMD_OP_MODIFY_XRQ:
557 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
558 MLX5_GET(arm_xrq_in, in, xrqn));
559 break;
560 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
561 obj_id = get_enc_obj_id
562 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
563 MLX5_GET(query_packet_reformat_context_in,
564 in, packet_reformat_id));
565 break;
566 default:
567 obj_id = 0;
568 }
569
570 return obj_id;
571 }
572
573 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
574 struct ib_uobject *uobj, const void *in)
575 {
576 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
577 u64 obj_id = devx_get_obj_id(in);
578
579 if (!obj_id)
580 return false;
581
582 switch (uobj_get_object_id(uobj)) {
583 case UVERBS_OBJECT_CQ:
584 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
585 to_mcq(uobj->object)->mcq.cqn) ==
586 obj_id;
587
588 case UVERBS_OBJECT_SRQ:
589 {
590 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
591 u16 opcode;
592
593 switch (srq->common.res) {
594 case MLX5_RES_XSRQ:
595 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
596 break;
597 case MLX5_RES_XRQ:
598 opcode = MLX5_CMD_OP_CREATE_XRQ;
599 break;
600 default:
601 if (!dev->mdev->issi)
602 opcode = MLX5_CMD_OP_CREATE_SRQ;
603 else
604 opcode = MLX5_CMD_OP_CREATE_RMP;
605 }
606
607 return get_enc_obj_id(opcode,
608 to_msrq(uobj->object)->msrq.srqn) ==
609 obj_id;
610 }
611
612 case UVERBS_OBJECT_QP:
613 {
614 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
615 enum ib_qp_type qp_type = qp->ibqp.qp_type;
616
617 if (qp_type == IB_QPT_RAW_PACKET ||
618 (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
619 struct mlx5_ib_raw_packet_qp *raw_packet_qp =
620 &qp->raw_packet_qp;
621 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
622 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
623
624 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
625 rq->base.mqp.qpn) == obj_id ||
626 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
627 sq->base.mqp.qpn) == obj_id ||
628 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
629 rq->tirn) == obj_id ||
630 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
631 sq->tisn) == obj_id);
632 }
633
634 if (qp_type == MLX5_IB_QPT_DCT)
635 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
636 qp->dct.mdct.mqp.qpn) == obj_id;
637
638 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
639 qp->ibqp.qp_num) == obj_id;
640 }
641
642 case UVERBS_OBJECT_WQ:
643 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
644 to_mrwq(uobj->object)->core_qp.qpn) ==
645 obj_id;
646
647 case UVERBS_OBJECT_RWQ_IND_TBL:
648 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
649 to_mrwq_ind_table(uobj->object)->rqtn) ==
650 obj_id;
651
652 case MLX5_IB_OBJECT_DEVX_OBJ:
653 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
654
655 default:
656 return false;
657 }
658 }
659
660 static void devx_set_umem_valid(const void *in)
661 {
662 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
663
664 switch (opcode) {
665 case MLX5_CMD_OP_CREATE_MKEY:
666 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
667 break;
668 case MLX5_CMD_OP_CREATE_CQ:
669 {
670 void *cqc;
671
672 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
673 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
674 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
675 break;
676 }
677 case MLX5_CMD_OP_CREATE_QP:
678 {
679 void *qpc;
680
681 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
682 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
683 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
684 break;
685 }
686
687 case MLX5_CMD_OP_CREATE_RQ:
688 {
689 void *rqc, *wq;
690
691 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
692 wq = MLX5_ADDR_OF(rqc, rqc, wq);
693 MLX5_SET(wq, wq, dbr_umem_valid, 1);
694 MLX5_SET(wq, wq, wq_umem_valid, 1);
695 break;
696 }
697
698 case MLX5_CMD_OP_CREATE_SQ:
699 {
700 void *sqc, *wq;
701
702 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
703 wq = MLX5_ADDR_OF(sqc, sqc, wq);
704 MLX5_SET(wq, wq, dbr_umem_valid, 1);
705 MLX5_SET(wq, wq, wq_umem_valid, 1);
706 break;
707 }
708
709 case MLX5_CMD_OP_MODIFY_CQ:
710 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
711 break;
712
713 case MLX5_CMD_OP_CREATE_RMP:
714 {
715 void *rmpc, *wq;
716
717 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
718 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
719 MLX5_SET(wq, wq, dbr_umem_valid, 1);
720 MLX5_SET(wq, wq, wq_umem_valid, 1);
721 break;
722 }
723
724 case MLX5_CMD_OP_CREATE_XRQ:
725 {
726 void *xrqc, *wq;
727
728 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
729 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
730 MLX5_SET(wq, wq, dbr_umem_valid, 1);
731 MLX5_SET(wq, wq, wq_umem_valid, 1);
732 break;
733 }
734
735 case MLX5_CMD_OP_CREATE_XRC_SRQ:
736 {
737 void *xrc_srqc;
738
739 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
740 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
741 xrc_srq_context_entry);
742 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
743 break;
744 }
745
746 default:
747 return;
748 }
749 }
750
751 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
752 {
753 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
754
755 switch (*opcode) {
756 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
757 case MLX5_CMD_OP_CREATE_MKEY:
758 case MLX5_CMD_OP_CREATE_CQ:
759 case MLX5_CMD_OP_ALLOC_PD:
760 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
761 case MLX5_CMD_OP_CREATE_RMP:
762 case MLX5_CMD_OP_CREATE_SQ:
763 case MLX5_CMD_OP_CREATE_RQ:
764 case MLX5_CMD_OP_CREATE_RQT:
765 case MLX5_CMD_OP_CREATE_TIR:
766 case MLX5_CMD_OP_CREATE_TIS:
767 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
768 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
769 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
770 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
771 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
772 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
773 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
774 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
775 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
776 case MLX5_CMD_OP_CREATE_QP:
777 case MLX5_CMD_OP_CREATE_SRQ:
778 case MLX5_CMD_OP_CREATE_XRC_SRQ:
779 case MLX5_CMD_OP_CREATE_DCT:
780 case MLX5_CMD_OP_CREATE_XRQ:
781 case MLX5_CMD_OP_ATTACH_TO_MCG:
782 case MLX5_CMD_OP_ALLOC_XRCD:
783 return true;
784 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
785 {
786 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
787 if (op_mod == 0)
788 return true;
789 return false;
790 }
791 case MLX5_CMD_OP_CREATE_PSV:
792 {
793 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
794
795 if (num_psv == 1)
796 return true;
797 return false;
798 }
799 default:
800 return false;
801 }
802 }
803
804 static bool devx_is_obj_modify_cmd(const void *in)
805 {
806 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
807
808 switch (opcode) {
809 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
810 case MLX5_CMD_OP_MODIFY_CQ:
811 case MLX5_CMD_OP_MODIFY_RMP:
812 case MLX5_CMD_OP_MODIFY_SQ:
813 case MLX5_CMD_OP_MODIFY_RQ:
814 case MLX5_CMD_OP_MODIFY_RQT:
815 case MLX5_CMD_OP_MODIFY_TIR:
816 case MLX5_CMD_OP_MODIFY_TIS:
817 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
818 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
819 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
820 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
821 case MLX5_CMD_OP_RST2INIT_QP:
822 case MLX5_CMD_OP_INIT2RTR_QP:
823 case MLX5_CMD_OP_RTR2RTS_QP:
824 case MLX5_CMD_OP_RTS2RTS_QP:
825 case MLX5_CMD_OP_SQERR2RTS_QP:
826 case MLX5_CMD_OP_2ERR_QP:
827 case MLX5_CMD_OP_2RST_QP:
828 case MLX5_CMD_OP_ARM_XRC_SRQ:
829 case MLX5_CMD_OP_ARM_RQ:
830 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
831 case MLX5_CMD_OP_ARM_XRQ:
832 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
833 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
834 case MLX5_CMD_OP_MODIFY_XRQ:
835 return true;
836 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
837 {
838 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
839
840 if (op_mod == 1)
841 return true;
842 return false;
843 }
844 default:
845 return false;
846 }
847 }
848
849 static bool devx_is_obj_query_cmd(const void *in)
850 {
851 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
852
853 switch (opcode) {
854 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
855 case MLX5_CMD_OP_QUERY_MKEY:
856 case MLX5_CMD_OP_QUERY_CQ:
857 case MLX5_CMD_OP_QUERY_RMP:
858 case MLX5_CMD_OP_QUERY_SQ:
859 case MLX5_CMD_OP_QUERY_RQ:
860 case MLX5_CMD_OP_QUERY_RQT:
861 case MLX5_CMD_OP_QUERY_TIR:
862 case MLX5_CMD_OP_QUERY_TIS:
863 case MLX5_CMD_OP_QUERY_Q_COUNTER:
864 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
865 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
866 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
867 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
868 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
869 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
870 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
871 case MLX5_CMD_OP_QUERY_QP:
872 case MLX5_CMD_OP_QUERY_SRQ:
873 case MLX5_CMD_OP_QUERY_XRC_SRQ:
874 case MLX5_CMD_OP_QUERY_DCT:
875 case MLX5_CMD_OP_QUERY_XRQ:
876 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
877 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
878 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
879 return true;
880 default:
881 return false;
882 }
883 }
884
885 static bool devx_is_whitelist_cmd(void *in)
886 {
887 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
888
889 switch (opcode) {
890 case MLX5_CMD_OP_QUERY_HCA_CAP:
891 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
892 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
893 return true;
894 default:
895 return false;
896 }
897 }
898
899 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
900 {
901 if (devx_is_whitelist_cmd(cmd_in)) {
902 struct mlx5_ib_dev *dev;
903
904 if (c->devx_uid)
905 return c->devx_uid;
906
907 dev = to_mdev(c->ibucontext.device);
908 if (dev->devx_whitelist_uid)
909 return dev->devx_whitelist_uid;
910
911 return -EOPNOTSUPP;
912 }
913
914 if (!c->devx_uid)
915 return -EINVAL;
916
917 return c->devx_uid;
918 }
919
920 static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
921 {
922 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
923
924 /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
925 if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
926 MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
927 (opcode >= MLX5_CMD_OP_GENERAL_START &&
928 opcode < MLX5_CMD_OP_GENERAL_END))
929 return true;
930
931 switch (opcode) {
932 case MLX5_CMD_OP_QUERY_HCA_CAP:
933 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
934 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
935 case MLX5_CMD_OP_QUERY_VPORT_STATE:
936 case MLX5_CMD_OP_QUERY_ADAPTER:
937 case MLX5_CMD_OP_QUERY_ISSI:
938 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
939 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
940 case MLX5_CMD_OP_QUERY_VNIC_ENV:
941 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
942 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
943 case MLX5_CMD_OP_NOP:
944 case MLX5_CMD_OP_QUERY_CONG_STATUS:
945 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
946 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
947 case MLX5_CMD_OP_QUERY_LAG:
948 return true;
949 default:
950 return false;
951 }
952 }
953
954 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
955 struct uverbs_attr_bundle *attrs)
956 {
957 struct mlx5_ib_ucontext *c;
958 struct mlx5_ib_dev *dev;
959 int user_vector;
960 int dev_eqn;
961 unsigned int irqn;
962 int err;
963
964 if (uverbs_copy_from(&user_vector, attrs,
965 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
966 return -EFAULT;
967
968 c = devx_ufile2uctx(attrs);
969 if (IS_ERR(c))
970 return PTR_ERR(c);
971 dev = to_mdev(c->ibucontext.device);
972
973 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
974 if (err < 0)
975 return err;
976
977 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
978 &dev_eqn, sizeof(dev_eqn)))
979 return -EFAULT;
980
981 return 0;
982 }
983
984 /*
985 *Security note:
986 * The hardware protection mechanism works like this: Each device object that
987 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
988 * the device specification manual) upon its creation. Then upon doorbell,
989 * hardware fetches the object context for which the doorbell was rang, and
990 * validates that the UAR through which the DB was rang matches the UAR ID
991 * of the object.
992 * If no match the doorbell is silently ignored by the hardware. Of course,
993 * the user cannot ring a doorbell on a UAR that was not mapped to it.
994 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
995 * mailboxes (except tagging them with UID), we expose to the user its UAR
996 * ID, so it can embed it in these objects in the expected specification
997 * format. So the only thing the user can do is hurt itself by creating a
998 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
999 * may ring a doorbell on its objects.
1000 * The consequence of that will be that another user can schedule a QP/SQ
1001 * of the buggy user for execution (just insert it to the hardware schedule
1002 * queue or arm its CQ for event generation), no further harm is expected.
1003 */
1004 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1005 struct uverbs_attr_bundle *attrs)
1006 {
1007 struct mlx5_ib_ucontext *c;
1008 struct mlx5_ib_dev *dev;
1009 u32 user_idx;
1010 s32 dev_idx;
1011
1012 c = devx_ufile2uctx(attrs);
1013 if (IS_ERR(c))
1014 return PTR_ERR(c);
1015 dev = to_mdev(c->ibucontext.device);
1016
1017 if (uverbs_copy_from(&user_idx, attrs,
1018 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1019 return -EFAULT;
1020
1021 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1022 if (dev_idx < 0)
1023 return dev_idx;
1024
1025 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1026 &dev_idx, sizeof(dev_idx)))
1027 return -EFAULT;
1028
1029 return 0;
1030 }
1031
1032 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1033 struct uverbs_attr_bundle *attrs)
1034 {
1035 struct mlx5_ib_ucontext *c;
1036 struct mlx5_ib_dev *dev;
1037 void *cmd_in = uverbs_attr_get_alloced_ptr(
1038 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1039 int cmd_out_len = uverbs_attr_get_len(attrs,
1040 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1041 void *cmd_out;
1042 int err;
1043 int uid;
1044
1045 c = devx_ufile2uctx(attrs);
1046 if (IS_ERR(c))
1047 return PTR_ERR(c);
1048 dev = to_mdev(c->ibucontext.device);
1049
1050 uid = devx_get_uid(c, cmd_in);
1051 if (uid < 0)
1052 return uid;
1053
1054 /* Only white list of some general HCA commands are allowed for this method. */
1055 if (!devx_is_general_cmd(cmd_in, dev))
1056 return -EINVAL;
1057
1058 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1059 if (IS_ERR(cmd_out))
1060 return PTR_ERR(cmd_out);
1061
1062 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1063 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1064 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1065 cmd_out, cmd_out_len);
1066 if (err)
1067 return err;
1068
1069 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1070 cmd_out_len);
1071 }
1072
1073 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1074 u32 *dinlen,
1075 u32 *obj_id)
1076 {
1077 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
1078 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1079
1080 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1081 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1082
1083 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1084 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1085
1086 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
1087 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1088 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1089 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
1090 break;
1091
1092 case MLX5_CMD_OP_CREATE_UMEM:
1093 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1094 MLX5_CMD_OP_DESTROY_UMEM);
1095 break;
1096 case MLX5_CMD_OP_CREATE_MKEY:
1097 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
1098 break;
1099 case MLX5_CMD_OP_CREATE_CQ:
1100 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1101 break;
1102 case MLX5_CMD_OP_ALLOC_PD:
1103 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1104 break;
1105 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1106 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1107 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1108 break;
1109 case MLX5_CMD_OP_CREATE_RMP:
1110 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1111 break;
1112 case MLX5_CMD_OP_CREATE_SQ:
1113 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1114 break;
1115 case MLX5_CMD_OP_CREATE_RQ:
1116 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1117 break;
1118 case MLX5_CMD_OP_CREATE_RQT:
1119 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1120 break;
1121 case MLX5_CMD_OP_CREATE_TIR:
1122 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1123 break;
1124 case MLX5_CMD_OP_CREATE_TIS:
1125 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1126 break;
1127 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1128 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1129 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1130 break;
1131 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1132 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1133 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
1134 MLX5_SET(destroy_flow_table_in, din, other_vport,
1135 MLX5_GET(create_flow_table_in, in, other_vport));
1136 MLX5_SET(destroy_flow_table_in, din, vport_number,
1137 MLX5_GET(create_flow_table_in, in, vport_number));
1138 MLX5_SET(destroy_flow_table_in, din, table_type,
1139 MLX5_GET(create_flow_table_in, in, table_type));
1140 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1141 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1142 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1143 break;
1144 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1145 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1146 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
1147 MLX5_SET(destroy_flow_group_in, din, other_vport,
1148 MLX5_GET(create_flow_group_in, in, other_vport));
1149 MLX5_SET(destroy_flow_group_in, din, vport_number,
1150 MLX5_GET(create_flow_group_in, in, vport_number));
1151 MLX5_SET(destroy_flow_group_in, din, table_type,
1152 MLX5_GET(create_flow_group_in, in, table_type));
1153 MLX5_SET(destroy_flow_group_in, din, table_id,
1154 MLX5_GET(create_flow_group_in, in, table_id));
1155 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1156 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1157 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1158 break;
1159 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1160 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1161 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
1162 MLX5_SET(delete_fte_in, din, other_vport,
1163 MLX5_GET(set_fte_in, in, other_vport));
1164 MLX5_SET(delete_fte_in, din, vport_number,
1165 MLX5_GET(set_fte_in, in, vport_number));
1166 MLX5_SET(delete_fte_in, din, table_type,
1167 MLX5_GET(set_fte_in, in, table_type));
1168 MLX5_SET(delete_fte_in, din, table_id,
1169 MLX5_GET(set_fte_in, in, table_id));
1170 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1171 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1172 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1173 break;
1174 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1175 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1176 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1177 break;
1178 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1179 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1180 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1181 break;
1182 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1183 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1184 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1185 break;
1186 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1187 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1188 *obj_id = MLX5_GET(create_scheduling_element_out, out,
1189 scheduling_element_id);
1190 MLX5_SET(destroy_scheduling_element_in, din,
1191 scheduling_hierarchy,
1192 MLX5_GET(create_scheduling_element_in, in,
1193 scheduling_hierarchy));
1194 MLX5_SET(destroy_scheduling_element_in, din,
1195 scheduling_element_id, *obj_id);
1196 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1197 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1198 break;
1199 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1200 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1201 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
1202 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1203 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1204 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1205 break;
1206 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1207 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1208 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1209 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1210 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1211 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1212 break;
1213 case MLX5_CMD_OP_CREATE_QP:
1214 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1215 break;
1216 case MLX5_CMD_OP_CREATE_SRQ:
1217 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1218 break;
1219 case MLX5_CMD_OP_CREATE_XRC_SRQ:
1220 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1221 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1222 break;
1223 case MLX5_CMD_OP_CREATE_DCT:
1224 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1225 break;
1226 case MLX5_CMD_OP_CREATE_XRQ:
1227 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1228 break;
1229 case MLX5_CMD_OP_ATTACH_TO_MCG:
1230 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1231 MLX5_SET(detach_from_mcg_in, din, qpn,
1232 MLX5_GET(attach_to_mcg_in, in, qpn));
1233 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1234 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1235 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1236 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1237 break;
1238 case MLX5_CMD_OP_ALLOC_XRCD:
1239 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1240 break;
1241 case MLX5_CMD_OP_CREATE_PSV:
1242 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1243 MLX5_CMD_OP_DESTROY_PSV);
1244 MLX5_SET(destroy_psv_in, din, psvn,
1245 MLX5_GET(create_psv_out, out, psv0_index));
1246 break;
1247 default:
1248 /* The entry must match to one of the devx_is_obj_create_cmd */
1249 WARN_ON(true);
1250 break;
1251 }
1252 }
1253
1254 static int devx_handle_mkey_indirect(struct devx_obj *obj,
1255 struct mlx5_ib_dev *dev,
1256 void *in, void *out)
1257 {
1258 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
1259 struct mlx5_core_mkey *mkey;
1260 void *mkc;
1261 u8 key;
1262
1263 mkey = &devx_mr->mmkey;
1264 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1265 key = MLX5_GET(mkc, mkc, mkey_7_0);
1266 mkey->key = mlx5_idx_to_mkey(
1267 MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1268 mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1269 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1270 mkey->size = MLX5_GET64(mkc, mkc, len);
1271 mkey->pd = MLX5_GET(mkc, mkc, pd);
1272 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1273
1274 return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mkey->key), mkey,
1275 GFP_KERNEL));
1276 }
1277
1278 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1279 struct devx_obj *obj,
1280 void *in, int in_len)
1281 {
1282 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1283 MLX5_FLD_SZ_BYTES(create_mkey_in,
1284 memory_key_mkey_entry);
1285 void *mkc;
1286 u8 access_mode;
1287
1288 if (in_len < min_len)
1289 return -EINVAL;
1290
1291 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1292
1293 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1294 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1295
1296 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1297 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1298 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1299 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1300 return 0;
1301 }
1302
1303 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1304 return 0;
1305 }
1306
1307 static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1308 struct devx_event_subscription *sub)
1309 {
1310 struct devx_event *event;
1311 struct devx_obj_event *xa_val_level2;
1312
1313 if (sub->is_cleaned)
1314 return;
1315
1316 sub->is_cleaned = 1;
1317 list_del_rcu(&sub->xa_list);
1318
1319 if (list_empty(&sub->obj_list))
1320 return;
1321
1322 list_del_rcu(&sub->obj_list);
1323 /* check whether key level 1 for this obj_sub_list is empty */
1324 event = xa_load(&dev->devx_event_table.event_xa,
1325 sub->xa_key_level1);
1326 WARN_ON(!event);
1327
1328 xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1329 if (list_empty(&xa_val_level2->obj_sub_list)) {
1330 xa_erase(&event->object_ids,
1331 sub->xa_key_level2);
1332 kfree_rcu(xa_val_level2, rcu);
1333 }
1334 }
1335
1336 static int devx_obj_cleanup(struct ib_uobject *uobject,
1337 enum rdma_remove_reason why,
1338 struct uverbs_attr_bundle *attrs)
1339 {
1340 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1341 struct mlx5_devx_event_table *devx_event_table;
1342 struct devx_obj *obj = uobject->object;
1343 struct devx_event_subscription *sub_entry, *tmp;
1344 struct mlx5_ib_dev *dev;
1345 int ret;
1346
1347 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1348 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1349 /*
1350 * The pagefault_single_data_segment() does commands against
1351 * the mmkey, we must wait for that to stop before freeing the
1352 * mkey, as another allocation could get the same mkey #.
1353 */
1354 xa_erase(&obj->ib_dev->odp_mkeys,
1355 mlx5_base_mkey(obj->devx_mr.mmkey.key));
1356 synchronize_srcu(&dev->odp_srcu);
1357 }
1358
1359 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1360 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1361 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1362 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1363 else
1364 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1365 obj->dinlen, out, sizeof(out));
1366 if (ib_is_destroy_retryable(ret, why, uobject))
1367 return ret;
1368
1369 devx_event_table = &dev->devx_event_table;
1370
1371 mutex_lock(&devx_event_table->event_xa_lock);
1372 list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1373 devx_cleanup_subscription(dev, sub_entry);
1374 mutex_unlock(&devx_event_table->event_xa_lock);
1375
1376 kfree(obj);
1377 return ret;
1378 }
1379
1380 static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1381 {
1382 struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1383 struct mlx5_devx_event_table *table;
1384 struct devx_event *event;
1385 struct devx_obj_event *obj_event;
1386 u32 obj_id = mcq->cqn;
1387
1388 table = &obj->ib_dev->devx_event_table;
1389 rcu_read_lock();
1390 event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1391 if (!event)
1392 goto out;
1393
1394 obj_event = xa_load(&event->object_ids, obj_id);
1395 if (!obj_event)
1396 goto out;
1397
1398 dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1399 out:
1400 rcu_read_unlock();
1401 }
1402
1403 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1404 struct uverbs_attr_bundle *attrs)
1405 {
1406 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1407 int cmd_out_len = uverbs_attr_get_len(attrs,
1408 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1409 int cmd_in_len = uverbs_attr_get_len(attrs,
1410 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1411 void *cmd_out;
1412 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1413 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1414 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1415 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1416 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1417 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1418 struct devx_obj *obj;
1419 u16 obj_type = 0;
1420 int err;
1421 int uid;
1422 u32 obj_id;
1423 u16 opcode;
1424
1425 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1426 return -EINVAL;
1427
1428 uid = devx_get_uid(c, cmd_in);
1429 if (uid < 0)
1430 return uid;
1431
1432 if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1433 return -EINVAL;
1434
1435 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1436 if (IS_ERR(cmd_out))
1437 return PTR_ERR(cmd_out);
1438
1439 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1440 if (!obj)
1441 return -ENOMEM;
1442
1443 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1444 if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1445 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1446 if (err)
1447 goto obj_free;
1448 } else {
1449 devx_set_umem_valid(cmd_in);
1450 }
1451
1452 if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1453 obj->flags |= DEVX_OBJ_FLAGS_DCT;
1454 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1455 cmd_in_len, cmd_out, cmd_out_len);
1456 } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
1457 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1458 obj->core_cq.comp = devx_cq_comp;
1459 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1460 cmd_in, cmd_in_len, cmd_out,
1461 cmd_out_len);
1462 } else {
1463 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1464 cmd_in_len,
1465 cmd_out, cmd_out_len);
1466 }
1467
1468 if (err)
1469 goto obj_free;
1470
1471 if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1472 u8 bulk = MLX5_GET(alloc_flow_counter_in,
1473 cmd_in,
1474 flow_counter_bulk);
1475 obj->flow_counter_bulk_size = 128UL * bulk;
1476 }
1477
1478 uobj->object = obj;
1479 INIT_LIST_HEAD(&obj->event_sub);
1480 obj->ib_dev = dev;
1481 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1482 &obj_id);
1483 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1484
1485 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1486 if (err)
1487 goto obj_destroy;
1488
1489 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1490 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1491 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1492
1493 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1494 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1495 if (err)
1496 goto obj_destroy;
1497 }
1498 return 0;
1499
1500 obj_destroy:
1501 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1502 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1503 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1504 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1505 else
1506 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1507 sizeof(out));
1508 obj_free:
1509 kfree(obj);
1510 return err;
1511 }
1512
1513 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1514 struct uverbs_attr_bundle *attrs)
1515 {
1516 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1517 int cmd_out_len = uverbs_attr_get_len(attrs,
1518 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1519 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1520 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1521 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1522 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1523 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1524 void *cmd_out;
1525 int err;
1526 int uid;
1527
1528 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1529 return -EINVAL;
1530
1531 uid = devx_get_uid(c, cmd_in);
1532 if (uid < 0)
1533 return uid;
1534
1535 if (!devx_is_obj_modify_cmd(cmd_in))
1536 return -EINVAL;
1537
1538 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1539 return -EINVAL;
1540
1541 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1542 if (IS_ERR(cmd_out))
1543 return PTR_ERR(cmd_out);
1544
1545 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1546 devx_set_umem_valid(cmd_in);
1547
1548 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1549 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1550 cmd_out, cmd_out_len);
1551 if (err)
1552 return err;
1553
1554 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1555 cmd_out, cmd_out_len);
1556 }
1557
1558 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1559 struct uverbs_attr_bundle *attrs)
1560 {
1561 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1562 int cmd_out_len = uverbs_attr_get_len(attrs,
1563 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1564 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1565 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1566 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1567 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1568 void *cmd_out;
1569 int err;
1570 int uid;
1571 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1572
1573 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1574 return -EINVAL;
1575
1576 uid = devx_get_uid(c, cmd_in);
1577 if (uid < 0)
1578 return uid;
1579
1580 if (!devx_is_obj_query_cmd(cmd_in))
1581 return -EINVAL;
1582
1583 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1584 return -EINVAL;
1585
1586 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1587 if (IS_ERR(cmd_out))
1588 return PTR_ERR(cmd_out);
1589
1590 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1591 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
1592 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1593 cmd_out, cmd_out_len);
1594 if (err)
1595 return err;
1596
1597 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1598 cmd_out, cmd_out_len);
1599 }
1600
1601 struct devx_async_event_queue {
1602 spinlock_t lock;
1603 wait_queue_head_t poll_wait;
1604 struct list_head event_list;
1605 atomic_t bytes_in_use;
1606 u8 is_destroyed:1;
1607 };
1608
1609 struct devx_async_cmd_event_file {
1610 struct ib_uobject uobj;
1611 struct devx_async_event_queue ev_queue;
1612 struct mlx5_async_ctx async_ctx;
1613 };
1614
1615 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1616 {
1617 spin_lock_init(&ev_queue->lock);
1618 INIT_LIST_HEAD(&ev_queue->event_list);
1619 init_waitqueue_head(&ev_queue->poll_wait);
1620 atomic_set(&ev_queue->bytes_in_use, 0);
1621 ev_queue->is_destroyed = 0;
1622 }
1623
1624 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1625 struct uverbs_attr_bundle *attrs)
1626 {
1627 struct devx_async_cmd_event_file *ev_file;
1628
1629 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1630 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1631 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1632
1633 ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1634 uobj);
1635 devx_init_event_queue(&ev_file->ev_queue);
1636 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1637 return 0;
1638 }
1639
1640 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1641 struct uverbs_attr_bundle *attrs)
1642 {
1643 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1644 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1645 struct devx_async_event_file *ev_file;
1646 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1647 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1648 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1649 u32 flags;
1650 int err;
1651
1652 err = uverbs_get_flags32(&flags, attrs,
1653 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1654 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1655
1656 if (err)
1657 return err;
1658
1659 ev_file = container_of(uobj, struct devx_async_event_file,
1660 uobj);
1661 spin_lock_init(&ev_file->lock);
1662 INIT_LIST_HEAD(&ev_file->event_list);
1663 init_waitqueue_head(&ev_file->poll_wait);
1664 if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1665 ev_file->omit_data = 1;
1666 INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1667 ev_file->dev = dev;
1668 get_device(&dev->ib_dev.dev);
1669 return 0;
1670 }
1671
1672 static void devx_query_callback(int status, struct mlx5_async_work *context)
1673 {
1674 struct devx_async_data *async_data =
1675 container_of(context, struct devx_async_data, cb_work);
1676 struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1677 struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1678 unsigned long flags;
1679
1680 /*
1681 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1682 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1683 * routine returns, ensuring that it always remains valid here.
1684 */
1685 spin_lock_irqsave(&ev_queue->lock, flags);
1686 list_add_tail(&async_data->list, &ev_queue->event_list);
1687 spin_unlock_irqrestore(&ev_queue->lock, flags);
1688
1689 wake_up_interruptible(&ev_queue->poll_wait);
1690 }
1691
1692 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1693
1694 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1695 struct uverbs_attr_bundle *attrs)
1696 {
1697 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1698 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1699 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1700 attrs,
1701 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1702 u16 cmd_out_len;
1703 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1704 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1705 struct ib_uobject *fd_uobj;
1706 int err;
1707 int uid;
1708 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1709 struct devx_async_cmd_event_file *ev_file;
1710 struct devx_async_data *async_data;
1711
1712 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1713 return -EINVAL;
1714
1715 uid = devx_get_uid(c, cmd_in);
1716 if (uid < 0)
1717 return uid;
1718
1719 if (!devx_is_obj_query_cmd(cmd_in))
1720 return -EINVAL;
1721
1722 err = uverbs_get_const(&cmd_out_len, attrs,
1723 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1724 if (err)
1725 return err;
1726
1727 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1728 return -EINVAL;
1729
1730 fd_uobj = uverbs_attr_get_uobject(attrs,
1731 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1732 if (IS_ERR(fd_uobj))
1733 return PTR_ERR(fd_uobj);
1734
1735 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1736 uobj);
1737
1738 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1739 MAX_ASYNC_BYTES_IN_USE) {
1740 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1741 return -EAGAIN;
1742 }
1743
1744 async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1745 cmd_out_len), GFP_KERNEL);
1746 if (!async_data) {
1747 err = -ENOMEM;
1748 goto sub_bytes;
1749 }
1750
1751 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1752 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1753 if (err)
1754 goto free_async;
1755
1756 async_data->cmd_out_len = cmd_out_len;
1757 async_data->mdev = mdev;
1758 async_data->ev_file = ev_file;
1759
1760 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1761 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1762 uverbs_attr_get_len(attrs,
1763 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1764 async_data->hdr.out_data,
1765 async_data->cmd_out_len,
1766 devx_query_callback, &async_data->cb_work);
1767
1768 if (err)
1769 goto free_async;
1770
1771 return 0;
1772
1773 free_async:
1774 kvfree(async_data);
1775 sub_bytes:
1776 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1777 return err;
1778 }
1779
1780 static void
1781 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1782 u32 key_level1,
1783 bool is_level2,
1784 u32 key_level2)
1785 {
1786 struct devx_event *event;
1787 struct devx_obj_event *xa_val_level2;
1788
1789 /* Level 1 is valid for future use, no need to free */
1790 if (!is_level2)
1791 return;
1792
1793 event = xa_load(&devx_event_table->event_xa, key_level1);
1794 WARN_ON(!event);
1795
1796 xa_val_level2 = xa_load(&event->object_ids,
1797 key_level2);
1798 if (list_empty(&xa_val_level2->obj_sub_list)) {
1799 xa_erase(&event->object_ids,
1800 key_level2);
1801 kfree_rcu(xa_val_level2, rcu);
1802 }
1803 }
1804
1805 static int
1806 subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1807 u32 key_level1,
1808 bool is_level2,
1809 u32 key_level2)
1810 {
1811 struct devx_obj_event *obj_event;
1812 struct devx_event *event;
1813 int err;
1814
1815 event = xa_load(&devx_event_table->event_xa, key_level1);
1816 if (!event) {
1817 event = kzalloc(sizeof(*event), GFP_KERNEL);
1818 if (!event)
1819 return -ENOMEM;
1820
1821 INIT_LIST_HEAD(&event->unaffiliated_list);
1822 xa_init(&event->object_ids);
1823
1824 err = xa_insert(&devx_event_table->event_xa,
1825 key_level1,
1826 event,
1827 GFP_KERNEL);
1828 if (err) {
1829 kfree(event);
1830 return err;
1831 }
1832 }
1833
1834 if (!is_level2)
1835 return 0;
1836
1837 obj_event = xa_load(&event->object_ids, key_level2);
1838 if (!obj_event) {
1839 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1840 if (!obj_event)
1841 /* Level1 is valid for future use, no need to free */
1842 return -ENOMEM;
1843
1844 err = xa_insert(&event->object_ids,
1845 key_level2,
1846 obj_event,
1847 GFP_KERNEL);
1848 if (err)
1849 return err;
1850 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1851 }
1852
1853 return 0;
1854 }
1855
1856 static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1857 struct devx_obj *obj)
1858 {
1859 int i;
1860
1861 for (i = 0; i < num_events; i++) {
1862 if (obj) {
1863 if (!is_legacy_obj_event_num(event_type_num_list[i]))
1864 return false;
1865 } else if (!is_legacy_unaffiliated_event_num(
1866 event_type_num_list[i])) {
1867 return false;
1868 }
1869 }
1870
1871 return true;
1872 }
1873
1874 #define MAX_SUPP_EVENT_NUM 255
1875 static bool is_valid_events(struct mlx5_core_dev *dev,
1876 int num_events, u16 *event_type_num_list,
1877 struct devx_obj *obj)
1878 {
1879 __be64 *aff_events;
1880 __be64 *unaff_events;
1881 int mask_entry;
1882 int mask_bit;
1883 int i;
1884
1885 if (MLX5_CAP_GEN(dev, event_cap)) {
1886 aff_events = MLX5_CAP_DEV_EVENT(dev,
1887 user_affiliated_events);
1888 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1889 user_unaffiliated_events);
1890 } else {
1891 return is_valid_events_legacy(num_events, event_type_num_list,
1892 obj);
1893 }
1894
1895 for (i = 0; i < num_events; i++) {
1896 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1897 return false;
1898
1899 mask_entry = event_type_num_list[i] / 64;
1900 mask_bit = event_type_num_list[i] % 64;
1901
1902 if (obj) {
1903 /* CQ completion */
1904 if (event_type_num_list[i] == 0)
1905 continue;
1906
1907 if (!(be64_to_cpu(aff_events[mask_entry]) &
1908 (1ull << mask_bit)))
1909 return false;
1910
1911 continue;
1912 }
1913
1914 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1915 (1ull << mask_bit)))
1916 return false;
1917 }
1918
1919 return true;
1920 }
1921
1922 #define MAX_NUM_EVENTS 16
1923 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1924 struct uverbs_attr_bundle *attrs)
1925 {
1926 struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1927 attrs,
1928 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1929 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1930 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1931 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1932 struct ib_uobject *fd_uobj;
1933 struct devx_obj *obj = NULL;
1934 struct devx_async_event_file *ev_file;
1935 struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1936 u16 *event_type_num_list;
1937 struct devx_event_subscription *event_sub, *tmp_sub;
1938 struct list_head sub_list;
1939 int redirect_fd;
1940 bool use_eventfd = false;
1941 int num_events;
1942 int num_alloc_xa_entries = 0;
1943 u16 obj_type = 0;
1944 u64 cookie = 0;
1945 u32 obj_id = 0;
1946 int err;
1947 int i;
1948
1949 if (!c->devx_uid)
1950 return -EINVAL;
1951
1952 if (!IS_ERR(devx_uobj)) {
1953 obj = (struct devx_obj *)devx_uobj->object;
1954 if (obj)
1955 obj_id = get_dec_obj_id(obj->obj_id);
1956 }
1957
1958 fd_uobj = uverbs_attr_get_uobject(attrs,
1959 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
1960 if (IS_ERR(fd_uobj))
1961 return PTR_ERR(fd_uobj);
1962
1963 ev_file = container_of(fd_uobj, struct devx_async_event_file,
1964 uobj);
1965
1966 if (uverbs_attr_is_valid(attrs,
1967 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
1968 err = uverbs_copy_from(&redirect_fd, attrs,
1969 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
1970 if (err)
1971 return err;
1972
1973 use_eventfd = true;
1974 }
1975
1976 if (uverbs_attr_is_valid(attrs,
1977 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
1978 if (use_eventfd)
1979 return -EINVAL;
1980
1981 err = uverbs_copy_from(&cookie, attrs,
1982 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
1983 if (err)
1984 return err;
1985 }
1986
1987 num_events = uverbs_attr_ptr_get_array_size(
1988 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
1989 sizeof(u16));
1990
1991 if (num_events < 0)
1992 return num_events;
1993
1994 if (num_events > MAX_NUM_EVENTS)
1995 return -EINVAL;
1996
1997 event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
1998 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
1999
2000 if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2001 return -EINVAL;
2002
2003 INIT_LIST_HEAD(&sub_list);
2004
2005 /* Protect from concurrent subscriptions to same XA entries to allow
2006 * both to succeed
2007 */
2008 mutex_lock(&devx_event_table->event_xa_lock);
2009 for (i = 0; i < num_events; i++) {
2010 u32 key_level1;
2011
2012 if (obj)
2013 obj_type = get_dec_obj_type(obj,
2014 event_type_num_list[i]);
2015 key_level1 = event_type_num_list[i] | obj_type << 16;
2016
2017 err = subscribe_event_xa_alloc(devx_event_table,
2018 key_level1,
2019 obj,
2020 obj_id);
2021 if (err)
2022 goto err;
2023
2024 num_alloc_xa_entries++;
2025 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2026 if (!event_sub)
2027 goto err;
2028
2029 list_add_tail(&event_sub->event_list, &sub_list);
2030 uverbs_uobject_get(&ev_file->uobj);
2031 if (use_eventfd) {
2032 event_sub->eventfd =
2033 eventfd_ctx_fdget(redirect_fd);
2034
2035 if (IS_ERR(event_sub->eventfd)) {
2036 err = PTR_ERR(event_sub->eventfd);
2037 event_sub->eventfd = NULL;
2038 goto err;
2039 }
2040 }
2041
2042 event_sub->cookie = cookie;
2043 event_sub->ev_file = ev_file;
2044 /* May be needed upon cleanup the devx object/subscription */
2045 event_sub->xa_key_level1 = key_level1;
2046 event_sub->xa_key_level2 = obj_id;
2047 INIT_LIST_HEAD(&event_sub->obj_list);
2048 }
2049
2050 /* Once all the allocations and the XA data insertions were done we
2051 * can go ahead and add all the subscriptions to the relevant lists
2052 * without concern of a failure.
2053 */
2054 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2055 struct devx_event *event;
2056 struct devx_obj_event *obj_event;
2057
2058 list_del_init(&event_sub->event_list);
2059
2060 spin_lock_irq(&ev_file->lock);
2061 list_add_tail_rcu(&event_sub->file_list,
2062 &ev_file->subscribed_events_list);
2063 spin_unlock_irq(&ev_file->lock);
2064
2065 event = xa_load(&devx_event_table->event_xa,
2066 event_sub->xa_key_level1);
2067 WARN_ON(!event);
2068
2069 if (!obj) {
2070 list_add_tail_rcu(&event_sub->xa_list,
2071 &event->unaffiliated_list);
2072 continue;
2073 }
2074
2075 obj_event = xa_load(&event->object_ids, obj_id);
2076 WARN_ON(!obj_event);
2077 list_add_tail_rcu(&event_sub->xa_list,
2078 &obj_event->obj_sub_list);
2079 list_add_tail_rcu(&event_sub->obj_list,
2080 &obj->event_sub);
2081 }
2082
2083 mutex_unlock(&devx_event_table->event_xa_lock);
2084 return 0;
2085
2086 err:
2087 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2088 list_del(&event_sub->event_list);
2089
2090 subscribe_event_xa_dealloc(devx_event_table,
2091 event_sub->xa_key_level1,
2092 obj,
2093 obj_id);
2094
2095 if (event_sub->eventfd)
2096 eventfd_ctx_put(event_sub->eventfd);
2097 uverbs_uobject_put(&event_sub->ev_file->uobj);
2098 kfree(event_sub);
2099 }
2100
2101 mutex_unlock(&devx_event_table->event_xa_lock);
2102 return err;
2103 }
2104
2105 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2106 struct uverbs_attr_bundle *attrs,
2107 struct devx_umem *obj)
2108 {
2109 u64 addr;
2110 size_t size;
2111 u32 access;
2112 int npages;
2113 int err;
2114 u32 page_mask;
2115
2116 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2117 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2118 return -EFAULT;
2119
2120 err = uverbs_get_flags32(&access, attrs,
2121 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2122 IB_ACCESS_LOCAL_WRITE |
2123 IB_ACCESS_REMOTE_WRITE |
2124 IB_ACCESS_REMOTE_READ);
2125 if (err)
2126 return err;
2127
2128 err = ib_check_mr_access(access);
2129 if (err)
2130 return err;
2131
2132 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
2133 if (IS_ERR(obj->umem))
2134 return PTR_ERR(obj->umem);
2135
2136 mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2137 MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2138 &obj->page_shift, &obj->ncont, NULL);
2139
2140 if (!npages) {
2141 ib_umem_release(obj->umem);
2142 return -EINVAL;
2143 }
2144
2145 page_mask = (1 << obj->page_shift) - 1;
2146 obj->page_offset = obj->umem->address & page_mask;
2147
2148 return 0;
2149 }
2150
2151 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2152 struct devx_umem *obj,
2153 struct devx_umem_reg_cmd *cmd)
2154 {
2155 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2156 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
2157 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2158 return PTR_ERR_OR_ZERO(cmd->in);
2159 }
2160
2161 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2162 struct devx_umem *obj,
2163 struct devx_umem_reg_cmd *cmd)
2164 {
2165 void *umem;
2166 __be64 *mtt;
2167
2168 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2169 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2170
2171 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2172 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2173 MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2174 MLX5_ADAPTER_PAGE_SHIFT);
2175 MLX5_SET(umem, umem, page_offset, obj->page_offset);
2176 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2177 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2178 MLX5_IB_MTT_READ);
2179 }
2180
2181 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2182 struct uverbs_attr_bundle *attrs)
2183 {
2184 struct devx_umem_reg_cmd cmd;
2185 struct devx_umem *obj;
2186 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2187 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2188 u32 obj_id;
2189 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2190 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2191 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2192 int err;
2193
2194 if (!c->devx_uid)
2195 return -EINVAL;
2196
2197 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2198 if (!obj)
2199 return -ENOMEM;
2200
2201 err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2202 if (err)
2203 goto err_obj_free;
2204
2205 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
2206 if (err)
2207 goto err_umem_release;
2208
2209 devx_umem_reg_cmd_build(dev, obj, &cmd);
2210
2211 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2212 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2213 sizeof(cmd.out));
2214 if (err)
2215 goto err_umem_release;
2216
2217 obj->mdev = dev->mdev;
2218 uobj->object = obj;
2219 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2220 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
2221 if (err)
2222 goto err_umem_destroy;
2223
2224 return 0;
2225
2226 err_umem_destroy:
2227 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
2228 err_umem_release:
2229 ib_umem_release(obj->umem);
2230 err_obj_free:
2231 kfree(obj);
2232 return err;
2233 }
2234
2235 static int devx_umem_cleanup(struct ib_uobject *uobject,
2236 enum rdma_remove_reason why,
2237 struct uverbs_attr_bundle *attrs)
2238 {
2239 struct devx_umem *obj = uobject->object;
2240 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2241 int err;
2242
2243 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2244 if (ib_is_destroy_retryable(err, why, uobject))
2245 return err;
2246
2247 ib_umem_release(obj->umem);
2248 kfree(obj);
2249 return 0;
2250 }
2251
2252 static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2253 unsigned long event_type)
2254 {
2255 __be64 *unaff_events;
2256 int mask_entry;
2257 int mask_bit;
2258
2259 if (!MLX5_CAP_GEN(dev, event_cap))
2260 return is_legacy_unaffiliated_event_num(event_type);
2261
2262 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2263 user_unaffiliated_events);
2264 WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2265
2266 mask_entry = event_type / 64;
2267 mask_bit = event_type % 64;
2268
2269 if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2270 return false;
2271
2272 return true;
2273 }
2274
2275 static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2276 {
2277 struct mlx5_eqe *eqe = data;
2278 u32 obj_id = 0;
2279
2280 switch (event_type) {
2281 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2282 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2283 case MLX5_EVENT_TYPE_PATH_MIG:
2284 case MLX5_EVENT_TYPE_COMM_EST:
2285 case MLX5_EVENT_TYPE_SQ_DRAINED:
2286 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2287 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2288 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2289 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2290 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2291 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2292 break;
2293 case MLX5_EVENT_TYPE_XRQ_ERROR:
2294 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2295 break;
2296 case MLX5_EVENT_TYPE_DCT_DRAINED:
2297 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2298 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2299 break;
2300 case MLX5_EVENT_TYPE_CQ_ERROR:
2301 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2302 break;
2303 default:
2304 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2305 break;
2306 }
2307
2308 return obj_id;
2309 }
2310
2311 static int deliver_event(struct devx_event_subscription *event_sub,
2312 const void *data)
2313 {
2314 struct devx_async_event_file *ev_file;
2315 struct devx_async_event_data *event_data;
2316 unsigned long flags;
2317
2318 ev_file = event_sub->ev_file;
2319
2320 if (ev_file->omit_data) {
2321 spin_lock_irqsave(&ev_file->lock, flags);
2322 if (!list_empty(&event_sub->event_list) ||
2323 ev_file->is_destroyed) {
2324 spin_unlock_irqrestore(&ev_file->lock, flags);
2325 return 0;
2326 }
2327
2328 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2329 spin_unlock_irqrestore(&ev_file->lock, flags);
2330 wake_up_interruptible(&ev_file->poll_wait);
2331 return 0;
2332 }
2333
2334 event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2335 GFP_ATOMIC);
2336 if (!event_data) {
2337 spin_lock_irqsave(&ev_file->lock, flags);
2338 ev_file->is_overflow_err = 1;
2339 spin_unlock_irqrestore(&ev_file->lock, flags);
2340 return -ENOMEM;
2341 }
2342
2343 event_data->hdr.cookie = event_sub->cookie;
2344 memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2345
2346 spin_lock_irqsave(&ev_file->lock, flags);
2347 if (!ev_file->is_destroyed)
2348 list_add_tail(&event_data->list, &ev_file->event_list);
2349 else
2350 kfree(event_data);
2351 spin_unlock_irqrestore(&ev_file->lock, flags);
2352 wake_up_interruptible(&ev_file->poll_wait);
2353
2354 return 0;
2355 }
2356
2357 static void dispatch_event_fd(struct list_head *fd_list,
2358 const void *data)
2359 {
2360 struct devx_event_subscription *item;
2361
2362 list_for_each_entry_rcu(item, fd_list, xa_list) {
2363 if (item->eventfd)
2364 eventfd_signal(item->eventfd, 1);
2365 else
2366 deliver_event(item, data);
2367 }
2368 }
2369
2370 static int devx_event_notifier(struct notifier_block *nb,
2371 unsigned long event_type, void *data)
2372 {
2373 struct mlx5_devx_event_table *table;
2374 struct mlx5_ib_dev *dev;
2375 struct devx_event *event;
2376 struct devx_obj_event *obj_event;
2377 u16 obj_type = 0;
2378 bool is_unaffiliated;
2379 u32 obj_id;
2380
2381 /* Explicit filtering to kernel events which may occur frequently */
2382 if (event_type == MLX5_EVENT_TYPE_CMD ||
2383 event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2384 return NOTIFY_OK;
2385
2386 table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2387 dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2388 is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2389
2390 if (!is_unaffiliated)
2391 obj_type = get_event_obj_type(event_type, data);
2392
2393 rcu_read_lock();
2394 event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2395 if (!event) {
2396 rcu_read_unlock();
2397 return NOTIFY_DONE;
2398 }
2399
2400 if (is_unaffiliated) {
2401 dispatch_event_fd(&event->unaffiliated_list, data);
2402 rcu_read_unlock();
2403 return NOTIFY_OK;
2404 }
2405
2406 obj_id = devx_get_obj_id_from_event(event_type, data);
2407 obj_event = xa_load(&event->object_ids, obj_id);
2408 if (!obj_event) {
2409 rcu_read_unlock();
2410 return NOTIFY_DONE;
2411 }
2412
2413 dispatch_event_fd(&obj_event->obj_sub_list, data);
2414
2415 rcu_read_unlock();
2416 return NOTIFY_OK;
2417 }
2418
2419 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
2420 {
2421 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2422
2423 xa_init(&table->event_xa);
2424 mutex_init(&table->event_xa_lock);
2425 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2426 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2427 }
2428
2429 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
2430 {
2431 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2432 struct devx_event_subscription *sub, *tmp;
2433 struct devx_event *event;
2434 void *entry;
2435 unsigned long id;
2436
2437 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2438 mutex_lock(&dev->devx_event_table.event_xa_lock);
2439 xa_for_each(&table->event_xa, id, entry) {
2440 event = entry;
2441 list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
2442 xa_list)
2443 devx_cleanup_subscription(dev, sub);
2444 kfree(entry);
2445 }
2446 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2447 xa_destroy(&table->event_xa);
2448 }
2449
2450 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2451 size_t count, loff_t *pos)
2452 {
2453 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2454 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2455 struct devx_async_data *event;
2456 int ret = 0;
2457 size_t eventsz;
2458
2459 spin_lock_irq(&ev_queue->lock);
2460
2461 while (list_empty(&ev_queue->event_list)) {
2462 spin_unlock_irq(&ev_queue->lock);
2463
2464 if (filp->f_flags & O_NONBLOCK)
2465 return -EAGAIN;
2466
2467 if (wait_event_interruptible(
2468 ev_queue->poll_wait,
2469 (!list_empty(&ev_queue->event_list) ||
2470 ev_queue->is_destroyed))) {
2471 return -ERESTARTSYS;
2472 }
2473
2474 spin_lock_irq(&ev_queue->lock);
2475 if (ev_queue->is_destroyed) {
2476 spin_unlock_irq(&ev_queue->lock);
2477 return -EIO;
2478 }
2479 }
2480
2481 event = list_entry(ev_queue->event_list.next,
2482 struct devx_async_data, list);
2483 eventsz = event->cmd_out_len +
2484 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2485
2486 if (eventsz > count) {
2487 spin_unlock_irq(&ev_queue->lock);
2488 return -ENOSPC;
2489 }
2490
2491 list_del(ev_queue->event_list.next);
2492 spin_unlock_irq(&ev_queue->lock);
2493
2494 if (copy_to_user(buf, &event->hdr, eventsz))
2495 ret = -EFAULT;
2496 else
2497 ret = eventsz;
2498
2499 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2500 kvfree(event);
2501 return ret;
2502 }
2503
2504 static __poll_t devx_async_cmd_event_poll(struct file *filp,
2505 struct poll_table_struct *wait)
2506 {
2507 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2508 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2509 __poll_t pollflags = 0;
2510
2511 poll_wait(filp, &ev_queue->poll_wait, wait);
2512
2513 spin_lock_irq(&ev_queue->lock);
2514 if (ev_queue->is_destroyed)
2515 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2516 else if (!list_empty(&ev_queue->event_list))
2517 pollflags = EPOLLIN | EPOLLRDNORM;
2518 spin_unlock_irq(&ev_queue->lock);
2519
2520 return pollflags;
2521 }
2522
2523 static const struct file_operations devx_async_cmd_event_fops = {
2524 .owner = THIS_MODULE,
2525 .read = devx_async_cmd_event_read,
2526 .poll = devx_async_cmd_event_poll,
2527 .release = uverbs_uobject_fd_release,
2528 .llseek = no_llseek,
2529 };
2530
2531 static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2532 size_t count, loff_t *pos)
2533 {
2534 struct devx_async_event_file *ev_file = filp->private_data;
2535 struct devx_event_subscription *event_sub;
2536 struct devx_async_event_data *uninitialized_var(event);
2537 int ret = 0;
2538 size_t eventsz;
2539 bool omit_data;
2540 void *event_data;
2541
2542 omit_data = ev_file->omit_data;
2543
2544 spin_lock_irq(&ev_file->lock);
2545
2546 if (ev_file->is_overflow_err) {
2547 ev_file->is_overflow_err = 0;
2548 spin_unlock_irq(&ev_file->lock);
2549 return -EOVERFLOW;
2550 }
2551
2552
2553 while (list_empty(&ev_file->event_list)) {
2554 spin_unlock_irq(&ev_file->lock);
2555
2556 if (filp->f_flags & O_NONBLOCK)
2557 return -EAGAIN;
2558
2559 if (wait_event_interruptible(ev_file->poll_wait,
2560 (!list_empty(&ev_file->event_list) ||
2561 ev_file->is_destroyed))) {
2562 return -ERESTARTSYS;
2563 }
2564
2565 spin_lock_irq(&ev_file->lock);
2566 if (ev_file->is_destroyed) {
2567 spin_unlock_irq(&ev_file->lock);
2568 return -EIO;
2569 }
2570 }
2571
2572 if (omit_data) {
2573 event_sub = list_first_entry(&ev_file->event_list,
2574 struct devx_event_subscription,
2575 event_list);
2576 eventsz = sizeof(event_sub->cookie);
2577 event_data = &event_sub->cookie;
2578 } else {
2579 event = list_first_entry(&ev_file->event_list,
2580 struct devx_async_event_data, list);
2581 eventsz = sizeof(struct mlx5_eqe) +
2582 sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2583 event_data = &event->hdr;
2584 }
2585
2586 if (eventsz > count) {
2587 spin_unlock_irq(&ev_file->lock);
2588 return -EINVAL;
2589 }
2590
2591 if (omit_data)
2592 list_del_init(&event_sub->event_list);
2593 else
2594 list_del(&event->list);
2595
2596 spin_unlock_irq(&ev_file->lock);
2597
2598 if (copy_to_user(buf, event_data, eventsz))
2599 /* This points to an application issue, not a kernel concern */
2600 ret = -EFAULT;
2601 else
2602 ret = eventsz;
2603
2604 if (!omit_data)
2605 kfree(event);
2606 return ret;
2607 }
2608
2609 static __poll_t devx_async_event_poll(struct file *filp,
2610 struct poll_table_struct *wait)
2611 {
2612 struct devx_async_event_file *ev_file = filp->private_data;
2613 __poll_t pollflags = 0;
2614
2615 poll_wait(filp, &ev_file->poll_wait, wait);
2616
2617 spin_lock_irq(&ev_file->lock);
2618 if (ev_file->is_destroyed)
2619 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2620 else if (!list_empty(&ev_file->event_list))
2621 pollflags = EPOLLIN | EPOLLRDNORM;
2622 spin_unlock_irq(&ev_file->lock);
2623
2624 return pollflags;
2625 }
2626
2627 static void devx_free_subscription(struct rcu_head *rcu)
2628 {
2629 struct devx_event_subscription *event_sub =
2630 container_of(rcu, struct devx_event_subscription, rcu);
2631
2632 if (event_sub->eventfd)
2633 eventfd_ctx_put(event_sub->eventfd);
2634 uverbs_uobject_put(&event_sub->ev_file->uobj);
2635 kfree(event_sub);
2636 }
2637
2638 static const struct file_operations devx_async_event_fops = {
2639 .owner = THIS_MODULE,
2640 .read = devx_async_event_read,
2641 .poll = devx_async_event_poll,
2642 .release = uverbs_uobject_fd_release,
2643 .llseek = no_llseek,
2644 };
2645
2646 static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2647 enum rdma_remove_reason why)
2648 {
2649 struct devx_async_cmd_event_file *comp_ev_file =
2650 container_of(uobj, struct devx_async_cmd_event_file,
2651 uobj);
2652 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2653 struct devx_async_data *entry, *tmp;
2654
2655 spin_lock_irq(&ev_queue->lock);
2656 ev_queue->is_destroyed = 1;
2657 spin_unlock_irq(&ev_queue->lock);
2658 wake_up_interruptible(&ev_queue->poll_wait);
2659
2660 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2661
2662 spin_lock_irq(&comp_ev_file->ev_queue.lock);
2663 list_for_each_entry_safe(entry, tmp,
2664 &comp_ev_file->ev_queue.event_list, list) {
2665 list_del(&entry->list);
2666 kvfree(entry);
2667 }
2668 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2669 return 0;
2670 };
2671
2672 static int devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2673 enum rdma_remove_reason why)
2674 {
2675 struct devx_async_event_file *ev_file =
2676 container_of(uobj, struct devx_async_event_file,
2677 uobj);
2678 struct devx_event_subscription *event_sub, *event_sub_tmp;
2679 struct mlx5_ib_dev *dev = ev_file->dev;
2680
2681 spin_lock_irq(&ev_file->lock);
2682 ev_file->is_destroyed = 1;
2683
2684 /* free the pending events allocation */
2685 if (ev_file->omit_data) {
2686 struct devx_event_subscription *event_sub, *tmp;
2687
2688 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2689 event_list)
2690 list_del_init(&event_sub->event_list);
2691
2692 } else {
2693 struct devx_async_event_data *entry, *tmp;
2694
2695 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2696 list) {
2697 list_del(&entry->list);
2698 kfree(entry);
2699 }
2700 }
2701
2702 spin_unlock_irq(&ev_file->lock);
2703 wake_up_interruptible(&ev_file->poll_wait);
2704
2705 mutex_lock(&dev->devx_event_table.event_xa_lock);
2706 /* delete the subscriptions which are related to this FD */
2707 list_for_each_entry_safe(event_sub, event_sub_tmp,
2708 &ev_file->subscribed_events_list, file_list) {
2709 devx_cleanup_subscription(dev, event_sub);
2710 list_del_rcu(&event_sub->file_list);
2711 /* subscription may not be used by the read API any more */
2712 call_rcu(&event_sub->rcu, devx_free_subscription);
2713 }
2714 mutex_unlock(&dev->devx_event_table.event_xa_lock);
2715
2716 put_device(&dev->ib_dev.dev);
2717 return 0;
2718 };
2719
2720 DECLARE_UVERBS_NAMED_METHOD(
2721 MLX5_IB_METHOD_DEVX_UMEM_REG,
2722 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2723 MLX5_IB_OBJECT_DEVX_UMEM,
2724 UVERBS_ACCESS_NEW,
2725 UA_MANDATORY),
2726 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2727 UVERBS_ATTR_TYPE(u64),
2728 UA_MANDATORY),
2729 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2730 UVERBS_ATTR_TYPE(u64),
2731 UA_MANDATORY),
2732 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2733 enum ib_access_flags),
2734 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2735 UVERBS_ATTR_TYPE(u32),
2736 UA_MANDATORY));
2737
2738 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2739 MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2740 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2741 MLX5_IB_OBJECT_DEVX_UMEM,
2742 UVERBS_ACCESS_DESTROY,
2743 UA_MANDATORY));
2744
2745 DECLARE_UVERBS_NAMED_METHOD(
2746 MLX5_IB_METHOD_DEVX_QUERY_EQN,
2747 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2748 UVERBS_ATTR_TYPE(u32),
2749 UA_MANDATORY),
2750 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2751 UVERBS_ATTR_TYPE(u32),
2752 UA_MANDATORY));
2753
2754 DECLARE_UVERBS_NAMED_METHOD(
2755 MLX5_IB_METHOD_DEVX_QUERY_UAR,
2756 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2757 UVERBS_ATTR_TYPE(u32),
2758 UA_MANDATORY),
2759 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2760 UVERBS_ATTR_TYPE(u32),
2761 UA_MANDATORY));
2762
2763 DECLARE_UVERBS_NAMED_METHOD(
2764 MLX5_IB_METHOD_DEVX_OTHER,
2765 UVERBS_ATTR_PTR_IN(
2766 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2767 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2768 UA_MANDATORY,
2769 UA_ALLOC_AND_COPY),
2770 UVERBS_ATTR_PTR_OUT(
2771 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2772 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2773 UA_MANDATORY));
2774
2775 DECLARE_UVERBS_NAMED_METHOD(
2776 MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2777 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2778 MLX5_IB_OBJECT_DEVX_OBJ,
2779 UVERBS_ACCESS_NEW,
2780 UA_MANDATORY),
2781 UVERBS_ATTR_PTR_IN(
2782 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2783 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2784 UA_MANDATORY,
2785 UA_ALLOC_AND_COPY),
2786 UVERBS_ATTR_PTR_OUT(
2787 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2788 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2789 UA_MANDATORY));
2790
2791 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2792 MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2793 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2794 MLX5_IB_OBJECT_DEVX_OBJ,
2795 UVERBS_ACCESS_DESTROY,
2796 UA_MANDATORY));
2797
2798 DECLARE_UVERBS_NAMED_METHOD(
2799 MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2800 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
2801 UVERBS_IDR_ANY_OBJECT,
2802 UVERBS_ACCESS_WRITE,
2803 UA_MANDATORY),
2804 UVERBS_ATTR_PTR_IN(
2805 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2806 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2807 UA_MANDATORY,
2808 UA_ALLOC_AND_COPY),
2809 UVERBS_ATTR_PTR_OUT(
2810 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2811 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2812 UA_MANDATORY));
2813
2814 DECLARE_UVERBS_NAMED_METHOD(
2815 MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2816 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2817 UVERBS_IDR_ANY_OBJECT,
2818 UVERBS_ACCESS_READ,
2819 UA_MANDATORY),
2820 UVERBS_ATTR_PTR_IN(
2821 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2822 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2823 UA_MANDATORY,
2824 UA_ALLOC_AND_COPY),
2825 UVERBS_ATTR_PTR_OUT(
2826 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2827 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
2828 UA_MANDATORY));
2829
2830 DECLARE_UVERBS_NAMED_METHOD(
2831 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2832 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2833 UVERBS_IDR_ANY_OBJECT,
2834 UVERBS_ACCESS_READ,
2835 UA_MANDATORY),
2836 UVERBS_ATTR_PTR_IN(
2837 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2838 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2839 UA_MANDATORY,
2840 UA_ALLOC_AND_COPY),
2841 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2842 u16, UA_MANDATORY),
2843 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2844 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2845 UVERBS_ACCESS_READ,
2846 UA_MANDATORY),
2847 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2848 UVERBS_ATTR_TYPE(u64),
2849 UA_MANDATORY));
2850
2851 DECLARE_UVERBS_NAMED_METHOD(
2852 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2853 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2854 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2855 UVERBS_ACCESS_READ,
2856 UA_MANDATORY),
2857 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2858 MLX5_IB_OBJECT_DEVX_OBJ,
2859 UVERBS_ACCESS_READ,
2860 UA_OPTIONAL),
2861 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2862 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2863 UA_MANDATORY,
2864 UA_ALLOC_AND_COPY),
2865 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2866 UVERBS_ATTR_TYPE(u64),
2867 UA_OPTIONAL),
2868 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2869 UVERBS_ATTR_TYPE(u32),
2870 UA_OPTIONAL));
2871
2872 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
2873 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2874 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
2875 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2876 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
2877
2878 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
2879 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2880 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2881 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2882 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
2883 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2884 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
2885
2886 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
2887 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2888 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2889 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
2890
2891
2892 DECLARE_UVERBS_NAMED_METHOD(
2893 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2894 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2895 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2896 UVERBS_ACCESS_NEW,
2897 UA_MANDATORY));
2898
2899 DECLARE_UVERBS_NAMED_OBJECT(
2900 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2901 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
2902 devx_async_cmd_event_destroy_uobj,
2903 &devx_async_cmd_event_fops, "[devx_async_cmd]",
2904 O_RDONLY),
2905 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
2906
2907 DECLARE_UVERBS_NAMED_METHOD(
2908 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
2909 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
2910 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2911 UVERBS_ACCESS_NEW,
2912 UA_MANDATORY),
2913 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
2914 enum mlx5_ib_uapi_devx_create_event_channel_flags,
2915 UA_MANDATORY));
2916
2917 DECLARE_UVERBS_NAMED_OBJECT(
2918 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2919 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
2920 devx_async_event_destroy_uobj,
2921 &devx_async_event_fops, "[devx_async_event]",
2922 O_RDONLY),
2923 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
2924
2925 static bool devx_is_supported(struct ib_device *device)
2926 {
2927 struct mlx5_ib_dev *dev = to_mdev(device);
2928
2929 return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
2930 }
2931
2932 const struct uapi_definition mlx5_ib_devx_defs[] = {
2933 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2934 MLX5_IB_OBJECT_DEVX,
2935 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2936 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2937 MLX5_IB_OBJECT_DEVX_OBJ,
2938 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2939 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2940 MLX5_IB_OBJECT_DEVX_UMEM,
2941 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2942 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2943 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2944 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2945 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2946 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2947 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2948 {},
2949 };
Ubuntu Hirsute Linux kernel mirror