1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring.h>
6 #include <linux/security.h>
7 #include <linux/nospec.h>
9 #include <uapi/linux/io_uring.h>
13 #include "uring_cmd.h"
15 static void io_uring_cmd_work(struct io_kiocb
*req
, bool *locked
)
17 struct io_uring_cmd
*ioucmd
= io_kiocb_to_cmd(req
, struct io_uring_cmd
);
19 ioucmd
->task_work_cb(ioucmd
);
22 void io_uring_cmd_complete_in_task(struct io_uring_cmd
*ioucmd
,
23 void (*task_work_cb
)(struct io_uring_cmd
*))
25 struct io_kiocb
*req
= cmd_to_io_kiocb(ioucmd
);
27 ioucmd
->task_work_cb
= task_work_cb
;
28 req
->io_task_work
.func
= io_uring_cmd_work
;
29 io_req_task_work_add(req
);
31 EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task
);
33 static inline void io_req_set_cqe32_extra(struct io_kiocb
*req
,
34 u64 extra1
, u64 extra2
)
38 req
->flags
|= REQ_F_CQE32_INIT
;
42 * Called by consumers of io_uring_cmd, if they originally returned
43 * -EIOCBQUEUED upon receiving the command.
45 void io_uring_cmd_done(struct io_uring_cmd
*ioucmd
, ssize_t ret
, ssize_t res2
)
47 struct io_kiocb
*req
= cmd_to_io_kiocb(ioucmd
);
52 io_req_set_res(req
, ret
, 0);
53 if (req
->ctx
->flags
& IORING_SETUP_CQE32
)
54 io_req_set_cqe32_extra(req
, res2
, 0);
55 if (req
->ctx
->flags
& IORING_SETUP_IOPOLL
)
56 /* order with io_iopoll_req_issued() checking ->iopoll_complete */
57 smp_store_release(&req
->iopoll_completed
, 1);
59 io_req_complete_post(req
, 0);
61 EXPORT_SYMBOL_GPL(io_uring_cmd_done
);
63 int io_uring_cmd_prep_async(struct io_kiocb
*req
)
65 struct io_uring_cmd
*ioucmd
= io_kiocb_to_cmd(req
, struct io_uring_cmd
);
68 BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16);
69 BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80);
71 cmd_size
= uring_cmd_pdu_size(req
->ctx
->flags
& IORING_SETUP_SQE128
);
73 memcpy(req
->async_data
, ioucmd
->cmd
, cmd_size
);
77 int io_uring_cmd_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
79 struct io_uring_cmd
*ioucmd
= io_kiocb_to_cmd(req
, struct io_uring_cmd
);
84 ioucmd
->flags
= READ_ONCE(sqe
->uring_cmd_flags
);
85 if (ioucmd
->flags
& ~IORING_URING_CMD_FIXED
)
88 if (ioucmd
->flags
& IORING_URING_CMD_FIXED
) {
89 struct io_ring_ctx
*ctx
= req
->ctx
;
92 req
->buf_index
= READ_ONCE(sqe
->buf_index
);
93 if (unlikely(req
->buf_index
>= ctx
->nr_user_bufs
))
95 index
= array_index_nospec(req
->buf_index
, ctx
->nr_user_bufs
);
96 req
->imu
= ctx
->user_bufs
[index
];
97 io_req_set_rsrc_node(req
, ctx
, 0);
99 ioucmd
->cmd
= sqe
->cmd
;
100 ioucmd
->cmd_op
= READ_ONCE(sqe
->cmd_op
);
104 int io_uring_cmd(struct io_kiocb
*req
, unsigned int issue_flags
)
106 struct io_uring_cmd
*ioucmd
= io_kiocb_to_cmd(req
, struct io_uring_cmd
);
107 struct io_ring_ctx
*ctx
= req
->ctx
;
108 struct file
*file
= req
->file
;
111 if (!req
->file
->f_op
->uring_cmd
)
114 ret
= security_uring_cmd(ioucmd
);
118 if (ctx
->flags
& IORING_SETUP_SQE128
)
119 issue_flags
|= IO_URING_F_SQE128
;
120 if (ctx
->flags
& IORING_SETUP_CQE32
)
121 issue_flags
|= IO_URING_F_CQE32
;
122 if (ctx
->flags
& IORING_SETUP_IOPOLL
) {
123 issue_flags
|= IO_URING_F_IOPOLL
;
124 req
->iopoll_completed
= 0;
125 WRITE_ONCE(ioucmd
->cookie
, NULL
);
128 if (req_has_async_data(req
))
129 ioucmd
->cmd
= req
->async_data
;
131 ret
= file
->f_op
->uring_cmd(ioucmd
, issue_flags
);
132 if (ret
== -EAGAIN
) {
133 if (!req_has_async_data(req
)) {
134 if (io_alloc_async_data(req
))
136 io_uring_cmd_prep_async(req
);
141 if (ret
!= -EIOCBQUEUED
) {
144 io_req_set_res(req
, ret
, 0);
148 return IOU_ISSUE_SKIP_COMPLETE
;
151 int io_uring_cmd_import_fixed(u64 ubuf
, unsigned long len
, int rw
,
152 struct iov_iter
*iter
, void *ioucmd
)
154 struct io_kiocb
*req
= cmd_to_io_kiocb(ioucmd
);
156 return io_import_fixed(rw
, iter
, req
->imu
, ubuf
, len
);
158 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed
);