]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - fs/io_uring.c
io_uring: enable req cache for task_work items
[mirror_ubuntu-jammy-kernel.git] / fs / io_uring.c
CommitLineData
2b188cc1
JA
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
1e84b97b
SB
7 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
2b188cc1
JA
29 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
c992fe29 40 * Copyright (c) 2018-2019 Christoph Hellwig
2b188cc1
JA
41 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
52de1fe1 47#include <net/compat.h>
2b188cc1
JA
48#include <linux/refcount.h>
49#include <linux/uio.h>
6b47ee6e 50#include <linux/bits.h>
2b188cc1
JA
51
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
2b188cc1
JA
58#include <linux/percpu.h>
59#include <linux/slab.h>
6c271ce2 60#include <linux/kthread.h>
2b188cc1 61#include <linux/blkdev.h>
edafccee 62#include <linux/bvec.h>
2b188cc1
JA
63#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
6b06314c 66#include <net/scm.h>
2b188cc1
JA
67#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
edafccee
JA
71#include <linux/sizes.h>
72#include <linux/hugetlb.h>
aa4c3967 73#include <linux/highmem.h>
15b71abe
JA
74#include <linux/namei.h>
75#include <linux/fsnotify.h>
4840e418 76#include <linux/fadvise.h>
3e4827b0 77#include <linux/eventpoll.h>
ff002b30 78#include <linux/fs_struct.h>
7d67af2c 79#include <linux/splice.h>
b41e9852 80#include <linux/task_work.h>
bcf5a063 81#include <linux/pagemap.h>
0f212204 82#include <linux/io_uring.h>
91d8f519 83#include <linux/blk-cgroup.h>
4ea33a97 84#include <linux/audit.h>
2b188cc1 85
c826bd7a
DD
86#define CREATE_TRACE_POINTS
87#include <trace/events/io_uring.h>
88
2b188cc1
JA
89#include <uapi/linux/io_uring.h>
90
91#include "internal.h"
561fb04a 92#include "io-wq.h"
2b188cc1 93
5277deaa 94#define IORING_MAX_ENTRIES 32768
33a107f0 95#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
65e19f54
JA
96
97/*
98 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
99 */
100#define IORING_FILE_TABLE_SHIFT 9
101#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
102#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
103#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
21b55dbc
SG
104#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
105 IORING_REGISTER_LAST + IORING_OP_LAST)
2b188cc1
JA
106
107struct io_uring {
108 u32 head ____cacheline_aligned_in_smp;
109 u32 tail ____cacheline_aligned_in_smp;
110};
111
1e84b97b 112/*
75b28aff
HV
113 * This data is shared with the application through the mmap at offsets
114 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
1e84b97b
SB
115 *
116 * The offsets to the member fields are published through struct
117 * io_sqring_offsets when calling io_uring_setup.
118 */
75b28aff 119struct io_rings {
1e84b97b
SB
120 /*
121 * Head and tail offsets into the ring; the offsets need to be
122 * masked to get valid indices.
123 *
75b28aff
HV
124 * The kernel controls head of the sq ring and the tail of the cq ring,
125 * and the application controls tail of the sq ring and the head of the
126 * cq ring.
1e84b97b 127 */
75b28aff 128 struct io_uring sq, cq;
1e84b97b 129 /*
75b28aff 130 * Bitmasks to apply to head and tail offsets (constant, equals
1e84b97b
SB
131 * ring_entries - 1)
132 */
75b28aff
HV
133 u32 sq_ring_mask, cq_ring_mask;
134 /* Ring sizes (constant, power of 2) */
135 u32 sq_ring_entries, cq_ring_entries;
1e84b97b
SB
136 /*
137 * Number of invalid entries dropped by the kernel due to
138 * invalid index stored in array
139 *
140 * Written by the kernel, shouldn't be modified by the
141 * application (i.e. get number of "new events" by comparing to
142 * cached value).
143 *
144 * After a new SQ head value was read by the application this
145 * counter includes all submissions that were dropped reaching
146 * the new SQ head (and possibly more).
147 */
75b28aff 148 u32 sq_dropped;
1e84b97b 149 /*
0d9b5b3a 150 * Runtime SQ flags
1e84b97b
SB
151 *
152 * Written by the kernel, shouldn't be modified by the
153 * application.
154 *
155 * The application needs a full memory barrier before checking
156 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
157 */
75b28aff 158 u32 sq_flags;
0d9b5b3a
SG
159 /*
160 * Runtime CQ flags
161 *
162 * Written by the application, shouldn't be modified by the
163 * kernel.
164 */
165 u32 cq_flags;
1e84b97b
SB
166 /*
167 * Number of completion events lost because the queue was full;
168 * this should be avoided by the application by making sure
0b4295b5 169 * there are not more requests pending than there is space in
1e84b97b
SB
170 * the completion queue.
171 *
172 * Written by the kernel, shouldn't be modified by the
173 * application (i.e. get number of "new events" by comparing to
174 * cached value).
175 *
176 * As completion events come in out of order this counter is not
177 * ordered with any other data.
178 */
75b28aff 179 u32 cq_overflow;
1e84b97b
SB
180 /*
181 * Ring buffer of completion events.
182 *
183 * The kernel writes completion events fresh every time they are
184 * produced, so the application is allowed to modify pending
185 * entries.
186 */
75b28aff 187 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
2b188cc1
JA
188};
189
45d189c6
PB
190enum io_uring_cmd_flags {
191 IO_URING_F_NONBLOCK = 1,
889fca73 192 IO_URING_F_COMPLETE_DEFER = 2,
45d189c6
PB
193};
194
edafccee
JA
195struct io_mapped_ubuf {
196 u64 ubuf;
197 size_t len;
198 struct bio_vec *bvec;
199 unsigned int nr_bvecs;
de293938 200 unsigned long acct_pages;
edafccee
JA
201};
202
50238531
BM
203struct io_ring_ctx;
204
269bbe5f
BM
205struct io_rsrc_put {
206 struct list_head list;
50238531
BM
207 union {
208 void *rsrc;
209 struct file *file;
210 };
269bbe5f
BM
211};
212
213struct fixed_rsrc_table {
65e19f54 214 struct file **files;
31b51510
JA
215};
216
269bbe5f 217struct fixed_rsrc_ref_node {
05589553
XW
218 struct percpu_ref refs;
219 struct list_head node;
269bbe5f
BM
220 struct list_head rsrc_list;
221 struct fixed_rsrc_data *rsrc_data;
50238531
BM
222 void (*rsrc_put)(struct io_ring_ctx *ctx,
223 struct io_rsrc_put *prsrc);
4a38aed2 224 struct llist_node llist;
e297822b 225 bool done;
05589553
XW
226};
227
269bbe5f
BM
228struct fixed_rsrc_data {
229 struct fixed_rsrc_table *table;
05f3fb3c
JA
230 struct io_ring_ctx *ctx;
231
269bbe5f 232 struct fixed_rsrc_ref_node *node;
05f3fb3c 233 struct percpu_ref refs;
05f3fb3c
JA
234 struct completion done;
235};
236
5a2e745d
JA
237struct io_buffer {
238 struct list_head list;
239 __u64 addr;
240 __s32 len;
241 __u16 bid;
242};
243
21b55dbc
SG
244struct io_restriction {
245 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
246 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
247 u8 sqe_flags_allowed;
248 u8 sqe_flags_required;
7e84e1c7 249 bool registered;
21b55dbc
SG
250};
251
534ca6d6
JA
252struct io_sq_data {
253 refcount_t refs;
69fb2131
JA
254 struct mutex lock;
255
256 /* ctx's that are using this sqd */
257 struct list_head ctx_list;
258 struct list_head ctx_new_list;
259 struct mutex ctx_lock;
260
534ca6d6
JA
261 struct task_struct *thread;
262 struct wait_queue_head wait;
08369246
XW
263
264 unsigned sq_thread_idle;
534ca6d6
JA
265};
266
258b29a9 267#define IO_IOPOLL_BATCH 8
6dd0be1e 268#define IO_COMPL_BATCH 32
6ff119a6 269#define IO_REQ_CACHE_SIZE 32
bf019da7 270#define IO_REQ_ALLOC_BATCH 8
258b29a9
PB
271
272struct io_comp_state {
6dd0be1e 273 struct io_kiocb *reqs[IO_COMPL_BATCH];
1b4c351f
JA
274 unsigned int nr;
275 struct list_head free_list;
258b29a9
PB
276};
277
278struct io_submit_state {
279 struct blk_plug plug;
280
281 /*
282 * io_kiocb alloc cache
283 */
bf019da7 284 void *reqs[IO_REQ_CACHE_SIZE];
258b29a9
PB
285 unsigned int free_reqs;
286
287 bool plug_started;
288
289 /*
290 * Batch completion logic
291 */
292 struct io_comp_state comp;
293
294 /*
295 * File reference cache
296 */
297 struct file *file;
298 unsigned int fd;
299 unsigned int file_refs;
300 unsigned int ios_left;
301};
302
2b188cc1
JA
303struct io_ring_ctx {
304 struct {
305 struct percpu_ref refs;
306 } ____cacheline_aligned_in_smp;
307
308 struct {
309 unsigned int flags;
e1d85334 310 unsigned int compat: 1;
aad5d8da 311 unsigned int limit_mem: 1;
e1d85334
RD
312 unsigned int cq_overflow_flushed: 1;
313 unsigned int drain_next: 1;
314 unsigned int eventfd_async: 1;
21b55dbc 315 unsigned int restricted: 1;
d9d05217 316 unsigned int sqo_dead: 1;
2b188cc1 317
75b28aff
HV
318 /*
319 * Ring buffer of indices into array of io_uring_sqe, which is
320 * mmapped by the application using the IORING_OFF_SQES offset.
321 *
322 * This indirection could e.g. be used to assign fixed
323 * io_uring_sqe entries to operations and only submit them to
324 * the queue when needed.
325 *
326 * The kernel modifies neither the indices array nor the entries
327 * array.
328 */
329 u32 *sq_array;
2b188cc1
JA
330 unsigned cached_sq_head;
331 unsigned sq_entries;
332 unsigned sq_mask;
6c271ce2 333 unsigned sq_thread_idle;
498ccd9e 334 unsigned cached_sq_dropped;
2c3bac6d 335 unsigned cached_cq_overflow;
ad3eb2c8 336 unsigned long sq_check_overflow;
de0617e4
JA
337
338 struct list_head defer_list;
5262f567 339 struct list_head timeout_list;
1d7bb1d5 340 struct list_head cq_overflow_list;
fcb323cc 341
ad3eb2c8 342 struct io_uring_sqe *sq_sqes;
2b188cc1
JA
343 } ____cacheline_aligned_in_smp;
344
206aefde
JA
345 struct io_rings *rings;
346
2b188cc1 347 /* IO offload */
561fb04a 348 struct io_wq *io_wq;
2aede0e4
JA
349
350 /*
351 * For SQPOLL usage - we hold a reference to the parent task, so we
352 * have access to the ->files
353 */
354 struct task_struct *sqo_task;
355
356 /* Only used for accounting purposes */
357 struct mm_struct *mm_account;
358
91d8f519
DZ
359#ifdef CONFIG_BLK_CGROUP
360 struct cgroup_subsys_state *sqo_blkcg_css;
361#endif
362
534ca6d6
JA
363 struct io_sq_data *sq_data; /* if using sq thread polling */
364
90554200 365 struct wait_queue_head sqo_sq_wait;
69fb2131 366 struct list_head sqd_list;
75b28aff 367
6b06314c
JA
368 /*
369 * If used, fixed file set. Writers must ensure that ->refs is dead,
370 * readers must ensure that ->refs is alive as long as the file* is
371 * used. Only updated through io_uring_register(2).
372 */
269bbe5f 373 struct fixed_rsrc_data *file_data;
6b06314c
JA
374 unsigned nr_user_files;
375
edafccee
JA
376 /* if used, fixed mapped user buffers */
377 unsigned nr_user_bufs;
378 struct io_mapped_ubuf *user_bufs;
379
2b188cc1
JA
380 struct user_struct *user;
381
0b8c0ec7 382 const struct cred *creds;
181e448d 383
4ea33a97
JA
384#ifdef CONFIG_AUDIT
385 kuid_t loginuid;
386 unsigned int sessionid;
387#endif
388
0f158b4c
JA
389 struct completion ref_comp;
390 struct completion sq_thread_comp;
206aefde
JA
391
392#if defined(CONFIG_UNIX)
393 struct socket *ring_sock;
394#endif
395
5a2e745d
JA
396 struct idr io_buffer_idr;
397
071698e1
JA
398 struct idr personality_idr;
399
206aefde
JA
400 struct {
401 unsigned cached_cq_tail;
402 unsigned cq_entries;
403 unsigned cq_mask;
404 atomic_t cq_timeouts;
f010505b 405 unsigned cq_last_tm_flush;
ad3eb2c8 406 unsigned long cq_check_overflow;
206aefde
JA
407 struct wait_queue_head cq_wait;
408 struct fasync_struct *cq_fasync;
409 struct eventfd_ctx *cq_ev_fd;
410 } ____cacheline_aligned_in_smp;
2b188cc1
JA
411
412 struct {
413 struct mutex uring_lock;
414 wait_queue_head_t wait;
415 } ____cacheline_aligned_in_smp;
416
417 struct {
418 spinlock_t completion_lock;
e94f141b 419
def596e9 420 /*
540e32a0 421 * ->iopoll_list is protected by the ctx->uring_lock for
def596e9
JA
422 * io_uring instances that don't use IORING_SETUP_SQPOLL.
423 * For SQPOLL, only the single threaded io_sq_thread() will
424 * manipulate the list, hence no extra locking is needed there.
425 */
540e32a0 426 struct list_head iopoll_list;
78076bb6
JA
427 struct hlist_head *cancel_hash;
428 unsigned cancel_hash_bits;
e94f141b 429 bool poll_multi_file;
31b51510 430
fcb323cc
JA
431 spinlock_t inflight_lock;
432 struct list_head inflight_list;
2b188cc1 433 } ____cacheline_aligned_in_smp;
85faa7b8 434
269bbe5f
BM
435 struct delayed_work rsrc_put_work;
436 struct llist_head rsrc_put_llist;
d67d2263
BM
437 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;
4a38aed2 439
85faa7b8 440 struct work_struct exit_work;
21b55dbc 441 struct io_restriction restrictions;
258b29a9 442 struct io_submit_state submit_state;
2b188cc1
JA
443};
444
09bb8394
JA
445/*
446 * First field must be the file pointer in all the
447 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
448 */
221c5eb2
JA
449struct io_poll_iocb {
450 struct file *file;
018043be 451 struct wait_queue_head *head;
221c5eb2 452 __poll_t events;
8c838788 453 bool done;
221c5eb2 454 bool canceled;
392edb45 455 struct wait_queue_entry wait;
221c5eb2
JA
456};
457
018043be
PB
458struct io_poll_remove {
459 struct file *file;
460 u64 addr;
461};
462
b5dba59e
JA
463struct io_close {
464 struct file *file;
b5dba59e
JA
465 int fd;
466};
467
ad8a48ac
JA
468struct io_timeout_data {
469 struct io_kiocb *req;
470 struct hrtimer timer;
471 struct timespec64 ts;
472 enum hrtimer_mode mode;
473};
474
8ed8d3c3
JA
475struct io_accept {
476 struct file *file;
477 struct sockaddr __user *addr;
478 int __user *addr_len;
479 int flags;
09952e3e 480 unsigned long nofile;
8ed8d3c3
JA
481};
482
483struct io_sync {
484 struct file *file;
485 loff_t len;
486 loff_t off;
487 int flags;
d63d1b5e 488 int mode;
8ed8d3c3
JA
489};
490
fbf23849
JA
491struct io_cancel {
492 struct file *file;
493 u64 addr;
494};
495
b29472ee
JA
496struct io_timeout {
497 struct file *file;
bfe68a22
PB
498 u32 off;
499 u32 target_seq;
135fcde8 500 struct list_head list;
90cd7e42
PB
501 /* head of the link, used by linked timeouts only */
502 struct io_kiocb *head;
b29472ee
JA
503};
504
0bdf7a2d
PB
505struct io_timeout_rem {
506 struct file *file;
507 u64 addr;
9c8e11b3
PB
508
509 /* timeout update */
510 struct timespec64 ts;
511 u32 flags;
0bdf7a2d
PB
512};
513
9adbd45d
JA
514struct io_rw {
515 /* NOTE: kiocb has the file as the first member, so don't do it here */
516 struct kiocb kiocb;
517 u64 addr;
518 u64 len;
519};
520
3fbb51c1
JA
521struct io_connect {
522 struct file *file;
523 struct sockaddr __user *addr;
524 int addr_len;
525};
526
e47293fd
JA
527struct io_sr_msg {
528 struct file *file;
fddaface 529 union {
270a5940 530 struct user_msghdr __user *umsg;
fddaface
JA
531 void __user *buf;
532 };
e47293fd 533 int msg_flags;
bcda7baa 534 int bgid;
fddaface 535 size_t len;
bcda7baa 536 struct io_buffer *kbuf;
e47293fd
JA
537};
538
15b71abe
JA
539struct io_open {
540 struct file *file;
541 int dfd;
15b71abe 542 struct filename *filename;
c12cedf2 543 struct open_how how;
4022e7af 544 unsigned long nofile;
15b71abe
JA
545};
546
269bbe5f 547struct io_rsrc_update {
05f3fb3c
JA
548 struct file *file;
549 u64 arg;
550 u32 nr_args;
551 u32 offset;
552};
553
4840e418
JA
554struct io_fadvise {
555 struct file *file;
556 u64 offset;
557 u32 len;
558 u32 advice;
559};
560
c1ca757b
JA
561struct io_madvise {
562 struct file *file;
563 u64 addr;
564 u32 len;
565 u32 advice;
566};
567
3e4827b0
JA
568struct io_epoll {
569 struct file *file;
570 int epfd;
571 int op;
572 int fd;
573 struct epoll_event event;
e47293fd
JA
574};
575
7d67af2c
PB
576struct io_splice {
577 struct file *file_out;
578 struct file *file_in;
579 loff_t off_out;
580 loff_t off_in;
581 u64 len;
582 unsigned int flags;
583};
584
ddf0322d
JA
585struct io_provide_buf {
586 struct file *file;
587 __u64 addr;
588 __s32 len;
589 __u32 bgid;
590 __u16 nbufs;
591 __u16 bid;
592};
593
1d9e1288
BM
594struct io_statx {
595 struct file *file;
596 int dfd;
597 unsigned int mask;
598 unsigned int flags;
e62753e4 599 const char __user *filename;
1d9e1288
BM
600 struct statx __user *buffer;
601};
602
36f4fa68
JA
603struct io_shutdown {
604 struct file *file;
605 int how;
606};
607
80a261fd
JA
608struct io_rename {
609 struct file *file;
610 int old_dfd;
611 int new_dfd;
612 struct filename *oldpath;
613 struct filename *newpath;
614 int flags;
615};
616
14a1143b
JA
617struct io_unlink {
618 struct file *file;
619 int dfd;
620 int flags;
621 struct filename *filename;
622};
623
3ca405eb
PB
624struct io_completion {
625 struct file *file;
626 struct list_head list;
0f7e466b 627 int cflags;
3ca405eb
PB
628};
629
f499a021
JA
630struct io_async_connect {
631 struct sockaddr_storage address;
632};
633
03b1230c
JA
634struct io_async_msghdr {
635 struct iovec fast_iov[UIO_FASTIOV];
257e84a5
PB
636 /* points to an allocated iov, if NULL we use fast_iov instead */
637 struct iovec *free_iov;
03b1230c
JA
638 struct sockaddr __user *uaddr;
639 struct msghdr msg;
b537916c 640 struct sockaddr_storage addr;
03b1230c
JA
641};
642
f67676d1
JA
643struct io_async_rw {
644 struct iovec fast_iov[UIO_FASTIOV];
ff6165b2
JA
645 const struct iovec *free_iovec;
646 struct iov_iter iter;
227c0c96 647 size_t bytes_done;
bcf5a063 648 struct wait_page_queue wpq;
f67676d1
JA
649};
650
6b47ee6e
PB
651enum {
652 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
653 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
654 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
655 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
656 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
bcda7baa 657 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
6b47ee6e 658
6b47ee6e
PB
659 REQ_F_FAIL_LINK_BIT,
660 REQ_F_INFLIGHT_BIT,
661 REQ_F_CUR_POS_BIT,
662 REQ_F_NOWAIT_BIT,
6b47ee6e 663 REQ_F_LINK_TIMEOUT_BIT,
6b47ee6e 664 REQ_F_ISREG_BIT,
99bc4c38 665 REQ_F_NEED_CLEANUP_BIT,
d7718a9d 666 REQ_F_POLLED_BIT,
bcda7baa 667 REQ_F_BUFFER_SELECTED_BIT,
5b0bbee4 668 REQ_F_NO_FILE_TABLE_BIT,
7cdaf587 669 REQ_F_WORK_INITIALIZED_BIT,
900fad45 670 REQ_F_LTIMEOUT_ACTIVE_BIT,
e342c807 671 REQ_F_COMPLETE_INLINE_BIT,
84557871
JA
672
673 /* not a real bit, just to check we're not overflowing the space */
674 __REQ_F_LAST_BIT,
6b47ee6e
PB
675};
676
677enum {
678 /* ctx owns file */
679 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
680 /* drain existing IO first */
681 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
682 /* linked sqes */
683 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
684 /* doesn't sever on completion < 0 */
685 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
686 /* IOSQE_ASYNC */
687 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
bcda7baa
JA
688 /* IOSQE_BUFFER_SELECT */
689 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
6b47ee6e 690
6b47ee6e
PB
691 /* fail rest of links */
692 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
693 /* on inflight list */
694 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
695 /* read/write uses file position */
696 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
697 /* must not punt to workers */
698 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
900fad45 699 /* has or had linked timeout */
6b47ee6e 700 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
6b47ee6e
PB
701 /* regular file */
702 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
99bc4c38
PB
703 /* needs cleanup */
704 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
d7718a9d
JA
705 /* already went through poll handler */
706 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
bcda7baa
JA
707 /* buffer already selected */
708 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
5b0bbee4
JA
709 /* doesn't need file table for this request */
710 REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
7cdaf587
XW
711 /* io_wq_work is initialized */
712 REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
900fad45
PB
713 /* linked timeout is active, i.e. prepared by link's head */
714 REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
e342c807
PB
715 /* completion is deferred through io_comp_state */
716 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
d7718a9d
JA
717};
718
719struct async_poll {
720 struct io_poll_iocb poll;
807abcb0 721 struct io_poll_iocb *double_poll;
6b47ee6e
PB
722};
723
7cbf1722
JA
724struct io_task_work {
725 struct io_wq_work_node node;
726 task_work_func_t func;
727};
728
09bb8394
JA
729/*
730 * NOTE! Each of the iocb union members has the file pointer
731 * as the first entry in their struct definition. So you can
732 * access the file pointer through any of the sub-structs,
733 * or directly as just 'ki_filp' in this struct.
734 */
2b188cc1 735struct io_kiocb {
221c5eb2 736 union {
09bb8394 737 struct file *file;
9adbd45d 738 struct io_rw rw;
221c5eb2 739 struct io_poll_iocb poll;
018043be 740 struct io_poll_remove poll_remove;
8ed8d3c3
JA
741 struct io_accept accept;
742 struct io_sync sync;
fbf23849 743 struct io_cancel cancel;
b29472ee 744 struct io_timeout timeout;
0bdf7a2d 745 struct io_timeout_rem timeout_rem;
3fbb51c1 746 struct io_connect connect;
e47293fd 747 struct io_sr_msg sr_msg;
15b71abe 748 struct io_open open;
b5dba59e 749 struct io_close close;
269bbe5f 750 struct io_rsrc_update rsrc_update;
4840e418 751 struct io_fadvise fadvise;
c1ca757b 752 struct io_madvise madvise;
3e4827b0 753 struct io_epoll epoll;
7d67af2c 754 struct io_splice splice;
ddf0322d 755 struct io_provide_buf pbuf;
1d9e1288 756 struct io_statx statx;
36f4fa68 757 struct io_shutdown shutdown;
80a261fd 758 struct io_rename rename;
14a1143b 759 struct io_unlink unlink;
3ca405eb
PB
760 /* use only after cleaning per-op data, see io_clean_op() */
761 struct io_completion compl;
221c5eb2 762 };
2b188cc1 763
e8c2bc1f
JA
764 /* opcode allocated if it needs to store data for async defer */
765 void *async_data;
d625c6ee 766 u8 opcode;
65a6543d
XW
767 /* polled IO has completed */
768 u8 iopoll_completed;
2b188cc1 769
4f4eeba8 770 u16 buf_index;
9cf7c104 771 u32 result;
4f4eeba8 772
010e8e6b
PB
773 struct io_ring_ctx *ctx;
774 unsigned int flags;
775 refcount_t refs;
776 struct task_struct *task;
777 u64 user_data;
d7718a9d 778
f2f87370 779 struct io_kiocb *link;
269bbe5f 780 struct percpu_ref *fixed_rsrc_refs;
fcb323cc 781
d21ffe7e
PB
782 /*
783 * 1. used with ctx->iopoll_list with reads/writes
784 * 2. to track reqs with ->files (see io_op_def::file_table)
785 */
010e8e6b 786 struct list_head inflight_entry;
7cbf1722
JA
787 union {
788 struct io_task_work io_task_work;
789 struct callback_head task_work;
790 };
010e8e6b
PB
791 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
792 struct hlist_node hash_node;
793 struct async_poll *apoll;
794 struct io_wq_work work;
2b188cc1 795};
05589553 796
27dc8338
PB
797struct io_defer_entry {
798 struct list_head list;
799 struct io_kiocb *req;
9cf7c104 800 u32 seq;
2b188cc1
JA
801};
802
d3656344 803struct io_op_def {
d3656344
JA
804 /* needs req->file assigned */
805 unsigned needs_file : 1;
d3656344
JA
806 /* hash wq insertion if file is a regular file */
807 unsigned hash_reg_file : 1;
808 /* unbound wq insertion if file is a non-regular file */
809 unsigned unbound_nonreg_file : 1;
66f4af93
JA
810 /* opcode is not supported by this kernel */
811 unsigned not_supported : 1;
8a72758c
JA
812 /* set if opcode supports polled "wait" */
813 unsigned pollin : 1;
814 unsigned pollout : 1;
bcda7baa
JA
815 /* op supports buffer selection */
816 unsigned buffer_select : 1;
e8c2bc1f
JA
817 /* must always have async data allocated */
818 unsigned needs_async_data : 1;
27926b68
JA
819 /* should block plug */
820 unsigned plug : 1;
e8c2bc1f
JA
821 /* size of async data needed, if any */
822 unsigned short async_size;
0f203765 823 unsigned work_flags;
d3656344
JA
824};
825
0918682b 826static const struct io_op_def io_op_defs[] = {
0463b6c5
PB
827 [IORING_OP_NOP] = {},
828 [IORING_OP_READV] = {
d3656344
JA
829 .needs_file = 1,
830 .unbound_nonreg_file = 1,
8a72758c 831 .pollin = 1,
4d954c25 832 .buffer_select = 1,
e8c2bc1f 833 .needs_async_data = 1,
27926b68 834 .plug = 1,
e8c2bc1f 835 .async_size = sizeof(struct io_async_rw),
0f203765 836 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
d3656344 837 },
0463b6c5 838 [IORING_OP_WRITEV] = {
d3656344
JA
839 .needs_file = 1,
840 .hash_reg_file = 1,
841 .unbound_nonreg_file = 1,
8a72758c 842 .pollout = 1,
e8c2bc1f 843 .needs_async_data = 1,
27926b68 844 .plug = 1,
e8c2bc1f 845 .async_size = sizeof(struct io_async_rw),
69228338
JA
846 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG |
847 IO_WQ_WORK_FSIZE,
d3656344 848 },
0463b6c5 849 [IORING_OP_FSYNC] = {
d3656344 850 .needs_file = 1,
0f203765 851 .work_flags = IO_WQ_WORK_BLKCG,
d3656344 852 },
0463b6c5 853 [IORING_OP_READ_FIXED] = {
d3656344
JA
854 .needs_file = 1,
855 .unbound_nonreg_file = 1,
8a72758c 856 .pollin = 1,
27926b68 857 .plug = 1,
e8c2bc1f 858 .async_size = sizeof(struct io_async_rw),
4017eb91 859 .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM,
d3656344 860 },
0463b6c5 861 [IORING_OP_WRITE_FIXED] = {
d3656344
JA
862 .needs_file = 1,
863 .hash_reg_file = 1,
864 .unbound_nonreg_file = 1,
8a72758c 865 .pollout = 1,
27926b68 866 .plug = 1,
e8c2bc1f 867 .async_size = sizeof(struct io_async_rw),
4017eb91
JA
868 .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE |
869 IO_WQ_WORK_MM,
d3656344 870 },
0463b6c5 871 [IORING_OP_POLL_ADD] = {
d3656344
JA
872 .needs_file = 1,
873 .unbound_nonreg_file = 1,
874 },
0463b6c5
PB
875 [IORING_OP_POLL_REMOVE] = {},
876 [IORING_OP_SYNC_FILE_RANGE] = {
d3656344 877 .needs_file = 1,
0f203765 878 .work_flags = IO_WQ_WORK_BLKCG,
d3656344 879 },
0463b6c5 880 [IORING_OP_SENDMSG] = {
d3656344
JA
881 .needs_file = 1,
882 .unbound_nonreg_file = 1,
8a72758c 883 .pollout = 1,
e8c2bc1f
JA
884 .needs_async_data = 1,
885 .async_size = sizeof(struct io_async_msghdr),
10cad2c4 886 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
d3656344 887 },
0463b6c5 888 [IORING_OP_RECVMSG] = {
d3656344
JA
889 .needs_file = 1,
890 .unbound_nonreg_file = 1,
8a72758c 891 .pollin = 1,
52de1fe1 892 .buffer_select = 1,
e8c2bc1f
JA
893 .needs_async_data = 1,
894 .async_size = sizeof(struct io_async_msghdr),
10cad2c4 895 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
d3656344 896 },
0463b6c5 897 [IORING_OP_TIMEOUT] = {
e8c2bc1f
JA
898 .needs_async_data = 1,
899 .async_size = sizeof(struct io_timeout_data),
0f203765 900 .work_flags = IO_WQ_WORK_MM,
d3656344 901 },
9c8e11b3
PB
902 [IORING_OP_TIMEOUT_REMOVE] = {
903 /* used by timeout updates' prep() */
904 .work_flags = IO_WQ_WORK_MM,
905 },
0463b6c5 906 [IORING_OP_ACCEPT] = {
d3656344
JA
907 .needs_file = 1,
908 .unbound_nonreg_file = 1,
8a72758c 909 .pollin = 1,
0f203765 910 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES,
d3656344 911 },
0463b6c5
PB
912 [IORING_OP_ASYNC_CANCEL] = {},
913 [IORING_OP_LINK_TIMEOUT] = {
e8c2bc1f
JA
914 .needs_async_data = 1,
915 .async_size = sizeof(struct io_timeout_data),
0f203765 916 .work_flags = IO_WQ_WORK_MM,
d3656344 917 },
0463b6c5 918 [IORING_OP_CONNECT] = {
d3656344
JA
919 .needs_file = 1,
920 .unbound_nonreg_file = 1,
8a72758c 921 .pollout = 1,
e8c2bc1f
JA
922 .needs_async_data = 1,
923 .async_size = sizeof(struct io_async_connect),
0f203765 924 .work_flags = IO_WQ_WORK_MM,
d3656344 925 },
0463b6c5 926 [IORING_OP_FALLOCATE] = {
d3656344 927 .needs_file = 1,
69228338 928 .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE,
d3656344 929 },
0463b6c5 930 [IORING_OP_OPENAT] = {
0f203765 931 .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG |
14587a46 932 IO_WQ_WORK_FS | IO_WQ_WORK_MM,
d3656344 933 },
0463b6c5 934 [IORING_OP_CLOSE] = {
0f203765 935 .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG,
d3656344 936 },
0463b6c5 937 [IORING_OP_FILES_UPDATE] = {
0f203765 938 .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM,
d3656344 939 },
0463b6c5 940 [IORING_OP_STATX] = {
0f203765
JA
941 .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM |
942 IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG,
d3656344 943 },
0463b6c5 944 [IORING_OP_READ] = {
3a6820f2
JA
945 .needs_file = 1,
946 .unbound_nonreg_file = 1,
8a72758c 947 .pollin = 1,
bcda7baa 948 .buffer_select = 1,
27926b68 949 .plug = 1,
e8c2bc1f 950 .async_size = sizeof(struct io_async_rw),
0f203765 951 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
3a6820f2 952 },
0463b6c5 953 [IORING_OP_WRITE] = {
3a6820f2
JA
954 .needs_file = 1,
955 .unbound_nonreg_file = 1,
8a72758c 956 .pollout = 1,
27926b68 957 .plug = 1,
e8c2bc1f 958 .async_size = sizeof(struct io_async_rw),
69228338
JA
959 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG |
960 IO_WQ_WORK_FSIZE,
3a6820f2 961 },
0463b6c5 962 [IORING_OP_FADVISE] = {
4840e418 963 .needs_file = 1,
0f203765 964 .work_flags = IO_WQ_WORK_BLKCG,
4840e418 965 },
0463b6c5 966 [IORING_OP_MADVISE] = {
0f203765 967 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
c1ca757b 968 },
0463b6c5 969 [IORING_OP_SEND] = {
fddaface
JA
970 .needs_file = 1,
971 .unbound_nonreg_file = 1,
8a72758c 972 .pollout = 1,
0f203765 973 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
fddaface 974 },
0463b6c5 975 [IORING_OP_RECV] = {
fddaface
JA
976 .needs_file = 1,
977 .unbound_nonreg_file = 1,
8a72758c 978 .pollin = 1,
bcda7baa 979 .buffer_select = 1,
0f203765 980 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG,
fddaface 981 },
0463b6c5 982 [IORING_OP_OPENAT2] = {
0f203765 983 .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_FS |
14587a46 984 IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM,
cebdb986 985 },
3e4827b0
JA
986 [IORING_OP_EPOLL_CTL] = {
987 .unbound_nonreg_file = 1,
0f203765 988 .work_flags = IO_WQ_WORK_FILES,
3e4827b0 989 },
7d67af2c
PB
990 [IORING_OP_SPLICE] = {
991 .needs_file = 1,
992 .hash_reg_file = 1,
993 .unbound_nonreg_file = 1,
0f203765 994 .work_flags = IO_WQ_WORK_BLKCG,
ddf0322d
JA
995 },
996 [IORING_OP_PROVIDE_BUFFERS] = {},
067524e9 997 [IORING_OP_REMOVE_BUFFERS] = {},
f2a8d5c7
PB
998 [IORING_OP_TEE] = {
999 .needs_file = 1,
1000 .hash_reg_file = 1,
1001 .unbound_nonreg_file = 1,
1002 },
36f4fa68
JA
1003 [IORING_OP_SHUTDOWN] = {
1004 .needs_file = 1,
1005 },
80a261fd
JA
1006 [IORING_OP_RENAMEAT] = {
1007 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES |
1008 IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG,
1009 },
14a1143b
JA
1010 [IORING_OP_UNLINKAT] = {
1011 .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES |
1012 IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG,
1013 },
d3656344
JA
1014};
1015
2e0464d4
BM
1016enum io_mem_account {
1017 ACCT_LOCKED,
1018 ACCT_PINNED,
1019};
1020
9936c7c2
PB
1021static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1022 struct task_struct *task,
1023 struct files_struct *files);
269bbe5f 1024static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node);
bc9744cd 1025static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(
1ffc5422 1026 struct io_ring_ctx *ctx);
bc9744cd
PB
1027static void init_fixed_file_ref_node(struct io_ring_ctx *ctx,
1028 struct fixed_rsrc_ref_node *ref_node);
1ffc5422 1029
81b68a5c 1030static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
889fca73 1031 unsigned int issue_flags);
78e19bbe 1032static void io_cqring_fill_event(struct io_kiocb *req, long res);
ec9c02ad 1033static void io_put_req(struct io_kiocb *req);
216578e5 1034static void io_put_req_deferred(struct io_kiocb *req, int nr);
c40f6379 1035static void io_double_put_req(struct io_kiocb *req);
94ae5e77 1036static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
7271ef3a 1037static void __io_queue_linked_timeout(struct io_kiocb *req);
94ae5e77 1038static void io_queue_linked_timeout(struct io_kiocb *req);
05f3fb3c 1039static int __io_sqe_files_update(struct io_ring_ctx *ctx,
269bbe5f 1040 struct io_uring_rsrc_update *ip,
05f3fb3c 1041 unsigned nr_args);
3ca405eb 1042static void __io_clean_op(struct io_kiocb *req);
8371adf5
PB
1043static struct file *io_file_get(struct io_submit_state *state,
1044 struct io_kiocb *req, int fd, bool fixed);
c1379e24 1045static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs);
269bbe5f 1046static void io_rsrc_put_work(struct work_struct *work);
de0617e4 1047
847595de
PB
1048static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
1049 struct iov_iter *iter, bool needs_lock);
ff6165b2
JA
1050static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
1051 const struct iovec *fast_iov,
227c0c96 1052 struct iov_iter *iter, bool force);
907d1df3 1053static void io_req_task_queue(struct io_kiocb *req);
65453d1e
JA
1054static void io_submit_flush_completions(struct io_comp_state *cs,
1055 struct io_ring_ctx *ctx);
de0617e4 1056
2b188cc1
JA
1057static struct kmem_cache *req_cachep;
1058
0918682b 1059static const struct file_operations io_uring_fops;
2b188cc1
JA
1060
1061struct sock *io_uring_get_socket(struct file *file)
1062{
1063#if defined(CONFIG_UNIX)
1064 if (file->f_op == &io_uring_fops) {
1065 struct io_ring_ctx *ctx = file->private_data;
1066
1067 return ctx->ring_sock->sk;
1068 }
1069#endif
1070 return NULL;
1071}
1072EXPORT_SYMBOL(io_uring_get_socket);
1073
f2f87370
PB
1074#define io_for_each_link(pos, head) \
1075 for (pos = (head); pos; pos = pos->link)
1076
3ca405eb
PB
1077static inline void io_clean_op(struct io_kiocb *req)
1078{
9d5c8190 1079 if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED))
3ca405eb
PB
1080 __io_clean_op(req);
1081}
1082
36f72fe2
PB
1083static inline void io_set_resource_node(struct io_kiocb *req)
1084{
1085 struct io_ring_ctx *ctx = req->ctx;
1086
269bbe5f
BM
1087 if (!req->fixed_rsrc_refs) {
1088 req->fixed_rsrc_refs = &ctx->file_data->node->refs;
1089 percpu_ref_get(req->fixed_rsrc_refs);
36f72fe2
PB
1090 }
1091}
1092
08d23634
PB
1093static bool io_match_task(struct io_kiocb *head,
1094 struct task_struct *task,
1095 struct files_struct *files)
1096{
1097 struct io_kiocb *req;
1098
84965ff8
JA
1099 if (task && head->task != task) {
1100 /* in terms of cancelation, always match if req task is dead */
1101 if (head->task->flags & PF_EXITING)
1102 return true;
08d23634 1103 return false;
84965ff8 1104 }
08d23634
PB
1105 if (!files)
1106 return true;
1107
1108 io_for_each_link(req, head) {
02a13674
JA
1109 if (!(req->flags & REQ_F_WORK_INITIALIZED))
1110 continue;
1111 if (req->file && req->file->f_op == &io_uring_fops)
1112 return true;
1113 if ((req->work.flags & IO_WQ_WORK_FILES) &&
08d23634
PB
1114 req->work.identity->files == files)
1115 return true;
1116 }
1117 return false;
1118}
1119
28cea78a 1120static void io_sq_thread_drop_mm_files(void)
c40f6379 1121{
28cea78a 1122 struct files_struct *files = current->files;
c40f6379
JA
1123 struct mm_struct *mm = current->mm;
1124
1125 if (mm) {
1126 kthread_unuse_mm(mm);
1127 mmput(mm);
4b70cf9d 1128 current->mm = NULL;
c40f6379 1129 }
28cea78a
JA
1130 if (files) {
1131 struct nsproxy *nsproxy = current->nsproxy;
1132
1133 task_lock(current);
1134 current->files = NULL;
1135 current->nsproxy = NULL;
1136 task_unlock(current);
1137 put_files_struct(files);
1138 put_nsproxy(nsproxy);
1139 }
1140}
1141
1a38ffc9 1142static int __io_sq_thread_acquire_files(struct io_ring_ctx *ctx)
28cea78a 1143{
621fadc2
PB
1144 if (current->flags & PF_EXITING)
1145 return -EFAULT;
1146
28cea78a
JA
1147 if (!current->files) {
1148 struct files_struct *files;
1149 struct nsproxy *nsproxy;
1150
1151 task_lock(ctx->sqo_task);
1152 files = ctx->sqo_task->files;
1153 if (!files) {
1154 task_unlock(ctx->sqo_task);
1a38ffc9 1155 return -EOWNERDEAD;
28cea78a
JA
1156 }
1157 atomic_inc(&files->count);
1158 get_nsproxy(ctx->sqo_task->nsproxy);
1159 nsproxy = ctx->sqo_task->nsproxy;
1160 task_unlock(ctx->sqo_task);
1161
1162 task_lock(current);
1163 current->files = files;
1164 current->nsproxy = nsproxy;
1165 task_unlock(current);
1166 }
1a38ffc9 1167 return 0;
c40f6379
JA
1168}
1169
1170static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
1171{
4b70cf9d
JA
1172 struct mm_struct *mm;
1173
621fadc2
PB
1174 if (current->flags & PF_EXITING)
1175 return -EFAULT;
4b70cf9d
JA
1176 if (current->mm)
1177 return 0;
1178
1179 /* Should never happen */
1180 if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL)))
1181 return -EFAULT;
1182
1183 task_lock(ctx->sqo_task);
1184 mm = ctx->sqo_task->mm;
1185 if (unlikely(!mm || !mmget_not_zero(mm)))
1186 mm = NULL;
1187 task_unlock(ctx->sqo_task);
1188
1189 if (mm) {
1190 kthread_use_mm(mm);
1191 return 0;
c40f6379
JA
1192 }
1193
4b70cf9d 1194 return -EFAULT;
c40f6379
JA
1195}
1196
28cea78a
JA
1197static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx,
1198 struct io_kiocb *req)
c40f6379 1199{
28cea78a 1200 const struct io_op_def *def = &io_op_defs[req->opcode];
1a38ffc9 1201 int ret;
28cea78a
JA
1202
1203 if (def->work_flags & IO_WQ_WORK_MM) {
1a38ffc9 1204 ret = __io_sq_thread_acquire_mm(ctx);
28cea78a
JA
1205 if (unlikely(ret))
1206 return ret;
1207 }
1208
1a38ffc9
PB
1209 if (def->needs_file || (def->work_flags & IO_WQ_WORK_FILES)) {
1210 ret = __io_sq_thread_acquire_files(ctx);
1211 if (unlikely(ret))
1212 return ret;
1213 }
28cea78a
JA
1214
1215 return 0;
c40f6379
JA
1216}
1217
91d8f519
DZ
1218static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx,
1219 struct cgroup_subsys_state **cur_css)
1220
1221{
1222#ifdef CONFIG_BLK_CGROUP
1223 /* puts the old one when swapping */
1224 if (*cur_css != ctx->sqo_blkcg_css) {
1225 kthread_associate_blkcg(ctx->sqo_blkcg_css);
1226 *cur_css = ctx->sqo_blkcg_css;
1227 }
1228#endif
1229}
1230
1231static void io_sq_thread_unassociate_blkcg(void)
1232{
1233#ifdef CONFIG_BLK_CGROUP
1234 kthread_associate_blkcg(NULL);
1235#endif
1236}
1237
c40f6379
JA
1238static inline void req_set_fail_links(struct io_kiocb *req)
1239{
1240 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
1241 req->flags |= REQ_F_FAIL_LINK;
1242}
4a38aed2 1243
1e6fa521
JA
1244/*
1245 * None of these are dereferenced, they are simply used to check if any of
1246 * them have changed. If we're under current and check they are still the
1247 * same, we're fine to grab references to them for actual out-of-line use.
1248 */
1249static void io_init_identity(struct io_identity *id)
1250{
1251 id->files = current->files;
1252 id->mm = current->mm;
1253#ifdef CONFIG_BLK_CGROUP
1254 rcu_read_lock();
1255 id->blkcg_css = blkcg_css();
1256 rcu_read_unlock();
1257#endif
1258 id->creds = current_cred();
1259 id->nsproxy = current->nsproxy;
1260 id->fs = current->fs;
1261 id->fsize = rlimit(RLIMIT_FSIZE);
4ea33a97
JA
1262#ifdef CONFIG_AUDIT
1263 id->loginuid = current->loginuid;
1264 id->sessionid = current->sessionid;
1265#endif
1e6fa521
JA
1266 refcount_set(&id->count, 1);
1267}
1268
ec99ca6c
PB
1269static inline void __io_req_init_async(struct io_kiocb *req)
1270{
1271 memset(&req->work, 0, sizeof(req->work));
1272 req->flags |= REQ_F_WORK_INITIALIZED;
1273}
1274
7cdaf587
XW
1275/*
1276 * Note: must call io_req_init_async() for the first time you
1277 * touch any members of io_wq_work.
1278 */
1279static inline void io_req_init_async(struct io_kiocb *req)
1280{
500a373d
JA
1281 struct io_uring_task *tctx = current->io_uring;
1282
7cdaf587
XW
1283 if (req->flags & REQ_F_WORK_INITIALIZED)
1284 return;
1285
ec99ca6c 1286 __io_req_init_async(req);
500a373d
JA
1287
1288 /* Grab a ref if this isn't our static identity */
1289 req->work.identity = tctx->identity;
1290 if (tctx->identity != &tctx->__identity)
1291 refcount_inc(&req->work.identity->count);
7cdaf587
XW
1292}
1293
2b188cc1
JA
1294static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1295{
1296 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1297
0f158b4c 1298 complete(&ctx->ref_comp);
2b188cc1
JA
1299}
1300
8eb7e2d0
PB
1301static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1302{
1303 return !req->timeout.off;
1304}
1305
2b188cc1
JA
1306static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1307{
1308 struct io_ring_ctx *ctx;
78076bb6 1309 int hash_bits;
2b188cc1
JA
1310
1311 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1312 if (!ctx)
1313 return NULL;
1314
78076bb6
JA
1315 /*
1316 * Use 5 bits less than the max cq entries, that should give us around
1317 * 32 entries per hash list if totally full and uniformly spread.
1318 */
1319 hash_bits = ilog2(p->cq_entries);
1320 hash_bits -= 5;
1321 if (hash_bits <= 0)
1322 hash_bits = 1;
1323 ctx->cancel_hash_bits = hash_bits;
1324 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1325 GFP_KERNEL);
1326 if (!ctx->cancel_hash)
1327 goto err;
1328 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1329
21482896 1330 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
206aefde
JA
1331 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1332 goto err;
2b188cc1
JA
1333
1334 ctx->flags = p->flags;
90554200 1335 init_waitqueue_head(&ctx->sqo_sq_wait);
69fb2131 1336 INIT_LIST_HEAD(&ctx->sqd_list);
2b188cc1 1337 init_waitqueue_head(&ctx->cq_wait);
1d7bb1d5 1338 INIT_LIST_HEAD(&ctx->cq_overflow_list);
0f158b4c
JA
1339 init_completion(&ctx->ref_comp);
1340 init_completion(&ctx->sq_thread_comp);
5a2e745d 1341 idr_init(&ctx->io_buffer_idr);
071698e1 1342 idr_init(&ctx->personality_idr);
2b188cc1
JA
1343 mutex_init(&ctx->uring_lock);
1344 init_waitqueue_head(&ctx->wait);
1345 spin_lock_init(&ctx->completion_lock);
540e32a0 1346 INIT_LIST_HEAD(&ctx->iopoll_list);
de0617e4 1347 INIT_LIST_HEAD(&ctx->defer_list);
5262f567 1348 INIT_LIST_HEAD(&ctx->timeout_list);
fcb323cc
JA
1349 spin_lock_init(&ctx->inflight_lock);
1350 INIT_LIST_HEAD(&ctx->inflight_list);
d67d2263
BM
1351 spin_lock_init(&ctx->rsrc_ref_lock);
1352 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
269bbe5f
BM
1353 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1354 init_llist_head(&ctx->rsrc_put_llist);
1b4c351f 1355 INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
2b188cc1 1356 return ctx;
206aefde 1357err:
78076bb6 1358 kfree(ctx->cancel_hash);
206aefde
JA
1359 kfree(ctx);
1360 return NULL;
2b188cc1
JA
1361}
1362
9cf7c104 1363static bool req_need_defer(struct io_kiocb *req, u32 seq)
7adf4eaf 1364{
2bc9930e
JA
1365 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1366 struct io_ring_ctx *ctx = req->ctx;
a197f664 1367
9cf7c104 1368 return seq != ctx->cached_cq_tail
2c3bac6d 1369 + READ_ONCE(ctx->cached_cq_overflow);
2bc9930e 1370 }
de0617e4 1371
9d858b21 1372 return false;
de0617e4
JA
1373}
1374
5c3462cf 1375static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)
1e6fa521 1376{
500a373d 1377 if (req->work.identity == &tctx->__identity)
1e6fa521
JA
1378 return;
1379 if (refcount_dec_and_test(&req->work.identity->count))
1380 kfree(req->work.identity);
1381}
1382
4edf20f9 1383static void io_req_clean_work(struct io_kiocb *req)
18d9be1a 1384{
7cdaf587 1385 if (!(req->flags & REQ_F_WORK_INITIALIZED))
4edf20f9 1386 return;
51a4cc11 1387
e86d0047 1388 if (req->work.flags & IO_WQ_WORK_MM)
98447d65 1389 mmdrop(req->work.identity->mm);
91d8f519 1390#ifdef CONFIG_BLK_CGROUP
e86d0047 1391 if (req->work.flags & IO_WQ_WORK_BLKCG)
98447d65 1392 css_put(req->work.identity->blkcg_css);
91d8f519 1393#endif
e86d0047 1394 if (req->work.flags & IO_WQ_WORK_CREDS)
98447d65 1395 put_cred(req->work.identity->creds);
dfead8a8 1396 if (req->work.flags & IO_WQ_WORK_FS) {
98447d65 1397 struct fs_struct *fs = req->work.identity->fs;
51a4cc11 1398
98447d65 1399 spin_lock(&req->work.identity->fs->lock);
ff002b30
JA
1400 if (--fs->users)
1401 fs = NULL;
98447d65 1402 spin_unlock(&req->work.identity->fs->lock);
ff002b30
JA
1403 if (fs)
1404 free_fs_struct(fs);
1405 }
34e08fed
PB
1406 if (req->work.flags & IO_WQ_WORK_FILES) {
1407 put_files_struct(req->work.identity->files);
1408 put_nsproxy(req->work.identity->nsproxy);
34e08fed
PB
1409 }
1410 if (req->flags & REQ_F_INFLIGHT) {
1411 struct io_ring_ctx *ctx = req->ctx;
1412 struct io_uring_task *tctx = req->task->io_uring;
1413 unsigned long flags;
1414
1415 spin_lock_irqsave(&ctx->inflight_lock, flags);
1416 list_del(&req->inflight_entry);
1417 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1418 req->flags &= ~REQ_F_INFLIGHT;
1419 if (atomic_read(&tctx->in_idle))
1420 wake_up(&tctx->wait);
1421 }
51a4cc11 1422
e86d0047
PB
1423 req->flags &= ~REQ_F_WORK_INITIALIZED;
1424 req->work.flags &= ~(IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | IO_WQ_WORK_FS |
1425 IO_WQ_WORK_CREDS | IO_WQ_WORK_FILES);
5c3462cf 1426 io_put_identity(req->task->io_uring, req);
561fb04a
JA
1427}
1428
1e6fa521
JA
1429/*
1430 * Create a private copy of io_identity, since some fields don't match
1431 * the current context.
1432 */
1433static bool io_identity_cow(struct io_kiocb *req)
1434{
5c3462cf 1435 struct io_uring_task *tctx = current->io_uring;
1e6fa521
JA
1436 const struct cred *creds = NULL;
1437 struct io_identity *id;
1438
1439 if (req->work.flags & IO_WQ_WORK_CREDS)
1440 creds = req->work.identity->creds;
1441
1442 id = kmemdup(req->work.identity, sizeof(*id), GFP_KERNEL);
1443 if (unlikely(!id)) {
1444 req->work.flags |= IO_WQ_WORK_CANCEL;
1445 return false;
1446 }
1447
1448 /*
1449 * We can safely just re-init the creds we copied Either the field
1450 * matches the current one, or we haven't grabbed it yet. The only
1451 * exception is ->creds, through registered personalities, so handle
1452 * that one separately.
1453 */
1454 io_init_identity(id);
1455 if (creds)
e8c954df 1456 id->creds = creds;
1e6fa521
JA
1457
1458 /* add one for this request */
1459 refcount_inc(&id->count);
1460
cb8a8ae3
JA
1461 /* drop tctx and req identity references, if needed */
1462 if (tctx->identity != &tctx->__identity &&
1463 refcount_dec_and_test(&tctx->identity->count))
1464 kfree(tctx->identity);
1465 if (req->work.identity != &tctx->__identity &&
1466 refcount_dec_and_test(&req->work.identity->count))
1e6fa521
JA
1467 kfree(req->work.identity);
1468
1469 req->work.identity = id;
500a373d 1470 tctx->identity = id;
1e6fa521
JA
1471 return true;
1472}
1473
ce3d5aae
PB
1474static void io_req_track_inflight(struct io_kiocb *req)
1475{
1476 struct io_ring_ctx *ctx = req->ctx;
1477
1478 if (!(req->flags & REQ_F_INFLIGHT)) {
1479 io_req_init_async(req);
1480 req->flags |= REQ_F_INFLIGHT;
1481
1482 spin_lock_irq(&ctx->inflight_lock);
1483 list_add(&req->inflight_entry, &ctx->inflight_list);
1484 spin_unlock_irq(&ctx->inflight_lock);
1485 }
1486}
1487
1e6fa521 1488static bool io_grab_identity(struct io_kiocb *req)
18d9be1a 1489{
d3656344 1490 const struct io_op_def *def = &io_op_defs[req->opcode];
5c3462cf 1491 struct io_identity *id = req->work.identity;
54a91f3b 1492
69228338
JA
1493 if (def->work_flags & IO_WQ_WORK_FSIZE) {
1494 if (id->fsize != rlimit(RLIMIT_FSIZE))
1495 return false;
1496 req->work.flags |= IO_WQ_WORK_FSIZE;
1497 }
91d8f519 1498#ifdef CONFIG_BLK_CGROUP
dfead8a8
JA
1499 if (!(req->work.flags & IO_WQ_WORK_BLKCG) &&
1500 (def->work_flags & IO_WQ_WORK_BLKCG)) {
91d8f519 1501 rcu_read_lock();
1e6fa521
JA
1502 if (id->blkcg_css != blkcg_css()) {
1503 rcu_read_unlock();
1504 return false;
1505 }
91d8f519
DZ
1506 /*
1507 * This should be rare, either the cgroup is dying or the task
1508 * is moving cgroups. Just punt to root for the handful of ios.
1509 */
1e6fa521 1510 if (css_tryget_online(id->blkcg_css))
dfead8a8 1511 req->work.flags |= IO_WQ_WORK_BLKCG;
91d8f519
DZ
1512 rcu_read_unlock();
1513 }
1514#endif
dfead8a8 1515 if (!(req->work.flags & IO_WQ_WORK_CREDS)) {
1e6fa521
JA
1516 if (id->creds != current_cred())
1517 return false;
1518 get_cred(id->creds);
dfead8a8
JA
1519 req->work.flags |= IO_WQ_WORK_CREDS;
1520 }
4ea33a97
JA
1521#ifdef CONFIG_AUDIT
1522 if (!uid_eq(current->loginuid, id->loginuid) ||
1523 current->sessionid != id->sessionid)
1524 return false;
1525#endif
dfead8a8
JA
1526 if (!(req->work.flags & IO_WQ_WORK_FS) &&
1527 (def->work_flags & IO_WQ_WORK_FS)) {
1e6fa521
JA
1528 if (current->fs != id->fs)
1529 return false;
1530 spin_lock(&id->fs->lock);
1531 if (!id->fs->in_exec) {
1532 id->fs->users++;
dfead8a8 1533 req->work.flags |= IO_WQ_WORK_FS;
dca9cf8b
PB
1534 } else {
1535 req->work.flags |= IO_WQ_WORK_CANCEL;
1536 }
1537 spin_unlock(&current->fs->lock);
1538 }
af604703
PB
1539 if (!(req->work.flags & IO_WQ_WORK_FILES) &&
1540 (def->work_flags & IO_WQ_WORK_FILES) &&
1541 !(req->flags & REQ_F_NO_FILE_TABLE)) {
1542 if (id->files != current->files ||
1543 id->nsproxy != current->nsproxy)
1544 return false;
1545 atomic_inc(&id->files->count);
1546 get_nsproxy(id->nsproxy);
af604703 1547 req->work.flags |= IO_WQ_WORK_FILES;
ce3d5aae 1548 io_req_track_inflight(req);
af604703 1549 }
77788775
JA
1550 if (!(req->work.flags & IO_WQ_WORK_MM) &&
1551 (def->work_flags & IO_WQ_WORK_MM)) {
1552 if (id->mm != current->mm)
1553 return false;
1554 mmgrab(id->mm);
1555 req->work.flags |= IO_WQ_WORK_MM;
1556 }
1e6fa521
JA
1557
1558 return true;
1559}
1560
1561static void io_prep_async_work(struct io_kiocb *req)
1562{
1563 const struct io_op_def *def = &io_op_defs[req->opcode];
1e6fa521
JA
1564 struct io_ring_ctx *ctx = req->ctx;
1565
1566 io_req_init_async(req);
1567
feaadc4f
PB
1568 if (req->flags & REQ_F_FORCE_ASYNC)
1569 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1570
1e6fa521
JA
1571 if (req->flags & REQ_F_ISREG) {
1572 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
1573 io_wq_hash_work(&req->work, file_inode(req->file));
1574 } else {
1575 if (def->unbound_nonreg_file)
1576 req->work.flags |= IO_WQ_WORK_UNBOUND;
1577 }
1578
1e6fa521
JA
1579 /* if we fail grabbing identity, we must COW, regrab, and retry */
1580 if (io_grab_identity(req))
1581 return;
1582
1583 if (!io_identity_cow(req))
1584 return;
1585
1586 /* can't fail at this point */
1587 if (!io_grab_identity(req))
1588 WARN_ON(1);
561fb04a 1589}
cccf0ee8 1590
cbdcb435 1591static void io_prep_async_link(struct io_kiocb *req)
561fb04a 1592{
cbdcb435 1593 struct io_kiocb *cur;
54a91f3b 1594
f2f87370
PB
1595 io_for_each_link(cur, req)
1596 io_prep_async_work(cur);
561fb04a
JA
1597}
1598
7271ef3a 1599static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req)
561fb04a 1600{
a197f664 1601 struct io_ring_ctx *ctx = req->ctx;
cbdcb435 1602 struct io_kiocb *link = io_prep_linked_timeout(req);
561fb04a 1603
8766dd51
PB
1604 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1605 &req->work, req->flags);
1606 io_wq_enqueue(ctx->io_wq, &req->work);
7271ef3a 1607 return link;
18d9be1a
JA
1608}
1609
cbdcb435
PB
1610static void io_queue_async_work(struct io_kiocb *req)
1611{
7271ef3a
JA
1612 struct io_kiocb *link;
1613
cbdcb435
PB
1614 /* init ->work of the whole link before punting */
1615 io_prep_async_link(req);
7271ef3a
JA
1616 link = __io_queue_async_work(req);
1617
1618 if (link)
1619 io_queue_linked_timeout(link);
cbdcb435
PB
1620}
1621
5262f567
JA
1622static void io_kill_timeout(struct io_kiocb *req)
1623{
e8c2bc1f 1624 struct io_timeout_data *io = req->async_data;
5262f567
JA
1625 int ret;
1626
e8c2bc1f 1627 ret = hrtimer_try_to_cancel(&io->timer);
5262f567 1628 if (ret != -1) {
01cec8c1
PB
1629 atomic_set(&req->ctx->cq_timeouts,
1630 atomic_read(&req->ctx->cq_timeouts) + 1);
135fcde8 1631 list_del_init(&req->timeout.list);
78e19bbe 1632 io_cqring_fill_event(req, 0);
216578e5 1633 io_put_req_deferred(req, 1);
5262f567
JA
1634 }
1635}
1636
76e1b642
JA
1637/*
1638 * Returns true if we found and killed one or more timeouts
1639 */
6b81928d
PB
1640static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
1641 struct files_struct *files)
5262f567
JA
1642{
1643 struct io_kiocb *req, *tmp;
76e1b642 1644 int canceled = 0;
5262f567
JA
1645
1646 spin_lock_irq(&ctx->completion_lock);
f3606e3a 1647 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
6b81928d 1648 if (io_match_task(req, tsk, files)) {
f3606e3a 1649 io_kill_timeout(req);
76e1b642
JA
1650 canceled++;
1651 }
f3606e3a 1652 }
5262f567 1653 spin_unlock_irq(&ctx->completion_lock);
76e1b642 1654 return canceled != 0;
5262f567
JA
1655}
1656
04518945 1657static void __io_queue_deferred(struct io_ring_ctx *ctx)
de0617e4 1658{
04518945 1659 do {
27dc8338
PB
1660 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1661 struct io_defer_entry, list);
de0617e4 1662
9cf7c104 1663 if (req_need_defer(de->req, de->seq))
04518945 1664 break;
27dc8338 1665 list_del_init(&de->list);
907d1df3 1666 io_req_task_queue(de->req);
27dc8338 1667 kfree(de);
04518945
PB
1668 } while (!list_empty(&ctx->defer_list));
1669}
1670
360428f8 1671static void io_flush_timeouts(struct io_ring_ctx *ctx)
de0617e4 1672{
f010505b
MDG
1673 u32 seq;
1674
1675 if (list_empty(&ctx->timeout_list))
1676 return;
1677
1678 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1679
1680 do {
1681 u32 events_needed, events_got;
360428f8 1682 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
135fcde8 1683 struct io_kiocb, timeout.list);
de0617e4 1684
8eb7e2d0 1685 if (io_is_timeout_noseq(req))
360428f8 1686 break;
f010505b
MDG
1687
1688 /*
1689 * Since seq can easily wrap around over time, subtract
1690 * the last seq at which timeouts were flushed before comparing.
1691 * Assuming not more than 2^31-1 events have happened since,
1692 * these subtractions won't have wrapped, so we can check if
1693 * target is in [last_seq, current_seq] by comparing the two.
1694 */
1695 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1696 events_got = seq - ctx->cq_last_tm_flush;
1697 if (events_got < events_needed)
360428f8 1698 break;
bfe68a22 1699
135fcde8 1700 list_del_init(&req->timeout.list);
5262f567 1701 io_kill_timeout(req);
f010505b
MDG
1702 } while (!list_empty(&ctx->timeout_list));
1703
1704 ctx->cq_last_tm_flush = seq;
360428f8 1705}
5262f567 1706
360428f8
PB
1707static void io_commit_cqring(struct io_ring_ctx *ctx)
1708{
1709 io_flush_timeouts(ctx);
ec30e04b
PB
1710
1711 /* order cqe stores with ring update */
1712 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
de0617e4 1713
04518945
PB
1714 if (unlikely(!list_empty(&ctx->defer_list)))
1715 __io_queue_deferred(ctx);
de0617e4
JA
1716}
1717
90554200
JA
1718static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1719{
1720 struct io_rings *r = ctx->rings;
1721
1722 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries;
1723}
1724
888aae2e
PB
1725static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1726{
1727 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1728}
1729
2b188cc1
JA
1730static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
1731{
75b28aff 1732 struct io_rings *rings = ctx->rings;
2b188cc1
JA
1733 unsigned tail;
1734
115e12e5
SB
1735 /*
1736 * writes to the cq entry need to come after reading head; the
1737 * control dependency is enough as we're using WRITE_ONCE to
1738 * fill the cq entry
1739 */
888aae2e 1740 if (__io_cqring_events(ctx) == rings->cq_ring_entries)
2b188cc1
JA
1741 return NULL;
1742
888aae2e 1743 tail = ctx->cached_cq_tail++;
75b28aff 1744 return &rings->cqes[tail & ctx->cq_mask];
2b188cc1
JA
1745}
1746
f2842ab5
JA
1747static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1748{
f0b493e6
JA
1749 if (!ctx->cq_ev_fd)
1750 return false;
7e55a19c
SG
1751 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1752 return false;
f2842ab5
JA
1753 if (!ctx->eventfd_async)
1754 return true;
b41e9852 1755 return io_wq_current_is_worker();
f2842ab5
JA
1756}
1757
b41e9852 1758static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1d7bb1d5 1759{
b1445e59
PB
1760 /* see waitqueue_active() comment */
1761 smp_mb();
1762
1d7bb1d5
JA
1763 if (waitqueue_active(&ctx->wait))
1764 wake_up(&ctx->wait);
534ca6d6
JA
1765 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1766 wake_up(&ctx->sq_data->wait);
b41e9852 1767 if (io_should_trigger_evfd(ctx))
1d7bb1d5 1768 eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59 1769 if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2f
PB
1770 wake_up_interruptible(&ctx->cq_wait);
1771 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1772 }
1d7bb1d5
JA
1773}
1774
80c18e4a
PB
1775static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1776{
b1445e59
PB
1777 /* see waitqueue_active() comment */
1778 smp_mb();
1779
80c18e4a
PB
1780 if (ctx->flags & IORING_SETUP_SQPOLL) {
1781 if (waitqueue_active(&ctx->wait))
1782 wake_up(&ctx->wait);
1783 }
1784 if (io_should_trigger_evfd(ctx))
1785 eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59 1786 if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2f
PB
1787 wake_up_interruptible(&ctx->cq_wait);
1788 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1789 }
80c18e4a
PB
1790}
1791
c4a2ed72 1792/* Returns true if there are no backlogged entries after the flush */
6c503150
PB
1793static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force,
1794 struct task_struct *tsk,
1795 struct files_struct *files)
1d7bb1d5
JA
1796{
1797 struct io_rings *rings = ctx->rings;
e6c8aa9a 1798 struct io_kiocb *req, *tmp;
1d7bb1d5 1799 struct io_uring_cqe *cqe;
1d7bb1d5 1800 unsigned long flags;
b18032bb 1801 bool all_flushed, posted;
1d7bb1d5
JA
1802 LIST_HEAD(list);
1803
e23de15f
PB
1804 if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries)
1805 return false;
1d7bb1d5 1806
b18032bb 1807 posted = false;
1d7bb1d5 1808 spin_lock_irqsave(&ctx->completion_lock, flags);
e6c8aa9a 1809 list_for_each_entry_safe(req, tmp, &ctx->cq_overflow_list, compl.list) {
08d23634 1810 if (!io_match_task(req, tsk, files))
e6c8aa9a
JA
1811 continue;
1812
1d7bb1d5
JA
1813 cqe = io_get_cqring(ctx);
1814 if (!cqe && !force)
1815 break;
1816
40d8ddd4 1817 list_move(&req->compl.list, &list);
1d7bb1d5
JA
1818 if (cqe) {
1819 WRITE_ONCE(cqe->user_data, req->user_data);
1820 WRITE_ONCE(cqe->res, req->result);
0f7e466b 1821 WRITE_ONCE(cqe->flags, req->compl.cflags);
1d7bb1d5 1822 } else {
2c3bac6d 1823 ctx->cached_cq_overflow++;
1d7bb1d5 1824 WRITE_ONCE(ctx->rings->cq_overflow,
2c3bac6d 1825 ctx->cached_cq_overflow);
1d7bb1d5 1826 }
b18032bb 1827 posted = true;
1d7bb1d5
JA
1828 }
1829
09e88404
PB
1830 all_flushed = list_empty(&ctx->cq_overflow_list);
1831 if (all_flushed) {
1832 clear_bit(0, &ctx->sq_check_overflow);
1833 clear_bit(0, &ctx->cq_check_overflow);
1834 ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1835 }
46930143 1836
b18032bb
JA
1837 if (posted)
1838 io_commit_cqring(ctx);
1d7bb1d5 1839 spin_unlock_irqrestore(&ctx->completion_lock, flags);
b18032bb
JA
1840 if (posted)
1841 io_cqring_ev_posted(ctx);
1d7bb1d5
JA
1842
1843 while (!list_empty(&list)) {
40d8ddd4
PB
1844 req = list_first_entry(&list, struct io_kiocb, compl.list);
1845 list_del(&req->compl.list);
ec9c02ad 1846 io_put_req(req);
1d7bb1d5 1847 }
c4a2ed72 1848
09e88404 1849 return all_flushed;
1d7bb1d5
JA
1850}
1851
6c503150
PB
1852static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force,
1853 struct task_struct *tsk,
1854 struct files_struct *files)
1855{
1856 if (test_bit(0, &ctx->cq_check_overflow)) {
1857 /* iopoll syncs against uring_lock, not completion_lock */
1858 if (ctx->flags & IORING_SETUP_IOPOLL)
1859 mutex_lock(&ctx->uring_lock);
1860 __io_cqring_overflow_flush(ctx, force, tsk, files);
1861 if (ctx->flags & IORING_SETUP_IOPOLL)
1862 mutex_unlock(&ctx->uring_lock);
1863 }
1864}
1865
bcda7baa 1866static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
2b188cc1 1867{
78e19bbe 1868 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA
1869 struct io_uring_cqe *cqe;
1870
78e19bbe 1871 trace_io_uring_complete(ctx, req->user_data, res);
51c3ff62 1872
2b188cc1
JA
1873 /*
1874 * If we can't get a cq entry, userspace overflowed the
1875 * submission (by quite a lot). Increment the overflow count in
1876 * the ring.
1877 */
1878 cqe = io_get_cqring(ctx);
1d7bb1d5 1879 if (likely(cqe)) {
78e19bbe 1880 WRITE_ONCE(cqe->user_data, req->user_data);
2b188cc1 1881 WRITE_ONCE(cqe->res, res);
bcda7baa 1882 WRITE_ONCE(cqe->flags, cflags);
fdaf083c
JA
1883 } else if (ctx->cq_overflow_flushed ||
1884 atomic_read(&req->task->io_uring->in_idle)) {
0f212204
JA
1885 /*
1886 * If we're in ring overflow flush mode, or in task cancel mode,
1887 * then we cannot store the request for later flushing, we need
1888 * to drop it on the floor.
1889 */
2c3bac6d
PB
1890 ctx->cached_cq_overflow++;
1891 WRITE_ONCE(ctx->rings->cq_overflow, ctx->cached_cq_overflow);
1d7bb1d5 1892 } else {
ad3eb2c8
JA
1893 if (list_empty(&ctx->cq_overflow_list)) {
1894 set_bit(0, &ctx->sq_check_overflow);
1895 set_bit(0, &ctx->cq_check_overflow);
6d5f9049 1896 ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
ad3eb2c8 1897 }
40d8ddd4 1898 io_clean_op(req);
1d7bb1d5 1899 req->result = res;
0f7e466b 1900 req->compl.cflags = cflags;
40d8ddd4
PB
1901 refcount_inc(&req->refs);
1902 list_add_tail(&req->compl.list, &ctx->cq_overflow_list);
2b188cc1
JA
1903 }
1904}
1905
bcda7baa
JA
1906static void io_cqring_fill_event(struct io_kiocb *req, long res)
1907{
1908 __io_cqring_fill_event(req, res, 0);
1909}
1910
9ae1f8dd
PB
1911static void io_req_complete_post(struct io_kiocb *req, long res,
1912 unsigned int cflags)
2b188cc1 1913{
78e19bbe 1914 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA
1915 unsigned long flags;
1916
1917 spin_lock_irqsave(&ctx->completion_lock, flags);
bcda7baa 1918 __io_cqring_fill_event(req, res, cflags);
2b188cc1
JA
1919 io_commit_cqring(ctx);
1920 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1921
8c838788 1922 io_cqring_ev_posted(ctx);
9ae1f8dd
PB
1923}
1924
1925static inline void io_req_complete_nostate(struct io_kiocb *req, long res,
1926 unsigned int cflags)
1927{
1928 io_req_complete_post(req, res, cflags);
a38d68db 1929 io_put_req(req);
2b188cc1
JA
1930}
1931
a38d68db 1932static void io_req_complete_state(struct io_kiocb *req, long res,
889fca73 1933 unsigned int cflags)
229a7b63 1934{
a38d68db
PB
1935 io_clean_op(req);
1936 req->result = res;
1937 req->compl.cflags = cflags;
e342c807 1938 req->flags |= REQ_F_COMPLETE_INLINE;
a38d68db
PB
1939}
1940
889fca73
PB
1941static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1942 long res, unsigned cflags)
a38d68db 1943{
889fca73
PB
1944 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1945 io_req_complete_state(req, res, cflags);
a38d68db 1946 else
889fca73 1947 io_req_complete_nostate(req, res, cflags);
e1e16097
JA
1948}
1949
a38d68db 1950static inline void io_req_complete(struct io_kiocb *req, long res)
bcda7baa 1951{
889fca73 1952 __io_req_complete(req, 0, res, 0);
bcda7baa
JA
1953}
1954
258b29a9 1955static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
2b188cc1 1956{
258b29a9
PB
1957 struct io_submit_state *state = &ctx->submit_state;
1958
bf019da7
PB
1959 BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs));
1960
1b4c351f
JA
1961 if (!list_empty(&state->comp.free_list)) {
1962 struct io_kiocb *req;
1963
1964 req = list_first_entry(&state->comp.free_list, struct io_kiocb,
1965 compl.list);
1966 list_del(&req->compl.list);
1967 return req;
1968 }
1969
f6b6c7d6 1970 if (!state->free_reqs) {
291b2821 1971 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
2579f913
JA
1972 int ret;
1973
bf019da7
PB
1974 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1975 state->reqs);
fd6fab2c
JA
1976
1977 /*
1978 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1979 * retry single alloc to be on the safe side.
1980 */
1981 if (unlikely(ret <= 0)) {
1982 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1983 if (!state->reqs[0])
3893f39f 1984 return NULL;
fd6fab2c
JA
1985 ret = 1;
1986 }
291b2821 1987 state->free_reqs = ret;
2b188cc1
JA
1988 }
1989
291b2821
PB
1990 state->free_reqs--;
1991 return state->reqs[state->free_reqs];
2b188cc1
JA
1992}
1993
8da11c19
PB
1994static inline void io_put_file(struct io_kiocb *req, struct file *file,
1995 bool fixed)
1996{
36f72fe2 1997 if (!fixed)
8da11c19
PB
1998 fput(file);
1999}
2000
4edf20f9 2001static void io_dismantle_req(struct io_kiocb *req)
2b188cc1 2002{
3ca405eb 2003 io_clean_op(req);
929a3af9 2004
e8c2bc1f
JA
2005 if (req->async_data)
2006 kfree(req->async_data);
8da11c19
PB
2007 if (req->file)
2008 io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
269bbe5f
BM
2009 if (req->fixed_rsrc_refs)
2010 percpu_ref_put(req->fixed_rsrc_refs);
4edf20f9 2011 io_req_clean_work(req);
e65ef56d
JA
2012}
2013
7c660731
PB
2014static inline void io_put_task(struct task_struct *task, int nr)
2015{
2016 struct io_uring_task *tctx = task->io_uring;
2017
2018 percpu_counter_sub(&tctx->inflight, nr);
2019 if (unlikely(atomic_read(&tctx->in_idle)))
2020 wake_up(&tctx->wait);
2021 put_task_struct_many(task, nr);
2022}
2023
216578e5 2024static void __io_free_req(struct io_kiocb *req)
c6ca97b3 2025{
51a4cc11 2026 struct io_ring_ctx *ctx = req->ctx;
c6ca97b3 2027
216578e5 2028 io_dismantle_req(req);
7c660731 2029 io_put_task(req->task, 1);
e3bc8e9d 2030
3893f39f 2031 kmem_cache_free(req_cachep, req);
ecfc5177 2032 percpu_ref_put(&ctx->refs);
e65ef56d
JA
2033}
2034
f2f87370
PB
2035static inline void io_remove_next_linked(struct io_kiocb *req)
2036{
2037 struct io_kiocb *nxt = req->link;
2038
2039 req->link = nxt->link;
2040 nxt->link = NULL;
2041}
2042
c9abd7ad 2043static void io_kill_linked_timeout(struct io_kiocb *req)
2665abfd 2044{
a197f664 2045 struct io_ring_ctx *ctx = req->ctx;
7c86ffee 2046 struct io_kiocb *link;
c9abd7ad
PB
2047 bool cancelled = false;
2048 unsigned long flags;
7c86ffee 2049
c9abd7ad 2050 spin_lock_irqsave(&ctx->completion_lock, flags);
f2f87370
PB
2051 link = req->link;
2052
900fad45
PB
2053 /*
2054 * Can happen if a linked timeout fired and link had been like
2055 * req -> link t-out -> link t-out [-> ...]
2056 */
c9abd7ad
PB
2057 if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
2058 struct io_timeout_data *io = link->async_data;
2059 int ret;
7c86ffee 2060
f2f87370 2061 io_remove_next_linked(req);
90cd7e42 2062 link->timeout.head = NULL;
c9abd7ad
PB
2063 ret = hrtimer_try_to_cancel(&io->timer);
2064 if (ret != -1) {
2065 io_cqring_fill_event(link, -ECANCELED);
2066 io_commit_cqring(ctx);
2067 cancelled = true;
2068 }
2069 }
7c86ffee 2070 req->flags &= ~REQ_F_LINK_TIMEOUT;
216578e5 2071 spin_unlock_irqrestore(&ctx->completion_lock, flags);
ab0b6451 2072
c9abd7ad 2073 if (cancelled) {
7c86ffee 2074 io_cqring_ev_posted(ctx);
c9abd7ad
PB
2075 io_put_req(link);
2076 }
7c86ffee
PB
2077}
2078
9e645e11 2079
d148ca4b 2080static void io_fail_links(struct io_kiocb *req)
9e645e11 2081{
f2f87370 2082 struct io_kiocb *link, *nxt;
2665abfd 2083 struct io_ring_ctx *ctx = req->ctx;
d148ca4b 2084 unsigned long flags;
9e645e11 2085
d148ca4b 2086 spin_lock_irqsave(&ctx->completion_lock, flags);
f2f87370
PB
2087 link = req->link;
2088 req->link = NULL;
9e645e11 2089
f2f87370
PB
2090 while (link) {
2091 nxt = link->link;
2092 link->link = NULL;
2665abfd 2093
f2f87370 2094 trace_io_uring_fail_link(req, link);
7c86ffee 2095 io_cqring_fill_event(link, -ECANCELED);
216578e5
PB
2096
2097 /*
2098 * It's ok to free under spinlock as they're not linked anymore,
2099 * but avoid REQ_F_WORK_INITIALIZED because it may deadlock on
2100 * work.fs->lock.
2101 */
2102 if (link->flags & REQ_F_WORK_INITIALIZED)
2103 io_put_req_deferred(link, 2);
2104 else
2105 io_double_put_req(link);
f2f87370 2106 link = nxt;
9e645e11 2107 }
2665abfd 2108 io_commit_cqring(ctx);
216578e5 2109 spin_unlock_irqrestore(&ctx->completion_lock, flags);
9e645e11 2110
2665abfd 2111 io_cqring_ev_posted(ctx);
9e645e11
JA
2112}
2113
3fa5e0f3 2114static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
c69f8dbe 2115{
7c86ffee
PB
2116 if (req->flags & REQ_F_LINK_TIMEOUT)
2117 io_kill_linked_timeout(req);
944e58bf 2118
9e645e11
JA
2119 /*
2120 * If LINK is set, we have dependent requests in this chain. If we
2121 * didn't fail this request, queue the first one up, moving any other
2122 * dependencies to the next request. In case of failure, fail the rest
2123 * of the chain.
2124 */
f2f87370
PB
2125 if (likely(!(req->flags & REQ_F_FAIL_LINK))) {
2126 struct io_kiocb *nxt = req->link;
2127
2128 req->link = NULL;
2129 return nxt;
2130 }
9b5f7bd9
PB
2131 io_fail_links(req);
2132 return NULL;
4d7dd462 2133}
9e645e11 2134
f2f87370 2135static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
3fa5e0f3 2136{
f2f87370 2137 if (likely(!(req->link) && !(req->flags & REQ_F_LINK_TIMEOUT)))
3fa5e0f3
PB
2138 return NULL;
2139 return __io_req_find_next(req);
2140}
2141
7cbf1722
JA
2142static bool __tctx_task_work(struct io_uring_task *tctx)
2143{
65453d1e 2144 struct io_ring_ctx *ctx = NULL;
7cbf1722
JA
2145 struct io_wq_work_list list;
2146 struct io_wq_work_node *node;
2147
2148 if (wq_list_empty(&tctx->task_list))
2149 return false;
2150
2151 spin_lock(&tctx->task_lock);
2152 list = tctx->task_list;
2153 INIT_WQ_LIST(&tctx->task_list);
2154 spin_unlock(&tctx->task_lock);
2155
2156 node = list.first;
2157 while (node) {
2158 struct io_wq_work_node *next = node->next;
65453d1e 2159 struct io_ring_ctx *this_ctx;
7cbf1722
JA
2160 struct io_kiocb *req;
2161
2162 req = container_of(node, struct io_kiocb, io_task_work.node);
65453d1e 2163 this_ctx = req->ctx;
7cbf1722
JA
2164 req->task_work.func(&req->task_work);
2165 node = next;
65453d1e
JA
2166
2167 if (!ctx) {
2168 ctx = this_ctx;
2169 } else if (ctx != this_ctx) {
2170 mutex_lock(&ctx->uring_lock);
2171 io_submit_flush_completions(&ctx->submit_state.comp, ctx);
2172 mutex_unlock(&ctx->uring_lock);
2173 ctx = this_ctx;
2174 }
2175 }
2176
2177 if (ctx && ctx->submit_state.comp.nr) {
2178 mutex_lock(&ctx->uring_lock);
2179 io_submit_flush_completions(&ctx->submit_state.comp, ctx);
2180 mutex_unlock(&ctx->uring_lock);
7cbf1722
JA
2181 }
2182
2183 return list.first != NULL;
2184}
2185
2186static void tctx_task_work(struct callback_head *cb)
2187{
2188 struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work);
2189
2190 while (__tctx_task_work(tctx))
2191 cond_resched();
2192
2193 clear_bit(0, &tctx->task_state);
2194}
2195
2196static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req,
2197 enum task_work_notify_mode notify)
2198{
2199 struct io_uring_task *tctx = tsk->io_uring;
2200 struct io_wq_work_node *node, *prev;
2201 int ret;
2202
2203 WARN_ON_ONCE(!tctx);
2204
2205 spin_lock(&tctx->task_lock);
2206 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
2207 spin_unlock(&tctx->task_lock);
2208
2209 /* task_work already pending, we're done */
2210 if (test_bit(0, &tctx->task_state) ||
2211 test_and_set_bit(0, &tctx->task_state))
2212 return 0;
2213
2214 if (!task_work_add(tsk, &tctx->task_work, notify))
2215 return 0;
2216
2217 /*
2218 * Slow path - we failed, find and delete work. if the work is not
2219 * in the list, it got run and we're fine.
2220 */
2221 ret = 0;
2222 spin_lock(&tctx->task_lock);
2223 wq_list_for_each(node, prev, &tctx->task_list) {
2224 if (&req->io_task_work.node == node) {
2225 wq_list_del(&tctx->task_list, node, prev);
2226 ret = 1;
2227 break;
2228 }
2229 }
2230 spin_unlock(&tctx->task_lock);
2231 clear_bit(0, &tctx->task_state);
2232 return ret;
2233}
2234
355fb9e2 2235static int io_req_task_work_add(struct io_kiocb *req)
c2c4c83c
JA
2236{
2237 struct task_struct *tsk = req->task;
2238 struct io_ring_ctx *ctx = req->ctx;
91989c70
JA
2239 enum task_work_notify_mode notify;
2240 int ret;
c2c4c83c 2241
6200b0ae
JA
2242 if (tsk->flags & PF_EXITING)
2243 return -ESRCH;
2244
c2c4c83c 2245 /*
0ba9c9ed
JA
2246 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2247 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2248 * processing task_work. There's no reliable way to tell if TWA_RESUME
2249 * will do the job.
c2c4c83c 2250 */
91989c70 2251 notify = TWA_NONE;
355fb9e2 2252 if (!(ctx->flags & IORING_SETUP_SQPOLL))
c2c4c83c
JA
2253 notify = TWA_SIGNAL;
2254
7cbf1722 2255 ret = io_task_work_add(tsk, req, notify);
c2c4c83c
JA
2256 if (!ret)
2257 wake_up_process(tsk);
0ba9c9ed 2258
c2c4c83c
JA
2259 return ret;
2260}
2261
eab30c4d 2262static void io_req_task_work_add_fallback(struct io_kiocb *req,
7cbf1722 2263 task_work_func_t cb)
eab30c4d
PB
2264{
2265 struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq);
2266
2267 init_task_work(&req->task_work, cb);
2268 task_work_add(tsk, &req->task_work, TWA_NONE);
2269 wake_up_process(tsk);
2270}
2271
c40f6379
JA
2272static void __io_req_task_cancel(struct io_kiocb *req, int error)
2273{
2274 struct io_ring_ctx *ctx = req->ctx;
2275
2276 spin_lock_irq(&ctx->completion_lock);
2277 io_cqring_fill_event(req, error);
2278 io_commit_cqring(ctx);
2279 spin_unlock_irq(&ctx->completion_lock);
2280
2281 io_cqring_ev_posted(ctx);
2282 req_set_fail_links(req);
2283 io_double_put_req(req);
2284}
2285
2286static void io_req_task_cancel(struct callback_head *cb)
2287{
2288 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
87ceb6a6 2289 struct io_ring_ctx *ctx = req->ctx;
c40f6379
JA
2290
2291 __io_req_task_cancel(req, -ECANCELED);
87ceb6a6 2292 percpu_ref_put(&ctx->refs);
c40f6379
JA
2293}
2294
2295static void __io_req_task_submit(struct io_kiocb *req)
2296{
2297 struct io_ring_ctx *ctx = req->ctx;
2298
81b6d05c 2299 mutex_lock(&ctx->uring_lock);
d9d05217
PB
2300 if (!ctx->sqo_dead &&
2301 !__io_sq_thread_acquire_mm(ctx) &&
2302 !__io_sq_thread_acquire_files(ctx))
65453d1e 2303 __io_queue_sqe(req, &ctx->submit_state.comp);
81b6d05c 2304 else
c40f6379 2305 __io_req_task_cancel(req, -EFAULT);
81b6d05c 2306 mutex_unlock(&ctx->uring_lock);
c40f6379
JA
2307}
2308
2309static void io_req_task_submit(struct callback_head *cb)
2310{
2311 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
6d816e08 2312 struct io_ring_ctx *ctx = req->ctx;
c40f6379
JA
2313
2314 __io_req_task_submit(req);
6d816e08 2315 percpu_ref_put(&ctx->refs);
c40f6379
JA
2316}
2317
2318static void io_req_task_queue(struct io_kiocb *req)
2319{
c40f6379
JA
2320 int ret;
2321
7cbf1722 2322 req->task_work.func = io_req_task_submit;
6d816e08 2323 percpu_ref_get(&req->ctx->refs);
c40f6379 2324
355fb9e2 2325 ret = io_req_task_work_add(req);
eab30c4d
PB
2326 if (unlikely(ret))
2327 io_req_task_work_add_fallback(req, io_req_task_cancel);
c40f6379
JA
2328}
2329
f2f87370 2330static inline void io_queue_next(struct io_kiocb *req)
c69f8dbe 2331{
9b5f7bd9 2332 struct io_kiocb *nxt = io_req_find_next(req);
944e58bf
PB
2333
2334 if (nxt)
906a8c3f 2335 io_req_task_queue(nxt);
c69f8dbe
JL
2336}
2337
c3524383 2338static void io_free_req(struct io_kiocb *req)
7a743e22 2339{
c3524383
PB
2340 io_queue_next(req);
2341 __io_free_req(req);
2342}
8766dd51 2343
2d6500d4 2344struct req_batch {
5af1d13e
PB
2345 struct task_struct *task;
2346 int task_refs;
1b4c351f 2347 int ctx_refs;
2d6500d4
PB
2348};
2349
5af1d13e
PB
2350static inline void io_init_req_batch(struct req_batch *rb)
2351{
5af1d13e 2352 rb->task_refs = 0;
9ae72463 2353 rb->ctx_refs = 0;
5af1d13e
PB
2354 rb->task = NULL;
2355}
2356
2d6500d4
PB
2357static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2358 struct req_batch *rb)
2359{
5af1d13e 2360 if (rb->task) {
7c660731 2361 io_put_task(rb->task, rb->task_refs);
5af1d13e
PB
2362 rb->task = NULL;
2363 }
9ae72463
PB
2364 if (rb->ctx_refs)
2365 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
2d6500d4
PB
2366}
2367
6ff119a6
PB
2368static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2369 struct io_submit_state *state)
2d6500d4 2370{
f2f87370 2371 io_queue_next(req);
2d6500d4 2372
e3bc8e9d 2373 if (req->task != rb->task) {
7c660731
PB
2374 if (rb->task)
2375 io_put_task(rb->task, rb->task_refs);
e3bc8e9d
JA
2376 rb->task = req->task;
2377 rb->task_refs = 0;
5af1d13e 2378 }
e3bc8e9d 2379 rb->task_refs++;
9ae72463 2380 rb->ctx_refs++;
5af1d13e 2381
4edf20f9 2382 io_dismantle_req(req);
6ff119a6
PB
2383 if (state->free_reqs != ARRAY_SIZE(state->reqs)) {
2384 state->reqs[state->free_reqs++] = req;
2385 } else {
1b4c351f
JA
2386 struct io_comp_state *cs = &req->ctx->submit_state.comp;
2387
2388 list_add(&req->compl.list, &cs->free_list);
6ff119a6 2389 }
7a743e22
PB
2390}
2391
905c172f
PB
2392static void io_submit_flush_completions(struct io_comp_state *cs,
2393 struct io_ring_ctx *ctx)
2394{
2395 int i, nr = cs->nr;
2396 struct io_kiocb *req;
2397 struct req_batch rb;
2398
2399 io_init_req_batch(&rb);
2400 spin_lock_irq(&ctx->completion_lock);
2401 for (i = 0; i < nr; i++) {
2402 req = cs->reqs[i];
2403 __io_cqring_fill_event(req, req->result, req->compl.cflags);
2404 }
2405 io_commit_cqring(ctx);
2406 spin_unlock_irq(&ctx->completion_lock);
2407
2408 io_cqring_ev_posted(ctx);
2409 for (i = 0; i < nr; i++) {
2410 req = cs->reqs[i];
2411
2412 /* submission and completion refs */
2413 if (refcount_sub_and_test(2, &req->refs))
6ff119a6 2414 io_req_free_batch(&rb, req, &ctx->submit_state);
905c172f
PB
2415 }
2416
2417 io_req_free_batch_finish(ctx, &rb);
2418 cs->nr = 0;
2419}
2420
ba816ad6
JA
2421/*
2422 * Drop reference to request, return next in chain (if there is one) if this
2423 * was the last reference to this request.
2424 */
9b5f7bd9 2425static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
e65ef56d 2426{
9b5f7bd9
PB
2427 struct io_kiocb *nxt = NULL;
2428
2a44f467 2429 if (refcount_dec_and_test(&req->refs)) {
9b5f7bd9 2430 nxt = io_req_find_next(req);
4d7dd462 2431 __io_free_req(req);
2a44f467 2432 }
9b5f7bd9 2433 return nxt;
2b188cc1
JA
2434}
2435
e65ef56d
JA
2436static void io_put_req(struct io_kiocb *req)
2437{
2438 if (refcount_dec_and_test(&req->refs))
2439 io_free_req(req);
2b188cc1
JA
2440}
2441
216578e5
PB
2442static void io_put_req_deferred_cb(struct callback_head *cb)
2443{
2444 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2445
2446 io_free_req(req);
2447}
2448
2449static void io_free_req_deferred(struct io_kiocb *req)
2450{
2451 int ret;
2452
7cbf1722 2453 req->task_work.func = io_put_req_deferred_cb;
355fb9e2 2454 ret = io_req_task_work_add(req);
eab30c4d
PB
2455 if (unlikely(ret))
2456 io_req_task_work_add_fallback(req, io_put_req_deferred_cb);
216578e5
PB
2457}
2458
2459static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
2460{
2461 if (refcount_sub_and_test(refs, &req->refs))
2462 io_free_req_deferred(req);
2463}
2464
978db57e
JA
2465static void io_double_put_req(struct io_kiocb *req)
2466{
2467 /* drop both submit and complete references */
2468 if (refcount_sub_and_test(2, &req->refs))
2469 io_free_req(req);
2470}
2471
6c503150 2472static unsigned io_cqring_events(struct io_ring_ctx *ctx)
a3a0e43f
JA
2473{
2474 /* See comment at the top of this file */
2475 smp_rmb();
e23de15f 2476 return __io_cqring_events(ctx);
a3a0e43f
JA
2477}
2478
fb5ccc98
PB
2479static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2480{
2481 struct io_rings *rings = ctx->rings;
2482
2483 /* make sure SQ entry isn't read before tail */
2484 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2485}
2486
8ff069bf 2487static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
e94f141b 2488{
8ff069bf 2489 unsigned int cflags;
e94f141b 2490
bcda7baa
JA
2491 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2492 cflags |= IORING_CQE_F_BUFFER;
0e1b6fe3 2493 req->flags &= ~REQ_F_BUFFER_SELECTED;
bcda7baa
JA
2494 kfree(kbuf);
2495 return cflags;
e94f141b
JA
2496}
2497
8ff069bf 2498static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
bcda7baa 2499{
4d954c25 2500 struct io_buffer *kbuf;
bcda7baa 2501
4d954c25 2502 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
8ff069bf
PB
2503 return io_put_kbuf(req, kbuf);
2504}
2505
4c6e277c
JA
2506static inline bool io_run_task_work(void)
2507{
6200b0ae
JA
2508 /*
2509 * Not safe to run on exiting task, and the task_work handling will
2510 * not add work to such a task.
2511 */
2512 if (unlikely(current->flags & PF_EXITING))
2513 return false;
4c6e277c
JA
2514 if (current->task_works) {
2515 __set_current_state(TASK_RUNNING);
2516 task_work_run();
2517 return true;
2518 }
2519
2520 return false;
bcda7baa
JA
2521}
2522
bbde017a
XW
2523static void io_iopoll_queue(struct list_head *again)
2524{
2525 struct io_kiocb *req;
2526
2527 do {
d21ffe7e
PB
2528 req = list_first_entry(again, struct io_kiocb, inflight_entry);
2529 list_del(&req->inflight_entry);
889fca73 2530 __io_complete_rw(req, -EAGAIN, 0, 0);
bbde017a
XW
2531 } while (!list_empty(again));
2532}
2533
def596e9
JA
2534/*
2535 * Find and free completed poll iocbs
2536 */
2537static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
2538 struct list_head *done)
2539{
8237e045 2540 struct req_batch rb;
def596e9 2541 struct io_kiocb *req;
bbde017a
XW
2542 LIST_HEAD(again);
2543
2544 /* order with ->result store in io_complete_rw_iopoll() */
2545 smp_rmb();
def596e9 2546
5af1d13e 2547 io_init_req_batch(&rb);
def596e9 2548 while (!list_empty(done)) {
bcda7baa
JA
2549 int cflags = 0;
2550
d21ffe7e 2551 req = list_first_entry(done, struct io_kiocb, inflight_entry);
bbde017a 2552 if (READ_ONCE(req->result) == -EAGAIN) {
56450c20 2553 req->result = 0;
bbde017a 2554 req->iopoll_completed = 0;
d21ffe7e 2555 list_move_tail(&req->inflight_entry, &again);
bbde017a
XW
2556 continue;
2557 }
d21ffe7e 2558 list_del(&req->inflight_entry);
def596e9 2559
bcda7baa 2560 if (req->flags & REQ_F_BUFFER_SELECTED)
8ff069bf 2561 cflags = io_put_rw_kbuf(req);
bcda7baa
JA
2562
2563 __io_cqring_fill_event(req, req->result, cflags);
def596e9
JA
2564 (*nr_events)++;
2565
c3524383 2566 if (refcount_dec_and_test(&req->refs))
6ff119a6 2567 io_req_free_batch(&rb, req, &ctx->submit_state);
def596e9 2568 }
def596e9 2569
09bb8394 2570 io_commit_cqring(ctx);
80c18e4a 2571 io_cqring_ev_posted_iopoll(ctx);
2d6500d4 2572 io_req_free_batch_finish(ctx, &rb);
581f9810 2573
bbde017a
XW
2574 if (!list_empty(&again))
2575 io_iopoll_queue(&again);
581f9810
BM
2576}
2577
def596e9
JA
2578static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
2579 long min)
2580{
2581 struct io_kiocb *req, *tmp;
2582 LIST_HEAD(done);
2583 bool spin;
2584 int ret;
2585
2586 /*
2587 * Only spin for completions if we don't have multiple devices hanging
2588 * off our complete list, and we're under the requested amount.
2589 */
2590 spin = !ctx->poll_multi_file && *nr_events < min;
2591
2592 ret = 0;
d21ffe7e 2593 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
9adbd45d 2594 struct kiocb *kiocb = &req->rw.kiocb;
def596e9
JA
2595
2596 /*
581f9810
BM
2597 * Move completed and retryable entries to our local lists.
2598 * If we find a request that requires polling, break out
2599 * and complete those lists first, if we have entries there.
def596e9 2600 */
65a6543d 2601 if (READ_ONCE(req->iopoll_completed)) {
d21ffe7e 2602 list_move_tail(&req->inflight_entry, &done);
def596e9
JA
2603 continue;
2604 }
2605 if (!list_empty(&done))
2606 break;
2607
2608 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
2609 if (ret < 0)
2610 break;
2611
3aadc23e
PB
2612 /* iopoll may have completed current req */
2613 if (READ_ONCE(req->iopoll_completed))
d21ffe7e 2614 list_move_tail(&req->inflight_entry, &done);
3aadc23e 2615
def596e9
JA
2616 if (ret && spin)
2617 spin = false;
2618 ret = 0;
2619 }
2620
2621 if (!list_empty(&done))
2622 io_iopoll_complete(ctx, nr_events, &done);
2623
2624 return ret;
2625}
2626
2627/*
d195a66e 2628 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
def596e9
JA
2629 * non-spinning poll check - we'll still enter the driver poll loop, but only
2630 * as a non-spinning completion check.
2631 */
2632static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
2633 long min)
2634{
540e32a0 2635 while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
def596e9
JA
2636 int ret;
2637
2638 ret = io_do_iopoll(ctx, nr_events, min);
2639 if (ret < 0)
2640 return ret;
eba0a4dd 2641 if (*nr_events >= min)
def596e9
JA
2642 return 0;
2643 }
2644
2645 return 1;
2646}
2647
2648/*
2649 * We can't just wait for polled events to come to us, we have to actively
2650 * find and complete them.
2651 */
b2edc0a7 2652static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
def596e9
JA
2653{
2654 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2655 return;
2656
2657 mutex_lock(&ctx->uring_lock);
540e32a0 2658 while (!list_empty(&ctx->iopoll_list)) {
def596e9
JA
2659 unsigned int nr_events = 0;
2660
b2edc0a7 2661 io_do_iopoll(ctx, &nr_events, 0);
08f5439f 2662
b2edc0a7
PB
2663 /* let it sleep and repeat later if can't complete a request */
2664 if (nr_events == 0)
2665 break;
08f5439f
JA
2666 /*
2667 * Ensure we allow local-to-the-cpu processing to take place,
2668 * in this case we need to ensure that we reap all events.
3fcee5a6 2669 * Also let task_work, etc. to progress by releasing the mutex
08f5439f 2670 */
3fcee5a6
PB
2671 if (need_resched()) {
2672 mutex_unlock(&ctx->uring_lock);
2673 cond_resched();
2674 mutex_lock(&ctx->uring_lock);
2675 }
def596e9
JA
2676 }
2677 mutex_unlock(&ctx->uring_lock);
2678}
2679
7668b92a 2680static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
def596e9 2681{
7668b92a 2682 unsigned int nr_events = 0;
2b2ed975 2683 int iters = 0, ret = 0;
500f9fba 2684
c7849be9
XW
2685 /*
2686 * We disallow the app entering submit/complete with polling, but we
2687 * still need to lock the ring to prevent racing with polled issue
2688 * that got punted to a workqueue.
2689 */
2690 mutex_lock(&ctx->uring_lock);
def596e9 2691 do {
a3a0e43f
JA
2692 /*
2693 * Don't enter poll loop if we already have events pending.
2694 * If we do, we can potentially be spinning for commands that
2695 * already triggered a CQE (eg in error).
2696 */
6c503150
PB
2697 if (test_bit(0, &ctx->cq_check_overflow))
2698 __io_cqring_overflow_flush(ctx, false, NULL, NULL);
2699 if (io_cqring_events(ctx))
a3a0e43f
JA
2700 break;
2701
500f9fba
JA
2702 /*
2703 * If a submit got punted to a workqueue, we can have the
2704 * application entering polling for a command before it gets
2705 * issued. That app will hold the uring_lock for the duration
2706 * of the poll right here, so we need to take a breather every
2707 * now and then to ensure that the issue has a chance to add
2708 * the poll to the issued list. Otherwise we can spin here
2709 * forever, while the workqueue is stuck trying to acquire the
2710 * very same mutex.
2711 */
2712 if (!(++iters & 7)) {
2713 mutex_unlock(&ctx->uring_lock);
4c6e277c 2714 io_run_task_work();
500f9fba
JA
2715 mutex_lock(&ctx->uring_lock);
2716 }
2717
7668b92a 2718 ret = io_iopoll_getevents(ctx, &nr_events, min);
def596e9
JA
2719 if (ret <= 0)
2720 break;
2721 ret = 0;
7668b92a 2722 } while (min && !nr_events && !need_resched());
def596e9 2723
500f9fba 2724 mutex_unlock(&ctx->uring_lock);
def596e9
JA
2725 return ret;
2726}
2727
491381ce 2728static void kiocb_end_write(struct io_kiocb *req)
2b188cc1 2729{
491381ce
JA
2730 /*
2731 * Tell lockdep we inherited freeze protection from submission
2732 * thread.
2733 */
2734 if (req->flags & REQ_F_ISREG) {
2735 struct inode *inode = file_inode(req->file);
2b188cc1 2736
491381ce 2737 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
2b188cc1 2738 }
491381ce 2739 file_end_write(req->file);
2b188cc1
JA
2740}
2741
a1d7c393 2742static void io_complete_rw_common(struct kiocb *kiocb, long res,
889fca73 2743 unsigned int issue_flags)
2b188cc1 2744{
9adbd45d 2745 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
bcda7baa 2746 int cflags = 0;
2b188cc1 2747
491381ce
JA
2748 if (kiocb->ki_flags & IOCB_WRITE)
2749 kiocb_end_write(req);
2b188cc1 2750
4e88d6e7
JA
2751 if (res != req->result)
2752 req_set_fail_links(req);
bcda7baa 2753 if (req->flags & REQ_F_BUFFER_SELECTED)
8ff069bf 2754 cflags = io_put_rw_kbuf(req);
889fca73 2755 __io_req_complete(req, issue_flags, res, cflags);
ba816ad6
JA
2756}
2757
b63534c4 2758#ifdef CONFIG_BLOCK
dc2a6e9a 2759static bool io_resubmit_prep(struct io_kiocb *req)
b63534c4
JA
2760{
2761 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
847595de 2762 int rw, ret = -ECANCELED;
b63534c4 2763 struct iov_iter iter;
b63534c4 2764
dc2a6e9a
PB
2765 /* already prepared */
2766 if (req->async_data)
2767 return true;
b63534c4
JA
2768
2769 switch (req->opcode) {
2770 case IORING_OP_READV:
2771 case IORING_OP_READ_FIXED:
2772 case IORING_OP_READ:
2773 rw = READ;
2774 break;
2775 case IORING_OP_WRITEV:
2776 case IORING_OP_WRITE_FIXED:
2777 case IORING_OP_WRITE:
2778 rw = WRITE;
2779 break;
2780 default:
2781 printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
2782 req->opcode);
dc2a6e9a 2783 return false;
b63534c4
JA
2784 }
2785
dc2a6e9a
PB
2786 ret = io_import_iovec(rw, req, &iovec, &iter, false);
2787 if (ret < 0)
2788 return false;
6bf985dc 2789 return !io_setup_async_rw(req, iovec, inline_vecs, &iter, false);
b63534c4 2790}
b63534c4
JA
2791#endif
2792
2793static bool io_rw_reissue(struct io_kiocb *req, long res)
2794{
2795#ifdef CONFIG_BLOCK
bf6182b6 2796 umode_t mode;
b63534c4
JA
2797 int ret;
2798
bf6182b6 2799 if (res != -EAGAIN && res != -EOPNOTSUPP)
355afaeb 2800 return false;
bf6182b6 2801 mode = file_inode(req->file)->i_mode;
75c668cd
PB
2802 if (!S_ISBLK(mode) && !S_ISREG(mode))
2803 return false;
2804 if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker())
b63534c4
JA
2805 return false;
2806
55e6ac1e
PB
2807 lockdep_assert_held(&req->ctx->uring_lock);
2808
28cea78a 2809 ret = io_sq_thread_acquire_mm_files(req->ctx, req);
6d816e08 2810
dc2a6e9a 2811 if (!ret && io_resubmit_prep(req)) {
fdee946d
JA
2812 refcount_inc(&req->refs);
2813 io_queue_async_work(req);
b63534c4 2814 return true;
fdee946d 2815 }
dc2a6e9a 2816 req_set_fail_links(req);
b63534c4
JA
2817#endif
2818 return false;
2819}
2820
a1d7c393 2821static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
889fca73 2822 unsigned int issue_flags)
a1d7c393
JA
2823{
2824 if (!io_rw_reissue(req, res))
889fca73 2825 io_complete_rw_common(&req->rw.kiocb, res, issue_flags);
ba816ad6
JA
2826}
2827
2828static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2829{
9adbd45d 2830 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
ba816ad6 2831
889fca73 2832 __io_complete_rw(req, res, res2, 0);
2b188cc1
JA
2833}
2834
def596e9
JA
2835static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2836{
9adbd45d 2837 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
def596e9 2838
491381ce
JA
2839 if (kiocb->ki_flags & IOCB_WRITE)
2840 kiocb_end_write(req);
def596e9 2841
2d7d6792 2842 if (res != -EAGAIN && res != req->result)
4e88d6e7 2843 req_set_fail_links(req);
bbde017a
XW
2844
2845 WRITE_ONCE(req->result, res);
2846 /* order with io_poll_complete() checking ->result */
cd664b0e
PB
2847 smp_wmb();
2848 WRITE_ONCE(req->iopoll_completed, 1);
def596e9
JA
2849}
2850
2851/*
2852 * After the iocb has been issued, it's safe to be found on the poll list.
2853 * Adding the kiocb to the list AFTER submission ensures that we don't
2854 * find it from a io_iopoll_getevents() thread before the issuer is done
2855 * accessing the kiocb cookie.
2856 */
2e9dbe90 2857static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async)
def596e9
JA
2858{
2859 struct io_ring_ctx *ctx = req->ctx;
2860
2861 /*
2862 * Track whether we have multiple files in our lists. This will impact
2863 * how we do polling eventually, not spinning if we're on potentially
2864 * different devices.
2865 */
540e32a0 2866 if (list_empty(&ctx->iopoll_list)) {
def596e9
JA
2867 ctx->poll_multi_file = false;
2868 } else if (!ctx->poll_multi_file) {
2869 struct io_kiocb *list_req;
2870
540e32a0 2871 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
d21ffe7e 2872 inflight_entry);
9adbd45d 2873 if (list_req->file != req->file)
def596e9
JA
2874 ctx->poll_multi_file = true;
2875 }
2876
2877 /*
2878 * For fast devices, IO may have already completed. If it has, add
2879 * it to the front so we find it first.
2880 */
65a6543d 2881 if (READ_ONCE(req->iopoll_completed))
d21ffe7e 2882 list_add(&req->inflight_entry, &ctx->iopoll_list);
def596e9 2883 else
d21ffe7e 2884 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
bdcd3eab 2885
2e9dbe90
XW
2886 /*
2887 * If IORING_SETUP_SQPOLL is enabled, sqes are either handled in sq thread
2888 * task context or in io worker task context. If current task context is
2889 * sq thread, we don't need to check whether should wake up sq thread.
2890 */
2891 if (in_async && (ctx->flags & IORING_SETUP_SQPOLL) &&
534ca6d6
JA
2892 wq_has_sleeper(&ctx->sq_data->wait))
2893 wake_up(&ctx->sq_data->wait);
def596e9
JA
2894}
2895
9f13c35b
PB
2896static inline void io_state_file_put(struct io_submit_state *state)
2897{
02b23a9a
PB
2898 if (state->file_refs) {
2899 fput_many(state->file, state->file_refs);
2900 state->file_refs = 0;
2901 }
9a56a232
JA
2902}
2903
2904/*
2905 * Get as many references to a file as we have IOs left in this submission,
2906 * assuming most submissions are for one file, or at least that each file
2907 * has more than one submission.
2908 */
8da11c19 2909static struct file *__io_file_get(struct io_submit_state *state, int fd)
9a56a232
JA
2910{
2911 if (!state)
2912 return fget(fd);
2913
6e1271e6 2914 if (state->file_refs) {
9a56a232 2915 if (state->fd == fd) {
6e1271e6 2916 state->file_refs--;
9a56a232
JA
2917 return state->file;
2918 }
02b23a9a 2919 io_state_file_put(state);
9a56a232
JA
2920 }
2921 state->file = fget_many(fd, state->ios_left);
6e1271e6 2922 if (unlikely(!state->file))
9a56a232
JA
2923 return NULL;
2924
2925 state->fd = fd;
6e1271e6 2926 state->file_refs = state->ios_left - 1;
9a56a232
JA
2927 return state->file;
2928}
2929
4503b767
JA
2930static bool io_bdev_nowait(struct block_device *bdev)
2931{
9ba0d0c8 2932 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
4503b767
JA
2933}
2934
2b188cc1
JA
2935/*
2936 * If we tracked the file through the SCM inflight mechanism, we could support
2937 * any file. For now, just ensure that anything potentially problematic is done
2938 * inline.
2939 */
af197f50 2940static bool io_file_supports_async(struct file *file, int rw)
2b188cc1
JA
2941{
2942 umode_t mode = file_inode(file)->i_mode;
2943
4503b767 2944 if (S_ISBLK(mode)) {
4e7b5671
CH
2945 if (IS_ENABLED(CONFIG_BLOCK) &&
2946 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
4503b767
JA
2947 return true;
2948 return false;
2949 }
2950 if (S_ISCHR(mode) || S_ISSOCK(mode))
2b188cc1 2951 return true;
4503b767 2952 if (S_ISREG(mode)) {
4e7b5671
CH
2953 if (IS_ENABLED(CONFIG_BLOCK) &&
2954 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
4503b767
JA
2955 file->f_op != &io_uring_fops)
2956 return true;
2957 return false;
2958 }
2b188cc1 2959
c5b85625
JA
2960 /* any ->read/write should understand O_NONBLOCK */
2961 if (file->f_flags & O_NONBLOCK)
2962 return true;
2963
af197f50
JA
2964 if (!(file->f_mode & FMODE_NOWAIT))
2965 return false;
2966
2967 if (rw == READ)
2968 return file->f_op->read_iter != NULL;
2969
2970 return file->f_op->write_iter != NULL;
2b188cc1
JA
2971}
2972
a88fc400 2973static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2b188cc1 2974{
def596e9 2975 struct io_ring_ctx *ctx = req->ctx;
9adbd45d 2976 struct kiocb *kiocb = &req->rw.kiocb;
75c668cd 2977 struct file *file = req->file;
09bb8394
JA
2978 unsigned ioprio;
2979 int ret;
2b188cc1 2980
75c668cd 2981 if (S_ISREG(file_inode(file)->i_mode))
491381ce
JA
2982 req->flags |= REQ_F_ISREG;
2983
2b188cc1 2984 kiocb->ki_pos = READ_ONCE(sqe->off);
75c668cd 2985 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
ba04291e 2986 req->flags |= REQ_F_CUR_POS;
75c668cd 2987 kiocb->ki_pos = file->f_pos;
ba04291e 2988 }
2b188cc1 2989 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
3e577dcd
PB
2990 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2991 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2992 if (unlikely(ret))
2993 return ret;
2b188cc1 2994
75c668cd
PB
2995 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2996 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2997 req->flags |= REQ_F_NOWAIT;
2998
2b188cc1
JA
2999 ioprio = READ_ONCE(sqe->ioprio);
3000 if (ioprio) {
3001 ret = ioprio_check_cap(ioprio);
3002 if (ret)
09bb8394 3003 return ret;
2b188cc1
JA
3004
3005 kiocb->ki_ioprio = ioprio;
3006 } else
3007 kiocb->ki_ioprio = get_current_ioprio();
3008
def596e9 3009 if (ctx->flags & IORING_SETUP_IOPOLL) {
def596e9
JA
3010 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
3011 !kiocb->ki_filp->f_op->iopoll)
09bb8394 3012 return -EOPNOTSUPP;
2b188cc1 3013
def596e9
JA
3014 kiocb->ki_flags |= IOCB_HIPRI;
3015 kiocb->ki_complete = io_complete_rw_iopoll;
65a6543d 3016 req->iopoll_completed = 0;
def596e9 3017 } else {
09bb8394
JA
3018 if (kiocb->ki_flags & IOCB_HIPRI)
3019 return -EINVAL;
def596e9
JA
3020 kiocb->ki_complete = io_complete_rw;
3021 }
9adbd45d 3022
3529d8c2
JA
3023 req->rw.addr = READ_ONCE(sqe->addr);
3024 req->rw.len = READ_ONCE(sqe->len);
4f4eeba8 3025 req->buf_index = READ_ONCE(sqe->buf_index);
2b188cc1 3026 return 0;
2b188cc1
JA
3027}
3028
3029static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
3030{
3031 switch (ret) {
3032 case -EIOCBQUEUED:
3033 break;
3034 case -ERESTARTSYS:
3035 case -ERESTARTNOINTR:
3036 case -ERESTARTNOHAND:
3037 case -ERESTART_RESTARTBLOCK:
3038 /*
3039 * We can't just restart the syscall, since previously
3040 * submitted sqes may already be in progress. Just fail this
3041 * IO with EINTR.
3042 */
3043 ret = -EINTR;
df561f66 3044 fallthrough;
2b188cc1
JA
3045 default:
3046 kiocb->ki_complete(kiocb, ret, 0);
3047 }
3048}
3049
a1d7c393 3050static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
889fca73 3051 unsigned int issue_flags)
ba816ad6 3052{
ba04291e 3053 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
e8c2bc1f 3054 struct io_async_rw *io = req->async_data;
ba04291e 3055
227c0c96 3056 /* add previously done IO, if any */
e8c2bc1f 3057 if (io && io->bytes_done > 0) {
227c0c96 3058 if (ret < 0)
e8c2bc1f 3059 ret = io->bytes_done;
227c0c96 3060 else
e8c2bc1f 3061 ret += io->bytes_done;
227c0c96
JA
3062 }
3063
ba04291e
JA
3064 if (req->flags & REQ_F_CUR_POS)
3065 req->file->f_pos = kiocb->ki_pos;
bcaec089 3066 if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
889fca73 3067 __io_complete_rw(req, ret, 0, issue_flags);
ba816ad6
JA
3068 else
3069 io_rw_done(kiocb, ret);
3070}
3071
847595de 3072static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
edafccee 3073{
9adbd45d
JA
3074 struct io_ring_ctx *ctx = req->ctx;
3075 size_t len = req->rw.len;
edafccee 3076 struct io_mapped_ubuf *imu;
4be1c615 3077 u16 index, buf_index = req->buf_index;
edafccee
JA
3078 size_t offset;
3079 u64 buf_addr;
3080
edafccee
JA
3081 if (unlikely(buf_index >= ctx->nr_user_bufs))
3082 return -EFAULT;
edafccee
JA
3083 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
3084 imu = &ctx->user_bufs[index];
9adbd45d 3085 buf_addr = req->rw.addr;
edafccee
JA
3086
3087 /* overflow */
3088 if (buf_addr + len < buf_addr)
3089 return -EFAULT;
3090 /* not inside the mapped region */
3091 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
3092 return -EFAULT;
3093
3094 /*
3095 * May not be a start of buffer, set size appropriately
3096 * and advance us to the beginning.
3097 */
3098 offset = buf_addr - imu->ubuf;
3099 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
bd11b3a3
JA
3100
3101 if (offset) {
3102 /*
3103 * Don't use iov_iter_advance() here, as it's really slow for
3104 * using the latter parts of a big fixed buffer - it iterates
3105 * over each segment manually. We can cheat a bit here, because
3106 * we know that:
3107 *
3108 * 1) it's a BVEC iter, we set it up
3109 * 2) all bvecs are PAGE_SIZE in size, except potentially the
3110 * first and last bvec
3111 *
3112 * So just find our index, and adjust the iterator afterwards.
3113 * If the offset is within the first bvec (or the whole first
3114 * bvec, just use iov_iter_advance(). This makes it easier
3115 * since we can just skip the first segment, which may not
3116 * be PAGE_SIZE aligned.
3117 */
3118 const struct bio_vec *bvec = imu->bvec;
3119
3120 if (offset <= bvec->bv_len) {
3121 iov_iter_advance(iter, offset);
3122 } else {
3123 unsigned long seg_skip;
3124
3125 /* skip first vec */
3126 offset -= bvec->bv_len;
3127 seg_skip = 1 + (offset >> PAGE_SHIFT);
3128
3129 iter->bvec = bvec + seg_skip;
3130 iter->nr_segs -= seg_skip;
99c79f66 3131 iter->count -= bvec->bv_len + offset;
bd11b3a3 3132 iter->iov_offset = offset & ~PAGE_MASK;
bd11b3a3
JA
3133 }
3134 }
3135
847595de 3136 return 0;
edafccee
JA
3137}
3138
bcda7baa
JA
3139static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
3140{
3141 if (needs_lock)
3142 mutex_unlock(&ctx->uring_lock);
3143}
3144
3145static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
3146{
3147 /*
3148 * "Normal" inline submissions always hold the uring_lock, since we
3149 * grab it from the system call. Same is true for the SQPOLL offload.
3150 * The only exception is when we've detached the request and issue it
3151 * from an async worker thread, grab the lock for that case.
3152 */
3153 if (needs_lock)
3154 mutex_lock(&ctx->uring_lock);
3155}
3156
3157static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
3158 int bgid, struct io_buffer *kbuf,
3159 bool needs_lock)
3160{
3161 struct io_buffer *head;
3162
3163 if (req->flags & REQ_F_BUFFER_SELECTED)
3164 return kbuf;
3165
3166 io_ring_submit_lock(req->ctx, needs_lock);
3167
3168 lockdep_assert_held(&req->ctx->uring_lock);
3169
3170 head = idr_find(&req->ctx->io_buffer_idr, bgid);
3171 if (head) {
3172 if (!list_empty(&head->list)) {
3173 kbuf = list_last_entry(&head->list, struct io_buffer,
3174 list);
3175 list_del(&kbuf->list);
3176 } else {
3177 kbuf = head;
3178 idr_remove(&req->ctx->io_buffer_idr, bgid);
3179 }
3180 if (*len > kbuf->len)
3181 *len = kbuf->len;
3182 } else {
3183 kbuf = ERR_PTR(-ENOBUFS);
3184 }
3185
3186 io_ring_submit_unlock(req->ctx, needs_lock);
3187
3188 return kbuf;
3189}
3190
4d954c25
JA
3191static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
3192 bool needs_lock)
3193{
3194 struct io_buffer *kbuf;
4f4eeba8 3195 u16 bgid;
4d954c25
JA
3196
3197 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
4f4eeba8 3198 bgid = req->buf_index;
4d954c25
JA
3199 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
3200 if (IS_ERR(kbuf))
3201 return kbuf;
3202 req->rw.addr = (u64) (unsigned long) kbuf;
3203 req->flags |= REQ_F_BUFFER_SELECTED;
3204 return u64_to_user_ptr(kbuf->addr);
3205}
3206
3207#ifdef CONFIG_COMPAT
3208static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
3209 bool needs_lock)
3210{
3211 struct compat_iovec __user *uiov;
3212 compat_ssize_t clen;
3213 void __user *buf;
3214 ssize_t len;
3215
3216 uiov = u64_to_user_ptr(req->rw.addr);
3217 if (!access_ok(uiov, sizeof(*uiov)))
3218 return -EFAULT;
3219 if (__get_user(clen, &uiov->iov_len))
3220 return -EFAULT;
3221 if (clen < 0)
3222 return -EINVAL;
3223
3224 len = clen;
3225 buf = io_rw_buffer_select(req, &len, needs_lock);
3226 if (IS_ERR(buf))
3227 return PTR_ERR(buf);
3228 iov[0].iov_base = buf;
3229 iov[0].iov_len = (compat_size_t) len;
3230 return 0;
3231}
3232#endif
3233
3234static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3235 bool needs_lock)
3236{
3237 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
3238 void __user *buf;
3239 ssize_t len;
3240
3241 if (copy_from_user(iov, uiov, sizeof(*uiov)))
3242 return -EFAULT;
3243
3244 len = iov[0].iov_len;
3245 if (len < 0)
3246 return -EINVAL;
3247 buf = io_rw_buffer_select(req, &len, needs_lock);
3248 if (IS_ERR(buf))
3249 return PTR_ERR(buf);
3250 iov[0].iov_base = buf;
3251 iov[0].iov_len = len;
3252 return 0;
3253}
3254
3255static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3256 bool needs_lock)
3257{
dddb3e26
JA
3258 if (req->flags & REQ_F_BUFFER_SELECTED) {
3259 struct io_buffer *kbuf;
3260
3261 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
3262 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3263 iov[0].iov_len = kbuf->len;
4d954c25 3264 return 0;
dddb3e26 3265 }
dd201662 3266 if (req->rw.len != 1)
4d954c25
JA
3267 return -EINVAL;
3268
3269#ifdef CONFIG_COMPAT
3270 if (req->ctx->compat)
3271 return io_compat_import(req, iov, needs_lock);
3272#endif
3273
3274 return __io_iov_buffer_select(req, iov, needs_lock);
3275}
3276
847595de
PB
3277static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
3278 struct iov_iter *iter, bool needs_lock)
2b188cc1 3279{
9adbd45d
JA
3280 void __user *buf = u64_to_user_ptr(req->rw.addr);
3281 size_t sqe_len = req->rw.len;
847595de 3282 u8 opcode = req->opcode;
4d954c25 3283 ssize_t ret;
edafccee 3284
7d009165 3285 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
edafccee 3286 *iovec = NULL;
9adbd45d 3287 return io_import_fixed(req, rw, iter);
edafccee 3288 }
2b188cc1 3289
bcda7baa 3290 /* buffer index only valid with fixed read/write, or buffer select */
4f4eeba8 3291 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
9adbd45d
JA
3292 return -EINVAL;
3293
3a6820f2 3294 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
bcda7baa 3295 if (req->flags & REQ_F_BUFFER_SELECT) {
4d954c25 3296 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
867a23ea 3297 if (IS_ERR(buf))
4d954c25 3298 return PTR_ERR(buf);
3f9d6441 3299 req->rw.len = sqe_len;
bcda7baa
JA
3300 }
3301
3a6820f2
JA
3302 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3303 *iovec = NULL;
10fc72e4 3304 return ret;
3a6820f2
JA
3305 }
3306
4d954c25
JA
3307 if (req->flags & REQ_F_BUFFER_SELECT) {
3308 ret = io_iov_buffer_select(req, *iovec, needs_lock);
847595de
PB
3309 if (!ret)
3310 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
4d954c25
JA
3311 *iovec = NULL;
3312 return ret;
3313 }
3314
89cd35c5
CH
3315 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3316 req->ctx->compat);
2b188cc1
JA
3317}
3318
0fef9483
JA
3319static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3320{
5b09e37e 3321 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
0fef9483
JA
3322}
3323
31b51510 3324/*
32960613
JA
3325 * For files that don't have ->read_iter() and ->write_iter(), handle them
3326 * by looping over ->read() or ->write() manually.
31b51510 3327 */
4017eb91 3328static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
32960613 3329{
4017eb91
JA
3330 struct kiocb *kiocb = &req->rw.kiocb;
3331 struct file *file = req->file;
32960613
JA
3332 ssize_t ret = 0;
3333
3334 /*
3335 * Don't support polled IO through this interface, and we can't
3336 * support non-blocking either. For the latter, this just causes
3337 * the kiocb to be handled from an async context.
3338 */
3339 if (kiocb->ki_flags & IOCB_HIPRI)
3340 return -EOPNOTSUPP;
3341 if (kiocb->ki_flags & IOCB_NOWAIT)
3342 return -EAGAIN;
3343
3344 while (iov_iter_count(iter)) {
311ae9e1 3345 struct iovec iovec;
32960613
JA
3346 ssize_t nr;
3347
311ae9e1
PB
3348 if (!iov_iter_is_bvec(iter)) {
3349 iovec = iov_iter_iovec(iter);
3350 } else {
4017eb91
JA
3351 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3352 iovec.iov_len = req->rw.len;
311ae9e1
PB
3353 }
3354
32960613
JA
3355 if (rw == READ) {
3356 nr = file->f_op->read(file, iovec.iov_base,
0fef9483 3357 iovec.iov_len, io_kiocb_ppos(kiocb));
32960613
JA
3358 } else {
3359 nr = file->f_op->write(file, iovec.iov_base,
0fef9483 3360 iovec.iov_len, io_kiocb_ppos(kiocb));
32960613
JA
3361 }
3362
3363 if (nr < 0) {
3364 if (!ret)
3365 ret = nr;
3366 break;
3367 }
3368 ret += nr;
3369 if (nr != iovec.iov_len)
3370 break;
4017eb91
JA
3371 req->rw.len -= nr;
3372 req->rw.addr += nr;
32960613
JA
3373 iov_iter_advance(iter, nr);
3374 }
3375
3376 return ret;
3377}
3378
ff6165b2
JA
3379static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3380 const struct iovec *fast_iov, struct iov_iter *iter)
f67676d1 3381{
e8c2bc1f 3382 struct io_async_rw *rw = req->async_data;
b64e3444 3383
ff6165b2 3384 memcpy(&rw->iter, iter, sizeof(*iter));
afb87658 3385 rw->free_iovec = iovec;
227c0c96 3386 rw->bytes_done = 0;
ff6165b2 3387 /* can only be fixed buffers, no need to do anything */
9c3a205c 3388 if (iov_iter_is_bvec(iter))
ff6165b2 3389 return;
b64e3444 3390 if (!iovec) {
ff6165b2
JA
3391 unsigned iov_off = 0;
3392
3393 rw->iter.iov = rw->fast_iov;
3394 if (iter->iov != fast_iov) {
3395 iov_off = iter->iov - fast_iov;
3396 rw->iter.iov += iov_off;
3397 }
3398 if (rw->fast_iov != fast_iov)
3399 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
45097dae 3400 sizeof(struct iovec) * iter->nr_segs);
99bc4c38
PB
3401 } else {
3402 req->flags |= REQ_F_NEED_CLEANUP;
f67676d1
JA
3403 }
3404}
3405
e8c2bc1f 3406static inline int __io_alloc_async_data(struct io_kiocb *req)
3d9932a8 3407{
e8c2bc1f
JA
3408 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3409 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3410 return req->async_data == NULL;
3d9932a8
XW
3411}
3412
e8c2bc1f 3413static int io_alloc_async_data(struct io_kiocb *req)
f67676d1 3414{
e8c2bc1f 3415 if (!io_op_defs[req->opcode].needs_async_data)
d3656344 3416 return 0;
3d9932a8 3417
e8c2bc1f 3418 return __io_alloc_async_data(req);
b7bb4f7d
JA
3419}
3420
ff6165b2
JA
3421static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3422 const struct iovec *fast_iov,
227c0c96 3423 struct iov_iter *iter, bool force)
b7bb4f7d 3424{
e8c2bc1f 3425 if (!force && !io_op_defs[req->opcode].needs_async_data)
74566df3 3426 return 0;
e8c2bc1f 3427 if (!req->async_data) {
6bf985dc
PB
3428 if (__io_alloc_async_data(req)) {
3429 kfree(iovec);
5d204bcf 3430 return -ENOMEM;
6bf985dc 3431 }
b7bb4f7d 3432
ff6165b2 3433 io_req_map_rw(req, iovec, fast_iov, iter);
5d204bcf 3434 }
b7bb4f7d 3435 return 0;
f67676d1
JA
3436}
3437
73debe68 3438static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
c3e330a4 3439{
e8c2bc1f 3440 struct io_async_rw *iorw = req->async_data;
f4bff104 3441 struct iovec *iov = iorw->fast_iov;
847595de 3442 int ret;
c3e330a4 3443
2846c481 3444 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
c3e330a4
PB
3445 if (unlikely(ret < 0))
3446 return ret;
3447
ab0b196c
PB
3448 iorw->bytes_done = 0;
3449 iorw->free_iovec = iov;
3450 if (iov)
3451 req->flags |= REQ_F_NEED_CLEANUP;
c3e330a4
PB
3452 return 0;
3453}
3454
73debe68 3455static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1
JA
3456{
3457 ssize_t ret;
3458
a88fc400 3459 ret = io_prep_rw(req, sqe);
3529d8c2
JA
3460 if (ret)
3461 return ret;
f67676d1 3462
3529d8c2
JA
3463 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3464 return -EBADF;
f67676d1 3465
5f798bea 3466 /* either don't need iovec imported or already have it */
2d199895 3467 if (!req->async_data)
3529d8c2 3468 return 0;
73debe68 3469 return io_rw_prep_async(req, READ);
f67676d1
JA
3470}
3471
c1dd91d1
JA
3472/*
3473 * This is our waitqueue callback handler, registered through lock_page_async()
3474 * when we initially tried to do the IO with the iocb armed our waitqueue.
3475 * This gets called when the page is unlocked, and we generally expect that to
3476 * happen when the page IO is completed and the page is now uptodate. This will
3477 * queue a task_work based retry of the operation, attempting to copy the data
3478 * again. If the latter fails because the page was NOT uptodate, then we will
3479 * do a thread based blocking retry of the operation. That's the unexpected
3480 * slow path.
3481 */
bcf5a063
JA
3482static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3483 int sync, void *arg)
3484{
3485 struct wait_page_queue *wpq;
3486 struct io_kiocb *req = wait->private;
bcf5a063 3487 struct wait_page_key *key = arg;
bcf5a063
JA
3488 int ret;
3489
3490 wpq = container_of(wait, struct wait_page_queue, wait);
3491
cdc8fcb4
LT
3492 if (!wake_page_match(wpq, key))
3493 return 0;
3494
c8d317aa 3495 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
bcf5a063
JA
3496 list_del_init(&wait->entry);
3497
7cbf1722 3498 req->task_work.func = io_req_task_submit;
6d816e08
JA
3499 percpu_ref_get(&req->ctx->refs);
3500
bcf5a063
JA
3501 /* submit ref gets dropped, acquire a new one */
3502 refcount_inc(&req->refs);
355fb9e2 3503 ret = io_req_task_work_add(req);
eab30c4d
PB
3504 if (unlikely(ret))
3505 io_req_task_work_add_fallback(req, io_req_task_cancel);
bcf5a063
JA
3506 return 1;
3507}
3508
c1dd91d1
JA
3509/*
3510 * This controls whether a given IO request should be armed for async page
3511 * based retry. If we return false here, the request is handed to the async
3512 * worker threads for retry. If we're doing buffered reads on a regular file,
3513 * we prepare a private wait_page_queue entry and retry the operation. This
3514 * will either succeed because the page is now uptodate and unlocked, or it
3515 * will register a callback when the page is unlocked at IO completion. Through
3516 * that callback, io_uring uses task_work to setup a retry of the operation.
3517 * That retry will attempt the buffered read again. The retry will generally
3518 * succeed, or in rare cases where it fails, we then fall back to using the
3519 * async worker threads for a blocking retry.
3520 */
227c0c96 3521static bool io_rw_should_retry(struct io_kiocb *req)
f67676d1 3522{
e8c2bc1f
JA
3523 struct io_async_rw *rw = req->async_data;
3524 struct wait_page_queue *wait = &rw->wpq;
bcf5a063 3525 struct kiocb *kiocb = &req->rw.kiocb;
f67676d1 3526
bcf5a063
JA
3527 /* never retry for NOWAIT, we just complete with -EAGAIN */
3528 if (req->flags & REQ_F_NOWAIT)
3529 return false;
f67676d1 3530
227c0c96 3531 /* Only for buffered IO */
3b2a4439 3532 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
bcf5a063 3533 return false;
3b2a4439 3534
bcf5a063
JA
3535 /*
3536 * just use poll if we can, and don't attempt if the fs doesn't
3537 * support callback based unlocks
3538 */
3539 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3540 return false;
f67676d1 3541
3b2a4439
JA
3542 wait->wait.func = io_async_buf_func;
3543 wait->wait.private = req;
3544 wait->wait.flags = 0;
3545 INIT_LIST_HEAD(&wait->wait.entry);
3546 kiocb->ki_flags |= IOCB_WAITQ;
c8d317aa 3547 kiocb->ki_flags &= ~IOCB_NOWAIT;
3b2a4439 3548 kiocb->ki_waitq = wait;
3b2a4439 3549 return true;
bcf5a063
JA
3550}
3551
3552static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
3553{
3554 if (req->file->f_op->read_iter)
3555 return call_read_iter(req->file, &req->rw.kiocb, iter);
2dd2111d 3556 else if (req->file->f_op->read)
4017eb91 3557 return loop_rw_iter(READ, req, iter);
2dd2111d
GH
3558 else
3559 return -EINVAL;
f67676d1
JA
3560}
3561
889fca73 3562static int io_read(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3563{
3564 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 3565 struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2 3566 struct iov_iter __iter, *iter = &__iter;
e8c2bc1f 3567 struct io_async_rw *rw = req->async_data;
227c0c96 3568 ssize_t io_size, ret, ret2;
45d189c6 3569 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ff6165b2 3570
2846c481 3571 if (rw) {
e8c2bc1f 3572 iter = &rw->iter;
2846c481
PB
3573 iovec = NULL;
3574 } else {
3575 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3576 if (ret < 0)
3577 return ret;
3578 }
632546c4 3579 io_size = iov_iter_count(iter);
fa15bafb 3580 req->result = io_size;
2b188cc1 3581
fd6c2e4c
JA
3582 /* Ensure we clear previously set non-block flag */
3583 if (!force_nonblock)
29de5f6a 3584 kiocb->ki_flags &= ~IOCB_NOWAIT;
a88fc400
PB
3585 else
3586 kiocb->ki_flags |= IOCB_NOWAIT;
3587
24c74678 3588 /* If the file doesn't support async, just async punt */
6713e7a6
PB
3589 if (force_nonblock && !io_file_supports_async(req->file, READ)) {
3590 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc 3591 return ret ?: -EAGAIN;
6713e7a6 3592 }
9e645e11 3593
632546c4 3594 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
5ea5dd45
PB
3595 if (unlikely(ret)) {
3596 kfree(iovec);
3597 return ret;
3598 }
2b188cc1 3599
227c0c96 3600 ret = io_iter_do_read(req, iter);
32960613 3601
57cd657b 3602 if (ret == -EIOCBQUEUED) {
5ea5dd45
PB
3603 /* it's faster to check here then delegate to kfree */
3604 if (iovec)
3605 kfree(iovec);
3606 return 0;
227c0c96 3607 } else if (ret == -EAGAIN) {
eefdf30f
JA
3608 /* IOPOLL retry should happen for io-wq threads */
3609 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
f91daf56 3610 goto done;
75c668cd
PB
3611 /* no retry on NONBLOCK nor RWF_NOWAIT */
3612 if (req->flags & REQ_F_NOWAIT)
355afaeb 3613 goto done;
84216315 3614 /* some cases will consume bytes even on error returns */
632546c4 3615 iov_iter_revert(iter, io_size - iov_iter_count(iter));
f38c7e3a 3616 ret = 0;
7335e3bf 3617 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
75c668cd 3618 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
7335e3bf 3619 /* read all, failed, already did sync or don't want to retry */
00d23d51 3620 goto done;
1a2cc0ce 3621 }
227c0c96 3622
227c0c96 3623 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc
PB
3624 if (ret2)
3625 return ret2;
3626
e8c2bc1f 3627 rw = req->async_data;
227c0c96 3628 /* now use our persistent iterator, if we aren't already */
e8c2bc1f 3629 iter = &rw->iter;
227c0c96 3630
b23df91b
PB
3631 do {
3632 io_size -= ret;
3633 rw->bytes_done += ret;
3634 /* if we can retry, do so with the callbacks armed */
3635 if (!io_rw_should_retry(req)) {
3636 kiocb->ki_flags &= ~IOCB_WAITQ;
3637 return -EAGAIN;
3638 }
3639
3640 /*
3641 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3642 * we get -EIOCBQUEUED, then we'll get a notification when the
3643 * desired page gets unlocked. We can also get a partial read
3644 * here, and if we do, then just retry at the new offset.
3645 */
3646 ret = io_iter_do_read(req, iter);
3647 if (ret == -EIOCBQUEUED)
3648 return 0;
3649 /* we got some bytes, but not all. retry. */
3650 } while (ret > 0 && ret < io_size);
227c0c96 3651done:
889fca73 3652 kiocb_done(kiocb, ret, issue_flags);
5ea5dd45 3653 return 0;
2b188cc1
JA
3654}
3655
73debe68 3656static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1
JA
3657{
3658 ssize_t ret;
3659
a88fc400 3660 ret = io_prep_rw(req, sqe);
3529d8c2
JA
3661 if (ret)
3662 return ret;
f67676d1 3663
3529d8c2
JA
3664 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3665 return -EBADF;
f67676d1 3666
5f798bea 3667 /* either don't need iovec imported or already have it */
2d199895 3668 if (!req->async_data)
3529d8c2 3669 return 0;
73debe68 3670 return io_rw_prep_async(req, WRITE);
f67676d1
JA
3671}
3672
889fca73 3673static int io_write(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3674{
3675 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 3676 struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2 3677 struct iov_iter __iter, *iter = &__iter;
e8c2bc1f 3678 struct io_async_rw *rw = req->async_data;
fa15bafb 3679 ssize_t ret, ret2, io_size;
45d189c6 3680 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
2b188cc1 3681
2846c481 3682 if (rw) {
e8c2bc1f 3683 iter = &rw->iter;
2846c481
PB
3684 iovec = NULL;
3685 } else {
3686 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3687 if (ret < 0)
3688 return ret;
3689 }
632546c4 3690 io_size = iov_iter_count(iter);
fa15bafb 3691 req->result = io_size;
2b188cc1 3692
fd6c2e4c
JA
3693 /* Ensure we clear previously set non-block flag */
3694 if (!force_nonblock)
a88fc400
PB
3695 kiocb->ki_flags &= ~IOCB_NOWAIT;
3696 else
3697 kiocb->ki_flags |= IOCB_NOWAIT;
fd6c2e4c 3698
24c74678 3699 /* If the file doesn't support async, just async punt */
af197f50 3700 if (force_nonblock && !io_file_supports_async(req->file, WRITE))
f67676d1 3701 goto copy_iov;
31b51510 3702
10d59345
JA
3703 /* file path doesn't support NOWAIT for non-direct_IO */
3704 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3705 (req->flags & REQ_F_ISREG))
f67676d1 3706 goto copy_iov;
31b51510 3707
632546c4 3708 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
fa15bafb
PB
3709 if (unlikely(ret))
3710 goto out_free;
4ed734b0 3711
fa15bafb
PB
3712 /*
3713 * Open-code file_start_write here to grab freeze protection,
3714 * which will be released by another thread in
3715 * io_complete_rw(). Fool lockdep by telling it the lock got
3716 * released so that it doesn't complain about the held lock when
3717 * we return to userspace.
3718 */
3719 if (req->flags & REQ_F_ISREG) {
8a3c84b6 3720 sb_start_write(file_inode(req->file)->i_sb);
fa15bafb
PB
3721 __sb_writers_release(file_inode(req->file)->i_sb,
3722 SB_FREEZE_WRITE);
3723 }
3724 kiocb->ki_flags |= IOCB_WRITE;
4ed734b0 3725
fa15bafb 3726 if (req->file->f_op->write_iter)
ff6165b2 3727 ret2 = call_write_iter(req->file, kiocb, iter);
2dd2111d 3728 else if (req->file->f_op->write)
4017eb91 3729 ret2 = loop_rw_iter(WRITE, req, iter);
2dd2111d
GH
3730 else
3731 ret2 = -EINVAL;
4ed734b0 3732
fa15bafb
PB
3733 /*
3734 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3735 * retry them without IOCB_NOWAIT.
3736 */
3737 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3738 ret2 = -EAGAIN;
75c668cd
PB
3739 /* no retry on NONBLOCK nor RWF_NOWAIT */
3740 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
355afaeb 3741 goto done;
fa15bafb 3742 if (!force_nonblock || ret2 != -EAGAIN) {
eefdf30f
JA
3743 /* IOPOLL retry should happen for io-wq threads */
3744 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3745 goto copy_iov;
355afaeb 3746done:
889fca73 3747 kiocb_done(kiocb, ret2, issue_flags);
fa15bafb 3748 } else {
f67676d1 3749copy_iov:
84216315 3750 /* some cases will consume bytes even on error returns */
632546c4 3751 iov_iter_revert(iter, io_size - iov_iter_count(iter));
227c0c96 3752 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
6bf985dc 3753 return ret ?: -EAGAIN;
2b188cc1 3754 }
31b51510 3755out_free:
f261c168 3756 /* it's reportedly faster than delegating the null check to kfree() */
252917c3 3757 if (iovec)
6f2cc166 3758 kfree(iovec);
2b188cc1
JA
3759 return ret;
3760}
3761
80a261fd
JA
3762static int io_renameat_prep(struct io_kiocb *req,
3763 const struct io_uring_sqe *sqe)
3764{
3765 struct io_rename *ren = &req->rename;
3766 const char __user *oldf, *newf;
3767
3768 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3769 return -EBADF;
3770
3771 ren->old_dfd = READ_ONCE(sqe->fd);
3772 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3773 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3774 ren->new_dfd = READ_ONCE(sqe->len);
3775 ren->flags = READ_ONCE(sqe->rename_flags);
3776
3777 ren->oldpath = getname(oldf);
3778 if (IS_ERR(ren->oldpath))
3779 return PTR_ERR(ren->oldpath);
3780
3781 ren->newpath = getname(newf);
3782 if (IS_ERR(ren->newpath)) {
3783 putname(ren->oldpath);
3784 return PTR_ERR(ren->newpath);
3785 }
3786
3787 req->flags |= REQ_F_NEED_CLEANUP;
3788 return 0;
3789}
3790
45d189c6 3791static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
80a261fd
JA
3792{
3793 struct io_rename *ren = &req->rename;
3794 int ret;
3795
45d189c6 3796 if (issue_flags & IO_URING_F_NONBLOCK)
80a261fd
JA
3797 return -EAGAIN;
3798
3799 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3800 ren->newpath, ren->flags);
3801
3802 req->flags &= ~REQ_F_NEED_CLEANUP;
3803 if (ret < 0)
3804 req_set_fail_links(req);
3805 io_req_complete(req, ret);
3806 return 0;
3807}
3808
14a1143b
JA
3809static int io_unlinkat_prep(struct io_kiocb *req,
3810 const struct io_uring_sqe *sqe)
3811{
3812 struct io_unlink *un = &req->unlink;
3813 const char __user *fname;
3814
3815 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3816 return -EBADF;
3817
3818 un->dfd = READ_ONCE(sqe->fd);
3819
3820 un->flags = READ_ONCE(sqe->unlink_flags);
3821 if (un->flags & ~AT_REMOVEDIR)
3822 return -EINVAL;
3823
3824 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3825 un->filename = getname(fname);
3826 if (IS_ERR(un->filename))
3827 return PTR_ERR(un->filename);
3828
3829 req->flags |= REQ_F_NEED_CLEANUP;
3830 return 0;
3831}
3832
45d189c6 3833static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
14a1143b
JA
3834{
3835 struct io_unlink *un = &req->unlink;
3836 int ret;
3837
45d189c6 3838 if (issue_flags & IO_URING_F_NONBLOCK)
14a1143b
JA
3839 return -EAGAIN;
3840
3841 if (un->flags & AT_REMOVEDIR)
3842 ret = do_rmdir(un->dfd, un->filename);
3843 else
3844 ret = do_unlinkat(un->dfd, un->filename);
3845
3846 req->flags &= ~REQ_F_NEED_CLEANUP;
3847 if (ret < 0)
3848 req_set_fail_links(req);
3849 io_req_complete(req, ret);
3850 return 0;
3851}
3852
36f4fa68
JA
3853static int io_shutdown_prep(struct io_kiocb *req,
3854 const struct io_uring_sqe *sqe)
3855{
3856#if defined(CONFIG_NET)
3857 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3858 return -EINVAL;
3859 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3860 sqe->buf_index)
3861 return -EINVAL;
3862
3863 req->shutdown.how = READ_ONCE(sqe->len);
3864 return 0;
3865#else
3866 return -EOPNOTSUPP;
3867#endif
3868}
3869
45d189c6 3870static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
36f4fa68
JA
3871{
3872#if defined(CONFIG_NET)
3873 struct socket *sock;
3874 int ret;
3875
45d189c6 3876 if (issue_flags & IO_URING_F_NONBLOCK)
36f4fa68
JA
3877 return -EAGAIN;
3878
48aba79b 3879 sock = sock_from_file(req->file);
36f4fa68 3880 if (unlikely(!sock))
48aba79b 3881 return -ENOTSOCK;
36f4fa68
JA
3882
3883 ret = __sys_shutdown_sock(sock, req->shutdown.how);
a146468d
JA
3884 if (ret < 0)
3885 req_set_fail_links(req);
36f4fa68
JA
3886 io_req_complete(req, ret);
3887 return 0;
3888#else
3889 return -EOPNOTSUPP;
3890#endif
3891}
3892
f2a8d5c7
PB
3893static int __io_splice_prep(struct io_kiocb *req,
3894 const struct io_uring_sqe *sqe)
7d67af2c
PB
3895{
3896 struct io_splice* sp = &req->splice;
3897 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
7d67af2c 3898
3232dd02
PB
3899 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3900 return -EINVAL;
7d67af2c
PB
3901
3902 sp->file_in = NULL;
7d67af2c
PB
3903 sp->len = READ_ONCE(sqe->len);
3904 sp->flags = READ_ONCE(sqe->splice_flags);
3905
3906 if (unlikely(sp->flags & ~valid_flags))
3907 return -EINVAL;
3908
8371adf5
PB
3909 sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
3910 (sp->flags & SPLICE_F_FD_IN_FIXED));
3911 if (!sp->file_in)
3912 return -EBADF;
7d67af2c
PB
3913 req->flags |= REQ_F_NEED_CLEANUP;
3914
7cdaf587
XW
3915 if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
3916 /*
3917 * Splice operation will be punted aync, and here need to
3918 * modify io_wq_work.flags, so initialize io_wq_work firstly.
3919 */
3920 io_req_init_async(req);
7d67af2c 3921 req->work.flags |= IO_WQ_WORK_UNBOUND;
7cdaf587 3922 }
7d67af2c
PB
3923
3924 return 0;
3925}
3926
f2a8d5c7
PB
3927static int io_tee_prep(struct io_kiocb *req,
3928 const struct io_uring_sqe *sqe)
3929{
3930 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3931 return -EINVAL;
3932 return __io_splice_prep(req, sqe);
3933}
3934
45d189c6 3935static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
f2a8d5c7
PB
3936{
3937 struct io_splice *sp = &req->splice;
3938 struct file *in = sp->file_in;
3939 struct file *out = sp->file_out;
3940 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3941 long ret = 0;
3942
45d189c6 3943 if (issue_flags & IO_URING_F_NONBLOCK)
f2a8d5c7
PB
3944 return -EAGAIN;
3945 if (sp->len)
3946 ret = do_tee(in, out, sp->len, flags);
3947
3948 io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
3949 req->flags &= ~REQ_F_NEED_CLEANUP;
3950
f2a8d5c7
PB
3951 if (ret != sp->len)
3952 req_set_fail_links(req);
e1e16097 3953 io_req_complete(req, ret);
f2a8d5c7
PB
3954 return 0;
3955}
3956
3957static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3958{
3959 struct io_splice* sp = &req->splice;
3960
3961 sp->off_in = READ_ONCE(sqe->splice_off_in);
3962 sp->off_out = READ_ONCE(sqe->off);
3963 return __io_splice_prep(req, sqe);
3964}
3965
45d189c6 3966static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
7d67af2c
PB
3967{
3968 struct io_splice *sp = &req->splice;
3969 struct file *in = sp->file_in;
3970 struct file *out = sp->file_out;
3971 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3972 loff_t *poff_in, *poff_out;
c9687426 3973 long ret = 0;
7d67af2c 3974
45d189c6 3975 if (issue_flags & IO_URING_F_NONBLOCK)
2fb3e822 3976 return -EAGAIN;
7d67af2c
PB
3977
3978 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3979 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
c9687426 3980
948a7749 3981 if (sp->len)
c9687426 3982 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
7d67af2c
PB
3983
3984 io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
3985 req->flags &= ~REQ_F_NEED_CLEANUP;
3986
7d67af2c
PB
3987 if (ret != sp->len)
3988 req_set_fail_links(req);
e1e16097 3989 io_req_complete(req, ret);
7d67af2c
PB
3990 return 0;
3991}
3992
2b188cc1
JA
3993/*
3994 * IORING_OP_NOP just posts a completion event, nothing else.
3995 */
889fca73 3996static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3997{
3998 struct io_ring_ctx *ctx = req->ctx;
2b188cc1 3999
def596e9
JA
4000 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4001 return -EINVAL;
4002
889fca73 4003 __io_req_complete(req, issue_flags, 0, 0);
2b188cc1
JA
4004 return 0;
4005}
4006
3529d8c2 4007static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
c992fe29 4008{
6b06314c 4009 struct io_ring_ctx *ctx = req->ctx;
c992fe29 4010
09bb8394
JA
4011 if (!req->file)
4012 return -EBADF;
c992fe29 4013
6b06314c 4014 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
def596e9 4015 return -EINVAL;
edafccee 4016 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
c992fe29
CH
4017 return -EINVAL;
4018
8ed8d3c3
JA
4019 req->sync.flags = READ_ONCE(sqe->fsync_flags);
4020 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
4021 return -EINVAL;
4022
4023 req->sync.off = READ_ONCE(sqe->off);
4024 req->sync.len = READ_ONCE(sqe->len);
c992fe29
CH
4025 return 0;
4026}
4027
45d189c6 4028static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3 4029{
8ed8d3c3 4030 loff_t end = req->sync.off + req->sync.len;
8ed8d3c3
JA
4031 int ret;
4032
ac45abc0 4033 /* fsync always requires a blocking context */
45d189c6 4034 if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0
PB
4035 return -EAGAIN;
4036
9adbd45d 4037 ret = vfs_fsync_range(req->file, req->sync.off,
8ed8d3c3
JA
4038 end > 0 ? end : LLONG_MAX,
4039 req->sync.flags & IORING_FSYNC_DATASYNC);
4040 if (ret < 0)
4041 req_set_fail_links(req);
e1e16097 4042 io_req_complete(req, ret);
c992fe29
CH
4043 return 0;
4044}
4045
d63d1b5e
JA
4046static int io_fallocate_prep(struct io_kiocb *req,
4047 const struct io_uring_sqe *sqe)
4048{
4049 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
4050 return -EINVAL;
3232dd02
PB
4051 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4052 return -EINVAL;
d63d1b5e
JA
4053
4054 req->sync.off = READ_ONCE(sqe->off);
4055 req->sync.len = READ_ONCE(sqe->addr);
4056 req->sync.mode = READ_ONCE(sqe->len);
4057 return 0;
4058}
4059
45d189c6 4060static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
5d17b4a4 4061{
ac45abc0
PB
4062 int ret;
4063
d63d1b5e 4064 /* fallocate always requiring blocking context */
45d189c6 4065 if (issue_flags & IO_URING_F_NONBLOCK)
5d17b4a4 4066 return -EAGAIN;
ac45abc0
PB
4067 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
4068 req->sync.len);
ac45abc0
PB
4069 if (ret < 0)
4070 req_set_fail_links(req);
e1e16097 4071 io_req_complete(req, ret);
5d17b4a4
JA
4072 return 0;
4073}
4074
ec65fea5 4075static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
b7bb4f7d 4076{
f8748881 4077 const char __user *fname;
15b71abe 4078 int ret;
b7bb4f7d 4079
ec65fea5 4080 if (unlikely(sqe->ioprio || sqe->buf_index))
15b71abe 4081 return -EINVAL;
ec65fea5 4082 if (unlikely(req->flags & REQ_F_FIXED_FILE))
cf3040ca 4083 return -EBADF;
03b1230c 4084
ec65fea5
PB
4085 /* open.how should be already initialised */
4086 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
08a1d26e 4087 req->open.how.flags |= O_LARGEFILE;
3529d8c2 4088
25e72d10
PB
4089 req->open.dfd = READ_ONCE(sqe->fd);
4090 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
f8748881 4091 req->open.filename = getname(fname);
15b71abe
JA
4092 if (IS_ERR(req->open.filename)) {
4093 ret = PTR_ERR(req->open.filename);
4094 req->open.filename = NULL;
4095 return ret;
4096 }
4022e7af 4097 req->open.nofile = rlimit(RLIMIT_NOFILE);
8fef80bf 4098 req->flags |= REQ_F_NEED_CLEANUP;
15b71abe 4099 return 0;
03b1230c
JA
4100}
4101
ec65fea5
PB
4102static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4103{
4104 u64 flags, mode;
4105
14587a46 4106 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded 4107 return -EINVAL;
ec65fea5
PB
4108 mode = READ_ONCE(sqe->len);
4109 flags = READ_ONCE(sqe->open_flags);
4110 req->open.how = build_open_how(flags, mode);
4111 return __io_openat_prep(req, sqe);
4112}
4113
cebdb986 4114static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
aa1fa28f 4115{
cebdb986 4116 struct open_how __user *how;
cebdb986 4117 size_t len;
0fa03c62
JA
4118 int ret;
4119
14587a46 4120 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded 4121 return -EINVAL;
cebdb986
JA
4122 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4123 len = READ_ONCE(sqe->len);
cebdb986
JA
4124 if (len < OPEN_HOW_SIZE_VER0)
4125 return -EINVAL;
3529d8c2 4126
cebdb986
JA
4127 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
4128 len);
4129 if (ret)
4130 return ret;
3529d8c2 4131
ec65fea5 4132 return __io_openat_prep(req, sqe);
cebdb986
JA
4133}
4134
45d189c6 4135static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
15b71abe
JA
4136{
4137 struct open_flags op;
15b71abe 4138 struct file *file;
3a81fd02
JA
4139 bool nonblock_set;
4140 bool resolve_nonblock;
15b71abe
JA
4141 int ret;
4142
cebdb986 4143 ret = build_open_flags(&req->open.how, &op);
15b71abe
JA
4144 if (ret)
4145 goto err;
3a81fd02
JA
4146 nonblock_set = op.open_flag & O_NONBLOCK;
4147 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
45d189c6 4148 if (issue_flags & IO_URING_F_NONBLOCK) {
3a81fd02
JA
4149 /*
4150 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
4151 * it'll always -EAGAIN
4152 */
4153 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
4154 return -EAGAIN;
4155 op.lookup_flags |= LOOKUP_CACHED;
4156 op.open_flag |= O_NONBLOCK;
4157 }
15b71abe 4158
4022e7af 4159 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
15b71abe
JA
4160 if (ret < 0)
4161 goto err;
4162
4163 file = do_filp_open(req->open.dfd, req->open.filename, &op);
3a81fd02 4164 /* only retry if RESOLVE_CACHED wasn't already set by application */
45d189c6
PB
4165 if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) &&
4166 file == ERR_PTR(-EAGAIN)) {
3a81fd02
JA
4167 /*
4168 * We could hang on to this 'fd', but seems like marginal
4169 * gain for something that is now known to be a slower path.
4170 * So just put it, and we'll get a new one when we retry.
4171 */
4172 put_unused_fd(ret);
4173 return -EAGAIN;
4174 }
4175
15b71abe
JA
4176 if (IS_ERR(file)) {
4177 put_unused_fd(ret);
4178 ret = PTR_ERR(file);
4179 } else {
45d189c6 4180 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3a81fd02 4181 file->f_flags &= ~O_NONBLOCK;
15b71abe
JA
4182 fsnotify_open(file);
4183 fd_install(ret, file);
4184 }
4185err:
4186 putname(req->open.filename);
8fef80bf 4187 req->flags &= ~REQ_F_NEED_CLEANUP;
15b71abe
JA
4188 if (ret < 0)
4189 req_set_fail_links(req);
e1e16097 4190 io_req_complete(req, ret);
15b71abe
JA
4191 return 0;
4192}
4193
45d189c6 4194static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
cebdb986 4195{
45d189c6 4196 return io_openat2(req, issue_flags & IO_URING_F_NONBLOCK);
cebdb986
JA
4197}
4198
067524e9
JA
4199static int io_remove_buffers_prep(struct io_kiocb *req,
4200 const struct io_uring_sqe *sqe)
4201{
4202 struct io_provide_buf *p = &req->pbuf;
4203 u64 tmp;
4204
4205 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
4206 return -EINVAL;
4207
4208 tmp = READ_ONCE(sqe->fd);
4209 if (!tmp || tmp > USHRT_MAX)
4210 return -EINVAL;
4211
4212 memset(p, 0, sizeof(*p));
4213 p->nbufs = tmp;
4214 p->bgid = READ_ONCE(sqe->buf_group);
4215 return 0;
4216}
4217
4218static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
4219 int bgid, unsigned nbufs)
4220{
4221 unsigned i = 0;
4222
4223 /* shouldn't happen */
4224 if (!nbufs)
4225 return 0;
4226
4227 /* the head kbuf is the list itself */
4228 while (!list_empty(&buf->list)) {
4229 struct io_buffer *nxt;
4230
4231 nxt = list_first_entry(&buf->list, struct io_buffer, list);
4232 list_del(&nxt->list);
4233 kfree(nxt);
4234 if (++i == nbufs)
4235 return i;
4236 }
4237 i++;
4238 kfree(buf);
4239 idr_remove(&ctx->io_buffer_idr, bgid);
4240
4241 return i;
4242}
4243
889fca73 4244static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
067524e9
JA
4245{
4246 struct io_provide_buf *p = &req->pbuf;
4247 struct io_ring_ctx *ctx = req->ctx;
4248 struct io_buffer *head;
4249 int ret = 0;
45d189c6 4250 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
067524e9
JA
4251
4252 io_ring_submit_lock(ctx, !force_nonblock);
4253
4254 lockdep_assert_held(&ctx->uring_lock);
4255
4256 ret = -ENOENT;
4257 head = idr_find(&ctx->io_buffer_idr, p->bgid);
4258 if (head)
4259 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
067524e9
JA
4260 if (ret < 0)
4261 req_set_fail_links(req);
067524e9 4262
31bff9a5
PB
4263 /* need to hold the lock to complete IOPOLL requests */
4264 if (ctx->flags & IORING_SETUP_IOPOLL) {
889fca73 4265 __io_req_complete(req, issue_flags, ret, 0);
31bff9a5
PB
4266 io_ring_submit_unlock(ctx, !force_nonblock);
4267 } else {
4268 io_ring_submit_unlock(ctx, !force_nonblock);
889fca73 4269 __io_req_complete(req, issue_flags, ret, 0);
31bff9a5 4270 }
067524e9
JA
4271 return 0;
4272}
4273
ddf0322d
JA
4274static int io_provide_buffers_prep(struct io_kiocb *req,
4275 const struct io_uring_sqe *sqe)
4276{
4277 struct io_provide_buf *p = &req->pbuf;
4278 u64 tmp;
4279
4280 if (sqe->ioprio || sqe->rw_flags)
4281 return -EINVAL;
4282
4283 tmp = READ_ONCE(sqe->fd);
4284 if (!tmp || tmp > USHRT_MAX)
4285 return -E2BIG;
4286 p->nbufs = tmp;
4287 p->addr = READ_ONCE(sqe->addr);
4288 p->len = READ_ONCE(sqe->len);
4289
efe68c1c 4290 if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
ddf0322d
JA
4291 return -EFAULT;
4292
4293 p->bgid = READ_ONCE(sqe->buf_group);
4294 tmp = READ_ONCE(sqe->off);
4295 if (tmp > USHRT_MAX)
4296 return -E2BIG;
4297 p->bid = tmp;
4298 return 0;
4299}
4300
4301static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4302{
4303 struct io_buffer *buf;
4304 u64 addr = pbuf->addr;
4305 int i, bid = pbuf->bid;
4306
4307 for (i = 0; i < pbuf->nbufs; i++) {
4308 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
4309 if (!buf)
4310 break;
4311
4312 buf->addr = addr;
4313 buf->len = pbuf->len;
4314 buf->bid = bid;
4315 addr += pbuf->len;
4316 bid++;
4317 if (!*head) {
4318 INIT_LIST_HEAD(&buf->list);
4319 *head = buf;
4320 } else {
4321 list_add_tail(&buf->list, &(*head)->list);
4322 }
4323 }
4324
4325 return i ? i : -ENOMEM;
4326}
4327
889fca73 4328static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
ddf0322d
JA
4329{
4330 struct io_provide_buf *p = &req->pbuf;
4331 struct io_ring_ctx *ctx = req->ctx;
4332 struct io_buffer *head, *list;
4333 int ret = 0;
45d189c6 4334 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ddf0322d
JA
4335
4336 io_ring_submit_lock(ctx, !force_nonblock);
4337
4338 lockdep_assert_held(&ctx->uring_lock);
4339
4340 list = head = idr_find(&ctx->io_buffer_idr, p->bgid);
4341
4342 ret = io_add_buffers(p, &head);
4343 if (ret < 0)
4344 goto out;
4345
4346 if (!list) {
4347 ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
4348 GFP_KERNEL);
4349 if (ret < 0) {
067524e9 4350 __io_remove_buffers(ctx, head, p->bgid, -1U);
ddf0322d
JA
4351 goto out;
4352 }
4353 }
4354out:
ddf0322d
JA
4355 if (ret < 0)
4356 req_set_fail_links(req);
31bff9a5
PB
4357
4358 /* need to hold the lock to complete IOPOLL requests */
4359 if (ctx->flags & IORING_SETUP_IOPOLL) {
889fca73 4360 __io_req_complete(req, issue_flags, ret, 0);
31bff9a5
PB
4361 io_ring_submit_unlock(ctx, !force_nonblock);
4362 } else {
4363 io_ring_submit_unlock(ctx, !force_nonblock);
889fca73 4364 __io_req_complete(req, issue_flags, ret, 0);
31bff9a5 4365 }
ddf0322d 4366 return 0;
cebdb986
JA
4367}
4368
3e4827b0
JA
4369static int io_epoll_ctl_prep(struct io_kiocb *req,
4370 const struct io_uring_sqe *sqe)
4371{
4372#if defined(CONFIG_EPOLL)
4373 if (sqe->ioprio || sqe->buf_index)
4374 return -EINVAL;
6ca56f84 4375 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL)))
3232dd02 4376 return -EINVAL;
3e4827b0
JA
4377
4378 req->epoll.epfd = READ_ONCE(sqe->fd);
4379 req->epoll.op = READ_ONCE(sqe->len);
4380 req->epoll.fd = READ_ONCE(sqe->off);
4381
4382 if (ep_op_has_event(req->epoll.op)) {
4383 struct epoll_event __user *ev;
4384
4385 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4386 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4387 return -EFAULT;
4388 }
4389
4390 return 0;
4391#else
4392 return -EOPNOTSUPP;
4393#endif
4394}
4395
889fca73 4396static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
3e4827b0
JA
4397{
4398#if defined(CONFIG_EPOLL)
4399 struct io_epoll *ie = &req->epoll;
4400 int ret;
45d189c6 4401 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3e4827b0
JA
4402
4403 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4404 if (force_nonblock && ret == -EAGAIN)
4405 return -EAGAIN;
4406
4407 if (ret < 0)
4408 req_set_fail_links(req);
889fca73 4409 __io_req_complete(req, issue_flags, ret, 0);
3e4827b0
JA
4410 return 0;
4411#else
4412 return -EOPNOTSUPP;
4413#endif
4414}
4415
c1ca757b
JA
4416static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4417{
4418#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4419 if (sqe->ioprio || sqe->buf_index || sqe->off)
4420 return -EINVAL;
3232dd02
PB
4421 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4422 return -EINVAL;
c1ca757b
JA
4423
4424 req->madvise.addr = READ_ONCE(sqe->addr);
4425 req->madvise.len = READ_ONCE(sqe->len);
4426 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4427 return 0;
4428#else
4429 return -EOPNOTSUPP;
4430#endif
4431}
4432
45d189c6 4433static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
c1ca757b
JA
4434{
4435#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4436 struct io_madvise *ma = &req->madvise;
4437 int ret;
4438
45d189c6 4439 if (issue_flags & IO_URING_F_NONBLOCK)
c1ca757b
JA
4440 return -EAGAIN;
4441
0726b01e 4442 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
c1ca757b
JA
4443 if (ret < 0)
4444 req_set_fail_links(req);
e1e16097 4445 io_req_complete(req, ret);
c1ca757b
JA
4446 return 0;
4447#else
4448 return -EOPNOTSUPP;
4449#endif
4450}
4451
4840e418
JA
4452static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4453{
4454 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4455 return -EINVAL;
3232dd02
PB
4456 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4457 return -EINVAL;
4840e418
JA
4458
4459 req->fadvise.offset = READ_ONCE(sqe->off);
4460 req->fadvise.len = READ_ONCE(sqe->len);
4461 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4462 return 0;
4463}
4464
45d189c6 4465static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
4840e418
JA
4466{
4467 struct io_fadvise *fa = &req->fadvise;
4468 int ret;
4469
45d189c6 4470 if (issue_flags & IO_URING_F_NONBLOCK) {
3e69426d
JA
4471 switch (fa->advice) {
4472 case POSIX_FADV_NORMAL:
4473 case POSIX_FADV_RANDOM:
4474 case POSIX_FADV_SEQUENTIAL:
4475 break;
4476 default:
4477 return -EAGAIN;
4478 }
4479 }
4840e418
JA
4480
4481 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4482 if (ret < 0)
4483 req_set_fail_links(req);
e1e16097 4484 io_req_complete(req, ret);
4840e418
JA
4485 return 0;
4486}
4487
eddc7ef5
JA
4488static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4489{
6ca56f84 4490 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL)))
3232dd02 4491 return -EINVAL;
eddc7ef5
JA
4492 if (sqe->ioprio || sqe->buf_index)
4493 return -EINVAL;
9c280f90 4494 if (req->flags & REQ_F_FIXED_FILE)
cf3040ca 4495 return -EBADF;
eddc7ef5 4496
1d9e1288
BM
4497 req->statx.dfd = READ_ONCE(sqe->fd);
4498 req->statx.mask = READ_ONCE(sqe->len);
e62753e4 4499 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
1d9e1288
BM
4500 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4501 req->statx.flags = READ_ONCE(sqe->statx_flags);
eddc7ef5
JA
4502
4503 return 0;
4504}
4505
45d189c6 4506static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
eddc7ef5 4507{
1d9e1288 4508 struct io_statx *ctx = &req->statx;
eddc7ef5
JA
4509 int ret;
4510
45d189c6 4511 if (issue_flags & IO_URING_F_NONBLOCK) {
5b0bbee4
JA
4512 /* only need file table for an actual valid fd */
4513 if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD)
4514 req->flags |= REQ_F_NO_FILE_TABLE;
eddc7ef5 4515 return -EAGAIN;
5b0bbee4 4516 }
eddc7ef5 4517
e62753e4
BM
4518 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4519 ctx->buffer);
eddc7ef5 4520
eddc7ef5
JA
4521 if (ret < 0)
4522 req_set_fail_links(req);
e1e16097 4523 io_req_complete(req, ret);
eddc7ef5
JA
4524 return 0;
4525}
4526
b5dba59e
JA
4527static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4528{
14587a46 4529 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02 4530 return -EINVAL;
b5dba59e
JA
4531 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4532 sqe->rw_flags || sqe->buf_index)
4533 return -EINVAL;
9c280f90 4534 if (req->flags & REQ_F_FIXED_FILE)
cf3040ca 4535 return -EBADF;
b5dba59e
JA
4536
4537 req->close.fd = READ_ONCE(sqe->fd);
b5dba59e 4538 return 0;
b5dba59e
JA
4539}
4540
889fca73 4541static int io_close(struct io_kiocb *req, unsigned int issue_flags)
b5dba59e 4542{
9eac1904 4543 struct files_struct *files = current->files;
3af73b28 4544 struct io_close *close = &req->close;
9eac1904
JA
4545 struct fdtable *fdt;
4546 struct file *file;
b5dba59e
JA
4547 int ret;
4548
9eac1904
JA
4549 file = NULL;
4550 ret = -EBADF;
4551 spin_lock(&files->file_lock);
4552 fdt = files_fdtable(files);
4553 if (close->fd >= fdt->max_fds) {
4554 spin_unlock(&files->file_lock);
4555 goto err;
4556 }
4557 file = fdt->fd[close->fd];
4558 if (!file) {
4559 spin_unlock(&files->file_lock);
4560 goto err;
4561 }
4562
4563 if (file->f_op == &io_uring_fops) {
4564 spin_unlock(&files->file_lock);
4565 file = NULL;
4566 goto err;
3af73b28 4567 }
b5dba59e
JA
4568
4569 /* if the file has a flush method, be safe and punt to async */
45d189c6 4570 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
9eac1904 4571 spin_unlock(&files->file_lock);
0bf0eefd 4572 return -EAGAIN;
a2100672 4573 }
b5dba59e 4574
9eac1904
JA
4575 ret = __close_fd_get_file(close->fd, &file);
4576 spin_unlock(&files->file_lock);
4577 if (ret < 0) {
4578 if (ret == -ENOENT)
4579 ret = -EBADF;
4580 goto err;
4581 }
4582
3af73b28 4583 /* No ->flush() or already async, safely close from here */
9eac1904
JA
4584 ret = filp_close(file, current->files);
4585err:
3af73b28
PB
4586 if (ret < 0)
4587 req_set_fail_links(req);
9eac1904
JA
4588 if (file)
4589 fput(file);
889fca73 4590 __io_req_complete(req, issue_flags, ret, 0);
1a417f4e 4591 return 0;
b5dba59e
JA
4592}
4593
3529d8c2 4594static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5d17b4a4
JA
4595{
4596 struct io_ring_ctx *ctx = req->ctx;
5d17b4a4
JA
4597
4598 if (!req->file)
4599 return -EBADF;
5d17b4a4
JA
4600
4601 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4602 return -EINVAL;
4603 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4604 return -EINVAL;
4605
8ed8d3c3
JA
4606 req->sync.off = READ_ONCE(sqe->off);
4607 req->sync.len = READ_ONCE(sqe->len);
4608 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
8ed8d3c3
JA
4609 return 0;
4610}
4611
45d189c6 4612static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3 4613{
8ed8d3c3
JA
4614 int ret;
4615
ac45abc0 4616 /* sync_file_range always requires a blocking context */
45d189c6 4617 if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0
PB
4618 return -EAGAIN;
4619
9adbd45d 4620 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
8ed8d3c3
JA
4621 req->sync.flags);
4622 if (ret < 0)
4623 req_set_fail_links(req);
e1e16097 4624 io_req_complete(req, ret);
5d17b4a4
JA
4625 return 0;
4626}
4627
469956e8 4628#if defined(CONFIG_NET)
02d27d89
PB
4629static int io_setup_async_msg(struct io_kiocb *req,
4630 struct io_async_msghdr *kmsg)
4631{
e8c2bc1f
JA
4632 struct io_async_msghdr *async_msg = req->async_data;
4633
4634 if (async_msg)
02d27d89 4635 return -EAGAIN;
e8c2bc1f 4636 if (io_alloc_async_data(req)) {
257e84a5 4637 kfree(kmsg->free_iov);
02d27d89
PB
4638 return -ENOMEM;
4639 }
e8c2bc1f 4640 async_msg = req->async_data;
02d27d89 4641 req->flags |= REQ_F_NEED_CLEANUP;
e8c2bc1f 4642 memcpy(async_msg, kmsg, sizeof(*kmsg));
2a780802 4643 async_msg->msg.msg_name = &async_msg->addr;
257e84a5
PB
4644 /* if were using fast_iov, set it to the new one */
4645 if (!async_msg->free_iov)
4646 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4647
02d27d89
PB
4648 return -EAGAIN;
4649}
4650
2ae523ed
PB
4651static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4652 struct io_async_msghdr *iomsg)
4653{
2ae523ed 4654 iomsg->msg.msg_name = &iomsg->addr;
257e84a5 4655 iomsg->free_iov = iomsg->fast_iov;
2ae523ed 4656 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
257e84a5 4657 req->sr_msg.msg_flags, &iomsg->free_iov);
2ae523ed
PB
4658}
4659
3529d8c2 4660static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
03b1230c 4661{
e8c2bc1f 4662 struct io_async_msghdr *async_msg = req->async_data;
e47293fd 4663 struct io_sr_msg *sr = &req->sr_msg;
99bc4c38 4664 int ret;
03b1230c 4665
d2b6f48b
PB
4666 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4667 return -EINVAL;
4668
e47293fd 4669 sr->msg_flags = READ_ONCE(sqe->msg_flags);
270a5940 4670 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
fddaface 4671 sr->len = READ_ONCE(sqe->len);
3529d8c2 4672
d8768362
JA
4673#ifdef CONFIG_COMPAT
4674 if (req->ctx->compat)
4675 sr->msg_flags |= MSG_CMSG_COMPAT;
4676#endif
4677
e8c2bc1f 4678 if (!async_msg || !io_op_defs[req->opcode].needs_async_data)
3529d8c2 4679 return 0;
e8c2bc1f 4680 ret = io_sendmsg_copy_hdr(req, async_msg);
99bc4c38
PB
4681 if (!ret)
4682 req->flags |= REQ_F_NEED_CLEANUP;
4683 return ret;
03b1230c
JA
4684}
4685
889fca73 4686static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28f 4687{
6b754c8b 4688 struct io_async_msghdr iomsg, *kmsg;
0fa03c62 4689 struct socket *sock;
7a7cacba 4690 unsigned flags;
0fa03c62
JA
4691 int ret;
4692
dba4a925 4693 sock = sock_from_file(req->file);
7a7cacba 4694 if (unlikely(!sock))
dba4a925 4695 return -ENOTSOCK;
3529d8c2 4696
257e84a5
PB
4697 kmsg = req->async_data;
4698 if (!kmsg) {
7a7cacba
PB
4699 ret = io_sendmsg_copy_hdr(req, &iomsg);
4700 if (ret)
4701 return ret;
4702 kmsg = &iomsg;
0fa03c62 4703 }
0fa03c62 4704
7a7cacba
PB
4705 flags = req->sr_msg.msg_flags;
4706 if (flags & MSG_DONTWAIT)
4707 req->flags |= REQ_F_NOWAIT;
45d189c6 4708 else if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba 4709 flags |= MSG_DONTWAIT;
e47293fd 4710
7a7cacba 4711 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
45d189c6 4712 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba
PB
4713 return io_setup_async_msg(req, kmsg);
4714 if (ret == -ERESTARTSYS)
4715 ret = -EINTR;
0fa03c62 4716
257e84a5
PB
4717 /* fast path, check for non-NULL to avoid function call */
4718 if (kmsg->free_iov)
4719 kfree(kmsg->free_iov);
99bc4c38 4720 req->flags &= ~REQ_F_NEED_CLEANUP;
4e88d6e7
JA
4721 if (ret < 0)
4722 req_set_fail_links(req);
889fca73 4723 __io_req_complete(req, issue_flags, ret, 0);
5d17b4a4 4724 return 0;
03b1230c 4725}
aa1fa28f 4726
889fca73 4727static int io_send(struct io_kiocb *req, unsigned int issue_flags)
fddaface 4728{
7a7cacba
PB
4729 struct io_sr_msg *sr = &req->sr_msg;
4730 struct msghdr msg;
4731 struct iovec iov;
fddaface 4732 struct socket *sock;
7a7cacba 4733 unsigned flags;
fddaface
JA
4734 int ret;
4735
dba4a925 4736 sock = sock_from_file(req->file);
7a7cacba 4737 if (unlikely(!sock))
dba4a925 4738 return -ENOTSOCK;
fddaface 4739
7a7cacba
PB
4740 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4741 if (unlikely(ret))
14db8411 4742 return ret;
fddaface 4743
7a7cacba
PB
4744 msg.msg_name = NULL;
4745 msg.msg_control = NULL;
4746 msg.msg_controllen = 0;
4747 msg.msg_namelen = 0;
fddaface 4748
7a7cacba
PB
4749 flags = req->sr_msg.msg_flags;
4750 if (flags & MSG_DONTWAIT)
4751 req->flags |= REQ_F_NOWAIT;
45d189c6 4752 else if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba 4753 flags |= MSG_DONTWAIT;
fddaface 4754
7a7cacba
PB
4755 msg.msg_flags = flags;
4756 ret = sock_sendmsg(sock, &msg);
45d189c6 4757 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba
PB
4758 return -EAGAIN;
4759 if (ret == -ERESTARTSYS)
4760 ret = -EINTR;
fddaface 4761
fddaface
JA
4762 if (ret < 0)
4763 req_set_fail_links(req);
889fca73 4764 __io_req_complete(req, issue_flags, ret, 0);
fddaface 4765 return 0;
fddaface
JA
4766}
4767
1400e697
PB
4768static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4769 struct io_async_msghdr *iomsg)
52de1fe1
JA
4770{
4771 struct io_sr_msg *sr = &req->sr_msg;
4772 struct iovec __user *uiov;
4773 size_t iov_len;
4774 int ret;
4775
1400e697
PB
4776 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4777 &iomsg->uaddr, &uiov, &iov_len);
52de1fe1
JA
4778 if (ret)
4779 return ret;
4780
4781 if (req->flags & REQ_F_BUFFER_SELECT) {
4782 if (iov_len > 1)
4783 return -EINVAL;
5476dfed 4784 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
52de1fe1 4785 return -EFAULT;
5476dfed 4786 sr->len = iomsg->fast_iov[0].iov_len;
257e84a5 4787 iomsg->free_iov = NULL;
52de1fe1 4788 } else {
257e84a5 4789 iomsg->free_iov = iomsg->fast_iov;
89cd35c5 4790 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
257e84a5 4791 &iomsg->free_iov, &iomsg->msg.msg_iter,
89cd35c5 4792 false);
52de1fe1
JA
4793 if (ret > 0)
4794 ret = 0;
4795 }
4796
4797 return ret;
4798}
4799
4800#ifdef CONFIG_COMPAT
4801static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
1400e697 4802 struct io_async_msghdr *iomsg)
52de1fe1
JA
4803{
4804 struct compat_msghdr __user *msg_compat;
4805 struct io_sr_msg *sr = &req->sr_msg;
4806 struct compat_iovec __user *uiov;
4807 compat_uptr_t ptr;
4808 compat_size_t len;
4809 int ret;
4810
270a5940 4811 msg_compat = (struct compat_msghdr __user *) sr->umsg;
1400e697 4812 ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
52de1fe1
JA
4813 &ptr, &len);
4814 if (ret)
4815 return ret;
4816
4817 uiov = compat_ptr(ptr);
4818 if (req->flags & REQ_F_BUFFER_SELECT) {
4819 compat_ssize_t clen;
4820
4821 if (len > 1)
4822 return -EINVAL;
4823 if (!access_ok(uiov, sizeof(*uiov)))
4824 return -EFAULT;
4825 if (__get_user(clen, &uiov->iov_len))
4826 return -EFAULT;
4827 if (clen < 0)
4828 return -EINVAL;
2d280bc8 4829 sr->len = clen;
257e84a5 4830 iomsg->free_iov = NULL;
52de1fe1 4831 } else {
257e84a5 4832 iomsg->free_iov = iomsg->fast_iov;
89cd35c5 4833 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
257e84a5 4834 UIO_FASTIOV, &iomsg->free_iov,
89cd35c5 4835 &iomsg->msg.msg_iter, true);
52de1fe1
JA
4836 if (ret < 0)
4837 return ret;
4838 }
4839
4840 return 0;
4841}
4842#endif
4843
1400e697
PB
4844static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4845 struct io_async_msghdr *iomsg)
52de1fe1 4846{
1400e697 4847 iomsg->msg.msg_name = &iomsg->addr;
52de1fe1
JA
4848
4849#ifdef CONFIG_COMPAT
4850 if (req->ctx->compat)
1400e697 4851 return __io_compat_recvmsg_copy_hdr(req, iomsg);
fddaface 4852#endif
52de1fe1 4853
1400e697 4854 return __io_recvmsg_copy_hdr(req, iomsg);
52de1fe1
JA
4855}
4856
bcda7baa 4857static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
7fbb1b54 4858 bool needs_lock)
bcda7baa
JA
4859{
4860 struct io_sr_msg *sr = &req->sr_msg;
4861 struct io_buffer *kbuf;
4862
bcda7baa
JA
4863 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4864 if (IS_ERR(kbuf))
4865 return kbuf;
4866
4867 sr->kbuf = kbuf;
4868 req->flags |= REQ_F_BUFFER_SELECTED;
bcda7baa 4869 return kbuf;
fddaface
JA
4870}
4871
7fbb1b54
PB
4872static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4873{
4874 return io_put_kbuf(req, req->sr_msg.kbuf);
4875}
4876
3529d8c2
JA
4877static int io_recvmsg_prep(struct io_kiocb *req,
4878 const struct io_uring_sqe *sqe)
aa1fa28f 4879{
e8c2bc1f 4880 struct io_async_msghdr *async_msg = req->async_data;
e47293fd 4881 struct io_sr_msg *sr = &req->sr_msg;
99bc4c38 4882 int ret;
3529d8c2 4883
d2b6f48b
PB
4884 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4885 return -EINVAL;
4886
3529d8c2 4887 sr->msg_flags = READ_ONCE(sqe->msg_flags);
270a5940 4888 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
0b7b21e4 4889 sr->len = READ_ONCE(sqe->len);
bcda7baa 4890 sr->bgid = READ_ONCE(sqe->buf_group);
06b76d44 4891
d8768362
JA
4892#ifdef CONFIG_COMPAT
4893 if (req->ctx->compat)
4894 sr->msg_flags |= MSG_CMSG_COMPAT;
4895#endif
4896
e8c2bc1f 4897 if (!async_msg || !io_op_defs[req->opcode].needs_async_data)
5f798bea 4898 return 0;
e8c2bc1f 4899 ret = io_recvmsg_copy_hdr(req, async_msg);
99bc4c38
PB
4900 if (!ret)
4901 req->flags |= REQ_F_NEED_CLEANUP;
4902 return ret;
aa1fa28f
JA
4903}
4904
889fca73 4905static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28f 4906{
6b754c8b 4907 struct io_async_msghdr iomsg, *kmsg;
03b1230c 4908 struct socket *sock;
7fbb1b54 4909 struct io_buffer *kbuf;
7a7cacba 4910 unsigned flags;
52de1fe1 4911 int ret, cflags = 0;
45d189c6 4912 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
03b1230c 4913
dba4a925 4914 sock = sock_from_file(req->file);
7a7cacba 4915 if (unlikely(!sock))
dba4a925 4916 return -ENOTSOCK;
3529d8c2 4917
257e84a5
PB
4918 kmsg = req->async_data;
4919 if (!kmsg) {
7a7cacba
PB
4920 ret = io_recvmsg_copy_hdr(req, &iomsg);
4921 if (ret)
681fda8d 4922 return ret;
7a7cacba
PB
4923 kmsg = &iomsg;
4924 }
03b1230c 4925
bc02ef33 4926 if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b54 4927 kbuf = io_recv_buffer_select(req, !force_nonblock);
bc02ef33 4928 if (IS_ERR(kbuf))
52de1fe1 4929 return PTR_ERR(kbuf);
7a7cacba 4930 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
5476dfed
PB
4931 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4932 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
7a7cacba
PB
4933 1, req->sr_msg.len);
4934 }
52de1fe1 4935
7a7cacba
PB
4936 flags = req->sr_msg.msg_flags;
4937 if (flags & MSG_DONTWAIT)
4938 req->flags |= REQ_F_NOWAIT;
4939 else if (force_nonblock)
4940 flags |= MSG_DONTWAIT;
e47293fd 4941
7a7cacba
PB
4942 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4943 kmsg->uaddr, flags);
0e1b6fe3
PB
4944 if (force_nonblock && ret == -EAGAIN)
4945 return io_setup_async_msg(req, kmsg);
7a7cacba
PB
4946 if (ret == -ERESTARTSYS)
4947 ret = -EINTR;
03b1230c 4948
7fbb1b54
PB
4949 if (req->flags & REQ_F_BUFFER_SELECTED)
4950 cflags = io_put_recv_kbuf(req);
257e84a5
PB
4951 /* fast path, check for non-NULL to avoid function call */
4952 if (kmsg->free_iov)
4953 kfree(kmsg->free_iov);
99bc4c38 4954 req->flags &= ~REQ_F_NEED_CLEANUP;
4e88d6e7
JA
4955 if (ret < 0)
4956 req_set_fail_links(req);
889fca73 4957 __io_req_complete(req, issue_flags, ret, cflags);
03b1230c 4958 return 0;
0fa03c62 4959}
5d17b4a4 4960
889fca73 4961static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
fddaface 4962{
6b754c8b 4963 struct io_buffer *kbuf;
7a7cacba
PB
4964 struct io_sr_msg *sr = &req->sr_msg;
4965 struct msghdr msg;
4966 void __user *buf = sr->buf;
fddaface 4967 struct socket *sock;
7a7cacba
PB
4968 struct iovec iov;
4969 unsigned flags;
bcda7baa 4970 int ret, cflags = 0;
45d189c6 4971 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
fddaface 4972
dba4a925 4973 sock = sock_from_file(req->file);
7a7cacba 4974 if (unlikely(!sock))
dba4a925 4975 return -ENOTSOCK;
fddaface 4976
bc02ef33 4977 if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b54 4978 kbuf = io_recv_buffer_select(req, !force_nonblock);
bcda7baa
JA
4979 if (IS_ERR(kbuf))
4980 return PTR_ERR(kbuf);
7a7cacba 4981 buf = u64_to_user_ptr(kbuf->addr);
bc02ef33 4982 }
bcda7baa 4983
7a7cacba 4984 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
14c32eee
PB
4985 if (unlikely(ret))
4986 goto out_free;
fddaface 4987
7a7cacba
PB
4988 msg.msg_name = NULL;
4989 msg.msg_control = NULL;
4990 msg.msg_controllen = 0;
4991 msg.msg_namelen = 0;
4992 msg.msg_iocb = NULL;
4993 msg.msg_flags = 0;
fddaface 4994
7a7cacba
PB
4995 flags = req->sr_msg.msg_flags;
4996 if (flags & MSG_DONTWAIT)
4997 req->flags |= REQ_F_NOWAIT;
4998 else if (force_nonblock)
4999 flags |= MSG_DONTWAIT;
5000
5001 ret = sock_recvmsg(sock, &msg, flags);
5002 if (force_nonblock && ret == -EAGAIN)
5003 return -EAGAIN;
5004 if (ret == -ERESTARTSYS)
5005 ret = -EINTR;
14c32eee 5006out_free:
7fbb1b54
PB
5007 if (req->flags & REQ_F_BUFFER_SELECTED)
5008 cflags = io_put_recv_kbuf(req);
fddaface
JA
5009 if (ret < 0)
5010 req_set_fail_links(req);
889fca73 5011 __io_req_complete(req, issue_flags, ret, cflags);
fddaface 5012 return 0;
fddaface
JA
5013}
5014
3529d8c2 5015static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
17f2fe35 5016{
8ed8d3c3
JA
5017 struct io_accept *accept = &req->accept;
5018
14587a46 5019 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
17f2fe35 5020 return -EINVAL;
8042d6ce 5021 if (sqe->ioprio || sqe->len || sqe->buf_index)
17f2fe35
JA
5022 return -EINVAL;
5023
d55e5f5b
JA
5024 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5025 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
8ed8d3c3 5026 accept->flags = READ_ONCE(sqe->accept_flags);
09952e3e 5027 accept->nofile = rlimit(RLIMIT_NOFILE);
8ed8d3c3 5028 return 0;
8ed8d3c3 5029}
17f2fe35 5030
889fca73 5031static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3
JA
5032{
5033 struct io_accept *accept = &req->accept;
45d189c6 5034 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ac45abc0 5035 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
8ed8d3c3
JA
5036 int ret;
5037
e697deed
JX
5038 if (req->file->f_flags & O_NONBLOCK)
5039 req->flags |= REQ_F_NOWAIT;
5040
8ed8d3c3 5041 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
09952e3e
JA
5042 accept->addr_len, accept->flags,
5043 accept->nofile);
8ed8d3c3 5044 if (ret == -EAGAIN && force_nonblock)
17f2fe35 5045 return -EAGAIN;
ac45abc0
PB
5046 if (ret < 0) {
5047 if (ret == -ERESTARTSYS)
5048 ret = -EINTR;
4e88d6e7 5049 req_set_fail_links(req);
ac45abc0 5050 }
889fca73 5051 __io_req_complete(req, issue_flags, ret, 0);
17f2fe35 5052 return 0;
8ed8d3c3
JA
5053}
5054
3529d8c2 5055static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f499a021 5056{
3529d8c2 5057 struct io_connect *conn = &req->connect;
e8c2bc1f 5058 struct io_async_connect *io = req->async_data;
f499a021 5059
14587a46 5060 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3fbb51c1
JA
5061 return -EINVAL;
5062 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
5063 return -EINVAL;
5064
3529d8c2
JA
5065 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5066 conn->addr_len = READ_ONCE(sqe->addr2);
5067
5068 if (!io)
5069 return 0;
5070
5071 return move_addr_to_kernel(conn->addr, conn->addr_len,
e8c2bc1f 5072 &io->address);
f499a021
JA
5073}
5074
889fca73 5075static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
f8e85cf2 5076{
e8c2bc1f 5077 struct io_async_connect __io, *io;
f8e85cf2 5078 unsigned file_flags;
3fbb51c1 5079 int ret;
45d189c6 5080 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
f8e85cf2 5081
e8c2bc1f
JA
5082 if (req->async_data) {
5083 io = req->async_data;
f499a021 5084 } else {
3529d8c2
JA
5085 ret = move_addr_to_kernel(req->connect.addr,
5086 req->connect.addr_len,
e8c2bc1f 5087 &__io.address);
f499a021
JA
5088 if (ret)
5089 goto out;
5090 io = &__io;
5091 }
5092
3fbb51c1
JA
5093 file_flags = force_nonblock ? O_NONBLOCK : 0;
5094
e8c2bc1f 5095 ret = __sys_connect_file(req->file, &io->address,
3fbb51c1 5096 req->connect.addr_len, file_flags);
87f80d62 5097 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
e8c2bc1f 5098 if (req->async_data)
b7bb4f7d 5099 return -EAGAIN;
e8c2bc1f 5100 if (io_alloc_async_data(req)) {
f499a021
JA
5101 ret = -ENOMEM;
5102 goto out;
5103 }
e8c2bc1f
JA
5104 io = req->async_data;
5105 memcpy(req->async_data, &__io, sizeof(__io));
f8e85cf2 5106 return -EAGAIN;
f499a021 5107 }
f8e85cf2
JA
5108 if (ret == -ERESTARTSYS)
5109 ret = -EINTR;
f499a021 5110out:
4e88d6e7
JA
5111 if (ret < 0)
5112 req_set_fail_links(req);
889fca73 5113 __io_req_complete(req, issue_flags, ret, 0);
f8e85cf2 5114 return 0;
469956e8
Y
5115}
5116#else /* !CONFIG_NET */
5117static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5118{
f8e85cf2 5119 return -EOPNOTSUPP;
f8e85cf2
JA
5120}
5121
889fca73 5122static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
469956e8
Y
5123{
5124 return -EOPNOTSUPP;
5125}
5126
889fca73 5127static int io_send(struct io_kiocb *req, unsigned int issue_flags)
469956e8
Y
5128{
5129 return -EOPNOTSUPP;
5130}
5131
5132static int io_recvmsg_prep(struct io_kiocb *req,
5133 const struct io_uring_sqe *sqe)
5134{
5135 return -EOPNOTSUPP;
5136}
5137
889fca73 5138static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
469956e8
Y
5139{
5140 return -EOPNOTSUPP;
5141}
5142
889fca73 5143static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
469956e8
Y
5144{
5145 return -EOPNOTSUPP;
5146}
5147
5148static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5149{
5150 return -EOPNOTSUPP;
5151}
5152
889fca73 5153static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
469956e8
Y
5154{
5155 return -EOPNOTSUPP;
5156}
ce593a6c 5157
469956e8
Y
5158static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5159{
5160 return -EOPNOTSUPP;
5161}
5162
889fca73 5163static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
469956e8 5164{
f8e85cf2 5165 return -EOPNOTSUPP;
ce593a6c 5166}
469956e8 5167#endif /* CONFIG_NET */
f8e85cf2 5168
d7718a9d
JA
5169struct io_poll_table {
5170 struct poll_table_struct pt;
5171 struct io_kiocb *req;
5172 int error;
5173};
ce593a6c 5174
d7718a9d
JA
5175static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
5176 __poll_t mask, task_work_func_t func)
5177{
aa96bf8a 5178 int ret;
d7718a9d
JA
5179
5180 /* for instances that support it check for an event match first: */
5181 if (mask && !(mask & poll->events))
5182 return 0;
5183
5184 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
5185
5186 list_del_init(&poll->wait.entry);
5187
d7718a9d 5188 req->result = mask;
7cbf1722 5189 req->task_work.func = func;
6d816e08
JA
5190 percpu_ref_get(&req->ctx->refs);
5191
d7718a9d 5192 /*
e3aabf95
JA
5193 * If this fails, then the task is exiting. When a task exits, the
5194 * work gets canceled, so just cancel this request as well instead
5195 * of executing it. We can't safely execute it anyway, as we may not
5196 * have the needed state needed for it anyway.
d7718a9d 5197 */
355fb9e2 5198 ret = io_req_task_work_add(req);
aa96bf8a 5199 if (unlikely(ret)) {
e3aabf95 5200 WRITE_ONCE(poll->canceled, true);
eab30c4d 5201 io_req_task_work_add_fallback(req, func);
aa96bf8a 5202 }
d7718a9d
JA
5203 return 1;
5204}
5205
74ce6ce4
JA
5206static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
5207 __acquires(&req->ctx->completion_lock)
5208{
5209 struct io_ring_ctx *ctx = req->ctx;
5210
5211 if (!req->result && !READ_ONCE(poll->canceled)) {
5212 struct poll_table_struct pt = { ._key = poll->events };
5213
5214 req->result = vfs_poll(req->file, &pt) & poll->events;
5215 }
5216
5217 spin_lock_irq(&ctx->completion_lock);
5218 if (!req->result && !READ_ONCE(poll->canceled)) {
5219 add_wait_queue(poll->head, &poll->wait);
5220 return true;
5221 }
5222
5223 return false;
5224}
5225
d4e7cd36 5226static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
18bceab1 5227{
e8c2bc1f 5228 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
d4e7cd36 5229 if (req->opcode == IORING_OP_POLL_ADD)
e8c2bc1f 5230 return req->async_data;
d4e7cd36
JA
5231 return req->apoll->double_poll;
5232}
5233
5234static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
5235{
5236 if (req->opcode == IORING_OP_POLL_ADD)
5237 return &req->poll;
5238 return &req->apoll->poll;
5239}
5240
5241static void io_poll_remove_double(struct io_kiocb *req)
5242{
5243 struct io_poll_iocb *poll = io_poll_get_double(req);
18bceab1
JA
5244
5245 lockdep_assert_held(&req->ctx->completion_lock);
5246
5247 if (poll && poll->head) {
5248 struct wait_queue_head *head = poll->head;
5249
5250 spin_lock(&head->lock);
5251 list_del_init(&poll->wait.entry);
5252 if (poll->wait.private)
5253 refcount_dec(&req->refs);
5254 poll->head = NULL;
5255 spin_unlock(&head->lock);
5256 }
5257}
5258
5259static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
5260{
5261 struct io_ring_ctx *ctx = req->ctx;
5262
d4e7cd36 5263 io_poll_remove_double(req);
18bceab1
JA
5264 req->poll.done = true;
5265 io_cqring_fill_event(req, error ? error : mangle_poll(mask));
5266 io_commit_cqring(ctx);
5267}
5268
dd221f46 5269static void io_poll_task_func(struct callback_head *cb)
18bceab1 5270{
dd221f46 5271 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
18bceab1 5272 struct io_ring_ctx *ctx = req->ctx;
dd221f46 5273 struct io_kiocb *nxt;
18bceab1
JA
5274
5275 if (io_poll_rewait(req, &req->poll)) {
5276 spin_unlock_irq(&ctx->completion_lock);
dd221f46
PB
5277 } else {
5278 hash_del(&req->hash_node);
5279 io_poll_complete(req, req->result, 0);
5280 spin_unlock_irq(&ctx->completion_lock);
18bceab1 5281
dd221f46
PB
5282 nxt = io_put_req_find_next(req);
5283 io_cqring_ev_posted(ctx);
5284 if (nxt)
5285 __io_req_task_submit(nxt);
5286 }
18bceab1 5287
6d816e08 5288 percpu_ref_put(&ctx->refs);
18bceab1
JA
5289}
5290
5291static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
5292 int sync, void *key)
5293{
5294 struct io_kiocb *req = wait->private;
d4e7cd36 5295 struct io_poll_iocb *poll = io_poll_get_single(req);
18bceab1
JA
5296 __poll_t mask = key_to_poll(key);
5297
5298 /* for instances that support it check for an event match first: */
5299 if (mask && !(mask & poll->events))
5300 return 0;
5301
8706e04e
JA
5302 list_del_init(&wait->entry);
5303
807abcb0 5304 if (poll && poll->head) {
18bceab1
JA
5305 bool done;
5306
807abcb0
JA
5307 spin_lock(&poll->head->lock);
5308 done = list_empty(&poll->wait.entry);
18bceab1 5309 if (!done)
807abcb0 5310 list_del_init(&poll->wait.entry);
d4e7cd36
JA
5311 /* make sure double remove sees this as being gone */
5312 wait->private = NULL;
807abcb0 5313 spin_unlock(&poll->head->lock);
c8b5e260
JA
5314 if (!done) {
5315 /* use wait func handler, so it matches the rq type */
5316 poll->wait.func(&poll->wait, mode, sync, key);
5317 }
18bceab1
JA
5318 }
5319 refcount_dec(&req->refs);
5320 return 1;
5321}
5322
5323static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
5324 wait_queue_func_t wake_func)
5325{
5326 poll->head = NULL;
5327 poll->done = false;
5328 poll->canceled = false;
5329 poll->events = events;
5330 INIT_LIST_HEAD(&poll->wait.entry);
5331 init_waitqueue_func_entry(&poll->wait, wake_func);
5332}
5333
5334static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
807abcb0
JA
5335 struct wait_queue_head *head,
5336 struct io_poll_iocb **poll_ptr)
18bceab1
JA
5337{
5338 struct io_kiocb *req = pt->req;
5339
5340 /*
5341 * If poll->head is already set, it's because the file being polled
5342 * uses multiple waitqueues for poll handling (eg one for read, one
5343 * for write). Setup a separate io_poll_iocb if this happens.
5344 */
5345 if (unlikely(poll->head)) {
58852d4d
PB
5346 struct io_poll_iocb *poll_one = poll;
5347
18bceab1 5348 /* already have a 2nd entry, fail a third attempt */
807abcb0 5349 if (*poll_ptr) {
18bceab1
JA
5350 pt->error = -EINVAL;
5351 return;
5352 }
5353 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5354 if (!poll) {
5355 pt->error = -ENOMEM;
5356 return;
5357 }
58852d4d 5358 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
18bceab1
JA
5359 refcount_inc(&req->refs);
5360 poll->wait.private = req;
807abcb0 5361 *poll_ptr = poll;
18bceab1
JA
5362 }
5363
5364 pt->error = 0;
5365 poll->head = head;
a31eb4a2
JX
5366
5367 if (poll->events & EPOLLEXCLUSIVE)
5368 add_wait_queue_exclusive(head, &poll->wait);
5369 else
5370 add_wait_queue(head, &poll->wait);
18bceab1
JA
5371}
5372
5373static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5374 struct poll_table_struct *p)
5375{
5376 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
807abcb0 5377 struct async_poll *apoll = pt->req->apoll;
18bceab1 5378
807abcb0 5379 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
18bceab1
JA
5380}
5381
d7718a9d
JA
5382static void io_async_task_func(struct callback_head *cb)
5383{
5384 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
5385 struct async_poll *apoll = req->apoll;
5386 struct io_ring_ctx *ctx = req->ctx;
5387
5388 trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
5389
74ce6ce4 5390 if (io_poll_rewait(req, &apoll->poll)) {
d7718a9d 5391 spin_unlock_irq(&ctx->completion_lock);
6d816e08 5392 percpu_ref_put(&ctx->refs);
74ce6ce4 5393 return;
d7718a9d
JA
5394 }
5395
31067255 5396 /* If req is still hashed, it cannot have been canceled. Don't check. */
0be0b0e3 5397 if (hash_hashed(&req->hash_node))
74ce6ce4 5398 hash_del(&req->hash_node);
2bae047e 5399
d4e7cd36 5400 io_poll_remove_double(req);
74ce6ce4
JA
5401 spin_unlock_irq(&ctx->completion_lock);
5402
0be0b0e3
PB
5403 if (!READ_ONCE(apoll->poll.canceled))
5404 __io_req_task_submit(req);
5405 else
5406 __io_req_task_cancel(req, -ECANCELED);
aa340845 5407
6d816e08 5408 percpu_ref_put(&ctx->refs);
807abcb0 5409 kfree(apoll->double_poll);
31067255 5410 kfree(apoll);
d7718a9d
JA
5411}
5412
5413static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5414 void *key)
5415{
5416 struct io_kiocb *req = wait->private;
5417 struct io_poll_iocb *poll = &req->apoll->poll;
5418
5419 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5420 key_to_poll(key));
5421
5422 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5423}
5424
5425static void io_poll_req_insert(struct io_kiocb *req)
5426{
5427 struct io_ring_ctx *ctx = req->ctx;
5428 struct hlist_head *list;
5429
5430 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5431 hlist_add_head(&req->hash_node, list);
5432}
5433
5434static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5435 struct io_poll_iocb *poll,
5436 struct io_poll_table *ipt, __poll_t mask,
5437 wait_queue_func_t wake_func)
5438 __acquires(&ctx->completion_lock)
5439{
5440 struct io_ring_ctx *ctx = req->ctx;
5441 bool cancel = false;
5442
4d52f338 5443 INIT_HLIST_NODE(&req->hash_node);
18bceab1 5444 io_init_poll_iocb(poll, mask, wake_func);
b90cd197 5445 poll->file = req->file;
18bceab1 5446 poll->wait.private = req;
d7718a9d
JA
5447
5448 ipt->pt._key = mask;
5449 ipt->req = req;
5450 ipt->error = -EINVAL;
5451
d7718a9d
JA
5452 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5453
5454 spin_lock_irq(&ctx->completion_lock);
5455 if (likely(poll->head)) {
5456 spin_lock(&poll->head->lock);
5457 if (unlikely(list_empty(&poll->wait.entry))) {
5458 if (ipt->error)
5459 cancel = true;
5460 ipt->error = 0;
5461 mask = 0;
5462 }
5463 if (mask || ipt->error)
5464 list_del_init(&poll->wait.entry);
5465 else if (cancel)
5466 WRITE_ONCE(poll->canceled, true);
5467 else if (!poll->done) /* actually waiting for an event */
5468 io_poll_req_insert(req);
5469 spin_unlock(&poll->head->lock);
5470 }
5471
5472 return mask;
5473}
5474
5475static bool io_arm_poll_handler(struct io_kiocb *req)
5476{
5477 const struct io_op_def *def = &io_op_defs[req->opcode];
5478 struct io_ring_ctx *ctx = req->ctx;
5479 struct async_poll *apoll;
5480 struct io_poll_table ipt;
5481 __poll_t mask, ret;
9dab14b8 5482 int rw;
d7718a9d
JA
5483
5484 if (!req->file || !file_can_poll(req->file))
5485 return false;
24c74678 5486 if (req->flags & REQ_F_POLLED)
d7718a9d 5487 return false;
9dab14b8
JA
5488 if (def->pollin)
5489 rw = READ;
5490 else if (def->pollout)
5491 rw = WRITE;
5492 else
5493 return false;
5494 /* if we can't nonblock try, then no point in arming a poll handler */
5495 if (!io_file_supports_async(req->file, rw))
d7718a9d
JA
5496 return false;
5497
5498 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5499 if (unlikely(!apoll))
5500 return false;
807abcb0 5501 apoll->double_poll = NULL;
d7718a9d
JA
5502
5503 req->flags |= REQ_F_POLLED;
d7718a9d 5504 req->apoll = apoll;
d7718a9d 5505
8755d97a 5506 mask = 0;
d7718a9d 5507 if (def->pollin)
8755d97a 5508 mask |= POLLIN | POLLRDNORM;
d7718a9d
JA
5509 if (def->pollout)
5510 mask |= POLLOUT | POLLWRNORM;
901341bb
LH
5511
5512 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5513 if ((req->opcode == IORING_OP_RECVMSG) &&
5514 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5515 mask &= ~POLLIN;
5516
d7718a9d
JA
5517 mask |= POLLERR | POLLPRI;
5518
5519 ipt.pt._qproc = io_async_queue_proc;
5520
5521 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5522 io_async_wake);
a36da65c 5523 if (ret || ipt.error) {
d4e7cd36 5524 io_poll_remove_double(req);
d7718a9d 5525 spin_unlock_irq(&ctx->completion_lock);
807abcb0 5526 kfree(apoll->double_poll);
d7718a9d
JA
5527 kfree(apoll);
5528 return false;
5529 }
5530 spin_unlock_irq(&ctx->completion_lock);
5531 trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
5532 apoll->poll.events);
5533 return true;
5534}
5535
5536static bool __io_poll_remove_one(struct io_kiocb *req,
5537 struct io_poll_iocb *poll)
221c5eb2 5538{
b41e9852 5539 bool do_complete = false;
221c5eb2
JA
5540
5541 spin_lock(&poll->head->lock);
5542 WRITE_ONCE(poll->canceled, true);
392edb45
JA
5543 if (!list_empty(&poll->wait.entry)) {
5544 list_del_init(&poll->wait.entry);
b41e9852 5545 do_complete = true;
221c5eb2
JA
5546 }
5547 spin_unlock(&poll->head->lock);
3bfa5bcb 5548 hash_del(&req->hash_node);
d7718a9d
JA
5549 return do_complete;
5550}
5551
5552static bool io_poll_remove_one(struct io_kiocb *req)
5553{
5554 bool do_complete;
5555
d4e7cd36
JA
5556 io_poll_remove_double(req);
5557
d7718a9d
JA
5558 if (req->opcode == IORING_OP_POLL_ADD) {
5559 do_complete = __io_poll_remove_one(req, &req->poll);
5560 } else {
3bfa5bcb
JA
5561 struct async_poll *apoll = req->apoll;
5562
d7718a9d 5563 /* non-poll requests have submit ref still */
3bfa5bcb
JA
5564 do_complete = __io_poll_remove_one(req, &apoll->poll);
5565 if (do_complete) {
d7718a9d 5566 io_put_req(req);
807abcb0 5567 kfree(apoll->double_poll);
3bfa5bcb
JA
5568 kfree(apoll);
5569 }
b1f573bd
XW
5570 }
5571
b41e9852
JA
5572 if (do_complete) {
5573 io_cqring_fill_event(req, -ECANCELED);
5574 io_commit_cqring(req->ctx);
f254ac04 5575 req_set_fail_links(req);
216578e5 5576 io_put_req_deferred(req, 1);
b41e9852
JA
5577 }
5578
5579 return do_complete;
221c5eb2
JA
5580}
5581
76e1b642
JA
5582/*
5583 * Returns true if we found and killed one or more poll requests
5584 */
6b81928d
PB
5585static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
5586 struct files_struct *files)
221c5eb2 5587{
78076bb6 5588 struct hlist_node *tmp;
221c5eb2 5589 struct io_kiocb *req;
8e2e1faf 5590 int posted = 0, i;
221c5eb2
JA
5591
5592 spin_lock_irq(&ctx->completion_lock);
78076bb6
JA
5593 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5594 struct hlist_head *list;
5595
5596 list = &ctx->cancel_hash[i];
f3606e3a 5597 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
6b81928d 5598 if (io_match_task(req, tsk, files))
f3606e3a
JA
5599 posted += io_poll_remove_one(req);
5600 }
221c5eb2
JA
5601 }
5602 spin_unlock_irq(&ctx->completion_lock);
b41e9852 5603
8e2e1faf
JA
5604 if (posted)
5605 io_cqring_ev_posted(ctx);
76e1b642
JA
5606
5607 return posted != 0;
221c5eb2
JA
5608}
5609
47f46768
JA
5610static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
5611{
78076bb6 5612 struct hlist_head *list;
47f46768
JA
5613 struct io_kiocb *req;
5614
78076bb6
JA
5615 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5616 hlist_for_each_entry(req, list, hash_node) {
b41e9852
JA
5617 if (sqe_addr != req->user_data)
5618 continue;
5619 if (io_poll_remove_one(req))
eac406c6 5620 return 0;
b41e9852 5621 return -EALREADY;
47f46768
JA
5622 }
5623
5624 return -ENOENT;
5625}
5626
3529d8c2
JA
5627static int io_poll_remove_prep(struct io_kiocb *req,
5628 const struct io_uring_sqe *sqe)
0969e783 5629{
0969e783
JA
5630 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5631 return -EINVAL;
5632 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
5633 sqe->poll_events)
5634 return -EINVAL;
5635
018043be 5636 req->poll_remove.addr = READ_ONCE(sqe->addr);
0969e783
JA
5637 return 0;
5638}
5639
221c5eb2
JA
5640/*
5641 * Find a running poll command that matches one specified in sqe->addr,
5642 * and remove it if found.
5643 */
61e98203 5644static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
221c5eb2
JA
5645{
5646 struct io_ring_ctx *ctx = req->ctx;
47f46768 5647 int ret;
221c5eb2 5648
221c5eb2 5649 spin_lock_irq(&ctx->completion_lock);
018043be 5650 ret = io_poll_cancel(ctx, req->poll_remove.addr);
221c5eb2
JA
5651 spin_unlock_irq(&ctx->completion_lock);
5652
4e88d6e7
JA
5653 if (ret < 0)
5654 req_set_fail_links(req);
e1e16097 5655 io_req_complete(req, ret);
221c5eb2
JA
5656 return 0;
5657}
5658
221c5eb2
JA
5659static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5660 void *key)
5661{
c2f2eb7d
JA
5662 struct io_kiocb *req = wait->private;
5663 struct io_poll_iocb *poll = &req->poll;
221c5eb2 5664
d7718a9d 5665 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
221c5eb2
JA
5666}
5667
221c5eb2
JA
5668static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5669 struct poll_table_struct *p)
5670{
5671 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5672
e8c2bc1f 5673 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
eac406c6
JA
5674}
5675
3529d8c2 5676static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
221c5eb2
JA
5677{
5678 struct io_poll_iocb *poll = &req->poll;
5769a351 5679 u32 events;
221c5eb2
JA
5680
5681 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5682 return -EINVAL;
5683 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
5684 return -EINVAL;
5685
5769a351
JX
5686 events = READ_ONCE(sqe->poll32_events);
5687#ifdef __BIG_ENDIAN
5688 events = swahw32(events);
5689#endif
a31eb4a2
JX
5690 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
5691 (events & EPOLLEXCLUSIVE);
0969e783
JA
5692 return 0;
5693}
5694
61e98203 5695static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
0969e783
JA
5696{
5697 struct io_poll_iocb *poll = &req->poll;
5698 struct io_ring_ctx *ctx = req->ctx;
5699 struct io_poll_table ipt;
0969e783 5700 __poll_t mask;
0969e783 5701
d7718a9d 5702 ipt.pt._qproc = io_poll_queue_proc;
36703247 5703
d7718a9d
JA
5704 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5705 io_poll_wake);
221c5eb2 5706
8c838788 5707 if (mask) { /* no async, we'd stolen it */
221c5eb2 5708 ipt.error = 0;
b0dd8a41 5709 io_poll_complete(req, mask, 0);
221c5eb2 5710 }
221c5eb2
JA
5711 spin_unlock_irq(&ctx->completion_lock);
5712
8c838788
JA
5713 if (mask) {
5714 io_cqring_ev_posted(ctx);
014db007 5715 io_put_req(req);
221c5eb2 5716 }
8c838788 5717 return ipt.error;
221c5eb2
JA
5718}
5719
5262f567
JA
5720static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5721{
ad8a48ac
JA
5722 struct io_timeout_data *data = container_of(timer,
5723 struct io_timeout_data, timer);
5724 struct io_kiocb *req = data->req;
5725 struct io_ring_ctx *ctx = req->ctx;
5262f567
JA
5726 unsigned long flags;
5727
5262f567 5728 spin_lock_irqsave(&ctx->completion_lock, flags);
a71976f3 5729 list_del_init(&req->timeout.list);
01cec8c1
PB
5730 atomic_set(&req->ctx->cq_timeouts,
5731 atomic_read(&req->ctx->cq_timeouts) + 1);
5732
78e19bbe 5733 io_cqring_fill_event(req, -ETIME);
5262f567
JA
5734 io_commit_cqring(ctx);
5735 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5736
5737 io_cqring_ev_posted(ctx);
4e88d6e7 5738 req_set_fail_links(req);
5262f567
JA
5739 io_put_req(req);
5740 return HRTIMER_NORESTART;
5741}
5742
fbd15848
PB
5743static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5744 __u64 user_data)
f254ac04 5745{
fbd15848 5746 struct io_timeout_data *io;
47f46768
JA
5747 struct io_kiocb *req;
5748 int ret = -ENOENT;
f254ac04 5749
135fcde8 5750 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
47f46768 5751 if (user_data == req->user_data) {
47f46768
JA
5752 ret = 0;
5753 break;
5754 }
5755 }
5756
5757 if (ret == -ENOENT)
fbd15848
PB
5758 return ERR_PTR(ret);
5759
5760 io = req->async_data;
e8c2bc1f 5761 ret = hrtimer_try_to_cancel(&io->timer);
f254ac04 5762 if (ret == -1)
fbd15848 5763 return ERR_PTR(-EALREADY);
a71976f3 5764 list_del_init(&req->timeout.list);
fbd15848
PB
5765 return req;
5766}
47f46768 5767
fbd15848
PB
5768static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
5769{
5770 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5771
5772 if (IS_ERR(req))
5773 return PTR_ERR(req);
f254ac04
JA
5774
5775 req_set_fail_links(req);
f254ac04 5776 io_cqring_fill_event(req, -ECANCELED);
216578e5 5777 io_put_req_deferred(req, 1);
f254ac04
JA
5778 return 0;
5779}
5780
9c8e11b3
PB
5781static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5782 struct timespec64 *ts, enum hrtimer_mode mode)
47f46768 5783{
9c8e11b3
PB
5784 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5785 struct io_timeout_data *data;
47f46768 5786
9c8e11b3
PB
5787 if (IS_ERR(req))
5788 return PTR_ERR(req);
47f46768 5789
9c8e11b3
PB
5790 req->timeout.off = 0; /* noseq */
5791 data = req->async_data;
5792 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5793 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5794 data->timer.function = io_timeout_fn;
5795 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5796 return 0;
47f46768
JA
5797}
5798
3529d8c2
JA
5799static int io_timeout_remove_prep(struct io_kiocb *req,
5800 const struct io_uring_sqe *sqe)
b29472ee 5801{
9c8e11b3
PB
5802 struct io_timeout_rem *tr = &req->timeout_rem;
5803
b29472ee
JA
5804 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5805 return -EINVAL;
61710e43
DA
5806 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5807 return -EINVAL;
9c8e11b3 5808 if (sqe->ioprio || sqe->buf_index || sqe->len)
b29472ee
JA
5809 return -EINVAL;
5810
9c8e11b3
PB
5811 tr->addr = READ_ONCE(sqe->addr);
5812 tr->flags = READ_ONCE(sqe->timeout_flags);
5813 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5814 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5815 return -EINVAL;
5816 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5817 return -EFAULT;
5818 } else if (tr->flags) {
5819 /* timeout removal doesn't support flags */
b29472ee 5820 return -EINVAL;
9c8e11b3 5821 }
b29472ee 5822
b29472ee
JA
5823 return 0;
5824}
5825
8662daec
PB
5826static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5827{
5828 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5829 : HRTIMER_MODE_REL;
5830}
5831
11365043
JA
5832/*
5833 * Remove or update an existing timeout command
5834 */
61e98203 5835static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
11365043 5836{
9c8e11b3 5837 struct io_timeout_rem *tr = &req->timeout_rem;
11365043 5838 struct io_ring_ctx *ctx = req->ctx;
47f46768 5839 int ret;
11365043 5840
11365043 5841 spin_lock_irq(&ctx->completion_lock);
8662daec 5842 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
9c8e11b3 5843 ret = io_timeout_cancel(ctx, tr->addr);
8662daec
PB
5844 else
5845 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5846 io_translate_timeout_mode(tr->flags));
11365043 5847
47f46768 5848 io_cqring_fill_event(req, ret);
11365043
JA
5849 io_commit_cqring(ctx);
5850 spin_unlock_irq(&ctx->completion_lock);
5262f567 5851 io_cqring_ev_posted(ctx);
4e88d6e7
JA
5852 if (ret < 0)
5853 req_set_fail_links(req);
ec9c02ad 5854 io_put_req(req);
11365043 5855 return 0;
5262f567
JA
5856}
5857
3529d8c2 5858static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2d28390a 5859 bool is_timeout_link)
5262f567 5860{
ad8a48ac 5861 struct io_timeout_data *data;
a41525ab 5862 unsigned flags;
56080b02 5863 u32 off = READ_ONCE(sqe->off);
5262f567 5864
ad8a48ac 5865 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5262f567 5866 return -EINVAL;
ad8a48ac 5867 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
a41525ab 5868 return -EINVAL;
56080b02 5869 if (off && is_timeout_link)
2d28390a 5870 return -EINVAL;
a41525ab
JA
5871 flags = READ_ONCE(sqe->timeout_flags);
5872 if (flags & ~IORING_TIMEOUT_ABS)
5262f567 5873 return -EINVAL;
bdf20073 5874
bfe68a22 5875 req->timeout.off = off;
26a61679 5876
e8c2bc1f 5877 if (!req->async_data && io_alloc_async_data(req))
26a61679
JA
5878 return -ENOMEM;
5879
e8c2bc1f 5880 data = req->async_data;
ad8a48ac 5881 data->req = req;
ad8a48ac
JA
5882
5883 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
5262f567
JA
5884 return -EFAULT;
5885
8662daec 5886 data->mode = io_translate_timeout_mode(flags);
ad8a48ac
JA
5887 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
5888 return 0;
5889}
5890
61e98203 5891static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
ad8a48ac 5892{
ad8a48ac 5893 struct io_ring_ctx *ctx = req->ctx;
e8c2bc1f 5894 struct io_timeout_data *data = req->async_data;
ad8a48ac 5895 struct list_head *entry;
bfe68a22 5896 u32 tail, off = req->timeout.off;
ad8a48ac 5897
733f5c95 5898 spin_lock_irq(&ctx->completion_lock);
93bd25bb 5899
5262f567
JA
5900 /*
5901 * sqe->off holds how many events that need to occur for this
93bd25bb
JA
5902 * timeout event to be satisfied. If it isn't set, then this is
5903 * a pure timeout request, sequence isn't used.
5262f567 5904 */
8eb7e2d0 5905 if (io_is_timeout_noseq(req)) {
93bd25bb
JA
5906 entry = ctx->timeout_list.prev;
5907 goto add;
5908 }
5262f567 5909
bfe68a22
PB
5910 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5911 req->timeout.target_seq = tail + off;
5262f567 5912
f010505b
MDG
5913 /* Update the last seq here in case io_flush_timeouts() hasn't.
5914 * This is safe because ->completion_lock is held, and submissions
5915 * and completions are never mixed in the same ->completion_lock section.
5916 */
5917 ctx->cq_last_tm_flush = tail;
5918
5262f567
JA
5919 /*
5920 * Insertion sort, ensuring the first entry in the list is always
5921 * the one we need first.
5922 */
5262f567 5923 list_for_each_prev(entry, &ctx->timeout_list) {
135fcde8
PB
5924 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5925 timeout.list);
5262f567 5926
8eb7e2d0 5927 if (io_is_timeout_noseq(nxt))
93bd25bb 5928 continue;
bfe68a22
PB
5929 /* nxt.seq is behind @tail, otherwise would've been completed */
5930 if (off >= nxt->timeout.target_seq - tail)
5262f567
JA
5931 break;
5932 }
93bd25bb 5933add:
135fcde8 5934 list_add(&req->timeout.list, entry);
ad8a48ac
JA
5935 data->timer.function = io_timeout_fn;
5936 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
5262f567 5937 spin_unlock_irq(&ctx->completion_lock);
5262f567
JA
5938 return 0;
5939}
5262f567 5940
62755e35
JA
5941static bool io_cancel_cb(struct io_wq_work *work, void *data)
5942{
5943 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5944
5945 return req->user_data == (unsigned long) data;
5946}
5947
e977d6d3 5948static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
62755e35 5949{
62755e35 5950 enum io_wq_cancel cancel_ret;
62755e35
JA
5951 int ret = 0;
5952
4f26bda1 5953 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
62755e35
JA
5954 switch (cancel_ret) {
5955 case IO_WQ_CANCEL_OK:
5956 ret = 0;
5957 break;
5958 case IO_WQ_CANCEL_RUNNING:
5959 ret = -EALREADY;
5960 break;
5961 case IO_WQ_CANCEL_NOTFOUND:
5962 ret = -ENOENT;
5963 break;
5964 }
5965
e977d6d3
JA
5966 return ret;
5967}
5968
47f46768
JA
5969static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
5970 struct io_kiocb *req, __u64 sqe_addr,
014db007 5971 int success_ret)
47f46768
JA
5972{
5973 unsigned long flags;
5974 int ret;
5975
5976 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
5977 if (ret != -ENOENT) {
5978 spin_lock_irqsave(&ctx->completion_lock, flags);
5979 goto done;
5980 }
5981
5982 spin_lock_irqsave(&ctx->completion_lock, flags);
5983 ret = io_timeout_cancel(ctx, sqe_addr);
5984 if (ret != -ENOENT)
5985 goto done;
5986 ret = io_poll_cancel(ctx, sqe_addr);
5987done:
b0dd8a41
JA
5988 if (!ret)
5989 ret = success_ret;
47f46768
JA
5990 io_cqring_fill_event(req, ret);
5991 io_commit_cqring(ctx);
5992 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5993 io_cqring_ev_posted(ctx);
5994
4e88d6e7
JA
5995 if (ret < 0)
5996 req_set_fail_links(req);
014db007 5997 io_put_req(req);
47f46768
JA
5998}
5999
3529d8c2
JA
6000static int io_async_cancel_prep(struct io_kiocb *req,
6001 const struct io_uring_sqe *sqe)
e977d6d3 6002{
fbf23849 6003 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
e977d6d3 6004 return -EINVAL;
61710e43
DA
6005 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6006 return -EINVAL;
6007 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
e977d6d3
JA
6008 return -EINVAL;
6009
fbf23849
JA
6010 req->cancel.addr = READ_ONCE(sqe->addr);
6011 return 0;
6012}
6013
61e98203 6014static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
fbf23849
JA
6015{
6016 struct io_ring_ctx *ctx = req->ctx;
fbf23849 6017
014db007 6018 io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
5262f567
JA
6019 return 0;
6020}
6021
269bbe5f 6022static int io_rsrc_update_prep(struct io_kiocb *req,
05f3fb3c
JA
6023 const struct io_uring_sqe *sqe)
6024{
6ca56f84
JA
6025 if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL))
6026 return -EINVAL;
61710e43
DA
6027 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6028 return -EINVAL;
6029 if (sqe->ioprio || sqe->rw_flags)
05f3fb3c
JA
6030 return -EINVAL;
6031
269bbe5f
BM
6032 req->rsrc_update.offset = READ_ONCE(sqe->off);
6033 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
6034 if (!req->rsrc_update.nr_args)
05f3fb3c 6035 return -EINVAL;
269bbe5f 6036 req->rsrc_update.arg = READ_ONCE(sqe->addr);
05f3fb3c
JA
6037 return 0;
6038}
6039
889fca73 6040static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
fbf23849
JA
6041{
6042 struct io_ring_ctx *ctx = req->ctx;
269bbe5f 6043 struct io_uring_rsrc_update up;
05f3fb3c 6044 int ret;
fbf23849 6045
45d189c6 6046 if (issue_flags & IO_URING_F_NONBLOCK)
05f3fb3c 6047 return -EAGAIN;
05f3fb3c 6048
269bbe5f
BM
6049 up.offset = req->rsrc_update.offset;
6050 up.data = req->rsrc_update.arg;
05f3fb3c
JA
6051
6052 mutex_lock(&ctx->uring_lock);
269bbe5f 6053 ret = __io_sqe_files_update(ctx, &up, req->rsrc_update.nr_args);
05f3fb3c
JA
6054 mutex_unlock(&ctx->uring_lock);
6055
6056 if (ret < 0)
6057 req_set_fail_links(req);
889fca73 6058 __io_req_complete(req, issue_flags, ret, 0);
5262f567
JA
6059 return 0;
6060}
6061
bfe76559 6062static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1 6063{
d625c6ee 6064 switch (req->opcode) {
e781573e 6065 case IORING_OP_NOP:
bfe76559 6066 return 0;
f67676d1
JA
6067 case IORING_OP_READV:
6068 case IORING_OP_READ_FIXED:
3a6820f2 6069 case IORING_OP_READ:
bfe76559 6070 return io_read_prep(req, sqe);
f67676d1
JA
6071 case IORING_OP_WRITEV:
6072 case IORING_OP_WRITE_FIXED:
3a6820f2 6073 case IORING_OP_WRITE:
bfe76559 6074 return io_write_prep(req, sqe);
0969e783 6075 case IORING_OP_POLL_ADD:
bfe76559 6076 return io_poll_add_prep(req, sqe);
0969e783 6077 case IORING_OP_POLL_REMOVE:
bfe76559 6078 return io_poll_remove_prep(req, sqe);
8ed8d3c3 6079 case IORING_OP_FSYNC:
bfe76559 6080 return io_prep_fsync(req, sqe);
8ed8d3c3 6081 case IORING_OP_SYNC_FILE_RANGE:
bfe76559 6082 return io_prep_sfr(req, sqe);
03b1230c 6083 case IORING_OP_SENDMSG:
fddaface 6084 case IORING_OP_SEND:
bfe76559 6085 return io_sendmsg_prep(req, sqe);
03b1230c 6086 case IORING_OP_RECVMSG:
fddaface 6087 case IORING_OP_RECV:
bfe76559 6088 return io_recvmsg_prep(req, sqe);
f499a021 6089 case IORING_OP_CONNECT:
bfe76559 6090 return io_connect_prep(req, sqe);
2d28390a 6091 case IORING_OP_TIMEOUT:
bfe76559 6092 return io_timeout_prep(req, sqe, false);
b29472ee 6093 case IORING_OP_TIMEOUT_REMOVE:
bfe76559 6094 return io_timeout_remove_prep(req, sqe);
fbf23849 6095 case IORING_OP_ASYNC_CANCEL:
bfe76559 6096 return io_async_cancel_prep(req, sqe);
2d28390a 6097 case IORING_OP_LINK_TIMEOUT:
bfe76559 6098 return io_timeout_prep(req, sqe, true);
8ed8d3c3 6099 case IORING_OP_ACCEPT:
bfe76559 6100 return io_accept_prep(req, sqe);
d63d1b5e 6101 case IORING_OP_FALLOCATE:
bfe76559 6102 return io_fallocate_prep(req, sqe);
15b71abe 6103 case IORING_OP_OPENAT:
bfe76559 6104 return io_openat_prep(req, sqe);
b5dba59e 6105 case IORING_OP_CLOSE:
bfe76559 6106 return io_close_prep(req, sqe);
05f3fb3c 6107 case IORING_OP_FILES_UPDATE:
269bbe5f 6108 return io_rsrc_update_prep(req, sqe);
eddc7ef5 6109 case IORING_OP_STATX:
bfe76559 6110 return io_statx_prep(req, sqe);
4840e418 6111 case IORING_OP_FADVISE:
bfe76559 6112 return io_fadvise_prep(req, sqe);
c1ca757b 6113 case IORING_OP_MADVISE:
bfe76559 6114 return io_madvise_prep(req, sqe);
cebdb986 6115 case IORING_OP_OPENAT2:
bfe76559 6116 return io_openat2_prep(req, sqe);
3e4827b0 6117 case IORING_OP_EPOLL_CTL:
bfe76559 6118 return io_epoll_ctl_prep(req, sqe);
7d67af2c 6119 case IORING_OP_SPLICE:
bfe76559 6120 return io_splice_prep(req, sqe);
ddf0322d 6121 case IORING_OP_PROVIDE_BUFFERS:
bfe76559 6122 return io_provide_buffers_prep(req, sqe);
067524e9 6123 case IORING_OP_REMOVE_BUFFERS:
bfe76559 6124 return io_remove_buffers_prep(req, sqe);
f2a8d5c7 6125 case IORING_OP_TEE:
bfe76559 6126 return io_tee_prep(req, sqe);
36f4fa68
JA
6127 case IORING_OP_SHUTDOWN:
6128 return io_shutdown_prep(req, sqe);
80a261fd
JA
6129 case IORING_OP_RENAMEAT:
6130 return io_renameat_prep(req, sqe);
14a1143b
JA
6131 case IORING_OP_UNLINKAT:
6132 return io_unlinkat_prep(req, sqe);
f67676d1
JA
6133 }
6134
bfe76559
PB
6135 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
6136 req->opcode);
6137 return-EINVAL;
6138}
6139
6140static int io_req_defer_prep(struct io_kiocb *req,
6141 const struct io_uring_sqe *sqe)
6142{
bfe76559
PB
6143 if (!sqe)
6144 return 0;
6145 if (io_alloc_async_data(req))
6146 return -EAGAIN;
bfe76559 6147 return io_req_prep(req, sqe);
f67676d1
JA
6148}
6149
9cf7c104
PB
6150static u32 io_get_sequence(struct io_kiocb *req)
6151{
6152 struct io_kiocb *pos;
6153 struct io_ring_ctx *ctx = req->ctx;
f2f87370 6154 u32 total_submitted, nr_reqs = 0;
9cf7c104 6155
f2f87370
PB
6156 io_for_each_link(pos, req)
6157 nr_reqs++;
9cf7c104
PB
6158
6159 total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
6160 return total_submitted - nr_reqs;
6161}
6162
3529d8c2 6163static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
de0617e4 6164{
a197f664 6165 struct io_ring_ctx *ctx = req->ctx;
27dc8338 6166 struct io_defer_entry *de;
f67676d1 6167 int ret;
9cf7c104 6168 u32 seq;
de0617e4 6169
9d858b21 6170 /* Still need defer if there is pending req in defer list. */
9cf7c104
PB
6171 if (likely(list_empty_careful(&ctx->defer_list) &&
6172 !(req->flags & REQ_F_IO_DRAIN)))
6173 return 0;
6174
6175 seq = io_get_sequence(req);
6176 /* Still a chance to pass the sequence check */
6177 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
de0617e4
JA
6178 return 0;
6179
e8c2bc1f 6180 if (!req->async_data) {
650b5481 6181 ret = io_req_defer_prep(req, sqe);
327d6d96 6182 if (ret)
650b5481
PB
6183 return ret;
6184 }
cbdcb435 6185 io_prep_async_link(req);
27dc8338
PB
6186 de = kmalloc(sizeof(*de), GFP_KERNEL);
6187 if (!de)
6188 return -ENOMEM;
2d28390a 6189
de0617e4 6190 spin_lock_irq(&ctx->completion_lock);
9cf7c104 6191 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
de0617e4 6192 spin_unlock_irq(&ctx->completion_lock);
27dc8338 6193 kfree(de);
ae34817b
PB
6194 io_queue_async_work(req);
6195 return -EIOCBQUEUED;
de0617e4
JA
6196 }
6197
915967f6 6198 trace_io_uring_defer(ctx, req, req->user_data);
27dc8338 6199 de->req = req;
9cf7c104 6200 de->seq = seq;
27dc8338 6201 list_add_tail(&de->list, &ctx->defer_list);
de0617e4
JA
6202 spin_unlock_irq(&ctx->completion_lock);
6203 return -EIOCBQUEUED;
6204}
6205
3ca405eb 6206static void __io_clean_op(struct io_kiocb *req)
99bc4c38 6207{
0e1b6fe3
PB
6208 if (req->flags & REQ_F_BUFFER_SELECTED) {
6209 switch (req->opcode) {
6210 case IORING_OP_READV:
6211 case IORING_OP_READ_FIXED:
6212 case IORING_OP_READ:
bcda7baa 6213 kfree((void *)(unsigned long)req->rw.addr);
0e1b6fe3
PB
6214 break;
6215 case IORING_OP_RECVMSG:
6216 case IORING_OP_RECV:
bcda7baa 6217 kfree(req->sr_msg.kbuf);
0e1b6fe3
PB
6218 break;
6219 }
6220 req->flags &= ~REQ_F_BUFFER_SELECTED;
99bc4c38
PB
6221 }
6222
0e1b6fe3
PB
6223 if (req->flags & REQ_F_NEED_CLEANUP) {
6224 switch (req->opcode) {
6225 case IORING_OP_READV:
6226 case IORING_OP_READ_FIXED:
6227 case IORING_OP_READ:
6228 case IORING_OP_WRITEV:
6229 case IORING_OP_WRITE_FIXED:
e8c2bc1f
JA
6230 case IORING_OP_WRITE: {
6231 struct io_async_rw *io = req->async_data;
6232 if (io->free_iovec)
6233 kfree(io->free_iovec);
0e1b6fe3 6234 break;
e8c2bc1f 6235 }
0e1b6fe3 6236 case IORING_OP_RECVMSG:
e8c2bc1f
JA
6237 case IORING_OP_SENDMSG: {
6238 struct io_async_msghdr *io = req->async_data;
257e84a5
PB
6239
6240 kfree(io->free_iov);
0e1b6fe3 6241 break;
e8c2bc1f 6242 }
0e1b6fe3
PB
6243 case IORING_OP_SPLICE:
6244 case IORING_OP_TEE:
6245 io_put_file(req, req->splice.file_in,
6246 (req->splice.flags & SPLICE_F_FD_IN_FIXED));
6247 break;
f3cd4850
JA
6248 case IORING_OP_OPENAT:
6249 case IORING_OP_OPENAT2:
6250 if (req->open.filename)
6251 putname(req->open.filename);
6252 break;
80a261fd
JA
6253 case IORING_OP_RENAMEAT:
6254 putname(req->rename.oldpath);
6255 putname(req->rename.newpath);
6256 break;
14a1143b
JA
6257 case IORING_OP_UNLINKAT:
6258 putname(req->unlink.filename);
6259 break;
0e1b6fe3
PB
6260 }
6261 req->flags &= ~REQ_F_NEED_CLEANUP;
99bc4c38 6262 }
99bc4c38
PB
6263}
6264
889fca73 6265static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1 6266{
a197f664 6267 struct io_ring_ctx *ctx = req->ctx;
d625c6ee 6268 int ret;
2b188cc1 6269
d625c6ee 6270 switch (req->opcode) {
2b188cc1 6271 case IORING_OP_NOP:
889fca73 6272 ret = io_nop(req, issue_flags);
2b188cc1
JA
6273 break;
6274 case IORING_OP_READV:
edafccee 6275 case IORING_OP_READ_FIXED:
3a6820f2 6276 case IORING_OP_READ:
889fca73 6277 ret = io_read(req, issue_flags);
edafccee 6278 break;
3529d8c2 6279 case IORING_OP_WRITEV:
edafccee 6280 case IORING_OP_WRITE_FIXED:
3a6820f2 6281 case IORING_OP_WRITE:
889fca73 6282 ret = io_write(req, issue_flags);
2b188cc1 6283 break;
c992fe29 6284 case IORING_OP_FSYNC:
45d189c6 6285 ret = io_fsync(req, issue_flags);
c992fe29 6286 break;
221c5eb2 6287 case IORING_OP_POLL_ADD:
61e98203 6288 ret = io_poll_add(req, issue_flags);
221c5eb2
JA
6289 break;
6290 case IORING_OP_POLL_REMOVE:
61e98203 6291 ret = io_poll_remove(req, issue_flags);
221c5eb2 6292 break;
5d17b4a4 6293 case IORING_OP_SYNC_FILE_RANGE:
45d189c6 6294 ret = io_sync_file_range(req, issue_flags);
5d17b4a4 6295 break;
0fa03c62 6296 case IORING_OP_SENDMSG:
889fca73 6297 ret = io_sendmsg(req, issue_flags);
062d04d7 6298 break;
fddaface 6299 case IORING_OP_SEND:
889fca73 6300 ret = io_send(req, issue_flags);
0fa03c62 6301 break;
aa1fa28f 6302 case IORING_OP_RECVMSG:
889fca73 6303 ret = io_recvmsg(req, issue_flags);
062d04d7 6304 break;
fddaface 6305 case IORING_OP_RECV:
889fca73 6306 ret = io_recv(req, issue_flags);
aa1fa28f 6307 break;
5262f567 6308 case IORING_OP_TIMEOUT:
61e98203 6309 ret = io_timeout(req, issue_flags);
5262f567 6310 break;
11365043 6311 case IORING_OP_TIMEOUT_REMOVE:
61e98203 6312 ret = io_timeout_remove(req, issue_flags);
11365043 6313 break;
17f2fe35 6314 case IORING_OP_ACCEPT:
889fca73 6315 ret = io_accept(req, issue_flags);
17f2fe35 6316 break;
f8e85cf2 6317 case IORING_OP_CONNECT:
889fca73 6318 ret = io_connect(req, issue_flags);
f8e85cf2 6319 break;
62755e35 6320 case IORING_OP_ASYNC_CANCEL:
61e98203 6321 ret = io_async_cancel(req, issue_flags);
62755e35 6322 break;
d63d1b5e 6323 case IORING_OP_FALLOCATE:
45d189c6 6324 ret = io_fallocate(req, issue_flags);
d63d1b5e 6325 break;
15b71abe 6326 case IORING_OP_OPENAT:
45d189c6 6327 ret = io_openat(req, issue_flags);
15b71abe 6328 break;
b5dba59e 6329 case IORING_OP_CLOSE:
889fca73 6330 ret = io_close(req, issue_flags);
b5dba59e 6331 break;
05f3fb3c 6332 case IORING_OP_FILES_UPDATE:
889fca73 6333 ret = io_files_update(req, issue_flags);
05f3fb3c 6334 break;
eddc7ef5 6335 case IORING_OP_STATX:
45d189c6 6336 ret = io_statx(req, issue_flags);
eddc7ef5 6337 break;
4840e418 6338 case IORING_OP_FADVISE:
45d189c6 6339 ret = io_fadvise(req, issue_flags);
4840e418 6340 break;
c1ca757b 6341 case IORING_OP_MADVISE:
45d189c6 6342 ret = io_madvise(req, issue_flags);
c1ca757b 6343 break;
cebdb986 6344 case IORING_OP_OPENAT2:
45d189c6 6345 ret = io_openat2(req, issue_flags);
cebdb986 6346 break;
3e4827b0 6347 case IORING_OP_EPOLL_CTL:
889fca73 6348 ret = io_epoll_ctl(req, issue_flags);
3e4827b0 6349 break;
7d67af2c 6350 case IORING_OP_SPLICE:
45d189c6 6351 ret = io_splice(req, issue_flags);
7d67af2c 6352 break;
ddf0322d 6353 case IORING_OP_PROVIDE_BUFFERS:
889fca73 6354 ret = io_provide_buffers(req, issue_flags);
ddf0322d 6355 break;
067524e9 6356 case IORING_OP_REMOVE_BUFFERS:
889fca73 6357 ret = io_remove_buffers(req, issue_flags);
3e4827b0 6358 break;
f2a8d5c7 6359 case IORING_OP_TEE:
45d189c6 6360 ret = io_tee(req, issue_flags);
f2a8d5c7 6361 break;
36f4fa68 6362 case IORING_OP_SHUTDOWN:
45d189c6 6363 ret = io_shutdown(req, issue_flags);
36f4fa68 6364 break;
80a261fd 6365 case IORING_OP_RENAMEAT:
45d189c6 6366 ret = io_renameat(req, issue_flags);
80a261fd 6367 break;
14a1143b 6368 case IORING_OP_UNLINKAT:
45d189c6 6369 ret = io_unlinkat(req, issue_flags);
14a1143b 6370 break;
2b188cc1
JA
6371 default:
6372 ret = -EINVAL;
6373 break;
6374 }
6375
def596e9
JA
6376 if (ret)
6377 return ret;
6378
b532576e
JA
6379 /* If the op doesn't have a file, we're not polling for it */
6380 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
11ba820b
JA
6381 const bool in_async = io_wq_current_is_worker();
6382
11ba820b
JA
6383 /* workqueue context doesn't hold uring_lock, grab it now */
6384 if (in_async)
6385 mutex_lock(&ctx->uring_lock);
6386
2e9dbe90 6387 io_iopoll_req_issued(req, in_async);
11ba820b
JA
6388
6389 if (in_async)
6390 mutex_unlock(&ctx->uring_lock);
def596e9
JA
6391 }
6392
6393 return 0;
2b188cc1
JA
6394}
6395
5280f7e5 6396static void io_wq_submit_work(struct io_wq_work *work)
2b188cc1
JA
6397{
6398 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6df1db6b 6399 struct io_kiocb *timeout;
561fb04a 6400 int ret = 0;
2b188cc1 6401
6df1db6b
PB
6402 timeout = io_prep_linked_timeout(req);
6403 if (timeout)
6404 io_queue_linked_timeout(timeout);
d4c81f38 6405
4014d943 6406 if (work->flags & IO_WQ_WORK_CANCEL)
561fb04a 6407 ret = -ECANCELED;
31b51510 6408
561fb04a 6409 if (!ret) {
561fb04a 6410 do {
889fca73 6411 ret = io_issue_sqe(req, 0);
561fb04a
JA
6412 /*
6413 * We can get EAGAIN for polled IO even though we're
6414 * forcing a sync submission from here, since we can't
6415 * wait for request slots on the block side.
6416 */
6417 if (ret != -EAGAIN)
6418 break;
6419 cond_resched();
6420 } while (1);
6421 }
31b51510 6422
561fb04a 6423 if (ret) {
c07e6719
XW
6424 struct io_ring_ctx *lock_ctx = NULL;
6425
6426 if (req->ctx->flags & IORING_SETUP_IOPOLL)
6427 lock_ctx = req->ctx;
6428
dad1b124 6429 /*
c07e6719
XW
6430 * io_iopoll_complete() does not hold completion_lock to
6431 * complete polled io, so here for polled io, we can not call
6432 * io_req_complete() directly, otherwise there maybe concurrent
6433 * access to cqring, defer_list, etc, which is not safe. Given
6434 * that io_iopoll_complete() is always called under uring_lock,
6435 * so here for polled io, we also get uring_lock to complete
6436 * it.
dad1b124 6437 */
c07e6719
XW
6438 if (lock_ctx)
6439 mutex_lock(&lock_ctx->uring_lock);
dad1b124 6440
c07e6719
XW
6441 req_set_fail_links(req);
6442 io_req_complete(req, ret);
6443
6444 if (lock_ctx)
6445 mutex_unlock(&lock_ctx->uring_lock);
edafccee 6446 }
2b188cc1
JA
6447}
6448
65e19f54
JA
6449static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6450 int index)
6451{
269bbe5f 6452 struct fixed_rsrc_table *table;
65e19f54 6453
05f3fb3c 6454 table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
84695089 6455 return table->files[index & IORING_FILE_TABLE_MASK];
65e19f54
JA
6456}
6457
8371adf5
PB
6458static struct file *io_file_get(struct io_submit_state *state,
6459 struct io_kiocb *req, int fd, bool fixed)
09bb8394 6460{
a197f664 6461 struct io_ring_ctx *ctx = req->ctx;
8da11c19 6462 struct file *file;
09bb8394 6463
8da11c19 6464 if (fixed) {
479f517b 6465 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
8371adf5 6466 return NULL;
b7620121 6467 fd = array_index_nospec(fd, ctx->nr_user_files);
8da11c19 6468 file = io_file_from_index(ctx, fd);
36f72fe2 6469 io_set_resource_node(req);
09bb8394 6470 } else {
c826bd7a 6471 trace_io_uring_file_get(ctx, fd);
8da11c19 6472 file = __io_file_get(state, fd);
09bb8394
JA
6473 }
6474
ce3d5aae
PB
6475 if (file && unlikely(file->f_op == &io_uring_fops))
6476 io_req_track_inflight(req);
8371adf5 6477 return file;
09bb8394
JA
6478}
6479
2665abfd 6480static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2b188cc1 6481{
ad8a48ac
JA
6482 struct io_timeout_data *data = container_of(timer,
6483 struct io_timeout_data, timer);
90cd7e42 6484 struct io_kiocb *prev, *req = data->req;
2665abfd 6485 struct io_ring_ctx *ctx = req->ctx;
2665abfd 6486 unsigned long flags;
2665abfd
JA
6487
6488 spin_lock_irqsave(&ctx->completion_lock, flags);
90cd7e42
PB
6489 prev = req->timeout.head;
6490 req->timeout.head = NULL;
2665abfd
JA
6491
6492 /*
6493 * We don't expect the list to be empty, that will only happen if we
6494 * race with the completion of the linked work.
6495 */
90cd7e42 6496 if (prev && refcount_inc_not_zero(&prev->refs))
f2f87370 6497 io_remove_next_linked(prev);
90cd7e42
PB
6498 else
6499 prev = NULL;
2665abfd
JA
6500 spin_unlock_irqrestore(&ctx->completion_lock, flags);
6501
6502 if (prev) {
4e88d6e7 6503 req_set_fail_links(prev);
014db007 6504 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
9ae1f8dd 6505 io_put_req_deferred(prev, 1);
47f46768 6506 } else {
9ae1f8dd
PB
6507 io_req_complete_post(req, -ETIME, 0);
6508 io_put_req_deferred(req, 1);
2665abfd 6509 }
2665abfd
JA
6510 return HRTIMER_NORESTART;
6511}
6512
7271ef3a 6513static void __io_queue_linked_timeout(struct io_kiocb *req)
2665abfd 6514{
76a46e06 6515 /*
f2f87370
PB
6516 * If the back reference is NULL, then our linked request finished
6517 * before we got a chance to setup the timer
76a46e06 6518 */
90cd7e42 6519 if (req->timeout.head) {
e8c2bc1f 6520 struct io_timeout_data *data = req->async_data;
94ae5e77 6521
ad8a48ac
JA
6522 data->timer.function = io_link_timeout_fn;
6523 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6524 data->mode);
2665abfd 6525 }
7271ef3a
JA
6526}
6527
6528static void io_queue_linked_timeout(struct io_kiocb *req)
6529{
6530 struct io_ring_ctx *ctx = req->ctx;
6531
6532 spin_lock_irq(&ctx->completion_lock);
6533 __io_queue_linked_timeout(req);
76a46e06 6534 spin_unlock_irq(&ctx->completion_lock);
2665abfd 6535
2665abfd 6536 /* drop submission reference */
76a46e06
JA
6537 io_put_req(req);
6538}
2665abfd 6539
ad8a48ac 6540static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
2665abfd 6541{
f2f87370 6542 struct io_kiocb *nxt = req->link;
2665abfd 6543
f2f87370
PB
6544 if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) ||
6545 nxt->opcode != IORING_OP_LINK_TIMEOUT)
76a46e06 6546 return NULL;
2665abfd 6547
90cd7e42 6548 nxt->timeout.head = req;
900fad45 6549 nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
76a46e06 6550 req->flags |= REQ_F_LINK_TIMEOUT;
76a46e06 6551 return nxt;
2665abfd
JA
6552}
6553
c1379e24 6554static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs)
2b188cc1 6555{
4a0a7a18 6556 struct io_kiocb *linked_timeout;
193155c8 6557 const struct cred *old_creds = NULL;
889fca73 6558 int ret, issue_flags = IO_URING_F_NONBLOCK;
2b188cc1 6559
889fca73
PB
6560 if (cs)
6561 issue_flags |= IO_URING_F_COMPLETE_DEFER;
4a0a7a18
JA
6562again:
6563 linked_timeout = io_prep_linked_timeout(req);
6564
2e5aa6cb
PB
6565 if ((req->flags & REQ_F_WORK_INITIALIZED) &&
6566 (req->work.flags & IO_WQ_WORK_CREDS) &&
98447d65 6567 req->work.identity->creds != current_cred()) {
193155c8
JA
6568 if (old_creds)
6569 revert_creds(old_creds);
98447d65 6570 if (old_creds == req->work.identity->creds)
193155c8
JA
6571 old_creds = NULL; /* restored original creds */
6572 else
98447d65 6573 old_creds = override_creds(req->work.identity->creds);
193155c8
JA
6574 }
6575
889fca73 6576 ret = io_issue_sqe(req, issue_flags);
491381ce
JA
6577
6578 /*
6579 * We async punt it if the file wasn't marked NOWAIT, or if the file
6580 * doesn't support non-blocking read/write attempts
6581 */
24c74678 6582 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
f063c547 6583 if (!io_arm_poll_handler(req)) {
f063c547
PB
6584 /*
6585 * Queued up for async execution, worker will release
6586 * submit reference when the iocb is actually submitted.
6587 */
6588 io_queue_async_work(req);
2b188cc1 6589 }
bbad27b2 6590
f063c547
PB
6591 if (linked_timeout)
6592 io_queue_linked_timeout(linked_timeout);
0d63c148
PB
6593 } else if (likely(!ret)) {
6594 /* drop submission reference */
e342c807 6595 if (req->flags & REQ_F_COMPLETE_INLINE) {
6dd0be1e
PB
6596 cs->reqs[cs->nr++] = req;
6597 if (cs->nr == IO_COMPL_BATCH)
ba88ff11 6598 io_submit_flush_completions(cs, req->ctx);
9affd664
PB
6599 req = NULL;
6600 } else {
6601 req = io_put_req_find_next(req);
6602 }
6603
0d63c148
PB
6604 if (linked_timeout)
6605 io_queue_linked_timeout(linked_timeout);
e65ef56d 6606
0d63c148
PB
6607 if (req) {
6608 if (!(req->flags & REQ_F_FORCE_ASYNC))
6609 goto again;
6610 io_queue_async_work(req);
6611 }
6612 } else {
652532ad
PB
6613 /* un-prep timeout, so it'll be killed as any other linked */
6614 req->flags &= ~REQ_F_LINK_TIMEOUT;
4e88d6e7 6615 req_set_fail_links(req);
e65ef56d 6616 io_put_req(req);
e1e16097 6617 io_req_complete(req, ret);
9e645e11 6618 }
652532ad 6619
193155c8
JA
6620 if (old_creds)
6621 revert_creds(old_creds);
2b188cc1
JA
6622}
6623
f13fad7b
JA
6624static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
6625 struct io_comp_state *cs)
4fe2c963
JL
6626{
6627 int ret;
6628
3529d8c2 6629 ret = io_req_defer(req, sqe);
4fe2c963
JL
6630 if (ret) {
6631 if (ret != -EIOCBQUEUED) {
1118591a 6632fail_req:
4e88d6e7 6633 req_set_fail_links(req);
e1e16097
JA
6634 io_put_req(req);
6635 io_req_complete(req, ret);
4fe2c963 6636 }
2550878f 6637 } else if (req->flags & REQ_F_FORCE_ASYNC) {
e8c2bc1f 6638 if (!req->async_data) {
bd2ab18a 6639 ret = io_req_defer_prep(req, sqe);
327d6d96 6640 if (unlikely(ret))
bd2ab18a
PB
6641 goto fail_req;
6642 }
ce35a47a
JA
6643 io_queue_async_work(req);
6644 } else {
c1379e24
PB
6645 if (sqe) {
6646 ret = io_req_prep(req, sqe);
6647 if (unlikely(ret))
6648 goto fail_req;
6649 }
6650 __io_queue_sqe(req, cs);
ce35a47a 6651 }
4fe2c963
JL
6652}
6653
f13fad7b
JA
6654static inline void io_queue_link_head(struct io_kiocb *req,
6655 struct io_comp_state *cs)
4fe2c963 6656{
94ae5e77 6657 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
e1e16097
JA
6658 io_put_req(req);
6659 io_req_complete(req, -ECANCELED);
1b4a51b6 6660 } else
f13fad7b 6661 io_queue_sqe(req, NULL, cs);
4fe2c963
JL
6662}
6663
863e0560
PB
6664struct io_submit_link {
6665 struct io_kiocb *head;
6666 struct io_kiocb *last;
6667};
6668
1d4240cc 6669static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
863e0560 6670 struct io_submit_link *link, struct io_comp_state *cs)
9e645e11 6671{
a197f664 6672 struct io_ring_ctx *ctx = req->ctx;
ef4ff581 6673 int ret;
9e645e11 6674
9e645e11
JA
6675 /*
6676 * If we already have a head request, queue this one for async
6677 * submittal once the head completes. If we don't have a head but
6678 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6679 * submitted sync once the chain is complete. If none of those
6680 * conditions are true (normal request), then just queue it.
6681 */
863e0560
PB
6682 if (link->head) {
6683 struct io_kiocb *head = link->head;
4e88d6e7 6684
8cdf2193
PB
6685 /*
6686 * Taking sequential execution of a link, draining both sides
6687 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
6688 * requests in the link. So, it drains the head and the
6689 * next after the link request. The last one is done via
6690 * drain_next flag to persist the effect across calls.
6691 */
ef4ff581 6692 if (req->flags & REQ_F_IO_DRAIN) {
711be031
PB
6693 head->flags |= REQ_F_IO_DRAIN;
6694 ctx->drain_next = 1;
6695 }
3529d8c2 6696 ret = io_req_defer_prep(req, sqe);
327d6d96 6697 if (unlikely(ret)) {
4e88d6e7 6698 /* fail even hard links since we don't submit */
9d76377f 6699 head->flags |= REQ_F_FAIL_LINK;
1d4240cc 6700 return ret;
2d28390a 6701 }
9d76377f 6702 trace_io_uring_link(ctx, req, head);
f2f87370 6703 link->last->link = req;
863e0560 6704 link->last = req;
32fe525b
PB
6705
6706 /* last request of a link, enqueue the link */
ef4ff581 6707 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
f13fad7b 6708 io_queue_link_head(head, cs);
863e0560 6709 link->head = NULL;
32fe525b 6710 }
9e645e11 6711 } else {
711be031
PB
6712 if (unlikely(ctx->drain_next)) {
6713 req->flags |= REQ_F_IO_DRAIN;
ef4ff581 6714 ctx->drain_next = 0;
711be031 6715 }
ef4ff581 6716 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
711be031 6717 ret = io_req_defer_prep(req, sqe);
327d6d96 6718 if (unlikely(ret))
711be031 6719 req->flags |= REQ_F_FAIL_LINK;
863e0560
PB
6720 link->head = req;
6721 link->last = req;
711be031 6722 } else {
f13fad7b 6723 io_queue_sqe(req, sqe, cs);
711be031 6724 }
9e645e11 6725 }
2e6e1fde 6726
1d4240cc 6727 return 0;
9e645e11
JA
6728}
6729
9a56a232
JA
6730/*
6731 * Batched submission is done, ensure local IO is flushed out.
6732 */
ba88ff11
PB
6733static void io_submit_state_end(struct io_submit_state *state,
6734 struct io_ring_ctx *ctx)
9a56a232 6735{
6dd0be1e 6736 if (state->comp.nr)
ba88ff11 6737 io_submit_flush_completions(&state->comp, ctx);
27926b68
JA
6738 if (state->plug_started)
6739 blk_finish_plug(&state->plug);
9f13c35b 6740 io_state_file_put(state);
9a56a232
JA
6741}
6742
6743/*
6744 * Start submission side cache.
6745 */
6746static void io_submit_state_start(struct io_submit_state *state,
ba88ff11 6747 unsigned int max_ios)
9a56a232 6748{
27926b68 6749 state->plug_started = false;
9a56a232
JA
6750 state->ios_left = max_ios;
6751}
6752
2b188cc1
JA
6753static void io_commit_sqring(struct io_ring_ctx *ctx)
6754{
75b28aff 6755 struct io_rings *rings = ctx->rings;
2b188cc1 6756
caf582c6
PB
6757 /*
6758 * Ensure any loads from the SQEs are done at this point,
6759 * since once we write the new head, the application could
6760 * write new data to them.
6761 */
6762 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2b188cc1
JA
6763}
6764
2b188cc1 6765/*
3529d8c2 6766 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
2b188cc1
JA
6767 * that is mapped by userspace. This means that care needs to be taken to
6768 * ensure that reads are stable, as we cannot rely on userspace always
6769 * being a good citizen. If members of the sqe are validated and then later
6770 * used, it's important that those reads are done through READ_ONCE() to
6771 * prevent a re-load down the line.
6772 */
709b302f 6773static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
2b188cc1 6774{
75b28aff 6775 u32 *sq_array = ctx->sq_array;
2b188cc1
JA
6776 unsigned head;
6777
6778 /*
6779 * The cached sq head (or cq tail) serves two purposes:
6780 *
6781 * 1) allows us to batch the cost of updating the user visible
6782 * head updates.
6783 * 2) allows the kernel side to track the head on its own, even
6784 * though the application is the one updating it.
6785 */
ee7d46d9 6786 head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
709b302f
PB
6787 if (likely(head < ctx->sq_entries))
6788 return &ctx->sq_sqes[head];
2b188cc1
JA
6789
6790 /* drop invalid entries */
498ccd9e 6791 ctx->cached_sq_dropped++;
ee7d46d9 6792 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
709b302f
PB
6793 return NULL;
6794}
6795
6796static inline void io_consume_sqe(struct io_ring_ctx *ctx)
6797{
6798 ctx->cached_sq_head++;
2b188cc1
JA
6799}
6800
21b55dbc
SG
6801/*
6802 * Check SQE restrictions (opcode and flags).
6803 *
6804 * Returns 'true' if SQE is allowed, 'false' otherwise.
6805 */
6806static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6807 struct io_kiocb *req,
6808 unsigned int sqe_flags)
6809{
6810 if (!ctx->restricted)
6811 return true;
6812
6813 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6814 return false;
6815
6816 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6817 ctx->restrictions.sqe_flags_required)
6818 return false;
6819
6820 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6821 ctx->restrictions.sqe_flags_required))
6822 return false;
6823
6824 return true;
6825}
6826
ef4ff581
PB
6827#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
6828 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
6829 IOSQE_BUFFER_SELECT)
6830
6831static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
258b29a9 6832 const struct io_uring_sqe *sqe)
0553b8bd 6833{
258b29a9 6834 struct io_submit_state *state;
ef4ff581 6835 unsigned int sqe_flags;
71b547c0 6836 int id, ret;
ef4ff581 6837
0553b8bd
PB
6838 req->opcode = READ_ONCE(sqe->opcode);
6839 req->user_data = READ_ONCE(sqe->user_data);
e8c2bc1f 6840 req->async_data = NULL;
0553b8bd
PB
6841 req->file = NULL;
6842 req->ctx = ctx;
6843 req->flags = 0;
f2f87370 6844 req->link = NULL;
269bbe5f 6845 req->fixed_rsrc_refs = NULL;
0553b8bd
PB
6846 /* one is dropped after submission, the other at completion */
6847 refcount_set(&req->refs, 2);
4dd2824d 6848 req->task = current;
0553b8bd 6849 req->result = 0;
ef4ff581
PB
6850
6851 if (unlikely(req->opcode >= IORING_OP_LAST))
6852 return -EINVAL;
6853
28cea78a 6854 if (unlikely(io_sq_thread_acquire_mm_files(ctx, req)))
9d8426a0 6855 return -EFAULT;
ef4ff581
PB
6856
6857 sqe_flags = READ_ONCE(sqe->flags);
6858 /* enforce forwards compatibility on users */
6859 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
6860 return -EINVAL;
6861
21b55dbc
SG
6862 if (unlikely(!io_check_restriction(ctx, req, sqe_flags)))
6863 return -EACCES;
6864
ef4ff581
PB
6865 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6866 !io_op_defs[req->opcode].buffer_select)
6867 return -EOPNOTSUPP;
6868
6869 id = READ_ONCE(sqe->personality);
6870 if (id) {
1e6fa521
JA
6871 struct io_identity *iod;
6872
1e6fa521
JA
6873 iod = idr_find(&ctx->personality_idr, id);
6874 if (unlikely(!iod))
ef4ff581 6875 return -EINVAL;
1e6fa521 6876 refcount_inc(&iod->count);
ec99ca6c
PB
6877
6878 __io_req_init_async(req);
1e6fa521
JA
6879 get_cred(iod->creds);
6880 req->work.identity = iod;
dfead8a8 6881 req->work.flags |= IO_WQ_WORK_CREDS;
ef4ff581
PB
6882 }
6883
6884 /* same numerical values with corresponding REQ_F_*, safe to copy */
c11368a5 6885 req->flags |= sqe_flags;
258b29a9 6886 state = &ctx->submit_state;
ef4ff581 6887
27926b68
JA
6888 /*
6889 * Plug now if we have more than 1 IO left after this, and the target
6890 * is potentially a read/write to block based storage.
6891 */
6892 if (!state->plug_started && state->ios_left > 1 &&
6893 io_op_defs[req->opcode].plug) {
6894 blk_start_plug(&state->plug);
6895 state->plug_started = true;
6896 }
6897
bd5bbda7
PB
6898 ret = 0;
6899 if (io_op_defs[req->opcode].needs_file) {
6900 bool fixed = req->flags & REQ_F_FIXED_FILE;
6901
6902 req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
ba13e23f 6903 if (unlikely(!req->file))
bd5bbda7
PB
6904 ret = -EBADF;
6905 }
63ff8223 6906
71b547c0
PB
6907 state->ios_left--;
6908 return ret;
0553b8bd
PB
6909}
6910
0f212204 6911static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
6c271ce2 6912{
863e0560 6913 struct io_submit_link link;
9e645e11 6914 int i, submitted = 0;
6c271ce2 6915
c4a2ed72 6916 /* if we have a backlog and couldn't flush it all, return BUSY */
ad3eb2c8 6917 if (test_bit(0, &ctx->sq_check_overflow)) {
6c503150 6918 if (!__io_cqring_overflow_flush(ctx, false, NULL, NULL))
ad3eb2c8
JA
6919 return -EBUSY;
6920 }
6c271ce2 6921
ee7d46d9
PB
6922 /* make sure SQ entry isn't read before tail */
6923 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
9ef4f124 6924
2b85edfc
PB
6925 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6926 return -EAGAIN;
6c271ce2 6927
d8a6df10 6928 percpu_counter_add(&current->io_uring->inflight, nr);
faf7b51c 6929 refcount_add(nr, &current->usage);
6c271ce2 6930
ba88ff11 6931 io_submit_state_start(&ctx->submit_state, nr);
863e0560 6932 link.head = NULL;
b14cca0c 6933
6c271ce2 6934 for (i = 0; i < nr; i++) {
3529d8c2 6935 const struct io_uring_sqe *sqe;
196be95c 6936 struct io_kiocb *req;
1cb1edb2 6937 int err;
fb5ccc98 6938
b1e50e54
PB
6939 sqe = io_get_sqe(ctx);
6940 if (unlikely(!sqe)) {
6941 io_consume_sqe(ctx);
6942 break;
6943 }
258b29a9 6944 req = io_alloc_req(ctx);
196be95c
PB
6945 if (unlikely(!req)) {
6946 if (!submitted)
6947 submitted = -EAGAIN;
fb5ccc98 6948 break;
196be95c 6949 }
709b302f 6950 io_consume_sqe(ctx);
d3656344
JA
6951 /* will complete beyond this point, count as submitted */
6952 submitted++;
6953
258b29a9 6954 err = io_init_req(ctx, req, sqe);
ef4ff581 6955 if (unlikely(err)) {
1cb1edb2 6956fail_req:
e1e16097
JA
6957 io_put_req(req);
6958 io_req_complete(req, err);
196be95c
PB
6959 break;
6960 }
fb5ccc98 6961
354420f7 6962 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
2d7e9358 6963 true, ctx->flags & IORING_SETUP_SQPOLL);
258b29a9 6964 err = io_submit_sqe(req, sqe, &link, &ctx->submit_state.comp);
1d4240cc
PB
6965 if (err)
6966 goto fail_req;
6c271ce2
JA
6967 }
6968
9466f437
PB
6969 if (unlikely(submitted != nr)) {
6970 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
d8a6df10
JA
6971 struct io_uring_task *tctx = current->io_uring;
6972 int unused = nr - ref_used;
9466f437 6973
d8a6df10
JA
6974 percpu_ref_put_many(&ctx->refs, unused);
6975 percpu_counter_sub(&tctx->inflight, unused);
6976 put_task_struct_many(current, unused);
9466f437 6977 }
863e0560 6978 if (link.head)
258b29a9 6979 io_queue_link_head(link.head, &ctx->submit_state.comp);
ba88ff11 6980 io_submit_state_end(&ctx->submit_state, ctx);
6c271ce2 6981
ae9428ca
PB
6982 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6983 io_commit_sqring(ctx);
6984
6c271ce2
JA
6985 return submitted;
6986}
6987
23b3628e
XW
6988static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6989{
6990 /* Tell userspace we may need a wakeup call */
6991 spin_lock_irq(&ctx->completion_lock);
6992 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
6993 spin_unlock_irq(&ctx->completion_lock);
6994}
6995
6996static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6997{
6998 spin_lock_irq(&ctx->completion_lock);
6999 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
7000 spin_unlock_irq(&ctx->completion_lock);
7001}
7002
08369246 7003static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
6c271ce2 7004{
c8d1ba58 7005 unsigned int to_submit;
bdcd3eab 7006 int ret = 0;
6c271ce2 7007
c8d1ba58 7008 to_submit = io_sqring_entries(ctx);
e95eee2d
JA
7009 /* if we're handling multiple rings, cap submit size for fairness */
7010 if (cap_entries && to_submit > 8)
7011 to_submit = 8;
7012
906a3c6f 7013 if (!list_empty(&ctx->iopoll_list) || to_submit) {
c8d1ba58 7014 unsigned nr_events = 0;
a4c0b3de 7015
c8d1ba58 7016 mutex_lock(&ctx->uring_lock);
906a3c6f 7017 if (!list_empty(&ctx->iopoll_list))
c8d1ba58 7018 io_do_iopoll(ctx, &nr_events, 0);
906a3c6f 7019
d9d05217
PB
7020 if (to_submit && !ctx->sqo_dead &&
7021 likely(!percpu_ref_is_dying(&ctx->refs)))
08369246 7022 ret = io_submit_sqes(ctx, to_submit);
c8d1ba58
JA
7023 mutex_unlock(&ctx->uring_lock);
7024 }
6c271ce2 7025
90554200
JA
7026 if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait))
7027 wake_up(&ctx->sqo_sq_wait);
6c271ce2 7028
08369246
XW
7029 return ret;
7030}
6c271ce2 7031
08369246
XW
7032static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
7033{
7034 struct io_ring_ctx *ctx;
7035 unsigned sq_thread_idle = 0;
6c271ce2 7036
08369246
XW
7037 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
7038 if (sq_thread_idle < ctx->sq_thread_idle)
7039 sq_thread_idle = ctx->sq_thread_idle;
c8d1ba58 7040 }
c1edbf5f 7041
08369246 7042 sqd->sq_thread_idle = sq_thread_idle;
c8d1ba58 7043}
6c271ce2 7044
69fb2131
JA
7045static void io_sqd_init_new(struct io_sq_data *sqd)
7046{
7047 struct io_ring_ctx *ctx;
7048
7049 while (!list_empty(&sqd->ctx_new_list)) {
7050 ctx = list_first_entry(&sqd->ctx_new_list, struct io_ring_ctx, sqd_list);
69fb2131
JA
7051 list_move_tail(&ctx->sqd_list, &sqd->ctx_list);
7052 complete(&ctx->sq_thread_comp);
7053 }
08369246
XW
7054
7055 io_sqd_update_thread_idle(sqd);
69fb2131
JA
7056}
7057
c8d1ba58
JA
7058static int io_sq_thread(void *data)
7059{
91d8f519 7060 struct cgroup_subsys_state *cur_css = NULL;
28cea78a
JA
7061 struct files_struct *old_files = current->files;
7062 struct nsproxy *old_nsproxy = current->nsproxy;
69fb2131
JA
7063 const struct cred *old_cred = NULL;
7064 struct io_sq_data *sqd = data;
7065 struct io_ring_ctx *ctx;
a0d9205f 7066 unsigned long timeout = 0;
08369246 7067 DEFINE_WAIT(wait);
6c271ce2 7068
28cea78a
JA
7069 task_lock(current);
7070 current->files = NULL;
7071 current->nsproxy = NULL;
7072 task_unlock(current);
6c271ce2 7073
69fb2131 7074 while (!kthread_should_stop()) {
08369246
XW
7075 int ret;
7076 bool cap_entries, sqt_spin, needs_sched;
c1edbf5f
JA
7077
7078 /*
69fb2131
JA
7079 * Any changes to the sqd lists are synchronized through the
7080 * kthread parking. This synchronizes the thread vs users,
7081 * the users are synchronized on the sqd->ctx_lock.
c1edbf5f 7082 */
65b2b213 7083 if (kthread_should_park()) {
69fb2131 7084 kthread_parkme();
65b2b213
XW
7085 /*
7086 * When sq thread is unparked, in case the previous park operation
7087 * comes from io_put_sq_data(), which means that sq thread is going
7088 * to be stopped, so here needs to have a check.
7089 */
7090 if (kthread_should_stop())
7091 break;
7092 }
7143b5ac 7093
08369246 7094 if (unlikely(!list_empty(&sqd->ctx_new_list))) {
69fb2131 7095 io_sqd_init_new(sqd);
08369246
XW
7096 timeout = jiffies + sqd->sq_thread_idle;
7097 }
6c271ce2 7098
08369246 7099 sqt_spin = false;
e95eee2d 7100 cap_entries = !list_is_singular(&sqd->ctx_list);
69fb2131
JA
7101 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
7102 if (current->cred != ctx->creds) {
7103 if (old_cred)
7104 revert_creds(old_cred);
7105 old_cred = override_creds(ctx->creds);
bdcd3eab 7106 }
91d8f519 7107 io_sq_thread_associate_blkcg(ctx, &cur_css);
4ea33a97
JA
7108#ifdef CONFIG_AUDIT
7109 current->loginuid = ctx->loginuid;
7110 current->sessionid = ctx->sessionid;
7111#endif
bdcd3eab 7112
08369246
XW
7113 ret = __io_sq_thread(ctx, cap_entries);
7114 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
7115 sqt_spin = true;
6c271ce2 7116
28cea78a 7117 io_sq_thread_drop_mm_files();
69fb2131 7118 }
6c271ce2 7119
08369246 7120 if (sqt_spin || !time_after(jiffies, timeout)) {
c8d1ba58 7121 io_run_task_work();
d434ab6d 7122 io_sq_thread_drop_mm_files();
c8d1ba58 7123 cond_resched();
08369246
XW
7124 if (sqt_spin)
7125 timeout = jiffies + sqd->sq_thread_idle;
7126 continue;
7127 }
7128
08369246
XW
7129 needs_sched = true;
7130 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
7131 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
7132 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
7133 !list_empty_careful(&ctx->iopoll_list)) {
7134 needs_sched = false;
7135 break;
7136 }
7137 if (io_sqring_entries(ctx)) {
7138 needs_sched = false;
7139 break;
7140 }
7141 }
7142
8b28fdf2 7143 if (needs_sched && !kthread_should_park()) {
69fb2131
JA
7144 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7145 io_ring_set_wakeup_flag(ctx);
08369246 7146
69fb2131 7147 schedule();
69fb2131
JA
7148 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7149 io_ring_clear_wakeup_flag(ctx);
6c271ce2 7150 }
08369246
XW
7151
7152 finish_wait(&sqd->wait, &wait);
7153 timeout = jiffies + sqd->sq_thread_idle;
6c271ce2
JA
7154 }
7155
4c6e277c 7156 io_run_task_work();
d434ab6d 7157 io_sq_thread_drop_mm_files();
b41e9852 7158
91d8f519
DZ
7159 if (cur_css)
7160 io_sq_thread_unassociate_blkcg();
69fb2131
JA
7161 if (old_cred)
7162 revert_creds(old_cred);
06058632 7163
28cea78a
JA
7164 task_lock(current);
7165 current->files = old_files;
7166 current->nsproxy = old_nsproxy;
7167 task_unlock(current);
7168
2bbcd6d3 7169 kthread_parkme();
06058632 7170
6c271ce2
JA
7171 return 0;
7172}
7173
bda52162
JA
7174struct io_wait_queue {
7175 struct wait_queue_entry wq;
7176 struct io_ring_ctx *ctx;
7177 unsigned to_wait;
7178 unsigned nr_timeouts;
7179};
7180
6c503150 7181static inline bool io_should_wake(struct io_wait_queue *iowq)
bda52162
JA
7182{
7183 struct io_ring_ctx *ctx = iowq->ctx;
7184
7185 /*
d195a66e 7186 * Wake up if we have enough events, or if a timeout occurred since we
bda52162
JA
7187 * started waiting. For timeouts, we always want to return to userspace,
7188 * regardless of event count.
7189 */
6c503150 7190 return io_cqring_events(ctx) >= iowq->to_wait ||
bda52162
JA
7191 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
7192}
7193
7194static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
7195 int wake_flags, void *key)
7196{
7197 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
7198 wq);
7199
6c503150
PB
7200 /*
7201 * Cannot safely flush overflowed CQEs from here, ensure we wake up
7202 * the task, and the next invocation will do it.
7203 */
7204 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow))
7205 return autoremove_wake_function(curr, mode, wake_flags, key);
7206 return -1;
bda52162
JA
7207}
7208
af9c1a44
JA
7209static int io_run_task_work_sig(void)
7210{
7211 if (io_run_task_work())
7212 return 1;
7213 if (!signal_pending(current))
7214 return 0;
792ee0f6
JA
7215 if (test_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL))
7216 return -ERESTARTSYS;
af9c1a44
JA
7217 return -EINTR;
7218}
7219
eeb60b9a
PB
7220/* when returns >0, the caller should retry */
7221static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7222 struct io_wait_queue *iowq,
7223 signed long *timeout)
7224{
7225 int ret;
7226
7227 /* make sure we run task_work before checking for signals */
7228 ret = io_run_task_work_sig();
7229 if (ret || io_should_wake(iowq))
7230 return ret;
7231 /* let the caller flush overflows, retry */
7232 if (test_bit(0, &ctx->cq_check_overflow))
7233 return 1;
7234
7235 *timeout = schedule_timeout(*timeout);
7236 return !*timeout ? -ETIME : 1;
7237}
7238
2b188cc1
JA
7239/*
7240 * Wait until events become available, if we don't already have some. The
7241 * application must reap them itself, as they reside on the shared cq ring.
7242 */
7243static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
c73ebb68
HX
7244 const sigset_t __user *sig, size_t sigsz,
7245 struct __kernel_timespec __user *uts)
2b188cc1 7246{
bda52162
JA
7247 struct io_wait_queue iowq = {
7248 .wq = {
7249 .private = current,
7250 .func = io_wake_function,
7251 .entry = LIST_HEAD_INIT(iowq.wq.entry),
7252 },
7253 .ctx = ctx,
7254 .to_wait = min_events,
7255 };
75b28aff 7256 struct io_rings *rings = ctx->rings;
c1d5a224
PB
7257 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7258 int ret;
2b188cc1 7259
b41e9852 7260 do {
6c503150
PB
7261 io_cqring_overflow_flush(ctx, false, NULL, NULL);
7262 if (io_cqring_events(ctx) >= min_events)
b41e9852 7263 return 0;
4c6e277c 7264 if (!io_run_task_work())
b41e9852 7265 break;
b41e9852 7266 } while (1);
2b188cc1
JA
7267
7268 if (sig) {
9e75ad5d
AB
7269#ifdef CONFIG_COMPAT
7270 if (in_compat_syscall())
7271 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
b772434b 7272 sigsz);
9e75ad5d
AB
7273 else
7274#endif
b772434b 7275 ret = set_user_sigmask(sig, sigsz);
9e75ad5d 7276
2b188cc1
JA
7277 if (ret)
7278 return ret;
7279 }
7280
c73ebb68 7281 if (uts) {
c1d5a224
PB
7282 struct timespec64 ts;
7283
c73ebb68
HX
7284 if (get_timespec64(&ts, uts))
7285 return -EFAULT;
7286 timeout = timespec64_to_jiffies(&ts);
7287 }
7288
bda52162 7289 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
c826bd7a 7290 trace_io_uring_cqring_wait(ctx, min_events);
bda52162 7291 do {
6c503150 7292 io_cqring_overflow_flush(ctx, false, NULL, NULL);
bda52162
JA
7293 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
7294 TASK_INTERRUPTIBLE);
eeb60b9a
PB
7295 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
7296 finish_wait(&ctx->wait, &iowq.wq);
7297 } while (ret > 0);
bda52162 7298
b7db41c9 7299 restore_saved_sigmask_unless(ret == -EINTR);
2b188cc1 7300
75b28aff 7301 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2b188cc1
JA
7302}
7303
6b06314c
JA
7304static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7305{
7306#if defined(CONFIG_UNIX)
7307 if (ctx->ring_sock) {
7308 struct sock *sock = ctx->ring_sock->sk;
7309 struct sk_buff *skb;
7310
7311 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7312 kfree_skb(skb);
7313 }
7314#else
7315 int i;
7316
65e19f54
JA
7317 for (i = 0; i < ctx->nr_user_files; i++) {
7318 struct file *file;
7319
7320 file = io_file_from_index(ctx, i);
7321 if (file)
7322 fput(file);
7323 }
6b06314c
JA
7324#endif
7325}
7326
00835dce 7327static void io_rsrc_data_ref_zero(struct percpu_ref *ref)
05f3fb3c 7328{
269bbe5f 7329 struct fixed_rsrc_data *data;
05f3fb3c 7330
269bbe5f 7331 data = container_of(ref, struct fixed_rsrc_data, refs);
05f3fb3c
JA
7332 complete(&data->done);
7333}
7334
2a63b2d9
BM
7335static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx)
7336{
7337 spin_lock_bh(&ctx->rsrc_ref_lock);
7338}
7339
7340static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx)
7341{
7342 spin_unlock_bh(&ctx->rsrc_ref_lock);
7343}
7344
d67d2263
BM
7345static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx,
7346 struct fixed_rsrc_data *rsrc_data,
269bbe5f 7347 struct fixed_rsrc_ref_node *ref_node)
1642b445 7348{
2a63b2d9 7349 io_rsrc_ref_lock(ctx);
269bbe5f 7350 rsrc_data->node = ref_node;
d67d2263 7351 list_add_tail(&ref_node->node, &ctx->rsrc_ref_list);
2a63b2d9 7352 io_rsrc_ref_unlock(ctx);
269bbe5f 7353 percpu_ref_get(&rsrc_data->refs);
1642b445
PB
7354}
7355
d7954b2b
BM
7356static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data,
7357 struct io_ring_ctx *ctx,
7358 struct fixed_rsrc_ref_node *backup_node)
6b06314c 7359{
d7954b2b 7360 struct fixed_rsrc_ref_node *ref_node;
1ffc5422 7361 int ret;
65e19f54 7362
2a63b2d9 7363 io_rsrc_ref_lock(ctx);
1e5d770b 7364 ref_node = data->node;
2a63b2d9 7365 io_rsrc_ref_unlock(ctx);
05589553
XW
7366 if (ref_node)
7367 percpu_ref_kill(&ref_node->refs);
7368
7369 percpu_ref_kill(&data->refs);
7370
7371 /* wait for all refs nodes to complete */
269bbe5f 7372 flush_delayed_work(&ctx->rsrc_put_work);
1ffc5422
PB
7373 do {
7374 ret = wait_for_completion_interruptible(&data->done);
7375 if (!ret)
7376 break;
7377 ret = io_run_task_work_sig();
7378 if (ret < 0) {
7379 percpu_ref_resurrect(&data->refs);
7380 reinit_completion(&data->done);
d67d2263 7381 io_sqe_rsrc_set_node(ctx, data, backup_node);
1ffc5422
PB
7382 return ret;
7383 }
7384 } while (1);
05f3fb3c 7385
d7954b2b
BM
7386 destroy_fixed_rsrc_ref_node(backup_node);
7387 return 0;
7388}
7389
1ad555c6
BM
7390static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx)
7391{
7392 struct fixed_rsrc_data *data;
7393
7394 data = kzalloc(sizeof(*data), GFP_KERNEL);
7395 if (!data)
7396 return NULL;
7397
00835dce 7398 if (percpu_ref_init(&data->refs, io_rsrc_data_ref_zero,
1ad555c6
BM
7399 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
7400 kfree(data);
7401 return NULL;
7402 }
7403 data->ctx = ctx;
7404 init_completion(&data->done);
7405 return data;
7406}
7407
7408static void free_fixed_rsrc_data(struct fixed_rsrc_data *data)
7409{
7410 percpu_ref_exit(&data->refs);
7411 kfree(data->table);
7412 kfree(data);
7413}
7414
d7954b2b
BM
7415static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7416{
7417 struct fixed_rsrc_data *data = ctx->file_data;
7418 struct fixed_rsrc_ref_node *backup_node;
7419 unsigned nr_tables, i;
7420 int ret;
7421
7422 if (!data)
7423 return -ENXIO;
7424 backup_node = alloc_fixed_rsrc_ref_node(ctx);
7425 if (!backup_node)
7426 return -ENOMEM;
7427 init_fixed_file_ref_node(ctx, backup_node);
7428
7429 ret = io_rsrc_ref_quiesce(data, ctx, backup_node);
7430 if (ret)
7431 return ret;
7432
6b06314c 7433 __io_sqe_files_unregister(ctx);
65e19f54
JA
7434 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
7435 for (i = 0; i < nr_tables; i++)
05f3fb3c 7436 kfree(data->table[i].files);
1ad555c6 7437 free_fixed_rsrc_data(data);
05f3fb3c 7438 ctx->file_data = NULL;
6b06314c
JA
7439 ctx->nr_user_files = 0;
7440 return 0;
7441}
7442
534ca6d6 7443static void io_put_sq_data(struct io_sq_data *sqd)
6c271ce2 7444{
534ca6d6 7445 if (refcount_dec_and_test(&sqd->refs)) {
2bbcd6d3
RP
7446 /*
7447 * The park is a bit of a work-around, without it we get
7448 * warning spews on shutdown with SQPOLL set and affinity
7449 * set to a single CPU.
7450 */
534ca6d6
JA
7451 if (sqd->thread) {
7452 kthread_park(sqd->thread);
7453 kthread_stop(sqd->thread);
7454 }
7455
7456 kfree(sqd);
7457 }
7458}
7459
aa06165d
JA
7460static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7461{
7462 struct io_ring_ctx *ctx_attach;
7463 struct io_sq_data *sqd;
7464 struct fd f;
7465
7466 f = fdget(p->wq_fd);
7467 if (!f.file)
7468 return ERR_PTR(-ENXIO);
7469 if (f.file->f_op != &io_uring_fops) {
7470 fdput(f);
7471 return ERR_PTR(-EINVAL);
7472 }
7473
7474 ctx_attach = f.file->private_data;
7475 sqd = ctx_attach->sq_data;
7476 if (!sqd) {
7477 fdput(f);
7478 return ERR_PTR(-EINVAL);
7479 }
7480
7481 refcount_inc(&sqd->refs);
7482 fdput(f);
7483 return sqd;
7484}
7485
534ca6d6
JA
7486static struct io_sq_data *io_get_sq_data(struct io_uring_params *p)
7487{
7488 struct io_sq_data *sqd;
7489
aa06165d
JA
7490 if (p->flags & IORING_SETUP_ATTACH_WQ)
7491 return io_attach_sq_data(p);
7492
534ca6d6
JA
7493 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7494 if (!sqd)
7495 return ERR_PTR(-ENOMEM);
7496
7497 refcount_set(&sqd->refs, 1);
69fb2131
JA
7498 INIT_LIST_HEAD(&sqd->ctx_list);
7499 INIT_LIST_HEAD(&sqd->ctx_new_list);
7500 mutex_init(&sqd->ctx_lock);
7501 mutex_init(&sqd->lock);
534ca6d6
JA
7502 init_waitqueue_head(&sqd->wait);
7503 return sqd;
7504}
7505
69fb2131
JA
7506static void io_sq_thread_unpark(struct io_sq_data *sqd)
7507 __releases(&sqd->lock)
7508{
7509 if (!sqd->thread)
7510 return;
7511 kthread_unpark(sqd->thread);
7512 mutex_unlock(&sqd->lock);
7513}
7514
7515static void io_sq_thread_park(struct io_sq_data *sqd)
7516 __acquires(&sqd->lock)
7517{
7518 if (!sqd->thread)
7519 return;
7520 mutex_lock(&sqd->lock);
7521 kthread_park(sqd->thread);
7522}
7523
534ca6d6
JA
7524static void io_sq_thread_stop(struct io_ring_ctx *ctx)
7525{
7526 struct io_sq_data *sqd = ctx->sq_data;
7527
7528 if (sqd) {
7529 if (sqd->thread) {
7530 /*
7531 * We may arrive here from the error branch in
7532 * io_sq_offload_create() where the kthread is created
7533 * without being waked up, thus wake it up now to make
7534 * sure the wait will complete.
7535 */
7536 wake_up_process(sqd->thread);
7537 wait_for_completion(&ctx->sq_thread_comp);
69fb2131
JA
7538
7539 io_sq_thread_park(sqd);
7540 }
7541
7542 mutex_lock(&sqd->ctx_lock);
7543 list_del(&ctx->sqd_list);
08369246 7544 io_sqd_update_thread_idle(sqd);
69fb2131
JA
7545 mutex_unlock(&sqd->ctx_lock);
7546
08369246 7547 if (sqd->thread)
69fb2131 7548 io_sq_thread_unpark(sqd);
534ca6d6
JA
7549
7550 io_put_sq_data(sqd);
7551 ctx->sq_data = NULL;
6c271ce2
JA
7552 }
7553}
7554
6b06314c
JA
7555static void io_finish_async(struct io_ring_ctx *ctx)
7556{
6c271ce2
JA
7557 io_sq_thread_stop(ctx);
7558
561fb04a
JA
7559 if (ctx->io_wq) {
7560 io_wq_destroy(ctx->io_wq);
7561 ctx->io_wq = NULL;
6b06314c
JA
7562 }
7563}
7564
7565#if defined(CONFIG_UNIX)
6b06314c
JA
7566/*
7567 * Ensure the UNIX gc is aware of our file set, so we are certain that
7568 * the io_uring can be safely unregistered on process exit, even if we have
7569 * loops in the file referencing.
7570 */
7571static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7572{
7573 struct sock *sk = ctx->ring_sock->sk;
7574 struct scm_fp_list *fpl;
7575 struct sk_buff *skb;
08a45173 7576 int i, nr_files;
6b06314c 7577
6b06314c
JA
7578 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7579 if (!fpl)
7580 return -ENOMEM;
7581
7582 skb = alloc_skb(0, GFP_KERNEL);
7583 if (!skb) {
7584 kfree(fpl);
7585 return -ENOMEM;
7586 }
7587
7588 skb->sk = sk;
6b06314c 7589
08a45173 7590 nr_files = 0;
6b06314c
JA
7591 fpl->user = get_uid(ctx->user);
7592 for (i = 0; i < nr; i++) {
65e19f54
JA
7593 struct file *file = io_file_from_index(ctx, i + offset);
7594
7595 if (!file)
08a45173 7596 continue;
65e19f54 7597 fpl->fp[nr_files] = get_file(file);
08a45173
JA
7598 unix_inflight(fpl->user, fpl->fp[nr_files]);
7599 nr_files++;
6b06314c
JA
7600 }
7601
08a45173
JA
7602 if (nr_files) {
7603 fpl->max = SCM_MAX_FD;
7604 fpl->count = nr_files;
7605 UNIXCB(skb).fp = fpl;
05f3fb3c 7606 skb->destructor = unix_destruct_scm;
08a45173
JA
7607 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7608 skb_queue_head(&sk->sk_receive_queue, skb);
6b06314c 7609
08a45173
JA
7610 for (i = 0; i < nr_files; i++)
7611 fput(fpl->fp[i]);
7612 } else {
7613 kfree_skb(skb);
7614 kfree(fpl);
7615 }
6b06314c
JA
7616
7617 return 0;
7618}
7619
7620/*
7621 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7622 * causes regular reference counting to break down. We rely on the UNIX
7623 * garbage collection to take care of this problem for us.
7624 */
7625static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7626{
7627 unsigned left, total;
7628 int ret = 0;
7629
7630 total = 0;
7631 left = ctx->nr_user_files;
7632 while (left) {
7633 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
6b06314c
JA
7634
7635 ret = __io_sqe_files_scm(ctx, this_files, total);
7636 if (ret)
7637 break;
7638 left -= this_files;
7639 total += this_files;
7640 }
7641
7642 if (!ret)
7643 return 0;
7644
7645 while (total < ctx->nr_user_files) {
65e19f54
JA
7646 struct file *file = io_file_from_index(ctx, total);
7647
7648 if (file)
7649 fput(file);
6b06314c
JA
7650 total++;
7651 }
7652
7653 return ret;
7654}
7655#else
7656static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7657{
7658 return 0;
7659}
7660#endif
7661
269bbe5f 7662static int io_sqe_alloc_file_tables(struct fixed_rsrc_data *file_data,
5398ae69 7663 unsigned nr_tables, unsigned nr_files)
65e19f54
JA
7664{
7665 int i;
7666
7667 for (i = 0; i < nr_tables; i++) {
269bbe5f 7668 struct fixed_rsrc_table *table = &file_data->table[i];
65e19f54
JA
7669 unsigned this_files;
7670
7671 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
7672 table->files = kcalloc(this_files, sizeof(struct file *),
7673 GFP_KERNEL);
7674 if (!table->files)
7675 break;
7676 nr_files -= this_files;
7677 }
7678
7679 if (i == nr_tables)
7680 return 0;
7681
7682 for (i = 0; i < nr_tables; i++) {
269bbe5f 7683 struct fixed_rsrc_table *table = &file_data->table[i];
65e19f54
JA
7684 kfree(table->files);
7685 }
7686 return 1;
7687}
7688
50238531 7689static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
05f3fb3c 7690{
50238531 7691 struct file *file = prsrc->file;
05f3fb3c
JA
7692#if defined(CONFIG_UNIX)
7693 struct sock *sock = ctx->ring_sock->sk;
7694 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7695 struct sk_buff *skb;
7696 int i;
7697
7698 __skb_queue_head_init(&list);
7699
7700 /*
7701 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7702 * remove this entry and rearrange the file array.
7703 */
7704 skb = skb_dequeue(head);
7705 while (skb) {
7706 struct scm_fp_list *fp;
7707
7708 fp = UNIXCB(skb).fp;
7709 for (i = 0; i < fp->count; i++) {
7710 int left;
7711
7712 if (fp->fp[i] != file)
7713 continue;
7714
7715 unix_notinflight(fp->user, fp->fp[i]);
7716 left = fp->count - 1 - i;
7717 if (left) {
7718 memmove(&fp->fp[i], &fp->fp[i + 1],
7719 left * sizeof(struct file *));
7720 }
7721 fp->count--;
7722 if (!fp->count) {
7723 kfree_skb(skb);
7724 skb = NULL;
7725 } else {
7726 __skb_queue_tail(&list, skb);
7727 }
7728 fput(file);
7729 file = NULL;
7730 break;
7731 }
7732
7733 if (!file)
7734 break;
7735
7736 __skb_queue_tail(&list, skb);
7737
7738 skb = skb_dequeue(head);
7739 }
7740
7741 if (skb_peek(&list)) {
7742 spin_lock_irq(&head->lock);
7743 while ((skb = __skb_dequeue(&list)) != NULL)
7744 __skb_queue_tail(head, skb);
7745 spin_unlock_irq(&head->lock);
7746 }
7747#else
7748 fput(file);
7749#endif
7750}
7751
269bbe5f 7752static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node)
65e19f54 7753{
269bbe5f
BM
7754 struct fixed_rsrc_data *rsrc_data = ref_node->rsrc_data;
7755 struct io_ring_ctx *ctx = rsrc_data->ctx;
7756 struct io_rsrc_put *prsrc, *tmp;
05589553 7757
269bbe5f
BM
7758 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7759 list_del(&prsrc->list);
50238531 7760 ref_node->rsrc_put(ctx, prsrc);
269bbe5f 7761 kfree(prsrc);
65e19f54 7762 }
05589553 7763
05589553
XW
7764 percpu_ref_exit(&ref_node->refs);
7765 kfree(ref_node);
269bbe5f 7766 percpu_ref_put(&rsrc_data->refs);
2faf852d 7767}
65e19f54 7768
269bbe5f 7769static void io_rsrc_put_work(struct work_struct *work)
4a38aed2
JA
7770{
7771 struct io_ring_ctx *ctx;
7772 struct llist_node *node;
7773
269bbe5f
BM
7774 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7775 node = llist_del_all(&ctx->rsrc_put_llist);
4a38aed2
JA
7776
7777 while (node) {
269bbe5f 7778 struct fixed_rsrc_ref_node *ref_node;
4a38aed2
JA
7779 struct llist_node *next = node->next;
7780
269bbe5f
BM
7781 ref_node = llist_entry(node, struct fixed_rsrc_ref_node, llist);
7782 __io_rsrc_put_work(ref_node);
4a38aed2
JA
7783 node = next;
7784 }
7785}
7786
ea64ec02
PB
7787static struct file **io_fixed_file_slot(struct fixed_rsrc_data *file_data,
7788 unsigned i)
7789{
7790 struct fixed_rsrc_table *table;
7791
7792 table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT];
7793 return &table->files[i & IORING_FILE_TABLE_MASK];
7794}
7795
00835dce 7796static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
2faf852d 7797{
269bbe5f
BM
7798 struct fixed_rsrc_ref_node *ref_node;
7799 struct fixed_rsrc_data *data;
4a38aed2 7800 struct io_ring_ctx *ctx;
e297822b 7801 bool first_add = false;
4a38aed2 7802 int delay = HZ;
65e19f54 7803
269bbe5f
BM
7804 ref_node = container_of(ref, struct fixed_rsrc_ref_node, refs);
7805 data = ref_node->rsrc_data;
e297822b
PB
7806 ctx = data->ctx;
7807
2a63b2d9 7808 io_rsrc_ref_lock(ctx);
e297822b
PB
7809 ref_node->done = true;
7810
d67d2263
BM
7811 while (!list_empty(&ctx->rsrc_ref_list)) {
7812 ref_node = list_first_entry(&ctx->rsrc_ref_list,
269bbe5f 7813 struct fixed_rsrc_ref_node, node);
e297822b
PB
7814 /* recycle ref nodes in order */
7815 if (!ref_node->done)
7816 break;
7817 list_del(&ref_node->node);
269bbe5f 7818 first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist);
e297822b 7819 }
2a63b2d9 7820 io_rsrc_ref_unlock(ctx);
05589553 7821
e297822b 7822 if (percpu_ref_is_dying(&data->refs))
4a38aed2 7823 delay = 0;
05589553 7824
4a38aed2 7825 if (!delay)
269bbe5f 7826 mod_delayed_work(system_wq, &ctx->rsrc_put_work, 0);
4a38aed2 7827 else if (first_add)
269bbe5f 7828 queue_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
05f3fb3c 7829}
65e19f54 7830
6802535d 7831static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(
05589553 7832 struct io_ring_ctx *ctx)
05f3fb3c 7833{
269bbe5f 7834 struct fixed_rsrc_ref_node *ref_node;
05f3fb3c 7835
05589553
XW
7836 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7837 if (!ref_node)
3e2224c5 7838 return NULL;
05f3fb3c 7839
00835dce 7840 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
05589553
XW
7841 0, GFP_KERNEL)) {
7842 kfree(ref_node);
3e2224c5 7843 return NULL;
05589553
XW
7844 }
7845 INIT_LIST_HEAD(&ref_node->node);
269bbe5f 7846 INIT_LIST_HEAD(&ref_node->rsrc_list);
6802535d
BM
7847 ref_node->done = false;
7848 return ref_node;
7849}
7850
bc9744cd
PB
7851static void init_fixed_file_ref_node(struct io_ring_ctx *ctx,
7852 struct fixed_rsrc_ref_node *ref_node)
6802535d 7853{
269bbe5f 7854 ref_node->rsrc_data = ctx->file_data;
50238531 7855 ref_node->rsrc_put = io_ring_file_put;
05589553
XW
7856}
7857
269bbe5f 7858static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node)
05589553
XW
7859{
7860 percpu_ref_exit(&ref_node->refs);
7861 kfree(ref_node);
65e19f54
JA
7862}
7863
ea64ec02 7864
6b06314c
JA
7865static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
7866 unsigned nr_args)
7867{
7868 __s32 __user *fds = (__s32 __user *) arg;
600cf3f8 7869 unsigned nr_tables, i;
05f3fb3c 7870 struct file *file;
600cf3f8 7871 int fd, ret = -ENOMEM;
269bbe5f
BM
7872 struct fixed_rsrc_ref_node *ref_node;
7873 struct fixed_rsrc_data *file_data;
6b06314c 7874
05f3fb3c 7875 if (ctx->file_data)
6b06314c
JA
7876 return -EBUSY;
7877 if (!nr_args)
7878 return -EINVAL;
7879 if (nr_args > IORING_MAX_FIXED_FILES)
7880 return -EMFILE;
7881
1ad555c6 7882 file_data = alloc_fixed_rsrc_data(ctx);
5398ae69 7883 if (!file_data)
05f3fb3c 7884 return -ENOMEM;
13770a71 7885 ctx->file_data = file_data;
05f3fb3c 7886
65e19f54 7887 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
035fbafc 7888 file_data->table = kcalloc(nr_tables, sizeof(*file_data->table),
5398ae69 7889 GFP_KERNEL);
600cf3f8
PB
7890 if (!file_data->table)
7891 goto out_free;
05f3fb3c 7892
600cf3f8 7893 if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args))
1ad555c6 7894 goto out_free;
65e19f54 7895
08a45173 7896 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
600cf3f8
PB
7897 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
7898 ret = -EFAULT;
7899 goto out_fput;
7900 }
08a45173 7901 /* allow sparse sets */
600cf3f8 7902 if (fd == -1)
08a45173 7903 continue;
6b06314c 7904
05f3fb3c 7905 file = fget(fd);
6b06314c 7906 ret = -EBADF;
05f3fb3c 7907 if (!file)
600cf3f8 7908 goto out_fput;
05f3fb3c 7909
6b06314c
JA
7910 /*
7911 * Don't allow io_uring instances to be registered. If UNIX
7912 * isn't enabled, then this causes a reference cycle and this
7913 * instance can never get freed. If UNIX is enabled we'll
7914 * handle it just fine, but there's still no point in allowing
7915 * a ring fd as it doesn't support regular read/write anyway.
7916 */
05f3fb3c
JA
7917 if (file->f_op == &io_uring_fops) {
7918 fput(file);
600cf3f8 7919 goto out_fput;
6b06314c 7920 }
ea64ec02 7921 *io_fixed_file_slot(file_data, i) = file;
6b06314c
JA
7922 }
7923
6b06314c 7924 ret = io_sqe_files_scm(ctx);
05589553 7925 if (ret) {
6b06314c 7926 io_sqe_files_unregister(ctx);
05589553
XW
7927 return ret;
7928 }
6b06314c 7929
bc9744cd 7930 ref_node = alloc_fixed_rsrc_ref_node(ctx);
3e2224c5 7931 if (!ref_node) {
05589553 7932 io_sqe_files_unregister(ctx);
3e2224c5 7933 return -ENOMEM;
05589553 7934 }
bc9744cd 7935 init_fixed_file_ref_node(ctx, ref_node);
05589553 7936
d67d2263 7937 io_sqe_rsrc_set_node(ctx, file_data, ref_node);
6b06314c 7938 return ret;
600cf3f8
PB
7939out_fput:
7940 for (i = 0; i < ctx->nr_user_files; i++) {
7941 file = io_file_from_index(ctx, i);
7942 if (file)
7943 fput(file);
7944 }
7945 for (i = 0; i < nr_tables; i++)
7946 kfree(file_data->table[i].files);
7947 ctx->nr_user_files = 0;
600cf3f8 7948out_free:
1ad555c6 7949 free_fixed_rsrc_data(ctx->file_data);
55cbc256 7950 ctx->file_data = NULL;
6b06314c
JA
7951 return ret;
7952}
7953
c3a31e60
JA
7954static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7955 int index)
7956{
7957#if defined(CONFIG_UNIX)
7958 struct sock *sock = ctx->ring_sock->sk;
7959 struct sk_buff_head *head = &sock->sk_receive_queue;
7960 struct sk_buff *skb;
7961
7962 /*
7963 * See if we can merge this file into an existing skb SCM_RIGHTS
7964 * file set. If there's no room, fall back to allocating a new skb
7965 * and filling it in.
7966 */
7967 spin_lock_irq(&head->lock);
7968 skb = skb_peek(head);
7969 if (skb) {
7970 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7971
7972 if (fpl->count < SCM_MAX_FD) {
7973 __skb_unlink(skb, head);
7974 spin_unlock_irq(&head->lock);
7975 fpl->fp[fpl->count] = get_file(file);
7976 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7977 fpl->count++;
7978 spin_lock_irq(&head->lock);
7979 __skb_queue_head(head, skb);
7980 } else {
7981 skb = NULL;
7982 }
7983 }
7984 spin_unlock_irq(&head->lock);
7985
7986 if (skb) {
7987 fput(file);
7988 return 0;
7989 }
7990
7991 return __io_sqe_files_scm(ctx, 1, index);
7992#else
7993 return 0;
7994#endif
7995}
7996
50238531 7997static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, void *rsrc)
05f3fb3c 7998{
269bbe5f
BM
7999 struct io_rsrc_put *prsrc;
8000 struct fixed_rsrc_ref_node *ref_node = data->node;
05f3fb3c 8001
269bbe5f
BM
8002 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
8003 if (!prsrc)
a5318d3c 8004 return -ENOMEM;
05f3fb3c 8005
50238531 8006 prsrc->rsrc = rsrc;
269bbe5f 8007 list_add(&prsrc->list, &ref_node->rsrc_list);
05589553 8008
a5318d3c 8009 return 0;
05f3fb3c
JA
8010}
8011
269bbe5f
BM
8012static inline int io_queue_file_removal(struct fixed_rsrc_data *data,
8013 struct file *file)
8014{
50238531 8015 return io_queue_rsrc_removal(data, (void *)file);
269bbe5f
BM
8016}
8017
05f3fb3c 8018static int __io_sqe_files_update(struct io_ring_ctx *ctx,
269bbe5f 8019 struct io_uring_rsrc_update *up,
05f3fb3c
JA
8020 unsigned nr_args)
8021{
269bbe5f
BM
8022 struct fixed_rsrc_data *data = ctx->file_data;
8023 struct fixed_rsrc_ref_node *ref_node;
ea64ec02 8024 struct file *file, **file_slot;
c3a31e60
JA
8025 __s32 __user *fds;
8026 int fd, i, err;
8027 __u32 done;
05589553 8028 bool needs_switch = false;
c3a31e60 8029
05f3fb3c 8030 if (check_add_overflow(up->offset, nr_args, &done))
c3a31e60
JA
8031 return -EOVERFLOW;
8032 if (done > ctx->nr_user_files)
8033 return -EINVAL;
8034
bc9744cd 8035 ref_node = alloc_fixed_rsrc_ref_node(ctx);
3e2224c5
MWO
8036 if (!ref_node)
8037 return -ENOMEM;
bc9744cd 8038 init_fixed_file_ref_node(ctx, ref_node);
05589553 8039
269bbe5f 8040 fds = u64_to_user_ptr(up->data);
67973b93 8041 for (done = 0; done < nr_args; done++) {
c3a31e60
JA
8042 err = 0;
8043 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
8044 err = -EFAULT;
8045 break;
8046 }
4e0377a1 8047 if (fd == IORING_REGISTER_FILES_SKIP)
8048 continue;
8049
67973b93 8050 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
ea64ec02
PB
8051 file_slot = io_fixed_file_slot(ctx->file_data, i);
8052
8053 if (*file_slot) {
8054 err = io_queue_file_removal(data, *file_slot);
a5318d3c
HD
8055 if (err)
8056 break;
ea64ec02 8057 *file_slot = NULL;
05589553 8058 needs_switch = true;
c3a31e60
JA
8059 }
8060 if (fd != -1) {
c3a31e60
JA
8061 file = fget(fd);
8062 if (!file) {
8063 err = -EBADF;
8064 break;
8065 }
8066 /*
8067 * Don't allow io_uring instances to be registered. If
8068 * UNIX isn't enabled, then this causes a reference
8069 * cycle and this instance can never get freed. If UNIX
8070 * is enabled we'll handle it just fine, but there's
8071 * still no point in allowing a ring fd as it doesn't
8072 * support regular read/write anyway.
8073 */
8074 if (file->f_op == &io_uring_fops) {
8075 fput(file);
8076 err = -EBADF;
8077 break;
8078 }
c3a31e60 8079 err = io_sqe_file_register(ctx, file, i);
f3bd9dae
YY
8080 if (err) {
8081 fput(file);
c3a31e60 8082 break;
f3bd9dae 8083 }
ea64ec02 8084 *file_slot = file;
c3a31e60 8085 }
05f3fb3c
JA
8086 }
8087
05589553 8088 if (needs_switch) {
b2e96852 8089 percpu_ref_kill(&data->node->refs);
d67d2263 8090 io_sqe_rsrc_set_node(ctx, data, ref_node);
05589553 8091 } else
269bbe5f 8092 destroy_fixed_rsrc_ref_node(ref_node);
c3a31e60
JA
8093
8094 return done ? done : err;
8095}
05589553 8096
05f3fb3c
JA
8097static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
8098 unsigned nr_args)
8099{
269bbe5f 8100 struct io_uring_rsrc_update up;
05f3fb3c
JA
8101
8102 if (!ctx->file_data)
8103 return -ENXIO;
8104 if (!nr_args)
8105 return -EINVAL;
8106 if (copy_from_user(&up, arg, sizeof(up)))
8107 return -EFAULT;
8108 if (up.resv)
8109 return -EINVAL;
8110
8111 return __io_sqe_files_update(ctx, &up, nr_args);
8112}
c3a31e60 8113
5280f7e5 8114static struct io_wq_work *io_free_work(struct io_wq_work *work)
7d723065
JA
8115{
8116 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8117
5280f7e5
PB
8118 req = io_put_req_find_next(req);
8119 return req ? &req->work : NULL;
7d723065
JA
8120}
8121
24369c2e
PB
8122static int io_init_wq_offload(struct io_ring_ctx *ctx,
8123 struct io_uring_params *p)
8124{
8125 struct io_wq_data data;
8126 struct fd f;
8127 struct io_ring_ctx *ctx_attach;
8128 unsigned int concurrency;
8129 int ret = 0;
8130
8131 data.user = ctx->user;
e9fd9396 8132 data.free_work = io_free_work;
f5fa38c5 8133 data.do_work = io_wq_submit_work;
24369c2e
PB
8134
8135 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
8136 /* Do QD, or 4 * CPUS, whatever is smallest */
8137 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
8138
8139 ctx->io_wq = io_wq_create(concurrency, &data);
8140 if (IS_ERR(ctx->io_wq)) {
8141 ret = PTR_ERR(ctx->io_wq);
8142 ctx->io_wq = NULL;
8143 }
8144 return ret;
8145 }
8146
8147 f = fdget(p->wq_fd);
8148 if (!f.file)
8149 return -EBADF;
8150
8151 if (f.file->f_op != &io_uring_fops) {
8152 ret = -EINVAL;
8153 goto out_fput;
8154 }
8155
8156 ctx_attach = f.file->private_data;
8157 /* @io_wq is protected by holding the fd */
8158 if (!io_wq_get(ctx_attach->io_wq, &data)) {
8159 ret = -EINVAL;
8160 goto out_fput;
8161 }
8162
8163 ctx->io_wq = ctx_attach->io_wq;
8164out_fput:
8165 fdput(f);
8166 return ret;
8167}
8168
0f212204
JA
8169static int io_uring_alloc_task_context(struct task_struct *task)
8170{
8171 struct io_uring_task *tctx;
d8a6df10 8172 int ret;
0f212204
JA
8173
8174 tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
8175 if (unlikely(!tctx))
8176 return -ENOMEM;
8177
d8a6df10
JA
8178 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8179 if (unlikely(ret)) {
8180 kfree(tctx);
8181 return ret;
8182 }
8183
0f212204
JA
8184 xa_init(&tctx->xa);
8185 init_waitqueue_head(&tctx->wait);
8186 tctx->last = NULL;
fdaf083c
JA
8187 atomic_set(&tctx->in_idle, 0);
8188 tctx->sqpoll = false;
500a373d
JA
8189 io_init_identity(&tctx->__identity);
8190 tctx->identity = &tctx->__identity;
0f212204 8191 task->io_uring = tctx;
7cbf1722
JA
8192 spin_lock_init(&tctx->task_lock);
8193 INIT_WQ_LIST(&tctx->task_list);
8194 tctx->task_state = 0;
8195 init_task_work(&tctx->task_work, tctx_task_work);
0f212204
JA
8196 return 0;
8197}
8198
8199void __io_uring_free(struct task_struct *tsk)
8200{
8201 struct io_uring_task *tctx = tsk->io_uring;
8202
8203 WARN_ON_ONCE(!xa_empty(&tctx->xa));
500a373d
JA
8204 WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);
8205 if (tctx->identity != &tctx->__identity)
8206 kfree(tctx->identity);
d8a6df10 8207 percpu_counter_destroy(&tctx->inflight);
0f212204
JA
8208 kfree(tctx);
8209 tsk->io_uring = NULL;
8210}
8211
7e84e1c7
SG
8212static int io_sq_offload_create(struct io_ring_ctx *ctx,
8213 struct io_uring_params *p)
2b188cc1
JA
8214{
8215 int ret;
8216
6c271ce2 8217 if (ctx->flags & IORING_SETUP_SQPOLL) {
534ca6d6
JA
8218 struct io_sq_data *sqd;
8219
3ec482d1 8220 ret = -EPERM;
ce59fc69 8221 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_NICE))
3ec482d1
JA
8222 goto err;
8223
534ca6d6
JA
8224 sqd = io_get_sq_data(p);
8225 if (IS_ERR(sqd)) {
8226 ret = PTR_ERR(sqd);
8227 goto err;
8228 }
69fb2131 8229
534ca6d6 8230 ctx->sq_data = sqd;
69fb2131
JA
8231 io_sq_thread_park(sqd);
8232 mutex_lock(&sqd->ctx_lock);
8233 list_add(&ctx->sqd_list, &sqd->ctx_new_list);
8234 mutex_unlock(&sqd->ctx_lock);
8235 io_sq_thread_unpark(sqd);
534ca6d6 8236
917257da
JA
8237 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8238 if (!ctx->sq_thread_idle)
8239 ctx->sq_thread_idle = HZ;
8240
aa06165d
JA
8241 if (sqd->thread)
8242 goto done;
8243
6c271ce2 8244 if (p->flags & IORING_SETUP_SQ_AFF) {
44a9bd18 8245 int cpu = p->sq_thread_cpu;
6c271ce2 8246
917257da 8247 ret = -EINVAL;
44a9bd18
JA
8248 if (cpu >= nr_cpu_ids)
8249 goto err;
7889f44d 8250 if (!cpu_online(cpu))
917257da
JA
8251 goto err;
8252
69fb2131 8253 sqd->thread = kthread_create_on_cpu(io_sq_thread, sqd,
534ca6d6 8254 cpu, "io_uring-sq");
6c271ce2 8255 } else {
69fb2131 8256 sqd->thread = kthread_create(io_sq_thread, sqd,
6c271ce2
JA
8257 "io_uring-sq");
8258 }
534ca6d6
JA
8259 if (IS_ERR(sqd->thread)) {
8260 ret = PTR_ERR(sqd->thread);
8261 sqd->thread = NULL;
6c271ce2
JA
8262 goto err;
8263 }
534ca6d6 8264 ret = io_uring_alloc_task_context(sqd->thread);
0f212204
JA
8265 if (ret)
8266 goto err;
6c271ce2
JA
8267 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8268 /* Can't have SQ_AFF without SQPOLL */
8269 ret = -EINVAL;
8270 goto err;
8271 }
8272
aa06165d 8273done:
24369c2e
PB
8274 ret = io_init_wq_offload(ctx, p);
8275 if (ret)
2b188cc1 8276 goto err;
2b188cc1
JA
8277
8278 return 0;
8279err:
54a91f3b 8280 io_finish_async(ctx);
2b188cc1
JA
8281 return ret;
8282}
8283
7e84e1c7
SG
8284static void io_sq_offload_start(struct io_ring_ctx *ctx)
8285{
534ca6d6
JA
8286 struct io_sq_data *sqd = ctx->sq_data;
8287
8288 if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd->thread)
8289 wake_up_process(sqd->thread);
7e84e1c7
SG
8290}
8291
a087e2b5
BM
8292static inline void __io_unaccount_mem(struct user_struct *user,
8293 unsigned long nr_pages)
2b188cc1
JA
8294{
8295 atomic_long_sub(nr_pages, &user->locked_vm);
8296}
8297
a087e2b5
BM
8298static inline int __io_account_mem(struct user_struct *user,
8299 unsigned long nr_pages)
2b188cc1
JA
8300{
8301 unsigned long page_limit, cur_pages, new_pages;
8302
8303 /* Don't allow more pages than we can safely lock */
8304 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8305
8306 do {
8307 cur_pages = atomic_long_read(&user->locked_vm);
8308 new_pages = cur_pages + nr_pages;
8309 if (new_pages > page_limit)
8310 return -ENOMEM;
8311 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8312 new_pages) != cur_pages);
8313
8314 return 0;
8315}
8316
2e0464d4
BM
8317static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
8318 enum io_mem_account acct)
a087e2b5 8319{
aad5d8da 8320 if (ctx->limit_mem)
a087e2b5 8321 __io_unaccount_mem(ctx->user, nr_pages);
30975825 8322
2aede0e4 8323 if (ctx->mm_account) {
4bc4a912
JA
8324 if (acct == ACCT_LOCKED) {
8325 mmap_write_lock(ctx->mm_account);
2aede0e4 8326 ctx->mm_account->locked_vm -= nr_pages;
4bc4a912
JA
8327 mmap_write_unlock(ctx->mm_account);
8328 }else if (acct == ACCT_PINNED) {
2aede0e4 8329 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
4bc4a912 8330 }
2e0464d4 8331 }
a087e2b5
BM
8332}
8333
2e0464d4
BM
8334static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
8335 enum io_mem_account acct)
a087e2b5 8336{
30975825
BM
8337 int ret;
8338
8339 if (ctx->limit_mem) {
8340 ret = __io_account_mem(ctx->user, nr_pages);
8341 if (ret)
8342 return ret;
8343 }
8344
2aede0e4 8345 if (ctx->mm_account) {
4bc4a912
JA
8346 if (acct == ACCT_LOCKED) {
8347 mmap_write_lock(ctx->mm_account);
2aede0e4 8348 ctx->mm_account->locked_vm += nr_pages;
4bc4a912
JA
8349 mmap_write_unlock(ctx->mm_account);
8350 } else if (acct == ACCT_PINNED) {
2aede0e4 8351 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
4bc4a912 8352 }
2e0464d4 8353 }
a087e2b5
BM
8354
8355 return 0;
8356}
8357
2b188cc1
JA
8358static void io_mem_free(void *ptr)
8359{
52e04ef4
MR
8360 struct page *page;
8361
8362 if (!ptr)
8363 return;
2b188cc1 8364
52e04ef4 8365 page = virt_to_head_page(ptr);
2b188cc1
JA
8366 if (put_page_testzero(page))
8367 free_compound_page(page);
8368}
8369
8370static void *io_mem_alloc(size_t size)
8371{
8372 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
8373 __GFP_NORETRY;
8374
8375 return (void *) __get_free_pages(gfp_flags, get_order(size));
8376}
8377
75b28aff
HV
8378static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8379 size_t *sq_offset)
8380{
8381 struct io_rings *rings;
8382 size_t off, sq_array_size;
8383
8384 off = struct_size(rings, cqes, cq_entries);
8385 if (off == SIZE_MAX)
8386 return SIZE_MAX;
8387
8388#ifdef CONFIG_SMP
8389 off = ALIGN(off, SMP_CACHE_BYTES);
8390 if (off == 0)
8391 return SIZE_MAX;
8392#endif
8393
b36200f5
DV
8394 if (sq_offset)
8395 *sq_offset = off;
8396
75b28aff
HV
8397 sq_array_size = array_size(sizeof(u32), sq_entries);
8398 if (sq_array_size == SIZE_MAX)
8399 return SIZE_MAX;
8400
8401 if (check_add_overflow(off, sq_array_size, &off))
8402 return SIZE_MAX;
8403
75b28aff
HV
8404 return off;
8405}
8406
2b188cc1
JA
8407static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
8408{
75b28aff 8409 size_t pages;
2b188cc1 8410
75b28aff
HV
8411 pages = (size_t)1 << get_order(
8412 rings_size(sq_entries, cq_entries, NULL));
8413 pages += (size_t)1 << get_order(
8414 array_size(sizeof(struct io_uring_sqe), sq_entries));
2b188cc1 8415
75b28aff 8416 return pages;
2b188cc1
JA
8417}
8418
0a96bbe4 8419static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
edafccee
JA
8420{
8421 int i, j;
8422
8423 if (!ctx->user_bufs)
8424 return -ENXIO;
8425
8426 for (i = 0; i < ctx->nr_user_bufs; i++) {
8427 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
8428
8429 for (j = 0; j < imu->nr_bvecs; j++)
f1f6a7dd 8430 unpin_user_page(imu->bvec[j].bv_page);
edafccee 8431
de293938
JA
8432 if (imu->acct_pages)
8433 io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED);
d4ef6475 8434 kvfree(imu->bvec);
edafccee
JA
8435 imu->nr_bvecs = 0;
8436 }
8437
8438 kfree(ctx->user_bufs);
8439 ctx->user_bufs = NULL;
8440 ctx->nr_user_bufs = 0;
8441 return 0;
8442}
8443
8444static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8445 void __user *arg, unsigned index)
8446{
8447 struct iovec __user *src;
8448
8449#ifdef CONFIG_COMPAT
8450 if (ctx->compat) {
8451 struct compat_iovec __user *ciovs;
8452 struct compat_iovec ciov;
8453
8454 ciovs = (struct compat_iovec __user *) arg;
8455 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8456 return -EFAULT;
8457
d55e5f5b 8458 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
edafccee
JA
8459 dst->iov_len = ciov.iov_len;
8460 return 0;
8461 }
8462#endif
8463 src = (struct iovec __user *) arg;
8464 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8465 return -EFAULT;
8466 return 0;
8467}
8468
de293938
JA
8469/*
8470 * Not super efficient, but this is just a registration time. And we do cache
8471 * the last compound head, so generally we'll only do a full search if we don't
8472 * match that one.
8473 *
8474 * We check if the given compound head page has already been accounted, to
8475 * avoid double accounting it. This allows us to account the full size of the
8476 * page, not just the constituent pages of a huge page.
8477 */
8478static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8479 int nr_pages, struct page *hpage)
8480{
8481 int i, j;
8482
8483 /* check current page array */
8484 for (i = 0; i < nr_pages; i++) {
8485 if (!PageCompound(pages[i]))
8486 continue;
8487 if (compound_head(pages[i]) == hpage)
8488 return true;
8489 }
8490
8491 /* check previously registered pages */
8492 for (i = 0; i < ctx->nr_user_bufs; i++) {
8493 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
8494
8495 for (j = 0; j < imu->nr_bvecs; j++) {
8496 if (!PageCompound(imu->bvec[j].bv_page))
8497 continue;
8498 if (compound_head(imu->bvec[j].bv_page) == hpage)
8499 return true;
8500 }
8501 }
8502
8503 return false;
8504}
8505
8506static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8507 int nr_pages, struct io_mapped_ubuf *imu,
8508 struct page **last_hpage)
8509{
8510 int i, ret;
8511
8512 for (i = 0; i < nr_pages; i++) {
8513 if (!PageCompound(pages[i])) {
8514 imu->acct_pages++;
8515 } else {
8516 struct page *hpage;
8517
8518 hpage = compound_head(pages[i]);
8519 if (hpage == *last_hpage)
8520 continue;
8521 *last_hpage = hpage;
8522 if (headpage_already_acct(ctx, pages, i, hpage))
8523 continue;
8524 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8525 }
8526 }
8527
8528 if (!imu->acct_pages)
8529 return 0;
8530
8531 ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED);
8532 if (ret)
8533 imu->acct_pages = 0;
8534 return ret;
8535}
8536
0a96bbe4
BM
8537static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
8538 struct io_mapped_ubuf *imu,
8539 struct page **last_hpage)
edafccee
JA
8540{
8541 struct vm_area_struct **vmas = NULL;
8542 struct page **pages = NULL;
0a96bbe4
BM
8543 unsigned long off, start, end, ubuf;
8544 size_t size;
8545 int ret, pret, nr_pages, i;
8546
8547 ubuf = (unsigned long) iov->iov_base;
8548 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8549 start = ubuf >> PAGE_SHIFT;
8550 nr_pages = end - start;
8551
8552 ret = -ENOMEM;
8553
8554 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8555 if (!pages)
8556 goto done;
8557
8558 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8559 GFP_KERNEL);
8560 if (!vmas)
8561 goto done;
8562
8563 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
8564 GFP_KERNEL);
8565 if (!imu->bvec)
8566 goto done;
8567
8568 ret = 0;
8569 mmap_read_lock(current->mm);
8570 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8571 pages, vmas);
8572 if (pret == nr_pages) {
8573 /* don't support file backed memory */
8574 for (i = 0; i < nr_pages; i++) {
8575 struct vm_area_struct *vma = vmas[i];
8576
8577 if (vma->vm_file &&
8578 !is_file_hugepages(vma->vm_file)) {
8579 ret = -EOPNOTSUPP;
8580 break;
8581 }
8582 }
8583 } else {
8584 ret = pret < 0 ? pret : -EFAULT;
8585 }
8586 mmap_read_unlock(current->mm);
8587 if (ret) {
8588 /*
8589 * if we did partial map, or found file backed vmas,
8590 * release any pages we did get
8591 */
8592 if (pret > 0)
8593 unpin_user_pages(pages, pret);
8594 kvfree(imu->bvec);
8595 goto done;
8596 }
8597
8598 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8599 if (ret) {
8600 unpin_user_pages(pages, pret);
8601 kvfree(imu->bvec);
8602 goto done;
8603 }
8604
8605 off = ubuf & ~PAGE_MASK;
8606 size = iov->iov_len;
8607 for (i = 0; i < nr_pages; i++) {
8608 size_t vec_len;
8609
8610 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8611 imu->bvec[i].bv_page = pages[i];
8612 imu->bvec[i].bv_len = vec_len;
8613 imu->bvec[i].bv_offset = off;
8614 off = 0;
8615 size -= vec_len;
8616 }
8617 /* store original address for later verification */
8618 imu->ubuf = ubuf;
8619 imu->len = iov->iov_len;
8620 imu->nr_bvecs = nr_pages;
8621 ret = 0;
8622done:
8623 kvfree(pages);
8624 kvfree(vmas);
8625 return ret;
8626}
8627
2b358604 8628static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
0a96bbe4 8629{
edafccee
JA
8630 if (ctx->user_bufs)
8631 return -EBUSY;
8632 if (!nr_args || nr_args > UIO_MAXIOV)
8633 return -EINVAL;
8634
8635 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
8636 GFP_KERNEL);
8637 if (!ctx->user_bufs)
8638 return -ENOMEM;
8639
2b358604
BM
8640 return 0;
8641}
8642
8643static int io_buffer_validate(struct iovec *iov)
8644{
8645 /*
8646 * Don't impose further limits on the size and buffer
8647 * constraints here, we'll -EINVAL later when IO is
8648 * submitted if they are wrong.
8649 */
8650 if (!iov->iov_base || !iov->iov_len)
8651 return -EFAULT;
8652
8653 /* arbitrary limit, but we need something */
8654 if (iov->iov_len > SZ_1G)
8655 return -EFAULT;
8656
8657 return 0;
8658}
8659
8660static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
8661 unsigned int nr_args)
8662{
8663 int i, ret;
8664 struct iovec iov;
8665 struct page *last_hpage = NULL;
8666
8667 ret = io_buffers_map_alloc(ctx, nr_args);
8668 if (ret)
8669 return ret;
8670
edafccee
JA
8671 for (i = 0; i < nr_args; i++) {
8672 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
edafccee
JA
8673
8674 ret = io_copy_iov(ctx, &iov, arg, i);
8675 if (ret)
0a96bbe4 8676 break;
edafccee 8677
2b358604
BM
8678 ret = io_buffer_validate(&iov);
8679 if (ret)
0a96bbe4 8680 break;
edafccee 8681
0a96bbe4
BM
8682 ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage);
8683 if (ret)
8684 break;
edafccee
JA
8685
8686 ctx->nr_user_bufs++;
8687 }
0a96bbe4
BM
8688
8689 if (ret)
8690 io_sqe_buffers_unregister(ctx);
8691
edafccee
JA
8692 return ret;
8693}
8694
9b402849
JA
8695static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8696{
8697 __s32 __user *fds = arg;
8698 int fd;
8699
8700 if (ctx->cq_ev_fd)
8701 return -EBUSY;
8702
8703 if (copy_from_user(&fd, fds, sizeof(*fds)))
8704 return -EFAULT;
8705
8706 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8707 if (IS_ERR(ctx->cq_ev_fd)) {
8708 int ret = PTR_ERR(ctx->cq_ev_fd);
8709 ctx->cq_ev_fd = NULL;
8710 return ret;
8711 }
8712
8713 return 0;
8714}
8715
8716static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8717{
8718 if (ctx->cq_ev_fd) {
8719 eventfd_ctx_put(ctx->cq_ev_fd);
8720 ctx->cq_ev_fd = NULL;
8721 return 0;
8722 }
8723
8724 return -ENXIO;
8725}
8726
5a2e745d
JA
8727static int __io_destroy_buffers(int id, void *p, void *data)
8728{
8729 struct io_ring_ctx *ctx = data;
8730 struct io_buffer *buf = p;
8731
067524e9 8732 __io_remove_buffers(ctx, buf, id, -1U);
5a2e745d
JA
8733 return 0;
8734}
8735
8736static void io_destroy_buffers(struct io_ring_ctx *ctx)
8737{
8738 idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
8739 idr_destroy(&ctx->io_buffer_idr);
8740}
8741
1b4c351f
JA
8742static void io_req_cache_free(struct io_ring_ctx *ctx)
8743{
8744 struct io_comp_state *cs = &ctx->submit_state.comp;
8745
8746 while (!list_empty(&cs->free_list)) {
8747 struct io_kiocb *req;
8748
8749 req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list);
8750 list_del(&req->compl.list);
8751 kmem_cache_free(req_cachep, req);
8752 }
8753}
8754
2b188cc1
JA
8755static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8756{
bf019da7
PB
8757 struct io_submit_state *submit_state = &ctx->submit_state;
8758
6b06314c 8759 io_finish_async(ctx);
0a96bbe4 8760 io_sqe_buffers_unregister(ctx);
2aede0e4
JA
8761
8762 if (ctx->sqo_task) {
8763 put_task_struct(ctx->sqo_task);
8764 ctx->sqo_task = NULL;
8765 mmdrop(ctx->mm_account);
8766 ctx->mm_account = NULL;
30975825 8767 }
def596e9 8768
bf019da7
PB
8769 if (submit_state->free_reqs)
8770 kmem_cache_free_bulk(req_cachep, submit_state->free_reqs,
8771 submit_state->reqs);
8772
91d8f519
DZ
8773#ifdef CONFIG_BLK_CGROUP
8774 if (ctx->sqo_blkcg_css)
8775 css_put(ctx->sqo_blkcg_css);
8776#endif
8777
6b06314c 8778 io_sqe_files_unregister(ctx);
9b402849 8779 io_eventfd_unregister(ctx);
5a2e745d 8780 io_destroy_buffers(ctx);
41726c9a 8781 idr_destroy(&ctx->personality_idr);
def596e9 8782
2b188cc1 8783#if defined(CONFIG_UNIX)
355e8d26
EB
8784 if (ctx->ring_sock) {
8785 ctx->ring_sock->file = NULL; /* so that iput() is called */
2b188cc1 8786 sock_release(ctx->ring_sock);
355e8d26 8787 }
2b188cc1
JA
8788#endif
8789
75b28aff 8790 io_mem_free(ctx->rings);
2b188cc1 8791 io_mem_free(ctx->sq_sqes);
2b188cc1
JA
8792
8793 percpu_ref_exit(&ctx->refs);
2b188cc1 8794 free_uid(ctx->user);
181e448d 8795 put_cred(ctx->creds);
78076bb6 8796 kfree(ctx->cancel_hash);
1b4c351f 8797 io_req_cache_free(ctx);
2b188cc1
JA
8798 kfree(ctx);
8799}
8800
8801static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8802{
8803 struct io_ring_ctx *ctx = file->private_data;
8804 __poll_t mask = 0;
8805
8806 poll_wait(file, &ctx->cq_wait, wait);
4f7067c3
SB
8807 /*
8808 * synchronizes with barrier from wq_has_sleeper call in
8809 * io_commit_cqring
8810 */
2b188cc1 8811 smp_rmb();
90554200 8812 if (!io_sqring_full(ctx))
2b188cc1 8813 mask |= EPOLLOUT | EPOLLWRNORM;
6c503150
PB
8814 io_cqring_overflow_flush(ctx, false, NULL, NULL);
8815 if (io_cqring_events(ctx))
2b188cc1
JA
8816 mask |= EPOLLIN | EPOLLRDNORM;
8817
8818 return mask;
8819}
8820
8821static int io_uring_fasync(int fd, struct file *file, int on)
8822{
8823 struct io_ring_ctx *ctx = file->private_data;
8824
8825 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8826}
8827
0bead8cd 8828static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
071698e1 8829{
1e6fa521 8830 struct io_identity *iod;
071698e1 8831
1e6fa521
JA
8832 iod = idr_remove(&ctx->personality_idr, id);
8833 if (iod) {
8834 put_cred(iod->creds);
8835 if (refcount_dec_and_test(&iod->count))
8836 kfree(iod);
0bead8cd 8837 return 0;
1e6fa521 8838 }
0bead8cd
YD
8839
8840 return -EINVAL;
8841}
8842
8843static int io_remove_personalities(int id, void *p, void *data)
8844{
8845 struct io_ring_ctx *ctx = data;
8846
8847 io_unregister_personality(ctx, id);
071698e1
JA
8848 return 0;
8849}
8850
85faa7b8
JA
8851static void io_ring_exit_work(struct work_struct *work)
8852{
b2edc0a7
PB
8853 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
8854 exit_work);
85faa7b8 8855
56952e91
JA
8856 /*
8857 * If we're doing polled IO and end up having requests being
8858 * submitted async (out-of-line), then completions can come in while
8859 * we're waiting for refs to drop. We need to reap these manually,
8860 * as nobody else will be looking for them.
8861 */
b2edc0a7 8862 do {
9936c7c2 8863 io_uring_try_cancel_requests(ctx, NULL, NULL);
b2edc0a7 8864 } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
85faa7b8
JA
8865 io_ring_ctx_free(ctx);
8866}
8867
00c18640
JA
8868static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8869{
8870 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8871
8872 return req->ctx == data;
8873}
8874
2b188cc1
JA
8875static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8876{
8877 mutex_lock(&ctx->uring_lock);
8878 percpu_ref_kill(&ctx->refs);
d9d05217
PB
8879
8880 if (WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && !ctx->sqo_dead))
8881 ctx->sqo_dead = 1;
8882
cda286f0
PB
8883 /* if force is set, the ring is going away. always drop after that */
8884 ctx->cq_overflow_flushed = 1;
634578f8 8885 if (ctx->rings)
6c503150 8886 __io_cqring_overflow_flush(ctx, true, NULL, NULL);
5c766a90 8887 idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
2b188cc1
JA
8888 mutex_unlock(&ctx->uring_lock);
8889
6b81928d
PB
8890 io_kill_timeouts(ctx, NULL, NULL);
8891 io_poll_remove_all(ctx, NULL, NULL);
561fb04a
JA
8892
8893 if (ctx->io_wq)
00c18640 8894 io_wq_cancel_cb(ctx->io_wq, io_cancel_ctx_cb, ctx, true);
561fb04a 8895
15dff286 8896 /* if we failed setting up the ctx, we might not have any rings */
b2edc0a7 8897 io_iopoll_try_reap_events(ctx);
309fc03a
JA
8898
8899 /*
8900 * Do this upfront, so we won't have a grace period where the ring
8901 * is closed but resources aren't reaped yet. This can cause
8902 * spurious failure in setting up a new ring.
8903 */
760618f7
JA
8904 io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
8905 ACCT_LOCKED);
309fc03a 8906
85faa7b8 8907 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
fc666777
JA
8908 /*
8909 * Use system_unbound_wq to avoid spawning tons of event kworkers
8910 * if we're exiting a ton of rings at the same time. It just adds
8911 * noise and overhead, there's no discernable change in runtime
8912 * over using system_wq.
8913 */
8914 queue_work(system_unbound_wq, &ctx->exit_work);
2b188cc1
JA
8915}
8916
8917static int io_uring_release(struct inode *inode, struct file *file)
8918{
8919 struct io_ring_ctx *ctx = file->private_data;
8920
8921 file->private_data = NULL;
8922 io_ring_ctx_wait_and_kill(ctx);
8923 return 0;
8924}
8925
f6edbabb
PB
8926struct io_task_cancel {
8927 struct task_struct *task;
8928 struct files_struct *files;
8929};
f254ac04 8930
f6edbabb 8931static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
b711d4ea 8932{
9a472ef7 8933 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
f6edbabb 8934 struct io_task_cancel *cancel = data;
9a472ef7
PB
8935 bool ret;
8936
f6edbabb 8937 if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) {
9a472ef7
PB
8938 unsigned long flags;
8939 struct io_ring_ctx *ctx = req->ctx;
8940
8941 /* protect against races with linked timeouts */
8942 spin_lock_irqsave(&ctx->completion_lock, flags);
f6edbabb 8943 ret = io_match_task(req, cancel->task, cancel->files);
9a472ef7
PB
8944 spin_unlock_irqrestore(&ctx->completion_lock, flags);
8945 } else {
f6edbabb 8946 ret = io_match_task(req, cancel->task, cancel->files);
9a472ef7
PB
8947 }
8948 return ret;
b711d4ea
JA
8949}
8950
b7ddce3c 8951static void io_cancel_defer_files(struct io_ring_ctx *ctx,
ef9865a4 8952 struct task_struct *task,
b7ddce3c
PB
8953 struct files_struct *files)
8954{
8955 struct io_defer_entry *de = NULL;
8956 LIST_HEAD(list);
8957
8958 spin_lock_irq(&ctx->completion_lock);
8959 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
08d23634 8960 if (io_match_task(de->req, task, files)) {
b7ddce3c
PB
8961 list_cut_position(&list, &ctx->defer_list, &de->list);
8962 break;
8963 }
8964 }
8965 spin_unlock_irq(&ctx->completion_lock);
8966
8967 while (!list_empty(&list)) {
8968 de = list_first_entry(&list, struct io_defer_entry, list);
8969 list_del_init(&de->list);
8970 req_set_fail_links(de->req);
8971 io_put_req(de->req);
8972 io_req_complete(de->req, -ECANCELED);
8973 kfree(de);
8974 }
8975}
8976
9936c7c2
PB
8977static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
8978 struct task_struct *task,
8979 struct files_struct *files)
8980{
8981 struct io_task_cancel cancel = { .task = task, .files = files, };
8982
8983 while (1) {
8984 enum io_wq_cancel cret;
8985 bool ret = false;
8986
8987 if (ctx->io_wq) {
8988 cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb,
8989 &cancel, true);
8990 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8991 }
8992
8993 /* SQPOLL thread does its own polling */
8994 if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) {
8995 while (!list_empty_careful(&ctx->iopoll_list)) {
8996 io_iopoll_try_reap_events(ctx);
8997 ret = true;
8998 }
8999 }
9000
9001 ret |= io_poll_remove_all(ctx, task, files);
9002 ret |= io_kill_timeouts(ctx, task, files);
9003 ret |= io_run_task_work();
9004 io_cqring_overflow_flush(ctx, true, task, files);
9005 if (!ret)
9006 break;
9007 cond_resched();
9008 }
9009}
9010
ca70f00b
PB
9011static int io_uring_count_inflight(struct io_ring_ctx *ctx,
9012 struct task_struct *task,
9013 struct files_struct *files)
9014{
9015 struct io_kiocb *req;
9016 int cnt = 0;
9017
9018 spin_lock_irq(&ctx->inflight_lock);
9019 list_for_each_entry(req, &ctx->inflight_list, inflight_entry)
9020 cnt += io_match_task(req, task, files);
9021 spin_unlock_irq(&ctx->inflight_lock);
9022 return cnt;
9023}
9024
b52fda00 9025static void io_uring_cancel_files(struct io_ring_ctx *ctx,
df9923f9 9026 struct task_struct *task,
fcb323cc
JA
9027 struct files_struct *files)
9028{
fcb323cc 9029 while (!list_empty_careful(&ctx->inflight_list)) {
d8f1b971 9030 DEFINE_WAIT(wait);
ca70f00b 9031 int inflight;
fcb323cc 9032
ca70f00b
PB
9033 inflight = io_uring_count_inflight(ctx, task, files);
9034 if (!inflight)
fcb323cc 9035 break;
f6edbabb 9036
9936c7c2 9037 io_uring_try_cancel_requests(ctx, task, files);
ca70f00b
PB
9038 prepare_to_wait(&task->io_uring->wait, &wait,
9039 TASK_UNINTERRUPTIBLE);
9040 if (inflight == io_uring_count_inflight(ctx, task, files))
9041 schedule();
c98de08c 9042 finish_wait(&task->io_uring->wait, &wait);
fcb323cc 9043 }
44e728b8
PB
9044}
9045
d9d05217
PB
9046static void io_disable_sqo_submit(struct io_ring_ctx *ctx)
9047{
d9d05217
PB
9048 mutex_lock(&ctx->uring_lock);
9049 ctx->sqo_dead = 1;
9050 mutex_unlock(&ctx->uring_lock);
9051
9052 /* make sure callers enter the ring to get error */
b4411616
PB
9053 if (ctx->rings)
9054 io_ring_set_wakeup_flag(ctx);
d9d05217
PB
9055}
9056
0f212204
JA
9057/*
9058 * We need to iteratively cancel requests, in case a request has dependent
9059 * hard links. These persist even for failure of cancelations, hence keep
9060 * looping until none are found.
9061 */
9062static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
9063 struct files_struct *files)
9064{
9065 struct task_struct *task = current;
9066
fdaf083c 9067 if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) {
d9d05217 9068 io_disable_sqo_submit(ctx);
534ca6d6 9069 task = ctx->sq_data->thread;
fdaf083c
JA
9070 atomic_inc(&task->io_uring->in_idle);
9071 io_sq_thread_park(ctx->sq_data);
9072 }
0f212204 9073
df9923f9 9074 io_cancel_defer_files(ctx, task, files);
0f212204 9075
3a7efd1a 9076 io_uring_cancel_files(ctx, task, files);
b52fda00 9077 if (!files)
9936c7c2 9078 io_uring_try_cancel_requests(ctx, task, NULL);
fdaf083c
JA
9079
9080 if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) {
9081 atomic_dec(&task->io_uring->in_idle);
9082 /*
9083 * If the files that are going away are the ones in the thread
9084 * identity, clear them out.
9085 */
9086 if (task->io_uring->identity->files == files)
9087 task->io_uring->identity->files = NULL;
9088 io_sq_thread_unpark(ctx->sq_data);
9089 }
0f212204
JA
9090}
9091
9092/*
9093 * Note that this task has used io_uring. We use it for cancelation purposes.
9094 */
fdaf083c 9095static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file)
0f212204 9096{
236434c3 9097 struct io_uring_task *tctx = current->io_uring;
a528b04e 9098 int ret;
236434c3
MWO
9099
9100 if (unlikely(!tctx)) {
0f212204
JA
9101 ret = io_uring_alloc_task_context(current);
9102 if (unlikely(ret))
9103 return ret;
236434c3 9104 tctx = current->io_uring;
0f212204 9105 }
236434c3
MWO
9106 if (tctx->last != file) {
9107 void *old = xa_load(&tctx->xa, (unsigned long)file);
0f212204 9108
236434c3 9109 if (!old) {
0f212204 9110 get_file(file);
a528b04e
PB
9111 ret = xa_err(xa_store(&tctx->xa, (unsigned long)file,
9112 file, GFP_KERNEL));
9113 if (ret) {
9114 fput(file);
9115 return ret;
9116 }
ecfc8492
PB
9117
9118 /* one and only SQPOLL file note, held by sqo_task */
9119 WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) &&
9120 current != ctx->sqo_task);
0f212204 9121 }
236434c3 9122 tctx->last = file;
0f212204
JA
9123 }
9124
fdaf083c
JA
9125 /*
9126 * This is race safe in that the task itself is doing this, hence it
9127 * cannot be going through the exit/cancel paths at the same time.
9128 * This cannot be modified while exit/cancel is running.
9129 */
9130 if (!tctx->sqpoll && (ctx->flags & IORING_SETUP_SQPOLL))
9131 tctx->sqpoll = true;
9132
0f212204
JA
9133 return 0;
9134}
9135
9136/*
9137 * Remove this io_uring_file -> task mapping.
9138 */
9139static void io_uring_del_task_file(struct file *file)
9140{
9141 struct io_uring_task *tctx = current->io_uring;
0f212204
JA
9142
9143 if (tctx->last == file)
9144 tctx->last = NULL;
5e2ed8c4 9145 file = xa_erase(&tctx->xa, (unsigned long)file);
0f212204
JA
9146 if (file)
9147 fput(file);
9148}
9149
de7f1d9e
PB
9150static void io_uring_remove_task_files(struct io_uring_task *tctx)
9151{
9152 struct file *file;
9153 unsigned long index;
9154
9155 xa_for_each(&tctx->xa, index, file)
9156 io_uring_del_task_file(file);
9157}
9158
0f212204
JA
9159void __io_uring_files_cancel(struct files_struct *files)
9160{
9161 struct io_uring_task *tctx = current->io_uring;
ce765372
MWO
9162 struct file *file;
9163 unsigned long index;
0f212204
JA
9164
9165 /* make sure overflow events are dropped */
fdaf083c 9166 atomic_inc(&tctx->in_idle);
de7f1d9e
PB
9167 xa_for_each(&tctx->xa, index, file)
9168 io_uring_cancel_task_requests(file->private_data, files);
fdaf083c 9169 atomic_dec(&tctx->in_idle);
de7f1d9e
PB
9170
9171 if (files)
9172 io_uring_remove_task_files(tctx);
fdaf083c
JA
9173}
9174
9175static s64 tctx_inflight(struct io_uring_task *tctx)
9176{
0e9ddb39
PB
9177 return percpu_counter_sum(&tctx->inflight);
9178}
fdaf083c 9179
0e9ddb39
PB
9180static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx)
9181{
9182 struct io_uring_task *tctx;
9183 s64 inflight;
9184 DEFINE_WAIT(wait);
fdaf083c 9185
0e9ddb39
PB
9186 if (!ctx->sq_data)
9187 return;
9188 tctx = ctx->sq_data->thread->io_uring;
9189 io_disable_sqo_submit(ctx);
fdaf083c 9190
0e9ddb39
PB
9191 atomic_inc(&tctx->in_idle);
9192 do {
9193 /* read completions before cancelations */
9194 inflight = tctx_inflight(tctx);
9195 if (!inflight)
9196 break;
9197 io_uring_cancel_task_requests(ctx, NULL);
fdaf083c 9198
0e9ddb39
PB
9199 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9200 /*
9201 * If we've seen completions, retry without waiting. This
9202 * avoids a race where a completion comes in before we did
9203 * prepare_to_wait().
9204 */
9205 if (inflight == tctx_inflight(tctx))
9206 schedule();
9207 finish_wait(&tctx->wait, &wait);
9208 } while (1);
9209 atomic_dec(&tctx->in_idle);
0f212204
JA
9210}
9211
0f212204
JA
9212/*
9213 * Find any io_uring fd that this task has registered or done IO on, and cancel
9214 * requests.
9215 */
9216void __io_uring_task_cancel(void)
9217{
9218 struct io_uring_task *tctx = current->io_uring;
9219 DEFINE_WAIT(wait);
d8a6df10 9220 s64 inflight;
0f212204
JA
9221
9222 /* make sure overflow events are dropped */
fdaf083c 9223 atomic_inc(&tctx->in_idle);
0f212204 9224
0b5cd6c3 9225 /* trigger io_disable_sqo_submit() */
0e9ddb39
PB
9226 if (tctx->sqpoll) {
9227 struct file *file;
9228 unsigned long index;
9229
9230 xa_for_each(&tctx->xa, index, file)
9231 io_uring_cancel_sqpoll(file->private_data);
9232 }
0b5cd6c3 9233
d8a6df10 9234 do {
0f212204 9235 /* read completions before cancelations */
fdaf083c 9236 inflight = tctx_inflight(tctx);
d8a6df10
JA
9237 if (!inflight)
9238 break;
0f212204
JA
9239 __io_uring_files_cancel(NULL);
9240
9241 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9242
9243 /*
a1bb3cd5
PB
9244 * If we've seen completions, retry without waiting. This
9245 * avoids a race where a completion comes in before we did
9246 * prepare_to_wait().
0f212204 9247 */
a1bb3cd5
PB
9248 if (inflight == tctx_inflight(tctx))
9249 schedule();
f57555ed 9250 finish_wait(&tctx->wait, &wait);
d8a6df10 9251 } while (1);
0f212204 9252
fdaf083c 9253 atomic_dec(&tctx->in_idle);
de7f1d9e
PB
9254
9255 io_uring_remove_task_files(tctx);
44e728b8
PB
9256}
9257
fcb323cc
JA
9258static int io_uring_flush(struct file *file, void *data)
9259{
6b5733eb 9260 struct io_uring_task *tctx = current->io_uring;
d9d05217 9261 struct io_ring_ctx *ctx = file->private_data;
6b5733eb 9262
84965ff8
JA
9263 if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
9264 io_uring_cancel_task_requests(ctx, NULL);
9265
6b5733eb 9266 if (!tctx)
4f793dc4
PB
9267 return 0;
9268
6b5733eb
PB
9269 /* we should have cancelled and erased it before PF_EXITING */
9270 WARN_ON_ONCE((current->flags & PF_EXITING) &&
9271 xa_load(&tctx->xa, (unsigned long)file));
9272
4f793dc4
PB
9273 /*
9274 * fput() is pending, will be 2 if the only other ref is our potential
9275 * task file note. If the task is exiting, drop regardless of count.
9276 */
6b5733eb
PB
9277 if (atomic_long_read(&file->f_count) != 2)
9278 return 0;
4f793dc4 9279
d9d05217
PB
9280 if (ctx->flags & IORING_SETUP_SQPOLL) {
9281 /* there is only one file note, which is owned by sqo_task */
4325cb49
PB
9282 WARN_ON_ONCE(ctx->sqo_task != current &&
9283 xa_load(&tctx->xa, (unsigned long)file));
9284 /* sqo_dead check is for when this happens after cancellation */
9285 WARN_ON_ONCE(ctx->sqo_task == current && !ctx->sqo_dead &&
d9d05217
PB
9286 !xa_load(&tctx->xa, (unsigned long)file));
9287
9288 io_disable_sqo_submit(ctx);
9289 }
9290
9291 if (!(ctx->flags & IORING_SETUP_SQPOLL) || ctx->sqo_task == current)
9292 io_uring_del_task_file(file);
fcb323cc
JA
9293 return 0;
9294}
9295
6c5c240e
RP
9296static void *io_uring_validate_mmap_request(struct file *file,
9297 loff_t pgoff, size_t sz)
2b188cc1 9298{
2b188cc1 9299 struct io_ring_ctx *ctx = file->private_data;
6c5c240e 9300 loff_t offset = pgoff << PAGE_SHIFT;
2b188cc1
JA
9301 struct page *page;
9302 void *ptr;
9303
9304 switch (offset) {
9305 case IORING_OFF_SQ_RING:
75b28aff
HV
9306 case IORING_OFF_CQ_RING:
9307 ptr = ctx->rings;
2b188cc1
JA
9308 break;
9309 case IORING_OFF_SQES:
9310 ptr = ctx->sq_sqes;
9311 break;
2b188cc1 9312 default:
6c5c240e 9313 return ERR_PTR(-EINVAL);
2b188cc1
JA
9314 }
9315
9316 page = virt_to_head_page(ptr);
a50b854e 9317 if (sz > page_size(page))
6c5c240e
RP
9318 return ERR_PTR(-EINVAL);
9319
9320 return ptr;
9321}
9322
9323#ifdef CONFIG_MMU
9324
9325static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9326{
9327 size_t sz = vma->vm_end - vma->vm_start;
9328 unsigned long pfn;
9329 void *ptr;
9330
9331 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9332 if (IS_ERR(ptr))
9333 return PTR_ERR(ptr);
2b188cc1
JA
9334
9335 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9336 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9337}
9338
6c5c240e
RP
9339#else /* !CONFIG_MMU */
9340
9341static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9342{
9343 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9344}
9345
9346static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9347{
9348 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9349}
9350
9351static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9352 unsigned long addr, unsigned long len,
9353 unsigned long pgoff, unsigned long flags)
9354{
9355 void *ptr;
9356
9357 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9358 if (IS_ERR(ptr))
9359 return PTR_ERR(ptr);
9360
9361 return (unsigned long) ptr;
9362}
9363
9364#endif /* !CONFIG_MMU */
9365
d9d05217 9366static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
90554200 9367{
d9d05217 9368 int ret = 0;
90554200
JA
9369 DEFINE_WAIT(wait);
9370
9371 do {
9372 if (!io_sqring_full(ctx))
9373 break;
9374
9375 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9376
d9d05217
PB
9377 if (unlikely(ctx->sqo_dead)) {
9378 ret = -EOWNERDEAD;
9379 goto out;
9380 }
9381
90554200
JA
9382 if (!io_sqring_full(ctx))
9383 break;
9384
9385 schedule();
9386 } while (!signal_pending(current));
9387
9388 finish_wait(&ctx->sqo_sq_wait, &wait);
d9d05217
PB
9389out:
9390 return ret;
90554200
JA
9391}
9392
c73ebb68
HX
9393static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9394 struct __kernel_timespec __user **ts,
9395 const sigset_t __user **sig)
9396{
9397 struct io_uring_getevents_arg arg;
9398
9399 /*
9400 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9401 * is just a pointer to the sigset_t.
9402 */
9403 if (!(flags & IORING_ENTER_EXT_ARG)) {
9404 *sig = (const sigset_t __user *) argp;
9405 *ts = NULL;
9406 return 0;
9407 }
9408
9409 /*
9410 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9411 * timespec and sigset_t pointers if good.
9412 */
9413 if (*argsz != sizeof(arg))
9414 return -EINVAL;
9415 if (copy_from_user(&arg, argp, sizeof(arg)))
9416 return -EFAULT;
9417 *sig = u64_to_user_ptr(arg.sigmask);
9418 *argsz = arg.sigmask_sz;
9419 *ts = u64_to_user_ptr(arg.ts);
9420 return 0;
9421}
9422
2b188cc1 9423SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
c73ebb68
HX
9424 u32, min_complete, u32, flags, const void __user *, argp,
9425 size_t, argsz)
2b188cc1
JA
9426{
9427 struct io_ring_ctx *ctx;
9428 long ret = -EBADF;
9429 int submitted = 0;
9430 struct fd f;
9431
4c6e277c 9432 io_run_task_work();
b41e9852 9433
90554200 9434 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
c73ebb68 9435 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG))
2b188cc1
JA
9436 return -EINVAL;
9437
9438 f = fdget(fd);
9439 if (!f.file)
9440 return -EBADF;
9441
9442 ret = -EOPNOTSUPP;
9443 if (f.file->f_op != &io_uring_fops)
9444 goto out_fput;
9445
9446 ret = -ENXIO;
9447 ctx = f.file->private_data;
9448 if (!percpu_ref_tryget(&ctx->refs))
9449 goto out_fput;
9450
7e84e1c7
SG
9451 ret = -EBADFD;
9452 if (ctx->flags & IORING_SETUP_R_DISABLED)
9453 goto out;
9454
6c271ce2
JA
9455 /*
9456 * For SQ polling, the thread will do all submissions and completions.
9457 * Just return the requested submit count, and wake the thread if
9458 * we were asked to.
9459 */
b2a9eada 9460 ret = 0;
6c271ce2 9461 if (ctx->flags & IORING_SETUP_SQPOLL) {
6c503150 9462 io_cqring_overflow_flush(ctx, false, NULL, NULL);
89448c47 9463
d9d05217
PB
9464 ret = -EOWNERDEAD;
9465 if (unlikely(ctx->sqo_dead))
9466 goto out;
6c271ce2 9467 if (flags & IORING_ENTER_SQ_WAKEUP)
534ca6d6 9468 wake_up(&ctx->sq_data->wait);
d9d05217
PB
9469 if (flags & IORING_ENTER_SQ_WAIT) {
9470 ret = io_sqpoll_wait_sq(ctx);
9471 if (ret)
9472 goto out;
9473 }
6c271ce2 9474 submitted = to_submit;
b2a9eada 9475 } else if (to_submit) {
fdaf083c 9476 ret = io_uring_add_task_file(ctx, f.file);
0f212204
JA
9477 if (unlikely(ret))
9478 goto out;
2b188cc1 9479 mutex_lock(&ctx->uring_lock);
0f212204 9480 submitted = io_submit_sqes(ctx, to_submit);
2b188cc1 9481 mutex_unlock(&ctx->uring_lock);
7c504e65
PB
9482
9483 if (submitted != to_submit)
9484 goto out;
2b188cc1
JA
9485 }
9486 if (flags & IORING_ENTER_GETEVENTS) {
c73ebb68
HX
9487 const sigset_t __user *sig;
9488 struct __kernel_timespec __user *ts;
9489
9490 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9491 if (unlikely(ret))
9492 goto out;
9493
2b188cc1
JA
9494 min_complete = min(min_complete, ctx->cq_entries);
9495
32b2244a
XW
9496 /*
9497 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9498 * space applications don't need to do io completion events
9499 * polling again, they can rely on io_sq_thread to do polling
9500 * work, which can reduce cpu usage and uring_lock contention.
9501 */
9502 if (ctx->flags & IORING_SETUP_IOPOLL &&
9503 !(ctx->flags & IORING_SETUP_SQPOLL)) {
7668b92a 9504 ret = io_iopoll_check(ctx, min_complete);
def596e9 9505 } else {
c73ebb68 9506 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
def596e9 9507 }
2b188cc1
JA
9508 }
9509
7c504e65 9510out:
6805b32e 9511 percpu_ref_put(&ctx->refs);
2b188cc1
JA
9512out_fput:
9513 fdput(f);
9514 return submitted ? submitted : ret;
9515}
9516
bebdb65e 9517#ifdef CONFIG_PROC_FS
87ce955b
JA
9518static int io_uring_show_cred(int id, void *p, void *data)
9519{
6b47ab81
JA
9520 struct io_identity *iod = p;
9521 const struct cred *cred = iod->creds;
87ce955b
JA
9522 struct seq_file *m = data;
9523 struct user_namespace *uns = seq_user_ns(m);
9524 struct group_info *gi;
9525 kernel_cap_t cap;
9526 unsigned __capi;
9527 int g;
9528
9529 seq_printf(m, "%5d\n", id);
9530 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9531 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9532 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9533 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9534 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9535 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9536 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9537 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9538 seq_puts(m, "\n\tGroups:\t");
9539 gi = cred->group_info;
9540 for (g = 0; g < gi->ngroups; g++) {
9541 seq_put_decimal_ull(m, g ? " " : "",
9542 from_kgid_munged(uns, gi->gid[g]));
9543 }
9544 seq_puts(m, "\n\tCapEff:\t");
9545 cap = cred->cap_effective;
9546 CAP_FOR_EACH_U32(__capi)
9547 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9548 seq_putc(m, '\n');
9549 return 0;
9550}
9551
9552static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9553{
dbbe9c64 9554 struct io_sq_data *sq = NULL;
fad8e0de 9555 bool has_lock;
87ce955b
JA
9556 int i;
9557
fad8e0de
JA
9558 /*
9559 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9560 * since fdinfo case grabs it in the opposite direction of normal use
9561 * cases. If we fail to get the lock, we just don't iterate any
9562 * structures that could be going away outside the io_uring mutex.
9563 */
9564 has_lock = mutex_trylock(&ctx->uring_lock);
9565
dbbe9c64
JQ
9566 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL))
9567 sq = ctx->sq_data;
9568
9569 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9570 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
87ce955b 9571 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
fad8e0de 9572 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
ea64ec02 9573 struct file *f = *io_fixed_file_slot(ctx->file_data, i);
87ce955b 9574
87ce955b
JA
9575 if (f)
9576 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9577 else
9578 seq_printf(m, "%5u: <none>\n", i);
9579 }
9580 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
fad8e0de 9581 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
87ce955b
JA
9582 struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
9583
9584 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
9585 (unsigned int) buf->len);
9586 }
fad8e0de 9587 if (has_lock && !idr_is_empty(&ctx->personality_idr)) {
87ce955b
JA
9588 seq_printf(m, "Personalities:\n");
9589 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
9590 }
d7718a9d
JA
9591 seq_printf(m, "PollList:\n");
9592 spin_lock_irq(&ctx->completion_lock);
9593 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9594 struct hlist_head *list = &ctx->cancel_hash[i];
9595 struct io_kiocb *req;
9596
9597 hlist_for_each_entry(req, list, hash_node)
9598 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9599 req->task->task_works != NULL);
9600 }
9601 spin_unlock_irq(&ctx->completion_lock);
fad8e0de
JA
9602 if (has_lock)
9603 mutex_unlock(&ctx->uring_lock);
87ce955b
JA
9604}
9605
9606static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9607{
9608 struct io_ring_ctx *ctx = f->private_data;
9609
9610 if (percpu_ref_tryget(&ctx->refs)) {
9611 __io_uring_show_fdinfo(ctx, m);
9612 percpu_ref_put(&ctx->refs);
9613 }
9614}
bebdb65e 9615#endif
87ce955b 9616
2b188cc1
JA
9617static const struct file_operations io_uring_fops = {
9618 .release = io_uring_release,
fcb323cc 9619 .flush = io_uring_flush,
2b188cc1 9620 .mmap = io_uring_mmap,
6c5c240e
RP
9621#ifndef CONFIG_MMU
9622 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9623 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9624#endif
2b188cc1
JA
9625 .poll = io_uring_poll,
9626 .fasync = io_uring_fasync,
bebdb65e 9627#ifdef CONFIG_PROC_FS
87ce955b 9628 .show_fdinfo = io_uring_show_fdinfo,
bebdb65e 9629#endif
2b188cc1
JA
9630};
9631
9632static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9633 struct io_uring_params *p)
9634{
75b28aff
HV
9635 struct io_rings *rings;
9636 size_t size, sq_array_offset;
2b188cc1 9637
bd740481
JA
9638 /* make sure these are sane, as we already accounted them */
9639 ctx->sq_entries = p->sq_entries;
9640 ctx->cq_entries = p->cq_entries;
9641
75b28aff
HV
9642 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9643 if (size == SIZE_MAX)
9644 return -EOVERFLOW;
9645
9646 rings = io_mem_alloc(size);
9647 if (!rings)
2b188cc1
JA
9648 return -ENOMEM;
9649
75b28aff
HV
9650 ctx->rings = rings;
9651 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9652 rings->sq_ring_mask = p->sq_entries - 1;
9653 rings->cq_ring_mask = p->cq_entries - 1;
9654 rings->sq_ring_entries = p->sq_entries;
9655 rings->cq_ring_entries = p->cq_entries;
9656 ctx->sq_mask = rings->sq_ring_mask;
9657 ctx->cq_mask = rings->cq_ring_mask;
2b188cc1
JA
9658
9659 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
eb065d30
JA
9660 if (size == SIZE_MAX) {
9661 io_mem_free(ctx->rings);
9662 ctx->rings = NULL;
2b188cc1 9663 return -EOVERFLOW;
eb065d30 9664 }
2b188cc1
JA
9665
9666 ctx->sq_sqes = io_mem_alloc(size);
eb065d30
JA
9667 if (!ctx->sq_sqes) {
9668 io_mem_free(ctx->rings);
9669 ctx->rings = NULL;
2b188cc1 9670 return -ENOMEM;
eb065d30 9671 }
2b188cc1 9672
2b188cc1
JA
9673 return 0;
9674}
9675
9faadcc8
PB
9676static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9677{
9678 int ret, fd;
9679
9680 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9681 if (fd < 0)
9682 return fd;
9683
9684 ret = io_uring_add_task_file(ctx, file);
9685 if (ret) {
9686 put_unused_fd(fd);
9687 return ret;
9688 }
9689 fd_install(fd, file);
9690 return fd;
9691}
9692
2b188cc1
JA
9693/*
9694 * Allocate an anonymous fd, this is what constitutes the application
9695 * visible backing of an io_uring instance. The application mmaps this
9696 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9697 * we have to tie this fd to a socket for file garbage collection purposes.
9698 */
9faadcc8 9699static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
2b188cc1
JA
9700{
9701 struct file *file;
9faadcc8 9702#if defined(CONFIG_UNIX)
2b188cc1
JA
9703 int ret;
9704
2b188cc1
JA
9705 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9706 &ctx->ring_sock);
9707 if (ret)
9faadcc8 9708 return ERR_PTR(ret);
2b188cc1
JA
9709#endif
9710
2b188cc1
JA
9711 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9712 O_RDWR | O_CLOEXEC);
2b188cc1 9713#if defined(CONFIG_UNIX)
9faadcc8
PB
9714 if (IS_ERR(file)) {
9715 sock_release(ctx->ring_sock);
9716 ctx->ring_sock = NULL;
9717 } else {
9718 ctx->ring_sock->file = file;
0f212204 9719 }
2b188cc1 9720#endif
9faadcc8 9721 return file;
2b188cc1
JA
9722}
9723
7f13657d
XW
9724static int io_uring_create(unsigned entries, struct io_uring_params *p,
9725 struct io_uring_params __user *params)
2b188cc1
JA
9726{
9727 struct user_struct *user = NULL;
9728 struct io_ring_ctx *ctx;
9faadcc8 9729 struct file *file;
aad5d8da 9730 bool limit_mem;
2b188cc1
JA
9731 int ret;
9732
8110c1a6 9733 if (!entries)
2b188cc1 9734 return -EINVAL;
8110c1a6
JA
9735 if (entries > IORING_MAX_ENTRIES) {
9736 if (!(p->flags & IORING_SETUP_CLAMP))
9737 return -EINVAL;
9738 entries = IORING_MAX_ENTRIES;
9739 }
2b188cc1
JA
9740
9741 /*
9742 * Use twice as many entries for the CQ ring. It's possible for the
9743 * application to drive a higher depth than the size of the SQ ring,
9744 * since the sqes are only used at submission time. This allows for
33a107f0
JA
9745 * some flexibility in overcommitting a bit. If the application has
9746 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9747 * of CQ ring entries manually.
2b188cc1
JA
9748 */
9749 p->sq_entries = roundup_pow_of_two(entries);
33a107f0
JA
9750 if (p->flags & IORING_SETUP_CQSIZE) {
9751 /*
9752 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9753 * to a power-of-two, if it isn't already. We do NOT impose
9754 * any cq vs sq ring sizing.
9755 */
eb2667b3 9756 if (!p->cq_entries)
33a107f0 9757 return -EINVAL;
8110c1a6
JA
9758 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9759 if (!(p->flags & IORING_SETUP_CLAMP))
9760 return -EINVAL;
9761 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9762 }
eb2667b3
JQ
9763 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9764 if (p->cq_entries < p->sq_entries)
9765 return -EINVAL;
33a107f0
JA
9766 } else {
9767 p->cq_entries = 2 * p->sq_entries;
9768 }
2b188cc1
JA
9769
9770 user = get_uid(current_user());
aad5d8da 9771 limit_mem = !capable(CAP_IPC_LOCK);
2b188cc1 9772
aad5d8da 9773 if (limit_mem) {
a087e2b5 9774 ret = __io_account_mem(user,
2b188cc1
JA
9775 ring_pages(p->sq_entries, p->cq_entries));
9776 if (ret) {
9777 free_uid(user);
9778 return ret;
9779 }
9780 }
9781
9782 ctx = io_ring_ctx_alloc(p);
9783 if (!ctx) {
aad5d8da 9784 if (limit_mem)
a087e2b5 9785 __io_unaccount_mem(user, ring_pages(p->sq_entries,
2b188cc1
JA
9786 p->cq_entries));
9787 free_uid(user);
9788 return -ENOMEM;
9789 }
9790 ctx->compat = in_compat_syscall();
2b188cc1 9791 ctx->user = user;
0b8c0ec7 9792 ctx->creds = get_current_cred();
4ea33a97
JA
9793#ifdef CONFIG_AUDIT
9794 ctx->loginuid = current->loginuid;
9795 ctx->sessionid = current->sessionid;
9796#endif
2aede0e4
JA
9797 ctx->sqo_task = get_task_struct(current);
9798
9799 /*
9800 * This is just grabbed for accounting purposes. When a process exits,
9801 * the mm is exited and dropped before the files, hence we need to hang
9802 * on to this mm purely for the purposes of being able to unaccount
9803 * memory (locked/pinned vm). It's not used for anything else.
9804 */
6b7898eb 9805 mmgrab(current->mm);
2aede0e4 9806 ctx->mm_account = current->mm;
6b7898eb 9807
91d8f519
DZ
9808#ifdef CONFIG_BLK_CGROUP
9809 /*
9810 * The sq thread will belong to the original cgroup it was inited in.
9811 * If the cgroup goes offline (e.g. disabling the io controller), then
9812 * issued bios will be associated with the closest cgroup later in the
9813 * block layer.
9814 */
9815 rcu_read_lock();
9816 ctx->sqo_blkcg_css = blkcg_css();
9817 ret = css_tryget_online(ctx->sqo_blkcg_css);
9818 rcu_read_unlock();
9819 if (!ret) {
9820 /* don't init against a dying cgroup, have the user try again */
9821 ctx->sqo_blkcg_css = NULL;
9822 ret = -ENODEV;
9823 goto err;
9824 }
9825#endif
6b7898eb 9826
f74441e6
JA
9827 /*
9828 * Account memory _before_ installing the file descriptor. Once
9829 * the descriptor is installed, it can get closed at any time. Also
9830 * do this before hitting the general error path, as ring freeing
9831 * will un-account as well.
9832 */
9833 io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
9834 ACCT_LOCKED);
9835 ctx->limit_mem = limit_mem;
9836
2b188cc1
JA
9837 ret = io_allocate_scq_urings(ctx, p);
9838 if (ret)
9839 goto err;
9840
7e84e1c7 9841 ret = io_sq_offload_create(ctx, p);
2b188cc1
JA
9842 if (ret)
9843 goto err;
9844
7e84e1c7
SG
9845 if (!(p->flags & IORING_SETUP_R_DISABLED))
9846 io_sq_offload_start(ctx);
9847
2b188cc1 9848 memset(&p->sq_off, 0, sizeof(p->sq_off));
75b28aff
HV
9849 p->sq_off.head = offsetof(struct io_rings, sq.head);
9850 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9851 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9852 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9853 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9854 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9855 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
2b188cc1
JA
9856
9857 memset(&p->cq_off, 0, sizeof(p->cq_off));
75b28aff
HV
9858 p->cq_off.head = offsetof(struct io_rings, cq.head);
9859 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9860 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9861 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9862 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9863 p->cq_off.cqes = offsetof(struct io_rings, cqes);
0d9b5b3a 9864 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
ac90f249 9865
7f13657d
XW
9866 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9867 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
5769a351 9868 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
c73ebb68
HX
9869 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
9870 IORING_FEAT_EXT_ARG;
7f13657d
XW
9871
9872 if (copy_to_user(params, p, sizeof(*p))) {
9873 ret = -EFAULT;
9874 goto err;
9875 }
d1719f70 9876
9faadcc8
PB
9877 file = io_uring_get_file(ctx);
9878 if (IS_ERR(file)) {
9879 ret = PTR_ERR(file);
9880 goto err;
9881 }
9882
044c1ab3
JA
9883 /*
9884 * Install ring fd as the very last thing, so we don't risk someone
9885 * having closed it before we finish setup
9886 */
9faadcc8
PB
9887 ret = io_uring_install_fd(ctx, file);
9888 if (ret < 0) {
06585c49 9889 io_disable_sqo_submit(ctx);
9faadcc8
PB
9890 /* fput will clean it up */
9891 fput(file);
9892 return ret;
9893 }
044c1ab3 9894
c826bd7a 9895 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
2b188cc1
JA
9896 return ret;
9897err:
d9d05217 9898 io_disable_sqo_submit(ctx);
2b188cc1
JA
9899 io_ring_ctx_wait_and_kill(ctx);
9900 return ret;
9901}
9902
9903/*
9904 * Sets up an aio uring context, and returns the fd. Applications asks for a
9905 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9906 * params structure passed in.
9907 */
9908static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9909{
9910 struct io_uring_params p;
2b188cc1
JA
9911 int i;
9912
9913 if (copy_from_user(&p, params, sizeof(p)))
9914 return -EFAULT;
9915 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9916 if (p.resv[i])
9917 return -EINVAL;
9918 }
9919
6c271ce2 9920 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8110c1a6 9921 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
7e84e1c7
SG
9922 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9923 IORING_SETUP_R_DISABLED))
2b188cc1
JA
9924 return -EINVAL;
9925
7f13657d 9926 return io_uring_create(entries, &p, params);
2b188cc1
JA
9927}
9928
9929SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9930 struct io_uring_params __user *, params)
9931{
9932 return io_uring_setup(entries, params);
9933}
9934
66f4af93
JA
9935static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9936{
9937 struct io_uring_probe *p;
9938 size_t size;
9939 int i, ret;
9940
9941 size = struct_size(p, ops, nr_args);
9942 if (size == SIZE_MAX)
9943 return -EOVERFLOW;
9944 p = kzalloc(size, GFP_KERNEL);
9945 if (!p)
9946 return -ENOMEM;
9947
9948 ret = -EFAULT;
9949 if (copy_from_user(p, arg, size))
9950 goto out;
9951 ret = -EINVAL;
9952 if (memchr_inv(p, 0, size))
9953 goto out;
9954
9955 p->last_op = IORING_OP_LAST - 1;
9956 if (nr_args > IORING_OP_LAST)
9957 nr_args = IORING_OP_LAST;
9958
9959 for (i = 0; i < nr_args; i++) {
9960 p->ops[i].op = i;
9961 if (!io_op_defs[i].not_supported)
9962 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9963 }
9964 p->ops_len = i;
9965
9966 ret = 0;
9967 if (copy_to_user(arg, p, size))
9968 ret = -EFAULT;
9969out:
9970 kfree(p);
9971 return ret;
9972}
9973
071698e1
JA
9974static int io_register_personality(struct io_ring_ctx *ctx)
9975{
1e6fa521
JA
9976 struct io_identity *id;
9977 int ret;
071698e1 9978
1e6fa521
JA
9979 id = kmalloc(sizeof(*id), GFP_KERNEL);
9980 if (unlikely(!id))
9981 return -ENOMEM;
9982
9983 io_init_identity(id);
9984 id->creds = get_current_cred();
9985
9986 ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL);
9987 if (ret < 0) {
9988 put_cred(id->creds);
9989 kfree(id);
9990 }
9991 return ret;
071698e1
JA
9992}
9993
21b55dbc
SG
9994static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9995 unsigned int nr_args)
9996{
9997 struct io_uring_restriction *res;
9998 size_t size;
9999 int i, ret;
10000
7e84e1c7
SG
10001 /* Restrictions allowed only if rings started disabled */
10002 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10003 return -EBADFD;
10004
21b55dbc 10005 /* We allow only a single restrictions registration */
7e84e1c7 10006 if (ctx->restrictions.registered)
21b55dbc
SG
10007 return -EBUSY;
10008
10009 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
10010 return -EINVAL;
10011
10012 size = array_size(nr_args, sizeof(*res));
10013 if (size == SIZE_MAX)
10014 return -EOVERFLOW;
10015
10016 res = memdup_user(arg, size);
10017 if (IS_ERR(res))
10018 return PTR_ERR(res);
10019
10020 ret = 0;
10021
10022 for (i = 0; i < nr_args; i++) {
10023 switch (res[i].opcode) {
10024 case IORING_RESTRICTION_REGISTER_OP:
10025 if (res[i].register_op >= IORING_REGISTER_LAST) {
10026 ret = -EINVAL;
10027 goto out;
10028 }
10029
10030 __set_bit(res[i].register_op,
10031 ctx->restrictions.register_op);
10032 break;
10033 case IORING_RESTRICTION_SQE_OP:
10034 if (res[i].sqe_op >= IORING_OP_LAST) {
10035 ret = -EINVAL;
10036 goto out;
10037 }
10038
10039 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
10040 break;
10041 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
10042 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
10043 break;
10044 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
10045 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
10046 break;
10047 default:
10048 ret = -EINVAL;
10049 goto out;
10050 }
10051 }
10052
10053out:
10054 /* Reset all restrictions if an error happened */
10055 if (ret != 0)
10056 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
10057 else
7e84e1c7 10058 ctx->restrictions.registered = true;
21b55dbc
SG
10059
10060 kfree(res);
10061 return ret;
10062}
10063
7e84e1c7
SG
10064static int io_register_enable_rings(struct io_ring_ctx *ctx)
10065{
10066 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10067 return -EBADFD;
10068
10069 if (ctx->restrictions.registered)
10070 ctx->restricted = 1;
10071
10072 ctx->flags &= ~IORING_SETUP_R_DISABLED;
10073
10074 io_sq_offload_start(ctx);
10075
10076 return 0;
10077}
10078
071698e1
JA
10079static bool io_register_op_must_quiesce(int op)
10080{
10081 switch (op) {
10082 case IORING_UNREGISTER_FILES:
10083 case IORING_REGISTER_FILES_UPDATE:
10084 case IORING_REGISTER_PROBE:
10085 case IORING_REGISTER_PERSONALITY:
10086 case IORING_UNREGISTER_PERSONALITY:
10087 return false;
10088 default:
10089 return true;
10090 }
10091}
10092
edafccee
JA
10093static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10094 void __user *arg, unsigned nr_args)
b19062a5
JA
10095 __releases(ctx->uring_lock)
10096 __acquires(ctx->uring_lock)
edafccee
JA
10097{
10098 int ret;
10099
35fa71a0
JA
10100 /*
10101 * We're inside the ring mutex, if the ref is already dying, then
10102 * someone else killed the ctx or is already going through
10103 * io_uring_register().
10104 */
10105 if (percpu_ref_is_dying(&ctx->refs))
10106 return -ENXIO;
10107
071698e1 10108 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 10109 percpu_ref_kill(&ctx->refs);
b19062a5 10110
05f3fb3c
JA
10111 /*
10112 * Drop uring mutex before waiting for references to exit. If
10113 * another thread is currently inside io_uring_enter() it might
10114 * need to grab the uring_lock to make progress. If we hold it
10115 * here across the drain wait, then we can deadlock. It's safe
10116 * to drop the mutex here, since no new references will come in
10117 * after we've killed the percpu ref.
10118 */
10119 mutex_unlock(&ctx->uring_lock);
af9c1a44
JA
10120 do {
10121 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10122 if (!ret)
10123 break;
ed6930c9
JA
10124 ret = io_run_task_work_sig();
10125 if (ret < 0)
10126 break;
af9c1a44
JA
10127 } while (1);
10128
05f3fb3c 10129 mutex_lock(&ctx->uring_lock);
af9c1a44 10130
c150368b
JA
10131 if (ret) {
10132 percpu_ref_resurrect(&ctx->refs);
21b55dbc
SG
10133 goto out_quiesce;
10134 }
10135 }
10136
10137 if (ctx->restricted) {
10138 if (opcode >= IORING_REGISTER_LAST) {
10139 ret = -EINVAL;
10140 goto out;
10141 }
10142
10143 if (!test_bit(opcode, ctx->restrictions.register_op)) {
10144 ret = -EACCES;
c150368b
JA
10145 goto out;
10146 }
05f3fb3c 10147 }
edafccee
JA
10148
10149 switch (opcode) {
10150 case IORING_REGISTER_BUFFERS:
0a96bbe4 10151 ret = io_sqe_buffers_register(ctx, arg, nr_args);
edafccee
JA
10152 break;
10153 case IORING_UNREGISTER_BUFFERS:
10154 ret = -EINVAL;
10155 if (arg || nr_args)
10156 break;
0a96bbe4 10157 ret = io_sqe_buffers_unregister(ctx);
edafccee 10158 break;
6b06314c
JA
10159 case IORING_REGISTER_FILES:
10160 ret = io_sqe_files_register(ctx, arg, nr_args);
10161 break;
10162 case IORING_UNREGISTER_FILES:
10163 ret = -EINVAL;
10164 if (arg || nr_args)
10165 break;
10166 ret = io_sqe_files_unregister(ctx);
10167 break;
c3a31e60
JA
10168 case IORING_REGISTER_FILES_UPDATE:
10169 ret = io_sqe_files_update(ctx, arg, nr_args);
10170 break;
9b402849 10171 case IORING_REGISTER_EVENTFD:
f2842ab5 10172 case IORING_REGISTER_EVENTFD_ASYNC:
9b402849
JA
10173 ret = -EINVAL;
10174 if (nr_args != 1)
10175 break;
10176 ret = io_eventfd_register(ctx, arg);
f2842ab5
JA
10177 if (ret)
10178 break;
10179 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10180 ctx->eventfd_async = 1;
10181 else
10182 ctx->eventfd_async = 0;
9b402849
JA
10183 break;
10184 case IORING_UNREGISTER_EVENTFD:
10185 ret = -EINVAL;
10186 if (arg || nr_args)
10187 break;
10188 ret = io_eventfd_unregister(ctx);
10189 break;
66f4af93
JA
10190 case IORING_REGISTER_PROBE:
10191 ret = -EINVAL;
10192 if (!arg || nr_args > 256)
10193 break;
10194 ret = io_probe(ctx, arg, nr_args);
10195 break;
071698e1
JA
10196 case IORING_REGISTER_PERSONALITY:
10197 ret = -EINVAL;
10198 if (arg || nr_args)
10199 break;
10200 ret = io_register_personality(ctx);
10201 break;
10202 case IORING_UNREGISTER_PERSONALITY:
10203 ret = -EINVAL;
10204 if (arg)
10205 break;
10206 ret = io_unregister_personality(ctx, nr_args);
10207 break;
7e84e1c7
SG
10208 case IORING_REGISTER_ENABLE_RINGS:
10209 ret = -EINVAL;
10210 if (arg || nr_args)
10211 break;
10212 ret = io_register_enable_rings(ctx);
10213 break;
21b55dbc
SG
10214 case IORING_REGISTER_RESTRICTIONS:
10215 ret = io_register_restrictions(ctx, arg, nr_args);
10216 break;
edafccee
JA
10217 default:
10218 ret = -EINVAL;
10219 break;
10220 }
10221
21b55dbc 10222out:
071698e1 10223 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 10224 /* bring the ctx back to life */
05f3fb3c 10225 percpu_ref_reinit(&ctx->refs);
21b55dbc 10226out_quiesce:
0f158b4c 10227 reinit_completion(&ctx->ref_comp);
05f3fb3c 10228 }
edafccee
JA
10229 return ret;
10230}
10231
10232SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10233 void __user *, arg, unsigned int, nr_args)
10234{
10235 struct io_ring_ctx *ctx;
10236 long ret = -EBADF;
10237 struct fd f;
10238
10239 f = fdget(fd);
10240 if (!f.file)
10241 return -EBADF;
10242
10243 ret = -EOPNOTSUPP;
10244 if (f.file->f_op != &io_uring_fops)
10245 goto out_fput;
10246
10247 ctx = f.file->private_data;
10248
10249 mutex_lock(&ctx->uring_lock);
10250 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10251 mutex_unlock(&ctx->uring_lock);
c826bd7a
DD
10252 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10253 ctx->cq_ev_fd != NULL, ret);
edafccee
JA
10254out_fput:
10255 fdput(f);
10256 return ret;
10257}
10258
2b188cc1
JA
10259static int __init io_uring_init(void)
10260{
d7f62e82
SM
10261#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10262 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10263 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10264} while (0)
10265
10266#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10267 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10268 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10269 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10270 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10271 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10272 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10273 BUILD_BUG_SQE_ELEM(8, __u64, off);
10274 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10275 BUILD_BUG_SQE_ELEM(16, __u64, addr);
7d67af2c 10276 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
d7f62e82
SM
10277 BUILD_BUG_SQE_ELEM(24, __u32, len);
10278 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10279 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10280 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10281 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
5769a351
JX
10282 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10283 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
d7f62e82
SM
10284 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10285 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10286 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10287 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10288 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10289 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10290 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10291 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
7d67af2c 10292 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
d7f62e82
SM
10293 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10294 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
10295 BUILD_BUG_SQE_ELEM(42, __u16, personality);
7d67af2c 10296 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
d7f62e82 10297
d3656344 10298 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
84557871 10299 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
2b188cc1
JA
10300 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
10301 return 0;
10302};
10303__initcall(io_uring_init);