]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - fs/io_uring.c
io_uring: encapsulate rsrc node manipulations
[mirror_ubuntu-jammy-kernel.git] / fs / io_uring.c
CommitLineData
2b188cc1
JA
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
1e84b97b
SB
7 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
2b188cc1
JA
29 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
c992fe29 40 * Copyright (c) 2018-2019 Christoph Hellwig
2b188cc1
JA
41 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
52de1fe1 47#include <net/compat.h>
2b188cc1
JA
48#include <linux/refcount.h>
49#include <linux/uio.h>
6b47ee6e 50#include <linux/bits.h>
2b188cc1
JA
51
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
2b188cc1
JA
58#include <linux/percpu.h>
59#include <linux/slab.h>
2b188cc1 60#include <linux/blkdev.h>
edafccee 61#include <linux/bvec.h>
2b188cc1
JA
62#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
6b06314c 65#include <net/scm.h>
2b188cc1
JA
66#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
edafccee
JA
70#include <linux/sizes.h>
71#include <linux/hugetlb.h>
aa4c3967 72#include <linux/highmem.h>
15b71abe
JA
73#include <linux/namei.h>
74#include <linux/fsnotify.h>
4840e418 75#include <linux/fadvise.h>
3e4827b0 76#include <linux/eventpoll.h>
7d67af2c 77#include <linux/splice.h>
b41e9852 78#include <linux/task_work.h>
bcf5a063 79#include <linux/pagemap.h>
0f212204 80#include <linux/io_uring.h>
2b188cc1 81
c826bd7a
DD
82#define CREATE_TRACE_POINTS
83#include <trace/events/io_uring.h>
84
2b188cc1
JA
85#include <uapi/linux/io_uring.h>
86
87#include "internal.h"
561fb04a 88#include "io-wq.h"
2b188cc1 89
5277deaa 90#define IORING_MAX_ENTRIES 32768
33a107f0 91#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
65e19f54
JA
92
93/*
94 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
95 */
96#define IORING_FILE_TABLE_SHIFT 9
97#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
98#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
99#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
21b55dbc
SG
100#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
101 IORING_REGISTER_LAST + IORING_OP_LAST)
2b188cc1 102
b16fed66
PB
103#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
104 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
105 IOSQE_BUFFER_SELECT)
106
2b188cc1
JA
107struct io_uring {
108 u32 head ____cacheline_aligned_in_smp;
109 u32 tail ____cacheline_aligned_in_smp;
110};
111
1e84b97b 112/*
75b28aff
HV
113 * This data is shared with the application through the mmap at offsets
114 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
1e84b97b
SB
115 *
116 * The offsets to the member fields are published through struct
117 * io_sqring_offsets when calling io_uring_setup.
118 */
75b28aff 119struct io_rings {
1e84b97b
SB
120 /*
121 * Head and tail offsets into the ring; the offsets need to be
122 * masked to get valid indices.
123 *
75b28aff
HV
124 * The kernel controls head of the sq ring and the tail of the cq ring,
125 * and the application controls tail of the sq ring and the head of the
126 * cq ring.
1e84b97b 127 */
75b28aff 128 struct io_uring sq, cq;
1e84b97b 129 /*
75b28aff 130 * Bitmasks to apply to head and tail offsets (constant, equals
1e84b97b
SB
131 * ring_entries - 1)
132 */
75b28aff
HV
133 u32 sq_ring_mask, cq_ring_mask;
134 /* Ring sizes (constant, power of 2) */
135 u32 sq_ring_entries, cq_ring_entries;
1e84b97b
SB
136 /*
137 * Number of invalid entries dropped by the kernel due to
138 * invalid index stored in array
139 *
140 * Written by the kernel, shouldn't be modified by the
141 * application (i.e. get number of "new events" by comparing to
142 * cached value).
143 *
144 * After a new SQ head value was read by the application this
145 * counter includes all submissions that were dropped reaching
146 * the new SQ head (and possibly more).
147 */
75b28aff 148 u32 sq_dropped;
1e84b97b 149 /*
0d9b5b3a 150 * Runtime SQ flags
1e84b97b
SB
151 *
152 * Written by the kernel, shouldn't be modified by the
153 * application.
154 *
155 * The application needs a full memory barrier before checking
156 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
157 */
75b28aff 158 u32 sq_flags;
0d9b5b3a
SG
159 /*
160 * Runtime CQ flags
161 *
162 * Written by the application, shouldn't be modified by the
163 * kernel.
164 */
165 u32 cq_flags;
1e84b97b
SB
166 /*
167 * Number of completion events lost because the queue was full;
168 * this should be avoided by the application by making sure
0b4295b5 169 * there are not more requests pending than there is space in
1e84b97b
SB
170 * the completion queue.
171 *
172 * Written by the kernel, shouldn't be modified by the
173 * application (i.e. get number of "new events" by comparing to
174 * cached value).
175 *
176 * As completion events come in out of order this counter is not
177 * ordered with any other data.
178 */
75b28aff 179 u32 cq_overflow;
1e84b97b
SB
180 /*
181 * Ring buffer of completion events.
182 *
183 * The kernel writes completion events fresh every time they are
184 * produced, so the application is allowed to modify pending
185 * entries.
186 */
75b28aff 187 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
2b188cc1
JA
188};
189
45d189c6
PB
190enum io_uring_cmd_flags {
191 IO_URING_F_NONBLOCK = 1,
889fca73 192 IO_URING_F_COMPLETE_DEFER = 2,
45d189c6
PB
193};
194
edafccee
JA
195struct io_mapped_ubuf {
196 u64 ubuf;
197 size_t len;
198 struct bio_vec *bvec;
199 unsigned int nr_bvecs;
de293938 200 unsigned long acct_pages;
edafccee
JA
201};
202
50238531
BM
203struct io_ring_ctx;
204
6c2450ae
PB
205struct io_overflow_cqe {
206 struct io_uring_cqe cqe;
207 struct list_head list;
208};
209
269bbe5f
BM
210struct io_rsrc_put {
211 struct list_head list;
50238531
BM
212 union {
213 void *rsrc;
214 struct file *file;
215 };
269bbe5f
BM
216};
217
218struct fixed_rsrc_table {
65e19f54 219 struct file **files;
31b51510
JA
220};
221
b895c9a6 222struct io_rsrc_node {
05589553
XW
223 struct percpu_ref refs;
224 struct list_head node;
269bbe5f 225 struct list_head rsrc_list;
b895c9a6 226 struct io_rsrc_data *rsrc_data;
50238531
BM
227 void (*rsrc_put)(struct io_ring_ctx *ctx,
228 struct io_rsrc_put *prsrc);
4a38aed2 229 struct llist_node llist;
e297822b 230 bool done;
05589553
XW
231};
232
b895c9a6 233struct io_rsrc_data {
269bbe5f 234 struct fixed_rsrc_table *table;
05f3fb3c
JA
235 struct io_ring_ctx *ctx;
236
b895c9a6 237 struct io_rsrc_node *node;
05f3fb3c 238 struct percpu_ref refs;
05f3fb3c 239 struct completion done;
8bad28d8 240 bool quiesce;
05f3fb3c
JA
241};
242
5a2e745d
JA
243struct io_buffer {
244 struct list_head list;
245 __u64 addr;
246 __s32 len;
247 __u16 bid;
248};
249
21b55dbc
SG
250struct io_restriction {
251 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
252 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
253 u8 sqe_flags_allowed;
254 u8 sqe_flags_required;
7e84e1c7 255 bool registered;
21b55dbc
SG
256};
257
37d1e2e3
JA
258enum {
259 IO_SQ_THREAD_SHOULD_STOP = 0,
260 IO_SQ_THREAD_SHOULD_PARK,
261};
262
534ca6d6
JA
263struct io_sq_data {
264 refcount_t refs;
9e138a48 265 atomic_t park_pending;
09a6f4ef 266 struct mutex lock;
69fb2131
JA
267
268 /* ctx's that are using this sqd */
269 struct list_head ctx_list;
69fb2131 270
534ca6d6
JA
271 struct task_struct *thread;
272 struct wait_queue_head wait;
08369246
XW
273
274 unsigned sq_thread_idle;
37d1e2e3
JA
275 int sq_cpu;
276 pid_t task_pid;
5c2469e0 277 pid_t task_tgid;
37d1e2e3
JA
278
279 unsigned long state;
37d1e2e3 280 struct completion exited;
b7f5a0bf 281 struct callback_head *park_task_work;
534ca6d6
JA
282};
283
258b29a9 284#define IO_IOPOLL_BATCH 8
6dd0be1e 285#define IO_COMPL_BATCH 32
6ff119a6 286#define IO_REQ_CACHE_SIZE 32
bf019da7 287#define IO_REQ_ALLOC_BATCH 8
258b29a9
PB
288
289struct io_comp_state {
6dd0be1e 290 struct io_kiocb *reqs[IO_COMPL_BATCH];
1b4c351f 291 unsigned int nr;
c7dae4ba
JA
292 unsigned int locked_free_nr;
293 /* inline/task_work completion list, under ->uring_lock */
1b4c351f 294 struct list_head free_list;
c7dae4ba
JA
295 /* IRQ completion list, under ->completion_lock */
296 struct list_head locked_free_list;
258b29a9
PB
297};
298
a1ab7b35
PB
299struct io_submit_link {
300 struct io_kiocb *head;
301 struct io_kiocb *last;
302};
303
258b29a9
PB
304struct io_submit_state {
305 struct blk_plug plug;
a1ab7b35 306 struct io_submit_link link;
258b29a9
PB
307
308 /*
309 * io_kiocb alloc cache
310 */
bf019da7 311 void *reqs[IO_REQ_CACHE_SIZE];
258b29a9
PB
312 unsigned int free_reqs;
313
314 bool plug_started;
315
316 /*
317 * Batch completion logic
318 */
319 struct io_comp_state comp;
320
321 /*
322 * File reference cache
323 */
324 struct file *file;
325 unsigned int fd;
326 unsigned int file_refs;
327 unsigned int ios_left;
328};
329
2b188cc1
JA
330struct io_ring_ctx {
331 struct {
332 struct percpu_ref refs;
333 } ____cacheline_aligned_in_smp;
334
335 struct {
336 unsigned int flags;
e1d85334 337 unsigned int compat: 1;
e1d85334
RD
338 unsigned int cq_overflow_flushed: 1;
339 unsigned int drain_next: 1;
340 unsigned int eventfd_async: 1;
21b55dbc 341 unsigned int restricted: 1;
2b188cc1 342
75b28aff
HV
343 /*
344 * Ring buffer of indices into array of io_uring_sqe, which is
345 * mmapped by the application using the IORING_OFF_SQES offset.
346 *
347 * This indirection could e.g. be used to assign fixed
348 * io_uring_sqe entries to operations and only submit them to
349 * the queue when needed.
350 *
351 * The kernel modifies neither the indices array nor the entries
352 * array.
353 */
354 u32 *sq_array;
2b188cc1
JA
355 unsigned cached_sq_head;
356 unsigned sq_entries;
357 unsigned sq_mask;
6c271ce2 358 unsigned sq_thread_idle;
498ccd9e 359 unsigned cached_sq_dropped;
2c3bac6d 360 unsigned cached_cq_overflow;
ad3eb2c8 361 unsigned long sq_check_overflow;
de0617e4 362
e941894e
JA
363 /* hashed buffered write serialization */
364 struct io_wq_hash *hash_map;
365
de0617e4 366 struct list_head defer_list;
5262f567 367 struct list_head timeout_list;
1d7bb1d5 368 struct list_head cq_overflow_list;
fcb323cc 369
ad3eb2c8 370 struct io_uring_sqe *sq_sqes;
2b188cc1
JA
371 } ____cacheline_aligned_in_smp;
372
3c1a2ead
JA
373 struct {
374 struct mutex uring_lock;
375 wait_queue_head_t wait;
376 } ____cacheline_aligned_in_smp;
377
378 struct io_submit_state submit_state;
379
206aefde
JA
380 struct io_rings *rings;
381
2aede0e4
JA
382 /* Only used for accounting purposes */
383 struct mm_struct *mm_account;
384
7c30f36a 385 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
534ca6d6
JA
386 struct io_sq_data *sq_data; /* if using sq thread polling */
387
90554200 388 struct wait_queue_head sqo_sq_wait;
69fb2131 389 struct list_head sqd_list;
75b28aff 390
6b06314c
JA
391 /*
392 * If used, fixed file set. Writers must ensure that ->refs is dead,
393 * readers must ensure that ->refs is alive as long as the file* is
394 * used. Only updated through io_uring_register(2).
395 */
b895c9a6 396 struct io_rsrc_data *file_data;
6b06314c
JA
397 unsigned nr_user_files;
398
edafccee
JA
399 /* if used, fixed mapped user buffers */
400 unsigned nr_user_bufs;
401 struct io_mapped_ubuf *user_bufs;
402
2b188cc1
JA
403 struct user_struct *user;
404
0f158b4c 405 struct completion ref_comp;
206aefde
JA
406
407#if defined(CONFIG_UNIX)
408 struct socket *ring_sock;
409#endif
410
9e15c3a0 411 struct xarray io_buffers;
5a2e745d 412
61cf9370
MWO
413 struct xarray personalities;
414 u32 pers_next;
071698e1 415
206aefde
JA
416 struct {
417 unsigned cached_cq_tail;
418 unsigned cq_entries;
419 unsigned cq_mask;
420 atomic_t cq_timeouts;
f010505b 421 unsigned cq_last_tm_flush;
ad3eb2c8 422 unsigned long cq_check_overflow;
206aefde
JA
423 struct wait_queue_head cq_wait;
424 struct fasync_struct *cq_fasync;
425 struct eventfd_ctx *cq_ev_fd;
426 } ____cacheline_aligned_in_smp;
2b188cc1 427
2b188cc1
JA
428 struct {
429 spinlock_t completion_lock;
e94f141b 430
def596e9 431 /*
540e32a0 432 * ->iopoll_list is protected by the ctx->uring_lock for
def596e9
JA
433 * io_uring instances that don't use IORING_SETUP_SQPOLL.
434 * For SQPOLL, only the single threaded io_sq_thread() will
435 * manipulate the list, hence no extra locking is needed there.
436 */
540e32a0 437 struct list_head iopoll_list;
78076bb6
JA
438 struct hlist_head *cancel_hash;
439 unsigned cancel_hash_bits;
e94f141b 440 bool poll_multi_file;
31b51510 441
fcb323cc
JA
442 spinlock_t inflight_lock;
443 struct list_head inflight_list;
2b188cc1 444 } ____cacheline_aligned_in_smp;
85faa7b8 445
269bbe5f
BM
446 struct delayed_work rsrc_put_work;
447 struct llist_head rsrc_put_llist;
d67d2263
BM
448 struct list_head rsrc_ref_list;
449 spinlock_t rsrc_ref_lock;
b895c9a6 450 struct io_rsrc_node *rsrc_backup_node;
4a38aed2 451
21b55dbc 452 struct io_restriction restrictions;
3c1a2ead 453
7c25c0d1
JA
454 /* exit task_work */
455 struct callback_head *exit_task_work;
456
e941894e
JA
457 struct wait_queue_head hash_wait;
458
3c1a2ead
JA
459 /* Keep this last, we don't need it for the fast path */
460 struct work_struct exit_work;
13bf43f5 461 struct list_head tctx_list;
2b188cc1
JA
462};
463
53e043b2
SM
464struct io_uring_task {
465 /* submission side */
466 struct xarray xa;
467 struct wait_queue_head wait;
ee53fb2b
SM
468 const struct io_ring_ctx *last;
469 struct io_wq *io_wq;
53e043b2
SM
470 struct percpu_counter inflight;
471 atomic_t in_idle;
53e043b2
SM
472
473 spinlock_t task_lock;
474 struct io_wq_work_list task_list;
475 unsigned long task_state;
476 struct callback_head task_work;
477};
478
09bb8394
JA
479/*
480 * First field must be the file pointer in all the
481 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
482 */
221c5eb2
JA
483struct io_poll_iocb {
484 struct file *file;
018043be 485 struct wait_queue_head *head;
221c5eb2 486 __poll_t events;
8c838788 487 bool done;
221c5eb2 488 bool canceled;
b69de288
JA
489 bool update_events;
490 bool update_user_data;
491 union {
492 struct wait_queue_entry wait;
493 struct {
494 u64 old_user_data;
495 u64 new_user_data;
496 };
497 };
221c5eb2
JA
498};
499
018043be
PB
500struct io_poll_remove {
501 struct file *file;
502 u64 addr;
503};
504
b5dba59e
JA
505struct io_close {
506 struct file *file;
b5dba59e
JA
507 int fd;
508};
509
ad8a48ac
JA
510struct io_timeout_data {
511 struct io_kiocb *req;
512 struct hrtimer timer;
513 struct timespec64 ts;
514 enum hrtimer_mode mode;
515};
516
8ed8d3c3
JA
517struct io_accept {
518 struct file *file;
519 struct sockaddr __user *addr;
520 int __user *addr_len;
521 int flags;
09952e3e 522 unsigned long nofile;
8ed8d3c3
JA
523};
524
525struct io_sync {
526 struct file *file;
527 loff_t len;
528 loff_t off;
529 int flags;
d63d1b5e 530 int mode;
8ed8d3c3
JA
531};
532
fbf23849
JA
533struct io_cancel {
534 struct file *file;
535 u64 addr;
536};
537
b29472ee
JA
538struct io_timeout {
539 struct file *file;
bfe68a22
PB
540 u32 off;
541 u32 target_seq;
135fcde8 542 struct list_head list;
90cd7e42
PB
543 /* head of the link, used by linked timeouts only */
544 struct io_kiocb *head;
b29472ee
JA
545};
546
0bdf7a2d
PB
547struct io_timeout_rem {
548 struct file *file;
549 u64 addr;
9c8e11b3
PB
550
551 /* timeout update */
552 struct timespec64 ts;
553 u32 flags;
0bdf7a2d
PB
554};
555
9adbd45d
JA
556struct io_rw {
557 /* NOTE: kiocb has the file as the first member, so don't do it here */
558 struct kiocb kiocb;
559 u64 addr;
560 u64 len;
561};
562
3fbb51c1
JA
563struct io_connect {
564 struct file *file;
565 struct sockaddr __user *addr;
566 int addr_len;
567};
568
e47293fd
JA
569struct io_sr_msg {
570 struct file *file;
fddaface 571 union {
270a5940 572 struct user_msghdr __user *umsg;
fddaface
JA
573 void __user *buf;
574 };
e47293fd 575 int msg_flags;
bcda7baa 576 int bgid;
fddaface 577 size_t len;
bcda7baa 578 struct io_buffer *kbuf;
e47293fd
JA
579};
580
15b71abe
JA
581struct io_open {
582 struct file *file;
583 int dfd;
15b71abe 584 struct filename *filename;
c12cedf2 585 struct open_how how;
4022e7af 586 unsigned long nofile;
15b71abe
JA
587};
588
269bbe5f 589struct io_rsrc_update {
05f3fb3c
JA
590 struct file *file;
591 u64 arg;
592 u32 nr_args;
593 u32 offset;
594};
595
4840e418
JA
596struct io_fadvise {
597 struct file *file;
598 u64 offset;
599 u32 len;
600 u32 advice;
601};
602
c1ca757b
JA
603struct io_madvise {
604 struct file *file;
605 u64 addr;
606 u32 len;
607 u32 advice;
608};
609
3e4827b0
JA
610struct io_epoll {
611 struct file *file;
612 int epfd;
613 int op;
614 int fd;
615 struct epoll_event event;
e47293fd
JA
616};
617
7d67af2c
PB
618struct io_splice {
619 struct file *file_out;
620 struct file *file_in;
621 loff_t off_out;
622 loff_t off_in;
623 u64 len;
624 unsigned int flags;
625};
626
ddf0322d
JA
627struct io_provide_buf {
628 struct file *file;
629 __u64 addr;
630 __s32 len;
631 __u32 bgid;
632 __u16 nbufs;
633 __u16 bid;
634};
635
1d9e1288
BM
636struct io_statx {
637 struct file *file;
638 int dfd;
639 unsigned int mask;
640 unsigned int flags;
e62753e4 641 const char __user *filename;
1d9e1288
BM
642 struct statx __user *buffer;
643};
644
36f4fa68
JA
645struct io_shutdown {
646 struct file *file;
647 int how;
648};
649
80a261fd
JA
650struct io_rename {
651 struct file *file;
652 int old_dfd;
653 int new_dfd;
654 struct filename *oldpath;
655 struct filename *newpath;
656 int flags;
657};
658
14a1143b
JA
659struct io_unlink {
660 struct file *file;
661 int dfd;
662 int flags;
663 struct filename *filename;
664};
665
3ca405eb
PB
666struct io_completion {
667 struct file *file;
668 struct list_head list;
8c3f9cd1 669 u32 cflags;
3ca405eb
PB
670};
671
f499a021
JA
672struct io_async_connect {
673 struct sockaddr_storage address;
674};
675
03b1230c
JA
676struct io_async_msghdr {
677 struct iovec fast_iov[UIO_FASTIOV];
257e84a5
PB
678 /* points to an allocated iov, if NULL we use fast_iov instead */
679 struct iovec *free_iov;
03b1230c
JA
680 struct sockaddr __user *uaddr;
681 struct msghdr msg;
b537916c 682 struct sockaddr_storage addr;
03b1230c
JA
683};
684
f67676d1
JA
685struct io_async_rw {
686 struct iovec fast_iov[UIO_FASTIOV];
ff6165b2
JA
687 const struct iovec *free_iovec;
688 struct iov_iter iter;
227c0c96 689 size_t bytes_done;
bcf5a063 690 struct wait_page_queue wpq;
f67676d1
JA
691};
692
6b47ee6e
PB
693enum {
694 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
695 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
696 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
697 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
698 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
bcda7baa 699 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
6b47ee6e 700
6b47ee6e
PB
701 REQ_F_FAIL_LINK_BIT,
702 REQ_F_INFLIGHT_BIT,
703 REQ_F_CUR_POS_BIT,
704 REQ_F_NOWAIT_BIT,
6b47ee6e 705 REQ_F_LINK_TIMEOUT_BIT,
99bc4c38 706 REQ_F_NEED_CLEANUP_BIT,
d7718a9d 707 REQ_F_POLLED_BIT,
bcda7baa 708 REQ_F_BUFFER_SELECTED_BIT,
900fad45 709 REQ_F_LTIMEOUT_ACTIVE_BIT,
e342c807 710 REQ_F_COMPLETE_INLINE_BIT,
230d50d4 711 REQ_F_REISSUE_BIT,
8c130827 712 REQ_F_DONT_REISSUE_BIT,
7b29f92d
JA
713 /* keep async read/write and isreg together and in order */
714 REQ_F_ASYNC_READ_BIT,
715 REQ_F_ASYNC_WRITE_BIT,
716 REQ_F_ISREG_BIT,
84557871
JA
717
718 /* not a real bit, just to check we're not overflowing the space */
719 __REQ_F_LAST_BIT,
6b47ee6e
PB
720};
721
722enum {
723 /* ctx owns file */
724 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
725 /* drain existing IO first */
726 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
727 /* linked sqes */
728 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
729 /* doesn't sever on completion < 0 */
730 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
731 /* IOSQE_ASYNC */
732 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
bcda7baa
JA
733 /* IOSQE_BUFFER_SELECT */
734 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
6b47ee6e 735
6b47ee6e
PB
736 /* fail rest of links */
737 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
b05a1bcd 738 /* on inflight list, should be cancelled and waited on exit reliably */
6b47ee6e
PB
739 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
740 /* read/write uses file position */
741 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
742 /* must not punt to workers */
743 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
900fad45 744 /* has or had linked timeout */
6b47ee6e 745 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
99bc4c38
PB
746 /* needs cleanup */
747 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
d7718a9d
JA
748 /* already went through poll handler */
749 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
bcda7baa
JA
750 /* buffer already selected */
751 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
900fad45
PB
752 /* linked timeout is active, i.e. prepared by link's head */
753 REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
e342c807
PB
754 /* completion is deferred through io_comp_state */
755 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
230d50d4
JA
756 /* caller should reissue async */
757 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
8c130827
PB
758 /* don't attempt request reissue, see io_rw_reissue() */
759 REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
7b29f92d
JA
760 /* supports async reads */
761 REQ_F_ASYNC_READ = BIT(REQ_F_ASYNC_READ_BIT),
762 /* supports async writes */
763 REQ_F_ASYNC_WRITE = BIT(REQ_F_ASYNC_WRITE_BIT),
764 /* regular file */
765 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
d7718a9d
JA
766};
767
768struct async_poll {
769 struct io_poll_iocb poll;
807abcb0 770 struct io_poll_iocb *double_poll;
6b47ee6e
PB
771};
772
7cbf1722
JA
773struct io_task_work {
774 struct io_wq_work_node node;
775 task_work_func_t func;
776};
777
09bb8394
JA
778/*
779 * NOTE! Each of the iocb union members has the file pointer
780 * as the first entry in their struct definition. So you can
781 * access the file pointer through any of the sub-structs,
782 * or directly as just 'ki_filp' in this struct.
783 */
2b188cc1 784struct io_kiocb {
221c5eb2 785 union {
09bb8394 786 struct file *file;
9adbd45d 787 struct io_rw rw;
221c5eb2 788 struct io_poll_iocb poll;
018043be 789 struct io_poll_remove poll_remove;
8ed8d3c3
JA
790 struct io_accept accept;
791 struct io_sync sync;
fbf23849 792 struct io_cancel cancel;
b29472ee 793 struct io_timeout timeout;
0bdf7a2d 794 struct io_timeout_rem timeout_rem;
3fbb51c1 795 struct io_connect connect;
e47293fd 796 struct io_sr_msg sr_msg;
15b71abe 797 struct io_open open;
b5dba59e 798 struct io_close close;
269bbe5f 799 struct io_rsrc_update rsrc_update;
4840e418 800 struct io_fadvise fadvise;
c1ca757b 801 struct io_madvise madvise;
3e4827b0 802 struct io_epoll epoll;
7d67af2c 803 struct io_splice splice;
ddf0322d 804 struct io_provide_buf pbuf;
1d9e1288 805 struct io_statx statx;
36f4fa68 806 struct io_shutdown shutdown;
80a261fd 807 struct io_rename rename;
14a1143b 808 struct io_unlink unlink;
3ca405eb
PB
809 /* use only after cleaning per-op data, see io_clean_op() */
810 struct io_completion compl;
221c5eb2 811 };
2b188cc1 812
e8c2bc1f
JA
813 /* opcode allocated if it needs to store data for async defer */
814 void *async_data;
d625c6ee 815 u8 opcode;
65a6543d
XW
816 /* polled IO has completed */
817 u8 iopoll_completed;
2b188cc1 818
4f4eeba8 819 u16 buf_index;
9cf7c104 820 u32 result;
4f4eeba8 821
010e8e6b
PB
822 struct io_ring_ctx *ctx;
823 unsigned int flags;
abc54d63 824 atomic_t refs;
010e8e6b
PB
825 struct task_struct *task;
826 u64 user_data;
d7718a9d 827
f2f87370 828 struct io_kiocb *link;
269bbe5f 829 struct percpu_ref *fixed_rsrc_refs;
fcb323cc 830
d21ffe7e
PB
831 /*
832 * 1. used with ctx->iopoll_list with reads/writes
833 * 2. to track reqs with ->files (see io_op_def::file_table)
834 */
010e8e6b 835 struct list_head inflight_entry;
7cbf1722
JA
836 union {
837 struct io_task_work io_task_work;
838 struct callback_head task_work;
839 };
010e8e6b
PB
840 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
841 struct hlist_node hash_node;
842 struct async_poll *apoll;
843 struct io_wq_work work;
2b188cc1 844};
05589553 845
13bf43f5
PB
846struct io_tctx_node {
847 struct list_head ctx_node;
848 struct task_struct *task;
13bf43f5
PB
849 struct io_ring_ctx *ctx;
850};
851
27dc8338
PB
852struct io_defer_entry {
853 struct list_head list;
854 struct io_kiocb *req;
9cf7c104 855 u32 seq;
2b188cc1
JA
856};
857
d3656344 858struct io_op_def {
d3656344
JA
859 /* needs req->file assigned */
860 unsigned needs_file : 1;
d3656344
JA
861 /* hash wq insertion if file is a regular file */
862 unsigned hash_reg_file : 1;
863 /* unbound wq insertion if file is a non-regular file */
864 unsigned unbound_nonreg_file : 1;
66f4af93
JA
865 /* opcode is not supported by this kernel */
866 unsigned not_supported : 1;
8a72758c
JA
867 /* set if opcode supports polled "wait" */
868 unsigned pollin : 1;
869 unsigned pollout : 1;
bcda7baa
JA
870 /* op supports buffer selection */
871 unsigned buffer_select : 1;
26f0505a
PB
872 /* do prep async if is going to be punted */
873 unsigned needs_async_setup : 1;
27926b68
JA
874 /* should block plug */
875 unsigned plug : 1;
e8c2bc1f
JA
876 /* size of async data needed, if any */
877 unsigned short async_size;
d3656344
JA
878};
879
0918682b 880static const struct io_op_def io_op_defs[] = {
0463b6c5
PB
881 [IORING_OP_NOP] = {},
882 [IORING_OP_READV] = {
d3656344
JA
883 .needs_file = 1,
884 .unbound_nonreg_file = 1,
8a72758c 885 .pollin = 1,
4d954c25 886 .buffer_select = 1,
26f0505a 887 .needs_async_setup = 1,
27926b68 888 .plug = 1,
e8c2bc1f 889 .async_size = sizeof(struct io_async_rw),
d3656344 890 },
0463b6c5 891 [IORING_OP_WRITEV] = {
d3656344
JA
892 .needs_file = 1,
893 .hash_reg_file = 1,
894 .unbound_nonreg_file = 1,
8a72758c 895 .pollout = 1,
26f0505a 896 .needs_async_setup = 1,
27926b68 897 .plug = 1,
e8c2bc1f 898 .async_size = sizeof(struct io_async_rw),
d3656344 899 },
0463b6c5 900 [IORING_OP_FSYNC] = {
d3656344
JA
901 .needs_file = 1,
902 },
0463b6c5 903 [IORING_OP_READ_FIXED] = {
d3656344
JA
904 .needs_file = 1,
905 .unbound_nonreg_file = 1,
8a72758c 906 .pollin = 1,
27926b68 907 .plug = 1,
e8c2bc1f 908 .async_size = sizeof(struct io_async_rw),
d3656344 909 },
0463b6c5 910 [IORING_OP_WRITE_FIXED] = {
d3656344
JA
911 .needs_file = 1,
912 .hash_reg_file = 1,
913 .unbound_nonreg_file = 1,
8a72758c 914 .pollout = 1,
27926b68 915 .plug = 1,
e8c2bc1f 916 .async_size = sizeof(struct io_async_rw),
d3656344 917 },
0463b6c5 918 [IORING_OP_POLL_ADD] = {
d3656344
JA
919 .needs_file = 1,
920 .unbound_nonreg_file = 1,
921 },
0463b6c5
PB
922 [IORING_OP_POLL_REMOVE] = {},
923 [IORING_OP_SYNC_FILE_RANGE] = {
d3656344
JA
924 .needs_file = 1,
925 },
0463b6c5 926 [IORING_OP_SENDMSG] = {
d3656344
JA
927 .needs_file = 1,
928 .unbound_nonreg_file = 1,
8a72758c 929 .pollout = 1,
26f0505a 930 .needs_async_setup = 1,
e8c2bc1f 931 .async_size = sizeof(struct io_async_msghdr),
d3656344 932 },
0463b6c5 933 [IORING_OP_RECVMSG] = {
d3656344
JA
934 .needs_file = 1,
935 .unbound_nonreg_file = 1,
8a72758c 936 .pollin = 1,
52de1fe1 937 .buffer_select = 1,
26f0505a 938 .needs_async_setup = 1,
e8c2bc1f 939 .async_size = sizeof(struct io_async_msghdr),
d3656344 940 },
0463b6c5 941 [IORING_OP_TIMEOUT] = {
e8c2bc1f 942 .async_size = sizeof(struct io_timeout_data),
d3656344 943 },
9c8e11b3
PB
944 [IORING_OP_TIMEOUT_REMOVE] = {
945 /* used by timeout updates' prep() */
9c8e11b3 946 },
0463b6c5 947 [IORING_OP_ACCEPT] = {
d3656344
JA
948 .needs_file = 1,
949 .unbound_nonreg_file = 1,
8a72758c 950 .pollin = 1,
d3656344 951 },
0463b6c5
PB
952 [IORING_OP_ASYNC_CANCEL] = {},
953 [IORING_OP_LINK_TIMEOUT] = {
e8c2bc1f 954 .async_size = sizeof(struct io_timeout_data),
d3656344 955 },
0463b6c5 956 [IORING_OP_CONNECT] = {
d3656344
JA
957 .needs_file = 1,
958 .unbound_nonreg_file = 1,
8a72758c 959 .pollout = 1,
26f0505a 960 .needs_async_setup = 1,
e8c2bc1f 961 .async_size = sizeof(struct io_async_connect),
d3656344 962 },
0463b6c5 963 [IORING_OP_FALLOCATE] = {
d3656344 964 .needs_file = 1,
d3656344 965 },
44526bed
JA
966 [IORING_OP_OPENAT] = {},
967 [IORING_OP_CLOSE] = {},
968 [IORING_OP_FILES_UPDATE] = {},
969 [IORING_OP_STATX] = {},
0463b6c5 970 [IORING_OP_READ] = {
3a6820f2
JA
971 .needs_file = 1,
972 .unbound_nonreg_file = 1,
8a72758c 973 .pollin = 1,
bcda7baa 974 .buffer_select = 1,
27926b68 975 .plug = 1,
e8c2bc1f 976 .async_size = sizeof(struct io_async_rw),
3a6820f2 977 },
0463b6c5 978 [IORING_OP_WRITE] = {
3a6820f2
JA
979 .needs_file = 1,
980 .unbound_nonreg_file = 1,
8a72758c 981 .pollout = 1,
27926b68 982 .plug = 1,
e8c2bc1f 983 .async_size = sizeof(struct io_async_rw),
3a6820f2 984 },
0463b6c5 985 [IORING_OP_FADVISE] = {
4840e418 986 .needs_file = 1,
c1ca757b 987 },
44526bed 988 [IORING_OP_MADVISE] = {},
0463b6c5 989 [IORING_OP_SEND] = {
fddaface
JA
990 .needs_file = 1,
991 .unbound_nonreg_file = 1,
8a72758c 992 .pollout = 1,
fddaface 993 },
0463b6c5 994 [IORING_OP_RECV] = {
fddaface
JA
995 .needs_file = 1,
996 .unbound_nonreg_file = 1,
8a72758c 997 .pollin = 1,
bcda7baa 998 .buffer_select = 1,
fddaface 999 },
0463b6c5 1000 [IORING_OP_OPENAT2] = {
cebdb986 1001 },
3e4827b0
JA
1002 [IORING_OP_EPOLL_CTL] = {
1003 .unbound_nonreg_file = 1,
3e4827b0 1004 },
7d67af2c
PB
1005 [IORING_OP_SPLICE] = {
1006 .needs_file = 1,
1007 .hash_reg_file = 1,
1008 .unbound_nonreg_file = 1,
ddf0322d
JA
1009 },
1010 [IORING_OP_PROVIDE_BUFFERS] = {},
067524e9 1011 [IORING_OP_REMOVE_BUFFERS] = {},
f2a8d5c7
PB
1012 [IORING_OP_TEE] = {
1013 .needs_file = 1,
1014 .hash_reg_file = 1,
1015 .unbound_nonreg_file = 1,
1016 },
36f4fa68
JA
1017 [IORING_OP_SHUTDOWN] = {
1018 .needs_file = 1,
1019 },
44526bed
JA
1020 [IORING_OP_RENAMEAT] = {},
1021 [IORING_OP_UNLINKAT] = {},
d3656344
JA
1022};
1023
7a612350 1024static bool io_disarm_next(struct io_kiocb *req);
d56d938b 1025static void io_uring_del_task_file(unsigned long index);
9936c7c2
PB
1026static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1027 struct task_struct *task,
1028 struct files_struct *files);
37d1e2e3 1029static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx);
b895c9a6
PB
1030static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node);
1031static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
f2303b1f 1032static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
1ffc5422 1033
78e19bbe 1034static void io_cqring_fill_event(struct io_kiocb *req, long res);
ec9c02ad 1035static void io_put_req(struct io_kiocb *req);
216578e5 1036static void io_put_req_deferred(struct io_kiocb *req, int nr);
c7dae4ba
JA
1037static void io_dismantle_req(struct io_kiocb *req);
1038static void io_put_task(struct task_struct *task, int nr);
1039static void io_queue_next(struct io_kiocb *req);
94ae5e77
JA
1040static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
1041static void io_queue_linked_timeout(struct io_kiocb *req);
05f3fb3c 1042static int __io_sqe_files_update(struct io_ring_ctx *ctx,
269bbe5f 1043 struct io_uring_rsrc_update *ip,
05f3fb3c 1044 unsigned nr_args);
68fb8979 1045static void io_clean_op(struct io_kiocb *req);
8371adf5
PB
1046static struct file *io_file_get(struct io_submit_state *state,
1047 struct io_kiocb *req, int fd, bool fixed);
c5eef2b9 1048static void __io_queue_sqe(struct io_kiocb *req);
269bbe5f 1049static void io_rsrc_put_work(struct work_struct *work);
de0617e4 1050
907d1df3 1051static void io_req_task_queue(struct io_kiocb *req);
65453d1e
JA
1052static void io_submit_flush_completions(struct io_comp_state *cs,
1053 struct io_ring_ctx *ctx);
5082620f 1054static bool io_poll_remove_waitqs(struct io_kiocb *req);
179ae0d1 1055static int io_req_prep_async(struct io_kiocb *req);
de0617e4 1056
2b188cc1
JA
1057static struct kmem_cache *req_cachep;
1058
0918682b 1059static const struct file_operations io_uring_fops;
2b188cc1
JA
1060
1061struct sock *io_uring_get_socket(struct file *file)
1062{
1063#if defined(CONFIG_UNIX)
1064 if (file->f_op == &io_uring_fops) {
1065 struct io_ring_ctx *ctx = file->private_data;
1066
1067 return ctx->ring_sock->sk;
1068 }
1069#endif
1070 return NULL;
1071}
1072EXPORT_SYMBOL(io_uring_get_socket);
1073
f2f87370
PB
1074#define io_for_each_link(pos, head) \
1075 for (pos = (head); pos; pos = pos->link)
1076
b895c9a6 1077static inline void io_req_set_rsrc_node(struct io_kiocb *req)
36f72fe2
PB
1078{
1079 struct io_ring_ctx *ctx = req->ctx;
1080
269bbe5f
BM
1081 if (!req->fixed_rsrc_refs) {
1082 req->fixed_rsrc_refs = &ctx->file_data->node->refs;
1083 percpu_ref_get(req->fixed_rsrc_refs);
36f72fe2
PB
1084 }
1085}
1086
08d23634
PB
1087static bool io_match_task(struct io_kiocb *head,
1088 struct task_struct *task,
1089 struct files_struct *files)
1090{
1091 struct io_kiocb *req;
1092
68207680 1093 if (task && head->task != task)
08d23634
PB
1094 return false;
1095 if (!files)
1096 return true;
1097
1098 io_for_each_link(req, head) {
b05a1bcd 1099 if (req->flags & REQ_F_INFLIGHT)
02a13674 1100 return true;
08d23634
PB
1101 }
1102 return false;
1103}
1104
c40f6379
JA
1105static inline void req_set_fail_links(struct io_kiocb *req)
1106{
1107 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
1108 req->flags |= REQ_F_FAIL_LINK;
1109}
4a38aed2 1110
2b188cc1
JA
1111static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1112{
1113 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1114
0f158b4c 1115 complete(&ctx->ref_comp);
2b188cc1
JA
1116}
1117
8eb7e2d0
PB
1118static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1119{
1120 return !req->timeout.off;
1121}
1122
2b188cc1
JA
1123static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1124{
1125 struct io_ring_ctx *ctx;
78076bb6 1126 int hash_bits;
2b188cc1
JA
1127
1128 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1129 if (!ctx)
1130 return NULL;
1131
78076bb6
JA
1132 /*
1133 * Use 5 bits less than the max cq entries, that should give us around
1134 * 32 entries per hash list if totally full and uniformly spread.
1135 */
1136 hash_bits = ilog2(p->cq_entries);
1137 hash_bits -= 5;
1138 if (hash_bits <= 0)
1139 hash_bits = 1;
1140 ctx->cancel_hash_bits = hash_bits;
1141 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1142 GFP_KERNEL);
1143 if (!ctx->cancel_hash)
1144 goto err;
1145 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1146
21482896 1147 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
206aefde
JA
1148 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1149 goto err;
2b188cc1
JA
1150
1151 ctx->flags = p->flags;
90554200 1152 init_waitqueue_head(&ctx->sqo_sq_wait);
69fb2131 1153 INIT_LIST_HEAD(&ctx->sqd_list);
2b188cc1 1154 init_waitqueue_head(&ctx->cq_wait);
1d7bb1d5 1155 INIT_LIST_HEAD(&ctx->cq_overflow_list);
0f158b4c 1156 init_completion(&ctx->ref_comp);
9e15c3a0 1157 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
61cf9370 1158 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
2b188cc1
JA
1159 mutex_init(&ctx->uring_lock);
1160 init_waitqueue_head(&ctx->wait);
1161 spin_lock_init(&ctx->completion_lock);
540e32a0 1162 INIT_LIST_HEAD(&ctx->iopoll_list);
de0617e4 1163 INIT_LIST_HEAD(&ctx->defer_list);
5262f567 1164 INIT_LIST_HEAD(&ctx->timeout_list);
fcb323cc
JA
1165 spin_lock_init(&ctx->inflight_lock);
1166 INIT_LIST_HEAD(&ctx->inflight_list);
d67d2263
BM
1167 spin_lock_init(&ctx->rsrc_ref_lock);
1168 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
269bbe5f
BM
1169 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1170 init_llist_head(&ctx->rsrc_put_llist);
13bf43f5 1171 INIT_LIST_HEAD(&ctx->tctx_list);
1b4c351f 1172 INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
c7dae4ba 1173 INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list);
2b188cc1 1174 return ctx;
206aefde 1175err:
78076bb6 1176 kfree(ctx->cancel_hash);
206aefde
JA
1177 kfree(ctx);
1178 return NULL;
2b188cc1
JA
1179}
1180
9cf7c104 1181static bool req_need_defer(struct io_kiocb *req, u32 seq)
7adf4eaf 1182{
2bc9930e
JA
1183 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1184 struct io_ring_ctx *ctx = req->ctx;
a197f664 1185
9cf7c104 1186 return seq != ctx->cached_cq_tail
2c3bac6d 1187 + READ_ONCE(ctx->cached_cq_overflow);
2bc9930e 1188 }
de0617e4 1189
9d858b21 1190 return false;
de0617e4
JA
1191}
1192
ce3d5aae
PB
1193static void io_req_track_inflight(struct io_kiocb *req)
1194{
1195 struct io_ring_ctx *ctx = req->ctx;
1196
1197 if (!(req->flags & REQ_F_INFLIGHT)) {
ce3d5aae
PB
1198 req->flags |= REQ_F_INFLIGHT;
1199
1200 spin_lock_irq(&ctx->inflight_lock);
1201 list_add(&req->inflight_entry, &ctx->inflight_list);
1202 spin_unlock_irq(&ctx->inflight_lock);
1203 }
1204}
1205
1e6fa521
JA
1206static void io_prep_async_work(struct io_kiocb *req)
1207{
1208 const struct io_op_def *def = &io_op_defs[req->opcode];
1e6fa521
JA
1209 struct io_ring_ctx *ctx = req->ctx;
1210
003e8dcc
JA
1211 if (!req->work.creds)
1212 req->work.creds = get_current_cred();
1213
e1d675df
PB
1214 req->work.list.next = NULL;
1215 req->work.flags = 0;
feaadc4f
PB
1216 if (req->flags & REQ_F_FORCE_ASYNC)
1217 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1218
1e6fa521
JA
1219 if (req->flags & REQ_F_ISREG) {
1220 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
1221 io_wq_hash_work(&req->work, file_inode(req->file));
4b982bd0 1222 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
1e6fa521
JA
1223 if (def->unbound_nonreg_file)
1224 req->work.flags |= IO_WQ_WORK_UNBOUND;
1225 }
e1d675df
PB
1226
1227 switch (req->opcode) {
1228 case IORING_OP_SPLICE:
1229 case IORING_OP_TEE:
1230 /*
1231 * Splice operation will be punted aync, and here need to
1232 * modify io_wq_work.flags, so initialize io_wq_work firstly.
1233 */
1234 if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
1235 req->work.flags |= IO_WQ_WORK_UNBOUND;
1236 break;
1237 }
561fb04a 1238}
cccf0ee8 1239
cbdcb435 1240static void io_prep_async_link(struct io_kiocb *req)
561fb04a 1241{
cbdcb435 1242 struct io_kiocb *cur;
54a91f3b 1243
f2f87370
PB
1244 io_for_each_link(cur, req)
1245 io_prep_async_work(cur);
561fb04a
JA
1246}
1247
ebf93667 1248static void io_queue_async_work(struct io_kiocb *req)
561fb04a 1249{
a197f664 1250 struct io_ring_ctx *ctx = req->ctx;
cbdcb435 1251 struct io_kiocb *link = io_prep_linked_timeout(req);
5aa75ed5 1252 struct io_uring_task *tctx = req->task->io_uring;
561fb04a 1253
3bfe6106
JA
1254 BUG_ON(!tctx);
1255 BUG_ON(!tctx->io_wq);
561fb04a 1256
cbdcb435
PB
1257 /* init ->work of the whole link before punting */
1258 io_prep_async_link(req);
d07f1e8a
PB
1259 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1260 &req->work, req->flags);
ebf93667 1261 io_wq_enqueue(tctx->io_wq, &req->work);
7271ef3a
JA
1262 if (link)
1263 io_queue_linked_timeout(link);
cbdcb435
PB
1264}
1265
1ee4160c 1266static void io_kill_timeout(struct io_kiocb *req, int status)
5262f567 1267{
e8c2bc1f 1268 struct io_timeout_data *io = req->async_data;
5262f567
JA
1269 int ret;
1270
e8c2bc1f 1271 ret = hrtimer_try_to_cancel(&io->timer);
5262f567 1272 if (ret != -1) {
01cec8c1
PB
1273 atomic_set(&req->ctx->cq_timeouts,
1274 atomic_read(&req->ctx->cq_timeouts) + 1);
135fcde8 1275 list_del_init(&req->timeout.list);
1ee4160c 1276 io_cqring_fill_event(req, status);
216578e5 1277 io_put_req_deferred(req, 1);
5262f567
JA
1278 }
1279}
1280
04518945 1281static void __io_queue_deferred(struct io_ring_ctx *ctx)
de0617e4 1282{
04518945 1283 do {
27dc8338
PB
1284 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1285 struct io_defer_entry, list);
de0617e4 1286
9cf7c104 1287 if (req_need_defer(de->req, de->seq))
04518945 1288 break;
27dc8338 1289 list_del_init(&de->list);
907d1df3 1290 io_req_task_queue(de->req);
27dc8338 1291 kfree(de);
04518945
PB
1292 } while (!list_empty(&ctx->defer_list));
1293}
1294
360428f8 1295static void io_flush_timeouts(struct io_ring_ctx *ctx)
de0617e4 1296{
f010505b
MDG
1297 u32 seq;
1298
1299 if (list_empty(&ctx->timeout_list))
1300 return;
1301
1302 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1303
1304 do {
1305 u32 events_needed, events_got;
360428f8 1306 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
135fcde8 1307 struct io_kiocb, timeout.list);
de0617e4 1308
8eb7e2d0 1309 if (io_is_timeout_noseq(req))
360428f8 1310 break;
f010505b
MDG
1311
1312 /*
1313 * Since seq can easily wrap around over time, subtract
1314 * the last seq at which timeouts were flushed before comparing.
1315 * Assuming not more than 2^31-1 events have happened since,
1316 * these subtractions won't have wrapped, so we can check if
1317 * target is in [last_seq, current_seq] by comparing the two.
1318 */
1319 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1320 events_got = seq - ctx->cq_last_tm_flush;
1321 if (events_got < events_needed)
360428f8 1322 break;
bfe68a22 1323
135fcde8 1324 list_del_init(&req->timeout.list);
1ee4160c 1325 io_kill_timeout(req, 0);
f010505b
MDG
1326 } while (!list_empty(&ctx->timeout_list));
1327
1328 ctx->cq_last_tm_flush = seq;
360428f8 1329}
5262f567 1330
360428f8
PB
1331static void io_commit_cqring(struct io_ring_ctx *ctx)
1332{
1333 io_flush_timeouts(ctx);
ec30e04b
PB
1334
1335 /* order cqe stores with ring update */
1336 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
de0617e4 1337
04518945
PB
1338 if (unlikely(!list_empty(&ctx->defer_list)))
1339 __io_queue_deferred(ctx);
de0617e4
JA
1340}
1341
90554200
JA
1342static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1343{
1344 struct io_rings *r = ctx->rings;
1345
1346 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries;
1347}
1348
888aae2e
PB
1349static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1350{
1351 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1352}
1353
2b188cc1
JA
1354static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
1355{
75b28aff 1356 struct io_rings *rings = ctx->rings;
2b188cc1
JA
1357 unsigned tail;
1358
115e12e5
SB
1359 /*
1360 * writes to the cq entry need to come after reading head; the
1361 * control dependency is enough as we're using WRITE_ONCE to
1362 * fill the cq entry
1363 */
888aae2e 1364 if (__io_cqring_events(ctx) == rings->cq_ring_entries)
2b188cc1
JA
1365 return NULL;
1366
888aae2e 1367 tail = ctx->cached_cq_tail++;
75b28aff 1368 return &rings->cqes[tail & ctx->cq_mask];
2b188cc1
JA
1369}
1370
f2842ab5
JA
1371static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1372{
f0b493e6
JA
1373 if (!ctx->cq_ev_fd)
1374 return false;
7e55a19c
SG
1375 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1376 return false;
f2842ab5
JA
1377 if (!ctx->eventfd_async)
1378 return true;
b41e9852 1379 return io_wq_current_is_worker();
f2842ab5
JA
1380}
1381
b41e9852 1382static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1d7bb1d5 1383{
b1445e59
PB
1384 /* see waitqueue_active() comment */
1385 smp_mb();
1386
1d7bb1d5
JA
1387 if (waitqueue_active(&ctx->wait))
1388 wake_up(&ctx->wait);
534ca6d6
JA
1389 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1390 wake_up(&ctx->sq_data->wait);
b41e9852 1391 if (io_should_trigger_evfd(ctx))
1d7bb1d5 1392 eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59 1393 if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2f
PB
1394 wake_up_interruptible(&ctx->cq_wait);
1395 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1396 }
1d7bb1d5
JA
1397}
1398
80c18e4a
PB
1399static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1400{
b1445e59
PB
1401 /* see waitqueue_active() comment */
1402 smp_mb();
1403
80c18e4a
PB
1404 if (ctx->flags & IORING_SETUP_SQPOLL) {
1405 if (waitqueue_active(&ctx->wait))
1406 wake_up(&ctx->wait);
1407 }
1408 if (io_should_trigger_evfd(ctx))
1409 eventfd_signal(ctx->cq_ev_fd, 1);
b1445e59 1410 if (waitqueue_active(&ctx->cq_wait)) {
4aa84f2f
PB
1411 wake_up_interruptible(&ctx->cq_wait);
1412 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1413 }
80c18e4a
PB
1414}
1415
c4a2ed72 1416/* Returns true if there are no backlogged entries after the flush */
6c2450ae 1417static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1d7bb1d5
JA
1418{
1419 struct io_rings *rings = ctx->rings;
1d7bb1d5 1420 unsigned long flags;
b18032bb 1421 bool all_flushed, posted;
1d7bb1d5 1422
e23de15f
PB
1423 if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries)
1424 return false;
1d7bb1d5 1425
b18032bb 1426 posted = false;
1d7bb1d5 1427 spin_lock_irqsave(&ctx->completion_lock, flags);
6c2450ae
PB
1428 while (!list_empty(&ctx->cq_overflow_list)) {
1429 struct io_uring_cqe *cqe = io_get_cqring(ctx);
1430 struct io_overflow_cqe *ocqe;
e6c8aa9a 1431
1d7bb1d5
JA
1432 if (!cqe && !force)
1433 break;
6c2450ae
PB
1434 ocqe = list_first_entry(&ctx->cq_overflow_list,
1435 struct io_overflow_cqe, list);
1436 if (cqe)
1437 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1438 else
1d7bb1d5 1439 WRITE_ONCE(ctx->rings->cq_overflow,
6c2450ae 1440 ++ctx->cached_cq_overflow);
b18032bb 1441 posted = true;
6c2450ae
PB
1442 list_del(&ocqe->list);
1443 kfree(ocqe);
1d7bb1d5
JA
1444 }
1445
09e88404
PB
1446 all_flushed = list_empty(&ctx->cq_overflow_list);
1447 if (all_flushed) {
1448 clear_bit(0, &ctx->sq_check_overflow);
1449 clear_bit(0, &ctx->cq_check_overflow);
1450 ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1451 }
46930143 1452
b18032bb
JA
1453 if (posted)
1454 io_commit_cqring(ctx);
1d7bb1d5 1455 spin_unlock_irqrestore(&ctx->completion_lock, flags);
b18032bb
JA
1456 if (posted)
1457 io_cqring_ev_posted(ctx);
09e88404 1458 return all_flushed;
1d7bb1d5
JA
1459}
1460
6c2450ae 1461static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
6c503150 1462{
ca0a2651
JA
1463 bool ret = true;
1464
6c503150
PB
1465 if (test_bit(0, &ctx->cq_check_overflow)) {
1466 /* iopoll syncs against uring_lock, not completion_lock */
1467 if (ctx->flags & IORING_SETUP_IOPOLL)
1468 mutex_lock(&ctx->uring_lock);
6c2450ae 1469 ret = __io_cqring_overflow_flush(ctx, force);
6c503150
PB
1470 if (ctx->flags & IORING_SETUP_IOPOLL)
1471 mutex_unlock(&ctx->uring_lock);
1472 }
ca0a2651
JA
1473
1474 return ret;
6c503150
PB
1475}
1476
abc54d63
JA
1477/*
1478 * Shamelessly stolen from the mm implementation of page reference checking,
1479 * see commit f958d7b528b1 for details.
1480 */
1481#define req_ref_zero_or_close_to_overflow(req) \
1482 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1483
de9b4cca
JA
1484static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1485{
abc54d63 1486 return atomic_inc_not_zero(&req->refs);
de9b4cca
JA
1487}
1488
1489static inline bool req_ref_sub_and_test(struct io_kiocb *req, int refs)
1490{
abc54d63
JA
1491 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1492 return atomic_sub_and_test(refs, &req->refs);
de9b4cca
JA
1493}
1494
1495static inline bool req_ref_put_and_test(struct io_kiocb *req)
1496{
abc54d63
JA
1497 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1498 return atomic_dec_and_test(&req->refs);
de9b4cca
JA
1499}
1500
1501static inline void req_ref_put(struct io_kiocb *req)
1502{
abc54d63 1503 WARN_ON_ONCE(req_ref_put_and_test(req));
de9b4cca
JA
1504}
1505
1506static inline void req_ref_get(struct io_kiocb *req)
1507{
abc54d63
JA
1508 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1509 atomic_inc(&req->refs);
de9b4cca
JA
1510}
1511
5082620f 1512static bool __io_cqring_fill_event(struct io_kiocb *req, long res,
8c3f9cd1 1513 unsigned int cflags)
2b188cc1 1514{
78e19bbe 1515 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA
1516 struct io_uring_cqe *cqe;
1517
7471e1af 1518 trace_io_uring_complete(ctx, req->user_data, res, cflags);
51c3ff62 1519
2b188cc1
JA
1520 /*
1521 * If we can't get a cq entry, userspace overflowed the
1522 * submission (by quite a lot). Increment the overflow count in
1523 * the ring.
1524 */
1525 cqe = io_get_cqring(ctx);
1d7bb1d5 1526 if (likely(cqe)) {
78e19bbe 1527 WRITE_ONCE(cqe->user_data, req->user_data);
2b188cc1 1528 WRITE_ONCE(cqe->res, res);
bcda7baa 1529 WRITE_ONCE(cqe->flags, cflags);
5082620f 1530 return true;
6c2450ae
PB
1531 }
1532 if (!ctx->cq_overflow_flushed &&
1533 !atomic_read(&req->task->io_uring->in_idle)) {
1534 struct io_overflow_cqe *ocqe;
1535
1536 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1537 if (!ocqe)
1538 goto overflow;
ad3eb2c8
JA
1539 if (list_empty(&ctx->cq_overflow_list)) {
1540 set_bit(0, &ctx->sq_check_overflow);
1541 set_bit(0, &ctx->cq_check_overflow);
6d5f9049 1542 ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
ad3eb2c8 1543 }
6c2450ae
PB
1544 ocqe->cqe.user_data = req->user_data;
1545 ocqe->cqe.res = res;
1546 ocqe->cqe.flags = cflags;
1547 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
5082620f 1548 return true;
2b188cc1 1549 }
6c2450ae
PB
1550overflow:
1551 /*
1552 * If we're in ring overflow flush mode, or in task cancel mode,
1553 * or cannot allocate an overflow entry, then we need to drop it
1554 * on the floor.
1555 */
1556 WRITE_ONCE(ctx->rings->cq_overflow, ++ctx->cached_cq_overflow);
5082620f 1557 return false;
2b188cc1
JA
1558}
1559
bcda7baa
JA
1560static void io_cqring_fill_event(struct io_kiocb *req, long res)
1561{
1562 __io_cqring_fill_event(req, res, 0);
1563}
1564
7a612350
PB
1565static void io_req_complete_post(struct io_kiocb *req, long res,
1566 unsigned int cflags)
2b188cc1 1567{
78e19bbe 1568 struct io_ring_ctx *ctx = req->ctx;
2b188cc1
JA
1569 unsigned long flags;
1570
1571 spin_lock_irqsave(&ctx->completion_lock, flags);
bcda7baa 1572 __io_cqring_fill_event(req, res, cflags);
c7dae4ba
JA
1573 /*
1574 * If we're the last reference to this request, add to our locked
1575 * free_list cache.
1576 */
de9b4cca 1577 if (req_ref_put_and_test(req)) {
c7dae4ba
JA
1578 struct io_comp_state *cs = &ctx->submit_state.comp;
1579
7a612350
PB
1580 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
1581 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK))
1582 io_disarm_next(req);
1583 if (req->link) {
1584 io_req_task_queue(req->link);
1585 req->link = NULL;
1586 }
1587 }
c7dae4ba
JA
1588 io_dismantle_req(req);
1589 io_put_task(req->task, 1);
1590 list_add(&req->compl.list, &cs->locked_free_list);
1591 cs->locked_free_nr++;
180f829f
PB
1592 } else {
1593 if (!percpu_ref_tryget(&ctx->refs))
1594 req = NULL;
1595 }
7a612350 1596 io_commit_cqring(ctx);
2b188cc1 1597 spin_unlock_irqrestore(&ctx->completion_lock, flags);
7a612350 1598
180f829f
PB
1599 if (req) {
1600 io_cqring_ev_posted(ctx);
c7dae4ba 1601 percpu_ref_put(&ctx->refs);
180f829f 1602 }
229a7b63
JA
1603}
1604
a38d68db 1605static void io_req_complete_state(struct io_kiocb *req, long res,
889fca73 1606 unsigned int cflags)
229a7b63 1607{
68fb8979
PB
1608 if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED))
1609 io_clean_op(req);
a38d68db
PB
1610 req->result = res;
1611 req->compl.cflags = cflags;
e342c807 1612 req->flags |= REQ_F_COMPLETE_INLINE;
e1e16097
JA
1613}
1614
889fca73
PB
1615static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1616 long res, unsigned cflags)
bcda7baa 1617{
889fca73
PB
1618 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1619 io_req_complete_state(req, res, cflags);
a38d68db 1620 else
c7dae4ba 1621 io_req_complete_post(req, res, cflags);
bcda7baa
JA
1622}
1623
a38d68db 1624static inline void io_req_complete(struct io_kiocb *req, long res)
0ddf92e8 1625{
889fca73 1626 __io_req_complete(req, 0, res, 0);
0ddf92e8
JA
1627}
1628
f41db273
PB
1629static void io_req_complete_failed(struct io_kiocb *req, long res)
1630{
1631 req_set_fail_links(req);
1632 io_put_req(req);
1633 io_req_complete_post(req, res, 0);
1634}
1635
dac7a098
PB
1636static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
1637 struct io_comp_state *cs)
1638{
1639 spin_lock_irq(&ctx->completion_lock);
1640 list_splice_init(&cs->locked_free_list, &cs->free_list);
1641 cs->locked_free_nr = 0;
1642 spin_unlock_irq(&ctx->completion_lock);
1643}
1644
dd78f492 1645/* Returns true IFF there are requests in the cache */
c7dae4ba 1646static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
0ddf92e8 1647{
c7dae4ba
JA
1648 struct io_submit_state *state = &ctx->submit_state;
1649 struct io_comp_state *cs = &state->comp;
dd78f492 1650 int nr;
0ddf92e8 1651
c7dae4ba
JA
1652 /*
1653 * If we have more than a batch's worth of requests in our IRQ side
1654 * locked cache, grab the lock and move them over to our submission
1655 * side cache.
1656 */
dac7a098
PB
1657 if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH)
1658 io_flush_cached_locked_reqs(ctx, cs);
0ddf92e8 1659
dd78f492 1660 nr = state->free_reqs;
c7dae4ba 1661 while (!list_empty(&cs->free_list)) {
dd78f492
PB
1662 struct io_kiocb *req = list_first_entry(&cs->free_list,
1663 struct io_kiocb, compl.list);
1664
1b4c351f 1665 list_del(&req->compl.list);
dd78f492
PB
1666 state->reqs[nr++] = req;
1667 if (nr == ARRAY_SIZE(state->reqs))
e5d1bc0a 1668 break;
1b4c351f
JA
1669 }
1670
dd78f492
PB
1671 state->free_reqs = nr;
1672 return nr != 0;
0ddf92e8
JA
1673}
1674
e5d1bc0a 1675static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
2b188cc1 1676{
e5d1bc0a
PB
1677 struct io_submit_state *state = &ctx->submit_state;
1678
1679 BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs));
1680
f6b6c7d6 1681 if (!state->free_reqs) {
291b2821 1682 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
2579f913
JA
1683 int ret;
1684
c7dae4ba 1685 if (io_flush_cached_reqs(ctx))
e5d1bc0a
PB
1686 goto got_req;
1687
bf019da7
PB
1688 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1689 state->reqs);
fd6fab2c
JA
1690
1691 /*
1692 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1693 * retry single alloc to be on the safe side.
1694 */
1695 if (unlikely(ret <= 0)) {
1696 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1697 if (!state->reqs[0])
3893f39f 1698 return NULL;
fd6fab2c
JA
1699 ret = 1;
1700 }
291b2821 1701 state->free_reqs = ret;
2b188cc1 1702 }
e5d1bc0a 1703got_req:
291b2821
PB
1704 state->free_reqs--;
1705 return state->reqs[state->free_reqs];
2b188cc1
JA
1706}
1707
e1d767f0 1708static inline void io_put_file(struct file *file)
8da11c19 1709{
e1d767f0 1710 if (file)
8da11c19
PB
1711 fput(file);
1712}
1713
4edf20f9 1714static void io_dismantle_req(struct io_kiocb *req)
2b188cc1 1715{
094bae49
PB
1716 unsigned int flags = req->flags;
1717
e1d767f0
PB
1718 if (!(flags & REQ_F_FIXED_FILE))
1719 io_put_file(req->file);
094bae49
PB
1720 if (flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED |
1721 REQ_F_INFLIGHT)) {
1722 io_clean_op(req);
1723
1724 if (req->flags & REQ_F_INFLIGHT) {
1725 struct io_ring_ctx *ctx = req->ctx;
1726 unsigned long flags;
1727
1728 spin_lock_irqsave(&ctx->inflight_lock, flags);
1729 list_del(&req->inflight_entry);
1730 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1731 req->flags &= ~REQ_F_INFLIGHT;
1732 }
1733 }
269bbe5f
BM
1734 if (req->fixed_rsrc_refs)
1735 percpu_ref_put(req->fixed_rsrc_refs);
094bae49
PB
1736 if (req->async_data)
1737 kfree(req->async_data);
003e8dcc
JA
1738 if (req->work.creds) {
1739 put_cred(req->work.creds);
1740 req->work.creds = NULL;
1741 }
e65ef56d
JA
1742}
1743
b23fcf47 1744/* must to be called somewhat shortly after putting a request */
7c660731
PB
1745static inline void io_put_task(struct task_struct *task, int nr)
1746{
1747 struct io_uring_task *tctx = task->io_uring;
1748
1749 percpu_counter_sub(&tctx->inflight, nr);
1750 if (unlikely(atomic_read(&tctx->in_idle)))
1751 wake_up(&tctx->wait);
1752 put_task_struct_many(task, nr);
1753}
1754
216578e5 1755static void __io_free_req(struct io_kiocb *req)
c6ca97b3 1756{
51a4cc11 1757 struct io_ring_ctx *ctx = req->ctx;
c6ca97b3 1758
216578e5 1759 io_dismantle_req(req);
7c660731 1760 io_put_task(req->task, 1);
c6ca97b3 1761
3893f39f 1762 kmem_cache_free(req_cachep, req);
ecfc5177 1763 percpu_ref_put(&ctx->refs);
e65ef56d
JA
1764}
1765
f2f87370
PB
1766static inline void io_remove_next_linked(struct io_kiocb *req)
1767{
1768 struct io_kiocb *nxt = req->link;
1769
1770 req->link = nxt->link;
1771 nxt->link = NULL;
1772}
1773
33cc89a9
PB
1774static bool io_kill_linked_timeout(struct io_kiocb *req)
1775 __must_hold(&req->ctx->completion_lock)
2665abfd 1776{
33cc89a9 1777 struct io_kiocb *link = req->link;
f2f87370 1778
900fad45
PB
1779 /*
1780 * Can happen if a linked timeout fired and link had been like
1781 * req -> link t-out -> link t-out [-> ...]
1782 */
c9abd7ad
PB
1783 if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
1784 struct io_timeout_data *io = link->async_data;
1785 int ret;
7c86ffee 1786
f2f87370 1787 io_remove_next_linked(req);
90cd7e42 1788 link->timeout.head = NULL;
c9abd7ad
PB
1789 ret = hrtimer_try_to_cancel(&io->timer);
1790 if (ret != -1) {
1791 io_cqring_fill_event(link, -ECANCELED);
33cc89a9 1792 io_put_req_deferred(link, 1);
d4729fbd 1793 return true;
c9abd7ad
PB
1794 }
1795 }
d4729fbd 1796 return false;
7c86ffee
PB
1797}
1798
d148ca4b 1799static void io_fail_links(struct io_kiocb *req)
33cc89a9 1800 __must_hold(&req->ctx->completion_lock)
9e645e11 1801{
33cc89a9 1802 struct io_kiocb *nxt, *link = req->link;
9e645e11 1803
f2f87370 1804 req->link = NULL;
f2f87370
PB
1805 while (link) {
1806 nxt = link->link;
1807 link->link = NULL;
2665abfd 1808
f2f87370 1809 trace_io_uring_fail_link(req, link);
7c86ffee 1810 io_cqring_fill_event(link, -ECANCELED);
1575f21a 1811 io_put_req_deferred(link, 2);
f2f87370 1812 link = nxt;
9e645e11 1813 }
33cc89a9 1814}
9e645e11 1815
33cc89a9
PB
1816static bool io_disarm_next(struct io_kiocb *req)
1817 __must_hold(&req->ctx->completion_lock)
1818{
1819 bool posted = false;
1820
1821 if (likely(req->flags & REQ_F_LINK_TIMEOUT))
1822 posted = io_kill_linked_timeout(req);
1823 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
1824 posted |= (req->link != NULL);
1825 io_fail_links(req);
1826 }
1827 return posted;
9e645e11
JA
1828}
1829
3fa5e0f3 1830static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
c69f8dbe 1831{
33cc89a9 1832 struct io_kiocb *nxt;
944e58bf 1833
9e645e11
JA
1834 /*
1835 * If LINK is set, we have dependent requests in this chain. If we
1836 * didn't fail this request, queue the first one up, moving any other
1837 * dependencies to the next request. In case of failure, fail the rest
1838 * of the chain.
1839 */
33cc89a9
PB
1840 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) {
1841 struct io_ring_ctx *ctx = req->ctx;
1842 unsigned long flags;
1843 bool posted;
1844
1845 spin_lock_irqsave(&ctx->completion_lock, flags);
1846 posted = io_disarm_next(req);
1847 if (posted)
1848 io_commit_cqring(req->ctx);
1849 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1850 if (posted)
1851 io_cqring_ev_posted(ctx);
f2f87370 1852 }
33cc89a9
PB
1853 nxt = req->link;
1854 req->link = NULL;
1855 return nxt;
4d7dd462 1856}
9e645e11 1857
f2f87370 1858static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
3fa5e0f3 1859{
cdbff982 1860 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
3fa5e0f3
PB
1861 return NULL;
1862 return __io_req_find_next(req);
1863}
1864
2c32395d
PB
1865static void ctx_flush_and_put(struct io_ring_ctx *ctx)
1866{
1867 if (!ctx)
1868 return;
1869 if (ctx->submit_state.comp.nr) {
1870 mutex_lock(&ctx->uring_lock);
1871 io_submit_flush_completions(&ctx->submit_state.comp, ctx);
1872 mutex_unlock(&ctx->uring_lock);
1873 }
1874 percpu_ref_put(&ctx->refs);
1875}
1876
7cbf1722 1877static bool __tctx_task_work(struct io_uring_task *tctx)
c2c4c83c 1878{
65453d1e 1879 struct io_ring_ctx *ctx = NULL;
7cbf1722
JA
1880 struct io_wq_work_list list;
1881 struct io_wq_work_node *node;
c2c4c83c 1882
7cbf1722
JA
1883 if (wq_list_empty(&tctx->task_list))
1884 return false;
6200b0ae 1885
0b81e80c 1886 spin_lock_irq(&tctx->task_lock);
7cbf1722
JA
1887 list = tctx->task_list;
1888 INIT_WQ_LIST(&tctx->task_list);
0b81e80c 1889 spin_unlock_irq(&tctx->task_lock);
c2c4c83c 1890
7cbf1722
JA
1891 node = list.first;
1892 while (node) {
1893 struct io_wq_work_node *next = node->next;
1894 struct io_kiocb *req;
0ba9c9ed 1895
7cbf1722 1896 req = container_of(node, struct io_kiocb, io_task_work.node);
2c32395d
PB
1897 if (req->ctx != ctx) {
1898 ctx_flush_and_put(ctx);
1899 ctx = req->ctx;
1900 percpu_ref_get(&ctx->refs);
65453d1e 1901 }
65453d1e 1902
2c32395d
PB
1903 req->task_work.func(&req->task_work);
1904 node = next;
7cbf1722
JA
1905 }
1906
2c32395d 1907 ctx_flush_and_put(ctx);
7cbf1722 1908 return list.first != NULL;
c2c4c83c
JA
1909}
1910
7cbf1722 1911static void tctx_task_work(struct callback_head *cb)
c40f6379 1912{
7cbf1722 1913 struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work);
c40f6379 1914
1d5f360d
JA
1915 clear_bit(0, &tctx->task_state);
1916
7cbf1722
JA
1917 while (__tctx_task_work(tctx))
1918 cond_resched();
7cbf1722
JA
1919}
1920
c15b79de 1921static int io_req_task_work_add(struct io_kiocb *req)
7cbf1722 1922{
c15b79de 1923 struct task_struct *tsk = req->task;
7cbf1722 1924 struct io_uring_task *tctx = tsk->io_uring;
c15b79de 1925 enum task_work_notify_mode notify;
7cbf1722 1926 struct io_wq_work_node *node, *prev;
0b81e80c 1927 unsigned long flags;
c15b79de
PB
1928 int ret = 0;
1929
1930 if (unlikely(tsk->flags & PF_EXITING))
1931 return -ESRCH;
7cbf1722
JA
1932
1933 WARN_ON_ONCE(!tctx);
1934
0b81e80c 1935 spin_lock_irqsave(&tctx->task_lock, flags);
7cbf1722 1936 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
0b81e80c 1937 spin_unlock_irqrestore(&tctx->task_lock, flags);
7cbf1722
JA
1938
1939 /* task_work already pending, we're done */
1940 if (test_bit(0, &tctx->task_state) ||
1941 test_and_set_bit(0, &tctx->task_state))
1942 return 0;
1943
c15b79de
PB
1944 /*
1945 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
1946 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
1947 * processing task_work. There's no reliable way to tell if TWA_RESUME
1948 * will do the job.
1949 */
1950 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
1951
1952 if (!task_work_add(tsk, &tctx->task_work, notify)) {
1953 wake_up_process(tsk);
7cbf1722 1954 return 0;
c15b79de 1955 }
7cbf1722
JA
1956
1957 /*
1958 * Slow path - we failed, find and delete work. if the work is not
1959 * in the list, it got run and we're fine.
1960 */
0b81e80c 1961 spin_lock_irqsave(&tctx->task_lock, flags);
7cbf1722
JA
1962 wq_list_for_each(node, prev, &tctx->task_list) {
1963 if (&req->io_task_work.node == node) {
1964 wq_list_del(&tctx->task_list, node, prev);
1965 ret = 1;
1966 break;
1967 }
1968 }
0b81e80c 1969 spin_unlock_irqrestore(&tctx->task_lock, flags);
7cbf1722
JA
1970 clear_bit(0, &tctx->task_state);
1971 return ret;
1972}
1973
9b465711
PB
1974static bool io_run_task_work_head(struct callback_head **work_head)
1975{
1976 struct callback_head *work, *next;
1977 bool executed = false;
1978
1979 do {
1980 work = xchg(work_head, NULL);
1981 if (!work)
1982 break;
1983
1984 do {
1985 next = work->next;
1986 work->func(work);
1987 work = next;
1988 cond_resched();
1989 } while (work);
1990 executed = true;
1991 } while (1);
1992
1993 return executed;
1994}
1995
1996static void io_task_work_add_head(struct callback_head **work_head,
1997 struct callback_head *task_work)
eab30c4d 1998{
7c25c0d1 1999 struct callback_head *head;
eab30c4d 2000
7c25c0d1 2001 do {
9b465711
PB
2002 head = READ_ONCE(*work_head);
2003 task_work->next = head;
2004 } while (cmpxchg(work_head, head, task_work) != head);
2005}
2006
2007static void io_req_task_work_add_fallback(struct io_kiocb *req,
2008 task_work_func_t cb)
2009{
2010 init_task_work(&req->task_work, cb);
2011 io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work);
eab30c4d
PB
2012}
2013
c40f6379
JA
2014static void io_req_task_cancel(struct callback_head *cb)
2015{
2016 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
87ceb6a6 2017 struct io_ring_ctx *ctx = req->ctx;
c40f6379 2018
e83acd7d 2019 /* ctx is guaranteed to stay alive while we hold uring_lock */
792bb6eb 2020 mutex_lock(&ctx->uring_lock);
2593553a 2021 io_req_complete_failed(req, req->result);
792bb6eb 2022 mutex_unlock(&ctx->uring_lock);
c40f6379
JA
2023}
2024
2025static void __io_req_task_submit(struct io_kiocb *req)
2026{
2027 struct io_ring_ctx *ctx = req->ctx;
2028
04fc6c80 2029 /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
81b6d05c 2030 mutex_lock(&ctx->uring_lock);
70aacfe6 2031 if (!(current->flags & PF_EXITING) && !current->in_execve)
c5eef2b9 2032 __io_queue_sqe(req);
81b6d05c 2033 else
2593553a 2034 io_req_complete_failed(req, -EFAULT);
81b6d05c 2035 mutex_unlock(&ctx->uring_lock);
c40f6379
JA
2036}
2037
2038static void io_req_task_submit(struct callback_head *cb)
2039{
2040 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2041
2042 __io_req_task_submit(req);
2043}
2044
2c4b8eb6 2045static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
c40f6379 2046{
2c4b8eb6
PB
2047 req->result = ret;
2048 req->task_work.func = io_req_task_cancel;
c40f6379 2049
2c4b8eb6 2050 if (unlikely(io_req_task_work_add(req)))
eab30c4d 2051 io_req_task_work_add_fallback(req, io_req_task_cancel);
c40f6379
JA
2052}
2053
2c4b8eb6 2054static void io_req_task_queue(struct io_kiocb *req)
a3df7698 2055{
2c4b8eb6 2056 req->task_work.func = io_req_task_submit;
a3df7698
PB
2057
2058 if (unlikely(io_req_task_work_add(req)))
2c4b8eb6 2059 io_req_task_queue_fail(req, -ECANCELED);
a3df7698
PB
2060}
2061
f2f87370 2062static inline void io_queue_next(struct io_kiocb *req)
c69f8dbe 2063{
9b5f7bd9 2064 struct io_kiocb *nxt = io_req_find_next(req);
944e58bf
PB
2065
2066 if (nxt)
906a8c3f 2067 io_req_task_queue(nxt);
c69f8dbe
JL
2068}
2069
c3524383 2070static void io_free_req(struct io_kiocb *req)
7a743e22 2071{
c3524383
PB
2072 io_queue_next(req);
2073 __io_free_req(req);
2074}
8766dd51 2075
2d6500d4 2076struct req_batch {
5af1d13e
PB
2077 struct task_struct *task;
2078 int task_refs;
1b4c351f 2079 int ctx_refs;
2d6500d4
PB
2080};
2081
5af1d13e
PB
2082static inline void io_init_req_batch(struct req_batch *rb)
2083{
5af1d13e 2084 rb->task_refs = 0;
9ae72463 2085 rb->ctx_refs = 0;
5af1d13e
PB
2086 rb->task = NULL;
2087}
2088
2d6500d4
PB
2089static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2090 struct req_batch *rb)
2091{
6e833d53 2092 if (rb->task)
7c660731 2093 io_put_task(rb->task, rb->task_refs);
9ae72463
PB
2094 if (rb->ctx_refs)
2095 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
2d6500d4
PB
2096}
2097
6ff119a6
PB
2098static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2099 struct io_submit_state *state)
2d6500d4 2100{
f2f87370 2101 io_queue_next(req);
96670657 2102 io_dismantle_req(req);
2d6500d4 2103
e3bc8e9d 2104 if (req->task != rb->task) {
7c660731
PB
2105 if (rb->task)
2106 io_put_task(rb->task, rb->task_refs);
e3bc8e9d
JA
2107 rb->task = req->task;
2108 rb->task_refs = 0;
5af1d13e 2109 }
e3bc8e9d 2110 rb->task_refs++;
9ae72463 2111 rb->ctx_refs++;
5af1d13e 2112
bd759045 2113 if (state->free_reqs != ARRAY_SIZE(state->reqs))
6ff119a6 2114 state->reqs[state->free_reqs++] = req;
bd759045
PB
2115 else
2116 list_add(&req->compl.list, &state->comp.free_list);
7a743e22
PB
2117}
2118
905c172f
PB
2119static void io_submit_flush_completions(struct io_comp_state *cs,
2120 struct io_ring_ctx *ctx)
2121{
2122 int i, nr = cs->nr;
2123 struct io_kiocb *req;
2124 struct req_batch rb;
2125
2126 io_init_req_batch(&rb);
2127 spin_lock_irq(&ctx->completion_lock);
2128 for (i = 0; i < nr; i++) {
2129 req = cs->reqs[i];
2130 __io_cqring_fill_event(req, req->result, req->compl.cflags);
2131 }
2132 io_commit_cqring(ctx);
2133 spin_unlock_irq(&ctx->completion_lock);
2134
2135 io_cqring_ev_posted(ctx);
2136 for (i = 0; i < nr; i++) {
2137 req = cs->reqs[i];
2138
2139 /* submission and completion refs */
de9b4cca 2140 if (req_ref_sub_and_test(req, 2))
6ff119a6 2141 io_req_free_batch(&rb, req, &ctx->submit_state);
905c172f
PB
2142 }
2143
2144 io_req_free_batch_finish(ctx, &rb);
2145 cs->nr = 0;
7a743e22
PB
2146}
2147
ba816ad6
JA
2148/*
2149 * Drop reference to request, return next in chain (if there is one) if this
2150 * was the last reference to this request.
2151 */
0d85035a 2152static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
e65ef56d 2153{
9b5f7bd9
PB
2154 struct io_kiocb *nxt = NULL;
2155
de9b4cca 2156 if (req_ref_put_and_test(req)) {
9b5f7bd9 2157 nxt = io_req_find_next(req);
4d7dd462 2158 __io_free_req(req);
2a44f467 2159 }
9b5f7bd9 2160 return nxt;
2b188cc1
JA
2161}
2162
0d85035a 2163static inline void io_put_req(struct io_kiocb *req)
e65ef56d 2164{
de9b4cca 2165 if (req_ref_put_and_test(req))
e65ef56d 2166 io_free_req(req);
2b188cc1
JA
2167}
2168
216578e5
PB
2169static void io_put_req_deferred_cb(struct callback_head *cb)
2170{
2171 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2172
2173 io_free_req(req);
2174}
2175
2176static void io_free_req_deferred(struct io_kiocb *req)
2177{
7cbf1722 2178 req->task_work.func = io_put_req_deferred_cb;
a05432fb 2179 if (unlikely(io_req_task_work_add(req)))
eab30c4d 2180 io_req_task_work_add_fallback(req, io_put_req_deferred_cb);
216578e5
PB
2181}
2182
2183static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
2184{
de9b4cca 2185 if (req_ref_sub_and_test(req, refs))
216578e5
PB
2186 io_free_req_deferred(req);
2187}
2188
6c503150 2189static unsigned io_cqring_events(struct io_ring_ctx *ctx)
a3a0e43f
JA
2190{
2191 /* See comment at the top of this file */
2192 smp_rmb();
e23de15f 2193 return __io_cqring_events(ctx);
a3a0e43f
JA
2194}
2195
fb5ccc98
PB
2196static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2197{
2198 struct io_rings *rings = ctx->rings;
2199
2200 /* make sure SQ entry isn't read before tail */
2201 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2202}
2203
8ff069bf 2204static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
e94f141b 2205{
8ff069bf 2206 unsigned int cflags;
e94f141b 2207
bcda7baa
JA
2208 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2209 cflags |= IORING_CQE_F_BUFFER;
0e1b6fe3 2210 req->flags &= ~REQ_F_BUFFER_SELECTED;
bcda7baa
JA
2211 kfree(kbuf);
2212 return cflags;
e94f141b
JA
2213}
2214
8ff069bf 2215static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
bcda7baa 2216{
4d954c25 2217 struct io_buffer *kbuf;
bcda7baa 2218
4d954c25 2219 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
8ff069bf
PB
2220 return io_put_kbuf(req, kbuf);
2221}
2222
4c6e277c
JA
2223static inline bool io_run_task_work(void)
2224{
6200b0ae
JA
2225 /*
2226 * Not safe to run on exiting task, and the task_work handling will
2227 * not add work to such a task.
2228 */
2229 if (unlikely(current->flags & PF_EXITING))
2230 return false;
4c6e277c
JA
2231 if (current->task_works) {
2232 __set_current_state(TASK_RUNNING);
2233 task_work_run();
2234 return true;
2235 }
2236
2237 return false;
bcda7baa
JA
2238}
2239
def596e9
JA
2240/*
2241 * Find and free completed poll iocbs
2242 */
2243static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
2244 struct list_head *done)
2245{
8237e045 2246 struct req_batch rb;
def596e9 2247 struct io_kiocb *req;
bbde017a
XW
2248
2249 /* order with ->result store in io_complete_rw_iopoll() */
2250 smp_rmb();
def596e9 2251
5af1d13e 2252 io_init_req_batch(&rb);
def596e9 2253 while (!list_empty(done)) {
bcda7baa
JA
2254 int cflags = 0;
2255
d21ffe7e 2256 req = list_first_entry(done, struct io_kiocb, inflight_entry);
f161340d
PB
2257 list_del(&req->inflight_entry);
2258
8c130827
PB
2259 if (READ_ONCE(req->result) == -EAGAIN &&
2260 !(req->flags & REQ_F_DONT_REISSUE)) {
bbde017a 2261 req->iopoll_completed = 0;
8c130827
PB
2262 req_ref_get(req);
2263 io_queue_async_work(req);
2264 continue;
bbde017a 2265 }
def596e9 2266
bcda7baa 2267 if (req->flags & REQ_F_BUFFER_SELECTED)
8ff069bf 2268 cflags = io_put_rw_kbuf(req);
bcda7baa
JA
2269
2270 __io_cqring_fill_event(req, req->result, cflags);
def596e9
JA
2271 (*nr_events)++;
2272
de9b4cca 2273 if (req_ref_put_and_test(req))
6ff119a6 2274 io_req_free_batch(&rb, req, &ctx->submit_state);
def596e9 2275 }
def596e9 2276
09bb8394 2277 io_commit_cqring(ctx);
80c18e4a 2278 io_cqring_ev_posted_iopoll(ctx);
2d6500d4 2279 io_req_free_batch_finish(ctx, &rb);
581f9810
BM
2280}
2281
def596e9
JA
2282static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
2283 long min)
2284{
2285 struct io_kiocb *req, *tmp;
2286 LIST_HEAD(done);
2287 bool spin;
2288 int ret;
2289
2290 /*
2291 * Only spin for completions if we don't have multiple devices hanging
2292 * off our complete list, and we're under the requested amount.
2293 */
2294 spin = !ctx->poll_multi_file && *nr_events < min;
2295
2296 ret = 0;
d21ffe7e 2297 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
9adbd45d 2298 struct kiocb *kiocb = &req->rw.kiocb;
def596e9
JA
2299
2300 /*
581f9810
BM
2301 * Move completed and retryable entries to our local lists.
2302 * If we find a request that requires polling, break out
2303 * and complete those lists first, if we have entries there.
def596e9 2304 */
65a6543d 2305 if (READ_ONCE(req->iopoll_completed)) {
d21ffe7e 2306 list_move_tail(&req->inflight_entry, &done);
def596e9
JA
2307 continue;
2308 }
2309 if (!list_empty(&done))
2310 break;
2311
2312 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
2313 if (ret < 0)
2314 break;
2315
3aadc23e
PB
2316 /* iopoll may have completed current req */
2317 if (READ_ONCE(req->iopoll_completed))
d21ffe7e 2318 list_move_tail(&req->inflight_entry, &done);
3aadc23e 2319
def596e9
JA
2320 if (ret && spin)
2321 spin = false;
2322 ret = 0;
2323 }
2324
2325 if (!list_empty(&done))
2326 io_iopoll_complete(ctx, nr_events, &done);
2327
2328 return ret;
2329}
2330
2331/*
d195a66e 2332 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
def596e9
JA
2333 * non-spinning poll check - we'll still enter the driver poll loop, but only
2334 * as a non-spinning completion check.
2335 */
2336static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
2337 long min)
2338{
540e32a0 2339 while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
def596e9
JA
2340 int ret;
2341
2342 ret = io_do_iopoll(ctx, nr_events, min);
2343 if (ret < 0)
2344 return ret;
eba0a4dd 2345 if (*nr_events >= min)
def596e9
JA
2346 return 0;
2347 }
2348
2349 return 1;
2350}
2351
2352/*
2353 * We can't just wait for polled events to come to us, we have to actively
2354 * find and complete them.
2355 */
b2edc0a7 2356static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
def596e9
JA
2357{
2358 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2359 return;
2360
2361 mutex_lock(&ctx->uring_lock);
540e32a0 2362 while (!list_empty(&ctx->iopoll_list)) {
def596e9
JA
2363 unsigned int nr_events = 0;
2364
b2edc0a7 2365 io_do_iopoll(ctx, &nr_events, 0);
08f5439f 2366
b2edc0a7
PB
2367 /* let it sleep and repeat later if can't complete a request */
2368 if (nr_events == 0)
2369 break;
08f5439f
JA
2370 /*
2371 * Ensure we allow local-to-the-cpu processing to take place,
2372 * in this case we need to ensure that we reap all events.
3fcee5a6 2373 * Also let task_work, etc. to progress by releasing the mutex
08f5439f 2374 */
3fcee5a6
PB
2375 if (need_resched()) {
2376 mutex_unlock(&ctx->uring_lock);
2377 cond_resched();
2378 mutex_lock(&ctx->uring_lock);
2379 }
def596e9
JA
2380 }
2381 mutex_unlock(&ctx->uring_lock);
2382}
2383
7668b92a 2384static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
def596e9 2385{
7668b92a 2386 unsigned int nr_events = 0;
2b2ed975 2387 int iters = 0, ret = 0;
500f9fba 2388
c7849be9
XW
2389 /*
2390 * We disallow the app entering submit/complete with polling, but we
2391 * still need to lock the ring to prevent racing with polled issue
2392 * that got punted to a workqueue.
2393 */
2394 mutex_lock(&ctx->uring_lock);
def596e9 2395 do {
a3a0e43f
JA
2396 /*
2397 * Don't enter poll loop if we already have events pending.
2398 * If we do, we can potentially be spinning for commands that
2399 * already triggered a CQE (eg in error).
2400 */
6c503150 2401 if (test_bit(0, &ctx->cq_check_overflow))
6c2450ae 2402 __io_cqring_overflow_flush(ctx, false);
6c503150 2403 if (io_cqring_events(ctx))
a3a0e43f
JA
2404 break;
2405
500f9fba
JA
2406 /*
2407 * If a submit got punted to a workqueue, we can have the
2408 * application entering polling for a command before it gets
2409 * issued. That app will hold the uring_lock for the duration
2410 * of the poll right here, so we need to take a breather every
2411 * now and then to ensure that the issue has a chance to add
2412 * the poll to the issued list. Otherwise we can spin here
2413 * forever, while the workqueue is stuck trying to acquire the
2414 * very same mutex.
2415 */
2416 if (!(++iters & 7)) {
2417 mutex_unlock(&ctx->uring_lock);
4c6e277c 2418 io_run_task_work();
500f9fba
JA
2419 mutex_lock(&ctx->uring_lock);
2420 }
2421
7668b92a 2422 ret = io_iopoll_getevents(ctx, &nr_events, min);
def596e9
JA
2423 if (ret <= 0)
2424 break;
2425 ret = 0;
7668b92a 2426 } while (min && !nr_events && !need_resched());
def596e9 2427
500f9fba 2428 mutex_unlock(&ctx->uring_lock);
def596e9
JA
2429 return ret;
2430}
2431
491381ce 2432static void kiocb_end_write(struct io_kiocb *req)
2b188cc1 2433{
491381ce
JA
2434 /*
2435 * Tell lockdep we inherited freeze protection from submission
2436 * thread.
2437 */
2438 if (req->flags & REQ_F_ISREG) {
1c98679d 2439 struct super_block *sb = file_inode(req->file)->i_sb;
2b188cc1 2440
1c98679d
PB
2441 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2442 sb_end_write(sb);
2b188cc1
JA
2443 }
2444}
2445
b63534c4 2446#ifdef CONFIG_BLOCK
dc2a6e9a 2447static bool io_resubmit_prep(struct io_kiocb *req)
b63534c4 2448{
ab454438
PB
2449 struct io_async_rw *rw = req->async_data;
2450
2451 if (!rw)
2452 return !io_req_prep_async(req);
2453 /* may have left rw->iter inconsistent on -EIOCBQUEUED */
2454 iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
2455 return true;
b63534c4 2456}
b63534c4 2457
3e6a0d3c 2458static bool io_rw_should_reissue(struct io_kiocb *req)
b63534c4 2459{
355afaeb 2460 umode_t mode = file_inode(req->file)->i_mode;
3e6a0d3c 2461 struct io_ring_ctx *ctx = req->ctx;
b63534c4 2462
355afaeb
JA
2463 if (!S_ISBLK(mode) && !S_ISREG(mode))
2464 return false;
3e6a0d3c
JA
2465 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2466 !(ctx->flags & IORING_SETUP_IOPOLL)))
b63534c4 2467 return false;
7c977a58
JA
2468 /*
2469 * If ref is dying, we might be running poll reap from the exit work.
2470 * Don't attempt to reissue from that path, just let it fail with
2471 * -EAGAIN.
2472 */
3e6a0d3c
JA
2473 if (percpu_ref_is_dying(&ctx->refs))
2474 return false;
2475 return true;
2476}
e82ad485
JA
2477#else
2478static bool io_rw_should_reissue(struct io_kiocb *req)
2479{
2480 return false;
2481}
3e6a0d3c
JA
2482#endif
2483
a1d7c393 2484static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
889fca73 2485 unsigned int issue_flags)
a1d7c393 2486{
2f8e45f1
PB
2487 int cflags = 0;
2488
b65c128f
PB
2489 if (req->rw.kiocb.ki_flags & IOCB_WRITE)
2490 kiocb_end_write(req);
9532b99b
PB
2491 if (res != req->result) {
2492 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2493 io_rw_should_reissue(req)) {
2494 req->flags |= REQ_F_REISSUE;
2495 return;
2496 }
2f8e45f1 2497 req_set_fail_links(req);
9532b99b 2498 }
2f8e45f1
PB
2499 if (req->flags & REQ_F_BUFFER_SELECTED)
2500 cflags = io_put_rw_kbuf(req);
2501 __io_req_complete(req, issue_flags, res, cflags);
ba816ad6
JA
2502}
2503
2504static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2505{
9adbd45d 2506 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
ba816ad6 2507
889fca73 2508 __io_complete_rw(req, res, res2, 0);
2b188cc1
JA
2509}
2510
def596e9
JA
2511static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2512{
9adbd45d 2513 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
def596e9 2514
491381ce
JA
2515 if (kiocb->ki_flags & IOCB_WRITE)
2516 kiocb_end_write(req);
9532b99b
PB
2517 if (unlikely(res != req->result)) {
2518 bool fail = true;
def596e9 2519
9532b99b
PB
2520#ifdef CONFIG_BLOCK
2521 if (res == -EAGAIN && io_rw_should_reissue(req) &&
2522 io_resubmit_prep(req))
2523 fail = false;
2524#endif
2525 if (fail) {
2526 req_set_fail_links(req);
2527 req->flags |= REQ_F_DONT_REISSUE;
2528 }
8c130827 2529 }
bbde017a
XW
2530
2531 WRITE_ONCE(req->result, res);
b9b0e0d3 2532 /* order with io_iopoll_complete() checking ->result */
cd664b0e
PB
2533 smp_wmb();
2534 WRITE_ONCE(req->iopoll_completed, 1);
def596e9
JA
2535}
2536
2537/*
2538 * After the iocb has been issued, it's safe to be found on the poll list.
2539 * Adding the kiocb to the list AFTER submission ensures that we don't
2540 * find it from a io_iopoll_getevents() thread before the issuer is done
2541 * accessing the kiocb cookie.
2542 */
2e9dbe90 2543static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async)
def596e9
JA
2544{
2545 struct io_ring_ctx *ctx = req->ctx;
2546
2547 /*
2548 * Track whether we have multiple files in our lists. This will impact
2549 * how we do polling eventually, not spinning if we're on potentially
2550 * different devices.
2551 */
540e32a0 2552 if (list_empty(&ctx->iopoll_list)) {
def596e9
JA
2553 ctx->poll_multi_file = false;
2554 } else if (!ctx->poll_multi_file) {
2555 struct io_kiocb *list_req;
2556
540e32a0 2557 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
d21ffe7e 2558 inflight_entry);
9adbd45d 2559 if (list_req->file != req->file)
def596e9
JA
2560 ctx->poll_multi_file = true;
2561 }
2562
2563 /*
2564 * For fast devices, IO may have already completed. If it has, add
2565 * it to the front so we find it first.
2566 */
65a6543d 2567 if (READ_ONCE(req->iopoll_completed))
d21ffe7e 2568 list_add(&req->inflight_entry, &ctx->iopoll_list);
def596e9 2569 else
d21ffe7e 2570 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
bdcd3eab 2571
2e9dbe90
XW
2572 /*
2573 * If IORING_SETUP_SQPOLL is enabled, sqes are either handled in sq thread
2574 * task context or in io worker task context. If current task context is
2575 * sq thread, we don't need to check whether should wake up sq thread.
2576 */
2577 if (in_async && (ctx->flags & IORING_SETUP_SQPOLL) &&
534ca6d6
JA
2578 wq_has_sleeper(&ctx->sq_data->wait))
2579 wake_up(&ctx->sq_data->wait);
def596e9
JA
2580}
2581
9f13c35b
PB
2582static inline void io_state_file_put(struct io_submit_state *state)
2583{
02b23a9a
PB
2584 if (state->file_refs) {
2585 fput_many(state->file, state->file_refs);
2586 state->file_refs = 0;
2587 }
9a56a232
JA
2588}
2589
2590/*
2591 * Get as many references to a file as we have IOs left in this submission,
2592 * assuming most submissions are for one file, or at least that each file
2593 * has more than one submission.
2594 */
8da11c19 2595static struct file *__io_file_get(struct io_submit_state *state, int fd)
9a56a232
JA
2596{
2597 if (!state)
2598 return fget(fd);
2599
6e1271e6 2600 if (state->file_refs) {
9a56a232 2601 if (state->fd == fd) {
6e1271e6 2602 state->file_refs--;
9a56a232
JA
2603 return state->file;
2604 }
02b23a9a 2605 io_state_file_put(state);
9a56a232
JA
2606 }
2607 state->file = fget_many(fd, state->ios_left);
6e1271e6 2608 if (unlikely(!state->file))
9a56a232
JA
2609 return NULL;
2610
2611 state->fd = fd;
6e1271e6 2612 state->file_refs = state->ios_left - 1;
9a56a232
JA
2613 return state->file;
2614}
2615
4503b767
JA
2616static bool io_bdev_nowait(struct block_device *bdev)
2617{
9ba0d0c8 2618 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
4503b767
JA
2619}
2620
2b188cc1
JA
2621/*
2622 * If we tracked the file through the SCM inflight mechanism, we could support
2623 * any file. For now, just ensure that anything potentially problematic is done
2624 * inline.
2625 */
7b29f92d 2626static bool __io_file_supports_async(struct file *file, int rw)
2b188cc1
JA
2627{
2628 umode_t mode = file_inode(file)->i_mode;
2629
4503b767 2630 if (S_ISBLK(mode)) {
4e7b5671
CH
2631 if (IS_ENABLED(CONFIG_BLOCK) &&
2632 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
4503b767
JA
2633 return true;
2634 return false;
2635 }
2636 if (S_ISCHR(mode) || S_ISSOCK(mode))
2b188cc1 2637 return true;
4503b767 2638 if (S_ISREG(mode)) {
4e7b5671
CH
2639 if (IS_ENABLED(CONFIG_BLOCK) &&
2640 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
4503b767
JA
2641 file->f_op != &io_uring_fops)
2642 return true;
2643 return false;
2644 }
2b188cc1 2645
c5b85625
JA
2646 /* any ->read/write should understand O_NONBLOCK */
2647 if (file->f_flags & O_NONBLOCK)
2648 return true;
2649
af197f50
JA
2650 if (!(file->f_mode & FMODE_NOWAIT))
2651 return false;
2652
2653 if (rw == READ)
2654 return file->f_op->read_iter != NULL;
2655
2656 return file->f_op->write_iter != NULL;
2b188cc1
JA
2657}
2658
7b29f92d
JA
2659static bool io_file_supports_async(struct io_kiocb *req, int rw)
2660{
2661 if (rw == READ && (req->flags & REQ_F_ASYNC_READ))
2662 return true;
2663 else if (rw == WRITE && (req->flags & REQ_F_ASYNC_WRITE))
2664 return true;
2665
2666 return __io_file_supports_async(req->file, rw);
2667}
2668
a88fc400 2669static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2b188cc1 2670{
def596e9 2671 struct io_ring_ctx *ctx = req->ctx;
9adbd45d 2672 struct kiocb *kiocb = &req->rw.kiocb;
75c668cd 2673 struct file *file = req->file;
09bb8394
JA
2674 unsigned ioprio;
2675 int ret;
2b188cc1 2676
7b29f92d 2677 if (!(req->flags & REQ_F_ISREG) && S_ISREG(file_inode(file)->i_mode))
491381ce
JA
2678 req->flags |= REQ_F_ISREG;
2679
2b188cc1 2680 kiocb->ki_pos = READ_ONCE(sqe->off);
75c668cd 2681 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
ba04291e 2682 req->flags |= REQ_F_CUR_POS;
75c668cd 2683 kiocb->ki_pos = file->f_pos;
ba04291e 2684 }
2b188cc1 2685 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
3e577dcd
PB
2686 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2687 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2688 if (unlikely(ret))
2689 return ret;
2b188cc1 2690
75c668cd
PB
2691 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2692 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2693 req->flags |= REQ_F_NOWAIT;
2694
2b188cc1
JA
2695 ioprio = READ_ONCE(sqe->ioprio);
2696 if (ioprio) {
2697 ret = ioprio_check_cap(ioprio);
2698 if (ret)
09bb8394 2699 return ret;
2b188cc1
JA
2700
2701 kiocb->ki_ioprio = ioprio;
2702 } else
2703 kiocb->ki_ioprio = get_current_ioprio();
2704
def596e9 2705 if (ctx->flags & IORING_SETUP_IOPOLL) {
def596e9
JA
2706 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2707 !kiocb->ki_filp->f_op->iopoll)
09bb8394 2708 return -EOPNOTSUPP;
2b188cc1 2709
def596e9
JA
2710 kiocb->ki_flags |= IOCB_HIPRI;
2711 kiocb->ki_complete = io_complete_rw_iopoll;
65a6543d 2712 req->iopoll_completed = 0;
def596e9 2713 } else {
09bb8394
JA
2714 if (kiocb->ki_flags & IOCB_HIPRI)
2715 return -EINVAL;
def596e9
JA
2716 kiocb->ki_complete = io_complete_rw;
2717 }
9adbd45d 2718
3529d8c2
JA
2719 req->rw.addr = READ_ONCE(sqe->addr);
2720 req->rw.len = READ_ONCE(sqe->len);
4f4eeba8 2721 req->buf_index = READ_ONCE(sqe->buf_index);
2b188cc1 2722 return 0;
2b188cc1
JA
2723}
2724
2725static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2726{
2727 switch (ret) {
2728 case -EIOCBQUEUED:
2729 break;
2730 case -ERESTARTSYS:
2731 case -ERESTARTNOINTR:
2732 case -ERESTARTNOHAND:
2733 case -ERESTART_RESTARTBLOCK:
2734 /*
2735 * We can't just restart the syscall, since previously
2736 * submitted sqes may already be in progress. Just fail this
2737 * IO with EINTR.
2738 */
2739 ret = -EINTR;
df561f66 2740 fallthrough;
2b188cc1
JA
2741 default:
2742 kiocb->ki_complete(kiocb, ret, 0);
2743 }
2744}
2745
a1d7c393 2746static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
889fca73 2747 unsigned int issue_flags)
ba816ad6 2748{
ba04291e 2749 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
e8c2bc1f 2750 struct io_async_rw *io = req->async_data;
97284637 2751 bool check_reissue = kiocb->ki_complete == io_complete_rw;
ba04291e 2752
227c0c96 2753 /* add previously done IO, if any */
e8c2bc1f 2754 if (io && io->bytes_done > 0) {
227c0c96 2755 if (ret < 0)
e8c2bc1f 2756 ret = io->bytes_done;
227c0c96 2757 else
e8c2bc1f 2758 ret += io->bytes_done;
227c0c96
JA
2759 }
2760
ba04291e
JA
2761 if (req->flags & REQ_F_CUR_POS)
2762 req->file->f_pos = kiocb->ki_pos;
bcaec089 2763 if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
889fca73 2764 __io_complete_rw(req, ret, 0, issue_flags);
ba816ad6
JA
2765 else
2766 io_rw_done(kiocb, ret);
97284637
PB
2767
2768 if (check_reissue && req->flags & REQ_F_REISSUE) {
2769 req->flags &= ~REQ_F_REISSUE;
8c130827
PB
2770 if (!io_resubmit_prep(req)) {
2771 req_ref_get(req);
2772 io_queue_async_work(req);
2773 } else {
97284637
PB
2774 int cflags = 0;
2775
2776 req_set_fail_links(req);
2777 if (req->flags & REQ_F_BUFFER_SELECTED)
2778 cflags = io_put_rw_kbuf(req);
2779 __io_req_complete(req, issue_flags, ret, cflags);
2780 }
2781 }
ba816ad6
JA
2782}
2783
847595de 2784static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
edafccee 2785{
9adbd45d
JA
2786 struct io_ring_ctx *ctx = req->ctx;
2787 size_t len = req->rw.len;
edafccee 2788 struct io_mapped_ubuf *imu;
4be1c615 2789 u16 index, buf_index = req->buf_index;
edafccee
JA
2790 size_t offset;
2791 u64 buf_addr;
2792
edafccee
JA
2793 if (unlikely(buf_index >= ctx->nr_user_bufs))
2794 return -EFAULT;
edafccee
JA
2795 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2796 imu = &ctx->user_bufs[index];
9adbd45d 2797 buf_addr = req->rw.addr;
edafccee
JA
2798
2799 /* overflow */
2800 if (buf_addr + len < buf_addr)
2801 return -EFAULT;
2802 /* not inside the mapped region */
2803 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
2804 return -EFAULT;
2805
2806 /*
2807 * May not be a start of buffer, set size appropriately
2808 * and advance us to the beginning.
2809 */
2810 offset = buf_addr - imu->ubuf;
2811 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
bd11b3a3
JA
2812
2813 if (offset) {
2814 /*
2815 * Don't use iov_iter_advance() here, as it's really slow for
2816 * using the latter parts of a big fixed buffer - it iterates
2817 * over each segment manually. We can cheat a bit here, because
2818 * we know that:
2819 *
2820 * 1) it's a BVEC iter, we set it up
2821 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2822 * first and last bvec
2823 *
2824 * So just find our index, and adjust the iterator afterwards.
2825 * If the offset is within the first bvec (or the whole first
2826 * bvec, just use iov_iter_advance(). This makes it easier
2827 * since we can just skip the first segment, which may not
2828 * be PAGE_SIZE aligned.
2829 */
2830 const struct bio_vec *bvec = imu->bvec;
2831
2832 if (offset <= bvec->bv_len) {
2833 iov_iter_advance(iter, offset);
2834 } else {
2835 unsigned long seg_skip;
2836
2837 /* skip first vec */
2838 offset -= bvec->bv_len;
2839 seg_skip = 1 + (offset >> PAGE_SHIFT);
2840
2841 iter->bvec = bvec + seg_skip;
2842 iter->nr_segs -= seg_skip;
99c79f66 2843 iter->count -= bvec->bv_len + offset;
bd11b3a3 2844 iter->iov_offset = offset & ~PAGE_MASK;
bd11b3a3
JA
2845 }
2846 }
2847
847595de 2848 return 0;
edafccee
JA
2849}
2850
bcda7baa
JA
2851static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2852{
2853 if (needs_lock)
2854 mutex_unlock(&ctx->uring_lock);
2855}
2856
2857static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2858{
2859 /*
2860 * "Normal" inline submissions always hold the uring_lock, since we
2861 * grab it from the system call. Same is true for the SQPOLL offload.
2862 * The only exception is when we've detached the request and issue it
2863 * from an async worker thread, grab the lock for that case.
2864 */
2865 if (needs_lock)
2866 mutex_lock(&ctx->uring_lock);
2867}
2868
2869static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2870 int bgid, struct io_buffer *kbuf,
2871 bool needs_lock)
2872{
2873 struct io_buffer *head;
2874
2875 if (req->flags & REQ_F_BUFFER_SELECTED)
2876 return kbuf;
2877
2878 io_ring_submit_lock(req->ctx, needs_lock);
2879
2880 lockdep_assert_held(&req->ctx->uring_lock);
2881
9e15c3a0 2882 head = xa_load(&req->ctx->io_buffers, bgid);
bcda7baa
JA
2883 if (head) {
2884 if (!list_empty(&head->list)) {
2885 kbuf = list_last_entry(&head->list, struct io_buffer,
2886 list);
2887 list_del(&kbuf->list);
2888 } else {
2889 kbuf = head;
9e15c3a0 2890 xa_erase(&req->ctx->io_buffers, bgid);
bcda7baa
JA
2891 }
2892 if (*len > kbuf->len)
2893 *len = kbuf->len;
2894 } else {
2895 kbuf = ERR_PTR(-ENOBUFS);
2896 }
2897
2898 io_ring_submit_unlock(req->ctx, needs_lock);
2899
2900 return kbuf;
2901}
2902
4d954c25
JA
2903static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2904 bool needs_lock)
2905{
2906 struct io_buffer *kbuf;
4f4eeba8 2907 u16 bgid;
4d954c25
JA
2908
2909 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
4f4eeba8 2910 bgid = req->buf_index;
4d954c25
JA
2911 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2912 if (IS_ERR(kbuf))
2913 return kbuf;
2914 req->rw.addr = (u64) (unsigned long) kbuf;
2915 req->flags |= REQ_F_BUFFER_SELECTED;
2916 return u64_to_user_ptr(kbuf->addr);
2917}
2918
2919#ifdef CONFIG_COMPAT
2920static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2921 bool needs_lock)
2922{
2923 struct compat_iovec __user *uiov;
2924 compat_ssize_t clen;
2925 void __user *buf;
2926 ssize_t len;
2927
2928 uiov = u64_to_user_ptr(req->rw.addr);
2929 if (!access_ok(uiov, sizeof(*uiov)))
2930 return -EFAULT;
2931 if (__get_user(clen, &uiov->iov_len))
2932 return -EFAULT;
2933 if (clen < 0)
2934 return -EINVAL;
2935
2936 len = clen;
2937 buf = io_rw_buffer_select(req, &len, needs_lock);
2938 if (IS_ERR(buf))
2939 return PTR_ERR(buf);
2940 iov[0].iov_base = buf;
2941 iov[0].iov_len = (compat_size_t) len;
2942 return 0;
2943}
2944#endif
2945
2946static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2947 bool needs_lock)
2948{
2949 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
2950 void __user *buf;
2951 ssize_t len;
2952
2953 if (copy_from_user(iov, uiov, sizeof(*uiov)))
2954 return -EFAULT;
2955
2956 len = iov[0].iov_len;
2957 if (len < 0)
2958 return -EINVAL;
2959 buf = io_rw_buffer_select(req, &len, needs_lock);
2960 if (IS_ERR(buf))
2961 return PTR_ERR(buf);
2962 iov[0].iov_base = buf;
2963 iov[0].iov_len = len;
2964 return 0;
2965}
2966
2967static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2968 bool needs_lock)
2969{
dddb3e26
JA
2970 if (req->flags & REQ_F_BUFFER_SELECTED) {
2971 struct io_buffer *kbuf;
2972
2973 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2974 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
2975 iov[0].iov_len = kbuf->len;
4d954c25 2976 return 0;
dddb3e26 2977 }
dd201662 2978 if (req->rw.len != 1)
4d954c25
JA
2979 return -EINVAL;
2980
2981#ifdef CONFIG_COMPAT
2982 if (req->ctx->compat)
2983 return io_compat_import(req, iov, needs_lock);
2984#endif
2985
2986 return __io_iov_buffer_select(req, iov, needs_lock);
2987}
2988
847595de
PB
2989static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
2990 struct iov_iter *iter, bool needs_lock)
2b188cc1 2991{
9adbd45d
JA
2992 void __user *buf = u64_to_user_ptr(req->rw.addr);
2993 size_t sqe_len = req->rw.len;
847595de 2994 u8 opcode = req->opcode;
4d954c25 2995 ssize_t ret;
edafccee 2996
7d009165 2997 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
edafccee 2998 *iovec = NULL;
9adbd45d 2999 return io_import_fixed(req, rw, iter);
edafccee 3000 }
2b188cc1 3001
bcda7baa 3002 /* buffer index only valid with fixed read/write, or buffer select */
4f4eeba8 3003 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
9adbd45d
JA
3004 return -EINVAL;
3005
3a6820f2 3006 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
bcda7baa 3007 if (req->flags & REQ_F_BUFFER_SELECT) {
4d954c25 3008 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
867a23ea 3009 if (IS_ERR(buf))
4d954c25 3010 return PTR_ERR(buf);
3f9d6441 3011 req->rw.len = sqe_len;
bcda7baa
JA
3012 }
3013
3a6820f2
JA
3014 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3015 *iovec = NULL;
10fc72e4 3016 return ret;
3a6820f2
JA
3017 }
3018
4d954c25
JA
3019 if (req->flags & REQ_F_BUFFER_SELECT) {
3020 ret = io_iov_buffer_select(req, *iovec, needs_lock);
847595de
PB
3021 if (!ret)
3022 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
4d954c25
JA
3023 *iovec = NULL;
3024 return ret;
3025 }
3026
89cd35c5
CH
3027 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3028 req->ctx->compat);
2b188cc1
JA
3029}
3030
0fef9483
JA
3031static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3032{
5b09e37e 3033 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
0fef9483
JA
3034}
3035
31b51510 3036/*
32960613
JA
3037 * For files that don't have ->read_iter() and ->write_iter(), handle them
3038 * by looping over ->read() or ->write() manually.
31b51510 3039 */
4017eb91 3040static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
32960613 3041{
4017eb91
JA
3042 struct kiocb *kiocb = &req->rw.kiocb;
3043 struct file *file = req->file;
32960613
JA
3044 ssize_t ret = 0;
3045
3046 /*
3047 * Don't support polled IO through this interface, and we can't
3048 * support non-blocking either. For the latter, this just causes
3049 * the kiocb to be handled from an async context.
3050 */
3051 if (kiocb->ki_flags & IOCB_HIPRI)
3052 return -EOPNOTSUPP;
3053 if (kiocb->ki_flags & IOCB_NOWAIT)
3054 return -EAGAIN;
3055
3056 while (iov_iter_count(iter)) {
311ae9e1 3057 struct iovec iovec;
32960613
JA
3058 ssize_t nr;
3059
311ae9e1
PB
3060 if (!iov_iter_is_bvec(iter)) {
3061 iovec = iov_iter_iovec(iter);
3062 } else {
4017eb91
JA
3063 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3064 iovec.iov_len = req->rw.len;
311ae9e1
PB
3065 }
3066
32960613
JA
3067 if (rw == READ) {
3068 nr = file->f_op->read(file, iovec.iov_base,
0fef9483 3069 iovec.iov_len, io_kiocb_ppos(kiocb));
32960613
JA
3070 } else {
3071 nr = file->f_op->write(file, iovec.iov_base,
0fef9483 3072 iovec.iov_len, io_kiocb_ppos(kiocb));
32960613
JA
3073 }
3074
3075 if (nr < 0) {
3076 if (!ret)
3077 ret = nr;
3078 break;
3079 }
3080 ret += nr;
3081 if (nr != iovec.iov_len)
3082 break;
4017eb91
JA
3083 req->rw.len -= nr;
3084 req->rw.addr += nr;
32960613
JA
3085 iov_iter_advance(iter, nr);
3086 }
3087
3088 return ret;
3089}
3090
ff6165b2
JA
3091static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3092 const struct iovec *fast_iov, struct iov_iter *iter)
f67676d1 3093{
e8c2bc1f 3094 struct io_async_rw *rw = req->async_data;
b64e3444 3095
ff6165b2 3096 memcpy(&rw->iter, iter, sizeof(*iter));
afb87658 3097 rw->free_iovec = iovec;
227c0c96 3098 rw->bytes_done = 0;
ff6165b2 3099 /* can only be fixed buffers, no need to do anything */
9c3a205c 3100 if (iov_iter_is_bvec(iter))
ff6165b2 3101 return;
b64e3444 3102 if (!iovec) {
ff6165b2
JA
3103 unsigned iov_off = 0;
3104
3105 rw->iter.iov = rw->fast_iov;
3106 if (iter->iov != fast_iov) {
3107 iov_off = iter->iov - fast_iov;
3108 rw->iter.iov += iov_off;
3109 }
3110 if (rw->fast_iov != fast_iov)
3111 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
45097dae 3112 sizeof(struct iovec) * iter->nr_segs);
99bc4c38
PB
3113 } else {
3114 req->flags |= REQ_F_NEED_CLEANUP;
f67676d1
JA
3115 }
3116}
3117
6cb78689 3118static inline int io_alloc_async_data(struct io_kiocb *req)
3d9932a8 3119{
e8c2bc1f
JA
3120 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3121 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3122 return req->async_data == NULL;
3d9932a8
XW
3123}
3124
ff6165b2
JA
3125static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3126 const struct iovec *fast_iov,
227c0c96 3127 struct iov_iter *iter, bool force)
b7bb4f7d 3128{
26f0505a 3129 if (!force && !io_op_defs[req->opcode].needs_async_setup)
74566df3 3130 return 0;
e8c2bc1f 3131 if (!req->async_data) {
6cb78689 3132 if (io_alloc_async_data(req)) {
6bf985dc 3133 kfree(iovec);
5d204bcf 3134 return -ENOMEM;
6bf985dc 3135 }
b7bb4f7d 3136
ff6165b2 3137 io_req_map_rw(req, iovec, fast_iov, iter);
5d204bcf 3138 }
b7bb4f7d 3139 return 0;
f67676d1
JA
3140}
3141
73debe68 3142static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
c3e330a4 3143{
e8c2bc1f 3144 struct io_async_rw *iorw = req->async_data;
f4bff104 3145 struct iovec *iov = iorw->fast_iov;
847595de 3146 int ret;
c3e330a4 3147
2846c481 3148 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
c3e330a4
PB
3149 if (unlikely(ret < 0))
3150 return ret;
3151
ab0b196c
PB
3152 iorw->bytes_done = 0;
3153 iorw->free_iovec = iov;
3154 if (iov)
3155 req->flags |= REQ_F_NEED_CLEANUP;
c3e330a4
PB
3156 return 0;
3157}
3158
73debe68 3159static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1 3160{
3529d8c2
JA
3161 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3162 return -EBADF;
93642ef8 3163 return io_prep_rw(req, sqe);
f67676d1
JA
3164}
3165
c1dd91d1
JA
3166/*
3167 * This is our waitqueue callback handler, registered through lock_page_async()
3168 * when we initially tried to do the IO with the iocb armed our waitqueue.
3169 * This gets called when the page is unlocked, and we generally expect that to
3170 * happen when the page IO is completed and the page is now uptodate. This will
3171 * queue a task_work based retry of the operation, attempting to copy the data
3172 * again. If the latter fails because the page was NOT uptodate, then we will
3173 * do a thread based blocking retry of the operation. That's the unexpected
3174 * slow path.
3175 */
bcf5a063
JA
3176static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3177 int sync, void *arg)
3178{
3179 struct wait_page_queue *wpq;
3180 struct io_kiocb *req = wait->private;
bcf5a063 3181 struct wait_page_key *key = arg;
bcf5a063
JA
3182
3183 wpq = container_of(wait, struct wait_page_queue, wait);
3184
cdc8fcb4
LT
3185 if (!wake_page_match(wpq, key))
3186 return 0;
3187
c8d317aa 3188 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
bcf5a063
JA
3189 list_del_init(&wait->entry);
3190
bcf5a063 3191 /* submit ref gets dropped, acquire a new one */
de9b4cca 3192 req_ref_get(req);
921b9054 3193 io_req_task_queue(req);
bcf5a063
JA
3194 return 1;
3195}
3196
c1dd91d1
JA
3197/*
3198 * This controls whether a given IO request should be armed for async page
3199 * based retry. If we return false here, the request is handed to the async
3200 * worker threads for retry. If we're doing buffered reads on a regular file,
3201 * we prepare a private wait_page_queue entry and retry the operation. This
3202 * will either succeed because the page is now uptodate and unlocked, or it
3203 * will register a callback when the page is unlocked at IO completion. Through
3204 * that callback, io_uring uses task_work to setup a retry of the operation.
3205 * That retry will attempt the buffered read again. The retry will generally
3206 * succeed, or in rare cases where it fails, we then fall back to using the
3207 * async worker threads for a blocking retry.
3208 */
227c0c96 3209static bool io_rw_should_retry(struct io_kiocb *req)
f67676d1 3210{
e8c2bc1f
JA
3211 struct io_async_rw *rw = req->async_data;
3212 struct wait_page_queue *wait = &rw->wpq;
bcf5a063 3213 struct kiocb *kiocb = &req->rw.kiocb;
f67676d1 3214
bcf5a063
JA
3215 /* never retry for NOWAIT, we just complete with -EAGAIN */
3216 if (req->flags & REQ_F_NOWAIT)
3217 return false;
f67676d1 3218
227c0c96 3219 /* Only for buffered IO */
3b2a4439 3220 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
bcf5a063 3221 return false;
3b2a4439 3222
bcf5a063
JA
3223 /*
3224 * just use poll if we can, and don't attempt if the fs doesn't
3225 * support callback based unlocks
3226 */
3227 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3228 return false;
f67676d1 3229
3b2a4439
JA
3230 wait->wait.func = io_async_buf_func;
3231 wait->wait.private = req;
3232 wait->wait.flags = 0;
3233 INIT_LIST_HEAD(&wait->wait.entry);
3234 kiocb->ki_flags |= IOCB_WAITQ;
c8d317aa 3235 kiocb->ki_flags &= ~IOCB_NOWAIT;
3b2a4439 3236 kiocb->ki_waitq = wait;
3b2a4439 3237 return true;
bcf5a063
JA
3238}
3239
3240static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
3241{
3242 if (req->file->f_op->read_iter)
3243 return call_read_iter(req->file, &req->rw.kiocb, iter);
2dd2111d 3244 else if (req->file->f_op->read)
4017eb91 3245 return loop_rw_iter(READ, req, iter);
2dd2111d
GH
3246 else
3247 return -EINVAL;
f67676d1
JA
3248}
3249
889fca73 3250static int io_read(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3251{
3252 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 3253 struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2 3254 struct iov_iter __iter, *iter = &__iter;
e8c2bc1f 3255 struct io_async_rw *rw = req->async_data;
227c0c96 3256 ssize_t io_size, ret, ret2;
45d189c6 3257 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ff6165b2 3258
2846c481 3259 if (rw) {
e8c2bc1f 3260 iter = &rw->iter;
2846c481
PB
3261 iovec = NULL;
3262 } else {
3263 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3264 if (ret < 0)
3265 return ret;
3266 }
632546c4 3267 io_size = iov_iter_count(iter);
fa15bafb 3268 req->result = io_size;
2b188cc1 3269
fd6c2e4c
JA
3270 /* Ensure we clear previously set non-block flag */
3271 if (!force_nonblock)
29de5f6a 3272 kiocb->ki_flags &= ~IOCB_NOWAIT;
a88fc400
PB
3273 else
3274 kiocb->ki_flags |= IOCB_NOWAIT;
3275
24c74678 3276 /* If the file doesn't support async, just async punt */
7b29f92d 3277 if (force_nonblock && !io_file_supports_async(req, READ)) {
6713e7a6 3278 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc 3279 return ret ?: -EAGAIN;
6713e7a6 3280 }
9e645e11 3281
632546c4 3282 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
5ea5dd45
PB
3283 if (unlikely(ret)) {
3284 kfree(iovec);
3285 return ret;
3286 }
2b188cc1 3287
227c0c96 3288 ret = io_iter_do_read(req, iter);
32960613 3289
230d50d4 3290 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
6ad7f233 3291 req->flags &= ~REQ_F_REISSUE;
eefdf30f
JA
3292 /* IOPOLL retry should happen for io-wq threads */
3293 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
f91daf56 3294 goto done;
75c668cd
PB
3295 /* no retry on NONBLOCK nor RWF_NOWAIT */
3296 if (req->flags & REQ_F_NOWAIT)
355afaeb 3297 goto done;
84216315 3298 /* some cases will consume bytes even on error returns */
632546c4 3299 iov_iter_revert(iter, io_size - iov_iter_count(iter));
f38c7e3a 3300 ret = 0;
230d50d4
JA
3301 } else if (ret == -EIOCBQUEUED) {
3302 goto out_free;
7335e3bf 3303 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
75c668cd 3304 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
7335e3bf 3305 /* read all, failed, already did sync or don't want to retry */
00d23d51 3306 goto done;
227c0c96
JA
3307 }
3308
227c0c96 3309 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
6bf985dc
PB
3310 if (ret2)
3311 return ret2;
3312
fe1cdd55 3313 iovec = NULL;
e8c2bc1f 3314 rw = req->async_data;
227c0c96 3315 /* now use our persistent iterator, if we aren't already */
e8c2bc1f 3316 iter = &rw->iter;
227c0c96 3317
b23df91b
PB
3318 do {
3319 io_size -= ret;
3320 rw->bytes_done += ret;
3321 /* if we can retry, do so with the callbacks armed */
3322 if (!io_rw_should_retry(req)) {
3323 kiocb->ki_flags &= ~IOCB_WAITQ;
3324 return -EAGAIN;
3325 }
3326
3327 /*
3328 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3329 * we get -EIOCBQUEUED, then we'll get a notification when the
3330 * desired page gets unlocked. We can also get a partial read
3331 * here, and if we do, then just retry at the new offset.
3332 */
3333 ret = io_iter_do_read(req, iter);
3334 if (ret == -EIOCBQUEUED)
3335 return 0;
227c0c96 3336 /* we got some bytes, but not all. retry. */
b5b0ecb7 3337 kiocb->ki_flags &= ~IOCB_WAITQ;
b23df91b 3338 } while (ret > 0 && ret < io_size);
227c0c96 3339done:
889fca73 3340 kiocb_done(kiocb, ret, issue_flags);
fe1cdd55
PB
3341out_free:
3342 /* it's faster to check here then delegate to kfree */
3343 if (iovec)
3344 kfree(iovec);
5ea5dd45 3345 return 0;
2b188cc1
JA
3346}
3347
73debe68 3348static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1 3349{
3529d8c2
JA
3350 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3351 return -EBADF;
93642ef8 3352 return io_prep_rw(req, sqe);
f67676d1
JA
3353}
3354
889fca73 3355static int io_write(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3356{
3357 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
9adbd45d 3358 struct kiocb *kiocb = &req->rw.kiocb;
ff6165b2 3359 struct iov_iter __iter, *iter = &__iter;
e8c2bc1f 3360 struct io_async_rw *rw = req->async_data;
fa15bafb 3361 ssize_t ret, ret2, io_size;
45d189c6 3362 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
2b188cc1 3363
2846c481 3364 if (rw) {
e8c2bc1f 3365 iter = &rw->iter;
2846c481
PB
3366 iovec = NULL;
3367 } else {
3368 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3369 if (ret < 0)
3370 return ret;
3371 }
632546c4 3372 io_size = iov_iter_count(iter);
fa15bafb 3373 req->result = io_size;
2b188cc1 3374
fd6c2e4c
JA
3375 /* Ensure we clear previously set non-block flag */
3376 if (!force_nonblock)
a88fc400
PB
3377 kiocb->ki_flags &= ~IOCB_NOWAIT;
3378 else
3379 kiocb->ki_flags |= IOCB_NOWAIT;
fd6c2e4c 3380
24c74678 3381 /* If the file doesn't support async, just async punt */
7b29f92d 3382 if (force_nonblock && !io_file_supports_async(req, WRITE))
f67676d1 3383 goto copy_iov;
31b51510 3384
10d59345
JA
3385 /* file path doesn't support NOWAIT for non-direct_IO */
3386 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3387 (req->flags & REQ_F_ISREG))
f67676d1 3388 goto copy_iov;
31b51510 3389
632546c4 3390 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
fa15bafb
PB
3391 if (unlikely(ret))
3392 goto out_free;
4ed734b0 3393
fa15bafb
PB
3394 /*
3395 * Open-code file_start_write here to grab freeze protection,
3396 * which will be released by another thread in
3397 * io_complete_rw(). Fool lockdep by telling it the lock got
3398 * released so that it doesn't complain about the held lock when
3399 * we return to userspace.
3400 */
3401 if (req->flags & REQ_F_ISREG) {
8a3c84b6 3402 sb_start_write(file_inode(req->file)->i_sb);
fa15bafb
PB
3403 __sb_writers_release(file_inode(req->file)->i_sb,
3404 SB_FREEZE_WRITE);
3405 }
3406 kiocb->ki_flags |= IOCB_WRITE;
4ed734b0 3407
fa15bafb 3408 if (req->file->f_op->write_iter)
ff6165b2 3409 ret2 = call_write_iter(req->file, kiocb, iter);
2dd2111d 3410 else if (req->file->f_op->write)
4017eb91 3411 ret2 = loop_rw_iter(WRITE, req, iter);
2dd2111d
GH
3412 else
3413 ret2 = -EINVAL;
4ed734b0 3414
6ad7f233
PB
3415 if (req->flags & REQ_F_REISSUE) {
3416 req->flags &= ~REQ_F_REISSUE;
230d50d4 3417 ret2 = -EAGAIN;
6ad7f233 3418 }
230d50d4 3419
fa15bafb
PB
3420 /*
3421 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3422 * retry them without IOCB_NOWAIT.
3423 */
3424 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3425 ret2 = -EAGAIN;
75c668cd
PB
3426 /* no retry on NONBLOCK nor RWF_NOWAIT */
3427 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
355afaeb 3428 goto done;
fa15bafb 3429 if (!force_nonblock || ret2 != -EAGAIN) {
eefdf30f
JA
3430 /* IOPOLL retry should happen for io-wq threads */
3431 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3432 goto copy_iov;
355afaeb 3433done:
889fca73 3434 kiocb_done(kiocb, ret2, issue_flags);
fa15bafb 3435 } else {
f67676d1 3436copy_iov:
84216315 3437 /* some cases will consume bytes even on error returns */
632546c4 3438 iov_iter_revert(iter, io_size - iov_iter_count(iter));
227c0c96 3439 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
6bf985dc 3440 return ret ?: -EAGAIN;
2b188cc1 3441 }
31b51510 3442out_free:
f261c168 3443 /* it's reportedly faster than delegating the null check to kfree() */
252917c3 3444 if (iovec)
6f2cc166 3445 kfree(iovec);
2b188cc1
JA
3446 return ret;
3447}
3448
80a261fd
JA
3449static int io_renameat_prep(struct io_kiocb *req,
3450 const struct io_uring_sqe *sqe)
3451{
3452 struct io_rename *ren = &req->rename;
3453 const char __user *oldf, *newf;
3454
3455 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3456 return -EBADF;
3457
3458 ren->old_dfd = READ_ONCE(sqe->fd);
3459 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3460 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3461 ren->new_dfd = READ_ONCE(sqe->len);
3462 ren->flags = READ_ONCE(sqe->rename_flags);
3463
3464 ren->oldpath = getname(oldf);
3465 if (IS_ERR(ren->oldpath))
3466 return PTR_ERR(ren->oldpath);
3467
3468 ren->newpath = getname(newf);
3469 if (IS_ERR(ren->newpath)) {
3470 putname(ren->oldpath);
3471 return PTR_ERR(ren->newpath);
3472 }
3473
3474 req->flags |= REQ_F_NEED_CLEANUP;
3475 return 0;
3476}
3477
45d189c6 3478static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
80a261fd
JA
3479{
3480 struct io_rename *ren = &req->rename;
3481 int ret;
3482
45d189c6 3483 if (issue_flags & IO_URING_F_NONBLOCK)
80a261fd
JA
3484 return -EAGAIN;
3485
3486 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3487 ren->newpath, ren->flags);
3488
3489 req->flags &= ~REQ_F_NEED_CLEANUP;
3490 if (ret < 0)
3491 req_set_fail_links(req);
3492 io_req_complete(req, ret);
3493 return 0;
3494}
3495
14a1143b
JA
3496static int io_unlinkat_prep(struct io_kiocb *req,
3497 const struct io_uring_sqe *sqe)
3498{
3499 struct io_unlink *un = &req->unlink;
3500 const char __user *fname;
3501
3502 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3503 return -EBADF;
3504
3505 un->dfd = READ_ONCE(sqe->fd);
3506
3507 un->flags = READ_ONCE(sqe->unlink_flags);
3508 if (un->flags & ~AT_REMOVEDIR)
3509 return -EINVAL;
3510
3511 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3512 un->filename = getname(fname);
3513 if (IS_ERR(un->filename))
3514 return PTR_ERR(un->filename);
3515
3516 req->flags |= REQ_F_NEED_CLEANUP;
3517 return 0;
3518}
3519
45d189c6 3520static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
14a1143b
JA
3521{
3522 struct io_unlink *un = &req->unlink;
3523 int ret;
3524
45d189c6 3525 if (issue_flags & IO_URING_F_NONBLOCK)
14a1143b
JA
3526 return -EAGAIN;
3527
3528 if (un->flags & AT_REMOVEDIR)
3529 ret = do_rmdir(un->dfd, un->filename);
3530 else
3531 ret = do_unlinkat(un->dfd, un->filename);
3532
3533 req->flags &= ~REQ_F_NEED_CLEANUP;
3534 if (ret < 0)
3535 req_set_fail_links(req);
3536 io_req_complete(req, ret);
3537 return 0;
3538}
3539
36f4fa68
JA
3540static int io_shutdown_prep(struct io_kiocb *req,
3541 const struct io_uring_sqe *sqe)
3542{
3543#if defined(CONFIG_NET)
3544 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3545 return -EINVAL;
3546 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3547 sqe->buf_index)
3548 return -EINVAL;
3549
3550 req->shutdown.how = READ_ONCE(sqe->len);
3551 return 0;
3552#else
3553 return -EOPNOTSUPP;
3554#endif
3555}
3556
45d189c6 3557static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
36f4fa68
JA
3558{
3559#if defined(CONFIG_NET)
3560 struct socket *sock;
3561 int ret;
3562
45d189c6 3563 if (issue_flags & IO_URING_F_NONBLOCK)
36f4fa68
JA
3564 return -EAGAIN;
3565
48aba79b 3566 sock = sock_from_file(req->file);
36f4fa68 3567 if (unlikely(!sock))
48aba79b 3568 return -ENOTSOCK;
36f4fa68
JA
3569
3570 ret = __sys_shutdown_sock(sock, req->shutdown.how);
a146468d
JA
3571 if (ret < 0)
3572 req_set_fail_links(req);
36f4fa68
JA
3573 io_req_complete(req, ret);
3574 return 0;
3575#else
3576 return -EOPNOTSUPP;
3577#endif
3578}
3579
f2a8d5c7
PB
3580static int __io_splice_prep(struct io_kiocb *req,
3581 const struct io_uring_sqe *sqe)
7d67af2c
PB
3582{
3583 struct io_splice* sp = &req->splice;
3584 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
7d67af2c 3585
3232dd02
PB
3586 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3587 return -EINVAL;
7d67af2c
PB
3588
3589 sp->file_in = NULL;
7d67af2c
PB
3590 sp->len = READ_ONCE(sqe->len);
3591 sp->flags = READ_ONCE(sqe->splice_flags);
3592
3593 if (unlikely(sp->flags & ~valid_flags))
3594 return -EINVAL;
3595
8371adf5
PB
3596 sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
3597 (sp->flags & SPLICE_F_FD_IN_FIXED));
3598 if (!sp->file_in)
3599 return -EBADF;
7d67af2c 3600 req->flags |= REQ_F_NEED_CLEANUP;
7d67af2c
PB
3601 return 0;
3602}
3603
f2a8d5c7
PB
3604static int io_tee_prep(struct io_kiocb *req,
3605 const struct io_uring_sqe *sqe)
3606{
3607 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3608 return -EINVAL;
3609 return __io_splice_prep(req, sqe);
3610}
3611
45d189c6 3612static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
f2a8d5c7
PB
3613{
3614 struct io_splice *sp = &req->splice;
3615 struct file *in = sp->file_in;
3616 struct file *out = sp->file_out;
3617 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3618 long ret = 0;
3619
45d189c6 3620 if (issue_flags & IO_URING_F_NONBLOCK)
f2a8d5c7
PB
3621 return -EAGAIN;
3622 if (sp->len)
3623 ret = do_tee(in, out, sp->len, flags);
3624
e1d767f0
PB
3625 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3626 io_put_file(in);
f2a8d5c7
PB
3627 req->flags &= ~REQ_F_NEED_CLEANUP;
3628
f2a8d5c7
PB
3629 if (ret != sp->len)
3630 req_set_fail_links(req);
e1e16097 3631 io_req_complete(req, ret);
f2a8d5c7
PB
3632 return 0;
3633}
3634
3635static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3636{
3637 struct io_splice* sp = &req->splice;
3638
3639 sp->off_in = READ_ONCE(sqe->splice_off_in);
3640 sp->off_out = READ_ONCE(sqe->off);
3641 return __io_splice_prep(req, sqe);
3642}
3643
45d189c6 3644static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
7d67af2c
PB
3645{
3646 struct io_splice *sp = &req->splice;
3647 struct file *in = sp->file_in;
3648 struct file *out = sp->file_out;
3649 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3650 loff_t *poff_in, *poff_out;
c9687426 3651 long ret = 0;
7d67af2c 3652
45d189c6 3653 if (issue_flags & IO_URING_F_NONBLOCK)
2fb3e822 3654 return -EAGAIN;
7d67af2c
PB
3655
3656 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3657 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
c9687426 3658
948a7749 3659 if (sp->len)
c9687426 3660 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
7d67af2c 3661
e1d767f0
PB
3662 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3663 io_put_file(in);
7d67af2c
PB
3664 req->flags &= ~REQ_F_NEED_CLEANUP;
3665
7d67af2c
PB
3666 if (ret != sp->len)
3667 req_set_fail_links(req);
e1e16097 3668 io_req_complete(req, ret);
7d67af2c
PB
3669 return 0;
3670}
3671
2b188cc1
JA
3672/*
3673 * IORING_OP_NOP just posts a completion event, nothing else.
3674 */
889fca73 3675static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1
JA
3676{
3677 struct io_ring_ctx *ctx = req->ctx;
2b188cc1 3678
def596e9
JA
3679 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3680 return -EINVAL;
3681
889fca73 3682 __io_req_complete(req, issue_flags, 0, 0);
2b188cc1
JA
3683 return 0;
3684}
3685
1155c76a 3686static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
c992fe29 3687{
6b06314c 3688 struct io_ring_ctx *ctx = req->ctx;
c992fe29 3689
09bb8394
JA
3690 if (!req->file)
3691 return -EBADF;
c992fe29 3692
6b06314c 3693 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
def596e9 3694 return -EINVAL;
edafccee 3695 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
c992fe29
CH
3696 return -EINVAL;
3697
8ed8d3c3
JA
3698 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3699 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3700 return -EINVAL;
3701
3702 req->sync.off = READ_ONCE(sqe->off);
3703 req->sync.len = READ_ONCE(sqe->len);
c992fe29
CH
3704 return 0;
3705}
3706
45d189c6 3707static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3 3708{
8ed8d3c3 3709 loff_t end = req->sync.off + req->sync.len;
8ed8d3c3
JA
3710 int ret;
3711
ac45abc0 3712 /* fsync always requires a blocking context */
45d189c6 3713 if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0
PB
3714 return -EAGAIN;
3715
9adbd45d 3716 ret = vfs_fsync_range(req->file, req->sync.off,
8ed8d3c3
JA
3717 end > 0 ? end : LLONG_MAX,
3718 req->sync.flags & IORING_FSYNC_DATASYNC);
3719 if (ret < 0)
3720 req_set_fail_links(req);
e1e16097 3721 io_req_complete(req, ret);
c992fe29
CH
3722 return 0;
3723}
3724
d63d1b5e
JA
3725static int io_fallocate_prep(struct io_kiocb *req,
3726 const struct io_uring_sqe *sqe)
3727{
3728 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3729 return -EINVAL;
3232dd02
PB
3730 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3731 return -EINVAL;
d63d1b5e
JA
3732
3733 req->sync.off = READ_ONCE(sqe->off);
3734 req->sync.len = READ_ONCE(sqe->addr);
3735 req->sync.mode = READ_ONCE(sqe->len);
3736 return 0;
3737}
3738
45d189c6 3739static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
5d17b4a4 3740{
ac45abc0
PB
3741 int ret;
3742
d63d1b5e 3743 /* fallocate always requiring blocking context */
45d189c6 3744 if (issue_flags & IO_URING_F_NONBLOCK)
5d17b4a4 3745 return -EAGAIN;
ac45abc0
PB
3746 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3747 req->sync.len);
ac45abc0
PB
3748 if (ret < 0)
3749 req_set_fail_links(req);
e1e16097 3750 io_req_complete(req, ret);
5d17b4a4
JA
3751 return 0;
3752}
3753
ec65fea5 3754static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
b7bb4f7d 3755{
f8748881 3756 const char __user *fname;
15b71abe 3757 int ret;
b7bb4f7d 3758
ec65fea5 3759 if (unlikely(sqe->ioprio || sqe->buf_index))
15b71abe 3760 return -EINVAL;
ec65fea5 3761 if (unlikely(req->flags & REQ_F_FIXED_FILE))
cf3040ca 3762 return -EBADF;
03b1230c 3763
ec65fea5
PB
3764 /* open.how should be already initialised */
3765 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
08a1d26e 3766 req->open.how.flags |= O_LARGEFILE;
3529d8c2 3767
25e72d10
PB
3768 req->open.dfd = READ_ONCE(sqe->fd);
3769 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
f8748881 3770 req->open.filename = getname(fname);
15b71abe
JA
3771 if (IS_ERR(req->open.filename)) {
3772 ret = PTR_ERR(req->open.filename);
3773 req->open.filename = NULL;
3774 return ret;
3775 }
4022e7af 3776 req->open.nofile = rlimit(RLIMIT_NOFILE);
8fef80bf 3777 req->flags |= REQ_F_NEED_CLEANUP;
15b71abe 3778 return 0;
03b1230c
JA
3779}
3780
ec65fea5
PB
3781static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3782{
3783 u64 flags, mode;
3784
14587a46 3785 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded 3786 return -EINVAL;
ec65fea5
PB
3787 mode = READ_ONCE(sqe->len);
3788 flags = READ_ONCE(sqe->open_flags);
3789 req->open.how = build_open_how(flags, mode);
3790 return __io_openat_prep(req, sqe);
3791}
3792
cebdb986 3793static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
aa1fa28f 3794{
cebdb986 3795 struct open_how __user *how;
cebdb986 3796 size_t len;
0fa03c62
JA
3797 int ret;
3798
14587a46 3799 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4eb8dded 3800 return -EINVAL;
cebdb986
JA
3801 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3802 len = READ_ONCE(sqe->len);
cebdb986
JA
3803 if (len < OPEN_HOW_SIZE_VER0)
3804 return -EINVAL;
3529d8c2 3805
cebdb986
JA
3806 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3807 len);
3808 if (ret)
3809 return ret;
3529d8c2 3810
ec65fea5 3811 return __io_openat_prep(req, sqe);
cebdb986
JA
3812}
3813
45d189c6 3814static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
15b71abe
JA
3815{
3816 struct open_flags op;
15b71abe 3817 struct file *file;
3a81fd02
JA
3818 bool nonblock_set;
3819 bool resolve_nonblock;
15b71abe
JA
3820 int ret;
3821
cebdb986 3822 ret = build_open_flags(&req->open.how, &op);
15b71abe
JA
3823 if (ret)
3824 goto err;
3a81fd02
JA
3825 nonblock_set = op.open_flag & O_NONBLOCK;
3826 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
45d189c6 3827 if (issue_flags & IO_URING_F_NONBLOCK) {
3a81fd02
JA
3828 /*
3829 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3830 * it'll always -EAGAIN
3831 */
3832 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3833 return -EAGAIN;
3834 op.lookup_flags |= LOOKUP_CACHED;
3835 op.open_flag |= O_NONBLOCK;
3836 }
15b71abe 3837
4022e7af 3838 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
15b71abe
JA
3839 if (ret < 0)
3840 goto err;
3841
3842 file = do_filp_open(req->open.dfd, req->open.filename, &op);
3a81fd02 3843 /* only retry if RESOLVE_CACHED wasn't already set by application */
45d189c6
PB
3844 if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) &&
3845 file == ERR_PTR(-EAGAIN)) {
944d1444 3846 /*
3a81fd02
JA
3847 * We could hang on to this 'fd', but seems like marginal
3848 * gain for something that is now known to be a slower path.
3849 * So just put it, and we'll get a new one when we retry.
944d1444 3850 */
3a81fd02
JA
3851 put_unused_fd(ret);
3852 return -EAGAIN;
3853 }
3854
15b71abe
JA
3855 if (IS_ERR(file)) {
3856 put_unused_fd(ret);
3857 ret = PTR_ERR(file);
3858 } else {
45d189c6 3859 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3a81fd02 3860 file->f_flags &= ~O_NONBLOCK;
15b71abe
JA
3861 fsnotify_open(file);
3862 fd_install(ret, file);
3863 }
3864err:
3865 putname(req->open.filename);
8fef80bf 3866 req->flags &= ~REQ_F_NEED_CLEANUP;
15b71abe
JA
3867 if (ret < 0)
3868 req_set_fail_links(req);
e1e16097 3869 io_req_complete(req, ret);
15b71abe
JA
3870 return 0;
3871}
3872
45d189c6 3873static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
cebdb986 3874{
e45cff58 3875 return io_openat2(req, issue_flags);
cebdb986
JA
3876}
3877
067524e9
JA
3878static int io_remove_buffers_prep(struct io_kiocb *req,
3879 const struct io_uring_sqe *sqe)
3880{
3881 struct io_provide_buf *p = &req->pbuf;
3882 u64 tmp;
3883
3884 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3885 return -EINVAL;
3886
3887 tmp = READ_ONCE(sqe->fd);
3888 if (!tmp || tmp > USHRT_MAX)
3889 return -EINVAL;
3890
3891 memset(p, 0, sizeof(*p));
3892 p->nbufs = tmp;
3893 p->bgid = READ_ONCE(sqe->buf_group);
3894 return 0;
3895}
3896
3897static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3898 int bgid, unsigned nbufs)
3899{
3900 unsigned i = 0;
3901
3902 /* shouldn't happen */
3903 if (!nbufs)
3904 return 0;
3905
3906 /* the head kbuf is the list itself */
3907 while (!list_empty(&buf->list)) {
3908 struct io_buffer *nxt;
3909
3910 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3911 list_del(&nxt->list);
3912 kfree(nxt);
3913 if (++i == nbufs)
3914 return i;
3915 }
3916 i++;
3917 kfree(buf);
9e15c3a0 3918 xa_erase(&ctx->io_buffers, bgid);
067524e9
JA
3919
3920 return i;
3921}
3922
889fca73 3923static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
067524e9
JA
3924{
3925 struct io_provide_buf *p = &req->pbuf;
3926 struct io_ring_ctx *ctx = req->ctx;
3927 struct io_buffer *head;
3928 int ret = 0;
45d189c6 3929 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
067524e9
JA
3930
3931 io_ring_submit_lock(ctx, !force_nonblock);
3932
3933 lockdep_assert_held(&ctx->uring_lock);
3934
3935 ret = -ENOENT;
9e15c3a0 3936 head = xa_load(&ctx->io_buffers, p->bgid);
067524e9
JA
3937 if (head)
3938 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
067524e9
JA
3939 if (ret < 0)
3940 req_set_fail_links(req);
067524e9 3941
9fb8cb49
PB
3942 /* complete before unlock, IOPOLL may need the lock */
3943 __io_req_complete(req, issue_flags, ret, 0);
3944 io_ring_submit_unlock(ctx, !force_nonblock);
067524e9
JA
3945 return 0;
3946}
3947
ddf0322d
JA
3948static int io_provide_buffers_prep(struct io_kiocb *req,
3949 const struct io_uring_sqe *sqe)
3950{
d81269fe 3951 unsigned long size;
ddf0322d
JA
3952 struct io_provide_buf *p = &req->pbuf;
3953 u64 tmp;
3954
3955 if (sqe->ioprio || sqe->rw_flags)
3956 return -EINVAL;
3957
3958 tmp = READ_ONCE(sqe->fd);
3959 if (!tmp || tmp > USHRT_MAX)
3960 return -E2BIG;
3961 p->nbufs = tmp;
3962 p->addr = READ_ONCE(sqe->addr);
3963 p->len = READ_ONCE(sqe->len);
3964
d81269fe
PB
3965 size = (unsigned long)p->len * p->nbufs;
3966 if (!access_ok(u64_to_user_ptr(p->addr), size))
ddf0322d
JA
3967 return -EFAULT;
3968
3969 p->bgid = READ_ONCE(sqe->buf_group);
3970 tmp = READ_ONCE(sqe->off);
3971 if (tmp > USHRT_MAX)
3972 return -E2BIG;
3973 p->bid = tmp;
3974 return 0;
3975}
3976
3977static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
3978{
3979 struct io_buffer *buf;
3980 u64 addr = pbuf->addr;
3981 int i, bid = pbuf->bid;
3982
3983 for (i = 0; i < pbuf->nbufs; i++) {
3984 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
3985 if (!buf)
3986 break;
3987
3988 buf->addr = addr;
3989 buf->len = pbuf->len;
3990 buf->bid = bid;
3991 addr += pbuf->len;
3992 bid++;
3993 if (!*head) {
3994 INIT_LIST_HEAD(&buf->list);
3995 *head = buf;
3996 } else {
3997 list_add_tail(&buf->list, &(*head)->list);
3998 }
3999 }
4000
4001 return i ? i : -ENOMEM;
4002}
4003
889fca73 4004static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
ddf0322d
JA
4005{
4006 struct io_provide_buf *p = &req->pbuf;
4007 struct io_ring_ctx *ctx = req->ctx;
4008 struct io_buffer *head, *list;
4009 int ret = 0;
45d189c6 4010 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ddf0322d
JA
4011
4012 io_ring_submit_lock(ctx, !force_nonblock);
4013
4014 lockdep_assert_held(&ctx->uring_lock);
4015
9e15c3a0 4016 list = head = xa_load(&ctx->io_buffers, p->bgid);
ddf0322d
JA
4017
4018 ret = io_add_buffers(p, &head);
9e15c3a0
JA
4019 if (ret >= 0 && !list) {
4020 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4021 if (ret < 0)
067524e9 4022 __io_remove_buffers(ctx, head, p->bgid, -1U);
ddf0322d 4023 }
ddf0322d
JA
4024 if (ret < 0)
4025 req_set_fail_links(req);
9fb8cb49
PB
4026 /* complete before unlock, IOPOLL may need the lock */
4027 __io_req_complete(req, issue_flags, ret, 0);
4028 io_ring_submit_unlock(ctx, !force_nonblock);
ddf0322d 4029 return 0;
cebdb986
JA
4030}
4031
3e4827b0
JA
4032static int io_epoll_ctl_prep(struct io_kiocb *req,
4033 const struct io_uring_sqe *sqe)
4034{
4035#if defined(CONFIG_EPOLL)
4036 if (sqe->ioprio || sqe->buf_index)
4037 return -EINVAL;
6ca56f84 4038 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL)))
3232dd02 4039 return -EINVAL;
3e4827b0
JA
4040
4041 req->epoll.epfd = READ_ONCE(sqe->fd);
4042 req->epoll.op = READ_ONCE(sqe->len);
4043 req->epoll.fd = READ_ONCE(sqe->off);
4044
4045 if (ep_op_has_event(req->epoll.op)) {
4046 struct epoll_event __user *ev;
4047
4048 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4049 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4050 return -EFAULT;
4051 }
4052
4053 return 0;
4054#else
4055 return -EOPNOTSUPP;
4056#endif
4057}
4058
889fca73 4059static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
3e4827b0
JA
4060{
4061#if defined(CONFIG_EPOLL)
4062 struct io_epoll *ie = &req->epoll;
4063 int ret;
45d189c6 4064 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3e4827b0
JA
4065
4066 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4067 if (force_nonblock && ret == -EAGAIN)
4068 return -EAGAIN;
4069
4070 if (ret < 0)
4071 req_set_fail_links(req);
889fca73 4072 __io_req_complete(req, issue_flags, ret, 0);
3e4827b0
JA
4073 return 0;
4074#else
4075 return -EOPNOTSUPP;
4076#endif
4077}
4078
c1ca757b
JA
4079static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4080{
4081#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4082 if (sqe->ioprio || sqe->buf_index || sqe->off)
4083 return -EINVAL;
3232dd02
PB
4084 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4085 return -EINVAL;
c1ca757b
JA
4086
4087 req->madvise.addr = READ_ONCE(sqe->addr);
4088 req->madvise.len = READ_ONCE(sqe->len);
4089 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4090 return 0;
4091#else
4092 return -EOPNOTSUPP;
4093#endif
4094}
4095
45d189c6 4096static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
c1ca757b
JA
4097{
4098#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4099 struct io_madvise *ma = &req->madvise;
4100 int ret;
4101
45d189c6 4102 if (issue_flags & IO_URING_F_NONBLOCK)
c1ca757b
JA
4103 return -EAGAIN;
4104
0726b01e 4105 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
c1ca757b
JA
4106 if (ret < 0)
4107 req_set_fail_links(req);
e1e16097 4108 io_req_complete(req, ret);
c1ca757b
JA
4109 return 0;
4110#else
4111 return -EOPNOTSUPP;
4112#endif
4113}
4114
4840e418
JA
4115static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4116{
4117 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4118 return -EINVAL;
3232dd02
PB
4119 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4120 return -EINVAL;
4840e418
JA
4121
4122 req->fadvise.offset = READ_ONCE(sqe->off);
4123 req->fadvise.len = READ_ONCE(sqe->len);
4124 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4125 return 0;
4126}
4127
45d189c6 4128static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
4840e418
JA
4129{
4130 struct io_fadvise *fa = &req->fadvise;
4131 int ret;
4132
45d189c6 4133 if (issue_flags & IO_URING_F_NONBLOCK) {
3e69426d
JA
4134 switch (fa->advice) {
4135 case POSIX_FADV_NORMAL:
4136 case POSIX_FADV_RANDOM:
4137 case POSIX_FADV_SEQUENTIAL:
4138 break;
4139 default:
4140 return -EAGAIN;
4141 }
4142 }
4840e418
JA
4143
4144 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4145 if (ret < 0)
4146 req_set_fail_links(req);
e1e16097 4147 io_req_complete(req, ret);
4840e418
JA
4148 return 0;
4149}
4150
eddc7ef5
JA
4151static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4152{
6ca56f84 4153 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL)))
3232dd02 4154 return -EINVAL;
eddc7ef5
JA
4155 if (sqe->ioprio || sqe->buf_index)
4156 return -EINVAL;
9c280f90 4157 if (req->flags & REQ_F_FIXED_FILE)
cf3040ca 4158 return -EBADF;
eddc7ef5 4159
1d9e1288
BM
4160 req->statx.dfd = READ_ONCE(sqe->fd);
4161 req->statx.mask = READ_ONCE(sqe->len);
e62753e4 4162 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
1d9e1288
BM
4163 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4164 req->statx.flags = READ_ONCE(sqe->statx_flags);
eddc7ef5
JA
4165
4166 return 0;
4167}
4168
45d189c6 4169static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
eddc7ef5 4170{
1d9e1288 4171 struct io_statx *ctx = &req->statx;
eddc7ef5
JA
4172 int ret;
4173
59d70013 4174 if (issue_flags & IO_URING_F_NONBLOCK)
eddc7ef5
JA
4175 return -EAGAIN;
4176
e62753e4
BM
4177 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4178 ctx->buffer);
eddc7ef5 4179
eddc7ef5
JA
4180 if (ret < 0)
4181 req_set_fail_links(req);
e1e16097 4182 io_req_complete(req, ret);
eddc7ef5
JA
4183 return 0;
4184}
4185
b5dba59e
JA
4186static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4187{
14587a46 4188 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3232dd02 4189 return -EINVAL;
b5dba59e
JA
4190 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4191 sqe->rw_flags || sqe->buf_index)
4192 return -EINVAL;
9c280f90 4193 if (req->flags & REQ_F_FIXED_FILE)
cf3040ca 4194 return -EBADF;
b5dba59e
JA
4195
4196 req->close.fd = READ_ONCE(sqe->fd);
b5dba59e 4197 return 0;
b5dba59e
JA
4198}
4199
889fca73 4200static int io_close(struct io_kiocb *req, unsigned int issue_flags)
b5dba59e 4201{
9eac1904 4202 struct files_struct *files = current->files;
3af73b28 4203 struct io_close *close = &req->close;
9eac1904
JA
4204 struct fdtable *fdt;
4205 struct file *file;
b5dba59e
JA
4206 int ret;
4207
9eac1904
JA
4208 file = NULL;
4209 ret = -EBADF;
4210 spin_lock(&files->file_lock);
4211 fdt = files_fdtable(files);
4212 if (close->fd >= fdt->max_fds) {
4213 spin_unlock(&files->file_lock);
4214 goto err;
4215 }
4216 file = fdt->fd[close->fd];
4217 if (!file) {
4218 spin_unlock(&files->file_lock);
4219 goto err;
4220 }
4221
4222 if (file->f_op == &io_uring_fops) {
4223 spin_unlock(&files->file_lock);
4224 file = NULL;
4225 goto err;
3af73b28 4226 }
b5dba59e
JA
4227
4228 /* if the file has a flush method, be safe and punt to async */
45d189c6 4229 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
9eac1904 4230 spin_unlock(&files->file_lock);
0bf0eefd 4231 return -EAGAIN;
a2100672 4232 }
b5dba59e 4233
9eac1904
JA
4234 ret = __close_fd_get_file(close->fd, &file);
4235 spin_unlock(&files->file_lock);
4236 if (ret < 0) {
4237 if (ret == -ENOENT)
4238 ret = -EBADF;
4239 goto err;
4240 }
4241
3af73b28 4242 /* No ->flush() or already async, safely close from here */
9eac1904
JA
4243 ret = filp_close(file, current->files);
4244err:
3af73b28
PB
4245 if (ret < 0)
4246 req_set_fail_links(req);
9eac1904
JA
4247 if (file)
4248 fput(file);
889fca73 4249 __io_req_complete(req, issue_flags, ret, 0);
1a417f4e 4250 return 0;
b5dba59e
JA
4251}
4252
1155c76a 4253static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5d17b4a4
JA
4254{
4255 struct io_ring_ctx *ctx = req->ctx;
5d17b4a4 4256
5d17b4a4
JA
4257 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4258 return -EINVAL;
4259 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4260 return -EINVAL;
4261
8ed8d3c3
JA
4262 req->sync.off = READ_ONCE(sqe->off);
4263 req->sync.len = READ_ONCE(sqe->len);
4264 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
8ed8d3c3
JA
4265 return 0;
4266}
4267
45d189c6 4268static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3 4269{
8ed8d3c3
JA
4270 int ret;
4271
ac45abc0 4272 /* sync_file_range always requires a blocking context */
45d189c6 4273 if (issue_flags & IO_URING_F_NONBLOCK)
ac45abc0
PB
4274 return -EAGAIN;
4275
9adbd45d 4276 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
8ed8d3c3
JA
4277 req->sync.flags);
4278 if (ret < 0)
4279 req_set_fail_links(req);
e1e16097 4280 io_req_complete(req, ret);
5d17b4a4
JA
4281 return 0;
4282}
4283
469956e8 4284#if defined(CONFIG_NET)
02d27d89
PB
4285static int io_setup_async_msg(struct io_kiocb *req,
4286 struct io_async_msghdr *kmsg)
4287{
e8c2bc1f
JA
4288 struct io_async_msghdr *async_msg = req->async_data;
4289
4290 if (async_msg)
02d27d89 4291 return -EAGAIN;
e8c2bc1f 4292 if (io_alloc_async_data(req)) {
257e84a5 4293 kfree(kmsg->free_iov);
02d27d89
PB
4294 return -ENOMEM;
4295 }
e8c2bc1f 4296 async_msg = req->async_data;
02d27d89 4297 req->flags |= REQ_F_NEED_CLEANUP;
e8c2bc1f 4298 memcpy(async_msg, kmsg, sizeof(*kmsg));
2a780802 4299 async_msg->msg.msg_name = &async_msg->addr;
257e84a5
PB
4300 /* if were using fast_iov, set it to the new one */
4301 if (!async_msg->free_iov)
4302 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4303
02d27d89
PB
4304 return -EAGAIN;
4305}
4306
2ae523ed
PB
4307static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4308 struct io_async_msghdr *iomsg)
4309{
2ae523ed 4310 iomsg->msg.msg_name = &iomsg->addr;
257e84a5 4311 iomsg->free_iov = iomsg->fast_iov;
2ae523ed 4312 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
257e84a5 4313 req->sr_msg.msg_flags, &iomsg->free_iov);
2ae523ed
PB
4314}
4315
93642ef8
PB
4316static int io_sendmsg_prep_async(struct io_kiocb *req)
4317{
4318 int ret;
4319
93642ef8
PB
4320 ret = io_sendmsg_copy_hdr(req, req->async_data);
4321 if (!ret)
4322 req->flags |= REQ_F_NEED_CLEANUP;
4323 return ret;
4324}
4325
3529d8c2 4326static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
03b1230c 4327{
e47293fd 4328 struct io_sr_msg *sr = &req->sr_msg;
03b1230c 4329
d2b6f48b
PB
4330 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4331 return -EINVAL;
4332
e47293fd 4333 sr->msg_flags = READ_ONCE(sqe->msg_flags);
270a5940 4334 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
fddaface 4335 sr->len = READ_ONCE(sqe->len);
3529d8c2 4336
d8768362
JA
4337#ifdef CONFIG_COMPAT
4338 if (req->ctx->compat)
4339 sr->msg_flags |= MSG_CMSG_COMPAT;
4340#endif
93642ef8 4341 return 0;
03b1230c
JA
4342}
4343
889fca73 4344static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28f 4345{
6b754c8b 4346 struct io_async_msghdr iomsg, *kmsg;
0fa03c62 4347 struct socket *sock;
7a7cacba 4348 unsigned flags;
0031275d 4349 int min_ret = 0;
0fa03c62
JA
4350 int ret;
4351
dba4a925 4352 sock = sock_from_file(req->file);
7a7cacba 4353 if (unlikely(!sock))
dba4a925 4354 return -ENOTSOCK;
3529d8c2 4355
257e84a5
PB
4356 kmsg = req->async_data;
4357 if (!kmsg) {
7a7cacba
PB
4358 ret = io_sendmsg_copy_hdr(req, &iomsg);
4359 if (ret)
4360 return ret;
4361 kmsg = &iomsg;
0fa03c62 4362 }
0fa03c62 4363
76cd979f 4364 flags = req->sr_msg.msg_flags | MSG_NOSIGNAL;
7a7cacba
PB
4365 if (flags & MSG_DONTWAIT)
4366 req->flags |= REQ_F_NOWAIT;
45d189c6 4367 else if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba 4368 flags |= MSG_DONTWAIT;
e47293fd 4369
0031275d
SM
4370 if (flags & MSG_WAITALL)
4371 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4372
7a7cacba 4373 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
45d189c6 4374 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba
PB
4375 return io_setup_async_msg(req, kmsg);
4376 if (ret == -ERESTARTSYS)
4377 ret = -EINTR;
0fa03c62 4378
257e84a5
PB
4379 /* fast path, check for non-NULL to avoid function call */
4380 if (kmsg->free_iov)
4381 kfree(kmsg->free_iov);
99bc4c38 4382 req->flags &= ~REQ_F_NEED_CLEANUP;
0031275d 4383 if (ret < min_ret)
4e88d6e7 4384 req_set_fail_links(req);
889fca73 4385 __io_req_complete(req, issue_flags, ret, 0);
5d17b4a4 4386 return 0;
03b1230c 4387}
aa1fa28f 4388
889fca73 4389static int io_send(struct io_kiocb *req, unsigned int issue_flags)
fddaface 4390{
7a7cacba
PB
4391 struct io_sr_msg *sr = &req->sr_msg;
4392 struct msghdr msg;
4393 struct iovec iov;
fddaface 4394 struct socket *sock;
7a7cacba 4395 unsigned flags;
0031275d 4396 int min_ret = 0;
fddaface
JA
4397 int ret;
4398
dba4a925 4399 sock = sock_from_file(req->file);
7a7cacba 4400 if (unlikely(!sock))
dba4a925 4401 return -ENOTSOCK;
fddaface 4402
7a7cacba
PB
4403 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4404 if (unlikely(ret))
14db8411 4405 return ret;
fddaface 4406
7a7cacba
PB
4407 msg.msg_name = NULL;
4408 msg.msg_control = NULL;
4409 msg.msg_controllen = 0;
4410 msg.msg_namelen = 0;
fddaface 4411
76cd979f 4412 flags = req->sr_msg.msg_flags | MSG_NOSIGNAL;
7a7cacba
PB
4413 if (flags & MSG_DONTWAIT)
4414 req->flags |= REQ_F_NOWAIT;
45d189c6 4415 else if (issue_flags & IO_URING_F_NONBLOCK)
7a7cacba 4416 flags |= MSG_DONTWAIT;
fddaface 4417
0031275d
SM
4418 if (flags & MSG_WAITALL)
4419 min_ret = iov_iter_count(&msg.msg_iter);
4420
7a7cacba
PB
4421 msg.msg_flags = flags;
4422 ret = sock_sendmsg(sock, &msg);
45d189c6 4423 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
7a7cacba
PB
4424 return -EAGAIN;
4425 if (ret == -ERESTARTSYS)
4426 ret = -EINTR;
fddaface 4427
0031275d 4428 if (ret < min_ret)
fddaface 4429 req_set_fail_links(req);
889fca73 4430 __io_req_complete(req, issue_flags, ret, 0);
fddaface 4431 return 0;
fddaface
JA
4432}
4433
1400e697
PB
4434static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4435 struct io_async_msghdr *iomsg)
52de1fe1
JA
4436{
4437 struct io_sr_msg *sr = &req->sr_msg;
4438 struct iovec __user *uiov;
4439 size_t iov_len;
4440 int ret;
4441
1400e697
PB
4442 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4443 &iomsg->uaddr, &uiov, &iov_len);
52de1fe1
JA
4444 if (ret)
4445 return ret;
4446
4447 if (req->flags & REQ_F_BUFFER_SELECT) {
4448 if (iov_len > 1)
4449 return -EINVAL;
5476dfed 4450 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
52de1fe1 4451 return -EFAULT;
5476dfed 4452 sr->len = iomsg->fast_iov[0].iov_len;
257e84a5 4453 iomsg->free_iov = NULL;
52de1fe1 4454 } else {
257e84a5 4455 iomsg->free_iov = iomsg->fast_iov;
89cd35c5 4456 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
257e84a5 4457 &iomsg->free_iov, &iomsg->msg.msg_iter,
89cd35c5 4458 false);
52de1fe1
JA
4459 if (ret > 0)
4460 ret = 0;
4461 }
4462
4463 return ret;
4464}
4465
4466#ifdef CONFIG_COMPAT
4467static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
1400e697 4468 struct io_async_msghdr *iomsg)
52de1fe1
JA
4469{
4470 struct compat_msghdr __user *msg_compat;
4471 struct io_sr_msg *sr = &req->sr_msg;
4472 struct compat_iovec __user *uiov;
4473 compat_uptr_t ptr;
4474 compat_size_t len;
4475 int ret;
4476
270a5940 4477 msg_compat = (struct compat_msghdr __user *) sr->umsg;
1400e697 4478 ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
52de1fe1
JA
4479 &ptr, &len);
4480 if (ret)
4481 return ret;
4482
4483 uiov = compat_ptr(ptr);
4484 if (req->flags & REQ_F_BUFFER_SELECT) {
4485 compat_ssize_t clen;
4486
4487 if (len > 1)
4488 return -EINVAL;
4489 if (!access_ok(uiov, sizeof(*uiov)))
4490 return -EFAULT;
4491 if (__get_user(clen, &uiov->iov_len))
4492 return -EFAULT;
4493 if (clen < 0)
4494 return -EINVAL;
2d280bc8 4495 sr->len = clen;
257e84a5 4496 iomsg->free_iov = NULL;
52de1fe1 4497 } else {
257e84a5 4498 iomsg->free_iov = iomsg->fast_iov;
89cd35c5 4499 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
257e84a5 4500 UIO_FASTIOV, &iomsg->free_iov,
89cd35c5 4501 &iomsg->msg.msg_iter, true);
52de1fe1
JA
4502 if (ret < 0)
4503 return ret;
4504 }
4505
4506 return 0;
4507}
4508#endif
4509
1400e697
PB
4510static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4511 struct io_async_msghdr *iomsg)
52de1fe1 4512{
1400e697 4513 iomsg->msg.msg_name = &iomsg->addr;
52de1fe1
JA
4514
4515#ifdef CONFIG_COMPAT
4516 if (req->ctx->compat)
1400e697 4517 return __io_compat_recvmsg_copy_hdr(req, iomsg);
fddaface 4518#endif
52de1fe1 4519
1400e697 4520 return __io_recvmsg_copy_hdr(req, iomsg);
52de1fe1
JA
4521}
4522
bcda7baa 4523static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
7fbb1b54 4524 bool needs_lock)
bcda7baa
JA
4525{
4526 struct io_sr_msg *sr = &req->sr_msg;
4527 struct io_buffer *kbuf;
4528
bcda7baa
JA
4529 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4530 if (IS_ERR(kbuf))
4531 return kbuf;
4532
4533 sr->kbuf = kbuf;
4534 req->flags |= REQ_F_BUFFER_SELECTED;
bcda7baa 4535 return kbuf;
fddaface
JA
4536}
4537
7fbb1b54
PB
4538static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4539{
4540 return io_put_kbuf(req, req->sr_msg.kbuf);
4541}
4542
93642ef8 4543static int io_recvmsg_prep_async(struct io_kiocb *req)
aa1fa28f 4544{
99bc4c38 4545 int ret;
3529d8c2 4546
93642ef8
PB
4547 ret = io_recvmsg_copy_hdr(req, req->async_data);
4548 if (!ret)
4549 req->flags |= REQ_F_NEED_CLEANUP;
4550 return ret;
4551}
4552
4553static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4554{
4555 struct io_sr_msg *sr = &req->sr_msg;
4556
d2b6f48b
PB
4557 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4558 return -EINVAL;
4559
3529d8c2 4560 sr->msg_flags = READ_ONCE(sqe->msg_flags);
270a5940 4561 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
0b7b21e4 4562 sr->len = READ_ONCE(sqe->len);
bcda7baa 4563 sr->bgid = READ_ONCE(sqe->buf_group);
06b76d44 4564
d8768362
JA
4565#ifdef CONFIG_COMPAT
4566 if (req->ctx->compat)
4567 sr->msg_flags |= MSG_CMSG_COMPAT;
4568#endif
93642ef8 4569 return 0;
aa1fa28f
JA
4570}
4571
889fca73 4572static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
aa1fa28f 4573{
6b754c8b 4574 struct io_async_msghdr iomsg, *kmsg;
03b1230c 4575 struct socket *sock;
7fbb1b54 4576 struct io_buffer *kbuf;
7a7cacba 4577 unsigned flags;
0031275d 4578 int min_ret = 0;
52de1fe1 4579 int ret, cflags = 0;
45d189c6 4580 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
03b1230c 4581
dba4a925 4582 sock = sock_from_file(req->file);
7a7cacba 4583 if (unlikely(!sock))
dba4a925 4584 return -ENOTSOCK;
3529d8c2 4585
257e84a5
PB
4586 kmsg = req->async_data;
4587 if (!kmsg) {
7a7cacba
PB
4588 ret = io_recvmsg_copy_hdr(req, &iomsg);
4589 if (ret)
681fda8d 4590 return ret;
7a7cacba
PB
4591 kmsg = &iomsg;
4592 }
03b1230c 4593
bc02ef33 4594 if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b54 4595 kbuf = io_recv_buffer_select(req, !force_nonblock);
bc02ef33 4596 if (IS_ERR(kbuf))
52de1fe1 4597 return PTR_ERR(kbuf);
7a7cacba 4598 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
5476dfed
PB
4599 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4600 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
7a7cacba
PB
4601 1, req->sr_msg.len);
4602 }
52de1fe1 4603
76cd979f 4604 flags = req->sr_msg.msg_flags | MSG_NOSIGNAL;
7a7cacba
PB
4605 if (flags & MSG_DONTWAIT)
4606 req->flags |= REQ_F_NOWAIT;
4607 else if (force_nonblock)
4608 flags |= MSG_DONTWAIT;
e47293fd 4609
0031275d
SM
4610 if (flags & MSG_WAITALL)
4611 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4612
7a7cacba
PB
4613 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4614 kmsg->uaddr, flags);
0e1b6fe3
PB
4615 if (force_nonblock && ret == -EAGAIN)
4616 return io_setup_async_msg(req, kmsg);
7a7cacba
PB
4617 if (ret == -ERESTARTSYS)
4618 ret = -EINTR;
03b1230c 4619
7fbb1b54
PB
4620 if (req->flags & REQ_F_BUFFER_SELECTED)
4621 cflags = io_put_recv_kbuf(req);
257e84a5
PB
4622 /* fast path, check for non-NULL to avoid function call */
4623 if (kmsg->free_iov)
4624 kfree(kmsg->free_iov);
99bc4c38 4625 req->flags &= ~REQ_F_NEED_CLEANUP;
0031275d 4626 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
4e88d6e7 4627 req_set_fail_links(req);
889fca73 4628 __io_req_complete(req, issue_flags, ret, cflags);
03b1230c 4629 return 0;
0fa03c62 4630}
5d17b4a4 4631
889fca73 4632static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
fddaface 4633{
6b754c8b 4634 struct io_buffer *kbuf;
7a7cacba
PB
4635 struct io_sr_msg *sr = &req->sr_msg;
4636 struct msghdr msg;
4637 void __user *buf = sr->buf;
fddaface 4638 struct socket *sock;
7a7cacba
PB
4639 struct iovec iov;
4640 unsigned flags;
0031275d 4641 int min_ret = 0;
bcda7baa 4642 int ret, cflags = 0;
45d189c6 4643 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
fddaface 4644
dba4a925 4645 sock = sock_from_file(req->file);
7a7cacba 4646 if (unlikely(!sock))
dba4a925 4647 return -ENOTSOCK;
fddaface 4648
bc02ef33 4649 if (req->flags & REQ_F_BUFFER_SELECT) {
7fbb1b54 4650 kbuf = io_recv_buffer_select(req, !force_nonblock);
bcda7baa
JA
4651 if (IS_ERR(kbuf))
4652 return PTR_ERR(kbuf);
7a7cacba 4653 buf = u64_to_user_ptr(kbuf->addr);
bc02ef33 4654 }
bcda7baa 4655
7a7cacba 4656 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
14c32eee
PB
4657 if (unlikely(ret))
4658 goto out_free;
fddaface 4659
7a7cacba
PB
4660 msg.msg_name = NULL;
4661 msg.msg_control = NULL;
4662 msg.msg_controllen = 0;
4663 msg.msg_namelen = 0;
4664 msg.msg_iocb = NULL;
4665 msg.msg_flags = 0;
fddaface 4666
76cd979f 4667 flags = req->sr_msg.msg_flags | MSG_NOSIGNAL;
7a7cacba
PB
4668 if (flags & MSG_DONTWAIT)
4669 req->flags |= REQ_F_NOWAIT;
4670 else if (force_nonblock)
4671 flags |= MSG_DONTWAIT;
4672
0031275d
SM
4673 if (flags & MSG_WAITALL)
4674 min_ret = iov_iter_count(&msg.msg_iter);
4675
7a7cacba
PB
4676 ret = sock_recvmsg(sock, &msg, flags);
4677 if (force_nonblock && ret == -EAGAIN)
4678 return -EAGAIN;
4679 if (ret == -ERESTARTSYS)
4680 ret = -EINTR;
14c32eee 4681out_free:
7fbb1b54
PB
4682 if (req->flags & REQ_F_BUFFER_SELECTED)
4683 cflags = io_put_recv_kbuf(req);
0031275d 4684 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
fddaface 4685 req_set_fail_links(req);
889fca73 4686 __io_req_complete(req, issue_flags, ret, cflags);
fddaface 4687 return 0;
fddaface
JA
4688}
4689
3529d8c2 4690static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
17f2fe35 4691{
8ed8d3c3
JA
4692 struct io_accept *accept = &req->accept;
4693
14587a46 4694 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
17f2fe35 4695 return -EINVAL;
8042d6ce 4696 if (sqe->ioprio || sqe->len || sqe->buf_index)
17f2fe35
JA
4697 return -EINVAL;
4698
d55e5f5b
JA
4699 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4700 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
8ed8d3c3 4701 accept->flags = READ_ONCE(sqe->accept_flags);
09952e3e 4702 accept->nofile = rlimit(RLIMIT_NOFILE);
8ed8d3c3 4703 return 0;
8ed8d3c3 4704}
17f2fe35 4705
889fca73 4706static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
8ed8d3c3
JA
4707{
4708 struct io_accept *accept = &req->accept;
45d189c6 4709 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ac45abc0 4710 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
8ed8d3c3
JA
4711 int ret;
4712
e697deed
JX
4713 if (req->file->f_flags & O_NONBLOCK)
4714 req->flags |= REQ_F_NOWAIT;
4715
8ed8d3c3 4716 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
09952e3e
JA
4717 accept->addr_len, accept->flags,
4718 accept->nofile);
8ed8d3c3 4719 if (ret == -EAGAIN && force_nonblock)
17f2fe35 4720 return -EAGAIN;
ac45abc0
PB
4721 if (ret < 0) {
4722 if (ret == -ERESTARTSYS)
4723 ret = -EINTR;
4e88d6e7 4724 req_set_fail_links(req);
ac45abc0 4725 }
889fca73 4726 __io_req_complete(req, issue_flags, ret, 0);
17f2fe35 4727 return 0;
8ed8d3c3
JA
4728}
4729
93642ef8
PB
4730static int io_connect_prep_async(struct io_kiocb *req)
4731{
4732 struct io_async_connect *io = req->async_data;
4733 struct io_connect *conn = &req->connect;
4734
4735 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4736}
4737
3529d8c2 4738static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f499a021 4739{
3529d8c2 4740 struct io_connect *conn = &req->connect;
f499a021 4741
14587a46 4742 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3fbb51c1
JA
4743 return -EINVAL;
4744 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4745 return -EINVAL;
4746
3529d8c2
JA
4747 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4748 conn->addr_len = READ_ONCE(sqe->addr2);
93642ef8 4749 return 0;
f499a021
JA
4750}
4751
889fca73 4752static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
f8e85cf2 4753{
e8c2bc1f 4754 struct io_async_connect __io, *io;
f8e85cf2 4755 unsigned file_flags;
3fbb51c1 4756 int ret;
45d189c6 4757 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
f8e85cf2 4758
e8c2bc1f
JA
4759 if (req->async_data) {
4760 io = req->async_data;
f499a021 4761 } else {
3529d8c2
JA
4762 ret = move_addr_to_kernel(req->connect.addr,
4763 req->connect.addr_len,
e8c2bc1f 4764 &__io.address);
f499a021
JA
4765 if (ret)
4766 goto out;
4767 io = &__io;
4768 }
4769
3fbb51c1
JA
4770 file_flags = force_nonblock ? O_NONBLOCK : 0;
4771
e8c2bc1f 4772 ret = __sys_connect_file(req->file, &io->address,
3fbb51c1 4773 req->connect.addr_len, file_flags);
87f80d62 4774 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
e8c2bc1f 4775 if (req->async_data)
b7bb4f7d 4776 return -EAGAIN;
e8c2bc1f 4777 if (io_alloc_async_data(req)) {
f499a021
JA
4778 ret = -ENOMEM;
4779 goto out;
4780 }
e8c2bc1f 4781 memcpy(req->async_data, &__io, sizeof(__io));
f8e85cf2 4782 return -EAGAIN;
f499a021 4783 }
f8e85cf2
JA
4784 if (ret == -ERESTARTSYS)
4785 ret = -EINTR;
f499a021 4786out:
4e88d6e7
JA
4787 if (ret < 0)
4788 req_set_fail_links(req);
889fca73 4789 __io_req_complete(req, issue_flags, ret, 0);
f8e85cf2 4790 return 0;
469956e8
Y
4791}
4792#else /* !CONFIG_NET */
99a10081
JA
4793#define IO_NETOP_FN(op) \
4794static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4795{ \
4796 return -EOPNOTSUPP; \
4797}
4798
4799#define IO_NETOP_PREP(op) \
4800IO_NETOP_FN(op) \
4801static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4802{ \
4803 return -EOPNOTSUPP; \
4804} \
4805
4806#define IO_NETOP_PREP_ASYNC(op) \
4807IO_NETOP_PREP(op) \
4808static int io_##op##_prep_async(struct io_kiocb *req) \
4809{ \
4810 return -EOPNOTSUPP; \
4811}
4812
4813IO_NETOP_PREP_ASYNC(sendmsg);
4814IO_NETOP_PREP_ASYNC(recvmsg);
4815IO_NETOP_PREP_ASYNC(connect);
4816IO_NETOP_PREP(accept);
4817IO_NETOP_FN(send);
4818IO_NETOP_FN(recv);
469956e8 4819#endif /* CONFIG_NET */
f8e85cf2 4820
d7718a9d
JA
4821struct io_poll_table {
4822 struct poll_table_struct pt;
4823 struct io_kiocb *req;
4824 int error;
4825};
ce593a6c 4826
d7718a9d
JA
4827static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
4828 __poll_t mask, task_work_func_t func)
4829{
aa96bf8a 4830 int ret;
d7718a9d
JA
4831
4832 /* for instances that support it check for an event match first: */
4833 if (mask && !(mask & poll->events))
4834 return 0;
4835
4836 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4837
4838 list_del_init(&poll->wait.entry);
4839
d7718a9d 4840 req->result = mask;
7cbf1722 4841 req->task_work.func = func;
6d816e08 4842
d7718a9d 4843 /*
e3aabf95
JA
4844 * If this fails, then the task is exiting. When a task exits, the
4845 * work gets canceled, so just cancel this request as well instead
4846 * of executing it. We can't safely execute it anyway, as we may not
4847 * have the needed state needed for it anyway.
d7718a9d 4848 */
355fb9e2 4849 ret = io_req_task_work_add(req);
aa96bf8a 4850 if (unlikely(ret)) {
e3aabf95 4851 WRITE_ONCE(poll->canceled, true);
eab30c4d 4852 io_req_task_work_add_fallback(req, func);
aa96bf8a 4853 }
d7718a9d
JA
4854 return 1;
4855}
4856
74ce6ce4
JA
4857static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4858 __acquires(&req->ctx->completion_lock)
4859{
4860 struct io_ring_ctx *ctx = req->ctx;
4861
4862 if (!req->result && !READ_ONCE(poll->canceled)) {
4863 struct poll_table_struct pt = { ._key = poll->events };
4864
4865 req->result = vfs_poll(req->file, &pt) & poll->events;
4866 }
4867
4868 spin_lock_irq(&ctx->completion_lock);
4869 if (!req->result && !READ_ONCE(poll->canceled)) {
4870 add_wait_queue(poll->head, &poll->wait);
4871 return true;
4872 }
4873
4874 return false;
4875}
4876
d4e7cd36 4877static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
18bceab1 4878{
e8c2bc1f 4879 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
d4e7cd36 4880 if (req->opcode == IORING_OP_POLL_ADD)
e8c2bc1f 4881 return req->async_data;
d4e7cd36
JA
4882 return req->apoll->double_poll;
4883}
4884
4885static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
4886{
4887 if (req->opcode == IORING_OP_POLL_ADD)
4888 return &req->poll;
4889 return &req->apoll->poll;
4890}
4891
4892static void io_poll_remove_double(struct io_kiocb *req)
4893{
4894 struct io_poll_iocb *poll = io_poll_get_double(req);
18bceab1
JA
4895
4896 lockdep_assert_held(&req->ctx->completion_lock);
4897
4898 if (poll && poll->head) {
4899 struct wait_queue_head *head = poll->head;
4900
4901 spin_lock(&head->lock);
4902 list_del_init(&poll->wait.entry);
4903 if (poll->wait.private)
de9b4cca 4904 req_ref_put(req);
18bceab1
JA
4905 poll->head = NULL;
4906 spin_unlock(&head->lock);
4907 }
4908}
4909
88e41cf9 4910static bool io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
18bceab1
JA
4911{
4912 struct io_ring_ctx *ctx = req->ctx;
88e41cf9 4913 unsigned flags = IORING_CQE_F_MORE;
18bceab1 4914
88e41cf9 4915 if (!error && req->poll.canceled) {
45ab03b1 4916 error = -ECANCELED;
88e41cf9
JA
4917 req->poll.events |= EPOLLONESHOT;
4918 }
5082620f
JA
4919 if (!error)
4920 error = mangle_poll(mask);
b69de288
JA
4921 if (req->poll.events & EPOLLONESHOT)
4922 flags = 0;
4923 if (!__io_cqring_fill_event(req, error, flags)) {
5082620f 4924 io_poll_remove_waitqs(req);
88e41cf9
JA
4925 req->poll.done = true;
4926 flags = 0;
4927 }
18bceab1 4928 io_commit_cqring(ctx);
88e41cf9 4929 return !(flags & IORING_CQE_F_MORE);
18bceab1
JA
4930}
4931
dd221f46 4932static void io_poll_task_func(struct callback_head *cb)
18bceab1 4933{
dd221f46 4934 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
18bceab1 4935 struct io_ring_ctx *ctx = req->ctx;
dd221f46 4936 struct io_kiocb *nxt;
18bceab1
JA
4937
4938 if (io_poll_rewait(req, &req->poll)) {
4939 spin_unlock_irq(&ctx->completion_lock);
dd221f46 4940 } else {
88e41cf9
JA
4941 bool done, post_ev;
4942
4943 post_ev = done = io_poll_complete(req, req->result, 0);
4944 if (done) {
4945 hash_del(&req->hash_node);
4946 } else if (!(req->poll.events & EPOLLONESHOT)) {
4947 post_ev = true;
4948 req->result = 0;
4949 add_wait_queue(req->poll.head, &req->poll.wait);
4950 }
dd221f46 4951 spin_unlock_irq(&ctx->completion_lock);
18bceab1 4952
88e41cf9
JA
4953 if (post_ev)
4954 io_cqring_ev_posted(ctx);
4955 if (done) {
4956 nxt = io_put_req_find_next(req);
4957 if (nxt)
4958 __io_req_task_submit(nxt);
4959 }
dd221f46 4960 }
18bceab1
JA
4961}
4962
4963static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
4964 int sync, void *key)
4965{
4966 struct io_kiocb *req = wait->private;
d4e7cd36 4967 struct io_poll_iocb *poll = io_poll_get_single(req);
18bceab1
JA
4968 __poll_t mask = key_to_poll(key);
4969
4970 /* for instances that support it check for an event match first: */
4971 if (mask && !(mask & poll->events))
4972 return 0;
88e41cf9
JA
4973 if (!(poll->events & EPOLLONESHOT))
4974 return poll->wait.func(&poll->wait, mode, sync, key);
18bceab1 4975
8706e04e
JA
4976 list_del_init(&wait->entry);
4977
807abcb0 4978 if (poll && poll->head) {
18bceab1
JA
4979 bool done;
4980
807abcb0
JA
4981 spin_lock(&poll->head->lock);
4982 done = list_empty(&poll->wait.entry);
18bceab1 4983 if (!done)
807abcb0 4984 list_del_init(&poll->wait.entry);
d4e7cd36
JA
4985 /* make sure double remove sees this as being gone */
4986 wait->private = NULL;
807abcb0 4987 spin_unlock(&poll->head->lock);
c8b5e260
JA
4988 if (!done) {
4989 /* use wait func handler, so it matches the rq type */
4990 poll->wait.func(&poll->wait, mode, sync, key);
4991 }
18bceab1 4992 }
de9b4cca 4993 req_ref_put(req);
18bceab1
JA
4994 return 1;
4995}
4996
4997static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
4998 wait_queue_func_t wake_func)
4999{
5000 poll->head = NULL;
5001 poll->done = false;
5002 poll->canceled = false;
b69de288 5003 poll->update_events = poll->update_user_data = false;
464dca61
JA
5004#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
5005 /* mask in events that we always want/need */
5006 poll->events = events | IO_POLL_UNMASK;
18bceab1
JA
5007 INIT_LIST_HEAD(&poll->wait.entry);
5008 init_waitqueue_func_entry(&poll->wait, wake_func);
5009}
5010
5011static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
807abcb0
JA
5012 struct wait_queue_head *head,
5013 struct io_poll_iocb **poll_ptr)
18bceab1
JA
5014{
5015 struct io_kiocb *req = pt->req;
5016
5017 /*
5018 * If poll->head is already set, it's because the file being polled
5019 * uses multiple waitqueues for poll handling (eg one for read, one
5020 * for write). Setup a separate io_poll_iocb if this happens.
5021 */
5022 if (unlikely(poll->head)) {
58852d4d
PB
5023 struct io_poll_iocb *poll_one = poll;
5024
18bceab1 5025 /* already have a 2nd entry, fail a third attempt */
807abcb0 5026 if (*poll_ptr) {
18bceab1
JA
5027 pt->error = -EINVAL;
5028 return;
5029 }
1c3b3e65
JA
5030 /* double add on the same waitqueue head, ignore */
5031 if (poll->head == head)
5032 return;
18bceab1
JA
5033 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5034 if (!poll) {
5035 pt->error = -ENOMEM;
5036 return;
5037 }
58852d4d 5038 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
de9b4cca 5039 req_ref_get(req);
18bceab1 5040 poll->wait.private = req;
807abcb0 5041 *poll_ptr = poll;
18bceab1
JA
5042 }
5043
5044 pt->error = 0;
5045 poll->head = head;
a31eb4a2
JX
5046
5047 if (poll->events & EPOLLEXCLUSIVE)
5048 add_wait_queue_exclusive(head, &poll->wait);
5049 else
5050 add_wait_queue(head, &poll->wait);
18bceab1
JA
5051}
5052
5053static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5054 struct poll_table_struct *p)
5055{
5056 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
807abcb0 5057 struct async_poll *apoll = pt->req->apoll;
18bceab1 5058
807abcb0 5059 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
18bceab1
JA
5060}
5061
d7718a9d
JA
5062static void io_async_task_func(struct callback_head *cb)
5063{
5064 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
5065 struct async_poll *apoll = req->apoll;
5066 struct io_ring_ctx *ctx = req->ctx;
5067
5068 trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
5069
74ce6ce4 5070 if (io_poll_rewait(req, &apoll->poll)) {
d7718a9d 5071 spin_unlock_irq(&ctx->completion_lock);
74ce6ce4 5072 return;
d7718a9d
JA
5073 }
5074
31067255 5075 /* If req is still hashed, it cannot have been canceled. Don't check. */
0be0b0e3 5076 if (hash_hashed(&req->hash_node))
74ce6ce4 5077 hash_del(&req->hash_node);
2bae047e 5078
d4e7cd36 5079 io_poll_remove_double(req);
74ce6ce4
JA
5080 spin_unlock_irq(&ctx->completion_lock);
5081
0be0b0e3
PB
5082 if (!READ_ONCE(apoll->poll.canceled))
5083 __io_req_task_submit(req);
5084 else
2593553a 5085 io_req_complete_failed(req, -ECANCELED);
aa340845 5086
807abcb0 5087 kfree(apoll->double_poll);
31067255 5088 kfree(apoll);
d7718a9d
JA
5089}
5090
5091static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5092 void *key)
5093{
5094 struct io_kiocb *req = wait->private;
5095 struct io_poll_iocb *poll = &req->apoll->poll;
5096
5097 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5098 key_to_poll(key));
5099
5100 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5101}
5102
5103static void io_poll_req_insert(struct io_kiocb *req)
5104{
5105 struct io_ring_ctx *ctx = req->ctx;
5106 struct hlist_head *list;
5107
5108 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5109 hlist_add_head(&req->hash_node, list);
5110}
5111
5112static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5113 struct io_poll_iocb *poll,
5114 struct io_poll_table *ipt, __poll_t mask,
5115 wait_queue_func_t wake_func)
5116 __acquires(&ctx->completion_lock)
5117{
5118 struct io_ring_ctx *ctx = req->ctx;
5119 bool cancel = false;
5120
4d52f338 5121 INIT_HLIST_NODE(&req->hash_node);
18bceab1 5122 io_init_poll_iocb(poll, mask, wake_func);
b90cd197 5123 poll->file = req->file;
18bceab1 5124 poll->wait.private = req;
d7718a9d
JA
5125
5126 ipt->pt._key = mask;
5127 ipt->req = req;
5128 ipt->error = -EINVAL;
5129
d7718a9d
JA
5130 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5131
5132 spin_lock_irq(&ctx->completion_lock);
5133 if (likely(poll->head)) {
5134 spin_lock(&poll->head->lock);
5135 if (unlikely(list_empty(&poll->wait.entry))) {
5136 if (ipt->error)
5137 cancel = true;
5138 ipt->error = 0;
5139 mask = 0;
5140 }
88e41cf9 5141 if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
d7718a9d
JA
5142 list_del_init(&poll->wait.entry);
5143 else if (cancel)
5144 WRITE_ONCE(poll->canceled, true);
5145 else if (!poll->done) /* actually waiting for an event */
5146 io_poll_req_insert(req);
5147 spin_unlock(&poll->head->lock);
5148 }
5149
5150 return mask;
5151}
5152
5153static bool io_arm_poll_handler(struct io_kiocb *req)
5154{
5155 const struct io_op_def *def = &io_op_defs[req->opcode];
5156 struct io_ring_ctx *ctx = req->ctx;
5157 struct async_poll *apoll;
5158 struct io_poll_table ipt;
5159 __poll_t mask, ret;
9dab14b8 5160 int rw;
d7718a9d
JA
5161
5162 if (!req->file || !file_can_poll(req->file))
5163 return false;
24c74678 5164 if (req->flags & REQ_F_POLLED)
d7718a9d 5165 return false;
9dab14b8
JA
5166 if (def->pollin)
5167 rw = READ;
5168 else if (def->pollout)
5169 rw = WRITE;
5170 else
5171 return false;
5172 /* if we can't nonblock try, then no point in arming a poll handler */
7b29f92d 5173 if (!io_file_supports_async(req, rw))
d7718a9d
JA
5174 return false;
5175
5176 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5177 if (unlikely(!apoll))
5178 return false;
807abcb0 5179 apoll->double_poll = NULL;
d7718a9d
JA
5180
5181 req->flags |= REQ_F_POLLED;
d7718a9d 5182 req->apoll = apoll;
d7718a9d 5183
88e41cf9 5184 mask = EPOLLONESHOT;
d7718a9d 5185 if (def->pollin)
8755d97a 5186 mask |= POLLIN | POLLRDNORM;
d7718a9d
JA
5187 if (def->pollout)
5188 mask |= POLLOUT | POLLWRNORM;
901341bb
LH
5189
5190 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5191 if ((req->opcode == IORING_OP_RECVMSG) &&
5192 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5193 mask &= ~POLLIN;
5194
d7718a9d
JA
5195 mask |= POLLERR | POLLPRI;
5196
5197 ipt.pt._qproc = io_async_queue_proc;
5198
5199 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5200 io_async_wake);
a36da65c 5201 if (ret || ipt.error) {
d4e7cd36 5202 io_poll_remove_double(req);
d7718a9d 5203 spin_unlock_irq(&ctx->completion_lock);
807abcb0 5204 kfree(apoll->double_poll);
d7718a9d
JA
5205 kfree(apoll);
5206 return false;
5207 }
5208 spin_unlock_irq(&ctx->completion_lock);
5209 trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
5210 apoll->poll.events);
5211 return true;
5212}
5213
5214static bool __io_poll_remove_one(struct io_kiocb *req,
b2e720ac 5215 struct io_poll_iocb *poll, bool do_cancel)
221c5eb2 5216{
b41e9852 5217 bool do_complete = false;
221c5eb2 5218
5082620f
JA
5219 if (!poll->head)
5220 return false;
221c5eb2 5221 spin_lock(&poll->head->lock);
b2e720ac
JA
5222 if (do_cancel)
5223 WRITE_ONCE(poll->canceled, true);
392edb45
JA
5224 if (!list_empty(&poll->wait.entry)) {
5225 list_del_init(&poll->wait.entry);
b41e9852 5226 do_complete = true;
221c5eb2
JA
5227 }
5228 spin_unlock(&poll->head->lock);
3bfa5bcb 5229 hash_del(&req->hash_node);
d7718a9d
JA
5230 return do_complete;
5231}
5232
b2c3f7e1 5233static bool io_poll_remove_waitqs(struct io_kiocb *req)
d7718a9d
JA
5234{
5235 bool do_complete;
5236
d4e7cd36
JA
5237 io_poll_remove_double(req);
5238
d7718a9d 5239 if (req->opcode == IORING_OP_POLL_ADD) {
b2e720ac 5240 do_complete = __io_poll_remove_one(req, &req->poll, true);
d7718a9d 5241 } else {
3bfa5bcb
JA
5242 struct async_poll *apoll = req->apoll;
5243
d7718a9d 5244 /* non-poll requests have submit ref still */
b2e720ac 5245 do_complete = __io_poll_remove_one(req, &apoll->poll, true);
3bfa5bcb 5246 if (do_complete) {
d7718a9d 5247 io_put_req(req);
807abcb0 5248 kfree(apoll->double_poll);
3bfa5bcb
JA
5249 kfree(apoll);
5250 }
b1f573bd
XW
5251 }
5252
b2c3f7e1
JA
5253 return do_complete;
5254}
5255
5256static bool io_poll_remove_one(struct io_kiocb *req)
5257{
5258 bool do_complete;
5259
5260 do_complete = io_poll_remove_waitqs(req);
b41e9852
JA
5261 if (do_complete) {
5262 io_cqring_fill_event(req, -ECANCELED);
5263 io_commit_cqring(req->ctx);
f254ac04 5264 req_set_fail_links(req);
216578e5 5265 io_put_req_deferred(req, 1);
b41e9852
JA
5266 }
5267
5268 return do_complete;
221c5eb2
JA
5269}
5270
76e1b642
JA
5271/*
5272 * Returns true if we found and killed one or more poll requests
5273 */
6b81928d
PB
5274static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
5275 struct files_struct *files)
221c5eb2 5276{
78076bb6 5277 struct hlist_node *tmp;
221c5eb2 5278 struct io_kiocb *req;
8e2e1faf 5279 int posted = 0, i;
221c5eb2
JA
5280
5281 spin_lock_irq(&ctx->completion_lock);
78076bb6
JA
5282 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5283 struct hlist_head *list;
5284
5285 list = &ctx->cancel_hash[i];
f3606e3a 5286 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
6b81928d 5287 if (io_match_task(req, tsk, files))
f3606e3a
JA
5288 posted += io_poll_remove_one(req);
5289 }
221c5eb2
JA
5290 }
5291 spin_unlock_irq(&ctx->completion_lock);
b41e9852 5292
8e2e1faf
JA
5293 if (posted)
5294 io_cqring_ev_posted(ctx);
76e1b642
JA
5295
5296 return posted != 0;
221c5eb2
JA
5297}
5298
b2cb805f 5299static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr)
47f46768 5300{
78076bb6 5301 struct hlist_head *list;
47f46768
JA
5302 struct io_kiocb *req;
5303
78076bb6
JA
5304 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5305 hlist_for_each_entry(req, list, hash_node) {
b41e9852
JA
5306 if (sqe_addr != req->user_data)
5307 continue;
b2cb805f 5308 return req;
47f46768
JA
5309 }
5310
b2cb805f
JA
5311 return NULL;
5312}
5313
5314static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
5315{
5316 struct io_kiocb *req;
5317
5318 req = io_poll_find(ctx, sqe_addr);
5319 if (!req)
5320 return -ENOENT;
5321 if (io_poll_remove_one(req))
5322 return 0;
5323
5324 return -EALREADY;
47f46768
JA
5325}
5326
3529d8c2
JA
5327static int io_poll_remove_prep(struct io_kiocb *req,
5328 const struct io_uring_sqe *sqe)
0969e783 5329{
0969e783
JA
5330 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5331 return -EINVAL;
5332 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
5333 sqe->poll_events)
5334 return -EINVAL;
5335
018043be 5336 req->poll_remove.addr = READ_ONCE(sqe->addr);
0969e783
JA
5337 return 0;
5338}
5339
221c5eb2
JA
5340/*
5341 * Find a running poll command that matches one specified in sqe->addr,
5342 * and remove it if found.
5343 */
61e98203 5344static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
221c5eb2
JA
5345{
5346 struct io_ring_ctx *ctx = req->ctx;
47f46768 5347 int ret;
221c5eb2 5348
221c5eb2 5349 spin_lock_irq(&ctx->completion_lock);
018043be 5350 ret = io_poll_cancel(ctx, req->poll_remove.addr);
221c5eb2
JA
5351 spin_unlock_irq(&ctx->completion_lock);
5352
4e88d6e7
JA
5353 if (ret < 0)
5354 req_set_fail_links(req);
e1e16097 5355 io_req_complete(req, ret);
221c5eb2
JA
5356 return 0;
5357}
5358
221c5eb2
JA
5359static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5360 void *key)
5361{
c2f2eb7d
JA
5362 struct io_kiocb *req = wait->private;
5363 struct io_poll_iocb *poll = &req->poll;
221c5eb2 5364
d7718a9d 5365 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
221c5eb2
JA
5366}
5367
221c5eb2
JA
5368static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5369 struct poll_table_struct *p)
5370{
5371 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5372
e8c2bc1f 5373 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
eac406c6
JA
5374}
5375
3529d8c2 5376static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
221c5eb2
JA
5377{
5378 struct io_poll_iocb *poll = &req->poll;
88e41cf9 5379 u32 events, flags;
221c5eb2
JA
5380
5381 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5382 return -EINVAL;
b69de288 5383 if (sqe->ioprio || sqe->buf_index)
88e41cf9
JA
5384 return -EINVAL;
5385 flags = READ_ONCE(sqe->len);
b69de288
JA
5386 if (flags & ~(IORING_POLL_ADD_MULTI | IORING_POLL_UPDATE_EVENTS |
5387 IORING_POLL_UPDATE_USER_DATA))
221c5eb2 5388 return -EINVAL;
5769a351
JX
5389 events = READ_ONCE(sqe->poll32_events);
5390#ifdef __BIG_ENDIAN
5391 events = swahw32(events);
5392#endif
b69de288 5393 if (!(flags & IORING_POLL_ADD_MULTI))
88e41cf9 5394 events |= EPOLLONESHOT;
b69de288
JA
5395 poll->update_events = poll->update_user_data = false;
5396 if (flags & IORING_POLL_UPDATE_EVENTS) {
5397 poll->update_events = true;
5398 poll->old_user_data = READ_ONCE(sqe->addr);
5399 }
5400 if (flags & IORING_POLL_UPDATE_USER_DATA) {
5401 poll->update_user_data = true;
5402 poll->new_user_data = READ_ONCE(sqe->off);
5403 }
5404 if (!(poll->update_events || poll->update_user_data) &&
5405 (sqe->off || sqe->addr))
5406 return -EINVAL;
88e41cf9
JA
5407 poll->events = demangle_poll(events) |
5408 (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
0969e783
JA
5409 return 0;
5410}
5411
b69de288 5412static int __io_poll_add(struct io_kiocb *req)
0969e783
JA
5413{
5414 struct io_poll_iocb *poll = &req->poll;
5415 struct io_ring_ctx *ctx = req->ctx;
5416 struct io_poll_table ipt;
0969e783 5417 __poll_t mask;
0969e783 5418
d7718a9d 5419 ipt.pt._qproc = io_poll_queue_proc;
36703247 5420
d7718a9d
JA
5421 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5422 io_poll_wake);
221c5eb2 5423
8c838788 5424 if (mask) { /* no async, we'd stolen it */
221c5eb2 5425 ipt.error = 0;
b0dd8a41 5426 io_poll_complete(req, mask, 0);
221c5eb2 5427 }
221c5eb2
JA
5428 spin_unlock_irq(&ctx->completion_lock);
5429
8c838788
JA
5430 if (mask) {
5431 io_cqring_ev_posted(ctx);
88e41cf9
JA
5432 if (poll->events & EPOLLONESHOT)
5433 io_put_req(req);
221c5eb2 5434 }
8c838788 5435 return ipt.error;
221c5eb2
JA
5436}
5437
b69de288
JA
5438static int io_poll_update(struct io_kiocb *req)
5439{
5440 struct io_ring_ctx *ctx = req->ctx;
5441 struct io_kiocb *preq;
5442 int ret;
5443
5444 spin_lock_irq(&ctx->completion_lock);
5445 preq = io_poll_find(ctx, req->poll.old_user_data);
5446 if (!preq) {
5447 ret = -ENOENT;
5448 goto err;
5449 } else if (preq->opcode != IORING_OP_POLL_ADD) {
5450 /* don't allow internal poll updates */
5451 ret = -EACCES;
5452 goto err;
5453 }
b2e720ac
JA
5454 if (!__io_poll_remove_one(preq, &preq->poll, false)) {
5455 if (preq->poll.events & EPOLLONESHOT) {
5456 ret = -EALREADY;
5457 goto err;
5458 }
b69de288
JA
5459 }
5460 /* we now have a detached poll request. reissue. */
5461 ret = 0;
5462err:
5463 spin_unlock_irq(&ctx->completion_lock);
5464 if (ret < 0) {
5465 req_set_fail_links(req);
5466 io_req_complete(req, ret);
5467 return 0;
5468 }
5469 /* only mask one event flags, keep behavior flags */
5470 if (req->poll.update_events) {
5471 preq->poll.events &= ~0xffff;
5472 preq->poll.events |= req->poll.events & 0xffff;
5473 preq->poll.events |= IO_POLL_UNMASK;
5474 }
5475 if (req->poll.update_user_data)
5476 preq->user_data = req->poll.new_user_data;
5477
5478 /* complete update request, we're done with it */
5479 io_req_complete(req, ret);
5480
5481 ret = __io_poll_add(preq);
5482 if (ret < 0) {
5483 req_set_fail_links(preq);
5484 io_req_complete(preq, ret);
5485 }
5486 return 0;
5487}
5488
5489static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
5490{
5491 if (!req->poll.update_events && !req->poll.update_user_data)
5492 return __io_poll_add(req);
5493 return io_poll_update(req);
5494}
5495
5262f567
JA
5496static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5497{
ad8a48ac
JA
5498 struct io_timeout_data *data = container_of(timer,
5499 struct io_timeout_data, timer);
5500 struct io_kiocb *req = data->req;
5501 struct io_ring_ctx *ctx = req->ctx;
5262f567
JA
5502 unsigned long flags;
5503
5262f567 5504 spin_lock_irqsave(&ctx->completion_lock, flags);
a71976f3 5505 list_del_init(&req->timeout.list);
01cec8c1
PB
5506 atomic_set(&req->ctx->cq_timeouts,
5507 atomic_read(&req->ctx->cq_timeouts) + 1);
5508
78e19bbe 5509 io_cqring_fill_event(req, -ETIME);
5262f567
JA
5510 io_commit_cqring(ctx);
5511 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5512
5513 io_cqring_ev_posted(ctx);
4e88d6e7 5514 req_set_fail_links(req);
5262f567
JA
5515 io_put_req(req);
5516 return HRTIMER_NORESTART;
5517}
5518
fbd15848
PB
5519static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5520 __u64 user_data)
f254ac04 5521{
fbd15848 5522 struct io_timeout_data *io;
47f46768
JA
5523 struct io_kiocb *req;
5524 int ret = -ENOENT;
f254ac04 5525
135fcde8 5526 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
47f46768 5527 if (user_data == req->user_data) {
47f46768
JA
5528 ret = 0;
5529 break;
5530 }
5531 }
5532
5533 if (ret == -ENOENT)
fbd15848
PB
5534 return ERR_PTR(ret);
5535
5536 io = req->async_data;
e8c2bc1f 5537 ret = hrtimer_try_to_cancel(&io->timer);
f254ac04 5538 if (ret == -1)
fbd15848 5539 return ERR_PTR(-EALREADY);
a71976f3 5540 list_del_init(&req->timeout.list);
fbd15848
PB
5541 return req;
5542}
47f46768 5543
fbd15848
PB
5544static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
5545{
5546 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5547
5548 if (IS_ERR(req))
5549 return PTR_ERR(req);
f254ac04
JA
5550
5551 req_set_fail_links(req);
f254ac04 5552 io_cqring_fill_event(req, -ECANCELED);
216578e5 5553 io_put_req_deferred(req, 1);
f254ac04
JA
5554 return 0;
5555}
5556
9c8e11b3
PB
5557static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5558 struct timespec64 *ts, enum hrtimer_mode mode)
47f46768 5559{
9c8e11b3
PB
5560 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5561 struct io_timeout_data *data;
47f46768 5562
9c8e11b3
PB
5563 if (IS_ERR(req))
5564 return PTR_ERR(req);
47f46768 5565
9c8e11b3
PB
5566 req->timeout.off = 0; /* noseq */
5567 data = req->async_data;
5568 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5569 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5570 data->timer.function = io_timeout_fn;
5571 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5572 return 0;
47f46768
JA
5573}
5574
3529d8c2
JA
5575static int io_timeout_remove_prep(struct io_kiocb *req,
5576 const struct io_uring_sqe *sqe)
b29472ee 5577{
9c8e11b3
PB
5578 struct io_timeout_rem *tr = &req->timeout_rem;
5579
b29472ee
JA
5580 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5581 return -EINVAL;
61710e43
DA
5582 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5583 return -EINVAL;
9c8e11b3 5584 if (sqe->ioprio || sqe->buf_index || sqe->len)
b29472ee
JA
5585 return -EINVAL;
5586
9c8e11b3
PB
5587 tr->addr = READ_ONCE(sqe->addr);
5588 tr->flags = READ_ONCE(sqe->timeout_flags);
5589 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5590 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5591 return -EINVAL;
5592 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5593 return -EFAULT;
5594 } else if (tr->flags) {
5595 /* timeout removal doesn't support flags */
b29472ee 5596 return -EINVAL;
9c8e11b3 5597 }
b29472ee 5598
b29472ee
JA
5599 return 0;
5600}
5601
8662daec
PB
5602static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5603{
5604 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5605 : HRTIMER_MODE_REL;
5606}
5607
11365043
JA
5608/*
5609 * Remove or update an existing timeout command
5610 */
61e98203 5611static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
11365043 5612{
9c8e11b3 5613 struct io_timeout_rem *tr = &req->timeout_rem;
11365043 5614 struct io_ring_ctx *ctx = req->ctx;
47f46768 5615 int ret;
11365043 5616
11365043 5617 spin_lock_irq(&ctx->completion_lock);
8662daec 5618 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
9c8e11b3 5619 ret = io_timeout_cancel(ctx, tr->addr);
8662daec
PB
5620 else
5621 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5622 io_translate_timeout_mode(tr->flags));
11365043 5623
47f46768 5624 io_cqring_fill_event(req, ret);
11365043
JA
5625 io_commit_cqring(ctx);
5626 spin_unlock_irq(&ctx->completion_lock);
5262f567 5627 io_cqring_ev_posted(ctx);
4e88d6e7
JA
5628 if (ret < 0)
5629 req_set_fail_links(req);
ec9c02ad 5630 io_put_req(req);
11365043 5631 return 0;
5262f567
JA
5632}
5633
3529d8c2 5634static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2d28390a 5635 bool is_timeout_link)
5262f567 5636{
ad8a48ac 5637 struct io_timeout_data *data;
a41525ab 5638 unsigned flags;
56080b02 5639 u32 off = READ_ONCE(sqe->off);
5262f567 5640
ad8a48ac 5641 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5262f567 5642 return -EINVAL;
ad8a48ac 5643 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
a41525ab 5644 return -EINVAL;
56080b02 5645 if (off && is_timeout_link)
2d28390a 5646 return -EINVAL;
a41525ab
JA
5647 flags = READ_ONCE(sqe->timeout_flags);
5648 if (flags & ~IORING_TIMEOUT_ABS)
5262f567 5649 return -EINVAL;
bdf20073 5650
bfe68a22 5651 req->timeout.off = off;
26a61679 5652
e8c2bc1f 5653 if (!req->async_data && io_alloc_async_data(req))
26a61679
JA
5654 return -ENOMEM;
5655
e8c2bc1f 5656 data = req->async_data;
ad8a48ac 5657 data->req = req;
ad8a48ac
JA
5658
5659 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
5262f567
JA
5660 return -EFAULT;
5661
8662daec 5662 data->mode = io_translate_timeout_mode(flags);
ad8a48ac 5663 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
2482b58f
PB
5664 if (is_timeout_link)
5665 io_req_track_inflight(req);
ad8a48ac
JA
5666 return 0;
5667}
5668
61e98203 5669static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
ad8a48ac 5670{
ad8a48ac 5671 struct io_ring_ctx *ctx = req->ctx;
e8c2bc1f 5672 struct io_timeout_data *data = req->async_data;
ad8a48ac 5673 struct list_head *entry;
bfe68a22 5674 u32 tail, off = req->timeout.off;
ad8a48ac 5675
733f5c95 5676 spin_lock_irq(&ctx->completion_lock);
93bd25bb 5677
5262f567
JA
5678 /*
5679 * sqe->off holds how many events that need to occur for this
93bd25bb
JA
5680 * timeout event to be satisfied. If it isn't set, then this is
5681 * a pure timeout request, sequence isn't used.
5262f567 5682 */
8eb7e2d0 5683 if (io_is_timeout_noseq(req)) {
93bd25bb
JA
5684 entry = ctx->timeout_list.prev;
5685 goto add;
5686 }
5262f567 5687
bfe68a22
PB
5688 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5689 req->timeout.target_seq = tail + off;
5262f567 5690
f010505b
MDG
5691 /* Update the last seq here in case io_flush_timeouts() hasn't.
5692 * This is safe because ->completion_lock is held, and submissions
5693 * and completions are never mixed in the same ->completion_lock section.
5694 */
5695 ctx->cq_last_tm_flush = tail;
5696
5262f567
JA
5697 /*
5698 * Insertion sort, ensuring the first entry in the list is always
5699 * the one we need first.
5700 */
5262f567 5701 list_for_each_prev(entry, &ctx->timeout_list) {
135fcde8
PB
5702 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5703 timeout.list);
5262f567 5704
8eb7e2d0 5705 if (io_is_timeout_noseq(nxt))
93bd25bb 5706 continue;
bfe68a22
PB
5707 /* nxt.seq is behind @tail, otherwise would've been completed */
5708 if (off >= nxt->timeout.target_seq - tail)
5262f567
JA
5709 break;
5710 }
93bd25bb 5711add:
135fcde8 5712 list_add(&req->timeout.list, entry);
ad8a48ac
JA
5713 data->timer.function = io_timeout_fn;
5714 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
5262f567 5715 spin_unlock_irq(&ctx->completion_lock);
5262f567
JA
5716 return 0;
5717}
5262f567 5718
f458dd84
PB
5719struct io_cancel_data {
5720 struct io_ring_ctx *ctx;
5721 u64 user_data;
5722};
5723
62755e35
JA
5724static bool io_cancel_cb(struct io_wq_work *work, void *data)
5725{
5726 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
f458dd84 5727 struct io_cancel_data *cd = data;
62755e35 5728
f458dd84 5729 return req->ctx == cd->ctx && req->user_data == cd->user_data;
62755e35
JA
5730}
5731
f458dd84
PB
5732static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
5733 struct io_ring_ctx *ctx)
62755e35 5734{
f458dd84 5735 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
62755e35 5736 enum io_wq_cancel cancel_ret;
62755e35
JA
5737 int ret = 0;
5738
f458dd84 5739 if (!tctx || !tctx->io_wq)
5aa75ed5
JA
5740 return -ENOENT;
5741
f458dd84 5742 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
62755e35
JA
5743 switch (cancel_ret) {
5744 case IO_WQ_CANCEL_OK:
5745 ret = 0;
5746 break;
5747 case IO_WQ_CANCEL_RUNNING:
5748 ret = -EALREADY;
5749 break;
5750 case IO_WQ_CANCEL_NOTFOUND:
5751 ret = -ENOENT;
5752 break;
5753 }
5754
e977d6d3
JA
5755 return ret;
5756}
5757
47f46768
JA
5758static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
5759 struct io_kiocb *req, __u64 sqe_addr,
014db007 5760 int success_ret)
47f46768
JA
5761{
5762 unsigned long flags;
5763 int ret;
5764
f458dd84 5765 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
47f46768
JA
5766 if (ret != -ENOENT) {
5767 spin_lock_irqsave(&ctx->completion_lock, flags);
5768 goto done;
5769 }
5770
5771 spin_lock_irqsave(&ctx->completion_lock, flags);
5772 ret = io_timeout_cancel(ctx, sqe_addr);
5773 if (ret != -ENOENT)
5774 goto done;
5775 ret = io_poll_cancel(ctx, sqe_addr);
5776done:
b0dd8a41
JA
5777 if (!ret)
5778 ret = success_ret;
47f46768
JA
5779 io_cqring_fill_event(req, ret);
5780 io_commit_cqring(ctx);
5781 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5782 io_cqring_ev_posted(ctx);
5783
4e88d6e7
JA
5784 if (ret < 0)
5785 req_set_fail_links(req);
014db007 5786 io_put_req(req);
47f46768
JA
5787}
5788
3529d8c2
JA
5789static int io_async_cancel_prep(struct io_kiocb *req,
5790 const struct io_uring_sqe *sqe)
e977d6d3 5791{
fbf23849 5792 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
e977d6d3 5793 return -EINVAL;
61710e43
DA
5794 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5795 return -EINVAL;
5796 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
e977d6d3
JA
5797 return -EINVAL;
5798
fbf23849
JA
5799 req->cancel.addr = READ_ONCE(sqe->addr);
5800 return 0;
5801}
5802
61e98203 5803static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
fbf23849
JA
5804{
5805 struct io_ring_ctx *ctx = req->ctx;
58f99373
PB
5806 u64 sqe_addr = req->cancel.addr;
5807 struct io_tctx_node *node;
5808 int ret;
5809
5810 /* tasks should wait for their io-wq threads, so safe w/o sync */
5811 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
5812 spin_lock_irq(&ctx->completion_lock);
5813 if (ret != -ENOENT)
5814 goto done;
5815 ret = io_timeout_cancel(ctx, sqe_addr);
5816 if (ret != -ENOENT)
5817 goto done;
5818 ret = io_poll_cancel(ctx, sqe_addr);
5819 if (ret != -ENOENT)
5820 goto done;
5821 spin_unlock_irq(&ctx->completion_lock);
5822
5823 /* slow path, try all io-wq's */
5824 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5825 ret = -ENOENT;
5826 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
5827 struct io_uring_task *tctx = node->task->io_uring;
fbf23849 5828
58f99373
PB
5829 if (!tctx || !tctx->io_wq)
5830 continue;
5831 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
5832 if (ret != -ENOENT)
5833 break;
5834 }
5835 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5836
5837 spin_lock_irq(&ctx->completion_lock);
5838done:
5839 io_cqring_fill_event(req, ret);
5840 io_commit_cqring(ctx);
5841 spin_unlock_irq(&ctx->completion_lock);
5842 io_cqring_ev_posted(ctx);
5843
5844 if (ret < 0)
5845 req_set_fail_links(req);
5846 io_put_req(req);
5262f567
JA
5847 return 0;
5848}
5849
269bbe5f 5850static int io_rsrc_update_prep(struct io_kiocb *req,
05f3fb3c
JA
5851 const struct io_uring_sqe *sqe)
5852{
6ca56f84
JA
5853 if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL))
5854 return -EINVAL;
61710e43
DA
5855 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5856 return -EINVAL;
5857 if (sqe->ioprio || sqe->rw_flags)
05f3fb3c
JA
5858 return -EINVAL;
5859
269bbe5f
BM
5860 req->rsrc_update.offset = READ_ONCE(sqe->off);
5861 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
5862 if (!req->rsrc_update.nr_args)
05f3fb3c 5863 return -EINVAL;
269bbe5f 5864 req->rsrc_update.arg = READ_ONCE(sqe->addr);
05f3fb3c
JA
5865 return 0;
5866}
5867
889fca73 5868static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
fbf23849
JA
5869{
5870 struct io_ring_ctx *ctx = req->ctx;
269bbe5f 5871 struct io_uring_rsrc_update up;
05f3fb3c 5872 int ret;
fbf23849 5873
45d189c6 5874 if (issue_flags & IO_URING_F_NONBLOCK)
05f3fb3c 5875 return -EAGAIN;
05f3fb3c 5876
269bbe5f
BM
5877 up.offset = req->rsrc_update.offset;
5878 up.data = req->rsrc_update.arg;
05f3fb3c
JA
5879
5880 mutex_lock(&ctx->uring_lock);
269bbe5f 5881 ret = __io_sqe_files_update(ctx, &up, req->rsrc_update.nr_args);
05f3fb3c
JA
5882 mutex_unlock(&ctx->uring_lock);
5883
5884 if (ret < 0)
5885 req_set_fail_links(req);
889fca73 5886 __io_req_complete(req, issue_flags, ret, 0);
5262f567
JA
5887 return 0;
5888}
5889
bfe76559 5890static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
f67676d1 5891{
d625c6ee 5892 switch (req->opcode) {
e781573e 5893 case IORING_OP_NOP:
bfe76559 5894 return 0;
f67676d1
JA
5895 case IORING_OP_READV:
5896 case IORING_OP_READ_FIXED:
3a6820f2 5897 case IORING_OP_READ:
bfe76559 5898 return io_read_prep(req, sqe);
f67676d1
JA
5899 case IORING_OP_WRITEV:
5900 case IORING_OP_WRITE_FIXED:
3a6820f2 5901 case IORING_OP_WRITE:
bfe76559 5902 return io_write_prep(req, sqe);
0969e783 5903 case IORING_OP_POLL_ADD:
bfe76559 5904 return io_poll_add_prep(req, sqe);
0969e783 5905 case IORING_OP_POLL_REMOVE:
bfe76559 5906 return io_poll_remove_prep(req, sqe);
8ed8d3c3 5907 case IORING_OP_FSYNC:
1155c76a 5908 return io_fsync_prep(req, sqe);
8ed8d3c3 5909 case IORING_OP_SYNC_FILE_RANGE:
1155c76a 5910 return io_sfr_prep(req, sqe);
03b1230c 5911 case IORING_OP_SENDMSG:
fddaface 5912 case IORING_OP_SEND:
bfe76559 5913 return io_sendmsg_prep(req, sqe);
03b1230c 5914 case IORING_OP_RECVMSG:
fddaface 5915 case IORING_OP_RECV:
bfe76559 5916 return io_recvmsg_prep(req, sqe);
f499a021 5917 case IORING_OP_CONNECT:
bfe76559 5918 return io_connect_prep(req, sqe);
2d28390a 5919 case IORING_OP_TIMEOUT:
bfe76559 5920 return io_timeout_prep(req, sqe, false);
b29472ee 5921 case IORING_OP_TIMEOUT_REMOVE:
bfe76559 5922 return io_timeout_remove_prep(req, sqe);
fbf23849 5923 case IORING_OP_ASYNC_CANCEL:
bfe76559 5924 return io_async_cancel_prep(req, sqe);
2d28390a 5925 case IORING_OP_LINK_TIMEOUT:
bfe76559 5926 return io_timeout_prep(req, sqe, true);
8ed8d3c3 5927 case IORING_OP_ACCEPT:
bfe76559 5928 return io_accept_prep(req, sqe);
d63d1b5e 5929 case IORING_OP_FALLOCATE:
bfe76559 5930 return io_fallocate_prep(req, sqe);
15b71abe 5931 case IORING_OP_OPENAT:
bfe76559 5932 return io_openat_prep(req, sqe);
b5dba59e 5933 case IORING_OP_CLOSE:
bfe76559 5934 return io_close_prep(req, sqe);
05f3fb3c 5935 case IORING_OP_FILES_UPDATE:
269bbe5f 5936 return io_rsrc_update_prep(req, sqe);
eddc7ef5 5937 case IORING_OP_STATX:
bfe76559 5938 return io_statx_prep(req, sqe);
4840e418 5939 case IORING_OP_FADVISE:
bfe76559 5940 return io_fadvise_prep(req, sqe);
c1ca757b 5941 case IORING_OP_MADVISE:
bfe76559 5942 return io_madvise_prep(req, sqe);
cebdb986 5943 case IORING_OP_OPENAT2:
bfe76559 5944 return io_openat2_prep(req, sqe);
3e4827b0 5945 case IORING_OP_EPOLL_CTL:
bfe76559 5946 return io_epoll_ctl_prep(req, sqe);
7d67af2c 5947 case IORING_OP_SPLICE:
bfe76559 5948 return io_splice_prep(req, sqe);
ddf0322d 5949 case IORING_OP_PROVIDE_BUFFERS:
bfe76559 5950 return io_provide_buffers_prep(req, sqe);
067524e9 5951 case IORING_OP_REMOVE_BUFFERS:
bfe76559 5952 return io_remove_buffers_prep(req, sqe);
f2a8d5c7 5953 case IORING_OP_TEE:
bfe76559 5954 return io_tee_prep(req, sqe);
36f4fa68
JA
5955 case IORING_OP_SHUTDOWN:
5956 return io_shutdown_prep(req, sqe);
80a261fd
JA
5957 case IORING_OP_RENAMEAT:
5958 return io_renameat_prep(req, sqe);
14a1143b
JA
5959 case IORING_OP_UNLINKAT:
5960 return io_unlinkat_prep(req, sqe);
f67676d1
JA
5961 }
5962
bfe76559
PB
5963 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
5964 req->opcode);
5965 return-EINVAL;
5966}
5967
93642ef8 5968static int io_req_prep_async(struct io_kiocb *req)
bfe76559 5969{
b7e298d2
PB
5970 if (!io_op_defs[req->opcode].needs_async_setup)
5971 return 0;
5972 if (WARN_ON_ONCE(req->async_data))
5973 return -EFAULT;
5974 if (io_alloc_async_data(req))
5975 return -EAGAIN;
5976
93642ef8
PB
5977 switch (req->opcode) {
5978 case IORING_OP_READV:
93642ef8
PB
5979 return io_rw_prep_async(req, READ);
5980 case IORING_OP_WRITEV:
93642ef8
PB
5981 return io_rw_prep_async(req, WRITE);
5982 case IORING_OP_SENDMSG:
93642ef8
PB
5983 return io_sendmsg_prep_async(req);
5984 case IORING_OP_RECVMSG:
93642ef8
PB
5985 return io_recvmsg_prep_async(req);
5986 case IORING_OP_CONNECT:
5987 return io_connect_prep_async(req);
5988 }
b7e298d2
PB
5989 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
5990 req->opcode);
5991 return -EFAULT;
f67676d1
JA
5992}
5993
9cf7c104
PB
5994static u32 io_get_sequence(struct io_kiocb *req)
5995{
5996 struct io_kiocb *pos;
5997 struct io_ring_ctx *ctx = req->ctx;
f2f87370 5998 u32 total_submitted, nr_reqs = 0;
9cf7c104 5999
f2f87370
PB
6000 io_for_each_link(pos, req)
6001 nr_reqs++;
9cf7c104
PB
6002
6003 total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
6004 return total_submitted - nr_reqs;
6005}
6006
be7053b7 6007static int io_req_defer(struct io_kiocb *req)
de0617e4 6008{
a197f664 6009 struct io_ring_ctx *ctx = req->ctx;
27dc8338 6010 struct io_defer_entry *de;
f67676d1 6011 int ret;
9cf7c104 6012 u32 seq;
de0617e4 6013
9d858b21 6014 /* Still need defer if there is pending req in defer list. */
9cf7c104
PB
6015 if (likely(list_empty_careful(&ctx->defer_list) &&
6016 !(req->flags & REQ_F_IO_DRAIN)))
6017 return 0;
6018
6019 seq = io_get_sequence(req);
6020 /* Still a chance to pass the sequence check */
6021 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
de0617e4
JA
6022 return 0;
6023
b7e298d2 6024 ret = io_req_prep_async(req);
be7053b7
PB
6025 if (ret)
6026 return ret;
cbdcb435 6027 io_prep_async_link(req);
27dc8338
PB
6028 de = kmalloc(sizeof(*de), GFP_KERNEL);
6029 if (!de)
6030 return -ENOMEM;
2d28390a 6031
de0617e4 6032 spin_lock_irq(&ctx->completion_lock);
9cf7c104 6033 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
de0617e4 6034 spin_unlock_irq(&ctx->completion_lock);
27dc8338 6035 kfree(de);
ae34817b
PB
6036 io_queue_async_work(req);
6037 return -EIOCBQUEUED;
de0617e4
JA
6038 }
6039
915967f6 6040 trace_io_uring_defer(ctx, req, req->user_data);
27dc8338 6041 de->req = req;
9cf7c104 6042 de->seq = seq;
27dc8338 6043 list_add_tail(&de->list, &ctx->defer_list);
de0617e4
JA
6044 spin_unlock_irq(&ctx->completion_lock);
6045 return -EIOCBQUEUED;
6046}
6047
68fb8979 6048static void io_clean_op(struct io_kiocb *req)
99bc4c38 6049{
0e1b6fe3
PB
6050 if (req->flags & REQ_F_BUFFER_SELECTED) {
6051 switch (req->opcode) {
6052 case IORING_OP_READV:
6053 case IORING_OP_READ_FIXED:
6054 case IORING_OP_READ:
bcda7baa 6055 kfree((void *)(unsigned long)req->rw.addr);
0e1b6fe3
PB
6056 break;
6057 case IORING_OP_RECVMSG:
6058 case IORING_OP_RECV:
bcda7baa 6059 kfree(req->sr_msg.kbuf);
0e1b6fe3
PB
6060 break;
6061 }
6062 req->flags &= ~REQ_F_BUFFER_SELECTED;
99bc4c38
PB
6063 }
6064
0e1b6fe3
PB
6065 if (req->flags & REQ_F_NEED_CLEANUP) {
6066 switch (req->opcode) {
6067 case IORING_OP_READV:
6068 case IORING_OP_READ_FIXED:
6069 case IORING_OP_READ:
6070 case IORING_OP_WRITEV:
6071 case IORING_OP_WRITE_FIXED:
e8c2bc1f
JA
6072 case IORING_OP_WRITE: {
6073 struct io_async_rw *io = req->async_data;
6074 if (io->free_iovec)
6075 kfree(io->free_iovec);
0e1b6fe3 6076 break;
e8c2bc1f 6077 }
0e1b6fe3 6078 case IORING_OP_RECVMSG:
e8c2bc1f
JA
6079 case IORING_OP_SENDMSG: {
6080 struct io_async_msghdr *io = req->async_data;
257e84a5
PB
6081
6082 kfree(io->free_iov);
0e1b6fe3 6083 break;
e8c2bc1f 6084 }
0e1b6fe3
PB
6085 case IORING_OP_SPLICE:
6086 case IORING_OP_TEE:
e1d767f0
PB
6087 if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
6088 io_put_file(req->splice.file_in);
0e1b6fe3 6089 break;
f3cd4850
JA
6090 case IORING_OP_OPENAT:
6091 case IORING_OP_OPENAT2:
6092 if (req->open.filename)
6093 putname(req->open.filename);
6094 break;
80a261fd
JA
6095 case IORING_OP_RENAMEAT:
6096 putname(req->rename.oldpath);
6097 putname(req->rename.newpath);
6098 break;
14a1143b
JA
6099 case IORING_OP_UNLINKAT:
6100 putname(req->unlink.filename);
6101 break;
0e1b6fe3
PB
6102 }
6103 req->flags &= ~REQ_F_NEED_CLEANUP;
99bc4c38 6104 }
99bc4c38
PB
6105}
6106
889fca73 6107static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
2b188cc1 6108{
a197f664 6109 struct io_ring_ctx *ctx = req->ctx;
5730b27e 6110 const struct cred *creds = NULL;
d625c6ee 6111 int ret;
2b188cc1 6112
003e8dcc
JA
6113 if (req->work.creds && req->work.creds != current_cred())
6114 creds = override_creds(req->work.creds);
5730b27e 6115
d625c6ee 6116 switch (req->opcode) {
2b188cc1 6117 case IORING_OP_NOP:
889fca73 6118 ret = io_nop(req, issue_flags);
2b188cc1
JA
6119 break;
6120 case IORING_OP_READV:
edafccee 6121 case IORING_OP_READ_FIXED:
3a6820f2 6122 case IORING_OP_READ:
889fca73 6123 ret = io_read(req, issue_flags);
edafccee 6124 break;
3529d8c2 6125 case IORING_OP_WRITEV:
edafccee 6126 case IORING_OP_WRITE_FIXED:
3a6820f2 6127 case IORING_OP_WRITE:
889fca73 6128 ret = io_write(req, issue_flags);
2b188cc1 6129 break;
c992fe29 6130 case IORING_OP_FSYNC:
45d189c6 6131 ret = io_fsync(req, issue_flags);
c992fe29 6132 break;
221c5eb2 6133 case IORING_OP_POLL_ADD:
61e98203 6134 ret = io_poll_add(req, issue_flags);
221c5eb2
JA
6135 break;
6136 case IORING_OP_POLL_REMOVE:
61e98203 6137 ret = io_poll_remove(req, issue_flags);
221c5eb2 6138 break;
5d17b4a4 6139 case IORING_OP_SYNC_FILE_RANGE:
45d189c6 6140 ret = io_sync_file_range(req, issue_flags);
5d17b4a4 6141 break;
0fa03c62 6142 case IORING_OP_SENDMSG:
889fca73 6143 ret = io_sendmsg(req, issue_flags);
062d04d7 6144 break;
fddaface 6145 case IORING_OP_SEND:
889fca73 6146 ret = io_send(req, issue_flags);
0fa03c62 6147 break;
aa1fa28f 6148 case IORING_OP_RECVMSG:
889fca73 6149 ret = io_recvmsg(req, issue_flags);
062d04d7 6150 break;
fddaface 6151 case IORING_OP_RECV:
889fca73 6152 ret = io_recv(req, issue_flags);
aa1fa28f 6153 break;
5262f567 6154 case IORING_OP_TIMEOUT:
61e98203 6155 ret = io_timeout(req, issue_flags);
5262f567 6156 break;
11365043 6157 case IORING_OP_TIMEOUT_REMOVE:
61e98203 6158 ret = io_timeout_remove(req, issue_flags);
11365043 6159 break;
17f2fe35 6160 case IORING_OP_ACCEPT:
889fca73 6161 ret = io_accept(req, issue_flags);
17f2fe35 6162 break;
f8e85cf2 6163 case IORING_OP_CONNECT:
889fca73 6164 ret = io_connect(req, issue_flags);
f8e85cf2 6165 break;
62755e35 6166 case IORING_OP_ASYNC_CANCEL:
61e98203 6167 ret = io_async_cancel(req, issue_flags);
62755e35 6168 break;
d63d1b5e 6169 case IORING_OP_FALLOCATE:
45d189c6 6170 ret = io_fallocate(req, issue_flags);
d63d1b5e 6171 break;
15b71abe 6172 case IORING_OP_OPENAT:
45d189c6 6173 ret = io_openat(req, issue_flags);
15b71abe 6174 break;
b5dba59e 6175 case IORING_OP_CLOSE:
889fca73 6176 ret = io_close(req, issue_flags);
b5dba59e 6177 break;
05f3fb3c 6178 case IORING_OP_FILES_UPDATE:
889fca73 6179 ret = io_files_update(req, issue_flags);
05f3fb3c 6180 break;
eddc7ef5 6181 case IORING_OP_STATX:
45d189c6 6182 ret = io_statx(req, issue_flags);
eddc7ef5 6183 break;
4840e418 6184 case IORING_OP_FADVISE:
45d189c6 6185 ret = io_fadvise(req, issue_flags);
4840e418 6186 break;
c1ca757b 6187 case IORING_OP_MADVISE:
45d189c6 6188 ret = io_madvise(req, issue_flags);
c1ca757b 6189 break;
cebdb986 6190 case IORING_OP_OPENAT2:
45d189c6 6191 ret = io_openat2(req, issue_flags);
cebdb986 6192 break;
3e4827b0 6193 case IORING_OP_EPOLL_CTL:
889fca73 6194 ret = io_epoll_ctl(req, issue_flags);
3e4827b0 6195 break;
7d67af2c 6196 case IORING_OP_SPLICE:
45d189c6 6197 ret = io_splice(req, issue_flags);
7d67af2c 6198 break;
ddf0322d 6199 case IORING_OP_PROVIDE_BUFFERS:
889fca73 6200 ret = io_provide_buffers(req, issue_flags);
ddf0322d 6201 break;
067524e9 6202 case IORING_OP_REMOVE_BUFFERS:
889fca73 6203 ret = io_remove_buffers(req, issue_flags);
3e4827b0 6204 break;
f2a8d5c7 6205 case IORING_OP_TEE:
45d189c6 6206 ret = io_tee(req, issue_flags);
f2a8d5c7 6207 break;
36f4fa68 6208 case IORING_OP_SHUTDOWN:
45d189c6 6209 ret = io_shutdown(req, issue_flags);
36f4fa68 6210 break;
80a261fd 6211 case IORING_OP_RENAMEAT:
45d189c6 6212 ret = io_renameat(req, issue_flags);
80a261fd 6213 break;
14a1143b 6214 case IORING_OP_UNLINKAT:
45d189c6 6215 ret = io_unlinkat(req, issue_flags);
14a1143b 6216 break;
2b188cc1
JA
6217 default:
6218 ret = -EINVAL;
6219 break;
6220 }
6221
5730b27e
JA
6222 if (creds)
6223 revert_creds(creds);
6224
def596e9
JA
6225 if (ret)
6226 return ret;
6227
b532576e
JA
6228 /* If the op doesn't have a file, we're not polling for it */
6229 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
11ba820b
JA
6230 const bool in_async = io_wq_current_is_worker();
6231
11ba820b
JA
6232 /* workqueue context doesn't hold uring_lock, grab it now */
6233 if (in_async)
6234 mutex_lock(&ctx->uring_lock);
6235
2e9dbe90 6236 io_iopoll_req_issued(req, in_async);
11ba820b
JA
6237
6238 if (in_async)
6239 mutex_unlock(&ctx->uring_lock);
def596e9
JA
6240 }
6241
6242 return 0;
2b188cc1
JA
6243}
6244
5280f7e5 6245static void io_wq_submit_work(struct io_wq_work *work)
2b188cc1
JA
6246{
6247 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6df1db6b 6248 struct io_kiocb *timeout;
561fb04a 6249 int ret = 0;
2b188cc1 6250
6df1db6b
PB
6251 timeout = io_prep_linked_timeout(req);
6252 if (timeout)
6253 io_queue_linked_timeout(timeout);
d4c81f38 6254
4014d943 6255 if (work->flags & IO_WQ_WORK_CANCEL)
561fb04a 6256 ret = -ECANCELED;
31b51510 6257
561fb04a 6258 if (!ret) {
561fb04a 6259 do {
889fca73 6260 ret = io_issue_sqe(req, 0);
561fb04a
JA
6261 /*
6262 * We can get EAGAIN for polled IO even though we're
6263 * forcing a sync submission from here, since we can't
6264 * wait for request slots on the block side.
6265 */
6266 if (ret != -EAGAIN)
6267 break;
6268 cond_resched();
6269 } while (1);
6270 }
31b51510 6271
a3df7698 6272 /* avoid locking problems by failing it from a clean context */
561fb04a 6273 if (ret) {
a3df7698 6274 /* io-wq is going to take one down */
de9b4cca 6275 req_ref_get(req);
a3df7698 6276 io_req_task_queue_fail(req, ret);
edafccee 6277 }
2b188cc1
JA
6278}
6279
7b29f92d
JA
6280#define FFS_ASYNC_READ 0x1UL
6281#define FFS_ASYNC_WRITE 0x2UL
6282#ifdef CONFIG_64BIT
6283#define FFS_ISREG 0x4UL
6284#else
6285#define FFS_ISREG 0x0UL
6286#endif
6287#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
6288
b895c9a6 6289static inline struct file **io_fixed_file_slot(struct io_rsrc_data *file_data,
dafecf19 6290 unsigned i)
65e19f54 6291{
269bbe5f 6292 struct fixed_rsrc_table *table;
65e19f54 6293
dafecf19
PB
6294 table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT];
6295 return &table->files[i & IORING_FILE_TABLE_MASK];
6296}
6297
6298static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6299 int index)
6300{
7b29f92d
JA
6301 struct file **file_slot = io_fixed_file_slot(ctx->file_data, index);
6302
6303 return (struct file *) ((unsigned long) *file_slot & FFS_MASK);
65e19f54
JA
6304}
6305
8371adf5
PB
6306static struct file *io_file_get(struct io_submit_state *state,
6307 struct io_kiocb *req, int fd, bool fixed)
09bb8394 6308{
a197f664 6309 struct io_ring_ctx *ctx = req->ctx;
8da11c19 6310 struct file *file;
09bb8394 6311
8da11c19 6312 if (fixed) {
7b29f92d
JA
6313 unsigned long file_ptr;
6314
479f517b 6315 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
8371adf5 6316 return NULL;
b7620121 6317 fd = array_index_nospec(fd, ctx->nr_user_files);
7b29f92d
JA
6318 file_ptr = (unsigned long) *io_fixed_file_slot(ctx->file_data, fd);
6319 file = (struct file *) (file_ptr & FFS_MASK);
6320 file_ptr &= ~FFS_MASK;
6321 /* mask in overlapping REQ_F and FFS bits */
6322 req->flags |= (file_ptr << REQ_F_ASYNC_READ_BIT);
b895c9a6 6323 io_req_set_rsrc_node(req);
09bb8394 6324 } else {
c826bd7a 6325 trace_io_uring_file_get(ctx, fd);
8da11c19 6326 file = __io_file_get(state, fd);
d44f554e
JA
6327
6328 /* we don't allow fixed io_uring files */
6329 if (file && unlikely(file->f_op == &io_uring_fops))
6330 io_req_track_inflight(req);
09bb8394
JA
6331 }
6332
8371adf5 6333 return file;
09bb8394
JA
6334}
6335
2665abfd 6336static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2b188cc1 6337{
ad8a48ac
JA
6338 struct io_timeout_data *data = container_of(timer,
6339 struct io_timeout_data, timer);
90cd7e42 6340 struct io_kiocb *prev, *req = data->req;
2665abfd 6341 struct io_ring_ctx *ctx = req->ctx;
2665abfd 6342 unsigned long flags;
2665abfd
JA
6343
6344 spin_lock_irqsave(&ctx->completion_lock, flags);
90cd7e42
PB
6345 prev = req->timeout.head;
6346 req->timeout.head = NULL;
2665abfd
JA
6347
6348 /*
6349 * We don't expect the list to be empty, that will only happen if we
6350 * race with the completion of the linked work.
6351 */
de9b4cca 6352 if (prev && req_ref_inc_not_zero(prev))
f2f87370 6353 io_remove_next_linked(prev);
90cd7e42
PB
6354 else
6355 prev = NULL;
2665abfd
JA
6356 spin_unlock_irqrestore(&ctx->completion_lock, flags);
6357
6358 if (prev) {
014db007 6359 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
9ae1f8dd 6360 io_put_req_deferred(prev, 1);
47f46768 6361 } else {
9ae1f8dd
PB
6362 io_req_complete_post(req, -ETIME, 0);
6363 io_put_req_deferred(req, 1);
2665abfd 6364 }
2665abfd
JA
6365 return HRTIMER_NORESTART;
6366}
6367
de968c18 6368static void io_queue_linked_timeout(struct io_kiocb *req)
2665abfd 6369{
de968c18
PB
6370 struct io_ring_ctx *ctx = req->ctx;
6371
6372 spin_lock_irq(&ctx->completion_lock);
76a46e06 6373 /*
f2f87370
PB
6374 * If the back reference is NULL, then our linked request finished
6375 * before we got a chance to setup the timer
76a46e06 6376 */
90cd7e42 6377 if (req->timeout.head) {
e8c2bc1f 6378 struct io_timeout_data *data = req->async_data;
94ae5e77 6379
ad8a48ac
JA
6380 data->timer.function = io_link_timeout_fn;
6381 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6382 data->mode);
2665abfd 6383 }
76a46e06 6384 spin_unlock_irq(&ctx->completion_lock);
2665abfd 6385 /* drop submission reference */
76a46e06
JA
6386 io_put_req(req);
6387}
2665abfd 6388
ad8a48ac 6389static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
2665abfd 6390{
f2f87370 6391 struct io_kiocb *nxt = req->link;
2665abfd 6392
f2f87370
PB
6393 if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) ||
6394 nxt->opcode != IORING_OP_LINK_TIMEOUT)
76a46e06 6395 return NULL;
2665abfd 6396
90cd7e42 6397 nxt->timeout.head = req;
900fad45 6398 nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
76a46e06 6399 req->flags |= REQ_F_LINK_TIMEOUT;
76a46e06 6400 return nxt;
2665abfd
JA
6401}
6402
c5eef2b9 6403static void __io_queue_sqe(struct io_kiocb *req)
2b188cc1 6404{
d3d7298d 6405 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
e0c5c576 6406 int ret;
2b188cc1 6407
c5eef2b9 6408 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
193155c8 6409
491381ce
JA
6410 /*
6411 * We async punt it if the file wasn't marked NOWAIT, or if the file
6412 * doesn't support non-blocking read/write attempts
6413 */
1840038e 6414 if (likely(!ret)) {
0d63c148 6415 /* drop submission reference */
e342c807 6416 if (req->flags & REQ_F_COMPLETE_INLINE) {
c5eef2b9
PB
6417 struct io_ring_ctx *ctx = req->ctx;
6418 struct io_comp_state *cs = &ctx->submit_state.comp;
e65ef56d 6419
6dd0be1e 6420 cs->reqs[cs->nr++] = req;
d3d7298d 6421 if (cs->nr == ARRAY_SIZE(cs->reqs))
c5eef2b9 6422 io_submit_flush_completions(cs, ctx);
9affd664 6423 } else {
d3d7298d 6424 io_put_req(req);
0d63c148 6425 }
1840038e
PB
6426 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6427 if (!io_arm_poll_handler(req)) {
6428 /*
6429 * Queued up for async execution, worker will release
6430 * submit reference when the iocb is actually submitted.
6431 */
6432 io_queue_async_work(req);
6433 }
0d63c148 6434 } else {
f41db273 6435 io_req_complete_failed(req, ret);
9e645e11 6436 }
d3d7298d
PB
6437 if (linked_timeout)
6438 io_queue_linked_timeout(linked_timeout);
2b188cc1
JA
6439}
6440
be7053b7 6441static void io_queue_sqe(struct io_kiocb *req)
4fe2c963
JL
6442{
6443 int ret;
6444
be7053b7 6445 ret = io_req_defer(req);
4fe2c963
JL
6446 if (ret) {
6447 if (ret != -EIOCBQUEUED) {
1118591a 6448fail_req:
f41db273 6449 io_req_complete_failed(req, ret);
4fe2c963 6450 }
2550878f 6451 } else if (req->flags & REQ_F_FORCE_ASYNC) {
b7e298d2 6452 ret = io_req_prep_async(req);
be7053b7
PB
6453 if (unlikely(ret))
6454 goto fail_req;
ce35a47a
JA
6455 io_queue_async_work(req);
6456 } else {
c5eef2b9 6457 __io_queue_sqe(req);
ce35a47a 6458 }
4fe2c963
JL
6459}
6460
b16fed66
PB
6461/*
6462 * Check SQE restrictions (opcode and flags).
6463 *
6464 * Returns 'true' if SQE is allowed, 'false' otherwise.
6465 */
6466static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6467 struct io_kiocb *req,
6468 unsigned int sqe_flags)
4fe2c963 6469{
b16fed66
PB
6470 if (!ctx->restricted)
6471 return true;
6472
6473 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6474 return false;
6475
6476 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6477 ctx->restrictions.sqe_flags_required)
6478 return false;
6479
6480 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6481 ctx->restrictions.sqe_flags_required))
6482 return false;
6483
6484 return true;
4fe2c963
JL
6485}
6486
b16fed66
PB
6487static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
6488 const struct io_uring_sqe *sqe)
6489{
6490 struct io_submit_state *state;
6491 unsigned int sqe_flags;
003e8dcc 6492 int personality, ret = 0;
b16fed66
PB
6493
6494 req->opcode = READ_ONCE(sqe->opcode);
6495 /* same numerical values with corresponding REQ_F_*, safe to copy */
6496 req->flags = sqe_flags = READ_ONCE(sqe->flags);
6497 req->user_data = READ_ONCE(sqe->user_data);
6498 req->async_data = NULL;
6499 req->file = NULL;
6500 req->ctx = ctx;
6501 req->link = NULL;
6502 req->fixed_rsrc_refs = NULL;
6503 /* one is dropped after submission, the other at completion */
abc54d63 6504 atomic_set(&req->refs, 2);
b16fed66
PB
6505 req->task = current;
6506 req->result = 0;
93e68e03 6507 req->work.creds = NULL;
b16fed66
PB
6508
6509 /* enforce forwards compatibility on users */
ebf4a5db
PB
6510 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
6511 req->flags = 0;
b16fed66 6512 return -EINVAL;
ebf4a5db 6513 }
b16fed66
PB
6514
6515 if (unlikely(req->opcode >= IORING_OP_LAST))
6516 return -EINVAL;
6517
b16fed66
PB
6518 if (unlikely(!io_check_restriction(ctx, req, sqe_flags)))
6519 return -EACCES;
6520
6521 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6522 !io_op_defs[req->opcode].buffer_select)
6523 return -EOPNOTSUPP;
863e0560 6524
003e8dcc
JA
6525 personality = READ_ONCE(sqe->personality);
6526 if (personality) {
61cf9370 6527 req->work.creds = xa_load(&ctx->personalities, personality);
003e8dcc
JA
6528 if (!req->work.creds)
6529 return -EINVAL;
6530 get_cred(req->work.creds);
003e8dcc 6531 }
b16fed66
PB
6532 state = &ctx->submit_state;
6533
6534 /*
6535 * Plug now if we have more than 1 IO left after this, and the target
6536 * is potentially a read/write to block based storage.
6537 */
6538 if (!state->plug_started && state->ios_left > 1 &&
6539 io_op_defs[req->opcode].plug) {
6540 blk_start_plug(&state->plug);
6541 state->plug_started = true;
6542 }
6543
6544 if (io_op_defs[req->opcode].needs_file) {
6545 bool fixed = req->flags & REQ_F_FIXED_FILE;
6546
6547 req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
6548 if (unlikely(!req->file))
6549 ret = -EBADF;
6550 }
6551
6552 state->ios_left--;
6553 return ret;
6554}
6555
a6b8cadc 6556static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
a1ab7b35 6557 const struct io_uring_sqe *sqe)
9e645e11 6558{
a1ab7b35 6559 struct io_submit_link *link = &ctx->submit_state.link;
ef4ff581 6560 int ret;
9e645e11 6561
a6b8cadc
PB
6562 ret = io_init_req(ctx, req, sqe);
6563 if (unlikely(ret)) {
6564fail_req:
de59bc10
PB
6565 if (link->head) {
6566 /* fail even hard links since we don't submit */
cf109604 6567 link->head->flags |= REQ_F_FAIL_LINK;
f41db273 6568 io_req_complete_failed(link->head, -ECANCELED);
de59bc10
PB
6569 link->head = NULL;
6570 }
f41db273 6571 io_req_complete_failed(req, ret);
a6b8cadc
PB
6572 return ret;
6573 }
be7053b7
PB
6574 ret = io_req_prep(req, sqe);
6575 if (unlikely(ret))
6576 goto fail_req;
a6b8cadc 6577
be7053b7 6578 /* don't need @sqe from now on */
a6b8cadc
PB
6579 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6580 true, ctx->flags & IORING_SETUP_SQPOLL);
6581
9e645e11
JA
6582 /*
6583 * If we already have a head request, queue this one for async
6584 * submittal once the head completes. If we don't have a head but
6585 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6586 * submitted sync once the chain is complete. If none of those
6587 * conditions are true (normal request), then just queue it.
6588 */
863e0560
PB
6589 if (link->head) {
6590 struct io_kiocb *head = link->head;
4e88d6e7 6591
8cdf2193
PB
6592 /*
6593 * Taking sequential execution of a link, draining both sides
6594 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
6595 * requests in the link. So, it drains the head and the
6596 * next after the link request. The last one is done via
6597 * drain_next flag to persist the effect across calls.
6598 */
ef4ff581 6599 if (req->flags & REQ_F_IO_DRAIN) {
711be031
PB
6600 head->flags |= REQ_F_IO_DRAIN;
6601 ctx->drain_next = 1;
6602 }
b7e298d2 6603 ret = io_req_prep_async(req);
cf109604 6604 if (unlikely(ret))
a6b8cadc 6605 goto fail_req;
9d76377f 6606 trace_io_uring_link(ctx, req, head);
f2f87370 6607 link->last->link = req;
863e0560 6608 link->last = req;
32fe525b
PB
6609
6610 /* last request of a link, enqueue the link */
ef4ff581 6611 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
de59bc10 6612 io_queue_sqe(head);
863e0560 6613 link->head = NULL;
32fe525b 6614 }
9e645e11 6615 } else {
711be031
PB
6616 if (unlikely(ctx->drain_next)) {
6617 req->flags |= REQ_F_IO_DRAIN;
ef4ff581 6618 ctx->drain_next = 0;
711be031 6619 }
ef4ff581 6620 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
863e0560
PB
6621 link->head = req;
6622 link->last = req;
711be031 6623 } else {
be7053b7 6624 io_queue_sqe(req);
711be031 6625 }
9e645e11 6626 }
2e6e1fde 6627
1d4240cc 6628 return 0;
9e645e11
JA
6629}
6630
9a56a232
JA
6631/*
6632 * Batched submission is done, ensure local IO is flushed out.
6633 */
ba88ff11
PB
6634static void io_submit_state_end(struct io_submit_state *state,
6635 struct io_ring_ctx *ctx)
9a56a232 6636{
a1ab7b35 6637 if (state->link.head)
de59bc10 6638 io_queue_sqe(state->link.head);
6dd0be1e 6639 if (state->comp.nr)
ba88ff11 6640 io_submit_flush_completions(&state->comp, ctx);
27926b68
JA
6641 if (state->plug_started)
6642 blk_finish_plug(&state->plug);
9f13c35b 6643 io_state_file_put(state);
9a56a232
JA
6644}
6645
6646/*
6647 * Start submission side cache.
6648 */
6649static void io_submit_state_start(struct io_submit_state *state,
ba88ff11 6650 unsigned int max_ios)
9a56a232 6651{
27926b68 6652 state->plug_started = false;
9a56a232 6653 state->ios_left = max_ios;
a1ab7b35
PB
6654 /* set only head, no need to init link_last in advance */
6655 state->link.head = NULL;
9a56a232
JA
6656}
6657
2b188cc1
JA
6658static void io_commit_sqring(struct io_ring_ctx *ctx)
6659{
75b28aff 6660 struct io_rings *rings = ctx->rings;
2b188cc1 6661
caf582c6
PB
6662 /*
6663 * Ensure any loads from the SQEs are done at this point,
6664 * since once we write the new head, the application could
6665 * write new data to them.
6666 */
6667 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2b188cc1
JA
6668}
6669
2b188cc1 6670/*
3529d8c2 6671 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
2b188cc1
JA
6672 * that is mapped by userspace. This means that care needs to be taken to
6673 * ensure that reads are stable, as we cannot rely on userspace always
6674 * being a good citizen. If members of the sqe are validated and then later
6675 * used, it's important that those reads are done through READ_ONCE() to
6676 * prevent a re-load down the line.
6677 */
709b302f 6678static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
2b188cc1 6679{
75b28aff 6680 u32 *sq_array = ctx->sq_array;
2b188cc1
JA
6681 unsigned head;
6682
6683 /*
6684 * The cached sq head (or cq tail) serves two purposes:
6685 *
6686 * 1) allows us to batch the cost of updating the user visible
6687 * head updates.
6688 * 2) allows the kernel side to track the head on its own, even
6689 * though the application is the one updating it.
6690 */
4fccfcbb 6691 head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]);
709b302f
PB
6692 if (likely(head < ctx->sq_entries))
6693 return &ctx->sq_sqes[head];
2b188cc1
JA
6694
6695 /* drop invalid entries */
498ccd9e 6696 ctx->cached_sq_dropped++;
ee7d46d9 6697 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
709b302f
PB
6698 return NULL;
6699}
6700
0f212204 6701static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
6c271ce2 6702{
46c4e16a 6703 int submitted = 0;
6c271ce2 6704
c4a2ed72 6705 /* if we have a backlog and couldn't flush it all, return BUSY */
ad3eb2c8 6706 if (test_bit(0, &ctx->sq_check_overflow)) {
6c2450ae 6707 if (!__io_cqring_overflow_flush(ctx, false))
ad3eb2c8
JA
6708 return -EBUSY;
6709 }
6c271ce2 6710
ee7d46d9
PB
6711 /* make sure SQ entry isn't read before tail */
6712 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
9ef4f124 6713
2b85edfc
PB
6714 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6715 return -EAGAIN;
6c271ce2 6716
d8a6df10 6717 percpu_counter_add(&current->io_uring->inflight, nr);
faf7b51c 6718 refcount_add(nr, &current->usage);
ba88ff11 6719 io_submit_state_start(&ctx->submit_state, nr);
b14cca0c 6720
46c4e16a 6721 while (submitted < nr) {
3529d8c2 6722 const struct io_uring_sqe *sqe;
196be95c 6723 struct io_kiocb *req;
fb5ccc98 6724
258b29a9 6725 req = io_alloc_req(ctx);
196be95c
PB
6726 if (unlikely(!req)) {
6727 if (!submitted)
6728 submitted = -EAGAIN;
fb5ccc98 6729 break;
196be95c 6730 }
4fccfcbb
PB
6731 sqe = io_get_sqe(ctx);
6732 if (unlikely(!sqe)) {
6733 kmem_cache_free(req_cachep, req);
6734 break;
6735 }
d3656344
JA
6736 /* will complete beyond this point, count as submitted */
6737 submitted++;
a1ab7b35 6738 if (io_submit_sqe(ctx, req, sqe))
196be95c 6739 break;
6c271ce2
JA
6740 }
6741
9466f437
PB
6742 if (unlikely(submitted != nr)) {
6743 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
d8a6df10
JA
6744 struct io_uring_task *tctx = current->io_uring;
6745 int unused = nr - ref_used;
9466f437 6746
d8a6df10
JA
6747 percpu_ref_put_many(&ctx->refs, unused);
6748 percpu_counter_sub(&tctx->inflight, unused);
6749 put_task_struct_many(current, unused);
9466f437 6750 }
6c271ce2 6751
a1ab7b35 6752 io_submit_state_end(&ctx->submit_state, ctx);
ae9428ca
PB
6753 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6754 io_commit_sqring(ctx);
6755
6c271ce2
JA
6756 return submitted;
6757}
6758
23b3628e
XW
6759static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6760{
6761 /* Tell userspace we may need a wakeup call */
6762 spin_lock_irq(&ctx->completion_lock);
6763 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
6764 spin_unlock_irq(&ctx->completion_lock);
6765}
6766
6767static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6768{
6769 spin_lock_irq(&ctx->completion_lock);
6770 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6771 spin_unlock_irq(&ctx->completion_lock);
6772}
6773
08369246 6774static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
6c271ce2 6775{
c8d1ba58 6776 unsigned int to_submit;
bdcd3eab 6777 int ret = 0;
6c271ce2 6778
c8d1ba58 6779 to_submit = io_sqring_entries(ctx);
e95eee2d
JA
6780 /* if we're handling multiple rings, cap submit size for fairness */
6781 if (cap_entries && to_submit > 8)
6782 to_submit = 8;
6783
906a3c6f 6784 if (!list_empty(&ctx->iopoll_list) || to_submit) {
c8d1ba58 6785 unsigned nr_events = 0;
a4c0b3de 6786
c8d1ba58 6787 mutex_lock(&ctx->uring_lock);
906a3c6f 6788 if (!list_empty(&ctx->iopoll_list))
c8d1ba58 6789 io_do_iopoll(ctx, &nr_events, 0);
906a3c6f 6790
0298ef96
PB
6791 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
6792 !(ctx->flags & IORING_SETUP_R_DISABLED))
08369246 6793 ret = io_submit_sqes(ctx, to_submit);
c8d1ba58
JA
6794 mutex_unlock(&ctx->uring_lock);
6795 }
6c271ce2 6796
90554200
JA
6797 if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait))
6798 wake_up(&ctx->sqo_sq_wait);
6c271ce2 6799
08369246
XW
6800 return ret;
6801}
6c271ce2 6802
08369246
XW
6803static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
6804{
6805 struct io_ring_ctx *ctx;
6806 unsigned sq_thread_idle = 0;
6c271ce2 6807
c9dca27d
PB
6808 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6809 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
08369246 6810 sqd->sq_thread_idle = sq_thread_idle;
c8d1ba58 6811}
6c271ce2 6812
c8d1ba58
JA
6813static int io_sq_thread(void *data)
6814{
69fb2131
JA
6815 struct io_sq_data *sqd = data;
6816 struct io_ring_ctx *ctx;
a0d9205f 6817 unsigned long timeout = 0;
37d1e2e3 6818 char buf[TASK_COMM_LEN];
08369246 6819 DEFINE_WAIT(wait);
6c271ce2 6820
696ee88a 6821 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
37d1e2e3 6822 set_task_comm(current, buf);
37d1e2e3
JA
6823 current->pf_io_worker = NULL;
6824
6825 if (sqd->sq_cpu != -1)
6826 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
6827 else
6828 set_cpus_allowed_ptr(current, cpu_online_mask);
6829 current->flags |= PF_NO_SETAFFINITY;
6830
09a6f4ef 6831 mutex_lock(&sqd->lock);
05962f95 6832 while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) {
08369246
XW
6833 int ret;
6834 bool cap_entries, sqt_spin, needs_sched;
c1edbf5f 6835
82734c5b
JA
6836 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
6837 signal_pending(current)) {
6838 bool did_sig = false;
6839
09a6f4ef 6840 mutex_unlock(&sqd->lock);
82734c5b
JA
6841 if (signal_pending(current)) {
6842 struct ksignal ksig;
6843
6844 did_sig = get_signal(&ksig);
6845 }
05962f95 6846 cond_resched();
09a6f4ef 6847 mutex_lock(&sqd->lock);
82734c5b
JA
6848 if (did_sig)
6849 break;
521d6a73 6850 io_run_task_work();
b7f5a0bf 6851 io_run_task_work_head(&sqd->park_task_work);
08369246 6852 timeout = jiffies + sqd->sq_thread_idle;
7d41e854 6853 continue;
08369246 6854 }
08369246 6855 sqt_spin = false;
e95eee2d 6856 cap_entries = !list_is_singular(&sqd->ctx_list);
69fb2131 6857 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
7c30f36a
SM
6858 const struct cred *creds = NULL;
6859
6860 if (ctx->sq_creds != current_cred())
6861 creds = override_creds(ctx->sq_creds);
08369246 6862 ret = __io_sq_thread(ctx, cap_entries);
7c30f36a
SM
6863 if (creds)
6864 revert_creds(creds);
08369246
XW
6865 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
6866 sqt_spin = true;
69fb2131 6867 }
6c271ce2 6868
08369246 6869 if (sqt_spin || !time_after(jiffies, timeout)) {
c8d1ba58
JA
6870 io_run_task_work();
6871 cond_resched();
08369246
XW
6872 if (sqt_spin)
6873 timeout = jiffies + sqd->sq_thread_idle;
6874 continue;
6875 }
6876
08369246
XW
6877 needs_sched = true;
6878 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
6879 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
6880 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6881 !list_empty_careful(&ctx->iopoll_list)) {
6882 needs_sched = false;
6883 break;
6884 }
6885 if (io_sqring_entries(ctx)) {
6886 needs_sched = false;
6887 break;
6888 }
6889 }
6890
05962f95 6891 if (needs_sched && !test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) {
69fb2131
JA
6892 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6893 io_ring_set_wakeup_flag(ctx);
08369246 6894
09a6f4ef 6895 mutex_unlock(&sqd->lock);
69fb2131 6896 schedule();
09a6f4ef 6897 mutex_lock(&sqd->lock);
69fb2131
JA
6898 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6899 io_ring_clear_wakeup_flag(ctx);
6c271ce2 6900 }
08369246
XW
6901
6902 finish_wait(&sqd->wait, &wait);
b7f5a0bf 6903 io_run_task_work_head(&sqd->park_task_work);
08369246 6904 timeout = jiffies + sqd->sq_thread_idle;
6c271ce2 6905 }
28cea78a 6906
521d6a73
PB
6907 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6908 io_uring_cancel_sqpoll(ctx);
37d1e2e3 6909 sqd->thread = NULL;
05962f95 6910 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
5f3f26f9 6911 io_ring_set_wakeup_flag(ctx);
09a6f4ef 6912 mutex_unlock(&sqd->lock);
521d6a73
PB
6913
6914 io_run_task_work();
b7f5a0bf 6915 io_run_task_work_head(&sqd->park_task_work);
37d1e2e3
JA
6916 complete(&sqd->exited);
6917 do_exit(0);
6c271ce2
JA
6918}
6919
bda52162
JA
6920struct io_wait_queue {
6921 struct wait_queue_entry wq;
6922 struct io_ring_ctx *ctx;
6923 unsigned to_wait;
6924 unsigned nr_timeouts;
6925};
6926
6c503150 6927static inline bool io_should_wake(struct io_wait_queue *iowq)
bda52162
JA
6928{
6929 struct io_ring_ctx *ctx = iowq->ctx;
6930
6931 /*
d195a66e 6932 * Wake up if we have enough events, or if a timeout occurred since we
bda52162
JA
6933 * started waiting. For timeouts, we always want to return to userspace,
6934 * regardless of event count.
6935 */
6c503150 6936 return io_cqring_events(ctx) >= iowq->to_wait ||
bda52162
JA
6937 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
6938}
6939
6940static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
6941 int wake_flags, void *key)
6942{
6943 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
6944 wq);
6945
6c503150
PB
6946 /*
6947 * Cannot safely flush overflowed CQEs from here, ensure we wake up
6948 * the task, and the next invocation will do it.
6949 */
6950 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow))
6951 return autoremove_wake_function(curr, mode, wake_flags, key);
6952 return -1;
bda52162
JA
6953}
6954
af9c1a44
JA
6955static int io_run_task_work_sig(void)
6956{
6957 if (io_run_task_work())
6958 return 1;
6959 if (!signal_pending(current))
6960 return 0;
0b8cfa97 6961 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
792ee0f6 6962 return -ERESTARTSYS;
af9c1a44
JA
6963 return -EINTR;
6964}
6965
eeb60b9a
PB
6966/* when returns >0, the caller should retry */
6967static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
6968 struct io_wait_queue *iowq,
6969 signed long *timeout)
6970{
6971 int ret;
6972
6973 /* make sure we run task_work before checking for signals */
6974 ret = io_run_task_work_sig();
6975 if (ret || io_should_wake(iowq))
6976 return ret;
6977 /* let the caller flush overflows, retry */
6978 if (test_bit(0, &ctx->cq_check_overflow))
6979 return 1;
6980
6981 *timeout = schedule_timeout(*timeout);
6982 return !*timeout ? -ETIME : 1;
6983}
6984
2b188cc1
JA
6985/*
6986 * Wait until events become available, if we don't already have some. The
6987 * application must reap them itself, as they reside on the shared cq ring.
6988 */
6989static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
c73ebb68
HX
6990 const sigset_t __user *sig, size_t sigsz,
6991 struct __kernel_timespec __user *uts)
2b188cc1 6992{
bda52162
JA
6993 struct io_wait_queue iowq = {
6994 .wq = {
6995 .private = current,
6996 .func = io_wake_function,
6997 .entry = LIST_HEAD_INIT(iowq.wq.entry),
6998 },
6999 .ctx = ctx,
7000 .to_wait = min_events,
7001 };
75b28aff 7002 struct io_rings *rings = ctx->rings;
c1d5a224
PB
7003 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7004 int ret;
2b188cc1 7005
b41e9852 7006 do {
6c2450ae 7007 io_cqring_overflow_flush(ctx, false);
6c503150 7008 if (io_cqring_events(ctx) >= min_events)
b41e9852 7009 return 0;
4c6e277c 7010 if (!io_run_task_work())
b41e9852 7011 break;
b41e9852 7012 } while (1);
2b188cc1
JA
7013
7014 if (sig) {
9e75ad5d
AB
7015#ifdef CONFIG_COMPAT
7016 if (in_compat_syscall())
7017 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
b772434b 7018 sigsz);
9e75ad5d
AB
7019 else
7020#endif
b772434b 7021 ret = set_user_sigmask(sig, sigsz);
9e75ad5d 7022
2b188cc1
JA
7023 if (ret)
7024 return ret;
7025 }
7026
c73ebb68 7027 if (uts) {
c1d5a224
PB
7028 struct timespec64 ts;
7029
c73ebb68
HX
7030 if (get_timespec64(&ts, uts))
7031 return -EFAULT;
7032 timeout = timespec64_to_jiffies(&ts);
7033 }
7034
bda52162 7035 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
c826bd7a 7036 trace_io_uring_cqring_wait(ctx, min_events);
bda52162 7037 do {
ca0a2651 7038 /* if we can't even flush overflow, don't wait for more */
6c2450ae 7039 if (!io_cqring_overflow_flush(ctx, false)) {
ca0a2651
JA
7040 ret = -EBUSY;
7041 break;
7042 }
bda52162
JA
7043 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
7044 TASK_INTERRUPTIBLE);
eeb60b9a
PB
7045 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
7046 finish_wait(&ctx->wait, &iowq.wq);
ca0a2651 7047 cond_resched();
eeb60b9a 7048 } while (ret > 0);
bda52162 7049
b7db41c9 7050 restore_saved_sigmask_unless(ret == -EINTR);
2b188cc1 7051
75b28aff 7052 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2b188cc1
JA
7053}
7054
6b06314c
JA
7055static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7056{
7057#if defined(CONFIG_UNIX)
7058 if (ctx->ring_sock) {
7059 struct sock *sock = ctx->ring_sock->sk;
7060 struct sk_buff *skb;
7061
7062 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7063 kfree_skb(skb);
7064 }
7065#else
7066 int i;
7067
65e19f54
JA
7068 for (i = 0; i < ctx->nr_user_files; i++) {
7069 struct file *file;
7070
7071 file = io_file_from_index(ctx, i);
7072 if (file)
7073 fput(file);
7074 }
6b06314c
JA
7075#endif
7076}
7077
00835dce 7078static void io_rsrc_data_ref_zero(struct percpu_ref *ref)
05f3fb3c 7079{
b895c9a6 7080 struct io_rsrc_data *data = container_of(ref, struct io_rsrc_data, refs);
05f3fb3c 7081
05f3fb3c
JA
7082 complete(&data->done);
7083}
7084
2a63b2d9 7085static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx)
1642b445 7086{
2a63b2d9 7087 spin_lock_bh(&ctx->rsrc_ref_lock);
1642b445
PB
7088}
7089
2a63b2d9 7090static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx)
6b06314c 7091{
2a63b2d9
BM
7092 spin_unlock_bh(&ctx->rsrc_ref_lock);
7093}
65e19f54 7094
b895c9a6
PB
7095static void io_rsrc_node_set(struct io_ring_ctx *ctx,
7096 struct io_rsrc_data *rsrc_data,
82fbcfa9
PB
7097 void (*rsrc_put)(struct io_ring_ctx *ctx,
7098 struct io_rsrc_put *prsrc))
1642b445 7099{
82fbcfa9
PB
7100 struct io_rsrc_node *rsrc_node = ctx->rsrc_backup_node;
7101
7102 WARN_ON_ONCE(!rsrc_node);
7103
7104 ctx->rsrc_backup_node = NULL;
7105 rsrc_node->rsrc_data = rsrc_data;
7106 rsrc_node->rsrc_put = rsrc_put;
7107
2a63b2d9 7108 io_rsrc_ref_lock(ctx);
b895c9a6
PB
7109 rsrc_data->node = rsrc_node;
7110 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
2a63b2d9 7111 io_rsrc_ref_unlock(ctx);
269bbe5f 7112 percpu_ref_get(&rsrc_data->refs);
1642b445
PB
7113}
7114
b895c9a6 7115static void io_rsrc_node_kill(struct io_ring_ctx *ctx, struct io_rsrc_data *data)
6b06314c 7116{
b895c9a6 7117 struct io_rsrc_node *ref_node = NULL;
6b06314c 7118
2a63b2d9 7119 io_rsrc_ref_lock(ctx);
1e5d770b 7120 ref_node = data->node;
e6cb007c 7121 data->node = NULL;
2a63b2d9 7122 io_rsrc_ref_unlock(ctx);
05589553
XW
7123 if (ref_node)
7124 percpu_ref_kill(&ref_node->refs);
8bad28d8
HX
7125}
7126
b895c9a6 7127static int io_rsrc_node_prealloc(struct io_ring_ctx *ctx)
8dd03afe
PB
7128{
7129 if (ctx->rsrc_backup_node)
7130 return 0;
b895c9a6 7131 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
8dd03afe
PB
7132 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7133}
7134
b895c9a6 7135static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
8bad28d8 7136 struct io_ring_ctx *ctx,
f2303b1f
PB
7137 void (*rsrc_put)(struct io_ring_ctx *ctx,
7138 struct io_rsrc_put *prsrc))
8bad28d8
HX
7139{
7140 int ret;
05589553 7141
8bad28d8
HX
7142 if (data->quiesce)
7143 return -ENXIO;
05589553 7144
8bad28d8 7145 data->quiesce = true;
1ffc5422 7146 do {
b895c9a6 7147 ret = io_rsrc_node_prealloc(ctx);
8dd03afe 7148 if (ret)
f2303b1f 7149 break;
b895c9a6 7150 io_rsrc_node_kill(ctx, data);
8bad28d8
HX
7151 percpu_ref_kill(&data->refs);
7152 flush_delayed_work(&ctx->rsrc_put_work);
7153
1ffc5422
PB
7154 ret = wait_for_completion_interruptible(&data->done);
7155 if (!ret)
7156 break;
8bad28d8 7157
cb5e1b81 7158 percpu_ref_resurrect(&data->refs);
82fbcfa9 7159 io_rsrc_node_set(ctx, data, rsrc_put);
cb5e1b81 7160 reinit_completion(&data->done);
8dd03afe 7161
8bad28d8 7162 mutex_unlock(&ctx->uring_lock);
1ffc5422 7163 ret = io_run_task_work_sig();
8bad28d8 7164 mutex_lock(&ctx->uring_lock);
f2303b1f 7165 } while (ret >= 0);
8bad28d8 7166 data->quiesce = false;
05f3fb3c 7167
8bad28d8 7168 return ret;
d7954b2b
BM
7169}
7170
b895c9a6 7171static struct io_rsrc_data *io_rsrc_data_alloc(struct io_ring_ctx *ctx)
1ad555c6 7172{
b895c9a6 7173 struct io_rsrc_data *data;
1ad555c6
BM
7174
7175 data = kzalloc(sizeof(*data), GFP_KERNEL);
7176 if (!data)
7177 return NULL;
7178
00835dce 7179 if (percpu_ref_init(&data->refs, io_rsrc_data_ref_zero,
1ad555c6
BM
7180 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
7181 kfree(data);
7182 return NULL;
7183 }
7184 data->ctx = ctx;
7185 init_completion(&data->done);
7186 return data;
7187}
7188
b895c9a6 7189static void io_rsrc_data_free(struct io_rsrc_data *data)
1ad555c6
BM
7190{
7191 percpu_ref_exit(&data->refs);
7192 kfree(data->table);
7193 kfree(data);
7194}
7195
d7954b2b
BM
7196static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7197{
b895c9a6 7198 struct io_rsrc_data *data = ctx->file_data;
d7954b2b
BM
7199 unsigned nr_tables, i;
7200 int ret;
7201
8bad28d8
HX
7202 /*
7203 * percpu_ref_is_dying() is to stop parallel files unregister
7204 * Since we possibly drop uring lock later in this function to
7205 * run task work.
7206 */
7207 if (!data || percpu_ref_is_dying(&data->refs))
d7954b2b 7208 return -ENXIO;
f2303b1f 7209 ret = io_rsrc_ref_quiesce(data, ctx, io_ring_file_put);
d7954b2b
BM
7210 if (ret)
7211 return ret;
7212
6b06314c 7213 __io_sqe_files_unregister(ctx);
65e19f54
JA
7214 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
7215 for (i = 0; i < nr_tables; i++)
05f3fb3c 7216 kfree(data->table[i].files);
b895c9a6 7217 io_rsrc_data_free(data);
05f3fb3c 7218 ctx->file_data = NULL;
6b06314c
JA
7219 ctx->nr_user_files = 0;
7220 return 0;
7221}
7222
37d1e2e3 7223static void io_sq_thread_unpark(struct io_sq_data *sqd)
09a6f4ef 7224 __releases(&sqd->lock)
37d1e2e3 7225{
521d6a73
PB
7226 WARN_ON_ONCE(sqd->thread == current);
7227
9e138a48
PB
7228 /*
7229 * Do the dance but not conditional clear_bit() because it'd race with
7230 * other threads incrementing park_pending and setting the bit.
7231 */
37d1e2e3 7232 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
9e138a48
PB
7233 if (atomic_dec_return(&sqd->park_pending))
7234 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
09a6f4ef 7235 mutex_unlock(&sqd->lock);
37d1e2e3
JA
7236}
7237
86e0d676 7238static void io_sq_thread_park(struct io_sq_data *sqd)
09a6f4ef 7239 __acquires(&sqd->lock)
37d1e2e3 7240{
521d6a73
PB
7241 WARN_ON_ONCE(sqd->thread == current);
7242
9e138a48 7243 atomic_inc(&sqd->park_pending);
86e0d676 7244 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
09a6f4ef 7245 mutex_lock(&sqd->lock);
05962f95 7246 if (sqd->thread)
86e0d676 7247 wake_up_process(sqd->thread);
37d1e2e3
JA
7248}
7249
7250static void io_sq_thread_stop(struct io_sq_data *sqd)
7251{
521d6a73
PB
7252 WARN_ON_ONCE(sqd->thread == current);
7253
09a6f4ef 7254 mutex_lock(&sqd->lock);
05962f95 7255 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
e8f98f24
JA
7256 if (sqd->thread)
7257 wake_up_process(sqd->thread);
09a6f4ef 7258 mutex_unlock(&sqd->lock);
05962f95 7259 wait_for_completion(&sqd->exited);
37d1e2e3
JA
7260}
7261
534ca6d6 7262static void io_put_sq_data(struct io_sq_data *sqd)
6c271ce2 7263{
534ca6d6 7264 if (refcount_dec_and_test(&sqd->refs)) {
9e138a48
PB
7265 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7266
37d1e2e3
JA
7267 io_sq_thread_stop(sqd);
7268 kfree(sqd);
7269 }
7270}
7271
7272static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7273{
7274 struct io_sq_data *sqd = ctx->sq_data;
7275
7276 if (sqd) {
05962f95 7277 io_sq_thread_park(sqd);
521d6a73 7278 list_del_init(&ctx->sqd_list);
37d1e2e3 7279 io_sqd_update_thread_idle(sqd);
05962f95 7280 io_sq_thread_unpark(sqd);
37d1e2e3
JA
7281
7282 io_put_sq_data(sqd);
7283 ctx->sq_data = NULL;
7c30f36a
SM
7284 if (ctx->sq_creds)
7285 put_cred(ctx->sq_creds);
534ca6d6
JA
7286 }
7287}
7288
aa06165d
JA
7289static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7290{
7291 struct io_ring_ctx *ctx_attach;
7292 struct io_sq_data *sqd;
7293 struct fd f;
7294
7295 f = fdget(p->wq_fd);
7296 if (!f.file)
7297 return ERR_PTR(-ENXIO);
7298 if (f.file->f_op != &io_uring_fops) {
7299 fdput(f);
7300 return ERR_PTR(-EINVAL);
7301 }
7302
7303 ctx_attach = f.file->private_data;
7304 sqd = ctx_attach->sq_data;
7305 if (!sqd) {
7306 fdput(f);
7307 return ERR_PTR(-EINVAL);
7308 }
5c2469e0
JA
7309 if (sqd->task_tgid != current->tgid) {
7310 fdput(f);
7311 return ERR_PTR(-EPERM);
7312 }
aa06165d
JA
7313
7314 refcount_inc(&sqd->refs);
7315 fdput(f);
7316 return sqd;
7317}
7318
26984fbf
PB
7319static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
7320 bool *attached)
534ca6d6
JA
7321{
7322 struct io_sq_data *sqd;
7323
26984fbf 7324 *attached = false;
5c2469e0
JA
7325 if (p->flags & IORING_SETUP_ATTACH_WQ) {
7326 sqd = io_attach_sq_data(p);
26984fbf
PB
7327 if (!IS_ERR(sqd)) {
7328 *attached = true;
5c2469e0 7329 return sqd;
26984fbf 7330 }
5c2469e0
JA
7331 /* fall through for EPERM case, setup new sqd/task */
7332 if (PTR_ERR(sqd) != -EPERM)
7333 return sqd;
7334 }
aa06165d 7335
534ca6d6
JA
7336 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7337 if (!sqd)
7338 return ERR_PTR(-ENOMEM);
7339
9e138a48 7340 atomic_set(&sqd->park_pending, 0);
534ca6d6 7341 refcount_set(&sqd->refs, 1);
69fb2131 7342 INIT_LIST_HEAD(&sqd->ctx_list);
09a6f4ef 7343 mutex_init(&sqd->lock);
534ca6d6 7344 init_waitqueue_head(&sqd->wait);
37d1e2e3 7345 init_completion(&sqd->exited);
534ca6d6
JA
7346 return sqd;
7347}
7348
6b06314c 7349#if defined(CONFIG_UNIX)
6b06314c
JA
7350/*
7351 * Ensure the UNIX gc is aware of our file set, so we are certain that
7352 * the io_uring can be safely unregistered on process exit, even if we have
7353 * loops in the file referencing.
7354 */
7355static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7356{
7357 struct sock *sk = ctx->ring_sock->sk;
7358 struct scm_fp_list *fpl;
7359 struct sk_buff *skb;
08a45173 7360 int i, nr_files;
6b06314c 7361
6b06314c
JA
7362 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7363 if (!fpl)
7364 return -ENOMEM;
7365
7366 skb = alloc_skb(0, GFP_KERNEL);
7367 if (!skb) {
7368 kfree(fpl);
7369 return -ENOMEM;
7370 }
7371
7372 skb->sk = sk;
6b06314c 7373
08a45173 7374 nr_files = 0;
62e398be 7375 fpl->user = get_uid(current_user());
6b06314c 7376 for (i = 0; i < nr; i++) {
65e19f54
JA
7377 struct file *file = io_file_from_index(ctx, i + offset);
7378
7379 if (!file)
08a45173 7380 continue;
65e19f54 7381 fpl->fp[nr_files] = get_file(file);
08a45173
JA
7382 unix_inflight(fpl->user, fpl->fp[nr_files]);
7383 nr_files++;
6b06314c
JA
7384 }
7385
08a45173
JA
7386 if (nr_files) {
7387 fpl->max = SCM_MAX_FD;
7388 fpl->count = nr_files;
7389 UNIXCB(skb).fp = fpl;
05f3fb3c 7390 skb->destructor = unix_destruct_scm;
08a45173
JA
7391 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7392 skb_queue_head(&sk->sk_receive_queue, skb);
6b06314c 7393
08a45173
JA
7394 for (i = 0; i < nr_files; i++)
7395 fput(fpl->fp[i]);
7396 } else {
7397 kfree_skb(skb);
7398 kfree(fpl);
7399 }
6b06314c
JA
7400
7401 return 0;
7402}
7403
7404/*
7405 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7406 * causes regular reference counting to break down. We rely on the UNIX
7407 * garbage collection to take care of this problem for us.
7408 */
7409static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7410{
7411 unsigned left, total;
7412 int ret = 0;
7413
7414 total = 0;
7415 left = ctx->nr_user_files;
7416 while (left) {
7417 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
6b06314c
JA
7418
7419 ret = __io_sqe_files_scm(ctx, this_files, total);
7420 if (ret)
7421 break;
7422 left -= this_files;
7423 total += this_files;
7424 }
7425
7426 if (!ret)
7427 return 0;
7428
7429 while (total < ctx->nr_user_files) {
65e19f54
JA
7430 struct file *file = io_file_from_index(ctx, total);
7431
7432 if (file)
7433 fput(file);
6b06314c
JA
7434 total++;
7435 }
7436
7437 return ret;
7438}
7439#else
7440static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7441{
7442 return 0;
7443}
7444#endif
7445
b895c9a6 7446static int io_sqe_alloc_file_tables(struct io_rsrc_data *file_data,
5398ae69 7447 unsigned nr_tables, unsigned nr_files)
65e19f54
JA
7448{
7449 int i;
7450
7451 for (i = 0; i < nr_tables; i++) {
269bbe5f 7452 struct fixed_rsrc_table *table = &file_data->table[i];
65e19f54
JA
7453 unsigned this_files;
7454
7455 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
7456 table->files = kcalloc(this_files, sizeof(struct file *),
7457 GFP_KERNEL);
7458 if (!table->files)
7459 break;
7460 nr_files -= this_files;
7461 }
7462
7463 if (i == nr_tables)
7464 return 0;
7465
7466 for (i = 0; i < nr_tables; i++) {
269bbe5f 7467 struct fixed_rsrc_table *table = &file_data->table[i];
65e19f54
JA
7468 kfree(table->files);
7469 }
7470 return 1;
7471}
7472
50238531 7473static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
05f3fb3c 7474{
50238531 7475 struct file *file = prsrc->file;
05f3fb3c
JA
7476#if defined(CONFIG_UNIX)
7477 struct sock *sock = ctx->ring_sock->sk;
7478 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7479 struct sk_buff *skb;
7480 int i;
7481
7482 __skb_queue_head_init(&list);
7483
7484 /*
7485 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7486 * remove this entry and rearrange the file array.
7487 */
7488 skb = skb_dequeue(head);
7489 while (skb) {
7490 struct scm_fp_list *fp;
7491
7492 fp = UNIXCB(skb).fp;
7493 for (i = 0; i < fp->count; i++) {
7494 int left;
7495
7496 if (fp->fp[i] != file)
7497 continue;
7498
7499 unix_notinflight(fp->user, fp->fp[i]);
7500 left = fp->count - 1 - i;
7501 if (left) {
7502 memmove(&fp->fp[i], &fp->fp[i + 1],
7503 left * sizeof(struct file *));
7504 }
7505 fp->count--;
7506 if (!fp->count) {
7507 kfree_skb(skb);
7508 skb = NULL;
7509 } else {
7510 __skb_queue_tail(&list, skb);
7511 }
7512 fput(file);
7513 file = NULL;
7514 break;
7515 }
7516
7517 if (!file)
7518 break;
7519
7520 __skb_queue_tail(&list, skb);
7521
7522 skb = skb_dequeue(head);
7523 }
7524
7525 if (skb_peek(&list)) {
7526 spin_lock_irq(&head->lock);
7527 while ((skb = __skb_dequeue(&list)) != NULL)
7528 __skb_queue_tail(head, skb);
7529 spin_unlock_irq(&head->lock);
7530 }
7531#else
7532 fput(file);
7533#endif
7534}
7535
b895c9a6 7536static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
65e19f54 7537{
b895c9a6 7538 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
269bbe5f
BM
7539 struct io_ring_ctx *ctx = rsrc_data->ctx;
7540 struct io_rsrc_put *prsrc, *tmp;
05589553 7541
269bbe5f
BM
7542 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7543 list_del(&prsrc->list);
50238531 7544 ref_node->rsrc_put(ctx, prsrc);
269bbe5f 7545 kfree(prsrc);
65e19f54 7546 }
05589553 7547
05589553
XW
7548 percpu_ref_exit(&ref_node->refs);
7549 kfree(ref_node);
269bbe5f 7550 percpu_ref_put(&rsrc_data->refs);
2faf852d 7551}
65e19f54 7552
269bbe5f 7553static void io_rsrc_put_work(struct work_struct *work)
4a38aed2
JA
7554{
7555 struct io_ring_ctx *ctx;
7556 struct llist_node *node;
7557
269bbe5f
BM
7558 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7559 node = llist_del_all(&ctx->rsrc_put_llist);
4a38aed2
JA
7560
7561 while (node) {
b895c9a6 7562 struct io_rsrc_node *ref_node;
4a38aed2
JA
7563 struct llist_node *next = node->next;
7564
b895c9a6 7565 ref_node = llist_entry(node, struct io_rsrc_node, llist);
269bbe5f 7566 __io_rsrc_put_work(ref_node);
4a38aed2
JA
7567 node = next;
7568 }
7569}
7570
00835dce 7571static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
2faf852d 7572{
b895c9a6
PB
7573 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7574 struct io_rsrc_data *data = node->rsrc_data;
7575 struct io_ring_ctx *ctx = data->ctx;
e297822b 7576 bool first_add = false;
221aa924 7577 int delay;
65e19f54 7578
2a63b2d9 7579 io_rsrc_ref_lock(ctx);
b895c9a6 7580 node->done = true;
e297822b 7581
d67d2263 7582 while (!list_empty(&ctx->rsrc_ref_list)) {
b895c9a6
PB
7583 node = list_first_entry(&ctx->rsrc_ref_list,
7584 struct io_rsrc_node, node);
e297822b 7585 /* recycle ref nodes in order */
b895c9a6 7586 if (!node->done)
e297822b 7587 break;
b895c9a6
PB
7588 list_del(&node->node);
7589 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
e297822b 7590 }
2a63b2d9 7591 io_rsrc_ref_unlock(ctx);
05589553 7592
221aa924
PB
7593 delay = percpu_ref_is_dying(&data->refs) ? 0 : HZ;
7594 if (first_add || !delay)
7595 mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
05f3fb3c 7596}
65e19f54 7597
b895c9a6 7598static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
05f3fb3c 7599{
b895c9a6 7600 struct io_rsrc_node *ref_node;
05f3fb3c 7601
05589553
XW
7602 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7603 if (!ref_node)
3e2224c5 7604 return NULL;
05f3fb3c 7605
00835dce 7606 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
05589553
XW
7607 0, GFP_KERNEL)) {
7608 kfree(ref_node);
3e2224c5 7609 return NULL;
05589553
XW
7610 }
7611 INIT_LIST_HEAD(&ref_node->node);
269bbe5f 7612 INIT_LIST_HEAD(&ref_node->rsrc_list);
e297822b 7613 ref_node->done = false;
05589553 7614 return ref_node;
05589553
XW
7615}
7616
b895c9a6 7617static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
05589553
XW
7618{
7619 percpu_ref_exit(&ref_node->refs);
7620 kfree(ref_node);
65e19f54
JA
7621}
7622
6b06314c
JA
7623static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
7624 unsigned nr_args)
7625{
7626 __s32 __user *fds = (__s32 __user *) arg;
600cf3f8 7627 unsigned nr_tables, i;
05f3fb3c 7628 struct file *file;
f3baed39 7629 int fd, ret;
b895c9a6 7630 struct io_rsrc_data *file_data;
6b06314c 7631
05f3fb3c 7632 if (ctx->file_data)
6b06314c
JA
7633 return -EBUSY;
7634 if (!nr_args)
7635 return -EINVAL;
7636 if (nr_args > IORING_MAX_FIXED_FILES)
7637 return -EMFILE;
f3baed39
PB
7638 ret = io_rsrc_node_prealloc(ctx);
7639 if (ret)
7640 return ret;
6b06314c 7641
b895c9a6 7642 file_data = io_rsrc_data_alloc(ctx);
5398ae69 7643 if (!file_data)
05f3fb3c 7644 return -ENOMEM;
13770a71 7645 ctx->file_data = file_data;
05f3fb3c 7646
f3baed39 7647 ret = -ENOMEM;
65e19f54 7648 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
035fbafc 7649 file_data->table = kcalloc(nr_tables, sizeof(*file_data->table),
5398ae69 7650 GFP_KERNEL);
600cf3f8
PB
7651 if (!file_data->table)
7652 goto out_free;
05f3fb3c 7653
600cf3f8 7654 if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args))
1ad555c6 7655 goto out_free;
65e19f54 7656
08a45173 7657 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
7b29f92d
JA
7658 unsigned long file_ptr;
7659
600cf3f8
PB
7660 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
7661 ret = -EFAULT;
7662 goto out_fput;
7663 }
08a45173 7664 /* allow sparse sets */
600cf3f8 7665 if (fd == -1)
08a45173 7666 continue;
6b06314c 7667
05f3fb3c 7668 file = fget(fd);
6b06314c 7669 ret = -EBADF;
05f3fb3c 7670 if (!file)
600cf3f8 7671 goto out_fput;
05f3fb3c 7672
6b06314c
JA
7673 /*
7674 * Don't allow io_uring instances to be registered. If UNIX
7675 * isn't enabled, then this causes a reference cycle and this
7676 * instance can never get freed. If UNIX is enabled we'll
7677 * handle it just fine, but there's still no point in allowing
7678 * a ring fd as it doesn't support regular read/write anyway.
7679 */
05f3fb3c
JA
7680 if (file->f_op == &io_uring_fops) {
7681 fput(file);
600cf3f8 7682 goto out_fput;
6b06314c 7683 }
7b29f92d
JA
7684 file_ptr = (unsigned long) file;
7685 if (__io_file_supports_async(file, READ))
7686 file_ptr |= FFS_ASYNC_READ;
7687 if (__io_file_supports_async(file, WRITE))
7688 file_ptr |= FFS_ASYNC_WRITE;
7689 if (S_ISREG(file_inode(file)->i_mode))
7690 file_ptr |= FFS_ISREG;
7691 *io_fixed_file_slot(file_data, i) = (struct file *) file_ptr;
6b06314c
JA
7692 }
7693
6b06314c 7694 ret = io_sqe_files_scm(ctx);
05589553 7695 if (ret) {
6b06314c 7696 io_sqe_files_unregister(ctx);
05589553
XW
7697 return ret;
7698 }
6b06314c 7699
82fbcfa9 7700 io_rsrc_node_set(ctx, file_data, io_ring_file_put);
6b06314c 7701 return ret;
600cf3f8
PB
7702out_fput:
7703 for (i = 0; i < ctx->nr_user_files; i++) {
7704 file = io_file_from_index(ctx, i);
7705 if (file)
7706 fput(file);
7707 }
7708 for (i = 0; i < nr_tables; i++)
7709 kfree(file_data->table[i].files);
7710 ctx->nr_user_files = 0;
600cf3f8 7711out_free:
b895c9a6 7712 io_rsrc_data_free(ctx->file_data);
55cbc256 7713 ctx->file_data = NULL;
6b06314c
JA
7714 return ret;
7715}
7716
c3a31e60
JA
7717static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7718 int index)
7719{
7720#if defined(CONFIG_UNIX)
7721 struct sock *sock = ctx->ring_sock->sk;
7722 struct sk_buff_head *head = &sock->sk_receive_queue;
7723 struct sk_buff *skb;
7724
7725 /*
7726 * See if we can merge this file into an existing skb SCM_RIGHTS
7727 * file set. If there's no room, fall back to allocating a new skb
7728 * and filling it in.
7729 */
7730 spin_lock_irq(&head->lock);
7731 skb = skb_peek(head);
7732 if (skb) {
7733 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7734
7735 if (fpl->count < SCM_MAX_FD) {
7736 __skb_unlink(skb, head);
7737 spin_unlock_irq(&head->lock);
7738 fpl->fp[fpl->count] = get_file(file);
7739 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7740 fpl->count++;
7741 spin_lock_irq(&head->lock);
7742 __skb_queue_head(head, skb);
7743 } else {
7744 skb = NULL;
7745 }
7746 }
7747 spin_unlock_irq(&head->lock);
7748
7749 if (skb) {
7750 fput(file);
7751 return 0;
7752 }
7753
7754 return __io_sqe_files_scm(ctx, 1, index);
7755#else
7756 return 0;
7757#endif
7758}
7759
b895c9a6 7760static int io_queue_rsrc_removal(struct io_rsrc_data *data, void *rsrc)
05f3fb3c 7761{
269bbe5f 7762 struct io_rsrc_put *prsrc;
b895c9a6 7763 struct io_rsrc_node *ref_node = data->node;
05f3fb3c 7764
269bbe5f
BM
7765 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
7766 if (!prsrc)
a5318d3c 7767 return -ENOMEM;
05f3fb3c 7768
50238531 7769 prsrc->rsrc = rsrc;
269bbe5f 7770 list_add(&prsrc->list, &ref_node->rsrc_list);
05589553 7771
a5318d3c 7772 return 0;
05f3fb3c
JA
7773}
7774
b895c9a6 7775static inline int io_queue_file_removal(struct io_rsrc_data *data,
269bbe5f
BM
7776 struct file *file)
7777{
50238531 7778 return io_queue_rsrc_removal(data, (void *)file);
269bbe5f
BM
7779}
7780
05f3fb3c 7781static int __io_sqe_files_update(struct io_ring_ctx *ctx,
269bbe5f 7782 struct io_uring_rsrc_update *up,
05f3fb3c
JA
7783 unsigned nr_args)
7784{
b895c9a6 7785 struct io_rsrc_data *data = ctx->file_data;
ea64ec02 7786 struct file *file, **file_slot;
c3a31e60
JA
7787 __s32 __user *fds;
7788 int fd, i, err;
7789 __u32 done;
05589553 7790 bool needs_switch = false;
c3a31e60 7791
05f3fb3c 7792 if (check_add_overflow(up->offset, nr_args, &done))
c3a31e60
JA
7793 return -EOVERFLOW;
7794 if (done > ctx->nr_user_files)
7795 return -EINVAL;
b895c9a6 7796 err = io_rsrc_node_prealloc(ctx);
8dd03afe
PB
7797 if (err)
7798 return err;
05589553 7799
269bbe5f 7800 fds = u64_to_user_ptr(up->data);
67973b93 7801 for (done = 0; done < nr_args; done++) {
c3a31e60
JA
7802 err = 0;
7803 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
7804 err = -EFAULT;
7805 break;
7806 }
4e0377a1 7807 if (fd == IORING_REGISTER_FILES_SKIP)
7808 continue;
7809
67973b93 7810 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
ea64ec02
PB
7811 file_slot = io_fixed_file_slot(ctx->file_data, i);
7812
7813 if (*file_slot) {
7b29f92d
JA
7814 file = (struct file *) ((unsigned long) *file_slot & FFS_MASK);
7815 err = io_queue_file_removal(data, file);
a5318d3c
HD
7816 if (err)
7817 break;
ea64ec02 7818 *file_slot = NULL;
05589553 7819 needs_switch = true;
c3a31e60
JA
7820 }
7821 if (fd != -1) {
c3a31e60
JA
7822 file = fget(fd);
7823 if (!file) {
7824 err = -EBADF;
7825 break;
7826 }
7827 /*
7828 * Don't allow io_uring instances to be registered. If
7829 * UNIX isn't enabled, then this causes a reference
7830 * cycle and this instance can never get freed. If UNIX
7831 * is enabled we'll handle it just fine, but there's
7832 * still no point in allowing a ring fd as it doesn't
7833 * support regular read/write anyway.
7834 */
7835 if (file->f_op == &io_uring_fops) {
7836 fput(file);
7837 err = -EBADF;
7838 break;
7839 }
e68a3ff8 7840 *file_slot = file;
c3a31e60 7841 err = io_sqe_file_register(ctx, file, i);
f3bd9dae 7842 if (err) {
e68a3ff8 7843 *file_slot = NULL;
f3bd9dae 7844 fput(file);
c3a31e60 7845 break;
f3bd9dae 7846 }
c3a31e60 7847 }
05f3fb3c
JA
7848 }
7849
05589553 7850 if (needs_switch) {
b2e96852 7851 percpu_ref_kill(&data->node->refs);
82fbcfa9 7852 io_rsrc_node_set(ctx, data, io_ring_file_put);
8dd03afe 7853 }
c3a31e60
JA
7854 return done ? done : err;
7855}
05589553 7856
05f3fb3c
JA
7857static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
7858 unsigned nr_args)
7859{
269bbe5f 7860 struct io_uring_rsrc_update up;
05f3fb3c
JA
7861
7862 if (!ctx->file_data)
7863 return -ENXIO;
7864 if (!nr_args)
7865 return -EINVAL;
7866 if (copy_from_user(&up, arg, sizeof(up)))
7867 return -EFAULT;
7868 if (up.resv)
7869 return -EINVAL;
7870
7871 return __io_sqe_files_update(ctx, &up, nr_args);
7872}
c3a31e60 7873
5280f7e5 7874static struct io_wq_work *io_free_work(struct io_wq_work *work)
7d723065
JA
7875{
7876 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7877
5280f7e5
PB
7878 req = io_put_req_find_next(req);
7879 return req ? &req->work : NULL;
7d723065
JA
7880}
7881
685fe7fe
JA
7882static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
7883 struct task_struct *task)
24369c2e 7884{
e941894e 7885 struct io_wq_hash *hash;
24369c2e 7886 struct io_wq_data data;
24369c2e 7887 unsigned int concurrency;
24369c2e 7888
e941894e
JA
7889 hash = ctx->hash_map;
7890 if (!hash) {
7891 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
7892 if (!hash)
7893 return ERR_PTR(-ENOMEM);
7894 refcount_set(&hash->refs, 1);
7895 init_waitqueue_head(&hash->wait);
7896 ctx->hash_map = hash;
24369c2e
PB
7897 }
7898
e941894e 7899 data.hash = hash;
685fe7fe 7900 data.task = task;
e9fd9396 7901 data.free_work = io_free_work;
f5fa38c5 7902 data.do_work = io_wq_submit_work;
24369c2e 7903
d25e3a3d
JA
7904 /* Do QD, or 4 * CPUS, whatever is smallest */
7905 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
24369c2e 7906
5aa75ed5 7907 return io_wq_create(concurrency, &data);
24369c2e
PB
7908}
7909
5aa75ed5
JA
7910static int io_uring_alloc_task_context(struct task_struct *task,
7911 struct io_ring_ctx *ctx)
0f212204
JA
7912{
7913 struct io_uring_task *tctx;
d8a6df10 7914 int ret;
0f212204
JA
7915
7916 tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
7917 if (unlikely(!tctx))
7918 return -ENOMEM;
7919
d8a6df10
JA
7920 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
7921 if (unlikely(ret)) {
7922 kfree(tctx);
7923 return ret;
7924 }
7925
685fe7fe 7926 tctx->io_wq = io_init_wq_offload(ctx, task);
5aa75ed5
JA
7927 if (IS_ERR(tctx->io_wq)) {
7928 ret = PTR_ERR(tctx->io_wq);
7929 percpu_counter_destroy(&tctx->inflight);
7930 kfree(tctx);
7931 return ret;
7932 }
7933
0f212204
JA
7934 xa_init(&tctx->xa);
7935 init_waitqueue_head(&tctx->wait);
7936 tctx->last = NULL;
fdaf083c 7937 atomic_set(&tctx->in_idle, 0);
0f212204 7938 task->io_uring = tctx;
7cbf1722
JA
7939 spin_lock_init(&tctx->task_lock);
7940 INIT_WQ_LIST(&tctx->task_list);
7941 tctx->task_state = 0;
7942 init_task_work(&tctx->task_work, tctx_task_work);
0f212204
JA
7943 return 0;
7944}
7945
7946void __io_uring_free(struct task_struct *tsk)
7947{
7948 struct io_uring_task *tctx = tsk->io_uring;
7949
7950 WARN_ON_ONCE(!xa_empty(&tctx->xa));
ef8eaa4e
PB
7951 WARN_ON_ONCE(tctx->io_wq);
7952
d8a6df10 7953 percpu_counter_destroy(&tctx->inflight);
0f212204
JA
7954 kfree(tctx);
7955 tsk->io_uring = NULL;
7956}
7957
7e84e1c7
SG
7958static int io_sq_offload_create(struct io_ring_ctx *ctx,
7959 struct io_uring_params *p)
2b188cc1
JA
7960{
7961 int ret;
7962
d25e3a3d
JA
7963 /* Retain compatibility with failing for an invalid attach attempt */
7964 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
7965 IORING_SETUP_ATTACH_WQ) {
7966 struct fd f;
7967
7968 f = fdget(p->wq_fd);
7969 if (!f.file)
7970 return -ENXIO;
7971 if (f.file->f_op != &io_uring_fops) {
7972 fdput(f);
7973 return -EINVAL;
7974 }
7975 fdput(f);
7976 }
6c271ce2 7977 if (ctx->flags & IORING_SETUP_SQPOLL) {
46fe18b1 7978 struct task_struct *tsk;
534ca6d6 7979 struct io_sq_data *sqd;
26984fbf 7980 bool attached;
534ca6d6 7981
26984fbf 7982 sqd = io_get_sq_data(p, &attached);
534ca6d6
JA
7983 if (IS_ERR(sqd)) {
7984 ret = PTR_ERR(sqd);
7985 goto err;
7986 }
69fb2131 7987
7c30f36a 7988 ctx->sq_creds = get_current_cred();
534ca6d6 7989 ctx->sq_data = sqd;
917257da
JA
7990 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
7991 if (!ctx->sq_thread_idle)
7992 ctx->sq_thread_idle = HZ;
7993
26984fbf 7994 ret = 0;
78d7f6ba 7995 io_sq_thread_park(sqd);
de75a3d3
PB
7996 list_add(&ctx->sqd_list, &sqd->ctx_list);
7997 io_sqd_update_thread_idle(sqd);
26984fbf 7998 /* don't attach to a dying SQPOLL thread, would be racy */
de75a3d3 7999 if (attached && !sqd->thread)
26984fbf 8000 ret = -ENXIO;
78d7f6ba
PB
8001 io_sq_thread_unpark(sqd);
8002
de75a3d3
PB
8003 if (ret < 0)
8004 goto err;
8005 if (attached)
5aa75ed5 8006 return 0;
aa06165d 8007
6c271ce2 8008 if (p->flags & IORING_SETUP_SQ_AFF) {
44a9bd18 8009 int cpu = p->sq_thread_cpu;
6c271ce2 8010
917257da 8011 ret = -EINVAL;
44a9bd18 8012 if (cpu >= nr_cpu_ids)
e8f98f24 8013 goto err_sqpoll;
7889f44d 8014 if (!cpu_online(cpu))
e8f98f24 8015 goto err_sqpoll;
917257da 8016
37d1e2e3 8017 sqd->sq_cpu = cpu;
6c271ce2 8018 } else {
37d1e2e3 8019 sqd->sq_cpu = -1;
6c271ce2 8020 }
37d1e2e3
JA
8021
8022 sqd->task_pid = current->pid;
5c2469e0 8023 sqd->task_tgid = current->tgid;
46fe18b1
JA
8024 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8025 if (IS_ERR(tsk)) {
8026 ret = PTR_ERR(tsk);
e8f98f24 8027 goto err_sqpoll;
6c271ce2 8028 }
97a73a0f 8029
46fe18b1 8030 sqd->thread = tsk;
97a73a0f 8031 ret = io_uring_alloc_task_context(tsk, ctx);
46fe18b1 8032 wake_up_new_task(tsk);
0f212204
JA
8033 if (ret)
8034 goto err;
6c271ce2
JA
8035 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8036 /* Can't have SQ_AFF without SQPOLL */
8037 ret = -EINVAL;
8038 goto err;
8039 }
8040
2b188cc1
JA
8041 return 0;
8042err:
37d1e2e3 8043 io_sq_thread_finish(ctx);
2b188cc1 8044 return ret;
e8f98f24
JA
8045err_sqpoll:
8046 complete(&ctx->sq_data->exited);
8047 goto err;
2b188cc1
JA
8048}
8049
a087e2b5
BM
8050static inline void __io_unaccount_mem(struct user_struct *user,
8051 unsigned long nr_pages)
2b188cc1
JA
8052{
8053 atomic_long_sub(nr_pages, &user->locked_vm);
8054}
8055
a087e2b5
BM
8056static inline int __io_account_mem(struct user_struct *user,
8057 unsigned long nr_pages)
2b188cc1
JA
8058{
8059 unsigned long page_limit, cur_pages, new_pages;
8060
8061 /* Don't allow more pages than we can safely lock */
8062 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8063
8064 do {
8065 cur_pages = atomic_long_read(&user->locked_vm);
8066 new_pages = cur_pages + nr_pages;
8067 if (new_pages > page_limit)
8068 return -ENOMEM;
8069 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8070 new_pages) != cur_pages);
8071
8072 return 0;
8073}
8074
26bfa89e 8075static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
a087e2b5 8076{
62e398be 8077 if (ctx->user)
a087e2b5 8078 __io_unaccount_mem(ctx->user, nr_pages);
30975825 8079
26bfa89e
JA
8080 if (ctx->mm_account)
8081 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
a087e2b5
BM
8082}
8083
26bfa89e 8084static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
a087e2b5 8085{
30975825
BM
8086 int ret;
8087
62e398be 8088 if (ctx->user) {
30975825
BM
8089 ret = __io_account_mem(ctx->user, nr_pages);
8090 if (ret)
8091 return ret;
8092 }
8093
26bfa89e
JA
8094 if (ctx->mm_account)
8095 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
a087e2b5
BM
8096
8097 return 0;
8098}
8099
2b188cc1
JA
8100static void io_mem_free(void *ptr)
8101{
52e04ef4
MR
8102 struct page *page;
8103
8104 if (!ptr)
8105 return;
2b188cc1 8106
52e04ef4 8107 page = virt_to_head_page(ptr);
2b188cc1
JA
8108 if (put_page_testzero(page))
8109 free_compound_page(page);
8110}
8111
8112static void *io_mem_alloc(size_t size)
8113{
8114 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
26bfa89e 8115 __GFP_NORETRY | __GFP_ACCOUNT;
2b188cc1
JA
8116
8117 return (void *) __get_free_pages(gfp_flags, get_order(size));
8118}
8119
75b28aff
HV
8120static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8121 size_t *sq_offset)
8122{
8123 struct io_rings *rings;
8124 size_t off, sq_array_size;
8125
8126 off = struct_size(rings, cqes, cq_entries);
8127 if (off == SIZE_MAX)
8128 return SIZE_MAX;
8129
8130#ifdef CONFIG_SMP
8131 off = ALIGN(off, SMP_CACHE_BYTES);
8132 if (off == 0)
8133 return SIZE_MAX;
8134#endif
8135
b36200f5
DV
8136 if (sq_offset)
8137 *sq_offset = off;
8138
75b28aff
HV
8139 sq_array_size = array_size(sizeof(u32), sq_entries);
8140 if (sq_array_size == SIZE_MAX)
8141 return SIZE_MAX;
8142
8143 if (check_add_overflow(off, sq_array_size, &off))
8144 return SIZE_MAX;
8145
75b28aff
HV
8146 return off;
8147}
8148
0a96bbe4 8149static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
edafccee
JA
8150{
8151 int i, j;
8152
8153 if (!ctx->user_bufs)
8154 return -ENXIO;
8155
8156 for (i = 0; i < ctx->nr_user_bufs; i++) {
8157 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
8158
8159 for (j = 0; j < imu->nr_bvecs; j++)
f1f6a7dd 8160 unpin_user_page(imu->bvec[j].bv_page);
edafccee 8161
de293938 8162 if (imu->acct_pages)
26bfa89e 8163 io_unaccount_mem(ctx, imu->acct_pages);
d4ef6475 8164 kvfree(imu->bvec);
edafccee
JA
8165 imu->nr_bvecs = 0;
8166 }
8167
8168 kfree(ctx->user_bufs);
8169 ctx->user_bufs = NULL;
8170 ctx->nr_user_bufs = 0;
8171 return 0;
8172}
8173
8174static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8175 void __user *arg, unsigned index)
8176{
8177 struct iovec __user *src;
8178
8179#ifdef CONFIG_COMPAT
8180 if (ctx->compat) {
8181 struct compat_iovec __user *ciovs;
8182 struct compat_iovec ciov;
8183
8184 ciovs = (struct compat_iovec __user *) arg;
8185 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8186 return -EFAULT;
8187
d55e5f5b 8188 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
edafccee
JA
8189 dst->iov_len = ciov.iov_len;
8190 return 0;
8191 }
8192#endif
8193 src = (struct iovec __user *) arg;
8194 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8195 return -EFAULT;
8196 return 0;
8197}
8198
de293938
JA
8199/*
8200 * Not super efficient, but this is just a registration time. And we do cache
8201 * the last compound head, so generally we'll only do a full search if we don't
8202 * match that one.
8203 *
8204 * We check if the given compound head page has already been accounted, to
8205 * avoid double accounting it. This allows us to account the full size of the
8206 * page, not just the constituent pages of a huge page.
8207 */
8208static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8209 int nr_pages, struct page *hpage)
8210{
8211 int i, j;
8212
8213 /* check current page array */
8214 for (i = 0; i < nr_pages; i++) {
8215 if (!PageCompound(pages[i]))
8216 continue;
8217 if (compound_head(pages[i]) == hpage)
8218 return true;
8219 }
8220
8221 /* check previously registered pages */
8222 for (i = 0; i < ctx->nr_user_bufs; i++) {
8223 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
8224
8225 for (j = 0; j < imu->nr_bvecs; j++) {
8226 if (!PageCompound(imu->bvec[j].bv_page))
8227 continue;
8228 if (compound_head(imu->bvec[j].bv_page) == hpage)
8229 return true;
8230 }
8231 }
8232
8233 return false;
8234}
8235
8236static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8237 int nr_pages, struct io_mapped_ubuf *imu,
8238 struct page **last_hpage)
8239{
8240 int i, ret;
8241
8242 for (i = 0; i < nr_pages; i++) {
8243 if (!PageCompound(pages[i])) {
8244 imu->acct_pages++;
8245 } else {
8246 struct page *hpage;
8247
8248 hpage = compound_head(pages[i]);
8249 if (hpage == *last_hpage)
8250 continue;
8251 *last_hpage = hpage;
8252 if (headpage_already_acct(ctx, pages, i, hpage))
8253 continue;
8254 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8255 }
8256 }
8257
8258 if (!imu->acct_pages)
8259 return 0;
8260
26bfa89e 8261 ret = io_account_mem(ctx, imu->acct_pages);
de293938
JA
8262 if (ret)
8263 imu->acct_pages = 0;
8264 return ret;
8265}
8266
0a96bbe4
BM
8267static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
8268 struct io_mapped_ubuf *imu,
8269 struct page **last_hpage)
edafccee
JA
8270{
8271 struct vm_area_struct **vmas = NULL;
8272 struct page **pages = NULL;
0a96bbe4
BM
8273 unsigned long off, start, end, ubuf;
8274 size_t size;
8275 int ret, pret, nr_pages, i;
8276
8277 ubuf = (unsigned long) iov->iov_base;
8278 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8279 start = ubuf >> PAGE_SHIFT;
8280 nr_pages = end - start;
8281
8282 ret = -ENOMEM;
8283
8284 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8285 if (!pages)
8286 goto done;
8287
8288 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8289 GFP_KERNEL);
8290 if (!vmas)
8291 goto done;
edafccee 8292
0a96bbe4
BM
8293 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
8294 GFP_KERNEL);
8295 if (!imu->bvec)
8296 goto done;
8297
8298 ret = 0;
8299 mmap_read_lock(current->mm);
8300 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8301 pages, vmas);
8302 if (pret == nr_pages) {
8303 /* don't support file backed memory */
8304 for (i = 0; i < nr_pages; i++) {
8305 struct vm_area_struct *vma = vmas[i];
8306
8307 if (vma->vm_file &&
8308 !is_file_hugepages(vma->vm_file)) {
8309 ret = -EOPNOTSUPP;
8310 break;
8311 }
8312 }
8313 } else {
8314 ret = pret < 0 ? pret : -EFAULT;
8315 }
8316 mmap_read_unlock(current->mm);
8317 if (ret) {
8318 /*
8319 * if we did partial map, or found file backed vmas,
8320 * release any pages we did get
8321 */
8322 if (pret > 0)
8323 unpin_user_pages(pages, pret);
8324 kvfree(imu->bvec);
8325 goto done;
8326 }
8327
8328 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8329 if (ret) {
8330 unpin_user_pages(pages, pret);
8331 kvfree(imu->bvec);
8332 goto done;
8333 }
8334
8335 off = ubuf & ~PAGE_MASK;
8336 size = iov->iov_len;
8337 for (i = 0; i < nr_pages; i++) {
8338 size_t vec_len;
8339
8340 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8341 imu->bvec[i].bv_page = pages[i];
8342 imu->bvec[i].bv_len = vec_len;
8343 imu->bvec[i].bv_offset = off;
8344 off = 0;
8345 size -= vec_len;
8346 }
8347 /* store original address for later verification */
8348 imu->ubuf = ubuf;
8349 imu->len = iov->iov_len;
8350 imu->nr_bvecs = nr_pages;
8351 ret = 0;
8352done:
8353 kvfree(pages);
8354 kvfree(vmas);
8355 return ret;
8356}
8357
2b358604 8358static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
0a96bbe4 8359{
edafccee
JA
8360 if (ctx->user_bufs)
8361 return -EBUSY;
8362 if (!nr_args || nr_args > UIO_MAXIOV)
8363 return -EINVAL;
8364
8365 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
8366 GFP_KERNEL);
8367 if (!ctx->user_bufs)
8368 return -ENOMEM;
8369
2b358604
BM
8370 return 0;
8371}
edafccee 8372
2b358604
BM
8373static int io_buffer_validate(struct iovec *iov)
8374{
50e96989
PB
8375 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
8376
2b358604
BM
8377 /*
8378 * Don't impose further limits on the size and buffer
8379 * constraints here, we'll -EINVAL later when IO is
8380 * submitted if they are wrong.
8381 */
8382 if (!iov->iov_base || !iov->iov_len)
8383 return -EFAULT;
edafccee 8384
2b358604
BM
8385 /* arbitrary limit, but we need something */
8386 if (iov->iov_len > SZ_1G)
8387 return -EFAULT;
edafccee 8388
50e96989
PB
8389 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
8390 return -EOVERFLOW;
8391
2b358604
BM
8392 return 0;
8393}
edafccee 8394
2b358604
BM
8395static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
8396 unsigned int nr_args)
8397{
8398 int i, ret;
8399 struct iovec iov;
8400 struct page *last_hpage = NULL;
edafccee 8401
2b358604
BM
8402 ret = io_buffers_map_alloc(ctx, nr_args);
8403 if (ret)
8404 return ret;
edafccee 8405
edafccee
JA
8406 for (i = 0; i < nr_args; i++) {
8407 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
edafccee 8408
edafccee
JA
8409 ret = io_copy_iov(ctx, &iov, arg, i);
8410 if (ret)
0a96bbe4 8411 break;
de293938 8412
2b358604
BM
8413 ret = io_buffer_validate(&iov);
8414 if (ret)
0a96bbe4 8415 break;
edafccee 8416
0a96bbe4
BM
8417 ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage);
8418 if (ret)
8419 break;
edafccee
JA
8420
8421 ctx->nr_user_bufs++;
8422 }
0a96bbe4
BM
8423
8424 if (ret)
8425 io_sqe_buffers_unregister(ctx);
8426
edafccee
JA
8427 return ret;
8428}
8429
9b402849
JA
8430static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8431{
8432 __s32 __user *fds = arg;
8433 int fd;
8434
8435 if (ctx->cq_ev_fd)
8436 return -EBUSY;
8437
8438 if (copy_from_user(&fd, fds, sizeof(*fds)))
8439 return -EFAULT;
8440
8441 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8442 if (IS_ERR(ctx->cq_ev_fd)) {
8443 int ret = PTR_ERR(ctx->cq_ev_fd);
8444 ctx->cq_ev_fd = NULL;
8445 return ret;
8446 }
8447
8448 return 0;
8449}
8450
8451static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8452{
8453 if (ctx->cq_ev_fd) {
8454 eventfd_ctx_put(ctx->cq_ev_fd);
8455 ctx->cq_ev_fd = NULL;
8456 return 0;
8457 }
8458
8459 return -ENXIO;
8460}
8461
5a2e745d
JA
8462static void io_destroy_buffers(struct io_ring_ctx *ctx)
8463{
9e15c3a0
JA
8464 struct io_buffer *buf;
8465 unsigned long index;
8466
8467 xa_for_each(&ctx->io_buffers, index, buf)
8468 __io_remove_buffers(ctx, buf, index, -1U);
5a2e745d
JA
8469}
8470
68e68ee6 8471static void io_req_cache_free(struct list_head *list, struct task_struct *tsk)
1b4c351f 8472{
68e68ee6 8473 struct io_kiocb *req, *nxt;
1b4c351f 8474
68e68ee6
JA
8475 list_for_each_entry_safe(req, nxt, list, compl.list) {
8476 if (tsk && req->task != tsk)
8477 continue;
1b4c351f
JA
8478 list_del(&req->compl.list);
8479 kmem_cache_free(req_cachep, req);
8480 }
8481}
8482
4010fec4 8483static void io_req_caches_free(struct io_ring_ctx *ctx)
2b188cc1 8484{
bf019da7 8485 struct io_submit_state *submit_state = &ctx->submit_state;
e5547d2c 8486 struct io_comp_state *cs = &ctx->submit_state.comp;
bf019da7 8487
9a4fdbd8
JA
8488 mutex_lock(&ctx->uring_lock);
8489
8e5c66c4 8490 if (submit_state->free_reqs) {
9a4fdbd8
JA
8491 kmem_cache_free_bulk(req_cachep, submit_state->free_reqs,
8492 submit_state->reqs);
8e5c66c4
PB
8493 submit_state->free_reqs = 0;
8494 }
9a4fdbd8 8495
dac7a098 8496 io_flush_cached_locked_reqs(ctx, cs);
e5547d2c 8497 io_req_cache_free(&cs->free_list, NULL);
9a4fdbd8
JA
8498 mutex_unlock(&ctx->uring_lock);
8499}
8500
2b188cc1
JA
8501static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8502{
04fc6c80
PB
8503 /*
8504 * Some may use context even when all refs and requests have been put,
180f829f
PB
8505 * and they are free to do so while still holding uring_lock or
8506 * completion_lock, see __io_req_task_submit(). Wait for them to finish.
04fc6c80
PB
8507 */
8508 mutex_lock(&ctx->uring_lock);
8509 mutex_unlock(&ctx->uring_lock);
180f829f
PB
8510 spin_lock_irq(&ctx->completion_lock);
8511 spin_unlock_irq(&ctx->completion_lock);
04fc6c80 8512
37d1e2e3 8513 io_sq_thread_finish(ctx);
0a96bbe4 8514 io_sqe_buffers_unregister(ctx);
2aede0e4 8515
37d1e2e3 8516 if (ctx->mm_account) {
2aede0e4
JA
8517 mmdrop(ctx->mm_account);
8518 ctx->mm_account = NULL;
30975825 8519 }
def596e9 8520
8bad28d8 8521 mutex_lock(&ctx->uring_lock);
6b06314c 8522 io_sqe_files_unregister(ctx);
8bad28d8 8523 mutex_unlock(&ctx->uring_lock);
9b402849 8524 io_eventfd_unregister(ctx);
5a2e745d 8525 io_destroy_buffers(ctx);
def596e9 8526
8dd03afe 8527 if (ctx->rsrc_backup_node)
b895c9a6 8528 io_rsrc_node_destroy(ctx->rsrc_backup_node);
8dd03afe 8529
2b188cc1 8530#if defined(CONFIG_UNIX)
355e8d26
EB
8531 if (ctx->ring_sock) {
8532 ctx->ring_sock->file = NULL; /* so that iput() is called */
2b188cc1 8533 sock_release(ctx->ring_sock);
355e8d26 8534 }
2b188cc1
JA
8535#endif
8536
75b28aff 8537 io_mem_free(ctx->rings);
2b188cc1 8538 io_mem_free(ctx->sq_sqes);
2b188cc1
JA
8539
8540 percpu_ref_exit(&ctx->refs);
2b188cc1 8541 free_uid(ctx->user);
4010fec4 8542 io_req_caches_free(ctx);
e941894e
JA
8543 if (ctx->hash_map)
8544 io_wq_put_hash(ctx->hash_map);
78076bb6 8545 kfree(ctx->cancel_hash);
2b188cc1
JA
8546 kfree(ctx);
8547}
8548
8549static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8550{
8551 struct io_ring_ctx *ctx = file->private_data;
8552 __poll_t mask = 0;
8553
8554 poll_wait(file, &ctx->cq_wait, wait);
4f7067c3
SB
8555 /*
8556 * synchronizes with barrier from wq_has_sleeper call in
8557 * io_commit_cqring
8558 */
2b188cc1 8559 smp_rmb();
90554200 8560 if (!io_sqring_full(ctx))
2b188cc1 8561 mask |= EPOLLOUT | EPOLLWRNORM;
ed670c3f
HX
8562
8563 /*
8564 * Don't flush cqring overflow list here, just do a simple check.
8565 * Otherwise there could possible be ABBA deadlock:
8566 * CPU0 CPU1
8567 * ---- ----
8568 * lock(&ctx->uring_lock);
8569 * lock(&ep->mtx);
8570 * lock(&ctx->uring_lock);
8571 * lock(&ep->mtx);
8572 *
8573 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
8574 * pushs them to do the flush.
8575 */
8576 if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow))
2b188cc1
JA
8577 mask |= EPOLLIN | EPOLLRDNORM;
8578
8579 return mask;
8580}
8581
8582static int io_uring_fasync(int fd, struct file *file, int on)
8583{
8584 struct io_ring_ctx *ctx = file->private_data;
8585
8586 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8587}
8588
0bead8cd 8589static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
071698e1 8590{
4379bf8b 8591 const struct cred *creds;
071698e1 8592
61cf9370 8593 creds = xa_erase(&ctx->personalities, id);
4379bf8b
JA
8594 if (creds) {
8595 put_cred(creds);
0bead8cd 8596 return 0;
1e6fa521 8597 }
0bead8cd
YD
8598
8599 return -EINVAL;
8600}
8601
9b465711 8602static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx)
7c25c0d1 8603{
9b465711 8604 return io_run_task_work_head(&ctx->exit_task_work);
7c25c0d1
JA
8605}
8606
d56d938b
PB
8607struct io_tctx_exit {
8608 struct callback_head task_work;
8609 struct completion completion;
baf186c4 8610 struct io_ring_ctx *ctx;
d56d938b
PB
8611};
8612
8613static void io_tctx_exit_cb(struct callback_head *cb)
8614{
8615 struct io_uring_task *tctx = current->io_uring;
8616 struct io_tctx_exit *work;
8617
8618 work = container_of(cb, struct io_tctx_exit, task_work);
8619 /*
8620 * When @in_idle, we're in cancellation and it's racy to remove the
8621 * node. It'll be removed by the end of cancellation, just ignore it.
8622 */
8623 if (!atomic_read(&tctx->in_idle))
baf186c4 8624 io_uring_del_task_file((unsigned long)work->ctx);
d56d938b
PB
8625 complete(&work->completion);
8626}
8627
85faa7b8
JA
8628static void io_ring_exit_work(struct work_struct *work)
8629{
d56d938b 8630 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
b5bb3a24 8631 unsigned long timeout = jiffies + HZ * 60 * 5;
d56d938b
PB
8632 struct io_tctx_exit exit;
8633 struct io_tctx_node *node;
8634 int ret;
85faa7b8 8635
a185f1db
PB
8636 /* prevent SQPOLL from submitting new requests */
8637 if (ctx->sq_data) {
8638 io_sq_thread_park(ctx->sq_data);
8639 list_del_init(&ctx->sqd_list);
8640 io_sqd_update_thread_idle(ctx->sq_data);
8641 io_sq_thread_unpark(ctx->sq_data);
8642 }
8643
56952e91
JA
8644 /*
8645 * If we're doing polled IO and end up having requests being
8646 * submitted async (out-of-line), then completions can come in while
8647 * we're waiting for refs to drop. We need to reap these manually,
8648 * as nobody else will be looking for them.
8649 */
b2edc0a7 8650 do {
9936c7c2 8651 io_uring_try_cancel_requests(ctx, NULL, NULL);
b5bb3a24
PB
8652
8653 WARN_ON_ONCE(time_after(jiffies, timeout));
b2edc0a7 8654 } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
d56d938b
PB
8655
8656 mutex_lock(&ctx->uring_lock);
8657 while (!list_empty(&ctx->tctx_list)) {
b5bb3a24
PB
8658 WARN_ON_ONCE(time_after(jiffies, timeout));
8659
d56d938b
PB
8660 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
8661 ctx_node);
baf186c4 8662 exit.ctx = ctx;
d56d938b
PB
8663 init_completion(&exit.completion);
8664 init_task_work(&exit.task_work, io_tctx_exit_cb);
8665 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
8666 if (WARN_ON_ONCE(ret))
8667 continue;
8668 wake_up_process(node->task);
8669
8670 mutex_unlock(&ctx->uring_lock);
8671 wait_for_completion(&exit.completion);
8672 cond_resched();
8673 mutex_lock(&ctx->uring_lock);
8674 }
8675 mutex_unlock(&ctx->uring_lock);
8676
85faa7b8
JA
8677 io_ring_ctx_free(ctx);
8678}
8679
80c4cbdb
PB
8680/* Returns true if we found and killed one or more timeouts */
8681static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
8682 struct files_struct *files)
8683{
8684 struct io_kiocb *req, *tmp;
8685 int canceled = 0;
8686
8687 spin_lock_irq(&ctx->completion_lock);
8688 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
8689 if (io_match_task(req, tsk, files)) {
8690 io_kill_timeout(req, -ECANCELED);
8691 canceled++;
8692 }
8693 }
51520426
PB
8694 if (canceled != 0)
8695 io_commit_cqring(ctx);
80c4cbdb 8696 spin_unlock_irq(&ctx->completion_lock);
80c4cbdb
PB
8697 if (canceled != 0)
8698 io_cqring_ev_posted(ctx);
8699 return canceled != 0;
8700}
8701
2b188cc1
JA
8702static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8703{
61cf9370
MWO
8704 unsigned long index;
8705 struct creds *creds;
8706
2b188cc1
JA
8707 mutex_lock(&ctx->uring_lock);
8708 percpu_ref_kill(&ctx->refs);
cda286f0
PB
8709 /* if force is set, the ring is going away. always drop after that */
8710 ctx->cq_overflow_flushed = 1;
634578f8 8711 if (ctx->rings)
6c2450ae 8712 __io_cqring_overflow_flush(ctx, true);
61cf9370
MWO
8713 xa_for_each(&ctx->personalities, index, creds)
8714 io_unregister_personality(ctx, index);
2b188cc1
JA
8715 mutex_unlock(&ctx->uring_lock);
8716
6b81928d
PB
8717 io_kill_timeouts(ctx, NULL, NULL);
8718 io_poll_remove_all(ctx, NULL, NULL);
561fb04a 8719
15dff286 8720 /* if we failed setting up the ctx, we might not have any rings */
b2edc0a7 8721 io_iopoll_try_reap_events(ctx);
309fc03a 8722
85faa7b8 8723 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
fc666777
JA
8724 /*
8725 * Use system_unbound_wq to avoid spawning tons of event kworkers
8726 * if we're exiting a ton of rings at the same time. It just adds
8727 * noise and overhead, there's no discernable change in runtime
8728 * over using system_wq.
8729 */
8730 queue_work(system_unbound_wq, &ctx->exit_work);
2b188cc1
JA
8731}
8732
8733static int io_uring_release(struct inode *inode, struct file *file)
8734{
8735 struct io_ring_ctx *ctx = file->private_data;
8736
8737 file->private_data = NULL;
8738 io_ring_ctx_wait_and_kill(ctx);
8739 return 0;
8740}
8741
f6edbabb
PB
8742struct io_task_cancel {
8743 struct task_struct *task;
8744 struct files_struct *files;
8745};
f254ac04 8746
f6edbabb 8747static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
b711d4ea 8748{
9a472ef7 8749 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
f6edbabb 8750 struct io_task_cancel *cancel = data;
9a472ef7
PB
8751 bool ret;
8752
f6edbabb 8753 if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) {
9a472ef7
PB
8754 unsigned long flags;
8755 struct io_ring_ctx *ctx = req->ctx;
8756
8757 /* protect against races with linked timeouts */
8758 spin_lock_irqsave(&ctx->completion_lock, flags);
f6edbabb 8759 ret = io_match_task(req, cancel->task, cancel->files);
9a472ef7
PB
8760 spin_unlock_irqrestore(&ctx->completion_lock, flags);
8761 } else {
f6edbabb 8762 ret = io_match_task(req, cancel->task, cancel->files);
9a472ef7
PB
8763 }
8764 return ret;
b711d4ea
JA
8765}
8766
e1915f76 8767static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
ef9865a4 8768 struct task_struct *task,
b7ddce3c
PB
8769 struct files_struct *files)
8770{
e1915f76 8771 struct io_defer_entry *de;
b7ddce3c
PB
8772 LIST_HEAD(list);
8773
8774 spin_lock_irq(&ctx->completion_lock);
8775 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
08d23634 8776 if (io_match_task(de->req, task, files)) {
b7ddce3c
PB
8777 list_cut_position(&list, &ctx->defer_list, &de->list);
8778 break;
8779 }
8780 }
8781 spin_unlock_irq(&ctx->completion_lock);
e1915f76
PB
8782 if (list_empty(&list))
8783 return false;
b7ddce3c
PB
8784
8785 while (!list_empty(&list)) {
8786 de = list_first_entry(&list, struct io_defer_entry, list);
8787 list_del_init(&de->list);
f41db273 8788 io_req_complete_failed(de->req, -ECANCELED);
b7ddce3c
PB
8789 kfree(de);
8790 }
e1915f76 8791 return true;
b7ddce3c
PB
8792}
8793
1b00764f
PB
8794static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8795{
8796 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8797
8798 return req->ctx == data;
8799}
8800
8801static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
8802{
8803 struct io_tctx_node *node;
8804 enum io_wq_cancel cret;
8805 bool ret = false;
8806
8807 mutex_lock(&ctx->uring_lock);
8808 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
8809 struct io_uring_task *tctx = node->task->io_uring;
8810
8811 /*
8812 * io_wq will stay alive while we hold uring_lock, because it's
8813 * killed after ctx nodes, which requires to take the lock.
8814 */
8815 if (!tctx || !tctx->io_wq)
8816 continue;
8817 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
8818 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8819 }
8820 mutex_unlock(&ctx->uring_lock);
8821
8822 return ret;
8823}
8824
9936c7c2
PB
8825static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
8826 struct task_struct *task,
8827 struct files_struct *files)
8828{
8829 struct io_task_cancel cancel = { .task = task, .files = files, };
1b00764f 8830 struct io_uring_task *tctx = task ? task->io_uring : NULL;
9936c7c2
PB
8831
8832 while (1) {
8833 enum io_wq_cancel cret;
8834 bool ret = false;
8835
1b00764f
PB
8836 if (!task) {
8837 ret |= io_uring_try_cancel_iowq(ctx);
8838 } else if (tctx && tctx->io_wq) {
8839 /*
8840 * Cancels requests of all rings, not only @ctx, but
8841 * it's fine as the task is in exit/exec.
8842 */
5aa75ed5 8843 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
9936c7c2
PB
8844 &cancel, true);
8845 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8846 }
8847
8848 /* SQPOLL thread does its own polling */
d052d1d6
JA
8849 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && !files) ||
8850 (ctx->sq_data && ctx->sq_data->thread == current)) {
9936c7c2
PB
8851 while (!list_empty_careful(&ctx->iopoll_list)) {
8852 io_iopoll_try_reap_events(ctx);
8853 ret = true;
8854 }
8855 }
8856
e1915f76 8857 ret |= io_cancel_defer_files(ctx, task, files);
9936c7c2
PB
8858 ret |= io_poll_remove_all(ctx, task, files);
8859 ret |= io_kill_timeouts(ctx, task, files);
8860 ret |= io_run_task_work();
ba50a036 8861 ret |= io_run_ctx_fallback(ctx);
9936c7c2
PB
8862 if (!ret)
8863 break;
8864 cond_resched();
8865 }
8866}
8867
ca70f00b
PB
8868static int io_uring_count_inflight(struct io_ring_ctx *ctx,
8869 struct task_struct *task,
8870 struct files_struct *files)
8871{
8872 struct io_kiocb *req;
8873 int cnt = 0;
8874
8875 spin_lock_irq(&ctx->inflight_lock);
8876 list_for_each_entry(req, &ctx->inflight_list, inflight_entry)
8877 cnt += io_match_task(req, task, files);
8878 spin_unlock_irq(&ctx->inflight_lock);
8879 return cnt;
8880}
8881
b52fda00 8882static void io_uring_cancel_files(struct io_ring_ctx *ctx,
df9923f9 8883 struct task_struct *task,
fcb323cc
JA
8884 struct files_struct *files)
8885{
fcb323cc 8886 while (!list_empty_careful(&ctx->inflight_list)) {
d8f1b971 8887 DEFINE_WAIT(wait);
ca70f00b 8888 int inflight;
fcb323cc 8889
ca70f00b
PB
8890 inflight = io_uring_count_inflight(ctx, task, files);
8891 if (!inflight)
fcb323cc 8892 break;
f6edbabb 8893
9936c7c2 8894 io_uring_try_cancel_requests(ctx, task, files);
ca70f00b
PB
8895
8896 prepare_to_wait(&task->io_uring->wait, &wait,
8897 TASK_UNINTERRUPTIBLE);
8898 if (inflight == io_uring_count_inflight(ctx, task, files))
8899 schedule();
c98de08c 8900 finish_wait(&task->io_uring->wait, &wait);
0f212204 8901 }
0f212204
JA
8902}
8903
cf27f3b1 8904static int __io_uring_add_task_file(struct io_ring_ctx *ctx)
0f212204 8905{
236434c3 8906 struct io_uring_task *tctx = current->io_uring;
13bf43f5 8907 struct io_tctx_node *node;
a528b04e 8908 int ret;
236434c3
MWO
8909
8910 if (unlikely(!tctx)) {
5aa75ed5 8911 ret = io_uring_alloc_task_context(current, ctx);
0f212204
JA
8912 if (unlikely(ret))
8913 return ret;
236434c3 8914 tctx = current->io_uring;
0f212204 8915 }
cf27f3b1
PB
8916 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
8917 node = kmalloc(sizeof(*node), GFP_KERNEL);
8918 if (!node)
8919 return -ENOMEM;
8920 node->ctx = ctx;
8921 node->task = current;
13bf43f5 8922
cf27f3b1
PB
8923 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
8924 node, GFP_KERNEL));
8925 if (ret) {
8926 kfree(node);
8927 return ret;
0f212204 8928 }
cf27f3b1
PB
8929
8930 mutex_lock(&ctx->uring_lock);
8931 list_add(&node->ctx_node, &ctx->tctx_list);
8932 mutex_unlock(&ctx->uring_lock);
0f212204 8933 }
cf27f3b1 8934 tctx->last = ctx;
0f212204
JA
8935 return 0;
8936}
8937
cf27f3b1
PB
8938/*
8939 * Note that this task has used io_uring. We use it for cancelation purposes.
8940 */
8941static inline int io_uring_add_task_file(struct io_ring_ctx *ctx)
8942{
8943 struct io_uring_task *tctx = current->io_uring;
8944
8945 if (likely(tctx && tctx->last == ctx))
8946 return 0;
8947 return __io_uring_add_task_file(ctx);
8948}
8949
0f212204
JA
8950/*
8951 * Remove this io_uring_file -> task mapping.
8952 */
2941267b 8953static void io_uring_del_task_file(unsigned long index)
0f212204
JA
8954{
8955 struct io_uring_task *tctx = current->io_uring;
13bf43f5 8956 struct io_tctx_node *node;
2941267b 8957
eebd2e37
PB
8958 if (!tctx)
8959 return;
13bf43f5
PB
8960 node = xa_erase(&tctx->xa, index);
8961 if (!node)
2941267b 8962 return;
0f212204 8963
13bf43f5
PB
8964 WARN_ON_ONCE(current != node->task);
8965 WARN_ON_ONCE(list_empty(&node->ctx_node));
8966
8967 mutex_lock(&node->ctx->uring_lock);
8968 list_del(&node->ctx_node);
8969 mutex_unlock(&node->ctx->uring_lock);
8970
baf186c4 8971 if (tctx->last == node->ctx)
0f212204 8972 tctx->last = NULL;
13bf43f5 8973 kfree(node);
0f212204
JA
8974}
8975
8452d4a6 8976static void io_uring_clean_tctx(struct io_uring_task *tctx)
de7f1d9e 8977{
13bf43f5 8978 struct io_tctx_node *node;
de7f1d9e
PB
8979 unsigned long index;
8980
13bf43f5 8981 xa_for_each(&tctx->xa, index, node)
2941267b 8982 io_uring_del_task_file(index);
8452d4a6
PB
8983 if (tctx->io_wq) {
8984 io_wq_put_and_exit(tctx->io_wq);
8985 tctx->io_wq = NULL;
8986 }
de7f1d9e
PB
8987}
8988
521d6a73
PB
8989static s64 tctx_inflight(struct io_uring_task *tctx)
8990{
8991 return percpu_counter_sum(&tctx->inflight);
8992}
8993
8994static void io_sqpoll_cancel_cb(struct callback_head *cb)
8995{
8996 struct io_tctx_exit *work = container_of(cb, struct io_tctx_exit, task_work);
8997 struct io_ring_ctx *ctx = work->ctx;
8998 struct io_sq_data *sqd = ctx->sq_data;
8999
9000 if (sqd->thread)
9001 io_uring_cancel_sqpoll(ctx);
9002 complete(&work->completion);
9003}
9004
9005static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx)
9006{
9007 struct io_sq_data *sqd = ctx->sq_data;
9008 struct io_tctx_exit work = { .ctx = ctx, };
9009 struct task_struct *task;
9010
9011 io_sq_thread_park(sqd);
9012 list_del_init(&ctx->sqd_list);
9013 io_sqd_update_thread_idle(sqd);
9014 task = sqd->thread;
9015 if (task) {
9016 init_completion(&work.completion);
9017 init_task_work(&work.task_work, io_sqpoll_cancel_cb);
b7f5a0bf 9018 io_task_work_add_head(&sqd->park_task_work, &work.task_work);
521d6a73
PB
9019 wake_up_process(task);
9020 }
9021 io_sq_thread_unpark(sqd);
9022
9023 if (task)
9024 wait_for_completion(&work.completion);
9025}
9026
0f212204
JA
9027void __io_uring_files_cancel(struct files_struct *files)
9028{
9029 struct io_uring_task *tctx = current->io_uring;
13bf43f5 9030 struct io_tctx_node *node;
ce765372 9031 unsigned long index;
0f212204
JA
9032
9033 /* make sure overflow events are dropped */
fdaf083c 9034 atomic_inc(&tctx->in_idle);
521d6a73
PB
9035 xa_for_each(&tctx->xa, index, node) {
9036 struct io_ring_ctx *ctx = node->ctx;
9037
9038 if (ctx->sq_data) {
9039 io_sqpoll_cancel_sync(ctx);
9040 continue;
9041 }
9042 io_uring_cancel_files(ctx, current, files);
9043 if (!files)
9044 io_uring_try_cancel_requests(ctx, current, NULL);
9045 }
fdaf083c 9046 atomic_dec(&tctx->in_idle);
de7f1d9e 9047
8452d4a6
PB
9048 if (files)
9049 io_uring_clean_tctx(tctx);
fdaf083c
JA
9050}
9051
521d6a73 9052/* should only be called by SQPOLL task */
0e9ddb39
PB
9053static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx)
9054{
37d1e2e3 9055 struct io_sq_data *sqd = ctx->sq_data;
521d6a73 9056 struct io_uring_task *tctx = current->io_uring;
0e9ddb39
PB
9057 s64 inflight;
9058 DEFINE_WAIT(wait);
fdaf083c 9059
521d6a73
PB
9060 WARN_ON_ONCE(!sqd || ctx->sq_data->thread != current);
9061
0e9ddb39
PB
9062 atomic_inc(&tctx->in_idle);
9063 do {
9064 /* read completions before cancelations */
9065 inflight = tctx_inflight(tctx);
9066 if (!inflight)
9067 break;
521d6a73 9068 io_uring_try_cancel_requests(ctx, current, NULL);
fdaf083c 9069
0e9ddb39
PB
9070 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9071 /*
9072 * If we've seen completions, retry without waiting. This
9073 * avoids a race where a completion comes in before we did
9074 * prepare_to_wait().
9075 */
9076 if (inflight == tctx_inflight(tctx))
9077 schedule();
9078 finish_wait(&tctx->wait, &wait);
9079 } while (1);
9080 atomic_dec(&tctx->in_idle);
0f212204
JA
9081}
9082
0f212204
JA
9083/*
9084 * Find any io_uring fd that this task has registered or done IO on, and cancel
9085 * requests.
9086 */
9087void __io_uring_task_cancel(void)
9088{
9089 struct io_uring_task *tctx = current->io_uring;
9090 DEFINE_WAIT(wait);
d8a6df10 9091 s64 inflight;
0f212204
JA
9092
9093 /* make sure overflow events are dropped */
fdaf083c 9094 atomic_inc(&tctx->in_idle);
5a978dcf
PB
9095 __io_uring_files_cancel(NULL);
9096
d8a6df10 9097 do {
0f212204 9098 /* read completions before cancelations */
fdaf083c 9099 inflight = tctx_inflight(tctx);
d8a6df10
JA
9100 if (!inflight)
9101 break;
0f212204
JA
9102 __io_uring_files_cancel(NULL);
9103
9104 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9105
9106 /*
a1bb3cd5
PB
9107 * If we've seen completions, retry without waiting. This
9108 * avoids a race where a completion comes in before we did
9109 * prepare_to_wait().
0f212204 9110 */
a1bb3cd5
PB
9111 if (inflight == tctx_inflight(tctx))
9112 schedule();
f57555ed 9113 finish_wait(&tctx->wait, &wait);
d8a6df10 9114 } while (1);
0f212204 9115
fdaf083c 9116 atomic_dec(&tctx->in_idle);
de7f1d9e 9117
8452d4a6
PB
9118 io_uring_clean_tctx(tctx);
9119 /* all current's requests should be gone, we can kill tctx */
9120 __io_uring_free(current);
44e728b8
PB
9121}
9122
6c5c240e
RP
9123static void *io_uring_validate_mmap_request(struct file *file,
9124 loff_t pgoff, size_t sz)
2b188cc1 9125{
2b188cc1 9126 struct io_ring_ctx *ctx = file->private_data;
6c5c240e 9127 loff_t offset = pgoff << PAGE_SHIFT;
2b188cc1
JA
9128 struct page *page;
9129 void *ptr;
9130
9131 switch (offset) {
9132 case IORING_OFF_SQ_RING:
75b28aff
HV
9133 case IORING_OFF_CQ_RING:
9134 ptr = ctx->rings;
2b188cc1
JA
9135 break;
9136 case IORING_OFF_SQES:
9137 ptr = ctx->sq_sqes;
9138 break;
2b188cc1 9139 default:
6c5c240e 9140 return ERR_PTR(-EINVAL);
2b188cc1
JA
9141 }
9142
9143 page = virt_to_head_page(ptr);
a50b854e 9144 if (sz > page_size(page))
6c5c240e
RP
9145 return ERR_PTR(-EINVAL);
9146
9147 return ptr;
9148}
9149
9150#ifdef CONFIG_MMU
9151
9152static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9153{
9154 size_t sz = vma->vm_end - vma->vm_start;
9155 unsigned long pfn;
9156 void *ptr;
9157
9158 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9159 if (IS_ERR(ptr))
9160 return PTR_ERR(ptr);
2b188cc1
JA
9161
9162 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9163 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9164}
9165
6c5c240e
RP
9166#else /* !CONFIG_MMU */
9167
9168static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9169{
9170 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9171}
9172
9173static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9174{
9175 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9176}
9177
9178static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9179 unsigned long addr, unsigned long len,
9180 unsigned long pgoff, unsigned long flags)
9181{
9182 void *ptr;
9183
9184 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9185 if (IS_ERR(ptr))
9186 return PTR_ERR(ptr);
9187
9188 return (unsigned long) ptr;
9189}
9190
9191#endif /* !CONFIG_MMU */
9192
d9d05217 9193static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
90554200
JA
9194{
9195 DEFINE_WAIT(wait);
9196
9197 do {
9198 if (!io_sqring_full(ctx))
9199 break;
90554200
JA
9200 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9201
9202 if (!io_sqring_full(ctx))
9203 break;
90554200
JA
9204 schedule();
9205 } while (!signal_pending(current));
9206
9207 finish_wait(&ctx->sqo_sq_wait, &wait);
5199328a 9208 return 0;
90554200
JA
9209}
9210
c73ebb68
HX
9211static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9212 struct __kernel_timespec __user **ts,
9213 const sigset_t __user **sig)
9214{
9215 struct io_uring_getevents_arg arg;
9216
9217 /*
9218 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9219 * is just a pointer to the sigset_t.
9220 */
9221 if (!(flags & IORING_ENTER_EXT_ARG)) {
9222 *sig = (const sigset_t __user *) argp;
9223 *ts = NULL;
9224 return 0;
9225 }
9226
9227 /*
9228 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9229 * timespec and sigset_t pointers if good.
9230 */
9231 if (*argsz != sizeof(arg))
9232 return -EINVAL;
9233 if (copy_from_user(&arg, argp, sizeof(arg)))
9234 return -EFAULT;
9235 *sig = u64_to_user_ptr(arg.sigmask);
9236 *argsz = arg.sigmask_sz;
9237 *ts = u64_to_user_ptr(arg.ts);
9238 return 0;
9239}
9240
2b188cc1 9241SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
c73ebb68
HX
9242 u32, min_complete, u32, flags, const void __user *, argp,
9243 size_t, argsz)
2b188cc1
JA
9244{
9245 struct io_ring_ctx *ctx;
2b188cc1
JA
9246 int submitted = 0;
9247 struct fd f;
33f993da 9248 long ret;
2b188cc1 9249
4c6e277c 9250 io_run_task_work();
b41e9852 9251
33f993da
PB
9252 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9253 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
2b188cc1
JA
9254 return -EINVAL;
9255
9256 f = fdget(fd);
33f993da 9257 if (unlikely(!f.file))
2b188cc1
JA
9258 return -EBADF;
9259
9260 ret = -EOPNOTSUPP;
33f993da 9261 if (unlikely(f.file->f_op != &io_uring_fops))
2b188cc1
JA
9262 goto out_fput;
9263
9264 ret = -ENXIO;
9265 ctx = f.file->private_data;
33f993da 9266 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
2b188cc1
JA
9267 goto out_fput;
9268
7e84e1c7 9269 ret = -EBADFD;
33f993da 9270 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
7e84e1c7
SG
9271 goto out;
9272
6c271ce2
JA
9273 /*
9274 * For SQ polling, the thread will do all submissions and completions.
9275 * Just return the requested submit count, and wake the thread if
9276 * we were asked to.
9277 */
b2a9eada 9278 ret = 0;
6c271ce2 9279 if (ctx->flags & IORING_SETUP_SQPOLL) {
6c2450ae 9280 io_cqring_overflow_flush(ctx, false);
89448c47 9281
d9d05217 9282 ret = -EOWNERDEAD;
04147488
SM
9283 if (unlikely(ctx->sq_data->thread == NULL)) {
9284 goto out;
9285 }
6c271ce2 9286 if (flags & IORING_ENTER_SQ_WAKEUP)
534ca6d6 9287 wake_up(&ctx->sq_data->wait);
d9d05217
PB
9288 if (flags & IORING_ENTER_SQ_WAIT) {
9289 ret = io_sqpoll_wait_sq(ctx);
9290 if (ret)
9291 goto out;
9292 }
6c271ce2 9293 submitted = to_submit;
b2a9eada 9294 } else if (to_submit) {
baf186c4 9295 ret = io_uring_add_task_file(ctx);
0f212204
JA
9296 if (unlikely(ret))
9297 goto out;
2b188cc1 9298 mutex_lock(&ctx->uring_lock);
0f212204 9299 submitted = io_submit_sqes(ctx, to_submit);
2b188cc1 9300 mutex_unlock(&ctx->uring_lock);
7c504e65
PB
9301
9302 if (submitted != to_submit)
9303 goto out;
2b188cc1
JA
9304 }
9305 if (flags & IORING_ENTER_GETEVENTS) {
c73ebb68
HX
9306 const sigset_t __user *sig;
9307 struct __kernel_timespec __user *ts;
9308
9309 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9310 if (unlikely(ret))
9311 goto out;
9312
2b188cc1
JA
9313 min_complete = min(min_complete, ctx->cq_entries);
9314
32b2244a
XW
9315 /*
9316 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9317 * space applications don't need to do io completion events
9318 * polling again, they can rely on io_sq_thread to do polling
9319 * work, which can reduce cpu usage and uring_lock contention.
9320 */
9321 if (ctx->flags & IORING_SETUP_IOPOLL &&
9322 !(ctx->flags & IORING_SETUP_SQPOLL)) {
7668b92a 9323 ret = io_iopoll_check(ctx, min_complete);
def596e9 9324 } else {
c73ebb68 9325 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
def596e9 9326 }
2b188cc1
JA
9327 }
9328
7c504e65 9329out:
6805b32e 9330 percpu_ref_put(&ctx->refs);
2b188cc1
JA
9331out_fput:
9332 fdput(f);
9333 return submitted ? submitted : ret;
9334}
9335
bebdb65e 9336#ifdef CONFIG_PROC_FS
61cf9370
MWO
9337static int io_uring_show_cred(struct seq_file *m, unsigned int id,
9338 const struct cred *cred)
87ce955b 9339{
87ce955b
JA
9340 struct user_namespace *uns = seq_user_ns(m);
9341 struct group_info *gi;
9342 kernel_cap_t cap;
9343 unsigned __capi;
9344 int g;
9345
9346 seq_printf(m, "%5d\n", id);
9347 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9348 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9349 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9350 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9351 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9352 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9353 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9354 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9355 seq_puts(m, "\n\tGroups:\t");
9356 gi = cred->group_info;
9357 for (g = 0; g < gi->ngroups; g++) {
9358 seq_put_decimal_ull(m, g ? " " : "",
9359 from_kgid_munged(uns, gi->gid[g]));
9360 }
9361 seq_puts(m, "\n\tCapEff:\t");
9362 cap = cred->cap_effective;
9363 CAP_FOR_EACH_U32(__capi)
9364 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9365 seq_putc(m, '\n');
9366 return 0;
9367}
9368
9369static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9370{
dbbe9c64 9371 struct io_sq_data *sq = NULL;
fad8e0de 9372 bool has_lock;
87ce955b
JA
9373 int i;
9374
fad8e0de
JA
9375 /*
9376 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9377 * since fdinfo case grabs it in the opposite direction of normal use
9378 * cases. If we fail to get the lock, we just don't iterate any
9379 * structures that could be going away outside the io_uring mutex.
9380 */
9381 has_lock = mutex_trylock(&ctx->uring_lock);
9382
5f3f26f9 9383 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
dbbe9c64 9384 sq = ctx->sq_data;
5f3f26f9
JA
9385 if (!sq->thread)
9386 sq = NULL;
9387 }
dbbe9c64
JQ
9388
9389 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9390 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
87ce955b 9391 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
fad8e0de 9392 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
7b29f92d 9393 struct file *f = io_file_from_index(ctx, i);
87ce955b 9394
87ce955b
JA
9395 if (f)
9396 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9397 else
9398 seq_printf(m, "%5u: <none>\n", i);
9399 }
9400 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
fad8e0de 9401 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
87ce955b
JA
9402 struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
9403
9404 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
9405 (unsigned int) buf->len);
9406 }
61cf9370
MWO
9407 if (has_lock && !xa_empty(&ctx->personalities)) {
9408 unsigned long index;
9409 const struct cred *cred;
9410
87ce955b 9411 seq_printf(m, "Personalities:\n");
61cf9370
MWO
9412 xa_for_each(&ctx->personalities, index, cred)
9413 io_uring_show_cred(m, index, cred);
87ce955b 9414 }
d7718a9d
JA
9415 seq_printf(m, "PollList:\n");
9416 spin_lock_irq(&ctx->completion_lock);
9417 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9418 struct hlist_head *list = &ctx->cancel_hash[i];
9419 struct io_kiocb *req;
9420
9421 hlist_for_each_entry(req, list, hash_node)
9422 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9423 req->task->task_works != NULL);
9424 }
9425 spin_unlock_irq(&ctx->completion_lock);
fad8e0de
JA
9426 if (has_lock)
9427 mutex_unlock(&ctx->uring_lock);
87ce955b
JA
9428}
9429
9430static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9431{
9432 struct io_ring_ctx *ctx = f->private_data;
9433
9434 if (percpu_ref_tryget(&ctx->refs)) {
9435 __io_uring_show_fdinfo(ctx, m);
9436 percpu_ref_put(&ctx->refs);
9437 }
9438}
bebdb65e 9439#endif
87ce955b 9440
2b188cc1
JA
9441static const struct file_operations io_uring_fops = {
9442 .release = io_uring_release,
9443 .mmap = io_uring_mmap,
6c5c240e
RP
9444#ifndef CONFIG_MMU
9445 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9446 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9447#endif
2b188cc1
JA
9448 .poll = io_uring_poll,
9449 .fasync = io_uring_fasync,
bebdb65e 9450#ifdef CONFIG_PROC_FS
87ce955b 9451 .show_fdinfo = io_uring_show_fdinfo,
bebdb65e 9452#endif
2b188cc1
JA
9453};
9454
9455static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9456 struct io_uring_params *p)
9457{
75b28aff
HV
9458 struct io_rings *rings;
9459 size_t size, sq_array_offset;
2b188cc1 9460
bd740481
JA
9461 /* make sure these are sane, as we already accounted them */
9462 ctx->sq_entries = p->sq_entries;
9463 ctx->cq_entries = p->cq_entries;
9464
75b28aff
HV
9465 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9466 if (size == SIZE_MAX)
9467 return -EOVERFLOW;
9468
9469 rings = io_mem_alloc(size);
9470 if (!rings)
2b188cc1
JA
9471 return -ENOMEM;
9472
75b28aff
HV
9473 ctx->rings = rings;
9474 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9475 rings->sq_ring_mask = p->sq_entries - 1;
9476 rings->cq_ring_mask = p->cq_entries - 1;
9477 rings->sq_ring_entries = p->sq_entries;
9478 rings->cq_ring_entries = p->cq_entries;
9479 ctx->sq_mask = rings->sq_ring_mask;
9480 ctx->cq_mask = rings->cq_ring_mask;
2b188cc1
JA
9481
9482 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
eb065d30
JA
9483 if (size == SIZE_MAX) {
9484 io_mem_free(ctx->rings);
9485 ctx->rings = NULL;
2b188cc1 9486 return -EOVERFLOW;
eb065d30 9487 }
2b188cc1
JA
9488
9489 ctx->sq_sqes = io_mem_alloc(size);
eb065d30
JA
9490 if (!ctx->sq_sqes) {
9491 io_mem_free(ctx->rings);
9492 ctx->rings = NULL;
2b188cc1 9493 return -ENOMEM;
eb065d30 9494 }
2b188cc1 9495
2b188cc1
JA
9496 return 0;
9497}
9498
9faadcc8
PB
9499static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9500{
9501 int ret, fd;
9502
9503 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9504 if (fd < 0)
9505 return fd;
9506
baf186c4 9507 ret = io_uring_add_task_file(ctx);
9faadcc8
PB
9508 if (ret) {
9509 put_unused_fd(fd);
9510 return ret;
9511 }
9512 fd_install(fd, file);
9513 return fd;
9514}
9515
2b188cc1
JA
9516/*
9517 * Allocate an anonymous fd, this is what constitutes the application
9518 * visible backing of an io_uring instance. The application mmaps this
9519 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9520 * we have to tie this fd to a socket for file garbage collection purposes.
9521 */
9faadcc8 9522static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
2b188cc1
JA
9523{
9524 struct file *file;
9faadcc8 9525#if defined(CONFIG_UNIX)
2b188cc1
JA
9526 int ret;
9527
2b188cc1
JA
9528 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9529 &ctx->ring_sock);
9530 if (ret)
9faadcc8 9531 return ERR_PTR(ret);
2b188cc1
JA
9532#endif
9533
2b188cc1
JA
9534 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9535 O_RDWR | O_CLOEXEC);
2b188cc1 9536#if defined(CONFIG_UNIX)
9faadcc8
PB
9537 if (IS_ERR(file)) {
9538 sock_release(ctx->ring_sock);
9539 ctx->ring_sock = NULL;
9540 } else {
9541 ctx->ring_sock->file = file;
0f212204 9542 }
2b188cc1 9543#endif
9faadcc8 9544 return file;
2b188cc1
JA
9545}
9546
7f13657d
XW
9547static int io_uring_create(unsigned entries, struct io_uring_params *p,
9548 struct io_uring_params __user *params)
2b188cc1 9549{
2b188cc1 9550 struct io_ring_ctx *ctx;
9faadcc8 9551 struct file *file;
2b188cc1
JA
9552 int ret;
9553
8110c1a6 9554 if (!entries)
2b188cc1 9555 return -EINVAL;
8110c1a6
JA
9556 if (entries > IORING_MAX_ENTRIES) {
9557 if (!(p->flags & IORING_SETUP_CLAMP))
9558 return -EINVAL;
9559 entries = IORING_MAX_ENTRIES;
9560 }
2b188cc1
JA
9561
9562 /*
9563 * Use twice as many entries for the CQ ring. It's possible for the
9564 * application to drive a higher depth than the size of the SQ ring,
9565 * since the sqes are only used at submission time. This allows for
33a107f0
JA
9566 * some flexibility in overcommitting a bit. If the application has
9567 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9568 * of CQ ring entries manually.
2b188cc1
JA
9569 */
9570 p->sq_entries = roundup_pow_of_two(entries);
33a107f0
JA
9571 if (p->flags & IORING_SETUP_CQSIZE) {
9572 /*
9573 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9574 * to a power-of-two, if it isn't already. We do NOT impose
9575 * any cq vs sq ring sizing.
9576 */
eb2667b3 9577 if (!p->cq_entries)
33a107f0 9578 return -EINVAL;
8110c1a6
JA
9579 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9580 if (!(p->flags & IORING_SETUP_CLAMP))
9581 return -EINVAL;
9582 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9583 }
eb2667b3
JQ
9584 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9585 if (p->cq_entries < p->sq_entries)
9586 return -EINVAL;
33a107f0
JA
9587 } else {
9588 p->cq_entries = 2 * p->sq_entries;
9589 }
2b188cc1 9590
2b188cc1 9591 ctx = io_ring_ctx_alloc(p);
62e398be 9592 if (!ctx)
2b188cc1 9593 return -ENOMEM;
2b188cc1 9594 ctx->compat = in_compat_syscall();
62e398be
JA
9595 if (!capable(CAP_IPC_LOCK))
9596 ctx->user = get_uid(current_user());
2aede0e4
JA
9597
9598 /*
9599 * This is just grabbed for accounting purposes. When a process exits,
9600 * the mm is exited and dropped before the files, hence we need to hang
9601 * on to this mm purely for the purposes of being able to unaccount
9602 * memory (locked/pinned vm). It's not used for anything else.
9603 */
6b7898eb 9604 mmgrab(current->mm);
2aede0e4 9605 ctx->mm_account = current->mm;
6b7898eb 9606
2b188cc1
JA
9607 ret = io_allocate_scq_urings(ctx, p);
9608 if (ret)
9609 goto err;
9610
7e84e1c7 9611 ret = io_sq_offload_create(ctx, p);
2b188cc1
JA
9612 if (ret)
9613 goto err;
9614
2b188cc1 9615 memset(&p->sq_off, 0, sizeof(p->sq_off));
75b28aff
HV
9616 p->sq_off.head = offsetof(struct io_rings, sq.head);
9617 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9618 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9619 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9620 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9621 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9622 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
2b188cc1
JA
9623
9624 memset(&p->cq_off, 0, sizeof(p->cq_off));
75b28aff
HV
9625 p->cq_off.head = offsetof(struct io_rings, cq.head);
9626 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9627 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9628 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9629 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9630 p->cq_off.cqes = offsetof(struct io_rings, cqes);
0d9b5b3a 9631 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
ac90f249 9632
7f13657d
XW
9633 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9634 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
5769a351 9635 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
c73ebb68 9636 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
1c0aa1fa 9637 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS;
7f13657d
XW
9638
9639 if (copy_to_user(params, p, sizeof(*p))) {
9640 ret = -EFAULT;
9641 goto err;
9642 }
d1719f70 9643
9faadcc8
PB
9644 file = io_uring_get_file(ctx);
9645 if (IS_ERR(file)) {
9646 ret = PTR_ERR(file);
9647 goto err;
9648 }
9649
044c1ab3
JA
9650 /*
9651 * Install ring fd as the very last thing, so we don't risk someone
9652 * having closed it before we finish setup
9653 */
9faadcc8
PB
9654 ret = io_uring_install_fd(ctx, file);
9655 if (ret < 0) {
9656 /* fput will clean it up */
9657 fput(file);
9658 return ret;
9659 }
044c1ab3 9660
c826bd7a 9661 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
2b188cc1
JA
9662 return ret;
9663err:
9664 io_ring_ctx_wait_and_kill(ctx);
9665 return ret;
9666}
9667
9668/*
9669 * Sets up an aio uring context, and returns the fd. Applications asks for a
9670 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9671 * params structure passed in.
9672 */
9673static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9674{
9675 struct io_uring_params p;
2b188cc1
JA
9676 int i;
9677
9678 if (copy_from_user(&p, params, sizeof(p)))
9679 return -EFAULT;
9680 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9681 if (p.resv[i])
9682 return -EINVAL;
9683 }
9684
6c271ce2 9685 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
8110c1a6 9686 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
7e84e1c7
SG
9687 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9688 IORING_SETUP_R_DISABLED))
2b188cc1
JA
9689 return -EINVAL;
9690
7f13657d 9691 return io_uring_create(entries, &p, params);
2b188cc1
JA
9692}
9693
9694SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9695 struct io_uring_params __user *, params)
9696{
9697 return io_uring_setup(entries, params);
9698}
9699
66f4af93
JA
9700static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9701{
9702 struct io_uring_probe *p;
9703 size_t size;
9704 int i, ret;
9705
9706 size = struct_size(p, ops, nr_args);
9707 if (size == SIZE_MAX)
9708 return -EOVERFLOW;
9709 p = kzalloc(size, GFP_KERNEL);
9710 if (!p)
9711 return -ENOMEM;
9712
9713 ret = -EFAULT;
9714 if (copy_from_user(p, arg, size))
9715 goto out;
9716 ret = -EINVAL;
9717 if (memchr_inv(p, 0, size))
9718 goto out;
9719
9720 p->last_op = IORING_OP_LAST - 1;
9721 if (nr_args > IORING_OP_LAST)
9722 nr_args = IORING_OP_LAST;
9723
9724 for (i = 0; i < nr_args; i++) {
9725 p->ops[i].op = i;
9726 if (!io_op_defs[i].not_supported)
9727 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9728 }
9729 p->ops_len = i;
9730
9731 ret = 0;
9732 if (copy_to_user(arg, p, size))
9733 ret = -EFAULT;
9734out:
9735 kfree(p);
9736 return ret;
9737}
9738
071698e1
JA
9739static int io_register_personality(struct io_ring_ctx *ctx)
9740{
4379bf8b 9741 const struct cred *creds;
61cf9370 9742 u32 id;
1e6fa521 9743 int ret;
071698e1 9744
4379bf8b 9745 creds = get_current_cred();
1e6fa521 9746
61cf9370
MWO
9747 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
9748 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
9749 if (!ret)
9750 return id;
9751 put_cred(creds);
1e6fa521 9752 return ret;
071698e1
JA
9753}
9754
21b55dbc
SG
9755static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9756 unsigned int nr_args)
9757{
9758 struct io_uring_restriction *res;
9759 size_t size;
9760 int i, ret;
9761
7e84e1c7
SG
9762 /* Restrictions allowed only if rings started disabled */
9763 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9764 return -EBADFD;
9765
21b55dbc 9766 /* We allow only a single restrictions registration */
7e84e1c7 9767 if (ctx->restrictions.registered)
21b55dbc
SG
9768 return -EBUSY;
9769
9770 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
9771 return -EINVAL;
9772
9773 size = array_size(nr_args, sizeof(*res));
9774 if (size == SIZE_MAX)
9775 return -EOVERFLOW;
9776
9777 res = memdup_user(arg, size);
9778 if (IS_ERR(res))
9779 return PTR_ERR(res);
9780
9781 ret = 0;
9782
9783 for (i = 0; i < nr_args; i++) {
9784 switch (res[i].opcode) {
9785 case IORING_RESTRICTION_REGISTER_OP:
9786 if (res[i].register_op >= IORING_REGISTER_LAST) {
9787 ret = -EINVAL;
9788 goto out;
9789 }
9790
9791 __set_bit(res[i].register_op,
9792 ctx->restrictions.register_op);
9793 break;
9794 case IORING_RESTRICTION_SQE_OP:
9795 if (res[i].sqe_op >= IORING_OP_LAST) {
9796 ret = -EINVAL;
9797 goto out;
9798 }
9799
9800 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
9801 break;
9802 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
9803 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
9804 break;
9805 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
9806 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
9807 break;
9808 default:
9809 ret = -EINVAL;
9810 goto out;
9811 }
9812 }
9813
9814out:
9815 /* Reset all restrictions if an error happened */
9816 if (ret != 0)
9817 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
9818 else
7e84e1c7 9819 ctx->restrictions.registered = true;
21b55dbc
SG
9820
9821 kfree(res);
9822 return ret;
9823}
9824
7e84e1c7
SG
9825static int io_register_enable_rings(struct io_ring_ctx *ctx)
9826{
9827 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9828 return -EBADFD;
9829
9830 if (ctx->restrictions.registered)
9831 ctx->restricted = 1;
9832
0298ef96
PB
9833 ctx->flags &= ~IORING_SETUP_R_DISABLED;
9834 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
9835 wake_up(&ctx->sq_data->wait);
7e84e1c7
SG
9836 return 0;
9837}
9838
071698e1
JA
9839static bool io_register_op_must_quiesce(int op)
9840{
9841 switch (op) {
9842 case IORING_UNREGISTER_FILES:
9843 case IORING_REGISTER_FILES_UPDATE:
9844 case IORING_REGISTER_PROBE:
9845 case IORING_REGISTER_PERSONALITY:
9846 case IORING_UNREGISTER_PERSONALITY:
9847 return false;
9848 default:
9849 return true;
9850 }
9851}
9852
edafccee
JA
9853static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
9854 void __user *arg, unsigned nr_args)
b19062a5
JA
9855 __releases(ctx->uring_lock)
9856 __acquires(ctx->uring_lock)
edafccee
JA
9857{
9858 int ret;
9859
35fa71a0
JA
9860 /*
9861 * We're inside the ring mutex, if the ref is already dying, then
9862 * someone else killed the ctx or is already going through
9863 * io_uring_register().
9864 */
9865 if (percpu_ref_is_dying(&ctx->refs))
9866 return -ENXIO;
9867
071698e1 9868 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 9869 percpu_ref_kill(&ctx->refs);
b19062a5 9870
05f3fb3c
JA
9871 /*
9872 * Drop uring mutex before waiting for references to exit. If
9873 * another thread is currently inside io_uring_enter() it might
9874 * need to grab the uring_lock to make progress. If we hold it
9875 * here across the drain wait, then we can deadlock. It's safe
9876 * to drop the mutex here, since no new references will come in
9877 * after we've killed the percpu ref.
9878 */
9879 mutex_unlock(&ctx->uring_lock);
af9c1a44
JA
9880 do {
9881 ret = wait_for_completion_interruptible(&ctx->ref_comp);
9882 if (!ret)
9883 break;
ed6930c9
JA
9884 ret = io_run_task_work_sig();
9885 if (ret < 0)
9886 break;
af9c1a44
JA
9887 } while (1);
9888
05f3fb3c 9889 mutex_lock(&ctx->uring_lock);
af9c1a44 9890
c150368b
JA
9891 if (ret) {
9892 percpu_ref_resurrect(&ctx->refs);
21b55dbc
SG
9893 goto out_quiesce;
9894 }
9895 }
9896
9897 if (ctx->restricted) {
9898 if (opcode >= IORING_REGISTER_LAST) {
9899 ret = -EINVAL;
9900 goto out;
9901 }
9902
9903 if (!test_bit(opcode, ctx->restrictions.register_op)) {
9904 ret = -EACCES;
c150368b
JA
9905 goto out;
9906 }
05f3fb3c 9907 }
edafccee
JA
9908
9909 switch (opcode) {
9910 case IORING_REGISTER_BUFFERS:
0a96bbe4 9911 ret = io_sqe_buffers_register(ctx, arg, nr_args);
edafccee
JA
9912 break;
9913 case IORING_UNREGISTER_BUFFERS:
9914 ret = -EINVAL;
9915 if (arg || nr_args)
9916 break;
0a96bbe4 9917 ret = io_sqe_buffers_unregister(ctx);
edafccee 9918 break;
6b06314c
JA
9919 case IORING_REGISTER_FILES:
9920 ret = io_sqe_files_register(ctx, arg, nr_args);
9921 break;
9922 case IORING_UNREGISTER_FILES:
9923 ret = -EINVAL;
9924 if (arg || nr_args)
9925 break;
9926 ret = io_sqe_files_unregister(ctx);
9927 break;
c3a31e60
JA
9928 case IORING_REGISTER_FILES_UPDATE:
9929 ret = io_sqe_files_update(ctx, arg, nr_args);
9930 break;
9b402849 9931 case IORING_REGISTER_EVENTFD:
f2842ab5 9932 case IORING_REGISTER_EVENTFD_ASYNC:
9b402849
JA
9933 ret = -EINVAL;
9934 if (nr_args != 1)
9935 break;
9936 ret = io_eventfd_register(ctx, arg);
f2842ab5
JA
9937 if (ret)
9938 break;
9939 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
9940 ctx->eventfd_async = 1;
9941 else
9942 ctx->eventfd_async = 0;
9b402849
JA
9943 break;
9944 case IORING_UNREGISTER_EVENTFD:
9945 ret = -EINVAL;
9946 if (arg || nr_args)
9947 break;
9948 ret = io_eventfd_unregister(ctx);
9949 break;
66f4af93
JA
9950 case IORING_REGISTER_PROBE:
9951 ret = -EINVAL;
9952 if (!arg || nr_args > 256)
9953 break;
9954 ret = io_probe(ctx, arg, nr_args);
9955 break;
071698e1
JA
9956 case IORING_REGISTER_PERSONALITY:
9957 ret = -EINVAL;
9958 if (arg || nr_args)
9959 break;
9960 ret = io_register_personality(ctx);
9961 break;
9962 case IORING_UNREGISTER_PERSONALITY:
9963 ret = -EINVAL;
9964 if (arg)
9965 break;
9966 ret = io_unregister_personality(ctx, nr_args);
9967 break;
7e84e1c7
SG
9968 case IORING_REGISTER_ENABLE_RINGS:
9969 ret = -EINVAL;
9970 if (arg || nr_args)
9971 break;
9972 ret = io_register_enable_rings(ctx);
9973 break;
21b55dbc
SG
9974 case IORING_REGISTER_RESTRICTIONS:
9975 ret = io_register_restrictions(ctx, arg, nr_args);
9976 break;
edafccee
JA
9977 default:
9978 ret = -EINVAL;
9979 break;
9980 }
9981
21b55dbc 9982out:
071698e1 9983 if (io_register_op_must_quiesce(opcode)) {
05f3fb3c 9984 /* bring the ctx back to life */
05f3fb3c 9985 percpu_ref_reinit(&ctx->refs);
21b55dbc 9986out_quiesce:
0f158b4c 9987 reinit_completion(&ctx->ref_comp);
05f3fb3c 9988 }
edafccee
JA
9989 return ret;
9990}
9991
9992SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
9993 void __user *, arg, unsigned int, nr_args)
9994{
9995 struct io_ring_ctx *ctx;
9996 long ret = -EBADF;
9997 struct fd f;
9998
9999 f = fdget(fd);
10000 if (!f.file)
10001 return -EBADF;
10002
10003 ret = -EOPNOTSUPP;
10004 if (f.file->f_op != &io_uring_fops)
10005 goto out_fput;
10006
10007 ctx = f.file->private_data;
10008
b6c23dd5
PB
10009 io_run_task_work();
10010
edafccee
JA
10011 mutex_lock(&ctx->uring_lock);
10012 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10013 mutex_unlock(&ctx->uring_lock);
c826bd7a
DD
10014 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10015 ctx->cq_ev_fd != NULL, ret);
edafccee
JA
10016out_fput:
10017 fdput(f);
10018 return ret;
10019}
10020
2b188cc1
JA
10021static int __init io_uring_init(void)
10022{
d7f62e82
SM
10023#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10024 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10025 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10026} while (0)
10027
10028#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10029 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10030 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10031 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10032 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10033 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10034 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10035 BUILD_BUG_SQE_ELEM(8, __u64, off);
10036 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10037 BUILD_BUG_SQE_ELEM(16, __u64, addr);
7d67af2c 10038 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
d7f62e82
SM
10039 BUILD_BUG_SQE_ELEM(24, __u32, len);
10040 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10041 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10042 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10043 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
5769a351
JX
10044 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10045 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
d7f62e82
SM
10046 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10047 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10048 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10049 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10050 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10051 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10052 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10053 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
7d67af2c 10054 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
d7f62e82
SM
10055 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10056 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
10057 BUILD_BUG_SQE_ELEM(42, __u16, personality);
7d67af2c 10058 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
d7f62e82 10059
d3656344 10060 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
84557871 10061 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
91f245d5
JA
10062 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
10063 SLAB_ACCOUNT);
2b188cc1
JA
10064 return 0;
10065};
10066__initcall(io_uring_init);