]>
git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - io_uring/rw.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
5 #include <linux/file.h>
6 #include <linux/blk-mq.h>
8 #include <linux/slab.h>
9 #include <linux/fsnotify.h>
10 #include <linux/poll.h>
11 #include <linux/nospec.h>
12 #include <linux/compat.h>
13 #include <linux/io_uring.h>
15 #include <uapi/linux/io_uring.h>
24 /* NOTE: kiocb has the file as the first member, so don't do it here */
31 static inline bool io_file_supports_nowait(struct io_kiocb
*req
)
33 return req
->flags
& REQ_F_SUPPORT_NOWAIT
;
37 static int io_iov_compat_buffer_select_prep(struct io_rw
*rw
)
39 struct compat_iovec __user
*uiov
;
42 uiov
= u64_to_user_ptr(rw
->addr
);
43 if (!access_ok(uiov
, sizeof(*uiov
)))
45 if (__get_user(clen
, &uiov
->iov_len
))
55 static int io_iov_buffer_select_prep(struct io_kiocb
*req
)
57 struct iovec __user
*uiov
;
59 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
66 return io_iov_compat_buffer_select_prep(rw
);
69 uiov
= u64_to_user_ptr(rw
->addr
);
70 if (copy_from_user(&iov
, uiov
, sizeof(*uiov
)))
72 rw
->len
= iov
.iov_len
;
76 int io_prep_rw(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
78 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
82 rw
->kiocb
.ki_pos
= READ_ONCE(sqe
->off
);
83 /* used for fixed read/write too - just read unconditionally */
84 req
->buf_index
= READ_ONCE(sqe
->buf_index
);
86 if (req
->opcode
== IORING_OP_READ_FIXED
||
87 req
->opcode
== IORING_OP_WRITE_FIXED
) {
88 struct io_ring_ctx
*ctx
= req
->ctx
;
91 if (unlikely(req
->buf_index
>= ctx
->nr_user_bufs
))
93 index
= array_index_nospec(req
->buf_index
, ctx
->nr_user_bufs
);
94 req
->imu
= ctx
->user_bufs
[index
];
95 io_req_set_rsrc_node(req
, ctx
, 0);
98 ioprio
= READ_ONCE(sqe
->ioprio
);
100 ret
= ioprio_check_cap(ioprio
);
104 rw
->kiocb
.ki_ioprio
= ioprio
;
106 rw
->kiocb
.ki_ioprio
= get_current_ioprio();
109 rw
->addr
= READ_ONCE(sqe
->addr
);
110 rw
->len
= READ_ONCE(sqe
->len
);
111 rw
->flags
= READ_ONCE(sqe
->rw_flags
);
113 /* Have to do this validation here, as this is in io_read() rw->len might
114 * have chanaged due to buffer selection
116 if (req
->opcode
== IORING_OP_READV
&& req
->flags
& REQ_F_BUFFER_SELECT
) {
117 ret
= io_iov_buffer_select_prep(req
);
125 void io_readv_writev_cleanup(struct io_kiocb
*req
)
127 struct io_async_rw
*io
= req
->async_data
;
129 kfree(io
->free_iovec
);
132 static inline void io_rw_done(struct kiocb
*kiocb
, ssize_t ret
)
138 case -ERESTARTNOINTR
:
139 case -ERESTARTNOHAND
:
140 case -ERESTART_RESTARTBLOCK
:
142 * We can't just restart the syscall, since previously
143 * submitted sqes may already be in progress. Just fail this
149 kiocb
->ki_complete(kiocb
, ret
);
153 static inline loff_t
*io_kiocb_update_pos(struct io_kiocb
*req
)
155 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
157 if (rw
->kiocb
.ki_pos
!= -1)
158 return &rw
->kiocb
.ki_pos
;
160 if (!(req
->file
->f_mode
& FMODE_STREAM
)) {
161 req
->flags
|= REQ_F_CUR_POS
;
162 rw
->kiocb
.ki_pos
= req
->file
->f_pos
;
163 return &rw
->kiocb
.ki_pos
;
166 rw
->kiocb
.ki_pos
= 0;
170 static void io_req_task_queue_reissue(struct io_kiocb
*req
)
172 req
->io_task_work
.func
= io_queue_iowq
;
173 io_req_task_work_add(req
);
177 static bool io_resubmit_prep(struct io_kiocb
*req
)
179 struct io_async_rw
*io
= req
->async_data
;
181 if (!req_has_async_data(req
))
182 return !io_req_prep_async(req
);
183 iov_iter_restore(&io
->s
.iter
, &io
->s
.iter_state
);
187 static bool io_rw_should_reissue(struct io_kiocb
*req
)
189 umode_t mode
= file_inode(req
->file
)->i_mode
;
190 struct io_ring_ctx
*ctx
= req
->ctx
;
192 if (!S_ISBLK(mode
) && !S_ISREG(mode
))
194 if ((req
->flags
& REQ_F_NOWAIT
) || (io_wq_current_is_worker() &&
195 !(ctx
->flags
& IORING_SETUP_IOPOLL
)))
198 * If ref is dying, we might be running poll reap from the exit work.
199 * Don't attempt to reissue from that path, just let it fail with
202 if (percpu_ref_is_dying(&ctx
->refs
))
205 * Play it safe and assume not safe to re-import and reissue if we're
206 * not in the original thread group (or in task context).
208 if (!same_thread_group(req
->task
, current
) || !in_task())
213 static bool io_resubmit_prep(struct io_kiocb
*req
)
217 static bool io_rw_should_reissue(struct io_kiocb
*req
)
223 static void kiocb_end_write(struct io_kiocb
*req
)
226 * Tell lockdep we inherited freeze protection from submission
229 if (req
->flags
& REQ_F_ISREG
) {
230 struct super_block
*sb
= file_inode(req
->file
)->i_sb
;
232 __sb_writers_acquired(sb
, SB_FREEZE_WRITE
);
238 * Trigger the notifications after having done some IO, and finish the write
239 * accounting, if any.
241 static void io_req_io_end(struct io_kiocb
*req
)
243 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
245 if (rw
->kiocb
.ki_flags
& IOCB_WRITE
) {
246 kiocb_end_write(req
);
247 fsnotify_modify(req
->file
);
249 fsnotify_access(req
->file
);
253 static bool __io_complete_rw_common(struct io_kiocb
*req
, long res
)
255 if (unlikely(res
!= req
->cqe
.res
)) {
256 if ((res
== -EAGAIN
|| res
== -EOPNOTSUPP
) &&
257 io_rw_should_reissue(req
)) {
259 * Reissue will start accounting again, finish the
263 req
->flags
|= REQ_F_REISSUE
| REQ_F_PARTIAL_IO
;
272 static inline int io_fixup_rw_res(struct io_kiocb
*req
, long res
)
274 struct io_async_rw
*io
= req
->async_data
;
276 /* add previously done IO, if any */
277 if (req_has_async_data(req
) && io
->bytes_done
> 0) {
279 res
= io
->bytes_done
;
281 res
+= io
->bytes_done
;
286 static void io_req_rw_complete(struct io_kiocb
*req
, bool *locked
)
290 if (req
->flags
& (REQ_F_BUFFER_SELECTED
|REQ_F_BUFFER_RING
)) {
291 unsigned issue_flags
= *locked
? 0 : IO_URING_F_UNLOCKED
;
293 req
->cqe
.flags
|= io_put_kbuf(req
, issue_flags
);
295 io_req_task_complete(req
, locked
);
298 static void io_complete_rw(struct kiocb
*kiocb
, long res
)
300 struct io_rw
*rw
= container_of(kiocb
, struct io_rw
, kiocb
);
301 struct io_kiocb
*req
= cmd_to_io_kiocb(rw
);
303 if (__io_complete_rw_common(req
, res
))
305 io_req_set_res(req
, io_fixup_rw_res(req
, res
), 0);
306 req
->io_task_work
.func
= io_req_rw_complete
;
307 io_req_task_work_add(req
);
310 static void io_complete_rw_iopoll(struct kiocb
*kiocb
, long res
)
312 struct io_rw
*rw
= container_of(kiocb
, struct io_rw
, kiocb
);
313 struct io_kiocb
*req
= cmd_to_io_kiocb(rw
);
315 if (kiocb
->ki_flags
& IOCB_WRITE
)
316 kiocb_end_write(req
);
317 if (unlikely(res
!= req
->cqe
.res
)) {
318 if (res
== -EAGAIN
&& io_rw_should_reissue(req
)) {
319 req
->flags
|= REQ_F_REISSUE
| REQ_F_PARTIAL_IO
;
325 /* order with io_iopoll_complete() checking ->iopoll_completed */
326 smp_store_release(&req
->iopoll_completed
, 1);
329 static int kiocb_done(struct io_kiocb
*req
, ssize_t ret
,
330 unsigned int issue_flags
)
332 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
333 unsigned final_ret
= io_fixup_rw_res(req
, ret
);
335 if (req
->flags
& REQ_F_CUR_POS
)
336 req
->file
->f_pos
= rw
->kiocb
.ki_pos
;
337 if (ret
>= 0 && (rw
->kiocb
.ki_complete
== io_complete_rw
)) {
338 if (!__io_complete_rw_common(req
, ret
)) {
340 * Safe to call io_end from here as we're inline
341 * from the submission path.
344 io_req_set_res(req
, final_ret
,
345 io_put_kbuf(req
, issue_flags
));
349 io_rw_done(&rw
->kiocb
, ret
);
352 if (req
->flags
& REQ_F_REISSUE
) {
353 req
->flags
&= ~REQ_F_REISSUE
;
354 if (io_resubmit_prep(req
))
355 io_req_task_queue_reissue(req
);
357 io_req_task_queue_fail(req
, final_ret
);
359 return IOU_ISSUE_SKIP_COMPLETE
;
362 static struct iovec
*__io_import_iovec(int ddir
, struct io_kiocb
*req
,
363 struct io_rw_state
*s
,
364 unsigned int issue_flags
)
366 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
367 struct iov_iter
*iter
= &s
->iter
;
368 u8 opcode
= req
->opcode
;
374 if (opcode
== IORING_OP_READ_FIXED
|| opcode
== IORING_OP_WRITE_FIXED
) {
375 ret
= io_import_fixed(ddir
, iter
, req
->imu
, rw
->addr
, rw
->len
);
381 buf
= u64_to_user_ptr(rw
->addr
);
384 if (opcode
== IORING_OP_READ
|| opcode
== IORING_OP_WRITE
||
385 (req
->flags
& REQ_F_BUFFER_SELECT
)) {
386 if (io_do_buffer_select(req
)) {
387 buf
= io_buffer_select(req
, &sqe_len
, issue_flags
);
389 return ERR_PTR(-ENOBUFS
);
390 rw
->addr
= (unsigned long) buf
;
394 ret
= import_single_range(ddir
, buf
, sqe_len
, s
->fast_iov
, iter
);
401 ret
= __import_iovec(ddir
, buf
, sqe_len
, UIO_FASTIOV
, &iovec
, iter
,
403 if (unlikely(ret
< 0))
408 static inline int io_import_iovec(int rw
, struct io_kiocb
*req
,
409 struct iovec
**iovec
, struct io_rw_state
*s
,
410 unsigned int issue_flags
)
412 *iovec
= __io_import_iovec(rw
, req
, s
, issue_flags
);
413 if (unlikely(IS_ERR(*iovec
)))
414 return PTR_ERR(*iovec
);
416 iov_iter_save_state(&s
->iter
, &s
->iter_state
);
420 static inline loff_t
*io_kiocb_ppos(struct kiocb
*kiocb
)
422 return (kiocb
->ki_filp
->f_mode
& FMODE_STREAM
) ? NULL
: &kiocb
->ki_pos
;
426 * For files that don't have ->read_iter() and ->write_iter(), handle them
427 * by looping over ->read() or ->write() manually.
429 static ssize_t
loop_rw_iter(int ddir
, struct io_rw
*rw
, struct iov_iter
*iter
)
431 struct kiocb
*kiocb
= &rw
->kiocb
;
432 struct file
*file
= kiocb
->ki_filp
;
437 * Don't support polled IO through this interface, and we can't
438 * support non-blocking either. For the latter, this just causes
439 * the kiocb to be handled from an async context.
441 if (kiocb
->ki_flags
& IOCB_HIPRI
)
443 if ((kiocb
->ki_flags
& IOCB_NOWAIT
) &&
444 !(kiocb
->ki_filp
->f_flags
& O_NONBLOCK
))
447 ppos
= io_kiocb_ppos(kiocb
);
449 while (iov_iter_count(iter
)) {
453 if (!iov_iter_is_bvec(iter
)) {
454 iovec
= iov_iter_iovec(iter
);
456 iovec
.iov_base
= u64_to_user_ptr(rw
->addr
);
457 iovec
.iov_len
= rw
->len
;
461 nr
= file
->f_op
->read(file
, iovec
.iov_base
,
462 iovec
.iov_len
, ppos
);
464 nr
= file
->f_op
->write(file
, iovec
.iov_base
,
465 iovec
.iov_len
, ppos
);
474 if (!iov_iter_is_bvec(iter
)) {
475 iov_iter_advance(iter
, nr
);
482 if (nr
!= iovec
.iov_len
)
489 static void io_req_map_rw(struct io_kiocb
*req
, const struct iovec
*iovec
,
490 const struct iovec
*fast_iov
, struct iov_iter
*iter
)
492 struct io_async_rw
*io
= req
->async_data
;
494 memcpy(&io
->s
.iter
, iter
, sizeof(*iter
));
495 io
->free_iovec
= iovec
;
497 /* can only be fixed buffers, no need to do anything */
498 if (iov_iter_is_bvec(iter
))
501 unsigned iov_off
= 0;
503 io
->s
.iter
.iov
= io
->s
.fast_iov
;
504 if (iter
->iov
!= fast_iov
) {
505 iov_off
= iter
->iov
- fast_iov
;
506 io
->s
.iter
.iov
+= iov_off
;
508 if (io
->s
.fast_iov
!= fast_iov
)
509 memcpy(io
->s
.fast_iov
+ iov_off
, fast_iov
+ iov_off
,
510 sizeof(struct iovec
) * iter
->nr_segs
);
512 req
->flags
|= REQ_F_NEED_CLEANUP
;
516 static int io_setup_async_rw(struct io_kiocb
*req
, const struct iovec
*iovec
,
517 struct io_rw_state
*s
, bool force
)
519 if (!force
&& !io_op_defs
[req
->opcode
].prep_async
)
521 if (!req_has_async_data(req
)) {
522 struct io_async_rw
*iorw
;
524 if (io_alloc_async_data(req
)) {
529 io_req_map_rw(req
, iovec
, s
->fast_iov
, &s
->iter
);
530 iorw
= req
->async_data
;
531 /* we've copied and mapped the iter, ensure state is saved */
532 iov_iter_save_state(&iorw
->s
.iter
, &iorw
->s
.iter_state
);
537 static inline int io_rw_prep_async(struct io_kiocb
*req
, int rw
)
539 struct io_async_rw
*iorw
= req
->async_data
;
543 /* submission path, ->uring_lock should already be taken */
544 ret
= io_import_iovec(rw
, req
, &iov
, &iorw
->s
, 0);
545 if (unlikely(ret
< 0))
548 iorw
->bytes_done
= 0;
549 iorw
->free_iovec
= iov
;
551 req
->flags
|= REQ_F_NEED_CLEANUP
;
555 int io_readv_prep_async(struct io_kiocb
*req
)
557 return io_rw_prep_async(req
, ITER_DEST
);
560 int io_writev_prep_async(struct io_kiocb
*req
)
562 return io_rw_prep_async(req
, ITER_SOURCE
);
566 * This is our waitqueue callback handler, registered through __folio_lock_async()
567 * when we initially tried to do the IO with the iocb armed our waitqueue.
568 * This gets called when the page is unlocked, and we generally expect that to
569 * happen when the page IO is completed and the page is now uptodate. This will
570 * queue a task_work based retry of the operation, attempting to copy the data
571 * again. If the latter fails because the page was NOT uptodate, then we will
572 * do a thread based blocking retry of the operation. That's the unexpected
575 static int io_async_buf_func(struct wait_queue_entry
*wait
, unsigned mode
,
578 struct wait_page_queue
*wpq
;
579 struct io_kiocb
*req
= wait
->private;
580 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
581 struct wait_page_key
*key
= arg
;
583 wpq
= container_of(wait
, struct wait_page_queue
, wait
);
585 if (!wake_page_match(wpq
, key
))
588 rw
->kiocb
.ki_flags
&= ~IOCB_WAITQ
;
589 list_del_init(&wait
->entry
);
590 io_req_task_queue(req
);
595 * This controls whether a given IO request should be armed for async page
596 * based retry. If we return false here, the request is handed to the async
597 * worker threads for retry. If we're doing buffered reads on a regular file,
598 * we prepare a private wait_page_queue entry and retry the operation. This
599 * will either succeed because the page is now uptodate and unlocked, or it
600 * will register a callback when the page is unlocked at IO completion. Through
601 * that callback, io_uring uses task_work to setup a retry of the operation.
602 * That retry will attempt the buffered read again. The retry will generally
603 * succeed, or in rare cases where it fails, we then fall back to using the
604 * async worker threads for a blocking retry.
606 static bool io_rw_should_retry(struct io_kiocb
*req
)
608 struct io_async_rw
*io
= req
->async_data
;
609 struct wait_page_queue
*wait
= &io
->wpq
;
610 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
611 struct kiocb
*kiocb
= &rw
->kiocb
;
613 /* never retry for NOWAIT, we just complete with -EAGAIN */
614 if (req
->flags
& REQ_F_NOWAIT
)
617 /* Only for buffered IO */
618 if (kiocb
->ki_flags
& (IOCB_DIRECT
| IOCB_HIPRI
))
622 * just use poll if we can, and don't attempt if the fs doesn't
623 * support callback based unlocks
625 if (file_can_poll(req
->file
) || !(req
->file
->f_mode
& FMODE_BUF_RASYNC
))
628 wait
->wait
.func
= io_async_buf_func
;
629 wait
->wait
.private = req
;
630 wait
->wait
.flags
= 0;
631 INIT_LIST_HEAD(&wait
->wait
.entry
);
632 kiocb
->ki_flags
|= IOCB_WAITQ
;
633 kiocb
->ki_flags
&= ~IOCB_NOWAIT
;
634 kiocb
->ki_waitq
= wait
;
638 static inline int io_iter_do_read(struct io_rw
*rw
, struct iov_iter
*iter
)
640 struct file
*file
= rw
->kiocb
.ki_filp
;
642 if (likely(file
->f_op
->read_iter
))
643 return call_read_iter(file
, &rw
->kiocb
, iter
);
644 else if (file
->f_op
->read
)
645 return loop_rw_iter(READ
, rw
, iter
);
650 static bool need_complete_io(struct io_kiocb
*req
)
652 return req
->flags
& REQ_F_ISREG
||
653 S_ISBLK(file_inode(req
->file
)->i_mode
);
656 static int io_rw_init_file(struct io_kiocb
*req
, fmode_t mode
)
658 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
659 struct kiocb
*kiocb
= &rw
->kiocb
;
660 struct io_ring_ctx
*ctx
= req
->ctx
;
661 struct file
*file
= req
->file
;
664 if (unlikely(!file
|| !(file
->f_mode
& mode
)))
667 if (!io_req_ffs_set(req
))
668 req
->flags
|= io_file_get_flags(file
) << REQ_F_SUPPORT_NOWAIT_BIT
;
670 kiocb
->ki_flags
= file
->f_iocb_flags
;
671 ret
= kiocb_set_rw_flags(kiocb
, rw
->flags
);
674 kiocb
->ki_flags
|= IOCB_ALLOC_CACHE
;
677 * If the file is marked O_NONBLOCK, still allow retry for it if it
678 * supports async. Otherwise it's impossible to use O_NONBLOCK files
679 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
681 if ((kiocb
->ki_flags
& IOCB_NOWAIT
) ||
682 ((file
->f_flags
& O_NONBLOCK
) && !io_file_supports_nowait(req
)))
683 req
->flags
|= REQ_F_NOWAIT
;
685 if (ctx
->flags
& IORING_SETUP_IOPOLL
) {
686 if (!(kiocb
->ki_flags
& IOCB_DIRECT
) || !file
->f_op
->iopoll
)
689 kiocb
->private = NULL
;
690 kiocb
->ki_flags
|= IOCB_HIPRI
;
691 kiocb
->ki_complete
= io_complete_rw_iopoll
;
692 req
->iopoll_completed
= 0;
694 if (kiocb
->ki_flags
& IOCB_HIPRI
)
696 kiocb
->ki_complete
= io_complete_rw
;
702 int io_read(struct io_kiocb
*req
, unsigned int issue_flags
)
704 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
705 struct io_rw_state __s
, *s
= &__s
;
707 struct kiocb
*kiocb
= &rw
->kiocb
;
708 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
709 struct io_async_rw
*io
;
713 if (!req_has_async_data(req
)) {
714 ret
= io_import_iovec(ITER_DEST
, req
, &iovec
, s
, issue_flags
);
715 if (unlikely(ret
< 0))
718 io
= req
->async_data
;
722 * Safe and required to re-import if we're using provided
723 * buffers, as we dropped the selected one before retry.
725 if (io_do_buffer_select(req
)) {
726 ret
= io_import_iovec(ITER_DEST
, req
, &iovec
, s
, issue_flags
);
727 if (unlikely(ret
< 0))
732 * We come here from an earlier attempt, restore our state to
733 * match in case it doesn't. It's cheap enough that we don't
734 * need to make this conditional.
736 iov_iter_restore(&s
->iter
, &s
->iter_state
);
739 ret
= io_rw_init_file(req
, FMODE_READ
);
744 req
->cqe
.res
= iov_iter_count(&s
->iter
);
746 if (force_nonblock
) {
747 /* If the file doesn't support async, just async punt */
748 if (unlikely(!io_file_supports_nowait(req
))) {
749 ret
= io_setup_async_rw(req
, iovec
, s
, true);
750 return ret
?: -EAGAIN
;
752 kiocb
->ki_flags
|= IOCB_NOWAIT
;
754 /* Ensure we clear previously set non-block flag */
755 kiocb
->ki_flags
&= ~IOCB_NOWAIT
;
758 ppos
= io_kiocb_update_pos(req
);
760 ret
= rw_verify_area(READ
, req
->file
, ppos
, req
->cqe
.res
);
766 ret
= io_iter_do_read(rw
, &s
->iter
);
768 if (ret
== -EAGAIN
|| (req
->flags
& REQ_F_REISSUE
)) {
769 req
->flags
&= ~REQ_F_REISSUE
;
770 /* if we can poll, just do that */
771 if (req
->opcode
== IORING_OP_READ
&& file_can_poll(req
->file
))
773 /* IOPOLL retry should happen for io-wq threads */
774 if (!force_nonblock
&& !(req
->ctx
->flags
& IORING_SETUP_IOPOLL
))
776 /* no retry on NONBLOCK nor RWF_NOWAIT */
777 if (req
->flags
& REQ_F_NOWAIT
)
780 } else if (ret
== -EIOCBQUEUED
) {
783 return IOU_ISSUE_SKIP_COMPLETE
;
784 } else if (ret
== req
->cqe
.res
|| ret
<= 0 || !force_nonblock
||
785 (req
->flags
& REQ_F_NOWAIT
) || !need_complete_io(req
)) {
786 /* read all, failed, already did sync or don't want to retry */
791 * Don't depend on the iter state matching what was consumed, or being
792 * untouched in case of error. Restore it and we'll advance it
793 * manually if we need to.
795 iov_iter_restore(&s
->iter
, &s
->iter_state
);
797 ret2
= io_setup_async_rw(req
, iovec
, s
, true);
800 ret
= ret
> 0 ? ret
: ret2
;
804 io
= req
->async_data
;
807 * Now use our persistent iterator and state, if we aren't already.
808 * We've restored and mapped the iter to match.
813 * We end up here because of a partial read, either from
814 * above or inside this loop. Advance the iter by the bytes
815 * that were consumed.
817 iov_iter_advance(&s
->iter
, ret
);
818 if (!iov_iter_count(&s
->iter
))
820 io
->bytes_done
+= ret
;
821 iov_iter_save_state(&s
->iter
, &s
->iter_state
);
823 /* if we can retry, do so with the callbacks armed */
824 if (!io_rw_should_retry(req
)) {
825 kiocb
->ki_flags
&= ~IOCB_WAITQ
;
829 req
->cqe
.res
= iov_iter_count(&s
->iter
);
831 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
832 * we get -EIOCBQUEUED, then we'll get a notification when the
833 * desired page gets unlocked. We can also get a partial read
834 * here, and if we do, then just retry at the new offset.
836 ret
= io_iter_do_read(rw
, &s
->iter
);
837 if (ret
== -EIOCBQUEUED
)
838 return IOU_ISSUE_SKIP_COMPLETE
;
839 /* we got some bytes, but not all. retry. */
840 kiocb
->ki_flags
&= ~IOCB_WAITQ
;
841 iov_iter_restore(&s
->iter
, &s
->iter_state
);
844 /* it's faster to check here then delegate to kfree */
847 return kiocb_done(req
, ret
, issue_flags
);
850 int io_write(struct io_kiocb
*req
, unsigned int issue_flags
)
852 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
853 struct io_rw_state __s
, *s
= &__s
;
855 struct kiocb
*kiocb
= &rw
->kiocb
;
856 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
860 if (!req_has_async_data(req
)) {
861 ret
= io_import_iovec(ITER_SOURCE
, req
, &iovec
, s
, issue_flags
);
862 if (unlikely(ret
< 0))
865 struct io_async_rw
*io
= req
->async_data
;
868 iov_iter_restore(&s
->iter
, &s
->iter_state
);
871 ret
= io_rw_init_file(req
, FMODE_WRITE
);
876 req
->cqe
.res
= iov_iter_count(&s
->iter
);
878 if (force_nonblock
) {
879 /* If the file doesn't support async, just async punt */
880 if (unlikely(!io_file_supports_nowait(req
)))
883 /* File path supports NOWAIT for non-direct_IO only for block devices. */
884 if (!(kiocb
->ki_flags
& IOCB_DIRECT
) &&
885 !(kiocb
->ki_filp
->f_mode
& FMODE_BUF_WASYNC
) &&
886 (req
->flags
& REQ_F_ISREG
))
889 kiocb
->ki_flags
|= IOCB_NOWAIT
;
891 /* Ensure we clear previously set non-block flag */
892 kiocb
->ki_flags
&= ~IOCB_NOWAIT
;
895 ppos
= io_kiocb_update_pos(req
);
897 ret
= rw_verify_area(WRITE
, req
->file
, ppos
, req
->cqe
.res
);
904 * Open-code file_start_write here to grab freeze protection,
905 * which will be released by another thread in
906 * io_complete_rw(). Fool lockdep by telling it the lock got
907 * released so that it doesn't complain about the held lock when
908 * we return to userspace.
910 if (req
->flags
& REQ_F_ISREG
) {
911 sb_start_write(file_inode(req
->file
)->i_sb
);
912 __sb_writers_release(file_inode(req
->file
)->i_sb
,
915 kiocb
->ki_flags
|= IOCB_WRITE
;
917 if (likely(req
->file
->f_op
->write_iter
))
918 ret2
= call_write_iter(req
->file
, kiocb
, &s
->iter
);
919 else if (req
->file
->f_op
->write
)
920 ret2
= loop_rw_iter(WRITE
, rw
, &s
->iter
);
924 if (req
->flags
& REQ_F_REISSUE
) {
925 req
->flags
&= ~REQ_F_REISSUE
;
930 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
931 * retry them without IOCB_NOWAIT.
933 if (ret2
== -EOPNOTSUPP
&& (kiocb
->ki_flags
& IOCB_NOWAIT
))
935 /* no retry on NONBLOCK nor RWF_NOWAIT */
936 if (ret2
== -EAGAIN
&& (req
->flags
& REQ_F_NOWAIT
))
938 if (!force_nonblock
|| ret2
!= -EAGAIN
) {
939 /* IOPOLL retry should happen for io-wq threads */
940 if (ret2
== -EAGAIN
&& (req
->ctx
->flags
& IORING_SETUP_IOPOLL
))
943 if (ret2
!= req
->cqe
.res
&& ret2
>= 0 && need_complete_io(req
)) {
944 struct io_async_rw
*io
;
946 trace_io_uring_short_write(req
->ctx
, kiocb
->ki_pos
- ret2
,
949 /* This is a partial write. The file pos has already been
950 * updated, setup the async struct to complete the request
951 * in the worker. Also update bytes_done to account for
952 * the bytes already written.
954 iov_iter_save_state(&s
->iter
, &s
->iter_state
);
955 ret
= io_setup_async_rw(req
, iovec
, s
, true);
957 io
= req
->async_data
;
959 io
->bytes_done
+= ret2
;
961 if (kiocb
->ki_flags
& IOCB_WRITE
)
962 kiocb_end_write(req
);
963 return ret
? ret
: -EAGAIN
;
966 ret
= kiocb_done(req
, ret2
, issue_flags
);
969 iov_iter_restore(&s
->iter
, &s
->iter_state
);
970 ret
= io_setup_async_rw(req
, iovec
, s
, false);
972 if (kiocb
->ki_flags
& IOCB_WRITE
)
973 kiocb_end_write(req
);
978 /* it's reportedly faster than delegating the null check to kfree() */
984 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx
*ctx
)
986 io_commit_cqring_flush(ctx
);
987 if (ctx
->flags
& IORING_SETUP_SQPOLL
)
991 void io_rw_fail(struct io_kiocb
*req
)
995 res
= io_fixup_rw_res(req
, req
->cqe
.res
);
996 io_req_set_res(req
, res
, req
->cqe
.flags
);
999 int io_do_iopoll(struct io_ring_ctx
*ctx
, bool force_nonspin
)
1001 struct io_wq_work_node
*pos
, *start
, *prev
;
1002 unsigned int poll_flags
= BLK_POLL_NOSLEEP
;
1003 DEFINE_IO_COMP_BATCH(iob
);
1007 * Only spin for completions if we don't have multiple devices hanging
1008 * off our complete list.
1010 if (ctx
->poll_multi_queue
|| force_nonspin
)
1011 poll_flags
|= BLK_POLL_ONESHOT
;
1013 wq_list_for_each(pos
, start
, &ctx
->iopoll_list
) {
1014 struct io_kiocb
*req
= container_of(pos
, struct io_kiocb
, comp_list
);
1015 struct file
*file
= req
->file
;
1019 * Move completed and retryable entries to our local lists.
1020 * If we find a request that requires polling, break out
1021 * and complete those lists first, if we have entries there.
1023 if (READ_ONCE(req
->iopoll_completed
))
1026 if (req
->opcode
== IORING_OP_URING_CMD
) {
1027 struct io_uring_cmd
*ioucmd
;
1029 ioucmd
= io_kiocb_to_cmd(req
, struct io_uring_cmd
);
1030 ret
= file
->f_op
->uring_cmd_iopoll(ioucmd
, &iob
,
1033 struct io_rw
*rw
= io_kiocb_to_cmd(req
, struct io_rw
);
1035 ret
= file
->f_op
->iopoll(&rw
->kiocb
, &iob
, poll_flags
);
1037 if (unlikely(ret
< 0))
1040 poll_flags
|= BLK_POLL_ONESHOT
;
1042 /* iopoll may have completed current req */
1043 if (!rq_list_empty(iob
.req_list
) ||
1044 READ_ONCE(req
->iopoll_completed
))
1048 if (!rq_list_empty(iob
.req_list
))
1054 wq_list_for_each_resume(pos
, prev
) {
1055 struct io_kiocb
*req
= container_of(pos
, struct io_kiocb
, comp_list
);
1057 /* order with io_complete_rw_iopoll(), e.g. ->result updates */
1058 if (!smp_load_acquire(&req
->iopoll_completed
))
1061 if (unlikely(req
->flags
& REQ_F_CQE_SKIP
))
1064 req
->cqe
.flags
= io_put_kbuf(req
, 0);
1065 if (unlikely(!__io_fill_cqe_req(ctx
, req
))) {
1066 spin_lock(&ctx
->completion_lock
);
1067 io_req_cqe_overflow(req
);
1068 spin_unlock(&ctx
->completion_lock
);
1072 if (unlikely(!nr_events
))
1075 io_commit_cqring(ctx
);
1076 io_cqring_ev_posted_iopoll(ctx
);
1077 pos
= start
? start
->next
: ctx
->iopoll_list
.first
;
1078 wq_list_cut(&ctx
->iopoll_list
, prev
, start
);
1079 io_free_batch_list(ctx
, pos
);