io_uring: add IOSQE_ASYNC

io_uring defaults to always doing inline submissions, if at all
possible. But for larger copies, even if the data is fully cached, that
can take a long time. Add an IOSQE_ASYNC flag that the application can
set on the SQE - if set, it'll ensure that we always go async for those
kinds of requests. Use the io-wq IO_WQ_WORK_CONCURRENT flag to ensure we
get the concurrency we desire for this case.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 115c7d41337289004a3a6c25aa83a5537903f2ed..9ffcfdc6382b5e732ba5ec1ca8ce1db8b17ee4e3 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -483,6 +483,7 @@ struct io_kiocb {
 #define REQ_F_INFLIGHT         16384   /* on inflight list */
 #define REQ_F_COMP_LOCKED      32768   /* completion under lock */
 #define REQ_F_HARDLINK         65536   /* doesn't sever on completion < 0 */
+#define REQ_F_FORCE_ASYNC      131072  /* IOSQE_ASYNC */
        u64                     user_data;
        u32                     result;
        u32                     sequence;
@@ -4017,8 +4018,17 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
                        req_set_fail_links(req);
                        io_double_put_req(req);
                }
-       } else
+       } else if ((req->flags & REQ_F_FORCE_ASYNC) &&
+                  !io_wq_current_is_worker()) {
+               /*
+                * Never try inline submit of IOSQE_ASYNC is set, go straight
+                * to async execution.
+                */
+               req->work.flags |= IO_WQ_WORK_CONCURRENT;
+               io_queue_async_work(req);
+       } else {
                __io_queue_sqe(req, sqe);
+       }
 }
 
 static inline void io_queue_link_head(struct io_kiocb *req)
@@ -4031,7 +4041,7 @@ static inline void io_queue_link_head(struct io_kiocb *req)
 }
 
 #define SQE_VALID_FLAGS        (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
-                               IOSQE_IO_HARDLINK)
+                               IOSQE_IO_HARDLINK | IOSQE_ASYNC)
 
 static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
                          struct io_submit_state *state, struct io_kiocb **link)
@@ -4044,6 +4054,8 @@ static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
                ret = -EINVAL;
                goto err_req;
        }
+       if (sqe->flags & IOSQE_ASYNC)
+               req->flags |= REQ_F_FORCE_ASYNC;
 
        ret = io_req_set_file(state, req, sqe);
        if (unlikely(ret)) {

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 3f45f7c543de622e5eb5812308a4b77b55945120..d7ec50247a3a9fedc7ffb830e3441547d7612e5a 100644 (file)
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -51,6 +51,7 @@ struct io_uring_sqe {
 #define IOSQE_IO_DRAIN         (1U << 1)       /* issue after inflight IO */
 #define IOSQE_IO_LINK          (1U << 2)       /* links next sqe */
 #define IOSQE_IO_HARDLINK      (1U << 3)       /* like LINK, but stronger */
+#define IOSQE_ASYNC            (1U << 4)       /* always go async */
 
 /*
  * io_uring_setup() flags