io_uring: close lookup gap for dependent next work

When we find new work to process within the work handler, we queue the
linked timeout before we have issued the new work. This can be
problematic for very short timeouts, as we have a window where the new
work isn't visible.

Allow the work handler to store a callback function for this in the work
item, and flag it with IO_WQ_WORK_CB if the caller has done so. If that
is set, then io-wq will call the callback when it has setup the new work
item.

Reported-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index b4bc377dda619e6fbffffb772dcc83b883f8b63f..9b32b3c811f5a3a5b4236a8fc0febbf13ac6aede 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -427,6 +427,9 @@ next:
                worker->cur_work = work;
                spin_unlock_irq(&worker->lock);
 
+               if (work->flags & IO_WQ_WORK_CB)
+                       work->func(&work);
+
                if ((work->flags & IO_WQ_WORK_NEEDS_FILES) &&
                    current->files != work->files) {
                        task_lock(current);

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 4b29f922f80cfc2b916374eedf788a8526b93861..b68b11bf363318ba2be8552c1c227bd8e8ad38e7 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -11,6 +11,7 @@ enum {
        IO_WQ_WORK_NEEDS_FILES  = 16,
        IO_WQ_WORK_UNBOUND      = 32,
        IO_WQ_WORK_INTERNAL     = 64,
+       IO_WQ_WORK_CB           = 128,
 
        IO_WQ_HASH_SHIFT        = 24,   /* upper 8 bits are used for hash key */
 };
@@ -22,7 +23,10 @@ enum io_wq_cancel {
 };
 
 struct io_wq_work {
-       struct list_head list;
+       union {
+               struct list_head list;
+               void *data;
+       };
        void (*func)(struct io_wq_work **);
        unsigned flags;
        struct files_struct *files;

diff --git a/fs/io_uring.c b/fs/io_uring.c
index e9980c584120f60012505e59fedc911cc186dead..27fefb52910c7f3262fefc710d26b6b05d146277 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2679,6 +2679,15 @@ static int __io_submit_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
        return 0;
 }
 
+static void io_link_work_cb(struct io_wq_work **workptr)
+{
+       struct io_wq_work *work = *workptr;
+       struct io_kiocb *link = work->data;
+
+       io_queue_linked_timeout(link);
+       work->func = io_wq_submit_work;
+}
+
 static void io_wq_submit_work(struct io_wq_work **workptr)
 {
        struct io_wq_work *work = *workptr;
@@ -2725,8 +2734,11 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
 
                io_prep_async_work(nxt, &link);
                *workptr = &nxt->work;
-               if (link)
-                       io_queue_linked_timeout(link);
+               if (link) {
+                       nxt->work.flags |= IO_WQ_WORK_CB;
+                       nxt->work.func = io_link_work_cb;
+                       nxt->work.data = link;
+               }
        }
 }