io-wq: optimise locking in io_worker_handle_work()

There are 2 optimisations:
- Now, io_worker_handler_work() do io_assign_current_work() twice per
request, and each one adds lock/unlock(worker->lock) pair. The first is
to reset worker->cur_work to NULL, and the second to set a real work
shortly after. If there is a dependant work, set it immediately, that
effectively removes the extra NULL'ing.

- And there is no use in taking wqe->lock for linked works, as they are
not hashed now. Optimise it out.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index e438dc4d7cb384c148b04ee7fb2fcce4234a54f8..473af080470a342c475453f393f5a1deb50966fc 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -476,7 +476,7 @@ static void io_worker_handle_work(struct io_worker *worker)
        struct io_wq *wq = wqe->wq;
 
        do {
-               struct io_wq_work *work, *old_work;
+               struct io_wq_work *work;
                unsigned hash = -1U;
 
                /*
@@ -495,12 +495,13 @@ static void io_worker_handle_work(struct io_worker *worker)
                spin_unlock_irq(&wqe->lock);
                if (!work)
                        break;
+               io_assign_current_work(worker, work);
 
                /* handle a whole dependent link */
                do {
-                       io_assign_current_work(worker, work);
-                       io_impersonate_work(worker, work);
+                       struct io_wq_work *old_work;
 
+                       io_impersonate_work(worker, work);
                        /*
                         * OK to set IO_WQ_WORK_CANCEL even for uncancellable
                         * work, the worker function will do the right thing.
@@ -513,10 +514,8 @@ static void io_worker_handle_work(struct io_worker *worker)
 
                        old_work = work;
                        work->func(&work);
-
-                       spin_lock_irq(&worker->lock);
-                       worker->cur_work = NULL;
-                       spin_unlock_irq(&worker->lock);
+                       work = (old_work == work) ? NULL : work;
+                       io_assign_current_work(worker, work);
 
                        if (wq->put_work)
                                wq->put_work(old_work);
@@ -529,7 +528,7 @@ static void io_worker_handle_work(struct io_worker *worker)
                                /* dependent work is not hashed */
                                hash = -1U;
                        }
-               } while (work && work != old_work);
+               } while (work);
 
                spin_lock_irq(&wqe->lock);
        } while (1);