blk-wbt: fix has-sleeper queueing check

[mirror_ubuntu-bionic-kernel.git] / block / blk-wbt.c

diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index ae8de9780085ae7b8e99237ed16fc9cd02b233a5..b2fe2682b3f95a7337f51b2bf791513441fb519d 100644 (file)
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -101,9 +101,13 @@ static bool wb_recent_wait(struct rq_wb *rwb)
        return time_before(jiffies, wb->dirty_sleep + HZ);
 }
 
-static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, bool is_kswapd)
+static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
+                                         enum wbt_flags wb_acct)
 {
-       return &rwb->rq_wait[is_kswapd];
+       if (wb_acct & WBT_KSWAPD)
+               return &rwb->rq_wait[WBT_RWQ_KSWAPD];
+
+       return &rwb->rq_wait[WBT_RWQ_BG];
 }
 
 static void rwb_wake_all(struct rq_wb *rwb)
@@ -113,7 +117,7 @@ static void rwb_wake_all(struct rq_wb *rwb)
        for (i = 0; i < WBT_NUM_RWQ; i++) {
                struct rq_wait *rqw = &rwb->rq_wait[i];
 
-               if (waitqueue_active(&rqw->wait))
+               if (wq_has_sleeper(&rqw->wait))
                        wake_up_all(&rqw->wait);
        }
 }
@@ -126,7 +130,7 @@ void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
        if (!(wb_acct & WBT_TRACKED))
                return;
 
-       rqw = get_rq_wait(rwb, wb_acct & WBT_KSWAPD);
+       rqw = get_rq_wait(rwb, wb_acct);
        inflight = atomic_dec_return(&rqw->inflight);
 
        /*
@@ -153,11 +157,11 @@ void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
        if (inflight && inflight >= limit)
                return;
 
-       if (waitqueue_active(&rqw->wait)) {
+       if (wq_has_sleeper(&rqw->wait)) {
                int diff = limit - inflight;
 
                if (!inflight || diff >= rwb->wb_background / 2)
-                       wake_up_all(&rqw->wait);
+                       wake_up(&rqw->wait);
        }
 }
 
@@ -479,6 +483,13 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
 {
        unsigned int limit;
 
+       /*
+        * If we got disabled, just return UINT_MAX. This ensures that
+        * we'll properly inc a new IO, and dec+wakeup at the end.
+        */
+       if (!rwb_enabled(rwb))
+               return UINT_MAX;
+
        /*
         * At this point we know it's a buffered write. If this is
         * kswapd trying to free memory, or REQ_SYNC is set, then
@@ -501,49 +512,28 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
        return limit;
 }
 
-static inline bool may_queue(struct rq_wb *rwb, struct rq_wait *rqw,
-                            wait_queue_entry_t *wait, unsigned long rw)
-{
-       /*
-        * inc it here even if disabled, since we'll dec it at completion.
-        * this only happens if the task was sleeping in __wbt_wait(),
-        * and someone turned it off at the same time.
-        */
-       if (!rwb_enabled(rwb)) {
-               atomic_inc(&rqw->inflight);
-               return true;
-       }
-
-       /*
-        * If the waitqueue is already active and we are not the next
-        * in line to be woken up, wait for our turn.
-        */
-       if (waitqueue_active(&rqw->wait) &&
-           rqw->wait.head.next != &wait->entry)
-               return false;
-
-       return atomic_inc_below(&rqw->inflight, get_limit(rwb, rw));
-}
-
 /*
  * Block if we will exceed our limit, or if we are currently waiting for
  * the timer to kick off queuing again.
  */
-static void __wbt_wait(struct rq_wb *rwb, unsigned long rw, spinlock_t *lock)
+static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
+                      unsigned long rw, spinlock_t *lock)
        __releases(lock)
        __acquires(lock)
 {
-       struct rq_wait *rqw = get_rq_wait(rwb, current_is_kswapd());
-       DEFINE_WAIT(wait);
+       struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
+       DECLARE_WAITQUEUE(wait, current);
+       bool has_sleeper;
 
-       if (may_queue(rwb, rqw, &wait, rw))
+       has_sleeper = wq_has_sleeper(&rqw->wait);
+       if (!has_sleeper && atomic_inc_below(&rqw->inflight, get_limit(rwb, rw)))
                return;
 
+       add_wait_queue_exclusive(&rqw->wait, &wait);
        do {
-               prepare_to_wait_exclusive(&rqw->wait, &wait,
-                                               TASK_UNINTERRUPTIBLE);
+               set_current_state(TASK_UNINTERRUPTIBLE);
 
-               if (may_queue(rwb, rqw, &wait, rw))
+               if (!has_sleeper && atomic_inc_below(&rqw->inflight, get_limit(rwb, rw)))
                        break;
 
                if (lock) {
@@ -552,9 +542,11 @@ static void __wbt_wait(struct rq_wb *rwb, unsigned long rw, spinlock_t *lock)
                        spin_lock_irq(lock);
                } else
                        io_schedule();
+               has_sleeper = false;
        } while (1);
 
-       finish_wait(&rqw->wait, &wait);
+       __set_current_state(TASK_RUNNING);
+       remove_wait_queue(&rqw->wait, &wait);
 }
 
 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
@@ -584,7 +576,7 @@ static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
  */
 enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio, spinlock_t *lock)
 {
-       unsigned int ret = 0;
+       enum wbt_flags ret = 0;
 
        if (!rwb_enabled(rwb))
                return 0;
@@ -598,14 +590,14 @@ enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio, spinlock_t *lock)
                return ret;
        }
 
-       __wbt_wait(rwb, bio->bi_opf, lock);
+       if (current_is_kswapd())
+               ret |= WBT_KSWAPD;
+
+       __wbt_wait(rwb, ret, bio->bi_opf, lock);
 
        if (!blk_stat_is_active(rwb->cb))
                rwb_arm_timer(rwb);
 
-       if (current_is_kswapd())
-               ret |= WBT_KSWAPD;
-
        return ret | WBT_TRACKED;
 }
 
@@ -697,7 +689,15 @@ u64 wbt_default_latency_nsec(struct request_queue *q)
 
 static int wbt_data_dir(const struct request *rq)
 {
-       return rq_data_dir(rq);
+       const int op = req_op(rq);
+
+       if (op == REQ_OP_READ)
+               return READ;
+       else if (op == REQ_OP_WRITE || op == REQ_OP_FLUSH)
+               return WRITE;
+
+       /* don't account */
+       return -1;
 }
 
 int wbt_init(struct request_queue *q)