block/blk-rq-qos.c

   1 #include "blk-rq-qos.h"
   2
   3 /*
   4  * Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
   5  * false if 'v' + 1 would be bigger than 'below'.
   6  */
   7 static bool atomic_inc_below(atomic_t *v, unsigned int below)
   8 {
   9         unsigned int cur = atomic_read(v);
  10
  11         for (;;) {
  12                 unsigned int old;
  13
  14                 if (cur >= below)
  15                         return false;
  16                 old = atomic_cmpxchg(v, cur, cur + 1);
  17                 if (old == cur)
  18                         break;
  19                 cur = old;
  20         }
  21
  22         return true;
  23 }
  24
  25 bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit)
  26 {
  27         return atomic_inc_below(&rq_wait->inflight, limit);
  28 }
  29
  30 void __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio)
  31 {
  32         do {
  33                 if (rqos->ops->cleanup)
  34                         rqos->ops->cleanup(rqos, bio);
  35                 rqos = rqos->next;
  36         } while (rqos);
  37 }
  38
  39 void __rq_qos_done(struct rq_qos *rqos, struct request *rq)
  40 {
  41         do {
  42                 if (rqos->ops->done)
  43                         rqos->ops->done(rqos, rq);
  44                 rqos = rqos->next;
  45         } while (rqos);
  46 }
  47
  48 void __rq_qos_issue(struct rq_qos *rqos, struct request *rq)
  49 {
  50         do {
  51                 if (rqos->ops->issue)
  52                         rqos->ops->issue(rqos, rq);
  53                 rqos = rqos->next;
  54         } while (rqos);
  55 }
  56
  57 void __rq_qos_requeue(struct rq_qos *rqos, struct request *rq)
  58 {
  59         do {
  60                 if (rqos->ops->requeue)
  61                         rqos->ops->requeue(rqos, rq);
  62                 rqos = rqos->next;
  63         } while (rqos);
  64 }
  65
  66 void __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio)
  67 {
  68         do {
  69                 if (rqos->ops->throttle)
  70                         rqos->ops->throttle(rqos, bio);
  71                 rqos = rqos->next;
  72         } while (rqos);
  73 }
  74
  75 void __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
  76 {
  77         do {
  78                 if (rqos->ops->track)
  79                         rqos->ops->track(rqos, rq, bio);
  80                 rqos = rqos->next;
  81         } while (rqos);
  82 }
  83
  84 void __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio)
  85 {
  86         do {
  87                 if (rqos->ops->done_bio)
  88                         rqos->ops->done_bio(rqos, bio);
  89                 rqos = rqos->next;
  90         } while (rqos);
  91 }
  92
  93 /*
  94  * Return true, if we can't increase the depth further by scaling
  95  */
  96 bool rq_depth_calc_max_depth(struct rq_depth *rqd)
  97 {
  98         unsigned int depth;
  99         bool ret = false;
 100
 101         /*
 102          * For QD=1 devices, this is a special case. It's important for those
 103          * to have one request ready when one completes, so force a depth of
 104          * 2 for those devices. On the backend, it'll be a depth of 1 anyway,
 105          * since the device can't have more than that in flight. If we're
 106          * scaling down, then keep a setting of 1/1/1.
 107          */
 108         if (rqd->queue_depth == 1) {
 109                 if (rqd->scale_step > 0)
 110                         rqd->max_depth = 1;
 111                 else {
 112                         rqd->max_depth = 2;
 113                         ret = true;
 114                 }
 115         } else {
 116                 /*
 117                  * scale_step == 0 is our default state. If we have suffered
 118                  * latency spikes, step will be > 0, and we shrink the
 119                  * allowed write depths. If step is < 0, we're only doing
 120                  * writes, and we allow a temporarily higher depth to
 121                  * increase performance.
 122                  */
 123                 depth = min_t(unsigned int, rqd->default_depth,
 124                               rqd->queue_depth);
 125                 if (rqd->scale_step > 0)
 126                         depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));
 127                 else if (rqd->scale_step < 0) {
 128                         unsigned int maxd = 3 * rqd->queue_depth / 4;
 129
 130                         depth = 1 + ((depth - 1) << -rqd->scale_step);
 131                         if (depth > maxd) {
 132                                 depth = maxd;
 133                                 ret = true;
 134                         }
 135                 }
 136
 137                 rqd->max_depth = depth;
 138         }
 139
 140         return ret;
 141 }
 142
 143 void rq_depth_scale_up(struct rq_depth *rqd)
 144 {
 145         /*
 146          * Hit max in previous round, stop here
 147          */
 148         if (rqd->scaled_max)
 149                 return;
 150
 151         rqd->scale_step--;
 152
 153         rqd->scaled_max = rq_depth_calc_max_depth(rqd);
 154 }
 155
 156 /*
 157  * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
 158  * had a latency violation.
 159  */
 160 void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle)
 161 {
 162         /*
 163          * Stop scaling down when we've hit the limit. This also prevents
 164          * ->scale_step from going to crazy values, if the device can't
 165          * keep up.
 166          */
 167         if (rqd->max_depth == 1)
 168                 return;
 169
 170         if (rqd->scale_step < 0 && hard_throttle)
 171                 rqd->scale_step = 0;
 172         else
 173                 rqd->scale_step++;
 174
 175         rqd->scaled_max = false;
 176         rq_depth_calc_max_depth(rqd);
 177 }
 178
 179 void rq_qos_exit(struct request_queue *q)
 180 {
 181         while (q->rq_qos) {
 182                 struct rq_qos *rqos = q->rq_qos;
 183                 q->rq_qos = rqos->next;
 184                 rqos->ops->exit(rqos);
 185         }
 186 }