block: Abstract out bvec iterator

[mirror_ubuntu-bionic-kernel.git] / block / blk-throttle.c

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 8331aba9426f2c75ffb29ff65a3b89effa6b5262..20f82003777511798659a467666c3a8f6d4b8cbb 100644 (file)
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -256,6 +256,12 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
        }                                                               \
 } while (0)
 
+static void tg_stats_init(struct tg_stats_cpu *tg_stats)
+{
+       blkg_rwstat_init(&tg_stats->service_bytes);
+       blkg_rwstat_init(&tg_stats->serviced);
+}
+
 /*
  * Worker for allocating per cpu stat for tgs. This is scheduled on the
  * system_wq once there are some groups on the alloc_list waiting for
@@ -269,12 +275,16 @@ static void tg_stats_alloc_fn(struct work_struct *work)
 
 alloc_stats:
        if (!stats_cpu) {
+               int cpu;
+
                stats_cpu = alloc_percpu(struct tg_stats_cpu);
                if (!stats_cpu) {
                        /* allocation failed, try again after some time */
                        schedule_delayed_work(dwork, msecs_to_jiffies(10));
                        return;
                }
+               for_each_possible_cpu(cpu)
+                       tg_stats_init(per_cpu_ptr(stats_cpu, cpu));
        }
 
        spin_lock_irq(&tg_stats_alloc_lock);
@@ -867,14 +877,14 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
        do_div(tmp, HZ);
        bytes_allowed = tmp;
 
-       if (tg->bytes_disp[rw] + bio->bi_size <= bytes_allowed) {
+       if (tg->bytes_disp[rw] + bio->bi_iter.bi_size <= bytes_allowed) {
                if (wait)
                        *wait = 0;
                return 1;
        }
 
        /* Calc approx time to dispatch */
-       extra_bytes = tg->bytes_disp[rw] + bio->bi_size - bytes_allowed;
+       extra_bytes = tg->bytes_disp[rw] + bio->bi_iter.bi_size - bytes_allowed;
        jiffy_wait = div64_u64(extra_bytes * HZ, tg->bps[rw]);
 
        if (!jiffy_wait)
@@ -977,7 +987,7 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
        bool rw = bio_data_dir(bio);
 
        /* Charge the bio to the group */
-       tg->bytes_disp[rw] += bio->bi_size;
+       tg->bytes_disp[rw] += bio->bi_iter.bi_size;
        tg->io_disp[rw]++;
 
        /*
@@ -993,8 +1003,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
         */
        if (!(bio->bi_rw & REQ_THROTTLED)) {
                bio->bi_rw |= REQ_THROTTLED;
-               throtl_update_dispatch_stats(tg_to_blkg(tg), bio->bi_size,
-                                            bio->bi_rw);
+               throtl_update_dispatch_stats(tg_to_blkg(tg),
+                                            bio->bi_iter.bi_size, bio->bi_rw);
        }
 }
 
@@ -1498,7 +1508,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
        if (tg) {
                if (!tg->has_rules[rw]) {
                        throtl_update_dispatch_stats(tg_to_blkg(tg),
-                                                    bio->bi_size, bio->bi_rw);
+                                       bio->bi_iter.bi_size, bio->bi_rw);
                        goto out_unlock_rcu;
                }
        }
@@ -1554,7 +1564,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
        /* out-of-limit, queue to @tg */
        throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",
                   rw == READ ? 'R' : 'W',
-                  tg->bytes_disp[rw], bio->bi_size, tg->bps[rw],
+                  tg->bytes_disp[rw], bio->bi_iter.bi_size, tg->bps[rw],
                   tg->io_disp[rw], tg->iops[rw],
                   sq->nr_queued[READ], sq->nr_queued[WRITE]);