block: hook up writeback throttling

[mirror_ubuntu-artful-kernel.git] / block / blk-core.c

diff --git a/block/blk-core.c b/block/blk-core.c
index 216372b01624cd61dba5574bb27434ab19b1287d..59f8129a42950e0d23e0e4cc28d4ab8e8e67b63e 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -39,6 +39,7 @@
 
 #include "blk.h"
 #include "blk-mq.h"
+#include "blk-wbt.h"
 
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
@@ -882,6 +883,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
 
 fail:
        blk_free_flush_queue(q->fq);
+       wbt_exit(q);
        return NULL;
 }
 EXPORT_SYMBOL(blk_init_allocated_queue);
@@ -1344,6 +1346,7 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
        blk_delete_timer(rq);
        blk_clear_rq_complete(rq);
        trace_block_rq_requeue(q, rq);
+       wbt_requeue(q->rq_wb, &rq->issue_stat);
 
        if (rq->rq_flags & RQF_QUEUED)
                blk_queue_end_tag(q, rq);
@@ -1436,6 +1439,8 @@ void __blk_put_request(struct request_queue *q, struct request *req)
        /* this is a bio leak */
        WARN_ON(req->bio != NULL);
 
+       wbt_done(q->rq_wb, &req->issue_stat);
+
        /*
         * Request may not have originated from ll_rw_blk. if not,
         * it didn't come out of our reserved rq pools
@@ -1663,6 +1668,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
        int el_ret, where = ELEVATOR_INSERT_SORT;
        struct request *req;
        unsigned int request_count = 0;
+       unsigned int wb_acct;
 
        /*
         * low level driver can indicate that it wants pages above a
@@ -1715,17 +1721,22 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
        }
 
 get_rq:
+       wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
+
        /*
         * Grab a free request. This is might sleep but can not fail.
         * Returns with the queue unlocked.
         */
        req = get_request(q, bio->bi_opf, bio, GFP_NOIO);
        if (IS_ERR(req)) {
+               __wbt_done(q->rq_wb, wb_acct);
                bio->bi_error = PTR_ERR(req);
                bio_endio(bio);
                goto out_unlock;
        }
 
+       wbt_track(&req->issue_stat, wb_acct);
+
        /*
         * After dropping the lock and possibly sleeping here, our request
         * may now be mergeable after it had proven unmergeable (above).
@@ -2467,6 +2478,7 @@ void blk_start_request(struct request *req)
        if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
                blk_stat_set_issue_time(&req->issue_stat);
                req->rq_flags |= RQF_STATS;
+               wbt_issue(req->q->rq_wb, &req->issue_stat);
        }
 
        /*
@@ -2708,9 +2720,10 @@ void blk_finish_request(struct request *req, int error)
 
        blk_account_io_done(req);
 
-       if (req->end_io)
+       if (req->end_io) {
+               wbt_done(req->q->rq_wb, &req->issue_stat);
                req->end_io(req, error);
-       else {
+       } else {
                if (blk_bidi_rq(req))
                        __blk_put_request(req->next_rq->q, req->next_rq);