blk-mq: add blk_mq_start_hw_queues

[mirror_ubuntu-artful-kernel.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 9180052d42cc13ed476471350e2f7934b8b2f006..543bbc08a2614e8ee6f6eecb42fa096b9f0fe746 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -294,20 +294,27 @@ void blk_mq_clone_flush_request(struct request *flush_rq,
                hctx->cmd_size);
 }
 
-bool blk_mq_end_io_partial(struct request *rq, int error, unsigned int nr_bytes)
+inline void __blk_mq_end_io(struct request *rq, int error)
 {
-       if (blk_update_request(rq, error, blk_rq_bytes(rq)))
-               return true;
-
        blk_account_io_done(rq);
 
-       if (rq->end_io)
+       if (rq->end_io) {
                rq->end_io(rq, error);
-       else
+       } else {
+               if (unlikely(blk_bidi_rq(rq)))
+                       blk_mq_free_request(rq->next_rq);
                blk_mq_free_request(rq);
-       return false;
+       }
 }
-EXPORT_SYMBOL(blk_mq_end_io_partial);
+EXPORT_SYMBOL(__blk_mq_end_io);
+
+void blk_mq_end_io(struct request *rq, int error)
+{
+       if (blk_update_request(rq, error, blk_rq_bytes(rq)))
+               BUG();
+       __blk_mq_end_io(rq, error);
+}
+EXPORT_SYMBOL(blk_mq_end_io);
 
 static void __blk_mq_complete_request_remote(void *data)
 {
@@ -362,6 +369,8 @@ static void blk_mq_start_request(struct request *rq, bool last)
        trace_block_rq_issue(q, rq);
 
        rq->resid_len = blk_rq_bytes(rq);
+       if (unlikely(blk_bidi_rq(rq)))
+               rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 
        /*
         * Just mark start time and set the started bit. Due to memory
@@ -534,7 +543,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
        LIST_HEAD(rq_list);
        int bit, queued;
 
-       WARN_ON(!preempt_count());
+       WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 
        if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
                return;
@@ -631,7 +640,7 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
        if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
                __blk_mq_run_hw_queue(hctx);
        else if (hctx->queue->nr_hw_queues == 1)
-               kblockd_schedule_delayed_work(&hctx->delayed_work, 0);
+               kblockd_schedule_delayed_work(&hctx->run_work, 0);
        else {
                unsigned int cpu;
 
@@ -642,7 +651,7 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
                 * just queue on the first CPU.
                 */
                cpu = cpumask_first(hctx->cpumask);
-               kblockd_schedule_delayed_work_on(cpu, &hctx->delayed_work, 0);
+               kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
        }
 }
 
@@ -666,7 +675,8 @@ EXPORT_SYMBOL(blk_mq_run_queues);
 
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 {
-       cancel_delayed_work(&hctx->delayed_work);
+       cancel_delayed_work(&hctx->run_work);
+       cancel_delayed_work(&hctx->delay_work);
        set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 }
 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
@@ -691,7 +701,18 @@ void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 }
 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 
-void blk_mq_start_stopped_hw_queues(struct request_queue *q)
+void blk_mq_start_hw_queues(struct request_queue *q)
+{
+       struct blk_mq_hw_ctx *hctx;
+       int i;
+
+       queue_for_each_hw_ctx(q, hctx, i)
+               blk_mq_start_hw_queue(hctx);
+}
+EXPORT_SYMBOL(blk_mq_start_hw_queues);
+
+
+void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 {
        struct blk_mq_hw_ctx *hctx;
        int i;
@@ -702,23 +723,52 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q)
 
                clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
                preempt_disable();
-               blk_mq_run_hw_queue(hctx, true);
+               blk_mq_run_hw_queue(hctx, async);
                preempt_enable();
        }
 }
 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 
-static void blk_mq_work_fn(struct work_struct *work)
+static void blk_mq_run_work_fn(struct work_struct *work)
 {
        struct blk_mq_hw_ctx *hctx;
 
-       hctx = container_of(work, struct blk_mq_hw_ctx, delayed_work.work);
+       hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 
-       preempt_disable();
        __blk_mq_run_hw_queue(hctx);
-       preempt_enable();
 }
 
+static void blk_mq_delay_work_fn(struct work_struct *work)
+{
+       struct blk_mq_hw_ctx *hctx;
+
+       hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
+
+       if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
+               __blk_mq_run_hw_queue(hctx);
+}
+
+void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
+{
+       unsigned long tmo = msecs_to_jiffies(msecs);
+
+       if (hctx->queue->nr_hw_queues == 1)
+               kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
+       else {
+               unsigned int cpu;
+
+               /*
+                * It'd be great if the workqueue API had a way to pass
+                * in a mask and had some smarts for more clever placement
+                * than the first CPU. Or we could round-robin here. For now,
+                * just queue on the first CPU.
+                */
+               cpu = cpumask_first(hctx->cpumask);
+               kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
+       }
+}
+EXPORT_SYMBOL(blk_mq_delay_queue);
+
 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
                                    struct request *rq, bool at_head)
 {
@@ -1172,7 +1222,8 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
                if (node == NUMA_NO_NODE)
                        node = hctx->numa_node = set->numa_node;
 
-               INIT_DELAYED_WORK(&hctx->delayed_work, blk_mq_work_fn);
+               INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
+               INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
                spin_lock_init(&hctx->lock);
                INIT_LIST_HEAD(&hctx->dispatch);
                hctx->queue = q;