blk-mq: make blk_mq_alloc_request_hctx() allocate a scheduler request

[mirror_ubuntu-artful-kernel.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 9e6b064e533979446a936c45c18f500c6f87725b..85a7047d8b03d7a99be90942df8389a9c09d62aa 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -273,10 +273,9 @@ EXPORT_SYMBOL(blk_mq_alloc_request);
 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
                unsigned int flags, unsigned int hctx_idx)
 {
-       struct blk_mq_hw_ctx *hctx;
-       struct blk_mq_ctx *ctx;
+       struct blk_mq_alloc_data alloc_data = { .flags = flags };
        struct request *rq;
-       struct blk_mq_alloc_data alloc_data;
+       unsigned int cpu;
        int ret;
 
        /*
@@ -299,25 +298,23 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
         * Check if the hardware context is actually mapped to anything.
         * If not tell the caller that it should skip this queue.
         */
-       hctx = q->queue_hw_ctx[hctx_idx];
-       if (!blk_mq_hw_queue_mapped(hctx)) {
-               ret = -EXDEV;
-               goto out_queue_exit;
+       alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
+       if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
+               blk_queue_exit(q);
+               return ERR_PTR(-EXDEV);
        }
-       ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));
+       cpu = cpumask_first(alloc_data.hctx->cpumask);
+       alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
 
-       blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
-       rq = __blk_mq_alloc_request(&alloc_data, rw);
-       if (!rq) {
-               ret = -EWOULDBLOCK;
-               goto out_queue_exit;
-       }
-
-       return rq;
+       rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
 
-out_queue_exit:
+       blk_mq_put_ctx(alloc_data.ctx);
        blk_queue_exit(q);
-       return ERR_PTR(ret);
+
+       if (!rq)
+               return ERR_PTR(-EWOULDBLOCK);
+
+       return rq;
 }
 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
 
@@ -852,6 +849,9 @@ done:
                return true;
        }
 
+       if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
+               data.flags |= BLK_MQ_REQ_RESERVED;
+
        rq->tag = blk_mq_get_tag(&data);
        if (rq->tag >= 0) {
                if (blk_mq_tag_busy(data.hctx)) {
@@ -1713,16 +1713,20 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
                                        unsigned int reserved_tags)
 {
        struct blk_mq_tags *tags;
+       int node;
+
+       node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
+       if (node == NUMA_NO_NODE)
+               node = set->numa_node;
 
-       tags = blk_mq_init_tags(nr_tags, reserved_tags,
-                               set->numa_node,
+       tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
                                BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
        if (!tags)
                return NULL;
 
        tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
                                 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
-                                set->numa_node);
+                                node);
        if (!tags->rqs) {
                blk_mq_free_tags(tags);
                return NULL;
@@ -1730,7 +1734,7 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
 
        tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
                                 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
-                                set->numa_node);
+                                node);
        if (!tags->static_rqs) {
                kfree(tags->rqs);
                blk_mq_free_tags(tags);
@@ -1750,6 +1754,11 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
 {
        unsigned int i, j, entries_per_page, max_order = 4;
        size_t rq_size, left;
+       int node;
+
+       node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
+       if (node == NUMA_NO_NODE)
+               node = set->numa_node;
 
        INIT_LIST_HEAD(&tags->page_list);
 
@@ -1771,7 +1780,7 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                        this_order--;
 
                do {
-                       page = alloc_pages_node(set->numa_node,
+                       page = alloc_pages_node(node,
                                GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
                                this_order);
                        if (page)
@@ -1804,7 +1813,7 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                        if (set->ops->init_request) {
                                if (set->ops->init_request(set->driver_data,
                                                rq, hctx_idx, i,
-                                               set->numa_node)) {
+                                               node)) {
                                        tags->static_rqs[i] = NULL;
                                        goto fail;
                                }