blk-mq: add optional request->alloc_time_ns

There are currently two start time timestamps - start_time_ns and
io_start_time_ns.  The former marks the request allocation and and the
second issue-to-device time.  The planned io.weight controller needs
to measure the total time bios take to execute after it leaves rq_qos
including the time spent waiting for request to become available,
which can easily dominate on saturated devices.

This patch adds request->alloc_time_ns which records when the request
allocation attempt started.  As it isn't used for the usual stats,
make it optional behind CONFIG_BLK_RQ_ALLOC_TIME and
QUEUE_FLAG_RQ_ALLOC_TIME so that it can be compiled out when there are
no users and it's active only on queues which need it even when
compiled in.

v2: s/pre_start_time/alloc_time/ and add CONFIG_BLK_RQ_ALLOC_TIME
    gating as suggested by Jens.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/Kconfig b/block/Kconfig
index 8b5f8e560eb4eaf5fef02298edfc6ebc9c03efd6..1b62ad6d0e12460652070c1ce659c93b75903b27 100644 (file)
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -26,6 +26,9 @@ menuconfig BLOCK
 
 if BLOCK
 
+config BLK_RQ_ALLOC_TIME
+       bool
+
 config BLK_SCSI_REQUEST
        bool
 

diff --git a/block/blk-mq.c b/block/blk-mq.c
index cf768d0c295080664b16b20252724da36e4fa0ed..00441123603431843f65af0e7871d3caf227897b 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -291,7 +291,7 @@ static inline bool blk_mq_need_time_stamp(struct request *rq)
 }
 
 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
-               unsigned int tag, unsigned int op)
+               unsigned int tag, unsigned int op, u64 alloc_time_ns)
 {
        struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
        struct request *rq = tags->static_rqs[tag];
@@ -325,6 +325,9 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
        RB_CLEAR_NODE(&rq->rb_node);
        rq->rq_disk = NULL;
        rq->part = NULL;
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
+       rq->alloc_time_ns = alloc_time_ns;
+#endif
        if (blk_mq_need_time_stamp(rq))
                rq->start_time_ns = ktime_get_ns();
        else
@@ -356,8 +359,14 @@ static struct request *blk_mq_get_request(struct request_queue *q,
        struct request *rq;
        unsigned int tag;
        bool clear_ctx_on_error = false;
+       u64 alloc_time_ns = 0;
 
        blk_queue_enter_live(q);
+
+       /* alloc_time includes depth and tag waits */
+       if (blk_queue_rq_alloc_time(q))
+               alloc_time_ns = ktime_get_ns();
+
        data->q = q;
        if (likely(!data->ctx)) {
                data->ctx = blk_mq_get_ctx(q);
@@ -393,7 +402,7 @@ static struct request *blk_mq_get_request(struct request_queue *q,
                return NULL;
        }
 
-       rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags);
+       rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags, alloc_time_ns);
        if (!op_is_flush(data->cmd_flags)) {
                rq->elv.icq = NULL;
                if (e && e->type->ops.prepare_request) {

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 1ac790178787e3e9109937686993d610c4bbd3cb..d0ad21e4771b930a632c5ce24378b6300c5b2b85 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -194,7 +194,11 @@ struct request {
 
        struct gendisk *rq_disk;
        struct hd_struct *part;
-       /* Time that I/O was submitted to the kernel. */
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
+       /* Time that the first bio started allocating this request. */
+       u64 alloc_time_ns;
+#endif
+       /* Time that this request was allocated for this IO. */
        u64 start_time_ns;
        /* Time that I/O was submitted to the device. */
        u64 io_start_time_ns;
@@ -609,6 +613,7 @@ struct request_queue {
 #define QUEUE_FLAG_QUIESCED    24      /* queue has been quiesced */
 #define QUEUE_FLAG_PCI_P2PDMA  25      /* device supports PCI p2p requests */
 #define QUEUE_FLAG_ZONE_RESETALL 26    /* supports Zone Reset All */
+#define QUEUE_FLAG_RQ_ALLOC_TIME 27    /* record rq->alloc_time_ns */
 
 #define QUEUE_FLAG_MQ_DEFAULT  ((1 << QUEUE_FLAG_IO_STAT) |            \
                                 (1 << QUEUE_FLAG_SAME_COMP))
@@ -637,6 +642,12 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
        test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
 #define blk_queue_pci_p2pdma(q)        \
        test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
+#define blk_queue_rq_alloc_time(q)     \
+       test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
+#else
+#define blk_queue_rq_alloc_time(q)     false
+#endif
 
 #define blk_noretry_request(rq) \
        ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \