[mirror_ubuntu-bionic-kernel.git] / block / blk-mq-sched.c

/*
 * blk-mq scheduling framework
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/blk-mq.h>

#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-sched.h"
#include "blk-mq-tag.h"
#include "blk-wbt.h"

void blk_mq_sched_free_hctx_data(struct request_queue *q,
				 void (*exit)(struct blk_mq_hw_ctx *))
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (exit && hctx->sched_data)
			exit(hctx);
		kfree(hctx->sched_data);
		hctx->sched_data = NULL;
	}
}
EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);

int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
				int (*init)(struct blk_mq_hw_ctx *),
				void (*exit)(struct blk_mq_hw_ctx *))
{
	struct blk_mq_hw_ctx *hctx;
	int ret;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		hctx->sched_data = kmalloc_node(size, GFP_KERNEL, hctx->numa_node);
		if (!hctx->sched_data) {
			ret = -ENOMEM;
			goto error;
		}

		if (init) {
			ret = init(hctx);
			if (ret) {
				/*
				 * We don't want to give exit() a partially
				 * initialized sched_data. init() must clean up
				 * if it fails.
				 */
				kfree(hctx->sched_data);
				hctx->sched_data = NULL;
				goto error;
			}
		}
	}

	return 0;
error:
	blk_mq_sched_free_hctx_data(q, exit);
	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_sched_init_hctx_data);

static void __blk_mq_sched_assign_ioc(struct request_queue *q,
				      struct request *rq,
				      struct bio *bio,
				      struct io_context *ioc)
{
	struct io_cq *icq;

	spin_lock_irq(q->queue_lock);
	icq = ioc_lookup_icq(ioc, q);
	spin_unlock_irq(q->queue_lock);

	if (!icq) {
		icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
		if (!icq)
			return;
	}

	rq->elv.icq = icq;
	if (!blk_mq_sched_get_rq_priv(q, rq, bio)) {
		rq->rq_flags |= RQF_ELVPRIV;
		get_io_context(icq->ioc);
		return;
	}

	rq->elv.icq = NULL;
}

static void blk_mq_sched_assign_ioc(struct request_queue *q,
				    struct request *rq, struct bio *bio)
{
	struct io_context *ioc;

	ioc = rq_ioc(bio);
	if (ioc)
		__blk_mq_sched_assign_ioc(q, rq, bio, ioc);
}

struct request *blk_mq_sched_get_request(struct request_queue *q,
					 struct bio *bio,
					 unsigned int op,
					 struct blk_mq_alloc_data *data)
{
	struct elevator_queue *e = q->elevator;
	struct request *rq;

	blk_queue_enter_live(q);
	data->q = q;
	if (likely(!data->ctx))
		data->ctx = blk_mq_get_ctx(q);
	if (likely(!data->hctx))
		data->hctx = blk_mq_map_queue(q, data->ctx->cpu);

	if (e) {
		data->flags |= BLK_MQ_REQ_INTERNAL;

		/*
		 * Flush requests are special and go directly to the
		 * dispatch list.
		 */
		if (!op_is_flush(op) && e->type->ops.mq.get_request) {
			rq = e->type->ops.mq.get_request(q, op, data);
			if (rq)
				rq->rq_flags |= RQF_QUEUED;
		} else
			rq = __blk_mq_alloc_request(data, op);
	} else {
		rq = __blk_mq_alloc_request(data, op);
	}

	if (rq) {
		if (!op_is_flush(op)) {
			rq->elv.icq = NULL;
			if (e && e->type->icq_cache)
				blk_mq_sched_assign_ioc(q, rq, bio);
		}
		data->hctx->queued++;
		return rq;
	}

	blk_queue_exit(q);
	return NULL;
}

void blk_mq_sched_put_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (rq->rq_flags & RQF_ELVPRIV) {
		blk_mq_sched_put_rq_priv(rq->q, rq);
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}

	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
		e->type->ops.mq.put_request(rq);
	else
		blk_mq_finish_request(rq);
}

void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct elevator_queue *e = q->elevator;
	const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
	bool did_work = false;
	LIST_HEAD(rq_list);

	if (unlikely(blk_mq_hctx_stopped(hctx)))
		return;

	hctx->run++;

	/*
	 * If we have previous entries on our dispatch list, grab them first for
	 * more fair dispatch.
	 */
	if (!list_empty_careful(&hctx->dispatch)) {
		spin_lock(&hctx->lock);
		if (!list_empty(&hctx->dispatch))
			list_splice_init(&hctx->dispatch, &rq_list);
		spin_unlock(&hctx->lock);
	}

	/*
	 * Only ask the scheduler for requests, if we didn't have residual
	 * requests from the dispatch list. This is to avoid the case where
	 * we only ever dispatch a fraction of the requests available because
	 * of low device queue depth. Once we pull requests out of the IO
	 * scheduler, we can no longer merge or sort them. So it's best to
	 * leave them there for as long as we can. Mark the hw queue as
	 * needing a restart in that case.
	 */
	if (!list_empty(&rq_list)) {
		blk_mq_sched_mark_restart_hctx(hctx);
		did_work = blk_mq_dispatch_rq_list(q, &rq_list);
	} else if (!has_sched_dispatch) {
		blk_mq_flush_busy_ctxs(hctx, &rq_list);
		blk_mq_dispatch_rq_list(q, &rq_list);
	}

	/*
	 * We want to dispatch from the scheduler if we had no work left
	 * on the dispatch list, OR if we did have work but weren't able
	 * to make progress.
	 */
	if (!did_work && has_sched_dispatch) {
		do {
			struct request *rq;

			rq = e->type->ops.mq.dispatch_request(hctx);
			if (!rq)
				break;
			list_add(&rq->queuelist, &rq_list);
		} while (blk_mq_dispatch_rq_list(q, &rq_list));
	}
}

void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
				   struct list_head *rq_list,
				   struct request *(*get_rq)(struct blk_mq_hw_ctx *))
{
	do {
		struct request *rq;

		rq = get_rq(hctx);
		if (!rq)
			break;

		list_add_tail(&rq->queuelist, rq_list);
	} while (1);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);

bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
			    struct request **merged_request)
{
	struct request *rq;

	switch (elv_merge(q, &rq, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (!bio_attempt_back_merge(q, rq, bio))
			return false;
		*merged_request = attempt_back_merge(q, rq);
		if (!*merged_request)
			elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
		return true;
	case ELEVATOR_FRONT_MERGE:
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (!bio_attempt_front_merge(q, rq, bio))
			return false;
		*merged_request = attempt_front_merge(q, rq);
		if (!*merged_request)
			elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
		return true;
	default:
		return false;
	}
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);

bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.mq.bio_merge) {
		struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
		struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

		blk_mq_put_ctx(ctx);
		return e->type->ops.mq.bio_merge(hctx, bio);
	}

	return false;
}

bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq)
{
	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);

void blk_mq_sched_request_inserted(struct request *rq)
{
	trace_block_rq_insert(rq->q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);

static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	if (rq->tag == -1) {
		rq->rq_flags |= RQF_SORTED;
		return false;
	}

	/*
	 * If we already have a real request tag, send directly to
	 * the dispatch list.
	 */
	spin_lock(&hctx->lock);
	list_add(&rq->queuelist, &hctx->dispatch);
	spin_unlock(&hctx->lock);
	return true;
}

static void blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx)
{
	if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) {
		clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
		if (blk_mq_hctx_has_pending(hctx))
			blk_mq_run_hw_queue(hctx, true);
	}
}

void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	unsigned int i;

	if (test_bit(QUEUE_FLAG_RESTART, &q->queue_flags)) {
		if (test_and_clear_bit(QUEUE_FLAG_RESTART, &q->queue_flags)) {
			queue_for_each_hw_ctx(q, hctx, i)
				blk_mq_sched_restart_hctx(hctx);
		}
	} else {
		blk_mq_sched_restart_hctx(hctx);
	}
}

/*
 * Add flush/fua to the queue. If we fail getting a driver tag, then
 * punt to the requeue list. Requeue will re-invoke us from a context
 * that's safe to block from.
 */
static void blk_mq_sched_insert_flush(struct blk_mq_hw_ctx *hctx,
				      struct request *rq, bool can_block)
{
	if (blk_mq_get_driver_tag(rq, &hctx, can_block)) {
		blk_insert_flush(rq);
		blk_mq_run_hw_queue(hctx, true);
	} else
		blk_mq_add_to_requeue_list(rq, false, true);
}

void blk_mq_sched_insert_request(struct request *rq, bool at_head,
				 bool run_queue, bool async, bool can_block)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

	if (rq->tag == -1 && op_is_flush(rq->cmd_flags)) {
		blk_mq_sched_insert_flush(hctx, rq, can_block);
		return;
	}

	if (e && blk_mq_sched_bypass_insert(hctx, rq))
		goto run;

	if (e && e->type->ops.mq.insert_requests) {
		LIST_HEAD(list);

		list_add(&rq->queuelist, &list);
		e->type->ops.mq.insert_requests(hctx, &list, at_head);
	} else {
		spin_lock(&ctx->lock);
		__blk_mq_insert_request(hctx, rq, at_head);
		spin_unlock(&ctx->lock);
	}

run:
	if (run_queue)
		blk_mq_run_hw_queue(hctx, async);
}

void blk_mq_sched_insert_requests(struct request_queue *q,
				  struct blk_mq_ctx *ctx,
				  struct list_head *list, bool run_queue_async)
{
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	struct elevator_queue *e = hctx->queue->elevator;

	if (e) {
		struct request *rq, *next;

		/*
		 * We bypass requests that already have a driver tag assigned,
		 * which should only be flushes. Flushes are only ever inserted
		 * as single requests, so we shouldn't ever hit the
		 * WARN_ON_ONCE() below (but let's handle it just in case).
		 */
		list_for_each_entry_safe(rq, next, list, queuelist) {
			if (WARN_ON_ONCE(rq->tag != -1)) {
				list_del_init(&rq->queuelist);
				blk_mq_sched_bypass_insert(hctx, rq);
			}
		}
	}

	if (e && e->type->ops.mq.insert_requests)
		e->type->ops.mq.insert_requests(hctx, list, false);
	else
		blk_mq_insert_requests(hctx, ctx, list);

	blk_mq_run_hw_queue(hctx, run_queue_async);
}

static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	if (hctx->sched_tags) {
		blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
		blk_mq_free_rq_map(hctx->sched_tags);
		hctx->sched_tags = NULL;
	}
}

static int blk_mq_sched_alloc_tags(struct request_queue *q,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	struct blk_mq_tag_set *set = q->tag_set;
	int ret;

	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
					       set->reserved_tags);
	if (!hctx->sched_tags)
		return -ENOMEM;

	ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
	if (ret)
		blk_mq_sched_free_tags(set, hctx, hctx_idx);

	return ret;
}

void blk_mq_sched_teardown(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_sched_free_tags(set, hctx, i);
}

int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	int ret;

	if (!e) {
		q->elevator = NULL;
		return 0;
	}

	/*
	 * Default to 256, since we don't split into sync/async like the
	 * old code did. Additionally, this is a per-hw queue depth.
	 */
	q->nr_requests = 2 * BLKDEV_MAX_RQ;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_sched_alloc_tags(q, hctx, i);
		if (ret)
			goto err;
	}

	ret = e->ops.mq.init_sched(q, e);
	if (ret)
		goto err;

	return 0;

err:
	blk_mq_sched_teardown(q);
	return ret;
}

int blk_mq_sched_init(struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = elevator_init(q, NULL);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
bd166ef1 JA	1	/*
	2	* blk-mq scheduling framework
	3	*
	4	* Copyright (C) 2016 Jens Axboe
	5	*/
	6	#include <linux/kernel.h>
	7	#include <linux/module.h>
	8	#include <linux/blk-mq.h>
	9
	10	#include <trace/events/block.h>
	11
	12	#include "blk.h"
	13	#include "blk-mq.h"
	14	#include "blk-mq-sched.h"
	15	#include "blk-mq-tag.h"
	16	#include "blk-wbt.h"
	17
	18	void blk_mq_sched_free_hctx_data(struct request_queue *q,
	19	void (exit)(struct blk_mq_hw_ctx ))
	20	{
	21	struct blk_mq_hw_ctx *hctx;
	22	int i;
	23
	24	queue_for_each_hw_ctx(q, hctx, i) {
	25	if (exit && hctx->sched_data)
	26	exit(hctx);
	27	kfree(hctx->sched_data);
	28	hctx->sched_data = NULL;
	29	}
	30	}
	31	EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
	32
	33	int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
	34	int (init)(struct blk_mq_hw_ctx ),
	35	void (exit)(struct blk_mq_hw_ctx ))
	36	{
	37	struct blk_mq_hw_ctx *hctx;
	38	int ret;
	39	int i;
	40
	41	queue_for_each_hw_ctx(q, hctx, i) {
	42	hctx->sched_data = kmalloc_node(size, GFP_KERNEL, hctx->numa_node);
	43	if (!hctx->sched_data) {
	44	ret = -ENOMEM;
	45	goto error;
	46	}
	47
	48	if (init) {
	49	ret = init(hctx);
	50	if (ret) {
	51	/*
	52	* We don't want to give exit() a partially
	53	* initialized sched_data. init() must clean up
	54	* if it fails.
	55	*/
	56	kfree(hctx->sched_data);
	57	hctx->sched_data = NULL;
	58	goto error;
	59	}
	60	}
	61	}
	62
	63	return 0;
	64	error:
65	blk_mq_sched_free_hctx_data(q, exit);
66	return ret;
67	}
68	EXPORT_SYMBOL_GPL(blk_mq_sched_init_hctx_data);
69
70	static void __blk_mq_sched_assign_ioc(struct request_queue *q,
f1ba8261 PV	71	struct request *rq,
	72	struct bio *bio,
	73	struct io_context *ioc)
bd166ef1 JA	74	{
	75	struct io_cq *icq;
	76
	77	spin_lock_irq(q->queue_lock);
	78	icq = ioc_lookup_icq(ioc, q);
	79	spin_unlock_irq(q->queue_lock);
	80
	81	if (!icq) {
	82	icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
	83	if (!icq)
	84	return;
	85	}
	86
	87	rq->elv.icq = icq;
f1ba8261	88	if (!blk_mq_sched_get_rq_priv(q, rq, bio)) {
bd166ef1 JA	89	rq->rq_flags \|= RQF_ELVPRIV;
	90	get_io_context(icq->ioc);
	91	return;
	92	}
	93
	94	rq->elv.icq = NULL;
	95	}
	96
	97	static void blk_mq_sched_assign_ioc(struct request_queue *q,
	98	struct request rq, struct bio bio)
	99	{
	100	struct io_context *ioc;
	101
	102	ioc = rq_ioc(bio);
	103	if (ioc)
f1ba8261	104	__blk_mq_sched_assign_ioc(q, rq, bio, ioc);
bd166ef1 JA	105	}
	106
	107	struct request blk_mq_sched_get_request(struct request_queue q,
	108	struct bio *bio,
	109	unsigned int op,
	110	struct blk_mq_alloc_data *data)
	111	{
	112	struct elevator_queue *e = q->elevator;
bd166ef1	113	struct request *rq;
bd166ef1 JA	114
bd166ef1 JA	115	blk_queue_enter_live(q);
6d2809d5 OS	116	data->q = q;
	117	if (likely(!data->ctx))
	118	data->ctx = blk_mq_get_ctx(q);
	119	if (likely(!data->hctx))
	120	data->hctx = blk_mq_map_queue(q, data->ctx->cpu);
bd166ef1 JA	121
	122	if (e) {
	123	data->flags \|= BLK_MQ_REQ_INTERNAL;
	124
	125	/*
	126	* Flush requests are special and go directly to the
	127	* dispatch list.
	128	*/
f73f44eb	129	if (!op_is_flush(op) && e->type->ops.mq.get_request) {
bd166ef1 JA	130	rq = e->type->ops.mq.get_request(q, op, data);
	131	if (rq)
	132	rq->rq_flags \|= RQF_QUEUED;
	133	} else
	134	rq = __blk_mq_alloc_request(data, op);
	135	} else {
	136	rq = __blk_mq_alloc_request(data, op);
bd166ef1 JA	137	}
	138
	139	if (rq) {
f73f44eb	140	if (!op_is_flush(op)) {
bd166ef1 JA	141	rq->elv.icq = NULL;
	142	if (e && e->type->icq_cache)
	143	blk_mq_sched_assign_ioc(q, rq, bio);
	144	}
	145	data->hctx->queued++;
	146	return rq;
	147	}
	148
	149	blk_queue_exit(q);
	150	return NULL;
	151	}
	152
	153	void blk_mq_sched_put_request(struct request *rq)
	154	{
	155	struct request_queue *q = rq->q;
	156	struct elevator_queue *e = q->elevator;
	157
	158	if (rq->rq_flags & RQF_ELVPRIV) {
	159	blk_mq_sched_put_rq_priv(rq->q, rq);
	160	if (rq->elv.icq) {
	161	put_io_context(rq->elv.icq->ioc);
	162	rq->elv.icq = NULL;
	163	}
	164	}
	165
	166	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
	167	e->type->ops.mq.put_request(rq);
	168	else
	169	blk_mq_finish_request(rq);
	170	}
	171
	172	void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
	173	{
81380ca1 OS	174	struct request_queue *q = hctx->queue;
81380ca1 OS	175	struct elevator_queue *e = q->elevator;
64765a75 JA	176	const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
64765a75 JA	177	bool did_work = false;
bd166ef1 JA	178	LIST_HEAD(rq_list);
	179
	180	if (unlikely(blk_mq_hctx_stopped(hctx)))
	181	return;
	182
	183	hctx->run++;
	184
	185	/*
	186	* If we have previous entries on our dispatch list, grab them first for
	187	* more fair dispatch.
	188	*/
	189	if (!list_empty_careful(&hctx->dispatch)) {
	190	spin_lock(&hctx->lock);
	191	if (!list_empty(&hctx->dispatch))
	192	list_splice_init(&hctx->dispatch, &rq_list);
	193	spin_unlock(&hctx->lock);
	194	}
	195
	196	/*
	197	* Only ask the scheduler for requests, if we didn't have residual
	198	* requests from the dispatch list. This is to avoid the case where
	199	* we only ever dispatch a fraction of the requests available because
	200	* of low device queue depth. Once we pull requests out of the IO
	201	* scheduler, we can no longer merge or sort them. So it's best to
	202	* leave them there for as long as we can. Mark the hw queue as
	203	* needing a restart in that case.
	204	*/
c13660a0	205	if (!list_empty(&rq_list)) {
d38d3515	206	blk_mq_sched_mark_restart_hctx(hctx);
81380ca1	207	did_work = blk_mq_dispatch_rq_list(q, &rq_list);
64765a75	208	} else if (!has_sched_dispatch) {
c13660a0	209	blk_mq_flush_busy_ctxs(hctx, &rq_list);
81380ca1	210	blk_mq_dispatch_rq_list(q, &rq_list);
64765a75 JA	211	}
	212
	213	/*
	214	* We want to dispatch from the scheduler if we had no work left
	215	* on the dispatch list, OR if we did have work but weren't able
	216	* to make progress.
	217	*/
	218	if (!did_work && has_sched_dispatch) {
c13660a0 JA	219	do {
	220	struct request *rq;
	221
	222	rq = e->type->ops.mq.dispatch_request(hctx);
	223	if (!rq)
	224	break;
	225	list_add(&rq->queuelist, &rq_list);
81380ca1	226	} while (blk_mq_dispatch_rq_list(q, &rq_list));
c13660a0	227	}
bd166ef1 JA	228	}
	229
	230	void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
	231	struct list_head *rq_list,
	232	struct request (get_rq)(struct blk_mq_hw_ctx *))
	233	{
	234	do {
	235	struct request *rq;
	236
	237	rq = get_rq(hctx);
	238	if (!rq)
	239	break;
	240
	241	list_add_tail(&rq->queuelist, rq_list);
	242	} while (1);
	243	}
	244	EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);
	245
e4d750c9 JA	246	bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio,
e4d750c9 JA	247	struct request **merged_request)
bd166ef1 JA	248	{
bd166ef1 JA	249	struct request *rq;
bd166ef1	250
34fe7c05 CH	251	switch (elv_merge(q, &rq, bio)) {
34fe7c05 CH	252	case ELEVATOR_BACK_MERGE:
bd166ef1 JA	253	if (!blk_mq_sched_allow_merge(q, rq, bio))
bd166ef1 JA	254	return false;
34fe7c05 CH	255	if (!bio_attempt_back_merge(q, rq, bio))
	256	return false;
	257	*merged_request = attempt_back_merge(q, rq);
	258	if (!*merged_request)
	259	elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
	260	return true;
	261	case ELEVATOR_FRONT_MERGE:
bd166ef1 JA	262	if (!blk_mq_sched_allow_merge(q, rq, bio))
bd166ef1 JA	263	return false;
34fe7c05 CH	264	if (!bio_attempt_front_merge(q, rq, bio))
	265	return false;
	266	*merged_request = attempt_front_merge(q, rq);
	267	if (!*merged_request)
	268	elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
	269	return true;
	270	default:
	271	return false;
bd166ef1	272	}
bd166ef1 JA	273	}
	274	EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
	275
	276	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
	277	{
	278	struct elevator_queue *e = q->elevator;
	279
	280	if (e->type->ops.mq.bio_merge) {
	281	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
	282	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	283
	284	blk_mq_put_ctx(ctx);
	285	return e->type->ops.mq.bio_merge(hctx, bio);
	286	}
	287
	288	return false;
	289	}
	290
	291	bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq)
	292	{
	293	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
	294	}
	295	EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
	296
	297	void blk_mq_sched_request_inserted(struct request *rq)
	298	{
	299	trace_block_rq_insert(rq->q, rq);
	300	}
	301	EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
	302
0cacba6c OS	303	static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
0cacba6c OS	304	struct request *rq)
bd166ef1 JA	305	{
	306	if (rq->tag == -1) {
	307	rq->rq_flags \|= RQF_SORTED;
	308	return false;
	309	}
	310
	311	/*
	312	* If we already have a real request tag, send directly to
	313	* the dispatch list.
	314	*/
	315	spin_lock(&hctx->lock);
	316	list_add(&rq->queuelist, &hctx->dispatch);
	317	spin_unlock(&hctx->lock);
	318	return true;
	319	}
bd166ef1	320
50e1dab8 JA	321	static void blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx)
	322	{
	323	if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) {
	324	clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	325	if (blk_mq_hctx_has_pending(hctx))
	326	blk_mq_run_hw_queue(hctx, true);
	327	}
	328	}
	329
	330	void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx)
	331	{
d38d3515	332	struct request_queue *q = hctx->queue;
50e1dab8 JA	333	unsigned int i;
50e1dab8 JA	334
d38d3515 OS	335	if (test_bit(QUEUE_FLAG_RESTART, &q->queue_flags)) {
	336	if (test_and_clear_bit(QUEUE_FLAG_RESTART, &q->queue_flags)) {
	337	queue_for_each_hw_ctx(q, hctx, i)
	338	blk_mq_sched_restart_hctx(hctx);
	339	}
	340	} else {
50e1dab8	341	blk_mq_sched_restart_hctx(hctx);
50e1dab8 JA	342	}
	343	}
	344
bd6737f1 JA	345	/*
	346	* Add flush/fua to the queue. If we fail getting a driver tag, then
	347	* punt to the requeue list. Requeue will re-invoke us from a context
	348	* that's safe to block from.
	349	*/
	350	static void blk_mq_sched_insert_flush(struct blk_mq_hw_ctx *hctx,
	351	struct request *rq, bool can_block)
	352	{
	353	if (blk_mq_get_driver_tag(rq, &hctx, can_block)) {
	354	blk_insert_flush(rq);
	355	blk_mq_run_hw_queue(hctx, true);
	356	} else
c7a571b4	357	blk_mq_add_to_requeue_list(rq, false, true);
bd6737f1 JA	358	}
	359
	360	void blk_mq_sched_insert_request(struct request *rq, bool at_head,
	361	bool run_queue, bool async, bool can_block)
	362	{
	363	struct request_queue *q = rq->q;
	364	struct elevator_queue *e = q->elevator;
	365	struct blk_mq_ctx *ctx = rq->mq_ctx;
	366	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	367
f3a8ab7d	368	if (rq->tag == -1 && op_is_flush(rq->cmd_flags)) {
bd6737f1 JA	369	blk_mq_sched_insert_flush(hctx, rq, can_block);
	370	return;
	371	}
	372
0cacba6c OS	373	if (e && blk_mq_sched_bypass_insert(hctx, rq))
	374	goto run;
	375
bd6737f1 JA	376	if (e && e->type->ops.mq.insert_requests) {
	377	LIST_HEAD(list);
	378
	379	list_add(&rq->queuelist, &list);
	380	e->type->ops.mq.insert_requests(hctx, &list, at_head);
	381	} else {
	382	spin_lock(&ctx->lock);
	383	__blk_mq_insert_request(hctx, rq, at_head);
	384	spin_unlock(&ctx->lock);
	385	}
	386
0cacba6c	387	run:
bd6737f1 JA	388	if (run_queue)
	389	blk_mq_run_hw_queue(hctx, async);
	390	}
	391
	392	void blk_mq_sched_insert_requests(struct request_queue *q,
	393	struct blk_mq_ctx *ctx,
	394	struct list_head *list, bool run_queue_async)
	395	{
	396	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	397	struct elevator_queue *e = hctx->queue->elevator;
	398
0cacba6c OS	399	if (e) {
	400	struct request rq, next;
	401
	402	/*
	403	* We bypass requests that already have a driver tag assigned,
	404	* which should only be flushes. Flushes are only ever inserted
	405	* as single requests, so we shouldn't ever hit the
	406	* WARN_ON_ONCE() below (but let's handle it just in case).
	407	*/
	408	list_for_each_entry_safe(rq, next, list, queuelist) {
	409	if (WARN_ON_ONCE(rq->tag != -1)) {
	410	list_del_init(&rq->queuelist);
	411	blk_mq_sched_bypass_insert(hctx, rq);
	412	}
	413	}
	414	}
	415
bd6737f1 JA	416	if (e && e->type->ops.mq.insert_requests)
	417	e->type->ops.mq.insert_requests(hctx, list, false);
	418	else
	419	blk_mq_insert_requests(hctx, ctx, list);
	420
	421	blk_mq_run_hw_queue(hctx, run_queue_async);
	422	}
	423
bd166ef1 JA	424	static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
	425	struct blk_mq_hw_ctx *hctx,
	426	unsigned int hctx_idx)
	427	{
	428	if (hctx->sched_tags) {
	429	blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
	430	blk_mq_free_rq_map(hctx->sched_tags);
	431	hctx->sched_tags = NULL;
	432	}
	433	}
	434
6917ff0b OS	435	static int blk_mq_sched_alloc_tags(struct request_queue *q,
	436	struct blk_mq_hw_ctx *hctx,
	437	unsigned int hctx_idx)
	438	{
	439	struct blk_mq_tag_set *set = q->tag_set;
	440	int ret;
	441
	442	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
	443	set->reserved_tags);
	444	if (!hctx->sched_tags)
	445	return -ENOMEM;
	446
	447	ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
	448	if (ret)
	449	blk_mq_sched_free_tags(set, hctx, hctx_idx);
	450
	451	return ret;
	452	}
	453
	454	void blk_mq_sched_teardown(struct request_queue *q)
bd166ef1 JA	455	{
	456	struct blk_mq_tag_set *set = q->tag_set;
	457	struct blk_mq_hw_ctx *hctx;
6917ff0b OS	458	int i;
	459
	460	queue_for_each_hw_ctx(q, hctx, i)
	461	blk_mq_sched_free_tags(set, hctx, i);
	462	}
	463
	464	int blk_mq_init_sched(struct request_queue q, struct elevator_type e)
	465	{
	466	struct blk_mq_hw_ctx *hctx;
	467	unsigned int i;
	468	int ret;
	469
	470	if (!e) {
	471	q->elevator = NULL;
	472	return 0;
	473	}
bd166ef1 JA	474
	475	/*
	476	* Default to 256, since we don't split into sync/async like the
	477	* old code did. Additionally, this is a per-hw queue depth.
	478	*/
	479	q->nr_requests = 2 * BLKDEV_MAX_RQ;
	480
bd166ef1	481	queue_for_each_hw_ctx(q, hctx, i) {
6917ff0b	482	ret = blk_mq_sched_alloc_tags(q, hctx, i);
bd166ef1	483	if (ret)
6917ff0b	484	goto err;
bd166ef1 JA	485	}
bd166ef1 JA	486
6917ff0b OS	487	ret = e->ops.mq.init_sched(q, e);
	488	if (ret)
	489	goto err;
bd166ef1 JA	490
bd166ef1 JA	491	return 0;
bd166ef1	492
6917ff0b OS	493	err:
	494	blk_mq_sched_teardown(q);
	495	return ret;
bd166ef1	496	}
d3484991 JA	497
	498	int blk_mq_sched_init(struct request_queue *q)
	499	{
	500	int ret;
	501
d3484991 JA	502	mutex_lock(&q->sysfs_lock);
	503	ret = elevator_init(q, NULL);
	504	mutex_unlock(&q->sysfs_lock);
	505
	506	return ret;
	507	}