[mirror_ubuntu-artful-kernel.git] / block / blk-mq-sched.c

/*
 * blk-mq scheduling framework
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/blk-mq.h>

#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-sched.h"
#include "blk-mq-tag.h"
#include "blk-wbt.h"

void blk_mq_sched_free_hctx_data(struct request_queue *q,
				 void (*exit)(struct blk_mq_hw_ctx *))
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (exit && hctx->sched_data)
			exit(hctx);
		kfree(hctx->sched_data);
		hctx->sched_data = NULL;
	}
}
EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);

static void __blk_mq_sched_assign_ioc(struct request_queue *q,
				      struct request *rq,
				      struct bio *bio,
				      struct io_context *ioc)
{
	struct io_cq *icq;

	spin_lock_irq(q->queue_lock);
	icq = ioc_lookup_icq(ioc, q);
	spin_unlock_irq(q->queue_lock);

	if (!icq) {
		icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
		if (!icq)
			return;
	}

	rq->elv.icq = icq;
	if (!blk_mq_sched_get_rq_priv(q, rq, bio)) {
		rq->rq_flags |= RQF_ELVPRIV;
		get_io_context(icq->ioc);
		return;
	}

	rq->elv.icq = NULL;
}

static void blk_mq_sched_assign_ioc(struct request_queue *q,
				    struct request *rq, struct bio *bio)
{
	struct io_context *ioc;

	ioc = rq_ioc(bio);
	if (ioc)
		__blk_mq_sched_assign_ioc(q, rq, bio, ioc);
}

/*
 * Mark a hardware queue as needing a restart. For shared queues, maintain
 * a count of how many hardware queues are marked for restart.
 */
static void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx)
{
	if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
		return;

	if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
		struct request_queue *q = hctx->queue;

		if (!test_and_set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
			atomic_inc(&q->shared_hctx_restart);
	} else
		set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
}

static bool blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx)
{
	if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
		return false;

	if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
		struct request_queue *q = hctx->queue;

		if (test_and_clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
			atomic_dec(&q->shared_hctx_restart);
	} else
		clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);

	if (blk_mq_hctx_has_pending(hctx)) {
		blk_mq_run_hw_queue(hctx, true);
		return true;
	}

	return false;
}

struct request *blk_mq_sched_get_request(struct request_queue *q,
					 struct bio *bio,
					 unsigned int op,
					 struct blk_mq_alloc_data *data)
{
	struct elevator_queue *e = q->elevator;
	struct request *rq;

	blk_queue_enter_live(q);
	data->q = q;
	if (likely(!data->ctx))
		data->ctx = blk_mq_get_ctx(q);
	if (likely(!data->hctx))
		data->hctx = blk_mq_map_queue(q, data->ctx->cpu);

	if (e) {
		data->flags |= BLK_MQ_REQ_INTERNAL;

		/*
		 * Flush requests are special and go directly to the
		 * dispatch list.
		 */
		if (!op_is_flush(op) && e->type->ops.mq.get_request) {
			rq = e->type->ops.mq.get_request(q, op, data);
			if (rq)
				rq->rq_flags |= RQF_QUEUED;
		} else
			rq = __blk_mq_alloc_request(data, op);
	} else {
		rq = __blk_mq_alloc_request(data, op);
	}

	if (rq) {
		if (!op_is_flush(op)) {
			rq->elv.icq = NULL;
			if (e && e->type->icq_cache)
				blk_mq_sched_assign_ioc(q, rq, bio);
		}
		data->hctx->queued++;
		return rq;
	}

	blk_queue_exit(q);
	return NULL;
}

void blk_mq_sched_put_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (rq->rq_flags & RQF_ELVPRIV) {
		blk_mq_sched_put_rq_priv(rq->q, rq);
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}

	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
		e->type->ops.mq.put_request(rq);
	else
		blk_mq_finish_request(rq);
}

void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct elevator_queue *e = q->elevator;
	const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
	bool did_work = false;
	LIST_HEAD(rq_list);

	if (unlikely(blk_mq_hctx_stopped(hctx)))
		return;

	hctx->run++;

	/*
	 * If we have previous entries on our dispatch list, grab them first for
	 * more fair dispatch.
	 */
	if (!list_empty_careful(&hctx->dispatch)) {
		spin_lock(&hctx->lock);
		if (!list_empty(&hctx->dispatch))
			list_splice_init(&hctx->dispatch, &rq_list);
		spin_unlock(&hctx->lock);
	}

	/*
	 * Only ask the scheduler for requests, if we didn't have residual
	 * requests from the dispatch list. This is to avoid the case where
	 * we only ever dispatch a fraction of the requests available because
	 * of low device queue depth. Once we pull requests out of the IO
	 * scheduler, we can no longer merge or sort them. So it's best to
	 * leave them there for as long as we can. Mark the hw queue as
	 * needing a restart in that case.
	 */
	if (!list_empty(&rq_list)) {
		blk_mq_sched_mark_restart_hctx(hctx);
		did_work = blk_mq_dispatch_rq_list(q, &rq_list);
	} else if (!has_sched_dispatch) {
		blk_mq_flush_busy_ctxs(hctx, &rq_list);
		blk_mq_dispatch_rq_list(q, &rq_list);
	}

	/*
	 * We want to dispatch from the scheduler if we had no work left
	 * on the dispatch list, OR if we did have work but weren't able
	 * to make progress.
	 */
	if (!did_work && has_sched_dispatch) {
		do {
			struct request *rq;

			rq = e->type->ops.mq.dispatch_request(hctx);
			if (!rq)
				break;
			list_add(&rq->queuelist, &rq_list);
		} while (blk_mq_dispatch_rq_list(q, &rq_list));
	}
}

bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
			    struct request **merged_request)
{
	struct request *rq;

	switch (elv_merge(q, &rq, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (!bio_attempt_back_merge(q, rq, bio))
			return false;
		*merged_request = attempt_back_merge(q, rq);
		if (!*merged_request)
			elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
		return true;
	case ELEVATOR_FRONT_MERGE:
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (!bio_attempt_front_merge(q, rq, bio))
			return false;
		*merged_request = attempt_front_merge(q, rq);
		if (!*merged_request)
			elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
		return true;
	default:
		return false;
	}
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);

bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.mq.bio_merge) {
		struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
		struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

		blk_mq_put_ctx(ctx);
		return e->type->ops.mq.bio_merge(hctx, bio);
	}

	return false;
}

bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq)
{
	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);

void blk_mq_sched_request_inserted(struct request *rq)
{
	trace_block_rq_insert(rq->q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);

static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	if (rq->tag == -1) {
		rq->rq_flags |= RQF_SORTED;
		return false;
	}

	/*
	 * If we already have a real request tag, send directly to
	 * the dispatch list.
	 */
	spin_lock(&hctx->lock);
	list_add(&rq->queuelist, &hctx->dispatch);
	spin_unlock(&hctx->lock);
	return true;
}

/**
 * list_for_each_entry_rcu_rr - iterate in a round-robin fashion over rcu list
 * @pos:    loop cursor.
 * @skip:   the list element that will not be examined. Iteration starts at
 *          @skip->next.
 * @head:   head of the list to examine. This list must have at least one
 *          element, namely @skip.
 * @member: name of the list_head structure within typeof(*pos).
 */
#define list_for_each_entry_rcu_rr(pos, skip, head, member)		\
	for ((pos) = (skip);						\
	     (pos = (pos)->member.next != (head) ? list_entry_rcu(	\
			(pos)->member.next, typeof(*pos), member) :	\
	      list_entry_rcu((pos)->member.next->next, typeof(*pos), member)), \
	     (pos) != (skip); )

/*
 * Called after a driver tag has been freed to check whether a hctx needs to
 * be restarted. Restarts @hctx if its tag set is not shared. Restarts hardware
 * queues in a round-robin fashion if the tag set of @hctx is shared with other
 * hardware queues.
 */
void blk_mq_sched_restart(struct blk_mq_hw_ctx *const hctx)
{
	struct blk_mq_tags *const tags = hctx->tags;
	struct blk_mq_tag_set *const set = hctx->queue->tag_set;
	struct request_queue *const queue = hctx->queue, *q;
	struct blk_mq_hw_ctx *hctx2;
	unsigned int i, j;

	if (set->flags & BLK_MQ_F_TAG_SHARED) {
		/*
		 * If this is 0, then we know that no hardware queues
		 * have RESTART marked. We're done.
		 */
		if (!atomic_read(&queue->shared_hctx_restart))
			return;

		rcu_read_lock();
		list_for_each_entry_rcu_rr(q, queue, &set->tag_list,
					   tag_set_list) {
			queue_for_each_hw_ctx(q, hctx2, i)
				if (hctx2->tags == tags &&
				    blk_mq_sched_restart_hctx(hctx2))
					goto done;
		}
		j = hctx->queue_num + 1;
		for (i = 0; i < queue->nr_hw_queues; i++, j++) {
			if (j == queue->nr_hw_queues)
				j = 0;
			hctx2 = queue->queue_hw_ctx[j];
			if (hctx2->tags == tags &&
			    blk_mq_sched_restart_hctx(hctx2))
				break;
		}
done:
		rcu_read_unlock();
	} else {
		blk_mq_sched_restart_hctx(hctx);
	}
}

/*
 * Add flush/fua to the queue. If we fail getting a driver tag, then
 * punt to the requeue list. Requeue will re-invoke us from a context
 * that's safe to block from.
 */
static void blk_mq_sched_insert_flush(struct blk_mq_hw_ctx *hctx,
				      struct request *rq, bool can_block)
{
	if (blk_mq_get_driver_tag(rq, &hctx, can_block)) {
		blk_insert_flush(rq);
		blk_mq_run_hw_queue(hctx, true);
	} else
		blk_mq_add_to_requeue_list(rq, false, true);
}

void blk_mq_sched_insert_request(struct request *rq, bool at_head,
				 bool run_queue, bool async, bool can_block)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

	if (rq->tag == -1 && op_is_flush(rq->cmd_flags)) {
		blk_mq_sched_insert_flush(hctx, rq, can_block);
		return;
	}

	if (e && blk_mq_sched_bypass_insert(hctx, rq))
		goto run;

	if (e && e->type->ops.mq.insert_requests) {
		LIST_HEAD(list);

		list_add(&rq->queuelist, &list);
		e->type->ops.mq.insert_requests(hctx, &list, at_head);
	} else {
		spin_lock(&ctx->lock);
		__blk_mq_insert_request(hctx, rq, at_head);
		spin_unlock(&ctx->lock);
	}

run:
	if (run_queue)
		blk_mq_run_hw_queue(hctx, async);
}

void blk_mq_sched_insert_requests(struct request_queue *q,
				  struct blk_mq_ctx *ctx,
				  struct list_head *list, bool run_queue_async)
{
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	struct elevator_queue *e = hctx->queue->elevator;

	if (e) {
		struct request *rq, *next;

		/*
		 * We bypass requests that already have a driver tag assigned,
		 * which should only be flushes. Flushes are only ever inserted
		 * as single requests, so we shouldn't ever hit the
		 * WARN_ON_ONCE() below (but let's handle it just in case).
		 */
		list_for_each_entry_safe(rq, next, list, queuelist) {
			if (WARN_ON_ONCE(rq->tag != -1)) {
				list_del_init(&rq->queuelist);
				blk_mq_sched_bypass_insert(hctx, rq);
			}
		}
	}

	if (e && e->type->ops.mq.insert_requests)
		e->type->ops.mq.insert_requests(hctx, list, false);
	else
		blk_mq_insert_requests(hctx, ctx, list);

	blk_mq_run_hw_queue(hctx, run_queue_async);
}

static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	if (hctx->sched_tags) {
		blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
		blk_mq_free_rq_map(hctx->sched_tags);
		hctx->sched_tags = NULL;
	}
}

static int blk_mq_sched_alloc_tags(struct request_queue *q,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	struct blk_mq_tag_set *set = q->tag_set;
	int ret;

	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
					       set->reserved_tags);
	if (!hctx->sched_tags)
		return -ENOMEM;

	ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
	if (ret)
		blk_mq_sched_free_tags(set, hctx, hctx_idx);

	return ret;
}

static void blk_mq_sched_tags_teardown(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_sched_free_tags(set, hctx, i);
}

int blk_mq_sched_init_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
			   unsigned int hctx_idx)
{
	struct elevator_queue *e = q->elevator;
	int ret;

	if (!e)
		return 0;

	ret = blk_mq_sched_alloc_tags(q, hctx, hctx_idx);
	if (ret)
		return ret;

	if (e->type->ops.mq.init_hctx) {
		ret = e->type->ops.mq.init_hctx(hctx, hctx_idx);
		if (ret) {
			blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx);
			return ret;
		}
	}

	blk_mq_debugfs_register_sched_hctx(q, hctx);

	return 0;
}

void blk_mq_sched_exit_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
			    unsigned int hctx_idx)
{
	struct elevator_queue *e = q->elevator;

	if (!e)
		return;

	blk_mq_debugfs_unregister_sched_hctx(hctx);

	if (e->type->ops.mq.exit_hctx && hctx->sched_data) {
		e->type->ops.mq.exit_hctx(hctx, hctx_idx);
		hctx->sched_data = NULL;
	}

	blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx);
}

int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
{
	struct blk_mq_hw_ctx *hctx;
	struct elevator_queue *eq;
	unsigned int i;
	int ret;

	if (!e) {
		q->elevator = NULL;
		return 0;
	}

	/*
	 * Default to 256, since we don't split into sync/async like the
	 * old code did. Additionally, this is a per-hw queue depth.
	 */
	q->nr_requests = 2 * BLKDEV_MAX_RQ;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_sched_alloc_tags(q, hctx, i);
		if (ret)
			goto err;
	}

	ret = e->ops.mq.init_sched(q, e);
	if (ret)
		goto err;

	blk_mq_debugfs_register_sched(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		if (e->ops.mq.init_hctx) {
			ret = e->ops.mq.init_hctx(hctx, i);
			if (ret) {
				eq = q->elevator;
				blk_mq_exit_sched(q, eq);
				kobject_put(&eq->kobj);
				return ret;
			}
		}
		blk_mq_debugfs_register_sched_hctx(q, hctx);
	}

	return 0;

err:
	blk_mq_sched_tags_teardown(q);
	q->elevator = NULL;
	return ret;
}

void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		blk_mq_debugfs_unregister_sched_hctx(hctx);
		if (e->type->ops.mq.exit_hctx && hctx->sched_data) {
			e->type->ops.mq.exit_hctx(hctx, i);
			hctx->sched_data = NULL;
		}
	}
	blk_mq_debugfs_unregister_sched(q);
	if (e->type->ops.mq.exit_sched)
		e->type->ops.mq.exit_sched(e);
	blk_mq_sched_tags_teardown(q);
	q->elevator = NULL;
}

int blk_mq_sched_init(struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = elevator_init(q, NULL);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
bd166ef1 JA	1	/*
	2	* blk-mq scheduling framework
	3	*
	4	* Copyright (C) 2016 Jens Axboe
	5	*/
	6	#include <linux/kernel.h>
	7	#include <linux/module.h>
	8	#include <linux/blk-mq.h>
	9
	10	#include <trace/events/block.h>
	11
	12	#include "blk.h"
	13	#include "blk-mq.h"
d332ce09	14	#include "blk-mq-debugfs.h"
bd166ef1 JA	15	#include "blk-mq-sched.h"
	16	#include "blk-mq-tag.h"
	17	#include "blk-wbt.h"
	18
	19	void blk_mq_sched_free_hctx_data(struct request_queue *q,
	20	void (exit)(struct blk_mq_hw_ctx ))
	21	{
	22	struct blk_mq_hw_ctx *hctx;
	23	int i;
	24
	25	queue_for_each_hw_ctx(q, hctx, i) {
	26	if (exit && hctx->sched_data)
	27	exit(hctx);
	28	kfree(hctx->sched_data);
	29	hctx->sched_data = NULL;
	30	}
	31	}
	32	EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
	33
bd166ef1	34	static void __blk_mq_sched_assign_ioc(struct request_queue *q,
f1ba8261 PV	35	struct request *rq,
	36	struct bio *bio,
	37	struct io_context *ioc)
bd166ef1 JA	38	{
	39	struct io_cq *icq;
	40
	41	spin_lock_irq(q->queue_lock);
	42	icq = ioc_lookup_icq(ioc, q);
	43	spin_unlock_irq(q->queue_lock);
	44
	45	if (!icq) {
	46	icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
	47	if (!icq)
	48	return;
	49	}
	50
	51	rq->elv.icq = icq;
f1ba8261	52	if (!blk_mq_sched_get_rq_priv(q, rq, bio)) {
bd166ef1 JA	53	rq->rq_flags \|= RQF_ELVPRIV;
	54	get_io_context(icq->ioc);
	55	return;
	56	}
	57
	58	rq->elv.icq = NULL;
	59	}
	60
	61	static void blk_mq_sched_assign_ioc(struct request_queue *q,
	62	struct request rq, struct bio bio)
	63	{
	64	struct io_context *ioc;
	65
	66	ioc = rq_ioc(bio);
	67	if (ioc)
f1ba8261	68	__blk_mq_sched_assign_ioc(q, rq, bio, ioc);
bd166ef1 JA	69	}
bd166ef1 JA	70
8e8320c9 JA	71	/*
	72	* Mark a hardware queue as needing a restart. For shared queues, maintain
	73	* a count of how many hardware queues are marked for restart.
	74	*/
	75	static void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx)
	76	{
	77	if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
	78	return;
	79
	80	if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
	81	struct request_queue *q = hctx->queue;
	82
	83	if (!test_and_set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
	84	atomic_inc(&q->shared_hctx_restart);
	85	} else
	86	set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	87	}
	88
	89	static bool blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx)
	90	{
	91	if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
	92	return false;
	93
	94	if (hctx->flags & BLK_MQ_F_TAG_SHARED) {
	95	struct request_queue *q = hctx->queue;
	96
	97	if (test_and_clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
	98	atomic_dec(&q->shared_hctx_restart);
	99	} else
	100	clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
	101
	102	if (blk_mq_hctx_has_pending(hctx)) {
	103	blk_mq_run_hw_queue(hctx, true);
	104	return true;
	105	}
	106
	107	return false;
	108	}
	109
bd166ef1 JA	110	struct request blk_mq_sched_get_request(struct request_queue q,
	111	struct bio *bio,
	112	unsigned int op,
	113	struct blk_mq_alloc_data *data)
	114	{
	115	struct elevator_queue *e = q->elevator;
bd166ef1	116	struct request *rq;
bd166ef1 JA	117
bd166ef1 JA	118	blk_queue_enter_live(q);
6d2809d5 OS	119	data->q = q;
	120	if (likely(!data->ctx))
	121	data->ctx = blk_mq_get_ctx(q);
	122	if (likely(!data->hctx))
	123	data->hctx = blk_mq_map_queue(q, data->ctx->cpu);
bd166ef1	124
9f2779bf	125	if (e) {
bd166ef1 JA	126	data->flags \|= BLK_MQ_REQ_INTERNAL;
	127
	128	/*
	129	* Flush requests are special and go directly to the
	130	* dispatch list.
	131	*/
f73f44eb	132	if (!op_is_flush(op) && e->type->ops.mq.get_request) {
bd166ef1 JA	133	rq = e->type->ops.mq.get_request(q, op, data);
	134	if (rq)
	135	rq->rq_flags \|= RQF_QUEUED;
	136	} else
	137	rq = __blk_mq_alloc_request(data, op);
	138	} else {
	139	rq = __blk_mq_alloc_request(data, op);
bd166ef1 JA	140	}
	141
	142	if (rq) {
f73f44eb	143	if (!op_is_flush(op)) {
bd166ef1 JA	144	rq->elv.icq = NULL;
	145	if (e && e->type->icq_cache)
	146	blk_mq_sched_assign_ioc(q, rq, bio);
	147	}
	148	data->hctx->queued++;
	149	return rq;
	150	}
	151
	152	blk_queue_exit(q);
	153	return NULL;
	154	}
	155
	156	void blk_mq_sched_put_request(struct request *rq)
	157	{
	158	struct request_queue *q = rq->q;
	159	struct elevator_queue *e = q->elevator;
	160
	161	if (rq->rq_flags & RQF_ELVPRIV) {
	162	blk_mq_sched_put_rq_priv(rq->q, rq);
	163	if (rq->elv.icq) {
	164	put_io_context(rq->elv.icq->ioc);
	165	rq->elv.icq = NULL;
	166	}
	167	}
	168
	169	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
	170	e->type->ops.mq.put_request(rq);
	171	else
	172	blk_mq_finish_request(rq);
	173	}
	174
	175	void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
	176	{
81380ca1 OS	177	struct request_queue *q = hctx->queue;
81380ca1 OS	178	struct elevator_queue *e = q->elevator;
64765a75 JA	179	const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
64765a75 JA	180	bool did_work = false;
bd166ef1 JA	181	LIST_HEAD(rq_list);
	182
	183	if (unlikely(blk_mq_hctx_stopped(hctx)))
	184	return;
	185
	186	hctx->run++;
	187
	188	/*
	189	* If we have previous entries on our dispatch list, grab them first for
	190	* more fair dispatch.
	191	*/
	192	if (!list_empty_careful(&hctx->dispatch)) {
	193	spin_lock(&hctx->lock);
	194	if (!list_empty(&hctx->dispatch))
	195	list_splice_init(&hctx->dispatch, &rq_list);
	196	spin_unlock(&hctx->lock);
	197	}
	198
	199	/*
	200	* Only ask the scheduler for requests, if we didn't have residual
	201	* requests from the dispatch list. This is to avoid the case where
	202	* we only ever dispatch a fraction of the requests available because
	203	* of low device queue depth. Once we pull requests out of the IO
	204	* scheduler, we can no longer merge or sort them. So it's best to
	205	* leave them there for as long as we can. Mark the hw queue as
	206	* needing a restart in that case.
	207	*/
c13660a0	208	if (!list_empty(&rq_list)) {
d38d3515	209	blk_mq_sched_mark_restart_hctx(hctx);
81380ca1	210	did_work = blk_mq_dispatch_rq_list(q, &rq_list);
64765a75	211	} else if (!has_sched_dispatch) {
c13660a0	212	blk_mq_flush_busy_ctxs(hctx, &rq_list);
81380ca1	213	blk_mq_dispatch_rq_list(q, &rq_list);
64765a75 JA	214	}
	215
	216	/*
	217	* We want to dispatch from the scheduler if we had no work left
	218	* on the dispatch list, OR if we did have work but weren't able
	219	* to make progress.
	220	*/
	221	if (!did_work && has_sched_dispatch) {
c13660a0 JA	222	do {
	223	struct request *rq;
	224
	225	rq = e->type->ops.mq.dispatch_request(hctx);
	226	if (!rq)
	227	break;
	228	list_add(&rq->queuelist, &rq_list);
81380ca1	229	} while (blk_mq_dispatch_rq_list(q, &rq_list));
c13660a0	230	}
bd166ef1 JA	231	}
bd166ef1 JA	232
e4d750c9 JA	233	bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio,
e4d750c9 JA	234	struct request **merged_request)
bd166ef1 JA	235	{
bd166ef1 JA	236	struct request *rq;
bd166ef1	237
34fe7c05 CH	238	switch (elv_merge(q, &rq, bio)) {
34fe7c05 CH	239	case ELEVATOR_BACK_MERGE:
bd166ef1 JA	240	if (!blk_mq_sched_allow_merge(q, rq, bio))
bd166ef1 JA	241	return false;
34fe7c05 CH	242	if (!bio_attempt_back_merge(q, rq, bio))
	243	return false;
	244	*merged_request = attempt_back_merge(q, rq);
	245	if (!*merged_request)
	246	elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
	247	return true;
	248	case ELEVATOR_FRONT_MERGE:
bd166ef1 JA	249	if (!blk_mq_sched_allow_merge(q, rq, bio))
bd166ef1 JA	250	return false;
34fe7c05 CH	251	if (!bio_attempt_front_merge(q, rq, bio))
	252	return false;
	253	*merged_request = attempt_front_merge(q, rq);
	254	if (!*merged_request)
	255	elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
	256	return true;
	257	default:
	258	return false;
bd166ef1	259	}
bd166ef1 JA	260	}
	261	EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
	262
	263	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
	264	{
	265	struct elevator_queue *e = q->elevator;
	266
	267	if (e->type->ops.mq.bio_merge) {
	268	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
	269	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	270
	271	blk_mq_put_ctx(ctx);
	272	return e->type->ops.mq.bio_merge(hctx, bio);
	273	}
	274
	275	return false;
	276	}
	277
	278	bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq)
	279	{
	280	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
	281	}
	282	EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
	283
	284	void blk_mq_sched_request_inserted(struct request *rq)
	285	{
	286	trace_block_rq_insert(rq->q, rq);
	287	}
	288	EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
	289
0cacba6c OS	290	static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
0cacba6c OS	291	struct request *rq)
bd166ef1 JA	292	{
	293	if (rq->tag == -1) {
	294	rq->rq_flags \|= RQF_SORTED;
	295	return false;
	296	}
	297
	298	/*
	299	* If we already have a real request tag, send directly to
	300	* the dispatch list.
	301	*/
	302	spin_lock(&hctx->lock);
	303	list_add(&rq->queuelist, &hctx->dispatch);
	304	spin_unlock(&hctx->lock);
	305	return true;
	306	}
bd166ef1	307
6d8c6c0f BVA	308	/**
	309	* list_for_each_entry_rcu_rr - iterate in a round-robin fashion over rcu list
	310	* @pos: loop cursor.
	311	* @skip: the list element that will not be examined. Iteration starts at
	312	* @skip->next.
	313	* @head: head of the list to examine. This list must have at least one
	314	* element, namely @skip.
	315	* @member: name of the list_head structure within typeof(*pos).
	316	*/
	317	#define list_for_each_entry_rcu_rr(pos, skip, head, member) \
	318	for ((pos) = (skip); \
	319	(pos = (pos)->member.next != (head) ? list_entry_rcu( \
	320	(pos)->member.next, typeof(*pos), member) : \
	321	list_entry_rcu((pos)->member.next->next, typeof(*pos), member)), \
	322	(pos) != (skip); )
50e1dab8	323
6d8c6c0f BVA	324	/*
	325	* Called after a driver tag has been freed to check whether a hctx needs to
	326	* be restarted. Restarts @hctx if its tag set is not shared. Restarts hardware
	327	* queues in a round-robin fashion if the tag set of @hctx is shared with other
	328	* hardware queues.
	329	*/
	330	void blk_mq_sched_restart(struct blk_mq_hw_ctx *const hctx)
	331	{
	332	struct blk_mq_tags *const tags = hctx->tags;
	333	struct blk_mq_tag_set *const set = hctx->queue->tag_set;
	334	struct request_queue const queue = hctx->queue, q;
	335	struct blk_mq_hw_ctx *hctx2;
	336	unsigned int i, j;
	337
	338	if (set->flags & BLK_MQ_F_TAG_SHARED) {
8e8320c9 JA	339	/*
	340	* If this is 0, then we know that no hardware queues
	341	* have RESTART marked. We're done.
	342	*/
	343	if (!atomic_read(&queue->shared_hctx_restart))
	344	return;
	345
6d8c6c0f BVA	346	rcu_read_lock();
	347	list_for_each_entry_rcu_rr(q, queue, &set->tag_list,
	348	tag_set_list) {
	349	queue_for_each_hw_ctx(q, hctx2, i)
	350	if (hctx2->tags == tags &&
	351	blk_mq_sched_restart_hctx(hctx2))
	352	goto done;
	353	}
	354	j = hctx->queue_num + 1;
	355	for (i = 0; i < queue->nr_hw_queues; i++, j++) {
	356	if (j == queue->nr_hw_queues)
	357	j = 0;
	358	hctx2 = queue->queue_hw_ctx[j];
	359	if (hctx2->tags == tags &&
	360	blk_mq_sched_restart_hctx(hctx2))
	361	break;
d38d3515	362	}
6d8c6c0f BVA	363	done:
6d8c6c0f BVA	364	rcu_read_unlock();
d38d3515	365	} else {
50e1dab8	366	blk_mq_sched_restart_hctx(hctx);
50e1dab8 JA	367	}
	368	}
	369
bd6737f1 JA	370	/*
	371	* Add flush/fua to the queue. If we fail getting a driver tag, then
	372	* punt to the requeue list. Requeue will re-invoke us from a context
	373	* that's safe to block from.
	374	*/
	375	static void blk_mq_sched_insert_flush(struct blk_mq_hw_ctx *hctx,
	376	struct request *rq, bool can_block)
	377	{
	378	if (blk_mq_get_driver_tag(rq, &hctx, can_block)) {
	379	blk_insert_flush(rq);
	380	blk_mq_run_hw_queue(hctx, true);
	381	} else
c7a571b4	382	blk_mq_add_to_requeue_list(rq, false, true);
bd6737f1 JA	383	}
	384
	385	void blk_mq_sched_insert_request(struct request *rq, bool at_head,
	386	bool run_queue, bool async, bool can_block)
	387	{
	388	struct request_queue *q = rq->q;
	389	struct elevator_queue *e = q->elevator;
	390	struct blk_mq_ctx *ctx = rq->mq_ctx;
	391	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	392
f3a8ab7d	393	if (rq->tag == -1 && op_is_flush(rq->cmd_flags)) {
bd6737f1 JA	394	blk_mq_sched_insert_flush(hctx, rq, can_block);
	395	return;
	396	}
	397
0cacba6c OS	398	if (e && blk_mq_sched_bypass_insert(hctx, rq))
	399	goto run;
	400
bd6737f1 JA	401	if (e && e->type->ops.mq.insert_requests) {
	402	LIST_HEAD(list);
	403
	404	list_add(&rq->queuelist, &list);
	405	e->type->ops.mq.insert_requests(hctx, &list, at_head);
	406	} else {
	407	spin_lock(&ctx->lock);
	408	__blk_mq_insert_request(hctx, rq, at_head);
	409	spin_unlock(&ctx->lock);
	410	}
	411
0cacba6c	412	run:
bd6737f1 JA	413	if (run_queue)
	414	blk_mq_run_hw_queue(hctx, async);
	415	}
	416
	417	void blk_mq_sched_insert_requests(struct request_queue *q,
	418	struct blk_mq_ctx *ctx,
	419	struct list_head *list, bool run_queue_async)
	420	{
	421	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	422	struct elevator_queue *e = hctx->queue->elevator;
	423
0cacba6c OS	424	if (e) {
	425	struct request rq, next;
	426
	427	/*
	428	* We bypass requests that already have a driver tag assigned,
	429	* which should only be flushes. Flushes are only ever inserted
	430	* as single requests, so we shouldn't ever hit the
	431	* WARN_ON_ONCE() below (but let's handle it just in case).
	432	*/
	433	list_for_each_entry_safe(rq, next, list, queuelist) {
	434	if (WARN_ON_ONCE(rq->tag != -1)) {
	435	list_del_init(&rq->queuelist);
	436	blk_mq_sched_bypass_insert(hctx, rq);
	437	}
	438	}
	439	}
	440
bd6737f1 JA	441	if (e && e->type->ops.mq.insert_requests)
	442	e->type->ops.mq.insert_requests(hctx, list, false);
	443	else
	444	blk_mq_insert_requests(hctx, ctx, list);
	445
	446	blk_mq_run_hw_queue(hctx, run_queue_async);
	447	}
	448
bd166ef1 JA	449	static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
	450	struct blk_mq_hw_ctx *hctx,
	451	unsigned int hctx_idx)
	452	{
	453	if (hctx->sched_tags) {
	454	blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
	455	blk_mq_free_rq_map(hctx->sched_tags);
	456	hctx->sched_tags = NULL;
	457	}
	458	}
	459
6917ff0b OS	460	static int blk_mq_sched_alloc_tags(struct request_queue *q,
	461	struct blk_mq_hw_ctx *hctx,
	462	unsigned int hctx_idx)
	463	{
	464	struct blk_mq_tag_set *set = q->tag_set;
	465	int ret;
	466
	467	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
	468	set->reserved_tags);
	469	if (!hctx->sched_tags)
	470	return -ENOMEM;
	471
	472	ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
	473	if (ret)
	474	blk_mq_sched_free_tags(set, hctx, hctx_idx);
	475
	476	return ret;
	477	}
	478
54d5329d	479	static void blk_mq_sched_tags_teardown(struct request_queue *q)
bd166ef1 JA	480	{
	481	struct blk_mq_tag_set *set = q->tag_set;
	482	struct blk_mq_hw_ctx *hctx;
6917ff0b OS	483	int i;
	484
	485	queue_for_each_hw_ctx(q, hctx, i)
	486	blk_mq_sched_free_tags(set, hctx, i);
	487	}
	488
93252632 OS	489	int blk_mq_sched_init_hctx(struct request_queue q, struct blk_mq_hw_ctx hctx,
	490	unsigned int hctx_idx)
	491	{
	492	struct elevator_queue *e = q->elevator;
ee056f98	493	int ret;
93252632 OS	494
	495	if (!e)
	496	return 0;
	497
ee056f98 OS	498	ret = blk_mq_sched_alloc_tags(q, hctx, hctx_idx);
	499	if (ret)
	500	return ret;
	501
	502	if (e->type->ops.mq.init_hctx) {
	503	ret = e->type->ops.mq.init_hctx(hctx, hctx_idx);
	504	if (ret) {
	505	blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx);
	506	return ret;
	507	}
	508	}
	509
d332ce09 OS	510	blk_mq_debugfs_register_sched_hctx(q, hctx);
d332ce09 OS	511
ee056f98	512	return 0;
93252632 OS	513	}
	514
	515	void blk_mq_sched_exit_hctx(struct request_queue q, struct blk_mq_hw_ctx hctx,
	516	unsigned int hctx_idx)
	517	{
	518	struct elevator_queue *e = q->elevator;
	519
	520	if (!e)
	521	return;
	522
d332ce09 OS	523	blk_mq_debugfs_unregister_sched_hctx(hctx);
d332ce09 OS	524
ee056f98 OS	525	if (e->type->ops.mq.exit_hctx && hctx->sched_data) {
	526	e->type->ops.mq.exit_hctx(hctx, hctx_idx);
	527	hctx->sched_data = NULL;
	528	}
	529
93252632 OS	530	blk_mq_sched_free_tags(q->tag_set, hctx, hctx_idx);
	531	}
	532
6917ff0b OS	533	int blk_mq_init_sched(struct request_queue q, struct elevator_type e)
	534	{
	535	struct blk_mq_hw_ctx *hctx;
ee056f98	536	struct elevator_queue *eq;
6917ff0b OS	537	unsigned int i;
	538	int ret;
	539
	540	if (!e) {
	541	q->elevator = NULL;
	542	return 0;
	543	}
bd166ef1 JA	544
	545	/*
	546	* Default to 256, since we don't split into sync/async like the
	547	* old code did. Additionally, this is a per-hw queue depth.
	548	*/
	549	q->nr_requests = 2 * BLKDEV_MAX_RQ;
	550
bd166ef1	551	queue_for_each_hw_ctx(q, hctx, i) {
6917ff0b	552	ret = blk_mq_sched_alloc_tags(q, hctx, i);
bd166ef1	553	if (ret)
6917ff0b	554	goto err;
bd166ef1 JA	555	}
bd166ef1 JA	556
6917ff0b OS	557	ret = e->ops.mq.init_sched(q, e);
	558	if (ret)
	559	goto err;
bd166ef1	560
d332ce09 OS	561	blk_mq_debugfs_register_sched(q);
	562
	563	queue_for_each_hw_ctx(q, hctx, i) {
	564	if (e->ops.mq.init_hctx) {
ee056f98 OS	565	ret = e->ops.mq.init_hctx(hctx, i);
	566	if (ret) {
	567	eq = q->elevator;
	568	blk_mq_exit_sched(q, eq);
	569	kobject_put(&eq->kobj);
	570	return ret;
	571	}
	572	}
d332ce09	573	blk_mq_debugfs_register_sched_hctx(q, hctx);
ee056f98 OS	574	}
ee056f98 OS	575
bd166ef1	576	return 0;
bd166ef1	577
6917ff0b	578	err:
54d5329d OS	579	blk_mq_sched_tags_teardown(q);
54d5329d OS	580	q->elevator = NULL;
6917ff0b	581	return ret;
bd166ef1	582	}
d3484991	583
54d5329d OS	584	void blk_mq_exit_sched(struct request_queue q, struct elevator_queue e)
54d5329d OS	585	{
ee056f98 OS	586	struct blk_mq_hw_ctx *hctx;
	587	unsigned int i;
	588
d332ce09 OS	589	queue_for_each_hw_ctx(q, hctx, i) {
	590	blk_mq_debugfs_unregister_sched_hctx(hctx);
	591	if (e->type->ops.mq.exit_hctx && hctx->sched_data) {
	592	e->type->ops.mq.exit_hctx(hctx, i);
	593	hctx->sched_data = NULL;
ee056f98 OS	594	}
ee056f98 OS	595	}
d332ce09	596	blk_mq_debugfs_unregister_sched(q);
54d5329d OS	597	if (e->type->ops.mq.exit_sched)
	598	e->type->ops.mq.exit_sched(e);
	599	blk_mq_sched_tags_teardown(q);
	600	q->elevator = NULL;
	601	}
	602
d3484991 JA	603	int blk_mq_sched_init(struct request_queue *q)
	604	{
	605	int ret;
	606
d3484991 JA	607	mutex_lock(&q->sysfs_lock);
	608	ret = elevator_init(q, NULL);
	609	mutex_unlock(&q->sysfs_lock);
	610
	611	return ret;
	612	}