[mirror_ubuntu-artful-kernel.git] / block / blk-mq-sched.c

/*
 * blk-mq scheduling framework
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/blk-mq.h>

#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-sched.h"
#include "blk-mq-tag.h"
#include "blk-wbt.h"

void blk_mq_sched_free_hctx_data(struct request_queue *q,
				 void (*exit)(struct blk_mq_hw_ctx *))
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (exit && hctx->sched_data)
			exit(hctx);
		kfree(hctx->sched_data);
		hctx->sched_data = NULL;
	}
}
EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);

int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
				int (*init)(struct blk_mq_hw_ctx *),
				void (*exit)(struct blk_mq_hw_ctx *))
{
	struct blk_mq_hw_ctx *hctx;
	int ret;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		hctx->sched_data = kmalloc_node(size, GFP_KERNEL, hctx->numa_node);
		if (!hctx->sched_data) {
			ret = -ENOMEM;
			goto error;
		}

		if (init) {
			ret = init(hctx);
			if (ret) {
				/*
				 * We don't want to give exit() a partially
				 * initialized sched_data. init() must clean up
				 * if it fails.
				 */
				kfree(hctx->sched_data);
				hctx->sched_data = NULL;
				goto error;
			}
		}
	}

	return 0;
error:
	blk_mq_sched_free_hctx_data(q, exit);
	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_sched_init_hctx_data);

static void __blk_mq_sched_assign_ioc(struct request_queue *q,
				      struct request *rq, struct io_context *ioc)
{
	struct io_cq *icq;

	spin_lock_irq(q->queue_lock);
	icq = ioc_lookup_icq(ioc, q);
	spin_unlock_irq(q->queue_lock);

	if (!icq) {
		icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
		if (!icq)
			return;
	}

	rq->elv.icq = icq;
	if (!blk_mq_sched_get_rq_priv(q, rq)) {
		rq->rq_flags |= RQF_ELVPRIV;
		get_io_context(icq->ioc);
		return;
	}

	rq->elv.icq = NULL;
}

static void blk_mq_sched_assign_ioc(struct request_queue *q,
				    struct request *rq, struct bio *bio)
{
	struct io_context *ioc;

	ioc = rq_ioc(bio);
	if (ioc)
		__blk_mq_sched_assign_ioc(q, rq, ioc);
}

struct request *blk_mq_sched_get_request(struct request_queue *q,
					 struct bio *bio,
					 unsigned int op,
					 struct blk_mq_alloc_data *data)
{
	struct elevator_queue *e = q->elevator;
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	const bool is_flush = op & (REQ_PREFLUSH | REQ_FUA);

	blk_queue_enter_live(q);
	ctx = blk_mq_get_ctx(q);
	hctx = blk_mq_map_queue(q, ctx->cpu);

	blk_mq_set_alloc_data(data, q, data->flags, ctx, hctx);

	if (e) {
		data->flags |= BLK_MQ_REQ_INTERNAL;

		/*
		 * Flush requests are special and go directly to the
		 * dispatch list.
		 */
		if (!is_flush && e->type->ops.mq.get_request) {
			rq = e->type->ops.mq.get_request(q, op, data);
			if (rq)
				rq->rq_flags |= RQF_QUEUED;
		} else
			rq = __blk_mq_alloc_request(data, op);
	} else {
		rq = __blk_mq_alloc_request(data, op);
		if (rq)
			data->hctx->tags->rqs[rq->tag] = rq;
	}

	if (rq) {
		if (!is_flush) {
			rq->elv.icq = NULL;
			if (e && e->type->icq_cache)
				blk_mq_sched_assign_ioc(q, rq, bio);
		}
		data->hctx->queued++;
		return rq;
	}

	blk_queue_exit(q);
	return NULL;
}

void blk_mq_sched_put_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (rq->rq_flags & RQF_ELVPRIV) {
		blk_mq_sched_put_rq_priv(rq->q, rq);
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}

	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
		e->type->ops.mq.put_request(rq);
	else
		blk_mq_finish_request(rq);
}

void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
{
	struct elevator_queue *e = hctx->queue->elevator;
	LIST_HEAD(rq_list);

	if (unlikely(blk_mq_hctx_stopped(hctx)))
		return;

	hctx->run++;

	/*
	 * If we have previous entries on our dispatch list, grab them first for
	 * more fair dispatch.
	 */
	if (!list_empty_careful(&hctx->dispatch)) {
		spin_lock(&hctx->lock);
		if (!list_empty(&hctx->dispatch))
			list_splice_init(&hctx->dispatch, &rq_list);
		spin_unlock(&hctx->lock);
	}

	/*
	 * Only ask the scheduler for requests, if we didn't have residual
	 * requests from the dispatch list. This is to avoid the case where
	 * we only ever dispatch a fraction of the requests available because
	 * of low device queue depth. Once we pull requests out of the IO
	 * scheduler, we can no longer merge or sort them. So it's best to
	 * leave them there for as long as we can. Mark the hw queue as
	 * needing a restart in that case.
	 */
	if (list_empty(&rq_list)) {
		if (e && e->type->ops.mq.dispatch_requests)
			e->type->ops.mq.dispatch_requests(hctx, &rq_list);
		else
			blk_mq_flush_busy_ctxs(hctx, &rq_list);
	} else
		blk_mq_sched_mark_restart(hctx);

	blk_mq_dispatch_rq_list(hctx, &rq_list);
}

void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
				   struct list_head *rq_list,
				   struct request *(*get_rq)(struct blk_mq_hw_ctx *))
{
	do {
		struct request *rq;

		rq = get_rq(hctx);
		if (!rq)
			break;

		list_add_tail(&rq->queuelist, rq_list);
	} while (1);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);

bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
{
	struct request *rq;
	int ret;

	ret = elv_merge(q, &rq, bio);
	if (ret == ELEVATOR_BACK_MERGE) {
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (bio_attempt_back_merge(q, rq, bio)) {
			if (!attempt_back_merge(q, rq))
				elv_merged_request(q, rq, ret);
			return true;
		}
	} else if (ret == ELEVATOR_FRONT_MERGE) {
		if (!blk_mq_sched_allow_merge(q, rq, bio))
			return false;
		if (bio_attempt_front_merge(q, rq, bio)) {
			if (!attempt_front_merge(q, rq))
				elv_merged_request(q, rq, ret);
			return true;
		}
	}

	return false;
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);

bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.mq.bio_merge) {
		struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
		struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

		blk_mq_put_ctx(ctx);
		return e->type->ops.mq.bio_merge(hctx, bio);
	}

	return false;
}

bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq)
{
	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);

void blk_mq_sched_request_inserted(struct request *rq)
{
	trace_block_rq_insert(rq->q, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);

bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	if (rq->tag == -1) {
		rq->rq_flags |= RQF_SORTED;
		return false;
	}

	/*
	 * If we already have a real request tag, send directly to
	 * the dispatch list.
	 */
	spin_lock(&hctx->lock);
	list_add(&rq->queuelist, &hctx->dispatch);
	spin_unlock(&hctx->lock);
	return true;
}
EXPORT_SYMBOL_GPL(blk_mq_sched_bypass_insert);

static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	if (hctx->sched_tags) {
		blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
		blk_mq_free_rq_map(hctx->sched_tags);
		hctx->sched_tags = NULL;
	}
}

int blk_mq_sched_setup(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	/*
	 * Default to 256, since we don't split into sync/async like the
	 * old code did. Additionally, this is a per-hw queue depth.
	 */
	q->nr_requests = 2 * BLKDEV_MAX_RQ;

	/*
	 * We're switching to using an IO scheduler, so setup the hctx
	 * scheduler tags and switch the request map from the regular
	 * tags to scheduler tags. First allocate what we need, so we
	 * can safely fail and fallback, if needed.
	 */
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
		hctx->sched_tags = blk_mq_alloc_rq_map(set, i, q->nr_requests, 0);
		if (!hctx->sched_tags) {
			ret = -ENOMEM;
			break;
		}
		ret = blk_mq_alloc_rqs(set, hctx->sched_tags, i, q->nr_requests);
		if (ret)
			break;
	}

	/*
	 * If we failed, free what we did allocate
	 */
	if (ret) {
		queue_for_each_hw_ctx(q, hctx, i) {
			if (!hctx->sched_tags)
				continue;
			blk_mq_sched_free_tags(set, hctx, i);
		}

		return ret;
	}

	return 0;
}

void blk_mq_sched_teardown(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_sched_free_tags(set, hctx, i);
}

int blk_mq_sched_init(struct request_queue *q)
{
	int ret;

#if defined(CONFIG_DEFAULT_SQ_NONE)
	if (q->nr_hw_queues == 1)
		return 0;
#endif
#if defined(CONFIG_DEFAULT_MQ_NONE)
	if (q->nr_hw_queues > 1)
		return 0;
#endif

	mutex_lock(&q->sysfs_lock);
	ret = elevator_init(q, NULL);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
bd166ef1 JA	1	/*
	2	* blk-mq scheduling framework
	3	*
	4	* Copyright (C) 2016 Jens Axboe
	5	*/
	6	#include <linux/kernel.h>
	7	#include <linux/module.h>
	8	#include <linux/blk-mq.h>
	9
	10	#include <trace/events/block.h>
	11
	12	#include "blk.h"
	13	#include "blk-mq.h"
	14	#include "blk-mq-sched.h"
	15	#include "blk-mq-tag.h"
	16	#include "blk-wbt.h"
	17
	18	void blk_mq_sched_free_hctx_data(struct request_queue *q,
	19	void (exit)(struct blk_mq_hw_ctx ))
	20	{
	21	struct blk_mq_hw_ctx *hctx;
	22	int i;
	23
	24	queue_for_each_hw_ctx(q, hctx, i) {
	25	if (exit && hctx->sched_data)
	26	exit(hctx);
	27	kfree(hctx->sched_data);
	28	hctx->sched_data = NULL;
	29	}
	30	}
	31	EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
	32
	33	int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
	34	int (init)(struct blk_mq_hw_ctx ),
	35	void (exit)(struct blk_mq_hw_ctx ))
	36	{
	37	struct blk_mq_hw_ctx *hctx;
	38	int ret;
	39	int i;
	40
	41	queue_for_each_hw_ctx(q, hctx, i) {
	42	hctx->sched_data = kmalloc_node(size, GFP_KERNEL, hctx->numa_node);
	43	if (!hctx->sched_data) {
	44	ret = -ENOMEM;
	45	goto error;
	46	}
	47
	48	if (init) {
	49	ret = init(hctx);
	50	if (ret) {
	51	/*
	52	* We don't want to give exit() a partially
	53	* initialized sched_data. init() must clean up
	54	* if it fails.
	55	*/
	56	kfree(hctx->sched_data);
	57	hctx->sched_data = NULL;
	58	goto error;
	59	}
	60	}
	61	}
	62
	63	return 0;
	64	error:
65	blk_mq_sched_free_hctx_data(q, exit);
66	return ret;
67	}
68	EXPORT_SYMBOL_GPL(blk_mq_sched_init_hctx_data);
69
70	static void __blk_mq_sched_assign_ioc(struct request_queue *q,
71	struct request rq, struct io_context ioc)
72	{
73	struct io_cq *icq;
74
75	spin_lock_irq(q->queue_lock);
76	icq = ioc_lookup_icq(ioc, q);
77	spin_unlock_irq(q->queue_lock);
78
79	if (!icq) {
80	icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
81	if (!icq)
82	return;
83	}
84
85	rq->elv.icq = icq;
86	if (!blk_mq_sched_get_rq_priv(q, rq)) {
87	rq->rq_flags \|= RQF_ELVPRIV;
88	get_io_context(icq->ioc);
89	return;
90	}
91
92	rq->elv.icq = NULL;
93	}
94
95	static void blk_mq_sched_assign_ioc(struct request_queue *q,
96	struct request rq, struct bio bio)
97	{
98	struct io_context *ioc;
99
100	ioc = rq_ioc(bio);
101	if (ioc)
102	__blk_mq_sched_assign_ioc(q, rq, ioc);
103	}
104
105	struct request blk_mq_sched_get_request(struct request_queue q,
106	struct bio *bio,
107	unsigned int op,
108	struct blk_mq_alloc_data *data)
109	{
110	struct elevator_queue *e = q->elevator;
111	struct blk_mq_hw_ctx *hctx;
112	struct blk_mq_ctx *ctx;
113	struct request *rq;
114	const bool is_flush = op & (REQ_PREFLUSH \| REQ_FUA);
115
116	blk_queue_enter_live(q);
117	ctx = blk_mq_get_ctx(q);
118	hctx = blk_mq_map_queue(q, ctx->cpu);
119
5a797e00	120	blk_mq_set_alloc_data(data, q, data->flags, ctx, hctx);
bd166ef1 JA	121
	122	if (e) {
	123	data->flags \|= BLK_MQ_REQ_INTERNAL;
	124
	125	/*
	126	* Flush requests are special and go directly to the
	127	* dispatch list.
	128	*/
	129	if (!is_flush && e->type->ops.mq.get_request) {
	130	rq = e->type->ops.mq.get_request(q, op, data);
	131	if (rq)
	132	rq->rq_flags \|= RQF_QUEUED;
	133	} else
	134	rq = __blk_mq_alloc_request(data, op);
	135	} else {
	136	rq = __blk_mq_alloc_request(data, op);
b48fda09 JA	137	if (rq)
b48fda09 JA	138	data->hctx->tags->rqs[rq->tag] = rq;
bd166ef1 JA	139	}
	140
	141	if (rq) {
	142	if (!is_flush) {
	143	rq->elv.icq = NULL;
	144	if (e && e->type->icq_cache)
	145	blk_mq_sched_assign_ioc(q, rq, bio);
	146	}
	147	data->hctx->queued++;
	148	return rq;
	149	}
	150
	151	blk_queue_exit(q);
	152	return NULL;
	153	}
	154
	155	void blk_mq_sched_put_request(struct request *rq)
	156	{
	157	struct request_queue *q = rq->q;
	158	struct elevator_queue *e = q->elevator;
	159
	160	if (rq->rq_flags & RQF_ELVPRIV) {
	161	blk_mq_sched_put_rq_priv(rq->q, rq);
	162	if (rq->elv.icq) {
	163	put_io_context(rq->elv.icq->ioc);
	164	rq->elv.icq = NULL;
	165	}
	166	}
	167
	168	if ((rq->rq_flags & RQF_QUEUED) && e && e->type->ops.mq.put_request)
	169	e->type->ops.mq.put_request(rq);
	170	else
	171	blk_mq_finish_request(rq);
	172	}
	173
	174	void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
	175	{
	176	struct elevator_queue *e = hctx->queue->elevator;
	177	LIST_HEAD(rq_list);
	178
	179	if (unlikely(blk_mq_hctx_stopped(hctx)))
	180	return;
	181
	182	hctx->run++;
	183
	184	/*
	185	* If we have previous entries on our dispatch list, grab them first for
	186	* more fair dispatch.
	187	*/
	188	if (!list_empty_careful(&hctx->dispatch)) {
	189	spin_lock(&hctx->lock);
	190	if (!list_empty(&hctx->dispatch))
	191	list_splice_init(&hctx->dispatch, &rq_list);
	192	spin_unlock(&hctx->lock);
	193	}
	194
	195	/*
	196	* Only ask the scheduler for requests, if we didn't have residual
	197	* requests from the dispatch list. This is to avoid the case where
	198	* we only ever dispatch a fraction of the requests available because
	199	* of low device queue depth. Once we pull requests out of the IO
	200	* scheduler, we can no longer merge or sort them. So it's best to
	201	* leave them there for as long as we can. Mark the hw queue as
	202	* needing a restart in that case.
203	*/
204	if (list_empty(&rq_list)) {
205	if (e && e->type->ops.mq.dispatch_requests)
206	e->type->ops.mq.dispatch_requests(hctx, &rq_list);
207	else
208	blk_mq_flush_busy_ctxs(hctx, &rq_list);
209	} else
210	blk_mq_sched_mark_restart(hctx);
211
212	blk_mq_dispatch_rq_list(hctx, &rq_list);
213	}
214
215	void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
216	struct list_head *rq_list,
217	struct request (get_rq)(struct blk_mq_hw_ctx *))
218	{
219	do {
220	struct request *rq;
221
222	rq = get_rq(hctx);
223	if (!rq)
224	break;
225
226	list_add_tail(&rq->queuelist, rq_list);
227	} while (1);
228	}
229	EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);
230
231	bool blk_mq_sched_try_merge(struct request_queue q, struct bio bio)
232	{
233	struct request *rq;
234	int ret;
235
236	ret = elv_merge(q, &rq, bio);
237	if (ret == ELEVATOR_BACK_MERGE) {
238	if (!blk_mq_sched_allow_merge(q, rq, bio))
239	return false;
240	if (bio_attempt_back_merge(q, rq, bio)) {
241	if (!attempt_back_merge(q, rq))
242	elv_merged_request(q, rq, ret);
243	return true;
244	}
245	} else if (ret == ELEVATOR_FRONT_MERGE) {
246	if (!blk_mq_sched_allow_merge(q, rq, bio))
247	return false;
248	if (bio_attempt_front_merge(q, rq, bio)) {
249	if (!attempt_front_merge(q, rq))
250	elv_merged_request(q, rq, ret);
251	return true;
252	}
253	}
254
255	return false;
256	}
257	EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
258
259	bool __blk_mq_sched_bio_merge(struct request_queue q, struct bio bio)
260	{
261	struct elevator_queue *e = q->elevator;
262
263	if (e->type->ops.mq.bio_merge) {
264	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
265	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
266
267	blk_mq_put_ctx(ctx);
268	return e->type->ops.mq.bio_merge(hctx, bio);
269	}
270
271	return false;
272	}
273
274	bool blk_mq_sched_try_insert_merge(struct request_queue q, struct request rq)
275	{
276	return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
277	}
278	EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
279
280	void blk_mq_sched_request_inserted(struct request *rq)
281	{
282	trace_block_rq_insert(rq->q, rq);
283	}
284	EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
285
286	bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx hctx, struct request rq)
287	{
288	if (rq->tag == -1) {
289	rq->rq_flags \|= RQF_SORTED;
290	return false;
291	}
292
293	/*
294	* If we already have a real request tag, send directly to
295	* the dispatch list.
296	*/
297	spin_lock(&hctx->lock);
298	list_add(&rq->queuelist, &hctx->dispatch);
299	spin_unlock(&hctx->lock);
300	return true;
301	}
302	EXPORT_SYMBOL_GPL(blk_mq_sched_bypass_insert);
303
304	static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
305	struct blk_mq_hw_ctx *hctx,
306	unsigned int hctx_idx)
307	{
308	if (hctx->sched_tags) {
309	blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
310	blk_mq_free_rq_map(hctx->sched_tags);
311	hctx->sched_tags = NULL;
312	}
313	}
314
315	int blk_mq_sched_setup(struct request_queue *q)
316	{
317	struct blk_mq_tag_set *set = q->tag_set;
318	struct blk_mq_hw_ctx *hctx;
319	int ret, i;
320
321	/*
322	* Default to 256, since we don't split into sync/async like the
323	* old code did. Additionally, this is a per-hw queue depth.
324	*/
325	q->nr_requests = 2 * BLKDEV_MAX_RQ;
326
327	/*
328	* We're switching to using an IO scheduler, so setup the hctx
329	* scheduler tags and switch the request map from the regular
330	* tags to scheduler tags. First allocate what we need, so we
331	* can safely fail and fallback, if needed.
332	*/
333	ret = 0;
334	queue_for_each_hw_ctx(q, hctx, i) {
335	hctx->sched_tags = blk_mq_alloc_rq_map(set, i, q->nr_requests, 0);
336	if (!hctx->sched_tags) {
337	ret = -ENOMEM;
338	break;
339	}
340	ret = blk_mq_alloc_rqs(set, hctx->sched_tags, i, q->nr_requests);
341	if (ret)
342	break;
343	}
344
345	/*
346	* If we failed, free what we did allocate
347	*/
348	if (ret) {
349	queue_for_each_hw_ctx(q, hctx, i) {
350	if (!hctx->sched_tags)
351	continue;
352	blk_mq_sched_free_tags(set, hctx, i);
353	}
354
355	return ret;
356	}
357
358	return 0;
359	}
360
361	void blk_mq_sched_teardown(struct request_queue *q)
362	{
363	struct blk_mq_tag_set *set = q->tag_set;
364	struct blk_mq_hw_ctx *hctx;
365	int i;
366
367	queue_for_each_hw_ctx(q, hctx, i)
368	blk_mq_sched_free_tags(set, hctx, i);
369	}
d3484991 JA	370
	371	int blk_mq_sched_init(struct request_queue *q)
	372	{
	373	int ret;
	374
	375	#if defined(CONFIG_DEFAULT_SQ_NONE)
	376	if (q->nr_hw_queues == 1)
	377	return 0;
	378	#endif
	379	#if defined(CONFIG_DEFAULT_MQ_NONE)
	380	if (q->nr_hw_queues > 1)
	381	return 0;
	382	#endif
	383
	384	mutex_lock(&q->sysfs_lock);
	385	ret = elevator_init(q, NULL);
	386	mutex_unlock(&q->sysfs_lock);
	387
	388	return ret;
	389	}