[mirror_ubuntu-bionic-kernel.git] / block / blk-mq.h

#ifndef INT_BLK_MQ_H
#define INT_BLK_MQ_H

#include "blk-stat.h"

struct blk_mq_tag_set;

struct blk_mq_ctx {
	struct {
		spinlock_t		lock;
		struct list_head	rq_list;
	}  ____cacheline_aligned_in_smp;

	unsigned int		cpu;
	unsigned int		index_hw;

	/* incremented at dispatch time */
	unsigned long		rq_dispatched[2];
	unsigned long		rq_merged;

	/* incremented at completion time */
	unsigned long		____cacheline_aligned_in_smp rq_completed[2];
	struct blk_rq_stat	stat[2];

	struct request_queue	*queue;
	struct kobject		kobj;
} ____cacheline_aligned_in_smp;

void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_free_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
void blk_mq_wake_waiters(struct request_queue *q);
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *, struct list_head *);

/*
 * CPU hotplug helpers
 */
void blk_mq_enable_hotplug(void);
void blk_mq_disable_hotplug(void);

/*
 * CPU -> queue mappings
 */
extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);

static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
		int cpu)
{
	return q->queue_hw_ctx[q->mq_map[cpu]];
}

/*
 * sysfs helpers
 */
extern int blk_mq_sysfs_register(struct request_queue *q);
extern void blk_mq_sysfs_unregister(struct request_queue *q);
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);

extern void blk_mq_rq_timed_out(struct request *req, bool reserved);

void blk_mq_release(struct request_queue *q);

static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
					   unsigned int cpu)
{
	return per_cpu_ptr(q->queue_ctx, cpu);
}

/*
 * This assumes per-cpu software queueing queues. They could be per-node
 * as well, for instance. For now this is hardcoded as-is. Note that we don't
 * care about preemption, since we know the ctx's are persistent. This does
 * mean that we can't rely on ctx always matching the currently running CPU.
 */
static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
{
	return __blk_mq_get_ctx(q, get_cpu());
}

static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
{
	put_cpu();
}

struct blk_mq_alloc_data {
	/* input parameter */
	struct request_queue *q;
	unsigned int flags;

	/* input & output parameter */
	struct blk_mq_ctx *ctx;
	struct blk_mq_hw_ctx *hctx;
};

static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
		struct request_queue *q, unsigned int flags,
		struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx)
{
	data->q = q;
	data->flags = flags;
	data->ctx = ctx;
	data->hctx = hctx;
}

static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
{
	return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
}

static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
{
	return hctx->nr_ctx && hctx->tags;
}

#endif
Commit	Line	Data
	1	#ifndef INT_BLK_MQ_H
	2	#define INT_BLK_MQ_H
	3
	4	#include "blk-stat.h"
	5
	6	struct blk_mq_tag_set;
	7
	8	struct blk_mq_ctx {
	9	struct {
	10	spinlock_t lock;
	11	struct list_head rq_list;
	12	} ____cacheline_aligned_in_smp;
	13
	14	unsigned int cpu;
	15	unsigned int index_hw;
	16
	17	/* incremented at dispatch time */
	18	unsigned long rq_dispatched[2];
	19	unsigned long rq_merged;
	20
	21	/* incremented at completion time */
	22	unsigned long ____cacheline_aligned_in_smp rq_completed[2];
	23	struct blk_rq_stat stat[2];
	24
	25	struct request_queue *queue;
	26	struct kobject kobj;
	27	} ____cacheline_aligned_in_smp;
	28
	29	void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
	30	void blk_mq_freeze_queue(struct request_queue *q);
	31	void blk_mq_free_queue(struct request_queue *q);
	32	int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
	33	void blk_mq_wake_waiters(struct request_queue *q);
	34	bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx , struct list_head );
	35
	36	/*
	37	* CPU hotplug helpers
	38	*/
	39	void blk_mq_enable_hotplug(void);
	40	void blk_mq_disable_hotplug(void);
	41
	42	/*
	43	* CPU -> queue mappings
	44	*/
	45	extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
	46
	47	static inline struct blk_mq_hw_ctx blk_mq_map_queue(struct request_queue q,
	48	int cpu)
	49	{
	50	return q->queue_hw_ctx[q->mq_map[cpu]];
	51	}
	52
	53	/*
	54	* sysfs helpers
	55	*/
	56	extern int blk_mq_sysfs_register(struct request_queue *q);
	57	extern void blk_mq_sysfs_unregister(struct request_queue *q);
	58	extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
	59
	60	extern void blk_mq_rq_timed_out(struct request *req, bool reserved);
	61
	62	void blk_mq_release(struct request_queue *q);
	63
	64	static inline struct blk_mq_ctx __blk_mq_get_ctx(struct request_queue q,
	65	unsigned int cpu)
	66	{
	67	return per_cpu_ptr(q->queue_ctx, cpu);
	68	}
	69
	70	/*
	71	* This assumes per-cpu software queueing queues. They could be per-node
	72	* as well, for instance. For now this is hardcoded as-is. Note that we don't
	73	* care about preemption, since we know the ctx's are persistent. This does
	74	* mean that we can't rely on ctx always matching the currently running CPU.
	75	*/
	76	static inline struct blk_mq_ctx blk_mq_get_ctx(struct request_queue q)
	77	{
	78	return __blk_mq_get_ctx(q, get_cpu());
	79	}
	80
	81	static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
	82	{
	83	put_cpu();
	84	}
	85
	86	struct blk_mq_alloc_data {
	87	/* input parameter */
	88	struct request_queue *q;
	89	unsigned int flags;
	90
	91	/* input & output parameter */
	92	struct blk_mq_ctx *ctx;
	93	struct blk_mq_hw_ctx *hctx;
	94	};
	95
	96	static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
	97	struct request_queue *q, unsigned int flags,
	98	struct blk_mq_ctx ctx, struct blk_mq_hw_ctx hctx)
	99	{
	100	data->q = q;
	101	data->flags = flags;
	102	data->ctx = ctx;
	103	data->hctx = hctx;
	104	}
	105
	106	static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
	107	{
	108	return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
	109	}
	110
	111	static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
	112	{
	113	return hctx->nr_ctx && hctx->tags;
	114	}
	115
	116	#endif