[mirror_ubuntu-zesty-kernel.git] / block / blk-mq-sysfs.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
				ctx->rq_dispatched[0]);
}

static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu\n", ctx->rq_merged);
}

static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
				ctx->rq_completed[0]);
}

static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg)
{
	struct request *rq;
	int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg);

	list_for_each_entry(rq, list, queuelist) {
		const int rq_len = 2 * sizeof(rq) + 2;

		/* if the output will be truncated */
		if (PAGE_SIZE - 1 < len + rq_len) {
			/* backspacing if it can't hold '\t...\n' */
			if (PAGE_SIZE - 1 < len + 5)
				len -= rq_len;
			len += snprintf(page + len, PAGE_SIZE - 1 - len,
					"\t...\n");
			break;
		}
		len += snprintf(page + len, PAGE_SIZE - 1 - len,
				"\t%p\n", rq);
	}

	return len;
}

static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
{
	ssize_t ret;

	spin_lock(&ctx->lock);
	ret = sysfs_list_show(page, &ctx->rq_list, "CTX pending");
	spin_unlock(&ctx->lock);

	return ret;
}

static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
		       hctx->poll_considered, hctx->poll_invoked,
		       hctx->poll_success);
}

static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
					   char *page)
{
	return sprintf(page, "%lu\n", hctx->queued);
}

static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "%lu\n", hctx->run);
}

static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
					       char *page)
{
	char *start_page = page;
	int i;

	page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);

	for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER; i++) {
		unsigned long d = 1U << (i - 1);

		page += sprintf(page, "%8lu\t%lu\n", d, hctx->dispatched[i]);
	}

	return page - start_page;
}

static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	ssize_t ret;

	spin_lock(&hctx->lock);
	ret = sysfs_list_show(page, &hctx->dispatch, "HCTX pending");
	spin_unlock(&hctx->lock);

	return ret;
}

static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return blk_mq_tag_sysfs_show(hctx->tags, page);
}

static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
	.attr = {.name = "dispatched", .mode = S_IRUGO },
	.show = blk_mq_sysfs_dispatched_show,
};
static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
	.attr = {.name = "merged", .mode = S_IRUGO },
	.show = blk_mq_sysfs_merged_show,
};
static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
	.attr = {.name = "completed", .mode = S_IRUGO },
	.show = blk_mq_sysfs_completed_show,
};
static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list = {
	.attr = {.name = "rq_list", .mode = S_IRUGO },
	.show = blk_mq_sysfs_rq_list_show,
};

static struct attribute *default_ctx_attrs[] = {
	&blk_mq_sysfs_dispatched.attr,
	&blk_mq_sysfs_merged.attr,
	&blk_mq_sysfs_completed.attr,
	&blk_mq_sysfs_rq_list.attr,
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
	.attr = {.name = "queued", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_queued_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
	.attr = {.name = "run", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_run_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
	.attr = {.name = "dispatched", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_dispatched_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
	.attr = {.name = "active", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_active_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = {
	.attr = {.name = "pending", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_rq_list_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
	.attr = {.name = "tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_cpus_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
	.attr = {.name = "io_poll", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_poll_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_queued.attr,
	&blk_mq_hw_sysfs_run.attr,
	&blk_mq_hw_sysfs_dispatched.attr,
	&blk_mq_hw_sysfs_pending.attr,
	&blk_mq_hw_sysfs_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	&blk_mq_hw_sysfs_active.attr,
	&blk_mq_hw_sysfs_poll.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

static void __blk_mq_unregister_disk(struct gendisk *disk)
{
	struct request_queue *q = disk->queue;
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	int i, j;

	queue_for_each_hw_ctx(q, hctx, i) {
		blk_mq_unregister_hctx(hctx);

		hctx_for_each_ctx(hctx, ctx, j)
			kobject_put(&ctx->kobj);

		kobject_put(&hctx->kobj);
	}

	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	kobject_del(&q->mq_kobj);
	kobject_put(&q->mq_kobj);

	kobject_put(&disk_to_dev(disk)->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_unregister_disk(struct gendisk *disk)
{
	blk_mq_disable_hotplug();
	__blk_mq_unregister_disk(disk);
	blk_mq_enable_hotplug();
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

static void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(&q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int blk_mq_register_disk(struct gendisk *disk)
{
	struct device *dev = disk_to_dev(disk);
	struct request_queue *q = disk->queue;
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	blk_mq_disable_hotplug();

	blk_mq_sysfs_init(q);

	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(&q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

	if (ret)
		__blk_mq_unregister_disk(disk);
	else
		q->mq_sysfs_init_done = true;
out:
	blk_mq_enable_hotplug();

	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_register_disk);

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	if (!q->mq_sysfs_init_done)
		return;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	if (!q->mq_sysfs_init_done)
		return ret;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

	return ret;
}
Commit	Line	Data
320ae51f JA	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/backing-dev.h>
	4	#include <linux/bio.h>
	5	#include <linux/blkdev.h>
	6	#include <linux/mm.h>
	7	#include <linux/init.h>
	8	#include <linux/slab.h>
	9	#include <linux/workqueue.h>
	10	#include <linux/smp.h>
	11
	12	#include <linux/blk-mq.h>
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
	16	static void blk_mq_sysfs_release(struct kobject *kobj)
	17	{
	18	}
	19
	20	struct blk_mq_ctx_sysfs_entry {
	21	struct attribute attr;
	22	ssize_t (show)(struct blk_mq_ctx , char *);
	23	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	24	};
	25
	26	struct blk_mq_hw_ctx_sysfs_entry {
	27	struct attribute attr;
	28	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	29	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	30	};
	31
	32	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	33	char *page)
	34	{
	35	struct blk_mq_ctx_sysfs_entry *entry;
	36	struct blk_mq_ctx *ctx;
	37	struct request_queue *q;
	38	ssize_t res;
	39
	40	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	41	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	42	q = ctx->queue;
	43
	44	if (!entry->show)
	45	return -EIO;
	46
	47	res = -ENOENT;
	48	mutex_lock(&q->sysfs_lock);
	49	if (!blk_queue_dying(q))
	50	res = entry->show(ctx, page);
	51	mutex_unlock(&q->sysfs_lock);
	52	return res;
	53	}
	54
	55	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	56	const char *page, size_t length)
	57	{
	58	struct blk_mq_ctx_sysfs_entry *entry;
	59	struct blk_mq_ctx *ctx;
	60	struct request_queue *q;
	61	ssize_t res;
	62
	63	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	64	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
65	q = ctx->queue;
66
67	if (!entry->store)
68	return -EIO;
69
70	res = -ENOENT;
71	mutex_lock(&q->sysfs_lock);
72	if (!blk_queue_dying(q))
73	res = entry->store(ctx, page, length);
74	mutex_unlock(&q->sysfs_lock);
75	return res;
76	}
77
78	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
79	struct attribute attr, char page)
80	{
81	struct blk_mq_hw_ctx_sysfs_entry *entry;
82	struct blk_mq_hw_ctx *hctx;
83	struct request_queue *q;
84	ssize_t res;
85
86	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
87	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
88	q = hctx->queue;
89
90	if (!entry->show)
91	return -EIO;
92
93	res = -ENOENT;
94	mutex_lock(&q->sysfs_lock);
95	if (!blk_queue_dying(q))
96	res = entry->show(hctx, page);
97	mutex_unlock(&q->sysfs_lock);
98	return res;
99	}
100
101	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
102	struct attribute attr, const char page,
103	size_t length)
104	{
105	struct blk_mq_hw_ctx_sysfs_entry *entry;
106	struct blk_mq_hw_ctx *hctx;
107	struct request_queue *q;
108	ssize_t res;
109
110	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
111	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
112	q = hctx->queue;
113
114	if (!entry->store)
115	return -EIO;
116
117	res = -ENOENT;
118	mutex_lock(&q->sysfs_lock);
119	if (!blk_queue_dying(q))
120	res = entry->store(hctx, page, length);
121	mutex_unlock(&q->sysfs_lock);
122	return res;
123	}
124
125	static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx ctx, char page)
126	{
127	return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
128	ctx->rq_dispatched[0]);
129	}
130
131	static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx ctx, char page)
132	{
133	return sprintf(page, "%lu\n", ctx->rq_merged);
134	}
135
136	static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx ctx, char page)
137	{
138	return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
139	ctx->rq_completed[0]);
140	}
141
142	static ssize_t sysfs_list_show(char page, struct list_head list, char *msg)
143	{
320ae51f	144	struct request *rq;
596f5aad ML	145	int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg);
	146
	147	list_for_each_entry(rq, list, queuelist) {
	148	const int rq_len = 2 * sizeof(rq) + 2;
	149
	150	/* if the output will be truncated */
	151	if (PAGE_SIZE - 1 < len + rq_len) {
	152	/* backspacing if it can't hold '\t...\n' */
	153	if (PAGE_SIZE - 1 < len + 5)
	154	len -= rq_len;
	155	len += snprintf(page + len, PAGE_SIZE - 1 - len,
	156	"\t...\n");
	157	break;
	158	}
	159	len += snprintf(page + len, PAGE_SIZE - 1 - len,
	160	"\t%p\n", rq);
	161	}
	162
	163	return len;
320ae51f JA	164	}
	165
	166	static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx ctx, char page)
	167	{
	168	ssize_t ret;
	169
	170	spin_lock(&ctx->lock);
	171	ret = sysfs_list_show(page, &ctx->rq_list, "CTX pending");
	172	spin_unlock(&ctx->lock);
	173
	174	return ret;
	175	}
	176
05229bee JA	177	static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx hctx, char page)
05229bee JA	178	{
6e219353 SB	179	return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
	180	hctx->poll_considered, hctx->poll_invoked,
	181	hctx->poll_success);
05229bee JA	182	}
05229bee JA	183
320ae51f JA	184	static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
	185	char *page)
	186	{
	187	return sprintf(page, "%lu\n", hctx->queued);
	188	}
	189
	190	static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx hctx, char page)
	191	{
	192	return sprintf(page, "%lu\n", hctx->run);
	193	}
	194
	195	static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
	196	char *page)
	197	{
	198	char *start_page = page;
	199	int i;
	200
	201	page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
	202
	203	for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER; i++) {
	204	unsigned long d = 1U << (i - 1);
	205
	206	page += sprintf(page, "%8lu\t%lu\n", d, hctx->dispatched[i]);
	207	}
	208
	209	return page - start_page;
	210	}
	211
	212	static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx,
	213	char *page)
	214	{
	215	ssize_t ret;
	216
	217	spin_lock(&hctx->lock);
	218	ret = sysfs_list_show(page, &hctx->dispatch, "HCTX pending");
	219	spin_unlock(&hctx->lock);
	220
	221	return ret;
	222	}
	223
320ae51f JA	224	static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx hctx, char page)
	225	{
	226	return blk_mq_tag_sysfs_show(hctx->tags, page);
	227	}
	228
0d2602ca JA	229	static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx hctx, char page)
	230	{
	231	return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
	232	}
	233
676141e4 JA	234	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	235	{
cb2da43e	236	unsigned int i, first = 1;
676141e4 JA	237	ssize_t ret = 0;
676141e4 JA	238
cb2da43e	239	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	240	if (first)
	241	ret += sprintf(ret + page, "%u", i);
	242	else
	243	ret += sprintf(ret + page, ", %u", i);
	244
	245	first = 0;
	246	}
	247
676141e4 JA	248	ret += sprintf(ret + page, "\n");
	249	return ret;
	250	}
	251
320ae51f JA	252	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
	253	.attr = {.name = "dispatched", .mode = S_IRUGO },
	254	.show = blk_mq_sysfs_dispatched_show,
	255	};
	256	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
	257	.attr = {.name = "merged", .mode = S_IRUGO },
	258	.show = blk_mq_sysfs_merged_show,
	259	};
	260	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
	261	.attr = {.name = "completed", .mode = S_IRUGO },
	262	.show = blk_mq_sysfs_completed_show,
	263	};
	264	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list = {
	265	.attr = {.name = "rq_list", .mode = S_IRUGO },
	266	.show = blk_mq_sysfs_rq_list_show,
	267	};
	268
	269	static struct attribute *default_ctx_attrs[] = {
	270	&blk_mq_sysfs_dispatched.attr,
	271	&blk_mq_sysfs_merged.attr,
	272	&blk_mq_sysfs_completed.attr,
	273	&blk_mq_sysfs_rq_list.attr,
	274	NULL,
	275	};
	276
	277	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
	278	.attr = {.name = "queued", .mode = S_IRUGO },
	279	.show = blk_mq_hw_sysfs_queued_show,
	280	};
	281	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
	282	.attr = {.name = "run", .mode = S_IRUGO },
	283	.show = blk_mq_hw_sysfs_run_show,
	284	};
	285	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
	286	.attr = {.name = "dispatched", .mode = S_IRUGO },
	287	.show = blk_mq_hw_sysfs_dispatched_show,
	288	};
0d2602ca JA	289	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
	290	.attr = {.name = "active", .mode = S_IRUGO },
	291	.show = blk_mq_hw_sysfs_active_show,
	292	};
320ae51f JA	293	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = {
	294	.attr = {.name = "pending", .mode = S_IRUGO },
	295	.show = blk_mq_hw_sysfs_rq_list_show,
	296	};
320ae51f JA	297	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
	298	.attr = {.name = "tags", .mode = S_IRUGO },
	299	.show = blk_mq_hw_sysfs_tags_show,
	300	};
676141e4 JA	301	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	302	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	303	.show = blk_mq_hw_sysfs_cpus_show,
	304	};
05229bee JA	305	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
	306	.attr = {.name = "io_poll", .mode = S_IRUGO },
	307	.show = blk_mq_hw_sysfs_poll_show,
	308	};
320ae51f JA	309
	310	static struct attribute *default_hw_ctx_attrs[] = {
	311	&blk_mq_hw_sysfs_queued.attr,
	312	&blk_mq_hw_sysfs_run.attr,
	313	&blk_mq_hw_sysfs_dispatched.attr,
	314	&blk_mq_hw_sysfs_pending.attr,
320ae51f	315	&blk_mq_hw_sysfs_tags.attr,
676141e4	316	&blk_mq_hw_sysfs_cpus.attr,
0d2602ca	317	&blk_mq_hw_sysfs_active.attr,
05229bee	318	&blk_mq_hw_sysfs_poll.attr,
320ae51f JA	319	NULL,
	320	};
	321
	322	static const struct sysfs_ops blk_mq_sysfs_ops = {
	323	.show = blk_mq_sysfs_show,
	324	.store = blk_mq_sysfs_store,
	325	};
	326
	327	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	328	.show = blk_mq_hw_sysfs_show,
	329	.store = blk_mq_hw_sysfs_store,
	330	};
	331
	332	static struct kobj_type blk_mq_ktype = {
	333	.sysfs_ops = &blk_mq_sysfs_ops,
	334	.release = blk_mq_sysfs_release,
	335	};
	336
	337	static struct kobj_type blk_mq_ctx_ktype = {
	338	.sysfs_ops = &blk_mq_sysfs_ops,
	339	.default_attrs = default_ctx_attrs,
74170118	340	.release = blk_mq_sysfs_release,
320ae51f JA	341	};
	342
	343	static struct kobj_type blk_mq_hw_ktype = {
	344	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	345	.default_attrs = default_hw_ctx_attrs,
74170118	346	.release = blk_mq_sysfs_release,
320ae51f JA	347	};
320ae51f JA	348
ee3c5db0	349	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	350	{
	351	struct blk_mq_ctx *ctx;
	352	int i;
	353
4593fdbe	354	if (!hctx->nr_ctx)
67aec14c JA	355	return;
	356
	357	hctx_for_each_ctx(hctx, ctx, i)
	358	kobject_del(&ctx->kobj);
	359
	360	kobject_del(&hctx->kobj);
	361	}
	362
ee3c5db0	363	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	364	{
	365	struct request_queue *q = hctx->queue;
	366	struct blk_mq_ctx *ctx;
	367	int i, ret;
	368
4593fdbe	369	if (!hctx->nr_ctx)
67aec14c JA	370	return 0;
	371
	372	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	373	if (ret)
	374	return ret;
	375
	376	hctx_for_each_ctx(hctx, ctx, i) {
	377	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	378	if (ret)
	379	break;
	380	}
	381
	382	return ret;
	383	}
	384
c0f3fd2b	385	static void __blk_mq_unregister_disk(struct gendisk *disk)
320ae51f JA	386	{
320ae51f JA	387	struct request_queue *q = disk->queue;
85157366 AV	388	struct blk_mq_hw_ctx *hctx;
	389	struct blk_mq_ctx *ctx;
	390	int i, j;
	391
	392	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c JA	393	blk_mq_unregister_hctx(hctx);
	394
	395	hctx_for_each_ctx(hctx, ctx, j)
85157366	396	kobject_put(&ctx->kobj);
67aec14c	397
85157366 AV	398	kobject_put(&hctx->kobj);
85157366 AV	399	}
320ae51f JA	400
	401	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	402	kobject_del(&q->mq_kobj);
85157366	403	kobject_put(&q->mq_kobj);
320ae51f JA	404
320ae51f JA	405	kobject_put(&disk_to_dev(disk)->kobj);
4593fdbe AM	406
4593fdbe AM	407	q->mq_sysfs_init_done = false;
c0f3fd2b JA	408	}
	409
	410	void blk_mq_unregister_disk(struct gendisk *disk)
	411	{
	412	blk_mq_disable_hotplug();
	413	__blk_mq_unregister_disk(disk);
4593fdbe	414	blk_mq_enable_hotplug();
320ae51f JA	415	}
320ae51f JA	416
868f2f0b KB	417	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	418	{
	419	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	420	}
	421
67aec14c JA	422	static void blk_mq_sysfs_init(struct request_queue *q)
67aec14c JA	423	{
67aec14c	424	struct blk_mq_ctx *ctx;
897bb0c7	425	int cpu;
67aec14c JA	426
	427	kobject_init(&q->mq_kobj, &blk_mq_ktype);
	428
897bb0c7 TG	429	for_each_possible_cpu(cpu) {
897bb0c7 TG	430	ctx = per_cpu_ptr(q->queue_ctx, cpu);
06a41a99	431	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	432	}
67aec14c JA	433	}
67aec14c JA	434
320ae51f JA	435	int blk_mq_register_disk(struct gendisk *disk)
	436	{
	437	struct device *dev = disk_to_dev(disk);
	438	struct request_queue *q = disk->queue;
	439	struct blk_mq_hw_ctx *hctx;
67aec14c	440	int ret, i;
320ae51f	441
4593fdbe AM	442	blk_mq_disable_hotplug();
4593fdbe AM	443
67aec14c	444	blk_mq_sysfs_init(q);
320ae51f JA	445
	446	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	447	if (ret < 0)
4593fdbe	448	goto out;
320ae51f JA	449
	450	kobject_uevent(&q->mq_kobj, KOBJ_ADD);
	451
	452	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	453	ret = blk_mq_register_hctx(hctx);
320ae51f JA	454	if (ret)
320ae51f JA	455	break;
320ae51f JA	456	}
320ae51f JA	457
4593fdbe	458	if (ret)
c0f3fd2b	459	__blk_mq_unregister_disk(disk);
4593fdbe AM	460	else
	461	q->mq_sysfs_init_done = true;
	462	out:
	463	blk_mq_enable_hotplug();
320ae51f	464
4593fdbe	465	return ret;
320ae51f	466	}
b62c21b7	467	EXPORT_SYMBOL_GPL(blk_mq_register_disk);
67aec14c JA	468
	469	void blk_mq_sysfs_unregister(struct request_queue *q)
	470	{
	471	struct blk_mq_hw_ctx *hctx;
	472	int i;
	473
4593fdbe AM	474	if (!q->mq_sysfs_init_done)
	475	return;
	476
67aec14c JA	477	queue_for_each_hw_ctx(q, hctx, i)
	478	blk_mq_unregister_hctx(hctx);
	479	}
	480
	481	int blk_mq_sysfs_register(struct request_queue *q)
	482	{
	483	struct blk_mq_hw_ctx *hctx;
	484	int i, ret = 0;
	485
4593fdbe AM	486	if (!q->mq_sysfs_init_done)
	487	return ret;
	488
67aec14c JA	489	queue_for_each_hw_ctx(q, hctx, i) {
	490	ret = blk_mq_register_hctx(hctx);
	491	if (ret)
	492	break;
	493	}
	494
	495	return ret;
	496	}