[mirror_ubuntu-bionic-kernel.git] / block / blk-mq-sysfs.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
				ctx->rq_dispatched[0]);
}

static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu\n", ctx->rq_merged);
}

static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
{
	return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
				ctx->rq_completed[0]);
}

static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
		       hctx->poll_considered, hctx->poll_invoked,
		       hctx->poll_success);
}

static ssize_t blk_mq_hw_sysfs_poll_store(struct blk_mq_hw_ctx *hctx,
					  const char *page, size_t size)
{
	hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0;

	return size;
}

static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
					   char *page)
{
	return sprintf(page, "%lu\n", hctx->queued);
}

static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "%lu\n", hctx->run);
}

static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
					       char *page)
{
	char *start_page = page;
	int i;

	page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);

	for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) {
		unsigned int d = 1U << (i - 1);

		page += sprintf(page, "%8u\t%lu\n", d, hctx->dispatched[i]);
	}

	page += sprintf(page, "%8u+\t%lu\n", 1U << (i - 1),
						hctx->dispatched[i]);
	return page - start_page;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static void blk_mq_stat_clear(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	unsigned int i;

	hctx_for_each_ctx(hctx, ctx, i) {
		blk_stat_init(&ctx->stat[BLK_STAT_READ]);
		blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
	}
}

static ssize_t blk_mq_hw_sysfs_stat_store(struct blk_mq_hw_ctx *hctx,
					  const char *page, size_t count)
{
	blk_mq_stat_clear(hctx);
	return count;
}

static ssize_t print_stat(char *page, struct blk_rq_stat *stat, const char *pre)
{
	return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
			pre, (long long) stat->nr_samples,
			(long long) stat->mean, (long long) stat->min,
			(long long) stat->max);
}

static ssize_t blk_mq_hw_sysfs_stat_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	struct blk_rq_stat stat[2];
	ssize_t ret;

	blk_stat_init(&stat[BLK_STAT_READ]);
	blk_stat_init(&stat[BLK_STAT_WRITE]);

	blk_hctx_stat_get(hctx, stat);

	ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
	ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
	return ret;
}

static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
	.attr = {.name = "dispatched", .mode = S_IRUGO },
	.show = blk_mq_sysfs_dispatched_show,
};
static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
	.attr = {.name = "merged", .mode = S_IRUGO },
	.show = blk_mq_sysfs_merged_show,
};
static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
	.attr = {.name = "completed", .mode = S_IRUGO },
	.show = blk_mq_sysfs_completed_show,
};

static struct attribute *default_ctx_attrs[] = {
	&blk_mq_sysfs_dispatched.attr,
	&blk_mq_sysfs_merged.attr,
	&blk_mq_sysfs_completed.attr,
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
	.attr = {.name = "queued", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_queued_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
	.attr = {.name = "run", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_run_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
	.attr = {.name = "dispatched", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_dispatched_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
	.attr = {.name = "active", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_active_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_cpus_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
	.attr = {.name = "io_poll", .mode = S_IWUSR | S_IRUGO },
	.show = blk_mq_hw_sysfs_poll_show,
	.store = blk_mq_hw_sysfs_poll_store,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_stat = {
	.attr = {.name = "stats", .mode = S_IRUGO | S_IWUSR },
	.show = blk_mq_hw_sysfs_stat_show,
	.store = blk_mq_hw_sysfs_stat_store,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_queued.attr,
	&blk_mq_hw_sysfs_run.attr,
	&blk_mq_hw_sysfs_dispatched.attr,
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	&blk_mq_hw_sysfs_active.attr,
	&blk_mq_hw_sysfs_poll.attr,
	&blk_mq_hw_sysfs_stat.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	int i, j;

	queue_for_each_hw_ctx(q, hctx, i) {
		blk_mq_unregister_hctx(hctx);

		hctx_for_each_ctx(hctx, ctx, j)
			kobject_put(&ctx->kobj);

		kobject_put(&hctx->kobj);
	}

	blk_mq_debugfs_unregister(q);

	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	kobject_del(&q->mq_kobj);
	kobject_put(&q->mq_kobj);

	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	blk_mq_disable_hotplug();
	__blk_mq_unregister_dev(dev, q);
	blk_mq_enable_hotplug();
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

static void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(&q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	blk_mq_disable_hotplug();

	blk_mq_sysfs_init(q);

	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(&q->mq_kobj, KOBJ_ADD);

	blk_mq_debugfs_register(q, kobject_name(&dev->kobj));

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

	if (ret)
		__blk_mq_unregister_dev(dev, q);
	else
		q->mq_sysfs_init_done = true;
out:
	blk_mq_enable_hotplug();

	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_register_dev);

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	if (!q->mq_sysfs_init_done)
		return;

	blk_mq_debugfs_unregister_hctxs(q);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	if (!q->mq_sysfs_init_done)
		return ret;

	blk_mq_debugfs_register_hctxs(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

	return ret;
}
Commit	Line	Data
320ae51f JA	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/backing-dev.h>
	4	#include <linux/bio.h>
	5	#include <linux/blkdev.h>
	6	#include <linux/mm.h>
	7	#include <linux/init.h>
	8	#include <linux/slab.h>
	9	#include <linux/workqueue.h>
	10	#include <linux/smp.h>
	11
	12	#include <linux/blk-mq.h>
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
	16	static void blk_mq_sysfs_release(struct kobject *kobj)
	17	{
	18	}
	19
	20	struct blk_mq_ctx_sysfs_entry {
	21	struct attribute attr;
	22	ssize_t (show)(struct blk_mq_ctx , char *);
	23	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	24	};
	25
	26	struct blk_mq_hw_ctx_sysfs_entry {
	27	struct attribute attr;
	28	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	29	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	30	};
	31
	32	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	33	char *page)
	34	{
	35	struct blk_mq_ctx_sysfs_entry *entry;
	36	struct blk_mq_ctx *ctx;
	37	struct request_queue *q;
	38	ssize_t res;
	39
	40	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	41	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	42	q = ctx->queue;
	43
	44	if (!entry->show)
	45	return -EIO;
	46
	47	res = -ENOENT;
	48	mutex_lock(&q->sysfs_lock);
	49	if (!blk_queue_dying(q))
	50	res = entry->show(ctx, page);
	51	mutex_unlock(&q->sysfs_lock);
	52	return res;
	53	}
	54
	55	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	56	const char *page, size_t length)
	57	{
	58	struct blk_mq_ctx_sysfs_entry *entry;
	59	struct blk_mq_ctx *ctx;
	60	struct request_queue *q;
	61	ssize_t res;
	62
	63	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	64	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
65	q = ctx->queue;
66
67	if (!entry->store)
68	return -EIO;
69
70	res = -ENOENT;
71	mutex_lock(&q->sysfs_lock);
72	if (!blk_queue_dying(q))
73	res = entry->store(ctx, page, length);
74	mutex_unlock(&q->sysfs_lock);
75	return res;
76	}
77
78	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
79	struct attribute attr, char page)
80	{
81	struct blk_mq_hw_ctx_sysfs_entry *entry;
82	struct blk_mq_hw_ctx *hctx;
83	struct request_queue *q;
84	ssize_t res;
85
86	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
87	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
88	q = hctx->queue;
89
90	if (!entry->show)
91	return -EIO;
92
93	res = -ENOENT;
94	mutex_lock(&q->sysfs_lock);
95	if (!blk_queue_dying(q))
96	res = entry->show(hctx, page);
97	mutex_unlock(&q->sysfs_lock);
98	return res;
99	}
100
101	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
102	struct attribute attr, const char page,
103	size_t length)
104	{
105	struct blk_mq_hw_ctx_sysfs_entry *entry;
106	struct blk_mq_hw_ctx *hctx;
107	struct request_queue *q;
108	ssize_t res;
109
110	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
111	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
112	q = hctx->queue;
113
114	if (!entry->store)
115	return -EIO;
116
117	res = -ENOENT;
118	mutex_lock(&q->sysfs_lock);
119	if (!blk_queue_dying(q))
120	res = entry->store(hctx, page, length);
121	mutex_unlock(&q->sysfs_lock);
122	return res;
123	}
124
125	static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx ctx, char page)
126	{
127	return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
128	ctx->rq_dispatched[0]);
129	}
130
131	static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx ctx, char page)
132	{
133	return sprintf(page, "%lu\n", ctx->rq_merged);
134	}
135
136	static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx ctx, char page)
137	{
138	return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
139	ctx->rq_completed[0]);
140	}
141
05229bee JA	142	static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx hctx, char page)
05229bee JA	143	{
6e219353 SB	144	return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
	145	hctx->poll_considered, hctx->poll_invoked,
	146	hctx->poll_success);
05229bee JA	147	}
05229bee JA	148
d21ea4bc SB	149	static ssize_t blk_mq_hw_sysfs_poll_store(struct blk_mq_hw_ctx *hctx,
	150	const char *page, size_t size)
	151	{
	152	hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0;
	153
	154	return size;
	155	}
	156
320ae51f JA	157	static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
	158	char *page)
	159	{
	160	return sprintf(page, "%lu\n", hctx->queued);
	161	}
	162
	163	static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx hctx, char page)
	164	{
	165	return sprintf(page, "%lu\n", hctx->run);
	166	}
	167
	168	static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
	169	char *page)
	170	{
	171	char *start_page = page;
	172	int i;
	173
	174	page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
	175
703fd1c0 JA	176	for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) {
703fd1c0 JA	177	unsigned int d = 1U << (i - 1);
320ae51f	178
703fd1c0	179	page += sprintf(page, "%8u\t%lu\n", d, hctx->dispatched[i]);
320ae51f JA	180	}
320ae51f JA	181
703fd1c0 JA	182	page += sprintf(page, "%8u+\t%lu\n", 1U << (i - 1),
703fd1c0 JA	183	hctx->dispatched[i]);
320ae51f JA	184	return page - start_page;
	185	}
	186
d96b37c0 OS	187	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	188	char *page)
bd166ef1	189	{
d96b37c0	190	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	191	}
bd166ef1 JA	192
d96b37c0 OS	193	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	194	char *page)
320ae51f	195	{
d96b37c0	196	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	197	}
320ae51f JA	198
0d2602ca JA	199	static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx hctx, char page)
	200	{
	201	return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
	202	}
	203
676141e4 JA	204	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	205	{
cb2da43e	206	unsigned int i, first = 1;
676141e4 JA	207	ssize_t ret = 0;
676141e4 JA	208
cb2da43e	209	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	210	if (first)
	211	ret += sprintf(ret + page, "%u", i);
	212	else
	213	ret += sprintf(ret + page, ", %u", i);
	214
	215	first = 0;
	216	}
	217
676141e4 JA	218	ret += sprintf(ret + page, "\n");
	219	return ret;
	220	}
	221
cf43e6be JA	222	static void blk_mq_stat_clear(struct blk_mq_hw_ctx *hctx)
	223	{
	224	struct blk_mq_ctx *ctx;
	225	unsigned int i;
	226
	227	hctx_for_each_ctx(hctx, ctx, i) {
	228	blk_stat_init(&ctx->stat[BLK_STAT_READ]);
	229	blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
	230	}
	231	}
	232
	233	static ssize_t blk_mq_hw_sysfs_stat_store(struct blk_mq_hw_ctx *hctx,
	234	const char *page, size_t count)
	235	{
	236	blk_mq_stat_clear(hctx);
	237	return count;
	238	}
	239
	240	static ssize_t print_stat(char page, struct blk_rq_stat stat, const char *pre)
	241	{
	242	return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
	243	pre, (long long) stat->nr_samples,
	244	(long long) stat->mean, (long long) stat->min,
	245	(long long) stat->max);
	246	}
	247
	248	static ssize_t blk_mq_hw_sysfs_stat_show(struct blk_mq_hw_ctx hctx, char page)
	249	{
	250	struct blk_rq_stat stat[2];
	251	ssize_t ret;
	252
	253	blk_stat_init(&stat[BLK_STAT_READ]);
	254	blk_stat_init(&stat[BLK_STAT_WRITE]);
	255
	256	blk_hctx_stat_get(hctx, stat);
	257
	258	ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
	259	ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
	260	return ret;
	261	}
	262
320ae51f JA	263	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
	264	.attr = {.name = "dispatched", .mode = S_IRUGO },
	265	.show = blk_mq_sysfs_dispatched_show,
	266	};
	267	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
	268	.attr = {.name = "merged", .mode = S_IRUGO },
	269	.show = blk_mq_sysfs_merged_show,
	270	};
	271	static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
	272	.attr = {.name = "completed", .mode = S_IRUGO },
	273	.show = blk_mq_sysfs_completed_show,
	274	};
320ae51f JA	275
	276	static struct attribute *default_ctx_attrs[] = {
	277	&blk_mq_sysfs_dispatched.attr,
	278	&blk_mq_sysfs_merged.attr,
	279	&blk_mq_sysfs_completed.attr,
320ae51f JA	280	NULL,
	281	};
	282
	283	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
	284	.attr = {.name = "queued", .mode = S_IRUGO },
	285	.show = blk_mq_hw_sysfs_queued_show,
	286	};
	287	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
	288	.attr = {.name = "run", .mode = S_IRUGO },
	289	.show = blk_mq_hw_sysfs_run_show,
	290	};
	291	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
	292	.attr = {.name = "dispatched", .mode = S_IRUGO },
	293	.show = blk_mq_hw_sysfs_dispatched_show,
	294	};
d96b37c0 OS	295	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	296	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	297	.show = blk_mq_hw_sysfs_nr_tags_show,
	298	};
	299	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	300	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	301	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
	302	};
0d2602ca JA	303	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
	304	.attr = {.name = "active", .mode = S_IRUGO },
	305	.show = blk_mq_hw_sysfs_active_show,
	306	};
676141e4 JA	307	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	308	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	309	.show = blk_mq_hw_sysfs_cpus_show,
	310	};
05229bee	311	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
d21ea4bc	312	.attr = {.name = "io_poll", .mode = S_IWUSR \| S_IRUGO },
05229bee	313	.show = blk_mq_hw_sysfs_poll_show,
d21ea4bc	314	.store = blk_mq_hw_sysfs_poll_store,
05229bee	315	};
cf43e6be JA	316	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_stat = {
	317	.attr = {.name = "stats", .mode = S_IRUGO \| S_IWUSR },
	318	.show = blk_mq_hw_sysfs_stat_show,
	319	.store = blk_mq_hw_sysfs_stat_store,
	320	};
320ae51f JA	321
	322	static struct attribute *default_hw_ctx_attrs[] = {
	323	&blk_mq_hw_sysfs_queued.attr,
	324	&blk_mq_hw_sysfs_run.attr,
	325	&blk_mq_hw_sysfs_dispatched.attr,
d96b37c0 OS	326	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	327	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	328	&blk_mq_hw_sysfs_cpus.attr,
0d2602ca	329	&blk_mq_hw_sysfs_active.attr,
05229bee	330	&blk_mq_hw_sysfs_poll.attr,
cf43e6be	331	&blk_mq_hw_sysfs_stat.attr,
320ae51f JA	332	NULL,
	333	};
	334
	335	static const struct sysfs_ops blk_mq_sysfs_ops = {
	336	.show = blk_mq_sysfs_show,
	337	.store = blk_mq_sysfs_store,
	338	};
	339
	340	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	341	.show = blk_mq_hw_sysfs_show,
	342	.store = blk_mq_hw_sysfs_store,
	343	};
	344
	345	static struct kobj_type blk_mq_ktype = {
	346	.sysfs_ops = &blk_mq_sysfs_ops,
	347	.release = blk_mq_sysfs_release,
	348	};
	349
	350	static struct kobj_type blk_mq_ctx_ktype = {
	351	.sysfs_ops = &blk_mq_sysfs_ops,
	352	.default_attrs = default_ctx_attrs,
74170118	353	.release = blk_mq_sysfs_release,
320ae51f JA	354	};
	355
	356	static struct kobj_type blk_mq_hw_ktype = {
	357	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	358	.default_attrs = default_hw_ctx_attrs,
74170118	359	.release = blk_mq_sysfs_release,
320ae51f JA	360	};
320ae51f JA	361
ee3c5db0	362	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	363	{
	364	struct blk_mq_ctx *ctx;
	365	int i;
	366
4593fdbe	367	if (!hctx->nr_ctx)
67aec14c JA	368	return;
	369
	370	hctx_for_each_ctx(hctx, ctx, i)
	371	kobject_del(&ctx->kobj);
	372
	373	kobject_del(&hctx->kobj);
	374	}
	375
ee3c5db0	376	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	377	{
	378	struct request_queue *q = hctx->queue;
	379	struct blk_mq_ctx *ctx;
	380	int i, ret;
	381
4593fdbe	382	if (!hctx->nr_ctx)
67aec14c JA	383	return 0;
	384
	385	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	386	if (ret)
	387	return ret;
	388
	389	hctx_for_each_ctx(hctx, ctx, i) {
	390	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	391	if (ret)
	392	break;
	393	}
	394
	395	return ret;
	396	}
	397
b21d5b30	398	static void __blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	399	{
85157366 AV	400	struct blk_mq_hw_ctx *hctx;
	401	struct blk_mq_ctx *ctx;
	402	int i, j;
	403
	404	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c JA	405	blk_mq_unregister_hctx(hctx);
	406
	407	hctx_for_each_ctx(hctx, ctx, j)
85157366	408	kobject_put(&ctx->kobj);
67aec14c	409
85157366 AV	410	kobject_put(&hctx->kobj);
85157366 AV	411	}
320ae51f	412
07e4fead OS	413	blk_mq_debugfs_unregister(q);
07e4fead OS	414
320ae51f JA	415	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
320ae51f JA	416	kobject_del(&q->mq_kobj);
85157366	417	kobject_put(&q->mq_kobj);
320ae51f	418
b21d5b30	419	kobject_put(&dev->kobj);
4593fdbe AM	420
4593fdbe AM	421	q->mq_sysfs_init_done = false;
c0f3fd2b JA	422	}
c0f3fd2b JA	423
b21d5b30	424	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
c0f3fd2b JA	425	{
c0f3fd2b JA	426	blk_mq_disable_hotplug();
b21d5b30	427	__blk_mq_unregister_dev(dev, q);
4593fdbe	428	blk_mq_enable_hotplug();
320ae51f JA	429	}
320ae51f JA	430
868f2f0b KB	431	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	432	{
	433	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	434	}
	435
67aec14c JA	436	static void blk_mq_sysfs_init(struct request_queue *q)
67aec14c JA	437	{
67aec14c	438	struct blk_mq_ctx *ctx;
897bb0c7	439	int cpu;
67aec14c JA	440
	441	kobject_init(&q->mq_kobj, &blk_mq_ktype);
	442
897bb0c7 TG	443	for_each_possible_cpu(cpu) {
897bb0c7 TG	444	ctx = per_cpu_ptr(q->queue_ctx, cpu);
06a41a99	445	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	446	}
67aec14c JA	447	}
67aec14c JA	448
b21d5b30	449	int blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	450	{
320ae51f	451	struct blk_mq_hw_ctx *hctx;
67aec14c	452	int ret, i;
320ae51f	453
4593fdbe AM	454	blk_mq_disable_hotplug();
4593fdbe AM	455
67aec14c	456	blk_mq_sysfs_init(q);
320ae51f JA	457
	458	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	459	if (ret < 0)
4593fdbe	460	goto out;
320ae51f JA	461
	462	kobject_uevent(&q->mq_kobj, KOBJ_ADD);
	463
07e4fead OS	464	blk_mq_debugfs_register(q, kobject_name(&dev->kobj));
07e4fead OS	465
320ae51f	466	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	467	ret = blk_mq_register_hctx(hctx);
320ae51f JA	468	if (ret)
320ae51f JA	469	break;
320ae51f JA	470	}
320ae51f JA	471
4593fdbe	472	if (ret)
b21d5b30	473	__blk_mq_unregister_dev(dev, q);
4593fdbe AM	474	else
	475	q->mq_sysfs_init_done = true;
	476	out:
	477	blk_mq_enable_hotplug();
320ae51f	478
4593fdbe	479	return ret;
320ae51f	480	}
b21d5b30	481	EXPORT_SYMBOL_GPL(blk_mq_register_dev);
67aec14c JA	482
	483	void blk_mq_sysfs_unregister(struct request_queue *q)
	484	{
	485	struct blk_mq_hw_ctx *hctx;
	486	int i;
	487
4593fdbe AM	488	if (!q->mq_sysfs_init_done)
	489	return;
	490
07e4fead OS	491	blk_mq_debugfs_unregister_hctxs(q);
07e4fead OS	492
67aec14c JA	493	queue_for_each_hw_ctx(q, hctx, i)
	494	blk_mq_unregister_hctx(hctx);
	495	}
	496
	497	int blk_mq_sysfs_register(struct request_queue *q)
	498	{
	499	struct blk_mq_hw_ctx *hctx;
	500	int i, ret = 0;
	501
4593fdbe AM	502	if (!q->mq_sysfs_init_done)
	503	return ret;
	504
07e4fead OS	505	blk_mq_debugfs_register_hctxs(q);
07e4fead OS	506
67aec14c JA	507	queue_for_each_hw_ctx(q, hctx, i) {
	508	ret = blk_mq_register_hctx(hctx);
	509	if (ret)
	510	break;
	511	}
	512
	513	return ret;
	514	}