[mirror_ubuntu-bionic-kernel.git] / block / blk-mq-sysfs.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	unsigned int i, first = 1;
	ssize_t ret = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret += sprintf(ret + page, "%u", i);
		else
			ret += sprintf(ret + page, ", %u", i);

		first = 0;
	}

	ret += sprintf(ret + page, "\n");
	return ret;
}

static struct attribute *default_ctx_attrs[] = {
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	blk_mq_debugfs_unregister_mq(q);

	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	kobject_del(&q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	mutex_lock(&q->sysfs_lock);
	__blk_mq_unregister_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(&q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(&q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_lock);

	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(&q->mq_kobj, KOBJ_ADD);

	blk_mq_debugfs_register(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

	if (ret)
		__blk_mq_unregister_dev(dev, q);
	else
		q->mq_sysfs_init_done = true;

out:
	return ret;
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = __blk_mq_register_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_register_dev);

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	blk_mq_debugfs_unregister_mq(q);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	blk_mq_debugfs_register_mq(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
320ae51f JA	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/backing-dev.h>
	4	#include <linux/bio.h>
	5	#include <linux/blkdev.h>
	6	#include <linux/mm.h>
	7	#include <linux/init.h>
	8	#include <linux/slab.h>
	9	#include <linux/workqueue.h>
	10	#include <linux/smp.h>
	11
	12	#include <linux/blk-mq.h>
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
	16	static void blk_mq_sysfs_release(struct kobject *kobj)
	17	{
	18	}
	19
6c8b232e ML	20	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	21	{
	22	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	23	kobj);
01388df3	24	free_cpumask_var(hctx->cpumask);
6c8b232e ML	25	kfree(hctx->ctxs);
	26	kfree(hctx);
	27	}
	28
320ae51f JA	29	struct blk_mq_ctx_sysfs_entry {
	30	struct attribute attr;
	31	ssize_t (show)(struct blk_mq_ctx , char *);
	32	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	33	};
	34
	35	struct blk_mq_hw_ctx_sysfs_entry {
	36	struct attribute attr;
	37	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	38	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	39	};
	40
	41	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	42	char *page)
	43	{
	44	struct blk_mq_ctx_sysfs_entry *entry;
	45	struct blk_mq_ctx *ctx;
	46	struct request_queue *q;
	47	ssize_t res;
	48
	49	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	50	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	51	q = ctx->queue;
	52
	53	if (!entry->show)
	54	return -EIO;
	55
	56	res = -ENOENT;
	57	mutex_lock(&q->sysfs_lock);
	58	if (!blk_queue_dying(q))
	59	res = entry->show(ctx, page);
	60	mutex_unlock(&q->sysfs_lock);
	61	return res;
	62	}
	63
	64	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	65	const char *page, size_t length)
	66	{
	67	struct blk_mq_ctx_sysfs_entry *entry;
	68	struct blk_mq_ctx *ctx;
	69	struct request_queue *q;
	70	ssize_t res;
	71
	72	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	73	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	74	q = ctx->queue;
	75
	76	if (!entry->store)
	77	return -EIO;
	78
	79	res = -ENOENT;
	80	mutex_lock(&q->sysfs_lock);
	81	if (!blk_queue_dying(q))
	82	res = entry->store(ctx, page, length);
	83	mutex_unlock(&q->sysfs_lock);
	84	return res;
	85	}
	86
	87	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	88	struct attribute attr, char page)
	89	{
	90	struct blk_mq_hw_ctx_sysfs_entry *entry;
	91	struct blk_mq_hw_ctx *hctx;
	92	struct request_queue *q;
93	ssize_t res;
94
95	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
96	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
97	q = hctx->queue;
98
99	if (!entry->show)
100	return -EIO;
101
102	res = -ENOENT;
103	mutex_lock(&q->sysfs_lock);
104	if (!blk_queue_dying(q))
105	res = entry->show(hctx, page);
106	mutex_unlock(&q->sysfs_lock);
107	return res;
108	}
109
110	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
111	struct attribute attr, const char page,
112	size_t length)
113	{
114	struct blk_mq_hw_ctx_sysfs_entry *entry;
115	struct blk_mq_hw_ctx *hctx;
116	struct request_queue *q;
117	ssize_t res;
118
119	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
120	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
121	q = hctx->queue;
122
123	if (!entry->store)
124	return -EIO;
125
126	res = -ENOENT;
127	mutex_lock(&q->sysfs_lock);
128	if (!blk_queue_dying(q))
129	res = entry->store(hctx, page, length);
130	mutex_unlock(&q->sysfs_lock);
131	return res;
132	}
133
d96b37c0 OS	134	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	135	char *page)
bd166ef1	136	{
d96b37c0	137	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	138	}
bd166ef1 JA	139
d96b37c0 OS	140	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	141	char *page)
320ae51f	142	{
d96b37c0	143	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	144	}
320ae51f JA	145
676141e4 JA	146	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	147	{
cb2da43e	148	unsigned int i, first = 1;
676141e4 JA	149	ssize_t ret = 0;
676141e4 JA	150
cb2da43e	151	for_each_cpu(i, hctx->cpumask) {
676141e4 JA	152	if (first)
	153	ret += sprintf(ret + page, "%u", i);
	154	else
	155	ret += sprintf(ret + page, ", %u", i);
	156
	157	first = 0;
	158	}
	159
676141e4 JA	160	ret += sprintf(ret + page, "\n");
	161	return ret;
	162	}
	163
320ae51f	164	static struct attribute *default_ctx_attrs[] = {
320ae51f JA	165	NULL,
	166	};
	167
d96b37c0 OS	168	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	169	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	170	.show = blk_mq_hw_sysfs_nr_tags_show,
	171	};
	172	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	173	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	174	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
	175	};
676141e4 JA	176	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	177	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	178	.show = blk_mq_hw_sysfs_cpus_show,
	179	};
320ae51f JA	180
320ae51f JA	181	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	182	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	183	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	184	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	185	NULL,
	186	};
	187
	188	static const struct sysfs_ops blk_mq_sysfs_ops = {
	189	.show = blk_mq_sysfs_show,
	190	.store = blk_mq_sysfs_store,
	191	};
	192
	193	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	194	.show = blk_mq_hw_sysfs_show,
	195	.store = blk_mq_hw_sysfs_store,
	196	};
	197
	198	static struct kobj_type blk_mq_ktype = {
	199	.sysfs_ops = &blk_mq_sysfs_ops,
	200	.release = blk_mq_sysfs_release,
	201	};
	202
	203	static struct kobj_type blk_mq_ctx_ktype = {
	204	.sysfs_ops = &blk_mq_sysfs_ops,
	205	.default_attrs = default_ctx_attrs,
74170118	206	.release = blk_mq_sysfs_release,
320ae51f JA	207	};
	208
	209	static struct kobj_type blk_mq_hw_ktype = {
	210	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	211	.default_attrs = default_hw_ctx_attrs,
6c8b232e	212	.release = blk_mq_hw_sysfs_release,
320ae51f JA	213	};
320ae51f JA	214
ee3c5db0	215	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	216	{
	217	struct blk_mq_ctx *ctx;
	218	int i;
	219
4593fdbe	220	if (!hctx->nr_ctx)
67aec14c JA	221	return;
	222
	223	hctx_for_each_ctx(hctx, ctx, i)
	224	kobject_del(&ctx->kobj);
	225
	226	kobject_del(&hctx->kobj);
	227	}
	228
ee3c5db0	229	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	230	{
	231	struct request_queue *q = hctx->queue;
	232	struct blk_mq_ctx *ctx;
	233	int i, ret;
	234
4593fdbe	235	if (!hctx->nr_ctx)
67aec14c JA	236	return 0;
	237
	238	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	239	if (ret)
	240	return ret;
	241
	242	hctx_for_each_ctx(hctx, ctx, i) {
	243	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	244	if (ret)
	245	break;
	246	}
	247
	248	return ret;
	249	}
	250
b21d5b30	251	static void __blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	252	{
85157366	253	struct blk_mq_hw_ctx *hctx;
7ea5fe31	254	int i;
85157366	255
2d0364c8 BVA	256	lockdep_assert_held(&q->sysfs_lock);
2d0364c8 BVA	257
6c8b232e	258	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	259	blk_mq_unregister_hctx(hctx);
67aec14c JA	260
62d6c949	261	blk_mq_debugfs_unregister_mq(q);
07e4fead	262
320ae51f JA	263	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
320ae51f JA	264	kobject_del(&q->mq_kobj);
b21d5b30	265	kobject_put(&dev->kobj);
4593fdbe AM	266
4593fdbe AM	267	q->mq_sysfs_init_done = false;
c0f3fd2b JA	268	}
c0f3fd2b JA	269
b21d5b30	270	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
c0f3fd2b	271	{
2d0364c8	272	mutex_lock(&q->sysfs_lock);
b21d5b30	273	__blk_mq_unregister_dev(dev, q);
2d0364c8	274	mutex_unlock(&q->sysfs_lock);
320ae51f JA	275	}
320ae51f JA	276
868f2f0b KB	277	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	278	{
	279	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	280	}
	281
7ea5fe31 ML	282	void blk_mq_sysfs_deinit(struct request_queue *q)
	283	{
	284	struct blk_mq_ctx *ctx;
	285	int cpu;
	286
	287	for_each_possible_cpu(cpu) {
	288	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	289	kobject_put(&ctx->kobj);
	290	}
	291	kobject_put(&q->mq_kobj);
	292	}
	293
737f98cf	294	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	295	{
67aec14c	296	struct blk_mq_ctx *ctx;
897bb0c7	297	int cpu;
67aec14c JA	298
	299	kobject_init(&q->mq_kobj, &blk_mq_ktype);
	300
897bb0c7 TG	301	for_each_possible_cpu(cpu) {
897bb0c7 TG	302	ctx = per_cpu_ptr(q->queue_ctx, cpu);
06a41a99	303	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	304	}
67aec14c JA	305	}
67aec14c JA	306
2d0364c8	307	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	308	{
320ae51f	309	struct blk_mq_hw_ctx *hctx;
67aec14c	310	int ret, i;
320ae51f	311
2d0364c8 BVA	312	WARN_ON_ONCE(!q->kobj.parent);
2d0364c8 BVA	313	lockdep_assert_held(&q->sysfs_lock);
4593fdbe	314
320ae51f JA	315	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f JA	316	if (ret < 0)
4593fdbe	317	goto out;
320ae51f JA	318
	319	kobject_uevent(&q->mq_kobj, KOBJ_ADD);
	320
4c9e4019	321	blk_mq_debugfs_register(q);
07e4fead	322
320ae51f	323	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	324	ret = blk_mq_register_hctx(hctx);
320ae51f JA	325	if (ret)
320ae51f JA	326	break;
320ae51f JA	327	}
320ae51f JA	328
4593fdbe	329	if (ret)
b21d5b30	330	__blk_mq_unregister_dev(dev, q);
4593fdbe AM	331	else
4593fdbe AM	332	q->mq_sysfs_init_done = true;
2d0364c8	333
4593fdbe	334	out:
2d0364c8 BVA	335	return ret;
	336	}
	337
	338	int blk_mq_register_dev(struct device dev, struct request_queue q)
	339	{
	340	int ret;
	341
	342	mutex_lock(&q->sysfs_lock);
	343	ret = __blk_mq_register_dev(dev, q);
	344	mutex_unlock(&q->sysfs_lock);
320ae51f	345
4593fdbe	346	return ret;
320ae51f	347	}
b21d5b30	348	EXPORT_SYMBOL_GPL(blk_mq_register_dev);
67aec14c JA	349
	350	void blk_mq_sysfs_unregister(struct request_queue *q)
	351	{
	352	struct blk_mq_hw_ctx *hctx;
	353	int i;
	354
2d0364c8	355	mutex_lock(&q->sysfs_lock);
4593fdbe	356	if (!q->mq_sysfs_init_done)
2d0364c8	357	goto unlock;
4593fdbe	358
62d6c949	359	blk_mq_debugfs_unregister_mq(q);
07e4fead	360
67aec14c JA	361	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	362	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	363
	364	unlock:
	365	mutex_unlock(&q->sysfs_lock);
67aec14c JA	366	}
	367
	368	int blk_mq_sysfs_register(struct request_queue *q)
	369	{
	370	struct blk_mq_hw_ctx *hctx;
	371	int i, ret = 0;
	372
2d0364c8	373	mutex_lock(&q->sysfs_lock);
4593fdbe	374	if (!q->mq_sysfs_init_done)
2d0364c8	375	goto unlock;
4593fdbe	376
62d6c949	377	blk_mq_debugfs_register_mq(q);
07e4fead	378
67aec14c JA	379	queue_for_each_hw_ctx(q, hctx, i) {
	380	ret = blk_mq_register_hctx(hctx);
	381	if (ret)
	382	break;
	383	}
	384
2d0364c8 BVA	385	unlock:
	386	mutex_unlock(&q->sysfs_lock);
	387
67aec14c JA	388	return ret;
67aec14c JA	389	}