[mirror_ubuntu-focal-kernel.git] / block / blk-mq-sysfs.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);

	free_percpu(ctxs->queue_ctx);
	kfree(ctxs);
}

static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);

	/* ctx->ctxs won't be released until all ctx are freed */
	kobject_put(&ctx->ctxs->kobj);
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);

	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(hctx->srcu);
	blk_free_flush_queue(hctx->fq);
	sbitmap_free(&hctx->ctx_map);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	const size_t size = PAGE_SIZE - 1;
	unsigned int i, first = 1;
	int ret = 0, pos = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret = snprintf(pos + page, size - pos, "%u", i);
		else
			ret = snprintf(pos + page, size - pos, ", %u", i);

		if (ret >= size - pos)
			break;

		first = 0;
		pos += ret;
	}

	ret = snprintf(pos + page, size + 1 - pos, "\n");
	return pos + ret;
}

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = 0444 },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};
ATTRIBUTE_GROUPS(default_hw_ctx);

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_ctx_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_groups = default_hw_ctx_groups,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_dir_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);

		kobject_get(q->mq_kobj);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_dir_lock);

	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	while (--i >= 0)
		blk_mq_unregister_hctx(q->queue_hw_ctx[i]);

	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
	kobject_del(q->mq_kobj);
	kobject_put(&dev->kobj);
	return ret;
}

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_dir_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_dir_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_dir_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_dir_lock);

	return ret;
}
Commit	Line	Data
3dcf60bc	1	// SPDX-License-Identifier: GPL-2.0
320ae51f JA	2	#include <linux/kernel.h>
	3	#include <linux/module.h>
	4	#include <linux/backing-dev.h>
	5	#include <linux/bio.h>
	6	#include <linux/blkdev.h>
	7	#include <linux/mm.h>
	8	#include <linux/init.h>
	9	#include <linux/slab.h>
	10	#include <linux/workqueue.h>
	11	#include <linux/smp.h>
	12
	13	#include <linux/blk-mq.h>
c7e2d94b	14	#include "blk.h"
320ae51f JA	15	#include "blk-mq.h"
	16	#include "blk-mq-tag.h"
	17
	18	static void blk_mq_sysfs_release(struct kobject *kobj)
	19	{
1db4909e ML	20	struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
	21
	22	free_percpu(ctxs->queue_ctx);
	23	kfree(ctxs);
	24	}
	25
	26	static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
	27	{
	28	struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	29
	30	/* ctx->ctxs won't be released until all ctx are freed */
	31	kobject_put(&ctx->ctxs->kobj);
320ae51f JA	32	}
320ae51f JA	33
6c8b232e ML	34	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	35	{
	36	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	37	kobj);
c7e2d94b ML	38
	39	if (hctx->flags & BLK_MQ_F_BLOCKING)
	40	cleanup_srcu_struct(hctx->srcu);
	41	blk_free_flush_queue(hctx->fq);
	42	sbitmap_free(&hctx->ctx_map);
01388df3	43	free_cpumask_var(hctx->cpumask);
6c8b232e ML	44	kfree(hctx->ctxs);
	45	kfree(hctx);
	46	}
	47
320ae51f JA	48	struct blk_mq_ctx_sysfs_entry {
	49	struct attribute attr;
	50	ssize_t (show)(struct blk_mq_ctx , char *);
	51	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	52	};
	53
	54	struct blk_mq_hw_ctx_sysfs_entry {
	55	struct attribute attr;
	56	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	57	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	58	};
	59
	60	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	61	char *page)
	62	{
	63	struct blk_mq_ctx_sysfs_entry *entry;
	64	struct blk_mq_ctx *ctx;
	65	struct request_queue *q;
	66	ssize_t res;
	67
	68	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	69	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	70	q = ctx->queue;
	71
	72	if (!entry->show)
	73	return -EIO;
	74
	75	res = -ENOENT;
	76	mutex_lock(&q->sysfs_lock);
	77	if (!blk_queue_dying(q))
	78	res = entry->show(ctx, page);
	79	mutex_unlock(&q->sysfs_lock);
	80	return res;
	81	}
	82
	83	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	84	const char *page, size_t length)
	85	{
	86	struct blk_mq_ctx_sysfs_entry *entry;
	87	struct blk_mq_ctx *ctx;
	88	struct request_queue *q;
	89	ssize_t res;
	90
	91	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	92	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	93	q = ctx->queue;
	94
	95	if (!entry->store)
	96	return -EIO;
	97
	98	res = -ENOENT;
	99	mutex_lock(&q->sysfs_lock);
	100	if (!blk_queue_dying(q))
	101	res = entry->store(ctx, page, length);
	102	mutex_unlock(&q->sysfs_lock);
	103	return res;
	104	}
	105
	106	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	107	struct attribute attr, char page)
	108	{
	109	struct blk_mq_hw_ctx_sysfs_entry *entry;
	110	struct blk_mq_hw_ctx *hctx;
	111	struct request_queue *q;
112	ssize_t res;
113
114	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
115	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
116	q = hctx->queue;
117
118	if (!entry->show)
119	return -EIO;
120
121	res = -ENOENT;
122	mutex_lock(&q->sysfs_lock);
123	if (!blk_queue_dying(q))
124	res = entry->show(hctx, page);
125	mutex_unlock(&q->sysfs_lock);
126	return res;
127	}
128
129	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
130	struct attribute attr, const char page,
131	size_t length)
132	{
133	struct blk_mq_hw_ctx_sysfs_entry *entry;
134	struct blk_mq_hw_ctx *hctx;
135	struct request_queue *q;
136	ssize_t res;
137
138	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
139	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
140	q = hctx->queue;
141
142	if (!entry->store)
143	return -EIO;
144
145	res = -ENOENT;
146	mutex_lock(&q->sysfs_lock);
147	if (!blk_queue_dying(q))
148	res = entry->store(hctx, page, length);
149	mutex_unlock(&q->sysfs_lock);
150	return res;
151	}
152
d96b37c0 OS	153	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	154	char *page)
bd166ef1	155	{
d96b37c0	156	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	157	}
bd166ef1 JA	158
d96b37c0 OS	159	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	160	char *page)
320ae51f	161	{
d96b37c0	162	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	163	}
320ae51f JA	164
676141e4 JA	165	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	166	{
4fd488c3	167	const size_t size = PAGE_SIZE - 1;
cb2da43e	168	unsigned int i, first = 1;
4fd488c3	169	int ret = 0, pos = 0;
676141e4	170
cb2da43e	171	for_each_cpu(i, hctx->cpumask) {
676141e4	172	if (first)
4fd488c3	173	ret = snprintf(pos + page, size - pos, "%u", i);
676141e4	174	else
4fd488c3 ML	175	ret = snprintf(pos + page, size - pos, ", %u", i);
	176
	177	if (ret >= size - pos)
	178	break;
676141e4 JA	179
676141e4 JA	180	first = 0;
4fd488c3	181	pos += ret;
676141e4 JA	182	}
676141e4 JA	183
5dbf8b6c	184	ret = snprintf(pos + page, size + 1 - pos, "\n");
4fd488c3	185	return pos + ret;
676141e4 JA	186	}
676141e4 JA	187
d96b37c0	188	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
5657a819	189	.attr = {.name = "nr_tags", .mode = 0444 },
d96b37c0 OS	190	.show = blk_mq_hw_sysfs_nr_tags_show,
	191	};
	192	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
5657a819	193	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
d96b37c0 OS	194	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
d96b37c0 OS	195	};
676141e4	196	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
5657a819	197	.attr = {.name = "cpu_list", .mode = 0444 },
676141e4 JA	198	.show = blk_mq_hw_sysfs_cpus_show,
676141e4 JA	199	};
320ae51f JA	200
320ae51f JA	201	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	202	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	203	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	204	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	205	NULL,
320ae51f JA	206	};
800f5aa1	207	ATTRIBUTE_GROUPS(default_hw_ctx);
320ae51f JA	208
	209	static const struct sysfs_ops blk_mq_sysfs_ops = {
	210	.show = blk_mq_sysfs_show,
	211	.store = blk_mq_sysfs_store,
	212	};
	213
	214	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	215	.show = blk_mq_hw_sysfs_show,
	216	.store = blk_mq_hw_sysfs_store,
	217	};
	218
	219	static struct kobj_type blk_mq_ktype = {
	220	.sysfs_ops = &blk_mq_sysfs_ops,
	221	.release = blk_mq_sysfs_release,
	222	};
	223
	224	static struct kobj_type blk_mq_ctx_ktype = {
	225	.sysfs_ops = &blk_mq_sysfs_ops,
1db4909e	226	.release = blk_mq_ctx_sysfs_release,
320ae51f JA	227	};
	228
	229	static struct kobj_type blk_mq_hw_ktype = {
	230	.sysfs_ops = &blk_mq_hw_sysfs_ops,
800f5aa1	231	.default_groups = default_hw_ctx_groups,
6c8b232e	232	.release = blk_mq_hw_sysfs_release,
320ae51f JA	233	};
320ae51f JA	234
ee3c5db0	235	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	236	{
	237	struct blk_mq_ctx *ctx;
	238	int i;
	239
4593fdbe	240	if (!hctx->nr_ctx)
67aec14c JA	241	return;
	242
	243	hctx_for_each_ctx(hctx, ctx, i)
	244	kobject_del(&ctx->kobj);
	245
	246	kobject_del(&hctx->kobj);
	247	}
	248
ee3c5db0	249	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	250	{
	251	struct request_queue *q = hctx->queue;
	252	struct blk_mq_ctx *ctx;
	253	int i, ret;
	254
4593fdbe	255	if (!hctx->nr_ctx)
67aec14c JA	256	return 0;
67aec14c JA	257
1db4909e	258	ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
67aec14c JA	259	if (ret)
	260	return ret;
	261
	262	hctx_for_each_ctx(hctx, ctx, i) {
	263	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	264	if (ret)
	265	break;
	266	}
	267
	268	return ret;
	269	}
	270
667257e8	271	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	272	{
85157366	273	struct blk_mq_hw_ctx *hctx;
7ea5fe31	274	int i;
85157366	275
cecf5d87	276	lockdep_assert_held(&q->sysfs_dir_lock);
2d0364c8	277
6c8b232e	278	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	279	blk_mq_unregister_hctx(hctx);
67aec14c JA	280
1db4909e ML	281	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	282	kobject_del(q->mq_kobj);
b21d5b30	283	kobject_put(&dev->kobj);
4593fdbe AM	284
4593fdbe AM	285	q->mq_sysfs_init_done = false;
c0f3fd2b JA	286	}
c0f3fd2b JA	287
868f2f0b KB	288	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	289	{
	290	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	291	}
	292
7ea5fe31 ML	293	void blk_mq_sysfs_deinit(struct request_queue *q)
	294	{
	295	struct blk_mq_ctx *ctx;
	296	int cpu;
	297
	298	for_each_possible_cpu(cpu) {
	299	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	300	kobject_put(&ctx->kobj);
	301	}
1db4909e	302	kobject_put(q->mq_kobj);
7ea5fe31 ML	303	}
7ea5fe31 ML	304
737f98cf	305	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	306	{
67aec14c	307	struct blk_mq_ctx *ctx;
897bb0c7	308	int cpu;
67aec14c	309
1db4909e	310	kobject_init(q->mq_kobj, &blk_mq_ktype);
67aec14c	311
897bb0c7 TG	312	for_each_possible_cpu(cpu) {
897bb0c7 TG	313	ctx = per_cpu_ptr(q->queue_ctx, cpu);
1db4909e ML	314
1db4909e ML	315	kobject_get(q->mq_kobj);
06a41a99	316	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	317	}
67aec14c JA	318	}
67aec14c JA	319
2d0364c8	320	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	321	{
320ae51f	322	struct blk_mq_hw_ctx *hctx;
67aec14c	323	int ret, i;
320ae51f	324
2d0364c8	325	WARN_ON_ONCE(!q->kobj.parent);
cecf5d87	326	lockdep_assert_held(&q->sysfs_dir_lock);
4593fdbe	327
1db4909e	328	ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f	329	if (ret < 0)
4593fdbe	330	goto out;
320ae51f	331
1db4909e	332	kobject_uevent(q->mq_kobj, KOBJ_ADD);
320ae51f JA	333
320ae51f JA	334	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	335	ret = blk_mq_register_hctx(hctx);
320ae51f	336	if (ret)
f05d1ba7	337	goto unreg;
320ae51f JA	338	}
320ae51f JA	339
f05d1ba7	340	q->mq_sysfs_init_done = true;
2d0364c8	341
4593fdbe	342	out:
2d0364c8	343	return ret;
f05d1ba7 BVA	344
	345	unreg:
	346	while (--i >= 0)
	347	blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
	348
1db4909e ML	349	kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
1db4909e ML	350	kobject_del(q->mq_kobj);
f05d1ba7 BVA	351	kobject_put(&dev->kobj);
f05d1ba7 BVA	352	return ret;
2d0364c8 BVA	353	}
2d0364c8 BVA	354
67aec14c JA	355	void blk_mq_sysfs_unregister(struct request_queue *q)
	356	{
	357	struct blk_mq_hw_ctx *hctx;
	358	int i;
	359
cecf5d87	360	mutex_lock(&q->sysfs_dir_lock);
4593fdbe	361	if (!q->mq_sysfs_init_done)
2d0364c8	362	goto unlock;
4593fdbe	363
67aec14c JA	364	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	365	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	366
2d0364c8 BVA	367	unlock:
cecf5d87	368	mutex_unlock(&q->sysfs_dir_lock);
67aec14c JA	369	}
	370
	371	int blk_mq_sysfs_register(struct request_queue *q)
	372	{
	373	struct blk_mq_hw_ctx *hctx;
	374	int i, ret = 0;
	375
cecf5d87	376	mutex_lock(&q->sysfs_dir_lock);
4593fdbe	377	if (!q->mq_sysfs_init_done)
2d0364c8	378	goto unlock;
4593fdbe	379
67aec14c JA	380	queue_for_each_hw_ctx(q, hctx, i) {
	381	ret = blk_mq_register_hctx(hctx);
	382	if (ret)
	383	break;
	384	}
	385
2d0364c8	386	unlock:
cecf5d87	387	mutex_unlock(&q->sysfs_dir_lock);
2d0364c8	388
67aec14c JA	389	return ret;
67aec14c JA	390	}