[mirror_ubuntu-bionic-kernel.git] / block / blk-mq-sysfs.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static void blk_mq_sysfs_release(struct kobject *kobj)
{
}

static void blk_mq_hw_sysfs_release(struct kobject *kobj)
{
	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
						  kobj);

	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(hctx->queue_rq_srcu);
	blk_free_flush_queue(hctx->fq);
	sbitmap_free(&hctx->ctx_map);
	free_cpumask_var(hctx->cpumask);
	kfree(hctx->ctxs);
	kfree(hctx);
}

struct blk_mq_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_ctx *, char *);
	ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
};

struct blk_mq_hw_ctx_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
	ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
};

static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
				 char *page)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(ctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
				  const char *page, size_t length)
{
	struct blk_mq_ctx_sysfs_entry *entry;
	struct blk_mq_ctx *ctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	q = ctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(ctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
				    struct attribute *attr, char *page)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->show)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->show(hctx, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
				     struct attribute *attr, const char *page,
				     size_t length)
{
	struct blk_mq_hw_ctx_sysfs_entry *entry;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	ssize_t res;

	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
	q = hctx->queue;

	if (!entry->store)
		return -EIO;

	res = -ENOENT;
	mutex_lock(&q->sysfs_lock);
	if (!blk_queue_dying(q))
		res = entry->store(hctx, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
					    char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_tags);
}

static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
						     char *page)
{
	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
}

static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
{
	const size_t size = PAGE_SIZE - 1;
	unsigned int i, first = 1;
	int ret = 0, pos = 0;

	for_each_cpu(i, hctx->cpumask) {
		if (first)
			ret = snprintf(pos + page, size - pos, "%u", i);
		else
			ret = snprintf(pos + page, size - pos, ", %u", i);

		if (ret >= size - pos)
			break;

		first = 0;
		pos += ret;
	}

	ret = snprintf(pos + page, size + 1 - pos, "\n");
	return pos + ret;
}

static struct attribute *default_ctx_attrs[] = {
	NULL,
};

static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	.show = blk_mq_hw_sysfs_cpus_show,
};

static struct attribute *default_hw_ctx_attrs[] = {
	&blk_mq_hw_sysfs_nr_tags.attr,
	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
	&blk_mq_hw_sysfs_cpus.attr,
	NULL,
};

static const struct sysfs_ops blk_mq_sysfs_ops = {
	.show	= blk_mq_sysfs_show,
	.store	= blk_mq_sysfs_store,
};

static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	.show	= blk_mq_hw_sysfs_show,
	.store	= blk_mq_hw_sysfs_store,
};

static struct kobj_type blk_mq_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_ctx_ktype = {
	.sysfs_ops	= &blk_mq_sysfs_ops,
	.default_attrs	= default_ctx_attrs,
	.release	= blk_mq_sysfs_release,
};

static struct kobj_type blk_mq_hw_ktype = {
	.sysfs_ops	= &blk_mq_hw_sysfs_ops,
	.default_attrs	= default_hw_ctx_attrs,
	.release	= blk_mq_hw_sysfs_release,
};

static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_ctx *ctx;
	int i;

	if (!hctx->nr_ctx)
		return;

	hctx_for_each_ctx(hctx, ctx, i)
		kobject_del(&ctx->kobj);

	kobject_del(&hctx->kobj);
}

static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	int i, ret;

	if (!hctx->nr_ctx)
		return 0;

	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	if (ret)
		return ret;

	hctx_for_each_ctx(hctx, ctx, i) {
		ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
		if (ret)
			break;
	}

	return ret;
}

static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	lockdep_assert_held(&q->sysfs_lock);

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	kobject_del(&q->mq_kobj);
	kobject_put(&dev->kobj);

	q->mq_sysfs_init_done = false;
}

void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
{
	mutex_lock(&q->sysfs_lock);
	__blk_mq_unregister_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);
}

void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
{
	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
}

void blk_mq_sysfs_deinit(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_put(&ctx->kobj);
	}
	kobject_put(&q->mq_kobj);
}

void blk_mq_sysfs_init(struct request_queue *q)
{
	struct blk_mq_ctx *ctx;
	int cpu;

	kobject_init(&q->mq_kobj, &blk_mq_ktype);

	for_each_possible_cpu(cpu) {
		ctx = per_cpu_ptr(q->queue_ctx, cpu);
		kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
	}
}

int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int ret, i;

	WARN_ON_ONCE(!q->kobj.parent);
	lockdep_assert_held(&q->sysfs_lock);

	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
	if (ret < 0)
		goto out;

	kobject_uevent(&q->mq_kobj, KOBJ_ADD);

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			goto unreg;
	}

	q->mq_sysfs_init_done = true;

out:
	return ret;

unreg:
	while (--i >= 0)
		blk_mq_unregister_hctx(q->queue_hw_ctx[i]);

	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	kobject_del(&q->mq_kobj);
	kobject_put(&dev->kobj);
	return ret;
}

int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
	int ret;

	mutex_lock(&q->sysfs_lock);
	ret = __blk_mq_register_dev(dev, q);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_register_dev);

void blk_mq_sysfs_unregister(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_unregister_hctx(hctx);

unlock:
	mutex_unlock(&q->sysfs_lock);
}

int blk_mq_sysfs_register(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i, ret = 0;

	mutex_lock(&q->sysfs_lock);
	if (!q->mq_sysfs_init_done)
		goto unlock;

	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_register_hctx(hctx);
		if (ret)
			break;
	}

unlock:
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Commit	Line	Data
320ae51f JA	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/backing-dev.h>
	4	#include <linux/bio.h>
	5	#include <linux/blkdev.h>
	6	#include <linux/mm.h>
	7	#include <linux/init.h>
	8	#include <linux/slab.h>
	9	#include <linux/workqueue.h>
	10	#include <linux/smp.h>
	11
	12	#include <linux/blk-mq.h>
84c8ff7f	13	#include "blk.h"
320ae51f JA	14	#include "blk-mq.h"
	15	#include "blk-mq-tag.h"
	16
	17	static void blk_mq_sysfs_release(struct kobject *kobj)
	18	{
	19	}
	20
6c8b232e ML	21	static void blk_mq_hw_sysfs_release(struct kobject *kobj)
	22	{
	23	struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
	24	kobj);
84c8ff7f ML	25
	26	if (hctx->flags & BLK_MQ_F_BLOCKING)
	27	cleanup_srcu_struct(hctx->queue_rq_srcu);
	28	blk_free_flush_queue(hctx->fq);
	29	sbitmap_free(&hctx->ctx_map);
01388df3	30	free_cpumask_var(hctx->cpumask);
6c8b232e ML	31	kfree(hctx->ctxs);
	32	kfree(hctx);
	33	}
	34
320ae51f JA	35	struct blk_mq_ctx_sysfs_entry {
	36	struct attribute attr;
	37	ssize_t (show)(struct blk_mq_ctx , char *);
	38	ssize_t (store)(struct blk_mq_ctx , const char *, size_t);
	39	};
	40
	41	struct blk_mq_hw_ctx_sysfs_entry {
	42	struct attribute attr;
	43	ssize_t (show)(struct blk_mq_hw_ctx , char *);
	44	ssize_t (store)(struct blk_mq_hw_ctx , const char *, size_t);
	45	};
	46
	47	static ssize_t blk_mq_sysfs_show(struct kobject kobj, struct attribute attr,
	48	char *page)
	49	{
	50	struct blk_mq_ctx_sysfs_entry *entry;
	51	struct blk_mq_ctx *ctx;
	52	struct request_queue *q;
	53	ssize_t res;
	54
	55	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	56	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	57	q = ctx->queue;
	58
	59	if (!entry->show)
	60	return -EIO;
	61
	62	res = -ENOENT;
	63	mutex_lock(&q->sysfs_lock);
	64	if (!blk_queue_dying(q))
	65	res = entry->show(ctx, page);
	66	mutex_unlock(&q->sysfs_lock);
	67	return res;
	68	}
	69
	70	static ssize_t blk_mq_sysfs_store(struct kobject kobj, struct attribute attr,
	71	const char *page, size_t length)
	72	{
	73	struct blk_mq_ctx_sysfs_entry *entry;
	74	struct blk_mq_ctx *ctx;
	75	struct request_queue *q;
	76	ssize_t res;
	77
	78	entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
	79	ctx = container_of(kobj, struct blk_mq_ctx, kobj);
	80	q = ctx->queue;
	81
	82	if (!entry->store)
	83	return -EIO;
	84
	85	res = -ENOENT;
	86	mutex_lock(&q->sysfs_lock);
	87	if (!blk_queue_dying(q))
	88	res = entry->store(ctx, page, length);
	89	mutex_unlock(&q->sysfs_lock);
	90	return res;
	91	}
	92
	93	static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
	94	struct attribute attr, char page)
	95	{
	96	struct blk_mq_hw_ctx_sysfs_entry *entry;
	97	struct blk_mq_hw_ctx *hctx;
	98	struct request_queue *q;
99	ssize_t res;
100
101	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
102	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
103	q = hctx->queue;
104
105	if (!entry->show)
106	return -EIO;
107
108	res = -ENOENT;
109	mutex_lock(&q->sysfs_lock);
110	if (!blk_queue_dying(q))
111	res = entry->show(hctx, page);
112	mutex_unlock(&q->sysfs_lock);
113	return res;
114	}
115
116	static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
117	struct attribute attr, const char page,
118	size_t length)
119	{
120	struct blk_mq_hw_ctx_sysfs_entry *entry;
121	struct blk_mq_hw_ctx *hctx;
122	struct request_queue *q;
123	ssize_t res;
124
125	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
126	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
127	q = hctx->queue;
128
129	if (!entry->store)
130	return -EIO;
131
132	res = -ENOENT;
133	mutex_lock(&q->sysfs_lock);
134	if (!blk_queue_dying(q))
135	res = entry->store(hctx, page, length);
136	mutex_unlock(&q->sysfs_lock);
137	return res;
138	}
139
d96b37c0 OS	140	static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	141	char *page)
bd166ef1	142	{
d96b37c0	143	return sprintf(page, "%u\n", hctx->tags->nr_tags);
bd166ef1 JA	144	}
bd166ef1 JA	145
d96b37c0 OS	146	static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
d96b37c0 OS	147	char *page)
320ae51f	148	{
d96b37c0	149	return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
320ae51f JA	150	}
320ae51f JA	151
676141e4 JA	152	static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx hctx, char page)
676141e4 JA	153	{
d00dc13c	154	const size_t size = PAGE_SIZE - 1;
cb2da43e	155	unsigned int i, first = 1;
d00dc13c	156	int ret = 0, pos = 0;
676141e4	157
cb2da43e	158	for_each_cpu(i, hctx->cpumask) {
676141e4	159	if (first)
d00dc13c	160	ret = snprintf(pos + page, size - pos, "%u", i);
676141e4	161	else
d00dc13c ML	162	ret = snprintf(pos + page, size - pos, ", %u", i);
	163
	164	if (ret >= size - pos)
	165	break;
676141e4 JA	166
676141e4 JA	167	first = 0;
d00dc13c	168	pos += ret;
676141e4 JA	169	}
676141e4 JA	170
61fcd9bc	171	ret = snprintf(pos + page, size + 1 - pos, "\n");
d00dc13c	172	return pos + ret;
676141e4 JA	173	}
676141e4 JA	174
320ae51f	175	static struct attribute *default_ctx_attrs[] = {
320ae51f JA	176	NULL,
	177	};
	178
d96b37c0 OS	179	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
	180	.attr = {.name = "nr_tags", .mode = S_IRUGO },
	181	.show = blk_mq_hw_sysfs_nr_tags_show,
	182	};
	183	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
	184	.attr = {.name = "nr_reserved_tags", .mode = S_IRUGO },
	185	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
	186	};
676141e4 JA	187	static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
	188	.attr = {.name = "cpu_list", .mode = S_IRUGO },
	189	.show = blk_mq_hw_sysfs_cpus_show,
	190	};
320ae51f JA	191
320ae51f JA	192	static struct attribute *default_hw_ctx_attrs[] = {
d96b37c0 OS	193	&blk_mq_hw_sysfs_nr_tags.attr,
d96b37c0 OS	194	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
676141e4	195	&blk_mq_hw_sysfs_cpus.attr,
320ae51f JA	196	NULL,
	197	};
	198
	199	static const struct sysfs_ops blk_mq_sysfs_ops = {
	200	.show = blk_mq_sysfs_show,
	201	.store = blk_mq_sysfs_store,
	202	};
	203
	204	static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
	205	.show = blk_mq_hw_sysfs_show,
	206	.store = blk_mq_hw_sysfs_store,
	207	};
	208
	209	static struct kobj_type blk_mq_ktype = {
	210	.sysfs_ops = &blk_mq_sysfs_ops,
	211	.release = blk_mq_sysfs_release,
	212	};
	213
	214	static struct kobj_type blk_mq_ctx_ktype = {
	215	.sysfs_ops = &blk_mq_sysfs_ops,
	216	.default_attrs = default_ctx_attrs,
74170118	217	.release = blk_mq_sysfs_release,
320ae51f JA	218	};
	219
	220	static struct kobj_type blk_mq_hw_ktype = {
	221	.sysfs_ops = &blk_mq_hw_sysfs_ops,
	222	.default_attrs = default_hw_ctx_attrs,
6c8b232e	223	.release = blk_mq_hw_sysfs_release,
320ae51f JA	224	};
320ae51f JA	225
ee3c5db0	226	static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	227	{
	228	struct blk_mq_ctx *ctx;
	229	int i;
	230
4593fdbe	231	if (!hctx->nr_ctx)
67aec14c JA	232	return;
	233
	234	hctx_for_each_ctx(hctx, ctx, i)
	235	kobject_del(&ctx->kobj);
	236
	237	kobject_del(&hctx->kobj);
	238	}
	239
ee3c5db0	240	static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
67aec14c JA	241	{
	242	struct request_queue *q = hctx->queue;
	243	struct blk_mq_ctx *ctx;
	244	int i, ret;
	245
4593fdbe	246	if (!hctx->nr_ctx)
67aec14c JA	247	return 0;
	248
	249	ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
	250	if (ret)
	251	return ret;
	252
	253	hctx_for_each_ctx(hctx, ctx, i) {
	254	ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
	255	if (ret)
	256	break;
	257	}
	258
	259	return ret;
	260	}
	261
b21d5b30	262	static void __blk_mq_unregister_dev(struct device dev, struct request_queue q)
320ae51f	263	{
85157366	264	struct blk_mq_hw_ctx *hctx;
7ea5fe31	265	int i;
85157366	266
2d0364c8 BVA	267	lockdep_assert_held(&q->sysfs_lock);
2d0364c8 BVA	268
6c8b232e	269	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	270	blk_mq_unregister_hctx(hctx);
67aec14c JA	271
320ae51f JA	272	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
320ae51f JA	273	kobject_del(&q->mq_kobj);
b21d5b30	274	kobject_put(&dev->kobj);
4593fdbe AM	275
4593fdbe AM	276	q->mq_sysfs_init_done = false;
c0f3fd2b JA	277	}
c0f3fd2b JA	278
b21d5b30	279	void blk_mq_unregister_dev(struct device dev, struct request_queue q)
c0f3fd2b	280	{
2d0364c8	281	mutex_lock(&q->sysfs_lock);
b21d5b30	282	__blk_mq_unregister_dev(dev, q);
2d0364c8	283	mutex_unlock(&q->sysfs_lock);
320ae51f JA	284	}
320ae51f JA	285
868f2f0b KB	286	void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
	287	{
	288	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
	289	}
	290
7ea5fe31 ML	291	void blk_mq_sysfs_deinit(struct request_queue *q)
	292	{
	293	struct blk_mq_ctx *ctx;
	294	int cpu;
	295
	296	for_each_possible_cpu(cpu) {
	297	ctx = per_cpu_ptr(q->queue_ctx, cpu);
	298	kobject_put(&ctx->kobj);
	299	}
	300	kobject_put(&q->mq_kobj);
	301	}
	302
737f98cf	303	void blk_mq_sysfs_init(struct request_queue *q)
67aec14c	304	{
67aec14c	305	struct blk_mq_ctx *ctx;
897bb0c7	306	int cpu;
67aec14c JA	307
	308	kobject_init(&q->mq_kobj, &blk_mq_ktype);
	309
897bb0c7 TG	310	for_each_possible_cpu(cpu) {
897bb0c7 TG	311	ctx = per_cpu_ptr(q->queue_ctx, cpu);
06a41a99	312	kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
897bb0c7	313	}
67aec14c JA	314	}
67aec14c JA	315
2d0364c8	316	int __blk_mq_register_dev(struct device dev, struct request_queue q)
320ae51f	317	{
320ae51f	318	struct blk_mq_hw_ctx *hctx;
67aec14c	319	int ret, i;
320ae51f	320
2d0364c8 BVA	321	WARN_ON_ONCE(!q->kobj.parent);
2d0364c8 BVA	322	lockdep_assert_held(&q->sysfs_lock);
4593fdbe	323
320ae51f JA	324	ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
320ae51f JA	325	if (ret < 0)
4593fdbe	326	goto out;
320ae51f JA	327
	328	kobject_uevent(&q->mq_kobj, KOBJ_ADD);
	329
	330	queue_for_each_hw_ctx(q, hctx, i) {
67aec14c	331	ret = blk_mq_register_hctx(hctx);
320ae51f	332	if (ret)
f05d1ba7	333	goto unreg;
320ae51f JA	334	}
320ae51f JA	335
f05d1ba7	336	q->mq_sysfs_init_done = true;
2d0364c8	337
4593fdbe	338	out:
2d0364c8	339	return ret;
f05d1ba7 BVA	340
	341	unreg:
	342	while (--i >= 0)
	343	blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
	344
f05d1ba7 BVA	345	kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
	346	kobject_del(&q->mq_kobj);
	347	kobject_put(&dev->kobj);
	348	return ret;
2d0364c8 BVA	349	}
	350
	351	int blk_mq_register_dev(struct device dev, struct request_queue q)
	352	{
	353	int ret;
	354
	355	mutex_lock(&q->sysfs_lock);
	356	ret = __blk_mq_register_dev(dev, q);
	357	mutex_unlock(&q->sysfs_lock);
320ae51f	358
4593fdbe	359	return ret;
320ae51f	360	}
b21d5b30	361	EXPORT_SYMBOL_GPL(blk_mq_register_dev);
67aec14c JA	362
	363	void blk_mq_sysfs_unregister(struct request_queue *q)
	364	{
	365	struct blk_mq_hw_ctx *hctx;
	366	int i;
	367
2d0364c8	368	mutex_lock(&q->sysfs_lock);
4593fdbe	369	if (!q->mq_sysfs_init_done)
2d0364c8	370	goto unlock;
4593fdbe	371
67aec14c JA	372	queue_for_each_hw_ctx(q, hctx, i)
67aec14c JA	373	blk_mq_unregister_hctx(hctx);
2d0364c8 BVA	374
	375	unlock:
	376	mutex_unlock(&q->sysfs_lock);
67aec14c JA	377	}
	378
	379	int blk_mq_sysfs_register(struct request_queue *q)
	380	{
	381	struct blk_mq_hw_ctx *hctx;
	382	int i, ret = 0;
	383
2d0364c8	384	mutex_lock(&q->sysfs_lock);
4593fdbe	385	if (!q->mq_sysfs_init_done)
2d0364c8	386	goto unlock;
4593fdbe	387
67aec14c JA	388	queue_for_each_hw_ctx(q, hctx, i) {
	389	ret = blk_mq_register_hctx(hctx);
	390	if (ret)
	391	break;
	392	}
	393
2d0364c8 BVA	394	unlock:
	395	mutex_unlock(&q->sysfs_lock);
	396
67aec14c JA	397	return ret;
67aec14c JA	398	}