1 #include <linux/kernel.h>
2 #include <linux/module.h>
3 #include <linux/backing-dev.h>
5 #include <linux/blkdev.h>
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/workqueue.h>
10 #include <linux/smp.h>
12 #include <linux/blk-mq.h>
15 #include "blk-mq-tag.h"
17 static void blk_mq_sysfs_release(struct kobject
*kobj
)
21 static void blk_mq_hw_sysfs_release(struct kobject
*kobj
)
23 struct blk_mq_hw_ctx
*hctx
= container_of(kobj
, struct blk_mq_hw_ctx
,
26 if (hctx
->flags
& BLK_MQ_F_BLOCKING
)
27 cleanup_srcu_struct(hctx
->queue_rq_srcu
);
28 blk_free_flush_queue(hctx
->fq
);
29 sbitmap_free(&hctx
->ctx_map
);
30 free_cpumask_var(hctx
->cpumask
);
35 struct blk_mq_ctx_sysfs_entry
{
36 struct attribute attr
;
37 ssize_t (*show
)(struct blk_mq_ctx
*, char *);
38 ssize_t (*store
)(struct blk_mq_ctx
*, const char *, size_t);
41 struct blk_mq_hw_ctx_sysfs_entry
{
42 struct attribute attr
;
43 ssize_t (*show
)(struct blk_mq_hw_ctx
*, char *);
44 ssize_t (*store
)(struct blk_mq_hw_ctx
*, const char *, size_t);
47 static ssize_t
blk_mq_sysfs_show(struct kobject
*kobj
, struct attribute
*attr
,
50 struct blk_mq_ctx_sysfs_entry
*entry
;
51 struct blk_mq_ctx
*ctx
;
52 struct request_queue
*q
;
55 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
56 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
63 mutex_lock(&q
->sysfs_lock
);
64 if (!blk_queue_dying(q
))
65 res
= entry
->show(ctx
, page
);
66 mutex_unlock(&q
->sysfs_lock
);
70 static ssize_t
blk_mq_sysfs_store(struct kobject
*kobj
, struct attribute
*attr
,
71 const char *page
, size_t length
)
73 struct blk_mq_ctx_sysfs_entry
*entry
;
74 struct blk_mq_ctx
*ctx
;
75 struct request_queue
*q
;
78 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
79 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
86 mutex_lock(&q
->sysfs_lock
);
87 if (!blk_queue_dying(q
))
88 res
= entry
->store(ctx
, page
, length
);
89 mutex_unlock(&q
->sysfs_lock
);
93 static ssize_t
blk_mq_hw_sysfs_show(struct kobject
*kobj
,
94 struct attribute
*attr
, char *page
)
96 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
97 struct blk_mq_hw_ctx
*hctx
;
98 struct request_queue
*q
;
101 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
102 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
109 mutex_lock(&q
->sysfs_lock
);
110 if (!blk_queue_dying(q
))
111 res
= entry
->show(hctx
, page
);
112 mutex_unlock(&q
->sysfs_lock
);
116 static ssize_t
blk_mq_hw_sysfs_store(struct kobject
*kobj
,
117 struct attribute
*attr
, const char *page
,
120 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
121 struct blk_mq_hw_ctx
*hctx
;
122 struct request_queue
*q
;
125 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
126 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
133 mutex_lock(&q
->sysfs_lock
);
134 if (!blk_queue_dying(q
))
135 res
= entry
->store(hctx
, page
, length
);
136 mutex_unlock(&q
->sysfs_lock
);
140 static ssize_t
blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx
*hctx
,
143 return sprintf(page
, "%u\n", hctx
->tags
->nr_tags
);
146 static ssize_t
blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx
*hctx
,
149 return sprintf(page
, "%u\n", hctx
->tags
->nr_reserved_tags
);
152 static ssize_t
blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
154 unsigned int i
, first
= 1;
157 for_each_cpu(i
, hctx
->cpumask
) {
159 ret
+= sprintf(ret
+ page
, "%u", i
);
161 ret
+= sprintf(ret
+ page
, ", %u", i
);
166 ret
+= sprintf(ret
+ page
, "\n");
170 static struct attribute
*default_ctx_attrs
[] = {
174 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags
= {
175 .attr
= {.name
= "nr_tags", .mode
= S_IRUGO
},
176 .show
= blk_mq_hw_sysfs_nr_tags_show
,
178 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags
= {
179 .attr
= {.name
= "nr_reserved_tags", .mode
= S_IRUGO
},
180 .show
= blk_mq_hw_sysfs_nr_reserved_tags_show
,
182 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus
= {
183 .attr
= {.name
= "cpu_list", .mode
= S_IRUGO
},
184 .show
= blk_mq_hw_sysfs_cpus_show
,
187 static struct attribute
*default_hw_ctx_attrs
[] = {
188 &blk_mq_hw_sysfs_nr_tags
.attr
,
189 &blk_mq_hw_sysfs_nr_reserved_tags
.attr
,
190 &blk_mq_hw_sysfs_cpus
.attr
,
194 static const struct sysfs_ops blk_mq_sysfs_ops
= {
195 .show
= blk_mq_sysfs_show
,
196 .store
= blk_mq_sysfs_store
,
199 static const struct sysfs_ops blk_mq_hw_sysfs_ops
= {
200 .show
= blk_mq_hw_sysfs_show
,
201 .store
= blk_mq_hw_sysfs_store
,
204 static struct kobj_type blk_mq_ktype
= {
205 .sysfs_ops
= &blk_mq_sysfs_ops
,
206 .release
= blk_mq_sysfs_release
,
209 static struct kobj_type blk_mq_ctx_ktype
= {
210 .sysfs_ops
= &blk_mq_sysfs_ops
,
211 .default_attrs
= default_ctx_attrs
,
212 .release
= blk_mq_sysfs_release
,
215 static struct kobj_type blk_mq_hw_ktype
= {
216 .sysfs_ops
= &blk_mq_hw_sysfs_ops
,
217 .default_attrs
= default_hw_ctx_attrs
,
218 .release
= blk_mq_hw_sysfs_release
,
221 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx
*hctx
)
223 struct blk_mq_ctx
*ctx
;
229 hctx_for_each_ctx(hctx
, ctx
, i
)
230 kobject_del(&ctx
->kobj
);
232 kobject_del(&hctx
->kobj
);
235 static int blk_mq_register_hctx(struct blk_mq_hw_ctx
*hctx
)
237 struct request_queue
*q
= hctx
->queue
;
238 struct blk_mq_ctx
*ctx
;
244 ret
= kobject_add(&hctx
->kobj
, &q
->mq_kobj
, "%u", hctx
->queue_num
);
248 hctx_for_each_ctx(hctx
, ctx
, i
) {
249 ret
= kobject_add(&ctx
->kobj
, &hctx
->kobj
, "cpu%u", ctx
->cpu
);
257 static void __blk_mq_unregister_dev(struct device
*dev
, struct request_queue
*q
)
259 struct blk_mq_hw_ctx
*hctx
;
262 lockdep_assert_held(&q
->sysfs_lock
);
264 queue_for_each_hw_ctx(q
, hctx
, i
)
265 blk_mq_unregister_hctx(hctx
);
267 kobject_uevent(&q
->mq_kobj
, KOBJ_REMOVE
);
268 kobject_del(&q
->mq_kobj
);
269 kobject_put(&dev
->kobj
);
271 q
->mq_sysfs_init_done
= false;
274 void blk_mq_unregister_dev(struct device
*dev
, struct request_queue
*q
)
276 mutex_lock(&q
->sysfs_lock
);
277 __blk_mq_unregister_dev(dev
, q
);
278 mutex_unlock(&q
->sysfs_lock
);
281 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx
*hctx
)
283 kobject_init(&hctx
->kobj
, &blk_mq_hw_ktype
);
286 void blk_mq_sysfs_deinit(struct request_queue
*q
)
288 struct blk_mq_ctx
*ctx
;
291 for_each_possible_cpu(cpu
) {
292 ctx
= per_cpu_ptr(q
->queue_ctx
, cpu
);
293 kobject_put(&ctx
->kobj
);
295 kobject_put(&q
->mq_kobj
);
298 void blk_mq_sysfs_init(struct request_queue
*q
)
300 struct blk_mq_ctx
*ctx
;
303 kobject_init(&q
->mq_kobj
, &blk_mq_ktype
);
305 for_each_possible_cpu(cpu
) {
306 ctx
= per_cpu_ptr(q
->queue_ctx
, cpu
);
307 kobject_init(&ctx
->kobj
, &blk_mq_ctx_ktype
);
311 int __blk_mq_register_dev(struct device
*dev
, struct request_queue
*q
)
313 struct blk_mq_hw_ctx
*hctx
;
316 WARN_ON_ONCE(!q
->kobj
.parent
);
317 lockdep_assert_held(&q
->sysfs_lock
);
319 ret
= kobject_add(&q
->mq_kobj
, kobject_get(&dev
->kobj
), "%s", "mq");
323 kobject_uevent(&q
->mq_kobj
, KOBJ_ADD
);
325 queue_for_each_hw_ctx(q
, hctx
, i
) {
326 ret
= blk_mq_register_hctx(hctx
);
331 q
->mq_sysfs_init_done
= true;
338 blk_mq_unregister_hctx(q
->queue_hw_ctx
[i
]);
340 kobject_uevent(&q
->mq_kobj
, KOBJ_REMOVE
);
341 kobject_del(&q
->mq_kobj
);
342 kobject_put(&dev
->kobj
);
346 int blk_mq_register_dev(struct device
*dev
, struct request_queue
*q
)
350 mutex_lock(&q
->sysfs_lock
);
351 ret
= __blk_mq_register_dev(dev
, q
);
352 mutex_unlock(&q
->sysfs_lock
);
356 EXPORT_SYMBOL_GPL(blk_mq_register_dev
);
358 void blk_mq_sysfs_unregister(struct request_queue
*q
)
360 struct blk_mq_hw_ctx
*hctx
;
363 mutex_lock(&q
->sysfs_lock
);
364 if (!q
->mq_sysfs_init_done
)
367 queue_for_each_hw_ctx(q
, hctx
, i
)
368 blk_mq_unregister_hctx(hctx
);
371 mutex_unlock(&q
->sysfs_lock
);
374 int blk_mq_sysfs_register(struct request_queue
*q
)
376 struct blk_mq_hw_ctx
*hctx
;
379 mutex_lock(&q
->sysfs_lock
);
380 if (!q
->mq_sysfs_init_done
)
383 queue_for_each_hw_ctx(q
, hctx
, i
) {
384 ret
= blk_mq_register_hctx(hctx
);
390 mutex_unlock(&q
->sysfs_lock
);