1 #include <linux/kernel.h>
2 #include <linux/module.h>
3 #include <linux/backing-dev.h>
5 #include <linux/blkdev.h>
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/workqueue.h>
10 #include <linux/smp.h>
12 #include <linux/blk-mq.h>
14 #include "blk-mq-tag.h"
16 static void blk_mq_sysfs_release(struct kobject
*kobj
)
18 struct request_queue
*q
;
20 q
= container_of(kobj
, struct request_queue
, mq_kobj
);
21 free_percpu(q
->queue_ctx
);
24 static void blk_mq_ctx_release(struct kobject
*kobj
)
26 struct blk_mq_ctx
*ctx
;
28 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
29 kobject_put(&ctx
->queue
->mq_kobj
);
32 static void blk_mq_hctx_release(struct kobject
*kobj
)
34 struct blk_mq_hw_ctx
*hctx
;
36 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
40 struct blk_mq_ctx_sysfs_entry
{
41 struct attribute attr
;
42 ssize_t (*show
)(struct blk_mq_ctx
*, char *);
43 ssize_t (*store
)(struct blk_mq_ctx
*, const char *, size_t);
46 struct blk_mq_hw_ctx_sysfs_entry
{
47 struct attribute attr
;
48 ssize_t (*show
)(struct blk_mq_hw_ctx
*, char *);
49 ssize_t (*store
)(struct blk_mq_hw_ctx
*, const char *, size_t);
52 static ssize_t
blk_mq_sysfs_show(struct kobject
*kobj
, struct attribute
*attr
,
55 struct blk_mq_ctx_sysfs_entry
*entry
;
56 struct blk_mq_ctx
*ctx
;
57 struct request_queue
*q
;
60 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
61 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
68 mutex_lock(&q
->sysfs_lock
);
69 if (!blk_queue_dying(q
))
70 res
= entry
->show(ctx
, page
);
71 mutex_unlock(&q
->sysfs_lock
);
75 static ssize_t
blk_mq_sysfs_store(struct kobject
*kobj
, struct attribute
*attr
,
76 const char *page
, size_t length
)
78 struct blk_mq_ctx_sysfs_entry
*entry
;
79 struct blk_mq_ctx
*ctx
;
80 struct request_queue
*q
;
83 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
84 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
91 mutex_lock(&q
->sysfs_lock
);
92 if (!blk_queue_dying(q
))
93 res
= entry
->store(ctx
, page
, length
);
94 mutex_unlock(&q
->sysfs_lock
);
98 static ssize_t
blk_mq_hw_sysfs_show(struct kobject
*kobj
,
99 struct attribute
*attr
, char *page
)
101 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
102 struct blk_mq_hw_ctx
*hctx
;
103 struct request_queue
*q
;
106 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
107 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
114 mutex_lock(&q
->sysfs_lock
);
115 if (!blk_queue_dying(q
))
116 res
= entry
->show(hctx
, page
);
117 mutex_unlock(&q
->sysfs_lock
);
121 static ssize_t
blk_mq_hw_sysfs_store(struct kobject
*kobj
,
122 struct attribute
*attr
, const char *page
,
125 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
126 struct blk_mq_hw_ctx
*hctx
;
127 struct request_queue
*q
;
130 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
131 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
138 mutex_lock(&q
->sysfs_lock
);
139 if (!blk_queue_dying(q
))
140 res
= entry
->store(hctx
, page
, length
);
141 mutex_unlock(&q
->sysfs_lock
);
145 static ssize_t
blk_mq_sysfs_dispatched_show(struct blk_mq_ctx
*ctx
, char *page
)
147 return sprintf(page
, "%lu %lu\n", ctx
->rq_dispatched
[1],
148 ctx
->rq_dispatched
[0]);
151 static ssize_t
blk_mq_sysfs_merged_show(struct blk_mq_ctx
*ctx
, char *page
)
153 return sprintf(page
, "%lu\n", ctx
->rq_merged
);
156 static ssize_t
blk_mq_sysfs_completed_show(struct blk_mq_ctx
*ctx
, char *page
)
158 return sprintf(page
, "%lu %lu\n", ctx
->rq_completed
[1],
159 ctx
->rq_completed
[0]);
162 static ssize_t
sysfs_list_show(char *page
, struct list_head
*list
, char *msg
)
164 char *start_page
= page
;
167 page
+= sprintf(page
, "%s:\n", msg
);
169 list_for_each_entry(rq
, list
, queuelist
)
170 page
+= sprintf(page
, "\t%p\n", rq
);
172 return page
- start_page
;
175 static ssize_t
blk_mq_sysfs_rq_list_show(struct blk_mq_ctx
*ctx
, char *page
)
179 spin_lock(&ctx
->lock
);
180 ret
= sysfs_list_show(page
, &ctx
->rq_list
, "CTX pending");
181 spin_unlock(&ctx
->lock
);
186 static ssize_t
blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx
*hctx
,
189 return sprintf(page
, "%lu\n", hctx
->queued
);
192 static ssize_t
blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
194 return sprintf(page
, "%lu\n", hctx
->run
);
197 static ssize_t
blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx
*hctx
,
200 char *start_page
= page
;
203 page
+= sprintf(page
, "%8u\t%lu\n", 0U, hctx
->dispatched
[0]);
205 for (i
= 1; i
< BLK_MQ_MAX_DISPATCH_ORDER
; i
++) {
206 unsigned long d
= 1U << (i
- 1);
208 page
+= sprintf(page
, "%8lu\t%lu\n", d
, hctx
->dispatched
[i
]);
211 return page
- start_page
;
214 static ssize_t
blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx
*hctx
,
219 spin_lock(&hctx
->lock
);
220 ret
= sysfs_list_show(page
, &hctx
->dispatch
, "HCTX pending");
221 spin_unlock(&hctx
->lock
);
226 static ssize_t
blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
228 return blk_mq_tag_sysfs_show(hctx
->tags
, page
);
231 static ssize_t
blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
233 return sprintf(page
, "%u\n", atomic_read(&hctx
->nr_active
));
236 static ssize_t
blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
238 unsigned int i
, first
= 1;
241 blk_mq_disable_hotplug();
243 for_each_cpu(i
, hctx
->cpumask
) {
245 ret
+= sprintf(ret
+ page
, "%u", i
);
247 ret
+= sprintf(ret
+ page
, ", %u", i
);
252 blk_mq_enable_hotplug();
254 ret
+= sprintf(ret
+ page
, "\n");
258 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched
= {
259 .attr
= {.name
= "dispatched", .mode
= S_IRUGO
},
260 .show
= blk_mq_sysfs_dispatched_show
,
262 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged
= {
263 .attr
= {.name
= "merged", .mode
= S_IRUGO
},
264 .show
= blk_mq_sysfs_merged_show
,
266 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed
= {
267 .attr
= {.name
= "completed", .mode
= S_IRUGO
},
268 .show
= blk_mq_sysfs_completed_show
,
270 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list
= {
271 .attr
= {.name
= "rq_list", .mode
= S_IRUGO
},
272 .show
= blk_mq_sysfs_rq_list_show
,
275 static struct attribute
*default_ctx_attrs
[] = {
276 &blk_mq_sysfs_dispatched
.attr
,
277 &blk_mq_sysfs_merged
.attr
,
278 &blk_mq_sysfs_completed
.attr
,
279 &blk_mq_sysfs_rq_list
.attr
,
283 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued
= {
284 .attr
= {.name
= "queued", .mode
= S_IRUGO
},
285 .show
= blk_mq_hw_sysfs_queued_show
,
287 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run
= {
288 .attr
= {.name
= "run", .mode
= S_IRUGO
},
289 .show
= blk_mq_hw_sysfs_run_show
,
291 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched
= {
292 .attr
= {.name
= "dispatched", .mode
= S_IRUGO
},
293 .show
= blk_mq_hw_sysfs_dispatched_show
,
295 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active
= {
296 .attr
= {.name
= "active", .mode
= S_IRUGO
},
297 .show
= blk_mq_hw_sysfs_active_show
,
299 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending
= {
300 .attr
= {.name
= "pending", .mode
= S_IRUGO
},
301 .show
= blk_mq_hw_sysfs_rq_list_show
,
303 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags
= {
304 .attr
= {.name
= "tags", .mode
= S_IRUGO
},
305 .show
= blk_mq_hw_sysfs_tags_show
,
307 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus
= {
308 .attr
= {.name
= "cpu_list", .mode
= S_IRUGO
},
309 .show
= blk_mq_hw_sysfs_cpus_show
,
312 static struct attribute
*default_hw_ctx_attrs
[] = {
313 &blk_mq_hw_sysfs_queued
.attr
,
314 &blk_mq_hw_sysfs_run
.attr
,
315 &blk_mq_hw_sysfs_dispatched
.attr
,
316 &blk_mq_hw_sysfs_pending
.attr
,
317 &blk_mq_hw_sysfs_tags
.attr
,
318 &blk_mq_hw_sysfs_cpus
.attr
,
319 &blk_mq_hw_sysfs_active
.attr
,
323 static const struct sysfs_ops blk_mq_sysfs_ops
= {
324 .show
= blk_mq_sysfs_show
,
325 .store
= blk_mq_sysfs_store
,
328 static const struct sysfs_ops blk_mq_hw_sysfs_ops
= {
329 .show
= blk_mq_hw_sysfs_show
,
330 .store
= blk_mq_hw_sysfs_store
,
333 static struct kobj_type blk_mq_ktype
= {
334 .sysfs_ops
= &blk_mq_sysfs_ops
,
335 .release
= blk_mq_sysfs_release
,
338 static struct kobj_type blk_mq_ctx_ktype
= {
339 .sysfs_ops
= &blk_mq_sysfs_ops
,
340 .default_attrs
= default_ctx_attrs
,
341 .release
= blk_mq_ctx_release
,
344 static struct kobj_type blk_mq_hw_ktype
= {
345 .sysfs_ops
= &blk_mq_hw_sysfs_ops
,
346 .default_attrs
= default_hw_ctx_attrs
,
347 .release
= blk_mq_hctx_release
,
350 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx
*hctx
)
352 struct blk_mq_ctx
*ctx
;
355 if (!hctx
->nr_ctx
|| !(hctx
->flags
& BLK_MQ_F_SYSFS_UP
))
358 hctx_for_each_ctx(hctx
, ctx
, i
)
359 kobject_del(&ctx
->kobj
);
361 kobject_del(&hctx
->kobj
);
364 static int blk_mq_register_hctx(struct blk_mq_hw_ctx
*hctx
)
366 struct request_queue
*q
= hctx
->queue
;
367 struct blk_mq_ctx
*ctx
;
370 if (!hctx
->nr_ctx
|| !(hctx
->flags
& BLK_MQ_F_SYSFS_UP
))
373 ret
= kobject_add(&hctx
->kobj
, &q
->mq_kobj
, "%u", hctx
->queue_num
);
377 hctx_for_each_ctx(hctx
, ctx
, i
) {
378 kobject_get(&q
->mq_kobj
);
379 ret
= kobject_add(&ctx
->kobj
, &hctx
->kobj
, "cpu%u", ctx
->cpu
);
387 void blk_mq_unregister_disk(struct gendisk
*disk
)
389 struct request_queue
*q
= disk
->queue
;
390 struct blk_mq_hw_ctx
*hctx
;
391 struct blk_mq_ctx
*ctx
;
394 queue_for_each_hw_ctx(q
, hctx
, i
) {
395 blk_mq_unregister_hctx(hctx
);
397 hctx_for_each_ctx(hctx
, ctx
, j
)
398 kobject_put(&ctx
->kobj
);
400 kobject_put(&hctx
->kobj
);
403 kobject_uevent(&q
->mq_kobj
, KOBJ_REMOVE
);
404 kobject_del(&q
->mq_kobj
);
405 kobject_put(&q
->mq_kobj
);
407 kobject_put(&disk_to_dev(disk
)->kobj
);
410 static void blk_mq_sysfs_init(struct request_queue
*q
)
412 struct blk_mq_hw_ctx
*hctx
;
413 struct blk_mq_ctx
*ctx
;
416 kobject_init(&q
->mq_kobj
, &blk_mq_ktype
);
418 queue_for_each_hw_ctx(q
, hctx
, i
)
419 kobject_init(&hctx
->kobj
, &blk_mq_hw_ktype
);
421 queue_for_each_ctx(q
, ctx
, i
)
422 kobject_init(&ctx
->kobj
, &blk_mq_ctx_ktype
);
425 /* see blk_register_queue() */
426 void blk_mq_finish_init(struct request_queue
*q
)
428 percpu_ref_switch_to_percpu(&q
->mq_usage_counter
);
431 int blk_mq_register_disk(struct gendisk
*disk
)
433 struct device
*dev
= disk_to_dev(disk
);
434 struct request_queue
*q
= disk
->queue
;
435 struct blk_mq_hw_ctx
*hctx
;
438 blk_mq_sysfs_init(q
);
440 ret
= kobject_add(&q
->mq_kobj
, kobject_get(&dev
->kobj
), "%s", "mq");
444 kobject_uevent(&q
->mq_kobj
, KOBJ_ADD
);
446 queue_for_each_hw_ctx(q
, hctx
, i
) {
447 hctx
->flags
|= BLK_MQ_F_SYSFS_UP
;
448 ret
= blk_mq_register_hctx(hctx
);
454 blk_mq_unregister_disk(disk
);
461 void blk_mq_sysfs_unregister(struct request_queue
*q
)
463 struct blk_mq_hw_ctx
*hctx
;
466 queue_for_each_hw_ctx(q
, hctx
, i
)
467 blk_mq_unregister_hctx(hctx
);
470 int blk_mq_sysfs_register(struct request_queue
*q
)
472 struct blk_mq_hw_ctx
*hctx
;
475 queue_for_each_hw_ctx(q
, hctx
, i
) {
476 ret
= blk_mq_register_hctx(hctx
);