block/blk-mq-sysfs.c

   1 #include <linux/kernel.h>
   2 #include <linux/module.h>
   3 #include <linux/backing-dev.h>
   4 #include <linux/bio.h>
   5 #include <linux/blkdev.h>
   6 #include <linux/mm.h>
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/smp.h>
  11
  12 #include <linux/blk-mq.h>
  13 #include "blk-mq.h"
  14 #include "blk-mq-tag.h"
  15
  16 static void blk_mq_sysfs_release(struct kobject *kobj)
  17 {
  18 }
  19
  20 struct blk_mq_ctx_sysfs_entry {
  21         struct attribute attr;
  22         ssize_t (*show)(struct blk_mq_ctx *, char *);
  23         ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
  24 };
  25
  26 struct blk_mq_hw_ctx_sysfs_entry {
  27         struct attribute attr;
  28         ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
  29         ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
  30 };
  31
  32 static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
  33                                  char *page)
  34 {
  35         struct blk_mq_ctx_sysfs_entry *entry;
  36         struct blk_mq_ctx *ctx;
  37         struct request_queue *q;
  38         ssize_t res;
  39
  40         entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
  41         ctx = container_of(kobj, struct blk_mq_ctx, kobj);
  42         q = ctx->queue;
  43
  44         if (!entry->show)
  45                 return -EIO;
  46
  47         res = -ENOENT;
  48         mutex_lock(&q->sysfs_lock);
  49         if (!blk_queue_dying(q))
  50                 res = entry->show(ctx, page);
  51         mutex_unlock(&q->sysfs_lock);
  52         return res;
  53 }
  54
  55 static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
  56                                   const char *page, size_t length)
  57 {
  58         struct blk_mq_ctx_sysfs_entry *entry;
  59         struct blk_mq_ctx *ctx;
  60         struct request_queue *q;
  61         ssize_t res;
  62
  63         entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
  64         ctx = container_of(kobj, struct blk_mq_ctx, kobj);
  65         q = ctx->queue;
  66
  67         if (!entry->store)
  68                 return -EIO;
  69
  70         res = -ENOENT;
  71         mutex_lock(&q->sysfs_lock);
  72         if (!blk_queue_dying(q))
  73                 res = entry->store(ctx, page, length);
  74         mutex_unlock(&q->sysfs_lock);
  75         return res;
  76 }
  77
  78 static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
  79                                     struct attribute *attr, char *page)
  80 {
  81         struct blk_mq_hw_ctx_sysfs_entry *entry;
  82         struct blk_mq_hw_ctx *hctx;
  83         struct request_queue *q;
  84         ssize_t res;
  85
  86         entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
  87         hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
  88         q = hctx->queue;
  89
  90         if (!entry->show)
  91                 return -EIO;
  92
  93         res = -ENOENT;
  94         mutex_lock(&q->sysfs_lock);
  95         if (!blk_queue_dying(q))
  96                 res = entry->show(hctx, page);
  97         mutex_unlock(&q->sysfs_lock);
  98         return res;
  99 }
 100
 101 static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
 102                                      struct attribute *attr, const char *page,
 103                                      size_t length)
 104 {
 105         struct blk_mq_hw_ctx_sysfs_entry *entry;
 106         struct blk_mq_hw_ctx *hctx;
 107         struct request_queue *q;
 108         ssize_t res;
 109
 110         entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
 111         hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
 112         q = hctx->queue;
 113
 114         if (!entry->store)
 115                 return -EIO;
 116
 117         res = -ENOENT;
 118         mutex_lock(&q->sysfs_lock);
 119         if (!blk_queue_dying(q))
 120                 res = entry->store(hctx, page, length);
 121         mutex_unlock(&q->sysfs_lock);
 122         return res;
 123 }
 124
 125 static ssize_t blk_mq_sysfs_dispatched_show(struct blk_mq_ctx *ctx, char *page)
 126 {
 127         return sprintf(page, "%lu %lu\n", ctx->rq_dispatched[1],
 128                                 ctx->rq_dispatched[0]);
 129 }
 130
 131 static ssize_t blk_mq_sysfs_merged_show(struct blk_mq_ctx *ctx, char *page)
 132 {
 133         return sprintf(page, "%lu\n", ctx->rq_merged);
 134 }
 135
 136 static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
 137 {
 138         return sprintf(page, "%lu %lu\n", ctx->rq_completed[1],
 139                                 ctx->rq_completed[0]);
 140 }
 141
 142 static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg)
 143 {
 144         struct request *rq;
 145         int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg);
 146
 147         list_for_each_entry(rq, list, queuelist) {
 148                 const int rq_len = 2 * sizeof(rq) + 2;
 149
 150                 /* if the output will be truncated */
 151                 if (PAGE_SIZE - 1 < len + rq_len) {
 152                         /* backspacing if it can't hold '\t...\n' */
 153                         if (PAGE_SIZE - 1 < len + 5)
 154                                 len -= rq_len;
 155                         len += snprintf(page + len, PAGE_SIZE - 1 - len,
 156                                         "\t...\n");
 157                         break;
 158                 }
 159                 len += snprintf(page + len, PAGE_SIZE - 1 - len,
 160                                 "\t%p\n", rq);
 161         }
 162
 163         return len;
 164 }
 165
 166 static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
 167 {
 168         ssize_t ret;
 169
 170         spin_lock(&ctx->lock);
 171         ret = sysfs_list_show(page, &ctx->rq_list, "CTX pending");
 172         spin_unlock(&ctx->lock);
 173
 174         return ret;
 175 }
 176
 177 static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
 178 {
 179         return sprintf(page, "considered=%lu, invoked=%lu, success=%lu\n",
 180                        hctx->poll_considered, hctx->poll_invoked,
 181                        hctx->poll_success);
 182 }
 183
 184 static ssize_t blk_mq_hw_sysfs_poll_store(struct blk_mq_hw_ctx *hctx,
 185                                           const char *page, size_t size)
 186 {
 187         hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0;
 188
 189         return size;
 190 }
 191
 192 static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
 193                                            char *page)
 194 {
 195         return sprintf(page, "%lu\n", hctx->queued);
 196 }
 197
 198 static ssize_t blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx *hctx, char *page)
 199 {
 200         return sprintf(page, "%lu\n", hctx->run);
 201 }
 202
 203 static ssize_t blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx *hctx,
 204                                                char *page)
 205 {
 206         char *start_page = page;
 207         int i;
 208
 209         page += sprintf(page, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
 210
 211         for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) {
 212                 unsigned int d = 1U << (i - 1);
 213
 214                 page += sprintf(page, "%8u\t%lu\n", d, hctx->dispatched[i]);
 215         }
 216
 217         page += sprintf(page, "%8u+\t%lu\n", 1U << (i - 1),
 218                                                 hctx->dispatched[i]);
 219         return page - start_page;
 220 }
 221
 222 static ssize_t blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx *hctx,
 223                                             char *page)
 224 {
 225         ssize_t ret;
 226
 227         spin_lock(&hctx->lock);
 228         ret = sysfs_list_show(page, &hctx->dispatch, "HCTX pending");
 229         spin_unlock(&hctx->lock);
 230
 231         return ret;
 232 }
 233
 234 static ssize_t blk_mq_hw_sysfs_sched_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
 235 {
 236         if (hctx->sched_tags)
 237                 return blk_mq_tag_sysfs_show(hctx->sched_tags, page);
 238
 239         return 0;
 240 }
 241
 242 static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
 243 {
 244         return blk_mq_tag_sysfs_show(hctx->tags, page);
 245 }
 246
 247 static ssize_t blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx *hctx, char *page)
 248 {
 249         return sprintf(page, "%u\n", atomic_read(&hctx->nr_active));
 250 }
 251
 252 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
 253 {
 254         unsigned int i, first = 1;
 255         ssize_t ret = 0;
 256
 257         for_each_cpu(i, hctx->cpumask) {
 258                 if (first)
 259                         ret += sprintf(ret + page, "%u", i);
 260                 else
 261                         ret += sprintf(ret + page, ", %u", i);
 262
 263                 first = 0;
 264         }
 265
 266         ret += sprintf(ret + page, "\n");
 267         return ret;
 268 }
 269
 270 static void blk_mq_stat_clear(struct blk_mq_hw_ctx *hctx)
 271 {
 272         struct blk_mq_ctx *ctx;
 273         unsigned int i;
 274
 275         hctx_for_each_ctx(hctx, ctx, i) {
 276                 blk_stat_init(&ctx->stat[BLK_STAT_READ]);
 277                 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
 278         }
 279 }
 280
 281 static ssize_t blk_mq_hw_sysfs_stat_store(struct blk_mq_hw_ctx *hctx,
 282                                           const char *page, size_t count)
 283 {
 284         blk_mq_stat_clear(hctx);
 285         return count;
 286 }
 287
 288 static ssize_t print_stat(char *page, struct blk_rq_stat *stat, const char *pre)
 289 {
 290         return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
 291                         pre, (long long) stat->nr_samples,
 292                         (long long) stat->mean, (long long) stat->min,
 293                         (long long) stat->max);
 294 }
 295
 296 static ssize_t blk_mq_hw_sysfs_stat_show(struct blk_mq_hw_ctx *hctx, char *page)
 297 {
 298         struct blk_rq_stat stat[2];
 299         ssize_t ret;
 300
 301         blk_stat_init(&stat[BLK_STAT_READ]);
 302         blk_stat_init(&stat[BLK_STAT_WRITE]);
 303
 304         blk_hctx_stat_get(hctx, stat);
 305
 306         ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
 307         ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
 308         return ret;
 309 }
 310
 311 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
 312         .attr = {.name = "dispatched", .mode = S_IRUGO },
 313         .show = blk_mq_sysfs_dispatched_show,
 314 };
 315 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged = {
 316         .attr = {.name = "merged", .mode = S_IRUGO },
 317         .show = blk_mq_sysfs_merged_show,
 318 };
 319 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed = {
 320         .attr = {.name = "completed", .mode = S_IRUGO },
 321         .show = blk_mq_sysfs_completed_show,
 322 };
 323 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list = {
 324         .attr = {.name = "rq_list", .mode = S_IRUGO },
 325         .show = blk_mq_sysfs_rq_list_show,
 326 };
 327
 328 static struct attribute *default_ctx_attrs[] = {
 329         &blk_mq_sysfs_dispatched.attr,
 330         &blk_mq_sysfs_merged.attr,
 331         &blk_mq_sysfs_completed.attr,
 332         &blk_mq_sysfs_rq_list.attr,
 333         NULL,
 334 };
 335
 336 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued = {
 337         .attr = {.name = "queued", .mode = S_IRUGO },
 338         .show = blk_mq_hw_sysfs_queued_show,
 339 };
 340 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run = {
 341         .attr = {.name = "run", .mode = S_IRUGO },
 342         .show = blk_mq_hw_sysfs_run_show,
 343 };
 344 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched = {
 345         .attr = {.name = "dispatched", .mode = S_IRUGO },
 346         .show = blk_mq_hw_sysfs_dispatched_show,
 347 };
 348 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active = {
 349         .attr = {.name = "active", .mode = S_IRUGO },
 350         .show = blk_mq_hw_sysfs_active_show,
 351 };
 352 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending = {
 353         .attr = {.name = "pending", .mode = S_IRUGO },
 354         .show = blk_mq_hw_sysfs_rq_list_show,
 355 };
 356 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_sched_tags = {
 357         .attr = {.name = "sched_tags", .mode = S_IRUGO },
 358         .show = blk_mq_hw_sysfs_sched_tags_show,
 359 };
 360 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
 361         .attr = {.name = "tags", .mode = S_IRUGO },
 362         .show = blk_mq_hw_sysfs_tags_show,
 363 };
 364 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
 365         .attr = {.name = "cpu_list", .mode = S_IRUGO },
 366         .show = blk_mq_hw_sysfs_cpus_show,
 367 };
 368 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
 369         .attr = {.name = "io_poll", .mode = S_IWUSR | S_IRUGO },
 370         .show = blk_mq_hw_sysfs_poll_show,
 371         .store = blk_mq_hw_sysfs_poll_store,
 372 };
 373 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_stat = {
 374         .attr = {.name = "stats", .mode = S_IRUGO | S_IWUSR },
 375         .show = blk_mq_hw_sysfs_stat_show,
 376         .store = blk_mq_hw_sysfs_stat_store,
 377 };
 378
 379 static struct attribute *default_hw_ctx_attrs[] = {
 380         &blk_mq_hw_sysfs_queued.attr,
 381         &blk_mq_hw_sysfs_run.attr,
 382         &blk_mq_hw_sysfs_dispatched.attr,
 383         &blk_mq_hw_sysfs_pending.attr,
 384         &blk_mq_hw_sysfs_tags.attr,
 385         &blk_mq_hw_sysfs_sched_tags.attr,
 386         &blk_mq_hw_sysfs_cpus.attr,
 387         &blk_mq_hw_sysfs_active.attr,
 388         &blk_mq_hw_sysfs_poll.attr,
 389         &blk_mq_hw_sysfs_stat.attr,
 390         NULL,
 391 };
 392
 393 static const struct sysfs_ops blk_mq_sysfs_ops = {
 394         .show   = blk_mq_sysfs_show,
 395         .store  = blk_mq_sysfs_store,
 396 };
 397
 398 static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
 399         .show   = blk_mq_hw_sysfs_show,
 400         .store  = blk_mq_hw_sysfs_store,
 401 };
 402
 403 static struct kobj_type blk_mq_ktype = {
 404         .sysfs_ops      = &blk_mq_sysfs_ops,
 405         .release        = blk_mq_sysfs_release,
 406 };
 407
 408 static struct kobj_type blk_mq_ctx_ktype = {
 409         .sysfs_ops      = &blk_mq_sysfs_ops,
 410         .default_attrs  = default_ctx_attrs,
 411         .release        = blk_mq_sysfs_release,
 412 };
 413
 414 static struct kobj_type blk_mq_hw_ktype = {
 415         .sysfs_ops      = &blk_mq_hw_sysfs_ops,
 416         .default_attrs  = default_hw_ctx_attrs,
 417         .release        = blk_mq_sysfs_release,
 418 };
 419
 420 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
 421 {
 422         struct blk_mq_ctx *ctx;
 423         int i;
 424
 425         if (!hctx->nr_ctx)
 426                 return;
 427
 428         hctx_for_each_ctx(hctx, ctx, i)
 429                 kobject_del(&ctx->kobj);
 430
 431         kobject_del(&hctx->kobj);
 432 }
 433
 434 static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
 435 {
 436         struct request_queue *q = hctx->queue;
 437         struct blk_mq_ctx *ctx;
 438         int i, ret;
 439
 440         if (!hctx->nr_ctx)
 441                 return 0;
 442
 443         ret = kobject_add(&hctx->kobj, &q->mq_kobj, "%u", hctx->queue_num);
 444         if (ret)
 445                 return ret;
 446
 447         hctx_for_each_ctx(hctx, ctx, i) {
 448                 ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
 449                 if (ret)
 450                         break;
 451         }
 452
 453         return ret;
 454 }
 455
 456 static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
 457 {
 458         struct blk_mq_hw_ctx *hctx;
 459         struct blk_mq_ctx *ctx;
 460         int i, j;
 461
 462         queue_for_each_hw_ctx(q, hctx, i) {
 463                 blk_mq_unregister_hctx(hctx);
 464
 465                 hctx_for_each_ctx(hctx, ctx, j)
 466                         kobject_put(&ctx->kobj);
 467
 468                 kobject_put(&hctx->kobj);
 469         }
 470
 471         kobject_uevent(&q->mq_kobj, KOBJ_REMOVE);
 472         kobject_del(&q->mq_kobj);
 473         kobject_put(&q->mq_kobj);
 474
 475         kobject_put(&dev->kobj);
 476
 477         q->mq_sysfs_init_done = false;
 478 }
 479
 480 void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
 481 {
 482         blk_mq_disable_hotplug();
 483         __blk_mq_unregister_dev(dev, q);
 484         blk_mq_enable_hotplug();
 485 }
 486
 487 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
 488 {
 489         kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
 490 }
 491
 492 static void blk_mq_sysfs_init(struct request_queue *q)
 493 {
 494         struct blk_mq_ctx *ctx;
 495         int cpu;
 496
 497         kobject_init(&q->mq_kobj, &blk_mq_ktype);
 498
 499         for_each_possible_cpu(cpu) {
 500                 ctx = per_cpu_ptr(q->queue_ctx, cpu);
 501                 kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
 502         }
 503 }
 504
 505 int blk_mq_register_dev(struct device *dev, struct request_queue *q)
 506 {
 507         struct blk_mq_hw_ctx *hctx;
 508         int ret, i;
 509
 510         blk_mq_disable_hotplug();
 511
 512         blk_mq_sysfs_init(q);
 513
 514         ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
 515         if (ret < 0)
 516                 goto out;
 517
 518         kobject_uevent(&q->mq_kobj, KOBJ_ADD);
 519
 520         queue_for_each_hw_ctx(q, hctx, i) {
 521                 ret = blk_mq_register_hctx(hctx);
 522                 if (ret)
 523                         break;
 524         }
 525
 526         if (ret)
 527                 __blk_mq_unregister_dev(dev, q);
 528         else
 529                 q->mq_sysfs_init_done = true;
 530 out:
 531         blk_mq_enable_hotplug();
 532
 533         return ret;
 534 }
 535 EXPORT_SYMBOL_GPL(blk_mq_register_dev);
 536
 537 void blk_mq_sysfs_unregister(struct request_queue *q)
 538 {
 539         struct blk_mq_hw_ctx *hctx;
 540         int i;
 541
 542         if (!q->mq_sysfs_init_done)
 543                 return;
 544
 545         queue_for_each_hw_ctx(q, hctx, i)
 546                 blk_mq_unregister_hctx(hctx);
 547 }
 548
 549 int blk_mq_sysfs_register(struct request_queue *q)
 550 {
 551         struct blk_mq_hw_ctx *hctx;
 552         int i, ret = 0;
 553
 554         if (!q->mq_sysfs_init_done)
 555                 return ret;
 556
 557         queue_for_each_hw_ctx(q, hctx, i) {
 558                 ret = blk_mq_register_hctx(hctx);
 559                 if (ret)
 560                         break;
 561         }
 562
 563         return ret;
 564 }