drivers/block/null_blk_zoned.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/vmalloc.h>
   3 #include <linux/bitmap.h>
   4 #include "null_blk.h"
   5
   6 #define CREATE_TRACE_POINTS
   7 #include "null_blk_trace.h"
   8
   9 /* zone_size in MBs to sectors. */
  10 #define ZONE_SIZE_SHIFT         11
  11
  12 static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect)
  13 {
  14         return sect >> ilog2(dev->zone_size_sects);
  15 }
  16
  17 int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q)
  18 {
  19         sector_t dev_size = (sector_t)dev->size * 1024 * 1024;
  20         sector_t sector = 0;
  21         unsigned int i;
  22
  23         if (!is_power_of_2(dev->zone_size)) {
  24                 pr_err("zone_size must be power-of-two\n");
  25                 return -EINVAL;
  26         }
  27         if (dev->zone_size > dev->size) {
  28                 pr_err("Zone size larger than device capacity\n");
  29                 return -EINVAL;
  30         }
  31
  32         if (!dev->zone_capacity)
  33                 dev->zone_capacity = dev->zone_size;
  34
  35         if (dev->zone_capacity > dev->zone_size) {
  36                 pr_err("null_blk: zone capacity (%lu MB) larger than zone size (%lu MB)\n",
  37                                         dev->zone_capacity, dev->zone_size);
  38                 return -EINVAL;
  39         }
  40
  41         dev->zone_size_sects = dev->zone_size << ZONE_SIZE_SHIFT;
  42         dev->nr_zones = dev_size >>
  43                                 (SECTOR_SHIFT + ilog2(dev->zone_size_sects));
  44         dev->zones = kvmalloc_array(dev->nr_zones, sizeof(struct blk_zone),
  45                         GFP_KERNEL | __GFP_ZERO);
  46         if (!dev->zones)
  47                 return -ENOMEM;
  48
  49         /*
  50          * With memory backing, the zone_lock spinlock needs to be temporarily
  51          * released to avoid scheduling in atomic context. To guarantee zone
  52          * information protection, use a bitmap to lock zones with
  53          * wait_on_bit_lock_io(). Sleeping on the lock is OK as memory backing
  54          * implies that the queue is marked with BLK_MQ_F_BLOCKING.
  55          */
  56         spin_lock_init(&dev->zone_lock);
  57         if (dev->memory_backed) {
  58                 dev->zone_locks = bitmap_zalloc(dev->nr_zones, GFP_KERNEL);
  59                 if (!dev->zone_locks) {
  60                         kvfree(dev->zones);
  61                         return -ENOMEM;
  62                 }
  63         }
  64
  65         if (dev->zone_nr_conv >= dev->nr_zones) {
  66                 dev->zone_nr_conv = dev->nr_zones - 1;
  67                 pr_info("changed the number of conventional zones to %u",
  68                         dev->zone_nr_conv);
  69         }
  70
  71         /* Max active zones has to be < nbr of seq zones in order to be enforceable */
  72         if (dev->zone_max_active >= dev->nr_zones - dev->zone_nr_conv) {
  73                 dev->zone_max_active = 0;
  74                 pr_info("zone_max_active limit disabled, limit >= zone count\n");
  75         }
  76
  77         /* Max open zones has to be <= max active zones */
  78         if (dev->zone_max_active && dev->zone_max_open > dev->zone_max_active) {
  79                 dev->zone_max_open = dev->zone_max_active;
  80                 pr_info("changed the maximum number of open zones to %u\n",
  81                         dev->nr_zones);
  82         } else if (dev->zone_max_open >= dev->nr_zones - dev->zone_nr_conv) {
  83                 dev->zone_max_open = 0;
  84                 pr_info("zone_max_open limit disabled, limit >= zone count\n");
  85         }
  86
  87         for (i = 0; i <  dev->zone_nr_conv; i++) {
  88                 struct blk_zone *zone = &dev->zones[i];
  89
  90                 zone->start = sector;
  91                 zone->len = dev->zone_size_sects;
  92                 zone->capacity = zone->len;
  93                 zone->wp = zone->start + zone->len;
  94                 zone->type = BLK_ZONE_TYPE_CONVENTIONAL;
  95                 zone->cond = BLK_ZONE_COND_NOT_WP;
  96
  97                 sector += dev->zone_size_sects;
  98         }
  99
 100         for (i = dev->zone_nr_conv; i < dev->nr_zones; i++) {
 101                 struct blk_zone *zone = &dev->zones[i];
 102
 103                 zone->start = zone->wp = sector;
 104                 zone->len = dev->zone_size_sects;
 105                 zone->capacity = dev->zone_capacity << ZONE_SIZE_SHIFT;
 106                 zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ;
 107                 zone->cond = BLK_ZONE_COND_EMPTY;
 108
 109                 sector += dev->zone_size_sects;
 110         }
 111
 112         q->limits.zoned = BLK_ZONED_HM;
 113         blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
 114         blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
 115
 116         return 0;
 117 }
 118
 119 int null_register_zoned_dev(struct nullb *nullb)
 120 {
 121         struct nullb_device *dev = nullb->dev;
 122         struct request_queue *q = nullb->q;
 123
 124         if (queue_is_mq(q)) {
 125                 int ret = blk_revalidate_disk_zones(nullb->disk, NULL);
 126
 127                 if (ret)
 128                         return ret;
 129         } else {
 130                 blk_queue_chunk_sectors(q, dev->zone_size_sects);
 131                 q->nr_zones = blkdev_nr_zones(nullb->disk);
 132         }
 133
 134         blk_queue_max_zone_append_sectors(q, dev->zone_size_sects);
 135         blk_queue_max_open_zones(q, dev->zone_max_open);
 136         blk_queue_max_active_zones(q, dev->zone_max_active);
 137
 138         return 0;
 139 }
 140
 141 void null_free_zoned_dev(struct nullb_device *dev)
 142 {
 143         bitmap_free(dev->zone_locks);
 144         kvfree(dev->zones);
 145 }
 146
 147 static inline void null_lock_zone(struct nullb_device *dev, unsigned int zno)
 148 {
 149         if (dev->memory_backed)
 150                 wait_on_bit_lock_io(dev->zone_locks, zno, TASK_UNINTERRUPTIBLE);
 151         spin_lock_irq(&dev->zone_lock);
 152 }
 153
 154 static inline void null_unlock_zone(struct nullb_device *dev, unsigned int zno)
 155 {
 156         spin_unlock_irq(&dev->zone_lock);
 157
 158         if (dev->memory_backed)
 159                 clear_and_wake_up_bit(zno, dev->zone_locks);
 160 }
 161
 162 int null_report_zones(struct gendisk *disk, sector_t sector,
 163                 unsigned int nr_zones, report_zones_cb cb, void *data)
 164 {
 165         struct nullb *nullb = disk->private_data;
 166         struct nullb_device *dev = nullb->dev;
 167         unsigned int first_zone, i, zno;
 168         struct blk_zone zone;
 169         int error;
 170
 171         first_zone = null_zone_no(dev, sector);
 172         if (first_zone >= dev->nr_zones)
 173                 return 0;
 174
 175         nr_zones = min(nr_zones, dev->nr_zones - first_zone);
 176         trace_nullb_report_zones(nullb, nr_zones);
 177
 178         zno = first_zone;
 179         for (i = 0; i < nr_zones; i++, zno++) {
 180                 /*
 181                  * Stacked DM target drivers will remap the zone information by
 182                  * modifying the zone information passed to the report callback.
 183                  * So use a local copy to avoid corruption of the device zone
 184                  * array.
 185                  */
 186                 null_lock_zone(dev, zno);
 187                 memcpy(&zone, &dev->zones[zno], sizeof(struct blk_zone));
 188                 null_unlock_zone(dev, zno);
 189
 190                 error = cb(&zone, i, data);
 191                 if (error)
 192                         return error;
 193         }
 194
 195         return nr_zones;
 196 }
 197
 198 /*
 199  * This is called in the case of memory backing from null_process_cmd()
 200  * with the target zone already locked.
 201  */
 202 size_t null_zone_valid_read_len(struct nullb *nullb,
 203                                 sector_t sector, unsigned int len)
 204 {
 205         struct nullb_device *dev = nullb->dev;
 206         struct blk_zone *zone = &dev->zones[null_zone_no(dev, sector)];
 207         unsigned int nr_sectors = len >> SECTOR_SHIFT;
 208
 209         /* Read must be below the write pointer position */
 210         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL ||
 211             sector + nr_sectors <= zone->wp)
 212                 return len;
 213
 214         if (sector > zone->wp)
 215                 return 0;
 216
 217         return (zone->wp - sector) << SECTOR_SHIFT;
 218 }
 219
 220 static blk_status_t null_close_zone(struct nullb_device *dev, struct blk_zone *zone)
 221 {
 222         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 223                 return BLK_STS_IOERR;
 224
 225         switch (zone->cond) {
 226         case BLK_ZONE_COND_CLOSED:
 227                 /* close operation on closed is not an error */
 228                 return BLK_STS_OK;
 229         case BLK_ZONE_COND_IMP_OPEN:
 230                 dev->nr_zones_imp_open--;
 231                 break;
 232         case BLK_ZONE_COND_EXP_OPEN:
 233                 dev->nr_zones_exp_open--;
 234                 break;
 235         case BLK_ZONE_COND_EMPTY:
 236         case BLK_ZONE_COND_FULL:
 237         default:
 238                 return BLK_STS_IOERR;
 239         }
 240
 241         if (zone->wp == zone->start) {
 242                 zone->cond = BLK_ZONE_COND_EMPTY;
 243         } else {
 244                 zone->cond = BLK_ZONE_COND_CLOSED;
 245                 dev->nr_zones_closed++;
 246         }
 247
 248         return BLK_STS_OK;
 249 }
 250
 251 static void null_close_first_imp_zone(struct nullb_device *dev)
 252 {
 253         unsigned int i;
 254
 255         for (i = dev->zone_nr_conv; i < dev->nr_zones; i++) {
 256                 if (dev->zones[i].cond == BLK_ZONE_COND_IMP_OPEN) {
 257                         null_close_zone(dev, &dev->zones[i]);
 258                         return;
 259                 }
 260         }
 261 }
 262
 263 static blk_status_t null_check_active(struct nullb_device *dev)
 264 {
 265         if (!dev->zone_max_active)
 266                 return BLK_STS_OK;
 267
 268         if (dev->nr_zones_exp_open + dev->nr_zones_imp_open +
 269                         dev->nr_zones_closed < dev->zone_max_active)
 270                 return BLK_STS_OK;
 271
 272         return BLK_STS_ZONE_ACTIVE_RESOURCE;
 273 }
 274
 275 static blk_status_t null_check_open(struct nullb_device *dev)
 276 {
 277         if (!dev->zone_max_open)
 278                 return BLK_STS_OK;
 279
 280         if (dev->nr_zones_exp_open + dev->nr_zones_imp_open < dev->zone_max_open)
 281                 return BLK_STS_OK;
 282
 283         if (dev->nr_zones_imp_open) {
 284                 if (null_check_active(dev) == BLK_STS_OK) {
 285                         null_close_first_imp_zone(dev);
 286                         return BLK_STS_OK;
 287                 }
 288         }
 289
 290         return BLK_STS_ZONE_OPEN_RESOURCE;
 291 }
 292
 293 /*
 294  * This function matches the manage open zone resources function in the ZBC standard,
 295  * with the addition of max active zones support (added in the ZNS standard).
 296  *
 297  * The function determines if a zone can transition to implicit open or explicit open,
 298  * while maintaining the max open zone (and max active zone) limit(s). It may close an
 299  * implicit open zone in order to make additional zone resources available.
 300  *
 301  * ZBC states that an implicit open zone shall be closed only if there is not
 302  * room within the open limit. However, with the addition of an active limit,
 303  * it is not certain that closing an implicit open zone will allow a new zone
 304  * to be opened, since we might already be at the active limit capacity.
 305  */
 306 static blk_status_t null_check_zone_resources(struct nullb_device *dev, struct blk_zone *zone)
 307 {
 308         blk_status_t ret;
 309
 310         switch (zone->cond) {
 311         case BLK_ZONE_COND_EMPTY:
 312                 ret = null_check_active(dev);
 313                 if (ret != BLK_STS_OK)
 314                         return ret;
 315                 fallthrough;
 316         case BLK_ZONE_COND_CLOSED:
 317                 return null_check_open(dev);
 318         default:
 319                 /* Should never be called for other states */
 320                 WARN_ON(1);
 321                 return BLK_STS_IOERR;
 322         }
 323 }
 324
 325 static blk_status_t null_zone_write(struct nullb_cmd *cmd, sector_t sector,
 326                                     unsigned int nr_sectors, bool append)
 327 {
 328         struct nullb_device *dev = cmd->nq->dev;
 329         unsigned int zno = null_zone_no(dev, sector);
 330         struct blk_zone *zone = &dev->zones[zno];
 331         blk_status_t ret;
 332
 333         trace_nullb_zone_op(cmd, zno, zone->cond);
 334
 335         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 336                 return null_process_cmd(cmd, REQ_OP_WRITE, sector, nr_sectors);
 337
 338         null_lock_zone(dev, zno);
 339
 340         switch (zone->cond) {
 341         case BLK_ZONE_COND_FULL:
 342                 /* Cannot write to a full zone */
 343                 ret = BLK_STS_IOERR;
 344                 goto unlock;
 345         case BLK_ZONE_COND_EMPTY:
 346         case BLK_ZONE_COND_CLOSED:
 347                 ret = null_check_zone_resources(dev, zone);
 348                 if (ret != BLK_STS_OK)
 349                         goto unlock;
 350                 break;
 351         case BLK_ZONE_COND_IMP_OPEN:
 352         case BLK_ZONE_COND_EXP_OPEN:
 353                 break;
 354         default:
 355                 /* Invalid zone condition */
 356                 ret = BLK_STS_IOERR;
 357                 goto unlock;
 358         }
 359
 360         /*
 361          * Regular writes must be at the write pointer position.
 362          * Zone append writes are automatically issued at the write
 363          * pointer and the position returned using the request or BIO
 364          * sector.
 365          */
 366         if (append) {
 367                 sector = zone->wp;
 368                 if (cmd->bio)
 369                         cmd->bio->bi_iter.bi_sector = sector;
 370                 else
 371                         cmd->rq->__sector = sector;
 372         } else if (sector != zone->wp) {
 373                 ret = BLK_STS_IOERR;
 374                 goto unlock;
 375         }
 376
 377         if (zone->wp + nr_sectors > zone->start + zone->capacity) {
 378                 ret = BLK_STS_IOERR;
 379                 goto unlock;
 380         }
 381
 382         if (zone->cond == BLK_ZONE_COND_CLOSED) {
 383                 dev->nr_zones_closed--;
 384                 dev->nr_zones_imp_open++;
 385         } else if (zone->cond == BLK_ZONE_COND_EMPTY) {
 386                 dev->nr_zones_imp_open++;
 387         }
 388         if (zone->cond != BLK_ZONE_COND_EXP_OPEN)
 389                 zone->cond = BLK_ZONE_COND_IMP_OPEN;
 390
 391         /*
 392          * Memory backing allocation may sleep: release the zone_lock spinlock
 393          * to avoid scheduling in atomic context. Zone operation atomicity is
 394          * still guaranteed through the zone_locks bitmap.
 395          */
 396         if (dev->memory_backed)
 397                 spin_unlock_irq(&dev->zone_lock);
 398         ret = null_process_cmd(cmd, REQ_OP_WRITE, sector, nr_sectors);
 399         if (dev->memory_backed)
 400                 spin_lock_irq(&dev->zone_lock);
 401
 402         if (ret != BLK_STS_OK)
 403                 goto unlock;
 404
 405         zone->wp += nr_sectors;
 406         if (zone->wp == zone->start + zone->capacity) {
 407                 if (zone->cond == BLK_ZONE_COND_EXP_OPEN)
 408                         dev->nr_zones_exp_open--;
 409                 else if (zone->cond == BLK_ZONE_COND_IMP_OPEN)
 410                         dev->nr_zones_imp_open--;
 411                 zone->cond = BLK_ZONE_COND_FULL;
 412         }
 413         ret = BLK_STS_OK;
 414
 415 unlock:
 416         null_unlock_zone(dev, zno);
 417
 418         return ret;
 419 }
 420
 421 static blk_status_t null_open_zone(struct nullb_device *dev, struct blk_zone *zone)
 422 {
 423         blk_status_t ret;
 424
 425         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 426                 return BLK_STS_IOERR;
 427
 428         switch (zone->cond) {
 429         case BLK_ZONE_COND_EXP_OPEN:
 430                 /* open operation on exp open is not an error */
 431                 return BLK_STS_OK;
 432         case BLK_ZONE_COND_EMPTY:
 433                 ret = null_check_zone_resources(dev, zone);
 434                 if (ret != BLK_STS_OK)
 435                         return ret;
 436                 break;
 437         case BLK_ZONE_COND_IMP_OPEN:
 438                 dev->nr_zones_imp_open--;
 439                 break;
 440         case BLK_ZONE_COND_CLOSED:
 441                 ret = null_check_zone_resources(dev, zone);
 442                 if (ret != BLK_STS_OK)
 443                         return ret;
 444                 dev->nr_zones_closed--;
 445                 break;
 446         case BLK_ZONE_COND_FULL:
 447         default:
 448                 return BLK_STS_IOERR;
 449         }
 450
 451         zone->cond = BLK_ZONE_COND_EXP_OPEN;
 452         dev->nr_zones_exp_open++;
 453
 454         return BLK_STS_OK;
 455 }
 456
 457 static blk_status_t null_finish_zone(struct nullb_device *dev, struct blk_zone *zone)
 458 {
 459         blk_status_t ret;
 460
 461         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 462                 return BLK_STS_IOERR;
 463
 464         switch (zone->cond) {
 465         case BLK_ZONE_COND_FULL:
 466                 /* finish operation on full is not an error */
 467                 return BLK_STS_OK;
 468         case BLK_ZONE_COND_EMPTY:
 469                 ret = null_check_zone_resources(dev, zone);
 470                 if (ret != BLK_STS_OK)
 471                         return ret;
 472                 break;
 473         case BLK_ZONE_COND_IMP_OPEN:
 474                 dev->nr_zones_imp_open--;
 475                 break;
 476         case BLK_ZONE_COND_EXP_OPEN:
 477                 dev->nr_zones_exp_open--;
 478                 break;
 479         case BLK_ZONE_COND_CLOSED:
 480                 ret = null_check_zone_resources(dev, zone);
 481                 if (ret != BLK_STS_OK)
 482                         return ret;
 483                 dev->nr_zones_closed--;
 484                 break;
 485         default:
 486                 return BLK_STS_IOERR;
 487         }
 488
 489         zone->cond = BLK_ZONE_COND_FULL;
 490         zone->wp = zone->start + zone->len;
 491
 492         return BLK_STS_OK;
 493 }
 494
 495 static blk_status_t null_reset_zone(struct nullb_device *dev, struct blk_zone *zone)
 496 {
 497         if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 498                 return BLK_STS_IOERR;
 499
 500         switch (zone->cond) {
 501         case BLK_ZONE_COND_EMPTY:
 502                 /* reset operation on empty is not an error */
 503                 return BLK_STS_OK;
 504         case BLK_ZONE_COND_IMP_OPEN:
 505                 dev->nr_zones_imp_open--;
 506                 break;
 507         case BLK_ZONE_COND_EXP_OPEN:
 508                 dev->nr_zones_exp_open--;
 509                 break;
 510         case BLK_ZONE_COND_CLOSED:
 511                 dev->nr_zones_closed--;
 512                 break;
 513         case BLK_ZONE_COND_FULL:
 514                 break;
 515         default:
 516                 return BLK_STS_IOERR;
 517         }
 518
 519         zone->cond = BLK_ZONE_COND_EMPTY;
 520         zone->wp = zone->start;
 521
 522         return BLK_STS_OK;
 523 }
 524
 525 static blk_status_t null_zone_mgmt(struct nullb_cmd *cmd, enum req_opf op,
 526                                    sector_t sector)
 527 {
 528         struct nullb_device *dev = cmd->nq->dev;
 529         unsigned int zone_no;
 530         struct blk_zone *zone;
 531         blk_status_t ret;
 532         size_t i;
 533
 534         if (op == REQ_OP_ZONE_RESET_ALL) {
 535                 for (i = dev->zone_nr_conv; i < dev->nr_zones; i++) {
 536                         null_lock_zone(dev, i);
 537                         zone = &dev->zones[i];
 538                         if (zone->cond != BLK_ZONE_COND_EMPTY) {
 539                                 null_reset_zone(dev, zone);
 540                                 trace_nullb_zone_op(cmd, i, zone->cond);
 541                         }
 542                         null_unlock_zone(dev, i);
 543                 }
 544                 return BLK_STS_OK;
 545         }
 546
 547         zone_no = null_zone_no(dev, sector);
 548         zone = &dev->zones[zone_no];
 549
 550         null_lock_zone(dev, zone_no);
 551
 552         switch (op) {
 553         case REQ_OP_ZONE_RESET:
 554                 ret = null_reset_zone(dev, zone);
 555                 break;
 556         case REQ_OP_ZONE_OPEN:
 557                 ret = null_open_zone(dev, zone);
 558                 break;
 559         case REQ_OP_ZONE_CLOSE:
 560                 ret = null_close_zone(dev, zone);
 561                 break;
 562         case REQ_OP_ZONE_FINISH:
 563                 ret = null_finish_zone(dev, zone);
 564                 break;
 565         default:
 566                 ret = BLK_STS_NOTSUPP;
 567                 break;
 568         }
 569
 570         if (ret == BLK_STS_OK)
 571                 trace_nullb_zone_op(cmd, zone_no, zone->cond);
 572
 573         null_unlock_zone(dev, zone_no);
 574
 575         return ret;
 576 }
 577
 578 blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_opf op,
 579                                     sector_t sector, sector_t nr_sectors)
 580 {
 581         struct nullb_device *dev = cmd->nq->dev;
 582         unsigned int zno = null_zone_no(dev, sector);
 583         blk_status_t sts;
 584
 585         switch (op) {
 586         case REQ_OP_WRITE:
 587                 sts = null_zone_write(cmd, sector, nr_sectors, false);
 588                 break;
 589         case REQ_OP_ZONE_APPEND:
 590                 sts = null_zone_write(cmd, sector, nr_sectors, true);
 591                 break;
 592         case REQ_OP_ZONE_RESET:
 593         case REQ_OP_ZONE_RESET_ALL:
 594         case REQ_OP_ZONE_OPEN:
 595         case REQ_OP_ZONE_CLOSE:
 596         case REQ_OP_ZONE_FINISH:
 597                 sts = null_zone_mgmt(cmd, op, sector);
 598                 break;
 599         default:
 600                 null_lock_zone(dev, zno);
 601                 sts = null_process_cmd(cmd, op, sector, nr_sectors);
 602                 null_unlock_zone(dev, zno);
 603         }
 604
 605         return sts;
 606 }