drivers/lightnvm/gennvm.c

   1 /*
   2  * Copyright (C) 2015 Matias Bjorling <m@bjorling.me>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License version
   6  * 2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; see the file COPYING.  If not, write to
  15  * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  16  * USA.
  17  *
  18  * Implementation of a general nvm manager for Open-Channel SSDs.
  19  */
  20
  21 #include "gennvm.h"
  22
  23 static struct nvm_target *gen_find_target(struct gen_dev *gn, const char *name)
  24 {
  25         struct nvm_target *tgt;
  26
  27         list_for_each_entry(tgt, &gn->targets, list)
  28                 if (!strcmp(name, tgt->disk->disk_name))
  29                         return tgt;
  30
  31         return NULL;
  32 }
  33
  34 static const struct block_device_operations gen_fops = {
  35         .owner          = THIS_MODULE,
  36 };
  37
  38 static int gen_reserve_luns(struct nvm_dev *dev, struct nvm_target *t,
  39                             int lun_begin, int lun_end)
  40 {
  41         struct gen_dev *gn = dev->mp;
  42         struct nvm_lun *lun;
  43         int i;
  44
  45         for (i = lun_begin; i <= lun_end; i++) {
  46                 if (test_and_set_bit(i, dev->lun_map)) {
  47                         pr_err("nvm: lun %d already allocated\n", i);
  48                         goto err;
  49                 }
  50
  51                 lun = &gn->luns[i];
  52                 list_add_tail(&lun->list, &t->lun_list);
  53         }
  54
  55         return 0;
  56
  57 err:
  58         while (--i > lun_begin) {
  59                 lun = &gn->luns[i];
  60                 clear_bit(i, dev->lun_map);
  61                 list_del(&lun->list);
  62         }
  63
  64         return -EBUSY;
  65 }
  66
  67 static void gen_release_luns(struct nvm_dev *dev, struct nvm_target *t)
  68 {
  69         struct nvm_lun *lun, *tmp;
  70
  71         list_for_each_entry_safe(lun, tmp, &t->lun_list, list) {
  72                 WARN_ON(!test_and_clear_bit(lun->id, dev->lun_map));
  73                 list_del(&lun->list);
  74         }
  75 }
  76
  77 static void gen_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev)
  78 {
  79         kfree(tgt_dev);
  80 }
  81
  82 static struct nvm_tgt_dev *gen_create_tgt_dev(struct nvm_dev *dev,
  83                                               int lun_begin, int lun_end)
  84 {
  85         struct nvm_tgt_dev *tgt_dev = NULL;
  86         int nr_luns = lun_end - lun_begin + 1;
  87
  88         tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL);
  89         if (!tgt_dev)
  90                 goto out;
  91
  92         memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo));
  93         tgt_dev->geo.nr_chnls = (nr_luns / (dev->geo.luns_per_chnl + 1)) + 1;
  94         tgt_dev->geo.nr_luns = nr_luns;
  95         tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun;
  96         tgt_dev->q = dev->q;
  97         tgt_dev->ops = dev->ops;
  98         tgt_dev->mt = dev->mt;
  99         memcpy(&tgt_dev->identity, &dev->identity, sizeof(struct nvm_id));
 100
 101         tgt_dev->parent = dev;
 102
 103 out:
 104         return tgt_dev;
 105 }
 106
 107 static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 108 {
 109         struct gen_dev *gn = dev->mp;
 110         struct nvm_ioctl_create_simple *s = &create->conf.s;
 111         struct request_queue *tqueue;
 112         struct gendisk *tdisk;
 113         struct nvm_tgt_type *tt;
 114         struct nvm_target *t;
 115         struct nvm_tgt_dev *tgt_dev;
 116         void *targetdata;
 117
 118         tt = nvm_find_target_type(create->tgttype, 1);
 119         if (!tt) {
 120                 pr_err("nvm: target type %s not found\n", create->tgttype);
 121                 return -EINVAL;
 122         }
 123
 124         mutex_lock(&gn->lock);
 125         t = gen_find_target(gn, create->tgtname);
 126         if (t) {
 127                 pr_err("nvm: target name already exists.\n");
 128                 mutex_unlock(&gn->lock);
 129                 return -EINVAL;
 130         }
 131         mutex_unlock(&gn->lock);
 132
 133         t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
 134         if (!t)
 135                 return -ENOMEM;
 136
 137         INIT_LIST_HEAD(&t->lun_list);
 138
 139         if (gen_reserve_luns(dev, t, s->lun_begin, s->lun_end))
 140                 goto err_t;
 141
 142         tgt_dev = gen_create_tgt_dev(dev, s->lun_begin, s->lun_end);
 143         if (!tgt_dev)
 144                 goto err_reserve;
 145
 146         tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
 147         if (!tqueue)
 148                 goto err_dev;
 149         blk_queue_make_request(tqueue, tt->make_rq);
 150
 151         tdisk = alloc_disk(0);
 152         if (!tdisk)
 153                 goto err_queue;
 154
 155         sprintf(tdisk->disk_name, "%s", create->tgtname);
 156         tdisk->flags = GENHD_FL_EXT_DEVT;
 157         tdisk->major = 0;
 158         tdisk->first_minor = 0;
 159         tdisk->fops = &gen_fops;
 160         tdisk->queue = tqueue;
 161
 162         targetdata = tt->init(tgt_dev, tdisk, &t->lun_list);
 163         if (IS_ERR(targetdata))
 164                 goto err_init;
 165
 166         tdisk->private_data = targetdata;
 167         tqueue->queuedata = targetdata;
 168
 169         blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
 170
 171         set_capacity(tdisk, tt->capacity(targetdata));
 172         add_disk(tdisk);
 173
 174         t->type = tt;
 175         t->disk = tdisk;
 176         t->dev = tgt_dev;
 177
 178         mutex_lock(&gn->lock);
 179         list_add_tail(&t->list, &gn->targets);
 180         mutex_unlock(&gn->lock);
 181
 182         return 0;
 183 err_init:
 184         put_disk(tdisk);
 185 err_queue:
 186         blk_cleanup_queue(tqueue);
 187 err_dev:
 188         kfree(tgt_dev);
 189 err_reserve:
 190         gen_release_luns(dev, t);
 191 err_t:
 192         kfree(t);
 193         return -ENOMEM;
 194 }
 195
 196 static void __gen_remove_target(struct nvm_target *t)
 197 {
 198         struct nvm_tgt_type *tt = t->type;
 199         struct gendisk *tdisk = t->disk;
 200         struct request_queue *q = tdisk->queue;
 201
 202         del_gendisk(tdisk);
 203         blk_cleanup_queue(q);
 204
 205         if (tt->exit)
 206                 tt->exit(tdisk->private_data);
 207
 208         gen_release_luns(t->dev->parent, t);
 209         gen_remove_tgt_dev(t->dev);
 210         put_disk(tdisk);
 211
 212         list_del(&t->list);
 213         kfree(t);
 214 }
 215
 216 /**
 217  * gen_remove_tgt - Removes a target from the media manager
 218  * @dev:        device
 219  * @remove:     ioctl structure with target name to remove.
 220  *
 221  * Returns:
 222  * 0: on success
 223  * 1: on not found
 224  * <0: on error
 225  */
 226 static int gen_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove)
 227 {
 228         struct gen_dev *gn = dev->mp;
 229         struct nvm_target *t;
 230
 231         if (!gn)
 232                 return 1;
 233
 234         mutex_lock(&gn->lock);
 235         t = gen_find_target(gn, remove->tgtname);
 236         if (!t) {
 237                 mutex_unlock(&gn->lock);
 238                 return 1;
 239         }
 240         __gen_remove_target(t);
 241         mutex_unlock(&gn->lock);
 242
 243         return 0;
 244 }
 245
 246 static int gen_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len)
 247 {
 248         struct nvm_geo *geo = &dev->geo;
 249         struct gen_dev *gn = dev->mp;
 250         struct gen_area *area, *prev, *next;
 251         sector_t begin = 0;
 252         sector_t max_sectors = (geo->sec_size * dev->total_secs) >> 9;
 253
 254         if (len > max_sectors)
 255                 return -EINVAL;
 256
 257         area = kmalloc(sizeof(struct gen_area), GFP_KERNEL);
 258         if (!area)
 259                 return -ENOMEM;
 260
 261         prev = NULL;
 262
 263         spin_lock(&dev->lock);
 264         list_for_each_entry(next, &gn->area_list, list) {
 265                 if (begin + len > next->begin) {
 266                         begin = next->end;
 267                         prev = next;
 268                         continue;
 269                 }
 270                 break;
 271         }
 272
 273         if ((begin + len) > max_sectors) {
 274                 spin_unlock(&dev->lock);
 275                 kfree(area);
 276                 return -EINVAL;
 277         }
 278
 279         area->begin = *lba = begin;
 280         area->end = begin + len;
 281
 282         if (prev) /* insert into sorted order */
 283                 list_add(&area->list, &prev->list);
 284         else
 285                 list_add(&area->list, &gn->area_list);
 286         spin_unlock(&dev->lock);
 287
 288         return 0;
 289 }
 290
 291 static void gen_put_area(struct nvm_dev *dev, sector_t begin)
 292 {
 293         struct gen_dev *gn = dev->mp;
 294         struct gen_area *area;
 295
 296         spin_lock(&dev->lock);
 297         list_for_each_entry(area, &gn->area_list, list) {
 298                 if (area->begin != begin)
 299                         continue;
 300
 301                 list_del(&area->list);
 302                 spin_unlock(&dev->lock);
 303                 kfree(area);
 304                 return;
 305         }
 306         spin_unlock(&dev->lock);
 307 }
 308
 309 static void gen_blocks_free(struct nvm_dev *dev)
 310 {
 311         struct gen_dev *gn = dev->mp;
 312         struct nvm_lun *lun;
 313         int i;
 314
 315         gen_for_each_lun(gn, lun, i) {
 316                 if (!lun->blocks)
 317                         break;
 318                 vfree(lun->blocks);
 319         }
 320 }
 321
 322 static void gen_luns_free(struct nvm_dev *dev)
 323 {
 324         struct gen_dev *gn = dev->mp;
 325
 326         kfree(gn->luns);
 327 }
 328
 329 static int gen_luns_init(struct nvm_dev *dev, struct gen_dev *gn)
 330 {
 331         struct nvm_geo *geo = &dev->geo;
 332         struct nvm_lun *lun;
 333         int i;
 334
 335         gn->luns = kcalloc(geo->nr_luns, sizeof(struct nvm_lun), GFP_KERNEL);
 336         if (!gn->luns)
 337                 return -ENOMEM;
 338
 339         gen_for_each_lun(gn, lun, i) {
 340                 INIT_LIST_HEAD(&lun->free_list);
 341                 INIT_LIST_HEAD(&lun->used_list);
 342                 INIT_LIST_HEAD(&lun->bb_list);
 343                 INIT_LIST_HEAD(&lun->list);
 344
 345                 spin_lock_init(&lun->lock);
 346
 347                 lun->id = i;
 348                 lun->lun_id = i % geo->luns_per_chnl;
 349                 lun->chnl_id = i / geo->luns_per_chnl;
 350                 lun->nr_free_blocks = geo->blks_per_lun;
 351         }
 352         return 0;
 353 }
 354
 355 static int gen_block_bb(struct gen_dev *gn, struct ppa_addr ppa,
 356                                                         u8 *blks, int nr_blks)
 357 {
 358         struct nvm_dev *dev = gn->dev;
 359         struct nvm_lun *lun;
 360         struct nvm_block *blk;
 361         int i;
 362
 363         nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
 364         if (nr_blks < 0)
 365                 return nr_blks;
 366
 367         lun = &gn->luns[(dev->geo.luns_per_chnl * ppa.g.ch) + ppa.g.lun];
 368
 369         for (i = 0; i < nr_blks; i++) {
 370                 if (blks[i] == NVM_BLK_T_FREE)
 371                         continue;
 372
 373                 blk = &lun->blocks[i];
 374                 list_move_tail(&blk->list, &lun->bb_list);
 375                 blk->state = NVM_BLK_ST_BAD;
 376                 lun->nr_free_blocks--;
 377         }
 378
 379         return 0;
 380 }
 381
 382 static int gen_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
 383 {
 384         struct nvm_dev *dev = private;
 385         struct nvm_geo *geo = &dev->geo;
 386         struct gen_dev *gn = dev->mp;
 387         u64 elba = slba + nlb;
 388         struct nvm_lun *lun;
 389         struct nvm_block *blk;
 390         u64 i;
 391         int lun_id;
 392
 393         if (unlikely(elba > dev->total_secs)) {
 394                 pr_err("gen: L2P data from device is out of bounds!\n");
 395                 return -EINVAL;
 396         }
 397
 398         for (i = 0; i < nlb; i++) {
 399                 u64 pba = le64_to_cpu(entries[i]);
 400
 401                 if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) {
 402                         pr_err("gen: L2P data entry is out of bounds!\n");
 403                         return -EINVAL;
 404                 }
 405
 406                 /* Address zero is a special one. The first page on a disk is
 407                  * protected. It often holds internal device boot
 408                  * information.
 409                  */
 410                 if (!pba)
 411                         continue;
 412
 413                 /* resolve block from physical address */
 414                 lun_id = div_u64(pba, geo->sec_per_lun);
 415                 lun = &gn->luns[lun_id];
 416
 417                 /* Calculate block offset into lun */
 418                 pba = pba - (geo->sec_per_lun * lun_id);
 419                 blk = &lun->blocks[div_u64(pba, geo->sec_per_blk)];
 420
 421                 if (!blk->state) {
 422                         /* at this point, we don't know anything about the
 423                          * block. It's up to the FTL on top to re-etablish the
 424                          * block state. The block is assumed to be open.
 425                          */
 426                         list_move_tail(&blk->list, &lun->used_list);
 427                         blk->state = NVM_BLK_ST_TGT;
 428                         lun->nr_free_blocks--;
 429                 }
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int gen_blocks_init(struct nvm_dev *dev, struct gen_dev *gn)
 436 {
 437         struct nvm_geo *geo = &dev->geo;
 438         struct nvm_lun *lun;
 439         struct nvm_block *block;
 440         sector_t lun_iter, blk_iter, cur_block_id = 0;
 441         int ret, nr_blks;
 442         u8 *blks;
 443
 444         nr_blks = geo->blks_per_lun * geo->plane_mode;
 445         blks = kmalloc(nr_blks, GFP_KERNEL);
 446         if (!blks)
 447                 return -ENOMEM;
 448
 449         gen_for_each_lun(gn, lun, lun_iter) {
 450                 lun->blocks = vzalloc(sizeof(struct nvm_block) *
 451                                                         geo->blks_per_lun);
 452                 if (!lun->blocks) {
 453                         kfree(blks);
 454                         return -ENOMEM;
 455                 }
 456
 457                 for (blk_iter = 0; blk_iter < geo->blks_per_lun; blk_iter++) {
 458                         block = &lun->blocks[blk_iter];
 459
 460                         INIT_LIST_HEAD(&block->list);
 461
 462                         block->lun = lun;
 463                         block->id = cur_block_id++;
 464
 465                         list_add_tail(&block->list, &lun->free_list);
 466                 }
 467
 468                 if (dev->ops->get_bb_tbl) {
 469                         struct ppa_addr ppa;
 470
 471                         ppa.ppa = 0;
 472                         ppa.g.ch = lun->chnl_id;
 473                         ppa.g.lun = lun->lun_id;
 474
 475                         ret = nvm_get_bb_tbl(dev, ppa, blks);
 476                         if (ret)
 477                                 pr_err("gen: could not get BB table\n");
 478
 479                         ret = gen_block_bb(gn, ppa, blks, nr_blks);
 480                         if (ret)
 481                                 pr_err("gen: BB table map failed\n");
 482                 }
 483         }
 484
 485         if ((dev->identity.dom & NVM_RSP_L2P) && dev->ops->get_l2p_tbl) {
 486                 ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_secs,
 487                                                         gen_block_map, dev);
 488                 if (ret) {
 489                         pr_err("gen: could not read L2P table.\n");
 490                         pr_warn("gen: default block initialization");
 491                 }
 492         }
 493
 494         kfree(blks);
 495         return 0;
 496 }
 497
 498 static void gen_free(struct nvm_dev *dev)
 499 {
 500         gen_blocks_free(dev);
 501         gen_luns_free(dev);
 502         kfree(dev->mp);
 503         dev->mp = NULL;
 504 }
 505
 506 static int gen_register(struct nvm_dev *dev)
 507 {
 508         struct gen_dev *gn;
 509         int ret;
 510
 511         if (!try_module_get(THIS_MODULE))
 512                 return -ENODEV;
 513
 514         gn = kzalloc(sizeof(struct gen_dev), GFP_KERNEL);
 515         if (!gn)
 516                 return -ENOMEM;
 517
 518         gn->dev = dev;
 519         gn->nr_luns = dev->geo.nr_luns;
 520         INIT_LIST_HEAD(&gn->area_list);
 521         mutex_init(&gn->lock);
 522         INIT_LIST_HEAD(&gn->targets);
 523         dev->mp = gn;
 524
 525         ret = gen_luns_init(dev, gn);
 526         if (ret) {
 527                 pr_err("gen: could not initialize luns\n");
 528                 goto err;
 529         }
 530
 531         ret = gen_blocks_init(dev, gn);
 532         if (ret) {
 533                 pr_err("gen: could not initialize blocks\n");
 534                 goto err;
 535         }
 536
 537         return 1;
 538 err:
 539         gen_free(dev);
 540         module_put(THIS_MODULE);
 541         return ret;
 542 }
 543
 544 static void gen_unregister(struct nvm_dev *dev)
 545 {
 546         struct gen_dev *gn = dev->mp;
 547         struct nvm_target *t, *tmp;
 548
 549         mutex_lock(&gn->lock);
 550         list_for_each_entry_safe(t, tmp, &gn->targets, list) {
 551                 if (t->dev->parent != dev)
 552                         continue;
 553                 __gen_remove_target(t);
 554         }
 555         mutex_unlock(&gn->lock);
 556
 557         gen_free(dev);
 558         module_put(THIS_MODULE);
 559 }
 560
 561 static void gen_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type)
 562 {
 563         struct nvm_geo *geo = &dev->geo;
 564         struct gen_dev *gn = dev->mp;
 565         struct nvm_lun *lun;
 566         struct nvm_block *blk;
 567
 568         pr_debug("gen: ppa  (ch: %u lun: %u blk: %u pg: %u) -> %u\n",
 569                         ppa.g.ch, ppa.g.lun, ppa.g.blk, ppa.g.pg, type);
 570
 571         if (unlikely(ppa.g.ch > geo->nr_chnls ||
 572                                         ppa.g.lun > geo->luns_per_chnl ||
 573                                         ppa.g.blk > geo->blks_per_lun)) {
 574                 WARN_ON_ONCE(1);
 575                 pr_err("gen: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u",
 576                                 ppa.g.ch, geo->nr_chnls,
 577                                 ppa.g.lun, geo->luns_per_chnl,
 578                                 ppa.g.blk, geo->blks_per_lun);
 579                 return;
 580         }
 581
 582         lun = &gn->luns[(geo->luns_per_chnl * ppa.g.ch) + ppa.g.lun];
 583         blk = &lun->blocks[ppa.g.blk];
 584
 585         /* will be moved to bb list on put_blk from target */
 586         blk->state = type;
 587 }
 588
 589 static void gen_end_io(struct nvm_rq *rqd)
 590 {
 591         struct nvm_tgt_instance *ins = rqd->ins;
 592
 593         ins->tt->end_io(rqd);
 594 }
 595
 596 static int gen_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 597 {
 598         if (!dev->ops->submit_io)
 599                 return -ENODEV;
 600
 601         /* Convert address space */
 602         nvm_generic_to_addr_mode(dev, rqd);
 603
 604         rqd->dev = dev;
 605         rqd->end_io = gen_end_io;
 606         return dev->ops->submit_io(dev, rqd);
 607 }
 608
 609 static int gen_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, int flags)
 610 {
 611         struct ppa_addr addr = block_to_ppa(dev, blk);
 612
 613         return nvm_erase_ppa(dev, &addr, 1, flags);
 614 }
 615
 616 static void gen_lun_info_print(struct nvm_dev *dev)
 617 {
 618         struct gen_dev *gn = dev->mp;
 619         struct nvm_lun *lun;
 620         unsigned int i;
 621
 622
 623         gen_for_each_lun(gn, lun, i) {
 624                 spin_lock(&lun->lock);
 625
 626                 pr_info("%s: lun%8u\t%u\n", dev->name, i,
 627                                                 lun->nr_free_blocks);
 628
 629                 spin_unlock(&lun->lock);
 630         }
 631 }
 632
 633 static struct nvmm_type gen = {
 634         .name                   = "gennvm",
 635         .version                = {0, 1, 0},
 636
 637         .register_mgr           = gen_register,
 638         .unregister_mgr         = gen_unregister,
 639
 640         .create_tgt             = gen_create_tgt,
 641         .remove_tgt             = gen_remove_tgt,
 642
 643         .submit_io              = gen_submit_io,
 644         .erase_blk              = gen_erase_blk,
 645
 646         .mark_blk               = gen_mark_blk,
 647
 648         .lun_info_print         = gen_lun_info_print,
 649
 650         .get_area               = gen_get_area,
 651         .put_area               = gen_put_area,
 652
 653 };
 654
 655 static int __init gen_module_init(void)
 656 {
 657         return nvm_register_mgr(&gen);
 658 }
 659
 660 static void gen_module_exit(void)
 661 {
 662         nvm_unregister_mgr(&gen);
 663 }
 664
 665 module_init(gen_module_init);
 666 module_exit(gen_module_exit);
 667 MODULE_LICENSE("GPL v2");
 668 MODULE_DESCRIPTION("General media manager for Open-Channel SSDs");