2 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
3 * Initial release: Matias Bjorling <m@bjorling.me>
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; see the file COPYING. If not, write to
16 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
21 #include <linux/list.h>
22 #include <linux/types.h>
23 #include <linux/sem.h>
24 #include <linux/bitmap.h>
25 #include <linux/moduleparam.h>
26 #include <linux/miscdevice.h>
27 #include <linux/lightnvm.h>
28 #include <linux/sched/sysctl.h>
30 static LIST_HEAD(nvm_tgt_types
);
31 static DECLARE_RWSEM(nvm_tgtt_lock
);
32 static LIST_HEAD(nvm_devices
);
33 static DECLARE_RWSEM(nvm_lock
);
35 /* Map between virtual and physical channel and lun */
43 struct nvm_ch_map
*chnls
;
48 struct list_head list
;
50 sector_t end
; /* end is excluded */
53 static struct nvm_target
*nvm_find_target(struct nvm_dev
*dev
, const char *name
)
55 struct nvm_target
*tgt
;
57 list_for_each_entry(tgt
, &dev
->targets
, list
)
58 if (!strcmp(name
, tgt
->disk
->disk_name
))
64 static int nvm_reserve_luns(struct nvm_dev
*dev
, int lun_begin
, int lun_end
)
68 for (i
= lun_begin
; i
<= lun_end
; i
++) {
69 if (test_and_set_bit(i
, dev
->lun_map
)) {
70 pr_err("nvm: lun %d already allocated\n", i
);
77 while (--i
> lun_begin
)
78 clear_bit(i
, dev
->lun_map
);
83 static void nvm_release_luns_err(struct nvm_dev
*dev
, int lun_begin
,
88 for (i
= lun_begin
; i
<= lun_end
; i
++)
89 WARN_ON(!test_and_clear_bit(i
, dev
->lun_map
));
92 static void nvm_remove_tgt_dev(struct nvm_tgt_dev
*tgt_dev
)
94 struct nvm_dev
*dev
= tgt_dev
->parent
;
95 struct nvm_dev_map
*dev_map
= tgt_dev
->map
;
98 for (i
= 0; i
< dev_map
->nr_chnls
; i
++) {
99 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[i
];
100 int *lun_offs
= ch_map
->lun_offs
;
101 int ch
= i
+ ch_map
->ch_off
;
103 for (j
= 0; j
< ch_map
->nr_luns
; j
++) {
104 int lun
= j
+ lun_offs
[j
];
105 int lunid
= (ch
* dev
->geo
.luns_per_chnl
) + lun
;
107 WARN_ON(!test_and_clear_bit(lunid
, dev
->lun_map
));
110 kfree(ch_map
->lun_offs
);
113 kfree(dev_map
->chnls
);
116 kfree(tgt_dev
->luns
);
120 static struct nvm_tgt_dev
*nvm_create_tgt_dev(struct nvm_dev
*dev
,
121 int lun_begin
, int lun_end
)
123 struct nvm_tgt_dev
*tgt_dev
= NULL
;
124 struct nvm_dev_map
*dev_rmap
= dev
->rmap
;
125 struct nvm_dev_map
*dev_map
;
126 struct ppa_addr
*luns
;
127 int nr_luns
= lun_end
- lun_begin
+ 1;
128 int luns_left
= nr_luns
;
129 int nr_chnls
= nr_luns
/ dev
->geo
.luns_per_chnl
;
130 int nr_chnls_mod
= nr_luns
% dev
->geo
.luns_per_chnl
;
131 int bch
= lun_begin
/ dev
->geo
.luns_per_chnl
;
132 int blun
= lun_begin
% dev
->geo
.luns_per_chnl
;
134 int lun_balanced
= 1;
138 nr_chnls
= nr_luns
/ dev
->geo
.luns_per_chnl
;
139 nr_chnls
= (nr_chnls_mod
== 0) ? nr_chnls
: nr_chnls
+ 1;
141 dev_map
= kmalloc(sizeof(struct nvm_dev_map
), GFP_KERNEL
);
145 dev_map
->chnls
= kcalloc(nr_chnls
, sizeof(struct nvm_ch_map
),
150 luns
= kcalloc(nr_luns
, sizeof(struct ppa_addr
), GFP_KERNEL
);
154 prev_nr_luns
= (luns_left
> dev
->geo
.luns_per_chnl
) ?
155 dev
->geo
.luns_per_chnl
: luns_left
;
156 for (i
= 0; i
< nr_chnls
; i
++) {
157 struct nvm_ch_map
*ch_rmap
= &dev_rmap
->chnls
[i
+ bch
];
158 int *lun_roffs
= ch_rmap
->lun_offs
;
159 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[i
];
161 int luns_in_chnl
= (luns_left
> dev
->geo
.luns_per_chnl
) ?
162 dev
->geo
.luns_per_chnl
: luns_left
;
164 if (lun_balanced
&& prev_nr_luns
!= luns_in_chnl
)
167 ch_map
->ch_off
= ch_rmap
->ch_off
= bch
;
168 ch_map
->nr_luns
= luns_in_chnl
;
170 lun_offs
= kcalloc(luns_in_chnl
, sizeof(int), GFP_KERNEL
);
174 for (j
= 0; j
< luns_in_chnl
; j
++) {
176 luns
[lunid
].g
.ch
= i
;
177 luns
[lunid
++].g
.lun
= j
;
180 lun_roffs
[j
+ blun
] = blun
;
183 ch_map
->lun_offs
= lun_offs
;
185 /* when starting a new channel, lun offset is reset */
187 luns_left
-= luns_in_chnl
;
190 dev_map
->nr_chnls
= nr_chnls
;
192 tgt_dev
= kmalloc(sizeof(struct nvm_tgt_dev
), GFP_KERNEL
);
196 memcpy(&tgt_dev
->geo
, &dev
->geo
, sizeof(struct nvm_geo
));
197 /* Target device only owns a portion of the physical device */
198 tgt_dev
->geo
.nr_chnls
= nr_chnls
;
199 tgt_dev
->geo
.nr_luns
= nr_luns
;
200 tgt_dev
->geo
.luns_per_chnl
= (lun_balanced
) ? prev_nr_luns
: -1;
201 tgt_dev
->total_secs
= nr_luns
* tgt_dev
->geo
.sec_per_lun
;
203 tgt_dev
->map
= dev_map
;
204 tgt_dev
->luns
= luns
;
205 memcpy(&tgt_dev
->identity
, &dev
->identity
, sizeof(struct nvm_id
));
207 tgt_dev
->parent
= dev
;
212 kfree(dev_map
->chnls
[i
].lun_offs
);
215 kfree(dev_map
->chnls
);
222 static const struct block_device_operations nvm_fops
= {
223 .owner
= THIS_MODULE
,
226 static int nvm_create_tgt(struct nvm_dev
*dev
, struct nvm_ioctl_create
*create
)
228 struct nvm_ioctl_create_simple
*s
= &create
->conf
.s
;
229 struct request_queue
*tqueue
;
230 struct gendisk
*tdisk
;
231 struct nvm_tgt_type
*tt
;
232 struct nvm_target
*t
;
233 struct nvm_tgt_dev
*tgt_dev
;
236 tt
= nvm_find_target_type(create
->tgttype
, 1);
238 pr_err("nvm: target type %s not found\n", create
->tgttype
);
242 mutex_lock(&dev
->mlock
);
243 t
= nvm_find_target(dev
, create
->tgtname
);
245 pr_err("nvm: target name already exists.\n");
246 mutex_unlock(&dev
->mlock
);
249 mutex_unlock(&dev
->mlock
);
251 if (nvm_reserve_luns(dev
, s
->lun_begin
, s
->lun_end
))
254 t
= kmalloc(sizeof(struct nvm_target
), GFP_KERNEL
);
258 tgt_dev
= nvm_create_tgt_dev(dev
, s
->lun_begin
, s
->lun_end
);
260 pr_err("nvm: could not create target device\n");
264 tqueue
= blk_alloc_queue_node(GFP_KERNEL
, dev
->q
->node
);
267 blk_queue_make_request(tqueue
, tt
->make_rq
);
269 tdisk
= alloc_disk(0);
273 sprintf(tdisk
->disk_name
, "%s", create
->tgtname
);
274 tdisk
->flags
= GENHD_FL_EXT_DEVT
;
276 tdisk
->first_minor
= 0;
277 tdisk
->fops
= &nvm_fops
;
278 tdisk
->queue
= tqueue
;
280 targetdata
= tt
->init(tgt_dev
, tdisk
);
281 if (IS_ERR(targetdata
))
284 tdisk
->private_data
= targetdata
;
285 tqueue
->queuedata
= targetdata
;
287 blk_queue_max_hw_sectors(tqueue
, 8 * dev
->ops
->max_phys_sect
);
289 set_capacity(tdisk
, tt
->capacity(targetdata
));
296 mutex_lock(&dev
->mlock
);
297 list_add_tail(&t
->list
, &dev
->targets
);
298 mutex_unlock(&dev
->mlock
);
304 blk_cleanup_queue(tqueue
);
306 nvm_remove_tgt_dev(tgt_dev
);
310 nvm_release_luns_err(dev
, s
->lun_begin
, s
->lun_end
);
314 static void __nvm_remove_target(struct nvm_target
*t
)
316 struct nvm_tgt_type
*tt
= t
->type
;
317 struct gendisk
*tdisk
= t
->disk
;
318 struct request_queue
*q
= tdisk
->queue
;
321 blk_cleanup_queue(q
);
324 tt
->exit(tdisk
->private_data
);
326 nvm_remove_tgt_dev(t
->dev
);
334 * nvm_remove_tgt - Removes a target from the media manager
336 * @remove: ioctl structure with target name to remove.
343 static int nvm_remove_tgt(struct nvm_dev
*dev
, struct nvm_ioctl_remove
*remove
)
345 struct nvm_target
*t
;
347 mutex_lock(&dev
->mlock
);
348 t
= nvm_find_target(dev
, remove
->tgtname
);
350 mutex_unlock(&dev
->mlock
);
353 __nvm_remove_target(t
);
354 mutex_unlock(&dev
->mlock
);
359 static int nvm_register_map(struct nvm_dev
*dev
)
361 struct nvm_dev_map
*rmap
;
364 rmap
= kmalloc(sizeof(struct nvm_dev_map
), GFP_KERNEL
);
368 rmap
->chnls
= kcalloc(dev
->geo
.nr_chnls
, sizeof(struct nvm_ch_map
),
373 for (i
= 0; i
< dev
->geo
.nr_chnls
; i
++) {
374 struct nvm_ch_map
*ch_rmap
;
376 int luns_in_chnl
= dev
->geo
.luns_per_chnl
;
378 ch_rmap
= &rmap
->chnls
[i
];
380 ch_rmap
->ch_off
= -1;
381 ch_rmap
->nr_luns
= luns_in_chnl
;
383 lun_roffs
= kcalloc(luns_in_chnl
, sizeof(int), GFP_KERNEL
);
387 for (j
= 0; j
< luns_in_chnl
; j
++)
390 ch_rmap
->lun_offs
= lun_roffs
;
398 kfree(rmap
->chnls
[i
].lun_offs
);
405 static void nvm_map_to_dev(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*p
)
407 struct nvm_dev_map
*dev_map
= tgt_dev
->map
;
408 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[p
->g
.ch
];
409 int lun_off
= ch_map
->lun_offs
[p
->g
.lun
];
411 p
->g
.ch
+= ch_map
->ch_off
;
415 static void nvm_map_to_tgt(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*p
)
417 struct nvm_dev
*dev
= tgt_dev
->parent
;
418 struct nvm_dev_map
*dev_rmap
= dev
->rmap
;
419 struct nvm_ch_map
*ch_rmap
= &dev_rmap
->chnls
[p
->g
.ch
];
420 int lun_roff
= ch_rmap
->lun_offs
[p
->g
.lun
];
422 p
->g
.ch
-= ch_rmap
->ch_off
;
423 p
->g
.lun
-= lun_roff
;
426 static void nvm_ppa_tgt_to_dev(struct nvm_tgt_dev
*tgt_dev
,
427 struct ppa_addr
*ppa_list
, int nr_ppas
)
431 for (i
= 0; i
< nr_ppas
; i
++) {
432 nvm_map_to_dev(tgt_dev
, &ppa_list
[i
]);
433 ppa_list
[i
] = generic_to_dev_addr(tgt_dev
, ppa_list
[i
]);
437 static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev
*tgt_dev
,
438 struct ppa_addr
*ppa_list
, int nr_ppas
)
442 for (i
= 0; i
< nr_ppas
; i
++) {
443 ppa_list
[i
] = dev_to_generic_addr(tgt_dev
, ppa_list
[i
]);
444 nvm_map_to_tgt(tgt_dev
, &ppa_list
[i
]);
448 static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
)
450 if (rqd
->nr_ppas
== 1) {
451 nvm_ppa_tgt_to_dev(tgt_dev
, &rqd
->ppa_addr
, 1);
455 nvm_ppa_tgt_to_dev(tgt_dev
, rqd
->ppa_list
, rqd
->nr_ppas
);
458 static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
)
460 if (rqd
->nr_ppas
== 1) {
461 nvm_ppa_dev_to_tgt(tgt_dev
, &rqd
->ppa_addr
, 1);
465 nvm_ppa_dev_to_tgt(tgt_dev
, rqd
->ppa_list
, rqd
->nr_ppas
);
468 void nvm_part_to_tgt(struct nvm_dev
*dev
, sector_t
*entries
,
471 struct nvm_geo
*geo
= &dev
->geo
;
472 struct nvm_dev_map
*dev_rmap
= dev
->rmap
;
475 for (i
= 0; i
< len
; i
++) {
476 struct nvm_ch_map
*ch_rmap
;
478 struct ppa_addr gaddr
;
479 u64 pba
= le64_to_cpu(entries
[i
]);
486 gaddr
= linear_to_generic_addr(geo
, pba
);
487 ch_rmap
= &dev_rmap
->chnls
[gaddr
.g
.ch
];
488 lun_roffs
= ch_rmap
->lun_offs
;
490 off
= gaddr
.g
.ch
* geo
->luns_per_chnl
+ gaddr
.g
.lun
;
492 diff
= ((ch_rmap
->ch_off
* geo
->luns_per_chnl
) +
493 (lun_roffs
[gaddr
.g
.lun
])) * geo
->sec_per_lun
;
495 entries
[i
] -= cpu_to_le64(diff
);
498 EXPORT_SYMBOL(nvm_part_to_tgt
);
500 struct nvm_tgt_type
*nvm_find_target_type(const char *name
, int lock
)
502 struct nvm_tgt_type
*tmp
, *tt
= NULL
;
505 down_write(&nvm_tgtt_lock
);
507 list_for_each_entry(tmp
, &nvm_tgt_types
, list
)
508 if (!strcmp(name
, tmp
->name
)) {
514 up_write(&nvm_tgtt_lock
);
517 EXPORT_SYMBOL(nvm_find_target_type
);
519 int nvm_register_tgt_type(struct nvm_tgt_type
*tt
)
523 down_write(&nvm_tgtt_lock
);
524 if (nvm_find_target_type(tt
->name
, 0))
527 list_add(&tt
->list
, &nvm_tgt_types
);
528 up_write(&nvm_tgtt_lock
);
532 EXPORT_SYMBOL(nvm_register_tgt_type
);
534 void nvm_unregister_tgt_type(struct nvm_tgt_type
*tt
)
539 down_write(&nvm_lock
);
543 EXPORT_SYMBOL(nvm_unregister_tgt_type
);
545 void *nvm_dev_dma_alloc(struct nvm_dev
*dev
, gfp_t mem_flags
,
546 dma_addr_t
*dma_handler
)
548 return dev
->ops
->dev_dma_alloc(dev
, dev
->dma_pool
, mem_flags
,
551 EXPORT_SYMBOL(nvm_dev_dma_alloc
);
553 void nvm_dev_dma_free(struct nvm_dev
*dev
, void *addr
, dma_addr_t dma_handler
)
555 dev
->ops
->dev_dma_free(dev
->dma_pool
, addr
, dma_handler
);
557 EXPORT_SYMBOL(nvm_dev_dma_free
);
559 static struct nvm_dev
*nvm_find_nvm_dev(const char *name
)
563 list_for_each_entry(dev
, &nvm_devices
, devices
)
564 if (!strcmp(name
, dev
->name
))
570 int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*ppas
,
571 int nr_ppas
, int type
)
573 struct nvm_dev
*dev
= tgt_dev
->parent
;
577 if (nr_ppas
> dev
->ops
->max_phys_sect
) {
578 pr_err("nvm: unable to update all blocks atomically\n");
582 memset(&rqd
, 0, sizeof(struct nvm_rq
));
584 nvm_set_rqd_ppalist(dev
, &rqd
, ppas
, nr_ppas
, 1);
585 nvm_rq_tgt_to_dev(tgt_dev
, &rqd
);
587 ret
= dev
->ops
->set_bb_tbl(dev
, &rqd
.ppa_addr
, rqd
.nr_ppas
, type
);
588 nvm_free_rqd_ppalist(dev
, &rqd
);
590 pr_err("nvm: failed bb mark\n");
596 EXPORT_SYMBOL(nvm_set_tgt_bb_tbl
);
598 int nvm_max_phys_sects(struct nvm_tgt_dev
*tgt_dev
)
600 struct nvm_dev
*dev
= tgt_dev
->parent
;
602 return dev
->ops
->max_phys_sect
;
604 EXPORT_SYMBOL(nvm_max_phys_sects
);
606 int nvm_submit_io(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
)
608 struct nvm_dev
*dev
= tgt_dev
->parent
;
610 if (!dev
->ops
->submit_io
)
613 nvm_rq_tgt_to_dev(tgt_dev
, rqd
);
616 return dev
->ops
->submit_io(dev
, rqd
);
618 EXPORT_SYMBOL(nvm_submit_io
);
620 int nvm_erase_blk(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*ppas
, int flags
)
622 struct nvm_dev
*dev
= tgt_dev
->parent
;
626 if (!dev
->ops
->erase_block
)
629 nvm_map_to_dev(tgt_dev
, ppas
);
631 memset(&rqd
, 0, sizeof(struct nvm_rq
));
633 ret
= nvm_set_rqd_ppalist(dev
, &rqd
, ppas
, 1, 1);
637 nvm_rq_tgt_to_dev(tgt_dev
, &rqd
);
641 ret
= dev
->ops
->erase_block(dev
, &rqd
);
643 nvm_free_rqd_ppalist(dev
, &rqd
);
647 EXPORT_SYMBOL(nvm_erase_blk
);
649 int nvm_get_l2p_tbl(struct nvm_tgt_dev
*tgt_dev
, u64 slba
, u32 nlb
,
650 nvm_l2p_update_fn
*update_l2p
, void *priv
)
652 struct nvm_dev
*dev
= tgt_dev
->parent
;
654 if (!dev
->ops
->get_l2p_tbl
)
657 return dev
->ops
->get_l2p_tbl(dev
, slba
, nlb
, update_l2p
, priv
);
659 EXPORT_SYMBOL(nvm_get_l2p_tbl
);
661 int nvm_get_area(struct nvm_tgt_dev
*tgt_dev
, sector_t
*lba
, sector_t len
)
663 struct nvm_dev
*dev
= tgt_dev
->parent
;
664 struct nvm_geo
*geo
= &dev
->geo
;
665 struct nvm_area
*area
, *prev
, *next
;
667 sector_t max_sectors
= (geo
->sec_size
* dev
->total_secs
) >> 9;
669 if (len
> max_sectors
)
672 area
= kmalloc(sizeof(struct nvm_area
), GFP_KERNEL
);
678 spin_lock(&dev
->lock
);
679 list_for_each_entry(next
, &dev
->area_list
, list
) {
680 if (begin
+ len
> next
->begin
) {
688 if ((begin
+ len
) > max_sectors
) {
689 spin_unlock(&dev
->lock
);
694 area
->begin
= *lba
= begin
;
695 area
->end
= begin
+ len
;
697 if (prev
) /* insert into sorted order */
698 list_add(&area
->list
, &prev
->list
);
700 list_add(&area
->list
, &dev
->area_list
);
701 spin_unlock(&dev
->lock
);
705 EXPORT_SYMBOL(nvm_get_area
);
707 void nvm_put_area(struct nvm_tgt_dev
*tgt_dev
, sector_t begin
)
709 struct nvm_dev
*dev
= tgt_dev
->parent
;
710 struct nvm_area
*area
;
712 spin_lock(&dev
->lock
);
713 list_for_each_entry(area
, &dev
->area_list
, list
) {
714 if (area
->begin
!= begin
)
717 list_del(&area
->list
);
718 spin_unlock(&dev
->lock
);
722 spin_unlock(&dev
->lock
);
724 EXPORT_SYMBOL(nvm_put_area
);
726 int nvm_set_rqd_ppalist(struct nvm_dev
*dev
, struct nvm_rq
*rqd
,
727 const struct ppa_addr
*ppas
, int nr_ppas
, int vblk
)
729 struct nvm_geo
*geo
= &dev
->geo
;
730 int i
, plane_cnt
, pl_idx
;
733 if ((!vblk
|| geo
->plane_mode
== NVM_PLANE_SINGLE
) && nr_ppas
== 1) {
734 rqd
->nr_ppas
= nr_ppas
;
735 rqd
->ppa_addr
= ppas
[0];
740 rqd
->nr_ppas
= nr_ppas
;
741 rqd
->ppa_list
= nvm_dev_dma_alloc(dev
, GFP_KERNEL
, &rqd
->dma_ppa_list
);
742 if (!rqd
->ppa_list
) {
743 pr_err("nvm: failed to allocate dma memory\n");
748 for (i
= 0; i
< nr_ppas
; i
++)
749 rqd
->ppa_list
[i
] = ppas
[i
];
751 plane_cnt
= geo
->plane_mode
;
752 rqd
->nr_ppas
*= plane_cnt
;
754 for (i
= 0; i
< nr_ppas
; i
++) {
755 for (pl_idx
= 0; pl_idx
< plane_cnt
; pl_idx
++) {
758 rqd
->ppa_list
[(pl_idx
* nr_ppas
) + i
] = ppa
;
765 EXPORT_SYMBOL(nvm_set_rqd_ppalist
);
767 void nvm_free_rqd_ppalist(struct nvm_dev
*dev
, struct nvm_rq
*rqd
)
772 nvm_dev_dma_free(dev
, rqd
->ppa_list
, rqd
->dma_ppa_list
);
774 EXPORT_SYMBOL(nvm_free_rqd_ppalist
);
776 void nvm_end_io(struct nvm_rq
*rqd
)
778 struct nvm_tgt_dev
*tgt_dev
= rqd
->dev
;
780 /* Convert address space */
782 nvm_rq_dev_to_tgt(tgt_dev
, rqd
);
787 EXPORT_SYMBOL(nvm_end_io
);
790 * folds a bad block list from its plane representation to its virtual
791 * block representation. The fold is done in place and reduced size is
794 * If any of the planes status are bad or grown bad block, the virtual block
795 * is marked bad. If not bad, the first plane state acts as the block state.
797 int nvm_bb_tbl_fold(struct nvm_dev
*dev
, u8
*blks
, int nr_blks
)
799 struct nvm_geo
*geo
= &dev
->geo
;
800 int blk
, offset
, pl
, blktype
;
802 if (nr_blks
!= geo
->blks_per_lun
* geo
->plane_mode
)
805 for (blk
= 0; blk
< geo
->blks_per_lun
; blk
++) {
806 offset
= blk
* geo
->plane_mode
;
807 blktype
= blks
[offset
];
809 /* Bad blocks on any planes take precedence over other types */
810 for (pl
= 0; pl
< geo
->plane_mode
; pl
++) {
811 if (blks
[offset
+ pl
] &
812 (NVM_BLK_T_BAD
|NVM_BLK_T_GRWN_BAD
)) {
813 blktype
= blks
[offset
+ pl
];
821 return geo
->blks_per_lun
;
823 EXPORT_SYMBOL(nvm_bb_tbl_fold
);
825 int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr ppa
,
828 struct nvm_dev
*dev
= tgt_dev
->parent
;
830 nvm_ppa_tgt_to_dev(tgt_dev
, &ppa
, 1);
832 return dev
->ops
->get_bb_tbl(dev
, ppa
, blks
);
834 EXPORT_SYMBOL(nvm_get_tgt_bb_tbl
);
836 static int nvm_init_slc_tbl(struct nvm_dev
*dev
, struct nvm_id_group
*grp
)
838 struct nvm_geo
*geo
= &dev
->geo
;
841 dev
->lps_per_blk
= geo
->pgs_per_blk
;
842 dev
->lptbl
= kcalloc(dev
->lps_per_blk
, sizeof(int), GFP_KERNEL
);
846 /* Just a linear array */
847 for (i
= 0; i
< dev
->lps_per_blk
; i
++)
853 static int nvm_init_mlc_tbl(struct nvm_dev
*dev
, struct nvm_id_group
*grp
)
856 struct nvm_id_lp_mlc
*mlc
= &grp
->lptbl
.mlc
;
861 dev
->lps_per_blk
= mlc
->num_pairs
;
862 dev
->lptbl
= kcalloc(dev
->lps_per_blk
, sizeof(int), GFP_KERNEL
);
866 /* The lower page table encoding consists of a list of bytes, where each
867 * has a lower and an upper half. The first half byte maintains the
868 * increment value and every value after is an offset added to the
869 * previous incrementation value
871 dev
->lptbl
[0] = mlc
->pairs
[0] & 0xF;
872 for (i
= 1; i
< dev
->lps_per_blk
; i
++) {
873 p
= mlc
->pairs
[i
>> 1];
874 if (i
& 0x1) /* upper */
875 dev
->lptbl
[i
] = dev
->lptbl
[i
- 1] + ((p
& 0xF0) >> 4);
877 dev
->lptbl
[i
] = dev
->lptbl
[i
- 1] + (p
& 0xF);
883 static int nvm_core_init(struct nvm_dev
*dev
)
885 struct nvm_id
*id
= &dev
->identity
;
886 struct nvm_id_group
*grp
= &id
->grp
;
887 struct nvm_geo
*geo
= &dev
->geo
;
890 /* Whole device values */
891 geo
->nr_chnls
= grp
->num_ch
;
892 geo
->luns_per_chnl
= grp
->num_lun
;
894 /* Generic device values */
895 geo
->pgs_per_blk
= grp
->num_pg
;
896 geo
->blks_per_lun
= grp
->num_blk
;
897 geo
->nr_planes
= grp
->num_pln
;
898 geo
->fpg_size
= grp
->fpg_sz
;
899 geo
->pfpg_size
= grp
->fpg_sz
* grp
->num_pln
;
900 geo
->sec_size
= grp
->csecs
;
901 geo
->oob_size
= grp
->sos
;
902 geo
->sec_per_pg
= grp
->fpg_sz
/ grp
->csecs
;
903 geo
->mccap
= grp
->mccap
;
904 memcpy(&geo
->ppaf
, &id
->ppaf
, sizeof(struct nvm_addr_format
));
906 geo
->plane_mode
= NVM_PLANE_SINGLE
;
907 geo
->max_rq_size
= dev
->ops
->max_phys_sect
* geo
->sec_size
;
909 if (grp
->mpos
& 0x020202)
910 geo
->plane_mode
= NVM_PLANE_DOUBLE
;
911 if (grp
->mpos
& 0x040404)
912 geo
->plane_mode
= NVM_PLANE_QUAD
;
914 if (grp
->mtype
!= 0) {
915 pr_err("nvm: memory type not supported\n");
919 /* calculated values */
920 geo
->sec_per_pl
= geo
->sec_per_pg
* geo
->nr_planes
;
921 geo
->sec_per_blk
= geo
->sec_per_pl
* geo
->pgs_per_blk
;
922 geo
->sec_per_lun
= geo
->sec_per_blk
* geo
->blks_per_lun
;
923 geo
->nr_luns
= geo
->luns_per_chnl
* geo
->nr_chnls
;
925 dev
->total_secs
= geo
->nr_luns
* geo
->sec_per_lun
;
926 dev
->lun_map
= kcalloc(BITS_TO_LONGS(geo
->nr_luns
),
927 sizeof(unsigned long), GFP_KERNEL
);
931 switch (grp
->fmtype
) {
932 case NVM_ID_FMTYPE_SLC
:
933 if (nvm_init_slc_tbl(dev
, grp
)) {
938 case NVM_ID_FMTYPE_MLC
:
939 if (nvm_init_mlc_tbl(dev
, grp
)) {
945 pr_err("nvm: flash type not supported\n");
950 INIT_LIST_HEAD(&dev
->area_list
);
951 INIT_LIST_HEAD(&dev
->targets
);
952 mutex_init(&dev
->mlock
);
953 spin_lock_init(&dev
->lock
);
955 ret
= nvm_register_map(dev
);
959 blk_queue_logical_block_size(dev
->q
, geo
->sec_size
);
966 void nvm_free(struct nvm_dev
*dev
)
972 dev
->ops
->destroy_dma_pool(dev
->dma_pool
);
980 static int nvm_init(struct nvm_dev
*dev
)
982 struct nvm_geo
*geo
= &dev
->geo
;
985 if (dev
->ops
->identity(dev
, &dev
->identity
)) {
986 pr_err("nvm: device could not be identified\n");
990 pr_debug("nvm: ver:%x nvm_vendor:%x\n",
991 dev
->identity
.ver_id
, dev
->identity
.vmnt
);
993 if (dev
->identity
.ver_id
!= 1) {
994 pr_err("nvm: device not supported by kernel.");
998 ret
= nvm_core_init(dev
);
1000 pr_err("nvm: could not initialize core structures.\n");
1004 pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
1005 dev
->name
, geo
->sec_per_pg
, geo
->nr_planes
,
1006 geo
->pgs_per_blk
, geo
->blks_per_lun
,
1007 geo
->nr_luns
, geo
->nr_chnls
);
1010 pr_err("nvm: failed to initialize nvm\n");
1014 struct nvm_dev
*nvm_alloc_dev(int node
)
1016 return kzalloc_node(sizeof(struct nvm_dev
), GFP_KERNEL
, node
);
1018 EXPORT_SYMBOL(nvm_alloc_dev
);
1020 int nvm_register(struct nvm_dev
*dev
)
1024 if (!dev
->q
|| !dev
->ops
)
1027 if (dev
->ops
->max_phys_sect
> 256) {
1028 pr_info("nvm: max sectors supported is 256.\n");
1032 if (dev
->ops
->max_phys_sect
> 1) {
1033 dev
->dma_pool
= dev
->ops
->create_dma_pool(dev
, "ppalist");
1034 if (!dev
->dma_pool
) {
1035 pr_err("nvm: could not create dma pool\n");
1040 ret
= nvm_init(dev
);
1044 /* register device with a supported media manager */
1045 down_write(&nvm_lock
);
1046 list_add(&dev
->devices
, &nvm_devices
);
1047 up_write(&nvm_lock
);
1051 dev
->ops
->destroy_dma_pool(dev
->dma_pool
);
1054 EXPORT_SYMBOL(nvm_register
);
1056 void nvm_unregister(struct nvm_dev
*dev
)
1058 struct nvm_target
*t
, *tmp
;
1060 mutex_lock(&dev
->mlock
);
1061 list_for_each_entry_safe(t
, tmp
, &dev
->targets
, list
) {
1062 if (t
->dev
->parent
!= dev
)
1064 __nvm_remove_target(t
);
1066 mutex_unlock(&dev
->mlock
);
1068 down_write(&nvm_lock
);
1069 list_del(&dev
->devices
);
1070 up_write(&nvm_lock
);
1074 EXPORT_SYMBOL(nvm_unregister
);
1076 static int __nvm_configure_create(struct nvm_ioctl_create
*create
)
1078 struct nvm_dev
*dev
;
1079 struct nvm_ioctl_create_simple
*s
;
1081 down_write(&nvm_lock
);
1082 dev
= nvm_find_nvm_dev(create
->dev
);
1083 up_write(&nvm_lock
);
1086 pr_err("nvm: device not found\n");
1090 if (create
->conf
.type
!= NVM_CONFIG_TYPE_SIMPLE
) {
1091 pr_err("nvm: config type not valid\n");
1094 s
= &create
->conf
.s
;
1096 if (s
->lun_begin
> s
->lun_end
|| s
->lun_end
> dev
->geo
.nr_luns
) {
1097 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
1098 s
->lun_begin
, s
->lun_end
, dev
->geo
.nr_luns
);
1102 return nvm_create_tgt(dev
, create
);
1105 static long nvm_ioctl_info(struct file
*file
, void __user
*arg
)
1107 struct nvm_ioctl_info
*info
;
1108 struct nvm_tgt_type
*tt
;
1111 if (!capable(CAP_SYS_ADMIN
))
1114 info
= memdup_user(arg
, sizeof(struct nvm_ioctl_info
));
1118 info
->version
[0] = NVM_VERSION_MAJOR
;
1119 info
->version
[1] = NVM_VERSION_MINOR
;
1120 info
->version
[2] = NVM_VERSION_PATCH
;
1122 down_write(&nvm_lock
);
1123 list_for_each_entry(tt
, &nvm_tgt_types
, list
) {
1124 struct nvm_ioctl_info_tgt
*tgt
= &info
->tgts
[tgt_iter
];
1126 tgt
->version
[0] = tt
->version
[0];
1127 tgt
->version
[1] = tt
->version
[1];
1128 tgt
->version
[2] = tt
->version
[2];
1129 strncpy(tgt
->tgtname
, tt
->name
, NVM_TTYPE_NAME_MAX
);
1134 info
->tgtsize
= tgt_iter
;
1135 up_write(&nvm_lock
);
1137 if (copy_to_user(arg
, info
, sizeof(struct nvm_ioctl_info
))) {
1146 static long nvm_ioctl_get_devices(struct file
*file
, void __user
*arg
)
1148 struct nvm_ioctl_get_devices
*devices
;
1149 struct nvm_dev
*dev
;
1152 if (!capable(CAP_SYS_ADMIN
))
1155 devices
= kzalloc(sizeof(struct nvm_ioctl_get_devices
), GFP_KERNEL
);
1159 down_write(&nvm_lock
);
1160 list_for_each_entry(dev
, &nvm_devices
, devices
) {
1161 struct nvm_ioctl_device_info
*info
= &devices
->info
[i
];
1163 sprintf(info
->devname
, "%s", dev
->name
);
1165 /* kept for compatibility */
1166 info
->bmversion
[0] = 1;
1167 info
->bmversion
[1] = 0;
1168 info
->bmversion
[2] = 0;
1169 sprintf(info
->bmname
, "%s", "gennvm");
1173 pr_err("nvm: max 31 devices can be reported.\n");
1177 up_write(&nvm_lock
);
1179 devices
->nr_devices
= i
;
1181 if (copy_to_user(arg
, devices
,
1182 sizeof(struct nvm_ioctl_get_devices
))) {
1191 static long nvm_ioctl_dev_create(struct file
*file
, void __user
*arg
)
1193 struct nvm_ioctl_create create
;
1195 if (!capable(CAP_SYS_ADMIN
))
1198 if (copy_from_user(&create
, arg
, sizeof(struct nvm_ioctl_create
)))
1201 create
.dev
[DISK_NAME_LEN
- 1] = '\0';
1202 create
.tgttype
[NVM_TTYPE_NAME_MAX
- 1] = '\0';
1203 create
.tgtname
[DISK_NAME_LEN
- 1] = '\0';
1205 if (create
.flags
!= 0) {
1206 pr_err("nvm: no flags supported\n");
1210 return __nvm_configure_create(&create
);
1213 static long nvm_ioctl_dev_remove(struct file
*file
, void __user
*arg
)
1215 struct nvm_ioctl_remove remove
;
1216 struct nvm_dev
*dev
;
1219 if (!capable(CAP_SYS_ADMIN
))
1222 if (copy_from_user(&remove
, arg
, sizeof(struct nvm_ioctl_remove
)))
1225 remove
.tgtname
[DISK_NAME_LEN
- 1] = '\0';
1227 if (remove
.flags
!= 0) {
1228 pr_err("nvm: no flags supported\n");
1232 list_for_each_entry(dev
, &nvm_devices
, devices
) {
1233 ret
= nvm_remove_tgt(dev
, &remove
);
1241 /* kept for compatibility reasons */
1242 static long nvm_ioctl_dev_init(struct file
*file
, void __user
*arg
)
1244 struct nvm_ioctl_dev_init init
;
1246 if (!capable(CAP_SYS_ADMIN
))
1249 if (copy_from_user(&init
, arg
, sizeof(struct nvm_ioctl_dev_init
)))
1252 if (init
.flags
!= 0) {
1253 pr_err("nvm: no flags supported\n");
1260 /* Kept for compatibility reasons */
1261 static long nvm_ioctl_dev_factory(struct file
*file
, void __user
*arg
)
1263 struct nvm_ioctl_dev_factory fact
;
1265 if (!capable(CAP_SYS_ADMIN
))
1268 if (copy_from_user(&fact
, arg
, sizeof(struct nvm_ioctl_dev_factory
)))
1271 fact
.dev
[DISK_NAME_LEN
- 1] = '\0';
1273 if (fact
.flags
& ~(NVM_FACTORY_NR_BITS
- 1))
1279 static long nvm_ctl_ioctl(struct file
*file
, uint cmd
, unsigned long arg
)
1281 void __user
*argp
= (void __user
*)arg
;
1285 return nvm_ioctl_info(file
, argp
);
1286 case NVM_GET_DEVICES
:
1287 return nvm_ioctl_get_devices(file
, argp
);
1288 case NVM_DEV_CREATE
:
1289 return nvm_ioctl_dev_create(file
, argp
);
1290 case NVM_DEV_REMOVE
:
1291 return nvm_ioctl_dev_remove(file
, argp
);
1293 return nvm_ioctl_dev_init(file
, argp
);
1294 case NVM_DEV_FACTORY
:
1295 return nvm_ioctl_dev_factory(file
, argp
);
1300 static const struct file_operations _ctl_fops
= {
1301 .open
= nonseekable_open
,
1302 .unlocked_ioctl
= nvm_ctl_ioctl
,
1303 .owner
= THIS_MODULE
,
1304 .llseek
= noop_llseek
,
1307 static struct miscdevice _nvm_misc
= {
1308 .minor
= MISC_DYNAMIC_MINOR
,
1310 .nodename
= "lightnvm/control",
1313 builtin_misc_device(_nvm_misc
);