1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
4 * Initial release: Matias Bjorling <m@bjorling.me>
7 #define pr_fmt(fmt) "nvm: " fmt
9 #include <linux/list.h>
10 #include <linux/types.h>
11 #include <linux/sem.h>
12 #include <linux/bitmap.h>
13 #include <linux/module.h>
14 #include <linux/moduleparam.h>
15 #include <linux/miscdevice.h>
16 #include <linux/lightnvm.h>
17 #include <linux/sched/sysctl.h>
19 static LIST_HEAD(nvm_tgt_types
);
20 static DECLARE_RWSEM(nvm_tgtt_lock
);
21 static LIST_HEAD(nvm_devices
);
22 static DECLARE_RWSEM(nvm_lock
);
24 /* Map between virtual and physical channel and lun */
32 struct nvm_ch_map
*chnls
;
36 static void nvm_free(struct kref
*ref
);
38 static struct nvm_target
*nvm_find_target(struct nvm_dev
*dev
, const char *name
)
40 struct nvm_target
*tgt
;
42 list_for_each_entry(tgt
, &dev
->targets
, list
)
43 if (!strcmp(name
, tgt
->disk
->disk_name
))
49 static bool nvm_target_exists(const char *name
)
52 struct nvm_target
*tgt
;
55 down_write(&nvm_lock
);
56 list_for_each_entry(dev
, &nvm_devices
, devices
) {
57 mutex_lock(&dev
->mlock
);
58 list_for_each_entry(tgt
, &dev
->targets
, list
) {
59 if (!strcmp(name
, tgt
->disk
->disk_name
)) {
61 mutex_unlock(&dev
->mlock
);
65 mutex_unlock(&dev
->mlock
);
73 static int nvm_reserve_luns(struct nvm_dev
*dev
, int lun_begin
, int lun_end
)
77 for (i
= lun_begin
; i
<= lun_end
; i
++) {
78 if (test_and_set_bit(i
, dev
->lun_map
)) {
79 pr_err("lun %d already allocated\n", i
);
86 while (--i
>= lun_begin
)
87 clear_bit(i
, dev
->lun_map
);
92 static void nvm_release_luns_err(struct nvm_dev
*dev
, int lun_begin
,
97 for (i
= lun_begin
; i
<= lun_end
; i
++)
98 WARN_ON(!test_and_clear_bit(i
, dev
->lun_map
));
101 static void nvm_remove_tgt_dev(struct nvm_tgt_dev
*tgt_dev
, int clear
)
103 struct nvm_dev
*dev
= tgt_dev
->parent
;
104 struct nvm_dev_map
*dev_map
= tgt_dev
->map
;
107 for (i
= 0; i
< dev_map
->num_ch
; i
++) {
108 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[i
];
109 int *lun_offs
= ch_map
->lun_offs
;
110 int ch
= i
+ ch_map
->ch_off
;
113 for (j
= 0; j
< ch_map
->num_lun
; j
++) {
114 int lun
= j
+ lun_offs
[j
];
115 int lunid
= (ch
* dev
->geo
.num_lun
) + lun
;
117 WARN_ON(!test_and_clear_bit(lunid
,
122 kfree(ch_map
->lun_offs
);
125 kfree(dev_map
->chnls
);
128 kfree(tgt_dev
->luns
);
132 static struct nvm_tgt_dev
*nvm_create_tgt_dev(struct nvm_dev
*dev
,
133 u16 lun_begin
, u16 lun_end
,
136 struct nvm_tgt_dev
*tgt_dev
= NULL
;
137 struct nvm_dev_map
*dev_rmap
= dev
->rmap
;
138 struct nvm_dev_map
*dev_map
;
139 struct ppa_addr
*luns
;
140 int num_lun
= lun_end
- lun_begin
+ 1;
141 int luns_left
= num_lun
;
142 int num_ch
= num_lun
/ dev
->geo
.num_lun
;
143 int num_ch_mod
= num_lun
% dev
->geo
.num_lun
;
144 int bch
= lun_begin
/ dev
->geo
.num_lun
;
145 int blun
= lun_begin
% dev
->geo
.num_lun
;
147 int lun_balanced
= 1;
148 int sec_per_lun
, prev_num_lun
;
151 num_ch
= (num_ch_mod
== 0) ? num_ch
: num_ch
+ 1;
153 dev_map
= kmalloc(sizeof(struct nvm_dev_map
), GFP_KERNEL
);
157 dev_map
->chnls
= kcalloc(num_ch
, sizeof(struct nvm_ch_map
), GFP_KERNEL
);
161 luns
= kcalloc(num_lun
, sizeof(struct ppa_addr
), GFP_KERNEL
);
165 prev_num_lun
= (luns_left
> dev
->geo
.num_lun
) ?
166 dev
->geo
.num_lun
: luns_left
;
167 for (i
= 0; i
< num_ch
; i
++) {
168 struct nvm_ch_map
*ch_rmap
= &dev_rmap
->chnls
[i
+ bch
];
169 int *lun_roffs
= ch_rmap
->lun_offs
;
170 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[i
];
172 int luns_in_chnl
= (luns_left
> dev
->geo
.num_lun
) ?
173 dev
->geo
.num_lun
: luns_left
;
175 if (lun_balanced
&& prev_num_lun
!= luns_in_chnl
)
178 ch_map
->ch_off
= ch_rmap
->ch_off
= bch
;
179 ch_map
->num_lun
= luns_in_chnl
;
181 lun_offs
= kcalloc(luns_in_chnl
, sizeof(int), GFP_KERNEL
);
185 for (j
= 0; j
< luns_in_chnl
; j
++) {
187 luns
[lunid
].a
.ch
= i
;
188 luns
[lunid
++].a
.lun
= j
;
191 lun_roffs
[j
+ blun
] = blun
;
194 ch_map
->lun_offs
= lun_offs
;
196 /* when starting a new channel, lun offset is reset */
198 luns_left
-= luns_in_chnl
;
201 dev_map
->num_ch
= num_ch
;
203 tgt_dev
= kmalloc(sizeof(struct nvm_tgt_dev
), GFP_KERNEL
);
207 /* Inherit device geometry from parent */
208 memcpy(&tgt_dev
->geo
, &dev
->geo
, sizeof(struct nvm_geo
));
210 /* Target device only owns a portion of the physical device */
211 tgt_dev
->geo
.num_ch
= num_ch
;
212 tgt_dev
->geo
.num_lun
= (lun_balanced
) ? prev_num_lun
: -1;
213 tgt_dev
->geo
.all_luns
= num_lun
;
214 tgt_dev
->geo
.all_chunks
= num_lun
* dev
->geo
.num_chk
;
216 tgt_dev
->geo
.op
= op
;
218 sec_per_lun
= dev
->geo
.clba
* dev
->geo
.num_chk
;
219 tgt_dev
->geo
.total_secs
= num_lun
* sec_per_lun
;
222 tgt_dev
->map
= dev_map
;
223 tgt_dev
->luns
= luns
;
224 tgt_dev
->parent
= dev
;
229 kfree(dev_map
->chnls
[i
].lun_offs
);
232 kfree(dev_map
->chnls
);
239 static struct nvm_tgt_type
*__nvm_find_target_type(const char *name
)
241 struct nvm_tgt_type
*tt
;
243 list_for_each_entry(tt
, &nvm_tgt_types
, list
)
244 if (!strcmp(name
, tt
->name
))
250 static struct nvm_tgt_type
*nvm_find_target_type(const char *name
)
252 struct nvm_tgt_type
*tt
;
254 down_write(&nvm_tgtt_lock
);
255 tt
= __nvm_find_target_type(name
);
256 up_write(&nvm_tgtt_lock
);
261 static int nvm_config_check_luns(struct nvm_geo
*geo
, int lun_begin
,
264 if (lun_begin
> lun_end
|| lun_end
>= geo
->all_luns
) {
265 pr_err("lun out of bound (%u:%u > %u)\n",
266 lun_begin
, lun_end
, geo
->all_luns
- 1);
273 static int __nvm_config_simple(struct nvm_dev
*dev
,
274 struct nvm_ioctl_create_simple
*s
)
276 struct nvm_geo
*geo
= &dev
->geo
;
278 if (s
->lun_begin
== -1 && s
->lun_end
== -1) {
280 s
->lun_end
= geo
->all_luns
- 1;
283 return nvm_config_check_luns(geo
, s
->lun_begin
, s
->lun_end
);
286 static int __nvm_config_extended(struct nvm_dev
*dev
,
287 struct nvm_ioctl_create_extended
*e
)
289 if (e
->lun_begin
== 0xFFFF && e
->lun_end
== 0xFFFF) {
291 e
->lun_end
= dev
->geo
.all_luns
- 1;
294 /* op not set falls into target's default */
295 if (e
->op
== 0xFFFF) {
296 e
->op
= NVM_TARGET_DEFAULT_OP
;
297 } else if (e
->op
< NVM_TARGET_MIN_OP
|| e
->op
> NVM_TARGET_MAX_OP
) {
298 pr_err("invalid over provisioning value\n");
302 return nvm_config_check_luns(&dev
->geo
, e
->lun_begin
, e
->lun_end
);
305 static int nvm_create_tgt(struct nvm_dev
*dev
, struct nvm_ioctl_create
*create
)
307 struct nvm_ioctl_create_extended e
;
308 struct request_queue
*tqueue
;
309 struct gendisk
*tdisk
;
310 struct nvm_tgt_type
*tt
;
311 struct nvm_target
*t
;
312 struct nvm_tgt_dev
*tgt_dev
;
317 switch (create
->conf
.type
) {
318 case NVM_CONFIG_TYPE_SIMPLE
:
319 ret
= __nvm_config_simple(dev
, &create
->conf
.s
);
323 e
.lun_begin
= create
->conf
.s
.lun_begin
;
324 e
.lun_end
= create
->conf
.s
.lun_end
;
325 e
.op
= NVM_TARGET_DEFAULT_OP
;
327 case NVM_CONFIG_TYPE_EXTENDED
:
328 ret
= __nvm_config_extended(dev
, &create
->conf
.e
);
335 pr_err("config type not valid\n");
339 tt
= nvm_find_target_type(create
->tgttype
);
341 pr_err("target type %s not found\n", create
->tgttype
);
345 if ((tt
->flags
& NVM_TGT_F_HOST_L2P
) != (dev
->geo
.dom
& NVM_RSP_L2P
)) {
346 pr_err("device is incompatible with target L2P type.\n");
350 if (nvm_target_exists(create
->tgtname
)) {
351 pr_err("target name already exists (%s)\n",
356 ret
= nvm_reserve_luns(dev
, e
.lun_begin
, e
.lun_end
);
360 t
= kmalloc(sizeof(struct nvm_target
), GFP_KERNEL
);
366 tgt_dev
= nvm_create_tgt_dev(dev
, e
.lun_begin
, e
.lun_end
, e
.op
);
368 pr_err("could not create target device\n");
373 tdisk
= alloc_disk(0);
379 tqueue
= blk_alloc_queue(dev
->q
->node
);
385 strlcpy(tdisk
->disk_name
, create
->tgtname
, sizeof(tdisk
->disk_name
));
386 tdisk
->flags
= GENHD_FL_EXT_DEVT
;
388 tdisk
->first_minor
= 0;
389 tdisk
->fops
= tt
->bops
;
390 tdisk
->queue
= tqueue
;
392 targetdata
= tt
->init(tgt_dev
, tdisk
, create
->flags
);
393 if (IS_ERR(targetdata
)) {
394 ret
= PTR_ERR(targetdata
);
398 tdisk
->private_data
= targetdata
;
399 tqueue
->queuedata
= targetdata
;
401 mdts
= (dev
->geo
.csecs
>> 9) * NVM_MAX_VLBA
;
403 mdts
= min_t(u32
, dev
->geo
.mdts
,
404 (dev
->geo
.csecs
>> 9) * NVM_MAX_VLBA
);
406 blk_queue_max_hw_sectors(tqueue
, mdts
);
408 set_capacity(tdisk
, tt
->capacity(targetdata
));
411 if (tt
->sysfs_init
&& tt
->sysfs_init(tdisk
)) {
420 mutex_lock(&dev
->mlock
);
421 list_add_tail(&t
->list
, &dev
->targets
);
422 mutex_unlock(&dev
->mlock
);
424 __module_get(tt
->owner
);
429 tt
->exit(targetdata
, true);
431 blk_cleanup_queue(tqueue
);
436 nvm_remove_tgt_dev(tgt_dev
, 0);
440 nvm_release_luns_err(dev
, e
.lun_begin
, e
.lun_end
);
444 static void __nvm_remove_target(struct nvm_target
*t
, bool graceful
)
446 struct nvm_tgt_type
*tt
= t
->type
;
447 struct gendisk
*tdisk
= t
->disk
;
448 struct request_queue
*q
= tdisk
->queue
;
451 blk_cleanup_queue(q
);
454 tt
->sysfs_exit(tdisk
);
457 tt
->exit(tdisk
->private_data
, graceful
);
459 nvm_remove_tgt_dev(t
->dev
, 1);
461 module_put(t
->type
->owner
);
468 * nvm_remove_tgt - Removes a target from the media manager
469 * @remove: ioctl structure with target name to remove.
476 static int nvm_remove_tgt(struct nvm_ioctl_remove
*remove
)
478 struct nvm_target
*t
= NULL
;
481 down_read(&nvm_lock
);
482 list_for_each_entry(dev
, &nvm_devices
, devices
) {
483 mutex_lock(&dev
->mlock
);
484 t
= nvm_find_target(dev
, remove
->tgtname
);
486 mutex_unlock(&dev
->mlock
);
489 mutex_unlock(&dev
->mlock
);
494 pr_err("failed to remove target %s\n",
499 __nvm_remove_target(t
, true);
500 kref_put(&dev
->ref
, nvm_free
);
505 static int nvm_register_map(struct nvm_dev
*dev
)
507 struct nvm_dev_map
*rmap
;
510 rmap
= kmalloc(sizeof(struct nvm_dev_map
), GFP_KERNEL
);
514 rmap
->chnls
= kcalloc(dev
->geo
.num_ch
, sizeof(struct nvm_ch_map
),
519 for (i
= 0; i
< dev
->geo
.num_ch
; i
++) {
520 struct nvm_ch_map
*ch_rmap
;
522 int luns_in_chnl
= dev
->geo
.num_lun
;
524 ch_rmap
= &rmap
->chnls
[i
];
526 ch_rmap
->ch_off
= -1;
527 ch_rmap
->num_lun
= luns_in_chnl
;
529 lun_roffs
= kcalloc(luns_in_chnl
, sizeof(int), GFP_KERNEL
);
533 for (j
= 0; j
< luns_in_chnl
; j
++)
536 ch_rmap
->lun_offs
= lun_roffs
;
544 kfree(rmap
->chnls
[i
].lun_offs
);
551 static void nvm_unregister_map(struct nvm_dev
*dev
)
553 struct nvm_dev_map
*rmap
= dev
->rmap
;
556 for (i
= 0; i
< dev
->geo
.num_ch
; i
++)
557 kfree(rmap
->chnls
[i
].lun_offs
);
563 static void nvm_map_to_dev(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*p
)
565 struct nvm_dev_map
*dev_map
= tgt_dev
->map
;
566 struct nvm_ch_map
*ch_map
= &dev_map
->chnls
[p
->a
.ch
];
567 int lun_off
= ch_map
->lun_offs
[p
->a
.lun
];
569 p
->a
.ch
+= ch_map
->ch_off
;
573 static void nvm_map_to_tgt(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*p
)
575 struct nvm_dev
*dev
= tgt_dev
->parent
;
576 struct nvm_dev_map
*dev_rmap
= dev
->rmap
;
577 struct nvm_ch_map
*ch_rmap
= &dev_rmap
->chnls
[p
->a
.ch
];
578 int lun_roff
= ch_rmap
->lun_offs
[p
->a
.lun
];
580 p
->a
.ch
-= ch_rmap
->ch_off
;
581 p
->a
.lun
-= lun_roff
;
584 static void nvm_ppa_tgt_to_dev(struct nvm_tgt_dev
*tgt_dev
,
585 struct ppa_addr
*ppa_list
, int nr_ppas
)
589 for (i
= 0; i
< nr_ppas
; i
++) {
590 nvm_map_to_dev(tgt_dev
, &ppa_list
[i
]);
591 ppa_list
[i
] = generic_to_dev_addr(tgt_dev
->parent
, ppa_list
[i
]);
595 static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev
*tgt_dev
,
596 struct ppa_addr
*ppa_list
, int nr_ppas
)
600 for (i
= 0; i
< nr_ppas
; i
++) {
601 ppa_list
[i
] = dev_to_generic_addr(tgt_dev
->parent
, ppa_list
[i
]);
602 nvm_map_to_tgt(tgt_dev
, &ppa_list
[i
]);
606 static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
)
608 struct ppa_addr
*ppa_list
= nvm_rq_to_ppa_list(rqd
);
610 nvm_ppa_tgt_to_dev(tgt_dev
, ppa_list
, rqd
->nr_ppas
);
613 static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
)
615 struct ppa_addr
*ppa_list
= nvm_rq_to_ppa_list(rqd
);
617 nvm_ppa_dev_to_tgt(tgt_dev
, ppa_list
, rqd
->nr_ppas
);
620 int nvm_register_tgt_type(struct nvm_tgt_type
*tt
)
624 down_write(&nvm_tgtt_lock
);
625 if (__nvm_find_target_type(tt
->name
))
628 list_add(&tt
->list
, &nvm_tgt_types
);
629 up_write(&nvm_tgtt_lock
);
633 EXPORT_SYMBOL(nvm_register_tgt_type
);
635 void nvm_unregister_tgt_type(struct nvm_tgt_type
*tt
)
640 down_write(&nvm_tgtt_lock
);
642 up_write(&nvm_tgtt_lock
);
644 EXPORT_SYMBOL(nvm_unregister_tgt_type
);
646 void *nvm_dev_dma_alloc(struct nvm_dev
*dev
, gfp_t mem_flags
,
647 dma_addr_t
*dma_handler
)
649 return dev
->ops
->dev_dma_alloc(dev
, dev
->dma_pool
, mem_flags
,
652 EXPORT_SYMBOL(nvm_dev_dma_alloc
);
654 void nvm_dev_dma_free(struct nvm_dev
*dev
, void *addr
, dma_addr_t dma_handler
)
656 dev
->ops
->dev_dma_free(dev
->dma_pool
, addr
, dma_handler
);
658 EXPORT_SYMBOL(nvm_dev_dma_free
);
660 static struct nvm_dev
*nvm_find_nvm_dev(const char *name
)
664 list_for_each_entry(dev
, &nvm_devices
, devices
)
665 if (!strcmp(name
, dev
->name
))
671 static int nvm_set_rqd_ppalist(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
,
672 const struct ppa_addr
*ppas
, int nr_ppas
)
674 struct nvm_dev
*dev
= tgt_dev
->parent
;
675 struct nvm_geo
*geo
= &tgt_dev
->geo
;
676 int i
, plane_cnt
, pl_idx
;
679 if (geo
->pln_mode
== NVM_PLANE_SINGLE
&& nr_ppas
== 1) {
680 rqd
->nr_ppas
= nr_ppas
;
681 rqd
->ppa_addr
= ppas
[0];
686 rqd
->nr_ppas
= nr_ppas
;
687 rqd
->ppa_list
= nvm_dev_dma_alloc(dev
, GFP_KERNEL
, &rqd
->dma_ppa_list
);
688 if (!rqd
->ppa_list
) {
689 pr_err("failed to allocate dma memory\n");
693 plane_cnt
= geo
->pln_mode
;
694 rqd
->nr_ppas
*= plane_cnt
;
696 for (i
= 0; i
< nr_ppas
; i
++) {
697 for (pl_idx
= 0; pl_idx
< plane_cnt
; pl_idx
++) {
700 rqd
->ppa_list
[(pl_idx
* nr_ppas
) + i
] = ppa
;
707 static void nvm_free_rqd_ppalist(struct nvm_tgt_dev
*tgt_dev
,
713 nvm_dev_dma_free(tgt_dev
->parent
, rqd
->ppa_list
, rqd
->dma_ppa_list
);
716 static int nvm_set_flags(struct nvm_geo
*geo
, struct nvm_rq
*rqd
)
720 if (geo
->version
== NVM_OCSSD_SPEC_20
)
724 flags
|= geo
->pln_mode
>> 1;
726 if (rqd
->opcode
== NVM_OP_PREAD
)
727 flags
|= (NVM_IO_SCRAMBLE_ENABLE
| NVM_IO_SUSPEND
);
728 else if (rqd
->opcode
== NVM_OP_PWRITE
)
729 flags
|= NVM_IO_SCRAMBLE_ENABLE
;
734 int nvm_submit_io(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
, void *buf
)
736 struct nvm_dev
*dev
= tgt_dev
->parent
;
739 if (!dev
->ops
->submit_io
)
742 nvm_rq_tgt_to_dev(tgt_dev
, rqd
);
745 rqd
->flags
= nvm_set_flags(&tgt_dev
->geo
, rqd
);
747 /* In case of error, fail with right address format */
748 ret
= dev
->ops
->submit_io(dev
, rqd
, buf
);
750 nvm_rq_dev_to_tgt(tgt_dev
, rqd
);
753 EXPORT_SYMBOL(nvm_submit_io
);
755 static void nvm_sync_end_io(struct nvm_rq
*rqd
)
757 struct completion
*waiting
= rqd
->private;
762 static int nvm_submit_io_wait(struct nvm_dev
*dev
, struct nvm_rq
*rqd
,
765 DECLARE_COMPLETION_ONSTACK(wait
);
768 rqd
->end_io
= nvm_sync_end_io
;
769 rqd
->private = &wait
;
771 ret
= dev
->ops
->submit_io(dev
, rqd
, buf
);
775 wait_for_completion_io(&wait
);
780 int nvm_submit_io_sync(struct nvm_tgt_dev
*tgt_dev
, struct nvm_rq
*rqd
,
783 struct nvm_dev
*dev
= tgt_dev
->parent
;
786 if (!dev
->ops
->submit_io
)
789 nvm_rq_tgt_to_dev(tgt_dev
, rqd
);
792 rqd
->flags
= nvm_set_flags(&tgt_dev
->geo
, rqd
);
794 ret
= nvm_submit_io_wait(dev
, rqd
, buf
);
798 EXPORT_SYMBOL(nvm_submit_io_sync
);
800 void nvm_end_io(struct nvm_rq
*rqd
)
802 struct nvm_tgt_dev
*tgt_dev
= rqd
->dev
;
804 /* Convert address space */
806 nvm_rq_dev_to_tgt(tgt_dev
, rqd
);
811 EXPORT_SYMBOL(nvm_end_io
);
813 static int nvm_submit_io_sync_raw(struct nvm_dev
*dev
, struct nvm_rq
*rqd
)
815 if (!dev
->ops
->submit_io
)
819 rqd
->flags
= nvm_set_flags(&dev
->geo
, rqd
);
821 return nvm_submit_io_wait(dev
, rqd
, NULL
);
824 static int nvm_bb_chunk_sense(struct nvm_dev
*dev
, struct ppa_addr ppa
)
826 struct nvm_rq rqd
= { NULL
};
828 struct bio_vec bio_vec
;
832 page
= alloc_page(GFP_KERNEL
);
836 bio_init(&bio
, &bio_vec
, 1);
837 bio_add_page(&bio
, page
, PAGE_SIZE
, 0);
838 bio_set_op_attrs(&bio
, REQ_OP_READ
, 0);
841 rqd
.opcode
= NVM_OP_PREAD
;
844 rqd
.ppa_addr
= generic_to_dev_addr(dev
, ppa
);
846 ret
= nvm_submit_io_sync_raw(dev
, &rqd
);
855 * Scans a 1.2 chunk first and last page to determine if its state.
856 * If the chunk is found to be open, also scan it to update the write
859 static int nvm_bb_chunk_scan(struct nvm_dev
*dev
, struct ppa_addr ppa
,
860 struct nvm_chk_meta
*meta
)
862 struct nvm_geo
*geo
= &dev
->geo
;
865 /* sense first page */
866 ret
= nvm_bb_chunk_sense(dev
, ppa
);
867 if (ret
< 0) /* io error */
869 else if (ret
== 0) /* valid data */
870 meta
->state
= NVM_CHK_ST_OPEN
;
873 * If empty page, the chunk is free, else it is an
874 * actual io error. In that case, mark it offline.
877 case NVM_RSP_ERR_EMPTYPAGE
:
878 meta
->state
= NVM_CHK_ST_FREE
;
880 case NVM_RSP_ERR_FAILCRC
:
881 case NVM_RSP_ERR_FAILECC
:
882 case NVM_RSP_WARN_HIGHECC
:
883 meta
->state
= NVM_CHK_ST_OPEN
;
886 return -ret
; /* other io error */
890 /* sense last page */
891 ppa
.g
.pg
= geo
->num_pg
- 1;
892 ppa
.g
.pl
= geo
->num_pln
- 1;
894 ret
= nvm_bb_chunk_sense(dev
, ppa
);
895 if (ret
< 0) /* io error */
897 else if (ret
== 0) { /* Chunk fully written */
898 meta
->state
= NVM_CHK_ST_CLOSED
;
899 meta
->wp
= geo
->clba
;
901 } else if (ret
> 0) {
903 case NVM_RSP_ERR_EMPTYPAGE
:
904 case NVM_RSP_ERR_FAILCRC
:
905 case NVM_RSP_ERR_FAILECC
:
906 case NVM_RSP_WARN_HIGHECC
:
907 meta
->state
= NVM_CHK_ST_OPEN
;
910 return -ret
; /* other io error */
916 * chunk is open, we scan sequentially to update the write pointer.
917 * We make the assumption that targets write data across all planes
918 * before moving to the next page.
920 for (pg
= 0; pg
< geo
->num_pg
; pg
++) {
921 for (pl
= 0; pl
< geo
->num_pln
; pl
++) {
925 ret
= nvm_bb_chunk_sense(dev
, ppa
);
926 if (ret
< 0) /* io error */
929 meta
->wp
+= geo
->ws_min
;
930 } else if (ret
> 0) {
932 case NVM_RSP_ERR_EMPTYPAGE
:
934 case NVM_RSP_ERR_FAILCRC
:
935 case NVM_RSP_ERR_FAILECC
:
936 case NVM_RSP_WARN_HIGHECC
:
937 meta
->wp
+= geo
->ws_min
;
940 return -ret
; /* other io error */
950 * folds a bad block list from its plane representation to its
951 * chunk representation.
953 * If any of the planes status are bad or grown bad, the chunk is marked
954 * offline. If not bad, the first plane state acts as the chunk state.
956 static int nvm_bb_to_chunk(struct nvm_dev
*dev
, struct ppa_addr ppa
,
957 u8
*blks
, int nr_blks
, struct nvm_chk_meta
*meta
)
959 struct nvm_geo
*geo
= &dev
->geo
;
960 int ret
, blk
, pl
, offset
, blktype
;
962 for (blk
= 0; blk
< geo
->num_chk
; blk
++) {
963 offset
= blk
* geo
->pln_mode
;
964 blktype
= blks
[offset
];
966 for (pl
= 0; pl
< geo
->pln_mode
; pl
++) {
967 if (blks
[offset
+ pl
] &
968 (NVM_BLK_T_BAD
|NVM_BLK_T_GRWN_BAD
)) {
969 blktype
= blks
[offset
+ pl
];
977 meta
->type
= NVM_CHK_TP_W_SEQ
;
979 meta
->slba
= generic_to_dev_addr(dev
, ppa
).ppa
;
980 meta
->cnlb
= dev
->geo
.clba
;
982 if (blktype
== NVM_BLK_T_FREE
) {
983 ret
= nvm_bb_chunk_scan(dev
, ppa
, meta
);
987 meta
->state
= NVM_CHK_ST_OFFLINE
;
996 static int nvm_get_bb_meta(struct nvm_dev
*dev
, sector_t slba
,
997 int nchks
, struct nvm_chk_meta
*meta
)
999 struct nvm_geo
*geo
= &dev
->geo
;
1000 struct ppa_addr ppa
;
1002 int ch
, lun
, nr_blks
;
1006 ppa
= dev_to_generic_addr(dev
, ppa
);
1011 if ((nchks
% geo
->num_chk
) != 0)
1014 nr_blks
= geo
->num_chk
* geo
->pln_mode
;
1016 blks
= kmalloc(nr_blks
, GFP_KERNEL
);
1020 for (ch
= ppa
.g
.ch
; ch
< geo
->num_ch
; ch
++) {
1021 for (lun
= ppa
.g
.lun
; lun
< geo
->num_lun
; lun
++) {
1022 struct ppa_addr ppa_gen
, ppa_dev
;
1029 ppa_gen
.g
.lun
= lun
;
1030 ppa_dev
= generic_to_dev_addr(dev
, ppa_gen
);
1032 ret
= dev
->ops
->get_bb_tbl(dev
, ppa_dev
, blks
);
1036 ret
= nvm_bb_to_chunk(dev
, ppa_gen
, blks
, nr_blks
,
1041 meta
+= geo
->num_chk
;
1042 nchks
-= geo
->num_chk
;
1050 int nvm_get_chunk_meta(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr ppa
,
1051 int nchks
, struct nvm_chk_meta
*meta
)
1053 struct nvm_dev
*dev
= tgt_dev
->parent
;
1055 nvm_ppa_tgt_to_dev(tgt_dev
, &ppa
, 1);
1057 if (dev
->geo
.version
== NVM_OCSSD_SPEC_12
)
1058 return nvm_get_bb_meta(dev
, (sector_t
)ppa
.ppa
, nchks
, meta
);
1060 return dev
->ops
->get_chk_meta(dev
, (sector_t
)ppa
.ppa
, nchks
, meta
);
1062 EXPORT_SYMBOL_GPL(nvm_get_chunk_meta
);
1064 int nvm_set_chunk_meta(struct nvm_tgt_dev
*tgt_dev
, struct ppa_addr
*ppas
,
1065 int nr_ppas
, int type
)
1067 struct nvm_dev
*dev
= tgt_dev
->parent
;
1071 if (dev
->geo
.version
== NVM_OCSSD_SPEC_20
)
1074 if (nr_ppas
> NVM_MAX_VLBA
) {
1075 pr_err("unable to update all blocks atomically\n");
1079 memset(&rqd
, 0, sizeof(struct nvm_rq
));
1081 nvm_set_rqd_ppalist(tgt_dev
, &rqd
, ppas
, nr_ppas
);
1082 nvm_rq_tgt_to_dev(tgt_dev
, &rqd
);
1084 ret
= dev
->ops
->set_bb_tbl(dev
, &rqd
.ppa_addr
, rqd
.nr_ppas
, type
);
1085 nvm_free_rqd_ppalist(tgt_dev
, &rqd
);
1091 EXPORT_SYMBOL_GPL(nvm_set_chunk_meta
);
1093 static int nvm_core_init(struct nvm_dev
*dev
)
1095 struct nvm_geo
*geo
= &dev
->geo
;
1098 dev
->lun_map
= kcalloc(BITS_TO_LONGS(geo
->all_luns
),
1099 sizeof(unsigned long), GFP_KERNEL
);
1103 INIT_LIST_HEAD(&dev
->area_list
);
1104 INIT_LIST_HEAD(&dev
->targets
);
1105 mutex_init(&dev
->mlock
);
1106 spin_lock_init(&dev
->lock
);
1108 ret
= nvm_register_map(dev
);
1114 kfree(dev
->lun_map
);
1118 static void nvm_free(struct kref
*ref
)
1120 struct nvm_dev
*dev
= container_of(ref
, struct nvm_dev
, ref
);
1123 dev
->ops
->destroy_dma_pool(dev
->dma_pool
);
1126 nvm_unregister_map(dev
);
1128 kfree(dev
->lun_map
);
1132 static int nvm_init(struct nvm_dev
*dev
)
1134 struct nvm_geo
*geo
= &dev
->geo
;
1137 if (dev
->ops
->identity(dev
)) {
1138 pr_err("device could not be identified\n");
1142 pr_debug("ver:%u.%u nvm_vendor:%x\n", geo
->major_ver_id
,
1143 geo
->minor_ver_id
, geo
->vmnt
);
1145 ret
= nvm_core_init(dev
);
1147 pr_err("could not initialize core structures.\n");
1151 pr_info("registered %s [%u/%u/%u/%u/%u]\n",
1152 dev
->name
, dev
->geo
.ws_min
, dev
->geo
.ws_opt
,
1153 dev
->geo
.num_chk
, dev
->geo
.all_luns
,
1157 pr_err("failed to initialize nvm\n");
1161 struct nvm_dev
*nvm_alloc_dev(int node
)
1163 struct nvm_dev
*dev
;
1165 dev
= kzalloc_node(sizeof(struct nvm_dev
), GFP_KERNEL
, node
);
1167 kref_init(&dev
->ref
);
1171 EXPORT_SYMBOL(nvm_alloc_dev
);
1173 int nvm_register(struct nvm_dev
*dev
)
1175 int ret
, exp_pool_size
;
1177 pr_warn_once("lightnvm support is deprecated and will be removed in Linux 5.15.\n");
1179 if (!dev
->q
|| !dev
->ops
) {
1180 kref_put(&dev
->ref
, nvm_free
);
1184 ret
= nvm_init(dev
);
1186 kref_put(&dev
->ref
, nvm_free
);
1190 exp_pool_size
= max_t(int, PAGE_SIZE
,
1191 (NVM_MAX_VLBA
* (sizeof(u64
) + dev
->geo
.sos
)));
1192 exp_pool_size
= round_up(exp_pool_size
, PAGE_SIZE
);
1194 dev
->dma_pool
= dev
->ops
->create_dma_pool(dev
, "ppalist",
1196 if (!dev
->dma_pool
) {
1197 pr_err("could not create dma pool\n");
1198 kref_put(&dev
->ref
, nvm_free
);
1202 /* register device with a supported media manager */
1203 down_write(&nvm_lock
);
1204 list_add(&dev
->devices
, &nvm_devices
);
1205 up_write(&nvm_lock
);
1209 EXPORT_SYMBOL(nvm_register
);
1211 void nvm_unregister(struct nvm_dev
*dev
)
1213 struct nvm_target
*t
, *tmp
;
1215 mutex_lock(&dev
->mlock
);
1216 list_for_each_entry_safe(t
, tmp
, &dev
->targets
, list
) {
1217 if (t
->dev
->parent
!= dev
)
1219 __nvm_remove_target(t
, false);
1220 kref_put(&dev
->ref
, nvm_free
);
1222 mutex_unlock(&dev
->mlock
);
1224 down_write(&nvm_lock
);
1225 list_del(&dev
->devices
);
1226 up_write(&nvm_lock
);
1228 kref_put(&dev
->ref
, nvm_free
);
1230 EXPORT_SYMBOL(nvm_unregister
);
1232 static int __nvm_configure_create(struct nvm_ioctl_create
*create
)
1234 struct nvm_dev
*dev
;
1237 down_write(&nvm_lock
);
1238 dev
= nvm_find_nvm_dev(create
->dev
);
1239 up_write(&nvm_lock
);
1242 pr_err("device not found\n");
1246 kref_get(&dev
->ref
);
1247 ret
= nvm_create_tgt(dev
, create
);
1249 kref_put(&dev
->ref
, nvm_free
);
1254 static long nvm_ioctl_info(struct file
*file
, void __user
*arg
)
1256 struct nvm_ioctl_info
*info
;
1257 struct nvm_tgt_type
*tt
;
1260 info
= memdup_user(arg
, sizeof(struct nvm_ioctl_info
));
1262 return PTR_ERR(info
);
1264 info
->version
[0] = NVM_VERSION_MAJOR
;
1265 info
->version
[1] = NVM_VERSION_MINOR
;
1266 info
->version
[2] = NVM_VERSION_PATCH
;
1268 down_write(&nvm_tgtt_lock
);
1269 list_for_each_entry(tt
, &nvm_tgt_types
, list
) {
1270 struct nvm_ioctl_info_tgt
*tgt
= &info
->tgts
[tgt_iter
];
1272 tgt
->version
[0] = tt
->version
[0];
1273 tgt
->version
[1] = tt
->version
[1];
1274 tgt
->version
[2] = tt
->version
[2];
1275 strncpy(tgt
->tgtname
, tt
->name
, NVM_TTYPE_NAME_MAX
);
1280 info
->tgtsize
= tgt_iter
;
1281 up_write(&nvm_tgtt_lock
);
1283 if (copy_to_user(arg
, info
, sizeof(struct nvm_ioctl_info
))) {
1292 static long nvm_ioctl_get_devices(struct file
*file
, void __user
*arg
)
1294 struct nvm_ioctl_get_devices
*devices
;
1295 struct nvm_dev
*dev
;
1298 devices
= kzalloc(sizeof(struct nvm_ioctl_get_devices
), GFP_KERNEL
);
1302 down_write(&nvm_lock
);
1303 list_for_each_entry(dev
, &nvm_devices
, devices
) {
1304 struct nvm_ioctl_device_info
*info
= &devices
->info
[i
];
1306 strlcpy(info
->devname
, dev
->name
, sizeof(info
->devname
));
1308 /* kept for compatibility */
1309 info
->bmversion
[0] = 1;
1310 info
->bmversion
[1] = 0;
1311 info
->bmversion
[2] = 0;
1312 strlcpy(info
->bmname
, "gennvm", sizeof(info
->bmname
));
1315 if (i
>= ARRAY_SIZE(devices
->info
)) {
1316 pr_err("max %zd devices can be reported.\n",
1317 ARRAY_SIZE(devices
->info
));
1321 up_write(&nvm_lock
);
1323 devices
->nr_devices
= i
;
1325 if (copy_to_user(arg
, devices
,
1326 sizeof(struct nvm_ioctl_get_devices
))) {
1335 static long nvm_ioctl_dev_create(struct file
*file
, void __user
*arg
)
1337 struct nvm_ioctl_create create
;
1339 if (copy_from_user(&create
, arg
, sizeof(struct nvm_ioctl_create
)))
1342 if (create
.conf
.type
== NVM_CONFIG_TYPE_EXTENDED
&&
1343 create
.conf
.e
.rsv
!= 0) {
1344 pr_err("reserved config field in use\n");
1348 create
.dev
[DISK_NAME_LEN
- 1] = '\0';
1349 create
.tgttype
[NVM_TTYPE_NAME_MAX
- 1] = '\0';
1350 create
.tgtname
[DISK_NAME_LEN
- 1] = '\0';
1352 if (create
.flags
!= 0) {
1353 __u32 flags
= create
.flags
;
1355 /* Check for valid flags */
1356 if (flags
& NVM_TARGET_FACTORY
)
1357 flags
&= ~NVM_TARGET_FACTORY
;
1360 pr_err("flag not supported\n");
1365 return __nvm_configure_create(&create
);
1368 static long nvm_ioctl_dev_remove(struct file
*file
, void __user
*arg
)
1370 struct nvm_ioctl_remove remove
;
1372 if (copy_from_user(&remove
, arg
, sizeof(struct nvm_ioctl_remove
)))
1375 remove
.tgtname
[DISK_NAME_LEN
- 1] = '\0';
1377 if (remove
.flags
!= 0) {
1378 pr_err("no flags supported\n");
1382 return nvm_remove_tgt(&remove
);
1385 /* kept for compatibility reasons */
1386 static long nvm_ioctl_dev_init(struct file
*file
, void __user
*arg
)
1388 struct nvm_ioctl_dev_init init
;
1390 if (copy_from_user(&init
, arg
, sizeof(struct nvm_ioctl_dev_init
)))
1393 if (init
.flags
!= 0) {
1394 pr_err("no flags supported\n");
1401 /* Kept for compatibility reasons */
1402 static long nvm_ioctl_dev_factory(struct file
*file
, void __user
*arg
)
1404 struct nvm_ioctl_dev_factory fact
;
1406 if (copy_from_user(&fact
, arg
, sizeof(struct nvm_ioctl_dev_factory
)))
1409 fact
.dev
[DISK_NAME_LEN
- 1] = '\0';
1411 if (fact
.flags
& ~(NVM_FACTORY_NR_BITS
- 1))
1417 static long nvm_ctl_ioctl(struct file
*file
, uint cmd
, unsigned long arg
)
1419 void __user
*argp
= (void __user
*)arg
;
1421 if (!capable(CAP_SYS_ADMIN
))
1426 return nvm_ioctl_info(file
, argp
);
1427 case NVM_GET_DEVICES
:
1428 return nvm_ioctl_get_devices(file
, argp
);
1429 case NVM_DEV_CREATE
:
1430 return nvm_ioctl_dev_create(file
, argp
);
1431 case NVM_DEV_REMOVE
:
1432 return nvm_ioctl_dev_remove(file
, argp
);
1434 return nvm_ioctl_dev_init(file
, argp
);
1435 case NVM_DEV_FACTORY
:
1436 return nvm_ioctl_dev_factory(file
, argp
);
1441 static const struct file_operations _ctl_fops
= {
1442 .open
= nonseekable_open
,
1443 .unlocked_ioctl
= nvm_ctl_ioctl
,
1444 .owner
= THIS_MODULE
,
1445 .llseek
= noop_llseek
,
1448 static struct miscdevice _nvm_misc
= {
1449 .minor
= MISC_DYNAMIC_MINOR
,
1451 .nodename
= "lightnvm/control",
1454 builtin_misc_device(_nvm_misc
);