]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/lightnvm/pblk-init.c
2 * Copyright (C) 2015 IT University of Copenhagen (rrpc.c)
3 * Copyright (C) 2016 CNEX Labs
4 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
5 * Matias Bjorling <matias@cnexlabs.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * Implementation of a physical block-device target for Open-channel SSDs.
18 * pblk-init.c - pblk's initialization.
23 static struct kmem_cache
*pblk_blk_ws_cache
, *pblk_rec_cache
, *pblk_r_rq_cache
,
24 *pblk_w_rq_cache
, *pblk_line_meta_cache
;
25 static DECLARE_RWSEM(pblk_lock
);
27 static int pblk_rw_io(struct request_queue
*q
, struct pblk
*pblk
,
32 /* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
33 * constraint. Writes can be of arbitrary size.
35 if (bio_data_dir(bio
) == READ
) {
36 blk_queue_split(q
, &bio
, q
->bio_split
);
37 ret
= pblk_submit_read(pblk
, bio
);
38 if (ret
== NVM_IO_DONE
&& bio_flagged(bio
, BIO_CLONED
))
44 /* Prevent deadlock in the case of a modest LUN configuration and large
45 * user I/Os. Unless stalled, the rate limiter leaves at least 256KB
46 * available for user I/O.
48 if (unlikely(pblk_get_secs(bio
) >= pblk_rl_sysfs_rate_show(&pblk
->rl
)))
49 blk_queue_split(q
, &bio
, q
->bio_split
);
51 return pblk_write_to_cache(pblk
, bio
, PBLK_IOTYPE_USER
);
54 static blk_qc_t
pblk_make_rq(struct request_queue
*q
, struct bio
*bio
)
56 struct pblk
*pblk
= q
->queuedata
;
58 if (bio_op(bio
) == REQ_OP_DISCARD
) {
59 pblk_discard(pblk
, bio
);
60 if (!(bio
->bi_opf
& REQ_PREFLUSH
)) {
66 switch (pblk_rw_io(q
, pblk
, bio
)) {
78 static void pblk_l2p_free(struct pblk
*pblk
)
80 vfree(pblk
->trans_map
);
83 static int pblk_l2p_init(struct pblk
*pblk
)
89 if (pblk
->ppaf_bitsize
< 32)
92 pblk
->trans_map
= vmalloc(entry_size
* pblk
->rl
.nr_secs
);
96 pblk_ppa_set_empty(&ppa
);
98 for (i
= 0; i
< pblk
->rl
.nr_secs
; i
++)
99 pblk_trans_map_set(pblk
, i
, ppa
);
104 static void pblk_rwb_free(struct pblk
*pblk
)
106 if (pblk_rb_tear_down_check(&pblk
->rwb
))
107 pr_err("pblk: write buffer error on tear down\n");
109 pblk_rb_data_free(&pblk
->rwb
);
110 vfree(pblk_rb_entries_ref(&pblk
->rwb
));
113 static int pblk_rwb_init(struct pblk
*pblk
)
115 struct nvm_tgt_dev
*dev
= pblk
->dev
;
116 struct nvm_geo
*geo
= &dev
->geo
;
117 struct pblk_rb_entry
*entries
;
118 unsigned long nr_entries
;
119 unsigned int power_size
, power_seg_sz
;
121 nr_entries
= pblk_rb_calculate_size(pblk
->pgs_in_buffer
);
123 entries
= vzalloc(nr_entries
* sizeof(struct pblk_rb_entry
));
127 power_size
= get_count_order(nr_entries
);
128 power_seg_sz
= get_count_order(geo
->sec_size
);
130 return pblk_rb_init(&pblk
->rwb
, entries
, power_size
, power_seg_sz
);
133 /* Minimum pages needed within a lun */
134 #define PAGE_POOL_SIZE 16
135 #define ADDR_POOL_SIZE 64
137 static int pblk_set_ppaf(struct pblk
*pblk
)
139 struct nvm_tgt_dev
*dev
= pblk
->dev
;
140 struct nvm_geo
*geo
= &dev
->geo
;
141 struct nvm_addr_format ppaf
= geo
->ppaf
;
144 /* Re-calculate channel and lun format to adapt to configuration */
145 power_len
= get_count_order(geo
->nr_chnls
);
146 if (1 << power_len
!= geo
->nr_chnls
) {
147 pr_err("pblk: supports only power-of-two channel config.\n");
150 ppaf
.ch_len
= power_len
;
152 power_len
= get_count_order(geo
->luns_per_chnl
);
153 if (1 << power_len
!= geo
->luns_per_chnl
) {
154 pr_err("pblk: supports only power-of-two LUN config.\n");
157 ppaf
.lun_len
= power_len
;
159 pblk
->ppaf
.sec_offset
= 0;
160 pblk
->ppaf
.pln_offset
= ppaf
.sect_len
;
161 pblk
->ppaf
.ch_offset
= pblk
->ppaf
.pln_offset
+ ppaf
.pln_len
;
162 pblk
->ppaf
.lun_offset
= pblk
->ppaf
.ch_offset
+ ppaf
.ch_len
;
163 pblk
->ppaf
.pg_offset
= pblk
->ppaf
.lun_offset
+ ppaf
.lun_len
;
164 pblk
->ppaf
.blk_offset
= pblk
->ppaf
.pg_offset
+ ppaf
.pg_len
;
165 pblk
->ppaf
.sec_mask
= (1ULL << ppaf
.sect_len
) - 1;
166 pblk
->ppaf
.pln_mask
= ((1ULL << ppaf
.pln_len
) - 1) <<
167 pblk
->ppaf
.pln_offset
;
168 pblk
->ppaf
.ch_mask
= ((1ULL << ppaf
.ch_len
) - 1) <<
169 pblk
->ppaf
.ch_offset
;
170 pblk
->ppaf
.lun_mask
= ((1ULL << ppaf
.lun_len
) - 1) <<
171 pblk
->ppaf
.lun_offset
;
172 pblk
->ppaf
.pg_mask
= ((1ULL << ppaf
.pg_len
) - 1) <<
173 pblk
->ppaf
.pg_offset
;
174 pblk
->ppaf
.blk_mask
= ((1ULL << ppaf
.blk_len
) - 1) <<
175 pblk
->ppaf
.blk_offset
;
177 pblk
->ppaf_bitsize
= pblk
->ppaf
.blk_offset
+ ppaf
.blk_len
;
182 static int pblk_init_global_caches(struct pblk
*pblk
)
184 char cache_name
[PBLK_CACHE_NAME_LEN
];
186 down_write(&pblk_lock
);
187 pblk_blk_ws_cache
= kmem_cache_create("pblk_blk_ws",
188 sizeof(struct pblk_line_ws
), 0, 0, NULL
);
189 if (!pblk_blk_ws_cache
) {
190 up_write(&pblk_lock
);
194 pblk_rec_cache
= kmem_cache_create("pblk_rec",
195 sizeof(struct pblk_rec_ctx
), 0, 0, NULL
);
196 if (!pblk_rec_cache
) {
197 kmem_cache_destroy(pblk_blk_ws_cache
);
198 up_write(&pblk_lock
);
202 pblk_r_rq_cache
= kmem_cache_create("pblk_r_rq", pblk_r_rq_size
,
204 if (!pblk_r_rq_cache
) {
205 kmem_cache_destroy(pblk_blk_ws_cache
);
206 kmem_cache_destroy(pblk_rec_cache
);
207 up_write(&pblk_lock
);
211 pblk_w_rq_cache
= kmem_cache_create("pblk_w_rq", pblk_w_rq_size
,
213 if (!pblk_w_rq_cache
) {
214 kmem_cache_destroy(pblk_blk_ws_cache
);
215 kmem_cache_destroy(pblk_rec_cache
);
216 kmem_cache_destroy(pblk_r_rq_cache
);
217 up_write(&pblk_lock
);
221 snprintf(cache_name
, sizeof(cache_name
), "pblk_line_m_%s",
222 pblk
->disk
->disk_name
);
223 pblk_line_meta_cache
= kmem_cache_create(cache_name
,
224 pblk
->lm
.sec_bitmap_len
, 0, 0, NULL
);
225 if (!pblk_line_meta_cache
) {
226 kmem_cache_destroy(pblk_blk_ws_cache
);
227 kmem_cache_destroy(pblk_rec_cache
);
228 kmem_cache_destroy(pblk_r_rq_cache
);
229 kmem_cache_destroy(pblk_w_rq_cache
);
230 up_write(&pblk_lock
);
233 up_write(&pblk_lock
);
238 static int pblk_core_init(struct pblk
*pblk
)
240 struct nvm_tgt_dev
*dev
= pblk
->dev
;
241 struct nvm_geo
*geo
= &dev
->geo
;
245 pblk
->min_write_pgs
= geo
->sec_per_pl
* (geo
->sec_size
/ PAGE_SIZE
);
246 max_write_ppas
= pblk
->min_write_pgs
* geo
->nr_luns
;
247 pblk
->max_write_pgs
= (max_write_ppas
< nvm_max_phys_sects(dev
)) ?
248 max_write_ppas
: nvm_max_phys_sects(dev
);
249 pblk
->pgs_in_buffer
= NVM_MEM_PAGE_WRITE
* geo
->sec_per_pg
*
250 geo
->nr_planes
* geo
->nr_luns
;
252 if (pblk
->max_write_pgs
> PBLK_MAX_REQ_ADDRS
) {
253 pr_err("pblk: cannot support device max_phys_sect\n");
257 div_u64_rem(geo
->sec_per_blk
, pblk
->min_write_pgs
, &mod
);
259 pr_err("pblk: bad configuration of sectors/pages\n");
263 if (pblk_init_global_caches(pblk
))
266 pblk
->page_pool
= mempool_create_page_pool(PAGE_POOL_SIZE
, 0);
267 if (!pblk
->page_pool
)
270 pblk
->line_ws_pool
= mempool_create_slab_pool(geo
->nr_luns
,
272 if (!pblk
->line_ws_pool
)
275 pblk
->rec_pool
= mempool_create_slab_pool(geo
->nr_luns
, pblk_rec_cache
);
277 goto free_blk_ws_pool
;
279 pblk
->r_rq_pool
= mempool_create_slab_pool(64, pblk_r_rq_cache
);
280 if (!pblk
->r_rq_pool
)
283 pblk
->w_rq_pool
= mempool_create_slab_pool(64, pblk_w_rq_cache
);
284 if (!pblk
->w_rq_pool
)
287 pblk
->line_meta_pool
=
288 mempool_create_slab_pool(16, pblk_line_meta_cache
);
289 if (!pblk
->line_meta_pool
)
292 pblk
->kw_wq
= alloc_workqueue("pblk-aux-wq",
293 WQ_MEM_RECLAIM
| WQ_UNBOUND
, 1);
295 goto free_line_meta_pool
;
297 if (pblk_set_ppaf(pblk
))
300 if (pblk_rwb_init(pblk
))
303 INIT_LIST_HEAD(&pblk
->compl_list
);
307 destroy_workqueue(pblk
->kw_wq
);
309 mempool_destroy(pblk
->line_meta_pool
);
311 mempool_destroy(pblk
->w_rq_pool
);
313 mempool_destroy(pblk
->r_rq_pool
);
315 mempool_destroy(pblk
->rec_pool
);
317 mempool_destroy(pblk
->line_ws_pool
);
319 mempool_destroy(pblk
->page_pool
);
323 static void pblk_core_free(struct pblk
*pblk
)
326 destroy_workqueue(pblk
->kw_wq
);
328 mempool_destroy(pblk
->page_pool
);
329 mempool_destroy(pblk
->line_ws_pool
);
330 mempool_destroy(pblk
->rec_pool
);
331 mempool_destroy(pblk
->r_rq_pool
);
332 mempool_destroy(pblk
->w_rq_pool
);
333 mempool_destroy(pblk
->line_meta_pool
);
335 kmem_cache_destroy(pblk_blk_ws_cache
);
336 kmem_cache_destroy(pblk_rec_cache
);
337 kmem_cache_destroy(pblk_r_rq_cache
);
338 kmem_cache_destroy(pblk_w_rq_cache
);
339 kmem_cache_destroy(pblk_line_meta_cache
);
342 static void pblk_luns_free(struct pblk
*pblk
)
347 static void pblk_lines_free(struct pblk
*pblk
)
349 struct pblk_line_mgmt
*l_mg
= &pblk
->l_mg
;
350 struct pblk_line
*line
;
353 spin_lock(&l_mg
->free_lock
);
354 for (i
= 0; i
< l_mg
->nr_lines
; i
++) {
355 line
= &pblk
->lines
[i
];
357 pblk_line_free(pblk
, line
);
358 kfree(line
->blk_bitmap
);
359 kfree(line
->erase_bitmap
);
361 spin_unlock(&l_mg
->free_lock
);
364 static void pblk_line_meta_free(struct pblk
*pblk
)
366 struct pblk_line_mgmt
*l_mg
= &pblk
->l_mg
;
369 kfree(l_mg
->bb_template
);
372 for (i
= 0; i
< PBLK_DATA_LINES
; i
++) {
373 pblk_mfree(l_mg
->sline_meta
[i
].meta
, l_mg
->smeta_alloc_type
);
374 pblk_mfree(l_mg
->eline_meta
[i
].meta
, l_mg
->emeta_alloc_type
);
380 static int pblk_bb_discovery(struct nvm_tgt_dev
*dev
, struct pblk_lun
*rlun
)
382 struct nvm_geo
*geo
= &dev
->geo
;
387 nr_blks
= geo
->blks_per_lun
* geo
->plane_mode
;
388 blks
= kmalloc(nr_blks
, GFP_KERNEL
);
393 ppa
.g
.ch
= rlun
->bppa
.g
.ch
;
394 ppa
.g
.lun
= rlun
->bppa
.g
.lun
;
396 ret
= nvm_get_tgt_bb_tbl(dev
, ppa
, blks
);
400 nr_blks
= nvm_bb_tbl_fold(dev
->parent
, blks
, nr_blks
);
406 rlun
->bb_list
= blks
;
414 static int pblk_bb_line(struct pblk
*pblk
, struct pblk_line
*line
)
416 struct pblk_line_meta
*lm
= &pblk
->lm
;
417 struct pblk_lun
*rlun
;
421 line
->blk_bitmap
= kzalloc(lm
->blk_bitmap_len
, GFP_KERNEL
);
422 if (!line
->blk_bitmap
)
425 line
->erase_bitmap
= kzalloc(lm
->blk_bitmap_len
, GFP_KERNEL
);
426 if (!line
->erase_bitmap
) {
427 kfree(line
->blk_bitmap
);
431 for (i
= 0; i
< lm
->blk_per_line
; i
++) {
432 rlun
= &pblk
->luns
[i
];
433 if (rlun
->bb_list
[line
->id
] == NVM_BLK_T_FREE
)
436 set_bit(i
, line
->blk_bitmap
);
443 static int pblk_luns_init(struct pblk
*pblk
, struct ppa_addr
*luns
)
445 struct nvm_tgt_dev
*dev
= pblk
->dev
;
446 struct nvm_geo
*geo
= &dev
->geo
;
447 struct pblk_lun
*rlun
;
450 /* TODO: Implement unbalanced LUN support */
451 if (geo
->luns_per_chnl
< 0) {
452 pr_err("pblk: unbalanced LUN config.\n");
456 pblk
->luns
= kcalloc(geo
->nr_luns
, sizeof(struct pblk_lun
), GFP_KERNEL
);
460 for (i
= 0; i
< geo
->nr_luns
; i
++) {
461 /* Stripe across channels */
462 int ch
= i
% geo
->nr_chnls
;
463 int lun_raw
= i
/ geo
->nr_chnls
;
464 int lunid
= lun_raw
+ ch
* geo
->luns_per_chnl
;
466 rlun
= &pblk
->luns
[i
];
467 rlun
->bppa
= luns
[lunid
];
469 sema_init(&rlun
->wr_sem
, 1);
471 ret
= pblk_bb_discovery(dev
, rlun
);
474 kfree(pblk
->luns
[i
].bb_list
);
482 static int pblk_lines_configure(struct pblk
*pblk
, int flags
)
484 struct pblk_line
*line
= NULL
;
487 if (!(flags
& NVM_TARGET_FACTORY
)) {
488 line
= pblk_recov_l2p(pblk
);
490 pr_err("pblk: could not recover l2p table\n");
496 /* Configure next line for user data */
497 line
= pblk_line_get_first_data(pblk
);
499 pr_err("pblk: line list corrupted\n");
507 /* See comment over struct line_emeta definition */
508 static unsigned int calc_emeta_len(struct pblk
*pblk
, struct pblk_line_meta
*lm
)
510 return (sizeof(struct line_emeta
) +
511 ((lm
->sec_per_line
- lm
->emeta_sec
) * sizeof(u64
)) +
512 (pblk
->l_mg
.nr_lines
* sizeof(u32
)) +
516 static void pblk_set_provision(struct pblk
*pblk
, long nr_free_blks
)
518 struct nvm_tgt_dev
*dev
= pblk
->dev
;
519 struct nvm_geo
*geo
= &dev
->geo
;
520 sector_t provisioned
;
524 provisioned
= nr_free_blks
;
525 provisioned
*= (100 - pblk
->over_pct
);
526 sector_div(provisioned
, 100);
528 /* Internally pblk manages all free blocks, but all calculations based
529 * on user capacity consider only provisioned blocks
531 pblk
->rl
.total_blocks
= nr_free_blks
;
532 pblk
->rl
.nr_secs
= nr_free_blks
* geo
->sec_per_blk
;
533 pblk
->capacity
= provisioned
* geo
->sec_per_blk
;
534 atomic_set(&pblk
->rl
.free_blocks
, nr_free_blks
);
537 static int pblk_lines_init(struct pblk
*pblk
)
539 struct nvm_tgt_dev
*dev
= pblk
->dev
;
540 struct nvm_geo
*geo
= &dev
->geo
;
541 struct pblk_line_mgmt
*l_mg
= &pblk
->l_mg
;
542 struct pblk_line_meta
*lm
= &pblk
->lm
;
543 struct pblk_line
*line
;
544 unsigned int smeta_len
, emeta_len
;
545 long nr_bad_blks
, nr_meta_blks
, nr_free_blks
;
550 lm
->sec_per_line
= geo
->sec_per_blk
* geo
->nr_luns
;
551 lm
->blk_per_line
= geo
->nr_luns
;
552 lm
->blk_bitmap_len
= BITS_TO_LONGS(geo
->nr_luns
) * sizeof(long);
553 lm
->sec_bitmap_len
= BITS_TO_LONGS(lm
->sec_per_line
) * sizeof(long);
554 lm
->lun_bitmap_len
= BITS_TO_LONGS(geo
->nr_luns
) * sizeof(long);
555 lm
->high_thrs
= lm
->sec_per_line
/ 2;
556 lm
->mid_thrs
= lm
->sec_per_line
/ 4;
558 /* Calculate necessary pages for smeta. See comment over struct
559 * line_smeta definition
561 lm
->smeta_len
= sizeof(struct line_smeta
) +
562 PBLK_LINE_NR_LUN_BITMAP
* lm
->lun_bitmap_len
;
566 lm
->smeta_sec
= i
* geo
->sec_per_pl
;
567 lm
->smeta_len
= lm
->smeta_sec
* geo
->sec_size
;
569 smeta_len
= sizeof(struct line_smeta
) +
570 PBLK_LINE_NR_LUN_BITMAP
* lm
->lun_bitmap_len
;
571 if (smeta_len
> lm
->smeta_len
) {
576 /* Calculate necessary pages for emeta. See comment over struct
577 * line_emeta definition
581 lm
->emeta_sec
= i
* geo
->sec_per_pl
;
582 lm
->emeta_len
= lm
->emeta_sec
* geo
->sec_size
;
584 emeta_len
= calc_emeta_len(pblk
, lm
);
585 if (emeta_len
> lm
->emeta_len
) {
589 lm
->emeta_bb
= geo
->nr_luns
- i
;
591 nr_meta_blks
= (lm
->smeta_sec
+ lm
->emeta_sec
+
592 (geo
->sec_per_blk
/ 2)) / geo
->sec_per_blk
;
593 lm
->min_blk_line
= nr_meta_blks
+ 1;
595 l_mg
->nr_lines
= geo
->blks_per_lun
;
596 l_mg
->log_line
= l_mg
->data_line
= NULL
;
597 l_mg
->l_seq_nr
= l_mg
->d_seq_nr
= 0;
598 l_mg
->nr_free_lines
= 0;
599 bitmap_zero(&l_mg
->meta_bitmap
, PBLK_DATA_LINES
);
601 /* smeta is always small enough to fit on a kmalloc memory allocation,
602 * emeta depends on the number of LUNs allocated to the pblk instance
604 l_mg
->smeta_alloc_type
= PBLK_KMALLOC_META
;
605 for (i
= 0; i
< PBLK_DATA_LINES
; i
++) {
606 l_mg
->sline_meta
[i
].meta
= kmalloc(lm
->smeta_len
, GFP_KERNEL
);
607 if (!l_mg
->sline_meta
[i
].meta
)
609 kfree(l_mg
->sline_meta
[i
].meta
);
615 if (lm
->emeta_len
> KMALLOC_MAX_CACHE_SIZE
) {
616 l_mg
->emeta_alloc_type
= PBLK_VMALLOC_META
;
618 for (i
= 0; i
< PBLK_DATA_LINES
; i
++) {
619 l_mg
->eline_meta
[i
].meta
= vmalloc(lm
->emeta_len
);
620 if (!l_mg
->eline_meta
[i
].meta
)
622 vfree(l_mg
->eline_meta
[i
].meta
);
628 l_mg
->emeta_alloc_type
= PBLK_KMALLOC_META
;
630 for (i
= 0; i
< PBLK_DATA_LINES
; i
++) {
631 l_mg
->eline_meta
[i
].meta
=
632 kmalloc(lm
->emeta_len
, GFP_KERNEL
);
633 if (!l_mg
->eline_meta
[i
].meta
)
635 kfree(l_mg
->eline_meta
[i
].meta
);
642 l_mg
->bb_template
= kzalloc(lm
->sec_bitmap_len
, GFP_KERNEL
);
643 if (!l_mg
->bb_template
) {
648 l_mg
->bb_aux
= kzalloc(lm
->sec_bitmap_len
, GFP_KERNEL
);
651 goto fail_free_bb_template
;
654 bb_distance
= (geo
->nr_luns
) * geo
->sec_per_pl
;
655 for (i
= 0; i
< lm
->sec_per_line
; i
+= bb_distance
)
656 bitmap_set(l_mg
->bb_template
, i
, geo
->sec_per_pl
);
658 INIT_LIST_HEAD(&l_mg
->free_list
);
659 INIT_LIST_HEAD(&l_mg
->corrupt_list
);
660 INIT_LIST_HEAD(&l_mg
->bad_list
);
661 INIT_LIST_HEAD(&l_mg
->gc_full_list
);
662 INIT_LIST_HEAD(&l_mg
->gc_high_list
);
663 INIT_LIST_HEAD(&l_mg
->gc_mid_list
);
664 INIT_LIST_HEAD(&l_mg
->gc_low_list
);
665 INIT_LIST_HEAD(&l_mg
->gc_empty_list
);
667 l_mg
->gc_lists
[0] = &l_mg
->gc_high_list
;
668 l_mg
->gc_lists
[1] = &l_mg
->gc_mid_list
;
669 l_mg
->gc_lists
[2] = &l_mg
->gc_low_list
;
671 spin_lock_init(&l_mg
->free_lock
);
672 spin_lock_init(&l_mg
->gc_lock
);
674 pblk
->lines
= kcalloc(l_mg
->nr_lines
, sizeof(struct pblk_line
),
678 goto fail_free_bb_aux
;
682 for (i
= 0; i
< l_mg
->nr_lines
; i
++) {
685 line
= &pblk
->lines
[i
];
689 line
->type
= PBLK_LINETYPE_FREE
;
690 line
->state
= PBLK_LINESTATE_FREE
;
691 line
->gc_group
= PBLK_LINEGC_NONE
;
692 spin_lock_init(&line
->lock
);
694 nr_bad_blks
= pblk_bb_line(pblk
, line
);
695 if (nr_bad_blks
< 0 || nr_bad_blks
> lm
->blk_per_line
) {
697 goto fail_free_lines
;
700 blk_in_line
= lm
->blk_per_line
- nr_bad_blks
;
701 if (blk_in_line
< lm
->min_blk_line
) {
702 line
->state
= PBLK_LINESTATE_BAD
;
703 list_add_tail(&line
->list
, &l_mg
->bad_list
);
707 nr_free_blks
+= blk_in_line
;
708 atomic_set(&line
->blk_in_line
, blk_in_line
);
710 l_mg
->nr_free_lines
++;
711 list_add_tail(&line
->list
, &l_mg
->free_list
);
714 pblk_set_provision(pblk
, nr_free_blks
);
716 sema_init(&pblk
->erase_sem
, 1);
718 /* Cleanup per-LUN bad block lists - managed within lines on run-time */
719 for (i
= 0; i
< geo
->nr_luns
; i
++)
720 kfree(pblk
->luns
[i
].bb_list
);
727 fail_free_bb_template
:
728 kfree(l_mg
->bb_template
);
730 for (i
= 0; i
< PBLK_DATA_LINES
; i
++) {
731 pblk_mfree(l_mg
->sline_meta
[i
].meta
, l_mg
->smeta_alloc_type
);
732 pblk_mfree(l_mg
->eline_meta
[i
].meta
, l_mg
->emeta_alloc_type
);
735 for (i
= 0; i
< geo
->nr_luns
; i
++)
736 kfree(pblk
->luns
[i
].bb_list
);
741 static int pblk_writer_init(struct pblk
*pblk
)
743 setup_timer(&pblk
->wtimer
, pblk_write_timer_fn
, (unsigned long)pblk
);
744 mod_timer(&pblk
->wtimer
, jiffies
+ msecs_to_jiffies(100));
746 pblk
->writer_ts
= kthread_create(pblk_write_ts
, pblk
, "pblk-writer-t");
747 if (IS_ERR(pblk
->writer_ts
)) {
748 pr_err("pblk: could not allocate writer kthread\n");
749 return PTR_ERR(pblk
->writer_ts
);
755 static void pblk_writer_stop(struct pblk
*pblk
)
758 kthread_stop(pblk
->writer_ts
);
759 del_timer(&pblk
->wtimer
);
762 static void pblk_free(struct pblk
*pblk
)
764 pblk_luns_free(pblk
);
765 pblk_lines_free(pblk
);
766 pblk_line_meta_free(pblk
);
767 pblk_core_free(pblk
);
773 static void pblk_tear_down(struct pblk
*pblk
)
775 pblk_flush_writer(pblk
);
776 pblk_writer_stop(pblk
);
777 pblk_rb_sync_l2p(&pblk
->rwb
);
778 pblk_recov_pad(pblk
);
780 pblk_rl_free(&pblk
->rl
);
782 pr_debug("pblk: consistent tear down\n");
785 static void pblk_exit(void *private)
787 struct pblk
*pblk
= private;
789 down_write(&pblk_lock
);
791 pblk_tear_down(pblk
);
793 up_write(&pblk_lock
);
796 static sector_t
pblk_capacity(void *private)
798 struct pblk
*pblk
= private;
800 return pblk
->capacity
* NR_PHY_IN_LOG
;
803 static void *pblk_init(struct nvm_tgt_dev
*dev
, struct gendisk
*tdisk
,
806 struct nvm_geo
*geo
= &dev
->geo
;
807 struct request_queue
*bqueue
= dev
->q
;
808 struct request_queue
*tqueue
= tdisk
->queue
;
812 if (dev
->identity
.dom
& NVM_RSP_L2P
) {
813 pr_err("pblk: device-side L2P table not supported. (%x)\n",
815 return ERR_PTR(-EINVAL
);
818 pblk
= kzalloc(sizeof(struct pblk
), GFP_KERNEL
);
820 return ERR_PTR(-ENOMEM
);
825 spin_lock_init(&pblk
->trans_lock
);
826 spin_lock_init(&pblk
->lock
);
828 if (flags
& NVM_TARGET_FACTORY
)
829 pblk_setup_uuid(pblk
);
831 #ifdef CONFIG_NVM_DEBUG
832 atomic_long_set(&pblk
->inflight_writes
, 0);
833 atomic_long_set(&pblk
->padded_writes
, 0);
834 atomic_long_set(&pblk
->padded_wb
, 0);
835 atomic_long_set(&pblk
->nr_flush
, 0);
836 atomic_long_set(&pblk
->req_writes
, 0);
837 atomic_long_set(&pblk
->sub_writes
, 0);
838 atomic_long_set(&pblk
->sync_writes
, 0);
839 atomic_long_set(&pblk
->compl_writes
, 0);
840 atomic_long_set(&pblk
->inflight_reads
, 0);
841 atomic_long_set(&pblk
->sync_reads
, 0);
842 atomic_long_set(&pblk
->recov_writes
, 0);
843 atomic_long_set(&pblk
->recov_writes
, 0);
844 atomic_long_set(&pblk
->recov_gc_writes
, 0);
847 atomic_long_set(&pblk
->read_failed
, 0);
848 atomic_long_set(&pblk
->read_empty
, 0);
849 atomic_long_set(&pblk
->read_high_ecc
, 0);
850 atomic_long_set(&pblk
->read_failed_gc
, 0);
851 atomic_long_set(&pblk
->write_failed
, 0);
852 atomic_long_set(&pblk
->erase_failed
, 0);
854 ret
= pblk_luns_init(pblk
, dev
->luns
);
856 pr_err("pblk: could not initialize luns\n");
860 ret
= pblk_lines_init(pblk
);
862 pr_err("pblk: could not initialize lines\n");
866 ret
= pblk_core_init(pblk
);
868 pr_err("pblk: could not initialize core\n");
869 goto fail_free_line_meta
;
872 ret
= pblk_l2p_init(pblk
);
874 pr_err("pblk: could not initialize maps\n");
878 ret
= pblk_lines_configure(pblk
, flags
);
880 pr_err("pblk: could not configure lines\n");
884 ret
= pblk_writer_init(pblk
);
886 pr_err("pblk: could not initialize write thread\n");
887 goto fail_free_lines
;
890 ret
= pblk_gc_init(pblk
);
892 pr_err("pblk: could not initialize gc\n");
893 goto fail_stop_writer
;
896 /* inherit the size from the underlying device */
897 blk_queue_logical_block_size(tqueue
, queue_physical_block_size(bqueue
));
898 blk_queue_max_hw_sectors(tqueue
, queue_max_hw_sectors(bqueue
));
900 blk_queue_write_cache(tqueue
, true, false);
902 tqueue
->limits
.discard_granularity
= geo
->pgs_per_blk
* geo
->pfpg_size
;
903 tqueue
->limits
.discard_alignment
= 0;
904 blk_queue_max_discard_sectors(tqueue
, UINT_MAX
>> 9);
905 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD
, tqueue
);
907 pr_info("pblk init: luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
908 geo
->nr_luns
, pblk
->l_mg
.nr_lines
,
909 (unsigned long long)pblk
->rl
.nr_secs
,
910 pblk
->rwb
.nr_entries
);
912 wake_up_process(pblk
->writer_ts
);
916 pblk_writer_stop(pblk
);
918 pblk_lines_free(pblk
);
922 pblk_core_free(pblk
);
924 pblk_line_meta_free(pblk
);
926 pblk_luns_free(pblk
);
932 /* physical block device target */
933 static struct nvm_tgt_type tt_pblk
= {
935 .version
= {1, 0, 0},
937 .make_rq
= pblk_make_rq
,
938 .capacity
= pblk_capacity
,
943 .sysfs_init
= pblk_sysfs_init
,
944 .sysfs_exit
= pblk_sysfs_exit
,
947 static int __init
pblk_module_init(void)
949 return nvm_register_tgt_type(&tt_pblk
);
952 static void pblk_module_exit(void)
954 nvm_unregister_tgt_type(&tt_pblk
);
957 module_init(pblk_module_init
);
958 module_exit(pblk_module_exit
);
959 MODULE_AUTHOR("Javier Gonzalez <javier@cnexlabs.com>");
960 MODULE_AUTHOR("Matias Bjorling <matias@cnexlabs.com>");
961 MODULE_LICENSE("GPL v2");
962 MODULE_DESCRIPTION("Physical Block-Device for Open-Channel SSDs");