2 * Copyright (C) 2015 IT University of Copenhagen (rrpc.h)
3 * Copyright (C) 2016 CNEX Labs
4 * Initial release: Matias Bjorling <matias@cnexlabs.com>
5 * Write buffering: Javier Gonzalez <javier@cnexlabs.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * Implementation of a Physical Block-device target for Open-channel SSDs.
23 #include <linux/blkdev.h>
24 #include <linux/blk-mq.h>
25 #include <linux/bio.h>
26 #include <linux/module.h>
27 #include <linux/kthread.h>
28 #include <linux/vmalloc.h>
29 #include <linux/crc32.h>
30 #include <linux/uuid.h>
32 #include <linux/lightnvm.h>
34 /* Run only GC if less than 1/X blocks are free */
35 #define GC_LIMIT_INVERSE 5
36 #define GC_TIME_MSECS 1000
38 #define PBLK_SECTOR (512)
39 #define PBLK_EXPOSED_PAGE_SIZE (4096)
40 #define PBLK_MAX_REQ_ADDRS (64)
41 #define PBLK_MAX_REQ_ADDRS_PW (6)
43 #define PBLK_NR_CLOSE_JOBS (4)
45 #define PBLK_CACHE_NAME_LEN (DISK_NAME_LEN + 16)
47 #define PBLK_COMMAND_TIMEOUT_MS 30000
49 /* Max 512 LUNs per device */
50 #define PBLK_MAX_LUNS_BITMAP (4)
52 #define NR_PHY_IN_LOG (PBLK_EXPOSED_PAGE_SIZE / PBLK_SECTOR)
54 /* Static pool sizes */
55 #define PBLK_GEN_WS_POOL_SIZE (2)
57 #define PBLK_DEFAULT_OP (11)
61 PBLK_WRITE
= WRITE
,/* Write from write buffer */
62 PBLK_WRITE_INT
, /* Internal write - no write buffer */
63 PBLK_READ_RECOV
, /* Recovery read - errors allowed */
69 PBLK_IOTYPE_USER
= 1 << 0,
70 PBLK_IOTYPE_GC
= 1 << 1,
72 /* Write buffer flags */
73 PBLK_FLUSH_ENTRY
= 1 << 2,
74 PBLK_WRITTEN_DATA
= 1 << 3,
75 PBLK_SUBMITTED_ENTRY
= 1 << 4,
76 PBLK_WRITABLE_ENTRY
= 1 << 5,
80 PBLK_BLK_ST_OPEN
= 0x1,
81 PBLK_BLK_ST_CLOSED
= 0x2,
84 struct pblk_sec_meta
{
89 /* The number of GC lists and the rate-limiter states go together. This way the
90 * rate-limiter can dictate how much GC is needed based on resource utilization.
92 #define PBLK_GC_NR_LISTS 3
100 #define pblk_dma_meta_size (sizeof(struct pblk_sec_meta) * PBLK_MAX_REQ_ADDRS)
101 #define pblk_dma_ppa_size (sizeof(u64) * PBLK_MAX_REQ_ADDRS)
103 /* write buffer completion context */
105 struct list_head list
; /* Head for out-of-order completion */
107 unsigned long *lun_bitmap
; /* Luns used on current request */
109 unsigned int nr_valid
;
110 unsigned int nr_padded
;
116 unsigned long start_time
;
123 struct completion wait
;
127 /* Recovery context */
128 struct pblk_rec_ctx
{
131 struct list_head failed
;
132 struct work_struct ws_rec
;
137 struct bio_list bios
; /* Original bios - used for completion
138 * in REQ_FUA, REQ_FLUSH case
140 u64 lba
; /* Logic addr. associated with entry */
141 struct ppa_addr ppa
; /* Physic addr. associated with entry */
142 int flags
; /* Write context flags */
145 struct pblk_rb_entry
{
146 struct ppa_addr cacheline
; /* Cacheline for this entry */
147 void *data
; /* Pointer to data on this entry */
148 struct pblk_w_ctx w_ctx
; /* Context for this entry */
149 struct list_head index
; /* List head to enable indexes */
152 #define EMPTY_ENTRY (~0U)
154 struct pblk_rb_pages
{
157 struct list_head list
;
161 struct pblk_rb_entry
*entries
; /* Ring buffer entries */
162 unsigned int mem
; /* Write offset - points to next
163 * writable entry in memory
165 unsigned int subm
; /* Read offset - points to last entry
166 * that has been submitted to the media
169 unsigned int sync
; /* Synced - backpointer that signals
170 * the last submitted entry that has
171 * been successfully persisted to media
173 unsigned int flush_point
; /* Sync point - last entry that must be
174 * flushed to the media. Used with
175 * REQ_FLUSH and REQ_FUA
177 unsigned int l2p_update
; /* l2p update point - next entry for
178 * which l2p mapping will be updated to
179 * contain a device ppa address (instead
182 unsigned int nr_entries
; /* Number of entries in write buffer -
183 * must be a power of two
185 unsigned int seg_size
; /* Size of the data segments being
186 * stored on each entry. Typically this
190 struct list_head pages
; /* List of data pages */
192 spinlock_t w_lock
; /* Write lock */
193 spinlock_t s_lock
; /* Sync lock */
195 #ifdef CONFIG_NVM_DEBUG
196 atomic_t inflight_flush_point
; /* Not served REQ_FLUSH | REQ_FUA */
200 #define PBLK_RECOVERY_SECTORS 16
203 struct ppa_addr bppa
;
204 struct semaphore wr_sem
;
208 struct pblk_line
*line
;
210 u64 paddr_list
[PBLK_MAX_REQ_ADDRS
];
211 u64 lba_list
[PBLK_MAX_REQ_ADDRS
];
214 struct list_head list
;
218 /* These states are not protected by a lock since (i) they are in the
219 * fast path, and (ii) they are not critical.
225 struct task_struct
*gc_ts
;
226 struct task_struct
*gc_writer_ts
;
227 struct task_struct
*gc_reader_ts
;
229 struct workqueue_struct
*gc_line_reader_wq
;
230 struct workqueue_struct
*gc_reader_wq
;
232 struct timer_list gc_timer
;
234 struct semaphore gc_sem
;
235 atomic_t read_inflight_gc
; /* Number of lines with inflight GC reads */
236 atomic_t pipeline_gc
; /* Number of lines in the GC pipeline -
237 * started reads to finished writes
241 struct list_head w_list
;
242 struct list_head r_list
;
250 unsigned int high
; /* Upper threshold for rate limiter (free run -
251 * user I/O rate limiter
253 unsigned int high_pw
; /* High rounded up as a power of 2 */
255 #define PBLK_USER_HIGH_THRS 8 /* Begin write limit at 12% available blks */
256 #define PBLK_USER_LOW_THRS 10 /* Aggressive GC at 10% available blocks */
258 int rb_windows_pw
; /* Number of rate windows in the write buffer
259 * given as a power-of-2. This guarantees that
260 * when user I/O is being rate limited, there
261 * will be reserved enough space for the GC to
262 * place its payload. A window is of
263 * pblk->max_write_pgs size, which in NVMe is
266 int rb_budget
; /* Total number of entries available for I/O */
267 int rb_user_max
; /* Max buffer entries available for user I/O */
268 int rb_gc_max
; /* Max buffer entries available for GC I/O */
269 int rb_gc_rsv
; /* Reserved buffer entries for GC I/O */
270 int rb_state
; /* Rate-limiter current state */
271 int rb_max_io
; /* Maximum size for an I/O giving the config */
273 atomic_t rb_user_cnt
; /* User I/O buffer counter */
274 atomic_t rb_gc_cnt
; /* GC I/O buffer counter */
275 atomic_t rb_space
; /* Space limit in case of reaching capacity */
277 int rsv_blocks
; /* Reserved blocks for GC */
282 struct timer_list u_timer
;
284 unsigned long long nr_secs
;
285 unsigned long total_blocks
;
287 atomic_t free_blocks
; /* Total number of free blocks (+ OP) */
288 atomic_t free_user_blocks
; /* Number of user free blocks (no OP) */
291 #define PBLK_LINE_EMPTY (~0U)
295 PBLK_LINETYPE_FREE
= 0,
296 PBLK_LINETYPE_LOG
= 1,
297 PBLK_LINETYPE_DATA
= 2,
300 PBLK_LINESTATE_NEW
= 9,
301 PBLK_LINESTATE_FREE
= 10,
302 PBLK_LINESTATE_OPEN
= 11,
303 PBLK_LINESTATE_CLOSED
= 12,
304 PBLK_LINESTATE_GC
= 13,
305 PBLK_LINESTATE_BAD
= 14,
306 PBLK_LINESTATE_CORRUPT
= 15,
309 PBLK_LINEGC_NONE
= 20,
310 PBLK_LINEGC_EMPTY
= 21,
311 PBLK_LINEGC_LOW
= 22,
312 PBLK_LINEGC_MID
= 23,
313 PBLK_LINEGC_HIGH
= 24,
314 PBLK_LINEGC_FULL
= 25,
317 #define PBLK_MAGIC 0x70626c6b /*pblk*/
319 /* emeta/smeta persistent storage format versions:
320 * Changes in major version requires offline migration.
321 * Changes in minor version are handled automatically during
325 #define SMETA_VERSION_MAJOR (0)
326 #define SMETA_VERSION_MINOR (1)
328 #define EMETA_VERSION_MAJOR (0)
329 #define EMETA_VERSION_MINOR (2)
333 __le32 identifier
; /* pblk identifier */
334 __u8 uuid
[16]; /* instance uuid */
335 __le16 type
; /* line type */
336 __u8 version_major
; /* version major */
337 __u8 version_minor
; /* version minor */
338 __le32 id
; /* line id for current line */
342 struct line_header header
;
344 __le32 crc
; /* Full structure including struct crc */
345 /* Previous line metadata */
346 __le32 prev_id
; /* Line id for previous line */
348 /* Current line metadata */
349 __le64 seq_nr
; /* Sequence number for current line */
352 __le32 window_wr_lun
; /* Number of parallel LUNs to write */
361 * Metadata layout in media:
363 * 1. struct line_emeta
364 * 2. bad block bitmap (u64 * window_wr_lun)
365 * 3. write amplification counters
366 * Mid sectors (start at lbas_sector):
367 * 3. nr_lbas (u64) forming lba list
368 * Last sectors (start at vsc_sector):
369 * 4. u32 valid sector count (vsc) for all lines (~0U: free line)
372 struct line_header header
;
374 __le32 crc
; /* Full structure including struct crc */
376 /* Previous line metadata */
377 __le32 prev_id
; /* Line id for prev line */
379 /* Current line metadata */
380 __le64 seq_nr
; /* Sequence number for current line */
383 __le32 window_wr_lun
; /* Number of parallel LUNs to write */
385 /* Bookkeeping for recovery */
386 __le32 next_id
; /* Line id for next line */
387 __le64 nr_lbas
; /* Number of lbas mapped in line */
388 __le64 nr_valid_lbas
; /* Number of valid lbas mapped in line */
389 __le64 bb_bitmap
[]; /* Updated bad block bitmap for line */
393 /* Write amplification counters stored on media */
395 __le64 user
; /* Number of user written sectors */
396 __le64 gc
; /* Number of sectors written by GC*/
397 __le64 pad
; /* Number of padded sectors */
401 struct line_emeta
*buf
; /* emeta buffer in media format */
402 int mem
; /* Write offset - points to next
403 * writable entry in memory
405 atomic_t sync
; /* Synced - backpointer that signals the
406 * last entry that has been successfully
409 unsigned int nr_entries
; /* Number of emeta entries */
413 struct line_smeta
*buf
; /* smeta buffer in persistent format */
418 unsigned int id
; /* Line number corresponds to the
421 unsigned int seq_nr
; /* Unique line sequence number */
423 int state
; /* PBLK_LINESTATE_X */
424 int type
; /* PBLK_LINETYPE_X */
425 int gc_group
; /* PBLK_LINEGC_X */
426 struct list_head list
; /* Free, GC lists */
428 unsigned long *lun_bitmap
; /* Bitmap for LUNs mapped in line */
430 struct nvm_chk_meta
*chks
; /* Chunks forming line */
432 struct pblk_smeta
*smeta
; /* Start metadata */
433 struct pblk_emeta
*emeta
; /* End medatada */
435 int meta_line
; /* Metadata line id */
436 int meta_distance
; /* Distance between data and metadata */
438 u64 smeta_ssec
; /* Sector where smeta starts */
439 u64 emeta_ssec
; /* Sector where emeta starts */
441 unsigned int sec_in_line
; /* Number of usable secs in line */
443 atomic_t blk_in_line
; /* Number of good blocks in line */
444 unsigned long *blk_bitmap
; /* Bitmap for valid/invalid blocks */
445 unsigned long *erase_bitmap
; /* Bitmap for erased blocks */
447 unsigned long *map_bitmap
; /* Bitmap for mapped sectors in line */
448 unsigned long *invalid_bitmap
; /* Bitmap for invalid sectors in line */
450 atomic_t left_eblks
; /* Blocks left for erasing */
451 atomic_t left_seblks
; /* Blocks left for sync erasing */
453 int left_msecs
; /* Sectors left for mapping */
454 unsigned int cur_sec
; /* Sector map pointer */
455 unsigned int nr_valid_lbas
; /* Number of valid lbas in line */
457 __le32
*vsc
; /* Valid sector count in line */
459 struct kref ref
; /* Write buffer L2P references */
461 spinlock_t lock
; /* Necessary for invalid_bitmap only */
464 #define PBLK_DATA_LINES 4
467 PBLK_KMALLOC_META
= 1,
468 PBLK_VMALLOC_META
= 2,
472 PBLK_EMETA_TYPE_HEADER
= 1, /* struct line_emeta first sector */
473 PBLK_EMETA_TYPE_LLBA
= 2, /* lba list - type: __le64 */
474 PBLK_EMETA_TYPE_VSC
= 3, /* vsc list - type: __le32 */
477 struct pblk_line_mgmt
{
478 int nr_lines
; /* Total number of full lines */
479 int nr_free_lines
; /* Number of full lines in free list */
481 /* Free lists - use free_lock */
482 struct list_head free_list
; /* Full lines ready to use */
483 struct list_head corrupt_list
; /* Full lines corrupted */
484 struct list_head bad_list
; /* Full lines bad */
486 /* GC lists - use gc_lock */
487 struct list_head
*gc_lists
[PBLK_GC_NR_LISTS
];
488 struct list_head gc_high_list
; /* Full lines ready to GC, high isc */
489 struct list_head gc_mid_list
; /* Full lines ready to GC, mid isc */
490 struct list_head gc_low_list
; /* Full lines ready to GC, low isc */
492 struct list_head gc_full_list
; /* Full lines ready to GC, no valid */
493 struct list_head gc_empty_list
; /* Full lines close, all valid */
495 struct pblk_line
*log_line
; /* Current FTL log line */
496 struct pblk_line
*data_line
; /* Current data line */
497 struct pblk_line
*log_next
; /* Next FTL log line */
498 struct pblk_line
*data_next
; /* Next data line */
500 struct list_head emeta_list
; /* Lines queued to schedule emeta */
502 __le32
*vsc_list
; /* Valid sector counts for all lines */
504 /* Metadata allocation type: VMALLOC | KMALLOC */
505 int emeta_alloc_type
;
507 /* Pre-allocated metadata for data lines */
508 struct pblk_smeta
*sline_meta
[PBLK_DATA_LINES
];
509 struct pblk_emeta
*eline_meta
[PBLK_DATA_LINES
];
510 unsigned long meta_bitmap
;
512 /* Helpers for fast bitmap calculations */
513 unsigned long *bb_template
;
514 unsigned long *bb_aux
;
516 unsigned long d_seq_nr
; /* Data line unique sequence number */
517 unsigned long l_seq_nr
; /* Log line unique sequence number */
519 spinlock_t free_lock
;
520 spinlock_t close_lock
;
524 struct pblk_line_meta
{
525 unsigned int smeta_len
; /* Total length for smeta */
526 unsigned int smeta_sec
; /* Sectors needed for smeta */
528 unsigned int emeta_len
[4]; /* Lengths for emeta:
530 * [1]: struct line_emeta +
531 * bb_bitmap + struct wa_counters
535 unsigned int emeta_sec
[4]; /* Sectors needed for emeta. Same layout
539 unsigned int emeta_bb
; /* Boundary for bb that affects emeta */
541 unsigned int vsc_list_len
; /* Length for vsc list */
542 unsigned int sec_bitmap_len
; /* Length for sector bitmap in line */
543 unsigned int blk_bitmap_len
; /* Length for block bitmap in line */
544 unsigned int lun_bitmap_len
; /* Length for lun bitmap in line */
546 unsigned int blk_per_line
; /* Number of blocks in a full line */
547 unsigned int sec_per_line
; /* Number of sectors in a line */
548 unsigned int dsec_per_line
; /* Number of data sectors in a line */
549 unsigned int min_blk_line
; /* Min. number of good blocks in line */
551 unsigned int mid_thrs
; /* Threshold for GC mid list */
552 unsigned int high_thrs
; /* Threshold for GC high list */
554 unsigned int meta_distance
; /* Distance between data and metadata */
558 PBLK_STATE_RUNNING
= 0,
559 PBLK_STATE_STOPPING
= 1,
560 PBLK_STATE_RECOVERING
= 2,
561 PBLK_STATE_STOPPED
= 3,
564 /* Internal format to support not power-of-2 device formats */
577 struct nvm_tgt_dev
*dev
;
578 struct gendisk
*disk
;
582 struct pblk_lun
*luns
;
584 struct pblk_line
*lines
; /* Line array */
585 struct pblk_line_mgmt l_mg
; /* Line management */
586 struct pblk_line_meta lm
; /* Line metadata */
588 struct nvm_addrf addrf
; /* Aligned address format */
589 struct pblk_addrf uaddrf
; /* Unaligned address format */
594 int state
; /* pblk line state */
596 int min_write_pgs
; /* Minimum amount of pages required by controller */
597 int max_write_pgs
; /* Maximum amount of pages supported by controller */
598 int pgs_in_buffer
; /* Number of pages that need to be held in buffer to
599 * guarantee successful reads.
602 sector_t capacity
; /* Device capacity when bad blocks are subtracted */
604 int op
; /* Percentage of device used for over-provisioning */
605 int op_blks
; /* Number of blocks used for over-provisioning */
607 /* pblk provisioning values. Used by rate limiter */
612 unsigned char instance_uuid
[16];
614 /* Persistent write amplification counters, 4kb sector I/Os */
615 atomic64_t user_wa
; /* Sectors written by user */
616 atomic64_t gc_wa
; /* Sectors written by GC */
617 atomic64_t pad_wa
; /* Padded sectors written */
619 /* Reset values for delta write amplification measurements */
624 /* Counters used for calculating padding distribution */
625 atomic64_t
*pad_dist
; /* Padding distribution buckets */
626 u64 nr_flush_rst
; /* Flushes reset value for pad dist.*/
627 atomic64_t nr_flush
; /* Number of flush/fua I/O */
629 #ifdef CONFIG_NVM_DEBUG
630 /* Non-persistent debug counters, 4kb sector I/Os */
631 atomic_long_t inflight_writes
; /* Inflight writes (user and gc) */
632 atomic_long_t padded_writes
; /* Sectors padded due to flush/fua */
633 atomic_long_t padded_wb
; /* Sectors padded in write buffer */
634 atomic_long_t req_writes
; /* Sectors stored on write buffer */
635 atomic_long_t sub_writes
; /* Sectors submitted from buffer */
636 atomic_long_t sync_writes
; /* Sectors synced to media */
637 atomic_long_t inflight_reads
; /* Inflight sector read requests */
638 atomic_long_t cache_reads
; /* Read requests that hit the cache */
639 atomic_long_t sync_reads
; /* Completed sector read requests */
640 atomic_long_t recov_writes
; /* Sectors submitted from recovery */
641 atomic_long_t recov_gc_writes
; /* Sectors submitted from write GC */
642 atomic_long_t recov_gc_reads
; /* Sectors submitted from read GC */
647 atomic_long_t read_failed
;
648 atomic_long_t read_empty
;
649 atomic_long_t read_high_ecc
;
650 atomic_long_t read_failed_gc
;
651 atomic_long_t write_failed
;
652 atomic_long_t erase_failed
;
654 atomic_t inflight_io
; /* General inflight I/O counter */
656 struct task_struct
*writer_ts
;
658 /* Simple translation map of logical addresses to physical addresses.
659 * The logical addresses is known by the host system, while the physical
660 * addresses are used when writing to the disk block device.
662 unsigned char *trans_map
;
663 spinlock_t trans_lock
;
665 struct list_head compl_list
;
667 mempool_t
*page_bio_pool
;
668 mempool_t
*gen_ws_pool
;
670 mempool_t
*r_rq_pool
;
671 mempool_t
*w_rq_pool
;
672 mempool_t
*e_rq_pool
;
674 struct workqueue_struct
*close_wq
;
675 struct workqueue_struct
*bb_wq
;
676 struct workqueue_struct
*r_end_wq
;
678 struct timer_list wtimer
;
683 struct pblk_line_ws
{
685 struct pblk_line
*line
;
687 struct work_struct ws
;
690 #define pblk_g_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_g_ctx))
691 #define pblk_w_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_c_ctx))
694 * pblk ring buffer operations
696 int pblk_rb_init(struct pblk_rb
*rb
, struct pblk_rb_entry
*rb_entry_base
,
697 unsigned int power_size
, unsigned int power_seg_sz
);
698 unsigned int pblk_rb_calculate_size(unsigned int nr_entries
);
699 void *pblk_rb_entries_ref(struct pblk_rb
*rb
);
700 int pblk_rb_may_write_user(struct pblk_rb
*rb
, struct bio
*bio
,
701 unsigned int nr_entries
, unsigned int *pos
);
702 int pblk_rb_may_write_gc(struct pblk_rb
*rb
, unsigned int nr_entries
,
704 void pblk_rb_write_entry_user(struct pblk_rb
*rb
, void *data
,
705 struct pblk_w_ctx w_ctx
, unsigned int pos
);
706 void pblk_rb_write_entry_gc(struct pblk_rb
*rb
, void *data
,
707 struct pblk_w_ctx w_ctx
, struct pblk_line
*line
,
708 u64 paddr
, unsigned int pos
);
709 struct pblk_w_ctx
*pblk_rb_w_ctx(struct pblk_rb
*rb
, unsigned int pos
);
710 void pblk_rb_flush(struct pblk_rb
*rb
);
712 void pblk_rb_sync_l2p(struct pblk_rb
*rb
);
713 unsigned int pblk_rb_read_to_bio(struct pblk_rb
*rb
, struct nvm_rq
*rqd
,
714 unsigned int pos
, unsigned int nr_entries
,
716 unsigned int pblk_rb_read_to_bio_list(struct pblk_rb
*rb
, struct bio
*bio
,
717 struct list_head
*list
,
719 int pblk_rb_copy_to_bio(struct pblk_rb
*rb
, struct bio
*bio
, sector_t lba
,
720 struct ppa_addr ppa
, int bio_iter
, bool advanced_bio
);
721 unsigned int pblk_rb_read_commit(struct pblk_rb
*rb
, unsigned int entries
);
723 unsigned int pblk_rb_sync_init(struct pblk_rb
*rb
, unsigned long *flags
);
724 unsigned int pblk_rb_sync_advance(struct pblk_rb
*rb
, unsigned int nr_entries
);
725 struct pblk_rb_entry
*pblk_rb_sync_scan_entry(struct pblk_rb
*rb
,
726 struct ppa_addr
*ppa
);
727 void pblk_rb_sync_end(struct pblk_rb
*rb
, unsigned long *flags
);
728 unsigned int pblk_rb_flush_point_count(struct pblk_rb
*rb
);
730 unsigned int pblk_rb_read_count(struct pblk_rb
*rb
);
731 unsigned int pblk_rb_sync_count(struct pblk_rb
*rb
);
732 unsigned int pblk_rb_wrap_pos(struct pblk_rb
*rb
, unsigned int pos
);
734 int pblk_rb_tear_down_check(struct pblk_rb
*rb
);
735 int pblk_rb_pos_oob(struct pblk_rb
*rb
, u64 pos
);
736 void pblk_rb_data_free(struct pblk_rb
*rb
);
737 ssize_t
pblk_rb_sysfs(struct pblk_rb
*rb
, char *buf
);
742 struct nvm_rq
*pblk_alloc_rqd(struct pblk
*pblk
, int type
);
743 void pblk_free_rqd(struct pblk
*pblk
, struct nvm_rq
*rqd
, int type
);
744 void pblk_set_sec_per_write(struct pblk
*pblk
, int sec_per_write
);
745 int pblk_setup_w_rec_rq(struct pblk
*pblk
, struct nvm_rq
*rqd
,
746 struct pblk_c_ctx
*c_ctx
);
747 void pblk_discard(struct pblk
*pblk
, struct bio
*bio
);
748 struct nvm_chk_meta
*pblk_chunk_get_info(struct pblk
*pblk
);
749 struct nvm_chk_meta
*pblk_chunk_get_off(struct pblk
*pblk
,
750 struct nvm_chk_meta
*lp
,
751 struct ppa_addr ppa
);
752 void pblk_log_write_err(struct pblk
*pblk
, struct nvm_rq
*rqd
);
753 void pblk_log_read_err(struct pblk
*pblk
, struct nvm_rq
*rqd
);
754 int pblk_submit_io(struct pblk
*pblk
, struct nvm_rq
*rqd
);
755 int pblk_submit_io_sync(struct pblk
*pblk
, struct nvm_rq
*rqd
);
756 int pblk_submit_meta_io(struct pblk
*pblk
, struct pblk_line
*meta_line
);
757 struct bio
*pblk_bio_map_addr(struct pblk
*pblk
, void *data
,
758 unsigned int nr_secs
, unsigned int len
,
759 int alloc_type
, gfp_t gfp_mask
);
760 struct pblk_line
*pblk_line_get(struct pblk
*pblk
);
761 struct pblk_line
*pblk_line_get_first_data(struct pblk
*pblk
);
762 struct pblk_line
*pblk_line_replace_data(struct pblk
*pblk
);
763 int pblk_line_recov_alloc(struct pblk
*pblk
, struct pblk_line
*line
);
764 void pblk_line_recov_close(struct pblk
*pblk
, struct pblk_line
*line
);
765 struct pblk_line
*pblk_line_get_data(struct pblk
*pblk
);
766 struct pblk_line
*pblk_line_get_erase(struct pblk
*pblk
);
767 int pblk_line_erase(struct pblk
*pblk
, struct pblk_line
*line
);
768 int pblk_line_is_full(struct pblk_line
*line
);
769 void pblk_line_free(struct pblk
*pblk
, struct pblk_line
*line
);
770 void pblk_line_close_meta(struct pblk
*pblk
, struct pblk_line
*line
);
771 void pblk_line_close(struct pblk
*pblk
, struct pblk_line
*line
);
772 void pblk_line_close_ws(struct work_struct
*work
);
773 void pblk_pipeline_stop(struct pblk
*pblk
);
774 void pblk_gen_run_ws(struct pblk
*pblk
, struct pblk_line
*line
, void *priv
,
775 void (*work
)(struct work_struct
*), gfp_t gfp_mask
,
776 struct workqueue_struct
*wq
);
777 u64
pblk_line_smeta_start(struct pblk
*pblk
, struct pblk_line
*line
);
778 int pblk_line_read_smeta(struct pblk
*pblk
, struct pblk_line
*line
);
779 int pblk_line_read_emeta(struct pblk
*pblk
, struct pblk_line
*line
,
781 int pblk_blk_erase_async(struct pblk
*pblk
, struct ppa_addr erase_ppa
);
782 void pblk_line_put(struct kref
*ref
);
783 void pblk_line_put_wq(struct kref
*ref
);
784 struct list_head
*pblk_line_gc_list(struct pblk
*pblk
, struct pblk_line
*line
);
785 u64
pblk_lookup_page(struct pblk
*pblk
, struct pblk_line
*line
);
786 void pblk_dealloc_page(struct pblk
*pblk
, struct pblk_line
*line
, int nr_secs
);
787 u64
pblk_alloc_page(struct pblk
*pblk
, struct pblk_line
*line
, int nr_secs
);
788 u64
__pblk_alloc_page(struct pblk
*pblk
, struct pblk_line
*line
, int nr_secs
);
789 int pblk_calc_secs(struct pblk
*pblk
, unsigned long secs_avail
,
790 unsigned long secs_to_flush
);
791 void pblk_up_page(struct pblk
*pblk
, struct ppa_addr
*ppa_list
, int nr_ppas
);
792 void pblk_down_rq(struct pblk
*pblk
, struct ppa_addr
*ppa_list
, int nr_ppas
,
793 unsigned long *lun_bitmap
);
794 void pblk_down_page(struct pblk
*pblk
, struct ppa_addr
*ppa_list
, int nr_ppas
);
795 void pblk_up_rq(struct pblk
*pblk
, struct ppa_addr
*ppa_list
, int nr_ppas
,
796 unsigned long *lun_bitmap
);
797 void pblk_end_io_sync(struct nvm_rq
*rqd
);
798 int pblk_bio_add_pages(struct pblk
*pblk
, struct bio
*bio
, gfp_t flags
,
800 void pblk_bio_free_pages(struct pblk
*pblk
, struct bio
*bio
, int off
,
802 void pblk_map_invalidate(struct pblk
*pblk
, struct ppa_addr ppa
);
803 void __pblk_map_invalidate(struct pblk
*pblk
, struct pblk_line
*line
,
805 void pblk_update_map(struct pblk
*pblk
, sector_t lba
, struct ppa_addr ppa
);
806 void pblk_update_map_cache(struct pblk
*pblk
, sector_t lba
,
807 struct ppa_addr ppa
);
808 void pblk_update_map_dev(struct pblk
*pblk
, sector_t lba
,
809 struct ppa_addr ppa
, struct ppa_addr entry_line
);
810 int pblk_update_map_gc(struct pblk
*pblk
, sector_t lba
, struct ppa_addr ppa
,
811 struct pblk_line
*gc_line
, u64 paddr
);
812 void pblk_lookup_l2p_rand(struct pblk
*pblk
, struct ppa_addr
*ppas
,
813 u64
*lba_list
, int nr_secs
);
814 void pblk_lookup_l2p_seq(struct pblk
*pblk
, struct ppa_addr
*ppas
,
815 sector_t blba
, int nr_secs
);
818 * pblk user I/O write path
820 int pblk_write_to_cache(struct pblk
*pblk
, struct bio
*bio
,
821 unsigned long flags
);
822 int pblk_write_gc_to_cache(struct pblk
*pblk
, struct pblk_gc_rq
*gc_rq
);
827 void pblk_map_erase_rq(struct pblk
*pblk
, struct nvm_rq
*rqd
,
828 unsigned int sentry
, unsigned long *lun_bitmap
,
829 unsigned int valid_secs
, struct ppa_addr
*erase_ppa
);
830 void pblk_map_rq(struct pblk
*pblk
, struct nvm_rq
*rqd
, unsigned int sentry
,
831 unsigned long *lun_bitmap
, unsigned int valid_secs
,
837 int pblk_write_ts(void *data
);
838 void pblk_write_timer_fn(struct timer_list
*t
);
839 void pblk_write_should_kick(struct pblk
*pblk
);
844 extern struct bio_set
*pblk_bio_set
;
845 int pblk_submit_read(struct pblk
*pblk
, struct bio
*bio
);
846 int pblk_submit_read_gc(struct pblk
*pblk
, struct pblk_gc_rq
*gc_rq
);
850 void pblk_submit_rec(struct work_struct
*work
);
851 struct pblk_line
*pblk_recov_l2p(struct pblk
*pblk
);
852 int pblk_recov_pad(struct pblk
*pblk
);
853 int pblk_recov_check_emeta(struct pblk
*pblk
, struct line_emeta
*emeta
);
854 int pblk_recov_setup_rq(struct pblk
*pblk
, struct pblk_c_ctx
*c_ctx
,
855 struct pblk_rec_ctx
*recovery
, u64
*comp_bits
,
861 #define PBLK_GC_MAX_READERS 8 /* Max number of outstanding GC reader jobs */
862 #define PBLK_GC_RQ_QD 128 /* Queue depth for inflight GC requests */
863 #define PBLK_GC_L_QD 4 /* Queue depth for inflight GC lines */
864 #define PBLK_GC_RSV_LINE 1 /* Reserved lines for GC */
866 int pblk_gc_init(struct pblk
*pblk
);
867 void pblk_gc_exit(struct pblk
*pblk
);
868 void pblk_gc_should_start(struct pblk
*pblk
);
869 void pblk_gc_should_stop(struct pblk
*pblk
);
870 void pblk_gc_should_kick(struct pblk
*pblk
);
871 void pblk_gc_free_full_lines(struct pblk
*pblk
);
872 void pblk_gc_sysfs_state_show(struct pblk
*pblk
, int *gc_enabled
,
874 int pblk_gc_sysfs_force(struct pblk
*pblk
, int force
);
879 void pblk_rl_init(struct pblk_rl
*rl
, int budget
);
880 void pblk_rl_free(struct pblk_rl
*rl
);
881 void pblk_rl_update_rates(struct pblk_rl
*rl
);
882 int pblk_rl_high_thrs(struct pblk_rl
*rl
);
883 unsigned long pblk_rl_nr_free_blks(struct pblk_rl
*rl
);
884 unsigned long pblk_rl_nr_user_free_blks(struct pblk_rl
*rl
);
885 int pblk_rl_user_may_insert(struct pblk_rl
*rl
, int nr_entries
);
886 void pblk_rl_inserted(struct pblk_rl
*rl
, int nr_entries
);
887 void pblk_rl_user_in(struct pblk_rl
*rl
, int nr_entries
);
888 int pblk_rl_gc_may_insert(struct pblk_rl
*rl
, int nr_entries
);
889 void pblk_rl_gc_in(struct pblk_rl
*rl
, int nr_entries
);
890 void pblk_rl_out(struct pblk_rl
*rl
, int nr_user
, int nr_gc
);
891 int pblk_rl_max_io(struct pblk_rl
*rl
);
892 void pblk_rl_free_lines_inc(struct pblk_rl
*rl
, struct pblk_line
*line
);
893 void pblk_rl_free_lines_dec(struct pblk_rl
*rl
, struct pblk_line
*line
,
895 int pblk_rl_is_limit(struct pblk_rl
*rl
);
900 int pblk_sysfs_init(struct gendisk
*tdisk
);
901 void pblk_sysfs_exit(struct gendisk
*tdisk
);
903 static inline void *pblk_malloc(size_t size
, int type
, gfp_t flags
)
905 if (type
== PBLK_KMALLOC_META
)
906 return kmalloc(size
, flags
);
907 return vmalloc(size
);
910 static inline void pblk_mfree(void *ptr
, int type
)
912 if (type
== PBLK_KMALLOC_META
)
918 static inline struct nvm_rq
*nvm_rq_from_c_ctx(void *c_ctx
)
920 return c_ctx
- sizeof(struct nvm_rq
);
923 static inline void *emeta_to_bb(struct line_emeta
*emeta
)
925 return emeta
->bb_bitmap
;
928 static inline void *emeta_to_wa(struct pblk_line_meta
*lm
,
929 struct line_emeta
*emeta
)
931 return emeta
->bb_bitmap
+ lm
->blk_bitmap_len
;
934 static inline void *emeta_to_lbas(struct pblk
*pblk
, struct line_emeta
*emeta
)
936 return ((void *)emeta
+ pblk
->lm
.emeta_len
[1]);
939 static inline void *emeta_to_vsc(struct pblk
*pblk
, struct line_emeta
*emeta
)
941 return (emeta_to_lbas(pblk
, emeta
) + pblk
->lm
.emeta_len
[2]);
944 static inline int pblk_line_vsc(struct pblk_line
*line
)
946 return le32_to_cpu(*line
->vsc
);
949 static inline int pblk_pad_distance(struct pblk
*pblk
)
951 struct nvm_tgt_dev
*dev
= pblk
->dev
;
952 struct nvm_geo
*geo
= &dev
->geo
;
954 return geo
->mw_cunits
* geo
->all_luns
* geo
->ws_opt
;
957 static inline int pblk_ppa_to_line(struct ppa_addr p
)
962 static inline int pblk_ppa_to_pos(struct nvm_geo
*geo
, struct ppa_addr p
)
964 return p
.a
.lun
* geo
->num_ch
+ p
.a
.ch
;
967 static inline struct ppa_addr
addr_to_gen_ppa(struct pblk
*pblk
, u64 paddr
,
970 struct nvm_tgt_dev
*dev
= pblk
->dev
;
971 struct nvm_geo
*geo
= &dev
->geo
;
974 if (geo
->version
== NVM_OCSSD_SPEC_12
) {
975 struct nvm_addrf_12
*ppaf
= (struct nvm_addrf_12
*)&pblk
->addrf
;
979 ppa
.g
.pg
= (paddr
& ppaf
->pg_mask
) >> ppaf
->pg_offset
;
980 ppa
.g
.lun
= (paddr
& ppaf
->lun_mask
) >> ppaf
->lun_offset
;
981 ppa
.g
.ch
= (paddr
& ppaf
->ch_mask
) >> ppaf
->ch_offset
;
982 ppa
.g
.pl
= (paddr
& ppaf
->pln_mask
) >> ppaf
->pln_offset
;
983 ppa
.g
.sec
= (paddr
& ppaf
->sec_mask
) >> ppaf
->sec_offset
;
985 struct pblk_addrf
*uaddrf
= &pblk
->uaddrf
;
986 int secs
, chnls
, luns
;
992 paddr
= div_u64_rem(paddr
, uaddrf
->sec_stripe
, &secs
);
995 paddr
= div_u64_rem(paddr
, uaddrf
->ch_stripe
, &chnls
);
998 paddr
= div_u64_rem(paddr
, uaddrf
->lun_stripe
, &luns
);
1001 ppa
.m
.sec
+= uaddrf
->sec_stripe
* paddr
;
1007 static inline u64
pblk_dev_ppa_to_line_addr(struct pblk
*pblk
,
1010 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1011 struct nvm_geo
*geo
= &dev
->geo
;
1014 if (geo
->version
== NVM_OCSSD_SPEC_12
) {
1015 struct nvm_addrf_12
*ppaf
= (struct nvm_addrf_12
*)&pblk
->addrf
;
1017 paddr
= (u64
)p
.g
.ch
<< ppaf
->ch_offset
;
1018 paddr
|= (u64
)p
.g
.lun
<< ppaf
->lun_offset
;
1019 paddr
|= (u64
)p
.g
.pg
<< ppaf
->pg_offset
;
1020 paddr
|= (u64
)p
.g
.pl
<< ppaf
->pln_offset
;
1021 paddr
|= (u64
)p
.g
.sec
<< ppaf
->sec_offset
;
1023 struct pblk_addrf
*uaddrf
= &pblk
->uaddrf
;
1027 paddr
= (u64
)p
.m
.grp
* uaddrf
->sec_stripe
;
1028 paddr
+= (u64
)p
.m
.pu
* uaddrf
->sec_lun_stripe
;
1030 secs
= div_u64_rem(secs
, uaddrf
->sec_stripe
, &sec_stripe
);
1031 paddr
+= secs
* uaddrf
->sec_ws_stripe
;
1032 paddr
+= sec_stripe
;
1038 static inline struct ppa_addr
pblk_ppa32_to_ppa64(struct pblk
*pblk
, u32 ppa32
)
1040 struct ppa_addr ppa64
;
1045 ppa64
.ppa
= ADDR_EMPTY
;
1046 } else if (ppa32
& (1U << 31)) {
1047 ppa64
.c
.line
= ppa32
& ((~0U) >> 1);
1048 ppa64
.c
.is_cached
= 1;
1050 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1051 struct nvm_geo
*geo
= &dev
->geo
;
1053 if (geo
->version
== NVM_OCSSD_SPEC_12
) {
1054 struct nvm_addrf_12
*ppaf
=
1055 (struct nvm_addrf_12
*)&pblk
->addrf
;
1057 ppa64
.g
.ch
= (ppa32
& ppaf
->ch_mask
) >>
1059 ppa64
.g
.lun
= (ppa32
& ppaf
->lun_mask
) >>
1061 ppa64
.g
.blk
= (ppa32
& ppaf
->blk_mask
) >>
1063 ppa64
.g
.pg
= (ppa32
& ppaf
->pg_mask
) >>
1065 ppa64
.g
.pl
= (ppa32
& ppaf
->pln_mask
) >>
1067 ppa64
.g
.sec
= (ppa32
& ppaf
->sec_mask
) >>
1070 struct nvm_addrf
*lbaf
= &pblk
->addrf
;
1072 ppa64
.m
.grp
= (ppa32
& lbaf
->ch_mask
) >>
1074 ppa64
.m
.pu
= (ppa32
& lbaf
->lun_mask
) >>
1076 ppa64
.m
.chk
= (ppa32
& lbaf
->chk_mask
) >>
1078 ppa64
.m
.sec
= (ppa32
& lbaf
->sec_mask
) >>
1086 static inline u32
pblk_ppa64_to_ppa32(struct pblk
*pblk
, struct ppa_addr ppa64
)
1090 if (ppa64
.ppa
== ADDR_EMPTY
) {
1092 } else if (ppa64
.c
.is_cached
) {
1093 ppa32
|= ppa64
.c
.line
;
1096 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1097 struct nvm_geo
*geo
= &dev
->geo
;
1099 if (geo
->version
== NVM_OCSSD_SPEC_12
) {
1100 struct nvm_addrf_12
*ppaf
=
1101 (struct nvm_addrf_12
*)&pblk
->addrf
;
1103 ppa32
|= ppa64
.g
.ch
<< ppaf
->ch_offset
;
1104 ppa32
|= ppa64
.g
.lun
<< ppaf
->lun_offset
;
1105 ppa32
|= ppa64
.g
.blk
<< ppaf
->blk_offset
;
1106 ppa32
|= ppa64
.g
.pg
<< ppaf
->pg_offset
;
1107 ppa32
|= ppa64
.g
.pl
<< ppaf
->pln_offset
;
1108 ppa32
|= ppa64
.g
.sec
<< ppaf
->sec_offset
;
1110 struct nvm_addrf
*lbaf
= &pblk
->addrf
;
1112 ppa32
|= ppa64
.m
.grp
<< lbaf
->ch_offset
;
1113 ppa32
|= ppa64
.m
.pu
<< lbaf
->lun_offset
;
1114 ppa32
|= ppa64
.m
.chk
<< lbaf
->chk_offset
;
1115 ppa32
|= ppa64
.m
.sec
<< lbaf
->sec_offset
;
1122 static inline struct ppa_addr
pblk_trans_map_get(struct pblk
*pblk
,
1125 struct ppa_addr ppa
;
1127 if (pblk
->addrf_len
< 32) {
1128 u32
*map
= (u32
*)pblk
->trans_map
;
1130 ppa
= pblk_ppa32_to_ppa64(pblk
, map
[lba
]);
1132 struct ppa_addr
*map
= (struct ppa_addr
*)pblk
->trans_map
;
1140 static inline void pblk_trans_map_set(struct pblk
*pblk
, sector_t lba
,
1141 struct ppa_addr ppa
)
1143 if (pblk
->addrf_len
< 32) {
1144 u32
*map
= (u32
*)pblk
->trans_map
;
1146 map
[lba
] = pblk_ppa64_to_ppa32(pblk
, ppa
);
1148 u64
*map
= (u64
*)pblk
->trans_map
;
1154 static inline int pblk_ppa_empty(struct ppa_addr ppa_addr
)
1156 return (ppa_addr
.ppa
== ADDR_EMPTY
);
1159 static inline void pblk_ppa_set_empty(struct ppa_addr
*ppa_addr
)
1161 ppa_addr
->ppa
= ADDR_EMPTY
;
1164 static inline bool pblk_ppa_comp(struct ppa_addr lppa
, struct ppa_addr rppa
)
1166 return (lppa
.ppa
== rppa
.ppa
);
1169 static inline int pblk_addr_in_cache(struct ppa_addr ppa
)
1171 return (ppa
.ppa
!= ADDR_EMPTY
&& ppa
.c
.is_cached
);
1174 static inline int pblk_addr_to_cacheline(struct ppa_addr ppa
)
1179 static inline struct ppa_addr
pblk_cacheline_to_addr(int addr
)
1189 static inline u32
pblk_calc_meta_header_crc(struct pblk
*pblk
,
1190 struct line_header
*header
)
1194 crc
= crc32_le(crc
, (unsigned char *)header
+ sizeof(crc
),
1195 sizeof(struct line_header
) - sizeof(crc
));
1200 static inline u32
pblk_calc_smeta_crc(struct pblk
*pblk
,
1201 struct line_smeta
*smeta
)
1203 struct pblk_line_meta
*lm
= &pblk
->lm
;
1206 crc
= crc32_le(crc
, (unsigned char *)smeta
+
1207 sizeof(struct line_header
) + sizeof(crc
),
1209 sizeof(struct line_header
) - sizeof(crc
));
1214 static inline u32
pblk_calc_emeta_crc(struct pblk
*pblk
,
1215 struct line_emeta
*emeta
)
1217 struct pblk_line_meta
*lm
= &pblk
->lm
;
1220 crc
= crc32_le(crc
, (unsigned char *)emeta
+
1221 sizeof(struct line_header
) + sizeof(crc
),
1223 sizeof(struct line_header
) - sizeof(crc
));
1228 static inline int pblk_set_progr_mode(struct pblk
*pblk
, int type
)
1230 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1231 struct nvm_geo
*geo
= &dev
->geo
;
1234 if (geo
->version
== NVM_OCSSD_SPEC_20
)
1237 flags
= geo
->pln_mode
>> 1;
1239 if (type
== PBLK_WRITE
)
1240 flags
|= NVM_IO_SCRAMBLE_ENABLE
;
1246 PBLK_READ_RANDOM
= 0,
1247 PBLK_READ_SEQUENTIAL
= 1,
1250 static inline int pblk_set_read_mode(struct pblk
*pblk
, int type
)
1252 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1253 struct nvm_geo
*geo
= &dev
->geo
;
1256 if (geo
->version
== NVM_OCSSD_SPEC_20
)
1259 flags
= NVM_IO_SUSPEND
| NVM_IO_SCRAMBLE_ENABLE
;
1260 if (type
== PBLK_READ_SEQUENTIAL
)
1261 flags
|= geo
->pln_mode
>> 1;
1266 static inline int pblk_io_aligned(struct pblk
*pblk
, int nr_secs
)
1268 return !(nr_secs
% pblk
->min_write_pgs
);
1271 #ifdef CONFIG_NVM_DEBUG
1272 static inline void print_ppa(struct nvm_geo
*geo
, struct ppa_addr
*p
,
1273 char *msg
, int error
)
1275 if (p
->c
.is_cached
) {
1276 pr_err("ppa: (%s: %x) cache line: %llu\n",
1277 msg
, error
, (u64
)p
->c
.line
);
1278 } else if (geo
->version
== NVM_OCSSD_SPEC_12
) {
1279 pr_err("ppa: (%s: %x):ch:%d,lun:%d,blk:%d,pg:%d,pl:%d,sec:%d\n",
1281 p
->g
.ch
, p
->g
.lun
, p
->g
.blk
,
1282 p
->g
.pg
, p
->g
.pl
, p
->g
.sec
);
1284 pr_err("ppa: (%s: %x):ch:%d,lun:%d,chk:%d,sec:%d\n",
1286 p
->m
.grp
, p
->m
.pu
, p
->m
.chk
, p
->m
.sec
);
1290 static inline void pblk_print_failed_rqd(struct pblk
*pblk
, struct nvm_rq
*rqd
,
1295 if (rqd
->nr_ppas
== 1) {
1296 print_ppa(&pblk
->dev
->geo
, &rqd
->ppa_addr
, "rqd", error
);
1300 while ((bit
= find_next_bit((void *)&rqd
->ppa_status
, rqd
->nr_ppas
,
1301 bit
+ 1)) < rqd
->nr_ppas
) {
1302 print_ppa(&pblk
->dev
->geo
, &rqd
->ppa_list
[bit
], "rqd", error
);
1305 pr_err("error:%d, ppa_status:%llx\n", error
, rqd
->ppa_status
);
1308 static inline int pblk_boundary_ppa_checks(struct nvm_tgt_dev
*tgt_dev
,
1309 struct ppa_addr
*ppas
, int nr_ppas
)
1311 struct nvm_geo
*geo
= &tgt_dev
->geo
;
1312 struct ppa_addr
*ppa
;
1315 for (i
= 0; i
< nr_ppas
; i
++) {
1318 if (geo
->version
== NVM_OCSSD_SPEC_12
) {
1319 if (!ppa
->c
.is_cached
&&
1320 ppa
->g
.ch
< geo
->num_ch
&&
1321 ppa
->g
.lun
< geo
->num_lun
&&
1322 ppa
->g
.pl
< geo
->num_pln
&&
1323 ppa
->g
.blk
< geo
->num_chk
&&
1324 ppa
->g
.pg
< geo
->num_pg
&&
1325 ppa
->g
.sec
< geo
->ws_min
)
1328 if (!ppa
->c
.is_cached
&&
1329 ppa
->m
.grp
< geo
->num_ch
&&
1330 ppa
->m
.pu
< geo
->num_lun
&&
1331 ppa
->m
.chk
< geo
->num_chk
&&
1332 ppa
->m
.sec
< geo
->clba
)
1336 print_ppa(geo
, ppa
, "boundary", i
);
1343 static inline int pblk_check_io(struct pblk
*pblk
, struct nvm_rq
*rqd
)
1345 struct nvm_tgt_dev
*dev
= pblk
->dev
;
1346 struct ppa_addr
*ppa_list
;
1348 ppa_list
= (rqd
->nr_ppas
> 1) ? rqd
->ppa_list
: &rqd
->ppa_addr
;
1350 if (pblk_boundary_ppa_checks(dev
, ppa_list
, rqd
->nr_ppas
)) {
1355 if (rqd
->opcode
== NVM_OP_PWRITE
) {
1356 struct pblk_line
*line
;
1357 struct ppa_addr ppa
;
1360 for (i
= 0; i
< rqd
->nr_ppas
; i
++) {
1362 line
= &pblk
->lines
[pblk_ppa_to_line(ppa
)];
1364 spin_lock(&line
->lock
);
1365 if (line
->state
!= PBLK_LINESTATE_OPEN
) {
1366 pr_err("pblk: bad ppa: line:%d,state:%d\n",
1367 line
->id
, line
->state
);
1369 spin_unlock(&line
->lock
);
1372 spin_unlock(&line
->lock
);
1380 static inline int pblk_boundary_paddr_checks(struct pblk
*pblk
, u64 paddr
)
1382 struct pblk_line_meta
*lm
= &pblk
->lm
;
1384 if (paddr
> lm
->sec_per_line
)
1390 static inline unsigned int pblk_get_bi_idx(struct bio
*bio
)
1392 return bio
->bi_iter
.bi_idx
;
1395 static inline sector_t
pblk_get_lba(struct bio
*bio
)
1397 return bio
->bi_iter
.bi_sector
/ NR_PHY_IN_LOG
;
1400 static inline unsigned int pblk_get_secs(struct bio
*bio
)
1402 return bio
->bi_iter
.bi_size
/ PBLK_EXPOSED_PAGE_SIZE
;
1405 static inline void pblk_setup_uuid(struct pblk
*pblk
)
1410 memcpy(pblk
->instance_uuid
, uuid
.b
, 16);
1412 #endif /* PBLK_H_ */