2 * Block Translation Table
3 * Copyright (c) 2014-2015, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/highmem.h>
15 #include <linux/debugfs.h>
16 #include <linux/blkdev.h>
17 #include <linux/module.h>
18 #include <linux/device.h>
19 #include <linux/mutex.h>
20 #include <linux/hdreg.h>
21 #include <linux/genhd.h>
22 #include <linux/sizes.h>
23 #include <linux/ndctl.h>
29 enum log_ent_request
{
34 static u64
adjust_initial_offset(struct nd_btt
*nd_btt
, u64 offset
)
36 return offset
+ nd_btt
->initial_offset
;
39 static int arena_read_bytes(struct arena_info
*arena
, resource_size_t offset
,
40 void *buf
, size_t n
, unsigned long flags
)
42 struct nd_btt
*nd_btt
= arena
->nd_btt
;
43 struct nd_namespace_common
*ndns
= nd_btt
->ndns
;
45 /* arena offsets may be shifted from the base of the device */
46 offset
= adjust_initial_offset(nd_btt
, offset
);
47 return nvdimm_read_bytes(ndns
, offset
, buf
, n
, flags
);
50 static int arena_write_bytes(struct arena_info
*arena
, resource_size_t offset
,
51 void *buf
, size_t n
, unsigned long flags
)
53 struct nd_btt
*nd_btt
= arena
->nd_btt
;
54 struct nd_namespace_common
*ndns
= nd_btt
->ndns
;
56 /* arena offsets may be shifted from the base of the device */
57 offset
= adjust_initial_offset(nd_btt
, offset
);
58 return nvdimm_write_bytes(ndns
, offset
, buf
, n
, flags
);
61 static int btt_info_write(struct arena_info
*arena
, struct btt_sb
*super
)
66 * infooff and info2off should always be at least 512B aligned.
67 * We rely on that to make sure rw_bytes does error clearing
68 * correctly, so make sure that is the case.
70 WARN_ON_ONCE(!IS_ALIGNED(arena
->infooff
, 512));
71 WARN_ON_ONCE(!IS_ALIGNED(arena
->info2off
, 512));
73 ret
= arena_write_bytes(arena
, arena
->info2off
, super
,
74 sizeof(struct btt_sb
), 0);
78 return arena_write_bytes(arena
, arena
->infooff
, super
,
79 sizeof(struct btt_sb
), 0);
82 static int btt_info_read(struct arena_info
*arena
, struct btt_sb
*super
)
85 return arena_read_bytes(arena
, arena
->infooff
, super
,
86 sizeof(struct btt_sb
), 0);
90 * 'raw' version of btt_map write
92 * mapping is in little-endian
93 * mapping contains 'E' and 'Z' flags as desired
95 static int __btt_map_write(struct arena_info
*arena
, u32 lba
, __le32 mapping
,
98 u64 ns_off
= arena
->mapoff
+ (lba
* MAP_ENT_SIZE
);
100 WARN_ON(lba
>= arena
->external_nlba
);
101 return arena_write_bytes(arena
, ns_off
, &mapping
, MAP_ENT_SIZE
, flags
);
104 static int btt_map_write(struct arena_info
*arena
, u32 lba
, u32 mapping
,
105 u32 z_flag
, u32 e_flag
, unsigned long rwb_flags
)
111 * This 'mapping' is supposed to be just the LBA mapping, without
112 * any flags set, so strip the flag bits.
114 mapping
= ent_lba(mapping
);
116 ze
= (z_flag
<< 1) + e_flag
;
120 * We want to set neither of the Z or E flags, and
121 * in the actual layout, this means setting the bit
122 * positions of both to '1' to indicate a 'normal'
125 mapping
|= MAP_ENT_NORMAL
;
128 mapping
|= (1 << MAP_ERR_SHIFT
);
131 mapping
|= (1 << MAP_TRIM_SHIFT
);
135 * The case where Z and E are both sent in as '1' could be
136 * construed as a valid 'normal' case, but we decide not to,
139 WARN_ONCE(1, "Invalid use of Z and E flags\n");
143 mapping_le
= cpu_to_le32(mapping
);
144 return __btt_map_write(arena
, lba
, mapping_le
, rwb_flags
);
147 static int btt_map_read(struct arena_info
*arena
, u32 lba
, u32
*mapping
,
148 int *trim
, int *error
, unsigned long rwb_flags
)
152 u32 raw_mapping
, postmap
, ze
, z_flag
, e_flag
;
153 u64 ns_off
= arena
->mapoff
+ (lba
* MAP_ENT_SIZE
);
155 WARN_ON(lba
>= arena
->external_nlba
);
157 ret
= arena_read_bytes(arena
, ns_off
, &in
, MAP_ENT_SIZE
, rwb_flags
);
161 raw_mapping
= le32_to_cpu(in
);
163 z_flag
= ent_z_flag(raw_mapping
);
164 e_flag
= ent_e_flag(raw_mapping
);
165 ze
= (z_flag
<< 1) + e_flag
;
166 postmap
= ent_lba(raw_mapping
);
168 /* Reuse the {z,e}_flag variables for *trim and *error */
174 /* Initial state. Return postmap = premap */
200 static int btt_log_read_pair(struct arena_info
*arena
, u32 lane
,
201 struct log_entry
*ent
)
204 return arena_read_bytes(arena
,
205 arena
->logoff
+ (2 * lane
* LOG_ENT_SIZE
), ent
,
206 2 * LOG_ENT_SIZE
, 0);
209 static struct dentry
*debugfs_root
;
211 static void arena_debugfs_init(struct arena_info
*a
, struct dentry
*parent
,
217 /* If for some reason, parent bttN was not created, exit */
221 snprintf(dirname
, 32, "arena%d", idx
);
222 d
= debugfs_create_dir(dirname
, parent
);
223 if (IS_ERR_OR_NULL(d
))
227 debugfs_create_x64("size", S_IRUGO
, d
, &a
->size
);
228 debugfs_create_x64("external_lba_start", S_IRUGO
, d
,
229 &a
->external_lba_start
);
230 debugfs_create_x32("internal_nlba", S_IRUGO
, d
, &a
->internal_nlba
);
231 debugfs_create_u32("internal_lbasize", S_IRUGO
, d
,
232 &a
->internal_lbasize
);
233 debugfs_create_x32("external_nlba", S_IRUGO
, d
, &a
->external_nlba
);
234 debugfs_create_u32("external_lbasize", S_IRUGO
, d
,
235 &a
->external_lbasize
);
236 debugfs_create_u32("nfree", S_IRUGO
, d
, &a
->nfree
);
237 debugfs_create_u16("version_major", S_IRUGO
, d
, &a
->version_major
);
238 debugfs_create_u16("version_minor", S_IRUGO
, d
, &a
->version_minor
);
239 debugfs_create_x64("nextoff", S_IRUGO
, d
, &a
->nextoff
);
240 debugfs_create_x64("infooff", S_IRUGO
, d
, &a
->infooff
);
241 debugfs_create_x64("dataoff", S_IRUGO
, d
, &a
->dataoff
);
242 debugfs_create_x64("mapoff", S_IRUGO
, d
, &a
->mapoff
);
243 debugfs_create_x64("logoff", S_IRUGO
, d
, &a
->logoff
);
244 debugfs_create_x64("info2off", S_IRUGO
, d
, &a
->info2off
);
245 debugfs_create_x32("flags", S_IRUGO
, d
, &a
->flags
);
248 static void btt_debugfs_init(struct btt
*btt
)
251 struct arena_info
*arena
;
253 btt
->debugfs_dir
= debugfs_create_dir(dev_name(&btt
->nd_btt
->dev
),
255 if (IS_ERR_OR_NULL(btt
->debugfs_dir
))
258 list_for_each_entry(arena
, &btt
->arena_list
, list
) {
259 arena_debugfs_init(arena
, btt
->debugfs_dir
, i
);
265 * This function accepts two log entries, and uses the
266 * sequence number to find the 'older' entry.
267 * It also updates the sequence number in this old entry to
268 * make it the 'new' one if the mark_flag is set.
269 * Finally, it returns which of the entries was the older one.
271 * TODO The logic feels a bit kludge-y. make it better..
273 static int btt_log_get_old(struct log_entry
*ent
)
278 * the first ever time this is seen, the entry goes into [0]
279 * the next time, the following logic works out to put this
280 * (next) entry into [1]
282 if (ent
[0].seq
== 0) {
283 ent
[0].seq
= cpu_to_le32(1);
287 if (ent
[0].seq
== ent
[1].seq
)
289 if (le32_to_cpu(ent
[0].seq
) + le32_to_cpu(ent
[1].seq
) > 5)
292 if (le32_to_cpu(ent
[0].seq
) < le32_to_cpu(ent
[1].seq
)) {
293 if (le32_to_cpu(ent
[1].seq
) - le32_to_cpu(ent
[0].seq
) == 1)
298 if (le32_to_cpu(ent
[0].seq
) - le32_to_cpu(ent
[1].seq
) == 1)
307 static struct device
*to_dev(struct arena_info
*arena
)
309 return &arena
->nd_btt
->dev
;
313 * This function copies the desired (old/new) log entry into ent if
314 * it is not NULL. It returns the sub-slot number (0 or 1)
315 * where the desired log entry was found. Negative return values
318 static int btt_log_read(struct arena_info
*arena
, u32 lane
,
319 struct log_entry
*ent
, int old_flag
)
322 int old_ent
, ret_ent
;
323 struct log_entry log
[2];
325 ret
= btt_log_read_pair(arena
, lane
, log
);
329 old_ent
= btt_log_get_old(log
);
330 if (old_ent
< 0 || old_ent
> 1) {
331 dev_err(to_dev(arena
),
332 "log corruption (%d): lane %d seq [%d, %d]\n",
333 old_ent
, lane
, log
[0].seq
, log
[1].seq
);
334 /* TODO set error state? */
338 ret_ent
= (old_flag
? old_ent
: (1 - old_ent
));
341 memcpy(ent
, &log
[ret_ent
], LOG_ENT_SIZE
);
347 * This function commits a log entry to media
348 * It does _not_ prepare the freelist entry for the next write
349 * btt_flog_write is the wrapper for updating the freelist elements
351 static int __btt_log_write(struct arena_info
*arena
, u32 lane
,
352 u32 sub
, struct log_entry
*ent
, unsigned long flags
)
356 * Ignore the padding in log_entry for calculating log_half.
357 * The entry is 'committed' when we write the sequence number,
358 * and we want to ensure that that is the last thing written.
359 * We don't bother writing the padding as that would be extra
360 * media wear and write amplification
362 unsigned int log_half
= (LOG_ENT_SIZE
- 2 * sizeof(u64
)) / 2;
363 u64 ns_off
= arena
->logoff
+ (((2 * lane
) + sub
) * LOG_ENT_SIZE
);
366 /* split the 16B write into atomic, durable halves */
367 ret
= arena_write_bytes(arena
, ns_off
, src
, log_half
, flags
);
373 return arena_write_bytes(arena
, ns_off
, src
, log_half
, flags
);
376 static int btt_flog_write(struct arena_info
*arena
, u32 lane
, u32 sub
,
377 struct log_entry
*ent
)
381 ret
= __btt_log_write(arena
, lane
, sub
, ent
, NVDIMM_IO_ATOMIC
);
385 /* prepare the next free entry */
386 arena
->freelist
[lane
].sub
= 1 - arena
->freelist
[lane
].sub
;
387 if (++(arena
->freelist
[lane
].seq
) == 4)
388 arena
->freelist
[lane
].seq
= 1;
389 if (ent_e_flag(ent
->old_map
))
390 arena
->freelist
[lane
].has_err
= 1;
391 arena
->freelist
[lane
].block
= le32_to_cpu(ent_lba(ent
->old_map
));
397 * This function initializes the BTT map to the initial state, which is
398 * all-zeroes, and indicates an identity mapping
400 static int btt_map_init(struct arena_info
*arena
)
405 size_t chunk_size
= SZ_2M
;
406 size_t mapsize
= arena
->logoff
- arena
->mapoff
;
408 zerobuf
= kzalloc(chunk_size
, GFP_KERNEL
);
413 * mapoff should always be at least 512B aligned. We rely on that to
414 * make sure rw_bytes does error clearing correctly, so make sure that
417 WARN_ON_ONCE(!IS_ALIGNED(arena
->mapoff
, 512));
420 size_t size
= min(mapsize
, chunk_size
);
422 WARN_ON_ONCE(size
< 512);
423 ret
= arena_write_bytes(arena
, arena
->mapoff
+ offset
, zerobuf
,
439 * This function initializes the BTT log with 'fake' entries pointing
440 * to the initial reserved set of blocks as being free
442 static int btt_log_init(struct arena_info
*arena
)
444 size_t logsize
= arena
->info2off
- arena
->logoff
;
445 size_t chunk_size
= SZ_4K
, offset
= 0;
446 struct log_entry log
;
451 zerobuf
= kzalloc(chunk_size
, GFP_KERNEL
);
455 * logoff should always be at least 512B aligned. We rely on that to
456 * make sure rw_bytes does error clearing correctly, so make sure that
459 WARN_ON_ONCE(!IS_ALIGNED(arena
->logoff
, 512));
462 size_t size
= min(logsize
, chunk_size
);
464 WARN_ON_ONCE(size
< 512);
465 ret
= arena_write_bytes(arena
, arena
->logoff
+ offset
, zerobuf
,
475 for (i
= 0; i
< arena
->nfree
; i
++) {
476 log
.lba
= cpu_to_le32(i
);
477 log
.old_map
= cpu_to_le32(arena
->external_nlba
+ i
);
478 log
.new_map
= cpu_to_le32(arena
->external_nlba
+ i
);
479 log
.seq
= cpu_to_le32(LOG_SEQ_INIT
);
480 ret
= __btt_log_write(arena
, i
, 0, &log
, 0);
490 static u64
to_namespace_offset(struct arena_info
*arena
, u64 lba
)
492 return arena
->dataoff
+ ((u64
)lba
* arena
->internal_lbasize
);
495 static int arena_clear_freelist_error(struct arena_info
*arena
, u32 lane
)
499 if (arena
->freelist
[lane
].has_err
) {
500 void *zero_page
= page_address(ZERO_PAGE(0));
501 u32 lba
= arena
->freelist
[lane
].block
;
502 u64 nsoff
= to_namespace_offset(arena
, lba
);
503 unsigned long len
= arena
->sector_size
;
505 mutex_lock(&arena
->err_lock
);
508 unsigned long chunk
= min(len
, PAGE_SIZE
);
510 ret
= arena_write_bytes(arena
, nsoff
, zero_page
,
517 arena
->freelist
[lane
].has_err
= 0;
519 mutex_unlock(&arena
->err_lock
);
524 static int btt_freelist_init(struct arena_info
*arena
)
528 struct log_entry log_new
, log_old
;
530 arena
->freelist
= kcalloc(arena
->nfree
, sizeof(struct free_entry
),
532 if (!arena
->freelist
)
535 for (i
= 0; i
< arena
->nfree
; i
++) {
536 old
= btt_log_read(arena
, i
, &log_old
, LOG_OLD_ENT
);
540 new = btt_log_read(arena
, i
, &log_new
, LOG_NEW_ENT
);
544 /* sub points to the next one to be overwritten */
545 arena
->freelist
[i
].sub
= 1 - new;
546 arena
->freelist
[i
].seq
= nd_inc_seq(le32_to_cpu(log_new
.seq
));
547 arena
->freelist
[i
].block
= le32_to_cpu(log_new
.old_map
);
550 * FIXME: if error clearing fails during init, we want to make
553 if (ent_e_flag(log_new
.old_map
)) {
554 ret
= arena_clear_freelist_error(arena
, i
);
556 WARN_ONCE(1, "Unable to clear known errors\n");
559 /* This implies a newly created or untouched flog entry */
560 if (log_new
.old_map
== log_new
.new_map
)
563 /* Check if map recovery is needed */
564 ret
= btt_map_read(arena
, le32_to_cpu(log_new
.lba
), &map_entry
,
568 if ((le32_to_cpu(log_new
.new_map
) != map_entry
) &&
569 (le32_to_cpu(log_new
.old_map
) == map_entry
)) {
571 * Last transaction wrote the flog, but wasn't able
572 * to complete the map write. So fix up the map.
574 ret
= btt_map_write(arena
, le32_to_cpu(log_new
.lba
),
575 le32_to_cpu(log_new
.new_map
), 0, 0, 0);
584 static int btt_rtt_init(struct arena_info
*arena
)
586 arena
->rtt
= kcalloc(arena
->nfree
, sizeof(u32
), GFP_KERNEL
);
587 if (arena
->rtt
== NULL
)
593 static int btt_maplocks_init(struct arena_info
*arena
)
597 arena
->map_locks
= kcalloc(arena
->nfree
, sizeof(struct aligned_lock
),
599 if (!arena
->map_locks
)
602 for (i
= 0; i
< arena
->nfree
; i
++)
603 spin_lock_init(&arena
->map_locks
[i
].lock
);
608 static struct arena_info
*alloc_arena(struct btt
*btt
, size_t size
,
609 size_t start
, size_t arena_off
)
611 struct arena_info
*arena
;
612 u64 logsize
, mapsize
, datasize
;
613 u64 available
= size
;
615 arena
= kzalloc(sizeof(struct arena_info
), GFP_KERNEL
);
618 arena
->nd_btt
= btt
->nd_btt
;
619 arena
->sector_size
= btt
->sector_size
;
625 arena
->external_lba_start
= start
;
626 arena
->external_lbasize
= btt
->lbasize
;
627 arena
->internal_lbasize
= roundup(arena
->external_lbasize
,
628 INT_LBASIZE_ALIGNMENT
);
629 arena
->nfree
= BTT_DEFAULT_NFREE
;
630 arena
->version_major
= btt
->nd_btt
->version_major
;
631 arena
->version_minor
= btt
->nd_btt
->version_minor
;
633 if (available
% BTT_PG_SIZE
)
634 available
-= (available
% BTT_PG_SIZE
);
636 /* Two pages are reserved for the super block and its copy */
637 available
-= 2 * BTT_PG_SIZE
;
639 /* The log takes a fixed amount of space based on nfree */
640 logsize
= roundup(2 * arena
->nfree
* sizeof(struct log_entry
),
642 available
-= logsize
;
644 /* Calculate optimal split between map and data area */
645 arena
->internal_nlba
= div_u64(available
- BTT_PG_SIZE
,
646 arena
->internal_lbasize
+ MAP_ENT_SIZE
);
647 arena
->external_nlba
= arena
->internal_nlba
- arena
->nfree
;
649 mapsize
= roundup((arena
->external_nlba
* MAP_ENT_SIZE
), BTT_PG_SIZE
);
650 datasize
= available
- mapsize
;
652 /* 'Absolute' values, relative to start of storage space */
653 arena
->infooff
= arena_off
;
654 arena
->dataoff
= arena
->infooff
+ BTT_PG_SIZE
;
655 arena
->mapoff
= arena
->dataoff
+ datasize
;
656 arena
->logoff
= arena
->mapoff
+ mapsize
;
657 arena
->info2off
= arena
->logoff
+ logsize
;
661 static void free_arenas(struct btt
*btt
)
663 struct arena_info
*arena
, *next
;
665 list_for_each_entry_safe(arena
, next
, &btt
->arena_list
, list
) {
666 list_del(&arena
->list
);
668 kfree(arena
->map_locks
);
669 kfree(arena
->freelist
);
670 debugfs_remove_recursive(arena
->debugfs_dir
);
676 * This function reads an existing valid btt superblock and
677 * populates the corresponding arena_info struct
679 static void parse_arena_meta(struct arena_info
*arena
, struct btt_sb
*super
,
682 arena
->internal_nlba
= le32_to_cpu(super
->internal_nlba
);
683 arena
->internal_lbasize
= le32_to_cpu(super
->internal_lbasize
);
684 arena
->external_nlba
= le32_to_cpu(super
->external_nlba
);
685 arena
->external_lbasize
= le32_to_cpu(super
->external_lbasize
);
686 arena
->nfree
= le32_to_cpu(super
->nfree
);
687 arena
->version_major
= le16_to_cpu(super
->version_major
);
688 arena
->version_minor
= le16_to_cpu(super
->version_minor
);
690 arena
->nextoff
= (super
->nextoff
== 0) ? 0 : (arena_off
+
691 le64_to_cpu(super
->nextoff
));
692 arena
->infooff
= arena_off
;
693 arena
->dataoff
= arena_off
+ le64_to_cpu(super
->dataoff
);
694 arena
->mapoff
= arena_off
+ le64_to_cpu(super
->mapoff
);
695 arena
->logoff
= arena_off
+ le64_to_cpu(super
->logoff
);
696 arena
->info2off
= arena_off
+ le64_to_cpu(super
->info2off
);
698 arena
->size
= (le64_to_cpu(super
->nextoff
) > 0)
699 ? (le64_to_cpu(super
->nextoff
))
700 : (arena
->info2off
- arena
->infooff
+ BTT_PG_SIZE
);
702 arena
->flags
= le32_to_cpu(super
->flags
);
705 static int discover_arenas(struct btt
*btt
)
708 struct arena_info
*arena
;
709 struct btt_sb
*super
;
710 size_t remaining
= btt
->rawsize
;
715 super
= kzalloc(sizeof(*super
), GFP_KERNEL
);
720 /* Alloc memory for arena */
721 arena
= alloc_arena(btt
, 0, 0, 0);
727 arena
->infooff
= cur_off
;
728 ret
= btt_info_read(arena
, super
);
732 if (!nd_btt_arena_is_valid(btt
->nd_btt
, super
)) {
733 if (remaining
== btt
->rawsize
) {
734 btt
->init_state
= INIT_NOTFOUND
;
735 dev_info(to_dev(arena
), "No existing arenas\n");
738 dev_err(to_dev(arena
),
739 "Found corrupted metadata!\n");
745 arena
->external_lba_start
= cur_nlba
;
746 parse_arena_meta(arena
, super
, cur_off
);
748 mutex_init(&arena
->err_lock
);
749 ret
= btt_freelist_init(arena
);
753 ret
= btt_rtt_init(arena
);
757 ret
= btt_maplocks_init(arena
);
761 list_add_tail(&arena
->list
, &btt
->arena_list
);
763 remaining
-= arena
->size
;
764 cur_off
+= arena
->size
;
765 cur_nlba
+= arena
->external_nlba
;
768 if (arena
->nextoff
== 0)
771 btt
->num_arenas
= num_arenas
;
772 btt
->nlba
= cur_nlba
;
773 btt
->init_state
= INIT_READY
;
786 static int create_arenas(struct btt
*btt
)
788 size_t remaining
= btt
->rawsize
;
792 struct arena_info
*arena
;
793 size_t arena_size
= min_t(u64
, ARENA_MAX_SIZE
, remaining
);
795 remaining
-= arena_size
;
796 if (arena_size
< ARENA_MIN_SIZE
)
799 arena
= alloc_arena(btt
, arena_size
, btt
->nlba
, cur_off
);
804 btt
->nlba
+= arena
->external_nlba
;
805 if (remaining
>= ARENA_MIN_SIZE
)
806 arena
->nextoff
= arena
->size
;
809 cur_off
+= arena_size
;
810 list_add_tail(&arena
->list
, &btt
->arena_list
);
817 * This function completes arena initialization by writing
819 * It is only called for an uninitialized arena when a write
820 * to that arena occurs for the first time.
822 static int btt_arena_write_layout(struct arena_info
*arena
)
826 struct btt_sb
*super
;
827 struct nd_btt
*nd_btt
= arena
->nd_btt
;
828 const u8
*parent_uuid
= nd_dev_to_uuid(&nd_btt
->ndns
->dev
);
830 ret
= btt_map_init(arena
);
834 ret
= btt_log_init(arena
);
838 super
= kzalloc(sizeof(struct btt_sb
), GFP_NOIO
);
842 strncpy(super
->signature
, BTT_SIG
, BTT_SIG_LEN
);
843 memcpy(super
->uuid
, nd_btt
->uuid
, 16);
844 memcpy(super
->parent_uuid
, parent_uuid
, 16);
845 super
->flags
= cpu_to_le32(arena
->flags
);
846 super
->version_major
= cpu_to_le16(arena
->version_major
);
847 super
->version_minor
= cpu_to_le16(arena
->version_minor
);
848 super
->external_lbasize
= cpu_to_le32(arena
->external_lbasize
);
849 super
->external_nlba
= cpu_to_le32(arena
->external_nlba
);
850 super
->internal_lbasize
= cpu_to_le32(arena
->internal_lbasize
);
851 super
->internal_nlba
= cpu_to_le32(arena
->internal_nlba
);
852 super
->nfree
= cpu_to_le32(arena
->nfree
);
853 super
->infosize
= cpu_to_le32(sizeof(struct btt_sb
));
854 super
->nextoff
= cpu_to_le64(arena
->nextoff
);
856 * Subtract arena->infooff (arena start) so numbers are relative
859 super
->dataoff
= cpu_to_le64(arena
->dataoff
- arena
->infooff
);
860 super
->mapoff
= cpu_to_le64(arena
->mapoff
- arena
->infooff
);
861 super
->logoff
= cpu_to_le64(arena
->logoff
- arena
->infooff
);
862 super
->info2off
= cpu_to_le64(arena
->info2off
- arena
->infooff
);
865 sum
= nd_sb_checksum((struct nd_gen_sb
*) super
);
866 super
->checksum
= cpu_to_le64(sum
);
868 ret
= btt_info_write(arena
, super
);
875 * This function completes the initialization for the BTT namespace
876 * such that it is ready to accept IOs
878 static int btt_meta_init(struct btt
*btt
)
881 struct arena_info
*arena
;
883 mutex_lock(&btt
->init_lock
);
884 list_for_each_entry(arena
, &btt
->arena_list
, list
) {
885 ret
= btt_arena_write_layout(arena
);
889 ret
= btt_freelist_init(arena
);
893 ret
= btt_rtt_init(arena
);
897 ret
= btt_maplocks_init(arena
);
902 btt
->init_state
= INIT_READY
;
905 mutex_unlock(&btt
->init_lock
);
909 static u32
btt_meta_size(struct btt
*btt
)
911 return btt
->lbasize
- btt
->sector_size
;
915 * This function calculates the arena in which the given LBA lies
916 * by doing a linear walk. This is acceptable since we expect only
917 * a few arenas. If we have backing devices that get much larger,
918 * we can construct a balanced binary tree of arenas at init time
919 * so that this range search becomes faster.
921 static int lba_to_arena(struct btt
*btt
, sector_t sector
, __u32
*premap
,
922 struct arena_info
**arena
)
924 struct arena_info
*arena_list
;
925 __u64 lba
= div_u64(sector
<< SECTOR_SHIFT
, btt
->sector_size
);
927 list_for_each_entry(arena_list
, &btt
->arena_list
, list
) {
928 if (lba
< arena_list
->external_nlba
) {
933 lba
-= arena_list
->external_nlba
;
940 * The following (lock_map, unlock_map) are mostly just to improve
941 * readability, since they index into an array of locks
943 static void lock_map(struct arena_info
*arena
, u32 premap
)
944 __acquires(&arena
->map_locks
[idx
].lock
)
946 u32 idx
= (premap
* MAP_ENT_SIZE
/ L1_CACHE_BYTES
) % arena
->nfree
;
948 spin_lock(&arena
->map_locks
[idx
].lock
);
951 static void unlock_map(struct arena_info
*arena
, u32 premap
)
952 __releases(&arena
->map_locks
[idx
].lock
)
954 u32 idx
= (premap
* MAP_ENT_SIZE
/ L1_CACHE_BYTES
) % arena
->nfree
;
956 spin_unlock(&arena
->map_locks
[idx
].lock
);
959 static int btt_data_read(struct arena_info
*arena
, struct page
*page
,
960 unsigned int off
, u32 lba
, u32 len
)
963 u64 nsoff
= to_namespace_offset(arena
, lba
);
964 void *mem
= kmap_atomic(page
);
966 ret
= arena_read_bytes(arena
, nsoff
, mem
+ off
, len
, NVDIMM_IO_ATOMIC
);
972 static int btt_data_write(struct arena_info
*arena
, u32 lba
,
973 struct page
*page
, unsigned int off
, u32 len
)
976 u64 nsoff
= to_namespace_offset(arena
, lba
);
977 void *mem
= kmap_atomic(page
);
979 ret
= arena_write_bytes(arena
, nsoff
, mem
+ off
, len
, NVDIMM_IO_ATOMIC
);
985 static void zero_fill_data(struct page
*page
, unsigned int off
, u32 len
)
987 void *mem
= kmap_atomic(page
);
989 memset(mem
+ off
, 0, len
);
993 #ifdef CONFIG_BLK_DEV_INTEGRITY
994 static int btt_rw_integrity(struct btt
*btt
, struct bio_integrity_payload
*bip
,
995 struct arena_info
*arena
, u32 postmap
, int rw
)
997 unsigned int len
= btt_meta_size(btt
);
1004 meta_nsoff
= to_namespace_offset(arena
, postmap
) + btt
->sector_size
;
1007 unsigned int cur_len
;
1011 bv
= bvec_iter_bvec(bip
->bip_vec
, bip
->bip_iter
);
1013 * The 'bv' obtained from bvec_iter_bvec has its .bv_len and
1014 * .bv_offset already adjusted for iter->bi_bvec_done, and we
1015 * can use those directly
1018 cur_len
= min(len
, bv
.bv_len
);
1019 mem
= kmap_atomic(bv
.bv_page
);
1021 ret
= arena_write_bytes(arena
, meta_nsoff
,
1022 mem
+ bv
.bv_offset
, cur_len
,
1025 ret
= arena_read_bytes(arena
, meta_nsoff
,
1026 mem
+ bv
.bv_offset
, cur_len
,
1034 meta_nsoff
+= cur_len
;
1035 if (!bvec_iter_advance(bip
->bip_vec
, &bip
->bip_iter
, cur_len
))
1042 #else /* CONFIG_BLK_DEV_INTEGRITY */
1043 static int btt_rw_integrity(struct btt
*btt
, struct bio_integrity_payload
*bip
,
1044 struct arena_info
*arena
, u32 postmap
, int rw
)
1050 static int btt_read_pg(struct btt
*btt
, struct bio_integrity_payload
*bip
,
1051 struct page
*page
, unsigned int off
, sector_t sector
,
1056 struct arena_info
*arena
= NULL
;
1057 u32 lane
= 0, premap
, postmap
;
1062 lane
= nd_region_acquire_lane(btt
->nd_region
);
1064 ret
= lba_to_arena(btt
, sector
, &premap
, &arena
);
1068 cur_len
= min(btt
->sector_size
, len
);
1070 ret
= btt_map_read(arena
, premap
, &postmap
, &t_flag
, &e_flag
,
1076 * We loop to make sure that the post map LBA didn't change
1077 * from under us between writing the RTT and doing the actual
1085 zero_fill_data(page
, off
, cur_len
);
1094 arena
->rtt
[lane
] = RTT_VALID
| postmap
;
1096 * Barrier to make sure this write is not reordered
1097 * to do the verification map_read before the RTT store
1101 ret
= btt_map_read(arena
, premap
, &new_map
, &new_t
,
1102 &new_e
, NVDIMM_IO_ATOMIC
);
1106 if ((postmap
== new_map
) && (t_flag
== new_t
) &&
1115 ret
= btt_data_read(arena
, page
, off
, postmap
, cur_len
);
1119 /* Media error - set the e_flag */
1120 rc
= btt_map_write(arena
, premap
, postmap
, 0, 1,
1126 ret
= btt_rw_integrity(btt
, bip
, arena
, postmap
, READ
);
1131 arena
->rtt
[lane
] = RTT_INVALID
;
1132 nd_region_release_lane(btt
->nd_region
, lane
);
1136 sector
+= btt
->sector_size
>> SECTOR_SHIFT
;
1142 arena
->rtt
[lane
] = RTT_INVALID
;
1144 nd_region_release_lane(btt
->nd_region
, lane
);
1149 * Normally, arena_{read,write}_bytes will take care of the initial offset
1150 * adjustment, but in the case of btt_is_badblock, where we query is_bad_pmem,
1151 * we need the final, raw namespace offset here
1153 static bool btt_is_badblock(struct btt
*btt
, struct arena_info
*arena
,
1156 u64 nsoff
= adjust_initial_offset(arena
->nd_btt
,
1157 to_namespace_offset(arena
, postmap
));
1158 sector_t phys_sector
= nsoff
>> 9;
1160 return is_bad_pmem(btt
->phys_bb
, phys_sector
, arena
->internal_lbasize
);
1163 static int btt_write_pg(struct btt
*btt
, struct bio_integrity_payload
*bip
,
1164 sector_t sector
, struct page
*page
, unsigned int off
,
1168 struct arena_info
*arena
= NULL
;
1169 u32 premap
= 0, old_postmap
, new_postmap
, lane
= 0, i
;
1170 struct log_entry log
;
1178 lane
= nd_region_acquire_lane(btt
->nd_region
);
1180 ret
= lba_to_arena(btt
, sector
, &premap
, &arena
);
1183 cur_len
= min(btt
->sector_size
, len
);
1185 if ((arena
->flags
& IB_FLAG_ERROR_MASK
) != 0) {
1190 if (btt_is_badblock(btt
, arena
, arena
->freelist
[lane
].block
))
1191 arena
->freelist
[lane
].has_err
= 1;
1193 if (mutex_is_locked(&arena
->err_lock
)
1194 || arena
->freelist
[lane
].has_err
) {
1195 nd_region_release_lane(btt
->nd_region
, lane
);
1197 ret
= arena_clear_freelist_error(arena
, lane
);
1201 /* OK to acquire a different lane/free block */
1205 new_postmap
= arena
->freelist
[lane
].block
;
1207 /* Wait if the new block is being read from */
1208 for (i
= 0; i
< arena
->nfree
; i
++)
1209 while (arena
->rtt
[i
] == (RTT_VALID
| new_postmap
))
1213 if (new_postmap
>= arena
->internal_nlba
) {
1218 ret
= btt_data_write(arena
, new_postmap
, page
, off
, cur_len
);
1223 ret
= btt_rw_integrity(btt
, bip
, arena
, new_postmap
,
1229 lock_map(arena
, premap
);
1230 ret
= btt_map_read(arena
, premap
, &old_postmap
, NULL
, &e_flag
,
1234 if (old_postmap
>= arena
->internal_nlba
) {
1239 set_e_flag(old_postmap
);
1241 log
.lba
= cpu_to_le32(premap
);
1242 log
.old_map
= cpu_to_le32(old_postmap
);
1243 log
.new_map
= cpu_to_le32(new_postmap
);
1244 log
.seq
= cpu_to_le32(arena
->freelist
[lane
].seq
);
1245 sub
= arena
->freelist
[lane
].sub
;
1246 ret
= btt_flog_write(arena
, lane
, sub
, &log
);
1250 ret
= btt_map_write(arena
, premap
, new_postmap
, 0, 0,
1255 unlock_map(arena
, premap
);
1256 nd_region_release_lane(btt
->nd_region
, lane
);
1259 ret
= arena_clear_freelist_error(arena
, lane
);
1266 sector
+= btt
->sector_size
>> SECTOR_SHIFT
;
1272 unlock_map(arena
, premap
);
1274 nd_region_release_lane(btt
->nd_region
, lane
);
1278 static int btt_do_bvec(struct btt
*btt
, struct bio_integrity_payload
*bip
,
1279 struct page
*page
, unsigned int len
, unsigned int off
,
1280 bool is_write
, sector_t sector
)
1285 ret
= btt_read_pg(btt
, bip
, page
, off
, sector
, len
);
1286 flush_dcache_page(page
);
1288 flush_dcache_page(page
);
1289 ret
= btt_write_pg(btt
, bip
, sector
, page
, off
, len
);
1295 static blk_qc_t
btt_make_request(struct request_queue
*q
, struct bio
*bio
)
1297 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
1298 struct btt
*btt
= q
->queuedata
;
1299 struct bvec_iter iter
;
1300 unsigned long start
;
1301 struct bio_vec bvec
;
1305 if (!bio_integrity_prep(bio
))
1306 return BLK_QC_T_NONE
;
1308 do_acct
= nd_iostat_start(bio
, &start
);
1309 bio_for_each_segment(bvec
, bio
, iter
) {
1310 unsigned int len
= bvec
.bv_len
;
1312 BUG_ON(len
> PAGE_SIZE
);
1313 /* Make sure len is in multiples of sector size. */
1314 /* XXX is this right? */
1315 BUG_ON(len
< btt
->sector_size
);
1316 BUG_ON(len
% btt
->sector_size
);
1318 err
= btt_do_bvec(btt
, bip
, bvec
.bv_page
, len
, bvec
.bv_offset
,
1319 op_is_write(bio_op(bio
)), iter
.bi_sector
);
1321 dev_err(&btt
->nd_btt
->dev
,
1322 "io error in %s sector %lld, len %d,\n",
1323 (op_is_write(bio_op(bio
))) ? "WRITE" :
1325 (unsigned long long) iter
.bi_sector
, len
);
1326 bio
->bi_status
= errno_to_blk_status(err
);
1331 nd_iostat_end(bio
, start
);
1334 return BLK_QC_T_NONE
;
1337 static int btt_rw_page(struct block_device
*bdev
, sector_t sector
,
1338 struct page
*page
, bool is_write
)
1340 struct btt
*btt
= bdev
->bd_disk
->private_data
;
1343 rc
= btt_do_bvec(btt
, NULL
, page
, PAGE_SIZE
, 0, is_write
, sector
);
1345 page_endio(page
, is_write
, 0);
1351 static int btt_getgeo(struct block_device
*bd
, struct hd_geometry
*geo
)
1353 /* some standard values */
1354 geo
->heads
= 1 << 6;
1355 geo
->sectors
= 1 << 5;
1356 geo
->cylinders
= get_capacity(bd
->bd_disk
) >> 11;
1360 static const struct block_device_operations btt_fops
= {
1361 .owner
= THIS_MODULE
,
1362 .rw_page
= btt_rw_page
,
1363 .getgeo
= btt_getgeo
,
1364 .revalidate_disk
= nvdimm_revalidate_disk
,
1367 static int btt_blk_init(struct btt
*btt
)
1369 struct nd_btt
*nd_btt
= btt
->nd_btt
;
1370 struct nd_namespace_common
*ndns
= nd_btt
->ndns
;
1372 /* create a new disk and request queue for btt */
1373 btt
->btt_queue
= blk_alloc_queue(GFP_KERNEL
);
1374 if (!btt
->btt_queue
)
1377 btt
->btt_disk
= alloc_disk(0);
1378 if (!btt
->btt_disk
) {
1379 blk_cleanup_queue(btt
->btt_queue
);
1383 nvdimm_namespace_disk_name(ndns
, btt
->btt_disk
->disk_name
);
1384 btt
->btt_disk
->first_minor
= 0;
1385 btt
->btt_disk
->fops
= &btt_fops
;
1386 btt
->btt_disk
->private_data
= btt
;
1387 btt
->btt_disk
->queue
= btt
->btt_queue
;
1388 btt
->btt_disk
->flags
= GENHD_FL_EXT_DEVT
;
1390 blk_queue_make_request(btt
->btt_queue
, btt_make_request
);
1391 blk_queue_logical_block_size(btt
->btt_queue
, btt
->sector_size
);
1392 blk_queue_max_hw_sectors(btt
->btt_queue
, UINT_MAX
);
1393 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, btt
->btt_queue
);
1394 btt
->btt_queue
->queuedata
= btt
;
1396 set_capacity(btt
->btt_disk
, 0);
1397 device_add_disk(&btt
->nd_btt
->dev
, btt
->btt_disk
);
1398 if (btt_meta_size(btt
)) {
1399 int rc
= nd_integrity_init(btt
->btt_disk
, btt_meta_size(btt
));
1402 del_gendisk(btt
->btt_disk
);
1403 put_disk(btt
->btt_disk
);
1404 blk_cleanup_queue(btt
->btt_queue
);
1408 set_capacity(btt
->btt_disk
, btt
->nlba
* btt
->sector_size
>> 9);
1409 btt
->nd_btt
->size
= btt
->nlba
* (u64
)btt
->sector_size
;
1410 revalidate_disk(btt
->btt_disk
);
1415 static void btt_blk_cleanup(struct btt
*btt
)
1417 del_gendisk(btt
->btt_disk
);
1418 put_disk(btt
->btt_disk
);
1419 blk_cleanup_queue(btt
->btt_queue
);
1423 * btt_init - initialize a block translation table for the given device
1424 * @nd_btt: device with BTT geometry and backing device info
1425 * @rawsize: raw size in bytes of the backing device
1426 * @lbasize: lba size of the backing device
1427 * @uuid: A uuid for the backing device - this is stored on media
1428 * @maxlane: maximum number of parallel requests the device can handle
1430 * Initialize a Block Translation Table on a backing device to provide
1431 * single sector power fail atomicity.
1437 * Pointer to a new struct btt on success, NULL on failure.
1439 static struct btt
*btt_init(struct nd_btt
*nd_btt
, unsigned long long rawsize
,
1440 u32 lbasize
, u8
*uuid
, struct nd_region
*nd_region
)
1444 struct nd_namespace_io
*nsio
;
1445 struct device
*dev
= &nd_btt
->dev
;
1447 btt
= devm_kzalloc(dev
, sizeof(struct btt
), GFP_KERNEL
);
1451 btt
->nd_btt
= nd_btt
;
1452 btt
->rawsize
= rawsize
;
1453 btt
->lbasize
= lbasize
;
1454 btt
->sector_size
= ((lbasize
>= 4096) ? 4096 : 512);
1455 INIT_LIST_HEAD(&btt
->arena_list
);
1456 mutex_init(&btt
->init_lock
);
1457 btt
->nd_region
= nd_region
;
1458 nsio
= to_nd_namespace_io(&nd_btt
->ndns
->dev
);
1459 btt
->phys_bb
= &nsio
->bb
;
1461 ret
= discover_arenas(btt
);
1463 dev_err(dev
, "init: error in arena_discover: %d\n", ret
);
1467 if (btt
->init_state
!= INIT_READY
&& nd_region
->ro
) {
1468 dev_warn(dev
, "%s is read-only, unable to init btt metadata\n",
1469 dev_name(&nd_region
->dev
));
1471 } else if (btt
->init_state
!= INIT_READY
) {
1472 btt
->num_arenas
= (rawsize
/ ARENA_MAX_SIZE
) +
1473 ((rawsize
% ARENA_MAX_SIZE
) ? 1 : 0);
1474 dev_dbg(dev
, "init: %d arenas for %llu rawsize\n",
1475 btt
->num_arenas
, rawsize
);
1477 ret
= create_arenas(btt
);
1479 dev_info(dev
, "init: create_arenas: %d\n", ret
);
1483 ret
= btt_meta_init(btt
);
1485 dev_err(dev
, "init: error in meta_init: %d\n", ret
);
1490 ret
= btt_blk_init(btt
);
1492 dev_err(dev
, "init: error in blk_init: %d\n", ret
);
1496 btt_debugfs_init(btt
);
1502 * btt_fini - de-initialize a BTT
1503 * @btt: the BTT handle that was generated by btt_init
1505 * De-initialize a Block Translation Table on device removal
1510 static void btt_fini(struct btt
*btt
)
1513 btt_blk_cleanup(btt
);
1515 debugfs_remove_recursive(btt
->debugfs_dir
);
1519 int nvdimm_namespace_attach_btt(struct nd_namespace_common
*ndns
)
1521 struct nd_btt
*nd_btt
= to_nd_btt(ndns
->claim
);
1522 struct nd_region
*nd_region
;
1523 struct btt_sb
*btt_sb
;
1527 if (!nd_btt
->uuid
|| !nd_btt
->ndns
|| !nd_btt
->lbasize
) {
1528 dev_dbg(&nd_btt
->dev
, "incomplete btt configuration\n");
1532 btt_sb
= devm_kzalloc(&nd_btt
->dev
, sizeof(*btt_sb
), GFP_KERNEL
);
1537 * If this returns < 0, that is ok as it just means there wasn't
1538 * an existing BTT, and we're creating a new one. We still need to
1539 * call this as we need the version dependent fields in nd_btt to be
1540 * set correctly based on the holder class
1542 nd_btt_version(nd_btt
, ndns
, btt_sb
);
1544 rawsize
= nvdimm_namespace_capacity(ndns
) - nd_btt
->initial_offset
;
1545 if (rawsize
< ARENA_MIN_SIZE
) {
1546 dev_dbg(&nd_btt
->dev
, "%s must be at least %ld bytes\n",
1547 dev_name(&ndns
->dev
),
1548 ARENA_MIN_SIZE
+ nd_btt
->initial_offset
);
1551 nd_region
= to_nd_region(nd_btt
->dev
.parent
);
1552 btt
= btt_init(nd_btt
, rawsize
, nd_btt
->lbasize
, nd_btt
->uuid
,
1560 EXPORT_SYMBOL(nvdimm_namespace_attach_btt
);
1562 int nvdimm_namespace_detach_btt(struct nd_btt
*nd_btt
)
1564 struct btt
*btt
= nd_btt
->btt
;
1571 EXPORT_SYMBOL(nvdimm_namespace_detach_btt
);
1573 static int __init
nd_btt_init(void)
1577 debugfs_root
= debugfs_create_dir("btt", NULL
);
1578 if (IS_ERR_OR_NULL(debugfs_root
))
1584 static void __exit
nd_btt_exit(void)
1586 debugfs_remove_recursive(debugfs_root
);
1589 MODULE_ALIAS_ND_DEVICE(ND_DEVICE_BTT
);
1590 MODULE_AUTHOR("Vishal Verma <vishal.l.verma@linux.intel.com>");
1591 MODULE_LICENSE("GPL v2");
1592 module_init(nd_btt_init
);
1593 module_exit(nd_btt_exit
);