2 * Copyright (C) 2012 Red Hat, Inc.
4 * This file is released under the GPL.
7 #include "dm-cache-metadata.h"
9 #include "persistent-data/dm-array.h"
10 #include "persistent-data/dm-bitset.h"
11 #include "persistent-data/dm-space-map.h"
12 #include "persistent-data/dm-space-map-disk.h"
13 #include "persistent-data/dm-transaction-manager.h"
15 #include <linux/device-mapper.h>
17 /*----------------------------------------------------------------*/
19 #define DM_MSG_PREFIX "cache metadata"
21 #define CACHE_SUPERBLOCK_MAGIC 06142003
22 #define CACHE_SUPERBLOCK_LOCATION 0
25 * defines a range of metadata versions that this module can handle.
27 #define MIN_CACHE_VERSION 1
28 #define MAX_CACHE_VERSION 2
31 * 3 for btree insert +
32 * 2 for btree lookup used within space map
34 #define CACHE_MAX_CONCURRENT_LOCKS 5
35 #define SPACE_MAP_ROOT_SIZE 128
37 enum superblock_flag_bits
{
38 /* for spotting crashes that would invalidate the dirty bitset */
40 /* metadata must be checked using the tools */
45 * Each mapping from cache block -> origin block carries a set of flags.
49 * A valid mapping. Because we're using an array we clear this
50 * flag for an non existant mapping.
55 * The data on the cache is different from that on the origin.
56 * This flag is only used by metadata format 1.
61 struct cache_disk_superblock
{
70 __u8 policy_name
[CACHE_POLICY_NAME_SIZE
];
71 __le32 policy_hint_size
;
73 __u8 metadata_space_map_root
[SPACE_MAP_ROOT_SIZE
];
78 __le64 discard_block_size
;
79 __le64 discard_nr_blocks
;
81 __le32 data_block_size
;
82 __le32 metadata_block_size
;
86 __le32 compat_ro_flags
;
87 __le32 incompat_flags
;
94 __le32 policy_version
[CACHE_POLICY_VERSION_SIZE
];
97 * Metadata format 2 fields.
102 struct dm_cache_metadata
{
104 struct list_head list
;
107 struct block_device
*bdev
;
108 struct dm_block_manager
*bm
;
109 struct dm_space_map
*metadata_sm
;
110 struct dm_transaction_manager
*tm
;
112 struct dm_array_info info
;
113 struct dm_array_info hint_info
;
114 struct dm_disk_bitset discard_info
;
116 struct rw_semaphore root_lock
;
119 dm_block_t hint_root
;
120 dm_block_t discard_root
;
122 sector_t discard_block_size
;
123 dm_dblock_t discard_nr_blocks
;
125 sector_t data_block_size
;
126 dm_cblock_t cache_blocks
;
128 bool clean_when_opened
:1;
130 char policy_name
[CACHE_POLICY_NAME_SIZE
];
131 unsigned policy_version
[CACHE_POLICY_VERSION_SIZE
];
132 size_t policy_hint_size
;
133 struct dm_cache_statistics stats
;
136 * Reading the space map root can fail, so we read it into this
137 * buffer before the superblock is locked and updated.
139 __u8 metadata_space_map_root
[SPACE_MAP_ROOT_SIZE
];
142 * Set if a transaction has to be aborted but the attempt to roll
143 * back to the previous (good) transaction failed. The only
144 * metadata operation permissible in this state is the closing of
150 * Metadata format 2 fields.
152 dm_block_t dirty_root
;
153 struct dm_disk_bitset dirty_info
;
156 * These structures are used when loading metadata. They're too
157 * big to put on the stack.
159 struct dm_array_cursor mapping_cursor
;
160 struct dm_array_cursor hint_cursor
;
161 struct dm_bitset_cursor dirty_cursor
;
164 /*-------------------------------------------------------------------
165 * superblock validator
166 *-----------------------------------------------------------------*/
168 #define SUPERBLOCK_CSUM_XOR 9031977
170 static void sb_prepare_for_write(struct dm_block_validator
*v
,
172 size_t sb_block_size
)
174 struct cache_disk_superblock
*disk_super
= dm_block_data(b
);
176 disk_super
->blocknr
= cpu_to_le64(dm_block_location(b
));
177 disk_super
->csum
= cpu_to_le32(dm_bm_checksum(&disk_super
->flags
,
178 sb_block_size
- sizeof(__le32
),
179 SUPERBLOCK_CSUM_XOR
));
182 static int check_metadata_version(struct cache_disk_superblock
*disk_super
)
184 uint32_t metadata_version
= le32_to_cpu(disk_super
->version
);
186 if (metadata_version
< MIN_CACHE_VERSION
|| metadata_version
> MAX_CACHE_VERSION
) {
187 DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
188 metadata_version
, MIN_CACHE_VERSION
, MAX_CACHE_VERSION
);
195 static int sb_check(struct dm_block_validator
*v
,
197 size_t sb_block_size
)
199 struct cache_disk_superblock
*disk_super
= dm_block_data(b
);
202 if (dm_block_location(b
) != le64_to_cpu(disk_super
->blocknr
)) {
203 DMERR("sb_check failed: blocknr %llu: wanted %llu",
204 le64_to_cpu(disk_super
->blocknr
),
205 (unsigned long long)dm_block_location(b
));
209 if (le64_to_cpu(disk_super
->magic
) != CACHE_SUPERBLOCK_MAGIC
) {
210 DMERR("sb_check failed: magic %llu: wanted %llu",
211 le64_to_cpu(disk_super
->magic
),
212 (unsigned long long)CACHE_SUPERBLOCK_MAGIC
);
216 csum_le
= cpu_to_le32(dm_bm_checksum(&disk_super
->flags
,
217 sb_block_size
- sizeof(__le32
),
218 SUPERBLOCK_CSUM_XOR
));
219 if (csum_le
!= disk_super
->csum
) {
220 DMERR("sb_check failed: csum %u: wanted %u",
221 le32_to_cpu(csum_le
), le32_to_cpu(disk_super
->csum
));
225 return check_metadata_version(disk_super
);
228 static struct dm_block_validator sb_validator
= {
229 .name
= "superblock",
230 .prepare_for_write
= sb_prepare_for_write
,
234 /*----------------------------------------------------------------*/
236 static int superblock_read_lock(struct dm_cache_metadata
*cmd
,
237 struct dm_block
**sblock
)
239 return dm_bm_read_lock(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
240 &sb_validator
, sblock
);
243 static int superblock_lock_zero(struct dm_cache_metadata
*cmd
,
244 struct dm_block
**sblock
)
246 return dm_bm_write_lock_zero(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
247 &sb_validator
, sblock
);
250 static int superblock_lock(struct dm_cache_metadata
*cmd
,
251 struct dm_block
**sblock
)
253 return dm_bm_write_lock(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
254 &sb_validator
, sblock
);
257 /*----------------------------------------------------------------*/
259 static int __superblock_all_zeroes(struct dm_block_manager
*bm
, bool *result
)
264 __le64
*data_le
, zero
= cpu_to_le64(0);
265 unsigned sb_block_size
= dm_bm_block_size(bm
) / sizeof(__le64
);
268 * We can't use a validator here - it may be all zeroes.
270 r
= dm_bm_read_lock(bm
, CACHE_SUPERBLOCK_LOCATION
, NULL
, &b
);
274 data_le
= dm_block_data(b
);
276 for (i
= 0; i
< sb_block_size
; i
++) {
277 if (data_le
[i
] != zero
) {
288 static void __setup_mapping_info(struct dm_cache_metadata
*cmd
)
290 struct dm_btree_value_type vt
;
293 vt
.size
= sizeof(__le64
);
297 dm_array_info_init(&cmd
->info
, cmd
->tm
, &vt
);
299 if (cmd
->policy_hint_size
) {
300 vt
.size
= sizeof(__le32
);
301 dm_array_info_init(&cmd
->hint_info
, cmd
->tm
, &vt
);
305 static int __save_sm_root(struct dm_cache_metadata
*cmd
)
310 r
= dm_sm_root_size(cmd
->metadata_sm
, &metadata_len
);
314 return dm_sm_copy_root(cmd
->metadata_sm
, &cmd
->metadata_space_map_root
,
318 static void __copy_sm_root(struct dm_cache_metadata
*cmd
,
319 struct cache_disk_superblock
*disk_super
)
321 memcpy(&disk_super
->metadata_space_map_root
,
322 &cmd
->metadata_space_map_root
,
323 sizeof(cmd
->metadata_space_map_root
));
326 static bool separate_dirty_bits(struct dm_cache_metadata
*cmd
)
328 return cmd
->version
>= 2;
331 static int __write_initial_superblock(struct dm_cache_metadata
*cmd
)
334 struct dm_block
*sblock
;
335 struct cache_disk_superblock
*disk_super
;
336 sector_t bdev_size
= i_size_read(cmd
->bdev
->bd_inode
) >> SECTOR_SHIFT
;
338 /* FIXME: see if we can lose the max sectors limit */
339 if (bdev_size
> DM_CACHE_METADATA_MAX_SECTORS
)
340 bdev_size
= DM_CACHE_METADATA_MAX_SECTORS
;
342 r
= dm_tm_pre_commit(cmd
->tm
);
347 * dm_sm_copy_root() can fail. So we need to do it before we start
348 * updating the superblock.
350 r
= __save_sm_root(cmd
);
354 r
= superblock_lock_zero(cmd
, &sblock
);
358 disk_super
= dm_block_data(sblock
);
359 disk_super
->flags
= 0;
360 memset(disk_super
->uuid
, 0, sizeof(disk_super
->uuid
));
361 disk_super
->magic
= cpu_to_le64(CACHE_SUPERBLOCK_MAGIC
);
362 disk_super
->version
= cpu_to_le32(cmd
->version
);
363 memset(disk_super
->policy_name
, 0, sizeof(disk_super
->policy_name
));
364 memset(disk_super
->policy_version
, 0, sizeof(disk_super
->policy_version
));
365 disk_super
->policy_hint_size
= 0;
367 __copy_sm_root(cmd
, disk_super
);
369 disk_super
->mapping_root
= cpu_to_le64(cmd
->root
);
370 disk_super
->hint_root
= cpu_to_le64(cmd
->hint_root
);
371 disk_super
->discard_root
= cpu_to_le64(cmd
->discard_root
);
372 disk_super
->discard_block_size
= cpu_to_le64(cmd
->discard_block_size
);
373 disk_super
->discard_nr_blocks
= cpu_to_le64(from_dblock(cmd
->discard_nr_blocks
));
374 disk_super
->metadata_block_size
= cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE
);
375 disk_super
->data_block_size
= cpu_to_le32(cmd
->data_block_size
);
376 disk_super
->cache_blocks
= cpu_to_le32(0);
378 disk_super
->read_hits
= cpu_to_le32(0);
379 disk_super
->read_misses
= cpu_to_le32(0);
380 disk_super
->write_hits
= cpu_to_le32(0);
381 disk_super
->write_misses
= cpu_to_le32(0);
383 if (separate_dirty_bits(cmd
))
384 disk_super
->dirty_root
= cpu_to_le64(cmd
->dirty_root
);
386 return dm_tm_commit(cmd
->tm
, sblock
);
389 static int __format_metadata(struct dm_cache_metadata
*cmd
)
393 r
= dm_tm_create_with_sm(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
394 &cmd
->tm
, &cmd
->metadata_sm
);
396 DMERR("tm_create_with_sm failed");
400 __setup_mapping_info(cmd
);
402 r
= dm_array_empty(&cmd
->info
, &cmd
->root
);
406 if (separate_dirty_bits(cmd
)) {
407 dm_disk_bitset_init(cmd
->tm
, &cmd
->dirty_info
);
408 r
= dm_bitset_empty(&cmd
->dirty_info
, &cmd
->dirty_root
);
413 dm_disk_bitset_init(cmd
->tm
, &cmd
->discard_info
);
414 r
= dm_bitset_empty(&cmd
->discard_info
, &cmd
->discard_root
);
418 cmd
->discard_block_size
= 0;
419 cmd
->discard_nr_blocks
= 0;
421 r
= __write_initial_superblock(cmd
);
425 cmd
->clean_when_opened
= true;
429 dm_tm_destroy(cmd
->tm
);
430 dm_sm_destroy(cmd
->metadata_sm
);
435 static int __check_incompat_features(struct cache_disk_superblock
*disk_super
,
436 struct dm_cache_metadata
*cmd
)
438 uint32_t incompat_flags
, features
;
440 incompat_flags
= le32_to_cpu(disk_super
->incompat_flags
);
441 features
= incompat_flags
& ~DM_CACHE_FEATURE_INCOMPAT_SUPP
;
443 DMERR("could not access metadata due to unsupported optional features (%lx).",
444 (unsigned long)features
);
449 * Check for read-only metadata to skip the following RDWR checks.
451 if (get_disk_ro(cmd
->bdev
->bd_disk
))
454 features
= le32_to_cpu(disk_super
->compat_ro_flags
) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP
;
456 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
457 (unsigned long)features
);
464 static int __open_metadata(struct dm_cache_metadata
*cmd
)
467 struct dm_block
*sblock
;
468 struct cache_disk_superblock
*disk_super
;
469 unsigned long sb_flags
;
471 r
= superblock_read_lock(cmd
, &sblock
);
473 DMERR("couldn't read lock superblock");
477 disk_super
= dm_block_data(sblock
);
479 /* Verify the data block size hasn't changed */
480 if (le32_to_cpu(disk_super
->data_block_size
) != cmd
->data_block_size
) {
481 DMERR("changing the data block size (from %u to %llu) is not supported",
482 le32_to_cpu(disk_super
->data_block_size
),
483 (unsigned long long)cmd
->data_block_size
);
488 r
= __check_incompat_features(disk_super
, cmd
);
492 r
= dm_tm_open_with_sm(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
493 disk_super
->metadata_space_map_root
,
494 sizeof(disk_super
->metadata_space_map_root
),
495 &cmd
->tm
, &cmd
->metadata_sm
);
497 DMERR("tm_open_with_sm failed");
501 __setup_mapping_info(cmd
);
502 dm_disk_bitset_init(cmd
->tm
, &cmd
->dirty_info
);
503 dm_disk_bitset_init(cmd
->tm
, &cmd
->discard_info
);
504 sb_flags
= le32_to_cpu(disk_super
->flags
);
505 cmd
->clean_when_opened
= test_bit(CLEAN_SHUTDOWN
, &sb_flags
);
506 dm_bm_unlock(sblock
);
511 dm_bm_unlock(sblock
);
515 static int __open_or_format_metadata(struct dm_cache_metadata
*cmd
,
519 bool unformatted
= false;
521 r
= __superblock_all_zeroes(cmd
->bm
, &unformatted
);
526 return format_device
? __format_metadata(cmd
) : -EPERM
;
528 return __open_metadata(cmd
);
531 static int __create_persistent_data_objects(struct dm_cache_metadata
*cmd
,
532 bool may_format_device
)
535 cmd
->bm
= dm_block_manager_create(cmd
->bdev
, DM_CACHE_METADATA_BLOCK_SIZE
<< SECTOR_SHIFT
,
536 CACHE_MAX_CONCURRENT_LOCKS
);
537 if (IS_ERR(cmd
->bm
)) {
538 DMERR("could not create block manager");
539 return PTR_ERR(cmd
->bm
);
542 r
= __open_or_format_metadata(cmd
, may_format_device
);
544 dm_block_manager_destroy(cmd
->bm
);
549 static void __destroy_persistent_data_objects(struct dm_cache_metadata
*cmd
)
551 dm_sm_destroy(cmd
->metadata_sm
);
552 dm_tm_destroy(cmd
->tm
);
553 dm_block_manager_destroy(cmd
->bm
);
556 typedef unsigned long (*flags_mutator
)(unsigned long);
558 static void update_flags(struct cache_disk_superblock
*disk_super
,
559 flags_mutator mutator
)
561 uint32_t sb_flags
= mutator(le32_to_cpu(disk_super
->flags
));
562 disk_super
->flags
= cpu_to_le32(sb_flags
);
565 static unsigned long set_clean_shutdown(unsigned long flags
)
567 set_bit(CLEAN_SHUTDOWN
, &flags
);
571 static unsigned long clear_clean_shutdown(unsigned long flags
)
573 clear_bit(CLEAN_SHUTDOWN
, &flags
);
577 static void read_superblock_fields(struct dm_cache_metadata
*cmd
,
578 struct cache_disk_superblock
*disk_super
)
580 cmd
->version
= le32_to_cpu(disk_super
->version
);
581 cmd
->flags
= le32_to_cpu(disk_super
->flags
);
582 cmd
->root
= le64_to_cpu(disk_super
->mapping_root
);
583 cmd
->hint_root
= le64_to_cpu(disk_super
->hint_root
);
584 cmd
->discard_root
= le64_to_cpu(disk_super
->discard_root
);
585 cmd
->discard_block_size
= le64_to_cpu(disk_super
->discard_block_size
);
586 cmd
->discard_nr_blocks
= to_dblock(le64_to_cpu(disk_super
->discard_nr_blocks
));
587 cmd
->data_block_size
= le32_to_cpu(disk_super
->data_block_size
);
588 cmd
->cache_blocks
= to_cblock(le32_to_cpu(disk_super
->cache_blocks
));
589 strncpy(cmd
->policy_name
, disk_super
->policy_name
, sizeof(cmd
->policy_name
));
590 cmd
->policy_version
[0] = le32_to_cpu(disk_super
->policy_version
[0]);
591 cmd
->policy_version
[1] = le32_to_cpu(disk_super
->policy_version
[1]);
592 cmd
->policy_version
[2] = le32_to_cpu(disk_super
->policy_version
[2]);
593 cmd
->policy_hint_size
= le32_to_cpu(disk_super
->policy_hint_size
);
595 cmd
->stats
.read_hits
= le32_to_cpu(disk_super
->read_hits
);
596 cmd
->stats
.read_misses
= le32_to_cpu(disk_super
->read_misses
);
597 cmd
->stats
.write_hits
= le32_to_cpu(disk_super
->write_hits
);
598 cmd
->stats
.write_misses
= le32_to_cpu(disk_super
->write_misses
);
600 if (separate_dirty_bits(cmd
))
601 cmd
->dirty_root
= le64_to_cpu(disk_super
->dirty_root
);
603 cmd
->changed
= false;
607 * The mutator updates the superblock flags.
609 static int __begin_transaction_flags(struct dm_cache_metadata
*cmd
,
610 flags_mutator mutator
)
613 struct cache_disk_superblock
*disk_super
;
614 struct dm_block
*sblock
;
616 r
= superblock_lock(cmd
, &sblock
);
620 disk_super
= dm_block_data(sblock
);
621 update_flags(disk_super
, mutator
);
622 read_superblock_fields(cmd
, disk_super
);
623 dm_bm_unlock(sblock
);
625 return dm_bm_flush(cmd
->bm
);
628 static int __begin_transaction(struct dm_cache_metadata
*cmd
)
631 struct cache_disk_superblock
*disk_super
;
632 struct dm_block
*sblock
;
635 * We re-read the superblock every time. Shouldn't need to do this
638 r
= superblock_read_lock(cmd
, &sblock
);
642 disk_super
= dm_block_data(sblock
);
643 read_superblock_fields(cmd
, disk_super
);
644 dm_bm_unlock(sblock
);
649 static int __commit_transaction(struct dm_cache_metadata
*cmd
,
650 flags_mutator mutator
)
653 struct cache_disk_superblock
*disk_super
;
654 struct dm_block
*sblock
;
657 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
659 BUILD_BUG_ON(sizeof(struct cache_disk_superblock
) > 512);
661 if (separate_dirty_bits(cmd
)) {
662 r
= dm_bitset_flush(&cmd
->dirty_info
, cmd
->dirty_root
,
668 r
= dm_bitset_flush(&cmd
->discard_info
, cmd
->discard_root
,
673 r
= dm_tm_pre_commit(cmd
->tm
);
677 r
= __save_sm_root(cmd
);
681 r
= superblock_lock(cmd
, &sblock
);
685 disk_super
= dm_block_data(sblock
);
687 disk_super
->flags
= cpu_to_le32(cmd
->flags
);
689 update_flags(disk_super
, mutator
);
691 disk_super
->mapping_root
= cpu_to_le64(cmd
->root
);
692 if (separate_dirty_bits(cmd
))
693 disk_super
->dirty_root
= cpu_to_le64(cmd
->dirty_root
);
694 disk_super
->hint_root
= cpu_to_le64(cmd
->hint_root
);
695 disk_super
->discard_root
= cpu_to_le64(cmd
->discard_root
);
696 disk_super
->discard_block_size
= cpu_to_le64(cmd
->discard_block_size
);
697 disk_super
->discard_nr_blocks
= cpu_to_le64(from_dblock(cmd
->discard_nr_blocks
));
698 disk_super
->cache_blocks
= cpu_to_le32(from_cblock(cmd
->cache_blocks
));
699 strncpy(disk_super
->policy_name
, cmd
->policy_name
, sizeof(disk_super
->policy_name
));
700 disk_super
->policy_version
[0] = cpu_to_le32(cmd
->policy_version
[0]);
701 disk_super
->policy_version
[1] = cpu_to_le32(cmd
->policy_version
[1]);
702 disk_super
->policy_version
[2] = cpu_to_le32(cmd
->policy_version
[2]);
704 disk_super
->read_hits
= cpu_to_le32(cmd
->stats
.read_hits
);
705 disk_super
->read_misses
= cpu_to_le32(cmd
->stats
.read_misses
);
706 disk_super
->write_hits
= cpu_to_le32(cmd
->stats
.write_hits
);
707 disk_super
->write_misses
= cpu_to_le32(cmd
->stats
.write_misses
);
708 __copy_sm_root(cmd
, disk_super
);
710 return dm_tm_commit(cmd
->tm
, sblock
);
713 /*----------------------------------------------------------------*/
716 * The mappings are held in a dm-array that has 64-bit values stored in
717 * little-endian format. The index is the cblock, the high 48bits of the
718 * value are the oblock and the low 16 bit the flags.
720 #define FLAGS_MASK ((1 << 16) - 1)
722 static __le64
pack_value(dm_oblock_t block
, unsigned flags
)
724 uint64_t value
= from_oblock(block
);
726 value
= value
| (flags
& FLAGS_MASK
);
727 return cpu_to_le64(value
);
730 static void unpack_value(__le64 value_le
, dm_oblock_t
*block
, unsigned *flags
)
732 uint64_t value
= le64_to_cpu(value_le
);
733 uint64_t b
= value
>> 16;
734 *block
= to_oblock(b
);
735 *flags
= value
& FLAGS_MASK
;
738 /*----------------------------------------------------------------*/
740 static struct dm_cache_metadata
*metadata_open(struct block_device
*bdev
,
741 sector_t data_block_size
,
742 bool may_format_device
,
743 size_t policy_hint_size
,
744 unsigned metadata_version
)
747 struct dm_cache_metadata
*cmd
;
749 cmd
= kzalloc(sizeof(*cmd
), GFP_KERNEL
);
751 DMERR("could not allocate metadata struct");
752 return ERR_PTR(-ENOMEM
);
755 cmd
->version
= metadata_version
;
756 atomic_set(&cmd
->ref_count
, 1);
757 init_rwsem(&cmd
->root_lock
);
759 cmd
->data_block_size
= data_block_size
;
760 cmd
->cache_blocks
= 0;
761 cmd
->policy_hint_size
= policy_hint_size
;
763 cmd
->fail_io
= false;
765 r
= __create_persistent_data_objects(cmd
, may_format_device
);
771 r
= __begin_transaction_flags(cmd
, clear_clean_shutdown
);
773 dm_cache_metadata_close(cmd
);
781 * We keep a little list of ref counted metadata objects to prevent two
782 * different target instances creating separate bufio instances. This is
783 * an issue if a table is reloaded before the suspend.
785 static DEFINE_MUTEX(table_lock
);
786 static LIST_HEAD(table
);
788 static struct dm_cache_metadata
*lookup(struct block_device
*bdev
)
790 struct dm_cache_metadata
*cmd
;
792 list_for_each_entry(cmd
, &table
, list
)
793 if (cmd
->bdev
== bdev
) {
794 atomic_inc(&cmd
->ref_count
);
801 static struct dm_cache_metadata
*lookup_or_open(struct block_device
*bdev
,
802 sector_t data_block_size
,
803 bool may_format_device
,
804 size_t policy_hint_size
,
805 unsigned metadata_version
)
807 struct dm_cache_metadata
*cmd
, *cmd2
;
809 mutex_lock(&table_lock
);
811 mutex_unlock(&table_lock
);
816 cmd
= metadata_open(bdev
, data_block_size
, may_format_device
,
817 policy_hint_size
, metadata_version
);
819 mutex_lock(&table_lock
);
822 mutex_unlock(&table_lock
);
823 __destroy_persistent_data_objects(cmd
);
827 list_add(&cmd
->list
, &table
);
828 mutex_unlock(&table_lock
);
834 static bool same_params(struct dm_cache_metadata
*cmd
, sector_t data_block_size
)
836 if (cmd
->data_block_size
!= data_block_size
) {
837 DMERR("data_block_size (%llu) different from that in metadata (%llu)",
838 (unsigned long long) data_block_size
,
839 (unsigned long long) cmd
->data_block_size
);
846 struct dm_cache_metadata
*dm_cache_metadata_open(struct block_device
*bdev
,
847 sector_t data_block_size
,
848 bool may_format_device
,
849 size_t policy_hint_size
,
850 unsigned metadata_version
)
852 struct dm_cache_metadata
*cmd
= lookup_or_open(bdev
, data_block_size
, may_format_device
,
853 policy_hint_size
, metadata_version
);
855 if (!IS_ERR(cmd
) && !same_params(cmd
, data_block_size
)) {
856 dm_cache_metadata_close(cmd
);
857 return ERR_PTR(-EINVAL
);
863 void dm_cache_metadata_close(struct dm_cache_metadata
*cmd
)
865 if (atomic_dec_and_test(&cmd
->ref_count
)) {
866 mutex_lock(&table_lock
);
867 list_del(&cmd
->list
);
868 mutex_unlock(&table_lock
);
871 __destroy_persistent_data_objects(cmd
);
877 * Checks that the given cache block is either unmapped or clean.
879 static int block_clean_combined_dirty(struct dm_cache_metadata
*cmd
, dm_cblock_t b
,
887 r
= dm_array_get_value(&cmd
->info
, cmd
->root
, from_cblock(b
), &value
);
891 unpack_value(value
, &ob
, &flags
);
892 *result
= !((flags
& M_VALID
) && (flags
& M_DIRTY
));
897 static int blocks_are_clean_combined_dirty(struct dm_cache_metadata
*cmd
,
898 dm_cblock_t begin
, dm_cblock_t end
,
904 while (begin
!= end
) {
905 r
= block_clean_combined_dirty(cmd
, begin
, result
);
907 DMERR("block_clean_combined_dirty failed");
912 DMERR("cache block %llu is dirty",
913 (unsigned long long) from_cblock(begin
));
917 begin
= to_cblock(from_cblock(begin
) + 1);
923 static int blocks_are_clean_separate_dirty(struct dm_cache_metadata
*cmd
,
924 dm_cblock_t begin
, dm_cblock_t end
,
931 r
= dm_bitset_cursor_begin(&cmd
->dirty_info
, cmd
->dirty_root
,
932 from_cblock(cmd
->cache_blocks
), &cmd
->dirty_cursor
);
934 DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__
);
938 r
= dm_bitset_cursor_skip(&cmd
->dirty_cursor
, from_cblock(begin
));
940 DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__
);
941 dm_bitset_cursor_end(&cmd
->dirty_cursor
);
945 while (begin
!= end
) {
947 * We assume that unmapped blocks have their dirty bit
950 dirty_flag
= dm_bitset_cursor_get_value(&cmd
->dirty_cursor
);
952 DMERR("%s: cache block %llu is dirty", __func__
,
953 (unsigned long long) from_cblock(begin
));
954 dm_bitset_cursor_end(&cmd
->dirty_cursor
);
959 begin
= to_cblock(from_cblock(begin
) + 1);
963 r
= dm_bitset_cursor_next(&cmd
->dirty_cursor
);
965 DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__
);
966 dm_bitset_cursor_end(&cmd
->dirty_cursor
);
971 dm_bitset_cursor_end(&cmd
->dirty_cursor
);
976 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata
*cmd
,
977 dm_cblock_t begin
, dm_cblock_t end
,
980 if (separate_dirty_bits(cmd
))
981 return blocks_are_clean_separate_dirty(cmd
, begin
, end
, result
);
983 return blocks_are_clean_combined_dirty(cmd
, begin
, end
, result
);
986 static bool cmd_write_lock(struct dm_cache_metadata
*cmd
)
988 down_write(&cmd
->root_lock
);
989 if (cmd
->fail_io
|| dm_bm_is_read_only(cmd
->bm
)) {
990 up_write(&cmd
->root_lock
);
996 #define WRITE_LOCK(cmd) \
998 if (!cmd_write_lock((cmd))) \
1002 #define WRITE_LOCK_VOID(cmd) \
1004 if (!cmd_write_lock((cmd))) \
1008 #define WRITE_UNLOCK(cmd) \
1009 up_write(&(cmd)->root_lock)
1011 static bool cmd_read_lock(struct dm_cache_metadata
*cmd
)
1013 down_read(&cmd
->root_lock
);
1015 up_read(&cmd
->root_lock
);
1021 #define READ_LOCK(cmd) \
1023 if (!cmd_read_lock((cmd))) \
1027 #define READ_LOCK_VOID(cmd) \
1029 if (!cmd_read_lock((cmd))) \
1033 #define READ_UNLOCK(cmd) \
1034 up_read(&(cmd)->root_lock)
1036 int dm_cache_resize(struct dm_cache_metadata
*cmd
, dm_cblock_t new_cache_size
)
1040 __le64 null_mapping
= pack_value(0, 0);
1043 __dm_bless_for_disk(&null_mapping
);
1045 if (from_cblock(new_cache_size
) < from_cblock(cmd
->cache_blocks
)) {
1046 r
= blocks_are_unmapped_or_clean(cmd
, new_cache_size
, cmd
->cache_blocks
, &clean
);
1048 __dm_unbless_for_disk(&null_mapping
);
1053 DMERR("unable to shrink cache due to dirty blocks");
1055 __dm_unbless_for_disk(&null_mapping
);
1060 r
= dm_array_resize(&cmd
->info
, cmd
->root
, from_cblock(cmd
->cache_blocks
),
1061 from_cblock(new_cache_size
),
1062 &null_mapping
, &cmd
->root
);
1066 if (separate_dirty_bits(cmd
)) {
1067 r
= dm_bitset_resize(&cmd
->dirty_info
, cmd
->dirty_root
,
1068 from_cblock(cmd
->cache_blocks
), from_cblock(new_cache_size
),
1069 false, &cmd
->dirty_root
);
1074 cmd
->cache_blocks
= new_cache_size
;
1075 cmd
->changed
= true;
1083 int dm_cache_discard_bitset_resize(struct dm_cache_metadata
*cmd
,
1084 sector_t discard_block_size
,
1085 dm_dblock_t new_nr_entries
)
1090 r
= dm_bitset_resize(&cmd
->discard_info
,
1092 from_dblock(cmd
->discard_nr_blocks
),
1093 from_dblock(new_nr_entries
),
1094 false, &cmd
->discard_root
);
1096 cmd
->discard_block_size
= discard_block_size
;
1097 cmd
->discard_nr_blocks
= new_nr_entries
;
1100 cmd
->changed
= true;
1106 static int __set_discard(struct dm_cache_metadata
*cmd
, dm_dblock_t b
)
1108 return dm_bitset_set_bit(&cmd
->discard_info
, cmd
->discard_root
,
1109 from_dblock(b
), &cmd
->discard_root
);
1112 static int __clear_discard(struct dm_cache_metadata
*cmd
, dm_dblock_t b
)
1114 return dm_bitset_clear_bit(&cmd
->discard_info
, cmd
->discard_root
,
1115 from_dblock(b
), &cmd
->discard_root
);
1118 static int __discard(struct dm_cache_metadata
*cmd
,
1119 dm_dblock_t dblock
, bool discard
)
1123 r
= (discard
? __set_discard
: __clear_discard
)(cmd
, dblock
);
1127 cmd
->changed
= true;
1131 int dm_cache_set_discard(struct dm_cache_metadata
*cmd
,
1132 dm_dblock_t dblock
, bool discard
)
1137 r
= __discard(cmd
, dblock
, discard
);
1143 static int __load_discards(struct dm_cache_metadata
*cmd
,
1144 load_discard_fn fn
, void *context
)
1148 struct dm_bitset_cursor c
;
1150 if (from_dblock(cmd
->discard_nr_blocks
) == 0)
1154 if (cmd
->clean_when_opened
) {
1155 r
= dm_bitset_flush(&cmd
->discard_info
, cmd
->discard_root
, &cmd
->discard_root
);
1159 r
= dm_bitset_cursor_begin(&cmd
->discard_info
, cmd
->discard_root
,
1160 from_dblock(cmd
->discard_nr_blocks
), &c
);
1164 for (b
= 0; b
< from_dblock(cmd
->discard_nr_blocks
); b
++) {
1165 r
= fn(context
, cmd
->discard_block_size
, to_dblock(b
),
1166 dm_bitset_cursor_get_value(&c
));
1171 dm_bitset_cursor_end(&c
);
1174 for (b
= 0; b
< from_dblock(cmd
->discard_nr_blocks
); b
++) {
1175 r
= fn(context
, cmd
->discard_block_size
, to_dblock(b
), false);
1184 int dm_cache_load_discards(struct dm_cache_metadata
*cmd
,
1185 load_discard_fn fn
, void *context
)
1190 r
= __load_discards(cmd
, fn
, context
);
1196 int dm_cache_size(struct dm_cache_metadata
*cmd
, dm_cblock_t
*result
)
1199 *result
= cmd
->cache_blocks
;
1205 static int __remove(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
)
1208 __le64 value
= pack_value(0, 0);
1210 __dm_bless_for_disk(&value
);
1211 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1212 &value
, &cmd
->root
);
1216 cmd
->changed
= true;
1220 int dm_cache_remove_mapping(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
)
1225 r
= __remove(cmd
, cblock
);
1231 static int __insert(struct dm_cache_metadata
*cmd
,
1232 dm_cblock_t cblock
, dm_oblock_t oblock
)
1235 __le64 value
= pack_value(oblock
, M_VALID
);
1236 __dm_bless_for_disk(&value
);
1238 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1239 &value
, &cmd
->root
);
1243 cmd
->changed
= true;
1247 int dm_cache_insert_mapping(struct dm_cache_metadata
*cmd
,
1248 dm_cblock_t cblock
, dm_oblock_t oblock
)
1253 r
= __insert(cmd
, cblock
, oblock
);
1263 struct dm_cache_metadata
*cmd
;
1264 bool respect_dirty_flags
;
1268 static bool policy_unchanged(struct dm_cache_metadata
*cmd
,
1269 struct dm_cache_policy
*policy
)
1271 const char *policy_name
= dm_cache_policy_get_name(policy
);
1272 const unsigned *policy_version
= dm_cache_policy_get_version(policy
);
1273 size_t policy_hint_size
= dm_cache_policy_get_hint_size(policy
);
1276 * Ensure policy names match.
1278 if (strncmp(cmd
->policy_name
, policy_name
, sizeof(cmd
->policy_name
)))
1282 * Ensure policy major versions match.
1284 if (cmd
->policy_version
[0] != policy_version
[0])
1288 * Ensure policy hint sizes match.
1290 if (cmd
->policy_hint_size
!= policy_hint_size
)
1296 static bool hints_array_initialized(struct dm_cache_metadata
*cmd
)
1298 return cmd
->hint_root
&& cmd
->policy_hint_size
;
1301 static bool hints_array_available(struct dm_cache_metadata
*cmd
,
1302 struct dm_cache_policy
*policy
)
1304 return cmd
->clean_when_opened
&& policy_unchanged(cmd
, policy
) &&
1305 hints_array_initialized(cmd
);
1308 static int __load_mapping_v1(struct dm_cache_metadata
*cmd
,
1309 uint64_t cb
, bool hints_valid
,
1310 struct dm_array_cursor
*mapping_cursor
,
1311 struct dm_array_cursor
*hint_cursor
,
1312 load_mapping_fn fn
, void *context
)
1319 __le64
*mapping_value_le
;
1320 __le32
*hint_value_le
;
1325 dm_array_cursor_get_value(mapping_cursor
, (void **) &mapping_value_le
);
1326 memcpy(&mapping
, mapping_value_le
, sizeof(mapping
));
1327 unpack_value(mapping
, &oblock
, &flags
);
1329 if (flags
& M_VALID
) {
1331 dm_array_cursor_get_value(hint_cursor
, (void **) &hint_value_le
);
1332 memcpy(&hint
, hint_value_le
, sizeof(hint
));
1335 r
= fn(context
, oblock
, to_cblock(cb
), flags
& M_DIRTY
,
1336 le32_to_cpu(hint
), hints_valid
);
1338 DMERR("policy couldn't load cache block %llu",
1339 (unsigned long long) from_cblock(to_cblock(cb
)));
1346 static int __load_mapping_v2(struct dm_cache_metadata
*cmd
,
1347 uint64_t cb
, bool hints_valid
,
1348 struct dm_array_cursor
*mapping_cursor
,
1349 struct dm_array_cursor
*hint_cursor
,
1350 struct dm_bitset_cursor
*dirty_cursor
,
1351 load_mapping_fn fn
, void *context
)
1358 __le64
*mapping_value_le
;
1359 __le32
*hint_value_le
;
1365 dm_array_cursor_get_value(mapping_cursor
, (void **) &mapping_value_le
);
1366 memcpy(&mapping
, mapping_value_le
, sizeof(mapping
));
1367 unpack_value(mapping
, &oblock
, &flags
);
1369 if (flags
& M_VALID
) {
1371 dm_array_cursor_get_value(hint_cursor
, (void **) &hint_value_le
);
1372 memcpy(&hint
, hint_value_le
, sizeof(hint
));
1375 dirty
= dm_bitset_cursor_get_value(dirty_cursor
);
1376 r
= fn(context
, oblock
, to_cblock(cb
), dirty
,
1377 le32_to_cpu(hint
), hints_valid
);
1379 DMERR("policy couldn't load cache block %llu",
1380 (unsigned long long) from_cblock(to_cblock(cb
)));
1387 static int __load_mappings(struct dm_cache_metadata
*cmd
,
1388 struct dm_cache_policy
*policy
,
1389 load_mapping_fn fn
, void *context
)
1394 bool hints_valid
= hints_array_available(cmd
, policy
);
1396 if (from_cblock(cmd
->cache_blocks
) == 0)
1400 r
= dm_array_cursor_begin(&cmd
->info
, cmd
->root
, &cmd
->mapping_cursor
);
1405 r
= dm_array_cursor_begin(&cmd
->hint_info
, cmd
->hint_root
, &cmd
->hint_cursor
);
1407 dm_array_cursor_end(&cmd
->mapping_cursor
);
1412 if (separate_dirty_bits(cmd
)) {
1413 r
= dm_bitset_cursor_begin(&cmd
->dirty_info
, cmd
->dirty_root
,
1414 from_cblock(cmd
->cache_blocks
),
1415 &cmd
->dirty_cursor
);
1417 dm_array_cursor_end(&cmd
->hint_cursor
);
1418 dm_array_cursor_end(&cmd
->mapping_cursor
);
1423 for (cb
= 0; ; cb
++) {
1424 if (separate_dirty_bits(cmd
))
1425 r
= __load_mapping_v2(cmd
, cb
, hints_valid
,
1426 &cmd
->mapping_cursor
,
1431 r
= __load_mapping_v1(cmd
, cb
, hints_valid
,
1432 &cmd
->mapping_cursor
, &cmd
->hint_cursor
,
1438 * We need to break out before we move the cursors.
1440 if (cb
>= (from_cblock(cmd
->cache_blocks
) - 1))
1443 r
= dm_array_cursor_next(&cmd
->mapping_cursor
);
1445 DMERR("dm_array_cursor_next for mapping failed");
1450 r
= dm_array_cursor_next(&cmd
->hint_cursor
);
1452 DMERR("dm_array_cursor_next for hint failed");
1457 if (separate_dirty_bits(cmd
)) {
1458 r
= dm_bitset_cursor_next(&cmd
->dirty_cursor
);
1460 DMERR("dm_bitset_cursor_next for dirty failed");
1466 dm_array_cursor_end(&cmd
->mapping_cursor
);
1468 dm_array_cursor_end(&cmd
->hint_cursor
);
1470 if (separate_dirty_bits(cmd
))
1471 dm_bitset_cursor_end(&cmd
->dirty_cursor
);
1476 int dm_cache_load_mappings(struct dm_cache_metadata
*cmd
,
1477 struct dm_cache_policy
*policy
,
1478 load_mapping_fn fn
, void *context
)
1483 r
= __load_mappings(cmd
, policy
, fn
, context
);
1489 static int __dump_mapping(void *context
, uint64_t cblock
, void *leaf
)
1496 memcpy(&value
, leaf
, sizeof(value
));
1497 unpack_value(value
, &oblock
, &flags
);
1502 static int __dump_mappings(struct dm_cache_metadata
*cmd
)
1504 return dm_array_walk(&cmd
->info
, cmd
->root
, __dump_mapping
, NULL
);
1507 void dm_cache_dump(struct dm_cache_metadata
*cmd
)
1509 READ_LOCK_VOID(cmd
);
1510 __dump_mappings(cmd
);
1514 int dm_cache_changed_this_transaction(struct dm_cache_metadata
*cmd
)
1525 static int __dirty(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
, bool dirty
)
1532 r
= dm_array_get_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
), &value
);
1536 unpack_value(value
, &oblock
, &flags
);
1538 if (((flags
& M_DIRTY
) && dirty
) || (!(flags
& M_DIRTY
) && !dirty
))
1539 /* nothing to be done */
1542 value
= pack_value(oblock
, (flags
& ~M_DIRTY
) | (dirty
? M_DIRTY
: 0));
1543 __dm_bless_for_disk(&value
);
1545 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1546 &value
, &cmd
->root
);
1550 cmd
->changed
= true;
1555 static int __set_dirty_bits_v1(struct dm_cache_metadata
*cmd
, unsigned nr_bits
, unsigned long *bits
)
1559 for (i
= 0; i
< nr_bits
; i
++) {
1560 r
= __dirty(cmd
, to_cblock(i
), test_bit(i
, bits
));
1568 static int is_dirty_callback(uint32_t index
, bool *value
, void *context
)
1570 unsigned long *bits
= context
;
1571 *value
= test_bit(index
, bits
);
1575 static int __set_dirty_bits_v2(struct dm_cache_metadata
*cmd
, unsigned nr_bits
, unsigned long *bits
)
1579 /* nr_bits is really just a sanity check */
1580 if (nr_bits
!= from_cblock(cmd
->cache_blocks
)) {
1581 DMERR("dirty bitset is wrong size");
1585 r
= dm_bitset_del(&cmd
->dirty_info
, cmd
->dirty_root
);
1589 cmd
->changed
= true;
1590 return dm_bitset_new(&cmd
->dirty_info
, &cmd
->dirty_root
, nr_bits
, is_dirty_callback
, bits
);
1593 int dm_cache_set_dirty_bits(struct dm_cache_metadata
*cmd
,
1595 unsigned long *bits
)
1600 if (separate_dirty_bits(cmd
))
1601 r
= __set_dirty_bits_v2(cmd
, nr_bits
, bits
);
1603 r
= __set_dirty_bits_v1(cmd
, nr_bits
, bits
);
1609 void dm_cache_metadata_get_stats(struct dm_cache_metadata
*cmd
,
1610 struct dm_cache_statistics
*stats
)
1612 READ_LOCK_VOID(cmd
);
1613 *stats
= cmd
->stats
;
1617 void dm_cache_metadata_set_stats(struct dm_cache_metadata
*cmd
,
1618 struct dm_cache_statistics
*stats
)
1620 WRITE_LOCK_VOID(cmd
);
1621 cmd
->stats
= *stats
;
1625 int dm_cache_commit(struct dm_cache_metadata
*cmd
, bool clean_shutdown
)
1628 flags_mutator mutator
= (clean_shutdown
? set_clean_shutdown
:
1629 clear_clean_shutdown
);
1635 r
= __commit_transaction(cmd
, mutator
);
1639 r
= __begin_transaction(cmd
);
1645 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata
*cmd
,
1652 r
= dm_sm_get_nr_free(cmd
->metadata_sm
, result
);
1658 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata
*cmd
,
1665 r
= dm_sm_get_nr_blocks(cmd
->metadata_sm
, result
);
1671 /*----------------------------------------------------------------*/
1673 static int get_hint(uint32_t index
, void *value_le
, void *context
)
1676 struct dm_cache_policy
*policy
= context
;
1678 value
= policy_get_hint(policy
, to_cblock(index
));
1679 *((__le32
*) value_le
) = cpu_to_le32(value
);
1685 * It's quicker to always delete the hint array, and recreate with
1688 static int write_hints(struct dm_cache_metadata
*cmd
, struct dm_cache_policy
*policy
)
1692 const char *policy_name
= dm_cache_policy_get_name(policy
);
1693 const unsigned *policy_version
= dm_cache_policy_get_version(policy
);
1695 if (!policy_name
[0] ||
1696 (strlen(policy_name
) > sizeof(cmd
->policy_name
) - 1))
1699 strncpy(cmd
->policy_name
, policy_name
, sizeof(cmd
->policy_name
));
1700 memcpy(cmd
->policy_version
, policy_version
, sizeof(cmd
->policy_version
));
1702 hint_size
= dm_cache_policy_get_hint_size(policy
);
1704 return 0; /* short-circuit hints initialization */
1705 cmd
->policy_hint_size
= hint_size
;
1707 if (cmd
->hint_root
) {
1708 r
= dm_array_del(&cmd
->hint_info
, cmd
->hint_root
);
1713 return dm_array_new(&cmd
->hint_info
, &cmd
->hint_root
,
1714 from_cblock(cmd
->cache_blocks
),
1718 int dm_cache_write_hints(struct dm_cache_metadata
*cmd
, struct dm_cache_policy
*policy
)
1723 r
= write_hints(cmd
, policy
);
1729 int dm_cache_metadata_all_clean(struct dm_cache_metadata
*cmd
, bool *result
)
1734 r
= blocks_are_unmapped_or_clean(cmd
, 0, cmd
->cache_blocks
, result
);
1740 void dm_cache_metadata_set_read_only(struct dm_cache_metadata
*cmd
)
1742 WRITE_LOCK_VOID(cmd
);
1743 dm_bm_set_read_only(cmd
->bm
);
1747 void dm_cache_metadata_set_read_write(struct dm_cache_metadata
*cmd
)
1749 WRITE_LOCK_VOID(cmd
);
1750 dm_bm_set_read_write(cmd
->bm
);
1754 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata
*cmd
)
1757 struct dm_block
*sblock
;
1758 struct cache_disk_superblock
*disk_super
;
1761 set_bit(NEEDS_CHECK
, &cmd
->flags
);
1763 r
= superblock_lock(cmd
, &sblock
);
1765 DMERR("couldn't read superblock");
1769 disk_super
= dm_block_data(sblock
);
1770 disk_super
->flags
= cpu_to_le32(cmd
->flags
);
1772 dm_bm_unlock(sblock
);
1779 int dm_cache_metadata_needs_check(struct dm_cache_metadata
*cmd
, bool *result
)
1782 *result
= !!test_bit(NEEDS_CHECK
, &cmd
->flags
);
1788 int dm_cache_metadata_abort(struct dm_cache_metadata
*cmd
)
1793 __destroy_persistent_data_objects(cmd
);
1794 r
= __create_persistent_data_objects(cmd
, false);
1796 cmd
->fail_io
= true;