4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2020, 2021, 2022 by Pawel Jakub Dawidek
26 #include <sys/zfs_context.h>
28 #include <sys/spa_impl.h>
31 #include <sys/brt_impl.h>
33 #include <sys/bitmap.h>
35 #include <sys/dmu_tx.h>
37 #include <sys/dsl_pool.h>
38 #include <sys/dsl_scan.h>
39 #include <sys/vdev_impl.h>
40 #include <sys/kstat.h>
41 #include <sys/wmsum.h>
44 * Block Cloning design.
46 * Block Cloning allows to manually clone a file (or a subset of its blocks)
47 * into another (or the same) file by just creating additional references to
48 * the data blocks without copying the data itself. Those references are kept
49 * in the Block Reference Tables (BRTs).
51 * In many ways this is similar to the existing deduplication, but there are
52 * some important differences:
54 * - Deduplication is automatic and Block Cloning is not - one has to use a
55 * dedicated system call(s) to clone the given file/blocks.
56 * - Deduplication keeps all data blocks in its table, even those referenced
57 * just once. Block Cloning creates an entry in its tables only when there
58 * are at least two references to the given data block. If the block was
59 * never explicitly cloned or the second to last reference was dropped,
60 * there will be neither space nor performance overhead.
61 * - Deduplication needs data to work - one needs to pass real data to the
62 * write(2) syscall, so hash can be calculated. Block Cloning doesn't require
63 * data, just block pointers to the data, so it is extremely fast, as we pay
64 * neither the cost of reading the data, nor the cost of writing the data -
65 * we operate exclusively on metadata.
66 * - If the D (dedup) bit is not set in the block pointer, it means that
67 * the block is not in the dedup table (DDT) and we won't consult the DDT
68 * when we need to free the block. Block Cloning must be consulted on every
69 * free, because we cannot modify the source BP (eg. by setting something
70 * similar to the D bit), thus we have no hint if the block is in the
71 * Block Reference Table (BRT), so we need to look into the BRT. There is
72 * an optimization in place that allows us to eliminate the majority of BRT
73 * lookups which is described below in the "Minimizing free penalty" section.
74 * - The BRT entry is much smaller than the DDT entry - for BRT we only store
75 * 64bit offset and 64bit reference counter.
76 * - Dedup keys are cryptographic hashes, so two blocks that are close to each
77 * other on disk are most likely in totally different parts of the DDT.
78 * The BRT entry keys are offsets into a single top-level VDEV, so data blocks
79 * from one file should have BRT entries close to each other.
80 * - Scrub will only do a single pass over a block that is referenced multiple
81 * times in the DDT. Unfortunately it is not currently (if at all) possible
82 * with Block Cloning and block referenced multiple times will be scrubbed
83 * multiple times. The new, sorted scrub should be able to eliminate
84 * duplicated reads given enough memory.
85 * - Deduplication requires cryptographically strong hash as a checksum or
86 * additional data verification. Block Cloning works with any checksum
87 * algorithm or even with checksumming disabled.
89 * As mentioned above, the BRT entries are much smaller than the DDT entries.
90 * To uniquely identify a block we just need its vdev id and offset. We also
91 * need to maintain a reference counter. The vdev id will often repeat, as there
92 * is a small number of top-level VDEVs and a large number of blocks stored in
93 * each VDEV. We take advantage of that to reduce the BRT entry size further by
94 * maintaining one BRT for each top-level VDEV, so we can then have only offset
95 * and counter as the BRT entry.
97 * Minimizing free penalty.
99 * Block Cloning allows creating additional references to any existing block.
100 * When we free a block there is no hint in the block pointer whether the block
101 * was cloned or not, so on each free we have to check if there is a
102 * corresponding entry in the BRT or not. If there is, we need to decrease
103 * the reference counter. Doing BRT lookup on every free can potentially be
104 * expensive by requiring additional I/Os if the BRT doesn't fit into memory.
105 * This is the main problem with deduplication, so we've learned our lesson and
106 * try not to repeat the same mistake here. How do we do that? We divide each
107 * top-level VDEV into 16MB regions. For each region we maintain a counter that
108 * is a sum of all the BRT entries that have offsets within the region. This
109 * creates the entries count array of 16bit numbers for each top-level VDEV.
110 * The entries count array is always kept in memory and updated on disk in the
111 * same transaction group as the BRT updates to keep everything in-sync. We can
112 * keep the array in memory, because it is very small. With 16MB regions and
113 * 1TB VDEV the array requires only 128kB of memory (we may decide to decrease
114 * the region size even further in the future). Now, when we want to free
115 * a block, we first consult the array. If the counter for the whole region is
116 * zero, there is no need to look for the BRT entry, as there isn't one for
117 * sure. If the counter for the region is greater than zero, only then we will
118 * do a BRT lookup and if an entry is found we will decrease the reference
119 * counter in the BRT entry and in the entry counters array.
121 * The entry counters array is small, but can potentially be larger for very
122 * large VDEVs or smaller regions. In this case we don't want to rewrite entire
123 * array on every change. We then divide the array into 32kB block and keep
124 * a bitmap of dirty blocks within a transaction group. When we sync the
125 * transaction group we can only update the parts of the entry counters array
126 * that were modified. Note: Keeping track of the dirty parts of the entry
127 * counters array is implemented, but updating only parts of the array on disk
128 * is not yet implemented - for now we will update entire array if there was
131 * The implementation tries to be economic: if BRT is not used, or no longer
132 * used, there will be no entries in the MOS and no additional memory used (eg.
133 * the entry counters array is only allocated if needed).
135 * Interaction between Deduplication and Block Cloning.
137 * If both functionalities are in use, we could end up with a block that is
138 * referenced multiple times in both DDT and BRT. When we free one of the
139 * references we couldn't tell where it belongs, so we would have to decide
140 * what table takes the precedence: do we first clear DDT references or BRT
141 * references? To avoid this dilemma BRT cooperates with DDT - if a given block
142 * is being cloned using BRT and the BP has the D (dedup) bit set, BRT will
143 * lookup DDT entry instead and increase the counter there. No BRT entry
144 * will be created for a block which has the D (dedup) bit set.
145 * BRT may be more efficient for manual deduplication, but if the block is
146 * already in the DDT, then creating additional BRT entry would be less
147 * efficient. This clever idea was proposed by Allan Jude.
149 * Block Cloning across datasets.
151 * Block Cloning is not limited to cloning blocks within the same dataset.
152 * It is possible (and very useful) to clone blocks between different datasets.
153 * One use case is recovering files from snapshots. By cloning the files into
154 * dataset we need no additional storage. Without Block Cloning we would need
155 * additional space for those files.
156 * Another interesting use case is moving the files between datasets
157 * (copying the file content to the new dataset and removing the source file).
158 * In that case Block Cloning will only be used briefly, because the BRT entries
159 * will be removed when the source is removed.
160 * Block Cloning across encrypted datasets is supported as long as both
161 * datasets share the same master key (e.g. snapshots and clones)
163 * Block Cloning flow through ZFS layers.
165 * Note: Block Cloning can be used both for cloning file system blocks and ZVOL
166 * blocks. As of this writing no interface is implemented that allows for block
167 * cloning within a ZVOL.
168 * FreeBSD and Linux provides copy_file_range(2) system call and we will use it
169 * for blocking cloning.
172 * copy_file_range(int infd, off_t *inoffp, int outfd, off_t *outoffp,
173 * size_t len, unsigned int flags);
175 * Even though offsets and length represent bytes, they have to be
176 * block-aligned or we will return an error so the upper layer can
177 * fallback to the generic mechanism that will just copy the data.
178 * Using copy_file_range(2) will call OS-independent zfs_clone_range() function.
179 * This function was implemented based on zfs_write(), but instead of writing
180 * the given data we first read block pointers using the new dmu_read_l0_bps()
181 * function from the source file. Once we have BPs from the source file we call
182 * the dmu_brt_clone() function on the destination file. This function
183 * allocates BPs for us. We iterate over all source BPs. If the given BP is
184 * a hole or an embedded block, we just copy BP as-is. If it points to a real
185 * data we place this BP on a BRT pending list using the brt_pending_add()
188 * We use this pending list to keep track of all BPs that got new references
189 * within this transaction group.
191 * Some special cases to consider and how we address them:
192 * - The block we want to clone may have been created within the same
193 * transaction group that we are trying to clone. Such block has no BP
194 * allocated yet, so cannot be immediately cloned. We return EAGAIN.
195 * - The block we want to clone may have been modified within the same
196 * transaction group. We return EAGAIN.
197 * - A block may be cloned multiple times during one transaction group (that's
198 * why pending list is actually a tree and not an append-only list - this
199 * way we can figure out faster if this block is cloned for the first time
200 * in this txg or consecutive time).
201 * - A block may be cloned and freed within the same transaction group
202 * (see dbuf_undirty()).
203 * - A block may be cloned and within the same transaction group the clone
204 * can be cloned again (see dmu_read_l0_bps()).
205 * - A file might have been deleted, but the caller still has a file descriptor
206 * open to this file and clones it.
208 * When we free a block we have an additional step in the ZIO pipeline where we
209 * call the zio_brt_free() function. We then call the brt_entry_decref()
210 * that loads the corresponding BRT entry (if one exists) and decreases
211 * reference counter. If this is not the last reference we will stop ZIO
212 * pipeline here. If this is the last reference or the block is not in the
213 * BRT, we continue the pipeline and free the block as usual.
215 * At the beginning of spa_sync() where there can be no more block cloning,
216 * but before issuing frees we call brt_pending_apply(). This function applies
217 * all the new clones to the BRT table - we load BRT entries and update
218 * reference counters. To sync new BRT entries to disk, we use brt_sync()
219 * function. This function will sync all dirty per-top-level-vdev BRTs,
220 * the entry counters arrays, etc.
222 * Block Cloning and ZIL.
224 * Every clone operation is divided into chunks (similar to write) and each
225 * chunk is cloned in a separate transaction. The chunk size is determined by
226 * how many BPs we can fit into a single ZIL entry.
227 * Replaying clone operation is different from the regular clone operation,
228 * as when we log clone operations we cannot use the source object - it may
229 * reside on a different dataset, so we log BPs we want to clone.
230 * The ZIL is replayed when we mount the given dataset, not when the pool is
231 * imported. Taking this into account it is possible that the pool is imported
232 * without mounting datasets and the source dataset is destroyed before the
233 * destination dataset is mounted and its ZIL replayed.
234 * To address this situation we leverage zil_claim() mechanism where ZFS will
235 * parse all the ZILs on pool import. When we come across TX_CLONE_RANGE
236 * entries, we will bump reference counters for their BPs in the BRT. Then
237 * on mount and ZIL replay we bump the reference counters once more, while the
238 * first references are dropped during ZIL destroy by zil_free_clone_range().
239 * It is possible that after zil_claim() we never mount the destination, so
240 * we never replay its ZIL and just destroy it. In this case the only taken
241 * references will be dropped by zil_free_clone_range(), since the cloning is
242 * not going to ever take place.
245 static kmem_cache_t
*brt_entry_cache
;
246 static kmem_cache_t
*brt_pending_entry_cache
;
249 * Enable/disable prefetching of BRT entries that we are going to modify.
251 int zfs_brt_prefetch
= 1;
254 #define BRT_DEBUG(...) do { \
255 if ((zfs_flags & ZFS_DEBUG_BRT) != 0) { \
256 __dprintf(B_TRUE, __FILE__, __func__, __LINE__, __VA_ARGS__); \
260 #define BRT_DEBUG(...) do { } while (0)
263 int brt_zap_leaf_blockshift
= 12;
264 int brt_zap_indirect_blockshift
= 12;
266 static kstat_t
*brt_ksp
;
268 typedef struct brt_stats
{
269 kstat_named_t brt_addref_entry_in_memory
;
270 kstat_named_t brt_addref_entry_not_on_disk
;
271 kstat_named_t brt_addref_entry_on_disk
;
272 kstat_named_t brt_addref_entry_read_lost_race
;
273 kstat_named_t brt_decref_entry_in_memory
;
274 kstat_named_t brt_decref_entry_loaded_from_disk
;
275 kstat_named_t brt_decref_entry_not_in_memory
;
276 kstat_named_t brt_decref_entry_not_on_disk
;
277 kstat_named_t brt_decref_entry_read_lost_race
;
278 kstat_named_t brt_decref_entry_still_referenced
;
279 kstat_named_t brt_decref_free_data_later
;
280 kstat_named_t brt_decref_free_data_now
;
281 kstat_named_t brt_decref_no_entry
;
284 static brt_stats_t brt_stats
= {
285 { "addref_entry_in_memory", KSTAT_DATA_UINT64
},
286 { "addref_entry_not_on_disk", KSTAT_DATA_UINT64
},
287 { "addref_entry_on_disk", KSTAT_DATA_UINT64
},
288 { "addref_entry_read_lost_race", KSTAT_DATA_UINT64
},
289 { "decref_entry_in_memory", KSTAT_DATA_UINT64
},
290 { "decref_entry_loaded_from_disk", KSTAT_DATA_UINT64
},
291 { "decref_entry_not_in_memory", KSTAT_DATA_UINT64
},
292 { "decref_entry_not_on_disk", KSTAT_DATA_UINT64
},
293 { "decref_entry_read_lost_race", KSTAT_DATA_UINT64
},
294 { "decref_entry_still_referenced", KSTAT_DATA_UINT64
},
295 { "decref_free_data_later", KSTAT_DATA_UINT64
},
296 { "decref_free_data_now", KSTAT_DATA_UINT64
},
297 { "decref_no_entry", KSTAT_DATA_UINT64
}
301 wmsum_t brt_addref_entry_in_memory
;
302 wmsum_t brt_addref_entry_not_on_disk
;
303 wmsum_t brt_addref_entry_on_disk
;
304 wmsum_t brt_addref_entry_read_lost_race
;
305 wmsum_t brt_decref_entry_in_memory
;
306 wmsum_t brt_decref_entry_loaded_from_disk
;
307 wmsum_t brt_decref_entry_not_in_memory
;
308 wmsum_t brt_decref_entry_not_on_disk
;
309 wmsum_t brt_decref_entry_read_lost_race
;
310 wmsum_t brt_decref_entry_still_referenced
;
311 wmsum_t brt_decref_free_data_later
;
312 wmsum_t brt_decref_free_data_now
;
313 wmsum_t brt_decref_no_entry
;
316 #define BRTSTAT_BUMP(stat) wmsum_add(&brt_sums.stat, 1)
318 static int brt_entry_compare(const void *x1
, const void *x2
);
319 static int brt_pending_entry_compare(const void *x1
, const void *x2
);
322 brt_rlock(brt_t
*brt
)
324 rw_enter(&brt
->brt_lock
, RW_READER
);
328 brt_wlock(brt_t
*brt
)
330 rw_enter(&brt
->brt_lock
, RW_WRITER
);
334 brt_unlock(brt_t
*brt
)
336 rw_exit(&brt
->brt_lock
);
340 brt_vdev_entcount_get(const brt_vdev_t
*brtvd
, uint64_t idx
)
343 ASSERT3U(idx
, <, brtvd
->bv_size
);
345 if (brtvd
->bv_need_byteswap
) {
346 return (BSWAP_16(brtvd
->bv_entcount
[idx
]));
348 return (brtvd
->bv_entcount
[idx
]);
353 brt_vdev_entcount_set(brt_vdev_t
*brtvd
, uint64_t idx
, uint16_t entcnt
)
356 ASSERT3U(idx
, <, brtvd
->bv_size
);
358 if (brtvd
->bv_need_byteswap
) {
359 brtvd
->bv_entcount
[idx
] = BSWAP_16(entcnt
);
361 brtvd
->bv_entcount
[idx
] = entcnt
;
366 brt_vdev_entcount_inc(brt_vdev_t
*brtvd
, uint64_t idx
)
370 ASSERT3U(idx
, <, brtvd
->bv_size
);
372 entcnt
= brt_vdev_entcount_get(brtvd
, idx
);
373 ASSERT(entcnt
< UINT16_MAX
);
375 brt_vdev_entcount_set(brtvd
, idx
, entcnt
+ 1);
379 brt_vdev_entcount_dec(brt_vdev_t
*brtvd
, uint64_t idx
)
383 ASSERT3U(idx
, <, brtvd
->bv_size
);
385 entcnt
= brt_vdev_entcount_get(brtvd
, idx
);
388 brt_vdev_entcount_set(brtvd
, idx
, entcnt
- 1);
393 brt_vdev_dump(brt_t
*brt
)
398 if ((zfs_flags
& ZFS_DEBUG_BRT
) == 0) {
402 if (brt
->brt_nvdevs
== 0) {
403 zfs_dbgmsg("BRT empty");
407 zfs_dbgmsg("BRT vdev dump:");
408 for (vdevid
= 0; vdevid
< brt
->brt_nvdevs
; vdevid
++) {
411 brtvd
= &brt
->brt_vdevs
[vdevid
];
412 zfs_dbgmsg(" vdevid=%llu/%llu meta_dirty=%d entcount_dirty=%d "
413 "size=%llu totalcount=%llu nblocks=%llu bitmapsize=%zu\n",
414 (u_longlong_t
)vdevid
, (u_longlong_t
)brtvd
->bv_vdevid
,
415 brtvd
->bv_meta_dirty
, brtvd
->bv_entcount_dirty
,
416 (u_longlong_t
)brtvd
->bv_size
,
417 (u_longlong_t
)brtvd
->bv_totalcount
,
418 (u_longlong_t
)brtvd
->bv_nblocks
,
419 (size_t)BT_SIZEOFMAP(brtvd
->bv_nblocks
));
420 if (brtvd
->bv_totalcount
> 0) {
421 zfs_dbgmsg(" entcounts:");
422 for (idx
= 0; idx
< brtvd
->bv_size
; idx
++) {
423 if (brt_vdev_entcount_get(brtvd
, idx
) > 0) {
424 zfs_dbgmsg(" [%04llu] %hu",
426 brt_vdev_entcount_get(brtvd
, idx
));
430 if (brtvd
->bv_entcount_dirty
) {
433 bitmap
= kmem_alloc(brtvd
->bv_nblocks
+ 1, KM_SLEEP
);
434 for (idx
= 0; idx
< brtvd
->bv_nblocks
; idx
++) {
436 BT_TEST(brtvd
->bv_bitmap
, idx
) ? 'x' : '.';
439 zfs_dbgmsg(" bitmap: %s", bitmap
);
440 kmem_free(bitmap
, brtvd
->bv_nblocks
+ 1);
447 brt_vdev(brt_t
*brt
, uint64_t vdevid
)
451 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
453 if (vdevid
< brt
->brt_nvdevs
) {
454 brtvd
= &brt
->brt_vdevs
[vdevid
];
463 brt_vdev_create(brt_t
*brt
, brt_vdev_t
*brtvd
, dmu_tx_t
*tx
)
467 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
468 ASSERT0(brtvd
->bv_mos_brtvdev
);
469 ASSERT0(brtvd
->bv_mos_entries
);
470 ASSERT(brtvd
->bv_entcount
!= NULL
);
471 ASSERT(brtvd
->bv_size
> 0);
472 ASSERT(brtvd
->bv_bitmap
!= NULL
);
473 ASSERT(brtvd
->bv_nblocks
> 0);
475 brtvd
->bv_mos_entries
= zap_create_flags(brt
->brt_mos
, 0,
476 ZAP_FLAG_HASH64
| ZAP_FLAG_UINT64_KEY
, DMU_OTN_ZAP_METADATA
,
477 brt_zap_leaf_blockshift
, brt_zap_indirect_blockshift
, DMU_OT_NONE
,
479 VERIFY(brtvd
->bv_mos_entries
!= 0);
480 BRT_DEBUG("MOS entries created, object=%llu",
481 (u_longlong_t
)brtvd
->bv_mos_entries
);
484 * We allocate DMU buffer to store the bv_entcount[] array.
485 * We will keep array size (bv_size) and cummulative count for all
486 * bv_entcount[]s (bv_totalcount) in the bonus buffer.
488 brtvd
->bv_mos_brtvdev
= dmu_object_alloc(brt
->brt_mos
,
489 DMU_OTN_UINT64_METADATA
, BRT_BLOCKSIZE
,
490 DMU_OTN_UINT64_METADATA
, sizeof (brt_vdev_phys_t
), tx
);
491 VERIFY(brtvd
->bv_mos_brtvdev
!= 0);
492 BRT_DEBUG("MOS BRT VDEV created, object=%llu",
493 (u_longlong_t
)brtvd
->bv_mos_brtvdev
);
495 snprintf(name
, sizeof (name
), "%s%llu", BRT_OBJECT_VDEV_PREFIX
,
496 (u_longlong_t
)brtvd
->bv_vdevid
);
497 VERIFY0(zap_add(brt
->brt_mos
, DMU_POOL_DIRECTORY_OBJECT
, name
,
498 sizeof (uint64_t), 1, &brtvd
->bv_mos_brtvdev
, tx
));
499 BRT_DEBUG("Pool directory object created, object=%s", name
);
501 spa_feature_incr(brt
->brt_spa
, SPA_FEATURE_BLOCK_CLONING
, tx
);
505 brt_vdev_realloc(brt_t
*brt
, brt_vdev_t
*brtvd
)
510 uint64_t nblocks
, size
;
512 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
514 spa_config_enter(brt
->brt_spa
, SCL_VDEV
, FTAG
, RW_READER
);
515 vd
= vdev_lookup_top(brt
->brt_spa
, brtvd
->bv_vdevid
);
516 size
= (vdev_get_min_asize(vd
) - 1) / brt
->brt_rangesize
+ 1;
517 spa_config_exit(brt
->brt_spa
, SCL_VDEV
, FTAG
);
519 entcount
= vmem_zalloc(sizeof (entcount
[0]) * size
, KM_SLEEP
);
520 nblocks
= BRT_RANGESIZE_TO_NBLOCKS(size
);
521 bitmap
= kmem_zalloc(BT_SIZEOFMAP(nblocks
), KM_SLEEP
);
523 if (!brtvd
->bv_initiated
) {
524 ASSERT0(brtvd
->bv_size
);
525 ASSERT(brtvd
->bv_entcount
== NULL
);
526 ASSERT(brtvd
->bv_bitmap
== NULL
);
527 ASSERT0(brtvd
->bv_nblocks
);
529 avl_create(&brtvd
->bv_tree
, brt_entry_compare
,
530 sizeof (brt_entry_t
), offsetof(brt_entry_t
, bre_node
));
532 ASSERT(brtvd
->bv_size
> 0);
533 ASSERT(brtvd
->bv_entcount
!= NULL
);
534 ASSERT(brtvd
->bv_bitmap
!= NULL
);
535 ASSERT(brtvd
->bv_nblocks
> 0);
537 * TODO: Allow vdev shrinking. We only need to implement
538 * shrinking the on-disk BRT VDEV object.
539 * dmu_free_range(brt->brt_mos, brtvd->bv_mos_brtvdev, offset,
542 ASSERT3U(brtvd
->bv_size
, <=, size
);
544 memcpy(entcount
, brtvd
->bv_entcount
,
545 sizeof (entcount
[0]) * MIN(size
, brtvd
->bv_size
));
546 memcpy(bitmap
, brtvd
->bv_bitmap
, MIN(BT_SIZEOFMAP(nblocks
),
547 BT_SIZEOFMAP(brtvd
->bv_nblocks
)));
548 vmem_free(brtvd
->bv_entcount
,
549 sizeof (entcount
[0]) * brtvd
->bv_size
);
550 kmem_free(brtvd
->bv_bitmap
, BT_SIZEOFMAP(brtvd
->bv_nblocks
));
553 brtvd
->bv_size
= size
;
554 brtvd
->bv_entcount
= entcount
;
555 brtvd
->bv_bitmap
= bitmap
;
556 brtvd
->bv_nblocks
= nblocks
;
557 if (!brtvd
->bv_initiated
) {
558 brtvd
->bv_need_byteswap
= FALSE
;
559 brtvd
->bv_initiated
= TRUE
;
560 BRT_DEBUG("BRT VDEV %llu initiated.",
561 (u_longlong_t
)brtvd
->bv_vdevid
);
566 brt_vdev_load(brt_t
*brt
, brt_vdev_t
*brtvd
)
570 brt_vdev_phys_t
*bvphys
;
573 snprintf(name
, sizeof (name
), "%s%llu", BRT_OBJECT_VDEV_PREFIX
,
574 (u_longlong_t
)brtvd
->bv_vdevid
);
575 error
= zap_lookup(brt
->brt_mos
, DMU_POOL_DIRECTORY_OBJECT
, name
,
576 sizeof (uint64_t), 1, &brtvd
->bv_mos_brtvdev
);
579 ASSERT(brtvd
->bv_mos_brtvdev
!= 0);
581 error
= dmu_bonus_hold(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, FTAG
, &db
);
586 bvphys
= db
->db_data
;
587 if (brt
->brt_rangesize
== 0) {
588 brt
->brt_rangesize
= bvphys
->bvp_rangesize
;
590 ASSERT3U(brt
->brt_rangesize
, ==, bvphys
->bvp_rangesize
);
593 ASSERT(!brtvd
->bv_initiated
);
594 brt_vdev_realloc(brt
, brtvd
);
596 /* TODO: We don't support VDEV shrinking. */
597 ASSERT3U(bvphys
->bvp_size
, <=, brtvd
->bv_size
);
600 * If VDEV grew, we will leave new bv_entcount[] entries zeroed out.
602 error
= dmu_read(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, 0,
603 MIN(brtvd
->bv_size
, bvphys
->bvp_size
) * sizeof (uint16_t),
604 brtvd
->bv_entcount
, DMU_READ_NO_PREFETCH
);
607 brtvd
->bv_mos_entries
= bvphys
->bvp_mos_entries
;
608 ASSERT(brtvd
->bv_mos_entries
!= 0);
609 brtvd
->bv_need_byteswap
=
610 (bvphys
->bvp_byteorder
!= BRT_NATIVE_BYTEORDER
);
611 brtvd
->bv_totalcount
= bvphys
->bvp_totalcount
;
612 brtvd
->bv_usedspace
= bvphys
->bvp_usedspace
;
613 brtvd
->bv_savedspace
= bvphys
->bvp_savedspace
;
614 brt
->brt_usedspace
+= brtvd
->bv_usedspace
;
615 brt
->brt_savedspace
+= brtvd
->bv_savedspace
;
617 dmu_buf_rele(db
, FTAG
);
619 BRT_DEBUG("MOS BRT VDEV %s loaded: mos_brtvdev=%llu, mos_entries=%llu",
620 name
, (u_longlong_t
)brtvd
->bv_mos_brtvdev
,
621 (u_longlong_t
)brtvd
->bv_mos_entries
);
625 brt_vdev_dealloc(brt_t
*brt
, brt_vdev_t
*brtvd
)
628 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
629 ASSERT(brtvd
->bv_initiated
);
631 vmem_free(brtvd
->bv_entcount
, sizeof (uint16_t) * brtvd
->bv_size
);
632 brtvd
->bv_entcount
= NULL
;
633 kmem_free(brtvd
->bv_bitmap
, BT_SIZEOFMAP(brtvd
->bv_nblocks
));
634 brtvd
->bv_bitmap
= NULL
;
635 ASSERT0(avl_numnodes(&brtvd
->bv_tree
));
636 avl_destroy(&brtvd
->bv_tree
);
639 brtvd
->bv_nblocks
= 0;
641 brtvd
->bv_initiated
= FALSE
;
642 BRT_DEBUG("BRT VDEV %llu deallocated.", (u_longlong_t
)brtvd
->bv_vdevid
);
646 brt_vdev_destroy(brt_t
*brt
, brt_vdev_t
*brtvd
, dmu_tx_t
*tx
)
651 brt_vdev_phys_t
*bvphys
;
653 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
654 ASSERT(brtvd
->bv_mos_brtvdev
!= 0);
655 ASSERT(brtvd
->bv_mos_entries
!= 0);
657 VERIFY0(zap_count(brt
->brt_mos
, brtvd
->bv_mos_entries
, &count
));
659 VERIFY0(zap_destroy(brt
->brt_mos
, brtvd
->bv_mos_entries
, tx
));
660 BRT_DEBUG("MOS entries destroyed, object=%llu",
661 (u_longlong_t
)brtvd
->bv_mos_entries
);
662 brtvd
->bv_mos_entries
= 0;
664 VERIFY0(dmu_bonus_hold(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, FTAG
, &db
));
665 bvphys
= db
->db_data
;
666 ASSERT0(bvphys
->bvp_totalcount
);
667 ASSERT0(bvphys
->bvp_usedspace
);
668 ASSERT0(bvphys
->bvp_savedspace
);
669 dmu_buf_rele(db
, FTAG
);
671 VERIFY0(dmu_object_free(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, tx
));
672 BRT_DEBUG("MOS BRT VDEV destroyed, object=%llu",
673 (u_longlong_t
)brtvd
->bv_mos_brtvdev
);
674 brtvd
->bv_mos_brtvdev
= 0;
676 snprintf(name
, sizeof (name
), "%s%llu", BRT_OBJECT_VDEV_PREFIX
,
677 (u_longlong_t
)brtvd
->bv_vdevid
);
678 VERIFY0(zap_remove(brt
->brt_mos
, DMU_POOL_DIRECTORY_OBJECT
, name
, tx
));
679 BRT_DEBUG("Pool directory object removed, object=%s", name
);
681 brt_vdev_dealloc(brt
, brtvd
);
683 spa_feature_decr(brt
->brt_spa
, SPA_FEATURE_BLOCK_CLONING
, tx
);
687 brt_vdevs_expand(brt_t
*brt
, uint64_t nvdevs
)
689 brt_vdev_t
*brtvd
, *vdevs
;
692 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
693 ASSERT3U(nvdevs
, >, brt
->brt_nvdevs
);
695 vdevs
= kmem_zalloc(sizeof (vdevs
[0]) * nvdevs
, KM_SLEEP
);
696 if (brt
->brt_nvdevs
> 0) {
697 ASSERT(brt
->brt_vdevs
!= NULL
);
699 memcpy(vdevs
, brt
->brt_vdevs
,
700 sizeof (brt_vdev_t
) * brt
->brt_nvdevs
);
701 kmem_free(brt
->brt_vdevs
,
702 sizeof (brt_vdev_t
) * brt
->brt_nvdevs
);
704 for (vdevid
= brt
->brt_nvdevs
; vdevid
< nvdevs
; vdevid
++) {
705 brtvd
= &vdevs
[vdevid
];
707 brtvd
->bv_vdevid
= vdevid
;
708 brtvd
->bv_initiated
= FALSE
;
711 BRT_DEBUG("BRT VDEVs expanded from %llu to %llu.",
712 (u_longlong_t
)brt
->brt_nvdevs
, (u_longlong_t
)nvdevs
);
714 brt
->brt_vdevs
= vdevs
;
715 brt
->brt_nvdevs
= nvdevs
;
719 brt_vdev_lookup(brt_t
*brt
, brt_vdev_t
*brtvd
, const brt_entry_t
*bre
)
723 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
725 idx
= bre
->bre_offset
/ brt
->brt_rangesize
;
726 if (brtvd
->bv_entcount
!= NULL
&& idx
< brtvd
->bv_size
) {
727 /* VDEV wasn't expanded. */
728 return (brt_vdev_entcount_get(brtvd
, idx
) > 0);
735 brt_vdev_addref(brt_t
*brt
, brt_vdev_t
*brtvd
, const brt_entry_t
*bre
,
740 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
741 ASSERT(brtvd
!= NULL
);
742 ASSERT(brtvd
->bv_entcount
!= NULL
);
744 brt
->brt_savedspace
+= dsize
;
745 brtvd
->bv_savedspace
+= dsize
;
746 brtvd
->bv_meta_dirty
= TRUE
;
748 if (bre
->bre_refcount
> 1) {
752 brt
->brt_usedspace
+= dsize
;
753 brtvd
->bv_usedspace
+= dsize
;
755 idx
= bre
->bre_offset
/ brt
->brt_rangesize
;
756 if (idx
>= brtvd
->bv_size
) {
757 /* VDEV has been expanded. */
758 brt_vdev_realloc(brt
, brtvd
);
761 ASSERT3U(idx
, <, brtvd
->bv_size
);
763 brtvd
->bv_totalcount
++;
764 brt_vdev_entcount_inc(brtvd
, idx
);
765 brtvd
->bv_entcount_dirty
= TRUE
;
766 idx
= idx
/ BRT_BLOCKSIZE
/ 8;
767 BT_SET(brtvd
->bv_bitmap
, idx
);
775 brt_vdev_decref(brt_t
*brt
, brt_vdev_t
*brtvd
, const brt_entry_t
*bre
,
780 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
781 ASSERT(brtvd
!= NULL
);
782 ASSERT(brtvd
->bv_entcount
!= NULL
);
784 brt
->brt_savedspace
-= dsize
;
785 brtvd
->bv_savedspace
-= dsize
;
786 brtvd
->bv_meta_dirty
= TRUE
;
788 if (bre
->bre_refcount
> 0) {
792 brt
->brt_usedspace
-= dsize
;
793 brtvd
->bv_usedspace
-= dsize
;
795 idx
= bre
->bre_offset
/ brt
->brt_rangesize
;
796 ASSERT3U(idx
, <, brtvd
->bv_size
);
798 ASSERT(brtvd
->bv_totalcount
> 0);
799 brtvd
->bv_totalcount
--;
800 brt_vdev_entcount_dec(brtvd
, idx
);
801 brtvd
->bv_entcount_dirty
= TRUE
;
802 idx
= idx
/ BRT_BLOCKSIZE
/ 8;
803 BT_SET(brtvd
->bv_bitmap
, idx
);
811 brt_vdev_sync(brt_t
*brt
, brt_vdev_t
*brtvd
, dmu_tx_t
*tx
)
814 brt_vdev_phys_t
*bvphys
;
816 ASSERT(brtvd
->bv_meta_dirty
);
817 ASSERT(brtvd
->bv_mos_brtvdev
!= 0);
818 ASSERT(dmu_tx_is_syncing(tx
));
820 VERIFY0(dmu_bonus_hold(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, FTAG
, &db
));
822 if (brtvd
->bv_entcount_dirty
) {
824 * TODO: Walk brtvd->bv_bitmap and write only the dirty blocks.
826 dmu_write(brt
->brt_mos
, brtvd
->bv_mos_brtvdev
, 0,
827 brtvd
->bv_size
* sizeof (brtvd
->bv_entcount
[0]),
828 brtvd
->bv_entcount
, tx
);
829 memset(brtvd
->bv_bitmap
, 0, BT_SIZEOFMAP(brtvd
->bv_nblocks
));
830 brtvd
->bv_entcount_dirty
= FALSE
;
833 dmu_buf_will_dirty(db
, tx
);
834 bvphys
= db
->db_data
;
835 bvphys
->bvp_mos_entries
= brtvd
->bv_mos_entries
;
836 bvphys
->bvp_size
= brtvd
->bv_size
;
837 if (brtvd
->bv_need_byteswap
) {
838 bvphys
->bvp_byteorder
= BRT_NON_NATIVE_BYTEORDER
;
840 bvphys
->bvp_byteorder
= BRT_NATIVE_BYTEORDER
;
842 bvphys
->bvp_totalcount
= brtvd
->bv_totalcount
;
843 bvphys
->bvp_rangesize
= brt
->brt_rangesize
;
844 bvphys
->bvp_usedspace
= brtvd
->bv_usedspace
;
845 bvphys
->bvp_savedspace
= brtvd
->bv_savedspace
;
846 dmu_buf_rele(db
, FTAG
);
848 brtvd
->bv_meta_dirty
= FALSE
;
852 brt_vdevs_alloc(brt_t
*brt
, boolean_t load
)
859 brt_vdevs_expand(brt
, brt
->brt_spa
->spa_root_vdev
->vdev_children
);
862 for (vdevid
= 0; vdevid
< brt
->brt_nvdevs
; vdevid
++) {
863 brtvd
= &brt
->brt_vdevs
[vdevid
];
864 ASSERT(brtvd
->bv_entcount
== NULL
);
866 brt_vdev_load(brt
, brtvd
);
870 if (brt
->brt_rangesize
== 0) {
871 brt
->brt_rangesize
= BRT_RANGESIZE
;
878 brt_vdevs_free(brt_t
*brt
)
885 for (vdevid
= 0; vdevid
< brt
->brt_nvdevs
; vdevid
++) {
886 brtvd
= &brt
->brt_vdevs
[vdevid
];
887 if (brtvd
->bv_initiated
)
888 brt_vdev_dealloc(brt
, brtvd
);
890 kmem_free(brt
->brt_vdevs
, sizeof (brt_vdev_t
) * brt
->brt_nvdevs
);
896 brt_entry_fill(const blkptr_t
*bp
, brt_entry_t
*bre
, uint64_t *vdevidp
)
899 bre
->bre_offset
= DVA_GET_OFFSET(&bp
->blk_dva
[0]);
900 bre
->bre_refcount
= 0;
902 *vdevidp
= DVA_GET_VDEV(&bp
->blk_dva
[0]);
906 brt_entry_compare(const void *x1
, const void *x2
)
908 const brt_entry_t
*bre1
= x1
;
909 const brt_entry_t
*bre2
= x2
;
911 return (TREE_CMP(bre1
->bre_offset
, bre2
->bre_offset
));
915 brt_entry_lookup(brt_t
*brt
, brt_vdev_t
*brtvd
, brt_entry_t
*bre
)
917 uint64_t mos_entries
;
918 uint64_t one
, physsize
;
921 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
923 if (!brt_vdev_lookup(brt
, brtvd
, bre
))
924 return (SET_ERROR(ENOENT
));
927 * Remember mos_entries object number. After we reacquire the BRT lock,
928 * the brtvd pointer may be invalid.
930 mos_entries
= brtvd
->bv_mos_entries
;
931 if (mos_entries
== 0)
932 return (SET_ERROR(ENOENT
));
936 error
= zap_length_uint64(brt
->brt_mos
, mos_entries
, &bre
->bre_offset
,
937 BRT_KEY_WORDS
, &one
, &physsize
);
939 ASSERT3U(one
, ==, 1);
940 ASSERT3U(physsize
, ==, sizeof (bre
->bre_refcount
));
942 error
= zap_lookup_uint64(brt
->brt_mos
, mos_entries
,
943 &bre
->bre_offset
, BRT_KEY_WORDS
, 1,
944 sizeof (bre
->bre_refcount
), &bre
->bre_refcount
);
945 BRT_DEBUG("ZAP lookup: object=%llu vdev=%llu offset=%llu "
946 "count=%llu error=%d", (u_longlong_t
)mos_entries
,
947 (u_longlong_t
)brtvd
->bv_vdevid
,
948 (u_longlong_t
)bre
->bre_offset
,
949 error
== 0 ? (u_longlong_t
)bre
->bre_refcount
: 0, error
);
958 brt_entry_prefetch(brt_t
*brt
, uint64_t vdevid
, brt_entry_t
*bre
)
961 uint64_t mos_entries
= 0;
964 brtvd
= brt_vdev(brt
, vdevid
);
966 mos_entries
= brtvd
->bv_mos_entries
;
969 if (mos_entries
== 0)
972 BRT_DEBUG("ZAP prefetch: object=%llu vdev=%llu offset=%llu",
973 (u_longlong_t
)mos_entries
, (u_longlong_t
)vdevid
,
974 (u_longlong_t
)bre
->bre_offset
);
975 (void) zap_prefetch_uint64(brt
->brt_mos
, mos_entries
,
976 (uint64_t *)&bre
->bre_offset
, BRT_KEY_WORDS
);
980 brt_entry_update(brt_t
*brt
, brt_vdev_t
*brtvd
, brt_entry_t
*bre
, dmu_tx_t
*tx
)
984 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
985 ASSERT(brtvd
->bv_mos_entries
!= 0);
986 ASSERT(bre
->bre_refcount
> 0);
988 error
= zap_update_uint64(brt
->brt_mos
, brtvd
->bv_mos_entries
,
989 (uint64_t *)&bre
->bre_offset
, BRT_KEY_WORDS
, 1,
990 sizeof (bre
->bre_refcount
), &bre
->bre_refcount
, tx
);
991 BRT_DEBUG("ZAP update: object=%llu vdev=%llu offset=%llu count=%llu "
992 "error=%d", (u_longlong_t
)brtvd
->bv_mos_entries
,
993 (u_longlong_t
)brtvd
->bv_vdevid
, (u_longlong_t
)bre
->bre_offset
,
994 (u_longlong_t
)bre
->bre_refcount
, error
);
1000 brt_entry_remove(brt_t
*brt
, brt_vdev_t
*brtvd
, brt_entry_t
*bre
, dmu_tx_t
*tx
)
1004 ASSERT(RW_LOCK_HELD(&brt
->brt_lock
));
1005 ASSERT(brtvd
->bv_mos_entries
!= 0);
1006 ASSERT0(bre
->bre_refcount
);
1008 error
= zap_remove_uint64(brt
->brt_mos
, brtvd
->bv_mos_entries
,
1009 (uint64_t *)&bre
->bre_offset
, BRT_KEY_WORDS
, tx
);
1010 BRT_DEBUG("ZAP remove: object=%llu vdev=%llu offset=%llu count=%llu "
1011 "error=%d", (u_longlong_t
)brtvd
->bv_mos_entries
,
1012 (u_longlong_t
)brtvd
->bv_vdevid
, (u_longlong_t
)bre
->bre_offset
,
1013 (u_longlong_t
)bre
->bre_refcount
, error
);
1019 * Return TRUE if we _can_ have BRT entry for this bp. It might be false
1020 * positive, but gives us quick answer if we should look into BRT, which
1021 * may require reads and thus will be more expensive.
1024 brt_maybe_exists(spa_t
*spa
, const blkptr_t
*bp
)
1026 brt_t
*brt
= spa
->spa_brt
;
1028 brt_entry_t bre_search
;
1029 boolean_t mayexists
= FALSE
;
1032 brt_entry_fill(bp
, &bre_search
, &vdevid
);
1036 brtvd
= brt_vdev(brt
, vdevid
);
1037 if (brtvd
!= NULL
&& brtvd
->bv_initiated
) {
1038 if (!avl_is_empty(&brtvd
->bv_tree
) ||
1039 brt_vdev_lookup(brt
, brtvd
, &bre_search
)) {
1050 brt_get_dspace(spa_t
*spa
)
1052 brt_t
*brt
= spa
->spa_brt
;
1057 return (brt
->brt_savedspace
);
1061 brt_get_used(spa_t
*spa
)
1063 brt_t
*brt
= spa
->spa_brt
;
1068 return (brt
->brt_usedspace
);
1072 brt_get_saved(spa_t
*spa
)
1074 brt_t
*brt
= spa
->spa_brt
;
1079 return (brt
->brt_savedspace
);
1083 brt_get_ratio(spa_t
*spa
)
1085 brt_t
*brt
= spa
->spa_brt
;
1087 if (brt
->brt_usedspace
== 0)
1090 return ((brt
->brt_usedspace
+ brt
->brt_savedspace
) * 100 /
1091 brt
->brt_usedspace
);
1095 brt_kstats_update(kstat_t
*ksp
, int rw
)
1097 brt_stats_t
*bs
= ksp
->ks_data
;
1099 if (rw
== KSTAT_WRITE
)
1102 bs
->brt_addref_entry_in_memory
.value
.ui64
=
1103 wmsum_value(&brt_sums
.brt_addref_entry_in_memory
);
1104 bs
->brt_addref_entry_not_on_disk
.value
.ui64
=
1105 wmsum_value(&brt_sums
.brt_addref_entry_not_on_disk
);
1106 bs
->brt_addref_entry_on_disk
.value
.ui64
=
1107 wmsum_value(&brt_sums
.brt_addref_entry_on_disk
);
1108 bs
->brt_addref_entry_read_lost_race
.value
.ui64
=
1109 wmsum_value(&brt_sums
.brt_addref_entry_read_lost_race
);
1110 bs
->brt_decref_entry_in_memory
.value
.ui64
=
1111 wmsum_value(&brt_sums
.brt_decref_entry_in_memory
);
1112 bs
->brt_decref_entry_loaded_from_disk
.value
.ui64
=
1113 wmsum_value(&brt_sums
.brt_decref_entry_loaded_from_disk
);
1114 bs
->brt_decref_entry_not_in_memory
.value
.ui64
=
1115 wmsum_value(&brt_sums
.brt_decref_entry_not_in_memory
);
1116 bs
->brt_decref_entry_not_on_disk
.value
.ui64
=
1117 wmsum_value(&brt_sums
.brt_decref_entry_not_on_disk
);
1118 bs
->brt_decref_entry_read_lost_race
.value
.ui64
=
1119 wmsum_value(&brt_sums
.brt_decref_entry_read_lost_race
);
1120 bs
->brt_decref_entry_still_referenced
.value
.ui64
=
1121 wmsum_value(&brt_sums
.brt_decref_entry_still_referenced
);
1122 bs
->brt_decref_free_data_later
.value
.ui64
=
1123 wmsum_value(&brt_sums
.brt_decref_free_data_later
);
1124 bs
->brt_decref_free_data_now
.value
.ui64
=
1125 wmsum_value(&brt_sums
.brt_decref_free_data_now
);
1126 bs
->brt_decref_no_entry
.value
.ui64
=
1127 wmsum_value(&brt_sums
.brt_decref_no_entry
);
1136 wmsum_init(&brt_sums
.brt_addref_entry_in_memory
, 0);
1137 wmsum_init(&brt_sums
.brt_addref_entry_not_on_disk
, 0);
1138 wmsum_init(&brt_sums
.brt_addref_entry_on_disk
, 0);
1139 wmsum_init(&brt_sums
.brt_addref_entry_read_lost_race
, 0);
1140 wmsum_init(&brt_sums
.brt_decref_entry_in_memory
, 0);
1141 wmsum_init(&brt_sums
.brt_decref_entry_loaded_from_disk
, 0);
1142 wmsum_init(&brt_sums
.brt_decref_entry_not_in_memory
, 0);
1143 wmsum_init(&brt_sums
.brt_decref_entry_not_on_disk
, 0);
1144 wmsum_init(&brt_sums
.brt_decref_entry_read_lost_race
, 0);
1145 wmsum_init(&brt_sums
.brt_decref_entry_still_referenced
, 0);
1146 wmsum_init(&brt_sums
.brt_decref_free_data_later
, 0);
1147 wmsum_init(&brt_sums
.brt_decref_free_data_now
, 0);
1148 wmsum_init(&brt_sums
.brt_decref_no_entry
, 0);
1150 brt_ksp
= kstat_create("zfs", 0, "brtstats", "misc", KSTAT_TYPE_NAMED
,
1151 sizeof (brt_stats
) / sizeof (kstat_named_t
), KSTAT_FLAG_VIRTUAL
);
1152 if (brt_ksp
!= NULL
) {
1153 brt_ksp
->ks_data
= &brt_stats
;
1154 brt_ksp
->ks_update
= brt_kstats_update
;
1155 kstat_install(brt_ksp
);
1162 if (brt_ksp
!= NULL
) {
1163 kstat_delete(brt_ksp
);
1167 wmsum_fini(&brt_sums
.brt_addref_entry_in_memory
);
1168 wmsum_fini(&brt_sums
.brt_addref_entry_not_on_disk
);
1169 wmsum_fini(&brt_sums
.brt_addref_entry_on_disk
);
1170 wmsum_fini(&brt_sums
.brt_addref_entry_read_lost_race
);
1171 wmsum_fini(&brt_sums
.brt_decref_entry_in_memory
);
1172 wmsum_fini(&brt_sums
.brt_decref_entry_loaded_from_disk
);
1173 wmsum_fini(&brt_sums
.brt_decref_entry_not_in_memory
);
1174 wmsum_fini(&brt_sums
.brt_decref_entry_not_on_disk
);
1175 wmsum_fini(&brt_sums
.brt_decref_entry_read_lost_race
);
1176 wmsum_fini(&brt_sums
.brt_decref_entry_still_referenced
);
1177 wmsum_fini(&brt_sums
.brt_decref_free_data_later
);
1178 wmsum_fini(&brt_sums
.brt_decref_free_data_now
);
1179 wmsum_fini(&brt_sums
.brt_decref_no_entry
);
1185 brt_entry_cache
= kmem_cache_create("brt_entry_cache",
1186 sizeof (brt_entry_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
1187 brt_pending_entry_cache
= kmem_cache_create("brt_pending_entry_cache",
1188 sizeof (brt_pending_entry_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
1198 kmem_cache_destroy(brt_entry_cache
);
1199 kmem_cache_destroy(brt_pending_entry_cache
);
1202 static brt_entry_t
*
1203 brt_entry_alloc(const brt_entry_t
*bre_init
)
1207 bre
= kmem_cache_alloc(brt_entry_cache
, KM_SLEEP
);
1208 bre
->bre_offset
= bre_init
->bre_offset
;
1209 bre
->bre_refcount
= bre_init
->bre_refcount
;
1215 brt_entry_free(brt_entry_t
*bre
)
1218 kmem_cache_free(brt_entry_cache
, bre
);
1222 brt_entry_addref(brt_t
*brt
, const blkptr_t
*bp
)
1225 brt_entry_t
*bre
, *racebre
;
1226 brt_entry_t bre_search
;
1231 ASSERT(!RW_WRITE_HELD(&brt
->brt_lock
));
1233 brt_entry_fill(bp
, &bre_search
, &vdevid
);
1237 brtvd
= brt_vdev(brt
, vdevid
);
1238 if (brtvd
== NULL
) {
1239 ASSERT3U(vdevid
, >=, brt
->brt_nvdevs
);
1241 /* New VDEV was added. */
1242 brt_vdevs_expand(brt
, vdevid
+ 1);
1243 brtvd
= brt_vdev(brt
, vdevid
);
1245 ASSERT(brtvd
!= NULL
);
1246 if (!brtvd
->bv_initiated
)
1247 brt_vdev_realloc(brt
, brtvd
);
1249 bre
= avl_find(&brtvd
->bv_tree
, &bre_search
, NULL
);
1251 BRTSTAT_BUMP(brt_addref_entry_in_memory
);
1254 * brt_entry_lookup() may drop the BRT (read) lock and
1255 * reacquire it (write).
1257 error
= brt_entry_lookup(brt
, brtvd
, &bre_search
);
1258 /* bre_search now contains correct bre_refcount */
1259 ASSERT(error
== 0 || error
== ENOENT
);
1261 BRTSTAT_BUMP(brt_addref_entry_on_disk
);
1263 BRTSTAT_BUMP(brt_addref_entry_not_on_disk
);
1265 * When the BRT lock was dropped, brt_vdevs[] may have been
1266 * expanded and reallocated, we need to update brtvd's pointer.
1268 brtvd
= brt_vdev(brt
, vdevid
);
1269 ASSERT(brtvd
!= NULL
);
1271 racebre
= avl_find(&brtvd
->bv_tree
, &bre_search
, &where
);
1272 if (racebre
== NULL
) {
1273 bre
= brt_entry_alloc(&bre_search
);
1274 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
1275 avl_insert(&brtvd
->bv_tree
, bre
, where
);
1276 brt
->brt_nentries
++;
1279 * The entry was added when the BRT lock was dropped in
1280 * brt_entry_lookup().
1282 BRTSTAT_BUMP(brt_addref_entry_read_lost_race
);
1286 bre
->bre_refcount
++;
1287 brt_vdev_addref(brt
, brtvd
, bre
, bp_get_dsize(brt
->brt_spa
, bp
));
1292 /* Return TRUE if block should be freed immediately. */
1294 brt_entry_decref(spa_t
*spa
, const blkptr_t
*bp
)
1296 brt_t
*brt
= spa
->spa_brt
;
1298 brt_entry_t
*bre
, *racebre
;
1299 brt_entry_t bre_search
;
1304 brt_entry_fill(bp
, &bre_search
, &vdevid
);
1308 brtvd
= brt_vdev(brt
, vdevid
);
1309 ASSERT(brtvd
!= NULL
);
1311 bre
= avl_find(&brtvd
->bv_tree
, &bre_search
, NULL
);
1313 BRTSTAT_BUMP(brt_decref_entry_in_memory
);
1316 BRTSTAT_BUMP(brt_decref_entry_not_in_memory
);
1320 * brt_entry_lookup() may drop the BRT lock and reacquire it.
1322 error
= brt_entry_lookup(brt
, brtvd
, &bre_search
);
1323 /* bre_search now contains correct bre_refcount */
1324 ASSERT(error
== 0 || error
== ENOENT
);
1326 * When the BRT lock was dropped, brt_vdevs[] may have been expanded
1327 * and reallocated, we need to update brtvd's pointer.
1329 brtvd
= brt_vdev(brt
, vdevid
);
1330 ASSERT(brtvd
!= NULL
);
1332 if (error
== ENOENT
) {
1333 BRTSTAT_BUMP(brt_decref_entry_not_on_disk
);
1338 racebre
= avl_find(&brtvd
->bv_tree
, &bre_search
, &where
);
1339 if (racebre
!= NULL
) {
1341 * The entry was added when the BRT lock was dropped in
1342 * brt_entry_lookup().
1344 BRTSTAT_BUMP(brt_decref_entry_read_lost_race
);
1349 BRTSTAT_BUMP(brt_decref_entry_loaded_from_disk
);
1350 bre
= brt_entry_alloc(&bre_search
);
1351 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
1352 avl_insert(&brtvd
->bv_tree
, bre
, where
);
1353 brt
->brt_nentries
++;
1358 * This is a free of a regular (not cloned) block.
1361 BRTSTAT_BUMP(brt_decref_no_entry
);
1364 if (bre
->bre_refcount
== 0) {
1366 BRTSTAT_BUMP(brt_decref_free_data_now
);
1370 ASSERT(bre
->bre_refcount
> 0);
1371 bre
->bre_refcount
--;
1372 if (bre
->bre_refcount
== 0)
1373 BRTSTAT_BUMP(brt_decref_free_data_later
);
1375 BRTSTAT_BUMP(brt_decref_entry_still_referenced
);
1376 brt_vdev_decref(brt
, brtvd
, bre
, bp_get_dsize(brt
->brt_spa
, bp
));
1384 brt_entry_get_refcount(spa_t
*spa
, const blkptr_t
*bp
)
1386 brt_t
*brt
= spa
->spa_brt
;
1388 brt_entry_t bre_search
, *bre
;
1389 uint64_t vdevid
, refcnt
;
1392 brt_entry_fill(bp
, &bre_search
, &vdevid
);
1396 brtvd
= brt_vdev(brt
, vdevid
);
1397 ASSERT(brtvd
!= NULL
);
1399 bre
= avl_find(&brtvd
->bv_tree
, &bre_search
, NULL
);
1401 error
= brt_entry_lookup(brt
, brtvd
, &bre_search
);
1402 ASSERT(error
== 0 || error
== ENOENT
);
1403 if (error
== ENOENT
)
1406 refcnt
= bre_search
.bre_refcount
;
1408 refcnt
= bre
->bre_refcount
;
1415 brt_prefetch(brt_t
*brt
, const blkptr_t
*bp
)
1422 if (!zfs_brt_prefetch
)
1425 brt_entry_fill(bp
, &bre
, &vdevid
);
1427 brt_entry_prefetch(brt
, vdevid
, &bre
);
1431 brt_pending_entry_compare(const void *x1
, const void *x2
)
1433 const brt_pending_entry_t
*bpe1
= x1
, *bpe2
= x2
;
1434 const blkptr_t
*bp1
= &bpe1
->bpe_bp
, *bp2
= &bpe2
->bpe_bp
;
1437 cmp
= TREE_CMP(BP_PHYSICAL_BIRTH(bp1
), BP_PHYSICAL_BIRTH(bp2
));
1439 cmp
= TREE_CMP(DVA_GET_VDEV(&bp1
->blk_dva
[0]),
1440 DVA_GET_VDEV(&bp2
->blk_dva
[0]));
1442 cmp
= TREE_CMP(DVA_GET_OFFSET(&bp1
->blk_dva
[0]),
1443 DVA_GET_OFFSET(&bp2
->blk_dva
[0]));
1451 brt_pending_add(spa_t
*spa
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
1454 avl_tree_t
*pending_tree
;
1455 kmutex_t
*pending_lock
;
1456 brt_pending_entry_t
*bpe
, *newbpe
;
1461 txg
= dmu_tx_get_txg(tx
);
1462 ASSERT3U(txg
, !=, 0);
1463 pending_tree
= &brt
->brt_pending_tree
[txg
& TXG_MASK
];
1464 pending_lock
= &brt
->brt_pending_lock
[txg
& TXG_MASK
];
1466 newbpe
= kmem_cache_alloc(brt_pending_entry_cache
, KM_SLEEP
);
1467 newbpe
->bpe_bp
= *bp
;
1468 newbpe
->bpe_count
= 1;
1470 mutex_enter(pending_lock
);
1472 bpe
= avl_find(pending_tree
, newbpe
, &where
);
1474 avl_insert(pending_tree
, newbpe
, where
);
1480 mutex_exit(pending_lock
);
1482 if (newbpe
!= NULL
) {
1483 ASSERT(bpe
!= NULL
);
1484 ASSERT(bpe
!= newbpe
);
1485 kmem_cache_free(brt_pending_entry_cache
, newbpe
);
1487 ASSERT(bpe
== NULL
);
1490 /* Prefetch BRT entry, as we will need it in the syncing context. */
1491 brt_prefetch(brt
, bp
);
1495 brt_pending_remove(spa_t
*spa
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
1498 avl_tree_t
*pending_tree
;
1499 kmutex_t
*pending_lock
;
1500 brt_pending_entry_t
*bpe
, bpe_search
;
1504 txg
= dmu_tx_get_txg(tx
);
1505 ASSERT3U(txg
, !=, 0);
1506 pending_tree
= &brt
->brt_pending_tree
[txg
& TXG_MASK
];
1507 pending_lock
= &brt
->brt_pending_lock
[txg
& TXG_MASK
];
1509 bpe_search
.bpe_bp
= *bp
;
1511 mutex_enter(pending_lock
);
1513 bpe
= avl_find(pending_tree
, &bpe_search
, NULL
);
1514 /* I believe we should always find bpe when this function is called. */
1516 ASSERT(bpe
->bpe_count
> 0);
1519 if (bpe
->bpe_count
== 0) {
1520 avl_remove(pending_tree
, bpe
);
1521 kmem_cache_free(brt_pending_entry_cache
, bpe
);
1525 mutex_exit(pending_lock
);
1529 brt_pending_apply(spa_t
*spa
, uint64_t txg
)
1532 brt_pending_entry_t
*bpe
;
1533 avl_tree_t
*pending_tree
;
1534 kmutex_t
*pending_lock
;
1537 ASSERT3U(txg
, !=, 0);
1540 pending_tree
= &brt
->brt_pending_tree
[txg
& TXG_MASK
];
1541 pending_lock
= &brt
->brt_pending_lock
[txg
& TXG_MASK
];
1543 mutex_enter(pending_lock
);
1546 while ((bpe
= avl_destroy_nodes(pending_tree
, &c
)) != NULL
) {
1547 boolean_t added_to_ddt
;
1549 mutex_exit(pending_lock
);
1551 for (int i
= 0; i
< bpe
->bpe_count
; i
++) {
1553 * If the block has DEDUP bit set, it means that it
1554 * already exists in the DEDUP table, so we can just
1555 * use that instead of creating new entry in
1558 if (BP_GET_DEDUP(&bpe
->bpe_bp
)) {
1559 added_to_ddt
= ddt_addref(spa
, &bpe
->bpe_bp
);
1561 added_to_ddt
= B_FALSE
;
1564 brt_entry_addref(brt
, &bpe
->bpe_bp
);
1567 kmem_cache_free(brt_pending_entry_cache
, bpe
);
1568 mutex_enter(pending_lock
);
1571 mutex_exit(pending_lock
);
1575 brt_sync_entry(brt_t
*brt
, brt_vdev_t
*brtvd
, brt_entry_t
*bre
, dmu_tx_t
*tx
)
1578 ASSERT(RW_WRITE_HELD(&brt
->brt_lock
));
1579 ASSERT(brtvd
->bv_mos_entries
!= 0);
1581 if (bre
->bre_refcount
== 0) {
1584 error
= brt_entry_remove(brt
, brtvd
, bre
, tx
);
1585 ASSERT(error
== 0 || error
== ENOENT
);
1587 * If error == ENOENT then zfs_clone_range() was done from a
1588 * removed (but opened) file (open(), unlink()).
1590 ASSERT(brt_entry_lookup(brt
, brtvd
, bre
) == ENOENT
);
1592 VERIFY0(brt_entry_update(brt
, brtvd
, bre
, tx
));
1597 brt_sync_table(brt_t
*brt
, dmu_tx_t
*tx
)
1606 for (vdevid
= 0; vdevid
< brt
->brt_nvdevs
; vdevid
++) {
1607 brtvd
= &brt
->brt_vdevs
[vdevid
];
1609 if (!brtvd
->bv_initiated
)
1612 if (!brtvd
->bv_meta_dirty
) {
1613 ASSERT(!brtvd
->bv_entcount_dirty
);
1614 ASSERT0(avl_numnodes(&brtvd
->bv_tree
));
1618 ASSERT(!brtvd
->bv_entcount_dirty
||
1619 avl_numnodes(&brtvd
->bv_tree
) != 0);
1621 if (brtvd
->bv_mos_brtvdev
== 0)
1622 brt_vdev_create(brt
, brtvd
, tx
);
1625 while ((bre
= avl_destroy_nodes(&brtvd
->bv_tree
, &c
)) != NULL
) {
1626 brt_sync_entry(brt
, brtvd
, bre
, tx
);
1627 brt_entry_free(bre
);
1628 ASSERT(brt
->brt_nentries
> 0);
1629 brt
->brt_nentries
--;
1632 brt_vdev_sync(brt
, brtvd
, tx
);
1634 if (brtvd
->bv_totalcount
== 0)
1635 brt_vdev_destroy(brt
, brtvd
, tx
);
1638 ASSERT0(brt
->brt_nentries
);
1644 brt_sync(spa_t
*spa
, uint64_t txg
)
1649 ASSERT(spa_syncing_txg(spa
) == txg
);
1653 if (brt
->brt_nentries
== 0) {
1660 tx
= dmu_tx_create_assigned(spa
->spa_dsl_pool
, txg
);
1662 brt_sync_table(brt
, tx
);
1668 brt_table_alloc(brt_t
*brt
)
1671 for (int i
= 0; i
< TXG_SIZE
; i
++) {
1672 avl_create(&brt
->brt_pending_tree
[i
],
1673 brt_pending_entry_compare
,
1674 sizeof (brt_pending_entry_t
),
1675 offsetof(brt_pending_entry_t
, bpe_node
));
1676 mutex_init(&brt
->brt_pending_lock
[i
], NULL
, MUTEX_DEFAULT
,
1682 brt_table_free(brt_t
*brt
)
1685 for (int i
= 0; i
< TXG_SIZE
; i
++) {
1686 ASSERT(avl_is_empty(&brt
->brt_pending_tree
[i
]));
1688 avl_destroy(&brt
->brt_pending_tree
[i
]);
1689 mutex_destroy(&brt
->brt_pending_lock
[i
]);
1694 brt_alloc(spa_t
*spa
)
1698 ASSERT(spa
->spa_brt
== NULL
);
1700 brt
= kmem_zalloc(sizeof (*brt
), KM_SLEEP
);
1701 rw_init(&brt
->brt_lock
, NULL
, RW_DEFAULT
, NULL
);
1703 brt
->brt_rangesize
= 0;
1704 brt
->brt_nentries
= 0;
1705 brt
->brt_vdevs
= NULL
;
1706 brt
->brt_nvdevs
= 0;
1707 brt_table_alloc(brt
);
1713 brt_create(spa_t
*spa
)
1717 brt_vdevs_alloc(spa
->spa_brt
, B_FALSE
);
1721 brt_load(spa_t
*spa
)
1725 brt_vdevs_alloc(spa
->spa_brt
, B_TRUE
);
1731 brt_unload(spa_t
*spa
)
1733 brt_t
*brt
= spa
->spa_brt
;
1738 brt_vdevs_free(brt
);
1739 brt_table_free(brt
);
1740 rw_destroy(&brt
->brt_lock
);
1741 kmem_free(brt
, sizeof (*brt
));
1742 spa
->spa_brt
= NULL
;
1746 ZFS_MODULE_PARAM(zfs_brt
, zfs_brt_
, prefetch
, INT
, ZMOD_RW
,
1747 "Enable prefetching of BRT entries");
1748 #ifdef ZFS_BRT_DEBUG
1749 ZFS_MODULE_PARAM(zfs_brt
, zfs_brt_
, debug
, INT
, ZMOD_RW
, "BRT debug");