]>
Commit | Line | Data |
---|---|---|
67a1b037 PJD |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
3a03c963 | 9 | * or https://opensource.org/licenses/CDDL-1.0. |
67a1b037 PJD |
10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | /* | |
23 | * Copyright (c) 2020, 2021, 2022 by Pawel Jakub Dawidek | |
24 | */ | |
25 | ||
26 | #include <sys/zfs_context.h> | |
27 | #include <sys/spa.h> | |
28 | #include <sys/spa_impl.h> | |
29 | #include <sys/zio.h> | |
30 | #include <sys/brt.h> | |
803a9c12 | 31 | #include <sys/brt_impl.h> |
67a1b037 PJD |
32 | #include <sys/ddt.h> |
33 | #include <sys/bitmap.h> | |
34 | #include <sys/zap.h> | |
35 | #include <sys/dmu_tx.h> | |
36 | #include <sys/arc.h> | |
37 | #include <sys/dsl_pool.h> | |
38 | #include <sys/dsl_scan.h> | |
39 | #include <sys/vdev_impl.h> | |
40 | #include <sys/kstat.h> | |
41 | #include <sys/wmsum.h> | |
42 | ||
43 | /* | |
44 | * Block Cloning design. | |
45 | * | |
46 | * Block Cloning allows to manually clone a file (or a subset of its blocks) | |
47 | * into another (or the same) file by just creating additional references to | |
48 | * the data blocks without copying the data itself. Those references are kept | |
49 | * in the Block Reference Tables (BRTs). | |
50 | * | |
51 | * In many ways this is similar to the existing deduplication, but there are | |
52 | * some important differences: | |
53 | * | |
54 | * - Deduplication is automatic and Block Cloning is not - one has to use a | |
55 | * dedicated system call(s) to clone the given file/blocks. | |
56 | * - Deduplication keeps all data blocks in its table, even those referenced | |
57 | * just once. Block Cloning creates an entry in its tables only when there | |
58 | * are at least two references to the given data block. If the block was | |
59 | * never explicitly cloned or the second to last reference was dropped, | |
60 | * there will be neither space nor performance overhead. | |
61 | * - Deduplication needs data to work - one needs to pass real data to the | |
62 | * write(2) syscall, so hash can be calculated. Block Cloning doesn't require | |
63 | * data, just block pointers to the data, so it is extremely fast, as we pay | |
64 | * neither the cost of reading the data, nor the cost of writing the data - | |
65 | * we operate exclusively on metadata. | |
66 | * - If the D (dedup) bit is not set in the block pointer, it means that | |
67 | * the block is not in the dedup table (DDT) and we won't consult the DDT | |
68 | * when we need to free the block. Block Cloning must be consulted on every | |
69 | * free, because we cannot modify the source BP (eg. by setting something | |
70 | * similar to the D bit), thus we have no hint if the block is in the | |
71 | * Block Reference Table (BRT), so we need to look into the BRT. There is | |
72 | * an optimization in place that allows us to eliminate the majority of BRT | |
73 | * lookups which is described below in the "Minimizing free penalty" section. | |
74 | * - The BRT entry is much smaller than the DDT entry - for BRT we only store | |
75 | * 64bit offset and 64bit reference counter. | |
76 | * - Dedup keys are cryptographic hashes, so two blocks that are close to each | |
77 | * other on disk are most likely in totally different parts of the DDT. | |
78 | * The BRT entry keys are offsets into a single top-level VDEV, so data blocks | |
79 | * from one file should have BRT entries close to each other. | |
80 | * - Scrub will only do a single pass over a block that is referenced multiple | |
81 | * times in the DDT. Unfortunately it is not currently (if at all) possible | |
82 | * with Block Cloning and block referenced multiple times will be scrubbed | |
83 | * multiple times. The new, sorted scrub should be able to eliminate | |
84 | * duplicated reads given enough memory. | |
85 | * - Deduplication requires cryptographically strong hash as a checksum or | |
86 | * additional data verification. Block Cloning works with any checksum | |
87 | * algorithm or even with checksumming disabled. | |
88 | * | |
89 | * As mentioned above, the BRT entries are much smaller than the DDT entries. | |
90 | * To uniquely identify a block we just need its vdev id and offset. We also | |
91 | * need to maintain a reference counter. The vdev id will often repeat, as there | |
92 | * is a small number of top-level VDEVs and a large number of blocks stored in | |
93 | * each VDEV. We take advantage of that to reduce the BRT entry size further by | |
94 | * maintaining one BRT for each top-level VDEV, so we can then have only offset | |
95 | * and counter as the BRT entry. | |
96 | * | |
97 | * Minimizing free penalty. | |
98 | * | |
99 | * Block Cloning allows creating additional references to any existing block. | |
100 | * When we free a block there is no hint in the block pointer whether the block | |
101 | * was cloned or not, so on each free we have to check if there is a | |
102 | * corresponding entry in the BRT or not. If there is, we need to decrease | |
103 | * the reference counter. Doing BRT lookup on every free can potentially be | |
104 | * expensive by requiring additional I/Os if the BRT doesn't fit into memory. | |
105 | * This is the main problem with deduplication, so we've learned our lesson and | |
106 | * try not to repeat the same mistake here. How do we do that? We divide each | |
107 | * top-level VDEV into 16MB regions. For each region we maintain a counter that | |
108 | * is a sum of all the BRT entries that have offsets within the region. This | |
109 | * creates the entries count array of 16bit numbers for each top-level VDEV. | |
110 | * The entries count array is always kept in memory and updated on disk in the | |
111 | * same transaction group as the BRT updates to keep everything in-sync. We can | |
112 | * keep the array in memory, because it is very small. With 16MB regions and | |
113 | * 1TB VDEV the array requires only 128kB of memory (we may decide to decrease | |
114 | * the region size even further in the future). Now, when we want to free | |
115 | * a block, we first consult the array. If the counter for the whole region is | |
116 | * zero, there is no need to look for the BRT entry, as there isn't one for | |
117 | * sure. If the counter for the region is greater than zero, only then we will | |
118 | * do a BRT lookup and if an entry is found we will decrease the reference | |
119 | * counter in the BRT entry and in the entry counters array. | |
120 | * | |
121 | * The entry counters array is small, but can potentially be larger for very | |
122 | * large VDEVs or smaller regions. In this case we don't want to rewrite entire | |
123 | * array on every change. We then divide the array into 32kB block and keep | |
124 | * a bitmap of dirty blocks within a transaction group. When we sync the | |
125 | * transaction group we can only update the parts of the entry counters array | |
126 | * that were modified. Note: Keeping track of the dirty parts of the entry | |
127 | * counters array is implemented, but updating only parts of the array on disk | |
128 | * is not yet implemented - for now we will update entire array if there was | |
129 | * any change. | |
130 | * | |
131 | * The implementation tries to be economic: if BRT is not used, or no longer | |
132 | * used, there will be no entries in the MOS and no additional memory used (eg. | |
133 | * the entry counters array is only allocated if needed). | |
134 | * | |
135 | * Interaction between Deduplication and Block Cloning. | |
136 | * | |
137 | * If both functionalities are in use, we could end up with a block that is | |
138 | * referenced multiple times in both DDT and BRT. When we free one of the | |
139 | * references we couldn't tell where it belongs, so we would have to decide | |
140 | * what table takes the precedence: do we first clear DDT references or BRT | |
141 | * references? To avoid this dilemma BRT cooperates with DDT - if a given block | |
142 | * is being cloned using BRT and the BP has the D (dedup) bit set, BRT will | |
143 | * lookup DDT entry instead and increase the counter there. No BRT entry | |
144 | * will be created for a block which has the D (dedup) bit set. | |
145 | * BRT may be more efficient for manual deduplication, but if the block is | |
146 | * already in the DDT, then creating additional BRT entry would be less | |
147 | * efficient. This clever idea was proposed by Allan Jude. | |
148 | * | |
149 | * Block Cloning across datasets. | |
150 | * | |
151 | * Block Cloning is not limited to cloning blocks within the same dataset. | |
152 | * It is possible (and very useful) to clone blocks between different datasets. | |
153 | * One use case is recovering files from snapshots. By cloning the files into | |
154 | * dataset we need no additional storage. Without Block Cloning we would need | |
155 | * additional space for those files. | |
156 | * Another interesting use case is moving the files between datasets | |
157 | * (copying the file content to the new dataset and removing the source file). | |
158 | * In that case Block Cloning will only be used briefly, because the BRT entries | |
159 | * will be removed when the source is removed. | |
c7b61192 KP |
160 | * Block Cloning across encrypted datasets is supported as long as both |
161 | * datasets share the same master key (e.g. snapshots and clones) | |
67a1b037 PJD |
162 | * |
163 | * Block Cloning flow through ZFS layers. | |
164 | * | |
165 | * Note: Block Cloning can be used both for cloning file system blocks and ZVOL | |
166 | * blocks. As of this writing no interface is implemented that allows for block | |
167 | * cloning within a ZVOL. | |
168 | * FreeBSD and Linux provides copy_file_range(2) system call and we will use it | |
169 | * for blocking cloning. | |
170 | * | |
171 | * ssize_t | |
172 | * copy_file_range(int infd, off_t *inoffp, int outfd, off_t *outoffp, | |
173 | * size_t len, unsigned int flags); | |
174 | * | |
175 | * Even though offsets and length represent bytes, they have to be | |
019dea0a | 176 | * block-aligned or we will return an error so the upper layer can |
67a1b037 PJD |
177 | * fallback to the generic mechanism that will just copy the data. |
178 | * Using copy_file_range(2) will call OS-independent zfs_clone_range() function. | |
179 | * This function was implemented based on zfs_write(), but instead of writing | |
180 | * the given data we first read block pointers using the new dmu_read_l0_bps() | |
181 | * function from the source file. Once we have BPs from the source file we call | |
182 | * the dmu_brt_clone() function on the destination file. This function | |
183 | * allocates BPs for us. We iterate over all source BPs. If the given BP is | |
184 | * a hole or an embedded block, we just copy BP as-is. If it points to a real | |
185 | * data we place this BP on a BRT pending list using the brt_pending_add() | |
186 | * function. | |
187 | * | |
188 | * We use this pending list to keep track of all BPs that got new references | |
189 | * within this transaction group. | |
190 | * | |
191 | * Some special cases to consider and how we address them: | |
192 | * - The block we want to clone may have been created within the same | |
193 | * transaction group that we are trying to clone. Such block has no BP | |
019dea0a | 194 | * allocated yet, so cannot be immediately cloned. We return EAGAIN. |
67a1b037 | 195 | * - The block we want to clone may have been modified within the same |
019dea0a | 196 | * transaction group. We return EAGAIN. |
67a1b037 PJD |
197 | * - A block may be cloned multiple times during one transaction group (that's |
198 | * why pending list is actually a tree and not an append-only list - this | |
199 | * way we can figure out faster if this block is cloned for the first time | |
200 | * in this txg or consecutive time). | |
201 | * - A block may be cloned and freed within the same transaction group | |
202 | * (see dbuf_undirty()). | |
203 | * - A block may be cloned and within the same transaction group the clone | |
204 | * can be cloned again (see dmu_read_l0_bps()). | |
205 | * - A file might have been deleted, but the caller still has a file descriptor | |
206 | * open to this file and clones it. | |
207 | * | |
208 | * When we free a block we have an additional step in the ZIO pipeline where we | |
209 | * call the zio_brt_free() function. We then call the brt_entry_decref() | |
210 | * that loads the corresponding BRT entry (if one exists) and decreases | |
211 | * reference counter. If this is not the last reference we will stop ZIO | |
212 | * pipeline here. If this is the last reference or the block is not in the | |
213 | * BRT, we continue the pipeline and free the block as usual. | |
214 | * | |
215 | * At the beginning of spa_sync() where there can be no more block cloning, | |
216 | * but before issuing frees we call brt_pending_apply(). This function applies | |
217 | * all the new clones to the BRT table - we load BRT entries and update | |
218 | * reference counters. To sync new BRT entries to disk, we use brt_sync() | |
219 | * function. This function will sync all dirty per-top-level-vdev BRTs, | |
220 | * the entry counters arrays, etc. | |
221 | * | |
222 | * Block Cloning and ZIL. | |
223 | * | |
224 | * Every clone operation is divided into chunks (similar to write) and each | |
225 | * chunk is cloned in a separate transaction. The chunk size is determined by | |
226 | * how many BPs we can fit into a single ZIL entry. | |
227 | * Replaying clone operation is different from the regular clone operation, | |
228 | * as when we log clone operations we cannot use the source object - it may | |
229 | * reside on a different dataset, so we log BPs we want to clone. | |
230 | * The ZIL is replayed when we mount the given dataset, not when the pool is | |
231 | * imported. Taking this into account it is possible that the pool is imported | |
232 | * without mounting datasets and the source dataset is destroyed before the | |
233 | * destination dataset is mounted and its ZIL replayed. | |
234 | * To address this situation we leverage zil_claim() mechanism where ZFS will | |
235 | * parse all the ZILs on pool import. When we come across TX_CLONE_RANGE | |
a03ebd9b AM |
236 | * entries, we will bump reference counters for their BPs in the BRT. Then |
237 | * on mount and ZIL replay we bump the reference counters once more, while the | |
238 | * first references are dropped during ZIL destroy by zil_free_clone_range(). | |
239 | * It is possible that after zil_claim() we never mount the destination, so | |
240 | * we never replay its ZIL and just destroy it. In this case the only taken | |
241 | * references will be dropped by zil_free_clone_range(), since the cloning is | |
242 | * not going to ever take place. | |
67a1b037 PJD |
243 | */ |
244 | ||
67a1b037 PJD |
245 | static kmem_cache_t *brt_entry_cache; |
246 | static kmem_cache_t *brt_pending_entry_cache; | |
247 | ||
248 | /* | |
249 | * Enable/disable prefetching of BRT entries that we are going to modify. | |
250 | */ | |
251 | int zfs_brt_prefetch = 1; | |
252 | ||
253 | #ifdef ZFS_DEBUG | |
254 | #define BRT_DEBUG(...) do { \ | |
255 | if ((zfs_flags & ZFS_DEBUG_BRT) != 0) { \ | |
256 | __dprintf(B_TRUE, __FILE__, __func__, __LINE__, __VA_ARGS__); \ | |
257 | } \ | |
258 | } while (0) | |
259 | #else | |
260 | #define BRT_DEBUG(...) do { } while (0) | |
261 | #endif | |
262 | ||
263 | int brt_zap_leaf_blockshift = 12; | |
264 | int brt_zap_indirect_blockshift = 12; | |
265 | ||
266 | static kstat_t *brt_ksp; | |
267 | ||
268 | typedef struct brt_stats { | |
269 | kstat_named_t brt_addref_entry_in_memory; | |
270 | kstat_named_t brt_addref_entry_not_on_disk; | |
271 | kstat_named_t brt_addref_entry_on_disk; | |
272 | kstat_named_t brt_addref_entry_read_lost_race; | |
273 | kstat_named_t brt_decref_entry_in_memory; | |
274 | kstat_named_t brt_decref_entry_loaded_from_disk; | |
275 | kstat_named_t brt_decref_entry_not_in_memory; | |
276 | kstat_named_t brt_decref_entry_not_on_disk; | |
277 | kstat_named_t brt_decref_entry_read_lost_race; | |
278 | kstat_named_t brt_decref_entry_still_referenced; | |
279 | kstat_named_t brt_decref_free_data_later; | |
280 | kstat_named_t brt_decref_free_data_now; | |
281 | kstat_named_t brt_decref_no_entry; | |
282 | } brt_stats_t; | |
283 | ||
284 | static brt_stats_t brt_stats = { | |
285 | { "addref_entry_in_memory", KSTAT_DATA_UINT64 }, | |
286 | { "addref_entry_not_on_disk", KSTAT_DATA_UINT64 }, | |
287 | { "addref_entry_on_disk", KSTAT_DATA_UINT64 }, | |
288 | { "addref_entry_read_lost_race", KSTAT_DATA_UINT64 }, | |
289 | { "decref_entry_in_memory", KSTAT_DATA_UINT64 }, | |
290 | { "decref_entry_loaded_from_disk", KSTAT_DATA_UINT64 }, | |
291 | { "decref_entry_not_in_memory", KSTAT_DATA_UINT64 }, | |
292 | { "decref_entry_not_on_disk", KSTAT_DATA_UINT64 }, | |
293 | { "decref_entry_read_lost_race", KSTAT_DATA_UINT64 }, | |
294 | { "decref_entry_still_referenced", KSTAT_DATA_UINT64 }, | |
295 | { "decref_free_data_later", KSTAT_DATA_UINT64 }, | |
296 | { "decref_free_data_now", KSTAT_DATA_UINT64 }, | |
297 | { "decref_no_entry", KSTAT_DATA_UINT64 } | |
298 | }; | |
299 | ||
300 | struct { | |
301 | wmsum_t brt_addref_entry_in_memory; | |
302 | wmsum_t brt_addref_entry_not_on_disk; | |
303 | wmsum_t brt_addref_entry_on_disk; | |
304 | wmsum_t brt_addref_entry_read_lost_race; | |
305 | wmsum_t brt_decref_entry_in_memory; | |
306 | wmsum_t brt_decref_entry_loaded_from_disk; | |
307 | wmsum_t brt_decref_entry_not_in_memory; | |
308 | wmsum_t brt_decref_entry_not_on_disk; | |
309 | wmsum_t brt_decref_entry_read_lost_race; | |
310 | wmsum_t brt_decref_entry_still_referenced; | |
311 | wmsum_t brt_decref_free_data_later; | |
312 | wmsum_t brt_decref_free_data_now; | |
313 | wmsum_t brt_decref_no_entry; | |
314 | } brt_sums; | |
315 | ||
316 | #define BRTSTAT_BUMP(stat) wmsum_add(&brt_sums.stat, 1) | |
317 | ||
318 | static int brt_entry_compare(const void *x1, const void *x2); | |
319 | static int brt_pending_entry_compare(const void *x1, const void *x2); | |
320 | ||
321 | static void | |
322 | brt_rlock(brt_t *brt) | |
323 | { | |
324 | rw_enter(&brt->brt_lock, RW_READER); | |
325 | } | |
326 | ||
327 | static void | |
328 | brt_wlock(brt_t *brt) | |
329 | { | |
330 | rw_enter(&brt->brt_lock, RW_WRITER); | |
331 | } | |
332 | ||
333 | static void | |
334 | brt_unlock(brt_t *brt) | |
335 | { | |
336 | rw_exit(&brt->brt_lock); | |
337 | } | |
338 | ||
339 | static uint16_t | |
340 | brt_vdev_entcount_get(const brt_vdev_t *brtvd, uint64_t idx) | |
341 | { | |
342 | ||
343 | ASSERT3U(idx, <, brtvd->bv_size); | |
344 | ||
9743d096 | 345 | if (unlikely(brtvd->bv_need_byteswap)) { |
67a1b037 PJD |
346 | return (BSWAP_16(brtvd->bv_entcount[idx])); |
347 | } else { | |
348 | return (brtvd->bv_entcount[idx]); | |
349 | } | |
350 | } | |
351 | ||
352 | static void | |
353 | brt_vdev_entcount_set(brt_vdev_t *brtvd, uint64_t idx, uint16_t entcnt) | |
354 | { | |
355 | ||
356 | ASSERT3U(idx, <, brtvd->bv_size); | |
357 | ||
9743d096 | 358 | if (unlikely(brtvd->bv_need_byteswap)) { |
67a1b037 PJD |
359 | brtvd->bv_entcount[idx] = BSWAP_16(entcnt); |
360 | } else { | |
361 | brtvd->bv_entcount[idx] = entcnt; | |
362 | } | |
363 | } | |
364 | ||
365 | static void | |
366 | brt_vdev_entcount_inc(brt_vdev_t *brtvd, uint64_t idx) | |
367 | { | |
368 | uint16_t entcnt; | |
369 | ||
370 | ASSERT3U(idx, <, brtvd->bv_size); | |
371 | ||
372 | entcnt = brt_vdev_entcount_get(brtvd, idx); | |
373 | ASSERT(entcnt < UINT16_MAX); | |
374 | ||
375 | brt_vdev_entcount_set(brtvd, idx, entcnt + 1); | |
376 | } | |
377 | ||
378 | static void | |
379 | brt_vdev_entcount_dec(brt_vdev_t *brtvd, uint64_t idx) | |
380 | { | |
381 | uint16_t entcnt; | |
382 | ||
383 | ASSERT3U(idx, <, brtvd->bv_size); | |
384 | ||
385 | entcnt = brt_vdev_entcount_get(brtvd, idx); | |
386 | ASSERT(entcnt > 0); | |
387 | ||
388 | brt_vdev_entcount_set(brtvd, idx, entcnt - 1); | |
389 | } | |
390 | ||
391 | #ifdef ZFS_DEBUG | |
392 | static void | |
9743d096 | 393 | brt_vdev_dump(brt_vdev_t *brtvd) |
67a1b037 | 394 | { |
9743d096 | 395 | uint64_t idx; |
67a1b037 | 396 | |
9743d096 AM |
397 | zfs_dbgmsg(" BRT vdevid=%llu meta_dirty=%d entcount_dirty=%d " |
398 | "size=%llu totalcount=%llu nblocks=%llu bitmapsize=%zu\n", | |
399 | (u_longlong_t)brtvd->bv_vdevid, | |
400 | brtvd->bv_meta_dirty, brtvd->bv_entcount_dirty, | |
401 | (u_longlong_t)brtvd->bv_size, | |
402 | (u_longlong_t)brtvd->bv_totalcount, | |
403 | (u_longlong_t)brtvd->bv_nblocks, | |
404 | (size_t)BT_SIZEOFMAP(brtvd->bv_nblocks)); | |
405 | if (brtvd->bv_totalcount > 0) { | |
406 | zfs_dbgmsg(" entcounts:"); | |
407 | for (idx = 0; idx < brtvd->bv_size; idx++) { | |
408 | uint16_t entcnt = brt_vdev_entcount_get(brtvd, idx); | |
409 | if (entcnt > 0) { | |
410 | zfs_dbgmsg(" [%04llu] %hu", | |
411 | (u_longlong_t)idx, entcnt); | |
67a1b037 PJD |
412 | } |
413 | } | |
9743d096 AM |
414 | } |
415 | if (brtvd->bv_entcount_dirty) { | |
416 | char *bitmap; | |
67a1b037 | 417 | |
9743d096 AM |
418 | bitmap = kmem_alloc(brtvd->bv_nblocks + 1, KM_SLEEP); |
419 | for (idx = 0; idx < brtvd->bv_nblocks; idx++) { | |
420 | bitmap[idx] = | |
421 | BT_TEST(brtvd->bv_bitmap, idx) ? 'x' : '.'; | |
67a1b037 | 422 | } |
9743d096 AM |
423 | bitmap[idx] = '\0'; |
424 | zfs_dbgmsg(" dirty: %s", bitmap); | |
425 | kmem_free(bitmap, brtvd->bv_nblocks + 1); | |
67a1b037 PJD |
426 | } |
427 | } | |
428 | #endif | |
429 | ||
430 | static brt_vdev_t * | |
431 | brt_vdev(brt_t *brt, uint64_t vdevid) | |
432 | { | |
433 | brt_vdev_t *brtvd; | |
434 | ||
435 | ASSERT(RW_LOCK_HELD(&brt->brt_lock)); | |
436 | ||
437 | if (vdevid < brt->brt_nvdevs) { | |
438 | brtvd = &brt->brt_vdevs[vdevid]; | |
439 | } else { | |
440 | brtvd = NULL; | |
441 | } | |
442 | ||
443 | return (brtvd); | |
444 | } | |
445 | ||
446 | static void | |
447 | brt_vdev_create(brt_t *brt, brt_vdev_t *brtvd, dmu_tx_t *tx) | |
448 | { | |
449 | char name[64]; | |
450 | ||
451 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
452 | ASSERT0(brtvd->bv_mos_brtvdev); | |
453 | ASSERT0(brtvd->bv_mos_entries); | |
454 | ASSERT(brtvd->bv_entcount != NULL); | |
455 | ASSERT(brtvd->bv_size > 0); | |
456 | ASSERT(brtvd->bv_bitmap != NULL); | |
457 | ASSERT(brtvd->bv_nblocks > 0); | |
458 | ||
459 | brtvd->bv_mos_entries = zap_create_flags(brt->brt_mos, 0, | |
460 | ZAP_FLAG_HASH64 | ZAP_FLAG_UINT64_KEY, DMU_OTN_ZAP_METADATA, | |
461 | brt_zap_leaf_blockshift, brt_zap_indirect_blockshift, DMU_OT_NONE, | |
462 | 0, tx); | |
463 | VERIFY(brtvd->bv_mos_entries != 0); | |
464 | BRT_DEBUG("MOS entries created, object=%llu", | |
465 | (u_longlong_t)brtvd->bv_mos_entries); | |
466 | ||
467 | /* | |
468 | * We allocate DMU buffer to store the bv_entcount[] array. | |
469 | * We will keep array size (bv_size) and cummulative count for all | |
470 | * bv_entcount[]s (bv_totalcount) in the bonus buffer. | |
471 | */ | |
472 | brtvd->bv_mos_brtvdev = dmu_object_alloc(brt->brt_mos, | |
473 | DMU_OTN_UINT64_METADATA, BRT_BLOCKSIZE, | |
474 | DMU_OTN_UINT64_METADATA, sizeof (brt_vdev_phys_t), tx); | |
475 | VERIFY(brtvd->bv_mos_brtvdev != 0); | |
476 | BRT_DEBUG("MOS BRT VDEV created, object=%llu", | |
477 | (u_longlong_t)brtvd->bv_mos_brtvdev); | |
478 | ||
479 | snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX, | |
480 | (u_longlong_t)brtvd->bv_vdevid); | |
481 | VERIFY0(zap_add(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name, | |
482 | sizeof (uint64_t), 1, &brtvd->bv_mos_brtvdev, tx)); | |
483 | BRT_DEBUG("Pool directory object created, object=%s", name); | |
484 | ||
485 | spa_feature_incr(brt->brt_spa, SPA_FEATURE_BLOCK_CLONING, tx); | |
486 | } | |
487 | ||
488 | static void | |
489 | brt_vdev_realloc(brt_t *brt, brt_vdev_t *brtvd) | |
490 | { | |
491 | vdev_t *vd; | |
492 | uint16_t *entcount; | |
493 | ulong_t *bitmap; | |
494 | uint64_t nblocks, size; | |
495 | ||
496 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
497 | ||
498 | spa_config_enter(brt->brt_spa, SCL_VDEV, FTAG, RW_READER); | |
499 | vd = vdev_lookup_top(brt->brt_spa, brtvd->bv_vdevid); | |
500 | size = (vdev_get_min_asize(vd) - 1) / brt->brt_rangesize + 1; | |
501 | spa_config_exit(brt->brt_spa, SCL_VDEV, FTAG); | |
502 | ||
87a6e135 | 503 | entcount = vmem_zalloc(sizeof (entcount[0]) * size, KM_SLEEP); |
67a1b037 PJD |
504 | nblocks = BRT_RANGESIZE_TO_NBLOCKS(size); |
505 | bitmap = kmem_zalloc(BT_SIZEOFMAP(nblocks), KM_SLEEP); | |
506 | ||
507 | if (!brtvd->bv_initiated) { | |
508 | ASSERT0(brtvd->bv_size); | |
509 | ASSERT(brtvd->bv_entcount == NULL); | |
510 | ASSERT(brtvd->bv_bitmap == NULL); | |
511 | ASSERT0(brtvd->bv_nblocks); | |
512 | ||
513 | avl_create(&brtvd->bv_tree, brt_entry_compare, | |
514 | sizeof (brt_entry_t), offsetof(brt_entry_t, bre_node)); | |
515 | } else { | |
516 | ASSERT(brtvd->bv_size > 0); | |
517 | ASSERT(brtvd->bv_entcount != NULL); | |
518 | ASSERT(brtvd->bv_bitmap != NULL); | |
519 | ASSERT(brtvd->bv_nblocks > 0); | |
520 | /* | |
521 | * TODO: Allow vdev shrinking. We only need to implement | |
522 | * shrinking the on-disk BRT VDEV object. | |
523 | * dmu_free_range(brt->brt_mos, brtvd->bv_mos_brtvdev, offset, | |
524 | * size, tx); | |
525 | */ | |
526 | ASSERT3U(brtvd->bv_size, <=, size); | |
527 | ||
528 | memcpy(entcount, brtvd->bv_entcount, | |
529 | sizeof (entcount[0]) * MIN(size, brtvd->bv_size)); | |
530 | memcpy(bitmap, brtvd->bv_bitmap, MIN(BT_SIZEOFMAP(nblocks), | |
531 | BT_SIZEOFMAP(brtvd->bv_nblocks))); | |
87a6e135 | 532 | vmem_free(brtvd->bv_entcount, |
67a1b037 PJD |
533 | sizeof (entcount[0]) * brtvd->bv_size); |
534 | kmem_free(brtvd->bv_bitmap, BT_SIZEOFMAP(brtvd->bv_nblocks)); | |
535 | } | |
536 | ||
537 | brtvd->bv_size = size; | |
538 | brtvd->bv_entcount = entcount; | |
539 | brtvd->bv_bitmap = bitmap; | |
540 | brtvd->bv_nblocks = nblocks; | |
541 | if (!brtvd->bv_initiated) { | |
542 | brtvd->bv_need_byteswap = FALSE; | |
543 | brtvd->bv_initiated = TRUE; | |
544 | BRT_DEBUG("BRT VDEV %llu initiated.", | |
545 | (u_longlong_t)brtvd->bv_vdevid); | |
546 | } | |
547 | } | |
548 | ||
549 | static void | |
550 | brt_vdev_load(brt_t *brt, brt_vdev_t *brtvd) | |
551 | { | |
552 | char name[64]; | |
553 | dmu_buf_t *db; | |
554 | brt_vdev_phys_t *bvphys; | |
555 | int error; | |
556 | ||
557 | snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX, | |
558 | (u_longlong_t)brtvd->bv_vdevid); | |
559 | error = zap_lookup(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name, | |
560 | sizeof (uint64_t), 1, &brtvd->bv_mos_brtvdev); | |
561 | if (error != 0) | |
562 | return; | |
563 | ASSERT(brtvd->bv_mos_brtvdev != 0); | |
564 | ||
565 | error = dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db); | |
566 | ASSERT0(error); | |
567 | if (error != 0) | |
568 | return; | |
569 | ||
570 | bvphys = db->db_data; | |
571 | if (brt->brt_rangesize == 0) { | |
572 | brt->brt_rangesize = bvphys->bvp_rangesize; | |
573 | } else { | |
574 | ASSERT3U(brt->brt_rangesize, ==, bvphys->bvp_rangesize); | |
575 | } | |
576 | ||
577 | ASSERT(!brtvd->bv_initiated); | |
578 | brt_vdev_realloc(brt, brtvd); | |
579 | ||
580 | /* TODO: We don't support VDEV shrinking. */ | |
581 | ASSERT3U(bvphys->bvp_size, <=, brtvd->bv_size); | |
582 | ||
583 | /* | |
584 | * If VDEV grew, we will leave new bv_entcount[] entries zeroed out. | |
585 | */ | |
586 | error = dmu_read(brt->brt_mos, brtvd->bv_mos_brtvdev, 0, | |
587 | MIN(brtvd->bv_size, bvphys->bvp_size) * sizeof (uint16_t), | |
588 | brtvd->bv_entcount, DMU_READ_NO_PREFETCH); | |
589 | ASSERT0(error); | |
590 | ||
591 | brtvd->bv_mos_entries = bvphys->bvp_mos_entries; | |
592 | ASSERT(brtvd->bv_mos_entries != 0); | |
593 | brtvd->bv_need_byteswap = | |
594 | (bvphys->bvp_byteorder != BRT_NATIVE_BYTEORDER); | |
595 | brtvd->bv_totalcount = bvphys->bvp_totalcount; | |
596 | brtvd->bv_usedspace = bvphys->bvp_usedspace; | |
597 | brtvd->bv_savedspace = bvphys->bvp_savedspace; | |
598 | brt->brt_usedspace += brtvd->bv_usedspace; | |
599 | brt->brt_savedspace += brtvd->bv_savedspace; | |
600 | ||
601 | dmu_buf_rele(db, FTAG); | |
602 | ||
603 | BRT_DEBUG("MOS BRT VDEV %s loaded: mos_brtvdev=%llu, mos_entries=%llu", | |
604 | name, (u_longlong_t)brtvd->bv_mos_brtvdev, | |
605 | (u_longlong_t)brtvd->bv_mos_entries); | |
606 | } | |
607 | ||
608 | static void | |
609 | brt_vdev_dealloc(brt_t *brt, brt_vdev_t *brtvd) | |
610 | { | |
611 | ||
612 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
613 | ASSERT(brtvd->bv_initiated); | |
614 | ||
87a6e135 | 615 | vmem_free(brtvd->bv_entcount, sizeof (uint16_t) * brtvd->bv_size); |
67a1b037 PJD |
616 | brtvd->bv_entcount = NULL; |
617 | kmem_free(brtvd->bv_bitmap, BT_SIZEOFMAP(brtvd->bv_nblocks)); | |
618 | brtvd->bv_bitmap = NULL; | |
619 | ASSERT0(avl_numnodes(&brtvd->bv_tree)); | |
620 | avl_destroy(&brtvd->bv_tree); | |
621 | ||
622 | brtvd->bv_size = 0; | |
623 | brtvd->bv_nblocks = 0; | |
624 | ||
625 | brtvd->bv_initiated = FALSE; | |
626 | BRT_DEBUG("BRT VDEV %llu deallocated.", (u_longlong_t)brtvd->bv_vdevid); | |
627 | } | |
628 | ||
629 | static void | |
630 | brt_vdev_destroy(brt_t *brt, brt_vdev_t *brtvd, dmu_tx_t *tx) | |
631 | { | |
632 | char name[64]; | |
633 | uint64_t count; | |
634 | dmu_buf_t *db; | |
635 | brt_vdev_phys_t *bvphys; | |
636 | ||
637 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
638 | ASSERT(brtvd->bv_mos_brtvdev != 0); | |
639 | ASSERT(brtvd->bv_mos_entries != 0); | |
640 | ||
641 | VERIFY0(zap_count(brt->brt_mos, brtvd->bv_mos_entries, &count)); | |
642 | VERIFY0(count); | |
643 | VERIFY0(zap_destroy(brt->brt_mos, brtvd->bv_mos_entries, tx)); | |
644 | BRT_DEBUG("MOS entries destroyed, object=%llu", | |
645 | (u_longlong_t)brtvd->bv_mos_entries); | |
646 | brtvd->bv_mos_entries = 0; | |
647 | ||
648 | VERIFY0(dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db)); | |
649 | bvphys = db->db_data; | |
650 | ASSERT0(bvphys->bvp_totalcount); | |
651 | ASSERT0(bvphys->bvp_usedspace); | |
652 | ASSERT0(bvphys->bvp_savedspace); | |
653 | dmu_buf_rele(db, FTAG); | |
654 | ||
655 | VERIFY0(dmu_object_free(brt->brt_mos, brtvd->bv_mos_brtvdev, tx)); | |
656 | BRT_DEBUG("MOS BRT VDEV destroyed, object=%llu", | |
657 | (u_longlong_t)brtvd->bv_mos_brtvdev); | |
658 | brtvd->bv_mos_brtvdev = 0; | |
659 | ||
660 | snprintf(name, sizeof (name), "%s%llu", BRT_OBJECT_VDEV_PREFIX, | |
661 | (u_longlong_t)brtvd->bv_vdevid); | |
662 | VERIFY0(zap_remove(brt->brt_mos, DMU_POOL_DIRECTORY_OBJECT, name, tx)); | |
663 | BRT_DEBUG("Pool directory object removed, object=%s", name); | |
664 | ||
665 | brt_vdev_dealloc(brt, brtvd); | |
666 | ||
667 | spa_feature_decr(brt->brt_spa, SPA_FEATURE_BLOCK_CLONING, tx); | |
668 | } | |
669 | ||
670 | static void | |
671 | brt_vdevs_expand(brt_t *brt, uint64_t nvdevs) | |
672 | { | |
673 | brt_vdev_t *brtvd, *vdevs; | |
674 | uint64_t vdevid; | |
675 | ||
676 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
677 | ASSERT3U(nvdevs, >, brt->brt_nvdevs); | |
678 | ||
679 | vdevs = kmem_zalloc(sizeof (vdevs[0]) * nvdevs, KM_SLEEP); | |
680 | if (brt->brt_nvdevs > 0) { | |
681 | ASSERT(brt->brt_vdevs != NULL); | |
682 | ||
683 | memcpy(vdevs, brt->brt_vdevs, | |
684 | sizeof (brt_vdev_t) * brt->brt_nvdevs); | |
685 | kmem_free(brt->brt_vdevs, | |
686 | sizeof (brt_vdev_t) * brt->brt_nvdevs); | |
687 | } | |
688 | for (vdevid = brt->brt_nvdevs; vdevid < nvdevs; vdevid++) { | |
689 | brtvd = &vdevs[vdevid]; | |
690 | ||
691 | brtvd->bv_vdevid = vdevid; | |
692 | brtvd->bv_initiated = FALSE; | |
693 | } | |
694 | ||
695 | BRT_DEBUG("BRT VDEVs expanded from %llu to %llu.", | |
696 | (u_longlong_t)brt->brt_nvdevs, (u_longlong_t)nvdevs); | |
697 | ||
698 | brt->brt_vdevs = vdevs; | |
699 | brt->brt_nvdevs = nvdevs; | |
700 | } | |
701 | ||
702 | static boolean_t | |
703 | brt_vdev_lookup(brt_t *brt, brt_vdev_t *brtvd, const brt_entry_t *bre) | |
704 | { | |
705 | uint64_t idx; | |
706 | ||
707 | ASSERT(RW_LOCK_HELD(&brt->brt_lock)); | |
708 | ||
709 | idx = bre->bre_offset / brt->brt_rangesize; | |
710 | if (brtvd->bv_entcount != NULL && idx < brtvd->bv_size) { | |
711 | /* VDEV wasn't expanded. */ | |
712 | return (brt_vdev_entcount_get(brtvd, idx) > 0); | |
713 | } | |
714 | ||
715 | return (FALSE); | |
716 | } | |
717 | ||
718 | static void | |
719 | brt_vdev_addref(brt_t *brt, brt_vdev_t *brtvd, const brt_entry_t *bre, | |
720 | uint64_t dsize) | |
721 | { | |
722 | uint64_t idx; | |
723 | ||
724 | ASSERT(RW_LOCK_HELD(&brt->brt_lock)); | |
725 | ASSERT(brtvd != NULL); | |
726 | ASSERT(brtvd->bv_entcount != NULL); | |
727 | ||
728 | brt->brt_savedspace += dsize; | |
729 | brtvd->bv_savedspace += dsize; | |
730 | brtvd->bv_meta_dirty = TRUE; | |
731 | ||
732 | if (bre->bre_refcount > 1) { | |
733 | return; | |
734 | } | |
735 | ||
736 | brt->brt_usedspace += dsize; | |
737 | brtvd->bv_usedspace += dsize; | |
738 | ||
739 | idx = bre->bre_offset / brt->brt_rangesize; | |
740 | if (idx >= brtvd->bv_size) { | |
741 | /* VDEV has been expanded. */ | |
742 | brt_vdev_realloc(brt, brtvd); | |
743 | } | |
744 | ||
745 | ASSERT3U(idx, <, brtvd->bv_size); | |
746 | ||
747 | brtvd->bv_totalcount++; | |
748 | brt_vdev_entcount_inc(brtvd, idx); | |
749 | brtvd->bv_entcount_dirty = TRUE; | |
750 | idx = idx / BRT_BLOCKSIZE / 8; | |
751 | BT_SET(brtvd->bv_bitmap, idx); | |
752 | ||
753 | #ifdef ZFS_DEBUG | |
9743d096 AM |
754 | if (zfs_flags & ZFS_DEBUG_BRT) |
755 | brt_vdev_dump(brtvd); | |
67a1b037 PJD |
756 | #endif |
757 | } | |
758 | ||
759 | static void | |
760 | brt_vdev_decref(brt_t *brt, brt_vdev_t *brtvd, const brt_entry_t *bre, | |
761 | uint64_t dsize) | |
762 | { | |
763 | uint64_t idx; | |
764 | ||
765 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
766 | ASSERT(brtvd != NULL); | |
767 | ASSERT(brtvd->bv_entcount != NULL); | |
768 | ||
769 | brt->brt_savedspace -= dsize; | |
770 | brtvd->bv_savedspace -= dsize; | |
771 | brtvd->bv_meta_dirty = TRUE; | |
772 | ||
773 | if (bre->bre_refcount > 0) { | |
774 | return; | |
775 | } | |
776 | ||
777 | brt->brt_usedspace -= dsize; | |
778 | brtvd->bv_usedspace -= dsize; | |
779 | ||
780 | idx = bre->bre_offset / brt->brt_rangesize; | |
781 | ASSERT3U(idx, <, brtvd->bv_size); | |
782 | ||
783 | ASSERT(brtvd->bv_totalcount > 0); | |
784 | brtvd->bv_totalcount--; | |
785 | brt_vdev_entcount_dec(brtvd, idx); | |
786 | brtvd->bv_entcount_dirty = TRUE; | |
787 | idx = idx / BRT_BLOCKSIZE / 8; | |
788 | BT_SET(brtvd->bv_bitmap, idx); | |
789 | ||
790 | #ifdef ZFS_DEBUG | |
9743d096 AM |
791 | if (zfs_flags & ZFS_DEBUG_BRT) |
792 | brt_vdev_dump(brtvd); | |
67a1b037 PJD |
793 | #endif |
794 | } | |
795 | ||
796 | static void | |
797 | brt_vdev_sync(brt_t *brt, brt_vdev_t *brtvd, dmu_tx_t *tx) | |
798 | { | |
799 | dmu_buf_t *db; | |
800 | brt_vdev_phys_t *bvphys; | |
801 | ||
802 | ASSERT(brtvd->bv_meta_dirty); | |
803 | ASSERT(brtvd->bv_mos_brtvdev != 0); | |
804 | ASSERT(dmu_tx_is_syncing(tx)); | |
805 | ||
806 | VERIFY0(dmu_bonus_hold(brt->brt_mos, brtvd->bv_mos_brtvdev, FTAG, &db)); | |
807 | ||
808 | if (brtvd->bv_entcount_dirty) { | |
809 | /* | |
810 | * TODO: Walk brtvd->bv_bitmap and write only the dirty blocks. | |
811 | */ | |
812 | dmu_write(brt->brt_mos, brtvd->bv_mos_brtvdev, 0, | |
813 | brtvd->bv_size * sizeof (brtvd->bv_entcount[0]), | |
814 | brtvd->bv_entcount, tx); | |
815 | memset(brtvd->bv_bitmap, 0, BT_SIZEOFMAP(brtvd->bv_nblocks)); | |
816 | brtvd->bv_entcount_dirty = FALSE; | |
817 | } | |
818 | ||
819 | dmu_buf_will_dirty(db, tx); | |
820 | bvphys = db->db_data; | |
821 | bvphys->bvp_mos_entries = brtvd->bv_mos_entries; | |
822 | bvphys->bvp_size = brtvd->bv_size; | |
823 | if (brtvd->bv_need_byteswap) { | |
824 | bvphys->bvp_byteorder = BRT_NON_NATIVE_BYTEORDER; | |
825 | } else { | |
826 | bvphys->bvp_byteorder = BRT_NATIVE_BYTEORDER; | |
827 | } | |
828 | bvphys->bvp_totalcount = brtvd->bv_totalcount; | |
829 | bvphys->bvp_rangesize = brt->brt_rangesize; | |
830 | bvphys->bvp_usedspace = brtvd->bv_usedspace; | |
831 | bvphys->bvp_savedspace = brtvd->bv_savedspace; | |
832 | dmu_buf_rele(db, FTAG); | |
833 | ||
834 | brtvd->bv_meta_dirty = FALSE; | |
835 | } | |
836 | ||
837 | static void | |
838 | brt_vdevs_alloc(brt_t *brt, boolean_t load) | |
839 | { | |
840 | brt_vdev_t *brtvd; | |
841 | uint64_t vdevid; | |
842 | ||
843 | brt_wlock(brt); | |
844 | ||
845 | brt_vdevs_expand(brt, brt->brt_spa->spa_root_vdev->vdev_children); | |
846 | ||
847 | if (load) { | |
848 | for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) { | |
849 | brtvd = &brt->brt_vdevs[vdevid]; | |
850 | ASSERT(brtvd->bv_entcount == NULL); | |
851 | ||
852 | brt_vdev_load(brt, brtvd); | |
853 | } | |
854 | } | |
855 | ||
856 | if (brt->brt_rangesize == 0) { | |
857 | brt->brt_rangesize = BRT_RANGESIZE; | |
858 | } | |
859 | ||
860 | brt_unlock(brt); | |
861 | } | |
862 | ||
863 | static void | |
864 | brt_vdevs_free(brt_t *brt) | |
865 | { | |
866 | brt_vdev_t *brtvd; | |
867 | uint64_t vdevid; | |
868 | ||
869 | brt_wlock(brt); | |
870 | ||
871 | for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) { | |
872 | brtvd = &brt->brt_vdevs[vdevid]; | |
873 | if (brtvd->bv_initiated) | |
874 | brt_vdev_dealloc(brt, brtvd); | |
875 | } | |
876 | kmem_free(brt->brt_vdevs, sizeof (brt_vdev_t) * brt->brt_nvdevs); | |
877 | ||
878 | brt_unlock(brt); | |
879 | } | |
880 | ||
881 | static void | |
882 | brt_entry_fill(const blkptr_t *bp, brt_entry_t *bre, uint64_t *vdevidp) | |
883 | { | |
884 | ||
885 | bre->bre_offset = DVA_GET_OFFSET(&bp->blk_dva[0]); | |
886 | bre->bre_refcount = 0; | |
887 | ||
888 | *vdevidp = DVA_GET_VDEV(&bp->blk_dva[0]); | |
889 | } | |
890 | ||
891 | static int | |
892 | brt_entry_compare(const void *x1, const void *x2) | |
893 | { | |
894 | const brt_entry_t *bre1 = x1; | |
895 | const brt_entry_t *bre2 = x2; | |
896 | ||
897 | return (TREE_CMP(bre1->bre_offset, bre2->bre_offset)); | |
898 | } | |
899 | ||
900 | static int | |
901 | brt_entry_lookup(brt_t *brt, brt_vdev_t *brtvd, brt_entry_t *bre) | |
902 | { | |
903 | uint64_t mos_entries; | |
904 | uint64_t one, physsize; | |
905 | int error; | |
906 | ||
907 | ASSERT(RW_LOCK_HELD(&brt->brt_lock)); | |
908 | ||
909 | if (!brt_vdev_lookup(brt, brtvd, bre)) | |
910 | return (SET_ERROR(ENOENT)); | |
911 | ||
912 | /* | |
913 | * Remember mos_entries object number. After we reacquire the BRT lock, | |
914 | * the brtvd pointer may be invalid. | |
915 | */ | |
916 | mos_entries = brtvd->bv_mos_entries; | |
917 | if (mos_entries == 0) | |
918 | return (SET_ERROR(ENOENT)); | |
919 | ||
920 | brt_unlock(brt); | |
921 | ||
922 | error = zap_length_uint64(brt->brt_mos, mos_entries, &bre->bre_offset, | |
923 | BRT_KEY_WORDS, &one, &physsize); | |
924 | if (error == 0) { | |
925 | ASSERT3U(one, ==, 1); | |
926 | ASSERT3U(physsize, ==, sizeof (bre->bre_refcount)); | |
927 | ||
928 | error = zap_lookup_uint64(brt->brt_mos, mos_entries, | |
929 | &bre->bre_offset, BRT_KEY_WORDS, 1, | |
930 | sizeof (bre->bre_refcount), &bre->bre_refcount); | |
931 | BRT_DEBUG("ZAP lookup: object=%llu vdev=%llu offset=%llu " | |
932 | "count=%llu error=%d", (u_longlong_t)mos_entries, | |
933 | (u_longlong_t)brtvd->bv_vdevid, | |
934 | (u_longlong_t)bre->bre_offset, | |
935 | error == 0 ? (u_longlong_t)bre->bre_refcount : 0, error); | |
936 | } | |
937 | ||
938 | brt_wlock(brt); | |
939 | ||
940 | return (error); | |
941 | } | |
942 | ||
943 | static void | |
944 | brt_entry_prefetch(brt_t *brt, uint64_t vdevid, brt_entry_t *bre) | |
945 | { | |
946 | brt_vdev_t *brtvd; | |
947 | uint64_t mos_entries = 0; | |
948 | ||
949 | brt_rlock(brt); | |
950 | brtvd = brt_vdev(brt, vdevid); | |
951 | if (brtvd != NULL) | |
952 | mos_entries = brtvd->bv_mos_entries; | |
953 | brt_unlock(brt); | |
954 | ||
955 | if (mos_entries == 0) | |
956 | return; | |
957 | ||
67a1b037 PJD |
958 | (void) zap_prefetch_uint64(brt->brt_mos, mos_entries, |
959 | (uint64_t *)&bre->bre_offset, BRT_KEY_WORDS); | |
960 | } | |
961 | ||
67a1b037 PJD |
962 | /* |
963 | * Return TRUE if we _can_ have BRT entry for this bp. It might be false | |
964 | * positive, but gives us quick answer if we should look into BRT, which | |
965 | * may require reads and thus will be more expensive. | |
966 | */ | |
967 | boolean_t | |
968 | brt_maybe_exists(spa_t *spa, const blkptr_t *bp) | |
969 | { | |
970 | brt_t *brt = spa->spa_brt; | |
971 | brt_vdev_t *brtvd; | |
972 | brt_entry_t bre_search; | |
973 | boolean_t mayexists = FALSE; | |
974 | uint64_t vdevid; | |
975 | ||
976 | brt_entry_fill(bp, &bre_search, &vdevid); | |
977 | ||
978 | brt_rlock(brt); | |
979 | ||
980 | brtvd = brt_vdev(brt, vdevid); | |
981 | if (brtvd != NULL && brtvd->bv_initiated) { | |
982 | if (!avl_is_empty(&brtvd->bv_tree) || | |
983 | brt_vdev_lookup(brt, brtvd, &bre_search)) { | |
984 | mayexists = TRUE; | |
985 | } | |
986 | } | |
987 | ||
988 | brt_unlock(brt); | |
989 | ||
990 | return (mayexists); | |
991 | } | |
992 | ||
993 | uint64_t | |
994 | brt_get_dspace(spa_t *spa) | |
995 | { | |
996 | brt_t *brt = spa->spa_brt; | |
997 | ||
998 | if (brt == NULL) | |
999 | return (0); | |
1000 | ||
1001 | return (brt->brt_savedspace); | |
1002 | } | |
1003 | ||
1004 | uint64_t | |
1005 | brt_get_used(spa_t *spa) | |
1006 | { | |
1007 | brt_t *brt = spa->spa_brt; | |
1008 | ||
1009 | if (brt == NULL) | |
1010 | return (0); | |
1011 | ||
1012 | return (brt->brt_usedspace); | |
1013 | } | |
1014 | ||
1015 | uint64_t | |
1016 | brt_get_saved(spa_t *spa) | |
1017 | { | |
1018 | brt_t *brt = spa->spa_brt; | |
1019 | ||
1020 | if (brt == NULL) | |
1021 | return (0); | |
1022 | ||
1023 | return (brt->brt_savedspace); | |
1024 | } | |
1025 | ||
1026 | uint64_t | |
1027 | brt_get_ratio(spa_t *spa) | |
1028 | { | |
1029 | brt_t *brt = spa->spa_brt; | |
1030 | ||
1031 | if (brt->brt_usedspace == 0) | |
1032 | return (100); | |
1033 | ||
1034 | return ((brt->brt_usedspace + brt->brt_savedspace) * 100 / | |
1035 | brt->brt_usedspace); | |
1036 | } | |
1037 | ||
1038 | static int | |
1039 | brt_kstats_update(kstat_t *ksp, int rw) | |
1040 | { | |
1041 | brt_stats_t *bs = ksp->ks_data; | |
1042 | ||
1043 | if (rw == KSTAT_WRITE) | |
1044 | return (EACCES); | |
1045 | ||
1046 | bs->brt_addref_entry_in_memory.value.ui64 = | |
1047 | wmsum_value(&brt_sums.brt_addref_entry_in_memory); | |
1048 | bs->brt_addref_entry_not_on_disk.value.ui64 = | |
1049 | wmsum_value(&brt_sums.brt_addref_entry_not_on_disk); | |
1050 | bs->brt_addref_entry_on_disk.value.ui64 = | |
1051 | wmsum_value(&brt_sums.brt_addref_entry_on_disk); | |
1052 | bs->brt_addref_entry_read_lost_race.value.ui64 = | |
1053 | wmsum_value(&brt_sums.brt_addref_entry_read_lost_race); | |
1054 | bs->brt_decref_entry_in_memory.value.ui64 = | |
1055 | wmsum_value(&brt_sums.brt_decref_entry_in_memory); | |
1056 | bs->brt_decref_entry_loaded_from_disk.value.ui64 = | |
1057 | wmsum_value(&brt_sums.brt_decref_entry_loaded_from_disk); | |
1058 | bs->brt_decref_entry_not_in_memory.value.ui64 = | |
1059 | wmsum_value(&brt_sums.brt_decref_entry_not_in_memory); | |
1060 | bs->brt_decref_entry_not_on_disk.value.ui64 = | |
1061 | wmsum_value(&brt_sums.brt_decref_entry_not_on_disk); | |
1062 | bs->brt_decref_entry_read_lost_race.value.ui64 = | |
1063 | wmsum_value(&brt_sums.brt_decref_entry_read_lost_race); | |
1064 | bs->brt_decref_entry_still_referenced.value.ui64 = | |
1065 | wmsum_value(&brt_sums.brt_decref_entry_still_referenced); | |
1066 | bs->brt_decref_free_data_later.value.ui64 = | |
1067 | wmsum_value(&brt_sums.brt_decref_free_data_later); | |
1068 | bs->brt_decref_free_data_now.value.ui64 = | |
1069 | wmsum_value(&brt_sums.brt_decref_free_data_now); | |
1070 | bs->brt_decref_no_entry.value.ui64 = | |
1071 | wmsum_value(&brt_sums.brt_decref_no_entry); | |
1072 | ||
1073 | return (0); | |
1074 | } | |
1075 | ||
1076 | static void | |
1077 | brt_stat_init(void) | |
1078 | { | |
1079 | ||
1080 | wmsum_init(&brt_sums.brt_addref_entry_in_memory, 0); | |
1081 | wmsum_init(&brt_sums.brt_addref_entry_not_on_disk, 0); | |
1082 | wmsum_init(&brt_sums.brt_addref_entry_on_disk, 0); | |
1083 | wmsum_init(&brt_sums.brt_addref_entry_read_lost_race, 0); | |
1084 | wmsum_init(&brt_sums.brt_decref_entry_in_memory, 0); | |
1085 | wmsum_init(&brt_sums.brt_decref_entry_loaded_from_disk, 0); | |
1086 | wmsum_init(&brt_sums.brt_decref_entry_not_in_memory, 0); | |
1087 | wmsum_init(&brt_sums.brt_decref_entry_not_on_disk, 0); | |
1088 | wmsum_init(&brt_sums.brt_decref_entry_read_lost_race, 0); | |
1089 | wmsum_init(&brt_sums.brt_decref_entry_still_referenced, 0); | |
1090 | wmsum_init(&brt_sums.brt_decref_free_data_later, 0); | |
1091 | wmsum_init(&brt_sums.brt_decref_free_data_now, 0); | |
1092 | wmsum_init(&brt_sums.brt_decref_no_entry, 0); | |
1093 | ||
1094 | brt_ksp = kstat_create("zfs", 0, "brtstats", "misc", KSTAT_TYPE_NAMED, | |
1095 | sizeof (brt_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); | |
1096 | if (brt_ksp != NULL) { | |
1097 | brt_ksp->ks_data = &brt_stats; | |
1098 | brt_ksp->ks_update = brt_kstats_update; | |
1099 | kstat_install(brt_ksp); | |
1100 | } | |
1101 | } | |
1102 | ||
1103 | static void | |
1104 | brt_stat_fini(void) | |
1105 | { | |
1106 | if (brt_ksp != NULL) { | |
1107 | kstat_delete(brt_ksp); | |
1108 | brt_ksp = NULL; | |
1109 | } | |
1110 | ||
1111 | wmsum_fini(&brt_sums.brt_addref_entry_in_memory); | |
1112 | wmsum_fini(&brt_sums.brt_addref_entry_not_on_disk); | |
1113 | wmsum_fini(&brt_sums.brt_addref_entry_on_disk); | |
1114 | wmsum_fini(&brt_sums.brt_addref_entry_read_lost_race); | |
1115 | wmsum_fini(&brt_sums.brt_decref_entry_in_memory); | |
1116 | wmsum_fini(&brt_sums.brt_decref_entry_loaded_from_disk); | |
1117 | wmsum_fini(&brt_sums.brt_decref_entry_not_in_memory); | |
1118 | wmsum_fini(&brt_sums.brt_decref_entry_not_on_disk); | |
1119 | wmsum_fini(&brt_sums.brt_decref_entry_read_lost_race); | |
1120 | wmsum_fini(&brt_sums.brt_decref_entry_still_referenced); | |
1121 | wmsum_fini(&brt_sums.brt_decref_free_data_later); | |
1122 | wmsum_fini(&brt_sums.brt_decref_free_data_now); | |
1123 | wmsum_fini(&brt_sums.brt_decref_no_entry); | |
1124 | } | |
1125 | ||
1126 | void | |
1127 | brt_init(void) | |
1128 | { | |
1129 | brt_entry_cache = kmem_cache_create("brt_entry_cache", | |
1130 | sizeof (brt_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | |
1131 | brt_pending_entry_cache = kmem_cache_create("brt_pending_entry_cache", | |
1132 | sizeof (brt_pending_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | |
1133 | ||
1134 | brt_stat_init(); | |
1135 | } | |
1136 | ||
1137 | void | |
1138 | brt_fini(void) | |
1139 | { | |
1140 | brt_stat_fini(); | |
1141 | ||
1142 | kmem_cache_destroy(brt_entry_cache); | |
1143 | kmem_cache_destroy(brt_pending_entry_cache); | |
1144 | } | |
1145 | ||
1146 | static brt_entry_t * | |
1147 | brt_entry_alloc(const brt_entry_t *bre_init) | |
1148 | { | |
1149 | brt_entry_t *bre; | |
1150 | ||
1151 | bre = kmem_cache_alloc(brt_entry_cache, KM_SLEEP); | |
1152 | bre->bre_offset = bre_init->bre_offset; | |
1153 | bre->bre_refcount = bre_init->bre_refcount; | |
1154 | ||
1155 | return (bre); | |
1156 | } | |
1157 | ||
1158 | static void | |
1159 | brt_entry_free(brt_entry_t *bre) | |
1160 | { | |
1161 | ||
1162 | kmem_cache_free(brt_entry_cache, bre); | |
1163 | } | |
1164 | ||
1165 | static void | |
1166 | brt_entry_addref(brt_t *brt, const blkptr_t *bp) | |
1167 | { | |
1168 | brt_vdev_t *brtvd; | |
1169 | brt_entry_t *bre, *racebre; | |
1170 | brt_entry_t bre_search; | |
1171 | avl_index_t where; | |
1172 | uint64_t vdevid; | |
1173 | int error; | |
1174 | ||
1175 | ASSERT(!RW_WRITE_HELD(&brt->brt_lock)); | |
1176 | ||
1177 | brt_entry_fill(bp, &bre_search, &vdevid); | |
1178 | ||
1179 | brt_wlock(brt); | |
1180 | ||
1181 | brtvd = brt_vdev(brt, vdevid); | |
1182 | if (brtvd == NULL) { | |
1183 | ASSERT3U(vdevid, >=, brt->brt_nvdevs); | |
1184 | ||
1185 | /* New VDEV was added. */ | |
1186 | brt_vdevs_expand(brt, vdevid + 1); | |
1187 | brtvd = brt_vdev(brt, vdevid); | |
1188 | } | |
1189 | ASSERT(brtvd != NULL); | |
1190 | if (!brtvd->bv_initiated) | |
1191 | brt_vdev_realloc(brt, brtvd); | |
1192 | ||
1193 | bre = avl_find(&brtvd->bv_tree, &bre_search, NULL); | |
1194 | if (bre != NULL) { | |
1195 | BRTSTAT_BUMP(brt_addref_entry_in_memory); | |
1196 | } else { | |
1197 | /* | |
1198 | * brt_entry_lookup() may drop the BRT (read) lock and | |
1199 | * reacquire it (write). | |
1200 | */ | |
1201 | error = brt_entry_lookup(brt, brtvd, &bre_search); | |
1202 | /* bre_search now contains correct bre_refcount */ | |
1203 | ASSERT(error == 0 || error == ENOENT); | |
1204 | if (error == 0) | |
1205 | BRTSTAT_BUMP(brt_addref_entry_on_disk); | |
1206 | else | |
1207 | BRTSTAT_BUMP(brt_addref_entry_not_on_disk); | |
1208 | /* | |
1209 | * When the BRT lock was dropped, brt_vdevs[] may have been | |
1210 | * expanded and reallocated, we need to update brtvd's pointer. | |
1211 | */ | |
1212 | brtvd = brt_vdev(brt, vdevid); | |
1213 | ASSERT(brtvd != NULL); | |
1214 | ||
1215 | racebre = avl_find(&brtvd->bv_tree, &bre_search, &where); | |
1216 | if (racebre == NULL) { | |
1217 | bre = brt_entry_alloc(&bre_search); | |
1218 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
1219 | avl_insert(&brtvd->bv_tree, bre, where); | |
1220 | brt->brt_nentries++; | |
1221 | } else { | |
1222 | /* | |
1223 | * The entry was added when the BRT lock was dropped in | |
1224 | * brt_entry_lookup(). | |
1225 | */ | |
1226 | BRTSTAT_BUMP(brt_addref_entry_read_lost_race); | |
1227 | bre = racebre; | |
1228 | } | |
1229 | } | |
1230 | bre->bre_refcount++; | |
1231 | brt_vdev_addref(brt, brtvd, bre, bp_get_dsize(brt->brt_spa, bp)); | |
1232 | ||
1233 | brt_unlock(brt); | |
1234 | } | |
1235 | ||
1236 | /* Return TRUE if block should be freed immediately. */ | |
1237 | boolean_t | |
1238 | brt_entry_decref(spa_t *spa, const blkptr_t *bp) | |
1239 | { | |
1240 | brt_t *brt = spa->spa_brt; | |
1241 | brt_vdev_t *brtvd; | |
1242 | brt_entry_t *bre, *racebre; | |
1243 | brt_entry_t bre_search; | |
1244 | avl_index_t where; | |
1245 | uint64_t vdevid; | |
1246 | int error; | |
1247 | ||
1248 | brt_entry_fill(bp, &bre_search, &vdevid); | |
1249 | ||
1250 | brt_wlock(brt); | |
1251 | ||
1252 | brtvd = brt_vdev(brt, vdevid); | |
1253 | ASSERT(brtvd != NULL); | |
1254 | ||
1255 | bre = avl_find(&brtvd->bv_tree, &bre_search, NULL); | |
1256 | if (bre != NULL) { | |
1257 | BRTSTAT_BUMP(brt_decref_entry_in_memory); | |
1258 | goto out; | |
1259 | } else { | |
1260 | BRTSTAT_BUMP(brt_decref_entry_not_in_memory); | |
1261 | } | |
1262 | ||
1263 | /* | |
1264 | * brt_entry_lookup() may drop the BRT lock and reacquire it. | |
1265 | */ | |
1266 | error = brt_entry_lookup(brt, brtvd, &bre_search); | |
1267 | /* bre_search now contains correct bre_refcount */ | |
1268 | ASSERT(error == 0 || error == ENOENT); | |
1269 | /* | |
1270 | * When the BRT lock was dropped, brt_vdevs[] may have been expanded | |
1271 | * and reallocated, we need to update brtvd's pointer. | |
1272 | */ | |
1273 | brtvd = brt_vdev(brt, vdevid); | |
1274 | ASSERT(brtvd != NULL); | |
1275 | ||
1276 | if (error == ENOENT) { | |
1277 | BRTSTAT_BUMP(brt_decref_entry_not_on_disk); | |
1278 | bre = NULL; | |
1279 | goto out; | |
1280 | } | |
1281 | ||
1282 | racebre = avl_find(&brtvd->bv_tree, &bre_search, &where); | |
1283 | if (racebre != NULL) { | |
1284 | /* | |
1285 | * The entry was added when the BRT lock was dropped in | |
1286 | * brt_entry_lookup(). | |
1287 | */ | |
1288 | BRTSTAT_BUMP(brt_decref_entry_read_lost_race); | |
1289 | bre = racebre; | |
1290 | goto out; | |
1291 | } | |
1292 | ||
1293 | BRTSTAT_BUMP(brt_decref_entry_loaded_from_disk); | |
1294 | bre = brt_entry_alloc(&bre_search); | |
1295 | ASSERT(RW_WRITE_HELD(&brt->brt_lock)); | |
1296 | avl_insert(&brtvd->bv_tree, bre, where); | |
1297 | brt->brt_nentries++; | |
1298 | ||
1299 | out: | |
1300 | if (bre == NULL) { | |
1301 | /* | |
1302 | * This is a free of a regular (not cloned) block. | |
1303 | */ | |
1304 | brt_unlock(brt); | |
1305 | BRTSTAT_BUMP(brt_decref_no_entry); | |
1306 | return (B_TRUE); | |
1307 | } | |
1308 | if (bre->bre_refcount == 0) { | |
1309 | brt_unlock(brt); | |
1310 | BRTSTAT_BUMP(brt_decref_free_data_now); | |
1311 | return (B_TRUE); | |
1312 | } | |
1313 | ||
1314 | ASSERT(bre->bre_refcount > 0); | |
1315 | bre->bre_refcount--; | |
1316 | if (bre->bre_refcount == 0) | |
1317 | BRTSTAT_BUMP(brt_decref_free_data_later); | |
1318 | else | |
1319 | BRTSTAT_BUMP(brt_decref_entry_still_referenced); | |
1320 | brt_vdev_decref(brt, brtvd, bre, bp_get_dsize(brt->brt_spa, bp)); | |
1321 | ||
1322 | brt_unlock(brt); | |
1323 | ||
1324 | return (B_FALSE); | |
1325 | } | |
1326 | ||
114a3996 RN |
1327 | uint64_t |
1328 | brt_entry_get_refcount(spa_t *spa, const blkptr_t *bp) | |
1329 | { | |
1330 | brt_t *brt = spa->spa_brt; | |
1331 | brt_vdev_t *brtvd; | |
1332 | brt_entry_t bre_search, *bre; | |
1333 | uint64_t vdevid, refcnt; | |
1334 | int error; | |
1335 | ||
1336 | brt_entry_fill(bp, &bre_search, &vdevid); | |
1337 | ||
1338 | brt_rlock(brt); | |
1339 | ||
1340 | brtvd = brt_vdev(brt, vdevid); | |
1341 | ASSERT(brtvd != NULL); | |
1342 | ||
1343 | bre = avl_find(&brtvd->bv_tree, &bre_search, NULL); | |
1344 | if (bre == NULL) { | |
1345 | error = brt_entry_lookup(brt, brtvd, &bre_search); | |
1346 | ASSERT(error == 0 || error == ENOENT); | |
1347 | if (error == ENOENT) | |
1348 | refcnt = 0; | |
1349 | else | |
1350 | refcnt = bre_search.bre_refcount; | |
1351 | } else | |
1352 | refcnt = bre->bre_refcount; | |
1353 | ||
1354 | brt_unlock(brt); | |
1355 | return (refcnt); | |
1356 | } | |
1357 | ||
67a1b037 PJD |
1358 | static void |
1359 | brt_prefetch(brt_t *brt, const blkptr_t *bp) | |
1360 | { | |
1361 | brt_entry_t bre; | |
1362 | uint64_t vdevid; | |
1363 | ||
1364 | ASSERT(bp != NULL); | |
1365 | ||
1366 | if (!zfs_brt_prefetch) | |
1367 | return; | |
1368 | ||
1369 | brt_entry_fill(bp, &bre, &vdevid); | |
1370 | ||
1371 | brt_entry_prefetch(brt, vdevid, &bre); | |
1372 | } | |
1373 | ||
1374 | static int | |
1375 | brt_pending_entry_compare(const void *x1, const void *x2) | |
1376 | { | |
1377 | const brt_pending_entry_t *bpe1 = x1, *bpe2 = x2; | |
1378 | const blkptr_t *bp1 = &bpe1->bpe_bp, *bp2 = &bpe2->bpe_bp; | |
1379 | int cmp; | |
1380 | ||
2c01cae8 AM |
1381 | cmp = TREE_CMP(DVA_GET_VDEV(&bp1->blk_dva[0]), |
1382 | DVA_GET_VDEV(&bp2->blk_dva[0])); | |
67a1b037 | 1383 | if (cmp == 0) { |
2c01cae8 AM |
1384 | cmp = TREE_CMP(DVA_GET_OFFSET(&bp1->blk_dva[0]), |
1385 | DVA_GET_OFFSET(&bp2->blk_dva[0])); | |
1386 | if (unlikely(cmp == 0)) { | |
1387 | cmp = TREE_CMP(BP_PHYSICAL_BIRTH(bp1), | |
1388 | BP_PHYSICAL_BIRTH(bp2)); | |
67a1b037 PJD |
1389 | } |
1390 | } | |
1391 | ||
1392 | return (cmp); | |
1393 | } | |
1394 | ||
1395 | void | |
1396 | brt_pending_add(spa_t *spa, const blkptr_t *bp, dmu_tx_t *tx) | |
1397 | { | |
1398 | brt_t *brt; | |
1399 | avl_tree_t *pending_tree; | |
1400 | kmutex_t *pending_lock; | |
1401 | brt_pending_entry_t *bpe, *newbpe; | |
1402 | avl_index_t where; | |
1403 | uint64_t txg; | |
1404 | ||
1405 | brt = spa->spa_brt; | |
1406 | txg = dmu_tx_get_txg(tx); | |
1407 | ASSERT3U(txg, !=, 0); | |
1408 | pending_tree = &brt->brt_pending_tree[txg & TXG_MASK]; | |
1409 | pending_lock = &brt->brt_pending_lock[txg & TXG_MASK]; | |
1410 | ||
1411 | newbpe = kmem_cache_alloc(brt_pending_entry_cache, KM_SLEEP); | |
1412 | newbpe->bpe_bp = *bp; | |
1413 | newbpe->bpe_count = 1; | |
1414 | ||
1415 | mutex_enter(pending_lock); | |
1416 | ||
1417 | bpe = avl_find(pending_tree, newbpe, &where); | |
1418 | if (bpe == NULL) { | |
1419 | avl_insert(pending_tree, newbpe, where); | |
1420 | newbpe = NULL; | |
1421 | } else { | |
1422 | bpe->bpe_count++; | |
1423 | } | |
1424 | ||
1425 | mutex_exit(pending_lock); | |
1426 | ||
1427 | if (newbpe != NULL) { | |
1428 | ASSERT(bpe != NULL); | |
1429 | ASSERT(bpe != newbpe); | |
1430 | kmem_cache_free(brt_pending_entry_cache, newbpe); | |
1431 | } else { | |
1432 | ASSERT(bpe == NULL); | |
67a1b037 | 1433 | |
bf8f7235 AM |
1434 | /* Prefetch BRT entry for the syncing context. */ |
1435 | brt_prefetch(brt, bp); | |
1436 | } | |
67a1b037 PJD |
1437 | } |
1438 | ||
1439 | void | |
1440 | brt_pending_remove(spa_t *spa, const blkptr_t *bp, dmu_tx_t *tx) | |
1441 | { | |
1442 | brt_t *brt; | |
1443 | avl_tree_t *pending_tree; | |
1444 | kmutex_t *pending_lock; | |
1445 | brt_pending_entry_t *bpe, bpe_search; | |
1446 | uint64_t txg; | |
1447 | ||
1448 | brt = spa->spa_brt; | |
1449 | txg = dmu_tx_get_txg(tx); | |
1450 | ASSERT3U(txg, !=, 0); | |
1451 | pending_tree = &brt->brt_pending_tree[txg & TXG_MASK]; | |
1452 | pending_lock = &brt->brt_pending_lock[txg & TXG_MASK]; | |
1453 | ||
1454 | bpe_search.bpe_bp = *bp; | |
1455 | ||
1456 | mutex_enter(pending_lock); | |
1457 | ||
1458 | bpe = avl_find(pending_tree, &bpe_search, NULL); | |
1459 | /* I believe we should always find bpe when this function is called. */ | |
1460 | if (bpe != NULL) { | |
1461 | ASSERT(bpe->bpe_count > 0); | |
1462 | ||
1463 | bpe->bpe_count--; | |
1464 | if (bpe->bpe_count == 0) { | |
1465 | avl_remove(pending_tree, bpe); | |
1466 | kmem_cache_free(brt_pending_entry_cache, bpe); | |
1467 | } | |
1468 | } | |
1469 | ||
1470 | mutex_exit(pending_lock); | |
1471 | } | |
1472 | ||
1473 | void | |
1474 | brt_pending_apply(spa_t *spa, uint64_t txg) | |
1475 | { | |
1476 | brt_t *brt; | |
1477 | brt_pending_entry_t *bpe; | |
1478 | avl_tree_t *pending_tree; | |
1479 | kmutex_t *pending_lock; | |
1480 | void *c; | |
1481 | ||
1482 | ASSERT3U(txg, !=, 0); | |
1483 | ||
1484 | brt = spa->spa_brt; | |
1485 | pending_tree = &brt->brt_pending_tree[txg & TXG_MASK]; | |
1486 | pending_lock = &brt->brt_pending_lock[txg & TXG_MASK]; | |
1487 | ||
1488 | mutex_enter(pending_lock); | |
1489 | ||
1490 | c = NULL; | |
1491 | while ((bpe = avl_destroy_nodes(pending_tree, &c)) != NULL) { | |
1492 | boolean_t added_to_ddt; | |
1493 | ||
1494 | mutex_exit(pending_lock); | |
1495 | ||
1496 | for (int i = 0; i < bpe->bpe_count; i++) { | |
1497 | /* | |
1498 | * If the block has DEDUP bit set, it means that it | |
1499 | * already exists in the DEDUP table, so we can just | |
1500 | * use that instead of creating new entry in | |
1501 | * the BRT table. | |
1502 | */ | |
1503 | if (BP_GET_DEDUP(&bpe->bpe_bp)) { | |
1504 | added_to_ddt = ddt_addref(spa, &bpe->bpe_bp); | |
1505 | } else { | |
1506 | added_to_ddt = B_FALSE; | |
1507 | } | |
1508 | if (!added_to_ddt) | |
1509 | brt_entry_addref(brt, &bpe->bpe_bp); | |
1510 | } | |
1511 | ||
1512 | kmem_cache_free(brt_pending_entry_cache, bpe); | |
1513 | mutex_enter(pending_lock); | |
1514 | } | |
1515 | ||
1516 | mutex_exit(pending_lock); | |
1517 | } | |
1518 | ||
1519 | static void | |
80cc5162 | 1520 | brt_sync_entry(dnode_t *dn, brt_entry_t *bre, dmu_tx_t *tx) |
67a1b037 | 1521 | { |
67a1b037 | 1522 | if (bre->bre_refcount == 0) { |
80cc5162 AM |
1523 | int error = zap_remove_uint64_by_dnode(dn, &bre->bre_offset, |
1524 | BRT_KEY_WORDS, tx); | |
1525 | VERIFY(error == 0 || error == ENOENT); | |
67a1b037 | 1526 | } else { |
80cc5162 AM |
1527 | VERIFY0(zap_update_uint64_by_dnode(dn, &bre->bre_offset, |
1528 | BRT_KEY_WORDS, 1, sizeof (bre->bre_refcount), | |
1529 | &bre->bre_refcount, tx)); | |
67a1b037 PJD |
1530 | } |
1531 | } | |
1532 | ||
1533 | static void | |
1534 | brt_sync_table(brt_t *brt, dmu_tx_t *tx) | |
1535 | { | |
1536 | brt_vdev_t *brtvd; | |
1537 | brt_entry_t *bre; | |
80cc5162 | 1538 | dnode_t *dn; |
67a1b037 PJD |
1539 | uint64_t vdevid; |
1540 | void *c; | |
1541 | ||
1542 | brt_wlock(brt); | |
1543 | ||
1544 | for (vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) { | |
1545 | brtvd = &brt->brt_vdevs[vdevid]; | |
1546 | ||
1547 | if (!brtvd->bv_initiated) | |
1548 | continue; | |
1549 | ||
1550 | if (!brtvd->bv_meta_dirty) { | |
1551 | ASSERT(!brtvd->bv_entcount_dirty); | |
1552 | ASSERT0(avl_numnodes(&brtvd->bv_tree)); | |
1553 | continue; | |
1554 | } | |
1555 | ||
1556 | ASSERT(!brtvd->bv_entcount_dirty || | |
1557 | avl_numnodes(&brtvd->bv_tree) != 0); | |
1558 | ||
1559 | if (brtvd->bv_mos_brtvdev == 0) | |
1560 | brt_vdev_create(brt, brtvd, tx); | |
1561 | ||
80cc5162 AM |
1562 | VERIFY0(dnode_hold(brt->brt_mos, brtvd->bv_mos_entries, |
1563 | FTAG, &dn)); | |
1564 | ||
67a1b037 PJD |
1565 | c = NULL; |
1566 | while ((bre = avl_destroy_nodes(&brtvd->bv_tree, &c)) != NULL) { | |
80cc5162 | 1567 | brt_sync_entry(dn, bre, tx); |
67a1b037 PJD |
1568 | brt_entry_free(bre); |
1569 | ASSERT(brt->brt_nentries > 0); | |
1570 | brt->brt_nentries--; | |
1571 | } | |
1572 | ||
80cc5162 AM |
1573 | dnode_rele(dn, FTAG); |
1574 | ||
67a1b037 PJD |
1575 | brt_vdev_sync(brt, brtvd, tx); |
1576 | ||
1577 | if (brtvd->bv_totalcount == 0) | |
1578 | brt_vdev_destroy(brt, brtvd, tx); | |
1579 | } | |
1580 | ||
1581 | ASSERT0(brt->brt_nentries); | |
1582 | ||
1583 | brt_unlock(brt); | |
1584 | } | |
1585 | ||
1586 | void | |
1587 | brt_sync(spa_t *spa, uint64_t txg) | |
1588 | { | |
1589 | dmu_tx_t *tx; | |
1590 | brt_t *brt; | |
1591 | ||
1592 | ASSERT(spa_syncing_txg(spa) == txg); | |
1593 | ||
1594 | brt = spa->spa_brt; | |
1595 | brt_rlock(brt); | |
1596 | if (brt->brt_nentries == 0) { | |
1597 | /* No changes. */ | |
1598 | brt_unlock(brt); | |
1599 | return; | |
1600 | } | |
1601 | brt_unlock(brt); | |
1602 | ||
1603 | tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); | |
1604 | ||
1605 | brt_sync_table(brt, tx); | |
1606 | ||
1607 | dmu_tx_commit(tx); | |
1608 | } | |
1609 | ||
1610 | static void | |
1611 | brt_table_alloc(brt_t *brt) | |
1612 | { | |
1613 | ||
1614 | for (int i = 0; i < TXG_SIZE; i++) { | |
1615 | avl_create(&brt->brt_pending_tree[i], | |
1616 | brt_pending_entry_compare, | |
1617 | sizeof (brt_pending_entry_t), | |
1618 | offsetof(brt_pending_entry_t, bpe_node)); | |
1619 | mutex_init(&brt->brt_pending_lock[i], NULL, MUTEX_DEFAULT, | |
1620 | NULL); | |
1621 | } | |
1622 | } | |
1623 | ||
1624 | static void | |
1625 | brt_table_free(brt_t *brt) | |
1626 | { | |
1627 | ||
1628 | for (int i = 0; i < TXG_SIZE; i++) { | |
1629 | ASSERT(avl_is_empty(&brt->brt_pending_tree[i])); | |
1630 | ||
1631 | avl_destroy(&brt->brt_pending_tree[i]); | |
1632 | mutex_destroy(&brt->brt_pending_lock[i]); | |
1633 | } | |
1634 | } | |
1635 | ||
1636 | static void | |
1637 | brt_alloc(spa_t *spa) | |
1638 | { | |
1639 | brt_t *brt; | |
1640 | ||
1641 | ASSERT(spa->spa_brt == NULL); | |
1642 | ||
1643 | brt = kmem_zalloc(sizeof (*brt), KM_SLEEP); | |
1644 | rw_init(&brt->brt_lock, NULL, RW_DEFAULT, NULL); | |
1645 | brt->brt_spa = spa; | |
1646 | brt->brt_rangesize = 0; | |
1647 | brt->brt_nentries = 0; | |
1648 | brt->brt_vdevs = NULL; | |
1649 | brt->brt_nvdevs = 0; | |
1650 | brt_table_alloc(brt); | |
1651 | ||
1652 | spa->spa_brt = brt; | |
1653 | } | |
1654 | ||
1655 | void | |
1656 | brt_create(spa_t *spa) | |
1657 | { | |
1658 | ||
1659 | brt_alloc(spa); | |
1660 | brt_vdevs_alloc(spa->spa_brt, B_FALSE); | |
1661 | } | |
1662 | ||
1663 | int | |
1664 | brt_load(spa_t *spa) | |
1665 | { | |
1666 | ||
1667 | brt_alloc(spa); | |
1668 | brt_vdevs_alloc(spa->spa_brt, B_TRUE); | |
1669 | ||
1670 | return (0); | |
1671 | } | |
1672 | ||
1673 | void | |
1674 | brt_unload(spa_t *spa) | |
1675 | { | |
1676 | brt_t *brt = spa->spa_brt; | |
1677 | ||
1678 | if (brt == NULL) | |
1679 | return; | |
1680 | ||
1681 | brt_vdevs_free(brt); | |
1682 | brt_table_free(brt); | |
1683 | rw_destroy(&brt->brt_lock); | |
1684 | kmem_free(brt, sizeof (*brt)); | |
1685 | spa->spa_brt = NULL; | |
1686 | } | |
1687 | ||
1688 | /* BEGIN CSTYLED */ | |
1689 | ZFS_MODULE_PARAM(zfs_brt, zfs_brt_, prefetch, INT, ZMOD_RW, | |
1690 | "Enable prefetching of BRT entries"); | |
1691 | #ifdef ZFS_BRT_DEBUG | |
1692 | ZFS_MODULE_PARAM(zfs_brt, zfs_brt_, debug, INT, ZMOD_RW, "BRT debug"); | |
1693 | #endif | |
1694 | /* END CSTYLED */ |