4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
25 * Copyright (c) 2022 by Pawel Jakub Dawidek
28 #include <sys/zfs_context.h>
30 #include <sys/spa_impl.h>
34 #include <sys/dmu_tx.h>
36 #include <sys/dsl_pool.h>
37 #include <sys/zio_checksum.h>
38 #include <sys/dsl_scan.h>
41 static kmem_cache_t
*ddt_cache
;
42 static kmem_cache_t
*ddt_entry_cache
;
45 * Enable/disable prefetching of dedup-ed blocks which are going to be freed.
47 int zfs_dedup_prefetch
= 0;
49 static const ddt_ops_t
*const ddt_ops
[DDT_TYPES
] = {
53 static const char *const ddt_class_name
[DDT_CLASSES
] = {
60 ddt_object_create(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
63 spa_t
*spa
= ddt
->ddt_spa
;
64 objset_t
*os
= ddt
->ddt_os
;
65 uint64_t *objectp
= &ddt
->ddt_object
[type
][class];
66 boolean_t prehash
= zio_checksum_table
[ddt
->ddt_checksum
].ci_flags
&
68 char name
[DDT_NAMELEN
];
70 ddt_object_name(ddt
, type
, class, name
);
72 ASSERT3U(*objectp
, ==, 0);
73 VERIFY0(ddt_ops
[type
]->ddt_op_create(os
, objectp
, tx
, prehash
));
74 ASSERT3U(*objectp
, !=, 0);
76 VERIFY0(zap_add(os
, DMU_POOL_DIRECTORY_OBJECT
, name
,
77 sizeof (uint64_t), 1, objectp
, tx
));
79 VERIFY0(zap_add(os
, spa
->spa_ddt_stat_object
, name
,
80 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
81 &ddt
->ddt_histogram
[type
][class], tx
));
85 ddt_object_destroy(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
88 spa_t
*spa
= ddt
->ddt_spa
;
89 objset_t
*os
= ddt
->ddt_os
;
90 uint64_t *objectp
= &ddt
->ddt_object
[type
][class];
92 char name
[DDT_NAMELEN
];
94 ddt_object_name(ddt
, type
, class, name
);
96 ASSERT3U(*objectp
, !=, 0);
97 ASSERT(ddt_histogram_empty(&ddt
->ddt_histogram
[type
][class]));
98 VERIFY0(ddt_object_count(ddt
, type
, class, &count
));
100 VERIFY0(zap_remove(os
, DMU_POOL_DIRECTORY_OBJECT
, name
, tx
));
101 VERIFY0(zap_remove(os
, spa
->spa_ddt_stat_object
, name
, tx
));
102 VERIFY0(ddt_ops
[type
]->ddt_op_destroy(os
, *objectp
, tx
));
103 memset(&ddt
->ddt_object_stats
[type
][class], 0, sizeof (ddt_object_t
));
109 ddt_object_load(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class)
111 ddt_object_t
*ddo
= &ddt
->ddt_object_stats
[type
][class];
112 dmu_object_info_t doi
;
114 char name
[DDT_NAMELEN
];
117 ddt_object_name(ddt
, type
, class, name
);
119 error
= zap_lookup(ddt
->ddt_os
, DMU_POOL_DIRECTORY_OBJECT
, name
,
120 sizeof (uint64_t), 1, &ddt
->ddt_object
[type
][class]);
124 error
= zap_lookup(ddt
->ddt_os
, ddt
->ddt_spa
->spa_ddt_stat_object
, name
,
125 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
126 &ddt
->ddt_histogram
[type
][class]);
131 * Seed the cached statistics.
133 error
= ddt_object_info(ddt
, type
, class, &doi
);
137 error
= ddt_object_count(ddt
, type
, class, &count
);
141 ddo
->ddo_count
= count
;
142 ddo
->ddo_dspace
= doi
.doi_physical_blocks_512
<< 9;
143 ddo
->ddo_mspace
= doi
.doi_fill_count
* doi
.doi_data_block_size
;
149 ddt_object_sync(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
152 ddt_object_t
*ddo
= &ddt
->ddt_object_stats
[type
][class];
153 dmu_object_info_t doi
;
155 char name
[DDT_NAMELEN
];
157 ddt_object_name(ddt
, type
, class, name
);
159 VERIFY0(zap_update(ddt
->ddt_os
, ddt
->ddt_spa
->spa_ddt_stat_object
, name
,
160 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
161 &ddt
->ddt_histogram
[type
][class], tx
));
164 * Cache DDT statistics; this is the only time they'll change.
166 VERIFY0(ddt_object_info(ddt
, type
, class, &doi
));
167 VERIFY0(ddt_object_count(ddt
, type
, class, &count
));
169 ddo
->ddo_count
= count
;
170 ddo
->ddo_dspace
= doi
.doi_physical_blocks_512
<< 9;
171 ddo
->ddo_mspace
= doi
.doi_fill_count
* doi
.doi_data_block_size
;
175 ddt_object_lookup(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
178 if (!ddt_object_exists(ddt
, type
, class))
179 return (SET_ERROR(ENOENT
));
181 return (ddt_ops
[type
]->ddt_op_lookup(ddt
->ddt_os
,
182 ddt
->ddt_object
[type
][class], dde
));
186 ddt_object_prefetch(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
189 if (!ddt_object_exists(ddt
, type
, class))
192 ddt_ops
[type
]->ddt_op_prefetch(ddt
->ddt_os
,
193 ddt
->ddt_object
[type
][class], dde
);
197 ddt_object_update(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
198 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
200 ASSERT(ddt_object_exists(ddt
, type
, class));
202 return (ddt_ops
[type
]->ddt_op_update(ddt
->ddt_os
,
203 ddt
->ddt_object
[type
][class], dde
, tx
));
207 ddt_object_remove(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
208 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
210 ASSERT(ddt_object_exists(ddt
, type
, class));
212 return (ddt_ops
[type
]->ddt_op_remove(ddt
->ddt_os
,
213 ddt
->ddt_object
[type
][class], dde
, tx
));
217 ddt_object_walk(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
218 uint64_t *walk
, ddt_entry_t
*dde
)
220 ASSERT(ddt_object_exists(ddt
, type
, class));
222 return (ddt_ops
[type
]->ddt_op_walk(ddt
->ddt_os
,
223 ddt
->ddt_object
[type
][class], dde
, walk
));
227 ddt_object_count(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
230 ASSERT(ddt_object_exists(ddt
, type
, class));
232 return (ddt_ops
[type
]->ddt_op_count(ddt
->ddt_os
,
233 ddt
->ddt_object
[type
][class], count
));
237 ddt_object_info(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
238 dmu_object_info_t
*doi
)
240 if (!ddt_object_exists(ddt
, type
, class))
241 return (SET_ERROR(ENOENT
));
243 return (dmu_object_info(ddt
->ddt_os
, ddt
->ddt_object
[type
][class],
248 ddt_object_exists(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class)
250 return (!!ddt
->ddt_object
[type
][class]);
254 ddt_object_name(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
257 (void) snprintf(name
, DDT_NAMELEN
, DMU_POOL_DDT
,
258 zio_checksum_table
[ddt
->ddt_checksum
].ci_name
,
259 ddt_ops
[type
]->ddt_op_name
, ddt_class_name
[class]);
263 ddt_bp_fill(const ddt_phys_t
*ddp
, blkptr_t
*bp
, uint64_t txg
)
265 ASSERT3U(txg
, !=, 0);
267 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++)
268 bp
->blk_dva
[d
] = ddp
->ddp_dva
[d
];
269 BP_SET_BIRTH(bp
, txg
, ddp
->ddp_phys_birth
);
273 * The bp created via this function may be used for repairs and scrub, but it
274 * will be missing the salt / IV required to do a full decrypting read.
277 ddt_bp_create(enum zio_checksum checksum
,
278 const ddt_key_t
*ddk
, const ddt_phys_t
*ddp
, blkptr_t
*bp
)
283 ddt_bp_fill(ddp
, bp
, ddp
->ddp_phys_birth
);
285 bp
->blk_cksum
= ddk
->ddk_cksum
;
287 BP_SET_LSIZE(bp
, DDK_GET_LSIZE(ddk
));
288 BP_SET_PSIZE(bp
, DDK_GET_PSIZE(ddk
));
289 BP_SET_COMPRESS(bp
, DDK_GET_COMPRESS(ddk
));
290 BP_SET_CRYPT(bp
, DDK_GET_CRYPT(ddk
));
292 BP_SET_CHECKSUM(bp
, checksum
);
293 BP_SET_TYPE(bp
, DMU_OT_DEDUP
);
296 BP_SET_BYTEORDER(bp
, ZFS_HOST_BYTEORDER
);
300 ddt_key_fill(ddt_key_t
*ddk
, const blkptr_t
*bp
)
302 ddk
->ddk_cksum
= bp
->blk_cksum
;
305 ASSERT(BP_IS_ENCRYPTED(bp
) || !BP_USES_CRYPT(bp
));
307 DDK_SET_LSIZE(ddk
, BP_GET_LSIZE(bp
));
308 DDK_SET_PSIZE(ddk
, BP_GET_PSIZE(bp
));
309 DDK_SET_COMPRESS(ddk
, BP_GET_COMPRESS(bp
));
310 DDK_SET_CRYPT(ddk
, BP_USES_CRYPT(bp
));
314 ddt_phys_fill(ddt_phys_t
*ddp
, const blkptr_t
*bp
)
316 ASSERT0(ddp
->ddp_phys_birth
);
318 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++)
319 ddp
->ddp_dva
[d
] = bp
->blk_dva
[d
];
320 ddp
->ddp_phys_birth
= BP_PHYSICAL_BIRTH(bp
);
324 ddt_phys_clear(ddt_phys_t
*ddp
)
326 memset(ddp
, 0, sizeof (*ddp
));
330 ddt_phys_addref(ddt_phys_t
*ddp
)
336 ddt_phys_decref(ddt_phys_t
*ddp
)
339 ASSERT3U(ddp
->ddp_refcnt
, >, 0);
345 ddt_phys_free(ddt_t
*ddt
, ddt_key_t
*ddk
, ddt_phys_t
*ddp
, uint64_t txg
)
349 ddt_bp_create(ddt
->ddt_checksum
, ddk
, ddp
, &blk
);
352 * We clear the dedup bit so that zio_free() will actually free the
353 * space, rather than just decrementing the refcount in the DDT.
355 BP_SET_DEDUP(&blk
, 0);
358 zio_free(ddt
->ddt_spa
, txg
, &blk
);
362 ddt_phys_select(const ddt_entry_t
*dde
, const blkptr_t
*bp
)
364 ddt_phys_t
*ddp
= (ddt_phys_t
*)dde
->dde_phys
;
366 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
367 if (DVA_EQUAL(BP_IDENTITY(bp
), &ddp
->ddp_dva
[0]) &&
368 BP_PHYSICAL_BIRTH(bp
) == ddp
->ddp_phys_birth
)
375 ddt_phys_total_refcnt(const ddt_entry_t
*dde
)
379 for (int p
= DDT_PHYS_SINGLE
; p
<= DDT_PHYS_TRIPLE
; p
++)
380 refcnt
+= dde
->dde_phys
[p
].ddp_refcnt
;
386 ddt_select(spa_t
*spa
, const blkptr_t
*bp
)
388 return (spa
->spa_ddt
[BP_GET_CHECKSUM(bp
)]);
392 ddt_enter(ddt_t
*ddt
)
394 mutex_enter(&ddt
->ddt_lock
);
400 mutex_exit(&ddt
->ddt_lock
);
406 ddt_cache
= kmem_cache_create("ddt_cache",
407 sizeof (ddt_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
408 ddt_entry_cache
= kmem_cache_create("ddt_entry_cache",
409 sizeof (ddt_entry_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
415 kmem_cache_destroy(ddt_entry_cache
);
416 kmem_cache_destroy(ddt_cache
);
420 ddt_alloc(const ddt_key_t
*ddk
)
424 dde
= kmem_cache_alloc(ddt_entry_cache
, KM_SLEEP
);
425 memset(dde
, 0, sizeof (ddt_entry_t
));
426 cv_init(&dde
->dde_cv
, NULL
, CV_DEFAULT
, NULL
);
434 ddt_free(ddt_entry_t
*dde
)
436 ASSERT(!dde
->dde_loading
);
438 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++)
439 ASSERT3P(dde
->dde_lead_zio
[p
], ==, NULL
);
441 if (dde
->dde_repair_abd
!= NULL
)
442 abd_free(dde
->dde_repair_abd
);
444 cv_destroy(&dde
->dde_cv
);
445 kmem_cache_free(ddt_entry_cache
, dde
);
449 ddt_remove(ddt_t
*ddt
, ddt_entry_t
*dde
)
451 ASSERT(MUTEX_HELD(&ddt
->ddt_lock
));
453 avl_remove(&ddt
->ddt_tree
, dde
);
458 ddt_lookup(ddt_t
*ddt
, const blkptr_t
*bp
, boolean_t add
)
463 enum ddt_class
class;
467 ASSERT(MUTEX_HELD(&ddt
->ddt_lock
));
469 ddt_key_fill(&search
, bp
);
471 dde
= avl_find(&ddt
->ddt_tree
, &search
, &where
);
475 dde
= ddt_alloc(&search
);
476 avl_insert(&ddt
->ddt_tree
, dde
, where
);
479 while (dde
->dde_loading
)
480 cv_wait(&dde
->dde_cv
, &ddt
->ddt_lock
);
485 dde
->dde_loading
= B_TRUE
;
491 for (type
= 0; type
< DDT_TYPES
; type
++) {
492 for (class = 0; class < DDT_CLASSES
; class++) {
493 error
= ddt_object_lookup(ddt
, type
, class, dde
);
494 if (error
!= ENOENT
) {
505 ASSERT(!dde
->dde_loaded
);
506 ASSERT(dde
->dde_loading
);
508 dde
->dde_type
= type
; /* will be DDT_TYPES if no entry found */
509 dde
->dde_class
= class; /* will be DDT_CLASSES if no entry found */
510 dde
->dde_loaded
= B_TRUE
;
511 dde
->dde_loading
= B_FALSE
;
514 ddt_stat_update(ddt
, dde
, -1ULL);
516 cv_broadcast(&dde
->dde_cv
);
522 ddt_prefetch(spa_t
*spa
, const blkptr_t
*bp
)
527 if (!zfs_dedup_prefetch
|| bp
== NULL
|| !BP_GET_DEDUP(bp
))
531 * We only remove the DDT once all tables are empty and only
532 * prefetch dedup blocks when there are entries in the DDT.
533 * Thus no locking is required as the DDT can't disappear on us.
535 ddt
= ddt_select(spa
, bp
);
536 ddt_key_fill(&dde
.dde_key
, bp
);
538 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
539 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
540 ddt_object_prefetch(ddt
, type
, class, &dde
);
546 * Key comparison. Any struct wanting to make use of this function must have
547 * the key as the first element.
549 #define DDT_KEY_CMP_LEN (sizeof (ddt_key_t) / sizeof (uint16_t))
551 typedef struct ddt_key_cmp
{
552 uint16_t u16
[DDT_KEY_CMP_LEN
];
556 ddt_key_compare(const void *x1
, const void *x2
)
558 const ddt_key_cmp_t
*k1
= (const ddt_key_cmp_t
*)x1
;
559 const ddt_key_cmp_t
*k2
= (const ddt_key_cmp_t
*)x2
;
562 for (int i
= 0; i
< DDT_KEY_CMP_LEN
; i
++) {
563 cmp
= (int32_t)k1
->u16
[i
] - (int32_t)k2
->u16
[i
];
568 return (TREE_ISIGN(cmp
));
572 ddt_table_alloc(spa_t
*spa
, enum zio_checksum c
)
576 ddt
= kmem_cache_alloc(ddt_cache
, KM_SLEEP
);
577 memset(ddt
, 0, sizeof (ddt_t
));
579 mutex_init(&ddt
->ddt_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
580 avl_create(&ddt
->ddt_tree
, ddt_key_compare
,
581 sizeof (ddt_entry_t
), offsetof(ddt_entry_t
, dde_node
));
582 avl_create(&ddt
->ddt_repair_tree
, ddt_key_compare
,
583 sizeof (ddt_entry_t
), offsetof(ddt_entry_t
, dde_node
));
584 ddt
->ddt_checksum
= c
;
586 ddt
->ddt_os
= spa
->spa_meta_objset
;
592 ddt_table_free(ddt_t
*ddt
)
594 ASSERT0(avl_numnodes(&ddt
->ddt_tree
));
595 ASSERT0(avl_numnodes(&ddt
->ddt_repair_tree
));
596 avl_destroy(&ddt
->ddt_tree
);
597 avl_destroy(&ddt
->ddt_repair_tree
);
598 mutex_destroy(&ddt
->ddt_lock
);
599 kmem_cache_free(ddt_cache
, ddt
);
603 ddt_create(spa_t
*spa
)
605 spa
->spa_dedup_checksum
= ZIO_DEDUPCHECKSUM
;
607 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++)
608 spa
->spa_ddt
[c
] = ddt_table_alloc(spa
, c
);
618 error
= zap_lookup(spa
->spa_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
619 DMU_POOL_DDT_STATS
, sizeof (uint64_t), 1,
620 &spa
->spa_ddt_stat_object
);
623 return (error
== ENOENT
? 0 : error
);
625 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
626 ddt_t
*ddt
= spa
->spa_ddt
[c
];
627 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
628 for (enum ddt_class
class = 0; class < DDT_CLASSES
;
630 error
= ddt_object_load(ddt
, type
, class);
631 if (error
!= 0 && error
!= ENOENT
)
637 * Seed the cached histograms.
639 memcpy(&ddt
->ddt_histogram_cache
, ddt
->ddt_histogram
,
640 sizeof (ddt
->ddt_histogram
));
641 spa
->spa_dedup_dspace
= ~0ULL;
648 ddt_unload(spa_t
*spa
)
650 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
651 if (spa
->spa_ddt
[c
]) {
652 ddt_table_free(spa
->spa_ddt
[c
]);
653 spa
->spa_ddt
[c
] = NULL
;
659 ddt_class_contains(spa_t
*spa
, enum ddt_class max_class
, const blkptr_t
*bp
)
664 if (!BP_GET_DEDUP(bp
))
667 if (max_class
== DDT_CLASS_UNIQUE
)
670 ddt
= spa
->spa_ddt
[BP_GET_CHECKSUM(bp
)];
671 dde
= kmem_cache_alloc(ddt_entry_cache
, KM_SLEEP
);
673 ddt_key_fill(&(dde
->dde_key
), bp
);
675 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
676 for (enum ddt_class
class = 0; class <= max_class
; class++) {
677 if (ddt_object_lookup(ddt
, type
, class, dde
) == 0) {
678 kmem_cache_free(ddt_entry_cache
, dde
);
684 kmem_cache_free(ddt_entry_cache
, dde
);
689 ddt_repair_start(ddt_t
*ddt
, const blkptr_t
*bp
)
694 ddt_key_fill(&ddk
, bp
);
696 dde
= ddt_alloc(&ddk
);
698 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
699 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
701 * We can only do repair if there are multiple copies
702 * of the block. For anything in the UNIQUE class,
703 * there's definitely only one copy, so don't even try.
705 if (class != DDT_CLASS_UNIQUE
&&
706 ddt_object_lookup(ddt
, type
, class, dde
) == 0)
711 memset(dde
->dde_phys
, 0, sizeof (dde
->dde_phys
));
717 ddt_repair_done(ddt_t
*ddt
, ddt_entry_t
*dde
)
723 if (dde
->dde_repair_abd
!= NULL
&& spa_writeable(ddt
->ddt_spa
) &&
724 avl_find(&ddt
->ddt_repair_tree
, dde
, &where
) == NULL
)
725 avl_insert(&ddt
->ddt_repair_tree
, dde
, where
);
733 ddt_repair_entry_done(zio_t
*zio
)
735 ddt_entry_t
*rdde
= zio
->io_private
;
741 ddt_repair_entry(ddt_t
*ddt
, ddt_entry_t
*dde
, ddt_entry_t
*rdde
, zio_t
*rio
)
743 ddt_phys_t
*ddp
= dde
->dde_phys
;
744 ddt_phys_t
*rddp
= rdde
->dde_phys
;
745 ddt_key_t
*ddk
= &dde
->dde_key
;
746 ddt_key_t
*rddk
= &rdde
->dde_key
;
750 zio
= zio_null(rio
, rio
->io_spa
, NULL
,
751 ddt_repair_entry_done
, rdde
, rio
->io_flags
);
753 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++, rddp
++) {
754 if (ddp
->ddp_phys_birth
== 0 ||
755 ddp
->ddp_phys_birth
!= rddp
->ddp_phys_birth
||
756 memcmp(ddp
->ddp_dva
, rddp
->ddp_dva
, sizeof (ddp
->ddp_dva
)))
758 ddt_bp_create(ddt
->ddt_checksum
, ddk
, ddp
, &blk
);
759 zio_nowait(zio_rewrite(zio
, zio
->io_spa
, 0, &blk
,
760 rdde
->dde_repair_abd
, DDK_GET_PSIZE(rddk
), NULL
, NULL
,
761 ZIO_PRIORITY_SYNC_WRITE
, ZIO_DDT_CHILD_FLAGS(zio
), NULL
));
768 ddt_repair_table(ddt_t
*ddt
, zio_t
*rio
)
770 spa_t
*spa
= ddt
->ddt_spa
;
771 ddt_entry_t
*dde
, *rdde_next
, *rdde
;
772 avl_tree_t
*t
= &ddt
->ddt_repair_tree
;
775 if (spa_sync_pass(spa
) > 1)
779 for (rdde
= avl_first(t
); rdde
!= NULL
; rdde
= rdde_next
) {
780 rdde_next
= AVL_NEXT(t
, rdde
);
781 avl_remove(&ddt
->ddt_repair_tree
, rdde
);
783 ddt_bp_create(ddt
->ddt_checksum
, &rdde
->dde_key
, NULL
, &blk
);
784 dde
= ddt_repair_start(ddt
, &blk
);
785 ddt_repair_entry(ddt
, dde
, rdde
, rio
);
786 ddt_repair_done(ddt
, dde
);
793 ddt_sync_entry(ddt_t
*ddt
, ddt_entry_t
*dde
, dmu_tx_t
*tx
, uint64_t txg
)
795 dsl_pool_t
*dp
= ddt
->ddt_spa
->spa_dsl_pool
;
796 ddt_phys_t
*ddp
= dde
->dde_phys
;
797 ddt_key_t
*ddk
= &dde
->dde_key
;
798 enum ddt_type otype
= dde
->dde_type
;
799 enum ddt_type ntype
= DDT_TYPE_CURRENT
;
800 enum ddt_class oclass
= dde
->dde_class
;
801 enum ddt_class nclass
;
802 uint64_t total_refcnt
= 0;
804 ASSERT(dde
->dde_loaded
);
805 ASSERT(!dde
->dde_loading
);
807 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
808 ASSERT3P(dde
->dde_lead_zio
[p
], ==, NULL
);
809 if (ddp
->ddp_phys_birth
== 0) {
810 ASSERT0(ddp
->ddp_refcnt
);
813 if (p
== DDT_PHYS_DITTO
) {
815 * Note, we no longer create DDT-DITTO blocks, but we
816 * don't want to leak any written by older software.
818 ddt_phys_free(ddt
, ddk
, ddp
, txg
);
821 if (ddp
->ddp_refcnt
== 0)
822 ddt_phys_free(ddt
, ddk
, ddp
, txg
);
823 total_refcnt
+= ddp
->ddp_refcnt
;
826 /* We do not create new DDT-DITTO blocks. */
827 ASSERT0(dde
->dde_phys
[DDT_PHYS_DITTO
].ddp_phys_birth
);
828 if (total_refcnt
> 1)
829 nclass
= DDT_CLASS_DUPLICATE
;
831 nclass
= DDT_CLASS_UNIQUE
;
833 if (otype
!= DDT_TYPES
&&
834 (otype
!= ntype
|| oclass
!= nclass
|| total_refcnt
== 0)) {
835 VERIFY0(ddt_object_remove(ddt
, otype
, oclass
, dde
, tx
));
837 ddt_object_lookup(ddt
, otype
, oclass
, dde
), ==, ENOENT
);
840 if (total_refcnt
!= 0) {
841 dde
->dde_type
= ntype
;
842 dde
->dde_class
= nclass
;
843 ddt_stat_update(ddt
, dde
, 0);
844 if (!ddt_object_exists(ddt
, ntype
, nclass
))
845 ddt_object_create(ddt
, ntype
, nclass
, tx
);
846 VERIFY0(ddt_object_update(ddt
, ntype
, nclass
, dde
, tx
));
849 * If the class changes, the order that we scan this bp
850 * changes. If it decreases, we could miss it, so
851 * scan it right now. (This covers both class changing
852 * while we are doing ddt_walk(), and when we are
855 if (nclass
< oclass
) {
856 dsl_scan_ddt_entry(dp
->dp_scan
,
857 ddt
->ddt_checksum
, dde
, tx
);
863 ddt_sync_table(ddt_t
*ddt
, dmu_tx_t
*tx
, uint64_t txg
)
865 spa_t
*spa
= ddt
->ddt_spa
;
869 if (avl_numnodes(&ddt
->ddt_tree
) == 0)
872 ASSERT3U(spa
->spa_uberblock
.ub_version
, >=, SPA_VERSION_DEDUP
);
874 if (spa
->spa_ddt_stat_object
== 0) {
875 spa
->spa_ddt_stat_object
= zap_create_link(ddt
->ddt_os
,
876 DMU_OT_DDT_STATS
, DMU_POOL_DIRECTORY_OBJECT
,
877 DMU_POOL_DDT_STATS
, tx
);
880 while ((dde
= avl_destroy_nodes(&ddt
->ddt_tree
, &cookie
)) != NULL
) {
881 ddt_sync_entry(ddt
, dde
, tx
, txg
);
885 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
886 uint64_t add
, count
= 0;
887 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
888 if (ddt_object_exists(ddt
, type
, class)) {
889 ddt_object_sync(ddt
, type
, class, tx
);
890 VERIFY0(ddt_object_count(ddt
, type
, class,
895 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
896 if (count
== 0 && ddt_object_exists(ddt
, type
, class))
897 ddt_object_destroy(ddt
, type
, class, tx
);
901 memcpy(&ddt
->ddt_histogram_cache
, ddt
->ddt_histogram
,
902 sizeof (ddt
->ddt_histogram
));
903 spa
->spa_dedup_dspace
= ~0ULL;
907 ddt_sync(spa_t
*spa
, uint64_t txg
)
909 dsl_scan_t
*scn
= spa
->spa_dsl_pool
->dp_scan
;
913 ASSERT3U(spa_syncing_txg(spa
), ==, txg
);
915 tx
= dmu_tx_create_assigned(spa
->spa_dsl_pool
, txg
);
917 rio
= zio_root(spa
, NULL
, NULL
,
918 ZIO_FLAG_CANFAIL
| ZIO_FLAG_SPECULATIVE
| ZIO_FLAG_SELF_HEAL
);
921 * This function may cause an immediate scan of ddt blocks (see
922 * the comment above dsl_scan_ddt() for details). We set the
923 * scan's root zio here so that we can wait for any scan IOs in
924 * addition to the regular ddt IOs.
926 ASSERT3P(scn
->scn_zio_root
, ==, NULL
);
927 scn
->scn_zio_root
= rio
;
929 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
930 ddt_t
*ddt
= spa
->spa_ddt
[c
];
933 ddt_sync_table(ddt
, tx
, txg
);
934 ddt_repair_table(ddt
, rio
);
937 (void) zio_wait(rio
);
938 scn
->scn_zio_root
= NULL
;
944 ddt_walk(spa_t
*spa
, ddt_bookmark_t
*ddb
, ddt_entry_t
*dde
)
949 ddt_t
*ddt
= spa
->spa_ddt
[ddb
->ddb_checksum
];
951 if (ddt_object_exists(ddt
, ddb
->ddb_type
,
953 error
= ddt_object_walk(ddt
,
954 ddb
->ddb_type
, ddb
->ddb_class
,
955 &ddb
->ddb_cursor
, dde
);
957 dde
->dde_type
= ddb
->ddb_type
;
958 dde
->dde_class
= ddb
->ddb_class
;
964 } while (++ddb
->ddb_checksum
< ZIO_CHECKSUM_FUNCTIONS
);
965 ddb
->ddb_checksum
= 0;
966 } while (++ddb
->ddb_type
< DDT_TYPES
);
968 } while (++ddb
->ddb_class
< DDT_CLASSES
);
970 return (SET_ERROR(ENOENT
));
974 * This function is used by Block Cloning (brt.c) to increase reference
975 * counter for the DDT entry if the block is already in DDT.
977 * Return false if the block, despite having the D bit set, is not present
978 * in the DDT. Currently this is not possible but might be in the future.
979 * See the comment below.
982 ddt_addref(spa_t
*spa
, const blkptr_t
*bp
)
988 spa_config_enter(spa
, SCL_ZIO
, FTAG
, RW_READER
);
989 ddt
= ddt_select(spa
, bp
);
992 dde
= ddt_lookup(ddt
, bp
, B_TRUE
);
993 ASSERT3P(dde
, !=, NULL
);
995 if (dde
->dde_type
< DDT_TYPES
) {
998 ASSERT3S(dde
->dde_class
, <, DDT_CLASSES
);
1000 ddp
= &dde
->dde_phys
[BP_GET_NDVAS(bp
)];
1003 * This entry already existed (dde_type is real), so it must
1004 * have refcnt >0 at the start of this txg. We are called from
1005 * brt_pending_apply(), before frees are issued, so the refcnt
1006 * can't be lowered yet. Therefore, it must be >0. We assert
1007 * this because if the order of BRT and DDT interactions were
1008 * ever to change and the refcnt was ever zero here, then
1009 * likely further action is required to fill out the DDT entry,
1010 * and this is a place that is likely to be missed in testing.
1012 ASSERT3U(ddp
->ddp_refcnt
, >, 0);
1014 ddt_phys_addref(ddp
);
1018 * At the time of implementating this if the block has the
1019 * DEDUP flag set it must exist in the DEDUP table, but
1020 * there are many advocates that want ability to remove
1021 * entries from DDT with refcnt=1. If this will happen,
1022 * we may have a block with the DEDUP set, but which doesn't
1023 * have a corresponding entry in the DDT. Be ready.
1025 ASSERT3S(dde
->dde_class
, ==, DDT_CLASSES
);
1026 ddt_remove(ddt
, dde
);
1031 spa_config_exit(spa
, SCL_ZIO
, FTAG
);
1036 ZFS_MODULE_PARAM(zfs_dedup
, zfs_dedup_
, prefetch
, INT
, ZMOD_RW
,
1037 "Enable prefetching dedup-ed blks");