4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2016 Gary Mills
25 * Copyright (c) 2017 Datto Inc.
26 * Copyright 2017 Joyent, Inc.
29 #include <sys/dsl_scan.h>
30 #include <sys/dsl_pool.h>
31 #include <sys/dsl_dataset.h>
32 #include <sys/dsl_prop.h>
33 #include <sys/dsl_dir.h>
34 #include <sys/dsl_synctask.h>
35 #include <sys/dnode.h>
36 #include <sys/dmu_tx.h>
37 #include <sys/dmu_objset.h>
41 #include <sys/zfs_context.h>
42 #include <sys/fs/zfs.h>
43 #include <sys/zfs_znode.h>
44 #include <sys/spa_impl.h>
45 #include <sys/vdev_impl.h>
46 #include <sys/zil_impl.h>
47 #include <sys/zio_checksum.h>
50 #include <sys/sa_impl.h>
51 #include <sys/zfeature.h>
53 #include <sys/range_tree.h>
55 #include <sys/zfs_vfsops.h>
59 * Grand theory statement on scan queue sorting
61 * Scanning is implemented by recursively traversing all indirection levels
62 * in an object and reading all blocks referenced from said objects. This
63 * results in us approximately traversing the object from lowest logical
64 * offset to the highest. For best performance, we would want the logical
65 * blocks to be physically contiguous. However, this is frequently not the
66 * case with pools given the allocation patterns of copy-on-write filesystems.
67 * So instead, we put the I/Os into a reordering queue and issue them in a
68 * way that will most benefit physical disks (LBA-order).
72 * Ideally, we would want to scan all metadata and queue up all block I/O
73 * prior to starting to issue it, because that allows us to do an optimal
74 * sorting job. This can however consume large amounts of memory. Therefore
75 * we continuously monitor the size of the queues and constrain them to 5%
76 * (zfs_scan_mem_lim_fact) of physmem. If the queues grow larger than this
77 * limit, we clear out a few of the largest extents at the head of the queues
78 * to make room for more scanning. Hopefully, these extents will be fairly
79 * large and contiguous, allowing us to approach sequential I/O throughput
80 * even without a fully sorted tree.
82 * Metadata scanning takes place in dsl_scan_visit(), which is called from
83 * dsl_scan_sync() every spa_sync(). If we have either fully scanned all
84 * metadata on the pool, or we need to make room in memory because our
85 * queues are too large, dsl_scan_visit() is postponed and
86 * scan_io_queues_run() is called from dsl_scan_sync() instead. This implies
87 * that metadata scanning and queued I/O issuing are mutually exclusive. This
88 * allows us to provide maximum sequential I/O throughput for the majority of
89 * I/O's issued since sequential I/O performance is significantly negatively
90 * impacted if it is interleaved with random I/O.
92 * Implementation Notes
94 * One side effect of the queued scanning algorithm is that the scanning code
95 * needs to be notified whenever a block is freed. This is needed to allow
96 * the scanning code to remove these I/Os from the issuing queue. Additionally,
97 * we do not attempt to queue gang blocks to be issued sequentially since this
98 * is very hard to do and would have an extremely limitted performance benefit.
99 * Instead, we simply issue gang I/Os as soon as we find them using the legacy
102 * Backwards compatibility
104 * This new algorithm is backwards compatible with the legacy on-disk data
105 * structures (and therefore does not require a new feature flag).
106 * Periodically during scanning (see zfs_scan_checkpoint_intval), the scan
107 * will stop scanning metadata (in logical order) and wait for all outstanding
108 * sorted I/O to complete. Once this is done, we write out a checkpoint
109 * bookmark, indicating that we have scanned everything logically before it.
110 * If the pool is imported on a machine without the new sorting algorithm,
111 * the scan simply resumes from the last checkpoint using the legacy algorithm.
114 typedef int (scan_cb_t
)(dsl_pool_t
*, const blkptr_t
*,
115 const zbookmark_phys_t
*);
117 static scan_cb_t dsl_scan_scrub_cb
;
119 static int scan_ds_queue_compare(const void *a
, const void *b
);
120 static int scan_prefetch_queue_compare(const void *a
, const void *b
);
121 static void scan_ds_queue_clear(dsl_scan_t
*scn
);
122 static boolean_t
scan_ds_queue_contains(dsl_scan_t
*scn
, uint64_t dsobj
,
124 static void scan_ds_queue_insert(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t txg
);
125 static void scan_ds_queue_remove(dsl_scan_t
*scn
, uint64_t dsobj
);
126 static void scan_ds_queue_sync(dsl_scan_t
*scn
, dmu_tx_t
*tx
);
128 extern int zfs_vdev_async_write_active_min_dirty_percent
;
131 * By default zfs will check to ensure it is not over the hard memory
132 * limit before each txg. If finer-grained control of this is needed
133 * this value can be set to 1 to enable checking before scanning each
136 int zfs_scan_strict_mem_lim
= B_FALSE
;
139 * Maximum number of parallelly executed bytes per leaf vdev. We attempt
140 * to strike a balance here between keeping the vdev queues full of I/Os
141 * at all times and not overflowing the queues to cause long latency,
142 * which would cause long txg sync times. No matter what, we will not
143 * overload the drives with I/O, since that is protected by
144 * zfs_vdev_scrub_max_active.
146 unsigned long zfs_scan_vdev_limit
= 4 << 20;
148 int zfs_scan_issue_strategy
= 0;
149 int zfs_scan_legacy
= B_FALSE
; /* don't queue & sort zios, go direct */
150 uint64_t zfs_scan_max_ext_gap
= 2 << 20; /* in bytes */
153 * fill_weight is non-tunable at runtime, so we copy it at module init from
154 * zfs_scan_fill_weight. Runtime adjustments to zfs_scan_fill_weight would
155 * break queue sorting.
157 int zfs_scan_fill_weight
= 3;
158 static uint64_t fill_weight
;
160 /* See dsl_scan_should_clear() for details on the memory limit tunables */
161 uint64_t zfs_scan_mem_lim_min
= 16 << 20; /* bytes */
162 uint64_t zfs_scan_mem_lim_soft_max
= 128 << 20; /* bytes */
163 int zfs_scan_mem_lim_fact
= 20; /* fraction of physmem */
164 int zfs_scan_mem_lim_soft_fact
= 20; /* fraction of mem lim above */
166 int zfs_scrub_min_time_ms
= 1000; /* min millisecs to scrub per txg */
167 int zfs_free_min_time_ms
= 1000; /* min millisecs to free per txg */
168 int zfs_resilver_min_time_ms
= 3000; /* min millisecs to resilver per txg */
169 int zfs_scan_checkpoint_intval
= 7200; /* in seconds */
170 int zfs_no_scrub_io
= B_FALSE
; /* set to disable scrub i/o */
171 int zfs_no_scrub_prefetch
= B_FALSE
; /* set to disable scrub prefetch */
172 enum ddt_class zfs_scrub_ddt_class_max
= DDT_CLASS_DUPLICATE
;
173 /* max number of blocks to free in a single TXG */
174 unsigned long zfs_free_max_blocks
= 100000;
177 * We wait a few txgs after importing a pool to begin scanning so that
178 * the import / mounting code isn't held up by scrub / resilver IO.
179 * Unfortunately, it is a bit difficult to determine exactly how long
180 * this will take since userspace will trigger fs mounts asynchronously
181 * and the kernel will create zvol minors asynchronously. As a result,
182 * the value provided here is a bit arbitrary, but represents a
183 * reasonable estimate of how many txgs it will take to finish fully
186 #define SCAN_IMPORT_WAIT_TXGS 5
188 #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \
189 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
190 (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
193 * Enable/disable the processing of the free_bpobj object.
195 int zfs_free_bpobj_enabled
= 1;
197 /* the order has to match pool_scan_type */
198 static scan_cb_t
*scan_funcs
[POOL_SCAN_FUNCS
] = {
200 dsl_scan_scrub_cb
, /* POOL_SCAN_SCRUB */
201 dsl_scan_scrub_cb
, /* POOL_SCAN_RESILVER */
204 /* In core node for the scn->scn_queue. Represents a dataset to be scanned */
212 * This controls what conditions are placed on dsl_scan_sync_state():
213 * SYNC_OPTIONAL) write out scn_phys iff scn_bytes_pending == 0
214 * SYNC_MANDATORY) write out scn_phys always. scn_bytes_pending must be 0.
215 * SYNC_CACHED) if scn_bytes_pending == 0, write out scn_phys. Otherwise
216 * write out the scn_phys_cached version.
217 * See dsl_scan_sync_state for details.
226 * This struct represents the minimum information needed to reconstruct a
227 * zio for sequential scanning. This is useful because many of these will
228 * accumulate in the sequential IO queues before being issued, so saving
229 * memory matters here.
231 typedef struct scan_io
{
232 /* fields from blkptr_t */
234 uint64_t sio_blk_prop
;
235 uint64_t sio_phys_birth
;
237 zio_cksum_t sio_cksum
;
240 /* fields from zio_t */
242 zbookmark_phys_t sio_zb
;
244 /* members for queue sorting */
246 avl_node_t sio_addr_node
; /* link into issueing queue */
247 list_node_t sio_list_node
; /* link for issuing to disk */
251 struct dsl_scan_io_queue
{
252 dsl_scan_t
*q_scn
; /* associated dsl_scan_t */
253 vdev_t
*q_vd
; /* top-level vdev that this queue represents */
255 /* trees used for sorting I/Os and extents of I/Os */
256 range_tree_t
*q_exts_by_addr
;
257 avl_tree_t q_exts_by_size
;
258 avl_tree_t q_sios_by_addr
;
260 /* members for zio rate limiting */
261 uint64_t q_maxinflight_bytes
;
262 uint64_t q_inflight_bytes
;
263 kcondvar_t q_zio_cv
; /* used under vd->vdev_scan_io_queue_lock */
265 /* per txg statistics */
266 uint64_t q_total_seg_size_this_txg
;
267 uint64_t q_segs_this_txg
;
268 uint64_t q_total_zio_size_this_txg
;
269 uint64_t q_zios_this_txg
;
272 /* private data for dsl_scan_prefetch_cb() */
273 typedef struct scan_prefetch_ctx
{
274 refcount_t spc_refcnt
; /* refcount for memory management */
275 dsl_scan_t
*spc_scn
; /* dsl_scan_t for the pool */
276 boolean_t spc_root
; /* is this prefetch for an objset? */
277 uint8_t spc_indblkshift
; /* dn_indblkshift of current dnode */
278 uint16_t spc_datablkszsec
; /* dn_idatablkszsec of current dnode */
279 } scan_prefetch_ctx_t
;
281 /* private data for dsl_scan_prefetch() */
282 typedef struct scan_prefetch_issue_ctx
{
283 avl_node_t spic_avl_node
; /* link into scn->scn_prefetch_queue */
284 scan_prefetch_ctx_t
*spic_spc
; /* spc for the callback */
285 blkptr_t spic_bp
; /* bp to prefetch */
286 zbookmark_phys_t spic_zb
; /* bookmark to prefetch */
287 } scan_prefetch_issue_ctx_t
;
289 static void scan_exec_io(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
290 const zbookmark_phys_t
*zb
, dsl_scan_io_queue_t
*queue
);
291 static void scan_io_queue_insert_impl(dsl_scan_io_queue_t
*queue
,
294 static dsl_scan_io_queue_t
*scan_io_queue_create(vdev_t
*vd
);
295 static void scan_io_queues_destroy(dsl_scan_t
*scn
);
297 static kmem_cache_t
*sio_cache
;
303 * This is used in ext_size_compare() to weight segments
304 * based on how sparse they are. This cannot be changed
305 * mid-scan and the tree comparison functions don't currently
306 * have a mechansim for passing additional context to the
307 * compare functions. Thus we store this value globally and
308 * we only allow it to be set at module intiailization time
310 fill_weight
= zfs_scan_fill_weight
;
312 sio_cache
= kmem_cache_create("sio_cache",
313 sizeof (scan_io_t
), 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
319 kmem_cache_destroy(sio_cache
);
322 static inline boolean_t
323 dsl_scan_is_running(const dsl_scan_t
*scn
)
325 return (scn
->scn_phys
.scn_state
== DSS_SCANNING
);
329 dsl_scan_resilvering(dsl_pool_t
*dp
)
331 return (dsl_scan_is_running(dp
->dp_scan
) &&
332 dp
->dp_scan
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
);
336 sio2bp(const scan_io_t
*sio
, blkptr_t
*bp
, uint64_t vdev_id
)
338 bzero(bp
, sizeof (*bp
));
339 DVA_SET_ASIZE(&bp
->blk_dva
[0], sio
->sio_asize
);
340 DVA_SET_VDEV(&bp
->blk_dva
[0], vdev_id
);
341 DVA_SET_OFFSET(&bp
->blk_dva
[0], sio
->sio_offset
);
342 bp
->blk_prop
= sio
->sio_blk_prop
;
343 bp
->blk_phys_birth
= sio
->sio_phys_birth
;
344 bp
->blk_birth
= sio
->sio_birth
;
345 bp
->blk_fill
= 1; /* we always only work with data pointers */
346 bp
->blk_cksum
= sio
->sio_cksum
;
350 bp2sio(const blkptr_t
*bp
, scan_io_t
*sio
, int dva_i
)
352 /* we discard the vdev id, since we can deduce it from the queue */
353 sio
->sio_offset
= DVA_GET_OFFSET(&bp
->blk_dva
[dva_i
]);
354 sio
->sio_asize
= DVA_GET_ASIZE(&bp
->blk_dva
[dva_i
]);
355 sio
->sio_blk_prop
= bp
->blk_prop
;
356 sio
->sio_phys_birth
= bp
->blk_phys_birth
;
357 sio
->sio_birth
= bp
->blk_birth
;
358 sio
->sio_cksum
= bp
->blk_cksum
;
362 dsl_scan_init(dsl_pool_t
*dp
, uint64_t txg
)
366 spa_t
*spa
= dp
->dp_spa
;
369 scn
= dp
->dp_scan
= kmem_zalloc(sizeof (dsl_scan_t
), KM_SLEEP
);
373 * It's possible that we're resuming a scan after a reboot so
374 * make sure that the scan_async_destroying flag is initialized
377 ASSERT(!scn
->scn_async_destroying
);
378 scn
->scn_async_destroying
= spa_feature_is_active(dp
->dp_spa
,
379 SPA_FEATURE_ASYNC_DESTROY
);
381 bcopy(&scn
->scn_phys
, &scn
->scn_phys_cached
, sizeof (scn
->scn_phys
));
382 avl_create(&scn
->scn_queue
, scan_ds_queue_compare
, sizeof (scan_ds_t
),
383 offsetof(scan_ds_t
, sds_node
));
384 avl_create(&scn
->scn_prefetch_queue
, scan_prefetch_queue_compare
,
385 sizeof (scan_prefetch_issue_ctx_t
),
386 offsetof(scan_prefetch_issue_ctx_t
, spic_avl_node
));
388 err
= zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
389 "scrub_func", sizeof (uint64_t), 1, &f
);
392 * There was an old-style scrub in progress. Restart a
393 * new-style scrub from the beginning.
395 scn
->scn_restart_txg
= txg
;
396 zfs_dbgmsg("old-style scrub was in progress; "
397 "restarting new-style scrub in txg %llu",
398 (longlong_t
)scn
->scn_restart_txg
);
401 * Load the queue obj from the old location so that it
402 * can be freed by dsl_scan_done().
404 (void) zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
405 "scrub_queue", sizeof (uint64_t), 1,
406 &scn
->scn_phys
.scn_queue_obj
);
408 err
= zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
409 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
412 * Detect if the pool contains the signature of #2094. If it
413 * does properly update the scn->scn_phys structure and notify
414 * the administrator by setting an errata for the pool.
416 if (err
== EOVERFLOW
) {
417 uint64_t zaptmp
[SCAN_PHYS_NUMINTS
+ 1];
418 VERIFY3S(SCAN_PHYS_NUMINTS
, ==, 24);
419 VERIFY3S(offsetof(dsl_scan_phys_t
, scn_flags
), ==,
420 (23 * sizeof (uint64_t)));
422 err
= zap_lookup(dp
->dp_meta_objset
,
423 DMU_POOL_DIRECTORY_OBJECT
, DMU_POOL_SCAN
,
424 sizeof (uint64_t), SCAN_PHYS_NUMINTS
+ 1, &zaptmp
);
426 uint64_t overflow
= zaptmp
[SCAN_PHYS_NUMINTS
];
428 if (overflow
& ~DSL_SCAN_FLAGS_MASK
||
429 scn
->scn_async_destroying
) {
431 ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY
;
435 bcopy(zaptmp
, &scn
->scn_phys
,
436 SCAN_PHYS_NUMINTS
* sizeof (uint64_t));
437 scn
->scn_phys
.scn_flags
= overflow
;
439 /* Required scrub already in progress. */
440 if (scn
->scn_phys
.scn_state
== DSS_FINISHED
||
441 scn
->scn_phys
.scn_state
== DSS_CANCELED
)
443 ZPOOL_ERRATA_ZOL_2094_SCRUB
;
453 * We might be restarting after a reboot, so jump the issued
454 * counter to how far we've scanned. We know we're consistent
457 scn
->scn_issued_before_pass
= scn
->scn_phys
.scn_examined
;
459 if (dsl_scan_is_running(scn
) &&
460 spa_prev_software_version(dp
->dp_spa
) < SPA_VERSION_SCAN
) {
462 * A new-type scrub was in progress on an old
463 * pool, and the pool was accessed by old
464 * software. Restart from the beginning, since
465 * the old software may have changed the pool in
468 scn
->scn_restart_txg
= txg
;
469 zfs_dbgmsg("new-style scrub was modified "
470 "by old software; restarting in txg %llu",
471 (longlong_t
)scn
->scn_restart_txg
);
475 /* reload the queue into the in-core state */
476 if (scn
->scn_phys
.scn_queue_obj
!= 0) {
480 for (zap_cursor_init(&zc
, dp
->dp_meta_objset
,
481 scn
->scn_phys
.scn_queue_obj
);
482 zap_cursor_retrieve(&zc
, &za
) == 0;
483 (void) zap_cursor_advance(&zc
)) {
484 scan_ds_queue_insert(scn
,
485 zfs_strtonum(za
.za_name
, NULL
),
486 za
.za_first_integer
);
488 zap_cursor_fini(&zc
);
491 spa_scan_stat_init(spa
);
496 dsl_scan_fini(dsl_pool_t
*dp
)
498 if (dp
->dp_scan
!= NULL
) {
499 dsl_scan_t
*scn
= dp
->dp_scan
;
501 if (scn
->scn_taskq
!= NULL
)
502 taskq_destroy(scn
->scn_taskq
);
503 scan_ds_queue_clear(scn
);
504 avl_destroy(&scn
->scn_queue
);
505 avl_destroy(&scn
->scn_prefetch_queue
);
507 kmem_free(dp
->dp_scan
, sizeof (dsl_scan_t
));
513 dsl_scan_restarting(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
515 return (scn
->scn_restart_txg
!= 0 &&
516 scn
->scn_restart_txg
<= tx
->tx_txg
);
520 dsl_scan_scrubbing(const dsl_pool_t
*dp
)
522 dsl_scan_phys_t
*scn_phys
= &dp
->dp_scan
->scn_phys
;
524 return (scn_phys
->scn_state
== DSS_SCANNING
&&
525 scn_phys
->scn_func
== POOL_SCAN_SCRUB
);
529 dsl_scan_is_paused_scrub(const dsl_scan_t
*scn
)
531 return (dsl_scan_scrubbing(scn
->scn_dp
) &&
532 scn
->scn_phys
.scn_flags
& DSF_SCRUB_PAUSED
);
536 * Writes out a persistent dsl_scan_phys_t record to the pool directory.
537 * Because we can be running in the block sorting algorithm, we do not always
538 * want to write out the record, only when it is "safe" to do so. This safety
539 * condition is achieved by making sure that the sorting queues are empty
540 * (scn_bytes_pending == 0). When this condition is not true, the sync'd state
541 * is inconsistent with how much actual scanning progress has been made. The
542 * kind of sync to be performed is specified by the sync_type argument. If the
543 * sync is optional, we only sync if the queues are empty. If the sync is
544 * mandatory, we do a hard ASSERT to make sure that the queues are empty. The
545 * third possible state is a "cached" sync. This is done in response to:
546 * 1) The dataset that was in the last sync'd dsl_scan_phys_t having been
547 * destroyed, so we wouldn't be able to restart scanning from it.
548 * 2) The snapshot that was in the last sync'd dsl_scan_phys_t having been
549 * superseded by a newer snapshot.
550 * 3) The dataset that was in the last sync'd dsl_scan_phys_t having been
551 * swapped with its clone.
552 * In all cases, a cached sync simply rewrites the last record we've written,
553 * just slightly modified. For the modifications that are performed to the
554 * last written dsl_scan_phys_t, see dsl_scan_ds_destroyed,
555 * dsl_scan_ds_snapshotted and dsl_scan_ds_clone_swapped.
558 dsl_scan_sync_state(dsl_scan_t
*scn
, dmu_tx_t
*tx
, state_sync_type_t sync_type
)
561 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
563 ASSERT(sync_type
!= SYNC_MANDATORY
|| scn
->scn_bytes_pending
== 0);
564 if (scn
->scn_bytes_pending
== 0) {
565 for (i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
566 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
567 dsl_scan_io_queue_t
*q
= vd
->vdev_scan_io_queue
;
572 mutex_enter(&vd
->vdev_scan_io_queue_lock
);
573 ASSERT3P(avl_first(&q
->q_sios_by_addr
), ==, NULL
);
574 ASSERT3P(avl_first(&q
->q_exts_by_size
), ==, NULL
);
575 ASSERT3P(range_tree_first(q
->q_exts_by_addr
), ==, NULL
);
576 mutex_exit(&vd
->vdev_scan_io_queue_lock
);
579 if (scn
->scn_phys
.scn_queue_obj
!= 0)
580 scan_ds_queue_sync(scn
, tx
);
581 VERIFY0(zap_update(scn
->scn_dp
->dp_meta_objset
,
582 DMU_POOL_DIRECTORY_OBJECT
,
583 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
584 &scn
->scn_phys
, tx
));
585 bcopy(&scn
->scn_phys
, &scn
->scn_phys_cached
,
586 sizeof (scn
->scn_phys
));
588 if (scn
->scn_checkpointing
)
589 zfs_dbgmsg("finish scan checkpoint");
591 scn
->scn_checkpointing
= B_FALSE
;
592 scn
->scn_last_checkpoint
= ddi_get_lbolt();
593 } else if (sync_type
== SYNC_CACHED
) {
594 VERIFY0(zap_update(scn
->scn_dp
->dp_meta_objset
,
595 DMU_POOL_DIRECTORY_OBJECT
,
596 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
597 &scn
->scn_phys_cached
, tx
));
603 dsl_scan_setup_check(void *arg
, dmu_tx_t
*tx
)
605 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
607 if (dsl_scan_is_running(scn
))
608 return (SET_ERROR(EBUSY
));
614 dsl_scan_setup_sync(void *arg
, dmu_tx_t
*tx
)
616 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
617 pool_scan_func_t
*funcp
= arg
;
618 dmu_object_type_t ot
= 0;
619 dsl_pool_t
*dp
= scn
->scn_dp
;
620 spa_t
*spa
= dp
->dp_spa
;
622 ASSERT(!dsl_scan_is_running(scn
));
623 ASSERT(*funcp
> POOL_SCAN_NONE
&& *funcp
< POOL_SCAN_FUNCS
);
624 bzero(&scn
->scn_phys
, sizeof (scn
->scn_phys
));
625 scn
->scn_phys
.scn_func
= *funcp
;
626 scn
->scn_phys
.scn_state
= DSS_SCANNING
;
627 scn
->scn_phys
.scn_min_txg
= 0;
628 scn
->scn_phys
.scn_max_txg
= tx
->tx_txg
;
629 scn
->scn_phys
.scn_ddt_class_max
= DDT_CLASSES
- 1; /* the entire DDT */
630 scn
->scn_phys
.scn_start_time
= gethrestime_sec();
631 scn
->scn_phys
.scn_errors
= 0;
632 scn
->scn_phys
.scn_to_examine
= spa
->spa_root_vdev
->vdev_stat
.vs_alloc
;
633 scn
->scn_issued_before_pass
= 0;
634 scn
->scn_restart_txg
= 0;
635 scn
->scn_done_txg
= 0;
636 scn
->scn_last_checkpoint
= 0;
637 scn
->scn_checkpointing
= B_FALSE
;
638 spa_scan_stat_init(spa
);
640 if (DSL_SCAN_IS_SCRUB_RESILVER(scn
)) {
641 scn
->scn_phys
.scn_ddt_class_max
= zfs_scrub_ddt_class_max
;
643 /* rewrite all disk labels */
644 vdev_config_dirty(spa
->spa_root_vdev
);
646 if (vdev_resilver_needed(spa
->spa_root_vdev
,
647 &scn
->scn_phys
.scn_min_txg
, &scn
->scn_phys
.scn_max_txg
)) {
648 spa_event_notify(spa
, NULL
, NULL
,
649 ESC_ZFS_RESILVER_START
);
651 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_SCRUB_START
);
654 spa
->spa_scrub_started
= B_TRUE
;
656 * If this is an incremental scrub, limit the DDT scrub phase
657 * to just the auto-ditto class (for correctness); the rest
658 * of the scrub should go faster using top-down pruning.
660 if (scn
->scn_phys
.scn_min_txg
> TXG_INITIAL
)
661 scn
->scn_phys
.scn_ddt_class_max
= DDT_CLASS_DITTO
;
665 /* back to the generic stuff */
667 if (dp
->dp_blkstats
== NULL
) {
669 vmem_alloc(sizeof (zfs_all_blkstats_t
), KM_SLEEP
);
670 mutex_init(&dp
->dp_blkstats
->zab_lock
, NULL
,
671 MUTEX_DEFAULT
, NULL
);
673 bzero(&dp
->dp_blkstats
->zab_type
, sizeof (dp
->dp_blkstats
->zab_type
));
675 if (spa_version(spa
) < SPA_VERSION_DSL_SCRUB
)
676 ot
= DMU_OT_ZAP_OTHER
;
678 scn
->scn_phys
.scn_queue_obj
= zap_create(dp
->dp_meta_objset
,
679 ot
? ot
: DMU_OT_SCAN_QUEUE
, DMU_OT_NONE
, 0, tx
);
681 bcopy(&scn
->scn_phys
, &scn
->scn_phys_cached
, sizeof (scn
->scn_phys
));
683 dsl_scan_sync_state(scn
, tx
, SYNC_MANDATORY
);
685 spa_history_log_internal(spa
, "scan setup", tx
,
686 "func=%u mintxg=%llu maxtxg=%llu",
687 *funcp
, scn
->scn_phys
.scn_min_txg
, scn
->scn_phys
.scn_max_txg
);
691 * Called by the ZFS_IOC_POOL_SCAN ioctl to start a scrub or resilver.
692 * Can also be called to resume a paused scrub.
695 dsl_scan(dsl_pool_t
*dp
, pool_scan_func_t func
)
697 spa_t
*spa
= dp
->dp_spa
;
698 dsl_scan_t
*scn
= dp
->dp_scan
;
701 * Purge all vdev caches and probe all devices. We do this here
702 * rather than in sync context because this requires a writer lock
703 * on the spa_config lock, which we can't do from sync context. The
704 * spa_scrub_reopen flag indicates that vdev_open() should not
705 * attempt to start another scrub.
707 spa_vdev_state_enter(spa
, SCL_NONE
);
708 spa
->spa_scrub_reopen
= B_TRUE
;
709 vdev_reopen(spa
->spa_root_vdev
);
710 spa
->spa_scrub_reopen
= B_FALSE
;
711 (void) spa_vdev_state_exit(spa
, NULL
, 0);
713 if (func
== POOL_SCAN_SCRUB
&& dsl_scan_is_paused_scrub(scn
)) {
714 /* got scrub start cmd, resume paused scrub */
715 int err
= dsl_scrub_set_pause_resume(scn
->scn_dp
,
720 return (SET_ERROR(err
));
723 return (dsl_sync_task(spa_name(spa
), dsl_scan_setup_check
,
724 dsl_scan_setup_sync
, &func
, 0, ZFS_SPACE_CHECK_NONE
));
729 dsl_scan_done(dsl_scan_t
*scn
, boolean_t complete
, dmu_tx_t
*tx
)
731 static const char *old_names
[] = {
733 "scrub_ddt_bookmark",
734 "scrub_ddt_class_max",
743 dsl_pool_t
*dp
= scn
->scn_dp
;
744 spa_t
*spa
= dp
->dp_spa
;
747 /* Remove any remnants of an old-style scrub. */
748 for (i
= 0; old_names
[i
]; i
++) {
749 (void) zap_remove(dp
->dp_meta_objset
,
750 DMU_POOL_DIRECTORY_OBJECT
, old_names
[i
], tx
);
753 if (scn
->scn_phys
.scn_queue_obj
!= 0) {
754 VERIFY0(dmu_object_free(dp
->dp_meta_objset
,
755 scn
->scn_phys
.scn_queue_obj
, tx
));
756 scn
->scn_phys
.scn_queue_obj
= 0;
758 scan_ds_queue_clear(scn
);
760 scn
->scn_phys
.scn_flags
&= ~DSF_SCRUB_PAUSED
;
763 * If we were "restarted" from a stopped state, don't bother
764 * with anything else.
766 if (!dsl_scan_is_running(scn
)) {
767 ASSERT(!scn
->scn_is_sorted
);
771 if (scn
->scn_is_sorted
) {
772 scan_io_queues_destroy(scn
);
773 scn
->scn_is_sorted
= B_FALSE
;
775 if (scn
->scn_taskq
!= NULL
) {
776 taskq_destroy(scn
->scn_taskq
);
777 scn
->scn_taskq
= NULL
;
781 scn
->scn_phys
.scn_state
= complete
? DSS_FINISHED
: DSS_CANCELED
;
783 if (dsl_scan_restarting(scn
, tx
))
784 spa_history_log_internal(spa
, "scan aborted, restarting", tx
,
785 "errors=%llu", spa_get_errlog_size(spa
));
787 spa_history_log_internal(spa
, "scan cancelled", tx
,
788 "errors=%llu", spa_get_errlog_size(spa
));
790 spa_history_log_internal(spa
, "scan done", tx
,
791 "errors=%llu", spa_get_errlog_size(spa
));
793 if (DSL_SCAN_IS_SCRUB_RESILVER(scn
)) {
794 spa
->spa_scrub_started
= B_FALSE
;
795 spa
->spa_scrub_active
= B_FALSE
;
798 * If the scrub/resilver completed, update all DTLs to
799 * reflect this. Whether it succeeded or not, vacate
800 * all temporary scrub DTLs.
802 vdev_dtl_reassess(spa
->spa_root_vdev
, tx
->tx_txg
,
803 complete
? scn
->scn_phys
.scn_max_txg
: 0, B_TRUE
);
805 spa_event_notify(spa
, NULL
, NULL
,
806 scn
->scn_phys
.scn_min_txg
?
807 ESC_ZFS_RESILVER_FINISH
: ESC_ZFS_SCRUB_FINISH
);
809 spa_errlog_rotate(spa
);
812 * We may have finished replacing a device.
813 * Let the async thread assess this and handle the detach.
815 spa_async_request(spa
, SPA_ASYNC_RESILVER_DONE
);
818 scn
->scn_phys
.scn_end_time
= gethrestime_sec();
820 if (spa
->spa_errata
== ZPOOL_ERRATA_ZOL_2094_SCRUB
)
823 ASSERT(!dsl_scan_is_running(scn
));
828 dsl_scan_cancel_check(void *arg
, dmu_tx_t
*tx
)
830 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
832 if (!dsl_scan_is_running(scn
))
833 return (SET_ERROR(ENOENT
));
839 dsl_scan_cancel_sync(void *arg
, dmu_tx_t
*tx
)
841 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
843 dsl_scan_done(scn
, B_FALSE
, tx
);
844 dsl_scan_sync_state(scn
, tx
, SYNC_MANDATORY
);
848 dsl_scan_cancel(dsl_pool_t
*dp
)
850 return (dsl_sync_task(spa_name(dp
->dp_spa
), dsl_scan_cancel_check
,
851 dsl_scan_cancel_sync
, NULL
, 3, ZFS_SPACE_CHECK_RESERVED
));
855 dsl_scrub_pause_resume_check(void *arg
, dmu_tx_t
*tx
)
857 pool_scrub_cmd_t
*cmd
= arg
;
858 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
859 dsl_scan_t
*scn
= dp
->dp_scan
;
861 if (*cmd
== POOL_SCRUB_PAUSE
) {
862 /* can't pause a scrub when there is no in-progress scrub */
863 if (!dsl_scan_scrubbing(dp
))
864 return (SET_ERROR(ENOENT
));
866 /* can't pause a paused scrub */
867 if (dsl_scan_is_paused_scrub(scn
))
868 return (SET_ERROR(EBUSY
));
869 } else if (*cmd
!= POOL_SCRUB_NORMAL
) {
870 return (SET_ERROR(ENOTSUP
));
877 dsl_scrub_pause_resume_sync(void *arg
, dmu_tx_t
*tx
)
879 pool_scrub_cmd_t
*cmd
= arg
;
880 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
881 spa_t
*spa
= dp
->dp_spa
;
882 dsl_scan_t
*scn
= dp
->dp_scan
;
884 if (*cmd
== POOL_SCRUB_PAUSE
) {
885 /* can't pause a scrub when there is no in-progress scrub */
886 spa
->spa_scan_pass_scrub_pause
= gethrestime_sec();
887 scn
->scn_phys
.scn_flags
|= DSF_SCRUB_PAUSED
;
888 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
890 ASSERT3U(*cmd
, ==, POOL_SCRUB_NORMAL
);
891 if (dsl_scan_is_paused_scrub(scn
)) {
893 * We need to keep track of how much time we spend
894 * paused per pass so that we can adjust the scrub rate
895 * shown in the output of 'zpool status'
897 spa
->spa_scan_pass_scrub_spent_paused
+=
898 gethrestime_sec() - spa
->spa_scan_pass_scrub_pause
;
899 spa
->spa_scan_pass_scrub_pause
= 0;
900 scn
->scn_phys
.scn_flags
&= ~DSF_SCRUB_PAUSED
;
901 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
907 * Set scrub pause/resume state if it makes sense to do so
910 dsl_scrub_set_pause_resume(const dsl_pool_t
*dp
, pool_scrub_cmd_t cmd
)
912 return (dsl_sync_task(spa_name(dp
->dp_spa
),
913 dsl_scrub_pause_resume_check
, dsl_scrub_pause_resume_sync
, &cmd
, 3,
914 ZFS_SPACE_CHECK_RESERVED
));
918 /* start a new scan, or restart an existing one. */
920 dsl_resilver_restart(dsl_pool_t
*dp
, uint64_t txg
)
924 tx
= dmu_tx_create_dd(dp
->dp_mos_dir
);
925 VERIFY(0 == dmu_tx_assign(tx
, TXG_WAIT
));
927 txg
= dmu_tx_get_txg(tx
);
928 dp
->dp_scan
->scn_restart_txg
= txg
;
931 dp
->dp_scan
->scn_restart_txg
= txg
;
933 zfs_dbgmsg("restarting resilver txg=%llu", (longlong_t
)txg
);
937 dsl_free(dsl_pool_t
*dp
, uint64_t txg
, const blkptr_t
*bp
)
939 zio_free(dp
->dp_spa
, txg
, bp
);
943 dsl_free_sync(zio_t
*pio
, dsl_pool_t
*dp
, uint64_t txg
, const blkptr_t
*bpp
)
945 ASSERT(dsl_pool_sync_context(dp
));
946 zio_nowait(zio_free_sync(pio
, dp
->dp_spa
, txg
, bpp
, pio
->io_flags
));
950 scan_ds_queue_compare(const void *a
, const void *b
)
952 const scan_ds_t
*sds_a
= a
, *sds_b
= b
;
954 if (sds_a
->sds_dsobj
< sds_b
->sds_dsobj
)
956 if (sds_a
->sds_dsobj
== sds_b
->sds_dsobj
)
962 scan_ds_queue_clear(dsl_scan_t
*scn
)
966 while ((sds
= avl_destroy_nodes(&scn
->scn_queue
, &cookie
)) != NULL
) {
967 kmem_free(sds
, sizeof (*sds
));
972 scan_ds_queue_contains(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t *txg
)
974 scan_ds_t srch
, *sds
;
976 srch
.sds_dsobj
= dsobj
;
977 sds
= avl_find(&scn
->scn_queue
, &srch
, NULL
);
978 if (sds
!= NULL
&& txg
!= NULL
)
980 return (sds
!= NULL
);
984 scan_ds_queue_insert(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t txg
)
989 sds
= kmem_zalloc(sizeof (*sds
), KM_SLEEP
);
990 sds
->sds_dsobj
= dsobj
;
993 VERIFY3P(avl_find(&scn
->scn_queue
, sds
, &where
), ==, NULL
);
994 avl_insert(&scn
->scn_queue
, sds
, where
);
998 scan_ds_queue_remove(dsl_scan_t
*scn
, uint64_t dsobj
)
1000 scan_ds_t srch
, *sds
;
1002 srch
.sds_dsobj
= dsobj
;
1004 sds
= avl_find(&scn
->scn_queue
, &srch
, NULL
);
1005 VERIFY(sds
!= NULL
);
1006 avl_remove(&scn
->scn_queue
, sds
);
1007 kmem_free(sds
, sizeof (*sds
));
1011 scan_ds_queue_sync(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
1013 dsl_pool_t
*dp
= scn
->scn_dp
;
1014 spa_t
*spa
= dp
->dp_spa
;
1015 dmu_object_type_t ot
= (spa_version(spa
) >= SPA_VERSION_DSL_SCRUB
) ?
1016 DMU_OT_SCAN_QUEUE
: DMU_OT_ZAP_OTHER
;
1018 ASSERT0(scn
->scn_bytes_pending
);
1019 ASSERT(scn
->scn_phys
.scn_queue_obj
!= 0);
1021 VERIFY0(dmu_object_free(dp
->dp_meta_objset
,
1022 scn
->scn_phys
.scn_queue_obj
, tx
));
1023 scn
->scn_phys
.scn_queue_obj
= zap_create(dp
->dp_meta_objset
, ot
,
1024 DMU_OT_NONE
, 0, tx
);
1025 for (scan_ds_t
*sds
= avl_first(&scn
->scn_queue
);
1026 sds
!= NULL
; sds
= AVL_NEXT(&scn
->scn_queue
, sds
)) {
1027 VERIFY0(zap_add_int_key(dp
->dp_meta_objset
,
1028 scn
->scn_phys
.scn_queue_obj
, sds
->sds_dsobj
,
1034 * Computes the memory limit state that we're currently in. A sorted scan
1035 * needs quite a bit of memory to hold the sorting queue, so we need to
1036 * reasonably constrain the size so it doesn't impact overall system
1037 * performance. We compute two limits:
1038 * 1) Hard memory limit: if the amount of memory used by the sorting
1039 * queues on a pool gets above this value, we stop the metadata
1040 * scanning portion and start issuing the queued up and sorted
1041 * I/Os to reduce memory usage.
1042 * This limit is calculated as a fraction of physmem (by default 5%).
1043 * We constrain the lower bound of the hard limit to an absolute
1044 * minimum of zfs_scan_mem_lim_min (default: 16 MiB). We also constrain
1045 * the upper bound to 5% of the total pool size - no chance we'll
1046 * ever need that much memory, but just to keep the value in check.
1047 * 2) Soft memory limit: once we hit the hard memory limit, we start
1048 * issuing I/O to reduce queue memory usage, but we don't want to
1049 * completely empty out the queues, since we might be able to find I/Os
1050 * that will fill in the gaps of our non-sequential IOs at some point
1051 * in the future. So we stop the issuing of I/Os once the amount of
1052 * memory used drops below the soft limit (at which point we stop issuing
1053 * I/O and start scanning metadata again).
1055 * This limit is calculated by subtracting a fraction of the hard
1056 * limit from the hard limit. By default this fraction is 5%, so
1057 * the soft limit is 95% of the hard limit. We cap the size of the
1058 * difference between the hard and soft limits at an absolute
1059 * maximum of zfs_scan_mem_lim_soft_max (default: 128 MiB) - this is
1060 * sufficient to not cause too frequent switching between the
1061 * metadata scan and I/O issue (even at 2k recordsize, 128 MiB's
1062 * worth of queues is about 1.2 GiB of on-pool data, so scanning
1063 * that should take at least a decent fraction of a second).
1066 dsl_scan_should_clear(dsl_scan_t
*scn
)
1068 vdev_t
*rvd
= scn
->scn_dp
->dp_spa
->spa_root_vdev
;
1069 uint64_t mlim_hard
, mlim_soft
, mused
;
1070 uint64_t alloc
= metaslab_class_get_alloc(spa_normal_class(
1071 scn
->scn_dp
->dp_spa
));
1073 mlim_hard
= MAX((physmem
/ zfs_scan_mem_lim_fact
) * PAGESIZE
,
1074 zfs_scan_mem_lim_min
);
1075 mlim_hard
= MIN(mlim_hard
, alloc
/ 20);
1076 mlim_soft
= mlim_hard
- MIN(mlim_hard
/ zfs_scan_mem_lim_soft_fact
,
1077 zfs_scan_mem_lim_soft_max
);
1079 for (uint64_t i
= 0; i
< rvd
->vdev_children
; i
++) {
1080 vdev_t
*tvd
= rvd
->vdev_child
[i
];
1081 dsl_scan_io_queue_t
*queue
;
1083 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
1084 queue
= tvd
->vdev_scan_io_queue
;
1085 if (queue
!= NULL
) {
1086 /* #extents in exts_by_size = # in exts_by_addr */
1087 mused
+= avl_numnodes(&queue
->q_exts_by_size
) *
1088 sizeof (range_seg_t
) +
1089 avl_numnodes(&queue
->q_sios_by_addr
) *
1092 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
1095 dprintf("current scan memory usage: %llu bytes\n", (longlong_t
)mused
);
1098 ASSERT0(scn
->scn_bytes_pending
);
1101 * If we are above our hard limit, we need to clear out memory.
1102 * If we are below our soft limit, we need to accumulate sequential IOs.
1103 * Otherwise, we should keep doing whatever we are currently doing.
1105 if (mused
>= mlim_hard
)
1107 else if (mused
< mlim_soft
)
1110 return (scn
->scn_clearing
);
1114 dsl_scan_check_suspend(dsl_scan_t
*scn
, const zbookmark_phys_t
*zb
)
1116 /* we never skip user/group accounting objects */
1117 if (zb
&& (int64_t)zb
->zb_object
< 0)
1120 if (scn
->scn_suspending
)
1121 return (B_TRUE
); /* we're already suspending */
1123 if (!ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
))
1124 return (B_FALSE
); /* we're resuming */
1126 /* We only know how to resume from level-0 blocks. */
1127 if (zb
&& zb
->zb_level
!= 0)
1132 * - we have scanned for at least the minimum time (default 1 sec
1133 * for scrub, 3 sec for resilver), and either we have sufficient
1134 * dirty data that we are starting to write more quickly
1135 * (default 30%), someone is explicitly waiting for this txg
1136 * to complete, or we have used up all of the time in the txg
1137 * timeout (default 5 sec).
1139 * - the spa is shutting down because this pool is being exported
1140 * or the machine is rebooting.
1142 * - the scan queue has reached its memory use limit
1144 uint64_t curr_time_ns
= gethrtime();
1145 uint64_t scan_time_ns
= curr_time_ns
- scn
->scn_sync_start_time
;
1146 uint64_t sync_time_ns
= curr_time_ns
-
1147 scn
->scn_dp
->dp_spa
->spa_sync_starttime
;
1148 int dirty_pct
= scn
->scn_dp
->dp_dirty_total
* 100 / zfs_dirty_data_max
;
1149 int mintime
= (scn
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
) ?
1150 zfs_resilver_min_time_ms
: zfs_scrub_min_time_ms
;
1152 if ((NSEC2MSEC(scan_time_ns
) > mintime
&&
1153 (dirty_pct
>= zfs_vdev_async_write_active_min_dirty_percent
||
1154 txg_sync_waiting(scn
->scn_dp
) ||
1155 NSEC2SEC(sync_time_ns
) >= zfs_txg_timeout
)) ||
1156 spa_shutting_down(scn
->scn_dp
->dp_spa
) ||
1157 (zfs_scan_strict_mem_lim
&& dsl_scan_should_clear(scn
))) {
1159 dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n",
1160 (longlong_t
)zb
->zb_objset
,
1161 (longlong_t
)zb
->zb_object
,
1162 (longlong_t
)zb
->zb_level
,
1163 (longlong_t
)zb
->zb_blkid
);
1164 scn
->scn_phys
.scn_bookmark
= *zb
;
1166 dsl_scan_phys_t
*scnp
= &scn
->scn_phys
;
1168 dprintf("suspending at at DDT bookmark "
1169 "%llx/%llx/%llx/%llx\n",
1170 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_class
,
1171 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_type
,
1172 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_checksum
,
1173 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_cursor
);
1175 scn
->scn_suspending
= B_TRUE
;
1181 typedef struct zil_scan_arg
{
1183 zil_header_t
*zsa_zh
;
1188 dsl_scan_zil_block(zilog_t
*zilog
, blkptr_t
*bp
, void *arg
, uint64_t claim_txg
)
1190 zil_scan_arg_t
*zsa
= arg
;
1191 dsl_pool_t
*dp
= zsa
->zsa_dp
;
1192 dsl_scan_t
*scn
= dp
->dp_scan
;
1193 zil_header_t
*zh
= zsa
->zsa_zh
;
1194 zbookmark_phys_t zb
;
1196 if (BP_IS_HOLE(bp
) || bp
->blk_birth
<= scn
->scn_phys
.scn_cur_min_txg
)
1200 * One block ("stubby") can be allocated a long time ago; we
1201 * want to visit that one because it has been allocated
1202 * (on-disk) even if it hasn't been claimed (even though for
1203 * scrub there's nothing to do to it).
1205 if (claim_txg
== 0 && bp
->blk_birth
>= spa_first_txg(dp
->dp_spa
))
1208 SET_BOOKMARK(&zb
, zh
->zh_log
.blk_cksum
.zc_word
[ZIL_ZC_OBJSET
],
1209 ZB_ZIL_OBJECT
, ZB_ZIL_LEVEL
, bp
->blk_cksum
.zc_word
[ZIL_ZC_SEQ
]);
1211 VERIFY(0 == scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, &zb
));
1217 dsl_scan_zil_record(zilog_t
*zilog
, lr_t
*lrc
, void *arg
, uint64_t claim_txg
)
1219 if (lrc
->lrc_txtype
== TX_WRITE
) {
1220 zil_scan_arg_t
*zsa
= arg
;
1221 dsl_pool_t
*dp
= zsa
->zsa_dp
;
1222 dsl_scan_t
*scn
= dp
->dp_scan
;
1223 zil_header_t
*zh
= zsa
->zsa_zh
;
1224 lr_write_t
*lr
= (lr_write_t
*)lrc
;
1225 blkptr_t
*bp
= &lr
->lr_blkptr
;
1226 zbookmark_phys_t zb
;
1228 if (BP_IS_HOLE(bp
) ||
1229 bp
->blk_birth
<= scn
->scn_phys
.scn_cur_min_txg
)
1233 * birth can be < claim_txg if this record's txg is
1234 * already txg sync'ed (but this log block contains
1235 * other records that are not synced)
1237 if (claim_txg
== 0 || bp
->blk_birth
< claim_txg
)
1240 SET_BOOKMARK(&zb
, zh
->zh_log
.blk_cksum
.zc_word
[ZIL_ZC_OBJSET
],
1241 lr
->lr_foid
, ZB_ZIL_LEVEL
,
1242 lr
->lr_offset
/ BP_GET_LSIZE(bp
));
1244 VERIFY(0 == scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, &zb
));
1250 dsl_scan_zil(dsl_pool_t
*dp
, zil_header_t
*zh
)
1252 uint64_t claim_txg
= zh
->zh_claim_txg
;
1253 zil_scan_arg_t zsa
= { dp
, zh
};
1257 * We only want to visit blocks that have been claimed but not yet
1258 * replayed (or, in read-only mode, blocks that *would* be claimed).
1260 if (claim_txg
== 0 && spa_writeable(dp
->dp_spa
))
1263 zilog
= zil_alloc(dp
->dp_meta_objset
, zh
);
1265 (void) zil_parse(zilog
, dsl_scan_zil_block
, dsl_scan_zil_record
, &zsa
,
1266 claim_txg
, B_FALSE
);
1272 * We compare scan_prefetch_issue_ctx_t's based on their bookmarks. The idea
1273 * here is to sort the AVL tree by the order each block will be needed.
1276 scan_prefetch_queue_compare(const void *a
, const void *b
)
1278 const scan_prefetch_issue_ctx_t
*spic_a
= a
, *spic_b
= b
;
1279 const scan_prefetch_ctx_t
*spc_a
= spic_a
->spic_spc
;
1280 const scan_prefetch_ctx_t
*spc_b
= spic_b
->spic_spc
;
1282 return (zbookmark_compare(spc_a
->spc_datablkszsec
,
1283 spc_a
->spc_indblkshift
, spc_b
->spc_datablkszsec
,
1284 spc_b
->spc_indblkshift
, &spic_a
->spic_zb
, &spic_b
->spic_zb
));
1288 scan_prefetch_ctx_rele(scan_prefetch_ctx_t
*spc
, void *tag
)
1290 if (refcount_remove(&spc
->spc_refcnt
, tag
) == 0) {
1291 refcount_destroy(&spc
->spc_refcnt
);
1292 kmem_free(spc
, sizeof (scan_prefetch_ctx_t
));
1296 static scan_prefetch_ctx_t
*
1297 scan_prefetch_ctx_create(dsl_scan_t
*scn
, dnode_phys_t
*dnp
, void *tag
)
1299 scan_prefetch_ctx_t
*spc
;
1301 spc
= kmem_alloc(sizeof (scan_prefetch_ctx_t
), KM_SLEEP
);
1302 refcount_create(&spc
->spc_refcnt
);
1303 refcount_add(&spc
->spc_refcnt
, tag
);
1306 spc
->spc_datablkszsec
= dnp
->dn_datablkszsec
;
1307 spc
->spc_indblkshift
= dnp
->dn_indblkshift
;
1308 spc
->spc_root
= B_FALSE
;
1310 spc
->spc_datablkszsec
= 0;
1311 spc
->spc_indblkshift
= 0;
1312 spc
->spc_root
= B_TRUE
;
1319 scan_prefetch_ctx_add_ref(scan_prefetch_ctx_t
*spc
, void *tag
)
1321 refcount_add(&spc
->spc_refcnt
, tag
);
1325 dsl_scan_check_prefetch_resume(scan_prefetch_ctx_t
*spc
,
1326 const zbookmark_phys_t
*zb
)
1328 zbookmark_phys_t
*last_zb
= &spc
->spc_scn
->scn_prefetch_bookmark
;
1329 dnode_phys_t tmp_dnp
;
1330 dnode_phys_t
*dnp
= (spc
->spc_root
) ? NULL
: &tmp_dnp
;
1332 if (zb
->zb_objset
!= last_zb
->zb_objset
)
1334 if ((int64_t)zb
->zb_object
< 0)
1337 tmp_dnp
.dn_datablkszsec
= spc
->spc_datablkszsec
;
1338 tmp_dnp
.dn_indblkshift
= spc
->spc_indblkshift
;
1340 if (zbookmark_subtree_completed(dnp
, zb
, last_zb
))
1347 dsl_scan_prefetch(scan_prefetch_ctx_t
*spc
, blkptr_t
*bp
, zbookmark_phys_t
*zb
)
1350 dsl_scan_t
*scn
= spc
->spc_scn
;
1351 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1352 scan_prefetch_issue_ctx_t
*spic
;
1354 if (zfs_no_scrub_prefetch
)
1357 if (BP_IS_HOLE(bp
) || bp
->blk_birth
<= scn
->scn_phys
.scn_cur_min_txg
||
1358 (BP_GET_LEVEL(bp
) == 0 && BP_GET_TYPE(bp
) != DMU_OT_DNODE
&&
1359 BP_GET_TYPE(bp
) != DMU_OT_OBJSET
))
1362 if (dsl_scan_check_prefetch_resume(spc
, zb
))
1365 scan_prefetch_ctx_add_ref(spc
, scn
);
1366 spic
= kmem_alloc(sizeof (scan_prefetch_issue_ctx_t
), KM_SLEEP
);
1367 spic
->spic_spc
= spc
;
1368 spic
->spic_bp
= *bp
;
1369 spic
->spic_zb
= *zb
;
1372 * Add the IO to the queue of blocks to prefetch. This allows us to
1373 * prioritize blocks that we will need first for the main traversal
1376 mutex_enter(&spa
->spa_scrub_lock
);
1377 if (avl_find(&scn
->scn_prefetch_queue
, spic
, &idx
) != NULL
) {
1378 /* this block is already queued for prefetch */
1379 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
1380 scan_prefetch_ctx_rele(spc
, scn
);
1381 mutex_exit(&spa
->spa_scrub_lock
);
1385 avl_insert(&scn
->scn_prefetch_queue
, spic
, idx
);
1386 cv_broadcast(&spa
->spa_scrub_io_cv
);
1387 mutex_exit(&spa
->spa_scrub_lock
);
1391 dsl_scan_prefetch_dnode(dsl_scan_t
*scn
, dnode_phys_t
*dnp
,
1392 uint64_t objset
, uint64_t object
)
1395 zbookmark_phys_t zb
;
1396 scan_prefetch_ctx_t
*spc
;
1398 if (dnp
->dn_nblkptr
== 0 && !(dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
))
1401 SET_BOOKMARK(&zb
, objset
, object
, 0, 0);
1403 spc
= scan_prefetch_ctx_create(scn
, dnp
, FTAG
);
1405 for (i
= 0; i
< dnp
->dn_nblkptr
; i
++) {
1406 zb
.zb_level
= BP_GET_LEVEL(&dnp
->dn_blkptr
[i
]);
1408 dsl_scan_prefetch(spc
, &dnp
->dn_blkptr
[i
], &zb
);
1411 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1413 zb
.zb_blkid
= DMU_SPILL_BLKID
;
1414 dsl_scan_prefetch(spc
, DN_SPILL_BLKPTR(dnp
), &zb
);
1417 scan_prefetch_ctx_rele(spc
, FTAG
);
1421 dsl_scan_prefetch_cb(zio_t
*zio
, const zbookmark_phys_t
*zb
, const blkptr_t
*bp
,
1422 arc_buf_t
*buf
, void *private)
1424 scan_prefetch_ctx_t
*spc
= private;
1425 dsl_scan_t
*scn
= spc
->spc_scn
;
1426 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1428 /* broadcast that the IO has completed for rate limitting purposes */
1429 mutex_enter(&spa
->spa_scrub_lock
);
1430 ASSERT3U(spa
->spa_scrub_inflight
, >=, BP_GET_PSIZE(bp
));
1431 spa
->spa_scrub_inflight
-= BP_GET_PSIZE(bp
);
1432 cv_broadcast(&spa
->spa_scrub_io_cv
);
1433 mutex_exit(&spa
->spa_scrub_lock
);
1435 /* if there was an error or we are done prefetching, just cleanup */
1436 if (buf
== NULL
|| scn
->scn_suspending
)
1439 if (BP_GET_LEVEL(bp
) > 0) {
1442 int epb
= BP_GET_LSIZE(bp
) >> SPA_BLKPTRSHIFT
;
1443 zbookmark_phys_t czb
;
1445 for (i
= 0, cbp
= buf
->b_data
; i
< epb
; i
++, cbp
++) {
1446 SET_BOOKMARK(&czb
, zb
->zb_objset
, zb
->zb_object
,
1447 zb
->zb_level
- 1, zb
->zb_blkid
* epb
+ i
);
1448 dsl_scan_prefetch(spc
, cbp
, &czb
);
1450 } else if (BP_GET_TYPE(bp
) == DMU_OT_DNODE
) {
1453 int epb
= BP_GET_LSIZE(bp
) >> DNODE_SHIFT
;
1455 for (i
= 0, cdnp
= buf
->b_data
; i
< epb
;
1456 i
+= cdnp
->dn_extra_slots
+ 1,
1457 cdnp
+= cdnp
->dn_extra_slots
+ 1) {
1458 dsl_scan_prefetch_dnode(scn
, cdnp
,
1459 zb
->zb_objset
, zb
->zb_blkid
* epb
+ i
);
1461 } else if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
1462 objset_phys_t
*osp
= buf
->b_data
;
1464 dsl_scan_prefetch_dnode(scn
, &osp
->os_meta_dnode
,
1465 zb
->zb_objset
, DMU_META_DNODE_OBJECT
);
1467 if (OBJSET_BUF_HAS_USERUSED(buf
)) {
1468 dsl_scan_prefetch_dnode(scn
,
1469 &osp
->os_groupused_dnode
, zb
->zb_objset
,
1470 DMU_GROUPUSED_OBJECT
);
1471 dsl_scan_prefetch_dnode(scn
,
1472 &osp
->os_userused_dnode
, zb
->zb_objset
,
1473 DMU_USERUSED_OBJECT
);
1479 arc_buf_destroy(buf
, private);
1480 scan_prefetch_ctx_rele(spc
, scn
);
1485 dsl_scan_prefetch_thread(void *arg
)
1487 dsl_scan_t
*scn
= arg
;
1488 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1489 scan_prefetch_issue_ctx_t
*spic
;
1491 /* loop until we are told to stop */
1492 while (!scn
->scn_prefetch_stop
) {
1493 arc_flags_t flags
= ARC_FLAG_NOWAIT
|
1494 ARC_FLAG_PRESCIENT_PREFETCH
| ARC_FLAG_PREFETCH
;
1495 int zio_flags
= ZIO_FLAG_CANFAIL
| ZIO_FLAG_SCAN_THREAD
;
1497 mutex_enter(&spa
->spa_scrub_lock
);
1500 * Wait until we have an IO to issue and are not above our
1501 * maximum in flight limit.
1503 while (!scn
->scn_prefetch_stop
&&
1504 (avl_numnodes(&scn
->scn_prefetch_queue
) == 0 ||
1505 spa
->spa_scrub_inflight
>= scn
->scn_maxinflight_bytes
)) {
1506 cv_wait(&spa
->spa_scrub_io_cv
, &spa
->spa_scrub_lock
);
1509 /* recheck if we should stop since we waited for the cv */
1510 if (scn
->scn_prefetch_stop
) {
1511 mutex_exit(&spa
->spa_scrub_lock
);
1515 /* remove the prefetch IO from the tree */
1516 spic
= avl_first(&scn
->scn_prefetch_queue
);
1517 spa
->spa_scrub_inflight
+= BP_GET_PSIZE(&spic
->spic_bp
);
1518 avl_remove(&scn
->scn_prefetch_queue
, spic
);
1520 mutex_exit(&spa
->spa_scrub_lock
);
1522 if (BP_IS_PROTECTED(&spic
->spic_bp
)) {
1523 ASSERT(BP_GET_TYPE(&spic
->spic_bp
) == DMU_OT_DNODE
||
1524 BP_GET_TYPE(&spic
->spic_bp
) == DMU_OT_OBJSET
);
1525 ASSERT3U(BP_GET_LEVEL(&spic
->spic_bp
), ==, 0);
1526 zio_flags
|= ZIO_FLAG_RAW
;
1529 /* issue the prefetch asynchronously */
1530 (void) arc_read(scn
->scn_zio_root
, scn
->scn_dp
->dp_spa
,
1531 &spic
->spic_bp
, dsl_scan_prefetch_cb
, spic
->spic_spc
,
1532 ZIO_PRIORITY_ASYNC_READ
, zio_flags
, &flags
, &spic
->spic_zb
);
1534 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
1537 ASSERT(scn
->scn_prefetch_stop
);
1539 /* free any prefetches we didn't get to complete */
1540 mutex_enter(&spa
->spa_scrub_lock
);
1541 while ((spic
= avl_first(&scn
->scn_prefetch_queue
)) != NULL
) {
1542 avl_remove(&scn
->scn_prefetch_queue
, spic
);
1543 scan_prefetch_ctx_rele(spic
->spic_spc
, scn
);
1544 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
1546 ASSERT0(avl_numnodes(&scn
->scn_prefetch_queue
));
1547 mutex_exit(&spa
->spa_scrub_lock
);
1551 dsl_scan_check_resume(dsl_scan_t
*scn
, const dnode_phys_t
*dnp
,
1552 const zbookmark_phys_t
*zb
)
1555 * We never skip over user/group accounting objects (obj<0)
1557 if (!ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
) &&
1558 (int64_t)zb
->zb_object
>= 0) {
1560 * If we already visited this bp & everything below (in
1561 * a prior txg sync), don't bother doing it again.
1563 if (zbookmark_subtree_completed(dnp
, zb
,
1564 &scn
->scn_phys
.scn_bookmark
))
1568 * If we found the block we're trying to resume from, or
1569 * we went past it to a different object, zero it out to
1570 * indicate that it's OK to start checking for suspending
1573 if (bcmp(zb
, &scn
->scn_phys
.scn_bookmark
, sizeof (*zb
)) == 0 ||
1574 zb
->zb_object
> scn
->scn_phys
.scn_bookmark
.zb_object
) {
1575 dprintf("resuming at %llx/%llx/%llx/%llx\n",
1576 (longlong_t
)zb
->zb_objset
,
1577 (longlong_t
)zb
->zb_object
,
1578 (longlong_t
)zb
->zb_level
,
1579 (longlong_t
)zb
->zb_blkid
);
1580 bzero(&scn
->scn_phys
.scn_bookmark
, sizeof (*zb
));
1586 static void dsl_scan_visitbp(blkptr_t
*bp
, const zbookmark_phys_t
*zb
,
1587 dnode_phys_t
*dnp
, dsl_dataset_t
*ds
, dsl_scan_t
*scn
,
1588 dmu_objset_type_t ostype
, dmu_tx_t
*tx
);
1589 inline __attribute__((always_inline
)) static void dsl_scan_visitdnode(
1590 dsl_scan_t
*, dsl_dataset_t
*ds
, dmu_objset_type_t ostype
,
1591 dnode_phys_t
*dnp
, uint64_t object
, dmu_tx_t
*tx
);
1594 * Return nonzero on i/o error.
1595 * Return new buf to write out in *bufp.
1597 inline __attribute__((always_inline
)) static int
1598 dsl_scan_recurse(dsl_scan_t
*scn
, dsl_dataset_t
*ds
, dmu_objset_type_t ostype
,
1599 dnode_phys_t
*dnp
, const blkptr_t
*bp
,
1600 const zbookmark_phys_t
*zb
, dmu_tx_t
*tx
)
1602 dsl_pool_t
*dp
= scn
->scn_dp
;
1603 int zio_flags
= ZIO_FLAG_CANFAIL
| ZIO_FLAG_SCAN_THREAD
;
1606 if (BP_GET_LEVEL(bp
) > 0) {
1607 arc_flags_t flags
= ARC_FLAG_WAIT
;
1610 int epb
= BP_GET_LSIZE(bp
) >> SPA_BLKPTRSHIFT
;
1613 err
= arc_read(NULL
, dp
->dp_spa
, bp
, arc_getbuf_func
, &buf
,
1614 ZIO_PRIORITY_ASYNC_READ
, zio_flags
, &flags
, zb
);
1616 scn
->scn_phys
.scn_errors
++;
1619 for (i
= 0, cbp
= buf
->b_data
; i
< epb
; i
++, cbp
++) {
1620 zbookmark_phys_t czb
;
1622 SET_BOOKMARK(&czb
, zb
->zb_objset
, zb
->zb_object
,
1624 zb
->zb_blkid
* epb
+ i
);
1625 dsl_scan_visitbp(cbp
, &czb
, dnp
,
1626 ds
, scn
, ostype
, tx
);
1628 arc_buf_destroy(buf
, &buf
);
1629 } else if (BP_GET_TYPE(bp
) == DMU_OT_DNODE
) {
1630 arc_flags_t flags
= ARC_FLAG_WAIT
;
1633 int epb
= BP_GET_LSIZE(bp
) >> DNODE_SHIFT
;
1636 if (BP_IS_PROTECTED(bp
)) {
1637 ASSERT3U(BP_GET_COMPRESS(bp
), ==, ZIO_COMPRESS_OFF
);
1638 zio_flags
|= ZIO_FLAG_RAW
;
1641 err
= arc_read(NULL
, dp
->dp_spa
, bp
, arc_getbuf_func
, &buf
,
1642 ZIO_PRIORITY_ASYNC_READ
, zio_flags
, &flags
, zb
);
1644 scn
->scn_phys
.scn_errors
++;
1647 for (i
= 0, cdnp
= buf
->b_data
; i
< epb
;
1648 i
+= cdnp
->dn_extra_slots
+ 1,
1649 cdnp
+= cdnp
->dn_extra_slots
+ 1) {
1650 dsl_scan_visitdnode(scn
, ds
, ostype
,
1651 cdnp
, zb
->zb_blkid
* epb
+ i
, tx
);
1654 arc_buf_destroy(buf
, &buf
);
1655 } else if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
1656 arc_flags_t flags
= ARC_FLAG_WAIT
;
1660 err
= arc_read(NULL
, dp
->dp_spa
, bp
, arc_getbuf_func
, &buf
,
1661 ZIO_PRIORITY_ASYNC_READ
, zio_flags
, &flags
, zb
);
1663 scn
->scn_phys
.scn_errors
++;
1669 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
1670 &osp
->os_meta_dnode
, DMU_META_DNODE_OBJECT
, tx
);
1672 if (OBJSET_BUF_HAS_USERUSED(buf
)) {
1674 * We also always visit user/group accounting
1675 * objects, and never skip them, even if we are
1676 * suspending. This is necessary so that the
1677 * space deltas from this txg get integrated.
1679 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
1680 &osp
->os_groupused_dnode
,
1681 DMU_GROUPUSED_OBJECT
, tx
);
1682 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
1683 &osp
->os_userused_dnode
,
1684 DMU_USERUSED_OBJECT
, tx
);
1686 arc_buf_destroy(buf
, &buf
);
1692 inline __attribute__((always_inline
)) static void
1693 dsl_scan_visitdnode(dsl_scan_t
*scn
, dsl_dataset_t
*ds
,
1694 dmu_objset_type_t ostype
, dnode_phys_t
*dnp
,
1695 uint64_t object
, dmu_tx_t
*tx
)
1699 for (j
= 0; j
< dnp
->dn_nblkptr
; j
++) {
1700 zbookmark_phys_t czb
;
1702 SET_BOOKMARK(&czb
, ds
? ds
->ds_object
: 0, object
,
1703 dnp
->dn_nlevels
- 1, j
);
1704 dsl_scan_visitbp(&dnp
->dn_blkptr
[j
],
1705 &czb
, dnp
, ds
, scn
, ostype
, tx
);
1708 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1709 zbookmark_phys_t czb
;
1710 SET_BOOKMARK(&czb
, ds
? ds
->ds_object
: 0, object
,
1711 0, DMU_SPILL_BLKID
);
1712 dsl_scan_visitbp(DN_SPILL_BLKPTR(dnp
),
1713 &czb
, dnp
, ds
, scn
, ostype
, tx
);
1718 * The arguments are in this order because mdb can only print the
1719 * first 5; we want them to be useful.
1722 dsl_scan_visitbp(blkptr_t
*bp
, const zbookmark_phys_t
*zb
,
1723 dnode_phys_t
*dnp
, dsl_dataset_t
*ds
, dsl_scan_t
*scn
,
1724 dmu_objset_type_t ostype
, dmu_tx_t
*tx
)
1726 dsl_pool_t
*dp
= scn
->scn_dp
;
1727 blkptr_t
*bp_toread
= NULL
;
1729 if (dsl_scan_check_suspend(scn
, zb
))
1732 if (dsl_scan_check_resume(scn
, dnp
, zb
))
1735 scn
->scn_visited_this_txg
++;
1738 * This debugging is commented out to conserve stack space. This
1739 * function is called recursively and the debugging addes several
1740 * bytes to the stack for each call. It can be commented back in
1741 * if required to debug an issue in dsl_scan_visitbp().
1744 * "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p",
1745 * ds, ds ? ds->ds_object : 0,
1746 * zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
1750 if (BP_IS_HOLE(bp
)) {
1751 scn
->scn_holes_this_txg
++;
1755 if (bp
->blk_birth
<= scn
->scn_phys
.scn_cur_min_txg
) {
1756 scn
->scn_lt_min_this_txg
++;
1760 bp_toread
= kmem_alloc(sizeof (blkptr_t
), KM_SLEEP
);
1763 if (dsl_scan_recurse(scn
, ds
, ostype
, dnp
, bp_toread
, zb
, tx
) != 0)
1767 * If dsl_scan_ddt() has already visited this block, it will have
1768 * already done any translations or scrubbing, so don't call the
1771 if (ddt_class_contains(dp
->dp_spa
,
1772 scn
->scn_phys
.scn_ddt_class_max
, bp
)) {
1773 scn
->scn_ddt_contained_this_txg
++;
1778 * If this block is from the future (after cur_max_txg), then we
1779 * are doing this on behalf of a deleted snapshot, and we will
1780 * revisit the future block on the next pass of this dataset.
1781 * Don't scan it now unless we need to because something
1782 * under it was modified.
1784 if (BP_PHYSICAL_BIRTH(bp
) > scn
->scn_phys
.scn_cur_max_txg
) {
1785 scn
->scn_gt_max_this_txg
++;
1789 scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, zb
);
1792 kmem_free(bp_toread
, sizeof (blkptr_t
));
1796 dsl_scan_visit_rootbp(dsl_scan_t
*scn
, dsl_dataset_t
*ds
, blkptr_t
*bp
,
1799 zbookmark_phys_t zb
;
1800 scan_prefetch_ctx_t
*spc
;
1802 SET_BOOKMARK(&zb
, ds
? ds
->ds_object
: DMU_META_OBJSET
,
1803 ZB_ROOT_OBJECT
, ZB_ROOT_LEVEL
, ZB_ROOT_BLKID
);
1805 if (ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
)) {
1806 SET_BOOKMARK(&scn
->scn_prefetch_bookmark
,
1807 zb
.zb_objset
, 0, 0, 0);
1809 scn
->scn_prefetch_bookmark
= scn
->scn_phys
.scn_bookmark
;
1812 scn
->scn_objsets_visited_this_txg
++;
1814 spc
= scan_prefetch_ctx_create(scn
, NULL
, FTAG
);
1815 dsl_scan_prefetch(spc
, bp
, &zb
);
1816 scan_prefetch_ctx_rele(spc
, FTAG
);
1818 dsl_scan_visitbp(bp
, &zb
, NULL
, ds
, scn
, DMU_OST_NONE
, tx
);
1820 dprintf_ds(ds
, "finished scan%s", "");
1824 ds_destroyed_scn_phys(dsl_dataset_t
*ds
, dsl_scan_phys_t
*scn_phys
)
1826 if (scn_phys
->scn_bookmark
.zb_objset
== ds
->ds_object
) {
1827 if (ds
->ds_is_snapshot
) {
1830 * - scn_cur_{min,max}_txg stays the same.
1831 * - Setting the flag is not really necessary if
1832 * scn_cur_max_txg == scn_max_txg, because there
1833 * is nothing after this snapshot that we care
1834 * about. However, we set it anyway and then
1835 * ignore it when we retraverse it in
1836 * dsl_scan_visitds().
1838 scn_phys
->scn_bookmark
.zb_objset
=
1839 dsl_dataset_phys(ds
)->ds_next_snap_obj
;
1840 zfs_dbgmsg("destroying ds %llu; currently traversing; "
1841 "reset zb_objset to %llu",
1842 (u_longlong_t
)ds
->ds_object
,
1843 (u_longlong_t
)dsl_dataset_phys(ds
)->
1845 scn_phys
->scn_flags
|= DSF_VISIT_DS_AGAIN
;
1847 SET_BOOKMARK(&scn_phys
->scn_bookmark
,
1848 ZB_DESTROYED_OBJSET
, 0, 0, 0);
1849 zfs_dbgmsg("destroying ds %llu; currently traversing; "
1850 "reset bookmark to -1,0,0,0",
1851 (u_longlong_t
)ds
->ds_object
);
1857 * Invoked when a dataset is destroyed. We need to make sure that:
1859 * 1) If it is the dataset that was currently being scanned, we write
1860 * a new dsl_scan_phys_t and marking the objset reference in it
1862 * 2) Remove it from the work queue, if it was present.
1864 * If the dataset was actually a snapshot, instead of marking the dataset
1865 * as destroyed, we instead substitute the next snapshot in line.
1868 dsl_scan_ds_destroyed(dsl_dataset_t
*ds
, dmu_tx_t
*tx
)
1870 dsl_pool_t
*dp
= ds
->ds_dir
->dd_pool
;
1871 dsl_scan_t
*scn
= dp
->dp_scan
;
1874 if (!dsl_scan_is_running(scn
))
1877 ds_destroyed_scn_phys(ds
, &scn
->scn_phys
);
1878 ds_destroyed_scn_phys(ds
, &scn
->scn_phys_cached
);
1880 if (scan_ds_queue_contains(scn
, ds
->ds_object
, &mintxg
)) {
1881 scan_ds_queue_remove(scn
, ds
->ds_object
);
1882 if (ds
->ds_is_snapshot
)
1883 scan_ds_queue_insert(scn
,
1884 dsl_dataset_phys(ds
)->ds_next_snap_obj
, mintxg
);
1887 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
1888 ds
->ds_object
, &mintxg
) == 0) {
1889 ASSERT3U(dsl_dataset_phys(ds
)->ds_num_children
, <=, 1);
1890 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
1891 scn
->scn_phys
.scn_queue_obj
, ds
->ds_object
, tx
));
1892 if (ds
->ds_is_snapshot
) {
1894 * We keep the same mintxg; it could be >
1895 * ds_creation_txg if the previous snapshot was
1898 VERIFY(zap_add_int_key(dp
->dp_meta_objset
,
1899 scn
->scn_phys
.scn_queue_obj
,
1900 dsl_dataset_phys(ds
)->ds_next_snap_obj
,
1902 zfs_dbgmsg("destroying ds %llu; in queue; "
1903 "replacing with %llu",
1904 (u_longlong_t
)ds
->ds_object
,
1905 (u_longlong_t
)dsl_dataset_phys(ds
)->
1908 zfs_dbgmsg("destroying ds %llu; in queue; removing",
1909 (u_longlong_t
)ds
->ds_object
);
1914 * dsl_scan_sync() should be called after this, and should sync
1915 * out our changed state, but just to be safe, do it here.
1917 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
1921 ds_snapshotted_bookmark(dsl_dataset_t
*ds
, zbookmark_phys_t
*scn_bookmark
)
1923 if (scn_bookmark
->zb_objset
== ds
->ds_object
) {
1924 scn_bookmark
->zb_objset
=
1925 dsl_dataset_phys(ds
)->ds_prev_snap_obj
;
1926 zfs_dbgmsg("snapshotting ds %llu; currently traversing; "
1927 "reset zb_objset to %llu",
1928 (u_longlong_t
)ds
->ds_object
,
1929 (u_longlong_t
)dsl_dataset_phys(ds
)->ds_prev_snap_obj
);
1934 * Called when a dataset is snapshotted. If we were currently traversing
1935 * this snapshot, we reset our bookmark to point at the newly created
1936 * snapshot. We also modify our work queue to remove the old snapshot and
1937 * replace with the new one.
1940 dsl_scan_ds_snapshotted(dsl_dataset_t
*ds
, dmu_tx_t
*tx
)
1942 dsl_pool_t
*dp
= ds
->ds_dir
->dd_pool
;
1943 dsl_scan_t
*scn
= dp
->dp_scan
;
1946 if (!dsl_scan_is_running(scn
))
1949 ASSERT(dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= 0);
1951 ds_snapshotted_bookmark(ds
, &scn
->scn_phys
.scn_bookmark
);
1952 ds_snapshotted_bookmark(ds
, &scn
->scn_phys_cached
.scn_bookmark
);
1954 if (scan_ds_queue_contains(scn
, ds
->ds_object
, &mintxg
)) {
1955 scan_ds_queue_remove(scn
, ds
->ds_object
);
1956 scan_ds_queue_insert(scn
,
1957 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, mintxg
);
1960 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
1961 ds
->ds_object
, &mintxg
) == 0) {
1962 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
1963 scn
->scn_phys
.scn_queue_obj
, ds
->ds_object
, tx
));
1964 VERIFY(zap_add_int_key(dp
->dp_meta_objset
,
1965 scn
->scn_phys
.scn_queue_obj
,
1966 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, mintxg
, tx
) == 0);
1967 zfs_dbgmsg("snapshotting ds %llu; in queue; "
1968 "replacing with %llu",
1969 (u_longlong_t
)ds
->ds_object
,
1970 (u_longlong_t
)dsl_dataset_phys(ds
)->ds_prev_snap_obj
);
1973 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
1977 ds_clone_swapped_bookmark(dsl_dataset_t
*ds1
, dsl_dataset_t
*ds2
,
1978 zbookmark_phys_t
*scn_bookmark
)
1980 if (scn_bookmark
->zb_objset
== ds1
->ds_object
) {
1981 scn_bookmark
->zb_objset
= ds2
->ds_object
;
1982 zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
1983 "reset zb_objset to %llu",
1984 (u_longlong_t
)ds1
->ds_object
,
1985 (u_longlong_t
)ds2
->ds_object
);
1986 } else if (scn_bookmark
->zb_objset
== ds2
->ds_object
) {
1987 scn_bookmark
->zb_objset
= ds1
->ds_object
;
1988 zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
1989 "reset zb_objset to %llu",
1990 (u_longlong_t
)ds2
->ds_object
,
1991 (u_longlong_t
)ds1
->ds_object
);
1996 * Called when a parent dataset and its clone are swapped. If we were
1997 * currently traversing the dataset, we need to switch to traversing the
1998 * newly promoted parent.
2001 dsl_scan_ds_clone_swapped(dsl_dataset_t
*ds1
, dsl_dataset_t
*ds2
, dmu_tx_t
*tx
)
2003 dsl_pool_t
*dp
= ds1
->ds_dir
->dd_pool
;
2004 dsl_scan_t
*scn
= dp
->dp_scan
;
2007 if (!dsl_scan_is_running(scn
))
2010 ds_clone_swapped_bookmark(ds1
, ds2
, &scn
->scn_phys
.scn_bookmark
);
2011 ds_clone_swapped_bookmark(ds1
, ds2
, &scn
->scn_phys_cached
.scn_bookmark
);
2013 if (scan_ds_queue_contains(scn
, ds1
->ds_object
, &mintxg
)) {
2014 scan_ds_queue_remove(scn
, ds1
->ds_object
);
2015 scan_ds_queue_insert(scn
, ds2
->ds_object
, mintxg
);
2017 if (scan_ds_queue_contains(scn
, ds2
->ds_object
, &mintxg
)) {
2018 scan_ds_queue_remove(scn
, ds2
->ds_object
);
2019 scan_ds_queue_insert(scn
, ds1
->ds_object
, mintxg
);
2022 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
2023 ds1
->ds_object
, &mintxg
) == 0) {
2025 ASSERT3U(mintxg
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2026 ASSERT3U(mintxg
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2027 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2028 scn
->scn_phys
.scn_queue_obj
, ds1
->ds_object
, tx
));
2029 err
= zap_add_int_key(dp
->dp_meta_objset
,
2030 scn
->scn_phys
.scn_queue_obj
, ds2
->ds_object
, mintxg
, tx
);
2031 VERIFY(err
== 0 || err
== EEXIST
);
2032 if (err
== EEXIST
) {
2033 /* Both were there to begin with */
2034 VERIFY(0 == zap_add_int_key(dp
->dp_meta_objset
,
2035 scn
->scn_phys
.scn_queue_obj
,
2036 ds1
->ds_object
, mintxg
, tx
));
2038 zfs_dbgmsg("clone_swap ds %llu; in queue; "
2039 "replacing with %llu",
2040 (u_longlong_t
)ds1
->ds_object
,
2041 (u_longlong_t
)ds2
->ds_object
);
2043 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
2044 ds2
->ds_object
, &mintxg
) == 0) {
2045 ASSERT3U(mintxg
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2046 ASSERT3U(mintxg
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2047 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2048 scn
->scn_phys
.scn_queue_obj
, ds2
->ds_object
, tx
));
2049 VERIFY(0 == zap_add_int_key(dp
->dp_meta_objset
,
2050 scn
->scn_phys
.scn_queue_obj
, ds1
->ds_object
, mintxg
, tx
));
2051 zfs_dbgmsg("clone_swap ds %llu; in queue; "
2052 "replacing with %llu",
2053 (u_longlong_t
)ds2
->ds_object
,
2054 (u_longlong_t
)ds1
->ds_object
);
2057 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
2062 enqueue_clones_cb(dsl_pool_t
*dp
, dsl_dataset_t
*hds
, void *arg
)
2064 uint64_t originobj
= *(uint64_t *)arg
;
2067 dsl_scan_t
*scn
= dp
->dp_scan
;
2069 if (dsl_dir_phys(hds
->ds_dir
)->dd_origin_obj
!= originobj
)
2072 err
= dsl_dataset_hold_obj(dp
, hds
->ds_object
, FTAG
, &ds
);
2076 while (dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= originobj
) {
2077 dsl_dataset_t
*prev
;
2078 err
= dsl_dataset_hold_obj(dp
,
2079 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, FTAG
, &prev
);
2081 dsl_dataset_rele(ds
, FTAG
);
2086 scan_ds_queue_insert(scn
, ds
->ds_object
,
2087 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
2088 dsl_dataset_rele(ds
, FTAG
);
2093 dsl_scan_visitds(dsl_scan_t
*scn
, uint64_t dsobj
, dmu_tx_t
*tx
)
2095 dsl_pool_t
*dp
= scn
->scn_dp
;
2099 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp
, dsobj
, FTAG
, &ds
));
2101 if (scn
->scn_phys
.scn_cur_min_txg
>=
2102 scn
->scn_phys
.scn_max_txg
) {
2104 * This can happen if this snapshot was created after the
2105 * scan started, and we already completed a previous snapshot
2106 * that was created after the scan started. This snapshot
2107 * only references blocks with:
2109 * birth < our ds_creation_txg
2110 * cur_min_txg is no less than ds_creation_txg.
2111 * We have already visited these blocks.
2113 * birth > scn_max_txg
2114 * The scan requested not to visit these blocks.
2116 * Subsequent snapshots (and clones) can reference our
2117 * blocks, or blocks with even higher birth times.
2118 * Therefore we do not need to visit them either,
2119 * so we do not add them to the work queue.
2121 * Note that checking for cur_min_txg >= cur_max_txg
2122 * is not sufficient, because in that case we may need to
2123 * visit subsequent snapshots. This happens when min_txg > 0,
2124 * which raises cur_min_txg. In this case we will visit
2125 * this dataset but skip all of its blocks, because the
2126 * rootbp's birth time is < cur_min_txg. Then we will
2127 * add the next snapshots/clones to the work queue.
2129 char *dsname
= kmem_alloc(ZFS_MAX_DATASET_NAME_LEN
, KM_SLEEP
);
2130 dsl_dataset_name(ds
, dsname
);
2131 zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because "
2132 "cur_min_txg (%llu) >= max_txg (%llu)",
2133 (longlong_t
)dsobj
, dsname
,
2134 (longlong_t
)scn
->scn_phys
.scn_cur_min_txg
,
2135 (longlong_t
)scn
->scn_phys
.scn_max_txg
);
2136 kmem_free(dsname
, MAXNAMELEN
);
2141 if (dmu_objset_from_ds(ds
, &os
))
2145 * Only the ZIL in the head (non-snapshot) is valid. Even though
2146 * snapshots can have ZIL block pointers (which may be the same
2147 * BP as in the head), they must be ignored. So we traverse the
2148 * ZIL here, rather than in scan_recurse(), because the regular
2149 * snapshot block-sharing rules don't apply to it.
2151 if (!ds
->ds_is_snapshot
)
2152 dsl_scan_zil(dp
, &os
->os_zil_header
);
2155 * Iterate over the bps in this ds.
2157 dmu_buf_will_dirty(ds
->ds_dbuf
, tx
);
2158 rrw_enter(&ds
->ds_bp_rwlock
, RW_READER
, FTAG
);
2159 dsl_scan_visit_rootbp(scn
, ds
, &dsl_dataset_phys(ds
)->ds_bp
, tx
);
2160 rrw_exit(&ds
->ds_bp_rwlock
, FTAG
);
2162 char *dsname
= kmem_alloc(ZFS_MAX_DATASET_NAME_LEN
, KM_SLEEP
);
2163 dsl_dataset_name(ds
, dsname
);
2164 zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; "
2166 (longlong_t
)dsobj
, dsname
,
2167 (longlong_t
)scn
->scn_phys
.scn_cur_min_txg
,
2168 (longlong_t
)scn
->scn_phys
.scn_cur_max_txg
,
2169 (int)scn
->scn_suspending
);
2170 kmem_free(dsname
, ZFS_MAX_DATASET_NAME_LEN
);
2172 if (scn
->scn_suspending
)
2176 * We've finished this pass over this dataset.
2180 * If we did not completely visit this dataset, do another pass.
2182 if (scn
->scn_phys
.scn_flags
& DSF_VISIT_DS_AGAIN
) {
2183 zfs_dbgmsg("incomplete pass; visiting again");
2184 scn
->scn_phys
.scn_flags
&= ~DSF_VISIT_DS_AGAIN
;
2185 scan_ds_queue_insert(scn
, ds
->ds_object
,
2186 scn
->scn_phys
.scn_cur_max_txg
);
2191 * Add descendent datasets to work queue.
2193 if (dsl_dataset_phys(ds
)->ds_next_snap_obj
!= 0) {
2194 scan_ds_queue_insert(scn
,
2195 dsl_dataset_phys(ds
)->ds_next_snap_obj
,
2196 dsl_dataset_phys(ds
)->ds_creation_txg
);
2198 if (dsl_dataset_phys(ds
)->ds_num_children
> 1) {
2199 boolean_t usenext
= B_FALSE
;
2200 if (dsl_dataset_phys(ds
)->ds_next_clones_obj
!= 0) {
2203 * A bug in a previous version of the code could
2204 * cause upgrade_clones_cb() to not set
2205 * ds_next_snap_obj when it should, leading to a
2206 * missing entry. Therefore we can only use the
2207 * next_clones_obj when its count is correct.
2209 int err
= zap_count(dp
->dp_meta_objset
,
2210 dsl_dataset_phys(ds
)->ds_next_clones_obj
, &count
);
2212 count
== dsl_dataset_phys(ds
)->ds_num_children
- 1)
2219 for (zap_cursor_init(&zc
, dp
->dp_meta_objset
,
2220 dsl_dataset_phys(ds
)->ds_next_clones_obj
);
2221 zap_cursor_retrieve(&zc
, &za
) == 0;
2222 (void) zap_cursor_advance(&zc
)) {
2223 scan_ds_queue_insert(scn
,
2224 zfs_strtonum(za
.za_name
, NULL
),
2225 dsl_dataset_phys(ds
)->ds_creation_txg
);
2227 zap_cursor_fini(&zc
);
2229 VERIFY0(dmu_objset_find_dp(dp
, dp
->dp_root_dir_obj
,
2230 enqueue_clones_cb
, &ds
->ds_object
,
2236 dsl_dataset_rele(ds
, FTAG
);
2241 enqueue_cb(dsl_pool_t
*dp
, dsl_dataset_t
*hds
, void *arg
)
2245 dsl_scan_t
*scn
= dp
->dp_scan
;
2247 err
= dsl_dataset_hold_obj(dp
, hds
->ds_object
, FTAG
, &ds
);
2251 while (dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= 0) {
2252 dsl_dataset_t
*prev
;
2253 err
= dsl_dataset_hold_obj(dp
,
2254 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, FTAG
, &prev
);
2256 dsl_dataset_rele(ds
, FTAG
);
2261 * If this is a clone, we don't need to worry about it for now.
2263 if (dsl_dataset_phys(prev
)->ds_next_snap_obj
!= ds
->ds_object
) {
2264 dsl_dataset_rele(ds
, FTAG
);
2265 dsl_dataset_rele(prev
, FTAG
);
2268 dsl_dataset_rele(ds
, FTAG
);
2272 scan_ds_queue_insert(scn
, ds
->ds_object
,
2273 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
2274 dsl_dataset_rele(ds
, FTAG
);
2280 dsl_scan_ddt_entry(dsl_scan_t
*scn
, enum zio_checksum checksum
,
2281 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
2283 const ddt_key_t
*ddk
= &dde
->dde_key
;
2284 ddt_phys_t
*ddp
= dde
->dde_phys
;
2286 zbookmark_phys_t zb
= { 0 };
2289 if (scn
->scn_phys
.scn_state
!= DSS_SCANNING
)
2292 for (p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
2293 if (ddp
->ddp_phys_birth
== 0 ||
2294 ddp
->ddp_phys_birth
> scn
->scn_phys
.scn_max_txg
)
2296 ddt_bp_create(checksum
, ddk
, ddp
, &bp
);
2298 scn
->scn_visited_this_txg
++;
2299 scan_funcs
[scn
->scn_phys
.scn_func
](scn
->scn_dp
, &bp
, &zb
);
2304 * Scrub/dedup interaction.
2306 * If there are N references to a deduped block, we don't want to scrub it
2307 * N times -- ideally, we should scrub it exactly once.
2309 * We leverage the fact that the dde's replication class (enum ddt_class)
2310 * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest
2311 * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order.
2313 * To prevent excess scrubbing, the scrub begins by walking the DDT
2314 * to find all blocks with refcnt > 1, and scrubs each of these once.
2315 * Since there are two replication classes which contain blocks with
2316 * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first.
2317 * Finally the top-down scrub begins, only visiting blocks with refcnt == 1.
2319 * There would be nothing more to say if a block's refcnt couldn't change
2320 * during a scrub, but of course it can so we must account for changes
2321 * in a block's replication class.
2323 * Here's an example of what can occur:
2325 * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1
2326 * when visited during the top-down scrub phase, it will be scrubbed twice.
2327 * This negates our scrub optimization, but is otherwise harmless.
2329 * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1
2330 * on each visit during the top-down scrub phase, it will never be scrubbed.
2331 * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's
2332 * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to
2333 * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1
2334 * while a scrub is in progress, it scrubs the block right then.
2337 dsl_scan_ddt(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
2339 ddt_bookmark_t
*ddb
= &scn
->scn_phys
.scn_ddt_bookmark
;
2344 bzero(&dde
, sizeof (ddt_entry_t
));
2346 while ((error
= ddt_walk(scn
->scn_dp
->dp_spa
, ddb
, &dde
)) == 0) {
2349 if (ddb
->ddb_class
> scn
->scn_phys
.scn_ddt_class_max
)
2351 dprintf("visiting ddb=%llu/%llu/%llu/%llx\n",
2352 (longlong_t
)ddb
->ddb_class
,
2353 (longlong_t
)ddb
->ddb_type
,
2354 (longlong_t
)ddb
->ddb_checksum
,
2355 (longlong_t
)ddb
->ddb_cursor
);
2357 /* There should be no pending changes to the dedup table */
2358 ddt
= scn
->scn_dp
->dp_spa
->spa_ddt
[ddb
->ddb_checksum
];
2359 ASSERT(avl_first(&ddt
->ddt_tree
) == NULL
);
2361 dsl_scan_ddt_entry(scn
, ddb
->ddb_checksum
, &dde
, tx
);
2364 if (dsl_scan_check_suspend(scn
, NULL
))
2368 zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; "
2369 "suspending=%u", (longlong_t
)n
,
2370 (int)scn
->scn_phys
.scn_ddt_class_max
, (int)scn
->scn_suspending
);
2372 ASSERT(error
== 0 || error
== ENOENT
);
2373 ASSERT(error
!= ENOENT
||
2374 ddb
->ddb_class
> scn
->scn_phys
.scn_ddt_class_max
);
2378 dsl_scan_ds_maxtxg(dsl_dataset_t
*ds
)
2380 uint64_t smt
= ds
->ds_dir
->dd_pool
->dp_scan
->scn_phys
.scn_max_txg
;
2381 if (ds
->ds_is_snapshot
)
2382 return (MIN(smt
, dsl_dataset_phys(ds
)->ds_creation_txg
));
2387 dsl_scan_visit(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
2390 dsl_pool_t
*dp
= scn
->scn_dp
;
2392 if (scn
->scn_phys
.scn_ddt_bookmark
.ddb_class
<=
2393 scn
->scn_phys
.scn_ddt_class_max
) {
2394 scn
->scn_phys
.scn_cur_min_txg
= scn
->scn_phys
.scn_min_txg
;
2395 scn
->scn_phys
.scn_cur_max_txg
= scn
->scn_phys
.scn_max_txg
;
2396 dsl_scan_ddt(scn
, tx
);
2397 if (scn
->scn_suspending
)
2401 if (scn
->scn_phys
.scn_bookmark
.zb_objset
== DMU_META_OBJSET
) {
2402 /* First do the MOS & ORIGIN */
2404 scn
->scn_phys
.scn_cur_min_txg
= scn
->scn_phys
.scn_min_txg
;
2405 scn
->scn_phys
.scn_cur_max_txg
= scn
->scn_phys
.scn_max_txg
;
2406 dsl_scan_visit_rootbp(scn
, NULL
,
2407 &dp
->dp_meta_rootbp
, tx
);
2408 spa_set_rootblkptr(dp
->dp_spa
, &dp
->dp_meta_rootbp
);
2409 if (scn
->scn_suspending
)
2412 if (spa_version(dp
->dp_spa
) < SPA_VERSION_DSL_SCRUB
) {
2413 VERIFY0(dmu_objset_find_dp(dp
, dp
->dp_root_dir_obj
,
2414 enqueue_cb
, NULL
, DS_FIND_CHILDREN
));
2416 dsl_scan_visitds(scn
,
2417 dp
->dp_origin_snap
->ds_object
, tx
);
2419 ASSERT(!scn
->scn_suspending
);
2420 } else if (scn
->scn_phys
.scn_bookmark
.zb_objset
!=
2421 ZB_DESTROYED_OBJSET
) {
2422 uint64_t dsobj
= scn
->scn_phys
.scn_bookmark
.zb_objset
;
2424 * If we were suspended, continue from here. Note if the
2425 * ds we were suspended on was deleted, the zb_objset may
2426 * be -1, so we will skip this and find a new objset
2429 dsl_scan_visitds(scn
, dsobj
, tx
);
2430 if (scn
->scn_suspending
)
2435 * In case we suspended right at the end of the ds, zero the
2436 * bookmark so we don't think that we're still trying to resume.
2438 bzero(&scn
->scn_phys
.scn_bookmark
, sizeof (zbookmark_phys_t
));
2441 * Keep pulling things out of the dataset avl queue. Updates to the
2442 * persistent zap-object-as-queue happen only at checkpoints.
2444 while ((sds
= avl_first(&scn
->scn_queue
)) != NULL
) {
2446 uint64_t dsobj
= sds
->sds_dsobj
;
2447 uint64_t txg
= sds
->sds_txg
;
2449 /* dequeue and free the ds from the queue */
2450 scan_ds_queue_remove(scn
, dsobj
);
2453 /* set up min / max txg */
2454 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp
, dsobj
, FTAG
, &ds
));
2456 scn
->scn_phys
.scn_cur_min_txg
=
2457 MAX(scn
->scn_phys
.scn_min_txg
, txg
);
2459 scn
->scn_phys
.scn_cur_min_txg
=
2460 MAX(scn
->scn_phys
.scn_min_txg
,
2461 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
2463 scn
->scn_phys
.scn_cur_max_txg
= dsl_scan_ds_maxtxg(ds
);
2464 dsl_dataset_rele(ds
, FTAG
);
2466 dsl_scan_visitds(scn
, dsobj
, tx
);
2467 if (scn
->scn_suspending
)
2471 /* No more objsets to fetch, we're done */
2472 scn
->scn_phys
.scn_bookmark
.zb_objset
= ZB_DESTROYED_OBJSET
;
2473 ASSERT0(scn
->scn_suspending
);
2477 dsl_scan_count_leaves(vdev_t
*vd
)
2479 uint64_t i
, leaves
= 0;
2481 /* we only count leaves that belong to the main pool and are readable */
2482 if (vd
->vdev_islog
|| vd
->vdev_isspare
||
2483 vd
->vdev_isl2cache
|| !vdev_readable(vd
))
2486 if (vd
->vdev_ops
->vdev_op_leaf
)
2489 for (i
= 0; i
< vd
->vdev_children
; i
++) {
2490 leaves
+= dsl_scan_count_leaves(vd
->vdev_child
[i
]);
2497 scan_io_queues_update_zio_stats(dsl_scan_io_queue_t
*q
, const blkptr_t
*bp
)
2500 uint64_t cur_size
= 0;
2502 for (i
= 0; i
< BP_GET_NDVAS(bp
); i
++) {
2503 cur_size
+= DVA_GET_ASIZE(&bp
->blk_dva
[i
]);
2506 q
->q_total_zio_size_this_txg
+= cur_size
;
2507 q
->q_zios_this_txg
++;
2511 scan_io_queues_update_seg_stats(dsl_scan_io_queue_t
*q
, uint64_t start
,
2514 q
->q_total_seg_size_this_txg
+= end
- start
;
2515 q
->q_segs_this_txg
++;
2519 scan_io_queue_check_suspend(dsl_scan_t
*scn
)
2521 /* See comment in dsl_scan_check_suspend() */
2522 uint64_t curr_time_ns
= gethrtime();
2523 uint64_t scan_time_ns
= curr_time_ns
- scn
->scn_sync_start_time
;
2524 uint64_t sync_time_ns
= curr_time_ns
-
2525 scn
->scn_dp
->dp_spa
->spa_sync_starttime
;
2526 int dirty_pct
= scn
->scn_dp
->dp_dirty_total
* 100 / zfs_dirty_data_max
;
2527 int mintime
= (scn
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
) ?
2528 zfs_resilver_min_time_ms
: zfs_scrub_min_time_ms
;
2530 return ((NSEC2MSEC(scan_time_ns
) > mintime
&&
2531 (dirty_pct
>= zfs_vdev_async_write_active_min_dirty_percent
||
2532 txg_sync_waiting(scn
->scn_dp
) ||
2533 NSEC2SEC(sync_time_ns
) >= zfs_txg_timeout
)) ||
2534 spa_shutting_down(scn
->scn_dp
->dp_spa
));
2538 * Given a list of scan_io_t's in io_list, this issues the io's out to
2539 * disk. This consumes the io_list and frees the scan_io_t's. This is
2540 * called when emptying queues, either when we're up against the memory
2541 * limit or when we have finished scanning. Returns B_TRUE if we stopped
2542 * processing the list before we finished. Any zios that were not issued
2543 * will remain in the io_list.
2546 scan_io_queue_issue(dsl_scan_io_queue_t
*queue
, list_t
*io_list
)
2548 dsl_scan_t
*scn
= queue
->q_scn
;
2550 int64_t bytes_issued
= 0;
2551 boolean_t suspended
= B_FALSE
;
2553 while ((sio
= list_head(io_list
)) != NULL
) {
2556 if (scan_io_queue_check_suspend(scn
)) {
2561 sio2bp(sio
, &bp
, queue
->q_vd
->vdev_id
);
2562 bytes_issued
+= sio
->sio_asize
;
2563 scan_exec_io(scn
->scn_dp
, &bp
, sio
->sio_flags
,
2564 &sio
->sio_zb
, queue
);
2565 (void) list_remove_head(io_list
);
2566 scan_io_queues_update_zio_stats(queue
, &bp
);
2567 kmem_cache_free(sio_cache
, sio
);
2570 atomic_add_64(&scn
->scn_bytes_pending
, -bytes_issued
);
2576 * This function removes sios from an IO queue which reside within a given
2577 * range_seg_t and inserts them (in offset order) into a list. Note that
2578 * we only ever return a maximum of 32 sios at once. If there are more sios
2579 * to process within this segment that did not make it onto the list we
2580 * return B_TRUE and otherwise B_FALSE.
2583 scan_io_queue_gather(dsl_scan_io_queue_t
*queue
, range_seg_t
*rs
, list_t
*list
)
2585 scan_io_t srch_sio
, *sio
, *next_sio
;
2587 uint_t num_sios
= 0;
2588 int64_t bytes_issued
= 0;
2591 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
2593 srch_sio
.sio_offset
= rs
->rs_start
;
2596 * The exact start of the extent might not contain any matching zios,
2597 * so if that's the case, examine the next one in the tree.
2599 sio
= avl_find(&queue
->q_sios_by_addr
, &srch_sio
, &idx
);
2601 sio
= avl_nearest(&queue
->q_sios_by_addr
, idx
, AVL_AFTER
);
2603 while (sio
!= NULL
&& sio
->sio_offset
< rs
->rs_end
&& num_sios
<= 32) {
2604 ASSERT3U(sio
->sio_offset
, >=, rs
->rs_start
);
2605 ASSERT3U(sio
->sio_offset
+ sio
->sio_asize
, <=, rs
->rs_end
);
2607 next_sio
= AVL_NEXT(&queue
->q_sios_by_addr
, sio
);
2608 avl_remove(&queue
->q_sios_by_addr
, sio
);
2610 bytes_issued
+= sio
->sio_asize
;
2612 list_insert_tail(list
, sio
);
2617 * We limit the number of sios we process at once to 32 to avoid
2618 * biting off more than we can chew. If we didn't take everything
2619 * in the segment we update it to reflect the work we were able to
2620 * complete. Otherwise, we remove it from the range tree entirely.
2622 if (sio
!= NULL
&& sio
->sio_offset
< rs
->rs_end
) {
2623 range_tree_adjust_fill(queue
->q_exts_by_addr
, rs
,
2625 range_tree_resize_segment(queue
->q_exts_by_addr
, rs
,
2626 sio
->sio_offset
, rs
->rs_end
- sio
->sio_offset
);
2630 range_tree_remove(queue
->q_exts_by_addr
, rs
->rs_start
,
2631 rs
->rs_end
- rs
->rs_start
);
2637 * This is called from the queue emptying thread and selects the next
2638 * extent from which we are to issue io's. The behavior of this function
2639 * depends on the state of the scan, the current memory consumption and
2640 * whether or not we are performing a scan shutdown.
2641 * 1) We select extents in an elevator algorithm (LBA-order) if the scan
2642 * needs to perform a checkpoint
2643 * 2) We select the largest available extent if we are up against the
2645 * 3) Otherwise we don't select any extents.
2647 static range_seg_t
*
2648 scan_io_queue_fetch_ext(dsl_scan_io_queue_t
*queue
)
2650 dsl_scan_t
*scn
= queue
->q_scn
;
2652 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
2653 ASSERT(scn
->scn_is_sorted
);
2655 /* handle tunable overrides */
2656 if (scn
->scn_checkpointing
|| scn
->scn_clearing
) {
2657 if (zfs_scan_issue_strategy
== 1) {
2658 return (range_tree_first(queue
->q_exts_by_addr
));
2659 } else if (zfs_scan_issue_strategy
== 2) {
2660 return (avl_first(&queue
->q_exts_by_size
));
2665 * During normal clearing, we want to issue our largest segments
2666 * first, keeping IO as sequential as possible, and leaving the
2667 * smaller extents for later with the hope that they might eventually
2668 * grow to larger sequential segments. However, when the scan is
2669 * checkpointing, no new extents will be added to the sorting queue,
2670 * so the way we are sorted now is as good as it will ever get.
2671 * In this case, we instead switch to issuing extents in LBA order.
2673 if (scn
->scn_checkpointing
) {
2674 return (range_tree_first(queue
->q_exts_by_addr
));
2675 } else if (scn
->scn_clearing
) {
2676 return (avl_first(&queue
->q_exts_by_size
));
2683 scan_io_queues_run_one(void *arg
)
2685 dsl_scan_io_queue_t
*queue
= arg
;
2686 kmutex_t
*q_lock
= &queue
->q_vd
->vdev_scan_io_queue_lock
;
2687 boolean_t suspended
= B_FALSE
;
2688 range_seg_t
*rs
= NULL
;
2689 scan_io_t
*sio
= NULL
;
2691 uint64_t bytes_per_leaf
= zfs_scan_vdev_limit
;
2692 uint64_t nr_leaves
= dsl_scan_count_leaves(queue
->q_vd
);
2694 ASSERT(queue
->q_scn
->scn_is_sorted
);
2696 list_create(&sio_list
, sizeof (scan_io_t
),
2697 offsetof(scan_io_t
, sio_nodes
.sio_list_node
));
2698 mutex_enter(q_lock
);
2700 /* calculate maximum in-flight bytes for this txg (min 1MB) */
2701 queue
->q_maxinflight_bytes
=
2702 MAX(nr_leaves
* bytes_per_leaf
, 1ULL << 20);
2704 /* reset per-queue scan statistics for this txg */
2705 queue
->q_total_seg_size_this_txg
= 0;
2706 queue
->q_segs_this_txg
= 0;
2707 queue
->q_total_zio_size_this_txg
= 0;
2708 queue
->q_zios_this_txg
= 0;
2710 /* loop until we run out of time or sios */
2711 while ((rs
= scan_io_queue_fetch_ext(queue
)) != NULL
) {
2712 uint64_t seg_start
= 0, seg_end
= 0;
2713 boolean_t more_left
= B_TRUE
;
2715 ASSERT(list_is_empty(&sio_list
));
2717 /* loop while we still have sios left to process in this rs */
2719 scan_io_t
*first_sio
, *last_sio
;
2722 * We have selected which extent needs to be
2723 * processed next. Gather up the corresponding sios.
2725 more_left
= scan_io_queue_gather(queue
, rs
, &sio_list
);
2726 ASSERT(!list_is_empty(&sio_list
));
2727 first_sio
= list_head(&sio_list
);
2728 last_sio
= list_tail(&sio_list
);
2730 seg_end
= last_sio
->sio_offset
+ last_sio
->sio_asize
;
2732 seg_start
= first_sio
->sio_offset
;
2735 * Issuing sios can take a long time so drop the
2736 * queue lock. The sio queue won't be updated by
2737 * other threads since we're in syncing context so
2738 * we can be sure that our trees will remain exactly
2742 suspended
= scan_io_queue_issue(queue
, &sio_list
);
2743 mutex_enter(q_lock
);
2749 /* update statistics for debugging purposes */
2750 scan_io_queues_update_seg_stats(queue
, seg_start
, seg_end
);
2757 * If we were suspended in the middle of processing,
2758 * requeue any unfinished sios and exit.
2760 while ((sio
= list_head(&sio_list
)) != NULL
) {
2761 list_remove(&sio_list
, sio
);
2762 scan_io_queue_insert_impl(queue
, sio
);
2766 list_destroy(&sio_list
);
2770 * Performs an emptying run on all scan queues in the pool. This just
2771 * punches out one thread per top-level vdev, each of which processes
2772 * only that vdev's scan queue. We can parallelize the I/O here because
2773 * we know that each queue's io's only affect its own top-level vdev.
2775 * This function waits for the queue runs to complete, and must be
2776 * called from dsl_scan_sync (or in general, syncing context).
2779 scan_io_queues_run(dsl_scan_t
*scn
)
2781 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
2783 ASSERT(scn
->scn_is_sorted
);
2784 ASSERT(spa_config_held(spa
, SCL_CONFIG
, RW_READER
));
2786 if (scn
->scn_bytes_pending
== 0)
2789 if (scn
->scn_taskq
== NULL
) {
2790 int nthreads
= spa
->spa_root_vdev
->vdev_children
;
2793 * We need to make this taskq *always* execute as many
2794 * threads in parallel as we have top-level vdevs and no
2795 * less, otherwise strange serialization of the calls to
2796 * scan_io_queues_run_one can occur during spa_sync runs
2797 * and that significantly impacts performance.
2799 scn
->scn_taskq
= taskq_create("dsl_scan_iss", nthreads
,
2800 minclsyspri
, nthreads
, nthreads
, TASKQ_PREPOPULATE
);
2803 for (uint64_t i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
2804 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
2806 mutex_enter(&vd
->vdev_scan_io_queue_lock
);
2807 if (vd
->vdev_scan_io_queue
!= NULL
) {
2808 VERIFY(taskq_dispatch(scn
->scn_taskq
,
2809 scan_io_queues_run_one
, vd
->vdev_scan_io_queue
,
2810 TQ_SLEEP
) != TASKQID_INVALID
);
2812 mutex_exit(&vd
->vdev_scan_io_queue_lock
);
2816 * Wait for the queues to finish issuing thir IOs for this run
2817 * before we return. There may still be IOs in flight at this
2820 taskq_wait(scn
->scn_taskq
);
2824 dsl_scan_free_should_suspend(dsl_scan_t
*scn
)
2826 uint64_t elapsed_nanosecs
;
2831 if (scn
->scn_visited_this_txg
>= zfs_free_max_blocks
)
2834 elapsed_nanosecs
= gethrtime() - scn
->scn_sync_start_time
;
2835 return (elapsed_nanosecs
/ NANOSEC
> zfs_txg_timeout
||
2836 (NSEC2MSEC(elapsed_nanosecs
) > zfs_free_min_time_ms
&&
2837 txg_sync_waiting(scn
->scn_dp
)) ||
2838 spa_shutting_down(scn
->scn_dp
->dp_spa
));
2842 dsl_scan_free_block_cb(void *arg
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
2844 dsl_scan_t
*scn
= arg
;
2846 if (!scn
->scn_is_bptree
||
2847 (BP_GET_LEVEL(bp
) == 0 && BP_GET_TYPE(bp
) != DMU_OT_OBJSET
)) {
2848 if (dsl_scan_free_should_suspend(scn
))
2849 return (SET_ERROR(ERESTART
));
2852 zio_nowait(zio_free_sync(scn
->scn_zio_root
, scn
->scn_dp
->dp_spa
,
2853 dmu_tx_get_txg(tx
), bp
, 0));
2854 dsl_dir_diduse_space(tx
->tx_pool
->dp_free_dir
, DD_USED_HEAD
,
2855 -bp_get_dsize_sync(scn
->scn_dp
->dp_spa
, bp
),
2856 -BP_GET_PSIZE(bp
), -BP_GET_UCSIZE(bp
), tx
);
2857 scn
->scn_visited_this_txg
++;
2862 dsl_scan_update_stats(dsl_scan_t
*scn
)
2864 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
2866 uint64_t seg_size_total
= 0, zio_size_total
= 0;
2867 uint64_t seg_count_total
= 0, zio_count_total
= 0;
2869 for (i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
2870 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
2871 dsl_scan_io_queue_t
*queue
= vd
->vdev_scan_io_queue
;
2876 seg_size_total
+= queue
->q_total_seg_size_this_txg
;
2877 zio_size_total
+= queue
->q_total_zio_size_this_txg
;
2878 seg_count_total
+= queue
->q_segs_this_txg
;
2879 zio_count_total
+= queue
->q_zios_this_txg
;
2882 if (seg_count_total
== 0 || zio_count_total
== 0) {
2883 scn
->scn_avg_seg_size_this_txg
= 0;
2884 scn
->scn_avg_zio_size_this_txg
= 0;
2885 scn
->scn_segs_this_txg
= 0;
2886 scn
->scn_zios_this_txg
= 0;
2890 scn
->scn_avg_seg_size_this_txg
= seg_size_total
/ seg_count_total
;
2891 scn
->scn_avg_zio_size_this_txg
= zio_size_total
/ zio_count_total
;
2892 scn
->scn_segs_this_txg
= seg_count_total
;
2893 scn
->scn_zios_this_txg
= zio_count_total
;
2897 dsl_scan_active(dsl_scan_t
*scn
)
2899 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
2900 uint64_t used
= 0, comp
, uncomp
;
2902 if (spa
->spa_load_state
!= SPA_LOAD_NONE
)
2904 if (spa_shutting_down(spa
))
2906 if ((dsl_scan_is_running(scn
) && !dsl_scan_is_paused_scrub(scn
)) ||
2907 (scn
->scn_async_destroying
&& !scn
->scn_async_stalled
))
2910 if (spa_version(scn
->scn_dp
->dp_spa
) >= SPA_VERSION_DEADLISTS
) {
2911 (void) bpobj_space(&scn
->scn_dp
->dp_free_bpobj
,
2912 &used
, &comp
, &uncomp
);
2918 dsl_scan_need_resilver(spa_t
*spa
, const dva_t
*dva
, size_t psize
,
2919 uint64_t phys_birth
)
2923 if (DVA_GET_GANG(dva
)) {
2925 * Gang members may be spread across multiple
2926 * vdevs, so the best estimate we have is the
2927 * scrub range, which has already been checked.
2928 * XXX -- it would be better to change our
2929 * allocation policy to ensure that all
2930 * gang members reside on the same vdev.
2935 vd
= vdev_lookup_top(spa
, DVA_GET_VDEV(dva
));
2938 * Check if the txg falls within the range which must be
2939 * resilvered. DVAs outside this range can always be skipped.
2941 if (!vdev_dtl_contains(vd
, DTL_PARTIAL
, phys_birth
, 1))
2945 * Check if the top-level vdev must resilver this offset.
2946 * When the offset does not intersect with a dirty leaf DTL
2947 * then it may be possible to skip the resilver IO. The psize
2948 * is provided instead of asize to simplify the check for RAIDZ.
2950 if (!vdev_dtl_need_resilver(vd
, DVA_GET_OFFSET(dva
), psize
))
2957 * This is the primary entry point for scans that is called from syncing
2958 * context. Scans must happen entirely during syncing context so that we
2959 * cna guarantee that blocks we are currently scanning will not change out
2960 * from under us. While a scan is active, this funciton controls how quickly
2961 * transaction groups proceed, instead of the normal handling provided by
2962 * txg_sync_thread().
2965 dsl_scan_sync(dsl_pool_t
*dp
, dmu_tx_t
*tx
)
2968 dsl_scan_t
*scn
= dp
->dp_scan
;
2969 spa_t
*spa
= dp
->dp_spa
;
2970 state_sync_type_t sync_type
= SYNC_OPTIONAL
;
2973 * Check for scn_restart_txg before checking spa_load_state, so
2974 * that we can restart an old-style scan while the pool is being
2975 * imported (see dsl_scan_init).
2977 if (dsl_scan_restarting(scn
, tx
)) {
2978 pool_scan_func_t func
= POOL_SCAN_SCRUB
;
2979 dsl_scan_done(scn
, B_FALSE
, tx
);
2980 if (vdev_resilver_needed(spa
->spa_root_vdev
, NULL
, NULL
))
2981 func
= POOL_SCAN_RESILVER
;
2982 zfs_dbgmsg("restarting scan func=%u txg=%llu",
2983 func
, (longlong_t
)tx
->tx_txg
);
2984 dsl_scan_setup_sync(&func
, tx
);
2988 * Only process scans in sync pass 1.
2990 if (spa_sync_pass(spa
) > 1)
2994 * If the spa is shutting down, then stop scanning. This will
2995 * ensure that the scan does not dirty any new data during the
2998 if (spa_shutting_down(spa
))
3002 * If the scan is inactive due to a stalled async destroy, try again.
3004 if (!scn
->scn_async_stalled
&& !dsl_scan_active(scn
))
3007 /* reset scan statistics */
3008 scn
->scn_visited_this_txg
= 0;
3009 scn
->scn_holes_this_txg
= 0;
3010 scn
->scn_lt_min_this_txg
= 0;
3011 scn
->scn_gt_max_this_txg
= 0;
3012 scn
->scn_ddt_contained_this_txg
= 0;
3013 scn
->scn_objsets_visited_this_txg
= 0;
3014 scn
->scn_avg_seg_size_this_txg
= 0;
3015 scn
->scn_segs_this_txg
= 0;
3016 scn
->scn_avg_zio_size_this_txg
= 0;
3017 scn
->scn_zios_this_txg
= 0;
3018 scn
->scn_suspending
= B_FALSE
;
3019 scn
->scn_sync_start_time
= gethrtime();
3020 spa
->spa_scrub_active
= B_TRUE
;
3023 * First process the async destroys. If we suspend, don't do
3024 * any scrubbing or resilvering. This ensures that there are no
3025 * async destroys while we are scanning, so the scan code doesn't
3026 * have to worry about traversing it. It is also faster to free the
3027 * blocks than to scrub them.
3029 if (zfs_free_bpobj_enabled
&&
3030 spa_version(spa
) >= SPA_VERSION_DEADLISTS
) {
3031 scn
->scn_is_bptree
= B_FALSE
;
3032 scn
->scn_zio_root
= zio_root(spa
, NULL
,
3033 NULL
, ZIO_FLAG_MUSTSUCCEED
);
3034 err
= bpobj_iterate(&dp
->dp_free_bpobj
,
3035 dsl_scan_free_block_cb
, scn
, tx
);
3036 VERIFY0(zio_wait(scn
->scn_zio_root
));
3037 scn
->scn_zio_root
= NULL
;
3039 if (err
!= 0 && err
!= ERESTART
)
3040 zfs_panic_recover("error %u from bpobj_iterate()", err
);
3043 if (err
== 0 && spa_feature_is_active(spa
, SPA_FEATURE_ASYNC_DESTROY
)) {
3044 ASSERT(scn
->scn_async_destroying
);
3045 scn
->scn_is_bptree
= B_TRUE
;
3046 scn
->scn_zio_root
= zio_root(spa
, NULL
,
3047 NULL
, ZIO_FLAG_MUSTSUCCEED
);
3048 err
= bptree_iterate(dp
->dp_meta_objset
,
3049 dp
->dp_bptree_obj
, B_TRUE
, dsl_scan_free_block_cb
, scn
, tx
);
3050 VERIFY0(zio_wait(scn
->scn_zio_root
));
3051 scn
->scn_zio_root
= NULL
;
3053 if (err
== EIO
|| err
== ECKSUM
) {
3055 } else if (err
!= 0 && err
!= ERESTART
) {
3056 zfs_panic_recover("error %u from "
3057 "traverse_dataset_destroyed()", err
);
3060 if (bptree_is_empty(dp
->dp_meta_objset
, dp
->dp_bptree_obj
)) {
3061 /* finished; deactivate async destroy feature */
3062 spa_feature_decr(spa
, SPA_FEATURE_ASYNC_DESTROY
, tx
);
3063 ASSERT(!spa_feature_is_active(spa
,
3064 SPA_FEATURE_ASYNC_DESTROY
));
3065 VERIFY0(zap_remove(dp
->dp_meta_objset
,
3066 DMU_POOL_DIRECTORY_OBJECT
,
3067 DMU_POOL_BPTREE_OBJ
, tx
));
3068 VERIFY0(bptree_free(dp
->dp_meta_objset
,
3069 dp
->dp_bptree_obj
, tx
));
3070 dp
->dp_bptree_obj
= 0;
3071 scn
->scn_async_destroying
= B_FALSE
;
3072 scn
->scn_async_stalled
= B_FALSE
;
3075 * If we didn't make progress, mark the async
3076 * destroy as stalled, so that we will not initiate
3077 * a spa_sync() on its behalf. Note that we only
3078 * check this if we are not finished, because if the
3079 * bptree had no blocks for us to visit, we can
3080 * finish without "making progress".
3082 scn
->scn_async_stalled
=
3083 (scn
->scn_visited_this_txg
== 0);
3086 if (scn
->scn_visited_this_txg
) {
3087 zfs_dbgmsg("freed %llu blocks in %llums from "
3088 "free_bpobj/bptree txg %llu; err=%u",
3089 (longlong_t
)scn
->scn_visited_this_txg
,
3091 NSEC2MSEC(gethrtime() - scn
->scn_sync_start_time
),
3092 (longlong_t
)tx
->tx_txg
, err
);
3093 scn
->scn_visited_this_txg
= 0;
3096 * Write out changes to the DDT that may be required as a
3097 * result of the blocks freed. This ensures that the DDT
3098 * is clean when a scrub/resilver runs.
3100 ddt_sync(spa
, tx
->tx_txg
);
3104 if (dp
->dp_free_dir
!= NULL
&& !scn
->scn_async_destroying
&&
3105 zfs_free_leak_on_eio
&&
3106 (dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
!= 0 ||
3107 dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
!= 0 ||
3108 dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
!= 0)) {
3110 * We have finished background destroying, but there is still
3111 * some space left in the dp_free_dir. Transfer this leaked
3112 * space to the dp_leak_dir.
3114 if (dp
->dp_leak_dir
== NULL
) {
3115 rrw_enter(&dp
->dp_config_rwlock
, RW_WRITER
, FTAG
);
3116 (void) dsl_dir_create_sync(dp
, dp
->dp_root_dir
,
3118 VERIFY0(dsl_pool_open_special_dir(dp
,
3119 LEAK_DIR_NAME
, &dp
->dp_leak_dir
));
3120 rrw_exit(&dp
->dp_config_rwlock
, FTAG
);
3122 dsl_dir_diduse_space(dp
->dp_leak_dir
, DD_USED_HEAD
,
3123 dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
,
3124 dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
,
3125 dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
, tx
);
3126 dsl_dir_diduse_space(dp
->dp_free_dir
, DD_USED_HEAD
,
3127 -dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
,
3128 -dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
,
3129 -dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
, tx
);
3131 if (dp
->dp_free_dir
!= NULL
&& !scn
->scn_async_destroying
) {
3132 /* finished; verify that space accounting went to zero */
3133 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
);
3134 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
);
3135 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
);
3138 if (!dsl_scan_is_running(scn
) || dsl_scan_is_paused_scrub(scn
))
3142 * Wait a few txgs after importing to begin scanning so that
3143 * we can get the pool imported quickly.
3145 if (spa
->spa_syncing_txg
< spa
->spa_first_txg
+ SCAN_IMPORT_WAIT_TXGS
)
3149 * It is possible to switch from unsorted to sorted at any time,
3150 * but afterwards the scan will remain sorted unless reloaded from
3151 * a checkpoint after a reboot.
3153 if (!zfs_scan_legacy
) {
3154 scn
->scn_is_sorted
= B_TRUE
;
3155 if (scn
->scn_last_checkpoint
== 0)
3156 scn
->scn_last_checkpoint
= ddi_get_lbolt();
3160 * For sorted scans, determine what kind of work we will be doing
3161 * this txg based on our memory limitations and whether or not we
3162 * need to perform a checkpoint.
3164 if (scn
->scn_is_sorted
) {
3166 * If we are over our checkpoint interval, set scn_clearing
3167 * so that we can begin checkpointing immediately. The
3168 * checkpoint allows us to save a consisent bookmark
3169 * representing how much data we have scrubbed so far.
3170 * Otherwise, use the memory limit to determine if we should
3171 * scan for metadata or start issue scrub IOs. We accumulate
3172 * metadata until we hit our hard memory limit at which point
3173 * we issue scrub IOs until we are at our soft memory limit.
3175 if (scn
->scn_checkpointing
||
3176 ddi_get_lbolt() - scn
->scn_last_checkpoint
>
3177 SEC_TO_TICK(zfs_scan_checkpoint_intval
)) {
3178 if (!scn
->scn_checkpointing
)
3179 zfs_dbgmsg("begin scan checkpoint");
3181 scn
->scn_checkpointing
= B_TRUE
;
3182 scn
->scn_clearing
= B_TRUE
;
3184 boolean_t should_clear
= dsl_scan_should_clear(scn
);
3185 if (should_clear
&& !scn
->scn_clearing
) {
3186 zfs_dbgmsg("begin scan clearing");
3187 scn
->scn_clearing
= B_TRUE
;
3188 } else if (!should_clear
&& scn
->scn_clearing
) {
3189 zfs_dbgmsg("finish scan clearing");
3190 scn
->scn_clearing
= B_FALSE
;
3194 ASSERT0(scn
->scn_checkpointing
);
3195 ASSERT0(scn
->scn_clearing
);
3198 if (!scn
->scn_clearing
&& scn
->scn_done_txg
== 0) {
3199 /* Need to scan metadata for more blocks to scrub */
3200 dsl_scan_phys_t
*scnp
= &scn
->scn_phys
;
3201 taskqid_t prefetch_tqid
;
3202 uint64_t bytes_per_leaf
= zfs_scan_vdev_limit
;
3203 uint64_t nr_leaves
= dsl_scan_count_leaves(spa
->spa_root_vdev
);
3206 * Calculate the max number of in-flight bytes for pool-wide
3207 * scanning operations (minimum 1MB). Limits for the issuing
3208 * phase are done per top-level vdev and are handled separately.
3210 scn
->scn_maxinflight_bytes
=
3211 MAX(nr_leaves
* bytes_per_leaf
, 1ULL << 20);
3213 if (scnp
->scn_ddt_bookmark
.ddb_class
<=
3214 scnp
->scn_ddt_class_max
) {
3215 ASSERT(ZB_IS_ZERO(&scnp
->scn_bookmark
));
3216 zfs_dbgmsg("doing scan sync txg %llu; "
3217 "ddt bm=%llu/%llu/%llu/%llx",
3218 (longlong_t
)tx
->tx_txg
,
3219 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_class
,
3220 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_type
,
3221 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_checksum
,
3222 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_cursor
);
3224 zfs_dbgmsg("doing scan sync txg %llu; "
3225 "bm=%llu/%llu/%llu/%llu",
3226 (longlong_t
)tx
->tx_txg
,
3227 (longlong_t
)scnp
->scn_bookmark
.zb_objset
,
3228 (longlong_t
)scnp
->scn_bookmark
.zb_object
,
3229 (longlong_t
)scnp
->scn_bookmark
.zb_level
,
3230 (longlong_t
)scnp
->scn_bookmark
.zb_blkid
);
3233 scn
->scn_zio_root
= zio_root(dp
->dp_spa
, NULL
,
3234 NULL
, ZIO_FLAG_CANFAIL
);
3236 scn
->scn_prefetch_stop
= B_FALSE
;
3237 prefetch_tqid
= taskq_dispatch(dp
->dp_sync_taskq
,
3238 dsl_scan_prefetch_thread
, scn
, TQ_SLEEP
);
3239 ASSERT(prefetch_tqid
!= TASKQID_INVALID
);
3241 dsl_pool_config_enter(dp
, FTAG
);
3242 dsl_scan_visit(scn
, tx
);
3243 dsl_pool_config_exit(dp
, FTAG
);
3245 mutex_enter(&dp
->dp_spa
->spa_scrub_lock
);
3246 scn
->scn_prefetch_stop
= B_TRUE
;
3247 cv_broadcast(&spa
->spa_scrub_io_cv
);
3248 mutex_exit(&dp
->dp_spa
->spa_scrub_lock
);
3250 taskq_wait_id(dp
->dp_sync_taskq
, prefetch_tqid
);
3251 (void) zio_wait(scn
->scn_zio_root
);
3252 scn
->scn_zio_root
= NULL
;
3254 zfs_dbgmsg("scan visited %llu blocks in %llums "
3255 "(%llu os's, %llu holes, %llu < mintxg, "
3256 "%llu in ddt, %llu > maxtxg)",
3257 (longlong_t
)scn
->scn_visited_this_txg
,
3258 (longlong_t
)NSEC2MSEC(gethrtime() -
3259 scn
->scn_sync_start_time
),
3260 (longlong_t
)scn
->scn_objsets_visited_this_txg
,
3261 (longlong_t
)scn
->scn_holes_this_txg
,
3262 (longlong_t
)scn
->scn_lt_min_this_txg
,
3263 (longlong_t
)scn
->scn_ddt_contained_this_txg
,
3264 (longlong_t
)scn
->scn_gt_max_this_txg
);
3266 if (!scn
->scn_suspending
) {
3267 ASSERT0(avl_numnodes(&scn
->scn_queue
));
3268 scn
->scn_done_txg
= tx
->tx_txg
+ 1;
3269 if (scn
->scn_is_sorted
) {
3270 scn
->scn_checkpointing
= B_TRUE
;
3271 scn
->scn_clearing
= B_TRUE
;
3273 zfs_dbgmsg("scan complete txg %llu",
3274 (longlong_t
)tx
->tx_txg
);
3276 } else if (scn
->scn_is_sorted
&& scn
->scn_bytes_pending
!= 0) {
3277 /* need to issue scrubbing IOs from per-vdev queues */
3278 scn
->scn_zio_root
= zio_root(dp
->dp_spa
, NULL
,
3279 NULL
, ZIO_FLAG_CANFAIL
);
3280 scan_io_queues_run(scn
);
3281 (void) zio_wait(scn
->scn_zio_root
);
3282 scn
->scn_zio_root
= NULL
;
3284 /* calculate and dprintf the current memory usage */
3285 (void) dsl_scan_should_clear(scn
);
3286 dsl_scan_update_stats(scn
);
3288 zfs_dbgmsg("scan issued %llu blocks (%llu segs) in %llums "
3289 "(avg_block_size = %llu, avg_seg_size = %llu)",
3290 (longlong_t
)scn
->scn_zios_this_txg
,
3291 (longlong_t
)scn
->scn_segs_this_txg
,
3292 (longlong_t
)NSEC2MSEC(gethrtime() -
3293 scn
->scn_sync_start_time
),
3294 (longlong_t
)scn
->scn_avg_zio_size_this_txg
,
3295 (longlong_t
)scn
->scn_avg_seg_size_this_txg
);
3296 } else if (scn
->scn_done_txg
!= 0 && scn
->scn_done_txg
<= tx
->tx_txg
) {
3297 /* Finished with everything. Mark the scrub as complete */
3298 zfs_dbgmsg("scan issuing complete txg %llu",
3299 (longlong_t
)tx
->tx_txg
);
3300 ASSERT3U(scn
->scn_done_txg
, !=, 0);
3301 ASSERT0(spa
->spa_scrub_inflight
);
3302 ASSERT0(scn
->scn_bytes_pending
);
3303 dsl_scan_done(scn
, B_TRUE
, tx
);
3304 sync_type
= SYNC_MANDATORY
;
3307 dsl_scan_sync_state(scn
, tx
, sync_type
);
3311 count_block(dsl_scan_t
*scn
, zfs_all_blkstats_t
*zab
, const blkptr_t
*bp
)
3315 /* update the spa's stats on how many bytes we have issued */
3316 for (i
= 0; i
< BP_GET_NDVAS(bp
); i
++) {
3317 atomic_add_64(&scn
->scn_dp
->dp_spa
->spa_scan_pass_issued
,
3318 DVA_GET_ASIZE(&bp
->blk_dva
[i
]));
3322 * If we resume after a reboot, zab will be NULL; don't record
3323 * incomplete stats in that case.
3328 mutex_enter(&zab
->zab_lock
);
3330 for (i
= 0; i
< 4; i
++) {
3331 int l
= (i
< 2) ? BP_GET_LEVEL(bp
) : DN_MAX_LEVELS
;
3332 int t
= (i
& 1) ? BP_GET_TYPE(bp
) : DMU_OT_TOTAL
;
3334 if (t
& DMU_OT_NEWTYPE
)
3336 zfs_blkstat_t
*zb
= &zab
->zab_type
[l
][t
];
3340 zb
->zb_asize
+= BP_GET_ASIZE(bp
);
3341 zb
->zb_lsize
+= BP_GET_LSIZE(bp
);
3342 zb
->zb_psize
+= BP_GET_PSIZE(bp
);
3343 zb
->zb_gangs
+= BP_COUNT_GANG(bp
);
3345 switch (BP_GET_NDVAS(bp
)) {
3347 if (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
3348 DVA_GET_VDEV(&bp
->blk_dva
[1]))
3349 zb
->zb_ditto_2_of_2_samevdev
++;
3352 equal
= (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
3353 DVA_GET_VDEV(&bp
->blk_dva
[1])) +
3354 (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
3355 DVA_GET_VDEV(&bp
->blk_dva
[2])) +
3356 (DVA_GET_VDEV(&bp
->blk_dva
[1]) ==
3357 DVA_GET_VDEV(&bp
->blk_dva
[2]));
3359 zb
->zb_ditto_2_of_3_samevdev
++;
3360 else if (equal
== 3)
3361 zb
->zb_ditto_3_of_3_samevdev
++;
3366 mutex_exit(&zab
->zab_lock
);
3370 scan_io_queue_insert_impl(dsl_scan_io_queue_t
*queue
, scan_io_t
*sio
)
3373 int64_t asize
= sio
->sio_asize
;
3374 dsl_scan_t
*scn
= queue
->q_scn
;
3376 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
3378 if (avl_find(&queue
->q_sios_by_addr
, sio
, &idx
) != NULL
) {
3379 /* block is already scheduled for reading */
3380 atomic_add_64(&scn
->scn_bytes_pending
, -asize
);
3381 kmem_cache_free(sio_cache
, sio
);
3384 avl_insert(&queue
->q_sios_by_addr
, sio
, idx
);
3385 range_tree_add(queue
->q_exts_by_addr
, sio
->sio_offset
, asize
);
3389 * Given all the info we got from our metadata scanning process, we
3390 * construct a scan_io_t and insert it into the scan sorting queue. The
3391 * I/O must already be suitable for us to process. This is controlled
3392 * by dsl_scan_enqueue().
3395 scan_io_queue_insert(dsl_scan_io_queue_t
*queue
, const blkptr_t
*bp
, int dva_i
,
3396 int zio_flags
, const zbookmark_phys_t
*zb
)
3398 dsl_scan_t
*scn
= queue
->q_scn
;
3399 scan_io_t
*sio
= kmem_cache_alloc(sio_cache
, KM_SLEEP
);
3401 ASSERT0(BP_IS_GANG(bp
));
3402 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
3404 bp2sio(bp
, sio
, dva_i
);
3405 sio
->sio_flags
= zio_flags
;
3409 * Increment the bytes pending counter now so that we can't
3410 * get an integer underflow in case the worker processes the
3411 * zio before we get to incrementing this counter.
3413 atomic_add_64(&scn
->scn_bytes_pending
, sio
->sio_asize
);
3415 scan_io_queue_insert_impl(queue
, sio
);
3419 * Given a set of I/O parameters as discovered by the metadata traversal
3420 * process, attempts to place the I/O into the sorted queues (if allowed),
3421 * or immediately executes the I/O.
3424 dsl_scan_enqueue(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
3425 const zbookmark_phys_t
*zb
)
3427 spa_t
*spa
= dp
->dp_spa
;
3429 ASSERT(!BP_IS_EMBEDDED(bp
));
3432 * Gang blocks are hard to issue sequentially, so we just issue them
3433 * here immediately instead of queuing them.
3435 if (!dp
->dp_scan
->scn_is_sorted
|| BP_IS_GANG(bp
)) {
3436 scan_exec_io(dp
, bp
, zio_flags
, zb
, NULL
);
3440 for (int i
= 0; i
< BP_GET_NDVAS(bp
); i
++) {
3444 dva
= bp
->blk_dva
[i
];
3445 vdev
= vdev_lookup_top(spa
, DVA_GET_VDEV(&dva
));
3446 ASSERT(vdev
!= NULL
);
3448 mutex_enter(&vdev
->vdev_scan_io_queue_lock
);
3449 if (vdev
->vdev_scan_io_queue
== NULL
)
3450 vdev
->vdev_scan_io_queue
= scan_io_queue_create(vdev
);
3451 ASSERT(dp
->dp_scan
!= NULL
);
3452 scan_io_queue_insert(vdev
->vdev_scan_io_queue
, bp
,
3454 mutex_exit(&vdev
->vdev_scan_io_queue_lock
);
3459 dsl_scan_scrub_cb(dsl_pool_t
*dp
,
3460 const blkptr_t
*bp
, const zbookmark_phys_t
*zb
)
3462 dsl_scan_t
*scn
= dp
->dp_scan
;
3463 spa_t
*spa
= dp
->dp_spa
;
3464 uint64_t phys_birth
= BP_PHYSICAL_BIRTH(bp
);
3465 size_t psize
= BP_GET_PSIZE(bp
);
3466 boolean_t needs_io
= B_FALSE
;
3467 int zio_flags
= ZIO_FLAG_SCAN_THREAD
| ZIO_FLAG_RAW
| ZIO_FLAG_CANFAIL
;
3469 if (phys_birth
<= scn
->scn_phys
.scn_min_txg
||
3470 phys_birth
>= scn
->scn_phys
.scn_max_txg
)
3473 if (BP_IS_EMBEDDED(bp
)) {
3474 count_block(scn
, dp
->dp_blkstats
, bp
);
3478 ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn
));
3479 if (scn
->scn_phys
.scn_func
== POOL_SCAN_SCRUB
) {
3480 zio_flags
|= ZIO_FLAG_SCRUB
;
3483 ASSERT3U(scn
->scn_phys
.scn_func
, ==, POOL_SCAN_RESILVER
);
3484 zio_flags
|= ZIO_FLAG_RESILVER
;
3488 /* If it's an intent log block, failure is expected. */
3489 if (zb
->zb_level
== ZB_ZIL_LEVEL
)
3490 zio_flags
|= ZIO_FLAG_SPECULATIVE
;
3492 for (int d
= 0; d
< BP_GET_NDVAS(bp
); d
++) {
3493 const dva_t
*dva
= &bp
->blk_dva
[d
];
3496 * Keep track of how much data we've examined so that
3497 * zpool(1M) status can make useful progress reports.
3499 scn
->scn_phys
.scn_examined
+= DVA_GET_ASIZE(dva
);
3500 spa
->spa_scan_pass_exam
+= DVA_GET_ASIZE(dva
);
3502 /* if it's a resilver, this may not be in the target range */
3504 needs_io
= dsl_scan_need_resilver(spa
, dva
, psize
,
3508 if (needs_io
&& !zfs_no_scrub_io
) {
3509 dsl_scan_enqueue(dp
, bp
, zio_flags
, zb
);
3511 count_block(scn
, dp
->dp_blkstats
, bp
);
3514 /* do not relocate this block */
3519 dsl_scan_scrub_done(zio_t
*zio
)
3521 spa_t
*spa
= zio
->io_spa
;
3522 blkptr_t
*bp
= zio
->io_bp
;
3523 dsl_scan_io_queue_t
*queue
= zio
->io_private
;
3525 abd_free(zio
->io_abd
);
3527 if (queue
== NULL
) {
3528 mutex_enter(&spa
->spa_scrub_lock
);
3529 ASSERT3U(spa
->spa_scrub_inflight
, >=, BP_GET_PSIZE(bp
));
3530 spa
->spa_scrub_inflight
-= BP_GET_PSIZE(bp
);
3531 cv_broadcast(&spa
->spa_scrub_io_cv
);
3532 mutex_exit(&spa
->spa_scrub_lock
);
3534 mutex_enter(&queue
->q_vd
->vdev_scan_io_queue_lock
);
3535 ASSERT3U(queue
->q_inflight_bytes
, >=, BP_GET_PSIZE(bp
));
3536 queue
->q_inflight_bytes
-= BP_GET_PSIZE(bp
);
3537 cv_broadcast(&queue
->q_zio_cv
);
3538 mutex_exit(&queue
->q_vd
->vdev_scan_io_queue_lock
);
3541 if (zio
->io_error
&& (zio
->io_error
!= ECKSUM
||
3542 !(zio
->io_flags
& ZIO_FLAG_SPECULATIVE
))) {
3543 atomic_inc_64(&spa
->spa_dsl_pool
->dp_scan
->scn_phys
.scn_errors
);
3548 * Given a scanning zio's information, executes the zio. The zio need
3549 * not necessarily be only sortable, this function simply executes the
3550 * zio, no matter what it is. The optional queue argument allows the
3551 * caller to specify that they want per top level vdev IO rate limiting
3552 * instead of the legacy global limiting.
3555 scan_exec_io(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
3556 const zbookmark_phys_t
*zb
, dsl_scan_io_queue_t
*queue
)
3558 spa_t
*spa
= dp
->dp_spa
;
3559 dsl_scan_t
*scn
= dp
->dp_scan
;
3560 size_t size
= BP_GET_PSIZE(bp
);
3561 abd_t
*data
= abd_alloc_for_io(size
, B_FALSE
);
3563 if (queue
== NULL
) {
3564 mutex_enter(&spa
->spa_scrub_lock
);
3565 while (spa
->spa_scrub_inflight
>= scn
->scn_maxinflight_bytes
)
3566 cv_wait(&spa
->spa_scrub_io_cv
, &spa
->spa_scrub_lock
);
3567 spa
->spa_scrub_inflight
+= BP_GET_PSIZE(bp
);
3568 mutex_exit(&spa
->spa_scrub_lock
);
3570 kmutex_t
*q_lock
= &queue
->q_vd
->vdev_scan_io_queue_lock
;
3572 mutex_enter(q_lock
);
3573 while (queue
->q_inflight_bytes
>= queue
->q_maxinflight_bytes
)
3574 cv_wait(&queue
->q_zio_cv
, q_lock
);
3575 queue
->q_inflight_bytes
+= BP_GET_PSIZE(bp
);
3579 count_block(scn
, dp
->dp_blkstats
, bp
);
3580 zio_nowait(zio_read(scn
->scn_zio_root
, spa
, bp
, data
, size
,
3581 dsl_scan_scrub_done
, queue
, ZIO_PRIORITY_SCRUB
, zio_flags
, zb
));
3585 * This is the primary extent sorting algorithm. We balance two parameters:
3586 * 1) how many bytes of I/O are in an extent
3587 * 2) how well the extent is filled with I/O (as a fraction of its total size)
3588 * Since we allow extents to have gaps between their constituent I/Os, it's
3589 * possible to have a fairly large extent that contains the same amount of
3590 * I/O bytes than a much smaller extent, which just packs the I/O more tightly.
3591 * The algorithm sorts based on a score calculated from the extent's size,
3592 * the relative fill volume (in %) and a "fill weight" parameter that controls
3593 * the split between whether we prefer larger extents or more well populated
3596 * SCORE = FILL_IN_BYTES + (FILL_IN_PERCENT * FILL_IN_BYTES * FILL_WEIGHT)
3599 * 1) assume extsz = 64 MiB
3600 * 2) assume fill = 32 MiB (extent is half full)
3601 * 3) assume fill_weight = 3
3602 * 4) SCORE = 32M + (((32M * 100) / 64M) * 3 * 32M) / 100
3603 * SCORE = 32M + (50 * 3 * 32M) / 100
3604 * SCORE = 32M + (4800M / 100)
3607 * | +--- final total relative fill-based score
3608 * +--------- final total fill-based score
3611 * As can be seen, at fill_ratio=3, the algorithm is slightly biased towards
3612 * extents that are more completely filled (in a 3:2 ratio) vs just larger.
3613 * Note that as an optimization, we replace multiplication and division by
3614 * 100 with bitshifting by 7 (which effecitvely multiplies and divides by 128).
3617 ext_size_compare(const void *x
, const void *y
)
3619 const range_seg_t
*rsa
= x
, *rsb
= y
;
3620 uint64_t sa
= rsa
->rs_end
- rsa
->rs_start
,
3621 sb
= rsb
->rs_end
- rsb
->rs_start
;
3622 uint64_t score_a
, score_b
;
3624 score_a
= rsa
->rs_fill
+ ((((rsa
->rs_fill
<< 7) / sa
) *
3625 fill_weight
* rsa
->rs_fill
) >> 7);
3626 score_b
= rsb
->rs_fill
+ ((((rsb
->rs_fill
<< 7) / sb
) *
3627 fill_weight
* rsb
->rs_fill
) >> 7);
3629 if (score_a
> score_b
)
3631 if (score_a
== score_b
) {
3632 if (rsa
->rs_start
< rsb
->rs_start
)
3634 if (rsa
->rs_start
== rsb
->rs_start
)
3642 * Comparator for the q_sios_by_addr tree. Sorting is simply performed
3643 * based on LBA-order (from lowest to highest).
3646 sio_addr_compare(const void *x
, const void *y
)
3648 const scan_io_t
*a
= x
, *b
= y
;
3650 if (a
->sio_offset
< b
->sio_offset
)
3652 if (a
->sio_offset
== b
->sio_offset
)
3657 /* IO queues are created on demand when they are needed. */
3658 static dsl_scan_io_queue_t
*
3659 scan_io_queue_create(vdev_t
*vd
)
3661 dsl_scan_t
*scn
= vd
->vdev_spa
->spa_dsl_pool
->dp_scan
;
3662 dsl_scan_io_queue_t
*q
= kmem_zalloc(sizeof (*q
), KM_SLEEP
);
3666 cv_init(&q
->q_zio_cv
, NULL
, CV_DEFAULT
, NULL
);
3667 q
->q_exts_by_addr
= range_tree_create_impl(&rt_avl_ops
,
3668 &q
->q_exts_by_size
, ext_size_compare
,
3669 &q
->q_vd
->vdev_scan_io_queue_lock
, zfs_scan_max_ext_gap
);
3670 avl_create(&q
->q_sios_by_addr
, sio_addr_compare
,
3671 sizeof (scan_io_t
), offsetof(scan_io_t
, sio_nodes
.sio_addr_node
));
3677 * Destroys a scan queue and all segments and scan_io_t's contained in it.
3678 * No further execution of I/O occurs, anything pending in the queue is
3679 * simply freed without being executed.
3682 dsl_scan_io_queue_destroy(dsl_scan_io_queue_t
*queue
)
3684 dsl_scan_t
*scn
= queue
->q_scn
;
3686 void *cookie
= NULL
;
3687 int64_t bytes_dequeued
= 0;
3689 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
3691 while ((sio
= avl_destroy_nodes(&queue
->q_sios_by_addr
, &cookie
)) !=
3693 ASSERT(range_tree_contains(queue
->q_exts_by_addr
,
3694 sio
->sio_offset
, sio
->sio_asize
));
3695 bytes_dequeued
+= sio
->sio_asize
;
3696 kmem_cache_free(sio_cache
, sio
);
3699 atomic_add_64(&scn
->scn_bytes_pending
, -bytes_dequeued
);
3700 range_tree_vacate(queue
->q_exts_by_addr
, NULL
, queue
);
3701 range_tree_destroy(queue
->q_exts_by_addr
);
3702 avl_destroy(&queue
->q_sios_by_addr
);
3703 cv_destroy(&queue
->q_zio_cv
);
3705 kmem_free(queue
, sizeof (*queue
));
3709 * Properly transfers a dsl_scan_queue_t from `svd' to `tvd'. This is
3710 * called on behalf of vdev_top_transfer when creating or destroying
3711 * a mirror vdev due to zpool attach/detach.
3714 dsl_scan_io_queue_vdev_xfer(vdev_t
*svd
, vdev_t
*tvd
)
3716 mutex_enter(&svd
->vdev_scan_io_queue_lock
);
3717 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
3719 VERIFY3P(tvd
->vdev_scan_io_queue
, ==, NULL
);
3720 tvd
->vdev_scan_io_queue
= svd
->vdev_scan_io_queue
;
3721 svd
->vdev_scan_io_queue
= NULL
;
3722 if (tvd
->vdev_scan_io_queue
!= NULL
) {
3723 tvd
->vdev_scan_io_queue
->q_vd
= tvd
;
3724 range_tree_set_lock(tvd
->vdev_scan_io_queue
->q_exts_by_addr
,
3725 &tvd
->vdev_scan_io_queue_lock
);
3728 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
3729 mutex_exit(&svd
->vdev_scan_io_queue_lock
);
3733 scan_io_queues_destroy(dsl_scan_t
*scn
)
3735 vdev_t
*rvd
= scn
->scn_dp
->dp_spa
->spa_root_vdev
;
3737 for (uint64_t i
= 0; i
< rvd
->vdev_children
; i
++) {
3738 vdev_t
*tvd
= rvd
->vdev_child
[i
];
3740 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
3741 if (tvd
->vdev_scan_io_queue
!= NULL
)
3742 dsl_scan_io_queue_destroy(tvd
->vdev_scan_io_queue
);
3743 tvd
->vdev_scan_io_queue
= NULL
;
3744 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
3749 dsl_scan_freed_dva(spa_t
*spa
, const blkptr_t
*bp
, int dva_i
)
3751 dsl_pool_t
*dp
= spa
->spa_dsl_pool
;
3752 dsl_scan_t
*scn
= dp
->dp_scan
;
3755 dsl_scan_io_queue_t
*queue
;
3756 scan_io_t srch
, *sio
;
3758 uint64_t start
, size
;
3760 vdev
= vdev_lookup_top(spa
, DVA_GET_VDEV(&bp
->blk_dva
[dva_i
]));
3761 ASSERT(vdev
!= NULL
);
3762 q_lock
= &vdev
->vdev_scan_io_queue_lock
;
3763 queue
= vdev
->vdev_scan_io_queue
;
3765 mutex_enter(q_lock
);
3766 if (queue
== NULL
) {
3771 bp2sio(bp
, &srch
, dva_i
);
3772 start
= srch
.sio_offset
;
3773 size
= srch
.sio_asize
;
3776 * We can find the zio in two states:
3777 * 1) Cold, just sitting in the queue of zio's to be issued at
3778 * some point in the future. In this case, all we do is
3779 * remove the zio from the q_sios_by_addr tree, decrement
3780 * its data volume from the containing range_seg_t and
3781 * resort the q_exts_by_size tree to reflect that the
3782 * range_seg_t has lost some of its 'fill'. We don't shorten
3783 * the range_seg_t - this is usually rare enough not to be
3784 * worth the extra hassle of trying keep track of precise
3785 * extent boundaries.
3786 * 2) Hot, where the zio is currently in-flight in
3787 * dsl_scan_issue_ios. In this case, we can't simply
3788 * reach in and stop the in-flight zio's, so we instead
3789 * block the caller. Eventually, dsl_scan_issue_ios will
3790 * be done with issuing the zio's it gathered and will
3793 sio
= avl_find(&queue
->q_sios_by_addr
, &srch
, &idx
);
3795 int64_t asize
= sio
->sio_asize
;
3798 /* Got it while it was cold in the queue */
3799 ASSERT3U(start
, ==, sio
->sio_offset
);
3800 ASSERT3U(size
, ==, asize
);
3801 avl_remove(&queue
->q_sios_by_addr
, sio
);
3803 ASSERT(range_tree_contains(queue
->q_exts_by_addr
, start
, size
));
3804 range_tree_remove_fill(queue
->q_exts_by_addr
, start
, size
);
3807 * We only update scn_bytes_pending in the cold path,
3808 * otherwise it will already have been accounted for as
3809 * part of the zio's execution.
3811 atomic_add_64(&scn
->scn_bytes_pending
, -asize
);
3813 /* count the block as though we issued it */
3814 sio2bp(sio
, &tmpbp
, dva_i
);
3815 count_block(scn
, dp
->dp_blkstats
, &tmpbp
);
3817 kmem_cache_free(sio_cache
, sio
);
3823 * Callback invoked when a zio_free() zio is executing. This needs to be
3824 * intercepted to prevent the zio from deallocating a particular portion
3825 * of disk space and it then getting reallocated and written to, while we
3826 * still have it queued up for processing.
3829 dsl_scan_freed(spa_t
*spa
, const blkptr_t
*bp
)
3831 dsl_pool_t
*dp
= spa
->spa_dsl_pool
;
3832 dsl_scan_t
*scn
= dp
->dp_scan
;
3834 ASSERT(!BP_IS_EMBEDDED(bp
));
3835 ASSERT(scn
!= NULL
);
3836 if (!dsl_scan_is_running(scn
))
3839 for (int i
= 0; i
< BP_GET_NDVAS(bp
); i
++)
3840 dsl_scan_freed_dva(spa
, bp
, i
);
3843 #if defined(_KERNEL) && defined(HAVE_SPL)
3845 module_param(zfs_scan_vdev_limit
, ulong
, 0644);
3846 MODULE_PARM_DESC(zfs_scan_vdev_limit
,
3847 "Max bytes in flight per leaf vdev for scrubs and resilvers");
3849 module_param(zfs_scrub_min_time_ms
, int, 0644);
3850 MODULE_PARM_DESC(zfs_scrub_min_time_ms
, "Min millisecs to scrub per txg");
3852 module_param(zfs_free_min_time_ms
, int, 0644);
3853 MODULE_PARM_DESC(zfs_free_min_time_ms
, "Min millisecs to free per txg");
3855 module_param(zfs_resilver_min_time_ms
, int, 0644);
3856 MODULE_PARM_DESC(zfs_resilver_min_time_ms
, "Min millisecs to resilver per txg");
3858 module_param(zfs_no_scrub_io
, int, 0644);
3859 MODULE_PARM_DESC(zfs_no_scrub_io
, "Set to disable scrub I/O");
3861 module_param(zfs_no_scrub_prefetch
, int, 0644);
3862 MODULE_PARM_DESC(zfs_no_scrub_prefetch
, "Set to disable scrub prefetching");
3865 module_param(zfs_free_max_blocks
, ulong
, 0644);
3866 MODULE_PARM_DESC(zfs_free_max_blocks
, "Max number of blocks freed in one txg");
3868 module_param(zfs_free_bpobj_enabled
, int, 0644);
3869 MODULE_PARM_DESC(zfs_free_bpobj_enabled
, "Enable processing of the free_bpobj");
3871 module_param(zfs_scan_mem_lim_fact
, int, 0644);
3872 MODULE_PARM_DESC(zfs_scan_mem_lim_fact
, "Fraction of RAM for scan hard limit");
3874 module_param(zfs_scan_issue_strategy
, int, 0644);
3875 MODULE_PARM_DESC(zfs_scan_issue_strategy
,
3876 "IO issuing strategy during scrubbing. 0 = default, 1 = LBA, 2 = size");
3878 module_param(zfs_scan_legacy
, int, 0644);
3879 MODULE_PARM_DESC(zfs_scan_legacy
, "Scrub using legacy non-sequential method");
3881 module_param(zfs_scan_checkpoint_intval
, int, 0644);
3882 MODULE_PARM_DESC(zfs_scan_checkpoint_intval
,
3883 "Scan progress on-disk checkpointing interval");
3885 module_param(zfs_scan_mem_lim_soft_fact
, int, 0644);
3886 MODULE_PARM_DESC(zfs_scan_mem_lim_soft_fact
,
3887 "Fraction of hard limit used as soft limit");
3889 module_param(zfs_scan_strict_mem_lim
, int, 0644);
3890 MODULE_PARM_DESC(zfs_scan_strict_mem_lim
,
3891 "Tunable to attempt to reduce lock contention");
3893 module_param(zfs_scan_fill_weight
, int, 0644);
3894 MODULE_PARM_DESC(zfs_scan_fill_weight
,
3895 "Tunable to adjust bias towards more filled segments during scans");