4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2021 by Delphix. All rights reserved.
24 * Copyright 2016 Gary Mills
25 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
26 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
27 * Copyright 2019 Joyent, Inc.
30 #include <sys/dsl_scan.h>
31 #include <sys/dsl_pool.h>
32 #include <sys/dsl_dataset.h>
33 #include <sys/dsl_prop.h>
34 #include <sys/dsl_dir.h>
35 #include <sys/dsl_synctask.h>
36 #include <sys/dnode.h>
37 #include <sys/dmu_tx.h>
38 #include <sys/dmu_objset.h>
40 #include <sys/arc_impl.h>
43 #include <sys/zfs_context.h>
44 #include <sys/fs/zfs.h>
45 #include <sys/zfs_znode.h>
46 #include <sys/spa_impl.h>
47 #include <sys/vdev_impl.h>
48 #include <sys/zil_impl.h>
49 #include <sys/zio_checksum.h>
53 #include <sys/sa_impl.h>
54 #include <sys/zfeature.h>
56 #include <sys/range_tree.h>
59 #include <sys/zfs_vfsops.h>
63 * Grand theory statement on scan queue sorting
65 * Scanning is implemented by recursively traversing all indirection levels
66 * in an object and reading all blocks referenced from said objects. This
67 * results in us approximately traversing the object from lowest logical
68 * offset to the highest. For best performance, we would want the logical
69 * blocks to be physically contiguous. However, this is frequently not the
70 * case with pools given the allocation patterns of copy-on-write filesystems.
71 * So instead, we put the I/Os into a reordering queue and issue them in a
72 * way that will most benefit physical disks (LBA-order).
76 * Ideally, we would want to scan all metadata and queue up all block I/O
77 * prior to starting to issue it, because that allows us to do an optimal
78 * sorting job. This can however consume large amounts of memory. Therefore
79 * we continuously monitor the size of the queues and constrain them to 5%
80 * (zfs_scan_mem_lim_fact) of physmem. If the queues grow larger than this
81 * limit, we clear out a few of the largest extents at the head of the queues
82 * to make room for more scanning. Hopefully, these extents will be fairly
83 * large and contiguous, allowing us to approach sequential I/O throughput
84 * even without a fully sorted tree.
86 * Metadata scanning takes place in dsl_scan_visit(), which is called from
87 * dsl_scan_sync() every spa_sync(). If we have either fully scanned all
88 * metadata on the pool, or we need to make room in memory because our
89 * queues are too large, dsl_scan_visit() is postponed and
90 * scan_io_queues_run() is called from dsl_scan_sync() instead. This implies
91 * that metadata scanning and queued I/O issuing are mutually exclusive. This
92 * allows us to provide maximum sequential I/O throughput for the majority of
93 * I/O's issued since sequential I/O performance is significantly negatively
94 * impacted if it is interleaved with random I/O.
96 * Implementation Notes
98 * One side effect of the queued scanning algorithm is that the scanning code
99 * needs to be notified whenever a block is freed. This is needed to allow
100 * the scanning code to remove these I/Os from the issuing queue. Additionally,
101 * we do not attempt to queue gang blocks to be issued sequentially since this
102 * is very hard to do and would have an extremely limited performance benefit.
103 * Instead, we simply issue gang I/Os as soon as we find them using the legacy
106 * Backwards compatibility
108 * This new algorithm is backwards compatible with the legacy on-disk data
109 * structures (and therefore does not require a new feature flag).
110 * Periodically during scanning (see zfs_scan_checkpoint_intval), the scan
111 * will stop scanning metadata (in logical order) and wait for all outstanding
112 * sorted I/O to complete. Once this is done, we write out a checkpoint
113 * bookmark, indicating that we have scanned everything logically before it.
114 * If the pool is imported on a machine without the new sorting algorithm,
115 * the scan simply resumes from the last checkpoint using the legacy algorithm.
118 typedef int (scan_cb_t
)(dsl_pool_t
*, const blkptr_t
*,
119 const zbookmark_phys_t
*);
121 static scan_cb_t dsl_scan_scrub_cb
;
123 static int scan_ds_queue_compare(const void *a
, const void *b
);
124 static int scan_prefetch_queue_compare(const void *a
, const void *b
);
125 static void scan_ds_queue_clear(dsl_scan_t
*scn
);
126 static void scan_ds_prefetch_queue_clear(dsl_scan_t
*scn
);
127 static boolean_t
scan_ds_queue_contains(dsl_scan_t
*scn
, uint64_t dsobj
,
129 static void scan_ds_queue_insert(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t txg
);
130 static void scan_ds_queue_remove(dsl_scan_t
*scn
, uint64_t dsobj
);
131 static void scan_ds_queue_sync(dsl_scan_t
*scn
, dmu_tx_t
*tx
);
132 static uint64_t dsl_scan_count_data_disks(spa_t
*spa
);
133 static void read_by_block_level(dsl_scan_t
*scn
, zbookmark_phys_t zb
);
135 extern uint_t zfs_vdev_async_write_active_min_dirty_percent
;
136 static int zfs_scan_blkstats
= 0;
139 * 'zpool status' uses bytes processed per pass to report throughput and
140 * estimate time remaining. We define a pass to start when the scanning
141 * phase completes for a sequential resilver. Optionally, this value
142 * may be used to reset the pass statistics every N txgs to provide an
143 * estimated completion time based on currently observed performance.
145 static uint_t zfs_scan_report_txgs
= 0;
148 * By default zfs will check to ensure it is not over the hard memory
149 * limit before each txg. If finer-grained control of this is needed
150 * this value can be set to 1 to enable checking before scanning each
153 static int zfs_scan_strict_mem_lim
= B_FALSE
;
156 * Maximum number of parallelly executed bytes per leaf vdev. We attempt
157 * to strike a balance here between keeping the vdev queues full of I/Os
158 * at all times and not overflowing the queues to cause long latency,
159 * which would cause long txg sync times. No matter what, we will not
160 * overload the drives with I/O, since that is protected by
161 * zfs_vdev_scrub_max_active.
163 static uint64_t zfs_scan_vdev_limit
= 16 << 20;
165 static uint_t zfs_scan_issue_strategy
= 0;
167 /* don't queue & sort zios, go direct */
168 static int zfs_scan_legacy
= B_FALSE
;
169 static uint64_t zfs_scan_max_ext_gap
= 2 << 20; /* in bytes */
172 * fill_weight is non-tunable at runtime, so we copy it at module init from
173 * zfs_scan_fill_weight. Runtime adjustments to zfs_scan_fill_weight would
174 * break queue sorting.
176 static uint_t zfs_scan_fill_weight
= 3;
177 static uint64_t fill_weight
;
179 /* See dsl_scan_should_clear() for details on the memory limit tunables */
180 static const uint64_t zfs_scan_mem_lim_min
= 16 << 20; /* bytes */
181 static const uint64_t zfs_scan_mem_lim_soft_max
= 128 << 20; /* bytes */
184 /* fraction of physmem */
185 static uint_t zfs_scan_mem_lim_fact
= 20;
187 /* fraction of mem lim above */
188 static uint_t zfs_scan_mem_lim_soft_fact
= 20;
190 /* minimum milliseconds to scrub per txg */
191 static uint_t zfs_scrub_min_time_ms
= 1000;
193 /* minimum milliseconds to obsolete per txg */
194 static uint_t zfs_obsolete_min_time_ms
= 500;
196 /* minimum milliseconds to free per txg */
197 static uint_t zfs_free_min_time_ms
= 1000;
199 /* minimum milliseconds to resilver per txg */
200 static uint_t zfs_resilver_min_time_ms
= 3000;
202 static uint_t zfs_scan_checkpoint_intval
= 7200; /* in seconds */
203 int zfs_scan_suspend_progress
= 0; /* set to prevent scans from progressing */
204 static int zfs_no_scrub_io
= B_FALSE
; /* set to disable scrub i/o */
205 static int zfs_no_scrub_prefetch
= B_FALSE
; /* set to disable scrub prefetch */
206 static const ddt_class_t zfs_scrub_ddt_class_max
= DDT_CLASS_DUPLICATE
;
207 /* max number of blocks to free in a single TXG */
208 static uint64_t zfs_async_block_max_blocks
= UINT64_MAX
;
209 /* max number of dedup blocks to free in a single TXG */
210 static uint64_t zfs_max_async_dedup_frees
= 100000;
212 /* set to disable resilver deferring */
213 static int zfs_resilver_disable_defer
= B_FALSE
;
216 * We wait a few txgs after importing a pool to begin scanning so that
217 * the import / mounting code isn't held up by scrub / resilver IO.
218 * Unfortunately, it is a bit difficult to determine exactly how long
219 * this will take since userspace will trigger fs mounts asynchronously
220 * and the kernel will create zvol minors asynchronously. As a result,
221 * the value provided here is a bit arbitrary, but represents a
222 * reasonable estimate of how many txgs it will take to finish fully
225 #define SCAN_IMPORT_WAIT_TXGS 5
227 #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \
228 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
229 (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
232 * Enable/disable the processing of the free_bpobj object.
234 static int zfs_free_bpobj_enabled
= 1;
236 /* Error blocks to be scrubbed in one txg. */
237 static uint_t zfs_scrub_error_blocks_per_txg
= 1 << 12;
239 /* the order has to match pool_scan_type */
240 static scan_cb_t
*scan_funcs
[POOL_SCAN_FUNCS
] = {
242 dsl_scan_scrub_cb
, /* POOL_SCAN_SCRUB */
243 dsl_scan_scrub_cb
, /* POOL_SCAN_RESILVER */
246 /* In core node for the scn->scn_queue. Represents a dataset to be scanned */
254 * This controls what conditions are placed on dsl_scan_sync_state():
255 * SYNC_OPTIONAL) write out scn_phys iff scn_queues_pending == 0
256 * SYNC_MANDATORY) write out scn_phys always. scn_queues_pending must be 0.
257 * SYNC_CACHED) if scn_queues_pending == 0, write out scn_phys. Otherwise
258 * write out the scn_phys_cached version.
259 * See dsl_scan_sync_state for details.
268 * This struct represents the minimum information needed to reconstruct a
269 * zio for sequential scanning. This is useful because many of these will
270 * accumulate in the sequential IO queues before being issued, so saving
271 * memory matters here.
273 typedef struct scan_io
{
274 /* fields from blkptr_t */
275 uint64_t sio_blk_prop
;
276 uint64_t sio_phys_birth
;
278 zio_cksum_t sio_cksum
;
279 uint32_t sio_nr_dvas
;
281 /* fields from zio_t */
283 zbookmark_phys_t sio_zb
;
285 /* members for queue sorting */
287 avl_node_t sio_addr_node
; /* link into issuing queue */
288 list_node_t sio_list_node
; /* link for issuing to disk */
292 * There may be up to SPA_DVAS_PER_BP DVAs here from the bp,
293 * depending on how many were in the original bp. Only the
294 * first DVA is really used for sorting and issuing purposes.
295 * The other DVAs (if provided) simply exist so that the zio
296 * layer can find additional copies to repair from in the
297 * event of an error. This array must go at the end of the
298 * struct to allow this for the variable number of elements.
303 #define SIO_SET_OFFSET(sio, x) DVA_SET_OFFSET(&(sio)->sio_dva[0], x)
304 #define SIO_SET_ASIZE(sio, x) DVA_SET_ASIZE(&(sio)->sio_dva[0], x)
305 #define SIO_GET_OFFSET(sio) DVA_GET_OFFSET(&(sio)->sio_dva[0])
306 #define SIO_GET_ASIZE(sio) DVA_GET_ASIZE(&(sio)->sio_dva[0])
307 #define SIO_GET_END_OFFSET(sio) \
308 (SIO_GET_OFFSET(sio) + SIO_GET_ASIZE(sio))
309 #define SIO_GET_MUSED(sio) \
310 (sizeof (scan_io_t) + ((sio)->sio_nr_dvas * sizeof (dva_t)))
312 struct dsl_scan_io_queue
{
313 dsl_scan_t
*q_scn
; /* associated dsl_scan_t */
314 vdev_t
*q_vd
; /* top-level vdev that this queue represents */
315 zio_t
*q_zio
; /* scn_zio_root child for waiting on IO */
317 /* trees used for sorting I/Os and extents of I/Os */
318 range_tree_t
*q_exts_by_addr
;
319 zfs_btree_t q_exts_by_size
;
320 avl_tree_t q_sios_by_addr
;
321 uint64_t q_sio_memused
;
322 uint64_t q_last_ext_addr
;
324 /* members for zio rate limiting */
325 uint64_t q_maxinflight_bytes
;
326 uint64_t q_inflight_bytes
;
327 kcondvar_t q_zio_cv
; /* used under vd->vdev_scan_io_queue_lock */
329 /* per txg statistics */
330 uint64_t q_total_seg_size_this_txg
;
331 uint64_t q_segs_this_txg
;
332 uint64_t q_total_zio_size_this_txg
;
333 uint64_t q_zios_this_txg
;
336 /* private data for dsl_scan_prefetch_cb() */
337 typedef struct scan_prefetch_ctx
{
338 zfs_refcount_t spc_refcnt
; /* refcount for memory management */
339 dsl_scan_t
*spc_scn
; /* dsl_scan_t for the pool */
340 boolean_t spc_root
; /* is this prefetch for an objset? */
341 uint8_t spc_indblkshift
; /* dn_indblkshift of current dnode */
342 uint16_t spc_datablkszsec
; /* dn_idatablkszsec of current dnode */
343 } scan_prefetch_ctx_t
;
345 /* private data for dsl_scan_prefetch() */
346 typedef struct scan_prefetch_issue_ctx
{
347 avl_node_t spic_avl_node
; /* link into scn->scn_prefetch_queue */
348 scan_prefetch_ctx_t
*spic_spc
; /* spc for the callback */
349 blkptr_t spic_bp
; /* bp to prefetch */
350 zbookmark_phys_t spic_zb
; /* bookmark to prefetch */
351 } scan_prefetch_issue_ctx_t
;
353 static void scan_exec_io(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
354 const zbookmark_phys_t
*zb
, dsl_scan_io_queue_t
*queue
);
355 static void scan_io_queue_insert_impl(dsl_scan_io_queue_t
*queue
,
358 static dsl_scan_io_queue_t
*scan_io_queue_create(vdev_t
*vd
);
359 static void scan_io_queues_destroy(dsl_scan_t
*scn
);
361 static kmem_cache_t
*sio_cache
[SPA_DVAS_PER_BP
];
363 /* sio->sio_nr_dvas must be set so we know which cache to free from */
365 sio_free(scan_io_t
*sio
)
367 ASSERT3U(sio
->sio_nr_dvas
, >, 0);
368 ASSERT3U(sio
->sio_nr_dvas
, <=, SPA_DVAS_PER_BP
);
370 kmem_cache_free(sio_cache
[sio
->sio_nr_dvas
- 1], sio
);
373 /* It is up to the caller to set sio->sio_nr_dvas for freeing */
375 sio_alloc(unsigned short nr_dvas
)
377 ASSERT3U(nr_dvas
, >, 0);
378 ASSERT3U(nr_dvas
, <=, SPA_DVAS_PER_BP
);
380 return (kmem_cache_alloc(sio_cache
[nr_dvas
- 1], KM_SLEEP
));
387 * This is used in ext_size_compare() to weight segments
388 * based on how sparse they are. This cannot be changed
389 * mid-scan and the tree comparison functions don't currently
390 * have a mechanism for passing additional context to the
391 * compare functions. Thus we store this value globally and
392 * we only allow it to be set at module initialization time
394 fill_weight
= zfs_scan_fill_weight
;
396 for (int i
= 0; i
< SPA_DVAS_PER_BP
; i
++) {
399 (void) snprintf(name
, sizeof (name
), "sio_cache_%d", i
);
400 sio_cache
[i
] = kmem_cache_create(name
,
401 (sizeof (scan_io_t
) + ((i
+ 1) * sizeof (dva_t
))),
402 0, NULL
, NULL
, NULL
, NULL
, NULL
, 0);
409 for (int i
= 0; i
< SPA_DVAS_PER_BP
; i
++) {
410 kmem_cache_destroy(sio_cache
[i
]);
414 static inline boolean_t
415 dsl_scan_is_running(const dsl_scan_t
*scn
)
417 return (scn
->scn_phys
.scn_state
== DSS_SCANNING
);
421 dsl_scan_resilvering(dsl_pool_t
*dp
)
423 return (dsl_scan_is_running(dp
->dp_scan
) &&
424 dp
->dp_scan
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
);
428 sio2bp(const scan_io_t
*sio
, blkptr_t
*bp
)
430 memset(bp
, 0, sizeof (*bp
));
431 bp
->blk_prop
= sio
->sio_blk_prop
;
432 BP_SET_PHYSICAL_BIRTH(bp
, sio
->sio_phys_birth
);
433 BP_SET_LOGICAL_BIRTH(bp
, sio
->sio_birth
);
434 bp
->blk_fill
= 1; /* we always only work with data pointers */
435 bp
->blk_cksum
= sio
->sio_cksum
;
437 ASSERT3U(sio
->sio_nr_dvas
, >, 0);
438 ASSERT3U(sio
->sio_nr_dvas
, <=, SPA_DVAS_PER_BP
);
440 memcpy(bp
->blk_dva
, sio
->sio_dva
, sio
->sio_nr_dvas
* sizeof (dva_t
));
444 bp2sio(const blkptr_t
*bp
, scan_io_t
*sio
, int dva_i
)
446 sio
->sio_blk_prop
= bp
->blk_prop
;
447 sio
->sio_phys_birth
= BP_GET_PHYSICAL_BIRTH(bp
);
448 sio
->sio_birth
= BP_GET_LOGICAL_BIRTH(bp
);
449 sio
->sio_cksum
= bp
->blk_cksum
;
450 sio
->sio_nr_dvas
= BP_GET_NDVAS(bp
);
453 * Copy the DVAs to the sio. We need all copies of the block so
454 * that the self healing code can use the alternate copies if the
455 * first is corrupted. We want the DVA at index dva_i to be first
456 * in the sio since this is the primary one that we want to issue.
458 for (int i
= 0, j
= dva_i
; i
< sio
->sio_nr_dvas
; i
++, j
++) {
459 sio
->sio_dva
[i
] = bp
->blk_dva
[j
% sio
->sio_nr_dvas
];
464 dsl_scan_init(dsl_pool_t
*dp
, uint64_t txg
)
468 spa_t
*spa
= dp
->dp_spa
;
471 scn
= dp
->dp_scan
= kmem_zalloc(sizeof (dsl_scan_t
), KM_SLEEP
);
475 * It's possible that we're resuming a scan after a reboot so
476 * make sure that the scan_async_destroying flag is initialized
479 ASSERT(!scn
->scn_async_destroying
);
480 scn
->scn_async_destroying
= spa_feature_is_active(dp
->dp_spa
,
481 SPA_FEATURE_ASYNC_DESTROY
);
484 * Calculate the max number of in-flight bytes for pool-wide
485 * scanning operations (minimum 1MB, maximum 1/4 of arc_c_max).
486 * Limits for the issuing phase are done per top-level vdev and
487 * are handled separately.
489 scn
->scn_maxinflight_bytes
= MIN(arc_c_max
/ 4, MAX(1ULL << 20,
490 zfs_scan_vdev_limit
* dsl_scan_count_data_disks(spa
)));
492 avl_create(&scn
->scn_queue
, scan_ds_queue_compare
, sizeof (scan_ds_t
),
493 offsetof(scan_ds_t
, sds_node
));
494 avl_create(&scn
->scn_prefetch_queue
, scan_prefetch_queue_compare
,
495 sizeof (scan_prefetch_issue_ctx_t
),
496 offsetof(scan_prefetch_issue_ctx_t
, spic_avl_node
));
498 err
= zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
499 "scrub_func", sizeof (uint64_t), 1, &f
);
502 * There was an old-style scrub in progress. Restart a
503 * new-style scrub from the beginning.
505 scn
->scn_restart_txg
= txg
;
506 zfs_dbgmsg("old-style scrub was in progress for %s; "
507 "restarting new-style scrub in txg %llu",
509 (longlong_t
)scn
->scn_restart_txg
);
512 * Load the queue obj from the old location so that it
513 * can be freed by dsl_scan_done().
515 (void) zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
516 "scrub_queue", sizeof (uint64_t), 1,
517 &scn
->scn_phys
.scn_queue_obj
);
519 err
= zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
520 DMU_POOL_ERRORSCRUB
, sizeof (uint64_t),
521 ERRORSCRUB_PHYS_NUMINTS
, &scn
->errorscrub_phys
);
523 if (err
!= 0 && err
!= ENOENT
)
526 err
= zap_lookup(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
527 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
531 * Detect if the pool contains the signature of #2094. If it
532 * does properly update the scn->scn_phys structure and notify
533 * the administrator by setting an errata for the pool.
535 if (err
== EOVERFLOW
) {
536 uint64_t zaptmp
[SCAN_PHYS_NUMINTS
+ 1];
537 VERIFY3S(SCAN_PHYS_NUMINTS
, ==, 24);
538 VERIFY3S(offsetof(dsl_scan_phys_t
, scn_flags
), ==,
539 (23 * sizeof (uint64_t)));
541 err
= zap_lookup(dp
->dp_meta_objset
,
542 DMU_POOL_DIRECTORY_OBJECT
, DMU_POOL_SCAN
,
543 sizeof (uint64_t), SCAN_PHYS_NUMINTS
+ 1, &zaptmp
);
545 uint64_t overflow
= zaptmp
[SCAN_PHYS_NUMINTS
];
547 if (overflow
& ~DSL_SCAN_FLAGS_MASK
||
548 scn
->scn_async_destroying
) {
550 ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY
;
554 memcpy(&scn
->scn_phys
, zaptmp
,
555 SCAN_PHYS_NUMINTS
* sizeof (uint64_t));
556 scn
->scn_phys
.scn_flags
= overflow
;
558 /* Required scrub already in progress. */
559 if (scn
->scn_phys
.scn_state
== DSS_FINISHED
||
560 scn
->scn_phys
.scn_state
== DSS_CANCELED
)
562 ZPOOL_ERRATA_ZOL_2094_SCRUB
;
572 * We might be restarting after a reboot, so jump the issued
573 * counter to how far we've scanned. We know we're consistent
576 scn
->scn_issued_before_pass
= scn
->scn_phys
.scn_examined
-
577 scn
->scn_phys
.scn_skipped
;
579 if (dsl_scan_is_running(scn
) &&
580 spa_prev_software_version(dp
->dp_spa
) < SPA_VERSION_SCAN
) {
582 * A new-type scrub was in progress on an old
583 * pool, and the pool was accessed by old
584 * software. Restart from the beginning, since
585 * the old software may have changed the pool in
588 scn
->scn_restart_txg
= txg
;
589 zfs_dbgmsg("new-style scrub for %s was modified "
590 "by old software; restarting in txg %llu",
592 (longlong_t
)scn
->scn_restart_txg
);
593 } else if (dsl_scan_resilvering(dp
)) {
595 * If a resilver is in progress and there are already
596 * errors, restart it instead of finishing this scan and
597 * then restarting it. If there haven't been any errors
598 * then remember that the incore DTL is valid.
600 if (scn
->scn_phys
.scn_errors
> 0) {
601 scn
->scn_restart_txg
= txg
;
602 zfs_dbgmsg("resilver can't excise DTL_MISSING "
603 "when finished; restarting on %s in txg "
606 (u_longlong_t
)scn
->scn_restart_txg
);
608 /* it's safe to excise DTL when finished */
609 spa
->spa_scrub_started
= B_TRUE
;
614 memcpy(&scn
->scn_phys_cached
, &scn
->scn_phys
, sizeof (scn
->scn_phys
));
616 /* reload the queue into the in-core state */
617 if (scn
->scn_phys
.scn_queue_obj
!= 0) {
621 for (zap_cursor_init(&zc
, dp
->dp_meta_objset
,
622 scn
->scn_phys
.scn_queue_obj
);
623 zap_cursor_retrieve(&zc
, &za
) == 0;
624 (void) zap_cursor_advance(&zc
)) {
625 scan_ds_queue_insert(scn
,
626 zfs_strtonum(za
.za_name
, NULL
),
627 za
.za_first_integer
);
629 zap_cursor_fini(&zc
);
632 spa_scan_stat_init(spa
);
633 vdev_scan_stat_init(spa
->spa_root_vdev
);
639 dsl_scan_fini(dsl_pool_t
*dp
)
641 if (dp
->dp_scan
!= NULL
) {
642 dsl_scan_t
*scn
= dp
->dp_scan
;
644 if (scn
->scn_taskq
!= NULL
)
645 taskq_destroy(scn
->scn_taskq
);
647 scan_ds_queue_clear(scn
);
648 avl_destroy(&scn
->scn_queue
);
649 scan_ds_prefetch_queue_clear(scn
);
650 avl_destroy(&scn
->scn_prefetch_queue
);
652 kmem_free(dp
->dp_scan
, sizeof (dsl_scan_t
));
658 dsl_scan_restarting(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
660 return (scn
->scn_restart_txg
!= 0 &&
661 scn
->scn_restart_txg
<= tx
->tx_txg
);
665 dsl_scan_resilver_scheduled(dsl_pool_t
*dp
)
667 return ((dp
->dp_scan
&& dp
->dp_scan
->scn_restart_txg
!= 0) ||
668 (spa_async_tasks(dp
->dp_spa
) & SPA_ASYNC_RESILVER
));
672 dsl_scan_scrubbing(const dsl_pool_t
*dp
)
674 dsl_scan_phys_t
*scn_phys
= &dp
->dp_scan
->scn_phys
;
676 return (scn_phys
->scn_state
== DSS_SCANNING
&&
677 scn_phys
->scn_func
== POOL_SCAN_SCRUB
);
681 dsl_errorscrubbing(const dsl_pool_t
*dp
)
683 dsl_errorscrub_phys_t
*errorscrub_phys
= &dp
->dp_scan
->errorscrub_phys
;
685 return (errorscrub_phys
->dep_state
== DSS_ERRORSCRUBBING
&&
686 errorscrub_phys
->dep_func
== POOL_SCAN_ERRORSCRUB
);
690 dsl_errorscrub_is_paused(const dsl_scan_t
*scn
)
692 return (dsl_errorscrubbing(scn
->scn_dp
) &&
693 scn
->errorscrub_phys
.dep_paused_flags
);
697 dsl_scan_is_paused_scrub(const dsl_scan_t
*scn
)
699 return (dsl_scan_scrubbing(scn
->scn_dp
) &&
700 scn
->scn_phys
.scn_flags
& DSF_SCRUB_PAUSED
);
704 dsl_errorscrub_sync_state(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
706 scn
->errorscrub_phys
.dep_cursor
=
707 zap_cursor_serialize(&scn
->errorscrub_cursor
);
709 VERIFY0(zap_update(scn
->scn_dp
->dp_meta_objset
,
710 DMU_POOL_DIRECTORY_OBJECT
,
711 DMU_POOL_ERRORSCRUB
, sizeof (uint64_t), ERRORSCRUB_PHYS_NUMINTS
,
712 &scn
->errorscrub_phys
, tx
));
716 dsl_errorscrub_setup_sync(void *arg
, dmu_tx_t
*tx
)
718 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
719 pool_scan_func_t
*funcp
= arg
;
720 dsl_pool_t
*dp
= scn
->scn_dp
;
721 spa_t
*spa
= dp
->dp_spa
;
723 ASSERT(!dsl_scan_is_running(scn
));
724 ASSERT(!dsl_errorscrubbing(scn
->scn_dp
));
725 ASSERT(*funcp
> POOL_SCAN_NONE
&& *funcp
< POOL_SCAN_FUNCS
);
727 memset(&scn
->errorscrub_phys
, 0, sizeof (scn
->errorscrub_phys
));
728 scn
->errorscrub_phys
.dep_func
= *funcp
;
729 scn
->errorscrub_phys
.dep_state
= DSS_ERRORSCRUBBING
;
730 scn
->errorscrub_phys
.dep_start_time
= gethrestime_sec();
731 scn
->errorscrub_phys
.dep_to_examine
= spa_get_last_errlog_size(spa
);
732 scn
->errorscrub_phys
.dep_examined
= 0;
733 scn
->errorscrub_phys
.dep_errors
= 0;
734 scn
->errorscrub_phys
.dep_cursor
= 0;
735 zap_cursor_init_serialized(&scn
->errorscrub_cursor
,
736 spa
->spa_meta_objset
, spa
->spa_errlog_last
,
737 scn
->errorscrub_phys
.dep_cursor
);
739 vdev_config_dirty(spa
->spa_root_vdev
);
740 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_ERRORSCRUB_START
);
742 dsl_errorscrub_sync_state(scn
, tx
);
744 spa_history_log_internal(spa
, "error scrub setup", tx
,
745 "func=%u mintxg=%u maxtxg=%llu",
746 *funcp
, 0, (u_longlong_t
)tx
->tx_txg
);
750 dsl_errorscrub_setup_check(void *arg
, dmu_tx_t
*tx
)
753 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
755 if (dsl_scan_is_running(scn
) || (dsl_errorscrubbing(scn
->scn_dp
))) {
756 return (SET_ERROR(EBUSY
));
759 if (spa_get_last_errlog_size(scn
->scn_dp
->dp_spa
) == 0) {
766 * Writes out a persistent dsl_scan_phys_t record to the pool directory.
767 * Because we can be running in the block sorting algorithm, we do not always
768 * want to write out the record, only when it is "safe" to do so. This safety
769 * condition is achieved by making sure that the sorting queues are empty
770 * (scn_queues_pending == 0). When this condition is not true, the sync'd state
771 * is inconsistent with how much actual scanning progress has been made. The
772 * kind of sync to be performed is specified by the sync_type argument. If the
773 * sync is optional, we only sync if the queues are empty. If the sync is
774 * mandatory, we do a hard ASSERT to make sure that the queues are empty. The
775 * third possible state is a "cached" sync. This is done in response to:
776 * 1) The dataset that was in the last sync'd dsl_scan_phys_t having been
777 * destroyed, so we wouldn't be able to restart scanning from it.
778 * 2) The snapshot that was in the last sync'd dsl_scan_phys_t having been
779 * superseded by a newer snapshot.
780 * 3) The dataset that was in the last sync'd dsl_scan_phys_t having been
781 * swapped with its clone.
782 * In all cases, a cached sync simply rewrites the last record we've written,
783 * just slightly modified. For the modifications that are performed to the
784 * last written dsl_scan_phys_t, see dsl_scan_ds_destroyed,
785 * dsl_scan_ds_snapshotted and dsl_scan_ds_clone_swapped.
788 dsl_scan_sync_state(dsl_scan_t
*scn
, dmu_tx_t
*tx
, state_sync_type_t sync_type
)
791 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
793 ASSERT(sync_type
!= SYNC_MANDATORY
|| scn
->scn_queues_pending
== 0);
794 if (scn
->scn_queues_pending
== 0) {
795 for (i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
796 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
797 dsl_scan_io_queue_t
*q
= vd
->vdev_scan_io_queue
;
802 mutex_enter(&vd
->vdev_scan_io_queue_lock
);
803 ASSERT3P(avl_first(&q
->q_sios_by_addr
), ==, NULL
);
804 ASSERT3P(zfs_btree_first(&q
->q_exts_by_size
, NULL
), ==,
806 ASSERT3P(range_tree_first(q
->q_exts_by_addr
), ==, NULL
);
807 mutex_exit(&vd
->vdev_scan_io_queue_lock
);
810 if (scn
->scn_phys
.scn_queue_obj
!= 0)
811 scan_ds_queue_sync(scn
, tx
);
812 VERIFY0(zap_update(scn
->scn_dp
->dp_meta_objset
,
813 DMU_POOL_DIRECTORY_OBJECT
,
814 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
815 &scn
->scn_phys
, tx
));
816 memcpy(&scn
->scn_phys_cached
, &scn
->scn_phys
,
817 sizeof (scn
->scn_phys
));
819 if (scn
->scn_checkpointing
)
820 zfs_dbgmsg("finish scan checkpoint for %s",
823 scn
->scn_checkpointing
= B_FALSE
;
824 scn
->scn_last_checkpoint
= ddi_get_lbolt();
825 } else if (sync_type
== SYNC_CACHED
) {
826 VERIFY0(zap_update(scn
->scn_dp
->dp_meta_objset
,
827 DMU_POOL_DIRECTORY_OBJECT
,
828 DMU_POOL_SCAN
, sizeof (uint64_t), SCAN_PHYS_NUMINTS
,
829 &scn
->scn_phys_cached
, tx
));
834 dsl_scan_setup_check(void *arg
, dmu_tx_t
*tx
)
837 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
838 vdev_t
*rvd
= scn
->scn_dp
->dp_spa
->spa_root_vdev
;
840 if (dsl_scan_is_running(scn
) || vdev_rebuild_active(rvd
) ||
841 dsl_errorscrubbing(scn
->scn_dp
))
842 return (SET_ERROR(EBUSY
));
848 dsl_scan_setup_sync(void *arg
, dmu_tx_t
*tx
)
851 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
852 pool_scan_func_t
*funcp
= arg
;
853 dmu_object_type_t ot
= 0;
854 dsl_pool_t
*dp
= scn
->scn_dp
;
855 spa_t
*spa
= dp
->dp_spa
;
857 ASSERT(!dsl_scan_is_running(scn
));
858 ASSERT(*funcp
> POOL_SCAN_NONE
&& *funcp
< POOL_SCAN_FUNCS
);
859 memset(&scn
->scn_phys
, 0, sizeof (scn
->scn_phys
));
862 * If we are starting a fresh scrub, we erase the error scrub
863 * information from disk.
865 memset(&scn
->errorscrub_phys
, 0, sizeof (scn
->errorscrub_phys
));
866 dsl_errorscrub_sync_state(scn
, tx
);
868 scn
->scn_phys
.scn_func
= *funcp
;
869 scn
->scn_phys
.scn_state
= DSS_SCANNING
;
870 scn
->scn_phys
.scn_min_txg
= 0;
871 scn
->scn_phys
.scn_max_txg
= tx
->tx_txg
;
872 scn
->scn_phys
.scn_ddt_class_max
= DDT_CLASSES
- 1; /* the entire DDT */
873 scn
->scn_phys
.scn_start_time
= gethrestime_sec();
874 scn
->scn_phys
.scn_errors
= 0;
875 scn
->scn_phys
.scn_to_examine
= spa
->spa_root_vdev
->vdev_stat
.vs_alloc
;
876 scn
->scn_issued_before_pass
= 0;
877 scn
->scn_restart_txg
= 0;
878 scn
->scn_done_txg
= 0;
879 scn
->scn_last_checkpoint
= 0;
880 scn
->scn_checkpointing
= B_FALSE
;
881 spa_scan_stat_init(spa
);
882 vdev_scan_stat_init(spa
->spa_root_vdev
);
884 if (DSL_SCAN_IS_SCRUB_RESILVER(scn
)) {
885 scn
->scn_phys
.scn_ddt_class_max
= zfs_scrub_ddt_class_max
;
887 /* rewrite all disk labels */
888 vdev_config_dirty(spa
->spa_root_vdev
);
890 if (vdev_resilver_needed(spa
->spa_root_vdev
,
891 &scn
->scn_phys
.scn_min_txg
, &scn
->scn_phys
.scn_max_txg
)) {
892 nvlist_t
*aux
= fnvlist_alloc();
893 fnvlist_add_string(aux
, ZFS_EV_RESILVER_TYPE
,
895 spa_event_notify(spa
, NULL
, aux
,
896 ESC_ZFS_RESILVER_START
);
899 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_SCRUB_START
);
902 spa
->spa_scrub_started
= B_TRUE
;
904 * If this is an incremental scrub, limit the DDT scrub phase
905 * to just the auto-ditto class (for correctness); the rest
906 * of the scrub should go faster using top-down pruning.
908 if (scn
->scn_phys
.scn_min_txg
> TXG_INITIAL
)
909 scn
->scn_phys
.scn_ddt_class_max
= DDT_CLASS_DITTO
;
912 * When starting a resilver clear any existing rebuild state.
913 * This is required to prevent stale rebuild status from
914 * being reported when a rebuild is run, then a resilver and
915 * finally a scrub. In which case only the scrub status
916 * should be reported by 'zpool status'.
918 if (scn
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
) {
919 vdev_t
*rvd
= spa
->spa_root_vdev
;
920 for (uint64_t i
= 0; i
< rvd
->vdev_children
; i
++) {
921 vdev_t
*vd
= rvd
->vdev_child
[i
];
922 vdev_rebuild_clear_sync(
923 (void *)(uintptr_t)vd
->vdev_id
, tx
);
928 /* back to the generic stuff */
930 if (zfs_scan_blkstats
) {
931 if (dp
->dp_blkstats
== NULL
) {
933 vmem_alloc(sizeof (zfs_all_blkstats_t
), KM_SLEEP
);
935 memset(&dp
->dp_blkstats
->zab_type
, 0,
936 sizeof (dp
->dp_blkstats
->zab_type
));
938 if (dp
->dp_blkstats
) {
939 vmem_free(dp
->dp_blkstats
, sizeof (zfs_all_blkstats_t
));
940 dp
->dp_blkstats
= NULL
;
944 if (spa_version(spa
) < SPA_VERSION_DSL_SCRUB
)
945 ot
= DMU_OT_ZAP_OTHER
;
947 scn
->scn_phys
.scn_queue_obj
= zap_create(dp
->dp_meta_objset
,
948 ot
? ot
: DMU_OT_SCAN_QUEUE
, DMU_OT_NONE
, 0, tx
);
950 memcpy(&scn
->scn_phys_cached
, &scn
->scn_phys
, sizeof (scn
->scn_phys
));
952 dsl_scan_sync_state(scn
, tx
, SYNC_MANDATORY
);
954 spa_history_log_internal(spa
, "scan setup", tx
,
955 "func=%u mintxg=%llu maxtxg=%llu",
956 *funcp
, (u_longlong_t
)scn
->scn_phys
.scn_min_txg
,
957 (u_longlong_t
)scn
->scn_phys
.scn_max_txg
);
961 * Called by ZFS_IOC_POOL_SCRUB and ZFS_IOC_POOL_SCAN ioctl to start a scrub,
962 * error scrub or resilver. Can also be called to resume a paused scrub or
966 dsl_scan(dsl_pool_t
*dp
, pool_scan_func_t func
)
968 spa_t
*spa
= dp
->dp_spa
;
969 dsl_scan_t
*scn
= dp
->dp_scan
;
972 * Purge all vdev caches and probe all devices. We do this here
973 * rather than in sync context because this requires a writer lock
974 * on the spa_config lock, which we can't do from sync context. The
975 * spa_scrub_reopen flag indicates that vdev_open() should not
976 * attempt to start another scrub.
978 spa_vdev_state_enter(spa
, SCL_NONE
);
979 spa
->spa_scrub_reopen
= B_TRUE
;
980 vdev_reopen(spa
->spa_root_vdev
);
981 spa
->spa_scrub_reopen
= B_FALSE
;
982 (void) spa_vdev_state_exit(spa
, NULL
, 0);
984 if (func
== POOL_SCAN_RESILVER
) {
985 dsl_scan_restart_resilver(spa
->spa_dsl_pool
, 0);
989 if (func
== POOL_SCAN_ERRORSCRUB
) {
990 if (dsl_errorscrub_is_paused(dp
->dp_scan
)) {
992 * got error scrub start cmd, resume paused error scrub.
994 int err
= dsl_scrub_set_pause_resume(scn
->scn_dp
,
997 spa_event_notify(spa
, NULL
, NULL
,
998 ESC_ZFS_ERRORSCRUB_RESUME
);
1001 return (SET_ERROR(err
));
1004 return (dsl_sync_task(spa_name(dp
->dp_spa
),
1005 dsl_errorscrub_setup_check
, dsl_errorscrub_setup_sync
,
1006 &func
, 0, ZFS_SPACE_CHECK_RESERVED
));
1009 if (func
== POOL_SCAN_SCRUB
&& dsl_scan_is_paused_scrub(scn
)) {
1010 /* got scrub start cmd, resume paused scrub */
1011 int err
= dsl_scrub_set_pause_resume(scn
->scn_dp
,
1014 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_SCRUB_RESUME
);
1015 return (SET_ERROR(ECANCELED
));
1017 return (SET_ERROR(err
));
1020 return (dsl_sync_task(spa_name(spa
), dsl_scan_setup_check
,
1021 dsl_scan_setup_sync
, &func
, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED
));
1025 dsl_errorscrub_done(dsl_scan_t
*scn
, boolean_t complete
, dmu_tx_t
*tx
)
1027 dsl_pool_t
*dp
= scn
->scn_dp
;
1028 spa_t
*spa
= dp
->dp_spa
;
1031 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_ERRORSCRUB_FINISH
);
1032 spa_history_log_internal(spa
, "error scrub done", tx
,
1033 "errors=%llu", (u_longlong_t
)spa_approx_errlog_size(spa
));
1035 spa_history_log_internal(spa
, "error scrub canceled", tx
,
1036 "errors=%llu", (u_longlong_t
)spa_approx_errlog_size(spa
));
1039 scn
->errorscrub_phys
.dep_state
= complete
? DSS_FINISHED
: DSS_CANCELED
;
1040 spa
->spa_scrub_active
= B_FALSE
;
1041 spa_errlog_rotate(spa
);
1042 scn
->errorscrub_phys
.dep_end_time
= gethrestime_sec();
1043 zap_cursor_fini(&scn
->errorscrub_cursor
);
1045 if (spa
->spa_errata
== ZPOOL_ERRATA_ZOL_2094_SCRUB
)
1046 spa
->spa_errata
= 0;
1048 ASSERT(!dsl_errorscrubbing(scn
->scn_dp
));
1052 dsl_scan_done(dsl_scan_t
*scn
, boolean_t complete
, dmu_tx_t
*tx
)
1054 static const char *old_names
[] = {
1056 "scrub_ddt_bookmark",
1057 "scrub_ddt_class_max",
1066 dsl_pool_t
*dp
= scn
->scn_dp
;
1067 spa_t
*spa
= dp
->dp_spa
;
1070 /* Remove any remnants of an old-style scrub. */
1071 for (i
= 0; old_names
[i
]; i
++) {
1072 (void) zap_remove(dp
->dp_meta_objset
,
1073 DMU_POOL_DIRECTORY_OBJECT
, old_names
[i
], tx
);
1076 if (scn
->scn_phys
.scn_queue_obj
!= 0) {
1077 VERIFY0(dmu_object_free(dp
->dp_meta_objset
,
1078 scn
->scn_phys
.scn_queue_obj
, tx
));
1079 scn
->scn_phys
.scn_queue_obj
= 0;
1081 scan_ds_queue_clear(scn
);
1082 scan_ds_prefetch_queue_clear(scn
);
1084 scn
->scn_phys
.scn_flags
&= ~DSF_SCRUB_PAUSED
;
1087 * If we were "restarted" from a stopped state, don't bother
1088 * with anything else.
1090 if (!dsl_scan_is_running(scn
)) {
1091 ASSERT(!scn
->scn_is_sorted
);
1095 if (scn
->scn_is_sorted
) {
1096 scan_io_queues_destroy(scn
);
1097 scn
->scn_is_sorted
= B_FALSE
;
1099 if (scn
->scn_taskq
!= NULL
) {
1100 taskq_destroy(scn
->scn_taskq
);
1101 scn
->scn_taskq
= NULL
;
1105 scn
->scn_phys
.scn_state
= complete
? DSS_FINISHED
: DSS_CANCELED
;
1107 spa_notify_waiters(spa
);
1109 if (dsl_scan_restarting(scn
, tx
))
1110 spa_history_log_internal(spa
, "scan aborted, restarting", tx
,
1111 "errors=%llu", (u_longlong_t
)spa_approx_errlog_size(spa
));
1113 spa_history_log_internal(spa
, "scan cancelled", tx
,
1114 "errors=%llu", (u_longlong_t
)spa_approx_errlog_size(spa
));
1116 spa_history_log_internal(spa
, "scan done", tx
,
1117 "errors=%llu", (u_longlong_t
)spa_approx_errlog_size(spa
));
1119 if (DSL_SCAN_IS_SCRUB_RESILVER(scn
)) {
1120 spa
->spa_scrub_active
= B_FALSE
;
1123 * If the scrub/resilver completed, update all DTLs to
1124 * reflect this. Whether it succeeded or not, vacate
1125 * all temporary scrub DTLs.
1127 * As the scrub does not currently support traversing
1128 * data that have been freed but are part of a checkpoint,
1129 * we don't mark the scrub as done in the DTLs as faults
1130 * may still exist in those vdevs.
1133 !spa_feature_is_active(spa
, SPA_FEATURE_POOL_CHECKPOINT
)) {
1134 vdev_dtl_reassess(spa
->spa_root_vdev
, tx
->tx_txg
,
1135 scn
->scn_phys
.scn_max_txg
, B_TRUE
, B_FALSE
);
1137 if (scn
->scn_phys
.scn_min_txg
) {
1138 nvlist_t
*aux
= fnvlist_alloc();
1139 fnvlist_add_string(aux
, ZFS_EV_RESILVER_TYPE
,
1141 spa_event_notify(spa
, NULL
, aux
,
1142 ESC_ZFS_RESILVER_FINISH
);
1145 spa_event_notify(spa
, NULL
, NULL
,
1146 ESC_ZFS_SCRUB_FINISH
);
1149 vdev_dtl_reassess(spa
->spa_root_vdev
, tx
->tx_txg
,
1150 0, B_TRUE
, B_FALSE
);
1152 spa_errlog_rotate(spa
);
1155 * Don't clear flag until after vdev_dtl_reassess to ensure that
1156 * DTL_MISSING will get updated when possible.
1158 spa
->spa_scrub_started
= B_FALSE
;
1161 * We may have finished replacing a device.
1162 * Let the async thread assess this and handle the detach.
1164 spa_async_request(spa
, SPA_ASYNC_RESILVER_DONE
);
1167 * Clear any resilver_deferred flags in the config.
1168 * If there are drives that need resilvering, kick
1169 * off an asynchronous request to start resilver.
1170 * vdev_clear_resilver_deferred() may update the config
1171 * before the resilver can restart. In the event of
1172 * a crash during this period, the spa loading code
1173 * will find the drives that need to be resilvered
1174 * and start the resilver then.
1176 if (spa_feature_is_enabled(spa
, SPA_FEATURE_RESILVER_DEFER
) &&
1177 vdev_clear_resilver_deferred(spa
->spa_root_vdev
, tx
)) {
1178 spa_history_log_internal(spa
,
1179 "starting deferred resilver", tx
, "errors=%llu",
1180 (u_longlong_t
)spa_approx_errlog_size(spa
));
1181 spa_async_request(spa
, SPA_ASYNC_RESILVER
);
1184 /* Clear recent error events (i.e. duplicate events tracking) */
1186 zfs_ereport_clear(spa
, NULL
);
1189 scn
->scn_phys
.scn_end_time
= gethrestime_sec();
1191 if (spa
->spa_errata
== ZPOOL_ERRATA_ZOL_2094_SCRUB
)
1192 spa
->spa_errata
= 0;
1194 ASSERT(!dsl_scan_is_running(scn
));
1198 dsl_errorscrub_pause_resume_check(void *arg
, dmu_tx_t
*tx
)
1200 pool_scrub_cmd_t
*cmd
= arg
;
1201 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
1202 dsl_scan_t
*scn
= dp
->dp_scan
;
1204 if (*cmd
== POOL_SCRUB_PAUSE
) {
1206 * can't pause a error scrub when there is no in-progress
1209 if (!dsl_errorscrubbing(dp
))
1210 return (SET_ERROR(ENOENT
));
1212 /* can't pause a paused error scrub */
1213 if (dsl_errorscrub_is_paused(scn
))
1214 return (SET_ERROR(EBUSY
));
1215 } else if (*cmd
!= POOL_SCRUB_NORMAL
) {
1216 return (SET_ERROR(ENOTSUP
));
1223 dsl_errorscrub_pause_resume_sync(void *arg
, dmu_tx_t
*tx
)
1225 pool_scrub_cmd_t
*cmd
= arg
;
1226 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
1227 spa_t
*spa
= dp
->dp_spa
;
1228 dsl_scan_t
*scn
= dp
->dp_scan
;
1230 if (*cmd
== POOL_SCRUB_PAUSE
) {
1231 spa
->spa_scan_pass_errorscrub_pause
= gethrestime_sec();
1232 scn
->errorscrub_phys
.dep_paused_flags
= B_TRUE
;
1233 dsl_errorscrub_sync_state(scn
, tx
);
1234 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_ERRORSCRUB_PAUSED
);
1236 ASSERT3U(*cmd
, ==, POOL_SCRUB_NORMAL
);
1237 if (dsl_errorscrub_is_paused(scn
)) {
1239 * We need to keep track of how much time we spend
1240 * paused per pass so that we can adjust the error scrub
1241 * rate shown in the output of 'zpool status'.
1243 spa
->spa_scan_pass_errorscrub_spent_paused
+=
1245 spa
->spa_scan_pass_errorscrub_pause
;
1247 spa
->spa_scan_pass_errorscrub_pause
= 0;
1248 scn
->errorscrub_phys
.dep_paused_flags
= B_FALSE
;
1250 zap_cursor_init_serialized(
1251 &scn
->errorscrub_cursor
,
1252 spa
->spa_meta_objset
, spa
->spa_errlog_last
,
1253 scn
->errorscrub_phys
.dep_cursor
);
1255 dsl_errorscrub_sync_state(scn
, tx
);
1261 dsl_errorscrub_cancel_check(void *arg
, dmu_tx_t
*tx
)
1264 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
1265 /* can't cancel a error scrub when there is no one in-progress */
1266 if (!dsl_errorscrubbing(scn
->scn_dp
))
1267 return (SET_ERROR(ENOENT
));
1272 dsl_errorscrub_cancel_sync(void *arg
, dmu_tx_t
*tx
)
1275 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
1277 dsl_errorscrub_done(scn
, B_FALSE
, tx
);
1278 dsl_errorscrub_sync_state(scn
, tx
);
1279 spa_event_notify(scn
->scn_dp
->dp_spa
, NULL
, NULL
,
1280 ESC_ZFS_ERRORSCRUB_ABORT
);
1284 dsl_scan_cancel_check(void *arg
, dmu_tx_t
*tx
)
1287 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
1289 if (!dsl_scan_is_running(scn
))
1290 return (SET_ERROR(ENOENT
));
1295 dsl_scan_cancel_sync(void *arg
, dmu_tx_t
*tx
)
1298 dsl_scan_t
*scn
= dmu_tx_pool(tx
)->dp_scan
;
1300 dsl_scan_done(scn
, B_FALSE
, tx
);
1301 dsl_scan_sync_state(scn
, tx
, SYNC_MANDATORY
);
1302 spa_event_notify(scn
->scn_dp
->dp_spa
, NULL
, NULL
, ESC_ZFS_SCRUB_ABORT
);
1306 dsl_scan_cancel(dsl_pool_t
*dp
)
1308 if (dsl_errorscrubbing(dp
)) {
1309 return (dsl_sync_task(spa_name(dp
->dp_spa
),
1310 dsl_errorscrub_cancel_check
, dsl_errorscrub_cancel_sync
,
1311 NULL
, 3, ZFS_SPACE_CHECK_RESERVED
));
1313 return (dsl_sync_task(spa_name(dp
->dp_spa
), dsl_scan_cancel_check
,
1314 dsl_scan_cancel_sync
, NULL
, 3, ZFS_SPACE_CHECK_RESERVED
));
1318 dsl_scrub_pause_resume_check(void *arg
, dmu_tx_t
*tx
)
1320 pool_scrub_cmd_t
*cmd
= arg
;
1321 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
1322 dsl_scan_t
*scn
= dp
->dp_scan
;
1324 if (*cmd
== POOL_SCRUB_PAUSE
) {
1325 /* can't pause a scrub when there is no in-progress scrub */
1326 if (!dsl_scan_scrubbing(dp
))
1327 return (SET_ERROR(ENOENT
));
1329 /* can't pause a paused scrub */
1330 if (dsl_scan_is_paused_scrub(scn
))
1331 return (SET_ERROR(EBUSY
));
1332 } else if (*cmd
!= POOL_SCRUB_NORMAL
) {
1333 return (SET_ERROR(ENOTSUP
));
1340 dsl_scrub_pause_resume_sync(void *arg
, dmu_tx_t
*tx
)
1342 pool_scrub_cmd_t
*cmd
= arg
;
1343 dsl_pool_t
*dp
= dmu_tx_pool(tx
);
1344 spa_t
*spa
= dp
->dp_spa
;
1345 dsl_scan_t
*scn
= dp
->dp_scan
;
1347 if (*cmd
== POOL_SCRUB_PAUSE
) {
1348 /* can't pause a scrub when there is no in-progress scrub */
1349 spa
->spa_scan_pass_scrub_pause
= gethrestime_sec();
1350 scn
->scn_phys
.scn_flags
|= DSF_SCRUB_PAUSED
;
1351 scn
->scn_phys_cached
.scn_flags
|= DSF_SCRUB_PAUSED
;
1352 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
1353 spa_event_notify(spa
, NULL
, NULL
, ESC_ZFS_SCRUB_PAUSED
);
1354 spa_notify_waiters(spa
);
1356 ASSERT3U(*cmd
, ==, POOL_SCRUB_NORMAL
);
1357 if (dsl_scan_is_paused_scrub(scn
)) {
1359 * We need to keep track of how much time we spend
1360 * paused per pass so that we can adjust the scrub rate
1361 * shown in the output of 'zpool status'
1363 spa
->spa_scan_pass_scrub_spent_paused
+=
1364 gethrestime_sec() - spa
->spa_scan_pass_scrub_pause
;
1365 spa
->spa_scan_pass_scrub_pause
= 0;
1366 scn
->scn_phys
.scn_flags
&= ~DSF_SCRUB_PAUSED
;
1367 scn
->scn_phys_cached
.scn_flags
&= ~DSF_SCRUB_PAUSED
;
1368 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
1374 * Set scrub pause/resume state if it makes sense to do so
1377 dsl_scrub_set_pause_resume(const dsl_pool_t
*dp
, pool_scrub_cmd_t cmd
)
1379 if (dsl_errorscrubbing(dp
)) {
1380 return (dsl_sync_task(spa_name(dp
->dp_spa
),
1381 dsl_errorscrub_pause_resume_check
,
1382 dsl_errorscrub_pause_resume_sync
, &cmd
, 3,
1383 ZFS_SPACE_CHECK_RESERVED
));
1385 return (dsl_sync_task(spa_name(dp
->dp_spa
),
1386 dsl_scrub_pause_resume_check
, dsl_scrub_pause_resume_sync
, &cmd
, 3,
1387 ZFS_SPACE_CHECK_RESERVED
));
1391 /* start a new scan, or restart an existing one. */
1393 dsl_scan_restart_resilver(dsl_pool_t
*dp
, uint64_t txg
)
1397 tx
= dmu_tx_create_dd(dp
->dp_mos_dir
);
1398 VERIFY(0 == dmu_tx_assign(tx
, TXG_WAIT
));
1400 txg
= dmu_tx_get_txg(tx
);
1401 dp
->dp_scan
->scn_restart_txg
= txg
;
1404 dp
->dp_scan
->scn_restart_txg
= txg
;
1406 zfs_dbgmsg("restarting resilver for %s at txg=%llu",
1407 dp
->dp_spa
->spa_name
, (longlong_t
)txg
);
1411 dsl_free(dsl_pool_t
*dp
, uint64_t txg
, const blkptr_t
*bp
)
1413 zio_free(dp
->dp_spa
, txg
, bp
);
1417 dsl_free_sync(zio_t
*pio
, dsl_pool_t
*dp
, uint64_t txg
, const blkptr_t
*bpp
)
1419 ASSERT(dsl_pool_sync_context(dp
));
1420 zio_nowait(zio_free_sync(pio
, dp
->dp_spa
, txg
, bpp
, pio
->io_flags
));
1424 scan_ds_queue_compare(const void *a
, const void *b
)
1426 const scan_ds_t
*sds_a
= a
, *sds_b
= b
;
1428 if (sds_a
->sds_dsobj
< sds_b
->sds_dsobj
)
1430 if (sds_a
->sds_dsobj
== sds_b
->sds_dsobj
)
1436 scan_ds_queue_clear(dsl_scan_t
*scn
)
1438 void *cookie
= NULL
;
1440 while ((sds
= avl_destroy_nodes(&scn
->scn_queue
, &cookie
)) != NULL
) {
1441 kmem_free(sds
, sizeof (*sds
));
1446 scan_ds_queue_contains(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t *txg
)
1448 scan_ds_t srch
, *sds
;
1450 srch
.sds_dsobj
= dsobj
;
1451 sds
= avl_find(&scn
->scn_queue
, &srch
, NULL
);
1452 if (sds
!= NULL
&& txg
!= NULL
)
1453 *txg
= sds
->sds_txg
;
1454 return (sds
!= NULL
);
1458 scan_ds_queue_insert(dsl_scan_t
*scn
, uint64_t dsobj
, uint64_t txg
)
1463 sds
= kmem_zalloc(sizeof (*sds
), KM_SLEEP
);
1464 sds
->sds_dsobj
= dsobj
;
1467 VERIFY3P(avl_find(&scn
->scn_queue
, sds
, &where
), ==, NULL
);
1468 avl_insert(&scn
->scn_queue
, sds
, where
);
1472 scan_ds_queue_remove(dsl_scan_t
*scn
, uint64_t dsobj
)
1474 scan_ds_t srch
, *sds
;
1476 srch
.sds_dsobj
= dsobj
;
1478 sds
= avl_find(&scn
->scn_queue
, &srch
, NULL
);
1479 VERIFY(sds
!= NULL
);
1480 avl_remove(&scn
->scn_queue
, sds
);
1481 kmem_free(sds
, sizeof (*sds
));
1485 scan_ds_queue_sync(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
1487 dsl_pool_t
*dp
= scn
->scn_dp
;
1488 spa_t
*spa
= dp
->dp_spa
;
1489 dmu_object_type_t ot
= (spa_version(spa
) >= SPA_VERSION_DSL_SCRUB
) ?
1490 DMU_OT_SCAN_QUEUE
: DMU_OT_ZAP_OTHER
;
1492 ASSERT0(scn
->scn_queues_pending
);
1493 ASSERT(scn
->scn_phys
.scn_queue_obj
!= 0);
1495 VERIFY0(dmu_object_free(dp
->dp_meta_objset
,
1496 scn
->scn_phys
.scn_queue_obj
, tx
));
1497 scn
->scn_phys
.scn_queue_obj
= zap_create(dp
->dp_meta_objset
, ot
,
1498 DMU_OT_NONE
, 0, tx
);
1499 for (scan_ds_t
*sds
= avl_first(&scn
->scn_queue
);
1500 sds
!= NULL
; sds
= AVL_NEXT(&scn
->scn_queue
, sds
)) {
1501 VERIFY0(zap_add_int_key(dp
->dp_meta_objset
,
1502 scn
->scn_phys
.scn_queue_obj
, sds
->sds_dsobj
,
1508 * Computes the memory limit state that we're currently in. A sorted scan
1509 * needs quite a bit of memory to hold the sorting queue, so we need to
1510 * reasonably constrain the size so it doesn't impact overall system
1511 * performance. We compute two limits:
1512 * 1) Hard memory limit: if the amount of memory used by the sorting
1513 * queues on a pool gets above this value, we stop the metadata
1514 * scanning portion and start issuing the queued up and sorted
1515 * I/Os to reduce memory usage.
1516 * This limit is calculated as a fraction of physmem (by default 5%).
1517 * We constrain the lower bound of the hard limit to an absolute
1518 * minimum of zfs_scan_mem_lim_min (default: 16 MiB). We also constrain
1519 * the upper bound to 5% of the total pool size - no chance we'll
1520 * ever need that much memory, but just to keep the value in check.
1521 * 2) Soft memory limit: once we hit the hard memory limit, we start
1522 * issuing I/O to reduce queue memory usage, but we don't want to
1523 * completely empty out the queues, since we might be able to find I/Os
1524 * that will fill in the gaps of our non-sequential IOs at some point
1525 * in the future. So we stop the issuing of I/Os once the amount of
1526 * memory used drops below the soft limit (at which point we stop issuing
1527 * I/O and start scanning metadata again).
1529 * This limit is calculated by subtracting a fraction of the hard
1530 * limit from the hard limit. By default this fraction is 5%, so
1531 * the soft limit is 95% of the hard limit. We cap the size of the
1532 * difference between the hard and soft limits at an absolute
1533 * maximum of zfs_scan_mem_lim_soft_max (default: 128 MiB) - this is
1534 * sufficient to not cause too frequent switching between the
1535 * metadata scan and I/O issue (even at 2k recordsize, 128 MiB's
1536 * worth of queues is about 1.2 GiB of on-pool data, so scanning
1537 * that should take at least a decent fraction of a second).
1540 dsl_scan_should_clear(dsl_scan_t
*scn
)
1542 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1543 vdev_t
*rvd
= scn
->scn_dp
->dp_spa
->spa_root_vdev
;
1544 uint64_t alloc
, mlim_hard
, mlim_soft
, mused
;
1546 alloc
= metaslab_class_get_alloc(spa_normal_class(spa
));
1547 alloc
+= metaslab_class_get_alloc(spa_special_class(spa
));
1548 alloc
+= metaslab_class_get_alloc(spa_dedup_class(spa
));
1550 mlim_hard
= MAX((physmem
/ zfs_scan_mem_lim_fact
) * PAGESIZE
,
1551 zfs_scan_mem_lim_min
);
1552 mlim_hard
= MIN(mlim_hard
, alloc
/ 20);
1553 mlim_soft
= mlim_hard
- MIN(mlim_hard
/ zfs_scan_mem_lim_soft_fact
,
1554 zfs_scan_mem_lim_soft_max
);
1556 for (uint64_t i
= 0; i
< rvd
->vdev_children
; i
++) {
1557 vdev_t
*tvd
= rvd
->vdev_child
[i
];
1558 dsl_scan_io_queue_t
*queue
;
1560 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
1561 queue
= tvd
->vdev_scan_io_queue
;
1562 if (queue
!= NULL
) {
1564 * # of extents in exts_by_addr = # in exts_by_size.
1565 * B-tree efficiency is ~75%, but can be as low as 50%.
1567 mused
+= zfs_btree_numnodes(&queue
->q_exts_by_size
) *
1568 ((sizeof (range_seg_gap_t
) + sizeof (uint64_t)) *
1569 3 / 2) + queue
->q_sio_memused
;
1571 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
1574 dprintf("current scan memory usage: %llu bytes\n", (longlong_t
)mused
);
1577 ASSERT0(scn
->scn_queues_pending
);
1580 * If we are above our hard limit, we need to clear out memory.
1581 * If we are below our soft limit, we need to accumulate sequential IOs.
1582 * Otherwise, we should keep doing whatever we are currently doing.
1584 if (mused
>= mlim_hard
)
1586 else if (mused
< mlim_soft
)
1589 return (scn
->scn_clearing
);
1593 dsl_scan_check_suspend(dsl_scan_t
*scn
, const zbookmark_phys_t
*zb
)
1595 /* we never skip user/group accounting objects */
1596 if (zb
&& (int64_t)zb
->zb_object
< 0)
1599 if (scn
->scn_suspending
)
1600 return (B_TRUE
); /* we're already suspending */
1602 if (!ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
))
1603 return (B_FALSE
); /* we're resuming */
1605 /* We only know how to resume from level-0 and objset blocks. */
1606 if (zb
&& (zb
->zb_level
!= 0 && zb
->zb_level
!= ZB_ROOT_LEVEL
))
1611 * - we have scanned for at least the minimum time (default 1 sec
1612 * for scrub, 3 sec for resilver), and either we have sufficient
1613 * dirty data that we are starting to write more quickly
1614 * (default 30%), someone is explicitly waiting for this txg
1615 * to complete, or we have used up all of the time in the txg
1616 * timeout (default 5 sec).
1618 * - the spa is shutting down because this pool is being exported
1619 * or the machine is rebooting.
1621 * - the scan queue has reached its memory use limit
1623 uint64_t curr_time_ns
= gethrtime();
1624 uint64_t scan_time_ns
= curr_time_ns
- scn
->scn_sync_start_time
;
1625 uint64_t sync_time_ns
= curr_time_ns
-
1626 scn
->scn_dp
->dp_spa
->spa_sync_starttime
;
1627 uint64_t dirty_min_bytes
= zfs_dirty_data_max
*
1628 zfs_vdev_async_write_active_min_dirty_percent
/ 100;
1629 uint_t mintime
= (scn
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
) ?
1630 zfs_resilver_min_time_ms
: zfs_scrub_min_time_ms
;
1632 if ((NSEC2MSEC(scan_time_ns
) > mintime
&&
1633 (scn
->scn_dp
->dp_dirty_total
>= dirty_min_bytes
||
1634 txg_sync_waiting(scn
->scn_dp
) ||
1635 NSEC2SEC(sync_time_ns
) >= zfs_txg_timeout
)) ||
1636 spa_shutting_down(scn
->scn_dp
->dp_spa
) ||
1637 (zfs_scan_strict_mem_lim
&& dsl_scan_should_clear(scn
))) {
1638 if (zb
&& zb
->zb_level
== ZB_ROOT_LEVEL
) {
1639 dprintf("suspending at first available bookmark "
1640 "%llx/%llx/%llx/%llx\n",
1641 (longlong_t
)zb
->zb_objset
,
1642 (longlong_t
)zb
->zb_object
,
1643 (longlong_t
)zb
->zb_level
,
1644 (longlong_t
)zb
->zb_blkid
);
1645 SET_BOOKMARK(&scn
->scn_phys
.scn_bookmark
,
1646 zb
->zb_objset
, 0, 0, 0);
1647 } else if (zb
!= NULL
) {
1648 dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n",
1649 (longlong_t
)zb
->zb_objset
,
1650 (longlong_t
)zb
->zb_object
,
1651 (longlong_t
)zb
->zb_level
,
1652 (longlong_t
)zb
->zb_blkid
);
1653 scn
->scn_phys
.scn_bookmark
= *zb
;
1656 dsl_scan_phys_t
*scnp
= &scn
->scn_phys
;
1657 dprintf("suspending at at DDT bookmark "
1658 "%llx/%llx/%llx/%llx\n",
1659 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_class
,
1660 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_type
,
1661 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_checksum
,
1662 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_cursor
);
1665 scn
->scn_suspending
= B_TRUE
;
1672 dsl_error_scrub_check_suspend(dsl_scan_t
*scn
, const zbookmark_phys_t
*zb
)
1676 * - we have scrubbed for at least the minimum time (default 1 sec
1677 * for error scrub), someone is explicitly waiting for this txg
1678 * to complete, or we have used up all of the time in the txg
1679 * timeout (default 5 sec).
1681 * - the spa is shutting down because this pool is being exported
1682 * or the machine is rebooting.
1684 uint64_t curr_time_ns
= gethrtime();
1685 uint64_t error_scrub_time_ns
= curr_time_ns
- scn
->scn_sync_start_time
;
1686 uint64_t sync_time_ns
= curr_time_ns
-
1687 scn
->scn_dp
->dp_spa
->spa_sync_starttime
;
1688 int mintime
= zfs_scrub_min_time_ms
;
1690 if ((NSEC2MSEC(error_scrub_time_ns
) > mintime
&&
1691 (txg_sync_waiting(scn
->scn_dp
) ||
1692 NSEC2SEC(sync_time_ns
) >= zfs_txg_timeout
)) ||
1693 spa_shutting_down(scn
->scn_dp
->dp_spa
)) {
1695 dprintf("error scrub suspending at bookmark "
1696 "%llx/%llx/%llx/%llx\n",
1697 (longlong_t
)zb
->zb_objset
,
1698 (longlong_t
)zb
->zb_object
,
1699 (longlong_t
)zb
->zb_level
,
1700 (longlong_t
)zb
->zb_blkid
);
1707 typedef struct zil_scan_arg
{
1709 zil_header_t
*zsa_zh
;
1713 dsl_scan_zil_block(zilog_t
*zilog
, const blkptr_t
*bp
, void *arg
,
1717 zil_scan_arg_t
*zsa
= arg
;
1718 dsl_pool_t
*dp
= zsa
->zsa_dp
;
1719 dsl_scan_t
*scn
= dp
->dp_scan
;
1720 zil_header_t
*zh
= zsa
->zsa_zh
;
1721 zbookmark_phys_t zb
;
1723 ASSERT(!BP_IS_REDACTED(bp
));
1724 if (BP_IS_HOLE(bp
) ||
1725 BP_GET_LOGICAL_BIRTH(bp
) <= scn
->scn_phys
.scn_cur_min_txg
)
1729 * One block ("stubby") can be allocated a long time ago; we
1730 * want to visit that one because it has been allocated
1731 * (on-disk) even if it hasn't been claimed (even though for
1732 * scrub there's nothing to do to it).
1734 if (claim_txg
== 0 &&
1735 BP_GET_LOGICAL_BIRTH(bp
) >= spa_min_claim_txg(dp
->dp_spa
))
1738 SET_BOOKMARK(&zb
, zh
->zh_log
.blk_cksum
.zc_word
[ZIL_ZC_OBJSET
],
1739 ZB_ZIL_OBJECT
, ZB_ZIL_LEVEL
, bp
->blk_cksum
.zc_word
[ZIL_ZC_SEQ
]);
1741 VERIFY(0 == scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, &zb
));
1746 dsl_scan_zil_record(zilog_t
*zilog
, const lr_t
*lrc
, void *arg
,
1750 if (lrc
->lrc_txtype
== TX_WRITE
) {
1751 zil_scan_arg_t
*zsa
= arg
;
1752 dsl_pool_t
*dp
= zsa
->zsa_dp
;
1753 dsl_scan_t
*scn
= dp
->dp_scan
;
1754 zil_header_t
*zh
= zsa
->zsa_zh
;
1755 const lr_write_t
*lr
= (const lr_write_t
*)lrc
;
1756 const blkptr_t
*bp
= &lr
->lr_blkptr
;
1757 zbookmark_phys_t zb
;
1759 ASSERT(!BP_IS_REDACTED(bp
));
1760 if (BP_IS_HOLE(bp
) ||
1761 BP_GET_LOGICAL_BIRTH(bp
) <= scn
->scn_phys
.scn_cur_min_txg
)
1765 * birth can be < claim_txg if this record's txg is
1766 * already txg sync'ed (but this log block contains
1767 * other records that are not synced)
1769 if (claim_txg
== 0 || BP_GET_LOGICAL_BIRTH(bp
) < claim_txg
)
1772 ASSERT3U(BP_GET_LSIZE(bp
), !=, 0);
1773 SET_BOOKMARK(&zb
, zh
->zh_log
.blk_cksum
.zc_word
[ZIL_ZC_OBJSET
],
1774 lr
->lr_foid
, ZB_ZIL_LEVEL
,
1775 lr
->lr_offset
/ BP_GET_LSIZE(bp
));
1777 VERIFY(0 == scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, &zb
));
1783 dsl_scan_zil(dsl_pool_t
*dp
, zil_header_t
*zh
)
1785 uint64_t claim_txg
= zh
->zh_claim_txg
;
1786 zil_scan_arg_t zsa
= { dp
, zh
};
1789 ASSERT(spa_writeable(dp
->dp_spa
));
1792 * We only want to visit blocks that have been claimed but not yet
1793 * replayed (or, in read-only mode, blocks that *would* be claimed).
1798 zilog
= zil_alloc(dp
->dp_meta_objset
, zh
);
1800 (void) zil_parse(zilog
, dsl_scan_zil_block
, dsl_scan_zil_record
, &zsa
,
1801 claim_txg
, B_FALSE
);
1807 * We compare scan_prefetch_issue_ctx_t's based on their bookmarks. The idea
1808 * here is to sort the AVL tree by the order each block will be needed.
1811 scan_prefetch_queue_compare(const void *a
, const void *b
)
1813 const scan_prefetch_issue_ctx_t
*spic_a
= a
, *spic_b
= b
;
1814 const scan_prefetch_ctx_t
*spc_a
= spic_a
->spic_spc
;
1815 const scan_prefetch_ctx_t
*spc_b
= spic_b
->spic_spc
;
1817 return (zbookmark_compare(spc_a
->spc_datablkszsec
,
1818 spc_a
->spc_indblkshift
, spc_b
->spc_datablkszsec
,
1819 spc_b
->spc_indblkshift
, &spic_a
->spic_zb
, &spic_b
->spic_zb
));
1823 scan_prefetch_ctx_rele(scan_prefetch_ctx_t
*spc
, const void *tag
)
1825 if (zfs_refcount_remove(&spc
->spc_refcnt
, tag
) == 0) {
1826 zfs_refcount_destroy(&spc
->spc_refcnt
);
1827 kmem_free(spc
, sizeof (scan_prefetch_ctx_t
));
1831 static scan_prefetch_ctx_t
*
1832 scan_prefetch_ctx_create(dsl_scan_t
*scn
, dnode_phys_t
*dnp
, const void *tag
)
1834 scan_prefetch_ctx_t
*spc
;
1836 spc
= kmem_alloc(sizeof (scan_prefetch_ctx_t
), KM_SLEEP
);
1837 zfs_refcount_create(&spc
->spc_refcnt
);
1838 zfs_refcount_add(&spc
->spc_refcnt
, tag
);
1841 spc
->spc_datablkszsec
= dnp
->dn_datablkszsec
;
1842 spc
->spc_indblkshift
= dnp
->dn_indblkshift
;
1843 spc
->spc_root
= B_FALSE
;
1845 spc
->spc_datablkszsec
= 0;
1846 spc
->spc_indblkshift
= 0;
1847 spc
->spc_root
= B_TRUE
;
1854 scan_prefetch_ctx_add_ref(scan_prefetch_ctx_t
*spc
, const void *tag
)
1856 zfs_refcount_add(&spc
->spc_refcnt
, tag
);
1860 scan_ds_prefetch_queue_clear(dsl_scan_t
*scn
)
1862 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1863 void *cookie
= NULL
;
1864 scan_prefetch_issue_ctx_t
*spic
= NULL
;
1866 mutex_enter(&spa
->spa_scrub_lock
);
1867 while ((spic
= avl_destroy_nodes(&scn
->scn_prefetch_queue
,
1868 &cookie
)) != NULL
) {
1869 scan_prefetch_ctx_rele(spic
->spic_spc
, scn
);
1870 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
1872 mutex_exit(&spa
->spa_scrub_lock
);
1876 dsl_scan_check_prefetch_resume(scan_prefetch_ctx_t
*spc
,
1877 const zbookmark_phys_t
*zb
)
1879 zbookmark_phys_t
*last_zb
= &spc
->spc_scn
->scn_prefetch_bookmark
;
1880 dnode_phys_t tmp_dnp
;
1881 dnode_phys_t
*dnp
= (spc
->spc_root
) ? NULL
: &tmp_dnp
;
1883 if (zb
->zb_objset
!= last_zb
->zb_objset
)
1885 if ((int64_t)zb
->zb_object
< 0)
1888 tmp_dnp
.dn_datablkszsec
= spc
->spc_datablkszsec
;
1889 tmp_dnp
.dn_indblkshift
= spc
->spc_indblkshift
;
1891 if (zbookmark_subtree_completed(dnp
, zb
, last_zb
))
1898 dsl_scan_prefetch(scan_prefetch_ctx_t
*spc
, blkptr_t
*bp
, zbookmark_phys_t
*zb
)
1901 dsl_scan_t
*scn
= spc
->spc_scn
;
1902 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1903 scan_prefetch_issue_ctx_t
*spic
;
1905 if (zfs_no_scrub_prefetch
|| BP_IS_REDACTED(bp
))
1908 if (BP_IS_HOLE(bp
) ||
1909 BP_GET_LOGICAL_BIRTH(bp
) <= scn
->scn_phys
.scn_cur_min_txg
||
1910 (BP_GET_LEVEL(bp
) == 0 && BP_GET_TYPE(bp
) != DMU_OT_DNODE
&&
1911 BP_GET_TYPE(bp
) != DMU_OT_OBJSET
))
1914 if (dsl_scan_check_prefetch_resume(spc
, zb
))
1917 scan_prefetch_ctx_add_ref(spc
, scn
);
1918 spic
= kmem_alloc(sizeof (scan_prefetch_issue_ctx_t
), KM_SLEEP
);
1919 spic
->spic_spc
= spc
;
1920 spic
->spic_bp
= *bp
;
1921 spic
->spic_zb
= *zb
;
1924 * Add the IO to the queue of blocks to prefetch. This allows us to
1925 * prioritize blocks that we will need first for the main traversal
1928 mutex_enter(&spa
->spa_scrub_lock
);
1929 if (avl_find(&scn
->scn_prefetch_queue
, spic
, &idx
) != NULL
) {
1930 /* this block is already queued for prefetch */
1931 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
1932 scan_prefetch_ctx_rele(spc
, scn
);
1933 mutex_exit(&spa
->spa_scrub_lock
);
1937 avl_insert(&scn
->scn_prefetch_queue
, spic
, idx
);
1938 cv_broadcast(&spa
->spa_scrub_io_cv
);
1939 mutex_exit(&spa
->spa_scrub_lock
);
1943 dsl_scan_prefetch_dnode(dsl_scan_t
*scn
, dnode_phys_t
*dnp
,
1944 uint64_t objset
, uint64_t object
)
1947 zbookmark_phys_t zb
;
1948 scan_prefetch_ctx_t
*spc
;
1950 if (dnp
->dn_nblkptr
== 0 && !(dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
))
1953 SET_BOOKMARK(&zb
, objset
, object
, 0, 0);
1955 spc
= scan_prefetch_ctx_create(scn
, dnp
, FTAG
);
1957 for (i
= 0; i
< dnp
->dn_nblkptr
; i
++) {
1958 zb
.zb_level
= BP_GET_LEVEL(&dnp
->dn_blkptr
[i
]);
1960 dsl_scan_prefetch(spc
, &dnp
->dn_blkptr
[i
], &zb
);
1963 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1965 zb
.zb_blkid
= DMU_SPILL_BLKID
;
1966 dsl_scan_prefetch(spc
, DN_SPILL_BLKPTR(dnp
), &zb
);
1969 scan_prefetch_ctx_rele(spc
, FTAG
);
1973 dsl_scan_prefetch_cb(zio_t
*zio
, const zbookmark_phys_t
*zb
, const blkptr_t
*bp
,
1974 arc_buf_t
*buf
, void *private)
1977 scan_prefetch_ctx_t
*spc
= private;
1978 dsl_scan_t
*scn
= spc
->spc_scn
;
1979 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
1981 /* broadcast that the IO has completed for rate limiting purposes */
1982 mutex_enter(&spa
->spa_scrub_lock
);
1983 ASSERT3U(spa
->spa_scrub_inflight
, >=, BP_GET_PSIZE(bp
));
1984 spa
->spa_scrub_inflight
-= BP_GET_PSIZE(bp
);
1985 cv_broadcast(&spa
->spa_scrub_io_cv
);
1986 mutex_exit(&spa
->spa_scrub_lock
);
1988 /* if there was an error or we are done prefetching, just cleanup */
1989 if (buf
== NULL
|| scn
->scn_prefetch_stop
)
1992 if (BP_GET_LEVEL(bp
) > 0) {
1995 int epb
= BP_GET_LSIZE(bp
) >> SPA_BLKPTRSHIFT
;
1996 zbookmark_phys_t czb
;
1998 for (i
= 0, cbp
= buf
->b_data
; i
< epb
; i
++, cbp
++) {
1999 SET_BOOKMARK(&czb
, zb
->zb_objset
, zb
->zb_object
,
2000 zb
->zb_level
- 1, zb
->zb_blkid
* epb
+ i
);
2001 dsl_scan_prefetch(spc
, cbp
, &czb
);
2003 } else if (BP_GET_TYPE(bp
) == DMU_OT_DNODE
) {
2006 int epb
= BP_GET_LSIZE(bp
) >> DNODE_SHIFT
;
2008 for (i
= 0, cdnp
= buf
->b_data
; i
< epb
;
2009 i
+= cdnp
->dn_extra_slots
+ 1,
2010 cdnp
+= cdnp
->dn_extra_slots
+ 1) {
2011 dsl_scan_prefetch_dnode(scn
, cdnp
,
2012 zb
->zb_objset
, zb
->zb_blkid
* epb
+ i
);
2014 } else if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
2015 objset_phys_t
*osp
= buf
->b_data
;
2017 dsl_scan_prefetch_dnode(scn
, &osp
->os_meta_dnode
,
2018 zb
->zb_objset
, DMU_META_DNODE_OBJECT
);
2020 if (OBJSET_BUF_HAS_USERUSED(buf
)) {
2021 if (OBJSET_BUF_HAS_PROJECTUSED(buf
)) {
2022 dsl_scan_prefetch_dnode(scn
,
2023 &osp
->os_projectused_dnode
, zb
->zb_objset
,
2024 DMU_PROJECTUSED_OBJECT
);
2026 dsl_scan_prefetch_dnode(scn
,
2027 &osp
->os_groupused_dnode
, zb
->zb_objset
,
2028 DMU_GROUPUSED_OBJECT
);
2029 dsl_scan_prefetch_dnode(scn
,
2030 &osp
->os_userused_dnode
, zb
->zb_objset
,
2031 DMU_USERUSED_OBJECT
);
2037 arc_buf_destroy(buf
, private);
2038 scan_prefetch_ctx_rele(spc
, scn
);
2042 dsl_scan_prefetch_thread(void *arg
)
2044 dsl_scan_t
*scn
= arg
;
2045 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
2046 scan_prefetch_issue_ctx_t
*spic
;
2048 /* loop until we are told to stop */
2049 while (!scn
->scn_prefetch_stop
) {
2050 arc_flags_t flags
= ARC_FLAG_NOWAIT
|
2051 ARC_FLAG_PRESCIENT_PREFETCH
| ARC_FLAG_PREFETCH
;
2052 int zio_flags
= ZIO_FLAG_CANFAIL
| ZIO_FLAG_SCAN_THREAD
;
2054 mutex_enter(&spa
->spa_scrub_lock
);
2057 * Wait until we have an IO to issue and are not above our
2058 * maximum in flight limit.
2060 while (!scn
->scn_prefetch_stop
&&
2061 (avl_numnodes(&scn
->scn_prefetch_queue
) == 0 ||
2062 spa
->spa_scrub_inflight
>= scn
->scn_maxinflight_bytes
)) {
2063 cv_wait(&spa
->spa_scrub_io_cv
, &spa
->spa_scrub_lock
);
2066 /* recheck if we should stop since we waited for the cv */
2067 if (scn
->scn_prefetch_stop
) {
2068 mutex_exit(&spa
->spa_scrub_lock
);
2072 /* remove the prefetch IO from the tree */
2073 spic
= avl_first(&scn
->scn_prefetch_queue
);
2074 spa
->spa_scrub_inflight
+= BP_GET_PSIZE(&spic
->spic_bp
);
2075 avl_remove(&scn
->scn_prefetch_queue
, spic
);
2077 mutex_exit(&spa
->spa_scrub_lock
);
2079 if (BP_IS_PROTECTED(&spic
->spic_bp
)) {
2080 ASSERT(BP_GET_TYPE(&spic
->spic_bp
) == DMU_OT_DNODE
||
2081 BP_GET_TYPE(&spic
->spic_bp
) == DMU_OT_OBJSET
);
2082 ASSERT3U(BP_GET_LEVEL(&spic
->spic_bp
), ==, 0);
2083 zio_flags
|= ZIO_FLAG_RAW
;
2086 /* We don't need data L1 buffer since we do not prefetch L0. */
2087 blkptr_t
*bp
= &spic
->spic_bp
;
2088 if (BP_GET_LEVEL(bp
) == 1 && BP_GET_TYPE(bp
) != DMU_OT_DNODE
&&
2089 BP_GET_TYPE(bp
) != DMU_OT_OBJSET
)
2090 flags
|= ARC_FLAG_NO_BUF
;
2092 /* issue the prefetch asynchronously */
2093 (void) arc_read(scn
->scn_zio_root
, spa
, bp
,
2094 dsl_scan_prefetch_cb
, spic
->spic_spc
, ZIO_PRIORITY_SCRUB
,
2095 zio_flags
, &flags
, &spic
->spic_zb
);
2097 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
2100 ASSERT(scn
->scn_prefetch_stop
);
2102 /* free any prefetches we didn't get to complete */
2103 mutex_enter(&spa
->spa_scrub_lock
);
2104 while ((spic
= avl_first(&scn
->scn_prefetch_queue
)) != NULL
) {
2105 avl_remove(&scn
->scn_prefetch_queue
, spic
);
2106 scan_prefetch_ctx_rele(spic
->spic_spc
, scn
);
2107 kmem_free(spic
, sizeof (scan_prefetch_issue_ctx_t
));
2109 ASSERT0(avl_numnodes(&scn
->scn_prefetch_queue
));
2110 mutex_exit(&spa
->spa_scrub_lock
);
2114 dsl_scan_check_resume(dsl_scan_t
*scn
, const dnode_phys_t
*dnp
,
2115 const zbookmark_phys_t
*zb
)
2118 * We never skip over user/group accounting objects (obj<0)
2120 if (!ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
) &&
2121 (int64_t)zb
->zb_object
>= 0) {
2123 * If we already visited this bp & everything below (in
2124 * a prior txg sync), don't bother doing it again.
2126 if (zbookmark_subtree_completed(dnp
, zb
,
2127 &scn
->scn_phys
.scn_bookmark
))
2131 * If we found the block we're trying to resume from, or
2132 * we went past it, zero it out to indicate that it's OK
2133 * to start checking for suspending again.
2135 if (zbookmark_subtree_tbd(dnp
, zb
,
2136 &scn
->scn_phys
.scn_bookmark
)) {
2137 dprintf("resuming at %llx/%llx/%llx/%llx\n",
2138 (longlong_t
)zb
->zb_objset
,
2139 (longlong_t
)zb
->zb_object
,
2140 (longlong_t
)zb
->zb_level
,
2141 (longlong_t
)zb
->zb_blkid
);
2142 memset(&scn
->scn_phys
.scn_bookmark
, 0, sizeof (*zb
));
2148 static void dsl_scan_visitbp(const blkptr_t
*bp
, const zbookmark_phys_t
*zb
,
2149 dnode_phys_t
*dnp
, dsl_dataset_t
*ds
, dsl_scan_t
*scn
,
2150 dmu_objset_type_t ostype
, dmu_tx_t
*tx
);
2151 inline __attribute__((always_inline
)) static void dsl_scan_visitdnode(
2152 dsl_scan_t
*, dsl_dataset_t
*ds
, dmu_objset_type_t ostype
,
2153 dnode_phys_t
*dnp
, uint64_t object
, dmu_tx_t
*tx
);
2156 * Return nonzero on i/o error.
2157 * Return new buf to write out in *bufp.
2159 inline __attribute__((always_inline
)) static int
2160 dsl_scan_recurse(dsl_scan_t
*scn
, dsl_dataset_t
*ds
, dmu_objset_type_t ostype
,
2161 dnode_phys_t
*dnp
, const blkptr_t
*bp
,
2162 const zbookmark_phys_t
*zb
, dmu_tx_t
*tx
)
2164 dsl_pool_t
*dp
= scn
->scn_dp
;
2165 spa_t
*spa
= dp
->dp_spa
;
2166 int zio_flags
= ZIO_FLAG_CANFAIL
| ZIO_FLAG_SCAN_THREAD
;
2169 ASSERT(!BP_IS_REDACTED(bp
));
2172 * There is an unlikely case of encountering dnodes with contradicting
2173 * dn_bonuslen and DNODE_FLAG_SPILL_BLKPTR flag before in files created
2174 * or modified before commit 4254acb was merged. As it is not possible
2175 * to know which of the two is correct, report an error.
2178 dnp
->dn_bonuslen
> DN_MAX_BONUS_LEN(dnp
)) {
2179 scn
->scn_phys
.scn_errors
++;
2180 spa_log_error(spa
, zb
, BP_GET_LOGICAL_BIRTH(bp
));
2181 return (SET_ERROR(EINVAL
));
2184 if (BP_GET_LEVEL(bp
) > 0) {
2185 arc_flags_t flags
= ARC_FLAG_WAIT
;
2188 int epb
= BP_GET_LSIZE(bp
) >> SPA_BLKPTRSHIFT
;
2191 err
= arc_read(NULL
, spa
, bp
, arc_getbuf_func
, &buf
,
2192 ZIO_PRIORITY_SCRUB
, zio_flags
, &flags
, zb
);
2194 scn
->scn_phys
.scn_errors
++;
2197 for (i
= 0, cbp
= buf
->b_data
; i
< epb
; i
++, cbp
++) {
2198 zbookmark_phys_t czb
;
2200 SET_BOOKMARK(&czb
, zb
->zb_objset
, zb
->zb_object
,
2202 zb
->zb_blkid
* epb
+ i
);
2203 dsl_scan_visitbp(cbp
, &czb
, dnp
,
2204 ds
, scn
, ostype
, tx
);
2206 arc_buf_destroy(buf
, &buf
);
2207 } else if (BP_GET_TYPE(bp
) == DMU_OT_DNODE
) {
2208 arc_flags_t flags
= ARC_FLAG_WAIT
;
2211 int epb
= BP_GET_LSIZE(bp
) >> DNODE_SHIFT
;
2214 if (BP_IS_PROTECTED(bp
)) {
2215 ASSERT3U(BP_GET_COMPRESS(bp
), ==, ZIO_COMPRESS_OFF
);
2216 zio_flags
|= ZIO_FLAG_RAW
;
2219 err
= arc_read(NULL
, spa
, bp
, arc_getbuf_func
, &buf
,
2220 ZIO_PRIORITY_SCRUB
, zio_flags
, &flags
, zb
);
2222 scn
->scn_phys
.scn_errors
++;
2225 for (i
= 0, cdnp
= buf
->b_data
; i
< epb
;
2226 i
+= cdnp
->dn_extra_slots
+ 1,
2227 cdnp
+= cdnp
->dn_extra_slots
+ 1) {
2228 dsl_scan_visitdnode(scn
, ds
, ostype
,
2229 cdnp
, zb
->zb_blkid
* epb
+ i
, tx
);
2232 arc_buf_destroy(buf
, &buf
);
2233 } else if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
2234 arc_flags_t flags
= ARC_FLAG_WAIT
;
2238 err
= arc_read(NULL
, spa
, bp
, arc_getbuf_func
, &buf
,
2239 ZIO_PRIORITY_SCRUB
, zio_flags
, &flags
, zb
);
2241 scn
->scn_phys
.scn_errors
++;
2247 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
2248 &osp
->os_meta_dnode
, DMU_META_DNODE_OBJECT
, tx
);
2250 if (OBJSET_BUF_HAS_USERUSED(buf
)) {
2252 * We also always visit user/group/project accounting
2253 * objects, and never skip them, even if we are
2254 * suspending. This is necessary so that the
2255 * space deltas from this txg get integrated.
2257 if (OBJSET_BUF_HAS_PROJECTUSED(buf
))
2258 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
2259 &osp
->os_projectused_dnode
,
2260 DMU_PROJECTUSED_OBJECT
, tx
);
2261 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
2262 &osp
->os_groupused_dnode
,
2263 DMU_GROUPUSED_OBJECT
, tx
);
2264 dsl_scan_visitdnode(scn
, ds
, osp
->os_type
,
2265 &osp
->os_userused_dnode
,
2266 DMU_USERUSED_OBJECT
, tx
);
2268 arc_buf_destroy(buf
, &buf
);
2269 } else if (!zfs_blkptr_verify(spa
, bp
,
2270 BLK_CONFIG_NEEDED
, BLK_VERIFY_LOG
)) {
2272 * Sanity check the block pointer contents, this is handled
2273 * by arc_read() for the cases above.
2275 scn
->scn_phys
.scn_errors
++;
2276 spa_log_error(spa
, zb
, BP_GET_LOGICAL_BIRTH(bp
));
2277 return (SET_ERROR(EINVAL
));
2283 inline __attribute__((always_inline
)) static void
2284 dsl_scan_visitdnode(dsl_scan_t
*scn
, dsl_dataset_t
*ds
,
2285 dmu_objset_type_t ostype
, dnode_phys_t
*dnp
,
2286 uint64_t object
, dmu_tx_t
*tx
)
2290 for (j
= 0; j
< dnp
->dn_nblkptr
; j
++) {
2291 zbookmark_phys_t czb
;
2293 SET_BOOKMARK(&czb
, ds
? ds
->ds_object
: 0, object
,
2294 dnp
->dn_nlevels
- 1, j
);
2295 dsl_scan_visitbp(&dnp
->dn_blkptr
[j
],
2296 &czb
, dnp
, ds
, scn
, ostype
, tx
);
2299 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
2300 zbookmark_phys_t czb
;
2301 SET_BOOKMARK(&czb
, ds
? ds
->ds_object
: 0, object
,
2302 0, DMU_SPILL_BLKID
);
2303 dsl_scan_visitbp(DN_SPILL_BLKPTR(dnp
),
2304 &czb
, dnp
, ds
, scn
, ostype
, tx
);
2309 * The arguments are in this order because mdb can only print the
2310 * first 5; we want them to be useful.
2313 dsl_scan_visitbp(const blkptr_t
*bp
, const zbookmark_phys_t
*zb
,
2314 dnode_phys_t
*dnp
, dsl_dataset_t
*ds
, dsl_scan_t
*scn
,
2315 dmu_objset_type_t ostype
, dmu_tx_t
*tx
)
2317 dsl_pool_t
*dp
= scn
->scn_dp
;
2319 if (dsl_scan_check_suspend(scn
, zb
))
2322 if (dsl_scan_check_resume(scn
, dnp
, zb
))
2325 scn
->scn_visited_this_txg
++;
2327 if (BP_IS_HOLE(bp
)) {
2328 scn
->scn_holes_this_txg
++;
2332 if (BP_IS_REDACTED(bp
)) {
2333 ASSERT(dsl_dataset_feature_is_active(ds
,
2334 SPA_FEATURE_REDACTED_DATASETS
));
2339 * Check if this block contradicts any filesystem flags.
2341 spa_feature_t f
= SPA_FEATURE_LARGE_BLOCKS
;
2342 if (BP_GET_LSIZE(bp
) > SPA_OLD_MAXBLOCKSIZE
)
2343 ASSERT(dsl_dataset_feature_is_active(ds
, f
));
2345 f
= zio_checksum_to_feature(BP_GET_CHECKSUM(bp
));
2346 if (f
!= SPA_FEATURE_NONE
)
2347 ASSERT(dsl_dataset_feature_is_active(ds
, f
));
2349 f
= zio_compress_to_feature(BP_GET_COMPRESS(bp
));
2350 if (f
!= SPA_FEATURE_NONE
)
2351 ASSERT(dsl_dataset_feature_is_active(ds
, f
));
2353 if (BP_GET_LOGICAL_BIRTH(bp
) <= scn
->scn_phys
.scn_cur_min_txg
) {
2354 scn
->scn_lt_min_this_txg
++;
2358 if (dsl_scan_recurse(scn
, ds
, ostype
, dnp
, bp
, zb
, tx
) != 0)
2362 * If dsl_scan_ddt() has already visited this block, it will have
2363 * already done any translations or scrubbing, so don't call the
2366 if (ddt_class_contains(dp
->dp_spa
,
2367 scn
->scn_phys
.scn_ddt_class_max
, bp
)) {
2368 scn
->scn_ddt_contained_this_txg
++;
2373 * If this block is from the future (after cur_max_txg), then we
2374 * are doing this on behalf of a deleted snapshot, and we will
2375 * revisit the future block on the next pass of this dataset.
2376 * Don't scan it now unless we need to because something
2377 * under it was modified.
2379 if (BP_GET_BIRTH(bp
) > scn
->scn_phys
.scn_cur_max_txg
) {
2380 scn
->scn_gt_max_this_txg
++;
2384 scan_funcs
[scn
->scn_phys
.scn_func
](dp
, bp
, zb
);
2388 dsl_scan_visit_rootbp(dsl_scan_t
*scn
, dsl_dataset_t
*ds
, blkptr_t
*bp
,
2391 zbookmark_phys_t zb
;
2392 scan_prefetch_ctx_t
*spc
;
2394 SET_BOOKMARK(&zb
, ds
? ds
->ds_object
: DMU_META_OBJSET
,
2395 ZB_ROOT_OBJECT
, ZB_ROOT_LEVEL
, ZB_ROOT_BLKID
);
2397 if (ZB_IS_ZERO(&scn
->scn_phys
.scn_bookmark
)) {
2398 SET_BOOKMARK(&scn
->scn_prefetch_bookmark
,
2399 zb
.zb_objset
, 0, 0, 0);
2401 scn
->scn_prefetch_bookmark
= scn
->scn_phys
.scn_bookmark
;
2404 scn
->scn_objsets_visited_this_txg
++;
2406 spc
= scan_prefetch_ctx_create(scn
, NULL
, FTAG
);
2407 dsl_scan_prefetch(spc
, bp
, &zb
);
2408 scan_prefetch_ctx_rele(spc
, FTAG
);
2410 dsl_scan_visitbp(bp
, &zb
, NULL
, ds
, scn
, DMU_OST_NONE
, tx
);
2412 dprintf_ds(ds
, "finished scan%s", "");
2416 ds_destroyed_scn_phys(dsl_dataset_t
*ds
, dsl_scan_phys_t
*scn_phys
)
2418 if (scn_phys
->scn_bookmark
.zb_objset
== ds
->ds_object
) {
2419 if (ds
->ds_is_snapshot
) {
2422 * - scn_cur_{min,max}_txg stays the same.
2423 * - Setting the flag is not really necessary if
2424 * scn_cur_max_txg == scn_max_txg, because there
2425 * is nothing after this snapshot that we care
2426 * about. However, we set it anyway and then
2427 * ignore it when we retraverse it in
2428 * dsl_scan_visitds().
2430 scn_phys
->scn_bookmark
.zb_objset
=
2431 dsl_dataset_phys(ds
)->ds_next_snap_obj
;
2432 zfs_dbgmsg("destroying ds %llu on %s; currently "
2433 "traversing; reset zb_objset to %llu",
2434 (u_longlong_t
)ds
->ds_object
,
2435 ds
->ds_dir
->dd_pool
->dp_spa
->spa_name
,
2436 (u_longlong_t
)dsl_dataset_phys(ds
)->
2438 scn_phys
->scn_flags
|= DSF_VISIT_DS_AGAIN
;
2440 SET_BOOKMARK(&scn_phys
->scn_bookmark
,
2441 ZB_DESTROYED_OBJSET
, 0, 0, 0);
2442 zfs_dbgmsg("destroying ds %llu on %s; currently "
2443 "traversing; reset bookmark to -1,0,0,0",
2444 (u_longlong_t
)ds
->ds_object
,
2445 ds
->ds_dir
->dd_pool
->dp_spa
->spa_name
);
2451 * Invoked when a dataset is destroyed. We need to make sure that:
2453 * 1) If it is the dataset that was currently being scanned, we write
2454 * a new dsl_scan_phys_t and marking the objset reference in it
2456 * 2) Remove it from the work queue, if it was present.
2458 * If the dataset was actually a snapshot, instead of marking the dataset
2459 * as destroyed, we instead substitute the next snapshot in line.
2462 dsl_scan_ds_destroyed(dsl_dataset_t
*ds
, dmu_tx_t
*tx
)
2464 dsl_pool_t
*dp
= ds
->ds_dir
->dd_pool
;
2465 dsl_scan_t
*scn
= dp
->dp_scan
;
2468 if (!dsl_scan_is_running(scn
))
2471 ds_destroyed_scn_phys(ds
, &scn
->scn_phys
);
2472 ds_destroyed_scn_phys(ds
, &scn
->scn_phys_cached
);
2474 if (scan_ds_queue_contains(scn
, ds
->ds_object
, &mintxg
)) {
2475 scan_ds_queue_remove(scn
, ds
->ds_object
);
2476 if (ds
->ds_is_snapshot
)
2477 scan_ds_queue_insert(scn
,
2478 dsl_dataset_phys(ds
)->ds_next_snap_obj
, mintxg
);
2481 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
2482 ds
->ds_object
, &mintxg
) == 0) {
2483 ASSERT3U(dsl_dataset_phys(ds
)->ds_num_children
, <=, 1);
2484 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2485 scn
->scn_phys
.scn_queue_obj
, ds
->ds_object
, tx
));
2486 if (ds
->ds_is_snapshot
) {
2488 * We keep the same mintxg; it could be >
2489 * ds_creation_txg if the previous snapshot was
2492 VERIFY(zap_add_int_key(dp
->dp_meta_objset
,
2493 scn
->scn_phys
.scn_queue_obj
,
2494 dsl_dataset_phys(ds
)->ds_next_snap_obj
,
2496 zfs_dbgmsg("destroying ds %llu on %s; in queue; "
2497 "replacing with %llu",
2498 (u_longlong_t
)ds
->ds_object
,
2499 dp
->dp_spa
->spa_name
,
2500 (u_longlong_t
)dsl_dataset_phys(ds
)->
2503 zfs_dbgmsg("destroying ds %llu on %s; in queue; "
2505 (u_longlong_t
)ds
->ds_object
,
2506 dp
->dp_spa
->spa_name
);
2511 * dsl_scan_sync() should be called after this, and should sync
2512 * out our changed state, but just to be safe, do it here.
2514 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
2518 ds_snapshotted_bookmark(dsl_dataset_t
*ds
, zbookmark_phys_t
*scn_bookmark
)
2520 if (scn_bookmark
->zb_objset
== ds
->ds_object
) {
2521 scn_bookmark
->zb_objset
=
2522 dsl_dataset_phys(ds
)->ds_prev_snap_obj
;
2523 zfs_dbgmsg("snapshotting ds %llu on %s; currently traversing; "
2524 "reset zb_objset to %llu",
2525 (u_longlong_t
)ds
->ds_object
,
2526 ds
->ds_dir
->dd_pool
->dp_spa
->spa_name
,
2527 (u_longlong_t
)dsl_dataset_phys(ds
)->ds_prev_snap_obj
);
2532 * Called when a dataset is snapshotted. If we were currently traversing
2533 * this snapshot, we reset our bookmark to point at the newly created
2534 * snapshot. We also modify our work queue to remove the old snapshot and
2535 * replace with the new one.
2538 dsl_scan_ds_snapshotted(dsl_dataset_t
*ds
, dmu_tx_t
*tx
)
2540 dsl_pool_t
*dp
= ds
->ds_dir
->dd_pool
;
2541 dsl_scan_t
*scn
= dp
->dp_scan
;
2544 if (!dsl_scan_is_running(scn
))
2547 ASSERT(dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= 0);
2549 ds_snapshotted_bookmark(ds
, &scn
->scn_phys
.scn_bookmark
);
2550 ds_snapshotted_bookmark(ds
, &scn
->scn_phys_cached
.scn_bookmark
);
2552 if (scan_ds_queue_contains(scn
, ds
->ds_object
, &mintxg
)) {
2553 scan_ds_queue_remove(scn
, ds
->ds_object
);
2554 scan_ds_queue_insert(scn
,
2555 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, mintxg
);
2558 if (zap_lookup_int_key(dp
->dp_meta_objset
, scn
->scn_phys
.scn_queue_obj
,
2559 ds
->ds_object
, &mintxg
) == 0) {
2560 VERIFY3U(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2561 scn
->scn_phys
.scn_queue_obj
, ds
->ds_object
, tx
));
2562 VERIFY(zap_add_int_key(dp
->dp_meta_objset
,
2563 scn
->scn_phys
.scn_queue_obj
,
2564 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, mintxg
, tx
) == 0);
2565 zfs_dbgmsg("snapshotting ds %llu on %s; in queue; "
2566 "replacing with %llu",
2567 (u_longlong_t
)ds
->ds_object
,
2568 dp
->dp_spa
->spa_name
,
2569 (u_longlong_t
)dsl_dataset_phys(ds
)->ds_prev_snap_obj
);
2572 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
2576 ds_clone_swapped_bookmark(dsl_dataset_t
*ds1
, dsl_dataset_t
*ds2
,
2577 zbookmark_phys_t
*scn_bookmark
)
2579 if (scn_bookmark
->zb_objset
== ds1
->ds_object
) {
2580 scn_bookmark
->zb_objset
= ds2
->ds_object
;
2581 zfs_dbgmsg("clone_swap ds %llu on %s; currently traversing; "
2582 "reset zb_objset to %llu",
2583 (u_longlong_t
)ds1
->ds_object
,
2584 ds1
->ds_dir
->dd_pool
->dp_spa
->spa_name
,
2585 (u_longlong_t
)ds2
->ds_object
);
2586 } else if (scn_bookmark
->zb_objset
== ds2
->ds_object
) {
2587 scn_bookmark
->zb_objset
= ds1
->ds_object
;
2588 zfs_dbgmsg("clone_swap ds %llu on %s; currently traversing; "
2589 "reset zb_objset to %llu",
2590 (u_longlong_t
)ds2
->ds_object
,
2591 ds2
->ds_dir
->dd_pool
->dp_spa
->spa_name
,
2592 (u_longlong_t
)ds1
->ds_object
);
2597 * Called when an origin dataset and its clone are swapped. If we were
2598 * currently traversing the dataset, we need to switch to traversing the
2599 * newly promoted clone.
2602 dsl_scan_ds_clone_swapped(dsl_dataset_t
*ds1
, dsl_dataset_t
*ds2
, dmu_tx_t
*tx
)
2604 dsl_pool_t
*dp
= ds1
->ds_dir
->dd_pool
;
2605 dsl_scan_t
*scn
= dp
->dp_scan
;
2606 uint64_t mintxg1
, mintxg2
;
2607 boolean_t ds1_queued
, ds2_queued
;
2609 if (!dsl_scan_is_running(scn
))
2612 ds_clone_swapped_bookmark(ds1
, ds2
, &scn
->scn_phys
.scn_bookmark
);
2613 ds_clone_swapped_bookmark(ds1
, ds2
, &scn
->scn_phys_cached
.scn_bookmark
);
2616 * Handle the in-memory scan queue.
2618 ds1_queued
= scan_ds_queue_contains(scn
, ds1
->ds_object
, &mintxg1
);
2619 ds2_queued
= scan_ds_queue_contains(scn
, ds2
->ds_object
, &mintxg2
);
2621 /* Sanity checking. */
2623 ASSERT3U(mintxg1
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2624 ASSERT3U(mintxg1
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2627 ASSERT3U(mintxg2
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2628 ASSERT3U(mintxg2
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2631 if (ds1_queued
&& ds2_queued
) {
2633 * If both are queued, we don't need to do anything.
2634 * The swapping code below would not handle this case correctly,
2635 * since we can't insert ds2 if it is already there. That's
2636 * because scan_ds_queue_insert() prohibits a duplicate insert
2639 } else if (ds1_queued
) {
2640 scan_ds_queue_remove(scn
, ds1
->ds_object
);
2641 scan_ds_queue_insert(scn
, ds2
->ds_object
, mintxg1
);
2642 } else if (ds2_queued
) {
2643 scan_ds_queue_remove(scn
, ds2
->ds_object
);
2644 scan_ds_queue_insert(scn
, ds1
->ds_object
, mintxg2
);
2648 * Handle the on-disk scan queue.
2649 * The on-disk state is an out-of-date version of the in-memory state,
2650 * so the in-memory and on-disk values for ds1_queued and ds2_queued may
2651 * be different. Therefore we need to apply the swap logic to the
2652 * on-disk state independently of the in-memory state.
2654 ds1_queued
= zap_lookup_int_key(dp
->dp_meta_objset
,
2655 scn
->scn_phys
.scn_queue_obj
, ds1
->ds_object
, &mintxg1
) == 0;
2656 ds2_queued
= zap_lookup_int_key(dp
->dp_meta_objset
,
2657 scn
->scn_phys
.scn_queue_obj
, ds2
->ds_object
, &mintxg2
) == 0;
2659 /* Sanity checking. */
2661 ASSERT3U(mintxg1
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2662 ASSERT3U(mintxg1
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2665 ASSERT3U(mintxg2
, ==, dsl_dataset_phys(ds1
)->ds_prev_snap_txg
);
2666 ASSERT3U(mintxg2
, ==, dsl_dataset_phys(ds2
)->ds_prev_snap_txg
);
2669 if (ds1_queued
&& ds2_queued
) {
2671 * If both are queued, we don't need to do anything.
2672 * Alternatively, we could check for EEXIST from
2673 * zap_add_int_key() and back out to the original state, but
2674 * that would be more work than checking for this case upfront.
2676 } else if (ds1_queued
) {
2677 VERIFY3S(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2678 scn
->scn_phys
.scn_queue_obj
, ds1
->ds_object
, tx
));
2679 VERIFY3S(0, ==, zap_add_int_key(dp
->dp_meta_objset
,
2680 scn
->scn_phys
.scn_queue_obj
, ds2
->ds_object
, mintxg1
, tx
));
2681 zfs_dbgmsg("clone_swap ds %llu on %s; in queue; "
2682 "replacing with %llu",
2683 (u_longlong_t
)ds1
->ds_object
,
2684 dp
->dp_spa
->spa_name
,
2685 (u_longlong_t
)ds2
->ds_object
);
2686 } else if (ds2_queued
) {
2687 VERIFY3S(0, ==, zap_remove_int(dp
->dp_meta_objset
,
2688 scn
->scn_phys
.scn_queue_obj
, ds2
->ds_object
, tx
));
2689 VERIFY3S(0, ==, zap_add_int_key(dp
->dp_meta_objset
,
2690 scn
->scn_phys
.scn_queue_obj
, ds1
->ds_object
, mintxg2
, tx
));
2691 zfs_dbgmsg("clone_swap ds %llu on %s; in queue; "
2692 "replacing with %llu",
2693 (u_longlong_t
)ds2
->ds_object
,
2694 dp
->dp_spa
->spa_name
,
2695 (u_longlong_t
)ds1
->ds_object
);
2698 dsl_scan_sync_state(scn
, tx
, SYNC_CACHED
);
2702 enqueue_clones_cb(dsl_pool_t
*dp
, dsl_dataset_t
*hds
, void *arg
)
2704 uint64_t originobj
= *(uint64_t *)arg
;
2707 dsl_scan_t
*scn
= dp
->dp_scan
;
2709 if (dsl_dir_phys(hds
->ds_dir
)->dd_origin_obj
!= originobj
)
2712 err
= dsl_dataset_hold_obj(dp
, hds
->ds_object
, FTAG
, &ds
);
2716 while (dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= originobj
) {
2717 dsl_dataset_t
*prev
;
2718 err
= dsl_dataset_hold_obj(dp
,
2719 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, FTAG
, &prev
);
2721 dsl_dataset_rele(ds
, FTAG
);
2726 scan_ds_queue_insert(scn
, ds
->ds_object
,
2727 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
2728 dsl_dataset_rele(ds
, FTAG
);
2733 dsl_scan_visitds(dsl_scan_t
*scn
, uint64_t dsobj
, dmu_tx_t
*tx
)
2735 dsl_pool_t
*dp
= scn
->scn_dp
;
2738 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp
, dsobj
, FTAG
, &ds
));
2740 if (scn
->scn_phys
.scn_cur_min_txg
>=
2741 scn
->scn_phys
.scn_max_txg
) {
2743 * This can happen if this snapshot was created after the
2744 * scan started, and we already completed a previous snapshot
2745 * that was created after the scan started. This snapshot
2746 * only references blocks with:
2748 * birth < our ds_creation_txg
2749 * cur_min_txg is no less than ds_creation_txg.
2750 * We have already visited these blocks.
2752 * birth > scn_max_txg
2753 * The scan requested not to visit these blocks.
2755 * Subsequent snapshots (and clones) can reference our
2756 * blocks, or blocks with even higher birth times.
2757 * Therefore we do not need to visit them either,
2758 * so we do not add them to the work queue.
2760 * Note that checking for cur_min_txg >= cur_max_txg
2761 * is not sufficient, because in that case we may need to
2762 * visit subsequent snapshots. This happens when min_txg > 0,
2763 * which raises cur_min_txg. In this case we will visit
2764 * this dataset but skip all of its blocks, because the
2765 * rootbp's birth time is < cur_min_txg. Then we will
2766 * add the next snapshots/clones to the work queue.
2768 char *dsname
= kmem_alloc(ZFS_MAX_DATASET_NAME_LEN
, KM_SLEEP
);
2769 dsl_dataset_name(ds
, dsname
);
2770 zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because "
2771 "cur_min_txg (%llu) >= max_txg (%llu)",
2772 (longlong_t
)dsobj
, dsname
,
2773 (longlong_t
)scn
->scn_phys
.scn_cur_min_txg
,
2774 (longlong_t
)scn
->scn_phys
.scn_max_txg
);
2775 kmem_free(dsname
, MAXNAMELEN
);
2781 * Only the ZIL in the head (non-snapshot) is valid. Even though
2782 * snapshots can have ZIL block pointers (which may be the same
2783 * BP as in the head), they must be ignored. In addition, $ORIGIN
2784 * doesn't have a objset (i.e. its ds_bp is a hole) so we don't
2785 * need to look for a ZIL in it either. So we traverse the ZIL here,
2786 * rather than in scan_recurse(), because the regular snapshot
2787 * block-sharing rules don't apply to it.
2789 if (!dsl_dataset_is_snapshot(ds
) &&
2790 (dp
->dp_origin_snap
== NULL
||
2791 ds
->ds_dir
!= dp
->dp_origin_snap
->ds_dir
)) {
2793 if (dmu_objset_from_ds(ds
, &os
) != 0) {
2796 dsl_scan_zil(dp
, &os
->os_zil_header
);
2800 * Iterate over the bps in this ds.
2802 dmu_buf_will_dirty(ds
->ds_dbuf
, tx
);
2803 rrw_enter(&ds
->ds_bp_rwlock
, RW_READER
, FTAG
);
2804 dsl_scan_visit_rootbp(scn
, ds
, &dsl_dataset_phys(ds
)->ds_bp
, tx
);
2805 rrw_exit(&ds
->ds_bp_rwlock
, FTAG
);
2807 char *dsname
= kmem_alloc(ZFS_MAX_DATASET_NAME_LEN
, KM_SLEEP
);
2808 dsl_dataset_name(ds
, dsname
);
2809 zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; "
2811 (longlong_t
)dsobj
, dsname
,
2812 (longlong_t
)scn
->scn_phys
.scn_cur_min_txg
,
2813 (longlong_t
)scn
->scn_phys
.scn_cur_max_txg
,
2814 (int)scn
->scn_suspending
);
2815 kmem_free(dsname
, ZFS_MAX_DATASET_NAME_LEN
);
2817 if (scn
->scn_suspending
)
2821 * We've finished this pass over this dataset.
2825 * If we did not completely visit this dataset, do another pass.
2827 if (scn
->scn_phys
.scn_flags
& DSF_VISIT_DS_AGAIN
) {
2828 zfs_dbgmsg("incomplete pass on %s; visiting again",
2829 dp
->dp_spa
->spa_name
);
2830 scn
->scn_phys
.scn_flags
&= ~DSF_VISIT_DS_AGAIN
;
2831 scan_ds_queue_insert(scn
, ds
->ds_object
,
2832 scn
->scn_phys
.scn_cur_max_txg
);
2837 * Add descendant datasets to work queue.
2839 if (dsl_dataset_phys(ds
)->ds_next_snap_obj
!= 0) {
2840 scan_ds_queue_insert(scn
,
2841 dsl_dataset_phys(ds
)->ds_next_snap_obj
,
2842 dsl_dataset_phys(ds
)->ds_creation_txg
);
2844 if (dsl_dataset_phys(ds
)->ds_num_children
> 1) {
2845 boolean_t usenext
= B_FALSE
;
2846 if (dsl_dataset_phys(ds
)->ds_next_clones_obj
!= 0) {
2849 * A bug in a previous version of the code could
2850 * cause upgrade_clones_cb() to not set
2851 * ds_next_snap_obj when it should, leading to a
2852 * missing entry. Therefore we can only use the
2853 * next_clones_obj when its count is correct.
2855 int err
= zap_count(dp
->dp_meta_objset
,
2856 dsl_dataset_phys(ds
)->ds_next_clones_obj
, &count
);
2858 count
== dsl_dataset_phys(ds
)->ds_num_children
- 1)
2865 for (zap_cursor_init(&zc
, dp
->dp_meta_objset
,
2866 dsl_dataset_phys(ds
)->ds_next_clones_obj
);
2867 zap_cursor_retrieve(&zc
, &za
) == 0;
2868 (void) zap_cursor_advance(&zc
)) {
2869 scan_ds_queue_insert(scn
,
2870 zfs_strtonum(za
.za_name
, NULL
),
2871 dsl_dataset_phys(ds
)->ds_creation_txg
);
2873 zap_cursor_fini(&zc
);
2875 VERIFY0(dmu_objset_find_dp(dp
, dp
->dp_root_dir_obj
,
2876 enqueue_clones_cb
, &ds
->ds_object
,
2882 dsl_dataset_rele(ds
, FTAG
);
2886 enqueue_cb(dsl_pool_t
*dp
, dsl_dataset_t
*hds
, void *arg
)
2891 dsl_scan_t
*scn
= dp
->dp_scan
;
2893 err
= dsl_dataset_hold_obj(dp
, hds
->ds_object
, FTAG
, &ds
);
2897 while (dsl_dataset_phys(ds
)->ds_prev_snap_obj
!= 0) {
2898 dsl_dataset_t
*prev
;
2899 err
= dsl_dataset_hold_obj(dp
,
2900 dsl_dataset_phys(ds
)->ds_prev_snap_obj
, FTAG
, &prev
);
2902 dsl_dataset_rele(ds
, FTAG
);
2907 * If this is a clone, we don't need to worry about it for now.
2909 if (dsl_dataset_phys(prev
)->ds_next_snap_obj
!= ds
->ds_object
) {
2910 dsl_dataset_rele(ds
, FTAG
);
2911 dsl_dataset_rele(prev
, FTAG
);
2914 dsl_dataset_rele(ds
, FTAG
);
2918 scan_ds_queue_insert(scn
, ds
->ds_object
,
2919 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
2920 dsl_dataset_rele(ds
, FTAG
);
2925 dsl_scan_ddt_entry(dsl_scan_t
*scn
, enum zio_checksum checksum
,
2926 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
2929 const ddt_key_t
*ddk
= &dde
->dde_key
;
2930 ddt_phys_t
*ddp
= dde
->dde_phys
;
2932 zbookmark_phys_t zb
= { 0 };
2934 if (!dsl_scan_is_running(scn
))
2938 * This function is special because it is the only thing
2939 * that can add scan_io_t's to the vdev scan queues from
2940 * outside dsl_scan_sync(). For the most part this is ok
2941 * as long as it is called from within syncing context.
2942 * However, dsl_scan_sync() expects that no new sio's will
2943 * be added between when all the work for a scan is done
2944 * and the next txg when the scan is actually marked as
2945 * completed. This check ensures we do not issue new sio's
2946 * during this period.
2948 if (scn
->scn_done_txg
!= 0)
2951 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
2952 if (ddp
->ddp_phys_birth
== 0 ||
2953 ddp
->ddp_phys_birth
> scn
->scn_phys
.scn_max_txg
)
2955 ddt_bp_create(checksum
, ddk
, ddp
, &bp
);
2957 scn
->scn_visited_this_txg
++;
2958 scan_funcs
[scn
->scn_phys
.scn_func
](scn
->scn_dp
, &bp
, &zb
);
2963 * Scrub/dedup interaction.
2965 * If there are N references to a deduped block, we don't want to scrub it
2966 * N times -- ideally, we should scrub it exactly once.
2968 * We leverage the fact that the dde's replication class (ddt_class_t)
2969 * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest
2970 * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order.
2972 * To prevent excess scrubbing, the scrub begins by walking the DDT
2973 * to find all blocks with refcnt > 1, and scrubs each of these once.
2974 * Since there are two replication classes which contain blocks with
2975 * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first.
2976 * Finally the top-down scrub begins, only visiting blocks with refcnt == 1.
2978 * There would be nothing more to say if a block's refcnt couldn't change
2979 * during a scrub, but of course it can so we must account for changes
2980 * in a block's replication class.
2982 * Here's an example of what can occur:
2984 * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1
2985 * when visited during the top-down scrub phase, it will be scrubbed twice.
2986 * This negates our scrub optimization, but is otherwise harmless.
2988 * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1
2989 * on each visit during the top-down scrub phase, it will never be scrubbed.
2990 * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's
2991 * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to
2992 * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1
2993 * while a scrub is in progress, it scrubs the block right then.
2996 dsl_scan_ddt(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
2998 ddt_bookmark_t
*ddb
= &scn
->scn_phys
.scn_ddt_bookmark
;
2999 ddt_entry_t dde
= {{{{0}}}};
3003 while ((error
= ddt_walk(scn
->scn_dp
->dp_spa
, ddb
, &dde
)) == 0) {
3006 if (ddb
->ddb_class
> scn
->scn_phys
.scn_ddt_class_max
)
3008 dprintf("visiting ddb=%llu/%llu/%llu/%llx\n",
3009 (longlong_t
)ddb
->ddb_class
,
3010 (longlong_t
)ddb
->ddb_type
,
3011 (longlong_t
)ddb
->ddb_checksum
,
3012 (longlong_t
)ddb
->ddb_cursor
);
3014 /* There should be no pending changes to the dedup table */
3015 ddt
= scn
->scn_dp
->dp_spa
->spa_ddt
[ddb
->ddb_checksum
];
3016 ASSERT(avl_first(&ddt
->ddt_tree
) == NULL
);
3018 dsl_scan_ddt_entry(scn
, ddb
->ddb_checksum
, &dde
, tx
);
3021 if (dsl_scan_check_suspend(scn
, NULL
))
3025 zfs_dbgmsg("scanned %llu ddt entries on %s with class_max = %u; "
3026 "suspending=%u", (longlong_t
)n
, scn
->scn_dp
->dp_spa
->spa_name
,
3027 (int)scn
->scn_phys
.scn_ddt_class_max
, (int)scn
->scn_suspending
);
3029 ASSERT(error
== 0 || error
== ENOENT
);
3030 ASSERT(error
!= ENOENT
||
3031 ddb
->ddb_class
> scn
->scn_phys
.scn_ddt_class_max
);
3035 dsl_scan_ds_maxtxg(dsl_dataset_t
*ds
)
3037 uint64_t smt
= ds
->ds_dir
->dd_pool
->dp_scan
->scn_phys
.scn_max_txg
;
3038 if (ds
->ds_is_snapshot
)
3039 return (MIN(smt
, dsl_dataset_phys(ds
)->ds_creation_txg
));
3044 dsl_scan_visit(dsl_scan_t
*scn
, dmu_tx_t
*tx
)
3047 dsl_pool_t
*dp
= scn
->scn_dp
;
3049 if (scn
->scn_phys
.scn_ddt_bookmark
.ddb_class
<=
3050 scn
->scn_phys
.scn_ddt_class_max
) {
3051 scn
->scn_phys
.scn_cur_min_txg
= scn
->scn_phys
.scn_min_txg
;
3052 scn
->scn_phys
.scn_cur_max_txg
= scn
->scn_phys
.scn_max_txg
;
3053 dsl_scan_ddt(scn
, tx
);
3054 if (scn
->scn_suspending
)
3058 if (scn
->scn_phys
.scn_bookmark
.zb_objset
== DMU_META_OBJSET
) {
3059 /* First do the MOS & ORIGIN */
3061 scn
->scn_phys
.scn_cur_min_txg
= scn
->scn_phys
.scn_min_txg
;
3062 scn
->scn_phys
.scn_cur_max_txg
= scn
->scn_phys
.scn_max_txg
;
3063 dsl_scan_visit_rootbp(scn
, NULL
,
3064 &dp
->dp_meta_rootbp
, tx
);
3065 if (scn
->scn_suspending
)
3068 if (spa_version(dp
->dp_spa
) < SPA_VERSION_DSL_SCRUB
) {
3069 VERIFY0(dmu_objset_find_dp(dp
, dp
->dp_root_dir_obj
,
3070 enqueue_cb
, NULL
, DS_FIND_CHILDREN
));
3072 dsl_scan_visitds(scn
,
3073 dp
->dp_origin_snap
->ds_object
, tx
);
3075 ASSERT(!scn
->scn_suspending
);
3076 } else if (scn
->scn_phys
.scn_bookmark
.zb_objset
!=
3077 ZB_DESTROYED_OBJSET
) {
3078 uint64_t dsobj
= scn
->scn_phys
.scn_bookmark
.zb_objset
;
3080 * If we were suspended, continue from here. Note if the
3081 * ds we were suspended on was deleted, the zb_objset may
3082 * be -1, so we will skip this and find a new objset
3085 dsl_scan_visitds(scn
, dsobj
, tx
);
3086 if (scn
->scn_suspending
)
3091 * In case we suspended right at the end of the ds, zero the
3092 * bookmark so we don't think that we're still trying to resume.
3094 memset(&scn
->scn_phys
.scn_bookmark
, 0, sizeof (zbookmark_phys_t
));
3097 * Keep pulling things out of the dataset avl queue. Updates to the
3098 * persistent zap-object-as-queue happen only at checkpoints.
3100 while ((sds
= avl_first(&scn
->scn_queue
)) != NULL
) {
3102 uint64_t dsobj
= sds
->sds_dsobj
;
3103 uint64_t txg
= sds
->sds_txg
;
3105 /* dequeue and free the ds from the queue */
3106 scan_ds_queue_remove(scn
, dsobj
);
3109 /* set up min / max txg */
3110 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp
, dsobj
, FTAG
, &ds
));
3112 scn
->scn_phys
.scn_cur_min_txg
=
3113 MAX(scn
->scn_phys
.scn_min_txg
, txg
);
3115 scn
->scn_phys
.scn_cur_min_txg
=
3116 MAX(scn
->scn_phys
.scn_min_txg
,
3117 dsl_dataset_phys(ds
)->ds_prev_snap_txg
);
3119 scn
->scn_phys
.scn_cur_max_txg
= dsl_scan_ds_maxtxg(ds
);
3120 dsl_dataset_rele(ds
, FTAG
);
3122 dsl_scan_visitds(scn
, dsobj
, tx
);
3123 if (scn
->scn_suspending
)
3127 /* No more objsets to fetch, we're done */
3128 scn
->scn_phys
.scn_bookmark
.zb_objset
= ZB_DESTROYED_OBJSET
;
3129 ASSERT0(scn
->scn_suspending
);
3133 dsl_scan_count_data_disks(spa_t
*spa
)
3135 vdev_t
*rvd
= spa
->spa_root_vdev
;
3136 uint64_t i
, leaves
= 0;
3138 for (i
= 0; i
< rvd
->vdev_children
; i
++) {
3139 vdev_t
*vd
= rvd
->vdev_child
[i
];
3140 if (vd
->vdev_islog
|| vd
->vdev_isspare
|| vd
->vdev_isl2cache
)
3142 leaves
+= vdev_get_ndisks(vd
) - vdev_get_nparity(vd
);
3148 scan_io_queues_update_zio_stats(dsl_scan_io_queue_t
*q
, const blkptr_t
*bp
)
3151 uint64_t cur_size
= 0;
3153 for (i
= 0; i
< BP_GET_NDVAS(bp
); i
++) {
3154 cur_size
+= DVA_GET_ASIZE(&bp
->blk_dva
[i
]);
3157 q
->q_total_zio_size_this_txg
+= cur_size
;
3158 q
->q_zios_this_txg
++;
3162 scan_io_queues_update_seg_stats(dsl_scan_io_queue_t
*q
, uint64_t start
,
3165 q
->q_total_seg_size_this_txg
+= end
- start
;
3166 q
->q_segs_this_txg
++;
3170 scan_io_queue_check_suspend(dsl_scan_t
*scn
)
3172 /* See comment in dsl_scan_check_suspend() */
3173 uint64_t curr_time_ns
= gethrtime();
3174 uint64_t scan_time_ns
= curr_time_ns
- scn
->scn_sync_start_time
;
3175 uint64_t sync_time_ns
= curr_time_ns
-
3176 scn
->scn_dp
->dp_spa
->spa_sync_starttime
;
3177 uint64_t dirty_min_bytes
= zfs_dirty_data_max
*
3178 zfs_vdev_async_write_active_min_dirty_percent
/ 100;
3179 uint_t mintime
= (scn
->scn_phys
.scn_func
== POOL_SCAN_RESILVER
) ?
3180 zfs_resilver_min_time_ms
: zfs_scrub_min_time_ms
;
3182 return ((NSEC2MSEC(scan_time_ns
) > mintime
&&
3183 (scn
->scn_dp
->dp_dirty_total
>= dirty_min_bytes
||
3184 txg_sync_waiting(scn
->scn_dp
) ||
3185 NSEC2SEC(sync_time_ns
) >= zfs_txg_timeout
)) ||
3186 spa_shutting_down(scn
->scn_dp
->dp_spa
));
3190 * Given a list of scan_io_t's in io_list, this issues the I/Os out to
3191 * disk. This consumes the io_list and frees the scan_io_t's. This is
3192 * called when emptying queues, either when we're up against the memory
3193 * limit or when we have finished scanning. Returns B_TRUE if we stopped
3194 * processing the list before we finished. Any sios that were not issued
3195 * will remain in the io_list.
3198 scan_io_queue_issue(dsl_scan_io_queue_t
*queue
, list_t
*io_list
)
3200 dsl_scan_t
*scn
= queue
->q_scn
;
3202 boolean_t suspended
= B_FALSE
;
3204 while ((sio
= list_head(io_list
)) != NULL
) {
3207 if (scan_io_queue_check_suspend(scn
)) {
3213 scan_exec_io(scn
->scn_dp
, &bp
, sio
->sio_flags
,
3214 &sio
->sio_zb
, queue
);
3215 (void) list_remove_head(io_list
);
3216 scan_io_queues_update_zio_stats(queue
, &bp
);
3223 * This function removes sios from an IO queue which reside within a given
3224 * range_seg_t and inserts them (in offset order) into a list. Note that
3225 * we only ever return a maximum of 32 sios at once. If there are more sios
3226 * to process within this segment that did not make it onto the list we
3227 * return B_TRUE and otherwise B_FALSE.
3230 scan_io_queue_gather(dsl_scan_io_queue_t
*queue
, range_seg_t
*rs
, list_t
*list
)
3232 scan_io_t
*srch_sio
, *sio
, *next_sio
;
3234 uint_t num_sios
= 0;
3235 int64_t bytes_issued
= 0;
3238 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
3240 srch_sio
= sio_alloc(1);
3241 srch_sio
->sio_nr_dvas
= 1;
3242 SIO_SET_OFFSET(srch_sio
, rs_get_start(rs
, queue
->q_exts_by_addr
));
3245 * The exact start of the extent might not contain any matching zios,
3246 * so if that's the case, examine the next one in the tree.
3248 sio
= avl_find(&queue
->q_sios_by_addr
, srch_sio
, &idx
);
3252 sio
= avl_nearest(&queue
->q_sios_by_addr
, idx
, AVL_AFTER
);
3254 while (sio
!= NULL
&& SIO_GET_OFFSET(sio
) < rs_get_end(rs
,
3255 queue
->q_exts_by_addr
) && num_sios
<= 32) {
3256 ASSERT3U(SIO_GET_OFFSET(sio
), >=, rs_get_start(rs
,
3257 queue
->q_exts_by_addr
));
3258 ASSERT3U(SIO_GET_END_OFFSET(sio
), <=, rs_get_end(rs
,
3259 queue
->q_exts_by_addr
));
3261 next_sio
= AVL_NEXT(&queue
->q_sios_by_addr
, sio
);
3262 avl_remove(&queue
->q_sios_by_addr
, sio
);
3263 if (avl_is_empty(&queue
->q_sios_by_addr
))
3264 atomic_add_64(&queue
->q_scn
->scn_queues_pending
, -1);
3265 queue
->q_sio_memused
-= SIO_GET_MUSED(sio
);
3267 bytes_issued
+= SIO_GET_ASIZE(sio
);
3269 list_insert_tail(list
, sio
);
3274 * We limit the number of sios we process at once to 32 to avoid
3275 * biting off more than we can chew. If we didn't take everything
3276 * in the segment we update it to reflect the work we were able to
3277 * complete. Otherwise, we remove it from the range tree entirely.
3279 if (sio
!= NULL
&& SIO_GET_OFFSET(sio
) < rs_get_end(rs
,
3280 queue
->q_exts_by_addr
)) {
3281 range_tree_adjust_fill(queue
->q_exts_by_addr
, rs
,
3283 range_tree_resize_segment(queue
->q_exts_by_addr
, rs
,
3284 SIO_GET_OFFSET(sio
), rs_get_end(rs
,
3285 queue
->q_exts_by_addr
) - SIO_GET_OFFSET(sio
));
3286 queue
->q_last_ext_addr
= SIO_GET_OFFSET(sio
);
3289 uint64_t rstart
= rs_get_start(rs
, queue
->q_exts_by_addr
);
3290 uint64_t rend
= rs_get_end(rs
, queue
->q_exts_by_addr
);
3291 range_tree_remove(queue
->q_exts_by_addr
, rstart
, rend
- rstart
);
3292 queue
->q_last_ext_addr
= -1;
3298 * This is called from the queue emptying thread and selects the next
3299 * extent from which we are to issue I/Os. The behavior of this function
3300 * depends on the state of the scan, the current memory consumption and
3301 * whether or not we are performing a scan shutdown.
3302 * 1) We select extents in an elevator algorithm (LBA-order) if the scan
3303 * needs to perform a checkpoint
3304 * 2) We select the largest available extent if we are up against the
3306 * 3) Otherwise we don't select any extents.
3308 static range_seg_t
*
3309 scan_io_queue_fetch_ext(dsl_scan_io_queue_t
*queue
)
3311 dsl_scan_t
*scn
= queue
->q_scn
;
3312 range_tree_t
*rt
= queue
->q_exts_by_addr
;
3314 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
3315 ASSERT(scn
->scn_is_sorted
);
3317 if (!scn
->scn_checkpointing
&& !scn
->scn_clearing
)
3321 * During normal clearing, we want to issue our largest segments
3322 * first, keeping IO as sequential as possible, and leaving the
3323 * smaller extents for later with the hope that they might eventually
3324 * grow to larger sequential segments. However, when the scan is
3325 * checkpointing, no new extents will be added to the sorting queue,
3326 * so the way we are sorted now is as good as it will ever get.
3327 * In this case, we instead switch to issuing extents in LBA order.
3329 if ((zfs_scan_issue_strategy
< 1 && scn
->scn_checkpointing
) ||
3330 zfs_scan_issue_strategy
== 1)
3331 return (range_tree_first(rt
));
3334 * Try to continue previous extent if it is not completed yet. After
3335 * shrink in scan_io_queue_gather() it may no longer be the best, but
3336 * otherwise we leave shorter remnant every txg.
3339 uint64_t size
= 1ULL << rt
->rt_shift
;
3340 range_seg_t
*addr_rs
;
3341 if (queue
->q_last_ext_addr
!= -1) {
3342 start
= queue
->q_last_ext_addr
;
3343 addr_rs
= range_tree_find(rt
, start
, size
);
3344 if (addr_rs
!= NULL
)
3349 * Nothing to continue, so find new best extent.
3351 uint64_t *v
= zfs_btree_first(&queue
->q_exts_by_size
, NULL
);
3354 queue
->q_last_ext_addr
= start
= *v
<< rt
->rt_shift
;
3357 * We need to get the original entry in the by_addr tree so we can
3360 addr_rs
= range_tree_find(rt
, start
, size
);
3361 ASSERT3P(addr_rs
, !=, NULL
);
3362 ASSERT3U(rs_get_start(addr_rs
, rt
), ==, start
);
3363 ASSERT3U(rs_get_end(addr_rs
, rt
), >, start
);
3368 scan_io_queues_run_one(void *arg
)
3370 dsl_scan_io_queue_t
*queue
= arg
;
3371 kmutex_t
*q_lock
= &queue
->q_vd
->vdev_scan_io_queue_lock
;
3372 boolean_t suspended
= B_FALSE
;
3378 ASSERT(queue
->q_scn
->scn_is_sorted
);
3380 list_create(&sio_list
, sizeof (scan_io_t
),
3381 offsetof(scan_io_t
, sio_nodes
.sio_list_node
));
3382 zio
= zio_null(queue
->q_scn
->scn_zio_root
, queue
->q_scn
->scn_dp
->dp_spa
,
3383 NULL
, NULL
, NULL
, ZIO_FLAG_CANFAIL
);
3384 mutex_enter(q_lock
);
3387 /* Calculate maximum in-flight bytes for this vdev. */
3388 queue
->q_maxinflight_bytes
= MAX(1, zfs_scan_vdev_limit
*
3389 (vdev_get_ndisks(queue
->q_vd
) - vdev_get_nparity(queue
->q_vd
)));
3391 /* reset per-queue scan statistics for this txg */
3392 queue
->q_total_seg_size_this_txg
= 0;
3393 queue
->q_segs_this_txg
= 0;
3394 queue
->q_total_zio_size_this_txg
= 0;
3395 queue
->q_zios_this_txg
= 0;
3397 /* loop until we run out of time or sios */
3398 while ((rs
= scan_io_queue_fetch_ext(queue
)) != NULL
) {
3399 uint64_t seg_start
= 0, seg_end
= 0;
3400 boolean_t more_left
;
3402 ASSERT(list_is_empty(&sio_list
));
3404 /* loop while we still have sios left to process in this rs */
3406 scan_io_t
*first_sio
, *last_sio
;
3409 * We have selected which extent needs to be
3410 * processed next. Gather up the corresponding sios.
3412 more_left
= scan_io_queue_gather(queue
, rs
, &sio_list
);
3413 ASSERT(!list_is_empty(&sio_list
));
3414 first_sio
= list_head(&sio_list
);
3415 last_sio
= list_tail(&sio_list
);
3417 seg_end
= SIO_GET_END_OFFSET(last_sio
);
3419 seg_start
= SIO_GET_OFFSET(first_sio
);
3422 * Issuing sios can take a long time so drop the
3423 * queue lock. The sio queue won't be updated by
3424 * other threads since we're in syncing context so
3425 * we can be sure that our trees will remain exactly
3429 suspended
= scan_io_queue_issue(queue
, &sio_list
);
3430 mutex_enter(q_lock
);
3434 } while (more_left
);
3436 /* update statistics for debugging purposes */
3437 scan_io_queues_update_seg_stats(queue
, seg_start
, seg_end
);
3444 * If we were suspended in the middle of processing,
3445 * requeue any unfinished sios and exit.
3447 while ((sio
= list_remove_head(&sio_list
)) != NULL
)
3448 scan_io_queue_insert_impl(queue
, sio
);
3450 queue
->q_zio
= NULL
;
3453 list_destroy(&sio_list
);
3457 * Performs an emptying run on all scan queues in the pool. This just
3458 * punches out one thread per top-level vdev, each of which processes
3459 * only that vdev's scan queue. We can parallelize the I/O here because
3460 * we know that each queue's I/Os only affect its own top-level vdev.
3462 * This function waits for the queue runs to complete, and must be
3463 * called from dsl_scan_sync (or in general, syncing context).
3466 scan_io_queues_run(dsl_scan_t
*scn
)
3468 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
3470 ASSERT(scn
->scn_is_sorted
);
3471 ASSERT(spa_config_held(spa
, SCL_CONFIG
, RW_READER
));
3473 if (scn
->scn_queues_pending
== 0)
3476 if (scn
->scn_taskq
== NULL
) {
3477 int nthreads
= spa
->spa_root_vdev
->vdev_children
;
3480 * We need to make this taskq *always* execute as many
3481 * threads in parallel as we have top-level vdevs and no
3482 * less, otherwise strange serialization of the calls to
3483 * scan_io_queues_run_one can occur during spa_sync runs
3484 * and that significantly impacts performance.
3486 scn
->scn_taskq
= taskq_create("dsl_scan_iss", nthreads
,
3487 minclsyspri
, nthreads
, nthreads
, TASKQ_PREPOPULATE
);
3490 for (uint64_t i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
3491 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
3493 mutex_enter(&vd
->vdev_scan_io_queue_lock
);
3494 if (vd
->vdev_scan_io_queue
!= NULL
) {
3495 VERIFY(taskq_dispatch(scn
->scn_taskq
,
3496 scan_io_queues_run_one
, vd
->vdev_scan_io_queue
,
3497 TQ_SLEEP
) != TASKQID_INVALID
);
3499 mutex_exit(&vd
->vdev_scan_io_queue_lock
);
3503 * Wait for the queues to finish issuing their IOs for this run
3504 * before we return. There may still be IOs in flight at this
3507 taskq_wait(scn
->scn_taskq
);
3511 dsl_scan_async_block_should_pause(dsl_scan_t
*scn
)
3513 uint64_t elapsed_nanosecs
;
3518 if (zfs_async_block_max_blocks
!= 0 &&
3519 scn
->scn_visited_this_txg
>= zfs_async_block_max_blocks
) {
3523 if (zfs_max_async_dedup_frees
!= 0 &&
3524 scn
->scn_dedup_frees_this_txg
>= zfs_max_async_dedup_frees
) {
3528 elapsed_nanosecs
= gethrtime() - scn
->scn_sync_start_time
;
3529 return (elapsed_nanosecs
/ NANOSEC
> zfs_txg_timeout
||
3530 (NSEC2MSEC(elapsed_nanosecs
) > scn
->scn_async_block_min_time_ms
&&
3531 txg_sync_waiting(scn
->scn_dp
)) ||
3532 spa_shutting_down(scn
->scn_dp
->dp_spa
));
3536 dsl_scan_free_block_cb(void *arg
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
3538 dsl_scan_t
*scn
= arg
;
3540 if (!scn
->scn_is_bptree
||
3541 (BP_GET_LEVEL(bp
) == 0 && BP_GET_TYPE(bp
) != DMU_OT_OBJSET
)) {
3542 if (dsl_scan_async_block_should_pause(scn
))
3543 return (SET_ERROR(ERESTART
));
3546 zio_nowait(zio_free_sync(scn
->scn_zio_root
, scn
->scn_dp
->dp_spa
,
3547 dmu_tx_get_txg(tx
), bp
, 0));
3548 dsl_dir_diduse_space(tx
->tx_pool
->dp_free_dir
, DD_USED_HEAD
,
3549 -bp_get_dsize_sync(scn
->scn_dp
->dp_spa
, bp
),
3550 -BP_GET_PSIZE(bp
), -BP_GET_UCSIZE(bp
), tx
);
3551 scn
->scn_visited_this_txg
++;
3552 if (BP_GET_DEDUP(bp
))
3553 scn
->scn_dedup_frees_this_txg
++;
3558 dsl_scan_update_stats(dsl_scan_t
*scn
)
3560 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
3562 uint64_t seg_size_total
= 0, zio_size_total
= 0;
3563 uint64_t seg_count_total
= 0, zio_count_total
= 0;
3565 for (i
= 0; i
< spa
->spa_root_vdev
->vdev_children
; i
++) {
3566 vdev_t
*vd
= spa
->spa_root_vdev
->vdev_child
[i
];
3567 dsl_scan_io_queue_t
*queue
= vd
->vdev_scan_io_queue
;
3572 seg_size_total
+= queue
->q_total_seg_size_this_txg
;
3573 zio_size_total
+= queue
->q_total_zio_size_this_txg
;
3574 seg_count_total
+= queue
->q_segs_this_txg
;
3575 zio_count_total
+= queue
->q_zios_this_txg
;
3578 if (seg_count_total
== 0 || zio_count_total
== 0) {
3579 scn
->scn_avg_seg_size_this_txg
= 0;
3580 scn
->scn_avg_zio_size_this_txg
= 0;
3581 scn
->scn_segs_this_txg
= 0;
3582 scn
->scn_zios_this_txg
= 0;
3586 scn
->scn_avg_seg_size_this_txg
= seg_size_total
/ seg_count_total
;
3587 scn
->scn_avg_zio_size_this_txg
= zio_size_total
/ zio_count_total
;
3588 scn
->scn_segs_this_txg
= seg_count_total
;
3589 scn
->scn_zios_this_txg
= zio_count_total
;
3593 bpobj_dsl_scan_free_block_cb(void *arg
, const blkptr_t
*bp
, boolean_t bp_freed
,
3597 return (dsl_scan_free_block_cb(arg
, bp
, tx
));
3601 dsl_scan_obsolete_block_cb(void *arg
, const blkptr_t
*bp
, boolean_t bp_freed
,
3605 dsl_scan_t
*scn
= arg
;
3606 const dva_t
*dva
= &bp
->blk_dva
[0];
3608 if (dsl_scan_async_block_should_pause(scn
))
3609 return (SET_ERROR(ERESTART
));
3611 spa_vdev_indirect_mark_obsolete(scn
->scn_dp
->dp_spa
,
3612 DVA_GET_VDEV(dva
), DVA_GET_OFFSET(dva
),
3613 DVA_GET_ASIZE(dva
), tx
);
3614 scn
->scn_visited_this_txg
++;
3619 dsl_scan_active(dsl_scan_t
*scn
)
3621 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
3622 uint64_t used
= 0, comp
, uncomp
;
3623 boolean_t clones_left
;
3625 if (spa
->spa_load_state
!= SPA_LOAD_NONE
)
3627 if (spa_shutting_down(spa
))
3629 if ((dsl_scan_is_running(scn
) && !dsl_scan_is_paused_scrub(scn
)) ||
3630 (scn
->scn_async_destroying
&& !scn
->scn_async_stalled
))
3633 if (spa_version(scn
->scn_dp
->dp_spa
) >= SPA_VERSION_DEADLISTS
) {
3634 (void) bpobj_space(&scn
->scn_dp
->dp_free_bpobj
,
3635 &used
, &comp
, &uncomp
);
3637 clones_left
= spa_livelist_delete_check(spa
);
3638 return ((used
!= 0) || (clones_left
));
3642 dsl_errorscrub_active(dsl_scan_t
*scn
)
3644 spa_t
*spa
= scn
->scn_dp
->dp_spa
;
3645 if (spa
->spa_load_state
!= SPA_LOAD_NONE
)
3647 if (spa_shutting_down(spa
))
3649 if (dsl_errorscrubbing(scn
->scn_dp
))
3655 dsl_scan_check_deferred(vdev_t
*vd
)
3657 boolean_t need_resilver
= B_FALSE
;
3659 for (int c
= 0; c
< vd
->vdev_children
; c
++) {
3661 dsl_scan_check_deferred(vd
->vdev_child
[c
]);
3664 if (!vdev_is_concrete(vd
) || vd
->vdev_aux
||
3665 !vd
->vdev_ops
->vdev_op_leaf
)
3666 return (need_resilver
);
3668 if (!vd
->vdev_resilver_deferred
)
3669 need_resilver
= B_TRUE
;
3671 return (need_resilver
);
3675 dsl_scan_need_resilver(spa_t
*spa
, const dva_t
*dva
, size_t psize
,
3676 uint64_t phys_birth
)
3680 vd
= vdev_lookup_top(spa
, DVA_GET_VDEV(dva
));
3682 if (vd
->vdev_ops
== &vdev_indirect_ops
) {
3684 * The indirect vdev can point to multiple
3685 * vdevs. For simplicity, always create
3686 * the resilver zio_t. zio_vdev_io_start()
3687 * will bypass the child resilver i/o's if
3688 * they are on vdevs that don't have DTL's.
3693 if (DVA_GET_GANG(dva
)) {
3695 * Gang members may be spread across multiple
3696 * vdevs, so the best estimate we have is the
3697 * scrub range, which has already been checked.
3698 * XXX -- it would be better to change our
3699 * allocation policy to ensure that all
3700 * gang members reside on the same vdev.
3706 * Check if the top-level vdev must resilver this offset.
3707 * When the offset does not intersect with a dirty leaf DTL
3708 * then it may be possible to skip the resilver IO. The psize
3709 * is provided instead of asize to simplify the check for RAIDZ.
3711 if (!vdev_dtl_need_resilver(vd
, dva
, psize
, phys_birth
))
3715 * Check that this top-level vdev has a device under it which
3716 * is resilvering and is not deferred.
3718 if (!dsl_scan_check_deferred(vd
))
3725 dsl_process_async_destroys(dsl_pool_t
*dp
, dmu_tx_t
*tx
)
3727 dsl_scan_t
*scn
= dp
->dp_scan
;
3728 spa_t
*spa
= dp
->dp_spa
;
3731 if (spa_suspend_async_destroy(spa
))
3734 if (zfs_free_bpobj_enabled
&&
3735 spa_version(spa
) >= SPA_VERSION_DEADLISTS
) {
3736 scn
->scn_is_bptree
= B_FALSE
;
3737 scn
->scn_async_block_min_time_ms
= zfs_free_min_time_ms
;
3738 scn
->scn_zio_root
= zio_root(spa
, NULL
,
3739 NULL
, ZIO_FLAG_MUSTSUCCEED
);
3740 err
= bpobj_iterate(&dp
->dp_free_bpobj
,
3741 bpobj_dsl_scan_free_block_cb
, scn
, tx
);
3742 VERIFY0(zio_wait(scn
->scn_zio_root
));
3743 scn
->scn_zio_root
= NULL
;
3745 if (err
!= 0 && err
!= ERESTART
)
3746 zfs_panic_recover("error %u from bpobj_iterate()", err
);
3749 if (err
== 0 && spa_feature_is_active(spa
, SPA_FEATURE_ASYNC_DESTROY
)) {
3750 ASSERT(scn
->scn_async_destroying
);
3751 scn
->scn_is_bptree
= B_TRUE
;
3752 scn
->scn_zio_root
= zio_root(spa
, NULL
,
3753 NULL
, ZIO_FLAG_MUSTSUCCEED
);
3754 err
= bptree_iterate(dp
->dp_meta_objset
,
3755 dp
->dp_bptree_obj
, B_TRUE
, dsl_scan_free_block_cb
, scn
, tx
);
3756 VERIFY0(zio_wait(scn
->scn_zio_root
));
3757 scn
->scn_zio_root
= NULL
;
3759 if (err
== EIO
|| err
== ECKSUM
) {
3761 } else if (err
!= 0 && err
!= ERESTART
) {
3762 zfs_panic_recover("error %u from "
3763 "traverse_dataset_destroyed()", err
);
3766 if (bptree_is_empty(dp
->dp_meta_objset
, dp
->dp_bptree_obj
)) {
3767 /* finished; deactivate async destroy feature */
3768 spa_feature_decr(spa
, SPA_FEATURE_ASYNC_DESTROY
, tx
);
3769 ASSERT(!spa_feature_is_active(spa
,
3770 SPA_FEATURE_ASYNC_DESTROY
));
3771 VERIFY0(zap_remove(dp
->dp_meta_objset
,
3772 DMU_POOL_DIRECTORY_OBJECT
,
3773 DMU_POOL_BPTREE_OBJ
, tx
));
3774 VERIFY0(bptree_free(dp
->dp_meta_objset
,
3775 dp
->dp_bptree_obj
, tx
));
3776 dp
->dp_bptree_obj
= 0;
3777 scn
->scn_async_destroying
= B_FALSE
;
3778 scn
->scn_async_stalled
= B_FALSE
;
3781 * If we didn't make progress, mark the async
3782 * destroy as stalled, so that we will not initiate
3783 * a spa_sync() on its behalf. Note that we only
3784 * check this if we are not finished, because if the
3785 * bptree had no blocks for us to visit, we can
3786 * finish without "making progress".
3788 scn
->scn_async_stalled
=
3789 (scn
->scn_visited_this_txg
== 0);
3792 if (scn
->scn_visited_this_txg
) {
3793 zfs_dbgmsg("freed %llu blocks in %llums from "
3794 "free_bpobj/bptree on %s in txg %llu; err=%u",
3795 (longlong_t
)scn
->scn_visited_this_txg
,
3797 NSEC2MSEC(gethrtime() - scn
->scn_sync_start_time
),
3798 spa
->spa_name
, (longlong_t
)tx
->tx_txg
, err
);
3799 scn
->scn_visited_this_txg
= 0;
3800 scn
->scn_dedup_frees_this_txg
= 0;
3803 * Write out changes to the DDT and the BRT that may be required
3804 * as a result of the blocks freed. This ensures that the DDT
3805 * and the BRT are clean when a scrub/resilver runs.
3807 ddt_sync(spa
, tx
->tx_txg
);
3808 brt_sync(spa
, tx
->tx_txg
);
3812 if (dp
->dp_free_dir
!= NULL
&& !scn
->scn_async_destroying
&&
3813 zfs_free_leak_on_eio
&&
3814 (dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
!= 0 ||
3815 dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
!= 0 ||
3816 dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
!= 0)) {
3818 * We have finished background destroying, but there is still
3819 * some space left in the dp_free_dir. Transfer this leaked
3820 * space to the dp_leak_dir.
3822 if (dp
->dp_leak_dir
== NULL
) {
3823 rrw_enter(&dp
->dp_config_rwlock
, RW_WRITER
, FTAG
);
3824 (void) dsl_dir_create_sync(dp
, dp
->dp_root_dir
,
3826 VERIFY0(dsl_pool_open_special_dir(dp
,
3827 LEAK_DIR_NAME
, &dp
->dp_leak_dir
));
3828 rrw_exit(&dp
->dp_config_rwlock
, FTAG
);
3830 dsl_dir_diduse_space(dp
->dp_leak_dir
, DD_USED_HEAD
,
3831 dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
,
3832 dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
,
3833 dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
, tx
);
3834 dsl_dir_diduse_space(dp
->dp_free_dir
, DD_USED_HEAD
,
3835 -dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
,
3836 -dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
,
3837 -dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
, tx
);
3840 if (dp
->dp_free_dir
!= NULL
&& !scn
->scn_async_destroying
&&
3841 !spa_livelist_delete_check(spa
)) {
3842 /* finished; verify that space accounting went to zero */
3843 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_used_bytes
);
3844 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_compressed_bytes
);
3845 ASSERT0(dsl_dir_phys(dp
->dp_free_dir
)->dd_uncompressed_bytes
);
3848 spa_notify_waiters(spa
);
3850 EQUIV(bpobj_is_open(&dp
->dp_obsolete_bpobj
),
3851 0 == zap_contains(dp
->dp_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
3852 DMU_POOL_OBSOLETE_BPOBJ
));
3853 if (err
== 0 && bpobj_is_open(&dp
->dp_obsolete_bpobj
)) {
3854 ASSERT(spa_feature_is_active(dp
->dp_spa
,
3855 SPA_FEATURE_OBSOLETE_COUNTS
));
3857 scn
->scn_is_bptree
= B_FALSE
;
3858 scn
->scn_async_block_min_time_ms
= zfs_obsolete_min_time_ms
;
3859 err
= bpobj_iterate(&dp
->dp_obsolete_bpobj
,
3860 dsl_scan_obsolete_block_cb
, scn
, tx
);
3861 if (err
!= 0 && err
!= ERESTART
)
3862 zfs_panic_recover("error %u from bpobj_iterate()", err
);
3864 if (bpobj_is_empty(&dp
->dp_obsolete_bpobj
))
3865 dsl_pool_destroy_obsolete_bpobj(dp
, tx
);
3871 name_to_bookmark(char *buf
, zbookmark_phys_t
*zb
)
3873 zb
->zb_objset
= zfs_strtonum(buf
, &buf
);
3874 ASSERT(*buf
== ':');
3875 zb
->zb_object
= zfs_strtonum(buf
+ 1, &buf
);
3876 ASSERT(*buf
== ':');
3877 zb
->zb_level
= (int)zfs_strtonum(buf
+ 1, &buf
);
3878 ASSERT(*buf
== ':');
3879 zb
->zb_blkid
= zfs_strtonum(buf
+ 1, &buf
);
3880 ASSERT(*buf
== '\0');
3884 name_to_object(char *buf
, uint64_t *obj
)
3886 *obj
= zfs_strtonum(buf
, &buf
);
3887 ASSERT(*buf
== '\0');
3891 read_by_block_level(dsl_scan_t
*scn
, zbookmark_phys_t zb
)
3893 dsl_pool_t
*dp
= scn
->scn_dp
;
3896 if (dsl_dataset_hold_obj(dp
, zb
.zb_objset
, FTAG
, &ds
) != 0)
3899 if (dmu_objset_from_ds(ds
, &os
) != 0) {
3900 dsl_dataset_rele(ds
, FTAG
);
3905 * If the key is not loaded dbuf_dnode_findbp() will error out with
3906 * EACCES. However in that case dnode_hold() will eventually call
3907 * dbuf_read()->zio_wait() which may call spa_log_error(). This will
3908 * lead to a deadlock due to us holding the mutex spa_errlist_lock.
3909 * Avoid this by checking here if the keys are loaded, if not return.
3910 * If the keys are not loaded the head_errlog feature is meaningless
3911 * as we cannot figure out the birth txg of the block pointer.
3913 if (dsl_dataset_get_keystatus(ds
->ds_dir
) ==
3914 ZFS_KEYSTATUS_UNAVAILABLE
) {
3915 dsl_dataset_rele(ds
, FTAG
);
3922 if (dnode_hold(os
, zb
.zb_object
, FTAG
, &dn
) != 0) {
3923 dsl_dataset_rele(ds
, FTAG
);
3927 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
3928 int error
= dbuf_dnode_findbp(dn
, zb
.zb_level
, zb
.zb_blkid
, &bp
, NULL
,
3932 rw_exit(&dn
->dn_struct_rwlock
);
3933 dnode_rele(dn
, FTAG
);
3934 dsl_dataset_rele(ds
, FTAG
);
3938 if (!error
&& BP_IS_HOLE(&bp
)) {
3939 rw_exit(&dn
->dn_struct_rwlock
);
3940 dnode_rele(dn
, FTAG
);
3941 dsl_dataset_rele(ds
, FTAG
);
3945 int zio_flags
= ZIO_FLAG_SCAN_THREAD
| ZIO_FLAG_RAW
|
3946 ZIO_FLAG_CANFAIL
| ZIO_FLAG_SCRUB
;
3948 /* If it's an intent log block, failure is expected. */
3949 if (zb
.zb_level
== ZB_ZIL_LEVEL
)
3950 zio_flags
|= ZIO_FLAG_SPECULATIVE
;
3952 ASSERT(!BP_IS_EMBEDDED(&bp
));
3953 scan_exec_io(dp
, &bp
, zio_flags
, &zb
, NULL
);
3954 rw_exit(&dn
->dn_struct_rwlock
);
3955 dnode_rele(dn
, FTAG
);
3956 dsl_dataset_rele(ds
, FTAG
);
3960 * We keep track of the scrubbed error blocks in "count". This will be used
3961 * when deciding whether we exceeded zfs_scrub_error_blocks_per_txg. This
3962 * function is modelled after check_filesystem().
3965 scrub_filesystem(spa_t
*spa
, uint64_t fs
, zbookmark_err_phys_t
*zep
,
3969 dsl_pool_t
*dp
= spa
->spa_dsl_pool
;
3970 dsl_scan_t
*scn
= dp
->dp_scan
;
3972 int error
= dsl_dataset_hold_obj(dp
, fs
, FTAG
, &ds
);
3976 uint64_t latest_txg
;
3977 uint64_t txg_to_consider
= spa
->spa_syncing_txg
;
3978 boolean_t check_snapshot
= B_TRUE
;
3980 error
= find_birth_txg(ds
, zep
, &latest_txg
);
3983 * If find_birth_txg() errors out, then err on the side of caution and
3984 * proceed. In worst case scenario scrub all objects. If zep->zb_birth
3985 * is 0 (e.g. in case of encryption with unloaded keys) also proceed to
3986 * scrub all objects.
3988 if (error
== 0 && zep
->zb_birth
== latest_txg
) {
3989 /* Block neither free nor re written. */
3990 zbookmark_phys_t zb
;
3991 zep_to_zb(fs
, zep
, &zb
);
3992 scn
->scn_zio_root
= zio_root(spa
, NULL
, NULL
,
3994 /* We have already acquired the config lock for spa */
3995 read_by_block_level(scn
, zb
);
3997 (void) zio_wait(scn
->scn_zio_root
);
3998 scn
->scn_zio_root
= NULL
;
4000 scn
->errorscrub_phys
.dep_examined
++;
4001 scn
->errorscrub_phys
.dep_to_examine
--;
4003 if ((*count
) == zfs_scrub_error_blocks_per_txg
||
4004 dsl_error_scrub_check_suspend(scn
, &zb
)) {
4005 dsl_dataset_rele(ds
, FTAG
);
4006 return (SET_ERROR(EFAULT
));
4009 check_snapshot
= B_FALSE
;
4010 } else if (error
== 0) {
4011 txg_to_consider
= latest_txg
;
4015 * Retrieve the number of snapshots if the dataset is not a snapshot.
4017 uint64_t snap_count
= 0;
4018 if (dsl_dataset_phys(ds
)->ds_snapnames_zapobj
!= 0) {
4020 error
= zap_count(spa
->spa_meta_objset
,
4021 dsl_dataset_phys(ds
)->ds_snapnames_zapobj
, &snap_count
);
4024 dsl_dataset_rele(ds
, FTAG
);
4029 if (snap_count
== 0) {
4030 /* Filesystem without snapshots. */
4031 dsl_dataset_rele(ds
, FTAG
);
4035 uint64_t snap_obj
= dsl_dataset_phys(ds
)->ds_prev_snap_obj
;
4036 uint64_t snap_obj_txg
= dsl_dataset_phys(ds
)->ds_prev_snap_txg
;
4038 dsl_dataset_rele(ds
, FTAG
);
4040 /* Check only snapshots created from this file system. */
4041 while (snap_obj
!= 0 && zep
->zb_birth
< snap_obj_txg
&&
4042 snap_obj_txg
<= txg_to_consider
) {
4044 error
= dsl_dataset_hold_obj(dp
, snap_obj
, FTAG
, &ds
);
4048 if (dsl_dir_phys(ds
->ds_dir
)->dd_head_dataset_obj
!= fs
) {
4049 snap_obj
= dsl_dataset_phys(ds
)->ds_prev_snap_obj
;
4050 snap_obj_txg
= dsl_dataset_phys(ds
)->ds_prev_snap_txg
;
4051 dsl_dataset_rele(ds
, FTAG
);
4055 boolean_t affected
= B_TRUE
;
4056 if (check_snapshot
) {
4058 error
= find_birth_txg(ds
, zep
, &blk_txg
);
4061 * Scrub the snapshot also when zb_birth == 0 or when
4062 * find_birth_txg() returns an error.
4064 affected
= (error
== 0 && zep
->zb_birth
== blk_txg
) ||
4065 (error
!= 0) || (zep
->zb_birth
== 0);
4068 /* Scrub snapshots. */
4070 zbookmark_phys_t zb
;
4071 zep_to_zb(snap_obj
, zep
, &zb
);
4072 scn
->scn_zio_root
= zio_root(spa
, NULL
, NULL
,
4074 /* We have already acquired the config lock for spa */
4075 read_by_block_level(scn
, zb
);
4077 (void) zio_wait(scn
->scn_zio_root
);
4078 scn
->scn_zio_root
= NULL
;
4080 scn
->errorscrub_phys
.dep_examined
++;
4081 scn
->errorscrub_phys
.dep_to_examine
--;
4083 if ((*count
) == zfs_scrub_error_blocks_per_txg
||
4084 dsl_error_scrub_check_suspend(scn
, &zb
)) {
4085 dsl_dataset_rele(ds
, FTAG
);
4089 snap_obj_txg
= dsl_dataset_phys(ds
)->ds_prev_snap_txg
;
4090 snap_obj
= dsl_dataset_phys(ds
)->ds_prev_snap_obj
;
4091 dsl_dataset_rele(ds
, FTAG
);
4097 dsl_errorscrub_sync(dsl_pool_t
*dp
, dmu_tx_t
*tx
)
4099 spa_t
*spa
= dp
->dp_spa
;
4100 dsl_scan_t
*scn
= dp
->dp_scan
;
4103 * Only process scans in sync pass 1.
4106 if (spa_sync_pass(spa
) > 1)
4110 * If the spa is shutting down, then stop scanning. This will
4111 * ensure that the scan does not dirty any new data during the
4114 if (spa_shutting_down(spa
))
4117 if (!dsl_errorscrub_active(scn
) || dsl_errorscrub_is_paused(scn
)) {
4121 if (dsl_scan_resilvering(scn
->scn_dp
)) {
4122 /* cancel the error scrub if resilver started */
4123 dsl_scan_cancel(scn
->scn_dp
);
4127 spa
->spa_scrub_active
= B_TRUE
;
4128 scn
->scn_sync_start_time
= gethrtime();
4131 * zfs_scan_suspend_progress can be set to disable scrub progress.
4132 * See more detailed comment in dsl_scan_sync().
4134 if (zfs_scan_suspend_progress
) {
4135 uint64_t scan_time_ns
= gethrtime() - scn
->scn_sync_start_time
;
4136 int mintime
= zfs_scrub_min_time_ms
;
4138 while (zfs_scan_suspend_progress
&&
4139 !txg_sync_waiting(scn
->scn_dp
) &&
4140 !spa_shutting_down(scn
->scn_dp
->dp_spa
) &&
4141 NSEC2MSEC(scan_time_ns
) < mintime
) {
4143 scan_time_ns
= gethrtime() - scn
->scn_sync_start_time
;
4149 zap_attribute_t
*za
;
4150 zbookmark_phys_t
*zb
;
4151 boolean_t limit_exceeded
= B_FALSE
;
4153 za
= kmem_zalloc(sizeof (zap_attribute_t
), KM_SLEEP
);
4154 zb
= kmem_zalloc(sizeof (zbookmark_phys_t
), KM_SLEEP
);
4156 if (!spa_feature_is_enabled(spa
, SPA_FEATURE_HEAD_ERRLOG
)) {
4157 for (; zap_cursor_retrieve(&scn
->errorscrub_cursor
, za
) == 0;
4158 zap_cursor_advance(&scn
->errorscrub_cursor
)) {
4159 name_to_bookmark(za
->za_name
, zb
);
4161 scn
->scn_zio_root
= zio_root(dp
->dp_spa
, NULL
,
4162 NULL
, ZIO_FLAG_CANFAIL
);
4163 dsl_pool_config_enter(dp
, FTAG
);
4164 read_by_block_level(scn
, *zb
);
4165 dsl_pool_config_exit(dp
, FTAG
);
4167 (void) zio_wait(scn
->scn_zio_root
);
4168 scn
->scn_zio_root
= NULL
;
4170 scn
->errorscrub_phys
.dep_examined
+= 1;
4171 scn
->errorscrub_phys
.dep_to_examine
-= 1;
4173 if (i
== zfs_scrub_error_blocks_per_txg
||
4174 dsl_error_scrub_check_suspend(scn
, zb
)) {
4175 limit_exceeded
= B_TRUE
;
4180 if (!limit_exceeded
)
4181 dsl_errorscrub_done(scn
, B_TRUE
, tx
);
4183 dsl_errorscrub_sync_state(scn
, tx
);
4184 kmem_free(za
, sizeof (*za
));
4185 kmem_free(zb
, sizeof (*zb
));
4190 for (; zap_cursor_retrieve(&scn
->errorscrub_cursor
, za
) == 0;
4191 zap_cursor_advance(&scn
->errorscrub_cursor
)) {
4193 zap_cursor_t
*head_ds_cursor
;
4194 zap_attribute_t
*head_ds_attr
;
4195 zbookmark_err_phys_t head_ds_block
;
4197 head_ds_cursor
= kmem_zalloc(sizeof (zap_cursor_t
), KM_SLEEP
);
4198 head_ds_attr
= kmem_zalloc(sizeof (zap_attribute_t
), KM_SLEEP
);
4200 uint64_t head_ds_err_obj
= za
->za_first_integer
;
4202 name_to_object(za
->za_name
, &head_ds
);
4203 boolean_t config_held
= B_FALSE
;
4204 uint64_t top_affected_fs
;
4206 for (zap_cursor_init(head_ds_cursor
, spa
->spa_meta_objset
,
4207 head_ds_err_obj
); zap_cursor_retrieve(head_ds_cursor
,
4208 head_ds_attr
) == 0; zap_cursor_advance(head_ds_cursor
)) {
4210 name_to_errphys(head_ds_attr
->za_name
, &head_ds_block
);
4213 * In case we are called from spa_sync the pool
4214 * config is already held.
4216 if (!dsl_pool_config_held(dp
)) {
4217 dsl_pool_config_enter(dp
, FTAG
);
4218 config_held
= B_TRUE
;
4221 error
= find_top_affected_fs(spa
,
4222 head_ds
, &head_ds_block
, &top_affected_fs
);
4226 error
= scrub_filesystem(spa
, top_affected_fs
,
4227 &head_ds_block
, &i
);
4229 if (error
== SET_ERROR(EFAULT
)) {
4230 limit_exceeded
= B_TRUE
;
4235 zap_cursor_fini(head_ds_cursor
);
4236 kmem_free(head_ds_cursor
, sizeof (*head_ds_cursor
));
4237 kmem_free(head_ds_attr
, sizeof (*head_ds_attr
));
4240 dsl_pool_config_exit(dp
, FTAG
);
4243 kmem_free(za
, sizeof (*za
));
4244 kmem_free(zb
, sizeof (*zb
));
4245 if (!limit_exceeded
)
4246 dsl_errorscrub_done(scn
, B_TRUE
, tx
);
4248 dsl_errorscrub_sync_state(scn
, tx
);
4252 * This is the primary entry point for scans that is called from syncing
4253 * context. Scans must happen entirely during syncing context so that we
4254 * can guarantee that blocks we are currently scanning will not change out
4255 * from under us. While a scan is active, this function controls how quickly
4256 * transaction groups proceed, instead of the normal handling provided by
4257 * txg_sync_thread().
4260 dsl_scan_sync(dsl_pool_t
*dp
, dmu_tx_t
*tx
)
4263 dsl_scan_t
*scn
= dp
->dp_scan
;
4264 spa_t
*spa
= dp
->dp_spa
;
4265 state_sync_type_t sync_type
= SYNC_OPTIONAL
;
4267 if (spa
->spa_resilver_deferred
&&
4268 !spa_feature_is_active(dp
->dp_spa
, SPA_FEATURE_RESILVER_DEFER
))
4269 spa_feature_incr(spa
, SPA_FEATURE_RESILVER_DEFER
, tx
);
4272 * Check for scn_restart_txg before checking spa_load_state, so
4273 * that we can restart an old-style scan while the pool is being
4274 * imported (see dsl_scan_init). We also restart scans if there
4275 * is a deferred resilver and the user has manually disabled
4276 * deferred resilvers via the tunable.
4278 if (dsl_scan_restarting(scn
, tx
) ||
4279 (spa
->spa_resilver_deferred
&& zfs_resilver_disable_defer
)) {
4280 pool_scan_func_t func
= POOL_SCAN_SCRUB
;
4281 dsl_scan_done(scn
, B_FALSE
, tx
);
4282 if (vdev_resilver_needed(spa
->spa_root_vdev
, NULL
, NULL
))
4283 func
= POOL_SCAN_RESILVER
;
4284 zfs_dbgmsg("restarting scan func=%u on %s txg=%llu",
4285 func
, dp
->dp_spa
->spa_name
, (longlong_t
)tx
->tx_txg
);
4286 dsl_scan_setup_sync(&func
, tx
);
4290 * Only process scans in sync pass 1.
4292 if (spa_sync_pass(spa
) > 1)
4296 * If the spa is shutting down, then stop scanning. This will
4297 * ensure that the scan does not dirty any new data during the
4300 if (spa_shutting_down(spa
))
4304 * If the scan is inactive due to a stalled async destroy, try again.
4306 if (!scn
->scn_async_stalled
&& !dsl_scan_active(scn
))
4309 /* reset scan statistics */
4310 scn
->scn_visited_this_txg
= 0;
4311 scn
->scn_dedup_frees_this_txg
= 0;
4312 scn
->scn_holes_this_txg
= 0;
4313 scn
->scn_lt_min_this_txg
= 0;
4314 scn
->scn_gt_max_this_txg
= 0;
4315 scn
->scn_ddt_contained_this_txg
= 0;
4316 scn
->scn_objsets_visited_this_txg
= 0;
4317 scn
->scn_avg_seg_size_this_txg
= 0;
4318 scn
->scn_segs_this_txg
= 0;
4319 scn
->scn_avg_zio_size_this_txg
= 0;
4320 scn
->scn_zios_this_txg
= 0;
4321 scn
->scn_suspending
= B_FALSE
;
4322 scn
->scn_sync_start_time
= gethrtime();
4323 spa
->spa_scrub_active
= B_TRUE
;
4326 * First process the async destroys. If we suspend, don't do
4327 * any scrubbing or resilvering. This ensures that there are no
4328 * async destroys while we are scanning, so the scan code doesn't
4329 * have to worry about traversing it. It is also faster to free the
4330 * blocks than to scrub them.
4332 err
= dsl_process_async_destroys(dp
, tx
);
4336 if (!dsl_scan_is_running(scn
) || dsl_scan_is_paused_scrub(scn
))
4340 * Wait a few txgs after importing to begin scanning so that
4341 * we can get the pool imported quickly.
4343 if (spa
->spa_syncing_txg
< spa
->spa_first_txg
+ SCAN_IMPORT_WAIT_TXGS
)
4347 * zfs_scan_suspend_progress can be set to disable scan progress.
4348 * We don't want to spin the txg_sync thread, so we add a delay
4349 * here to simulate the time spent doing a scan. This is mostly
4350 * useful for testing and debugging.
4352 if (zfs_scan_suspend_progress
) {
4353 uint64_t scan_time_ns
= gethrtime() - scn
->scn_sync_start_time
;
4354 uint_t mintime
= (scn
->scn_phys
.scn_func
==
4355 POOL_SCAN_RESILVER
) ? zfs_resilver_min_time_ms
:
4356 zfs_scrub_min_time_ms
;
4358 while (zfs_scan_suspend_progress
&&
4359 !txg_sync_waiting(scn
->scn_dp
) &&
4360 !spa_shutting_down(scn
->scn_dp
->dp_spa
) &&
4361 NSEC2MSEC(scan_time_ns
) < mintime
) {
4363 scan_time_ns
= gethrtime() - scn
->scn_sync_start_time
;
4369 * Disabled by default, set zfs_scan_report_txgs to report
4370 * average performance over the last zfs_scan_report_txgs TXGs.
4372 if (zfs_scan_report_txgs
!= 0 &&
4373 tx
->tx_txg
% zfs_scan_report_txgs
== 0) {
4374 scn
->scn_issued_before_pass
+= spa
->spa_scan_pass_issued
;
4375 spa_scan_stat_init(spa
);
4379 * It is possible to switch from unsorted to sorted at any time,
4380 * but afterwards the scan will remain sorted unless reloaded from
4381 * a checkpoint after a reboot.
4383 if (!zfs_scan_legacy
) {
4384 scn
->scn_is_sorted
= B_TRUE
;
4385 if (scn
->scn_last_checkpoint
== 0)
4386 scn
->scn_last_checkpoint
= ddi_get_lbolt();
4390 * For sorted scans, determine what kind of work we will be doing
4391 * this txg based on our memory limitations and whether or not we
4392 * need to perform a checkpoint.
4394 if (scn
->scn_is_sorted
) {
4396 * If we are over our checkpoint interval, set scn_clearing
4397 * so that we can begin checkpointing immediately. The
4398 * checkpoint allows us to save a consistent bookmark
4399 * representing how much data we have scrubbed so far.
4400 * Otherwise, use the memory limit to determine if we should
4401 * scan for metadata or start issue scrub IOs. We accumulate
4402 * metadata until we hit our hard memory limit at which point
4403 * we issue scrub IOs until we are at our soft memory limit.
4405 if (scn
->scn_checkpointing
||
4406 ddi_get_lbolt() - scn
->scn_last_checkpoint
>
4407 SEC_TO_TICK(zfs_scan_checkpoint_intval
)) {
4408 if (!scn
->scn_checkpointing
)
4409 zfs_dbgmsg("begin scan checkpoint for %s",
4412 scn
->scn_checkpointing
= B_TRUE
;
4413 scn
->scn_clearing
= B_TRUE
;
4415 boolean_t should_clear
= dsl_scan_should_clear(scn
);
4416 if (should_clear
&& !scn
->scn_clearing
) {
4417 zfs_dbgmsg("begin scan clearing for %s",
4419 scn
->scn_clearing
= B_TRUE
;
4420 } else if (!should_clear
&& scn
->scn_clearing
) {
4421 zfs_dbgmsg("finish scan clearing for %s",
4423 scn
->scn_clearing
= B_FALSE
;
4427 ASSERT0(scn
->scn_checkpointing
);
4428 ASSERT0(scn
->scn_clearing
);
4431 if (!scn
->scn_clearing
&& scn
->scn_done_txg
== 0) {
4432 /* Need to scan metadata for more blocks to scrub */
4433 dsl_scan_phys_t
*scnp
= &scn
->scn_phys
;
4434 taskqid_t prefetch_tqid
;
4437 * Calculate the max number of in-flight bytes for pool-wide
4438 * scanning operations (minimum 1MB, maximum 1/4 of arc_c_max).
4439 * Limits for the issuing phase are done per top-level vdev and
4440 * are handled separately.
4442 scn
->scn_maxinflight_bytes
= MIN(arc_c_max
/ 4, MAX(1ULL << 20,
4443 zfs_scan_vdev_limit
* dsl_scan_count_data_disks(spa
)));
4445 if (scnp
->scn_ddt_bookmark
.ddb_class
<=
4446 scnp
->scn_ddt_class_max
) {
4447 ASSERT(ZB_IS_ZERO(&scnp
->scn_bookmark
));
4448 zfs_dbgmsg("doing scan sync for %s txg %llu; "
4449 "ddt bm=%llu/%llu/%llu/%llx",
4451 (longlong_t
)tx
->tx_txg
,
4452 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_class
,
4453 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_type
,
4454 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_checksum
,
4455 (longlong_t
)scnp
->scn_ddt_bookmark
.ddb_cursor
);
4457 zfs_dbgmsg("doing scan sync for %s txg %llu; "
4458 "bm=%llu/%llu/%llu/%llu",
4460 (longlong_t
)tx
->tx_txg
,
4461 (longlong_t
)scnp
->scn_bookmark
.zb_objset
,
4462 (longlong_t
)scnp
->scn_bookmark
.zb_object
,
4463 (longlong_t
)scnp
->scn_bookmark
.zb_level
,
4464 (longlong_t
)scnp
->scn_bookmark
.zb_blkid
);
4467 scn
->scn_zio_root
= zio_root(dp
->dp_spa
, NULL
,
4468 NULL
, ZIO_FLAG_CANFAIL
);
4470 scn
->scn_prefetch_stop
= B_FALSE
;
4471 prefetch_tqid
= taskq_dispatch(dp
->dp_sync_taskq
,
4472 dsl_scan_prefetch_thread
, scn
, TQ_SLEEP
);
4473 ASSERT(prefetch_tqid
!= TASKQID_INVALID
);
4475 dsl_pool_config_enter(dp
, FTAG
);
4476 dsl_scan_visit(scn
, tx
);
4477 dsl_pool_config_exit(dp
, FTAG
);
4479 mutex_enter(&dp
->dp_spa
->spa_scrub_lock
);
4480 scn
->scn_prefetch_stop
= B_TRUE
;
4481 cv_broadcast(&spa
->spa_scrub_io_cv
);
4482 mutex_exit(&dp
->dp_spa
->spa_scrub_lock
);
4484 taskq_wait_id(dp
->dp_sync_taskq
, prefetch_tqid
);
4485 (void) zio_wait(scn
->scn_zio_root
);
4486 scn
->scn_zio_root
= NULL
;
4488 zfs_dbgmsg("scan visited %llu blocks of %s in %llums "
4489 "(%llu os's, %llu holes, %llu < mintxg, "
4490 "%llu in ddt, %llu > maxtxg)",
4491 (longlong_t
)scn
->scn_visited_this_txg
,
4493 (longlong_t
)NSEC2MSEC(gethrtime() -
4494 scn
->scn_sync_start_time
),
4495 (longlong_t
)scn
->scn_objsets_visited_this_txg
,
4496 (longlong_t
)scn
->scn_holes_this_txg
,
4497 (longlong_t
)scn
->scn_lt_min_this_txg
,
4498 (longlong_t
)scn
->scn_ddt_contained_this_txg
,
4499 (longlong_t
)scn
->scn_gt_max_this_txg
);
4501 if (!scn
->scn_suspending
) {
4502 ASSERT0(avl_numnodes(&scn
->scn_queue
));
4503 scn
->scn_done_txg
= tx
->tx_txg
+ 1;
4504 if (scn
->scn_is_sorted
) {
4505 scn
->scn_checkpointing
= B_TRUE
;
4506 scn
->scn_clearing
= B_TRUE
;
4507 scn
->scn_issued_before_pass
+=
4508 spa
->spa_scan_pass_issued
;
4509 spa_scan_stat_init(spa
);
4511 zfs_dbgmsg("scan complete for %s txg %llu",
4513 (longlong_t
)tx
->tx_txg
);
4515 } else if (scn
->scn_is_sorted
&& scn
->scn_queues_pending
!= 0) {
4516 ASSERT(scn
->scn_clearing
);
4518 /* need to issue scrubbing IOs from per-vdev queues */
4519 scn
->scn_zio_root
= zio_root(dp
->dp_spa
, NULL
,
4520 NULL
, ZIO_FLAG_CANFAIL
);
4521 scan_io_queues_run(scn
);
4522 (void) zio_wait(scn
->scn_zio_root
);
4523 scn
->scn_zio_root
= NULL
;
4525 /* calculate and dprintf the current memory usage */
4526 (void) dsl_scan_should_clear(scn
);
4527 dsl_scan_update_stats(scn
);
4529 zfs_dbgmsg("scan issued %llu blocks for %s (%llu segs) "
4530 "in %llums (avg_block_size = %llu, avg_seg_size = %llu)",
4531 (longlong_t
)scn
->scn_zios_this_txg
,
4533 (longlong_t
)scn
->scn_segs_this_txg
,
4534 (longlong_t
)NSEC2MSEC(gethrtime() -
4535 scn
->scn_sync_start_time
),
4536 (longlong_t
)scn
->scn_avg_zio_size_this_txg
,
4537 (longlong_t
)scn
->scn_avg_seg_size_this_txg
);
4538 } else if (scn
->scn_done_txg
!= 0 && scn
->scn_done_txg
<= tx
->tx_txg
) {
4539 /* Finished with everything. Mark the scrub as complete */
4540 zfs_dbgmsg("scan issuing complete txg %llu for %s",
4541 (longlong_t
)tx
->tx_txg
,
4543 ASSERT3U(scn
->scn_done_txg
, !=, 0);
4544 ASSERT0(spa
->spa_scrub_inflight
);
4545 ASSERT0(scn
->scn_queues_pending
);
4546 dsl_scan_done(scn
, B_TRUE
, tx
);
4547 sync_type
= SYNC_MANDATORY
;
4550 dsl_scan_sync_state(scn
, tx
, sync_type
);
4554 count_block_issued(spa_t
*spa
, const blkptr_t
*bp
, boolean_t all
)
4557 * Don't count embedded bp's, since we already did the work of
4558 * scanning these when we scanned the containing block.
4560 if (BP_IS_EMBEDDED(bp
))
4564 * Update the spa's stats on how many bytes we have issued.
4565 * Sequential scrubs create a zio for each DVA of the bp. Each
4566 * of these will include all DVAs for repair purposes, but the
4567 * zio code will only try the first one unless there is an issue.
4568 * Therefore, we should only count the first DVA for these IOs.
4570 atomic_add_64(&spa
->spa_scan_pass_issued
,
4571 all
? BP_GET_ASIZE(bp
) : DVA_GET_ASIZE(&bp
->blk_dva
[0]));
4575 count_block_skipped(dsl_scan_t
*scn
, const blkptr_t
*bp
, boolean_t all
)
4577 if (BP_IS_EMBEDDED(bp
))
4579 atomic_add_64(&scn
->scn_phys
.scn_skipped
,
4580 all
? BP_GET_ASIZE(bp
) : DVA_GET_ASIZE(&bp
->blk_dva
[0]));
4584 count_block(zfs_all_blkstats_t
*zab
, const blkptr_t
*bp
)
4587 * If we resume after a reboot, zab will be NULL; don't record
4588 * incomplete stats in that case.
4593 for (int i
= 0; i
< 4; i
++) {
4594 int l
= (i
< 2) ? BP_GET_LEVEL(bp
) : DN_MAX_LEVELS
;
4595 int t
= (i
& 1) ? BP_GET_TYPE(bp
) : DMU_OT_TOTAL
;
4597 if (t
& DMU_OT_NEWTYPE
)
4599 zfs_blkstat_t
*zb
= &zab
->zab_type
[l
][t
];
4603 zb
->zb_asize
+= BP_GET_ASIZE(bp
);
4604 zb
->zb_lsize
+= BP_GET_LSIZE(bp
);
4605 zb
->zb_psize
+= BP_GET_PSIZE(bp
);
4606 zb
->zb_gangs
+= BP_COUNT_GANG(bp
);
4608 switch (BP_GET_NDVAS(bp
)) {
4610 if (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
4611 DVA_GET_VDEV(&bp
->blk_dva
[1]))
4612 zb
->zb_ditto_2_of_2_samevdev
++;
4615 equal
= (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
4616 DVA_GET_VDEV(&bp
->blk_dva
[1])) +
4617 (DVA_GET_VDEV(&bp
->blk_dva
[0]) ==
4618 DVA_GET_VDEV(&bp
->blk_dva
[2])) +
4619 (DVA_GET_VDEV(&bp
->blk_dva
[1]) ==
4620 DVA_GET_VDEV(&bp
->blk_dva
[2]));
4622 zb
->zb_ditto_2_of_3_samevdev
++;
4623 else if (equal
== 3)
4624 zb
->zb_ditto_3_of_3_samevdev
++;
4631 scan_io_queue_insert_impl(dsl_scan_io_queue_t
*queue
, scan_io_t
*sio
)
4634 dsl_scan_t
*scn
= queue
->q_scn
;
4636 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
4638 if (unlikely(avl_is_empty(&queue
->q_sios_by_addr
)))
4639 atomic_add_64(&scn
->scn_queues_pending
, 1);
4640 if (avl_find(&queue
->q_sios_by_addr
, sio
, &idx
) != NULL
) {
4641 /* block is already scheduled for reading */
4645 avl_insert(&queue
->q_sios_by_addr
, sio
, idx
);
4646 queue
->q_sio_memused
+= SIO_GET_MUSED(sio
);
4647 range_tree_add(queue
->q_exts_by_addr
, SIO_GET_OFFSET(sio
),
4648 SIO_GET_ASIZE(sio
));
4652 * Given all the info we got from our metadata scanning process, we
4653 * construct a scan_io_t and insert it into the scan sorting queue. The
4654 * I/O must already be suitable for us to process. This is controlled
4655 * by dsl_scan_enqueue().
4658 scan_io_queue_insert(dsl_scan_io_queue_t
*queue
, const blkptr_t
*bp
, int dva_i
,
4659 int zio_flags
, const zbookmark_phys_t
*zb
)
4661 scan_io_t
*sio
= sio_alloc(BP_GET_NDVAS(bp
));
4663 ASSERT0(BP_IS_GANG(bp
));
4664 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
4666 bp2sio(bp
, sio
, dva_i
);
4667 sio
->sio_flags
= zio_flags
;
4670 queue
->q_last_ext_addr
= -1;
4671 scan_io_queue_insert_impl(queue
, sio
);
4675 * Given a set of I/O parameters as discovered by the metadata traversal
4676 * process, attempts to place the I/O into the sorted queues (if allowed),
4677 * or immediately executes the I/O.
4680 dsl_scan_enqueue(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
4681 const zbookmark_phys_t
*zb
)
4683 spa_t
*spa
= dp
->dp_spa
;
4685 ASSERT(!BP_IS_EMBEDDED(bp
));
4688 * Gang blocks are hard to issue sequentially, so we just issue them
4689 * here immediately instead of queuing them.
4691 if (!dp
->dp_scan
->scn_is_sorted
|| BP_IS_GANG(bp
)) {
4692 scan_exec_io(dp
, bp
, zio_flags
, zb
, NULL
);
4696 for (int i
= 0; i
< BP_GET_NDVAS(bp
); i
++) {
4700 dva
= bp
->blk_dva
[i
];
4701 vdev
= vdev_lookup_top(spa
, DVA_GET_VDEV(&dva
));
4702 ASSERT(vdev
!= NULL
);
4704 mutex_enter(&vdev
->vdev_scan_io_queue_lock
);
4705 if (vdev
->vdev_scan_io_queue
== NULL
)
4706 vdev
->vdev_scan_io_queue
= scan_io_queue_create(vdev
);
4707 ASSERT(dp
->dp_scan
!= NULL
);
4708 scan_io_queue_insert(vdev
->vdev_scan_io_queue
, bp
,
4710 mutex_exit(&vdev
->vdev_scan_io_queue_lock
);
4715 dsl_scan_scrub_cb(dsl_pool_t
*dp
,
4716 const blkptr_t
*bp
, const zbookmark_phys_t
*zb
)
4718 dsl_scan_t
*scn
= dp
->dp_scan
;
4719 spa_t
*spa
= dp
->dp_spa
;
4720 uint64_t phys_birth
= BP_GET_BIRTH(bp
);
4721 size_t psize
= BP_GET_PSIZE(bp
);
4722 boolean_t needs_io
= B_FALSE
;
4723 int zio_flags
= ZIO_FLAG_SCAN_THREAD
| ZIO_FLAG_RAW
| ZIO_FLAG_CANFAIL
;
4725 count_block(dp
->dp_blkstats
, bp
);
4726 if (phys_birth
<= scn
->scn_phys
.scn_min_txg
||
4727 phys_birth
>= scn
->scn_phys
.scn_max_txg
) {
4728 count_block_skipped(scn
, bp
, B_TRUE
);
4732 /* Embedded BP's have phys_birth==0, so we reject them above. */
4733 ASSERT(!BP_IS_EMBEDDED(bp
));
4735 ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn
));
4736 if (scn
->scn_phys
.scn_func
== POOL_SCAN_SCRUB
) {
4737 zio_flags
|= ZIO_FLAG_SCRUB
;
4740 ASSERT3U(scn
->scn_phys
.scn_func
, ==, POOL_SCAN_RESILVER
);
4741 zio_flags
|= ZIO_FLAG_RESILVER
;
4745 /* If it's an intent log block, failure is expected. */
4746 if (zb
->zb_level
== ZB_ZIL_LEVEL
)
4747 zio_flags
|= ZIO_FLAG_SPECULATIVE
;
4749 for (int d
= 0; d
< BP_GET_NDVAS(bp
); d
++) {
4750 const dva_t
*dva
= &bp
->blk_dva
[d
];
4753 * Keep track of how much data we've examined so that
4754 * zpool(8) status can make useful progress reports.
4756 uint64_t asize
= DVA_GET_ASIZE(dva
);
4757 scn
->scn_phys
.scn_examined
+= asize
;
4758 spa
->spa_scan_pass_exam
+= asize
;
4760 /* if it's a resilver, this may not be in the target range */
4762 needs_io
= dsl_scan_need_resilver(spa
, dva
, psize
,
4766 if (needs_io
&& !zfs_no_scrub_io
) {
4767 dsl_scan_enqueue(dp
, bp
, zio_flags
, zb
);
4769 count_block_skipped(scn
, bp
, B_TRUE
);
4772 /* do not relocate this block */
4777 dsl_scan_scrub_done(zio_t
*zio
)
4779 spa_t
*spa
= zio
->io_spa
;
4780 blkptr_t
*bp
= zio
->io_bp
;
4781 dsl_scan_io_queue_t
*queue
= zio
->io_private
;
4783 abd_free(zio
->io_abd
);
4785 if (queue
== NULL
) {
4786 mutex_enter(&spa
->spa_scrub_lock
);
4787 ASSERT3U(spa
->spa_scrub_inflight
, >=, BP_GET_PSIZE(bp
));
4788 spa
->spa_scrub_inflight
-= BP_GET_PSIZE(bp
);
4789 cv_broadcast(&spa
->spa_scrub_io_cv
);
4790 mutex_exit(&spa
->spa_scrub_lock
);
4792 mutex_enter(&queue
->q_vd
->vdev_scan_io_queue_lock
);
4793 ASSERT3U(queue
->q_inflight_bytes
, >=, BP_GET_PSIZE(bp
));
4794 queue
->q_inflight_bytes
-= BP_GET_PSIZE(bp
);
4795 cv_broadcast(&queue
->q_zio_cv
);
4796 mutex_exit(&queue
->q_vd
->vdev_scan_io_queue_lock
);
4799 if (zio
->io_error
&& (zio
->io_error
!= ECKSUM
||
4800 !(zio
->io_flags
& ZIO_FLAG_SPECULATIVE
))) {
4801 if (dsl_errorscrubbing(spa
->spa_dsl_pool
) &&
4802 !dsl_errorscrub_is_paused(spa
->spa_dsl_pool
->dp_scan
)) {
4803 atomic_inc_64(&spa
->spa_dsl_pool
->dp_scan
4804 ->errorscrub_phys
.dep_errors
);
4806 atomic_inc_64(&spa
->spa_dsl_pool
->dp_scan
->scn_phys
4813 * Given a scanning zio's information, executes the zio. The zio need
4814 * not necessarily be only sortable, this function simply executes the
4815 * zio, no matter what it is. The optional queue argument allows the
4816 * caller to specify that they want per top level vdev IO rate limiting
4817 * instead of the legacy global limiting.
4820 scan_exec_io(dsl_pool_t
*dp
, const blkptr_t
*bp
, int zio_flags
,
4821 const zbookmark_phys_t
*zb
, dsl_scan_io_queue_t
*queue
)
4823 spa_t
*spa
= dp
->dp_spa
;
4824 dsl_scan_t
*scn
= dp
->dp_scan
;
4825 size_t size
= BP_GET_PSIZE(bp
);
4826 abd_t
*data
= abd_alloc_for_io(size
, B_FALSE
);
4829 if (queue
== NULL
) {
4830 ASSERT3U(scn
->scn_maxinflight_bytes
, >, 0);
4831 mutex_enter(&spa
->spa_scrub_lock
);
4832 while (spa
->spa_scrub_inflight
>= scn
->scn_maxinflight_bytes
)
4833 cv_wait(&spa
->spa_scrub_io_cv
, &spa
->spa_scrub_lock
);
4834 spa
->spa_scrub_inflight
+= BP_GET_PSIZE(bp
);
4835 mutex_exit(&spa
->spa_scrub_lock
);
4836 pio
= scn
->scn_zio_root
;
4838 kmutex_t
*q_lock
= &queue
->q_vd
->vdev_scan_io_queue_lock
;
4840 ASSERT3U(queue
->q_maxinflight_bytes
, >, 0);
4841 mutex_enter(q_lock
);
4842 while (queue
->q_inflight_bytes
>= queue
->q_maxinflight_bytes
)
4843 cv_wait(&queue
->q_zio_cv
, q_lock
);
4844 queue
->q_inflight_bytes
+= BP_GET_PSIZE(bp
);
4849 ASSERT(pio
!= NULL
);
4850 count_block_issued(spa
, bp
, queue
== NULL
);
4851 zio_nowait(zio_read(pio
, spa
, bp
, data
, size
, dsl_scan_scrub_done
,
4852 queue
, ZIO_PRIORITY_SCRUB
, zio_flags
, zb
));
4856 * This is the primary extent sorting algorithm. We balance two parameters:
4857 * 1) how many bytes of I/O are in an extent
4858 * 2) how well the extent is filled with I/O (as a fraction of its total size)
4859 * Since we allow extents to have gaps between their constituent I/Os, it's
4860 * possible to have a fairly large extent that contains the same amount of
4861 * I/O bytes than a much smaller extent, which just packs the I/O more tightly.
4862 * The algorithm sorts based on a score calculated from the extent's size,
4863 * the relative fill volume (in %) and a "fill weight" parameter that controls
4864 * the split between whether we prefer larger extents or more well populated
4867 * SCORE = FILL_IN_BYTES + (FILL_IN_PERCENT * FILL_IN_BYTES * FILL_WEIGHT)
4870 * 1) assume extsz = 64 MiB
4871 * 2) assume fill = 32 MiB (extent is half full)
4872 * 3) assume fill_weight = 3
4873 * 4) SCORE = 32M + (((32M * 100) / 64M) * 3 * 32M) / 100
4874 * SCORE = 32M + (50 * 3 * 32M) / 100
4875 * SCORE = 32M + (4800M / 100)
4878 * | +--- final total relative fill-based score
4879 * +--------- final total fill-based score
4882 * As can be seen, at fill_ratio=3, the algorithm is slightly biased towards
4883 * extents that are more completely filled (in a 3:2 ratio) vs just larger.
4884 * Note that as an optimization, we replace multiplication and division by
4885 * 100 with bitshifting by 7 (which effectively multiplies and divides by 128).
4887 * Since we do not care if one extent is only few percent better than another,
4888 * compress the score into 6 bits via binary logarithm AKA highbit64() and
4889 * put into otherwise unused due to ashift high bits of offset. This allows
4890 * to reduce q_exts_by_size B-tree elements to only 64 bits and compare them
4891 * with single operation. Plus it makes scrubs more sequential and reduces
4892 * chances that minor extent change move it within the B-tree.
4894 __attribute__((always_inline
)) inline
4896 ext_size_compare(const void *x
, const void *y
)
4898 const uint64_t *a
= x
, *b
= y
;
4900 return (TREE_CMP(*a
, *b
));
4903 ZFS_BTREE_FIND_IN_BUF_FUNC(ext_size_find_in_buf
, uint64_t,
4907 ext_size_create(range_tree_t
*rt
, void *arg
)
4910 zfs_btree_t
*size_tree
= arg
;
4912 zfs_btree_create(size_tree
, ext_size_compare
, ext_size_find_in_buf
,
4917 ext_size_destroy(range_tree_t
*rt
, void *arg
)
4920 zfs_btree_t
*size_tree
= arg
;
4921 ASSERT0(zfs_btree_numnodes(size_tree
));
4923 zfs_btree_destroy(size_tree
);
4927 ext_size_value(range_tree_t
*rt
, range_seg_gap_t
*rsg
)
4930 uint64_t size
= rsg
->rs_end
- rsg
->rs_start
;
4931 uint64_t score
= rsg
->rs_fill
+ ((((rsg
->rs_fill
<< 7) / size
) *
4932 fill_weight
* rsg
->rs_fill
) >> 7);
4933 ASSERT3U(rt
->rt_shift
, >=, 8);
4934 return (((uint64_t)(64 - highbit64(score
)) << 56) | rsg
->rs_start
);
4938 ext_size_add(range_tree_t
*rt
, range_seg_t
*rs
, void *arg
)
4940 zfs_btree_t
*size_tree
= arg
;
4941 ASSERT3U(rt
->rt_type
, ==, RANGE_SEG_GAP
);
4942 uint64_t v
= ext_size_value(rt
, (range_seg_gap_t
*)rs
);
4943 zfs_btree_add(size_tree
, &v
);
4947 ext_size_remove(range_tree_t
*rt
, range_seg_t
*rs
, void *arg
)
4949 zfs_btree_t
*size_tree
= arg
;
4950 ASSERT3U(rt
->rt_type
, ==, RANGE_SEG_GAP
);
4951 uint64_t v
= ext_size_value(rt
, (range_seg_gap_t
*)rs
);
4952 zfs_btree_remove(size_tree
, &v
);
4956 ext_size_vacate(range_tree_t
*rt
, void *arg
)
4958 zfs_btree_t
*size_tree
= arg
;
4959 zfs_btree_clear(size_tree
);
4960 zfs_btree_destroy(size_tree
);
4962 ext_size_create(rt
, arg
);
4965 static const range_tree_ops_t ext_size_ops
= {
4966 .rtop_create
= ext_size_create
,
4967 .rtop_destroy
= ext_size_destroy
,
4968 .rtop_add
= ext_size_add
,
4969 .rtop_remove
= ext_size_remove
,
4970 .rtop_vacate
= ext_size_vacate
4974 * Comparator for the q_sios_by_addr tree. Sorting is simply performed
4975 * based on LBA-order (from lowest to highest).
4978 sio_addr_compare(const void *x
, const void *y
)
4980 const scan_io_t
*a
= x
, *b
= y
;
4982 return (TREE_CMP(SIO_GET_OFFSET(a
), SIO_GET_OFFSET(b
)));
4985 /* IO queues are created on demand when they are needed. */
4986 static dsl_scan_io_queue_t
*
4987 scan_io_queue_create(vdev_t
*vd
)
4989 dsl_scan_t
*scn
= vd
->vdev_spa
->spa_dsl_pool
->dp_scan
;
4990 dsl_scan_io_queue_t
*q
= kmem_zalloc(sizeof (*q
), KM_SLEEP
);
4994 q
->q_sio_memused
= 0;
4995 q
->q_last_ext_addr
= -1;
4996 cv_init(&q
->q_zio_cv
, NULL
, CV_DEFAULT
, NULL
);
4997 q
->q_exts_by_addr
= range_tree_create_gap(&ext_size_ops
, RANGE_SEG_GAP
,
4998 &q
->q_exts_by_size
, 0, vd
->vdev_ashift
, zfs_scan_max_ext_gap
);
4999 avl_create(&q
->q_sios_by_addr
, sio_addr_compare
,
5000 sizeof (scan_io_t
), offsetof(scan_io_t
, sio_nodes
.sio_addr_node
));
5006 * Destroys a scan queue and all segments and scan_io_t's contained in it.
5007 * No further execution of I/O occurs, anything pending in the queue is
5008 * simply freed without being executed.
5011 dsl_scan_io_queue_destroy(dsl_scan_io_queue_t
*queue
)
5013 dsl_scan_t
*scn
= queue
->q_scn
;
5015 void *cookie
= NULL
;
5017 ASSERT(MUTEX_HELD(&queue
->q_vd
->vdev_scan_io_queue_lock
));
5019 if (!avl_is_empty(&queue
->q_sios_by_addr
))
5020 atomic_add_64(&scn
->scn_queues_pending
, -1);
5021 while ((sio
= avl_destroy_nodes(&queue
->q_sios_by_addr
, &cookie
)) !=
5023 ASSERT(range_tree_contains(queue
->q_exts_by_addr
,
5024 SIO_GET_OFFSET(sio
), SIO_GET_ASIZE(sio
)));
5025 queue
->q_sio_memused
-= SIO_GET_MUSED(sio
);
5029 ASSERT0(queue
->q_sio_memused
);
5030 range_tree_vacate(queue
->q_exts_by_addr
, NULL
, queue
);
5031 range_tree_destroy(queue
->q_exts_by_addr
);
5032 avl_destroy(&queue
->q_sios_by_addr
);
5033 cv_destroy(&queue
->q_zio_cv
);
5035 kmem_free(queue
, sizeof (*queue
));
5039 * Properly transfers a dsl_scan_queue_t from `svd' to `tvd'. This is
5040 * called on behalf of vdev_top_transfer when creating or destroying
5041 * a mirror vdev due to zpool attach/detach.
5044 dsl_scan_io_queue_vdev_xfer(vdev_t
*svd
, vdev_t
*tvd
)
5046 mutex_enter(&svd
->vdev_scan_io_queue_lock
);
5047 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
5049 VERIFY3P(tvd
->vdev_scan_io_queue
, ==, NULL
);
5050 tvd
->vdev_scan_io_queue
= svd
->vdev_scan_io_queue
;
5051 svd
->vdev_scan_io_queue
= NULL
;
5052 if (tvd
->vdev_scan_io_queue
!= NULL
)
5053 tvd
->vdev_scan_io_queue
->q_vd
= tvd
;
5055 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
5056 mutex_exit(&svd
->vdev_scan_io_queue_lock
);
5060 scan_io_queues_destroy(dsl_scan_t
*scn
)
5062 vdev_t
*rvd
= scn
->scn_dp
->dp_spa
->spa_root_vdev
;
5064 for (uint64_t i
= 0; i
< rvd
->vdev_children
; i
++) {
5065 vdev_t
*tvd
= rvd
->vdev_child
[i
];
5067 mutex_enter(&tvd
->vdev_scan_io_queue_lock
);
5068 if (tvd
->vdev_scan_io_queue
!= NULL
)
5069 dsl_scan_io_queue_destroy(tvd
->vdev_scan_io_queue
);
5070 tvd
->vdev_scan_io_queue
= NULL
;
5071 mutex_exit(&tvd
->vdev_scan_io_queue_lock
);
5076 dsl_scan_freed_dva(spa_t
*spa
, const blkptr_t
*bp
, int dva_i
)
5078 dsl_pool_t
*dp
= spa
->spa_dsl_pool
;
5079 dsl_scan_t
*scn
= dp
->dp_scan
;
5082 dsl_scan_io_queue_t
*queue
;
5083 scan_io_t
*srch_sio
, *sio
;
5085 uint64_t start
, size
;
5087 vdev
= vdev_lookup_top(spa
, DVA_GET_VDEV(&bp
->blk_dva
[dva_i
]));
5088 ASSERT(vdev
!= NULL
);
5089 q_lock
= &vdev
->vdev_scan_io_queue_lock
;
5090 queue
= vdev
->vdev_scan_io_queue
;
5092 mutex_enter(q_lock
);
5093 if (queue
== NULL
) {
5098 srch_sio
= sio_alloc(BP_GET_NDVAS(bp
));
5099 bp2sio(bp
, srch_sio
, dva_i
);
5100 start
= SIO_GET_OFFSET(srch_sio
);
5101 size
= SIO_GET_ASIZE(srch_sio
);
5104 * We can find the zio in two states:
5105 * 1) Cold, just sitting in the queue of zio's to be issued at
5106 * some point in the future. In this case, all we do is
5107 * remove the zio from the q_sios_by_addr tree, decrement
5108 * its data volume from the containing range_seg_t and
5109 * resort the q_exts_by_size tree to reflect that the
5110 * range_seg_t has lost some of its 'fill'. We don't shorten
5111 * the range_seg_t - this is usually rare enough not to be
5112 * worth the extra hassle of trying keep track of precise
5113 * extent boundaries.
5114 * 2) Hot, where the zio is currently in-flight in
5115 * dsl_scan_issue_ios. In this case, we can't simply
5116 * reach in and stop the in-flight zio's, so we instead
5117 * block the caller. Eventually, dsl_scan_issue_ios will
5118 * be done with issuing the zio's it gathered and will
5121 sio
= avl_find(&queue
->q_sios_by_addr
, srch_sio
, &idx
);
5127 /* Got it while it was cold in the queue */
5128 ASSERT3U(start
, ==, SIO_GET_OFFSET(sio
));
5129 ASSERT3U(size
, ==, SIO_GET_ASIZE(sio
));
5130 avl_remove(&queue
->q_sios_by_addr
, sio
);
5131 if (avl_is_empty(&queue
->q_sios_by_addr
))
5132 atomic_add_64(&scn
->scn_queues_pending
, -1);
5133 queue
->q_sio_memused
-= SIO_GET_MUSED(sio
);
5135 ASSERT(range_tree_contains(queue
->q_exts_by_addr
, start
, size
));
5136 range_tree_remove_fill(queue
->q_exts_by_addr
, start
, size
);
5138 /* count the block as though we skipped it */
5139 sio2bp(sio
, &tmpbp
);
5140 count_block_skipped(scn
, &tmpbp
, B_FALSE
);
5148 * Callback invoked when a zio_free() zio is executing. This needs to be
5149 * intercepted to prevent the zio from deallocating a particular portion
5150 * of disk space and it then getting reallocated and written to, while we
5151 * still have it queued up for processing.
5154 dsl_scan_freed(spa_t
*spa
, const blkptr_t
*bp
)
5156 dsl_pool_t
*dp
= spa
->spa_dsl_pool
;
5157 dsl_scan_t
*scn
= dp
->dp_scan
;
5159 ASSERT(!BP_IS_EMBEDDED(bp
));
5160 ASSERT(scn
!= NULL
);
5161 if (!dsl_scan_is_running(scn
))
5164 for (int i
= 0; i
< BP_GET_NDVAS(bp
); i
++)
5165 dsl_scan_freed_dva(spa
, bp
, i
);
5169 * Check if a vdev needs resilvering (non-empty DTL), if so, and resilver has
5170 * not started, start it. Otherwise, only restart if max txg in DTL range is
5171 * greater than the max txg in the current scan. If the DTL max is less than
5172 * the scan max, then the vdev has not missed any new data since the resilver
5173 * started, so a restart is not needed.
5176 dsl_scan_assess_vdev(dsl_pool_t
*dp
, vdev_t
*vd
)
5180 if (!vdev_resilver_needed(vd
, &min
, &max
))
5183 if (!dsl_scan_resilvering(dp
)) {
5184 spa_async_request(dp
->dp_spa
, SPA_ASYNC_RESILVER
);
5188 if (max
<= dp
->dp_scan
->scn_phys
.scn_max_txg
)
5191 /* restart is needed, check if it can be deferred */
5192 if (spa_feature_is_enabled(dp
->dp_spa
, SPA_FEATURE_RESILVER_DEFER
))
5193 vdev_defer_resilver(vd
);
5195 spa_async_request(dp
->dp_spa
, SPA_ASYNC_RESILVER
);
5198 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_vdev_limit
, U64
, ZMOD_RW
,
5199 "Max bytes in flight per leaf vdev for scrubs and resilvers");
5201 ZFS_MODULE_PARAM(zfs
, zfs_
, scrub_min_time_ms
, UINT
, ZMOD_RW
,
5202 "Min millisecs to scrub per txg");
5204 ZFS_MODULE_PARAM(zfs
, zfs_
, obsolete_min_time_ms
, UINT
, ZMOD_RW
,
5205 "Min millisecs to obsolete per txg");
5207 ZFS_MODULE_PARAM(zfs
, zfs_
, free_min_time_ms
, UINT
, ZMOD_RW
,
5208 "Min millisecs to free per txg");
5210 ZFS_MODULE_PARAM(zfs
, zfs_
, resilver_min_time_ms
, UINT
, ZMOD_RW
,
5211 "Min millisecs to resilver per txg");
5213 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_suspend_progress
, INT
, ZMOD_RW
,
5214 "Set to prevent scans from progressing");
5216 ZFS_MODULE_PARAM(zfs
, zfs_
, no_scrub_io
, INT
, ZMOD_RW
,
5217 "Set to disable scrub I/O");
5219 ZFS_MODULE_PARAM(zfs
, zfs_
, no_scrub_prefetch
, INT
, ZMOD_RW
,
5220 "Set to disable scrub prefetching");
5222 ZFS_MODULE_PARAM(zfs
, zfs_
, async_block_max_blocks
, U64
, ZMOD_RW
,
5223 "Max number of blocks freed in one txg");
5225 ZFS_MODULE_PARAM(zfs
, zfs_
, max_async_dedup_frees
, U64
, ZMOD_RW
,
5226 "Max number of dedup blocks freed in one txg");
5228 ZFS_MODULE_PARAM(zfs
, zfs_
, free_bpobj_enabled
, INT
, ZMOD_RW
,
5229 "Enable processing of the free_bpobj");
5231 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_blkstats
, INT
, ZMOD_RW
,
5232 "Enable block statistics calculation during scrub");
5234 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_mem_lim_fact
, UINT
, ZMOD_RW
,
5235 "Fraction of RAM for scan hard limit");
5237 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_issue_strategy
, UINT
, ZMOD_RW
,
5238 "IO issuing strategy during scrubbing. 0 = default, 1 = LBA, 2 = size");
5240 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_legacy
, INT
, ZMOD_RW
,
5241 "Scrub using legacy non-sequential method");
5243 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_checkpoint_intval
, UINT
, ZMOD_RW
,
5244 "Scan progress on-disk checkpointing interval");
5246 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_max_ext_gap
, U64
, ZMOD_RW
,
5247 "Max gap in bytes between sequential scrub / resilver I/Os");
5249 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_mem_lim_soft_fact
, UINT
, ZMOD_RW
,
5250 "Fraction of hard limit used as soft limit");
5252 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_strict_mem_lim
, INT
, ZMOD_RW
,
5253 "Tunable to attempt to reduce lock contention");
5255 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_fill_weight
, UINT
, ZMOD_RW
,
5256 "Tunable to adjust bias towards more filled segments during scans");
5258 ZFS_MODULE_PARAM(zfs
, zfs_
, scan_report_txgs
, UINT
, ZMOD_RW
,
5259 "Tunable to report resilver performance over the last N txgs");
5261 ZFS_MODULE_PARAM(zfs
, zfs_
, resilver_disable_defer
, INT
, ZMOD_RW
,
5262 "Process all resilvers immediately");
5264 ZFS_MODULE_PARAM(zfs
, zfs_
, scrub_error_blocks_per_txg
, UINT
, ZMOD_RW
,
5265 "Error blocks to be scrubbed in one txg");