4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
9 * A full copy of the text of the CDDL should have accompanied this
10 * source. A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
17 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
20 #include <sys/zfs_context.h>
22 #include <sys/spa_impl.h>
23 #include <sys/vdev_impl.h>
24 #include <sys/fs/zfs.h>
26 #include <sys/zio_checksum.h>
27 #include <sys/metaslab.h>
28 #include <sys/refcount.h>
30 #include <sys/vdev_indirect_mapping.h>
31 #include <sys/dmu_tx.h>
32 #include <sys/dsl_synctask.h>
38 * An indirect vdev corresponds to a vdev that has been removed. Since
39 * we cannot rewrite block pointers of snapshots, etc., we keep a
40 * mapping from old location on the removed device to the new location
41 * on another device in the pool and use this mapping whenever we need
42 * to access the DVA. Unfortunately, this mapping did not respect
43 * logical block boundaries when it was first created, and so a DVA on
44 * this indirect vdev may be "split" into multiple sections that each
45 * map to a different location. As a consequence, not all DVAs can be
46 * translated to an equivalent new DVA. Instead we must provide a
47 * "vdev_remap" operation that executes a callback on each contiguous
48 * segment of the new location. This function is used in multiple ways:
50 * - i/os to this vdev use the callback to determine where the
51 * data is now located, and issue child i/os for each segment's new
54 * - frees and claims to this vdev use the callback to free or claim
55 * each mapped segment. (Note that we don't actually need to claim
56 * log blocks on indirect vdevs, because we don't allocate to
57 * removing vdevs. However, zdb uses zio_claim() for its leak
62 * "Big theory statement" for how we mark blocks obsolete.
64 * When a block on an indirect vdev is freed or remapped, a section of
65 * that vdev's mapping may no longer be referenced (aka "obsolete"). We
66 * keep track of how much of each mapping entry is obsolete. When
67 * an entry becomes completely obsolete, we can remove it, thus reducing
68 * the memory used by the mapping. The complete picture of obsolescence
69 * is given by the following data structures, described below:
70 * - the entry-specific obsolete count
71 * - the vdev-specific obsolete spacemap
72 * - the pool-specific obsolete bpobj
74 * == On disk data structures used ==
76 * We track the obsolete space for the pool using several objects. Each
77 * of these objects is created on demand and freed when no longer
78 * needed, and is assumed to be empty if it does not exist.
79 * SPA_FEATURE_OBSOLETE_COUNTS includes the count of these objects.
81 * - Each vic_mapping_object (associated with an indirect vdev) can
82 * have a vimp_counts_object. This is an array of uint32_t's
83 * with the same number of entries as the vic_mapping_object. When
84 * the mapping is condensed, entries from the vic_obsolete_sm_object
85 * (see below) are folded into the counts. Therefore, each
86 * obsolete_counts entry tells us the number of bytes in the
87 * corresponding mapping entry that were not referenced when the
88 * mapping was last condensed.
90 * - Each indirect or removing vdev can have a vic_obsolete_sm_object.
91 * This is a space map containing an alloc entry for every DVA that
92 * has been obsoleted since the last time this indirect vdev was
93 * condensed. We use this object in order to improve performance
94 * when marking a DVA as obsolete. Instead of modifying an arbitrary
95 * offset of the vimp_counts_object, we only need to append an entry
96 * to the end of this object. When a DVA becomes obsolete, it is
97 * added to the obsolete space map. This happens when the DVA is
98 * freed, remapped and not referenced by a snapshot, or the last
99 * snapshot referencing it is destroyed.
101 * - Each dataset can have a ds_remap_deadlist object. This is a
102 * deadlist object containing all blocks that were remapped in this
103 * dataset but referenced in a previous snapshot. Blocks can *only*
104 * appear on this list if they were remapped (dsl_dataset_block_remapped);
105 * blocks that were killed in a head dataset are put on the normal
106 * ds_deadlist and marked obsolete when they are freed.
108 * - The pool can have a dp_obsolete_bpobj. This is a list of blocks
109 * in the pool that need to be marked obsolete. When a snapshot is
110 * destroyed, we move some of the ds_remap_deadlist to the obsolete
111 * bpobj (see dsl_destroy_snapshot_handle_remaps()). We then
112 * asynchronously process the obsolete bpobj, moving its entries to
113 * the specific vdevs' obsolete space maps.
115 * == Summary of how we mark blocks as obsolete ==
117 * - When freeing a block: if any DVA is on an indirect vdev, append to
118 * vic_obsolete_sm_object.
119 * - When remapping a block, add dva to ds_remap_deadlist (if prev snap
120 * references; otherwise append to vic_obsolete_sm_object).
121 * - When freeing a snapshot: move parts of ds_remap_deadlist to
122 * dp_obsolete_bpobj (same algorithm as ds_deadlist).
123 * - When syncing the spa: process dp_obsolete_bpobj, moving ranges to
124 * individual vdev's vic_obsolete_sm_object.
128 * "Big theory statement" for how we condense indirect vdevs.
130 * Condensing an indirect vdev's mapping is the process of determining
131 * the precise counts of obsolete space for each mapping entry (by
132 * integrating the obsolete spacemap into the obsolete counts) and
133 * writing out a new mapping that contains only referenced entries.
135 * We condense a vdev when we expect the mapping to shrink (see
136 * vdev_indirect_should_condense()), but only perform one condense at a
137 * time to limit the memory usage. In addition, we use a separate
138 * open-context thread (spa_condense_indirect_thread) to incrementally
139 * create the new mapping object in a way that minimizes the impact on
140 * the rest of the system.
142 * == Generating a new mapping ==
144 * To generate a new mapping, we follow these steps:
146 * 1. Save the old obsolete space map and create a new mapping object
147 * (see spa_condense_indirect_start_sync()). This initializes the
148 * spa_condensing_indirect_phys with the "previous obsolete space map",
149 * which is now read only. Newly obsolete DVAs will be added to a
150 * new (initially empty) obsolete space map, and will not be
151 * considered as part of this condense operation.
153 * 2. Construct in memory the precise counts of obsolete space for each
154 * mapping entry, by incorporating the obsolete space map into the
155 * counts. (See vdev_indirect_mapping_load_obsolete_{counts,spacemap}().)
157 * 3. Iterate through each mapping entry, writing to the new mapping any
158 * entries that are not completely obsolete (i.e. which don't have
159 * obsolete count == mapping length). (See
160 * spa_condense_indirect_generate_new_mapping().)
162 * 4. Destroy the old mapping object and switch over to the new one
163 * (spa_condense_indirect_complete_sync).
165 * == Restarting from failure ==
167 * To restart the condense when we import/open the pool, we must start
168 * at the 2nd step above: reconstruct the precise counts in memory,
169 * based on the space map + counts. Then in the 3rd step, we start
170 * iterating where we left off: at vimp_max_offset of the new mapping
174 int zfs_condense_indirect_vdevs_enable
= B_TRUE
;
177 * Condense if at least this percent of the bytes in the mapping is
178 * obsolete. With the default of 25%, the amount of space mapped
179 * will be reduced to 1% of its original size after at most 16
180 * condenses. Higher values will condense less often (causing less
181 * i/o); lower values will reduce the mapping size more quickly.
183 int zfs_indirect_condense_obsolete_pct
= 25;
186 * Condense if the obsolete space map takes up more than this amount of
187 * space on disk (logically). This limits the amount of disk space
188 * consumed by the obsolete space map; the default of 1GB is small enough
189 * that we typically don't mind "wasting" it.
191 unsigned long zfs_condense_max_obsolete_bytes
= 1024 * 1024 * 1024;
194 * Don't bother condensing if the mapping uses less than this amount of
195 * memory. The default of 128KB is considered a "trivial" amount of
196 * memory and not worth reducing.
198 unsigned long zfs_condense_min_mapping_bytes
= 128 * 1024;
201 * This is used by the test suite so that it can ensure that certain
202 * actions happen while in the middle of a condense (which might otherwise
203 * complete too quickly). If used to reduce the performance impact of
204 * condensing in production, a maximum value of 1 should be sufficient.
206 int zfs_condense_indirect_commit_entry_delay_ms
= 0;
209 * If an indirect split block contains more than this many possible unique
210 * combinations when being reconstructed, consider it too computationally
211 * expensive to check them all. Instead, try at most 100 randomly-selected
212 * combinations each time the block is accessed. This allows all segment
213 * copies to participate fairly in the reconstruction when all combinations
214 * cannot be checked and prevents repeated use of one bad copy.
216 int zfs_reconstruct_indirect_combinations_max
= 256;
220 * Enable to simulate damaged segments and validate reconstruction. This
221 * is intentionally not exposed as a module parameter.
223 unsigned long zfs_reconstruct_indirect_damage_fraction
= 0;
226 * The indirect_child_t represents the vdev that we will read from, when we
227 * need to read all copies of the data (e.g. for scrub or reconstruction).
228 * For plain (non-mirror) top-level vdevs (i.e. is_vdev is not a mirror),
229 * ic_vdev is the same as is_vdev. However, for mirror top-level vdevs,
230 * ic_vdev is a child of the mirror.
232 typedef struct indirect_child
{
237 * ic_duplicate is NULL when the ic_data contents are unique, when it
238 * is determined to be a duplicate it references the primary child.
240 struct indirect_child
*ic_duplicate
;
241 list_node_t ic_node
; /* node on is_unique_child */
245 * The indirect_split_t represents one mapped segment of an i/o to the
246 * indirect vdev. For non-split (contiguously-mapped) blocks, there will be
247 * only one indirect_split_t, with is_split_offset==0 and is_size==io_size.
248 * For split blocks, there will be several of these.
250 typedef struct indirect_split
{
251 list_node_t is_node
; /* link on iv_splits */
254 * is_split_offset is the offset into the i/o.
255 * This is the sum of the previous splits' is_size's.
257 uint64_t is_split_offset
;
259 vdev_t
*is_vdev
; /* top-level vdev */
260 uint64_t is_target_offset
; /* offset on is_vdev */
262 int is_children
; /* number of entries in is_child[] */
263 int is_unique_children
; /* number of entries in is_unique_child */
264 list_t is_unique_child
;
267 * is_good_child is the child that we are currently using to
268 * attempt reconstruction.
270 indirect_child_t
*is_good_child
;
272 indirect_child_t is_child
[1]; /* variable-length */
276 * The indirect_vsd_t is associated with each i/o to the indirect vdev.
277 * It is the "Vdev-Specific Data" in the zio_t's io_vsd.
279 typedef struct indirect_vsd
{
280 boolean_t iv_split_block
;
281 boolean_t iv_reconstruct
;
282 uint64_t iv_unique_combinations
;
283 uint64_t iv_attempts
;
284 uint64_t iv_attempts_max
;
286 list_t iv_splits
; /* list of indirect_split_t's */
290 vdev_indirect_map_free(zio_t
*zio
)
292 indirect_vsd_t
*iv
= zio
->io_vsd
;
294 indirect_split_t
*is
;
295 while ((is
= list_head(&iv
->iv_splits
)) != NULL
) {
296 for (int c
= 0; c
< is
->is_children
; c
++) {
297 indirect_child_t
*ic
= &is
->is_child
[c
];
298 if (ic
->ic_data
!= NULL
)
299 abd_free(ic
->ic_data
);
301 list_remove(&iv
->iv_splits
, is
);
303 indirect_child_t
*ic
;
304 while ((ic
= list_head(&is
->is_unique_child
)) != NULL
)
305 list_remove(&is
->is_unique_child
, ic
);
307 list_destroy(&is
->is_unique_child
);
310 offsetof(indirect_split_t
, is_child
[is
->is_children
]));
312 kmem_free(iv
, sizeof (*iv
));
315 static const zio_vsd_ops_t vdev_indirect_vsd_ops
= {
316 .vsd_free
= vdev_indirect_map_free
,
317 .vsd_cksum_report
= zio_vsd_default_cksum_report
321 * Mark the given offset and size as being obsolete.
324 vdev_indirect_mark_obsolete(vdev_t
*vd
, uint64_t offset
, uint64_t size
)
326 spa_t
*spa
= vd
->vdev_spa
;
328 ASSERT3U(vd
->vdev_indirect_config
.vic_mapping_object
, !=, 0);
329 ASSERT(vd
->vdev_removing
|| vd
->vdev_ops
== &vdev_indirect_ops
);
331 VERIFY(vdev_indirect_mapping_entry_for_offset(
332 vd
->vdev_indirect_mapping
, offset
) != NULL
);
334 if (spa_feature_is_enabled(spa
, SPA_FEATURE_OBSOLETE_COUNTS
)) {
335 mutex_enter(&vd
->vdev_obsolete_lock
);
336 range_tree_add(vd
->vdev_obsolete_segments
, offset
, size
);
337 mutex_exit(&vd
->vdev_obsolete_lock
);
338 vdev_dirty(vd
, 0, NULL
, spa_syncing_txg(spa
));
343 * Mark the DVA vdev_id:offset:size as being obsolete in the given tx. This
344 * wrapper is provided because the DMU does not know about vdev_t's and
345 * cannot directly call vdev_indirect_mark_obsolete.
348 spa_vdev_indirect_mark_obsolete(spa_t
*spa
, uint64_t vdev_id
, uint64_t offset
,
349 uint64_t size
, dmu_tx_t
*tx
)
351 vdev_t
*vd
= vdev_lookup_top(spa
, vdev_id
);
352 ASSERT(dmu_tx_is_syncing(tx
));
354 /* The DMU can only remap indirect vdevs. */
355 ASSERT3P(vd
->vdev_ops
, ==, &vdev_indirect_ops
);
356 vdev_indirect_mark_obsolete(vd
, offset
, size
);
359 static spa_condensing_indirect_t
*
360 spa_condensing_indirect_create(spa_t
*spa
)
362 spa_condensing_indirect_phys_t
*scip
=
363 &spa
->spa_condensing_indirect_phys
;
364 spa_condensing_indirect_t
*sci
= kmem_zalloc(sizeof (*sci
), KM_SLEEP
);
365 objset_t
*mos
= spa
->spa_meta_objset
;
367 for (int i
= 0; i
< TXG_SIZE
; i
++) {
368 list_create(&sci
->sci_new_mapping_entries
[i
],
369 sizeof (vdev_indirect_mapping_entry_t
),
370 offsetof(vdev_indirect_mapping_entry_t
, vime_node
));
373 sci
->sci_new_mapping
=
374 vdev_indirect_mapping_open(mos
, scip
->scip_next_mapping_object
);
380 spa_condensing_indirect_destroy(spa_condensing_indirect_t
*sci
)
382 for (int i
= 0; i
< TXG_SIZE
; i
++)
383 list_destroy(&sci
->sci_new_mapping_entries
[i
]);
385 if (sci
->sci_new_mapping
!= NULL
)
386 vdev_indirect_mapping_close(sci
->sci_new_mapping
);
388 kmem_free(sci
, sizeof (*sci
));
392 vdev_indirect_should_condense(vdev_t
*vd
)
394 vdev_indirect_mapping_t
*vim
= vd
->vdev_indirect_mapping
;
395 spa_t
*spa
= vd
->vdev_spa
;
397 ASSERT(dsl_pool_sync_context(spa
->spa_dsl_pool
));
399 if (!zfs_condense_indirect_vdevs_enable
)
403 * We can only condense one indirect vdev at a time.
405 if (spa
->spa_condensing_indirect
!= NULL
)
408 if (spa_shutting_down(spa
))
412 * The mapping object size must not change while we are
413 * condensing, so we can only condense indirect vdevs
414 * (not vdevs that are still in the middle of being removed).
416 if (vd
->vdev_ops
!= &vdev_indirect_ops
)
420 * If nothing new has been marked obsolete, there is no
421 * point in condensing.
423 ASSERTV(uint64_t obsolete_sm_obj
);
424 ASSERT0(vdev_obsolete_sm_object(vd
, &obsolete_sm_obj
));
425 if (vd
->vdev_obsolete_sm
== NULL
) {
426 ASSERT0(obsolete_sm_obj
);
430 ASSERT(vd
->vdev_obsolete_sm
!= NULL
);
432 ASSERT3U(obsolete_sm_obj
, ==, space_map_object(vd
->vdev_obsolete_sm
));
434 uint64_t bytes_mapped
= vdev_indirect_mapping_bytes_mapped(vim
);
435 uint64_t bytes_obsolete
= space_map_allocated(vd
->vdev_obsolete_sm
);
436 uint64_t mapping_size
= vdev_indirect_mapping_size(vim
);
437 uint64_t obsolete_sm_size
= space_map_length(vd
->vdev_obsolete_sm
);
439 ASSERT3U(bytes_obsolete
, <=, bytes_mapped
);
442 * If a high percentage of the bytes that are mapped have become
443 * obsolete, condense (unless the mapping is already small enough).
444 * This has a good chance of reducing the amount of memory used
447 if (bytes_obsolete
* 100 / bytes_mapped
>=
448 zfs_indirect_condense_obsolete_pct
&&
449 mapping_size
> zfs_condense_min_mapping_bytes
) {
450 zfs_dbgmsg("should condense vdev %llu because obsolete "
451 "spacemap covers %d%% of %lluMB mapping",
452 (u_longlong_t
)vd
->vdev_id
,
453 (int)(bytes_obsolete
* 100 / bytes_mapped
),
454 (u_longlong_t
)bytes_mapped
/ 1024 / 1024);
459 * If the obsolete space map takes up too much space on disk,
460 * condense in order to free up this disk space.
462 if (obsolete_sm_size
>= zfs_condense_max_obsolete_bytes
) {
463 zfs_dbgmsg("should condense vdev %llu because obsolete sm "
464 "length %lluMB >= max size %lluMB",
465 (u_longlong_t
)vd
->vdev_id
,
466 (u_longlong_t
)obsolete_sm_size
/ 1024 / 1024,
467 (u_longlong_t
)zfs_condense_max_obsolete_bytes
/
476 * This sync task completes (finishes) a condense, deleting the old
477 * mapping and replacing it with the new one.
480 spa_condense_indirect_complete_sync(void *arg
, dmu_tx_t
*tx
)
482 spa_condensing_indirect_t
*sci
= arg
;
483 spa_t
*spa
= dmu_tx_pool(tx
)->dp_spa
;
484 spa_condensing_indirect_phys_t
*scip
=
485 &spa
->spa_condensing_indirect_phys
;
486 vdev_t
*vd
= vdev_lookup_top(spa
, scip
->scip_vdev
);
487 vdev_indirect_config_t
*vic
= &vd
->vdev_indirect_config
;
488 objset_t
*mos
= spa
->spa_meta_objset
;
489 vdev_indirect_mapping_t
*old_mapping
= vd
->vdev_indirect_mapping
;
490 uint64_t old_count
= vdev_indirect_mapping_num_entries(old_mapping
);
492 vdev_indirect_mapping_num_entries(sci
->sci_new_mapping
);
494 ASSERT(dmu_tx_is_syncing(tx
));
495 ASSERT3P(vd
->vdev_ops
, ==, &vdev_indirect_ops
);
496 ASSERT3P(sci
, ==, spa
->spa_condensing_indirect
);
497 for (int i
= 0; i
< TXG_SIZE
; i
++) {
498 ASSERT(list_is_empty(&sci
->sci_new_mapping_entries
[i
]));
500 ASSERT(vic
->vic_mapping_object
!= 0);
501 ASSERT3U(vd
->vdev_id
, ==, scip
->scip_vdev
);
502 ASSERT(scip
->scip_next_mapping_object
!= 0);
503 ASSERT(scip
->scip_prev_obsolete_sm_object
!= 0);
506 * Reset vdev_indirect_mapping to refer to the new object.
508 rw_enter(&vd
->vdev_indirect_rwlock
, RW_WRITER
);
509 vdev_indirect_mapping_close(vd
->vdev_indirect_mapping
);
510 vd
->vdev_indirect_mapping
= sci
->sci_new_mapping
;
511 rw_exit(&vd
->vdev_indirect_rwlock
);
513 sci
->sci_new_mapping
= NULL
;
514 vdev_indirect_mapping_free(mos
, vic
->vic_mapping_object
, tx
);
515 vic
->vic_mapping_object
= scip
->scip_next_mapping_object
;
516 scip
->scip_next_mapping_object
= 0;
518 space_map_free_obj(mos
, scip
->scip_prev_obsolete_sm_object
, tx
);
519 spa_feature_decr(spa
, SPA_FEATURE_OBSOLETE_COUNTS
, tx
);
520 scip
->scip_prev_obsolete_sm_object
= 0;
524 VERIFY0(zap_remove(mos
, DMU_POOL_DIRECTORY_OBJECT
,
525 DMU_POOL_CONDENSING_INDIRECT
, tx
));
526 spa_condensing_indirect_destroy(spa
->spa_condensing_indirect
);
527 spa
->spa_condensing_indirect
= NULL
;
529 zfs_dbgmsg("finished condense of vdev %llu in txg %llu: "
530 "new mapping object %llu has %llu entries "
531 "(was %llu entries)",
532 vd
->vdev_id
, dmu_tx_get_txg(tx
), vic
->vic_mapping_object
,
533 new_count
, old_count
);
535 vdev_config_dirty(spa
->spa_root_vdev
);
539 * This sync task appends entries to the new mapping object.
542 spa_condense_indirect_commit_sync(void *arg
, dmu_tx_t
*tx
)
544 spa_condensing_indirect_t
*sci
= arg
;
545 uint64_t txg
= dmu_tx_get_txg(tx
);
546 ASSERTV(spa_t
*spa
= dmu_tx_pool(tx
)->dp_spa
);
548 ASSERT(dmu_tx_is_syncing(tx
));
549 ASSERT3P(sci
, ==, spa
->spa_condensing_indirect
);
551 vdev_indirect_mapping_add_entries(sci
->sci_new_mapping
,
552 &sci
->sci_new_mapping_entries
[txg
& TXG_MASK
], tx
);
553 ASSERT(list_is_empty(&sci
->sci_new_mapping_entries
[txg
& TXG_MASK
]));
557 * Open-context function to add one entry to the new mapping. The new
558 * entry will be remembered and written from syncing context.
561 spa_condense_indirect_commit_entry(spa_t
*spa
,
562 vdev_indirect_mapping_entry_phys_t
*vimep
, uint32_t count
)
564 spa_condensing_indirect_t
*sci
= spa
->spa_condensing_indirect
;
566 ASSERT3U(count
, <, DVA_GET_ASIZE(&vimep
->vimep_dst
));
568 dmu_tx_t
*tx
= dmu_tx_create_dd(spa_get_dsl(spa
)->dp_mos_dir
);
569 dmu_tx_hold_space(tx
, sizeof (*vimep
) + sizeof (count
));
570 VERIFY0(dmu_tx_assign(tx
, TXG_WAIT
));
571 int txgoff
= dmu_tx_get_txg(tx
) & TXG_MASK
;
574 * If we are the first entry committed this txg, kick off the sync
575 * task to write to the MOS on our behalf.
577 if (list_is_empty(&sci
->sci_new_mapping_entries
[txgoff
])) {
578 dsl_sync_task_nowait(dmu_tx_pool(tx
),
579 spa_condense_indirect_commit_sync
, sci
,
580 0, ZFS_SPACE_CHECK_NONE
, tx
);
583 vdev_indirect_mapping_entry_t
*vime
=
584 kmem_alloc(sizeof (*vime
), KM_SLEEP
);
585 vime
->vime_mapping
= *vimep
;
586 vime
->vime_obsolete_count
= count
;
587 list_insert_tail(&sci
->sci_new_mapping_entries
[txgoff
], vime
);
593 spa_condense_indirect_generate_new_mapping(vdev_t
*vd
,
594 uint32_t *obsolete_counts
, uint64_t start_index
, zthr_t
*zthr
)
596 spa_t
*spa
= vd
->vdev_spa
;
597 uint64_t mapi
= start_index
;
598 vdev_indirect_mapping_t
*old_mapping
= vd
->vdev_indirect_mapping
;
599 uint64_t old_num_entries
=
600 vdev_indirect_mapping_num_entries(old_mapping
);
602 ASSERT3P(vd
->vdev_ops
, ==, &vdev_indirect_ops
);
603 ASSERT3U(vd
->vdev_id
, ==, spa
->spa_condensing_indirect_phys
.scip_vdev
);
605 zfs_dbgmsg("starting condense of vdev %llu from index %llu",
606 (u_longlong_t
)vd
->vdev_id
,
609 while (mapi
< old_num_entries
) {
611 if (zthr_iscancelled(zthr
)) {
612 zfs_dbgmsg("pausing condense of vdev %llu "
613 "at index %llu", (u_longlong_t
)vd
->vdev_id
,
618 vdev_indirect_mapping_entry_phys_t
*entry
=
619 &old_mapping
->vim_entries
[mapi
];
620 uint64_t entry_size
= DVA_GET_ASIZE(&entry
->vimep_dst
);
621 ASSERT3U(obsolete_counts
[mapi
], <=, entry_size
);
622 if (obsolete_counts
[mapi
] < entry_size
) {
623 spa_condense_indirect_commit_entry(spa
, entry
,
624 obsolete_counts
[mapi
]);
627 * This delay may be requested for testing, debugging,
628 * or performance reasons.
630 hrtime_t now
= gethrtime();
631 hrtime_t sleep_until
= now
+ MSEC2NSEC(
632 zfs_condense_indirect_commit_entry_delay_ms
);
633 zfs_sleep_until(sleep_until
);
642 spa_condense_indirect_thread_check(void *arg
, zthr_t
*zthr
)
646 return (spa
->spa_condensing_indirect
!= NULL
);
651 spa_condense_indirect_thread(void *arg
, zthr_t
*zthr
)
656 ASSERT3P(spa
->spa_condensing_indirect
, !=, NULL
);
657 spa_config_enter(spa
, SCL_VDEV
, FTAG
, RW_READER
);
658 vd
= vdev_lookup_top(spa
, spa
->spa_condensing_indirect_phys
.scip_vdev
);
659 ASSERT3P(vd
, !=, NULL
);
660 spa_config_exit(spa
, SCL_VDEV
, FTAG
);
662 spa_condensing_indirect_t
*sci
= spa
->spa_condensing_indirect
;
663 spa_condensing_indirect_phys_t
*scip
=
664 &spa
->spa_condensing_indirect_phys
;
666 uint64_t start_index
;
667 vdev_indirect_mapping_t
*old_mapping
= vd
->vdev_indirect_mapping
;
668 space_map_t
*prev_obsolete_sm
= NULL
;
670 ASSERT3U(vd
->vdev_id
, ==, scip
->scip_vdev
);
671 ASSERT(scip
->scip_next_mapping_object
!= 0);
672 ASSERT(scip
->scip_prev_obsolete_sm_object
!= 0);
673 ASSERT3P(vd
->vdev_ops
, ==, &vdev_indirect_ops
);
675 for (int i
= 0; i
< TXG_SIZE
; i
++) {
677 * The list must start out empty in order for the
678 * _commit_sync() sync task to be properly registered
679 * on the first call to _commit_entry(); so it's wise
680 * to double check and ensure we actually are starting
683 ASSERT(list_is_empty(&sci
->sci_new_mapping_entries
[i
]));
686 VERIFY0(space_map_open(&prev_obsolete_sm
, spa
->spa_meta_objset
,
687 scip
->scip_prev_obsolete_sm_object
, 0, vd
->vdev_asize
, 0));
688 space_map_update(prev_obsolete_sm
);
689 counts
= vdev_indirect_mapping_load_obsolete_counts(old_mapping
);
690 if (prev_obsolete_sm
!= NULL
) {
691 vdev_indirect_mapping_load_obsolete_spacemap(old_mapping
,
692 counts
, prev_obsolete_sm
);
694 space_map_close(prev_obsolete_sm
);
697 * Generate new mapping. Determine what index to continue from
698 * based on the max offset that we've already written in the
701 uint64_t max_offset
=
702 vdev_indirect_mapping_max_offset(sci
->sci_new_mapping
);
703 if (max_offset
== 0) {
704 /* We haven't written anything to the new mapping yet. */
708 * Pick up from where we left off. _entry_for_offset()
709 * returns a pointer into the vim_entries array. If
710 * max_offset is greater than any of the mappings
711 * contained in the table NULL will be returned and
712 * that indicates we've exhausted our iteration of the
716 vdev_indirect_mapping_entry_phys_t
*entry
=
717 vdev_indirect_mapping_entry_for_offset_or_next(old_mapping
,
722 * We've already written the whole new mapping.
723 * This special value will cause us to skip the
724 * generate_new_mapping step and just do the sync
725 * task to complete the condense.
727 start_index
= UINT64_MAX
;
729 start_index
= entry
- old_mapping
->vim_entries
;
730 ASSERT3U(start_index
, <,
731 vdev_indirect_mapping_num_entries(old_mapping
));
735 spa_condense_indirect_generate_new_mapping(vd
, counts
,
738 vdev_indirect_mapping_free_obsolete_counts(old_mapping
, counts
);
741 * If the zthr has received a cancellation signal while running
742 * in generate_new_mapping() or at any point after that, then bail
743 * early. We don't want to complete the condense if the spa is
746 if (zthr_iscancelled(zthr
))
749 VERIFY0(dsl_sync_task(spa_name(spa
), NULL
,
750 spa_condense_indirect_complete_sync
, sci
, 0,
751 ZFS_SPACE_CHECK_EXTRA_RESERVED
));
755 * Sync task to begin the condensing process.
758 spa_condense_indirect_start_sync(vdev_t
*vd
, dmu_tx_t
*tx
)
760 spa_t
*spa
= vd
->vdev_spa
;
761 spa_condensing_indirect_phys_t
*scip
=
762 &spa
->spa_condensing_indirect_phys
;
764 ASSERT0(scip
->scip_next_mapping_object
);
765 ASSERT0(scip
->scip_prev_obsolete_sm_object
);
766 ASSERT0(scip
->scip_vdev
);
767 ASSERT(dmu_tx_is_syncing(tx
));
768 ASSERT3P(vd
->vdev_ops
, ==, &vdev_indirect_ops
);
769 ASSERT(spa_feature_is_active(spa
, SPA_FEATURE_OBSOLETE_COUNTS
));
770 ASSERT(vdev_indirect_mapping_num_entries(vd
->vdev_indirect_mapping
));
772 uint64_t obsolete_sm_obj
;
773 VERIFY0(vdev_obsolete_sm_object(vd
, &obsolete_sm_obj
));
774 ASSERT3U(obsolete_sm_obj
, !=, 0);
776 scip
->scip_vdev
= vd
->vdev_id
;
777 scip
->scip_next_mapping_object
=
778 vdev_indirect_mapping_alloc(spa
->spa_meta_objset
, tx
);
780 scip
->scip_prev_obsolete_sm_object
= obsolete_sm_obj
;
783 * We don't need to allocate a new space map object, since
784 * vdev_indirect_sync_obsolete will allocate one when needed.
786 space_map_close(vd
->vdev_obsolete_sm
);
787 vd
->vdev_obsolete_sm
= NULL
;
788 VERIFY0(zap_remove(spa
->spa_meta_objset
, vd
->vdev_top_zap
,
789 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM
, tx
));
791 VERIFY0(zap_add(spa
->spa_dsl_pool
->dp_meta_objset
,
792 DMU_POOL_DIRECTORY_OBJECT
,
793 DMU_POOL_CONDENSING_INDIRECT
, sizeof (uint64_t),
794 sizeof (*scip
) / sizeof (uint64_t), scip
, tx
));
796 ASSERT3P(spa
->spa_condensing_indirect
, ==, NULL
);
797 spa
->spa_condensing_indirect
= spa_condensing_indirect_create(spa
);
799 zfs_dbgmsg("starting condense of vdev %llu in txg %llu: "
801 vd
->vdev_id
, dmu_tx_get_txg(tx
),
802 (u_longlong_t
)scip
->scip_prev_obsolete_sm_object
,
803 (u_longlong_t
)scip
->scip_next_mapping_object
);
805 zthr_wakeup(spa
->spa_condense_zthr
);
809 * Sync to the given vdev's obsolete space map any segments that are no longer
810 * referenced as of the given txg.
812 * If the obsolete space map doesn't exist yet, create and open it.
815 vdev_indirect_sync_obsolete(vdev_t
*vd
, dmu_tx_t
*tx
)
817 spa_t
*spa
= vd
->vdev_spa
;
818 ASSERTV(vdev_indirect_config_t
*vic
= &vd
->vdev_indirect_config
);
820 ASSERT3U(vic
->vic_mapping_object
, !=, 0);
821 ASSERT(range_tree_space(vd
->vdev_obsolete_segments
) > 0);
822 ASSERT(vd
->vdev_removing
|| vd
->vdev_ops
== &vdev_indirect_ops
);
823 ASSERT(spa_feature_is_enabled(spa
, SPA_FEATURE_OBSOLETE_COUNTS
));
825 uint64_t obsolete_sm_object
;
826 VERIFY0(vdev_obsolete_sm_object(vd
, &obsolete_sm_object
));
827 if (obsolete_sm_object
== 0) {
828 obsolete_sm_object
= space_map_alloc(spa
->spa_meta_objset
,
829 vdev_standard_sm_blksz
, tx
);
831 ASSERT(vd
->vdev_top_zap
!= 0);
832 VERIFY0(zap_add(vd
->vdev_spa
->spa_meta_objset
, vd
->vdev_top_zap
,
833 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM
,
834 sizeof (obsolete_sm_object
), 1, &obsolete_sm_object
, tx
));
835 ASSERT0(vdev_obsolete_sm_object(vd
, &obsolete_sm_object
));
836 ASSERT3U(obsolete_sm_object
, !=, 0);
838 spa_feature_incr(spa
, SPA_FEATURE_OBSOLETE_COUNTS
, tx
);
839 VERIFY0(space_map_open(&vd
->vdev_obsolete_sm
,
840 spa
->spa_meta_objset
, obsolete_sm_object
,
841 0, vd
->vdev_asize
, 0));
842 space_map_update(vd
->vdev_obsolete_sm
);
845 ASSERT(vd
->vdev_obsolete_sm
!= NULL
);
846 ASSERT3U(obsolete_sm_object
, ==,
847 space_map_object(vd
->vdev_obsolete_sm
));
849 space_map_write(vd
->vdev_obsolete_sm
,
850 vd
->vdev_obsolete_segments
, SM_ALLOC
, SM_NO_VDEVID
, tx
);
851 space_map_update(vd
->vdev_obsolete_sm
);
852 range_tree_vacate(vd
->vdev_obsolete_segments
, NULL
, NULL
);
856 spa_condense_init(spa_t
*spa
)
858 int error
= zap_lookup(spa
->spa_meta_objset
,
859 DMU_POOL_DIRECTORY_OBJECT
,
860 DMU_POOL_CONDENSING_INDIRECT
, sizeof (uint64_t),
861 sizeof (spa
->spa_condensing_indirect_phys
) / sizeof (uint64_t),
862 &spa
->spa_condensing_indirect_phys
);
864 if (spa_writeable(spa
)) {
865 spa
->spa_condensing_indirect
=
866 spa_condensing_indirect_create(spa
);
869 } else if (error
== ENOENT
) {
877 spa_condense_fini(spa_t
*spa
)
879 if (spa
->spa_condensing_indirect
!= NULL
) {
880 spa_condensing_indirect_destroy(spa
->spa_condensing_indirect
);
881 spa
->spa_condensing_indirect
= NULL
;
886 spa_start_indirect_condensing_thread(spa_t
*spa
)
888 ASSERT3P(spa
->spa_condense_zthr
, ==, NULL
);
889 spa
->spa_condense_zthr
= zthr_create(spa_condense_indirect_thread_check
,
890 spa_condense_indirect_thread
, spa
);
894 * Gets the obsolete spacemap object from the vdev's ZAP. On success sm_obj
895 * will contain either the obsolete spacemap object or zero if none exists.
896 * All other errors are returned to the caller.
899 vdev_obsolete_sm_object(vdev_t
*vd
, uint64_t *sm_obj
)
901 ASSERT0(spa_config_held(vd
->vdev_spa
, SCL_ALL
, RW_WRITER
));
903 if (vd
->vdev_top_zap
== 0) {
908 int error
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
, vd
->vdev_top_zap
,
909 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM
, sizeof (sm_obj
), 1, sm_obj
);
910 if (error
== ENOENT
) {
919 * Gets the obsolete count are precise spacemap object from the vdev's ZAP.
920 * On success are_precise will be set to reflect if the counts are precise.
921 * All other errors are returned to the caller.
924 vdev_obsolete_counts_are_precise(vdev_t
*vd
, boolean_t
*are_precise
)
926 ASSERT0(spa_config_held(vd
->vdev_spa
, SCL_ALL
, RW_WRITER
));
928 if (vd
->vdev_top_zap
== 0) {
929 *are_precise
= B_FALSE
;
934 int error
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
, vd
->vdev_top_zap
,
935 VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE
, sizeof (val
), 1, &val
);
937 *are_precise
= (val
!= 0);
938 } else if (error
== ENOENT
) {
939 *are_precise
= B_FALSE
;
948 vdev_indirect_close(vdev_t
*vd
)
954 vdev_indirect_open(vdev_t
*vd
, uint64_t *psize
, uint64_t *max_psize
,
957 *psize
= *max_psize
= vd
->vdev_asize
+
958 VDEV_LABEL_START_SIZE
+ VDEV_LABEL_END_SIZE
;
959 *ashift
= vd
->vdev_ashift
;
963 typedef struct remap_segment
{
967 uint64_t rs_split_offset
;
972 rs_alloc(vdev_t
*vd
, uint64_t offset
, uint64_t asize
, uint64_t split_offset
)
974 remap_segment_t
*rs
= kmem_alloc(sizeof (remap_segment_t
), KM_SLEEP
);
976 rs
->rs_offset
= offset
;
977 rs
->rs_asize
= asize
;
978 rs
->rs_split_offset
= split_offset
;
983 * Given an indirect vdev and an extent on that vdev, it duplicates the
984 * physical entries of the indirect mapping that correspond to the extent
985 * to a new array and returns a pointer to it. In addition, copied_entries
986 * is populated with the number of mapping entries that were duplicated.
988 * Note that the function assumes that the caller holds vdev_indirect_rwlock.
989 * This ensures that the mapping won't change due to condensing as we
990 * copy over its contents.
992 * Finally, since we are doing an allocation, it is up to the caller to
993 * free the array allocated in this function.
995 vdev_indirect_mapping_entry_phys_t
*
996 vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t
*vd
, uint64_t offset
,
997 uint64_t asize
, uint64_t *copied_entries
)
999 vdev_indirect_mapping_entry_phys_t
*duplicate_mappings
= NULL
;
1000 vdev_indirect_mapping_t
*vim
= vd
->vdev_indirect_mapping
;
1001 uint64_t entries
= 0;
1003 ASSERT(RW_READ_HELD(&vd
->vdev_indirect_rwlock
));
1005 vdev_indirect_mapping_entry_phys_t
*first_mapping
=
1006 vdev_indirect_mapping_entry_for_offset(vim
, offset
);
1007 ASSERT3P(first_mapping
, !=, NULL
);
1009 vdev_indirect_mapping_entry_phys_t
*m
= first_mapping
;
1011 uint64_t size
= DVA_GET_ASIZE(&m
->vimep_dst
);
1013 ASSERT3U(offset
, >=, DVA_MAPPING_GET_SRC_OFFSET(m
));
1014 ASSERT3U(offset
, <, DVA_MAPPING_GET_SRC_OFFSET(m
) + size
);
1016 uint64_t inner_offset
= offset
- DVA_MAPPING_GET_SRC_OFFSET(m
);
1017 uint64_t inner_size
= MIN(asize
, size
- inner_offset
);
1019 offset
+= inner_size
;
1020 asize
-= inner_size
;
1025 size_t copy_length
= entries
* sizeof (*first_mapping
);
1026 duplicate_mappings
= kmem_alloc(copy_length
, KM_SLEEP
);
1027 bcopy(first_mapping
, duplicate_mappings
, copy_length
);
1028 *copied_entries
= entries
;
1030 return (duplicate_mappings
);
1034 * Goes through the relevant indirect mappings until it hits a concrete vdev
1035 * and issues the callback. On the way to the concrete vdev, if any other
1036 * indirect vdevs are encountered, then the callback will also be called on
1037 * each of those indirect vdevs. For example, if the segment is mapped to
1038 * segment A on indirect vdev 1, and then segment A on indirect vdev 1 is
1039 * mapped to segment B on concrete vdev 2, then the callback will be called on
1040 * both vdev 1 and vdev 2.
1042 * While the callback passed to vdev_indirect_remap() is called on every vdev
1043 * the function encounters, certain callbacks only care about concrete vdevs.
1044 * These types of callbacks should return immediately and explicitly when they
1045 * are called on an indirect vdev.
1047 * Because there is a possibility that a DVA section in the indirect device
1048 * has been split into multiple sections in our mapping, we keep track
1049 * of the relevant contiguous segments of the new location (remap_segment_t)
1050 * in a stack. This way we can call the callback for each of the new sections
1051 * created by a single section of the indirect device. Note though, that in
1052 * this scenario the callbacks in each split block won't occur in-order in
1053 * terms of offset, so callers should not make any assumptions about that.
1055 * For callbacks that don't handle split blocks and immediately return when
1056 * they encounter them (as is the case for remap_blkptr_cb), the caller can
1057 * assume that its callback will be applied from the first indirect vdev
1058 * encountered to the last one and then the concrete vdev, in that order.
1061 vdev_indirect_remap(vdev_t
*vd
, uint64_t offset
, uint64_t asize
,
1062 void (*func
)(uint64_t, vdev_t
*, uint64_t, uint64_t, void *), void *arg
)
1065 spa_t
*spa
= vd
->vdev_spa
;
1067 list_create(&stack
, sizeof (remap_segment_t
),
1068 offsetof(remap_segment_t
, rs_node
));
1070 for (remap_segment_t
*rs
= rs_alloc(vd
, offset
, asize
, 0);
1071 rs
!= NULL
; rs
= list_remove_head(&stack
)) {
1072 vdev_t
*v
= rs
->rs_vd
;
1073 uint64_t num_entries
= 0;
1075 ASSERT(spa_config_held(spa
, SCL_ALL
, RW_READER
) != 0);
1076 ASSERT(rs
->rs_asize
> 0);
1079 * Note: As this function can be called from open context
1080 * (e.g. zio_read()), we need the following rwlock to
1081 * prevent the mapping from being changed by condensing.
1083 * So we grab the lock and we make a copy of the entries
1084 * that are relevant to the extent that we are working on.
1085 * Once that is done, we drop the lock and iterate over
1086 * our copy of the mapping. Once we are done with the with
1087 * the remap segment and we free it, we also free our copy
1088 * of the indirect mapping entries that are relevant to it.
1090 * This way we don't need to wait until the function is
1091 * finished with a segment, to condense it. In addition, we
1092 * don't need a recursive rwlock for the case that a call to
1093 * vdev_indirect_remap() needs to call itself (through the
1094 * codepath of its callback) for the same vdev in the middle
1097 rw_enter(&v
->vdev_indirect_rwlock
, RW_READER
);
1098 ASSERT3P(v
->vdev_indirect_mapping
, !=, NULL
);
1100 vdev_indirect_mapping_entry_phys_t
*mapping
=
1101 vdev_indirect_mapping_duplicate_adjacent_entries(v
,
1102 rs
->rs_offset
, rs
->rs_asize
, &num_entries
);
1103 ASSERT3P(mapping
, !=, NULL
);
1104 ASSERT3U(num_entries
, >, 0);
1105 rw_exit(&v
->vdev_indirect_rwlock
);
1107 for (uint64_t i
= 0; i
< num_entries
; i
++) {
1109 * Note: the vdev_indirect_mapping can not change
1110 * while we are running. It only changes while the
1111 * removal is in progress, and then only from syncing
1112 * context. While a removal is in progress, this
1113 * function is only called for frees, which also only
1114 * happen from syncing context.
1116 vdev_indirect_mapping_entry_phys_t
*m
= &mapping
[i
];
1118 ASSERT3P(m
, !=, NULL
);
1119 ASSERT3U(rs
->rs_asize
, >, 0);
1121 uint64_t size
= DVA_GET_ASIZE(&m
->vimep_dst
);
1122 uint64_t dst_offset
= DVA_GET_OFFSET(&m
->vimep_dst
);
1123 uint64_t dst_vdev
= DVA_GET_VDEV(&m
->vimep_dst
);
1125 ASSERT3U(rs
->rs_offset
, >=,
1126 DVA_MAPPING_GET_SRC_OFFSET(m
));
1127 ASSERT3U(rs
->rs_offset
, <,
1128 DVA_MAPPING_GET_SRC_OFFSET(m
) + size
);
1129 ASSERT3U(dst_vdev
, !=, v
->vdev_id
);
1131 uint64_t inner_offset
= rs
->rs_offset
-
1132 DVA_MAPPING_GET_SRC_OFFSET(m
);
1133 uint64_t inner_size
=
1134 MIN(rs
->rs_asize
, size
- inner_offset
);
1136 vdev_t
*dst_v
= vdev_lookup_top(spa
, dst_vdev
);
1137 ASSERT3P(dst_v
, !=, NULL
);
1139 if (dst_v
->vdev_ops
== &vdev_indirect_ops
) {
1140 list_insert_head(&stack
,
1141 rs_alloc(dst_v
, dst_offset
+ inner_offset
,
1142 inner_size
, rs
->rs_split_offset
));
1146 if ((zfs_flags
& ZFS_DEBUG_INDIRECT_REMAP
) &&
1147 IS_P2ALIGNED(inner_size
, 2 * SPA_MINBLOCKSIZE
)) {
1149 * Note: This clause exists only solely for
1150 * testing purposes. We use it to ensure that
1151 * split blocks work and that the callbacks
1152 * using them yield the same result if issued
1155 uint64_t inner_half
= inner_size
/ 2;
1157 func(rs
->rs_split_offset
+ inner_half
, dst_v
,
1158 dst_offset
+ inner_offset
+ inner_half
,
1161 func(rs
->rs_split_offset
, dst_v
,
1162 dst_offset
+ inner_offset
,
1165 func(rs
->rs_split_offset
, dst_v
,
1166 dst_offset
+ inner_offset
,
1170 rs
->rs_offset
+= inner_size
;
1171 rs
->rs_asize
-= inner_size
;
1172 rs
->rs_split_offset
+= inner_size
;
1174 VERIFY0(rs
->rs_asize
);
1176 kmem_free(mapping
, num_entries
* sizeof (*mapping
));
1177 kmem_free(rs
, sizeof (remap_segment_t
));
1179 list_destroy(&stack
);
1183 vdev_indirect_child_io_done(zio_t
*zio
)
1185 zio_t
*pio
= zio
->io_private
;
1187 mutex_enter(&pio
->io_lock
);
1188 pio
->io_error
= zio_worst_error(pio
->io_error
, zio
->io_error
);
1189 mutex_exit(&pio
->io_lock
);
1191 abd_put(zio
->io_abd
);
1195 * This is a callback for vdev_indirect_remap() which allocates an
1196 * indirect_split_t for each split segment and adds it to iv_splits.
1199 vdev_indirect_gather_splits(uint64_t split_offset
, vdev_t
*vd
, uint64_t offset
,
1200 uint64_t size
, void *arg
)
1203 indirect_vsd_t
*iv
= zio
->io_vsd
;
1205 ASSERT3P(vd
, !=, NULL
);
1207 if (vd
->vdev_ops
== &vdev_indirect_ops
)
1211 if (vd
->vdev_ops
== &vdev_mirror_ops
)
1212 n
= vd
->vdev_children
;
1214 indirect_split_t
*is
=
1215 kmem_zalloc(offsetof(indirect_split_t
, is_child
[n
]), KM_SLEEP
);
1217 is
->is_children
= n
;
1219 is
->is_split_offset
= split_offset
;
1220 is
->is_target_offset
= offset
;
1222 list_create(&is
->is_unique_child
, sizeof (indirect_child_t
),
1223 offsetof(indirect_child_t
, ic_node
));
1226 * Note that we only consider multiple copies of the data for
1227 * *mirror* vdevs. We don't for "replacing" or "spare" vdevs, even
1228 * though they use the same ops as mirror, because there's only one
1229 * "good" copy under the replacing/spare.
1231 if (vd
->vdev_ops
== &vdev_mirror_ops
) {
1232 for (int i
= 0; i
< n
; i
++) {
1233 is
->is_child
[i
].ic_vdev
= vd
->vdev_child
[i
];
1234 list_link_init(&is
->is_child
[i
].ic_node
);
1237 is
->is_child
[0].ic_vdev
= vd
;
1240 list_insert_tail(&iv
->iv_splits
, is
);
1244 vdev_indirect_read_split_done(zio_t
*zio
)
1246 indirect_child_t
*ic
= zio
->io_private
;
1248 if (zio
->io_error
!= 0) {
1250 * Clear ic_data to indicate that we do not have data for this
1253 abd_free(ic
->ic_data
);
1259 * Issue reads for all copies (mirror children) of all splits.
1262 vdev_indirect_read_all(zio_t
*zio
)
1264 indirect_vsd_t
*iv
= zio
->io_vsd
;
1266 ASSERT3U(zio
->io_type
, ==, ZIO_TYPE_READ
);
1268 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1269 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1270 for (int i
= 0; i
< is
->is_children
; i
++) {
1271 indirect_child_t
*ic
= &is
->is_child
[i
];
1273 if (!vdev_readable(ic
->ic_vdev
))
1277 * Note, we may read from a child whose DTL
1278 * indicates that the data may not be present here.
1279 * While this might result in a few i/os that will
1280 * likely return incorrect data, it simplifies the
1281 * code since we can treat scrub and resilver
1282 * identically. (The incorrect data will be
1283 * detected and ignored when we verify the
1287 ic
->ic_data
= abd_alloc_sametype(zio
->io_abd
,
1289 ic
->ic_duplicate
= NULL
;
1291 zio_nowait(zio_vdev_child_io(zio
, NULL
,
1292 ic
->ic_vdev
, is
->is_target_offset
, ic
->ic_data
,
1293 is
->is_size
, zio
->io_type
, zio
->io_priority
, 0,
1294 vdev_indirect_read_split_done
, ic
));
1297 iv
->iv_reconstruct
= B_TRUE
;
1301 vdev_indirect_io_start(zio_t
*zio
)
1303 ASSERTV(spa_t
*spa
= zio
->io_spa
);
1304 indirect_vsd_t
*iv
= kmem_zalloc(sizeof (*iv
), KM_SLEEP
);
1305 list_create(&iv
->iv_splits
,
1306 sizeof (indirect_split_t
), offsetof(indirect_split_t
, is_node
));
1309 zio
->io_vsd_ops
= &vdev_indirect_vsd_ops
;
1311 ASSERT(spa_config_held(spa
, SCL_ALL
, RW_READER
) != 0);
1312 if (zio
->io_type
!= ZIO_TYPE_READ
) {
1313 ASSERT3U(zio
->io_type
, ==, ZIO_TYPE_WRITE
);
1315 * Note: this code can handle other kinds of writes,
1316 * but we don't expect them.
1318 ASSERT((zio
->io_flags
& (ZIO_FLAG_SELF_HEAL
|
1319 ZIO_FLAG_RESILVER
| ZIO_FLAG_INDUCE_DAMAGE
)) != 0);
1322 vdev_indirect_remap(zio
->io_vd
, zio
->io_offset
, zio
->io_size
,
1323 vdev_indirect_gather_splits
, zio
);
1325 indirect_split_t
*first
= list_head(&iv
->iv_splits
);
1326 if (first
->is_size
== zio
->io_size
) {
1328 * This is not a split block; we are pointing to the entire
1329 * data, which will checksum the same as the original data.
1330 * Pass the BP down so that the child i/o can verify the
1331 * checksum, and try a different location if available
1332 * (e.g. on a mirror).
1334 * While this special case could be handled the same as the
1335 * general (split block) case, doing it this way ensures
1336 * that the vast majority of blocks on indirect vdevs
1337 * (which are not split) are handled identically to blocks
1338 * on non-indirect vdevs. This allows us to be less strict
1339 * about performance in the general (but rare) case.
1341 ASSERT0(first
->is_split_offset
);
1342 ASSERT3P(list_next(&iv
->iv_splits
, first
), ==, NULL
);
1343 zio_nowait(zio_vdev_child_io(zio
, zio
->io_bp
,
1344 first
->is_vdev
, first
->is_target_offset
,
1345 abd_get_offset(zio
->io_abd
, 0),
1346 zio
->io_size
, zio
->io_type
, zio
->io_priority
, 0,
1347 vdev_indirect_child_io_done
, zio
));
1349 iv
->iv_split_block
= B_TRUE
;
1350 if (zio
->io_type
== ZIO_TYPE_READ
&&
1351 zio
->io_flags
& (ZIO_FLAG_SCRUB
| ZIO_FLAG_RESILVER
)) {
1353 * Read all copies. Note that for simplicity,
1354 * we don't bother consulting the DTL in the
1357 vdev_indirect_read_all(zio
);
1360 * If this is a read zio, we read one copy of each
1361 * split segment, from the top-level vdev. Since
1362 * we don't know the checksum of each split
1363 * individually, the child zio can't ensure that
1364 * we get the right data. E.g. if it's a mirror,
1365 * it will just read from a random (healthy) leaf
1366 * vdev. We have to verify the checksum in
1367 * vdev_indirect_io_done().
1369 * For write zios, the vdev code will ensure we write
1372 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1373 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1374 zio_nowait(zio_vdev_child_io(zio
, NULL
,
1375 is
->is_vdev
, is
->is_target_offset
,
1376 abd_get_offset(zio
->io_abd
,
1377 is
->is_split_offset
), is
->is_size
,
1378 zio
->io_type
, zio
->io_priority
, 0,
1379 vdev_indirect_child_io_done
, zio
));
1389 * Report a checksum error for a child.
1392 vdev_indirect_checksum_error(zio_t
*zio
,
1393 indirect_split_t
*is
, indirect_child_t
*ic
)
1395 vdev_t
*vd
= ic
->ic_vdev
;
1397 if (zio
->io_flags
& ZIO_FLAG_SPECULATIVE
)
1400 mutex_enter(&vd
->vdev_stat_lock
);
1401 vd
->vdev_stat
.vs_checksum_errors
++;
1402 mutex_exit(&vd
->vdev_stat_lock
);
1404 zio_bad_cksum_t zbc
= {{{ 0 }}};
1405 abd_t
*bad_abd
= ic
->ic_data
;
1406 abd_t
*good_abd
= is
->is_good_child
->ic_data
;
1407 zfs_ereport_post_checksum(zio
->io_spa
, vd
, NULL
, zio
,
1408 is
->is_target_offset
, is
->is_size
, good_abd
, bad_abd
, &zbc
);
1412 * Issue repair i/os for any incorrect copies. We do this by comparing
1413 * each split segment's correct data (is_good_child's ic_data) with each
1414 * other copy of the data. If they differ, then we overwrite the bad data
1415 * with the good copy. Note that we do this without regard for the DTL's,
1416 * which simplifies this code and also issues the optimal number of writes
1417 * (based on which copies actually read bad data, as opposed to which we
1418 * think might be wrong). For the same reason, we always use
1419 * ZIO_FLAG_SELF_HEAL, to bypass the DTL check in zio_vdev_io_start().
1422 vdev_indirect_repair(zio_t
*zio
)
1424 indirect_vsd_t
*iv
= zio
->io_vsd
;
1426 enum zio_flag flags
= ZIO_FLAG_IO_REPAIR
;
1428 if (!(zio
->io_flags
& (ZIO_FLAG_SCRUB
| ZIO_FLAG_RESILVER
)))
1429 flags
|= ZIO_FLAG_SELF_HEAL
;
1431 if (!spa_writeable(zio
->io_spa
))
1434 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1435 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1436 for (int c
= 0; c
< is
->is_children
; c
++) {
1437 indirect_child_t
*ic
= &is
->is_child
[c
];
1438 if (ic
== is
->is_good_child
)
1440 if (ic
->ic_data
== NULL
)
1442 if (ic
->ic_duplicate
== is
->is_good_child
)
1445 zio_nowait(zio_vdev_child_io(zio
, NULL
,
1446 ic
->ic_vdev
, is
->is_target_offset
,
1447 is
->is_good_child
->ic_data
, is
->is_size
,
1448 ZIO_TYPE_WRITE
, ZIO_PRIORITY_ASYNC_WRITE
,
1449 ZIO_FLAG_IO_REPAIR
| ZIO_FLAG_SELF_HEAL
,
1452 vdev_indirect_checksum_error(zio
, is
, ic
);
1458 * Report checksum errors on all children that we read from.
1461 vdev_indirect_all_checksum_errors(zio_t
*zio
)
1463 indirect_vsd_t
*iv
= zio
->io_vsd
;
1465 if (zio
->io_flags
& ZIO_FLAG_SPECULATIVE
)
1468 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1469 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1470 for (int c
= 0; c
< is
->is_children
; c
++) {
1471 indirect_child_t
*ic
= &is
->is_child
[c
];
1473 if (ic
->ic_data
== NULL
)
1476 vdev_t
*vd
= ic
->ic_vdev
;
1478 mutex_enter(&vd
->vdev_stat_lock
);
1479 vd
->vdev_stat
.vs_checksum_errors
++;
1480 mutex_exit(&vd
->vdev_stat_lock
);
1482 zfs_ereport_post_checksum(zio
->io_spa
, vd
, NULL
, zio
,
1483 is
->is_target_offset
, is
->is_size
,
1490 * Copy data from all the splits to a main zio then validate the checksum.
1491 * If then checksum is successfully validated return success.
1494 vdev_indirect_splits_checksum_validate(indirect_vsd_t
*iv
, zio_t
*zio
)
1496 zio_bad_cksum_t zbc
;
1498 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1499 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1501 ASSERT3P(is
->is_good_child
->ic_data
, !=, NULL
);
1502 ASSERT3P(is
->is_good_child
->ic_duplicate
, ==, NULL
);
1504 abd_copy_off(zio
->io_abd
, is
->is_good_child
->ic_data
,
1505 is
->is_split_offset
, 0, is
->is_size
);
1508 return (zio_checksum_error(zio
, &zbc
));
1512 * There are relatively few possible combinations making it feasible to
1513 * deterministically check them all. We do this by setting the good_child
1514 * to the next unique split version. If we reach the end of the list then
1515 * "carry over" to the next unique split version (like counting in base
1516 * is_unique_children, but each digit can have a different base).
1519 vdev_indirect_splits_enumerate_all(indirect_vsd_t
*iv
, zio_t
*zio
)
1521 boolean_t more
= B_TRUE
;
1523 iv
->iv_attempts
= 0;
1525 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1526 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
))
1527 is
->is_good_child
= list_head(&is
->is_unique_child
);
1529 while (more
== B_TRUE
) {
1533 if (vdev_indirect_splits_checksum_validate(iv
, zio
) == 0)
1536 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1537 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1538 is
->is_good_child
= list_next(&is
->is_unique_child
,
1540 if (is
->is_good_child
!= NULL
) {
1545 is
->is_good_child
= list_head(&is
->is_unique_child
);
1549 ASSERT3S(iv
->iv_attempts
, <=, iv
->iv_unique_combinations
);
1551 return (SET_ERROR(ECKSUM
));
1555 * There are too many combinations to try all of them in a reasonable amount
1556 * of time. So try a fixed number of random combinations from the unique
1557 * split versions, after which we'll consider the block unrecoverable.
1560 vdev_indirect_splits_enumerate_randomly(indirect_vsd_t
*iv
, zio_t
*zio
)
1562 iv
->iv_attempts
= 0;
1564 while (iv
->iv_attempts
< iv
->iv_attempts_max
) {
1567 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1568 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1569 indirect_child_t
*ic
= list_head(&is
->is_unique_child
);
1570 int children
= is
->is_unique_children
;
1572 for (int i
= spa_get_random(children
); i
> 0; i
--)
1573 ic
= list_next(&is
->is_unique_child
, ic
);
1575 ASSERT3P(ic
, !=, NULL
);
1576 is
->is_good_child
= ic
;
1579 if (vdev_indirect_splits_checksum_validate(iv
, zio
) == 0)
1583 return (SET_ERROR(ECKSUM
));
1587 * This is a validation function for reconstruction. It randomly selects
1588 * a good combination, if one can be found, and then it intentionally
1589 * damages all other segment copes by zeroing them. This forces the
1590 * reconstruction algorithm to locate the one remaining known good copy.
1593 vdev_indirect_splits_damage(indirect_vsd_t
*iv
, zio_t
*zio
)
1597 /* Presume all the copies are unique for initial selection. */
1598 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1599 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1600 is
->is_unique_children
= 0;
1602 for (int i
= 0; i
< is
->is_children
; i
++) {
1603 indirect_child_t
*ic
= &is
->is_child
[i
];
1604 if (ic
->ic_data
!= NULL
) {
1605 is
->is_unique_children
++;
1606 list_insert_tail(&is
->is_unique_child
, ic
);
1610 if (list_is_empty(&is
->is_unique_child
)) {
1611 error
= SET_ERROR(EIO
);
1617 * Set each is_good_child to a randomly-selected child which
1618 * is known to contain validated data.
1620 error
= vdev_indirect_splits_enumerate_randomly(iv
, zio
);
1625 * Damage all but the known good copy by zeroing it. This will
1626 * result in two or less unique copies per indirect_child_t.
1627 * Both may need to be checked in order to reconstruct the block.
1628 * Set iv->iv_attempts_max such that all unique combinations will
1629 * enumerated, but limit the damage to at most 12 indirect splits.
1631 iv
->iv_attempts_max
= 1;
1633 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1634 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1635 for (int c
= 0; c
< is
->is_children
; c
++) {
1636 indirect_child_t
*ic
= &is
->is_child
[c
];
1638 if (ic
== is
->is_good_child
)
1640 if (ic
->ic_data
== NULL
)
1643 abd_zero(ic
->ic_data
, ic
->ic_data
->abd_size
);
1646 iv
->iv_attempts_max
*= 2;
1647 if (iv
->iv_attempts_max
>= (1ULL << 12)) {
1648 iv
->iv_attempts_max
= UINT64_MAX
;
1654 /* Empty the unique children lists so they can be reconstructed. */
1655 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1656 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1657 indirect_child_t
*ic
;
1658 while ((ic
= list_head(&is
->is_unique_child
)) != NULL
)
1659 list_remove(&is
->is_unique_child
, ic
);
1661 is
->is_unique_children
= 0;
1668 * This function is called when we have read all copies of the data and need
1669 * to try to find a combination of copies that gives us the right checksum.
1671 * If we pointed to any mirror vdevs, this effectively does the job of the
1672 * mirror. The mirror vdev code can't do its own job because we don't know
1673 * the checksum of each split segment individually.
1675 * We have to try every unique combination of copies of split segments, until
1676 * we find one that checksums correctly. Duplicate segment copies are first
1677 * identified and latter skipped during reconstruction. This optimization
1678 * reduces the search space and ensures that of the remaining combinations
1679 * at most one is correct.
1681 * When the total number of combinations is small they can all be checked.
1682 * For example, if we have 3 segments in the split, and each points to a
1683 * 2-way mirror with unique copies, we will have the following pieces of data:
1687 * ======|=====================
1688 * A | data_A_0 data_A_1
1689 * B | data_B_0 data_B_1
1690 * C | data_C_0 data_C_1
1692 * We will try the following (mirror children)^(number of splits) (2^3=8)
1693 * combinations, which is similar to bitwise-little-endian counting in
1694 * binary. In general each "digit" corresponds to a split segment, and the
1695 * base of each digit is is_children, which can be different for each
1698 * "low bit" "high bit"
1700 * data_A_0 data_B_0 data_C_0
1701 * data_A_1 data_B_0 data_C_0
1702 * data_A_0 data_B_1 data_C_0
1703 * data_A_1 data_B_1 data_C_0
1704 * data_A_0 data_B_0 data_C_1
1705 * data_A_1 data_B_0 data_C_1
1706 * data_A_0 data_B_1 data_C_1
1707 * data_A_1 data_B_1 data_C_1
1709 * Note that the split segments may be on the same or different top-level
1710 * vdevs. In either case, we may need to try lots of combinations (see
1711 * zfs_reconstruct_indirect_combinations_max). This ensures that if a mirror
1712 * has small silent errors on all of its children, we can still reconstruct
1713 * the correct data, as long as those errors are at sufficiently-separated
1714 * offsets (specifically, separated by the largest block size - default of
1715 * 128KB, but up to 16MB).
1718 vdev_indirect_reconstruct_io_done(zio_t
*zio
)
1720 indirect_vsd_t
*iv
= zio
->io_vsd
;
1721 boolean_t known_good
= B_FALSE
;
1724 iv
->iv_unique_combinations
= 1;
1725 iv
->iv_attempts_max
= UINT64_MAX
;
1727 if (zfs_reconstruct_indirect_combinations_max
> 0)
1728 iv
->iv_attempts_max
= zfs_reconstruct_indirect_combinations_max
;
1731 * If nonzero, every 1/x blocks will be damaged, in order to validate
1732 * reconstruction when there are split segments with damaged copies.
1733 * Known_good will be TRUE when reconstruction is known to be possible.
1735 if (zfs_reconstruct_indirect_damage_fraction
!= 0 &&
1736 spa_get_random(zfs_reconstruct_indirect_damage_fraction
) == 0)
1737 known_good
= (vdev_indirect_splits_damage(iv
, zio
) == 0);
1740 * Determine the unique children for a split segment and add them
1741 * to the is_unique_child list. By restricting reconstruction
1742 * to these children, only unique combinations will be considered.
1743 * This can vastly reduce the search space when there are a large
1744 * number of indirect splits.
1746 for (indirect_split_t
*is
= list_head(&iv
->iv_splits
);
1747 is
!= NULL
; is
= list_next(&iv
->iv_splits
, is
)) {
1748 is
->is_unique_children
= 0;
1750 for (int i
= 0; i
< is
->is_children
; i
++) {
1751 indirect_child_t
*ic_i
= &is
->is_child
[i
];
1753 if (ic_i
->ic_data
== NULL
||
1754 ic_i
->ic_duplicate
!= NULL
)
1757 for (int j
= i
+ 1; j
< is
->is_children
; j
++) {
1758 indirect_child_t
*ic_j
= &is
->is_child
[j
];
1760 if (ic_j
->ic_data
== NULL
||
1761 ic_j
->ic_duplicate
!= NULL
)
1764 if (abd_cmp(ic_i
->ic_data
, ic_j
->ic_data
) == 0)
1765 ic_j
->ic_duplicate
= ic_i
;
1768 is
->is_unique_children
++;
1769 list_insert_tail(&is
->is_unique_child
, ic_i
);
1772 /* Reconstruction is impossible, no valid children */
1773 EQUIV(list_is_empty(&is
->is_unique_child
),
1774 is
->is_unique_children
== 0);
1775 if (list_is_empty(&is
->is_unique_child
)) {
1776 zio
->io_error
= EIO
;
1777 vdev_indirect_all_checksum_errors(zio
);
1778 zio_checksum_verified(zio
);
1782 iv
->iv_unique_combinations
*= is
->is_unique_children
;
1785 if (iv
->iv_unique_combinations
<= iv
->iv_attempts_max
)
1786 error
= vdev_indirect_splits_enumerate_all(iv
, zio
);
1788 error
= vdev_indirect_splits_enumerate_randomly(iv
, zio
);
1791 /* All attempted combinations failed. */
1792 ASSERT3B(known_good
, ==, B_FALSE
);
1793 zio
->io_error
= error
;
1794 vdev_indirect_all_checksum_errors(zio
);
1797 * The checksum has been successfully validated. Issue
1798 * repair I/Os to any copies of splits which don't match
1799 * the validated version.
1801 ASSERT0(vdev_indirect_splits_checksum_validate(iv
, zio
));
1802 vdev_indirect_repair(zio
);
1803 zio_checksum_verified(zio
);
1808 vdev_indirect_io_done(zio_t
*zio
)
1810 indirect_vsd_t
*iv
= zio
->io_vsd
;
1812 if (iv
->iv_reconstruct
) {
1814 * We have read all copies of the data (e.g. from mirrors),
1815 * either because this was a scrub/resilver, or because the
1816 * one-copy read didn't checksum correctly.
1818 vdev_indirect_reconstruct_io_done(zio
);
1822 if (!iv
->iv_split_block
) {
1824 * This was not a split block, so we passed the BP down,
1825 * and the checksum was handled by the (one) child zio.
1830 zio_bad_cksum_t zbc
;
1831 int ret
= zio_checksum_error(zio
, &zbc
);
1833 zio_checksum_verified(zio
);
1838 * The checksum didn't match. Read all copies of all splits, and
1839 * then we will try to reconstruct. The next time
1840 * vdev_indirect_io_done() is called, iv_reconstruct will be set.
1842 vdev_indirect_read_all(zio
);
1844 zio_vdev_io_redone(zio
);
1847 vdev_ops_t vdev_indirect_ops
= {
1849 vdev_indirect_close
,
1851 vdev_indirect_io_start
,
1852 vdev_indirect_io_done
,
1857 vdev_indirect_remap
,
1859 VDEV_TYPE_INDIRECT
, /* name of this vdev type */
1860 B_FALSE
/* leaf vdev */
1863 #if defined(_KERNEL)
1864 EXPORT_SYMBOL(rs_alloc
);
1865 EXPORT_SYMBOL(spa_condense_fini
);
1866 EXPORT_SYMBOL(spa_start_indirect_condensing_thread
);
1867 EXPORT_SYMBOL(spa_condense_indirect_start_sync
);
1868 EXPORT_SYMBOL(spa_condense_init
);
1869 EXPORT_SYMBOL(spa_vdev_indirect_mark_obsolete
);
1870 EXPORT_SYMBOL(vdev_indirect_mark_obsolete
);
1871 EXPORT_SYMBOL(vdev_indirect_should_condense
);
1872 EXPORT_SYMBOL(vdev_indirect_sync_obsolete
);
1873 EXPORT_SYMBOL(vdev_obsolete_counts_are_precise
);
1874 EXPORT_SYMBOL(vdev_obsolete_sm_object
);
1876 module_param(zfs_condense_indirect_vdevs_enable
, int, 0644);
1877 MODULE_PARM_DESC(zfs_condense_indirect_vdevs_enable
,
1878 "Whether to attempt condensing indirect vdev mappings");
1881 module_param(zfs_condense_min_mapping_bytes
, ulong
, 0644);
1882 MODULE_PARM_DESC(zfs_condense_min_mapping_bytes
,
1883 "Minimum size of vdev mapping to condense");
1886 module_param(zfs_condense_max_obsolete_bytes
, ulong
, 0644);
1887 MODULE_PARM_DESC(zfs_condense_max_obsolete_bytes
,
1888 "Minimum size obsolete spacemap to attempt condensing");
1890 module_param(zfs_condense_indirect_commit_entry_delay_ms
, int, 0644);
1891 MODULE_PARM_DESC(zfs_condense_indirect_commit_entry_delay_ms
,
1892 "Delay while condensing vdev mapping");
1894 module_param(zfs_reconstruct_indirect_combinations_max
, int, 0644);
1895 MODULE_PARM_DESC(zfs_reconstruct_indirect_combinations_max
,
1896 "Maximum number of combinations when reconstructing split segments");