4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
27 #include <sys/spa_impl.h>
29 #include <sys/vdev_impl.h>
30 #include <sys/refcount.h>
31 #include <sys/metaslab_impl.h>
32 #include <sys/dsl_synctask.h>
34 #include <sys/dmu_tx.h>
37 * Value that is written to disk during initialization.
40 unsigned long zfs_initialize_value
= 0xdeadbeefUL
;
42 unsigned long zfs_initialize_value
= 0xdeadbeefdeadbeeeULL
;
45 /* maximum number of I/Os outstanding per leaf vdev */
46 int zfs_initialize_limit
= 1;
48 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
49 uint64_t zfs_initialize_chunk_size
= 1024 * 1024;
52 vdev_initialize_should_stop(vdev_t
*vd
)
54 return (vd
->vdev_initialize_exit_wanted
|| !vdev_writeable(vd
) ||
55 vd
->vdev_detached
|| vd
->vdev_top
->vdev_removing
);
59 vdev_initialize_zap_update_sync(void *arg
, dmu_tx_t
*tx
)
62 * We pass in the guid instead of the vdev_t since the vdev may
63 * have been freed prior to the sync task being processed. This
64 * happens when a vdev is detached as we call spa_config_vdev_exit(),
65 * stop the initializing thread, schedule the sync task, and free
66 * the vdev. Later when the scheduled sync task is invoked, it would
67 * find that the vdev has been freed.
69 uint64_t guid
= *(uint64_t *)arg
;
70 uint64_t txg
= dmu_tx_get_txg(tx
);
71 kmem_free(arg
, sizeof (uint64_t));
73 vdev_t
*vd
= spa_lookup_by_guid(tx
->tx_pool
->dp_spa
, guid
, B_FALSE
);
74 if (vd
== NULL
|| vd
->vdev_top
->vdev_removing
|| !vdev_is_concrete(vd
))
77 uint64_t last_offset
= vd
->vdev_initialize_offset
[txg
& TXG_MASK
];
78 vd
->vdev_initialize_offset
[txg
& TXG_MASK
] = 0;
80 VERIFY(vd
->vdev_leaf_zap
!= 0);
82 objset_t
*mos
= vd
->vdev_spa
->spa_meta_objset
;
84 if (last_offset
> 0) {
85 vd
->vdev_initialize_last_offset
= last_offset
;
86 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
87 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET
,
88 sizeof (last_offset
), 1, &last_offset
, tx
));
90 if (vd
->vdev_initialize_action_time
> 0) {
91 uint64_t val
= (uint64_t)vd
->vdev_initialize_action_time
;
92 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
93 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME
, sizeof (val
),
97 uint64_t initialize_state
= vd
->vdev_initialize_state
;
98 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
99 VDEV_LEAF_ZAP_INITIALIZE_STATE
, sizeof (initialize_state
), 1,
100 &initialize_state
, tx
));
104 vdev_initialize_change_state(vdev_t
*vd
, vdev_initializing_state_t new_state
)
106 ASSERT(MUTEX_HELD(&vd
->vdev_initialize_lock
));
107 spa_t
*spa
= vd
->vdev_spa
;
109 if (new_state
== vd
->vdev_initialize_state
)
113 * Copy the vd's guid, this will be freed by the sync task.
115 uint64_t *guid
= kmem_zalloc(sizeof (uint64_t), KM_SLEEP
);
116 *guid
= vd
->vdev_guid
;
119 * If we're suspending, then preserving the original start time.
121 if (vd
->vdev_initialize_state
!= VDEV_INITIALIZE_SUSPENDED
) {
122 vd
->vdev_initialize_action_time
= gethrestime_sec();
124 vd
->vdev_initialize_state
= new_state
;
126 dmu_tx_t
*tx
= dmu_tx_create_dd(spa_get_dsl(spa
)->dp_mos_dir
);
127 VERIFY0(dmu_tx_assign(tx
, TXG_WAIT
));
128 dsl_sync_task_nowait(spa_get_dsl(spa
), vdev_initialize_zap_update_sync
,
129 guid
, 2, ZFS_SPACE_CHECK_NONE
, tx
);
132 case VDEV_INITIALIZE_ACTIVE
:
133 spa_history_log_internal(spa
, "initialize", tx
,
134 "vdev=%s activated", vd
->vdev_path
);
136 case VDEV_INITIALIZE_SUSPENDED
:
137 spa_history_log_internal(spa
, "initialize", tx
,
138 "vdev=%s suspended", vd
->vdev_path
);
140 case VDEV_INITIALIZE_CANCELED
:
141 spa_history_log_internal(spa
, "initialize", tx
,
142 "vdev=%s canceled", vd
->vdev_path
);
144 case VDEV_INITIALIZE_COMPLETE
:
145 spa_history_log_internal(spa
, "initialize", tx
,
146 "vdev=%s complete", vd
->vdev_path
);
149 panic("invalid state %llu", (unsigned long long)new_state
);
156 vdev_initialize_cb(zio_t
*zio
)
158 vdev_t
*vd
= zio
->io_vd
;
159 mutex_enter(&vd
->vdev_initialize_io_lock
);
160 if (zio
->io_error
== ENXIO
&& !vdev_writeable(vd
)) {
162 * The I/O failed because the vdev was unavailable; roll the
163 * last offset back. (This works because spa_sync waits on
164 * spa_txg_zio before it runs sync tasks.)
167 &vd
->vdev_initialize_offset
[zio
->io_txg
& TXG_MASK
];
168 *off
= MIN(*off
, zio
->io_offset
);
171 * Since initializing is best-effort, we ignore I/O errors and
172 * rely on vdev_probe to determine if the errors are more
175 if (zio
->io_error
!= 0)
176 vd
->vdev_stat
.vs_initialize_errors
++;
178 vd
->vdev_initialize_bytes_done
+= zio
->io_orig_size
;
180 ASSERT3U(vd
->vdev_initialize_inflight
, >, 0);
181 vd
->vdev_initialize_inflight
--;
182 cv_broadcast(&vd
->vdev_initialize_io_cv
);
183 mutex_exit(&vd
->vdev_initialize_io_lock
);
185 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
188 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
190 vdev_initialize_write(vdev_t
*vd
, uint64_t start
, uint64_t size
, abd_t
*data
)
192 spa_t
*spa
= vd
->vdev_spa
;
194 /* Limit inflight initializing I/Os */
195 mutex_enter(&vd
->vdev_initialize_io_lock
);
196 while (vd
->vdev_initialize_inflight
>= zfs_initialize_limit
) {
197 cv_wait(&vd
->vdev_initialize_io_cv
,
198 &vd
->vdev_initialize_io_lock
);
200 vd
->vdev_initialize_inflight
++;
201 mutex_exit(&vd
->vdev_initialize_io_lock
);
203 dmu_tx_t
*tx
= dmu_tx_create_dd(spa_get_dsl(spa
)->dp_mos_dir
);
204 VERIFY0(dmu_tx_assign(tx
, TXG_WAIT
));
205 uint64_t txg
= dmu_tx_get_txg(tx
);
207 spa_config_enter(spa
, SCL_STATE_ALL
, vd
, RW_READER
);
208 mutex_enter(&vd
->vdev_initialize_lock
);
210 if (vd
->vdev_initialize_offset
[txg
& TXG_MASK
] == 0) {
211 uint64_t *guid
= kmem_zalloc(sizeof (uint64_t), KM_SLEEP
);
212 *guid
= vd
->vdev_guid
;
214 /* This is the first write of this txg. */
215 dsl_sync_task_nowait(spa_get_dsl(spa
),
216 vdev_initialize_zap_update_sync
, guid
, 2,
217 ZFS_SPACE_CHECK_RESERVED
, tx
);
221 * We know the vdev struct will still be around since all
222 * consumers of vdev_free must stop the initialization first.
224 if (vdev_initialize_should_stop(vd
)) {
225 mutex_enter(&vd
->vdev_initialize_io_lock
);
226 ASSERT3U(vd
->vdev_initialize_inflight
, >, 0);
227 vd
->vdev_initialize_inflight
--;
228 mutex_exit(&vd
->vdev_initialize_io_lock
);
229 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
230 mutex_exit(&vd
->vdev_initialize_lock
);
232 return (SET_ERROR(EINTR
));
234 mutex_exit(&vd
->vdev_initialize_lock
);
236 vd
->vdev_initialize_offset
[txg
& TXG_MASK
] = start
+ size
;
237 zio_nowait(zio_write_phys(spa
->spa_txg_zio
[txg
& TXG_MASK
], vd
, start
,
238 size
, data
, ZIO_CHECKSUM_OFF
, vdev_initialize_cb
, NULL
,
239 ZIO_PRIORITY_INITIALIZING
, ZIO_FLAG_CANFAIL
, B_FALSE
));
240 /* vdev_initialize_cb releases SCL_STATE_ALL */
248 * Callback to fill each ABD chunk with zfs_initialize_value. len must be
249 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
250 * allocation will guarantee these for us.
254 vdev_initialize_block_fill(void *buf
, size_t len
, void *unused
)
256 ASSERT0(len
% sizeof (uint64_t));
258 for (uint64_t i
= 0; i
< len
; i
+= sizeof (uint32_t)) {
259 *(uint32_t *)((char *)(buf
) + i
) = zfs_initialize_value
;
262 for (uint64_t i
= 0; i
< len
; i
+= sizeof (uint64_t)) {
263 *(uint64_t *)((char *)(buf
) + i
) = zfs_initialize_value
;
270 vdev_initialize_block_alloc(void)
272 /* Allocate ABD for filler data */
273 abd_t
*data
= abd_alloc_for_io(zfs_initialize_chunk_size
, B_FALSE
);
275 ASSERT0(zfs_initialize_chunk_size
% sizeof (uint64_t));
276 (void) abd_iterate_func(data
, 0, zfs_initialize_chunk_size
,
277 vdev_initialize_block_fill
, NULL
);
283 vdev_initialize_block_free(abd_t
*data
)
289 vdev_initialize_ranges(vdev_t
*vd
, abd_t
*data
)
291 avl_tree_t
*rt
= &vd
->vdev_initialize_tree
->rt_root
;
293 for (range_seg_t
*rs
= avl_first(rt
); rs
!= NULL
;
294 rs
= AVL_NEXT(rt
, rs
)) {
295 uint64_t size
= rs
->rs_end
- rs
->rs_start
;
297 /* Split range into legally-sized physical chunks */
298 uint64_t writes_required
=
299 ((size
- 1) / zfs_initialize_chunk_size
) + 1;
301 for (uint64_t w
= 0; w
< writes_required
; w
++) {
304 error
= vdev_initialize_write(vd
,
305 VDEV_LABEL_START_SIZE
+ rs
->rs_start
+
306 (w
* zfs_initialize_chunk_size
),
307 MIN(size
- (w
* zfs_initialize_chunk_size
),
308 zfs_initialize_chunk_size
), data
);
317 vdev_initialize_calculate_progress(vdev_t
*vd
)
319 ASSERT(spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_READER
) ||
320 spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_WRITER
));
321 ASSERT(vd
->vdev_leaf_zap
!= 0);
323 vd
->vdev_initialize_bytes_est
= 0;
324 vd
->vdev_initialize_bytes_done
= 0;
326 for (uint64_t i
= 0; i
< vd
->vdev_top
->vdev_ms_count
; i
++) {
327 metaslab_t
*msp
= vd
->vdev_top
->vdev_ms
[i
];
328 mutex_enter(&msp
->ms_lock
);
330 uint64_t ms_free
= msp
->ms_size
-
331 metaslab_allocated_space(msp
);
333 if (vd
->vdev_top
->vdev_ops
== &vdev_raidz_ops
)
334 ms_free
/= vd
->vdev_top
->vdev_children
;
337 * Convert the metaslab range to a physical range
338 * on our vdev. We use this to determine if we are
339 * in the middle of this metaslab range.
341 range_seg_t logical_rs
, physical_rs
;
342 logical_rs
.rs_start
= msp
->ms_start
;
343 logical_rs
.rs_end
= msp
->ms_start
+ msp
->ms_size
;
344 vdev_xlate(vd
, &logical_rs
, &physical_rs
);
346 if (vd
->vdev_initialize_last_offset
<= physical_rs
.rs_start
) {
347 vd
->vdev_initialize_bytes_est
+= ms_free
;
348 mutex_exit(&msp
->ms_lock
);
350 } else if (vd
->vdev_initialize_last_offset
>
351 physical_rs
.rs_end
) {
352 vd
->vdev_initialize_bytes_done
+= ms_free
;
353 vd
->vdev_initialize_bytes_est
+= ms_free
;
354 mutex_exit(&msp
->ms_lock
);
359 * If we get here, we're in the middle of initializing this
360 * metaslab. Load it and walk the free tree for more accurate
361 * progress estimation.
363 VERIFY0(metaslab_load(msp
));
365 for (range_seg_t
*rs
= avl_first(&msp
->ms_allocatable
->rt_root
);
366 rs
; rs
= AVL_NEXT(&msp
->ms_allocatable
->rt_root
, rs
)) {
367 logical_rs
.rs_start
= rs
->rs_start
;
368 logical_rs
.rs_end
= rs
->rs_end
;
369 vdev_xlate(vd
, &logical_rs
, &physical_rs
);
371 uint64_t size
= physical_rs
.rs_end
-
372 physical_rs
.rs_start
;
373 vd
->vdev_initialize_bytes_est
+= size
;
374 if (vd
->vdev_initialize_last_offset
>
375 physical_rs
.rs_end
) {
376 vd
->vdev_initialize_bytes_done
+= size
;
377 } else if (vd
->vdev_initialize_last_offset
>
378 physical_rs
.rs_start
&&
379 vd
->vdev_initialize_last_offset
<
380 physical_rs
.rs_end
) {
381 vd
->vdev_initialize_bytes_done
+=
382 vd
->vdev_initialize_last_offset
-
383 physical_rs
.rs_start
;
386 mutex_exit(&msp
->ms_lock
);
391 vdev_initialize_load(vdev_t
*vd
)
394 ASSERT(spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_READER
) ||
395 spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_WRITER
));
396 ASSERT(vd
->vdev_leaf_zap
!= 0);
398 if (vd
->vdev_initialize_state
== VDEV_INITIALIZE_ACTIVE
||
399 vd
->vdev_initialize_state
== VDEV_INITIALIZE_SUSPENDED
) {
400 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
401 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET
,
402 sizeof (vd
->vdev_initialize_last_offset
), 1,
403 &vd
->vdev_initialize_last_offset
);
405 vd
->vdev_initialize_last_offset
= 0;
410 vdev_initialize_calculate_progress(vd
);
415 * Convert the logical range into a physical range and add it to our
419 vdev_initialize_range_add(void *arg
, uint64_t start
, uint64_t size
)
422 range_seg_t logical_rs
, physical_rs
;
423 logical_rs
.rs_start
= start
;
424 logical_rs
.rs_end
= start
+ size
;
426 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
427 vdev_xlate(vd
, &logical_rs
, &physical_rs
);
429 IMPLY(vd
->vdev_top
== vd
,
430 logical_rs
.rs_start
== physical_rs
.rs_start
);
431 IMPLY(vd
->vdev_top
== vd
,
432 logical_rs
.rs_end
== physical_rs
.rs_end
);
434 /* Only add segments that we have not visited yet */
435 if (physical_rs
.rs_end
<= vd
->vdev_initialize_last_offset
)
438 /* Pick up where we left off mid-range. */
439 if (vd
->vdev_initialize_last_offset
> physical_rs
.rs_start
) {
440 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
441 "(%llu, %llu)", vd
->vdev_path
,
442 (u_longlong_t
)physical_rs
.rs_start
,
443 (u_longlong_t
)physical_rs
.rs_end
,
444 (u_longlong_t
)vd
->vdev_initialize_last_offset
,
445 (u_longlong_t
)physical_rs
.rs_end
);
446 ASSERT3U(physical_rs
.rs_end
, >,
447 vd
->vdev_initialize_last_offset
);
448 physical_rs
.rs_start
= vd
->vdev_initialize_last_offset
;
450 ASSERT3U(physical_rs
.rs_end
, >=, physical_rs
.rs_start
);
453 * With raidz, it's possible that the logical range does not live on
454 * this leaf vdev. We only add the physical range to this vdev's if it
455 * has a length greater than 0.
457 if (physical_rs
.rs_end
> physical_rs
.rs_start
) {
458 range_tree_add(vd
->vdev_initialize_tree
, physical_rs
.rs_start
,
459 physical_rs
.rs_end
- physical_rs
.rs_start
);
461 ASSERT3U(physical_rs
.rs_end
, ==, physical_rs
.rs_start
);
466 vdev_initialize_thread(void *arg
)
469 spa_t
*spa
= vd
->vdev_spa
;
471 uint64_t ms_count
= 0;
473 ASSERT(vdev_is_concrete(vd
));
474 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
476 vd
->vdev_initialize_last_offset
= 0;
477 VERIFY0(vdev_initialize_load(vd
));
479 abd_t
*deadbeef
= vdev_initialize_block_alloc();
481 vd
->vdev_initialize_tree
= range_tree_create(NULL
, NULL
);
483 for (uint64_t i
= 0; !vd
->vdev_detached
&&
484 i
< vd
->vdev_top
->vdev_ms_count
; i
++) {
485 metaslab_t
*msp
= vd
->vdev_top
->vdev_ms
[i
];
486 boolean_t unload_when_done
= B_FALSE
;
489 * If we've expanded the top-level vdev or it's our
490 * first pass, calculate our progress.
492 if (vd
->vdev_top
->vdev_ms_count
!= ms_count
) {
493 vdev_initialize_calculate_progress(vd
);
494 ms_count
= vd
->vdev_top
->vdev_ms_count
;
497 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
498 metaslab_disable(msp
);
499 mutex_enter(&msp
->ms_lock
);
500 if (!msp
->ms_loaded
&& !msp
->ms_loading
)
501 unload_when_done
= B_TRUE
;
502 VERIFY0(metaslab_load(msp
));
504 range_tree_walk(msp
->ms_allocatable
, vdev_initialize_range_add
,
506 mutex_exit(&msp
->ms_lock
);
508 error
= vdev_initialize_ranges(vd
, deadbeef
);
509 metaslab_enable(msp
, B_TRUE
, unload_when_done
);
510 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
512 range_tree_vacate(vd
->vdev_initialize_tree
, NULL
, NULL
);
517 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
518 mutex_enter(&vd
->vdev_initialize_io_lock
);
519 while (vd
->vdev_initialize_inflight
> 0) {
520 cv_wait(&vd
->vdev_initialize_io_cv
,
521 &vd
->vdev_initialize_io_lock
);
523 mutex_exit(&vd
->vdev_initialize_io_lock
);
525 range_tree_destroy(vd
->vdev_initialize_tree
);
526 vdev_initialize_block_free(deadbeef
);
527 vd
->vdev_initialize_tree
= NULL
;
529 mutex_enter(&vd
->vdev_initialize_lock
);
530 if (!vd
->vdev_initialize_exit_wanted
&& vdev_writeable(vd
)) {
531 vdev_initialize_change_state(vd
, VDEV_INITIALIZE_COMPLETE
);
533 ASSERT(vd
->vdev_initialize_thread
!= NULL
||
534 vd
->vdev_initialize_inflight
== 0);
537 * Drop the vdev_initialize_lock while we sync out the
538 * txg since it's possible that a device might be trying to
539 * come online and must check to see if it needs to restart an
540 * initialization. That thread will be holding the spa_config_lock
541 * which would prevent the txg_wait_synced from completing.
543 mutex_exit(&vd
->vdev_initialize_lock
);
544 txg_wait_synced(spa_get_dsl(spa
), 0);
545 mutex_enter(&vd
->vdev_initialize_lock
);
547 vd
->vdev_initialize_thread
= NULL
;
548 cv_broadcast(&vd
->vdev_initialize_cv
);
549 mutex_exit(&vd
->vdev_initialize_lock
);
553 * Initiates a device. Caller must hold vdev_initialize_lock.
554 * Device must be a leaf and not already be initializing.
557 vdev_initialize(vdev_t
*vd
)
559 ASSERT(MUTEX_HELD(&vd
->vdev_initialize_lock
));
560 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
561 ASSERT(vdev_is_concrete(vd
));
562 ASSERT3P(vd
->vdev_initialize_thread
, ==, NULL
);
563 ASSERT(!vd
->vdev_detached
);
564 ASSERT(!vd
->vdev_initialize_exit_wanted
);
565 ASSERT(!vd
->vdev_top
->vdev_removing
);
567 vdev_initialize_change_state(vd
, VDEV_INITIALIZE_ACTIVE
);
568 vd
->vdev_initialize_thread
= thread_create(NULL
, 0,
569 vdev_initialize_thread
, vd
, 0, &p0
, TS_RUN
, maxclsyspri
);
573 * Wait for the initialize thread to be terminated (cancelled or stopped).
576 vdev_initialize_stop_wait_impl(vdev_t
*vd
)
578 ASSERT(MUTEX_HELD(&vd
->vdev_initialize_lock
));
580 while (vd
->vdev_initialize_thread
!= NULL
)
581 cv_wait(&vd
->vdev_initialize_cv
, &vd
->vdev_initialize_lock
);
583 ASSERT3P(vd
->vdev_initialize_thread
, ==, NULL
);
584 vd
->vdev_initialize_exit_wanted
= B_FALSE
;
588 * Wait for vdev initialize threads which were either to cleanly exit.
591 vdev_initialize_stop_wait(spa_t
*spa
, list_t
*vd_list
)
595 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
597 while ((vd
= list_remove_head(vd_list
)) != NULL
) {
598 mutex_enter(&vd
->vdev_initialize_lock
);
599 vdev_initialize_stop_wait_impl(vd
);
600 mutex_exit(&vd
->vdev_initialize_lock
);
605 * Stop initializing a device, with the resultant initializing state being
606 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when
607 * a list_t is provided the stopping vdev is inserted in to the list. Callers
608 * are then required to call vdev_initialize_stop_wait() to block for all the
609 * initialization threads to exit. The caller must hold vdev_initialize_lock
610 * and must not be writing to the spa config, as the initializing thread may
611 * try to enter the config as a reader before exiting.
614 vdev_initialize_stop(vdev_t
*vd
, vdev_initializing_state_t tgt_state
,
617 ASSERT(!spa_config_held(vd
->vdev_spa
, SCL_CONFIG
|SCL_STATE
, RW_WRITER
));
618 ASSERT(MUTEX_HELD(&vd
->vdev_initialize_lock
));
619 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
620 ASSERT(vdev_is_concrete(vd
));
623 * Allow cancel requests to proceed even if the initialize thread
626 if (vd
->vdev_initialize_thread
== NULL
&&
627 tgt_state
!= VDEV_INITIALIZE_CANCELED
) {
631 vdev_initialize_change_state(vd
, tgt_state
);
632 vd
->vdev_initialize_exit_wanted
= B_TRUE
;
634 if (vd_list
== NULL
) {
635 vdev_initialize_stop_wait_impl(vd
);
637 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
638 list_insert_tail(vd_list
, vd
);
643 vdev_initialize_stop_all_impl(vdev_t
*vd
, vdev_initializing_state_t tgt_state
,
646 if (vd
->vdev_ops
->vdev_op_leaf
&& vdev_is_concrete(vd
)) {
647 mutex_enter(&vd
->vdev_initialize_lock
);
648 vdev_initialize_stop(vd
, tgt_state
, vd_list
);
649 mutex_exit(&vd
->vdev_initialize_lock
);
653 for (uint64_t i
= 0; i
< vd
->vdev_children
; i
++) {
654 vdev_initialize_stop_all_impl(vd
->vdev_child
[i
], tgt_state
,
660 * Convenience function to stop initializing of a vdev tree and set all
661 * initialize thread pointers to NULL.
664 vdev_initialize_stop_all(vdev_t
*vd
, vdev_initializing_state_t tgt_state
)
666 spa_t
*spa
= vd
->vdev_spa
;
669 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
671 list_create(&vd_list
, sizeof (vdev_t
),
672 offsetof(vdev_t
, vdev_initialize_node
));
674 vdev_initialize_stop_all_impl(vd
, tgt_state
, &vd_list
);
675 vdev_initialize_stop_wait(spa
, &vd_list
);
677 if (vd
->vdev_spa
->spa_sync_on
) {
678 /* Make sure that our state has been synced to disk */
679 txg_wait_synced(spa_get_dsl(vd
->vdev_spa
), 0);
682 list_destroy(&vd_list
);
686 vdev_initialize_restart(vdev_t
*vd
)
688 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
689 ASSERT(!spa_config_held(vd
->vdev_spa
, SCL_ALL
, RW_WRITER
));
691 if (vd
->vdev_leaf_zap
!= 0) {
692 mutex_enter(&vd
->vdev_initialize_lock
);
693 uint64_t initialize_state
= VDEV_INITIALIZE_NONE
;
694 int err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
695 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_INITIALIZE_STATE
,
696 sizeof (initialize_state
), 1, &initialize_state
);
697 ASSERT(err
== 0 || err
== ENOENT
);
698 vd
->vdev_initialize_state
= initialize_state
;
700 uint64_t timestamp
= 0;
701 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
702 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME
,
703 sizeof (timestamp
), 1, ×tamp
);
704 ASSERT(err
== 0 || err
== ENOENT
);
705 vd
->vdev_initialize_action_time
= (time_t)timestamp
;
707 if (vd
->vdev_initialize_state
== VDEV_INITIALIZE_SUSPENDED
||
709 /* load progress for reporting, but don't resume */
710 VERIFY0(vdev_initialize_load(vd
));
711 } else if (vd
->vdev_initialize_state
==
712 VDEV_INITIALIZE_ACTIVE
&& vdev_writeable(vd
) &&
713 !vd
->vdev_top
->vdev_removing
&&
714 vd
->vdev_initialize_thread
== NULL
) {
718 mutex_exit(&vd
->vdev_initialize_lock
);
721 for (uint64_t i
= 0; i
< vd
->vdev_children
; i
++) {
722 vdev_initialize_restart(vd
->vdev_child
[i
]);
726 EXPORT_SYMBOL(vdev_initialize
);
727 EXPORT_SYMBOL(vdev_initialize_stop
);
728 EXPORT_SYMBOL(vdev_initialize_stop_all
);
729 EXPORT_SYMBOL(vdev_initialize_stop_wait
);
730 EXPORT_SYMBOL(vdev_initialize_restart
);
733 ZFS_MODULE_PARAM(zfs
, zfs_
, initialize_value
, ULONG
, ZMOD_RW
,
734 "Value written during zpool initialize");