4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2016 by Delphix. All rights reserved.
24 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
25 * Copyright (c) 2021 Hewlett Packard Enterprise Development LP
26 * Copyright 2023 RackTop Systems, Inc.
30 #include <sys/spa_impl.h>
32 #include <sys/vdev_impl.h>
33 #include <sys/vdev_trim.h>
34 #include <sys/metaslab_impl.h>
35 #include <sys/dsl_synctask.h>
37 #include <sys/dmu_tx.h>
38 #include <sys/arc_impl.h>
41 * TRIM is a feature which is used to notify a SSD that some previously
42 * written space is no longer allocated by the pool. This is useful because
43 * writes to a SSD must be performed to blocks which have first been erased.
44 * Ensuring the SSD always has a supply of erased blocks for new writes
45 * helps prevent the performance from deteriorating.
47 * There are two supported TRIM methods; manual and automatic.
51 * A manual TRIM is initiated by running the 'zpool trim' command. A single
52 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
53 * managing that vdev TRIM process. This involves iterating over all the
54 * metaslabs, calculating the unallocated space ranges, and then issuing the
57 * While a metaslab is being actively trimmed it is not eligible to perform
58 * new allocations. After traversing all of the metaslabs the thread is
59 * terminated. Finally, both the requested options and current progress of
60 * the TRIM are regularly written to the pool. This allows the TRIM to be
61 * suspended and resumed as needed.
65 * An automatic TRIM is enabled by setting the 'autotrim' pool property
66 * to 'on'. When enabled, a `vdev_autotrim' thread is created for each
67 * top-level (not leaf) vdev in the pool. These threads perform the same
68 * core TRIM process as a manual TRIM, but with a few key differences.
70 * 1) Automatic TRIM happens continuously in the background and operates
71 * solely on recently freed blocks (ms_trim not ms_allocatable).
73 * 2) Each thread is associated with a top-level (not leaf) vdev. This has
74 * the benefit of simplifying the threading model, it makes it easier
75 * to coordinate administrative commands, and it ensures only a single
76 * metaslab is disabled at a time. Unlike manual TRIM, this means each
77 * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
80 * 3) There is no automatic TRIM progress information stored on disk, nor
81 * is it reported by 'zpool status'.
83 * While the automatic TRIM process is highly effective it is more likely
84 * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to
85 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
86 * TRIM and are skipped. This means small amounts of freed space may not
87 * be automatically trimmed.
89 * Furthermore, devices with attached hot spares and devices being actively
90 * replaced are skipped. This is done to avoid adding additional stress to
91 * a potentially unhealthy device and to minimize the required rebuild time.
93 * For this reason it may be beneficial to occasionally manually TRIM a pool
94 * even when automatic TRIM is enabled.
98 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
100 static unsigned int zfs_trim_extent_bytes_max
= 128 * 1024 * 1024;
103 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
105 static unsigned int zfs_trim_extent_bytes_min
= 32 * 1024;
108 * Skip uninitialized metaslabs during the TRIM process. This option is
109 * useful for pools constructed from large thinly-provisioned devices where
110 * TRIM operations are slow. As a pool ages an increasing fraction of
111 * the pools metaslabs will be initialized progressively degrading the
112 * usefulness of this option. This setting is stored when starting a
113 * manual TRIM and will persist for the duration of the requested TRIM.
115 unsigned int zfs_trim_metaslab_skip
= 0;
118 * Maximum number of queued TRIM I/Os per leaf vdev. The number of
119 * concurrent TRIM I/Os issued to the device is controlled by the
120 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
122 static unsigned int zfs_trim_queue_limit
= 10;
125 * The minimum number of transaction groups between automatic trims of a
126 * metaslab. This setting represents a trade-off between issuing more
127 * efficient TRIM operations, by allowing them to be aggregated longer,
128 * and issuing them promptly so the trimmed space is available. Note
129 * that this value is a minimum; metaslabs can be trimmed less frequently
130 * when there are a large number of ranges which need to be trimmed.
132 * Increasing this value will allow frees to be aggregated for a longer
133 * time. This can result is larger TRIM operations, and increased memory
134 * usage in order to track the ranges to be trimmed. Decreasing this value
135 * has the opposite effect. The default value of 32 was determined though
136 * testing to be a reasonable compromise.
138 static unsigned int zfs_trim_txg_batch
= 32;
141 * The trim_args are a control structure which describe how a leaf vdev
142 * should be trimmed. The core elements are the vdev, the metaslab being
143 * trimmed and a range tree containing the extents to TRIM. All provided
144 * ranges must be within the metaslab.
146 typedef struct trim_args
{
148 * These fields are set by the caller of vdev_trim_ranges().
150 vdev_t
*trim_vdev
; /* Leaf vdev to TRIM */
151 metaslab_t
*trim_msp
; /* Disabled metaslab */
152 range_tree_t
*trim_tree
; /* TRIM ranges (in metaslab) */
153 trim_type_t trim_type
; /* Manual or auto TRIM */
154 uint64_t trim_extent_bytes_max
; /* Maximum TRIM I/O size */
155 uint64_t trim_extent_bytes_min
; /* Minimum TRIM I/O size */
156 enum trim_flag trim_flags
; /* TRIM flags (secure) */
159 * These fields are updated by vdev_trim_ranges().
161 hrtime_t trim_start_time
; /* Start time */
162 uint64_t trim_bytes_done
; /* Bytes trimmed */
166 * Determines whether a vdev_trim_thread() should be stopped.
169 vdev_trim_should_stop(vdev_t
*vd
)
171 return (vd
->vdev_trim_exit_wanted
|| !vdev_writeable(vd
) ||
172 vd
->vdev_detached
|| vd
->vdev_top
->vdev_removing
||
173 vd
->vdev_top
->vdev_rz_expanding
);
177 * Determines whether a vdev_autotrim_thread() should be stopped.
180 vdev_autotrim_should_stop(vdev_t
*tvd
)
182 return (tvd
->vdev_autotrim_exit_wanted
||
183 !vdev_writeable(tvd
) || tvd
->vdev_removing
||
184 tvd
->vdev_rz_expanding
||
185 spa_get_autotrim(tvd
->vdev_spa
) == SPA_AUTOTRIM_OFF
);
189 * Wait for given number of kicks, return true if the wait is aborted due to
190 * vdev_autotrim_exit_wanted.
193 vdev_autotrim_wait_kick(vdev_t
*vd
, int num_of_kick
)
195 mutex_enter(&vd
->vdev_autotrim_lock
);
196 for (int i
= 0; i
< num_of_kick
; i
++) {
197 if (vd
->vdev_autotrim_exit_wanted
)
199 cv_wait(&vd
->vdev_autotrim_kick_cv
, &vd
->vdev_autotrim_lock
);
201 boolean_t exit_wanted
= vd
->vdev_autotrim_exit_wanted
;
202 mutex_exit(&vd
->vdev_autotrim_lock
);
204 return (exit_wanted
);
208 * The sync task for updating the on-disk state of a manual TRIM. This
209 * is scheduled by vdev_trim_change_state().
212 vdev_trim_zap_update_sync(void *arg
, dmu_tx_t
*tx
)
215 * We pass in the guid instead of the vdev_t since the vdev may
216 * have been freed prior to the sync task being processed. This
217 * happens when a vdev is detached as we call spa_config_vdev_exit(),
218 * stop the trimming thread, schedule the sync task, and free
219 * the vdev. Later when the scheduled sync task is invoked, it would
220 * find that the vdev has been freed.
222 uint64_t guid
= *(uint64_t *)arg
;
223 uint64_t txg
= dmu_tx_get_txg(tx
);
224 kmem_free(arg
, sizeof (uint64_t));
226 vdev_t
*vd
= spa_lookup_by_guid(tx
->tx_pool
->dp_spa
, guid
, B_FALSE
);
227 if (vd
== NULL
|| vd
->vdev_top
->vdev_removing
||
228 !vdev_is_concrete(vd
) || vd
->vdev_top
->vdev_rz_expanding
)
231 uint64_t last_offset
= vd
->vdev_trim_offset
[txg
& TXG_MASK
];
232 vd
->vdev_trim_offset
[txg
& TXG_MASK
] = 0;
234 VERIFY3U(vd
->vdev_leaf_zap
, !=, 0);
236 objset_t
*mos
= vd
->vdev_spa
->spa_meta_objset
;
238 if (last_offset
> 0 || vd
->vdev_trim_last_offset
== UINT64_MAX
) {
240 if (vd
->vdev_trim_last_offset
== UINT64_MAX
)
243 vd
->vdev_trim_last_offset
= last_offset
;
244 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
245 VDEV_LEAF_ZAP_TRIM_LAST_OFFSET
,
246 sizeof (last_offset
), 1, &last_offset
, tx
));
249 if (vd
->vdev_trim_action_time
> 0) {
250 uint64_t val
= (uint64_t)vd
->vdev_trim_action_time
;
251 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
252 VDEV_LEAF_ZAP_TRIM_ACTION_TIME
, sizeof (val
),
256 if (vd
->vdev_trim_rate
> 0) {
257 uint64_t rate
= (uint64_t)vd
->vdev_trim_rate
;
259 if (rate
== UINT64_MAX
)
262 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
,
263 VDEV_LEAF_ZAP_TRIM_RATE
, sizeof (rate
), 1, &rate
, tx
));
266 uint64_t partial
= vd
->vdev_trim_partial
;
267 if (partial
== UINT64_MAX
)
270 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_PARTIAL
,
271 sizeof (partial
), 1, &partial
, tx
));
273 uint64_t secure
= vd
->vdev_trim_secure
;
274 if (secure
== UINT64_MAX
)
277 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_SECURE
,
278 sizeof (secure
), 1, &secure
, tx
));
281 uint64_t trim_state
= vd
->vdev_trim_state
;
282 VERIFY0(zap_update(mos
, vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_STATE
,
283 sizeof (trim_state
), 1, &trim_state
, tx
));
287 * Update the on-disk state of a manual TRIM. This is called to request
288 * that a TRIM be started/suspended/canceled, or to change one of the
289 * TRIM options (partial, secure, rate).
292 vdev_trim_change_state(vdev_t
*vd
, vdev_trim_state_t new_state
,
293 uint64_t rate
, boolean_t partial
, boolean_t secure
)
295 ASSERT(MUTEX_HELD(&vd
->vdev_trim_lock
));
296 spa_t
*spa
= vd
->vdev_spa
;
298 if (new_state
== vd
->vdev_trim_state
)
302 * Copy the vd's guid, this will be freed by the sync task.
304 uint64_t *guid
= kmem_zalloc(sizeof (uint64_t), KM_SLEEP
);
305 *guid
= vd
->vdev_guid
;
308 * If we're suspending, then preserve the original start time.
310 if (vd
->vdev_trim_state
!= VDEV_TRIM_SUSPENDED
) {
311 vd
->vdev_trim_action_time
= gethrestime_sec();
315 * If we're activating, then preserve the requested rate and trim
316 * method. Setting the last offset and rate to UINT64_MAX is used
317 * as a sentinel to indicate they should be reset to default values.
319 if (new_state
== VDEV_TRIM_ACTIVE
) {
320 if (vd
->vdev_trim_state
== VDEV_TRIM_COMPLETE
||
321 vd
->vdev_trim_state
== VDEV_TRIM_CANCELED
) {
322 vd
->vdev_trim_last_offset
= UINT64_MAX
;
323 vd
->vdev_trim_rate
= UINT64_MAX
;
324 vd
->vdev_trim_partial
= UINT64_MAX
;
325 vd
->vdev_trim_secure
= UINT64_MAX
;
329 vd
->vdev_trim_rate
= rate
;
332 vd
->vdev_trim_partial
= partial
;
335 vd
->vdev_trim_secure
= secure
;
338 vdev_trim_state_t old_state
= vd
->vdev_trim_state
;
339 boolean_t resumed
= (old_state
== VDEV_TRIM_SUSPENDED
);
340 vd
->vdev_trim_state
= new_state
;
342 dmu_tx_t
*tx
= dmu_tx_create_dd(spa_get_dsl(spa
)->dp_mos_dir
);
343 VERIFY0(dmu_tx_assign(tx
, TXG_WAIT
));
344 dsl_sync_task_nowait(spa_get_dsl(spa
), vdev_trim_zap_update_sync
,
348 case VDEV_TRIM_ACTIVE
:
349 spa_event_notify(spa
, vd
, NULL
,
350 resumed
? ESC_ZFS_TRIM_RESUME
: ESC_ZFS_TRIM_START
);
351 spa_history_log_internal(spa
, "trim", tx
,
352 "vdev=%s activated", vd
->vdev_path
);
354 case VDEV_TRIM_SUSPENDED
:
355 spa_event_notify(spa
, vd
, NULL
, ESC_ZFS_TRIM_SUSPEND
);
356 spa_history_log_internal(spa
, "trim", tx
,
357 "vdev=%s suspended", vd
->vdev_path
);
359 case VDEV_TRIM_CANCELED
:
360 if (old_state
== VDEV_TRIM_ACTIVE
||
361 old_state
== VDEV_TRIM_SUSPENDED
) {
362 spa_event_notify(spa
, vd
, NULL
, ESC_ZFS_TRIM_CANCEL
);
363 spa_history_log_internal(spa
, "trim", tx
,
364 "vdev=%s canceled", vd
->vdev_path
);
367 case VDEV_TRIM_COMPLETE
:
368 spa_event_notify(spa
, vd
, NULL
, ESC_ZFS_TRIM_FINISH
);
369 spa_history_log_internal(spa
, "trim", tx
,
370 "vdev=%s complete", vd
->vdev_path
);
373 panic("invalid state %llu", (unsigned long long)new_state
);
378 if (new_state
!= VDEV_TRIM_ACTIVE
)
379 spa_notify_waiters(spa
);
383 * The zio_done_func_t done callback for each manual TRIM issued. It is
384 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
385 * and limiting the number of in flight TRIM I/Os.
388 vdev_trim_cb(zio_t
*zio
)
390 vdev_t
*vd
= zio
->io_vd
;
392 mutex_enter(&vd
->vdev_trim_io_lock
);
393 if (zio
->io_error
== ENXIO
&& !vdev_writeable(vd
)) {
395 * The I/O failed because the vdev was unavailable; roll the
396 * last offset back. (This works because spa_sync waits on
397 * spa_txg_zio before it runs sync tasks.)
400 &vd
->vdev_trim_offset
[zio
->io_txg
& TXG_MASK
];
401 *offset
= MIN(*offset
, zio
->io_offset
);
403 if (zio
->io_error
!= 0) {
404 vd
->vdev_stat
.vs_trim_errors
++;
405 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_MANUAL
,
406 0, 0, 0, 0, 1, zio
->io_orig_size
);
408 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_MANUAL
,
409 1, zio
->io_orig_size
, 0, 0, 0, 0);
412 vd
->vdev_trim_bytes_done
+= zio
->io_orig_size
;
415 ASSERT3U(vd
->vdev_trim_inflight
[TRIM_TYPE_MANUAL
], >, 0);
416 vd
->vdev_trim_inflight
[TRIM_TYPE_MANUAL
]--;
417 cv_broadcast(&vd
->vdev_trim_io_cv
);
418 mutex_exit(&vd
->vdev_trim_io_lock
);
420 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
424 * The zio_done_func_t done callback for each automatic TRIM issued. It
425 * is responsible for updating the TRIM stats and limiting the number of
426 * in flight TRIM I/Os. Automatic TRIM I/Os are best effort and are
427 * never reissued on failure.
430 vdev_autotrim_cb(zio_t
*zio
)
432 vdev_t
*vd
= zio
->io_vd
;
434 mutex_enter(&vd
->vdev_trim_io_lock
);
436 if (zio
->io_error
!= 0) {
437 vd
->vdev_stat
.vs_trim_errors
++;
438 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_AUTO
,
439 0, 0, 0, 0, 1, zio
->io_orig_size
);
441 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_AUTO
,
442 1, zio
->io_orig_size
, 0, 0, 0, 0);
445 ASSERT3U(vd
->vdev_trim_inflight
[TRIM_TYPE_AUTO
], >, 0);
446 vd
->vdev_trim_inflight
[TRIM_TYPE_AUTO
]--;
447 cv_broadcast(&vd
->vdev_trim_io_cv
);
448 mutex_exit(&vd
->vdev_trim_io_lock
);
450 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
454 * The zio_done_func_t done callback for each TRIM issued via
455 * vdev_trim_simple(). It is responsible for updating the TRIM stats and
456 * limiting the number of in flight TRIM I/Os. Simple TRIM I/Os are best
457 * effort and are never reissued on failure.
460 vdev_trim_simple_cb(zio_t
*zio
)
462 vdev_t
*vd
= zio
->io_vd
;
464 mutex_enter(&vd
->vdev_trim_io_lock
);
466 if (zio
->io_error
!= 0) {
467 vd
->vdev_stat
.vs_trim_errors
++;
468 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_SIMPLE
,
469 0, 0, 0, 0, 1, zio
->io_orig_size
);
471 spa_iostats_trim_add(vd
->vdev_spa
, TRIM_TYPE_SIMPLE
,
472 1, zio
->io_orig_size
, 0, 0, 0, 0);
475 ASSERT3U(vd
->vdev_trim_inflight
[TRIM_TYPE_SIMPLE
], >, 0);
476 vd
->vdev_trim_inflight
[TRIM_TYPE_SIMPLE
]--;
477 cv_broadcast(&vd
->vdev_trim_io_cv
);
478 mutex_exit(&vd
->vdev_trim_io_lock
);
480 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
483 * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
486 vdev_trim_calculate_rate(trim_args_t
*ta
)
488 return (ta
->trim_bytes_done
* 1000 /
489 (NSEC2MSEC(gethrtime() - ta
->trim_start_time
) + 1));
493 * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
494 * and number of concurrent TRIM I/Os.
497 vdev_trim_range(trim_args_t
*ta
, uint64_t start
, uint64_t size
)
499 vdev_t
*vd
= ta
->trim_vdev
;
500 spa_t
*spa
= vd
->vdev_spa
;
503 mutex_enter(&vd
->vdev_trim_io_lock
);
506 * Limit manual TRIM I/Os to the requested rate. This does not
507 * apply to automatic TRIM since no per vdev rate can be specified.
509 if (ta
->trim_type
== TRIM_TYPE_MANUAL
) {
510 while (vd
->vdev_trim_rate
!= 0 && !vdev_trim_should_stop(vd
) &&
511 vdev_trim_calculate_rate(ta
) > vd
->vdev_trim_rate
) {
512 cv_timedwait_idle(&vd
->vdev_trim_io_cv
,
513 &vd
->vdev_trim_io_lock
, ddi_get_lbolt() +
517 ta
->trim_bytes_done
+= size
;
519 /* Limit in flight trimming I/Os */
520 while (vd
->vdev_trim_inflight
[0] + vd
->vdev_trim_inflight
[1] +
521 vd
->vdev_trim_inflight
[2] >= zfs_trim_queue_limit
) {
522 cv_wait(&vd
->vdev_trim_io_cv
, &vd
->vdev_trim_io_lock
);
524 vd
->vdev_trim_inflight
[ta
->trim_type
]++;
525 mutex_exit(&vd
->vdev_trim_io_lock
);
527 dmu_tx_t
*tx
= dmu_tx_create_dd(spa_get_dsl(spa
)->dp_mos_dir
);
528 VERIFY0(dmu_tx_assign(tx
, TXG_WAIT
));
529 uint64_t txg
= dmu_tx_get_txg(tx
);
531 spa_config_enter(spa
, SCL_STATE_ALL
, vd
, RW_READER
);
532 mutex_enter(&vd
->vdev_trim_lock
);
534 if (ta
->trim_type
== TRIM_TYPE_MANUAL
&&
535 vd
->vdev_trim_offset
[txg
& TXG_MASK
] == 0) {
536 uint64_t *guid
= kmem_zalloc(sizeof (uint64_t), KM_SLEEP
);
537 *guid
= vd
->vdev_guid
;
539 /* This is the first write of this txg. */
540 dsl_sync_task_nowait(spa_get_dsl(spa
),
541 vdev_trim_zap_update_sync
, guid
, tx
);
545 * We know the vdev_t will still be around since all consumers of
546 * vdev_free must stop the trimming first.
548 if ((ta
->trim_type
== TRIM_TYPE_MANUAL
&&
549 vdev_trim_should_stop(vd
)) ||
550 (ta
->trim_type
== TRIM_TYPE_AUTO
&&
551 vdev_autotrim_should_stop(vd
->vdev_top
))) {
552 mutex_enter(&vd
->vdev_trim_io_lock
);
553 vd
->vdev_trim_inflight
[ta
->trim_type
]--;
554 mutex_exit(&vd
->vdev_trim_io_lock
);
555 spa_config_exit(vd
->vdev_spa
, SCL_STATE_ALL
, vd
);
556 mutex_exit(&vd
->vdev_trim_lock
);
558 return (SET_ERROR(EINTR
));
560 mutex_exit(&vd
->vdev_trim_lock
);
562 if (ta
->trim_type
== TRIM_TYPE_MANUAL
)
563 vd
->vdev_trim_offset
[txg
& TXG_MASK
] = start
+ size
;
565 if (ta
->trim_type
== TRIM_TYPE_MANUAL
) {
567 } else if (ta
->trim_type
== TRIM_TYPE_AUTO
) {
568 cb
= vdev_autotrim_cb
;
570 cb
= vdev_trim_simple_cb
;
573 zio_nowait(zio_trim(spa
->spa_txg_zio
[txg
& TXG_MASK
], vd
,
574 start
, size
, cb
, NULL
, ZIO_PRIORITY_TRIM
, ZIO_FLAG_CANFAIL
,
576 /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
584 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
585 * Additional parameters describing how the TRIM should be performed must
586 * be set in the trim_args structure. See the trim_args definition for
587 * additional information.
590 vdev_trim_ranges(trim_args_t
*ta
)
592 vdev_t
*vd
= ta
->trim_vdev
;
593 zfs_btree_t
*t
= &ta
->trim_tree
->rt_root
;
594 zfs_btree_index_t idx
;
595 uint64_t extent_bytes_max
= ta
->trim_extent_bytes_max
;
596 uint64_t extent_bytes_min
= ta
->trim_extent_bytes_min
;
597 spa_t
*spa
= vd
->vdev_spa
;
600 ta
->trim_start_time
= gethrtime();
601 ta
->trim_bytes_done
= 0;
603 for (range_seg_t
*rs
= zfs_btree_first(t
, &idx
); rs
!= NULL
;
604 rs
= zfs_btree_next(t
, &idx
, &idx
)) {
605 uint64_t size
= rs_get_end(rs
, ta
->trim_tree
) - rs_get_start(rs
,
608 if (extent_bytes_min
&& size
< extent_bytes_min
) {
609 spa_iostats_trim_add(spa
, ta
->trim_type
,
610 0, 0, 1, size
, 0, 0);
614 /* Split range into legally-sized physical chunks */
615 uint64_t writes_required
= ((size
- 1) / extent_bytes_max
) + 1;
617 for (uint64_t w
= 0; w
< writes_required
; w
++) {
618 error
= vdev_trim_range(ta
, VDEV_LABEL_START_SIZE
+
619 rs_get_start(rs
, ta
->trim_tree
) +
620 (w
*extent_bytes_max
), MIN(size
-
621 (w
* extent_bytes_max
), extent_bytes_max
));
630 * Make sure all TRIMs for this metaslab have completed before
631 * returning. TRIM zios have lower priority over regular or syncing
632 * zios, so all TRIM zios for this metaslab must complete before the
633 * metaslab is re-enabled. Otherwise it's possible write zios to
634 * this metaslab could cut ahead of still queued TRIM zios for this
635 * metaslab causing corruption if the ranges overlap.
637 mutex_enter(&vd
->vdev_trim_io_lock
);
638 while (vd
->vdev_trim_inflight
[0] > 0) {
639 cv_wait(&vd
->vdev_trim_io_cv
, &vd
->vdev_trim_io_lock
);
641 mutex_exit(&vd
->vdev_trim_io_lock
);
647 vdev_trim_xlate_last_rs_end(void *arg
, range_seg64_t
*physical_rs
)
649 uint64_t *last_rs_end
= (uint64_t *)arg
;
651 if (physical_rs
->rs_end
> *last_rs_end
)
652 *last_rs_end
= physical_rs
->rs_end
;
656 vdev_trim_xlate_progress(void *arg
, range_seg64_t
*physical_rs
)
658 vdev_t
*vd
= (vdev_t
*)arg
;
660 uint64_t size
= physical_rs
->rs_end
- physical_rs
->rs_start
;
661 vd
->vdev_trim_bytes_est
+= size
;
663 if (vd
->vdev_trim_last_offset
>= physical_rs
->rs_end
) {
664 vd
->vdev_trim_bytes_done
+= size
;
665 } else if (vd
->vdev_trim_last_offset
> physical_rs
->rs_start
&&
666 vd
->vdev_trim_last_offset
<= physical_rs
->rs_end
) {
667 vd
->vdev_trim_bytes_done
+=
668 vd
->vdev_trim_last_offset
- physical_rs
->rs_start
;
673 * Calculates the completion percentage of a manual TRIM.
676 vdev_trim_calculate_progress(vdev_t
*vd
)
678 ASSERT(spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_READER
) ||
679 spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_WRITER
));
680 ASSERT(vd
->vdev_leaf_zap
!= 0);
682 vd
->vdev_trim_bytes_est
= 0;
683 vd
->vdev_trim_bytes_done
= 0;
685 for (uint64_t i
= 0; i
< vd
->vdev_top
->vdev_ms_count
; i
++) {
686 metaslab_t
*msp
= vd
->vdev_top
->vdev_ms
[i
];
687 mutex_enter(&msp
->ms_lock
);
689 uint64_t ms_free
= (msp
->ms_size
-
690 metaslab_allocated_space(msp
)) /
691 vdev_get_ndisks(vd
->vdev_top
);
694 * Convert the metaslab range to a physical range
695 * on our vdev. We use this to determine if we are
696 * in the middle of this metaslab range.
698 range_seg64_t logical_rs
, physical_rs
, remain_rs
;
699 logical_rs
.rs_start
= msp
->ms_start
;
700 logical_rs
.rs_end
= msp
->ms_start
+ msp
->ms_size
;
702 /* Metaslab space after this offset has not been trimmed. */
703 vdev_xlate(vd
, &logical_rs
, &physical_rs
, &remain_rs
);
704 if (vd
->vdev_trim_last_offset
<= physical_rs
.rs_start
) {
705 vd
->vdev_trim_bytes_est
+= ms_free
;
706 mutex_exit(&msp
->ms_lock
);
710 /* Metaslab space before this offset has been trimmed */
711 uint64_t last_rs_end
= physical_rs
.rs_end
;
712 if (!vdev_xlate_is_empty(&remain_rs
)) {
713 vdev_xlate_walk(vd
, &remain_rs
,
714 vdev_trim_xlate_last_rs_end
, &last_rs_end
);
717 if (vd
->vdev_trim_last_offset
> last_rs_end
) {
718 vd
->vdev_trim_bytes_done
+= ms_free
;
719 vd
->vdev_trim_bytes_est
+= ms_free
;
720 mutex_exit(&msp
->ms_lock
);
725 * If we get here, we're in the middle of trimming this
726 * metaslab. Load it and walk the free tree for more
727 * accurate progress estimation.
729 VERIFY0(metaslab_load(msp
));
731 range_tree_t
*rt
= msp
->ms_allocatable
;
732 zfs_btree_t
*bt
= &rt
->rt_root
;
733 zfs_btree_index_t idx
;
734 for (range_seg_t
*rs
= zfs_btree_first(bt
, &idx
);
735 rs
!= NULL
; rs
= zfs_btree_next(bt
, &idx
, &idx
)) {
736 logical_rs
.rs_start
= rs_get_start(rs
, rt
);
737 logical_rs
.rs_end
= rs_get_end(rs
, rt
);
739 vdev_xlate_walk(vd
, &logical_rs
,
740 vdev_trim_xlate_progress
, vd
);
742 mutex_exit(&msp
->ms_lock
);
747 * Load from disk the vdev's manual TRIM information. This includes the
748 * state, progress, and options provided when initiating the manual TRIM.
751 vdev_trim_load(vdev_t
*vd
)
754 ASSERT(spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_READER
) ||
755 spa_config_held(vd
->vdev_spa
, SCL_CONFIG
, RW_WRITER
));
756 ASSERT(vd
->vdev_leaf_zap
!= 0);
758 if (vd
->vdev_trim_state
== VDEV_TRIM_ACTIVE
||
759 vd
->vdev_trim_state
== VDEV_TRIM_SUSPENDED
) {
760 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
761 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET
,
762 sizeof (vd
->vdev_trim_last_offset
), 1,
763 &vd
->vdev_trim_last_offset
);
765 vd
->vdev_trim_last_offset
= 0;
770 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
771 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_RATE
,
772 sizeof (vd
->vdev_trim_rate
), 1,
773 &vd
->vdev_trim_rate
);
775 vd
->vdev_trim_rate
= 0;
781 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
782 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_PARTIAL
,
783 sizeof (vd
->vdev_trim_partial
), 1,
784 &vd
->vdev_trim_partial
);
786 vd
->vdev_trim_partial
= 0;
792 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
793 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_SECURE
,
794 sizeof (vd
->vdev_trim_secure
), 1,
795 &vd
->vdev_trim_secure
);
797 vd
->vdev_trim_secure
= 0;
803 vdev_trim_calculate_progress(vd
);
809 vdev_trim_xlate_range_add(void *arg
, range_seg64_t
*physical_rs
)
811 trim_args_t
*ta
= arg
;
812 vdev_t
*vd
= ta
->trim_vdev
;
815 * Only a manual trim will be traversing the vdev sequentially.
816 * For an auto trim all valid ranges should be added.
818 if (ta
->trim_type
== TRIM_TYPE_MANUAL
) {
820 /* Only add segments that we have not visited yet */
821 if (physical_rs
->rs_end
<= vd
->vdev_trim_last_offset
)
824 /* Pick up where we left off mid-range. */
825 if (vd
->vdev_trim_last_offset
> physical_rs
->rs_start
) {
826 ASSERT3U(physical_rs
->rs_end
, >,
827 vd
->vdev_trim_last_offset
);
828 physical_rs
->rs_start
= vd
->vdev_trim_last_offset
;
832 ASSERT3U(physical_rs
->rs_end
, >, physical_rs
->rs_start
);
834 range_tree_add(ta
->trim_tree
, physical_rs
->rs_start
,
835 physical_rs
->rs_end
- physical_rs
->rs_start
);
839 * Convert the logical range into physical ranges and add them to the
840 * range tree passed in the trim_args_t.
843 vdev_trim_range_add(void *arg
, uint64_t start
, uint64_t size
)
845 trim_args_t
*ta
= arg
;
846 vdev_t
*vd
= ta
->trim_vdev
;
847 range_seg64_t logical_rs
;
848 logical_rs
.rs_start
= start
;
849 logical_rs
.rs_end
= start
+ size
;
852 * Every range to be trimmed must be part of ms_allocatable.
853 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
854 * is always the case.
856 if (zfs_flags
& ZFS_DEBUG_TRIM
) {
857 metaslab_t
*msp
= ta
->trim_msp
;
858 VERIFY0(metaslab_load(msp
));
859 VERIFY3B(msp
->ms_loaded
, ==, B_TRUE
);
860 VERIFY(range_tree_contains(msp
->ms_allocatable
, start
, size
));
863 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
864 vdev_xlate_walk(vd
, &logical_rs
, vdev_trim_xlate_range_add
, arg
);
868 * Each manual TRIM thread is responsible for trimming the unallocated
869 * space for each leaf vdev. This is accomplished by sequentially iterating
870 * over its top-level metaslabs and issuing TRIM I/O for the space described
871 * by its ms_allocatable. While a metaslab is undergoing trimming it is
872 * not eligible for new allocations.
874 static __attribute__((noreturn
)) void
875 vdev_trim_thread(void *arg
)
878 spa_t
*spa
= vd
->vdev_spa
;
883 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
884 * vdev_trim(). Wait for the updated values to be reflected
885 * in the zap in order to start with the requested settings.
887 txg_wait_synced(spa_get_dsl(vd
->vdev_spa
), 0);
889 ASSERT(vdev_is_concrete(vd
));
890 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
892 vd
->vdev_trim_last_offset
= 0;
893 vd
->vdev_trim_rate
= 0;
894 vd
->vdev_trim_partial
= 0;
895 vd
->vdev_trim_secure
= 0;
897 VERIFY0(vdev_trim_load(vd
));
900 ta
.trim_extent_bytes_max
= zfs_trim_extent_bytes_max
;
901 ta
.trim_extent_bytes_min
= zfs_trim_extent_bytes_min
;
902 ta
.trim_tree
= range_tree_create(NULL
, RANGE_SEG64
, NULL
, 0, 0);
903 ta
.trim_type
= TRIM_TYPE_MANUAL
;
907 * When a secure TRIM has been requested infer that the intent
908 * is that everything must be trimmed. Override the default
909 * minimum TRIM size to prevent ranges from being skipped.
911 if (vd
->vdev_trim_secure
) {
912 ta
.trim_flags
|= ZIO_TRIM_SECURE
;
913 ta
.trim_extent_bytes_min
= SPA_MINBLOCKSIZE
;
916 uint64_t ms_count
= 0;
917 for (uint64_t i
= 0; !vd
->vdev_detached
&&
918 i
< vd
->vdev_top
->vdev_ms_count
; i
++) {
919 metaslab_t
*msp
= vd
->vdev_top
->vdev_ms
[i
];
922 * If we've expanded the top-level vdev or it's our
923 * first pass, calculate our progress.
925 if (vd
->vdev_top
->vdev_ms_count
!= ms_count
) {
926 vdev_trim_calculate_progress(vd
);
927 ms_count
= vd
->vdev_top
->vdev_ms_count
;
930 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
931 metaslab_disable(msp
);
932 mutex_enter(&msp
->ms_lock
);
933 VERIFY0(metaslab_load(msp
));
936 * If a partial TRIM was requested skip metaslabs which have
937 * never been initialized and thus have never been written.
939 if (msp
->ms_sm
== NULL
&& vd
->vdev_trim_partial
) {
940 mutex_exit(&msp
->ms_lock
);
941 metaslab_enable(msp
, B_FALSE
, B_FALSE
);
942 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
943 vdev_trim_calculate_progress(vd
);
948 range_tree_walk(msp
->ms_allocatable
, vdev_trim_range_add
, &ta
);
949 range_tree_vacate(msp
->ms_trim
, NULL
, NULL
);
950 mutex_exit(&msp
->ms_lock
);
952 error
= vdev_trim_ranges(&ta
);
953 metaslab_enable(msp
, B_TRUE
, B_FALSE
);
954 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
956 range_tree_vacate(ta
.trim_tree
, NULL
, NULL
);
961 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
963 range_tree_destroy(ta
.trim_tree
);
965 mutex_enter(&vd
->vdev_trim_lock
);
966 if (!vd
->vdev_trim_exit_wanted
) {
967 if (vdev_writeable(vd
)) {
968 vdev_trim_change_state(vd
, VDEV_TRIM_COMPLETE
,
969 vd
->vdev_trim_rate
, vd
->vdev_trim_partial
,
970 vd
->vdev_trim_secure
);
971 } else if (vd
->vdev_faulted
) {
972 vdev_trim_change_state(vd
, VDEV_TRIM_CANCELED
,
973 vd
->vdev_trim_rate
, vd
->vdev_trim_partial
,
974 vd
->vdev_trim_secure
);
977 ASSERT(vd
->vdev_trim_thread
!= NULL
|| vd
->vdev_trim_inflight
[0] == 0);
980 * Drop the vdev_trim_lock while we sync out the txg since it's
981 * possible that a device might be trying to come online and must
982 * check to see if it needs to restart a trim. That thread will be
983 * holding the spa_config_lock which would prevent the txg_wait_synced
986 mutex_exit(&vd
->vdev_trim_lock
);
987 txg_wait_synced(spa_get_dsl(spa
), 0);
988 mutex_enter(&vd
->vdev_trim_lock
);
990 vd
->vdev_trim_thread
= NULL
;
991 cv_broadcast(&vd
->vdev_trim_cv
);
992 mutex_exit(&vd
->vdev_trim_lock
);
998 * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock,
999 * the vdev_t must be a leaf and cannot already be manually trimming.
1002 vdev_trim(vdev_t
*vd
, uint64_t rate
, boolean_t partial
, boolean_t secure
)
1004 ASSERT(MUTEX_HELD(&vd
->vdev_trim_lock
));
1005 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
1006 ASSERT(vdev_is_concrete(vd
));
1007 ASSERT3P(vd
->vdev_trim_thread
, ==, NULL
);
1008 ASSERT(!vd
->vdev_detached
);
1009 ASSERT(!vd
->vdev_trim_exit_wanted
);
1010 ASSERT(!vd
->vdev_top
->vdev_removing
);
1011 ASSERT(!vd
->vdev_rz_expanding
);
1013 vdev_trim_change_state(vd
, VDEV_TRIM_ACTIVE
, rate
, partial
, secure
);
1014 vd
->vdev_trim_thread
= thread_create(NULL
, 0,
1015 vdev_trim_thread
, vd
, 0, &p0
, TS_RUN
, maxclsyspri
);
1019 * Wait for the trimming thread to be terminated (canceled or stopped).
1022 vdev_trim_stop_wait_impl(vdev_t
*vd
)
1024 ASSERT(MUTEX_HELD(&vd
->vdev_trim_lock
));
1026 while (vd
->vdev_trim_thread
!= NULL
)
1027 cv_wait(&vd
->vdev_trim_cv
, &vd
->vdev_trim_lock
);
1029 ASSERT3P(vd
->vdev_trim_thread
, ==, NULL
);
1030 vd
->vdev_trim_exit_wanted
= B_FALSE
;
1034 * Wait for vdev trim threads which were listed to cleanly exit.
1037 vdev_trim_stop_wait(spa_t
*spa
, list_t
*vd_list
)
1042 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1044 while ((vd
= list_remove_head(vd_list
)) != NULL
) {
1045 mutex_enter(&vd
->vdev_trim_lock
);
1046 vdev_trim_stop_wait_impl(vd
);
1047 mutex_exit(&vd
->vdev_trim_lock
);
1052 * Stop trimming a device, with the resultant trimming state being tgt_state.
1053 * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is
1054 * provided the stopping vdev is inserted in to the list. Callers are then
1055 * required to call vdev_trim_stop_wait() to block for all the trim threads
1056 * to exit. The caller must hold vdev_trim_lock and must not be writing to
1057 * the spa config, as the trimming thread may try to enter the config as a
1058 * reader before exiting.
1061 vdev_trim_stop(vdev_t
*vd
, vdev_trim_state_t tgt_state
, list_t
*vd_list
)
1063 ASSERT(!spa_config_held(vd
->vdev_spa
, SCL_CONFIG
|SCL_STATE
, RW_WRITER
));
1064 ASSERT(MUTEX_HELD(&vd
->vdev_trim_lock
));
1065 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
1066 ASSERT(vdev_is_concrete(vd
));
1069 * Allow cancel requests to proceed even if the trim thread has
1072 if (vd
->vdev_trim_thread
== NULL
&& tgt_state
!= VDEV_TRIM_CANCELED
)
1075 vdev_trim_change_state(vd
, tgt_state
, 0, 0, 0);
1076 vd
->vdev_trim_exit_wanted
= B_TRUE
;
1078 if (vd_list
== NULL
) {
1079 vdev_trim_stop_wait_impl(vd
);
1081 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1082 list_insert_tail(vd_list
, vd
);
1087 * Requests that all listed vdevs stop trimming.
1090 vdev_trim_stop_all_impl(vdev_t
*vd
, vdev_trim_state_t tgt_state
,
1093 if (vd
->vdev_ops
->vdev_op_leaf
&& vdev_is_concrete(vd
)) {
1094 mutex_enter(&vd
->vdev_trim_lock
);
1095 vdev_trim_stop(vd
, tgt_state
, vd_list
);
1096 mutex_exit(&vd
->vdev_trim_lock
);
1100 for (uint64_t i
= 0; i
< vd
->vdev_children
; i
++) {
1101 vdev_trim_stop_all_impl(vd
->vdev_child
[i
], tgt_state
,
1107 * Convenience function to stop trimming of a vdev tree and set all trim
1108 * thread pointers to NULL.
1111 vdev_trim_stop_all(vdev_t
*vd
, vdev_trim_state_t tgt_state
)
1113 spa_t
*spa
= vd
->vdev_spa
;
1117 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1119 list_create(&vd_list
, sizeof (vdev_t
),
1120 offsetof(vdev_t
, vdev_trim_node
));
1122 vdev_trim_stop_all_impl(vd
, tgt_state
, &vd_list
);
1125 * Iterate over cache devices and request stop trimming the
1126 * whole device in case we export the pool or remove the cache
1127 * device prematurely.
1129 for (int i
= 0; i
< spa
->spa_l2cache
.sav_count
; i
++) {
1130 vd_l2cache
= spa
->spa_l2cache
.sav_vdevs
[i
];
1131 vdev_trim_stop_all_impl(vd_l2cache
, tgt_state
, &vd_list
);
1134 vdev_trim_stop_wait(spa
, &vd_list
);
1136 if (vd
->vdev_spa
->spa_sync_on
) {
1137 /* Make sure that our state has been synced to disk */
1138 txg_wait_synced(spa_get_dsl(vd
->vdev_spa
), 0);
1141 list_destroy(&vd_list
);
1145 * Conditionally restarts a manual TRIM given its on-disk state.
1148 vdev_trim_restart(vdev_t
*vd
)
1150 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1151 ASSERT(!spa_config_held(vd
->vdev_spa
, SCL_ALL
, RW_WRITER
));
1153 if (vd
->vdev_leaf_zap
!= 0) {
1154 mutex_enter(&vd
->vdev_trim_lock
);
1155 uint64_t trim_state
= VDEV_TRIM_NONE
;
1156 int err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
1157 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_STATE
,
1158 sizeof (trim_state
), 1, &trim_state
);
1159 ASSERT(err
== 0 || err
== ENOENT
);
1160 vd
->vdev_trim_state
= trim_state
;
1162 uint64_t timestamp
= 0;
1163 err
= zap_lookup(vd
->vdev_spa
->spa_meta_objset
,
1164 vd
->vdev_leaf_zap
, VDEV_LEAF_ZAP_TRIM_ACTION_TIME
,
1165 sizeof (timestamp
), 1, ×tamp
);
1166 ASSERT(err
== 0 || err
== ENOENT
);
1167 vd
->vdev_trim_action_time
= timestamp
;
1169 if ((vd
->vdev_trim_state
== VDEV_TRIM_SUSPENDED
||
1170 vd
->vdev_offline
) && !vd
->vdev_top
->vdev_rz_expanding
) {
1171 /* load progress for reporting, but don't resume */
1172 VERIFY0(vdev_trim_load(vd
));
1173 } else if (vd
->vdev_trim_state
== VDEV_TRIM_ACTIVE
&&
1174 vdev_writeable(vd
) && !vd
->vdev_top
->vdev_removing
&&
1175 !vd
->vdev_top
->vdev_rz_expanding
&&
1176 vd
->vdev_trim_thread
== NULL
) {
1177 VERIFY0(vdev_trim_load(vd
));
1178 vdev_trim(vd
, vd
->vdev_trim_rate
,
1179 vd
->vdev_trim_partial
, vd
->vdev_trim_secure
);
1182 mutex_exit(&vd
->vdev_trim_lock
);
1185 for (uint64_t i
= 0; i
< vd
->vdev_children
; i
++) {
1186 vdev_trim_restart(vd
->vdev_child
[i
]);
1191 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1192 * every TRIM range is contained within ms_allocatable.
1195 vdev_trim_range_verify(void *arg
, uint64_t start
, uint64_t size
)
1197 trim_args_t
*ta
= arg
;
1198 metaslab_t
*msp
= ta
->trim_msp
;
1200 VERIFY3B(msp
->ms_loaded
, ==, B_TRUE
);
1201 VERIFY3U(msp
->ms_disabled
, >, 0);
1202 VERIFY(range_tree_contains(msp
->ms_allocatable
, start
, size
));
1206 * Each automatic TRIM thread is responsible for managing the trimming of a
1207 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk.
1209 * N.B. This behavior is different from a manual TRIM where a thread
1210 * is created for each leaf vdev, instead of each top-level vdev.
1212 static __attribute__((noreturn
)) void
1213 vdev_autotrim_thread(void *arg
)
1216 spa_t
*spa
= vd
->vdev_spa
;
1219 mutex_enter(&vd
->vdev_autotrim_lock
);
1220 ASSERT3P(vd
->vdev_top
, ==, vd
);
1221 ASSERT3P(vd
->vdev_autotrim_thread
, !=, NULL
);
1222 mutex_exit(&vd
->vdev_autotrim_lock
);
1223 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
1225 while (!vdev_autotrim_should_stop(vd
)) {
1226 int txgs_per_trim
= MAX(zfs_trim_txg_batch
, 1);
1227 uint64_t extent_bytes_max
= zfs_trim_extent_bytes_max
;
1228 uint64_t extent_bytes_min
= zfs_trim_extent_bytes_min
;
1231 * All of the metaslabs are divided in to groups of size
1232 * num_metaslabs / zfs_trim_txg_batch. Each of these groups
1233 * is composed of metaslabs which are spread evenly over the
1236 * For example, when zfs_trim_txg_batch = 32 (default) then
1237 * group 0 will contain metaslabs 0, 32, 64, ...;
1238 * group 1 will contain metaslabs 1, 33, 65, ...;
1239 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1241 * On each pass through the while() loop one of these groups
1242 * is selected. This is accomplished by using a shift value
1243 * to select the starting metaslab, then striding over the
1244 * metaslabs using the zfs_trim_txg_batch size. This is
1245 * done to accomplish two things.
1247 * 1) By dividing the metaslabs in to groups, and making sure
1248 * that each group takes a minimum of one txg to process.
1249 * Then zfs_trim_txg_batch controls the minimum number of
1250 * txgs which must occur before a metaslab is revisited.
1252 * 2) Selecting non-consecutive metaslabs distributes the
1253 * TRIM commands for a group evenly over the entire device.
1254 * This can be advantageous for certain types of devices.
1256 for (uint64_t i
= shift
% txgs_per_trim
; i
< vd
->vdev_ms_count
;
1257 i
+= txgs_per_trim
) {
1258 metaslab_t
*msp
= vd
->vdev_ms
[i
];
1259 range_tree_t
*trim_tree
;
1260 boolean_t issued_trim
= B_FALSE
;
1261 boolean_t wait_aborted
= B_FALSE
;
1263 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
1264 metaslab_disable(msp
);
1265 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
1267 mutex_enter(&msp
->ms_lock
);
1270 * Skip the metaslab when it has never been allocated
1271 * or when there are no recent frees to trim.
1273 if (msp
->ms_sm
== NULL
||
1274 range_tree_is_empty(msp
->ms_trim
)) {
1275 mutex_exit(&msp
->ms_lock
);
1276 metaslab_enable(msp
, B_FALSE
, B_FALSE
);
1281 * Skip the metaslab when it has already been disabled.
1282 * This may happen when a manual TRIM or initialize
1283 * operation is running concurrently. In the case
1284 * of a manual TRIM, the ms_trim tree will have been
1285 * vacated. Only ranges added after the manual TRIM
1286 * disabled the metaslab will be included in the tree.
1287 * These will be processed when the automatic TRIM
1288 * next revisits this metaslab.
1290 if (msp
->ms_disabled
> 1) {
1291 mutex_exit(&msp
->ms_lock
);
1292 metaslab_enable(msp
, B_FALSE
, B_FALSE
);
1297 * Allocate an empty range tree which is swapped in
1298 * for the existing ms_trim tree while it is processed.
1300 trim_tree
= range_tree_create(NULL
, RANGE_SEG64
, NULL
,
1302 range_tree_swap(&msp
->ms_trim
, &trim_tree
);
1303 ASSERT(range_tree_is_empty(msp
->ms_trim
));
1306 * There are two cases when constructing the per-vdev
1307 * trim trees for a metaslab. If the top-level vdev
1308 * has no children then it is also a leaf and should
1309 * be trimmed. Otherwise our children are the leaves
1310 * and a trim tree should be constructed for each.
1313 uint64_t children
= vd
->vdev_children
;
1314 if (children
== 0) {
1316 tap
= kmem_zalloc(sizeof (trim_args_t
) *
1317 children
, KM_SLEEP
);
1318 tap
[0].trim_vdev
= vd
;
1320 tap
= kmem_zalloc(sizeof (trim_args_t
) *
1321 children
, KM_SLEEP
);
1323 for (uint64_t c
= 0; c
< children
; c
++) {
1324 tap
[c
].trim_vdev
= vd
->vdev_child
[c
];
1328 for (uint64_t c
= 0; c
< children
; c
++) {
1329 trim_args_t
*ta
= &tap
[c
];
1330 vdev_t
*cvd
= ta
->trim_vdev
;
1333 ta
->trim_extent_bytes_max
= extent_bytes_max
;
1334 ta
->trim_extent_bytes_min
= extent_bytes_min
;
1335 ta
->trim_type
= TRIM_TYPE_AUTO
;
1338 if (cvd
->vdev_detached
||
1339 !vdev_writeable(cvd
) ||
1340 !cvd
->vdev_has_trim
||
1341 cvd
->vdev_trim_thread
!= NULL
) {
1346 * When a device has an attached hot spare, or
1347 * is being replaced it will not be trimmed.
1348 * This is done to avoid adding additional
1349 * stress to a potentially unhealthy device,
1350 * and to minimize the required rebuild time.
1352 if (!cvd
->vdev_ops
->vdev_op_leaf
)
1355 ta
->trim_tree
= range_tree_create(NULL
,
1356 RANGE_SEG64
, NULL
, 0, 0);
1357 range_tree_walk(trim_tree
,
1358 vdev_trim_range_add
, ta
);
1361 mutex_exit(&msp
->ms_lock
);
1362 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
1365 * Issue the TRIM I/Os for all ranges covered by the
1366 * TRIM trees. These ranges are safe to TRIM because
1367 * no new allocations will be performed until the call
1368 * to metaslab_enabled() below.
1370 for (uint64_t c
= 0; c
< children
; c
++) {
1371 trim_args_t
*ta
= &tap
[c
];
1374 * Always yield to a manual TRIM if one has
1375 * been started for the child vdev.
1377 if (ta
->trim_tree
== NULL
||
1378 ta
->trim_vdev
->vdev_trim_thread
!= NULL
) {
1383 * After this point metaslab_enable() must be
1384 * called with the sync flag set. This is done
1385 * here because vdev_trim_ranges() is allowed
1386 * to be interrupted (EINTR) before issuing all
1387 * of the required TRIM I/Os.
1389 issued_trim
= B_TRUE
;
1391 int error
= vdev_trim_ranges(ta
);
1397 * Verify every range which was trimmed is still
1398 * contained within the ms_allocatable tree.
1400 if (zfs_flags
& ZFS_DEBUG_TRIM
) {
1401 mutex_enter(&msp
->ms_lock
);
1402 VERIFY0(metaslab_load(msp
));
1403 VERIFY3P(tap
[0].trim_msp
, ==, msp
);
1404 range_tree_walk(trim_tree
,
1405 vdev_trim_range_verify
, &tap
[0]);
1406 mutex_exit(&msp
->ms_lock
);
1409 range_tree_vacate(trim_tree
, NULL
, NULL
);
1410 range_tree_destroy(trim_tree
);
1413 * Wait for couples of kicks, to ensure the trim io is
1414 * synced. If the wait is aborted due to
1415 * vdev_autotrim_exit_wanted, we need to signal
1416 * metaslab_enable() to wait for sync.
1419 wait_aborted
= vdev_autotrim_wait_kick(vd
,
1420 TXG_CONCURRENT_STATES
+ TXG_DEFER_SIZE
);
1423 metaslab_enable(msp
, wait_aborted
, B_FALSE
);
1424 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
1426 for (uint64_t c
= 0; c
< children
; c
++) {
1427 trim_args_t
*ta
= &tap
[c
];
1429 if (ta
->trim_tree
== NULL
)
1432 range_tree_vacate(ta
->trim_tree
, NULL
, NULL
);
1433 range_tree_destroy(ta
->trim_tree
);
1436 kmem_free(tap
, sizeof (trim_args_t
) * children
);
1438 if (vdev_autotrim_should_stop(vd
))
1442 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
1444 vdev_autotrim_wait_kick(vd
, 1);
1447 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
1450 for (uint64_t c
= 0; c
< vd
->vdev_children
; c
++) {
1451 vdev_t
*cvd
= vd
->vdev_child
[c
];
1452 mutex_enter(&cvd
->vdev_trim_io_lock
);
1454 while (cvd
->vdev_trim_inflight
[1] > 0) {
1455 cv_wait(&cvd
->vdev_trim_io_cv
,
1456 &cvd
->vdev_trim_io_lock
);
1458 mutex_exit(&cvd
->vdev_trim_io_lock
);
1461 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
1464 * When exiting because the autotrim property was set to off, then
1465 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1467 if (spa_get_autotrim(spa
) == SPA_AUTOTRIM_OFF
) {
1468 for (uint64_t i
= 0; i
< vd
->vdev_ms_count
; i
++) {
1469 metaslab_t
*msp
= vd
->vdev_ms
[i
];
1471 mutex_enter(&msp
->ms_lock
);
1472 range_tree_vacate(msp
->ms_trim
, NULL
, NULL
);
1473 mutex_exit(&msp
->ms_lock
);
1477 mutex_enter(&vd
->vdev_autotrim_lock
);
1478 ASSERT(vd
->vdev_autotrim_thread
!= NULL
);
1479 vd
->vdev_autotrim_thread
= NULL
;
1480 cv_broadcast(&vd
->vdev_autotrim_cv
);
1481 mutex_exit(&vd
->vdev_autotrim_lock
);
1487 * Starts an autotrim thread, if needed, for each top-level vdev which can be
1488 * trimmed. A top-level vdev which has been evacuated will never be trimmed.
1491 vdev_autotrim(spa_t
*spa
)
1493 vdev_t
*root_vd
= spa
->spa_root_vdev
;
1495 for (uint64_t i
= 0; i
< root_vd
->vdev_children
; i
++) {
1496 vdev_t
*tvd
= root_vd
->vdev_child
[i
];
1498 mutex_enter(&tvd
->vdev_autotrim_lock
);
1499 if (vdev_writeable(tvd
) && !tvd
->vdev_removing
&&
1500 tvd
->vdev_autotrim_thread
== NULL
&&
1501 !tvd
->vdev_rz_expanding
) {
1502 ASSERT3P(tvd
->vdev_top
, ==, tvd
);
1504 tvd
->vdev_autotrim_thread
= thread_create(NULL
, 0,
1505 vdev_autotrim_thread
, tvd
, 0, &p0
, TS_RUN
,
1507 ASSERT(tvd
->vdev_autotrim_thread
!= NULL
);
1509 mutex_exit(&tvd
->vdev_autotrim_lock
);
1514 * Wait for the vdev_autotrim_thread associated with the passed top-level
1515 * vdev to be terminated (canceled or stopped).
1518 vdev_autotrim_stop_wait(vdev_t
*tvd
)
1520 mutex_enter(&tvd
->vdev_autotrim_lock
);
1521 if (tvd
->vdev_autotrim_thread
!= NULL
) {
1522 tvd
->vdev_autotrim_exit_wanted
= B_TRUE
;
1523 cv_broadcast(&tvd
->vdev_autotrim_kick_cv
);
1524 cv_wait(&tvd
->vdev_autotrim_cv
,
1525 &tvd
->vdev_autotrim_lock
);
1527 ASSERT3P(tvd
->vdev_autotrim_thread
, ==, NULL
);
1528 tvd
->vdev_autotrim_exit_wanted
= B_FALSE
;
1530 mutex_exit(&tvd
->vdev_autotrim_lock
);
1534 vdev_autotrim_kick(spa_t
*spa
)
1536 ASSERT(spa_config_held(spa
, SCL_CONFIG
, RW_READER
));
1538 vdev_t
*root_vd
= spa
->spa_root_vdev
;
1541 for (uint64_t i
= 0; i
< root_vd
->vdev_children
; i
++) {
1542 tvd
= root_vd
->vdev_child
[i
];
1544 mutex_enter(&tvd
->vdev_autotrim_lock
);
1545 if (tvd
->vdev_autotrim_thread
!= NULL
)
1546 cv_broadcast(&tvd
->vdev_autotrim_kick_cv
);
1547 mutex_exit(&tvd
->vdev_autotrim_lock
);
1552 * Wait for all of the vdev_autotrim_thread associated with the pool to
1553 * be terminated (canceled or stopped).
1556 vdev_autotrim_stop_all(spa_t
*spa
)
1558 vdev_t
*root_vd
= spa
->spa_root_vdev
;
1560 for (uint64_t i
= 0; i
< root_vd
->vdev_children
; i
++)
1561 vdev_autotrim_stop_wait(root_vd
->vdev_child
[i
]);
1565 * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1568 vdev_autotrim_restart(spa_t
*spa
)
1570 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1572 if (spa
->spa_autotrim
)
1576 static __attribute__((noreturn
)) void
1577 vdev_trim_l2arc_thread(void *arg
)
1580 spa_t
*spa
= vd
->vdev_spa
;
1581 l2arc_dev_t
*dev
= l2arc_vdev_get(vd
);
1582 trim_args_t ta
= {0};
1583 range_seg64_t physical_rs
;
1585 ASSERT(vdev_is_concrete(vd
));
1586 spa_config_enter(spa
, SCL_CONFIG
, FTAG
, RW_READER
);
1588 vd
->vdev_trim_last_offset
= 0;
1589 vd
->vdev_trim_rate
= 0;
1590 vd
->vdev_trim_partial
= 0;
1591 vd
->vdev_trim_secure
= 0;
1594 ta
.trim_tree
= range_tree_create(NULL
, RANGE_SEG64
, NULL
, 0, 0);
1595 ta
.trim_type
= TRIM_TYPE_MANUAL
;
1596 ta
.trim_extent_bytes_max
= zfs_trim_extent_bytes_max
;
1597 ta
.trim_extent_bytes_min
= SPA_MINBLOCKSIZE
;
1600 physical_rs
.rs_start
= vd
->vdev_trim_bytes_done
= 0;
1601 physical_rs
.rs_end
= vd
->vdev_trim_bytes_est
=
1602 vdev_get_min_asize(vd
);
1604 range_tree_add(ta
.trim_tree
, physical_rs
.rs_start
,
1605 physical_rs
.rs_end
- physical_rs
.rs_start
);
1607 mutex_enter(&vd
->vdev_trim_lock
);
1608 vdev_trim_change_state(vd
, VDEV_TRIM_ACTIVE
, 0, 0, 0);
1609 mutex_exit(&vd
->vdev_trim_lock
);
1611 (void) vdev_trim_ranges(&ta
);
1613 spa_config_exit(spa
, SCL_CONFIG
, FTAG
);
1614 mutex_enter(&vd
->vdev_trim_io_lock
);
1615 while (vd
->vdev_trim_inflight
[TRIM_TYPE_MANUAL
] > 0) {
1616 cv_wait(&vd
->vdev_trim_io_cv
, &vd
->vdev_trim_io_lock
);
1618 mutex_exit(&vd
->vdev_trim_io_lock
);
1620 range_tree_vacate(ta
.trim_tree
, NULL
, NULL
);
1621 range_tree_destroy(ta
.trim_tree
);
1623 mutex_enter(&vd
->vdev_trim_lock
);
1624 if (!vd
->vdev_trim_exit_wanted
&& vdev_writeable(vd
)) {
1625 vdev_trim_change_state(vd
, VDEV_TRIM_COMPLETE
,
1626 vd
->vdev_trim_rate
, vd
->vdev_trim_partial
,
1627 vd
->vdev_trim_secure
);
1629 ASSERT(vd
->vdev_trim_thread
!= NULL
||
1630 vd
->vdev_trim_inflight
[TRIM_TYPE_MANUAL
] == 0);
1633 * Drop the vdev_trim_lock while we sync out the txg since it's
1634 * possible that a device might be trying to come online and
1635 * must check to see if it needs to restart a trim. That thread
1636 * will be holding the spa_config_lock which would prevent the
1637 * txg_wait_synced from completing. Same strategy as in
1638 * vdev_trim_thread().
1640 mutex_exit(&vd
->vdev_trim_lock
);
1641 txg_wait_synced(spa_get_dsl(vd
->vdev_spa
), 0);
1642 mutex_enter(&vd
->vdev_trim_lock
);
1645 * Update the header of the cache device here, before
1646 * broadcasting vdev_trim_cv which may lead to the removal
1647 * of the device. The same applies for setting l2ad_trim_all to
1650 spa_config_enter(vd
->vdev_spa
, SCL_L2ARC
, vd
,
1652 memset(dev
->l2ad_dev_hdr
, 0, dev
->l2ad_dev_hdr_asize
);
1653 l2arc_dev_hdr_update(dev
);
1654 spa_config_exit(vd
->vdev_spa
, SCL_L2ARC
, vd
);
1656 vd
->vdev_trim_thread
= NULL
;
1657 if (vd
->vdev_trim_state
== VDEV_TRIM_COMPLETE
)
1658 dev
->l2ad_trim_all
= B_FALSE
;
1660 cv_broadcast(&vd
->vdev_trim_cv
);
1661 mutex_exit(&vd
->vdev_trim_lock
);
1667 * Punches out TRIM threads for the L2ARC devices in a spa and assigns them
1668 * to vd->vdev_trim_thread variable. This facilitates the management of
1669 * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition
1670 * to a pool or pool creation or when the header of the device is invalid.
1673 vdev_trim_l2arc(spa_t
*spa
)
1675 ASSERT(MUTEX_HELD(&spa_namespace_lock
));
1678 * Locate the spa's l2arc devices and kick off TRIM threads.
1680 for (int i
= 0; i
< spa
->spa_l2cache
.sav_count
; i
++) {
1681 vdev_t
*vd
= spa
->spa_l2cache
.sav_vdevs
[i
];
1682 l2arc_dev_t
*dev
= l2arc_vdev_get(vd
);
1684 if (dev
== NULL
|| !dev
->l2ad_trim_all
) {
1686 * Don't attempt TRIM if the vdev is UNAVAIL or if the
1687 * cache device was not marked for whole device TRIM
1688 * (ie l2arc_trim_ahead = 0, or the L2ARC device header
1689 * is valid with trim_state = VDEV_TRIM_COMPLETE and
1690 * l2ad_log_entries > 0).
1695 mutex_enter(&vd
->vdev_trim_lock
);
1696 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
1697 ASSERT(vdev_is_concrete(vd
));
1698 ASSERT3P(vd
->vdev_trim_thread
, ==, NULL
);
1699 ASSERT(!vd
->vdev_detached
);
1700 ASSERT(!vd
->vdev_trim_exit_wanted
);
1701 ASSERT(!vd
->vdev_top
->vdev_removing
);
1702 vdev_trim_change_state(vd
, VDEV_TRIM_ACTIVE
, 0, 0, 0);
1703 vd
->vdev_trim_thread
= thread_create(NULL
, 0,
1704 vdev_trim_l2arc_thread
, vd
, 0, &p0
, TS_RUN
, maxclsyspri
);
1705 mutex_exit(&vd
->vdev_trim_lock
);
1710 * A wrapper which calls vdev_trim_ranges(). It is intended to be called
1714 vdev_trim_simple(vdev_t
*vd
, uint64_t start
, uint64_t size
)
1716 trim_args_t ta
= {0};
1717 range_seg64_t physical_rs
;
1719 physical_rs
.rs_start
= start
;
1720 physical_rs
.rs_end
= start
+ size
;
1722 ASSERT(vdev_is_concrete(vd
));
1723 ASSERT(vd
->vdev_ops
->vdev_op_leaf
);
1724 ASSERT(!vd
->vdev_detached
);
1725 ASSERT(!vd
->vdev_top
->vdev_removing
);
1726 ASSERT(!vd
->vdev_top
->vdev_rz_expanding
);
1729 ta
.trim_tree
= range_tree_create(NULL
, RANGE_SEG64
, NULL
, 0, 0);
1730 ta
.trim_type
= TRIM_TYPE_SIMPLE
;
1731 ta
.trim_extent_bytes_max
= zfs_trim_extent_bytes_max
;
1732 ta
.trim_extent_bytes_min
= SPA_MINBLOCKSIZE
;
1735 ASSERT3U(physical_rs
.rs_end
, >=, physical_rs
.rs_start
);
1737 if (physical_rs
.rs_end
> physical_rs
.rs_start
) {
1738 range_tree_add(ta
.trim_tree
, physical_rs
.rs_start
,
1739 physical_rs
.rs_end
- physical_rs
.rs_start
);
1741 ASSERT3U(physical_rs
.rs_end
, ==, physical_rs
.rs_start
);
1744 error
= vdev_trim_ranges(&ta
);
1746 mutex_enter(&vd
->vdev_trim_io_lock
);
1747 while (vd
->vdev_trim_inflight
[TRIM_TYPE_SIMPLE
] > 0) {
1748 cv_wait(&vd
->vdev_trim_io_cv
, &vd
->vdev_trim_io_lock
);
1750 mutex_exit(&vd
->vdev_trim_io_lock
);
1752 range_tree_vacate(ta
.trim_tree
, NULL
, NULL
);
1753 range_tree_destroy(ta
.trim_tree
);
1758 EXPORT_SYMBOL(vdev_trim
);
1759 EXPORT_SYMBOL(vdev_trim_stop
);
1760 EXPORT_SYMBOL(vdev_trim_stop_all
);
1761 EXPORT_SYMBOL(vdev_trim_stop_wait
);
1762 EXPORT_SYMBOL(vdev_trim_restart
);
1763 EXPORT_SYMBOL(vdev_autotrim
);
1764 EXPORT_SYMBOL(vdev_autotrim_stop_all
);
1765 EXPORT_SYMBOL(vdev_autotrim_stop_wait
);
1766 EXPORT_SYMBOL(vdev_autotrim_restart
);
1767 EXPORT_SYMBOL(vdev_trim_l2arc
);
1768 EXPORT_SYMBOL(vdev_trim_simple
);
1770 ZFS_MODULE_PARAM(zfs_trim
, zfs_trim_
, extent_bytes_max
, UINT
, ZMOD_RW
,
1771 "Max size of TRIM commands, larger will be split");
1773 ZFS_MODULE_PARAM(zfs_trim
, zfs_trim_
, extent_bytes_min
, UINT
, ZMOD_RW
,
1774 "Min size of TRIM commands, smaller will be skipped");
1776 ZFS_MODULE_PARAM(zfs_trim
, zfs_trim_
, metaslab_skip
, UINT
, ZMOD_RW
,
1777 "Skip metaslabs which have never been initialized");
1779 ZFS_MODULE_PARAM(zfs_trim
, zfs_trim_
, txg_batch
, UINT
, ZMOD_RW
,
1780 "Min number of txgs to aggregate frees before issuing TRIM");
1782 ZFS_MODULE_PARAM(zfs_trim
, zfs_trim_
, queue_limit
, UINT
, ZMOD_RW
,
1783 "Max queued TRIMs outstanding per leaf vdev");