4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2013 by Delphix. All rights reserved.
26 #include <sys/zfs_context.h>
27 #include <sys/zfeature.h>
29 #include <sys/nvpair.h>
31 #include <sys/dmu_tx.h>
32 #include "zfeature_common.h"
33 #include <sys/spa_impl.h>
39 * ZFS feature flags are used to provide fine-grained versioning to the ZFS
40 * on-disk format. Once enabled on a pool feature flags replace the old
41 * spa_version() number.
43 * Each new on-disk format change will be given a uniquely identifying string
44 * guid rather than a version number. This avoids the problem of different
45 * organizations creating new on-disk formats with the same version number. To
46 * keep feature guids unique they should consist of the reverse dns name of the
47 * organization which implemented the feature and a short name for the feature,
48 * separated by a colon (e.g. com.delphix:async_destroy).
53 * Within each pool features can be in one of three states: disabled, enabled,
54 * or active. These states are differentiated by a reference count stored on
55 * disk for each feature:
57 * 1) If there is no reference count stored on disk the feature is disabled.
58 * 2) If the reference count is 0 a system administrator has enabled the
59 * feature, but the feature has not been used yet, so no on-disk
60 * format changes have been made.
61 * 3) If the reference count is greater than 0 the feature is active.
62 * The format changes required by the feature are currently on disk.
63 * Note that if the feature's format changes are reversed the feature
64 * may choose to set its reference count back to 0.
66 * Feature flags makes no differentiation between non-zero reference counts
67 * for an active feature (e.g. a reference count of 1 means the same thing as a
68 * reference count of 27834721), but feature implementations may choose to use
69 * the reference count to store meaningful information. For example, a new RAID
70 * implementation might set the reference count to the number of vdevs using
71 * it. If all those disks are removed from the pool the feature goes back to
72 * having a reference count of 0.
74 * It is the responsibility of the individual features to maintain a non-zero
75 * reference count as long as the feature's format changes are present on disk.
80 * Each feature may depend on other features. The only effect of this
81 * relationship is that when a feature is enabled all of its dependencies are
82 * automatically enabled as well. Any future work to support disabling of
83 * features would need to ensure that features cannot be disabled if other
84 * enabled features depend on them.
89 * When feature flags are enabled spa_version() is set to SPA_VERSION_FEATURES
90 * (5000). In order for this to work the pool is automatically upgraded to
91 * SPA_VERSION_BEFORE_FEATURES (28) first, so all pre-feature flags on disk
92 * format changes will be in use.
94 * Information about features is stored in 3 ZAP objects in the pool's MOS.
95 * These objects are linked to by the following names in the pool directory
98 * 1) features_for_read: feature guid -> reference count
99 * Features needed to open the pool for reading.
100 * 2) features_for_write: feature guid -> reference count
101 * Features needed to open the pool for writing.
102 * 3) feature_descriptions: feature guid -> descriptive string
103 * A human readable string.
105 * All enabled features appear in either features_for_read or
106 * features_for_write, but not both.
108 * To open a pool in read-only mode only the features listed in
109 * features_for_read need to be supported.
111 * To open the pool in read-write mode features in both features_for_read and
112 * features_for_write need to be supported.
114 * Some features may be required to read the ZAP objects containing feature
115 * information. To allow software to check for compatibility with these features
116 * before the pool is opened their names must be stored in the label in a
117 * new "features_for_read" entry (note that features that are only required
118 * to write to a pool never need to be stored in the label since the
119 * features_for_write ZAP object can be read before the pool is written to).
120 * To save space in the label features must be explicitly marked as needing to
121 * be written to the label. Also, reference counts are not stored in the label,
122 * instead any feature whose reference count drops to 0 is removed from the
125 * Adding New Features
126 * -------------------
128 * Features must be registered in zpool_feature_init() function in
129 * zfeature_common.c using the zfeature_register() function. This function
130 * has arguments to specify if the feature should be stored in the
131 * features_for_read or features_for_write ZAP object and if it needs to be
132 * written to the label when active.
134 * Once a feature is registered it will appear as a "feature@<feature name>"
135 * property which can be set by an administrator. Feature implementors should
136 * use the spa_feature_is_enabled() and spa_feature_is_active() functions to
137 * query the state of a feature and the spa_feature_incr() and
138 * spa_feature_decr() functions to change an enabled feature's reference count.
139 * Reference counts may only be updated in the syncing context.
141 * Features may not perform enable-time initialization. Instead, any such
142 * initialization should occur when the feature is first used. This design
143 * enforces that on-disk changes be made only when features are used. Code
144 * should only check if a feature is enabled using spa_feature_is_enabled(),
145 * not by relying on any feature specific metadata existing. If a feature is
146 * enabled, but the feature's metadata is not on disk yet then it should be
149 * As an example, consider the com.delphix:async_destroy feature. This feature
150 * relies on the existence of a bptree in the MOS that store blocks for
151 * asynchronous freeing. This bptree is not created when async_destroy is
152 * enabled. Instead, when a dataset is destroyed spa_feature_is_enabled() is
153 * called to check if async_destroy is enabled. If it is and the bptree object
154 * does not exist yet, the bptree object is created as part of the dataset
155 * destroy and async_destroy's reference count is incremented to indicate it
156 * has made an on-disk format change. Later, after the destroyed dataset's
157 * blocks have all been asynchronously freed there is no longer any use for the
158 * bptree object, so it is destroyed and async_destroy's reference count is
159 * decremented back to 0 to indicate that it has undone its on-disk format
169 * Checks that the active features in the pool are supported by
170 * this software. Adds each unsupported feature (name -> description) to
171 * the supplied nvlist.
174 spa_features_check(spa_t
*spa
, boolean_t for_write
,
175 nvlist_t
*unsup_feat
, nvlist_t
*enabled_feat
)
177 objset_t
*os
= spa
->spa_meta_objset
;
181 uint64_t obj
= for_write
?
182 spa
->spa_feat_for_write_obj
: spa
->spa_feat_for_read_obj
;
185 zc
= kmem_alloc(sizeof (zap_cursor_t
), KM_SLEEP
);
186 za
= kmem_alloc(sizeof (zap_attribute_t
), KM_SLEEP
);
187 buf
= kmem_alloc(MAXPATHLEN
, KM_SLEEP
);
190 for (zap_cursor_init(zc
, os
, obj
);
191 zap_cursor_retrieve(zc
, za
) == 0;
192 zap_cursor_advance(zc
)) {
193 ASSERT(za
->za_integer_length
== sizeof (uint64_t) &&
194 za
->za_num_integers
== 1);
196 if (NULL
!= enabled_feat
) {
197 fnvlist_add_uint64(enabled_feat
, za
->za_name
,
198 za
->za_first_integer
);
201 if (za
->za_first_integer
!= 0 &&
202 !zfeature_is_supported(za
->za_name
)) {
205 if (NULL
!= unsup_feat
) {
208 if (zap_lookup(os
, spa
->spa_feat_desc_obj
,
209 za
->za_name
, 1, MAXPATHLEN
, buf
) == 0)
212 VERIFY(nvlist_add_string(unsup_feat
,
213 za
->za_name
, desc
) == 0);
219 kmem_free(buf
, MAXPATHLEN
);
220 kmem_free(za
, sizeof (zap_attribute_t
));
221 kmem_free(zc
, sizeof (zap_cursor_t
));
227 * Use an in-memory cache of feature refcounts for quick retrieval.
229 * Note: well-designed features will not need to use this; they should
230 * use spa_feature_is_enabled() and spa_feature_is_active() instead.
231 * However, this is non-static for zdb, zhack, and spa_add_feature_stats().
234 feature_get_refcount(spa_t
*spa
, zfeature_info_t
*feature
, uint64_t *res
)
236 ASSERT(VALID_FEATURE_FID(feature
->fi_feature
));
237 if (spa
->spa_feat_refcount_cache
[feature
->fi_feature
] ==
238 SPA_FEATURE_DISABLED
) {
239 return (SET_ERROR(ENOTSUP
));
241 *res
= spa
->spa_feat_refcount_cache
[feature
->fi_feature
];
246 * Note: well-designed features will not need to use this; they should
247 * use spa_feature_is_enabled() and spa_feature_is_active() instead.
248 * However, this is non-static for zdb and zhack.
251 feature_get_refcount_from_disk(spa_t
*spa
, zfeature_info_t
*feature
,
256 uint64_t zapobj
= feature
->fi_can_readonly
?
257 spa
->spa_feat_for_write_obj
: spa
->spa_feat_for_read_obj
;
260 * If the pool is currently being created, the feature objects may not
261 * have been allocated yet. Act as though all features are disabled.
264 return (SET_ERROR(ENOTSUP
));
266 err
= zap_lookup(spa
->spa_meta_objset
, zapobj
,
267 feature
->fi_guid
, sizeof (uint64_t), 1, &refcount
);
270 return (SET_ERROR(ENOTSUP
));
280 feature_get_enabled_txg(spa_t
*spa
, zfeature_info_t
*feature
, uint64_t *res
) {
281 ASSERTV(uint64_t enabled_txg_obj
= spa
->spa_feat_enabled_txg_obj
);
283 ASSERT(zfeature_depends_on(feature
->fi_feature
,
284 SPA_FEATURE_ENABLED_TXG
));
286 if (!spa_feature_is_enabled(spa
, feature
->fi_feature
)) {
287 return (SET_ERROR(ENOTSUP
));
290 ASSERT(enabled_txg_obj
!= 0);
292 VERIFY0(zap_lookup(spa
->spa_meta_objset
, spa
->spa_feat_enabled_txg_obj
,
293 feature
->fi_guid
, sizeof (uint64_t), 1, res
));
299 * This function is non-static for zhack; it should otherwise not be used
303 feature_sync(spa_t
*spa
, zfeature_info_t
*feature
, uint64_t refcount
,
308 ASSERT(VALID_FEATURE_OR_NONE(feature
->fi_feature
));
309 zapobj
= feature
->fi_can_readonly
?
310 spa
->spa_feat_for_write_obj
: spa
->spa_feat_for_read_obj
;
311 VERIFY0(zap_update(spa
->spa_meta_objset
, zapobj
, feature
->fi_guid
,
312 sizeof (uint64_t), 1, &refcount
, tx
));
315 * feature_sync is called directly from zhack, allowing the
316 * creation of arbitrary features whose fi_feature field may
317 * be greater than SPA_FEATURES. When called from zhack, the
318 * zfeature_info_t object's fi_feature field will be set to
321 if (feature
->fi_feature
!= SPA_FEATURE_NONE
) {
322 uint64_t *refcount_cache
=
323 &spa
->spa_feat_refcount_cache
[feature
->fi_feature
];
324 VERIFY3U(*refcount_cache
, ==,
325 atomic_swap_64(refcount_cache
, refcount
));
329 spa_deactivate_mos_feature(spa
, feature
->fi_guid
);
330 else if (feature
->fi_mos
)
331 spa_activate_mos_feature(spa
, feature
->fi_guid
, tx
);
335 * This function is non-static for zhack; it should otherwise not be used
339 feature_enable_sync(spa_t
*spa
, zfeature_info_t
*feature
, dmu_tx_t
*tx
)
341 uint64_t initial_refcount
= feature
->fi_activate_on_enable
? 1 : 0;
342 uint64_t zapobj
= feature
->fi_can_readonly
?
343 spa
->spa_feat_for_write_obj
: spa
->spa_feat_for_read_obj
;
347 ASSERT(zfeature_is_valid_guid(feature
->fi_guid
));
348 ASSERT3U(spa_version(spa
), >=, SPA_VERSION_FEATURES
);
351 * If the feature is already enabled, ignore the request.
353 if (zap_contains(spa
->spa_meta_objset
, zapobj
, feature
->fi_guid
) == 0)
356 for (i
= 0; feature
->fi_depends
[i
] != SPA_FEATURE_NONE
; i
++)
357 spa_feature_enable(spa
, feature
->fi_depends
[i
], tx
);
359 VERIFY0(zap_update(spa
->spa_meta_objset
, spa
->spa_feat_desc_obj
,
360 feature
->fi_guid
, 1, strlen(feature
->fi_desc
) + 1,
361 feature
->fi_desc
, tx
));
363 feature_sync(spa
, feature
, initial_refcount
, tx
);
365 if (spa_feature_is_enabled(spa
, SPA_FEATURE_ENABLED_TXG
)) {
366 uint64_t enabling_txg
= dmu_tx_get_txg(tx
);
368 if (spa
->spa_feat_enabled_txg_obj
== 0ULL) {
369 spa
->spa_feat_enabled_txg_obj
=
370 zap_create_link(spa
->spa_meta_objset
,
371 DMU_OTN_ZAP_METADATA
, DMU_POOL_DIRECTORY_OBJECT
,
372 DMU_POOL_FEATURE_ENABLED_TXG
, tx
);
374 spa_feature_incr(spa
, SPA_FEATURE_ENABLED_TXG
, tx
);
376 VERIFY0(zap_add(spa
->spa_meta_objset
,
377 spa
->spa_feat_enabled_txg_obj
, feature
->fi_guid
,
378 sizeof (uint64_t), 1, &enabling_txg
, tx
));
383 feature_do_action(spa_t
*spa
, spa_feature_t fid
, feature_action_t action
,
386 uint64_t refcount
= 0;
387 zfeature_info_t
*feature
= &spa_feature_table
[fid
];
388 ASSERTV(uint64_t zapobj
= feature
->fi_can_readonly
?
389 spa
->spa_feat_for_write_obj
: spa
->spa_feat_for_read_obj
);
391 ASSERT(VALID_FEATURE_FID(fid
));
393 ASSERT(zfeature_is_valid_guid(feature
->fi_guid
));
395 ASSERT(dmu_tx_is_syncing(tx
));
396 ASSERT3U(spa_version(spa
), >=, SPA_VERSION_FEATURES
);
398 VERIFY3U(feature_get_refcount(spa
, feature
, &refcount
), !=, ENOTSUP
);
401 case FEATURE_ACTION_INCR
:
402 VERIFY3U(refcount
, !=, UINT64_MAX
);
405 case FEATURE_ACTION_DECR
:
406 VERIFY3U(refcount
, !=, 0);
414 feature_sync(spa
, feature
, refcount
, tx
);
418 spa_feature_create_zap_objects(spa_t
*spa
, dmu_tx_t
*tx
)
421 * We create feature flags ZAP objects in two instances: during pool
422 * creation and during pool upgrade.
424 ASSERT(dsl_pool_sync_context(spa_get_dsl(spa
)) || (!spa
->spa_sync_on
&&
425 tx
->tx_txg
== TXG_INITIAL
));
427 spa
->spa_feat_for_read_obj
= zap_create_link(spa
->spa_meta_objset
,
428 DMU_OTN_ZAP_METADATA
, DMU_POOL_DIRECTORY_OBJECT
,
429 DMU_POOL_FEATURES_FOR_READ
, tx
);
430 spa
->spa_feat_for_write_obj
= zap_create_link(spa
->spa_meta_objset
,
431 DMU_OTN_ZAP_METADATA
, DMU_POOL_DIRECTORY_OBJECT
,
432 DMU_POOL_FEATURES_FOR_WRITE
, tx
);
433 spa
->spa_feat_desc_obj
= zap_create_link(spa
->spa_meta_objset
,
434 DMU_OTN_ZAP_METADATA
, DMU_POOL_DIRECTORY_OBJECT
,
435 DMU_POOL_FEATURE_DESCRIPTIONS
, tx
);
439 * Enable any required dependencies, then enable the requested feature.
442 spa_feature_enable(spa_t
*spa
, spa_feature_t fid
, dmu_tx_t
*tx
)
444 ASSERT3U(spa_version(spa
), >=, SPA_VERSION_FEATURES
);
445 ASSERT(VALID_FEATURE_FID(fid
));
446 feature_enable_sync(spa
, &spa_feature_table
[fid
], tx
);
450 spa_feature_incr(spa_t
*spa
, spa_feature_t fid
, dmu_tx_t
*tx
)
452 feature_do_action(spa
, fid
, FEATURE_ACTION_INCR
, tx
);
456 spa_feature_decr(spa_t
*spa
, spa_feature_t fid
, dmu_tx_t
*tx
)
458 feature_do_action(spa
, fid
, FEATURE_ACTION_DECR
, tx
);
462 spa_feature_is_enabled(spa_t
*spa
, spa_feature_t fid
)
465 uint64_t refcount
= 0;
467 ASSERT(VALID_FEATURE_FID(fid
));
468 if (spa_version(spa
) < SPA_VERSION_FEATURES
)
471 err
= feature_get_refcount(spa
, &spa_feature_table
[fid
], &refcount
);
472 ASSERT(err
== 0 || err
== ENOTSUP
);
477 spa_feature_is_active(spa_t
*spa
, spa_feature_t fid
)
480 uint64_t refcount
= 0;
482 ASSERT(VALID_FEATURE_FID(fid
));
483 if (spa_version(spa
) < SPA_VERSION_FEATURES
)
486 err
= feature_get_refcount(spa
, &spa_feature_table
[fid
], &refcount
);
487 ASSERT(err
== 0 || err
== ENOTSUP
);
488 return (err
== 0 && refcount
> 0);
492 * For the feature specified by fid (which must depend on
493 * SPA_FEATURE_ENABLED_TXG), return the TXG at which it was enabled in the
496 * Returns B_TRUE if the feature is enabled, in which case txg will be filled
497 * with the transaction group in which the specified feature was enabled.
498 * Returns B_FALSE otherwise (i.e. if the feature is not enabled).
501 spa_feature_enabled_txg(spa_t
*spa
, spa_feature_t fid
, uint64_t *txg
) {
504 ASSERT(VALID_FEATURE_FID(fid
));
505 if (spa_version(spa
) < SPA_VERSION_FEATURES
)
508 err
= feature_get_enabled_txg(spa
, &spa_feature_table
[fid
], txg
);
509 ASSERT(err
== 0 || err
== ENOTSUP
);