4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
9 * A full copy of the text of the CDDL should have accompanied this
10 * source. A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
17 * Copyright (c) 2017, Datto, Inc. All rights reserved.
20 #include <sys/zio_crypt.h>
22 #include <sys/dmu_objset.h>
23 #include <sys/dnode.h>
24 #include <sys/fs/zfs.h>
31 * This file is responsible for handling all of the details of generating
32 * encryption parameters and performing encryption and authentication.
34 * BLOCK ENCRYPTION PARAMETERS:
35 * Encryption /Authentication Algorithm Suite (crypt):
36 * The encryption algorithm, mode, and key length we are going to use. We
37 * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
38 * keys. All authentication is currently done with SHA512-HMAC.
41 * The unencrypted data that we want to encrypt.
43 * Initialization Vector (IV):
44 * An initialization vector for the encryption algorithms. This is used to
45 * "tweak" the encryption algorithms so that two blocks of the same data are
46 * encrypted into different ciphertext outputs, thus obfuscating block patterns.
47 * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
48 * never reused with the same encryption key. This value is stored unencrypted
49 * and must simply be provided to the decryption function. We use a 96 bit IV
50 * (as recommended by NIST) for all block encryption. For non-dedup blocks we
51 * derive the IV randomly. The first 64 bits of the IV are stored in the second
52 * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
53 * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
54 * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
55 * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
56 * level 0 blocks is the number of allocated dnodes in that block. The on-disk
57 * format supports at most 2^15 slots per L0 dnode block, because the maximum
58 * block size is 16MB (2^24). In either case, for level 0 blocks this number
59 * will still be smaller than UINT32_MAX so it is safe to store the IV in the
60 * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
64 * This is the most important secret data of an encrypted dataset. It is used
65 * along with the salt to generate that actual encryption keys via HKDF. We
66 * do not use the master key to directly encrypt any data because there are
67 * theoretical limits on how much data can actually be safely encrypted with
68 * any encryption mode. The master key is stored encrypted on disk with the
69 * user's wrapping key. Its length is determined by the encryption algorithm.
70 * For details on how this is stored see the block comment in dsl_crypt.c
73 * Used as an input to the HKDF function, along with the master key. We use a
74 * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
75 * can be used for encrypting many blocks, so we cache the current salt and the
76 * associated derived key in zio_crypt_t so we do not need to derive it again
80 * A secret binary key, generated from an HKDF function used to encrypt and
83 * Message Authenication Code (MAC)
84 * The MAC is an output of authenticated encryption modes such as AES-GCM and
85 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
86 * data on disk and return garbage to the application. Effectively, it is a
87 * checksum that can not be reproduced by an attacker. We store the MAC in the
88 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
89 * regular checksum of the ciphertext which can be used for scrubbing.
91 * OBJECT AUTHENTICATION:
92 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
93 * they contain some info that always needs to be readable. To prevent this
94 * data from being altered, we authenticate this data using SHA512-HMAC. This
95 * will produce a MAC (similar to the one produced via encryption) which can
96 * be used to verify the object was not modified. HMACs do not require key
97 * rotation or IVs, so we can keep up to the full 3 copies of authenticated
101 * ZIL blocks have their bp written to disk ahead of the associated data, so we
102 * cannot store the MAC there as we normally do. For these blocks the MAC is
103 * stored in the embedded checksum within the zil_chain_t header. The salt and
104 * IV are generated for the block on bp allocation instead of at encryption
105 * time. In addition, ZIL blocks have some pieces that must be left in plaintext
106 * for claiming even though all of the sensitive user data still needs to be
107 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
108 * pieces of the block need to be encrypted. All data that is not encrypted is
109 * authenticated using the AAD mechanisms that the supported encryption modes
110 * provide for. In order to preserve the semantics of the ZIL for encrypted
111 * datasets, the ZIL is not protected at the objset level as described below.
114 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
115 * in plaintext for scrubbing and claiming, but the bonus buffers might contain
116 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
117 * which which pieces of the block need to be encrypted. For more details about
118 * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
120 * OBJECT SET AUTHENTICATION:
121 * Up to this point, everything we have encrypted and authenticated has been
122 * at level 0 (or -2 for the ZIL). If we did not do any further work the
123 * on-disk format would be susceptible to attacks that deleted or rearrannged
124 * the order of level 0 blocks. Ideally, the cleanest solution would be to
125 * maintain a tree of authentication MACs going up the bp tree. However, this
126 * presents a problem for raw sends. Send files do not send information about
127 * indirect blocks so there would be no convenient way to transfer the MACs and
128 * they cannot be recalculated on the receive side without the master key which
129 * would defeat one of the purposes of raw sends in the first place. Instead,
130 * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
131 * from the level below. We also include some portable fields from blk_prop such
132 * as the lsize and compression algorithm to prevent the data from being
135 * At the objset level, we maintain 2 seperate 256 bit MACs in the
136 * objset_phys_t. The first one is "portable" and is the logical root of the
137 * MAC tree maintianed in the metadnode's bps. The second, is "local" and is
138 * used as the root MAC for the user accounting objects, which are also not
139 * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
140 * of the send file. The useraccounting code ensures that the useraccounting
141 * info is not present upon a receive, so the local MAC can simply be cleared
142 * out at that time. For more info about objset_phys_t authentication, see
143 * zio_crypt_do_objset_hmacs().
145 * CONSIDERATIONS FOR DEDUP:
146 * In order for dedup to work, blocks that we want to dedup with one another
147 * need to use the same IV and encryption key, so that they will have the same
148 * ciphertext. Normally, one should never reuse an IV with the same encryption
149 * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
150 * blocks. In this case, however, since we are using the same plaindata as
151 * well all that we end up with is a duplicate of the original ciphertext we
152 * already had. As a result, an attacker with read access to the raw disk will
153 * be able to tell which blocks are the same but this information is given away
154 * by dedup anyway. In order to get the same IVs and encryption keys for
155 * equivalent blocks of data we use an HMAC of the plaindata. We use an HMAC
156 * here so that a reproducible checksum of the plaindata is never available to
157 * the attacker. The HMAC key is kept alongside the master key, encrypted on
158 * disk. The first 64 bits of the HMAC are used in place of the random salt, and
159 * the next 96 bits are used as the IV. As a result of this mechanism, dedup
160 * will only work within a clone family since encrypted dedup requires use of
161 * the same master and HMAC keys.
165 * After encrypting many blocks with the same key we may start to run up
166 * against the theoretical limits of how much data can securely be encrypted
167 * with a single key using the supported encryption modes. The most obvious
168 * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
169 * the more IVs we generate (which both GCM and CCM modes strictly forbid).
170 * This risk actually grows surprisingly quickly over time according to the
171 * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
172 * generated n IVs with a cryptographically secure RNG, the approximate
173 * probability p(n) of a collision is given as:
175 * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
177 * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
179 * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
180 * we must not write more than 398,065,730 blocks with the same encryption key.
181 * Therefore, we rotate our keys after 400,000,000 blocks have been written by
182 * generating a new random 64 bit salt for our HKDF encryption key generation
185 #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000
186 #define ZFS_CURRENT_MAX_SALT_USES \
187 (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
188 unsigned long zfs_key_max_salt_uses
= ZFS_KEY_MAX_SALT_USES_DEFAULT
;
190 zio_crypt_info_t zio_crypt_table
[ZIO_CRYPT_FUNCTIONS
] = {
191 {"", ZC_TYPE_NONE
, 0, "inherit"},
192 {"", ZC_TYPE_NONE
, 0, "on"},
193 {"", ZC_TYPE_NONE
, 0, "off"},
194 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 16, "aes-128-ccm"},
195 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 24, "aes-192-ccm"},
196 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 32, "aes-256-ccm"},
197 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 16, "aes-128-gcm"},
198 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 24, "aes-192-gcm"},
199 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 32, "aes-256-gcm"}
203 zio_crypt_key_destroy(zio_crypt_key_t
*key
)
205 rw_destroy(&key
->zk_salt_lock
);
207 /* free crypto templates */
208 crypto_destroy_ctx_template(key
->zk_current_tmpl
);
209 crypto_destroy_ctx_template(key
->zk_hmac_tmpl
);
211 /* zero out sensitive data */
212 bzero(key
, sizeof (zio_crypt_key_t
));
216 zio_crypt_key_init(uint64_t crypt
, zio_crypt_key_t
*key
)
219 crypto_mechanism_t mech
;
223 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
225 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
226 bzero(key
, sizeof (zio_crypt_key_t
));
228 /* fill keydata buffers and salt with random data */
229 ret
= random_get_bytes((uint8_t *)&key
->zk_guid
, sizeof (uint64_t));
233 ret
= random_get_bytes(key
->zk_master_keydata
, keydata_len
);
237 ret
= random_get_bytes(key
->zk_hmac_keydata
, SHA512_HMAC_KEYLEN
);
241 ret
= random_get_bytes(key
->zk_salt
, ZIO_DATA_SALT_LEN
);
245 /* derive the current key from the master key */
246 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
247 key
->zk_salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
,
252 /* initialize keys for the ICP */
253 key
->zk_current_key
.ck_format
= CRYPTO_KEY_RAW
;
254 key
->zk_current_key
.ck_data
= key
->zk_current_keydata
;
255 key
->zk_current_key
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
257 key
->zk_hmac_key
.ck_format
= CRYPTO_KEY_RAW
;
258 key
->zk_hmac_key
.ck_data
= &key
->zk_hmac_key
;
259 key
->zk_hmac_key
.ck_length
= CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN
);
262 * Initialize the crypto templates. It's ok if this fails because
263 * this is just an optimization.
265 mech
.cm_type
= crypto_mech2id(zio_crypt_table
[crypt
].ci_mechname
);
266 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
267 &key
->zk_current_tmpl
, KM_SLEEP
);
268 if (ret
!= CRYPTO_SUCCESS
)
269 key
->zk_current_tmpl
= NULL
;
271 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
272 ret
= crypto_create_ctx_template(&mech
, &key
->zk_hmac_key
,
273 &key
->zk_hmac_tmpl
, KM_SLEEP
);
274 if (ret
!= CRYPTO_SUCCESS
)
275 key
->zk_hmac_tmpl
= NULL
;
277 key
->zk_crypt
= crypt
;
278 key
->zk_salt_count
= 0;
279 rw_init(&key
->zk_salt_lock
, NULL
, RW_DEFAULT
, NULL
);
284 zio_crypt_key_destroy(key
);
289 zio_crypt_key_change_salt(zio_crypt_key_t
*key
)
292 uint8_t salt
[ZIO_DATA_SALT_LEN
];
293 crypto_mechanism_t mech
;
294 uint_t keydata_len
= zio_crypt_table
[key
->zk_crypt
].ci_keylen
;
296 /* generate a new salt */
297 ret
= random_get_bytes(salt
, ZIO_DATA_SALT_LEN
);
301 rw_enter(&key
->zk_salt_lock
, RW_WRITER
);
303 /* someone beat us to the salt rotation, just unlock and return */
304 if (key
->zk_salt_count
< ZFS_CURRENT_MAX_SALT_USES
)
307 /* derive the current key from the master key and the new salt */
308 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
309 salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
, keydata_len
);
313 /* assign the salt and reset the usage count */
314 bcopy(salt
, key
->zk_salt
, ZIO_DATA_SALT_LEN
);
315 key
->zk_salt_count
= 0;
317 /* destroy the old context template and create the new one */
318 crypto_destroy_ctx_template(key
->zk_current_tmpl
);
319 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
320 &key
->zk_current_tmpl
, KM_SLEEP
);
321 if (ret
!= CRYPTO_SUCCESS
)
322 key
->zk_current_tmpl
= NULL
;
324 rw_exit(&key
->zk_salt_lock
);
329 rw_exit(&key
->zk_salt_lock
);
334 /* See comment above zfs_key_max_salt_uses definition for details */
336 zio_crypt_key_get_salt(zio_crypt_key_t
*key
, uint8_t *salt
)
339 boolean_t salt_change
;
341 rw_enter(&key
->zk_salt_lock
, RW_READER
);
343 bcopy(key
->zk_salt
, salt
, ZIO_DATA_SALT_LEN
);
344 salt_change
= (atomic_inc_64_nv(&key
->zk_salt_count
) >=
345 ZFS_CURRENT_MAX_SALT_USES
);
347 rw_exit(&key
->zk_salt_lock
);
350 ret
= zio_crypt_key_change_salt(key
);
362 * This function handles all encryption and decryption in zfs. When
363 * encrypting it expects puio to reference the plaintext and cuio to
364 * reference the cphertext. cuio must have enough space for the
365 * ciphertext + room for a MAC. datalen should be the length of the
366 * plaintext / ciphertext alone.
369 zio_do_crypt_uio(boolean_t encrypt
, uint64_t crypt
, crypto_key_t
*key
,
370 crypto_ctx_template_t tmpl
, uint8_t *ivbuf
, uint_t datalen
,
371 uio_t
*puio
, uio_t
*cuio
, uint8_t *authbuf
, uint_t auth_len
)
374 crypto_data_t plaindata
, cipherdata
;
375 CK_AES_CCM_PARAMS ccmp
;
376 CK_AES_GCM_PARAMS gcmp
;
377 crypto_mechanism_t mech
;
378 zio_crypt_info_t crypt_info
;
379 uint_t plain_full_len
, maclen
;
381 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
382 ASSERT3U(key
->ck_format
, ==, CRYPTO_KEY_RAW
);
384 /* lookup the encryption info */
385 crypt_info
= zio_crypt_table
[crypt
];
387 /* the mac will always be the last iovec_t in the cipher uio */
388 maclen
= cuio
->uio_iov
[cuio
->uio_iovcnt
- 1].iov_len
;
390 ASSERT(maclen
<= ZIO_DATA_MAC_LEN
);
392 /* setup encryption mechanism (same as crypt) */
393 mech
.cm_type
= crypto_mech2id(crypt_info
.ci_mechname
);
396 * Strangely, the ICP requires that plain_full_len must include
397 * the MAC length when decrypting, even though the UIO does not
398 * need to have the extra space allocated.
401 plain_full_len
= datalen
;
403 plain_full_len
= datalen
+ maclen
;
407 * setup encryption params (currently only AES CCM and AES GCM
410 if (crypt_info
.ci_crypt_type
== ZC_TYPE_CCM
) {
411 ccmp
.ulNonceSize
= ZIO_DATA_IV_LEN
;
412 ccmp
.ulAuthDataSize
= auth_len
;
413 ccmp
.authData
= authbuf
;
414 ccmp
.ulMACSize
= maclen
;
416 ccmp
.ulDataSize
= plain_full_len
;
418 mech
.cm_param
= (char *)(&ccmp
);
419 mech
.cm_param_len
= sizeof (CK_AES_CCM_PARAMS
);
421 gcmp
.ulIvLen
= ZIO_DATA_IV_LEN
;
422 gcmp
.ulIvBits
= CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN
);
423 gcmp
.ulAADLen
= auth_len
;
425 gcmp
.ulTagBits
= CRYPTO_BYTES2BITS(maclen
);
428 mech
.cm_param
= (char *)(&gcmp
);
429 mech
.cm_param_len
= sizeof (CK_AES_GCM_PARAMS
);
432 /* populate the cipher and plain data structs. */
433 plaindata
.cd_format
= CRYPTO_DATA_UIO
;
434 plaindata
.cd_offset
= 0;
435 plaindata
.cd_uio
= puio
;
436 plaindata
.cd_miscdata
= NULL
;
437 plaindata
.cd_length
= plain_full_len
;
439 cipherdata
.cd_format
= CRYPTO_DATA_UIO
;
440 cipherdata
.cd_offset
= 0;
441 cipherdata
.cd_uio
= cuio
;
442 cipherdata
.cd_miscdata
= NULL
;
443 cipherdata
.cd_length
= datalen
+ maclen
;
445 /* perform the actual encryption */
447 ret
= crypto_encrypt(&mech
, &plaindata
, key
, tmpl
, &cipherdata
,
449 if (ret
!= CRYPTO_SUCCESS
) {
450 ret
= SET_ERROR(EIO
);
454 ret
= crypto_decrypt(&mech
, &cipherdata
, key
, tmpl
, &plaindata
,
456 if (ret
!= CRYPTO_SUCCESS
) {
457 ASSERT3U(ret
, ==, CRYPTO_INVALID_MAC
);
458 ret
= SET_ERROR(ECKSUM
);
470 zio_crypt_key_wrap(crypto_key_t
*cwkey
, zio_crypt_key_t
*key
, uint8_t *iv
,
471 uint8_t *mac
, uint8_t *keydata_out
, uint8_t *hmac_keydata_out
)
475 iovec_t plain_iovecs
[2], cipher_iovecs
[3];
476 uint64_t crypt
= key
->zk_crypt
;
477 uint64_t le_guid
= LE_64(key
->zk_guid
);
478 uint_t enc_len
, keydata_len
;
480 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
481 ASSERT3U(cwkey
->ck_format
, ==, CRYPTO_KEY_RAW
);
483 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
485 /* generate iv for wrapping the master and hmac key */
486 ret
= random_get_pseudo_bytes(iv
, WRAPPING_IV_LEN
);
490 /* initialize uio_ts */
491 plain_iovecs
[0].iov_base
= key
->zk_master_keydata
;
492 plain_iovecs
[0].iov_len
= keydata_len
;
493 plain_iovecs
[1].iov_base
= key
->zk_hmac_keydata
;
494 plain_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
496 cipher_iovecs
[0].iov_base
= keydata_out
;
497 cipher_iovecs
[0].iov_len
= keydata_len
;
498 cipher_iovecs
[1].iov_base
= hmac_keydata_out
;
499 cipher_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
500 cipher_iovecs
[2].iov_base
= mac
;
501 cipher_iovecs
[2].iov_len
= WRAPPING_MAC_LEN
;
503 enc_len
= zio_crypt_table
[crypt
].ci_keylen
+ SHA512_HMAC_KEYLEN
;
504 puio
.uio_iov
= plain_iovecs
;
506 puio
.uio_segflg
= UIO_SYSSPACE
;
507 cuio
.uio_iov
= cipher_iovecs
;
509 cuio
.uio_segflg
= UIO_SYSSPACE
;
511 /* encrypt the keys and store the resulting ciphertext and mac */
512 ret
= zio_do_crypt_uio(B_TRUE
, crypt
, cwkey
, NULL
, iv
, enc_len
,
513 &puio
, &cuio
, (uint8_t *)&le_guid
, sizeof (uint64_t));
524 zio_crypt_key_unwrap(crypto_key_t
*cwkey
, uint64_t crypt
, uint64_t guid
,
525 uint8_t *keydata
, uint8_t *hmac_keydata
, uint8_t *iv
, uint8_t *mac
,
526 zio_crypt_key_t
*key
)
529 crypto_mechanism_t mech
;
531 iovec_t plain_iovecs
[2], cipher_iovecs
[3];
532 uint_t enc_len
, keydata_len
;
533 uint64_t le_guid
= LE_64(guid
);
535 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
536 ASSERT3U(cwkey
->ck_format
, ==, CRYPTO_KEY_RAW
);
538 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
540 /* initialize uio_ts */
541 plain_iovecs
[0].iov_base
= key
->zk_master_keydata
;
542 plain_iovecs
[0].iov_len
= keydata_len
;
543 plain_iovecs
[1].iov_base
= key
->zk_hmac_keydata
;
544 plain_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
546 cipher_iovecs
[0].iov_base
= keydata
;
547 cipher_iovecs
[0].iov_len
= keydata_len
;
548 cipher_iovecs
[1].iov_base
= hmac_keydata
;
549 cipher_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
550 cipher_iovecs
[2].iov_base
= mac
;
551 cipher_iovecs
[2].iov_len
= WRAPPING_MAC_LEN
;
553 enc_len
= keydata_len
+ SHA512_HMAC_KEYLEN
;
554 puio
.uio_iov
= plain_iovecs
;
555 puio
.uio_segflg
= UIO_SYSSPACE
;
557 cuio
.uio_iov
= cipher_iovecs
;
559 cuio
.uio_segflg
= UIO_SYSSPACE
;
561 /* decrypt the keys and store the result in the output buffers */
562 ret
= zio_do_crypt_uio(B_FALSE
, crypt
, cwkey
, NULL
, iv
, enc_len
,
563 &puio
, &cuio
, (uint8_t *)&le_guid
, sizeof (uint64_t));
567 /* generate a fresh salt */
568 ret
= random_get_bytes(key
->zk_salt
, ZIO_DATA_SALT_LEN
);
572 /* derive the current key from the master key */
573 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
574 key
->zk_salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
,
579 /* initialize keys for ICP */
580 key
->zk_current_key
.ck_format
= CRYPTO_KEY_RAW
;
581 key
->zk_current_key
.ck_data
= key
->zk_current_keydata
;
582 key
->zk_current_key
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
584 key
->zk_hmac_key
.ck_format
= CRYPTO_KEY_RAW
;
585 key
->zk_hmac_key
.ck_data
= key
->zk_hmac_keydata
;
586 key
->zk_hmac_key
.ck_length
= CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN
);
589 * Initialize the crypto templates. It's ok if this fails because
590 * this is just an optimization.
592 mech
.cm_type
= crypto_mech2id(zio_crypt_table
[crypt
].ci_mechname
);
593 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
594 &key
->zk_current_tmpl
, KM_SLEEP
);
595 if (ret
!= CRYPTO_SUCCESS
)
596 key
->zk_current_tmpl
= NULL
;
598 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
599 ret
= crypto_create_ctx_template(&mech
, &key
->zk_hmac_key
,
600 &key
->zk_hmac_tmpl
, KM_SLEEP
);
601 if (ret
!= CRYPTO_SUCCESS
)
602 key
->zk_hmac_tmpl
= NULL
;
604 key
->zk_crypt
= crypt
;
606 key
->zk_salt_count
= 0;
607 rw_init(&key
->zk_salt_lock
, NULL
, RW_DEFAULT
, NULL
);
612 zio_crypt_key_destroy(key
);
617 zio_crypt_generate_iv(uint8_t *ivbuf
)
621 /* randomly generate the IV */
622 ret
= random_get_pseudo_bytes(ivbuf
, ZIO_DATA_IV_LEN
);
629 bzero(ivbuf
, ZIO_DATA_IV_LEN
);
634 zio_crypt_do_hmac(zio_crypt_key_t
*key
, uint8_t *data
, uint_t datalen
,
635 uint8_t *digestbuf
, uint_t digestlen
)
638 crypto_mechanism_t mech
;
639 crypto_data_t in_data
, digest_data
;
640 uint8_t raw_digestbuf
[SHA512_DIGEST_LENGTH
];
642 ASSERT3U(digestlen
, <=, SHA512_DIGEST_LENGTH
);
644 /* initialize sha512-hmac mechanism and crypto data */
645 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
646 mech
.cm_param
= NULL
;
647 mech
.cm_param_len
= 0;
649 /* initialize the crypto data */
650 in_data
.cd_format
= CRYPTO_DATA_RAW
;
651 in_data
.cd_offset
= 0;
652 in_data
.cd_length
= datalen
;
653 in_data
.cd_raw
.iov_base
= (char *)data
;
654 in_data
.cd_raw
.iov_len
= in_data
.cd_length
;
656 digest_data
.cd_format
= CRYPTO_DATA_RAW
;
657 digest_data
.cd_offset
= 0;
658 digest_data
.cd_length
= SHA512_DIGEST_LENGTH
;
659 digest_data
.cd_raw
.iov_base
= (char *)raw_digestbuf
;
660 digest_data
.cd_raw
.iov_len
= digest_data
.cd_length
;
662 /* generate the hmac */
663 ret
= crypto_mac(&mech
, &in_data
, &key
->zk_hmac_key
, key
->zk_hmac_tmpl
,
665 if (ret
!= CRYPTO_SUCCESS
) {
666 ret
= SET_ERROR(EIO
);
670 bcopy(raw_digestbuf
, digestbuf
, digestlen
);
675 bzero(digestbuf
, digestlen
);
680 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t
*key
, uint8_t *data
,
681 uint_t datalen
, uint8_t *ivbuf
, uint8_t *salt
)
684 uint8_t digestbuf
[SHA512_DIGEST_LENGTH
];
686 ret
= zio_crypt_do_hmac(key
, data
, datalen
,
687 digestbuf
, SHA512_DIGEST_LENGTH
);
691 bcopy(digestbuf
, salt
, ZIO_DATA_SALT_LEN
);
692 bcopy(digestbuf
+ ZIO_DATA_SALT_LEN
, ivbuf
, ZIO_DATA_IV_LEN
);
698 * The following functions are used to encode and decode encryption parameters
699 * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
700 * byte strings, which normally means that these strings would not need to deal
701 * with byteswapping at all. However, both blkptr_t and zil_header_t may be
702 * byteswapped by lower layers and so we must "undo" that byteswap here upon
706 zio_crypt_encode_params_bp(blkptr_t
*bp
, uint8_t *salt
, uint8_t *iv
)
710 ASSERT(BP_IS_ENCRYPTED(bp
));
712 bcopy(salt
, &bp
->blk_dva
[2].dva_word
[0], sizeof (uint64_t));
713 bcopy(iv
, &bp
->blk_dva
[2].dva_word
[1], sizeof (uint64_t));
714 bcopy(iv
+ sizeof (uint64_t), &val32
, sizeof (uint32_t));
715 BP_SET_IV2(bp
, val32
);
719 zio_crypt_decode_params_bp(const blkptr_t
*bp
, uint8_t *salt
, uint8_t *iv
)
724 ASSERT(BP_IS_PROTECTED(bp
));
726 /* for convenience, so callers don't need to check */
727 if (BP_IS_AUTHENTICATED(bp
)) {
728 bzero(salt
, ZIO_DATA_SALT_LEN
);
729 bzero(iv
, ZIO_DATA_IV_LEN
);
733 if (!BP_SHOULD_BYTESWAP(bp
)) {
734 bcopy(&bp
->blk_dva
[2].dva_word
[0], salt
, sizeof (uint64_t));
735 bcopy(&bp
->blk_dva
[2].dva_word
[1], iv
, sizeof (uint64_t));
737 val32
= (uint32_t)BP_GET_IV2(bp
);
738 bcopy(&val32
, iv
+ sizeof (uint64_t), sizeof (uint32_t));
740 val64
= BSWAP_64(bp
->blk_dva
[2].dva_word
[0]);
741 bcopy(&val64
, salt
, sizeof (uint64_t));
743 val64
= BSWAP_64(bp
->blk_dva
[2].dva_word
[1]);
744 bcopy(&val64
, iv
, sizeof (uint64_t));
746 val32
= BSWAP_32((uint32_t)BP_GET_IV2(bp
));
747 bcopy(&val32
, iv
+ sizeof (uint64_t), sizeof (uint32_t));
752 zio_crypt_encode_mac_bp(blkptr_t
*bp
, uint8_t *mac
)
754 ASSERT(BP_USES_CRYPT(bp
));
755 ASSERT3U(BP_GET_TYPE(bp
), !=, DMU_OT_OBJSET
);
757 bcopy(mac
, &bp
->blk_cksum
.zc_word
[2], sizeof (uint64_t));
758 bcopy(mac
+ sizeof (uint64_t), &bp
->blk_cksum
.zc_word
[3],
763 zio_crypt_decode_mac_bp(const blkptr_t
*bp
, uint8_t *mac
)
767 ASSERT(BP_USES_CRYPT(bp
) || BP_IS_HOLE(bp
));
769 /* for convenience, so callers don't need to check */
770 if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
771 bzero(mac
, ZIO_DATA_MAC_LEN
);
775 if (!BP_SHOULD_BYTESWAP(bp
)) {
776 bcopy(&bp
->blk_cksum
.zc_word
[2], mac
, sizeof (uint64_t));
777 bcopy(&bp
->blk_cksum
.zc_word
[3], mac
+ sizeof (uint64_t),
780 val64
= BSWAP_64(bp
->blk_cksum
.zc_word
[2]);
781 bcopy(&val64
, mac
, sizeof (uint64_t));
783 val64
= BSWAP_64(bp
->blk_cksum
.zc_word
[3]);
784 bcopy(&val64
, mac
+ sizeof (uint64_t), sizeof (uint64_t));
789 zio_crypt_encode_mac_zil(void *data
, uint8_t *mac
)
791 zil_chain_t
*zilc
= data
;
793 bcopy(mac
, &zilc
->zc_eck
.zec_cksum
.zc_word
[2], sizeof (uint64_t));
794 bcopy(mac
+ sizeof (uint64_t), &zilc
->zc_eck
.zec_cksum
.zc_word
[3],
799 zio_crypt_decode_mac_zil(const void *data
, uint8_t *mac
)
802 * The ZIL MAC is embedded in the block it protects, which will
803 * not have been byteswapped by the time this function has been called.
804 * As a result, we don't need to worry about byteswapping the MAC.
806 const zil_chain_t
*zilc
= data
;
808 bcopy(&zilc
->zc_eck
.zec_cksum
.zc_word
[2], mac
, sizeof (uint64_t));
809 bcopy(&zilc
->zc_eck
.zec_cksum
.zc_word
[3], mac
+ sizeof (uint64_t),
814 * This routine takes a block of dnodes (src_abd) and copies only the bonus
815 * buffers to the same offsets in the dst buffer. datalen should be the size
816 * of both the src_abd and the dst buffer (not just the length of the bonus
820 zio_crypt_copy_dnode_bonus(abd_t
*src_abd
, uint8_t *dst
, uint_t datalen
)
822 uint_t i
, max_dnp
= datalen
>> DNODE_SHIFT
;
824 dnode_phys_t
*dnp
, *sdnp
, *ddnp
;
826 src
= abd_borrow_buf_copy(src_abd
, datalen
);
828 sdnp
= (dnode_phys_t
*)src
;
829 ddnp
= (dnode_phys_t
*)dst
;
831 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
833 if (dnp
->dn_type
!= DMU_OT_NONE
&&
834 DMU_OT_IS_ENCRYPTED(dnp
->dn_bonustype
) &&
835 dnp
->dn_bonuslen
!= 0) {
836 bcopy(DN_BONUS(dnp
), DN_BONUS(&ddnp
[i
]),
837 DN_MAX_BONUS_LEN(dnp
));
841 abd_return_buf(src_abd
, src
, datalen
);
845 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t
*bp
)
848 BP_SET_CHECKSUM(bp
, 0);
851 * psize cannot be set to zero or it will trigger asserts, but the
852 * value doesn't really matter as long as it is constant.
854 BP_SET_PSIZE(bp
, SPA_MINBLOCKSIZE
);
858 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx
, boolean_t should_bswap
,
864 blkptr_t tmpbp
= *bp
;
865 uint8_t mac
[ZIO_DATA_MAC_LEN
];
867 cd
.cd_format
= CRYPTO_DATA_RAW
;
871 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
873 ASSERT(BP_USES_CRYPT(&tmpbp
) || BP_IS_HOLE(&tmpbp
));
874 ASSERT0(BP_IS_EMBEDDED(&tmpbp
));
875 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp
);
877 le_blkprop
= (ZFS_HOST_BYTEORDER
) ?
878 tmpbp
.blk_prop
: BSWAP_64(tmpbp
.blk_prop
);
880 cd
.cd_length
= sizeof (uint64_t);
881 cd
.cd_raw
.iov_base
= (char *)&le_blkprop
;
882 cd
.cd_raw
.iov_len
= cd
.cd_length
;
884 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
885 if (ret
!= CRYPTO_SUCCESS
) {
886 ret
= SET_ERROR(EIO
);
890 zio_crypt_decode_mac_bp(&tmpbp
, mac
);
891 cd
.cd_length
= ZIO_DATA_MAC_LEN
;
892 cd
.cd_raw
.iov_base
= (char *)mac
;
893 cd
.cd_raw
.iov_len
= cd
.cd_length
;
895 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
896 if (ret
!= CRYPTO_SUCCESS
) {
897 ret
= SET_ERROR(EIO
);
908 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX
*ctx
, boolean_t should_bswap
,
911 blkptr_t tmpbp
= *bp
;
912 uint8_t mac
[ZIO_DATA_MAC_LEN
];
915 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
917 ASSERT(BP_USES_CRYPT(&tmpbp
) || BP_IS_HOLE(&tmpbp
));
918 ASSERT0(BP_IS_EMBEDDED(&tmpbp
));
919 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp
);
920 zio_crypt_decode_mac_bp(&tmpbp
, mac
);
923 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
925 SHA2Update(ctx
, &tmpbp
.blk_prop
, sizeof (uint64_t));
926 SHA2Update(ctx
, mac
, ZIO_DATA_MAC_LEN
);
930 zio_crypt_bp_do_aad_updates(uint8_t **aadp
, uint_t
*aad_len
,
931 boolean_t should_bswap
, blkptr_t
*bp
)
934 blkptr_t tmpbp
= *bp
;
935 uint8_t mac
[ZIO_DATA_MAC_LEN
];
938 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
940 ASSERT(BP_USES_CRYPT(&tmpbp
) || BP_IS_HOLE(&tmpbp
));
941 ASSERT0(BP_IS_EMBEDDED(&tmpbp
));
942 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp
);
943 zio_crypt_decode_mac_bp(&tmpbp
, mac
);
946 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
948 crypt_len
= sizeof (uint64_t);
949 bcopy(&tmpbp
.blk_prop
, *aadp
, crypt_len
);
951 *aad_len
+= crypt_len
;
953 crypt_len
= ZIO_DATA_MAC_LEN
;
954 bcopy(mac
, *aadp
, crypt_len
);
956 *aad_len
+= crypt_len
;
960 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx
, boolean_t should_bswap
,
965 boolean_t le_bswap
= (should_bswap
== ZFS_HOST_BYTEORDER
);
967 uint8_t tmp_dncore
[offsetof(dnode_phys_t
, dn_blkptr
)];
969 cd
.cd_format
= CRYPTO_DATA_RAW
;
972 /* authenticate the core dnode (masking out non-portable bits) */
973 bcopy(dnp
, tmp_dncore
, sizeof (tmp_dncore
));
974 adnp
= (dnode_phys_t
*)tmp_dncore
;
976 adnp
->dn_datablkszsec
= BSWAP_16(adnp
->dn_datablkszsec
);
977 adnp
->dn_bonuslen
= BSWAP_16(adnp
->dn_bonuslen
);
978 adnp
->dn_maxblkid
= BSWAP_64(adnp
->dn_maxblkid
);
979 adnp
->dn_used
= BSWAP_64(adnp
->dn_used
);
981 adnp
->dn_flags
&= DNODE_CRYPT_PORTABLE_FLAGS_MASK
;
984 cd
.cd_length
= sizeof (tmp_dncore
);
985 cd
.cd_raw
.iov_base
= (char *)adnp
;
986 cd
.cd_raw
.iov_len
= cd
.cd_length
;
988 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
989 if (ret
!= CRYPTO_SUCCESS
) {
990 ret
= SET_ERROR(EIO
);
994 for (i
= 0; i
< dnp
->dn_nblkptr
; i
++) {
995 ret
= zio_crypt_bp_do_hmac_updates(ctx
,
996 should_bswap
, &dnp
->dn_blkptr
[i
]);
1001 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1002 ret
= zio_crypt_bp_do_hmac_updates(ctx
,
1003 should_bswap
, DN_SPILL_BLKPTR(dnp
));
1015 * objset_phys_t blocks introduce a number of exceptions to the normal
1016 * authentication process. objset_phys_t's contain 2 seperate HMACS for
1017 * protecting the integrity of their data. The portable_mac protects the
1018 * the metadnode. This MAC can be sent with a raw send and protects against
1019 * reordering of data within the metadnode. The local_mac protects the user
1020 * accounting objects which are not sent from one system to another.
1022 * In addition, objset blocks are the only blocks that can be modified and
1023 * written to disk without the key loaded under certain circumstances. During
1024 * zil_claim() we need to be able to update the zil_header_t to complete
1025 * claiming log blocks and during raw receives we need to write out the
1026 * portable_mac from the send file. Both of these actions are possible
1027 * because these fields are not protected by either MAC so neither one will
1028 * need to modify the MACs without the key. However, when the modified blocks
1029 * are written out they will be byteswapped into the host machine's native
1030 * endianness which will modify fields protected by the MAC. As a result, MAC
1031 * calculation for objset blocks works slightly differently from other block
1032 * types. Where other block types MAC the data in whatever endianness is
1033 * written to disk, objset blocks always MAC little endian version of their
1034 * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1035 * and le_bswap indicates whether a byteswap is needed to get this block
1036 * into little endian format.
1039 zio_crypt_do_objset_hmacs(zio_crypt_key_t
*key
, void *data
, uint_t datalen
,
1040 boolean_t should_bswap
, uint8_t *portable_mac
, uint8_t *local_mac
)
1043 crypto_mechanism_t mech
;
1044 crypto_context_t ctx
;
1046 objset_phys_t
*osp
= data
;
1048 boolean_t le_bswap
= (should_bswap
== ZFS_HOST_BYTEORDER
);
1049 uint8_t raw_portable_mac
[SHA512_DIGEST_LENGTH
];
1050 uint8_t raw_local_mac
[SHA512_DIGEST_LENGTH
];
1052 /* initialize HMAC mechanism */
1053 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
1054 mech
.cm_param
= NULL
;
1055 mech
.cm_param_len
= 0;
1057 cd
.cd_format
= CRYPTO_DATA_RAW
;
1060 /* calculate the portable MAC from the portable fields and metadnode */
1061 ret
= crypto_mac_init(&mech
, &key
->zk_hmac_key
, NULL
, &ctx
, NULL
);
1062 if (ret
!= CRYPTO_SUCCESS
) {
1063 ret
= SET_ERROR(EIO
);
1067 /* add in the os_type */
1068 intval
= (le_bswap
) ? osp
->os_type
: BSWAP_64(osp
->os_type
);
1069 cd
.cd_length
= sizeof (uint64_t);
1070 cd
.cd_raw
.iov_base
= (char *)&intval
;
1071 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1073 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1074 if (ret
!= CRYPTO_SUCCESS
) {
1075 ret
= SET_ERROR(EIO
);
1079 /* add in the portable os_flags */
1080 intval
= osp
->os_flags
;
1082 intval
= BSWAP_64(intval
);
1083 intval
&= OBJSET_CRYPT_PORTABLE_FLAGS_MASK
;
1084 if (!ZFS_HOST_BYTEORDER
)
1085 intval
= BSWAP_64(intval
);
1087 cd
.cd_length
= sizeof (uint64_t);
1088 cd
.cd_raw
.iov_base
= (char *)&intval
;
1089 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1091 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1092 if (ret
!= CRYPTO_SUCCESS
) {
1093 ret
= SET_ERROR(EIO
);
1097 /* add in fields from the metadnode */
1098 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, should_bswap
,
1099 &osp
->os_meta_dnode
);
1103 /* store the final digest in a temporary buffer and copy what we need */
1104 cd
.cd_length
= SHA512_DIGEST_LENGTH
;
1105 cd
.cd_raw
.iov_base
= (char *)raw_portable_mac
;
1106 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1108 ret
= crypto_mac_final(ctx
, &cd
, NULL
);
1109 if (ret
!= CRYPTO_SUCCESS
) {
1110 ret
= SET_ERROR(EIO
);
1114 bcopy(raw_portable_mac
, portable_mac
, ZIO_OBJSET_MAC_LEN
);
1117 * The local MAC protects the user and group accounting. If these
1118 * objects are not present, the local MAC is zeroed out.
1120 if (osp
->os_userused_dnode
.dn_type
== DMU_OT_NONE
&&
1121 osp
->os_groupused_dnode
.dn_type
== DMU_OT_NONE
) {
1122 bzero(local_mac
, ZIO_OBJSET_MAC_LEN
);
1126 /* calculate the local MAC from the userused and groupused dnodes */
1127 ret
= crypto_mac_init(&mech
, &key
->zk_hmac_key
, NULL
, &ctx
, NULL
);
1128 if (ret
!= CRYPTO_SUCCESS
) {
1129 ret
= SET_ERROR(EIO
);
1133 /* add in the non-portable os_flags */
1134 intval
= osp
->os_flags
;
1136 intval
= BSWAP_64(intval
);
1137 intval
&= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK
;
1138 if (!ZFS_HOST_BYTEORDER
)
1139 intval
= BSWAP_64(intval
);
1141 cd
.cd_length
= sizeof (uint64_t);
1142 cd
.cd_raw
.iov_base
= (char *)&intval
;
1143 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1145 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1146 if (ret
!= CRYPTO_SUCCESS
) {
1147 ret
= SET_ERROR(EIO
);
1151 /* add in fields from the user accounting dnodes */
1152 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, should_bswap
,
1153 &osp
->os_userused_dnode
);
1157 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, should_bswap
,
1158 &osp
->os_groupused_dnode
);
1162 /* store the final digest in a temporary buffer and copy what we need */
1163 cd
.cd_length
= SHA512_DIGEST_LENGTH
;
1164 cd
.cd_raw
.iov_base
= (char *)raw_local_mac
;
1165 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1167 ret
= crypto_mac_final(ctx
, &cd
, NULL
);
1168 if (ret
!= CRYPTO_SUCCESS
) {
1169 ret
= SET_ERROR(EIO
);
1173 bcopy(raw_local_mac
, local_mac
, ZIO_OBJSET_MAC_LEN
);
1178 bzero(portable_mac
, ZIO_OBJSET_MAC_LEN
);
1179 bzero(local_mac
, ZIO_OBJSET_MAC_LEN
);
1184 zio_crypt_destroy_uio(uio_t
*uio
)
1187 kmem_free(uio
->uio_iov
, uio
->uio_iovcnt
* sizeof (iovec_t
));
1191 * This function parses an uncompressed indirect block and returns a checksum
1192 * of all the portable fields from all of the contained bps. The portable
1193 * fields are the MAC and all of the fields from blk_prop except for the dedup,
1194 * checksum, and psize bits. For an explanation of the purpose of this, see
1195 * the comment block on object set authentication.
1198 zio_crypt_do_indirect_mac_checksum(boolean_t generate
, void *buf
,
1199 uint_t datalen
, boolean_t byteswap
, uint8_t *cksum
)
1202 int i
, epb
= datalen
>> SPA_BLKPTRSHIFT
;
1204 uint8_t digestbuf
[SHA512_DIGEST_LENGTH
];
1206 /* checksum all of the MACs from the layer below */
1207 SHA2Init(SHA512
, &ctx
);
1208 for (i
= 0, bp
= buf
; i
< epb
; i
++, bp
++) {
1209 zio_crypt_bp_do_indrect_checksum_updates(&ctx
, byteswap
, bp
);
1211 SHA2Final(digestbuf
, &ctx
);
1214 bcopy(digestbuf
, cksum
, ZIO_DATA_MAC_LEN
);
1218 if (bcmp(digestbuf
, cksum
, ZIO_DATA_MAC_LEN
) != 0)
1219 return (SET_ERROR(ECKSUM
));
1225 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate
, abd_t
*abd
,
1226 uint_t datalen
, boolean_t byteswap
, uint8_t *cksum
)
1232 buf
= abd_borrow_buf_copy(abd
, datalen
);
1233 ret
= zio_crypt_do_indirect_mac_checksum(generate
, buf
, datalen
,
1235 abd_return_buf(abd
, buf
, datalen
);
1241 * Special case handling routine for encrypting / decrypting ZIL blocks.
1242 * We do not check for the older ZIL chain because the encryption feature
1243 * was not available before the newer ZIL chain was introduced. The goal
1244 * here is to encrypt everything except the blkptr_t of a lr_write_t and
1245 * the zil_chain_t header. Everything that is not encrypted is authenticated.
1248 zio_crypt_init_uios_zil(boolean_t encrypt
, uint8_t *plainbuf
,
1249 uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
, uio_t
*puio
,
1250 uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
, uint_t
*auth_len
,
1251 boolean_t
*no_crypt
)
1255 uint_t nr_src
, nr_dst
, lr_len
, crypt_len
;
1256 uint_t aad_len
= 0, nr_iovecs
= 0, total_len
= 0;
1257 iovec_t
*src_iovecs
= NULL
, *dst_iovecs
= NULL
;
1258 uint8_t *src
, *dst
, *slrp
, *dlrp
, *blkend
, *aadp
;
1261 uint8_t *aadbuf
= zio_buf_alloc(datalen
);
1263 /* cipherbuf always needs an extra iovec for the MAC */
1276 /* find the start and end record of the log block */
1277 zilc
= (zil_chain_t
*)src
;
1278 slrp
= src
+ sizeof (zil_chain_t
);
1280 blkend
= src
+ ((byteswap
) ? BSWAP_64(zilc
->zc_nused
) : zilc
->zc_nused
);
1282 /* calculate the number of encrypted iovecs we will need */
1283 for (; slrp
< blkend
; slrp
+= lr_len
) {
1287 txtype
= lr
->lrc_txtype
;
1288 lr_len
= lr
->lrc_reclen
;
1290 txtype
= BSWAP_64(lr
->lrc_txtype
);
1291 lr_len
= BSWAP_64(lr
->lrc_reclen
);
1295 if (txtype
== TX_WRITE
&& lr_len
!= sizeof (lr_write_t
))
1299 nr_src
+= nr_iovecs
;
1300 nr_dst
+= nr_iovecs
;
1302 /* allocate the iovec arrays */
1304 src_iovecs
= kmem_alloc(nr_src
* sizeof (iovec_t
), KM_SLEEP
);
1305 if (src_iovecs
== NULL
) {
1306 ret
= SET_ERROR(ENOMEM
);
1312 dst_iovecs
= kmem_alloc(nr_dst
* sizeof (iovec_t
), KM_SLEEP
);
1313 if (dst_iovecs
== NULL
) {
1314 ret
= SET_ERROR(ENOMEM
);
1320 * Copy the plain zil header over and authenticate everything except
1321 * the checksum that will store our MAC. If we are writing the data
1322 * the embedded checksum will not have been calculated yet, so we don't
1323 * authenticate that.
1325 bcopy(src
, dst
, sizeof (zil_chain_t
));
1326 bcopy(src
, aadp
, sizeof (zil_chain_t
) - sizeof (zio_eck_t
));
1327 aadp
+= sizeof (zil_chain_t
) - sizeof (zio_eck_t
);
1328 aad_len
+= sizeof (zil_chain_t
) - sizeof (zio_eck_t
);
1330 /* loop over records again, filling in iovecs */
1332 slrp
= src
+ sizeof (zil_chain_t
);
1333 dlrp
= dst
+ sizeof (zil_chain_t
);
1335 for (; slrp
< blkend
; slrp
+= lr_len
, dlrp
+= lr_len
) {
1339 txtype
= lr
->lrc_txtype
;
1340 lr_len
= lr
->lrc_reclen
;
1342 txtype
= BSWAP_64(lr
->lrc_txtype
);
1343 lr_len
= BSWAP_64(lr
->lrc_reclen
);
1346 /* copy the common lr_t */
1347 bcopy(slrp
, dlrp
, sizeof (lr_t
));
1348 bcopy(slrp
, aadp
, sizeof (lr_t
));
1349 aadp
+= sizeof (lr_t
);
1350 aad_len
+= sizeof (lr_t
);
1352 ASSERT3P(src_iovecs
, !=, NULL
);
1353 ASSERT3P(dst_iovecs
, !=, NULL
);
1356 * If this is a TX_WRITE record we want to encrypt everything
1357 * except the bp if exists. If the bp does exist we want to
1360 if (txtype
== TX_WRITE
) {
1361 crypt_len
= sizeof (lr_write_t
) -
1362 sizeof (lr_t
) - sizeof (blkptr_t
);
1363 src_iovecs
[nr_iovecs
].iov_base
= slrp
+ sizeof (lr_t
);
1364 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1365 dst_iovecs
[nr_iovecs
].iov_base
= dlrp
+ sizeof (lr_t
);
1366 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1368 /* copy the bp now since it will not be encrypted */
1369 bcopy(slrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1370 dlrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1372 bcopy(slrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1373 aadp
, sizeof (blkptr_t
));
1374 aadp
+= sizeof (blkptr_t
);
1375 aad_len
+= sizeof (blkptr_t
);
1377 total_len
+= crypt_len
;
1379 if (lr_len
!= sizeof (lr_write_t
)) {
1380 crypt_len
= lr_len
- sizeof (lr_write_t
);
1381 src_iovecs
[nr_iovecs
].iov_base
=
1382 slrp
+ sizeof (lr_write_t
);
1383 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1384 dst_iovecs
[nr_iovecs
].iov_base
=
1385 dlrp
+ sizeof (lr_write_t
);
1386 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1388 total_len
+= crypt_len
;
1391 crypt_len
= lr_len
- sizeof (lr_t
);
1392 src_iovecs
[nr_iovecs
].iov_base
= slrp
+ sizeof (lr_t
);
1393 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1394 dst_iovecs
[nr_iovecs
].iov_base
= dlrp
+ sizeof (lr_t
);
1395 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1397 total_len
+= crypt_len
;
1401 *no_crypt
= (nr_iovecs
== 0);
1402 *enc_len
= total_len
;
1404 *auth_len
= aad_len
;
1407 puio
->uio_iov
= src_iovecs
;
1408 puio
->uio_iovcnt
= nr_src
;
1409 cuio
->uio_iov
= dst_iovecs
;
1410 cuio
->uio_iovcnt
= nr_dst
;
1412 puio
->uio_iov
= dst_iovecs
;
1413 puio
->uio_iovcnt
= nr_dst
;
1414 cuio
->uio_iov
= src_iovecs
;
1415 cuio
->uio_iovcnt
= nr_src
;
1421 zio_buf_free(aadbuf
, datalen
);
1422 if (src_iovecs
!= NULL
)
1423 kmem_free(src_iovecs
, nr_src
* sizeof (iovec_t
));
1424 if (dst_iovecs
!= NULL
)
1425 kmem_free(dst_iovecs
, nr_dst
* sizeof (iovec_t
));
1430 *no_crypt
= B_FALSE
;
1431 puio
->uio_iov
= NULL
;
1432 puio
->uio_iovcnt
= 0;
1433 cuio
->uio_iov
= NULL
;
1434 cuio
->uio_iovcnt
= 0;
1439 * Special case handling routine for encrypting / decrypting dnode blocks.
1442 zio_crypt_init_uios_dnode(boolean_t encrypt
, uint8_t *plainbuf
,
1443 uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
, uio_t
*puio
,
1444 uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
, uint_t
*auth_len
,
1445 boolean_t
*no_crypt
)
1448 uint_t nr_src
, nr_dst
, crypt_len
;
1449 uint_t aad_len
= 0, nr_iovecs
= 0, total_len
= 0;
1450 uint_t i
, j
, max_dnp
= datalen
>> DNODE_SHIFT
;
1451 iovec_t
*src_iovecs
= NULL
, *dst_iovecs
= NULL
;
1452 uint8_t *src
, *dst
, *aadp
;
1453 dnode_phys_t
*dnp
, *adnp
, *sdnp
, *ddnp
;
1454 uint8_t *aadbuf
= zio_buf_alloc(datalen
);
1468 sdnp
= (dnode_phys_t
*)src
;
1469 ddnp
= (dnode_phys_t
*)dst
;
1473 * Count the number of iovecs we will need to do the encryption by
1474 * counting the number of bonus buffers that need to be encrypted.
1476 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
1478 * This block may still be byteswapped. However, all of the
1479 * values we use are either uint8_t's (for which byteswapping
1480 * is a noop) or a * != 0 check, which will work regardless
1481 * of whether or not we byteswap.
1483 if (sdnp
[i
].dn_type
!= DMU_OT_NONE
&&
1484 DMU_OT_IS_ENCRYPTED(sdnp
[i
].dn_bonustype
) &&
1485 sdnp
[i
].dn_bonuslen
!= 0) {
1490 nr_src
+= nr_iovecs
;
1491 nr_dst
+= nr_iovecs
;
1494 src_iovecs
= kmem_alloc(nr_src
* sizeof (iovec_t
), KM_SLEEP
);
1495 if (src_iovecs
== NULL
) {
1496 ret
= SET_ERROR(ENOMEM
);
1502 dst_iovecs
= kmem_alloc(nr_dst
* sizeof (iovec_t
), KM_SLEEP
);
1503 if (dst_iovecs
== NULL
) {
1504 ret
= SET_ERROR(ENOMEM
);
1512 * Iterate through the dnodes again, this time filling in the uios
1513 * we allocated earlier. We also concatenate any data we want to
1514 * authenticate onto aadbuf.
1516 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
1519 /* copy over the core fields and blkptrs (kept as plaintext) */
1520 bcopy(dnp
, &ddnp
[i
], (uint8_t *)DN_BONUS(dnp
) - (uint8_t *)dnp
);
1522 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1523 bcopy(DN_SPILL_BLKPTR(dnp
), DN_SPILL_BLKPTR(&ddnp
[i
]),
1528 * Handle authenticated data. We authenticate everything in
1529 * the dnode that can be brought over when we do a raw send.
1530 * This includes all of the core fields as well as the MACs
1531 * stored in the bp checksums and all of the portable bits
1532 * from blk_prop. We include the dnode padding here in case it
1533 * ever gets used in the future. Some dn_flags and dn_used are
1534 * not portable so we mask those out values out of the
1535 * authenticated data.
1537 crypt_len
= offsetof(dnode_phys_t
, dn_blkptr
);
1538 bcopy(dnp
, aadp
, crypt_len
);
1539 adnp
= (dnode_phys_t
*)aadp
;
1540 adnp
->dn_flags
&= DNODE_CRYPT_PORTABLE_FLAGS_MASK
;
1543 aad_len
+= crypt_len
;
1545 for (j
= 0; j
< dnp
->dn_nblkptr
; j
++) {
1546 zio_crypt_bp_do_aad_updates(&aadp
, &aad_len
,
1547 byteswap
, &dnp
->dn_blkptr
[j
]);
1550 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1551 zio_crypt_bp_do_aad_updates(&aadp
, &aad_len
,
1552 byteswap
, DN_SPILL_BLKPTR(dnp
));
1556 * If this bonus buffer needs to be encrypted, we prepare an
1557 * iovec_t. The encryption / decryption functions will fill
1558 * this in for us with the encrypted or decrypted data.
1559 * Otherwise we add the bonus buffer to the authenticated
1560 * data buffer and copy it over to the destination. The
1561 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1562 * we can guarantee alignment with the AES block size
1565 crypt_len
= DN_MAX_BONUS_LEN(dnp
);
1566 if (dnp
->dn_type
!= DMU_OT_NONE
&&
1567 DMU_OT_IS_ENCRYPTED(dnp
->dn_bonustype
) &&
1568 dnp
->dn_bonuslen
!= 0) {
1569 ASSERT3U(nr_iovecs
, <, nr_src
);
1570 ASSERT3U(nr_iovecs
, <, nr_dst
);
1571 ASSERT3P(src_iovecs
, !=, NULL
);
1572 ASSERT3P(dst_iovecs
, !=, NULL
);
1573 src_iovecs
[nr_iovecs
].iov_base
= DN_BONUS(dnp
);
1574 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1575 dst_iovecs
[nr_iovecs
].iov_base
= DN_BONUS(&ddnp
[i
]);
1576 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1579 total_len
+= crypt_len
;
1581 bcopy(DN_BONUS(dnp
), DN_BONUS(&ddnp
[i
]), crypt_len
);
1582 bcopy(DN_BONUS(dnp
), aadp
, crypt_len
);
1584 aad_len
+= crypt_len
;
1588 *no_crypt
= (nr_iovecs
== 0);
1589 *enc_len
= total_len
;
1591 *auth_len
= aad_len
;
1594 puio
->uio_iov
= src_iovecs
;
1595 puio
->uio_iovcnt
= nr_src
;
1596 cuio
->uio_iov
= dst_iovecs
;
1597 cuio
->uio_iovcnt
= nr_dst
;
1599 puio
->uio_iov
= dst_iovecs
;
1600 puio
->uio_iovcnt
= nr_dst
;
1601 cuio
->uio_iov
= src_iovecs
;
1602 cuio
->uio_iovcnt
= nr_src
;
1608 zio_buf_free(aadbuf
, datalen
);
1609 if (src_iovecs
!= NULL
)
1610 kmem_free(src_iovecs
, nr_src
* sizeof (iovec_t
));
1611 if (dst_iovecs
!= NULL
)
1612 kmem_free(dst_iovecs
, nr_dst
* sizeof (iovec_t
));
1617 *no_crypt
= B_FALSE
;
1618 puio
->uio_iov
= NULL
;
1619 puio
->uio_iovcnt
= 0;
1620 cuio
->uio_iov
= NULL
;
1621 cuio
->uio_iovcnt
= 0;
1626 zio_crypt_init_uios_normal(boolean_t encrypt
, uint8_t *plainbuf
,
1627 uint8_t *cipherbuf
, uint_t datalen
, uio_t
*puio
, uio_t
*cuio
,
1631 uint_t nr_plain
= 1, nr_cipher
= 2;
1632 iovec_t
*plain_iovecs
= NULL
, *cipher_iovecs
= NULL
;
1634 /* allocate the iovecs for the plain and cipher data */
1635 plain_iovecs
= kmem_alloc(nr_plain
* sizeof (iovec_t
),
1637 if (!plain_iovecs
) {
1638 ret
= SET_ERROR(ENOMEM
);
1642 cipher_iovecs
= kmem_alloc(nr_cipher
* sizeof (iovec_t
),
1644 if (!cipher_iovecs
) {
1645 ret
= SET_ERROR(ENOMEM
);
1649 plain_iovecs
[0].iov_base
= plainbuf
;
1650 plain_iovecs
[0].iov_len
= datalen
;
1651 cipher_iovecs
[0].iov_base
= cipherbuf
;
1652 cipher_iovecs
[0].iov_len
= datalen
;
1655 puio
->uio_iov
= plain_iovecs
;
1656 puio
->uio_iovcnt
= nr_plain
;
1657 cuio
->uio_iov
= cipher_iovecs
;
1658 cuio
->uio_iovcnt
= nr_cipher
;
1663 if (plain_iovecs
!= NULL
)
1664 kmem_free(plain_iovecs
, nr_plain
* sizeof (iovec_t
));
1665 if (cipher_iovecs
!= NULL
)
1666 kmem_free(cipher_iovecs
, nr_cipher
* sizeof (iovec_t
));
1669 puio
->uio_iov
= NULL
;
1670 puio
->uio_iovcnt
= 0;
1671 cuio
->uio_iov
= NULL
;
1672 cuio
->uio_iovcnt
= 0;
1677 * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1678 * that they can be used for encryption and decryption by zio_do_crypt_uio().
1679 * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1680 * requiring special handling to parse out pieces that are to be encrypted. The
1681 * authbuf is used by these special cases to store additional authenticated
1682 * data (AAD) for the encryption modes.
1685 zio_crypt_init_uios(boolean_t encrypt
, dmu_object_type_t ot
, uint8_t *plainbuf
,
1686 uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
, uint8_t *mac
,
1687 uio_t
*puio
, uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
,
1688 uint_t
*auth_len
, boolean_t
*no_crypt
)
1693 ASSERT(DMU_OT_IS_ENCRYPTED(ot
) || ot
== DMU_OT_NONE
);
1695 /* route to handler */
1697 case DMU_OT_INTENT_LOG
:
1698 ret
= zio_crypt_init_uios_zil(encrypt
, plainbuf
, cipherbuf
,
1699 datalen
, byteswap
, puio
, cuio
, enc_len
, authbuf
, auth_len
,
1703 ret
= zio_crypt_init_uios_dnode(encrypt
, plainbuf
, cipherbuf
,
1704 datalen
, byteswap
, puio
, cuio
, enc_len
, authbuf
, auth_len
,
1708 ret
= zio_crypt_init_uios_normal(encrypt
, plainbuf
, cipherbuf
,
1709 datalen
, puio
, cuio
, enc_len
);
1712 *no_crypt
= B_FALSE
;
1719 /* populate the uios */
1720 puio
->uio_segflg
= UIO_SYSSPACE
;
1721 cuio
->uio_segflg
= UIO_SYSSPACE
;
1723 mac_iov
= ((iovec_t
*)&cuio
->uio_iov
[cuio
->uio_iovcnt
- 1]);
1724 mac_iov
->iov_base
= mac
;
1725 mac_iov
->iov_len
= ZIO_DATA_MAC_LEN
;
1734 * Primary encryption / decryption entrypoint for zio data.
1737 zio_do_crypt_data(boolean_t encrypt
, zio_crypt_key_t
*key
, uint8_t *salt
,
1738 dmu_object_type_t ot
, uint8_t *iv
, uint8_t *mac
, uint_t datalen
,
1739 boolean_t byteswap
, uint8_t *plainbuf
, uint8_t *cipherbuf
,
1740 boolean_t
*no_crypt
)
1743 boolean_t locked
= B_FALSE
;
1744 uint64_t crypt
= key
->zk_crypt
;
1745 uint_t keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
1746 uint_t enc_len
, auth_len
;
1748 uint8_t enc_keydata
[MASTER_KEY_MAX_LEN
];
1749 crypto_key_t tmp_ckey
, *ckey
= NULL
;
1750 crypto_ctx_template_t tmpl
;
1751 uint8_t *authbuf
= NULL
;
1753 bzero(&puio
, sizeof (uio_t
));
1754 bzero(&cuio
, sizeof (uio_t
));
1756 /* create uios for encryption */
1757 ret
= zio_crypt_init_uios(encrypt
, ot
, plainbuf
, cipherbuf
, datalen
,
1758 byteswap
, mac
, &puio
, &cuio
, &enc_len
, &authbuf
, &auth_len
,
1764 * If the needed key is the current one, just use it. Otherwise we
1765 * need to generate a temporary one from the given salt + master key.
1766 * If we are encrypting, we must return a copy of the current salt
1767 * so that it can be stored in the blkptr_t.
1769 rw_enter(&key
->zk_salt_lock
, RW_READER
);
1772 if (bcmp(salt
, key
->zk_salt
, ZIO_DATA_SALT_LEN
) == 0) {
1773 ckey
= &key
->zk_current_key
;
1774 tmpl
= key
->zk_current_tmpl
;
1776 rw_exit(&key
->zk_salt_lock
);
1779 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
1780 salt
, ZIO_DATA_SALT_LEN
, enc_keydata
, keydata_len
);
1784 tmp_ckey
.ck_format
= CRYPTO_KEY_RAW
;
1785 tmp_ckey
.ck_data
= enc_keydata
;
1786 tmp_ckey
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
1792 /* perform the encryption / decryption */
1793 ret
= zio_do_crypt_uio(encrypt
, key
->zk_crypt
, ckey
, tmpl
, iv
, enc_len
,
1794 &puio
, &cuio
, authbuf
, auth_len
);
1799 rw_exit(&key
->zk_salt_lock
);
1803 if (authbuf
!= NULL
)
1804 zio_buf_free(authbuf
, datalen
);
1805 if (ckey
== &tmp_ckey
)
1806 bzero(enc_keydata
, keydata_len
);
1807 zio_crypt_destroy_uio(&puio
);
1808 zio_crypt_destroy_uio(&cuio
);
1814 rw_exit(&key
->zk_salt_lock
);
1815 if (authbuf
!= NULL
)
1816 zio_buf_free(authbuf
, datalen
);
1817 if (ckey
== &tmp_ckey
)
1818 bzero(enc_keydata
, keydata_len
);
1819 zio_crypt_destroy_uio(&puio
);
1820 zio_crypt_destroy_uio(&cuio
);
1826 * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1830 zio_do_crypt_abd(boolean_t encrypt
, zio_crypt_key_t
*key
, uint8_t *salt
,
1831 dmu_object_type_t ot
, uint8_t *iv
, uint8_t *mac
, uint_t datalen
,
1832 boolean_t byteswap
, abd_t
*pabd
, abd_t
*cabd
, boolean_t
*no_crypt
)
1838 ptmp
= abd_borrow_buf_copy(pabd
, datalen
);
1839 ctmp
= abd_borrow_buf(cabd
, datalen
);
1841 ptmp
= abd_borrow_buf(pabd
, datalen
);
1842 ctmp
= abd_borrow_buf_copy(cabd
, datalen
);
1845 ret
= zio_do_crypt_data(encrypt
, key
, salt
, ot
, iv
, mac
,
1846 datalen
, byteswap
, ptmp
, ctmp
, no_crypt
);
1851 abd_return_buf(pabd
, ptmp
, datalen
);
1852 abd_return_buf_copy(cabd
, ctmp
, datalen
);
1854 abd_return_buf_copy(pabd
, ptmp
, datalen
);
1855 abd_return_buf(cabd
, ctmp
, datalen
);
1862 abd_return_buf(pabd
, ptmp
, datalen
);
1863 abd_return_buf_copy(cabd
, ctmp
, datalen
);
1865 abd_return_buf_copy(pabd
, ptmp
, datalen
);
1866 abd_return_buf(cabd
, ctmp
, datalen
);
1872 #if defined(_KERNEL) && defined(HAVE_SPL)
1874 module_param(zfs_key_max_salt_uses
, ulong
, 0644);
1875 MODULE_PARM_DESC(zfs_key_max_salt_uses
, "Max number of times a salt value "
1876 "can be used for generating encryption keys before it is rotated");