4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
9 * A full copy of the text of the CDDL should have accompanied this
10 * source. A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
17 * Copyright (c) 2017, Datto, Inc. All rights reserved.
20 #include <sys/zio_crypt.h>
22 #include <sys/dmu_objset.h>
23 #include <sys/dnode.h>
24 #include <sys/fs/zfs.h>
32 * This file is responsible for handling all of the details of generating
33 * encryption parameters and performing encryption and authentication.
35 * BLOCK ENCRYPTION PARAMETERS:
36 * Encryption /Authentication Algorithm Suite (crypt):
37 * The encryption algorithm, mode, and key length we are going to use. We
38 * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
39 * keys. All authentication is currently done with SHA512-HMAC.
42 * The unencrypted data that we want to encrypt.
44 * Initialization Vector (IV):
45 * An initialization vector for the encryption algorithms. This is used to
46 * "tweak" the encryption algorithms so that two blocks of the same data are
47 * encrypted into different ciphertext outputs, thus obfuscating block patterns.
48 * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
49 * never reused with the same encryption key. This value is stored unencrypted
50 * and must simply be provided to the decryption function. We use a 96 bit IV
51 * (as recommended by NIST) for all block encryption. For non-dedup blocks we
52 * derive the IV randomly. The first 64 bits of the IV are stored in the second
53 * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
54 * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
55 * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
56 * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
57 * level 0 blocks is the number of allocated dnodes in that block. The on-disk
58 * format supports at most 2^15 slots per L0 dnode block, because the maximum
59 * block size is 16MB (2^24). In either case, for level 0 blocks this number
60 * will still be smaller than UINT32_MAX so it is safe to store the IV in the
61 * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
65 * This is the most important secret data of an encrypted dataset. It is used
66 * along with the salt to generate that actual encryption keys via HKDF. We
67 * do not use the master key to directly encrypt any data because there are
68 * theoretical limits on how much data can actually be safely encrypted with
69 * any encryption mode. The master key is stored encrypted on disk with the
70 * user's wrapping key. Its length is determined by the encryption algorithm.
71 * For details on how this is stored see the block comment in dsl_crypt.c
74 * Used as an input to the HKDF function, along with the master key. We use a
75 * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
76 * can be used for encrypting many blocks, so we cache the current salt and the
77 * associated derived key in zio_crypt_t so we do not need to derive it again
81 * A secret binary key, generated from an HKDF function used to encrypt and
84 * Message Authentication Code (MAC)
85 * The MAC is an output of authenticated encryption modes such as AES-GCM and
86 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
87 * data on disk and return garbage to the application. Effectively, it is a
88 * checksum that can not be reproduced by an attacker. We store the MAC in the
89 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
90 * regular checksum of the ciphertext which can be used for scrubbing.
92 * OBJECT AUTHENTICATION:
93 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
94 * they contain some info that always needs to be readable. To prevent this
95 * data from being altered, we authenticate this data using SHA512-HMAC. This
96 * will produce a MAC (similar to the one produced via encryption) which can
97 * be used to verify the object was not modified. HMACs do not require key
98 * rotation or IVs, so we can keep up to the full 3 copies of authenticated
102 * ZIL blocks have their bp written to disk ahead of the associated data, so we
103 * cannot store the MAC there as we normally do. For these blocks the MAC is
104 * stored in the embedded checksum within the zil_chain_t header. The salt and
105 * IV are generated for the block on bp allocation instead of at encryption
106 * time. In addition, ZIL blocks have some pieces that must be left in plaintext
107 * for claiming even though all of the sensitive user data still needs to be
108 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
109 * pieces of the block need to be encrypted. All data that is not encrypted is
110 * authenticated using the AAD mechanisms that the supported encryption modes
111 * provide for. In order to preserve the semantics of the ZIL for encrypted
112 * datasets, the ZIL is not protected at the objset level as described below.
115 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
116 * in plaintext for scrubbing and claiming, but the bonus buffers might contain
117 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
118 * which which pieces of the block need to be encrypted. For more details about
119 * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
121 * OBJECT SET AUTHENTICATION:
122 * Up to this point, everything we have encrypted and authenticated has been
123 * at level 0 (or -2 for the ZIL). If we did not do any further work the
124 * on-disk format would be susceptible to attacks that deleted or rearranged
125 * the order of level 0 blocks. Ideally, the cleanest solution would be to
126 * maintain a tree of authentication MACs going up the bp tree. However, this
127 * presents a problem for raw sends. Send files do not send information about
128 * indirect blocks so there would be no convenient way to transfer the MACs and
129 * they cannot be recalculated on the receive side without the master key which
130 * would defeat one of the purposes of raw sends in the first place. Instead,
131 * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
132 * from the level below. We also include some portable fields from blk_prop such
133 * as the lsize and compression algorithm to prevent the data from being
136 * At the objset level, we maintain 2 separate 256 bit MACs in the
137 * objset_phys_t. The first one is "portable" and is the logical root of the
138 * MAC tree maintained in the metadnode's bps. The second, is "local" and is
139 * used as the root MAC for the user accounting objects, which are also not
140 * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
141 * of the send file. The useraccounting code ensures that the useraccounting
142 * info is not present upon a receive, so the local MAC can simply be cleared
143 * out at that time. For more info about objset_phys_t authentication, see
144 * zio_crypt_do_objset_hmacs().
146 * CONSIDERATIONS FOR DEDUP:
147 * In order for dedup to work, blocks that we want to dedup with one another
148 * need to use the same IV and encryption key, so that they will have the same
149 * ciphertext. Normally, one should never reuse an IV with the same encryption
150 * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
151 * blocks. In this case, however, since we are using the same plaintext as
152 * well all that we end up with is a duplicate of the original ciphertext we
153 * already had. As a result, an attacker with read access to the raw disk will
154 * be able to tell which blocks are the same but this information is given away
155 * by dedup anyway. In order to get the same IVs and encryption keys for
156 * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC
157 * here so that a reproducible checksum of the plaintext is never available to
158 * the attacker. The HMAC key is kept alongside the master key, encrypted on
159 * disk. The first 64 bits of the HMAC are used in place of the random salt, and
160 * the next 96 bits are used as the IV. As a result of this mechanism, dedup
161 * will only work within a clone family since encrypted dedup requires use of
162 * the same master and HMAC keys.
166 * After encrypting many blocks with the same key we may start to run up
167 * against the theoretical limits of how much data can securely be encrypted
168 * with a single key using the supported encryption modes. The most obvious
169 * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
170 * the more IVs we generate (which both GCM and CCM modes strictly forbid).
171 * This risk actually grows surprisingly quickly over time according to the
172 * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
173 * generated n IVs with a cryptographically secure RNG, the approximate
174 * probability p(n) of a collision is given as:
176 * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
178 * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
180 * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
181 * we must not write more than 398,065,730 blocks with the same encryption key.
182 * Therefore, we rotate our keys after 400,000,000 blocks have been written by
183 * generating a new random 64 bit salt for our HKDF encryption key generation
186 #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000
187 #define ZFS_CURRENT_MAX_SALT_USES \
188 (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
189 unsigned long zfs_key_max_salt_uses
= ZFS_KEY_MAX_SALT_USES_DEFAULT
;
191 typedef struct blkptr_auth_buf
{
192 uint64_t bab_prop
; /* blk_prop - portable mask */
193 uint8_t bab_mac
[ZIO_DATA_MAC_LEN
]; /* MAC from blk_cksum */
194 uint64_t bab_pad
; /* reserved for future use */
197 zio_crypt_info_t zio_crypt_table
[ZIO_CRYPT_FUNCTIONS
] = {
198 {"", ZC_TYPE_NONE
, 0, "inherit"},
199 {"", ZC_TYPE_NONE
, 0, "on"},
200 {"", ZC_TYPE_NONE
, 0, "off"},
201 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 16, "aes-128-ccm"},
202 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 24, "aes-192-ccm"},
203 {SUN_CKM_AES_CCM
, ZC_TYPE_CCM
, 32, "aes-256-ccm"},
204 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 16, "aes-128-gcm"},
205 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 24, "aes-192-gcm"},
206 {SUN_CKM_AES_GCM
, ZC_TYPE_GCM
, 32, "aes-256-gcm"}
210 zio_crypt_key_destroy(zio_crypt_key_t
*key
)
212 rw_destroy(&key
->zk_salt_lock
);
214 /* free crypto templates */
215 crypto_destroy_ctx_template(key
->zk_current_tmpl
);
216 crypto_destroy_ctx_template(key
->zk_hmac_tmpl
);
218 /* zero out sensitive data */
219 bzero(key
, sizeof (zio_crypt_key_t
));
223 zio_crypt_key_init(uint64_t crypt
, zio_crypt_key_t
*key
)
226 crypto_mechanism_t mech
;
230 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
232 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
233 bzero(key
, sizeof (zio_crypt_key_t
));
235 /* fill keydata buffers and salt with random data */
236 ret
= random_get_bytes((uint8_t *)&key
->zk_guid
, sizeof (uint64_t));
240 ret
= random_get_bytes(key
->zk_master_keydata
, keydata_len
);
244 ret
= random_get_bytes(key
->zk_hmac_keydata
, SHA512_HMAC_KEYLEN
);
248 ret
= random_get_bytes(key
->zk_salt
, ZIO_DATA_SALT_LEN
);
252 /* derive the current key from the master key */
253 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
254 key
->zk_salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
,
259 /* initialize keys for the ICP */
260 key
->zk_current_key
.ck_format
= CRYPTO_KEY_RAW
;
261 key
->zk_current_key
.ck_data
= key
->zk_current_keydata
;
262 key
->zk_current_key
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
264 key
->zk_hmac_key
.ck_format
= CRYPTO_KEY_RAW
;
265 key
->zk_hmac_key
.ck_data
= &key
->zk_hmac_key
;
266 key
->zk_hmac_key
.ck_length
= CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN
);
269 * Initialize the crypto templates. It's ok if this fails because
270 * this is just an optimization.
272 mech
.cm_type
= crypto_mech2id(zio_crypt_table
[crypt
].ci_mechname
);
273 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
274 &key
->zk_current_tmpl
, KM_SLEEP
);
275 if (ret
!= CRYPTO_SUCCESS
)
276 key
->zk_current_tmpl
= NULL
;
278 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
279 ret
= crypto_create_ctx_template(&mech
, &key
->zk_hmac_key
,
280 &key
->zk_hmac_tmpl
, KM_SLEEP
);
281 if (ret
!= CRYPTO_SUCCESS
)
282 key
->zk_hmac_tmpl
= NULL
;
284 key
->zk_crypt
= crypt
;
285 key
->zk_version
= ZIO_CRYPT_KEY_CURRENT_VERSION
;
286 key
->zk_salt_count
= 0;
287 rw_init(&key
->zk_salt_lock
, NULL
, RW_DEFAULT
, NULL
);
292 zio_crypt_key_destroy(key
);
297 zio_crypt_key_change_salt(zio_crypt_key_t
*key
)
300 uint8_t salt
[ZIO_DATA_SALT_LEN
];
301 crypto_mechanism_t mech
;
302 uint_t keydata_len
= zio_crypt_table
[key
->zk_crypt
].ci_keylen
;
304 /* generate a new salt */
305 ret
= random_get_bytes(salt
, ZIO_DATA_SALT_LEN
);
309 rw_enter(&key
->zk_salt_lock
, RW_WRITER
);
311 /* someone beat us to the salt rotation, just unlock and return */
312 if (key
->zk_salt_count
< ZFS_CURRENT_MAX_SALT_USES
)
315 /* derive the current key from the master key and the new salt */
316 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
317 salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
, keydata_len
);
321 /* assign the salt and reset the usage count */
322 bcopy(salt
, key
->zk_salt
, ZIO_DATA_SALT_LEN
);
323 key
->zk_salt_count
= 0;
325 /* destroy the old context template and create the new one */
326 crypto_destroy_ctx_template(key
->zk_current_tmpl
);
327 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
328 &key
->zk_current_tmpl
, KM_SLEEP
);
329 if (ret
!= CRYPTO_SUCCESS
)
330 key
->zk_current_tmpl
= NULL
;
332 rw_exit(&key
->zk_salt_lock
);
337 rw_exit(&key
->zk_salt_lock
);
342 /* See comment above zfs_key_max_salt_uses definition for details */
344 zio_crypt_key_get_salt(zio_crypt_key_t
*key
, uint8_t *salt
)
347 boolean_t salt_change
;
349 rw_enter(&key
->zk_salt_lock
, RW_READER
);
351 bcopy(key
->zk_salt
, salt
, ZIO_DATA_SALT_LEN
);
352 salt_change
= (atomic_inc_64_nv(&key
->zk_salt_count
) >=
353 ZFS_CURRENT_MAX_SALT_USES
);
355 rw_exit(&key
->zk_salt_lock
);
358 ret
= zio_crypt_key_change_salt(key
);
370 * This function handles all encryption and decryption in zfs. When
371 * encrypting it expects puio to reference the plaintext and cuio to
372 * reference the cphertext. cuio must have enough space for the
373 * ciphertext + room for a MAC. datalen should be the length of the
374 * plaintext / ciphertext alone.
377 zio_do_crypt_uio(boolean_t encrypt
, uint64_t crypt
, crypto_key_t
*key
,
378 crypto_ctx_template_t tmpl
, uint8_t *ivbuf
, uint_t datalen
,
379 uio_t
*puio
, uio_t
*cuio
, uint8_t *authbuf
, uint_t auth_len
)
382 crypto_data_t plaindata
, cipherdata
;
383 CK_AES_CCM_PARAMS ccmp
;
384 CK_AES_GCM_PARAMS gcmp
;
385 crypto_mechanism_t mech
;
386 zio_crypt_info_t crypt_info
;
387 uint_t plain_full_len
, maclen
;
389 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
390 ASSERT3U(key
->ck_format
, ==, CRYPTO_KEY_RAW
);
392 /* lookup the encryption info */
393 crypt_info
= zio_crypt_table
[crypt
];
395 /* the mac will always be the last iovec_t in the cipher uio */
396 maclen
= cuio
->uio_iov
[cuio
->uio_iovcnt
- 1].iov_len
;
398 ASSERT(maclen
<= ZIO_DATA_MAC_LEN
);
400 /* setup encryption mechanism (same as crypt) */
401 mech
.cm_type
= crypto_mech2id(crypt_info
.ci_mechname
);
404 * Strangely, the ICP requires that plain_full_len must include
405 * the MAC length when decrypting, even though the UIO does not
406 * need to have the extra space allocated.
409 plain_full_len
= datalen
;
411 plain_full_len
= datalen
+ maclen
;
415 * setup encryption params (currently only AES CCM and AES GCM
418 if (crypt_info
.ci_crypt_type
== ZC_TYPE_CCM
) {
419 ccmp
.ulNonceSize
= ZIO_DATA_IV_LEN
;
420 ccmp
.ulAuthDataSize
= auth_len
;
421 ccmp
.authData
= authbuf
;
422 ccmp
.ulMACSize
= maclen
;
424 ccmp
.ulDataSize
= plain_full_len
;
426 mech
.cm_param
= (char *)(&ccmp
);
427 mech
.cm_param_len
= sizeof (CK_AES_CCM_PARAMS
);
429 gcmp
.ulIvLen
= ZIO_DATA_IV_LEN
;
430 gcmp
.ulIvBits
= CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN
);
431 gcmp
.ulAADLen
= auth_len
;
433 gcmp
.ulTagBits
= CRYPTO_BYTES2BITS(maclen
);
436 mech
.cm_param
= (char *)(&gcmp
);
437 mech
.cm_param_len
= sizeof (CK_AES_GCM_PARAMS
);
440 /* populate the cipher and plain data structs. */
441 plaindata
.cd_format
= CRYPTO_DATA_UIO
;
442 plaindata
.cd_offset
= 0;
443 plaindata
.cd_uio
= puio
;
444 plaindata
.cd_miscdata
= NULL
;
445 plaindata
.cd_length
= plain_full_len
;
447 cipherdata
.cd_format
= CRYPTO_DATA_UIO
;
448 cipherdata
.cd_offset
= 0;
449 cipherdata
.cd_uio
= cuio
;
450 cipherdata
.cd_miscdata
= NULL
;
451 cipherdata
.cd_length
= datalen
+ maclen
;
453 /* perform the actual encryption */
455 ret
= crypto_encrypt(&mech
, &plaindata
, key
, tmpl
, &cipherdata
,
457 if (ret
!= CRYPTO_SUCCESS
) {
458 ret
= SET_ERROR(EIO
);
462 ret
= crypto_decrypt(&mech
, &cipherdata
, key
, tmpl
, &plaindata
,
464 if (ret
!= CRYPTO_SUCCESS
) {
465 ASSERT3U(ret
, ==, CRYPTO_INVALID_MAC
);
466 ret
= SET_ERROR(ECKSUM
);
478 zio_crypt_key_wrap(crypto_key_t
*cwkey
, zio_crypt_key_t
*key
, uint8_t *iv
,
479 uint8_t *mac
, uint8_t *keydata_out
, uint8_t *hmac_keydata_out
)
484 iovec_t plain_iovecs
[2], cipher_iovecs
[3];
485 uint64_t crypt
= key
->zk_crypt
;
486 uint_t enc_len
, keydata_len
, aad_len
;
488 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
489 ASSERT3U(cwkey
->ck_format
, ==, CRYPTO_KEY_RAW
);
491 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
493 /* generate iv for wrapping the master and hmac key */
494 ret
= random_get_pseudo_bytes(iv
, WRAPPING_IV_LEN
);
498 /* initialize uio_ts */
499 plain_iovecs
[0].iov_base
= key
->zk_master_keydata
;
500 plain_iovecs
[0].iov_len
= keydata_len
;
501 plain_iovecs
[1].iov_base
= key
->zk_hmac_keydata
;
502 plain_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
504 cipher_iovecs
[0].iov_base
= keydata_out
;
505 cipher_iovecs
[0].iov_len
= keydata_len
;
506 cipher_iovecs
[1].iov_base
= hmac_keydata_out
;
507 cipher_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
508 cipher_iovecs
[2].iov_base
= mac
;
509 cipher_iovecs
[2].iov_len
= WRAPPING_MAC_LEN
;
512 * Although we don't support writing to the old format, we do
513 * support rewrapping the key so that the user can move and
514 * quarantine datasets on the old format.
516 if (key
->zk_version
== 0) {
517 aad_len
= sizeof (uint64_t);
518 aad
[0] = LE_64(key
->zk_guid
);
520 ASSERT3U(key
->zk_version
, ==, ZIO_CRYPT_KEY_CURRENT_VERSION
);
521 aad_len
= sizeof (uint64_t) * 3;
522 aad
[0] = LE_64(key
->zk_guid
);
523 aad
[1] = LE_64(crypt
);
524 aad
[2] = LE_64(key
->zk_version
);
527 enc_len
= zio_crypt_table
[crypt
].ci_keylen
+ SHA512_HMAC_KEYLEN
;
528 puio
.uio_iov
= plain_iovecs
;
530 puio
.uio_segflg
= UIO_SYSSPACE
;
531 cuio
.uio_iov
= cipher_iovecs
;
533 cuio
.uio_segflg
= UIO_SYSSPACE
;
535 /* encrypt the keys and store the resulting ciphertext and mac */
536 ret
= zio_do_crypt_uio(B_TRUE
, crypt
, cwkey
, NULL
, iv
, enc_len
,
537 &puio
, &cuio
, (uint8_t *)aad
, aad_len
);
548 zio_crypt_key_unwrap(crypto_key_t
*cwkey
, uint64_t crypt
, uint64_t version
,
549 uint64_t guid
, uint8_t *keydata
, uint8_t *hmac_keydata
, uint8_t *iv
,
550 uint8_t *mac
, zio_crypt_key_t
*key
)
553 crypto_mechanism_t mech
;
556 iovec_t plain_iovecs
[2], cipher_iovecs
[3];
557 uint_t enc_len
, keydata_len
, aad_len
;
559 ASSERT3U(crypt
, <, ZIO_CRYPT_FUNCTIONS
);
560 ASSERT3U(cwkey
->ck_format
, ==, CRYPTO_KEY_RAW
);
562 rw_init(&key
->zk_salt_lock
, NULL
, RW_DEFAULT
, NULL
);
564 keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
566 /* initialize uio_ts */
567 plain_iovecs
[0].iov_base
= key
->zk_master_keydata
;
568 plain_iovecs
[0].iov_len
= keydata_len
;
569 plain_iovecs
[1].iov_base
= key
->zk_hmac_keydata
;
570 plain_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
572 cipher_iovecs
[0].iov_base
= keydata
;
573 cipher_iovecs
[0].iov_len
= keydata_len
;
574 cipher_iovecs
[1].iov_base
= hmac_keydata
;
575 cipher_iovecs
[1].iov_len
= SHA512_HMAC_KEYLEN
;
576 cipher_iovecs
[2].iov_base
= mac
;
577 cipher_iovecs
[2].iov_len
= WRAPPING_MAC_LEN
;
580 aad_len
= sizeof (uint64_t);
581 aad
[0] = LE_64(guid
);
583 ASSERT3U(version
, ==, ZIO_CRYPT_KEY_CURRENT_VERSION
);
584 aad_len
= sizeof (uint64_t) * 3;
585 aad
[0] = LE_64(guid
);
586 aad
[1] = LE_64(crypt
);
587 aad
[2] = LE_64(version
);
590 enc_len
= keydata_len
+ SHA512_HMAC_KEYLEN
;
591 puio
.uio_iov
= plain_iovecs
;
592 puio
.uio_segflg
= UIO_SYSSPACE
;
594 cuio
.uio_iov
= cipher_iovecs
;
596 cuio
.uio_segflg
= UIO_SYSSPACE
;
598 /* decrypt the keys and store the result in the output buffers */
599 ret
= zio_do_crypt_uio(B_FALSE
, crypt
, cwkey
, NULL
, iv
, enc_len
,
600 &puio
, &cuio
, (uint8_t *)aad
, aad_len
);
604 /* generate a fresh salt */
605 ret
= random_get_bytes(key
->zk_salt
, ZIO_DATA_SALT_LEN
);
609 /* derive the current key from the master key */
610 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
611 key
->zk_salt
, ZIO_DATA_SALT_LEN
, key
->zk_current_keydata
,
616 /* initialize keys for ICP */
617 key
->zk_current_key
.ck_format
= CRYPTO_KEY_RAW
;
618 key
->zk_current_key
.ck_data
= key
->zk_current_keydata
;
619 key
->zk_current_key
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
621 key
->zk_hmac_key
.ck_format
= CRYPTO_KEY_RAW
;
622 key
->zk_hmac_key
.ck_data
= key
->zk_hmac_keydata
;
623 key
->zk_hmac_key
.ck_length
= CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN
);
626 * Initialize the crypto templates. It's ok if this fails because
627 * this is just an optimization.
629 mech
.cm_type
= crypto_mech2id(zio_crypt_table
[crypt
].ci_mechname
);
630 ret
= crypto_create_ctx_template(&mech
, &key
->zk_current_key
,
631 &key
->zk_current_tmpl
, KM_SLEEP
);
632 if (ret
!= CRYPTO_SUCCESS
)
633 key
->zk_current_tmpl
= NULL
;
635 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
636 ret
= crypto_create_ctx_template(&mech
, &key
->zk_hmac_key
,
637 &key
->zk_hmac_tmpl
, KM_SLEEP
);
638 if (ret
!= CRYPTO_SUCCESS
)
639 key
->zk_hmac_tmpl
= NULL
;
641 key
->zk_crypt
= crypt
;
642 key
->zk_version
= version
;
644 key
->zk_salt_count
= 0;
649 zio_crypt_key_destroy(key
);
654 zio_crypt_generate_iv(uint8_t *ivbuf
)
658 /* randomly generate the IV */
659 ret
= random_get_pseudo_bytes(ivbuf
, ZIO_DATA_IV_LEN
);
666 bzero(ivbuf
, ZIO_DATA_IV_LEN
);
671 zio_crypt_do_hmac(zio_crypt_key_t
*key
, uint8_t *data
, uint_t datalen
,
672 uint8_t *digestbuf
, uint_t digestlen
)
675 crypto_mechanism_t mech
;
676 crypto_data_t in_data
, digest_data
;
677 uint8_t raw_digestbuf
[SHA512_DIGEST_LENGTH
];
679 ASSERT3U(digestlen
, <=, SHA512_DIGEST_LENGTH
);
681 /* initialize sha512-hmac mechanism and crypto data */
682 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
683 mech
.cm_param
= NULL
;
684 mech
.cm_param_len
= 0;
686 /* initialize the crypto data */
687 in_data
.cd_format
= CRYPTO_DATA_RAW
;
688 in_data
.cd_offset
= 0;
689 in_data
.cd_length
= datalen
;
690 in_data
.cd_raw
.iov_base
= (char *)data
;
691 in_data
.cd_raw
.iov_len
= in_data
.cd_length
;
693 digest_data
.cd_format
= CRYPTO_DATA_RAW
;
694 digest_data
.cd_offset
= 0;
695 digest_data
.cd_length
= SHA512_DIGEST_LENGTH
;
696 digest_data
.cd_raw
.iov_base
= (char *)raw_digestbuf
;
697 digest_data
.cd_raw
.iov_len
= digest_data
.cd_length
;
699 /* generate the hmac */
700 ret
= crypto_mac(&mech
, &in_data
, &key
->zk_hmac_key
, key
->zk_hmac_tmpl
,
702 if (ret
!= CRYPTO_SUCCESS
) {
703 ret
= SET_ERROR(EIO
);
707 bcopy(raw_digestbuf
, digestbuf
, digestlen
);
712 bzero(digestbuf
, digestlen
);
717 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t
*key
, uint8_t *data
,
718 uint_t datalen
, uint8_t *ivbuf
, uint8_t *salt
)
721 uint8_t digestbuf
[SHA512_DIGEST_LENGTH
];
723 ret
= zio_crypt_do_hmac(key
, data
, datalen
,
724 digestbuf
, SHA512_DIGEST_LENGTH
);
728 bcopy(digestbuf
, salt
, ZIO_DATA_SALT_LEN
);
729 bcopy(digestbuf
+ ZIO_DATA_SALT_LEN
, ivbuf
, ZIO_DATA_IV_LEN
);
735 * The following functions are used to encode and decode encryption parameters
736 * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
737 * byte strings, which normally means that these strings would not need to deal
738 * with byteswapping at all. However, both blkptr_t and zil_header_t may be
739 * byteswapped by lower layers and so we must "undo" that byteswap here upon
740 * decoding and encoding in a non-native byteorder. These functions require
741 * that the byteorder bit is correct before being called.
744 zio_crypt_encode_params_bp(blkptr_t
*bp
, uint8_t *salt
, uint8_t *iv
)
749 ASSERT(BP_IS_ENCRYPTED(bp
));
751 if (!BP_SHOULD_BYTESWAP(bp
)) {
752 bcopy(salt
, &bp
->blk_dva
[2].dva_word
[0], sizeof (uint64_t));
753 bcopy(iv
, &bp
->blk_dva
[2].dva_word
[1], sizeof (uint64_t));
754 bcopy(iv
+ sizeof (uint64_t), &val32
, sizeof (uint32_t));
755 BP_SET_IV2(bp
, val32
);
757 bcopy(salt
, &val64
, sizeof (uint64_t));
758 bp
->blk_dva
[2].dva_word
[0] = BSWAP_64(val64
);
760 bcopy(iv
, &val64
, sizeof (uint64_t));
761 bp
->blk_dva
[2].dva_word
[1] = BSWAP_64(val64
);
763 bcopy(iv
+ sizeof (uint64_t), &val32
, sizeof (uint32_t));
764 BP_SET_IV2(bp
, BSWAP_32(val32
));
769 zio_crypt_decode_params_bp(const blkptr_t
*bp
, uint8_t *salt
, uint8_t *iv
)
774 ASSERT(BP_IS_PROTECTED(bp
));
776 /* for convenience, so callers don't need to check */
777 if (BP_IS_AUTHENTICATED(bp
)) {
778 bzero(salt
, ZIO_DATA_SALT_LEN
);
779 bzero(iv
, ZIO_DATA_IV_LEN
);
783 if (!BP_SHOULD_BYTESWAP(bp
)) {
784 bcopy(&bp
->blk_dva
[2].dva_word
[0], salt
, sizeof (uint64_t));
785 bcopy(&bp
->blk_dva
[2].dva_word
[1], iv
, sizeof (uint64_t));
787 val32
= (uint32_t)BP_GET_IV2(bp
);
788 bcopy(&val32
, iv
+ sizeof (uint64_t), sizeof (uint32_t));
790 val64
= BSWAP_64(bp
->blk_dva
[2].dva_word
[0]);
791 bcopy(&val64
, salt
, sizeof (uint64_t));
793 val64
= BSWAP_64(bp
->blk_dva
[2].dva_word
[1]);
794 bcopy(&val64
, iv
, sizeof (uint64_t));
796 val32
= BSWAP_32((uint32_t)BP_GET_IV2(bp
));
797 bcopy(&val32
, iv
+ sizeof (uint64_t), sizeof (uint32_t));
802 zio_crypt_encode_mac_bp(blkptr_t
*bp
, uint8_t *mac
)
806 ASSERT(BP_USES_CRYPT(bp
));
807 ASSERT3U(BP_GET_TYPE(bp
), !=, DMU_OT_OBJSET
);
809 if (!BP_SHOULD_BYTESWAP(bp
)) {
810 bcopy(mac
, &bp
->blk_cksum
.zc_word
[2], sizeof (uint64_t));
811 bcopy(mac
+ sizeof (uint64_t), &bp
->blk_cksum
.zc_word
[3],
814 bcopy(mac
, &val64
, sizeof (uint64_t));
815 bp
->blk_cksum
.zc_word
[2] = BSWAP_64(val64
);
817 bcopy(mac
+ sizeof (uint64_t), &val64
, sizeof (uint64_t));
818 bp
->blk_cksum
.zc_word
[3] = BSWAP_64(val64
);
823 zio_crypt_decode_mac_bp(const blkptr_t
*bp
, uint8_t *mac
)
827 ASSERT(BP_USES_CRYPT(bp
) || BP_IS_HOLE(bp
));
829 /* for convenience, so callers don't need to check */
830 if (BP_GET_TYPE(bp
) == DMU_OT_OBJSET
) {
831 bzero(mac
, ZIO_DATA_MAC_LEN
);
835 if (!BP_SHOULD_BYTESWAP(bp
)) {
836 bcopy(&bp
->blk_cksum
.zc_word
[2], mac
, sizeof (uint64_t));
837 bcopy(&bp
->blk_cksum
.zc_word
[3], mac
+ sizeof (uint64_t),
840 val64
= BSWAP_64(bp
->blk_cksum
.zc_word
[2]);
841 bcopy(&val64
, mac
, sizeof (uint64_t));
843 val64
= BSWAP_64(bp
->blk_cksum
.zc_word
[3]);
844 bcopy(&val64
, mac
+ sizeof (uint64_t), sizeof (uint64_t));
849 zio_crypt_encode_mac_zil(void *data
, uint8_t *mac
)
851 zil_chain_t
*zilc
= data
;
853 bcopy(mac
, &zilc
->zc_eck
.zec_cksum
.zc_word
[2], sizeof (uint64_t));
854 bcopy(mac
+ sizeof (uint64_t), &zilc
->zc_eck
.zec_cksum
.zc_word
[3],
859 zio_crypt_decode_mac_zil(const void *data
, uint8_t *mac
)
862 * The ZIL MAC is embedded in the block it protects, which will
863 * not have been byteswapped by the time this function has been called.
864 * As a result, we don't need to worry about byteswapping the MAC.
866 const zil_chain_t
*zilc
= data
;
868 bcopy(&zilc
->zc_eck
.zec_cksum
.zc_word
[2], mac
, sizeof (uint64_t));
869 bcopy(&zilc
->zc_eck
.zec_cksum
.zc_word
[3], mac
+ sizeof (uint64_t),
874 * This routine takes a block of dnodes (src_abd) and copies only the bonus
875 * buffers to the same offsets in the dst buffer. datalen should be the size
876 * of both the src_abd and the dst buffer (not just the length of the bonus
880 zio_crypt_copy_dnode_bonus(abd_t
*src_abd
, uint8_t *dst
, uint_t datalen
)
882 uint_t i
, max_dnp
= datalen
>> DNODE_SHIFT
;
884 dnode_phys_t
*dnp
, *sdnp
, *ddnp
;
886 src
= abd_borrow_buf_copy(src_abd
, datalen
);
888 sdnp
= (dnode_phys_t
*)src
;
889 ddnp
= (dnode_phys_t
*)dst
;
891 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
893 if (dnp
->dn_type
!= DMU_OT_NONE
&&
894 DMU_OT_IS_ENCRYPTED(dnp
->dn_bonustype
) &&
895 dnp
->dn_bonuslen
!= 0) {
896 bcopy(DN_BONUS(dnp
), DN_BONUS(&ddnp
[i
]),
897 DN_MAX_BONUS_LEN(dnp
));
901 abd_return_buf(src_abd
, src
, datalen
);
905 * This function decides what fields from blk_prop are included in
906 * the on-disk various MAC algorithms.
909 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t
*bp
, uint64_t version
)
912 * Version 0 did not properly zero out all non-portable fields
913 * as it should have done. We maintain this code so that we can
914 * do read-only imports of pools on this version.
918 BP_SET_CHECKSUM(bp
, 0);
919 BP_SET_PSIZE(bp
, SPA_MINBLOCKSIZE
);
923 ASSERT3U(version
, ==, ZIO_CRYPT_KEY_CURRENT_VERSION
);
926 * The hole_birth feature might set these fields even if this bp
927 * is a hole. We zero them out here to guarantee that raw sends
928 * will function with or without the feature.
930 if (BP_IS_HOLE(bp
)) {
936 * At L0 we want to verify these fields to ensure that data blocks
937 * can not be reinterpretted. For instance, we do not want an attacker
938 * to trick us into returning raw lz4 compressed data to the user
939 * by modifying the compression bits. At higher levels, we cannot
940 * enforce this policy since raw sends do not convey any information
941 * about indirect blocks, so these values might be different on the
942 * receive side. Fortunately, this does not open any new attack
943 * vectors, since any alterations that can be made to a higher level
944 * bp must still verify the correct order of the layer below it.
946 if (BP_GET_LEVEL(bp
) != 0) {
947 BP_SET_BYTEORDER(bp
, 0);
948 BP_SET_COMPRESS(bp
, 0);
951 * psize cannot be set to zero or it will trigger
952 * asserts, but the value doesn't really matter as
953 * long as it is constant.
955 BP_SET_PSIZE(bp
, SPA_MINBLOCKSIZE
);
959 BP_SET_CHECKSUM(bp
, 0);
963 zio_crypt_bp_auth_init(uint64_t version
, boolean_t should_bswap
, blkptr_t
*bp
,
964 blkptr_auth_buf_t
*bab
, uint_t
*bab_len
)
966 blkptr_t tmpbp
= *bp
;
969 byteswap_uint64_array(&tmpbp
, sizeof (blkptr_t
));
971 ASSERT(BP_USES_CRYPT(&tmpbp
) || BP_IS_HOLE(&tmpbp
));
972 ASSERT0(BP_IS_EMBEDDED(&tmpbp
));
974 zio_crypt_decode_mac_bp(&tmpbp
, bab
->bab_mac
);
977 * We always MAC blk_prop in LE to ensure portability. This
978 * must be done after decoding the mac, since the endianness
979 * will get zero'd out here.
981 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp
, version
);
982 bab
->bab_prop
= LE_64(tmpbp
.blk_prop
);
985 /* version 0 did not include the padding */
986 *bab_len
= sizeof (blkptr_auth_buf_t
);
988 *bab_len
-= sizeof (uint64_t);
992 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx
, uint64_t version
,
993 boolean_t should_bswap
, blkptr_t
*bp
)
997 blkptr_auth_buf_t bab
;
1000 zio_crypt_bp_auth_init(version
, should_bswap
, bp
, &bab
, &bab_len
);
1001 cd
.cd_format
= CRYPTO_DATA_RAW
;
1003 cd
.cd_length
= bab_len
;
1004 cd
.cd_raw
.iov_base
= (char *)&bab
;
1005 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1007 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1008 if (ret
!= CRYPTO_SUCCESS
) {
1009 ret
= SET_ERROR(EIO
);
1020 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX
*ctx
, uint64_t version
,
1021 boolean_t should_bswap
, blkptr_t
*bp
)
1024 blkptr_auth_buf_t bab
;
1026 zio_crypt_bp_auth_init(version
, should_bswap
, bp
, &bab
, &bab_len
);
1027 SHA2Update(ctx
, &bab
, bab_len
);
1031 zio_crypt_bp_do_aad_updates(uint8_t **aadp
, uint_t
*aad_len
, uint64_t version
,
1032 boolean_t should_bswap
, blkptr_t
*bp
)
1035 blkptr_auth_buf_t bab
;
1037 zio_crypt_bp_auth_init(version
, should_bswap
, bp
, &bab
, &bab_len
);
1038 bcopy(&bab
, *aadp
, bab_len
);
1040 *aad_len
+= bab_len
;
1044 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx
, uint64_t version
,
1045 boolean_t should_bswap
, dnode_phys_t
*dnp
)
1049 boolean_t le_bswap
= (should_bswap
== ZFS_HOST_BYTEORDER
);
1051 uint8_t tmp_dncore
[offsetof(dnode_phys_t
, dn_blkptr
)];
1053 cd
.cd_format
= CRYPTO_DATA_RAW
;
1056 /* authenticate the core dnode (masking out non-portable bits) */
1057 bcopy(dnp
, tmp_dncore
, sizeof (tmp_dncore
));
1058 adnp
= (dnode_phys_t
*)tmp_dncore
;
1060 adnp
->dn_datablkszsec
= BSWAP_16(adnp
->dn_datablkszsec
);
1061 adnp
->dn_bonuslen
= BSWAP_16(adnp
->dn_bonuslen
);
1062 adnp
->dn_maxblkid
= BSWAP_64(adnp
->dn_maxblkid
);
1063 adnp
->dn_used
= BSWAP_64(adnp
->dn_used
);
1065 adnp
->dn_flags
&= DNODE_CRYPT_PORTABLE_FLAGS_MASK
;
1068 cd
.cd_length
= sizeof (tmp_dncore
);
1069 cd
.cd_raw
.iov_base
= (char *)adnp
;
1070 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1072 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1073 if (ret
!= CRYPTO_SUCCESS
) {
1074 ret
= SET_ERROR(EIO
);
1078 for (i
= 0; i
< dnp
->dn_nblkptr
; i
++) {
1079 ret
= zio_crypt_bp_do_hmac_updates(ctx
, version
,
1080 should_bswap
, &dnp
->dn_blkptr
[i
]);
1085 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1086 ret
= zio_crypt_bp_do_hmac_updates(ctx
, version
,
1087 should_bswap
, DN_SPILL_BLKPTR(dnp
));
1099 * objset_phys_t blocks introduce a number of exceptions to the normal
1100 * authentication process. objset_phys_t's contain 2 separate HMACS for
1101 * protecting the integrity of their data. The portable_mac protects the
1102 * metadnode. This MAC can be sent with a raw send and protects against
1103 * reordering of data within the metadnode. The local_mac protects the user
1104 * accounting objects which are not sent from one system to another.
1106 * In addition, objset blocks are the only blocks that can be modified and
1107 * written to disk without the key loaded under certain circumstances. During
1108 * zil_claim() we need to be able to update the zil_header_t to complete
1109 * claiming log blocks and during raw receives we need to write out the
1110 * portable_mac from the send file. Both of these actions are possible
1111 * because these fields are not protected by either MAC so neither one will
1112 * need to modify the MACs without the key. However, when the modified blocks
1113 * are written out they will be byteswapped into the host machine's native
1114 * endianness which will modify fields protected by the MAC. As a result, MAC
1115 * calculation for objset blocks works slightly differently from other block
1116 * types. Where other block types MAC the data in whatever endianness is
1117 * written to disk, objset blocks always MAC little endian version of their
1118 * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1119 * and le_bswap indicates whether a byteswap is needed to get this block
1120 * into little endian format.
1123 zio_crypt_do_objset_hmacs(zio_crypt_key_t
*key
, void *data
, uint_t datalen
,
1124 boolean_t should_bswap
, uint8_t *portable_mac
, uint8_t *local_mac
)
1127 crypto_mechanism_t mech
;
1128 crypto_context_t ctx
;
1130 objset_phys_t
*osp
= data
;
1132 boolean_t le_bswap
= (should_bswap
== ZFS_HOST_BYTEORDER
);
1133 uint8_t raw_portable_mac
[SHA512_DIGEST_LENGTH
];
1134 uint8_t raw_local_mac
[SHA512_DIGEST_LENGTH
];
1136 /* initialize HMAC mechanism */
1137 mech
.cm_type
= crypto_mech2id(SUN_CKM_SHA512_HMAC
);
1138 mech
.cm_param
= NULL
;
1139 mech
.cm_param_len
= 0;
1141 cd
.cd_format
= CRYPTO_DATA_RAW
;
1144 /* calculate the portable MAC from the portable fields and metadnode */
1145 ret
= crypto_mac_init(&mech
, &key
->zk_hmac_key
, NULL
, &ctx
, NULL
);
1146 if (ret
!= CRYPTO_SUCCESS
) {
1147 ret
= SET_ERROR(EIO
);
1151 /* add in the os_type */
1152 intval
= (le_bswap
) ? osp
->os_type
: BSWAP_64(osp
->os_type
);
1153 cd
.cd_length
= sizeof (uint64_t);
1154 cd
.cd_raw
.iov_base
= (char *)&intval
;
1155 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1157 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1158 if (ret
!= CRYPTO_SUCCESS
) {
1159 ret
= SET_ERROR(EIO
);
1163 /* add in the portable os_flags */
1164 intval
= osp
->os_flags
;
1166 intval
= BSWAP_64(intval
);
1167 intval
&= OBJSET_CRYPT_PORTABLE_FLAGS_MASK
;
1168 if (!ZFS_HOST_BYTEORDER
)
1169 intval
= BSWAP_64(intval
);
1171 cd
.cd_length
= sizeof (uint64_t);
1172 cd
.cd_raw
.iov_base
= (char *)&intval
;
1173 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1175 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1176 if (ret
!= CRYPTO_SUCCESS
) {
1177 ret
= SET_ERROR(EIO
);
1181 /* add in fields from the metadnode */
1182 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, key
->zk_version
,
1183 should_bswap
, &osp
->os_meta_dnode
);
1187 /* store the final digest in a temporary buffer and copy what we need */
1188 cd
.cd_length
= SHA512_DIGEST_LENGTH
;
1189 cd
.cd_raw
.iov_base
= (char *)raw_portable_mac
;
1190 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1192 ret
= crypto_mac_final(ctx
, &cd
, NULL
);
1193 if (ret
!= CRYPTO_SUCCESS
) {
1194 ret
= SET_ERROR(EIO
);
1198 bcopy(raw_portable_mac
, portable_mac
, ZIO_OBJSET_MAC_LEN
);
1201 * The local MAC protects the user, group and project accounting.
1202 * If these objects are not present, the local MAC is zeroed out.
1204 if ((datalen
>= OBJSET_PHYS_SIZE_V3
&&
1205 osp
->os_userused_dnode
.dn_type
== DMU_OT_NONE
&&
1206 osp
->os_groupused_dnode
.dn_type
== DMU_OT_NONE
&&
1207 osp
->os_projectused_dnode
.dn_type
== DMU_OT_NONE
) ||
1208 (datalen
>= OBJSET_PHYS_SIZE_V2
&&
1209 osp
->os_userused_dnode
.dn_type
== DMU_OT_NONE
&&
1210 osp
->os_groupused_dnode
.dn_type
== DMU_OT_NONE
) ||
1211 (datalen
<= OBJSET_PHYS_SIZE_V1
)) {
1212 bzero(local_mac
, ZIO_OBJSET_MAC_LEN
);
1216 /* calculate the local MAC from the userused and groupused dnodes */
1217 ret
= crypto_mac_init(&mech
, &key
->zk_hmac_key
, NULL
, &ctx
, NULL
);
1218 if (ret
!= CRYPTO_SUCCESS
) {
1219 ret
= SET_ERROR(EIO
);
1223 /* add in the non-portable os_flags */
1224 intval
= osp
->os_flags
;
1226 intval
= BSWAP_64(intval
);
1227 intval
&= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK
;
1228 if (!ZFS_HOST_BYTEORDER
)
1229 intval
= BSWAP_64(intval
);
1231 cd
.cd_length
= sizeof (uint64_t);
1232 cd
.cd_raw
.iov_base
= (char *)&intval
;
1233 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1235 ret
= crypto_mac_update(ctx
, &cd
, NULL
);
1236 if (ret
!= CRYPTO_SUCCESS
) {
1237 ret
= SET_ERROR(EIO
);
1241 /* add in fields from the user accounting dnodes */
1242 if (osp
->os_userused_dnode
.dn_type
!= DMU_OT_NONE
) {
1243 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, key
->zk_version
,
1244 should_bswap
, &osp
->os_userused_dnode
);
1249 if (osp
->os_groupused_dnode
.dn_type
!= DMU_OT_NONE
) {
1250 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, key
->zk_version
,
1251 should_bswap
, &osp
->os_groupused_dnode
);
1256 if (osp
->os_projectused_dnode
.dn_type
!= DMU_OT_NONE
&&
1257 datalen
>= OBJSET_PHYS_SIZE_V3
) {
1258 ret
= zio_crypt_do_dnode_hmac_updates(ctx
, key
->zk_version
,
1259 should_bswap
, &osp
->os_projectused_dnode
);
1264 /* store the final digest in a temporary buffer and copy what we need */
1265 cd
.cd_length
= SHA512_DIGEST_LENGTH
;
1266 cd
.cd_raw
.iov_base
= (char *)raw_local_mac
;
1267 cd
.cd_raw
.iov_len
= cd
.cd_length
;
1269 ret
= crypto_mac_final(ctx
, &cd
, NULL
);
1270 if (ret
!= CRYPTO_SUCCESS
) {
1271 ret
= SET_ERROR(EIO
);
1275 bcopy(raw_local_mac
, local_mac
, ZIO_OBJSET_MAC_LEN
);
1280 bzero(portable_mac
, ZIO_OBJSET_MAC_LEN
);
1281 bzero(local_mac
, ZIO_OBJSET_MAC_LEN
);
1286 zio_crypt_destroy_uio(uio_t
*uio
)
1289 kmem_free(uio
->uio_iov
, uio
->uio_iovcnt
* sizeof (iovec_t
));
1293 * This function parses an uncompressed indirect block and returns a checksum
1294 * of all the portable fields from all of the contained bps. The portable
1295 * fields are the MAC and all of the fields from blk_prop except for the dedup,
1296 * checksum, and psize bits. For an explanation of the purpose of this, see
1297 * the comment block on object set authentication.
1300 zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate
, void *buf
,
1301 uint_t datalen
, uint64_t version
, boolean_t byteswap
, uint8_t *cksum
)
1304 int i
, epb
= datalen
>> SPA_BLKPTRSHIFT
;
1306 uint8_t digestbuf
[SHA512_DIGEST_LENGTH
];
1308 /* checksum all of the MACs from the layer below */
1309 SHA2Init(SHA512
, &ctx
);
1310 for (i
= 0, bp
= buf
; i
< epb
; i
++, bp
++) {
1311 zio_crypt_bp_do_indrect_checksum_updates(&ctx
, version
,
1314 SHA2Final(digestbuf
, &ctx
);
1317 bcopy(digestbuf
, cksum
, ZIO_DATA_MAC_LEN
);
1321 if (bcmp(digestbuf
, cksum
, ZIO_DATA_MAC_LEN
) != 0)
1322 return (SET_ERROR(ECKSUM
));
1328 zio_crypt_do_indirect_mac_checksum(boolean_t generate
, void *buf
,
1329 uint_t datalen
, boolean_t byteswap
, uint8_t *cksum
)
1334 * Unfortunately, callers of this function will not always have
1335 * easy access to the on-disk format version. This info is
1336 * normally found in the DSL Crypto Key, but the checksum-of-MACs
1337 * is expected to be verifiable even when the key isn't loaded.
1338 * Here, instead of doing a ZAP lookup for the version for each
1339 * zio, we simply try both existing formats.
1341 ret
= zio_crypt_do_indirect_mac_checksum_impl(generate
, buf
,
1342 datalen
, ZIO_CRYPT_KEY_CURRENT_VERSION
, byteswap
, cksum
);
1343 if (ret
== ECKSUM
) {
1345 ret
= zio_crypt_do_indirect_mac_checksum_impl(generate
,
1346 buf
, datalen
, 0, byteswap
, cksum
);
1353 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate
, abd_t
*abd
,
1354 uint_t datalen
, boolean_t byteswap
, uint8_t *cksum
)
1359 buf
= abd_borrow_buf_copy(abd
, datalen
);
1360 ret
= zio_crypt_do_indirect_mac_checksum(generate
, buf
, datalen
,
1362 abd_return_buf(abd
, buf
, datalen
);
1368 * Special case handling routine for encrypting / decrypting ZIL blocks.
1369 * We do not check for the older ZIL chain because the encryption feature
1370 * was not available before the newer ZIL chain was introduced. The goal
1371 * here is to encrypt everything except the blkptr_t of a lr_write_t and
1372 * the zil_chain_t header. Everything that is not encrypted is authenticated.
1375 zio_crypt_init_uios_zil(boolean_t encrypt
, uint8_t *plainbuf
,
1376 uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
, uio_t
*puio
,
1377 uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
, uint_t
*auth_len
,
1378 boolean_t
*no_crypt
)
1381 uint64_t txtype
, lr_len
;
1382 uint_t nr_src
, nr_dst
, crypt_len
;
1383 uint_t aad_len
= 0, nr_iovecs
= 0, total_len
= 0;
1384 iovec_t
*src_iovecs
= NULL
, *dst_iovecs
= NULL
;
1385 uint8_t *src
, *dst
, *slrp
, *dlrp
, *blkend
, *aadp
;
1388 uint8_t *aadbuf
= zio_buf_alloc(datalen
);
1390 /* cipherbuf always needs an extra iovec for the MAC */
1403 /* find the start and end record of the log block */
1404 zilc
= (zil_chain_t
*)src
;
1405 slrp
= src
+ sizeof (zil_chain_t
);
1407 blkend
= src
+ ((byteswap
) ? BSWAP_64(zilc
->zc_nused
) : zilc
->zc_nused
);
1409 /* calculate the number of encrypted iovecs we will need */
1410 for (; slrp
< blkend
; slrp
+= lr_len
) {
1414 txtype
= lr
->lrc_txtype
;
1415 lr_len
= lr
->lrc_reclen
;
1417 txtype
= BSWAP_64(lr
->lrc_txtype
);
1418 lr_len
= BSWAP_64(lr
->lrc_reclen
);
1422 if (txtype
== TX_WRITE
&& lr_len
!= sizeof (lr_write_t
))
1426 nr_src
+= nr_iovecs
;
1427 nr_dst
+= nr_iovecs
;
1429 /* allocate the iovec arrays */
1431 src_iovecs
= kmem_alloc(nr_src
* sizeof (iovec_t
), KM_SLEEP
);
1432 if (src_iovecs
== NULL
) {
1433 ret
= SET_ERROR(ENOMEM
);
1439 dst_iovecs
= kmem_alloc(nr_dst
* sizeof (iovec_t
), KM_SLEEP
);
1440 if (dst_iovecs
== NULL
) {
1441 ret
= SET_ERROR(ENOMEM
);
1447 * Copy the plain zil header over and authenticate everything except
1448 * the checksum that will store our MAC. If we are writing the data
1449 * the embedded checksum will not have been calculated yet, so we don't
1450 * authenticate that.
1452 bcopy(src
, dst
, sizeof (zil_chain_t
));
1453 bcopy(src
, aadp
, sizeof (zil_chain_t
) - sizeof (zio_eck_t
));
1454 aadp
+= sizeof (zil_chain_t
) - sizeof (zio_eck_t
);
1455 aad_len
+= sizeof (zil_chain_t
) - sizeof (zio_eck_t
);
1457 /* loop over records again, filling in iovecs */
1459 slrp
= src
+ sizeof (zil_chain_t
);
1460 dlrp
= dst
+ sizeof (zil_chain_t
);
1462 for (; slrp
< blkend
; slrp
+= lr_len
, dlrp
+= lr_len
) {
1466 txtype
= lr
->lrc_txtype
;
1467 lr_len
= lr
->lrc_reclen
;
1469 txtype
= BSWAP_64(lr
->lrc_txtype
);
1470 lr_len
= BSWAP_64(lr
->lrc_reclen
);
1473 /* copy the common lr_t */
1474 bcopy(slrp
, dlrp
, sizeof (lr_t
));
1475 bcopy(slrp
, aadp
, sizeof (lr_t
));
1476 aadp
+= sizeof (lr_t
);
1477 aad_len
+= sizeof (lr_t
);
1479 ASSERT3P(src_iovecs
, !=, NULL
);
1480 ASSERT3P(dst_iovecs
, !=, NULL
);
1483 * If this is a TX_WRITE record we want to encrypt everything
1484 * except the bp if exists. If the bp does exist we want to
1487 if (txtype
== TX_WRITE
) {
1488 crypt_len
= sizeof (lr_write_t
) -
1489 sizeof (lr_t
) - sizeof (blkptr_t
);
1490 src_iovecs
[nr_iovecs
].iov_base
= slrp
+ sizeof (lr_t
);
1491 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1492 dst_iovecs
[nr_iovecs
].iov_base
= dlrp
+ sizeof (lr_t
);
1493 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1495 /* copy the bp now since it will not be encrypted */
1496 bcopy(slrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1497 dlrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1499 bcopy(slrp
+ sizeof (lr_write_t
) - sizeof (blkptr_t
),
1500 aadp
, sizeof (blkptr_t
));
1501 aadp
+= sizeof (blkptr_t
);
1502 aad_len
+= sizeof (blkptr_t
);
1504 total_len
+= crypt_len
;
1506 if (lr_len
!= sizeof (lr_write_t
)) {
1507 crypt_len
= lr_len
- sizeof (lr_write_t
);
1508 src_iovecs
[nr_iovecs
].iov_base
=
1509 slrp
+ sizeof (lr_write_t
);
1510 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1511 dst_iovecs
[nr_iovecs
].iov_base
=
1512 dlrp
+ sizeof (lr_write_t
);
1513 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1515 total_len
+= crypt_len
;
1518 crypt_len
= lr_len
- sizeof (lr_t
);
1519 src_iovecs
[nr_iovecs
].iov_base
= slrp
+ sizeof (lr_t
);
1520 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1521 dst_iovecs
[nr_iovecs
].iov_base
= dlrp
+ sizeof (lr_t
);
1522 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1524 total_len
+= crypt_len
;
1528 *no_crypt
= (nr_iovecs
== 0);
1529 *enc_len
= total_len
;
1531 *auth_len
= aad_len
;
1534 puio
->uio_iov
= src_iovecs
;
1535 puio
->uio_iovcnt
= nr_src
;
1536 cuio
->uio_iov
= dst_iovecs
;
1537 cuio
->uio_iovcnt
= nr_dst
;
1539 puio
->uio_iov
= dst_iovecs
;
1540 puio
->uio_iovcnt
= nr_dst
;
1541 cuio
->uio_iov
= src_iovecs
;
1542 cuio
->uio_iovcnt
= nr_src
;
1548 zio_buf_free(aadbuf
, datalen
);
1549 if (src_iovecs
!= NULL
)
1550 kmem_free(src_iovecs
, nr_src
* sizeof (iovec_t
));
1551 if (dst_iovecs
!= NULL
)
1552 kmem_free(dst_iovecs
, nr_dst
* sizeof (iovec_t
));
1557 *no_crypt
= B_FALSE
;
1558 puio
->uio_iov
= NULL
;
1559 puio
->uio_iovcnt
= 0;
1560 cuio
->uio_iov
= NULL
;
1561 cuio
->uio_iovcnt
= 0;
1566 * Special case handling routine for encrypting / decrypting dnode blocks.
1569 zio_crypt_init_uios_dnode(boolean_t encrypt
, uint64_t version
,
1570 uint8_t *plainbuf
, uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
,
1571 uio_t
*puio
, uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
,
1572 uint_t
*auth_len
, boolean_t
*no_crypt
)
1575 uint_t nr_src
, nr_dst
, crypt_len
;
1576 uint_t aad_len
= 0, nr_iovecs
= 0, total_len
= 0;
1577 uint_t i
, j
, max_dnp
= datalen
>> DNODE_SHIFT
;
1578 iovec_t
*src_iovecs
= NULL
, *dst_iovecs
= NULL
;
1579 uint8_t *src
, *dst
, *aadp
;
1580 dnode_phys_t
*dnp
, *adnp
, *sdnp
, *ddnp
;
1581 uint8_t *aadbuf
= zio_buf_alloc(datalen
);
1595 sdnp
= (dnode_phys_t
*)src
;
1596 ddnp
= (dnode_phys_t
*)dst
;
1600 * Count the number of iovecs we will need to do the encryption by
1601 * counting the number of bonus buffers that need to be encrypted.
1603 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
1605 * This block may still be byteswapped. However, all of the
1606 * values we use are either uint8_t's (for which byteswapping
1607 * is a noop) or a * != 0 check, which will work regardless
1608 * of whether or not we byteswap.
1610 if (sdnp
[i
].dn_type
!= DMU_OT_NONE
&&
1611 DMU_OT_IS_ENCRYPTED(sdnp
[i
].dn_bonustype
) &&
1612 sdnp
[i
].dn_bonuslen
!= 0) {
1617 nr_src
+= nr_iovecs
;
1618 nr_dst
+= nr_iovecs
;
1621 src_iovecs
= kmem_alloc(nr_src
* sizeof (iovec_t
), KM_SLEEP
);
1622 if (src_iovecs
== NULL
) {
1623 ret
= SET_ERROR(ENOMEM
);
1629 dst_iovecs
= kmem_alloc(nr_dst
* sizeof (iovec_t
), KM_SLEEP
);
1630 if (dst_iovecs
== NULL
) {
1631 ret
= SET_ERROR(ENOMEM
);
1639 * Iterate through the dnodes again, this time filling in the uios
1640 * we allocated earlier. We also concatenate any data we want to
1641 * authenticate onto aadbuf.
1643 for (i
= 0; i
< max_dnp
; i
+= sdnp
[i
].dn_extra_slots
+ 1) {
1646 /* copy over the core fields and blkptrs (kept as plaintext) */
1647 bcopy(dnp
, &ddnp
[i
], (uint8_t *)DN_BONUS(dnp
) - (uint8_t *)dnp
);
1649 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1650 bcopy(DN_SPILL_BLKPTR(dnp
), DN_SPILL_BLKPTR(&ddnp
[i
]),
1655 * Handle authenticated data. We authenticate everything in
1656 * the dnode that can be brought over when we do a raw send.
1657 * This includes all of the core fields as well as the MACs
1658 * stored in the bp checksums and all of the portable bits
1659 * from blk_prop. We include the dnode padding here in case it
1660 * ever gets used in the future. Some dn_flags and dn_used are
1661 * not portable so we mask those out values out of the
1662 * authenticated data.
1664 crypt_len
= offsetof(dnode_phys_t
, dn_blkptr
);
1665 bcopy(dnp
, aadp
, crypt_len
);
1666 adnp
= (dnode_phys_t
*)aadp
;
1667 adnp
->dn_flags
&= DNODE_CRYPT_PORTABLE_FLAGS_MASK
;
1670 aad_len
+= crypt_len
;
1672 for (j
= 0; j
< dnp
->dn_nblkptr
; j
++) {
1673 zio_crypt_bp_do_aad_updates(&aadp
, &aad_len
,
1674 version
, byteswap
, &dnp
->dn_blkptr
[j
]);
1677 if (dnp
->dn_flags
& DNODE_FLAG_SPILL_BLKPTR
) {
1678 zio_crypt_bp_do_aad_updates(&aadp
, &aad_len
,
1679 version
, byteswap
, DN_SPILL_BLKPTR(dnp
));
1683 * If this bonus buffer needs to be encrypted, we prepare an
1684 * iovec_t. The encryption / decryption functions will fill
1685 * this in for us with the encrypted or decrypted data.
1686 * Otherwise we add the bonus buffer to the authenticated
1687 * data buffer and copy it over to the destination. The
1688 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1689 * we can guarantee alignment with the AES block size
1692 crypt_len
= DN_MAX_BONUS_LEN(dnp
);
1693 if (dnp
->dn_type
!= DMU_OT_NONE
&&
1694 DMU_OT_IS_ENCRYPTED(dnp
->dn_bonustype
) &&
1695 dnp
->dn_bonuslen
!= 0) {
1696 ASSERT3U(nr_iovecs
, <, nr_src
);
1697 ASSERT3U(nr_iovecs
, <, nr_dst
);
1698 ASSERT3P(src_iovecs
, !=, NULL
);
1699 ASSERT3P(dst_iovecs
, !=, NULL
);
1700 src_iovecs
[nr_iovecs
].iov_base
= DN_BONUS(dnp
);
1701 src_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1702 dst_iovecs
[nr_iovecs
].iov_base
= DN_BONUS(&ddnp
[i
]);
1703 dst_iovecs
[nr_iovecs
].iov_len
= crypt_len
;
1706 total_len
+= crypt_len
;
1708 bcopy(DN_BONUS(dnp
), DN_BONUS(&ddnp
[i
]), crypt_len
);
1709 bcopy(DN_BONUS(dnp
), aadp
, crypt_len
);
1711 aad_len
+= crypt_len
;
1715 *no_crypt
= (nr_iovecs
== 0);
1716 *enc_len
= total_len
;
1718 *auth_len
= aad_len
;
1721 puio
->uio_iov
= src_iovecs
;
1722 puio
->uio_iovcnt
= nr_src
;
1723 cuio
->uio_iov
= dst_iovecs
;
1724 cuio
->uio_iovcnt
= nr_dst
;
1726 puio
->uio_iov
= dst_iovecs
;
1727 puio
->uio_iovcnt
= nr_dst
;
1728 cuio
->uio_iov
= src_iovecs
;
1729 cuio
->uio_iovcnt
= nr_src
;
1735 zio_buf_free(aadbuf
, datalen
);
1736 if (src_iovecs
!= NULL
)
1737 kmem_free(src_iovecs
, nr_src
* sizeof (iovec_t
));
1738 if (dst_iovecs
!= NULL
)
1739 kmem_free(dst_iovecs
, nr_dst
* sizeof (iovec_t
));
1744 *no_crypt
= B_FALSE
;
1745 puio
->uio_iov
= NULL
;
1746 puio
->uio_iovcnt
= 0;
1747 cuio
->uio_iov
= NULL
;
1748 cuio
->uio_iovcnt
= 0;
1753 zio_crypt_init_uios_normal(boolean_t encrypt
, uint8_t *plainbuf
,
1754 uint8_t *cipherbuf
, uint_t datalen
, uio_t
*puio
, uio_t
*cuio
,
1758 uint_t nr_plain
= 1, nr_cipher
= 2;
1759 iovec_t
*plain_iovecs
= NULL
, *cipher_iovecs
= NULL
;
1761 /* allocate the iovecs for the plain and cipher data */
1762 plain_iovecs
= kmem_alloc(nr_plain
* sizeof (iovec_t
),
1764 if (!plain_iovecs
) {
1765 ret
= SET_ERROR(ENOMEM
);
1769 cipher_iovecs
= kmem_alloc(nr_cipher
* sizeof (iovec_t
),
1771 if (!cipher_iovecs
) {
1772 ret
= SET_ERROR(ENOMEM
);
1776 plain_iovecs
[0].iov_base
= plainbuf
;
1777 plain_iovecs
[0].iov_len
= datalen
;
1778 cipher_iovecs
[0].iov_base
= cipherbuf
;
1779 cipher_iovecs
[0].iov_len
= datalen
;
1782 puio
->uio_iov
= plain_iovecs
;
1783 puio
->uio_iovcnt
= nr_plain
;
1784 cuio
->uio_iov
= cipher_iovecs
;
1785 cuio
->uio_iovcnt
= nr_cipher
;
1790 if (plain_iovecs
!= NULL
)
1791 kmem_free(plain_iovecs
, nr_plain
* sizeof (iovec_t
));
1792 if (cipher_iovecs
!= NULL
)
1793 kmem_free(cipher_iovecs
, nr_cipher
* sizeof (iovec_t
));
1796 puio
->uio_iov
= NULL
;
1797 puio
->uio_iovcnt
= 0;
1798 cuio
->uio_iov
= NULL
;
1799 cuio
->uio_iovcnt
= 0;
1804 * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1805 * that they can be used for encryption and decryption by zio_do_crypt_uio().
1806 * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1807 * requiring special handling to parse out pieces that are to be encrypted. The
1808 * authbuf is used by these special cases to store additional authenticated
1809 * data (AAD) for the encryption modes.
1812 zio_crypt_init_uios(boolean_t encrypt
, uint64_t version
, dmu_object_type_t ot
,
1813 uint8_t *plainbuf
, uint8_t *cipherbuf
, uint_t datalen
, boolean_t byteswap
,
1814 uint8_t *mac
, uio_t
*puio
, uio_t
*cuio
, uint_t
*enc_len
, uint8_t **authbuf
,
1815 uint_t
*auth_len
, boolean_t
*no_crypt
)
1820 ASSERT(DMU_OT_IS_ENCRYPTED(ot
) || ot
== DMU_OT_NONE
);
1822 /* route to handler */
1824 case DMU_OT_INTENT_LOG
:
1825 ret
= zio_crypt_init_uios_zil(encrypt
, plainbuf
, cipherbuf
,
1826 datalen
, byteswap
, puio
, cuio
, enc_len
, authbuf
, auth_len
,
1830 ret
= zio_crypt_init_uios_dnode(encrypt
, version
, plainbuf
,
1831 cipherbuf
, datalen
, byteswap
, puio
, cuio
, enc_len
, authbuf
,
1832 auth_len
, no_crypt
);
1835 ret
= zio_crypt_init_uios_normal(encrypt
, plainbuf
, cipherbuf
,
1836 datalen
, puio
, cuio
, enc_len
);
1839 *no_crypt
= B_FALSE
;
1846 /* populate the uios */
1847 puio
->uio_segflg
= UIO_SYSSPACE
;
1848 cuio
->uio_segflg
= UIO_SYSSPACE
;
1850 mac_iov
= ((iovec_t
*)&cuio
->uio_iov
[cuio
->uio_iovcnt
- 1]);
1851 mac_iov
->iov_base
= mac
;
1852 mac_iov
->iov_len
= ZIO_DATA_MAC_LEN
;
1861 * Primary encryption / decryption entrypoint for zio data.
1864 zio_do_crypt_data(boolean_t encrypt
, zio_crypt_key_t
*key
,
1865 dmu_object_type_t ot
, boolean_t byteswap
, uint8_t *salt
, uint8_t *iv
,
1866 uint8_t *mac
, uint_t datalen
, uint8_t *plainbuf
, uint8_t *cipherbuf
,
1867 boolean_t
*no_crypt
)
1870 boolean_t locked
= B_FALSE
;
1871 uint64_t crypt
= key
->zk_crypt
;
1872 uint_t keydata_len
= zio_crypt_table
[crypt
].ci_keylen
;
1873 uint_t enc_len
, auth_len
;
1875 uint8_t enc_keydata
[MASTER_KEY_MAX_LEN
];
1876 crypto_key_t tmp_ckey
, *ckey
= NULL
;
1877 crypto_ctx_template_t tmpl
;
1878 uint8_t *authbuf
= NULL
;
1881 * If the needed key is the current one, just use it. Otherwise we
1882 * need to generate a temporary one from the given salt + master key.
1883 * If we are encrypting, we must return a copy of the current salt
1884 * so that it can be stored in the blkptr_t.
1886 rw_enter(&key
->zk_salt_lock
, RW_READER
);
1889 if (bcmp(salt
, key
->zk_salt
, ZIO_DATA_SALT_LEN
) == 0) {
1890 ckey
= &key
->zk_current_key
;
1891 tmpl
= key
->zk_current_tmpl
;
1893 rw_exit(&key
->zk_salt_lock
);
1896 ret
= hkdf_sha512(key
->zk_master_keydata
, keydata_len
, NULL
, 0,
1897 salt
, ZIO_DATA_SALT_LEN
, enc_keydata
, keydata_len
);
1901 tmp_ckey
.ck_format
= CRYPTO_KEY_RAW
;
1902 tmp_ckey
.ck_data
= enc_keydata
;
1903 tmp_ckey
.ck_length
= CRYPTO_BYTES2BITS(keydata_len
);
1910 * Attempt to use QAT acceleration if we can. We currently don't
1911 * do this for metadnode and ZIL blocks, since they have a much
1912 * more involved buffer layout and the qat_crypt() function only
1915 if (qat_crypt_use_accel(datalen
) &&
1916 ot
!= DMU_OT_INTENT_LOG
&& ot
!= DMU_OT_DNODE
) {
1917 uint8_t *srcbuf
, *dstbuf
;
1927 ret
= qat_crypt((encrypt
) ? QAT_ENCRYPT
: QAT_DECRYPT
, srcbuf
,
1928 dstbuf
, NULL
, 0, iv
, mac
, ckey
, key
->zk_crypt
, datalen
);
1929 if (ret
== CPA_STATUS_SUCCESS
) {
1931 rw_exit(&key
->zk_salt_lock
);
1937 /* If the hardware implementation fails fall back to software */
1940 bzero(&puio
, sizeof (uio_t
));
1941 bzero(&cuio
, sizeof (uio_t
));
1943 /* create uios for encryption */
1944 ret
= zio_crypt_init_uios(encrypt
, key
->zk_version
, ot
, plainbuf
,
1945 cipherbuf
, datalen
, byteswap
, mac
, &puio
, &cuio
, &enc_len
,
1946 &authbuf
, &auth_len
, no_crypt
);
1950 /* perform the encryption / decryption in software */
1951 ret
= zio_do_crypt_uio(encrypt
, key
->zk_crypt
, ckey
, tmpl
, iv
, enc_len
,
1952 &puio
, &cuio
, authbuf
, auth_len
);
1957 rw_exit(&key
->zk_salt_lock
);
1961 if (authbuf
!= NULL
)
1962 zio_buf_free(authbuf
, datalen
);
1963 if (ckey
== &tmp_ckey
)
1964 bzero(enc_keydata
, keydata_len
);
1965 zio_crypt_destroy_uio(&puio
);
1966 zio_crypt_destroy_uio(&cuio
);
1972 rw_exit(&key
->zk_salt_lock
);
1973 if (authbuf
!= NULL
)
1974 zio_buf_free(authbuf
, datalen
);
1975 if (ckey
== &tmp_ckey
)
1976 bzero(enc_keydata
, keydata_len
);
1977 zio_crypt_destroy_uio(&puio
);
1978 zio_crypt_destroy_uio(&cuio
);
1984 * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1988 zio_do_crypt_abd(boolean_t encrypt
, zio_crypt_key_t
*key
, dmu_object_type_t ot
,
1989 boolean_t byteswap
, uint8_t *salt
, uint8_t *iv
, uint8_t *mac
,
1990 uint_t datalen
, abd_t
*pabd
, abd_t
*cabd
, boolean_t
*no_crypt
)
1996 ptmp
= abd_borrow_buf_copy(pabd
, datalen
);
1997 ctmp
= abd_borrow_buf(cabd
, datalen
);
1999 ptmp
= abd_borrow_buf(pabd
, datalen
);
2000 ctmp
= abd_borrow_buf_copy(cabd
, datalen
);
2003 ret
= zio_do_crypt_data(encrypt
, key
, ot
, byteswap
, salt
, iv
, mac
,
2004 datalen
, ptmp
, ctmp
, no_crypt
);
2009 abd_return_buf(pabd
, ptmp
, datalen
);
2010 abd_return_buf_copy(cabd
, ctmp
, datalen
);
2012 abd_return_buf_copy(pabd
, ptmp
, datalen
);
2013 abd_return_buf(cabd
, ctmp
, datalen
);
2020 abd_return_buf(pabd
, ptmp
, datalen
);
2021 abd_return_buf_copy(cabd
, ctmp
, datalen
);
2023 abd_return_buf_copy(pabd
, ptmp
, datalen
);
2024 abd_return_buf(cabd
, ctmp
, datalen
);
2030 #if defined(_KERNEL)
2032 module_param(zfs_key_max_salt_uses
, ulong
, 0644);
2033 MODULE_PARM_DESC(zfs_key_max_salt_uses
, "Max number of times a salt value "
2034 "can be used for generating encryption keys before it is rotated");