]>
Commit | Line | Data |
---|---|---|
b5256303 TC |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * This file and its contents are supplied under the terms of the | |
5 | * Common Development and Distribution License ("CDDL"), version 1.0. | |
6 | * You may only use this file in accordance with the terms of version | |
7 | * 1.0 of the CDDL. | |
8 | * | |
9 | * A full copy of the text of the CDDL should have accompanied this | |
10 | * source. A copy of the CDDL is also available via the Internet at | |
11 | * http://www.illumos.org/license/CDDL. | |
12 | * | |
13 | * CDDL HEADER END | |
14 | */ | |
15 | ||
16 | /* | |
17 | * Copyright (c) 2017, Datto, Inc. All rights reserved. | |
18 | */ | |
19 | ||
20 | #include <sys/zio_crypt.h> | |
21 | #include <sys/dmu.h> | |
22 | #include <sys/dmu_objset.h> | |
23 | #include <sys/dnode.h> | |
24 | #include <sys/fs/zfs.h> | |
25 | #include <sys/zio.h> | |
26 | #include <sys/zil.h> | |
27 | #include <sys/sha2.h> | |
4807c0ba | 28 | #include <sys/hkdf.h> |
b5256303 TC |
29 | |
30 | /* | |
31 | * This file is responsible for handling all of the details of generating | |
32 | * encryption parameters and performing encryption and authentication. | |
33 | * | |
34 | * BLOCK ENCRYPTION PARAMETERS: | |
35 | * Encryption /Authentication Algorithm Suite (crypt): | |
36 | * The encryption algorithm, mode, and key length we are going to use. We | |
37 | * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit | |
38 | * keys. All authentication is currently done with SHA512-HMAC. | |
39 | * | |
40 | * Plaintext: | |
41 | * The unencrypted data that we want to encrypt. | |
42 | * | |
43 | * Initialization Vector (IV): | |
44 | * An initialization vector for the encryption algorithms. This is used to | |
45 | * "tweak" the encryption algorithms so that two blocks of the same data are | |
46 | * encrypted into different ciphertext outputs, thus obfuscating block patterns. | |
47 | * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is | |
48 | * never reused with the same encryption key. This value is stored unencrypted | |
49 | * and must simply be provided to the decryption function. We use a 96 bit IV | |
50 | * (as recommended by NIST) for all block encryption. For non-dedup blocks we | |
51 | * derive the IV randomly. The first 64 bits of the IV are stored in the second | |
52 | * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of | |
53 | * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits | |
54 | * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count | |
55 | * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of | |
56 | * level 0 blocks is the number of allocated dnodes in that block. The on-disk | |
57 | * format supports at most 2^15 slots per L0 dnode block, because the maximum | |
58 | * block size is 16MB (2^24). In either case, for level 0 blocks this number | |
59 | * will still be smaller than UINT32_MAX so it is safe to store the IV in the | |
60 | * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count | |
61 | * for the dnode code. | |
62 | * | |
63 | * Master key: | |
64 | * This is the most important secret data of an encrypted dataset. It is used | |
65 | * along with the salt to generate that actual encryption keys via HKDF. We | |
66 | * do not use the master key to directly encrypt any data because there are | |
67 | * theoretical limits on how much data can actually be safely encrypted with | |
68 | * any encryption mode. The master key is stored encrypted on disk with the | |
69 | * user's wrapping key. Its length is determined by the encryption algorithm. | |
70 | * For details on how this is stored see the block comment in dsl_crypt.c | |
71 | * | |
72 | * Salt: | |
73 | * Used as an input to the HKDF function, along with the master key. We use a | |
74 | * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt | |
75 | * can be used for encrypting many blocks, so we cache the current salt and the | |
76 | * associated derived key in zio_crypt_t so we do not need to derive it again | |
77 | * needlessly. | |
78 | * | |
79 | * Encryption Key: | |
80 | * A secret binary key, generated from an HKDF function used to encrypt and | |
81 | * decrypt data. | |
82 | * | |
83 | * Message Authenication Code (MAC) | |
84 | * The MAC is an output of authenticated encryption modes such as AES-GCM and | |
85 | * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted | |
86 | * data on disk and return garbage to the application. Effectively, it is a | |
87 | * checksum that can not be reproduced by an attacker. We store the MAC in the | |
88 | * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated | |
89 | * regular checksum of the ciphertext which can be used for scrubbing. | |
90 | * | |
91 | * OBJECT AUTHENTICATION: | |
92 | * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because | |
93 | * they contain some info that always needs to be readable. To prevent this | |
94 | * data from being altered, we authenticate this data using SHA512-HMAC. This | |
95 | * will produce a MAC (similar to the one produced via encryption) which can | |
96 | * be used to verify the object was not modified. HMACs do not require key | |
97 | * rotation or IVs, so we can keep up to the full 3 copies of authenticated | |
98 | * data. | |
99 | * | |
100 | * ZIL ENCRYPTION: | |
101 | * ZIL blocks have their bp written to disk ahead of the associated data, so we | |
102 | * cannot store the MAC there as we normally do. For these blocks the MAC is | |
103 | * stored in the embedded checksum within the zil_chain_t header. The salt and | |
104 | * IV are generated for the block on bp allocation instead of at encryption | |
105 | * time. In addition, ZIL blocks have some pieces that must be left in plaintext | |
106 | * for claiming even though all of the sensitive user data still needs to be | |
107 | * encrypted. The function zio_crypt_init_uios_zil() handles parsing which | |
108 | * pieces of the block need to be encrypted. All data that is not encrypted is | |
109 | * authenticated using the AAD mechanisms that the supported encryption modes | |
110 | * provide for. In order to preserve the semantics of the ZIL for encrypted | |
111 | * datasets, the ZIL is not protected at the objset level as described below. | |
112 | * | |
113 | * DNODE ENCRYPTION: | |
114 | * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left | |
115 | * in plaintext for scrubbing and claiming, but the bonus buffers might contain | |
116 | * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing | |
117 | * which which pieces of the block need to be encrypted. For more details about | |
118 | * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). | |
119 | * | |
120 | * OBJECT SET AUTHENTICATION: | |
121 | * Up to this point, everything we have encrypted and authenticated has been | |
122 | * at level 0 (or -2 for the ZIL). If we did not do any further work the | |
123 | * on-disk format would be susceptible to attacks that deleted or rearrannged | |
124 | * the order of level 0 blocks. Ideally, the cleanest solution would be to | |
125 | * maintain a tree of authentication MACs going up the bp tree. However, this | |
126 | * presents a problem for raw sends. Send files do not send information about | |
127 | * indirect blocks so there would be no convenient way to transfer the MACs and | |
128 | * they cannot be recalculated on the receive side without the master key which | |
129 | * would defeat one of the purposes of raw sends in the first place. Instead, | |
130 | * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs | |
131 | * from the level below. We also include some portable fields from blk_prop such | |
132 | * as the lsize and compression algorithm to prevent the data from being | |
133 | * misinterpretted. | |
134 | * | |
135 | * At the objset level, we maintain 2 seperate 256 bit MACs in the | |
136 | * objset_phys_t. The first one is "portable" and is the logical root of the | |
137 | * MAC tree maintianed in the metadnode's bps. The second, is "local" and is | |
138 | * used as the root MAC for the user accounting objects, which are also not | |
139 | * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload | |
140 | * of the send file. The useraccounting code ensures that the useraccounting | |
141 | * info is not present upon a receive, so the local MAC can simply be cleared | |
142 | * out at that time. For more info about objset_phys_t authentication, see | |
143 | * zio_crypt_do_objset_hmacs(). | |
144 | * | |
145 | * CONSIDERATIONS FOR DEDUP: | |
146 | * In order for dedup to work, blocks that we want to dedup with one another | |
147 | * need to use the same IV and encryption key, so that they will have the same | |
148 | * ciphertext. Normally, one should never reuse an IV with the same encryption | |
149 | * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both | |
150 | * blocks. In this case, however, since we are using the same plaindata as | |
151 | * well all that we end up with is a duplicate of the original ciphertext we | |
152 | * already had. As a result, an attacker with read access to the raw disk will | |
153 | * be able to tell which blocks are the same but this information is given away | |
154 | * by dedup anyway. In order to get the same IVs and encryption keys for | |
155 | * equivalent blocks of data we use an HMAC of the plaindata. We use an HMAC | |
156 | * here so that a reproducible checksum of the plaindata is never available to | |
157 | * the attacker. The HMAC key is kept alongside the master key, encrypted on | |
158 | * disk. The first 64 bits of the HMAC are used in place of the random salt, and | |
159 | * the next 96 bits are used as the IV. As a result of this mechanism, dedup | |
160 | * will only work within a clone family since encrypted dedup requires use of | |
161 | * the same master and HMAC keys. | |
162 | */ | |
163 | ||
164 | /* | |
165 | * After encrypting many blocks with the same key we may start to run up | |
166 | * against the theoretical limits of how much data can securely be encrypted | |
167 | * with a single key using the supported encryption modes. The most obvious | |
168 | * limitation is that our risk of generating 2 equivalent 96 bit IVs increases | |
169 | * the more IVs we generate (which both GCM and CCM modes strictly forbid). | |
170 | * This risk actually grows surprisingly quickly over time according to the | |
171 | * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have | |
172 | * generated n IVs with a cryptographically secure RNG, the approximate | |
173 | * probability p(n) of a collision is given as: | |
174 | * | |
175 | * p(n) ~= e^(-n*(n-1)/(2*(2^96))) | |
176 | * | |
177 | * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] | |
178 | * | |
179 | * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion | |
180 | * we must not write more than 398,065,730 blocks with the same encryption key. | |
181 | * Therefore, we rotate our keys after 400,000,000 blocks have been written by | |
182 | * generating a new random 64 bit salt for our HKDF encryption key generation | |
183 | * function. | |
184 | */ | |
185 | #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 | |
186 | #define ZFS_CURRENT_MAX_SALT_USES \ | |
187 | (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) | |
188 | unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; | |
189 | ||
ae76f45c TC |
190 | typedef struct blkptr_auth_buf { |
191 | uint64_t bab_prop; /* blk_prop - portable mask */ | |
192 | uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */\r | |
193 | uint64_t bab_pad; /* reserved for future use */ | |
194 | } blkptr_auth_buf_t; | |
195 | ||
b5256303 TC |
196 | zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { |
197 | {"", ZC_TYPE_NONE, 0, "inherit"}, | |
198 | {"", ZC_TYPE_NONE, 0, "on"}, | |
199 | {"", ZC_TYPE_NONE, 0, "off"}, | |
200 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, | |
201 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, | |
202 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, | |
203 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, | |
204 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, | |
205 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} | |
206 | }; | |
207 | ||
b5256303 TC |
208 | void |
209 | zio_crypt_key_destroy(zio_crypt_key_t *key) | |
210 | { | |
211 | rw_destroy(&key->zk_salt_lock); | |
212 | ||
213 | /* free crypto templates */ | |
214 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
215 | crypto_destroy_ctx_template(key->zk_hmac_tmpl); | |
216 | ||
217 | /* zero out sensitive data */ | |
218 | bzero(key, sizeof (zio_crypt_key_t)); | |
219 | } | |
220 | ||
221 | int | |
222 | zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) | |
223 | { | |
224 | int ret; | |
225 | crypto_mechanism_t mech; | |
226 | uint_t keydata_len; | |
227 | ||
228 | ASSERT(key != NULL); | |
229 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
230 | ||
231 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
232 | bzero(key, sizeof (zio_crypt_key_t)); | |
233 | ||
234 | /* fill keydata buffers and salt with random data */ | |
235 | ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); | |
236 | if (ret != 0) | |
237 | goto error; | |
238 | ||
239 | ret = random_get_bytes(key->zk_master_keydata, keydata_len); | |
240 | if (ret != 0) | |
241 | goto error; | |
242 | ||
243 | ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); | |
244 | if (ret != 0) | |
245 | goto error; | |
246 | ||
247 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
248 | if (ret != 0) | |
249 | goto error; | |
250 | ||
251 | /* derive the current key from the master key */ | |
252 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
253 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
254 | keydata_len); | |
255 | if (ret != 0) | |
256 | goto error; | |
257 | ||
258 | /* initialize keys for the ICP */ | |
259 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
260 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 261 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
262 | |
263 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
264 | key->zk_hmac_key.ck_data = &key->zk_hmac_key; | |
4807c0ba | 265 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 TC |
266 | |
267 | /* | |
268 | * Initialize the crypto templates. It's ok if this fails because | |
269 | * this is just an optimization. | |
270 | */ | |
271 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
272 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
273 | &key->zk_current_tmpl, KM_SLEEP); | |
274 | if (ret != CRYPTO_SUCCESS) | |
275 | key->zk_current_tmpl = NULL; | |
276 | ||
277 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
278 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
279 | &key->zk_hmac_tmpl, KM_SLEEP); | |
280 | if (ret != CRYPTO_SUCCESS) | |
281 | key->zk_hmac_tmpl = NULL; | |
282 | ||
283 | key->zk_crypt = crypt; | |
ae76f45c | 284 | key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; |
b5256303 TC |
285 | key->zk_salt_count = 0; |
286 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); | |
287 | ||
288 | return (0); | |
289 | ||
290 | error: | |
291 | zio_crypt_key_destroy(key); | |
292 | return (ret); | |
293 | } | |
294 | ||
295 | static int | |
296 | zio_crypt_key_change_salt(zio_crypt_key_t *key) | |
297 | { | |
298 | int ret = 0; | |
299 | uint8_t salt[ZIO_DATA_SALT_LEN]; | |
300 | crypto_mechanism_t mech; | |
301 | uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; | |
302 | ||
303 | /* generate a new salt */ | |
304 | ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); | |
305 | if (ret != 0) | |
306 | goto error; | |
307 | ||
308 | rw_enter(&key->zk_salt_lock, RW_WRITER); | |
309 | ||
310 | /* someone beat us to the salt rotation, just unlock and return */ | |
311 | if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) | |
312 | goto out_unlock; | |
313 | ||
314 | /* derive the current key from the master key and the new salt */ | |
315 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
316 | salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); | |
317 | if (ret != 0) | |
318 | goto out_unlock; | |
319 | ||
320 | /* assign the salt and reset the usage count */ | |
321 | bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); | |
322 | key->zk_salt_count = 0; | |
323 | ||
324 | /* destroy the old context template and create the new one */ | |
325 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
326 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
327 | &key->zk_current_tmpl, KM_SLEEP); | |
328 | if (ret != CRYPTO_SUCCESS) | |
329 | key->zk_current_tmpl = NULL; | |
330 | ||
331 | rw_exit(&key->zk_salt_lock); | |
332 | ||
333 | return (0); | |
334 | ||
335 | out_unlock: | |
336 | rw_exit(&key->zk_salt_lock); | |
337 | error: | |
338 | return (ret); | |
339 | } | |
340 | ||
341 | /* See comment above zfs_key_max_salt_uses definition for details */ | |
342 | int | |
343 | zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) | |
344 | { | |
345 | int ret; | |
346 | boolean_t salt_change; | |
347 | ||
348 | rw_enter(&key->zk_salt_lock, RW_READER); | |
349 | ||
350 | bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); | |
351 | salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= | |
352 | ZFS_CURRENT_MAX_SALT_USES); | |
353 | ||
354 | rw_exit(&key->zk_salt_lock); | |
355 | ||
356 | if (salt_change) { | |
357 | ret = zio_crypt_key_change_salt(key); | |
358 | if (ret != 0) | |
359 | goto error; | |
360 | } | |
361 | ||
362 | return (0); | |
363 | ||
364 | error: | |
365 | return (ret); | |
366 | } | |
367 | ||
368 | /* | |
369 | * This function handles all encryption and decryption in zfs. When | |
370 | * encrypting it expects puio to reference the plaintext and cuio to | |
371 | * reference the cphertext. cuio must have enough space for the | |
372 | * ciphertext + room for a MAC. datalen should be the length of the | |
373 | * plaintext / ciphertext alone. | |
374 | */ | |
375 | static int | |
376 | zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, | |
377 | crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, | |
378 | uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len) | |
379 | { | |
380 | int ret; | |
381 | crypto_data_t plaindata, cipherdata; | |
382 | CK_AES_CCM_PARAMS ccmp; | |
383 | CK_AES_GCM_PARAMS gcmp; | |
384 | crypto_mechanism_t mech; | |
385 | zio_crypt_info_t crypt_info; | |
386 | uint_t plain_full_len, maclen; | |
387 | ||
388 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
389 | ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW); | |
390 | ||
391 | /* lookup the encryption info */ | |
392 | crypt_info = zio_crypt_table[crypt]; | |
393 | ||
394 | /* the mac will always be the last iovec_t in the cipher uio */ | |
395 | maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; | |
396 | ||
397 | ASSERT(maclen <= ZIO_DATA_MAC_LEN); | |
398 | ||
399 | /* setup encryption mechanism (same as crypt) */ | |
400 | mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); | |
401 | ||
402 | /* | |
403 | * Strangely, the ICP requires that plain_full_len must include | |
404 | * the MAC length when decrypting, even though the UIO does not | |
405 | * need to have the extra space allocated. | |
406 | */ | |
407 | if (encrypt) { | |
408 | plain_full_len = datalen; | |
409 | } else { | |
410 | plain_full_len = datalen + maclen; | |
411 | } | |
412 | ||
413 | /* | |
414 | * setup encryption params (currently only AES CCM and AES GCM | |
415 | * are supported) | |
416 | */ | |
417 | if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { | |
418 | ccmp.ulNonceSize = ZIO_DATA_IV_LEN; | |
419 | ccmp.ulAuthDataSize = auth_len; | |
420 | ccmp.authData = authbuf; | |
421 | ccmp.ulMACSize = maclen; | |
422 | ccmp.nonce = ivbuf; | |
423 | ccmp.ulDataSize = plain_full_len; | |
424 | ||
425 | mech.cm_param = (char *)(&ccmp); | |
426 | mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); | |
427 | } else { | |
428 | gcmp.ulIvLen = ZIO_DATA_IV_LEN; | |
4807c0ba | 429 | gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); |
b5256303 TC |
430 | gcmp.ulAADLen = auth_len; |
431 | gcmp.pAAD = authbuf; | |
4807c0ba | 432 | gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); |
b5256303 TC |
433 | gcmp.pIv = ivbuf; |
434 | ||
435 | mech.cm_param = (char *)(&gcmp); | |
436 | mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); | |
437 | } | |
438 | ||
439 | /* populate the cipher and plain data structs. */ | |
440 | plaindata.cd_format = CRYPTO_DATA_UIO; | |
441 | plaindata.cd_offset = 0; | |
442 | plaindata.cd_uio = puio; | |
443 | plaindata.cd_miscdata = NULL; | |
444 | plaindata.cd_length = plain_full_len; | |
445 | ||
446 | cipherdata.cd_format = CRYPTO_DATA_UIO; | |
447 | cipherdata.cd_offset = 0; | |
448 | cipherdata.cd_uio = cuio; | |
449 | cipherdata.cd_miscdata = NULL; | |
450 | cipherdata.cd_length = datalen + maclen; | |
451 | ||
452 | /* perform the actual encryption */ | |
453 | if (encrypt) { | |
454 | ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata, | |
455 | NULL); | |
456 | if (ret != CRYPTO_SUCCESS) { | |
457 | ret = SET_ERROR(EIO); | |
458 | goto error; | |
459 | } | |
460 | } else { | |
461 | ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata, | |
462 | NULL); | |
463 | if (ret != CRYPTO_SUCCESS) { | |
464 | ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); | |
465 | ret = SET_ERROR(ECKSUM); | |
466 | goto error; | |
467 | } | |
468 | } | |
469 | ||
470 | return (0); | |
471 | ||
472 | error: | |
473 | return (ret); | |
474 | } | |
475 | ||
476 | int | |
477 | zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, | |
478 | uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) | |
479 | { | |
480 | int ret; | |
481 | uio_t puio, cuio; | |
ae76f45c | 482 | uint64_t aad[3]; |
b5256303 TC |
483 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
484 | uint64_t crypt = key->zk_crypt; | |
ae76f45c | 485 | uint_t enc_len, keydata_len, aad_len; |
b5256303 TC |
486 | |
487 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
488 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
489 | ||
490 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
491 | ||
492 | /* generate iv for wrapping the master and hmac key */ | |
493 | ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); | |
494 | if (ret != 0) | |
495 | goto error; | |
496 | ||
497 | /* initialize uio_ts */ | |
498 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
499 | plain_iovecs[0].iov_len = keydata_len; | |
500 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
501 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
502 | ||
503 | cipher_iovecs[0].iov_base = keydata_out; | |
504 | cipher_iovecs[0].iov_len = keydata_len; | |
505 | cipher_iovecs[1].iov_base = hmac_keydata_out; | |
506 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
507 | cipher_iovecs[2].iov_base = mac; | |
508 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
509 | ||
ae76f45c TC |
510 | /* |
511 | * Although we don't support writing to the old format, we do | |
512 | * support rewrapping the key so that the user can move and | |
513 | * quarantine datasets on the old format. | |
514 | */ | |
515 | if (key->zk_version == 0) { | |
516 | aad_len = sizeof (uint64_t); | |
517 | aad[0] = LE_64(key->zk_guid); | |
518 | } else { | |
519 | ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
520 | aad_len = sizeof (uint64_t) * 3; | |
521 | aad[0] = LE_64(key->zk_guid); | |
522 | aad[1] = LE_64(crypt); | |
523 | aad[2] = LE_64(key->zk_version); | |
524 | } | |
525 | ||
b5256303 TC |
526 | enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; |
527 | puio.uio_iov = plain_iovecs; | |
528 | puio.uio_iovcnt = 2; | |
529 | puio.uio_segflg = UIO_SYSSPACE; | |
530 | cuio.uio_iov = cipher_iovecs; | |
531 | cuio.uio_iovcnt = 3; | |
532 | cuio.uio_segflg = UIO_SYSSPACE; | |
533 | ||
534 | /* encrypt the keys and store the resulting ciphertext and mac */ | |
535 | ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, | |
ae76f45c | 536 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
537 | if (ret != 0) |
538 | goto error; | |
539 | ||
540 | return (0); | |
541 | ||
542 | error: | |
543 | return (ret); | |
544 | } | |
545 | ||
546 | int | |
ae76f45c TC |
547 | zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, |
548 | uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, | |
549 | uint8_t *mac, zio_crypt_key_t *key) | |
b5256303 TC |
550 | { |
551 | int ret; | |
552 | crypto_mechanism_t mech; | |
553 | uio_t puio, cuio; | |
ae76f45c | 554 | uint64_t aad[3]; |
b5256303 | 555 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
ae76f45c | 556 | uint_t enc_len, keydata_len, aad_len; |
b5256303 TC |
557 | |
558 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
559 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
560 | ||
561 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
562 | ||
563 | /* initialize uio_ts */ | |
564 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
565 | plain_iovecs[0].iov_len = keydata_len; | |
566 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
567 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
568 | ||
569 | cipher_iovecs[0].iov_base = keydata; | |
570 | cipher_iovecs[0].iov_len = keydata_len; | |
571 | cipher_iovecs[1].iov_base = hmac_keydata; | |
572 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
573 | cipher_iovecs[2].iov_base = mac; | |
574 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
575 | ||
ae76f45c TC |
576 | if (version == 0) { |
577 | aad_len = sizeof (uint64_t); | |
578 | aad[0] = LE_64(guid); | |
579 | } else { | |
580 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
581 | aad_len = sizeof (uint64_t) * 3; | |
582 | aad[0] = LE_64(guid); | |
583 | aad[1] = LE_64(crypt); | |
584 | aad[2] = LE_64(version); | |
585 | } | |
586 | ||
b5256303 TC |
587 | enc_len = keydata_len + SHA512_HMAC_KEYLEN; |
588 | puio.uio_iov = plain_iovecs; | |
589 | puio.uio_segflg = UIO_SYSSPACE; | |
590 | puio.uio_iovcnt = 2; | |
591 | cuio.uio_iov = cipher_iovecs; | |
592 | cuio.uio_iovcnt = 3; | |
593 | cuio.uio_segflg = UIO_SYSSPACE; | |
594 | ||
595 | /* decrypt the keys and store the result in the output buffers */ | |
596 | ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, | |
ae76f45c | 597 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
598 | if (ret != 0) |
599 | goto error; | |
600 | ||
601 | /* generate a fresh salt */ | |
602 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
603 | if (ret != 0) | |
604 | goto error; | |
605 | ||
606 | /* derive the current key from the master key */ | |
607 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
608 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
609 | keydata_len); | |
610 | if (ret != 0) | |
611 | goto error; | |
612 | ||
613 | /* initialize keys for ICP */ | |
614 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
615 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 616 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
617 | |
618 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
619 | key->zk_hmac_key.ck_data = key->zk_hmac_keydata; | |
4807c0ba | 620 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 TC |
621 | |
622 | /* | |
623 | * Initialize the crypto templates. It's ok if this fails because | |
624 | * this is just an optimization. | |
625 | */ | |
626 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
627 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
628 | &key->zk_current_tmpl, KM_SLEEP); | |
629 | if (ret != CRYPTO_SUCCESS) | |
630 | key->zk_current_tmpl = NULL; | |
631 | ||
632 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
633 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
634 | &key->zk_hmac_tmpl, KM_SLEEP); | |
635 | if (ret != CRYPTO_SUCCESS) | |
636 | key->zk_hmac_tmpl = NULL; | |
637 | ||
638 | key->zk_crypt = crypt; | |
ae76f45c | 639 | key->zk_version = version; |
b5256303 TC |
640 | key->zk_guid = guid; |
641 | key->zk_salt_count = 0; | |
642 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); | |
643 | ||
644 | return (0); | |
645 | ||
646 | error: | |
647 | zio_crypt_key_destroy(key); | |
648 | return (ret); | |
649 | } | |
650 | ||
651 | int | |
652 | zio_crypt_generate_iv(uint8_t *ivbuf) | |
653 | { | |
654 | int ret; | |
655 | ||
656 | /* randomly generate the IV */ | |
657 | ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); | |
658 | if (ret != 0) | |
659 | goto error; | |
660 | ||
661 | return (0); | |
662 | ||
663 | error: | |
664 | bzero(ivbuf, ZIO_DATA_IV_LEN); | |
665 | return (ret); | |
666 | } | |
667 | ||
668 | int | |
669 | zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, | |
4807c0ba | 670 | uint8_t *digestbuf, uint_t digestlen) |
b5256303 TC |
671 | { |
672 | int ret; | |
673 | crypto_mechanism_t mech; | |
674 | crypto_data_t in_data, digest_data; | |
4807c0ba TC |
675 | uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; |
676 | ||
677 | ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
678 | |
679 | /* initialize sha512-hmac mechanism and crypto data */ | |
680 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
681 | mech.cm_param = NULL; | |
682 | mech.cm_param_len = 0; | |
683 | ||
684 | /* initialize the crypto data */ | |
685 | in_data.cd_format = CRYPTO_DATA_RAW; | |
686 | in_data.cd_offset = 0; | |
687 | in_data.cd_length = datalen; | |
688 | in_data.cd_raw.iov_base = (char *)data; | |
689 | in_data.cd_raw.iov_len = in_data.cd_length; | |
690 | ||
691 | digest_data.cd_format = CRYPTO_DATA_RAW; | |
692 | digest_data.cd_offset = 0; | |
4807c0ba | 693 | digest_data.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
694 | digest_data.cd_raw.iov_base = (char *)raw_digestbuf; |
695 | digest_data.cd_raw.iov_len = digest_data.cd_length; | |
696 | ||
697 | /* generate the hmac */ | |
698 | ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, | |
699 | &digest_data, NULL); | |
700 | if (ret != CRYPTO_SUCCESS) { | |
701 | ret = SET_ERROR(EIO); | |
702 | goto error; | |
703 | } | |
704 | ||
4807c0ba | 705 | bcopy(raw_digestbuf, digestbuf, digestlen); |
b5256303 TC |
706 | |
707 | return (0); | |
708 | ||
709 | error: | |
4807c0ba | 710 | bzero(digestbuf, digestlen); |
b5256303 TC |
711 | return (ret); |
712 | } | |
713 | ||
714 | int | |
715 | zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, | |
716 | uint_t datalen, uint8_t *ivbuf, uint8_t *salt) | |
717 | { | |
718 | int ret; | |
4807c0ba | 719 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 | 720 | |
4807c0ba TC |
721 | ret = zio_crypt_do_hmac(key, data, datalen, |
722 | digestbuf, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
723 | if (ret != 0) |
724 | return (ret); | |
725 | ||
726 | bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); | |
727 | bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); | |
728 | ||
729 | return (0); | |
730 | } | |
731 | ||
732 | /* | |
733 | * The following functions are used to encode and decode encryption parameters | |
734 | * into blkptr_t and zil_header_t. The ICP wants to use these parameters as | |
735 | * byte strings, which normally means that these strings would not need to deal | |
736 | * with byteswapping at all. However, both blkptr_t and zil_header_t may be | |
737 | * byteswapped by lower layers and so we must "undo" that byteswap here upon | |
ae76f45c TC |
738 | * decoding and encoding in a non-native byteorder. These functions require |
739 | * that the byteorder bit is correct before being called. | |
b5256303 TC |
740 | */ |
741 | void | |
742 | zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
743 | { | |
ae76f45c | 744 | uint64_t val64; |
b5256303 TC |
745 | uint32_t val32; |
746 | ||
747 | ASSERT(BP_IS_ENCRYPTED(bp)); | |
748 | ||
ae76f45c TC |
749 | if (!BP_SHOULD_BYTESWAP(bp)) { |
750 | bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); | |
751 | bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); | |
752 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
753 | BP_SET_IV2(bp, val32); | |
754 | } else { | |
755 | bcopy(salt, &val64, sizeof (uint64_t)); | |
756 | bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); | |
757 | ||
758 | bcopy(iv, &val64, sizeof (uint64_t)); | |
759 | bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); | |
760 | ||
761 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
762 | BP_SET_IV2(bp, BSWAP_32(val32)); | |
763 | } | |
b5256303 TC |
764 | } |
765 | ||
766 | void | |
767 | zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
768 | { | |
769 | uint64_t val64; | |
770 | uint32_t val32; | |
771 | ||
772 | ASSERT(BP_IS_PROTECTED(bp)); | |
773 | ||
774 | /* for convenience, so callers don't need to check */ | |
775 | if (BP_IS_AUTHENTICATED(bp)) { | |
776 | bzero(salt, ZIO_DATA_SALT_LEN); | |
777 | bzero(iv, ZIO_DATA_IV_LEN); | |
778 | return; | |
779 | } | |
780 | ||
781 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
782 | bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); | |
783 | bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); | |
784 | ||
785 | val32 = (uint32_t)BP_GET_IV2(bp); | |
786 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
787 | } else { | |
788 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); | |
789 | bcopy(&val64, salt, sizeof (uint64_t)); | |
790 | ||
791 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); | |
792 | bcopy(&val64, iv, sizeof (uint64_t)); | |
793 | ||
794 | val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); | |
795 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
796 | } | |
797 | } | |
798 | ||
799 | void | |
800 | zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) | |
801 | { | |
ae76f45c TC |
802 | uint64_t val64; |
803 | ||
b5256303 TC |
804 | ASSERT(BP_USES_CRYPT(bp)); |
805 | ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); | |
806 | ||
ae76f45c TC |
807 | if (!BP_SHOULD_BYTESWAP(bp)) { |
808 | bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); | |
809 | bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], | |
810 | sizeof (uint64_t)); | |
811 | } else { | |
812 | bcopy(mac, &val64, sizeof (uint64_t)); | |
813 | bp->blk_cksum.zc_word[2] = BSWAP_64(val64); | |
814 | ||
815 | bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); | |
816 | bp->blk_cksum.zc_word[3] = BSWAP_64(val64); | |
817 | } | |
b5256303 TC |
818 | } |
819 | ||
820 | void | |
821 | zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) | |
822 | { | |
823 | uint64_t val64; | |
824 | ||
825 | ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); | |
826 | ||
827 | /* for convenience, so callers don't need to check */ | |
828 | if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { | |
829 | bzero(mac, ZIO_DATA_MAC_LEN); | |
830 | return; | |
831 | } | |
832 | ||
833 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
834 | bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
835 | bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), | |
836 | sizeof (uint64_t)); | |
837 | } else { | |
838 | val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); | |
839 | bcopy(&val64, mac, sizeof (uint64_t)); | |
840 | ||
841 | val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); | |
842 | bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); | |
843 | } | |
844 | } | |
845 | ||
846 | void | |
847 | zio_crypt_encode_mac_zil(void *data, uint8_t *mac) | |
848 | { | |
849 | zil_chain_t *zilc = data; | |
850 | ||
851 | bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); | |
852 | bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], | |
853 | sizeof (uint64_t)); | |
854 | } | |
855 | ||
856 | void | |
857 | zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) | |
858 | { | |
859 | /* | |
860 | * The ZIL MAC is embedded in the block it protects, which will | |
861 | * not have been byteswapped by the time this function has been called. | |
862 | * As a result, we don't need to worry about byteswapping the MAC. | |
863 | */ | |
864 | const zil_chain_t *zilc = data; | |
865 | ||
866 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
867 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), | |
868 | sizeof (uint64_t)); | |
869 | } | |
870 | ||
871 | /* | |
872 | * This routine takes a block of dnodes (src_abd) and copies only the bonus | |
873 | * buffers to the same offsets in the dst buffer. datalen should be the size | |
874 | * of both the src_abd and the dst buffer (not just the length of the bonus | |
875 | * buffers). | |
876 | */ | |
877 | void | |
878 | zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) | |
879 | { | |
880 | uint_t i, max_dnp = datalen >> DNODE_SHIFT; | |
881 | uint8_t *src; | |
882 | dnode_phys_t *dnp, *sdnp, *ddnp; | |
883 | ||
884 | src = abd_borrow_buf_copy(src_abd, datalen); | |
885 | ||
886 | sdnp = (dnode_phys_t *)src; | |
887 | ddnp = (dnode_phys_t *)dst; | |
888 | ||
889 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
890 | dnp = &sdnp[i]; | |
891 | if (dnp->dn_type != DMU_OT_NONE && | |
892 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
893 | dnp->dn_bonuslen != 0) { | |
894 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), | |
895 | DN_MAX_BONUS_LEN(dnp)); | |
896 | } | |
897 | } | |
898 | ||
899 | abd_return_buf(src_abd, src, datalen); | |
900 | } | |
901 | ||
ae76f45c TC |
902 | /* |
903 | * This function decides what fields from blk_prop are included in | |
904 | * the on-disk various MAC algorithms. | |
905 | */ | |
b5256303 | 906 | static void |
ae76f45c | 907 | zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) |
b5256303 | 908 | { |
ae76f45c TC |
909 | /* |
910 | * Version 0 did not properly zero out all non-portable fields | |
911 | * as it should have done. We maintain this code so that we can | |
912 | * do read-only imports of pools on this version. | |
913 | */ | |
914 | if (version == 0) { | |
915 | BP_SET_DEDUP(bp, 0); | |
916 | BP_SET_CHECKSUM(bp, 0); | |
917 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
918 | return; | |
919 | } | |
920 | ||
921 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
922 | ||
923 | /* | |
924 | * The hole_birth feature might set these fields even if this bp | |
925 | * is a hole. We zero them out here to guarantee that raw sends | |
926 | * will function with or without the feature. | |
927 | */ | |
928 | if (BP_IS_HOLE(bp)) { | |
929 | bp->blk_prop = 0ULL; | |
930 | return; | |
931 | } | |
b5256303 TC |
932 | |
933 | /* | |
ae76f45c TC |
934 | * At L0 we want to verify these fields to ensure that data blocks |
935 | * can not be reinterpretted. For instance, we do not want an attacker | |
936 | * to trick us into returning raw lz4 compressed data to the user | |
937 | * by modifying the compression bits. At higher levels, we cannot | |
938 | * enforce this policy since raw sends do not convey any information | |
939 | * about indirect blocks, so these values might be different on the | |
940 | * receive side. Fortunately, this does not open any new attack | |
941 | * vectors, since any alterations that can be made to a higher level | |
942 | * bp must still verify the correct order of the layer below it. | |
b5256303 | 943 | */ |
ae76f45c TC |
944 | if (BP_GET_LEVEL(bp) != 0) { |
945 | BP_SET_BYTEORDER(bp, 0); | |
946 | BP_SET_COMPRESS(bp, 0); | |
947 | ||
948 | /* | |
949 | * psize cannot be set to zero or it will trigger | |
950 | * asserts, but the value doesn't really matter as | |
951 | * long as it is constant. | |
952 | */ | |
953 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
954 | } | |
955 | ||
956 | BP_SET_DEDUP(bp, 0); | |
957 | BP_SET_CHECKSUM(bp, 0); | |
b5256303 TC |
958 | } |
959 | ||
ae76f45c TC |
960 | static void |
961 | zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, | |
962 | blkptr_auth_buf_t *bab, uint_t *bab_len) | |
b5256303 | 963 | { |
b5256303 | 964 | blkptr_t tmpbp = *bp; |
b5256303 TC |
965 | |
966 | if (should_bswap) | |
967 | byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); | |
968 | ||
969 | ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); | |
970 | ASSERT0(BP_IS_EMBEDDED(&tmpbp)); | |
b5256303 | 971 | |
ae76f45c | 972 | zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); |
b5256303 | 973 | |
ae76f45c TC |
974 | /* |
975 | * We always MAC blk_prop in LE to ensure portability. This | |
976 | * must be done after decoding the mac, since the endianness | |
977 | * will get zero'd out here. | |
978 | */ | |
979 | zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); | |
980 | bab->bab_prop = LE_64(tmpbp.blk_prop); | |
981 | bab->bab_pad = 0ULL; | |
982 | ||
983 | /* version 0 did not include the padding */ | |
984 | *bab_len = sizeof (blkptr_auth_buf_t); | |
985 | if (version == 0) | |
986 | *bab_len -= sizeof (uint64_t); | |
987 | } | |
b5256303 | 988 | |
ae76f45c TC |
989 | static int |
990 | zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, | |
991 | boolean_t should_bswap, blkptr_t *bp) | |
992 | { | |
993 | int ret; | |
994 | uint_t bab_len; | |
995 | blkptr_auth_buf_t bab; | |
996 | crypto_data_t cd; | |
b5256303 | 997 | |
ae76f45c TC |
998 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
999 | cd.cd_format = CRYPTO_DATA_RAW; | |
1000 | cd.cd_offset = 0; | |
1001 | cd.cd_length = bab_len; | |
1002 | cd.cd_raw.iov_base = (char *)&bab; | |
b5256303 TC |
1003 | cd.cd_raw.iov_len = cd.cd_length; |
1004 | ||
1005 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1006 | if (ret != CRYPTO_SUCCESS) { | |
1007 | ret = SET_ERROR(EIO); | |
1008 | goto error; | |
1009 | } | |
1010 | ||
1011 | return (0); | |
1012 | ||
1013 | error: | |
1014 | return (ret); | |
1015 | } | |
1016 | ||
1017 | static void | |
ae76f45c TC |
1018 | zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, |
1019 | boolean_t should_bswap, blkptr_t *bp) | |
b5256303 | 1020 | { |
ae76f45c TC |
1021 | uint_t bab_len; |
1022 | blkptr_auth_buf_t bab; | |
b5256303 | 1023 | |
ae76f45c TC |
1024 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1025 | SHA2Update(ctx, &bab, bab_len); | |
b5256303 TC |
1026 | } |
1027 | ||
1028 | static void | |
ae76f45c | 1029 | zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, |
b5256303 TC |
1030 | boolean_t should_bswap, blkptr_t *bp) |
1031 | { | |
ae76f45c TC |
1032 | uint_t bab_len; |
1033 | blkptr_auth_buf_t bab; | |
b5256303 | 1034 | |
ae76f45c TC |
1035 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1036 | bcopy(&bab, *aadp, bab_len); | |
1037 | *aadp += bab_len; | |
1038 | *aad_len += bab_len; | |
b5256303 TC |
1039 | } |
1040 | ||
1041 | static int | |
ae76f45c TC |
1042 | zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, |
1043 | boolean_t should_bswap, dnode_phys_t *dnp) | |
b5256303 TC |
1044 | { |
1045 | int ret, i; | |
1046 | dnode_phys_t *adnp; | |
1047 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
1048 | crypto_data_t cd; | |
1049 | uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; | |
1050 | ||
1051 | cd.cd_format = CRYPTO_DATA_RAW; | |
1052 | cd.cd_offset = 0; | |
1053 | ||
1054 | /* authenticate the core dnode (masking out non-portable bits) */ | |
1055 | bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); | |
1056 | adnp = (dnode_phys_t *)tmp_dncore; | |
1057 | if (le_bswap) { | |
1058 | adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); | |
1059 | adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); | |
1060 | adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); | |
1061 | adnp->dn_used = BSWAP_64(adnp->dn_used); | |
1062 | } | |
1063 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1064 | adnp->dn_used = 0; | |
1065 | ||
1066 | cd.cd_length = sizeof (tmp_dncore); | |
1067 | cd.cd_raw.iov_base = (char *)adnp; | |
1068 | cd.cd_raw.iov_len = cd.cd_length; | |
1069 | ||
1070 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1071 | if (ret != CRYPTO_SUCCESS) { | |
1072 | ret = SET_ERROR(EIO); | |
1073 | goto error; | |
1074 | } | |
1075 | ||
1076 | for (i = 0; i < dnp->dn_nblkptr; i++) { | |
ae76f45c | 1077 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1078 | should_bswap, &dnp->dn_blkptr[i]); |
1079 | if (ret != 0) | |
1080 | goto error; | |
1081 | } | |
1082 | ||
1083 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
ae76f45c | 1084 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1085 | should_bswap, DN_SPILL_BLKPTR(dnp)); |
1086 | if (ret != 0) | |
1087 | goto error; | |
1088 | } | |
1089 | ||
1090 | return (0); | |
1091 | ||
1092 | error: | |
1093 | return (ret); | |
1094 | } | |
1095 | ||
1096 | /* | |
1097 | * objset_phys_t blocks introduce a number of exceptions to the normal | |
1098 | * authentication process. objset_phys_t's contain 2 seperate HMACS for | |
1099 | * protecting the integrity of their data. The portable_mac protects the | |
1100 | * the metadnode. This MAC can be sent with a raw send and protects against | |
1101 | * reordering of data within the metadnode. The local_mac protects the user | |
1102 | * accounting objects which are not sent from one system to another. | |
1103 | * | |
1104 | * In addition, objset blocks are the only blocks that can be modified and | |
1105 | * written to disk without the key loaded under certain circumstances. During | |
1106 | * zil_claim() we need to be able to update the zil_header_t to complete | |
1107 | * claiming log blocks and during raw receives we need to write out the | |
1108 | * portable_mac from the send file. Both of these actions are possible | |
1109 | * because these fields are not protected by either MAC so neither one will | |
1110 | * need to modify the MACs without the key. However, when the modified blocks | |
1111 | * are written out they will be byteswapped into the host machine's native | |
1112 | * endianness which will modify fields protected by the MAC. As a result, MAC | |
1113 | * calculation for objset blocks works slightly differently from other block | |
1114 | * types. Where other block types MAC the data in whatever endianness is | |
1115 | * written to disk, objset blocks always MAC little endian version of their | |
1116 | * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() | |
1117 | * and le_bswap indicates whether a byteswap is needed to get this block | |
1118 | * into little endian format. | |
1119 | */ | |
1120 | int | |
1121 | zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, | |
1122 | boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) | |
1123 | { | |
1124 | int ret; | |
1125 | crypto_mechanism_t mech; | |
1126 | crypto_context_t ctx; | |
1127 | crypto_data_t cd; | |
1128 | objset_phys_t *osp = data; | |
1129 | uint64_t intval; | |
1130 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
4807c0ba TC |
1131 | uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; |
1132 | uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; | |
b5256303 TC |
1133 | |
1134 | /* initialize HMAC mechanism */ | |
1135 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
1136 | mech.cm_param = NULL; | |
1137 | mech.cm_param_len = 0; | |
1138 | ||
1139 | cd.cd_format = CRYPTO_DATA_RAW; | |
1140 | cd.cd_offset = 0; | |
1141 | ||
1142 | /* calculate the portable MAC from the portable fields and metadnode */ | |
1143 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1144 | if (ret != CRYPTO_SUCCESS) { | |
1145 | ret = SET_ERROR(EIO); | |
1146 | goto error; | |
1147 | } | |
1148 | ||
1149 | /* add in the os_type */ | |
1150 | intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); | |
1151 | cd.cd_length = sizeof (uint64_t); | |
1152 | cd.cd_raw.iov_base = (char *)&intval; | |
1153 | cd.cd_raw.iov_len = cd.cd_length; | |
1154 | ||
1155 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1156 | if (ret != CRYPTO_SUCCESS) { | |
1157 | ret = SET_ERROR(EIO); | |
1158 | goto error; | |
1159 | } | |
1160 | ||
1161 | /* add in the portable os_flags */ | |
1162 | intval = osp->os_flags; | |
1163 | if (should_bswap) | |
1164 | intval = BSWAP_64(intval); | |
1165 | intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1166 | if (!ZFS_HOST_BYTEORDER) | |
1167 | intval = BSWAP_64(intval); | |
1168 | ||
1169 | cd.cd_length = sizeof (uint64_t); | |
1170 | cd.cd_raw.iov_base = (char *)&intval; | |
1171 | cd.cd_raw.iov_len = cd.cd_length; | |
1172 | ||
1173 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1174 | if (ret != CRYPTO_SUCCESS) { | |
1175 | ret = SET_ERROR(EIO); | |
1176 | goto error; | |
1177 | } | |
1178 | ||
1179 | /* add in fields from the metadnode */ | |
ae76f45c TC |
1180 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, |
1181 | should_bswap, &osp->os_meta_dnode); | |
b5256303 TC |
1182 | if (ret) |
1183 | goto error; | |
1184 | ||
1185 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1186 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1187 | cd.cd_raw.iov_base = (char *)raw_portable_mac; |
1188 | cd.cd_raw.iov_len = cd.cd_length; | |
1189 | ||
1190 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1191 | if (ret != CRYPTO_SUCCESS) { | |
1192 | ret = SET_ERROR(EIO); | |
1193 | goto error; | |
1194 | } | |
1195 | ||
1196 | bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); | |
1197 | ||
1198 | /* | |
1199 | * The local MAC protects the user and group accounting. If these | |
1200 | * objects are not present, the local MAC is zeroed out. | |
1201 | */ | |
1202 | if (osp->os_userused_dnode.dn_type == DMU_OT_NONE && | |
4807c0ba | 1203 | osp->os_groupused_dnode.dn_type == DMU_OT_NONE) { |
b5256303 TC |
1204 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); |
1205 | return (0); | |
1206 | } | |
1207 | ||
1208 | /* calculate the local MAC from the userused and groupused dnodes */ | |
1209 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1210 | if (ret != CRYPTO_SUCCESS) { | |
1211 | ret = SET_ERROR(EIO); | |
1212 | goto error; | |
1213 | } | |
1214 | ||
1215 | /* add in the non-portable os_flags */ | |
1216 | intval = osp->os_flags; | |
1217 | if (should_bswap) | |
1218 | intval = BSWAP_64(intval); | |
1219 | intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1220 | if (!ZFS_HOST_BYTEORDER) | |
1221 | intval = BSWAP_64(intval); | |
1222 | ||
1223 | cd.cd_length = sizeof (uint64_t); | |
1224 | cd.cd_raw.iov_base = (char *)&intval; | |
1225 | cd.cd_raw.iov_len = cd.cd_length; | |
1226 | ||
1227 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1228 | if (ret != CRYPTO_SUCCESS) { | |
1229 | ret = SET_ERROR(EIO); | |
1230 | goto error; | |
1231 | } | |
1232 | ||
1233 | /* add in fields from the user accounting dnodes */ | |
ae76f45c TC |
1234 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, |
1235 | should_bswap, &osp->os_userused_dnode); | |
b5256303 TC |
1236 | if (ret) |
1237 | goto error; | |
1238 | ||
ae76f45c TC |
1239 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, |
1240 | should_bswap, &osp->os_groupused_dnode); | |
b5256303 TC |
1241 | if (ret) |
1242 | goto error; | |
1243 | ||
1244 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1245 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1246 | cd.cd_raw.iov_base = (char *)raw_local_mac; |
1247 | cd.cd_raw.iov_len = cd.cd_length; | |
1248 | ||
1249 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1250 | if (ret != CRYPTO_SUCCESS) { | |
1251 | ret = SET_ERROR(EIO); | |
1252 | goto error; | |
1253 | } | |
1254 | ||
1255 | bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); | |
1256 | ||
1257 | return (0); | |
1258 | ||
1259 | error: | |
1260 | bzero(portable_mac, ZIO_OBJSET_MAC_LEN); | |
1261 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); | |
1262 | return (ret); | |
1263 | } | |
1264 | ||
1265 | static void | |
1266 | zio_crypt_destroy_uio(uio_t *uio) | |
1267 | { | |
1268 | if (uio->uio_iov) | |
1269 | kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); | |
1270 | } | |
1271 | ||
1272 | /* | |
1273 | * This function parses an uncompressed indirect block and returns a checksum | |
1274 | * of all the portable fields from all of the contained bps. The portable | |
1275 | * fields are the MAC and all of the fields from blk_prop except for the dedup, | |
1276 | * checksum, and psize bits. For an explanation of the purpose of this, see | |
1277 | * the comment block on object set authentication. | |
1278 | */ | |
ae76f45c TC |
1279 | static int |
1280 | zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, | |
1281 | uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) | |
b5256303 TC |
1282 | { |
1283 | blkptr_t *bp; | |
1284 | int i, epb = datalen >> SPA_BLKPTRSHIFT; | |
1285 | SHA2_CTX ctx; | |
4807c0ba | 1286 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 TC |
1287 | |
1288 | /* checksum all of the MACs from the layer below */ | |
1289 | SHA2Init(SHA512, &ctx); | |
1290 | for (i = 0, bp = buf; i < epb; i++, bp++) { | |
ae76f45c TC |
1291 | zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, |
1292 | byteswap, bp); | |
b5256303 TC |
1293 | } |
1294 | SHA2Final(digestbuf, &ctx); | |
1295 | ||
1296 | if (generate) { | |
1297 | bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); | |
1298 | return (0); | |
1299 | } | |
1300 | ||
1301 | if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) | |
1302 | return (SET_ERROR(ECKSUM)); | |
1303 | ||
1304 | return (0); | |
1305 | } | |
1306 | ||
1307 | int | |
ae76f45c | 1308 | zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, |
b5256303 TC |
1309 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) |
1310 | { | |
ae76f45c | 1311 | int ret; |
b5256303 | 1312 | |
ae76f45c TC |
1313 | /* |
1314 | * Unfortunately, callers of this function will not always have | |
1315 | * easy access to the on-disk format version. This info is | |
1316 | * normally found in the DSL Crypto Key, but the checksum-of-MACs | |
1317 | * is expected to be verifiable even when the key isn't loaded. | |
1318 | * Here, instead of doing a ZAP lookup for the version for each | |
1319 | * zio, we simply try both existing formats. | |
1320 | */ | |
1321 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, | |
1322 | datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); | |
1323 | if (ret == ECKSUM) { | |
1324 | ASSERT(!generate); | |
1325 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, | |
1326 | buf, datalen, 0, byteswap, cksum); | |
1327 | } | |
1328 | ||
1329 | return (ret); | |
1330 | } | |
1331 | ||
1332 | int | |
1333 | zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, | |
1334 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) | |
1335 | { | |
b5256303 TC |
1336 | int ret; |
1337 | void *buf; | |
1338 | ||
1339 | buf = abd_borrow_buf_copy(abd, datalen); | |
1340 | ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, | |
1341 | byteswap, cksum); | |
1342 | abd_return_buf(abd, buf, datalen); | |
1343 | ||
1344 | return (ret); | |
1345 | } | |
1346 | ||
1347 | /* | |
1348 | * Special case handling routine for encrypting / decrypting ZIL blocks. | |
1349 | * We do not check for the older ZIL chain because the encryption feature | |
1350 | * was not available before the newer ZIL chain was introduced. The goal | |
1351 | * here is to encrypt everything except the blkptr_t of a lr_write_t and | |
1352 | * the zil_chain_t header. Everything that is not encrypted is authenticated. | |
1353 | */ | |
1354 | static int | |
1355 | zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, | |
1356 | uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio, | |
1357 | uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, | |
1358 | boolean_t *no_crypt) | |
1359 | { | |
1360 | int ret; | |
9bae371c TC |
1361 | uint64_t txtype, lr_len; |
1362 | uint_t nr_src, nr_dst, crypt_len; | |
b5256303 TC |
1363 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; |
1364 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1365 | uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; | |
1366 | zil_chain_t *zilc; | |
1367 | lr_t *lr; | |
1368 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1369 | ||
1370 | /* cipherbuf always needs an extra iovec for the MAC */ | |
1371 | if (encrypt) { | |
1372 | src = plainbuf; | |
1373 | dst = cipherbuf; | |
1374 | nr_src = 0; | |
1375 | nr_dst = 1; | |
1376 | } else { | |
1377 | src = cipherbuf; | |
1378 | dst = plainbuf; | |
1379 | nr_src = 1; | |
1380 | nr_dst = 0; | |
1381 | } | |
1382 | ||
1383 | /* find the start and end record of the log block */ | |
1384 | zilc = (zil_chain_t *)src; | |
1385 | slrp = src + sizeof (zil_chain_t); | |
1386 | aadp = aadbuf; | |
1387 | blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); | |
1388 | ||
1389 | /* calculate the number of encrypted iovecs we will need */ | |
1390 | for (; slrp < blkend; slrp += lr_len) { | |
1391 | lr = (lr_t *)slrp; | |
1392 | ||
1393 | if (!byteswap) { | |
1394 | txtype = lr->lrc_txtype; | |
1395 | lr_len = lr->lrc_reclen; | |
1396 | } else { | |
1397 | txtype = BSWAP_64(lr->lrc_txtype); | |
1398 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1399 | } | |
1400 | ||
1401 | nr_iovecs++; | |
1402 | if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) | |
1403 | nr_iovecs++; | |
1404 | } | |
1405 | ||
1406 | nr_src += nr_iovecs; | |
1407 | nr_dst += nr_iovecs; | |
1408 | ||
1409 | /* allocate the iovec arrays */ | |
1410 | if (nr_src != 0) { | |
1411 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1412 | if (src_iovecs == NULL) { |
b5256303 TC |
1413 | ret = SET_ERROR(ENOMEM); |
1414 | goto error; | |
1415 | } | |
1416 | } | |
1417 | ||
1418 | if (nr_dst != 0) { | |
1419 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1420 | if (dst_iovecs == NULL) { |
b5256303 TC |
1421 | ret = SET_ERROR(ENOMEM); |
1422 | goto error; | |
1423 | } | |
1424 | } | |
1425 | ||
1426 | /* | |
1427 | * Copy the plain zil header over and authenticate everything except | |
1428 | * the checksum that will store our MAC. If we are writing the data | |
1429 | * the embedded checksum will not have been calculated yet, so we don't | |
1430 | * authenticate that. | |
1431 | */ | |
1432 | bcopy(src, dst, sizeof (zil_chain_t)); | |
1433 | bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); | |
1434 | aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1435 | aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1436 | ||
1437 | /* loop over records again, filling in iovecs */ | |
1438 | nr_iovecs = 0; | |
1439 | slrp = src + sizeof (zil_chain_t); | |
1440 | dlrp = dst + sizeof (zil_chain_t); | |
1441 | ||
1442 | for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { | |
1443 | lr = (lr_t *)slrp; | |
1444 | ||
1445 | if (!byteswap) { | |
1446 | txtype = lr->lrc_txtype; | |
1447 | lr_len = lr->lrc_reclen; | |
1448 | } else { | |
1449 | txtype = BSWAP_64(lr->lrc_txtype); | |
1450 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1451 | } | |
1452 | ||
1453 | /* copy the common lr_t */ | |
1454 | bcopy(slrp, dlrp, sizeof (lr_t)); | |
1455 | bcopy(slrp, aadp, sizeof (lr_t)); | |
1456 | aadp += sizeof (lr_t); | |
1457 | aad_len += sizeof (lr_t); | |
1458 | ||
4807c0ba TC |
1459 | ASSERT3P(src_iovecs, !=, NULL); |
1460 | ASSERT3P(dst_iovecs, !=, NULL); | |
1461 | ||
b5256303 TC |
1462 | /* |
1463 | * If this is a TX_WRITE record we want to encrypt everything | |
1464 | * except the bp if exists. If the bp does exist we want to | |
1465 | * authenticate it. | |
1466 | */ | |
1467 | if (txtype == TX_WRITE) { | |
1468 | crypt_len = sizeof (lr_write_t) - | |
1469 | sizeof (lr_t) - sizeof (blkptr_t); | |
1470 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1471 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1472 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1473 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1474 | ||
1475 | /* copy the bp now since it will not be encrypted */ | |
1476 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1477 | dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1478 | sizeof (blkptr_t)); | |
1479 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1480 | aadp, sizeof (blkptr_t)); | |
1481 | aadp += sizeof (blkptr_t); | |
1482 | aad_len += sizeof (blkptr_t); | |
1483 | nr_iovecs++; | |
1484 | total_len += crypt_len; | |
1485 | ||
1486 | if (lr_len != sizeof (lr_write_t)) { | |
1487 | crypt_len = lr_len - sizeof (lr_write_t); | |
1488 | src_iovecs[nr_iovecs].iov_base = | |
1489 | slrp + sizeof (lr_write_t); | |
1490 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1491 | dst_iovecs[nr_iovecs].iov_base = | |
1492 | dlrp + sizeof (lr_write_t); | |
1493 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1494 | nr_iovecs++; | |
1495 | total_len += crypt_len; | |
1496 | } | |
1497 | } else { | |
1498 | crypt_len = lr_len - sizeof (lr_t); | |
1499 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1500 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1501 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1502 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1503 | nr_iovecs++; | |
1504 | total_len += crypt_len; | |
1505 | } | |
1506 | } | |
1507 | ||
1508 | *no_crypt = (nr_iovecs == 0); | |
1509 | *enc_len = total_len; | |
1510 | *authbuf = aadbuf; | |
1511 | *auth_len = aad_len; | |
1512 | ||
1513 | if (encrypt) { | |
1514 | puio->uio_iov = src_iovecs; | |
1515 | puio->uio_iovcnt = nr_src; | |
1516 | cuio->uio_iov = dst_iovecs; | |
1517 | cuio->uio_iovcnt = nr_dst; | |
1518 | } else { | |
1519 | puio->uio_iov = dst_iovecs; | |
1520 | puio->uio_iovcnt = nr_dst; | |
1521 | cuio->uio_iov = src_iovecs; | |
1522 | cuio->uio_iovcnt = nr_src; | |
1523 | } | |
1524 | ||
1525 | return (0); | |
1526 | ||
1527 | error: | |
1528 | zio_buf_free(aadbuf, datalen); | |
1529 | if (src_iovecs != NULL) | |
1530 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1531 | if (dst_iovecs != NULL) | |
1532 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1533 | ||
1534 | *enc_len = 0; | |
1535 | *authbuf = NULL; | |
1536 | *auth_len = 0; | |
1537 | *no_crypt = B_FALSE; | |
1538 | puio->uio_iov = NULL; | |
1539 | puio->uio_iovcnt = 0; | |
1540 | cuio->uio_iov = NULL; | |
1541 | cuio->uio_iovcnt = 0; | |
1542 | return (ret); | |
1543 | } | |
1544 | ||
1545 | /* | |
1546 | * Special case handling routine for encrypting / decrypting dnode blocks. | |
1547 | */ | |
1548 | static int | |
ae76f45c TC |
1549 | zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, |
1550 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1551 | uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
1552 | uint_t *auth_len, boolean_t *no_crypt) | |
b5256303 TC |
1553 | { |
1554 | int ret; | |
1555 | uint_t nr_src, nr_dst, crypt_len; | |
1556 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; | |
1557 | uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; | |
1558 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1559 | uint8_t *src, *dst, *aadp; | |
1560 | dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; | |
1561 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1562 | ||
1563 | if (encrypt) { | |
1564 | src = plainbuf; | |
1565 | dst = cipherbuf; | |
1566 | nr_src = 0; | |
1567 | nr_dst = 1; | |
1568 | } else { | |
1569 | src = cipherbuf; | |
1570 | dst = plainbuf; | |
1571 | nr_src = 1; | |
1572 | nr_dst = 0; | |
1573 | } | |
1574 | ||
1575 | sdnp = (dnode_phys_t *)src; | |
1576 | ddnp = (dnode_phys_t *)dst; | |
1577 | aadp = aadbuf; | |
1578 | ||
1579 | /* | |
1580 | * Count the number of iovecs we will need to do the encryption by | |
1581 | * counting the number of bonus buffers that need to be encrypted. | |
1582 | */ | |
1583 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1584 | /* | |
1585 | * This block may still be byteswapped. However, all of the | |
1586 | * values we use are either uint8_t's (for which byteswapping | |
1587 | * is a noop) or a * != 0 check, which will work regardless | |
1588 | * of whether or not we byteswap. | |
1589 | */ | |
1590 | if (sdnp[i].dn_type != DMU_OT_NONE && | |
1591 | DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && | |
1592 | sdnp[i].dn_bonuslen != 0) { | |
1593 | nr_iovecs++; | |
1594 | } | |
1595 | } | |
1596 | ||
1597 | nr_src += nr_iovecs; | |
1598 | nr_dst += nr_iovecs; | |
1599 | ||
1600 | if (nr_src != 0) { | |
1601 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1602 | if (src_iovecs == NULL) { |
b5256303 TC |
1603 | ret = SET_ERROR(ENOMEM); |
1604 | goto error; | |
1605 | } | |
1606 | } | |
1607 | ||
1608 | if (nr_dst != 0) { | |
1609 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1610 | if (dst_iovecs == NULL) { |
b5256303 TC |
1611 | ret = SET_ERROR(ENOMEM); |
1612 | goto error; | |
1613 | } | |
1614 | } | |
1615 | ||
1616 | nr_iovecs = 0; | |
1617 | ||
1618 | /* | |
1619 | * Iterate through the dnodes again, this time filling in the uios | |
1620 | * we allocated earlier. We also concatenate any data we want to | |
1621 | * authenticate onto aadbuf. | |
1622 | */ | |
1623 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1624 | dnp = &sdnp[i]; | |
1625 | ||
1626 | /* copy over the core fields and blkptrs (kept as plaintext) */ | |
1627 | bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); | |
1628 | ||
1629 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1630 | bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), | |
1631 | sizeof (blkptr_t)); | |
1632 | } | |
1633 | ||
1634 | /* | |
1635 | * Handle authenticated data. We authenticate everything in | |
1636 | * the dnode that can be brought over when we do a raw send. | |
1637 | * This includes all of the core fields as well as the MACs | |
1638 | * stored in the bp checksums and all of the portable bits | |
1639 | * from blk_prop. We include the dnode padding here in case it | |
1640 | * ever gets used in the future. Some dn_flags and dn_used are | |
1641 | * not portable so we mask those out values out of the | |
1642 | * authenticated data. | |
1643 | */ | |
1644 | crypt_len = offsetof(dnode_phys_t, dn_blkptr); | |
1645 | bcopy(dnp, aadp, crypt_len); | |
1646 | adnp = (dnode_phys_t *)aadp; | |
1647 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1648 | adnp->dn_used = 0; | |
1649 | aadp += crypt_len; | |
1650 | aad_len += crypt_len; | |
1651 | ||
1652 | for (j = 0; j < dnp->dn_nblkptr; j++) { | |
1653 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1654 | version, byteswap, &dnp->dn_blkptr[j]); |
b5256303 TC |
1655 | } |
1656 | ||
1657 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1658 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1659 | version, byteswap, DN_SPILL_BLKPTR(dnp)); |
b5256303 TC |
1660 | } |
1661 | ||
1662 | /* | |
1663 | * If this bonus buffer needs to be encrypted, we prepare an | |
1664 | * iovec_t. The encryption / decryption functions will fill | |
1665 | * this in for us with the encrypted or decrypted data. | |
1666 | * Otherwise we add the bonus buffer to the authenticated | |
1667 | * data buffer and copy it over to the destination. The | |
1668 | * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that | |
1669 | * we can guarantee alignment with the AES block size | |
1670 | * (128 bits). | |
1671 | */ | |
1672 | crypt_len = DN_MAX_BONUS_LEN(dnp); | |
1673 | if (dnp->dn_type != DMU_OT_NONE && | |
1674 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
1675 | dnp->dn_bonuslen != 0) { | |
4807c0ba TC |
1676 | ASSERT3U(nr_iovecs, <, nr_src); |
1677 | ASSERT3U(nr_iovecs, <, nr_dst); | |
1678 | ASSERT3P(src_iovecs, !=, NULL); | |
1679 | ASSERT3P(dst_iovecs, !=, NULL); | |
b5256303 TC |
1680 | src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); |
1681 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1682 | dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); | |
1683 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1684 | ||
1685 | nr_iovecs++; | |
1686 | total_len += crypt_len; | |
1687 | } else { | |
1688 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); | |
1689 | bcopy(DN_BONUS(dnp), aadp, crypt_len); | |
1690 | aadp += crypt_len; | |
1691 | aad_len += crypt_len; | |
1692 | } | |
1693 | } | |
1694 | ||
1695 | *no_crypt = (nr_iovecs == 0); | |
1696 | *enc_len = total_len; | |
1697 | *authbuf = aadbuf; | |
1698 | *auth_len = aad_len; | |
1699 | ||
1700 | if (encrypt) { | |
1701 | puio->uio_iov = src_iovecs; | |
1702 | puio->uio_iovcnt = nr_src; | |
1703 | cuio->uio_iov = dst_iovecs; | |
1704 | cuio->uio_iovcnt = nr_dst; | |
1705 | } else { | |
1706 | puio->uio_iov = dst_iovecs; | |
1707 | puio->uio_iovcnt = nr_dst; | |
1708 | cuio->uio_iov = src_iovecs; | |
1709 | cuio->uio_iovcnt = nr_src; | |
1710 | } | |
1711 | ||
1712 | return (0); | |
1713 | ||
1714 | error: | |
1715 | zio_buf_free(aadbuf, datalen); | |
1716 | if (src_iovecs != NULL) | |
1717 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1718 | if (dst_iovecs != NULL) | |
1719 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1720 | ||
1721 | *enc_len = 0; | |
1722 | *authbuf = NULL; | |
1723 | *auth_len = 0; | |
1724 | *no_crypt = B_FALSE; | |
1725 | puio->uio_iov = NULL; | |
1726 | puio->uio_iovcnt = 0; | |
1727 | cuio->uio_iov = NULL; | |
1728 | cuio->uio_iovcnt = 0; | |
1729 | return (ret); | |
1730 | } | |
1731 | ||
1732 | static int | |
1733 | zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, | |
1734 | uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio, | |
1735 | uint_t *enc_len) | |
1736 | { | |
1737 | int ret; | |
1738 | uint_t nr_plain = 1, nr_cipher = 2; | |
1739 | iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; | |
1740 | ||
1741 | /* allocate the iovecs for the plain and cipher data */ | |
1742 | plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), | |
1743 | KM_SLEEP); | |
1744 | if (!plain_iovecs) { | |
1745 | ret = SET_ERROR(ENOMEM); | |
1746 | goto error; | |
1747 | } | |
1748 | ||
1749 | cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), | |
1750 | KM_SLEEP); | |
1751 | if (!cipher_iovecs) { | |
1752 | ret = SET_ERROR(ENOMEM); | |
1753 | goto error; | |
1754 | } | |
1755 | ||
1756 | plain_iovecs[0].iov_base = plainbuf; | |
1757 | plain_iovecs[0].iov_len = datalen; | |
1758 | cipher_iovecs[0].iov_base = cipherbuf; | |
1759 | cipher_iovecs[0].iov_len = datalen; | |
1760 | ||
1761 | *enc_len = datalen; | |
1762 | puio->uio_iov = plain_iovecs; | |
1763 | puio->uio_iovcnt = nr_plain; | |
1764 | cuio->uio_iov = cipher_iovecs; | |
1765 | cuio->uio_iovcnt = nr_cipher; | |
1766 | ||
1767 | return (0); | |
1768 | ||
1769 | error: | |
1770 | if (plain_iovecs != NULL) | |
1771 | kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); | |
1772 | if (cipher_iovecs != NULL) | |
1773 | kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); | |
1774 | ||
1775 | *enc_len = 0; | |
1776 | puio->uio_iov = NULL; | |
1777 | puio->uio_iovcnt = 0; | |
1778 | cuio->uio_iov = NULL; | |
1779 | cuio->uio_iovcnt = 0; | |
1780 | return (ret); | |
1781 | } | |
1782 | ||
1783 | /* | |
1784 | * This function builds up the plaintext (puio) and ciphertext (cuio) uios so | |
1785 | * that they can be used for encryption and decryption by zio_do_crypt_uio(). | |
1786 | * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks | |
1787 | * requiring special handling to parse out pieces that are to be encrypted. The | |
1788 | * authbuf is used by these special cases to store additional authenticated | |
1789 | * data (AAD) for the encryption modes. | |
1790 | */ | |
1791 | static int | |
ae76f45c TC |
1792 | zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, |
1793 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1794 | uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
b5256303 TC |
1795 | uint_t *auth_len, boolean_t *no_crypt) |
1796 | { | |
1797 | int ret; | |
1798 | iovec_t *mac_iov; | |
1799 | ||
1800 | ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); | |
1801 | ||
1802 | /* route to handler */ | |
1803 | switch (ot) { | |
1804 | case DMU_OT_INTENT_LOG: | |
1805 | ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, | |
1806 | datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, | |
1807 | no_crypt); | |
1808 | break; | |
1809 | case DMU_OT_DNODE: | |
ae76f45c TC |
1810 | ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, |
1811 | cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, | |
1812 | auth_len, no_crypt); | |
b5256303 TC |
1813 | break; |
1814 | default: | |
1815 | ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, | |
1816 | datalen, puio, cuio, enc_len); | |
1817 | *authbuf = NULL; | |
1818 | *auth_len = 0; | |
1819 | *no_crypt = B_FALSE; | |
1820 | break; | |
1821 | } | |
1822 | ||
1823 | if (ret != 0) | |
1824 | goto error; | |
1825 | ||
1826 | /* populate the uios */ | |
1827 | puio->uio_segflg = UIO_SYSSPACE; | |
1828 | cuio->uio_segflg = UIO_SYSSPACE; | |
1829 | ||
1830 | mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); | |
1831 | mac_iov->iov_base = mac; | |
1832 | mac_iov->iov_len = ZIO_DATA_MAC_LEN; | |
1833 | ||
1834 | return (0); | |
1835 | ||
1836 | error: | |
1837 | return (ret); | |
1838 | } | |
1839 | ||
1840 | /* | |
1841 | * Primary encryption / decryption entrypoint for zio data. | |
1842 | */ | |
1843 | int | |
1844 | zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt, | |
1845 | dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen, | |
1846 | boolean_t byteswap, uint8_t *plainbuf, uint8_t *cipherbuf, | |
1847 | boolean_t *no_crypt) | |
1848 | { | |
1849 | int ret; | |
1850 | boolean_t locked = B_FALSE; | |
1851 | uint64_t crypt = key->zk_crypt; | |
1852 | uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; | |
1853 | uint_t enc_len, auth_len; | |
1854 | uio_t puio, cuio; | |
1855 | uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; | |
1856 | crypto_key_t tmp_ckey, *ckey = NULL; | |
1857 | crypto_ctx_template_t tmpl; | |
1858 | uint8_t *authbuf = NULL; | |
1859 | ||
1860 | bzero(&puio, sizeof (uio_t)); | |
1861 | bzero(&cuio, sizeof (uio_t)); | |
1862 | ||
1863 | /* create uios for encryption */ | |
ae76f45c TC |
1864 | ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, |
1865 | cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, | |
1866 | &authbuf, &auth_len, no_crypt); | |
b5256303 TC |
1867 | if (ret != 0) |
1868 | return (ret); | |
1869 | ||
1870 | /* | |
1871 | * If the needed key is the current one, just use it. Otherwise we | |
1872 | * need to generate a temporary one from the given salt + master key. | |
1873 | * If we are encrypting, we must return a copy of the current salt | |
1874 | * so that it can be stored in the blkptr_t. | |
1875 | */ | |
1876 | rw_enter(&key->zk_salt_lock, RW_READER); | |
1877 | locked = B_TRUE; | |
1878 | ||
1879 | if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { | |
1880 | ckey = &key->zk_current_key; | |
1881 | tmpl = key->zk_current_tmpl; | |
1882 | } else { | |
1883 | rw_exit(&key->zk_salt_lock); | |
1884 | locked = B_FALSE; | |
1885 | ||
1886 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
1887 | salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); | |
1888 | if (ret != 0) | |
1889 | goto error; | |
1890 | ||
1891 | tmp_ckey.ck_format = CRYPTO_KEY_RAW; | |
1892 | tmp_ckey.ck_data = enc_keydata; | |
4807c0ba | 1893 | tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
1894 | |
1895 | ckey = &tmp_ckey; | |
1896 | tmpl = NULL; | |
1897 | } | |
1898 | ||
1899 | /* perform the encryption / decryption */ | |
1900 | ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, | |
1901 | &puio, &cuio, authbuf, auth_len); | |
1902 | if (ret != 0) | |
1903 | goto error; | |
1904 | ||
1905 | if (locked) { | |
1906 | rw_exit(&key->zk_salt_lock); | |
1907 | locked = B_FALSE; | |
1908 | } | |
1909 | ||
1910 | if (authbuf != NULL) | |
1911 | zio_buf_free(authbuf, datalen); | |
1912 | if (ckey == &tmp_ckey) | |
1913 | bzero(enc_keydata, keydata_len); | |
1914 | zio_crypt_destroy_uio(&puio); | |
1915 | zio_crypt_destroy_uio(&cuio); | |
1916 | ||
1917 | return (0); | |
1918 | ||
1919 | error: | |
1920 | if (locked) | |
1921 | rw_exit(&key->zk_salt_lock); | |
1922 | if (authbuf != NULL) | |
1923 | zio_buf_free(authbuf, datalen); | |
1924 | if (ckey == &tmp_ckey) | |
1925 | bzero(enc_keydata, keydata_len); | |
1926 | zio_crypt_destroy_uio(&puio); | |
1927 | zio_crypt_destroy_uio(&cuio); | |
1928 | ||
1929 | return (ret); | |
1930 | } | |
1931 | ||
1932 | /* | |
1933 | * Simple wrapper around zio_do_crypt_data() to work with abd's instead of | |
1934 | * linear buffers. | |
1935 | */ | |
1936 | int | |
1937 | zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt, | |
1938 | dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen, | |
1939 | boolean_t byteswap, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) | |
1940 | { | |
1941 | int ret; | |
1942 | void *ptmp, *ctmp; | |
1943 | ||
1944 | if (encrypt) { | |
1945 | ptmp = abd_borrow_buf_copy(pabd, datalen); | |
1946 | ctmp = abd_borrow_buf(cabd, datalen); | |
1947 | } else { | |
1948 | ptmp = abd_borrow_buf(pabd, datalen); | |
1949 | ctmp = abd_borrow_buf_copy(cabd, datalen); | |
1950 | } | |
1951 | ||
1952 | ret = zio_do_crypt_data(encrypt, key, salt, ot, iv, mac, | |
1953 | datalen, byteswap, ptmp, ctmp, no_crypt); | |
1954 | if (ret != 0) | |
1955 | goto error; | |
1956 | ||
1957 | if (encrypt) { | |
1958 | abd_return_buf(pabd, ptmp, datalen); | |
1959 | abd_return_buf_copy(cabd, ctmp, datalen); | |
1960 | } else { | |
1961 | abd_return_buf_copy(pabd, ptmp, datalen); | |
1962 | abd_return_buf(cabd, ctmp, datalen); | |
1963 | } | |
1964 | ||
1965 | return (0); | |
1966 | ||
1967 | error: | |
1968 | if (encrypt) { | |
1969 | abd_return_buf(pabd, ptmp, datalen); | |
1970 | abd_return_buf_copy(cabd, ctmp, datalen); | |
1971 | } else { | |
1972 | abd_return_buf_copy(pabd, ptmp, datalen); | |
1973 | abd_return_buf(cabd, ctmp, datalen); | |
1974 | } | |
1975 | ||
1976 | return (ret); | |
1977 | } | |
1978 | ||
1979 | #if defined(_KERNEL) && defined(HAVE_SPL) | |
1980 | /* BEGIN CSTYLED */ | |
1981 | module_param(zfs_key_max_salt_uses, ulong, 0644); | |
1982 | MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " | |
1983 | "can be used for generating encryption keys before it is rotated"); | |
1984 | /* END CSTYLED */ | |
1985 | #endif |