]>
Commit | Line | Data |
---|---|---|
b5256303 TC |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * This file and its contents are supplied under the terms of the | |
5 | * Common Development and Distribution License ("CDDL"), version 1.0. | |
6 | * You may only use this file in accordance with the terms of version | |
7 | * 1.0 of the CDDL. | |
8 | * | |
9 | * A full copy of the text of the CDDL should have accompanied this | |
10 | * source. A copy of the CDDL is also available via the Internet at | |
11 | * http://www.illumos.org/license/CDDL. | |
12 | * | |
13 | * CDDL HEADER END | |
14 | */ | |
15 | ||
16 | /* | |
17 | * Copyright (c) 2017, Datto, Inc. All rights reserved. | |
18 | */ | |
19 | ||
20 | #include <sys/zio_crypt.h> | |
21 | #include <sys/dmu.h> | |
22 | #include <sys/dmu_objset.h> | |
23 | #include <sys/dnode.h> | |
24 | #include <sys/fs/zfs.h> | |
25 | #include <sys/zio.h> | |
26 | #include <sys/zil.h> | |
27 | #include <sys/sha2.h> | |
4807c0ba | 28 | #include <sys/hkdf.h> |
cf637391 | 29 | #include "qat.h" |
b5256303 TC |
30 | |
31 | /* | |
32 | * This file is responsible for handling all of the details of generating | |
33 | * encryption parameters and performing encryption and authentication. | |
34 | * | |
35 | * BLOCK ENCRYPTION PARAMETERS: | |
36 | * Encryption /Authentication Algorithm Suite (crypt): | |
37 | * The encryption algorithm, mode, and key length we are going to use. We | |
38 | * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit | |
39 | * keys. All authentication is currently done with SHA512-HMAC. | |
40 | * | |
41 | * Plaintext: | |
42 | * The unencrypted data that we want to encrypt. | |
43 | * | |
44 | * Initialization Vector (IV): | |
45 | * An initialization vector for the encryption algorithms. This is used to | |
46 | * "tweak" the encryption algorithms so that two blocks of the same data are | |
47 | * encrypted into different ciphertext outputs, thus obfuscating block patterns. | |
48 | * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is | |
49 | * never reused with the same encryption key. This value is stored unencrypted | |
50 | * and must simply be provided to the decryption function. We use a 96 bit IV | |
51 | * (as recommended by NIST) for all block encryption. For non-dedup blocks we | |
52 | * derive the IV randomly. The first 64 bits of the IV are stored in the second | |
53 | * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of | |
54 | * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits | |
55 | * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count | |
56 | * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of | |
57 | * level 0 blocks is the number of allocated dnodes in that block. The on-disk | |
58 | * format supports at most 2^15 slots per L0 dnode block, because the maximum | |
59 | * block size is 16MB (2^24). In either case, for level 0 blocks this number | |
60 | * will still be smaller than UINT32_MAX so it is safe to store the IV in the | |
61 | * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count | |
62 | * for the dnode code. | |
63 | * | |
64 | * Master key: | |
65 | * This is the most important secret data of an encrypted dataset. It is used | |
66 | * along with the salt to generate that actual encryption keys via HKDF. We | |
67 | * do not use the master key to directly encrypt any data because there are | |
68 | * theoretical limits on how much data can actually be safely encrypted with | |
69 | * any encryption mode. The master key is stored encrypted on disk with the | |
70 | * user's wrapping key. Its length is determined by the encryption algorithm. | |
71 | * For details on how this is stored see the block comment in dsl_crypt.c | |
72 | * | |
73 | * Salt: | |
74 | * Used as an input to the HKDF function, along with the master key. We use a | |
75 | * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt | |
76 | * can be used for encrypting many blocks, so we cache the current salt and the | |
77 | * associated derived key in zio_crypt_t so we do not need to derive it again | |
78 | * needlessly. | |
79 | * | |
80 | * Encryption Key: | |
81 | * A secret binary key, generated from an HKDF function used to encrypt and | |
82 | * decrypt data. | |
83 | * | |
8d9e7c8f | 84 | * Message Authentication Code (MAC) |
b5256303 TC |
85 | * The MAC is an output of authenticated encryption modes such as AES-GCM and |
86 | * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted | |
87 | * data on disk and return garbage to the application. Effectively, it is a | |
88 | * checksum that can not be reproduced by an attacker. We store the MAC in the | |
89 | * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated | |
90 | * regular checksum of the ciphertext which can be used for scrubbing. | |
91 | * | |
92 | * OBJECT AUTHENTICATION: | |
93 | * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because | |
94 | * they contain some info that always needs to be readable. To prevent this | |
95 | * data from being altered, we authenticate this data using SHA512-HMAC. This | |
96 | * will produce a MAC (similar to the one produced via encryption) which can | |
97 | * be used to verify the object was not modified. HMACs do not require key | |
98 | * rotation or IVs, so we can keep up to the full 3 copies of authenticated | |
99 | * data. | |
100 | * | |
101 | * ZIL ENCRYPTION: | |
102 | * ZIL blocks have their bp written to disk ahead of the associated data, so we | |
103 | * cannot store the MAC there as we normally do. For these blocks the MAC is | |
104 | * stored in the embedded checksum within the zil_chain_t header. The salt and | |
105 | * IV are generated for the block on bp allocation instead of at encryption | |
106 | * time. In addition, ZIL blocks have some pieces that must be left in plaintext | |
107 | * for claiming even though all of the sensitive user data still needs to be | |
108 | * encrypted. The function zio_crypt_init_uios_zil() handles parsing which | |
109 | * pieces of the block need to be encrypted. All data that is not encrypted is | |
110 | * authenticated using the AAD mechanisms that the supported encryption modes | |
111 | * provide for. In order to preserve the semantics of the ZIL for encrypted | |
112 | * datasets, the ZIL is not protected at the objset level as described below. | |
113 | * | |
114 | * DNODE ENCRYPTION: | |
115 | * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left | |
116 | * in plaintext for scrubbing and claiming, but the bonus buffers might contain | |
117 | * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing | |
118 | * which which pieces of the block need to be encrypted. For more details about | |
119 | * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). | |
120 | * | |
121 | * OBJECT SET AUTHENTICATION: | |
122 | * Up to this point, everything we have encrypted and authenticated has been | |
123 | * at level 0 (or -2 for the ZIL). If we did not do any further work the | |
8d9e7c8f | 124 | * on-disk format would be susceptible to attacks that deleted or rearranged |
b5256303 TC |
125 | * the order of level 0 blocks. Ideally, the cleanest solution would be to |
126 | * maintain a tree of authentication MACs going up the bp tree. However, this | |
127 | * presents a problem for raw sends. Send files do not send information about | |
128 | * indirect blocks so there would be no convenient way to transfer the MACs and | |
129 | * they cannot be recalculated on the receive side without the master key which | |
130 | * would defeat one of the purposes of raw sends in the first place. Instead, | |
131 | * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs | |
132 | * from the level below. We also include some portable fields from blk_prop such | |
133 | * as the lsize and compression algorithm to prevent the data from being | |
8d9e7c8f | 134 | * misinterpreted. |
b5256303 | 135 | * |
8d9e7c8f | 136 | * At the objset level, we maintain 2 separate 256 bit MACs in the |
b5256303 | 137 | * objset_phys_t. The first one is "portable" and is the logical root of the |
8d9e7c8f | 138 | * MAC tree maintained in the metadnode's bps. The second, is "local" and is |
b5256303 TC |
139 | * used as the root MAC for the user accounting objects, which are also not |
140 | * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload | |
141 | * of the send file. The useraccounting code ensures that the useraccounting | |
142 | * info is not present upon a receive, so the local MAC can simply be cleared | |
143 | * out at that time. For more info about objset_phys_t authentication, see | |
144 | * zio_crypt_do_objset_hmacs(). | |
145 | * | |
146 | * CONSIDERATIONS FOR DEDUP: | |
147 | * In order for dedup to work, blocks that we want to dedup with one another | |
148 | * need to use the same IV and encryption key, so that they will have the same | |
149 | * ciphertext. Normally, one should never reuse an IV with the same encryption | |
150 | * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both | |
8d9e7c8f | 151 | * blocks. In this case, however, since we are using the same plaintext as |
b5256303 TC |
152 | * well all that we end up with is a duplicate of the original ciphertext we |
153 | * already had. As a result, an attacker with read access to the raw disk will | |
154 | * be able to tell which blocks are the same but this information is given away | |
155 | * by dedup anyway. In order to get the same IVs and encryption keys for | |
8d9e7c8f TC |
156 | * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC |
157 | * here so that a reproducible checksum of the plaintext is never available to | |
b5256303 TC |
158 | * the attacker. The HMAC key is kept alongside the master key, encrypted on |
159 | * disk. The first 64 bits of the HMAC are used in place of the random salt, and | |
160 | * the next 96 bits are used as the IV. As a result of this mechanism, dedup | |
161 | * will only work within a clone family since encrypted dedup requires use of | |
162 | * the same master and HMAC keys. | |
163 | */ | |
164 | ||
165 | /* | |
166 | * After encrypting many blocks with the same key we may start to run up | |
167 | * against the theoretical limits of how much data can securely be encrypted | |
168 | * with a single key using the supported encryption modes. The most obvious | |
169 | * limitation is that our risk of generating 2 equivalent 96 bit IVs increases | |
170 | * the more IVs we generate (which both GCM and CCM modes strictly forbid). | |
171 | * This risk actually grows surprisingly quickly over time according to the | |
172 | * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have | |
173 | * generated n IVs with a cryptographically secure RNG, the approximate | |
174 | * probability p(n) of a collision is given as: | |
175 | * | |
176 | * p(n) ~= e^(-n*(n-1)/(2*(2^96))) | |
177 | * | |
178 | * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] | |
179 | * | |
180 | * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion | |
181 | * we must not write more than 398,065,730 blocks with the same encryption key. | |
182 | * Therefore, we rotate our keys after 400,000,000 blocks have been written by | |
183 | * generating a new random 64 bit salt for our HKDF encryption key generation | |
184 | * function. | |
185 | */ | |
186 | #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 | |
187 | #define ZFS_CURRENT_MAX_SALT_USES \ | |
188 | (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) | |
189 | unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; | |
190 | ||
ae76f45c TC |
191 | typedef struct blkptr_auth_buf { |
192 | uint64_t bab_prop; /* blk_prop - portable mask */ | |
193 | uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */\r | |
194 | uint64_t bab_pad; /* reserved for future use */ | |
195 | } blkptr_auth_buf_t; | |
196 | ||
b5256303 TC |
197 | zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { |
198 | {"", ZC_TYPE_NONE, 0, "inherit"}, | |
199 | {"", ZC_TYPE_NONE, 0, "on"}, | |
200 | {"", ZC_TYPE_NONE, 0, "off"}, | |
201 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, | |
202 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, | |
203 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, | |
204 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, | |
205 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, | |
206 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} | |
207 | }; | |
208 | ||
b5256303 TC |
209 | void |
210 | zio_crypt_key_destroy(zio_crypt_key_t *key) | |
211 | { | |
212 | rw_destroy(&key->zk_salt_lock); | |
213 | ||
214 | /* free crypto templates */ | |
215 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
216 | crypto_destroy_ctx_template(key->zk_hmac_tmpl); | |
217 | ||
218 | /* zero out sensitive data */ | |
219 | bzero(key, sizeof (zio_crypt_key_t)); | |
220 | } | |
221 | ||
222 | int | |
223 | zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) | |
224 | { | |
225 | int ret; | |
226 | crypto_mechanism_t mech; | |
227 | uint_t keydata_len; | |
228 | ||
229 | ASSERT(key != NULL); | |
230 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
231 | ||
232 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
233 | bzero(key, sizeof (zio_crypt_key_t)); | |
234 | ||
235 | /* fill keydata buffers and salt with random data */ | |
236 | ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); | |
237 | if (ret != 0) | |
238 | goto error; | |
239 | ||
240 | ret = random_get_bytes(key->zk_master_keydata, keydata_len); | |
241 | if (ret != 0) | |
242 | goto error; | |
243 | ||
244 | ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); | |
245 | if (ret != 0) | |
246 | goto error; | |
247 | ||
248 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
249 | if (ret != 0) | |
250 | goto error; | |
251 | ||
252 | /* derive the current key from the master key */ | |
253 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
254 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
255 | keydata_len); | |
256 | if (ret != 0) | |
257 | goto error; | |
258 | ||
259 | /* initialize keys for the ICP */ | |
260 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
261 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 262 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
263 | |
264 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
265 | key->zk_hmac_key.ck_data = &key->zk_hmac_key; | |
4807c0ba | 266 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 TC |
267 | |
268 | /* | |
269 | * Initialize the crypto templates. It's ok if this fails because | |
270 | * this is just an optimization. | |
271 | */ | |
272 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
273 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
274 | &key->zk_current_tmpl, KM_SLEEP); | |
275 | if (ret != CRYPTO_SUCCESS) | |
276 | key->zk_current_tmpl = NULL; | |
277 | ||
278 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
279 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
280 | &key->zk_hmac_tmpl, KM_SLEEP); | |
281 | if (ret != CRYPTO_SUCCESS) | |
282 | key->zk_hmac_tmpl = NULL; | |
283 | ||
284 | key->zk_crypt = crypt; | |
ae76f45c | 285 | key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; |
b5256303 TC |
286 | key->zk_salt_count = 0; |
287 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); | |
288 | ||
289 | return (0); | |
290 | ||
291 | error: | |
292 | zio_crypt_key_destroy(key); | |
293 | return (ret); | |
294 | } | |
295 | ||
296 | static int | |
297 | zio_crypt_key_change_salt(zio_crypt_key_t *key) | |
298 | { | |
299 | int ret = 0; | |
300 | uint8_t salt[ZIO_DATA_SALT_LEN]; | |
301 | crypto_mechanism_t mech; | |
302 | uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; | |
303 | ||
304 | /* generate a new salt */ | |
305 | ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); | |
306 | if (ret != 0) | |
307 | goto error; | |
308 | ||
309 | rw_enter(&key->zk_salt_lock, RW_WRITER); | |
310 | ||
311 | /* someone beat us to the salt rotation, just unlock and return */ | |
312 | if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) | |
313 | goto out_unlock; | |
314 | ||
315 | /* derive the current key from the master key and the new salt */ | |
316 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
317 | salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); | |
318 | if (ret != 0) | |
319 | goto out_unlock; | |
320 | ||
321 | /* assign the salt and reset the usage count */ | |
322 | bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); | |
323 | key->zk_salt_count = 0; | |
324 | ||
325 | /* destroy the old context template and create the new one */ | |
326 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
327 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
328 | &key->zk_current_tmpl, KM_SLEEP); | |
329 | if (ret != CRYPTO_SUCCESS) | |
330 | key->zk_current_tmpl = NULL; | |
331 | ||
332 | rw_exit(&key->zk_salt_lock); | |
333 | ||
334 | return (0); | |
335 | ||
336 | out_unlock: | |
337 | rw_exit(&key->zk_salt_lock); | |
338 | error: | |
339 | return (ret); | |
340 | } | |
341 | ||
342 | /* See comment above zfs_key_max_salt_uses definition for details */ | |
343 | int | |
344 | zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) | |
345 | { | |
346 | int ret; | |
347 | boolean_t salt_change; | |
348 | ||
349 | rw_enter(&key->zk_salt_lock, RW_READER); | |
350 | ||
351 | bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); | |
352 | salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= | |
353 | ZFS_CURRENT_MAX_SALT_USES); | |
354 | ||
355 | rw_exit(&key->zk_salt_lock); | |
356 | ||
357 | if (salt_change) { | |
358 | ret = zio_crypt_key_change_salt(key); | |
359 | if (ret != 0) | |
360 | goto error; | |
361 | } | |
362 | ||
363 | return (0); | |
364 | ||
365 | error: | |
366 | return (ret); | |
367 | } | |
368 | ||
369 | /* | |
370 | * This function handles all encryption and decryption in zfs. When | |
371 | * encrypting it expects puio to reference the plaintext and cuio to | |
372 | * reference the cphertext. cuio must have enough space for the | |
373 | * ciphertext + room for a MAC. datalen should be the length of the | |
374 | * plaintext / ciphertext alone. | |
375 | */ | |
376 | static int | |
377 | zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, | |
378 | crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, | |
379 | uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len) | |
380 | { | |
381 | int ret; | |
382 | crypto_data_t plaindata, cipherdata; | |
383 | CK_AES_CCM_PARAMS ccmp; | |
384 | CK_AES_GCM_PARAMS gcmp; | |
385 | crypto_mechanism_t mech; | |
386 | zio_crypt_info_t crypt_info; | |
387 | uint_t plain_full_len, maclen; | |
388 | ||
389 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
390 | ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW); | |
391 | ||
392 | /* lookup the encryption info */ | |
393 | crypt_info = zio_crypt_table[crypt]; | |
394 | ||
395 | /* the mac will always be the last iovec_t in the cipher uio */ | |
396 | maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; | |
397 | ||
398 | ASSERT(maclen <= ZIO_DATA_MAC_LEN); | |
399 | ||
400 | /* setup encryption mechanism (same as crypt) */ | |
401 | mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); | |
402 | ||
403 | /* | |
404 | * Strangely, the ICP requires that plain_full_len must include | |
405 | * the MAC length when decrypting, even though the UIO does not | |
406 | * need to have the extra space allocated. | |
407 | */ | |
408 | if (encrypt) { | |
409 | plain_full_len = datalen; | |
410 | } else { | |
411 | plain_full_len = datalen + maclen; | |
412 | } | |
413 | ||
414 | /* | |
415 | * setup encryption params (currently only AES CCM and AES GCM | |
416 | * are supported) | |
417 | */ | |
418 | if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { | |
419 | ccmp.ulNonceSize = ZIO_DATA_IV_LEN; | |
420 | ccmp.ulAuthDataSize = auth_len; | |
421 | ccmp.authData = authbuf; | |
422 | ccmp.ulMACSize = maclen; | |
423 | ccmp.nonce = ivbuf; | |
424 | ccmp.ulDataSize = plain_full_len; | |
425 | ||
426 | mech.cm_param = (char *)(&ccmp); | |
427 | mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); | |
428 | } else { | |
429 | gcmp.ulIvLen = ZIO_DATA_IV_LEN; | |
4807c0ba | 430 | gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); |
b5256303 TC |
431 | gcmp.ulAADLen = auth_len; |
432 | gcmp.pAAD = authbuf; | |
4807c0ba | 433 | gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); |
b5256303 TC |
434 | gcmp.pIv = ivbuf; |
435 | ||
436 | mech.cm_param = (char *)(&gcmp); | |
437 | mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); | |
438 | } | |
439 | ||
440 | /* populate the cipher and plain data structs. */ | |
441 | plaindata.cd_format = CRYPTO_DATA_UIO; | |
442 | plaindata.cd_offset = 0; | |
443 | plaindata.cd_uio = puio; | |
444 | plaindata.cd_miscdata = NULL; | |
445 | plaindata.cd_length = plain_full_len; | |
446 | ||
447 | cipherdata.cd_format = CRYPTO_DATA_UIO; | |
448 | cipherdata.cd_offset = 0; | |
449 | cipherdata.cd_uio = cuio; | |
450 | cipherdata.cd_miscdata = NULL; | |
451 | cipherdata.cd_length = datalen + maclen; | |
452 | ||
453 | /* perform the actual encryption */ | |
454 | if (encrypt) { | |
455 | ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata, | |
456 | NULL); | |
457 | if (ret != CRYPTO_SUCCESS) { | |
458 | ret = SET_ERROR(EIO); | |
459 | goto error; | |
460 | } | |
461 | } else { | |
462 | ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata, | |
463 | NULL); | |
464 | if (ret != CRYPTO_SUCCESS) { | |
465 | ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); | |
466 | ret = SET_ERROR(ECKSUM); | |
467 | goto error; | |
468 | } | |
469 | } | |
470 | ||
471 | return (0); | |
472 | ||
473 | error: | |
474 | return (ret); | |
475 | } | |
476 | ||
477 | int | |
478 | zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, | |
479 | uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) | |
480 | { | |
481 | int ret; | |
482 | uio_t puio, cuio; | |
ae76f45c | 483 | uint64_t aad[3]; |
b5256303 TC |
484 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
485 | uint64_t crypt = key->zk_crypt; | |
ae76f45c | 486 | uint_t enc_len, keydata_len, aad_len; |
b5256303 TC |
487 | |
488 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
489 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
490 | ||
491 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
492 | ||
493 | /* generate iv for wrapping the master and hmac key */ | |
494 | ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); | |
495 | if (ret != 0) | |
496 | goto error; | |
497 | ||
498 | /* initialize uio_ts */ | |
499 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
500 | plain_iovecs[0].iov_len = keydata_len; | |
501 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
502 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
503 | ||
504 | cipher_iovecs[0].iov_base = keydata_out; | |
505 | cipher_iovecs[0].iov_len = keydata_len; | |
506 | cipher_iovecs[1].iov_base = hmac_keydata_out; | |
507 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
508 | cipher_iovecs[2].iov_base = mac; | |
509 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
510 | ||
ae76f45c TC |
511 | /* |
512 | * Although we don't support writing to the old format, we do | |
513 | * support rewrapping the key so that the user can move and | |
514 | * quarantine datasets on the old format. | |
515 | */ | |
516 | if (key->zk_version == 0) { | |
517 | aad_len = sizeof (uint64_t); | |
518 | aad[0] = LE_64(key->zk_guid); | |
519 | } else { | |
520 | ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
521 | aad_len = sizeof (uint64_t) * 3; | |
522 | aad[0] = LE_64(key->zk_guid); | |
523 | aad[1] = LE_64(crypt); | |
524 | aad[2] = LE_64(key->zk_version); | |
525 | } | |
526 | ||
b5256303 TC |
527 | enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; |
528 | puio.uio_iov = plain_iovecs; | |
529 | puio.uio_iovcnt = 2; | |
530 | puio.uio_segflg = UIO_SYSSPACE; | |
531 | cuio.uio_iov = cipher_iovecs; | |
532 | cuio.uio_iovcnt = 3; | |
533 | cuio.uio_segflg = UIO_SYSSPACE; | |
534 | ||
535 | /* encrypt the keys and store the resulting ciphertext and mac */ | |
536 | ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, | |
ae76f45c | 537 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
538 | if (ret != 0) |
539 | goto error; | |
540 | ||
541 | return (0); | |
542 | ||
543 | error: | |
544 | return (ret); | |
545 | } | |
546 | ||
547 | int | |
ae76f45c TC |
548 | zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, |
549 | uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, | |
550 | uint8_t *mac, zio_crypt_key_t *key) | |
b5256303 TC |
551 | { |
552 | int ret; | |
553 | crypto_mechanism_t mech; | |
554 | uio_t puio, cuio; | |
ae76f45c | 555 | uint64_t aad[3]; |
b5256303 | 556 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
ae76f45c | 557 | uint_t enc_len, keydata_len, aad_len; |
b5256303 TC |
558 | |
559 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
560 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
561 | ||
562 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
563 | ||
564 | /* initialize uio_ts */ | |
565 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
566 | plain_iovecs[0].iov_len = keydata_len; | |
567 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
568 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
569 | ||
570 | cipher_iovecs[0].iov_base = keydata; | |
571 | cipher_iovecs[0].iov_len = keydata_len; | |
572 | cipher_iovecs[1].iov_base = hmac_keydata; | |
573 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
574 | cipher_iovecs[2].iov_base = mac; | |
575 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
576 | ||
ae76f45c TC |
577 | if (version == 0) { |
578 | aad_len = sizeof (uint64_t); | |
579 | aad[0] = LE_64(guid); | |
580 | } else { | |
581 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
582 | aad_len = sizeof (uint64_t) * 3; | |
583 | aad[0] = LE_64(guid); | |
584 | aad[1] = LE_64(crypt); | |
585 | aad[2] = LE_64(version); | |
586 | } | |
587 | ||
b5256303 TC |
588 | enc_len = keydata_len + SHA512_HMAC_KEYLEN; |
589 | puio.uio_iov = plain_iovecs; | |
590 | puio.uio_segflg = UIO_SYSSPACE; | |
591 | puio.uio_iovcnt = 2; | |
592 | cuio.uio_iov = cipher_iovecs; | |
593 | cuio.uio_iovcnt = 3; | |
594 | cuio.uio_segflg = UIO_SYSSPACE; | |
595 | ||
596 | /* decrypt the keys and store the result in the output buffers */ | |
597 | ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, | |
ae76f45c | 598 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
599 | if (ret != 0) |
600 | goto error; | |
601 | ||
602 | /* generate a fresh salt */ | |
603 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
604 | if (ret != 0) | |
605 | goto error; | |
606 | ||
607 | /* derive the current key from the master key */ | |
608 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
609 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
610 | keydata_len); | |
611 | if (ret != 0) | |
612 | goto error; | |
613 | ||
614 | /* initialize keys for ICP */ | |
615 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
616 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 617 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
618 | |
619 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
620 | key->zk_hmac_key.ck_data = key->zk_hmac_keydata; | |
4807c0ba | 621 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 TC |
622 | |
623 | /* | |
624 | * Initialize the crypto templates. It's ok if this fails because | |
625 | * this is just an optimization. | |
626 | */ | |
627 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
628 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
629 | &key->zk_current_tmpl, KM_SLEEP); | |
630 | if (ret != CRYPTO_SUCCESS) | |
631 | key->zk_current_tmpl = NULL; | |
632 | ||
633 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
634 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
635 | &key->zk_hmac_tmpl, KM_SLEEP); | |
636 | if (ret != CRYPTO_SUCCESS) | |
637 | key->zk_hmac_tmpl = NULL; | |
638 | ||
639 | key->zk_crypt = crypt; | |
ae76f45c | 640 | key->zk_version = version; |
b5256303 TC |
641 | key->zk_guid = guid; |
642 | key->zk_salt_count = 0; | |
643 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); | |
644 | ||
645 | return (0); | |
646 | ||
647 | error: | |
648 | zio_crypt_key_destroy(key); | |
649 | return (ret); | |
650 | } | |
651 | ||
652 | int | |
653 | zio_crypt_generate_iv(uint8_t *ivbuf) | |
654 | { | |
655 | int ret; | |
656 | ||
657 | /* randomly generate the IV */ | |
658 | ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); | |
659 | if (ret != 0) | |
660 | goto error; | |
661 | ||
662 | return (0); | |
663 | ||
664 | error: | |
665 | bzero(ivbuf, ZIO_DATA_IV_LEN); | |
666 | return (ret); | |
667 | } | |
668 | ||
669 | int | |
670 | zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, | |
4807c0ba | 671 | uint8_t *digestbuf, uint_t digestlen) |
b5256303 TC |
672 | { |
673 | int ret; | |
674 | crypto_mechanism_t mech; | |
675 | crypto_data_t in_data, digest_data; | |
4807c0ba TC |
676 | uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; |
677 | ||
678 | ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
679 | |
680 | /* initialize sha512-hmac mechanism and crypto data */ | |
681 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
682 | mech.cm_param = NULL; | |
683 | mech.cm_param_len = 0; | |
684 | ||
685 | /* initialize the crypto data */ | |
686 | in_data.cd_format = CRYPTO_DATA_RAW; | |
687 | in_data.cd_offset = 0; | |
688 | in_data.cd_length = datalen; | |
689 | in_data.cd_raw.iov_base = (char *)data; | |
690 | in_data.cd_raw.iov_len = in_data.cd_length; | |
691 | ||
692 | digest_data.cd_format = CRYPTO_DATA_RAW; | |
693 | digest_data.cd_offset = 0; | |
4807c0ba | 694 | digest_data.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
695 | digest_data.cd_raw.iov_base = (char *)raw_digestbuf; |
696 | digest_data.cd_raw.iov_len = digest_data.cd_length; | |
697 | ||
698 | /* generate the hmac */ | |
699 | ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, | |
700 | &digest_data, NULL); | |
701 | if (ret != CRYPTO_SUCCESS) { | |
702 | ret = SET_ERROR(EIO); | |
703 | goto error; | |
704 | } | |
705 | ||
4807c0ba | 706 | bcopy(raw_digestbuf, digestbuf, digestlen); |
b5256303 TC |
707 | |
708 | return (0); | |
709 | ||
710 | error: | |
4807c0ba | 711 | bzero(digestbuf, digestlen); |
b5256303 TC |
712 | return (ret); |
713 | } | |
714 | ||
715 | int | |
716 | zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, | |
717 | uint_t datalen, uint8_t *ivbuf, uint8_t *salt) | |
718 | { | |
719 | int ret; | |
4807c0ba | 720 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 | 721 | |
4807c0ba TC |
722 | ret = zio_crypt_do_hmac(key, data, datalen, |
723 | digestbuf, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
724 | if (ret != 0) |
725 | return (ret); | |
726 | ||
727 | bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); | |
728 | bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); | |
729 | ||
730 | return (0); | |
731 | } | |
732 | ||
733 | /* | |
734 | * The following functions are used to encode and decode encryption parameters | |
735 | * into blkptr_t and zil_header_t. The ICP wants to use these parameters as | |
736 | * byte strings, which normally means that these strings would not need to deal | |
737 | * with byteswapping at all. However, both blkptr_t and zil_header_t may be | |
738 | * byteswapped by lower layers and so we must "undo" that byteswap here upon | |
ae76f45c TC |
739 | * decoding and encoding in a non-native byteorder. These functions require |
740 | * that the byteorder bit is correct before being called. | |
b5256303 TC |
741 | */ |
742 | void | |
743 | zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
744 | { | |
ae76f45c | 745 | uint64_t val64; |
b5256303 TC |
746 | uint32_t val32; |
747 | ||
748 | ASSERT(BP_IS_ENCRYPTED(bp)); | |
749 | ||
ae76f45c TC |
750 | if (!BP_SHOULD_BYTESWAP(bp)) { |
751 | bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); | |
752 | bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); | |
753 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
754 | BP_SET_IV2(bp, val32); | |
755 | } else { | |
756 | bcopy(salt, &val64, sizeof (uint64_t)); | |
757 | bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); | |
758 | ||
759 | bcopy(iv, &val64, sizeof (uint64_t)); | |
760 | bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); | |
761 | ||
762 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
763 | BP_SET_IV2(bp, BSWAP_32(val32)); | |
764 | } | |
b5256303 TC |
765 | } |
766 | ||
767 | void | |
768 | zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
769 | { | |
770 | uint64_t val64; | |
771 | uint32_t val32; | |
772 | ||
773 | ASSERT(BP_IS_PROTECTED(bp)); | |
774 | ||
775 | /* for convenience, so callers don't need to check */ | |
776 | if (BP_IS_AUTHENTICATED(bp)) { | |
777 | bzero(salt, ZIO_DATA_SALT_LEN); | |
778 | bzero(iv, ZIO_DATA_IV_LEN); | |
779 | return; | |
780 | } | |
781 | ||
782 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
783 | bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); | |
784 | bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); | |
785 | ||
786 | val32 = (uint32_t)BP_GET_IV2(bp); | |
787 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
788 | } else { | |
789 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); | |
790 | bcopy(&val64, salt, sizeof (uint64_t)); | |
791 | ||
792 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); | |
793 | bcopy(&val64, iv, sizeof (uint64_t)); | |
794 | ||
795 | val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); | |
796 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
797 | } | |
798 | } | |
799 | ||
800 | void | |
801 | zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) | |
802 | { | |
ae76f45c TC |
803 | uint64_t val64; |
804 | ||
b5256303 TC |
805 | ASSERT(BP_USES_CRYPT(bp)); |
806 | ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); | |
807 | ||
ae76f45c TC |
808 | if (!BP_SHOULD_BYTESWAP(bp)) { |
809 | bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); | |
810 | bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], | |
811 | sizeof (uint64_t)); | |
812 | } else { | |
813 | bcopy(mac, &val64, sizeof (uint64_t)); | |
814 | bp->blk_cksum.zc_word[2] = BSWAP_64(val64); | |
815 | ||
816 | bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); | |
817 | bp->blk_cksum.zc_word[3] = BSWAP_64(val64); | |
818 | } | |
b5256303 TC |
819 | } |
820 | ||
821 | void | |
822 | zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) | |
823 | { | |
824 | uint64_t val64; | |
825 | ||
826 | ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); | |
827 | ||
828 | /* for convenience, so callers don't need to check */ | |
829 | if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { | |
830 | bzero(mac, ZIO_DATA_MAC_LEN); | |
831 | return; | |
832 | } | |
833 | ||
834 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
835 | bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
836 | bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), | |
837 | sizeof (uint64_t)); | |
838 | } else { | |
839 | val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); | |
840 | bcopy(&val64, mac, sizeof (uint64_t)); | |
841 | ||
842 | val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); | |
843 | bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); | |
844 | } | |
845 | } | |
846 | ||
847 | void | |
848 | zio_crypt_encode_mac_zil(void *data, uint8_t *mac) | |
849 | { | |
850 | zil_chain_t *zilc = data; | |
851 | ||
852 | bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); | |
853 | bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], | |
854 | sizeof (uint64_t)); | |
855 | } | |
856 | ||
857 | void | |
858 | zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) | |
859 | { | |
860 | /* | |
861 | * The ZIL MAC is embedded in the block it protects, which will | |
862 | * not have been byteswapped by the time this function has been called. | |
863 | * As a result, we don't need to worry about byteswapping the MAC. | |
864 | */ | |
865 | const zil_chain_t *zilc = data; | |
866 | ||
867 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
868 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), | |
869 | sizeof (uint64_t)); | |
870 | } | |
871 | ||
872 | /* | |
873 | * This routine takes a block of dnodes (src_abd) and copies only the bonus | |
874 | * buffers to the same offsets in the dst buffer. datalen should be the size | |
875 | * of both the src_abd and the dst buffer (not just the length of the bonus | |
876 | * buffers). | |
877 | */ | |
878 | void | |
879 | zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) | |
880 | { | |
881 | uint_t i, max_dnp = datalen >> DNODE_SHIFT; | |
882 | uint8_t *src; | |
883 | dnode_phys_t *dnp, *sdnp, *ddnp; | |
884 | ||
885 | src = abd_borrow_buf_copy(src_abd, datalen); | |
886 | ||
887 | sdnp = (dnode_phys_t *)src; | |
888 | ddnp = (dnode_phys_t *)dst; | |
889 | ||
890 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
891 | dnp = &sdnp[i]; | |
892 | if (dnp->dn_type != DMU_OT_NONE && | |
893 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
894 | dnp->dn_bonuslen != 0) { | |
895 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), | |
896 | DN_MAX_BONUS_LEN(dnp)); | |
897 | } | |
898 | } | |
899 | ||
900 | abd_return_buf(src_abd, src, datalen); | |
901 | } | |
902 | ||
ae76f45c TC |
903 | /* |
904 | * This function decides what fields from blk_prop are included in | |
905 | * the on-disk various MAC algorithms. | |
906 | */ | |
b5256303 | 907 | static void |
ae76f45c | 908 | zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) |
b5256303 | 909 | { |
ae76f45c TC |
910 | /* |
911 | * Version 0 did not properly zero out all non-portable fields | |
912 | * as it should have done. We maintain this code so that we can | |
913 | * do read-only imports of pools on this version. | |
914 | */ | |
915 | if (version == 0) { | |
916 | BP_SET_DEDUP(bp, 0); | |
917 | BP_SET_CHECKSUM(bp, 0); | |
918 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
919 | return; | |
920 | } | |
921 | ||
922 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
923 | ||
924 | /* | |
925 | * The hole_birth feature might set these fields even if this bp | |
926 | * is a hole. We zero them out here to guarantee that raw sends | |
927 | * will function with or without the feature. | |
928 | */ | |
929 | if (BP_IS_HOLE(bp)) { | |
930 | bp->blk_prop = 0ULL; | |
931 | return; | |
932 | } | |
b5256303 TC |
933 | |
934 | /* | |
ae76f45c TC |
935 | * At L0 we want to verify these fields to ensure that data blocks |
936 | * can not be reinterpretted. For instance, we do not want an attacker | |
937 | * to trick us into returning raw lz4 compressed data to the user | |
938 | * by modifying the compression bits. At higher levels, we cannot | |
939 | * enforce this policy since raw sends do not convey any information | |
940 | * about indirect blocks, so these values might be different on the | |
941 | * receive side. Fortunately, this does not open any new attack | |
942 | * vectors, since any alterations that can be made to a higher level | |
943 | * bp must still verify the correct order of the layer below it. | |
b5256303 | 944 | */ |
ae76f45c TC |
945 | if (BP_GET_LEVEL(bp) != 0) { |
946 | BP_SET_BYTEORDER(bp, 0); | |
947 | BP_SET_COMPRESS(bp, 0); | |
948 | ||
949 | /* | |
950 | * psize cannot be set to zero or it will trigger | |
951 | * asserts, but the value doesn't really matter as | |
952 | * long as it is constant. | |
953 | */ | |
954 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
955 | } | |
956 | ||
957 | BP_SET_DEDUP(bp, 0); | |
958 | BP_SET_CHECKSUM(bp, 0); | |
b5256303 TC |
959 | } |
960 | ||
ae76f45c TC |
961 | static void |
962 | zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, | |
963 | blkptr_auth_buf_t *bab, uint_t *bab_len) | |
b5256303 | 964 | { |
b5256303 | 965 | blkptr_t tmpbp = *bp; |
b5256303 TC |
966 | |
967 | if (should_bswap) | |
968 | byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); | |
969 | ||
970 | ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); | |
971 | ASSERT0(BP_IS_EMBEDDED(&tmpbp)); | |
b5256303 | 972 | |
ae76f45c | 973 | zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); |
b5256303 | 974 | |
ae76f45c TC |
975 | /* |
976 | * We always MAC blk_prop in LE to ensure portability. This | |
977 | * must be done after decoding the mac, since the endianness | |
978 | * will get zero'd out here. | |
979 | */ | |
980 | zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); | |
981 | bab->bab_prop = LE_64(tmpbp.blk_prop); | |
982 | bab->bab_pad = 0ULL; | |
983 | ||
984 | /* version 0 did not include the padding */ | |
985 | *bab_len = sizeof (blkptr_auth_buf_t); | |
986 | if (version == 0) | |
987 | *bab_len -= sizeof (uint64_t); | |
988 | } | |
b5256303 | 989 | |
ae76f45c TC |
990 | static int |
991 | zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, | |
992 | boolean_t should_bswap, blkptr_t *bp) | |
993 | { | |
994 | int ret; | |
995 | uint_t bab_len; | |
996 | blkptr_auth_buf_t bab; | |
997 | crypto_data_t cd; | |
b5256303 | 998 | |
ae76f45c TC |
999 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1000 | cd.cd_format = CRYPTO_DATA_RAW; | |
1001 | cd.cd_offset = 0; | |
1002 | cd.cd_length = bab_len; | |
1003 | cd.cd_raw.iov_base = (char *)&bab; | |
b5256303 TC |
1004 | cd.cd_raw.iov_len = cd.cd_length; |
1005 | ||
1006 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1007 | if (ret != CRYPTO_SUCCESS) { | |
1008 | ret = SET_ERROR(EIO); | |
1009 | goto error; | |
1010 | } | |
1011 | ||
1012 | return (0); | |
1013 | ||
1014 | error: | |
1015 | return (ret); | |
1016 | } | |
1017 | ||
1018 | static void | |
ae76f45c TC |
1019 | zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, |
1020 | boolean_t should_bswap, blkptr_t *bp) | |
b5256303 | 1021 | { |
ae76f45c TC |
1022 | uint_t bab_len; |
1023 | blkptr_auth_buf_t bab; | |
b5256303 | 1024 | |
ae76f45c TC |
1025 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1026 | SHA2Update(ctx, &bab, bab_len); | |
b5256303 TC |
1027 | } |
1028 | ||
1029 | static void | |
ae76f45c | 1030 | zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, |
b5256303 TC |
1031 | boolean_t should_bswap, blkptr_t *bp) |
1032 | { | |
ae76f45c TC |
1033 | uint_t bab_len; |
1034 | blkptr_auth_buf_t bab; | |
b5256303 | 1035 | |
ae76f45c TC |
1036 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1037 | bcopy(&bab, *aadp, bab_len); | |
1038 | *aadp += bab_len; | |
1039 | *aad_len += bab_len; | |
b5256303 TC |
1040 | } |
1041 | ||
1042 | static int | |
ae76f45c TC |
1043 | zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, |
1044 | boolean_t should_bswap, dnode_phys_t *dnp) | |
b5256303 TC |
1045 | { |
1046 | int ret, i; | |
1047 | dnode_phys_t *adnp; | |
1048 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
1049 | crypto_data_t cd; | |
1050 | uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; | |
1051 | ||
1052 | cd.cd_format = CRYPTO_DATA_RAW; | |
1053 | cd.cd_offset = 0; | |
1054 | ||
1055 | /* authenticate the core dnode (masking out non-portable bits) */ | |
1056 | bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); | |
1057 | adnp = (dnode_phys_t *)tmp_dncore; | |
1058 | if (le_bswap) { | |
1059 | adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); | |
1060 | adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); | |
1061 | adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); | |
1062 | adnp->dn_used = BSWAP_64(adnp->dn_used); | |
1063 | } | |
1064 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1065 | adnp->dn_used = 0; | |
1066 | ||
1067 | cd.cd_length = sizeof (tmp_dncore); | |
1068 | cd.cd_raw.iov_base = (char *)adnp; | |
1069 | cd.cd_raw.iov_len = cd.cd_length; | |
1070 | ||
1071 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1072 | if (ret != CRYPTO_SUCCESS) { | |
1073 | ret = SET_ERROR(EIO); | |
1074 | goto error; | |
1075 | } | |
1076 | ||
1077 | for (i = 0; i < dnp->dn_nblkptr; i++) { | |
ae76f45c | 1078 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1079 | should_bswap, &dnp->dn_blkptr[i]); |
1080 | if (ret != 0) | |
1081 | goto error; | |
1082 | } | |
1083 | ||
1084 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
ae76f45c | 1085 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1086 | should_bswap, DN_SPILL_BLKPTR(dnp)); |
1087 | if (ret != 0) | |
1088 | goto error; | |
1089 | } | |
1090 | ||
1091 | return (0); | |
1092 | ||
1093 | error: | |
1094 | return (ret); | |
1095 | } | |
1096 | ||
1097 | /* | |
1098 | * objset_phys_t blocks introduce a number of exceptions to the normal | |
7b30ee6b | 1099 | * authentication process. objset_phys_t's contain 2 separate HMACS for |
b5256303 | 1100 | * protecting the integrity of their data. The portable_mac protects the |
7b30ee6b | 1101 | * metadnode. This MAC can be sent with a raw send and protects against |
b5256303 TC |
1102 | * reordering of data within the metadnode. The local_mac protects the user |
1103 | * accounting objects which are not sent from one system to another. | |
1104 | * | |
1105 | * In addition, objset blocks are the only blocks that can be modified and | |
1106 | * written to disk without the key loaded under certain circumstances. During | |
1107 | * zil_claim() we need to be able to update the zil_header_t to complete | |
1108 | * claiming log blocks and during raw receives we need to write out the | |
1109 | * portable_mac from the send file. Both of these actions are possible | |
1110 | * because these fields are not protected by either MAC so neither one will | |
1111 | * need to modify the MACs without the key. However, when the modified blocks | |
1112 | * are written out they will be byteswapped into the host machine's native | |
1113 | * endianness which will modify fields protected by the MAC. As a result, MAC | |
1114 | * calculation for objset blocks works slightly differently from other block | |
1115 | * types. Where other block types MAC the data in whatever endianness is | |
1116 | * written to disk, objset blocks always MAC little endian version of their | |
1117 | * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() | |
1118 | * and le_bswap indicates whether a byteswap is needed to get this block | |
1119 | * into little endian format. | |
1120 | */ | |
1121 | int | |
1122 | zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, | |
1123 | boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) | |
1124 | { | |
1125 | int ret; | |
1126 | crypto_mechanism_t mech; | |
1127 | crypto_context_t ctx; | |
1128 | crypto_data_t cd; | |
1129 | objset_phys_t *osp = data; | |
1130 | uint64_t intval; | |
1131 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
4807c0ba TC |
1132 | uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; |
1133 | uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; | |
b5256303 TC |
1134 | |
1135 | /* initialize HMAC mechanism */ | |
1136 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
1137 | mech.cm_param = NULL; | |
1138 | mech.cm_param_len = 0; | |
1139 | ||
1140 | cd.cd_format = CRYPTO_DATA_RAW; | |
1141 | cd.cd_offset = 0; | |
1142 | ||
1143 | /* calculate the portable MAC from the portable fields and metadnode */ | |
1144 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1145 | if (ret != CRYPTO_SUCCESS) { | |
1146 | ret = SET_ERROR(EIO); | |
1147 | goto error; | |
1148 | } | |
1149 | ||
1150 | /* add in the os_type */ | |
1151 | intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); | |
1152 | cd.cd_length = sizeof (uint64_t); | |
1153 | cd.cd_raw.iov_base = (char *)&intval; | |
1154 | cd.cd_raw.iov_len = cd.cd_length; | |
1155 | ||
1156 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1157 | if (ret != CRYPTO_SUCCESS) { | |
1158 | ret = SET_ERROR(EIO); | |
1159 | goto error; | |
1160 | } | |
1161 | ||
1162 | /* add in the portable os_flags */ | |
1163 | intval = osp->os_flags; | |
1164 | if (should_bswap) | |
1165 | intval = BSWAP_64(intval); | |
1166 | intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1167 | if (!ZFS_HOST_BYTEORDER) | |
1168 | intval = BSWAP_64(intval); | |
1169 | ||
1170 | cd.cd_length = sizeof (uint64_t); | |
1171 | cd.cd_raw.iov_base = (char *)&intval; | |
1172 | cd.cd_raw.iov_len = cd.cd_length; | |
1173 | ||
1174 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1175 | if (ret != CRYPTO_SUCCESS) { | |
1176 | ret = SET_ERROR(EIO); | |
1177 | goto error; | |
1178 | } | |
1179 | ||
1180 | /* add in fields from the metadnode */ | |
ae76f45c TC |
1181 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, |
1182 | should_bswap, &osp->os_meta_dnode); | |
b5256303 TC |
1183 | if (ret) |
1184 | goto error; | |
1185 | ||
1186 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1187 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1188 | cd.cd_raw.iov_base = (char *)raw_portable_mac; |
1189 | cd.cd_raw.iov_len = cd.cd_length; | |
1190 | ||
1191 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1192 | if (ret != CRYPTO_SUCCESS) { | |
1193 | ret = SET_ERROR(EIO); | |
1194 | goto error; | |
1195 | } | |
1196 | ||
1197 | bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); | |
1198 | ||
1199 | /* | |
f8478fc2 TC |
1200 | * The local MAC protects the user, group and project accounting. |
1201 | * If these objects are not present, the local MAC is zeroed out. | |
b5256303 | 1202 | */ |
f8478fc2 | 1203 | if ((datalen >= OBJSET_PHYS_SIZE_V3 && |
7b30ee6b TC |
1204 | osp->os_userused_dnode.dn_type == DMU_OT_NONE && |
1205 | osp->os_groupused_dnode.dn_type == DMU_OT_NONE && | |
f8478fc2 TC |
1206 | osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || |
1207 | (datalen >= OBJSET_PHYS_SIZE_V2 && | |
1208 | osp->os_userused_dnode.dn_type == DMU_OT_NONE && | |
1209 | osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || | |
1210 | (datalen <= OBJSET_PHYS_SIZE_V1)) { | |
b5256303 TC |
1211 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); |
1212 | return (0); | |
1213 | } | |
1214 | ||
1215 | /* calculate the local MAC from the userused and groupused dnodes */ | |
1216 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1217 | if (ret != CRYPTO_SUCCESS) { | |
1218 | ret = SET_ERROR(EIO); | |
1219 | goto error; | |
1220 | } | |
1221 | ||
1222 | /* add in the non-portable os_flags */ | |
1223 | intval = osp->os_flags; | |
1224 | if (should_bswap) | |
1225 | intval = BSWAP_64(intval); | |
1226 | intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1227 | if (!ZFS_HOST_BYTEORDER) | |
1228 | intval = BSWAP_64(intval); | |
1229 | ||
1230 | cd.cd_length = sizeof (uint64_t); | |
1231 | cd.cd_raw.iov_base = (char *)&intval; | |
1232 | cd.cd_raw.iov_len = cd.cd_length; | |
1233 | ||
1234 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1235 | if (ret != CRYPTO_SUCCESS) { | |
1236 | ret = SET_ERROR(EIO); | |
1237 | goto error; | |
1238 | } | |
1239 | ||
1240 | /* add in fields from the user accounting dnodes */ | |
7b30ee6b TC |
1241 | if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { |
1242 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1243 | should_bswap, &osp->os_userused_dnode); | |
1244 | if (ret) | |
1245 | goto error; | |
1246 | } | |
b5256303 | 1247 | |
7b30ee6b TC |
1248 | if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { |
1249 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1250 | should_bswap, &osp->os_groupused_dnode); | |
1251 | if (ret) | |
1252 | goto error; | |
1253 | } | |
1254 | ||
1255 | if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && | |
1256 | datalen >= OBJSET_PHYS_SIZE_V3) { | |
1257 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1258 | should_bswap, &osp->os_projectused_dnode); | |
1259 | if (ret) | |
1260 | goto error; | |
1261 | } | |
b5256303 TC |
1262 | |
1263 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1264 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1265 | cd.cd_raw.iov_base = (char *)raw_local_mac; |
1266 | cd.cd_raw.iov_len = cd.cd_length; | |
1267 | ||
1268 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1269 | if (ret != CRYPTO_SUCCESS) { | |
1270 | ret = SET_ERROR(EIO); | |
1271 | goto error; | |
1272 | } | |
1273 | ||
1274 | bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); | |
1275 | ||
1276 | return (0); | |
1277 | ||
1278 | error: | |
1279 | bzero(portable_mac, ZIO_OBJSET_MAC_LEN); | |
1280 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); | |
1281 | return (ret); | |
1282 | } | |
1283 | ||
1284 | static void | |
1285 | zio_crypt_destroy_uio(uio_t *uio) | |
1286 | { | |
1287 | if (uio->uio_iov) | |
1288 | kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); | |
1289 | } | |
1290 | ||
1291 | /* | |
1292 | * This function parses an uncompressed indirect block and returns a checksum | |
1293 | * of all the portable fields from all of the contained bps. The portable | |
1294 | * fields are the MAC and all of the fields from blk_prop except for the dedup, | |
1295 | * checksum, and psize bits. For an explanation of the purpose of this, see | |
1296 | * the comment block on object set authentication. | |
1297 | */ | |
ae76f45c TC |
1298 | static int |
1299 | zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, | |
1300 | uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) | |
b5256303 TC |
1301 | { |
1302 | blkptr_t *bp; | |
1303 | int i, epb = datalen >> SPA_BLKPTRSHIFT; | |
1304 | SHA2_CTX ctx; | |
4807c0ba | 1305 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 TC |
1306 | |
1307 | /* checksum all of the MACs from the layer below */ | |
1308 | SHA2Init(SHA512, &ctx); | |
1309 | for (i = 0, bp = buf; i < epb; i++, bp++) { | |
ae76f45c TC |
1310 | zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, |
1311 | byteswap, bp); | |
b5256303 TC |
1312 | } |
1313 | SHA2Final(digestbuf, &ctx); | |
1314 | ||
1315 | if (generate) { | |
1316 | bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); | |
1317 | return (0); | |
1318 | } | |
1319 | ||
1320 | if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) | |
1321 | return (SET_ERROR(ECKSUM)); | |
1322 | ||
1323 | return (0); | |
1324 | } | |
1325 | ||
1326 | int | |
ae76f45c | 1327 | zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, |
b5256303 TC |
1328 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) |
1329 | { | |
ae76f45c | 1330 | int ret; |
b5256303 | 1331 | |
ae76f45c TC |
1332 | /* |
1333 | * Unfortunately, callers of this function will not always have | |
1334 | * easy access to the on-disk format version. This info is | |
1335 | * normally found in the DSL Crypto Key, but the checksum-of-MACs | |
1336 | * is expected to be verifiable even when the key isn't loaded. | |
1337 | * Here, instead of doing a ZAP lookup for the version for each | |
1338 | * zio, we simply try both existing formats. | |
1339 | */ | |
1340 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, | |
1341 | datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); | |
1342 | if (ret == ECKSUM) { | |
1343 | ASSERT(!generate); | |
1344 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, | |
1345 | buf, datalen, 0, byteswap, cksum); | |
1346 | } | |
1347 | ||
1348 | return (ret); | |
1349 | } | |
1350 | ||
1351 | int | |
1352 | zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, | |
1353 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) | |
1354 | { | |
b5256303 TC |
1355 | int ret; |
1356 | void *buf; | |
1357 | ||
1358 | buf = abd_borrow_buf_copy(abd, datalen); | |
1359 | ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, | |
1360 | byteswap, cksum); | |
1361 | abd_return_buf(abd, buf, datalen); | |
1362 | ||
1363 | return (ret); | |
1364 | } | |
1365 | ||
1366 | /* | |
1367 | * Special case handling routine for encrypting / decrypting ZIL blocks. | |
1368 | * We do not check for the older ZIL chain because the encryption feature | |
1369 | * was not available before the newer ZIL chain was introduced. The goal | |
1370 | * here is to encrypt everything except the blkptr_t of a lr_write_t and | |
1371 | * the zil_chain_t header. Everything that is not encrypted is authenticated. | |
1372 | */ | |
1373 | static int | |
1374 | zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, | |
1375 | uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio, | |
1376 | uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, | |
1377 | boolean_t *no_crypt) | |
1378 | { | |
1379 | int ret; | |
9bae371c TC |
1380 | uint64_t txtype, lr_len; |
1381 | uint_t nr_src, nr_dst, crypt_len; | |
b5256303 TC |
1382 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; |
1383 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1384 | uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; | |
1385 | zil_chain_t *zilc; | |
1386 | lr_t *lr; | |
1387 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1388 | ||
1389 | /* cipherbuf always needs an extra iovec for the MAC */ | |
1390 | if (encrypt) { | |
1391 | src = plainbuf; | |
1392 | dst = cipherbuf; | |
1393 | nr_src = 0; | |
1394 | nr_dst = 1; | |
1395 | } else { | |
1396 | src = cipherbuf; | |
1397 | dst = plainbuf; | |
1398 | nr_src = 1; | |
1399 | nr_dst = 0; | |
1400 | } | |
1401 | ||
1402 | /* find the start and end record of the log block */ | |
1403 | zilc = (zil_chain_t *)src; | |
1404 | slrp = src + sizeof (zil_chain_t); | |
1405 | aadp = aadbuf; | |
1406 | blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); | |
1407 | ||
1408 | /* calculate the number of encrypted iovecs we will need */ | |
1409 | for (; slrp < blkend; slrp += lr_len) { | |
1410 | lr = (lr_t *)slrp; | |
1411 | ||
1412 | if (!byteswap) { | |
1413 | txtype = lr->lrc_txtype; | |
1414 | lr_len = lr->lrc_reclen; | |
1415 | } else { | |
1416 | txtype = BSWAP_64(lr->lrc_txtype); | |
1417 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1418 | } | |
1419 | ||
1420 | nr_iovecs++; | |
1421 | if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) | |
1422 | nr_iovecs++; | |
1423 | } | |
1424 | ||
1425 | nr_src += nr_iovecs; | |
1426 | nr_dst += nr_iovecs; | |
1427 | ||
1428 | /* allocate the iovec arrays */ | |
1429 | if (nr_src != 0) { | |
1430 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1431 | if (src_iovecs == NULL) { |
b5256303 TC |
1432 | ret = SET_ERROR(ENOMEM); |
1433 | goto error; | |
1434 | } | |
1435 | } | |
1436 | ||
1437 | if (nr_dst != 0) { | |
1438 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1439 | if (dst_iovecs == NULL) { |
b5256303 TC |
1440 | ret = SET_ERROR(ENOMEM); |
1441 | goto error; | |
1442 | } | |
1443 | } | |
1444 | ||
1445 | /* | |
1446 | * Copy the plain zil header over and authenticate everything except | |
1447 | * the checksum that will store our MAC. If we are writing the data | |
1448 | * the embedded checksum will not have been calculated yet, so we don't | |
1449 | * authenticate that. | |
1450 | */ | |
1451 | bcopy(src, dst, sizeof (zil_chain_t)); | |
1452 | bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); | |
1453 | aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1454 | aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1455 | ||
1456 | /* loop over records again, filling in iovecs */ | |
1457 | nr_iovecs = 0; | |
1458 | slrp = src + sizeof (zil_chain_t); | |
1459 | dlrp = dst + sizeof (zil_chain_t); | |
1460 | ||
1461 | for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { | |
1462 | lr = (lr_t *)slrp; | |
1463 | ||
1464 | if (!byteswap) { | |
1465 | txtype = lr->lrc_txtype; | |
1466 | lr_len = lr->lrc_reclen; | |
1467 | } else { | |
1468 | txtype = BSWAP_64(lr->lrc_txtype); | |
1469 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1470 | } | |
1471 | ||
1472 | /* copy the common lr_t */ | |
1473 | bcopy(slrp, dlrp, sizeof (lr_t)); | |
1474 | bcopy(slrp, aadp, sizeof (lr_t)); | |
1475 | aadp += sizeof (lr_t); | |
1476 | aad_len += sizeof (lr_t); | |
1477 | ||
4807c0ba TC |
1478 | ASSERT3P(src_iovecs, !=, NULL); |
1479 | ASSERT3P(dst_iovecs, !=, NULL); | |
1480 | ||
b5256303 TC |
1481 | /* |
1482 | * If this is a TX_WRITE record we want to encrypt everything | |
1483 | * except the bp if exists. If the bp does exist we want to | |
1484 | * authenticate it. | |
1485 | */ | |
1486 | if (txtype == TX_WRITE) { | |
1487 | crypt_len = sizeof (lr_write_t) - | |
1488 | sizeof (lr_t) - sizeof (blkptr_t); | |
1489 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1490 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1491 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1492 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1493 | ||
1494 | /* copy the bp now since it will not be encrypted */ | |
1495 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1496 | dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1497 | sizeof (blkptr_t)); | |
1498 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1499 | aadp, sizeof (blkptr_t)); | |
1500 | aadp += sizeof (blkptr_t); | |
1501 | aad_len += sizeof (blkptr_t); | |
1502 | nr_iovecs++; | |
1503 | total_len += crypt_len; | |
1504 | ||
1505 | if (lr_len != sizeof (lr_write_t)) { | |
1506 | crypt_len = lr_len - sizeof (lr_write_t); | |
1507 | src_iovecs[nr_iovecs].iov_base = | |
1508 | slrp + sizeof (lr_write_t); | |
1509 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1510 | dst_iovecs[nr_iovecs].iov_base = | |
1511 | dlrp + sizeof (lr_write_t); | |
1512 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1513 | nr_iovecs++; | |
1514 | total_len += crypt_len; | |
1515 | } | |
1516 | } else { | |
1517 | crypt_len = lr_len - sizeof (lr_t); | |
1518 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1519 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1520 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1521 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1522 | nr_iovecs++; | |
1523 | total_len += crypt_len; | |
1524 | } | |
1525 | } | |
1526 | ||
1527 | *no_crypt = (nr_iovecs == 0); | |
1528 | *enc_len = total_len; | |
1529 | *authbuf = aadbuf; | |
1530 | *auth_len = aad_len; | |
1531 | ||
1532 | if (encrypt) { | |
1533 | puio->uio_iov = src_iovecs; | |
1534 | puio->uio_iovcnt = nr_src; | |
1535 | cuio->uio_iov = dst_iovecs; | |
1536 | cuio->uio_iovcnt = nr_dst; | |
1537 | } else { | |
1538 | puio->uio_iov = dst_iovecs; | |
1539 | puio->uio_iovcnt = nr_dst; | |
1540 | cuio->uio_iov = src_iovecs; | |
1541 | cuio->uio_iovcnt = nr_src; | |
1542 | } | |
1543 | ||
1544 | return (0); | |
1545 | ||
1546 | error: | |
1547 | zio_buf_free(aadbuf, datalen); | |
1548 | if (src_iovecs != NULL) | |
1549 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1550 | if (dst_iovecs != NULL) | |
1551 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1552 | ||
1553 | *enc_len = 0; | |
1554 | *authbuf = NULL; | |
1555 | *auth_len = 0; | |
1556 | *no_crypt = B_FALSE; | |
1557 | puio->uio_iov = NULL; | |
1558 | puio->uio_iovcnt = 0; | |
1559 | cuio->uio_iov = NULL; | |
1560 | cuio->uio_iovcnt = 0; | |
1561 | return (ret); | |
1562 | } | |
1563 | ||
1564 | /* | |
1565 | * Special case handling routine for encrypting / decrypting dnode blocks. | |
1566 | */ | |
1567 | static int | |
ae76f45c TC |
1568 | zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, |
1569 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1570 | uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
1571 | uint_t *auth_len, boolean_t *no_crypt) | |
b5256303 TC |
1572 | { |
1573 | int ret; | |
1574 | uint_t nr_src, nr_dst, crypt_len; | |
1575 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; | |
1576 | uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; | |
1577 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1578 | uint8_t *src, *dst, *aadp; | |
1579 | dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; | |
1580 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1581 | ||
1582 | if (encrypt) { | |
1583 | src = plainbuf; | |
1584 | dst = cipherbuf; | |
1585 | nr_src = 0; | |
1586 | nr_dst = 1; | |
1587 | } else { | |
1588 | src = cipherbuf; | |
1589 | dst = plainbuf; | |
1590 | nr_src = 1; | |
1591 | nr_dst = 0; | |
1592 | } | |
1593 | ||
1594 | sdnp = (dnode_phys_t *)src; | |
1595 | ddnp = (dnode_phys_t *)dst; | |
1596 | aadp = aadbuf; | |
1597 | ||
1598 | /* | |
1599 | * Count the number of iovecs we will need to do the encryption by | |
1600 | * counting the number of bonus buffers that need to be encrypted. | |
1601 | */ | |
1602 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1603 | /* | |
1604 | * This block may still be byteswapped. However, all of the | |
1605 | * values we use are either uint8_t's (for which byteswapping | |
1606 | * is a noop) or a * != 0 check, which will work regardless | |
1607 | * of whether or not we byteswap. | |
1608 | */ | |
1609 | if (sdnp[i].dn_type != DMU_OT_NONE && | |
1610 | DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && | |
1611 | sdnp[i].dn_bonuslen != 0) { | |
1612 | nr_iovecs++; | |
1613 | } | |
1614 | } | |
1615 | ||
1616 | nr_src += nr_iovecs; | |
1617 | nr_dst += nr_iovecs; | |
1618 | ||
1619 | if (nr_src != 0) { | |
1620 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1621 | if (src_iovecs == NULL) { |
b5256303 TC |
1622 | ret = SET_ERROR(ENOMEM); |
1623 | goto error; | |
1624 | } | |
1625 | } | |
1626 | ||
1627 | if (nr_dst != 0) { | |
1628 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1629 | if (dst_iovecs == NULL) { |
b5256303 TC |
1630 | ret = SET_ERROR(ENOMEM); |
1631 | goto error; | |
1632 | } | |
1633 | } | |
1634 | ||
1635 | nr_iovecs = 0; | |
1636 | ||
1637 | /* | |
1638 | * Iterate through the dnodes again, this time filling in the uios | |
1639 | * we allocated earlier. We also concatenate any data we want to | |
1640 | * authenticate onto aadbuf. | |
1641 | */ | |
1642 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1643 | dnp = &sdnp[i]; | |
1644 | ||
1645 | /* copy over the core fields and blkptrs (kept as plaintext) */ | |
1646 | bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); | |
1647 | ||
1648 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1649 | bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), | |
1650 | sizeof (blkptr_t)); | |
1651 | } | |
1652 | ||
1653 | /* | |
1654 | * Handle authenticated data. We authenticate everything in | |
1655 | * the dnode that can be brought over when we do a raw send. | |
1656 | * This includes all of the core fields as well as the MACs | |
1657 | * stored in the bp checksums and all of the portable bits | |
1658 | * from blk_prop. We include the dnode padding here in case it | |
1659 | * ever gets used in the future. Some dn_flags and dn_used are | |
1660 | * not portable so we mask those out values out of the | |
1661 | * authenticated data. | |
1662 | */ | |
1663 | crypt_len = offsetof(dnode_phys_t, dn_blkptr); | |
1664 | bcopy(dnp, aadp, crypt_len); | |
1665 | adnp = (dnode_phys_t *)aadp; | |
1666 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1667 | adnp->dn_used = 0; | |
1668 | aadp += crypt_len; | |
1669 | aad_len += crypt_len; | |
1670 | ||
1671 | for (j = 0; j < dnp->dn_nblkptr; j++) { | |
1672 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1673 | version, byteswap, &dnp->dn_blkptr[j]); |
b5256303 TC |
1674 | } |
1675 | ||
1676 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1677 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1678 | version, byteswap, DN_SPILL_BLKPTR(dnp)); |
b5256303 TC |
1679 | } |
1680 | ||
1681 | /* | |
1682 | * If this bonus buffer needs to be encrypted, we prepare an | |
1683 | * iovec_t. The encryption / decryption functions will fill | |
1684 | * this in for us with the encrypted or decrypted data. | |
1685 | * Otherwise we add the bonus buffer to the authenticated | |
1686 | * data buffer and copy it over to the destination. The | |
1687 | * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that | |
1688 | * we can guarantee alignment with the AES block size | |
1689 | * (128 bits). | |
1690 | */ | |
1691 | crypt_len = DN_MAX_BONUS_LEN(dnp); | |
1692 | if (dnp->dn_type != DMU_OT_NONE && | |
1693 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
1694 | dnp->dn_bonuslen != 0) { | |
4807c0ba TC |
1695 | ASSERT3U(nr_iovecs, <, nr_src); |
1696 | ASSERT3U(nr_iovecs, <, nr_dst); | |
1697 | ASSERT3P(src_iovecs, !=, NULL); | |
1698 | ASSERT3P(dst_iovecs, !=, NULL); | |
b5256303 TC |
1699 | src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); |
1700 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1701 | dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); | |
1702 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1703 | ||
1704 | nr_iovecs++; | |
1705 | total_len += crypt_len; | |
1706 | } else { | |
1707 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); | |
1708 | bcopy(DN_BONUS(dnp), aadp, crypt_len); | |
1709 | aadp += crypt_len; | |
1710 | aad_len += crypt_len; | |
1711 | } | |
1712 | } | |
1713 | ||
1714 | *no_crypt = (nr_iovecs == 0); | |
1715 | *enc_len = total_len; | |
1716 | *authbuf = aadbuf; | |
1717 | *auth_len = aad_len; | |
1718 | ||
1719 | if (encrypt) { | |
1720 | puio->uio_iov = src_iovecs; | |
1721 | puio->uio_iovcnt = nr_src; | |
1722 | cuio->uio_iov = dst_iovecs; | |
1723 | cuio->uio_iovcnt = nr_dst; | |
1724 | } else { | |
1725 | puio->uio_iov = dst_iovecs; | |
1726 | puio->uio_iovcnt = nr_dst; | |
1727 | cuio->uio_iov = src_iovecs; | |
1728 | cuio->uio_iovcnt = nr_src; | |
1729 | } | |
1730 | ||
1731 | return (0); | |
1732 | ||
1733 | error: | |
1734 | zio_buf_free(aadbuf, datalen); | |
1735 | if (src_iovecs != NULL) | |
1736 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1737 | if (dst_iovecs != NULL) | |
1738 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1739 | ||
1740 | *enc_len = 0; | |
1741 | *authbuf = NULL; | |
1742 | *auth_len = 0; | |
1743 | *no_crypt = B_FALSE; | |
1744 | puio->uio_iov = NULL; | |
1745 | puio->uio_iovcnt = 0; | |
1746 | cuio->uio_iov = NULL; | |
1747 | cuio->uio_iovcnt = 0; | |
1748 | return (ret); | |
1749 | } | |
1750 | ||
1751 | static int | |
1752 | zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, | |
1753 | uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio, | |
1754 | uint_t *enc_len) | |
1755 | { | |
1756 | int ret; | |
1757 | uint_t nr_plain = 1, nr_cipher = 2; | |
1758 | iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; | |
1759 | ||
1760 | /* allocate the iovecs for the plain and cipher data */ | |
1761 | plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), | |
1762 | KM_SLEEP); | |
1763 | if (!plain_iovecs) { | |
1764 | ret = SET_ERROR(ENOMEM); | |
1765 | goto error; | |
1766 | } | |
1767 | ||
1768 | cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), | |
1769 | KM_SLEEP); | |
1770 | if (!cipher_iovecs) { | |
1771 | ret = SET_ERROR(ENOMEM); | |
1772 | goto error; | |
1773 | } | |
1774 | ||
1775 | plain_iovecs[0].iov_base = plainbuf; | |
1776 | plain_iovecs[0].iov_len = datalen; | |
1777 | cipher_iovecs[0].iov_base = cipherbuf; | |
1778 | cipher_iovecs[0].iov_len = datalen; | |
1779 | ||
1780 | *enc_len = datalen; | |
1781 | puio->uio_iov = plain_iovecs; | |
1782 | puio->uio_iovcnt = nr_plain; | |
1783 | cuio->uio_iov = cipher_iovecs; | |
1784 | cuio->uio_iovcnt = nr_cipher; | |
1785 | ||
1786 | return (0); | |
1787 | ||
1788 | error: | |
1789 | if (plain_iovecs != NULL) | |
1790 | kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); | |
1791 | if (cipher_iovecs != NULL) | |
1792 | kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); | |
1793 | ||
1794 | *enc_len = 0; | |
1795 | puio->uio_iov = NULL; | |
1796 | puio->uio_iovcnt = 0; | |
1797 | cuio->uio_iov = NULL; | |
1798 | cuio->uio_iovcnt = 0; | |
1799 | return (ret); | |
1800 | } | |
1801 | ||
1802 | /* | |
1803 | * This function builds up the plaintext (puio) and ciphertext (cuio) uios so | |
1804 | * that they can be used for encryption and decryption by zio_do_crypt_uio(). | |
1805 | * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks | |
1806 | * requiring special handling to parse out pieces that are to be encrypted. The | |
1807 | * authbuf is used by these special cases to store additional authenticated | |
1808 | * data (AAD) for the encryption modes. | |
1809 | */ | |
1810 | static int | |
ae76f45c TC |
1811 | zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, |
1812 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1813 | uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
b5256303 TC |
1814 | uint_t *auth_len, boolean_t *no_crypt) |
1815 | { | |
1816 | int ret; | |
1817 | iovec_t *mac_iov; | |
1818 | ||
1819 | ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); | |
1820 | ||
1821 | /* route to handler */ | |
1822 | switch (ot) { | |
1823 | case DMU_OT_INTENT_LOG: | |
1824 | ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, | |
1825 | datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, | |
1826 | no_crypt); | |
1827 | break; | |
1828 | case DMU_OT_DNODE: | |
ae76f45c TC |
1829 | ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, |
1830 | cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, | |
1831 | auth_len, no_crypt); | |
b5256303 TC |
1832 | break; |
1833 | default: | |
1834 | ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, | |
1835 | datalen, puio, cuio, enc_len); | |
1836 | *authbuf = NULL; | |
1837 | *auth_len = 0; | |
1838 | *no_crypt = B_FALSE; | |
1839 | break; | |
1840 | } | |
1841 | ||
1842 | if (ret != 0) | |
1843 | goto error; | |
1844 | ||
1845 | /* populate the uios */ | |
1846 | puio->uio_segflg = UIO_SYSSPACE; | |
1847 | cuio->uio_segflg = UIO_SYSSPACE; | |
1848 | ||
1849 | mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); | |
1850 | mac_iov->iov_base = mac; | |
1851 | mac_iov->iov_len = ZIO_DATA_MAC_LEN; | |
1852 | ||
1853 | return (0); | |
1854 | ||
1855 | error: | |
1856 | return (ret); | |
1857 | } | |
1858 | ||
1859 | /* | |
1860 | * Primary encryption / decryption entrypoint for zio data. | |
1861 | */ | |
1862 | int | |
1863 | zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt, | |
1864 | dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen, | |
1865 | boolean_t byteswap, uint8_t *plainbuf, uint8_t *cipherbuf, | |
1866 | boolean_t *no_crypt) | |
1867 | { | |
1868 | int ret; | |
1869 | boolean_t locked = B_FALSE; | |
1870 | uint64_t crypt = key->zk_crypt; | |
1871 | uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; | |
1872 | uint_t enc_len, auth_len; | |
1873 | uio_t puio, cuio; | |
1874 | uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; | |
1875 | crypto_key_t tmp_ckey, *ckey = NULL; | |
1876 | crypto_ctx_template_t tmpl; | |
1877 | uint8_t *authbuf = NULL; | |
1878 | ||
b5256303 TC |
1879 | /* |
1880 | * If the needed key is the current one, just use it. Otherwise we | |
1881 | * need to generate a temporary one from the given salt + master key. | |
1882 | * If we are encrypting, we must return a copy of the current salt | |
1883 | * so that it can be stored in the blkptr_t. | |
1884 | */ | |
1885 | rw_enter(&key->zk_salt_lock, RW_READER); | |
1886 | locked = B_TRUE; | |
1887 | ||
1888 | if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { | |
1889 | ckey = &key->zk_current_key; | |
1890 | tmpl = key->zk_current_tmpl; | |
1891 | } else { | |
1892 | rw_exit(&key->zk_salt_lock); | |
1893 | locked = B_FALSE; | |
1894 | ||
1895 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
1896 | salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); | |
1897 | if (ret != 0) | |
1898 | goto error; | |
1899 | ||
1900 | tmp_ckey.ck_format = CRYPTO_KEY_RAW; | |
1901 | tmp_ckey.ck_data = enc_keydata; | |
4807c0ba | 1902 | tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
1903 | |
1904 | ckey = &tmp_ckey; | |
1905 | tmpl = NULL; | |
1906 | } | |
1907 | ||
cf637391 TC |
1908 | /* |
1909 | * Attempt to use QAT acceleration if we can. We currently don't | |
1910 | * do this for metadnode and ZIL blocks, since they have a much | |
1911 | * more involved buffer layout and the qat_crypt() function only | |
1912 | * works in-place. | |
1913 | */ | |
1914 | if (qat_crypt_use_accel(datalen) && | |
1915 | ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { | |
1916 | uint8_t *srcbuf, *dstbuf; | |
1917 | ||
1918 | if (encrypt) { | |
1919 | srcbuf = plainbuf; | |
1920 | dstbuf = cipherbuf; | |
1921 | } else { | |
1922 | srcbuf = cipherbuf; | |
1923 | dstbuf = plainbuf; | |
1924 | } | |
1925 | ||
1926 | ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, | |
1927 | dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); | |
1928 | if (ret == CPA_STATUS_SUCCESS) { | |
1929 | if (locked) { | |
1930 | rw_exit(&key->zk_salt_lock); | |
1931 | locked = B_FALSE; | |
1932 | } | |
1933 | ||
1934 | return (0); | |
1935 | } | |
1936 | /* If the hardware implementation fails fall back to software */ | |
1937 | } | |
1938 | ||
1939 | bzero(&puio, sizeof (uio_t)); | |
1940 | bzero(&cuio, sizeof (uio_t)); | |
1941 | ||
1942 | /* create uios for encryption */ | |
1943 | ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, | |
1944 | cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, | |
1945 | &authbuf, &auth_len, no_crypt); | |
1946 | if (ret != 0) | |
1947 | goto error; | |
1948 | ||
1949 | /* perform the encryption / decryption in software */ | |
b5256303 TC |
1950 | ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, |
1951 | &puio, &cuio, authbuf, auth_len); | |
1952 | if (ret != 0) | |
1953 | goto error; | |
1954 | ||
1955 | if (locked) { | |
1956 | rw_exit(&key->zk_salt_lock); | |
1957 | locked = B_FALSE; | |
1958 | } | |
1959 | ||
1960 | if (authbuf != NULL) | |
1961 | zio_buf_free(authbuf, datalen); | |
1962 | if (ckey == &tmp_ckey) | |
1963 | bzero(enc_keydata, keydata_len); | |
1964 | zio_crypt_destroy_uio(&puio); | |
1965 | zio_crypt_destroy_uio(&cuio); | |
1966 | ||
1967 | return (0); | |
1968 | ||
1969 | error: | |
1970 | if (locked) | |
1971 | rw_exit(&key->zk_salt_lock); | |
1972 | if (authbuf != NULL) | |
1973 | zio_buf_free(authbuf, datalen); | |
1974 | if (ckey == &tmp_ckey) | |
1975 | bzero(enc_keydata, keydata_len); | |
1976 | zio_crypt_destroy_uio(&puio); | |
1977 | zio_crypt_destroy_uio(&cuio); | |
1978 | ||
1979 | return (ret); | |
1980 | } | |
1981 | ||
1982 | /* | |
1983 | * Simple wrapper around zio_do_crypt_data() to work with abd's instead of | |
1984 | * linear buffers. | |
1985 | */ | |
1986 | int | |
1987 | zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *salt, | |
1988 | dmu_object_type_t ot, uint8_t *iv, uint8_t *mac, uint_t datalen, | |
1989 | boolean_t byteswap, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) | |
1990 | { | |
1991 | int ret; | |
1992 | void *ptmp, *ctmp; | |
1993 | ||
1994 | if (encrypt) { | |
1995 | ptmp = abd_borrow_buf_copy(pabd, datalen); | |
1996 | ctmp = abd_borrow_buf(cabd, datalen); | |
1997 | } else { | |
1998 | ptmp = abd_borrow_buf(pabd, datalen); | |
1999 | ctmp = abd_borrow_buf_copy(cabd, datalen); | |
2000 | } | |
2001 | ||
2002 | ret = zio_do_crypt_data(encrypt, key, salt, ot, iv, mac, | |
2003 | datalen, byteswap, ptmp, ctmp, no_crypt); | |
2004 | if (ret != 0) | |
2005 | goto error; | |
2006 | ||
2007 | if (encrypt) { | |
2008 | abd_return_buf(pabd, ptmp, datalen); | |
2009 | abd_return_buf_copy(cabd, ctmp, datalen); | |
2010 | } else { | |
2011 | abd_return_buf_copy(pabd, ptmp, datalen); | |
2012 | abd_return_buf(cabd, ctmp, datalen); | |
2013 | } | |
2014 | ||
2015 | return (0); | |
2016 | ||
2017 | error: | |
2018 | if (encrypt) { | |
2019 | abd_return_buf(pabd, ptmp, datalen); | |
2020 | abd_return_buf_copy(cabd, ctmp, datalen); | |
2021 | } else { | |
2022 | abd_return_buf_copy(pabd, ptmp, datalen); | |
2023 | abd_return_buf(cabd, ctmp, datalen); | |
2024 | } | |
2025 | ||
2026 | return (ret); | |
2027 | } | |
2028 | ||
2029 | #if defined(_KERNEL) && defined(HAVE_SPL) | |
2030 | /* BEGIN CSTYLED */ | |
2031 | module_param(zfs_key_max_salt_uses, ulong, 0644); | |
2032 | MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " | |
2033 | "can be used for generating encryption keys before it is rotated"); | |
2034 | /* END CSTYLED */ | |
2035 | #endif |