]>
Commit | Line | Data |
---|---|---|
b5256303 TC |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * This file and its contents are supplied under the terms of the | |
5 | * Common Development and Distribution License ("CDDL"), version 1.0. | |
6 | * You may only use this file in accordance with the terms of version | |
7 | * 1.0 of the CDDL. | |
8 | * | |
9 | * A full copy of the text of the CDDL should have accompanied this | |
10 | * source. A copy of the CDDL is also available via the Internet at | |
11 | * http://www.illumos.org/license/CDDL. | |
12 | * | |
13 | * CDDL HEADER END | |
14 | */ | |
15 | ||
16 | /* | |
17 | * Copyright (c) 2017, Datto, Inc. All rights reserved. | |
18 | */ | |
19 | ||
20 | #include <sys/zio_crypt.h> | |
21 | #include <sys/dmu.h> | |
22 | #include <sys/dmu_objset.h> | |
23 | #include <sys/dnode.h> | |
24 | #include <sys/fs/zfs.h> | |
25 | #include <sys/zio.h> | |
26 | #include <sys/zil.h> | |
27 | #include <sys/sha2.h> | |
4807c0ba | 28 | #include <sys/hkdf.h> |
bced7e3a | 29 | #include <sys/qat.h> |
b5256303 TC |
30 | |
31 | /* | |
32 | * This file is responsible for handling all of the details of generating | |
33 | * encryption parameters and performing encryption and authentication. | |
34 | * | |
35 | * BLOCK ENCRYPTION PARAMETERS: | |
36 | * Encryption /Authentication Algorithm Suite (crypt): | |
37 | * The encryption algorithm, mode, and key length we are going to use. We | |
38 | * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit | |
39 | * keys. All authentication is currently done with SHA512-HMAC. | |
40 | * | |
41 | * Plaintext: | |
42 | * The unencrypted data that we want to encrypt. | |
43 | * | |
44 | * Initialization Vector (IV): | |
45 | * An initialization vector for the encryption algorithms. This is used to | |
46 | * "tweak" the encryption algorithms so that two blocks of the same data are | |
47 | * encrypted into different ciphertext outputs, thus obfuscating block patterns. | |
48 | * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is | |
49 | * never reused with the same encryption key. This value is stored unencrypted | |
50 | * and must simply be provided to the decryption function. We use a 96 bit IV | |
51 | * (as recommended by NIST) for all block encryption. For non-dedup blocks we | |
52 | * derive the IV randomly. The first 64 bits of the IV are stored in the second | |
53 | * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of | |
54 | * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits | |
55 | * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count | |
56 | * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of | |
57 | * level 0 blocks is the number of allocated dnodes in that block. The on-disk | |
58 | * format supports at most 2^15 slots per L0 dnode block, because the maximum | |
59 | * block size is 16MB (2^24). In either case, for level 0 blocks this number | |
60 | * will still be smaller than UINT32_MAX so it is safe to store the IV in the | |
61 | * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count | |
62 | * for the dnode code. | |
63 | * | |
64 | * Master key: | |
65 | * This is the most important secret data of an encrypted dataset. It is used | |
66 | * along with the salt to generate that actual encryption keys via HKDF. We | |
67 | * do not use the master key to directly encrypt any data because there are | |
68 | * theoretical limits on how much data can actually be safely encrypted with | |
69 | * any encryption mode. The master key is stored encrypted on disk with the | |
70 | * user's wrapping key. Its length is determined by the encryption algorithm. | |
71 | * For details on how this is stored see the block comment in dsl_crypt.c | |
72 | * | |
73 | * Salt: | |
74 | * Used as an input to the HKDF function, along with the master key. We use a | |
75 | * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt | |
76 | * can be used for encrypting many blocks, so we cache the current salt and the | |
77 | * associated derived key in zio_crypt_t so we do not need to derive it again | |
78 | * needlessly. | |
79 | * | |
80 | * Encryption Key: | |
81 | * A secret binary key, generated from an HKDF function used to encrypt and | |
82 | * decrypt data. | |
83 | * | |
8d9e7c8f | 84 | * Message Authentication Code (MAC) |
b5256303 TC |
85 | * The MAC is an output of authenticated encryption modes such as AES-GCM and |
86 | * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted | |
87 | * data on disk and return garbage to the application. Effectively, it is a | |
88 | * checksum that can not be reproduced by an attacker. We store the MAC in the | |
89 | * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated | |
90 | * regular checksum of the ciphertext which can be used for scrubbing. | |
91 | * | |
92 | * OBJECT AUTHENTICATION: | |
93 | * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because | |
94 | * they contain some info that always needs to be readable. To prevent this | |
95 | * data from being altered, we authenticate this data using SHA512-HMAC. This | |
96 | * will produce a MAC (similar to the one produced via encryption) which can | |
97 | * be used to verify the object was not modified. HMACs do not require key | |
98 | * rotation or IVs, so we can keep up to the full 3 copies of authenticated | |
99 | * data. | |
100 | * | |
101 | * ZIL ENCRYPTION: | |
102 | * ZIL blocks have their bp written to disk ahead of the associated data, so we | |
103 | * cannot store the MAC there as we normally do. For these blocks the MAC is | |
104 | * stored in the embedded checksum within the zil_chain_t header. The salt and | |
105 | * IV are generated for the block on bp allocation instead of at encryption | |
106 | * time. In addition, ZIL blocks have some pieces that must be left in plaintext | |
107 | * for claiming even though all of the sensitive user data still needs to be | |
108 | * encrypted. The function zio_crypt_init_uios_zil() handles parsing which | |
109 | * pieces of the block need to be encrypted. All data that is not encrypted is | |
110 | * authenticated using the AAD mechanisms that the supported encryption modes | |
111 | * provide for. In order to preserve the semantics of the ZIL for encrypted | |
112 | * datasets, the ZIL is not protected at the objset level as described below. | |
113 | * | |
114 | * DNODE ENCRYPTION: | |
115 | * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left | |
116 | * in plaintext for scrubbing and claiming, but the bonus buffers might contain | |
117 | * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing | |
118 | * which which pieces of the block need to be encrypted. For more details about | |
119 | * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). | |
120 | * | |
121 | * OBJECT SET AUTHENTICATION: | |
122 | * Up to this point, everything we have encrypted and authenticated has been | |
123 | * at level 0 (or -2 for the ZIL). If we did not do any further work the | |
8d9e7c8f | 124 | * on-disk format would be susceptible to attacks that deleted or rearranged |
b5256303 TC |
125 | * the order of level 0 blocks. Ideally, the cleanest solution would be to |
126 | * maintain a tree of authentication MACs going up the bp tree. However, this | |
127 | * presents a problem for raw sends. Send files do not send information about | |
128 | * indirect blocks so there would be no convenient way to transfer the MACs and | |
129 | * they cannot be recalculated on the receive side without the master key which | |
130 | * would defeat one of the purposes of raw sends in the first place. Instead, | |
131 | * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs | |
132 | * from the level below. We also include some portable fields from blk_prop such | |
133 | * as the lsize and compression algorithm to prevent the data from being | |
8d9e7c8f | 134 | * misinterpreted. |
b5256303 | 135 | * |
8d9e7c8f | 136 | * At the objset level, we maintain 2 separate 256 bit MACs in the |
b5256303 | 137 | * objset_phys_t. The first one is "portable" and is the logical root of the |
8d9e7c8f | 138 | * MAC tree maintained in the metadnode's bps. The second, is "local" and is |
b5256303 TC |
139 | * used as the root MAC for the user accounting objects, which are also not |
140 | * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload | |
141 | * of the send file. The useraccounting code ensures that the useraccounting | |
142 | * info is not present upon a receive, so the local MAC can simply be cleared | |
143 | * out at that time. For more info about objset_phys_t authentication, see | |
144 | * zio_crypt_do_objset_hmacs(). | |
145 | * | |
146 | * CONSIDERATIONS FOR DEDUP: | |
147 | * In order for dedup to work, blocks that we want to dedup with one another | |
148 | * need to use the same IV and encryption key, so that they will have the same | |
149 | * ciphertext. Normally, one should never reuse an IV with the same encryption | |
150 | * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both | |
8d9e7c8f | 151 | * blocks. In this case, however, since we are using the same plaintext as |
b5256303 TC |
152 | * well all that we end up with is a duplicate of the original ciphertext we |
153 | * already had. As a result, an attacker with read access to the raw disk will | |
154 | * be able to tell which blocks are the same but this information is given away | |
155 | * by dedup anyway. In order to get the same IVs and encryption keys for | |
8d9e7c8f TC |
156 | * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC |
157 | * here so that a reproducible checksum of the plaintext is never available to | |
b5256303 TC |
158 | * the attacker. The HMAC key is kept alongside the master key, encrypted on |
159 | * disk. The first 64 bits of the HMAC are used in place of the random salt, and | |
160 | * the next 96 bits are used as the IV. As a result of this mechanism, dedup | |
161 | * will only work within a clone family since encrypted dedup requires use of | |
162 | * the same master and HMAC keys. | |
163 | */ | |
164 | ||
165 | /* | |
166 | * After encrypting many blocks with the same key we may start to run up | |
167 | * against the theoretical limits of how much data can securely be encrypted | |
168 | * with a single key using the supported encryption modes. The most obvious | |
169 | * limitation is that our risk of generating 2 equivalent 96 bit IVs increases | |
170 | * the more IVs we generate (which both GCM and CCM modes strictly forbid). | |
171 | * This risk actually grows surprisingly quickly over time according to the | |
172 | * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have | |
173 | * generated n IVs with a cryptographically secure RNG, the approximate | |
174 | * probability p(n) of a collision is given as: | |
175 | * | |
176 | * p(n) ~= e^(-n*(n-1)/(2*(2^96))) | |
177 | * | |
178 | * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] | |
179 | * | |
180 | * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion | |
181 | * we must not write more than 398,065,730 blocks with the same encryption key. | |
182 | * Therefore, we rotate our keys after 400,000,000 blocks have been written by | |
183 | * generating a new random 64 bit salt for our HKDF encryption key generation | |
184 | * function. | |
185 | */ | |
186 | #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 | |
187 | #define ZFS_CURRENT_MAX_SALT_USES \ | |
188 | (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) | |
189 | unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; | |
190 | ||
ae76f45c TC |
191 | typedef struct blkptr_auth_buf { |
192 | uint64_t bab_prop; /* blk_prop - portable mask */ | |
193 | uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */\r | |
194 | uint64_t bab_pad; /* reserved for future use */ | |
195 | } blkptr_auth_buf_t; | |
196 | ||
b5256303 TC |
197 | zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { |
198 | {"", ZC_TYPE_NONE, 0, "inherit"}, | |
199 | {"", ZC_TYPE_NONE, 0, "on"}, | |
200 | {"", ZC_TYPE_NONE, 0, "off"}, | |
201 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, | |
202 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, | |
203 | {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, | |
204 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, | |
205 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, | |
206 | {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} | |
207 | }; | |
208 | ||
b5256303 TC |
209 | void |
210 | zio_crypt_key_destroy(zio_crypt_key_t *key) | |
211 | { | |
212 | rw_destroy(&key->zk_salt_lock); | |
213 | ||
214 | /* free crypto templates */ | |
215 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
216 | crypto_destroy_ctx_template(key->zk_hmac_tmpl); | |
217 | ||
218 | /* zero out sensitive data */ | |
219 | bzero(key, sizeof (zio_crypt_key_t)); | |
220 | } | |
221 | ||
222 | int | |
223 | zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) | |
224 | { | |
225 | int ret; | |
226 | crypto_mechanism_t mech; | |
227 | uint_t keydata_len; | |
228 | ||
229 | ASSERT(key != NULL); | |
230 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
231 | ||
232 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
233 | bzero(key, sizeof (zio_crypt_key_t)); | |
234 | ||
235 | /* fill keydata buffers and salt with random data */ | |
236 | ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); | |
237 | if (ret != 0) | |
238 | goto error; | |
239 | ||
240 | ret = random_get_bytes(key->zk_master_keydata, keydata_len); | |
241 | if (ret != 0) | |
242 | goto error; | |
243 | ||
244 | ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); | |
245 | if (ret != 0) | |
246 | goto error; | |
247 | ||
248 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
249 | if (ret != 0) | |
250 | goto error; | |
251 | ||
252 | /* derive the current key from the master key */ | |
253 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
254 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
255 | keydata_len); | |
256 | if (ret != 0) | |
257 | goto error; | |
258 | ||
259 | /* initialize keys for the ICP */ | |
260 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
261 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 262 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
263 | |
264 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
265 | key->zk_hmac_key.ck_data = &key->zk_hmac_key; | |
4807c0ba | 266 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 TC |
267 | |
268 | /* | |
269 | * Initialize the crypto templates. It's ok if this fails because | |
270 | * this is just an optimization. | |
271 | */ | |
272 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
273 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
274 | &key->zk_current_tmpl, KM_SLEEP); | |
275 | if (ret != CRYPTO_SUCCESS) | |
276 | key->zk_current_tmpl = NULL; | |
277 | ||
278 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
279 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
280 | &key->zk_hmac_tmpl, KM_SLEEP); | |
281 | if (ret != CRYPTO_SUCCESS) | |
282 | key->zk_hmac_tmpl = NULL; | |
283 | ||
284 | key->zk_crypt = crypt; | |
ae76f45c | 285 | key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; |
b5256303 TC |
286 | key->zk_salt_count = 0; |
287 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); | |
288 | ||
289 | return (0); | |
290 | ||
291 | error: | |
292 | zio_crypt_key_destroy(key); | |
293 | return (ret); | |
294 | } | |
295 | ||
296 | static int | |
297 | zio_crypt_key_change_salt(zio_crypt_key_t *key) | |
298 | { | |
299 | int ret = 0; | |
300 | uint8_t salt[ZIO_DATA_SALT_LEN]; | |
301 | crypto_mechanism_t mech; | |
302 | uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; | |
303 | ||
304 | /* generate a new salt */ | |
305 | ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); | |
306 | if (ret != 0) | |
307 | goto error; | |
308 | ||
309 | rw_enter(&key->zk_salt_lock, RW_WRITER); | |
310 | ||
311 | /* someone beat us to the salt rotation, just unlock and return */ | |
312 | if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) | |
313 | goto out_unlock; | |
314 | ||
315 | /* derive the current key from the master key and the new salt */ | |
316 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
317 | salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); | |
318 | if (ret != 0) | |
319 | goto out_unlock; | |
320 | ||
321 | /* assign the salt and reset the usage count */ | |
322 | bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); | |
323 | key->zk_salt_count = 0; | |
324 | ||
325 | /* destroy the old context template and create the new one */ | |
326 | crypto_destroy_ctx_template(key->zk_current_tmpl); | |
327 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
328 | &key->zk_current_tmpl, KM_SLEEP); | |
329 | if (ret != CRYPTO_SUCCESS) | |
330 | key->zk_current_tmpl = NULL; | |
331 | ||
332 | rw_exit(&key->zk_salt_lock); | |
333 | ||
334 | return (0); | |
335 | ||
336 | out_unlock: | |
337 | rw_exit(&key->zk_salt_lock); | |
338 | error: | |
339 | return (ret); | |
340 | } | |
341 | ||
342 | /* See comment above zfs_key_max_salt_uses definition for details */ | |
343 | int | |
344 | zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) | |
345 | { | |
346 | int ret; | |
347 | boolean_t salt_change; | |
348 | ||
349 | rw_enter(&key->zk_salt_lock, RW_READER); | |
350 | ||
351 | bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); | |
352 | salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= | |
353 | ZFS_CURRENT_MAX_SALT_USES); | |
354 | ||
355 | rw_exit(&key->zk_salt_lock); | |
356 | ||
357 | if (salt_change) { | |
358 | ret = zio_crypt_key_change_salt(key); | |
359 | if (ret != 0) | |
360 | goto error; | |
361 | } | |
362 | ||
363 | return (0); | |
364 | ||
365 | error: | |
366 | return (ret); | |
367 | } | |
368 | ||
369 | /* | |
370 | * This function handles all encryption and decryption in zfs. When | |
371 | * encrypting it expects puio to reference the plaintext and cuio to | |
e1cfd73f | 372 | * reference the ciphertext. cuio must have enough space for the |
b5256303 TC |
373 | * ciphertext + room for a MAC. datalen should be the length of the |
374 | * plaintext / ciphertext alone. | |
375 | */ | |
376 | static int | |
10fa2545 | 377 | zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, |
b5256303 TC |
378 | crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, |
379 | uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len) | |
380 | { | |
381 | int ret; | |
382 | crypto_data_t plaindata, cipherdata; | |
383 | CK_AES_CCM_PARAMS ccmp; | |
384 | CK_AES_GCM_PARAMS gcmp; | |
385 | crypto_mechanism_t mech; | |
386 | zio_crypt_info_t crypt_info; | |
387 | uint_t plain_full_len, maclen; | |
388 | ||
389 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
390 | ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW); | |
391 | ||
392 | /* lookup the encryption info */ | |
393 | crypt_info = zio_crypt_table[crypt]; | |
394 | ||
395 | /* the mac will always be the last iovec_t in the cipher uio */ | |
396 | maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; | |
397 | ||
398 | ASSERT(maclen <= ZIO_DATA_MAC_LEN); | |
399 | ||
400 | /* setup encryption mechanism (same as crypt) */ | |
401 | mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); | |
402 | ||
403 | /* | |
404 | * Strangely, the ICP requires that plain_full_len must include | |
405 | * the MAC length when decrypting, even though the UIO does not | |
406 | * need to have the extra space allocated. | |
407 | */ | |
408 | if (encrypt) { | |
409 | plain_full_len = datalen; | |
410 | } else { | |
411 | plain_full_len = datalen + maclen; | |
412 | } | |
413 | ||
414 | /* | |
415 | * setup encryption params (currently only AES CCM and AES GCM | |
416 | * are supported) | |
417 | */ | |
418 | if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { | |
419 | ccmp.ulNonceSize = ZIO_DATA_IV_LEN; | |
420 | ccmp.ulAuthDataSize = auth_len; | |
421 | ccmp.authData = authbuf; | |
422 | ccmp.ulMACSize = maclen; | |
423 | ccmp.nonce = ivbuf; | |
424 | ccmp.ulDataSize = plain_full_len; | |
425 | ||
426 | mech.cm_param = (char *)(&ccmp); | |
427 | mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); | |
428 | } else { | |
429 | gcmp.ulIvLen = ZIO_DATA_IV_LEN; | |
4807c0ba | 430 | gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); |
b5256303 TC |
431 | gcmp.ulAADLen = auth_len; |
432 | gcmp.pAAD = authbuf; | |
4807c0ba | 433 | gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); |
b5256303 TC |
434 | gcmp.pIv = ivbuf; |
435 | ||
436 | mech.cm_param = (char *)(&gcmp); | |
437 | mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); | |
438 | } | |
439 | ||
440 | /* populate the cipher and plain data structs. */ | |
441 | plaindata.cd_format = CRYPTO_DATA_UIO; | |
442 | plaindata.cd_offset = 0; | |
443 | plaindata.cd_uio = puio; | |
444 | plaindata.cd_miscdata = NULL; | |
445 | plaindata.cd_length = plain_full_len; | |
446 | ||
447 | cipherdata.cd_format = CRYPTO_DATA_UIO; | |
448 | cipherdata.cd_offset = 0; | |
449 | cipherdata.cd_uio = cuio; | |
450 | cipherdata.cd_miscdata = NULL; | |
451 | cipherdata.cd_length = datalen + maclen; | |
452 | ||
453 | /* perform the actual encryption */ | |
454 | if (encrypt) { | |
455 | ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata, | |
456 | NULL); | |
457 | if (ret != CRYPTO_SUCCESS) { | |
458 | ret = SET_ERROR(EIO); | |
459 | goto error; | |
460 | } | |
461 | } else { | |
462 | ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata, | |
463 | NULL); | |
464 | if (ret != CRYPTO_SUCCESS) { | |
465 | ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); | |
466 | ret = SET_ERROR(ECKSUM); | |
467 | goto error; | |
468 | } | |
469 | } | |
470 | ||
471 | return (0); | |
472 | ||
473 | error: | |
474 | return (ret); | |
475 | } | |
476 | ||
477 | int | |
10fa2545 BB |
478 | zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, |
479 | uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) | |
b5256303 TC |
480 | { |
481 | int ret; | |
482 | uio_t puio, cuio; | |
ae76f45c | 483 | uint64_t aad[3]; |
b5256303 TC |
484 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
485 | uint64_t crypt = key->zk_crypt; | |
ae76f45c | 486 | uint_t enc_len, keydata_len, aad_len; |
b5256303 TC |
487 | |
488 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
489 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
490 | ||
491 | keydata_len = zio_crypt_table[crypt].ci_keylen; | |
492 | ||
493 | /* generate iv for wrapping the master and hmac key */ | |
494 | ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); | |
495 | if (ret != 0) | |
496 | goto error; | |
497 | ||
498 | /* initialize uio_ts */ | |
499 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
500 | plain_iovecs[0].iov_len = keydata_len; | |
501 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
502 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
503 | ||
504 | cipher_iovecs[0].iov_base = keydata_out; | |
505 | cipher_iovecs[0].iov_len = keydata_len; | |
506 | cipher_iovecs[1].iov_base = hmac_keydata_out; | |
507 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
508 | cipher_iovecs[2].iov_base = mac; | |
509 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
510 | ||
ae76f45c TC |
511 | /* |
512 | * Although we don't support writing to the old format, we do | |
513 | * support rewrapping the key so that the user can move and | |
514 | * quarantine datasets on the old format. | |
515 | */ | |
516 | if (key->zk_version == 0) { | |
517 | aad_len = sizeof (uint64_t); | |
518 | aad[0] = LE_64(key->zk_guid); | |
519 | } else { | |
520 | ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
521 | aad_len = sizeof (uint64_t) * 3; | |
522 | aad[0] = LE_64(key->zk_guid); | |
523 | aad[1] = LE_64(crypt); | |
524 | aad[2] = LE_64(key->zk_version); | |
525 | } | |
526 | ||
b5256303 TC |
527 | enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; |
528 | puio.uio_iov = plain_iovecs; | |
529 | puio.uio_iovcnt = 2; | |
530 | puio.uio_segflg = UIO_SYSSPACE; | |
531 | cuio.uio_iov = cipher_iovecs; | |
532 | cuio.uio_iovcnt = 3; | |
533 | cuio.uio_segflg = UIO_SYSSPACE; | |
534 | ||
535 | /* encrypt the keys and store the resulting ciphertext and mac */ | |
10fa2545 | 536 | ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, |
ae76f45c | 537 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
538 | if (ret != 0) |
539 | goto error; | |
540 | ||
541 | return (0); | |
542 | ||
543 | error: | |
544 | return (ret); | |
545 | } | |
546 | ||
b88ca2ac | 547 | int |
10fa2545 BB |
548 | zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, |
549 | uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, | |
550 | uint8_t *mac, zio_crypt_key_t *key) | |
b88ca2ac | 551 | { |
10fa2545 | 552 | crypto_mechanism_t mech; |
b5256303 | 553 | uio_t puio, cuio; |
ae76f45c | 554 | uint64_t aad[3]; |
b5256303 | 555 | iovec_t plain_iovecs[2], cipher_iovecs[3]; |
ae76f45c | 556 | uint_t enc_len, keydata_len, aad_len; |
b88ca2ac | 557 | int ret; |
b5256303 TC |
558 | |
559 | ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); | |
560 | ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); | |
561 | ||
48ed0f9d JL |
562 | rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); |
563 | ||
b5256303 TC |
564 | keydata_len = zio_crypt_table[crypt].ci_keylen; |
565 | ||
566 | /* initialize uio_ts */ | |
567 | plain_iovecs[0].iov_base = key->zk_master_keydata; | |
568 | plain_iovecs[0].iov_len = keydata_len; | |
569 | plain_iovecs[1].iov_base = key->zk_hmac_keydata; | |
570 | plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
571 | ||
572 | cipher_iovecs[0].iov_base = keydata; | |
573 | cipher_iovecs[0].iov_len = keydata_len; | |
574 | cipher_iovecs[1].iov_base = hmac_keydata; | |
575 | cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; | |
576 | cipher_iovecs[2].iov_base = mac; | |
577 | cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; | |
578 | ||
ae76f45c TC |
579 | if (version == 0) { |
580 | aad_len = sizeof (uint64_t); | |
581 | aad[0] = LE_64(guid); | |
582 | } else { | |
583 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
584 | aad_len = sizeof (uint64_t) * 3; | |
585 | aad[0] = LE_64(guid); | |
586 | aad[1] = LE_64(crypt); | |
587 | aad[2] = LE_64(version); | |
588 | } | |
589 | ||
b5256303 TC |
590 | enc_len = keydata_len + SHA512_HMAC_KEYLEN; |
591 | puio.uio_iov = plain_iovecs; | |
592 | puio.uio_segflg = UIO_SYSSPACE; | |
593 | puio.uio_iovcnt = 2; | |
594 | cuio.uio_iov = cipher_iovecs; | |
595 | cuio.uio_iovcnt = 3; | |
596 | cuio.uio_segflg = UIO_SYSSPACE; | |
597 | ||
598 | /* decrypt the keys and store the result in the output buffers */ | |
10fa2545 | 599 | ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, |
ae76f45c | 600 | &puio, &cuio, (uint8_t *)aad, aad_len); |
b5256303 TC |
601 | if (ret != 0) |
602 | goto error; | |
603 | ||
604 | /* generate a fresh salt */ | |
605 | ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); | |
606 | if (ret != 0) | |
607 | goto error; | |
608 | ||
609 | /* derive the current key from the master key */ | |
610 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
611 | key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, | |
612 | keydata_len); | |
613 | if (ret != 0) | |
614 | goto error; | |
615 | ||
616 | /* initialize keys for ICP */ | |
617 | key->zk_current_key.ck_format = CRYPTO_KEY_RAW; | |
618 | key->zk_current_key.ck_data = key->zk_current_keydata; | |
4807c0ba | 619 | key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
620 | |
621 | key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; | |
622 | key->zk_hmac_key.ck_data = key->zk_hmac_keydata; | |
4807c0ba | 623 | key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); |
b5256303 | 624 | |
10fa2545 BB |
625 | /* |
626 | * Initialize the crypto templates. It's ok if this fails because | |
627 | * this is just an optimization. | |
628 | */ | |
629 | mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); | |
630 | ret = crypto_create_ctx_template(&mech, &key->zk_current_key, | |
631 | &key->zk_current_tmpl, KM_SLEEP); | |
632 | if (ret != CRYPTO_SUCCESS) | |
633 | key->zk_current_tmpl = NULL; | |
634 | ||
635 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
636 | ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, | |
637 | &key->zk_hmac_tmpl, KM_SLEEP); | |
638 | if (ret != CRYPTO_SUCCESS) | |
639 | key->zk_hmac_tmpl = NULL; | |
640 | ||
b5256303 | 641 | key->zk_crypt = crypt; |
ae76f45c | 642 | key->zk_version = version; |
b5256303 TC |
643 | key->zk_guid = guid; |
644 | key->zk_salt_count = 0; | |
b5256303 TC |
645 | |
646 | return (0); | |
647 | ||
648 | error: | |
649 | zio_crypt_key_destroy(key); | |
650 | return (ret); | |
651 | } | |
652 | ||
653 | int | |
654 | zio_crypt_generate_iv(uint8_t *ivbuf) | |
655 | { | |
656 | int ret; | |
657 | ||
658 | /* randomly generate the IV */ | |
659 | ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); | |
660 | if (ret != 0) | |
661 | goto error; | |
662 | ||
663 | return (0); | |
664 | ||
665 | error: | |
666 | bzero(ivbuf, ZIO_DATA_IV_LEN); | |
667 | return (ret); | |
668 | } | |
669 | ||
670 | int | |
671 | zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, | |
4807c0ba | 672 | uint8_t *digestbuf, uint_t digestlen) |
b5256303 TC |
673 | { |
674 | int ret; | |
675 | crypto_mechanism_t mech; | |
676 | crypto_data_t in_data, digest_data; | |
4807c0ba TC |
677 | uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; |
678 | ||
679 | ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
680 | |
681 | /* initialize sha512-hmac mechanism and crypto data */ | |
682 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
683 | mech.cm_param = NULL; | |
684 | mech.cm_param_len = 0; | |
685 | ||
686 | /* initialize the crypto data */ | |
687 | in_data.cd_format = CRYPTO_DATA_RAW; | |
688 | in_data.cd_offset = 0; | |
689 | in_data.cd_length = datalen; | |
690 | in_data.cd_raw.iov_base = (char *)data; | |
691 | in_data.cd_raw.iov_len = in_data.cd_length; | |
692 | ||
693 | digest_data.cd_format = CRYPTO_DATA_RAW; | |
694 | digest_data.cd_offset = 0; | |
4807c0ba | 695 | digest_data.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
696 | digest_data.cd_raw.iov_base = (char *)raw_digestbuf; |
697 | digest_data.cd_raw.iov_len = digest_data.cd_length; | |
698 | ||
699 | /* generate the hmac */ | |
700 | ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, | |
701 | &digest_data, NULL); | |
702 | if (ret != CRYPTO_SUCCESS) { | |
703 | ret = SET_ERROR(EIO); | |
704 | goto error; | |
705 | } | |
706 | ||
4807c0ba | 707 | bcopy(raw_digestbuf, digestbuf, digestlen); |
b5256303 TC |
708 | |
709 | return (0); | |
710 | ||
711 | error: | |
4807c0ba | 712 | bzero(digestbuf, digestlen); |
b5256303 TC |
713 | return (ret); |
714 | } | |
715 | ||
716 | int | |
717 | zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, | |
718 | uint_t datalen, uint8_t *ivbuf, uint8_t *salt) | |
719 | { | |
720 | int ret; | |
4807c0ba | 721 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 | 722 | |
4807c0ba TC |
723 | ret = zio_crypt_do_hmac(key, data, datalen, |
724 | digestbuf, SHA512_DIGEST_LENGTH); | |
b5256303 TC |
725 | if (ret != 0) |
726 | return (ret); | |
727 | ||
728 | bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); | |
729 | bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); | |
730 | ||
731 | return (0); | |
732 | } | |
733 | ||
734 | /* | |
735 | * The following functions are used to encode and decode encryption parameters | |
736 | * into blkptr_t and zil_header_t. The ICP wants to use these parameters as | |
737 | * byte strings, which normally means that these strings would not need to deal | |
738 | * with byteswapping at all. However, both blkptr_t and zil_header_t may be | |
739 | * byteswapped by lower layers and so we must "undo" that byteswap here upon | |
ae76f45c TC |
740 | * decoding and encoding in a non-native byteorder. These functions require |
741 | * that the byteorder bit is correct before being called. | |
b5256303 TC |
742 | */ |
743 | void | |
744 | zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
745 | { | |
ae76f45c | 746 | uint64_t val64; |
b5256303 TC |
747 | uint32_t val32; |
748 | ||
749 | ASSERT(BP_IS_ENCRYPTED(bp)); | |
750 | ||
ae76f45c TC |
751 | if (!BP_SHOULD_BYTESWAP(bp)) { |
752 | bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); | |
753 | bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); | |
754 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
755 | BP_SET_IV2(bp, val32); | |
756 | } else { | |
757 | bcopy(salt, &val64, sizeof (uint64_t)); | |
758 | bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); | |
759 | ||
760 | bcopy(iv, &val64, sizeof (uint64_t)); | |
761 | bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); | |
762 | ||
763 | bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); | |
764 | BP_SET_IV2(bp, BSWAP_32(val32)); | |
765 | } | |
b5256303 TC |
766 | } |
767 | ||
768 | void | |
769 | zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) | |
770 | { | |
771 | uint64_t val64; | |
772 | uint32_t val32; | |
773 | ||
774 | ASSERT(BP_IS_PROTECTED(bp)); | |
775 | ||
776 | /* for convenience, so callers don't need to check */ | |
777 | if (BP_IS_AUTHENTICATED(bp)) { | |
778 | bzero(salt, ZIO_DATA_SALT_LEN); | |
779 | bzero(iv, ZIO_DATA_IV_LEN); | |
780 | return; | |
781 | } | |
782 | ||
783 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
784 | bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); | |
785 | bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); | |
786 | ||
787 | val32 = (uint32_t)BP_GET_IV2(bp); | |
788 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
789 | } else { | |
790 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); | |
791 | bcopy(&val64, salt, sizeof (uint64_t)); | |
792 | ||
793 | val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); | |
794 | bcopy(&val64, iv, sizeof (uint64_t)); | |
795 | ||
796 | val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); | |
797 | bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); | |
798 | } | |
799 | } | |
800 | ||
801 | void | |
802 | zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) | |
803 | { | |
ae76f45c TC |
804 | uint64_t val64; |
805 | ||
b5256303 TC |
806 | ASSERT(BP_USES_CRYPT(bp)); |
807 | ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); | |
808 | ||
ae76f45c TC |
809 | if (!BP_SHOULD_BYTESWAP(bp)) { |
810 | bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); | |
811 | bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], | |
812 | sizeof (uint64_t)); | |
813 | } else { | |
814 | bcopy(mac, &val64, sizeof (uint64_t)); | |
815 | bp->blk_cksum.zc_word[2] = BSWAP_64(val64); | |
816 | ||
817 | bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); | |
818 | bp->blk_cksum.zc_word[3] = BSWAP_64(val64); | |
819 | } | |
b5256303 TC |
820 | } |
821 | ||
822 | void | |
823 | zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) | |
824 | { | |
825 | uint64_t val64; | |
826 | ||
827 | ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); | |
828 | ||
829 | /* for convenience, so callers don't need to check */ | |
830 | if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { | |
831 | bzero(mac, ZIO_DATA_MAC_LEN); | |
832 | return; | |
833 | } | |
834 | ||
835 | if (!BP_SHOULD_BYTESWAP(bp)) { | |
836 | bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
837 | bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), | |
838 | sizeof (uint64_t)); | |
839 | } else { | |
840 | val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); | |
841 | bcopy(&val64, mac, sizeof (uint64_t)); | |
842 | ||
843 | val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); | |
844 | bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); | |
845 | } | |
846 | } | |
847 | ||
848 | void | |
849 | zio_crypt_encode_mac_zil(void *data, uint8_t *mac) | |
850 | { | |
851 | zil_chain_t *zilc = data; | |
852 | ||
853 | bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); | |
854 | bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], | |
855 | sizeof (uint64_t)); | |
856 | } | |
857 | ||
858 | void | |
859 | zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) | |
860 | { | |
861 | /* | |
862 | * The ZIL MAC is embedded in the block it protects, which will | |
863 | * not have been byteswapped by the time this function has been called. | |
864 | * As a result, we don't need to worry about byteswapping the MAC. | |
865 | */ | |
866 | const zil_chain_t *zilc = data; | |
867 | ||
868 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); | |
869 | bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), | |
870 | sizeof (uint64_t)); | |
871 | } | |
872 | ||
873 | /* | |
874 | * This routine takes a block of dnodes (src_abd) and copies only the bonus | |
875 | * buffers to the same offsets in the dst buffer. datalen should be the size | |
876 | * of both the src_abd and the dst buffer (not just the length of the bonus | |
877 | * buffers). | |
878 | */ | |
879 | void | |
880 | zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) | |
881 | { | |
882 | uint_t i, max_dnp = datalen >> DNODE_SHIFT; | |
883 | uint8_t *src; | |
884 | dnode_phys_t *dnp, *sdnp, *ddnp; | |
885 | ||
886 | src = abd_borrow_buf_copy(src_abd, datalen); | |
887 | ||
888 | sdnp = (dnode_phys_t *)src; | |
889 | ddnp = (dnode_phys_t *)dst; | |
890 | ||
891 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
892 | dnp = &sdnp[i]; | |
893 | if (dnp->dn_type != DMU_OT_NONE && | |
894 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
895 | dnp->dn_bonuslen != 0) { | |
896 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), | |
897 | DN_MAX_BONUS_LEN(dnp)); | |
898 | } | |
899 | } | |
900 | ||
901 | abd_return_buf(src_abd, src, datalen); | |
902 | } | |
903 | ||
ae76f45c TC |
904 | /* |
905 | * This function decides what fields from blk_prop are included in | |
906 | * the on-disk various MAC algorithms. | |
907 | */ | |
b5256303 | 908 | static void |
ae76f45c | 909 | zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) |
b5256303 | 910 | { |
ae76f45c TC |
911 | /* |
912 | * Version 0 did not properly zero out all non-portable fields | |
913 | * as it should have done. We maintain this code so that we can | |
914 | * do read-only imports of pools on this version. | |
915 | */ | |
916 | if (version == 0) { | |
917 | BP_SET_DEDUP(bp, 0); | |
918 | BP_SET_CHECKSUM(bp, 0); | |
919 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
920 | return; | |
921 | } | |
922 | ||
923 | ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); | |
924 | ||
925 | /* | |
926 | * The hole_birth feature might set these fields even if this bp | |
927 | * is a hole. We zero them out here to guarantee that raw sends | |
928 | * will function with or without the feature. | |
929 | */ | |
930 | if (BP_IS_HOLE(bp)) { | |
931 | bp->blk_prop = 0ULL; | |
932 | return; | |
933 | } | |
b5256303 TC |
934 | |
935 | /* | |
ae76f45c | 936 | * At L0 we want to verify these fields to ensure that data blocks |
e1cfd73f | 937 | * can not be reinterpreted. For instance, we do not want an attacker |
ae76f45c TC |
938 | * to trick us into returning raw lz4 compressed data to the user |
939 | * by modifying the compression bits. At higher levels, we cannot | |
940 | * enforce this policy since raw sends do not convey any information | |
941 | * about indirect blocks, so these values might be different on the | |
942 | * receive side. Fortunately, this does not open any new attack | |
943 | * vectors, since any alterations that can be made to a higher level | |
944 | * bp must still verify the correct order of the layer below it. | |
b5256303 | 945 | */ |
ae76f45c TC |
946 | if (BP_GET_LEVEL(bp) != 0) { |
947 | BP_SET_BYTEORDER(bp, 0); | |
948 | BP_SET_COMPRESS(bp, 0); | |
949 | ||
950 | /* | |
951 | * psize cannot be set to zero or it will trigger | |
952 | * asserts, but the value doesn't really matter as | |
953 | * long as it is constant. | |
954 | */ | |
955 | BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); | |
956 | } | |
957 | ||
958 | BP_SET_DEDUP(bp, 0); | |
959 | BP_SET_CHECKSUM(bp, 0); | |
b5256303 TC |
960 | } |
961 | ||
ae76f45c TC |
962 | static void |
963 | zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, | |
964 | blkptr_auth_buf_t *bab, uint_t *bab_len) | |
b5256303 | 965 | { |
b5256303 | 966 | blkptr_t tmpbp = *bp; |
b5256303 TC |
967 | |
968 | if (should_bswap) | |
969 | byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); | |
970 | ||
971 | ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); | |
972 | ASSERT0(BP_IS_EMBEDDED(&tmpbp)); | |
b5256303 | 973 | |
ae76f45c | 974 | zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); |
b5256303 | 975 | |
ae76f45c TC |
976 | /* |
977 | * We always MAC blk_prop in LE to ensure portability. This | |
978 | * must be done after decoding the mac, since the endianness | |
979 | * will get zero'd out here. | |
980 | */ | |
981 | zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); | |
982 | bab->bab_prop = LE_64(tmpbp.blk_prop); | |
983 | bab->bab_pad = 0ULL; | |
984 | ||
985 | /* version 0 did not include the padding */ | |
986 | *bab_len = sizeof (blkptr_auth_buf_t); | |
987 | if (version == 0) | |
988 | *bab_len -= sizeof (uint64_t); | |
989 | } | |
b5256303 | 990 | |
ae76f45c TC |
991 | static int |
992 | zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, | |
993 | boolean_t should_bswap, blkptr_t *bp) | |
994 | { | |
995 | int ret; | |
996 | uint_t bab_len; | |
997 | blkptr_auth_buf_t bab; | |
998 | crypto_data_t cd; | |
b5256303 | 999 | |
ae76f45c TC |
1000 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1001 | cd.cd_format = CRYPTO_DATA_RAW; | |
1002 | cd.cd_offset = 0; | |
1003 | cd.cd_length = bab_len; | |
1004 | cd.cd_raw.iov_base = (char *)&bab; | |
b5256303 TC |
1005 | cd.cd_raw.iov_len = cd.cd_length; |
1006 | ||
1007 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1008 | if (ret != CRYPTO_SUCCESS) { | |
1009 | ret = SET_ERROR(EIO); | |
1010 | goto error; | |
1011 | } | |
1012 | ||
1013 | return (0); | |
1014 | ||
1015 | error: | |
1016 | return (ret); | |
1017 | } | |
1018 | ||
1019 | static void | |
ae76f45c TC |
1020 | zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, |
1021 | boolean_t should_bswap, blkptr_t *bp) | |
b5256303 | 1022 | { |
ae76f45c TC |
1023 | uint_t bab_len; |
1024 | blkptr_auth_buf_t bab; | |
b5256303 | 1025 | |
ae76f45c TC |
1026 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1027 | SHA2Update(ctx, &bab, bab_len); | |
b5256303 TC |
1028 | } |
1029 | ||
1030 | static void | |
ae76f45c | 1031 | zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, |
b5256303 TC |
1032 | boolean_t should_bswap, blkptr_t *bp) |
1033 | { | |
ae76f45c TC |
1034 | uint_t bab_len; |
1035 | blkptr_auth_buf_t bab; | |
b5256303 | 1036 | |
ae76f45c TC |
1037 | zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); |
1038 | bcopy(&bab, *aadp, bab_len); | |
1039 | *aadp += bab_len; | |
1040 | *aad_len += bab_len; | |
b5256303 TC |
1041 | } |
1042 | ||
1043 | static int | |
ae76f45c TC |
1044 | zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, |
1045 | boolean_t should_bswap, dnode_phys_t *dnp) | |
b5256303 TC |
1046 | { |
1047 | int ret, i; | |
1048 | dnode_phys_t *adnp; | |
1049 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
1050 | crypto_data_t cd; | |
1051 | uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; | |
1052 | ||
1053 | cd.cd_format = CRYPTO_DATA_RAW; | |
1054 | cd.cd_offset = 0; | |
1055 | ||
1056 | /* authenticate the core dnode (masking out non-portable bits) */ | |
1057 | bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); | |
1058 | adnp = (dnode_phys_t *)tmp_dncore; | |
1059 | if (le_bswap) { | |
1060 | adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); | |
1061 | adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); | |
1062 | adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); | |
1063 | adnp->dn_used = BSWAP_64(adnp->dn_used); | |
1064 | } | |
1065 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1066 | adnp->dn_used = 0; | |
1067 | ||
1068 | cd.cd_length = sizeof (tmp_dncore); | |
1069 | cd.cd_raw.iov_base = (char *)adnp; | |
1070 | cd.cd_raw.iov_len = cd.cd_length; | |
1071 | ||
1072 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1073 | if (ret != CRYPTO_SUCCESS) { | |
1074 | ret = SET_ERROR(EIO); | |
1075 | goto error; | |
1076 | } | |
1077 | ||
1078 | for (i = 0; i < dnp->dn_nblkptr; i++) { | |
ae76f45c | 1079 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1080 | should_bswap, &dnp->dn_blkptr[i]); |
1081 | if (ret != 0) | |
1082 | goto error; | |
1083 | } | |
1084 | ||
1085 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
ae76f45c | 1086 | ret = zio_crypt_bp_do_hmac_updates(ctx, version, |
b5256303 TC |
1087 | should_bswap, DN_SPILL_BLKPTR(dnp)); |
1088 | if (ret != 0) | |
1089 | goto error; | |
1090 | } | |
1091 | ||
1092 | return (0); | |
1093 | ||
1094 | error: | |
1095 | return (ret); | |
1096 | } | |
1097 | ||
1098 | /* | |
1099 | * objset_phys_t blocks introduce a number of exceptions to the normal | |
7b30ee6b | 1100 | * authentication process. objset_phys_t's contain 2 separate HMACS for |
b5256303 | 1101 | * protecting the integrity of their data. The portable_mac protects the |
7b30ee6b | 1102 | * metadnode. This MAC can be sent with a raw send and protects against |
b5256303 TC |
1103 | * reordering of data within the metadnode. The local_mac protects the user |
1104 | * accounting objects which are not sent from one system to another. | |
1105 | * | |
1106 | * In addition, objset blocks are the only blocks that can be modified and | |
1107 | * written to disk without the key loaded under certain circumstances. During | |
1108 | * zil_claim() we need to be able to update the zil_header_t to complete | |
1109 | * claiming log blocks and during raw receives we need to write out the | |
1110 | * portable_mac from the send file. Both of these actions are possible | |
1111 | * because these fields are not protected by either MAC so neither one will | |
1112 | * need to modify the MACs without the key. However, when the modified blocks | |
1113 | * are written out they will be byteswapped into the host machine's native | |
1114 | * endianness which will modify fields protected by the MAC. As a result, MAC | |
1115 | * calculation for objset blocks works slightly differently from other block | |
1116 | * types. Where other block types MAC the data in whatever endianness is | |
1117 | * written to disk, objset blocks always MAC little endian version of their | |
1118 | * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() | |
1119 | * and le_bswap indicates whether a byteswap is needed to get this block | |
1120 | * into little endian format. | |
1121 | */ | |
1122 | int | |
1123 | zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, | |
1124 | boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) | |
1125 | { | |
1126 | int ret; | |
1127 | crypto_mechanism_t mech; | |
1128 | crypto_context_t ctx; | |
1129 | crypto_data_t cd; | |
1130 | objset_phys_t *osp = data; | |
1131 | uint64_t intval; | |
1132 | boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); | |
4807c0ba TC |
1133 | uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; |
1134 | uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; | |
b5256303 TC |
1135 | |
1136 | /* initialize HMAC mechanism */ | |
1137 | mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); | |
1138 | mech.cm_param = NULL; | |
1139 | mech.cm_param_len = 0; | |
1140 | ||
1141 | cd.cd_format = CRYPTO_DATA_RAW; | |
1142 | cd.cd_offset = 0; | |
1143 | ||
1144 | /* calculate the portable MAC from the portable fields and metadnode */ | |
1145 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1146 | if (ret != CRYPTO_SUCCESS) { | |
1147 | ret = SET_ERROR(EIO); | |
1148 | goto error; | |
1149 | } | |
1150 | ||
1151 | /* add in the os_type */ | |
1152 | intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); | |
1153 | cd.cd_length = sizeof (uint64_t); | |
1154 | cd.cd_raw.iov_base = (char *)&intval; | |
1155 | cd.cd_raw.iov_len = cd.cd_length; | |
1156 | ||
1157 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1158 | if (ret != CRYPTO_SUCCESS) { | |
1159 | ret = SET_ERROR(EIO); | |
1160 | goto error; | |
1161 | } | |
1162 | ||
1163 | /* add in the portable os_flags */ | |
1164 | intval = osp->os_flags; | |
1165 | if (should_bswap) | |
1166 | intval = BSWAP_64(intval); | |
1167 | intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1168 | if (!ZFS_HOST_BYTEORDER) | |
1169 | intval = BSWAP_64(intval); | |
1170 | ||
1171 | cd.cd_length = sizeof (uint64_t); | |
1172 | cd.cd_raw.iov_base = (char *)&intval; | |
1173 | cd.cd_raw.iov_len = cd.cd_length; | |
1174 | ||
1175 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1176 | if (ret != CRYPTO_SUCCESS) { | |
1177 | ret = SET_ERROR(EIO); | |
1178 | goto error; | |
1179 | } | |
1180 | ||
1181 | /* add in fields from the metadnode */ | |
ae76f45c TC |
1182 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, |
1183 | should_bswap, &osp->os_meta_dnode); | |
b5256303 TC |
1184 | if (ret) |
1185 | goto error; | |
1186 | ||
1187 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1188 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1189 | cd.cd_raw.iov_base = (char *)raw_portable_mac; |
1190 | cd.cd_raw.iov_len = cd.cd_length; | |
1191 | ||
1192 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1193 | if (ret != CRYPTO_SUCCESS) { | |
1194 | ret = SET_ERROR(EIO); | |
1195 | goto error; | |
1196 | } | |
1197 | ||
1198 | bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); | |
1199 | ||
1200 | /* | |
f8478fc2 TC |
1201 | * The local MAC protects the user, group and project accounting. |
1202 | * If these objects are not present, the local MAC is zeroed out. | |
b5256303 | 1203 | */ |
f8478fc2 | 1204 | if ((datalen >= OBJSET_PHYS_SIZE_V3 && |
7b30ee6b TC |
1205 | osp->os_userused_dnode.dn_type == DMU_OT_NONE && |
1206 | osp->os_groupused_dnode.dn_type == DMU_OT_NONE && | |
f8478fc2 TC |
1207 | osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || |
1208 | (datalen >= OBJSET_PHYS_SIZE_V2 && | |
1209 | osp->os_userused_dnode.dn_type == DMU_OT_NONE && | |
1210 | osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || | |
1211 | (datalen <= OBJSET_PHYS_SIZE_V1)) { | |
b5256303 TC |
1212 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); |
1213 | return (0); | |
1214 | } | |
1215 | ||
1216 | /* calculate the local MAC from the userused and groupused dnodes */ | |
1217 | ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); | |
1218 | if (ret != CRYPTO_SUCCESS) { | |
1219 | ret = SET_ERROR(EIO); | |
1220 | goto error; | |
1221 | } | |
1222 | ||
1223 | /* add in the non-portable os_flags */ | |
1224 | intval = osp->os_flags; | |
1225 | if (should_bswap) | |
1226 | intval = BSWAP_64(intval); | |
1227 | intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; | |
1228 | if (!ZFS_HOST_BYTEORDER) | |
1229 | intval = BSWAP_64(intval); | |
1230 | ||
1231 | cd.cd_length = sizeof (uint64_t); | |
1232 | cd.cd_raw.iov_base = (char *)&intval; | |
1233 | cd.cd_raw.iov_len = cd.cd_length; | |
1234 | ||
1235 | ret = crypto_mac_update(ctx, &cd, NULL); | |
1236 | if (ret != CRYPTO_SUCCESS) { | |
1237 | ret = SET_ERROR(EIO); | |
1238 | goto error; | |
1239 | } | |
1240 | ||
1241 | /* add in fields from the user accounting dnodes */ | |
7b30ee6b TC |
1242 | if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { |
1243 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1244 | should_bswap, &osp->os_userused_dnode); | |
1245 | if (ret) | |
1246 | goto error; | |
1247 | } | |
b5256303 | 1248 | |
7b30ee6b TC |
1249 | if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { |
1250 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1251 | should_bswap, &osp->os_groupused_dnode); | |
1252 | if (ret) | |
1253 | goto error; | |
1254 | } | |
1255 | ||
1256 | if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && | |
1257 | datalen >= OBJSET_PHYS_SIZE_V3) { | |
1258 | ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, | |
1259 | should_bswap, &osp->os_projectused_dnode); | |
1260 | if (ret) | |
1261 | goto error; | |
1262 | } | |
b5256303 TC |
1263 | |
1264 | /* store the final digest in a temporary buffer and copy what we need */ | |
4807c0ba | 1265 | cd.cd_length = SHA512_DIGEST_LENGTH; |
b5256303 TC |
1266 | cd.cd_raw.iov_base = (char *)raw_local_mac; |
1267 | cd.cd_raw.iov_len = cd.cd_length; | |
1268 | ||
1269 | ret = crypto_mac_final(ctx, &cd, NULL); | |
1270 | if (ret != CRYPTO_SUCCESS) { | |
1271 | ret = SET_ERROR(EIO); | |
1272 | goto error; | |
1273 | } | |
1274 | ||
1275 | bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); | |
1276 | ||
1277 | return (0); | |
1278 | ||
1279 | error: | |
1280 | bzero(portable_mac, ZIO_OBJSET_MAC_LEN); | |
1281 | bzero(local_mac, ZIO_OBJSET_MAC_LEN); | |
1282 | return (ret); | |
1283 | } | |
1284 | ||
1285 | static void | |
1286 | zio_crypt_destroy_uio(uio_t *uio) | |
1287 | { | |
1288 | if (uio->uio_iov) | |
1289 | kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); | |
1290 | } | |
1291 | ||
1292 | /* | |
1293 | * This function parses an uncompressed indirect block and returns a checksum | |
1294 | * of all the portable fields from all of the contained bps. The portable | |
1295 | * fields are the MAC and all of the fields from blk_prop except for the dedup, | |
1296 | * checksum, and psize bits. For an explanation of the purpose of this, see | |
1297 | * the comment block on object set authentication. | |
1298 | */ | |
ae76f45c TC |
1299 | static int |
1300 | zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, | |
1301 | uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) | |
b5256303 TC |
1302 | { |
1303 | blkptr_t *bp; | |
1304 | int i, epb = datalen >> SPA_BLKPTRSHIFT; | |
1305 | SHA2_CTX ctx; | |
4807c0ba | 1306 | uint8_t digestbuf[SHA512_DIGEST_LENGTH]; |
b5256303 TC |
1307 | |
1308 | /* checksum all of the MACs from the layer below */ | |
1309 | SHA2Init(SHA512, &ctx); | |
1310 | for (i = 0, bp = buf; i < epb; i++, bp++) { | |
ae76f45c TC |
1311 | zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, |
1312 | byteswap, bp); | |
b5256303 TC |
1313 | } |
1314 | SHA2Final(digestbuf, &ctx); | |
1315 | ||
1316 | if (generate) { | |
1317 | bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); | |
1318 | return (0); | |
1319 | } | |
1320 | ||
1321 | if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) | |
1322 | return (SET_ERROR(ECKSUM)); | |
1323 | ||
1324 | return (0); | |
1325 | } | |
1326 | ||
1327 | int | |
ae76f45c | 1328 | zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, |
b5256303 TC |
1329 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) |
1330 | { | |
ae76f45c | 1331 | int ret; |
b5256303 | 1332 | |
ae76f45c TC |
1333 | /* |
1334 | * Unfortunately, callers of this function will not always have | |
1335 | * easy access to the on-disk format version. This info is | |
1336 | * normally found in the DSL Crypto Key, but the checksum-of-MACs | |
1337 | * is expected to be verifiable even when the key isn't loaded. | |
1338 | * Here, instead of doing a ZAP lookup for the version for each | |
1339 | * zio, we simply try both existing formats. | |
1340 | */ | |
1341 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, | |
1342 | datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); | |
1343 | if (ret == ECKSUM) { | |
1344 | ASSERT(!generate); | |
1345 | ret = zio_crypt_do_indirect_mac_checksum_impl(generate, | |
1346 | buf, datalen, 0, byteswap, cksum); | |
1347 | } | |
1348 | ||
1349 | return (ret); | |
1350 | } | |
1351 | ||
1352 | int | |
1353 | zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, | |
1354 | uint_t datalen, boolean_t byteswap, uint8_t *cksum) | |
1355 | { | |
b5256303 TC |
1356 | int ret; |
1357 | void *buf; | |
1358 | ||
1359 | buf = abd_borrow_buf_copy(abd, datalen); | |
1360 | ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, | |
1361 | byteswap, cksum); | |
1362 | abd_return_buf(abd, buf, datalen); | |
1363 | ||
1364 | return (ret); | |
1365 | } | |
1366 | ||
1367 | /* | |
1368 | * Special case handling routine for encrypting / decrypting ZIL blocks. | |
1369 | * We do not check for the older ZIL chain because the encryption feature | |
1370 | * was not available before the newer ZIL chain was introduced. The goal | |
1371 | * here is to encrypt everything except the blkptr_t of a lr_write_t and | |
1372 | * the zil_chain_t header. Everything that is not encrypted is authenticated. | |
1373 | */ | |
1374 | static int | |
1375 | zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, | |
1376 | uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio, | |
1377 | uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, | |
1378 | boolean_t *no_crypt) | |
1379 | { | |
1380 | int ret; | |
9bae371c TC |
1381 | uint64_t txtype, lr_len; |
1382 | uint_t nr_src, nr_dst, crypt_len; | |
b5256303 TC |
1383 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; |
1384 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1385 | uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; | |
1386 | zil_chain_t *zilc; | |
1387 | lr_t *lr; | |
1388 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1389 | ||
1390 | /* cipherbuf always needs an extra iovec for the MAC */ | |
1391 | if (encrypt) { | |
1392 | src = plainbuf; | |
1393 | dst = cipherbuf; | |
1394 | nr_src = 0; | |
1395 | nr_dst = 1; | |
1396 | } else { | |
1397 | src = cipherbuf; | |
1398 | dst = plainbuf; | |
1399 | nr_src = 1; | |
1400 | nr_dst = 0; | |
1401 | } | |
e219935f | 1402 | bzero(dst, datalen); |
b5256303 TC |
1403 | |
1404 | /* find the start and end record of the log block */ | |
1405 | zilc = (zil_chain_t *)src; | |
1406 | slrp = src + sizeof (zil_chain_t); | |
1407 | aadp = aadbuf; | |
1408 | blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); | |
1409 | ||
1410 | /* calculate the number of encrypted iovecs we will need */ | |
1411 | for (; slrp < blkend; slrp += lr_len) { | |
1412 | lr = (lr_t *)slrp; | |
1413 | ||
1414 | if (!byteswap) { | |
1415 | txtype = lr->lrc_txtype; | |
1416 | lr_len = lr->lrc_reclen; | |
1417 | } else { | |
1418 | txtype = BSWAP_64(lr->lrc_txtype); | |
1419 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1420 | } | |
1421 | ||
1422 | nr_iovecs++; | |
1423 | if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) | |
1424 | nr_iovecs++; | |
1425 | } | |
1426 | ||
1427 | nr_src += nr_iovecs; | |
1428 | nr_dst += nr_iovecs; | |
1429 | ||
1430 | /* allocate the iovec arrays */ | |
1431 | if (nr_src != 0) { | |
1432 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1433 | if (src_iovecs == NULL) { |
b5256303 TC |
1434 | ret = SET_ERROR(ENOMEM); |
1435 | goto error; | |
1436 | } | |
1437 | } | |
1438 | ||
1439 | if (nr_dst != 0) { | |
1440 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1441 | if (dst_iovecs == NULL) { |
b5256303 TC |
1442 | ret = SET_ERROR(ENOMEM); |
1443 | goto error; | |
1444 | } | |
1445 | } | |
1446 | ||
1447 | /* | |
1448 | * Copy the plain zil header over and authenticate everything except | |
1449 | * the checksum that will store our MAC. If we are writing the data | |
1450 | * the embedded checksum will not have been calculated yet, so we don't | |
1451 | * authenticate that. | |
1452 | */ | |
1453 | bcopy(src, dst, sizeof (zil_chain_t)); | |
1454 | bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); | |
1455 | aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1456 | aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); | |
1457 | ||
1458 | /* loop over records again, filling in iovecs */ | |
1459 | nr_iovecs = 0; | |
1460 | slrp = src + sizeof (zil_chain_t); | |
1461 | dlrp = dst + sizeof (zil_chain_t); | |
1462 | ||
1463 | for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { | |
1464 | lr = (lr_t *)slrp; | |
1465 | ||
1466 | if (!byteswap) { | |
1467 | txtype = lr->lrc_txtype; | |
1468 | lr_len = lr->lrc_reclen; | |
1469 | } else { | |
1470 | txtype = BSWAP_64(lr->lrc_txtype); | |
1471 | lr_len = BSWAP_64(lr->lrc_reclen); | |
1472 | } | |
1473 | ||
1474 | /* copy the common lr_t */ | |
1475 | bcopy(slrp, dlrp, sizeof (lr_t)); | |
1476 | bcopy(slrp, aadp, sizeof (lr_t)); | |
1477 | aadp += sizeof (lr_t); | |
1478 | aad_len += sizeof (lr_t); | |
1479 | ||
4807c0ba TC |
1480 | ASSERT3P(src_iovecs, !=, NULL); |
1481 | ASSERT3P(dst_iovecs, !=, NULL); | |
1482 | ||
b5256303 TC |
1483 | /* |
1484 | * If this is a TX_WRITE record we want to encrypt everything | |
1485 | * except the bp if exists. If the bp does exist we want to | |
1486 | * authenticate it. | |
1487 | */ | |
1488 | if (txtype == TX_WRITE) { | |
1489 | crypt_len = sizeof (lr_write_t) - | |
1490 | sizeof (lr_t) - sizeof (blkptr_t); | |
1491 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1492 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1493 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1494 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1495 | ||
1496 | /* copy the bp now since it will not be encrypted */ | |
1497 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1498 | dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1499 | sizeof (blkptr_t)); | |
1500 | bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), | |
1501 | aadp, sizeof (blkptr_t)); | |
1502 | aadp += sizeof (blkptr_t); | |
1503 | aad_len += sizeof (blkptr_t); | |
1504 | nr_iovecs++; | |
1505 | total_len += crypt_len; | |
1506 | ||
1507 | if (lr_len != sizeof (lr_write_t)) { | |
1508 | crypt_len = lr_len - sizeof (lr_write_t); | |
1509 | src_iovecs[nr_iovecs].iov_base = | |
1510 | slrp + sizeof (lr_write_t); | |
1511 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1512 | dst_iovecs[nr_iovecs].iov_base = | |
1513 | dlrp + sizeof (lr_write_t); | |
1514 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1515 | nr_iovecs++; | |
1516 | total_len += crypt_len; | |
1517 | } | |
1518 | } else { | |
1519 | crypt_len = lr_len - sizeof (lr_t); | |
1520 | src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); | |
1521 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1522 | dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); | |
1523 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1524 | nr_iovecs++; | |
1525 | total_len += crypt_len; | |
1526 | } | |
1527 | } | |
1528 | ||
1529 | *no_crypt = (nr_iovecs == 0); | |
1530 | *enc_len = total_len; | |
1531 | *authbuf = aadbuf; | |
1532 | *auth_len = aad_len; | |
1533 | ||
1534 | if (encrypt) { | |
1535 | puio->uio_iov = src_iovecs; | |
1536 | puio->uio_iovcnt = nr_src; | |
1537 | cuio->uio_iov = dst_iovecs; | |
1538 | cuio->uio_iovcnt = nr_dst; | |
1539 | } else { | |
1540 | puio->uio_iov = dst_iovecs; | |
1541 | puio->uio_iovcnt = nr_dst; | |
1542 | cuio->uio_iov = src_iovecs; | |
1543 | cuio->uio_iovcnt = nr_src; | |
1544 | } | |
1545 | ||
1546 | return (0); | |
1547 | ||
1548 | error: | |
1549 | zio_buf_free(aadbuf, datalen); | |
1550 | if (src_iovecs != NULL) | |
1551 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1552 | if (dst_iovecs != NULL) | |
1553 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1554 | ||
1555 | *enc_len = 0; | |
1556 | *authbuf = NULL; | |
1557 | *auth_len = 0; | |
1558 | *no_crypt = B_FALSE; | |
1559 | puio->uio_iov = NULL; | |
1560 | puio->uio_iovcnt = 0; | |
1561 | cuio->uio_iov = NULL; | |
1562 | cuio->uio_iovcnt = 0; | |
1563 | return (ret); | |
1564 | } | |
1565 | ||
1566 | /* | |
1567 | * Special case handling routine for encrypting / decrypting dnode blocks. | |
1568 | */ | |
1569 | static int | |
ae76f45c TC |
1570 | zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, |
1571 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1572 | uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
1573 | uint_t *auth_len, boolean_t *no_crypt) | |
b5256303 TC |
1574 | { |
1575 | int ret; | |
1576 | uint_t nr_src, nr_dst, crypt_len; | |
1577 | uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; | |
1578 | uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; | |
1579 | iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; | |
1580 | uint8_t *src, *dst, *aadp; | |
1581 | dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; | |
1582 | uint8_t *aadbuf = zio_buf_alloc(datalen); | |
1583 | ||
1584 | if (encrypt) { | |
1585 | src = plainbuf; | |
1586 | dst = cipherbuf; | |
1587 | nr_src = 0; | |
1588 | nr_dst = 1; | |
1589 | } else { | |
1590 | src = cipherbuf; | |
1591 | dst = plainbuf; | |
1592 | nr_src = 1; | |
1593 | nr_dst = 0; | |
1594 | } | |
1595 | ||
1596 | sdnp = (dnode_phys_t *)src; | |
1597 | ddnp = (dnode_phys_t *)dst; | |
1598 | aadp = aadbuf; | |
1599 | ||
1600 | /* | |
1601 | * Count the number of iovecs we will need to do the encryption by | |
1602 | * counting the number of bonus buffers that need to be encrypted. | |
1603 | */ | |
1604 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1605 | /* | |
1606 | * This block may still be byteswapped. However, all of the | |
1607 | * values we use are either uint8_t's (for which byteswapping | |
1608 | * is a noop) or a * != 0 check, which will work regardless | |
1609 | * of whether or not we byteswap. | |
1610 | */ | |
1611 | if (sdnp[i].dn_type != DMU_OT_NONE && | |
1612 | DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && | |
1613 | sdnp[i].dn_bonuslen != 0) { | |
1614 | nr_iovecs++; | |
1615 | } | |
1616 | } | |
1617 | ||
1618 | nr_src += nr_iovecs; | |
1619 | nr_dst += nr_iovecs; | |
1620 | ||
1621 | if (nr_src != 0) { | |
1622 | src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1623 | if (src_iovecs == NULL) { |
b5256303 TC |
1624 | ret = SET_ERROR(ENOMEM); |
1625 | goto error; | |
1626 | } | |
1627 | } | |
1628 | ||
1629 | if (nr_dst != 0) { | |
1630 | dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); | |
4807c0ba | 1631 | if (dst_iovecs == NULL) { |
b5256303 TC |
1632 | ret = SET_ERROR(ENOMEM); |
1633 | goto error; | |
1634 | } | |
1635 | } | |
1636 | ||
1637 | nr_iovecs = 0; | |
1638 | ||
1639 | /* | |
1640 | * Iterate through the dnodes again, this time filling in the uios | |
1641 | * we allocated earlier. We also concatenate any data we want to | |
1642 | * authenticate onto aadbuf. | |
1643 | */ | |
1644 | for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { | |
1645 | dnp = &sdnp[i]; | |
1646 | ||
1647 | /* copy over the core fields and blkptrs (kept as plaintext) */ | |
1648 | bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); | |
1649 | ||
1650 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1651 | bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), | |
1652 | sizeof (blkptr_t)); | |
1653 | } | |
1654 | ||
1655 | /* | |
1656 | * Handle authenticated data. We authenticate everything in | |
1657 | * the dnode that can be brought over when we do a raw send. | |
1658 | * This includes all of the core fields as well as the MACs | |
1659 | * stored in the bp checksums and all of the portable bits | |
1660 | * from blk_prop. We include the dnode padding here in case it | |
1661 | * ever gets used in the future. Some dn_flags and dn_used are | |
1662 | * not portable so we mask those out values out of the | |
1663 | * authenticated data. | |
1664 | */ | |
1665 | crypt_len = offsetof(dnode_phys_t, dn_blkptr); | |
1666 | bcopy(dnp, aadp, crypt_len); | |
1667 | adnp = (dnode_phys_t *)aadp; | |
1668 | adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; | |
1669 | adnp->dn_used = 0; | |
1670 | aadp += crypt_len; | |
1671 | aad_len += crypt_len; | |
1672 | ||
1673 | for (j = 0; j < dnp->dn_nblkptr; j++) { | |
1674 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1675 | version, byteswap, &dnp->dn_blkptr[j]); |
b5256303 TC |
1676 | } |
1677 | ||
1678 | if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { | |
1679 | zio_crypt_bp_do_aad_updates(&aadp, &aad_len, | |
ae76f45c | 1680 | version, byteswap, DN_SPILL_BLKPTR(dnp)); |
b5256303 TC |
1681 | } |
1682 | ||
1683 | /* | |
1684 | * If this bonus buffer needs to be encrypted, we prepare an | |
1685 | * iovec_t. The encryption / decryption functions will fill | |
1686 | * this in for us with the encrypted or decrypted data. | |
1687 | * Otherwise we add the bonus buffer to the authenticated | |
1688 | * data buffer and copy it over to the destination. The | |
1689 | * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that | |
1690 | * we can guarantee alignment with the AES block size | |
1691 | * (128 bits). | |
1692 | */ | |
1693 | crypt_len = DN_MAX_BONUS_LEN(dnp); | |
1694 | if (dnp->dn_type != DMU_OT_NONE && | |
1695 | DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && | |
1696 | dnp->dn_bonuslen != 0) { | |
4807c0ba TC |
1697 | ASSERT3U(nr_iovecs, <, nr_src); |
1698 | ASSERT3U(nr_iovecs, <, nr_dst); | |
1699 | ASSERT3P(src_iovecs, !=, NULL); | |
1700 | ASSERT3P(dst_iovecs, !=, NULL); | |
b5256303 TC |
1701 | src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); |
1702 | src_iovecs[nr_iovecs].iov_len = crypt_len; | |
1703 | dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); | |
1704 | dst_iovecs[nr_iovecs].iov_len = crypt_len; | |
1705 | ||
1706 | nr_iovecs++; | |
1707 | total_len += crypt_len; | |
1708 | } else { | |
1709 | bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); | |
1710 | bcopy(DN_BONUS(dnp), aadp, crypt_len); | |
1711 | aadp += crypt_len; | |
1712 | aad_len += crypt_len; | |
1713 | } | |
1714 | } | |
1715 | ||
1716 | *no_crypt = (nr_iovecs == 0); | |
1717 | *enc_len = total_len; | |
1718 | *authbuf = aadbuf; | |
1719 | *auth_len = aad_len; | |
1720 | ||
1721 | if (encrypt) { | |
1722 | puio->uio_iov = src_iovecs; | |
1723 | puio->uio_iovcnt = nr_src; | |
1724 | cuio->uio_iov = dst_iovecs; | |
1725 | cuio->uio_iovcnt = nr_dst; | |
1726 | } else { | |
1727 | puio->uio_iov = dst_iovecs; | |
1728 | puio->uio_iovcnt = nr_dst; | |
1729 | cuio->uio_iov = src_iovecs; | |
1730 | cuio->uio_iovcnt = nr_src; | |
1731 | } | |
1732 | ||
1733 | return (0); | |
1734 | ||
1735 | error: | |
1736 | zio_buf_free(aadbuf, datalen); | |
1737 | if (src_iovecs != NULL) | |
1738 | kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); | |
1739 | if (dst_iovecs != NULL) | |
1740 | kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); | |
1741 | ||
1742 | *enc_len = 0; | |
1743 | *authbuf = NULL; | |
1744 | *auth_len = 0; | |
1745 | *no_crypt = B_FALSE; | |
1746 | puio->uio_iov = NULL; | |
1747 | puio->uio_iovcnt = 0; | |
1748 | cuio->uio_iov = NULL; | |
1749 | cuio->uio_iovcnt = 0; | |
1750 | return (ret); | |
1751 | } | |
1752 | ||
1753 | static int | |
1754 | zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, | |
1755 | uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio, | |
1756 | uint_t *enc_len) | |
1757 | { | |
1758 | int ret; | |
1759 | uint_t nr_plain = 1, nr_cipher = 2; | |
1760 | iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; | |
1761 | ||
1762 | /* allocate the iovecs for the plain and cipher data */ | |
1763 | plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), | |
1764 | KM_SLEEP); | |
1765 | if (!plain_iovecs) { | |
1766 | ret = SET_ERROR(ENOMEM); | |
1767 | goto error; | |
1768 | } | |
1769 | ||
1770 | cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), | |
1771 | KM_SLEEP); | |
1772 | if (!cipher_iovecs) { | |
1773 | ret = SET_ERROR(ENOMEM); | |
1774 | goto error; | |
1775 | } | |
1776 | ||
1777 | plain_iovecs[0].iov_base = plainbuf; | |
1778 | plain_iovecs[0].iov_len = datalen; | |
1779 | cipher_iovecs[0].iov_base = cipherbuf; | |
1780 | cipher_iovecs[0].iov_len = datalen; | |
1781 | ||
1782 | *enc_len = datalen; | |
1783 | puio->uio_iov = plain_iovecs; | |
1784 | puio->uio_iovcnt = nr_plain; | |
1785 | cuio->uio_iov = cipher_iovecs; | |
1786 | cuio->uio_iovcnt = nr_cipher; | |
1787 | ||
1788 | return (0); | |
1789 | ||
1790 | error: | |
1791 | if (plain_iovecs != NULL) | |
1792 | kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); | |
1793 | if (cipher_iovecs != NULL) | |
1794 | kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); | |
1795 | ||
1796 | *enc_len = 0; | |
1797 | puio->uio_iov = NULL; | |
1798 | puio->uio_iovcnt = 0; | |
1799 | cuio->uio_iov = NULL; | |
1800 | cuio->uio_iovcnt = 0; | |
1801 | return (ret); | |
1802 | } | |
1803 | ||
1804 | /* | |
1805 | * This function builds up the plaintext (puio) and ciphertext (cuio) uios so | |
1806 | * that they can be used for encryption and decryption by zio_do_crypt_uio(). | |
1807 | * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks | |
1808 | * requiring special handling to parse out pieces that are to be encrypted. The | |
1809 | * authbuf is used by these special cases to store additional authenticated | |
1810 | * data (AAD) for the encryption modes. | |
1811 | */ | |
1812 | static int | |
ae76f45c TC |
1813 | zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, |
1814 | uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, | |
1815 | uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, | |
b5256303 TC |
1816 | uint_t *auth_len, boolean_t *no_crypt) |
1817 | { | |
1818 | int ret; | |
1819 | iovec_t *mac_iov; | |
1820 | ||
1821 | ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); | |
1822 | ||
1823 | /* route to handler */ | |
1824 | switch (ot) { | |
1825 | case DMU_OT_INTENT_LOG: | |
1826 | ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, | |
1827 | datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, | |
1828 | no_crypt); | |
1829 | break; | |
1830 | case DMU_OT_DNODE: | |
ae76f45c TC |
1831 | ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, |
1832 | cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, | |
1833 | auth_len, no_crypt); | |
b5256303 TC |
1834 | break; |
1835 | default: | |
1836 | ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, | |
1837 | datalen, puio, cuio, enc_len); | |
1838 | *authbuf = NULL; | |
1839 | *auth_len = 0; | |
1840 | *no_crypt = B_FALSE; | |
1841 | break; | |
1842 | } | |
1843 | ||
1844 | if (ret != 0) | |
1845 | goto error; | |
1846 | ||
1847 | /* populate the uios */ | |
1848 | puio->uio_segflg = UIO_SYSSPACE; | |
1849 | cuio->uio_segflg = UIO_SYSSPACE; | |
1850 | ||
1851 | mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); | |
1852 | mac_iov->iov_base = mac; | |
1853 | mac_iov->iov_len = ZIO_DATA_MAC_LEN; | |
1854 | ||
1855 | return (0); | |
1856 | ||
1857 | error: | |
1858 | return (ret); | |
1859 | } | |
1860 | ||
1861 | /* | |
1862 | * Primary encryption / decryption entrypoint for zio data. | |
1863 | */ | |
1864 | int | |
10fa2545 | 1865 | zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, |
be9a5c35 TC |
1866 | dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, |
1867 | uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, | |
b5256303 TC |
1868 | boolean_t *no_crypt) |
1869 | { | |
1870 | int ret; | |
1871 | boolean_t locked = B_FALSE; | |
1872 | uint64_t crypt = key->zk_crypt; | |
1873 | uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; | |
1874 | uint_t enc_len, auth_len; | |
1875 | uio_t puio, cuio; | |
1876 | uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; | |
1877 | crypto_key_t tmp_ckey, *ckey = NULL; | |
1878 | crypto_ctx_template_t tmpl; | |
1879 | uint8_t *authbuf = NULL; | |
1880 | ||
b5256303 TC |
1881 | /* |
1882 | * If the needed key is the current one, just use it. Otherwise we | |
1883 | * need to generate a temporary one from the given salt + master key. | |
1884 | * If we are encrypting, we must return a copy of the current salt | |
1885 | * so that it can be stored in the blkptr_t. | |
1886 | */ | |
1887 | rw_enter(&key->zk_salt_lock, RW_READER); | |
1888 | locked = B_TRUE; | |
1889 | ||
1890 | if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { | |
1891 | ckey = &key->zk_current_key; | |
1892 | tmpl = key->zk_current_tmpl; | |
1893 | } else { | |
1894 | rw_exit(&key->zk_salt_lock); | |
1895 | locked = B_FALSE; | |
1896 | ||
1897 | ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, | |
1898 | salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); | |
1899 | if (ret != 0) | |
1900 | goto error; | |
1901 | ||
1902 | tmp_ckey.ck_format = CRYPTO_KEY_RAW; | |
1903 | tmp_ckey.ck_data = enc_keydata; | |
4807c0ba | 1904 | tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); |
b5256303 TC |
1905 | |
1906 | ckey = &tmp_ckey; | |
1907 | tmpl = NULL; | |
1908 | } | |
1909 | ||
cf637391 TC |
1910 | /* |
1911 | * Attempt to use QAT acceleration if we can. We currently don't | |
1912 | * do this for metadnode and ZIL blocks, since they have a much | |
1913 | * more involved buffer layout and the qat_crypt() function only | |
1914 | * works in-place. | |
1915 | */ | |
1916 | if (qat_crypt_use_accel(datalen) && | |
1917 | ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { | |
1918 | uint8_t *srcbuf, *dstbuf; | |
1919 | ||
1920 | if (encrypt) { | |
1921 | srcbuf = plainbuf; | |
1922 | dstbuf = cipherbuf; | |
1923 | } else { | |
1924 | srcbuf = cipherbuf; | |
1925 | dstbuf = plainbuf; | |
1926 | } | |
1927 | ||
1928 | ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, | |
1929 | dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); | |
1930 | if (ret == CPA_STATUS_SUCCESS) { | |
1931 | if (locked) { | |
1932 | rw_exit(&key->zk_salt_lock); | |
1933 | locked = B_FALSE; | |
1934 | } | |
1935 | ||
1936 | return (0); | |
1937 | } | |
1938 | /* If the hardware implementation fails fall back to software */ | |
1939 | } | |
1940 | ||
1941 | bzero(&puio, sizeof (uio_t)); | |
1942 | bzero(&cuio, sizeof (uio_t)); | |
1943 | ||
1944 | /* create uios for encryption */ | |
1945 | ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, | |
1946 | cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, | |
1947 | &authbuf, &auth_len, no_crypt); | |
1948 | if (ret != 0) | |
1949 | goto error; | |
1950 | ||
1951 | /* perform the encryption / decryption in software */ | |
10fa2545 BB |
1952 | ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, |
1953 | &puio, &cuio, authbuf, auth_len); | |
b5256303 TC |
1954 | if (ret != 0) |
1955 | goto error; | |
1956 | ||
1957 | if (locked) { | |
1958 | rw_exit(&key->zk_salt_lock); | |
1959 | locked = B_FALSE; | |
1960 | } | |
1961 | ||
1962 | if (authbuf != NULL) | |
1963 | zio_buf_free(authbuf, datalen); | |
1964 | if (ckey == &tmp_ckey) | |
1965 | bzero(enc_keydata, keydata_len); | |
1966 | zio_crypt_destroy_uio(&puio); | |
1967 | zio_crypt_destroy_uio(&cuio); | |
1968 | ||
1969 | return (0); | |
1970 | ||
1971 | error: | |
1972 | if (locked) | |
1973 | rw_exit(&key->zk_salt_lock); | |
1974 | if (authbuf != NULL) | |
1975 | zio_buf_free(authbuf, datalen); | |
1976 | if (ckey == &tmp_ckey) | |
1977 | bzero(enc_keydata, keydata_len); | |
1978 | zio_crypt_destroy_uio(&puio); | |
1979 | zio_crypt_destroy_uio(&cuio); | |
1980 | ||
1981 | return (ret); | |
1982 | } | |
1983 | ||
1984 | /* | |
1985 | * Simple wrapper around zio_do_crypt_data() to work with abd's instead of | |
1986 | * linear buffers. | |
1987 | */ | |
1988 | int | |
10fa2545 BB |
1989 | zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, |
1990 | boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, | |
1991 | uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) | |
b5256303 TC |
1992 | { |
1993 | int ret; | |
1994 | void *ptmp, *ctmp; | |
1995 | ||
1996 | if (encrypt) { | |
1997 | ptmp = abd_borrow_buf_copy(pabd, datalen); | |
1998 | ctmp = abd_borrow_buf(cabd, datalen); | |
1999 | } else { | |
2000 | ptmp = abd_borrow_buf(pabd, datalen); | |
2001 | ctmp = abd_borrow_buf_copy(cabd, datalen); | |
2002 | } | |
2003 | ||
10fa2545 | 2004 | ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, |
be9a5c35 | 2005 | datalen, ptmp, ctmp, no_crypt); |
b5256303 TC |
2006 | if (ret != 0) |
2007 | goto error; | |
2008 | ||
2009 | if (encrypt) { | |
2010 | abd_return_buf(pabd, ptmp, datalen); | |
2011 | abd_return_buf_copy(cabd, ctmp, datalen); | |
2012 | } else { | |
2013 | abd_return_buf_copy(pabd, ptmp, datalen); | |
2014 | abd_return_buf(cabd, ctmp, datalen); | |
2015 | } | |
2016 | ||
2017 | return (0); | |
2018 | ||
2019 | error: | |
2020 | if (encrypt) { | |
2021 | abd_return_buf(pabd, ptmp, datalen); | |
2022 | abd_return_buf_copy(cabd, ctmp, datalen); | |
2023 | } else { | |
2024 | abd_return_buf_copy(pabd, ptmp, datalen); | |
2025 | abd_return_buf(cabd, ctmp, datalen); | |
2026 | } | |
2027 | ||
2028 | return (ret); | |
2029 | } | |
2030 | ||
93ce2b4c | 2031 | #if defined(_KERNEL) |
b5256303 TC |
2032 | /* BEGIN CSTYLED */ |
2033 | module_param(zfs_key_max_salt_uses, ulong, 0644); | |
2034 | MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " | |
2035 | "can be used for generating encryption keys before it is rotated"); | |
2036 | /* END CSTYLED */ | |
2037 | #endif |