2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/types.h>
34 #include <sys/errno.h>
37 #include <sys/libkern.h>
38 #include <sys/malloc.h>
39 #include <sys/sysctl.h>
40 #include <opencrypto/cryptodev.h>
41 #include <opencrypto/xform.h>
46 #include <sys/zio_crypt.h>
47 #include <sys/fs/zfs.h>
50 #include <sys/freebsd_crypto.h>
52 #define SHA512_HMAC_BLOCK_SIZE 128
54 static int crypt_sessions
= 0;
55 SYSCTL_DECL(_vfs_zfs
);
56 SYSCTL_INT(_vfs_zfs
, OID_AUTO
, crypt_sessions
, CTLFLAG_RD
,
57 &crypt_sessions
, 0, "Number of cryptographic sessions created");
60 crypto_mac_init(struct hmac_ctx
*ctx
, const crypto_key_t
*c_key
)
62 uint8_t k_ipad
[SHA512_HMAC_BLOCK_SIZE
],
63 k_opad
[SHA512_HMAC_BLOCK_SIZE
],
64 key
[SHA512_HMAC_BLOCK_SIZE
];
67 size_t cl_bytes
= CRYPTO_BITS2BYTES(c_key
->ck_length
);
70 * This code is based on the similar code in geom/eli/g_eli_hmac.c
72 explicit_bzero(key
, sizeof (key
));
73 if (c_key
->ck_length
== 0)
75 else if (cl_bytes
<= SHA512_HMAC_BLOCK_SIZE
)
76 bcopy(c_key
->ck_data
, key
, cl_bytes
);
79 * If key is longer than 128 bytes reset it to
83 SHA512_Update(&lctx
, c_key
->ck_data
, cl_bytes
);
84 SHA512_Final(key
, &lctx
);
87 /* XOR key with ipad and opad values. */
88 for (i
= 0; i
< sizeof (key
); i
++) {
89 k_ipad
[i
] = key
[i
] ^ 0x36;
90 k_opad
[i
] = key
[i
] ^ 0x5c;
92 explicit_bzero(key
, sizeof (key
));
94 /* Start inner SHA512. */
95 SHA512_Init(&ctx
->innerctx
);
96 SHA512_Update(&ctx
->innerctx
, k_ipad
, sizeof (k_ipad
));
97 explicit_bzero(k_ipad
, sizeof (k_ipad
));
98 /* Start outer SHA512. */
99 SHA512_Init(&ctx
->outerctx
);
100 SHA512_Update(&ctx
->outerctx
, k_opad
, sizeof (k_opad
));
101 explicit_bzero(k_opad
, sizeof (k_opad
));
105 crypto_mac_update(struct hmac_ctx
*ctx
, const void *data
, size_t datasize
)
107 SHA512_Update(&ctx
->innerctx
, data
, datasize
);
111 crypto_mac_final(struct hmac_ctx
*ctx
, void *md
, size_t mdsize
)
113 uint8_t digest
[SHA512_DIGEST_LENGTH
];
115 /* Complete inner hash */
116 SHA512_Final(digest
, &ctx
->innerctx
);
118 /* Complete outer hash */
119 SHA512_Update(&ctx
->outerctx
, digest
, sizeof (digest
));
120 SHA512_Final(digest
, &ctx
->outerctx
);
122 explicit_bzero(ctx
, sizeof (*ctx
));
123 /* mdsize == 0 means "Give me the whole hash!" */
125 mdsize
= SHA512_DIGEST_LENGTH
;
126 bcopy(digest
, md
, mdsize
);
127 explicit_bzero(digest
, sizeof (digest
));
131 crypto_mac(const crypto_key_t
*key
, const void *in_data
, size_t in_data_size
,
132 void *out_data
, size_t out_data_size
)
136 crypto_mac_init(&ctx
, key
);
137 crypto_mac_update(&ctx
, in_data
, in_data_size
);
138 crypto_mac_final(&ctx
, out_data
, out_data_size
);
142 freebsd_zfs_crypt_done(struct cryptop
*crp
)
144 freebsd_crypt_session_t
*ses
;
146 ses
= crp
->crp_opaque
;
147 mtx_lock(&ses
->fs_lock
);
149 mtx_unlock(&ses
->fs_lock
);
155 freebsd_crypt_freesession(freebsd_crypt_session_t
*sess
)
157 mtx_destroy(&sess
->fs_lock
);
158 crypto_freesession(sess
->fs_sid
);
159 explicit_bzero(sess
, sizeof (*sess
));
163 zfs_crypto_dispatch(freebsd_crypt_session_t
*session
, struct cryptop
*crp
)
167 crp
->crp_opaque
= session
;
168 crp
->crp_callback
= freebsd_zfs_crypt_done
;
170 error
= crypto_dispatch(crp
);
173 mtx_lock(&session
->fs_lock
);
174 while (session
->fs_done
== false)
175 msleep(crp
, &session
->fs_lock
, PRIBIO
,
177 mtx_unlock(&session
->fs_lock
);
179 if (crp
->crp_etype
!= EAGAIN
) {
180 error
= crp
->crp_etype
;
184 crp
->crp_flags
&= ~CRYPTO_F_DONE
;
185 session
->fs_done
= false;
186 #if __FreeBSD_version < 1300087
188 * Session ID changed, so we should record that,
191 session
->fs_sid
= crp
->crp_session
;
197 freebsd_crypt_uio_debug_log(boolean_t encrypt
,
198 freebsd_crypt_session_t
*input_sessionp
,
199 struct zio_crypt_info
*c_info
,
207 struct cryptodesc
*crd
;
211 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, "
213 __FUNCTION__
, encrypt
? "encrypt" : "decrypt", input_sessionp
,
214 c_info
->ci_algname
, c_info
->ci_crypt_type
,
215 (unsigned int)c_info
->ci_keylen
, c_info
->ci_name
,
216 data_uio
, key
->ck_format
, key
->ck_data
,
217 (unsigned int)key
->ck_length
,
218 ivbuf
, (unsigned int)datalen
, (unsigned int)auth_len
);
219 printf("\tkey = { ");
220 for (int i
= 0; i
< key
->ck_length
/ 8; i
++) {
221 uint8_t *b
= (uint8_t *)key
->ck_data
;
222 printf("%02x ", b
[i
]);
225 for (int i
= 0; i
< data_uio
->uio_iovcnt
; i
++) {
226 printf("\tiovec #%d: <%p, %u>\n", i
,
227 data_uio
->uio_iov
[i
].iov_base
,
228 (unsigned int)data_uio
->uio_iov
[i
].iov_len
);
229 total
+= data_uio
->uio_iov
[i
].iov_len
;
231 data_uio
->uio_resid
= total
;
235 * Create a new cryptographic session. This should
236 * happen every time the key changes (including when
237 * it's first loaded).
239 #if __FreeBSD_version >= 1300087
241 freebsd_crypt_newsession(freebsd_crypt_session_t
*sessp
,
242 struct zio_crypt_info
*c_info
, crypto_key_t
*key
)
244 struct crypto_session_params csp
;
248 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
250 c_info
->ci_algname
, c_info
->ci_crypt_type
,
251 (unsigned int)c_info
->ci_keylen
, c_info
->ci_name
,
252 key
->ck_format
, key
->ck_data
, (unsigned int)key
->ck_length
);
253 printf("\tkey = { ");
254 for (int i
= 0; i
< key
->ck_length
/ 8; i
++) {
255 uint8_t *b
= (uint8_t *)key
->ck_data
;
256 printf("%02x ", b
[i
]);
260 bzero(&csp
, sizeof (csp
));
261 csp
.csp_mode
= CSP_MODE_AEAD
;
262 csp
.csp_cipher_key
= key
->ck_data
;
263 csp
.csp_cipher_klen
= key
->ck_length
/ 8;
264 switch (c_info
->ci_crypt_type
) {
266 csp
.csp_cipher_alg
= CRYPTO_AES_NIST_GCM_16
;
267 csp
.csp_ivlen
= AES_GCM_IV_LEN
;
268 switch (key
->ck_length
/8) {
269 case AES_128_GMAC_KEY_LEN
:
270 case AES_192_GMAC_KEY_LEN
:
271 case AES_256_GMAC_KEY_LEN
:
279 csp
.csp_cipher_alg
= CRYPTO_AES_CCM_16
;
280 csp
.csp_ivlen
= AES_CCM_IV_LEN
;
281 switch (key
->ck_length
/8) {
282 case AES_128_CBC_MAC_KEY_LEN
:
283 case AES_192_CBC_MAC_KEY_LEN
:
284 case AES_256_CBC_MAC_KEY_LEN
:
296 error
= crypto_newsession(&sessp
->fs_sid
, &csp
,
297 CRYPTOCAP_F_HARDWARE
| CRYPTOCAP_F_SOFTWARE
);
298 mtx_init(&sessp
->fs_lock
, "FreeBSD Cryptographic Session Lock",
304 printf("%s: returning error %d\n", __FUNCTION__
, error
);
310 freebsd_crypt_uio(boolean_t encrypt
,
311 freebsd_crypt_session_t
*input_sessionp
,
312 struct zio_crypt_info
*c_info
,
320 freebsd_crypt_session_t
*session
= NULL
;
324 freebsd_crypt_uio_debug_log(encrypt
, input_sessionp
, c_info
, data_uio
,
325 key
, ivbuf
, datalen
, auth_len
);
326 for (int i
= 0; i
< data_uio
->uio_iovcnt
; i
++)
327 total
+= data_uio
->uio_iov
[i
].iov_len
;
328 data_uio
->uio_resid
= total
;
329 if (input_sessionp
== NULL
) {
330 session
= kmem_zalloc(sizeof (*session
), KM_SLEEP
);
331 error
= freebsd_crypt_newsession(session
, c_info
, key
);
335 session
= input_sessionp
;
337 crp
= crypto_getreq(session
->fs_sid
, M_WAITOK
);
339 crp
->crp_op
= CRYPTO_OP_ENCRYPT
|
340 CRYPTO_OP_COMPUTE_DIGEST
;
342 crp
->crp_op
= CRYPTO_OP_DECRYPT
|
343 CRYPTO_OP_VERIFY_DIGEST
;
345 crp
->crp_flags
= CRYPTO_F_CBIFSYNC
| CRYPTO_F_IV_SEPARATE
;
346 crp
->crp_buf_type
= CRYPTO_BUF_UIO
;
347 crp
->crp_uio
= (void*)data_uio
;
348 crp
->crp_ilen
= data_uio
->uio_resid
;
350 crp
->crp_aad_start
= 0;
351 crp
->crp_aad_length
= auth_len
;
352 crp
->crp_payload_start
= auth_len
;
353 crp
->crp_payload_length
= datalen
;
354 crp
->crp_digest_start
= auth_len
+ datalen
;
356 bcopy(ivbuf
, crp
->crp_iv
, ZIO_DATA_IV_LEN
);
357 error
= zfs_crypto_dispatch(session
, crp
);
362 printf("%s: returning error %d\n", __FUNCTION__
, error
);
364 if (input_sessionp
== NULL
) {
365 freebsd_crypt_freesession(session
);
366 kmem_free(session
, sizeof (*session
));
373 freebsd_crypt_newsession(freebsd_crypt_session_t
*sessp
,
374 struct zio_crypt_info
*c_info
, crypto_key_t
*key
)
376 struct cryptoini cria
, crie
, *crip
;
377 struct enc_xform
*xform
;
378 struct auth_hash
*xauth
;
380 crypto_session_t sid
;
383 printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
385 c_info
->ci_algname
, c_info
->ci_crypt_type
,
386 (unsigned int)c_info
->ci_keylen
, c_info
->ci_name
,
387 key
->ck_format
, key
->ck_data
, (unsigned int)key
->ck_length
);
388 printf("\tkey = { ");
389 for (int i
= 0; i
< key
->ck_length
/ 8; i
++) {
390 uint8_t *b
= (uint8_t *)key
->ck_data
;
391 printf("%02x ", b
[i
]);
395 switch (c_info
->ci_crypt_type
) {
397 xform
= &enc_xform_aes_nist_gcm
;
398 switch (key
->ck_length
/8) {
399 case AES_128_GMAC_KEY_LEN
:
400 xauth
= &auth_hash_nist_gmac_aes_128
;
402 case AES_192_GMAC_KEY_LEN
:
403 xauth
= &auth_hash_nist_gmac_aes_192
;
405 case AES_256_GMAC_KEY_LEN
:
406 xauth
= &auth_hash_nist_gmac_aes_256
;
414 xform
= &enc_xform_ccm
;
415 switch (key
->ck_length
/8) {
416 case AES_128_CBC_MAC_KEY_LEN
:
417 xauth
= &auth_hash_ccm_cbc_mac_128
;
419 case AES_192_CBC_MAC_KEY_LEN
:
420 xauth
= &auth_hash_ccm_cbc_mac_192
;
422 case AES_256_CBC_MAC_KEY_LEN
:
423 xauth
= &auth_hash_ccm_cbc_mac_256
;
436 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
437 "auth %s (key length %d)\n",
438 __FUNCTION__
, __LINE__
,
439 xform
->name
, (unsigned int)key
->ck_length
,
440 (unsigned int)key
->ck_length
/8,
441 xauth
->name
, xauth
->keysize
);
444 bzero(&crie
, sizeof (crie
));
445 bzero(&cria
, sizeof (cria
));
447 crie
.cri_alg
= xform
->type
;
448 crie
.cri_key
= key
->ck_data
;
449 crie
.cri_klen
= key
->ck_length
;
451 cria
.cri_alg
= xauth
->type
;
452 cria
.cri_key
= key
->ck_data
;
453 cria
.cri_klen
= key
->ck_length
;
455 cria
.cri_next
= &crie
;
456 crie
.cri_next
= NULL
;
458 // Everything else is bzero'd
460 error
= crypto_newsession(&sid
, crip
,
461 CRYPTOCAP_F_HARDWARE
| CRYPTOCAP_F_SOFTWARE
);
463 printf("%s(%d): crypto_newsession failed with %d\n",
464 __FUNCTION__
, __LINE__
, error
);
468 mtx_init(&sessp
->fs_lock
, "FreeBSD Cryptographic Session Lock",
476 * The meat of encryption/decryption.
477 * If sessp is NULL, then it will create a
478 * temporary cryptographic session, and release
482 freebsd_crypt_uio(boolean_t encrypt
,
483 freebsd_crypt_session_t
*input_sessionp
,
484 struct zio_crypt_info
*c_info
,
492 struct cryptodesc
*enc_desc
, *auth_desc
;
493 struct enc_xform
*xform
;
494 struct auth_hash
*xauth
;
495 freebsd_crypt_session_t
*session
= NULL
;
498 freebsd_crypt_uio_debug_log(encrypt
, input_sessionp
, c_info
, data_uio
,
499 key
, ivbuf
, datalen
, auth_len
);
500 switch (c_info
->ci_crypt_type
) {
502 xform
= &enc_xform_aes_nist_gcm
;
503 switch (key
->ck_length
/8) {
504 case AES_128_GMAC_KEY_LEN
:
505 xauth
= &auth_hash_nist_gmac_aes_128
;
507 case AES_192_GMAC_KEY_LEN
:
508 xauth
= &auth_hash_nist_gmac_aes_192
;
510 case AES_256_GMAC_KEY_LEN
:
511 xauth
= &auth_hash_nist_gmac_aes_256
;
519 xform
= &enc_xform_ccm
;
520 switch (key
->ck_length
/8) {
521 case AES_128_CBC_MAC_KEY_LEN
:
522 xauth
= &auth_hash_ccm_cbc_mac_128
;
524 case AES_192_CBC_MAC_KEY_LEN
:
525 xauth
= &auth_hash_ccm_cbc_mac_192
;
527 case AES_256_CBC_MAC_KEY_LEN
:
528 xauth
= &auth_hash_ccm_cbc_mac_256
;
542 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
543 "auth %s (key length %d)\n",
544 __FUNCTION__
, __LINE__
,
545 xform
->name
, (unsigned int)key
->ck_length
,
546 (unsigned int)key
->ck_length
/8,
547 xauth
->name
, xauth
->keysize
);
550 if (input_sessionp
== NULL
) {
551 session
= kmem_zalloc(sizeof (*session
), KM_SLEEP
);
552 error
= freebsd_crypt_newsession(session
, c_info
, key
);
556 session
= input_sessionp
;
558 crp
= crypto_getreq(2);
564 auth_desc
= crp
->crp_desc
;
565 enc_desc
= auth_desc
->crd_next
;
567 crp
->crp_session
= session
->fs_sid
;
568 crp
->crp_ilen
= auth_len
+ datalen
;
569 crp
->crp_buf
= (void*)data_uio
;
570 crp
->crp_flags
= CRYPTO_F_IOV
| CRYPTO_F_CBIFSYNC
;
572 auth_desc
->crd_skip
= 0;
573 auth_desc
->crd_len
= auth_len
;
574 auth_desc
->crd_inject
= auth_len
+ datalen
;
575 auth_desc
->crd_alg
= xauth
->type
;
577 printf("%s: auth: skip = %u, len = %u, inject = %u\n",
578 __FUNCTION__
, auth_desc
->crd_skip
, auth_desc
->crd_len
,
579 auth_desc
->crd_inject
);
582 enc_desc
->crd_skip
= auth_len
;
583 enc_desc
->crd_len
= datalen
;
584 enc_desc
->crd_inject
= auth_len
;
585 enc_desc
->crd_alg
= xform
->type
;
586 enc_desc
->crd_flags
= CRD_F_IV_EXPLICIT
| CRD_F_IV_PRESENT
;
587 bcopy(ivbuf
, enc_desc
->crd_iv
, ZIO_DATA_IV_LEN
);
588 enc_desc
->crd_next
= NULL
;
591 printf("%s: enc: skip = %u, len = %u, inject = %u\n",
592 __FUNCTION__
, enc_desc
->crd_skip
, enc_desc
->crd_len
,
593 enc_desc
->crd_inject
);
597 enc_desc
->crd_flags
|= CRD_F_ENCRYPT
;
599 error
= zfs_crypto_dispatch(session
, crp
);
602 if (input_sessionp
== NULL
) {
603 freebsd_crypt_freesession(session
);
604 kmem_free(session
, sizeof (*session
));
609 printf("%s: returning error %d\n", __FUNCTION__
, error
);