[mirror_zfs.git] / module / os / freebsd / zfs / crypto_os.c

/*
 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/types.h>
#include <sys/errno.h>

#ifdef _KERNEL
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>
#else
#include <strings.h>
#endif

#include <sys/zio_crypt.h>
#include <sys/fs/zfs.h>
#include <sys/zio.h>

#include <sys/freebsd_crypto.h>

#define	SHA512_HMAC_BLOCK_SIZE	128

static int crypt_sessions = 0;
SYSCTL_DECL(_vfs_zfs);
SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
	&crypt_sessions, 0, "Number of cryptographic sessions created");

void
crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key)
{
	uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
	    k_opad[SHA512_HMAC_BLOCK_SIZE],
	    key[SHA512_HMAC_BLOCK_SIZE];
	SHA512_CTX lctx;
	int i;
	size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);

	/*
	 * This code is based on the similar code in geom/eli/g_eli_hmac.c
	 */
	explicit_bzero(key, sizeof (key));
	if (c_key->ck_length  == 0)
		/* do nothing */;
	else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
		bcopy(c_key->ck_data, key, cl_bytes);
	else {
		/*
		 * If key is longer than 128 bytes reset it to
		 * key = SHA512(key).
		 */
		SHA512_Init(&lctx);
		SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
		SHA512_Final(key, &lctx);
	}

	/* XOR key with ipad and opad values. */
	for (i = 0; i < sizeof (key); i++) {
		k_ipad[i] = key[i] ^ 0x36;
		k_opad[i] = key[i] ^ 0x5c;
	}
	explicit_bzero(key, sizeof (key));

	/* Start inner SHA512. */
	SHA512_Init(&ctx->innerctx);
	SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
	explicit_bzero(k_ipad, sizeof (k_ipad));
	/* Start outer SHA512. */
	SHA512_Init(&ctx->outerctx);
	SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
	explicit_bzero(k_opad, sizeof (k_opad));
}

void
crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize)
{
	SHA512_Update(&ctx->innerctx, data, datasize);
}

void
crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize)
{
	uint8_t digest[SHA512_DIGEST_LENGTH];

	/* Complete inner hash */
	SHA512_Final(digest, &ctx->innerctx);

	/* Complete outer hash */
	SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
	SHA512_Final(digest, &ctx->outerctx);

	explicit_bzero(ctx, sizeof (*ctx));
	/* mdsize == 0 means "Give me the whole hash!" */
	if (mdsize == 0)
		mdsize = SHA512_DIGEST_LENGTH;
	bcopy(digest, md, mdsize);
	explicit_bzero(digest, sizeof (digest));
}

void
crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size,
    void *out_data, size_t out_data_size)
{
	struct hmac_ctx ctx;

	crypto_mac_init(&ctx, key);
	crypto_mac_update(&ctx, in_data, in_data_size);
	crypto_mac_final(&ctx, out_data, out_data_size);
}

static int
freebsd_zfs_crypt_done(struct cryptop *crp)
{
	freebsd_crypt_session_t *ses;

	ses = crp->crp_opaque;
	mtx_lock(&ses->fs_lock);
	ses->fs_done = true;
	mtx_unlock(&ses->fs_lock);
	wakeup(crp);
	return (0);
}

void
freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
{
	mtx_destroy(&sess->fs_lock);
	crypto_freesession(sess->fs_sid);
	explicit_bzero(sess, sizeof (*sess));
}

static int
zfs_crypto_dispatch(freebsd_crypt_session_t *session, 	struct cryptop *crp)
{
	int error;

	crp->crp_opaque = session;
	crp->crp_callback = freebsd_zfs_crypt_done;
	for (;;) {
		error = crypto_dispatch(crp);
		if (error)
			break;
		mtx_lock(&session->fs_lock);
		while (session->fs_done == false)
			msleep(crp, &session->fs_lock, 0,
			    "zfs_crypto", 0);
		mtx_unlock(&session->fs_lock);

		if (crp->crp_etype == ENOMEM) {
			pause("zcrnomem", 1);
		} else if (crp->crp_etype != EAGAIN) {
			error = crp->crp_etype;
			break;
		}
		crp->crp_etype = 0;
		crp->crp_flags &= ~CRYPTO_F_DONE;
		session->fs_done = false;
#if __FreeBSD_version < 1300087
		/*
		 * Session ID changed, so we should record that,
		 * and try again
		 */
		session->fs_sid = crp->crp_session;
#endif
	}
	return (error);
}
static void
freebsd_crypt_uio_debug_log(boolean_t encrypt,
    freebsd_crypt_session_t *input_sessionp,
    const struct zio_crypt_info *c_info,
    zfs_uio_t *data_uio,
    crypto_key_t *key,
    uint8_t *ivbuf,
    size_t datalen,
    size_t auth_len)
{
#ifdef FCRYPTO_DEBUG
	struct cryptodesc *crd;
	uint8_t *p = NULL;
	size_t total = 0;

	printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, "
	    "%p, %u, %u)\n",
	    __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
	    c_info->ci_algname, c_info->ci_crypt_type,
	    (unsigned int)c_info->ci_keylen, c_info->ci_name,
	    data_uio, key->ck_format, key->ck_data,
	    (unsigned int)key->ck_length,
	    ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
	printf("\tkey = { ");
	for (int i = 0; i < key->ck_length / 8; i++) {
		uint8_t *b = (uint8_t *)key->ck_data;
		printf("%02x ", b[i]);
	}
	printf("}\n");
	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
		printf("\tiovec #%d: <%p, %u>\n", i,
		    zfs_uio_iovbase(data_uio, i),
		    (unsigned int)zfs_uio_iovlen(data_uio, i));
		total += zfs_uio_iovlen(data_uio, i);
	}
	zfs_uio_resid(data_uio) = total;
#endif
}
/*
 * Create a new cryptographic session.  This should
 * happen every time the key changes (including when
 * it's first loaded).
 */
#if __FreeBSD_version >= 1300087
int
freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
    const struct zio_crypt_info *c_info, crypto_key_t *key)
{
	struct crypto_session_params csp;
	int error = 0;

#ifdef FCRYPTO_DEBUG
	printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
	    __FUNCTION__, sessp,
	    c_info->ci_algname, c_info->ci_crypt_type,
	    (unsigned int)c_info->ci_keylen, c_info->ci_name,
	    key->ck_format, key->ck_data, (unsigned int)key->ck_length);
	printf("\tkey = { ");
	for (int i = 0; i < key->ck_length / 8; i++) {
		uint8_t *b = (uint8_t *)key->ck_data;
		printf("%02x ", b[i]);
	}
	printf("}\n");
#endif
	bzero(&csp, sizeof (csp));
	csp.csp_mode = CSP_MODE_AEAD;
	csp.csp_cipher_key = key->ck_data;
	csp.csp_cipher_klen = key->ck_length / 8;
	switch (c_info->ci_crypt_type) {
		case ZC_TYPE_GCM:
		csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
		csp.csp_ivlen = AES_GCM_IV_LEN;
		switch (key->ck_length/8) {
		case AES_128_GMAC_KEY_LEN:
		case AES_192_GMAC_KEY_LEN:
		case AES_256_GMAC_KEY_LEN:
			break;
		default:
			error = EINVAL;
			goto bad;
		}
		break;
	case ZC_TYPE_CCM:
		csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
		csp.csp_ivlen = AES_CCM_IV_LEN;
		switch (key->ck_length/8) {
		case AES_128_CBC_MAC_KEY_LEN:
		case AES_192_CBC_MAC_KEY_LEN:
		case AES_256_CBC_MAC_KEY_LEN:
			break;
		default:
			error = EINVAL;
			goto bad;
			break;
		}
		break;
	default:
		error = ENOTSUP;
		goto bad;
	}

	/*
	 * Disable the use of hardware drivers on FreeBSD 13 and later since
	 * common crypto offload drivers impose constraints on AES-GCM AAD
	 * lengths that make them unusable for ZFS, and we currently do not have
	 * a mechanism to fall back to a software driver for requests not
	 * handled by a hardware driver.
	 *
	 * On 12 we continue to permit the use of hardware drivers since
	 * CPU-accelerated drivers such as aesni(4) register themselves as
	 * hardware drivers.
	 */
	error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
	    NULL, MTX_DEF);
	crypt_sessions++;
bad:
#ifdef FCRYPTO_DEBUG
	if (error)
		printf("%s: returning error %d\n", __FUNCTION__, error);
#endif
	return (error);
}

int
freebsd_crypt_uio(boolean_t encrypt,
    freebsd_crypt_session_t *input_sessionp,
    const struct zio_crypt_info *c_info,
    zfs_uio_t *data_uio,
    crypto_key_t *key,
    uint8_t *ivbuf,
    size_t datalen,
    size_t auth_len)
{
	struct cryptop *crp;
	freebsd_crypt_session_t *session = NULL;
	int error = 0;
	size_t total = 0;

	freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
	    key, ivbuf, datalen, auth_len);
	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
		total += zfs_uio_iovlen(data_uio, i);
	zfs_uio_resid(data_uio) = total;
	if (input_sessionp == NULL) {
		session = kmem_zalloc(sizeof (*session), KM_SLEEP);
		error = freebsd_crypt_newsession(session, c_info, key);
		if (error)
			goto out;
	} else
		session = input_sessionp;

	crp = crypto_getreq(session->fs_sid, M_WAITOK);
	if (encrypt) {
		crp->crp_op = CRYPTO_OP_ENCRYPT |
		    CRYPTO_OP_COMPUTE_DIGEST;
	} else {
		crp->crp_op = CRYPTO_OP_DECRYPT |
		    CRYPTO_OP_VERIFY_DIGEST;
	}
	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
	crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));

	crp->crp_aad_start = 0;
	crp->crp_aad_length = auth_len;
	crp->crp_payload_start = auth_len;
	crp->crp_payload_length = datalen;
	crp->crp_digest_start = auth_len + datalen;

	bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN);
	error = zfs_crypto_dispatch(session, crp);
	crypto_freereq(crp);
out:
#ifdef FCRYPTO_DEBUG
	if (error)
		printf("%s: returning error %d\n", __FUNCTION__, error);
#endif
	if (input_sessionp == NULL) {
		freebsd_crypt_freesession(session);
		kmem_free(session, sizeof (*session));
	}
	return (error);
}

#else
int
freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
    const struct zio_crypt_info *c_info, crypto_key_t *key)
{
	struct cryptoini cria, crie, *crip;
	struct enc_xform *xform;
	struct auth_hash *xauth;
	int error = 0;
	crypto_session_t sid;

#ifdef FCRYPTO_DEBUG
	printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
	    __FUNCTION__, sessp,
	    c_info->ci_algname, c_info->ci_crypt_type,
	    (unsigned int)c_info->ci_keylen, c_info->ci_name,
	    key->ck_format, key->ck_data, (unsigned int)key->ck_length);
	printf("\tkey = { ");
	for (int i = 0; i < key->ck_length / 8; i++) {
		uint8_t *b = (uint8_t *)key->ck_data;
		printf("%02x ", b[i]);
	}
	printf("}\n");
#endif
	switch (c_info->ci_crypt_type) {
	case ZC_TYPE_GCM:
		xform = &enc_xform_aes_nist_gcm;
		switch (key->ck_length/8) {
		case AES_128_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_128;
			break;
		case AES_192_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_192;
			break;
		case AES_256_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_256;
			break;
		default:
			error = EINVAL;
			goto bad;
		}
		break;
	case ZC_TYPE_CCM:
		xform = &enc_xform_ccm;
		switch (key->ck_length/8) {
		case AES_128_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_128;
			break;
		case AES_192_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_192;
			break;
		case AES_256_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_256;
			break;
		default:
			error = EINVAL;
			goto bad;
			break;
		}
		break;
	default:
		error = ENOTSUP;
		goto bad;
	}
#ifdef FCRYPTO_DEBUG
	printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
	    "auth %s (key length %d)\n",
	    __FUNCTION__, __LINE__,
	    xform->name, (unsigned int)key->ck_length,
	    (unsigned int)key->ck_length/8,
	    xauth->name, xauth->keysize);
#endif

	bzero(&crie, sizeof (crie));
	bzero(&cria, sizeof (cria));

	crie.cri_alg = xform->type;
	crie.cri_key = key->ck_data;
	crie.cri_klen = key->ck_length;

	cria.cri_alg = xauth->type;
	cria.cri_key = key->ck_data;
	cria.cri_klen = key->ck_length;

	cria.cri_next = &crie;
	crie.cri_next = NULL;
	crip = &cria;
	// Everything else is bzero'd

	error = crypto_newsession(&sid, crip,
	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
	if (error != 0) {
		printf("%s(%d):  crypto_newsession failed with %d\n",
		    __FUNCTION__, __LINE__, error);
		goto bad;
	}
	sessp->fs_sid = sid;
	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
	    NULL, MTX_DEF);
	crypt_sessions++;
bad:
	return (error);
}

/*
 * The meat of encryption/decryption.
 * If sessp is NULL, then it will create a
 * temporary cryptographic session, and release
 * it when done.
 */
int
freebsd_crypt_uio(boolean_t encrypt,
    freebsd_crypt_session_t *input_sessionp,
    const struct zio_crypt_info *c_info,
    zfs_uio_t *data_uio,
    crypto_key_t *key,
    uint8_t *ivbuf,
    size_t datalen,
    size_t auth_len)
{
	struct cryptop *crp;
	struct cryptodesc *enc_desc, *auth_desc;
	struct enc_xform *xform;
	struct auth_hash *xauth;
	freebsd_crypt_session_t *session = NULL;
	int error;

	freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
	    key, ivbuf, datalen, auth_len);
	switch (c_info->ci_crypt_type) {
	case ZC_TYPE_GCM:
		xform = &enc_xform_aes_nist_gcm;
		switch (key->ck_length/8) {
		case AES_128_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_128;
			break;
		case AES_192_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_192;
			break;
		case AES_256_GMAC_KEY_LEN:
			xauth = &auth_hash_nist_gmac_aes_256;
			break;
		default:
			error = EINVAL;
			goto bad;
		}
		break;
	case ZC_TYPE_CCM:
		xform = &enc_xform_ccm;
		switch (key->ck_length/8) {
		case AES_128_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_128;
			break;
		case AES_192_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_192;
			break;
		case AES_256_CBC_MAC_KEY_LEN:
			xauth = &auth_hash_ccm_cbc_mac_256;
			break;
		default:
			error = EINVAL;
			goto bad;
			break;
		}
		break;
	default:
		error = ENOTSUP;
		goto bad;
	}

#ifdef FCRYPTO_DEBUG
	printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
	    "auth %s (key length %d)\n",
	    __FUNCTION__, __LINE__,
	    xform->name, (unsigned int)key->ck_length,
	    (unsigned int)key->ck_length/8,
	    xauth->name, xauth->keysize);
#endif

	if (input_sessionp == NULL) {
		session = kmem_zalloc(sizeof (*session), KM_SLEEP);
		error = freebsd_crypt_newsession(session, c_info, key);
		if (error)
			goto out;
	} else
		session = input_sessionp;

	crp = crypto_getreq(2);
	if (crp == NULL) {
		error = ENOMEM;
		goto bad;
	}

	auth_desc = crp->crp_desc;
	enc_desc = auth_desc->crd_next;

	crp->crp_session = session->fs_sid;
	crp->crp_ilen = auth_len + datalen;
	crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio);
	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC;

	auth_desc->crd_skip = 0;
	auth_desc->crd_len = auth_len;
	auth_desc->crd_inject = auth_len + datalen;
	auth_desc->crd_alg = xauth->type;
#ifdef FCRYPTO_DEBUG
	printf("%s: auth: skip = %u, len = %u, inject = %u\n",
	    __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len,
	    auth_desc->crd_inject);
#endif

	enc_desc->crd_skip = auth_len;
	enc_desc->crd_len = datalen;
	enc_desc->crd_inject = auth_len;
	enc_desc->crd_alg = xform->type;
	enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
	bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN);
	enc_desc->crd_next = NULL;

#ifdef FCRYPTO_DEBUG
	printf("%s: enc: skip = %u, len = %u, inject = %u\n",
	    __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len,
	    enc_desc->crd_inject);
#endif

	if (encrypt)
		enc_desc->crd_flags |= CRD_F_ENCRYPT;

	error = zfs_crypto_dispatch(session, crp);
	crypto_freereq(crp);
out:
	if (input_sessionp == NULL) {
		freebsd_crypt_freesession(session);
		kmem_free(session, sizeof (*session));
	}
bad:
#ifdef FCRYPTO_DEBUG
	if (error)
		printf("%s: returning error %d\n", __FUNCTION__, error);
#endif
	return (error);
}
#endif
Commit	Line	Data
9f0a21e6 MM	1	/*
	2	* Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
	3	* Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
	4	* All rights reserved.
	5	*
	6	* Redistribution and use in source and binary forms, with or without
	7	* modification, are permitted provided that the following conditions
	8	* are met:
	9	* 1. Redistributions of source code must retain the above copyright
	10	* notice, this list of conditions and the following disclaimer.
	11	* 2. Redistributions in binary form must reproduce the above copyright
	12	* notice, this list of conditions and the following disclaimer in the
	13	* documentation and/or other materials provided with the distribution.
	14	*
	15	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	16	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	17	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	18	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	19	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	20	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	21	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	22	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	23	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	24	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	25	* SUCH DAMAGE.
	26	*
	27	* Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
	28	*/
	29
	30	#include <sys/cdefs.h>
	31	__FBSDID("$FreeBSD$");
	32
	33	#include <sys/types.h>
	34	#include <sys/errno.h>
	35
	36	#ifdef _KERNEL
	37	#include <sys/libkern.h>
	38	#include <sys/malloc.h>
	39	#include <sys/sysctl.h>
	40	#include <opencrypto/cryptodev.h>
	41	#include <opencrypto/xform.h>
	42	#else
	43	#include <strings.h>
	44	#endif
	45
	46	#include <sys/zio_crypt.h>
	47	#include <sys/fs/zfs.h>
	48	#include <sys/zio.h>
	49
	50	#include <sys/freebsd_crypto.h>
	51
	52	#define SHA512_HMAC_BLOCK_SIZE 128
	53
	54	static int crypt_sessions = 0;
	55	SYSCTL_DECL(_vfs_zfs);
	56	SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
	57	&crypt_sessions, 0, "Number of cryptographic sessions created");
	58
	59	void
	60	crypto_mac_init(struct hmac_ctx ctx, const crypto_key_t c_key)
	61	{
	62	uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
	63	k_opad[SHA512_HMAC_BLOCK_SIZE],
	64	key[SHA512_HMAC_BLOCK_SIZE];
65	SHA512_CTX lctx;
66	int i;
67	size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);
68
69	/*
70	* This code is based on the similar code in geom/eli/g_eli_hmac.c
71	*/
72	explicit_bzero(key, sizeof (key));
73	if (c_key->ck_length == 0)
74	/* do nothing */;
75	else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
76	bcopy(c_key->ck_data, key, cl_bytes);
77	else {
78	/*
79	* If key is longer than 128 bytes reset it to
80	* key = SHA512(key).
81	*/
82	SHA512_Init(&lctx);
83	SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
84	SHA512_Final(key, &lctx);
85	}
86
87	/* XOR key with ipad and opad values. */
88	for (i = 0; i < sizeof (key); i++) {
89	k_ipad[i] = key[i] ^ 0x36;
90	k_opad[i] = key[i] ^ 0x5c;
91	}
92	explicit_bzero(key, sizeof (key));
93
94	/* Start inner SHA512. */
95	SHA512_Init(&ctx->innerctx);
96	SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
97	explicit_bzero(k_ipad, sizeof (k_ipad));
98	/* Start outer SHA512. */
99	SHA512_Init(&ctx->outerctx);
100	SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
101	explicit_bzero(k_opad, sizeof (k_opad));
102	}
103
104	void
105	crypto_mac_update(struct hmac_ctx ctx, const void data, size_t datasize)
106	{
107	SHA512_Update(&ctx->innerctx, data, datasize);
108	}
109
110	void
111	crypto_mac_final(struct hmac_ctx ctx, void md, size_t mdsize)
112	{
113	uint8_t digest[SHA512_DIGEST_LENGTH];
114
115	/* Complete inner hash */
116	SHA512_Final(digest, &ctx->innerctx);
117
118	/* Complete outer hash */
119	SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
120	SHA512_Final(digest, &ctx->outerctx);
121
122	explicit_bzero(ctx, sizeof (*ctx));
123	/* mdsize == 0 means "Give me the whole hash!" */
124	if (mdsize == 0)
125	mdsize = SHA512_DIGEST_LENGTH;
126	bcopy(digest, md, mdsize);
127	explicit_bzero(digest, sizeof (digest));
128	}
129
130	void
131	crypto_mac(const crypto_key_t key, const void in_data, size_t in_data_size,
132	void *out_data, size_t out_data_size)
133	{
134	struct hmac_ctx ctx;
135
136	crypto_mac_init(&ctx, key);
137	crypto_mac_update(&ctx, in_data, in_data_size);
138	crypto_mac_final(&ctx, out_data, out_data_size);
139	}
140
141	static int
142	freebsd_zfs_crypt_done(struct cryptop *crp)
143	{
144	freebsd_crypt_session_t *ses;
145
146	ses = crp->crp_opaque;
147	mtx_lock(&ses->fs_lock);
148	ses->fs_done = true;
149	mtx_unlock(&ses->fs_lock);
150	wakeup(crp);
151	return (0);
152	}
153
154	void
155	freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
156	{
157	mtx_destroy(&sess->fs_lock);
158	crypto_freesession(sess->fs_sid);
159	explicit_bzero(sess, sizeof (*sess));
160	}
161
162	static int
163	zfs_crypto_dispatch(freebsd_crypt_session_t session, struct cryptop crp)
164	{
165	int error;
166
167	crp->crp_opaque = session;
168	crp->crp_callback = freebsd_zfs_crypt_done;
169	for (;;) {
170	error = crypto_dispatch(crp);
171	if (error)
172	break;
173	mtx_lock(&session->fs_lock);
174	while (session->fs_done == false)
f8646c87 AM	175	msleep(crp, &session->fs_lock, 0,
f8646c87 AM	176	"zfs_crypto", 0);
9f0a21e6 MM	177	mtx_unlock(&session->fs_lock);
9f0a21e6 MM	178
f8646c87 AM	179	if (crp->crp_etype == ENOMEM) {
	180	pause("zcrnomem", 1);
	181	} else if (crp->crp_etype != EAGAIN) {
9f0a21e6 MM	182	error = crp->crp_etype;
	183	break;
	184	}
	185	crp->crp_etype = 0;
	186	crp->crp_flags &= ~CRYPTO_F_DONE;
	187	session->fs_done = false;
	188	#if __FreeBSD_version < 1300087
	189	/*
	190	* Session ID changed, so we should record that,
	191	* and try again
	192	*/
	193	session->fs_sid = crp->crp_session;
	194	#endif
	195	}
	196	return (error);
	197	}
	198	static void
	199	freebsd_crypt_uio_debug_log(boolean_t encrypt,
	200	freebsd_crypt_session_t *input_sessionp,
18168da7	201	const struct zio_crypt_info *c_info,
d0cd9a5c	202	zfs_uio_t *data_uio,
9f0a21e6 MM	203	crypto_key_t *key,
	204	uint8_t *ivbuf,
	205	size_t datalen,
	206	size_t auth_len)
	207	{
	208	#ifdef FCRYPTO_DEBUG
	209	struct cryptodesc *crd;
	210	uint8_t *p = NULL;
	211	size_t total = 0;
	212
	213	printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, "
	214	"%p, %u, %u)\n",
	215	__FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
	216	c_info->ci_algname, c_info->ci_crypt_type,
	217	(unsigned int)c_info->ci_keylen, c_info->ci_name,
	218	data_uio, key->ck_format, key->ck_data,
	219	(unsigned int)key->ck_length,
	220	ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
	221	printf("\tkey = { ");
	222	for (int i = 0; i < key->ck_length / 8; i++) {
	223	uint8_t b = (uint8_t )key->ck_data;
	224	printf("%02x ", b[i]);
	225	}
	226	printf("}\n");
d0cd9a5c	227	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
9f0a21e6	228	printf("\tiovec #%d: <%p, %u>\n", i,
d0cd9a5c BA	229	zfs_uio_iovbase(data_uio, i),
	230	(unsigned int)zfs_uio_iovlen(data_uio, i));
	231	total += zfs_uio_iovlen(data_uio, i);
9f0a21e6	232	}
d0cd9a5c	233	zfs_uio_resid(data_uio) = total;
9f0a21e6 MM	234	#endif
	235	}
	236	/*
	237	* Create a new cryptographic session. This should
	238	* happen every time the key changes (including when
	239	* it's first loaded).
	240	*/
	241	#if __FreeBSD_version >= 1300087
	242	int
	243	freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
18168da7	244	const struct zio_crypt_info c_info, crypto_key_t key)
9f0a21e6 MM	245	{
	246	struct crypto_session_params csp;
	247	int error = 0;
	248
	249	#ifdef FCRYPTO_DEBUG
	250	printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
	251	__FUNCTION__, sessp,
	252	c_info->ci_algname, c_info->ci_crypt_type,
	253	(unsigned int)c_info->ci_keylen, c_info->ci_name,
	254	key->ck_format, key->ck_data, (unsigned int)key->ck_length);
	255	printf("\tkey = { ");
	256	for (int i = 0; i < key->ck_length / 8; i++) {
	257	uint8_t b = (uint8_t )key->ck_data;
	258	printf("%02x ", b[i]);
	259	}
	260	printf("}\n");
	261	#endif
	262	bzero(&csp, sizeof (csp));
	263	csp.csp_mode = CSP_MODE_AEAD;
	264	csp.csp_cipher_key = key->ck_data;
	265	csp.csp_cipher_klen = key->ck_length / 8;
	266	switch (c_info->ci_crypt_type) {
	267	case ZC_TYPE_GCM:
	268	csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
	269	csp.csp_ivlen = AES_GCM_IV_LEN;
	270	switch (key->ck_length/8) {
	271	case AES_128_GMAC_KEY_LEN:
	272	case AES_192_GMAC_KEY_LEN:
	273	case AES_256_GMAC_KEY_LEN:
	274	break;
	275	default:
	276	error = EINVAL;
	277	goto bad;
	278	}
	279	break;
	280	case ZC_TYPE_CCM:
	281	csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
	282	csp.csp_ivlen = AES_CCM_IV_LEN;
	283	switch (key->ck_length/8) {
	284	case AES_128_CBC_MAC_KEY_LEN:
	285	case AES_192_CBC_MAC_KEY_LEN:
	286	case AES_256_CBC_MAC_KEY_LEN:
	287	break;
	288	default:
	289	error = EINVAL;
	290	goto bad;
	291	break;
	292	}
	293	break;
	294	default:
	295	error = ENOTSUP;
	296	goto bad;
	297	}
e7adccf7 MJ	298
	299	/*
	300	* Disable the use of hardware drivers on FreeBSD 13 and later since
	301	* common crypto offload drivers impose constraints on AES-GCM AAD
	302	* lengths that make them unusable for ZFS, and we currently do not have
	303	* a mechanism to fall back to a software driver for requests not
	304	* handled by a hardware driver.
	305	*
	306	* On 12 we continue to permit the use of hardware drivers since
	307	* CPU-accelerated drivers such as aesni(4) register themselves as
	308	* hardware drivers.
	309	*/
	310	error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
9f0a21e6 MM	311	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
	312	NULL, MTX_DEF);
	313	crypt_sessions++;
	314	bad:
	315	#ifdef FCRYPTO_DEBUG
	316	if (error)
	317	printf("%s: returning error %d\n", __FUNCTION__, error);
	318	#endif
	319	return (error);
	320	}
	321
	322	int
	323	freebsd_crypt_uio(boolean_t encrypt,
	324	freebsd_crypt_session_t *input_sessionp,
18168da7	325	const struct zio_crypt_info *c_info,
d0cd9a5c	326	zfs_uio_t *data_uio,
9f0a21e6 MM	327	crypto_key_t *key,
	328	uint8_t *ivbuf,
	329	size_t datalen,
	330	size_t auth_len)
	331	{
	332	struct cryptop *crp;
	333	freebsd_crypt_session_t *session = NULL;
	334	int error = 0;
	335	size_t total = 0;
	336
	337	freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
	338	key, ivbuf, datalen, auth_len);
d0cd9a5c BA	339	for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
	340	total += zfs_uio_iovlen(data_uio, i);
	341	zfs_uio_resid(data_uio) = total;
9f0a21e6 MM	342	if (input_sessionp == NULL) {
	343	session = kmem_zalloc(sizeof (*session), KM_SLEEP);
	344	error = freebsd_crypt_newsession(session, c_info, key);
	345	if (error)
	346	goto out;
	347	} else
	348	session = input_sessionp;
	349
	350	crp = crypto_getreq(session->fs_sid, M_WAITOK);
	351	if (encrypt) {
	352	crp->crp_op = CRYPTO_OP_ENCRYPT \|
	353	CRYPTO_OP_COMPUTE_DIGEST;
	354	} else {
	355	crp->crp_op = CRYPTO_OP_DECRYPT \|
	356	CRYPTO_OP_VERIFY_DIGEST;
	357	}
	358	crp->crp_flags = CRYPTO_F_CBIFSYNC \| CRYPTO_F_IV_SEPARATE;
d0cd9a5c	359	crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));
9f0a21e6 MM	360
	361	crp->crp_aad_start = 0;
	362	crp->crp_aad_length = auth_len;
	363	crp->crp_payload_start = auth_len;
	364	crp->crp_payload_length = datalen;
	365	crp->crp_digest_start = auth_len + datalen;
	366
	367	bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN);
	368	error = zfs_crypto_dispatch(session, crp);
	369	crypto_freereq(crp);
	370	out:
	371	#ifdef FCRYPTO_DEBUG
	372	if (error)
	373	printf("%s: returning error %d\n", __FUNCTION__, error);
	374	#endif
	375	if (input_sessionp == NULL) {
	376	freebsd_crypt_freesession(session);
	377	kmem_free(session, sizeof (*session));
	378	}
	379	return (error);
	380	}
	381
	382	#else
	383	int
	384	freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
18168da7	385	const struct zio_crypt_info c_info, crypto_key_t key)
9f0a21e6 MM	386	{
	387	struct cryptoini cria, crie, *crip;
	388	struct enc_xform *xform;
	389	struct auth_hash *xauth;
	390	int error = 0;
	391	crypto_session_t sid;
	392
	393	#ifdef FCRYPTO_DEBUG
	394	printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n",
	395	__FUNCTION__, sessp,
	396	c_info->ci_algname, c_info->ci_crypt_type,
	397	(unsigned int)c_info->ci_keylen, c_info->ci_name,
	398	key->ck_format, key->ck_data, (unsigned int)key->ck_length);
	399	printf("\tkey = { ");
	400	for (int i = 0; i < key->ck_length / 8; i++) {
	401	uint8_t b = (uint8_t )key->ck_data;
	402	printf("%02x ", b[i]);
	403	}
	404	printf("}\n");
	405	#endif
	406	switch (c_info->ci_crypt_type) {
	407	case ZC_TYPE_GCM:
	408	xform = &enc_xform_aes_nist_gcm;
	409	switch (key->ck_length/8) {
	410	case AES_128_GMAC_KEY_LEN:
	411	xauth = &auth_hash_nist_gmac_aes_128;
	412	break;
	413	case AES_192_GMAC_KEY_LEN:
	414	xauth = &auth_hash_nist_gmac_aes_192;
	415	break;
	416	case AES_256_GMAC_KEY_LEN:
	417	xauth = &auth_hash_nist_gmac_aes_256;
	418	break;
	419	default:
	420	error = EINVAL;
	421	goto bad;
	422	}
	423	break;
	424	case ZC_TYPE_CCM:
	425	xform = &enc_xform_ccm;
	426	switch (key->ck_length/8) {
	427	case AES_128_CBC_MAC_KEY_LEN:
	428	xauth = &auth_hash_ccm_cbc_mac_128;
	429	break;
	430	case AES_192_CBC_MAC_KEY_LEN:
	431	xauth = &auth_hash_ccm_cbc_mac_192;
	432	break;
	433	case AES_256_CBC_MAC_KEY_LEN:
	434	xauth = &auth_hash_ccm_cbc_mac_256;
	435	break;
	436	default:
	437	error = EINVAL;
	438	goto bad;
	439	break;
	440	}
	441	break;
	442	default:
	443	error = ENOTSUP;
	444	goto bad;
	445	}
	446	#ifdef FCRYPTO_DEBUG
	447	printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
	448	"auth %s (key length %d)\n",
	449	__FUNCTION__, __LINE__,
450	xform->name, (unsigned int)key->ck_length,
451	(unsigned int)key->ck_length/8,
452	xauth->name, xauth->keysize);
453	#endif
454
455	bzero(&crie, sizeof (crie));
456	bzero(&cria, sizeof (cria));
457
458	crie.cri_alg = xform->type;
459	crie.cri_key = key->ck_data;
460	crie.cri_klen = key->ck_length;
461
462	cria.cri_alg = xauth->type;
463	cria.cri_key = key->ck_data;
464	cria.cri_klen = key->ck_length;
465
466	cria.cri_next = &crie;
467	crie.cri_next = NULL;
468	crip = &cria;
469	// Everything else is bzero'd
470
471	error = crypto_newsession(&sid, crip,
472	CRYPTOCAP_F_HARDWARE \| CRYPTOCAP_F_SOFTWARE);
473	if (error != 0) {
474	printf("%s(%d): crypto_newsession failed with %d\n",
475	__FUNCTION__, __LINE__, error);
476	goto bad;
477	}
478	sessp->fs_sid = sid;
479	mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
480	NULL, MTX_DEF);
481	crypt_sessions++;
482	bad:
483	return (error);
484	}
485
486	/*
487	* The meat of encryption/decryption.
488	* If sessp is NULL, then it will create a
489	* temporary cryptographic session, and release
490	* it when done.
491	*/
492	int
493	freebsd_crypt_uio(boolean_t encrypt,
494	freebsd_crypt_session_t *input_sessionp,
18168da7	495	const struct zio_crypt_info *c_info,
d0cd9a5c	496	zfs_uio_t *data_uio,
9f0a21e6 MM	497	crypto_key_t *key,
	498	uint8_t *ivbuf,
	499	size_t datalen,
	500	size_t auth_len)
	501	{
	502	struct cryptop *crp;
	503	struct cryptodesc enc_desc, auth_desc;
	504	struct enc_xform *xform;
	505	struct auth_hash *xauth;
	506	freebsd_crypt_session_t *session = NULL;
	507	int error;
	508
	509	freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
	510	key, ivbuf, datalen, auth_len);
	511	switch (c_info->ci_crypt_type) {
	512	case ZC_TYPE_GCM:
	513	xform = &enc_xform_aes_nist_gcm;
	514	switch (key->ck_length/8) {
	515	case AES_128_GMAC_KEY_LEN:
	516	xauth = &auth_hash_nist_gmac_aes_128;
	517	break;
	518	case AES_192_GMAC_KEY_LEN:
	519	xauth = &auth_hash_nist_gmac_aes_192;
	520	break;
	521	case AES_256_GMAC_KEY_LEN:
	522	xauth = &auth_hash_nist_gmac_aes_256;
	523	break;
	524	default:
	525	error = EINVAL;
	526	goto bad;
	527	}
	528	break;
	529	case ZC_TYPE_CCM:
	530	xform = &enc_xform_ccm;
	531	switch (key->ck_length/8) {
	532	case AES_128_CBC_MAC_KEY_LEN:
	533	xauth = &auth_hash_ccm_cbc_mac_128;
	534	break;
	535	case AES_192_CBC_MAC_KEY_LEN:
	536	xauth = &auth_hash_ccm_cbc_mac_192;
	537	break;
	538	case AES_256_CBC_MAC_KEY_LEN:
	539	xauth = &auth_hash_ccm_cbc_mac_256;
	540	break;
	541	default:
	542	error = EINVAL;
	543	goto bad;
	544	break;
	545	}
	546	break;
	547	default:
	548	error = ENOTSUP;
	549	goto bad;
	550	}
	551
	552	#ifdef FCRYPTO_DEBUG
	553	printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
	554	"auth %s (key length %d)\n",
	555	__FUNCTION__, __LINE__,
	556	xform->name, (unsigned int)key->ck_length,
	557	(unsigned int)key->ck_length/8,
	558	xauth->name, xauth->keysize);
	559	#endif
	560
561	if (input_sessionp == NULL) {
562	session = kmem_zalloc(sizeof (*session), KM_SLEEP);
563	error = freebsd_crypt_newsession(session, c_info, key);
564	if (error)
565	goto out;
566	} else
567	session = input_sessionp;
568
569	crp = crypto_getreq(2);
570	if (crp == NULL) {
571	error = ENOMEM;
572	goto bad;
573	}
574
575	auth_desc = crp->crp_desc;
576	enc_desc = auth_desc->crd_next;
577
578	crp->crp_session = session->fs_sid;
579	crp->crp_ilen = auth_len + datalen;
d0cd9a5c	580	crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio);
9f0a21e6 MM	581	crp->crp_flags = CRYPTO_F_IOV \| CRYPTO_F_CBIFSYNC;
	582
	583	auth_desc->crd_skip = 0;
	584	auth_desc->crd_len = auth_len;
	585	auth_desc->crd_inject = auth_len + datalen;
	586	auth_desc->crd_alg = xauth->type;
	587	#ifdef FCRYPTO_DEBUG
	588	printf("%s: auth: skip = %u, len = %u, inject = %u\n",
	589	__FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len,
	590	auth_desc->crd_inject);
	591	#endif
	592
	593	enc_desc->crd_skip = auth_len;
	594	enc_desc->crd_len = datalen;
	595	enc_desc->crd_inject = auth_len;
	596	enc_desc->crd_alg = xform->type;
	597	enc_desc->crd_flags = CRD_F_IV_EXPLICIT \| CRD_F_IV_PRESENT;
	598	bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN);
	599	enc_desc->crd_next = NULL;
	600
	601	#ifdef FCRYPTO_DEBUG
	602	printf("%s: enc: skip = %u, len = %u, inject = %u\n",
	603	__FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len,
	604	enc_desc->crd_inject);
	605	#endif
	606
	607	if (encrypt)
	608	enc_desc->crd_flags \|= CRD_F_ENCRYPT;
	609
	610	error = zfs_crypto_dispatch(session, crp);
	611	crypto_freereq(crp);
	612	out:
	613	if (input_sessionp == NULL) {
	614	freebsd_crypt_freesession(session);
	615	kmem_free(session, sizeof (*session));
	616	}
	617	bad:
	618	#ifdef FCRYPTO_DEBUG
	619	if (error)
	620	printf("%s: returning error %d\n", __FUNCTION__, error);
	621	#endif
	622	return (error);
	623	}
	624	#endif