module/os/freebsd/zfs/crypto_os.c

   1 /*
   2  * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/types.h>
  34 #include <sys/errno.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/libkern.h>
  38 #include <sys/malloc.h>
  39 #include <sys/sysctl.h>
  40 #include <opencrypto/cryptodev.h>
  41 #include <opencrypto/xform.h>
  42 #else
  43 #include <strings.h>
  44 #endif
  45
  46 #include <sys/zio_crypt.h>
  47 #include <sys/fs/zfs.h>
  48 #include <sys/zio.h>
  49
  50 #include <sys/freebsd_crypto.h>
  51
  52 #define SHA512_HMAC_BLOCK_SIZE  128
  53
  54 static int crypt_sessions = 0;
  55 SYSCTL_DECL(_vfs_zfs);
  56 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
  57         &crypt_sessions, 0, "Number of cryptographic sessions created");
  58
  59 void
  60 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key)
  61 {
  62         uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
  63             k_opad[SHA512_HMAC_BLOCK_SIZE],
  64             key[SHA512_HMAC_BLOCK_SIZE];
  65         SHA512_CTX lctx;
  66         int i;
  67         size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);
  68
  69         /*
  70          * This code is based on the similar code in geom/eli/g_eli_hmac.c
  71          */
  72         memset(key, 0, sizeof (key));
  73         if (c_key->ck_length  == 0)
  74                 /* do nothing */;
  75         else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
  76                 memcpy(key, c_key->ck_data, cl_bytes);
  77         else {
  78                 /*
  79                  * If key is longer than 128 bytes reset it to
  80                  * key = SHA512(key).
  81                  */
  82                 SHA512_Init(&lctx);
  83                 SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
  84                 SHA512_Final(key, &lctx);
  85         }
  86
  87         /* XOR key with ipad and opad values. */
  88         for (i = 0; i < sizeof (key); i++) {
  89                 k_ipad[i] = key[i] ^ 0x36;
  90                 k_opad[i] = key[i] ^ 0x5c;
  91         }
  92         memset(key, 0, sizeof (key));
  93
  94         /* Start inner SHA512. */
  95         SHA512_Init(&ctx->innerctx);
  96         SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
  97         memset(k_ipad, 0, sizeof (k_ipad));
  98         /* Start outer SHA512. */
  99         SHA512_Init(&ctx->outerctx);
 100         SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
 101         memset(k_opad, 0, sizeof (k_opad));
 102 }
 103
 104 void
 105 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize)
 106 {
 107         SHA512_Update(&ctx->innerctx, data, datasize);
 108 }
 109
 110 void
 111 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize)
 112 {
 113         uint8_t digest[SHA512_DIGEST_LENGTH];
 114
 115         /* Complete inner hash */
 116         SHA512_Final(digest, &ctx->innerctx);
 117
 118         /* Complete outer hash */
 119         SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
 120         SHA512_Final(digest, &ctx->outerctx);
 121
 122         memset(ctx, 0, sizeof (*ctx));
 123         /* mdsize == 0 means "Give me the whole hash!" */
 124         if (mdsize == 0)
 125                 mdsize = SHA512_DIGEST_LENGTH;
 126         memcpy(md, digest, mdsize);
 127         memset(digest, 0, sizeof (digest));
 128 }
 129
 130 void
 131 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size,
 132     void *out_data, size_t out_data_size)
 133 {
 134         struct hmac_ctx ctx;
 135
 136         crypto_mac_init(&ctx, key);
 137         crypto_mac_update(&ctx, in_data, in_data_size);
 138         crypto_mac_final(&ctx, out_data, out_data_size);
 139 }
 140
 141 static int
 142 freebsd_zfs_crypt_done(struct cryptop *crp)
 143 {
 144         freebsd_crypt_session_t *ses;
 145
 146         ses = crp->crp_opaque;
 147         mtx_lock(&ses->fs_lock);
 148         ses->fs_done = true;
 149         mtx_unlock(&ses->fs_lock);
 150         wakeup(crp);
 151         return (0);
 152 }
 153
 154 void
 155 freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
 156 {
 157         mtx_destroy(&sess->fs_lock);
 158         crypto_freesession(sess->fs_sid);
 159         memset(sess, 0, sizeof (*sess));
 160 }
 161
 162 static int
 163 zfs_crypto_dispatch(freebsd_crypt_session_t *session,   struct cryptop *crp)
 164 {
 165         int error;
 166
 167         crp->crp_opaque = session;
 168         crp->crp_callback = freebsd_zfs_crypt_done;
 169         for (;;) {
 170                 error = crypto_dispatch(crp);
 171                 if (error)
 172                         break;
 173                 mtx_lock(&session->fs_lock);
 174                 while (session->fs_done == false)
 175                         msleep(crp, &session->fs_lock, 0,
 176                             "zfs_crypto", 0);
 177                 mtx_unlock(&session->fs_lock);
 178
 179                 if (crp->crp_etype == ENOMEM) {
 180                         pause("zcrnomem", 1);
 181                 } else if (crp->crp_etype != EAGAIN) {
 182                         error = crp->crp_etype;
 183                         break;
 184                 }
 185                 crp->crp_etype = 0;
 186                 crp->crp_flags &= ~CRYPTO_F_DONE;
 187                 session->fs_done = false;
 188 #if __FreeBSD_version < 1300087
 189                 /*
 190                  * Session ID changed, so we should record that,
 191                  * and try again
 192                  */
 193                 session->fs_sid = crp->crp_session;
 194 #endif
 195         }
 196         return (error);
 197 }
 198 static void
 199 freebsd_crypt_uio_debug_log(boolean_t encrypt,
 200     freebsd_crypt_session_t *input_sessionp,
 201     const struct zio_crypt_info *c_info,
 202     zfs_uio_t *data_uio,
 203     crypto_key_t *key,
 204     uint8_t *ivbuf,
 205     size_t datalen,
 206     size_t auth_len)
 207 {
 208 #ifdef FCRYPTO_DEBUG
 209         struct cryptodesc *crd;
 210         uint8_t *p = NULL;
 211         size_t total = 0;
 212
 213         printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, "
 214             "%p, %u, %u)\n",
 215             __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
 216             c_info->ci_algname, c_info->ci_crypt_type,
 217             (unsigned int)c_info->ci_keylen, c_info->ci_name,
 218             data_uio, key->ck_data,
 219             (unsigned int)key->ck_length,
 220             ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
 221         printf("\tkey = { ");
 222         for (int i = 0; i < key->ck_length / 8; i++) {
 223                 uint8_t *b = (uint8_t *)key->ck_data;
 224                 printf("%02x ", b[i]);
 225         }
 226         printf("}\n");
 227         for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
 228                 printf("\tiovec #%d: <%p, %u>\n", i,
 229                     zfs_uio_iovbase(data_uio, i),
 230                     (unsigned int)zfs_uio_iovlen(data_uio, i));
 231                 total += zfs_uio_iovlen(data_uio, i);
 232         }
 233         zfs_uio_resid(data_uio) = total;
 234 #endif
 235 }
 236 /*
 237  * Create a new cryptographic session.  This should
 238  * happen every time the key changes (including when
 239  * it's first loaded).
 240  */
 241 #if __FreeBSD_version >= 1300087
 242 int
 243 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 244     const struct zio_crypt_info *c_info, crypto_key_t *key)
 245 {
 246         struct crypto_session_params csp = {0};
 247         int error = 0;
 248
 249 #ifdef FCRYPTO_DEBUG
 250         printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n",
 251             __FUNCTION__, sessp,
 252             c_info->ci_algname, c_info->ci_crypt_type,
 253             (unsigned int)c_info->ci_keylen, c_info->ci_name,
 254             key->ck_data, (unsigned int)key->ck_length);
 255         printf("\tkey = { ");
 256         for (int i = 0; i < key->ck_length / 8; i++) {
 257                 uint8_t *b = (uint8_t *)key->ck_data;
 258                 printf("%02x ", b[i]);
 259         }
 260         printf("}\n");
 261 #endif
 262         csp.csp_mode = CSP_MODE_AEAD;
 263         csp.csp_cipher_key = key->ck_data;
 264         csp.csp_cipher_klen = key->ck_length / 8;
 265         switch (c_info->ci_crypt_type) {
 266                 case ZC_TYPE_GCM:
 267                 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
 268                 csp.csp_ivlen = AES_GCM_IV_LEN;
 269                 switch (key->ck_length/8) {
 270                 case AES_128_GMAC_KEY_LEN:
 271                 case AES_192_GMAC_KEY_LEN:
 272                 case AES_256_GMAC_KEY_LEN:
 273                         break;
 274                 default:
 275                         error = EINVAL;
 276                         goto bad;
 277                 }
 278                 break;
 279         case ZC_TYPE_CCM:
 280                 csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
 281                 csp.csp_ivlen = AES_CCM_IV_LEN;
 282                 switch (key->ck_length/8) {
 283                 case AES_128_CBC_MAC_KEY_LEN:
 284                 case AES_192_CBC_MAC_KEY_LEN:
 285                 case AES_256_CBC_MAC_KEY_LEN:
 286                         break;
 287                 default:
 288                         error = EINVAL;
 289                         goto bad;
 290                         break;
 291                 }
 292                 break;
 293         default:
 294                 error = ENOTSUP;
 295                 goto bad;
 296         }
 297
 298         /*
 299          * Disable the use of hardware drivers on FreeBSD 13 and later since
 300          * common crypto offload drivers impose constraints on AES-GCM AAD
 301          * lengths that make them unusable for ZFS, and we currently do not have
 302          * a mechanism to fall back to a software driver for requests not
 303          * handled by a hardware driver.
 304          *
 305          * On 12 we continue to permit the use of hardware drivers since
 306          * CPU-accelerated drivers such as aesni(4) register themselves as
 307          * hardware drivers.
 308          */
 309         error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
 310         mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
 311             NULL, MTX_DEF);
 312         crypt_sessions++;
 313 bad:
 314 #ifdef FCRYPTO_DEBUG
 315         if (error)
 316                 printf("%s: returning error %d\n", __FUNCTION__, error);
 317 #endif
 318         return (error);
 319 }
 320
 321 int
 322 freebsd_crypt_uio(boolean_t encrypt,
 323     freebsd_crypt_session_t *input_sessionp,
 324     const struct zio_crypt_info *c_info,
 325     zfs_uio_t *data_uio,
 326     crypto_key_t *key,
 327     uint8_t *ivbuf,
 328     size_t datalen,
 329     size_t auth_len)
 330 {
 331         struct cryptop *crp;
 332         freebsd_crypt_session_t *session = NULL;
 333         int error = 0;
 334         size_t total = 0;
 335
 336         freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
 337             key, ivbuf, datalen, auth_len);
 338         for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
 339                 total += zfs_uio_iovlen(data_uio, i);
 340         zfs_uio_resid(data_uio) = total;
 341         if (input_sessionp == NULL) {
 342                 session = kmem_zalloc(sizeof (*session), KM_SLEEP);
 343                 error = freebsd_crypt_newsession(session, c_info, key);
 344                 if (error)
 345                         goto out;
 346         } else
 347                 session = input_sessionp;
 348
 349         crp = crypto_getreq(session->fs_sid, M_WAITOK);
 350         if (encrypt) {
 351                 crp->crp_op = CRYPTO_OP_ENCRYPT |
 352                     CRYPTO_OP_COMPUTE_DIGEST;
 353         } else {
 354                 crp->crp_op = CRYPTO_OP_DECRYPT |
 355                     CRYPTO_OP_VERIFY_DIGEST;
 356         }
 357         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
 358         crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));
 359
 360         crp->crp_aad_start = 0;
 361         crp->crp_aad_length = auth_len;
 362         crp->crp_payload_start = auth_len;
 363         crp->crp_payload_length = datalen;
 364         crp->crp_digest_start = auth_len + datalen;
 365
 366         memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN);
 367         error = zfs_crypto_dispatch(session, crp);
 368         crypto_freereq(crp);
 369 out:
 370 #ifdef FCRYPTO_DEBUG
 371         if (error)
 372                 printf("%s: returning error %d\n", __FUNCTION__, error);
 373 #endif
 374         if (input_sessionp == NULL) {
 375                 freebsd_crypt_freesession(session);
 376                 kmem_free(session, sizeof (*session));
 377         }
 378         return (error);
 379 }
 380
 381 #else
 382 int
 383 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 384     const struct zio_crypt_info *c_info, crypto_key_t *key)
 385 {
 386         struct cryptoini cria = {0}, crie = {0}, *crip;
 387         struct enc_xform *xform;
 388         struct auth_hash *xauth;
 389         int error = 0;
 390         crypto_session_t sid;
 391
 392 #ifdef FCRYPTO_DEBUG
 393         printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n",
 394             __FUNCTION__, sessp,
 395             c_info->ci_algname, c_info->ci_crypt_type,
 396             (unsigned int)c_info->ci_keylen, c_info->ci_name,
 397             key->ck_data, (unsigned int)key->ck_length);
 398         printf("\tkey = { ");
 399         for (int i = 0; i < key->ck_length / 8; i++) {
 400                 uint8_t *b = (uint8_t *)key->ck_data;
 401                 printf("%02x ", b[i]);
 402         }
 403         printf("}\n");
 404 #endif
 405         switch (c_info->ci_crypt_type) {
 406         case ZC_TYPE_GCM:
 407                 xform = &enc_xform_aes_nist_gcm;
 408                 switch (key->ck_length/8) {
 409                 case AES_128_GMAC_KEY_LEN:
 410                         xauth = &auth_hash_nist_gmac_aes_128;
 411                         break;
 412                 case AES_192_GMAC_KEY_LEN:
 413                         xauth = &auth_hash_nist_gmac_aes_192;
 414                         break;
 415                 case AES_256_GMAC_KEY_LEN:
 416                         xauth = &auth_hash_nist_gmac_aes_256;
 417                         break;
 418                 default:
 419                         error = EINVAL;
 420                         goto bad;
 421                 }
 422                 break;
 423         case ZC_TYPE_CCM:
 424                 xform = &enc_xform_ccm;
 425                 switch (key->ck_length/8) {
 426                 case AES_128_CBC_MAC_KEY_LEN:
 427                         xauth = &auth_hash_ccm_cbc_mac_128;
 428                         break;
 429                 case AES_192_CBC_MAC_KEY_LEN:
 430                         xauth = &auth_hash_ccm_cbc_mac_192;
 431                         break;
 432                 case AES_256_CBC_MAC_KEY_LEN:
 433                         xauth = &auth_hash_ccm_cbc_mac_256;
 434                         break;
 435                 default:
 436                         error = EINVAL;
 437                         goto bad;
 438                         break;
 439                 }
 440                 break;
 441         default:
 442                 error = ENOTSUP;
 443                 goto bad;
 444         }
 445 #ifdef FCRYPTO_DEBUG
 446         printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
 447             "auth %s (key length %d)\n",
 448             __FUNCTION__, __LINE__,
 449             xform->name, (unsigned int)key->ck_length,
 450             (unsigned int)key->ck_length/8,
 451             xauth->name, xauth->keysize);
 452 #endif
 453
 454         crie.cri_alg = xform->type;
 455         crie.cri_key = key->ck_data;
 456         crie.cri_klen = key->ck_length;
 457
 458         cria.cri_alg = xauth->type;
 459         cria.cri_key = key->ck_data;
 460         cria.cri_klen = key->ck_length;
 461
 462         cria.cri_next = &crie;
 463         crie.cri_next = NULL;
 464         crip = &cria;
 465         // Everything else is zero-initialised
 466
 467         error = crypto_newsession(&sid, crip,
 468             CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 469         if (error != 0) {
 470                 printf("%s(%d):  crypto_newsession failed with %d\n",
 471                     __FUNCTION__, __LINE__, error);
 472                 goto bad;
 473         }
 474         sessp->fs_sid = sid;
 475         mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
 476             NULL, MTX_DEF);
 477         crypt_sessions++;
 478 bad:
 479         return (error);
 480 }
 481
 482 /*
 483  * The meat of encryption/decryption.
 484  * If sessp is NULL, then it will create a
 485  * temporary cryptographic session, and release
 486  * it when done.
 487  */
 488 int
 489 freebsd_crypt_uio(boolean_t encrypt,
 490     freebsd_crypt_session_t *input_sessionp,
 491     const struct zio_crypt_info *c_info,
 492     zfs_uio_t *data_uio,
 493     crypto_key_t *key,
 494     uint8_t *ivbuf,
 495     size_t datalen,
 496     size_t auth_len)
 497 {
 498         struct cryptop *crp;
 499         struct cryptodesc *enc_desc, *auth_desc;
 500         struct enc_xform *xform;
 501         struct auth_hash *xauth;
 502         freebsd_crypt_session_t *session = NULL;
 503         int error;
 504
 505         freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
 506             key, ivbuf, datalen, auth_len);
 507         switch (c_info->ci_crypt_type) {
 508         case ZC_TYPE_GCM:
 509                 xform = &enc_xform_aes_nist_gcm;
 510                 switch (key->ck_length/8) {
 511                 case AES_128_GMAC_KEY_LEN:
 512                         xauth = &auth_hash_nist_gmac_aes_128;
 513                         break;
 514                 case AES_192_GMAC_KEY_LEN:
 515                         xauth = &auth_hash_nist_gmac_aes_192;
 516                         break;
 517                 case AES_256_GMAC_KEY_LEN:
 518                         xauth = &auth_hash_nist_gmac_aes_256;
 519                         break;
 520                 default:
 521                         error = EINVAL;
 522                         goto bad;
 523                 }
 524                 break;
 525         case ZC_TYPE_CCM:
 526                 xform = &enc_xform_ccm;
 527                 switch (key->ck_length/8) {
 528                 case AES_128_CBC_MAC_KEY_LEN:
 529                         xauth = &auth_hash_ccm_cbc_mac_128;
 530                         break;
 531                 case AES_192_CBC_MAC_KEY_LEN:
 532                         xauth = &auth_hash_ccm_cbc_mac_192;
 533                         break;
 534                 case AES_256_CBC_MAC_KEY_LEN:
 535                         xauth = &auth_hash_ccm_cbc_mac_256;
 536                         break;
 537                 default:
 538                         error = EINVAL;
 539                         goto bad;
 540                         break;
 541                 }
 542                 break;
 543         default:
 544                 error = ENOTSUP;
 545                 goto bad;
 546         }
 547
 548 #ifdef FCRYPTO_DEBUG
 549         printf("%s(%d): Using crypt %s (key length %u [%u bytes]), "
 550             "auth %s (key length %d)\n",
 551             __FUNCTION__, __LINE__,
 552             xform->name, (unsigned int)key->ck_length,
 553             (unsigned int)key->ck_length/8,
 554             xauth->name, xauth->keysize);
 555 #endif
 556
 557         if (input_sessionp == NULL) {
 558                 session = kmem_zalloc(sizeof (*session), KM_SLEEP);
 559                 error = freebsd_crypt_newsession(session, c_info, key);
 560                 if (error)
 561                         goto out;
 562         } else
 563                 session = input_sessionp;
 564
 565         crp = crypto_getreq(2);
 566         if (crp == NULL) {
 567                 error = ENOMEM;
 568                 goto bad;
 569         }
 570
 571         auth_desc = crp->crp_desc;
 572         enc_desc = auth_desc->crd_next;
 573
 574         crp->crp_session = session->fs_sid;
 575         crp->crp_ilen = auth_len + datalen;
 576         crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio);
 577         crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC;
 578
 579         auth_desc->crd_skip = 0;
 580         auth_desc->crd_len = auth_len;
 581         auth_desc->crd_inject = auth_len + datalen;
 582         auth_desc->crd_alg = xauth->type;
 583 #ifdef FCRYPTO_DEBUG
 584         printf("%s: auth: skip = %u, len = %u, inject = %u\n",
 585             __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len,
 586             auth_desc->crd_inject);
 587 #endif
 588
 589         enc_desc->crd_skip = auth_len;
 590         enc_desc->crd_len = datalen;
 591         enc_desc->crd_inject = auth_len;
 592         enc_desc->crd_alg = xform->type;
 593         enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 594         memcpy(enc_desc->crd_iv, ivbuf, ZIO_DATA_IV_LEN);
 595         enc_desc->crd_next = NULL;
 596
 597 #ifdef FCRYPTO_DEBUG
 598         printf("%s: enc: skip = %u, len = %u, inject = %u\n",
 599             __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len,
 600             enc_desc->crd_inject);
 601 #endif
 602
 603         if (encrypt)
 604                 enc_desc->crd_flags |= CRD_F_ENCRYPT;
 605
 606         error = zfs_crypto_dispatch(session, crp);
 607         crypto_freereq(crp);
 608 out:
 609         if (input_sessionp == NULL) {
 610                 freebsd_crypt_freesession(session);
 611                 kmem_free(session, sizeof (*session));
 612         }
 613 bad:
 614 #ifdef FCRYPTO_DEBUG
 615         if (error)
 616                 printf("%s: returning error %d\n", __FUNCTION__, error);
 617 #endif
 618         return (error);
 619 }
 620 #endif