module/zfs/qat_crypt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * This file represents the QAT implementation of checksums and encryption.
  24  * Internally, QAT shares the same cryptographic instances for both of these
  25  * operations, so the code has been combined here. QAT data compression uses
  26  * compression instances, so that code is separated into qat_compress.c
  27  */
  28
  29 #if defined(_KERNEL) && defined(HAVE_QAT)
  30 #include <linux/slab.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/completion.h>
  34 #include <sys/zfs_context.h>
  35 #include <sys/zio_crypt.h>
  36 #include "lac/cpa_cy_im.h"
  37 #include "lac/cpa_cy_common.h"
  38 #include "qat.h"
  39
  40 /*
  41  * Max instances in a QAT device, each instance is a channel to submit
  42  * jobs to QAT hardware, this is only for pre-allocating instances
  43  * and session arrays; the actual number of instances are defined in
  44  * the QAT driver's configure file.
  45  */
  46 #define QAT_CRYPT_MAX_INSTANCES         48
  47
  48 #define MAX_PAGE_NUM                    1024
  49
  50 static Cpa32U inst_num = 0;
  51 static Cpa16U num_inst = 0;
  52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
  53 static boolean_t qat_crypt_init_done = B_FALSE;
  54 int zfs_qat_encrypt_disable = 0;
  55 int zfs_qat_checksum_disable = 0;
  56
  57 typedef struct cy_callback {
  58         CpaBoolean verify_result;
  59         struct completion complete;
  60 } cy_callback_t;
  61
  62 static void
  63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
  64     void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
  65 {
  66         cy_callback_t *cb = p_callback;
  67
  68         if (cb != NULL) {
  69                 /* indicate that the function has been called */
  70                 cb->verify_result = verify;
  71                 complete(&cb->complete);
  72         }
  73 }
  74
  75 boolean_t
  76 qat_crypt_use_accel(size_t s_len)
  77 {
  78         return (!zfs_qat_encrypt_disable &&
  79             qat_crypt_init_done &&
  80             s_len >= QAT_MIN_BUF_SIZE &&
  81             s_len <= QAT_MAX_BUF_SIZE);
  82 }
  83
  84 boolean_t
  85 qat_checksum_use_accel(size_t s_len)
  86 {
  87         return (!zfs_qat_checksum_disable &&
  88             qat_crypt_init_done &&
  89             s_len >= QAT_MIN_BUF_SIZE &&
  90             s_len <= QAT_MAX_BUF_SIZE);
  91 }
  92
  93 void
  94 qat_crypt_clean(void)
  95 {
  96         for (Cpa16U i = 0; i < num_inst; i++)
  97                 cpaCyStopInstance(cy_inst_handles[i]);
  98
  99         num_inst = 0;
 100         qat_crypt_init_done = B_FALSE;
 101 }
 102
 103 int
 104 qat_crypt_init(void)
 105 {
 106         CpaStatus status = CPA_STATUS_FAIL;
 107
 108         status = cpaCyGetNumInstances(&num_inst);
 109         if (status != CPA_STATUS_SUCCESS)
 110                 return (-1);
 111
 112         /* if the user has configured no QAT encryption units just return */
 113         if (num_inst == 0)
 114                 return (0);
 115
 116         if (num_inst > QAT_CRYPT_MAX_INSTANCES)
 117                 num_inst = QAT_CRYPT_MAX_INSTANCES;
 118
 119         status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
 120         if (status != CPA_STATUS_SUCCESS)
 121                 return (-1);
 122
 123         for (Cpa16U i = 0; i < num_inst; i++) {
 124                 status = cpaCySetAddressTranslation(cy_inst_handles[i],
 125                     (void *)virt_to_phys);
 126                 if (status != CPA_STATUS_SUCCESS)
 127                         goto error;
 128
 129                 status = cpaCyStartInstance(cy_inst_handles[i]);
 130                 if (status != CPA_STATUS_SUCCESS)
 131                         goto error;
 132         }
 133
 134         qat_crypt_init_done = B_TRUE;
 135         return (0);
 136
 137 error:
 138         qat_crypt_clean();
 139         return (-1);
 140 }
 141
 142 void
 143 qat_crypt_fini(void)
 144 {
 145         if (!qat_crypt_init_done)
 146                 return;
 147
 148         qat_crypt_clean();
 149 }
 150
 151 static CpaStatus
 152 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
 153     CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
 154     Cpa64U crypt, Cpa32U aad_len)
 155 {
 156         CpaStatus status = CPA_STATUS_SUCCESS;
 157         Cpa32U ctx_size;
 158         Cpa32U ciper_algorithm;
 159         Cpa32U hash_algorithm;
 160         CpaCySymSessionSetupData sd = { 0 };
 161
 162         if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
 163                 return (CPA_STATUS_FAIL);
 164         } else {
 165                 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
 166                 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
 167         }
 168
 169         sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
 170         sd.cipherSetupData.pCipherKey = key->ck_data;
 171         sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
 172         sd.hashSetupData.hashAlgorithm = hash_algorithm;
 173         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
 174         sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
 175         sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
 176         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
 177         sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 178         sd.digestIsAppended = CPA_FALSE;
 179         sd.verifyDigest = CPA_FALSE;
 180
 181         if (dir == QAT_ENCRYPT) {
 182                 sd.cipherSetupData.cipherDirection =
 183                     CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
 184                 sd.algChainOrder =
 185                     CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 186         } else {
 187                 ASSERT3U(dir, ==, QAT_DECRYPT);
 188                 sd.cipherSetupData.cipherDirection =
 189                     CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
 190                 sd.algChainOrder =
 191                     CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 192         }
 193
 194         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
 195         if (status != CPA_STATUS_SUCCESS)
 196                 return (status);
 197
 198         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
 199         if (status != CPA_STATUS_SUCCESS)
 200                 return (status);
 201
 202         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
 203             *cy_session_ctx);
 204         if (status != CPA_STATUS_SUCCESS) {
 205                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
 206                 return (status);
 207         }
 208
 209         return (CPA_STATUS_SUCCESS);
 210 }
 211
 212 static CpaStatus
 213 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
 214     CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
 215 {
 216         CpaStatus status = CPA_STATUS_SUCCESS;
 217         Cpa32U ctx_size;
 218         Cpa32U hash_algorithm;
 219         CpaCySymSessionSetupData sd = { 0 };
 220
 221         /*
 222          * ZFS's SHA512 checksum is actually SHA512/256, which uses
 223          * a different IV from standard SHA512. QAT does not support
 224          * SHA512/256, so we can only support SHA256.
 225          */
 226         if (cksum == ZIO_CHECKSUM_SHA256)
 227                 hash_algorithm = CPA_CY_SYM_HASH_SHA256;
 228         else
 229                 return (CPA_STATUS_FAIL);
 230
 231         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
 232         sd.symOperation = CPA_CY_SYM_OP_HASH;
 233         sd.hashSetupData.hashAlgorithm = hash_algorithm;
 234         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
 235         sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
 236         sd.digestIsAppended = CPA_FALSE;
 237         sd.verifyDigest = CPA_FALSE;
 238
 239         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
 240         if (status != CPA_STATUS_SUCCESS)
 241                 return (status);
 242
 243         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
 244         if (status != CPA_STATUS_SUCCESS)
 245                 return (status);
 246
 247         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
 248             *cy_session_ctx);
 249         if (status != CPA_STATUS_SUCCESS) {
 250                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
 251                 return (status);
 252         }
 253
 254         return (CPA_STATUS_SUCCESS);
 255 }
 256
 257 static CpaStatus
 258 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
 259     CpaBufferList *src, CpaBufferList *dst)
 260 {
 261         CpaStatus status = CPA_STATUS_SUCCESS;
 262         Cpa32U meta_size = 0;
 263
 264         status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
 265         if (status != CPA_STATUS_SUCCESS)
 266                 return (status);
 267
 268         status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
 269         if (status != CPA_STATUS_SUCCESS)
 270                 goto error;
 271
 272         if (src != dst) {
 273                 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
 274                     meta_size);
 275                 if (status != CPA_STATUS_SUCCESS)
 276                         goto error;
 277         }
 278
 279         return (CPA_STATUS_SUCCESS);
 280
 281 error:
 282         QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
 283         if (src != dst)
 284                 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
 285
 286         return (status);
 287 }
 288
 289 int
 290 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 291     uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
 292     crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
 293 {
 294         CpaStatus status = CPA_STATUS_SUCCESS;
 295         Cpa16U i;
 296         CpaInstanceHandle cy_inst_handle;
 297         Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
 298         Cpa32U bytes_left = 0;
 299         Cpa8S *data = NULL;
 300         CpaCySymSessionCtx *cy_session_ctx = NULL;
 301         cy_callback_t cb;
 302         CpaCySymOpData op_data = { 0 };
 303         CpaBufferList src_buffer_list = { 0 };
 304         CpaBufferList dst_buffer_list = { 0 };
 305         CpaFlatBuffer *flat_src_buf_array = NULL;
 306         CpaFlatBuffer *flat_src_buf = NULL;
 307         CpaFlatBuffer *flat_dst_buf_array = NULL;
 308         CpaFlatBuffer *flat_dst_buf = NULL;
 309         struct page *in_pages[MAX_PAGE_NUM];
 310         struct page *out_pages[MAX_PAGE_NUM];
 311         Cpa32U in_page_num = 0;
 312         Cpa32U out_page_num = 0;
 313         Cpa32U in_page_off = 0;
 314         Cpa32U out_page_off = 0;
 315
 316         if (dir == QAT_ENCRYPT) {
 317                 QAT_STAT_BUMP(encrypt_requests);
 318                 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
 319         } else {
 320                 QAT_STAT_BUMP(decrypt_requests);
 321                 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
 322         }
 323
 324         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
 325         cy_inst_handle = cy_inst_handles[i];
 326
 327         status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
 328             &cy_session_ctx, key, crypt, aad_len);
 329         if (status != CPA_STATUS_SUCCESS) {
 330                 /* don't count CCM as a failure since it's not supported */
 331                 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
 332                         QAT_STAT_BUMP(crypt_fails);
 333                 return (status);
 334         }
 335
 336         /*
 337          * We increment nr_bufs by 2 to allow us to handle non
 338          * page-aligned buffer addresses and buffers whose sizes
 339          * are not divisible by PAGE_SIZE.
 340          */
 341         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
 342             &src_buffer_list, &dst_buffer_list);
 343         if (status != CPA_STATUS_SUCCESS)
 344                 goto fail;
 345
 346         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
 347             nr_bufs * sizeof (CpaFlatBuffer));
 348         if (status != CPA_STATUS_SUCCESS)
 349                 goto fail;
 350         status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
 351             nr_bufs * sizeof (CpaFlatBuffer));
 352         if (status != CPA_STATUS_SUCCESS)
 353                 goto fail;
 354
 355         bytes_left = enc_len;
 356         data = src_buf;
 357         flat_src_buf = flat_src_buf_array;
 358         while (bytes_left > 0) {
 359                 in_page_off = ((long)data & ~PAGE_MASK);
 360                 in_pages[in_page_num] = qat_mem_to_page(data);
 361                 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
 362                 flat_src_buf->dataLenInBytes =
 363                     min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
 364                 data += flat_src_buf->dataLenInBytes;
 365                 bytes_left -= flat_src_buf->dataLenInBytes;
 366                 flat_src_buf++;
 367                 in_page_num++;
 368         }
 369         src_buffer_list.pBuffers = flat_src_buf_array;
 370         src_buffer_list.numBuffers = in_page_num;
 371
 372         bytes_left = enc_len;
 373         data = dst_buf;
 374         flat_dst_buf = flat_dst_buf_array;
 375         while (bytes_left > 0) {
 376                 out_page_off = ((long)data & ~PAGE_MASK);
 377                 out_pages[out_page_num] = qat_mem_to_page(data);
 378                 flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
 379                     out_page_off;
 380                 flat_dst_buf->dataLenInBytes =
 381                     min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
 382                 data += flat_dst_buf->dataLenInBytes;
 383                 bytes_left -= flat_dst_buf->dataLenInBytes;
 384                 flat_dst_buf++;
 385                 out_page_num++;
 386         }
 387         dst_buffer_list.pBuffers = flat_dst_buf_array;
 388         dst_buffer_list.numBuffers = out_page_num;
 389
 390         op_data.sessionCtx = cy_session_ctx;
 391         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 392         op_data.pIv = NULL; /* set this later as the J0 block */
 393         op_data.ivLenInBytes = 0;
 394         op_data.cryptoStartSrcOffsetInBytes = 0;
 395         op_data.messageLenToCipherInBytes = 0;
 396         op_data.hashStartSrcOffsetInBytes = 0;
 397         op_data.messageLenToHashInBytes = 0;
 398         op_data.pDigestResult = 0;
 399         op_data.messageLenToCipherInBytes = enc_len;
 400         op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
 401         op_data.pDigestResult = digest_buf;
 402         op_data.pAdditionalAuthData = aad_buf;
 403         op_data.pIv = iv_buf;
 404
 405         cb.verify_result = CPA_FALSE;
 406         init_completion(&cb.complete);
 407         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
 408             &src_buffer_list, &dst_buffer_list, NULL);
 409         if (status != CPA_STATUS_SUCCESS)
 410                 goto fail;
 411
 412         if (!wait_for_completion_interruptible_timeout(&cb.complete,
 413             QAT_TIMEOUT_MS)) {
 414                 status = CPA_STATUS_FAIL;
 415                 goto fail;
 416         }
 417
 418         if (cb.verify_result == CPA_FALSE) {
 419                 status = CPA_STATUS_FAIL;
 420                 goto fail;
 421         }
 422
 423         if (dir == QAT_ENCRYPT)
 424                 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
 425         else
 426                 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
 427
 428 fail:
 429         if (status != CPA_STATUS_SUCCESS)
 430                 QAT_STAT_BUMP(crypt_fails);
 431
 432         for (i = 0; i < in_page_num; i++)
 433                 kunmap(in_pages[i]);
 434         for (i = 0; i < out_page_num; i++)
 435                 kunmap(out_pages[i]);
 436
 437         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
 438         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 439         QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
 440         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
 441         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
 442         QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
 443
 444         return (status);
 445 }
 446
 447 int
 448 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
 449 {
 450         CpaStatus status;
 451         Cpa16U i;
 452         CpaInstanceHandle cy_inst_handle;
 453         Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
 454         Cpa32U bytes_left = 0;
 455         Cpa8S *data = NULL;
 456         CpaCySymSessionCtx *cy_session_ctx = NULL;
 457         cy_callback_t cb;
 458         Cpa8U *digest_buffer = NULL;
 459         CpaCySymOpData op_data = { 0 };
 460         CpaBufferList src_buffer_list = { 0 };
 461         CpaFlatBuffer *flat_src_buf_array = NULL;
 462         CpaFlatBuffer *flat_src_buf = NULL;
 463         struct page *in_pages[MAX_PAGE_NUM];
 464         Cpa32U page_num = 0;
 465         Cpa32U page_off = 0;
 466
 467         QAT_STAT_BUMP(cksum_requests);
 468         QAT_STAT_INCR(cksum_total_in_bytes, size);
 469
 470         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
 471         cy_inst_handle = cy_inst_handles[i];
 472
 473         status = qat_init_checksum_session_ctx(cy_inst_handle,
 474             &cy_session_ctx, cksum);
 475         if (status != CPA_STATUS_SUCCESS) {
 476                 /* don't count unsupported checksums as a failure */
 477                 if (cksum == ZIO_CHECKSUM_SHA256 ||
 478                     cksum == ZIO_CHECKSUM_SHA512)
 479                         QAT_STAT_BUMP(cksum_fails);
 480                 return (status);
 481         }
 482
 483         /*
 484          * We increment nr_bufs by 2 to allow us to handle non
 485          * page-aligned buffer addresses and buffers whose sizes
 486          * are not divisible by PAGE_SIZE.
 487          */
 488         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
 489             &src_buffer_list, &src_buffer_list);
 490         if (status != CPA_STATUS_SUCCESS)
 491                 goto fail;
 492
 493         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
 494             nr_bufs * sizeof (CpaFlatBuffer));
 495         if (status != CPA_STATUS_SUCCESS)
 496                 goto fail;
 497         status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
 498             sizeof (zio_cksum_t));
 499         if (status != CPA_STATUS_SUCCESS)
 500                 goto fail;
 501
 502         bytes_left = size;
 503         data = buf;
 504         flat_src_buf = flat_src_buf_array;
 505         while (bytes_left > 0) {
 506                 page_off = ((long)data & ~PAGE_MASK);
 507                 in_pages[page_num] = qat_mem_to_page(data);
 508                 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
 509                 flat_src_buf->dataLenInBytes =
 510                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
 511                 data += flat_src_buf->dataLenInBytes;
 512                 bytes_left -= flat_src_buf->dataLenInBytes;
 513                 flat_src_buf++;
 514                 page_num++;
 515         }
 516         src_buffer_list.pBuffers = flat_src_buf_array;
 517         src_buffer_list.numBuffers = page_num;
 518
 519         op_data.sessionCtx = cy_session_ctx;
 520         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 521         op_data.hashStartSrcOffsetInBytes = 0;
 522         op_data.messageLenToHashInBytes = size;
 523         op_data.pDigestResult = digest_buffer;
 524
 525         cb.verify_result = CPA_FALSE;
 526         init_completion(&cb.complete);
 527         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
 528             &src_buffer_list, &src_buffer_list, NULL);
 529         if (status != CPA_STATUS_SUCCESS)
 530                 goto fail;
 531
 532         if (!wait_for_completion_interruptible_timeout(&cb.complete,
 533             QAT_TIMEOUT_MS)) {
 534                 status = CPA_STATUS_FAIL;
 535                 goto fail;
 536         }
 537         if (cb.verify_result == CPA_FALSE) {
 538                 status = CPA_STATUS_FAIL;
 539                 goto fail;
 540         }
 541
 542         bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
 543
 544 fail:
 545         if (status != CPA_STATUS_SUCCESS)
 546                 QAT_STAT_BUMP(cksum_fails);
 547
 548         for (i = 0; i < page_num; i++)
 549                 kunmap(in_pages[i]);
 550
 551         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
 552         QAT_PHYS_CONTIG_FREE(digest_buffer);
 553         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 554         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
 555         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
 556
 557         return (status);
 558 }
 559
 560 module_param(zfs_qat_encrypt_disable, int, 0644);
 561 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
 562
 563 module_param(zfs_qat_checksum_disable, int, 0644);
 564 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
 565
 566 #endif