module/zfs/qat_crypt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 #if defined(_KERNEL) && defined(HAVE_QAT)
  23 #include <linux/slab.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/pagemap.h>
  26 #include <linux/completion.h>
  27 #include <sys/zfs_context.h>
  28 #include <sys/zio_crypt.h>
  29 #include "lac/cpa_cy_im.h"
  30 #include "qat.h"
  31
  32 /*
  33  * Max instances in QAT device, each instance is a channel to submit
  34  * jobs to QAT hardware, this is only for pre-allocating instance,
  35  * and session arrays, the actual number of instances are defined in
  36  * the QAT driver's configure file.
  37  */
  38 #define QAT_CRYPT_MAX_INSTANCES         48
  39
  40 #define MAX_PAGE_NUM                    1024
  41
  42 static boolean_t qat_crypt_init_done = B_FALSE;
  43 static Cpa16U inst_num = 0;
  44 static Cpa16U num_inst = 0;
  45 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
  46
  47 typedef struct cy_callback {
  48         CpaBoolean verify_result;
  49         struct completion complete;
  50 } cy_callback_t;
  51
  52 static void
  53 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
  54     void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
  55 {
  56         cy_callback_t *cb = p_callback;
  57
  58         if (cb != NULL) {
  59                 /* indicate that the function has been called */
  60                 cb->verify_result = verify;
  61                 complete(&cb->complete);
  62         }
  63 }
  64
  65 boolean_t
  66 qat_crypt_use_accel(size_t s_len)
  67 {
  68         return (!zfs_qat_disable &&
  69             qat_crypt_init_done &&
  70             s_len >= QAT_MIN_BUF_SIZE &&
  71             s_len <= QAT_MAX_BUF_SIZE);
  72 }
  73
  74 void
  75 qat_crypt_clean(void)
  76 {
  77         for (Cpa32U i = 0; i < num_inst; i++)
  78                 cpaCyStopInstance(cy_inst_handles[i]);
  79
  80         num_inst = 0;
  81         qat_crypt_init_done = B_FALSE;
  82 }
  83
  84 int
  85 qat_crypt_init(void)
  86 {
  87         Cpa32U i;
  88         CpaStatus status = CPA_STATUS_FAIL;
  89
  90         status = cpaCyGetNumInstances(&num_inst);
  91         if (status != CPA_STATUS_SUCCESS)
  92                 return (-1);
  93
  94         /* if the user has configured no QAT encryption units just return */
  95         if (num_inst == 0)
  96                 return (0);
  97
  98         if (num_inst > QAT_CRYPT_MAX_INSTANCES)
  99                 num_inst = QAT_CRYPT_MAX_INSTANCES;
 100
 101         status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
 102         if (status != CPA_STATUS_SUCCESS)
 103                 return (-1);
 104
 105         for (i = 0; i < num_inst; i++) {
 106                 status = cpaCySetAddressTranslation(cy_inst_handles[i],
 107                     (void *)virt_to_phys);
 108                 if (status != CPA_STATUS_SUCCESS)
 109                         goto error;
 110
 111                 status = cpaCyStartInstance(cy_inst_handles[i]);
 112                 if (status != CPA_STATUS_SUCCESS)
 113                         goto error;
 114         }
 115
 116         qat_crypt_init_done = B_TRUE;
 117         return (0);
 118
 119 error:
 120         qat_crypt_clean();
 121         return (-1);
 122 }
 123
 124 void
 125 qat_crypt_fini(void)
 126 {
 127         if (!qat_crypt_init_done)
 128                 return;
 129
 130         qat_crypt_clean();
 131 }
 132
 133 static CpaStatus
 134 init_cy_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
 135     CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
 136     Cpa64U crypt, Cpa32U aad_len)
 137 {
 138         CpaStatus status = CPA_STATUS_SUCCESS;
 139         Cpa32U ctx_size;
 140         Cpa32U ciper_algorithm;
 141         Cpa32U hash_algorithm;
 142         CpaCySymSessionSetupData sd = { 0 };
 143
 144         if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
 145                 return (CPA_STATUS_FAIL);
 146         } else {
 147                 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
 148                 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
 149         }
 150
 151         sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
 152         sd.cipherSetupData.pCipherKey = key->ck_data;
 153         sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
 154         sd.hashSetupData.hashAlgorithm = hash_algorithm;
 155         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
 156         sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
 157         sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
 158         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
 159         sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 160         sd.digestIsAppended = CPA_FALSE;
 161         sd.verifyDigest = CPA_FALSE;
 162
 163         if (dir == QAT_ENCRYPT) {
 164                 sd.cipherSetupData.cipherDirection =
 165                     CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
 166                 sd.algChainOrder =
 167                     CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 168         } else {
 169                 ASSERT3U(dir, ==, QAT_DECRYPT);
 170                 sd.cipherSetupData.cipherDirection =
 171                     CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
 172                 sd.algChainOrder =
 173                     CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 174         }
 175
 176         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
 177         if (status != CPA_STATUS_SUCCESS)
 178                 return (status);
 179
 180         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
 181         if (status != CPA_STATUS_SUCCESS)
 182                 return (status);
 183
 184         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
 185             *cy_session_ctx);
 186         if (status != CPA_STATUS_SUCCESS) {
 187                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
 188                 return (status);
 189         }
 190
 191         return (CPA_STATUS_SUCCESS);
 192 }
 193
 194 static CpaStatus
 195 init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
 196     CpaBufferList *src, CpaBufferList *dst)
 197 {
 198         CpaStatus status = CPA_STATUS_SUCCESS;
 199         Cpa32U meta_size = 0;
 200
 201         status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
 202         if (status != CPA_STATUS_SUCCESS)
 203                 return (status);
 204
 205         src->numBuffers = nr_bufs;
 206         status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
 207         if (status != CPA_STATUS_SUCCESS)
 208                 goto error;
 209
 210         if (src != dst) {
 211                 dst->numBuffers = nr_bufs;
 212                 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
 213                     meta_size);
 214                 if (status != CPA_STATUS_SUCCESS)
 215                         goto error;
 216         }
 217
 218         return (CPA_STATUS_SUCCESS);
 219
 220 error:
 221         QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
 222         if (src != dst)
 223                 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
 224
 225         return (status);
 226 }
 227
 228 int
 229 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 230     uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
 231     crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
 232 {
 233         CpaStatus status = CPA_STATUS_SUCCESS;
 234         Cpa16U i;
 235         CpaInstanceHandle cy_inst_handle;
 236         Cpa16U nr_bufs;
 237         Cpa32U bytes_left = 0;
 238         Cpa8S *in = NULL;
 239         Cpa8S *out = NULL;
 240         CpaCySymSessionCtx *cy_session_ctx = NULL;
 241         cy_callback_t cb;
 242         CpaCySymOpData op_data = { 0 };
 243         CpaBufferList src_buffer_list = { 0 };
 244         CpaBufferList dst_buffer_list = { 0 };
 245         CpaFlatBuffer *flat_src_buf_array = NULL;
 246         CpaFlatBuffer *flat_src_buf = NULL;
 247         CpaFlatBuffer *flat_dst_buf_array = NULL;
 248         CpaFlatBuffer *flat_dst_buf = NULL;
 249         struct page *in_pages[MAX_PAGE_NUM];
 250         struct page *out_pages[MAX_PAGE_NUM];
 251         Cpa32S page_num = 0;
 252
 253         if (dir == QAT_ENCRYPT) {
 254                 QAT_STAT_BUMP(encrypt_requests);
 255                 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
 256         } else {
 257                 QAT_STAT_BUMP(decrypt_requests);
 258                 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
 259         }
 260
 261         i = atomic_inc_32_nv(&inst_num) % num_inst;
 262         cy_inst_handle = cy_inst_handles[i];
 263
 264         status = init_cy_session_ctx(dir, cy_inst_handle, &cy_session_ctx, key,
 265             crypt, aad_len);
 266         if (status != CPA_STATUS_SUCCESS)
 267                 return (status);
 268
 269         nr_bufs = enc_len / PAGE_CACHE_SIZE +
 270             (enc_len % PAGE_CACHE_SIZE == 0 ? 0 : 1);
 271         status = init_cy_buffer_lists(cy_inst_handle, nr_bufs, &src_buffer_list,
 272             &dst_buffer_list);
 273         if (status != CPA_STATUS_SUCCESS)
 274                 goto fail;
 275
 276         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
 277             nr_bufs * sizeof (CpaFlatBuffer));
 278         if (status != CPA_STATUS_SUCCESS)
 279                 goto fail;
 280         status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
 281             nr_bufs * sizeof (CpaFlatBuffer));
 282         if (status != CPA_STATUS_SUCCESS)
 283                 goto fail;
 284
 285         bytes_left = enc_len;
 286         in = src_buf;
 287         out = dst_buf;
 288         flat_src_buf = flat_src_buf_array;
 289         flat_dst_buf = flat_dst_buf_array;
 290         while (bytes_left > 0) {
 291                 in_pages[page_num] = qat_mem_to_page(in);
 292                 out_pages[page_num] = qat_mem_to_page(out);
 293                 flat_src_buf->pData = kmap(in_pages[page_num]);
 294                 flat_dst_buf->pData = kmap(out_pages[page_num]);
 295                 flat_src_buf->dataLenInBytes = min((long)PAGE_CACHE_SIZE,
 296                     (long)bytes_left);
 297                 flat_dst_buf->dataLenInBytes = min((long)PAGE_CACHE_SIZE,
 298                     (long)bytes_left);
 299                 in += flat_src_buf->dataLenInBytes;
 300                 out += flat_dst_buf->dataLenInBytes;
 301                 bytes_left -= flat_src_buf->dataLenInBytes;
 302                 flat_src_buf++;
 303                 flat_dst_buf++;
 304                 page_num++;
 305         }
 306         src_buffer_list.pBuffers = flat_src_buf_array;
 307         dst_buffer_list.pBuffers = flat_dst_buf_array;
 308
 309         op_data.sessionCtx = cy_session_ctx;
 310         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 311         op_data.pIv = NULL; /* set this later as the J0 block */
 312         op_data.ivLenInBytes = 0;
 313         op_data.cryptoStartSrcOffsetInBytes = 0;
 314         op_data.messageLenToCipherInBytes = 0;
 315         op_data.hashStartSrcOffsetInBytes = 0;
 316         op_data.messageLenToHashInBytes = 0;
 317         op_data.pDigestResult = 0;
 318         op_data.messageLenToCipherInBytes = enc_len;
 319         op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
 320         op_data.pDigestResult = digest_buf;
 321         op_data.pAdditionalAuthData = aad_buf;
 322         op_data.pIv = iv_buf;
 323
 324         cb.verify_result = CPA_FALSE;
 325         init_completion(&cb.complete);
 326         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
 327             &src_buffer_list, &dst_buffer_list, NULL);
 328         if (status != CPA_STATUS_SUCCESS)
 329                 goto fail;
 330
 331         if (!wait_for_completion_interruptible_timeout(&cb.complete,
 332             QAT_TIMEOUT_MS)) {
 333                 status = CPA_STATUS_FAIL;
 334                 goto fail;
 335         }
 336
 337         if (cb.verify_result == CPA_FALSE) {
 338                 status = CPA_STATUS_FAIL;
 339                 goto fail;
 340         }
 341
 342         if (dir == QAT_ENCRYPT)
 343                 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
 344         else
 345                 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
 346
 347 fail:
 348         /* don't count CCM as a failure since it's not supported */
 349         if (status != CPA_STATUS_SUCCESS &&
 350             zio_crypt_table[crypt].ci_crypt_type != ZC_TYPE_CCM)
 351                 QAT_STAT_BUMP(crypt_fails);
 352
 353         for (i = 0; i < page_num; i ++) {
 354                 kunmap(in_pages[i]);
 355                 kunmap(out_pages[i]);
 356         }
 357
 358         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
 359         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 360         QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
 361         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
 362         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
 363         QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
 364
 365         return (status);
 366 }
 367
 368 module_param(zfs_qat_disable, int, 0644);
 369 MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT acceleration");
 370
 371 #endif