Code improvement and bug fixes for QAT support

[mirror_zfs.git] / module / zfs / qat_crypt.c

diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c
index 5a5113e68a5e1b40bcdc6e773801186bef0999d1..34c19b5823a8c4c84957de40255b4d07f252c477 100644 (file)
--- a/module/zfs/qat_crypt.c
+++ b/module/zfs/qat_crypt.c
@@ -31,6 +31,7 @@
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>
+#include <linux/mod_compat.h>
 #include <sys/zfs_context.h>
 #include <sys/zio_crypt.h>
 #include "lac/cpa_cy_im.h"
@@ -50,7 +51,7 @@
 static Cpa32U inst_num = 0;
 static Cpa16U num_inst = 0;
 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
-static boolean_t qat_crypt_init_done = B_FALSE;
+static boolean_t qat_cy_init_done = B_FALSE;
 int zfs_qat_encrypt_disable = 0;
 int zfs_qat_checksum_disable = 0;
 
@@ -76,7 +77,7 @@ boolean_t
 qat_crypt_use_accel(size_t s_len)
 {
        return (!zfs_qat_encrypt_disable &&
-           qat_crypt_init_done &&
+           qat_cy_init_done &&
            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }
@@ -85,26 +86,29 @@ boolean_t
 qat_checksum_use_accel(size_t s_len)
 {
        return (!zfs_qat_checksum_disable &&
-           qat_crypt_init_done &&
+           qat_cy_init_done &&
            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }
 
 void
-qat_crypt_clean(void)
+qat_cy_clean(void)
 {
        for (Cpa16U i = 0; i < num_inst; i++)
                cpaCyStopInstance(cy_inst_handles[i]);
 
        num_inst = 0;
-       qat_crypt_init_done = B_FALSE;
+       qat_cy_init_done = B_FALSE;
 }
 
 int
-qat_crypt_init(void)
+qat_cy_init(void)
 {
        CpaStatus status = CPA_STATUS_FAIL;
 
+       if (qat_cy_init_done)
+               return (0);
+
        status = cpaCyGetNumInstances(&num_inst);
        if (status != CPA_STATUS_SUCCESS)
                return (-1);
@@ -131,21 +135,21 @@ qat_crypt_init(void)
                        goto error;
        }
 
-       qat_crypt_init_done = B_TRUE;
+       qat_cy_init_done = B_TRUE;
        return (0);
 
 error:
-       qat_crypt_clean();
+       qat_cy_clean();
        return (-1);
 }
 
 void
-qat_crypt_fini(void)
+qat_cy_fini(void)
 {
-       if (!qat_crypt_init_done)
+       if (!qat_cy_init_done)
                return;
 
-       qat_crypt_clean();
+       qat_cy_clean();
 }
 
 static CpaStatus
@@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
            nr_bufs * sizeof (CpaFlatBuffer));
        if (status != CPA_STATUS_SUCCESS)
                goto fail;
+       status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
+           ZIO_DATA_MAC_LEN);
+       if (status != CPA_STATUS_SUCCESS)
+               goto fail;
+       status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
+           ZIO_DATA_IV_LEN);
+       if (status != CPA_STATUS_SUCCESS)
+               goto fail;
+       if (aad_len > 0) {
+               status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
+                   aad_len);
+               if (status != CPA_STATUS_SUCCESS)
+                       goto fail;
+               bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
+       }
 
        bytes_left = enc_len;
        data = src_buf;
@@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 
        op_data.sessionCtx = cy_session_ctx;
        op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
-       op_data.pIv = NULL; /* set this later as the J0 block */
-       op_data.ivLenInBytes = 0;
        op_data.cryptoStartSrcOffsetInBytes = 0;
        op_data.messageLenToCipherInBytes = 0;
        op_data.hashStartSrcOffsetInBytes = 0;
        op_data.messageLenToHashInBytes = 0;
-       op_data.pDigestResult = 0;
        op_data.messageLenToCipherInBytes = enc_len;
        op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
-       op_data.pDigestResult = digest_buf;
-       op_data.pAdditionalAuthData = aad_buf;
-       op_data.pIv = iv_buf;
+       bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
 
        cb.verify_result = CPA_FALSE;
        init_completion(&cb.complete);
@@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
                goto fail;
        }
 
+       /* save digest result to digest_buf */
+       bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
        if (dir == QAT_ENCRYPT)
                QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
        else
@@ -435,6 +451,10 @@ fail:
                kunmap(out_pages[i]);
 
        cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
+       if (aad_len > 0)
+               QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
+       QAT_PHYS_CONTIG_FREE(op_data.pIv);
+       QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
        QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(cy_session_ctx);
@@ -557,10 +577,56 @@ fail:
        return (status);
 }
 
-module_param(zfs_qat_encrypt_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
+static int
+param_set_qat_encrypt(const char *val, struct kernel_param *kp)
+{
+       int ret;
+       int *pvalue = kp->arg;
+       ret = param_set_int(val, kp);
+       if (ret)
+               return (ret);
+       /*
+        * zfs_qat_encrypt_disable = 0: enable qat encrypt
+        * try to initialize qat instance if it has not been done
+        */
+       if (*pvalue == 0 && !qat_cy_init_done) {
+               ret = qat_cy_init();
+               if (ret != 0) {
+                       zfs_qat_encrypt_disable = 1;
+                       return (ret);
+               }
+       }
+       return (ret);
+}
+
+static int
+param_set_qat_checksum(const char *val, struct kernel_param *kp)
+{
+       int ret;
+       int *pvalue = kp->arg;
+       ret = param_set_int(val, kp);
+       if (ret)
+               return (ret);
+       /*
+        * set_checksum_param_ops = 0: enable qat checksum
+        * try to initialize qat instance if it has not been done
+        */
+       if (*pvalue == 0 && !qat_cy_init_done) {
+               ret = qat_cy_init();
+               if (ret != 0) {
+                       zfs_qat_checksum_disable = 1;
+                       return (ret);
+               }
+       }
+       return (ret);
+}
+
+module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
+    param_get_int, &zfs_qat_encrypt_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
 
-module_param(zfs_qat_checksum_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
+module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
+    param_get_int, &zfs_qat_checksum_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
 
 #endif