Code improvement and bug fixes for QAT support

1. Support QAT when ZFS is root file-system:
   When ZFS module is loaded before QAT started, the QAT can
   be started again in post-process, e.g.:
   echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_encrypt_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_checksum_disable
2. Verify alder checksum of the de-compress result
3. Allocate Digest, IV and AAD buffer in physical contiguous
   memory by QAT_PHYS_CONTIG_ALLOC.
4. Update the documentation for zfs_qat_compress_disable,
   zfs_qat_checksum_disable, zfs_qat_encrypt_disable.

Reviewed-by: Tom Caputi <tcaputi@datto.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Chengfeix Zhu <chengfeix.zhu@intel.com>
Closes #8323
Closes #8610

diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5
index a1a586df1c164ce26d01504448f9a27be942e7ba..4a3ff1d49a1ea47daa1baf2e5b6526f76d44cf8c 100644 (file)
--- a/man/man5/zfs-module-parameters.5
+++ b/man/man5/zfs-module-parameters.5
@@ -2042,6 +2042,45 @@ can't hurt performance.
 Use \fB1\fR for yes and \fB0\fR for no (default).
 .RE
 
 Use \fB1\fR for yes and \fB0\fR for no (default).
 .RE
 

+.sp
+.ne 2
+.na
+\fBzfs_qat_checksum_disable\fR (int)
+.ad
+.RS 12n
+This tunable disables qat hardware acceleration for sha256 checksums. It
+may be set after the zfs modules have been loaded to initialize the qat
+hardware as long as support is compiled in and the qat driver is present.
+.sp
+Use \fB1\fR for yes and \fB0\fR for no (default).
+.RE
+
+.sp
+.ne 2
+.na
+\fBzfs_qat_compress_disable\fR (int)
+.ad
+.RS 12n
+This tunable disables qat hardware acceleration for gzip compression. It
+may be set after the zfs modules have been loaded to initialize the qat
+hardware as long as support is compiled in and the qat driver is present.
+.sp
+Use \fB1\fR for yes and \fB0\fR for no (default).
+.RE
+
+.sp
+.ne 2
+.na
+\fBzfs_qat_encrypt_disable\fR (int)
+.ad
+.RS 12n
+This tunable disables qat hardware acceleration for AES-GCM encryption. It
+may be set after the zfs modules have been loaded to initialize the qat
+hardware as long as support is compiled in and the qat driver is present.
+.sp
+Use \fB1\fR for yes and \fB0\fR for no (default).
+.RE
+

 .sp
 .ne 2
 .na
 .sp
 .ne 2
 .na


@@ -2961,19 +3000,6 @@ Valid values are \fB1\fR (full), \fB2\fR (dev) and \fB3\fR (none).
 Default value: \fB1\fR.
 .RE
 
 Default value: \fB1\fR.
 .RE
 

-.sp
-.ne 2
-.na
-\fBzfs_qat_disable\fR (int)
-.ad
-.RS 12n
-This tunable disables qat hardware acceleration for gzip compression and.
-AES-GCM encryption. It is available only if qat acceleration is compiled in
-and the qat driver is present.
-.sp
-Use \fB1\fR for yes and \fB0\fR for no (default).
-.RE
-

 .SH ZFS I/O SCHEDULER
 ZFS issues I/O operations to leaf vdevs to satisfy and complete I/Os.
 The I/O scheduler determines when and in what order those operations are
 .SH ZFS I/O SCHEDULER
 ZFS issues I/O operations to leaf vdevs to satisfy and complete I/Os.
 The I/O scheduler determines when and in what order those operations are

diff --git a/module/zfs/qat.c b/module/zfs/qat.c
index 0a4f9c72a3dfc77837f67fcc5a2629193830c20d..a6f024cb44d7c0d1b9dc75920f8bfb688b01d7c0 100644 (file)
--- a/module/zfs/qat.c
+++ b/module/zfs/qat.c
@@ -66,18 +66,6 @@ qat_mem_free_contig(void **pp_mem_addr)
 int
 qat_init(void)
 {
 int
 qat_init(void)
 {

-       int ret;
-
-       ret = qat_dc_init();
-       if (ret != 0)
-               return (ret);
-
-       ret = qat_crypt_init();
-       if (ret != 0) {
-               qat_dc_fini();
-               return (ret);
-       }
-

        qat_ksp = kstat_create("zfs", 0, "qat", "misc",
            KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
            KSTAT_FLAG_VIRTUAL);
        qat_ksp = kstat_create("zfs", 0, "qat", "misc",
            KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
            KSTAT_FLAG_VIRTUAL);


@@ -86,6 +74,19 @@ qat_init(void)
                kstat_install(qat_ksp);
        }
 
                kstat_install(qat_ksp);
        }
 

+       /*
+        * Just set the disable flag when qat init failed, qat can be
+        * turned on again in post-process after zfs module is loaded, e.g.:
+        * echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
+        */
+       if (qat_dc_init() != 0)
+               zfs_qat_compress_disable = 1;
+
+       if (qat_cy_init() != 0) {
+               zfs_qat_checksum_disable = 1;
+               zfs_qat_encrypt_disable = 1;
+       }
+

        return (0);
 }
 
        return (0);
 }
 


@@ -97,7 +98,7 @@ qat_fini(void)
                qat_ksp = NULL;
        }
 
                qat_ksp = NULL;
        }
 

-       qat_crypt_fini();
+       qat_cy_fini();

        qat_dc_fini();
 }
 
        qat_dc_fini();
 }
 

diff --git a/module/zfs/qat.h b/module/zfs/qat.h
index b2cd5a9c17a9af33e5ac5067a62b410198947f4f..9014c03148bad09c3128961904a7bba6b3f8a279 100644 (file)
--- a/module/zfs/qat.h
+++ b/module/zfs/qat.h
@@ -147,6 +147,9 @@ typedef struct qat_stats {
        QAT_STAT_INCR(stat, 1)
 
 extern qat_stats_t qat_stats;
        QAT_STAT_INCR(stat, 1)
 
 extern qat_stats_t qat_stats;

+extern int zfs_qat_compress_disable;
+extern int zfs_qat_checksum_disable;
+extern int zfs_qat_encrypt_disable;

 
 /* inlined for performance */
 static inline struct page *
 
 /* inlined for performance */
 static inline struct page *


@@ -167,8 +170,8 @@ void qat_mem_free_contig(void **pp_mem_addr);
 
 extern int qat_dc_init(void);
 extern void qat_dc_fini(void);
 
 extern int qat_dc_init(void);
 extern void qat_dc_fini(void);

-extern int qat_crypt_init(void);
-extern void qat_crypt_fini(void);
+extern int qat_cy_init(void);
+extern void qat_cy_fini(void);

 extern int qat_init(void);
 extern void qat_fini(void);
 
 extern int qat_init(void);
 extern void qat_fini(void);
 

diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c
index 8c1410c9ec01966d5f806ddd49a43f61254909ee..1c5c0a4e725684dca8fc39b32aae90e292493de5 100644 (file)
--- a/module/zfs/qat_compress.c
+++ b/module/zfs/qat_compress.c
@@ -24,7 +24,9 @@
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>

+#include <linux/mod_compat.h>

 #include <sys/zfs_context.h>
 #include <sys/zfs_context.h>

+#include <sys/byteorder.h>

 #include <sys/zio.h>
 #include "qat.h"
 
 #include <sys/zio.h>
 #include "qat.h"
 


@@ -111,6 +113,9 @@ qat_dc_init(void)
        Cpa32U buff_meta_size = 0;
        CpaDcSessionSetupData sd = {0};
 
        Cpa32U buff_meta_size = 0;
        CpaDcSessionSetupData sd = {0};
 

+       if (qat_dc_init_done)
+               return (0);
+

        status = cpaDcGetNumInstances(&num_inst);
        if (status != CPA_STATUS_SUCCESS)
                return (-1);
        status = cpaDcGetNumInstances(&num_inst);
        if (status != CPA_STATUS_SUCCESS)
                return (-1);


@@ -252,6 +257,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
        Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
        Cpa32U bytes_left;
        Cpa32U dst_pages = 0;
        Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
        Cpa32U bytes_left;
        Cpa32U dst_pages = 0;

+       Cpa32U adler32 = 0;

        char *data;
        struct page *page;
        struct page **in_pages = NULL;
        char *data;
        struct page *page;
        struct page **in_pages = NULL;


@@ -468,6 +474,12 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
                        goto fail;
                }
 
                        goto fail;
                }
 

+               /* verify adler checksum */
+               adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
+               if (adler32 != BSWAP_32(dc_results.checksum)) {
+                       status = CPA_STATUS_FAIL;
+                       goto fail;
+               }

                *c_len = dc_results.produced;
                QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
        }
                *c_len = dc_results.produced;
                QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
        }


@@ -534,7 +546,30 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
        return (ret);
 }
 
        return (ret);
 }
 

-module_param(zfs_qat_compress_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression");
+static int
+param_set_qat_compress(const char *val, struct kernel_param *kp)
+{
+       int ret;
+       int *pvalue = kp->arg;
+       ret = param_set_int(val, kp);
+       if (ret)
+               return (ret);
+       /*
+        * zfs_qat_compress_disable = 0: enable qat compress
+        * try to initialize qat instance if it has not been done
+        */
+       if (*pvalue == 0 && !qat_dc_init_done) {
+               ret = qat_dc_init();
+               if (ret != 0) {
+                       zfs_qat_compress_disable = 1;
+                       return (ret);
+               }
+       }
+       return (ret);
+}
+
+module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
+    param_get_int, &zfs_qat_compress_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");

 
 #endif
 
 #endif

diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c
index 5a5113e68a5e1b40bcdc6e773801186bef0999d1..34c19b5823a8c4c84957de40255b4d07f252c477 100644 (file)
--- a/module/zfs/qat_crypt.c
+++ b/module/zfs/qat_crypt.c
@@ -31,6 +31,7 @@
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/completion.h>

+#include <linux/mod_compat.h>

 #include <sys/zfs_context.h>
 #include <sys/zio_crypt.h>
 #include "lac/cpa_cy_im.h"
 #include <sys/zfs_context.h>
 #include <sys/zio_crypt.h>
 #include "lac/cpa_cy_im.h"


@@ -50,7 +51,7 @@
 static Cpa32U inst_num = 0;
 static Cpa16U num_inst = 0;
 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
 static Cpa32U inst_num = 0;
 static Cpa16U num_inst = 0;
 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];

-static boolean_t qat_crypt_init_done = B_FALSE;
+static boolean_t qat_cy_init_done = B_FALSE;

 int zfs_qat_encrypt_disable = 0;
 int zfs_qat_checksum_disable = 0;
 
 int zfs_qat_encrypt_disable = 0;
 int zfs_qat_checksum_disable = 0;
 


@@ -76,7 +77,7 @@ boolean_t
 qat_crypt_use_accel(size_t s_len)
 {
        return (!zfs_qat_encrypt_disable &&
 qat_crypt_use_accel(size_t s_len)
 {
        return (!zfs_qat_encrypt_disable &&

-           qat_crypt_init_done &&
+           qat_cy_init_done &&

            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }
            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }


@@ -85,26 +86,29 @@ boolean_t
 qat_checksum_use_accel(size_t s_len)
 {
        return (!zfs_qat_checksum_disable &&
 qat_checksum_use_accel(size_t s_len)
 {
        return (!zfs_qat_checksum_disable &&

-           qat_crypt_init_done &&
+           qat_cy_init_done &&

            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }
 
 void
            s_len >= QAT_MIN_BUF_SIZE &&
            s_len <= QAT_MAX_BUF_SIZE);
 }
 
 void

-qat_crypt_clean(void)
+qat_cy_clean(void)

 {
        for (Cpa16U i = 0; i < num_inst; i++)
                cpaCyStopInstance(cy_inst_handles[i]);
 
        num_inst = 0;
 {
        for (Cpa16U i = 0; i < num_inst; i++)
                cpaCyStopInstance(cy_inst_handles[i]);
 
        num_inst = 0;

-       qat_crypt_init_done = B_FALSE;
+       qat_cy_init_done = B_FALSE;

 }
 
 int
 }
 
 int

-qat_crypt_init(void)
+qat_cy_init(void)

 {
        CpaStatus status = CPA_STATUS_FAIL;
 
 {
        CpaStatus status = CPA_STATUS_FAIL;
 

+       if (qat_cy_init_done)
+               return (0);
+

        status = cpaCyGetNumInstances(&num_inst);
        if (status != CPA_STATUS_SUCCESS)
                return (-1);
        status = cpaCyGetNumInstances(&num_inst);
        if (status != CPA_STATUS_SUCCESS)
                return (-1);


@@ -131,21 +135,21 @@ qat_crypt_init(void)
                        goto error;
        }
 
                        goto error;
        }
 

-       qat_crypt_init_done = B_TRUE;
+       qat_cy_init_done = B_TRUE;

        return (0);
 
 error:
        return (0);
 
 error:

-       qat_crypt_clean();
+       qat_cy_clean();

        return (-1);
 }
 
 void
        return (-1);
 }
 
 void

-qat_crypt_fini(void)
+qat_cy_fini(void)

 {
 {

-       if (!qat_crypt_init_done)
+       if (!qat_cy_init_done)

                return;
 
                return;
 

-       qat_crypt_clean();
+       qat_cy_clean();

 }
 
 static CpaStatus
 }
 
 static CpaStatus


@@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
            nr_bufs * sizeof (CpaFlatBuffer));
        if (status != CPA_STATUS_SUCCESS)
                goto fail;
            nr_bufs * sizeof (CpaFlatBuffer));
        if (status != CPA_STATUS_SUCCESS)
                goto fail;

+       status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
+           ZIO_DATA_MAC_LEN);
+       if (status != CPA_STATUS_SUCCESS)
+               goto fail;
+       status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
+           ZIO_DATA_IV_LEN);
+       if (status != CPA_STATUS_SUCCESS)
+               goto fail;
+       if (aad_len > 0) {
+               status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
+                   aad_len);
+               if (status != CPA_STATUS_SUCCESS)
+                       goto fail;
+               bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
+       }

 
        bytes_left = enc_len;
        data = src_buf;
 
        bytes_left = enc_len;
        data = src_buf;


@@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 
        op_data.sessionCtx = cy_session_ctx;
        op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 
        op_data.sessionCtx = cy_session_ctx;
        op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;

-       op_data.pIv = NULL; /* set this later as the J0 block */
-       op_data.ivLenInBytes = 0;

        op_data.cryptoStartSrcOffsetInBytes = 0;
        op_data.messageLenToCipherInBytes = 0;
        op_data.hashStartSrcOffsetInBytes = 0;
        op_data.messageLenToHashInBytes = 0;
        op_data.cryptoStartSrcOffsetInBytes = 0;
        op_data.messageLenToCipherInBytes = 0;
        op_data.hashStartSrcOffsetInBytes = 0;
        op_data.messageLenToHashInBytes = 0;

-       op_data.pDigestResult = 0;

        op_data.messageLenToCipherInBytes = enc_len;
        op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
        op_data.messageLenToCipherInBytes = enc_len;
        op_data.ivLenInBytes = ZIO_DATA_IV_LEN;

-       op_data.pDigestResult = digest_buf;
-       op_data.pAdditionalAuthData = aad_buf;
-       op_data.pIv = iv_buf;
+       bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);

 
        cb.verify_result = CPA_FALSE;
        init_completion(&cb.complete);
 
        cb.verify_result = CPA_FALSE;
        init_completion(&cb.complete);


@@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
                goto fail;
        }
 
                goto fail;
        }
 

+       /* save digest result to digest_buf */
+       bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);

        if (dir == QAT_ENCRYPT)
                QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
        else
        if (dir == QAT_ENCRYPT)
                QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
        else


@@ -435,6 +451,10 @@ fail:
                kunmap(out_pages[i]);
 
        cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
                kunmap(out_pages[i]);
 
        cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);

+       if (aad_len > 0)
+               QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
+       QAT_PHYS_CONTIG_FREE(op_data.pIv);
+       QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);

        QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(cy_session_ctx);
        QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
        QAT_PHYS_CONTIG_FREE(cy_session_ctx);


@@ -557,10 +577,56 @@ fail:
        return (status);
 }
 
        return (status);
 }
 

-module_param(zfs_qat_encrypt_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
+static int
+param_set_qat_encrypt(const char *val, struct kernel_param *kp)
+{
+       int ret;
+       int *pvalue = kp->arg;
+       ret = param_set_int(val, kp);
+       if (ret)
+               return (ret);
+       /*
+        * zfs_qat_encrypt_disable = 0: enable qat encrypt
+        * try to initialize qat instance if it has not been done
+        */
+       if (*pvalue == 0 && !qat_cy_init_done) {
+               ret = qat_cy_init();
+               if (ret != 0) {
+                       zfs_qat_encrypt_disable = 1;
+                       return (ret);
+               }
+       }
+       return (ret);
+}
+
+static int
+param_set_qat_checksum(const char *val, struct kernel_param *kp)
+{
+       int ret;
+       int *pvalue = kp->arg;
+       ret = param_set_int(val, kp);
+       if (ret)
+               return (ret);
+       /*
+        * set_checksum_param_ops = 0: enable qat checksum
+        * try to initialize qat instance if it has not been done
+        */
+       if (*pvalue == 0 && !qat_cy_init_done) {
+               ret = qat_cy_init();
+               if (ret != 0) {
+                       zfs_qat_checksum_disable = 1;
+                       return (ret);
+               }
+       }
+       return (ret);
+}
+
+module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
+    param_get_int, &zfs_qat_encrypt_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");

 
 

-module_param(zfs_qat_checksum_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
+module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
+    param_get_int, &zfs_qat_checksum_disable, 0644);
+MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");

 
 #endif
 
 #endif