From: cfzhu Date: Tue, 16 Apr 2019 19:38:36 +0000 (+0800) Subject: Code improvement and bug fixes for QAT support X-Git-Tag: zfs-0.8.0~57 X-Git-Url: https://git.proxmox.com/?p=mirror_zfs.git;a=commitdiff_plain;h=5090f72743f5a587b1a8b64aaa1023913735d0bf Code improvement and bug fixes for QAT support 1. Support QAT when ZFS is root file-system: When ZFS module is loaded before QAT started, the QAT can be started again in post-process, e.g.: echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable echo 0 > /sys/module/zfs/parameters/zfs_qat_encrypt_disable echo 0 > /sys/module/zfs/parameters/zfs_qat_checksum_disable 2. Verify alder checksum of the de-compress result 3. Allocate Digest, IV and AAD buffer in physical contiguous memory by QAT_PHYS_CONTIG_ALLOC. 4. Update the documentation for zfs_qat_compress_disable, zfs_qat_checksum_disable, zfs_qat_encrypt_disable. Reviewed-by: Tom Caputi Reviewed-by: Brian Behlendorf Signed-off-by: Weigang Li Signed-off-by: Chengfeix Zhu Closes #8323 Closes #8610 --- diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index a1a586df1..4a3ff1d49 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -2042,6 +2042,45 @@ can't hurt performance. Use \fB1\fR for yes and \fB0\fR for no (default). .RE +.sp +.ne 2 +.na +\fBzfs_qat_checksum_disable\fR (int) +.ad +.RS 12n +This tunable disables qat hardware acceleration for sha256 checksums. It +may be set after the zfs modules have been loaded to initialize the qat +hardware as long as support is compiled in and the qat driver is present. +.sp +Use \fB1\fR for yes and \fB0\fR for no (default). +.RE + +.sp +.ne 2 +.na +\fBzfs_qat_compress_disable\fR (int) +.ad +.RS 12n +This tunable disables qat hardware acceleration for gzip compression. It +may be set after the zfs modules have been loaded to initialize the qat +hardware as long as support is compiled in and the qat driver is present. +.sp +Use \fB1\fR for yes and \fB0\fR for no (default). +.RE + +.sp +.ne 2 +.na +\fBzfs_qat_encrypt_disable\fR (int) +.ad +.RS 12n +This tunable disables qat hardware acceleration for AES-GCM encryption. It +may be set after the zfs modules have been loaded to initialize the qat +hardware as long as support is compiled in and the qat driver is present. +.sp +Use \fB1\fR for yes and \fB0\fR for no (default). +.RE + .sp .ne 2 .na @@ -2961,19 +3000,6 @@ Valid values are \fB1\fR (full), \fB2\fR (dev) and \fB3\fR (none). Default value: \fB1\fR. .RE -.sp -.ne 2 -.na -\fBzfs_qat_disable\fR (int) -.ad -.RS 12n -This tunable disables qat hardware acceleration for gzip compression and. -AES-GCM encryption. It is available only if qat acceleration is compiled in -and the qat driver is present. -.sp -Use \fB1\fR for yes and \fB0\fR for no (default). -.RE - .SH ZFS I/O SCHEDULER ZFS issues I/O operations to leaf vdevs to satisfy and complete I/Os. The I/O scheduler determines when and in what order those operations are diff --git a/module/zfs/qat.c b/module/zfs/qat.c index 0a4f9c72a..a6f024cb4 100644 --- a/module/zfs/qat.c +++ b/module/zfs/qat.c @@ -66,18 +66,6 @@ qat_mem_free_contig(void **pp_mem_addr) int qat_init(void) { - int ret; - - ret = qat_dc_init(); - if (ret != 0) - return (ret); - - ret = qat_crypt_init(); - if (ret != 0) { - qat_dc_fini(); - return (ret); - } - qat_ksp = kstat_create("zfs", 0, "qat", "misc", KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); @@ -86,6 +74,19 @@ qat_init(void) kstat_install(qat_ksp); } + /* + * Just set the disable flag when qat init failed, qat can be + * turned on again in post-process after zfs module is loaded, e.g.: + * echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable + */ + if (qat_dc_init() != 0) + zfs_qat_compress_disable = 1; + + if (qat_cy_init() != 0) { + zfs_qat_checksum_disable = 1; + zfs_qat_encrypt_disable = 1; + } + return (0); } @@ -97,7 +98,7 @@ qat_fini(void) qat_ksp = NULL; } - qat_crypt_fini(); + qat_cy_fini(); qat_dc_fini(); } diff --git a/module/zfs/qat.h b/module/zfs/qat.h index b2cd5a9c1..9014c0314 100644 --- a/module/zfs/qat.h +++ b/module/zfs/qat.h @@ -147,6 +147,9 @@ typedef struct qat_stats { QAT_STAT_INCR(stat, 1) extern qat_stats_t qat_stats; +extern int zfs_qat_compress_disable; +extern int zfs_qat_checksum_disable; +extern int zfs_qat_encrypt_disable; /* inlined for performance */ static inline struct page * @@ -167,8 +170,8 @@ void qat_mem_free_contig(void **pp_mem_addr); extern int qat_dc_init(void); extern void qat_dc_fini(void); -extern int qat_crypt_init(void); -extern void qat_crypt_fini(void); +extern int qat_cy_init(void); +extern void qat_cy_fini(void); extern int qat_init(void); extern void qat_fini(void); diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c index 8c1410c9e..1c5c0a4e7 100644 --- a/module/zfs/qat_compress.c +++ b/module/zfs/qat_compress.c @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include #include #include "qat.h" @@ -111,6 +113,9 @@ qat_dc_init(void) Cpa32U buff_meta_size = 0; CpaDcSessionSetupData sd = {0}; + if (qat_dc_init_done) + return (0); + status = cpaDcGetNumInstances(&num_inst); if (status != CPA_STATUS_SUCCESS) return (-1); @@ -252,6 +257,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2; Cpa32U bytes_left; Cpa32U dst_pages = 0; + Cpa32U adler32 = 0; char *data; struct page *page; struct page **in_pages = NULL; @@ -468,6 +474,12 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len, goto fail; } + /* verify adler checksum */ + adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ); + if (adler32 != BSWAP_32(dc_results.checksum)) { + status = CPA_STATUS_FAIL; + goto fail; + } *c_len = dc_results.produced; QAT_STAT_INCR(decomp_total_out_bytes, *c_len); } @@ -534,7 +546,30 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, return (ret); } -module_param(zfs_qat_compress_disable, int, 0644); -MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression"); +static int +param_set_qat_compress(const char *val, struct kernel_param *kp) +{ + int ret; + int *pvalue = kp->arg; + ret = param_set_int(val, kp); + if (ret) + return (ret); + /* + * zfs_qat_compress_disable = 0: enable qat compress + * try to initialize qat instance if it has not been done + */ + if (*pvalue == 0 && !qat_dc_init_done) { + ret = qat_dc_init(); + if (ret != 0) { + zfs_qat_compress_disable = 1; + return (ret); + } + } + return (ret); +} + +module_param_call(zfs_qat_compress_disable, param_set_qat_compress, + param_get_int, &zfs_qat_compress_disable, 0644); +MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression"); #endif diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c index 5a5113e68..34c19b582 100644 --- a/module/zfs/qat_crypt.c +++ b/module/zfs/qat_crypt.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include "lac/cpa_cy_im.h" @@ -50,7 +51,7 @@ static Cpa32U inst_num = 0; static Cpa16U num_inst = 0; static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; -static boolean_t qat_crypt_init_done = B_FALSE; +static boolean_t qat_cy_init_done = B_FALSE; int zfs_qat_encrypt_disable = 0; int zfs_qat_checksum_disable = 0; @@ -76,7 +77,7 @@ boolean_t qat_crypt_use_accel(size_t s_len) { return (!zfs_qat_encrypt_disable && - qat_crypt_init_done && + qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } @@ -85,26 +86,29 @@ boolean_t qat_checksum_use_accel(size_t s_len) { return (!zfs_qat_checksum_disable && - qat_crypt_init_done && + qat_cy_init_done && s_len >= QAT_MIN_BUF_SIZE && s_len <= QAT_MAX_BUF_SIZE); } void -qat_crypt_clean(void) +qat_cy_clean(void) { for (Cpa16U i = 0; i < num_inst; i++) cpaCyStopInstance(cy_inst_handles[i]); num_inst = 0; - qat_crypt_init_done = B_FALSE; + qat_cy_init_done = B_FALSE; } int -qat_crypt_init(void) +qat_cy_init(void) { CpaStatus status = CPA_STATUS_FAIL; + if (qat_cy_init_done) + return (0); + status = cpaCyGetNumInstances(&num_inst); if (status != CPA_STATUS_SUCCESS) return (-1); @@ -131,21 +135,21 @@ qat_crypt_init(void) goto error; } - qat_crypt_init_done = B_TRUE; + qat_cy_init_done = B_TRUE; return (0); error: - qat_crypt_clean(); + qat_cy_clean(); return (-1); } void -qat_crypt_fini(void) +qat_cy_fini(void) { - if (!qat_crypt_init_done) + if (!qat_cy_init_done) return; - qat_crypt_clean(); + qat_cy_clean(); } static CpaStatus @@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, nr_bufs * sizeof (CpaFlatBuffer)); if (status != CPA_STATUS_SUCCESS) goto fail; + status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult, + ZIO_DATA_MAC_LEN); + if (status != CPA_STATUS_SUCCESS) + goto fail; + status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv, + ZIO_DATA_IV_LEN); + if (status != CPA_STATUS_SUCCESS) + goto fail; + if (aad_len > 0) { + status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData, + aad_len); + if (status != CPA_STATUS_SUCCESS) + goto fail; + bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len); + } bytes_left = enc_len; data = src_buf; @@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, op_data.sessionCtx = cy_session_ctx; op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; - op_data.pIv = NULL; /* set this later as the J0 block */ - op_data.ivLenInBytes = 0; op_data.cryptoStartSrcOffsetInBytes = 0; op_data.messageLenToCipherInBytes = 0; op_data.hashStartSrcOffsetInBytes = 0; op_data.messageLenToHashInBytes = 0; - op_data.pDigestResult = 0; op_data.messageLenToCipherInBytes = enc_len; op_data.ivLenInBytes = ZIO_DATA_IV_LEN; - op_data.pDigestResult = digest_buf; - op_data.pAdditionalAuthData = aad_buf; - op_data.pIv = iv_buf; + bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN); cb.verify_result = CPA_FALSE; init_completion(&cb.complete); @@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, goto fail; } + /* save digest result to digest_buf */ + bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN); if (dir == QAT_ENCRYPT) QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); else @@ -435,6 +451,10 @@ fail: kunmap(out_pages[i]); cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); + if (aad_len > 0) + QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData); + QAT_PHYS_CONTIG_FREE(op_data.pIv); + QAT_PHYS_CONTIG_FREE(op_data.pDigestResult); QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData); QAT_PHYS_CONTIG_FREE(cy_session_ctx); @@ -557,10 +577,56 @@ fail: return (status); } -module_param(zfs_qat_encrypt_disable, int, 0644); -MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption"); +static int +param_set_qat_encrypt(const char *val, struct kernel_param *kp) +{ + int ret; + int *pvalue = kp->arg; + ret = param_set_int(val, kp); + if (ret) + return (ret); + /* + * zfs_qat_encrypt_disable = 0: enable qat encrypt + * try to initialize qat instance if it has not been done + */ + if (*pvalue == 0 && !qat_cy_init_done) { + ret = qat_cy_init(); + if (ret != 0) { + zfs_qat_encrypt_disable = 1; + return (ret); + } + } + return (ret); +} + +static int +param_set_qat_checksum(const char *val, struct kernel_param *kp) +{ + int ret; + int *pvalue = kp->arg; + ret = param_set_int(val, kp); + if (ret) + return (ret); + /* + * set_checksum_param_ops = 0: enable qat checksum + * try to initialize qat instance if it has not been done + */ + if (*pvalue == 0 && !qat_cy_init_done) { + ret = qat_cy_init(); + if (ret != 0) { + zfs_qat_checksum_disable = 1; + return (ret); + } + } + return (ret); +} + +module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt, + param_get_int, &zfs_qat_encrypt_disable, 0644); +MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption"); -module_param(zfs_qat_checksum_disable, int, 0644); -MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming"); +module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum, + param_get_int, &zfs_qat_checksum_disable, 0644); +MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming"); #endif