[mirror_zfs.git] / module / zfs / qat_crypt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * This file represents the QAT implementation of checksums and encryption.
 * Internally, QAT shares the same cryptographic instances for both of these
 * operations, so the code has been combined here. QAT data compression uses
 * compression instances, so that code is separated into qat_compress.c
 */

#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
#include "lac/cpa_cy_common.h"
#include "qat.h"

/*
 * Max instances in a QAT device, each instance is a channel to submit
 * jobs to QAT hardware, this is only for pre-allocating instances
 * and session arrays; the actual number of instances are defined in
 * the QAT driver's configure file.
 */
#define	QAT_CRYPT_MAX_INSTANCES		48

#define	MAX_PAGE_NUM			1024

static Cpa32U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_cy_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;

typedef struct cy_callback {
	CpaBoolean verify_result;
	struct completion complete;
} cy_callback_t;

static void
symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
    void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
{
	cy_callback_t *cb = p_callback;

	if (cb != NULL) {
		/* indicate that the function has been called */
		cb->verify_result = verify;
		complete(&cb->complete);
	}
}

boolean_t
qat_crypt_use_accel(size_t s_len)
{
	return (!zfs_qat_encrypt_disable &&
	    qat_cy_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

boolean_t
qat_checksum_use_accel(size_t s_len)
{
	return (!zfs_qat_checksum_disable &&
	    qat_cy_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

void
qat_cy_clean(void)
{
	for (Cpa16U i = 0; i < num_inst; i++)
		cpaCyStopInstance(cy_inst_handles[i]);

	num_inst = 0;
	qat_cy_init_done = B_FALSE;
}

int
qat_cy_init(void)
{
	CpaStatus status = CPA_STATUS_FAIL;

	if (qat_cy_init_done)
		return (0);

	status = cpaCyGetNumInstances(&num_inst);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	/* if the user has configured no QAT encryption units just return */
	if (num_inst == 0)
		return (0);

	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
		num_inst = QAT_CRYPT_MAX_INSTANCES;

	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	for (Cpa16U i = 0; i < num_inst; i++) {
		status = cpaCySetAddressTranslation(cy_inst_handles[i],
		    (void *)virt_to_phys);
		if (status != CPA_STATUS_SUCCESS)
			goto error;

		status = cpaCyStartInstance(cy_inst_handles[i]);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	qat_cy_init_done = B_TRUE;
	return (0);

error:
	qat_cy_clean();
	return (-1);
}

void
qat_cy_fini(void)
{
	if (!qat_cy_init_done)
		return;

	qat_cy_clean();
}

static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
    Cpa64U crypt, Cpa32U aad_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U ciper_algorithm;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
		return (CPA_STATUS_FAIL);
	} else {
		ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
		hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	}

	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	sd.cipherSetupData.pCipherKey = key->ck_data;
	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	if (dir == QAT_ENCRYPT) {
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	} else {
		ASSERT3U(dir, ==, QAT_DECRYPT);
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	}

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	/*
	 * ZFS's SHA512 checksum is actually SHA512/256, which uses
	 * a different IV from standard SHA512. QAT does not support
	 * SHA512/256, so we can only support SHA256.
	 */
	if (cksum == ZIO_CHECKSUM_SHA256)
		hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	else
		return (CPA_STATUS_FAIL);

	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_HASH;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
    CpaBufferList *src, CpaBufferList *dst)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U meta_size = 0;

	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	if (status != CPA_STATUS_SUCCESS)
		goto error;

	if (src != dst) {
		status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
		    meta_size);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	return (CPA_STATUS_SUCCESS);

error:
	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	if (src != dst)
		QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);

	return (status);
}

int
qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
    uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
    crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaBufferList dst_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	CpaFlatBuffer *flat_dst_buf_array = NULL;
	CpaFlatBuffer *flat_dst_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	struct page *out_pages[MAX_PAGE_NUM];
	Cpa32U in_page_num = 0;
	Cpa32U out_page_num = 0;
	Cpa32U in_page_off = 0;
	Cpa32U out_page_off = 0;

	if (dir == QAT_ENCRYPT) {
		QAT_STAT_BUMP(encrypt_requests);
		QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	} else {
		QAT_STAT_BUMP(decrypt_requests);
		QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	}

	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	    &cy_session_ctx, key, crypt, aad_len);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count CCM as a failure since it's not supported */
		if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
			QAT_STAT_BUMP(crypt_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &dst_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
	    ZIO_DATA_MAC_LEN);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
	    ZIO_DATA_IV_LEN);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	if (aad_len > 0) {
		status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
		    aad_len);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;
		bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
	}

	bytes_left = enc_len;
	data = src_buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		in_page_off = ((long)data & ~PAGE_MASK);
		in_pages[in_page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		in_page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = in_page_num;

	bytes_left = enc_len;
	data = dst_buf;
	flat_dst_buf = flat_dst_buf_array;
	while (bytes_left > 0) {
		out_page_off = ((long)data & ~PAGE_MASK);
		out_pages[out_page_num] = qat_mem_to_page(data);
		flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
		    out_page_off;
		flat_dst_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
		data += flat_dst_buf->dataLenInBytes;
		bytes_left -= flat_dst_buf->dataLenInBytes;
		flat_dst_buf++;
		out_page_num++;
	}
	dst_buffer_list.pBuffers = flat_dst_buf_array;
	dst_buffer_list.numBuffers = out_page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.cryptoStartSrcOffsetInBytes = 0;
	op_data.messageLenToCipherInBytes = 0;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = 0;
	op_data.messageLenToCipherInBytes = enc_len;
	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
	bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &dst_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	if (cb.verify_result == CPA_FALSE) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	/* save digest result to digest_buf */
	bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
	if (dir == QAT_ENCRYPT)
		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	else
		QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(crypt_fails);

	for (i = 0; i < in_page_num; i++)
		kunmap(in_pages[i]);
	for (i = 0; i < out_page_num; i++)
		kunmap(out_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	if (aad_len > 0)
		QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
	QAT_PHYS_CONTIG_FREE(op_data.pIv);
	QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);

	return (status);
}

int
qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
{
	CpaStatus status;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	Cpa8U *digest_buffer = NULL;
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	Cpa32U page_num = 0;
	Cpa32U page_off = 0;

	QAT_STAT_BUMP(cksum_requests);
	QAT_STAT_INCR(cksum_total_in_bytes, size);

	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_checksum_session_ctx(cy_inst_handle,
	    &cy_session_ctx, cksum);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count unsupported checksums as a failure */
		if (cksum == ZIO_CHECKSUM_SHA256 ||
		    cksum == ZIO_CHECKSUM_SHA512)
			QAT_STAT_BUMP(cksum_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &src_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
	    sizeof (zio_cksum_t));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	bytes_left = size;
	data = buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		in_pages[page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = size;
	op_data.pDigestResult = digest_buffer;

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &src_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}
	if (cb.verify_result == CPA_FALSE) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(cksum_fails);

	for (i = 0; i < page_num; i++)
		kunmap(in_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(digest_buffer);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);

	return (status);
}

static int
param_set_qat_encrypt(const char *val, struct kernel_param *kp)
{
	int ret;
	int *pvalue = kp->arg;
	ret = param_set_int(val, kp);
	if (ret)
		return (ret);
	/*
	 * zfs_qat_encrypt_disable = 0: enable qat encrypt
	 * try to initialize qat instance if it has not been done
	 */
	if (*pvalue == 0 && !qat_cy_init_done) {
		ret = qat_cy_init();
		if (ret != 0) {
			zfs_qat_encrypt_disable = 1;
			return (ret);
		}
	}
	return (ret);
}

static int
param_set_qat_checksum(const char *val, struct kernel_param *kp)
{
	int ret;
	int *pvalue = kp->arg;
	ret = param_set_int(val, kp);
	if (ret)
		return (ret);
	/*
	 * set_checksum_param_ops = 0: enable qat checksum
	 * try to initialize qat instance if it has not been done
	 */
	if (*pvalue == 0 && !qat_cy_init_done) {
		ret = qat_cy_init();
		if (ret != 0) {
			zfs_qat_checksum_disable = 1;
			return (ret);
		}
	}
	return (ret);
}

module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
    param_get_int, &zfs_qat_encrypt_disable, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");

module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
    param_get_int, &zfs_qat_checksum_disable, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");

#endif
Commit	Line	Data
cf637391 TC	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21
38742209 TC	22	/*
	23	* This file represents the QAT implementation of checksums and encryption.
	24	* Internally, QAT shares the same cryptographic instances for both of these
	25	* operations, so the code has been combined here. QAT data compression uses
	26	* compression instances, so that code is separated into qat_compress.c
	27	*/
	28
cf637391 TC	29	#if defined(_KERNEL) && defined(HAVE_QAT)
	30	#include <linux/slab.h>
	31	#include <linux/vmalloc.h>
	32	#include <linux/pagemap.h>
	33	#include <linux/completion.h>
5090f727	34	#include <linux/mod_compat.h>
cf637391 TC	35	#include <sys/zfs_context.h>
	36	#include <sys/zio_crypt.h>
	37	#include "lac/cpa_cy_im.h"
38742209	38	#include "lac/cpa_cy_common.h"
cf637391 TC	39	#include "qat.h"
	40
	41	/*
8d9e7c8f TC	42	* Max instances in a QAT device, each instance is a channel to submit
	43	* jobs to QAT hardware, this is only for pre-allocating instances
	44	* and session arrays; the actual number of instances are defined in
cf637391 TC	45	* the QAT driver's configure file.
	46	*/
	47	#define QAT_CRYPT_MAX_INSTANCES 48
	48
	49	#define MAX_PAGE_NUM 1024
	50
bc8a6a60	51	static Cpa32U inst_num = 0;
cf637391 TC	52	static Cpa16U num_inst = 0;
cf637391 TC	53	static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
5090f727	54	static boolean_t qat_cy_init_done = B_FALSE;
38742209 TC	55	int zfs_qat_encrypt_disable = 0;
38742209 TC	56	int zfs_qat_checksum_disable = 0;
cf637391 TC	57
	58	typedef struct cy_callback {
	59	CpaBoolean verify_result;
	60	struct completion complete;
	61	} cy_callback_t;
	62
	63	static void
	64	symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
	65	void op_data, CpaBufferList buf_list_dst, CpaBoolean verify)
	66	{
	67	cy_callback_t *cb = p_callback;
	68
	69	if (cb != NULL) {
	70	/* indicate that the function has been called */
	71	cb->verify_result = verify;
	72	complete(&cb->complete);
	73	}
	74	}
	75
	76	boolean_t
	77	qat_crypt_use_accel(size_t s_len)
	78	{
38742209	79	return (!zfs_qat_encrypt_disable &&
5090f727	80	qat_cy_init_done &&
38742209 TC	81	s_len >= QAT_MIN_BUF_SIZE &&
	82	s_len <= QAT_MAX_BUF_SIZE);
	83	}
	84
	85	boolean_t
	86	qat_checksum_use_accel(size_t s_len)
	87	{
	88	return (!zfs_qat_checksum_disable &&
5090f727	89	qat_cy_init_done &&
cf637391 TC	90	s_len >= QAT_MIN_BUF_SIZE &&
	91	s_len <= QAT_MAX_BUF_SIZE);
	92	}
	93
	94	void
5090f727	95	qat_cy_clean(void)
cf637391	96	{
bc8a6a60	97	for (Cpa16U i = 0; i < num_inst; i++)
cf637391 TC	98	cpaCyStopInstance(cy_inst_handles[i]);
	99
	100	num_inst = 0;
5090f727	101	qat_cy_init_done = B_FALSE;
cf637391 TC	102	}
	103
	104	int
5090f727	105	qat_cy_init(void)
cf637391	106	{
cf637391 TC	107	CpaStatus status = CPA_STATUS_FAIL;
cf637391 TC	108
5090f727 CZ	109	if (qat_cy_init_done)
	110	return (0);
	111
cf637391 TC	112	status = cpaCyGetNumInstances(&num_inst);
	113	if (status != CPA_STATUS_SUCCESS)
	114	return (-1);
	115
	116	/* if the user has configured no QAT encryption units just return */
	117	if (num_inst == 0)
	118	return (0);
	119
	120	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
	121	num_inst = QAT_CRYPT_MAX_INSTANCES;
	122
	123	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	124	if (status != CPA_STATUS_SUCCESS)
	125	return (-1);
	126
bc8a6a60	127	for (Cpa16U i = 0; i < num_inst; i++) {
cf637391 TC	128	status = cpaCySetAddressTranslation(cy_inst_handles[i],
	129	(void *)virt_to_phys);
	130	if (status != CPA_STATUS_SUCCESS)
	131	goto error;
	132
	133	status = cpaCyStartInstance(cy_inst_handles[i]);
	134	if (status != CPA_STATUS_SUCCESS)
	135	goto error;
	136	}
	137
5090f727	138	qat_cy_init_done = B_TRUE;
cf637391 TC	139	return (0);
	140
	141	error:
5090f727	142	qat_cy_clean();
cf637391 TC	143	return (-1);
	144	}
	145
	146	void
5090f727	147	qat_cy_fini(void)
cf637391	148	{
5090f727	149	if (!qat_cy_init_done)
cf637391 TC	150	return;
cf637391 TC	151
5090f727	152	qat_cy_clean();
cf637391 TC	153	}
	154
	155	static CpaStatus
38742209	156	qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
cf637391 TC	157	CpaCySymSessionCtx *cy_session_ctx, crypto_key_t key,
	158	Cpa64U crypt, Cpa32U aad_len)
	159	{
	160	CpaStatus status = CPA_STATUS_SUCCESS;
	161	Cpa32U ctx_size;
	162	Cpa32U ciper_algorithm;
	163	Cpa32U hash_algorithm;
	164	CpaCySymSessionSetupData sd = { 0 };
	165
	166	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
	167	return (CPA_STATUS_FAIL);
	168	} else {
	169	ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
	170	hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	171	}
	172
	173	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	174	sd.cipherSetupData.pCipherKey = key->ck_data;
	175	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	176	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	177	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	178	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	179	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	180	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	181	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	182	sd.digestIsAppended = CPA_FALSE;
	183	sd.verifyDigest = CPA_FALSE;
	184
	185	if (dir == QAT_ENCRYPT) {
	186	sd.cipherSetupData.cipherDirection =
	187	CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
	188	sd.algChainOrder =
	189	CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	190	} else {
	191	ASSERT3U(dir, ==, QAT_DECRYPT);
	192	sd.cipherSetupData.cipherDirection =
	193	CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
	194	sd.algChainOrder =
	195	CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	196	}
	197
	198	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	199	if (status != CPA_STATUS_SUCCESS)
	200	return (status);
	201
	202	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	203	if (status != CPA_STATUS_SUCCESS)
	204	return (status);
	205
	206	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	207	*cy_session_ctx);
	208	if (status != CPA_STATUS_SUCCESS) {
	209	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	210	return (status);
	211	}
	212
	213	return (CPA_STATUS_SUCCESS);
	214	}
	215
	216	static CpaStatus
38742209 TC	217	qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
	218	CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
	219	{
	220	CpaStatus status = CPA_STATUS_SUCCESS;
	221	Cpa32U ctx_size;
	222	Cpa32U hash_algorithm;
	223	CpaCySymSessionSetupData sd = { 0 };
	224
	225	/*
	226	* ZFS's SHA512 checksum is actually SHA512/256, which uses
	227	* a different IV from standard SHA512. QAT does not support
	228	* SHA512/256, so we can only support SHA256.
	229	*/
	230	if (cksum == ZIO_CHECKSUM_SHA256)
	231	hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	232	else
	233	return (CPA_STATUS_FAIL);
	234
	235	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	236	sd.symOperation = CPA_CY_SYM_OP_HASH;
	237	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	238	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	239	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	240	sd.digestIsAppended = CPA_FALSE;
	241	sd.verifyDigest = CPA_FALSE;
	242
	243	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	244	if (status != CPA_STATUS_SUCCESS)
	245	return (status);
	246
	247	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	248	if (status != CPA_STATUS_SUCCESS)
	249	return (status);
	250
	251	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	252	*cy_session_ctx);
	253	if (status != CPA_STATUS_SUCCESS) {
	254	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	255	return (status);
	256	}
	257
	258	return (CPA_STATUS_SUCCESS);
	259	}
	260
	261	static CpaStatus
	262	qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
cf637391 TC	263	CpaBufferList src, CpaBufferList dst)
	264	{
	265	CpaStatus status = CPA_STATUS_SUCCESS;
	266	Cpa32U meta_size = 0;
	267
	268	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	269	if (status != CPA_STATUS_SUCCESS)
	270	return (status);
	271
cf637391 TC	272	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	273	if (status != CPA_STATUS_SUCCESS)
	274	goto error;
	275
	276	if (src != dst) {
cf637391 TC	277	status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
	278	meta_size);
	279	if (status != CPA_STATUS_SUCCESS)
	280	goto error;
	281	}
	282
	283	return (CPA_STATUS_SUCCESS);
	284
	285	error:
	286	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	287	if (src != dst)
	288	QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
	289
	290	return (status);
	291	}
	292
	293	int
	294	qat_crypt(qat_encrypt_dir_t dir, uint8_t src_buf, uint8_t dst_buf,
	295	uint8_t aad_buf, uint32_t aad_len, uint8_t iv_buf, uint8_t *digest_buf,
	296	crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
	297	{
	298	CpaStatus status = CPA_STATUS_SUCCESS;
	299	Cpa16U i;
	300	CpaInstanceHandle cy_inst_handle;
17dd8835	301	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
cf637391	302	Cpa32U bytes_left = 0;
17dd8835	303	Cpa8S *data = NULL;
cf637391 TC	304	CpaCySymSessionCtx *cy_session_ctx = NULL;
	305	cy_callback_t cb;
	306	CpaCySymOpData op_data = { 0 };
	307	CpaBufferList src_buffer_list = { 0 };
	308	CpaBufferList dst_buffer_list = { 0 };
	309	CpaFlatBuffer *flat_src_buf_array = NULL;
	310	CpaFlatBuffer *flat_src_buf = NULL;
	311	CpaFlatBuffer *flat_dst_buf_array = NULL;
	312	CpaFlatBuffer *flat_dst_buf = NULL;
	313	struct page *in_pages[MAX_PAGE_NUM];
	314	struct page *out_pages[MAX_PAGE_NUM];
17dd8835 TC	315	Cpa32U in_page_num = 0;
17dd8835 TC	316	Cpa32U out_page_num = 0;
38742209 TC	317	Cpa32U in_page_off = 0;
38742209 TC	318	Cpa32U out_page_off = 0;
cf637391 TC	319
	320	if (dir == QAT_ENCRYPT) {
	321	QAT_STAT_BUMP(encrypt_requests);
	322	QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	323	} else {
	324	QAT_STAT_BUMP(decrypt_requests);
	325	QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	326	}
	327
bc8a6a60	328	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
cf637391 TC	329	cy_inst_handle = cy_inst_handles[i];
cf637391 TC	330
38742209 TC	331	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	332	&cy_session_ctx, key, crypt, aad_len);
	333	if (status != CPA_STATUS_SUCCESS) {
	334	/* don't count CCM as a failure since it's not supported */
	335	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
	336	QAT_STAT_BUMP(crypt_fails);
cf637391	337	return (status);
38742209	338	}
cf637391	339
17dd8835 TC	340	/*
	341	* We increment nr_bufs by 2 to allow us to handle non
	342	* page-aligned buffer addresses and buffers whose sizes
	343	* are not divisible by PAGE_SIZE.
	344	*/
38742209 TC	345	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
38742209 TC	346	&src_buffer_list, &dst_buffer_list);
cf637391 TC	347	if (status != CPA_STATUS_SUCCESS)
	348	goto fail;
	349
	350	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	351	nr_bufs * sizeof (CpaFlatBuffer));
	352	if (status != CPA_STATUS_SUCCESS)
	353	goto fail;
	354	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	355	nr_bufs * sizeof (CpaFlatBuffer));
	356	if (status != CPA_STATUS_SUCCESS)
	357	goto fail;
5090f727 CZ	358	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
	359	ZIO_DATA_MAC_LEN);
	360	if (status != CPA_STATUS_SUCCESS)
	361	goto fail;
	362	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
	363	ZIO_DATA_IV_LEN);
	364	if (status != CPA_STATUS_SUCCESS)
	365	goto fail;
	366	if (aad_len > 0) {
	367	status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
	368	aad_len);
	369	if (status != CPA_STATUS_SUCCESS)
	370	goto fail;
	371	bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
	372	}
cf637391 TC	373
cf637391 TC	374	bytes_left = enc_len;
17dd8835	375	data = src_buf;
cf637391	376	flat_src_buf = flat_src_buf_array;
cf637391	377	while (bytes_left > 0) {
17dd8835 TC	378	in_page_off = ((long)data & ~PAGE_MASK);
	379	in_pages[in_page_num] = qat_mem_to_page(data);
	380	flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
38742209	381	flat_src_buf->dataLenInBytes =
17dd8835 TC	382	min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
17dd8835 TC	383	data += flat_src_buf->dataLenInBytes;
cf637391 TC	384	bytes_left -= flat_src_buf->dataLenInBytes;
cf637391 TC	385	flat_src_buf++;
17dd8835	386	in_page_num++;
cf637391 TC	387	}
cf637391 TC	388	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835 TC	389	src_buffer_list.numBuffers = in_page_num;
	390
	391	bytes_left = enc_len;
	392	data = dst_buf;
	393	flat_dst_buf = flat_dst_buf_array;
	394	while (bytes_left > 0) {
	395	out_page_off = ((long)data & ~PAGE_MASK);
	396	out_pages[out_page_num] = qat_mem_to_page(data);
	397	flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
	398	out_page_off;
	399	flat_dst_buf->dataLenInBytes =
	400	min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
	401	data += flat_dst_buf->dataLenInBytes;
	402	bytes_left -= flat_dst_buf->dataLenInBytes;
	403	flat_dst_buf++;
	404	out_page_num++;
	405	}
cf637391	406	dst_buffer_list.pBuffers = flat_dst_buf_array;
17dd8835	407	dst_buffer_list.numBuffers = out_page_num;
cf637391 TC	408
	409	op_data.sessionCtx = cy_session_ctx;
	410	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
cf637391 TC	411	op_data.cryptoStartSrcOffsetInBytes = 0;
	412	op_data.messageLenToCipherInBytes = 0;
	413	op_data.hashStartSrcOffsetInBytes = 0;
	414	op_data.messageLenToHashInBytes = 0;
cf637391 TC	415	op_data.messageLenToCipherInBytes = enc_len;
cf637391 TC	416	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
5090f727	417	bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
cf637391 TC	418
	419	cb.verify_result = CPA_FALSE;
	420	init_completion(&cb.complete);
	421	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	422	&src_buffer_list, &dst_buffer_list, NULL);
	423	if (status != CPA_STATUS_SUCCESS)
	424	goto fail;
	425
	426	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	427	QAT_TIMEOUT_MS)) {
	428	status = CPA_STATUS_FAIL;
	429	goto fail;
	430	}
	431
	432	if (cb.verify_result == CPA_FALSE) {
	433	status = CPA_STATUS_FAIL;
	434	goto fail;
	435	}
	436
5090f727 CZ	437	/* save digest result to digest_buf */
5090f727 CZ	438	bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
cf637391 TC	439	if (dir == QAT_ENCRYPT)
	440	QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	441	else
	442	QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
	443
	444	fail:
38742209	445	if (status != CPA_STATUS_SUCCESS)
cf637391 TC	446	QAT_STAT_BUMP(crypt_fails);
cf637391 TC	447
17dd8835	448	for (i = 0; i < in_page_num; i++)
cf637391	449	kunmap(in_pages[i]);
17dd8835	450	for (i = 0; i < out_page_num; i++)
cf637391	451	kunmap(out_pages[i]);
cf637391 TC	452
cf637391 TC	453	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
5090f727 CZ	454	if (aad_len > 0)
	455	QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
	456	QAT_PHYS_CONTIG_FREE(op_data.pIv);
	457	QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
cf637391 TC	458	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	459	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	460	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	461	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	462	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
	463
	464	return (status);
	465	}
	466
38742209 TC	467	int
	468	qat_checksum(uint64_t cksum, uint8_t buf, uint64_t size, zio_cksum_t zcp)
	469	{
	470	CpaStatus status;
	471	Cpa16U i;
	472	CpaInstanceHandle cy_inst_handle;
17dd8835	473	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
38742209 TC	474	Cpa32U bytes_left = 0;
	475	Cpa8S *data = NULL;
	476	CpaCySymSessionCtx *cy_session_ctx = NULL;
	477	cy_callback_t cb;
45001b94	478	Cpa8U *digest_buffer = NULL;
38742209 TC	479	CpaCySymOpData op_data = { 0 };
	480	CpaBufferList src_buffer_list = { 0 };
	481	CpaFlatBuffer *flat_src_buf_array = NULL;
	482	CpaFlatBuffer *flat_src_buf = NULL;
	483	struct page *in_pages[MAX_PAGE_NUM];
17dd8835	484	Cpa32U page_num = 0;
38742209 TC	485	Cpa32U page_off = 0;
	486
	487	QAT_STAT_BUMP(cksum_requests);
	488	QAT_STAT_INCR(cksum_total_in_bytes, size);
	489
bc8a6a60	490	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
38742209 TC	491	cy_inst_handle = cy_inst_handles[i];
	492
	493	status = qat_init_checksum_session_ctx(cy_inst_handle,
	494	&cy_session_ctx, cksum);
	495	if (status != CPA_STATUS_SUCCESS) {
	496	/* don't count unsupported checksums as a failure */
	497	if (cksum == ZIO_CHECKSUM_SHA256 \|\|
	498	cksum == ZIO_CHECKSUM_SHA512)
	499	QAT_STAT_BUMP(cksum_fails);
	500	return (status);
	501	}
	502
17dd8835 TC	503	/*
	504	* We increment nr_bufs by 2 to allow us to handle non
	505	* page-aligned buffer addresses and buffers whose sizes
	506	* are not divisible by PAGE_SIZE.
	507	*/
38742209 TC	508	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	509	&src_buffer_list, &src_buffer_list);
	510	if (status != CPA_STATUS_SUCCESS)
	511	goto fail;
	512
	513	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	514	nr_bufs * sizeof (CpaFlatBuffer));
	515	if (status != CPA_STATUS_SUCCESS)
	516	goto fail;
45001b94 CZ	517	status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
	518	sizeof (zio_cksum_t));
	519	if (status != CPA_STATUS_SUCCESS)
	520	goto fail;
38742209 TC	521
	522	bytes_left = size;
	523	data = buf;
	524	flat_src_buf = flat_src_buf_array;
	525	while (bytes_left > 0) {
	526	page_off = ((long)data & ~PAGE_MASK);
	527	in_pages[page_num] = qat_mem_to_page(data);
	528	flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
	529	flat_src_buf->dataLenInBytes =
17dd8835	530	min((long)PAGE_SIZE - page_off, (long)bytes_left);
38742209 TC	531	data += flat_src_buf->dataLenInBytes;
	532	bytes_left -= flat_src_buf->dataLenInBytes;
	533	flat_src_buf++;
	534	page_num++;
	535	}
	536	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835	537	src_buffer_list.numBuffers = page_num;
38742209 TC	538
	539	op_data.sessionCtx = cy_session_ctx;
	540	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	541	op_data.hashStartSrcOffsetInBytes = 0;
	542	op_data.messageLenToHashInBytes = size;
	543	op_data.pDigestResult = digest_buffer;
	544
	545	cb.verify_result = CPA_FALSE;
	546	init_completion(&cb.complete);
	547	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	548	&src_buffer_list, &src_buffer_list, NULL);
	549	if (status != CPA_STATUS_SUCCESS)
	550	goto fail;
	551
	552	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	553	QAT_TIMEOUT_MS)) {
	554	status = CPA_STATUS_FAIL;
	555	goto fail;
	556	}
45001b94 CZ	557	if (cb.verify_result == CPA_FALSE) {
	558	status = CPA_STATUS_FAIL;
	559	goto fail;
	560	}
38742209 TC	561
	562	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
	563
	564	fail:
	565	if (status != CPA_STATUS_SUCCESS)
	566	QAT_STAT_BUMP(cksum_fails);
	567
	568	for (i = 0; i < page_num; i++)
	569	kunmap(in_pages[i]);
	570
	571	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
45001b94	572	QAT_PHYS_CONTIG_FREE(digest_buffer);
38742209 TC	573	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	574	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	575	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	576
	577	return (status);
	578	}
	579
5090f727 CZ	580	static int
	581	param_set_qat_encrypt(const char val, struct kernel_param kp)
	582	{
	583	int ret;
	584	int *pvalue = kp->arg;
	585	ret = param_set_int(val, kp);
	586	if (ret)
	587	return (ret);
	588	/*
	589	* zfs_qat_encrypt_disable = 0: enable qat encrypt
	590	* try to initialize qat instance if it has not been done
	591	*/
	592	if (*pvalue == 0 && !qat_cy_init_done) {
	593	ret = qat_cy_init();
	594	if (ret != 0) {
	595	zfs_qat_encrypt_disable = 1;
	596	return (ret);
	597	}
	598	}
	599	return (ret);
	600	}
	601
	602	static int
	603	param_set_qat_checksum(const char val, struct kernel_param kp)
	604	{
	605	int ret;
	606	int *pvalue = kp->arg;
	607	ret = param_set_int(val, kp);
	608	if (ret)
	609	return (ret);
	610	/*
	611	* set_checksum_param_ops = 0: enable qat checksum
	612	* try to initialize qat instance if it has not been done
	613	*/
	614	if (*pvalue == 0 && !qat_cy_init_done) {
	615	ret = qat_cy_init();
	616	if (ret != 0) {
	617	zfs_qat_checksum_disable = 1;
	618	return (ret);
	619	}
	620	}
	621	return (ret);
	622	}
	623
	624	module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
	625	param_get_int, &zfs_qat_encrypt_disable, 0644);
	626	MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
38742209	627
5090f727 CZ	628	module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
	629	param_get_int, &zfs_qat_checksum_disable, 0644);
	630	MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
cf637391 TC	631
cf637391 TC	632	#endif