[mirror_zfs.git] / module / zfs / qat_crypt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * This file represents the QAT implementation of checksums and encryption.
 * Internally, QAT shares the same cryptographic instances for both of these
 * operations, so the code has been combined here. QAT data compression uses
 * compression instances, so that code is separated into qat_compress.c
 */

#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
#include "lac/cpa_cy_common.h"
#include "qat.h"

/*
 * Max instances in QAT device, each instance is a channel to submit
 * jobs to QAT hardware, this is only for pre-allocating instance,
 * and session arrays, the actual number of instances are defined in
 * the QAT driver's configure file.
 */
#define	QAT_CRYPT_MAX_INSTANCES		48

#define	MAX_PAGE_NUM			1024

static Cpa16U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_crypt_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;

typedef struct cy_callback {
	CpaBoolean verify_result;
	struct completion complete;
} cy_callback_t;

static void
symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
    void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
{
	cy_callback_t *cb = p_callback;

	if (cb != NULL) {
		/* indicate that the function has been called */
		cb->verify_result = verify;
		complete(&cb->complete);
	}
}

boolean_t
qat_crypt_use_accel(size_t s_len)
{
	return (!zfs_qat_encrypt_disable &&
	    qat_crypt_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

boolean_t
qat_checksum_use_accel(size_t s_len)
{
	return (!zfs_qat_checksum_disable &&
	    qat_crypt_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

void
qat_crypt_clean(void)
{
	for (Cpa32U i = 0; i < num_inst; i++)
		cpaCyStopInstance(cy_inst_handles[i]);

	num_inst = 0;
	qat_crypt_init_done = B_FALSE;
}

int
qat_crypt_init(void)
{
	Cpa32U i;
	CpaStatus status = CPA_STATUS_FAIL;

	status = cpaCyGetNumInstances(&num_inst);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	/* if the user has configured no QAT encryption units just return */
	if (num_inst == 0)
		return (0);

	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
		num_inst = QAT_CRYPT_MAX_INSTANCES;

	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	for (i = 0; i < num_inst; i++) {
		status = cpaCySetAddressTranslation(cy_inst_handles[i],
		    (void *)virt_to_phys);
		if (status != CPA_STATUS_SUCCESS)
			goto error;

		status = cpaCyStartInstance(cy_inst_handles[i]);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	qat_crypt_init_done = B_TRUE;
	return (0);

error:
	qat_crypt_clean();
	return (-1);
}

void
qat_crypt_fini(void)
{
	if (!qat_crypt_init_done)
		return;

	qat_crypt_clean();
}

static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
    Cpa64U crypt, Cpa32U aad_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U ciper_algorithm;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
		return (CPA_STATUS_FAIL);
	} else {
		ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
		hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	}

	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	sd.cipherSetupData.pCipherKey = key->ck_data;
	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	if (dir == QAT_ENCRYPT) {
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	} else {
		ASSERT3U(dir, ==, QAT_DECRYPT);
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	}

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	/*
	 * ZFS's SHA512 checksum is actually SHA512/256, which uses
	 * a different IV from standard SHA512. QAT does not support
	 * SHA512/256, so we can only support SHA256.
	 */
	if (cksum == ZIO_CHECKSUM_SHA256)
		hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	else
		return (CPA_STATUS_FAIL);

	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_HASH;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
    CpaBufferList *src, CpaBufferList *dst)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U meta_size = 0;

	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	if (status != CPA_STATUS_SUCCESS)
		goto error;

	if (src != dst) {
		status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
		    meta_size);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	return (CPA_STATUS_SUCCESS);

error:
	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	if (src != dst)
		QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);

	return (status);
}

int
qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
    uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
    crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaBufferList dst_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	CpaFlatBuffer *flat_dst_buf_array = NULL;
	CpaFlatBuffer *flat_dst_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	struct page *out_pages[MAX_PAGE_NUM];
	Cpa32U in_page_num = 0;
	Cpa32U out_page_num = 0;
	Cpa32U in_page_off = 0;
	Cpa32U out_page_off = 0;

	if (dir == QAT_ENCRYPT) {
		QAT_STAT_BUMP(encrypt_requests);
		QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	} else {
		QAT_STAT_BUMP(decrypt_requests);
		QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	}

	i = atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	    &cy_session_ctx, key, crypt, aad_len);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count CCM as a failure since it's not supported */
		if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
			QAT_STAT_BUMP(crypt_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &dst_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	bytes_left = enc_len;
	data = src_buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		in_page_off = ((long)data & ~PAGE_MASK);
		in_pages[in_page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		in_page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = in_page_num;

	bytes_left = enc_len;
	data = dst_buf;
	flat_dst_buf = flat_dst_buf_array;
	while (bytes_left > 0) {
		out_page_off = ((long)data & ~PAGE_MASK);
		out_pages[out_page_num] = qat_mem_to_page(data);
		flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
		    out_page_off;
		flat_dst_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
		data += flat_dst_buf->dataLenInBytes;
		bytes_left -= flat_dst_buf->dataLenInBytes;
		flat_dst_buf++;
		out_page_num++;
	}
	dst_buffer_list.pBuffers = flat_dst_buf_array;
	dst_buffer_list.numBuffers = out_page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.pIv = NULL; /* set this later as the J0 block */
	op_data.ivLenInBytes = 0;
	op_data.cryptoStartSrcOffsetInBytes = 0;
	op_data.messageLenToCipherInBytes = 0;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = 0;
	op_data.pDigestResult = 0;
	op_data.messageLenToCipherInBytes = enc_len;
	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
	op_data.pDigestResult = digest_buf;
	op_data.pAdditionalAuthData = aad_buf;
	op_data.pIv = iv_buf;

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &dst_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	if (cb.verify_result == CPA_FALSE) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	if (dir == QAT_ENCRYPT)
		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	else
		QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(crypt_fails);

	for (i = 0; i < in_page_num; i++)
		kunmap(in_pages[i]);
	for (i = 0; i < out_page_num; i++)
		kunmap(out_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);

	return (status);
}

int
qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
{
	CpaStatus status;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	Cpa8U digest_buffer[sizeof (zio_cksum_t)];
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	Cpa32U page_num = 0;
	Cpa32U page_off = 0;

	QAT_STAT_BUMP(cksum_requests);
	QAT_STAT_INCR(cksum_total_in_bytes, size);

	i = atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_checksum_session_ctx(cy_inst_handle,
	    &cy_session_ctx, cksum);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count unsupported checksums as a failure */
		if (cksum == ZIO_CHECKSUM_SHA256 ||
		    cksum == ZIO_CHECKSUM_SHA512)
			QAT_STAT_BUMP(cksum_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &src_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	bytes_left = size;
	data = buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		in_pages[page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = size;
	op_data.pDigestResult = digest_buffer;

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &src_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(cksum_fails);

	for (i = 0; i < page_num; i++)
		kunmap(in_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);

	return (status);
}

module_param(zfs_qat_encrypt_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");

module_param(zfs_qat_checksum_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");

#endif
Commit	Line	Data
cf637391 TC	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21
38742209 TC	22	/*
	23	* This file represents the QAT implementation of checksums and encryption.
	24	* Internally, QAT shares the same cryptographic instances for both of these
	25	* operations, so the code has been combined here. QAT data compression uses
	26	* compression instances, so that code is separated into qat_compress.c
	27	*/
	28
cf637391 TC	29	#if defined(_KERNEL) && defined(HAVE_QAT)
	30	#include <linux/slab.h>
	31	#include <linux/vmalloc.h>
	32	#include <linux/pagemap.h>
	33	#include <linux/completion.h>
	34	#include <sys/zfs_context.h>
	35	#include <sys/zio_crypt.h>
	36	#include "lac/cpa_cy_im.h"
38742209	37	#include "lac/cpa_cy_common.h"
cf637391 TC	38	#include "qat.h"
	39
	40	/*
	41	* Max instances in QAT device, each instance is a channel to submit
	42	* jobs to QAT hardware, this is only for pre-allocating instance,
	43	* and session arrays, the actual number of instances are defined in
	44	* the QAT driver's configure file.
	45	*/
	46	#define QAT_CRYPT_MAX_INSTANCES 48
	47
	48	#define MAX_PAGE_NUM 1024
	49
cf637391 TC	50	static Cpa16U inst_num = 0;
	51	static Cpa16U num_inst = 0;
	52	static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
38742209 TC	53	static boolean_t qat_crypt_init_done = B_FALSE;
	54	int zfs_qat_encrypt_disable = 0;
	55	int zfs_qat_checksum_disable = 0;
cf637391 TC	56
	57	typedef struct cy_callback {
	58	CpaBoolean verify_result;
	59	struct completion complete;
	60	} cy_callback_t;
	61
	62	static void
	63	symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
	64	void op_data, CpaBufferList buf_list_dst, CpaBoolean verify)
	65	{
	66	cy_callback_t *cb = p_callback;
	67
	68	if (cb != NULL) {
	69	/* indicate that the function has been called */
	70	cb->verify_result = verify;
	71	complete(&cb->complete);
	72	}
	73	}
	74
	75	boolean_t
	76	qat_crypt_use_accel(size_t s_len)
	77	{
38742209 TC	78	return (!zfs_qat_encrypt_disable &&
	79	qat_crypt_init_done &&
	80	s_len >= QAT_MIN_BUF_SIZE &&
	81	s_len <= QAT_MAX_BUF_SIZE);
	82	}
	83
	84	boolean_t
	85	qat_checksum_use_accel(size_t s_len)
	86	{
	87	return (!zfs_qat_checksum_disable &&
cf637391 TC	88	qat_crypt_init_done &&
	89	s_len >= QAT_MIN_BUF_SIZE &&
	90	s_len <= QAT_MAX_BUF_SIZE);
	91	}
	92
	93	void
	94	qat_crypt_clean(void)
	95	{
	96	for (Cpa32U i = 0; i < num_inst; i++)
	97	cpaCyStopInstance(cy_inst_handles[i]);
	98
	99	num_inst = 0;
	100	qat_crypt_init_done = B_FALSE;
	101	}
	102
	103	int
	104	qat_crypt_init(void)
	105	{
	106	Cpa32U i;
	107	CpaStatus status = CPA_STATUS_FAIL;
	108
	109	status = cpaCyGetNumInstances(&num_inst);
	110	if (status != CPA_STATUS_SUCCESS)
	111	return (-1);
	112
	113	/* if the user has configured no QAT encryption units just return */
	114	if (num_inst == 0)
	115	return (0);
	116
	117	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
	118	num_inst = QAT_CRYPT_MAX_INSTANCES;
	119
	120	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	121	if (status != CPA_STATUS_SUCCESS)
	122	return (-1);
	123
	124	for (i = 0; i < num_inst; i++) {
	125	status = cpaCySetAddressTranslation(cy_inst_handles[i],
	126	(void *)virt_to_phys);
	127	if (status != CPA_STATUS_SUCCESS)
	128	goto error;
	129
	130	status = cpaCyStartInstance(cy_inst_handles[i]);
	131	if (status != CPA_STATUS_SUCCESS)
	132	goto error;
	133	}
	134
	135	qat_crypt_init_done = B_TRUE;
	136	return (0);
	137
	138	error:
	139	qat_crypt_clean();
	140	return (-1);
	141	}
	142
	143	void
	144	qat_crypt_fini(void)
	145	{
	146	if (!qat_crypt_init_done)
	147	return;
	148
	149	qat_crypt_clean();
	150	}
	151
152	static CpaStatus
38742209	153	qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
cf637391 TC	154	CpaCySymSessionCtx *cy_session_ctx, crypto_key_t key,
	155	Cpa64U crypt, Cpa32U aad_len)
	156	{
	157	CpaStatus status = CPA_STATUS_SUCCESS;
	158	Cpa32U ctx_size;
	159	Cpa32U ciper_algorithm;
	160	Cpa32U hash_algorithm;
	161	CpaCySymSessionSetupData sd = { 0 };
	162
	163	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
	164	return (CPA_STATUS_FAIL);
	165	} else {
	166	ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
	167	hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	168	}
	169
	170	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	171	sd.cipherSetupData.pCipherKey = key->ck_data;
	172	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	173	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	174	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	175	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	176	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	177	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	178	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	179	sd.digestIsAppended = CPA_FALSE;
	180	sd.verifyDigest = CPA_FALSE;
	181
	182	if (dir == QAT_ENCRYPT) {
	183	sd.cipherSetupData.cipherDirection =
	184	CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
	185	sd.algChainOrder =
	186	CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	187	} else {
	188	ASSERT3U(dir, ==, QAT_DECRYPT);
	189	sd.cipherSetupData.cipherDirection =
	190	CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
	191	sd.algChainOrder =
	192	CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	193	}
	194
	195	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	196	if (status != CPA_STATUS_SUCCESS)
	197	return (status);
	198
	199	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	200	if (status != CPA_STATUS_SUCCESS)
	201	return (status);
	202
	203	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	204	*cy_session_ctx);
	205	if (status != CPA_STATUS_SUCCESS) {
	206	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	207	return (status);
	208	}
	209
	210	return (CPA_STATUS_SUCCESS);
	211	}
	212
	213	static CpaStatus
38742209 TC	214	qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
	215	CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
	216	{
	217	CpaStatus status = CPA_STATUS_SUCCESS;
	218	Cpa32U ctx_size;
	219	Cpa32U hash_algorithm;
	220	CpaCySymSessionSetupData sd = { 0 };
	221
	222	/*
	223	* ZFS's SHA512 checksum is actually SHA512/256, which uses
	224	* a different IV from standard SHA512. QAT does not support
	225	* SHA512/256, so we can only support SHA256.
	226	*/
	227	if (cksum == ZIO_CHECKSUM_SHA256)
	228	hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	229	else
	230	return (CPA_STATUS_FAIL);
	231
	232	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	233	sd.symOperation = CPA_CY_SYM_OP_HASH;
	234	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	235	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	236	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	237	sd.digestIsAppended = CPA_FALSE;
	238	sd.verifyDigest = CPA_FALSE;
	239
	240	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	241	if (status != CPA_STATUS_SUCCESS)
	242	return (status);
	243
	244	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	245	if (status != CPA_STATUS_SUCCESS)
	246	return (status);
	247
	248	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	249	*cy_session_ctx);
	250	if (status != CPA_STATUS_SUCCESS) {
	251	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	252	return (status);
	253	}
	254
	255	return (CPA_STATUS_SUCCESS);
	256	}
	257
	258	static CpaStatus
	259	qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
cf637391 TC	260	CpaBufferList src, CpaBufferList dst)
	261	{
	262	CpaStatus status = CPA_STATUS_SUCCESS;
	263	Cpa32U meta_size = 0;
	264
	265	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	266	if (status != CPA_STATUS_SUCCESS)
	267	return (status);
	268
cf637391 TC	269	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	270	if (status != CPA_STATUS_SUCCESS)
	271	goto error;
	272
	273	if (src != dst) {
cf637391 TC	274	status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
	275	meta_size);
	276	if (status != CPA_STATUS_SUCCESS)
	277	goto error;
	278	}
	279
	280	return (CPA_STATUS_SUCCESS);
	281
	282	error:
	283	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	284	if (src != dst)
	285	QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
	286
	287	return (status);
	288	}
	289
	290	int
	291	qat_crypt(qat_encrypt_dir_t dir, uint8_t src_buf, uint8_t dst_buf,
	292	uint8_t aad_buf, uint32_t aad_len, uint8_t iv_buf, uint8_t *digest_buf,
	293	crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
	294	{
	295	CpaStatus status = CPA_STATUS_SUCCESS;
	296	Cpa16U i;
	297	CpaInstanceHandle cy_inst_handle;
17dd8835	298	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
cf637391	299	Cpa32U bytes_left = 0;
17dd8835	300	Cpa8S *data = NULL;
cf637391 TC	301	CpaCySymSessionCtx *cy_session_ctx = NULL;
	302	cy_callback_t cb;
	303	CpaCySymOpData op_data = { 0 };
	304	CpaBufferList src_buffer_list = { 0 };
	305	CpaBufferList dst_buffer_list = { 0 };
	306	CpaFlatBuffer *flat_src_buf_array = NULL;
	307	CpaFlatBuffer *flat_src_buf = NULL;
	308	CpaFlatBuffer *flat_dst_buf_array = NULL;
	309	CpaFlatBuffer *flat_dst_buf = NULL;
	310	struct page *in_pages[MAX_PAGE_NUM];
	311	struct page *out_pages[MAX_PAGE_NUM];
17dd8835 TC	312	Cpa32U in_page_num = 0;
17dd8835 TC	313	Cpa32U out_page_num = 0;
38742209 TC	314	Cpa32U in_page_off = 0;
38742209 TC	315	Cpa32U out_page_off = 0;
cf637391 TC	316
	317	if (dir == QAT_ENCRYPT) {
	318	QAT_STAT_BUMP(encrypt_requests);
	319	QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	320	} else {
	321	QAT_STAT_BUMP(decrypt_requests);
	322	QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	323	}
	324
	325	i = atomic_inc_32_nv(&inst_num) % num_inst;
	326	cy_inst_handle = cy_inst_handles[i];
	327
38742209 TC	328	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	329	&cy_session_ctx, key, crypt, aad_len);
	330	if (status != CPA_STATUS_SUCCESS) {
	331	/* don't count CCM as a failure since it's not supported */
	332	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
	333	QAT_STAT_BUMP(crypt_fails);
cf637391	334	return (status);
38742209	335	}
cf637391	336
17dd8835 TC	337	/*
	338	* We increment nr_bufs by 2 to allow us to handle non
	339	* page-aligned buffer addresses and buffers whose sizes
	340	* are not divisible by PAGE_SIZE.
	341	*/
38742209 TC	342	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
38742209 TC	343	&src_buffer_list, &dst_buffer_list);
cf637391 TC	344	if (status != CPA_STATUS_SUCCESS)
	345	goto fail;
	346
	347	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	348	nr_bufs * sizeof (CpaFlatBuffer));
	349	if (status != CPA_STATUS_SUCCESS)
	350	goto fail;
	351	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	352	nr_bufs * sizeof (CpaFlatBuffer));
	353	if (status != CPA_STATUS_SUCCESS)
	354	goto fail;
	355
	356	bytes_left = enc_len;
17dd8835	357	data = src_buf;
cf637391	358	flat_src_buf = flat_src_buf_array;
cf637391	359	while (bytes_left > 0) {
17dd8835 TC	360	in_page_off = ((long)data & ~PAGE_MASK);
	361	in_pages[in_page_num] = qat_mem_to_page(data);
	362	flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
38742209	363	flat_src_buf->dataLenInBytes =
17dd8835 TC	364	min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
17dd8835 TC	365	data += flat_src_buf->dataLenInBytes;
cf637391 TC	366	bytes_left -= flat_src_buf->dataLenInBytes;
cf637391 TC	367	flat_src_buf++;
17dd8835	368	in_page_num++;
cf637391 TC	369	}
cf637391 TC	370	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835 TC	371	src_buffer_list.numBuffers = in_page_num;
	372
	373	bytes_left = enc_len;
	374	data = dst_buf;
	375	flat_dst_buf = flat_dst_buf_array;
	376	while (bytes_left > 0) {
	377	out_page_off = ((long)data & ~PAGE_MASK);
	378	out_pages[out_page_num] = qat_mem_to_page(data);
	379	flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
	380	out_page_off;
	381	flat_dst_buf->dataLenInBytes =
	382	min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
	383	data += flat_dst_buf->dataLenInBytes;
	384	bytes_left -= flat_dst_buf->dataLenInBytes;
	385	flat_dst_buf++;
	386	out_page_num++;
	387	}
cf637391	388	dst_buffer_list.pBuffers = flat_dst_buf_array;
17dd8835	389	dst_buffer_list.numBuffers = out_page_num;
cf637391 TC	390
	391	op_data.sessionCtx = cy_session_ctx;
	392	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	393	op_data.pIv = NULL; /* set this later as the J0 block */
	394	op_data.ivLenInBytes = 0;
	395	op_data.cryptoStartSrcOffsetInBytes = 0;
	396	op_data.messageLenToCipherInBytes = 0;
	397	op_data.hashStartSrcOffsetInBytes = 0;
	398	op_data.messageLenToHashInBytes = 0;
	399	op_data.pDigestResult = 0;
	400	op_data.messageLenToCipherInBytes = enc_len;
	401	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
	402	op_data.pDigestResult = digest_buf;
	403	op_data.pAdditionalAuthData = aad_buf;
	404	op_data.pIv = iv_buf;
	405
	406	cb.verify_result = CPA_FALSE;
	407	init_completion(&cb.complete);
	408	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	409	&src_buffer_list, &dst_buffer_list, NULL);
	410	if (status != CPA_STATUS_SUCCESS)
	411	goto fail;
	412
	413	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	414	QAT_TIMEOUT_MS)) {
	415	status = CPA_STATUS_FAIL;
	416	goto fail;
	417	}
	418
	419	if (cb.verify_result == CPA_FALSE) {
	420	status = CPA_STATUS_FAIL;
	421	goto fail;
	422	}
	423
	424	if (dir == QAT_ENCRYPT)
	425	QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	426	else
	427	QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
	428
	429	fail:
38742209	430	if (status != CPA_STATUS_SUCCESS)
cf637391 TC	431	QAT_STAT_BUMP(crypt_fails);
cf637391 TC	432
17dd8835	433	for (i = 0; i < in_page_num; i++)
cf637391	434	kunmap(in_pages[i]);
17dd8835	435	for (i = 0; i < out_page_num; i++)
cf637391	436	kunmap(out_pages[i]);
cf637391 TC	437
	438	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	439	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	440	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	441	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	442	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	443	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
	444
	445	return (status);
	446	}
	447
38742209 TC	448	int
	449	qat_checksum(uint64_t cksum, uint8_t buf, uint64_t size, zio_cksum_t zcp)
	450	{
	451	CpaStatus status;
	452	Cpa16U i;
	453	CpaInstanceHandle cy_inst_handle;
17dd8835	454	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
38742209 TC	455	Cpa32U bytes_left = 0;
	456	Cpa8S *data = NULL;
	457	CpaCySymSessionCtx *cy_session_ctx = NULL;
	458	cy_callback_t cb;
	459	Cpa8U digest_buffer[sizeof (zio_cksum_t)];
	460	CpaCySymOpData op_data = { 0 };
	461	CpaBufferList src_buffer_list = { 0 };
	462	CpaFlatBuffer *flat_src_buf_array = NULL;
	463	CpaFlatBuffer *flat_src_buf = NULL;
	464	struct page *in_pages[MAX_PAGE_NUM];
17dd8835	465	Cpa32U page_num = 0;
38742209 TC	466	Cpa32U page_off = 0;
	467
	468	QAT_STAT_BUMP(cksum_requests);
	469	QAT_STAT_INCR(cksum_total_in_bytes, size);
	470
	471	i = atomic_inc_32_nv(&inst_num) % num_inst;
	472	cy_inst_handle = cy_inst_handles[i];
	473
	474	status = qat_init_checksum_session_ctx(cy_inst_handle,
	475	&cy_session_ctx, cksum);
	476	if (status != CPA_STATUS_SUCCESS) {
	477	/* don't count unsupported checksums as a failure */
	478	if (cksum == ZIO_CHECKSUM_SHA256 \|\|
	479	cksum == ZIO_CHECKSUM_SHA512)
	480	QAT_STAT_BUMP(cksum_fails);
	481	return (status);
	482	}
	483
17dd8835 TC	484	/*
	485	* We increment nr_bufs by 2 to allow us to handle non
	486	* page-aligned buffer addresses and buffers whose sizes
	487	* are not divisible by PAGE_SIZE.
	488	*/
38742209 TC	489	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	490	&src_buffer_list, &src_buffer_list);
	491	if (status != CPA_STATUS_SUCCESS)
	492	goto fail;
	493
	494	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	495	nr_bufs * sizeof (CpaFlatBuffer));
	496	if (status != CPA_STATUS_SUCCESS)
	497	goto fail;
	498
	499	bytes_left = size;
	500	data = buf;
	501	flat_src_buf = flat_src_buf_array;
	502	while (bytes_left > 0) {
	503	page_off = ((long)data & ~PAGE_MASK);
	504	in_pages[page_num] = qat_mem_to_page(data);
	505	flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
	506	flat_src_buf->dataLenInBytes =
17dd8835	507	min((long)PAGE_SIZE - page_off, (long)bytes_left);
38742209 TC	508	data += flat_src_buf->dataLenInBytes;
	509	bytes_left -= flat_src_buf->dataLenInBytes;
	510	flat_src_buf++;
	511	page_num++;
	512	}
	513	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835	514	src_buffer_list.numBuffers = page_num;
38742209 TC	515
	516	op_data.sessionCtx = cy_session_ctx;
	517	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	518	op_data.hashStartSrcOffsetInBytes = 0;
	519	op_data.messageLenToHashInBytes = size;
	520	op_data.pDigestResult = digest_buffer;
	521
	522	cb.verify_result = CPA_FALSE;
	523	init_completion(&cb.complete);
	524	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	525	&src_buffer_list, &src_buffer_list, NULL);
	526	if (status != CPA_STATUS_SUCCESS)
	527	goto fail;
	528
	529	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	530	QAT_TIMEOUT_MS)) {
	531	status = CPA_STATUS_FAIL;
	532	goto fail;
	533	}
	534
	535	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
	536
	537	fail:
	538	if (status != CPA_STATUS_SUCCESS)
	539	QAT_STAT_BUMP(cksum_fails);
	540
	541	for (i = 0; i < page_num; i++)
	542	kunmap(in_pages[i]);
	543
	544	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	545	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	546	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	547	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	548
	549	return (status);
	550	}
	551
	552	module_param(zfs_qat_encrypt_disable, int, 0644);
	553	MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
	554
	555	module_param(zfs_qat_checksum_disable, int, 0644);
	556	MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
cf637391 TC	557
cf637391 TC	558	#endif