[mirror_zfs.git] / module / zfs / qat_crypt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * This file represents the QAT implementation of checksums and encryption.
 * Internally, QAT shares the same cryptographic instances for both of these
 * operations, so the code has been combined here. QAT data compression uses
 * compression instances, so that code is separated into qat_compress.c
 */

#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
#include "lac/cpa_cy_common.h"
#include "qat.h"

/*
 * Max instances in a QAT device, each instance is a channel to submit
 * jobs to QAT hardware, this is only for pre-allocating instances
 * and session arrays; the actual number of instances are defined in
 * the QAT driver's configure file.
 */
#define	QAT_CRYPT_MAX_INSTANCES		48

#define	MAX_PAGE_NUM			1024

static Cpa32U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_crypt_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;

typedef struct cy_callback {
	CpaBoolean verify_result;
	struct completion complete;
} cy_callback_t;

static void
symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
    void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
{
	cy_callback_t *cb = p_callback;

	if (cb != NULL) {
		/* indicate that the function has been called */
		cb->verify_result = verify;
		complete(&cb->complete);
	}
}

boolean_t
qat_crypt_use_accel(size_t s_len)
{
	return (!zfs_qat_encrypt_disable &&
	    qat_crypt_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

boolean_t
qat_checksum_use_accel(size_t s_len)
{
	return (!zfs_qat_checksum_disable &&
	    qat_crypt_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

void
qat_crypt_clean(void)
{
	for (Cpa16U i = 0; i < num_inst; i++)
		cpaCyStopInstance(cy_inst_handles[i]);

	num_inst = 0;
	qat_crypt_init_done = B_FALSE;
}

int
qat_crypt_init(void)
{
	CpaStatus status = CPA_STATUS_FAIL;

	status = cpaCyGetNumInstances(&num_inst);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	/* if the user has configured no QAT encryption units just return */
	if (num_inst == 0)
		return (0);

	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
		num_inst = QAT_CRYPT_MAX_INSTANCES;

	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	for (Cpa16U i = 0; i < num_inst; i++) {
		status = cpaCySetAddressTranslation(cy_inst_handles[i],
		    (void *)virt_to_phys);
		if (status != CPA_STATUS_SUCCESS)
			goto error;

		status = cpaCyStartInstance(cy_inst_handles[i]);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	qat_crypt_init_done = B_TRUE;
	return (0);

error:
	qat_crypt_clean();
	return (-1);
}

void
qat_crypt_fini(void)
{
	if (!qat_crypt_init_done)
		return;

	qat_crypt_clean();
}

static CpaStatus
qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
    Cpa64U crypt, Cpa32U aad_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U ciper_algorithm;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
		return (CPA_STATUS_FAIL);
	} else {
		ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
		hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	}

	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	sd.cipherSetupData.pCipherKey = key->ck_data;
	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	if (dir == QAT_ENCRYPT) {
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	} else {
		ASSERT3U(dir, ==, QAT_DECRYPT);
		sd.cipherSetupData.cipherDirection =
		    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
		sd.algChainOrder =
		    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	}

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
    CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U ctx_size;
	Cpa32U hash_algorithm;
	CpaCySymSessionSetupData sd = { 0 };

	/*
	 * ZFS's SHA512 checksum is actually SHA512/256, which uses
	 * a different IV from standard SHA512. QAT does not support
	 * SHA512/256, so we can only support SHA256.
	 */
	if (cksum == ZIO_CHECKSUM_SHA256)
		hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	else
		return (CPA_STATUS_FAIL);

	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	sd.symOperation = CPA_CY_SYM_OP_HASH;
	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	sd.digestIsAppended = CPA_FALSE;
	sd.verifyDigest = CPA_FALSE;

	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	    *cy_session_ctx);
	if (status != CPA_STATUS_SUCCESS) {
		QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
		return (status);
	}

	return (CPA_STATUS_SUCCESS);
}

static CpaStatus
qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
    CpaBufferList *src, CpaBufferList *dst)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U meta_size = 0;

	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	if (status != CPA_STATUS_SUCCESS)
		return (status);

	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	if (status != CPA_STATUS_SUCCESS)
		goto error;

	if (src != dst) {
		status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
		    meta_size);
		if (status != CPA_STATUS_SUCCESS)
			goto error;
	}

	return (CPA_STATUS_SUCCESS);

error:
	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	if (src != dst)
		QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);

	return (status);
}

int
qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
    uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
    crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaBufferList dst_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	CpaFlatBuffer *flat_dst_buf_array = NULL;
	CpaFlatBuffer *flat_dst_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	struct page *out_pages[MAX_PAGE_NUM];
	Cpa32U in_page_num = 0;
	Cpa32U out_page_num = 0;
	Cpa32U in_page_off = 0;
	Cpa32U out_page_off = 0;

	if (dir == QAT_ENCRYPT) {
		QAT_STAT_BUMP(encrypt_requests);
		QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	} else {
		QAT_STAT_BUMP(decrypt_requests);
		QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	}

	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	    &cy_session_ctx, key, crypt, aad_len);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count CCM as a failure since it's not supported */
		if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
			QAT_STAT_BUMP(crypt_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &dst_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;
	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	bytes_left = enc_len;
	data = src_buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		in_page_off = ((long)data & ~PAGE_MASK);
		in_pages[in_page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		in_page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = in_page_num;

	bytes_left = enc_len;
	data = dst_buf;
	flat_dst_buf = flat_dst_buf_array;
	while (bytes_left > 0) {
		out_page_off = ((long)data & ~PAGE_MASK);
		out_pages[out_page_num] = qat_mem_to_page(data);
		flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
		    out_page_off;
		flat_dst_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
		data += flat_dst_buf->dataLenInBytes;
		bytes_left -= flat_dst_buf->dataLenInBytes;
		flat_dst_buf++;
		out_page_num++;
	}
	dst_buffer_list.pBuffers = flat_dst_buf_array;
	dst_buffer_list.numBuffers = out_page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.pIv = NULL; /* set this later as the J0 block */
	op_data.ivLenInBytes = 0;
	op_data.cryptoStartSrcOffsetInBytes = 0;
	op_data.messageLenToCipherInBytes = 0;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = 0;
	op_data.pDigestResult = 0;
	op_data.messageLenToCipherInBytes = enc_len;
	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
	op_data.pDigestResult = digest_buf;
	op_data.pAdditionalAuthData = aad_buf;
	op_data.pIv = iv_buf;

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &dst_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	if (cb.verify_result == CPA_FALSE) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	if (dir == QAT_ENCRYPT)
		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	else
		QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(crypt_fails);

	for (i = 0; i < in_page_num; i++)
		kunmap(in_pages[i]);
	for (i = 0; i < out_page_num; i++)
		kunmap(out_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);

	return (status);
}

int
qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
{
	CpaStatus status;
	Cpa16U i;
	CpaInstanceHandle cy_inst_handle;
	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left = 0;
	Cpa8S *data = NULL;
	CpaCySymSessionCtx *cy_session_ctx = NULL;
	cy_callback_t cb;
	Cpa8U digest_buffer[sizeof (zio_cksum_t)];
	CpaCySymOpData op_data = { 0 };
	CpaBufferList src_buffer_list = { 0 };
	CpaFlatBuffer *flat_src_buf_array = NULL;
	CpaFlatBuffer *flat_src_buf = NULL;
	struct page *in_pages[MAX_PAGE_NUM];
	Cpa32U page_num = 0;
	Cpa32U page_off = 0;

	QAT_STAT_BUMP(cksum_requests);
	QAT_STAT_INCR(cksum_total_in_bytes, size);

	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
	cy_inst_handle = cy_inst_handles[i];

	status = qat_init_checksum_session_ctx(cy_inst_handle,
	    &cy_session_ctx, cksum);
	if (status != CPA_STATUS_SUCCESS) {
		/* don't count unsupported checksums as a failure */
		if (cksum == ZIO_CHECKSUM_SHA256 ||
		    cksum == ZIO_CHECKSUM_SHA512)
			QAT_STAT_BUMP(cksum_fails);
		return (status);
	}

	/*
	 * We increment nr_bufs by 2 to allow us to handle non
	 * page-aligned buffer addresses and buffers whose sizes
	 * are not divisible by PAGE_SIZE.
	 */
	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	    &src_buffer_list, &src_buffer_list);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	    nr_bufs * sizeof (CpaFlatBuffer));
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	bytes_left = size;
	data = buf;
	flat_src_buf = flat_src_buf_array;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		in_pages[page_num] = qat_mem_to_page(data);
		flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
		flat_src_buf->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);
		data += flat_src_buf->dataLenInBytes;
		bytes_left -= flat_src_buf->dataLenInBytes;
		flat_src_buf++;
		page_num++;
	}
	src_buffer_list.pBuffers = flat_src_buf_array;
	src_buffer_list.numBuffers = page_num;

	op_data.sessionCtx = cy_session_ctx;
	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	op_data.hashStartSrcOffsetInBytes = 0;
	op_data.messageLenToHashInBytes = size;
	op_data.pDigestResult = digest_buffer;

	cb.verify_result = CPA_FALSE;
	init_completion(&cb.complete);
	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	    &src_buffer_list, &src_buffer_list, NULL);
	if (status != CPA_STATUS_SUCCESS)
		goto fail;

	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	    QAT_TIMEOUT_MS)) {
		status = CPA_STATUS_FAIL;
		goto fail;
	}

	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));

fail:
	if (status != CPA_STATUS_SUCCESS)
		QAT_STAT_BUMP(cksum_fails);

	for (i = 0; i < page_num; i++)
		kunmap(in_pages[i]);

	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);

	return (status);
}

module_param(zfs_qat_encrypt_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");

module_param(zfs_qat_checksum_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");

#endif
Commit	Line	Data
cf637391 TC	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21
38742209 TC	22	/*
	23	* This file represents the QAT implementation of checksums and encryption.
	24	* Internally, QAT shares the same cryptographic instances for both of these
	25	* operations, so the code has been combined here. QAT data compression uses
	26	* compression instances, so that code is separated into qat_compress.c
	27	*/
	28
cf637391 TC	29	#if defined(_KERNEL) && defined(HAVE_QAT)
	30	#include <linux/slab.h>
	31	#include <linux/vmalloc.h>
	32	#include <linux/pagemap.h>
	33	#include <linux/completion.h>
	34	#include <sys/zfs_context.h>
	35	#include <sys/zio_crypt.h>
	36	#include "lac/cpa_cy_im.h"
38742209	37	#include "lac/cpa_cy_common.h"
cf637391 TC	38	#include "qat.h"
	39
	40	/*
8d9e7c8f TC	41	* Max instances in a QAT device, each instance is a channel to submit
	42	* jobs to QAT hardware, this is only for pre-allocating instances
	43	* and session arrays; the actual number of instances are defined in
cf637391 TC	44	* the QAT driver's configure file.
	45	*/
	46	#define QAT_CRYPT_MAX_INSTANCES 48
	47
	48	#define MAX_PAGE_NUM 1024
	49
bc8a6a60	50	static Cpa32U inst_num = 0;
cf637391 TC	51	static Cpa16U num_inst = 0;
cf637391 TC	52	static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
38742209 TC	53	static boolean_t qat_crypt_init_done = B_FALSE;
	54	int zfs_qat_encrypt_disable = 0;
	55	int zfs_qat_checksum_disable = 0;
cf637391 TC	56
	57	typedef struct cy_callback {
	58	CpaBoolean verify_result;
	59	struct completion complete;
	60	} cy_callback_t;
	61
	62	static void
	63	symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
	64	void op_data, CpaBufferList buf_list_dst, CpaBoolean verify)
	65	{
	66	cy_callback_t *cb = p_callback;
	67
	68	if (cb != NULL) {
	69	/* indicate that the function has been called */
	70	cb->verify_result = verify;
	71	complete(&cb->complete);
	72	}
	73	}
	74
	75	boolean_t
	76	qat_crypt_use_accel(size_t s_len)
	77	{
38742209 TC	78	return (!zfs_qat_encrypt_disable &&
	79	qat_crypt_init_done &&
	80	s_len >= QAT_MIN_BUF_SIZE &&
	81	s_len <= QAT_MAX_BUF_SIZE);
	82	}
	83
	84	boolean_t
	85	qat_checksum_use_accel(size_t s_len)
	86	{
	87	return (!zfs_qat_checksum_disable &&
cf637391 TC	88	qat_crypt_init_done &&
	89	s_len >= QAT_MIN_BUF_SIZE &&
	90	s_len <= QAT_MAX_BUF_SIZE);
	91	}
	92
	93	void
	94	qat_crypt_clean(void)
	95	{
bc8a6a60	96	for (Cpa16U i = 0; i < num_inst; i++)
cf637391 TC	97	cpaCyStopInstance(cy_inst_handles[i]);
	98
	99	num_inst = 0;
	100	qat_crypt_init_done = B_FALSE;
	101	}
	102
	103	int
	104	qat_crypt_init(void)
	105	{
cf637391 TC	106	CpaStatus status = CPA_STATUS_FAIL;
	107
	108	status = cpaCyGetNumInstances(&num_inst);
	109	if (status != CPA_STATUS_SUCCESS)
	110	return (-1);
	111
	112	/* if the user has configured no QAT encryption units just return */
	113	if (num_inst == 0)
	114	return (0);
	115
	116	if (num_inst > QAT_CRYPT_MAX_INSTANCES)
	117	num_inst = QAT_CRYPT_MAX_INSTANCES;
	118
	119	status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
	120	if (status != CPA_STATUS_SUCCESS)
	121	return (-1);
	122
bc8a6a60	123	for (Cpa16U i = 0; i < num_inst; i++) {
cf637391 TC	124	status = cpaCySetAddressTranslation(cy_inst_handles[i],
	125	(void *)virt_to_phys);
	126	if (status != CPA_STATUS_SUCCESS)
	127	goto error;
	128
	129	status = cpaCyStartInstance(cy_inst_handles[i]);
	130	if (status != CPA_STATUS_SUCCESS)
	131	goto error;
	132	}
	133
	134	qat_crypt_init_done = B_TRUE;
	135	return (0);
	136
	137	error:
	138	qat_crypt_clean();
	139	return (-1);
	140	}
	141
	142	void
	143	qat_crypt_fini(void)
	144	{
	145	if (!qat_crypt_init_done)
	146	return;
	147
	148	qat_crypt_clean();
	149	}
	150
	151	static CpaStatus
38742209	152	qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
cf637391 TC	153	CpaCySymSessionCtx *cy_session_ctx, crypto_key_t key,
	154	Cpa64U crypt, Cpa32U aad_len)
	155	{
	156	CpaStatus status = CPA_STATUS_SUCCESS;
	157	Cpa32U ctx_size;
	158	Cpa32U ciper_algorithm;
	159	Cpa32U hash_algorithm;
	160	CpaCySymSessionSetupData sd = { 0 };
	161
	162	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
	163	return (CPA_STATUS_FAIL);
	164	} else {
	165	ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
	166	hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
	167	}
	168
	169	sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
	170	sd.cipherSetupData.pCipherKey = key->ck_data;
	171	sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
	172	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	173	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
	174	sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
	175	sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
	176	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	177	sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
	178	sd.digestIsAppended = CPA_FALSE;
	179	sd.verifyDigest = CPA_FALSE;
	180
	181	if (dir == QAT_ENCRYPT) {
	182	sd.cipherSetupData.cipherDirection =
	183	CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
	184	sd.algChainOrder =
	185	CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
	186	} else {
	187	ASSERT3U(dir, ==, QAT_DECRYPT);
	188	sd.cipherSetupData.cipherDirection =
	189	CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
	190	sd.algChainOrder =
	191	CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	192	}
	193
	194	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	195	if (status != CPA_STATUS_SUCCESS)
	196	return (status);
	197
	198	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	199	if (status != CPA_STATUS_SUCCESS)
	200	return (status);
	201
	202	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	203	*cy_session_ctx);
	204	if (status != CPA_STATUS_SUCCESS) {
	205	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	206	return (status);
	207	}
	208
	209	return (CPA_STATUS_SUCCESS);
	210	}
	211
	212	static CpaStatus
38742209 TC	213	qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
	214	CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
	215	{
	216	CpaStatus status = CPA_STATUS_SUCCESS;
	217	Cpa32U ctx_size;
	218	Cpa32U hash_algorithm;
	219	CpaCySymSessionSetupData sd = { 0 };
	220
	221	/*
	222	* ZFS's SHA512 checksum is actually SHA512/256, which uses
	223	* a different IV from standard SHA512. QAT does not support
	224	* SHA512/256, so we can only support SHA256.
	225	*/
	226	if (cksum == ZIO_CHECKSUM_SHA256)
	227	hash_algorithm = CPA_CY_SYM_HASH_SHA256;
	228	else
	229	return (CPA_STATUS_FAIL);
	230
	231	sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
	232	sd.symOperation = CPA_CY_SYM_OP_HASH;
	233	sd.hashSetupData.hashAlgorithm = hash_algorithm;
	234	sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
	235	sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
	236	sd.digestIsAppended = CPA_FALSE;
	237	sd.verifyDigest = CPA_FALSE;
	238
	239	status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
	240	if (status != CPA_STATUS_SUCCESS)
	241	return (status);
	242
	243	status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
	244	if (status != CPA_STATUS_SUCCESS)
	245	return (status);
	246
	247	status = cpaCySymInitSession(inst_handle, symcallback, &sd,
	248	*cy_session_ctx);
	249	if (status != CPA_STATUS_SUCCESS) {
	250	QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
	251	return (status);
	252	}
	253
	254	return (CPA_STATUS_SUCCESS);
	255	}
	256
	257	static CpaStatus
	258	qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
cf637391 TC	259	CpaBufferList src, CpaBufferList dst)
	260	{
	261	CpaStatus status = CPA_STATUS_SUCCESS;
	262	Cpa32U meta_size = 0;
	263
	264	status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
	265	if (status != CPA_STATUS_SUCCESS)
	266	return (status);
	267
cf637391 TC	268	status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
	269	if (status != CPA_STATUS_SUCCESS)
	270	goto error;
	271
	272	if (src != dst) {
cf637391 TC	273	status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
	274	meta_size);
	275	if (status != CPA_STATUS_SUCCESS)
	276	goto error;
	277	}
	278
	279	return (CPA_STATUS_SUCCESS);
	280
	281	error:
	282	QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
	283	if (src != dst)
	284	QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
	285
	286	return (status);
	287	}
	288
	289	int
	290	qat_crypt(qat_encrypt_dir_t dir, uint8_t src_buf, uint8_t dst_buf,
	291	uint8_t aad_buf, uint32_t aad_len, uint8_t iv_buf, uint8_t *digest_buf,
	292	crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
	293	{
	294	CpaStatus status = CPA_STATUS_SUCCESS;
	295	Cpa16U i;
	296	CpaInstanceHandle cy_inst_handle;
17dd8835	297	Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
cf637391	298	Cpa32U bytes_left = 0;
17dd8835	299	Cpa8S *data = NULL;
cf637391 TC	300	CpaCySymSessionCtx *cy_session_ctx = NULL;
	301	cy_callback_t cb;
	302	CpaCySymOpData op_data = { 0 };
	303	CpaBufferList src_buffer_list = { 0 };
	304	CpaBufferList dst_buffer_list = { 0 };
	305	CpaFlatBuffer *flat_src_buf_array = NULL;
	306	CpaFlatBuffer *flat_src_buf = NULL;
	307	CpaFlatBuffer *flat_dst_buf_array = NULL;
	308	CpaFlatBuffer *flat_dst_buf = NULL;
	309	struct page *in_pages[MAX_PAGE_NUM];
	310	struct page *out_pages[MAX_PAGE_NUM];
17dd8835 TC	311	Cpa32U in_page_num = 0;
17dd8835 TC	312	Cpa32U out_page_num = 0;
38742209 TC	313	Cpa32U in_page_off = 0;
38742209 TC	314	Cpa32U out_page_off = 0;
cf637391 TC	315
	316	if (dir == QAT_ENCRYPT) {
	317	QAT_STAT_BUMP(encrypt_requests);
	318	QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
	319	} else {
	320	QAT_STAT_BUMP(decrypt_requests);
	321	QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
	322	}
	323
bc8a6a60	324	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
cf637391 TC	325	cy_inst_handle = cy_inst_handles[i];
cf637391 TC	326
38742209 TC	327	status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
	328	&cy_session_ctx, key, crypt, aad_len);
	329	if (status != CPA_STATUS_SUCCESS) {
	330	/* don't count CCM as a failure since it's not supported */
	331	if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
	332	QAT_STAT_BUMP(crypt_fails);
cf637391	333	return (status);
38742209	334	}
cf637391	335
17dd8835 TC	336	/*
	337	* We increment nr_bufs by 2 to allow us to handle non
	338	* page-aligned buffer addresses and buffers whose sizes
	339	* are not divisible by PAGE_SIZE.
	340	*/
38742209 TC	341	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
38742209 TC	342	&src_buffer_list, &dst_buffer_list);
cf637391 TC	343	if (status != CPA_STATUS_SUCCESS)
	344	goto fail;
	345
	346	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	347	nr_bufs * sizeof (CpaFlatBuffer));
	348	if (status != CPA_STATUS_SUCCESS)
	349	goto fail;
	350	status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
	351	nr_bufs * sizeof (CpaFlatBuffer));
	352	if (status != CPA_STATUS_SUCCESS)
	353	goto fail;
	354
	355	bytes_left = enc_len;
17dd8835	356	data = src_buf;
cf637391	357	flat_src_buf = flat_src_buf_array;
cf637391	358	while (bytes_left > 0) {
17dd8835 TC	359	in_page_off = ((long)data & ~PAGE_MASK);
	360	in_pages[in_page_num] = qat_mem_to_page(data);
	361	flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
38742209	362	flat_src_buf->dataLenInBytes =
17dd8835 TC	363	min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
17dd8835 TC	364	data += flat_src_buf->dataLenInBytes;
cf637391 TC	365	bytes_left -= flat_src_buf->dataLenInBytes;
cf637391 TC	366	flat_src_buf++;
17dd8835	367	in_page_num++;
cf637391 TC	368	}
cf637391 TC	369	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835 TC	370	src_buffer_list.numBuffers = in_page_num;
	371
	372	bytes_left = enc_len;
	373	data = dst_buf;
	374	flat_dst_buf = flat_dst_buf_array;
	375	while (bytes_left > 0) {
	376	out_page_off = ((long)data & ~PAGE_MASK);
	377	out_pages[out_page_num] = qat_mem_to_page(data);
	378	flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
	379	out_page_off;
	380	flat_dst_buf->dataLenInBytes =
	381	min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
	382	data += flat_dst_buf->dataLenInBytes;
	383	bytes_left -= flat_dst_buf->dataLenInBytes;
	384	flat_dst_buf++;
	385	out_page_num++;
	386	}
cf637391	387	dst_buffer_list.pBuffers = flat_dst_buf_array;
17dd8835	388	dst_buffer_list.numBuffers = out_page_num;
cf637391 TC	389
	390	op_data.sessionCtx = cy_session_ctx;
	391	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	392	op_data.pIv = NULL; /* set this later as the J0 block */
	393	op_data.ivLenInBytes = 0;
	394	op_data.cryptoStartSrcOffsetInBytes = 0;
	395	op_data.messageLenToCipherInBytes = 0;
	396	op_data.hashStartSrcOffsetInBytes = 0;
	397	op_data.messageLenToHashInBytes = 0;
	398	op_data.pDigestResult = 0;
	399	op_data.messageLenToCipherInBytes = enc_len;
	400	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
	401	op_data.pDigestResult = digest_buf;
	402	op_data.pAdditionalAuthData = aad_buf;
	403	op_data.pIv = iv_buf;
	404
	405	cb.verify_result = CPA_FALSE;
	406	init_completion(&cb.complete);
	407	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	408	&src_buffer_list, &dst_buffer_list, NULL);
	409	if (status != CPA_STATUS_SUCCESS)
	410	goto fail;
	411
	412	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	413	QAT_TIMEOUT_MS)) {
	414	status = CPA_STATUS_FAIL;
	415	goto fail;
	416	}
	417
	418	if (cb.verify_result == CPA_FALSE) {
	419	status = CPA_STATUS_FAIL;
	420	goto fail;
	421	}
	422
	423	if (dir == QAT_ENCRYPT)
	424	QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
	425	else
	426	QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
	427
	428	fail:
38742209	429	if (status != CPA_STATUS_SUCCESS)
cf637391 TC	430	QAT_STAT_BUMP(crypt_fails);
cf637391 TC	431
17dd8835	432	for (i = 0; i < in_page_num; i++)
cf637391	433	kunmap(in_pages[i]);
17dd8835	434	for (i = 0; i < out_page_num; i++)
cf637391	435	kunmap(out_pages[i]);
cf637391 TC	436
	437	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	438	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	439	QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
	440	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	441	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	442	QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
	443
	444	return (status);
	445	}
	446
38742209 TC	447	int
	448	qat_checksum(uint64_t cksum, uint8_t buf, uint64_t size, zio_cksum_t zcp)
	449	{
	450	CpaStatus status;
	451	Cpa16U i;
	452	CpaInstanceHandle cy_inst_handle;
17dd8835	453	Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
38742209 TC	454	Cpa32U bytes_left = 0;
	455	Cpa8S *data = NULL;
	456	CpaCySymSessionCtx *cy_session_ctx = NULL;
	457	cy_callback_t cb;
	458	Cpa8U digest_buffer[sizeof (zio_cksum_t)];
	459	CpaCySymOpData op_data = { 0 };
	460	CpaBufferList src_buffer_list = { 0 };
	461	CpaFlatBuffer *flat_src_buf_array = NULL;
	462	CpaFlatBuffer *flat_src_buf = NULL;
	463	struct page *in_pages[MAX_PAGE_NUM];
17dd8835	464	Cpa32U page_num = 0;
38742209 TC	465	Cpa32U page_off = 0;
	466
	467	QAT_STAT_BUMP(cksum_requests);
	468	QAT_STAT_INCR(cksum_total_in_bytes, size);
	469
bc8a6a60	470	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
38742209 TC	471	cy_inst_handle = cy_inst_handles[i];
	472
	473	status = qat_init_checksum_session_ctx(cy_inst_handle,
	474	&cy_session_ctx, cksum);
	475	if (status != CPA_STATUS_SUCCESS) {
	476	/* don't count unsupported checksums as a failure */
	477	if (cksum == ZIO_CHECKSUM_SHA256 \|\|
	478	cksum == ZIO_CHECKSUM_SHA512)
	479	QAT_STAT_BUMP(cksum_fails);
	480	return (status);
	481	}
	482
17dd8835 TC	483	/*
	484	* We increment nr_bufs by 2 to allow us to handle non
	485	* page-aligned buffer addresses and buffers whose sizes
	486	* are not divisible by PAGE_SIZE.
	487	*/
38742209 TC	488	status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
	489	&src_buffer_list, &src_buffer_list);
	490	if (status != CPA_STATUS_SUCCESS)
	491	goto fail;
	492
	493	status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
	494	nr_bufs * sizeof (CpaFlatBuffer));
	495	if (status != CPA_STATUS_SUCCESS)
	496	goto fail;
	497
	498	bytes_left = size;
	499	data = buf;
	500	flat_src_buf = flat_src_buf_array;
	501	while (bytes_left > 0) {
	502	page_off = ((long)data & ~PAGE_MASK);
	503	in_pages[page_num] = qat_mem_to_page(data);
	504	flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
	505	flat_src_buf->dataLenInBytes =
17dd8835	506	min((long)PAGE_SIZE - page_off, (long)bytes_left);
38742209 TC	507	data += flat_src_buf->dataLenInBytes;
	508	bytes_left -= flat_src_buf->dataLenInBytes;
	509	flat_src_buf++;
	510	page_num++;
	511	}
	512	src_buffer_list.pBuffers = flat_src_buf_array;
17dd8835	513	src_buffer_list.numBuffers = page_num;
38742209 TC	514
	515	op_data.sessionCtx = cy_session_ctx;
	516	op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
	517	op_data.hashStartSrcOffsetInBytes = 0;
	518	op_data.messageLenToHashInBytes = size;
	519	op_data.pDigestResult = digest_buffer;
	520
	521	cb.verify_result = CPA_FALSE;
	522	init_completion(&cb.complete);
	523	status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
	524	&src_buffer_list, &src_buffer_list, NULL);
	525	if (status != CPA_STATUS_SUCCESS)
	526	goto fail;
	527
	528	if (!wait_for_completion_interruptible_timeout(&cb.complete,
	529	QAT_TIMEOUT_MS)) {
	530	status = CPA_STATUS_FAIL;
	531	goto fail;
	532	}
	533
	534	bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
	535
	536	fail:
	537	if (status != CPA_STATUS_SUCCESS)
	538	QAT_STAT_BUMP(cksum_fails);
	539
	540	for (i = 0; i < page_num; i++)
	541	kunmap(in_pages[i]);
	542
	543	cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
	544	QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
	545	QAT_PHYS_CONTIG_FREE(cy_session_ctx);
	546	QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
	547
	548	return (status);
	549	}
	550
	551	module_param(zfs_qat_encrypt_disable, int, 0644);
	552	MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
	553
	554	module_param(zfs_qat_checksum_disable, int, 0644);
	555	MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
cf637391 TC	556
cf637391 TC	557	#endif