[mirror_zfs.git] / module / zfs / qat_compress.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/byteorder.h>
#include <sys/zio.h>
#include "qat.h"

/*
 * Max instances in a QAT device, each instance is a channel to submit
 * jobs to QAT hardware, this is only for pre-allocating instance and
 * session arrays; the actual number of instances are defined in the
 * QAT driver's configuration file.
 */
#define	QAT_DC_MAX_INSTANCES	48

/*
 * ZLIB head and foot size
 */
#define	ZLIB_HEAD_SZ		2
#define	ZLIB_FOOT_SZ		4

static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
static Cpa16U num_inst = 0;
static Cpa32U inst_num = 0;
static boolean_t qat_dc_init_done = B_FALSE;
int zfs_qat_compress_disable = 0;

boolean_t
qat_dc_use_accel(size_t s_len)
{
	return (!zfs_qat_compress_disable &&
	    qat_dc_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

static void
qat_dc_callback(void *p_callback, CpaStatus status)
{
	if (p_callback != NULL)
		complete((struct completion *)p_callback);
}

static void
qat_dc_clean(void)
{
	Cpa16U buff_num = 0;
	Cpa16U num_inter_buff_lists = 0;

	for (Cpa16U i = 0; i < num_inst; i++) {
		cpaDcStopInstance(dc_inst_handles[i]);
		QAT_PHYS_CONTIG_FREE(session_handles[i]);
		/* free intermediate buffers  */
		if (buffer_array[i] != NULL) {
			cpaDcGetNumIntermediateBuffers(
			    dc_inst_handles[i], &num_inter_buff_lists);
			for (buff_num = 0; buff_num < num_inter_buff_lists;
			    buff_num++) {
				CpaBufferList *buffer_inter =
				    buffer_array[i][buff_num];
				if (buffer_inter->pBuffers) {
					QAT_PHYS_CONTIG_FREE(
					    buffer_inter->pBuffers->pData);
					QAT_PHYS_CONTIG_FREE(
					    buffer_inter->pBuffers);
				}
				QAT_PHYS_CONTIG_FREE(
				    buffer_inter->pPrivateMetaData);
				QAT_PHYS_CONTIG_FREE(buffer_inter);
			}
		}
	}

	num_inst = 0;
	qat_dc_init_done = B_FALSE;
}

int
qat_dc_init(void)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U sess_size = 0;
	Cpa32U ctx_size = 0;
	Cpa16U num_inter_buff_lists = 0;
	Cpa16U buff_num = 0;
	Cpa32U buff_meta_size = 0;
	CpaDcSessionSetupData sd = {0};

	if (qat_dc_init_done)
		return (0);

	status = cpaDcGetNumInstances(&num_inst);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	/* if the user has configured no QAT compression units just return */
	if (num_inst == 0)
		return (0);

	if (num_inst > QAT_DC_MAX_INSTANCES)
		num_inst = QAT_DC_MAX_INSTANCES;

	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	for (Cpa16U i = 0; i < num_inst; i++) {
		cpaDcSetAddressTranslation(dc_inst_handles[i],
		    (void*)virt_to_phys);

		status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
		    1, &buff_meta_size);

		if (status == CPA_STATUS_SUCCESS)
			status = cpaDcGetNumIntermediateBuffers(
			    dc_inst_handles[i], &num_inter_buff_lists);

		if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
			status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
			    num_inter_buff_lists *
			    sizeof (CpaBufferList *));

		for (buff_num = 0; buff_num < num_inter_buff_lists;
		    buff_num++) {
			if (status == CPA_STATUS_SUCCESS)
				status = QAT_PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num],
				    sizeof (CpaBufferList));

			if (status == CPA_STATUS_SUCCESS)
				status = QAT_PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->
				    pPrivateMetaData,
				    buff_meta_size);

			if (status == CPA_STATUS_SUCCESS)
				status = QAT_PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->pBuffers,
				    sizeof (CpaFlatBuffer));

			if (status == CPA_STATUS_SUCCESS) {
				/*
				 *  implementation requires an intermediate
				 *  buffer approximately twice the size of
				 *  output buffer, which is 2x max buffer
				 *  size here.
				 */
				status = QAT_PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->pBuffers->
				    pData, 2 * QAT_MAX_BUF_SIZE);
				if (status != CPA_STATUS_SUCCESS)
					goto fail;

				buffer_array[i][buff_num]->numBuffers = 1;
				buffer_array[i][buff_num]->pBuffers->
				    dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
			}
		}

		status = cpaDcStartInstance(dc_inst_handles[i],
		    num_inter_buff_lists, buffer_array[i]);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;

		sd.compLevel = CPA_DC_L1;
		sd.compType = CPA_DC_DEFLATE;
		sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
		sd.sessDirection = CPA_DC_DIR_COMBINED;
		sd.sessState = CPA_DC_STATELESS;
		sd.deflateWindowSize = 7;
		sd.checksum = CPA_DC_ADLER32;
		status = cpaDcGetSessionSize(dc_inst_handles[i],
		    &sd, &sess_size, &ctx_size);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;

		QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
		if (session_handles[i] == NULL)
			goto fail;

		status = cpaDcInitSession(dc_inst_handles[i],
		    session_handles[i],
		    &sd, NULL, qat_dc_callback);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;
	}

	qat_dc_init_done = B_TRUE;
	return (0);
fail:
	qat_dc_clean();
	return (-1);
}

void
qat_dc_fini(void)
{
	if (!qat_dc_init_done)
		return;

	qat_dc_clean();
}

/*
 * The "add" parameter is an additional buffer which is passed
 * to QAT as a scratch buffer alongside the destination buffer
 * in case the "compressed" data ends up being larger than the
 * original source data. This is necessary to prevent QAT from
 * generating buffer overflow warnings for incompressible data.
 */
static int
qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
    char *dst, int dst_len, char *add, int add_len, size_t *c_len)
{
	CpaInstanceHandle dc_inst_handle;
	CpaDcSessionHandle session_handle;
	CpaBufferList *buf_list_src = NULL;
	CpaBufferList *buf_list_dst = NULL;
	CpaFlatBuffer *flat_buf_src = NULL;
	CpaFlatBuffer *flat_buf_dst = NULL;
	Cpa8U *buffer_meta_src = NULL;
	Cpa8U *buffer_meta_dst = NULL;
	Cpa32U buffer_meta_size = 0;
	CpaDcRqResults dc_results;
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U hdr_sz = 0;
	Cpa32U compressed_sz;
	Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
	Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
	Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
	Cpa32U bytes_left;
	Cpa32U dst_pages = 0;
	Cpa32U adler32 = 0;
	char *data;
	struct page *page;
	struct page **in_pages = NULL;
	struct page **out_pages = NULL;
	struct page **add_pages = NULL;
	Cpa32U page_off = 0;
	struct completion complete;
	Cpa32U page_num = 0;
	Cpa16U i;

	/*
	 * We increment num_src_buf and num_dst_buf by 2 to allow
	 * us to handle non page-aligned buffer addresses and buffers
	 * whose sizes are not divisible by PAGE_SIZE.
	 */
	Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
	    (num_src_buf * sizeof (CpaFlatBuffer));
	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
	    ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));

	if (QAT_PHYS_CONTIG_ALLOC(&in_pages,
	    num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
		goto fail;

	if (QAT_PHYS_CONTIG_ALLOC(&out_pages,
	    num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
		goto fail;

	if (QAT_PHYS_CONTIG_ALLOC(&add_pages,
	    num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
		goto fail;

	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
	dc_inst_handle = dc_inst_handles[i];
	session_handle = session_handles[i];

	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
	    &buffer_meta_size);
	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
	    &buffer_meta_size);
	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	/* build source buffer list */
	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);

	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */

	/* build destination buffer list */
	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);

	buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */

	buf_list_src->numBuffers = 0;
	buf_list_src->pPrivateMetaData = buffer_meta_src;
	bytes_left = src_len;
	data = src;
	page_num = 0;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		page = qat_mem_to_page(data);
		in_pages[page_num] = page;
		flat_buf_src->pData = kmap(page) + page_off;
		flat_buf_src->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);

		bytes_left -= flat_buf_src->dataLenInBytes;
		data += flat_buf_src->dataLenInBytes;
		flat_buf_src++;
		buf_list_src->numBuffers++;
		page_num++;
	}

	buf_list_dst->numBuffers = 0;
	buf_list_dst->pPrivateMetaData = buffer_meta_dst;
	bytes_left = dst_len;
	data = dst;
	page_num = 0;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		page = qat_mem_to_page(data);
		flat_buf_dst->pData = kmap(page) + page_off;
		out_pages[page_num] = page;
		flat_buf_dst->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);

		bytes_left -= flat_buf_dst->dataLenInBytes;
		data += flat_buf_dst->dataLenInBytes;
		flat_buf_dst++;
		buf_list_dst->numBuffers++;
		page_num++;
		dst_pages++;
	}

	/* map additional scratch pages into the destination buffer list */
	bytes_left = add_len;
	data = add;
	page_num = 0;
	while (bytes_left > 0) {
		page_off = ((long)data & ~PAGE_MASK);
		page = qat_mem_to_page(data);
		flat_buf_dst->pData = kmap(page) + page_off;
		add_pages[page_num] = page;
		flat_buf_dst->dataLenInBytes =
		    min((long)PAGE_SIZE - page_off, (long)bytes_left);

		bytes_left -= flat_buf_dst->dataLenInBytes;
		data += flat_buf_dst->dataLenInBytes;
		flat_buf_dst++;
		buf_list_dst->numBuffers++;
		page_num++;
	}

	init_completion(&complete);

	if (dir == QAT_COMPRESS) {
		QAT_STAT_BUMP(comp_requests);
		QAT_STAT_INCR(comp_total_in_bytes, src_len);

		cpaDcGenerateHeader(session_handle,
		    buf_list_dst->pBuffers, &hdr_sz);
		buf_list_dst->pBuffers->pData += hdr_sz;
		buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
		status = cpaDcCompressData(
		    dc_inst_handle, session_handle,
		    buf_list_src, buf_list_dst,
		    &dc_results, CPA_DC_FLUSH_FINAL,
		    &complete);
		if (status != CPA_STATUS_SUCCESS) {
			goto fail;
		}

		/* we now wait until the completion of the operation. */
		if (!wait_for_completion_interruptible_timeout(&complete,
		    QAT_TIMEOUT_MS)) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		if (dc_results.status != CPA_STATUS_SUCCESS) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		compressed_sz = dc_results.produced;
		if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
			status = CPA_STATUS_INCOMPRESSIBLE;
			goto fail;
		}

		flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
		/* move to the last page */
		flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;

		/* no space for gzip footer in the last page */
		if (((compressed_sz + hdr_sz) % PAGE_SIZE)
		    + ZLIB_FOOT_SZ > PAGE_SIZE) {
			status = CPA_STATUS_INCOMPRESSIBLE;
			goto fail;
		}

		/* jump to the end of the buffer and append footer */
		flat_buf_dst->pData =
		    (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
		    + ((compressed_sz + hdr_sz) % PAGE_SIZE);
		flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;

		dc_results.produced = 0;
		status = cpaDcGenerateFooter(session_handle,
		    flat_buf_dst, &dc_results);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;

		*c_len = compressed_sz + dc_results.produced + hdr_sz;
		QAT_STAT_INCR(comp_total_out_bytes, *c_len);
	} else {
		ASSERT3U(dir, ==, QAT_DECOMPRESS);
		QAT_STAT_BUMP(decomp_requests);
		QAT_STAT_INCR(decomp_total_in_bytes, src_len);

		buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
		buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
		status = cpaDcDecompressData(dc_inst_handle, session_handle,
		    buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
		    &complete);

		if (CPA_STATUS_SUCCESS != status) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		/* we now wait until the completion of the operation. */
		if (!wait_for_completion_interruptible_timeout(&complete,
		    QAT_TIMEOUT_MS)) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		if (dc_results.status != CPA_STATUS_SUCCESS) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		/* verify adler checksum */
		adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
		if (adler32 != BSWAP_32(dc_results.checksum)) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}
		*c_len = dc_results.produced;
		QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
	}

fail:
	if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
		QAT_STAT_BUMP(dc_fails);

	if (in_pages) {
		for (page_num = 0;
		    page_num < buf_list_src->numBuffers;
		    page_num++) {
			kunmap(in_pages[page_num]);
		}
		QAT_PHYS_CONTIG_FREE(in_pages);
	}

	if (out_pages) {
		for (page_num = 0; page_num < dst_pages; page_num++) {
			kunmap(out_pages[page_num]);
		}
		QAT_PHYS_CONTIG_FREE(out_pages);
	}

	if (add_pages) {
		for (page_num = 0;
		    page_num < buf_list_dst->numBuffers - dst_pages;
		    page_num++) {
			kunmap(add_pages[page_num]);
		}
		QAT_PHYS_CONTIG_FREE(add_pages);
	}

	QAT_PHYS_CONTIG_FREE(buffer_meta_src);
	QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
	QAT_PHYS_CONTIG_FREE(buf_list_src);
	QAT_PHYS_CONTIG_FREE(buf_list_dst);

	return (status);
}

/*
 * Entry point for QAT accelerated compression / decompression.
 */
int
qat_compress(qat_compress_dir_t dir, char *src, int src_len,
    char *dst, int dst_len, size_t *c_len)
{
	int ret;
	size_t add_len = 0;
	void *add = NULL;

	if (dir == QAT_COMPRESS) {
		add_len = dst_len;
		add = zio_data_buf_alloc(add_len);
	}

	ret = qat_compress_impl(dir, src, src_len, dst,
	    dst_len, add, add_len, c_len);

	if (dir == QAT_COMPRESS)
		zio_data_buf_free(add, add_len);

	return (ret);
}

static int
param_set_qat_compress(const char *val, struct kernel_param *kp)
{
	int ret;
	int *pvalue = kp->arg;
	ret = param_set_int(val, kp);
	if (ret)
		return (ret);
	/*
	 * zfs_qat_compress_disable = 0: enable qat compress
	 * try to initialize qat instance if it has not been done
	 */
	if (*pvalue == 0 && !qat_dc_init_done) {
		ret = qat_dc_init();
		if (ret != 0) {
			zfs_qat_compress_disable = 1;
			return (ret);
		}
	}
	return (ret);
}

module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
    param_get_int, &zfs_qat_compress_disable, 0644);
MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");

#endif
Commit	Line	Data
6a9d6359	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21
	22	#if defined(_KERNEL) && defined(HAVE_QAT)
	23	#include <linux/slab.h>
	24	#include <linux/vmalloc.h>
	25	#include <linux/pagemap.h>
	26	#include <linux/completion.h>
5090f727	27	#include <linux/mod_compat.h>
6a9d6359	28	#include <sys/zfs_context.h>
5090f727	29	#include <sys/byteorder.h>
32dce2da	30	#include <sys/zio.h>
cf637391	31	#include "qat.h"
6a9d6359	32
6a9d6359	33	/*
8d9e7c8f TC	34	* Max instances in a QAT device, each instance is a channel to submit
	35	* jobs to QAT hardware, this is only for pre-allocating instance and
	36	* session arrays; the actual number of instances are defined in the
	37	* QAT driver's configuration file.
6a9d6359	38	*/
cf637391	39	#define QAT_DC_MAX_INSTANCES 48
6a9d6359	40
	41	/*
	42	* ZLIB head and foot size
	43	*/
	44	#define ZLIB_HEAD_SZ 2
	45	#define ZLIB_FOOT_SZ 4
	46
cf637391 TC	47	static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
	48	static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
	49	static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
1cfdb0e6	50	static Cpa16U num_inst = 0;
7a25f089	51	static Cpa32U inst_num = 0;
cf637391	52	static boolean_t qat_dc_init_done = B_FALSE;
38742209	53	int zfs_qat_compress_disable = 0;
6a9d6359	54
cf637391 TC	55	boolean_t
cf637391 TC	56	qat_dc_use_accel(size_t s_len)
6a9d6359	57	{
38742209	58	return (!zfs_qat_compress_disable &&
cf637391 TC	59	qat_dc_init_done &&
	60	s_len >= QAT_MIN_BUF_SIZE &&
	61	s_len <= QAT_MAX_BUF_SIZE);
6a9d6359	62	}
	63
	64	static void
	65	qat_dc_callback(void *p_callback, CpaStatus status)
	66	{
	67	if (p_callback != NULL)
	68	complete((struct completion *)p_callback);
	69	}
	70
6a9d6359	71	static void
cf637391	72	qat_dc_clean(void)
6a9d6359	73	{
	74	Cpa16U buff_num = 0;
	75	Cpa16U num_inter_buff_lists = 0;
6a9d6359	76
bc8a6a60	77	for (Cpa16U i = 0; i < num_inst; i++) {
6a9d6359	78	cpaDcStopInstance(dc_inst_handles[i]);
cf637391	79	QAT_PHYS_CONTIG_FREE(session_handles[i]);
6a9d6359	80	/* free intermediate buffers */
	81	if (buffer_array[i] != NULL) {
	82	cpaDcGetNumIntermediateBuffers(
	83	dc_inst_handles[i], &num_inter_buff_lists);
	84	for (buff_num = 0; buff_num < num_inter_buff_lists;
	85	buff_num++) {
	86	CpaBufferList *buffer_inter =
	87	buffer_array[i][buff_num];
	88	if (buffer_inter->pBuffers) {
cf637391	89	QAT_PHYS_CONTIG_FREE(
6a9d6359	90	buffer_inter->pBuffers->pData);
cf637391	91	QAT_PHYS_CONTIG_FREE(
6a9d6359	92	buffer_inter->pBuffers);
6a9d6359	93	}
cf637391	94	QAT_PHYS_CONTIG_FREE(
6a9d6359	95	buffer_inter->pPrivateMetaData);
cf637391	96	QAT_PHYS_CONTIG_FREE(buffer_inter);
6a9d6359	97	}
	98	}
	99	}
	100
	101	num_inst = 0;
cf637391	102	qat_dc_init_done = B_FALSE;
6a9d6359	103	}
	104
	105	int
cf637391	106	qat_dc_init(void)
6a9d6359	107	{
	108	CpaStatus status = CPA_STATUS_SUCCESS;
	109	Cpa32U sess_size = 0;
	110	Cpa32U ctx_size = 0;
	111	Cpa16U num_inter_buff_lists = 0;
	112	Cpa16U buff_num = 0;
	113	Cpa32U buff_meta_size = 0;
	114	CpaDcSessionSetupData sd = {0};
6a9d6359	115
5090f727 CZ	116	if (qat_dc_init_done)
	117	return (0);
	118
6a9d6359	119	status = cpaDcGetNumInstances(&num_inst);
cf637391	120	if (status != CPA_STATUS_SUCCESS)
6a9d6359	121	return (-1);
6a9d6359	122
cf637391 TC	123	/* if the user has configured no QAT compression units just return */
	124	if (num_inst == 0)
	125	return (0);
	126
	127	if (num_inst > QAT_DC_MAX_INSTANCES)
	128	num_inst = QAT_DC_MAX_INSTANCES;
6a9d6359	129
	130	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
	131	if (status != CPA_STATUS_SUCCESS)
	132	return (-1);
	133
bc8a6a60	134	for (Cpa16U i = 0; i < num_inst; i++) {
6a9d6359	135	cpaDcSetAddressTranslation(dc_inst_handles[i],
	136	(void*)virt_to_phys);
	137
	138	status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
	139	1, &buff_meta_size);
	140
	141	if (status == CPA_STATUS_SUCCESS)
	142	status = cpaDcGetNumIntermediateBuffers(
	143	dc_inst_handles[i], &num_inter_buff_lists);
	144
	145	if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
cf637391	146	status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
6a9d6359	147	num_inter_buff_lists *
	148	sizeof (CpaBufferList *));
	149
	150	for (buff_num = 0; buff_num < num_inter_buff_lists;
	151	buff_num++) {
	152	if (status == CPA_STATUS_SUCCESS)
cf637391	153	status = QAT_PHYS_CONTIG_ALLOC(
6a9d6359	154	&buffer_array[i][buff_num],
	155	sizeof (CpaBufferList));
	156
	157	if (status == CPA_STATUS_SUCCESS)
cf637391	158	status = QAT_PHYS_CONTIG_ALLOC(
6a9d6359	159	&buffer_array[i][buff_num]->
	160	pPrivateMetaData,
	161	buff_meta_size);
	162
	163	if (status == CPA_STATUS_SUCCESS)
cf637391	164	status = QAT_PHYS_CONTIG_ALLOC(
6a9d6359	165	&buffer_array[i][buff_num]->pBuffers,
	166	sizeof (CpaFlatBuffer));
	167
	168	if (status == CPA_STATUS_SUCCESS) {
	169	/*
	170	* implementation requires an intermediate
	171	* buffer approximately twice the size of
	172	* output buffer, which is 2x max buffer
	173	* size here.
	174	*/
cf637391	175	status = QAT_PHYS_CONTIG_ALLOC(
6a9d6359	176	&buffer_array[i][buff_num]->pBuffers->
	177	pData, 2 * QAT_MAX_BUF_SIZE);
	178	if (status != CPA_STATUS_SUCCESS)
	179	goto fail;
	180
	181	buffer_array[i][buff_num]->numBuffers = 1;
	182	buffer_array[i][buff_num]->pBuffers->
	183	dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
	184	}
	185	}
	186
	187	status = cpaDcStartInstance(dc_inst_handles[i],
	188	num_inter_buff_lists, buffer_array[i]);
	189	if (status != CPA_STATUS_SUCCESS)
	190	goto fail;
	191
	192	sd.compLevel = CPA_DC_L1;
	193	sd.compType = CPA_DC_DEFLATE;
	194	sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
	195	sd.sessDirection = CPA_DC_DIR_COMBINED;
	196	sd.sessState = CPA_DC_STATELESS;
	197	sd.deflateWindowSize = 7;
	198	sd.checksum = CPA_DC_ADLER32;
	199	status = cpaDcGetSessionSize(dc_inst_handles[i],
	200	&sd, &sess_size, &ctx_size);
	201	if (status != CPA_STATUS_SUCCESS)
	202	goto fail;
	203
cf637391	204	QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
6a9d6359	205	if (session_handles[i] == NULL)
	206	goto fail;
	207
	208	status = cpaDcInitSession(dc_inst_handles[i],
	209	session_handles[i],
	210	&sd, NULL, qat_dc_callback);
	211	if (status != CPA_STATUS_SUCCESS)
	212	goto fail;
	213	}
	214
cf637391	215	qat_dc_init_done = B_TRUE;
6a9d6359	216	return (0);
6a9d6359	217	fail:
cf637391	218	qat_dc_clean();
6a9d6359	219	return (-1);
	220	}
	221
	222	void
cf637391	223	qat_dc_fini(void)
6a9d6359	224	{
cf637391 TC	225	if (!qat_dc_init_done)
cf637391 TC	226	return;
6a9d6359	227
cf637391	228	qat_dc_clean();
6a9d6359	229	}
6a9d6359	230
32dce2da TC	231	/*
	232	* The "add" parameter is an additional buffer which is passed
	233	* to QAT as a scratch buffer alongside the destination buffer
	234	* in case the "compressed" data ends up being larger than the
	235	* original source data. This is necessary to prevent QAT from
	236	* generating buffer overflow warnings for incompressible data.
	237	*/
	238	static int
	239	qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
	240	char dst, int dst_len, char add, int add_len, size_t *c_len)
6a9d6359	241	{
	242	CpaInstanceHandle dc_inst_handle;
	243	CpaDcSessionHandle session_handle;
	244	CpaBufferList *buf_list_src = NULL;
	245	CpaBufferList *buf_list_dst = NULL;
	246	CpaFlatBuffer *flat_buf_src = NULL;
	247	CpaFlatBuffer *flat_buf_dst = NULL;
	248	Cpa8U *buffer_meta_src = NULL;
	249	Cpa8U *buffer_meta_dst = NULL;
	250	Cpa32U buffer_meta_size = 0;
	251	CpaDcRqResults dc_results;
	252	CpaStatus status = CPA_STATUS_SUCCESS;
	253	Cpa32U hdr_sz = 0;
	254	Cpa32U compressed_sz;
17dd8835 TC	255	Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
17dd8835 TC	256	Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
32dce2da	257	Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
6a9d6359	258	Cpa32U bytes_left;
32dce2da	259	Cpa32U dst_pages = 0;
5090f727	260	Cpa32U adler32 = 0;
6a9d6359	261	char *data;
32dce2da	262	struct page *page;
6a9d6359	263	struct page **in_pages = NULL;
6a9d6359	264	struct page **out_pages = NULL;
32dce2da	265	struct page **add_pages = NULL;
17dd8835	266	Cpa32U page_off = 0;
6a9d6359	267	struct completion complete;
17dd8835	268	Cpa32U page_num = 0;
6a9d6359	269	Cpa16U i;
6a9d6359	270
17dd8835 TC	271	/*
	272	* We increment num_src_buf and num_dst_buf by 2 to allow
	273	* us to handle non page-aligned buffer addresses and buffers
	274	* whose sizes are not divisible by PAGE_SIZE.
	275	*/
6a9d6359	276	Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
	277	(num_src_buf * sizeof (CpaFlatBuffer));
	278	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
32dce2da	279	((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
6a9d6359	280
cf637391	281	if (QAT_PHYS_CONTIG_ALLOC(&in_pages,
6a9d6359	282	num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
	283	goto fail;
	284
cf637391	285	if (QAT_PHYS_CONTIG_ALLOC(&out_pages,
6a9d6359	286	num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
	287	goto fail;
	288
32dce2da TC	289	if (QAT_PHYS_CONTIG_ALLOC(&add_pages,
	290	num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
	291	goto fail;
	292
bc8a6a60	293	i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
6a9d6359	294	dc_inst_handle = dc_inst_handles[i];
	295	session_handle = session_handles[i];
	296
	297	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
	298	&buffer_meta_size);
cf637391	299	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
6a9d6359	300	CPA_STATUS_SUCCESS)
	301	goto fail;
	302
32dce2da	303	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
6a9d6359	304	&buffer_meta_size);
cf637391	305	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
6a9d6359	306	CPA_STATUS_SUCCESS)
	307	goto fail;
	308
	309	/* build source buffer list */
cf637391	310	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
6a9d6359	311	CPA_STATUS_SUCCESS)
	312	goto fail;
	313
	314	flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
	315
	316	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
	317
	318	/* build destination buffer list */
cf637391	319	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
6a9d6359	320	CPA_STATUS_SUCCESS)
	321	goto fail;
	322
	323	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
	324
	325	buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
	326
	327	buf_list_src->numBuffers = 0;
	328	buf_list_src->pPrivateMetaData = buffer_meta_src;
	329	bytes_left = src_len;
	330	data = src;
	331	page_num = 0;
	332	while (bytes_left > 0) {
17dd8835	333	page_off = ((long)data & ~PAGE_MASK);
32dce2da TC	334	page = qat_mem_to_page(data);
	335	in_pages[page_num] = page;
	336	flat_buf_src->pData = kmap(page) + page_off;
6a9d6359	337	flat_buf_src->dataLenInBytes =
17dd8835	338	min((long)PAGE_SIZE - page_off, (long)bytes_left);
6a9d6359	339
	340	bytes_left -= flat_buf_src->dataLenInBytes;
	341	data += flat_buf_src->dataLenInBytes;
	342	flat_buf_src++;
	343	buf_list_src->numBuffers++;
	344	page_num++;
	345	}
	346
	347	buf_list_dst->numBuffers = 0;
	348	buf_list_dst->pPrivateMetaData = buffer_meta_dst;
	349	bytes_left = dst_len;
	350	data = dst;
	351	page_num = 0;
	352	while (bytes_left > 0) {
17dd8835	353	page_off = ((long)data & ~PAGE_MASK);
32dce2da TC	354	page = qat_mem_to_page(data);
	355	flat_buf_dst->pData = kmap(page) + page_off;
	356	out_pages[page_num] = page;
	357	flat_buf_dst->dataLenInBytes =
	358	min((long)PAGE_SIZE - page_off, (long)bytes_left);
	359
	360	bytes_left -= flat_buf_dst->dataLenInBytes;
	361	data += flat_buf_dst->dataLenInBytes;
	362	flat_buf_dst++;
	363	buf_list_dst->numBuffers++;
	364	page_num++;
	365	dst_pages++;
	366	}
	367
	368	/* map additional scratch pages into the destination buffer list */
	369	bytes_left = add_len;
	370	data = add;
	371	page_num = 0;
	372	while (bytes_left > 0) {
	373	page_off = ((long)data & ~PAGE_MASK);
	374	page = qat_mem_to_page(data);
	375	flat_buf_dst->pData = kmap(page) + page_off;
	376	add_pages[page_num] = page;
6a9d6359	377	flat_buf_dst->dataLenInBytes =
17dd8835	378	min((long)PAGE_SIZE - page_off, (long)bytes_left);
6a9d6359	379
	380	bytes_left -= flat_buf_dst->dataLenInBytes;
	381	data += flat_buf_dst->dataLenInBytes;
	382	flat_buf_dst++;
	383	buf_list_dst->numBuffers++;
	384	page_num++;
	385	}
	386
	387	init_completion(&complete);
	388
	389	if (dir == QAT_COMPRESS) {
	390	QAT_STAT_BUMP(comp_requests);
	391	QAT_STAT_INCR(comp_total_in_bytes, src_len);
	392
	393	cpaDcGenerateHeader(session_handle,
	394	buf_list_dst->pBuffers, &hdr_sz);
	395	buf_list_dst->pBuffers->pData += hdr_sz;
	396	buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
	397	status = cpaDcCompressData(
	398	dc_inst_handle, session_handle,
	399	buf_list_src, buf_list_dst,
	400	&dc_results, CPA_DC_FLUSH_FINAL,
	401	&complete);
	402	if (status != CPA_STATUS_SUCCESS) {
	403	goto fail;
	404	}
	405
	406	/* we now wait until the completion of the operation. */
	407	if (!wait_for_completion_interruptible_timeout(&complete,
cf637391	408	QAT_TIMEOUT_MS)) {
6a9d6359	409	status = CPA_STATUS_FAIL;
	410	goto fail;
	411	}
	412
	413	if (dc_results.status != CPA_STATUS_SUCCESS) {
	414	status = CPA_STATUS_FAIL;
	415	goto fail;
	416	}
	417
	418	compressed_sz = dc_results.produced;
	419	if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
32dce2da	420	status = CPA_STATUS_INCOMPRESSIBLE;
6a9d6359	421	goto fail;
	422	}
	423
	424	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
	425	/* move to the last page */
	426	flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
	427
8d9e7c8f	428	/* no space for gzip footer in the last page */
6a9d6359	429	if (((compressed_sz + hdr_sz) % PAGE_SIZE)
32dce2da TC	430	+ ZLIB_FOOT_SZ > PAGE_SIZE) {
32dce2da TC	431	status = CPA_STATUS_INCOMPRESSIBLE;
6a9d6359	432	goto fail;
32dce2da	433	}
6a9d6359	434
a3df7fa7	435	/* jump to the end of the buffer and append footer */
	436	flat_buf_dst->pData =
	437	(char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
	438	+ ((compressed_sz + hdr_sz) % PAGE_SIZE);
6a9d6359	439	flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
	440
	441	dc_results.produced = 0;
	442	status = cpaDcGenerateFooter(session_handle,
	443	flat_buf_dst, &dc_results);
32dce2da	444	if (status != CPA_STATUS_SUCCESS)
6a9d6359	445	goto fail;
6a9d6359	446
6a9d6359	447	*c_len = compressed_sz + dc_results.produced + hdr_sz;
6a9d6359	448	QAT_STAT_INCR(comp_total_out_bytes, *c_len);
cf637391 TC	449	} else {
cf637391 TC	450	ASSERT3U(dir, ==, QAT_DECOMPRESS);
6a9d6359	451	QAT_STAT_BUMP(decomp_requests);
	452	QAT_STAT_INCR(decomp_total_in_bytes, src_len);
	453
	454	buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
	455	buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
32dce2da TC	456	status = cpaDcDecompressData(dc_inst_handle, session_handle,
32dce2da TC	457	buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
6a9d6359	458	&complete);
	459
	460	if (CPA_STATUS_SUCCESS != status) {
	461	status = CPA_STATUS_FAIL;
	462	goto fail;
	463	}
	464
	465	/* we now wait until the completion of the operation. */
	466	if (!wait_for_completion_interruptible_timeout(&complete,
cf637391	467	QAT_TIMEOUT_MS)) {
6a9d6359	468	status = CPA_STATUS_FAIL;
	469	goto fail;
	470	}
	471
	472	if (dc_results.status != CPA_STATUS_SUCCESS) {
	473	status = CPA_STATUS_FAIL;
	474	goto fail;
	475	}
	476
5090f727 CZ	477	/* verify adler checksum */
	478	adler32 = (Cpa32U )(src + dc_results.consumed + ZLIB_HEAD_SZ);
	479	if (adler32 != BSWAP_32(dc_results.checksum)) {
	480	status = CPA_STATUS_FAIL;
	481	goto fail;
	482	}
6a9d6359	483	*c_len = dc_results.produced;
6a9d6359	484	QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
6a9d6359	485	}
	486
	487	fail:
32dce2da	488	if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
6a9d6359	489	QAT_STAT_BUMP(dc_fails);
6a9d6359	490
	491	if (in_pages) {
	492	for (page_num = 0;
	493	page_num < buf_list_src->numBuffers;
	494	page_num++) {
	495	kunmap(in_pages[page_num]);
	496	}
cf637391	497	QAT_PHYS_CONTIG_FREE(in_pages);
6a9d6359	498	}
	499
	500	if (out_pages) {
32dce2da	501	for (page_num = 0; page_num < dst_pages; page_num++) {
6a9d6359	502	kunmap(out_pages[page_num]);
6a9d6359	503	}
cf637391	504	QAT_PHYS_CONTIG_FREE(out_pages);
6a9d6359	505	}
6a9d6359	506
32dce2da TC	507	if (add_pages) {
	508	for (page_num = 0;
	509	page_num < buf_list_dst->numBuffers - dst_pages;
	510	page_num++) {
	511	kunmap(add_pages[page_num]);
	512	}
	513	QAT_PHYS_CONTIG_FREE(add_pages);
	514	}
	515
cf637391 TC	516	QAT_PHYS_CONTIG_FREE(buffer_meta_src);
	517	QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
	518	QAT_PHYS_CONTIG_FREE(buf_list_src);
	519	QAT_PHYS_CONTIG_FREE(buf_list_dst);
6a9d6359	520
32dce2da TC	521	return (status);
	522	}
	523
	524	/*
	525	* Entry point for QAT accelerated compression / decompression.
	526	*/
	527	int
	528	qat_compress(qat_compress_dir_t dir, char *src, int src_len,
	529	char dst, int dst_len, size_t c_len)
	530	{
	531	int ret;
	532	size_t add_len = 0;
	533	void *add = NULL;
	534
	535	if (dir == QAT_COMPRESS) {
	536	add_len = dst_len;
	537	add = zio_data_buf_alloc(add_len);
	538	}
	539
	540	ret = qat_compress_impl(dir, src, src_len, dst,
	541	dst_len, add, add_len, c_len);
	542
	543	if (dir == QAT_COMPRESS)
	544	zio_data_buf_free(add, add_len);
	545
6a9d6359	546	return (ret);
	547	}
	548
5090f727 CZ	549	static int
	550	param_set_qat_compress(const char val, struct kernel_param kp)
	551	{
	552	int ret;
	553	int *pvalue = kp->arg;
	554	ret = param_set_int(val, kp);
	555	if (ret)
	556	return (ret);
	557	/*
	558	* zfs_qat_compress_disable = 0: enable qat compress
	559	* try to initialize qat instance if it has not been done
	560	*/
	561	if (*pvalue == 0 && !qat_dc_init_done) {
	562	ret = qat_dc_init();
	563	if (ret != 0) {
	564	zfs_qat_compress_disable = 1;
	565	return (ret);
	566	}
	567	}
	568	return (ret);
	569	}
	570
	571	module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
	572	param_get_int, &zfs_qat_compress_disable, 0644);
	573	MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
38742209	574
6a9d6359	575	#endif