[mirror_zfs-debian.git] / module / zfs / qat_compress.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

#if defined(_KERNEL) && defined(HAVE_QAT)
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <sys/zfs_context.h>
#include "qat_compress.h"

/*
 * Timeout - no response from hardware after 0.5 seconds
 */
#define	TIMEOUT_MS		500

/*
 * Max instances in QAT device, each instance is a channel to submit
 * jobs to QAT hardware, this is only for pre-allocating instance,
 * and session arrays, the actual number of instances are defined in
 * the QAT driver's configure file.
 */
#define	MAX_INSTANCES		48

/*
 * ZLIB head and foot size
 */
#define	ZLIB_HEAD_SZ		2
#define	ZLIB_FOOT_SZ		4

/*
 * The minimal and maximal buffer size, which are not restricted
 * in the QAT hardware, but with the input buffer size between 4KB
 * and 128KB, the hardware can provide the optimal performance.
 */
#define	QAT_MIN_BUF_SIZE	(4*1024)
#define	QAT_MAX_BUF_SIZE	(128*1024)

/*
 * Used for qat kstat.
 */
typedef struct qat_stats {
	/*
	 * Number of jobs submitted to qat compression engine.
	 */
	kstat_named_t comp_requests;
	/*
	 * Total bytes sent to qat compression engine.
	 */
	kstat_named_t comp_total_in_bytes;
	/*
	 * Total bytes output from qat compression engine.
	 */
	kstat_named_t comp_total_out_bytes;
	/*
	 * Number of jobs submitted to qat de-compression engine.
	 */
	kstat_named_t decomp_requests;
	/*
	 * Total bytes sent to qat de-compression engine.
	 */
	kstat_named_t decomp_total_in_bytes;
	/*
	 * Total bytes output from qat de-compression engine.
	 */
	kstat_named_t decomp_total_out_bytes;
	/*
	 * Number of fails in qat engine.
	 * Note: when qat fail happens, it doesn't mean a critical hardware
	 * issue, sometimes it is because the output buffer is not big enough,
	 * and the compression job will be transfered to gzip software again,
	 * so the functionality of ZFS is not impacted.
	 */
	kstat_named_t dc_fails;
} qat_stats_t;

qat_stats_t qat_stats = {
	{ "comp_reqests",			KSTAT_DATA_UINT64 },
	{ "comp_total_in_bytes",		KSTAT_DATA_UINT64 },
	{ "comp_total_out_bytes",		KSTAT_DATA_UINT64 },
	{ "decomp_reqests",			KSTAT_DATA_UINT64 },
	{ "decomp_total_in_bytes",		KSTAT_DATA_UINT64 },
	{ "decomp_total_out_bytes",		KSTAT_DATA_UINT64 },
	{ "dc_fails",				KSTAT_DATA_UINT64 },
};

static kstat_t *qat_ksp;
static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES];
static CpaDcSessionHandle session_handles[MAX_INSTANCES];
static CpaBufferList **buffer_array[MAX_INSTANCES];
static Cpa16U num_inst = 0;
static Cpa32U inst_num = 0;
static boolean_t qat_init_done = B_FALSE;
int zfs_qat_disable = 0;

#define	QAT_STAT_INCR(stat, val) \
	atomic_add_64(&qat_stats.stat.value.ui64, (val));
#define	QAT_STAT_BUMP(stat) \
	QAT_STAT_INCR(stat, 1);

#define	PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes)	\
	mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))

#define	PHYS_CONTIG_FREE(p_mem_addr)	\
	mem_free_contig((void *)&(p_mem_addr))

static inline struct page *
mem_to_page(void *addr)
{
	if (!is_vmalloc_addr(addr))
		return (virt_to_page(addr));

	return (vmalloc_to_page(addr));
}

static void
qat_dc_callback(void *p_callback, CpaStatus status)
{
	if (p_callback != NULL)
		complete((struct completion *)p_callback);
}

static inline CpaStatus
mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
{
	*pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
	if (*pp_mem_addr == NULL)
		return (CPA_STATUS_RESOURCE);
	return (CPA_STATUS_SUCCESS);
}

static inline void
mem_free_contig(void **pp_mem_addr)
{
	if (*pp_mem_addr != NULL) {
		kfree(*pp_mem_addr);
		*pp_mem_addr = NULL;
	}
}

static void
qat_clean(void)
{
	Cpa16U buff_num = 0;
	Cpa16U num_inter_buff_lists = 0;
	Cpa16U i = 0;

	for (i = 0; i < num_inst; i++) {
		cpaDcStopInstance(dc_inst_handles[i]);
		PHYS_CONTIG_FREE(session_handles[i]);
		/* free intermediate buffers  */
		if (buffer_array[i] != NULL) {
			cpaDcGetNumIntermediateBuffers(
			    dc_inst_handles[i], &num_inter_buff_lists);
			for (buff_num = 0; buff_num < num_inter_buff_lists;
			    buff_num++) {
				CpaBufferList *buffer_inter =
				    buffer_array[i][buff_num];
				if (buffer_inter->pBuffers) {
					PHYS_CONTIG_FREE(
					    buffer_inter->pBuffers->pData);
					PHYS_CONTIG_FREE(
					    buffer_inter->pBuffers);
				}
				PHYS_CONTIG_FREE(
				    buffer_inter->pPrivateMetaData);
				PHYS_CONTIG_FREE(buffer_inter);
			}
		}
	}

	num_inst = 0;
	qat_init_done = B_FALSE;
}

int
qat_init(void)
{
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U sess_size = 0;
	Cpa32U ctx_size = 0;
	Cpa16U num_inter_buff_lists = 0;
	Cpa16U buff_num = 0;
	Cpa32U buff_meta_size = 0;
	CpaDcSessionSetupData sd = {0};
	Cpa16U i;

	status = cpaDcGetNumInstances(&num_inst);
	if (status != CPA_STATUS_SUCCESS || num_inst == 0)
		return (-1);

	if (num_inst > MAX_INSTANCES)
		num_inst = MAX_INSTANCES;

	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
	if (status != CPA_STATUS_SUCCESS)
		return (-1);

	for (i = 0; i < num_inst; i++) {
		cpaDcSetAddressTranslation(dc_inst_handles[i],
		    (void*)virt_to_phys);

		status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
		    1, &buff_meta_size);

		if (status == CPA_STATUS_SUCCESS)
			status = cpaDcGetNumIntermediateBuffers(
			    dc_inst_handles[i], &num_inter_buff_lists);

		if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
			status = PHYS_CONTIG_ALLOC(&buffer_array[i],
			    num_inter_buff_lists *
			    sizeof (CpaBufferList *));

		for (buff_num = 0; buff_num < num_inter_buff_lists;
		    buff_num++) {
			if (status == CPA_STATUS_SUCCESS)
				status = PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num],
				    sizeof (CpaBufferList));

			if (status == CPA_STATUS_SUCCESS)
				status = PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->
				    pPrivateMetaData,
				    buff_meta_size);

			if (status == CPA_STATUS_SUCCESS)
				status = PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->pBuffers,
				    sizeof (CpaFlatBuffer));

			if (status == CPA_STATUS_SUCCESS) {
				/*
				 *  implementation requires an intermediate
				 *  buffer approximately twice the size of
				 *  output buffer, which is 2x max buffer
				 *  size here.
				 */
				status = PHYS_CONTIG_ALLOC(
				    &buffer_array[i][buff_num]->pBuffers->
				    pData, 2 * QAT_MAX_BUF_SIZE);
				if (status != CPA_STATUS_SUCCESS)
					goto fail;

				buffer_array[i][buff_num]->numBuffers = 1;
				buffer_array[i][buff_num]->pBuffers->
				    dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
			}
		}

		status = cpaDcStartInstance(dc_inst_handles[i],
		    num_inter_buff_lists, buffer_array[i]);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;

		sd.compLevel = CPA_DC_L1;
		sd.compType = CPA_DC_DEFLATE;
		sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
		sd.sessDirection = CPA_DC_DIR_COMBINED;
		sd.sessState = CPA_DC_STATELESS;
		sd.deflateWindowSize = 7;
		sd.checksum = CPA_DC_ADLER32;
		status = cpaDcGetSessionSize(dc_inst_handles[i],
		    &sd, &sess_size, &ctx_size);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;

		PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
		if (session_handles[i] == NULL)
			goto fail;

		status = cpaDcInitSession(dc_inst_handles[i],
		    session_handles[i],
		    &sd, NULL, qat_dc_callback);
		if (status != CPA_STATUS_SUCCESS)
			goto fail;
	}

	qat_ksp = kstat_create("zfs", 0, "qat", "misc",
	    KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL);
	if (qat_ksp != NULL) {
		qat_ksp->ks_data = &qat_stats;
		kstat_install(qat_ksp);
	}

	qat_init_done = B_TRUE;
	return (0);
fail:
	qat_clean();
	return (-1);
}

void
qat_fini(void)
{
	qat_clean();

	if (qat_ksp != NULL) {
		kstat_delete(qat_ksp);
		qat_ksp = NULL;
	}
}

boolean_t
qat_use_accel(size_t s_len)
{
	return (!zfs_qat_disable &&
	    qat_init_done &&
	    s_len >= QAT_MIN_BUF_SIZE &&
	    s_len <= QAT_MAX_BUF_SIZE);
}

int
qat_compress(qat_compress_dir_t dir, char *src, int src_len,
    char *dst, int dst_len, size_t *c_len)
{
	CpaInstanceHandle dc_inst_handle;
	CpaDcSessionHandle session_handle;
	CpaBufferList *buf_list_src = NULL;
	CpaBufferList *buf_list_dst = NULL;
	CpaFlatBuffer *flat_buf_src = NULL;
	CpaFlatBuffer *flat_buf_dst = NULL;
	Cpa8U *buffer_meta_src = NULL;
	Cpa8U *buffer_meta_dst = NULL;
	Cpa32U buffer_meta_size = 0;
	CpaDcRqResults dc_results;
	CpaStatus status = CPA_STATUS_SUCCESS;
	Cpa32U hdr_sz = 0;
	Cpa32U compressed_sz;
	Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1;
	Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1;
	Cpa32U bytes_left;
	char *data;
	struct page *in_page, *out_page;
	struct page **in_pages = NULL;
	struct page **out_pages = NULL;
	struct completion complete;
	size_t ret = -1;
	Cpa16U page_num = 0;
	Cpa16U i;

	Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
	    (num_src_buf * sizeof (CpaFlatBuffer));
	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
	    (num_dst_buf * sizeof (CpaFlatBuffer));

	if (PHYS_CONTIG_ALLOC(&in_pages,
	    num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
		goto fail;

	if (PHYS_CONTIG_ALLOC(&out_pages,
	    num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
		goto fail;

	i = atomic_inc_32_nv(&inst_num) % num_inst;
	dc_inst_handle = dc_inst_handles[i];
	session_handle = session_handles[i];

	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
	    &buffer_meta_size);
	if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf,
	    &buffer_meta_size);
	if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	/* build source buffer list */
	if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);

	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */

	/* build destination buffer list */
	if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
	    CPA_STATUS_SUCCESS)
		goto fail;

	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);

	buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */

	buf_list_src->numBuffers = 0;
	buf_list_src->pPrivateMetaData = buffer_meta_src;
	bytes_left = src_len;
	data = src;
	page_num = 0;
	while (bytes_left > 0) {
		in_page = mem_to_page(data);
		in_pages[page_num] = in_page;
		flat_buf_src->pData = kmap(in_page);
		flat_buf_src->dataLenInBytes =
		    min((long)bytes_left, (long)PAGE_SIZE);

		bytes_left -= flat_buf_src->dataLenInBytes;
		data += flat_buf_src->dataLenInBytes;
		flat_buf_src++;
		buf_list_src->numBuffers++;
		page_num++;
	}

	buf_list_dst->numBuffers = 0;
	buf_list_dst->pPrivateMetaData = buffer_meta_dst;
	bytes_left = dst_len;
	data = dst;
	page_num = 0;
	while (bytes_left > 0) {
		out_page = mem_to_page(data);
		flat_buf_dst->pData = kmap(out_page);
		out_pages[page_num] = out_page;
		flat_buf_dst->dataLenInBytes =
		    min((long)bytes_left, (long)PAGE_SIZE);

		bytes_left -= flat_buf_dst->dataLenInBytes;
		data += flat_buf_dst->dataLenInBytes;
		flat_buf_dst++;
		buf_list_dst->numBuffers++;
		page_num++;
	}

	init_completion(&complete);

	if (dir == QAT_COMPRESS) {
		QAT_STAT_BUMP(comp_requests);
		QAT_STAT_INCR(comp_total_in_bytes, src_len);

		cpaDcGenerateHeader(session_handle,
		    buf_list_dst->pBuffers, &hdr_sz);
		buf_list_dst->pBuffers->pData += hdr_sz;
		buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
		status = cpaDcCompressData(
		    dc_inst_handle, session_handle,
		    buf_list_src, buf_list_dst,
		    &dc_results, CPA_DC_FLUSH_FINAL,
		    &complete);
		if (status != CPA_STATUS_SUCCESS) {
			goto fail;
		}

		/* we now wait until the completion of the operation. */
		if (!wait_for_completion_interruptible_timeout(&complete,
		    TIMEOUT_MS)) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		if (dc_results.status != CPA_STATUS_SUCCESS) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		compressed_sz = dc_results.produced;
		if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
			goto fail;
		}

		flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
		/* move to the last page */
		flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;

		/* no space for gzip foot in the last page */
		if (((compressed_sz + hdr_sz) % PAGE_SIZE)
		    + ZLIB_FOOT_SZ > PAGE_SIZE)
			goto fail;

		/* jump to the end of the buffer and append footer */
		flat_buf_dst->pData =
		    (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
		    + ((compressed_sz + hdr_sz) % PAGE_SIZE);
		flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;

		dc_results.produced = 0;
		status = cpaDcGenerateFooter(session_handle,
		    flat_buf_dst, &dc_results);
		if (status != CPA_STATUS_SUCCESS) {
			goto fail;
		}

		*c_len = compressed_sz + dc_results.produced + hdr_sz;

		QAT_STAT_INCR(comp_total_out_bytes, *c_len);

		ret = 0;

	} else if (dir == QAT_DECOMPRESS) {
		QAT_STAT_BUMP(decomp_requests);
		QAT_STAT_INCR(decomp_total_in_bytes, src_len);

		buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
		buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
		status = cpaDcDecompressData(dc_inst_handle,
		    session_handle,
		    buf_list_src,
		    buf_list_dst,
		    &dc_results,
		    CPA_DC_FLUSH_FINAL,
		    &complete);

		if (CPA_STATUS_SUCCESS != status) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		/* we now wait until the completion of the operation. */
		if (!wait_for_completion_interruptible_timeout(&complete,
		    TIMEOUT_MS)) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		if (dc_results.status != CPA_STATUS_SUCCESS) {
			status = CPA_STATUS_FAIL;
			goto fail;
		}

		*c_len = dc_results.produced;

		QAT_STAT_INCR(decomp_total_out_bytes, *c_len);

		ret = 0;
	}

fail:
	if (status != CPA_STATUS_SUCCESS) {
		QAT_STAT_BUMP(dc_fails);
	}

	if (in_pages) {
		for (page_num = 0;
		    page_num < buf_list_src->numBuffers;
		    page_num++) {
			kunmap(in_pages[page_num]);
		}
		PHYS_CONTIG_FREE(in_pages);
	}

	if (out_pages) {
		for (page_num = 0;
		    page_num < buf_list_dst->numBuffers;
		    page_num++) {
			kunmap(out_pages[page_num]);
		}
		PHYS_CONTIG_FREE(out_pages);
	}

	PHYS_CONTIG_FREE(buffer_meta_src);
	PHYS_CONTIG_FREE(buffer_meta_dst);
	PHYS_CONTIG_FREE(buf_list_src);
	PHYS_CONTIG_FREE(buf_list_dst);

	return (ret);
}

module_param(zfs_qat_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression");

#endif
Commit	Line	Data
cae5b340 AX	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21
	22	#if defined(_KERNEL) && defined(HAVE_QAT)
	23	#include <linux/slab.h>
	24	#include <linux/vmalloc.h>
	25	#include <linux/pagemap.h>
	26	#include <linux/completion.h>
	27	#include <sys/zfs_context.h>
	28	#include "qat_compress.h"
	29
	30	/*
	31	* Timeout - no response from hardware after 0.5 seconds
	32	*/
	33	#define TIMEOUT_MS 500
	34
	35	/*
	36	* Max instances in QAT device, each instance is a channel to submit
41d74433 AX	37	* jobs to QAT hardware, this is only for pre-allocating instance,
	38	* and session arrays, the actual number of instances are defined in
	39	* the QAT driver's configure file.
cae5b340	40	*/
41d74433	41	#define MAX_INSTANCES 48
cae5b340 AX	42
	43	/*
	44	* ZLIB head and foot size
	45	*/
	46	#define ZLIB_HEAD_SZ 2
	47	#define ZLIB_FOOT_SZ 4
	48
	49	/*
	50	* The minimal and maximal buffer size, which are not restricted
	51	* in the QAT hardware, but with the input buffer size between 4KB
	52	* and 128KB, the hardware can provide the optimal performance.
	53	*/
	54	#define QAT_MIN_BUF_SIZE (4*1024)
	55	#define QAT_MAX_BUF_SIZE (128*1024)
	56
	57	/*
	58	* Used for qat kstat.
	59	*/
	60	typedef struct qat_stats {
	61	/*
	62	* Number of jobs submitted to qat compression engine.
	63	*/
	64	kstat_named_t comp_requests;
	65	/*
	66	* Total bytes sent to qat compression engine.
	67	*/
	68	kstat_named_t comp_total_in_bytes;
	69	/*
	70	* Total bytes output from qat compression engine.
	71	*/
	72	kstat_named_t comp_total_out_bytes;
	73	/*
	74	* Number of jobs submitted to qat de-compression engine.
	75	*/
	76	kstat_named_t decomp_requests;
	77	/*
	78	* Total bytes sent to qat de-compression engine.
	79	*/
	80	kstat_named_t decomp_total_in_bytes;
	81	/*
	82	* Total bytes output from qat de-compression engine.
	83	*/
	84	kstat_named_t decomp_total_out_bytes;
	85	/*
	86	* Number of fails in qat engine.
	87	* Note: when qat fail happens, it doesn't mean a critical hardware
	88	* issue, sometimes it is because the output buffer is not big enough,
	89	* and the compression job will be transfered to gzip software again,
	90	* so the functionality of ZFS is not impacted.
	91	*/
	92	kstat_named_t dc_fails;
	93	} qat_stats_t;
	94
	95	qat_stats_t qat_stats = {
	96	{ "comp_reqests", KSTAT_DATA_UINT64 },
	97	{ "comp_total_in_bytes", KSTAT_DATA_UINT64 },
	98	{ "comp_total_out_bytes", KSTAT_DATA_UINT64 },
	99	{ "decomp_reqests", KSTAT_DATA_UINT64 },
	100	{ "decomp_total_in_bytes", KSTAT_DATA_UINT64 },
	101	{ "decomp_total_out_bytes", KSTAT_DATA_UINT64 },
	102	{ "dc_fails", KSTAT_DATA_UINT64 },
	103	};
	104
	105	static kstat_t *qat_ksp;
106	static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES];
107	static CpaDcSessionHandle session_handles[MAX_INSTANCES];
108	static CpaBufferList **buffer_array[MAX_INSTANCES];
41d74433	109	static Cpa16U num_inst = 0;
cae5b340 AX	110	static Cpa32U inst_num = 0;
	111	static boolean_t qat_init_done = B_FALSE;
	112	int zfs_qat_disable = 0;
	113
	114	#define QAT_STAT_INCR(stat, val) \
	115	atomic_add_64(&qat_stats.stat.value.ui64, (val));
	116	#define QAT_STAT_BUMP(stat) \
	117	QAT_STAT_INCR(stat, 1);
	118
	119	#define PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \
	120	mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
	121
	122	#define PHYS_CONTIG_FREE(p_mem_addr) \
	123	mem_free_contig((void *)&(p_mem_addr))
	124
	125	static inline struct page *
	126	mem_to_page(void *addr)
	127	{
	128	if (!is_vmalloc_addr(addr))
	129	return (virt_to_page(addr));
	130
	131	return (vmalloc_to_page(addr));
	132	}
	133
	134	static void
	135	qat_dc_callback(void *p_callback, CpaStatus status)
	136	{
	137	if (p_callback != NULL)
	138	complete((struct completion *)p_callback);
	139	}
	140
	141	static inline CpaStatus
	142	mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
	143	{
	144	*pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
	145	if (*pp_mem_addr == NULL)
	146	return (CPA_STATUS_RESOURCE);
	147	return (CPA_STATUS_SUCCESS);
	148	}
	149
	150	static inline void
	151	mem_free_contig(void **pp_mem_addr)
	152	{
	153	if (*pp_mem_addr != NULL) {
	154	kfree(*pp_mem_addr);
	155	*pp_mem_addr = NULL;
	156	}
	157	}
	158
	159	static void
	160	qat_clean(void)
	161	{
	162	Cpa16U buff_num = 0;
	163	Cpa16U num_inter_buff_lists = 0;
	164	Cpa16U i = 0;
	165
	166	for (i = 0; i < num_inst; i++) {
	167	cpaDcStopInstance(dc_inst_handles[i]);
	168	PHYS_CONTIG_FREE(session_handles[i]);
	169	/* free intermediate buffers */
	170	if (buffer_array[i] != NULL) {
	171	cpaDcGetNumIntermediateBuffers(
	172	dc_inst_handles[i], &num_inter_buff_lists);
	173	for (buff_num = 0; buff_num < num_inter_buff_lists;
174	buff_num++) {
175	CpaBufferList *buffer_inter =
176	buffer_array[i][buff_num];
177	if (buffer_inter->pBuffers) {
178	PHYS_CONTIG_FREE(
179	buffer_inter->pBuffers->pData);
180	PHYS_CONTIG_FREE(
181	buffer_inter->pBuffers);
182	}
183	PHYS_CONTIG_FREE(
184	buffer_inter->pPrivateMetaData);
185	PHYS_CONTIG_FREE(buffer_inter);
186	}
187	}
188	}
189
190	num_inst = 0;
191	qat_init_done = B_FALSE;
192	}
193
194	int
195	qat_init(void)
196	{
197	CpaStatus status = CPA_STATUS_SUCCESS;
198	Cpa32U sess_size = 0;
199	Cpa32U ctx_size = 0;
200	Cpa16U num_inter_buff_lists = 0;
201	Cpa16U buff_num = 0;
202	Cpa32U buff_meta_size = 0;
203	CpaDcSessionSetupData sd = {0};
204	Cpa16U i;
205
206	status = cpaDcGetNumInstances(&num_inst);
207	if (status != CPA_STATUS_SUCCESS \|\| num_inst == 0)
208	return (-1);
209
210	if (num_inst > MAX_INSTANCES)
211	num_inst = MAX_INSTANCES;
212
213	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
214	if (status != CPA_STATUS_SUCCESS)
215	return (-1);
216
217	for (i = 0; i < num_inst; i++) {
218	cpaDcSetAddressTranslation(dc_inst_handles[i],
219	(void*)virt_to_phys);
220
221	status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
222	1, &buff_meta_size);
223
224	if (status == CPA_STATUS_SUCCESS)
225	status = cpaDcGetNumIntermediateBuffers(
226	dc_inst_handles[i], &num_inter_buff_lists);
227
228	if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
229	status = PHYS_CONTIG_ALLOC(&buffer_array[i],
230	num_inter_buff_lists *
231	sizeof (CpaBufferList *));
232
233	for (buff_num = 0; buff_num < num_inter_buff_lists;
234	buff_num++) {
235	if (status == CPA_STATUS_SUCCESS)
236	status = PHYS_CONTIG_ALLOC(
237	&buffer_array[i][buff_num],
238	sizeof (CpaBufferList));
239
240	if (status == CPA_STATUS_SUCCESS)
241	status = PHYS_CONTIG_ALLOC(
242	&buffer_array[i][buff_num]->
243	pPrivateMetaData,
244	buff_meta_size);
245
246	if (status == CPA_STATUS_SUCCESS)
247	status = PHYS_CONTIG_ALLOC(
248	&buffer_array[i][buff_num]->pBuffers,
249	sizeof (CpaFlatBuffer));
250
251	if (status == CPA_STATUS_SUCCESS) {
252	/*
253	* implementation requires an intermediate
254	* buffer approximately twice the size of
255	* output buffer, which is 2x max buffer
256	* size here.
257	*/
258	status = PHYS_CONTIG_ALLOC(
259	&buffer_array[i][buff_num]->pBuffers->
260	pData, 2 * QAT_MAX_BUF_SIZE);
261	if (status != CPA_STATUS_SUCCESS)
262	goto fail;
263
264	buffer_array[i][buff_num]->numBuffers = 1;
265	buffer_array[i][buff_num]->pBuffers->
266	dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
267	}
268	}
269
270	status = cpaDcStartInstance(dc_inst_handles[i],
271	num_inter_buff_lists, buffer_array[i]);
272	if (status != CPA_STATUS_SUCCESS)
273	goto fail;
274
275	sd.compLevel = CPA_DC_L1;
276	sd.compType = CPA_DC_DEFLATE;
277	sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
278	sd.sessDirection = CPA_DC_DIR_COMBINED;
279	sd.sessState = CPA_DC_STATELESS;
280	sd.deflateWindowSize = 7;
281	sd.checksum = CPA_DC_ADLER32;
282	status = cpaDcGetSessionSize(dc_inst_handles[i],
283	&sd, &sess_size, &ctx_size);
284	if (status != CPA_STATUS_SUCCESS)
285	goto fail;
286
287	PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
288	if (session_handles[i] == NULL)
289	goto fail;
290
291	status = cpaDcInitSession(dc_inst_handles[i],
292	session_handles[i],
293	&sd, NULL, qat_dc_callback);
294	if (status != CPA_STATUS_SUCCESS)
295	goto fail;
296	}
297
298	qat_ksp = kstat_create("zfs", 0, "qat", "misc",
299	KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
300	KSTAT_FLAG_VIRTUAL);
301	if (qat_ksp != NULL) {
302	qat_ksp->ks_data = &qat_stats;
303	kstat_install(qat_ksp);
304	}
305
306	qat_init_done = B_TRUE;
307	return (0);
308	fail:
309	qat_clean();
310	return (-1);
311	}
312
313	void
314	qat_fini(void)
315	{
316	qat_clean();
317
318	if (qat_ksp != NULL) {
319	kstat_delete(qat_ksp);
320	qat_ksp = NULL;
321	}
322	}
323
324	boolean_t
325	qat_use_accel(size_t s_len)
326	{
327	return (!zfs_qat_disable &&
328	qat_init_done &&
329	s_len >= QAT_MIN_BUF_SIZE &&
330	s_len <= QAT_MAX_BUF_SIZE);
331	}
332
333	int
334	qat_compress(qat_compress_dir_t dir, char *src, int src_len,
335	char dst, int dst_len, size_t c_len)
336	{
337	CpaInstanceHandle dc_inst_handle;
338	CpaDcSessionHandle session_handle;
339	CpaBufferList *buf_list_src = NULL;
340	CpaBufferList *buf_list_dst = NULL;
341	CpaFlatBuffer *flat_buf_src = NULL;
342	CpaFlatBuffer *flat_buf_dst = NULL;
343	Cpa8U *buffer_meta_src = NULL;
344	Cpa8U *buffer_meta_dst = NULL;
345	Cpa32U buffer_meta_size = 0;
346	CpaDcRqResults dc_results;
347	CpaStatus status = CPA_STATUS_SUCCESS;
348	Cpa32U hdr_sz = 0;
349	Cpa32U compressed_sz;
350	Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1;
351	Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1;
352	Cpa32U bytes_left;
353	char *data;
354	struct page in_page, out_page;
355	struct page **in_pages = NULL;
356	struct page **out_pages = NULL;
357	struct completion complete;
358	size_t ret = -1;
359	Cpa16U page_num = 0;
360	Cpa16U i;
361
362	Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
363	(num_src_buf * sizeof (CpaFlatBuffer));
364	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
365	(num_dst_buf * sizeof (CpaFlatBuffer));
366
cae5b340 AX	367	if (PHYS_CONTIG_ALLOC(&in_pages,
	368	num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
	369	goto fail;
	370
	371	if (PHYS_CONTIG_ALLOC(&out_pages,
	372	num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
	373	goto fail;
	374
	375	i = atomic_inc_32_nv(&inst_num) % num_inst;
	376	dc_inst_handle = dc_inst_handles[i];
	377	session_handle = session_handles[i];
	378
	379	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
	380	&buffer_meta_size);
	381	if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
	382	CPA_STATUS_SUCCESS)
	383	goto fail;
	384
	385	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf,
	386	&buffer_meta_size);
	387	if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
	388	CPA_STATUS_SUCCESS)
	389	goto fail;
	390
	391	/* build source buffer list */
	392	if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
	393	CPA_STATUS_SUCCESS)
	394	goto fail;
	395
	396	flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
	397
	398	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
	399
	400	/* build destination buffer list */
	401	if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
	402	CPA_STATUS_SUCCESS)
	403	goto fail;
	404
	405	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
	406
	407	buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
	408
	409	buf_list_src->numBuffers = 0;
	410	buf_list_src->pPrivateMetaData = buffer_meta_src;
	411	bytes_left = src_len;
	412	data = src;
	413	page_num = 0;
	414	while (bytes_left > 0) {
	415	in_page = mem_to_page(data);
	416	in_pages[page_num] = in_page;
	417	flat_buf_src->pData = kmap(in_page);
	418	flat_buf_src->dataLenInBytes =
	419	min((long)bytes_left, (long)PAGE_SIZE);
	420
	421	bytes_left -= flat_buf_src->dataLenInBytes;
	422	data += flat_buf_src->dataLenInBytes;
	423	flat_buf_src++;
	424	buf_list_src->numBuffers++;
	425	page_num++;
	426	}
	427
	428	buf_list_dst->numBuffers = 0;
	429	buf_list_dst->pPrivateMetaData = buffer_meta_dst;
	430	bytes_left = dst_len;
431	data = dst;
432	page_num = 0;
433	while (bytes_left > 0) {
434	out_page = mem_to_page(data);
435	flat_buf_dst->pData = kmap(out_page);
436	out_pages[page_num] = out_page;
437	flat_buf_dst->dataLenInBytes =
438	min((long)bytes_left, (long)PAGE_SIZE);
439
440	bytes_left -= flat_buf_dst->dataLenInBytes;
441	data += flat_buf_dst->dataLenInBytes;
442	flat_buf_dst++;
443	buf_list_dst->numBuffers++;
444	page_num++;
445	}
446
447	init_completion(&complete);
448
449	if (dir == QAT_COMPRESS) {
450	QAT_STAT_BUMP(comp_requests);
451	QAT_STAT_INCR(comp_total_in_bytes, src_len);
452
453	cpaDcGenerateHeader(session_handle,
454	buf_list_dst->pBuffers, &hdr_sz);
455	buf_list_dst->pBuffers->pData += hdr_sz;
456	buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
457	status = cpaDcCompressData(
458	dc_inst_handle, session_handle,
459	buf_list_src, buf_list_dst,
460	&dc_results, CPA_DC_FLUSH_FINAL,
461	&complete);
462	if (status != CPA_STATUS_SUCCESS) {
463	goto fail;
464	}
465
466	/* we now wait until the completion of the operation. */
467	if (!wait_for_completion_interruptible_timeout(&complete,
468	TIMEOUT_MS)) {
469	status = CPA_STATUS_FAIL;
470	goto fail;
471	}
472
473	if (dc_results.status != CPA_STATUS_SUCCESS) {
474	status = CPA_STATUS_FAIL;
475	goto fail;
476	}
477
478	compressed_sz = dc_results.produced;
479	if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
480	goto fail;
481	}
482
483	flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
484	/* move to the last page */
485	flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
486
487	/* no space for gzip foot in the last page */
488	if (((compressed_sz + hdr_sz) % PAGE_SIZE)
489	+ ZLIB_FOOT_SZ > PAGE_SIZE)
490	goto fail;
491
41d74433 AX	492	/* jump to the end of the buffer and append footer */
	493	flat_buf_dst->pData =
	494	(char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
	495	+ ((compressed_sz + hdr_sz) % PAGE_SIZE);
cae5b340 AX	496	flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
	497
	498	dc_results.produced = 0;
	499	status = cpaDcGenerateFooter(session_handle,
	500	flat_buf_dst, &dc_results);
	501	if (status != CPA_STATUS_SUCCESS) {
	502	goto fail;
	503	}
	504
	505	*c_len = compressed_sz + dc_results.produced + hdr_sz;
	506
cae5b340 AX	507	QAT_STAT_INCR(comp_total_out_bytes, *c_len);
	508
	509	ret = 0;
	510
	511	} else if (dir == QAT_DECOMPRESS) {
	512	QAT_STAT_BUMP(decomp_requests);
	513	QAT_STAT_INCR(decomp_total_in_bytes, src_len);
	514
	515	buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
	516	buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
	517	status = cpaDcDecompressData(dc_inst_handle,
	518	session_handle,
	519	buf_list_src,
	520	buf_list_dst,
	521	&dc_results,
	522	CPA_DC_FLUSH_FINAL,
	523	&complete);
	524
	525	if (CPA_STATUS_SUCCESS != status) {
	526	status = CPA_STATUS_FAIL;
	527	goto fail;
	528	}
	529
	530	/* we now wait until the completion of the operation. */
	531	if (!wait_for_completion_interruptible_timeout(&complete,
	532	TIMEOUT_MS)) {
	533	status = CPA_STATUS_FAIL;
	534	goto fail;
	535	}
	536
	537	if (dc_results.status != CPA_STATUS_SUCCESS) {
	538	status = CPA_STATUS_FAIL;
	539	goto fail;
	540	}
	541
	542	*c_len = dc_results.produced;
	543
	544	QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
	545
	546	ret = 0;
	547	}
	548
	549	fail:
	550	if (status != CPA_STATUS_SUCCESS) {
	551	QAT_STAT_BUMP(dc_fails);
	552	}
	553
	554	if (in_pages) {
	555	for (page_num = 0;
	556	page_num < buf_list_src->numBuffers;
	557	page_num++) {
	558	kunmap(in_pages[page_num]);
	559	}
	560	PHYS_CONTIG_FREE(in_pages);
	561	}
	562
	563	if (out_pages) {
	564	for (page_num = 0;
	565	page_num < buf_list_dst->numBuffers;
	566	page_num++) {
	567	kunmap(out_pages[page_num]);
	568	}
	569	PHYS_CONTIG_FREE(out_pages);
	570	}
571
572	PHYS_CONTIG_FREE(buffer_meta_src);
573	PHYS_CONTIG_FREE(buffer_meta_dst);
574	PHYS_CONTIG_FREE(buf_list_src);
575	PHYS_CONTIG_FREE(buf_list_dst);
576
577	return (ret);
578	}
579
580	module_param(zfs_qat_disable, int, 0644);
581	MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression");
582
583	#endif