[mirror_zfs.git] / module / zfs / vdev_raidz_math_avx2.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
 */
#include <sys/isa_defs.h>

#if defined(__x86_64) && defined(HAVE_AVX2)

#include <sys/types.h>
#include <linux/simd_x86.h>

#define	__asm __asm__ __volatile__

#define	_REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
#define	REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)

#define	VR0_(REG, ...) "ymm"#REG
#define	VR1_(_1, REG, ...) "ymm"#REG
#define	VR2_(_1, _2, REG, ...) "ymm"#REG
#define	VR3_(_1, _2, _3, REG, ...) "ymm"#REG
#define	VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
#define	VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
#define	VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
#define	VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG

#define	VR0(r...) VR0_(r)
#define	VR1(r...) VR1_(r)
#define	VR2(r...) VR2_(r, 1)
#define	VR3(r...) VR3_(r, 1, 2)
#define	VR4(r...) VR4_(r, 1, 2)
#define	VR5(r...) VR5_(r, 1, 2, 3)
#define	VR6(r...) VR6_(r, 1, 2, 3, 4)
#define	VR7(r...) VR7_(r, 1, 2, 3, 4, 5)

#define	R_01(REG1, REG2, ...) REG1, REG2
#define	_R_23(_0, _1, REG2, REG3, ...) REG2, REG3
#define	R_23(REG...) _R_23(REG, 1, 2, 3)

#define	ZFS_ASM_BUG()	ASSERT(0)

extern const uint8_t gf_clmul_mod_lt[4*256][16];

#define	ELEM_SIZE 32

typedef struct v {
	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
} v_t;


#define	XOR_ACC(src, r...)						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
		    "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"	\
		    "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"	\
		    : : [SRC] "r" (src));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
		    : : [SRC] "r" (src));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	XOR(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		    "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"	\
		    "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"	\
		    "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"	\
		    "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r));		\
		break;							\
	case 4:								\
		__asm(							\
		    "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"	\
		    "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r));		\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	ZERO(r...)	XOR(r, r)

#define	COPY(r...) 							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		    "vmovdqa %" VR0(r) ", %" VR4(r) "\n"		\
		    "vmovdqa %" VR1(r) ", %" VR5(r) "\n"		\
		    "vmovdqa %" VR2(r) ", %" VR6(r) "\n"		\
		    "vmovdqa %" VR3(r) ", %" VR7(r));			\
		break;							\
	case 4:								\
		__asm(							\
		    "vmovdqa %" VR0(r) ", %" VR2(r) "\n"		\
		    "vmovdqa %" VR1(r) ", %" VR3(r));			\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	LOAD(src, r...) 						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
		    "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n"		\
		    "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n"		\
		    : : [SRC] "r" (src));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
		    : : [SRC] "r" (src));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	STORE(dst, r...)   						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
		    "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n"		\
		    "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n"		\
		    : : [DST] "r" (dst));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
		    : : [DST] "r" (dst));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	FLUSH()								\
{									\
	__asm("vzeroupper");						\
}

#define	MUL2_SETUP() 							\
{   									\
	__asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));	\
	__asm("vpbroadcastq %xmm14, %ymm14");				\
	__asm("vpxor        %ymm15, %ymm15 ,%ymm15");			\
}

#define	_MUL2(r...) 							\
{									\
	switch	(REG_CNT(r)) {						\
	case 2:								\
		__asm(							\
		    "vpcmpgtb %" VR0(r)", %ymm15,     %ymm12\n"		\
		    "vpcmpgtb %" VR1(r)", %ymm15,     %ymm13\n"		\
		    "vpaddb   %" VR0(r)", %" VR0(r)", %" VR0(r) "\n"	\
		    "vpaddb   %" VR1(r)", %" VR1(r)", %" VR1(r) "\n"	\
		    "vpand    %ymm14,     %ymm12,     %ymm12\n"		\
		    "vpand    %ymm14,     %ymm13,     %ymm13\n"		\
		    "vpxor    %ymm12,     %" VR0(r)", %" VR0(r) "\n"	\
		    "vpxor    %ymm13,     %" VR1(r)", %" VR1(r));	\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL2(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
	    _MUL2(R_01(r));						\
	    _MUL2(R_23(r));						\
	    break;							\
	case 2:								\
	    _MUL2(r);							\
	    break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL4(r...)							\
{									\
	MUL2(r);							\
	MUL2(r);							\
}

#define	_0f		"ymm15"
#define	_as		"ymm14"
#define	_bs		"ymm13"
#define	_ltmod		"ymm12"
#define	_ltmul		"ymm11"
#define	_ta		"ymm10"
#define	_tb		"ymm15"

static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;

#define	_MULx2(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 2:								\
		__asm(							\
		    "vpbroadcastb (%[mask]), %%" _0f "\n"		\
		    /* upper bits */					\
		    "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n"	\
		    "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n"	\
									\
		    "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"		\
		    "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"		\
		    "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    "vpand %%" _0f ", %%" _as ", %%" _as "\n"		\
		    "vpand %%" _0f ", %%" _bs ", %%" _bs "\n"		\
									\
		    "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"	\
		    "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"	\
		    "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"	\
		    "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"	\
		    /* lower bits */					\
		    "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n"	\
		    "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n"	\
									\
		    "vpxor %%" _ta ", %%" _as ", %%" _as "\n"		\
		    "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n"		\
									\
		    "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"	\
		    "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"	\
		    "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
		    "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
									\
		    "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    : : [mask] "r" (&_mul_mask),			\
		    [lt] "r" (gf_clmul_mod_lt[4*(c)]));			\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		_MULx2(c, R_01(r));					\
		_MULx2(c, R_23(r));					\
		break;							\
	case 2:								\
		_MULx2(c, R_01(r));					\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	raidz_math_begin()	kfpu_begin()
#define	raidz_math_end()						\
{									\
	FLUSH();							\
	kfpu_end();							\
}


#define	SYN_STRIDE		4

#define	ZERO_STRIDE		4
#define	ZERO_DEFINE()		{}
#define	ZERO_D			0, 1, 2, 3

#define	COPY_STRIDE		4
#define	COPY_DEFINE()		{}
#define	COPY_D			0, 1, 2, 3

#define	ADD_STRIDE		4
#define	ADD_DEFINE()		{}
#define	ADD_D 			0, 1, 2, 3

#define	MUL_STRIDE		4
#define	MUL_DEFINE() 		{}
#define	MUL_D			0, 1, 2, 3

#define	GEN_P_STRIDE		4
#define	GEN_P_DEFINE()		{}
#define	GEN_P_P			0, 1, 2, 3

#define	GEN_PQ_STRIDE		4
#define	GEN_PQ_DEFINE() 	{}
#define	GEN_PQ_D		0, 1, 2, 3
#define	GEN_PQ_C		4, 5, 6, 7

#define	GEN_PQR_STRIDE		4
#define	GEN_PQR_DEFINE() 	{}
#define	GEN_PQR_D		0, 1, 2, 3
#define	GEN_PQR_C		4, 5, 6, 7

#define	SYN_Q_DEFINE()		{}
#define	SYN_Q_D			0, 1, 2, 3
#define	SYN_Q_X			4, 5, 6, 7

#define	SYN_R_DEFINE()		{}
#define	SYN_R_D			0, 1, 2, 3
#define	SYN_R_X			4, 5, 6, 7

#define	SYN_PQ_DEFINE() 	{}
#define	SYN_PQ_D		0, 1, 2, 3
#define	SYN_PQ_X		4, 5, 6, 7

#define	REC_PQ_STRIDE		2
#define	REC_PQ_DEFINE() 	{}
#define	REC_PQ_X		0, 1
#define	REC_PQ_Y		2, 3
#define	REC_PQ_T		4, 5

#define	SYN_PR_DEFINE() 	{}
#define	SYN_PR_D		0, 1, 2, 3
#define	SYN_PR_X		4, 5, 6, 7

#define	REC_PR_STRIDE		2
#define	REC_PR_DEFINE() 	{}
#define	REC_PR_X		0, 1
#define	REC_PR_Y		2, 3
#define	REC_PR_T		4, 5

#define	SYN_QR_DEFINE() 	{}
#define	SYN_QR_D		0, 1, 2, 3
#define	SYN_QR_X		4, 5, 6, 7

#define	REC_QR_STRIDE		2
#define	REC_QR_DEFINE() 	{}
#define	REC_QR_X		0, 1
#define	REC_QR_Y		2, 3
#define	REC_QR_T		4, 5

#define	SYN_PQR_DEFINE() 	{}
#define	SYN_PQR_D		0, 1, 2, 3
#define	SYN_PQR_X		4, 5, 6, 7

#define	REC_PQR_STRIDE		2
#define	REC_PQR_DEFINE() 	{}
#define	REC_PQR_X		0, 1
#define	REC_PQR_Y		2, 3
#define	REC_PQR_Z		4, 5
#define	REC_PQR_XS		6, 7
#define	REC_PQR_YS		8, 9


#include <sys/vdev_raidz_impl.h>
#include "vdev_raidz_math_impl.h"

DEFINE_GEN_METHODS(avx2);
DEFINE_REC_METHODS(avx2);

static boolean_t
raidz_will_avx2_work(void)
{
	return (zfs_avx_available() && zfs_avx2_available());
}

const raidz_impl_ops_t vdev_raidz_avx2_impl = {
	.init = NULL,
	.fini = NULL,
	.gen = RAIDZ_GEN_METHODS(avx2),
	.rec = RAIDZ_REC_METHODS(avx2),
	.is_supported = &raidz_will_avx2_work,
	.name = "avx2"
};

#endif /* defined(__x86_64) && defined(HAVE_AVX2) */
Commit	Line	Data
ab9f4b0b GN	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright (C) 2016 Gvozden Nešković. All rights reserved.
	23	*/
ab9f4b0b GN	24	#include <sys/isa_defs.h>
	25
	26	#if defined(__x86_64) && defined(HAVE_AVX2)
	27
	28	#include <sys/types.h>
	29	#include <linux/simd_x86.h>
	30
	31	#define __asm __asm__ __volatile__
	32
	33	#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
	34	#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
	35
	36	#define VR0_(REG, ...) "ymm"#REG
	37	#define VR1_(_1, REG, ...) "ymm"#REG
	38	#define VR2_(_1, _2, REG, ...) "ymm"#REG
	39	#define VR3_(_1, _2, _3, REG, ...) "ymm"#REG
	40	#define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
	41	#define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
	42	#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
	43	#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
	44
	45	#define VR0(r...) VR0_(r)
	46	#define VR1(r...) VR1_(r)
	47	#define VR2(r...) VR2_(r, 1)
	48	#define VR3(r...) VR3_(r, 1, 2)
c9187d86 GN	49	#define VR4(r...) VR4_(r, 1, 2)
	50	#define VR5(r...) VR5_(r, 1, 2, 3)
	51	#define VR6(r...) VR6_(r, 1, 2, 3, 4)
	52	#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
ab9f4b0b GN	53
	54	#define R_01(REG1, REG2, ...) REG1, REG2
	55	#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
	56	#define R_23(REG...) _R_23(REG, 1, 2, 3)
	57
a20f43b5	58	#define ZFS_ASM_BUG() ASSERT(0)
ab9f4b0b GN	59
	60	extern const uint8_t gf_clmul_mod_lt[4*256][16];
	61
	62	#define ELEM_SIZE 32
	63
	64	typedef struct v {
	65	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
	66	} v_t;
	67
ab9f4b0b GN	68
	69	#define XOR_ACC(src, r...) \
	70	{ \
	71	switch (REG_CNT(r)) { \
	72	case 4: \
	73	__asm( \
	74	"vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
	75	"vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
	76	"vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \
	77	"vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \
	78	: : [SRC] "r" (src)); \
	79	break; \
	80	case 2: \
	81	__asm( \
	82	"vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
	83	"vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
	84	: : [SRC] "r" (src)); \
	85	break; \
	86	default: \
a20f43b5	87	ZFS_ASM_BUG(); \
ab9f4b0b GN	88	} \
	89	}
	90
	91	#define XOR(r...) \
	92	{ \
	93	switch (REG_CNT(r)) { \
	94	case 8: \
	95	__asm( \
	96	"vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \
	97	"vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \
	98	"vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \
	99	"vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \
	100	break; \
	101	case 4: \
	102	__asm( \
	103	"vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \
	104	"vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \
	105	break; \
	106	default: \
a20f43b5	107	ZFS_ASM_BUG(); \
ab9f4b0b GN	108	} \
	109	}
	110
65d71d42	111	#define ZERO(r...) XOR(r, r)
62a65a65	112
ab9f4b0b GN	113	#define COPY(r...) \
	114	{ \
	115	switch (REG_CNT(r)) { \
	116	case 8: \
	117	__asm( \
	118	"vmovdqa %" VR0(r) ", %" VR4(r) "\n" \
	119	"vmovdqa %" VR1(r) ", %" VR5(r) "\n" \
	120	"vmovdqa %" VR2(r) ", %" VR6(r) "\n" \
	121	"vmovdqa %" VR3(r) ", %" VR7(r)); \
	122	break; \
	123	case 4: \
	124	__asm( \
	125	"vmovdqa %" VR0(r) ", %" VR2(r) "\n" \
	126	"vmovdqa %" VR1(r) ", %" VR3(r)); \
	127	break; \
	128	default: \
a20f43b5	129	ZFS_ASM_BUG(); \
ab9f4b0b GN	130	} \
	131	}
	132
	133	#define LOAD(src, r...) \
	134	{ \
	135	switch (REG_CNT(r)) { \
	136	case 4: \
	137	__asm( \
	138	"vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
	139	"vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
	140	"vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \
	141	"vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \
	142	: : [SRC] "r" (src)); \
	143	break; \
	144	case 2: \
	145	__asm( \
	146	"vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
	147	"vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
	148	: : [SRC] "r" (src)); \
	149	break; \
	150	default: \
a20f43b5	151	ZFS_ASM_BUG(); \
ab9f4b0b GN	152	} \
	153	}
	154
	155	#define STORE(dst, r...) \
	156	{ \
	157	switch (REG_CNT(r)) { \
	158	case 4: \
	159	__asm( \
	160	"vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
	161	"vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
	162	"vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \
	163	"vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \
	164	: : [DST] "r" (dst)); \
	165	break; \
	166	case 2: \
	167	__asm( \
	168	"vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
	169	"vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
	170	: : [DST] "r" (dst)); \
	171	break; \
	172	default: \
a20f43b5	173	ZFS_ASM_BUG(); \
ab9f4b0b GN	174	} \
	175	}
	176
	177	#define FLUSH() \
	178	{ \
	179	__asm("vzeroupper"); \
	180	}
	181
	182	#define MUL2_SETUP() \
	183	{ \
	184	__asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \
	185	__asm("vpbroadcastq %xmm14, %ymm14"); \
	186	__asm("vpxor %ymm15, %ymm15 ,%ymm15"); \
	187	}
	188
	189	#define _MUL2(r...) \
	190	{ \
	191	switch (REG_CNT(r)) { \
	192	case 2: \
	193	__asm( \
	194	"vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \
	195	"vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \
	196	"vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \
	197	"vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \
	198	"vpand %ymm14, %ymm12, %ymm12\n" \
	199	"vpand %ymm14, %ymm13, %ymm13\n" \
	200	"vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \
	201	"vpxor %ymm13, %" VR1(r)", %" VR1(r)); \
	202	break; \
	203	default: \
a20f43b5	204	ZFS_ASM_BUG(); \
ab9f4b0b GN	205	} \
	206	}
	207
	208	#define MUL2(r...) \
	209	{ \
	210	switch (REG_CNT(r)) { \
	211	case 4: \
	212	_MUL2(R_01(r)); \
	213	_MUL2(R_23(r)); \
	214	break; \
	215	case 2: \
	216	_MUL2(r); \
	217	break; \
	218	default: \
a20f43b5	219	ZFS_ASM_BUG(); \
ab9f4b0b GN	220	} \
	221	}
	222
	223	#define MUL4(r...) \
	224	{ \
	225	MUL2(r); \
	226	MUL2(r); \
	227	}
	228
	229	#define _0f "ymm15"
	230	#define _as "ymm14"
	231	#define _bs "ymm13"
	232	#define _ltmod "ymm12"
	233	#define _ltmul "ymm11"
	234	#define _ta "ymm10"
	235	#define _tb "ymm15"
	236
	237	static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
	238
	239	#define _MULx2(c, r...) \
	240	{ \
	241	switch (REG_CNT(r)) { \
	242	case 2: \
	243	__asm( \
	244	"vpbroadcastb (%[mask]), %%" _0f "\n" \
	245	/* upper bits */ \
	246	"vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \
	247	"vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \
	248	\
	249	"vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \
	250	"vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \
	251	"vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \
	252	"vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \
	253	"vpand %%" _0f ", %%" _as ", %%" _as "\n" \
	254	"vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \
	255	\
	256	"vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \
	257	"vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \
	258	"vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \
	259	"vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \
	260	/* lower bits */ \
	261	"vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \
	262	"vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \
	263	\
	264	"vpxor %%" _ta ", %%" _as ", %%" _as "\n" \
	265	"vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \
	266	\
	267	"vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \
	268	"vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \
	269	"vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
	270	"vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
	271	\
	272	"vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \
	273	"vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \
	274	"vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \
	275	"vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \
	276	: : [mask] "r" (&_mul_mask), \
	277	[lt] "r" (gf_clmul_mod_lt[4*(c)])); \
	278	break; \
	279	default: \
a20f43b5	280	ZFS_ASM_BUG(); \
ab9f4b0b GN	281	} \
	282	}
	283
	284	#define MUL(c, r...) \
	285	{ \
	286	switch (REG_CNT(r)) { \
	287	case 4: \
	288	_MULx2(c, R_01(r)); \
	289	_MULx2(c, R_23(r)); \
	290	break; \
	291	case 2: \
	292	_MULx2(c, R_01(r)); \
	293	break; \
	294	default: \
a20f43b5	295	ZFS_ASM_BUG(); \
ab9f4b0b GN	296	} \
	297	}
	298
	299	#define raidz_math_begin() kfpu_begin()
	300	#define raidz_math_end() \
	301	{ \
	302	FLUSH(); \
	303	kfpu_end(); \
	304	}
	305
cbf484f8 GN	306
	307	#define SYN_STRIDE 4
	308
	309	#define ZERO_STRIDE 4
	310	#define ZERO_DEFINE() {}
	311	#define ZERO_D 0, 1, 2, 3
	312
	313	#define COPY_STRIDE 4
	314	#define COPY_DEFINE() {}
	315	#define COPY_D 0, 1, 2, 3
	316
	317	#define ADD_STRIDE 4
	318	#define ADD_DEFINE() {}
	319	#define ADD_D 0, 1, 2, 3
	320
	321	#define MUL_STRIDE 4
	322	#define MUL_DEFINE() {}
	323	#define MUL_D 0, 1, 2, 3
	324
ab9f4b0b	325	#define GEN_P_STRIDE 4
cbf484f8	326	#define GEN_P_DEFINE() {}
ab9f4b0b GN	327	#define GEN_P_P 0, 1, 2, 3
ab9f4b0b GN	328
ab9f4b0b	329	#define GEN_PQ_STRIDE 4
cbf484f8	330	#define GEN_PQ_DEFINE() {}
ab9f4b0b	331	#define GEN_PQ_D 0, 1, 2, 3
cbf484f8	332	#define GEN_PQ_C 4, 5, 6, 7
ab9f4b0b	333
cbf484f8	334	#define GEN_PQR_STRIDE 4
ab9f4b0b	335	#define GEN_PQR_DEFINE() {}
cbf484f8 GN	336	#define GEN_PQR_D 0, 1, 2, 3
cbf484f8 GN	337	#define GEN_PQR_C 4, 5, 6, 7
ab9f4b0b	338
cbf484f8 GN	339	#define SYN_Q_DEFINE() {}
	340	#define SYN_Q_D 0, 1, 2, 3
	341	#define SYN_Q_X 4, 5, 6, 7
ab9f4b0b	342
cbf484f8 GN	343	#define SYN_R_DEFINE() {}
	344	#define SYN_R_D 0, 1, 2, 3
	345	#define SYN_R_X 4, 5, 6, 7
ab9f4b0b	346
cbf484f8 GN	347	#define SYN_PQ_DEFINE() {}
	348	#define SYN_PQ_D 0, 1, 2, 3
	349	#define SYN_PQ_X 4, 5, 6, 7
ab9f4b0b	350
ab9f4b0b	351	#define REC_PQ_STRIDE 2
cbf484f8	352	#define REC_PQ_DEFINE() {}
ab9f4b0b GN	353	#define REC_PQ_X 0, 1
ab9f4b0b GN	354	#define REC_PQ_Y 2, 3
cbf484f8 GN	355	#define REC_PQ_T 4, 5
	356
	357	#define SYN_PR_DEFINE() {}
	358	#define SYN_PR_D 0, 1, 2, 3
	359	#define SYN_PR_X 4, 5, 6, 7
ab9f4b0b	360
ab9f4b0b	361	#define REC_PR_STRIDE 2
cbf484f8	362	#define REC_PR_DEFINE() {}
ab9f4b0b GN	363	#define REC_PR_X 0, 1
ab9f4b0b GN	364	#define REC_PR_Y 2, 3
cbf484f8 GN	365	#define REC_PR_T 4, 5
	366
	367	#define SYN_QR_DEFINE() {}
	368	#define SYN_QR_D 0, 1, 2, 3
	369	#define SYN_QR_X 4, 5, 6, 7
ab9f4b0b	370
ab9f4b0b	371	#define REC_QR_STRIDE 2
cbf484f8	372	#define REC_QR_DEFINE() {}
ab9f4b0b GN	373	#define REC_QR_X 0, 1
ab9f4b0b GN	374	#define REC_QR_Y 2, 3
cbf484f8 GN	375	#define REC_QR_T 4, 5
	376
	377	#define SYN_PQR_DEFINE() {}
	378	#define SYN_PQR_D 0, 1, 2, 3
	379	#define SYN_PQR_X 4, 5, 6, 7
ab9f4b0b	380
ab9f4b0b	381	#define REC_PQR_STRIDE 2
cbf484f8	382	#define REC_PQR_DEFINE() {}
ab9f4b0b GN	383	#define REC_PQR_X 0, 1
	384	#define REC_PQR_Y 2, 3
	385	#define REC_PQR_Z 4, 5
ab9f4b0b GN	386	#define REC_PQR_XS 6, 7
	387	#define REC_PQR_YS 8, 9
	388
	389
	390	#include <sys/vdev_raidz_impl.h>
	391	#include "vdev_raidz_math_impl.h"
	392
	393	DEFINE_GEN_METHODS(avx2);
	394	DEFINE_REC_METHODS(avx2);
	395
	396	static boolean_t
	397	raidz_will_avx2_work(void)
	398	{
	399	return (zfs_avx_available() && zfs_avx2_available());
	400	}
	401
	402	const raidz_impl_ops_t vdev_raidz_avx2_impl = {
	403	.init = NULL,
	404	.fini = NULL,
	405	.gen = RAIDZ_GEN_METHODS(avx2),
	406	.rec = RAIDZ_REC_METHODS(avx2),
	407	.is_supported = &raidz_will_avx2_work,
	408	.name = "avx2"
	409	};
	410
	411	#endif /* defined(__x86_64) && defined(HAVE_AVX2) */