[mirror_zfs.git] / module / zfs / vdev_raidz_math_aarch64_neonx2.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
 */

#include <sys/isa_defs.h>

#if defined(__aarch64__)

#include "vdev_raidz_math_aarch64_neon_common.h"

#define	SYN_STRIDE		4

#define	ZERO_STRIDE		8
#define	ZERO_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	ZERO_D			0, 1, 2, 3, 4, 5, 6, 7

#define	COPY_STRIDE		8
#define	COPY_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	COPY_D			0, 1, 2, 3, 4, 5, 6, 7

#define	ADD_STRIDE		8
#define	ADD_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	ADD_D			0, 1, 2, 3, 4, 5, 6, 7

#define	MUL_STRIDE		4
#define	MUL_DEFINE()	\
	GEN_X_DEFINE_0_3()  \
	GEN_X_DEFINE_33_36()
#define	MUL_D			0, 1, 2, 3

#define	GEN_P_DEFINE() \
	GEN_X_DEFINE_0_3() \
	GEN_X_DEFINE_33_36()
#define	GEN_P_STRIDE		4
#define	GEN_P_P			0, 1, 2, 3

#define	GEN_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	GEN_PQ_STRIDE		4
#define	GEN_PQ_D		0, 1, 2, 3
#define	GEN_PQ_C		4, 5, 6, 7

#define	GEN_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	GEN_PQR_STRIDE		4
#define	GEN_PQR_D		0, 1, 2, 3
#define	GEN_PQR_C		4, 5, 6, 7

#define	SYN_Q_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_Q_STRIDE		4
#define	SYN_Q_D			0, 1, 2, 3
#define	SYN_Q_X			4, 5, 6, 7

#define	SYN_R_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_R_STRIDE		4
#define	SYN_R_D			0, 1, 2, 3
#define	SYN_R_X			4, 5, 6, 7

#define	SYN_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PQ_STRIDE		4
#define	SYN_PQ_D		0, 1, 2, 3
#define	SYN_PQ_X		4, 5, 6, 7

#define	REC_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_PQ_STRIDE		4
#define	REC_PQ_X		0, 1, 2, 3
#define	REC_PQ_Y		4, 5, 6, 7
#define	REC_PQ_T		8, 9, 22, 23

#define	SYN_PR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PR_STRIDE		4
#define	SYN_PR_D		0, 1, 2, 3
#define	SYN_PR_X		4, 5, 6, 7

#define	REC_PR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_PR_STRIDE		4
#define	REC_PR_X		0, 1, 2, 3
#define	REC_PR_Y		4, 5, 6, 7
#define	REC_PR_T		8, 9, 22, 23

#define	SYN_QR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_QR_STRIDE		4
#define	SYN_QR_D		0, 1, 2, 3
#define	SYN_QR_X		4, 5, 6, 7

#define	REC_QR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_QR_STRIDE		4
#define	REC_QR_X		0, 1, 2, 3
#define	REC_QR_Y		4, 5, 6, 7
#define	REC_QR_T		8, 9, 22, 23

#define	SYN_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PQR_STRIDE		 4
#define	SYN_PQR_D		 0, 1, 2, 3
#define	SYN_PQR_X		 4, 5, 6, 7

#define	REC_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_31()	\
	GEN_X_DEFINE_32()	\
	GEN_X_DEFINE_33_36()
#define	REC_PQR_STRIDE		2
#define	REC_PQR_X		0, 1
#define	REC_PQR_Y		2, 3
#define	REC_PQR_Z		4, 5
#define	REC_PQR_XS		6, 7
#define	REC_PQR_YS		8, 9

#include <sys/vdev_raidz_impl.h>
#include "vdev_raidz_math_impl.h"

DEFINE_GEN_METHODS(aarch64_neonx2);
/*
 * If compiled with -O0, gcc doesn't do any stack frame coalescing
 * and -Wframe-larger-than=1024 is triggered in debug mode.
 */
#pragma GCC diagnostic ignored "-Wframe-larger-than="
DEFINE_REC_METHODS(aarch64_neonx2);
#pragma GCC diagnostic pop

static boolean_t
raidz_will_aarch64_neonx2_work(void)
{
	return (B_TRUE); // __arch64__ requires NEON
}

const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl = {
	.init = NULL,
	.fini = NULL,
	.gen = RAIDZ_GEN_METHODS(aarch64_neonx2),
	.rec = RAIDZ_REC_METHODS(aarch64_neonx2),
	.is_supported = &raidz_will_aarch64_neonx2_work,
	.name = "aarch64_neonx2"
};

#endif /* defined(__aarch64__) */
Commit	Line	Data
62a65a65 RD	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright (C) 2016 Romain Dolbeau. All rights reserved.
	23	*/
	24
	25	#include <sys/isa_defs.h>
	26
88cc2352	27	#if defined(__aarch64__)
62a65a65 RD	28
	29	#include "vdev_raidz_math_aarch64_neon_common.h"
	30
88cc2352 RD	31	#define SYN_STRIDE 4
	32
	33	#define ZERO_STRIDE 8
	34	#define ZERO_DEFINE() \
62a65a65 RD	35	GEN_X_DEFINE_0_3() \
	36	GEN_X_DEFINE_4_5() \
	37	GEN_X_DEFINE_6_7()
88cc2352	38	#define ZERO_D 0, 1, 2, 3, 4, 5, 6, 7
62a65a65	39
88cc2352 RD	40	#define COPY_STRIDE 8
	41	#define COPY_DEFINE() \
	42	GEN_X_DEFINE_0_3() \
	43	GEN_X_DEFINE_4_5() \
	44	GEN_X_DEFINE_6_7()
	45	#define COPY_D 0, 1, 2, 3, 4, 5, 6, 7
	46
	47	#define ADD_STRIDE 8
	48	#define ADD_DEFINE() \
	49	GEN_X_DEFINE_0_3() \
	50	GEN_X_DEFINE_4_5() \
	51	GEN_X_DEFINE_6_7()
	52	#define ADD_D 0, 1, 2, 3, 4, 5, 6, 7
	53
	54	#define MUL_STRIDE 4
	55	#define MUL_DEFINE() \
	56	GEN_X_DEFINE_0_3() \
	57	GEN_X_DEFINE_33_36()
	58	#define MUL_D 0, 1, 2, 3
	59
	60	#define GEN_P_DEFINE() \
	61	GEN_X_DEFINE_0_3() \
	62	GEN_X_DEFINE_33_36()
	63	#define GEN_P_STRIDE 4
	64	#define GEN_P_P 0, 1, 2, 3
	65
	66	#define GEN_PQ_DEFINE() \
62a65a65 RD	67	GEN_X_DEFINE_0_3() \
	68	GEN_X_DEFINE_4_5() \
	69	GEN_X_DEFINE_6_7() \
62a65a65 RD	70	GEN_X_DEFINE_16() \
	71	GEN_X_DEFINE_17() \
	72	GEN_X_DEFINE_33_36()
	73	#define GEN_PQ_STRIDE 4
	74	#define GEN_PQ_D 0, 1, 2, 3
88cc2352	75	#define GEN_PQ_C 4, 5, 6, 7
62a65a65 RD	76
	77	#define GEN_PQR_DEFINE() \
	78	GEN_X_DEFINE_0_3() \
	79	GEN_X_DEFINE_4_5() \
	80	GEN_X_DEFINE_6_7() \
62a65a65 RD	81	GEN_X_DEFINE_16() \
	82	GEN_X_DEFINE_17() \
	83	GEN_X_DEFINE_33_36()
	84	#define GEN_PQR_STRIDE 4
	85	#define GEN_PQR_D 0, 1, 2, 3
88cc2352	86	#define GEN_PQR_C 4, 5, 6, 7
62a65a65	87
88cc2352	88	#define SYN_Q_DEFINE() \
62a65a65	89	GEN_X_DEFINE_0_3() \
88cc2352 RD	90	GEN_X_DEFINE_4_5() \
	91	GEN_X_DEFINE_6_7() \
	92	GEN_X_DEFINE_16() \
	93	GEN_X_DEFINE_17() \
62a65a65	94	GEN_X_DEFINE_33_36()
88cc2352 RD	95	#define SYN_Q_STRIDE 4
	96	#define SYN_Q_D 0, 1, 2, 3
	97	#define SYN_Q_X 4, 5, 6, 7
62a65a65	98
88cc2352	99	#define SYN_R_DEFINE() \
62a65a65	100	GEN_X_DEFINE_0_3() \
88cc2352 RD	101	GEN_X_DEFINE_4_5() \
88cc2352 RD	102	GEN_X_DEFINE_6_7() \
62a65a65 RD	103	GEN_X_DEFINE_16() \
	104	GEN_X_DEFINE_17() \
	105	GEN_X_DEFINE_33_36()
88cc2352 RD	106	#define SYN_R_STRIDE 4
	107	#define SYN_R_D 0, 1, 2, 3
	108	#define SYN_R_X 4, 5, 6, 7
62a65a65	109
88cc2352	110	#define SYN_PQ_DEFINE() \
62a65a65	111	GEN_X_DEFINE_0_3() \
88cc2352 RD	112	GEN_X_DEFINE_4_5() \
88cc2352 RD	113	GEN_X_DEFINE_6_7() \
62a65a65 RD	114	GEN_X_DEFINE_16() \
	115	GEN_X_DEFINE_17() \
	116	GEN_X_DEFINE_33_36()
88cc2352 RD	117	#define SYN_PQ_STRIDE 4
	118	#define SYN_PQ_D 0, 1, 2, 3
	119	#define SYN_PQ_X 4, 5, 6, 7
62a65a65 RD	120
	121	#define REC_PQ_DEFINE() \
	122	GEN_X_DEFINE_0_3() \
	123	GEN_X_DEFINE_4_5() \
	124	GEN_X_DEFINE_6_7() \
	125	GEN_X_DEFINE_8_9() \
62a65a65 RD	126	GEN_X_DEFINE_22_23() \
	127	GEN_X_DEFINE_33_36()
	128	#define REC_PQ_STRIDE 4
	129	#define REC_PQ_X 0, 1, 2, 3
	130	#define REC_PQ_Y 4, 5, 6, 7
88cc2352	131	#define REC_PQ_T 8, 9, 22, 23
62a65a65	132
88cc2352 RD	133	#define SYN_PR_DEFINE() \
	134	GEN_X_DEFINE_0_3() \
	135	GEN_X_DEFINE_4_5() \
	136	GEN_X_DEFINE_6_7() \
	137	GEN_X_DEFINE_16() \
	138	GEN_X_DEFINE_17() \
	139	GEN_X_DEFINE_33_36()
	140	#define SYN_PR_STRIDE 4
	141	#define SYN_PR_D 0, 1, 2, 3
	142	#define SYN_PR_X 4, 5, 6, 7
	143
	144	#define REC_PR_DEFINE() \
	145	GEN_X_DEFINE_0_3() \
	146	GEN_X_DEFINE_4_5() \
	147	GEN_X_DEFINE_6_7() \
	148	GEN_X_DEFINE_8_9() \
	149	GEN_X_DEFINE_22_23() \
	150	GEN_X_DEFINE_33_36()
62a65a65 RD	151	#define REC_PR_STRIDE 4
	152	#define REC_PR_X 0, 1, 2, 3
	153	#define REC_PR_Y 4, 5, 6, 7
88cc2352	154	#define REC_PR_T 8, 9, 22, 23
62a65a65	155
88cc2352 RD	156	#define SYN_QR_DEFINE() \
	157	GEN_X_DEFINE_0_3() \
	158	GEN_X_DEFINE_4_5() \
	159	GEN_X_DEFINE_6_7() \
	160	GEN_X_DEFINE_16() \
	161	GEN_X_DEFINE_17() \
	162	GEN_X_DEFINE_33_36()
	163	#define SYN_QR_STRIDE 4
	164	#define SYN_QR_D 0, 1, 2, 3
	165	#define SYN_QR_X 4, 5, 6, 7
	166
	167	#define REC_QR_DEFINE() \
	168	GEN_X_DEFINE_0_3() \
	169	GEN_X_DEFINE_4_5() \
	170	GEN_X_DEFINE_6_7() \
	171	GEN_X_DEFINE_8_9() \
	172	GEN_X_DEFINE_22_23() \
	173	GEN_X_DEFINE_33_36()
62a65a65 RD	174	#define REC_QR_STRIDE 4
	175	#define REC_QR_X 0, 1, 2, 3
	176	#define REC_QR_Y 4, 5, 6, 7
88cc2352	177	#define REC_QR_T 8, 9, 22, 23
62a65a65	178
88cc2352	179	#define SYN_PQR_DEFINE() \
62a65a65 RD	180	GEN_X_DEFINE_0_3() \
	181	GEN_X_DEFINE_4_5() \
	182	GEN_X_DEFINE_6_7() \
62a65a65 RD	183	GEN_X_DEFINE_16() \
62a65a65 RD	184	GEN_X_DEFINE_17() \
62a65a65	185	GEN_X_DEFINE_33_36()
88cc2352 RD	186	#define SYN_PQR_STRIDE 4
	187	#define SYN_PQR_D 0, 1, 2, 3
	188	#define SYN_PQR_X 4, 5, 6, 7
62a65a65	189
88cc2352 RD	190	#define REC_PQR_DEFINE() \
	191	GEN_X_DEFINE_0_3() \
	192	GEN_X_DEFINE_4_5() \
	193	GEN_X_DEFINE_6_7() \
	194	GEN_X_DEFINE_8_9() \
	195	GEN_X_DEFINE_31() \
	196	GEN_X_DEFINE_32() \
	197	GEN_X_DEFINE_33_36()
	198	#define REC_PQR_STRIDE 2
	199	#define REC_PQR_X 0, 1
	200	#define REC_PQR_Y 2, 3
	201	#define REC_PQR_Z 4, 5
	202	#define REC_PQR_XS 6, 7
	203	#define REC_PQR_YS 8, 9
62a65a65 RD	204
	205	#include <sys/vdev_raidz_impl.h>
	206	#include "vdev_raidz_math_impl.h"
	207
	208	DEFINE_GEN_METHODS(aarch64_neonx2);
	209	/*
	210	* If compiled with -O0, gcc doesn't do any stack frame coalescing
	211	* and -Wframe-larger-than=1024 is triggered in debug mode.
	212	*/
	213	#pragma GCC diagnostic ignored "-Wframe-larger-than="
	214	DEFINE_REC_METHODS(aarch64_neonx2);
	215	#pragma GCC diagnostic pop
	216
	217	static boolean_t
	218	raidz_will_aarch64_neonx2_work(void)
	219	{
	220	return (B_TRUE); // __arch64__ requires NEON
	221	}
	222
	223	const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl = {
	224	.init = NULL,
	225	.fini = NULL,
	226	.gen = RAIDZ_GEN_METHODS(aarch64_neonx2),
	227	.rec = RAIDZ_REC_METHODS(aarch64_neonx2),
	228	.is_supported = &raidz_will_aarch64_neonx2_work,
	229	.name = "aarch64_neonx2"
	230	};
	231
	232	#endif /* defined(__aarch64__) */