[mirror_zfs-debian.git] / module / zfs / vdev_raidz_math_aarch64_neonx2.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
 */

#include <sys/isa_defs.h>

#if defined(__aarch64__)

#include "vdev_raidz_math_aarch64_neon_common.h"

#define	SYN_STRIDE		4

#define	ZERO_STRIDE		8
#define	ZERO_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	ZERO_D			0, 1, 2, 3, 4, 5, 6, 7

#define	COPY_STRIDE		8
#define	COPY_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	COPY_D			0, 1, 2, 3, 4, 5, 6, 7

#define	ADD_STRIDE		8
#define	ADD_DEFINE()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()
#define	ADD_D			0, 1, 2, 3, 4, 5, 6, 7

#define	MUL_STRIDE		4
#define	MUL_DEFINE()	\
	GEN_X_DEFINE_0_3()  \
	GEN_X_DEFINE_33_36()
#define	MUL_D			0, 1, 2, 3

#define	GEN_P_DEFINE() \
	GEN_X_DEFINE_0_3() \
	GEN_X_DEFINE_33_36()
#define	GEN_P_STRIDE		4
#define	GEN_P_P			0, 1, 2, 3

#define	GEN_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	GEN_PQ_STRIDE		4
#define	GEN_PQ_D		0, 1, 2, 3
#define	GEN_PQ_C		4, 5, 6, 7

#define	GEN_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	GEN_PQR_STRIDE		4
#define	GEN_PQR_D		0, 1, 2, 3
#define	GEN_PQR_C		4, 5, 6, 7

#define	SYN_Q_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_Q_STRIDE		4
#define	SYN_Q_D			0, 1, 2, 3
#define	SYN_Q_X			4, 5, 6, 7

#define	SYN_R_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_R_STRIDE		4
#define	SYN_R_D			0, 1, 2, 3
#define	SYN_R_X			4, 5, 6, 7

#define	SYN_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PQ_STRIDE		4
#define	SYN_PQ_D		0, 1, 2, 3
#define	SYN_PQ_X		4, 5, 6, 7

#define	REC_PQ_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_PQ_STRIDE		4
#define	REC_PQ_X		0, 1, 2, 3
#define	REC_PQ_Y		4, 5, 6, 7
#define	REC_PQ_T		8, 9, 22, 23

#define	SYN_PR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PR_STRIDE		4
#define	SYN_PR_D		0, 1, 2, 3
#define	SYN_PR_X		4, 5, 6, 7

#define	REC_PR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_PR_STRIDE		4
#define	REC_PR_X		0, 1, 2, 3
#define	REC_PR_Y		4, 5, 6, 7
#define	REC_PR_T		8, 9, 22, 23

#define	SYN_QR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_QR_STRIDE		4
#define	SYN_QR_D		0, 1, 2, 3
#define	SYN_QR_X		4, 5, 6, 7

#define	REC_QR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_33_36()
#define	REC_QR_STRIDE		4
#define	REC_QR_X		0, 1, 2, 3
#define	REC_QR_Y		4, 5, 6, 7
#define	REC_QR_T		8, 9, 22, 23

#define	SYN_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_33_36()
#define	SYN_PQR_STRIDE		 4
#define	SYN_PQR_D		 0, 1, 2, 3
#define	SYN_PQR_X		 4, 5, 6, 7

#define	REC_PQR_DEFINE() \
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_31()	\
	GEN_X_DEFINE_32()	\
	GEN_X_DEFINE_33_36()
#define	REC_PQR_STRIDE		2
#define	REC_PQR_X		0, 1
#define	REC_PQR_Y		2, 3
#define	REC_PQR_Z		4, 5
#define	REC_PQR_XS		6, 7
#define	REC_PQR_YS		8, 9

#include <sys/vdev_raidz_impl.h>
#include "vdev_raidz_math_impl.h"

DEFINE_GEN_METHODS(aarch64_neonx2);
/*
 * If compiled with -O0, gcc doesn't do any stack frame coalescing
 * and -Wframe-larger-than=1024 is triggered in debug mode.
 */
#pragma GCC diagnostic ignored "-Wframe-larger-than="
DEFINE_REC_METHODS(aarch64_neonx2);
#pragma GCC diagnostic pop

static boolean_t
raidz_will_aarch64_neonx2_work(void)
{
	return (B_TRUE); // __arch64__ requires NEON
}

const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl = {
	.init = NULL,
	.fini = NULL,
	.gen = RAIDZ_GEN_METHODS(aarch64_neonx2),
	.rec = RAIDZ_REC_METHODS(aarch64_neonx2),
	.is_supported = &raidz_will_aarch64_neonx2_work,
	.name = "aarch64_neonx2"
};

#endif /* defined(__aarch64__) */
Commit	Line	Data
cae5b340 AX	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright (C) 2016 Romain Dolbeau. All rights reserved.
	23	*/
	24
	25	#include <sys/isa_defs.h>
	26
	27	#if defined(__aarch64__)
	28
	29	#include "vdev_raidz_math_aarch64_neon_common.h"
	30
	31	#define SYN_STRIDE 4
	32
	33	#define ZERO_STRIDE 8
	34	#define ZERO_DEFINE() \
	35	GEN_X_DEFINE_0_3() \
	36	GEN_X_DEFINE_4_5() \
	37	GEN_X_DEFINE_6_7()
	38	#define ZERO_D 0, 1, 2, 3, 4, 5, 6, 7
	39
	40	#define COPY_STRIDE 8
	41	#define COPY_DEFINE() \
	42	GEN_X_DEFINE_0_3() \
	43	GEN_X_DEFINE_4_5() \
	44	GEN_X_DEFINE_6_7()
	45	#define COPY_D 0, 1, 2, 3, 4, 5, 6, 7
	46
	47	#define ADD_STRIDE 8
	48	#define ADD_DEFINE() \
	49	GEN_X_DEFINE_0_3() \
	50	GEN_X_DEFINE_4_5() \
	51	GEN_X_DEFINE_6_7()
	52	#define ADD_D 0, 1, 2, 3, 4, 5, 6, 7
	53
	54	#define MUL_STRIDE 4
	55	#define MUL_DEFINE() \
	56	GEN_X_DEFINE_0_3() \
	57	GEN_X_DEFINE_33_36()
	58	#define MUL_D 0, 1, 2, 3
	59
	60	#define GEN_P_DEFINE() \
	61	GEN_X_DEFINE_0_3() \
	62	GEN_X_DEFINE_33_36()
	63	#define GEN_P_STRIDE 4
	64	#define GEN_P_P 0, 1, 2, 3
65
66	#define GEN_PQ_DEFINE() \
67	GEN_X_DEFINE_0_3() \
68	GEN_X_DEFINE_4_5() \
69	GEN_X_DEFINE_6_7() \
70	GEN_X_DEFINE_16() \
71	GEN_X_DEFINE_17() \
72	GEN_X_DEFINE_33_36()
73	#define GEN_PQ_STRIDE 4
74	#define GEN_PQ_D 0, 1, 2, 3
75	#define GEN_PQ_C 4, 5, 6, 7
76
77	#define GEN_PQR_DEFINE() \
78	GEN_X_DEFINE_0_3() \
79	GEN_X_DEFINE_4_5() \
80	GEN_X_DEFINE_6_7() \
81	GEN_X_DEFINE_16() \
82	GEN_X_DEFINE_17() \
83	GEN_X_DEFINE_33_36()
84	#define GEN_PQR_STRIDE 4
85	#define GEN_PQR_D 0, 1, 2, 3
86	#define GEN_PQR_C 4, 5, 6, 7
87
88	#define SYN_Q_DEFINE() \
89	GEN_X_DEFINE_0_3() \
90	GEN_X_DEFINE_4_5() \
91	GEN_X_DEFINE_6_7() \
92	GEN_X_DEFINE_16() \
93	GEN_X_DEFINE_17() \
94	GEN_X_DEFINE_33_36()
95	#define SYN_Q_STRIDE 4
96	#define SYN_Q_D 0, 1, 2, 3
97	#define SYN_Q_X 4, 5, 6, 7
98
99	#define SYN_R_DEFINE() \
100	GEN_X_DEFINE_0_3() \
101	GEN_X_DEFINE_4_5() \
102	GEN_X_DEFINE_6_7() \
103	GEN_X_DEFINE_16() \
104	GEN_X_DEFINE_17() \
105	GEN_X_DEFINE_33_36()
106	#define SYN_R_STRIDE 4
107	#define SYN_R_D 0, 1, 2, 3
108	#define SYN_R_X 4, 5, 6, 7
109
110	#define SYN_PQ_DEFINE() \
111	GEN_X_DEFINE_0_3() \
112	GEN_X_DEFINE_4_5() \
113	GEN_X_DEFINE_6_7() \
114	GEN_X_DEFINE_16() \
115	GEN_X_DEFINE_17() \
116	GEN_X_DEFINE_33_36()
117	#define SYN_PQ_STRIDE 4
118	#define SYN_PQ_D 0, 1, 2, 3
119	#define SYN_PQ_X 4, 5, 6, 7
120
121	#define REC_PQ_DEFINE() \
122	GEN_X_DEFINE_0_3() \
123	GEN_X_DEFINE_4_5() \
124	GEN_X_DEFINE_6_7() \
125	GEN_X_DEFINE_8_9() \
126	GEN_X_DEFINE_22_23() \
127	GEN_X_DEFINE_33_36()
128	#define REC_PQ_STRIDE 4
129	#define REC_PQ_X 0, 1, 2, 3
130	#define REC_PQ_Y 4, 5, 6, 7
131	#define REC_PQ_T 8, 9, 22, 23
132
133	#define SYN_PR_DEFINE() \
134	GEN_X_DEFINE_0_3() \
135	GEN_X_DEFINE_4_5() \
136	GEN_X_DEFINE_6_7() \
137	GEN_X_DEFINE_16() \
138	GEN_X_DEFINE_17() \
139	GEN_X_DEFINE_33_36()
140	#define SYN_PR_STRIDE 4
141	#define SYN_PR_D 0, 1, 2, 3
142	#define SYN_PR_X 4, 5, 6, 7
143
144	#define REC_PR_DEFINE() \
145	GEN_X_DEFINE_0_3() \
146	GEN_X_DEFINE_4_5() \
147	GEN_X_DEFINE_6_7() \
148	GEN_X_DEFINE_8_9() \
149	GEN_X_DEFINE_22_23() \
150	GEN_X_DEFINE_33_36()
151	#define REC_PR_STRIDE 4
152	#define REC_PR_X 0, 1, 2, 3
153	#define REC_PR_Y 4, 5, 6, 7
154	#define REC_PR_T 8, 9, 22, 23
155
156	#define SYN_QR_DEFINE() \
157	GEN_X_DEFINE_0_3() \
158	GEN_X_DEFINE_4_5() \
159	GEN_X_DEFINE_6_7() \
160	GEN_X_DEFINE_16() \
161	GEN_X_DEFINE_17() \
162	GEN_X_DEFINE_33_36()
163	#define SYN_QR_STRIDE 4
164	#define SYN_QR_D 0, 1, 2, 3
165	#define SYN_QR_X 4, 5, 6, 7
166
167	#define REC_QR_DEFINE() \
168	GEN_X_DEFINE_0_3() \
169	GEN_X_DEFINE_4_5() \
170	GEN_X_DEFINE_6_7() \
171	GEN_X_DEFINE_8_9() \
172	GEN_X_DEFINE_22_23() \
173	GEN_X_DEFINE_33_36()
174	#define REC_QR_STRIDE 4
175	#define REC_QR_X 0, 1, 2, 3
176	#define REC_QR_Y 4, 5, 6, 7
177	#define REC_QR_T 8, 9, 22, 23
178
179	#define SYN_PQR_DEFINE() \
180	GEN_X_DEFINE_0_3() \
181	GEN_X_DEFINE_4_5() \
182	GEN_X_DEFINE_6_7() \
183	GEN_X_DEFINE_16() \
184	GEN_X_DEFINE_17() \
185	GEN_X_DEFINE_33_36()
186	#define SYN_PQR_STRIDE 4
187	#define SYN_PQR_D 0, 1, 2, 3
188	#define SYN_PQR_X 4, 5, 6, 7
189
190	#define REC_PQR_DEFINE() \
191	GEN_X_DEFINE_0_3() \
192	GEN_X_DEFINE_4_5() \
193	GEN_X_DEFINE_6_7() \
194	GEN_X_DEFINE_8_9() \
195	GEN_X_DEFINE_31() \
196	GEN_X_DEFINE_32() \
197	GEN_X_DEFINE_33_36()
198	#define REC_PQR_STRIDE 2
199	#define REC_PQR_X 0, 1
200	#define REC_PQR_Y 2, 3
201	#define REC_PQR_Z 4, 5
202	#define REC_PQR_XS 6, 7
203	#define REC_PQR_YS 8, 9
204
205	#include <sys/vdev_raidz_impl.h>
206	#include "vdev_raidz_math_impl.h"
207
208	DEFINE_GEN_METHODS(aarch64_neonx2);
209	/*
210	* If compiled with -O0, gcc doesn't do any stack frame coalescing
211	* and -Wframe-larger-than=1024 is triggered in debug mode.
212	*/
213	#pragma GCC diagnostic ignored "-Wframe-larger-than="
214	DEFINE_REC_METHODS(aarch64_neonx2);
215	#pragma GCC diagnostic pop
216
217	static boolean_t
218	raidz_will_aarch64_neonx2_work(void)
219	{
220	return (B_TRUE); // __arch64__ requires NEON
221	}
222
223	const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl = {
224	.init = NULL,
225	.fini = NULL,
226	.gen = RAIDZ_GEN_METHODS(aarch64_neonx2),
227	.rec = RAIDZ_REC_METHODS(aarch64_neonx2),
228	.is_supported = &raidz_will_aarch64_neonx2_work,
229	.name = "aarch64_neonx2"
230	};
231
232	#endif /* defined(__aarch64__) */