]>
Commit | Line | Data |
---|---|---|
7f547f85 RD |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | /* | |
22 | * Copyright (C) 2016 Romain Dolbeau. All rights reserved. | |
65d71d42 | 23 | * Copyright (C) 2016 Gvozden Nešković. All rights reserved. |
7f547f85 RD |
24 | */ |
25 | ||
26 | #include <sys/isa_defs.h> | |
27 | ||
65d71d42 | 28 | #if defined(__x86_64) && defined(HAVE_AVX512F) |
7f547f85 RD |
29 | |
30 | #include <sys/types.h> | |
31 | #include <linux/simd_x86.h> | |
32 | ||
33 | #define __asm __asm__ __volatile__ | |
34 | ||
35 | #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N | |
36 | #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) | |
37 | ||
38 | #define VR0_(REG, ...) "zmm"#REG | |
39 | #define VR1_(_1, REG, ...) "zmm"#REG | |
40 | #define VR2_(_1, _2, REG, ...) "zmm"#REG | |
41 | #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG | |
42 | #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG | |
43 | #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG | |
44 | #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG | |
45 | #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG | |
46 | ||
47 | #define VR0(r...) VR0_(r) | |
48 | #define VR1(r...) VR1_(r) | |
49 | #define VR2(r...) VR2_(r, 1) | |
50 | #define VR3(r...) VR3_(r, 1, 2) | |
51 | #define VR4(r...) VR4_(r, 1, 2) | |
52 | #define VR5(r...) VR5_(r, 1, 2, 3) | |
53 | #define VR6(r...) VR6_(r, 1, 2, 3, 4) | |
54 | #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) | |
55 | ||
56 | #define VRy0_(REG, ...) "ymm"#REG | |
57 | #define VRy1_(_1, REG, ...) "ymm"#REG | |
58 | #define VRy2_(_1, _2, REG, ...) "ymm"#REG | |
59 | #define VRy3_(_1, _2, _3, REG, ...) "ymm"#REG | |
60 | #define VRy4_(_1, _2, _3, _4, REG, ...) "ymm"#REG | |
61 | #define VRy5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG | |
62 | #define VRy6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG | |
63 | #define VRy7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG | |
64 | ||
65 | #define VRy0(r...) VRy0_(r) | |
66 | #define VRy1(r...) VRy1_(r) | |
67 | #define VRy2(r...) VRy2_(r, 1) | |
68 | #define VRy3(r...) VRy3_(r, 1, 2) | |
69 | #define VRy4(r...) VRy4_(r, 1, 2) | |
70 | #define VRy5(r...) VRy5_(r, 1, 2, 3) | |
71 | #define VRy6(r...) VRy6_(r, 1, 2, 3, 4) | |
72 | #define VRy7(r...) VRy7_(r, 1, 2, 3, 4, 5) | |
73 | ||
74 | #define R_01(REG1, REG2, ...) REG1, REG2 | |
75 | #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 | |
76 | #define R_23(REG...) _R_23(REG, 1, 2, 3) | |
77 | ||
7f547f85 RD |
78 | #define ELEM_SIZE 64 |
79 | ||
80 | typedef struct v { | |
81 | uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); | |
82 | } v_t; | |
83 | ||
7f547f85 RD |
84 | |
85 | #define XOR_ACC(src, r...) \ | |
86 | { \ | |
87 | switch (REG_CNT(r)) { \ | |
88 | case 4: \ | |
89 | __asm( \ | |
90 | "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ | |
91 | "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ | |
92 | "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ | |
93 | "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ | |
94 | : : [SRC] "r" (src)); \ | |
95 | break; \ | |
7f547f85 RD |
96 | } \ |
97 | } | |
98 | ||
99 | #define XOR(r...) \ | |
100 | { \ | |
101 | switch (REG_CNT(r)) { \ | |
102 | case 8: \ | |
103 | __asm( \ | |
104 | "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ | |
105 | "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ | |
106 | "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ | |
107 | "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ | |
108 | break; \ | |
109 | case 4: \ | |
110 | __asm( \ | |
111 | "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ | |
112 | "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ | |
113 | break; \ | |
7f547f85 RD |
114 | } \ |
115 | } | |
116 | ||
65d71d42 GN |
117 | |
118 | #define ZERO(r...) XOR(r, r) | |
119 | ||
7f547f85 RD |
120 | |
121 | #define COPY(r...) \ | |
122 | { \ | |
123 | switch (REG_CNT(r)) { \ | |
124 | case 8: \ | |
125 | __asm( \ | |
126 | "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ | |
127 | "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ | |
128 | "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ | |
129 | "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ | |
130 | break; \ | |
131 | case 4: \ | |
132 | __asm( \ | |
133 | "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ | |
134 | "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ | |
135 | break; \ | |
7f547f85 RD |
136 | } \ |
137 | } | |
138 | ||
139 | #define LOAD(src, r...) \ | |
140 | { \ | |
141 | switch (REG_CNT(r)) { \ | |
142 | case 4: \ | |
143 | __asm( \ | |
144 | "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ | |
145 | "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ | |
146 | "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ | |
147 | "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ | |
148 | : : [SRC] "r" (src)); \ | |
149 | break; \ | |
7f547f85 RD |
150 | } \ |
151 | } | |
152 | ||
153 | #define STORE(dst, r...) \ | |
154 | { \ | |
155 | switch (REG_CNT(r)) { \ | |
156 | case 4: \ | |
157 | __asm( \ | |
158 | "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ | |
159 | "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ | |
160 | "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ | |
161 | "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ | |
162 | : : [DST] "r" (dst)); \ | |
163 | break; \ | |
7f547f85 RD |
164 | } \ |
165 | } | |
166 | ||
7f547f85 RD |
167 | #define MUL2_SETUP() \ |
168 | { \ | |
65d71d42 GN |
169 | __asm("vmovq %0, %%xmm31" :: "r"(0x1d1d1d1d1d1d1d1d)); \ |
170 | __asm("vpbroadcastq %xmm31, %zmm31"); \ | |
171 | __asm("vmovq %0, %%xmm30" :: "r"(0x8080808080808080)); \ | |
172 | __asm("vpbroadcastq %xmm30, %zmm30"); \ | |
173 | __asm("vmovq %0, %%xmm29" :: "r"(0xfefefefefefefefe)); \ | |
174 | __asm("vpbroadcastq %xmm29, %zmm29"); \ | |
7f547f85 RD |
175 | } |
176 | ||
177 | #define _MUL2(r...) \ | |
178 | { \ | |
179 | switch (REG_CNT(r)) { \ | |
180 | case 2: \ | |
181 | __asm( \ | |
65d71d42 GN |
182 | "vpandq %" VR0(r)", %zmm30, %zmm26\n" \ |
183 | "vpandq %" VR1(r)", %zmm30, %zmm25\n" \ | |
184 | "vpsrlq $7, %zmm26, %zmm28\n" \ | |
185 | "vpsrlq $7, %zmm25, %zmm27\n" \ | |
186 | "vpsllq $1, %zmm26, %zmm26\n" \ | |
187 | "vpsllq $1, %zmm25, %zmm25\n" \ | |
188 | "vpsubq %zmm28, %zmm26, %zmm26\n" \ | |
189 | "vpsubq %zmm27, %zmm25, %zmm25\n" \ | |
7f547f85 RD |
190 | "vpsllq $1, %" VR0(r)", %" VR0(r) "\n" \ |
191 | "vpsllq $1, %" VR1(r)", %" VR1(r) "\n" \ | |
65d71d42 GN |
192 | "vpandq %zmm26, %zmm31, %zmm26\n" \ |
193 | "vpandq %zmm25, %zmm31, %zmm25\n" \ | |
194 | "vpternlogd $0x6c,%zmm29, %zmm26, %" VR0(r) "\n" \ | |
195 | "vpternlogd $0x6c,%zmm29, %zmm25, %" VR1(r)); \ | |
7f547f85 | 196 | break; \ |
7f547f85 RD |
197 | } \ |
198 | } | |
199 | ||
200 | #define MUL2(r...) \ | |
201 | { \ | |
202 | switch (REG_CNT(r)) { \ | |
203 | case 4: \ | |
204 | _MUL2(R_01(r)); \ | |
205 | _MUL2(R_23(r)); \ | |
206 | break; \ | |
207 | case 2: \ | |
208 | _MUL2(r); \ | |
209 | break; \ | |
7f547f85 RD |
210 | } \ |
211 | } | |
212 | ||
213 | #define MUL4(r...) \ | |
214 | { \ | |
215 | MUL2(r); \ | |
216 | MUL2(r); \ | |
217 | } | |
218 | ||
7f547f85 | 219 | |
65d71d42 | 220 | /* General multiplication by adding powers of two */ |
7f547f85 | 221 | |
65d71d42 GN |
222 | #define _mul_x2_in 21, 22 |
223 | #define _mul_x2_acc 23, 24 | |
7f547f85 | 224 | |
65d71d42 | 225 | #define _MUL_PARAM(x, in, acc) \ |
7f547f85 | 226 | { \ |
65d71d42 GN |
227 | if (x & 0x01) { COPY(in, acc); } else { ZERO(acc); } \ |
228 | if (x & 0xfe) { MUL2(in); } \ | |
229 | if (x & 0x02) { XOR(in, acc); } \ | |
230 | if (x & 0xfc) { MUL2(in); } \ | |
231 | if (x & 0x04) { XOR(in, acc); } \ | |
232 | if (x & 0xf8) { MUL2(in); } \ | |
233 | if (x & 0x08) { XOR(in, acc); } \ | |
234 | if (x & 0xf0) { MUL2(in); } \ | |
235 | if (x & 0x10) { XOR(in, acc); } \ | |
236 | if (x & 0xe0) { MUL2(in); } \ | |
237 | if (x & 0x20) { XOR(in, acc); } \ | |
238 | if (x & 0xc0) { MUL2(in); } \ | |
239 | if (x & 0x40) { XOR(in, acc); } \ | |
240 | if (x & 0x80) { MUL2(in); XOR(in, acc); } \ | |
7f547f85 RD |
241 | } |
242 | ||
65d71d42 GN |
243 | #define MUL_x2_DEFINE(x) \ |
244 | static void \ | |
245 | mul_x2_ ## x(void) { _MUL_PARAM(x, _mul_x2_in, _mul_x2_acc); } | |
246 | ||
247 | ||
248 | MUL_x2_DEFINE(0); MUL_x2_DEFINE(1); MUL_x2_DEFINE(2); MUL_x2_DEFINE(3); | |
249 | MUL_x2_DEFINE(4); MUL_x2_DEFINE(5); MUL_x2_DEFINE(6); MUL_x2_DEFINE(7); | |
250 | MUL_x2_DEFINE(8); MUL_x2_DEFINE(9); MUL_x2_DEFINE(10); MUL_x2_DEFINE(11); | |
251 | MUL_x2_DEFINE(12); MUL_x2_DEFINE(13); MUL_x2_DEFINE(14); MUL_x2_DEFINE(15); | |
252 | MUL_x2_DEFINE(16); MUL_x2_DEFINE(17); MUL_x2_DEFINE(18); MUL_x2_DEFINE(19); | |
253 | MUL_x2_DEFINE(20); MUL_x2_DEFINE(21); MUL_x2_DEFINE(22); MUL_x2_DEFINE(23); | |
254 | MUL_x2_DEFINE(24); MUL_x2_DEFINE(25); MUL_x2_DEFINE(26); MUL_x2_DEFINE(27); | |
255 | MUL_x2_DEFINE(28); MUL_x2_DEFINE(29); MUL_x2_DEFINE(30); MUL_x2_DEFINE(31); | |
256 | MUL_x2_DEFINE(32); MUL_x2_DEFINE(33); MUL_x2_DEFINE(34); MUL_x2_DEFINE(35); | |
257 | MUL_x2_DEFINE(36); MUL_x2_DEFINE(37); MUL_x2_DEFINE(38); MUL_x2_DEFINE(39); | |
258 | MUL_x2_DEFINE(40); MUL_x2_DEFINE(41); MUL_x2_DEFINE(42); MUL_x2_DEFINE(43); | |
259 | MUL_x2_DEFINE(44); MUL_x2_DEFINE(45); MUL_x2_DEFINE(46); MUL_x2_DEFINE(47); | |
260 | MUL_x2_DEFINE(48); MUL_x2_DEFINE(49); MUL_x2_DEFINE(50); MUL_x2_DEFINE(51); | |
261 | MUL_x2_DEFINE(52); MUL_x2_DEFINE(53); MUL_x2_DEFINE(54); MUL_x2_DEFINE(55); | |
262 | MUL_x2_DEFINE(56); MUL_x2_DEFINE(57); MUL_x2_DEFINE(58); MUL_x2_DEFINE(59); | |
263 | MUL_x2_DEFINE(60); MUL_x2_DEFINE(61); MUL_x2_DEFINE(62); MUL_x2_DEFINE(63); | |
264 | MUL_x2_DEFINE(64); MUL_x2_DEFINE(65); MUL_x2_DEFINE(66); MUL_x2_DEFINE(67); | |
265 | MUL_x2_DEFINE(68); MUL_x2_DEFINE(69); MUL_x2_DEFINE(70); MUL_x2_DEFINE(71); | |
266 | MUL_x2_DEFINE(72); MUL_x2_DEFINE(73); MUL_x2_DEFINE(74); MUL_x2_DEFINE(75); | |
267 | MUL_x2_DEFINE(76); MUL_x2_DEFINE(77); MUL_x2_DEFINE(78); MUL_x2_DEFINE(79); | |
268 | MUL_x2_DEFINE(80); MUL_x2_DEFINE(81); MUL_x2_DEFINE(82); MUL_x2_DEFINE(83); | |
269 | MUL_x2_DEFINE(84); MUL_x2_DEFINE(85); MUL_x2_DEFINE(86); MUL_x2_DEFINE(87); | |
270 | MUL_x2_DEFINE(88); MUL_x2_DEFINE(89); MUL_x2_DEFINE(90); MUL_x2_DEFINE(91); | |
271 | MUL_x2_DEFINE(92); MUL_x2_DEFINE(93); MUL_x2_DEFINE(94); MUL_x2_DEFINE(95); | |
272 | MUL_x2_DEFINE(96); MUL_x2_DEFINE(97); MUL_x2_DEFINE(98); MUL_x2_DEFINE(99); | |
273 | MUL_x2_DEFINE(100); MUL_x2_DEFINE(101); MUL_x2_DEFINE(102); MUL_x2_DEFINE(103); | |
274 | MUL_x2_DEFINE(104); MUL_x2_DEFINE(105); MUL_x2_DEFINE(106); MUL_x2_DEFINE(107); | |
275 | MUL_x2_DEFINE(108); MUL_x2_DEFINE(109); MUL_x2_DEFINE(110); MUL_x2_DEFINE(111); | |
276 | MUL_x2_DEFINE(112); MUL_x2_DEFINE(113); MUL_x2_DEFINE(114); MUL_x2_DEFINE(115); | |
277 | MUL_x2_DEFINE(116); MUL_x2_DEFINE(117); MUL_x2_DEFINE(118); MUL_x2_DEFINE(119); | |
278 | MUL_x2_DEFINE(120); MUL_x2_DEFINE(121); MUL_x2_DEFINE(122); MUL_x2_DEFINE(123); | |
279 | MUL_x2_DEFINE(124); MUL_x2_DEFINE(125); MUL_x2_DEFINE(126); MUL_x2_DEFINE(127); | |
280 | MUL_x2_DEFINE(128); MUL_x2_DEFINE(129); MUL_x2_DEFINE(130); MUL_x2_DEFINE(131); | |
281 | MUL_x2_DEFINE(132); MUL_x2_DEFINE(133); MUL_x2_DEFINE(134); MUL_x2_DEFINE(135); | |
282 | MUL_x2_DEFINE(136); MUL_x2_DEFINE(137); MUL_x2_DEFINE(138); MUL_x2_DEFINE(139); | |
283 | MUL_x2_DEFINE(140); MUL_x2_DEFINE(141); MUL_x2_DEFINE(142); MUL_x2_DEFINE(143); | |
284 | MUL_x2_DEFINE(144); MUL_x2_DEFINE(145); MUL_x2_DEFINE(146); MUL_x2_DEFINE(147); | |
285 | MUL_x2_DEFINE(148); MUL_x2_DEFINE(149); MUL_x2_DEFINE(150); MUL_x2_DEFINE(151); | |
286 | MUL_x2_DEFINE(152); MUL_x2_DEFINE(153); MUL_x2_DEFINE(154); MUL_x2_DEFINE(155); | |
287 | MUL_x2_DEFINE(156); MUL_x2_DEFINE(157); MUL_x2_DEFINE(158); MUL_x2_DEFINE(159); | |
288 | MUL_x2_DEFINE(160); MUL_x2_DEFINE(161); MUL_x2_DEFINE(162); MUL_x2_DEFINE(163); | |
289 | MUL_x2_DEFINE(164); MUL_x2_DEFINE(165); MUL_x2_DEFINE(166); MUL_x2_DEFINE(167); | |
290 | MUL_x2_DEFINE(168); MUL_x2_DEFINE(169); MUL_x2_DEFINE(170); MUL_x2_DEFINE(171); | |
291 | MUL_x2_DEFINE(172); MUL_x2_DEFINE(173); MUL_x2_DEFINE(174); MUL_x2_DEFINE(175); | |
292 | MUL_x2_DEFINE(176); MUL_x2_DEFINE(177); MUL_x2_DEFINE(178); MUL_x2_DEFINE(179); | |
293 | MUL_x2_DEFINE(180); MUL_x2_DEFINE(181); MUL_x2_DEFINE(182); MUL_x2_DEFINE(183); | |
294 | MUL_x2_DEFINE(184); MUL_x2_DEFINE(185); MUL_x2_DEFINE(186); MUL_x2_DEFINE(187); | |
295 | MUL_x2_DEFINE(188); MUL_x2_DEFINE(189); MUL_x2_DEFINE(190); MUL_x2_DEFINE(191); | |
296 | MUL_x2_DEFINE(192); MUL_x2_DEFINE(193); MUL_x2_DEFINE(194); MUL_x2_DEFINE(195); | |
297 | MUL_x2_DEFINE(196); MUL_x2_DEFINE(197); MUL_x2_DEFINE(198); MUL_x2_DEFINE(199); | |
298 | MUL_x2_DEFINE(200); MUL_x2_DEFINE(201); MUL_x2_DEFINE(202); MUL_x2_DEFINE(203); | |
299 | MUL_x2_DEFINE(204); MUL_x2_DEFINE(205); MUL_x2_DEFINE(206); MUL_x2_DEFINE(207); | |
300 | MUL_x2_DEFINE(208); MUL_x2_DEFINE(209); MUL_x2_DEFINE(210); MUL_x2_DEFINE(211); | |
301 | MUL_x2_DEFINE(212); MUL_x2_DEFINE(213); MUL_x2_DEFINE(214); MUL_x2_DEFINE(215); | |
302 | MUL_x2_DEFINE(216); MUL_x2_DEFINE(217); MUL_x2_DEFINE(218); MUL_x2_DEFINE(219); | |
303 | MUL_x2_DEFINE(220); MUL_x2_DEFINE(221); MUL_x2_DEFINE(222); MUL_x2_DEFINE(223); | |
304 | MUL_x2_DEFINE(224); MUL_x2_DEFINE(225); MUL_x2_DEFINE(226); MUL_x2_DEFINE(227); | |
305 | MUL_x2_DEFINE(228); MUL_x2_DEFINE(229); MUL_x2_DEFINE(230); MUL_x2_DEFINE(231); | |
306 | MUL_x2_DEFINE(232); MUL_x2_DEFINE(233); MUL_x2_DEFINE(234); MUL_x2_DEFINE(235); | |
307 | MUL_x2_DEFINE(236); MUL_x2_DEFINE(237); MUL_x2_DEFINE(238); MUL_x2_DEFINE(239); | |
308 | MUL_x2_DEFINE(240); MUL_x2_DEFINE(241); MUL_x2_DEFINE(242); MUL_x2_DEFINE(243); | |
309 | MUL_x2_DEFINE(244); MUL_x2_DEFINE(245); MUL_x2_DEFINE(246); MUL_x2_DEFINE(247); | |
310 | MUL_x2_DEFINE(248); MUL_x2_DEFINE(249); MUL_x2_DEFINE(250); MUL_x2_DEFINE(251); | |
311 | MUL_x2_DEFINE(252); MUL_x2_DEFINE(253); MUL_x2_DEFINE(254); MUL_x2_DEFINE(255); | |
312 | ||
313 | ||
314 | typedef void (*mul_fn_ptr_t)(void); | |
315 | ||
316 | static const mul_fn_ptr_t __attribute__((aligned(256))) | |
317 | gf_x2_mul_fns[256] = { | |
318 | mul_x2_0, mul_x2_1, mul_x2_2, mul_x2_3, mul_x2_4, mul_x2_5, | |
319 | mul_x2_6, mul_x2_7, mul_x2_8, mul_x2_9, mul_x2_10, mul_x2_11, | |
320 | mul_x2_12, mul_x2_13, mul_x2_14, mul_x2_15, mul_x2_16, mul_x2_17, | |
321 | mul_x2_18, mul_x2_19, mul_x2_20, mul_x2_21, mul_x2_22, mul_x2_23, | |
322 | mul_x2_24, mul_x2_25, mul_x2_26, mul_x2_27, mul_x2_28, mul_x2_29, | |
323 | mul_x2_30, mul_x2_31, mul_x2_32, mul_x2_33, mul_x2_34, mul_x2_35, | |
324 | mul_x2_36, mul_x2_37, mul_x2_38, mul_x2_39, mul_x2_40, mul_x2_41, | |
325 | mul_x2_42, mul_x2_43, mul_x2_44, mul_x2_45, mul_x2_46, mul_x2_47, | |
326 | mul_x2_48, mul_x2_49, mul_x2_50, mul_x2_51, mul_x2_52, mul_x2_53, | |
327 | mul_x2_54, mul_x2_55, mul_x2_56, mul_x2_57, mul_x2_58, mul_x2_59, | |
328 | mul_x2_60, mul_x2_61, mul_x2_62, mul_x2_63, mul_x2_64, mul_x2_65, | |
329 | mul_x2_66, mul_x2_67, mul_x2_68, mul_x2_69, mul_x2_70, mul_x2_71, | |
330 | mul_x2_72, mul_x2_73, mul_x2_74, mul_x2_75, mul_x2_76, mul_x2_77, | |
331 | mul_x2_78, mul_x2_79, mul_x2_80, mul_x2_81, mul_x2_82, mul_x2_83, | |
332 | mul_x2_84, mul_x2_85, mul_x2_86, mul_x2_87, mul_x2_88, mul_x2_89, | |
333 | mul_x2_90, mul_x2_91, mul_x2_92, mul_x2_93, mul_x2_94, mul_x2_95, | |
334 | mul_x2_96, mul_x2_97, mul_x2_98, mul_x2_99, mul_x2_100, mul_x2_101, | |
335 | mul_x2_102, mul_x2_103, mul_x2_104, mul_x2_105, mul_x2_106, mul_x2_107, | |
336 | mul_x2_108, mul_x2_109, mul_x2_110, mul_x2_111, mul_x2_112, mul_x2_113, | |
337 | mul_x2_114, mul_x2_115, mul_x2_116, mul_x2_117, mul_x2_118, mul_x2_119, | |
338 | mul_x2_120, mul_x2_121, mul_x2_122, mul_x2_123, mul_x2_124, mul_x2_125, | |
339 | mul_x2_126, mul_x2_127, mul_x2_128, mul_x2_129, mul_x2_130, mul_x2_131, | |
340 | mul_x2_132, mul_x2_133, mul_x2_134, mul_x2_135, mul_x2_136, mul_x2_137, | |
341 | mul_x2_138, mul_x2_139, mul_x2_140, mul_x2_141, mul_x2_142, mul_x2_143, | |
342 | mul_x2_144, mul_x2_145, mul_x2_146, mul_x2_147, mul_x2_148, mul_x2_149, | |
343 | mul_x2_150, mul_x2_151, mul_x2_152, mul_x2_153, mul_x2_154, mul_x2_155, | |
344 | mul_x2_156, mul_x2_157, mul_x2_158, mul_x2_159, mul_x2_160, mul_x2_161, | |
345 | mul_x2_162, mul_x2_163, mul_x2_164, mul_x2_165, mul_x2_166, mul_x2_167, | |
346 | mul_x2_168, mul_x2_169, mul_x2_170, mul_x2_171, mul_x2_172, mul_x2_173, | |
347 | mul_x2_174, mul_x2_175, mul_x2_176, mul_x2_177, mul_x2_178, mul_x2_179, | |
348 | mul_x2_180, mul_x2_181, mul_x2_182, mul_x2_183, mul_x2_184, mul_x2_185, | |
349 | mul_x2_186, mul_x2_187, mul_x2_188, mul_x2_189, mul_x2_190, mul_x2_191, | |
350 | mul_x2_192, mul_x2_193, mul_x2_194, mul_x2_195, mul_x2_196, mul_x2_197, | |
351 | mul_x2_198, mul_x2_199, mul_x2_200, mul_x2_201, mul_x2_202, mul_x2_203, | |
352 | mul_x2_204, mul_x2_205, mul_x2_206, mul_x2_207, mul_x2_208, mul_x2_209, | |
353 | mul_x2_210, mul_x2_211, mul_x2_212, mul_x2_213, mul_x2_214, mul_x2_215, | |
354 | mul_x2_216, mul_x2_217, mul_x2_218, mul_x2_219, mul_x2_220, mul_x2_221, | |
355 | mul_x2_222, mul_x2_223, mul_x2_224, mul_x2_225, mul_x2_226, mul_x2_227, | |
356 | mul_x2_228, mul_x2_229, mul_x2_230, mul_x2_231, mul_x2_232, mul_x2_233, | |
357 | mul_x2_234, mul_x2_235, mul_x2_236, mul_x2_237, mul_x2_238, mul_x2_239, | |
358 | mul_x2_240, mul_x2_241, mul_x2_242, mul_x2_243, mul_x2_244, mul_x2_245, | |
359 | mul_x2_246, mul_x2_247, mul_x2_248, mul_x2_249, mul_x2_250, mul_x2_251, | |
360 | mul_x2_252, mul_x2_253, mul_x2_254, mul_x2_255 | |
361 | }; | |
362 | ||
363 | #define MUL(c, r...) \ | |
7f547f85 RD |
364 | { \ |
365 | switch (REG_CNT(r)) { \ | |
366 | case 4: \ | |
65d71d42 GN |
367 | COPY(R_01(r), _mul_x2_in); \ |
368 | gf_x2_mul_fns[c](); \ | |
369 | COPY(_mul_x2_acc, R_01(r)); \ | |
370 | COPY(R_23(r), _mul_x2_in); \ | |
371 | gf_x2_mul_fns[c](); \ | |
372 | COPY(_mul_x2_acc, R_23(r)); \ | |
7f547f85 RD |
373 | } \ |
374 | } | |
375 | ||
65d71d42 | 376 | |
7f547f85 | 377 | #define raidz_math_begin() kfpu_begin() |
65d71d42 GN |
378 | #define raidz_math_end() kfpu_end() |
379 | ||
380 | ||
381 | #define SYN_STRIDE 4 | |
7f547f85 | 382 | |
cbf484f8 GN |
383 | #define ZERO_STRIDE 4 |
384 | #define ZERO_DEFINE() {} | |
65d71d42 | 385 | #define ZERO_D 0, 1, 2, 3 |
cbf484f8 GN |
386 | |
387 | #define COPY_STRIDE 4 | |
388 | #define COPY_DEFINE() {} | |
65d71d42 | 389 | #define COPY_D 0, 1, 2, 3 |
cbf484f8 GN |
390 | |
391 | #define ADD_STRIDE 4 | |
392 | #define ADD_DEFINE() {} | |
65d71d42 | 393 | #define ADD_D 0, 1, 2, 3 |
cbf484f8 GN |
394 | |
395 | #define MUL_STRIDE 4 | |
65d71d42 GN |
396 | #define MUL_DEFINE() MUL2_SETUP() |
397 | #define MUL_D 0, 1, 2, 3 | |
398 | ||
7f547f85 | 399 | #define GEN_P_STRIDE 4 |
65d71d42 GN |
400 | #define GEN_P_DEFINE() {} |
401 | #define GEN_P_P 0, 1, 2, 3 | |
7f547f85 | 402 | |
7f547f85 | 403 | #define GEN_PQ_STRIDE 4 |
65d71d42 GN |
404 | #define GEN_PQ_DEFINE() {} |
405 | #define GEN_PQ_D 0, 1, 2, 3 | |
406 | #define GEN_PQ_C 4, 5, 6, 7 | |
7f547f85 | 407 | |
65d71d42 | 408 | #define GEN_PQR_STRIDE 4 |
7f547f85 | 409 | #define GEN_PQR_DEFINE() {} |
65d71d42 GN |
410 | #define GEN_PQR_D 0, 1, 2, 3 |
411 | #define GEN_PQR_C 4, 5, 6, 7 | |
412 | ||
413 | #define SYN_Q_DEFINE() {} | |
414 | #define SYN_Q_D 0, 1, 2, 3 | |
415 | #define SYN_Q_X 4, 5, 6, 7 | |
416 | ||
417 | #define SYN_R_DEFINE() {} | |
418 | #define SYN_R_D 0, 1, 2, 3 | |
419 | #define SYN_R_X 4, 5, 6, 7 | |
420 | ||
421 | #define SYN_PQ_DEFINE() {} | |
422 | #define SYN_PQ_D 0, 1, 2, 3 | |
423 | #define SYN_PQ_X 4, 5, 6, 7 | |
424 | ||
425 | #define REC_PQ_STRIDE 4 | |
426 | #define REC_PQ_DEFINE() MUL2_SETUP() | |
427 | #define REC_PQ_X 0, 1, 2, 3 | |
428 | #define REC_PQ_Y 4, 5, 6, 7 | |
429 | #define REC_PQ_T 8, 9, 10, 11 | |
430 | ||
431 | #define SYN_PR_DEFINE() {} | |
432 | #define SYN_PR_D 0, 1, 2, 3 | |
433 | #define SYN_PR_X 4, 5, 6, 7 | |
434 | ||
435 | #define REC_PR_STRIDE 4 | |
436 | #define REC_PR_DEFINE() MUL2_SETUP() | |
437 | #define REC_PR_X 0, 1, 2, 3 | |
438 | #define REC_PR_Y 4, 5, 6, 7 | |
439 | #define REC_PR_T 8, 9, 10, 11 | |
440 | ||
441 | #define SYN_QR_DEFINE() {} | |
442 | #define SYN_QR_D 0, 1, 2, 3 | |
443 | #define SYN_QR_X 4, 5, 6, 7 | |
444 | ||
445 | #define REC_QR_STRIDE 4 | |
446 | #define REC_QR_DEFINE() MUL2_SETUP() | |
447 | #define REC_QR_X 0, 1, 2, 3 | |
448 | #define REC_QR_Y 4, 5, 6, 7 | |
449 | #define REC_QR_T 8, 9, 10, 11 | |
450 | ||
451 | #define SYN_PQR_DEFINE() {} | |
452 | #define SYN_PQR_D 0, 1, 2, 3 | |
453 | #define SYN_PQR_X 4, 5, 6, 7 | |
454 | ||
455 | #define REC_PQR_STRIDE 4 | |
456 | #define REC_PQR_DEFINE() MUL2_SETUP() | |
457 | #define REC_PQR_X 0, 1, 2, 3 | |
458 | #define REC_PQR_Y 4, 5, 6, 7 | |
459 | #define REC_PQR_Z 8, 9, 10, 11 | |
460 | #define REC_PQR_XS 12, 13, 14, 15 | |
461 | #define REC_PQR_YS 16, 17, 18, 19 | |
7f547f85 RD |
462 | |
463 | ||
464 | #include <sys/vdev_raidz_impl.h> | |
465 | #include "vdev_raidz_math_impl.h" | |
466 | ||
467 | DEFINE_GEN_METHODS(avx512f); | |
468 | DEFINE_REC_METHODS(avx512f); | |
469 | ||
470 | static boolean_t | |
471 | raidz_will_avx512f_work(void) | |
472 | { | |
473 | return (zfs_avx_available() && | |
65d71d42 | 474 | zfs_avx2_available() && |
7f547f85 RD |
475 | zfs_avx512f_available()); |
476 | } | |
477 | ||
478 | const raidz_impl_ops_t vdev_raidz_avx512f_impl = { | |
479 | .init = NULL, | |
480 | .fini = NULL, | |
481 | .gen = RAIDZ_GEN_METHODS(avx512f), | |
482 | .rec = RAIDZ_REC_METHODS(avx512f), | |
483 | .is_supported = &raidz_will_avx512f_work, | |
484 | .name = "avx512f" | |
485 | }; | |
486 | ||
487 | #endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ |