]>
Commit | Line | Data |
---|---|---|
7f547f85 RD |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | /* | |
22 | * Copyright (C) 2016 Romain Dolbeau. All rights reserved. | |
23 | */ | |
24 | ||
25 | #include <sys/isa_defs.h> | |
26 | ||
cbf484f8 | 27 | #if 0 // defined(__x86_64) && defined(HAVE_AVX512BW) |
7f547f85 RD |
28 | |
29 | #include <sys/types.h> | |
30 | #include <linux/simd_x86.h> | |
31 | ||
32 | #define __asm __asm__ __volatile__ | |
33 | ||
34 | #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N | |
35 | #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) | |
36 | ||
37 | #define VR0_(REG, ...) "zmm"#REG | |
38 | #define VR1_(_1, REG, ...) "zmm"#REG | |
39 | #define VR2_(_1, _2, REG, ...) "zmm"#REG | |
40 | #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG | |
41 | #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG | |
42 | #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG | |
43 | #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG | |
44 | #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG | |
45 | ||
46 | #define VR0(r...) VR0_(r) | |
47 | #define VR1(r...) VR1_(r) | |
48 | #define VR2(r...) VR2_(r, 1) | |
49 | #define VR3(r...) VR3_(r, 1, 2) | |
50 | #define VR4(r...) VR4_(r, 1, 2) | |
51 | #define VR5(r...) VR5_(r, 1, 2, 3) | |
52 | #define VR6(r...) VR6_(r, 1, 2, 3, 4) | |
53 | #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) | |
54 | ||
55 | #define R_01(REG1, REG2, ...) REG1, REG2 | |
56 | #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 | |
57 | #define R_23(REG...) _R_23(REG, 1, 2, 3) | |
58 | ||
59 | #define ASM_BUG() ASSERT(0) | |
60 | ||
61 | extern const uint8_t gf_clmul_mod_lt[4*256][16]; | |
62 | ||
63 | #define ELEM_SIZE 64 | |
64 | ||
65 | typedef struct v { | |
66 | uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); | |
67 | } v_t; | |
68 | ||
02730c33 | 69 | #define PREFETCHNTA(ptr, offset) \ |
7f547f85 RD |
70 | { \ |
71 | __asm( \ | |
72 | "prefetchnta " #offset "(%[MEM])\n" \ | |
73 | : : [MEM] "r" (ptr)); \ | |
74 | } | |
75 | ||
02730c33 | 76 | #define PREFETCH(ptr, offset) \ |
7f547f85 RD |
77 | { \ |
78 | __asm( \ | |
79 | "prefetcht0 " #offset "(%[MEM])\n" \ | |
80 | : : [MEM] "r" (ptr)); \ | |
81 | } | |
82 | ||
83 | #define XOR_ACC(src, r...) \ | |
84 | { \ | |
85 | switch (REG_CNT(r)) { \ | |
86 | case 4: \ | |
87 | __asm( \ | |
88 | "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ | |
89 | "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ | |
90 | "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ | |
91 | "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ | |
92 | : : [SRC] "r" (src)); \ | |
93 | break; \ | |
94 | case 2: \ | |
95 | __asm( \ | |
96 | "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ | |
97 | "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ | |
98 | : : [SRC] "r" (src)); \ | |
99 | break; \ | |
100 | default: \ | |
101 | ASM_BUG(); \ | |
102 | } \ | |
103 | } | |
104 | ||
105 | #define XOR(r...) \ | |
106 | { \ | |
107 | switch (REG_CNT(r)) { \ | |
108 | case 8: \ | |
109 | __asm( \ | |
110 | "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ | |
111 | "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ | |
112 | "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ | |
113 | "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ | |
114 | break; \ | |
115 | case 4: \ | |
116 | __asm( \ | |
117 | "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ | |
118 | "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ | |
119 | break; \ | |
120 | default: \ | |
121 | ASM_BUG(); \ | |
122 | } \ | |
123 | } | |
124 | ||
125 | #define ZERO(r...) \ | |
126 | { \ | |
127 | switch (REG_CNT(r)) { \ | |
128 | case 4: \ | |
129 | __asm( \ | |
130 | "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ | |
131 | "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r) "\n" \ | |
132 | "vpxorq %" VR2(r) ", %" VR2(r)", %" VR2(r) "\n" \ | |
133 | "vpxorq %" VR3(r) ", %" VR3(r)", %" VR3(r)); \ | |
134 | break; \ | |
135 | case 2: \ | |
136 | __asm( \ | |
137 | "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ | |
138 | "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r)); \ | |
139 | break; \ | |
140 | default: \ | |
141 | ASM_BUG(); \ | |
142 | } \ | |
143 | } | |
144 | ||
02730c33 | 145 | #define COPY(r...) \ |
7f547f85 RD |
146 | { \ |
147 | switch (REG_CNT(r)) { \ | |
148 | case 8: \ | |
149 | __asm( \ | |
150 | "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ | |
151 | "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ | |
152 | "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ | |
153 | "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ | |
154 | break; \ | |
155 | case 4: \ | |
156 | __asm( \ | |
157 | "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ | |
158 | "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ | |
159 | break; \ | |
160 | default: \ | |
161 | ASM_BUG(); \ | |
162 | } \ | |
163 | } | |
164 | ||
02730c33 | 165 | #define LOAD(src, r...) \ |
7f547f85 RD |
166 | { \ |
167 | switch (REG_CNT(r)) { \ | |
168 | case 4: \ | |
169 | __asm( \ | |
170 | "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ | |
171 | "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ | |
172 | "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ | |
173 | "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ | |
174 | : : [SRC] "r" (src)); \ | |
175 | break; \ | |
176 | case 2: \ | |
177 | __asm( \ | |
178 | "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ | |
179 | "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ | |
180 | : : [SRC] "r" (src)); \ | |
181 | break; \ | |
182 | default: \ | |
183 | ASM_BUG(); \ | |
184 | } \ | |
185 | } | |
186 | ||
02730c33 | 187 | #define STORE(dst, r...) \ |
7f547f85 RD |
188 | { \ |
189 | switch (REG_CNT(r)) { \ | |
190 | case 4: \ | |
191 | __asm( \ | |
192 | "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ | |
193 | "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ | |
194 | "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ | |
195 | "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ | |
196 | : : [DST] "r" (dst)); \ | |
197 | break; \ | |
198 | case 2: \ | |
199 | __asm( \ | |
200 | "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ | |
201 | "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ | |
202 | : : [DST] "r" (dst)); \ | |
203 | break; \ | |
204 | default: \ | |
205 | ASM_BUG(); \ | |
206 | } \ | |
207 | } | |
208 | ||
209 | #define FLUSH() \ | |
210 | { \ | |
211 | __asm("vzeroupper"); \ | |
212 | } | |
213 | ||
02730c33 BB |
214 | #define MUL2_SETUP() \ |
215 | { \ | |
7f547f85 RD |
216 | __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ |
217 | __asm("vpbroadcastq %xmm14, %zmm14"); \ | |
218 | __asm("vmovq %0, %%xmm13" :: "r"(0x8080808080808080)); \ | |
219 | __asm("vpbroadcastq %xmm13, %zmm13"); \ | |
220 | __asm("vmovq %0, %%xmm12" :: "r"(0xfefefefefefefefe)); \ | |
221 | __asm("vpbroadcastq %xmm12, %zmm12"); \ | |
222 | __asm("vpxorq %zmm15, %zmm15 ,%zmm15"); \ | |
223 | } | |
224 | ||
02730c33 | 225 | #define _MUL2(r...) \ |
7f547f85 RD |
226 | { \ |
227 | switch (REG_CNT(r)) { \ | |
228 | case 2: \ | |
229 | __asm( \ | |
230 | "vpandq %" VR0(r)", %zmm13, %zmm10\n" \ | |
231 | "vpandq %" VR1(r)", %zmm13, %zmm11\n" \ | |
232 | "vpsrlq $7, %zmm10, %zmm8\n" \ | |
233 | "vpsrlq $7, %zmm11, %zmm9\n" \ | |
234 | "vpsllq $1, %zmm10, %zmm10\n" \ | |
235 | "vpsllq $1, %zmm11, %zmm11\n" \ | |
236 | "vpsubq %zmm8, %zmm10, %zmm10\n" \ | |
237 | "vpsubq %zmm9, %zmm11, %zmm11\n" \ | |
238 | "vpsllq $1, %" VR0(r)", %" VR0(r) "\n" \ | |
239 | "vpsllq $1, %" VR1(r)", %" VR1(r) "\n" \ | |
02730c33 BB |
240 | "vpandq %zmm10, %zmm14, %zmm10\n" \ |
241 | "vpandq %zmm11, %zmm14, %zmm11\n" \ | |
7f547f85 RD |
242 | "vpternlogd $0x6c,%zmm12, %zmm10, %" VR0(r) "\n" \ |
243 | "vpternlogd $0x6c,%zmm12, %zmm11, %" VR1(r)); \ | |
244 | break; \ | |
245 | default: \ | |
246 | ASM_BUG(); \ | |
247 | } \ | |
248 | } | |
249 | ||
250 | #define MUL2(r...) \ | |
251 | { \ | |
252 | switch (REG_CNT(r)) { \ | |
253 | case 4: \ | |
254 | _MUL2(R_01(r)); \ | |
255 | _MUL2(R_23(r)); \ | |
256 | break; \ | |
257 | case 2: \ | |
258 | _MUL2(r); \ | |
259 | break; \ | |
260 | default: \ | |
261 | ASM_BUG(); \ | |
262 | } \ | |
263 | } | |
264 | ||
265 | #define MUL4(r...) \ | |
266 | { \ | |
267 | MUL2(r); \ | |
268 | MUL2(r); \ | |
269 | } | |
270 | ||
271 | #define _0f "zmm15" | |
272 | #define _as "zmm14" | |
273 | #define _bs "zmm13" | |
274 | #define _ltmod "zmm12" | |
275 | #define _ltmul "zmm11" | |
276 | #define _ta "zmm10" | |
277 | #define _tb "zmm15" | |
278 | ||
279 | static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; | |
280 | ||
281 | #define _MULx2(c, r...) \ | |
282 | { \ | |
283 | switch (REG_CNT(r)) { \ | |
284 | case 2: \ | |
285 | __asm( \ | |
286 | "vpbroadcastb (%[mask]), %%" _0f "\n" \ | |
287 | /* upper bits */ \ | |
288 | "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n" \ | |
289 | "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n" \ | |
290 | \ | |
291 | "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ | |
292 | "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ | |
293 | "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ | |
294 | "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ | |
295 | "vpandq %%" _0f ", %%" _as ", %%" _as "\n" \ | |
296 | "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n" \ | |
297 | \ | |
298 | "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ | |
299 | "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ | |
300 | "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ | |
301 | "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ | |
302 | /* lower bits */ \ | |
303 | "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n" \ | |
304 | "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n" \ | |
305 | \ | |
306 | "vpxorq %%" _ta ", %%" _as ", %%" _as "\n" \ | |
307 | "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n" \ | |
308 | \ | |
309 | "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ | |
310 | "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ | |
311 | "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ | |
312 | "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ | |
313 | \ | |
314 | "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ | |
315 | "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ | |
316 | "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ | |
317 | "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ | |
318 | : : [mask] "r" (&_mul_mask), \ | |
319 | [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ | |
320 | break; \ | |
321 | default: \ | |
322 | ASM_BUG(); \ | |
323 | } \ | |
324 | } | |
325 | ||
326 | #define MUL(c, r...) \ | |
327 | { \ | |
328 | switch (REG_CNT(r)) { \ | |
329 | case 4: \ | |
330 | _MULx2(c, R_01(r)); \ | |
331 | _MULx2(c, R_23(r)); \ | |
332 | break; \ | |
333 | case 2: \ | |
334 | _MULx2(c, R_01(r)); \ | |
335 | break; \ | |
336 | default: \ | |
337 | ASM_BUG(); \ | |
338 | } \ | |
339 | } | |
340 | ||
341 | #define raidz_math_begin() kfpu_begin() | |
342 | #define raidz_math_end() \ | |
343 | { \ | |
344 | FLUSH(); \ | |
345 | kfpu_end(); \ | |
346 | } | |
347 | ||
cbf484f8 GN |
348 | #define ZERO_STRIDE 4 |
349 | #define ZERO_DEFINE() {} | |
350 | #define ZERO_D 0, 1, 2, 3 | |
351 | ||
352 | #define COPY_STRIDE 4 | |
353 | #define COPY_DEFINE() {} | |
354 | #define COPY_D 0, 1, 2, 3 | |
355 | ||
356 | #define ADD_STRIDE 4 | |
357 | #define ADD_DEFINE() {} | |
02730c33 | 358 | #define ADD_D 0, 1, 2, 3 |
cbf484f8 GN |
359 | |
360 | #define MUL_STRIDE 4 | |
02730c33 | 361 | #define MUL_DEFINE() {} |
cbf484f8 GN |
362 | #define MUL_D 0, 1, 2, 3 |
363 | ||
7f547f85 RD |
364 | #define GEN_P_DEFINE() {} |
365 | #define GEN_P_STRIDE 4 | |
366 | #define GEN_P_P 0, 1, 2, 3 | |
367 | ||
02730c33 | 368 | #define GEN_PQ_DEFINE() {} |
7f547f85 RD |
369 | #define GEN_PQ_STRIDE 4 |
370 | #define GEN_PQ_D 0, 1, 2, 3 | |
371 | #define GEN_PQ_P 4, 5, 6, 7 | |
372 | #define GEN_PQ_Q 20, 21, 22, 23 | |
373 | ||
02730c33 | 374 | #define GEN_PQR_DEFINE() {} |
7f547f85 RD |
375 | #define GEN_PQR_STRIDE 2 |
376 | #define GEN_PQR_D 0, 1 | |
377 | #define GEN_PQR_P 2, 3 | |
378 | #define GEN_PQR_Q 4, 5 | |
379 | #define GEN_PQR_R 6, 7 | |
380 | ||
02730c33 | 381 | #define REC_P_DEFINE() {} |
7f547f85 RD |
382 | #define REC_P_STRIDE 4 |
383 | #define REC_P_X 0, 1, 2, 3 | |
384 | ||
02730c33 | 385 | #define REC_Q_DEFINE() {} |
7f547f85 RD |
386 | #define REC_Q_STRIDE 4 |
387 | #define REC_Q_X 0, 1, 2, 3 | |
388 | ||
02730c33 | 389 | #define REC_R_DEFINE() {} |
7f547f85 RD |
390 | #define REC_R_STRIDE 4 |
391 | #define REC_R_X 0, 1, 2, 3 | |
392 | ||
02730c33 | 393 | #define REC_PQ_DEFINE() {} |
7f547f85 RD |
394 | #define REC_PQ_STRIDE 4 |
395 | #define REC_PQ_X 0, 1, 2, 3 | |
396 | #define REC_PQ_Y 4, 5, 6, 7 | |
397 | #define REC_PQ_D 20, 21, 22, 23 | |
398 | ||
02730c33 | 399 | #define REC_PR_DEFINE() {} |
7f547f85 RD |
400 | #define REC_PR_STRIDE 4 |
401 | #define REC_PR_X 0, 1, 2, 3 | |
402 | #define REC_PR_Y 4, 5, 6, 7 | |
403 | #define REC_PR_D 20, 21, 22, 23 | |
404 | ||
02730c33 | 405 | #define REC_QR_DEFINE() {} |
7f547f85 RD |
406 | #define REC_QR_STRIDE 4 |
407 | #define REC_QR_X 0, 1, 2, 3 | |
408 | #define REC_QR_Y 4, 5, 6, 7 | |
409 | #define REC_QR_D 20, 21, 22, 23 | |
410 | ||
02730c33 | 411 | #define REC_PQR_DEFINE() {} |
7f547f85 RD |
412 | #define REC_PQR_STRIDE 2 |
413 | #define REC_PQR_X 0, 1 | |
414 | #define REC_PQR_Y 2, 3 | |
415 | #define REC_PQR_Z 4, 5 | |
416 | #define REC_PQR_D 6, 7 | |
417 | #define REC_PQR_XS 6, 7 | |
418 | #define REC_PQR_YS 8, 9 | |
419 | ||
420 | ||
421 | #include <sys/vdev_raidz_impl.h> | |
422 | #include "vdev_raidz_math_impl.h" | |
423 | ||
424 | DEFINE_GEN_METHODS(avx512bw); | |
425 | DEFINE_REC_METHODS(avx512bw); | |
426 | ||
427 | static boolean_t | |
428 | raidz_will_avx512bw_work(void) | |
429 | { | |
430 | return (zfs_avx_available() && | |
02730c33 BB |
431 | zfs_avx512f_available() && |
432 | zfs_avx512bw_available()); | |
7f547f85 RD |
433 | } |
434 | ||
435 | const raidz_impl_ops_t vdev_raidz_avx512bw_impl = { | |
436 | .init = NULL, | |
437 | .fini = NULL, | |
438 | .gen = RAIDZ_GEN_METHODS(avx512bw), | |
439 | .rec = RAIDZ_REC_METHODS(avx512bw), | |
440 | .is_supported = &raidz_will_avx512bw_work, | |
441 | .name = "avx512bw" | |
442 | }; | |
443 | ||
444 | #endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */ |