2 * Implement fast Fletcher4 with AVX2 instructions. (x86_64)
4 * Use the 256-bit AVX2 SIMD instructions and registers to compute
5 * Fletcher4 in four incremental 64-bit parallel accumulator streams,
6 * and then combine the streams to form the final four checksum words.
8 * Copyright (C) 2015 Intel Corporation.
11 * James Guilford <james.guilford@intel.com>
12 * Jinshan Xiong <jinshan.xiong@intel.com>
14 * This software is available to you under a choice of one of two
15 * licenses. You may choose to be licensed under the terms of the GNU
16 * General Public License (GPL) Version 2, available from the file
17 * COPYING in the main directory of this source tree, or the
18 * OpenIB.org BSD license below:
20 * Redistribution and use in source and binary forms, with or
21 * without modification, are permitted provided that the following
24 * - Redistributions of source code must retain the above
25 * copyright notice, this list of conditions and the following
28 * - Redistributions in binary form must reproduce the above
29 * copyright notice, this list of conditions and the following
30 * disclaimer in the documentation and/or other materials
31 * provided with the distribution.
33 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
34 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
35 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
36 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
37 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
38 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
39 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
43 #if defined(HAVE_AVX) && defined(HAVE_AVX2)
45 #include <sys/spa_checksum.h>
47 #include <sys/strings.h>
48 #include <zfs_fletcher.h>
50 ZFS_NO_SANITIZE_UNDEFINED
52 fletcher_4_avx2_init(fletcher_4_ctx_t
*ctx
)
54 memset(ctx
->avx
, 0, 4 * sizeof (zfs_fletcher_avx_t
));
57 ZFS_NO_SANITIZE_UNDEFINED
59 fletcher_4_avx2_fini(fletcher_4_ctx_t
*ctx
, zio_cksum_t
*zcp
)
63 A
= ctx
->avx
[0].v
[0] + ctx
->avx
[0].v
[1] +
64 ctx
->avx
[0].v
[2] + ctx
->avx
[0].v
[3];
65 B
= 0 - ctx
->avx
[0].v
[1] - 2 * ctx
->avx
[0].v
[2] - 3 * ctx
->avx
[0].v
[3] +
66 4 * ctx
->avx
[1].v
[0] + 4 * ctx
->avx
[1].v
[1] + 4 * ctx
->avx
[1].v
[2] +
69 C
= ctx
->avx
[0].v
[2] + 3 * ctx
->avx
[0].v
[3] - 6 * ctx
->avx
[1].v
[0] -
70 10 * ctx
->avx
[1].v
[1] - 14 * ctx
->avx
[1].v
[2] -
71 18 * ctx
->avx
[1].v
[3] + 16 * ctx
->avx
[2].v
[0] +
72 16 * ctx
->avx
[2].v
[1] + 16 * ctx
->avx
[2].v
[2] +
73 16 * ctx
->avx
[2].v
[3];
75 D
= 0 - ctx
->avx
[0].v
[3] + 4 * ctx
->avx
[1].v
[0] +
76 10 * ctx
->avx
[1].v
[1] + 20 * ctx
->avx
[1].v
[2] +
77 34 * ctx
->avx
[1].v
[3] - 48 * ctx
->avx
[2].v
[0] -
78 64 * ctx
->avx
[2].v
[1] - 80 * ctx
->avx
[2].v
[2] -
79 96 * ctx
->avx
[2].v
[3] + 64 * ctx
->avx
[3].v
[0] +
80 64 * ctx
->avx
[3].v
[1] + 64 * ctx
->avx
[3].v
[2] +
81 64 * ctx
->avx
[3].v
[3];
83 ZIO_SET_CHECKSUM(zcp
, A
, B
, C
, D
);
86 #define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \
88 asm volatile("vmovdqu %0, %%ymm0" :: "m" ((ctx)->avx[0])); \
89 asm volatile("vmovdqu %0, %%ymm1" :: "m" ((ctx)->avx[1])); \
90 asm volatile("vmovdqu %0, %%ymm2" :: "m" ((ctx)->avx[2])); \
91 asm volatile("vmovdqu %0, %%ymm3" :: "m" ((ctx)->avx[3])); \
94 #define FLETCHER_4_AVX2_SAVE_CTX(ctx) \
96 asm volatile("vmovdqu %%ymm0, %0" : "=m" ((ctx)->avx[0])); \
97 asm volatile("vmovdqu %%ymm1, %0" : "=m" ((ctx)->avx[1])); \
98 asm volatile("vmovdqu %%ymm2, %0" : "=m" ((ctx)->avx[2])); \
99 asm volatile("vmovdqu %%ymm3, %0" : "=m" ((ctx)->avx[3])); \
104 fletcher_4_avx2_native(fletcher_4_ctx_t
*ctx
, const void *buf
, uint64_t size
)
106 const uint64_t *ip
= buf
;
107 const uint64_t *ipend
= (uint64_t *)((uint8_t *)ip
+ size
);
111 FLETCHER_4_AVX2_RESTORE_CTX(ctx
);
113 for (; ip
< ipend
; ip
+= 2) {
114 asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip
));
115 asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
116 asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
117 asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
118 asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
121 FLETCHER_4_AVX2_SAVE_CTX(ctx
);
122 asm volatile("vzeroupper");
128 fletcher_4_avx2_byteswap(fletcher_4_ctx_t
*ctx
, const void *buf
, uint64_t size
)
130 static const zfs_fletcher_avx_t mask
= {
131 .v
= { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B,
132 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B }
134 const uint64_t *ip
= buf
;
135 const uint64_t *ipend
= (uint64_t *)((uint8_t *)ip
+ size
);
139 FLETCHER_4_AVX2_RESTORE_CTX(ctx
);
141 asm volatile("vmovdqu %0, %%ymm5" :: "m" (mask
));
143 for (; ip
< ipend
; ip
+= 2) {
144 asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip
));
145 asm volatile("vpshufb %ymm5, %ymm4, %ymm4");
147 asm volatile("vpaddq %ymm4, %ymm0, %ymm0");
148 asm volatile("vpaddq %ymm0, %ymm1, %ymm1");
149 asm volatile("vpaddq %ymm1, %ymm2, %ymm2");
150 asm volatile("vpaddq %ymm2, %ymm3, %ymm3");
153 FLETCHER_4_AVX2_SAVE_CTX(ctx
);
154 asm volatile("vzeroupper");
159 static boolean_t
fletcher_4_avx2_valid(void)
161 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
164 const fletcher_4_ops_t fletcher_4_avx2_ops
= {
165 .init_native
= fletcher_4_avx2_init
,
166 .fini_native
= fletcher_4_avx2_fini
,
167 .compute_native
= fletcher_4_avx2_native
,
168 .init_byteswap
= fletcher_4_avx2_init
,
169 .fini_byteswap
= fletcher_4_avx2_fini
,
170 .compute_byteswap
= fletcher_4_avx2_byteswap
,
171 .valid
= fletcher_4_avx2_valid
,
175 #endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */