]>
Commit | Line | Data |
---|---|---|
35a76a03 TS |
1 | /* |
2 | * Implement fast Fletcher4 with SSE2,SSSE3 instructions. (x86) | |
3 | * | |
4 | * Use the 128-bit SSE2/SSSE3 SIMD instructions and registers to compute | |
7f319493 | 5 | * Fletcher4 in two incremental 64-bit parallel accumulator streams, |
35a76a03 TS |
6 | * and then combine the streams to form the final four checksum words. |
7 | * This implementation is a derivative of the AVX SIMD implementation by | |
8 | * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c). | |
9 | * | |
10 | * Copyright (C) 2016 Tyler J. Stachecki. | |
11 | * | |
12 | * Authors: | |
13 | * Tyler J. Stachecki <stachecki.tyler@gmail.com> | |
14 | * | |
15 | * This software is available to you under a choice of one of two | |
16 | * licenses. You may choose to be licensed under the terms of the GNU | |
17 | * General Public License (GPL) Version 2, available from the file | |
18 | * COPYING in the main directory of this source tree, or the | |
19 | * OpenIB.org BSD license below: | |
20 | * | |
21 | * Redistribution and use in source and binary forms, with or | |
22 | * without modification, are permitted provided that the following | |
23 | * conditions are met: | |
24 | * | |
25 | * - Redistributions of source code must retain the above | |
26 | * copyright notice, this list of conditions and the following | |
27 | * disclaimer. | |
28 | * | |
29 | * - Redistributions in binary form must reproduce the above | |
30 | * copyright notice, this list of conditions and the following | |
31 | * disclaimer in the documentation and/or other materials | |
32 | * provided with the distribution. | |
33 | * | |
34 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
35 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
36 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
37 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
38 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
39 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
40 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
41 | * SOFTWARE. | |
42 | */ | |
43 | ||
44 | #if defined(HAVE_SSE2) | |
45 | ||
46 | #include <linux/simd_x86.h> | |
47 | #include <sys/spa_checksum.h> | |
5bf703b8 | 48 | #include <sys/byteorder.h> |
93ce2b4c | 49 | #include <sys/strings.h> |
35a76a03 | 50 | #include <zfs_fletcher.h> |
35a76a03 TS |
51 | |
52 | static void | |
4ea3f864 GM |
53 | fletcher_4_sse2_init(fletcher_4_ctx_t *ctx) |
54 | { | |
5bf703b8 | 55 | bzero(ctx->sse, 4 * sizeof (zfs_fletcher_sse_t)); |
35a76a03 TS |
56 | } |
57 | ||
58 | static void | |
4ea3f864 GM |
59 | fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) |
60 | { | |
35a76a03 TS |
61 | uint64_t A, B, C, D; |
62 | ||
35a76a03 TS |
63 | /* |
64 | * The mixing matrix for checksum calculation is: | |
65 | * a = a0 + a1 | |
66 | * b = 2b0 + 2b1 - a1 | |
67 | * c = 4c0 - b0 + 4c1 -3b1 | |
68 | * d = 8d0 - 4c0 + 8d1 - 8c1 + b1; | |
69 | * | |
70 | * c and d are multiplied by 4 and 8, respectively, | |
71 | * before spilling the vectors out to memory. | |
72 | */ | |
5bf703b8 GN |
73 | A = ctx->sse[0].v[0] + ctx->sse[0].v[1]; |
74 | B = 2 * ctx->sse[1].v[0] + 2 * ctx->sse[1].v[1] - ctx->sse[0].v[1]; | |
75 | C = 4 * ctx->sse[2].v[0] - ctx->sse[1].v[0] + 4 * ctx->sse[2].v[1] - | |
76 | 3 * ctx->sse[1].v[1]; | |
77 | D = 8 * ctx->sse[3].v[0] - 4 * ctx->sse[2].v[0] + 8 * ctx->sse[3].v[1] - | |
78 | 8 * ctx->sse[2].v[1] + ctx->sse[1].v[1]; | |
35a76a03 TS |
79 | |
80 | ZIO_SET_CHECKSUM(zcp, A, B, C, D); | |
81 | } | |
82 | ||
5bf703b8 GN |
83 | #define FLETCHER_4_SSE_RESTORE_CTX(ctx) \ |
84 | { \ | |
85 | asm volatile("movdqu %0, %%xmm0" :: "m" ((ctx)->sse[0])); \ | |
86 | asm volatile("movdqu %0, %%xmm1" :: "m" ((ctx)->sse[1])); \ | |
87 | asm volatile("movdqu %0, %%xmm2" :: "m" ((ctx)->sse[2])); \ | |
88 | asm volatile("movdqu %0, %%xmm3" :: "m" ((ctx)->sse[3])); \ | |
89 | } | |
90 | ||
91 | #define FLETCHER_4_SSE_SAVE_CTX(ctx) \ | |
92 | { \ | |
93 | asm volatile("movdqu %%xmm0, %0" : "=m" ((ctx)->sse[0])); \ | |
94 | asm volatile("movdqu %%xmm1, %0" : "=m" ((ctx)->sse[1])); \ | |
95 | asm volatile("movdqu %%xmm2, %0" : "=m" ((ctx)->sse[2])); \ | |
96 | asm volatile("movdqu %%xmm3, %0" : "=m" ((ctx)->sse[3])); \ | |
97 | } | |
98 | ||
35a76a03 | 99 | static void |
5bf703b8 | 100 | fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) |
35a76a03 TS |
101 | { |
102 | const uint64_t *ip = buf; | |
103 | const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); | |
104 | ||
5bf703b8 GN |
105 | kfpu_begin(); |
106 | ||
107 | FLETCHER_4_SSE_RESTORE_CTX(ctx); | |
108 | ||
35a76a03 TS |
109 | asm volatile("pxor %xmm4, %xmm4"); |
110 | ||
111 | for (; ip < ipend; ip += 2) { | |
112 | asm volatile("movdqu %0, %%xmm5" :: "m"(*ip)); | |
113 | asm volatile("movdqa %xmm5, %xmm6"); | |
114 | asm volatile("punpckldq %xmm4, %xmm5"); | |
115 | asm volatile("punpckhdq %xmm4, %xmm6"); | |
116 | asm volatile("paddq %xmm5, %xmm0"); | |
117 | asm volatile("paddq %xmm0, %xmm1"); | |
118 | asm volatile("paddq %xmm1, %xmm2"); | |
119 | asm volatile("paddq %xmm2, %xmm3"); | |
120 | asm volatile("paddq %xmm6, %xmm0"); | |
121 | asm volatile("paddq %xmm0, %xmm1"); | |
122 | asm volatile("paddq %xmm1, %xmm2"); | |
123 | asm volatile("paddq %xmm2, %xmm3"); | |
124 | } | |
5bf703b8 GN |
125 | |
126 | FLETCHER_4_SSE_SAVE_CTX(ctx); | |
127 | ||
128 | kfpu_end(); | |
35a76a03 TS |
129 | } |
130 | ||
131 | static void | |
5bf703b8 | 132 | fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) |
35a76a03 TS |
133 | { |
134 | const uint32_t *ip = buf; | |
135 | const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); | |
136 | ||
5bf703b8 GN |
137 | kfpu_begin(); |
138 | ||
139 | FLETCHER_4_SSE_RESTORE_CTX(ctx); | |
35a76a03 | 140 | |
5bf703b8 GN |
141 | for (; ip < ipend; ip += 2) { |
142 | uint32_t scratch1 = BSWAP_32(ip[0]); | |
143 | uint32_t scratch2 = BSWAP_32(ip[1]); | |
144 | asm volatile("movd %0, %%xmm5" :: "r"(scratch1)); | |
145 | asm volatile("movd %0, %%xmm6" :: "r"(scratch2)); | |
35a76a03 TS |
146 | asm volatile("punpcklqdq %xmm6, %xmm5"); |
147 | asm volatile("paddq %xmm5, %xmm0"); | |
148 | asm volatile("paddq %xmm0, %xmm1"); | |
149 | asm volatile("paddq %xmm1, %xmm2"); | |
150 | asm volatile("paddq %xmm2, %xmm3"); | |
151 | } | |
5bf703b8 GN |
152 | |
153 | FLETCHER_4_SSE_SAVE_CTX(ctx); | |
154 | ||
155 | kfpu_end(); | |
35a76a03 TS |
156 | } |
157 | ||
158 | static boolean_t fletcher_4_sse2_valid(void) | |
159 | { | |
160 | return (zfs_sse2_available()); | |
161 | } | |
162 | ||
163 | const fletcher_4_ops_t fletcher_4_sse2_ops = { | |
fc897b24 GN |
164 | .init_native = fletcher_4_sse2_init, |
165 | .fini_native = fletcher_4_sse2_fini, | |
166 | .compute_native = fletcher_4_sse2_native, | |
167 | .init_byteswap = fletcher_4_sse2_init, | |
168 | .fini_byteswap = fletcher_4_sse2_fini, | |
35a76a03 TS |
169 | .compute_byteswap = fletcher_4_sse2_byteswap, |
170 | .valid = fletcher_4_sse2_valid, | |
171 | .name = "sse2" | |
172 | }; | |
173 | ||
174 | #endif /* defined(HAVE_SSE2) */ | |
175 | ||
176 | #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) | |
177 | static void | |
5bf703b8 | 178 | fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) |
35a76a03 | 179 | { |
5bf703b8 | 180 | static const zfs_fletcher_sse_t mask = { |
35a76a03 TS |
181 | .v = { 0x0405060700010203, 0x0C0D0E0F08090A0B } |
182 | }; | |
183 | ||
184 | const uint64_t *ip = buf; | |
185 | const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); | |
186 | ||
5bf703b8 GN |
187 | kfpu_begin(); |
188 | ||
189 | FLETCHER_4_SSE_RESTORE_CTX(ctx); | |
190 | ||
3d11ecbd | 191 | asm volatile("movdqu %0, %%xmm7"::"m" (mask)); |
35a76a03 TS |
192 | asm volatile("pxor %xmm4, %xmm4"); |
193 | ||
194 | for (; ip < ipend; ip += 2) { | |
195 | asm volatile("movdqu %0, %%xmm5"::"m" (*ip)); | |
196 | asm volatile("pshufb %xmm7, %xmm5"); | |
197 | asm volatile("movdqa %xmm5, %xmm6"); | |
198 | asm volatile("punpckldq %xmm4, %xmm5"); | |
199 | asm volatile("punpckhdq %xmm4, %xmm6"); | |
200 | asm volatile("paddq %xmm5, %xmm0"); | |
201 | asm volatile("paddq %xmm0, %xmm1"); | |
202 | asm volatile("paddq %xmm1, %xmm2"); | |
203 | asm volatile("paddq %xmm2, %xmm3"); | |
204 | asm volatile("paddq %xmm6, %xmm0"); | |
205 | asm volatile("paddq %xmm0, %xmm1"); | |
206 | asm volatile("paddq %xmm1, %xmm2"); | |
207 | asm volatile("paddq %xmm2, %xmm3"); | |
208 | } | |
5bf703b8 GN |
209 | |
210 | FLETCHER_4_SSE_SAVE_CTX(ctx); | |
211 | ||
212 | kfpu_end(); | |
35a76a03 TS |
213 | } |
214 | ||
215 | static boolean_t fletcher_4_ssse3_valid(void) | |
216 | { | |
217 | return (zfs_sse2_available() && zfs_ssse3_available()); | |
218 | } | |
219 | ||
220 | const fletcher_4_ops_t fletcher_4_ssse3_ops = { | |
fc897b24 GN |
221 | .init_native = fletcher_4_sse2_init, |
222 | .fini_native = fletcher_4_sse2_fini, | |
223 | .compute_native = fletcher_4_sse2_native, | |
224 | .init_byteswap = fletcher_4_sse2_init, | |
225 | .fini_byteswap = fletcher_4_sse2_fini, | |
35a76a03 TS |
226 | .compute_byteswap = fletcher_4_ssse3_byteswap, |
227 | .valid = fletcher_4_ssse3_valid, | |
228 | .name = "ssse3" | |
229 | }; | |
230 | ||
231 | #endif /* defined(HAVE_SSE2) && defined(HAVE_SSSE3) */ |