]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support | |
3 | * | |
4 | * Copyright (c) 2005 Fabrice Bellard | |
5 | * Copyright (c) 2008 Intel Corporation <andrew.zaborowski@intel.com> | |
6 | * | |
7 | * This library is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU Lesser General Public | |
9 | * License as published by the Free Software Foundation; either | |
10 | * version 2.1 of the License, or (at your option) any later version. | |
11 | * | |
12 | * This library is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * Lesser General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU Lesser General Public | |
18 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | #include "crypto/aes.h" | |
22 | ||
23 | #if SHIFT == 0 | |
24 | #define Reg MMXReg | |
25 | #define XMM_ONLY(...) | |
26 | #define B(n) MMX_B(n) | |
27 | #define W(n) MMX_W(n) | |
28 | #define L(n) MMX_L(n) | |
29 | #define Q(n) MMX_Q(n) | |
30 | #define SUFFIX _mmx | |
31 | #else | |
32 | #define Reg ZMMReg | |
33 | #define XMM_ONLY(...) __VA_ARGS__ | |
34 | #define B(n) ZMM_B(n) | |
35 | #define W(n) ZMM_W(n) | |
36 | #define L(n) ZMM_L(n) | |
37 | #define Q(n) ZMM_Q(n) | |
38 | #if SHIFT == 1 | |
39 | #define SUFFIX _xmm | |
40 | #else | |
41 | #define SUFFIX _ymm | |
42 | #endif | |
43 | #endif | |
44 | ||
45 | #define LANE_WIDTH (SHIFT ? 16 : 8) | |
46 | #define PACK_WIDTH (LANE_WIDTH / 2) | |
47 | ||
48 | #if SHIFT == 0 | |
49 | #define FPSRL(x, c) ((x) >> shift) | |
50 | #define FPSRAW(x, c) ((int16_t)(x) >> shift) | |
51 | #define FPSRAL(x, c) ((int32_t)(x) >> shift) | |
52 | #define FPSLL(x, c) ((x) << shift) | |
53 | #endif | |
54 | ||
55 | void glue(helper_psrlw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
56 | { | |
57 | int shift; | |
58 | if (c->Q(0) > 15) { | |
59 | for (int i = 0; i < 1 << SHIFT; i++) { | |
60 | d->Q(i) = 0; | |
61 | } | |
62 | } else { | |
63 | shift = c->B(0); | |
64 | for (int i = 0; i < 4 << SHIFT; i++) { | |
65 | d->W(i) = FPSRL(s->W(i), shift); | |
66 | } | |
67 | } | |
68 | } | |
69 | ||
70 | void glue(helper_psllw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
71 | { | |
72 | int shift; | |
73 | if (c->Q(0) > 15) { | |
74 | for (int i = 0; i < 1 << SHIFT; i++) { | |
75 | d->Q(i) = 0; | |
76 | } | |
77 | } else { | |
78 | shift = c->B(0); | |
79 | for (int i = 0; i < 4 << SHIFT; i++) { | |
80 | d->W(i) = FPSLL(s->W(i), shift); | |
81 | } | |
82 | } | |
83 | } | |
84 | ||
85 | void glue(helper_psraw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
86 | { | |
87 | int shift; | |
88 | if (c->Q(0) > 15) { | |
89 | shift = 15; | |
90 | } else { | |
91 | shift = c->B(0); | |
92 | } | |
93 | for (int i = 0; i < 4 << SHIFT; i++) { | |
94 | d->W(i) = FPSRAW(s->W(i), shift); | |
95 | } | |
96 | } | |
97 | ||
98 | void glue(helper_psrld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
99 | { | |
100 | int shift; | |
101 | if (c->Q(0) > 31) { | |
102 | for (int i = 0; i < 1 << SHIFT; i++) { | |
103 | d->Q(i) = 0; | |
104 | } | |
105 | } else { | |
106 | shift = c->B(0); | |
107 | for (int i = 0; i < 2 << SHIFT; i++) { | |
108 | d->L(i) = FPSRL(s->L(i), shift); | |
109 | } | |
110 | } | |
111 | } | |
112 | ||
113 | void glue(helper_pslld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
114 | { | |
115 | int shift; | |
116 | if (c->Q(0) > 31) { | |
117 | for (int i = 0; i < 1 << SHIFT; i++) { | |
118 | d->Q(i) = 0; | |
119 | } | |
120 | } else { | |
121 | shift = c->B(0); | |
122 | for (int i = 0; i < 2 << SHIFT; i++) { | |
123 | d->L(i) = FPSLL(s->L(i), shift); | |
124 | } | |
125 | } | |
126 | } | |
127 | ||
128 | void glue(helper_psrad, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
129 | { | |
130 | int shift; | |
131 | if (c->Q(0) > 31) { | |
132 | shift = 31; | |
133 | } else { | |
134 | shift = c->B(0); | |
135 | } | |
136 | for (int i = 0; i < 2 << SHIFT; i++) { | |
137 | d->L(i) = FPSRAL(s->L(i), shift); | |
138 | } | |
139 | } | |
140 | ||
141 | void glue(helper_psrlq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
142 | { | |
143 | int shift; | |
144 | if (c->Q(0) > 63) { | |
145 | for (int i = 0; i < 1 << SHIFT; i++) { | |
146 | d->Q(i) = 0; | |
147 | } | |
148 | } else { | |
149 | shift = c->B(0); | |
150 | for (int i = 0; i < 1 << SHIFT; i++) { | |
151 | d->Q(i) = FPSRL(s->Q(i), shift); | |
152 | } | |
153 | } | |
154 | } | |
155 | ||
156 | void glue(helper_psllq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
157 | { | |
158 | int shift; | |
159 | if (c->Q(0) > 63) { | |
160 | for (int i = 0; i < 1 << SHIFT; i++) { | |
161 | d->Q(i) = 0; | |
162 | } | |
163 | } else { | |
164 | shift = c->B(0); | |
165 | for (int i = 0; i < 1 << SHIFT; i++) { | |
166 | d->Q(i) = FPSLL(s->Q(i), shift); | |
167 | } | |
168 | } | |
169 | } | |
170 | ||
171 | #if SHIFT >= 1 | |
172 | void glue(helper_psrldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
173 | { | |
174 | int shift, i, j; | |
175 | ||
176 | shift = c->L(0); | |
177 | if (shift > 16) { | |
178 | shift = 16; | |
179 | } | |
180 | for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) { | |
181 | for (i = 0; i < 16 - shift; i++) { | |
182 | d->B(j + i) = s->B(j + i + shift); | |
183 | } | |
184 | for (i = 16 - shift; i < 16; i++) { | |
185 | d->B(j + i) = 0; | |
186 | } | |
187 | } | |
188 | } | |
189 | ||
190 | void glue(helper_pslldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) | |
191 | { | |
192 | int shift, i, j; | |
193 | ||
194 | shift = c->L(0); | |
195 | if (shift > 16) { | |
196 | shift = 16; | |
197 | } | |
198 | for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) { | |
199 | for (i = 15; i >= shift; i--) { | |
200 | d->B(j + i) = s->B(j + i - shift); | |
201 | } | |
202 | for (i = 0; i < shift; i++) { | |
203 | d->B(j + i) = 0; | |
204 | } | |
205 | } | |
206 | } | |
207 | #endif | |
208 | ||
209 | #define SSE_HELPER_1(name, elem, num, F) \ | |
210 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) \ | |
211 | { \ | |
212 | int n = num; \ | |
213 | for (int i = 0; i < n; i++) { \ | |
214 | d->elem(i) = F(s->elem(i)); \ | |
215 | } \ | |
216 | } | |
217 | ||
218 | #define SSE_HELPER_2(name, elem, num, F) \ | |
219 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
220 | { \ | |
221 | int n = num; \ | |
222 | for (int i = 0; i < n; i++) { \ | |
223 | d->elem(i) = F(v->elem(i), s->elem(i)); \ | |
224 | } \ | |
225 | } | |
226 | ||
227 | #define SSE_HELPER_B(name, F) \ | |
228 | SSE_HELPER_2(name, B, 8 << SHIFT, F) | |
229 | ||
230 | #define SSE_HELPER_W(name, F) \ | |
231 | SSE_HELPER_2(name, W, 4 << SHIFT, F) | |
232 | ||
233 | #define SSE_HELPER_L(name, F) \ | |
234 | SSE_HELPER_2(name, L, 2 << SHIFT, F) | |
235 | ||
236 | #define SSE_HELPER_Q(name, F) \ | |
237 | SSE_HELPER_2(name, Q, 1 << SHIFT, F) | |
238 | ||
239 | #if SHIFT == 0 | |
240 | static inline int satub(int x) | |
241 | { | |
242 | if (x < 0) { | |
243 | return 0; | |
244 | } else if (x > 255) { | |
245 | return 255; | |
246 | } else { | |
247 | return x; | |
248 | } | |
249 | } | |
250 | ||
251 | static inline int satuw(int x) | |
252 | { | |
253 | if (x < 0) { | |
254 | return 0; | |
255 | } else if (x > 65535) { | |
256 | return 65535; | |
257 | } else { | |
258 | return x; | |
259 | } | |
260 | } | |
261 | ||
262 | static inline int satsb(int x) | |
263 | { | |
264 | if (x < -128) { | |
265 | return -128; | |
266 | } else if (x > 127) { | |
267 | return 127; | |
268 | } else { | |
269 | return x; | |
270 | } | |
271 | } | |
272 | ||
273 | static inline int satsw(int x) | |
274 | { | |
275 | if (x < -32768) { | |
276 | return -32768; | |
277 | } else if (x > 32767) { | |
278 | return 32767; | |
279 | } else { | |
280 | return x; | |
281 | } | |
282 | } | |
283 | ||
284 | #define FADD(a, b) ((a) + (b)) | |
285 | #define FADDUB(a, b) satub((a) + (b)) | |
286 | #define FADDUW(a, b) satuw((a) + (b)) | |
287 | #define FADDSB(a, b) satsb((int8_t)(a) + (int8_t)(b)) | |
288 | #define FADDSW(a, b) satsw((int16_t)(a) + (int16_t)(b)) | |
289 | ||
290 | #define FSUB(a, b) ((a) - (b)) | |
291 | #define FSUBUB(a, b) satub((a) - (b)) | |
292 | #define FSUBUW(a, b) satuw((a) - (b)) | |
293 | #define FSUBSB(a, b) satsb((int8_t)(a) - (int8_t)(b)) | |
294 | #define FSUBSW(a, b) satsw((int16_t)(a) - (int16_t)(b)) | |
295 | #define FMINUB(a, b) ((a) < (b)) ? (a) : (b) | |
296 | #define FMINSW(a, b) ((int16_t)(a) < (int16_t)(b)) ? (a) : (b) | |
297 | #define FMAXUB(a, b) ((a) > (b)) ? (a) : (b) | |
298 | #define FMAXSW(a, b) ((int16_t)(a) > (int16_t)(b)) ? (a) : (b) | |
299 | ||
300 | #define FMULHRW(a, b) (((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16) | |
301 | #define FMULHUW(a, b) ((a) * (b) >> 16) | |
302 | #define FMULHW(a, b) ((int16_t)(a) * (int16_t)(b) >> 16) | |
303 | ||
304 | #define FAVG(a, b) (((a) + (b) + 1) >> 1) | |
305 | #endif | |
306 | ||
307 | SSE_HELPER_W(helper_pmulhuw, FMULHUW) | |
308 | SSE_HELPER_W(helper_pmulhw, FMULHW) | |
309 | ||
310 | #if SHIFT == 0 | |
311 | void glue(helper_pmulhrw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
312 | { | |
313 | d->W(0) = FMULHRW(d->W(0), s->W(0)); | |
314 | d->W(1) = FMULHRW(d->W(1), s->W(1)); | |
315 | d->W(2) = FMULHRW(d->W(2), s->W(2)); | |
316 | d->W(3) = FMULHRW(d->W(3), s->W(3)); | |
317 | } | |
318 | #endif | |
319 | ||
320 | SSE_HELPER_B(helper_pavgb, FAVG) | |
321 | SSE_HELPER_W(helper_pavgw, FAVG) | |
322 | ||
323 | void glue(helper_pmuludq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
324 | { | |
325 | int i; | |
326 | ||
327 | for (i = 0; i < (1 << SHIFT); i++) { | |
328 | d->Q(i) = (uint64_t)s->L(i * 2) * (uint64_t)v->L(i * 2); | |
329 | } | |
330 | } | |
331 | ||
332 | void glue(helper_pmaddwd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
333 | { | |
334 | int i; | |
335 | ||
336 | for (i = 0; i < (2 << SHIFT); i++) { | |
337 | d->L(i) = (int16_t)s->W(2 * i) * (int16_t)v->W(2 * i) + | |
338 | (int16_t)s->W(2 * i + 1) * (int16_t)v->W(2 * i + 1); | |
339 | } | |
340 | } | |
341 | ||
342 | #if SHIFT == 0 | |
343 | static inline int abs1(int a) | |
344 | { | |
345 | if (a < 0) { | |
346 | return -a; | |
347 | } else { | |
348 | return a; | |
349 | } | |
350 | } | |
351 | #endif | |
352 | void glue(helper_psadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
353 | { | |
354 | int i; | |
355 | ||
356 | for (i = 0; i < (1 << SHIFT); i++) { | |
357 | unsigned int val = 0; | |
358 | val += abs1(v->B(8 * i + 0) - s->B(8 * i + 0)); | |
359 | val += abs1(v->B(8 * i + 1) - s->B(8 * i + 1)); | |
360 | val += abs1(v->B(8 * i + 2) - s->B(8 * i + 2)); | |
361 | val += abs1(v->B(8 * i + 3) - s->B(8 * i + 3)); | |
362 | val += abs1(v->B(8 * i + 4) - s->B(8 * i + 4)); | |
363 | val += abs1(v->B(8 * i + 5) - s->B(8 * i + 5)); | |
364 | val += abs1(v->B(8 * i + 6) - s->B(8 * i + 6)); | |
365 | val += abs1(v->B(8 * i + 7) - s->B(8 * i + 7)); | |
366 | d->Q(i) = val; | |
367 | } | |
368 | } | |
369 | ||
370 | #if SHIFT < 2 | |
371 | void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
372 | target_ulong a0) | |
373 | { | |
374 | int i; | |
375 | ||
376 | for (i = 0; i < (8 << SHIFT); i++) { | |
377 | if (s->B(i) & 0x80) { | |
378 | cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC()); | |
379 | } | |
380 | } | |
381 | } | |
382 | #endif | |
383 | ||
384 | #define SHUFFLE4(F, a, b, offset) do { \ | |
385 | r0 = a->F((order & 3) + offset); \ | |
386 | r1 = a->F(((order >> 2) & 3) + offset); \ | |
387 | r2 = b->F(((order >> 4) & 3) + offset); \ | |
388 | r3 = b->F(((order >> 6) & 3) + offset); \ | |
389 | d->F(offset) = r0; \ | |
390 | d->F(offset + 1) = r1; \ | |
391 | d->F(offset + 2) = r2; \ | |
392 | d->F(offset + 3) = r3; \ | |
393 | } while (0) | |
394 | ||
395 | #if SHIFT == 0 | |
396 | void glue(helper_pshufw, SUFFIX)(Reg *d, Reg *s, int order) | |
397 | { | |
398 | uint16_t r0, r1, r2, r3; | |
399 | ||
400 | SHUFFLE4(W, s, s, 0); | |
401 | } | |
402 | #else | |
403 | void glue(helper_shufps, SUFFIX)(Reg *d, Reg *v, Reg *s, int order) | |
404 | { | |
405 | uint32_t r0, r1, r2, r3; | |
406 | int i; | |
407 | ||
408 | for (i = 0; i < 2 << SHIFT; i += 4) { | |
409 | SHUFFLE4(L, v, s, i); | |
410 | } | |
411 | } | |
412 | ||
413 | void glue(helper_shufpd, SUFFIX)(Reg *d, Reg *v, Reg *s, int order) | |
414 | { | |
415 | uint64_t r0, r1; | |
416 | int i; | |
417 | ||
418 | for (i = 0; i < 1 << SHIFT; i += 2) { | |
419 | r0 = v->Q(((order & 1) & 1) + i); | |
420 | r1 = s->Q(((order >> 1) & 1) + i); | |
421 | d->Q(i) = r0; | |
422 | d->Q(i + 1) = r1; | |
423 | order >>= 2; | |
424 | } | |
425 | } | |
426 | ||
427 | void glue(helper_pshufd, SUFFIX)(Reg *d, Reg *s, int order) | |
428 | { | |
429 | uint32_t r0, r1, r2, r3; | |
430 | int i; | |
431 | ||
432 | for (i = 0; i < 2 << SHIFT; i += 4) { | |
433 | SHUFFLE4(L, s, s, i); | |
434 | } | |
435 | } | |
436 | ||
437 | void glue(helper_pshuflw, SUFFIX)(Reg *d, Reg *s, int order) | |
438 | { | |
439 | uint16_t r0, r1, r2, r3; | |
440 | int i, j; | |
441 | ||
442 | for (i = 0, j = 1; j < 1 << SHIFT; i += 8, j += 2) { | |
443 | SHUFFLE4(W, s, s, i); | |
444 | d->Q(j) = s->Q(j); | |
445 | } | |
446 | } | |
447 | ||
448 | void glue(helper_pshufhw, SUFFIX)(Reg *d, Reg *s, int order) | |
449 | { | |
450 | uint16_t r0, r1, r2, r3; | |
451 | int i, j; | |
452 | ||
453 | for (i = 4, j = 0; j < 1 << SHIFT; i += 8, j += 2) { | |
454 | d->Q(j) = s->Q(j); | |
455 | SHUFFLE4(W, s, s, i); | |
456 | } | |
457 | } | |
458 | #endif | |
459 | ||
460 | #if SHIFT >= 1 | |
461 | /* FPU ops */ | |
462 | /* XXX: not accurate */ | |
463 | ||
464 | #define SSE_HELPER_P(name, F) \ | |
465 | void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env, \ | |
466 | Reg *d, Reg *v, Reg *s) \ | |
467 | { \ | |
468 | int i; \ | |
469 | for (i = 0; i < 2 << SHIFT; i++) { \ | |
470 | d->ZMM_S(i) = F(32, v->ZMM_S(i), s->ZMM_S(i)); \ | |
471 | } \ | |
472 | } \ | |
473 | \ | |
474 | void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env, \ | |
475 | Reg *d, Reg *v, Reg *s) \ | |
476 | { \ | |
477 | int i; \ | |
478 | for (i = 0; i < 1 << SHIFT; i++) { \ | |
479 | d->ZMM_D(i) = F(64, v->ZMM_D(i), s->ZMM_D(i)); \ | |
480 | } \ | |
481 | } | |
482 | ||
483 | #if SHIFT == 1 | |
484 | ||
485 | #define SSE_HELPER_S(name, F) \ | |
486 | SSE_HELPER_P(name, F) \ | |
487 | \ | |
488 | void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s)\ | |
489 | { \ | |
490 | int i; \ | |
491 | d->ZMM_S(0) = F(32, v->ZMM_S(0), s->ZMM_S(0)); \ | |
492 | for (i = 1; i < 2 << SHIFT; i++) { \ | |
493 | d->ZMM_L(i) = v->ZMM_L(i); \ | |
494 | } \ | |
495 | } \ | |
496 | \ | |
497 | void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s)\ | |
498 | { \ | |
499 | int i; \ | |
500 | d->ZMM_D(0) = F(64, v->ZMM_D(0), s->ZMM_D(0)); \ | |
501 | for (i = 1; i < 1 << SHIFT; i++) { \ | |
502 | d->ZMM_Q(i) = v->ZMM_Q(i); \ | |
503 | } \ | |
504 | } | |
505 | ||
506 | #else | |
507 | ||
508 | #define SSE_HELPER_S(name, F) SSE_HELPER_P(name, F) | |
509 | ||
510 | #endif | |
511 | ||
512 | #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status) | |
513 | #define FPU_SUB(size, a, b) float ## size ## _sub(a, b, &env->sse_status) | |
514 | #define FPU_MUL(size, a, b) float ## size ## _mul(a, b, &env->sse_status) | |
515 | #define FPU_DIV(size, a, b) float ## size ## _div(a, b, &env->sse_status) | |
516 | ||
517 | /* Note that the choice of comparison op here is important to get the | |
518 | * special cases right: for min and max Intel specifies that (-0,0), | |
519 | * (NaN, anything) and (anything, NaN) return the second argument. | |
520 | */ | |
521 | #define FPU_MIN(size, a, b) \ | |
522 | (float ## size ## _lt(a, b, &env->sse_status) ? (a) : (b)) | |
523 | #define FPU_MAX(size, a, b) \ | |
524 | (float ## size ## _lt(b, a, &env->sse_status) ? (a) : (b)) | |
525 | ||
526 | SSE_HELPER_S(add, FPU_ADD) | |
527 | SSE_HELPER_S(sub, FPU_SUB) | |
528 | SSE_HELPER_S(mul, FPU_MUL) | |
529 | SSE_HELPER_S(div, FPU_DIV) | |
530 | SSE_HELPER_S(min, FPU_MIN) | |
531 | SSE_HELPER_S(max, FPU_MAX) | |
532 | ||
533 | void glue(helper_sqrtps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
534 | { | |
535 | int i; | |
536 | for (i = 0; i < 2 << SHIFT; i++) { | |
537 | d->ZMM_S(i) = float32_sqrt(s->ZMM_S(i), &env->sse_status); | |
538 | } | |
539 | } | |
540 | ||
541 | void glue(helper_sqrtpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
542 | { | |
543 | int i; | |
544 | for (i = 0; i < 1 << SHIFT; i++) { | |
545 | d->ZMM_D(i) = float64_sqrt(s->ZMM_D(i), &env->sse_status); | |
546 | } | |
547 | } | |
548 | ||
549 | #if SHIFT == 1 | |
550 | void helper_sqrtss(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
551 | { | |
552 | int i; | |
553 | d->ZMM_S(0) = float32_sqrt(s->ZMM_S(0), &env->sse_status); | |
554 | for (i = 1; i < 2 << SHIFT; i++) { | |
555 | d->ZMM_L(i) = v->ZMM_L(i); | |
556 | } | |
557 | } | |
558 | ||
559 | void helper_sqrtsd(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
560 | { | |
561 | int i; | |
562 | d->ZMM_D(0) = float64_sqrt(s->ZMM_D(0), &env->sse_status); | |
563 | for (i = 1; i < 1 << SHIFT; i++) { | |
564 | d->ZMM_Q(i) = v->ZMM_Q(i); | |
565 | } | |
566 | } | |
567 | #endif | |
568 | ||
569 | /* float to float conversions */ | |
570 | void glue(helper_cvtps2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
571 | { | |
572 | int i; | |
573 | for (i = 1 << SHIFT; --i >= 0; ) { | |
574 | d->ZMM_D(i) = float32_to_float64(s->ZMM_S(i), &env->sse_status); | |
575 | } | |
576 | } | |
577 | ||
578 | void glue(helper_cvtpd2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
579 | { | |
580 | int i; | |
581 | for (i = 0; i < 1 << SHIFT; i++) { | |
582 | d->ZMM_S(i) = float64_to_float32(s->ZMM_D(i), &env->sse_status); | |
583 | } | |
584 | for (i >>= 1; i < 1 << SHIFT; i++) { | |
585 | d->Q(i) = 0; | |
586 | } | |
587 | } | |
588 | ||
589 | #if SHIFT >= 1 | |
590 | void glue(helper_cvtph2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
591 | { | |
592 | int i; | |
593 | ||
594 | for (i = 2 << SHIFT; --i >= 0; ) { | |
595 | d->ZMM_S(i) = float16_to_float32(s->ZMM_H(i), true, &env->sse_status); | |
596 | } | |
597 | } | |
598 | ||
599 | void glue(helper_cvtps2ph, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, int mode) | |
600 | { | |
601 | int i; | |
602 | FloatRoundMode prev_rounding_mode = env->sse_status.float_rounding_mode; | |
603 | if (!(mode & (1 << 2))) { | |
604 | set_x86_rounding_mode(mode & 3, &env->sse_status); | |
605 | } | |
606 | ||
607 | for (i = 0; i < 2 << SHIFT; i++) { | |
608 | d->ZMM_H(i) = float32_to_float16(s->ZMM_S(i), true, &env->sse_status); | |
609 | } | |
610 | for (i >>= 2; i < 1 << SHIFT; i++) { | |
611 | d->Q(i) = 0; | |
612 | } | |
613 | ||
614 | env->sse_status.float_rounding_mode = prev_rounding_mode; | |
615 | } | |
616 | #endif | |
617 | ||
618 | #if SHIFT == 1 | |
619 | void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
620 | { | |
621 | int i; | |
622 | d->ZMM_D(0) = float32_to_float64(s->ZMM_S(0), &env->sse_status); | |
623 | for (i = 1; i < 1 << SHIFT; i++) { | |
624 | d->ZMM_Q(i) = v->ZMM_Q(i); | |
625 | } | |
626 | } | |
627 | ||
628 | void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
629 | { | |
630 | int i; | |
631 | d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status); | |
632 | for (i = 1; i < 2 << SHIFT; i++) { | |
633 | d->ZMM_L(i) = v->ZMM_L(i); | |
634 | } | |
635 | } | |
636 | #endif | |
637 | ||
638 | /* integer to float */ | |
639 | void glue(helper_cvtdq2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
640 | { | |
641 | int i; | |
642 | for (i = 0; i < 2 << SHIFT; i++) { | |
643 | d->ZMM_S(i) = int32_to_float32(s->ZMM_L(i), &env->sse_status); | |
644 | } | |
645 | } | |
646 | ||
647 | void glue(helper_cvtdq2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
648 | { | |
649 | int i; | |
650 | for (i = 1 << SHIFT; --i >= 0; ) { | |
651 | int32_t l = s->ZMM_L(i); | |
652 | d->ZMM_D(i) = int32_to_float64(l, &env->sse_status); | |
653 | } | |
654 | } | |
655 | ||
656 | #if SHIFT == 1 | |
657 | void helper_cvtpi2ps(CPUX86State *env, ZMMReg *d, MMXReg *s) | |
658 | { | |
659 | d->ZMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status); | |
660 | d->ZMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status); | |
661 | } | |
662 | ||
663 | void helper_cvtpi2pd(CPUX86State *env, ZMMReg *d, MMXReg *s) | |
664 | { | |
665 | d->ZMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status); | |
666 | d->ZMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status); | |
667 | } | |
668 | ||
669 | void helper_cvtsi2ss(CPUX86State *env, ZMMReg *d, uint32_t val) | |
670 | { | |
671 | d->ZMM_S(0) = int32_to_float32(val, &env->sse_status); | |
672 | } | |
673 | ||
674 | void helper_cvtsi2sd(CPUX86State *env, ZMMReg *d, uint32_t val) | |
675 | { | |
676 | d->ZMM_D(0) = int32_to_float64(val, &env->sse_status); | |
677 | } | |
678 | ||
679 | #ifdef TARGET_X86_64 | |
680 | void helper_cvtsq2ss(CPUX86State *env, ZMMReg *d, uint64_t val) | |
681 | { | |
682 | d->ZMM_S(0) = int64_to_float32(val, &env->sse_status); | |
683 | } | |
684 | ||
685 | void helper_cvtsq2sd(CPUX86State *env, ZMMReg *d, uint64_t val) | |
686 | { | |
687 | d->ZMM_D(0) = int64_to_float64(val, &env->sse_status); | |
688 | } | |
689 | #endif | |
690 | ||
691 | #endif | |
692 | ||
693 | /* float to integer */ | |
694 | ||
695 | #if SHIFT == 1 | |
696 | /* | |
697 | * x86 mandates that we return the indefinite integer value for the result | |
698 | * of any float-to-integer conversion that raises the 'invalid' exception. | |
699 | * Wrap the softfloat functions to get this behaviour. | |
700 | */ | |
701 | #define WRAP_FLOATCONV(RETTYPE, FN, FLOATTYPE, INDEFVALUE) \ | |
702 | static inline RETTYPE x86_##FN(FLOATTYPE a, float_status *s) \ | |
703 | { \ | |
704 | int oldflags, newflags; \ | |
705 | RETTYPE r; \ | |
706 | \ | |
707 | oldflags = get_float_exception_flags(s); \ | |
708 | set_float_exception_flags(0, s); \ | |
709 | r = FN(a, s); \ | |
710 | newflags = get_float_exception_flags(s); \ | |
711 | if (newflags & float_flag_invalid) { \ | |
712 | r = INDEFVALUE; \ | |
713 | } \ | |
714 | set_float_exception_flags(newflags | oldflags, s); \ | |
715 | return r; \ | |
716 | } | |
717 | ||
718 | WRAP_FLOATCONV(int32_t, float32_to_int32, float32, INT32_MIN) | |
719 | WRAP_FLOATCONV(int32_t, float32_to_int32_round_to_zero, float32, INT32_MIN) | |
720 | WRAP_FLOATCONV(int32_t, float64_to_int32, float64, INT32_MIN) | |
721 | WRAP_FLOATCONV(int32_t, float64_to_int32_round_to_zero, float64, INT32_MIN) | |
722 | WRAP_FLOATCONV(int64_t, float32_to_int64, float32, INT64_MIN) | |
723 | WRAP_FLOATCONV(int64_t, float32_to_int64_round_to_zero, float32, INT64_MIN) | |
724 | WRAP_FLOATCONV(int64_t, float64_to_int64, float64, INT64_MIN) | |
725 | WRAP_FLOATCONV(int64_t, float64_to_int64_round_to_zero, float64, INT64_MIN) | |
726 | #endif | |
727 | ||
728 | void glue(helper_cvtps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
729 | { | |
730 | int i; | |
731 | for (i = 0; i < 2 << SHIFT; i++) { | |
732 | d->ZMM_L(i) = x86_float32_to_int32(s->ZMM_S(i), &env->sse_status); | |
733 | } | |
734 | } | |
735 | ||
736 | void glue(helper_cvtpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
737 | { | |
738 | int i; | |
739 | for (i = 0; i < 1 << SHIFT; i++) { | |
740 | d->ZMM_L(i) = x86_float64_to_int32(s->ZMM_D(i), &env->sse_status); | |
741 | } | |
742 | for (i >>= 1; i < 1 << SHIFT; i++) { | |
743 | d->Q(i) = 0; | |
744 | } | |
745 | } | |
746 | ||
747 | #if SHIFT == 1 | |
748 | void helper_cvtps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) | |
749 | { | |
750 | d->MMX_L(0) = x86_float32_to_int32(s->ZMM_S(0), &env->sse_status); | |
751 | d->MMX_L(1) = x86_float32_to_int32(s->ZMM_S(1), &env->sse_status); | |
752 | } | |
753 | ||
754 | void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) | |
755 | { | |
756 | d->MMX_L(0) = x86_float64_to_int32(s->ZMM_D(0), &env->sse_status); | |
757 | d->MMX_L(1) = x86_float64_to_int32(s->ZMM_D(1), &env->sse_status); | |
758 | } | |
759 | ||
760 | int32_t helper_cvtss2si(CPUX86State *env, ZMMReg *s) | |
761 | { | |
762 | return x86_float32_to_int32(s->ZMM_S(0), &env->sse_status); | |
763 | } | |
764 | ||
765 | int32_t helper_cvtsd2si(CPUX86State *env, ZMMReg *s) | |
766 | { | |
767 | return x86_float64_to_int32(s->ZMM_D(0), &env->sse_status); | |
768 | } | |
769 | ||
770 | #ifdef TARGET_X86_64 | |
771 | int64_t helper_cvtss2sq(CPUX86State *env, ZMMReg *s) | |
772 | { | |
773 | return x86_float32_to_int64(s->ZMM_S(0), &env->sse_status); | |
774 | } | |
775 | ||
776 | int64_t helper_cvtsd2sq(CPUX86State *env, ZMMReg *s) | |
777 | { | |
778 | return x86_float64_to_int64(s->ZMM_D(0), &env->sse_status); | |
779 | } | |
780 | #endif | |
781 | #endif | |
782 | ||
783 | /* float to integer truncated */ | |
784 | void glue(helper_cvttps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
785 | { | |
786 | int i; | |
787 | for (i = 0; i < 2 << SHIFT; i++) { | |
788 | d->ZMM_L(i) = x86_float32_to_int32_round_to_zero(s->ZMM_S(i), | |
789 | &env->sse_status); | |
790 | } | |
791 | } | |
792 | ||
793 | void glue(helper_cvttpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
794 | { | |
795 | int i; | |
796 | for (i = 0; i < 1 << SHIFT; i++) { | |
797 | d->ZMM_L(i) = x86_float64_to_int32_round_to_zero(s->ZMM_D(i), | |
798 | &env->sse_status); | |
799 | } | |
800 | for (i >>= 1; i < 1 << SHIFT; i++) { | |
801 | d->Q(i) = 0; | |
802 | } | |
803 | } | |
804 | ||
805 | #if SHIFT == 1 | |
806 | void helper_cvttps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) | |
807 | { | |
808 | d->MMX_L(0) = x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status); | |
809 | d->MMX_L(1) = x86_float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status); | |
810 | } | |
811 | ||
812 | void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) | |
813 | { | |
814 | d->MMX_L(0) = x86_float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status); | |
815 | d->MMX_L(1) = x86_float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status); | |
816 | } | |
817 | ||
818 | int32_t helper_cvttss2si(CPUX86State *env, ZMMReg *s) | |
819 | { | |
820 | return x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status); | |
821 | } | |
822 | ||
823 | int32_t helper_cvttsd2si(CPUX86State *env, ZMMReg *s) | |
824 | { | |
825 | return x86_float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status); | |
826 | } | |
827 | ||
828 | #ifdef TARGET_X86_64 | |
829 | int64_t helper_cvttss2sq(CPUX86State *env, ZMMReg *s) | |
830 | { | |
831 | return x86_float32_to_int64_round_to_zero(s->ZMM_S(0), &env->sse_status); | |
832 | } | |
833 | ||
834 | int64_t helper_cvttsd2sq(CPUX86State *env, ZMMReg *s) | |
835 | { | |
836 | return x86_float64_to_int64_round_to_zero(s->ZMM_D(0), &env->sse_status); | |
837 | } | |
838 | #endif | |
839 | #endif | |
840 | ||
841 | void glue(helper_rsqrtps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
842 | { | |
843 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
844 | int i; | |
845 | for (i = 0; i < 2 << SHIFT; i++) { | |
846 | d->ZMM_S(i) = float32_div(float32_one, | |
847 | float32_sqrt(s->ZMM_S(i), &env->sse_status), | |
848 | &env->sse_status); | |
849 | } | |
850 | set_float_exception_flags(old_flags, &env->sse_status); | |
851 | } | |
852 | ||
853 | #if SHIFT == 1 | |
854 | void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s) | |
855 | { | |
856 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
857 | int i; | |
858 | d->ZMM_S(0) = float32_div(float32_one, | |
859 | float32_sqrt(s->ZMM_S(0), &env->sse_status), | |
860 | &env->sse_status); | |
861 | set_float_exception_flags(old_flags, &env->sse_status); | |
862 | for (i = 1; i < 2 << SHIFT; i++) { | |
863 | d->ZMM_L(i) = v->ZMM_L(i); | |
864 | } | |
865 | } | |
866 | #endif | |
867 | ||
868 | void glue(helper_rcpps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
869 | { | |
870 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
871 | int i; | |
872 | for (i = 0; i < 2 << SHIFT; i++) { | |
873 | d->ZMM_S(i) = float32_div(float32_one, s->ZMM_S(i), &env->sse_status); | |
874 | } | |
875 | set_float_exception_flags(old_flags, &env->sse_status); | |
876 | } | |
877 | ||
878 | #if SHIFT == 1 | |
879 | void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s) | |
880 | { | |
881 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
882 | int i; | |
883 | d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status); | |
884 | for (i = 1; i < 2 << SHIFT; i++) { | |
885 | d->ZMM_L(i) = v->ZMM_L(i); | |
886 | } | |
887 | set_float_exception_flags(old_flags, &env->sse_status); | |
888 | } | |
889 | #endif | |
890 | ||
891 | #if SHIFT == 1 | |
892 | static inline uint64_t helper_extrq(uint64_t src, int shift, int len) | |
893 | { | |
894 | uint64_t mask; | |
895 | ||
896 | if (len == 0) { | |
897 | mask = ~0LL; | |
898 | } else { | |
899 | mask = (1ULL << len) - 1; | |
900 | } | |
901 | return (src >> shift) & mask; | |
902 | } | |
903 | ||
904 | void helper_extrq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
905 | { | |
906 | d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1) & 63, s->ZMM_B(0) & 63); | |
907 | } | |
908 | ||
909 | void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length) | |
910 | { | |
911 | d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), index, length); | |
912 | } | |
913 | ||
914 | static inline uint64_t helper_insertq(uint64_t dest, uint64_t src, int shift, int len) | |
915 | { | |
916 | uint64_t mask; | |
917 | ||
918 | if (len == 0) { | |
919 | mask = ~0ULL; | |
920 | } else { | |
921 | mask = (1ULL << len) - 1; | |
922 | } | |
923 | return (dest & ~(mask << shift)) | ((src & mask) << shift); | |
924 | } | |
925 | ||
926 | void helper_insertq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s) | |
927 | { | |
928 | d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), s->ZMM_B(9) & 63, s->ZMM_B(8) & 63); | |
929 | } | |
930 | ||
931 | void helper_insertq_i(CPUX86State *env, ZMMReg *d, ZMMReg *s, int index, int length) | |
932 | { | |
933 | d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), index, length); | |
934 | } | |
935 | #endif | |
936 | ||
937 | #define SSE_HELPER_HPS(name, F) \ | |
938 | void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
939 | { \ | |
940 | float32 r[2 << SHIFT]; \ | |
941 | int i, j, k; \ | |
942 | for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) { \ | |
943 | for (i = j = 0; j < 4; i++, j += 2) { \ | |
944 | r[i + k] = F(v->ZMM_S(j + k), v->ZMM_S(j + k + 1), &env->sse_status); \ | |
945 | } \ | |
946 | for (j = 0; j < 4; i++, j += 2) { \ | |
947 | r[i + k] = F(s->ZMM_S(j + k), s->ZMM_S(j + k + 1), &env->sse_status); \ | |
948 | } \ | |
949 | } \ | |
950 | for (i = 0; i < 2 << SHIFT; i++) { \ | |
951 | d->ZMM_S(i) = r[i]; \ | |
952 | } \ | |
953 | } | |
954 | ||
955 | SSE_HELPER_HPS(haddps, float32_add) | |
956 | SSE_HELPER_HPS(hsubps, float32_sub) | |
957 | ||
958 | #define SSE_HELPER_HPD(name, F) \ | |
959 | void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
960 | { \ | |
961 | float64 r[1 << SHIFT]; \ | |
962 | int i, j, k; \ | |
963 | for (k = 0; k < 1 << SHIFT; k += LANE_WIDTH / 8) { \ | |
964 | for (i = j = 0; j < 2; i++, j += 2) { \ | |
965 | r[i + k] = F(v->ZMM_D(j + k), v->ZMM_D(j + k + 1), &env->sse_status); \ | |
966 | } \ | |
967 | for (j = 0; j < 2; i++, j += 2) { \ | |
968 | r[i + k] = F(s->ZMM_D(j + k), s->ZMM_D(j + k + 1), &env->sse_status); \ | |
969 | } \ | |
970 | } \ | |
971 | for (i = 0; i < 1 << SHIFT; i++) { \ | |
972 | d->ZMM_D(i) = r[i]; \ | |
973 | } \ | |
974 | } | |
975 | ||
976 | SSE_HELPER_HPD(haddpd, float64_add) | |
977 | SSE_HELPER_HPD(hsubpd, float64_sub) | |
978 | ||
979 | void glue(helper_addsubps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
980 | { | |
981 | int i; | |
982 | for (i = 0; i < 2 << SHIFT; i += 2) { | |
983 | d->ZMM_S(i) = float32_sub(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status); | |
984 | d->ZMM_S(i+1) = float32_add(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status); | |
985 | } | |
986 | } | |
987 | ||
988 | void glue(helper_addsubpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
989 | { | |
990 | int i; | |
991 | for (i = 0; i < 1 << SHIFT; i += 2) { | |
992 | d->ZMM_D(i) = float64_sub(v->ZMM_D(i), s->ZMM_D(i), &env->sse_status); | |
993 | d->ZMM_D(i+1) = float64_add(v->ZMM_D(i+1), s->ZMM_D(i+1), &env->sse_status); | |
994 | } | |
995 | } | |
996 | ||
997 | #define SSE_HELPER_CMP_P(name, F, C) \ | |
998 | void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env, \ | |
999 | Reg *d, Reg *v, Reg *s) \ | |
1000 | { \ | |
1001 | int i; \ | |
1002 | for (i = 0; i < 2 << SHIFT; i++) { \ | |
1003 | d->ZMM_L(i) = C(F(32, v->ZMM_S(i), s->ZMM_S(i))) ? -1 : 0; \ | |
1004 | } \ | |
1005 | } \ | |
1006 | \ | |
1007 | void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env, \ | |
1008 | Reg *d, Reg *v, Reg *s) \ | |
1009 | { \ | |
1010 | int i; \ | |
1011 | for (i = 0; i < 1 << SHIFT; i++) { \ | |
1012 | d->ZMM_Q(i) = C(F(64, v->ZMM_D(i), s->ZMM_D(i))) ? -1 : 0; \ | |
1013 | } \ | |
1014 | } | |
1015 | ||
1016 | #if SHIFT == 1 | |
1017 | #define SSE_HELPER_CMP(name, F, C) \ | |
1018 | SSE_HELPER_CMP_P(name, F, C) \ | |
1019 | void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
1020 | { \ | |
1021 | int i; \ | |
1022 | d->ZMM_L(0) = C(F(32, v->ZMM_S(0), s->ZMM_S(0))) ? -1 : 0; \ | |
1023 | for (i = 1; i < 2 << SHIFT; i++) { \ | |
1024 | d->ZMM_L(i) = v->ZMM_L(i); \ | |
1025 | } \ | |
1026 | } \ | |
1027 | \ | |
1028 | void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
1029 | { \ | |
1030 | int i; \ | |
1031 | d->ZMM_Q(0) = C(F(64, v->ZMM_D(0), s->ZMM_D(0))) ? -1 : 0; \ | |
1032 | for (i = 1; i < 1 << SHIFT; i++) { \ | |
1033 | d->ZMM_Q(i) = v->ZMM_Q(i); \ | |
1034 | } \ | |
1035 | } | |
1036 | ||
1037 | static inline bool FPU_EQU(FloatRelation x) | |
1038 | { | |
1039 | return (x == float_relation_equal || x == float_relation_unordered); | |
1040 | } | |
1041 | static inline bool FPU_GE(FloatRelation x) | |
1042 | { | |
1043 | return (x == float_relation_equal || x == float_relation_greater); | |
1044 | } | |
1045 | #define FPU_EQ(x) (x == float_relation_equal) | |
1046 | #define FPU_LT(x) (x == float_relation_less) | |
1047 | #define FPU_LE(x) (x <= float_relation_equal) | |
1048 | #define FPU_GT(x) (x == float_relation_greater) | |
1049 | #define FPU_UNORD(x) (x == float_relation_unordered) | |
1050 | /* We must make sure we evaluate the argument in case it is a signalling NAN */ | |
1051 | #define FPU_FALSE(x) (x == float_relation_equal && 0) | |
1052 | ||
1053 | #define FPU_CMPQ(size, a, b) \ | |
1054 | float ## size ## _compare_quiet(a, b, &env->sse_status) | |
1055 | #define FPU_CMPS(size, a, b) \ | |
1056 | float ## size ## _compare(a, b, &env->sse_status) | |
1057 | ||
1058 | #else | |
1059 | #define SSE_HELPER_CMP(name, F, C) SSE_HELPER_CMP_P(name, F, C) | |
1060 | #endif | |
1061 | ||
1062 | SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ) | |
1063 | SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT) | |
1064 | SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE) | |
1065 | SSE_HELPER_CMP(cmpunord, FPU_CMPQ, FPU_UNORD) | |
1066 | SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ) | |
1067 | SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT) | |
1068 | SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE) | |
1069 | SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD) | |
1070 | ||
1071 | SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU) | |
1072 | SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE) | |
1073 | SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT) | |
1074 | SSE_HELPER_CMP(cmpfalse, FPU_CMPQ, FPU_FALSE) | |
1075 | SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU) | |
1076 | SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE) | |
1077 | SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT) | |
1078 | SSE_HELPER_CMP(cmptrue, FPU_CMPQ, !FPU_FALSE) | |
1079 | ||
1080 | SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ) | |
1081 | SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT) | |
1082 | SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE) | |
1083 | SSE_HELPER_CMP(cmpunords, FPU_CMPS, FPU_UNORD) | |
1084 | SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ) | |
1085 | SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT) | |
1086 | SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE) | |
1087 | SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD) | |
1088 | ||
1089 | SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU) | |
1090 | SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE) | |
1091 | SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT) | |
1092 | SSE_HELPER_CMP(cmpfalses, FPU_CMPS, FPU_FALSE) | |
1093 | SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU) | |
1094 | SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE) | |
1095 | SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT) | |
1096 | SSE_HELPER_CMP(cmptrues, FPU_CMPS, !FPU_FALSE) | |
1097 | ||
1098 | #undef SSE_HELPER_CMP | |
1099 | ||
1100 | #if SHIFT == 1 | |
1101 | static const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C}; | |
1102 | ||
1103 | void helper_ucomiss(CPUX86State *env, Reg *d, Reg *s) | |
1104 | { | |
1105 | FloatRelation ret; | |
1106 | float32 s0, s1; | |
1107 | ||
1108 | s0 = d->ZMM_S(0); | |
1109 | s1 = s->ZMM_S(0); | |
1110 | ret = float32_compare_quiet(s0, s1, &env->sse_status); | |
1111 | CC_SRC = comis_eflags[ret + 1]; | |
1112 | } | |
1113 | ||
1114 | void helper_comiss(CPUX86State *env, Reg *d, Reg *s) | |
1115 | { | |
1116 | FloatRelation ret; | |
1117 | float32 s0, s1; | |
1118 | ||
1119 | s0 = d->ZMM_S(0); | |
1120 | s1 = s->ZMM_S(0); | |
1121 | ret = float32_compare(s0, s1, &env->sse_status); | |
1122 | CC_SRC = comis_eflags[ret + 1]; | |
1123 | } | |
1124 | ||
1125 | void helper_ucomisd(CPUX86State *env, Reg *d, Reg *s) | |
1126 | { | |
1127 | FloatRelation ret; | |
1128 | float64 d0, d1; | |
1129 | ||
1130 | d0 = d->ZMM_D(0); | |
1131 | d1 = s->ZMM_D(0); | |
1132 | ret = float64_compare_quiet(d0, d1, &env->sse_status); | |
1133 | CC_SRC = comis_eflags[ret + 1]; | |
1134 | } | |
1135 | ||
1136 | void helper_comisd(CPUX86State *env, Reg *d, Reg *s) | |
1137 | { | |
1138 | FloatRelation ret; | |
1139 | float64 d0, d1; | |
1140 | ||
1141 | d0 = d->ZMM_D(0); | |
1142 | d1 = s->ZMM_D(0); | |
1143 | ret = float64_compare(d0, d1, &env->sse_status); | |
1144 | CC_SRC = comis_eflags[ret + 1]; | |
1145 | } | |
1146 | #endif | |
1147 | ||
1148 | uint32_t glue(helper_movmskps, SUFFIX)(CPUX86State *env, Reg *s) | |
1149 | { | |
1150 | uint32_t mask; | |
1151 | int i; | |
1152 | ||
1153 | mask = 0; | |
1154 | for (i = 0; i < 2 << SHIFT; i++) { | |
1155 | mask |= (s->ZMM_L(i) >> (31 - i)) & (1 << i); | |
1156 | } | |
1157 | return mask; | |
1158 | } | |
1159 | ||
1160 | uint32_t glue(helper_movmskpd, SUFFIX)(CPUX86State *env, Reg *s) | |
1161 | { | |
1162 | uint32_t mask; | |
1163 | int i; | |
1164 | ||
1165 | mask = 0; | |
1166 | for (i = 0; i < 1 << SHIFT; i++) { | |
1167 | mask |= (s->ZMM_Q(i) >> (63 - i)) & (1 << i); | |
1168 | } | |
1169 | return mask; | |
1170 | } | |
1171 | ||
1172 | #endif | |
1173 | ||
1174 | #define PACK_HELPER_B(name, F) \ | |
1175 | void glue(helper_pack ## name, SUFFIX)(CPUX86State *env, \ | |
1176 | Reg *d, Reg *v, Reg *s) \ | |
1177 | { \ | |
1178 | uint8_t r[PACK_WIDTH * 2]; \ | |
1179 | int j, k; \ | |
1180 | for (j = 0; j < 4 << SHIFT; j += PACK_WIDTH) { \ | |
1181 | for (k = 0; k < PACK_WIDTH; k++) { \ | |
1182 | r[k] = F((int16_t)v->W(j + k)); \ | |
1183 | } \ | |
1184 | for (k = 0; k < PACK_WIDTH; k++) { \ | |
1185 | r[PACK_WIDTH + k] = F((int16_t)s->W(j + k)); \ | |
1186 | } \ | |
1187 | for (k = 0; k < PACK_WIDTH * 2; k++) { \ | |
1188 | d->B(2 * j + k) = r[k]; \ | |
1189 | } \ | |
1190 | } \ | |
1191 | } | |
1192 | ||
1193 | PACK_HELPER_B(sswb, satsb) | |
1194 | PACK_HELPER_B(uswb, satub) | |
1195 | ||
1196 | void glue(helper_packssdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
1197 | { | |
1198 | uint16_t r[PACK_WIDTH]; | |
1199 | int j, k; | |
1200 | ||
1201 | for (j = 0; j < 2 << SHIFT; j += PACK_WIDTH / 2) { | |
1202 | for (k = 0; k < PACK_WIDTH / 2; k++) { | |
1203 | r[k] = satsw(v->L(j + k)); | |
1204 | } | |
1205 | for (k = 0; k < PACK_WIDTH / 2; k++) { | |
1206 | r[PACK_WIDTH / 2 + k] = satsw(s->L(j + k)); | |
1207 | } | |
1208 | for (k = 0; k < PACK_WIDTH; k++) { | |
1209 | d->W(2 * j + k) = r[k]; | |
1210 | } | |
1211 | } | |
1212 | } | |
1213 | ||
1214 | #define UNPCK_OP(base_name, base) \ | |
1215 | \ | |
1216 | void glue(helper_punpck ## base_name ## bw, SUFFIX)(CPUX86State *env,\ | |
1217 | Reg *d, Reg *v, Reg *s) \ | |
1218 | { \ | |
1219 | uint8_t r[PACK_WIDTH * 2]; \ | |
1220 | int j, i; \ | |
1221 | \ | |
1222 | for (j = 0; j < 8 << SHIFT; ) { \ | |
1223 | int k = j + base * PACK_WIDTH; \ | |
1224 | for (i = 0; i < PACK_WIDTH; i++) { \ | |
1225 | r[2 * i] = v->B(k + i); \ | |
1226 | r[2 * i + 1] = s->B(k + i); \ | |
1227 | } \ | |
1228 | for (i = 0; i < PACK_WIDTH * 2; i++, j++) { \ | |
1229 | d->B(j) = r[i]; \ | |
1230 | } \ | |
1231 | } \ | |
1232 | } \ | |
1233 | \ | |
1234 | void glue(helper_punpck ## base_name ## wd, SUFFIX)(CPUX86State *env,\ | |
1235 | Reg *d, Reg *v, Reg *s) \ | |
1236 | { \ | |
1237 | uint16_t r[PACK_WIDTH]; \ | |
1238 | int j, i; \ | |
1239 | \ | |
1240 | for (j = 0; j < 4 << SHIFT; ) { \ | |
1241 | int k = j + base * PACK_WIDTH / 2; \ | |
1242 | for (i = 0; i < PACK_WIDTH / 2; i++) { \ | |
1243 | r[2 * i] = v->W(k + i); \ | |
1244 | r[2 * i + 1] = s->W(k + i); \ | |
1245 | } \ | |
1246 | for (i = 0; i < PACK_WIDTH; i++, j++) { \ | |
1247 | d->W(j) = r[i]; \ | |
1248 | } \ | |
1249 | } \ | |
1250 | } \ | |
1251 | \ | |
1252 | void glue(helper_punpck ## base_name ## dq, SUFFIX)(CPUX86State *env,\ | |
1253 | Reg *d, Reg *v, Reg *s) \ | |
1254 | { \ | |
1255 | uint32_t r[PACK_WIDTH / 2]; \ | |
1256 | int j, i; \ | |
1257 | \ | |
1258 | for (j = 0; j < 2 << SHIFT; ) { \ | |
1259 | int k = j + base * PACK_WIDTH / 4; \ | |
1260 | for (i = 0; i < PACK_WIDTH / 4; i++) { \ | |
1261 | r[2 * i] = v->L(k + i); \ | |
1262 | r[2 * i + 1] = s->L(k + i); \ | |
1263 | } \ | |
1264 | for (i = 0; i < PACK_WIDTH / 2; i++, j++) { \ | |
1265 | d->L(j) = r[i]; \ | |
1266 | } \ | |
1267 | } \ | |
1268 | } \ | |
1269 | \ | |
1270 | XMM_ONLY( \ | |
1271 | void glue(helper_punpck ## base_name ## qdq, SUFFIX)( \ | |
1272 | CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
1273 | { \ | |
1274 | uint64_t r[2]; \ | |
1275 | int i; \ | |
1276 | \ | |
1277 | for (i = 0; i < 1 << SHIFT; i += 2) { \ | |
1278 | r[0] = v->Q(base + i); \ | |
1279 | r[1] = s->Q(base + i); \ | |
1280 | d->Q(i) = r[0]; \ | |
1281 | d->Q(i + 1) = r[1]; \ | |
1282 | } \ | |
1283 | } \ | |
1284 | ) | |
1285 | ||
1286 | UNPCK_OP(l, 0) | |
1287 | UNPCK_OP(h, 1) | |
1288 | ||
1289 | #undef PACK_WIDTH | |
1290 | #undef PACK_HELPER_B | |
1291 | #undef UNPCK_OP | |
1292 | ||
1293 | ||
1294 | /* 3DNow! float ops */ | |
1295 | #if SHIFT == 0 | |
1296 | void helper_pi2fd(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1297 | { | |
1298 | d->MMX_S(0) = int32_to_float32(s->MMX_L(0), &env->mmx_status); | |
1299 | d->MMX_S(1) = int32_to_float32(s->MMX_L(1), &env->mmx_status); | |
1300 | } | |
1301 | ||
1302 | void helper_pi2fw(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1303 | { | |
1304 | d->MMX_S(0) = int32_to_float32((int16_t)s->MMX_W(0), &env->mmx_status); | |
1305 | d->MMX_S(1) = int32_to_float32((int16_t)s->MMX_W(2), &env->mmx_status); | |
1306 | } | |
1307 | ||
1308 | void helper_pf2id(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1309 | { | |
1310 | d->MMX_L(0) = float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status); | |
1311 | d->MMX_L(1) = float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status); | |
1312 | } | |
1313 | ||
1314 | void helper_pf2iw(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1315 | { | |
1316 | d->MMX_L(0) = satsw(float32_to_int32_round_to_zero(s->MMX_S(0), | |
1317 | &env->mmx_status)); | |
1318 | d->MMX_L(1) = satsw(float32_to_int32_round_to_zero(s->MMX_S(1), | |
1319 | &env->mmx_status)); | |
1320 | } | |
1321 | ||
1322 | void helper_pfacc(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1323 | { | |
1324 | float32 r; | |
1325 | ||
1326 | r = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); | |
1327 | d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); | |
1328 | d->MMX_S(0) = r; | |
1329 | } | |
1330 | ||
1331 | void helper_pfadd(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1332 | { | |
1333 | d->MMX_S(0) = float32_add(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); | |
1334 | d->MMX_S(1) = float32_add(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); | |
1335 | } | |
1336 | ||
1337 | void helper_pfcmpeq(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1338 | { | |
1339 | d->MMX_L(0) = float32_eq_quiet(d->MMX_S(0), s->MMX_S(0), | |
1340 | &env->mmx_status) ? -1 : 0; | |
1341 | d->MMX_L(1) = float32_eq_quiet(d->MMX_S(1), s->MMX_S(1), | |
1342 | &env->mmx_status) ? -1 : 0; | |
1343 | } | |
1344 | ||
1345 | void helper_pfcmpge(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1346 | { | |
1347 | d->MMX_L(0) = float32_le(s->MMX_S(0), d->MMX_S(0), | |
1348 | &env->mmx_status) ? -1 : 0; | |
1349 | d->MMX_L(1) = float32_le(s->MMX_S(1), d->MMX_S(1), | |
1350 | &env->mmx_status) ? -1 : 0; | |
1351 | } | |
1352 | ||
1353 | void helper_pfcmpgt(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1354 | { | |
1355 | d->MMX_L(0) = float32_lt(s->MMX_S(0), d->MMX_S(0), | |
1356 | &env->mmx_status) ? -1 : 0; | |
1357 | d->MMX_L(1) = float32_lt(s->MMX_S(1), d->MMX_S(1), | |
1358 | &env->mmx_status) ? -1 : 0; | |
1359 | } | |
1360 | ||
1361 | void helper_pfmax(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1362 | { | |
1363 | if (float32_lt(d->MMX_S(0), s->MMX_S(0), &env->mmx_status)) { | |
1364 | d->MMX_S(0) = s->MMX_S(0); | |
1365 | } | |
1366 | if (float32_lt(d->MMX_S(1), s->MMX_S(1), &env->mmx_status)) { | |
1367 | d->MMX_S(1) = s->MMX_S(1); | |
1368 | } | |
1369 | } | |
1370 | ||
1371 | void helper_pfmin(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1372 | { | |
1373 | if (float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status)) { | |
1374 | d->MMX_S(0) = s->MMX_S(0); | |
1375 | } | |
1376 | if (float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status)) { | |
1377 | d->MMX_S(1) = s->MMX_S(1); | |
1378 | } | |
1379 | } | |
1380 | ||
1381 | void helper_pfmul(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1382 | { | |
1383 | d->MMX_S(0) = float32_mul(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); | |
1384 | d->MMX_S(1) = float32_mul(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); | |
1385 | } | |
1386 | ||
1387 | void helper_pfnacc(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1388 | { | |
1389 | float32 r; | |
1390 | ||
1391 | r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); | |
1392 | d->MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); | |
1393 | d->MMX_S(0) = r; | |
1394 | } | |
1395 | ||
1396 | void helper_pfpnacc(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1397 | { | |
1398 | float32 r; | |
1399 | ||
1400 | r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); | |
1401 | d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); | |
1402 | d->MMX_S(0) = r; | |
1403 | } | |
1404 | ||
1405 | void helper_pfrcp(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1406 | { | |
1407 | d->MMX_S(0) = float32_div(float32_one, s->MMX_S(0), &env->mmx_status); | |
1408 | d->MMX_S(1) = d->MMX_S(0); | |
1409 | } | |
1410 | ||
1411 | void helper_pfrsqrt(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1412 | { | |
1413 | d->MMX_L(1) = s->MMX_L(0) & 0x7fffffff; | |
1414 | d->MMX_S(1) = float32_div(float32_one, | |
1415 | float32_sqrt(d->MMX_S(1), &env->mmx_status), | |
1416 | &env->mmx_status); | |
1417 | d->MMX_L(1) |= s->MMX_L(0) & 0x80000000; | |
1418 | d->MMX_L(0) = d->MMX_L(1); | |
1419 | } | |
1420 | ||
1421 | void helper_pfsub(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1422 | { | |
1423 | d->MMX_S(0) = float32_sub(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); | |
1424 | d->MMX_S(1) = float32_sub(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); | |
1425 | } | |
1426 | ||
1427 | void helper_pfsubr(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1428 | { | |
1429 | d->MMX_S(0) = float32_sub(s->MMX_S(0), d->MMX_S(0), &env->mmx_status); | |
1430 | d->MMX_S(1) = float32_sub(s->MMX_S(1), d->MMX_S(1), &env->mmx_status); | |
1431 | } | |
1432 | ||
1433 | void helper_pswapd(CPUX86State *env, MMXReg *d, MMXReg *s) | |
1434 | { | |
1435 | uint32_t r; | |
1436 | ||
1437 | r = s->MMX_L(0); | |
1438 | d->MMX_L(0) = s->MMX_L(1); | |
1439 | d->MMX_L(1) = r; | |
1440 | } | |
1441 | #endif | |
1442 | ||
1443 | /* SSSE3 op helpers */ | |
1444 | void glue(helper_pshufb, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
1445 | { | |
1446 | int i; | |
1447 | #if SHIFT == 0 | |
1448 | uint8_t r[8]; | |
1449 | ||
1450 | for (i = 0; i < 8; i++) { | |
1451 | r[i] = (s->B(i) & 0x80) ? 0 : (v->B(s->B(i) & 7)); | |
1452 | } | |
1453 | for (i = 0; i < 8; i++) { | |
1454 | d->B(i) = r[i]; | |
1455 | } | |
1456 | #else | |
1457 | uint8_t r[8 << SHIFT]; | |
1458 | ||
1459 | for (i = 0; i < 8 << SHIFT; i++) { | |
1460 | int j = i & ~0xf; | |
1461 | r[i] = (s->B(i) & 0x80) ? 0 : v->B(j | (s->B(i) & 0xf)); | |
1462 | } | |
1463 | for (i = 0; i < 8 << SHIFT; i++) { | |
1464 | d->B(i) = r[i]; | |
1465 | } | |
1466 | #endif | |
1467 | } | |
1468 | ||
1469 | #define SSE_HELPER_HW(name, F) \ | |
1470 | void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
1471 | { \ | |
1472 | uint16_t r[4 << SHIFT]; \ | |
1473 | int i, j, k; \ | |
1474 | for (k = 0; k < 4 << SHIFT; k += LANE_WIDTH / 2) { \ | |
1475 | for (i = j = 0; j < LANE_WIDTH / 2; i++, j += 2) { \ | |
1476 | r[i + k] = F(v->W(j + k), v->W(j + k + 1)); \ | |
1477 | } \ | |
1478 | for (j = 0; j < LANE_WIDTH / 2; i++, j += 2) { \ | |
1479 | r[i + k] = F(s->W(j + k), s->W(j + k + 1)); \ | |
1480 | } \ | |
1481 | } \ | |
1482 | for (i = 0; i < 4 << SHIFT; i++) { \ | |
1483 | d->W(i) = r[i]; \ | |
1484 | } \ | |
1485 | } | |
1486 | ||
1487 | #define SSE_HELPER_HL(name, F) \ | |
1488 | void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ | |
1489 | { \ | |
1490 | uint32_t r[2 << SHIFT]; \ | |
1491 | int i, j, k; \ | |
1492 | for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) { \ | |
1493 | for (i = j = 0; j < LANE_WIDTH / 4; i++, j += 2) { \ | |
1494 | r[i + k] = F(v->L(j + k), v->L(j + k + 1)); \ | |
1495 | } \ | |
1496 | for (j = 0; j < LANE_WIDTH / 4; i++, j += 2) { \ | |
1497 | r[i + k] = F(s->L(j + k), s->L(j + k + 1)); \ | |
1498 | } \ | |
1499 | } \ | |
1500 | for (i = 0; i < 2 << SHIFT; i++) { \ | |
1501 | d->L(i) = r[i]; \ | |
1502 | } \ | |
1503 | } | |
1504 | ||
1505 | SSE_HELPER_HW(phaddw, FADD) | |
1506 | SSE_HELPER_HW(phsubw, FSUB) | |
1507 | SSE_HELPER_HW(phaddsw, FADDSW) | |
1508 | SSE_HELPER_HW(phsubsw, FSUBSW) | |
1509 | SSE_HELPER_HL(phaddd, FADD) | |
1510 | SSE_HELPER_HL(phsubd, FSUB) | |
1511 | ||
1512 | #undef SSE_HELPER_HW | |
1513 | #undef SSE_HELPER_HL | |
1514 | ||
1515 | void glue(helper_pmaddubsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
1516 | { | |
1517 | int i; | |
1518 | for (i = 0; i < 4 << SHIFT; i++) { | |
1519 | d->W(i) = satsw((int8_t)s->B(i * 2) * (uint8_t)v->B(i * 2) + | |
1520 | (int8_t)s->B(i * 2 + 1) * (uint8_t)v->B(i * 2 + 1)); | |
1521 | } | |
1522 | } | |
1523 | ||
1524 | #define FMULHRSW(d, s) (((int16_t) d * (int16_t)s + 0x4000) >> 15) | |
1525 | SSE_HELPER_W(helper_pmulhrsw, FMULHRSW) | |
1526 | ||
1527 | #define FSIGNB(d, s) (s <= INT8_MAX ? s ? d : 0 : -(int8_t)d) | |
1528 | #define FSIGNW(d, s) (s <= INT16_MAX ? s ? d : 0 : -(int16_t)d) | |
1529 | #define FSIGNL(d, s) (s <= INT32_MAX ? s ? d : 0 : -(int32_t)d) | |
1530 | SSE_HELPER_B(helper_psignb, FSIGNB) | |
1531 | SSE_HELPER_W(helper_psignw, FSIGNW) | |
1532 | SSE_HELPER_L(helper_psignd, FSIGNL) | |
1533 | ||
1534 | void glue(helper_palignr, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
1535 | uint32_t imm) | |
1536 | { | |
1537 | int i; | |
1538 | ||
1539 | /* XXX could be checked during translation */ | |
1540 | if (imm >= (SHIFT ? 32 : 16)) { | |
1541 | for (i = 0; i < (1 << SHIFT); i++) { | |
1542 | d->Q(i) = 0; | |
1543 | } | |
1544 | } else { | |
1545 | int shift = imm * 8; | |
1546 | #define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0) | |
1547 | #if SHIFT == 0 | |
1548 | d->Q(0) = SHR(s->Q(0), shift - 0) | | |
1549 | SHR(v->Q(0), shift - 64); | |
1550 | #else | |
1551 | for (i = 0; i < (1 << SHIFT); i += 2) { | |
1552 | uint64_t r0, r1; | |
1553 | ||
1554 | r0 = SHR(s->Q(i), shift - 0) | | |
1555 | SHR(s->Q(i + 1), shift - 64) | | |
1556 | SHR(v->Q(i), shift - 128) | | |
1557 | SHR(v->Q(i + 1), shift - 192); | |
1558 | r1 = SHR(s->Q(i), shift + 64) | | |
1559 | SHR(s->Q(i + 1), shift - 0) | | |
1560 | SHR(v->Q(i), shift - 64) | | |
1561 | SHR(v->Q(i + 1), shift - 128); | |
1562 | d->Q(i) = r0; | |
1563 | d->Q(i + 1) = r1; | |
1564 | } | |
1565 | #endif | |
1566 | #undef SHR | |
1567 | } | |
1568 | } | |
1569 | ||
1570 | #if SHIFT >= 1 | |
1571 | ||
1572 | #define SSE_HELPER_V(name, elem, num, F) \ | |
1573 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, \ | |
1574 | Reg *m) \ | |
1575 | { \ | |
1576 | int i; \ | |
1577 | for (i = 0; i < num; i++) { \ | |
1578 | d->elem(i) = F(v->elem(i), s->elem(i), m->elem(i)); \ | |
1579 | } \ | |
1580 | } | |
1581 | ||
1582 | #define SSE_HELPER_I(name, elem, num, F) \ | |
1583 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, \ | |
1584 | uint32_t imm) \ | |
1585 | { \ | |
1586 | int i; \ | |
1587 | for (i = 0; i < num; i++) { \ | |
1588 | int j = i & 7; \ | |
1589 | d->elem(i) = F(v->elem(i), s->elem(i), (imm >> j) & 1); \ | |
1590 | } \ | |
1591 | } | |
1592 | ||
1593 | /* SSE4.1 op helpers */ | |
1594 | #define FBLENDVB(v, s, m) ((m & 0x80) ? s : v) | |
1595 | #define FBLENDVPS(v, s, m) ((m & 0x80000000) ? s : v) | |
1596 | #define FBLENDVPD(v, s, m) ((m & 0x8000000000000000LL) ? s : v) | |
1597 | SSE_HELPER_V(helper_pblendvb, B, 8 << SHIFT, FBLENDVB) | |
1598 | SSE_HELPER_V(helper_blendvps, L, 2 << SHIFT, FBLENDVPS) | |
1599 | SSE_HELPER_V(helper_blendvpd, Q, 1 << SHIFT, FBLENDVPD) | |
1600 | ||
1601 | void glue(helper_ptest, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
1602 | { | |
1603 | uint64_t zf = 0, cf = 0; | |
1604 | int i; | |
1605 | ||
1606 | for (i = 0; i < 1 << SHIFT; i++) { | |
1607 | zf |= (s->Q(i) & d->Q(i)); | |
1608 | cf |= (s->Q(i) & ~d->Q(i)); | |
1609 | } | |
1610 | CC_SRC = (zf ? 0 : CC_Z) | (cf ? 0 : CC_C); | |
1611 | } | |
1612 | ||
1613 | #define FMOVSLDUP(i) s->L((i) & ~1) | |
1614 | #define FMOVSHDUP(i) s->L((i) | 1) | |
1615 | #define FMOVDLDUP(i) s->Q((i) & ~1) | |
1616 | ||
1617 | #define SSE_HELPER_F(name, elem, num, F) \ | |
1618 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) \ | |
1619 | { \ | |
1620 | int n = num; \ | |
1621 | for (int i = n; --i >= 0; ) { \ | |
1622 | d->elem(i) = F(i); \ | |
1623 | } \ | |
1624 | } | |
1625 | ||
1626 | #if SHIFT > 0 | |
1627 | SSE_HELPER_F(helper_pmovsxbw, W, 4 << SHIFT, (int8_t) s->B) | |
1628 | SSE_HELPER_F(helper_pmovsxbd, L, 2 << SHIFT, (int8_t) s->B) | |
1629 | SSE_HELPER_F(helper_pmovsxbq, Q, 1 << SHIFT, (int8_t) s->B) | |
1630 | SSE_HELPER_F(helper_pmovsxwd, L, 2 << SHIFT, (int16_t) s->W) | |
1631 | SSE_HELPER_F(helper_pmovsxwq, Q, 1 << SHIFT, (int16_t) s->W) | |
1632 | SSE_HELPER_F(helper_pmovsxdq, Q, 1 << SHIFT, (int32_t) s->L) | |
1633 | SSE_HELPER_F(helper_pmovzxbw, W, 4 << SHIFT, s->B) | |
1634 | SSE_HELPER_F(helper_pmovzxbd, L, 2 << SHIFT, s->B) | |
1635 | SSE_HELPER_F(helper_pmovzxbq, Q, 1 << SHIFT, s->B) | |
1636 | SSE_HELPER_F(helper_pmovzxwd, L, 2 << SHIFT, s->W) | |
1637 | SSE_HELPER_F(helper_pmovzxwq, Q, 1 << SHIFT, s->W) | |
1638 | SSE_HELPER_F(helper_pmovzxdq, Q, 1 << SHIFT, s->L) | |
1639 | SSE_HELPER_F(helper_pmovsldup, L, 2 << SHIFT, FMOVSLDUP) | |
1640 | SSE_HELPER_F(helper_pmovshdup, L, 2 << SHIFT, FMOVSHDUP) | |
1641 | SSE_HELPER_F(helper_pmovdldup, Q, 1 << SHIFT, FMOVDLDUP) | |
1642 | #endif | |
1643 | ||
1644 | void glue(helper_pmuldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
1645 | { | |
1646 | int i; | |
1647 | ||
1648 | for (i = 0; i < 1 << SHIFT; i++) { | |
1649 | d->Q(i) = (int64_t)(int32_t) v->L(2 * i) * (int32_t) s->L(2 * i); | |
1650 | } | |
1651 | } | |
1652 | ||
1653 | void glue(helper_packusdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
1654 | { | |
1655 | uint16_t r[8]; | |
1656 | int i, j, k; | |
1657 | ||
1658 | for (i = 0, j = 0; i <= 2 << SHIFT; i += 8, j += 4) { | |
1659 | r[0] = satuw(v->L(j)); | |
1660 | r[1] = satuw(v->L(j + 1)); | |
1661 | r[2] = satuw(v->L(j + 2)); | |
1662 | r[3] = satuw(v->L(j + 3)); | |
1663 | r[4] = satuw(s->L(j)); | |
1664 | r[5] = satuw(s->L(j + 1)); | |
1665 | r[6] = satuw(s->L(j + 2)); | |
1666 | r[7] = satuw(s->L(j + 3)); | |
1667 | for (k = 0; k < 8; k++) { | |
1668 | d->W(i + k) = r[k]; | |
1669 | } | |
1670 | } | |
1671 | } | |
1672 | ||
1673 | #if SHIFT == 1 | |
1674 | void glue(helper_phminposuw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
1675 | { | |
1676 | int idx = 0; | |
1677 | ||
1678 | if (s->W(1) < s->W(idx)) { | |
1679 | idx = 1; | |
1680 | } | |
1681 | if (s->W(2) < s->W(idx)) { | |
1682 | idx = 2; | |
1683 | } | |
1684 | if (s->W(3) < s->W(idx)) { | |
1685 | idx = 3; | |
1686 | } | |
1687 | if (s->W(4) < s->W(idx)) { | |
1688 | idx = 4; | |
1689 | } | |
1690 | if (s->W(5) < s->W(idx)) { | |
1691 | idx = 5; | |
1692 | } | |
1693 | if (s->W(6) < s->W(idx)) { | |
1694 | idx = 6; | |
1695 | } | |
1696 | if (s->W(7) < s->W(idx)) { | |
1697 | idx = 7; | |
1698 | } | |
1699 | ||
1700 | d->W(0) = s->W(idx); | |
1701 | d->W(1) = idx; | |
1702 | d->L(1) = 0; | |
1703 | d->Q(1) = 0; | |
1704 | } | |
1705 | #endif | |
1706 | ||
1707 | void glue(helper_roundps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
1708 | uint32_t mode) | |
1709 | { | |
1710 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
1711 | signed char prev_rounding_mode; | |
1712 | int i; | |
1713 | ||
1714 | prev_rounding_mode = env->sse_status.float_rounding_mode; | |
1715 | if (!(mode & (1 << 2))) { | |
1716 | set_x86_rounding_mode(mode & 3, &env->sse_status); | |
1717 | } | |
1718 | ||
1719 | for (i = 0; i < 2 << SHIFT; i++) { | |
1720 | d->ZMM_S(i) = float32_round_to_int(s->ZMM_S(i), &env->sse_status); | |
1721 | } | |
1722 | ||
1723 | if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { | |
1724 | set_float_exception_flags(get_float_exception_flags(&env->sse_status) & | |
1725 | ~float_flag_inexact, | |
1726 | &env->sse_status); | |
1727 | } | |
1728 | env->sse_status.float_rounding_mode = prev_rounding_mode; | |
1729 | } | |
1730 | ||
1731 | void glue(helper_roundpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
1732 | uint32_t mode) | |
1733 | { | |
1734 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
1735 | signed char prev_rounding_mode; | |
1736 | int i; | |
1737 | ||
1738 | prev_rounding_mode = env->sse_status.float_rounding_mode; | |
1739 | if (!(mode & (1 << 2))) { | |
1740 | set_x86_rounding_mode(mode & 3, &env->sse_status); | |
1741 | } | |
1742 | ||
1743 | for (i = 0; i < 1 << SHIFT; i++) { | |
1744 | d->ZMM_D(i) = float64_round_to_int(s->ZMM_D(i), &env->sse_status); | |
1745 | } | |
1746 | ||
1747 | if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { | |
1748 | set_float_exception_flags(get_float_exception_flags(&env->sse_status) & | |
1749 | ~float_flag_inexact, | |
1750 | &env->sse_status); | |
1751 | } | |
1752 | env->sse_status.float_rounding_mode = prev_rounding_mode; | |
1753 | } | |
1754 | ||
1755 | #if SHIFT == 1 | |
1756 | void glue(helper_roundss, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
1757 | uint32_t mode) | |
1758 | { | |
1759 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
1760 | signed char prev_rounding_mode; | |
1761 | int i; | |
1762 | ||
1763 | prev_rounding_mode = env->sse_status.float_rounding_mode; | |
1764 | if (!(mode & (1 << 2))) { | |
1765 | set_x86_rounding_mode(mode & 3, &env->sse_status); | |
1766 | } | |
1767 | ||
1768 | d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status); | |
1769 | for (i = 1; i < 2 << SHIFT; i++) { | |
1770 | d->ZMM_L(i) = v->ZMM_L(i); | |
1771 | } | |
1772 | ||
1773 | if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { | |
1774 | set_float_exception_flags(get_float_exception_flags(&env->sse_status) & | |
1775 | ~float_flag_inexact, | |
1776 | &env->sse_status); | |
1777 | } | |
1778 | env->sse_status.float_rounding_mode = prev_rounding_mode; | |
1779 | } | |
1780 | ||
1781 | void glue(helper_roundsd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
1782 | uint32_t mode) | |
1783 | { | |
1784 | uint8_t old_flags = get_float_exception_flags(&env->sse_status); | |
1785 | signed char prev_rounding_mode; | |
1786 | int i; | |
1787 | ||
1788 | prev_rounding_mode = env->sse_status.float_rounding_mode; | |
1789 | if (!(mode & (1 << 2))) { | |
1790 | set_x86_rounding_mode(mode & 3, &env->sse_status); | |
1791 | } | |
1792 | ||
1793 | d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status); | |
1794 | for (i = 1; i < 1 << SHIFT; i++) { | |
1795 | d->ZMM_Q(i) = v->ZMM_Q(i); | |
1796 | } | |
1797 | ||
1798 | if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { | |
1799 | set_float_exception_flags(get_float_exception_flags(&env->sse_status) & | |
1800 | ~float_flag_inexact, | |
1801 | &env->sse_status); | |
1802 | } | |
1803 | env->sse_status.float_rounding_mode = prev_rounding_mode; | |
1804 | } | |
1805 | #endif | |
1806 | ||
1807 | #define FBLENDP(v, s, m) (m ? s : v) | |
1808 | SSE_HELPER_I(helper_blendps, L, 2 << SHIFT, FBLENDP) | |
1809 | SSE_HELPER_I(helper_blendpd, Q, 1 << SHIFT, FBLENDP) | |
1810 | SSE_HELPER_I(helper_pblendw, W, 4 << SHIFT, FBLENDP) | |
1811 | ||
1812 | void glue(helper_dpps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
1813 | uint32_t mask) | |
1814 | { | |
1815 | float32 prod1, prod2, temp2, temp3, temp4; | |
1816 | int i; | |
1817 | ||
1818 | for (i = 0; i < 2 << SHIFT; i += 4) { | |
1819 | /* | |
1820 | * We must evaluate (A+B)+(C+D), not ((A+B)+C)+D | |
1821 | * to correctly round the intermediate results | |
1822 | */ | |
1823 | if (mask & (1 << 4)) { | |
1824 | prod1 = float32_mul(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status); | |
1825 | } else { | |
1826 | prod1 = float32_zero; | |
1827 | } | |
1828 | if (mask & (1 << 5)) { | |
1829 | prod2 = float32_mul(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status); | |
1830 | } else { | |
1831 | prod2 = float32_zero; | |
1832 | } | |
1833 | temp2 = float32_add(prod1, prod2, &env->sse_status); | |
1834 | if (mask & (1 << 6)) { | |
1835 | prod1 = float32_mul(v->ZMM_S(i+2), s->ZMM_S(i+2), &env->sse_status); | |
1836 | } else { | |
1837 | prod1 = float32_zero; | |
1838 | } | |
1839 | if (mask & (1 << 7)) { | |
1840 | prod2 = float32_mul(v->ZMM_S(i+3), s->ZMM_S(i+3), &env->sse_status); | |
1841 | } else { | |
1842 | prod2 = float32_zero; | |
1843 | } | |
1844 | temp3 = float32_add(prod1, prod2, &env->sse_status); | |
1845 | temp4 = float32_add(temp2, temp3, &env->sse_status); | |
1846 | ||
1847 | d->ZMM_S(i) = (mask & (1 << 0)) ? temp4 : float32_zero; | |
1848 | d->ZMM_S(i+1) = (mask & (1 << 1)) ? temp4 : float32_zero; | |
1849 | d->ZMM_S(i+2) = (mask & (1 << 2)) ? temp4 : float32_zero; | |
1850 | d->ZMM_S(i+3) = (mask & (1 << 3)) ? temp4 : float32_zero; | |
1851 | } | |
1852 | } | |
1853 | ||
1854 | #if SHIFT == 1 | |
1855 | /* Oddly, there is no ymm version of dppd */ | |
1856 | void glue(helper_dppd, SUFFIX)(CPUX86State *env, | |
1857 | Reg *d, Reg *v, Reg *s, uint32_t mask) | |
1858 | { | |
1859 | float64 prod1, prod2, temp2; | |
1860 | ||
1861 | if (mask & (1 << 4)) { | |
1862 | prod1 = float64_mul(v->ZMM_D(0), s->ZMM_D(0), &env->sse_status); | |
1863 | } else { | |
1864 | prod1 = float64_zero; | |
1865 | } | |
1866 | if (mask & (1 << 5)) { | |
1867 | prod2 = float64_mul(v->ZMM_D(1), s->ZMM_D(1), &env->sse_status); | |
1868 | } else { | |
1869 | prod2 = float64_zero; | |
1870 | } | |
1871 | temp2 = float64_add(prod1, prod2, &env->sse_status); | |
1872 | d->ZMM_D(0) = (mask & (1 << 0)) ? temp2 : float64_zero; | |
1873 | d->ZMM_D(1) = (mask & (1 << 1)) ? temp2 : float64_zero; | |
1874 | } | |
1875 | #endif | |
1876 | ||
1877 | void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
1878 | uint32_t offset) | |
1879 | { | |
1880 | int i, j; | |
1881 | uint16_t r[8]; | |
1882 | ||
1883 | for (j = 0; j < 4 << SHIFT; ) { | |
1884 | int s0 = (j * 2) + ((offset & 3) << 2); | |
1885 | int d0 = (j * 2) + ((offset & 4) << 0); | |
1886 | for (i = 0; i < LANE_WIDTH / 2; i++, d0++) { | |
1887 | r[i] = 0; | |
1888 | r[i] += abs1(v->B(d0 + 0) - s->B(s0 + 0)); | |
1889 | r[i] += abs1(v->B(d0 + 1) - s->B(s0 + 1)); | |
1890 | r[i] += abs1(v->B(d0 + 2) - s->B(s0 + 2)); | |
1891 | r[i] += abs1(v->B(d0 + 3) - s->B(s0 + 3)); | |
1892 | } | |
1893 | for (i = 0; i < LANE_WIDTH / 2; i++, j++) { | |
1894 | d->W(j) = r[i]; | |
1895 | } | |
1896 | offset >>= 3; | |
1897 | } | |
1898 | } | |
1899 | ||
1900 | /* SSE4.2 op helpers */ | |
1901 | #if SHIFT == 1 | |
1902 | static inline int pcmp_elen(CPUX86State *env, int reg, uint32_t ctrl) | |
1903 | { | |
1904 | target_long val, limit; | |
1905 | ||
1906 | /* Presence of REX.W is indicated by a bit higher than 7 set */ | |
1907 | if (ctrl >> 8) { | |
1908 | val = (target_long)env->regs[reg]; | |
1909 | } else { | |
1910 | val = (int32_t)env->regs[reg]; | |
1911 | } | |
1912 | if (ctrl & 1) { | |
1913 | limit = 8; | |
1914 | } else { | |
1915 | limit = 16; | |
1916 | } | |
1917 | if ((val > limit) || (val < -limit)) { | |
1918 | return limit; | |
1919 | } | |
1920 | return abs1(val); | |
1921 | } | |
1922 | ||
1923 | static inline int pcmp_ilen(Reg *r, uint8_t ctrl) | |
1924 | { | |
1925 | int val = 0; | |
1926 | ||
1927 | if (ctrl & 1) { | |
1928 | while (val < 8 && r->W(val)) { | |
1929 | val++; | |
1930 | } | |
1931 | } else { | |
1932 | while (val < 16 && r->B(val)) { | |
1933 | val++; | |
1934 | } | |
1935 | } | |
1936 | ||
1937 | return val; | |
1938 | } | |
1939 | ||
1940 | static inline int pcmp_val(Reg *r, uint8_t ctrl, int i) | |
1941 | { | |
1942 | switch ((ctrl >> 0) & 3) { | |
1943 | case 0: | |
1944 | return r->B(i); | |
1945 | case 1: | |
1946 | return r->W(i); | |
1947 | case 2: | |
1948 | return (int8_t)r->B(i); | |
1949 | case 3: | |
1950 | default: | |
1951 | return (int16_t)r->W(i); | |
1952 | } | |
1953 | } | |
1954 | ||
1955 | static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s, | |
1956 | uint8_t ctrl, int valids, int validd) | |
1957 | { | |
1958 | unsigned int res = 0; | |
1959 | int v; | |
1960 | int j, i; | |
1961 | int upper = (ctrl & 1) ? 7 : 15; | |
1962 | ||
1963 | valids--; | |
1964 | validd--; | |
1965 | ||
1966 | CC_SRC = (valids < upper ? CC_Z : 0) | (validd < upper ? CC_S : 0); | |
1967 | ||
1968 | switch ((ctrl >> 2) & 3) { | |
1969 | case 0: | |
1970 | for (j = valids; j >= 0; j--) { | |
1971 | res <<= 1; | |
1972 | v = pcmp_val(s, ctrl, j); | |
1973 | for (i = validd; i >= 0; i--) { | |
1974 | res |= (v == pcmp_val(d, ctrl, i)); | |
1975 | } | |
1976 | } | |
1977 | break; | |
1978 | case 1: | |
1979 | for (j = valids; j >= 0; j--) { | |
1980 | res <<= 1; | |
1981 | v = pcmp_val(s, ctrl, j); | |
1982 | for (i = ((validd - 1) | 1); i >= 0; i -= 2) { | |
1983 | res |= (pcmp_val(d, ctrl, i - 0) >= v && | |
1984 | pcmp_val(d, ctrl, i - 1) <= v); | |
1985 | } | |
1986 | } | |
1987 | break; | |
1988 | case 2: | |
1989 | res = (1 << (upper - MAX(valids, validd))) - 1; | |
1990 | res <<= MAX(valids, validd) - MIN(valids, validd); | |
1991 | for (i = MIN(valids, validd); i >= 0; i--) { | |
1992 | res <<= 1; | |
1993 | v = pcmp_val(s, ctrl, i); | |
1994 | res |= (v == pcmp_val(d, ctrl, i)); | |
1995 | } | |
1996 | break; | |
1997 | case 3: | |
1998 | if (validd == -1) { | |
1999 | res = (2 << upper) - 1; | |
2000 | break; | |
2001 | } | |
2002 | for (j = valids == upper ? valids : valids - validd; j >= 0; j--) { | |
2003 | res <<= 1; | |
2004 | v = 1; | |
2005 | for (i = MIN(valids - j, validd); i >= 0; i--) { | |
2006 | v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i)); | |
2007 | } | |
2008 | res |= v; | |
2009 | } | |
2010 | break; | |
2011 | } | |
2012 | ||
2013 | switch ((ctrl >> 4) & 3) { | |
2014 | case 1: | |
2015 | res ^= (2 << upper) - 1; | |
2016 | break; | |
2017 | case 3: | |
2018 | res ^= (1 << (valids + 1)) - 1; | |
2019 | break; | |
2020 | } | |
2021 | ||
2022 | if (res) { | |
2023 | CC_SRC |= CC_C; | |
2024 | } | |
2025 | if (res & 1) { | |
2026 | CC_SRC |= CC_O; | |
2027 | } | |
2028 | ||
2029 | return res; | |
2030 | } | |
2031 | ||
2032 | void glue(helper_pcmpestri, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
2033 | uint32_t ctrl) | |
2034 | { | |
2035 | unsigned int res = pcmpxstrx(env, d, s, ctrl, | |
2036 | pcmp_elen(env, R_EDX, ctrl), | |
2037 | pcmp_elen(env, R_EAX, ctrl)); | |
2038 | ||
2039 | if (res) { | |
2040 | env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res); | |
2041 | } else { | |
2042 | env->regs[R_ECX] = 16 >> (ctrl & (1 << 0)); | |
2043 | } | |
2044 | } | |
2045 | ||
2046 | void glue(helper_pcmpestrm, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
2047 | uint32_t ctrl) | |
2048 | { | |
2049 | int i; | |
2050 | unsigned int res = pcmpxstrx(env, d, s, ctrl, | |
2051 | pcmp_elen(env, R_EDX, ctrl), | |
2052 | pcmp_elen(env, R_EAX, ctrl)); | |
2053 | ||
2054 | if ((ctrl >> 6) & 1) { | |
2055 | if (ctrl & 1) { | |
2056 | for (i = 0; i < 8; i++, res >>= 1) { | |
2057 | env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0; | |
2058 | } | |
2059 | } else { | |
2060 | for (i = 0; i < 16; i++, res >>= 1) { | |
2061 | env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0; | |
2062 | } | |
2063 | } | |
2064 | } else { | |
2065 | env->xmm_regs[0].Q(1) = 0; | |
2066 | env->xmm_regs[0].Q(0) = res; | |
2067 | } | |
2068 | } | |
2069 | ||
2070 | void glue(helper_pcmpistri, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
2071 | uint32_t ctrl) | |
2072 | { | |
2073 | unsigned int res = pcmpxstrx(env, d, s, ctrl, | |
2074 | pcmp_ilen(s, ctrl), | |
2075 | pcmp_ilen(d, ctrl)); | |
2076 | ||
2077 | if (res) { | |
2078 | env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res); | |
2079 | } else { | |
2080 | env->regs[R_ECX] = 16 >> (ctrl & (1 << 0)); | |
2081 | } | |
2082 | } | |
2083 | ||
2084 | void glue(helper_pcmpistrm, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
2085 | uint32_t ctrl) | |
2086 | { | |
2087 | int i; | |
2088 | unsigned int res = pcmpxstrx(env, d, s, ctrl, | |
2089 | pcmp_ilen(s, ctrl), | |
2090 | pcmp_ilen(d, ctrl)); | |
2091 | ||
2092 | if ((ctrl >> 6) & 1) { | |
2093 | if (ctrl & 1) { | |
2094 | for (i = 0; i < 8; i++, res >>= 1) { | |
2095 | env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0; | |
2096 | } | |
2097 | } else { | |
2098 | for (i = 0; i < 16; i++, res >>= 1) { | |
2099 | env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0; | |
2100 | } | |
2101 | } | |
2102 | } else { | |
2103 | env->xmm_regs[0].Q(1) = 0; | |
2104 | env->xmm_regs[0].Q(0) = res; | |
2105 | } | |
2106 | } | |
2107 | ||
2108 | #define CRCPOLY 0x1edc6f41 | |
2109 | #define CRCPOLY_BITREV 0x82f63b78 | |
2110 | target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len) | |
2111 | { | |
2112 | target_ulong crc = (msg & ((target_ulong) -1 >> | |
2113 | (TARGET_LONG_BITS - len))) ^ crc1; | |
2114 | ||
2115 | while (len--) { | |
2116 | crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_BITREV : 0); | |
2117 | } | |
2118 | ||
2119 | return crc; | |
2120 | } | |
2121 | ||
2122 | #endif | |
2123 | ||
2124 | #if SHIFT == 1 | |
2125 | static void clmulq(uint64_t *dest_l, uint64_t *dest_h, | |
2126 | uint64_t a, uint64_t b) | |
2127 | { | |
2128 | uint64_t al, ah, resh, resl; | |
2129 | ||
2130 | ah = 0; | |
2131 | al = a; | |
2132 | resh = resl = 0; | |
2133 | ||
2134 | while (b) { | |
2135 | if (b & 1) { | |
2136 | resl ^= al; | |
2137 | resh ^= ah; | |
2138 | } | |
2139 | ah = (ah << 1) | (al >> 63); | |
2140 | al <<= 1; | |
2141 | b >>= 1; | |
2142 | } | |
2143 | ||
2144 | *dest_l = resl; | |
2145 | *dest_h = resh; | |
2146 | } | |
2147 | #endif | |
2148 | ||
2149 | void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, | |
2150 | uint32_t ctrl) | |
2151 | { | |
2152 | uint64_t a, b; | |
2153 | int i; | |
2154 | ||
2155 | for (i = 0; i < 1 << SHIFT; i += 2) { | |
2156 | a = v->Q(((ctrl & 1) != 0) + i); | |
2157 | b = s->Q(((ctrl & 16) != 0) + i); | |
2158 | clmulq(&d->Q(i), &d->Q(i + 1), a, b); | |
2159 | } | |
2160 | } | |
2161 | ||
2162 | void glue(helper_aesdec, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2163 | { | |
2164 | int i; | |
2165 | Reg st = *v; | |
2166 | Reg rk = *s; | |
2167 | ||
2168 | for (i = 0 ; i < 2 << SHIFT ; i++) { | |
2169 | int j = i & 3; | |
2170 | d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(AES_ishifts[4 * j + 0])] ^ | |
2171 | AES_Td1[st.B(AES_ishifts[4 * j + 1])] ^ | |
2172 | AES_Td2[st.B(AES_ishifts[4 * j + 2])] ^ | |
2173 | AES_Td3[st.B(AES_ishifts[4 * j + 3])]); | |
2174 | } | |
2175 | } | |
2176 | ||
2177 | void glue(helper_aesdeclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2178 | { | |
2179 | int i; | |
2180 | Reg st = *v; | |
2181 | Reg rk = *s; | |
2182 | ||
2183 | for (i = 0; i < 8 << SHIFT; i++) { | |
2184 | d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i & 15] + (i & ~15))]); | |
2185 | } | |
2186 | } | |
2187 | ||
2188 | void glue(helper_aesenc, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2189 | { | |
2190 | int i; | |
2191 | Reg st = *v; | |
2192 | Reg rk = *s; | |
2193 | ||
2194 | for (i = 0 ; i < 2 << SHIFT ; i++) { | |
2195 | int j = i & 3; | |
2196 | d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(AES_shifts[4 * j + 0])] ^ | |
2197 | AES_Te1[st.B(AES_shifts[4 * j + 1])] ^ | |
2198 | AES_Te2[st.B(AES_shifts[4 * j + 2])] ^ | |
2199 | AES_Te3[st.B(AES_shifts[4 * j + 3])]); | |
2200 | } | |
2201 | } | |
2202 | ||
2203 | void glue(helper_aesenclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2204 | { | |
2205 | int i; | |
2206 | Reg st = *v; | |
2207 | Reg rk = *s; | |
2208 | ||
2209 | for (i = 0; i < 8 << SHIFT; i++) { | |
2210 | d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i & 15] + (i & ~15))]); | |
2211 | } | |
2212 | } | |
2213 | ||
2214 | #if SHIFT == 1 | |
2215 | void glue(helper_aesimc, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
2216 | { | |
2217 | int i; | |
2218 | Reg tmp = *s; | |
2219 | ||
2220 | for (i = 0 ; i < 4 ; i++) { | |
2221 | d->L(i) = bswap32(AES_imc[tmp.B(4 * i + 0)][0] ^ | |
2222 | AES_imc[tmp.B(4 * i + 1)][1] ^ | |
2223 | AES_imc[tmp.B(4 * i + 2)][2] ^ | |
2224 | AES_imc[tmp.B(4 * i + 3)][3]); | |
2225 | } | |
2226 | } | |
2227 | ||
2228 | void glue(helper_aeskeygenassist, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, | |
2229 | uint32_t ctrl) | |
2230 | { | |
2231 | int i; | |
2232 | Reg tmp = *s; | |
2233 | ||
2234 | for (i = 0 ; i < 4 ; i++) { | |
2235 | d->B(i) = AES_sbox[tmp.B(i + 4)]; | |
2236 | d->B(i + 8) = AES_sbox[tmp.B(i + 12)]; | |
2237 | } | |
2238 | d->L(1) = (d->L(0) << 24 | d->L(0) >> 8) ^ ctrl; | |
2239 | d->L(3) = (d->L(2) << 24 | d->L(2) >> 8) ^ ctrl; | |
2240 | } | |
2241 | #endif | |
2242 | #endif | |
2243 | ||
2244 | #if SHIFT >= 1 | |
2245 | void glue(helper_vpermilpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2246 | { | |
2247 | uint64_t r0, r1; | |
2248 | int i; | |
2249 | ||
2250 | for (i = 0; i < 1 << SHIFT; i += 2) { | |
2251 | r0 = v->Q(i + ((s->Q(i) >> 1) & 1)); | |
2252 | r1 = v->Q(i + ((s->Q(i+1) >> 1) & 1)); | |
2253 | d->Q(i) = r0; | |
2254 | d->Q(i+1) = r1; | |
2255 | } | |
2256 | } | |
2257 | ||
2258 | void glue(helper_vpermilps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2259 | { | |
2260 | uint32_t r0, r1, r2, r3; | |
2261 | int i; | |
2262 | ||
2263 | for (i = 0; i < 2 << SHIFT; i += 4) { | |
2264 | r0 = v->L(i + (s->L(i) & 3)); | |
2265 | r1 = v->L(i + (s->L(i+1) & 3)); | |
2266 | r2 = v->L(i + (s->L(i+2) & 3)); | |
2267 | r3 = v->L(i + (s->L(i+3) & 3)); | |
2268 | d->L(i) = r0; | |
2269 | d->L(i+1) = r1; | |
2270 | d->L(i+2) = r2; | |
2271 | d->L(i+3) = r3; | |
2272 | } | |
2273 | } | |
2274 | ||
2275 | void glue(helper_vpermilpd_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order) | |
2276 | { | |
2277 | uint64_t r0, r1; | |
2278 | int i; | |
2279 | ||
2280 | for (i = 0; i < 1 << SHIFT; i += 2) { | |
2281 | r0 = s->Q(i + ((order >> 0) & 1)); | |
2282 | r1 = s->Q(i + ((order >> 1) & 1)); | |
2283 | d->Q(i) = r0; | |
2284 | d->Q(i+1) = r1; | |
2285 | ||
2286 | order >>= 2; | |
2287 | } | |
2288 | } | |
2289 | ||
2290 | void glue(helper_vpermilps_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order) | |
2291 | { | |
2292 | uint32_t r0, r1, r2, r3; | |
2293 | int i; | |
2294 | ||
2295 | for (i = 0; i < 2 << SHIFT; i += 4) { | |
2296 | r0 = s->L(i + ((order >> 0) & 3)); | |
2297 | r1 = s->L(i + ((order >> 2) & 3)); | |
2298 | r2 = s->L(i + ((order >> 4) & 3)); | |
2299 | r3 = s->L(i + ((order >> 6) & 3)); | |
2300 | d->L(i) = r0; | |
2301 | d->L(i+1) = r1; | |
2302 | d->L(i+2) = r2; | |
2303 | d->L(i+3) = r3; | |
2304 | } | |
2305 | } | |
2306 | ||
2307 | #if SHIFT == 1 | |
2308 | #define FPSRLVD(x, c) (c < 32 ? ((x) >> c) : 0) | |
2309 | #define FPSRLVQ(x, c) (c < 64 ? ((x) >> c) : 0) | |
2310 | #define FPSRAVD(x, c) ((int32_t)(x) >> (c < 32 ? c : 31)) | |
2311 | #define FPSRAVQ(x, c) ((int64_t)(x) >> (c < 64 ? c : 63)) | |
2312 | #define FPSLLVD(x, c) (c < 32 ? ((x) << c) : 0) | |
2313 | #define FPSLLVQ(x, c) (c < 64 ? ((x) << c) : 0) | |
2314 | #endif | |
2315 | ||
2316 | SSE_HELPER_L(helper_vpsrlvd, FPSRLVD) | |
2317 | SSE_HELPER_L(helper_vpsravd, FPSRAVD) | |
2318 | SSE_HELPER_L(helper_vpsllvd, FPSLLVD) | |
2319 | ||
2320 | SSE_HELPER_Q(helper_vpsrlvq, FPSRLVQ) | |
2321 | SSE_HELPER_Q(helper_vpsravq, FPSRAVQ) | |
2322 | SSE_HELPER_Q(helper_vpsllvq, FPSLLVQ) | |
2323 | ||
2324 | void glue(helper_vtestps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
2325 | { | |
2326 | uint32_t zf = 0, cf = 0; | |
2327 | int i; | |
2328 | ||
2329 | for (i = 0; i < 2 << SHIFT; i++) { | |
2330 | zf |= (s->L(i) & d->L(i)); | |
2331 | cf |= (s->L(i) & ~d->L(i)); | |
2332 | } | |
2333 | CC_SRC = ((zf >> 31) ? 0 : CC_Z) | ((cf >> 31) ? 0 : CC_C); | |
2334 | } | |
2335 | ||
2336 | void glue(helper_vtestpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) | |
2337 | { | |
2338 | uint64_t zf = 0, cf = 0; | |
2339 | int i; | |
2340 | ||
2341 | for (i = 0; i < 1 << SHIFT; i++) { | |
2342 | zf |= (s->Q(i) & d->Q(i)); | |
2343 | cf |= (s->Q(i) & ~d->Q(i)); | |
2344 | } | |
2345 | CC_SRC = ((zf >> 63) ? 0 : CC_Z) | ((cf >> 63) ? 0 : CC_C); | |
2346 | } | |
2347 | ||
2348 | void glue(helper_vpmaskmovd_st, SUFFIX)(CPUX86State *env, | |
2349 | Reg *v, Reg *s, target_ulong a0) | |
2350 | { | |
2351 | int i; | |
2352 | ||
2353 | for (i = 0; i < (2 << SHIFT); i++) { | |
2354 | if (v->L(i) >> 31) { | |
2355 | cpu_stl_data_ra(env, a0 + i * 4, s->L(i), GETPC()); | |
2356 | } | |
2357 | } | |
2358 | } | |
2359 | ||
2360 | void glue(helper_vpmaskmovq_st, SUFFIX)(CPUX86State *env, | |
2361 | Reg *v, Reg *s, target_ulong a0) | |
2362 | { | |
2363 | int i; | |
2364 | ||
2365 | for (i = 0; i < (1 << SHIFT); i++) { | |
2366 | if (v->Q(i) >> 63) { | |
2367 | cpu_stq_data_ra(env, a0 + i * 8, s->Q(i), GETPC()); | |
2368 | } | |
2369 | } | |
2370 | } | |
2371 | ||
2372 | void glue(helper_vpmaskmovd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2373 | { | |
2374 | int i; | |
2375 | ||
2376 | for (i = 0; i < (2 << SHIFT); i++) { | |
2377 | d->L(i) = (v->L(i) >> 31) ? s->L(i) : 0; | |
2378 | } | |
2379 | } | |
2380 | ||
2381 | void glue(helper_vpmaskmovq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) | |
2382 | { | |
2383 | int i; | |
2384 | ||
2385 | for (i = 0; i < (1 << SHIFT); i++) { | |
2386 | d->Q(i) = (v->Q(i) >> 63) ? s->Q(i) : 0; | |
2387 | } | |
2388 | } | |
2389 | ||
2390 | void glue(helper_vpgatherdd, SUFFIX)(CPUX86State *env, | |
2391 | Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) | |
2392 | { | |
2393 | int i; | |
2394 | for (i = 0; i < (2 << SHIFT); i++) { | |
2395 | if (v->L(i) >> 31) { | |
2396 | target_ulong addr = a0 | |
2397 | + ((target_ulong)(int32_t)s->L(i) << scale); | |
2398 | d->L(i) = cpu_ldl_data_ra(env, addr, GETPC()); | |
2399 | } | |
2400 | v->L(i) = 0; | |
2401 | } | |
2402 | } | |
2403 | ||
2404 | void glue(helper_vpgatherdq, SUFFIX)(CPUX86State *env, | |
2405 | Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) | |
2406 | { | |
2407 | int i; | |
2408 | for (i = 0; i < (1 << SHIFT); i++) { | |
2409 | if (v->Q(i) >> 63) { | |
2410 | target_ulong addr = a0 | |
2411 | + ((target_ulong)(int32_t)s->L(i) << scale); | |
2412 | d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC()); | |
2413 | } | |
2414 | v->Q(i) = 0; | |
2415 | } | |
2416 | } | |
2417 | ||
2418 | void glue(helper_vpgatherqd, SUFFIX)(CPUX86State *env, | |
2419 | Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) | |
2420 | { | |
2421 | int i; | |
2422 | for (i = 0; i < (1 << SHIFT); i++) { | |
2423 | if (v->L(i) >> 31) { | |
2424 | target_ulong addr = a0 | |
2425 | + ((target_ulong)(int64_t)s->Q(i) << scale); | |
2426 | d->L(i) = cpu_ldl_data_ra(env, addr, GETPC()); | |
2427 | } | |
2428 | v->L(i) = 0; | |
2429 | } | |
2430 | for (i /= 2; i < 1 << SHIFT; i++) { | |
2431 | d->Q(i) = 0; | |
2432 | v->Q(i) = 0; | |
2433 | } | |
2434 | } | |
2435 | ||
2436 | void glue(helper_vpgatherqq, SUFFIX)(CPUX86State *env, | |
2437 | Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) | |
2438 | { | |
2439 | int i; | |
2440 | for (i = 0; i < (1 << SHIFT); i++) { | |
2441 | if (v->Q(i) >> 63) { | |
2442 | target_ulong addr = a0 | |
2443 | + ((target_ulong)(int64_t)s->Q(i) << scale); | |
2444 | d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC()); | |
2445 | } | |
2446 | v->Q(i) = 0; | |
2447 | } | |
2448 | } | |
2449 | #endif | |
2450 | ||
2451 | #if SHIFT >= 2 | |
2452 | void helper_vpermdq_ymm(Reg *d, Reg *v, Reg *s, uint32_t order) | |
2453 | { | |
2454 | uint64_t r0, r1, r2, r3; | |
2455 | ||
2456 | switch (order & 3) { | |
2457 | case 0: | |
2458 | r0 = v->Q(0); | |
2459 | r1 = v->Q(1); | |
2460 | break; | |
2461 | case 1: | |
2462 | r0 = v->Q(2); | |
2463 | r1 = v->Q(3); | |
2464 | break; | |
2465 | case 2: | |
2466 | r0 = s->Q(0); | |
2467 | r1 = s->Q(1); | |
2468 | break; | |
2469 | case 3: | |
2470 | r0 = s->Q(2); | |
2471 | r1 = s->Q(3); | |
2472 | break; | |
2473 | default: /* default case added to help the compiler to avoid warnings */ | |
2474 | g_assert_not_reached(); | |
2475 | } | |
2476 | switch ((order >> 4) & 3) { | |
2477 | case 0: | |
2478 | r2 = v->Q(0); | |
2479 | r3 = v->Q(1); | |
2480 | break; | |
2481 | case 1: | |
2482 | r2 = v->Q(2); | |
2483 | r3 = v->Q(3); | |
2484 | break; | |
2485 | case 2: | |
2486 | r2 = s->Q(0); | |
2487 | r3 = s->Q(1); | |
2488 | break; | |
2489 | case 3: | |
2490 | r2 = s->Q(2); | |
2491 | r3 = s->Q(3); | |
2492 | break; | |
2493 | default: /* default case added to help the compiler to avoid warnings */ | |
2494 | g_assert_not_reached(); | |
2495 | } | |
2496 | d->Q(0) = r0; | |
2497 | d->Q(1) = r1; | |
2498 | d->Q(2) = r2; | |
2499 | d->Q(3) = r3; | |
2500 | } | |
2501 | ||
2502 | void helper_vpermq_ymm(Reg *d, Reg *s, uint32_t order) | |
2503 | { | |
2504 | uint64_t r0, r1, r2, r3; | |
2505 | r0 = s->Q(order & 3); | |
2506 | r1 = s->Q((order >> 2) & 3); | |
2507 | r2 = s->Q((order >> 4) & 3); | |
2508 | r3 = s->Q((order >> 6) & 3); | |
2509 | d->Q(0) = r0; | |
2510 | d->Q(1) = r1; | |
2511 | d->Q(2) = r2; | |
2512 | d->Q(3) = r3; | |
2513 | } | |
2514 | ||
2515 | void helper_vpermd_ymm(Reg *d, Reg *v, Reg *s) | |
2516 | { | |
2517 | uint32_t r[8]; | |
2518 | int i; | |
2519 | ||
2520 | for (i = 0; i < 8; i++) { | |
2521 | r[i] = s->L(v->L(i) & 7); | |
2522 | } | |
2523 | for (i = 0; i < 8; i++) { | |
2524 | d->L(i) = r[i]; | |
2525 | } | |
2526 | } | |
2527 | #endif | |
2528 | ||
2529 | /* FMA3 op helpers */ | |
2530 | #if SHIFT == 1 | |
2531 | #define SSE_HELPER_FMAS(name, elem, F) \ | |
2532 | void name(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c, int flags) \ | |
2533 | { \ | |
2534 | d->elem(0) = F(a->elem(0), b->elem(0), c->elem(0), flags, &env->sse_status); \ | |
2535 | } | |
2536 | #define SSE_HELPER_FMAP(name, elem, num, F) \ | |
2537 | void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c, \ | |
2538 | int flags, int flip) \ | |
2539 | { \ | |
2540 | int i; \ | |
2541 | for (i = 0; i < num; i++) { \ | |
2542 | d->elem(i) = F(a->elem(i), b->elem(i), c->elem(i), flags, &env->sse_status); \ | |
2543 | flags ^= flip; \ | |
2544 | } \ | |
2545 | } | |
2546 | ||
2547 | SSE_HELPER_FMAS(helper_fma4ss, ZMM_S, float32_muladd) | |
2548 | SSE_HELPER_FMAS(helper_fma4sd, ZMM_D, float64_muladd) | |
2549 | #endif | |
2550 | ||
2551 | #if SHIFT >= 1 | |
2552 | SSE_HELPER_FMAP(helper_fma4ps, ZMM_S, 2 << SHIFT, float32_muladd) | |
2553 | SSE_HELPER_FMAP(helper_fma4pd, ZMM_D, 1 << SHIFT, float64_muladd) | |
2554 | #endif | |
2555 | ||
2556 | #undef SSE_HELPER_S | |
2557 | ||
2558 | #undef LANE_WIDTH | |
2559 | #undef SHIFT | |
2560 | #undef XMM_ONLY | |
2561 | #undef Reg | |
2562 | #undef B | |
2563 | #undef W | |
2564 | #undef L | |
2565 | #undef Q | |
2566 | #undef SUFFIX |