]> git.proxmox.com Git - qemu.git/blame - target-arm/neon_helper.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target-arm: Signal InvalidOp for Neon GE and GT compares of QNaN
[qemu.git] / target-arm / neon_helper.c
CommitLineData
e677137d
PB		 1/*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licenced under the GNU GPL v2.
8 */
ad69471c
PB		 9#include <stdlib.h>
10#include <stdio.h>
11
12#include "cpu.h"
2a3f75b4 13#include "exec.h"
7b59220e 14#include "helper.h"
ad69471c
PB		 15
16#define SIGNBIT (uint32_t)0x80000000
17#define SIGNBIT64 ((uint64_t)1 << 63)
18
19#define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
20
cc49f217 21#define NFS (&env->vfp.standard_fp_status)
ad69471c 22
ad69471c
PB		 23#define NEON_TYPE1(name, type) \
24typedef struct \
25{ \
26 type v1; \
27} neon_##name;
e2542fe2 28#ifdef HOST_WORDS_BIGENDIAN
ad69471c
PB		 29#define NEON_TYPE2(name, type) \
30typedef struct \
31{ \
32 type v2; \
33 type v1; \
34} neon_##name;
35#define NEON_TYPE4(name, type) \
36typedef struct \
37{ \
38 type v4; \
39 type v3; \
40 type v2; \
41 type v1; \
42} neon_##name;
43#else
44#define NEON_TYPE2(name, type) \
45typedef struct \
46{ \
47 type v1; \
48 type v2; \
49} neon_##name;
50#define NEON_TYPE4(name, type) \
51typedef struct \
52{ \
53 type v1; \
54 type v2; \
55 type v3; \
56 type v4; \
57} neon_##name;
58#endif
59
60NEON_TYPE4(s8, int8_t)
61NEON_TYPE4(u8, uint8_t)
62NEON_TYPE2(s16, int16_t)
63NEON_TYPE2(u16, uint16_t)
64NEON_TYPE1(s32, int32_t)
65NEON_TYPE1(u32, uint32_t)
66#undef NEON_TYPE4
67#undef NEON_TYPE2
68#undef NEON_TYPE1
69
70/* Copy from a uint32_t to a vector structure type. */
71#define NEON_UNPACK(vtype, dest, val) do { \
72 union { \
73 vtype v; \
74 uint32_t i; \
75 } conv_u; \
76 conv_u.i = (val); \
77 dest = conv_u.v; \
78 } while(0)
79
80/* Copy from a vector structure type to a uint32_t. */
81#define NEON_PACK(vtype, dest, val) do { \
82 union { \
83 vtype v; \
84 uint32_t i; \
85 } conv_u; \
86 conv_u.v = (val); \
87 dest = conv_u.i; \
88 } while(0)
89
90#define NEON_DO1 \
91 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
92#define NEON_DO2 \
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
95#define NEON_DO4 \
96 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
97 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
98 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
99 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
100
101#define NEON_VOP_BODY(vtype, n) \
102{ \
103 uint32_t res; \
104 vtype vsrc1; \
105 vtype vsrc2; \
106 vtype vdest; \
107 NEON_UNPACK(vtype, vsrc1, arg1); \
108 NEON_UNPACK(vtype, vsrc2, arg2); \
109 NEON_DO##n; \
110 NEON_PACK(vtype, res, vdest); \
111 return res; \
112}
113
114#define NEON_VOP(name, vtype, n) \
115uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
116NEON_VOP_BODY(vtype, n)
117
ad69471c
PB		 118/* Pairwise operations. */
119/* For 32-bit elements each segment only contains a single element, so
120 the elementwise and pairwise operations are the same. */
121#define NEON_PDO2 \
122 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
123 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
124#define NEON_PDO4 \
125 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
126 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
127 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
128 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
129
130#define NEON_POP(name, vtype, n) \
131uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
132{ \
133 uint32_t res; \
134 vtype vsrc1; \
135 vtype vsrc2; \
136 vtype vdest; \
137 NEON_UNPACK(vtype, vsrc1, arg1); \
138 NEON_UNPACK(vtype, vsrc2, arg2); \
139 NEON_PDO##n; \
140 NEON_PACK(vtype, res, vdest); \
141 return res; \
142}
143
144/* Unary operators. */
145#define NEON_VOP1(name, vtype, n) \
146uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
147{ \
148 vtype vsrc1; \
149 vtype vdest; \
150 NEON_UNPACK(vtype, vsrc1, arg); \
151 NEON_DO##n; \
152 NEON_PACK(vtype, arg, vdest); \
153 return arg; \
154}
155
156
157#define NEON_USAT(dest, src1, src2, type) do { \
158 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
159 if (tmp != (type)tmp) { \
160 SET_QC(); \
161 dest = ~0; \
162 } else { \
163 dest = tmp; \
164 }} while(0)
165#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
2a3f75b4 166NEON_VOP(qadd_u8, neon_u8, 4)
ad69471c
PB		 167#undef NEON_FN
168#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
2a3f75b4 169NEON_VOP(qadd_u16, neon_u16, 2)
ad69471c
PB		 170#undef NEON_FN
171#undef NEON_USAT
172
2a3f75b4 173uint32_t HELPER(neon_qadd_u32)(uint32_t a, uint32_t b)
72902672
CL		 174{
175 uint32_t res = a + b;
176 if (res < a) {
177 SET_QC();
178 res = ~0;
179 }
180 return res;
181}
182
2a3f75b4 183uint64_t HELPER(neon_qadd_u64)(uint64_t src1, uint64_t src2)
72902672
CL		 184{
185 uint64_t res;
186
187 res = src1 + src2;
188 if (res < src1) {
189 SET_QC();
190 res = ~(uint64_t)0;
191 }
192 return res;
193}
194
ad69471c
PB		 195#define NEON_SSAT(dest, src1, src2, type) do { \
196 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
197 if (tmp != (type)tmp) { \
198 SET_QC(); \
199 if (src2 > 0) { \
200 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
201 } else { \
202 tmp = 1 << (sizeof(type) * 8 - 1); \
203 } \
204 } \
205 dest = tmp; \
206 } while(0)
207#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
2a3f75b4 208NEON_VOP(qadd_s8, neon_s8, 4)
ad69471c
PB		 209#undef NEON_FN
210#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
2a3f75b4 211NEON_VOP(qadd_s16, neon_s16, 2)
ad69471c
PB		 212#undef NEON_FN
213#undef NEON_SSAT
214
2a3f75b4 215uint32_t HELPER(neon_qadd_s32)(uint32_t a, uint32_t b)
72902672
CL		 216{
217 uint32_t res = a + b;
218 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
219 SET_QC();
220 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
221 }
222 return res;
223}
224
2a3f75b4 225uint64_t HELPER(neon_qadd_s64)(uint64_t src1, uint64_t src2)
72902672
CL		 226{
227 uint64_t res;
228
229 res = src1 + src2;
230 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
231 SET_QC();
232 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
233 }
234 return res;
235}
236
ad69471c
PB		 237#define NEON_USAT(dest, src1, src2, type) do { \
238 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
239 if (tmp != (type)tmp) { \
240 SET_QC(); \
241 dest = 0; \
242 } else { \
243 dest = tmp; \
244 }} while(0)
245#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
2a3f75b4 246NEON_VOP(qsub_u8, neon_u8, 4)
ad69471c
PB		 247#undef NEON_FN
248#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
2a3f75b4 249NEON_VOP(qsub_u16, neon_u16, 2)
ad69471c
PB		 250#undef NEON_FN
251#undef NEON_USAT
252
2a3f75b4 253uint32_t HELPER(neon_qsub_u32)(uint32_t a, uint32_t b)
72902672
CL		 254{
255 uint32_t res = a - b;
256 if (res > a) {
257 SET_QC();
258 res = 0;
259 }
260 return res;
261}
262
2a3f75b4 263uint64_t HELPER(neon_qsub_u64)(uint64_t src1, uint64_t src2)
72902672
CL		 264{
265 uint64_t res;
266
267 if (src1 < src2) {
268 SET_QC();
269 res = 0;
270 } else {
271 res = src1 - src2;
272 }
273 return res;
274}
275
ad69471c
PB		 276#define NEON_SSAT(dest, src1, src2, type) do { \
277 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
278 if (tmp != (type)tmp) { \
279 SET_QC(); \
280 if (src2 < 0) { \
281 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
282 } else { \
283 tmp = 1 << (sizeof(type) * 8 - 1); \
284 } \
285 } \
286 dest = tmp; \
287 } while(0)
288#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
2a3f75b4 289NEON_VOP(qsub_s8, neon_s8, 4)
ad69471c
PB		 290#undef NEON_FN
291#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
2a3f75b4 292NEON_VOP(qsub_s16, neon_s16, 2)
ad69471c
PB		 293#undef NEON_FN
294#undef NEON_SSAT
295
2a3f75b4 296uint32_t HELPER(neon_qsub_s32)(uint32_t a, uint32_t b)
72902672
CL		 297{
298 uint32_t res = a - b;
299 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
300 SET_QC();
301 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
302 }
303 return res;
304}
305
2a3f75b4 306uint64_t HELPER(neon_qsub_s64)(uint64_t src1, uint64_t src2)
72902672
CL		 307{
308 uint64_t res;
309
310 res = src1 - src2;
311 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
312 SET_QC();
313 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
314 }
315 return res;
316}
317
ad69471c
PB		 318#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
319NEON_VOP(hadd_s8, neon_s8, 4)
320NEON_VOP(hadd_u8, neon_u8, 4)
321NEON_VOP(hadd_s16, neon_s16, 2)
322NEON_VOP(hadd_u16, neon_u16, 2)
323#undef NEON_FN
324
325int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
326{
327 int32_t dest;
328
329 dest = (src1 >> 1) + (src2 >> 1);
330 if (src1 & src2 & 1)
331 dest++;
332 return dest;
333}
334
335uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
336{
337 uint32_t dest;
338
339 dest = (src1 >> 1) + (src2 >> 1);
340 if (src1 & src2 & 1)
341 dest++;
342 return dest;
343}
344
345#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
346NEON_VOP(rhadd_s8, neon_s8, 4)
347NEON_VOP(rhadd_u8, neon_u8, 4)
348NEON_VOP(rhadd_s16, neon_s16, 2)
349NEON_VOP(rhadd_u16, neon_u16, 2)
350#undef NEON_FN
351
352int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
353{
354 int32_t dest;
355
356 dest = (src1 >> 1) + (src2 >> 1);
357 if ((src1 | src2) & 1)
358 dest++;
359 return dest;
360}
361
362uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
363{
364 uint32_t dest;
365
366 dest = (src1 >> 1) + (src2 >> 1);
367 if ((src1 | src2) & 1)
368 dest++;
369 return dest;
370}
371
372#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
373NEON_VOP(hsub_s8, neon_s8, 4)
374NEON_VOP(hsub_u8, neon_u8, 4)
375NEON_VOP(hsub_s16, neon_s16, 2)
376NEON_VOP(hsub_u16, neon_u16, 2)
377#undef NEON_FN
378
379int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
380{
381 int32_t dest;
382
383 dest = (src1 >> 1) - (src2 >> 1);
384 if ((~src1) & src2 & 1)
385 dest--;
386 return dest;
387}
388
389uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
390{
391 uint32_t dest;
392
393 dest = (src1 >> 1) - (src2 >> 1);
394 if ((~src1) & src2 & 1)
395 dest--;
396 return dest;
397}
398
399#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
400NEON_VOP(cgt_s8, neon_s8, 4)
401NEON_VOP(cgt_u8, neon_u8, 4)
402NEON_VOP(cgt_s16, neon_s16, 2)
403NEON_VOP(cgt_u16, neon_u16, 2)
404NEON_VOP(cgt_s32, neon_s32, 1)
405NEON_VOP(cgt_u32, neon_u32, 1)
406#undef NEON_FN
407
408#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
409NEON_VOP(cge_s8, neon_s8, 4)
410NEON_VOP(cge_u8, neon_u8, 4)
411NEON_VOP(cge_s16, neon_s16, 2)
412NEON_VOP(cge_u16, neon_u16, 2)
413NEON_VOP(cge_s32, neon_s32, 1)
414NEON_VOP(cge_u32, neon_u32, 1)
415#undef NEON_FN
416
417#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
418NEON_VOP(min_s8, neon_s8, 4)
419NEON_VOP(min_u8, neon_u8, 4)
420NEON_VOP(min_s16, neon_s16, 2)
421NEON_VOP(min_u16, neon_u16, 2)
422NEON_VOP(min_s32, neon_s32, 1)
423NEON_VOP(min_u32, neon_u32, 1)
424NEON_POP(pmin_s8, neon_s8, 4)
425NEON_POP(pmin_u8, neon_u8, 4)
426NEON_POP(pmin_s16, neon_s16, 2)
427NEON_POP(pmin_u16, neon_u16, 2)
428#undef NEON_FN
429
430#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
431NEON_VOP(max_s8, neon_s8, 4)
432NEON_VOP(max_u8, neon_u8, 4)
433NEON_VOP(max_s16, neon_s16, 2)
434NEON_VOP(max_u16, neon_u16, 2)
435NEON_VOP(max_s32, neon_s32, 1)
436NEON_VOP(max_u32, neon_u32, 1)
437NEON_POP(pmax_s8, neon_s8, 4)
438NEON_POP(pmax_u8, neon_u8, 4)
439NEON_POP(pmax_s16, neon_s16, 2)
440NEON_POP(pmax_u16, neon_u16, 2)
441#undef NEON_FN
442
443#define NEON_FN(dest, src1, src2) \
444 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
445NEON_VOP(abd_s8, neon_s8, 4)
446NEON_VOP(abd_u8, neon_u8, 4)
447NEON_VOP(abd_s16, neon_s16, 2)
448NEON_VOP(abd_u16, neon_u16, 2)
449NEON_VOP(abd_s32, neon_s32, 1)
450NEON_VOP(abd_u32, neon_u32, 1)
451#undef NEON_FN
452
453#define NEON_FN(dest, src1, src2) do { \
454 int8_t tmp; \
455 tmp = (int8_t)src2; \
50f67e95
JR		 456 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
457 tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 458 dest = 0; \
459 } else if (tmp < 0) { \
460 dest = src1 >> -tmp; \
461 } else { \
462 dest = src1 << tmp; \
463 }} while (0)
464NEON_VOP(shl_u8, neon_u8, 4)
465NEON_VOP(shl_u16, neon_u16, 2)
466NEON_VOP(shl_u32, neon_u32, 1)
467#undef NEON_FN
468
469uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
470{
471 int8_t shift = (int8_t)shiftop;
472 if (shift >= 64 || shift <= -64) {
473 val = 0;
474 } else if (shift < 0) {
475 val >>= -shift;
476 } else {
477 val <<= shift;
478 }
479 return val;
480}
481
482#define NEON_FN(dest, src1, src2) do { \
483 int8_t tmp; \
484 tmp = (int8_t)src2; \
50f67e95 485 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c 486 dest = 0; \
50f67e95 487 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 488 dest = src1 >> (sizeof(src1) * 8 - 1); \
489 } else if (tmp < 0) { \
490 dest = src1 >> -tmp; \
491 } else { \
492 dest = src1 << tmp; \
493 }} while (0)
494NEON_VOP(shl_s8, neon_s8, 4)
495NEON_VOP(shl_s16, neon_s16, 2)
496NEON_VOP(shl_s32, neon_s32, 1)
497#undef NEON_FN
498
499uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
500{
501 int8_t shift = (int8_t)shiftop;
502 int64_t val = valop;
503 if (shift >= 64) {
504 val = 0;
505 } else if (shift <= -64) {
506 val >>= 63;
507 } else if (shift < 0) {
508 val >>= -shift;
509 } else {
510 val <<= shift;
511 }
512 return val;
513}
514
515#define NEON_FN(dest, src1, src2) do { \
516 int8_t tmp; \
517 tmp = (int8_t)src2; \
0670a7b6
PM		 518 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
519 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
ad69471c 520 dest = 0; \
ad69471c
PB		 521 } else if (tmp < 0) { \
522 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
523 } else { \
524 dest = src1 << tmp; \
525 }} while (0)
526NEON_VOP(rshl_s8, neon_s8, 4)
527NEON_VOP(rshl_s16, neon_s16, 2)
ad69471c
PB		 528#undef NEON_FN
529
4bd4ee07
CL		 530/* The addition of the rounding constant may overflow, so we use an
531 * intermediate 64 bits accumulator. */
532uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
533{
534 int32_t dest;
535 int32_t val = (int32_t)valop;
536 int8_t shift = (int8_t)shiftop;
537 if ((shift >= 32) || (shift <= -32)) {
538 dest = 0;
539 } else if (shift < 0) {
540 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
541 dest = big_dest >> -shift;
542 } else {
543 dest = val << shift;
544 }
545 return dest;
546}
547
548/* Handling addition overflow with 64 bits inputs values is more
549 * tricky than with 32 bits values. */
ad69471c
PB		 550uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
551{
552 int8_t shift = (int8_t)shiftop;
553 int64_t val = valop;
0670a7b6 554 if ((shift >= 64) || (shift <= -64)) {
ad69471c 555 val = 0;
ad69471c 556 } else if (shift < 0) {
4bd4ee07
CL		 557 val >>= (-shift - 1);
558 if (val == INT64_MAX) {
559 /* In this case, it means that the rounding constant is 1,
560 * and the addition would overflow. Return the actual
561 * result directly. */
562 val = 0x4000000000000000LL;
563 } else {
564 val++;
565 val >>= 1;
566 }
ad69471c
PB		 567 } else {
568 val <<= shift;
569 }
570 return val;
571}
572
573#define NEON_FN(dest, src1, src2) do { \
574 int8_t tmp; \
575 tmp = (int8_t)src2; \
50f67e95
JR		 576 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
577 tmp < -(ssize_t)sizeof(src1) * 8) { \
ad69471c 578 dest = 0; \
50f67e95 579 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
b6c63b98 580 dest = src1 >> (-tmp - 1); \
ad69471c
PB		 581 } else if (tmp < 0) { \
582 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
583 } else { \
584 dest = src1 << tmp; \
585 }} while (0)
586NEON_VOP(rshl_u8, neon_u8, 4)
587NEON_VOP(rshl_u16, neon_u16, 2)
ad69471c
PB		 588#undef NEON_FN
589
4bd4ee07
CL		 590/* The addition of the rounding constant may overflow, so we use an
591 * intermediate 64 bits accumulator. */
592uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
593{
594 uint32_t dest;
595 int8_t shift = (int8_t)shiftop;
596 if (shift >= 32 || shift < -32) {
597 dest = 0;
598 } else if (shift == -32) {
599 dest = val >> 31;
600 } else if (shift < 0) {
601 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
602 dest = big_dest >> -shift;
603 } else {
604 dest = val << shift;
605 }
606 return dest;
607}
608
609/* Handling addition overflow with 64 bits inputs values is more
610 * tricky than with 32 bits values. */
ad69471c
PB		 611uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
612{
613 int8_t shift = (uint8_t)shiftop;
51e3930f 614 if (shift >= 64 || shift < -64) {
ad69471c
PB		 615 val = 0;
616 } else if (shift == -64) {
617 /* Rounding a 1-bit result just preserves that bit. */
618 val >>= 63;
4bd4ee07
CL		 619 } else if (shift < 0) {
620 val >>= (-shift - 1);
621 if (val == UINT64_MAX) {
622 /* In this case, it means that the rounding constant is 1,
623 * and the addition would overflow. Return the actual
624 * result directly. */
625 val = 0x8000000000000000ULL;
626 } else {
627 val++;
628 val >>= 1;
629 }
ad69471c
PB		 630 } else {
631 val <<= shift;
632 }
633 return val;
634}
635
636#define NEON_FN(dest, src1, src2) do { \
637 int8_t tmp; \
638 tmp = (int8_t)src2; \
50f67e95 639 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 640 if (src1) { \
641 SET_QC(); \
642 dest = ~0; \
643 } else { \
644 dest = 0; \
645 } \
50f67e95 646 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 647 dest = 0; \
648 } else if (tmp < 0) { \
649 dest = src1 >> -tmp; \
650 } else { \
651 dest = src1 << tmp; \
652 if ((dest >> tmp) != src1) { \
653 SET_QC(); \
654 dest = ~0; \
655 } \
656 }} while (0)
2a3f75b4
PM		 657NEON_VOP(qshl_u8, neon_u8, 4)
658NEON_VOP(qshl_u16, neon_u16, 2)
659NEON_VOP(qshl_u32, neon_u32, 1)
ad69471c
PB		 660#undef NEON_FN
661
2a3f75b4 662uint64_t HELPER(neon_qshl_u64)(uint64_t val, uint64_t shiftop)
ad69471c
PB		 663{
664 int8_t shift = (int8_t)shiftop;
665 if (shift >= 64) {
666 if (val) {
667 val = ~(uint64_t)0;
668 SET_QC();
ad69471c
PB		 669 }
670 } else if (shift <= -64) {
671 val = 0;
672 } else if (shift < 0) {
673 val >>= -shift;
674 } else {
675 uint64_t tmp = val;
676 val <<= shift;
677 if ((val >> shift) != tmp) {
678 SET_QC();
679 val = ~(uint64_t)0;
680 }
681 }
682 return val;
683}
684
685#define NEON_FN(dest, src1, src2) do { \
686 int8_t tmp; \
687 tmp = (int8_t)src2; \
50f67e95 688 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
a5d88f3e 689 if (src1) { \
ad69471c 690 SET_QC(); \
a5d88f3e
PM		 691 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
692 if (src1 > 0) { \
693 dest--; \
694 } \
695 } else { \
696 dest = src1; \
697 } \
50f67e95 698 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 699 dest = src1 >> 31; \
700 } else if (tmp < 0) { \
701 dest = src1 >> -tmp; \
702 } else { \
703 dest = src1 << tmp; \
704 if ((dest >> tmp) != src1) { \
705 SET_QC(); \
a5d88f3e
PM		 706 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
707 if (src1 > 0) { \
708 dest--; \
709 } \
ad69471c
PB		 710 } \
711 }} while (0)
2a3f75b4
PM		 712NEON_VOP(qshl_s8, neon_s8, 4)
713NEON_VOP(qshl_s16, neon_s16, 2)
714NEON_VOP(qshl_s32, neon_s32, 1)
ad69471c
PB		 715#undef NEON_FN
716
2a3f75b4 717uint64_t HELPER(neon_qshl_s64)(uint64_t valop, uint64_t shiftop)
ad69471c
PB		 718{
719 int8_t shift = (uint8_t)shiftop;
720 int64_t val = valop;
721 if (shift >= 64) {
722 if (val) {
723 SET_QC();
eb7a3d79 724 val = (val >> 63) ^ ~SIGNBIT64;
ad69471c 725 }
4c9b70ae 726 } else if (shift <= -64) {
ad69471c
PB		 727 val >>= 63;
728 } else if (shift < 0) {
729 val >>= -shift;
730 } else {
731 int64_t tmp = val;
732 val <<= shift;
733 if ((val >> shift) != tmp) {
734 SET_QC();
735 val = (tmp >> 63) ^ ~SIGNBIT64;
736 }
737 }
738 return val;
739}
740
4ca4502c
JR		 741#define NEON_FN(dest, src1, src2) do { \
742 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
743 SET_QC(); \
744 dest = 0; \
745 } else { \
746 int8_t tmp; \
747 tmp = (int8_t)src2; \
748 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
749 if (src1) { \
750 SET_QC(); \
751 dest = ~0; \
752 } else { \
753 dest = 0; \
754 } \
755 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
756 dest = 0; \
757 } else if (tmp < 0) { \
758 dest = src1 >> -tmp; \
759 } else { \
760 dest = src1 << tmp; \
761 if ((dest >> tmp) != src1) { \
762 SET_QC(); \
763 dest = ~0; \
764 } \
765 } \
766 }} while (0)
2a3f75b4
PM		 767NEON_VOP(qshlu_s8, neon_u8, 4)
768NEON_VOP(qshlu_s16, neon_u16, 2)
4ca4502c
JR		 769#undef NEON_FN
770
2a3f75b4 771uint32_t HELPER(neon_qshlu_s32)(uint32_t valop, uint32_t shiftop)
4ca4502c
JR		 772{
773 if ((int32_t)valop < 0) {
774 SET_QC();
775 return 0;
776 }
2a3f75b4 777 return helper_neon_qshl_u32(valop, shiftop);
4ca4502c
JR		 778}
779
2a3f75b4 780uint64_t HELPER(neon_qshlu_s64)(uint64_t valop, uint64_t shiftop)
4ca4502c
JR		 781{
782 if ((int64_t)valop < 0) {
783 SET_QC();
784 return 0;
785 }
2a3f75b4 786 return helper_neon_qshl_u64(valop, shiftop);
4ca4502c 787}
ad69471c
PB		 788
789/* FIXME: This is wrong. */
790#define NEON_FN(dest, src1, src2) do { \
791 int8_t tmp; \
792 tmp = (int8_t)src2; \
33ebc293
PM		 793 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
794 if (src1) { \
795 SET_QC(); \
796 dest = ~0; \
797 } else { \
798 dest = 0; \
799 } \
800 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
801 dest = 0; \
802 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
803 dest = src1 >> (sizeof(src1) * 8 - 1); \
804 } else if (tmp < 0) { \
ad69471c
PB		 805 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
806 } else { \
807 dest = src1 << tmp; \
808 if ((dest >> tmp) != src1) { \
809 SET_QC(); \
810 dest = ~0; \
811 } \
812 }} while (0)
2a3f75b4
PM		 813NEON_VOP(qrshl_u8, neon_u8, 4)
814NEON_VOP(qrshl_u16, neon_u16, 2)
ad69471c
PB		 815#undef NEON_FN
816
4bd4ee07
CL		 817/* The addition of the rounding constant may overflow, so we use an
818 * intermediate 64 bits accumulator. */
2a3f75b4 819uint32_t HELPER(neon_qrshl_u32)(uint32_t val, uint32_t shiftop)
4bd4ee07
CL		 820{
821 uint32_t dest;
822 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 823 if (shift >= 32) {
824 if (val) {
825 SET_QC();
826 dest = ~0;
827 } else {
828 dest = 0;
829 }
830 } else if (shift < -32) {
831 dest = 0;
832 } else if (shift == -32) {
833 dest = val >> 31;
834 } else if (shift < 0) {
4bd4ee07
CL		 835 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
836 dest = big_dest >> -shift;
837 } else {
838 dest = val << shift;
839 if ((dest >> shift) != val) {
840 SET_QC();
841 dest = ~0;
842 }
843 }
844 return dest;
845}
846
847/* Handling addition overflow with 64 bits inputs values is more
848 * tricky than with 32 bits values. */
2a3f75b4 849uint64_t HELPER(neon_qrshl_u64)(uint64_t val, uint64_t shiftop)
ad69471c
PB		 850{
851 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 852 if (shift >= 64) {
853 if (val) {
854 SET_QC();
855 val = ~0;
856 }
857 } else if (shift < -64) {
858 val = 0;
859 } else if (shift == -64) {
860 val >>= 63;
861 } else if (shift < 0) {
4bd4ee07
CL		 862 val >>= (-shift - 1);
863 if (val == UINT64_MAX) {
864 /* In this case, it means that the rounding constant is 1,
865 * and the addition would overflow. Return the actual
866 * result directly. */
867 val = 0x8000000000000000ULL;
868 } else {
869 val++;
870 val >>= 1;
871 }
ad69471c
PB		 872 } else { \
873 uint64_t tmp = val;
874 val <<= shift;
875 if ((val >> shift) != tmp) {
876 SET_QC();
877 val = ~0;
878 }
879 }
880 return val;
881}
882
883#define NEON_FN(dest, src1, src2) do { \
884 int8_t tmp; \
885 tmp = (int8_t)src2; \
7b6ecf5b
PM		 886 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
887 if (src1) { \
888 SET_QC(); \
889 dest = (1 << (sizeof(src1) * 8 - 1)); \
890 if (src1 > 0) { \
891 dest--; \
892 } \
893 } else { \
894 dest = 0; \
895 } \
896 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
897 dest = 0; \
898 } else if (tmp < 0) { \
ad69471c
PB		 899 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
900 } else { \
901 dest = src1 << tmp; \
902 if ((dest >> tmp) != src1) { \
903 SET_QC(); \
960e623b
PM		 904 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
905 if (src1 > 0) { \
906 dest--; \
907 } \
ad69471c
PB		 908 } \
909 }} while (0)
2a3f75b4
PM		 910NEON_VOP(qrshl_s8, neon_s8, 4)
911NEON_VOP(qrshl_s16, neon_s16, 2)
ad69471c
PB		 912#undef NEON_FN
913
4bd4ee07
CL		 914/* The addition of the rounding constant may overflow, so we use an
915 * intermediate 64 bits accumulator. */
2a3f75b4 916uint32_t HELPER(neon_qrshl_s32)(uint32_t valop, uint32_t shiftop)
4bd4ee07
CL		 917{
918 int32_t dest;
919 int32_t val = (int32_t)valop;
920 int8_t shift = (int8_t)shiftop;
7b6ecf5b
PM		 921 if (shift >= 32) {
922 if (val) {
923 SET_QC();
924 dest = (val >> 31) ^ ~SIGNBIT;
925 } else {
926 dest = 0;
927 }
928 } else if (shift <= -32) {
929 dest = 0;
930 } else if (shift < 0) {
4bd4ee07
CL		 931 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
932 dest = big_dest >> -shift;
933 } else {
934 dest = val << shift;
935 if ((dest >> shift) != val) {
936 SET_QC();
937 dest = (val >> 31) ^ ~SIGNBIT;
938 }
939 }
940 return dest;
941}
942
943/* Handling addition overflow with 64 bits inputs values is more
944 * tricky than with 32 bits values. */
2a3f75b4 945uint64_t HELPER(neon_qrshl_s64)(uint64_t valop, uint64_t shiftop)
ad69471c
PB		 946{
947 int8_t shift = (uint8_t)shiftop;
948 int64_t val = valop;
949
7b6ecf5b
PM		 950 if (shift >= 64) {
951 if (val) {
952 SET_QC();
953 val = (val >> 63) ^ ~SIGNBIT64;
954 }
955 } else if (shift <= -64) {
956 val = 0;
957 } else if (shift < 0) {
4bd4ee07
CL		 958 val >>= (-shift - 1);
959 if (val == INT64_MAX) {
960 /* In this case, it means that the rounding constant is 1,
961 * and the addition would overflow. Return the actual
962 * result directly. */
963 val = 0x4000000000000000ULL;
964 } else {
965 val++;
966 val >>= 1;
967 }
ad69471c 968 } else {
4bd4ee07 969 int64_t tmp = val;
ad69471c
PB		 970 val <<= shift;
971 if ((val >> shift) != tmp) {
972 SET_QC();
4bd4ee07 973 val = (tmp >> 63) ^ ~SIGNBIT64;
ad69471c
PB		 974 }
975 }
976 return val;
977}
978
979uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
980{
981 uint32_t mask;
982 mask = (a ^ b) & 0x80808080u;
983 a &= ~0x80808080u;
984 b &= ~0x80808080u;
985 return (a + b) ^ mask;
986}
987
988uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
989{
990 uint32_t mask;
991 mask = (a ^ b) & 0x80008000u;
992 a &= ~0x80008000u;
993 b &= ~0x80008000u;
994 return (a + b) ^ mask;
995}
996
997#define NEON_FN(dest, src1, src2) dest = src1 + src2
998NEON_POP(padd_u8, neon_u8, 4)
999NEON_POP(padd_u16, neon_u16, 2)
1000#undef NEON_FN
1001
1002#define NEON_FN(dest, src1, src2) dest = src1 - src2
1003NEON_VOP(sub_u8, neon_u8, 4)
1004NEON_VOP(sub_u16, neon_u16, 2)
1005#undef NEON_FN
1006
1007#define NEON_FN(dest, src1, src2) dest = src1 * src2
1008NEON_VOP(mul_u8, neon_u8, 4)
1009NEON_VOP(mul_u16, neon_u16, 2)
1010#undef NEON_FN
1011
1654b2d6 1012/* Polynomial multiplication is like integer multiplication except the
ad69471c
PB		 1013 partial products are XORed, not added. */
1014uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1015{
1016 uint32_t mask;
1017 uint32_t result;
1018 result = 0;
1019 while (op1) {
1020 mask = 0;
1021 if (op1 & 1)
1022 mask |= 0xff;
1023 if (op1 & (1 << 8))
1024 mask |= (0xff << 8);
1025 if (op1 & (1 << 16))
1026 mask |= (0xff << 16);
1027 if (op1 & (1 << 24))
1028 mask |= (0xff << 24);
1029 result ^= op2 & mask;
1030 op1 = (op1 >> 1) & 0x7f7f7f7f;
1031 op2 = (op2 << 1) & 0xfefefefe;
1032 }
1033 return result;
1034}
1035
e5ca24cb
PM		 1036uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1037{
1038 uint64_t result = 0;
1039 uint64_t mask;
1040 uint64_t op2ex = op2;
1041 op2ex = (op2ex & 0xff) |
1042 ((op2ex & 0xff00) << 8) |
1043 ((op2ex & 0xff0000) << 16) |
1044 ((op2ex & 0xff000000) << 24);
1045 while (op1) {
1046 mask = 0;
1047 if (op1 & 1) {
1048 mask |= 0xffff;
1049 }
1050 if (op1 & (1 << 8)) {
1051 mask |= (0xffffU << 16);
1052 }
1053 if (op1 & (1 << 16)) {
1054 mask |= (0xffffULL << 32);
1055 }
1056 if (op1 & (1 << 24)) {
1057 mask |= (0xffffULL << 48);
1058 }
1059 result ^= op2ex & mask;
1060 op1 = (op1 >> 1) & 0x7f7f7f7f;
1061 op2ex <<= 1;
1062 }
1063 return result;
1064}
1065
ad69471c
PB		 1066#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1067NEON_VOP(tst_u8, neon_u8, 4)
1068NEON_VOP(tst_u16, neon_u16, 2)
1069NEON_VOP(tst_u32, neon_u32, 1)
1070#undef NEON_FN
1071
1072#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1073NEON_VOP(ceq_u8, neon_u8, 4)
1074NEON_VOP(ceq_u16, neon_u16, 2)
1075NEON_VOP(ceq_u32, neon_u32, 1)
1076#undef NEON_FN
1077
1078#define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1079NEON_VOP1(abs_s8, neon_s8, 4)
1080NEON_VOP1(abs_s16, neon_s16, 2)
1081#undef NEON_FN
1082
1083/* Count Leading Sign/Zero Bits. */
1084static inline int do_clz8(uint8_t x)
1085{
1086 int n;
1087 for (n = 8; x; n--)
1088 x >>= 1;
1089 return n;
1090}
1091
1092static inline int do_clz16(uint16_t x)
1093{
1094 int n;
1095 for (n = 16; x; n--)
1096 x >>= 1;
1097 return n;
1098}
1099
1100#define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1101NEON_VOP1(clz_u8, neon_u8, 4)
1102#undef NEON_FN
1103
1104#define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1105NEON_VOP1(clz_u16, neon_u16, 2)
1106#undef NEON_FN
1107
1108#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1109NEON_VOP1(cls_s8, neon_s8, 4)
1110#undef NEON_FN
1111
1112#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1113NEON_VOP1(cls_s16, neon_s16, 2)
1114#undef NEON_FN
1115
1116uint32_t HELPER(neon_cls_s32)(uint32_t x)
1117{
1118 int count;
1119 if ((int32_t)x < 0)
1120 x = ~x;
1121 for (count = 32; x; count--)
1122 x = x >> 1;
1123 return count - 1;
1124}
1125
1126/* Bit count. */
1127uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1128{
1129 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1130 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1131 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1132 return x;
1133}
1134
1135#define NEON_QDMULH16(dest, src1, src2, round) do { \
1136 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1137 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1138 SET_QC(); \
1139 tmp = (tmp >> 31) ^ ~SIGNBIT; \
46eece9d
JR		 1140 } else { \
1141 tmp <<= 1; \
ad69471c 1142 } \
ad69471c
PB		 1143 if (round) { \
1144 int32_t old = tmp; \
1145 tmp += 1 << 15; \
1146 if ((int32_t)tmp < old) { \
1147 SET_QC(); \
1148 tmp = SIGNBIT - 1; \
1149 } \
1150 } \
1151 dest = tmp >> 16; \
1152 } while(0)
1153#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
2a3f75b4 1154NEON_VOP(qdmulh_s16, neon_s16, 2)
ad69471c
PB		 1155#undef NEON_FN
1156#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
2a3f75b4 1157NEON_VOP(qrdmulh_s16, neon_s16, 2)
ad69471c
PB		 1158#undef NEON_FN
1159#undef NEON_QDMULH16
1160
1161#define NEON_QDMULH32(dest, src1, src2, round) do { \
1162 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1163 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1164 SET_QC(); \
1165 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1166 } else { \
1167 tmp <<= 1; \
1168 } \
1169 if (round) { \
1170 int64_t old = tmp; \
1171 tmp += (int64_t)1 << 31; \
1172 if ((int64_t)tmp < old) { \
1173 SET_QC(); \
1174 tmp = SIGNBIT64 - 1; \
1175 } \
1176 } \
1177 dest = tmp >> 32; \
1178 } while(0)
1179#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
2a3f75b4 1180NEON_VOP(qdmulh_s32, neon_s32, 1)
ad69471c
PB		 1181#undef NEON_FN
1182#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
2a3f75b4 1183NEON_VOP(qrdmulh_s32, neon_s32, 1)
ad69471c
PB		 1184#undef NEON_FN
1185#undef NEON_QDMULH32
1186
1187uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1188{
1189 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1190 | ((x >> 24) & 0xff000000u);
1191}
1192
1193uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1194{
1195 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1196}
1197
1198uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1199{
1200 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1201 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1202}
1203
1204uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1205{
1206 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1207}
1208
1209uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1210{
1211 x &= 0xff80ff80ff80ff80ull;
1212 x += 0x0080008000800080ull;
1213 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1214 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1215}
1216
1217uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1218{
1219 x &= 0xffff8000ffff8000ull;
1220 x += 0x0000800000008000ull;
1221 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1222}
1223
2a3f75b4 1224uint32_t HELPER(neon_unarrow_sat8)(uint64_t x)
af1bbf30
JR		 1225{
1226 uint16_t s;
1227 uint8_t d;
1228 uint32_t res = 0;
1229#define SAT8(n) \
1230 s = x >> n; \
1231 if (s & 0x8000) { \
1232 SET_QC(); \
1233 } else { \
1234 if (s > 0xff) { \
1235 d = 0xff; \
1236 SET_QC(); \
1237 } else { \
1238 d = s; \
1239 } \
1240 res |= (uint32_t)d << (n / 2); \
1241 }
1242
1243 SAT8(0);
1244 SAT8(16);
1245 SAT8(32);
1246 SAT8(48);
1247#undef SAT8
1248 return res;
1249}
1250
2a3f75b4 1251uint32_t HELPER(neon_narrow_sat_u8)(uint64_t x)
ad69471c
PB		 1252{
1253 uint16_t s;
1254 uint8_t d;
1255 uint32_t res = 0;
1256#define SAT8(n) \
1257 s = x >> n; \
1258 if (s > 0xff) { \
1259 d = 0xff; \
1260 SET_QC(); \
1261 } else { \
1262 d = s; \
1263 } \
1264 res |= (uint32_t)d << (n / 2);
1265
1266 SAT8(0);
1267 SAT8(16);
1268 SAT8(32);
1269 SAT8(48);
1270#undef SAT8
1271 return res;
1272}
1273
2a3f75b4 1274uint32_t HELPER(neon_narrow_sat_s8)(uint64_t x)
ad69471c
PB		 1275{
1276 int16_t s;
1277 uint8_t d;
1278 uint32_t res = 0;
1279#define SAT8(n) \
1280 s = x >> n; \
1281 if (s != (int8_t)s) { \
1282 d = (s >> 15) ^ 0x7f; \
1283 SET_QC(); \
1284 } else { \
1285 d = s; \
1286 } \
1287 res |= (uint32_t)d << (n / 2);
1288
1289 SAT8(0);
1290 SAT8(16);
1291 SAT8(32);
1292 SAT8(48);
1293#undef SAT8
1294 return res;
1295}
1296
2a3f75b4 1297uint32_t HELPER(neon_unarrow_sat16)(uint64_t x)
af1bbf30
JR		 1298{
1299 uint32_t high;
1300 uint32_t low;
1301 low = x;
1302 if (low & 0x80000000) {
1303 low = 0;
1304 SET_QC();
1305 } else if (low > 0xffff) {
1306 low = 0xffff;
1307 SET_QC();
1308 }
1309 high = x >> 32;
1310 if (high & 0x80000000) {
1311 high = 0;
1312 SET_QC();
1313 } else if (high > 0xffff) {
1314 high = 0xffff;
1315 SET_QC();
1316 }
1317 return low | (high << 16);
1318}
1319
2a3f75b4 1320uint32_t HELPER(neon_narrow_sat_u16)(uint64_t x)
ad69471c
PB		 1321{
1322 uint32_t high;
1323 uint32_t low;
1324 low = x;
1325 if (low > 0xffff) {
1326 low = 0xffff;
1327 SET_QC();
1328 }
1329 high = x >> 32;
1330 if (high > 0xffff) {
1331 high = 0xffff;
1332 SET_QC();
1333 }
1334 return low | (high << 16);
1335}
1336
2a3f75b4 1337uint32_t HELPER(neon_narrow_sat_s16)(uint64_t x)
ad69471c
PB		 1338{
1339 int32_t low;
1340 int32_t high;
1341 low = x;
1342 if (low != (int16_t)low) {
1343 low = (low >> 31) ^ 0x7fff;
1344 SET_QC();
1345 }
1346 high = x >> 32;
1347 if (high != (int16_t)high) {
1348 high = (high >> 31) ^ 0x7fff;
1349 SET_QC();
1350 }
1351 return (uint16_t)low | (high << 16);
1352}
1353
2a3f75b4 1354uint32_t HELPER(neon_unarrow_sat32)(uint64_t x)
af1bbf30
JR		 1355{
1356 if (x & 0x8000000000000000ull) {
1357 SET_QC();
1358 return 0;
1359 }
1360 if (x > 0xffffffffu) {
1361 SET_QC();
1362 return 0xffffffffu;
1363 }
1364 return x;
1365}
1366
2a3f75b4 1367uint32_t HELPER(neon_narrow_sat_u32)(uint64_t x)
ad69471c
PB		 1368{
1369 if (x > 0xffffffffu) {
1370 SET_QC();
1371 return 0xffffffffu;
1372 }
1373 return x;
1374}
1375
2a3f75b4 1376uint32_t HELPER(neon_narrow_sat_s32)(uint64_t x)
ad69471c
PB		 1377{
1378 if ((int64_t)x != (int32_t)x) {
1379 SET_QC();
cc2212c2 1380 return ((int64_t)x >> 63) ^ 0x7fffffff;
ad69471c
PB		 1381 }
1382 return x;
1383}
1384
1385uint64_t HELPER(neon_widen_u8)(uint32_t x)
1386{
1387 uint64_t tmp;
1388 uint64_t ret;
1389 ret = (uint8_t)x;
1390 tmp = (uint8_t)(x >> 8);
1391 ret |= tmp << 16;
1392 tmp = (uint8_t)(x >> 16);
1393 ret |= tmp << 32;
1394 tmp = (uint8_t)(x >> 24);
1395 ret |= tmp << 48;
1396 return ret;
1397}
1398
1399uint64_t HELPER(neon_widen_s8)(uint32_t x)
1400{
1401 uint64_t tmp;
1402 uint64_t ret;
1403 ret = (uint16_t)(int8_t)x;
1404 tmp = (uint16_t)(int8_t)(x >> 8);
1405 ret |= tmp << 16;
1406 tmp = (uint16_t)(int8_t)(x >> 16);
1407 ret |= tmp << 32;
1408 tmp = (uint16_t)(int8_t)(x >> 24);
1409 ret |= tmp << 48;
1410 return ret;
1411}
1412
1413uint64_t HELPER(neon_widen_u16)(uint32_t x)
1414{
1415 uint64_t high = (uint16_t)(x >> 16);
1416 return ((uint16_t)x) | (high << 32);
1417}
1418
1419uint64_t HELPER(neon_widen_s16)(uint32_t x)
1420{
1421 uint64_t high = (int16_t)(x >> 16);
1422 return ((uint32_t)(int16_t)x) | (high << 32);
1423}
1424
1425uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1426{
1427 uint64_t mask;
1428 mask = (a ^ b) & 0x8000800080008000ull;
1429 a &= ~0x8000800080008000ull;
1430 b &= ~0x8000800080008000ull;
1431 return (a + b) ^ mask;
1432}
1433
1434uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1435{
1436 uint64_t mask;
1437 mask = (a ^ b) & 0x8000000080000000ull;
1438 a &= ~0x8000000080000000ull;
1439 b &= ~0x8000000080000000ull;
1440 return (a + b) ^ mask;
1441}
1442
1443uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1444{
1445 uint64_t tmp;
1446 uint64_t tmp2;
1447
1448 tmp = a & 0x0000ffff0000ffffull;
1449 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1450 tmp2 = b & 0xffff0000ffff0000ull;
1451 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1452 return ( tmp & 0xffff)
1453 | ((tmp >> 16) & 0xffff0000ull)
1454 | ((tmp2 << 16) & 0xffff00000000ull)
1455 | ( tmp2 & 0xffff000000000000ull);
1456}
1457
1458uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1459{
1460 uint32_t low = a + (a >> 32);
1461 uint32_t high = b + (b >> 32);
1462 return low + ((uint64_t)high << 32);
1463}
1464
1465uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1466{
1467 uint64_t mask;
1468 mask = (a ^ ~b) & 0x8000800080008000ull;
1469 a |= 0x8000800080008000ull;
1470 b &= ~0x8000800080008000ull;
1471 return (a - b) ^ mask;
1472}
1473
1474uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1475{
1476 uint64_t mask;
1477 mask = (a ^ ~b) & 0x8000000080000000ull;
1478 a |= 0x8000000080000000ull;
1479 b &= ~0x8000000080000000ull;
1480 return (a - b) ^ mask;
1481}
1482
2a3f75b4 1483uint64_t HELPER(neon_addl_saturate_s32)(uint64_t a, uint64_t b)
ad69471c
PB		 1484{
1485 uint32_t x, y;
1486 uint32_t low, high;
1487
1488 x = a;
1489 y = b;
1490 low = x + y;
1491 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1492 SET_QC();
1493 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1494 }
1495 x = a >> 32;
1496 y = b >> 32;
1497 high = x + y;
1498 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1499 SET_QC();
1500 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1501 }
1502 return low | ((uint64_t)high << 32);
1503}
1504
2a3f75b4 1505uint64_t HELPER(neon_addl_saturate_s64)(uint64_t a, uint64_t b)
ad69471c
PB		 1506{
1507 uint64_t result;
1508
1509 result = a + b;
1510 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1511 SET_QC();
1512 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1513 }
1514 return result;
1515}
1516
4d9ad7f7
PM		 1517/* We have to do the arithmetic in a larger type than
1518 * the input type, because for example with a signed 32 bit
1519 * op the absolute difference can overflow a signed 32 bit value.
1520 */
1521#define DO_ABD(dest, x, y, intype, arithtype) do { \
1522 arithtype tmp_x = (intype)(x); \
1523 arithtype tmp_y = (intype)(y); \
ad69471c
PB		 1524 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1525 } while(0)
1526
1527uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1528{
1529 uint64_t tmp;
1530 uint64_t result;
4d9ad7f7
PM		 1531 DO_ABD(result, a, b, uint8_t, uint32_t);
1532 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
ad69471c 1533 result |= tmp << 16;
4d9ad7f7 1534 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
ad69471c 1535 result |= tmp << 32;
4d9ad7f7 1536 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
ad69471c
PB		 1537 result |= tmp << 48;
1538 return result;
1539}
1540
1541uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1542{
1543 uint64_t tmp;
1544 uint64_t result;
4d9ad7f7
PM		 1545 DO_ABD(result, a, b, int8_t, int32_t);
1546 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
ad69471c 1547 result |= tmp << 16;
4d9ad7f7 1548 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
ad69471c 1549 result |= tmp << 32;
4d9ad7f7 1550 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
ad69471c
PB		 1551 result |= tmp << 48;
1552 return result;
1553}
1554
1555uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1556{
1557 uint64_t tmp;
1558 uint64_t result;
4d9ad7f7
PM		 1559 DO_ABD(result, a, b, uint16_t, uint32_t);
1560 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
ad69471c
PB		 1561 return result | (tmp << 32);
1562}
1563
1564uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1565{
1566 uint64_t tmp;
1567 uint64_t result;
4d9ad7f7
PM		 1568 DO_ABD(result, a, b, int16_t, int32_t);
1569 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
ad69471c
PB		 1570 return result | (tmp << 32);
1571}
1572
1573uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1574{
1575 uint64_t result;
4d9ad7f7 1576 DO_ABD(result, a, b, uint32_t, uint64_t);
ad69471c
PB		 1577 return result;
1578}
1579
1580uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1581{
1582 uint64_t result;
4d9ad7f7 1583 DO_ABD(result, a, b, int32_t, int64_t);
ad69471c
PB		 1584 return result;
1585}
1586#undef DO_ABD
1587
1588/* Widening multiply. Named type is the source type. */
1589#define DO_MULL(dest, x, y, type1, type2) do { \
1590 type1 tmp_x = x; \
1591 type1 tmp_y = y; \
1592 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1593 } while(0)
1594
1595uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1596{
1597 uint64_t tmp;
1598 uint64_t result;
1599
1600 DO_MULL(result, a, b, uint8_t, uint16_t);
1601 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1602 result |= tmp << 16;
1603 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1604 result |= tmp << 32;
1605 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1606 result |= tmp << 48;
1607 return result;
1608}
1609
1610uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1611{
1612 uint64_t tmp;
1613 uint64_t result;
1614
1615 DO_MULL(result, a, b, int8_t, uint16_t);
1616 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1617 result |= tmp << 16;
1618 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1619 result |= tmp << 32;
1620 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1621 result |= tmp << 48;
1622 return result;
1623}
1624
1625uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1626{
1627 uint64_t tmp;
1628 uint64_t result;
1629
1630 DO_MULL(result, a, b, uint16_t, uint32_t);
1631 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1632 return result | (tmp << 32);
1633}
1634
1635uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1636{
1637 uint64_t tmp;
1638 uint64_t result;
1639
1640 DO_MULL(result, a, b, int16_t, uint32_t);
1641 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1642 return result | (tmp << 32);
1643}
1644
1645uint64_t HELPER(neon_negl_u16)(uint64_t x)
1646{
1647 uint16_t tmp;
1648 uint64_t result;
1649 result = (uint16_t)-x;
1650 tmp = -(x >> 16);
1651 result |= (uint64_t)tmp << 16;
1652 tmp = -(x >> 32);
1653 result |= (uint64_t)tmp << 32;
1654 tmp = -(x >> 48);
1655 result |= (uint64_t)tmp << 48;
1656 return result;
1657}
1658
ad69471c
PB		 1659uint64_t HELPER(neon_negl_u32)(uint64_t x)
1660{
1661 uint32_t low = -x;
1662 uint32_t high = -(x >> 32);
1663 return low | ((uint64_t)high << 32);
1664}
1665
1666/* FIXME: There should be a native op for this. */
1667uint64_t HELPER(neon_negl_u64)(uint64_t x)
1668{
1669 return -x;
1670}
1671
1672/* Saturnating sign manuipulation. */
1673/* ??? Make these use NEON_VOP1 */
1674#define DO_QABS8(x) do { \
1675 if (x == (int8_t)0x80) { \
1676 x = 0x7f; \
1677 SET_QC(); \
1678 } else if (x < 0) { \
1679 x = -x; \
1680 }} while (0)
2a3f75b4 1681uint32_t HELPER(neon_qabs_s8)(uint32_t x)
ad69471c
PB		 1682{
1683 neon_s8 vec;
1684 NEON_UNPACK(neon_s8, vec, x);
1685 DO_QABS8(vec.v1);
1686 DO_QABS8(vec.v2);
1687 DO_QABS8(vec.v3);
1688 DO_QABS8(vec.v4);
1689 NEON_PACK(neon_s8, x, vec);
1690 return x;
1691}
1692#undef DO_QABS8
1693
1694#define DO_QNEG8(x) do { \
1695 if (x == (int8_t)0x80) { \
1696 x = 0x7f; \
1697 SET_QC(); \
1698 } else { \
1699 x = -x; \
1700 }} while (0)
2a3f75b4 1701uint32_t HELPER(neon_qneg_s8)(uint32_t x)
ad69471c
PB		 1702{
1703 neon_s8 vec;
1704 NEON_UNPACK(neon_s8, vec, x);
1705 DO_QNEG8(vec.v1);
1706 DO_QNEG8(vec.v2);
1707 DO_QNEG8(vec.v3);
1708 DO_QNEG8(vec.v4);
1709 NEON_PACK(neon_s8, x, vec);
1710 return x;
1711}
1712#undef DO_QNEG8
1713
1714#define DO_QABS16(x) do { \
1715 if (x == (int16_t)0x8000) { \
1716 x = 0x7fff; \
1717 SET_QC(); \
1718 } else if (x < 0) { \
1719 x = -x; \
1720 }} while (0)
2a3f75b4 1721uint32_t HELPER(neon_qabs_s16)(uint32_t x)
ad69471c
PB		 1722{
1723 neon_s16 vec;
1724 NEON_UNPACK(neon_s16, vec, x);
1725 DO_QABS16(vec.v1);
1726 DO_QABS16(vec.v2);
1727 NEON_PACK(neon_s16, x, vec);
1728 return x;
1729}
1730#undef DO_QABS16
1731
1732#define DO_QNEG16(x) do { \
1733 if (x == (int16_t)0x8000) { \
1734 x = 0x7fff; \
1735 SET_QC(); \
1736 } else { \
1737 x = -x; \
1738 }} while (0)
2a3f75b4 1739uint32_t HELPER(neon_qneg_s16)(uint32_t x)
ad69471c
PB		 1740{
1741 neon_s16 vec;
1742 NEON_UNPACK(neon_s16, vec, x);
1743 DO_QNEG16(vec.v1);
1744 DO_QNEG16(vec.v2);
1745 NEON_PACK(neon_s16, x, vec);
1746 return x;
1747}
1748#undef DO_QNEG16
1749
2a3f75b4 1750uint32_t HELPER(neon_qabs_s32)(uint32_t x)
ad69471c
PB		 1751{
1752 if (x == SIGNBIT) {
1753 SET_QC();
1754 x = ~SIGNBIT;
1755 } else if ((int32_t)x < 0) {
1756 x = -x;
1757 }
1758 return x;
1759}
1760
2a3f75b4 1761uint32_t HELPER(neon_qneg_s32)(uint32_t x)
ad69471c
PB		 1762{
1763 if (x == SIGNBIT) {
1764 SET_QC();
1765 x = ~SIGNBIT;
1766 } else {
1767 x = -x;
1768 }
1769 return x;
1770}
1771
1772/* NEON Float helpers. */
1773uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1774{
4a9f9cb2 1775 return float32_val(float32_min(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1776}
1777
1778uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1779{
4a9f9cb2 1780 return float32_val(float32_max(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1781}
1782
1783uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1784{
51d85267
PM		 1785 float32 f0 = make_float32(a);
1786 float32 f1 = make_float32(b);
79c18be7 1787 return float32_val(float32_abs(float32_sub(f0, f1, NFS)));
ad69471c
PB		 1788}
1789
1790uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1791{
51d85267 1792 return float32_val(float32_add(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1793}
1794
1795uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1796{
51d85267 1797 return float32_val(float32_sub(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1798}
1799
1800uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1801{
51d85267 1802 return float32_val(float32_mul(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1803}
1804
cab565c4
PM		 1805/* Floating point comparisons produce an integer result.
1806 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1807 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1808 */
1809uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b)
1810{
1811 return -float32_eq_quiet(make_float32(a), make_float32(b), NFS);
1812}
1813
1814uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b)
1815{
1816 return -float32_le(make_float32(b), make_float32(a), NFS);
ad69471c
PB		 1817}
1818
cab565c4
PM		 1819uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b)
1820{
1821 return -float32_lt(make_float32(b), make_float32(a), NFS);
1822}
ad69471c
PB		 1823
1824uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1825{
51d85267
PM		 1826 float32 f0 = float32_abs(make_float32(a));
1827 float32 f1 = float32_abs(make_float32(b));
cab565c4 1828 return -float32_le(f1, f0, NFS);
ad69471c
PB		 1829}
1830
1831uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1832{
51d85267
PM		 1833 float32 f0 = float32_abs(make_float32(a));
1834 float32 f1 = float32_abs(make_float32(b));
cab565c4 1835 return -float32_lt(f1, f0, NFS);
ad69471c 1836}
02acedf9
PM		 1837
1838#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1839
2a3f75b4 1840void HELPER(neon_qunzip8)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1841{
1842 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1843 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1844 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1845 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1846 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1847 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1848 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1849 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1850 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1851 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1852 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1853 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1854 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1855 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1856 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1857 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1858 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1859 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1860 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1861 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1862 env->vfp.regs[rm] = make_float64(m0);
1863 env->vfp.regs[rm + 1] = make_float64(m1);
1864 env->vfp.regs[rd] = make_float64(d0);
1865 env->vfp.regs[rd + 1] = make_float64(d1);
1866}
1867
2a3f75b4 1868void HELPER(neon_qunzip16)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1869{
1870 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1871 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1872 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1873 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1874 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1875 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1876 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1877 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1878 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1879 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1880 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1881 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1882 env->vfp.regs[rm] = make_float64(m0);
1883 env->vfp.regs[rm + 1] = make_float64(m1);
1884 env->vfp.regs[rd] = make_float64(d0);
1885 env->vfp.regs[rd + 1] = make_float64(d1);
1886}
1887
2a3f75b4 1888void HELPER(neon_qunzip32)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1889{
1890 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1891 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1892 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1893 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1894 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1895 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1896 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1897 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1898 env->vfp.regs[rm] = make_float64(m0);
1899 env->vfp.regs[rm + 1] = make_float64(m1);
1900 env->vfp.regs[rd] = make_float64(d0);
1901 env->vfp.regs[rd + 1] = make_float64(d1);
1902}
1903
2a3f75b4 1904void HELPER(neon_unzip8)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1905{
1906 uint64_t zm = float64_val(env->vfp.regs[rm]);
1907 uint64_t zd = float64_val(env->vfp.regs[rd]);
1908 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1909 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1910 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1911 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1912 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1913 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1914 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1915 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1916 env->vfp.regs[rm] = make_float64(m0);
1917 env->vfp.regs[rd] = make_float64(d0);
1918}
1919
2a3f75b4 1920void HELPER(neon_unzip16)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1921{
1922 uint64_t zm = float64_val(env->vfp.regs[rm]);
1923 uint64_t zd = float64_val(env->vfp.regs[rd]);
1924 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1925 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1926 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1927 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1928 env->vfp.regs[rm] = make_float64(m0);
1929 env->vfp.regs[rd] = make_float64(d0);
1930}
d68a6f3a 1931
2a3f75b4 1932void HELPER(neon_qzip8)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1933{
1934 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1935 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1936 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1937 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1938 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1939 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1940 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1941 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1942 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1943 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1944 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1945 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1946 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1947 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1948 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1949 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1950 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1951 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1952 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1953 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1954 env->vfp.regs[rm] = make_float64(m0);
1955 env->vfp.regs[rm + 1] = make_float64(m1);
1956 env->vfp.regs[rd] = make_float64(d0);
1957 env->vfp.regs[rd + 1] = make_float64(d1);
1958}
1959
2a3f75b4 1960void HELPER(neon_qzip16)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1961{
1962 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1963 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1964 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1965 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1966 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1967 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1968 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1969 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1970 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1971 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1972 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1973 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1974 env->vfp.regs[rm] = make_float64(m0);
1975 env->vfp.regs[rm + 1] = make_float64(m1);
1976 env->vfp.regs[rd] = make_float64(d0);
1977 env->vfp.regs[rd + 1] = make_float64(d1);
1978}
1979
2a3f75b4 1980void HELPER(neon_qzip32)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1981{
1982 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1983 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1984 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1985 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1986 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1987 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1988 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1989 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1990 env->vfp.regs[rm] = make_float64(m0);
1991 env->vfp.regs[rm + 1] = make_float64(m1);
1992 env->vfp.regs[rd] = make_float64(d0);
1993 env->vfp.regs[rd + 1] = make_float64(d1);
1994}
1995
2a3f75b4 1996void HELPER(neon_zip8)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1997{
1998 uint64_t zm = float64_val(env->vfp.regs[rm]);
1999 uint64_t zd = float64_val(env->vfp.regs[rd]);
2000 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2001 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2002 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2003 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2004 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2005 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2006 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2007 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2008 env->vfp.regs[rm] = make_float64(m0);
2009 env->vfp.regs[rd] = make_float64(d0);
2010}
2011
2a3f75b4 2012void HELPER(neon_zip16)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 2013{
2014 uint64_t zm = float64_val(env->vfp.regs[rm]);
2015 uint64_t zd = float64_val(env->vfp.regs[rd]);
2016 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2017 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2018 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2019 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2020 env->vfp.regs[rm] = make_float64(m0);
2021 env->vfp.regs[rd] = make_float64(d0);
2022}
Qemu copy for testing