]> git.proxmox.com Git - qemu.git/blame - target-arm/neon_helper.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target-arm: Use global env in neon_helper.c helpers
[qemu.git] / target-arm / neon_helper.c
CommitLineData
e677137d
PB		 1/*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licenced under the GNU GPL v2.
8 */
ad69471c
PB		 9#include <stdlib.h>
10#include <stdio.h>
11
12#include "cpu.h"
2a3f75b4 13#include "exec.h"
ad69471c
PB		 14#include "helpers.h"
15
16#define SIGNBIT (uint32_t)0x80000000
17#define SIGNBIT64 ((uint64_t)1 << 63)
18
19#define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
20
21static float_status neon_float_status;
22#define NFS &neon_float_status
23
ad69471c
PB		 24#define NEON_TYPE1(name, type) \
25typedef struct \
26{ \
27 type v1; \
28} neon_##name;
e2542fe2 29#ifdef HOST_WORDS_BIGENDIAN
ad69471c
PB		 30#define NEON_TYPE2(name, type) \
31typedef struct \
32{ \
33 type v2; \
34 type v1; \
35} neon_##name;
36#define NEON_TYPE4(name, type) \
37typedef struct \
38{ \
39 type v4; \
40 type v3; \
41 type v2; \
42 type v1; \
43} neon_##name;
44#else
45#define NEON_TYPE2(name, type) \
46typedef struct \
47{ \
48 type v1; \
49 type v2; \
50} neon_##name;
51#define NEON_TYPE4(name, type) \
52typedef struct \
53{ \
54 type v1; \
55 type v2; \
56 type v3; \
57 type v4; \
58} neon_##name;
59#endif
60
61NEON_TYPE4(s8, int8_t)
62NEON_TYPE4(u8, uint8_t)
63NEON_TYPE2(s16, int16_t)
64NEON_TYPE2(u16, uint16_t)
65NEON_TYPE1(s32, int32_t)
66NEON_TYPE1(u32, uint32_t)
67#undef NEON_TYPE4
68#undef NEON_TYPE2
69#undef NEON_TYPE1
70
71/* Copy from a uint32_t to a vector structure type. */
72#define NEON_UNPACK(vtype, dest, val) do { \
73 union { \
74 vtype v; \
75 uint32_t i; \
76 } conv_u; \
77 conv_u.i = (val); \
78 dest = conv_u.v; \
79 } while(0)
80
81/* Copy from a vector structure type to a uint32_t. */
82#define NEON_PACK(vtype, dest, val) do { \
83 union { \
84 vtype v; \
85 uint32_t i; \
86 } conv_u; \
87 conv_u.v = (val); \
88 dest = conv_u.i; \
89 } while(0)
90
91#define NEON_DO1 \
92 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
93#define NEON_DO2 \
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
96#define NEON_DO4 \
97 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
98 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
99 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
100 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
101
102#define NEON_VOP_BODY(vtype, n) \
103{ \
104 uint32_t res; \
105 vtype vsrc1; \
106 vtype vsrc2; \
107 vtype vdest; \
108 NEON_UNPACK(vtype, vsrc1, arg1); \
109 NEON_UNPACK(vtype, vsrc2, arg2); \
110 NEON_DO##n; \
111 NEON_PACK(vtype, res, vdest); \
112 return res; \
113}
114
115#define NEON_VOP(name, vtype, n) \
116uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
117NEON_VOP_BODY(vtype, n)
118
ad69471c
PB		 119/* Pairwise operations. */
120/* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
122#define NEON_PDO2 \
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
125#define NEON_PDO4 \
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
130
131#define NEON_POP(name, vtype, n) \
132uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
133{ \
134 uint32_t res; \
135 vtype vsrc1; \
136 vtype vsrc2; \
137 vtype vdest; \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
140 NEON_PDO##n; \
141 NEON_PACK(vtype, res, vdest); \
142 return res; \
143}
144
145/* Unary operators. */
146#define NEON_VOP1(name, vtype, n) \
147uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
148{ \
149 vtype vsrc1; \
150 vtype vdest; \
151 NEON_UNPACK(vtype, vsrc1, arg); \
152 NEON_DO##n; \
153 NEON_PACK(vtype, arg, vdest); \
154 return arg; \
155}
156
157
158#define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
161 SET_QC(); \
162 dest = ~0; \
163 } else { \
164 dest = tmp; \
165 }} while(0)
166#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
2a3f75b4 167NEON_VOP(qadd_u8, neon_u8, 4)
ad69471c
PB		 168#undef NEON_FN
169#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
2a3f75b4 170NEON_VOP(qadd_u16, neon_u16, 2)
ad69471c
PB		 171#undef NEON_FN
172#undef NEON_USAT
173
2a3f75b4 174uint32_t HELPER(neon_qadd_u32)(uint32_t a, uint32_t b)
72902672
CL		 175{
176 uint32_t res = a + b;
177 if (res < a) {
178 SET_QC();
179 res = ~0;
180 }
181 return res;
182}
183
2a3f75b4 184uint64_t HELPER(neon_qadd_u64)(uint64_t src1, uint64_t src2)
72902672
CL		 185{
186 uint64_t res;
187
188 res = src1 + src2;
189 if (res < src1) {
190 SET_QC();
191 res = ~(uint64_t)0;
192 }
193 return res;
194}
195
ad69471c
PB		 196#define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
199 SET_QC(); \
200 if (src2 > 0) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
202 } else { \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
204 } \
205 } \
206 dest = tmp; \
207 } while(0)
208#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
2a3f75b4 209NEON_VOP(qadd_s8, neon_s8, 4)
ad69471c
PB		 210#undef NEON_FN
211#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
2a3f75b4 212NEON_VOP(qadd_s16, neon_s16, 2)
ad69471c
PB		 213#undef NEON_FN
214#undef NEON_SSAT
215
2a3f75b4 216uint32_t HELPER(neon_qadd_s32)(uint32_t a, uint32_t b)
72902672
CL		 217{
218 uint32_t res = a + b;
219 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
220 SET_QC();
221 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
222 }
223 return res;
224}
225
2a3f75b4 226uint64_t HELPER(neon_qadd_s64)(uint64_t src1, uint64_t src2)
72902672
CL		 227{
228 uint64_t res;
229
230 res = src1 + src2;
231 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
232 SET_QC();
233 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
234 }
235 return res;
236}
237
ad69471c
PB		 238#define NEON_USAT(dest, src1, src2, type) do { \
239 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
240 if (tmp != (type)tmp) { \
241 SET_QC(); \
242 dest = 0; \
243 } else { \
244 dest = tmp; \
245 }} while(0)
246#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
2a3f75b4 247NEON_VOP(qsub_u8, neon_u8, 4)
ad69471c
PB		 248#undef NEON_FN
249#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
2a3f75b4 250NEON_VOP(qsub_u16, neon_u16, 2)
ad69471c
PB		 251#undef NEON_FN
252#undef NEON_USAT
253
2a3f75b4 254uint32_t HELPER(neon_qsub_u32)(uint32_t a, uint32_t b)
72902672
CL		 255{
256 uint32_t res = a - b;
257 if (res > a) {
258 SET_QC();
259 res = 0;
260 }
261 return res;
262}
263
2a3f75b4 264uint64_t HELPER(neon_qsub_u64)(uint64_t src1, uint64_t src2)
72902672
CL		 265{
266 uint64_t res;
267
268 if (src1 < src2) {
269 SET_QC();
270 res = 0;
271 } else {
272 res = src1 - src2;
273 }
274 return res;
275}
276
ad69471c
PB		 277#define NEON_SSAT(dest, src1, src2, type) do { \
278 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
279 if (tmp != (type)tmp) { \
280 SET_QC(); \
281 if (src2 < 0) { \
282 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
283 } else { \
284 tmp = 1 << (sizeof(type) * 8 - 1); \
285 } \
286 } \
287 dest = tmp; \
288 } while(0)
289#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
2a3f75b4 290NEON_VOP(qsub_s8, neon_s8, 4)
ad69471c
PB		 291#undef NEON_FN
292#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
2a3f75b4 293NEON_VOP(qsub_s16, neon_s16, 2)
ad69471c
PB		 294#undef NEON_FN
295#undef NEON_SSAT
296
2a3f75b4 297uint32_t HELPER(neon_qsub_s32)(uint32_t a, uint32_t b)
72902672
CL		 298{
299 uint32_t res = a - b;
300 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
301 SET_QC();
302 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
303 }
304 return res;
305}
306
2a3f75b4 307uint64_t HELPER(neon_qsub_s64)(uint64_t src1, uint64_t src2)
72902672
CL		 308{
309 uint64_t res;
310
311 res = src1 - src2;
312 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
313 SET_QC();
314 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
315 }
316 return res;
317}
318
ad69471c
PB		 319#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
320NEON_VOP(hadd_s8, neon_s8, 4)
321NEON_VOP(hadd_u8, neon_u8, 4)
322NEON_VOP(hadd_s16, neon_s16, 2)
323NEON_VOP(hadd_u16, neon_u16, 2)
324#undef NEON_FN
325
326int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
327{
328 int32_t dest;
329
330 dest = (src1 >> 1) + (src2 >> 1);
331 if (src1 & src2 & 1)
332 dest++;
333 return dest;
334}
335
336uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
337{
338 uint32_t dest;
339
340 dest = (src1 >> 1) + (src2 >> 1);
341 if (src1 & src2 & 1)
342 dest++;
343 return dest;
344}
345
346#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
347NEON_VOP(rhadd_s8, neon_s8, 4)
348NEON_VOP(rhadd_u8, neon_u8, 4)
349NEON_VOP(rhadd_s16, neon_s16, 2)
350NEON_VOP(rhadd_u16, neon_u16, 2)
351#undef NEON_FN
352
353int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
354{
355 int32_t dest;
356
357 dest = (src1 >> 1) + (src2 >> 1);
358 if ((src1 | src2) & 1)
359 dest++;
360 return dest;
361}
362
363uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
364{
365 uint32_t dest;
366
367 dest = (src1 >> 1) + (src2 >> 1);
368 if ((src1 | src2) & 1)
369 dest++;
370 return dest;
371}
372
373#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
374NEON_VOP(hsub_s8, neon_s8, 4)
375NEON_VOP(hsub_u8, neon_u8, 4)
376NEON_VOP(hsub_s16, neon_s16, 2)
377NEON_VOP(hsub_u16, neon_u16, 2)
378#undef NEON_FN
379
380int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
381{
382 int32_t dest;
383
384 dest = (src1 >> 1) - (src2 >> 1);
385 if ((~src1) & src2 & 1)
386 dest--;
387 return dest;
388}
389
390uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
391{
392 uint32_t dest;
393
394 dest = (src1 >> 1) - (src2 >> 1);
395 if ((~src1) & src2 & 1)
396 dest--;
397 return dest;
398}
399
400#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
401NEON_VOP(cgt_s8, neon_s8, 4)
402NEON_VOP(cgt_u8, neon_u8, 4)
403NEON_VOP(cgt_s16, neon_s16, 2)
404NEON_VOP(cgt_u16, neon_u16, 2)
405NEON_VOP(cgt_s32, neon_s32, 1)
406NEON_VOP(cgt_u32, neon_u32, 1)
407#undef NEON_FN
408
409#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
410NEON_VOP(cge_s8, neon_s8, 4)
411NEON_VOP(cge_u8, neon_u8, 4)
412NEON_VOP(cge_s16, neon_s16, 2)
413NEON_VOP(cge_u16, neon_u16, 2)
414NEON_VOP(cge_s32, neon_s32, 1)
415NEON_VOP(cge_u32, neon_u32, 1)
416#undef NEON_FN
417
418#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
419NEON_VOP(min_s8, neon_s8, 4)
420NEON_VOP(min_u8, neon_u8, 4)
421NEON_VOP(min_s16, neon_s16, 2)
422NEON_VOP(min_u16, neon_u16, 2)
423NEON_VOP(min_s32, neon_s32, 1)
424NEON_VOP(min_u32, neon_u32, 1)
425NEON_POP(pmin_s8, neon_s8, 4)
426NEON_POP(pmin_u8, neon_u8, 4)
427NEON_POP(pmin_s16, neon_s16, 2)
428NEON_POP(pmin_u16, neon_u16, 2)
429#undef NEON_FN
430
431#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
432NEON_VOP(max_s8, neon_s8, 4)
433NEON_VOP(max_u8, neon_u8, 4)
434NEON_VOP(max_s16, neon_s16, 2)
435NEON_VOP(max_u16, neon_u16, 2)
436NEON_VOP(max_s32, neon_s32, 1)
437NEON_VOP(max_u32, neon_u32, 1)
438NEON_POP(pmax_s8, neon_s8, 4)
439NEON_POP(pmax_u8, neon_u8, 4)
440NEON_POP(pmax_s16, neon_s16, 2)
441NEON_POP(pmax_u16, neon_u16, 2)
442#undef NEON_FN
443
444#define NEON_FN(dest, src1, src2) \
445 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
446NEON_VOP(abd_s8, neon_s8, 4)
447NEON_VOP(abd_u8, neon_u8, 4)
448NEON_VOP(abd_s16, neon_s16, 2)
449NEON_VOP(abd_u16, neon_u16, 2)
450NEON_VOP(abd_s32, neon_s32, 1)
451NEON_VOP(abd_u32, neon_u32, 1)
452#undef NEON_FN
453
454#define NEON_FN(dest, src1, src2) do { \
455 int8_t tmp; \
456 tmp = (int8_t)src2; \
50f67e95
JR		 457 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
458 tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 459 dest = 0; \
460 } else if (tmp < 0) { \
461 dest = src1 >> -tmp; \
462 } else { \
463 dest = src1 << tmp; \
464 }} while (0)
465NEON_VOP(shl_u8, neon_u8, 4)
466NEON_VOP(shl_u16, neon_u16, 2)
467NEON_VOP(shl_u32, neon_u32, 1)
468#undef NEON_FN
469
470uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
471{
472 int8_t shift = (int8_t)shiftop;
473 if (shift >= 64 || shift <= -64) {
474 val = 0;
475 } else if (shift < 0) {
476 val >>= -shift;
477 } else {
478 val <<= shift;
479 }
480 return val;
481}
482
483#define NEON_FN(dest, src1, src2) do { \
484 int8_t tmp; \
485 tmp = (int8_t)src2; \
50f67e95 486 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c 487 dest = 0; \
50f67e95 488 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 489 dest = src1 >> (sizeof(src1) * 8 - 1); \
490 } else if (tmp < 0) { \
491 dest = src1 >> -tmp; \
492 } else { \
493 dest = src1 << tmp; \
494 }} while (0)
495NEON_VOP(shl_s8, neon_s8, 4)
496NEON_VOP(shl_s16, neon_s16, 2)
497NEON_VOP(shl_s32, neon_s32, 1)
498#undef NEON_FN
499
500uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
501{
502 int8_t shift = (int8_t)shiftop;
503 int64_t val = valop;
504 if (shift >= 64) {
505 val = 0;
506 } else if (shift <= -64) {
507 val >>= 63;
508 } else if (shift < 0) {
509 val >>= -shift;
510 } else {
511 val <<= shift;
512 }
513 return val;
514}
515
516#define NEON_FN(dest, src1, src2) do { \
517 int8_t tmp; \
518 tmp = (int8_t)src2; \
0670a7b6
PM		 519 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
520 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
ad69471c 521 dest = 0; \
ad69471c
PB		 522 } else if (tmp < 0) { \
523 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
524 } else { \
525 dest = src1 << tmp; \
526 }} while (0)
527NEON_VOP(rshl_s8, neon_s8, 4)
528NEON_VOP(rshl_s16, neon_s16, 2)
ad69471c
PB		 529#undef NEON_FN
530
4bd4ee07
CL		 531/* The addition of the rounding constant may overflow, so we use an
532 * intermediate 64 bits accumulator. */
533uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
534{
535 int32_t dest;
536 int32_t val = (int32_t)valop;
537 int8_t shift = (int8_t)shiftop;
538 if ((shift >= 32) || (shift <= -32)) {
539 dest = 0;
540 } else if (shift < 0) {
541 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
542 dest = big_dest >> -shift;
543 } else {
544 dest = val << shift;
545 }
546 return dest;
547}
548
549/* Handling addition overflow with 64 bits inputs values is more
550 * tricky than with 32 bits values. */
ad69471c
PB		 551uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
552{
553 int8_t shift = (int8_t)shiftop;
554 int64_t val = valop;
0670a7b6 555 if ((shift >= 64) || (shift <= -64)) {
ad69471c 556 val = 0;
ad69471c 557 } else if (shift < 0) {
4bd4ee07
CL		 558 val >>= (-shift - 1);
559 if (val == INT64_MAX) {
560 /* In this case, it means that the rounding constant is 1,
561 * and the addition would overflow. Return the actual
562 * result directly. */
563 val = 0x4000000000000000LL;
564 } else {
565 val++;
566 val >>= 1;
567 }
ad69471c
PB		 568 } else {
569 val <<= shift;
570 }
571 return val;
572}
573
574#define NEON_FN(dest, src1, src2) do { \
575 int8_t tmp; \
576 tmp = (int8_t)src2; \
50f67e95
JR		 577 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
578 tmp < -(ssize_t)sizeof(src1) * 8) { \
ad69471c 579 dest = 0; \
50f67e95 580 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
b6c63b98 581 dest = src1 >> (-tmp - 1); \
ad69471c
PB		 582 } else if (tmp < 0) { \
583 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
584 } else { \
585 dest = src1 << tmp; \
586 }} while (0)
587NEON_VOP(rshl_u8, neon_u8, 4)
588NEON_VOP(rshl_u16, neon_u16, 2)
ad69471c
PB		 589#undef NEON_FN
590
4bd4ee07
CL		 591/* The addition of the rounding constant may overflow, so we use an
592 * intermediate 64 bits accumulator. */
593uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
594{
595 uint32_t dest;
596 int8_t shift = (int8_t)shiftop;
597 if (shift >= 32 || shift < -32) {
598 dest = 0;
599 } else if (shift == -32) {
600 dest = val >> 31;
601 } else if (shift < 0) {
602 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
603 dest = big_dest >> -shift;
604 } else {
605 dest = val << shift;
606 }
607 return dest;
608}
609
610/* Handling addition overflow with 64 bits inputs values is more
611 * tricky than with 32 bits values. */
ad69471c
PB		 612uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
613{
614 int8_t shift = (uint8_t)shiftop;
51e3930f 615 if (shift >= 64 || shift < -64) {
ad69471c
PB		 616 val = 0;
617 } else if (shift == -64) {
618 /* Rounding a 1-bit result just preserves that bit. */
619 val >>= 63;
4bd4ee07
CL		 620 } else if (shift < 0) {
621 val >>= (-shift - 1);
622 if (val == UINT64_MAX) {
623 /* In this case, it means that the rounding constant is 1,
624 * and the addition would overflow. Return the actual
625 * result directly. */
626 val = 0x8000000000000000ULL;
627 } else {
628 val++;
629 val >>= 1;
630 }
ad69471c
PB		 631 } else {
632 val <<= shift;
633 }
634 return val;
635}
636
637#define NEON_FN(dest, src1, src2) do { \
638 int8_t tmp; \
639 tmp = (int8_t)src2; \
50f67e95 640 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 641 if (src1) { \
642 SET_QC(); \
643 dest = ~0; \
644 } else { \
645 dest = 0; \
646 } \
50f67e95 647 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 648 dest = 0; \
649 } else if (tmp < 0) { \
650 dest = src1 >> -tmp; \
651 } else { \
652 dest = src1 << tmp; \
653 if ((dest >> tmp) != src1) { \
654 SET_QC(); \
655 dest = ~0; \
656 } \
657 }} while (0)
2a3f75b4
PM		 658NEON_VOP(qshl_u8, neon_u8, 4)
659NEON_VOP(qshl_u16, neon_u16, 2)
660NEON_VOP(qshl_u32, neon_u32, 1)
ad69471c
PB		 661#undef NEON_FN
662
2a3f75b4 663uint64_t HELPER(neon_qshl_u64)(uint64_t val, uint64_t shiftop)
ad69471c
PB		 664{
665 int8_t shift = (int8_t)shiftop;
666 if (shift >= 64) {
667 if (val) {
668 val = ~(uint64_t)0;
669 SET_QC();
ad69471c
PB		 670 }
671 } else if (shift <= -64) {
672 val = 0;
673 } else if (shift < 0) {
674 val >>= -shift;
675 } else {
676 uint64_t tmp = val;
677 val <<= shift;
678 if ((val >> shift) != tmp) {
679 SET_QC();
680 val = ~(uint64_t)0;
681 }
682 }
683 return val;
684}
685
686#define NEON_FN(dest, src1, src2) do { \
687 int8_t tmp; \
688 tmp = (int8_t)src2; \
50f67e95 689 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
a5d88f3e 690 if (src1) { \
ad69471c 691 SET_QC(); \
a5d88f3e
PM		 692 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
693 if (src1 > 0) { \
694 dest--; \
695 } \
696 } else { \
697 dest = src1; \
698 } \
50f67e95 699 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 700 dest = src1 >> 31; \
701 } else if (tmp < 0) { \
702 dest = src1 >> -tmp; \
703 } else { \
704 dest = src1 << tmp; \
705 if ((dest >> tmp) != src1) { \
706 SET_QC(); \
a5d88f3e
PM		 707 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
708 if (src1 > 0) { \
709 dest--; \
710 } \
ad69471c
PB		 711 } \
712 }} while (0)
2a3f75b4
PM		 713NEON_VOP(qshl_s8, neon_s8, 4)
714NEON_VOP(qshl_s16, neon_s16, 2)
715NEON_VOP(qshl_s32, neon_s32, 1)
ad69471c
PB		 716#undef NEON_FN
717
2a3f75b4 718uint64_t HELPER(neon_qshl_s64)(uint64_t valop, uint64_t shiftop)
ad69471c
PB		 719{
720 int8_t shift = (uint8_t)shiftop;
721 int64_t val = valop;
722 if (shift >= 64) {
723 if (val) {
724 SET_QC();
eb7a3d79 725 val = (val >> 63) ^ ~SIGNBIT64;
ad69471c 726 }
4c9b70ae 727 } else if (shift <= -64) {
ad69471c
PB		 728 val >>= 63;
729 } else if (shift < 0) {
730 val >>= -shift;
731 } else {
732 int64_t tmp = val;
733 val <<= shift;
734 if ((val >> shift) != tmp) {
735 SET_QC();
736 val = (tmp >> 63) ^ ~SIGNBIT64;
737 }
738 }
739 return val;
740}
741
4ca4502c
JR		 742#define NEON_FN(dest, src1, src2) do { \
743 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
744 SET_QC(); \
745 dest = 0; \
746 } else { \
747 int8_t tmp; \
748 tmp = (int8_t)src2; \
749 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
750 if (src1) { \
751 SET_QC(); \
752 dest = ~0; \
753 } else { \
754 dest = 0; \
755 } \
756 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
757 dest = 0; \
758 } else if (tmp < 0) { \
759 dest = src1 >> -tmp; \
760 } else { \
761 dest = src1 << tmp; \
762 if ((dest >> tmp) != src1) { \
763 SET_QC(); \
764 dest = ~0; \
765 } \
766 } \
767 }} while (0)
2a3f75b4
PM		 768NEON_VOP(qshlu_s8, neon_u8, 4)
769NEON_VOP(qshlu_s16, neon_u16, 2)
4ca4502c
JR		 770#undef NEON_FN
771
2a3f75b4 772uint32_t HELPER(neon_qshlu_s32)(uint32_t valop, uint32_t shiftop)
4ca4502c
JR		 773{
774 if ((int32_t)valop < 0) {
775 SET_QC();
776 return 0;
777 }
2a3f75b4 778 return helper_neon_qshl_u32(valop, shiftop);
4ca4502c
JR		 779}
780
2a3f75b4 781uint64_t HELPER(neon_qshlu_s64)(uint64_t valop, uint64_t shiftop)
4ca4502c
JR		 782{
783 if ((int64_t)valop < 0) {
784 SET_QC();
785 return 0;
786 }
2a3f75b4 787 return helper_neon_qshl_u64(valop, shiftop);
4ca4502c 788}
ad69471c
PB		 789
790/* FIXME: This is wrong. */
791#define NEON_FN(dest, src1, src2) do { \
792 int8_t tmp; \
793 tmp = (int8_t)src2; \
33ebc293
PM		 794 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
795 if (src1) { \
796 SET_QC(); \
797 dest = ~0; \
798 } else { \
799 dest = 0; \
800 } \
801 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
802 dest = 0; \
803 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
804 dest = src1 >> (sizeof(src1) * 8 - 1); \
805 } else if (tmp < 0) { \
ad69471c
PB		 806 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
807 } else { \
808 dest = src1 << tmp; \
809 if ((dest >> tmp) != src1) { \
810 SET_QC(); \
811 dest = ~0; \
812 } \
813 }} while (0)
2a3f75b4
PM		 814NEON_VOP(qrshl_u8, neon_u8, 4)
815NEON_VOP(qrshl_u16, neon_u16, 2)
ad69471c
PB		 816#undef NEON_FN
817
4bd4ee07
CL		 818/* The addition of the rounding constant may overflow, so we use an
819 * intermediate 64 bits accumulator. */
2a3f75b4 820uint32_t HELPER(neon_qrshl_u32)(uint32_t val, uint32_t shiftop)
4bd4ee07
CL		 821{
822 uint32_t dest;
823 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 824 if (shift >= 32) {
825 if (val) {
826 SET_QC();
827 dest = ~0;
828 } else {
829 dest = 0;
830 }
831 } else if (shift < -32) {
832 dest = 0;
833 } else if (shift == -32) {
834 dest = val >> 31;
835 } else if (shift < 0) {
4bd4ee07
CL		 836 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
837 dest = big_dest >> -shift;
838 } else {
839 dest = val << shift;
840 if ((dest >> shift) != val) {
841 SET_QC();
842 dest = ~0;
843 }
844 }
845 return dest;
846}
847
848/* Handling addition overflow with 64 bits inputs values is more
849 * tricky than with 32 bits values. */
2a3f75b4 850uint64_t HELPER(neon_qrshl_u64)(uint64_t val, uint64_t shiftop)
ad69471c
PB		 851{
852 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 853 if (shift >= 64) {
854 if (val) {
855 SET_QC();
856 val = ~0;
857 }
858 } else if (shift < -64) {
859 val = 0;
860 } else if (shift == -64) {
861 val >>= 63;
862 } else if (shift < 0) {
4bd4ee07
CL		 863 val >>= (-shift - 1);
864 if (val == UINT64_MAX) {
865 /* In this case, it means that the rounding constant is 1,
866 * and the addition would overflow. Return the actual
867 * result directly. */
868 val = 0x8000000000000000ULL;
869 } else {
870 val++;
871 val >>= 1;
872 }
ad69471c
PB		 873 } else { \
874 uint64_t tmp = val;
875 val <<= shift;
876 if ((val >> shift) != tmp) {
877 SET_QC();
878 val = ~0;
879 }
880 }
881 return val;
882}
883
884#define NEON_FN(dest, src1, src2) do { \
885 int8_t tmp; \
886 tmp = (int8_t)src2; \
7b6ecf5b
PM		 887 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
888 if (src1) { \
889 SET_QC(); \
890 dest = (1 << (sizeof(src1) * 8 - 1)); \
891 if (src1 > 0) { \
892 dest--; \
893 } \
894 } else { \
895 dest = 0; \
896 } \
897 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
898 dest = 0; \
899 } else if (tmp < 0) { \
ad69471c
PB		 900 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
901 } else { \
902 dest = src1 << tmp; \
903 if ((dest >> tmp) != src1) { \
904 SET_QC(); \
960e623b
PM		 905 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
906 if (src1 > 0) { \
907 dest--; \
908 } \
ad69471c
PB		 909 } \
910 }} while (0)
2a3f75b4
PM		 911NEON_VOP(qrshl_s8, neon_s8, 4)
912NEON_VOP(qrshl_s16, neon_s16, 2)
ad69471c
PB		 913#undef NEON_FN
914
4bd4ee07
CL		 915/* The addition of the rounding constant may overflow, so we use an
916 * intermediate 64 bits accumulator. */
2a3f75b4 917uint32_t HELPER(neon_qrshl_s32)(uint32_t valop, uint32_t shiftop)
4bd4ee07
CL		 918{
919 int32_t dest;
920 int32_t val = (int32_t)valop;
921 int8_t shift = (int8_t)shiftop;
7b6ecf5b
PM		 922 if (shift >= 32) {
923 if (val) {
924 SET_QC();
925 dest = (val >> 31) ^ ~SIGNBIT;
926 } else {
927 dest = 0;
928 }
929 } else if (shift <= -32) {
930 dest = 0;
931 } else if (shift < 0) {
4bd4ee07
CL		 932 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
933 dest = big_dest >> -shift;
934 } else {
935 dest = val << shift;
936 if ((dest >> shift) != val) {
937 SET_QC();
938 dest = (val >> 31) ^ ~SIGNBIT;
939 }
940 }
941 return dest;
942}
943
944/* Handling addition overflow with 64 bits inputs values is more
945 * tricky than with 32 bits values. */
2a3f75b4 946uint64_t HELPER(neon_qrshl_s64)(uint64_t valop, uint64_t shiftop)
ad69471c
PB		 947{
948 int8_t shift = (uint8_t)shiftop;
949 int64_t val = valop;
950
7b6ecf5b
PM		 951 if (shift >= 64) {
952 if (val) {
953 SET_QC();
954 val = (val >> 63) ^ ~SIGNBIT64;
955 }
956 } else if (shift <= -64) {
957 val = 0;
958 } else if (shift < 0) {
4bd4ee07
CL		 959 val >>= (-shift - 1);
960 if (val == INT64_MAX) {
961 /* In this case, it means that the rounding constant is 1,
962 * and the addition would overflow. Return the actual
963 * result directly. */
964 val = 0x4000000000000000ULL;
965 } else {
966 val++;
967 val >>= 1;
968 }
ad69471c 969 } else {
4bd4ee07 970 int64_t tmp = val;
ad69471c
PB		 971 val <<= shift;
972 if ((val >> shift) != tmp) {
973 SET_QC();
4bd4ee07 974 val = (tmp >> 63) ^ ~SIGNBIT64;
ad69471c
PB		 975 }
976 }
977 return val;
978}
979
980uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
981{
982 uint32_t mask;
983 mask = (a ^ b) & 0x80808080u;
984 a &= ~0x80808080u;
985 b &= ~0x80808080u;
986 return (a + b) ^ mask;
987}
988
989uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
990{
991 uint32_t mask;
992 mask = (a ^ b) & 0x80008000u;
993 a &= ~0x80008000u;
994 b &= ~0x80008000u;
995 return (a + b) ^ mask;
996}
997
998#define NEON_FN(dest, src1, src2) dest = src1 + src2
999NEON_POP(padd_u8, neon_u8, 4)
1000NEON_POP(padd_u16, neon_u16, 2)
1001#undef NEON_FN
1002
1003#define NEON_FN(dest, src1, src2) dest = src1 - src2
1004NEON_VOP(sub_u8, neon_u8, 4)
1005NEON_VOP(sub_u16, neon_u16, 2)
1006#undef NEON_FN
1007
1008#define NEON_FN(dest, src1, src2) dest = src1 * src2
1009NEON_VOP(mul_u8, neon_u8, 4)
1010NEON_VOP(mul_u16, neon_u16, 2)
1011#undef NEON_FN
1012
1654b2d6 1013/* Polynomial multiplication is like integer multiplication except the
ad69471c
PB		 1014 partial products are XORed, not added. */
1015uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1016{
1017 uint32_t mask;
1018 uint32_t result;
1019 result = 0;
1020 while (op1) {
1021 mask = 0;
1022 if (op1 & 1)
1023 mask |= 0xff;
1024 if (op1 & (1 << 8))
1025 mask |= (0xff << 8);
1026 if (op1 & (1 << 16))
1027 mask |= (0xff << 16);
1028 if (op1 & (1 << 24))
1029 mask |= (0xff << 24);
1030 result ^= op2 & mask;
1031 op1 = (op1 >> 1) & 0x7f7f7f7f;
1032 op2 = (op2 << 1) & 0xfefefefe;
1033 }
1034 return result;
1035}
1036
e5ca24cb
PM		 1037uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1038{
1039 uint64_t result = 0;
1040 uint64_t mask;
1041 uint64_t op2ex = op2;
1042 op2ex = (op2ex & 0xff) |
1043 ((op2ex & 0xff00) << 8) |
1044 ((op2ex & 0xff0000) << 16) |
1045 ((op2ex & 0xff000000) << 24);
1046 while (op1) {
1047 mask = 0;
1048 if (op1 & 1) {
1049 mask |= 0xffff;
1050 }
1051 if (op1 & (1 << 8)) {
1052 mask |= (0xffffU << 16);
1053 }
1054 if (op1 & (1 << 16)) {
1055 mask |= (0xffffULL << 32);
1056 }
1057 if (op1 & (1 << 24)) {
1058 mask |= (0xffffULL << 48);
1059 }
1060 result ^= op2ex & mask;
1061 op1 = (op1 >> 1) & 0x7f7f7f7f;
1062 op2ex <<= 1;
1063 }
1064 return result;
1065}
1066
ad69471c
PB		 1067#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1068NEON_VOP(tst_u8, neon_u8, 4)
1069NEON_VOP(tst_u16, neon_u16, 2)
1070NEON_VOP(tst_u32, neon_u32, 1)
1071#undef NEON_FN
1072
1073#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1074NEON_VOP(ceq_u8, neon_u8, 4)
1075NEON_VOP(ceq_u16, neon_u16, 2)
1076NEON_VOP(ceq_u32, neon_u32, 1)
1077#undef NEON_FN
1078
1079#define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1080NEON_VOP1(abs_s8, neon_s8, 4)
1081NEON_VOP1(abs_s16, neon_s16, 2)
1082#undef NEON_FN
1083
1084/* Count Leading Sign/Zero Bits. */
1085static inline int do_clz8(uint8_t x)
1086{
1087 int n;
1088 for (n = 8; x; n--)
1089 x >>= 1;
1090 return n;
1091}
1092
1093static inline int do_clz16(uint16_t x)
1094{
1095 int n;
1096 for (n = 16; x; n--)
1097 x >>= 1;
1098 return n;
1099}
1100
1101#define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1102NEON_VOP1(clz_u8, neon_u8, 4)
1103#undef NEON_FN
1104
1105#define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1106NEON_VOP1(clz_u16, neon_u16, 2)
1107#undef NEON_FN
1108
1109#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1110NEON_VOP1(cls_s8, neon_s8, 4)
1111#undef NEON_FN
1112
1113#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1114NEON_VOP1(cls_s16, neon_s16, 2)
1115#undef NEON_FN
1116
1117uint32_t HELPER(neon_cls_s32)(uint32_t x)
1118{
1119 int count;
1120 if ((int32_t)x < 0)
1121 x = ~x;
1122 for (count = 32; x; count--)
1123 x = x >> 1;
1124 return count - 1;
1125}
1126
1127/* Bit count. */
1128uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1129{
1130 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1131 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1132 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1133 return x;
1134}
1135
1136#define NEON_QDMULH16(dest, src1, src2, round) do { \
1137 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1138 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1139 SET_QC(); \
1140 tmp = (tmp >> 31) ^ ~SIGNBIT; \
46eece9d
JR		 1141 } else { \
1142 tmp <<= 1; \
ad69471c 1143 } \
ad69471c
PB		 1144 if (round) { \
1145 int32_t old = tmp; \
1146 tmp += 1 << 15; \
1147 if ((int32_t)tmp < old) { \
1148 SET_QC(); \
1149 tmp = SIGNBIT - 1; \
1150 } \
1151 } \
1152 dest = tmp >> 16; \
1153 } while(0)
1154#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
2a3f75b4 1155NEON_VOP(qdmulh_s16, neon_s16, 2)
ad69471c
PB		 1156#undef NEON_FN
1157#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
2a3f75b4 1158NEON_VOP(qrdmulh_s16, neon_s16, 2)
ad69471c
PB		 1159#undef NEON_FN
1160#undef NEON_QDMULH16
1161
1162#define NEON_QDMULH32(dest, src1, src2, round) do { \
1163 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1164 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1165 SET_QC(); \
1166 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1167 } else { \
1168 tmp <<= 1; \
1169 } \
1170 if (round) { \
1171 int64_t old = tmp; \
1172 tmp += (int64_t)1 << 31; \
1173 if ((int64_t)tmp < old) { \
1174 SET_QC(); \
1175 tmp = SIGNBIT64 - 1; \
1176 } \
1177 } \
1178 dest = tmp >> 32; \
1179 } while(0)
1180#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
2a3f75b4 1181NEON_VOP(qdmulh_s32, neon_s32, 1)
ad69471c
PB		 1182#undef NEON_FN
1183#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
2a3f75b4 1184NEON_VOP(qrdmulh_s32, neon_s32, 1)
ad69471c
PB		 1185#undef NEON_FN
1186#undef NEON_QDMULH32
1187
1188uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1189{
1190 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1191 | ((x >> 24) & 0xff000000u);
1192}
1193
1194uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1195{
1196 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1197}
1198
1199uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1200{
1201 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1202 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1203}
1204
1205uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1206{
1207 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1208}
1209
1210uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1211{
1212 x &= 0xff80ff80ff80ff80ull;
1213 x += 0x0080008000800080ull;
1214 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1215 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1216}
1217
1218uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1219{
1220 x &= 0xffff8000ffff8000ull;
1221 x += 0x0000800000008000ull;
1222 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1223}
1224
2a3f75b4 1225uint32_t HELPER(neon_unarrow_sat8)(uint64_t x)
af1bbf30
JR		 1226{
1227 uint16_t s;
1228 uint8_t d;
1229 uint32_t res = 0;
1230#define SAT8(n) \
1231 s = x >> n; \
1232 if (s & 0x8000) { \
1233 SET_QC(); \
1234 } else { \
1235 if (s > 0xff) { \
1236 d = 0xff; \
1237 SET_QC(); \
1238 } else { \
1239 d = s; \
1240 } \
1241 res |= (uint32_t)d << (n / 2); \
1242 }
1243
1244 SAT8(0);
1245 SAT8(16);
1246 SAT8(32);
1247 SAT8(48);
1248#undef SAT8
1249 return res;
1250}
1251
2a3f75b4 1252uint32_t HELPER(neon_narrow_sat_u8)(uint64_t x)
ad69471c
PB		 1253{
1254 uint16_t s;
1255 uint8_t d;
1256 uint32_t res = 0;
1257#define SAT8(n) \
1258 s = x >> n; \
1259 if (s > 0xff) { \
1260 d = 0xff; \
1261 SET_QC(); \
1262 } else { \
1263 d = s; \
1264 } \
1265 res |= (uint32_t)d << (n / 2);
1266
1267 SAT8(0);
1268 SAT8(16);
1269 SAT8(32);
1270 SAT8(48);
1271#undef SAT8
1272 return res;
1273}
1274
2a3f75b4 1275uint32_t HELPER(neon_narrow_sat_s8)(uint64_t x)
ad69471c
PB		 1276{
1277 int16_t s;
1278 uint8_t d;
1279 uint32_t res = 0;
1280#define SAT8(n) \
1281 s = x >> n; \
1282 if (s != (int8_t)s) { \
1283 d = (s >> 15) ^ 0x7f; \
1284 SET_QC(); \
1285 } else { \
1286 d = s; \
1287 } \
1288 res |= (uint32_t)d << (n / 2);
1289
1290 SAT8(0);
1291 SAT8(16);
1292 SAT8(32);
1293 SAT8(48);
1294#undef SAT8
1295 return res;
1296}
1297
2a3f75b4 1298uint32_t HELPER(neon_unarrow_sat16)(uint64_t x)
af1bbf30
JR		 1299{
1300 uint32_t high;
1301 uint32_t low;
1302 low = x;
1303 if (low & 0x80000000) {
1304 low = 0;
1305 SET_QC();
1306 } else if (low > 0xffff) {
1307 low = 0xffff;
1308 SET_QC();
1309 }
1310 high = x >> 32;
1311 if (high & 0x80000000) {
1312 high = 0;
1313 SET_QC();
1314 } else if (high > 0xffff) {
1315 high = 0xffff;
1316 SET_QC();
1317 }
1318 return low | (high << 16);
1319}
1320
2a3f75b4 1321uint32_t HELPER(neon_narrow_sat_u16)(uint64_t x)
ad69471c
PB		 1322{
1323 uint32_t high;
1324 uint32_t low;
1325 low = x;
1326 if (low > 0xffff) {
1327 low = 0xffff;
1328 SET_QC();
1329 }
1330 high = x >> 32;
1331 if (high > 0xffff) {
1332 high = 0xffff;
1333 SET_QC();
1334 }
1335 return low | (high << 16);
1336}
1337
2a3f75b4 1338uint32_t HELPER(neon_narrow_sat_s16)(uint64_t x)
ad69471c
PB		 1339{
1340 int32_t low;
1341 int32_t high;
1342 low = x;
1343 if (low != (int16_t)low) {
1344 low = (low >> 31) ^ 0x7fff;
1345 SET_QC();
1346 }
1347 high = x >> 32;
1348 if (high != (int16_t)high) {
1349 high = (high >> 31) ^ 0x7fff;
1350 SET_QC();
1351 }
1352 return (uint16_t)low | (high << 16);
1353}
1354
2a3f75b4 1355uint32_t HELPER(neon_unarrow_sat32)(uint64_t x)
af1bbf30
JR		 1356{
1357 if (x & 0x8000000000000000ull) {
1358 SET_QC();
1359 return 0;
1360 }
1361 if (x > 0xffffffffu) {
1362 SET_QC();
1363 return 0xffffffffu;
1364 }
1365 return x;
1366}
1367
2a3f75b4 1368uint32_t HELPER(neon_narrow_sat_u32)(uint64_t x)
ad69471c
PB		 1369{
1370 if (x > 0xffffffffu) {
1371 SET_QC();
1372 return 0xffffffffu;
1373 }
1374 return x;
1375}
1376
2a3f75b4 1377uint32_t HELPER(neon_narrow_sat_s32)(uint64_t x)
ad69471c
PB		 1378{
1379 if ((int64_t)x != (int32_t)x) {
1380 SET_QC();
cc2212c2 1381 return ((int64_t)x >> 63) ^ 0x7fffffff;
ad69471c
PB		 1382 }
1383 return x;
1384}
1385
1386uint64_t HELPER(neon_widen_u8)(uint32_t x)
1387{
1388 uint64_t tmp;
1389 uint64_t ret;
1390 ret = (uint8_t)x;
1391 tmp = (uint8_t)(x >> 8);
1392 ret |= tmp << 16;
1393 tmp = (uint8_t)(x >> 16);
1394 ret |= tmp << 32;
1395 tmp = (uint8_t)(x >> 24);
1396 ret |= tmp << 48;
1397 return ret;
1398}
1399
1400uint64_t HELPER(neon_widen_s8)(uint32_t x)
1401{
1402 uint64_t tmp;
1403 uint64_t ret;
1404 ret = (uint16_t)(int8_t)x;
1405 tmp = (uint16_t)(int8_t)(x >> 8);
1406 ret |= tmp << 16;
1407 tmp = (uint16_t)(int8_t)(x >> 16);
1408 ret |= tmp << 32;
1409 tmp = (uint16_t)(int8_t)(x >> 24);
1410 ret |= tmp << 48;
1411 return ret;
1412}
1413
1414uint64_t HELPER(neon_widen_u16)(uint32_t x)
1415{
1416 uint64_t high = (uint16_t)(x >> 16);
1417 return ((uint16_t)x) | (high << 32);
1418}
1419
1420uint64_t HELPER(neon_widen_s16)(uint32_t x)
1421{
1422 uint64_t high = (int16_t)(x >> 16);
1423 return ((uint32_t)(int16_t)x) | (high << 32);
1424}
1425
1426uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1427{
1428 uint64_t mask;
1429 mask = (a ^ b) & 0x8000800080008000ull;
1430 a &= ~0x8000800080008000ull;
1431 b &= ~0x8000800080008000ull;
1432 return (a + b) ^ mask;
1433}
1434
1435uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1436{
1437 uint64_t mask;
1438 mask = (a ^ b) & 0x8000000080000000ull;
1439 a &= ~0x8000000080000000ull;
1440 b &= ~0x8000000080000000ull;
1441 return (a + b) ^ mask;
1442}
1443
1444uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1445{
1446 uint64_t tmp;
1447 uint64_t tmp2;
1448
1449 tmp = a & 0x0000ffff0000ffffull;
1450 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1451 tmp2 = b & 0xffff0000ffff0000ull;
1452 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1453 return ( tmp & 0xffff)
1454 | ((tmp >> 16) & 0xffff0000ull)
1455 | ((tmp2 << 16) & 0xffff00000000ull)
1456 | ( tmp2 & 0xffff000000000000ull);
1457}
1458
1459uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1460{
1461 uint32_t low = a + (a >> 32);
1462 uint32_t high = b + (b >> 32);
1463 return low + ((uint64_t)high << 32);
1464}
1465
1466uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1467{
1468 uint64_t mask;
1469 mask = (a ^ ~b) & 0x8000800080008000ull;
1470 a |= 0x8000800080008000ull;
1471 b &= ~0x8000800080008000ull;
1472 return (a - b) ^ mask;
1473}
1474
1475uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1476{
1477 uint64_t mask;
1478 mask = (a ^ ~b) & 0x8000000080000000ull;
1479 a |= 0x8000000080000000ull;
1480 b &= ~0x8000000080000000ull;
1481 return (a - b) ^ mask;
1482}
1483
2a3f75b4 1484uint64_t HELPER(neon_addl_saturate_s32)(uint64_t a, uint64_t b)
ad69471c
PB		 1485{
1486 uint32_t x, y;
1487 uint32_t low, high;
1488
1489 x = a;
1490 y = b;
1491 low = x + y;
1492 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1493 SET_QC();
1494 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1495 }
1496 x = a >> 32;
1497 y = b >> 32;
1498 high = x + y;
1499 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1500 SET_QC();
1501 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1502 }
1503 return low | ((uint64_t)high << 32);
1504}
1505
2a3f75b4 1506uint64_t HELPER(neon_addl_saturate_s64)(uint64_t a, uint64_t b)
ad69471c
PB		 1507{
1508 uint64_t result;
1509
1510 result = a + b;
1511 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1512 SET_QC();
1513 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1514 }
1515 return result;
1516}
1517
1518#define DO_ABD(dest, x, y, type) do { \
1519 type tmp_x = x; \
1520 type tmp_y = y; \
1521 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1522 } while(0)
1523
1524uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1525{
1526 uint64_t tmp;
1527 uint64_t result;
1528 DO_ABD(result, a, b, uint8_t);
1529 DO_ABD(tmp, a >> 8, b >> 8, uint8_t);
1530 result |= tmp << 16;
1531 DO_ABD(tmp, a >> 16, b >> 16, uint8_t);
1532 result |= tmp << 32;
1533 DO_ABD(tmp, a >> 24, b >> 24, uint8_t);
1534 result |= tmp << 48;
1535 return result;
1536}
1537
1538uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1539{
1540 uint64_t tmp;
1541 uint64_t result;
1542 DO_ABD(result, a, b, int8_t);
1543 DO_ABD(tmp, a >> 8, b >> 8, int8_t);
1544 result |= tmp << 16;
1545 DO_ABD(tmp, a >> 16, b >> 16, int8_t);
1546 result |= tmp << 32;
1547 DO_ABD(tmp, a >> 24, b >> 24, int8_t);
1548 result |= tmp << 48;
1549 return result;
1550}
1551
1552uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1553{
1554 uint64_t tmp;
1555 uint64_t result;
1556 DO_ABD(result, a, b, uint16_t);
1557 DO_ABD(tmp, a >> 16, b >> 16, uint16_t);
1558 return result | (tmp << 32);
1559}
1560
1561uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1562{
1563 uint64_t tmp;
1564 uint64_t result;
1565 DO_ABD(result, a, b, int16_t);
1566 DO_ABD(tmp, a >> 16, b >> 16, int16_t);
1567 return result | (tmp << 32);
1568}
1569
1570uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1571{
1572 uint64_t result;
1573 DO_ABD(result, a, b, uint32_t);
1574 return result;
1575}
1576
1577uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1578{
1579 uint64_t result;
1580 DO_ABD(result, a, b, int32_t);
1581 return result;
1582}
1583#undef DO_ABD
1584
1585/* Widening multiply. Named type is the source type. */
1586#define DO_MULL(dest, x, y, type1, type2) do { \
1587 type1 tmp_x = x; \
1588 type1 tmp_y = y; \
1589 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1590 } while(0)
1591
1592uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1593{
1594 uint64_t tmp;
1595 uint64_t result;
1596
1597 DO_MULL(result, a, b, uint8_t, uint16_t);
1598 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1599 result |= tmp << 16;
1600 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1601 result |= tmp << 32;
1602 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1603 result |= tmp << 48;
1604 return result;
1605}
1606
1607uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1608{
1609 uint64_t tmp;
1610 uint64_t result;
1611
1612 DO_MULL(result, a, b, int8_t, uint16_t);
1613 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1614 result |= tmp << 16;
1615 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1616 result |= tmp << 32;
1617 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1618 result |= tmp << 48;
1619 return result;
1620}
1621
1622uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1623{
1624 uint64_t tmp;
1625 uint64_t result;
1626
1627 DO_MULL(result, a, b, uint16_t, uint32_t);
1628 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1629 return result | (tmp << 32);
1630}
1631
1632uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1633{
1634 uint64_t tmp;
1635 uint64_t result;
1636
1637 DO_MULL(result, a, b, int16_t, uint32_t);
1638 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1639 return result | (tmp << 32);
1640}
1641
1642uint64_t HELPER(neon_negl_u16)(uint64_t x)
1643{
1644 uint16_t tmp;
1645 uint64_t result;
1646 result = (uint16_t)-x;
1647 tmp = -(x >> 16);
1648 result |= (uint64_t)tmp << 16;
1649 tmp = -(x >> 32);
1650 result |= (uint64_t)tmp << 32;
1651 tmp = -(x >> 48);
1652 result |= (uint64_t)tmp << 48;
1653 return result;
1654}
1655
ad69471c
PB		 1656uint64_t HELPER(neon_negl_u32)(uint64_t x)
1657{
1658 uint32_t low = -x;
1659 uint32_t high = -(x >> 32);
1660 return low | ((uint64_t)high << 32);
1661}
1662
1663/* FIXME: There should be a native op for this. */
1664uint64_t HELPER(neon_negl_u64)(uint64_t x)
1665{
1666 return -x;
1667}
1668
1669/* Saturnating sign manuipulation. */
1670/* ??? Make these use NEON_VOP1 */
1671#define DO_QABS8(x) do { \
1672 if (x == (int8_t)0x80) { \
1673 x = 0x7f; \
1674 SET_QC(); \
1675 } else if (x < 0) { \
1676 x = -x; \
1677 }} while (0)
2a3f75b4 1678uint32_t HELPER(neon_qabs_s8)(uint32_t x)
ad69471c
PB		 1679{
1680 neon_s8 vec;
1681 NEON_UNPACK(neon_s8, vec, x);
1682 DO_QABS8(vec.v1);
1683 DO_QABS8(vec.v2);
1684 DO_QABS8(vec.v3);
1685 DO_QABS8(vec.v4);
1686 NEON_PACK(neon_s8, x, vec);
1687 return x;
1688}
1689#undef DO_QABS8
1690
1691#define DO_QNEG8(x) do { \
1692 if (x == (int8_t)0x80) { \
1693 x = 0x7f; \
1694 SET_QC(); \
1695 } else { \
1696 x = -x; \
1697 }} while (0)
2a3f75b4 1698uint32_t HELPER(neon_qneg_s8)(uint32_t x)
ad69471c
PB		 1699{
1700 neon_s8 vec;
1701 NEON_UNPACK(neon_s8, vec, x);
1702 DO_QNEG8(vec.v1);
1703 DO_QNEG8(vec.v2);
1704 DO_QNEG8(vec.v3);
1705 DO_QNEG8(vec.v4);
1706 NEON_PACK(neon_s8, x, vec);
1707 return x;
1708}
1709#undef DO_QNEG8
1710
1711#define DO_QABS16(x) do { \
1712 if (x == (int16_t)0x8000) { \
1713 x = 0x7fff; \
1714 SET_QC(); \
1715 } else if (x < 0) { \
1716 x = -x; \
1717 }} while (0)
2a3f75b4 1718uint32_t HELPER(neon_qabs_s16)(uint32_t x)
ad69471c
PB		 1719{
1720 neon_s16 vec;
1721 NEON_UNPACK(neon_s16, vec, x);
1722 DO_QABS16(vec.v1);
1723 DO_QABS16(vec.v2);
1724 NEON_PACK(neon_s16, x, vec);
1725 return x;
1726}
1727#undef DO_QABS16
1728
1729#define DO_QNEG16(x) do { \
1730 if (x == (int16_t)0x8000) { \
1731 x = 0x7fff; \
1732 SET_QC(); \
1733 } else { \
1734 x = -x; \
1735 }} while (0)
2a3f75b4 1736uint32_t HELPER(neon_qneg_s16)(uint32_t x)
ad69471c
PB		 1737{
1738 neon_s16 vec;
1739 NEON_UNPACK(neon_s16, vec, x);
1740 DO_QNEG16(vec.v1);
1741 DO_QNEG16(vec.v2);
1742 NEON_PACK(neon_s16, x, vec);
1743 return x;
1744}
1745#undef DO_QNEG16
1746
2a3f75b4 1747uint32_t HELPER(neon_qabs_s32)(uint32_t x)
ad69471c
PB		 1748{
1749 if (x == SIGNBIT) {
1750 SET_QC();
1751 x = ~SIGNBIT;
1752 } else if ((int32_t)x < 0) {
1753 x = -x;
1754 }
1755 return x;
1756}
1757
2a3f75b4 1758uint32_t HELPER(neon_qneg_s32)(uint32_t x)
ad69471c
PB		 1759{
1760 if (x == SIGNBIT) {
1761 SET_QC();
1762 x = ~SIGNBIT;
1763 } else {
1764 x = -x;
1765 }
1766 return x;
1767}
1768
1769/* NEON Float helpers. */
1770uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1771{
4a9f9cb2 1772 return float32_val(float32_min(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1773}
1774
1775uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1776{
4a9f9cb2 1777 return float32_val(float32_max(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1778}
1779
1780uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1781{
51d85267
PM		 1782 float32 f0 = make_float32(a);
1783 float32 f1 = make_float32(b);
79c18be7 1784 return float32_val(float32_abs(float32_sub(f0, f1, NFS)));
ad69471c
PB		 1785}
1786
1787uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1788{
51d85267 1789 return float32_val(float32_add(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1790}
1791
1792uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1793{
51d85267 1794 return float32_val(float32_sub(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1795}
1796
1797uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1798{
51d85267 1799 return float32_val(float32_mul(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1800}
1801
1802/* Floating point comparisons produce an integer result. */
c7498dae 1803#define NEON_VOP_FCMP(name, ok) \
ad69471c
PB		 1804uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1805{ \
c7498dae
PM		 1806 switch (float32_compare_quiet(make_float32(a), make_float32(b), NFS)) { \
1807 ok return ~0; \
1808 default: return 0; \
51d85267 1809 } \
ad69471c
PB		 1810}
1811
c7498dae
PM		 1812NEON_VOP_FCMP(ceq_f32, case float_relation_equal:)
1813NEON_VOP_FCMP(cge_f32, case float_relation_equal: case float_relation_greater:)
1814NEON_VOP_FCMP(cgt_f32, case float_relation_greater:)
ad69471c
PB		 1815
1816uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1817{
51d85267
PM		 1818 float32 f0 = float32_abs(make_float32(a));
1819 float32 f1 = float32_abs(make_float32(b));
c7498dae
PM		 1820 switch (float32_compare_quiet(f0, f1, NFS)) {
1821 case float_relation_equal:
1822 case float_relation_greater:
1823 return ~0;
1824 default:
1825 return 0;
1826 }
ad69471c
PB		 1827}
1828
1829uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1830{
51d85267
PM		 1831 float32 f0 = float32_abs(make_float32(a));
1832 float32 f1 = float32_abs(make_float32(b));
c7498dae
PM		 1833 if (float32_compare_quiet(f0, f1, NFS) == float_relation_greater) {
1834 return ~0;
1835 }
1836 return 0;
ad69471c 1837}
02acedf9
PM		 1838
1839#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1840
2a3f75b4 1841void HELPER(neon_qunzip8)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1842{
1843 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1844 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1845 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1846 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1847 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1848 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1849 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1850 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1851 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1852 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1853 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1854 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1855 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1856 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1857 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1858 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1859 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1860 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1861 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1862 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1863 env->vfp.regs[rm] = make_float64(m0);
1864 env->vfp.regs[rm + 1] = make_float64(m1);
1865 env->vfp.regs[rd] = make_float64(d0);
1866 env->vfp.regs[rd + 1] = make_float64(d1);
1867}
1868
2a3f75b4 1869void HELPER(neon_qunzip16)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1870{
1871 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1872 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1873 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1874 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1875 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1876 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1877 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1878 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1879 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1880 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1881 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1882 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1883 env->vfp.regs[rm] = make_float64(m0);
1884 env->vfp.regs[rm + 1] = make_float64(m1);
1885 env->vfp.regs[rd] = make_float64(d0);
1886 env->vfp.regs[rd + 1] = make_float64(d1);
1887}
1888
2a3f75b4 1889void HELPER(neon_qunzip32)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1890{
1891 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1892 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1893 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1894 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1895 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1896 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1897 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1898 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1899 env->vfp.regs[rm] = make_float64(m0);
1900 env->vfp.regs[rm + 1] = make_float64(m1);
1901 env->vfp.regs[rd] = make_float64(d0);
1902 env->vfp.regs[rd + 1] = make_float64(d1);
1903}
1904
2a3f75b4 1905void HELPER(neon_unzip8)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1906{
1907 uint64_t zm = float64_val(env->vfp.regs[rm]);
1908 uint64_t zd = float64_val(env->vfp.regs[rd]);
1909 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1910 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1911 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1912 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1913 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1914 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1915 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1916 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1917 env->vfp.regs[rm] = make_float64(m0);
1918 env->vfp.regs[rd] = make_float64(d0);
1919}
1920
2a3f75b4 1921void HELPER(neon_unzip16)(uint32_t rd, uint32_t rm)
02acedf9
PM		 1922{
1923 uint64_t zm = float64_val(env->vfp.regs[rm]);
1924 uint64_t zd = float64_val(env->vfp.regs[rd]);
1925 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1926 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1927 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1928 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1929 env->vfp.regs[rm] = make_float64(m0);
1930 env->vfp.regs[rd] = make_float64(d0);
1931}
d68a6f3a 1932
2a3f75b4 1933void HELPER(neon_qzip8)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1934{
1935 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1936 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1937 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1938 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1939 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1940 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1941 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1942 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1943 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1944 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1945 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1946 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1947 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1948 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1949 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1950 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1951 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1952 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1953 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1954 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1955 env->vfp.regs[rm] = make_float64(m0);
1956 env->vfp.regs[rm + 1] = make_float64(m1);
1957 env->vfp.regs[rd] = make_float64(d0);
1958 env->vfp.regs[rd + 1] = make_float64(d1);
1959}
1960
2a3f75b4 1961void HELPER(neon_qzip16)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1962{
1963 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1964 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1965 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1966 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1967 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1968 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1969 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1970 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1971 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1972 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1973 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1974 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1975 env->vfp.regs[rm] = make_float64(m0);
1976 env->vfp.regs[rm + 1] = make_float64(m1);
1977 env->vfp.regs[rd] = make_float64(d0);
1978 env->vfp.regs[rd + 1] = make_float64(d1);
1979}
1980
2a3f75b4 1981void HELPER(neon_qzip32)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1982{
1983 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1984 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1985 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1986 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1987 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1988 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1989 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1990 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1991 env->vfp.regs[rm] = make_float64(m0);
1992 env->vfp.regs[rm + 1] = make_float64(m1);
1993 env->vfp.regs[rd] = make_float64(d0);
1994 env->vfp.regs[rd + 1] = make_float64(d1);
1995}
1996
2a3f75b4 1997void HELPER(neon_zip8)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 1998{
1999 uint64_t zm = float64_val(env->vfp.regs[rm]);
2000 uint64_t zd = float64_val(env->vfp.regs[rd]);
2001 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2002 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2003 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2004 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2005 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2006 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2007 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2008 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2009 env->vfp.regs[rm] = make_float64(m0);
2010 env->vfp.regs[rd] = make_float64(d0);
2011}
2012
2a3f75b4 2013void HELPER(neon_zip16)(uint32_t rd, uint32_t rm)
d68a6f3a
PM		 2014{
2015 uint64_t zm = float64_val(env->vfp.regs[rm]);
2016 uint64_t zd = float64_val(env->vfp.regs[rd]);
2017 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2018 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2019 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2020 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2021 env->vfp.regs[rm] = make_float64(m0);
2022 env->vfp.regs[rd] = make_float64(d0);
2023}
Qemu copy for testing