]> git.proxmox.com Git - qemu.git/blame - target-arm/neon_helper.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Makefile.target: Allow target helpers to be in any *_helper.c file
[qemu.git] / target-arm / neon_helper.c
CommitLineData
e677137d
PB		 1/*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licenced under the GNU GPL v2.
8 */
ad69471c
PB		 9#include <stdlib.h>
10#include <stdio.h>
11
12#include "cpu.h"
13#include "exec-all.h"
14#include "helpers.h"
15
16#define SIGNBIT (uint32_t)0x80000000
17#define SIGNBIT64 ((uint64_t)1 << 63)
18
19#define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
20
21static float_status neon_float_status;
22#define NFS &neon_float_status
23
ad69471c
PB		 24#define NEON_TYPE1(name, type) \
25typedef struct \
26{ \
27 type v1; \
28} neon_##name;
e2542fe2 29#ifdef HOST_WORDS_BIGENDIAN
ad69471c
PB		 30#define NEON_TYPE2(name, type) \
31typedef struct \
32{ \
33 type v2; \
34 type v1; \
35} neon_##name;
36#define NEON_TYPE4(name, type) \
37typedef struct \
38{ \
39 type v4; \
40 type v3; \
41 type v2; \
42 type v1; \
43} neon_##name;
44#else
45#define NEON_TYPE2(name, type) \
46typedef struct \
47{ \
48 type v1; \
49 type v2; \
50} neon_##name;
51#define NEON_TYPE4(name, type) \
52typedef struct \
53{ \
54 type v1; \
55 type v2; \
56 type v3; \
57 type v4; \
58} neon_##name;
59#endif
60
61NEON_TYPE4(s8, int8_t)
62NEON_TYPE4(u8, uint8_t)
63NEON_TYPE2(s16, int16_t)
64NEON_TYPE2(u16, uint16_t)
65NEON_TYPE1(s32, int32_t)
66NEON_TYPE1(u32, uint32_t)
67#undef NEON_TYPE4
68#undef NEON_TYPE2
69#undef NEON_TYPE1
70
71/* Copy from a uint32_t to a vector structure type. */
72#define NEON_UNPACK(vtype, dest, val) do { \
73 union { \
74 vtype v; \
75 uint32_t i; \
76 } conv_u; \
77 conv_u.i = (val); \
78 dest = conv_u.v; \
79 } while(0)
80
81/* Copy from a vector structure type to a uint32_t. */
82#define NEON_PACK(vtype, dest, val) do { \
83 union { \
84 vtype v; \
85 uint32_t i; \
86 } conv_u; \
87 conv_u.v = (val); \
88 dest = conv_u.i; \
89 } while(0)
90
91#define NEON_DO1 \
92 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
93#define NEON_DO2 \
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
96#define NEON_DO4 \
97 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
98 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
99 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
100 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
101
102#define NEON_VOP_BODY(vtype, n) \
103{ \
104 uint32_t res; \
105 vtype vsrc1; \
106 vtype vsrc2; \
107 vtype vdest; \
108 NEON_UNPACK(vtype, vsrc1, arg1); \
109 NEON_UNPACK(vtype, vsrc2, arg2); \
110 NEON_DO##n; \
111 NEON_PACK(vtype, res, vdest); \
112 return res; \
113}
114
115#define NEON_VOP(name, vtype, n) \
116uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
117NEON_VOP_BODY(vtype, n)
118
119#define NEON_VOP_ENV(name, vtype, n) \
120uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
121NEON_VOP_BODY(vtype, n)
122
123/* Pairwise operations. */
124/* For 32-bit elements each segment only contains a single element, so
125 the elementwise and pairwise operations are the same. */
126#define NEON_PDO2 \
127 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
128 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
129#define NEON_PDO4 \
130 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
131 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
132 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
133 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
134
135#define NEON_POP(name, vtype, n) \
136uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
137{ \
138 uint32_t res; \
139 vtype vsrc1; \
140 vtype vsrc2; \
141 vtype vdest; \
142 NEON_UNPACK(vtype, vsrc1, arg1); \
143 NEON_UNPACK(vtype, vsrc2, arg2); \
144 NEON_PDO##n; \
145 NEON_PACK(vtype, res, vdest); \
146 return res; \
147}
148
149/* Unary operators. */
150#define NEON_VOP1(name, vtype, n) \
151uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
152{ \
153 vtype vsrc1; \
154 vtype vdest; \
155 NEON_UNPACK(vtype, vsrc1, arg); \
156 NEON_DO##n; \
157 NEON_PACK(vtype, arg, vdest); \
158 return arg; \
159}
160
161
162#define NEON_USAT(dest, src1, src2, type) do { \
163 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
164 if (tmp != (type)tmp) { \
165 SET_QC(); \
166 dest = ~0; \
167 } else { \
168 dest = tmp; \
169 }} while(0)
170#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
171NEON_VOP_ENV(qadd_u8, neon_u8, 4)
172#undef NEON_FN
173#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
174NEON_VOP_ENV(qadd_u16, neon_u16, 2)
175#undef NEON_FN
176#undef NEON_USAT
177
72902672
CL		 178uint32_t HELPER(neon_qadd_u32)(CPUState *env, uint32_t a, uint32_t b)
179{
180 uint32_t res = a + b;
181 if (res < a) {
182 SET_QC();
183 res = ~0;
184 }
185 return res;
186}
187
188uint64_t HELPER(neon_qadd_u64)(CPUState *env, uint64_t src1, uint64_t src2)
189{
190 uint64_t res;
191
192 res = src1 + src2;
193 if (res < src1) {
194 SET_QC();
195 res = ~(uint64_t)0;
196 }
197 return res;
198}
199
ad69471c
PB		 200#define NEON_SSAT(dest, src1, src2, type) do { \
201 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
202 if (tmp != (type)tmp) { \
203 SET_QC(); \
204 if (src2 > 0) { \
205 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
206 } else { \
207 tmp = 1 << (sizeof(type) * 8 - 1); \
208 } \
209 } \
210 dest = tmp; \
211 } while(0)
212#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
213NEON_VOP_ENV(qadd_s8, neon_s8, 4)
214#undef NEON_FN
215#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
216NEON_VOP_ENV(qadd_s16, neon_s16, 2)
217#undef NEON_FN
218#undef NEON_SSAT
219
72902672
CL		 220uint32_t HELPER(neon_qadd_s32)(CPUState *env, uint32_t a, uint32_t b)
221{
222 uint32_t res = a + b;
223 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
224 SET_QC();
225 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
226 }
227 return res;
228}
229
230uint64_t HELPER(neon_qadd_s64)(CPUState *env, uint64_t src1, uint64_t src2)
231{
232 uint64_t res;
233
234 res = src1 + src2;
235 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
236 SET_QC();
237 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
238 }
239 return res;
240}
241
ad69471c
PB		 242#define NEON_USAT(dest, src1, src2, type) do { \
243 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
244 if (tmp != (type)tmp) { \
245 SET_QC(); \
246 dest = 0; \
247 } else { \
248 dest = tmp; \
249 }} while(0)
250#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
251NEON_VOP_ENV(qsub_u8, neon_u8, 4)
252#undef NEON_FN
253#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
254NEON_VOP_ENV(qsub_u16, neon_u16, 2)
255#undef NEON_FN
256#undef NEON_USAT
257
72902672
CL		 258uint32_t HELPER(neon_qsub_u32)(CPUState *env, uint32_t a, uint32_t b)
259{
260 uint32_t res = a - b;
261 if (res > a) {
262 SET_QC();
263 res = 0;
264 }
265 return res;
266}
267
268uint64_t HELPER(neon_qsub_u64)(CPUState *env, uint64_t src1, uint64_t src2)
269{
270 uint64_t res;
271
272 if (src1 < src2) {
273 SET_QC();
274 res = 0;
275 } else {
276 res = src1 - src2;
277 }
278 return res;
279}
280
ad69471c
PB		 281#define NEON_SSAT(dest, src1, src2, type) do { \
282 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
283 if (tmp != (type)tmp) { \
284 SET_QC(); \
285 if (src2 < 0) { \
286 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
287 } else { \
288 tmp = 1 << (sizeof(type) * 8 - 1); \
289 } \
290 } \
291 dest = tmp; \
292 } while(0)
293#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
294NEON_VOP_ENV(qsub_s8, neon_s8, 4)
295#undef NEON_FN
296#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
297NEON_VOP_ENV(qsub_s16, neon_s16, 2)
298#undef NEON_FN
299#undef NEON_SSAT
300
72902672
CL		 301uint32_t HELPER(neon_qsub_s32)(CPUState *env, uint32_t a, uint32_t b)
302{
303 uint32_t res = a - b;
304 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
305 SET_QC();
306 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
307 }
308 return res;
309}
310
311uint64_t HELPER(neon_qsub_s64)(CPUState *env, uint64_t src1, uint64_t src2)
312{
313 uint64_t res;
314
315 res = src1 - src2;
316 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
317 SET_QC();
318 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
319 }
320 return res;
321}
322
ad69471c
PB		 323#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
324NEON_VOP(hadd_s8, neon_s8, 4)
325NEON_VOP(hadd_u8, neon_u8, 4)
326NEON_VOP(hadd_s16, neon_s16, 2)
327NEON_VOP(hadd_u16, neon_u16, 2)
328#undef NEON_FN
329
330int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
331{
332 int32_t dest;
333
334 dest = (src1 >> 1) + (src2 >> 1);
335 if (src1 & src2 & 1)
336 dest++;
337 return dest;
338}
339
340uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
341{
342 uint32_t dest;
343
344 dest = (src1 >> 1) + (src2 >> 1);
345 if (src1 & src2 & 1)
346 dest++;
347 return dest;
348}
349
350#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
351NEON_VOP(rhadd_s8, neon_s8, 4)
352NEON_VOP(rhadd_u8, neon_u8, 4)
353NEON_VOP(rhadd_s16, neon_s16, 2)
354NEON_VOP(rhadd_u16, neon_u16, 2)
355#undef NEON_FN
356
357int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
358{
359 int32_t dest;
360
361 dest = (src1 >> 1) + (src2 >> 1);
362 if ((src1 | src2) & 1)
363 dest++;
364 return dest;
365}
366
367uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
368{
369 uint32_t dest;
370
371 dest = (src1 >> 1) + (src2 >> 1);
372 if ((src1 | src2) & 1)
373 dest++;
374 return dest;
375}
376
377#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
378NEON_VOP(hsub_s8, neon_s8, 4)
379NEON_VOP(hsub_u8, neon_u8, 4)
380NEON_VOP(hsub_s16, neon_s16, 2)
381NEON_VOP(hsub_u16, neon_u16, 2)
382#undef NEON_FN
383
384int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
385{
386 int32_t dest;
387
388 dest = (src1 >> 1) - (src2 >> 1);
389 if ((~src1) & src2 & 1)
390 dest--;
391 return dest;
392}
393
394uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
395{
396 uint32_t dest;
397
398 dest = (src1 >> 1) - (src2 >> 1);
399 if ((~src1) & src2 & 1)
400 dest--;
401 return dest;
402}
403
404#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
405NEON_VOP(cgt_s8, neon_s8, 4)
406NEON_VOP(cgt_u8, neon_u8, 4)
407NEON_VOP(cgt_s16, neon_s16, 2)
408NEON_VOP(cgt_u16, neon_u16, 2)
409NEON_VOP(cgt_s32, neon_s32, 1)
410NEON_VOP(cgt_u32, neon_u32, 1)
411#undef NEON_FN
412
413#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
414NEON_VOP(cge_s8, neon_s8, 4)
415NEON_VOP(cge_u8, neon_u8, 4)
416NEON_VOP(cge_s16, neon_s16, 2)
417NEON_VOP(cge_u16, neon_u16, 2)
418NEON_VOP(cge_s32, neon_s32, 1)
419NEON_VOP(cge_u32, neon_u32, 1)
420#undef NEON_FN
421
422#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
423NEON_VOP(min_s8, neon_s8, 4)
424NEON_VOP(min_u8, neon_u8, 4)
425NEON_VOP(min_s16, neon_s16, 2)
426NEON_VOP(min_u16, neon_u16, 2)
427NEON_VOP(min_s32, neon_s32, 1)
428NEON_VOP(min_u32, neon_u32, 1)
429NEON_POP(pmin_s8, neon_s8, 4)
430NEON_POP(pmin_u8, neon_u8, 4)
431NEON_POP(pmin_s16, neon_s16, 2)
432NEON_POP(pmin_u16, neon_u16, 2)
433#undef NEON_FN
434
435#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
436NEON_VOP(max_s8, neon_s8, 4)
437NEON_VOP(max_u8, neon_u8, 4)
438NEON_VOP(max_s16, neon_s16, 2)
439NEON_VOP(max_u16, neon_u16, 2)
440NEON_VOP(max_s32, neon_s32, 1)
441NEON_VOP(max_u32, neon_u32, 1)
442NEON_POP(pmax_s8, neon_s8, 4)
443NEON_POP(pmax_u8, neon_u8, 4)
444NEON_POP(pmax_s16, neon_s16, 2)
445NEON_POP(pmax_u16, neon_u16, 2)
446#undef NEON_FN
447
448#define NEON_FN(dest, src1, src2) \
449 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
450NEON_VOP(abd_s8, neon_s8, 4)
451NEON_VOP(abd_u8, neon_u8, 4)
452NEON_VOP(abd_s16, neon_s16, 2)
453NEON_VOP(abd_u16, neon_u16, 2)
454NEON_VOP(abd_s32, neon_s32, 1)
455NEON_VOP(abd_u32, neon_u32, 1)
456#undef NEON_FN
457
458#define NEON_FN(dest, src1, src2) do { \
459 int8_t tmp; \
460 tmp = (int8_t)src2; \
50f67e95
JR		 461 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
462 tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 463 dest = 0; \
464 } else if (tmp < 0) { \
465 dest = src1 >> -tmp; \
466 } else { \
467 dest = src1 << tmp; \
468 }} while (0)
469NEON_VOP(shl_u8, neon_u8, 4)
470NEON_VOP(shl_u16, neon_u16, 2)
471NEON_VOP(shl_u32, neon_u32, 1)
472#undef NEON_FN
473
474uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
475{
476 int8_t shift = (int8_t)shiftop;
477 if (shift >= 64 || shift <= -64) {
478 val = 0;
479 } else if (shift < 0) {
480 val >>= -shift;
481 } else {
482 val <<= shift;
483 }
484 return val;
485}
486
487#define NEON_FN(dest, src1, src2) do { \
488 int8_t tmp; \
489 tmp = (int8_t)src2; \
50f67e95 490 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c 491 dest = 0; \
50f67e95 492 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 493 dest = src1 >> (sizeof(src1) * 8 - 1); \
494 } else if (tmp < 0) { \
495 dest = src1 >> -tmp; \
496 } else { \
497 dest = src1 << tmp; \
498 }} while (0)
499NEON_VOP(shl_s8, neon_s8, 4)
500NEON_VOP(shl_s16, neon_s16, 2)
501NEON_VOP(shl_s32, neon_s32, 1)
502#undef NEON_FN
503
504uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
505{
506 int8_t shift = (int8_t)shiftop;
507 int64_t val = valop;
508 if (shift >= 64) {
509 val = 0;
510 } else if (shift <= -64) {
511 val >>= 63;
512 } else if (shift < 0) {
513 val >>= -shift;
514 } else {
515 val <<= shift;
516 }
517 return val;
518}
519
520#define NEON_FN(dest, src1, src2) do { \
521 int8_t tmp; \
522 tmp = (int8_t)src2; \
0670a7b6
PM		 523 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
524 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
ad69471c 525 dest = 0; \
ad69471c
PB		 526 } else if (tmp < 0) { \
527 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
528 } else { \
529 dest = src1 << tmp; \
530 }} while (0)
531NEON_VOP(rshl_s8, neon_s8, 4)
532NEON_VOP(rshl_s16, neon_s16, 2)
ad69471c
PB		 533#undef NEON_FN
534
4bd4ee07
CL		 535/* The addition of the rounding constant may overflow, so we use an
536 * intermediate 64 bits accumulator. */
537uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
538{
539 int32_t dest;
540 int32_t val = (int32_t)valop;
541 int8_t shift = (int8_t)shiftop;
542 if ((shift >= 32) || (shift <= -32)) {
543 dest = 0;
544 } else if (shift < 0) {
545 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
546 dest = big_dest >> -shift;
547 } else {
548 dest = val << shift;
549 }
550 return dest;
551}
552
553/* Handling addition overflow with 64 bits inputs values is more
554 * tricky than with 32 bits values. */
ad69471c
PB		 555uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
556{
557 int8_t shift = (int8_t)shiftop;
558 int64_t val = valop;
0670a7b6 559 if ((shift >= 64) || (shift <= -64)) {
ad69471c 560 val = 0;
ad69471c 561 } else if (shift < 0) {
4bd4ee07
CL		 562 val >>= (-shift - 1);
563 if (val == INT64_MAX) {
564 /* In this case, it means that the rounding constant is 1,
565 * and the addition would overflow. Return the actual
566 * result directly. */
567 val = 0x4000000000000000LL;
568 } else {
569 val++;
570 val >>= 1;
571 }
ad69471c
PB		 572 } else {
573 val <<= shift;
574 }
575 return val;
576}
577
578#define NEON_FN(dest, src1, src2) do { \
579 int8_t tmp; \
580 tmp = (int8_t)src2; \
50f67e95
JR		 581 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
582 tmp < -(ssize_t)sizeof(src1) * 8) { \
ad69471c 583 dest = 0; \
50f67e95 584 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
b6c63b98 585 dest = src1 >> (-tmp - 1); \
ad69471c
PB		 586 } else if (tmp < 0) { \
587 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
588 } else { \
589 dest = src1 << tmp; \
590 }} while (0)
591NEON_VOP(rshl_u8, neon_u8, 4)
592NEON_VOP(rshl_u16, neon_u16, 2)
ad69471c
PB		 593#undef NEON_FN
594
4bd4ee07
CL		 595/* The addition of the rounding constant may overflow, so we use an
596 * intermediate 64 bits accumulator. */
597uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
598{
599 uint32_t dest;
600 int8_t shift = (int8_t)shiftop;
601 if (shift >= 32 || shift < -32) {
602 dest = 0;
603 } else if (shift == -32) {
604 dest = val >> 31;
605 } else if (shift < 0) {
606 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
607 dest = big_dest >> -shift;
608 } else {
609 dest = val << shift;
610 }
611 return dest;
612}
613
614/* Handling addition overflow with 64 bits inputs values is more
615 * tricky than with 32 bits values. */
ad69471c
PB		 616uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
617{
618 int8_t shift = (uint8_t)shiftop;
51e3930f 619 if (shift >= 64 || shift < -64) {
ad69471c
PB		 620 val = 0;
621 } else if (shift == -64) {
622 /* Rounding a 1-bit result just preserves that bit. */
623 val >>= 63;
4bd4ee07
CL		 624 } else if (shift < 0) {
625 val >>= (-shift - 1);
626 if (val == UINT64_MAX) {
627 /* In this case, it means that the rounding constant is 1,
628 * and the addition would overflow. Return the actual
629 * result directly. */
630 val = 0x8000000000000000ULL;
631 } else {
632 val++;
633 val >>= 1;
634 }
ad69471c
PB		 635 } else {
636 val <<= shift;
637 }
638 return val;
639}
640
641#define NEON_FN(dest, src1, src2) do { \
642 int8_t tmp; \
643 tmp = (int8_t)src2; \
50f67e95 644 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 645 if (src1) { \
646 SET_QC(); \
647 dest = ~0; \
648 } else { \
649 dest = 0; \
650 } \
50f67e95 651 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 652 dest = 0; \
653 } else if (tmp < 0) { \
654 dest = src1 >> -tmp; \
655 } else { \
656 dest = src1 << tmp; \
657 if ((dest >> tmp) != src1) { \
658 SET_QC(); \
659 dest = ~0; \
660 } \
661 }} while (0)
662NEON_VOP_ENV(qshl_u8, neon_u8, 4)
663NEON_VOP_ENV(qshl_u16, neon_u16, 2)
664NEON_VOP_ENV(qshl_u32, neon_u32, 1)
665#undef NEON_FN
666
667uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
668{
669 int8_t shift = (int8_t)shiftop;
670 if (shift >= 64) {
671 if (val) {
672 val = ~(uint64_t)0;
673 SET_QC();
ad69471c
PB		 674 }
675 } else if (shift <= -64) {
676 val = 0;
677 } else if (shift < 0) {
678 val >>= -shift;
679 } else {
680 uint64_t tmp = val;
681 val <<= shift;
682 if ((val >> shift) != tmp) {
683 SET_QC();
684 val = ~(uint64_t)0;
685 }
686 }
687 return val;
688}
689
690#define NEON_FN(dest, src1, src2) do { \
691 int8_t tmp; \
692 tmp = (int8_t)src2; \
50f67e95 693 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
a5d88f3e 694 if (src1) { \
ad69471c 695 SET_QC(); \
a5d88f3e
PM		 696 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
697 if (src1 > 0) { \
698 dest--; \
699 } \
700 } else { \
701 dest = src1; \
702 } \
50f67e95 703 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 704 dest = src1 >> 31; \
705 } else if (tmp < 0) { \
706 dest = src1 >> -tmp; \
707 } else { \
708 dest = src1 << tmp; \
709 if ((dest >> tmp) != src1) { \
710 SET_QC(); \
a5d88f3e
PM		 711 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
712 if (src1 > 0) { \
713 dest--; \
714 } \
ad69471c
PB		 715 } \
716 }} while (0)
717NEON_VOP_ENV(qshl_s8, neon_s8, 4)
718NEON_VOP_ENV(qshl_s16, neon_s16, 2)
719NEON_VOP_ENV(qshl_s32, neon_s32, 1)
720#undef NEON_FN
721
722uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
723{
724 int8_t shift = (uint8_t)shiftop;
725 int64_t val = valop;
726 if (shift >= 64) {
727 if (val) {
728 SET_QC();
eb7a3d79 729 val = (val >> 63) ^ ~SIGNBIT64;
ad69471c 730 }
4c9b70ae 731 } else if (shift <= -64) {
ad69471c
PB		 732 val >>= 63;
733 } else if (shift < 0) {
734 val >>= -shift;
735 } else {
736 int64_t tmp = val;
737 val <<= shift;
738 if ((val >> shift) != tmp) {
739 SET_QC();
740 val = (tmp >> 63) ^ ~SIGNBIT64;
741 }
742 }
743 return val;
744}
745
4ca4502c
JR		 746#define NEON_FN(dest, src1, src2) do { \
747 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
748 SET_QC(); \
749 dest = 0; \
750 } else { \
751 int8_t tmp; \
752 tmp = (int8_t)src2; \
753 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
754 if (src1) { \
755 SET_QC(); \
756 dest = ~0; \
757 } else { \
758 dest = 0; \
759 } \
760 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
761 dest = 0; \
762 } else if (tmp < 0) { \
763 dest = src1 >> -tmp; \
764 } else { \
765 dest = src1 << tmp; \
766 if ((dest >> tmp) != src1) { \
767 SET_QC(); \
768 dest = ~0; \
769 } \
770 } \
771 }} while (0)
772NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
773NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
774#undef NEON_FN
775
776uint32_t HELPER(neon_qshlu_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
777{
778 if ((int32_t)valop < 0) {
779 SET_QC();
780 return 0;
781 }
782 return helper_neon_qshl_u32(env, valop, shiftop);
783}
784
785uint64_t HELPER(neon_qshlu_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
786{
787 if ((int64_t)valop < 0) {
788 SET_QC();
789 return 0;
790 }
791 return helper_neon_qshl_u64(env, valop, shiftop);
792}
ad69471c
PB		 793
794/* FIXME: This is wrong. */
795#define NEON_FN(dest, src1, src2) do { \
796 int8_t tmp; \
797 tmp = (int8_t)src2; \
33ebc293
PM		 798 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
799 if (src1) { \
800 SET_QC(); \
801 dest = ~0; \
802 } else { \
803 dest = 0; \
804 } \
805 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
806 dest = 0; \
807 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
808 dest = src1 >> (sizeof(src1) * 8 - 1); \
809 } else if (tmp < 0) { \
ad69471c
PB		 810 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
811 } else { \
812 dest = src1 << tmp; \
813 if ((dest >> tmp) != src1) { \
814 SET_QC(); \
815 dest = ~0; \
816 } \
817 }} while (0)
818NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
819NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
ad69471c
PB		 820#undef NEON_FN
821
4bd4ee07
CL		 822/* The addition of the rounding constant may overflow, so we use an
823 * intermediate 64 bits accumulator. */
824uint32_t HELPER(neon_qrshl_u32)(CPUState *env, uint32_t val, uint32_t shiftop)
825{
826 uint32_t dest;
827 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 828 if (shift >= 32) {
829 if (val) {
830 SET_QC();
831 dest = ~0;
832 } else {
833 dest = 0;
834 }
835 } else if (shift < -32) {
836 dest = 0;
837 } else if (shift == -32) {
838 dest = val >> 31;
839 } else if (shift < 0) {
4bd4ee07
CL		 840 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
841 dest = big_dest >> -shift;
842 } else {
843 dest = val << shift;
844 if ((dest >> shift) != val) {
845 SET_QC();
846 dest = ~0;
847 }
848 }
849 return dest;
850}
851
852/* Handling addition overflow with 64 bits inputs values is more
853 * tricky than with 32 bits values. */
ad69471c
PB		 854uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
855{
856 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 857 if (shift >= 64) {
858 if (val) {
859 SET_QC();
860 val = ~0;
861 }
862 } else if (shift < -64) {
863 val = 0;
864 } else if (shift == -64) {
865 val >>= 63;
866 } else if (shift < 0) {
4bd4ee07
CL		 867 val >>= (-shift - 1);
868 if (val == UINT64_MAX) {
869 /* In this case, it means that the rounding constant is 1,
870 * and the addition would overflow. Return the actual
871 * result directly. */
872 val = 0x8000000000000000ULL;
873 } else {
874 val++;
875 val >>= 1;
876 }
ad69471c
PB		 877 } else { \
878 uint64_t tmp = val;
879 val <<= shift;
880 if ((val >> shift) != tmp) {
881 SET_QC();
882 val = ~0;
883 }
884 }
885 return val;
886}
887
888#define NEON_FN(dest, src1, src2) do { \
889 int8_t tmp; \
890 tmp = (int8_t)src2; \
7b6ecf5b
PM		 891 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
892 if (src1) { \
893 SET_QC(); \
894 dest = (1 << (sizeof(src1) * 8 - 1)); \
895 if (src1 > 0) { \
896 dest--; \
897 } \
898 } else { \
899 dest = 0; \
900 } \
901 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
902 dest = 0; \
903 } else if (tmp < 0) { \
ad69471c
PB		 904 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
905 } else { \
906 dest = src1 << tmp; \
907 if ((dest >> tmp) != src1) { \
908 SET_QC(); \
960e623b
PM		 909 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
910 if (src1 > 0) { \
911 dest--; \
912 } \
ad69471c
PB		 913 } \
914 }} while (0)
915NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
916NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
ad69471c
PB		 917#undef NEON_FN
918
4bd4ee07
CL		 919/* The addition of the rounding constant may overflow, so we use an
920 * intermediate 64 bits accumulator. */
921uint32_t HELPER(neon_qrshl_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
922{
923 int32_t dest;
924 int32_t val = (int32_t)valop;
925 int8_t shift = (int8_t)shiftop;
7b6ecf5b
PM		 926 if (shift >= 32) {
927 if (val) {
928 SET_QC();
929 dest = (val >> 31) ^ ~SIGNBIT;
930 } else {
931 dest = 0;
932 }
933 } else if (shift <= -32) {
934 dest = 0;
935 } else if (shift < 0) {
4bd4ee07
CL		 936 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
937 dest = big_dest >> -shift;
938 } else {
939 dest = val << shift;
940 if ((dest >> shift) != val) {
941 SET_QC();
942 dest = (val >> 31) ^ ~SIGNBIT;
943 }
944 }
945 return dest;
946}
947
948/* Handling addition overflow with 64 bits inputs values is more
949 * tricky than with 32 bits values. */
ad69471c
PB		 950uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
951{
952 int8_t shift = (uint8_t)shiftop;
953 int64_t val = valop;
954
7b6ecf5b
PM		 955 if (shift >= 64) {
956 if (val) {
957 SET_QC();
958 val = (val >> 63) ^ ~SIGNBIT64;
959 }
960 } else if (shift <= -64) {
961 val = 0;
962 } else if (shift < 0) {
4bd4ee07
CL		 963 val >>= (-shift - 1);
964 if (val == INT64_MAX) {
965 /* In this case, it means that the rounding constant is 1,
966 * and the addition would overflow. Return the actual
967 * result directly. */
968 val = 0x4000000000000000ULL;
969 } else {
970 val++;
971 val >>= 1;
972 }
ad69471c 973 } else {
4bd4ee07 974 int64_t tmp = val;
ad69471c
PB		 975 val <<= shift;
976 if ((val >> shift) != tmp) {
977 SET_QC();
4bd4ee07 978 val = (tmp >> 63) ^ ~SIGNBIT64;
ad69471c
PB		 979 }
980 }
981 return val;
982}
983
984uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
985{
986 uint32_t mask;
987 mask = (a ^ b) & 0x80808080u;
988 a &= ~0x80808080u;
989 b &= ~0x80808080u;
990 return (a + b) ^ mask;
991}
992
993uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
994{
995 uint32_t mask;
996 mask = (a ^ b) & 0x80008000u;
997 a &= ~0x80008000u;
998 b &= ~0x80008000u;
999 return (a + b) ^ mask;
1000}
1001
1002#define NEON_FN(dest, src1, src2) dest = src1 + src2
1003NEON_POP(padd_u8, neon_u8, 4)
1004NEON_POP(padd_u16, neon_u16, 2)
1005#undef NEON_FN
1006
1007#define NEON_FN(dest, src1, src2) dest = src1 - src2
1008NEON_VOP(sub_u8, neon_u8, 4)
1009NEON_VOP(sub_u16, neon_u16, 2)
1010#undef NEON_FN
1011
1012#define NEON_FN(dest, src1, src2) dest = src1 * src2
1013NEON_VOP(mul_u8, neon_u8, 4)
1014NEON_VOP(mul_u16, neon_u16, 2)
1015#undef NEON_FN
1016
1654b2d6 1017/* Polynomial multiplication is like integer multiplication except the
ad69471c
PB		 1018 partial products are XORed, not added. */
1019uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1020{
1021 uint32_t mask;
1022 uint32_t result;
1023 result = 0;
1024 while (op1) {
1025 mask = 0;
1026 if (op1 & 1)
1027 mask |= 0xff;
1028 if (op1 & (1 << 8))
1029 mask |= (0xff << 8);
1030 if (op1 & (1 << 16))
1031 mask |= (0xff << 16);
1032 if (op1 & (1 << 24))
1033 mask |= (0xff << 24);
1034 result ^= op2 & mask;
1035 op1 = (op1 >> 1) & 0x7f7f7f7f;
1036 op2 = (op2 << 1) & 0xfefefefe;
1037 }
1038 return result;
1039}
1040
e5ca24cb
PM		 1041uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1042{
1043 uint64_t result = 0;
1044 uint64_t mask;
1045 uint64_t op2ex = op2;
1046 op2ex = (op2ex & 0xff) |
1047 ((op2ex & 0xff00) << 8) |
1048 ((op2ex & 0xff0000) << 16) |
1049 ((op2ex & 0xff000000) << 24);
1050 while (op1) {
1051 mask = 0;
1052 if (op1 & 1) {
1053 mask |= 0xffff;
1054 }
1055 if (op1 & (1 << 8)) {
1056 mask |= (0xffffU << 16);
1057 }
1058 if (op1 & (1 << 16)) {
1059 mask |= (0xffffULL << 32);
1060 }
1061 if (op1 & (1 << 24)) {
1062 mask |= (0xffffULL << 48);
1063 }
1064 result ^= op2ex & mask;
1065 op1 = (op1 >> 1) & 0x7f7f7f7f;
1066 op2ex <<= 1;
1067 }
1068 return result;
1069}
1070
ad69471c
PB		 1071#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1072NEON_VOP(tst_u8, neon_u8, 4)
1073NEON_VOP(tst_u16, neon_u16, 2)
1074NEON_VOP(tst_u32, neon_u32, 1)
1075#undef NEON_FN
1076
1077#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1078NEON_VOP(ceq_u8, neon_u8, 4)
1079NEON_VOP(ceq_u16, neon_u16, 2)
1080NEON_VOP(ceq_u32, neon_u32, 1)
1081#undef NEON_FN
1082
1083#define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1084NEON_VOP1(abs_s8, neon_s8, 4)
1085NEON_VOP1(abs_s16, neon_s16, 2)
1086#undef NEON_FN
1087
1088/* Count Leading Sign/Zero Bits. */
1089static inline int do_clz8(uint8_t x)
1090{
1091 int n;
1092 for (n = 8; x; n--)
1093 x >>= 1;
1094 return n;
1095}
1096
1097static inline int do_clz16(uint16_t x)
1098{
1099 int n;
1100 for (n = 16; x; n--)
1101 x >>= 1;
1102 return n;
1103}
1104
1105#define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1106NEON_VOP1(clz_u8, neon_u8, 4)
1107#undef NEON_FN
1108
1109#define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1110NEON_VOP1(clz_u16, neon_u16, 2)
1111#undef NEON_FN
1112
1113#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1114NEON_VOP1(cls_s8, neon_s8, 4)
1115#undef NEON_FN
1116
1117#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1118NEON_VOP1(cls_s16, neon_s16, 2)
1119#undef NEON_FN
1120
1121uint32_t HELPER(neon_cls_s32)(uint32_t x)
1122{
1123 int count;
1124 if ((int32_t)x < 0)
1125 x = ~x;
1126 for (count = 32; x; count--)
1127 x = x >> 1;
1128 return count - 1;
1129}
1130
1131/* Bit count. */
1132uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1133{
1134 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1135 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1136 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1137 return x;
1138}
1139
1140#define NEON_QDMULH16(dest, src1, src2, round) do { \
1141 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1142 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1143 SET_QC(); \
1144 tmp = (tmp >> 31) ^ ~SIGNBIT; \
46eece9d
JR		 1145 } else { \
1146 tmp <<= 1; \
ad69471c 1147 } \
ad69471c
PB		 1148 if (round) { \
1149 int32_t old = tmp; \
1150 tmp += 1 << 15; \
1151 if ((int32_t)tmp < old) { \
1152 SET_QC(); \
1153 tmp = SIGNBIT - 1; \
1154 } \
1155 } \
1156 dest = tmp >> 16; \
1157 } while(0)
1158#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1159NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1160#undef NEON_FN
1161#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1162NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1163#undef NEON_FN
1164#undef NEON_QDMULH16
1165
1166#define NEON_QDMULH32(dest, src1, src2, round) do { \
1167 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1168 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1169 SET_QC(); \
1170 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1171 } else { \
1172 tmp <<= 1; \
1173 } \
1174 if (round) { \
1175 int64_t old = tmp; \
1176 tmp += (int64_t)1 << 31; \
1177 if ((int64_t)tmp < old) { \
1178 SET_QC(); \
1179 tmp = SIGNBIT64 - 1; \
1180 } \
1181 } \
1182 dest = tmp >> 32; \
1183 } while(0)
1184#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1185NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1186#undef NEON_FN
1187#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1188NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1189#undef NEON_FN
1190#undef NEON_QDMULH32
1191
1192uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1193{
1194 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1195 | ((x >> 24) & 0xff000000u);
1196}
1197
1198uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1199{
1200 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1201}
1202
1203uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1204{
1205 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1206 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1207}
1208
1209uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1210{
1211 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1212}
1213
1214uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1215{
1216 x &= 0xff80ff80ff80ff80ull;
1217 x += 0x0080008000800080ull;
1218 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1219 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1220}
1221
1222uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1223{
1224 x &= 0xffff8000ffff8000ull;
1225 x += 0x0000800000008000ull;
1226 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1227}
1228
af1bbf30
JR		 1229uint32_t HELPER(neon_unarrow_sat8)(CPUState *env, uint64_t x)
1230{
1231 uint16_t s;
1232 uint8_t d;
1233 uint32_t res = 0;
1234#define SAT8(n) \
1235 s = x >> n; \
1236 if (s & 0x8000) { \
1237 SET_QC(); \
1238 } else { \
1239 if (s > 0xff) { \
1240 d = 0xff; \
1241 SET_QC(); \
1242 } else { \
1243 d = s; \
1244 } \
1245 res |= (uint32_t)d << (n / 2); \
1246 }
1247
1248 SAT8(0);
1249 SAT8(16);
1250 SAT8(32);
1251 SAT8(48);
1252#undef SAT8
1253 return res;
1254}
1255
ad69471c
PB		 1256uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
1257{
1258 uint16_t s;
1259 uint8_t d;
1260 uint32_t res = 0;
1261#define SAT8(n) \
1262 s = x >> n; \
1263 if (s > 0xff) { \
1264 d = 0xff; \
1265 SET_QC(); \
1266 } else { \
1267 d = s; \
1268 } \
1269 res |= (uint32_t)d << (n / 2);
1270
1271 SAT8(0);
1272 SAT8(16);
1273 SAT8(32);
1274 SAT8(48);
1275#undef SAT8
1276 return res;
1277}
1278
1279uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
1280{
1281 int16_t s;
1282 uint8_t d;
1283 uint32_t res = 0;
1284#define SAT8(n) \
1285 s = x >> n; \
1286 if (s != (int8_t)s) { \
1287 d = (s >> 15) ^ 0x7f; \
1288 SET_QC(); \
1289 } else { \
1290 d = s; \
1291 } \
1292 res |= (uint32_t)d << (n / 2);
1293
1294 SAT8(0);
1295 SAT8(16);
1296 SAT8(32);
1297 SAT8(48);
1298#undef SAT8
1299 return res;
1300}
1301
af1bbf30
JR		 1302uint32_t HELPER(neon_unarrow_sat16)(CPUState *env, uint64_t x)
1303{
1304 uint32_t high;
1305 uint32_t low;
1306 low = x;
1307 if (low & 0x80000000) {
1308 low = 0;
1309 SET_QC();
1310 } else if (low > 0xffff) {
1311 low = 0xffff;
1312 SET_QC();
1313 }
1314 high = x >> 32;
1315 if (high & 0x80000000) {
1316 high = 0;
1317 SET_QC();
1318 } else if (high > 0xffff) {
1319 high = 0xffff;
1320 SET_QC();
1321 }
1322 return low | (high << 16);
1323}
1324
ad69471c
PB		 1325uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
1326{
1327 uint32_t high;
1328 uint32_t low;
1329 low = x;
1330 if (low > 0xffff) {
1331 low = 0xffff;
1332 SET_QC();
1333 }
1334 high = x >> 32;
1335 if (high > 0xffff) {
1336 high = 0xffff;
1337 SET_QC();
1338 }
1339 return low | (high << 16);
1340}
1341
1342uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
1343{
1344 int32_t low;
1345 int32_t high;
1346 low = x;
1347 if (low != (int16_t)low) {
1348 low = (low >> 31) ^ 0x7fff;
1349 SET_QC();
1350 }
1351 high = x >> 32;
1352 if (high != (int16_t)high) {
1353 high = (high >> 31) ^ 0x7fff;
1354 SET_QC();
1355 }
1356 return (uint16_t)low | (high << 16);
1357}
1358
af1bbf30
JR		 1359uint32_t HELPER(neon_unarrow_sat32)(CPUState *env, uint64_t x)
1360{
1361 if (x & 0x8000000000000000ull) {
1362 SET_QC();
1363 return 0;
1364 }
1365 if (x > 0xffffffffu) {
1366 SET_QC();
1367 return 0xffffffffu;
1368 }
1369 return x;
1370}
1371
ad69471c
PB		 1372uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
1373{
1374 if (x > 0xffffffffu) {
1375 SET_QC();
1376 return 0xffffffffu;
1377 }
1378 return x;
1379}
1380
1381uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)
1382{
1383 if ((int64_t)x != (int32_t)x) {
1384 SET_QC();
cc2212c2 1385 return ((int64_t)x >> 63) ^ 0x7fffffff;
ad69471c
PB		 1386 }
1387 return x;
1388}
1389
1390uint64_t HELPER(neon_widen_u8)(uint32_t x)
1391{
1392 uint64_t tmp;
1393 uint64_t ret;
1394 ret = (uint8_t)x;
1395 tmp = (uint8_t)(x >> 8);
1396 ret |= tmp << 16;
1397 tmp = (uint8_t)(x >> 16);
1398 ret |= tmp << 32;
1399 tmp = (uint8_t)(x >> 24);
1400 ret |= tmp << 48;
1401 return ret;
1402}
1403
1404uint64_t HELPER(neon_widen_s8)(uint32_t x)
1405{
1406 uint64_t tmp;
1407 uint64_t ret;
1408 ret = (uint16_t)(int8_t)x;
1409 tmp = (uint16_t)(int8_t)(x >> 8);
1410 ret |= tmp << 16;
1411 tmp = (uint16_t)(int8_t)(x >> 16);
1412 ret |= tmp << 32;
1413 tmp = (uint16_t)(int8_t)(x >> 24);
1414 ret |= tmp << 48;
1415 return ret;
1416}
1417
1418uint64_t HELPER(neon_widen_u16)(uint32_t x)
1419{
1420 uint64_t high = (uint16_t)(x >> 16);
1421 return ((uint16_t)x) | (high << 32);
1422}
1423
1424uint64_t HELPER(neon_widen_s16)(uint32_t x)
1425{
1426 uint64_t high = (int16_t)(x >> 16);
1427 return ((uint32_t)(int16_t)x) | (high << 32);
1428}
1429
1430uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1431{
1432 uint64_t mask;
1433 mask = (a ^ b) & 0x8000800080008000ull;
1434 a &= ~0x8000800080008000ull;
1435 b &= ~0x8000800080008000ull;
1436 return (a + b) ^ mask;
1437}
1438
1439uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1440{
1441 uint64_t mask;
1442 mask = (a ^ b) & 0x8000000080000000ull;
1443 a &= ~0x8000000080000000ull;
1444 b &= ~0x8000000080000000ull;
1445 return (a + b) ^ mask;
1446}
1447
1448uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1449{
1450 uint64_t tmp;
1451 uint64_t tmp2;
1452
1453 tmp = a & 0x0000ffff0000ffffull;
1454 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1455 tmp2 = b & 0xffff0000ffff0000ull;
1456 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1457 return ( tmp & 0xffff)
1458 | ((tmp >> 16) & 0xffff0000ull)
1459 | ((tmp2 << 16) & 0xffff00000000ull)
1460 | ( tmp2 & 0xffff000000000000ull);
1461}
1462
1463uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1464{
1465 uint32_t low = a + (a >> 32);
1466 uint32_t high = b + (b >> 32);
1467 return low + ((uint64_t)high << 32);
1468}
1469
1470uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1471{
1472 uint64_t mask;
1473 mask = (a ^ ~b) & 0x8000800080008000ull;
1474 a |= 0x8000800080008000ull;
1475 b &= ~0x8000800080008000ull;
1476 return (a - b) ^ mask;
1477}
1478
1479uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1480{
1481 uint64_t mask;
1482 mask = (a ^ ~b) & 0x8000000080000000ull;
1483 a |= 0x8000000080000000ull;
1484 b &= ~0x8000000080000000ull;
1485 return (a - b) ^ mask;
1486}
1487
1488uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
1489{
1490 uint32_t x, y;
1491 uint32_t low, high;
1492
1493 x = a;
1494 y = b;
1495 low = x + y;
1496 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1497 SET_QC();
1498 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1499 }
1500 x = a >> 32;
1501 y = b >> 32;
1502 high = x + y;
1503 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1504 SET_QC();
1505 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1506 }
1507 return low | ((uint64_t)high << 32);
1508}
1509
1510uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
1511{
1512 uint64_t result;
1513
1514 result = a + b;
1515 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1516 SET_QC();
1517 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1518 }
1519 return result;
1520}
1521
1522#define DO_ABD(dest, x, y, type) do { \
1523 type tmp_x = x; \
1524 type tmp_y = y; \
1525 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1526 } while(0)
1527
1528uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1529{
1530 uint64_t tmp;
1531 uint64_t result;
1532 DO_ABD(result, a, b, uint8_t);
1533 DO_ABD(tmp, a >> 8, b >> 8, uint8_t);
1534 result |= tmp << 16;
1535 DO_ABD(tmp, a >> 16, b >> 16, uint8_t);
1536 result |= tmp << 32;
1537 DO_ABD(tmp, a >> 24, b >> 24, uint8_t);
1538 result |= tmp << 48;
1539 return result;
1540}
1541
1542uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1543{
1544 uint64_t tmp;
1545 uint64_t result;
1546 DO_ABD(result, a, b, int8_t);
1547 DO_ABD(tmp, a >> 8, b >> 8, int8_t);
1548 result |= tmp << 16;
1549 DO_ABD(tmp, a >> 16, b >> 16, int8_t);
1550 result |= tmp << 32;
1551 DO_ABD(tmp, a >> 24, b >> 24, int8_t);
1552 result |= tmp << 48;
1553 return result;
1554}
1555
1556uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1557{
1558 uint64_t tmp;
1559 uint64_t result;
1560 DO_ABD(result, a, b, uint16_t);
1561 DO_ABD(tmp, a >> 16, b >> 16, uint16_t);
1562 return result | (tmp << 32);
1563}
1564
1565uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1566{
1567 uint64_t tmp;
1568 uint64_t result;
1569 DO_ABD(result, a, b, int16_t);
1570 DO_ABD(tmp, a >> 16, b >> 16, int16_t);
1571 return result | (tmp << 32);
1572}
1573
1574uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1575{
1576 uint64_t result;
1577 DO_ABD(result, a, b, uint32_t);
1578 return result;
1579}
1580
1581uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1582{
1583 uint64_t result;
1584 DO_ABD(result, a, b, int32_t);
1585 return result;
1586}
1587#undef DO_ABD
1588
1589/* Widening multiply. Named type is the source type. */
1590#define DO_MULL(dest, x, y, type1, type2) do { \
1591 type1 tmp_x = x; \
1592 type1 tmp_y = y; \
1593 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1594 } while(0)
1595
1596uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1597{
1598 uint64_t tmp;
1599 uint64_t result;
1600
1601 DO_MULL(result, a, b, uint8_t, uint16_t);
1602 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1603 result |= tmp << 16;
1604 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1605 result |= tmp << 32;
1606 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1607 result |= tmp << 48;
1608 return result;
1609}
1610
1611uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1612{
1613 uint64_t tmp;
1614 uint64_t result;
1615
1616 DO_MULL(result, a, b, int8_t, uint16_t);
1617 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1618 result |= tmp << 16;
1619 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1620 result |= tmp << 32;
1621 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1622 result |= tmp << 48;
1623 return result;
1624}
1625
1626uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1627{
1628 uint64_t tmp;
1629 uint64_t result;
1630
1631 DO_MULL(result, a, b, uint16_t, uint32_t);
1632 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1633 return result | (tmp << 32);
1634}
1635
1636uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1637{
1638 uint64_t tmp;
1639 uint64_t result;
1640
1641 DO_MULL(result, a, b, int16_t, uint32_t);
1642 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1643 return result | (tmp << 32);
1644}
1645
1646uint64_t HELPER(neon_negl_u16)(uint64_t x)
1647{
1648 uint16_t tmp;
1649 uint64_t result;
1650 result = (uint16_t)-x;
1651 tmp = -(x >> 16);
1652 result |= (uint64_t)tmp << 16;
1653 tmp = -(x >> 32);
1654 result |= (uint64_t)tmp << 32;
1655 tmp = -(x >> 48);
1656 result |= (uint64_t)tmp << 48;
1657 return result;
1658}
1659
ad69471c
PB		 1660uint64_t HELPER(neon_negl_u32)(uint64_t x)
1661{
1662 uint32_t low = -x;
1663 uint32_t high = -(x >> 32);
1664 return low | ((uint64_t)high << 32);
1665}
1666
1667/* FIXME: There should be a native op for this. */
1668uint64_t HELPER(neon_negl_u64)(uint64_t x)
1669{
1670 return -x;
1671}
1672
1673/* Saturnating sign manuipulation. */
1674/* ??? Make these use NEON_VOP1 */
1675#define DO_QABS8(x) do { \
1676 if (x == (int8_t)0x80) { \
1677 x = 0x7f; \
1678 SET_QC(); \
1679 } else if (x < 0) { \
1680 x = -x; \
1681 }} while (0)
1682uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
1683{
1684 neon_s8 vec;
1685 NEON_UNPACK(neon_s8, vec, x);
1686 DO_QABS8(vec.v1);
1687 DO_QABS8(vec.v2);
1688 DO_QABS8(vec.v3);
1689 DO_QABS8(vec.v4);
1690 NEON_PACK(neon_s8, x, vec);
1691 return x;
1692}
1693#undef DO_QABS8
1694
1695#define DO_QNEG8(x) do { \
1696 if (x == (int8_t)0x80) { \
1697 x = 0x7f; \
1698 SET_QC(); \
1699 } else { \
1700 x = -x; \
1701 }} while (0)
1702uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
1703{
1704 neon_s8 vec;
1705 NEON_UNPACK(neon_s8, vec, x);
1706 DO_QNEG8(vec.v1);
1707 DO_QNEG8(vec.v2);
1708 DO_QNEG8(vec.v3);
1709 DO_QNEG8(vec.v4);
1710 NEON_PACK(neon_s8, x, vec);
1711 return x;
1712}
1713#undef DO_QNEG8
1714
1715#define DO_QABS16(x) do { \
1716 if (x == (int16_t)0x8000) { \
1717 x = 0x7fff; \
1718 SET_QC(); \
1719 } else if (x < 0) { \
1720 x = -x; \
1721 }} while (0)
1722uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
1723{
1724 neon_s16 vec;
1725 NEON_UNPACK(neon_s16, vec, x);
1726 DO_QABS16(vec.v1);
1727 DO_QABS16(vec.v2);
1728 NEON_PACK(neon_s16, x, vec);
1729 return x;
1730}
1731#undef DO_QABS16
1732
1733#define DO_QNEG16(x) do { \
1734 if (x == (int16_t)0x8000) { \
1735 x = 0x7fff; \
1736 SET_QC(); \
1737 } else { \
1738 x = -x; \
1739 }} while (0)
1740uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
1741{
1742 neon_s16 vec;
1743 NEON_UNPACK(neon_s16, vec, x);
1744 DO_QNEG16(vec.v1);
1745 DO_QNEG16(vec.v2);
1746 NEON_PACK(neon_s16, x, vec);
1747 return x;
1748}
1749#undef DO_QNEG16
1750
1751uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
1752{
1753 if (x == SIGNBIT) {
1754 SET_QC();
1755 x = ~SIGNBIT;
1756 } else if ((int32_t)x < 0) {
1757 x = -x;
1758 }
1759 return x;
1760}
1761
1762uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
1763{
1764 if (x == SIGNBIT) {
1765 SET_QC();
1766 x = ~SIGNBIT;
1767 } else {
1768 x = -x;
1769 }
1770 return x;
1771}
1772
1773/* NEON Float helpers. */
1774uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1775{
4a9f9cb2 1776 return float32_val(float32_min(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1777}
1778
1779uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1780{
4a9f9cb2 1781 return float32_val(float32_max(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1782}
1783
1784uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1785{
51d85267
PM		 1786 float32 f0 = make_float32(a);
1787 float32 f1 = make_float32(b);
79c18be7 1788 return float32_val(float32_abs(float32_sub(f0, f1, NFS)));
ad69471c
PB		 1789}
1790
1791uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1792{
51d85267 1793 return float32_val(float32_add(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1794}
1795
1796uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1797{
51d85267 1798 return float32_val(float32_sub(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1799}
1800
1801uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1802{
51d85267 1803 return float32_val(float32_mul(make_float32(a), make_float32(b), NFS));
ad69471c
PB		 1804}
1805
1806/* Floating point comparisons produce an integer result. */
c7498dae 1807#define NEON_VOP_FCMP(name, ok) \
ad69471c
PB		 1808uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1809{ \
c7498dae
PM		 1810 switch (float32_compare_quiet(make_float32(a), make_float32(b), NFS)) { \
1811 ok return ~0; \
1812 default: return 0; \
51d85267 1813 } \
ad69471c
PB		 1814}
1815
c7498dae
PM		 1816NEON_VOP_FCMP(ceq_f32, case float_relation_equal:)
1817NEON_VOP_FCMP(cge_f32, case float_relation_equal: case float_relation_greater:)
1818NEON_VOP_FCMP(cgt_f32, case float_relation_greater:)
ad69471c
PB		 1819
1820uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1821{
51d85267
PM		 1822 float32 f0 = float32_abs(make_float32(a));
1823 float32 f1 = float32_abs(make_float32(b));
c7498dae
PM		 1824 switch (float32_compare_quiet(f0, f1, NFS)) {
1825 case float_relation_equal:
1826 case float_relation_greater:
1827 return ~0;
1828 default:
1829 return 0;
1830 }
ad69471c
PB		 1831}
1832
1833uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1834{
51d85267
PM		 1835 float32 f0 = float32_abs(make_float32(a));
1836 float32 f1 = float32_abs(make_float32(b));
c7498dae
PM		 1837 if (float32_compare_quiet(f0, f1, NFS) == float_relation_greater) {
1838 return ~0;
1839 }
1840 return 0;
ad69471c 1841}
02acedf9
PM		 1842
1843#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1844
1845void HELPER(neon_qunzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1846{
1847 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1848 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1849 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1850 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1851 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1852 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1853 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1854 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1855 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1856 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1857 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1858 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1859 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1860 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1861 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1862 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1863 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1864 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1865 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1866 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1867 env->vfp.regs[rm] = make_float64(m0);
1868 env->vfp.regs[rm + 1] = make_float64(m1);
1869 env->vfp.regs[rd] = make_float64(d0);
1870 env->vfp.regs[rd + 1] = make_float64(d1);
1871}
1872
1873void HELPER(neon_qunzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1874{
1875 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1876 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1877 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1878 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1879 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1880 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1881 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1882 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1883 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1884 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1885 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1886 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1887 env->vfp.regs[rm] = make_float64(m0);
1888 env->vfp.regs[rm + 1] = make_float64(m1);
1889 env->vfp.regs[rd] = make_float64(d0);
1890 env->vfp.regs[rd + 1] = make_float64(d1);
1891}
1892
1893void HELPER(neon_qunzip32)(CPUState *env, uint32_t rd, uint32_t rm)
1894{
1895 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1896 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1897 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1898 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1899 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1900 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1901 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1902 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1903 env->vfp.regs[rm] = make_float64(m0);
1904 env->vfp.regs[rm + 1] = make_float64(m1);
1905 env->vfp.regs[rd] = make_float64(d0);
1906 env->vfp.regs[rd + 1] = make_float64(d1);
1907}
1908
1909void HELPER(neon_unzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1910{
1911 uint64_t zm = float64_val(env->vfp.regs[rm]);
1912 uint64_t zd = float64_val(env->vfp.regs[rd]);
1913 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1914 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1915 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1916 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1917 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1918 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1919 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1920 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1921 env->vfp.regs[rm] = make_float64(m0);
1922 env->vfp.regs[rd] = make_float64(d0);
1923}
1924
1925void HELPER(neon_unzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1926{
1927 uint64_t zm = float64_val(env->vfp.regs[rm]);
1928 uint64_t zd = float64_val(env->vfp.regs[rd]);
1929 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1930 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1931 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1932 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1933 env->vfp.regs[rm] = make_float64(m0);
1934 env->vfp.regs[rd] = make_float64(d0);
1935}
d68a6f3a
PM		 1936
1937void HELPER(neon_qzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1938{
1939 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1940 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1941 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1942 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1943 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1944 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1945 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1946 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1947 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1948 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1949 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1950 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1951 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1952 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1953 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1954 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1955 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1956 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1957 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1958 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1959 env->vfp.regs[rm] = make_float64(m0);
1960 env->vfp.regs[rm + 1] = make_float64(m1);
1961 env->vfp.regs[rd] = make_float64(d0);
1962 env->vfp.regs[rd + 1] = make_float64(d1);
1963}
1964
1965void HELPER(neon_qzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1966{
1967 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1968 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1969 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1970 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1971 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1972 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1973 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1974 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1975 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1976 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1977 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1978 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1979 env->vfp.regs[rm] = make_float64(m0);
1980 env->vfp.regs[rm + 1] = make_float64(m1);
1981 env->vfp.regs[rd] = make_float64(d0);
1982 env->vfp.regs[rd + 1] = make_float64(d1);
1983}
1984
1985void HELPER(neon_qzip32)(CPUState *env, uint32_t rd, uint32_t rm)
1986{
1987 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1988 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1989 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1990 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1991 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1992 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1993 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1994 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1995 env->vfp.regs[rm] = make_float64(m0);
1996 env->vfp.regs[rm + 1] = make_float64(m1);
1997 env->vfp.regs[rd] = make_float64(d0);
1998 env->vfp.regs[rd + 1] = make_float64(d1);
1999}
2000
2001void HELPER(neon_zip8)(CPUState *env, uint32_t rd, uint32_t rm)
2002{
2003 uint64_t zm = float64_val(env->vfp.regs[rm]);
2004 uint64_t zd = float64_val(env->vfp.regs[rd]);
2005 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2006 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2007 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2008 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2009 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2010 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2011 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2012 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2013 env->vfp.regs[rm] = make_float64(m0);
2014 env->vfp.regs[rd] = make_float64(d0);
2015}
2016
2017void HELPER(neon_zip16)(CPUState *env, uint32_t rd, uint32_t rm)
2018{
2019 uint64_t zm = float64_val(env->vfp.regs[rm]);
2020 uint64_t zd = float64_val(env->vfp.regs[rd]);
2021 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2022 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2023 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2024 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2025 env->vfp.regs[rm] = make_float64(m0);
2026 env->vfp.regs[rd] = make_float64(d0);
2027}
Qemu copy for testing