2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licenced under the GNU GPL v2.
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
21 static float_status neon_float_status
;
22 #define NFS &neon_float_status
24 #define NEON_TYPE1(name, type) \
29 #ifdef HOST_WORDS_BIGENDIAN
30 #define NEON_TYPE2(name, type) \
36 #define NEON_TYPE4(name, type) \
45 #define NEON_TYPE2(name, type) \
51 #define NEON_TYPE4(name, type) \
61 NEON_TYPE4(s8
, int8_t)
62 NEON_TYPE4(u8
, uint8_t)
63 NEON_TYPE2(s16
, int16_t)
64 NEON_TYPE2(u16
, uint16_t)
65 NEON_TYPE1(s32
, int32_t)
66 NEON_TYPE1(u32
, uint32_t)
71 /* Copy from a uint32_t to a vector structure type. */
72 #define NEON_UNPACK(vtype, dest, val) do { \
81 /* Copy from a vector structure type to a uint32_t. */
82 #define NEON_PACK(vtype, dest, val) do { \
92 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
97 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
98 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
99 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
100 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
102 #define NEON_VOP_BODY(vtype, n) \
108 NEON_UNPACK(vtype, vsrc1, arg1); \
109 NEON_UNPACK(vtype, vsrc2, arg2); \
111 NEON_PACK(vtype, res, vdest); \
115 #define NEON_VOP(name, vtype, n) \
116 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
117 NEON_VOP_BODY(vtype, n)
119 /* Pairwise operations. */
120 /* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
131 #define NEON_POP(name, vtype, n) \
132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
141 NEON_PACK(vtype, res, vdest); \
145 /* Unary operators. */
146 #define NEON_VOP1(name, vtype, n) \
147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
151 NEON_UNPACK(vtype, vsrc1, arg); \
153 NEON_PACK(vtype, arg, vdest); \
158 #define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
167 NEON_VOP(qadd_u8
, neon_u8
, 4)
169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
170 NEON_VOP(qadd_u16
, neon_u16
, 2)
174 uint32_t HELPER(neon_qadd_u32
)(uint32_t a
, uint32_t b
)
176 uint32_t res
= a
+ b
;
184 uint64_t HELPER(neon_qadd_u64
)(uint64_t src1
, uint64_t src2
)
196 #define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
209 NEON_VOP(qadd_s8
, neon_s8
, 4)
211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
212 NEON_VOP(qadd_s16
, neon_s16
, 2)
216 uint32_t HELPER(neon_qadd_s32
)(uint32_t a
, uint32_t b
)
218 uint32_t res
= a
+ b
;
219 if (((res
^ a
) & SIGNBIT
) && !((a
^ b
) & SIGNBIT
)) {
221 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
226 uint64_t HELPER(neon_qadd_s64
)(uint64_t src1
, uint64_t src2
)
231 if (((res
^ src1
) & SIGNBIT64
) && !((src1
^ src2
) & SIGNBIT64
)) {
233 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
238 #define NEON_USAT(dest, src1, src2, type) do { \
239 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
240 if (tmp != (type)tmp) { \
246 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
247 NEON_VOP(qsub_u8
, neon_u8
, 4)
249 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
250 NEON_VOP(qsub_u16
, neon_u16
, 2)
254 uint32_t HELPER(neon_qsub_u32
)(uint32_t a
, uint32_t b
)
256 uint32_t res
= a
- b
;
264 uint64_t HELPER(neon_qsub_u64
)(uint64_t src1
, uint64_t src2
)
277 #define NEON_SSAT(dest, src1, src2, type) do { \
278 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
279 if (tmp != (type)tmp) { \
282 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
284 tmp = 1 << (sizeof(type) * 8 - 1); \
289 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
290 NEON_VOP(qsub_s8
, neon_s8
, 4)
292 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
293 NEON_VOP(qsub_s16
, neon_s16
, 2)
297 uint32_t HELPER(neon_qsub_s32
)(uint32_t a
, uint32_t b
)
299 uint32_t res
= a
- b
;
300 if (((res
^ a
) & SIGNBIT
) && ((a
^ b
) & SIGNBIT
)) {
302 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
307 uint64_t HELPER(neon_qsub_s64
)(uint64_t src1
, uint64_t src2
)
312 if (((res
^ src1
) & SIGNBIT64
) && ((src1
^ src2
) & SIGNBIT64
)) {
314 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
319 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
320 NEON_VOP(hadd_s8
, neon_s8
, 4)
321 NEON_VOP(hadd_u8
, neon_u8
, 4)
322 NEON_VOP(hadd_s16
, neon_s16
, 2)
323 NEON_VOP(hadd_u16
, neon_u16
, 2)
326 int32_t HELPER(neon_hadd_s32
)(int32_t src1
, int32_t src2
)
330 dest
= (src1
>> 1) + (src2
>> 1);
336 uint32_t HELPER(neon_hadd_u32
)(uint32_t src1
, uint32_t src2
)
340 dest
= (src1
>> 1) + (src2
>> 1);
346 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
347 NEON_VOP(rhadd_s8
, neon_s8
, 4)
348 NEON_VOP(rhadd_u8
, neon_u8
, 4)
349 NEON_VOP(rhadd_s16
, neon_s16
, 2)
350 NEON_VOP(rhadd_u16
, neon_u16
, 2)
353 int32_t HELPER(neon_rhadd_s32
)(int32_t src1
, int32_t src2
)
357 dest
= (src1
>> 1) + (src2
>> 1);
358 if ((src1
| src2
) & 1)
363 uint32_t HELPER(neon_rhadd_u32
)(uint32_t src1
, uint32_t src2
)
367 dest
= (src1
>> 1) + (src2
>> 1);
368 if ((src1
| src2
) & 1)
373 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
374 NEON_VOP(hsub_s8
, neon_s8
, 4)
375 NEON_VOP(hsub_u8
, neon_u8
, 4)
376 NEON_VOP(hsub_s16
, neon_s16
, 2)
377 NEON_VOP(hsub_u16
, neon_u16
, 2)
380 int32_t HELPER(neon_hsub_s32
)(int32_t src1
, int32_t src2
)
384 dest
= (src1
>> 1) - (src2
>> 1);
385 if ((~src1
) & src2
& 1)
390 uint32_t HELPER(neon_hsub_u32
)(uint32_t src1
, uint32_t src2
)
394 dest
= (src1
>> 1) - (src2
>> 1);
395 if ((~src1
) & src2
& 1)
400 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
401 NEON_VOP(cgt_s8
, neon_s8
, 4)
402 NEON_VOP(cgt_u8
, neon_u8
, 4)
403 NEON_VOP(cgt_s16
, neon_s16
, 2)
404 NEON_VOP(cgt_u16
, neon_u16
, 2)
405 NEON_VOP(cgt_s32
, neon_s32
, 1)
406 NEON_VOP(cgt_u32
, neon_u32
, 1)
409 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
410 NEON_VOP(cge_s8
, neon_s8
, 4)
411 NEON_VOP(cge_u8
, neon_u8
, 4)
412 NEON_VOP(cge_s16
, neon_s16
, 2)
413 NEON_VOP(cge_u16
, neon_u16
, 2)
414 NEON_VOP(cge_s32
, neon_s32
, 1)
415 NEON_VOP(cge_u32
, neon_u32
, 1)
418 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
419 NEON_VOP(min_s8
, neon_s8
, 4)
420 NEON_VOP(min_u8
, neon_u8
, 4)
421 NEON_VOP(min_s16
, neon_s16
, 2)
422 NEON_VOP(min_u16
, neon_u16
, 2)
423 NEON_VOP(min_s32
, neon_s32
, 1)
424 NEON_VOP(min_u32
, neon_u32
, 1)
425 NEON_POP(pmin_s8
, neon_s8
, 4)
426 NEON_POP(pmin_u8
, neon_u8
, 4)
427 NEON_POP(pmin_s16
, neon_s16
, 2)
428 NEON_POP(pmin_u16
, neon_u16
, 2)
431 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
432 NEON_VOP(max_s8
, neon_s8
, 4)
433 NEON_VOP(max_u8
, neon_u8
, 4)
434 NEON_VOP(max_s16
, neon_s16
, 2)
435 NEON_VOP(max_u16
, neon_u16
, 2)
436 NEON_VOP(max_s32
, neon_s32
, 1)
437 NEON_VOP(max_u32
, neon_u32
, 1)
438 NEON_POP(pmax_s8
, neon_s8
, 4)
439 NEON_POP(pmax_u8
, neon_u8
, 4)
440 NEON_POP(pmax_s16
, neon_s16
, 2)
441 NEON_POP(pmax_u16
, neon_u16
, 2)
444 #define NEON_FN(dest, src1, src2) \
445 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
446 NEON_VOP(abd_s8
, neon_s8
, 4)
447 NEON_VOP(abd_u8
, neon_u8
, 4)
448 NEON_VOP(abd_s16
, neon_s16
, 2)
449 NEON_VOP(abd_u16
, neon_u16
, 2)
450 NEON_VOP(abd_s32
, neon_s32
, 1)
451 NEON_VOP(abd_u32
, neon_u32
, 1)
454 #define NEON_FN(dest, src1, src2) do { \
456 tmp = (int8_t)src2; \
457 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
458 tmp <= -(ssize_t)sizeof(src1) * 8) { \
460 } else if (tmp < 0) { \
461 dest = src1 >> -tmp; \
463 dest = src1 << tmp; \
465 NEON_VOP(shl_u8
, neon_u8
, 4)
466 NEON_VOP(shl_u16
, neon_u16
, 2)
467 NEON_VOP(shl_u32
, neon_u32
, 1)
470 uint64_t HELPER(neon_shl_u64
)(uint64_t val
, uint64_t shiftop
)
472 int8_t shift
= (int8_t)shiftop
;
473 if (shift
>= 64 || shift
<= -64) {
475 } else if (shift
< 0) {
483 #define NEON_FN(dest, src1, src2) do { \
485 tmp = (int8_t)src2; \
486 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
488 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
489 dest = src1 >> (sizeof(src1) * 8 - 1); \
490 } else if (tmp < 0) { \
491 dest = src1 >> -tmp; \
493 dest = src1 << tmp; \
495 NEON_VOP(shl_s8
, neon_s8
, 4)
496 NEON_VOP(shl_s16
, neon_s16
, 2)
497 NEON_VOP(shl_s32
, neon_s32
, 1)
500 uint64_t HELPER(neon_shl_s64
)(uint64_t valop
, uint64_t shiftop
)
502 int8_t shift
= (int8_t)shiftop
;
506 } else if (shift
<= -64) {
508 } else if (shift
< 0) {
516 #define NEON_FN(dest, src1, src2) do { \
518 tmp = (int8_t)src2; \
519 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
520 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
522 } else if (tmp < 0) { \
523 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
525 dest = src1 << tmp; \
527 NEON_VOP(rshl_s8
, neon_s8
, 4)
528 NEON_VOP(rshl_s16
, neon_s16
, 2)
531 /* The addition of the rounding constant may overflow, so we use an
532 * intermediate 64 bits accumulator. */
533 uint32_t HELPER(neon_rshl_s32
)(uint32_t valop
, uint32_t shiftop
)
536 int32_t val
= (int32_t)valop
;
537 int8_t shift
= (int8_t)shiftop
;
538 if ((shift
>= 32) || (shift
<= -32)) {
540 } else if (shift
< 0) {
541 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
542 dest
= big_dest
>> -shift
;
549 /* Handling addition overflow with 64 bits inputs values is more
550 * tricky than with 32 bits values. */
551 uint64_t HELPER(neon_rshl_s64
)(uint64_t valop
, uint64_t shiftop
)
553 int8_t shift
= (int8_t)shiftop
;
555 if ((shift
>= 64) || (shift
<= -64)) {
557 } else if (shift
< 0) {
558 val
>>= (-shift
- 1);
559 if (val
== INT64_MAX
) {
560 /* In this case, it means that the rounding constant is 1,
561 * and the addition would overflow. Return the actual
562 * result directly. */
563 val
= 0x4000000000000000LL
;
574 #define NEON_FN(dest, src1, src2) do { \
576 tmp = (int8_t)src2; \
577 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
578 tmp < -(ssize_t)sizeof(src1) * 8) { \
580 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
581 dest = src1 >> (-tmp - 1); \
582 } else if (tmp < 0) { \
583 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
585 dest = src1 << tmp; \
587 NEON_VOP(rshl_u8
, neon_u8
, 4)
588 NEON_VOP(rshl_u16
, neon_u16
, 2)
591 /* The addition of the rounding constant may overflow, so we use an
592 * intermediate 64 bits accumulator. */
593 uint32_t HELPER(neon_rshl_u32
)(uint32_t val
, uint32_t shiftop
)
596 int8_t shift
= (int8_t)shiftop
;
597 if (shift
>= 32 || shift
< -32) {
599 } else if (shift
== -32) {
601 } else if (shift
< 0) {
602 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
603 dest
= big_dest
>> -shift
;
610 /* Handling addition overflow with 64 bits inputs values is more
611 * tricky than with 32 bits values. */
612 uint64_t HELPER(neon_rshl_u64
)(uint64_t val
, uint64_t shiftop
)
614 int8_t shift
= (uint8_t)shiftop
;
615 if (shift
>= 64 || shift
< -64) {
617 } else if (shift
== -64) {
618 /* Rounding a 1-bit result just preserves that bit. */
620 } else if (shift
< 0) {
621 val
>>= (-shift
- 1);
622 if (val
== UINT64_MAX
) {
623 /* In this case, it means that the rounding constant is 1,
624 * and the addition would overflow. Return the actual
625 * result directly. */
626 val
= 0x8000000000000000ULL
;
637 #define NEON_FN(dest, src1, src2) do { \
639 tmp = (int8_t)src2; \
640 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
647 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
649 } else if (tmp < 0) { \
650 dest = src1 >> -tmp; \
652 dest = src1 << tmp; \
653 if ((dest >> tmp) != src1) { \
658 NEON_VOP(qshl_u8
, neon_u8
, 4)
659 NEON_VOP(qshl_u16
, neon_u16
, 2)
660 NEON_VOP(qshl_u32
, neon_u32
, 1)
663 uint64_t HELPER(neon_qshl_u64
)(uint64_t val
, uint64_t shiftop
)
665 int8_t shift
= (int8_t)shiftop
;
671 } else if (shift
<= -64) {
673 } else if (shift
< 0) {
678 if ((val
>> shift
) != tmp
) {
686 #define NEON_FN(dest, src1, src2) do { \
688 tmp = (int8_t)src2; \
689 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
692 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
699 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
701 } else if (tmp < 0) { \
702 dest = src1 >> -tmp; \
704 dest = src1 << tmp; \
705 if ((dest >> tmp) != src1) { \
707 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
713 NEON_VOP(qshl_s8
, neon_s8
, 4)
714 NEON_VOP(qshl_s16
, neon_s16
, 2)
715 NEON_VOP(qshl_s32
, neon_s32
, 1)
718 uint64_t HELPER(neon_qshl_s64
)(uint64_t valop
, uint64_t shiftop
)
720 int8_t shift
= (uint8_t)shiftop
;
725 val
= (val
>> 63) ^ ~SIGNBIT64
;
727 } else if (shift
<= -64) {
729 } else if (shift
< 0) {
734 if ((val
>> shift
) != tmp
) {
736 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
742 #define NEON_FN(dest, src1, src2) do { \
743 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
748 tmp = (int8_t)src2; \
749 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
756 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
758 } else if (tmp < 0) { \
759 dest = src1 >> -tmp; \
761 dest = src1 << tmp; \
762 if ((dest >> tmp) != src1) { \
768 NEON_VOP(qshlu_s8
, neon_u8
, 4)
769 NEON_VOP(qshlu_s16
, neon_u16
, 2)
772 uint32_t HELPER(neon_qshlu_s32
)(uint32_t valop
, uint32_t shiftop
)
774 if ((int32_t)valop
< 0) {
778 return helper_neon_qshl_u32(valop
, shiftop
);
781 uint64_t HELPER(neon_qshlu_s64
)(uint64_t valop
, uint64_t shiftop
)
783 if ((int64_t)valop
< 0) {
787 return helper_neon_qshl_u64(valop
, shiftop
);
790 /* FIXME: This is wrong. */
791 #define NEON_FN(dest, src1, src2) do { \
793 tmp = (int8_t)src2; \
794 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
801 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
803 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
804 dest = src1 >> (sizeof(src1) * 8 - 1); \
805 } else if (tmp < 0) { \
806 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
808 dest = src1 << tmp; \
809 if ((dest >> tmp) != src1) { \
814 NEON_VOP(qrshl_u8
, neon_u8
, 4)
815 NEON_VOP(qrshl_u16
, neon_u16
, 2)
818 /* The addition of the rounding constant may overflow, so we use an
819 * intermediate 64 bits accumulator. */
820 uint32_t HELPER(neon_qrshl_u32
)(uint32_t val
, uint32_t shiftop
)
823 int8_t shift
= (int8_t)shiftop
;
831 } else if (shift
< -32) {
833 } else if (shift
== -32) {
835 } else if (shift
< 0) {
836 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
837 dest
= big_dest
>> -shift
;
840 if ((dest
>> shift
) != val
) {
848 /* Handling addition overflow with 64 bits inputs values is more
849 * tricky than with 32 bits values. */
850 uint64_t HELPER(neon_qrshl_u64
)(uint64_t val
, uint64_t shiftop
)
852 int8_t shift
= (int8_t)shiftop
;
858 } else if (shift
< -64) {
860 } else if (shift
== -64) {
862 } else if (shift
< 0) {
863 val
>>= (-shift
- 1);
864 if (val
== UINT64_MAX
) {
865 /* In this case, it means that the rounding constant is 1,
866 * and the addition would overflow. Return the actual
867 * result directly. */
868 val
= 0x8000000000000000ULL
;
876 if ((val
>> shift
) != tmp
) {
884 #define NEON_FN(dest, src1, src2) do { \
886 tmp = (int8_t)src2; \
887 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
890 dest = (1 << (sizeof(src1) * 8 - 1)); \
897 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
899 } else if (tmp < 0) { \
900 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
902 dest = src1 << tmp; \
903 if ((dest >> tmp) != src1) { \
905 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
911 NEON_VOP(qrshl_s8
, neon_s8
, 4)
912 NEON_VOP(qrshl_s16
, neon_s16
, 2)
915 /* The addition of the rounding constant may overflow, so we use an
916 * intermediate 64 bits accumulator. */
917 uint32_t HELPER(neon_qrshl_s32
)(uint32_t valop
, uint32_t shiftop
)
920 int32_t val
= (int32_t)valop
;
921 int8_t shift
= (int8_t)shiftop
;
925 dest
= (val
>> 31) ^ ~SIGNBIT
;
929 } else if (shift
<= -32) {
931 } else if (shift
< 0) {
932 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
933 dest
= big_dest
>> -shift
;
936 if ((dest
>> shift
) != val
) {
938 dest
= (val
>> 31) ^ ~SIGNBIT
;
944 /* Handling addition overflow with 64 bits inputs values is more
945 * tricky than with 32 bits values. */
946 uint64_t HELPER(neon_qrshl_s64
)(uint64_t valop
, uint64_t shiftop
)
948 int8_t shift
= (uint8_t)shiftop
;
954 val
= (val
>> 63) ^ ~SIGNBIT64
;
956 } else if (shift
<= -64) {
958 } else if (shift
< 0) {
959 val
>>= (-shift
- 1);
960 if (val
== INT64_MAX
) {
961 /* In this case, it means that the rounding constant is 1,
962 * and the addition would overflow. Return the actual
963 * result directly. */
964 val
= 0x4000000000000000ULL
;
972 if ((val
>> shift
) != tmp
) {
974 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
980 uint32_t HELPER(neon_add_u8
)(uint32_t a
, uint32_t b
)
983 mask
= (a
^ b
) & 0x80808080u
;
986 return (a
+ b
) ^ mask
;
989 uint32_t HELPER(neon_add_u16
)(uint32_t a
, uint32_t b
)
992 mask
= (a
^ b
) & 0x80008000u
;
995 return (a
+ b
) ^ mask
;
998 #define NEON_FN(dest, src1, src2) dest = src1 + src2
999 NEON_POP(padd_u8
, neon_u8
, 4)
1000 NEON_POP(padd_u16
, neon_u16
, 2)
1003 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1004 NEON_VOP(sub_u8
, neon_u8
, 4)
1005 NEON_VOP(sub_u16
, neon_u16
, 2)
1008 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1009 NEON_VOP(mul_u8
, neon_u8
, 4)
1010 NEON_VOP(mul_u16
, neon_u16
, 2)
1013 /* Polynomial multiplication is like integer multiplication except the
1014 partial products are XORed, not added. */
1015 uint32_t HELPER(neon_mul_p8
)(uint32_t op1
, uint32_t op2
)
1025 mask
|= (0xff << 8);
1026 if (op1
& (1 << 16))
1027 mask
|= (0xff << 16);
1028 if (op1
& (1 << 24))
1029 mask
|= (0xff << 24);
1030 result
^= op2
& mask
;
1031 op1
= (op1
>> 1) & 0x7f7f7f7f;
1032 op2
= (op2
<< 1) & 0xfefefefe;
1037 uint64_t HELPER(neon_mull_p8
)(uint32_t op1
, uint32_t op2
)
1039 uint64_t result
= 0;
1041 uint64_t op2ex
= op2
;
1042 op2ex
= (op2ex
& 0xff) |
1043 ((op2ex
& 0xff00) << 8) |
1044 ((op2ex
& 0xff0000) << 16) |
1045 ((op2ex
& 0xff000000) << 24);
1051 if (op1
& (1 << 8)) {
1052 mask
|= (0xffffU
<< 16);
1054 if (op1
& (1 << 16)) {
1055 mask
|= (0xffffULL
<< 32);
1057 if (op1
& (1 << 24)) {
1058 mask
|= (0xffffULL
<< 48);
1060 result
^= op2ex
& mask
;
1061 op1
= (op1
>> 1) & 0x7f7f7f7f;
1067 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1068 NEON_VOP(tst_u8
, neon_u8
, 4)
1069 NEON_VOP(tst_u16
, neon_u16
, 2)
1070 NEON_VOP(tst_u32
, neon_u32
, 1)
1073 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1074 NEON_VOP(ceq_u8
, neon_u8
, 4)
1075 NEON_VOP(ceq_u16
, neon_u16
, 2)
1076 NEON_VOP(ceq_u32
, neon_u32
, 1)
1079 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1080 NEON_VOP1(abs_s8
, neon_s8
, 4)
1081 NEON_VOP1(abs_s16
, neon_s16
, 2)
1084 /* Count Leading Sign/Zero Bits. */
1085 static inline int do_clz8(uint8_t x
)
1093 static inline int do_clz16(uint16_t x
)
1096 for (n
= 16; x
; n
--)
1101 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1102 NEON_VOP1(clz_u8
, neon_u8
, 4)
1105 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1106 NEON_VOP1(clz_u16
, neon_u16
, 2)
1109 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1110 NEON_VOP1(cls_s8
, neon_s8
, 4)
1113 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1114 NEON_VOP1(cls_s16
, neon_s16
, 2)
1117 uint32_t HELPER(neon_cls_s32
)(uint32_t x
)
1122 for (count
= 32; x
; count
--)
1128 uint32_t HELPER(neon_cnt_u8
)(uint32_t x
)
1130 x
= (x
& 0x55555555) + ((x
>> 1) & 0x55555555);
1131 x
= (x
& 0x33333333) + ((x
>> 2) & 0x33333333);
1132 x
= (x
& 0x0f0f0f0f) + ((x
>> 4) & 0x0f0f0f0f);
1136 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1137 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1138 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1140 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1145 int32_t old = tmp; \
1147 if ((int32_t)tmp < old) { \
1149 tmp = SIGNBIT - 1; \
1154 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1155 NEON_VOP(qdmulh_s16
, neon_s16
, 2)
1157 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1158 NEON_VOP(qrdmulh_s16
, neon_s16
, 2)
1160 #undef NEON_QDMULH16
1162 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1163 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1164 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1166 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1171 int64_t old = tmp; \
1172 tmp += (int64_t)1 << 31; \
1173 if ((int64_t)tmp < old) { \
1175 tmp = SIGNBIT64 - 1; \
1180 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1181 NEON_VOP(qdmulh_s32
, neon_s32
, 1)
1183 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1184 NEON_VOP(qrdmulh_s32
, neon_s32
, 1)
1186 #undef NEON_QDMULH32
1188 uint32_t HELPER(neon_narrow_u8
)(uint64_t x
)
1190 return (x
& 0xffu
) | ((x
>> 8) & 0xff00u
) | ((x
>> 16) & 0xff0000u
)
1191 | ((x
>> 24) & 0xff000000u
);
1194 uint32_t HELPER(neon_narrow_u16
)(uint64_t x
)
1196 return (x
& 0xffffu
) | ((x
>> 16) & 0xffff0000u
);
1199 uint32_t HELPER(neon_narrow_high_u8
)(uint64_t x
)
1201 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1202 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1205 uint32_t HELPER(neon_narrow_high_u16
)(uint64_t x
)
1207 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1210 uint32_t HELPER(neon_narrow_round_high_u8
)(uint64_t x
)
1212 x
&= 0xff80ff80ff80ff80ull
;
1213 x
+= 0x0080008000800080ull
;
1214 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1215 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1218 uint32_t HELPER(neon_narrow_round_high_u16
)(uint64_t x
)
1220 x
&= 0xffff8000ffff8000ull
;
1221 x
+= 0x0000800000008000ull
;
1222 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1225 uint32_t HELPER(neon_unarrow_sat8
)(uint64_t x
)
1241 res |= (uint32_t)d << (n / 2); \
1252 uint32_t HELPER(neon_narrow_sat_u8
)(uint64_t x
)
1265 res |= (uint32_t)d << (n / 2);
1275 uint32_t HELPER(neon_narrow_sat_s8
)(uint64_t x
)
1282 if (s != (int8_t)s) { \
1283 d = (s >> 15) ^ 0x7f; \
1288 res |= (uint32_t)d << (n / 2);
1298 uint32_t HELPER(neon_unarrow_sat16
)(uint64_t x
)
1303 if (low
& 0x80000000) {
1306 } else if (low
> 0xffff) {
1311 if (high
& 0x80000000) {
1314 } else if (high
> 0xffff) {
1318 return low
| (high
<< 16);
1321 uint32_t HELPER(neon_narrow_sat_u16
)(uint64_t x
)
1331 if (high
> 0xffff) {
1335 return low
| (high
<< 16);
1338 uint32_t HELPER(neon_narrow_sat_s16
)(uint64_t x
)
1343 if (low
!= (int16_t)low
) {
1344 low
= (low
>> 31) ^ 0x7fff;
1348 if (high
!= (int16_t)high
) {
1349 high
= (high
>> 31) ^ 0x7fff;
1352 return (uint16_t)low
| (high
<< 16);
1355 uint32_t HELPER(neon_unarrow_sat32
)(uint64_t x
)
1357 if (x
& 0x8000000000000000ull
) {
1361 if (x
> 0xffffffffu
) {
1368 uint32_t HELPER(neon_narrow_sat_u32
)(uint64_t x
)
1370 if (x
> 0xffffffffu
) {
1377 uint32_t HELPER(neon_narrow_sat_s32
)(uint64_t x
)
1379 if ((int64_t)x
!= (int32_t)x
) {
1381 return ((int64_t)x
>> 63) ^ 0x7fffffff;
1386 uint64_t HELPER(neon_widen_u8
)(uint32_t x
)
1391 tmp
= (uint8_t)(x
>> 8);
1393 tmp
= (uint8_t)(x
>> 16);
1395 tmp
= (uint8_t)(x
>> 24);
1400 uint64_t HELPER(neon_widen_s8
)(uint32_t x
)
1404 ret
= (uint16_t)(int8_t)x
;
1405 tmp
= (uint16_t)(int8_t)(x
>> 8);
1407 tmp
= (uint16_t)(int8_t)(x
>> 16);
1409 tmp
= (uint16_t)(int8_t)(x
>> 24);
1414 uint64_t HELPER(neon_widen_u16
)(uint32_t x
)
1416 uint64_t high
= (uint16_t)(x
>> 16);
1417 return ((uint16_t)x
) | (high
<< 32);
1420 uint64_t HELPER(neon_widen_s16
)(uint32_t x
)
1422 uint64_t high
= (int16_t)(x
>> 16);
1423 return ((uint32_t)(int16_t)x
) | (high
<< 32);
1426 uint64_t HELPER(neon_addl_u16
)(uint64_t a
, uint64_t b
)
1429 mask
= (a
^ b
) & 0x8000800080008000ull
;
1430 a
&= ~0x8000800080008000ull
;
1431 b
&= ~0x8000800080008000ull
;
1432 return (a
+ b
) ^ mask
;
1435 uint64_t HELPER(neon_addl_u32
)(uint64_t a
, uint64_t b
)
1438 mask
= (a
^ b
) & 0x8000000080000000ull
;
1439 a
&= ~0x8000000080000000ull
;
1440 b
&= ~0x8000000080000000ull
;
1441 return (a
+ b
) ^ mask
;
1444 uint64_t HELPER(neon_paddl_u16
)(uint64_t a
, uint64_t b
)
1449 tmp
= a
& 0x0000ffff0000ffffull
;
1450 tmp
+= (a
>> 16) & 0x0000ffff0000ffffull
;
1451 tmp2
= b
& 0xffff0000ffff0000ull
;
1452 tmp2
+= (b
<< 16) & 0xffff0000ffff0000ull
;
1453 return ( tmp
& 0xffff)
1454 | ((tmp
>> 16) & 0xffff0000ull
)
1455 | ((tmp2
<< 16) & 0xffff00000000ull
)
1456 | ( tmp2
& 0xffff000000000000ull
);
1459 uint64_t HELPER(neon_paddl_u32
)(uint64_t a
, uint64_t b
)
1461 uint32_t low
= a
+ (a
>> 32);
1462 uint32_t high
= b
+ (b
>> 32);
1463 return low
+ ((uint64_t)high
<< 32);
1466 uint64_t HELPER(neon_subl_u16
)(uint64_t a
, uint64_t b
)
1469 mask
= (a
^ ~b
) & 0x8000800080008000ull
;
1470 a
|= 0x8000800080008000ull
;
1471 b
&= ~0x8000800080008000ull
;
1472 return (a
- b
) ^ mask
;
1475 uint64_t HELPER(neon_subl_u32
)(uint64_t a
, uint64_t b
)
1478 mask
= (a
^ ~b
) & 0x8000000080000000ull
;
1479 a
|= 0x8000000080000000ull
;
1480 b
&= ~0x8000000080000000ull
;
1481 return (a
- b
) ^ mask
;
1484 uint64_t HELPER(neon_addl_saturate_s32
)(uint64_t a
, uint64_t b
)
1492 if (((low
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1494 low
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1499 if (((high
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1501 high
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1503 return low
| ((uint64_t)high
<< 32);
1506 uint64_t HELPER(neon_addl_saturate_s64
)(uint64_t a
, uint64_t b
)
1511 if (((result
^ a
) & SIGNBIT64
) && !((a
^ b
) & SIGNBIT64
)) {
1513 result
= ((int64_t)a
>> 63) ^ ~SIGNBIT64
;
1518 #define DO_ABD(dest, x, y, type) do { \
1521 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1524 uint64_t HELPER(neon_abdl_u16
)(uint32_t a
, uint32_t b
)
1528 DO_ABD(result
, a
, b
, uint8_t);
1529 DO_ABD(tmp
, a
>> 8, b
>> 8, uint8_t);
1530 result
|= tmp
<< 16;
1531 DO_ABD(tmp
, a
>> 16, b
>> 16, uint8_t);
1532 result
|= tmp
<< 32;
1533 DO_ABD(tmp
, a
>> 24, b
>> 24, uint8_t);
1534 result
|= tmp
<< 48;
1538 uint64_t HELPER(neon_abdl_s16
)(uint32_t a
, uint32_t b
)
1542 DO_ABD(result
, a
, b
, int8_t);
1543 DO_ABD(tmp
, a
>> 8, b
>> 8, int8_t);
1544 result
|= tmp
<< 16;
1545 DO_ABD(tmp
, a
>> 16, b
>> 16, int8_t);
1546 result
|= tmp
<< 32;
1547 DO_ABD(tmp
, a
>> 24, b
>> 24, int8_t);
1548 result
|= tmp
<< 48;
1552 uint64_t HELPER(neon_abdl_u32
)(uint32_t a
, uint32_t b
)
1556 DO_ABD(result
, a
, b
, uint16_t);
1557 DO_ABD(tmp
, a
>> 16, b
>> 16, uint16_t);
1558 return result
| (tmp
<< 32);
1561 uint64_t HELPER(neon_abdl_s32
)(uint32_t a
, uint32_t b
)
1565 DO_ABD(result
, a
, b
, int16_t);
1566 DO_ABD(tmp
, a
>> 16, b
>> 16, int16_t);
1567 return result
| (tmp
<< 32);
1570 uint64_t HELPER(neon_abdl_u64
)(uint32_t a
, uint32_t b
)
1573 DO_ABD(result
, a
, b
, uint32_t);
1577 uint64_t HELPER(neon_abdl_s64
)(uint32_t a
, uint32_t b
)
1580 DO_ABD(result
, a
, b
, int32_t);
1585 /* Widening multiply. Named type is the source type. */
1586 #define DO_MULL(dest, x, y, type1, type2) do { \
1589 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1592 uint64_t HELPER(neon_mull_u8
)(uint32_t a
, uint32_t b
)
1597 DO_MULL(result
, a
, b
, uint8_t, uint16_t);
1598 DO_MULL(tmp
, a
>> 8, b
>> 8, uint8_t, uint16_t);
1599 result
|= tmp
<< 16;
1600 DO_MULL(tmp
, a
>> 16, b
>> 16, uint8_t, uint16_t);
1601 result
|= tmp
<< 32;
1602 DO_MULL(tmp
, a
>> 24, b
>> 24, uint8_t, uint16_t);
1603 result
|= tmp
<< 48;
1607 uint64_t HELPER(neon_mull_s8
)(uint32_t a
, uint32_t b
)
1612 DO_MULL(result
, a
, b
, int8_t, uint16_t);
1613 DO_MULL(tmp
, a
>> 8, b
>> 8, int8_t, uint16_t);
1614 result
|= tmp
<< 16;
1615 DO_MULL(tmp
, a
>> 16, b
>> 16, int8_t, uint16_t);
1616 result
|= tmp
<< 32;
1617 DO_MULL(tmp
, a
>> 24, b
>> 24, int8_t, uint16_t);
1618 result
|= tmp
<< 48;
1622 uint64_t HELPER(neon_mull_u16
)(uint32_t a
, uint32_t b
)
1627 DO_MULL(result
, a
, b
, uint16_t, uint32_t);
1628 DO_MULL(tmp
, a
>> 16, b
>> 16, uint16_t, uint32_t);
1629 return result
| (tmp
<< 32);
1632 uint64_t HELPER(neon_mull_s16
)(uint32_t a
, uint32_t b
)
1637 DO_MULL(result
, a
, b
, int16_t, uint32_t);
1638 DO_MULL(tmp
, a
>> 16, b
>> 16, int16_t, uint32_t);
1639 return result
| (tmp
<< 32);
1642 uint64_t HELPER(neon_negl_u16
)(uint64_t x
)
1646 result
= (uint16_t)-x
;
1648 result
|= (uint64_t)tmp
<< 16;
1650 result
|= (uint64_t)tmp
<< 32;
1652 result
|= (uint64_t)tmp
<< 48;
1656 uint64_t HELPER(neon_negl_u32
)(uint64_t x
)
1659 uint32_t high
= -(x
>> 32);
1660 return low
| ((uint64_t)high
<< 32);
1663 /* FIXME: There should be a native op for this. */
1664 uint64_t HELPER(neon_negl_u64
)(uint64_t x
)
1669 /* Saturnating sign manuipulation. */
1670 /* ??? Make these use NEON_VOP1 */
1671 #define DO_QABS8(x) do { \
1672 if (x == (int8_t)0x80) { \
1675 } else if (x < 0) { \
1678 uint32_t HELPER(neon_qabs_s8
)(uint32_t x
)
1681 NEON_UNPACK(neon_s8
, vec
, x
);
1686 NEON_PACK(neon_s8
, x
, vec
);
1691 #define DO_QNEG8(x) do { \
1692 if (x == (int8_t)0x80) { \
1698 uint32_t HELPER(neon_qneg_s8
)(uint32_t x
)
1701 NEON_UNPACK(neon_s8
, vec
, x
);
1706 NEON_PACK(neon_s8
, x
, vec
);
1711 #define DO_QABS16(x) do { \
1712 if (x == (int16_t)0x8000) { \
1715 } else if (x < 0) { \
1718 uint32_t HELPER(neon_qabs_s16
)(uint32_t x
)
1721 NEON_UNPACK(neon_s16
, vec
, x
);
1724 NEON_PACK(neon_s16
, x
, vec
);
1729 #define DO_QNEG16(x) do { \
1730 if (x == (int16_t)0x8000) { \
1736 uint32_t HELPER(neon_qneg_s16
)(uint32_t x
)
1739 NEON_UNPACK(neon_s16
, vec
, x
);
1742 NEON_PACK(neon_s16
, x
, vec
);
1747 uint32_t HELPER(neon_qabs_s32
)(uint32_t x
)
1752 } else if ((int32_t)x
< 0) {
1758 uint32_t HELPER(neon_qneg_s32
)(uint32_t x
)
1769 /* NEON Float helpers. */
1770 uint32_t HELPER(neon_min_f32
)(uint32_t a
, uint32_t b
)
1772 return float32_val(float32_min(make_float32(a
), make_float32(b
), NFS
));
1775 uint32_t HELPER(neon_max_f32
)(uint32_t a
, uint32_t b
)
1777 return float32_val(float32_max(make_float32(a
), make_float32(b
), NFS
));
1780 uint32_t HELPER(neon_abd_f32
)(uint32_t a
, uint32_t b
)
1782 float32 f0
= make_float32(a
);
1783 float32 f1
= make_float32(b
);
1784 return float32_val(float32_abs(float32_sub(f0
, f1
, NFS
)));
1787 uint32_t HELPER(neon_add_f32
)(uint32_t a
, uint32_t b
)
1789 return float32_val(float32_add(make_float32(a
), make_float32(b
), NFS
));
1792 uint32_t HELPER(neon_sub_f32
)(uint32_t a
, uint32_t b
)
1794 return float32_val(float32_sub(make_float32(a
), make_float32(b
), NFS
));
1797 uint32_t HELPER(neon_mul_f32
)(uint32_t a
, uint32_t b
)
1799 return float32_val(float32_mul(make_float32(a
), make_float32(b
), NFS
));
1802 /* Floating point comparisons produce an integer result. */
1803 #define NEON_VOP_FCMP(name, ok) \
1804 uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1806 switch (float32_compare_quiet(make_float32(a), make_float32(b), NFS)) { \
1808 default: return 0; \
1812 NEON_VOP_FCMP(ceq_f32
, case float_relation_equal
:)
1813 NEON_VOP_FCMP(cge_f32
, case float_relation_equal
: case float_relation_greater
:)
1814 NEON_VOP_FCMP(cgt_f32
, case float_relation_greater
:)
1816 uint32_t HELPER(neon_acge_f32
)(uint32_t a
, uint32_t b
)
1818 float32 f0
= float32_abs(make_float32(a
));
1819 float32 f1
= float32_abs(make_float32(b
));
1820 switch (float32_compare_quiet(f0
, f1
, NFS
)) {
1821 case float_relation_equal
:
1822 case float_relation_greater
:
1829 uint32_t HELPER(neon_acgt_f32
)(uint32_t a
, uint32_t b
)
1831 float32 f0
= float32_abs(make_float32(a
));
1832 float32 f1
= float32_abs(make_float32(b
));
1833 if (float32_compare_quiet(f0
, f1
, NFS
) == float_relation_greater
) {
1839 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1841 void HELPER(neon_qunzip8
)(uint32_t rd
, uint32_t rm
)
1843 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1844 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1845 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1846 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1847 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zd0
, 2, 8) << 8)
1848 | (ELEM(zd0
, 4, 8) << 16) | (ELEM(zd0
, 6, 8) << 24)
1849 | (ELEM(zd1
, 0, 8) << 32) | (ELEM(zd1
, 2, 8) << 40)
1850 | (ELEM(zd1
, 4, 8) << 48) | (ELEM(zd1
, 6, 8) << 56);
1851 uint64_t d1
= ELEM(zm0
, 0, 8) | (ELEM(zm0
, 2, 8) << 8)
1852 | (ELEM(zm0
, 4, 8) << 16) | (ELEM(zm0
, 6, 8) << 24)
1853 | (ELEM(zm1
, 0, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
1854 | (ELEM(zm1
, 4, 8) << 48) | (ELEM(zm1
, 6, 8) << 56);
1855 uint64_t m0
= ELEM(zd0
, 1, 8) | (ELEM(zd0
, 3, 8) << 8)
1856 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zd0
, 7, 8) << 24)
1857 | (ELEM(zd1
, 1, 8) << 32) | (ELEM(zd1
, 3, 8) << 40)
1858 | (ELEM(zd1
, 5, 8) << 48) | (ELEM(zd1
, 7, 8) << 56);
1859 uint64_t m1
= ELEM(zm0
, 1, 8) | (ELEM(zm0
, 3, 8) << 8)
1860 | (ELEM(zm0
, 5, 8) << 16) | (ELEM(zm0
, 7, 8) << 24)
1861 | (ELEM(zm1
, 1, 8) << 32) | (ELEM(zm1
, 3, 8) << 40)
1862 | (ELEM(zm1
, 5, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
1863 env
->vfp
.regs
[rm
] = make_float64(m0
);
1864 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1865 env
->vfp
.regs
[rd
] = make_float64(d0
);
1866 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1869 void HELPER(neon_qunzip16
)(uint32_t rd
, uint32_t rm
)
1871 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1872 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1873 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1874 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1875 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zd0
, 2, 16) << 16)
1876 | (ELEM(zd1
, 0, 16) << 32) | (ELEM(zd1
, 2, 16) << 48);
1877 uint64_t d1
= ELEM(zm0
, 0, 16) | (ELEM(zm0
, 2, 16) << 16)
1878 | (ELEM(zm1
, 0, 16) << 32) | (ELEM(zm1
, 2, 16) << 48);
1879 uint64_t m0
= ELEM(zd0
, 1, 16) | (ELEM(zd0
, 3, 16) << 16)
1880 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zd1
, 3, 16) << 48);
1881 uint64_t m1
= ELEM(zm0
, 1, 16) | (ELEM(zm0
, 3, 16) << 16)
1882 | (ELEM(zm1
, 1, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
1883 env
->vfp
.regs
[rm
] = make_float64(m0
);
1884 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1885 env
->vfp
.regs
[rd
] = make_float64(d0
);
1886 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1889 void HELPER(neon_qunzip32
)(uint32_t rd
, uint32_t rm
)
1891 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1892 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1893 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1894 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1895 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zd1
, 0, 32) << 32);
1896 uint64_t d1
= ELEM(zm0
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
1897 uint64_t m0
= ELEM(zd0
, 1, 32) | (ELEM(zd1
, 1, 32) << 32);
1898 uint64_t m1
= ELEM(zm0
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
1899 env
->vfp
.regs
[rm
] = make_float64(m0
);
1900 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1901 env
->vfp
.regs
[rd
] = make_float64(d0
);
1902 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1905 void HELPER(neon_unzip8
)(uint32_t rd
, uint32_t rm
)
1907 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
1908 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
1909 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zd
, 2, 8) << 8)
1910 | (ELEM(zd
, 4, 8) << 16) | (ELEM(zd
, 6, 8) << 24)
1911 | (ELEM(zm
, 0, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
1912 | (ELEM(zm
, 4, 8) << 48) | (ELEM(zm
, 6, 8) << 56);
1913 uint64_t m0
= ELEM(zd
, 1, 8) | (ELEM(zd
, 3, 8) << 8)
1914 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zd
, 7, 8) << 24)
1915 | (ELEM(zm
, 1, 8) << 32) | (ELEM(zm
, 3, 8) << 40)
1916 | (ELEM(zm
, 5, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
1917 env
->vfp
.regs
[rm
] = make_float64(m0
);
1918 env
->vfp
.regs
[rd
] = make_float64(d0
);
1921 void HELPER(neon_unzip16
)(uint32_t rd
, uint32_t rm
)
1923 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
1924 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
1925 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zd
, 2, 16) << 16)
1926 | (ELEM(zm
, 0, 16) << 32) | (ELEM(zm
, 2, 16) << 48);
1927 uint64_t m0
= ELEM(zd
, 1, 16) | (ELEM(zd
, 3, 16) << 16)
1928 | (ELEM(zm
, 1, 16) << 32) | (ELEM(zm
, 3, 16) << 48);
1929 env
->vfp
.regs
[rm
] = make_float64(m0
);
1930 env
->vfp
.regs
[rd
] = make_float64(d0
);
1933 void HELPER(neon_qzip8
)(uint32_t rd
, uint32_t rm
)
1935 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1936 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1937 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1938 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1939 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zm0
, 0, 8) << 8)
1940 | (ELEM(zd0
, 1, 8) << 16) | (ELEM(zm0
, 1, 8) << 24)
1941 | (ELEM(zd0
, 2, 8) << 32) | (ELEM(zm0
, 2, 8) << 40)
1942 | (ELEM(zd0
, 3, 8) << 48) | (ELEM(zm0
, 3, 8) << 56);
1943 uint64_t d1
= ELEM(zd0
, 4, 8) | (ELEM(zm0
, 4, 8) << 8)
1944 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zm0
, 5, 8) << 24)
1945 | (ELEM(zd0
, 6, 8) << 32) | (ELEM(zm0
, 6, 8) << 40)
1946 | (ELEM(zd0
, 7, 8) << 48) | (ELEM(zm0
, 7, 8) << 56);
1947 uint64_t m0
= ELEM(zd1
, 0, 8) | (ELEM(zm1
, 0, 8) << 8)
1948 | (ELEM(zd1
, 1, 8) << 16) | (ELEM(zm1
, 1, 8) << 24)
1949 | (ELEM(zd1
, 2, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
1950 | (ELEM(zd1
, 3, 8) << 48) | (ELEM(zm1
, 3, 8) << 56);
1951 uint64_t m1
= ELEM(zd1
, 4, 8) | (ELEM(zm1
, 4, 8) << 8)
1952 | (ELEM(zd1
, 5, 8) << 16) | (ELEM(zm1
, 5, 8) << 24)
1953 | (ELEM(zd1
, 6, 8) << 32) | (ELEM(zm1
, 6, 8) << 40)
1954 | (ELEM(zd1
, 7, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
1955 env
->vfp
.regs
[rm
] = make_float64(m0
);
1956 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1957 env
->vfp
.regs
[rd
] = make_float64(d0
);
1958 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1961 void HELPER(neon_qzip16
)(uint32_t rd
, uint32_t rm
)
1963 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1964 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1965 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1966 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1967 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zm0
, 0, 16) << 16)
1968 | (ELEM(zd0
, 1, 16) << 32) | (ELEM(zm0
, 1, 16) << 48);
1969 uint64_t d1
= ELEM(zd0
, 2, 16) | (ELEM(zm0
, 2, 16) << 16)
1970 | (ELEM(zd0
, 3, 16) << 32) | (ELEM(zm0
, 3, 16) << 48);
1971 uint64_t m0
= ELEM(zd1
, 0, 16) | (ELEM(zm1
, 0, 16) << 16)
1972 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zm1
, 1, 16) << 48);
1973 uint64_t m1
= ELEM(zd1
, 2, 16) | (ELEM(zm1
, 2, 16) << 16)
1974 | (ELEM(zd1
, 3, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
1975 env
->vfp
.regs
[rm
] = make_float64(m0
);
1976 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1977 env
->vfp
.regs
[rd
] = make_float64(d0
);
1978 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1981 void HELPER(neon_qzip32
)(uint32_t rd
, uint32_t rm
)
1983 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1984 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1985 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1986 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1987 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zm0
, 0, 32) << 32);
1988 uint64_t d1
= ELEM(zd0
, 1, 32) | (ELEM(zm0
, 1, 32) << 32);
1989 uint64_t m0
= ELEM(zd1
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
1990 uint64_t m1
= ELEM(zd1
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
1991 env
->vfp
.regs
[rm
] = make_float64(m0
);
1992 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1993 env
->vfp
.regs
[rd
] = make_float64(d0
);
1994 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1997 void HELPER(neon_zip8
)(uint32_t rd
, uint32_t rm
)
1999 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
2000 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
2001 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zm
, 0, 8) << 8)
2002 | (ELEM(zd
, 1, 8) << 16) | (ELEM(zm
, 1, 8) << 24)
2003 | (ELEM(zd
, 2, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
2004 | (ELEM(zd
, 3, 8) << 48) | (ELEM(zm
, 3, 8) << 56);
2005 uint64_t m0
= ELEM(zd
, 4, 8) | (ELEM(zm
, 4, 8) << 8)
2006 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zm
, 5, 8) << 24)
2007 | (ELEM(zd
, 6, 8) << 32) | (ELEM(zm
, 6, 8) << 40)
2008 | (ELEM(zd
, 7, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
2009 env
->vfp
.regs
[rm
] = make_float64(m0
);
2010 env
->vfp
.regs
[rd
] = make_float64(d0
);
2013 void HELPER(neon_zip16
)(uint32_t rd
, uint32_t rm
)
2015 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
2016 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
2017 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zm
, 0, 16) << 16)
2018 | (ELEM(zd
, 1, 16) << 32) | (ELEM(zm
, 1, 16) << 48);
2019 uint64_t m0
= ELEM(zd
, 2, 16) | (ELEM(zm
, 2, 16) << 16)
2020 | (ELEM(zd
, 3, 16) << 32) | (ELEM(zm
, 3, 16) << 48);
2021 env
->vfp
.regs
[rm
] = make_float64(m0
);
2022 env
->vfp
.regs
[rd
] = make_float64(d0
);