2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licenced under the GNU GPL v2.
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
21 static float_status neon_float_status
;
22 #define NFS &neon_float_status
24 /* Helper routines to perform bitwise copies between float and int. */
25 static inline float32
vfp_itos(uint32_t i
)
36 static inline uint32_t vfp_stoi(float32 s
)
47 #define NEON_TYPE1(name, type) \
52 #ifdef HOST_WORDS_BIGENDIAN
53 #define NEON_TYPE2(name, type) \
59 #define NEON_TYPE4(name, type) \
68 #define NEON_TYPE2(name, type) \
74 #define NEON_TYPE4(name, type) \
84 NEON_TYPE4(s8
, int8_t)
85 NEON_TYPE4(u8
, uint8_t)
86 NEON_TYPE2(s16
, int16_t)
87 NEON_TYPE2(u16
, uint16_t)
88 NEON_TYPE1(s32
, int32_t)
89 NEON_TYPE1(u32
, uint32_t)
94 /* Copy from a uint32_t to a vector structure type. */
95 #define NEON_UNPACK(vtype, dest, val) do { \
104 /* Copy from a vector structure type to a uint32_t. */
105 #define NEON_PACK(vtype, dest, val) do { \
115 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
117 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
118 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
120 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
121 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
122 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
123 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
125 #define NEON_VOP_BODY(vtype, n) \
131 NEON_UNPACK(vtype, vsrc1, arg1); \
132 NEON_UNPACK(vtype, vsrc2, arg2); \
134 NEON_PACK(vtype, res, vdest); \
138 #define NEON_VOP(name, vtype, n) \
139 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
140 NEON_VOP_BODY(vtype, n)
142 #define NEON_VOP_ENV(name, vtype, n) \
143 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
144 NEON_VOP_BODY(vtype, n)
146 /* Pairwise operations. */
147 /* For 32-bit elements each segment only contains a single element, so
148 the elementwise and pairwise operations are the same. */
150 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
151 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
153 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
154 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
155 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
156 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
158 #define NEON_POP(name, vtype, n) \
159 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
165 NEON_UNPACK(vtype, vsrc1, arg1); \
166 NEON_UNPACK(vtype, vsrc2, arg2); \
168 NEON_PACK(vtype, res, vdest); \
172 /* Unary operators. */
173 #define NEON_VOP1(name, vtype, n) \
174 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
178 NEON_UNPACK(vtype, vsrc1, arg); \
180 NEON_PACK(vtype, arg, vdest); \
185 #define NEON_USAT(dest, src1, src2, type) do { \
186 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
187 if (tmp != (type)tmp) { \
193 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
194 NEON_VOP_ENV(qadd_u8
, neon_u8
, 4)
196 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
197 NEON_VOP_ENV(qadd_u16
, neon_u16
, 2)
201 #define NEON_SSAT(dest, src1, src2, type) do { \
202 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
203 if (tmp != (type)tmp) { \
206 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
208 tmp = 1 << (sizeof(type) * 8 - 1); \
213 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
214 NEON_VOP_ENV(qadd_s8
, neon_s8
, 4)
216 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
217 NEON_VOP_ENV(qadd_s16
, neon_s16
, 2)
221 #define NEON_USAT(dest, src1, src2, type) do { \
222 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
223 if (tmp != (type)tmp) { \
229 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
230 NEON_VOP_ENV(qsub_u8
, neon_u8
, 4)
232 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
233 NEON_VOP_ENV(qsub_u16
, neon_u16
, 2)
237 #define NEON_SSAT(dest, src1, src2, type) do { \
238 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
239 if (tmp != (type)tmp) { \
242 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
244 tmp = 1 << (sizeof(type) * 8 - 1); \
249 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
250 NEON_VOP_ENV(qsub_s8
, neon_s8
, 4)
252 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
253 NEON_VOP_ENV(qsub_s16
, neon_s16
, 2)
257 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
258 NEON_VOP(hadd_s8
, neon_s8
, 4)
259 NEON_VOP(hadd_u8
, neon_u8
, 4)
260 NEON_VOP(hadd_s16
, neon_s16
, 2)
261 NEON_VOP(hadd_u16
, neon_u16
, 2)
264 int32_t HELPER(neon_hadd_s32
)(int32_t src1
, int32_t src2
)
268 dest
= (src1
>> 1) + (src2
>> 1);
274 uint32_t HELPER(neon_hadd_u32
)(uint32_t src1
, uint32_t src2
)
278 dest
= (src1
>> 1) + (src2
>> 1);
284 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
285 NEON_VOP(rhadd_s8
, neon_s8
, 4)
286 NEON_VOP(rhadd_u8
, neon_u8
, 4)
287 NEON_VOP(rhadd_s16
, neon_s16
, 2)
288 NEON_VOP(rhadd_u16
, neon_u16
, 2)
291 int32_t HELPER(neon_rhadd_s32
)(int32_t src1
, int32_t src2
)
295 dest
= (src1
>> 1) + (src2
>> 1);
296 if ((src1
| src2
) & 1)
301 uint32_t HELPER(neon_rhadd_u32
)(uint32_t src1
, uint32_t src2
)
305 dest
= (src1
>> 1) + (src2
>> 1);
306 if ((src1
| src2
) & 1)
311 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
312 NEON_VOP(hsub_s8
, neon_s8
, 4)
313 NEON_VOP(hsub_u8
, neon_u8
, 4)
314 NEON_VOP(hsub_s16
, neon_s16
, 2)
315 NEON_VOP(hsub_u16
, neon_u16
, 2)
318 int32_t HELPER(neon_hsub_s32
)(int32_t src1
, int32_t src2
)
322 dest
= (src1
>> 1) - (src2
>> 1);
323 if ((~src1
) & src2
& 1)
328 uint32_t HELPER(neon_hsub_u32
)(uint32_t src1
, uint32_t src2
)
332 dest
= (src1
>> 1) - (src2
>> 1);
333 if ((~src1
) & src2
& 1)
338 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
339 NEON_VOP(cgt_s8
, neon_s8
, 4)
340 NEON_VOP(cgt_u8
, neon_u8
, 4)
341 NEON_VOP(cgt_s16
, neon_s16
, 2)
342 NEON_VOP(cgt_u16
, neon_u16
, 2)
343 NEON_VOP(cgt_s32
, neon_s32
, 1)
344 NEON_VOP(cgt_u32
, neon_u32
, 1)
347 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
348 NEON_VOP(cge_s8
, neon_s8
, 4)
349 NEON_VOP(cge_u8
, neon_u8
, 4)
350 NEON_VOP(cge_s16
, neon_s16
, 2)
351 NEON_VOP(cge_u16
, neon_u16
, 2)
352 NEON_VOP(cge_s32
, neon_s32
, 1)
353 NEON_VOP(cge_u32
, neon_u32
, 1)
356 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
357 NEON_VOP(min_s8
, neon_s8
, 4)
358 NEON_VOP(min_u8
, neon_u8
, 4)
359 NEON_VOP(min_s16
, neon_s16
, 2)
360 NEON_VOP(min_u16
, neon_u16
, 2)
361 NEON_VOP(min_s32
, neon_s32
, 1)
362 NEON_VOP(min_u32
, neon_u32
, 1)
363 NEON_POP(pmin_s8
, neon_s8
, 4)
364 NEON_POP(pmin_u8
, neon_u8
, 4)
365 NEON_POP(pmin_s16
, neon_s16
, 2)
366 NEON_POP(pmin_u16
, neon_u16
, 2)
369 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
370 NEON_VOP(max_s8
, neon_s8
, 4)
371 NEON_VOP(max_u8
, neon_u8
, 4)
372 NEON_VOP(max_s16
, neon_s16
, 2)
373 NEON_VOP(max_u16
, neon_u16
, 2)
374 NEON_VOP(max_s32
, neon_s32
, 1)
375 NEON_VOP(max_u32
, neon_u32
, 1)
376 NEON_POP(pmax_s8
, neon_s8
, 4)
377 NEON_POP(pmax_u8
, neon_u8
, 4)
378 NEON_POP(pmax_s16
, neon_s16
, 2)
379 NEON_POP(pmax_u16
, neon_u16
, 2)
382 #define NEON_FN(dest, src1, src2) \
383 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
384 NEON_VOP(abd_s8
, neon_s8
, 4)
385 NEON_VOP(abd_u8
, neon_u8
, 4)
386 NEON_VOP(abd_s16
, neon_s16
, 2)
387 NEON_VOP(abd_u16
, neon_u16
, 2)
388 NEON_VOP(abd_s32
, neon_s32
, 1)
389 NEON_VOP(abd_u32
, neon_u32
, 1)
392 #define NEON_FN(dest, src1, src2) do { \
394 tmp = (int8_t)src2; \
395 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
396 tmp <= -(ssize_t)sizeof(src1) * 8) { \
398 } else if (tmp < 0) { \
399 dest = src1 >> -tmp; \
401 dest = src1 << tmp; \
403 NEON_VOP(shl_u8
, neon_u8
, 4)
404 NEON_VOP(shl_u16
, neon_u16
, 2)
405 NEON_VOP(shl_u32
, neon_u32
, 1)
408 uint64_t HELPER(neon_shl_u64
)(uint64_t val
, uint64_t shiftop
)
410 int8_t shift
= (int8_t)shiftop
;
411 if (shift
>= 64 || shift
<= -64) {
413 } else if (shift
< 0) {
421 #define NEON_FN(dest, src1, src2) do { \
423 tmp = (int8_t)src2; \
424 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
426 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
427 dest = src1 >> (sizeof(src1) * 8 - 1); \
428 } else if (tmp < 0) { \
429 dest = src1 >> -tmp; \
431 dest = src1 << tmp; \
433 NEON_VOP(shl_s8
, neon_s8
, 4)
434 NEON_VOP(shl_s16
, neon_s16
, 2)
435 NEON_VOP(shl_s32
, neon_s32
, 1)
438 uint64_t HELPER(neon_shl_s64
)(uint64_t valop
, uint64_t shiftop
)
440 int8_t shift
= (int8_t)shiftop
;
444 } else if (shift
<= -64) {
446 } else if (shift
< 0) {
454 #define NEON_FN(dest, src1, src2) do { \
456 tmp = (int8_t)src2; \
457 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
459 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
460 dest = src1 >> (sizeof(src1) * 8 - 1); \
461 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
462 dest = src1 >> (tmp - 1); \
465 } else if (tmp < 0) { \
466 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
468 dest = src1 << tmp; \
470 NEON_VOP(rshl_s8
, neon_s8
, 4)
471 NEON_VOP(rshl_s16
, neon_s16
, 2)
472 NEON_VOP(rshl_s32
, neon_s32
, 1)
475 uint64_t HELPER(neon_rshl_s64
)(uint64_t valop
, uint64_t shiftop
)
477 int8_t shift
= (int8_t)shiftop
;
481 } else if (shift
< -64) {
483 } else if (shift
== -63) {
487 } else if (shift
< 0) {
488 val
= (val
+ ((int64_t)1 << (-1 - shift
))) >> -shift
;
495 #define NEON_FN(dest, src1, src2) do { \
497 tmp = (int8_t)src2; \
498 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
499 tmp < -(ssize_t)sizeof(src1) * 8) { \
501 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
502 dest = src1 >> (tmp - 1); \
503 } else if (tmp < 0) { \
504 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
506 dest = src1 << tmp; \
508 NEON_VOP(rshl_u8
, neon_u8
, 4)
509 NEON_VOP(rshl_u16
, neon_u16
, 2)
510 NEON_VOP(rshl_u32
, neon_u32
, 1)
513 uint64_t HELPER(neon_rshl_u64
)(uint64_t val
, uint64_t shiftop
)
515 int8_t shift
= (uint8_t)shiftop
;
516 if (shift
>= 64 || shift
< 64) {
518 } else if (shift
== -64) {
519 /* Rounding a 1-bit result just preserves that bit. */
522 val
= (val
+ ((uint64_t)1 << (-1 - shift
))) >> -shift
;
530 #define NEON_FN(dest, src1, src2) do { \
532 tmp = (int8_t)src2; \
533 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
540 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
542 } else if (tmp < 0) { \
543 dest = src1 >> -tmp; \
545 dest = src1 << tmp; \
546 if ((dest >> tmp) != src1) { \
551 NEON_VOP_ENV(qshl_u8
, neon_u8
, 4)
552 NEON_VOP_ENV(qshl_u16
, neon_u16
, 2)
553 NEON_VOP_ENV(qshl_u32
, neon_u32
, 1)
556 uint64_t HELPER(neon_qshl_u64
)(CPUState
*env
, uint64_t val
, uint64_t shiftop
)
558 int8_t shift
= (int8_t)shiftop
;
564 } else if (shift
<= -64) {
566 } else if (shift
< 0) {
571 if ((val
>> shift
) != tmp
) {
579 #define NEON_FN(dest, src1, src2) do { \
581 tmp = (int8_t)src2; \
582 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
585 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
592 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
594 } else if (tmp < 0) { \
595 dest = src1 >> -tmp; \
597 dest = src1 << tmp; \
598 if ((dest >> tmp) != src1) { \
600 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
606 NEON_VOP_ENV(qshl_s8
, neon_s8
, 4)
607 NEON_VOP_ENV(qshl_s16
, neon_s16
, 2)
608 NEON_VOP_ENV(qshl_s32
, neon_s32
, 1)
611 uint64_t HELPER(neon_qshl_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
613 int8_t shift
= (uint8_t)shiftop
;
618 val
= (val
>> 63) ^ ~SIGNBIT64
;
620 } else if (shift
<= -64) {
622 } else if (shift
< 0) {
627 if ((val
>> shift
) != tmp
) {
629 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
635 #define NEON_FN(dest, src1, src2) do { \
636 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
641 tmp = (int8_t)src2; \
642 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
649 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
651 } else if (tmp < 0) { \
652 dest = src1 >> -tmp; \
654 dest = src1 << tmp; \
655 if ((dest >> tmp) != src1) { \
661 NEON_VOP_ENV(qshlu_s8
, neon_u8
, 4)
662 NEON_VOP_ENV(qshlu_s16
, neon_u16
, 2)
665 uint32_t HELPER(neon_qshlu_s32
)(CPUState
*env
, uint32_t valop
, uint32_t shiftop
)
667 if ((int32_t)valop
< 0) {
671 return helper_neon_qshl_u32(env
, valop
, shiftop
);
674 uint64_t HELPER(neon_qshlu_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
676 if ((int64_t)valop
< 0) {
680 return helper_neon_qshl_u64(env
, valop
, shiftop
);
683 /* FIXME: This is wrong. */
684 #define NEON_FN(dest, src1, src2) do { \
686 tmp = (int8_t)src2; \
688 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
690 dest = src1 << tmp; \
691 if ((dest >> tmp) != src1) { \
696 NEON_VOP_ENV(qrshl_u8
, neon_u8
, 4)
697 NEON_VOP_ENV(qrshl_u16
, neon_u16
, 2)
698 NEON_VOP_ENV(qrshl_u32
, neon_u32
, 1)
701 uint64_t HELPER(neon_qrshl_u64
)(CPUState
*env
, uint64_t val
, uint64_t shiftop
)
703 int8_t shift
= (int8_t)shiftop
;
705 val
= (val
+ (1 << (-1 - shift
))) >> -shift
;
709 if ((val
>> shift
) != tmp
) {
717 #define NEON_FN(dest, src1, src2) do { \
719 tmp = (int8_t)src2; \
721 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
723 dest = src1 << tmp; \
724 if ((dest >> tmp) != src1) { \
729 NEON_VOP_ENV(qrshl_s8
, neon_s8
, 4)
730 NEON_VOP_ENV(qrshl_s16
, neon_s16
, 2)
731 NEON_VOP_ENV(qrshl_s32
, neon_s32
, 1)
734 uint64_t HELPER(neon_qrshl_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
736 int8_t shift
= (uint8_t)shiftop
;
740 val
= (val
+ (1 << (-1 - shift
))) >> -shift
;
744 if ((val
>> shift
) != tmp
) {
752 uint32_t HELPER(neon_add_u8
)(uint32_t a
, uint32_t b
)
755 mask
= (a
^ b
) & 0x80808080u
;
758 return (a
+ b
) ^ mask
;
761 uint32_t HELPER(neon_add_u16
)(uint32_t a
, uint32_t b
)
764 mask
= (a
^ b
) & 0x80008000u
;
767 return (a
+ b
) ^ mask
;
770 #define NEON_FN(dest, src1, src2) dest = src1 + src2
771 NEON_POP(padd_u8
, neon_u8
, 4)
772 NEON_POP(padd_u16
, neon_u16
, 2)
775 #define NEON_FN(dest, src1, src2) dest = src1 - src2
776 NEON_VOP(sub_u8
, neon_u8
, 4)
777 NEON_VOP(sub_u16
, neon_u16
, 2)
780 #define NEON_FN(dest, src1, src2) dest = src1 * src2
781 NEON_VOP(mul_u8
, neon_u8
, 4)
782 NEON_VOP(mul_u16
, neon_u16
, 2)
785 /* Polynomial multiplication is like integer multiplication except the
786 partial products are XORed, not added. */
787 uint32_t HELPER(neon_mul_p8
)(uint32_t op1
, uint32_t op2
)
799 mask
|= (0xff << 16);
801 mask
|= (0xff << 24);
802 result
^= op2
& mask
;
803 op1
= (op1
>> 1) & 0x7f7f7f7f;
804 op2
= (op2
<< 1) & 0xfefefefe;
809 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
810 NEON_VOP(tst_u8
, neon_u8
, 4)
811 NEON_VOP(tst_u16
, neon_u16
, 2)
812 NEON_VOP(tst_u32
, neon_u32
, 1)
815 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
816 NEON_VOP(ceq_u8
, neon_u8
, 4)
817 NEON_VOP(ceq_u16
, neon_u16
, 2)
818 NEON_VOP(ceq_u32
, neon_u32
, 1)
821 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
822 NEON_VOP1(abs_s8
, neon_s8
, 4)
823 NEON_VOP1(abs_s16
, neon_s16
, 2)
826 /* Count Leading Sign/Zero Bits. */
827 static inline int do_clz8(uint8_t x
)
835 static inline int do_clz16(uint16_t x
)
843 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
844 NEON_VOP1(clz_u8
, neon_u8
, 4)
847 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
848 NEON_VOP1(clz_u16
, neon_u16
, 2)
851 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
852 NEON_VOP1(cls_s8
, neon_s8
, 4)
855 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
856 NEON_VOP1(cls_s16
, neon_s16
, 2)
859 uint32_t HELPER(neon_cls_s32
)(uint32_t x
)
864 for (count
= 32; x
; count
--)
870 uint32_t HELPER(neon_cnt_u8
)(uint32_t x
)
872 x
= (x
& 0x55555555) + ((x
>> 1) & 0x55555555);
873 x
= (x
& 0x33333333) + ((x
>> 2) & 0x33333333);
874 x
= (x
& 0x0f0f0f0f) + ((x
>> 4) & 0x0f0f0f0f);
878 #define NEON_QDMULH16(dest, src1, src2, round) do { \
879 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
880 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
882 tmp = (tmp >> 31) ^ ~SIGNBIT; \
888 if ((int32_t)tmp < old) { \
895 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
896 NEON_VOP_ENV(qdmulh_s16
, neon_s16
, 2)
898 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
899 NEON_VOP_ENV(qrdmulh_s16
, neon_s16
, 2)
903 #define NEON_QDMULH32(dest, src1, src2, round) do { \
904 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
905 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
907 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
913 tmp += (int64_t)1 << 31; \
914 if ((int64_t)tmp < old) { \
916 tmp = SIGNBIT64 - 1; \
921 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
922 NEON_VOP_ENV(qdmulh_s32
, neon_s32
, 1)
924 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
925 NEON_VOP_ENV(qrdmulh_s32
, neon_s32
, 1)
929 uint32_t HELPER(neon_narrow_u8
)(uint64_t x
)
931 return (x
& 0xffu
) | ((x
>> 8) & 0xff00u
) | ((x
>> 16) & 0xff0000u
)
932 | ((x
>> 24) & 0xff000000u
);
935 uint32_t HELPER(neon_narrow_u16
)(uint64_t x
)
937 return (x
& 0xffffu
) | ((x
>> 16) & 0xffff0000u
);
940 uint32_t HELPER(neon_narrow_high_u8
)(uint64_t x
)
942 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
943 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
946 uint32_t HELPER(neon_narrow_high_u16
)(uint64_t x
)
948 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
951 uint32_t HELPER(neon_narrow_round_high_u8
)(uint64_t x
)
953 x
&= 0xff80ff80ff80ff80ull
;
954 x
+= 0x0080008000800080ull
;
955 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
956 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
959 uint32_t HELPER(neon_narrow_round_high_u16
)(uint64_t x
)
961 x
&= 0xffff8000ffff8000ull
;
962 x
+= 0x0000800000008000ull
;
963 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
966 uint32_t HELPER(neon_narrow_sat_u8
)(CPUState
*env
, uint64_t x
)
979 res |= (uint32_t)d << (n / 2);
989 uint32_t HELPER(neon_narrow_sat_s8
)(CPUState
*env
, uint64_t x
)
996 if (s != (int8_t)s) { \
997 d = (s >> 15) ^ 0x7f; \
1002 res |= (uint32_t)d << (n / 2);
1012 uint32_t HELPER(neon_narrow_sat_u16
)(CPUState
*env
, uint64_t x
)
1022 if (high
> 0xffff) {
1026 return low
| (high
<< 16);
1029 uint32_t HELPER(neon_narrow_sat_s16
)(CPUState
*env
, uint64_t x
)
1034 if (low
!= (int16_t)low
) {
1035 low
= (low
>> 31) ^ 0x7fff;
1039 if (high
!= (int16_t)high
) {
1040 high
= (high
>> 31) ^ 0x7fff;
1043 return (uint16_t)low
| (high
<< 16);
1046 uint32_t HELPER(neon_narrow_sat_u32
)(CPUState
*env
, uint64_t x
)
1048 if (x
> 0xffffffffu
) {
1055 uint32_t HELPER(neon_narrow_sat_s32
)(CPUState
*env
, uint64_t x
)
1057 if ((int64_t)x
!= (int32_t)x
) {
1059 return (x
>> 63) ^ 0x7fffffff;
1064 uint64_t HELPER(neon_widen_u8
)(uint32_t x
)
1069 tmp
= (uint8_t)(x
>> 8);
1071 tmp
= (uint8_t)(x
>> 16);
1073 tmp
= (uint8_t)(x
>> 24);
1078 uint64_t HELPER(neon_widen_s8
)(uint32_t x
)
1082 ret
= (uint16_t)(int8_t)x
;
1083 tmp
= (uint16_t)(int8_t)(x
>> 8);
1085 tmp
= (uint16_t)(int8_t)(x
>> 16);
1087 tmp
= (uint16_t)(int8_t)(x
>> 24);
1092 uint64_t HELPER(neon_widen_u16
)(uint32_t x
)
1094 uint64_t high
= (uint16_t)(x
>> 16);
1095 return ((uint16_t)x
) | (high
<< 32);
1098 uint64_t HELPER(neon_widen_s16
)(uint32_t x
)
1100 uint64_t high
= (int16_t)(x
>> 16);
1101 return ((uint32_t)(int16_t)x
) | (high
<< 32);
1104 uint64_t HELPER(neon_addl_u16
)(uint64_t a
, uint64_t b
)
1107 mask
= (a
^ b
) & 0x8000800080008000ull
;
1108 a
&= ~0x8000800080008000ull
;
1109 b
&= ~0x8000800080008000ull
;
1110 return (a
+ b
) ^ mask
;
1113 uint64_t HELPER(neon_addl_u32
)(uint64_t a
, uint64_t b
)
1116 mask
= (a
^ b
) & 0x8000000080000000ull
;
1117 a
&= ~0x8000000080000000ull
;
1118 b
&= ~0x8000000080000000ull
;
1119 return (a
+ b
) ^ mask
;
1122 uint64_t HELPER(neon_paddl_u16
)(uint64_t a
, uint64_t b
)
1127 tmp
= a
& 0x0000ffff0000ffffull
;
1128 tmp
+= (a
>> 16) & 0x0000ffff0000ffffull
;
1129 tmp2
= b
& 0xffff0000ffff0000ull
;
1130 tmp2
+= (b
<< 16) & 0xffff0000ffff0000ull
;
1131 return ( tmp
& 0xffff)
1132 | ((tmp
>> 16) & 0xffff0000ull
)
1133 | ((tmp2
<< 16) & 0xffff00000000ull
)
1134 | ( tmp2
& 0xffff000000000000ull
);
1137 uint64_t HELPER(neon_paddl_u32
)(uint64_t a
, uint64_t b
)
1139 uint32_t low
= a
+ (a
>> 32);
1140 uint32_t high
= b
+ (b
>> 32);
1141 return low
+ ((uint64_t)high
<< 32);
1144 uint64_t HELPER(neon_subl_u16
)(uint64_t a
, uint64_t b
)
1147 mask
= (a
^ ~b
) & 0x8000800080008000ull
;
1148 a
|= 0x8000800080008000ull
;
1149 b
&= ~0x8000800080008000ull
;
1150 return (a
- b
) ^ mask
;
1153 uint64_t HELPER(neon_subl_u32
)(uint64_t a
, uint64_t b
)
1156 mask
= (a
^ ~b
) & 0x8000000080000000ull
;
1157 a
|= 0x8000000080000000ull
;
1158 b
&= ~0x8000000080000000ull
;
1159 return (a
- b
) ^ mask
;
1162 uint64_t HELPER(neon_addl_saturate_s32
)(CPUState
*env
, uint64_t a
, uint64_t b
)
1170 if (((low
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1172 low
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1177 if (((high
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1179 high
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1181 return low
| ((uint64_t)high
<< 32);
1184 uint64_t HELPER(neon_addl_saturate_s64
)(CPUState
*env
, uint64_t a
, uint64_t b
)
1189 if (((result
^ a
) & SIGNBIT64
) && !((a
^ b
) & SIGNBIT64
)) {
1191 result
= ((int64_t)a
>> 63) ^ ~SIGNBIT64
;
1196 #define DO_ABD(dest, x, y, type) do { \
1199 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1202 uint64_t HELPER(neon_abdl_u16
)(uint32_t a
, uint32_t b
)
1206 DO_ABD(result
, a
, b
, uint8_t);
1207 DO_ABD(tmp
, a
>> 8, b
>> 8, uint8_t);
1208 result
|= tmp
<< 16;
1209 DO_ABD(tmp
, a
>> 16, b
>> 16, uint8_t);
1210 result
|= tmp
<< 32;
1211 DO_ABD(tmp
, a
>> 24, b
>> 24, uint8_t);
1212 result
|= tmp
<< 48;
1216 uint64_t HELPER(neon_abdl_s16
)(uint32_t a
, uint32_t b
)
1220 DO_ABD(result
, a
, b
, int8_t);
1221 DO_ABD(tmp
, a
>> 8, b
>> 8, int8_t);
1222 result
|= tmp
<< 16;
1223 DO_ABD(tmp
, a
>> 16, b
>> 16, int8_t);
1224 result
|= tmp
<< 32;
1225 DO_ABD(tmp
, a
>> 24, b
>> 24, int8_t);
1226 result
|= tmp
<< 48;
1230 uint64_t HELPER(neon_abdl_u32
)(uint32_t a
, uint32_t b
)
1234 DO_ABD(result
, a
, b
, uint16_t);
1235 DO_ABD(tmp
, a
>> 16, b
>> 16, uint16_t);
1236 return result
| (tmp
<< 32);
1239 uint64_t HELPER(neon_abdl_s32
)(uint32_t a
, uint32_t b
)
1243 DO_ABD(result
, a
, b
, int16_t);
1244 DO_ABD(tmp
, a
>> 16, b
>> 16, int16_t);
1245 return result
| (tmp
<< 32);
1248 uint64_t HELPER(neon_abdl_u64
)(uint32_t a
, uint32_t b
)
1251 DO_ABD(result
, a
, b
, uint32_t);
1255 uint64_t HELPER(neon_abdl_s64
)(uint32_t a
, uint32_t b
)
1258 DO_ABD(result
, a
, b
, int32_t);
1263 /* Widening multiply. Named type is the source type. */
1264 #define DO_MULL(dest, x, y, type1, type2) do { \
1267 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1270 uint64_t HELPER(neon_mull_u8
)(uint32_t a
, uint32_t b
)
1275 DO_MULL(result
, a
, b
, uint8_t, uint16_t);
1276 DO_MULL(tmp
, a
>> 8, b
>> 8, uint8_t, uint16_t);
1277 result
|= tmp
<< 16;
1278 DO_MULL(tmp
, a
>> 16, b
>> 16, uint8_t, uint16_t);
1279 result
|= tmp
<< 32;
1280 DO_MULL(tmp
, a
>> 24, b
>> 24, uint8_t, uint16_t);
1281 result
|= tmp
<< 48;
1285 uint64_t HELPER(neon_mull_s8
)(uint32_t a
, uint32_t b
)
1290 DO_MULL(result
, a
, b
, int8_t, uint16_t);
1291 DO_MULL(tmp
, a
>> 8, b
>> 8, int8_t, uint16_t);
1292 result
|= tmp
<< 16;
1293 DO_MULL(tmp
, a
>> 16, b
>> 16, int8_t, uint16_t);
1294 result
|= tmp
<< 32;
1295 DO_MULL(tmp
, a
>> 24, b
>> 24, int8_t, uint16_t);
1296 result
|= tmp
<< 48;
1300 uint64_t HELPER(neon_mull_u16
)(uint32_t a
, uint32_t b
)
1305 DO_MULL(result
, a
, b
, uint16_t, uint32_t);
1306 DO_MULL(tmp
, a
>> 16, b
>> 16, uint16_t, uint32_t);
1307 return result
| (tmp
<< 32);
1310 uint64_t HELPER(neon_mull_s16
)(uint32_t a
, uint32_t b
)
1315 DO_MULL(result
, a
, b
, int16_t, uint32_t);
1316 DO_MULL(tmp
, a
>> 16, b
>> 16, int16_t, uint32_t);
1317 return result
| (tmp
<< 32);
1320 uint64_t HELPER(neon_negl_u16
)(uint64_t x
)
1324 result
= (uint16_t)-x
;
1326 result
|= (uint64_t)tmp
<< 16;
1328 result
|= (uint64_t)tmp
<< 32;
1330 result
|= (uint64_t)tmp
<< 48;
1335 uint64_t HELPER(neon_negl_u32
)(uint64_t x
)
1338 uint32_t high
= -(x
>> 32);
1339 return low
| ((uint64_t)high
<< 32);
1342 /* FIXME: There should be a native op for this. */
1343 uint64_t HELPER(neon_negl_u64
)(uint64_t x
)
1348 /* Saturnating sign manuipulation. */
1349 /* ??? Make these use NEON_VOP1 */
1350 #define DO_QABS8(x) do { \
1351 if (x == (int8_t)0x80) { \
1354 } else if (x < 0) { \
1357 uint32_t HELPER(neon_qabs_s8
)(CPUState
*env
, uint32_t x
)
1360 NEON_UNPACK(neon_s8
, vec
, x
);
1365 NEON_PACK(neon_s8
, x
, vec
);
1370 #define DO_QNEG8(x) do { \
1371 if (x == (int8_t)0x80) { \
1377 uint32_t HELPER(neon_qneg_s8
)(CPUState
*env
, uint32_t x
)
1380 NEON_UNPACK(neon_s8
, vec
, x
);
1385 NEON_PACK(neon_s8
, x
, vec
);
1390 #define DO_QABS16(x) do { \
1391 if (x == (int16_t)0x8000) { \
1394 } else if (x < 0) { \
1397 uint32_t HELPER(neon_qabs_s16
)(CPUState
*env
, uint32_t x
)
1400 NEON_UNPACK(neon_s16
, vec
, x
);
1403 NEON_PACK(neon_s16
, x
, vec
);
1408 #define DO_QNEG16(x) do { \
1409 if (x == (int16_t)0x8000) { \
1415 uint32_t HELPER(neon_qneg_s16
)(CPUState
*env
, uint32_t x
)
1418 NEON_UNPACK(neon_s16
, vec
, x
);
1421 NEON_PACK(neon_s16
, x
, vec
);
1426 uint32_t HELPER(neon_qabs_s32
)(CPUState
*env
, uint32_t x
)
1431 } else if ((int32_t)x
< 0) {
1437 uint32_t HELPER(neon_qneg_s32
)(CPUState
*env
, uint32_t x
)
1448 /* NEON Float helpers. */
1449 uint32_t HELPER(neon_min_f32
)(uint32_t a
, uint32_t b
)
1451 float32 f0
= vfp_itos(a
);
1452 float32 f1
= vfp_itos(b
);
1453 return (float32_compare_quiet(f0
, f1
, NFS
) == -1) ? a
: b
;
1456 uint32_t HELPER(neon_max_f32
)(uint32_t a
, uint32_t b
)
1458 float32 f0
= vfp_itos(a
);
1459 float32 f1
= vfp_itos(b
);
1460 return (float32_compare_quiet(f0
, f1
, NFS
) == 1) ? a
: b
;
1463 uint32_t HELPER(neon_abd_f32
)(uint32_t a
, uint32_t b
)
1465 float32 f0
= vfp_itos(a
);
1466 float32 f1
= vfp_itos(b
);
1467 return vfp_stoi((float32_compare_quiet(f0
, f1
, NFS
) == 1)
1468 ? float32_sub(f0
, f1
, NFS
)
1469 : float32_sub(f1
, f0
, NFS
));
1472 uint32_t HELPER(neon_add_f32
)(uint32_t a
, uint32_t b
)
1474 return vfp_stoi(float32_add(vfp_itos(a
), vfp_itos(b
), NFS
));
1477 uint32_t HELPER(neon_sub_f32
)(uint32_t a
, uint32_t b
)
1479 return vfp_stoi(float32_sub(vfp_itos(a
), vfp_itos(b
), NFS
));
1482 uint32_t HELPER(neon_mul_f32
)(uint32_t a
, uint32_t b
)
1484 return vfp_stoi(float32_mul(vfp_itos(a
), vfp_itos(b
), NFS
));
1487 /* Floating point comparisons produce an integer result. */
1488 #define NEON_VOP_FCMP(name, cmp) \
1489 uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1491 if (float32_compare_quiet(vfp_itos(a), vfp_itos(b), NFS) cmp 0) \
1497 NEON_VOP_FCMP(ceq_f32
, ==)
1498 NEON_VOP_FCMP(cge_f32
, >=)
1499 NEON_VOP_FCMP(cgt_f32
, >)
1501 uint32_t HELPER(neon_acge_f32
)(uint32_t a
, uint32_t b
)
1503 float32 f0
= float32_abs(vfp_itos(a
));
1504 float32 f1
= float32_abs(vfp_itos(b
));
1505 return (float32_compare_quiet(f0
, f1
,NFS
) >= 0) ? ~0 : 0;
1508 uint32_t HELPER(neon_acgt_f32
)(uint32_t a
, uint32_t b
)
1510 float32 f0
= float32_abs(vfp_itos(a
));
1511 float32 f1
= float32_abs(vfp_itos(b
));
1512 return (float32_compare_quiet(f0
, f1
, NFS
) > 0) ? ~0 : 0;