2 * Optimizations for Tiny Code Generator for QEMU
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 #include "qemu/osdep.h"
27 #include "qemu/int128.h"
28 #include "qemu/interval-tree.h"
29 #include "tcg/tcg-op-common.h"
30 #include "tcg-internal.h"
32 #define CASE_OP_32_64(x) \
33 glue(glue(case INDEX_op_, x), _i32): \
34 glue(glue(case INDEX_op_, x), _i64)
36 #define CASE_OP_32_64_VEC(x) \
37 glue(glue(case INDEX_op_, x), _i32): \
38 glue(glue(case INDEX_op_, x), _i64): \
39 glue(glue(case INDEX_op_, x), _vec)
41 typedef struct MemCopyInfo
{
42 IntervalTreeNode itree
;
43 QSIMPLEQ_ENTRY (MemCopyInfo
) next
;
48 typedef struct TempOptInfo
{
52 QSIMPLEQ_HEAD(, MemCopyInfo
) mem_copy
;
54 uint64_t z_mask
; /* mask bit is 0 if and only if value bit is 0 */
55 uint64_t s_mask
; /* a left-aligned mask of clrsb(value) bits. */
58 typedef struct OptContext
{
61 TCGTempSet temps_used
;
63 IntervalTreeRoot mem_copy
;
64 QSIMPLEQ_HEAD(, MemCopyInfo
) mem_free
;
66 /* In flight values from optimization. */
67 uint64_t a_mask
; /* mask bit is 0 iff value identical to first input */
68 uint64_t z_mask
; /* mask bit is 0 iff value bit is 0 */
69 uint64_t s_mask
; /* mask of clrsb(value) bits */
73 /* Calculate the smask for a specific value. */
74 static uint64_t smask_from_value(uint64_t value
)
76 int rep
= clrsb64(value
);
77 return ~(~0ull >> rep
);
81 * Calculate the smask for a given set of known-zeros.
82 * If there are lots of zeros on the left, we can consider the remainder
83 * an unsigned field, and thus the corresponding signed field is one bit
86 static uint64_t smask_from_zmask(uint64_t zmask
)
89 * Only the 0 bits are significant for zmask, thus the msb itself
90 * must be zero, else we have no sign information.
92 int rep
= clz64(zmask
);
97 return ~(~0ull >> rep
);
101 * Recreate a properly left-aligned smask after manipulation.
102 * Some bit-shuffling, particularly shifts and rotates, may
103 * retain sign bits on the left, but may scatter disconnected
104 * sign bits on the right. Retain only what remains to the left.
106 static uint64_t smask_from_smask(int64_t smask
)
108 /* Only the 1 bits are significant for smask */
109 return smask_from_zmask(~smask
);
112 static inline TempOptInfo
*ts_info(TCGTemp
*ts
)
114 return ts
->state_ptr
;
117 static inline TempOptInfo
*arg_info(TCGArg arg
)
119 return ts_info(arg_temp(arg
));
122 static inline bool ts_is_const(TCGTemp
*ts
)
124 return ts_info(ts
)->is_const
;
127 static inline bool arg_is_const(TCGArg arg
)
129 return ts_is_const(arg_temp(arg
));
132 static inline bool ts_is_copy(TCGTemp
*ts
)
134 return ts_info(ts
)->next_copy
!= ts
;
137 static TCGTemp
*cmp_better_copy(TCGTemp
*a
, TCGTemp
*b
)
139 return a
->kind
< b
->kind
? b
: a
;
142 /* Initialize and activate a temporary. */
143 static void init_ts_info(OptContext
*ctx
, TCGTemp
*ts
)
145 size_t idx
= temp_idx(ts
);
148 if (test_bit(idx
, ctx
->temps_used
.l
)) {
151 set_bit(idx
, ctx
->temps_used
.l
);
155 ti
= tcg_malloc(sizeof(TempOptInfo
));
161 QSIMPLEQ_INIT(&ti
->mem_copy
);
162 if (ts
->kind
== TEMP_CONST
) {
165 ti
->z_mask
= ts
->val
;
166 ti
->s_mask
= smask_from_value(ts
->val
);
168 ti
->is_const
= false;
174 static MemCopyInfo
*mem_copy_first(OptContext
*ctx
, intptr_t s
, intptr_t l
)
176 IntervalTreeNode
*r
= interval_tree_iter_first(&ctx
->mem_copy
, s
, l
);
177 return r
? container_of(r
, MemCopyInfo
, itree
) : NULL
;
180 static MemCopyInfo
*mem_copy_next(MemCopyInfo
*mem
, intptr_t s
, intptr_t l
)
182 IntervalTreeNode
*r
= interval_tree_iter_next(&mem
->itree
, s
, l
);
183 return r
? container_of(r
, MemCopyInfo
, itree
) : NULL
;
186 static void remove_mem_copy(OptContext
*ctx
, MemCopyInfo
*mc
)
188 TCGTemp
*ts
= mc
->ts
;
189 TempOptInfo
*ti
= ts_info(ts
);
191 interval_tree_remove(&mc
->itree
, &ctx
->mem_copy
);
192 QSIMPLEQ_REMOVE(&ti
->mem_copy
, mc
, MemCopyInfo
, next
);
193 QSIMPLEQ_INSERT_TAIL(&ctx
->mem_free
, mc
, next
);
196 static void remove_mem_copy_in(OptContext
*ctx
, intptr_t s
, intptr_t l
)
199 MemCopyInfo
*mc
= mem_copy_first(ctx
, s
, l
);
203 remove_mem_copy(ctx
, mc
);
207 static void remove_mem_copy_all(OptContext
*ctx
)
209 remove_mem_copy_in(ctx
, 0, -1);
210 tcg_debug_assert(interval_tree_is_empty(&ctx
->mem_copy
));
213 static TCGTemp
*find_better_copy(TCGTemp
*ts
)
217 /* If this is already readonly, we can't do better. */
218 if (temp_readonly(ts
)) {
223 for (i
= ts_info(ts
)->next_copy
; i
!= ts
; i
= ts_info(i
)->next_copy
) {
224 ret
= cmp_better_copy(ret
, i
);
229 static void move_mem_copies(TCGTemp
*dst_ts
, TCGTemp
*src_ts
)
231 TempOptInfo
*si
= ts_info(src_ts
);
232 TempOptInfo
*di
= ts_info(dst_ts
);
235 QSIMPLEQ_FOREACH(mc
, &si
->mem_copy
, next
) {
236 tcg_debug_assert(mc
->ts
== src_ts
);
239 QSIMPLEQ_CONCAT(&di
->mem_copy
, &si
->mem_copy
);
242 /* Reset TEMP's state, possibly removing the temp for the list of copies. */
243 static void reset_ts(OptContext
*ctx
, TCGTemp
*ts
)
245 TempOptInfo
*ti
= ts_info(ts
);
246 TCGTemp
*pts
= ti
->prev_copy
;
247 TCGTemp
*nts
= ti
->next_copy
;
248 TempOptInfo
*pi
= ts_info(pts
);
249 TempOptInfo
*ni
= ts_info(nts
);
251 ni
->prev_copy
= ti
->prev_copy
;
252 pi
->next_copy
= ti
->next_copy
;
255 ti
->is_const
= false;
259 if (!QSIMPLEQ_EMPTY(&ti
->mem_copy
)) {
261 /* Last temp copy being removed, the mem copies die. */
263 QSIMPLEQ_FOREACH(mc
, &ti
->mem_copy
, next
) {
264 interval_tree_remove(&mc
->itree
, &ctx
->mem_copy
);
266 QSIMPLEQ_CONCAT(&ctx
->mem_free
, &ti
->mem_copy
);
268 move_mem_copies(find_better_copy(nts
), ts
);
273 static void reset_temp(OptContext
*ctx
, TCGArg arg
)
275 reset_ts(ctx
, arg_temp(arg
));
278 static void record_mem_copy(OptContext
*ctx
, TCGType type
,
279 TCGTemp
*ts
, intptr_t start
, intptr_t last
)
284 mc
= QSIMPLEQ_FIRST(&ctx
->mem_free
);
286 QSIMPLEQ_REMOVE_HEAD(&ctx
->mem_free
, next
);
288 mc
= tcg_malloc(sizeof(*mc
));
291 memset(mc
, 0, sizeof(*mc
));
292 mc
->itree
.start
= start
;
293 mc
->itree
.last
= last
;
295 interval_tree_insert(&mc
->itree
, &ctx
->mem_copy
);
297 ts
= find_better_copy(ts
);
300 QSIMPLEQ_INSERT_TAIL(&ti
->mem_copy
, mc
, next
);
303 static bool ts_are_copies(TCGTemp
*ts1
, TCGTemp
*ts2
)
311 if (!ts_is_copy(ts1
) || !ts_is_copy(ts2
)) {
315 for (i
= ts_info(ts1
)->next_copy
; i
!= ts1
; i
= ts_info(i
)->next_copy
) {
324 static bool args_are_copies(TCGArg arg1
, TCGArg arg2
)
326 return ts_are_copies(arg_temp(arg1
), arg_temp(arg2
));
329 static TCGTemp
*find_mem_copy_for(OptContext
*ctx
, TCGType type
, intptr_t s
)
333 for (mc
= mem_copy_first(ctx
, s
, s
); mc
; mc
= mem_copy_next(mc
, s
, s
)) {
334 if (mc
->itree
.start
== s
&& mc
->type
== type
) {
335 return find_better_copy(mc
->ts
);
341 static TCGArg
arg_new_constant(OptContext
*ctx
, uint64_t val
)
343 TCGType type
= ctx
->type
;
346 if (type
== TCG_TYPE_I32
) {
350 ts
= tcg_constant_internal(type
, val
);
351 init_ts_info(ctx
, ts
);
356 static bool tcg_opt_gen_mov(OptContext
*ctx
, TCGOp
*op
, TCGArg dst
, TCGArg src
)
358 TCGTemp
*dst_ts
= arg_temp(dst
);
359 TCGTemp
*src_ts
= arg_temp(src
);
364 if (ts_are_copies(dst_ts
, src_ts
)) {
365 tcg_op_remove(ctx
->tcg
, op
);
369 reset_ts(ctx
, dst_ts
);
370 di
= ts_info(dst_ts
);
371 si
= ts_info(src_ts
);
375 new_op
= INDEX_op_mov_i32
;
378 new_op
= INDEX_op_mov_i64
;
383 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
384 new_op
= INDEX_op_mov_vec
;
387 g_assert_not_reached();
393 di
->z_mask
= si
->z_mask
;
394 di
->s_mask
= si
->s_mask
;
396 if (src_ts
->type
== dst_ts
->type
) {
397 TempOptInfo
*ni
= ts_info(si
->next_copy
);
399 di
->next_copy
= si
->next_copy
;
400 di
->prev_copy
= src_ts
;
401 ni
->prev_copy
= dst_ts
;
402 si
->next_copy
= dst_ts
;
403 di
->is_const
= si
->is_const
;
406 if (!QSIMPLEQ_EMPTY(&si
->mem_copy
)
407 && cmp_better_copy(src_ts
, dst_ts
) == dst_ts
) {
408 move_mem_copies(dst_ts
, src_ts
);
414 static bool tcg_opt_gen_movi(OptContext
*ctx
, TCGOp
*op
,
415 TCGArg dst
, uint64_t val
)
417 /* Convert movi to mov with constant temp. */
418 return tcg_opt_gen_mov(ctx
, op
, dst
, arg_new_constant(ctx
, val
));
421 static uint64_t do_constant_folding_2(TCGOpcode op
, uint64_t x
, uint64_t y
)
435 CASE_OP_32_64_VEC(and):
438 CASE_OP_32_64_VEC(or):
441 CASE_OP_32_64_VEC(xor):
444 case INDEX_op_shl_i32
:
445 return (uint32_t)x
<< (y
& 31);
447 case INDEX_op_shl_i64
:
448 return (uint64_t)x
<< (y
& 63);
450 case INDEX_op_shr_i32
:
451 return (uint32_t)x
>> (y
& 31);
453 case INDEX_op_shr_i64
:
454 return (uint64_t)x
>> (y
& 63);
456 case INDEX_op_sar_i32
:
457 return (int32_t)x
>> (y
& 31);
459 case INDEX_op_sar_i64
:
460 return (int64_t)x
>> (y
& 63);
462 case INDEX_op_rotr_i32
:
463 return ror32(x
, y
& 31);
465 case INDEX_op_rotr_i64
:
466 return ror64(x
, y
& 63);
468 case INDEX_op_rotl_i32
:
469 return rol32(x
, y
& 31);
471 case INDEX_op_rotl_i64
:
472 return rol64(x
, y
& 63);
474 CASE_OP_32_64_VEC(not):
480 CASE_OP_32_64_VEC(andc
):
483 CASE_OP_32_64_VEC(orc
):
486 CASE_OP_32_64_VEC(eqv
):
489 CASE_OP_32_64_VEC(nand
):
492 CASE_OP_32_64_VEC(nor
):
495 case INDEX_op_clz_i32
:
496 return (uint32_t)x
? clz32(x
) : y
;
498 case INDEX_op_clz_i64
:
499 return x
? clz64(x
) : y
;
501 case INDEX_op_ctz_i32
:
502 return (uint32_t)x
? ctz32(x
) : y
;
504 case INDEX_op_ctz_i64
:
505 return x
? ctz64(x
) : y
;
507 case INDEX_op_ctpop_i32
:
510 case INDEX_op_ctpop_i64
:
513 CASE_OP_32_64(ext8s
):
516 CASE_OP_32_64(ext16s
):
519 CASE_OP_32_64(ext8u
):
522 CASE_OP_32_64(ext16u
):
525 CASE_OP_32_64(bswap16
):
527 return y
& TCG_BSWAP_OS
? (int16_t)x
: x
;
529 CASE_OP_32_64(bswap32
):
531 return y
& TCG_BSWAP_OS
? (int32_t)x
: x
;
533 case INDEX_op_bswap64_i64
:
536 case INDEX_op_ext_i32_i64
:
537 case INDEX_op_ext32s_i64
:
540 case INDEX_op_extu_i32_i64
:
541 case INDEX_op_extrl_i64_i32
:
542 case INDEX_op_ext32u_i64
:
545 case INDEX_op_extrh_i64_i32
:
546 return (uint64_t)x
>> 32;
548 case INDEX_op_muluh_i32
:
549 return ((uint64_t)(uint32_t)x
* (uint32_t)y
) >> 32;
550 case INDEX_op_mulsh_i32
:
551 return ((int64_t)(int32_t)x
* (int32_t)y
) >> 32;
553 case INDEX_op_muluh_i64
:
554 mulu64(&l64
, &h64
, x
, y
);
556 case INDEX_op_mulsh_i64
:
557 muls64(&l64
, &h64
, x
, y
);
560 case INDEX_op_div_i32
:
561 /* Avoid crashing on divide by zero, otherwise undefined. */
562 return (int32_t)x
/ ((int32_t)y
? : 1);
563 case INDEX_op_divu_i32
:
564 return (uint32_t)x
/ ((uint32_t)y
? : 1);
565 case INDEX_op_div_i64
:
566 return (int64_t)x
/ ((int64_t)y
? : 1);
567 case INDEX_op_divu_i64
:
568 return (uint64_t)x
/ ((uint64_t)y
? : 1);
570 case INDEX_op_rem_i32
:
571 return (int32_t)x
% ((int32_t)y
? : 1);
572 case INDEX_op_remu_i32
:
573 return (uint32_t)x
% ((uint32_t)y
? : 1);
574 case INDEX_op_rem_i64
:
575 return (int64_t)x
% ((int64_t)y
? : 1);
576 case INDEX_op_remu_i64
:
577 return (uint64_t)x
% ((uint64_t)y
? : 1);
580 g_assert_not_reached();
584 static uint64_t do_constant_folding(TCGOpcode op
, TCGType type
,
585 uint64_t x
, uint64_t y
)
587 uint64_t res
= do_constant_folding_2(op
, x
, y
);
588 if (type
== TCG_TYPE_I32
) {
594 static bool do_constant_folding_cond_32(uint32_t x
, uint32_t y
, TCGCond c
)
602 return (int32_t)x
< (int32_t)y
;
604 return (int32_t)x
>= (int32_t)y
;
606 return (int32_t)x
<= (int32_t)y
;
608 return (int32_t)x
> (int32_t)y
;
618 g_assert_not_reached();
622 static bool do_constant_folding_cond_64(uint64_t x
, uint64_t y
, TCGCond c
)
630 return (int64_t)x
< (int64_t)y
;
632 return (int64_t)x
>= (int64_t)y
;
634 return (int64_t)x
<= (int64_t)y
;
636 return (int64_t)x
> (int64_t)y
;
646 g_assert_not_reached();
650 static bool do_constant_folding_cond_eq(TCGCond c
)
666 g_assert_not_reached();
671 * Return -1 if the condition can't be simplified,
672 * and the result of the condition (0 or 1) if it can.
674 static int do_constant_folding_cond(TCGType type
, TCGArg x
,
677 if (arg_is_const(x
) && arg_is_const(y
)) {
678 uint64_t xv
= arg_info(x
)->val
;
679 uint64_t yv
= arg_info(y
)->val
;
683 return do_constant_folding_cond_32(xv
, yv
, c
);
685 return do_constant_folding_cond_64(xv
, yv
, c
);
687 /* Only scalar comparisons are optimizable */
690 } else if (args_are_copies(x
, y
)) {
691 return do_constant_folding_cond_eq(c
);
692 } else if (arg_is_const(y
) && arg_info(y
)->val
== 0) {
706 * Return -1 if the condition can't be simplified,
707 * and the result of the condition (0 or 1) if it can.
709 static int do_constant_folding_cond2(TCGArg
*p1
, TCGArg
*p2
, TCGCond c
)
711 TCGArg al
= p1
[0], ah
= p1
[1];
712 TCGArg bl
= p2
[0], bh
= p2
[1];
714 if (arg_is_const(bl
) && arg_is_const(bh
)) {
715 tcg_target_ulong blv
= arg_info(bl
)->val
;
716 tcg_target_ulong bhv
= arg_info(bh
)->val
;
717 uint64_t b
= deposit64(blv
, 32, 32, bhv
);
719 if (arg_is_const(al
) && arg_is_const(ah
)) {
720 tcg_target_ulong alv
= arg_info(al
)->val
;
721 tcg_target_ulong ahv
= arg_info(ah
)->val
;
722 uint64_t a
= deposit64(alv
, 32, 32, ahv
);
723 return do_constant_folding_cond_64(a
, b
, c
);
736 if (args_are_copies(al
, bl
) && args_are_copies(ah
, bh
)) {
737 return do_constant_folding_cond_eq(c
);
744 * @dest: TCGArg of the destination argument, or NO_DEST.
745 * @p1: first paired argument
746 * @p2: second paired argument
748 * If *@p1 is a constant and *@p2 is not, swap.
749 * If *@p2 matches @dest, swap.
750 * Return true if a swap was performed.
753 #define NO_DEST temp_arg(NULL)
755 static bool swap_commutative(TCGArg dest
, TCGArg
*p1
, TCGArg
*p2
)
757 TCGArg a1
= *p1
, a2
= *p2
;
759 sum
+= arg_is_const(a1
);
760 sum
-= arg_is_const(a2
);
762 /* Prefer the constant in second argument, and then the form
763 op a, a, b, which is better handled on non-RISC hosts. */
764 if (sum
> 0 || (sum
== 0 && dest
== a2
)) {
772 static bool swap_commutative2(TCGArg
*p1
, TCGArg
*p2
)
775 sum
+= arg_is_const(p1
[0]);
776 sum
+= arg_is_const(p1
[1]);
777 sum
-= arg_is_const(p2
[0]);
778 sum
-= arg_is_const(p2
[1]);
781 t
= p1
[0], p1
[0] = p2
[0], p2
[0] = t
;
782 t
= p1
[1], p1
[1] = p2
[1], p2
[1] = t
;
788 static void init_arguments(OptContext
*ctx
, TCGOp
*op
, int nb_args
)
790 for (int i
= 0; i
< nb_args
; i
++) {
791 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
792 init_ts_info(ctx
, ts
);
796 static void copy_propagate(OptContext
*ctx
, TCGOp
*op
,
797 int nb_oargs
, int nb_iargs
)
799 for (int i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
800 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
801 if (ts_is_copy(ts
)) {
802 op
->args
[i
] = temp_arg(find_better_copy(ts
));
807 static void finish_folding(OptContext
*ctx
, TCGOp
*op
)
809 const TCGOpDef
*def
= &tcg_op_defs
[op
->opc
];
813 * We only optimize extended basic blocks. If the opcode ends a BB
814 * and is not a conditional branch, reset all temp data.
816 if (def
->flags
& TCG_OPF_BB_END
) {
818 if (!(def
->flags
& TCG_OPF_COND_BRANCH
)) {
819 memset(&ctx
->temps_used
, 0, sizeof(ctx
->temps_used
));
820 remove_mem_copy_all(ctx
);
825 nb_oargs
= def
->nb_oargs
;
826 for (i
= 0; i
< nb_oargs
; i
++) {
827 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
830 * Save the corresponding known-zero/sign bits mask for the
831 * first output argument (only one supported so far).
834 ts_info(ts
)->z_mask
= ctx
->z_mask
;
835 ts_info(ts
)->s_mask
= ctx
->s_mask
;
841 * The fold_* functions return true when processing is complete,
842 * usually by folding the operation to a constant or to a copy,
843 * and calling tcg_opt_gen_{mov,movi}. They may do other things,
844 * like collect information about the value produced, for use in
845 * optimizing a subsequent operation.
847 * These first fold_* functions are all helpers, used by other
848 * folders for more specific operations.
851 static bool fold_const1(OptContext
*ctx
, TCGOp
*op
)
853 if (arg_is_const(op
->args
[1])) {
856 t
= arg_info(op
->args
[1])->val
;
857 t
= do_constant_folding(op
->opc
, ctx
->type
, t
, 0);
858 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
863 static bool fold_const2(OptContext
*ctx
, TCGOp
*op
)
865 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
866 uint64_t t1
= arg_info(op
->args
[1])->val
;
867 uint64_t t2
= arg_info(op
->args
[2])->val
;
869 t1
= do_constant_folding(op
->opc
, ctx
->type
, t1
, t2
);
870 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t1
);
875 static bool fold_commutative(OptContext
*ctx
, TCGOp
*op
)
877 swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2]);
881 static bool fold_const2_commutative(OptContext
*ctx
, TCGOp
*op
)
883 swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2]);
884 return fold_const2(ctx
, op
);
887 static bool fold_masks(OptContext
*ctx
, TCGOp
*op
)
889 uint64_t a_mask
= ctx
->a_mask
;
890 uint64_t z_mask
= ctx
->z_mask
;
891 uint64_t s_mask
= ctx
->s_mask
;
894 * 32-bit ops generate 32-bit results, which for the purpose of
895 * simplifying tcg are sign-extended. Certainly that's how we
896 * represent our constants elsewhere. Note that the bits will
897 * be reset properly for a 64-bit value when encountering the
898 * type changing opcodes.
900 if (ctx
->type
== TCG_TYPE_I32
) {
901 a_mask
= (int32_t)a_mask
;
902 z_mask
= (int32_t)z_mask
;
903 s_mask
|= MAKE_64BIT_MASK(32, 32);
904 ctx
->z_mask
= z_mask
;
905 ctx
->s_mask
= s_mask
;
909 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], 0);
912 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[1]);
918 * Convert @op to NOT, if NOT is supported by the host.
919 * Return true f the conversion is successful, which will still
920 * indicate that the processing is complete.
922 static bool fold_not(OptContext
*ctx
, TCGOp
*op
);
923 static bool fold_to_not(OptContext
*ctx
, TCGOp
*op
, int idx
)
930 not_op
= INDEX_op_not_i32
;
931 have_not
= TCG_TARGET_HAS_not_i32
;
934 not_op
= INDEX_op_not_i64
;
935 have_not
= TCG_TARGET_HAS_not_i64
;
940 not_op
= INDEX_op_not_vec
;
941 have_not
= TCG_TARGET_HAS_not_vec
;
944 g_assert_not_reached();
948 op
->args
[1] = op
->args
[idx
];
949 return fold_not(ctx
, op
);
954 /* If the binary operation has first argument @i, fold to @i. */
955 static bool fold_ix_to_i(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
957 if (arg_is_const(op
->args
[1]) && arg_info(op
->args
[1])->val
== i
) {
958 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], i
);
963 /* If the binary operation has first argument @i, fold to NOT. */
964 static bool fold_ix_to_not(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
966 if (arg_is_const(op
->args
[1]) && arg_info(op
->args
[1])->val
== i
) {
967 return fold_to_not(ctx
, op
, 2);
972 /* If the binary operation has second argument @i, fold to @i. */
973 static bool fold_xi_to_i(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
975 if (arg_is_const(op
->args
[2]) && arg_info(op
->args
[2])->val
== i
) {
976 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], i
);
981 /* If the binary operation has second argument @i, fold to identity. */
982 static bool fold_xi_to_x(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
984 if (arg_is_const(op
->args
[2]) && arg_info(op
->args
[2])->val
== i
) {
985 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[1]);
990 /* If the binary operation has second argument @i, fold to NOT. */
991 static bool fold_xi_to_not(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
993 if (arg_is_const(op
->args
[2]) && arg_info(op
->args
[2])->val
== i
) {
994 return fold_to_not(ctx
, op
, 1);
999 /* If the binary operation has both arguments equal, fold to @i. */
1000 static bool fold_xx_to_i(OptContext
*ctx
, TCGOp
*op
, uint64_t i
)
1002 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1003 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], i
);
1008 /* If the binary operation has both arguments equal, fold to identity. */
1009 static bool fold_xx_to_x(OptContext
*ctx
, TCGOp
*op
)
1011 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1012 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[1]);
1018 * These outermost fold_<op> functions are sorted alphabetically.
1020 * The ordering of the transformations should be:
1021 * 1) those that produce a constant
1022 * 2) those that produce a copy
1023 * 3) those that produce information about the result value.
1026 static bool fold_add(OptContext
*ctx
, TCGOp
*op
)
1028 if (fold_const2_commutative(ctx
, op
) ||
1029 fold_xi_to_x(ctx
, op
, 0)) {
1035 /* We cannot as yet do_constant_folding with vectors. */
1036 static bool fold_add_vec(OptContext
*ctx
, TCGOp
*op
)
1038 if (fold_commutative(ctx
, op
) ||
1039 fold_xi_to_x(ctx
, op
, 0)) {
1045 static bool fold_addsub2(OptContext
*ctx
, TCGOp
*op
, bool add
)
1047 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3]) &&
1048 arg_is_const(op
->args
[4]) && arg_is_const(op
->args
[5])) {
1049 uint64_t al
= arg_info(op
->args
[2])->val
;
1050 uint64_t ah
= arg_info(op
->args
[3])->val
;
1051 uint64_t bl
= arg_info(op
->args
[4])->val
;
1052 uint64_t bh
= arg_info(op
->args
[5])->val
;
1056 if (ctx
->type
== TCG_TYPE_I32
) {
1057 uint64_t a
= deposit64(al
, 32, 32, ah
);
1058 uint64_t b
= deposit64(bl
, 32, 32, bh
);
1066 al
= sextract64(a
, 0, 32);
1067 ah
= sextract64(a
, 32, 32);
1069 Int128 a
= int128_make128(al
, ah
);
1070 Int128 b
= int128_make128(bl
, bh
);
1073 a
= int128_add(a
, b
);
1075 a
= int128_sub(a
, b
);
1078 al
= int128_getlo(a
);
1079 ah
= int128_gethi(a
);
1085 /* The proper opcode is supplied by tcg_opt_gen_mov. */
1086 op2
= tcg_op_insert_before(ctx
->tcg
, op
, 0, 2);
1088 tcg_opt_gen_movi(ctx
, op
, rl
, al
);
1089 tcg_opt_gen_movi(ctx
, op2
, rh
, ah
);
1095 static bool fold_add2(OptContext
*ctx
, TCGOp
*op
)
1097 /* Note that the high and low parts may be independently swapped. */
1098 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[4]);
1099 swap_commutative(op
->args
[1], &op
->args
[3], &op
->args
[5]);
1101 return fold_addsub2(ctx
, op
, true);
1104 static bool fold_and(OptContext
*ctx
, TCGOp
*op
)
1108 if (fold_const2_commutative(ctx
, op
) ||
1109 fold_xi_to_i(ctx
, op
, 0) ||
1110 fold_xi_to_x(ctx
, op
, -1) ||
1111 fold_xx_to_x(ctx
, op
)) {
1115 z1
= arg_info(op
->args
[1])->z_mask
;
1116 z2
= arg_info(op
->args
[2])->z_mask
;
1117 ctx
->z_mask
= z1
& z2
;
1120 * Sign repetitions are perforce all identical, whether they are 1 or 0.
1121 * Bitwise operations preserve the relative quantity of the repetitions.
1123 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1124 & arg_info(op
->args
[2])->s_mask
;
1127 * Known-zeros does not imply known-ones. Therefore unless
1128 * arg2 is constant, we can't infer affected bits from it.
1130 if (arg_is_const(op
->args
[2])) {
1131 ctx
->a_mask
= z1
& ~z2
;
1134 return fold_masks(ctx
, op
);
1137 static bool fold_andc(OptContext
*ctx
, TCGOp
*op
)
1141 if (fold_const2(ctx
, op
) ||
1142 fold_xx_to_i(ctx
, op
, 0) ||
1143 fold_xi_to_x(ctx
, op
, 0) ||
1144 fold_ix_to_not(ctx
, op
, -1)) {
1148 z1
= arg_info(op
->args
[1])->z_mask
;
1151 * Known-zeros does not imply known-ones. Therefore unless
1152 * arg2 is constant, we can't infer anything from it.
1154 if (arg_is_const(op
->args
[2])) {
1155 uint64_t z2
= ~arg_info(op
->args
[2])->z_mask
;
1156 ctx
->a_mask
= z1
& ~z2
;
1161 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1162 & arg_info(op
->args
[2])->s_mask
;
1163 return fold_masks(ctx
, op
);
1166 static bool fold_brcond(OptContext
*ctx
, TCGOp
*op
)
1168 TCGCond cond
= op
->args
[2];
1171 if (swap_commutative(NO_DEST
, &op
->args
[0], &op
->args
[1])) {
1172 op
->args
[2] = cond
= tcg_swap_cond(cond
);
1175 i
= do_constant_folding_cond(ctx
->type
, op
->args
[0], op
->args
[1], cond
);
1177 tcg_op_remove(ctx
->tcg
, op
);
1181 op
->opc
= INDEX_op_br
;
1182 op
->args
[0] = op
->args
[3];
1187 static bool fold_brcond2(OptContext
*ctx
, TCGOp
*op
)
1189 TCGCond cond
= op
->args
[4];
1190 TCGArg label
= op
->args
[5];
1193 if (swap_commutative2(&op
->args
[0], &op
->args
[2])) {
1194 op
->args
[4] = cond
= tcg_swap_cond(cond
);
1197 i
= do_constant_folding_cond2(&op
->args
[0], &op
->args
[2], cond
);
1199 goto do_brcond_const
;
1206 * Simplify LT/GE comparisons vs zero to a single compare
1207 * vs the high word of the input.
1209 if (arg_is_const(op
->args
[2]) && arg_info(op
->args
[2])->val
== 0 &&
1210 arg_is_const(op
->args
[3]) && arg_info(op
->args
[3])->val
== 0) {
1211 goto do_brcond_high
;
1220 * Simplify EQ/NE comparisons where one of the pairs
1221 * can be simplified.
1223 i
= do_constant_folding_cond(TCG_TYPE_I32
, op
->args
[0],
1227 goto do_brcond_const
;
1229 goto do_brcond_high
;
1232 i
= do_constant_folding_cond(TCG_TYPE_I32
, op
->args
[1],
1236 goto do_brcond_const
;
1238 op
->opc
= INDEX_op_brcond_i32
;
1239 op
->args
[1] = op
->args
[2];
1241 op
->args
[3] = label
;
1250 op
->opc
= INDEX_op_brcond_i32
;
1251 op
->args
[0] = op
->args
[1];
1252 op
->args
[1] = op
->args
[3];
1254 op
->args
[3] = label
;
1259 tcg_op_remove(ctx
->tcg
, op
);
1262 op
->opc
= INDEX_op_br
;
1263 op
->args
[0] = label
;
1269 static bool fold_bswap(OptContext
*ctx
, TCGOp
*op
)
1271 uint64_t z_mask
, s_mask
, sign
;
1273 if (arg_is_const(op
->args
[1])) {
1274 uint64_t t
= arg_info(op
->args
[1])->val
;
1276 t
= do_constant_folding(op
->opc
, ctx
->type
, t
, op
->args
[2]);
1277 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
1280 z_mask
= arg_info(op
->args
[1])->z_mask
;
1283 case INDEX_op_bswap16_i32
:
1284 case INDEX_op_bswap16_i64
:
1285 z_mask
= bswap16(z_mask
);
1288 case INDEX_op_bswap32_i32
:
1289 case INDEX_op_bswap32_i64
:
1290 z_mask
= bswap32(z_mask
);
1293 case INDEX_op_bswap64_i64
:
1294 z_mask
= bswap64(z_mask
);
1298 g_assert_not_reached();
1300 s_mask
= smask_from_zmask(z_mask
);
1302 switch (op
->args
[2] & (TCG_BSWAP_OZ
| TCG_BSWAP_OS
)) {
1306 /* If the sign bit may be 1, force all the bits above to 1. */
1307 if (z_mask
& sign
) {
1313 /* The high bits are undefined: force all bits above the sign to 1. */
1314 z_mask
|= sign
<< 1;
1318 ctx
->z_mask
= z_mask
;
1319 ctx
->s_mask
= s_mask
;
1321 return fold_masks(ctx
, op
);
1324 static bool fold_call(OptContext
*ctx
, TCGOp
*op
)
1326 TCGContext
*s
= ctx
->tcg
;
1327 int nb_oargs
= TCGOP_CALLO(op
);
1328 int nb_iargs
= TCGOP_CALLI(op
);
1331 init_arguments(ctx
, op
, nb_oargs
+ nb_iargs
);
1332 copy_propagate(ctx
, op
, nb_oargs
, nb_iargs
);
1334 /* If the function reads or writes globals, reset temp data. */
1335 flags
= tcg_call_flags(op
);
1336 if (!(flags
& (TCG_CALL_NO_READ_GLOBALS
| TCG_CALL_NO_WRITE_GLOBALS
))) {
1337 int nb_globals
= s
->nb_globals
;
1339 for (i
= 0; i
< nb_globals
; i
++) {
1340 if (test_bit(i
, ctx
->temps_used
.l
)) {
1341 reset_ts(ctx
, &ctx
->tcg
->temps
[i
]);
1346 /* If the function has side effects, reset mem data. */
1347 if (!(flags
& TCG_CALL_NO_SIDE_EFFECTS
)) {
1348 remove_mem_copy_all(ctx
);
1351 /* Reset temp data for outputs. */
1352 for (i
= 0; i
< nb_oargs
; i
++) {
1353 reset_temp(ctx
, op
->args
[i
]);
1356 /* Stop optimizing MB across calls. */
1357 ctx
->prev_mb
= NULL
;
1361 static bool fold_count_zeros(OptContext
*ctx
, TCGOp
*op
)
1365 if (arg_is_const(op
->args
[1])) {
1366 uint64_t t
= arg_info(op
->args
[1])->val
;
1369 t
= do_constant_folding(op
->opc
, ctx
->type
, t
, 0);
1370 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
1372 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[2]);
1375 switch (ctx
->type
) {
1383 g_assert_not_reached();
1385 ctx
->z_mask
= arg_info(op
->args
[2])->z_mask
| z_mask
;
1386 ctx
->s_mask
= smask_from_zmask(ctx
->z_mask
);
1390 static bool fold_ctpop(OptContext
*ctx
, TCGOp
*op
)
1392 if (fold_const1(ctx
, op
)) {
1396 switch (ctx
->type
) {
1398 ctx
->z_mask
= 32 | 31;
1401 ctx
->z_mask
= 64 | 63;
1404 g_assert_not_reached();
1406 ctx
->s_mask
= smask_from_zmask(ctx
->z_mask
);
1410 static bool fold_deposit(OptContext
*ctx
, TCGOp
*op
)
1414 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1415 uint64_t t1
= arg_info(op
->args
[1])->val
;
1416 uint64_t t2
= arg_info(op
->args
[2])->val
;
1418 t1
= deposit64(t1
, op
->args
[3], op
->args
[4], t2
);
1419 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t1
);
1422 switch (ctx
->type
) {
1424 and_opc
= INDEX_op_and_i32
;
1427 and_opc
= INDEX_op_and_i64
;
1430 g_assert_not_reached();
1433 /* Inserting a value into zero at offset 0. */
1434 if (arg_is_const(op
->args
[1])
1435 && arg_info(op
->args
[1])->val
== 0
1436 && op
->args
[3] == 0) {
1437 uint64_t mask
= MAKE_64BIT_MASK(0, op
->args
[4]);
1440 op
->args
[1] = op
->args
[2];
1441 op
->args
[2] = arg_new_constant(ctx
, mask
);
1442 ctx
->z_mask
= mask
& arg_info(op
->args
[1])->z_mask
;
1446 /* Inserting zero into a value. */
1447 if (arg_is_const(op
->args
[2])
1448 && arg_info(op
->args
[2])->val
== 0) {
1449 uint64_t mask
= deposit64(-1, op
->args
[3], op
->args
[4], 0);
1452 op
->args
[2] = arg_new_constant(ctx
, mask
);
1453 ctx
->z_mask
= mask
& arg_info(op
->args
[1])->z_mask
;
1457 ctx
->z_mask
= deposit64(arg_info(op
->args
[1])->z_mask
,
1458 op
->args
[3], op
->args
[4],
1459 arg_info(op
->args
[2])->z_mask
);
1463 static bool fold_divide(OptContext
*ctx
, TCGOp
*op
)
1465 if (fold_const2(ctx
, op
) ||
1466 fold_xi_to_x(ctx
, op
, 1)) {
1472 static bool fold_dup(OptContext
*ctx
, TCGOp
*op
)
1474 if (arg_is_const(op
->args
[1])) {
1475 uint64_t t
= arg_info(op
->args
[1])->val
;
1476 t
= dup_const(TCGOP_VECE(op
), t
);
1477 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
1482 static bool fold_dup2(OptContext
*ctx
, TCGOp
*op
)
1484 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1485 uint64_t t
= deposit64(arg_info(op
->args
[1])->val
, 32, 32,
1486 arg_info(op
->args
[2])->val
);
1487 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
1490 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1491 op
->opc
= INDEX_op_dup_vec
;
1492 TCGOP_VECE(op
) = MO_32
;
1497 static bool fold_eqv(OptContext
*ctx
, TCGOp
*op
)
1499 if (fold_const2_commutative(ctx
, op
) ||
1500 fold_xi_to_x(ctx
, op
, -1) ||
1501 fold_xi_to_not(ctx
, op
, 0)) {
1505 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1506 & arg_info(op
->args
[2])->s_mask
;
1510 static bool fold_extract(OptContext
*ctx
, TCGOp
*op
)
1512 uint64_t z_mask_old
, z_mask
;
1513 int pos
= op
->args
[2];
1514 int len
= op
->args
[3];
1516 if (arg_is_const(op
->args
[1])) {
1519 t
= arg_info(op
->args
[1])->val
;
1520 t
= extract64(t
, pos
, len
);
1521 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
1524 z_mask_old
= arg_info(op
->args
[1])->z_mask
;
1525 z_mask
= extract64(z_mask_old
, pos
, len
);
1527 ctx
->a_mask
= z_mask_old
^ z_mask
;
1529 ctx
->z_mask
= z_mask
;
1530 ctx
->s_mask
= smask_from_zmask(z_mask
);
1532 return fold_masks(ctx
, op
);
1535 static bool fold_extract2(OptContext
*ctx
, TCGOp
*op
)
1537 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1538 uint64_t v1
= arg_info(op
->args
[1])->val
;
1539 uint64_t v2
= arg_info(op
->args
[2])->val
;
1540 int shr
= op
->args
[3];
1542 if (op
->opc
== INDEX_op_extract2_i64
) {
1546 v1
= (uint32_t)v1
>> shr
;
1547 v2
= (uint64_t)((int32_t)v2
<< (32 - shr
));
1549 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], v1
| v2
);
1554 static bool fold_exts(OptContext
*ctx
, TCGOp
*op
)
1556 uint64_t s_mask_old
, s_mask
, z_mask
, sign
;
1557 bool type_change
= false;
1559 if (fold_const1(ctx
, op
)) {
1563 z_mask
= arg_info(op
->args
[1])->z_mask
;
1564 s_mask
= arg_info(op
->args
[1])->s_mask
;
1565 s_mask_old
= s_mask
;
1568 CASE_OP_32_64(ext8s
):
1570 z_mask
= (uint8_t)z_mask
;
1572 CASE_OP_32_64(ext16s
):
1574 z_mask
= (uint16_t)z_mask
;
1576 case INDEX_op_ext_i32_i64
:
1579 case INDEX_op_ext32s_i64
:
1581 z_mask
= (uint32_t)z_mask
;
1584 g_assert_not_reached();
1587 if (z_mask
& sign
) {
1590 s_mask
|= sign
<< 1;
1592 ctx
->z_mask
= z_mask
;
1593 ctx
->s_mask
= s_mask
;
1595 ctx
->a_mask
= s_mask
& ~s_mask_old
;
1598 return fold_masks(ctx
, op
);
1601 static bool fold_extu(OptContext
*ctx
, TCGOp
*op
)
1603 uint64_t z_mask_old
, z_mask
;
1604 bool type_change
= false;
1606 if (fold_const1(ctx
, op
)) {
1610 z_mask_old
= z_mask
= arg_info(op
->args
[1])->z_mask
;
1613 CASE_OP_32_64(ext8u
):
1614 z_mask
= (uint8_t)z_mask
;
1616 CASE_OP_32_64(ext16u
):
1617 z_mask
= (uint16_t)z_mask
;
1619 case INDEX_op_extrl_i64_i32
:
1620 case INDEX_op_extu_i32_i64
:
1623 case INDEX_op_ext32u_i64
:
1624 z_mask
= (uint32_t)z_mask
;
1626 case INDEX_op_extrh_i64_i32
:
1631 g_assert_not_reached();
1634 ctx
->z_mask
= z_mask
;
1635 ctx
->s_mask
= smask_from_zmask(z_mask
);
1637 ctx
->a_mask
= z_mask_old
^ z_mask
;
1639 return fold_masks(ctx
, op
);
1642 static bool fold_mb(OptContext
*ctx
, TCGOp
*op
)
1644 /* Eliminate duplicate and redundant fence instructions. */
1647 * Merge two barriers of the same type into one,
1648 * or a weaker barrier into a stronger one,
1649 * or two weaker barriers into a stronger one.
1650 * mb X; mb Y => mb X|Y
1651 * mb; strl => mb; st
1652 * ldaq; mb => ld; mb
1653 * ldaq; strl => ld; mb; st
1654 * Other combinations are also merged into a strong
1655 * barrier. This is stricter than specified but for
1656 * the purposes of TCG is better than not optimizing.
1658 ctx
->prev_mb
->args
[0] |= op
->args
[0];
1659 tcg_op_remove(ctx
->tcg
, op
);
1666 static bool fold_mov(OptContext
*ctx
, TCGOp
*op
)
1668 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[1]);
1671 static bool fold_movcond(OptContext
*ctx
, TCGOp
*op
)
1673 TCGCond cond
= op
->args
[5];
1676 if (swap_commutative(NO_DEST
, &op
->args
[1], &op
->args
[2])) {
1677 op
->args
[5] = cond
= tcg_swap_cond(cond
);
1680 * Canonicalize the "false" input reg to match the destination reg so
1681 * that the tcg backend can implement a "move if true" operation.
1683 if (swap_commutative(op
->args
[0], &op
->args
[4], &op
->args
[3])) {
1684 op
->args
[5] = cond
= tcg_invert_cond(cond
);
1687 i
= do_constant_folding_cond(ctx
->type
, op
->args
[1], op
->args
[2], cond
);
1689 return tcg_opt_gen_mov(ctx
, op
, op
->args
[0], op
->args
[4 - i
]);
1692 ctx
->z_mask
= arg_info(op
->args
[3])->z_mask
1693 | arg_info(op
->args
[4])->z_mask
;
1694 ctx
->s_mask
= arg_info(op
->args
[3])->s_mask
1695 & arg_info(op
->args
[4])->s_mask
;
1697 if (arg_is_const(op
->args
[3]) && arg_is_const(op
->args
[4])) {
1698 uint64_t tv
= arg_info(op
->args
[3])->val
;
1699 uint64_t fv
= arg_info(op
->args
[4])->val
;
1700 TCGOpcode opc
, negopc
= 0;
1702 switch (ctx
->type
) {
1704 opc
= INDEX_op_setcond_i32
;
1705 if (TCG_TARGET_HAS_negsetcond_i32
) {
1706 negopc
= INDEX_op_negsetcond_i32
;
1712 opc
= INDEX_op_setcond_i64
;
1713 if (TCG_TARGET_HAS_negsetcond_i64
) {
1714 negopc
= INDEX_op_negsetcond_i64
;
1718 g_assert_not_reached();
1721 if (tv
== 1 && fv
== 0) {
1724 } else if (fv
== 1 && tv
== 0) {
1726 op
->args
[3] = tcg_invert_cond(cond
);
1727 } else if (negopc
) {
1728 if (tv
== -1 && fv
== 0) {
1731 } else if (fv
== -1 && tv
== 0) {
1733 op
->args
[3] = tcg_invert_cond(cond
);
1740 static bool fold_mul(OptContext
*ctx
, TCGOp
*op
)
1742 if (fold_const2(ctx
, op
) ||
1743 fold_xi_to_i(ctx
, op
, 0) ||
1744 fold_xi_to_x(ctx
, op
, 1)) {
1750 static bool fold_mul_highpart(OptContext
*ctx
, TCGOp
*op
)
1752 if (fold_const2_commutative(ctx
, op
) ||
1753 fold_xi_to_i(ctx
, op
, 0)) {
1759 static bool fold_multiply2(OptContext
*ctx
, TCGOp
*op
)
1761 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[3]);
1763 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3])) {
1764 uint64_t a
= arg_info(op
->args
[2])->val
;
1765 uint64_t b
= arg_info(op
->args
[3])->val
;
1771 case INDEX_op_mulu2_i32
:
1772 l
= (uint64_t)(uint32_t)a
* (uint32_t)b
;
1773 h
= (int32_t)(l
>> 32);
1776 case INDEX_op_muls2_i32
:
1777 l
= (int64_t)(int32_t)a
* (int32_t)b
;
1781 case INDEX_op_mulu2_i64
:
1782 mulu64(&l
, &h
, a
, b
);
1784 case INDEX_op_muls2_i64
:
1785 muls64(&l
, &h
, a
, b
);
1788 g_assert_not_reached();
1794 /* The proper opcode is supplied by tcg_opt_gen_mov. */
1795 op2
= tcg_op_insert_before(ctx
->tcg
, op
, 0, 2);
1797 tcg_opt_gen_movi(ctx
, op
, rl
, l
);
1798 tcg_opt_gen_movi(ctx
, op2
, rh
, h
);
1804 static bool fold_nand(OptContext
*ctx
, TCGOp
*op
)
1806 if (fold_const2_commutative(ctx
, op
) ||
1807 fold_xi_to_not(ctx
, op
, -1)) {
1811 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1812 & arg_info(op
->args
[2])->s_mask
;
1816 static bool fold_neg(OptContext
*ctx
, TCGOp
*op
)
1820 if (fold_const1(ctx
, op
)) {
1824 /* Set to 1 all bits to the left of the rightmost. */
1825 z_mask
= arg_info(op
->args
[1])->z_mask
;
1826 ctx
->z_mask
= -(z_mask
& -z_mask
);
1829 * Because of fold_sub_to_neg, we want to always return true,
1830 * via finish_folding.
1832 finish_folding(ctx
, op
);
1836 static bool fold_nor(OptContext
*ctx
, TCGOp
*op
)
1838 if (fold_const2_commutative(ctx
, op
) ||
1839 fold_xi_to_not(ctx
, op
, 0)) {
1843 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1844 & arg_info(op
->args
[2])->s_mask
;
1848 static bool fold_not(OptContext
*ctx
, TCGOp
*op
)
1850 if (fold_const1(ctx
, op
)) {
1854 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
;
1856 /* Because of fold_to_not, we want to always return true, via finish. */
1857 finish_folding(ctx
, op
);
1861 static bool fold_or(OptContext
*ctx
, TCGOp
*op
)
1863 if (fold_const2_commutative(ctx
, op
) ||
1864 fold_xi_to_x(ctx
, op
, 0) ||
1865 fold_xx_to_x(ctx
, op
)) {
1869 ctx
->z_mask
= arg_info(op
->args
[1])->z_mask
1870 | arg_info(op
->args
[2])->z_mask
;
1871 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1872 & arg_info(op
->args
[2])->s_mask
;
1873 return fold_masks(ctx
, op
);
1876 static bool fold_orc(OptContext
*ctx
, TCGOp
*op
)
1878 if (fold_const2(ctx
, op
) ||
1879 fold_xx_to_i(ctx
, op
, -1) ||
1880 fold_xi_to_x(ctx
, op
, -1) ||
1881 fold_ix_to_not(ctx
, op
, 0)) {
1885 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
1886 & arg_info(op
->args
[2])->s_mask
;
1890 static bool fold_qemu_ld(OptContext
*ctx
, TCGOp
*op
)
1892 const TCGOpDef
*def
= &tcg_op_defs
[op
->opc
];
1893 MemOpIdx oi
= op
->args
[def
->nb_oargs
+ def
->nb_iargs
];
1894 MemOp mop
= get_memop(oi
);
1895 int width
= 8 * memop_size(mop
);
1898 ctx
->s_mask
= MAKE_64BIT_MASK(width
, 64 - width
);
1899 if (!(mop
& MO_SIGN
)) {
1900 ctx
->z_mask
= MAKE_64BIT_MASK(0, width
);
1905 /* Opcodes that touch guest memory stop the mb optimization. */
1906 ctx
->prev_mb
= NULL
;
1910 static bool fold_qemu_st(OptContext
*ctx
, TCGOp
*op
)
1912 /* Opcodes that touch guest memory stop the mb optimization. */
1913 ctx
->prev_mb
= NULL
;
1917 static bool fold_remainder(OptContext
*ctx
, TCGOp
*op
)
1919 if (fold_const2(ctx
, op
) ||
1920 fold_xx_to_i(ctx
, op
, 0)) {
1926 static bool fold_setcond(OptContext
*ctx
, TCGOp
*op
)
1928 TCGCond cond
= op
->args
[3];
1931 if (swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2])) {
1932 op
->args
[3] = cond
= tcg_swap_cond(cond
);
1935 i
= do_constant_folding_cond(ctx
->type
, op
->args
[1], op
->args
[2], cond
);
1937 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], i
);
1941 ctx
->s_mask
= smask_from_zmask(1);
1945 static bool fold_negsetcond(OptContext
*ctx
, TCGOp
*op
)
1947 TCGCond cond
= op
->args
[3];
1950 if (swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2])) {
1951 op
->args
[3] = cond
= tcg_swap_cond(cond
);
1954 i
= do_constant_folding_cond(ctx
->type
, op
->args
[1], op
->args
[2], cond
);
1956 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], -i
);
1959 /* Value is {0,-1} so all bits are repetitions of the sign. */
1965 static bool fold_setcond2(OptContext
*ctx
, TCGOp
*op
)
1967 TCGCond cond
= op
->args
[5];
1970 if (swap_commutative2(&op
->args
[1], &op
->args
[3])) {
1971 op
->args
[5] = cond
= tcg_swap_cond(cond
);
1974 i
= do_constant_folding_cond2(&op
->args
[1], &op
->args
[3], cond
);
1976 goto do_setcond_const
;
1983 * Simplify LT/GE comparisons vs zero to a single compare
1984 * vs the high word of the input.
1986 if (arg_is_const(op
->args
[3]) && arg_info(op
->args
[3])->val
== 0 &&
1987 arg_is_const(op
->args
[4]) && arg_info(op
->args
[4])->val
== 0) {
1988 goto do_setcond_high
;
1997 * Simplify EQ/NE comparisons where one of the pairs
1998 * can be simplified.
2000 i
= do_constant_folding_cond(TCG_TYPE_I32
, op
->args
[1],
2004 goto do_setcond_const
;
2006 goto do_setcond_high
;
2009 i
= do_constant_folding_cond(TCG_TYPE_I32
, op
->args
[2],
2013 goto do_setcond_const
;
2015 op
->args
[2] = op
->args
[3];
2017 op
->opc
= INDEX_op_setcond_i32
;
2026 op
->args
[1] = op
->args
[2];
2027 op
->args
[2] = op
->args
[4];
2029 op
->opc
= INDEX_op_setcond_i32
;
2034 ctx
->s_mask
= smask_from_zmask(1);
2038 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], i
);
2041 static bool fold_sextract(OptContext
*ctx
, TCGOp
*op
)
2043 uint64_t z_mask
, s_mask
, s_mask_old
;
2044 int pos
= op
->args
[2];
2045 int len
= op
->args
[3];
2047 if (arg_is_const(op
->args
[1])) {
2050 t
= arg_info(op
->args
[1])->val
;
2051 t
= sextract64(t
, pos
, len
);
2052 return tcg_opt_gen_movi(ctx
, op
, op
->args
[0], t
);
2055 z_mask
= arg_info(op
->args
[1])->z_mask
;
2056 z_mask
= sextract64(z_mask
, pos
, len
);
2057 ctx
->z_mask
= z_mask
;
2059 s_mask_old
= arg_info(op
->args
[1])->s_mask
;
2060 s_mask
= sextract64(s_mask_old
, pos
, len
);
2061 s_mask
|= MAKE_64BIT_MASK(len
, 64 - len
);
2062 ctx
->s_mask
= s_mask
;
2065 ctx
->a_mask
= s_mask
& ~s_mask_old
;
2068 return fold_masks(ctx
, op
);
2071 static bool fold_shift(OptContext
*ctx
, TCGOp
*op
)
2073 uint64_t s_mask
, z_mask
, sign
;
2075 if (fold_const2(ctx
, op
) ||
2076 fold_ix_to_i(ctx
, op
, 0) ||
2077 fold_xi_to_x(ctx
, op
, 0)) {
2081 s_mask
= arg_info(op
->args
[1])->s_mask
;
2082 z_mask
= arg_info(op
->args
[1])->z_mask
;
2084 if (arg_is_const(op
->args
[2])) {
2085 int sh
= arg_info(op
->args
[2])->val
;
2087 ctx
->z_mask
= do_constant_folding(op
->opc
, ctx
->type
, z_mask
, sh
);
2089 s_mask
= do_constant_folding(op
->opc
, ctx
->type
, s_mask
, sh
);
2090 ctx
->s_mask
= smask_from_smask(s_mask
);
2092 return fold_masks(ctx
, op
);
2098 * Arithmetic right shift will not reduce the number of
2099 * input sign repetitions.
2101 ctx
->s_mask
= s_mask
;
2105 * If the sign bit is known zero, then logical right shift
2106 * will not reduced the number of input sign repetitions.
2108 sign
= (s_mask
& -s_mask
) >> 1;
2109 if (!(z_mask
& sign
)) {
2110 ctx
->s_mask
= s_mask
;
2120 static bool fold_sub_to_neg(OptContext
*ctx
, TCGOp
*op
)
2125 if (!arg_is_const(op
->args
[1]) || arg_info(op
->args
[1])->val
!= 0) {
2129 switch (ctx
->type
) {
2131 neg_op
= INDEX_op_neg_i32
;
2135 neg_op
= INDEX_op_neg_i64
;
2141 neg_op
= INDEX_op_neg_vec
;
2142 have_neg
= (TCG_TARGET_HAS_neg_vec
&&
2143 tcg_can_emit_vec_op(neg_op
, ctx
->type
, TCGOP_VECE(op
)) > 0);
2146 g_assert_not_reached();
2150 op
->args
[1] = op
->args
[2];
2151 return fold_neg(ctx
, op
);
2156 /* We cannot as yet do_constant_folding with vectors. */
2157 static bool fold_sub_vec(OptContext
*ctx
, TCGOp
*op
)
2159 if (fold_xx_to_i(ctx
, op
, 0) ||
2160 fold_xi_to_x(ctx
, op
, 0) ||
2161 fold_sub_to_neg(ctx
, op
)) {
2167 static bool fold_sub(OptContext
*ctx
, TCGOp
*op
)
2169 return fold_const2(ctx
, op
) || fold_sub_vec(ctx
, op
);
2172 static bool fold_sub2(OptContext
*ctx
, TCGOp
*op
)
2174 return fold_addsub2(ctx
, op
, false);
2177 static bool fold_tcg_ld(OptContext
*ctx
, TCGOp
*op
)
2179 /* We can't do any folding with a load, but we can record bits. */
2181 CASE_OP_32_64(ld8s
):
2182 ctx
->s_mask
= MAKE_64BIT_MASK(8, 56);
2184 CASE_OP_32_64(ld8u
):
2185 ctx
->z_mask
= MAKE_64BIT_MASK(0, 8);
2186 ctx
->s_mask
= MAKE_64BIT_MASK(9, 55);
2188 CASE_OP_32_64(ld16s
):
2189 ctx
->s_mask
= MAKE_64BIT_MASK(16, 48);
2191 CASE_OP_32_64(ld16u
):
2192 ctx
->z_mask
= MAKE_64BIT_MASK(0, 16);
2193 ctx
->s_mask
= MAKE_64BIT_MASK(17, 47);
2195 case INDEX_op_ld32s_i64
:
2196 ctx
->s_mask
= MAKE_64BIT_MASK(32, 32);
2198 case INDEX_op_ld32u_i64
:
2199 ctx
->z_mask
= MAKE_64BIT_MASK(0, 32);
2200 ctx
->s_mask
= MAKE_64BIT_MASK(33, 31);
2203 g_assert_not_reached();
2208 static bool fold_tcg_ld_memcopy(OptContext
*ctx
, TCGOp
*op
)
2214 if (op
->args
[1] != tcgv_ptr_arg(tcg_env
)) {
2220 dst
= arg_temp(op
->args
[0]);
2221 src
= find_mem_copy_for(ctx
, type
, ofs
);
2222 if (src
&& src
->base_type
== type
) {
2223 return tcg_opt_gen_mov(ctx
, op
, temp_arg(dst
), temp_arg(src
));
2227 record_mem_copy(ctx
, type
, dst
, ofs
, ofs
+ tcg_type_size(type
) - 1);
2231 static bool fold_tcg_st(OptContext
*ctx
, TCGOp
*op
)
2233 intptr_t ofs
= op
->args
[2];
2236 if (op
->args
[1] != tcgv_ptr_arg(tcg_env
)) {
2237 remove_mem_copy_all(ctx
);
2245 CASE_OP_32_64(st16
):
2248 case INDEX_op_st32_i64
:
2249 case INDEX_op_st_i32
:
2252 case INDEX_op_st_i64
:
2255 case INDEX_op_st_vec
:
2256 lm1
= tcg_type_size(ctx
->type
) - 1;
2259 g_assert_not_reached();
2261 remove_mem_copy_in(ctx
, ofs
, ofs
+ lm1
);
2265 static bool fold_tcg_st_memcopy(OptContext
*ctx
, TCGOp
*op
)
2271 if (op
->args
[1] != tcgv_ptr_arg(tcg_env
)) {
2272 fold_tcg_st(ctx
, op
);
2276 src
= arg_temp(op
->args
[0]);
2281 * Eliminate duplicate stores of a constant.
2282 * This happens frequently when the target ISA zero-extends.
2284 if (ts_is_const(src
)) {
2285 TCGTemp
*prev
= find_mem_copy_for(ctx
, type
, ofs
);
2287 tcg_op_remove(ctx
->tcg
, op
);
2292 last
= ofs
+ tcg_type_size(type
) - 1;
2293 remove_mem_copy_in(ctx
, ofs
, last
);
2294 record_mem_copy(ctx
, type
, src
, ofs
, last
);
2298 static bool fold_xor(OptContext
*ctx
, TCGOp
*op
)
2300 if (fold_const2_commutative(ctx
, op
) ||
2301 fold_xx_to_i(ctx
, op
, 0) ||
2302 fold_xi_to_x(ctx
, op
, 0) ||
2303 fold_xi_to_not(ctx
, op
, -1)) {
2307 ctx
->z_mask
= arg_info(op
->args
[1])->z_mask
2308 | arg_info(op
->args
[2])->z_mask
;
2309 ctx
->s_mask
= arg_info(op
->args
[1])->s_mask
2310 & arg_info(op
->args
[2])->s_mask
;
2311 return fold_masks(ctx
, op
);
2314 /* Propagate constants and copies, fold constant expressions. */
2315 void tcg_optimize(TCGContext
*s
)
2318 TCGOp
*op
, *op_next
;
2319 OptContext ctx
= { .tcg
= s
};
2321 QSIMPLEQ_INIT(&ctx
.mem_free
);
2323 /* Array VALS has an element for each temp.
2324 If this temp holds a constant then its value is kept in VALS' element.
2325 If this temp is a copy of other ones then the other copies are
2326 available through the doubly linked circular list. */
2328 nb_temps
= s
->nb_temps
;
2329 for (i
= 0; i
< nb_temps
; ++i
) {
2330 s
->temps
[i
].state_ptr
= NULL
;
2333 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2334 TCGOpcode opc
= op
->opc
;
2335 const TCGOpDef
*def
;
2338 /* Calls are special. */
2339 if (opc
== INDEX_op_call
) {
2340 fold_call(&ctx
, op
);
2344 def
= &tcg_op_defs
[opc
];
2345 init_arguments(&ctx
, op
, def
->nb_oargs
+ def
->nb_iargs
);
2346 copy_propagate(&ctx
, op
, def
->nb_oargs
, def
->nb_iargs
);
2348 /* Pre-compute the type of the operation. */
2349 if (def
->flags
& TCG_OPF_VECTOR
) {
2350 ctx
.type
= TCG_TYPE_V64
+ TCGOP_VECL(op
);
2351 } else if (def
->flags
& TCG_OPF_64BIT
) {
2352 ctx
.type
= TCG_TYPE_I64
;
2354 ctx
.type
= TCG_TYPE_I32
;
2357 /* Assume all bits affected, no bits known zero, no sign reps. */
2363 * Process each opcode.
2364 * Sorted alphabetically by opcode as much as possible.
2368 done
= fold_add(&ctx
, op
);
2370 case INDEX_op_add_vec
:
2371 done
= fold_add_vec(&ctx
, op
);
2373 CASE_OP_32_64(add2
):
2374 done
= fold_add2(&ctx
, op
);
2376 CASE_OP_32_64_VEC(and):
2377 done
= fold_and(&ctx
, op
);
2379 CASE_OP_32_64_VEC(andc
):
2380 done
= fold_andc(&ctx
, op
);
2382 CASE_OP_32_64(brcond
):
2383 done
= fold_brcond(&ctx
, op
);
2385 case INDEX_op_brcond2_i32
:
2386 done
= fold_brcond2(&ctx
, op
);
2388 CASE_OP_32_64(bswap16
):
2389 CASE_OP_32_64(bswap32
):
2390 case INDEX_op_bswap64_i64
:
2391 done
= fold_bswap(&ctx
, op
);
2395 done
= fold_count_zeros(&ctx
, op
);
2397 CASE_OP_32_64(ctpop
):
2398 done
= fold_ctpop(&ctx
, op
);
2400 CASE_OP_32_64(deposit
):
2401 done
= fold_deposit(&ctx
, op
);
2404 CASE_OP_32_64(divu
):
2405 done
= fold_divide(&ctx
, op
);
2407 case INDEX_op_dup_vec
:
2408 done
= fold_dup(&ctx
, op
);
2410 case INDEX_op_dup2_vec
:
2411 done
= fold_dup2(&ctx
, op
);
2413 CASE_OP_32_64_VEC(eqv
):
2414 done
= fold_eqv(&ctx
, op
);
2416 CASE_OP_32_64(extract
):
2417 done
= fold_extract(&ctx
, op
);
2419 CASE_OP_32_64(extract2
):
2420 done
= fold_extract2(&ctx
, op
);
2422 CASE_OP_32_64(ext8s
):
2423 CASE_OP_32_64(ext16s
):
2424 case INDEX_op_ext32s_i64
:
2425 case INDEX_op_ext_i32_i64
:
2426 done
= fold_exts(&ctx
, op
);
2428 CASE_OP_32_64(ext8u
):
2429 CASE_OP_32_64(ext16u
):
2430 case INDEX_op_ext32u_i64
:
2431 case INDEX_op_extu_i32_i64
:
2432 case INDEX_op_extrl_i64_i32
:
2433 case INDEX_op_extrh_i64_i32
:
2434 done
= fold_extu(&ctx
, op
);
2436 CASE_OP_32_64(ld8s
):
2437 CASE_OP_32_64(ld8u
):
2438 CASE_OP_32_64(ld16s
):
2439 CASE_OP_32_64(ld16u
):
2440 case INDEX_op_ld32s_i64
:
2441 case INDEX_op_ld32u_i64
:
2442 done
= fold_tcg_ld(&ctx
, op
);
2444 case INDEX_op_ld_i32
:
2445 case INDEX_op_ld_i64
:
2446 case INDEX_op_ld_vec
:
2447 done
= fold_tcg_ld_memcopy(&ctx
, op
);
2450 CASE_OP_32_64(st16
):
2451 case INDEX_op_st32_i64
:
2452 done
= fold_tcg_st(&ctx
, op
);
2454 case INDEX_op_st_i32
:
2455 case INDEX_op_st_i64
:
2456 case INDEX_op_st_vec
:
2457 done
= fold_tcg_st_memcopy(&ctx
, op
);
2460 done
= fold_mb(&ctx
, op
);
2462 CASE_OP_32_64_VEC(mov
):
2463 done
= fold_mov(&ctx
, op
);
2465 CASE_OP_32_64(movcond
):
2466 done
= fold_movcond(&ctx
, op
);
2469 done
= fold_mul(&ctx
, op
);
2471 CASE_OP_32_64(mulsh
):
2472 CASE_OP_32_64(muluh
):
2473 done
= fold_mul_highpart(&ctx
, op
);
2475 CASE_OP_32_64(muls2
):
2476 CASE_OP_32_64(mulu2
):
2477 done
= fold_multiply2(&ctx
, op
);
2479 CASE_OP_32_64_VEC(nand
):
2480 done
= fold_nand(&ctx
, op
);
2483 done
= fold_neg(&ctx
, op
);
2485 CASE_OP_32_64_VEC(nor
):
2486 done
= fold_nor(&ctx
, op
);
2488 CASE_OP_32_64_VEC(not):
2489 done
= fold_not(&ctx
, op
);
2491 CASE_OP_32_64_VEC(or):
2492 done
= fold_or(&ctx
, op
);
2494 CASE_OP_32_64_VEC(orc
):
2495 done
= fold_orc(&ctx
, op
);
2497 case INDEX_op_qemu_ld_a32_i32
:
2498 case INDEX_op_qemu_ld_a64_i32
:
2499 case INDEX_op_qemu_ld_a32_i64
:
2500 case INDEX_op_qemu_ld_a64_i64
:
2501 case INDEX_op_qemu_ld_a32_i128
:
2502 case INDEX_op_qemu_ld_a64_i128
:
2503 done
= fold_qemu_ld(&ctx
, op
);
2505 case INDEX_op_qemu_st8_a32_i32
:
2506 case INDEX_op_qemu_st8_a64_i32
:
2507 case INDEX_op_qemu_st_a32_i32
:
2508 case INDEX_op_qemu_st_a64_i32
:
2509 case INDEX_op_qemu_st_a32_i64
:
2510 case INDEX_op_qemu_st_a64_i64
:
2511 case INDEX_op_qemu_st_a32_i128
:
2512 case INDEX_op_qemu_st_a64_i128
:
2513 done
= fold_qemu_st(&ctx
, op
);
2516 CASE_OP_32_64(remu
):
2517 done
= fold_remainder(&ctx
, op
);
2519 CASE_OP_32_64(rotl
):
2520 CASE_OP_32_64(rotr
):
2524 done
= fold_shift(&ctx
, op
);
2526 CASE_OP_32_64(setcond
):
2527 done
= fold_setcond(&ctx
, op
);
2529 CASE_OP_32_64(negsetcond
):
2530 done
= fold_negsetcond(&ctx
, op
);
2532 case INDEX_op_setcond2_i32
:
2533 done
= fold_setcond2(&ctx
, op
);
2535 CASE_OP_32_64(sextract
):
2536 done
= fold_sextract(&ctx
, op
);
2539 done
= fold_sub(&ctx
, op
);
2541 case INDEX_op_sub_vec
:
2542 done
= fold_sub_vec(&ctx
, op
);
2544 CASE_OP_32_64(sub2
):
2545 done
= fold_sub2(&ctx
, op
);
2547 CASE_OP_32_64_VEC(xor):
2548 done
= fold_xor(&ctx
, op
);
2555 finish_folding(&ctx
, op
);