2 * Optimizations for Tiny Code Generator for QEMU
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 #include "qemu/osdep.h"
27 #include "tcg/tcg-op.h"
29 #define CASE_OP_32_64(x) \
30 glue(glue(case INDEX_op_, x), _i32): \
31 glue(glue(case INDEX_op_, x), _i64)
33 #define CASE_OP_32_64_VEC(x) \
34 glue(glue(case INDEX_op_, x), _i32): \
35 glue(glue(case INDEX_op_, x), _i64): \
36 glue(glue(case INDEX_op_, x), _vec)
38 typedef struct TempOptInfo
{
46 static inline TempOptInfo
*ts_info(TCGTemp
*ts
)
51 static inline TempOptInfo
*arg_info(TCGArg arg
)
53 return ts_info(arg_temp(arg
));
56 static inline bool ts_is_const(TCGTemp
*ts
)
58 return ts_info(ts
)->is_const
;
61 static inline bool arg_is_const(TCGArg arg
)
63 return ts_is_const(arg_temp(arg
));
66 static inline bool ts_is_copy(TCGTemp
*ts
)
68 return ts_info(ts
)->next_copy
!= ts
;
71 /* Reset TEMP's state, possibly removing the temp for the list of copies. */
72 static void reset_ts(TCGTemp
*ts
)
74 TempOptInfo
*ti
= ts_info(ts
);
75 TempOptInfo
*pi
= ts_info(ti
->prev_copy
);
76 TempOptInfo
*ni
= ts_info(ti
->next_copy
);
78 ni
->prev_copy
= ti
->prev_copy
;
79 pi
->next_copy
= ti
->next_copy
;
86 static void reset_temp(TCGArg arg
)
88 reset_ts(arg_temp(arg
));
91 /* Initialize and activate a temporary. */
92 static void init_ts_info(TempOptInfo
*infos
,
93 TCGTempSet
*temps_used
, TCGTemp
*ts
)
95 size_t idx
= temp_idx(ts
);
96 if (!test_bit(idx
, temps_used
->l
)) {
97 TempOptInfo
*ti
= &infos
[idx
];
102 if (ts
->kind
== TEMP_CONST
) {
104 ti
->val
= ti
->mask
= ts
->val
;
105 if (TCG_TARGET_REG_BITS
> 32 && ts
->type
== TCG_TYPE_I32
) {
106 /* High bits of a 32-bit quantity are garbage. */
107 ti
->mask
|= ~0xffffffffull
;
110 ti
->is_const
= false;
113 set_bit(idx
, temps_used
->l
);
117 static void init_arg_info(TempOptInfo
*infos
,
118 TCGTempSet
*temps_used
, TCGArg arg
)
120 init_ts_info(infos
, temps_used
, arg_temp(arg
));
123 static TCGTemp
*find_better_copy(TCGContext
*s
, TCGTemp
*ts
)
127 /* If this is already a global, we can't do better. */
128 if (ts
->kind
>= TEMP_GLOBAL
) {
132 /* Search for a global first. */
133 for (i
= ts_info(ts
)->next_copy
; i
!= ts
; i
= ts_info(i
)->next_copy
) {
134 if (i
->kind
>= TEMP_GLOBAL
) {
139 /* If it is a temp, search for a temp local. */
140 if (ts
->kind
== TEMP_NORMAL
) {
141 for (i
= ts_info(ts
)->next_copy
; i
!= ts
; i
= ts_info(i
)->next_copy
) {
142 if (i
->kind
>= TEMP_LOCAL
) {
148 /* Failure to find a better representation, return the same temp. */
152 static bool ts_are_copies(TCGTemp
*ts1
, TCGTemp
*ts2
)
160 if (!ts_is_copy(ts1
) || !ts_is_copy(ts2
)) {
164 for (i
= ts_info(ts1
)->next_copy
; i
!= ts1
; i
= ts_info(i
)->next_copy
) {
173 static bool args_are_copies(TCGArg arg1
, TCGArg arg2
)
175 return ts_are_copies(arg_temp(arg1
), arg_temp(arg2
));
178 static void tcg_opt_gen_movi(TCGContext
*s
, TCGOp
*op
, TCGArg dst
, uint64_t val
)
183 TempOptInfo
*di
= arg_info(dst
);
185 def
= &tcg_op_defs
[op
->opc
];
186 if (def
->flags
& TCG_OPF_VECTOR
) {
187 new_op
= INDEX_op_dupi_vec
;
188 } else if (def
->flags
& TCG_OPF_64BIT
) {
189 new_op
= INDEX_op_movi_i64
;
191 new_op
= INDEX_op_movi_i32
;
194 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
202 if (TCG_TARGET_REG_BITS
> 32 && new_op
== INDEX_op_movi_i32
) {
203 /* High bits of the destination are now garbage. */
204 mask
|= ~0xffffffffull
;
209 static void tcg_opt_gen_mov(TCGContext
*s
, TCGOp
*op
, TCGArg dst
, TCGArg src
)
211 TCGTemp
*dst_ts
= arg_temp(dst
);
212 TCGTemp
*src_ts
= arg_temp(src
);
219 if (ts_are_copies(dst_ts
, src_ts
)) {
220 tcg_op_remove(s
, op
);
225 di
= ts_info(dst_ts
);
226 si
= ts_info(src_ts
);
227 def
= &tcg_op_defs
[op
->opc
];
228 if (def
->flags
& TCG_OPF_VECTOR
) {
229 new_op
= INDEX_op_mov_vec
;
230 } else if (def
->flags
& TCG_OPF_64BIT
) {
231 new_op
= INDEX_op_mov_i64
;
233 new_op
= INDEX_op_mov_i32
;
236 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
241 if (TCG_TARGET_REG_BITS
> 32 && new_op
== INDEX_op_mov_i32
) {
242 /* High bits of the destination are now garbage. */
243 mask
|= ~0xffffffffull
;
247 if (src_ts
->type
== dst_ts
->type
) {
248 TempOptInfo
*ni
= ts_info(si
->next_copy
);
250 di
->next_copy
= si
->next_copy
;
251 di
->prev_copy
= src_ts
;
252 ni
->prev_copy
= dst_ts
;
253 si
->next_copy
= dst_ts
;
254 di
->is_const
= si
->is_const
;
259 static uint64_t do_constant_folding_2(TCGOpcode op
, uint64_t x
, uint64_t y
)
282 case INDEX_op_shl_i32
:
283 return (uint32_t)x
<< (y
& 31);
285 case INDEX_op_shl_i64
:
286 return (uint64_t)x
<< (y
& 63);
288 case INDEX_op_shr_i32
:
289 return (uint32_t)x
>> (y
& 31);
291 case INDEX_op_shr_i64
:
292 return (uint64_t)x
>> (y
& 63);
294 case INDEX_op_sar_i32
:
295 return (int32_t)x
>> (y
& 31);
297 case INDEX_op_sar_i64
:
298 return (int64_t)x
>> (y
& 63);
300 case INDEX_op_rotr_i32
:
301 return ror32(x
, y
& 31);
303 case INDEX_op_rotr_i64
:
304 return ror64(x
, y
& 63);
306 case INDEX_op_rotl_i32
:
307 return rol32(x
, y
& 31);
309 case INDEX_op_rotl_i64
:
310 return rol64(x
, y
& 63);
333 case INDEX_op_clz_i32
:
334 return (uint32_t)x
? clz32(x
) : y
;
336 case INDEX_op_clz_i64
:
337 return x
? clz64(x
) : y
;
339 case INDEX_op_ctz_i32
:
340 return (uint32_t)x
? ctz32(x
) : y
;
342 case INDEX_op_ctz_i64
:
343 return x
? ctz64(x
) : y
;
345 case INDEX_op_ctpop_i32
:
348 case INDEX_op_ctpop_i64
:
351 CASE_OP_32_64(ext8s
):
354 CASE_OP_32_64(ext16s
):
357 CASE_OP_32_64(ext8u
):
360 CASE_OP_32_64(ext16u
):
363 CASE_OP_32_64(bswap16
):
366 CASE_OP_32_64(bswap32
):
369 case INDEX_op_bswap64_i64
:
372 case INDEX_op_ext_i32_i64
:
373 case INDEX_op_ext32s_i64
:
376 case INDEX_op_extu_i32_i64
:
377 case INDEX_op_extrl_i64_i32
:
378 case INDEX_op_ext32u_i64
:
381 case INDEX_op_extrh_i64_i32
:
382 return (uint64_t)x
>> 32;
384 case INDEX_op_muluh_i32
:
385 return ((uint64_t)(uint32_t)x
* (uint32_t)y
) >> 32;
386 case INDEX_op_mulsh_i32
:
387 return ((int64_t)(int32_t)x
* (int32_t)y
) >> 32;
389 case INDEX_op_muluh_i64
:
390 mulu64(&l64
, &h64
, x
, y
);
392 case INDEX_op_mulsh_i64
:
393 muls64(&l64
, &h64
, x
, y
);
396 case INDEX_op_div_i32
:
397 /* Avoid crashing on divide by zero, otherwise undefined. */
398 return (int32_t)x
/ ((int32_t)y
? : 1);
399 case INDEX_op_divu_i32
:
400 return (uint32_t)x
/ ((uint32_t)y
? : 1);
401 case INDEX_op_div_i64
:
402 return (int64_t)x
/ ((int64_t)y
? : 1);
403 case INDEX_op_divu_i64
:
404 return (uint64_t)x
/ ((uint64_t)y
? : 1);
406 case INDEX_op_rem_i32
:
407 return (int32_t)x
% ((int32_t)y
? : 1);
408 case INDEX_op_remu_i32
:
409 return (uint32_t)x
% ((uint32_t)y
? : 1);
410 case INDEX_op_rem_i64
:
411 return (int64_t)x
% ((int64_t)y
? : 1);
412 case INDEX_op_remu_i64
:
413 return (uint64_t)x
% ((uint64_t)y
? : 1);
417 "Unrecognized operation %d in do_constant_folding.\n", op
);
422 static uint64_t do_constant_folding(TCGOpcode op
, uint64_t x
, uint64_t y
)
424 const TCGOpDef
*def
= &tcg_op_defs
[op
];
425 uint64_t res
= do_constant_folding_2(op
, x
, y
);
426 if (!(def
->flags
& TCG_OPF_64BIT
)) {
432 static bool do_constant_folding_cond_32(uint32_t x
, uint32_t y
, TCGCond c
)
440 return (int32_t)x
< (int32_t)y
;
442 return (int32_t)x
>= (int32_t)y
;
444 return (int32_t)x
<= (int32_t)y
;
446 return (int32_t)x
> (int32_t)y
;
460 static bool do_constant_folding_cond_64(uint64_t x
, uint64_t y
, TCGCond c
)
468 return (int64_t)x
< (int64_t)y
;
470 return (int64_t)x
>= (int64_t)y
;
472 return (int64_t)x
<= (int64_t)y
;
474 return (int64_t)x
> (int64_t)y
;
488 static bool do_constant_folding_cond_eq(TCGCond c
)
508 /* Return 2 if the condition can't be simplified, and the result
509 of the condition (0 or 1) if it can */
510 static TCGArg
do_constant_folding_cond(TCGOpcode op
, TCGArg x
,
513 uint64_t xv
= arg_info(x
)->val
;
514 uint64_t yv
= arg_info(y
)->val
;
516 if (arg_is_const(x
) && arg_is_const(y
)) {
517 const TCGOpDef
*def
= &tcg_op_defs
[op
];
518 tcg_debug_assert(!(def
->flags
& TCG_OPF_VECTOR
));
519 if (def
->flags
& TCG_OPF_64BIT
) {
520 return do_constant_folding_cond_64(xv
, yv
, c
);
522 return do_constant_folding_cond_32(xv
, yv
, c
);
524 } else if (args_are_copies(x
, y
)) {
525 return do_constant_folding_cond_eq(c
);
526 } else if (arg_is_const(y
) && yv
== 0) {
539 /* Return 2 if the condition can't be simplified, and the result
540 of the condition (0 or 1) if it can */
541 static TCGArg
do_constant_folding_cond2(TCGArg
*p1
, TCGArg
*p2
, TCGCond c
)
543 TCGArg al
= p1
[0], ah
= p1
[1];
544 TCGArg bl
= p2
[0], bh
= p2
[1];
546 if (arg_is_const(bl
) && arg_is_const(bh
)) {
547 tcg_target_ulong blv
= arg_info(bl
)->val
;
548 tcg_target_ulong bhv
= arg_info(bh
)->val
;
549 uint64_t b
= deposit64(blv
, 32, 32, bhv
);
551 if (arg_is_const(al
) && arg_is_const(ah
)) {
552 tcg_target_ulong alv
= arg_info(al
)->val
;
553 tcg_target_ulong ahv
= arg_info(ah
)->val
;
554 uint64_t a
= deposit64(alv
, 32, 32, ahv
);
555 return do_constant_folding_cond_64(a
, b
, c
);
568 if (args_are_copies(al
, bl
) && args_are_copies(ah
, bh
)) {
569 return do_constant_folding_cond_eq(c
);
574 static bool swap_commutative(TCGArg dest
, TCGArg
*p1
, TCGArg
*p2
)
576 TCGArg a1
= *p1
, a2
= *p2
;
578 sum
+= arg_is_const(a1
);
579 sum
-= arg_is_const(a2
);
581 /* Prefer the constant in second argument, and then the form
582 op a, a, b, which is better handled on non-RISC hosts. */
583 if (sum
> 0 || (sum
== 0 && dest
== a2
)) {
591 static bool swap_commutative2(TCGArg
*p1
, TCGArg
*p2
)
594 sum
+= arg_is_const(p1
[0]);
595 sum
+= arg_is_const(p1
[1]);
596 sum
-= arg_is_const(p2
[0]);
597 sum
-= arg_is_const(p2
[1]);
600 t
= p1
[0], p1
[0] = p2
[0], p2
[0] = t
;
601 t
= p1
[1], p1
[1] = p2
[1], p2
[1] = t
;
607 /* Propagate constants and copies, fold constant expressions. */
608 void tcg_optimize(TCGContext
*s
)
610 int nb_temps
, nb_globals
;
611 TCGOp
*op
, *op_next
, *prev_mb
= NULL
;
613 TCGTempSet temps_used
;
615 /* Array VALS has an element for each temp.
616 If this temp holds a constant then its value is kept in VALS' element.
617 If this temp is a copy of other ones then the other copies are
618 available through the doubly linked circular list. */
620 nb_temps
= s
->nb_temps
;
621 nb_globals
= s
->nb_globals
;
622 bitmap_zero(temps_used
.l
, nb_temps
);
623 infos
= tcg_malloc(sizeof(TempOptInfo
) * nb_temps
);
625 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
626 uint64_t mask
, partmask
, affected
, tmp
;
627 int nb_oargs
, nb_iargs
, i
;
628 TCGOpcode opc
= op
->opc
;
629 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
631 /* Count the arguments, and initialize the temps that are
633 if (opc
== INDEX_op_call
) {
634 nb_oargs
= TCGOP_CALLO(op
);
635 nb_iargs
= TCGOP_CALLI(op
);
636 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
637 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
639 init_ts_info(infos
, &temps_used
, ts
);
643 nb_oargs
= def
->nb_oargs
;
644 nb_iargs
= def
->nb_iargs
;
645 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
646 init_arg_info(infos
, &temps_used
, op
->args
[i
]);
650 /* Do copy propagation */
651 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
652 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
653 if (ts
&& ts_is_copy(ts
)) {
654 op
->args
[i
] = temp_arg(find_better_copy(s
, ts
));
658 /* For commutative operations make constant second argument */
660 CASE_OP_32_64_VEC(add
):
661 CASE_OP_32_64_VEC(mul
):
662 CASE_OP_32_64_VEC(and):
663 CASE_OP_32_64_VEC(or):
664 CASE_OP_32_64_VEC(xor):
668 CASE_OP_32_64(muluh
):
669 CASE_OP_32_64(mulsh
):
670 swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2]);
672 CASE_OP_32_64(brcond
):
673 if (swap_commutative(-1, &op
->args
[0], &op
->args
[1])) {
674 op
->args
[2] = tcg_swap_cond(op
->args
[2]);
677 CASE_OP_32_64(setcond
):
678 if (swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2])) {
679 op
->args
[3] = tcg_swap_cond(op
->args
[3]);
682 CASE_OP_32_64(movcond
):
683 if (swap_commutative(-1, &op
->args
[1], &op
->args
[2])) {
684 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
686 /* For movcond, we canonicalize the "false" input reg to match
687 the destination reg so that the tcg backend can implement
688 a "move if true" operation. */
689 if (swap_commutative(op
->args
[0], &op
->args
[4], &op
->args
[3])) {
690 op
->args
[5] = tcg_invert_cond(op
->args
[5]);
694 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[4]);
695 swap_commutative(op
->args
[1], &op
->args
[3], &op
->args
[5]);
697 CASE_OP_32_64(mulu2
):
698 CASE_OP_32_64(muls2
):
699 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[3]);
701 case INDEX_op_brcond2_i32
:
702 if (swap_commutative2(&op
->args
[0], &op
->args
[2])) {
703 op
->args
[4] = tcg_swap_cond(op
->args
[4]);
706 case INDEX_op_setcond2_i32
:
707 if (swap_commutative2(&op
->args
[1], &op
->args
[3])) {
708 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
715 /* Simplify expressions for "shift/rot r, 0, a => movi r, 0",
716 and "sub r, 0, a => neg r, a" case. */
723 if (arg_is_const(op
->args
[1])
724 && arg_info(op
->args
[1])->val
== 0) {
725 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
729 CASE_OP_32_64_VEC(sub
):
734 if (arg_is_const(op
->args
[2])) {
735 /* Proceed with possible constant folding. */
738 if (opc
== INDEX_op_sub_i32
) {
739 neg_op
= INDEX_op_neg_i32
;
740 have_neg
= TCG_TARGET_HAS_neg_i32
;
741 } else if (opc
== INDEX_op_sub_i64
) {
742 neg_op
= INDEX_op_neg_i64
;
743 have_neg
= TCG_TARGET_HAS_neg_i64
;
744 } else if (TCG_TARGET_HAS_neg_vec
) {
745 TCGType type
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
746 unsigned vece
= TCGOP_VECE(op
);
747 neg_op
= INDEX_op_neg_vec
;
748 have_neg
= tcg_can_emit_vec_op(neg_op
, type
, vece
) > 0;
755 if (arg_is_const(op
->args
[1])
756 && arg_info(op
->args
[1])->val
== 0) {
758 reset_temp(op
->args
[0]);
759 op
->args
[1] = op
->args
[2];
764 CASE_OP_32_64_VEC(xor):
766 if (!arg_is_const(op
->args
[1])
767 && arg_is_const(op
->args
[2])
768 && arg_info(op
->args
[2])->val
== -1) {
774 if (!arg_is_const(op
->args
[1])
775 && arg_is_const(op
->args
[2])
776 && arg_info(op
->args
[2])->val
== 0) {
781 CASE_OP_32_64_VEC(andc
):
782 if (!arg_is_const(op
->args
[2])
783 && arg_is_const(op
->args
[1])
784 && arg_info(op
->args
[1])->val
== -1) {
789 CASE_OP_32_64_VEC(orc
):
791 if (!arg_is_const(op
->args
[2])
792 && arg_is_const(op
->args
[1])
793 && arg_info(op
->args
[1])->val
== 0) {
803 if (def
->flags
& TCG_OPF_VECTOR
) {
804 not_op
= INDEX_op_not_vec
;
805 have_not
= TCG_TARGET_HAS_not_vec
;
806 } else if (def
->flags
& TCG_OPF_64BIT
) {
807 not_op
= INDEX_op_not_i64
;
808 have_not
= TCG_TARGET_HAS_not_i64
;
810 not_op
= INDEX_op_not_i32
;
811 have_not
= TCG_TARGET_HAS_not_i32
;
817 reset_temp(op
->args
[0]);
818 op
->args
[1] = op
->args
[i
];
825 /* Simplify expression for "op r, a, const => mov r, a" cases */
827 CASE_OP_32_64_VEC(add
):
828 CASE_OP_32_64_VEC(sub
):
829 CASE_OP_32_64_VEC(or):
830 CASE_OP_32_64_VEC(xor):
831 CASE_OP_32_64_VEC(andc
):
837 if (!arg_is_const(op
->args
[1])
838 && arg_is_const(op
->args
[2])
839 && arg_info(op
->args
[2])->val
== 0) {
840 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
844 CASE_OP_32_64_VEC(and):
845 CASE_OP_32_64_VEC(orc
):
847 if (!arg_is_const(op
->args
[1])
848 && arg_is_const(op
->args
[2])
849 && arg_info(op
->args
[2])->val
== -1) {
850 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
858 /* Simplify using known-zero bits. Currently only ops with a single
859 output argument is supported. */
863 CASE_OP_32_64(ext8s
):
864 if ((arg_info(op
->args
[1])->mask
& 0x80) != 0) {
868 CASE_OP_32_64(ext8u
):
871 CASE_OP_32_64(ext16s
):
872 if ((arg_info(op
->args
[1])->mask
& 0x8000) != 0) {
876 CASE_OP_32_64(ext16u
):
879 case INDEX_op_ext32s_i64
:
880 if ((arg_info(op
->args
[1])->mask
& 0x80000000) != 0) {
884 case INDEX_op_ext32u_i64
:
889 mask
= arg_info(op
->args
[2])->mask
;
890 if (arg_is_const(op
->args
[2])) {
892 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
894 mask
= arg_info(op
->args
[1])->mask
& mask
;
897 case INDEX_op_ext_i32_i64
:
898 if ((arg_info(op
->args
[1])->mask
& 0x80000000) != 0) {
902 case INDEX_op_extu_i32_i64
:
903 /* We do not compute affected as it is a size changing op. */
904 mask
= (uint32_t)arg_info(op
->args
[1])->mask
;
908 /* Known-zeros does not imply known-ones. Therefore unless
909 op->args[2] is constant, we can't infer anything from it. */
910 if (arg_is_const(op
->args
[2])) {
911 mask
= ~arg_info(op
->args
[2])->mask
;
914 /* But we certainly know nothing outside args[1] may be set. */
915 mask
= arg_info(op
->args
[1])->mask
;
918 case INDEX_op_sar_i32
:
919 if (arg_is_const(op
->args
[2])) {
920 tmp
= arg_info(op
->args
[2])->val
& 31;
921 mask
= (int32_t)arg_info(op
->args
[1])->mask
>> tmp
;
924 case INDEX_op_sar_i64
:
925 if (arg_is_const(op
->args
[2])) {
926 tmp
= arg_info(op
->args
[2])->val
& 63;
927 mask
= (int64_t)arg_info(op
->args
[1])->mask
>> tmp
;
931 case INDEX_op_shr_i32
:
932 if (arg_is_const(op
->args
[2])) {
933 tmp
= arg_info(op
->args
[2])->val
& 31;
934 mask
= (uint32_t)arg_info(op
->args
[1])->mask
>> tmp
;
937 case INDEX_op_shr_i64
:
938 if (arg_is_const(op
->args
[2])) {
939 tmp
= arg_info(op
->args
[2])->val
& 63;
940 mask
= (uint64_t)arg_info(op
->args
[1])->mask
>> tmp
;
944 case INDEX_op_extrl_i64_i32
:
945 mask
= (uint32_t)arg_info(op
->args
[1])->mask
;
947 case INDEX_op_extrh_i64_i32
:
948 mask
= (uint64_t)arg_info(op
->args
[1])->mask
>> 32;
952 if (arg_is_const(op
->args
[2])) {
953 tmp
= arg_info(op
->args
[2])->val
& (TCG_TARGET_REG_BITS
- 1);
954 mask
= arg_info(op
->args
[1])->mask
<< tmp
;
959 /* Set to 1 all bits to the left of the rightmost. */
960 mask
= -(arg_info(op
->args
[1])->mask
961 & -arg_info(op
->args
[1])->mask
);
964 CASE_OP_32_64(deposit
):
965 mask
= deposit64(arg_info(op
->args
[1])->mask
,
966 op
->args
[3], op
->args
[4],
967 arg_info(op
->args
[2])->mask
);
970 CASE_OP_32_64(extract
):
971 mask
= extract64(arg_info(op
->args
[1])->mask
,
972 op
->args
[2], op
->args
[3]);
973 if (op
->args
[2] == 0) {
974 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
977 CASE_OP_32_64(sextract
):
978 mask
= sextract64(arg_info(op
->args
[1])->mask
,
979 op
->args
[2], op
->args
[3]);
980 if (op
->args
[2] == 0 && (tcg_target_long
)mask
>= 0) {
981 affected
= arg_info(op
->args
[1])->mask
& ~mask
;
987 mask
= arg_info(op
->args
[1])->mask
| arg_info(op
->args
[2])->mask
;
990 case INDEX_op_clz_i32
:
991 case INDEX_op_ctz_i32
:
992 mask
= arg_info(op
->args
[2])->mask
| 31;
995 case INDEX_op_clz_i64
:
996 case INDEX_op_ctz_i64
:
997 mask
= arg_info(op
->args
[2])->mask
| 63;
1000 case INDEX_op_ctpop_i32
:
1003 case INDEX_op_ctpop_i64
:
1007 CASE_OP_32_64(setcond
):
1008 case INDEX_op_setcond2_i32
:
1012 CASE_OP_32_64(movcond
):
1013 mask
= arg_info(op
->args
[3])->mask
| arg_info(op
->args
[4])->mask
;
1016 CASE_OP_32_64(ld8u
):
1019 CASE_OP_32_64(ld16u
):
1022 case INDEX_op_ld32u_i64
:
1026 CASE_OP_32_64(qemu_ld
):
1028 TCGMemOpIdx oi
= op
->args
[nb_oargs
+ nb_iargs
];
1029 MemOp mop
= get_memop(oi
);
1030 if (!(mop
& MO_SIGN
)) {
1031 mask
= (2ULL << ((8 << (mop
& MO_SIZE
)) - 1)) - 1;
1040 /* 32-bit ops generate 32-bit results. For the result is zero test
1041 below, we can ignore high bits, but for further optimizations we
1042 need to record that the high bits contain garbage. */
1044 if (!(def
->flags
& TCG_OPF_64BIT
)) {
1045 mask
|= ~(tcg_target_ulong
)0xffffffffu
;
1046 partmask
&= 0xffffffffu
;
1047 affected
&= 0xffffffffu
;
1050 if (partmask
== 0) {
1051 tcg_debug_assert(nb_oargs
== 1);
1052 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
1055 if (affected
== 0) {
1056 tcg_debug_assert(nb_oargs
== 1);
1057 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1061 /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
1063 CASE_OP_32_64_VEC(and):
1064 CASE_OP_32_64_VEC(mul
):
1065 CASE_OP_32_64(muluh
):
1066 CASE_OP_32_64(mulsh
):
1067 if (arg_is_const(op
->args
[2])
1068 && arg_info(op
->args
[2])->val
== 0) {
1069 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
1077 /* Simplify expression for "op r, a, a => mov r, a" cases */
1079 CASE_OP_32_64_VEC(or):
1080 CASE_OP_32_64_VEC(and):
1081 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1082 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1090 /* Simplify expression for "op r, a, a => movi r, 0" cases */
1092 CASE_OP_32_64_VEC(andc
):
1093 CASE_OP_32_64_VEC(sub
):
1094 CASE_OP_32_64_VEC(xor):
1095 if (args_are_copies(op
->args
[1], op
->args
[2])) {
1096 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
1104 /* Propagate constants through copy operations and do constant
1105 folding. Constants will be substituted to arguments by register
1106 allocator where needed and possible. Also detect copies. */
1108 CASE_OP_32_64_VEC(mov
):
1109 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1111 CASE_OP_32_64(movi
):
1112 case INDEX_op_dupi_vec
:
1113 tcg_opt_gen_movi(s
, op
, op
->args
[0], op
->args
[1]);
1116 case INDEX_op_dup_vec
:
1117 if (arg_is_const(op
->args
[1])) {
1118 tmp
= arg_info(op
->args
[1])->val
;
1119 tmp
= dup_const(TCGOP_VECE(op
), tmp
);
1120 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1125 case INDEX_op_dup2_vec
:
1126 assert(TCG_TARGET_REG_BITS
== 32);
1127 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1128 tmp
= arg_info(op
->args
[1])->val
;
1129 if (tmp
== arg_info(op
->args
[2])->val
) {
1130 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1133 } else if (args_are_copies(op
->args
[1], op
->args
[2])) {
1134 op
->opc
= INDEX_op_dup_vec
;
1135 TCGOP_VECE(op
) = MO_32
;
1142 CASE_OP_32_64(ext8s
):
1143 CASE_OP_32_64(ext8u
):
1144 CASE_OP_32_64(ext16s
):
1145 CASE_OP_32_64(ext16u
):
1146 CASE_OP_32_64(ctpop
):
1147 CASE_OP_32_64(bswap16
):
1148 CASE_OP_32_64(bswap32
):
1149 case INDEX_op_bswap64_i64
:
1150 case INDEX_op_ext32s_i64
:
1151 case INDEX_op_ext32u_i64
:
1152 case INDEX_op_ext_i32_i64
:
1153 case INDEX_op_extu_i32_i64
:
1154 case INDEX_op_extrl_i64_i32
:
1155 case INDEX_op_extrh_i64_i32
:
1156 if (arg_is_const(op
->args
[1])) {
1157 tmp
= do_constant_folding(opc
, arg_info(op
->args
[1])->val
, 0);
1158 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1172 CASE_OP_32_64(rotl
):
1173 CASE_OP_32_64(rotr
):
1174 CASE_OP_32_64(andc
):
1177 CASE_OP_32_64(nand
):
1179 CASE_OP_32_64(muluh
):
1180 CASE_OP_32_64(mulsh
):
1182 CASE_OP_32_64(divu
):
1184 CASE_OP_32_64(remu
):
1185 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1186 tmp
= do_constant_folding(opc
, arg_info(op
->args
[1])->val
,
1187 arg_info(op
->args
[2])->val
);
1188 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1195 if (arg_is_const(op
->args
[1])) {
1196 TCGArg v
= arg_info(op
->args
[1])->val
;
1198 tmp
= do_constant_folding(opc
, v
, 0);
1199 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1201 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[2]);
1207 CASE_OP_32_64(deposit
):
1208 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1209 tmp
= deposit64(arg_info(op
->args
[1])->val
,
1210 op
->args
[3], op
->args
[4],
1211 arg_info(op
->args
[2])->val
);
1212 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1217 CASE_OP_32_64(extract
):
1218 if (arg_is_const(op
->args
[1])) {
1219 tmp
= extract64(arg_info(op
->args
[1])->val
,
1220 op
->args
[2], op
->args
[3]);
1221 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1226 CASE_OP_32_64(sextract
):
1227 if (arg_is_const(op
->args
[1])) {
1228 tmp
= sextract64(arg_info(op
->args
[1])->val
,
1229 op
->args
[2], op
->args
[3]);
1230 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1235 CASE_OP_32_64(extract2
):
1236 if (arg_is_const(op
->args
[1]) && arg_is_const(op
->args
[2])) {
1237 uint64_t v1
= arg_info(op
->args
[1])->val
;
1238 uint64_t v2
= arg_info(op
->args
[2])->val
;
1239 int shr
= op
->args
[3];
1241 if (opc
== INDEX_op_extract2_i64
) {
1242 tmp
= (v1
>> shr
) | (v2
<< (64 - shr
));
1244 tmp
= (int32_t)(((uint32_t)v1
>> shr
) |
1245 ((uint32_t)v2
<< (32 - shr
)));
1247 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1252 CASE_OP_32_64(setcond
):
1253 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1254 op
->args
[2], op
->args
[3]);
1256 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1261 CASE_OP_32_64(brcond
):
1262 tmp
= do_constant_folding_cond(opc
, op
->args
[0],
1263 op
->args
[1], op
->args
[2]);
1266 bitmap_zero(temps_used
.l
, nb_temps
);
1267 op
->opc
= INDEX_op_br
;
1268 op
->args
[0] = op
->args
[3];
1270 tcg_op_remove(s
, op
);
1276 CASE_OP_32_64(movcond
):
1277 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1278 op
->args
[2], op
->args
[5]);
1280 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[4-tmp
]);
1283 if (arg_is_const(op
->args
[3]) && arg_is_const(op
->args
[4])) {
1284 uint64_t tv
= arg_info(op
->args
[3])->val
;
1285 uint64_t fv
= arg_info(op
->args
[4])->val
;
1286 TCGCond cond
= op
->args
[5];
1288 if (fv
== 1 && tv
== 0) {
1289 cond
= tcg_invert_cond(cond
);
1290 } else if (!(tv
== 1 && fv
== 0)) {
1294 op
->opc
= opc
= (opc
== INDEX_op_movcond_i32
1295 ? INDEX_op_setcond_i32
1296 : INDEX_op_setcond_i64
);
1301 case INDEX_op_add2_i32
:
1302 case INDEX_op_sub2_i32
:
1303 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3])
1304 && arg_is_const(op
->args
[4]) && arg_is_const(op
->args
[5])) {
1305 uint32_t al
= arg_info(op
->args
[2])->val
;
1306 uint32_t ah
= arg_info(op
->args
[3])->val
;
1307 uint32_t bl
= arg_info(op
->args
[4])->val
;
1308 uint32_t bh
= arg_info(op
->args
[5])->val
;
1309 uint64_t a
= ((uint64_t)ah
<< 32) | al
;
1310 uint64_t b
= ((uint64_t)bh
<< 32) | bl
;
1312 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_movi_i32
);
1314 if (opc
== INDEX_op_add2_i32
) {
1322 tcg_opt_gen_movi(s
, op
, rl
, (int32_t)a
);
1323 tcg_opt_gen_movi(s
, op2
, rh
, (int32_t)(a
>> 32));
1328 case INDEX_op_mulu2_i32
:
1329 if (arg_is_const(op
->args
[2]) && arg_is_const(op
->args
[3])) {
1330 uint32_t a
= arg_info(op
->args
[2])->val
;
1331 uint32_t b
= arg_info(op
->args
[3])->val
;
1332 uint64_t r
= (uint64_t)a
* b
;
1334 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_movi_i32
);
1338 tcg_opt_gen_movi(s
, op
, rl
, (int32_t)r
);
1339 tcg_opt_gen_movi(s
, op2
, rh
, (int32_t)(r
>> 32));
1344 case INDEX_op_brcond2_i32
:
1345 tmp
= do_constant_folding_cond2(&op
->args
[0], &op
->args
[2],
1350 bitmap_zero(temps_used
.l
, nb_temps
);
1351 op
->opc
= INDEX_op_br
;
1352 op
->args
[0] = op
->args
[5];
1355 tcg_op_remove(s
, op
);
1357 } else if ((op
->args
[4] == TCG_COND_LT
1358 || op
->args
[4] == TCG_COND_GE
)
1359 && arg_is_const(op
->args
[2])
1360 && arg_info(op
->args
[2])->val
== 0
1361 && arg_is_const(op
->args
[3])
1362 && arg_info(op
->args
[3])->val
== 0) {
1363 /* Simplify LT/GE comparisons vs zero to a single compare
1364 vs the high word of the input. */
1366 bitmap_zero(temps_used
.l
, nb_temps
);
1367 op
->opc
= INDEX_op_brcond_i32
;
1368 op
->args
[0] = op
->args
[1];
1369 op
->args
[1] = op
->args
[3];
1370 op
->args
[2] = op
->args
[4];
1371 op
->args
[3] = op
->args
[5];
1372 } else if (op
->args
[4] == TCG_COND_EQ
) {
1373 /* Simplify EQ comparisons where one of the pairs
1374 can be simplified. */
1375 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1376 op
->args
[0], op
->args
[2],
1379 goto do_brcond_false
;
1380 } else if (tmp
== 1) {
1381 goto do_brcond_high
;
1383 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1384 op
->args
[1], op
->args
[3],
1387 goto do_brcond_false
;
1388 } else if (tmp
!= 1) {
1392 bitmap_zero(temps_used
.l
, nb_temps
);
1393 op
->opc
= INDEX_op_brcond_i32
;
1394 op
->args
[1] = op
->args
[2];
1395 op
->args
[2] = op
->args
[4];
1396 op
->args
[3] = op
->args
[5];
1397 } else if (op
->args
[4] == TCG_COND_NE
) {
1398 /* Simplify NE comparisons where one of the pairs
1399 can be simplified. */
1400 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1401 op
->args
[0], op
->args
[2],
1404 goto do_brcond_high
;
1405 } else if (tmp
== 1) {
1406 goto do_brcond_true
;
1408 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1409 op
->args
[1], op
->args
[3],
1413 } else if (tmp
== 1) {
1414 goto do_brcond_true
;
1422 case INDEX_op_setcond2_i32
:
1423 tmp
= do_constant_folding_cond2(&op
->args
[1], &op
->args
[3],
1427 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1428 } else if ((op
->args
[5] == TCG_COND_LT
1429 || op
->args
[5] == TCG_COND_GE
)
1430 && arg_is_const(op
->args
[3])
1431 && arg_info(op
->args
[3])->val
== 0
1432 && arg_is_const(op
->args
[4])
1433 && arg_info(op
->args
[4])->val
== 0) {
1434 /* Simplify LT/GE comparisons vs zero to a single compare
1435 vs the high word of the input. */
1437 reset_temp(op
->args
[0]);
1438 arg_info(op
->args
[0])->mask
= 1;
1439 op
->opc
= INDEX_op_setcond_i32
;
1440 op
->args
[1] = op
->args
[2];
1441 op
->args
[2] = op
->args
[4];
1442 op
->args
[3] = op
->args
[5];
1443 } else if (op
->args
[5] == TCG_COND_EQ
) {
1444 /* Simplify EQ comparisons where one of the pairs
1445 can be simplified. */
1446 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1447 op
->args
[1], op
->args
[3],
1450 goto do_setcond_const
;
1451 } else if (tmp
== 1) {
1452 goto do_setcond_high
;
1454 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1455 op
->args
[2], op
->args
[4],
1458 goto do_setcond_high
;
1459 } else if (tmp
!= 1) {
1463 reset_temp(op
->args
[0]);
1464 arg_info(op
->args
[0])->mask
= 1;
1465 op
->opc
= INDEX_op_setcond_i32
;
1466 op
->args
[2] = op
->args
[3];
1467 op
->args
[3] = op
->args
[5];
1468 } else if (op
->args
[5] == TCG_COND_NE
) {
1469 /* Simplify NE comparisons where one of the pairs
1470 can be simplified. */
1471 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1472 op
->args
[1], op
->args
[3],
1475 goto do_setcond_high
;
1476 } else if (tmp
== 1) {
1477 goto do_setcond_const
;
1479 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1480 op
->args
[2], op
->args
[4],
1483 goto do_setcond_low
;
1484 } else if (tmp
== 1) {
1485 goto do_setcond_const
;
1494 if (!(op
->args
[nb_oargs
+ nb_iargs
+ 1]
1495 & (TCG_CALL_NO_READ_GLOBALS
| TCG_CALL_NO_WRITE_GLOBALS
))) {
1496 for (i
= 0; i
< nb_globals
; i
++) {
1497 if (test_bit(i
, temps_used
.l
)) {
1498 reset_ts(&s
->temps
[i
]);
1502 goto do_reset_output
;
1506 /* Default case: we know nothing about operation (or were unable
1507 to compute the operation result) so no propagation is done.
1508 We trash everything if the operation is the end of a basic
1509 block, otherwise we only trash the output args. "mask" is
1510 the non-zero bits mask for the first output arg. */
1511 if (def
->flags
& TCG_OPF_BB_END
) {
1512 bitmap_zero(temps_used
.l
, nb_temps
);
1515 for (i
= 0; i
< nb_oargs
; i
++) {
1516 reset_temp(op
->args
[i
]);
1517 /* Save the corresponding known-zero bits mask for the
1518 first output argument (only one supported so far). */
1520 arg_info(op
->args
[i
])->mask
= mask
;
1527 /* Eliminate duplicate and redundant fence instructions. */
1531 /* Merge two barriers of the same type into one,
1532 * or a weaker barrier into a stronger one,
1533 * or two weaker barriers into a stronger one.
1534 * mb X; mb Y => mb X|Y
1535 * mb; strl => mb; st
1536 * ldaq; mb => ld; mb
1537 * ldaq; strl => ld; mb; st
1538 * Other combinations are also merged into a strong
1539 * barrier. This is stricter than specified but for
1540 * the purposes of TCG is better than not optimizing.
1542 prev_mb
->args
[0] |= op
->args
[0];
1543 tcg_op_remove(s
, op
);
1547 /* Opcodes that end the block stop the optimization. */
1548 if ((def
->flags
& TCG_OPF_BB_END
) == 0) {
1552 case INDEX_op_qemu_ld_i32
:
1553 case INDEX_op_qemu_ld_i64
:
1554 case INDEX_op_qemu_st_i32
:
1555 case INDEX_op_qemu_st8_i32
:
1556 case INDEX_op_qemu_st_i64
:
1558 /* Opcodes that touch guest memory stop the optimization. */
1562 } else if (opc
== INDEX_op_mb
) {