2 * Optimizations for Tiny Code Generator for QEMU
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 #include "qemu/osdep.h"
27 #include "qemu-common.h"
28 #include "exec/cpu-common.h"
31 #define CASE_OP_32_64(x) \
32 glue(glue(case INDEX_op_, x), _i32): \
33 glue(glue(case INDEX_op_, x), _i64)
35 struct tcg_temp_info
{
40 tcg_target_ulong mask
;
43 static struct tcg_temp_info temps
[TCG_MAX_TEMPS
];
44 static TCGTempSet temps_used
;
46 static inline bool temp_is_const(TCGArg arg
)
48 return temps
[arg
].is_const
;
51 static inline bool temp_is_copy(TCGArg arg
)
53 return temps
[arg
].next_copy
!= arg
;
56 /* Reset TEMP's state, possibly removing the temp for the list of copies. */
57 static void reset_temp(TCGArg temp
)
59 temps
[temps
[temp
].next_copy
].prev_copy
= temps
[temp
].prev_copy
;
60 temps
[temps
[temp
].prev_copy
].next_copy
= temps
[temp
].next_copy
;
61 temps
[temp
].next_copy
= temp
;
62 temps
[temp
].prev_copy
= temp
;
63 temps
[temp
].is_const
= false;
64 temps
[temp
].mask
= -1;
67 /* Reset all temporaries, given that there are NB_TEMPS of them. */
68 static void reset_all_temps(int nb_temps
)
70 bitmap_zero(temps_used
.l
, nb_temps
);
73 /* Initialize and activate a temporary. */
74 static void init_temp_info(TCGArg temp
)
76 if (!test_bit(temp
, temps_used
.l
)) {
77 temps
[temp
].next_copy
= temp
;
78 temps
[temp
].prev_copy
= temp
;
79 temps
[temp
].is_const
= false;
80 temps
[temp
].mask
= -1;
81 set_bit(temp
, temps_used
.l
);
85 static int op_bits(TCGOpcode op
)
87 const TCGOpDef
*def
= &tcg_op_defs
[op
];
88 return def
->flags
& TCG_OPF_64BIT
? 64 : 32;
91 static TCGOpcode
op_to_mov(TCGOpcode op
)
93 switch (op_bits(op
)) {
95 return INDEX_op_mov_i32
;
97 return INDEX_op_mov_i64
;
99 fprintf(stderr
, "op_to_mov: unexpected return value of "
100 "function op_bits.\n");
105 static TCGOpcode
op_to_movi(TCGOpcode op
)
107 switch (op_bits(op
)) {
109 return INDEX_op_movi_i32
;
111 return INDEX_op_movi_i64
;
113 fprintf(stderr
, "op_to_movi: unexpected return value of "
114 "function op_bits.\n");
119 static TCGArg
find_better_copy(TCGContext
*s
, TCGArg arg
)
121 TCGTemp
*ts
= arg_temp(arg
);
124 /* If this is already a global, we can't do better. */
125 if (ts
->temp_global
) {
129 /* Search for a global first. */
130 for (i
= temps
[arg
].next_copy
; i
!= arg
; i
= temps
[i
].next_copy
) {
131 if (i
< s
->nb_globals
) {
136 /* If it is a temp, search for a temp local. */
137 if (!ts
->temp_local
) {
138 for (i
= temps
[arg
].next_copy
; i
!= arg
; i
= temps
[i
].next_copy
) {
139 if (s
->temps
[i
].temp_local
) {
145 /* Failure to find a better representation, return the same temp. */
149 static bool temps_are_copies(TCGArg arg1
, TCGArg arg2
)
157 if (!temp_is_copy(arg1
) || !temp_is_copy(arg2
)) {
161 for (i
= temps
[arg1
].next_copy
; i
!= arg1
; i
= temps
[i
].next_copy
) {
170 static void tcg_opt_gen_movi(TCGContext
*s
, TCGOp
*op
, TCGArg dst
, TCGArg val
)
172 TCGOpcode new_op
= op_to_movi(op
->opc
);
173 tcg_target_ulong mask
;
178 temps
[dst
].is_const
= true;
179 temps
[dst
].val
= val
;
181 if (TCG_TARGET_REG_BITS
> 32 && new_op
== INDEX_op_movi_i32
) {
182 /* High bits of the destination are now garbage. */
183 mask
|= ~0xffffffffull
;
185 temps
[dst
].mask
= mask
;
191 static void tcg_opt_gen_mov(TCGContext
*s
, TCGOp
*op
, TCGArg dst
, TCGArg src
)
193 if (temps_are_copies(dst
, src
)) {
194 tcg_op_remove(s
, op
);
198 TCGOpcode new_op
= op_to_mov(op
->opc
);
199 tcg_target_ulong mask
;
204 mask
= temps
[src
].mask
;
205 if (TCG_TARGET_REG_BITS
> 32 && new_op
== INDEX_op_mov_i32
) {
206 /* High bits of the destination are now garbage. */
207 mask
|= ~0xffffffffull
;
209 temps
[dst
].mask
= mask
;
211 if (arg_temp(src
)->type
== arg_temp(dst
)->type
) {
212 temps
[dst
].next_copy
= temps
[src
].next_copy
;
213 temps
[dst
].prev_copy
= src
;
214 temps
[temps
[dst
].next_copy
].prev_copy
= dst
;
215 temps
[src
].next_copy
= dst
;
216 temps
[dst
].is_const
= temps
[src
].is_const
;
217 temps
[dst
].val
= temps
[src
].val
;
224 static TCGArg
do_constant_folding_2(TCGOpcode op
, TCGArg x
, TCGArg y
)
247 case INDEX_op_shl_i32
:
248 return (uint32_t)x
<< (y
& 31);
250 case INDEX_op_shl_i64
:
251 return (uint64_t)x
<< (y
& 63);
253 case INDEX_op_shr_i32
:
254 return (uint32_t)x
>> (y
& 31);
256 case INDEX_op_shr_i64
:
257 return (uint64_t)x
>> (y
& 63);
259 case INDEX_op_sar_i32
:
260 return (int32_t)x
>> (y
& 31);
262 case INDEX_op_sar_i64
:
263 return (int64_t)x
>> (y
& 63);
265 case INDEX_op_rotr_i32
:
266 return ror32(x
, y
& 31);
268 case INDEX_op_rotr_i64
:
269 return ror64(x
, y
& 63);
271 case INDEX_op_rotl_i32
:
272 return rol32(x
, y
& 31);
274 case INDEX_op_rotl_i64
:
275 return rol64(x
, y
& 63);
298 case INDEX_op_clz_i32
:
299 return (uint32_t)x
? clz32(x
) : y
;
301 case INDEX_op_clz_i64
:
302 return x
? clz64(x
) : y
;
304 case INDEX_op_ctz_i32
:
305 return (uint32_t)x
? ctz32(x
) : y
;
307 case INDEX_op_ctz_i64
:
308 return x
? ctz64(x
) : y
;
310 case INDEX_op_ctpop_i32
:
313 case INDEX_op_ctpop_i64
:
316 CASE_OP_32_64(ext8s
):
319 CASE_OP_32_64(ext16s
):
322 CASE_OP_32_64(ext8u
):
325 CASE_OP_32_64(ext16u
):
328 case INDEX_op_ext_i32_i64
:
329 case INDEX_op_ext32s_i64
:
332 case INDEX_op_extu_i32_i64
:
333 case INDEX_op_extrl_i64_i32
:
334 case INDEX_op_ext32u_i64
:
337 case INDEX_op_extrh_i64_i32
:
338 return (uint64_t)x
>> 32;
340 case INDEX_op_muluh_i32
:
341 return ((uint64_t)(uint32_t)x
* (uint32_t)y
) >> 32;
342 case INDEX_op_mulsh_i32
:
343 return ((int64_t)(int32_t)x
* (int32_t)y
) >> 32;
345 case INDEX_op_muluh_i64
:
346 mulu64(&l64
, &h64
, x
, y
);
348 case INDEX_op_mulsh_i64
:
349 muls64(&l64
, &h64
, x
, y
);
352 case INDEX_op_div_i32
:
353 /* Avoid crashing on divide by zero, otherwise undefined. */
354 return (int32_t)x
/ ((int32_t)y
? : 1);
355 case INDEX_op_divu_i32
:
356 return (uint32_t)x
/ ((uint32_t)y
? : 1);
357 case INDEX_op_div_i64
:
358 return (int64_t)x
/ ((int64_t)y
? : 1);
359 case INDEX_op_divu_i64
:
360 return (uint64_t)x
/ ((uint64_t)y
? : 1);
362 case INDEX_op_rem_i32
:
363 return (int32_t)x
% ((int32_t)y
? : 1);
364 case INDEX_op_remu_i32
:
365 return (uint32_t)x
% ((uint32_t)y
? : 1);
366 case INDEX_op_rem_i64
:
367 return (int64_t)x
% ((int64_t)y
? : 1);
368 case INDEX_op_remu_i64
:
369 return (uint64_t)x
% ((uint64_t)y
? : 1);
373 "Unrecognized operation %d in do_constant_folding.\n", op
);
378 static TCGArg
do_constant_folding(TCGOpcode op
, TCGArg x
, TCGArg y
)
380 TCGArg res
= do_constant_folding_2(op
, x
, y
);
381 if (op_bits(op
) == 32) {
387 static bool do_constant_folding_cond_32(uint32_t x
, uint32_t y
, TCGCond c
)
395 return (int32_t)x
< (int32_t)y
;
397 return (int32_t)x
>= (int32_t)y
;
399 return (int32_t)x
<= (int32_t)y
;
401 return (int32_t)x
> (int32_t)y
;
415 static bool do_constant_folding_cond_64(uint64_t x
, uint64_t y
, TCGCond c
)
423 return (int64_t)x
< (int64_t)y
;
425 return (int64_t)x
>= (int64_t)y
;
427 return (int64_t)x
<= (int64_t)y
;
429 return (int64_t)x
> (int64_t)y
;
443 static bool do_constant_folding_cond_eq(TCGCond c
)
463 /* Return 2 if the condition can't be simplified, and the result
464 of the condition (0 or 1) if it can */
465 static TCGArg
do_constant_folding_cond(TCGOpcode op
, TCGArg x
,
468 if (temp_is_const(x
) && temp_is_const(y
)) {
469 switch (op_bits(op
)) {
471 return do_constant_folding_cond_32(temps
[x
].val
, temps
[y
].val
, c
);
473 return do_constant_folding_cond_64(temps
[x
].val
, temps
[y
].val
, c
);
477 } else if (temps_are_copies(x
, y
)) {
478 return do_constant_folding_cond_eq(c
);
479 } else if (temp_is_const(y
) && temps
[y
].val
== 0) {
492 /* Return 2 if the condition can't be simplified, and the result
493 of the condition (0 or 1) if it can */
494 static TCGArg
do_constant_folding_cond2(TCGArg
*p1
, TCGArg
*p2
, TCGCond c
)
496 TCGArg al
= p1
[0], ah
= p1
[1];
497 TCGArg bl
= p2
[0], bh
= p2
[1];
499 if (temp_is_const(bl
) && temp_is_const(bh
)) {
500 uint64_t b
= ((uint64_t)temps
[bh
].val
<< 32) | (uint32_t)temps
[bl
].val
;
502 if (temp_is_const(al
) && temp_is_const(ah
)) {
504 a
= ((uint64_t)temps
[ah
].val
<< 32) | (uint32_t)temps
[al
].val
;
505 return do_constant_folding_cond_64(a
, b
, c
);
518 if (temps_are_copies(al
, bl
) && temps_are_copies(ah
, bh
)) {
519 return do_constant_folding_cond_eq(c
);
524 static bool swap_commutative(TCGArg dest
, TCGArg
*p1
, TCGArg
*p2
)
526 TCGArg a1
= *p1
, a2
= *p2
;
528 sum
+= temp_is_const(a1
);
529 sum
-= temp_is_const(a2
);
531 /* Prefer the constant in second argument, and then the form
532 op a, a, b, which is better handled on non-RISC hosts. */
533 if (sum
> 0 || (sum
== 0 && dest
== a2
)) {
541 static bool swap_commutative2(TCGArg
*p1
, TCGArg
*p2
)
544 sum
+= temp_is_const(p1
[0]);
545 sum
+= temp_is_const(p1
[1]);
546 sum
-= temp_is_const(p2
[0]);
547 sum
-= temp_is_const(p2
[1]);
550 t
= p1
[0], p1
[0] = p2
[0], p2
[0] = t
;
551 t
= p1
[1], p1
[1] = p2
[1], p2
[1] = t
;
557 /* Propagate constants and copies, fold constant expressions. */
558 void tcg_optimize(TCGContext
*s
)
560 int oi
, oi_next
, nb_temps
, nb_globals
;
561 TCGOp
*prev_mb
= NULL
;
563 /* Array VALS has an element for each temp.
564 If this temp holds a constant then its value is kept in VALS' element.
565 If this temp is a copy of other ones then the other copies are
566 available through the doubly linked circular list. */
568 nb_temps
= s
->nb_temps
;
569 nb_globals
= s
->nb_globals
;
570 reset_all_temps(nb_temps
);
572 for (oi
= s
->gen_op_buf
[0].next
; oi
!= 0; oi
= oi_next
) {
573 tcg_target_ulong mask
, partmask
, affected
;
574 int nb_oargs
, nb_iargs
, i
;
577 TCGOp
* const op
= &s
->gen_op_buf
[oi
];
578 TCGOpcode opc
= op
->opc
;
579 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
583 /* Count the arguments, and initialize the temps that are
585 if (opc
== INDEX_op_call
) {
586 nb_oargs
= op
->callo
;
587 nb_iargs
= op
->calli
;
588 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
590 if (tmp
!= TCG_CALL_DUMMY_ARG
) {
595 nb_oargs
= def
->nb_oargs
;
596 nb_iargs
= def
->nb_iargs
;
597 for (i
= 0; i
< nb_oargs
+ nb_iargs
; i
++) {
598 init_temp_info(op
->args
[i
]);
602 /* Do copy propagation */
603 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
604 if (temp_is_copy(op
->args
[i
])) {
605 op
->args
[i
] = find_better_copy(s
, op
->args
[i
]);
609 /* For commutative operations make constant second argument */
619 CASE_OP_32_64(muluh
):
620 CASE_OP_32_64(mulsh
):
621 swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2]);
623 CASE_OP_32_64(brcond
):
624 if (swap_commutative(-1, &op
->args
[0], &op
->args
[1])) {
625 op
->args
[2] = tcg_swap_cond(op
->args
[2]);
628 CASE_OP_32_64(setcond
):
629 if (swap_commutative(op
->args
[0], &op
->args
[1], &op
->args
[2])) {
630 op
->args
[3] = tcg_swap_cond(op
->args
[3]);
633 CASE_OP_32_64(movcond
):
634 if (swap_commutative(-1, &op
->args
[1], &op
->args
[2])) {
635 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
637 /* For movcond, we canonicalize the "false" input reg to match
638 the destination reg so that the tcg backend can implement
639 a "move if true" operation. */
640 if (swap_commutative(op
->args
[0], &op
->args
[4], &op
->args
[3])) {
641 op
->args
[5] = tcg_invert_cond(op
->args
[5]);
645 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[4]);
646 swap_commutative(op
->args
[1], &op
->args
[3], &op
->args
[5]);
648 CASE_OP_32_64(mulu2
):
649 CASE_OP_32_64(muls2
):
650 swap_commutative(op
->args
[0], &op
->args
[2], &op
->args
[3]);
652 case INDEX_op_brcond2_i32
:
653 if (swap_commutative2(&op
->args
[0], &op
->args
[2])) {
654 op
->args
[4] = tcg_swap_cond(op
->args
[4]);
657 case INDEX_op_setcond2_i32
:
658 if (swap_commutative2(&op
->args
[1], &op
->args
[3])) {
659 op
->args
[5] = tcg_swap_cond(op
->args
[5]);
666 /* Simplify expressions for "shift/rot r, 0, a => movi r, 0",
667 and "sub r, 0, a => neg r, a" case. */
674 if (temp_is_const(op
->args
[1]) && temps
[op
->args
[1]].val
== 0) {
675 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
684 if (temp_is_const(op
->args
[2])) {
685 /* Proceed with possible constant folding. */
688 if (opc
== INDEX_op_sub_i32
) {
689 neg_op
= INDEX_op_neg_i32
;
690 have_neg
= TCG_TARGET_HAS_neg_i32
;
692 neg_op
= INDEX_op_neg_i64
;
693 have_neg
= TCG_TARGET_HAS_neg_i64
;
698 if (temp_is_const(op
->args
[1])
699 && temps
[op
->args
[1]].val
== 0) {
701 reset_temp(op
->args
[0]);
702 op
->args
[1] = op
->args
[2];
709 if (!temp_is_const(op
->args
[1])
710 && temp_is_const(op
->args
[2])
711 && temps
[op
->args
[2]].val
== -1) {
717 if (!temp_is_const(op
->args
[1])
718 && temp_is_const(op
->args
[2])
719 && temps
[op
->args
[2]].val
== 0) {
725 if (!temp_is_const(op
->args
[2])
726 && temp_is_const(op
->args
[1])
727 && temps
[op
->args
[1]].val
== -1) {
734 if (!temp_is_const(op
->args
[2])
735 && temp_is_const(op
->args
[1])
736 && temps
[op
->args
[1]].val
== 0) {
746 if (def
->flags
& TCG_OPF_64BIT
) {
747 not_op
= INDEX_op_not_i64
;
748 have_not
= TCG_TARGET_HAS_not_i64
;
750 not_op
= INDEX_op_not_i32
;
751 have_not
= TCG_TARGET_HAS_not_i32
;
757 reset_temp(op
->args
[0]);
758 op
->args
[1] = op
->args
[i
];
765 /* Simplify expression for "op r, a, const => mov r, a" cases */
777 if (!temp_is_const(op
->args
[1])
778 && temp_is_const(op
->args
[2])
779 && temps
[op
->args
[2]].val
== 0) {
780 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
787 if (!temp_is_const(op
->args
[1])
788 && temp_is_const(op
->args
[2])
789 && temps
[op
->args
[2]].val
== -1) {
790 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
798 /* Simplify using known-zero bits. Currently only ops with a single
799 output argument is supported. */
803 CASE_OP_32_64(ext8s
):
804 if ((temps
[op
->args
[1]].mask
& 0x80) != 0) {
807 CASE_OP_32_64(ext8u
):
810 CASE_OP_32_64(ext16s
):
811 if ((temps
[op
->args
[1]].mask
& 0x8000) != 0) {
814 CASE_OP_32_64(ext16u
):
817 case INDEX_op_ext32s_i64
:
818 if ((temps
[op
->args
[1]].mask
& 0x80000000) != 0) {
821 case INDEX_op_ext32u_i64
:
826 mask
= temps
[op
->args
[2]].mask
;
827 if (temp_is_const(op
->args
[2])) {
829 affected
= temps
[op
->args
[1]].mask
& ~mask
;
831 mask
= temps
[op
->args
[1]].mask
& mask
;
834 case INDEX_op_ext_i32_i64
:
835 if ((temps
[op
->args
[1]].mask
& 0x80000000) != 0) {
838 case INDEX_op_extu_i32_i64
:
839 /* We do not compute affected as it is a size changing op. */
840 mask
= (uint32_t)temps
[op
->args
[1]].mask
;
844 /* Known-zeros does not imply known-ones. Therefore unless
845 op->args[2] is constant, we can't infer anything from it. */
846 if (temp_is_const(op
->args
[2])) {
847 mask
= ~temps
[op
->args
[2]].mask
;
850 /* But we certainly know nothing outside op->args[1] may be set. */
851 mask
= temps
[op
->args
[1]].mask
;
854 case INDEX_op_sar_i32
:
855 if (temp_is_const(op
->args
[2])) {
856 tmp
= temps
[op
->args
[2]].val
& 31;
857 mask
= (int32_t)temps
[op
->args
[1]].mask
>> tmp
;
860 case INDEX_op_sar_i64
:
861 if (temp_is_const(op
->args
[2])) {
862 tmp
= temps
[op
->args
[2]].val
& 63;
863 mask
= (int64_t)temps
[op
->args
[1]].mask
>> tmp
;
867 case INDEX_op_shr_i32
:
868 if (temp_is_const(op
->args
[2])) {
869 tmp
= temps
[op
->args
[2]].val
& 31;
870 mask
= (uint32_t)temps
[op
->args
[1]].mask
>> tmp
;
873 case INDEX_op_shr_i64
:
874 if (temp_is_const(op
->args
[2])) {
875 tmp
= temps
[op
->args
[2]].val
& 63;
876 mask
= (uint64_t)temps
[op
->args
[1]].mask
>> tmp
;
880 case INDEX_op_extrl_i64_i32
:
881 mask
= (uint32_t)temps
[op
->args
[1]].mask
;
883 case INDEX_op_extrh_i64_i32
:
884 mask
= (uint64_t)temps
[op
->args
[1]].mask
>> 32;
888 if (temp_is_const(op
->args
[2])) {
889 tmp
= temps
[op
->args
[2]].val
& (TCG_TARGET_REG_BITS
- 1);
890 mask
= temps
[op
->args
[1]].mask
<< tmp
;
895 /* Set to 1 all bits to the left of the rightmost. */
896 mask
= -(temps
[op
->args
[1]].mask
& -temps
[op
->args
[1]].mask
);
899 CASE_OP_32_64(deposit
):
900 mask
= deposit64(temps
[op
->args
[1]].mask
, op
->args
[3],
901 op
->args
[4], temps
[op
->args
[2]].mask
);
904 CASE_OP_32_64(extract
):
905 mask
= extract64(temps
[op
->args
[1]].mask
, op
->args
[2], op
->args
[3]);
906 if (op
->args
[2] == 0) {
907 affected
= temps
[op
->args
[1]].mask
& ~mask
;
910 CASE_OP_32_64(sextract
):
911 mask
= sextract64(temps
[op
->args
[1]].mask
,
912 op
->args
[2], op
->args
[3]);
913 if (op
->args
[2] == 0 && (tcg_target_long
)mask
>= 0) {
914 affected
= temps
[op
->args
[1]].mask
& ~mask
;
920 mask
= temps
[op
->args
[1]].mask
| temps
[op
->args
[2]].mask
;
923 case INDEX_op_clz_i32
:
924 case INDEX_op_ctz_i32
:
925 mask
= temps
[op
->args
[2]].mask
| 31;
928 case INDEX_op_clz_i64
:
929 case INDEX_op_ctz_i64
:
930 mask
= temps
[op
->args
[2]].mask
| 63;
933 case INDEX_op_ctpop_i32
:
936 case INDEX_op_ctpop_i64
:
940 CASE_OP_32_64(setcond
):
941 case INDEX_op_setcond2_i32
:
945 CASE_OP_32_64(movcond
):
946 mask
= temps
[op
->args
[3]].mask
| temps
[op
->args
[4]].mask
;
952 CASE_OP_32_64(ld16u
):
955 case INDEX_op_ld32u_i64
:
959 CASE_OP_32_64(qemu_ld
):
961 TCGMemOpIdx oi
= op
->args
[nb_oargs
+ nb_iargs
];
962 TCGMemOp mop
= get_memop(oi
);
963 if (!(mop
& MO_SIGN
)) {
964 mask
= (2ULL << ((8 << (mop
& MO_SIZE
)) - 1)) - 1;
973 /* 32-bit ops generate 32-bit results. For the result is zero test
974 below, we can ignore high bits, but for further optimizations we
975 need to record that the high bits contain garbage. */
977 if (!(def
->flags
& TCG_OPF_64BIT
)) {
978 mask
|= ~(tcg_target_ulong
)0xffffffffu
;
979 partmask
&= 0xffffffffu
;
980 affected
&= 0xffffffffu
;
984 tcg_debug_assert(nb_oargs
== 1);
985 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
989 tcg_debug_assert(nb_oargs
== 1);
990 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
994 /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
998 CASE_OP_32_64(muluh
):
999 CASE_OP_32_64(mulsh
):
1000 if ((temp_is_const(op
->args
[2]) && temps
[op
->args
[2]].val
== 0)) {
1001 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
1009 /* Simplify expression for "op r, a, a => mov r, a" cases */
1013 if (temps_are_copies(op
->args
[1], op
->args
[2])) {
1014 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1022 /* Simplify expression for "op r, a, a => movi r, 0" cases */
1024 CASE_OP_32_64(andc
):
1027 if (temps_are_copies(op
->args
[1], op
->args
[2])) {
1028 tcg_opt_gen_movi(s
, op
, op
->args
[0], 0);
1036 /* Propagate constants through copy operations and do constant
1037 folding. Constants will be substituted to arguments by register
1038 allocator where needed and possible. Also detect copies. */
1041 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[1]);
1043 CASE_OP_32_64(movi
):
1044 tcg_opt_gen_movi(s
, op
, op
->args
[0], op
->args
[1]);
1049 CASE_OP_32_64(ext8s
):
1050 CASE_OP_32_64(ext8u
):
1051 CASE_OP_32_64(ext16s
):
1052 CASE_OP_32_64(ext16u
):
1053 CASE_OP_32_64(ctpop
):
1054 case INDEX_op_ext32s_i64
:
1055 case INDEX_op_ext32u_i64
:
1056 case INDEX_op_ext_i32_i64
:
1057 case INDEX_op_extu_i32_i64
:
1058 case INDEX_op_extrl_i64_i32
:
1059 case INDEX_op_extrh_i64_i32
:
1060 if (temp_is_const(op
->args
[1])) {
1061 tmp
= do_constant_folding(opc
, temps
[op
->args
[1]].val
, 0);
1062 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1076 CASE_OP_32_64(rotl
):
1077 CASE_OP_32_64(rotr
):
1078 CASE_OP_32_64(andc
):
1081 CASE_OP_32_64(nand
):
1083 CASE_OP_32_64(muluh
):
1084 CASE_OP_32_64(mulsh
):
1086 CASE_OP_32_64(divu
):
1088 CASE_OP_32_64(remu
):
1089 if (temp_is_const(op
->args
[1]) && temp_is_const(op
->args
[2])) {
1090 tmp
= do_constant_folding(opc
, temps
[op
->args
[1]].val
,
1091 temps
[op
->args
[2]].val
);
1092 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1099 if (temp_is_const(op
->args
[1])) {
1100 TCGArg v
= temps
[op
->args
[1]].val
;
1102 tmp
= do_constant_folding(opc
, v
, 0);
1103 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1105 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[2]);
1111 CASE_OP_32_64(deposit
):
1112 if (temp_is_const(op
->args
[1]) && temp_is_const(op
->args
[2])) {
1113 tmp
= deposit64(temps
[op
->args
[1]].val
, op
->args
[3],
1114 op
->args
[4], temps
[op
->args
[2]].val
);
1115 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1120 CASE_OP_32_64(extract
):
1121 if (temp_is_const(op
->args
[1])) {
1122 tmp
= extract64(temps
[op
->args
[1]].val
,
1123 op
->args
[2], op
->args
[3]);
1124 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1129 CASE_OP_32_64(sextract
):
1130 if (temp_is_const(op
->args
[1])) {
1131 tmp
= sextract64(temps
[op
->args
[1]].val
,
1132 op
->args
[2], op
->args
[3]);
1133 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1138 CASE_OP_32_64(setcond
):
1139 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1140 op
->args
[2], op
->args
[3]);
1142 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1147 CASE_OP_32_64(brcond
):
1148 tmp
= do_constant_folding_cond(opc
, op
->args
[0],
1149 op
->args
[1], op
->args
[2]);
1152 reset_all_temps(nb_temps
);
1153 op
->opc
= INDEX_op_br
;
1154 op
->args
[0] = op
->args
[3];
1156 tcg_op_remove(s
, op
);
1162 CASE_OP_32_64(movcond
):
1163 tmp
= do_constant_folding_cond(opc
, op
->args
[1],
1164 op
->args
[2], op
->args
[5]);
1166 tcg_opt_gen_mov(s
, op
, op
->args
[0], op
->args
[4-tmp
]);
1169 if (temp_is_const(op
->args
[3]) && temp_is_const(op
->args
[4])) {
1170 tcg_target_ulong tv
= temps
[op
->args
[3]].val
;
1171 tcg_target_ulong fv
= temps
[op
->args
[4]].val
;
1172 TCGCond cond
= op
->args
[5];
1173 if (fv
== 1 && tv
== 0) {
1174 cond
= tcg_invert_cond(cond
);
1175 } else if (!(tv
== 1 && fv
== 0)) {
1179 op
->opc
= opc
= (opc
== INDEX_op_movcond_i32
1180 ? INDEX_op_setcond_i32
1181 : INDEX_op_setcond_i64
);
1186 case INDEX_op_add2_i32
:
1187 case INDEX_op_sub2_i32
:
1188 if (temp_is_const(op
->args
[2]) && temp_is_const(op
->args
[3])
1189 && temp_is_const(op
->args
[4]) && temp_is_const(op
->args
[5])) {
1190 uint32_t al
= temps
[op
->args
[2]].val
;
1191 uint32_t ah
= temps
[op
->args
[3]].val
;
1192 uint32_t bl
= temps
[op
->args
[4]].val
;
1193 uint32_t bh
= temps
[op
->args
[5]].val
;
1194 uint64_t a
= ((uint64_t)ah
<< 32) | al
;
1195 uint64_t b
= ((uint64_t)bh
<< 32) | bl
;
1197 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_movi_i32
, 2);
1199 if (opc
== INDEX_op_add2_i32
) {
1207 tcg_opt_gen_movi(s
, op
, rl
, (int32_t)a
);
1208 tcg_opt_gen_movi(s
, op2
, rh
, (int32_t)(a
>> 32));
1210 /* We've done all we need to do with the movi. Skip it. */
1211 oi_next
= op2
->next
;
1216 case INDEX_op_mulu2_i32
:
1217 if (temp_is_const(op
->args
[2]) && temp_is_const(op
->args
[3])) {
1218 uint32_t a
= temps
[op
->args
[2]].val
;
1219 uint32_t b
= temps
[op
->args
[3]].val
;
1220 uint64_t r
= (uint64_t)a
* b
;
1222 TCGOp
*op2
= tcg_op_insert_before(s
, op
, INDEX_op_movi_i32
, 2);
1226 tcg_opt_gen_movi(s
, op
, rl
, (int32_t)r
);
1227 tcg_opt_gen_movi(s
, op2
, rh
, (int32_t)(r
>> 32));
1229 /* We've done all we need to do with the movi. Skip it. */
1230 oi_next
= op2
->next
;
1235 case INDEX_op_brcond2_i32
:
1236 tmp
= do_constant_folding_cond2(&op
->args
[0], &op
->args
[2],
1241 reset_all_temps(nb_temps
);
1242 op
->opc
= INDEX_op_br
;
1243 op
->args
[0] = op
->args
[5];
1246 tcg_op_remove(s
, op
);
1248 } else if ((op
->args
[4] == TCG_COND_LT
1249 || op
->args
[4] == TCG_COND_GE
)
1250 && temp_is_const(op
->args
[2])
1251 && temps
[op
->args
[2]].val
== 0
1252 && temp_is_const(op
->args
[3])
1253 && temps
[op
->args
[3]].val
== 0) {
1254 /* Simplify LT/GE comparisons vs zero to a single compare
1255 vs the high word of the input. */
1257 reset_all_temps(nb_temps
);
1258 op
->opc
= INDEX_op_brcond_i32
;
1259 op
->args
[0] = op
->args
[1];
1260 op
->args
[1] = op
->args
[3];
1261 op
->args
[2] = op
->args
[4];
1262 op
->args
[3] = op
->args
[5];
1263 } else if (op
->args
[4] == TCG_COND_EQ
) {
1264 /* Simplify EQ comparisons where one of the pairs
1265 can be simplified. */
1266 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1267 op
->args
[0], op
->args
[2],
1270 goto do_brcond_false
;
1271 } else if (tmp
== 1) {
1272 goto do_brcond_high
;
1274 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1275 op
->args
[1], op
->args
[3],
1278 goto do_brcond_false
;
1279 } else if (tmp
!= 1) {
1283 reset_all_temps(nb_temps
);
1284 op
->opc
= INDEX_op_brcond_i32
;
1285 op
->args
[1] = op
->args
[2];
1286 op
->args
[2] = op
->args
[4];
1287 op
->args
[3] = op
->args
[5];
1288 } else if (op
->args
[4] == TCG_COND_NE
) {
1289 /* Simplify NE comparisons where one of the pairs
1290 can be simplified. */
1291 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1292 op
->args
[0], op
->args
[2],
1295 goto do_brcond_high
;
1296 } else if (tmp
== 1) {
1297 goto do_brcond_true
;
1299 tmp
= do_constant_folding_cond(INDEX_op_brcond_i32
,
1300 op
->args
[1], op
->args
[3],
1304 } else if (tmp
== 1) {
1305 goto do_brcond_true
;
1313 case INDEX_op_setcond2_i32
:
1314 tmp
= do_constant_folding_cond2(&op
->args
[1], &op
->args
[3],
1318 tcg_opt_gen_movi(s
, op
, op
->args
[0], tmp
);
1319 } else if ((op
->args
[5] == TCG_COND_LT
1320 || op
->args
[5] == TCG_COND_GE
)
1321 && temp_is_const(op
->args
[3])
1322 && temps
[op
->args
[3]].val
== 0
1323 && temp_is_const(op
->args
[4])
1324 && temps
[op
->args
[4]].val
== 0) {
1325 /* Simplify LT/GE comparisons vs zero to a single compare
1326 vs the high word of the input. */
1328 reset_temp(op
->args
[0]);
1329 temps
[op
->args
[0]].mask
= 1;
1330 op
->opc
= INDEX_op_setcond_i32
;
1331 op
->args
[1] = op
->args
[2];
1332 op
->args
[2] = op
->args
[4];
1333 op
->args
[3] = op
->args
[5];
1334 } else if (op
->args
[5] == TCG_COND_EQ
) {
1335 /* Simplify EQ comparisons where one of the pairs
1336 can be simplified. */
1337 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1338 op
->args
[1], op
->args
[3],
1341 goto do_setcond_const
;
1342 } else if (tmp
== 1) {
1343 goto do_setcond_high
;
1345 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1346 op
->args
[2], op
->args
[4],
1349 goto do_setcond_high
;
1350 } else if (tmp
!= 1) {
1354 reset_temp(op
->args
[0]);
1355 temps
[op
->args
[0]].mask
= 1;
1356 op
->opc
= INDEX_op_setcond_i32
;
1357 op
->args
[2] = op
->args
[3];
1358 op
->args
[3] = op
->args
[5];
1359 } else if (op
->args
[5] == TCG_COND_NE
) {
1360 /* Simplify NE comparisons where one of the pairs
1361 can be simplified. */
1362 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1363 op
->args
[1], op
->args
[3],
1366 goto do_setcond_high
;
1367 } else if (tmp
== 1) {
1368 goto do_setcond_const
;
1370 tmp
= do_constant_folding_cond(INDEX_op_setcond_i32
,
1371 op
->args
[2], op
->args
[4],
1374 goto do_setcond_low
;
1375 } else if (tmp
== 1) {
1376 goto do_setcond_const
;
1385 if (!(op
->args
[nb_oargs
+ nb_iargs
+ 1]
1386 & (TCG_CALL_NO_READ_GLOBALS
| TCG_CALL_NO_WRITE_GLOBALS
))) {
1387 for (i
= 0; i
< nb_globals
; i
++) {
1388 if (test_bit(i
, temps_used
.l
)) {
1393 goto do_reset_output
;
1397 /* Default case: we know nothing about operation (or were unable
1398 to compute the operation result) so no propagation is done.
1399 We trash everything if the operation is the end of a basic
1400 block, otherwise we only trash the output args. "mask" is
1401 the non-zero bits mask for the first output arg. */
1402 if (def
->flags
& TCG_OPF_BB_END
) {
1403 reset_all_temps(nb_temps
);
1406 for (i
= 0; i
< nb_oargs
; i
++) {
1407 reset_temp(op
->args
[i
]);
1408 /* Save the corresponding known-zero bits mask for the
1409 first output argument (only one supported so far). */
1411 temps
[op
->args
[i
]].mask
= mask
;
1418 /* Eliminate duplicate and redundant fence instructions. */
1422 /* Merge two barriers of the same type into one,
1423 * or a weaker barrier into a stronger one,
1424 * or two weaker barriers into a stronger one.
1425 * mb X; mb Y => mb X|Y
1426 * mb; strl => mb; st
1427 * ldaq; mb => ld; mb
1428 * ldaq; strl => ld; mb; st
1429 * Other combinations are also merged into a strong
1430 * barrier. This is stricter than specified but for
1431 * the purposes of TCG is better than not optimizing.
1433 prev_mb
->args
[0] |= op
->args
[0];
1434 tcg_op_remove(s
, op
);
1438 /* Opcodes that end the block stop the optimization. */
1439 if ((def
->flags
& TCG_OPF_BB_END
) == 0) {
1443 case INDEX_op_qemu_ld_i32
:
1444 case INDEX_op_qemu_ld_i64
:
1445 case INDEX_op_qemu_st_i32
:
1446 case INDEX_op_qemu_st_i64
:
1448 /* Opcodes that touch guest memory stop the optimization. */
1452 } else if (opc
== INDEX_op_mb
) {