2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38 #include "qemu/cacheflush.h"
39 #include "qemu/cacheinfo.h"
41 /* Note: the long term plan is to reduce the dependencies on the QEMU
42 CPU definitions. Currently they are used for qemu_ld/st
44 #define NO_CPU_IO_DEFS
46 #include "exec/exec-all.h"
47 #include "tcg/tcg-op.h"
49 #if UINTPTR_MAX == UINT32_MAX
50 # define ELF_CLASS ELFCLASS32
52 # define ELF_CLASS ELFCLASS64
55 # define ELF_DATA ELFDATA2MSB
57 # define ELF_DATA ELFDATA2LSB
62 #include "tcg/tcg-ldst.h"
63 #include "tcg-internal.h"
64 #include "accel/tcg/perf.h"
66 /* Forward declarations for functions declared in tcg-target.c.inc and
68 static void tcg_target_init(TCGContext
*s
);
69 static void tcg_target_qemu_prologue(TCGContext
*s
);
70 static bool patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
71 intptr_t value
, intptr_t addend
);
73 /* The CIE and FDE header definitions will be common to all hosts. */
75 uint32_t len
__attribute__((aligned((sizeof(void *)))));
81 uint8_t return_column
;
84 typedef struct QEMU_PACKED
{
85 uint32_t len
__attribute__((aligned((sizeof(void *)))));
89 } DebugFrameFDEHeader
;
91 typedef struct QEMU_PACKED
{
93 DebugFrameFDEHeader fde
;
96 static void tcg_register_jit_int(const void *buf
, size_t size
,
97 const void *debug_frame
,
98 size_t debug_frame_size
)
99 __attribute__((unused
));
101 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
102 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
104 static bool tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
105 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
106 TCGReg ret
, tcg_target_long arg
);
107 static void tcg_out_exit_tb(TCGContext
*s
, uintptr_t arg
);
108 static void tcg_out_goto_tb(TCGContext
*s
, int which
);
109 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
,
110 const TCGArg args
[TCG_MAX_OP_ARGS
],
111 const int const_args
[TCG_MAX_OP_ARGS
]);
112 #if TCG_TARGET_MAYBE_vec
113 static bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
114 TCGReg dst
, TCGReg src
);
115 static bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
116 TCGReg dst
, TCGReg base
, intptr_t offset
);
117 static void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
118 TCGReg dst
, int64_t arg
);
119 static void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
,
120 unsigned vecl
, unsigned vece
,
121 const TCGArg args
[TCG_MAX_OP_ARGS
],
122 const int const_args
[TCG_MAX_OP_ARGS
]);
124 static inline bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
125 TCGReg dst
, TCGReg src
)
127 g_assert_not_reached();
129 static inline bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
130 TCGReg dst
, TCGReg base
, intptr_t offset
)
132 g_assert_not_reached();
134 static inline void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
135 TCGReg dst
, int64_t arg
)
137 g_assert_not_reached();
139 static inline void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
,
140 unsigned vecl
, unsigned vece
,
141 const TCGArg args
[TCG_MAX_OP_ARGS
],
142 const int const_args
[TCG_MAX_OP_ARGS
])
144 g_assert_not_reached();
147 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
149 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
150 TCGReg base
, intptr_t ofs
);
151 static void tcg_out_call(TCGContext
*s
, const tcg_insn_unit
*target
,
152 const TCGHelperInfo
*info
);
153 static bool tcg_target_const_match(int64_t val
, TCGType type
, int ct
);
154 #ifdef TCG_TARGET_NEED_LDST_LABELS
155 static int tcg_out_ldst_finalize(TCGContext
*s
);
158 TCGContext tcg_init_ctx
;
159 __thread TCGContext
*tcg_ctx
;
161 TCGContext
**tcg_ctxs
;
162 unsigned int tcg_cur_ctxs
;
163 unsigned int tcg_max_ctxs
;
164 TCGv_env cpu_env
= 0;
165 const void *tcg_code_gen_epilogue
;
166 uintptr_t tcg_splitwx_diff
;
168 #ifndef CONFIG_TCG_INTERPRETER
169 tcg_prologue_fn
*tcg_qemu_tb_exec
;
172 static TCGRegSet tcg_target_available_regs
[TCG_TYPE_COUNT
];
173 static TCGRegSet tcg_target_call_clobber_regs
;
175 #if TCG_TARGET_INSN_UNIT_SIZE == 1
176 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
181 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
188 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
189 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
191 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
194 tcg_insn_unit
*p
= s
->code_ptr
;
195 memcpy(p
, &v
, sizeof(v
));
196 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
200 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
203 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
206 memcpy(p
, &v
, sizeof(v
));
211 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
212 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
214 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
217 tcg_insn_unit
*p
= s
->code_ptr
;
218 memcpy(p
, &v
, sizeof(v
));
219 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
223 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
226 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
229 memcpy(p
, &v
, sizeof(v
));
234 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
235 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
237 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
240 tcg_insn_unit
*p
= s
->code_ptr
;
241 memcpy(p
, &v
, sizeof(v
));
242 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
246 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
249 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
252 memcpy(p
, &v
, sizeof(v
));
257 /* label relocation processing */
259 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
260 TCGLabel
*l
, intptr_t addend
)
262 TCGRelocation
*r
= tcg_malloc(sizeof(TCGRelocation
));
267 QSIMPLEQ_INSERT_TAIL(&l
->relocs
, r
, next
);
270 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
)
272 tcg_debug_assert(!l
->has_value
);
274 l
->u
.value_ptr
= tcg_splitwx_to_rx(s
->code_ptr
);
277 TCGLabel
*gen_new_label(void)
279 TCGContext
*s
= tcg_ctx
;
280 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
282 memset(l
, 0, sizeof(TCGLabel
));
283 l
->id
= s
->nb_labels
++;
284 QSIMPLEQ_INIT(&l
->relocs
);
286 QSIMPLEQ_INSERT_TAIL(&s
->labels
, l
, next
);
291 static bool tcg_resolve_relocs(TCGContext
*s
)
295 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
297 uintptr_t value
= l
->u
.value
;
299 QSIMPLEQ_FOREACH(r
, &l
->relocs
, next
) {
300 if (!patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
)) {
308 static void set_jmp_reset_offset(TCGContext
*s
, int which
)
311 * We will check for overflow at the end of the opcode loop in
312 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
314 s
->gen_tb
->jmp_reset_offset
[which
] = tcg_current_code_size(s
);
317 static void G_GNUC_UNUSED
set_jmp_insn_offset(TCGContext
*s
, int which
)
320 * We will check for overflow at the end of the opcode loop in
321 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
323 s
->gen_tb
->jmp_insn_offset
[which
] = tcg_current_code_size(s
);
326 static uintptr_t G_GNUC_UNUSED
get_jmp_target_addr(TCGContext
*s
, int which
)
329 * Return the read-execute version of the pointer, for the benefit
330 * of any pc-relative addressing mode.
332 return (uintptr_t)tcg_splitwx_to_rx(&s
->gen_tb
->jmp_target_addr
[which
]);
335 /* Signal overflow, starting over with fewer guest insns. */
337 void tcg_raise_tb_overflow(TCGContext
*s
)
339 siglongjmp(s
->jmp_trans
, -2);
342 #define C_PFX1(P, A) P##A
343 #define C_PFX2(P, A, B) P##A##_##B
344 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
345 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
346 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
347 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
349 /* Define an enumeration for the various combinations. */
351 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
352 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
353 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
354 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
356 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
357 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
358 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
359 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
361 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
363 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
364 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
365 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
366 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
369 #include "tcg-target-con-set.h"
370 } TCGConstraintSetIndex
;
372 static TCGConstraintSetIndex
tcg_target_op_def(TCGOpcode
);
388 /* Put all of the constraint sets into an array, indexed by the enum. */
390 #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
391 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
392 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
393 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
395 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
396 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
397 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
398 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
400 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
402 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
403 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
404 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
405 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
407 static const TCGTargetOpDef constraint_sets
[] = {
408 #include "tcg-target-con-set.h"
426 /* Expand the enumerator to be returned from tcg_target_op_def(). */
428 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
429 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
430 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
431 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
433 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
434 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
435 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
436 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
438 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
440 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
441 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
442 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
443 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
445 #include "tcg-target.c.inc"
447 static void alloc_tcg_plugin_context(TCGContext
*s
)
450 s
->plugin_tb
= g_new0(struct qemu_plugin_tb
, 1);
451 s
->plugin_tb
->insns
=
452 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn
);
457 * All TCG threads except the parent (i.e. the one that called tcg_context_init
458 * and registered the target's TCG globals) must register with this function
459 * before initiating translation.
461 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
462 * of tcg_region_init() for the reasoning behind this.
464 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
465 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
466 * is not used anymore for translation once this function is called.
468 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
469 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
471 #ifdef CONFIG_USER_ONLY
472 void tcg_register_thread(void)
474 tcg_ctx
= &tcg_init_ctx
;
477 void tcg_register_thread(void)
479 TCGContext
*s
= g_malloc(sizeof(*s
));
484 /* Relink mem_base. */
485 for (i
= 0, n
= tcg_init_ctx
.nb_globals
; i
< n
; ++i
) {
486 if (tcg_init_ctx
.temps
[i
].mem_base
) {
487 ptrdiff_t b
= tcg_init_ctx
.temps
[i
].mem_base
- tcg_init_ctx
.temps
;
488 tcg_debug_assert(b
>= 0 && b
< n
);
489 s
->temps
[i
].mem_base
= &s
->temps
[b
];
493 /* Claim an entry in tcg_ctxs */
494 n
= qatomic_fetch_inc(&tcg_cur_ctxs
);
495 g_assert(n
< tcg_max_ctxs
);
496 qatomic_set(&tcg_ctxs
[n
], s
);
499 alloc_tcg_plugin_context(s
);
500 tcg_region_initial_alloc(s
);
505 #endif /* !CONFIG_USER_ONLY */
507 /* pool based memory allocation */
508 void *tcg_malloc_internal(TCGContext
*s
, int size
)
513 if (size
> TCG_POOL_CHUNK_SIZE
) {
514 /* big malloc: insert a new pool (XXX: could optimize) */
515 p
= g_malloc(sizeof(TCGPool
) + size
);
517 p
->next
= s
->pool_first_large
;
518 s
->pool_first_large
= p
;
529 pool_size
= TCG_POOL_CHUNK_SIZE
;
530 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
533 if (s
->pool_current
) {
534 s
->pool_current
->next
= p
;
544 s
->pool_cur
= p
->data
+ size
;
545 s
->pool_end
= p
->data
+ p
->size
;
549 void tcg_pool_reset(TCGContext
*s
)
552 for (p
= s
->pool_first_large
; p
; p
= t
) {
556 s
->pool_first_large
= NULL
;
557 s
->pool_cur
= s
->pool_end
= NULL
;
558 s
->pool_current
= NULL
;
561 #include "exec/helper-proto.h"
563 static TCGHelperInfo all_helpers
[] = {
564 #include "exec/helper-tcg.h"
566 static GHashTable
*helper_table
;
568 #ifdef CONFIG_TCG_INTERPRETER
569 static ffi_type
*typecode_to_ffi(int argmask
)
572 case dh_typecode_void
:
573 return &ffi_type_void
;
574 case dh_typecode_i32
:
575 return &ffi_type_uint32
;
576 case dh_typecode_s32
:
577 return &ffi_type_sint32
;
578 case dh_typecode_i64
:
579 return &ffi_type_uint64
;
580 case dh_typecode_s64
:
581 return &ffi_type_sint64
;
582 case dh_typecode_ptr
:
583 return &ffi_type_pointer
;
585 g_assert_not_reached();
588 static void init_ffi_layouts(void)
590 /* g_direct_hash/equal for direct comparisons on uint32_t. */
591 GHashTable
*ffi_table
= g_hash_table_new(NULL
, NULL
);
593 for (int i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
594 TCGHelperInfo
*info
= &all_helpers
[i
];
595 unsigned typemask
= info
->typemask
;
596 gpointer hash
= (gpointer
)(uintptr_t)typemask
;
605 cif
= g_hash_table_lookup(ffi_table
, hash
);
611 /* Ignoring the return type, find the last non-zero field. */
612 nargs
= 32 - clz32(typemask
>> 3);
613 nargs
= DIV_ROUND_UP(nargs
, 3);
615 ca
= g_malloc0(sizeof(*ca
) + nargs
* sizeof(ffi_type
*));
616 ca
->cif
.rtype
= typecode_to_ffi(typemask
& 7);
617 ca
->cif
.nargs
= nargs
;
620 ca
->cif
.arg_types
= ca
->args
;
621 for (int j
= 0; j
< nargs
; ++j
) {
622 int typecode
= extract32(typemask
, (j
+ 1) * 3, 3);
623 ca
->args
[j
] = typecode_to_ffi(typecode
);
627 status
= ffi_prep_cif(&ca
->cif
, FFI_DEFAULT_ABI
, nargs
,
628 ca
->cif
.rtype
, ca
->cif
.arg_types
);
629 assert(status
== FFI_OK
);
633 g_hash_table_insert(ffi_table
, hash
, (gpointer
)cif
);
636 g_hash_table_destroy(ffi_table
);
638 #endif /* CONFIG_TCG_INTERPRETER */
640 typedef struct TCGCumulativeArgs
{
641 int arg_idx
; /* tcg_gen_callN args[] */
642 int info_in_idx
; /* TCGHelperInfo in[] */
643 int arg_slot
; /* regs+stack slot */
644 int ref_slot
; /* stack slots for references */
647 static void layout_arg_even(TCGCumulativeArgs
*cum
)
649 cum
->arg_slot
+= cum
->arg_slot
& 1;
652 static void layout_arg_1(TCGCumulativeArgs
*cum
, TCGHelperInfo
*info
,
653 TCGCallArgumentKind kind
)
655 TCGCallArgumentLoc
*loc
= &info
->in
[cum
->info_in_idx
];
657 *loc
= (TCGCallArgumentLoc
){
659 .arg_idx
= cum
->arg_idx
,
660 .arg_slot
= cum
->arg_slot
,
666 static void layout_arg_normal_n(TCGCumulativeArgs
*cum
,
667 TCGHelperInfo
*info
, int n
)
669 TCGCallArgumentLoc
*loc
= &info
->in
[cum
->info_in_idx
];
671 for (int i
= 0; i
< n
; ++i
) {
672 /* Layout all using the same arg_idx, adjusting the subindex. */
673 loc
[i
] = (TCGCallArgumentLoc
){
674 .kind
= TCG_CALL_ARG_NORMAL
,
675 .arg_idx
= cum
->arg_idx
,
677 .arg_slot
= cum
->arg_slot
+ i
,
680 cum
->info_in_idx
+= n
;
684 static void init_call_layout(TCGHelperInfo
*info
)
686 int max_reg_slots
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
687 int max_stk_slots
= TCG_STATIC_CALL_ARGS_SIZE
/ sizeof(tcg_target_long
);
688 unsigned typemask
= info
->typemask
;
690 TCGCumulativeArgs cum
= { };
693 * Parse and place any function return value.
695 typecode
= typemask
& 7;
697 case dh_typecode_void
:
700 case dh_typecode_i32
:
701 case dh_typecode_s32
:
702 case dh_typecode_ptr
:
704 info
->out_kind
= TCG_CALL_RET_NORMAL
;
706 case dh_typecode_i64
:
707 case dh_typecode_s64
:
708 info
->nr_out
= 64 / TCG_TARGET_REG_BITS
;
709 info
->out_kind
= TCG_CALL_RET_NORMAL
;
712 g_assert_not_reached();
714 assert(info
->nr_out
<= ARRAY_SIZE(tcg_target_call_oarg_regs
));
717 * Parse and place function arguments.
719 for (typemask
>>= 3; typemask
; typemask
>>= 3, cum
.arg_idx
++) {
720 TCGCallArgumentKind kind
;
723 typecode
= typemask
& 7;
725 case dh_typecode_i32
:
726 case dh_typecode_s32
:
729 case dh_typecode_i64
:
730 case dh_typecode_s64
:
733 case dh_typecode_ptr
:
737 g_assert_not_reached();
742 switch (TCG_TARGET_CALL_ARG_I32
) {
743 case TCG_CALL_ARG_EVEN
:
744 layout_arg_even(&cum
);
746 case TCG_CALL_ARG_NORMAL
:
747 layout_arg_1(&cum
, info
, TCG_CALL_ARG_NORMAL
);
749 case TCG_CALL_ARG_EXTEND
:
750 kind
= TCG_CALL_ARG_EXTEND_U
+ (typecode
& 1);
751 layout_arg_1(&cum
, info
, kind
);
754 qemu_build_not_reached();
759 switch (TCG_TARGET_CALL_ARG_I64
) {
760 case TCG_CALL_ARG_EVEN
:
761 layout_arg_even(&cum
);
763 case TCG_CALL_ARG_NORMAL
:
764 if (TCG_TARGET_REG_BITS
== 32) {
765 layout_arg_normal_n(&cum
, info
, 2);
767 layout_arg_1(&cum
, info
, TCG_CALL_ARG_NORMAL
);
771 qemu_build_not_reached();
776 g_assert_not_reached();
779 info
->nr_in
= cum
.info_in_idx
;
781 /* Validate that we didn't overrun the input array. */
782 assert(cum
.info_in_idx
<= ARRAY_SIZE(info
->in
));
783 /* Validate the backend has enough argument space. */
784 assert(cum
.arg_slot
<= max_reg_slots
+ max_stk_slots
);
785 assert(cum
.ref_slot
<= max_stk_slots
);
788 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
789 static void process_op_defs(TCGContext
*s
);
790 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
791 TCGReg reg
, const char *name
);
793 static void tcg_context_init(unsigned max_cpus
)
795 TCGContext
*s
= &tcg_init_ctx
;
796 int op
, total_args
, n
, i
;
798 TCGArgConstraint
*args_ct
;
801 memset(s
, 0, sizeof(*s
));
804 /* Count total number of arguments and allocate the corresponding
807 for(op
= 0; op
< NB_OPS
; op
++) {
808 def
= &tcg_op_defs
[op
];
809 n
= def
->nb_iargs
+ def
->nb_oargs
;
813 args_ct
= g_new0(TCGArgConstraint
, total_args
);
815 for(op
= 0; op
< NB_OPS
; op
++) {
816 def
= &tcg_op_defs
[op
];
817 def
->args_ct
= args_ct
;
818 n
= def
->nb_iargs
+ def
->nb_oargs
;
822 /* Register helpers. */
823 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
824 helper_table
= g_hash_table_new(NULL
, NULL
);
826 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
827 init_call_layout(&all_helpers
[i
]);
828 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
829 (gpointer
)&all_helpers
[i
]);
832 #ifdef CONFIG_TCG_INTERPRETER
839 /* Reverse the order of the saved registers, assuming they're all at
840 the start of tcg_target_reg_alloc_order. */
841 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
842 int r
= tcg_target_reg_alloc_order
[n
];
843 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
847 for (i
= 0; i
< n
; ++i
) {
848 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
850 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
851 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
854 alloc_tcg_plugin_context(s
);
858 * In user-mode we simply share the init context among threads, since we
859 * use a single region. See the documentation tcg_region_init() for the
860 * reasoning behind this.
861 * In softmmu we will have at most max_cpus TCG threads.
863 #ifdef CONFIG_USER_ONLY
868 tcg_max_ctxs
= max_cpus
;
869 tcg_ctxs
= g_new0(TCGContext
*, max_cpus
);
872 tcg_debug_assert(!tcg_regset_test_reg(s
->reserved_regs
, TCG_AREG0
));
873 ts
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, TCG_AREG0
, "env");
874 cpu_env
= temp_tcgv_ptr(ts
);
877 void tcg_init(size_t tb_size
, int splitwx
, unsigned max_cpus
)
879 tcg_context_init(max_cpus
);
880 tcg_region_init(tb_size
, splitwx
, max_cpus
);
884 * Allocate TBs right before their corresponding translated code, making
885 * sure that TBs and code are on different cache lines.
887 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
889 uintptr_t align
= qemu_icache_linesize
;
890 TranslationBlock
*tb
;
894 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
895 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
897 if (unlikely(next
> s
->code_gen_highwater
)) {
898 if (tcg_region_alloc(s
)) {
903 qatomic_set(&s
->code_gen_ptr
, next
);
904 s
->data_gen_ptr
= NULL
;
908 void tcg_prologue_init(TCGContext
*s
)
910 size_t prologue_size
;
912 s
->code_ptr
= s
->code_gen_ptr
;
913 s
->code_buf
= s
->code_gen_ptr
;
914 s
->data_gen_ptr
= NULL
;
916 #ifndef CONFIG_TCG_INTERPRETER
917 tcg_qemu_tb_exec
= (tcg_prologue_fn
*)tcg_splitwx_to_rx(s
->code_ptr
);
920 #ifdef TCG_TARGET_NEED_POOL_LABELS
921 s
->pool_labels
= NULL
;
924 qemu_thread_jit_write();
925 /* Generate the prologue. */
926 tcg_target_qemu_prologue(s
);
928 #ifdef TCG_TARGET_NEED_POOL_LABELS
929 /* Allow the prologue to put e.g. guest_base into a pool entry. */
931 int result
= tcg_out_pool_finalize(s
);
932 tcg_debug_assert(result
== 0);
936 prologue_size
= tcg_current_code_size(s
);
937 perf_report_prologue(s
->code_gen_ptr
, prologue_size
);
939 #ifndef CONFIG_TCG_INTERPRETER
940 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s
->code_buf
),
941 (uintptr_t)s
->code_buf
, prologue_size
);
945 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
946 FILE *logfile
= qemu_log_trylock();
948 fprintf(logfile
, "PROLOGUE: [size=%zu]\n", prologue_size
);
949 if (s
->data_gen_ptr
) {
950 size_t code_size
= s
->data_gen_ptr
- s
->code_gen_ptr
;
951 size_t data_size
= prologue_size
- code_size
;
954 disas(logfile
, s
->code_gen_ptr
, code_size
);
956 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
957 if (sizeof(tcg_target_ulong
) == 8) {
959 "0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
960 (uintptr_t)s
->data_gen_ptr
+ i
,
961 *(uint64_t *)(s
->data_gen_ptr
+ i
));
964 "0x%08" PRIxPTR
": .long 0x%08x\n",
965 (uintptr_t)s
->data_gen_ptr
+ i
,
966 *(uint32_t *)(s
->data_gen_ptr
+ i
));
970 disas(logfile
, s
->code_gen_ptr
, prologue_size
);
972 fprintf(logfile
, "\n");
973 qemu_log_unlock(logfile
);
978 #ifndef CONFIG_TCG_INTERPRETER
980 * Assert that goto_ptr is implemented completely, setting an epilogue.
981 * For tci, we use NULL as the signal to return from the interpreter,
982 * so skip this check.
984 tcg_debug_assert(tcg_code_gen_epilogue
!= NULL
);
987 tcg_region_prologue_set(s
);
990 void tcg_func_start(TCGContext
*s
)
993 s
->nb_temps
= s
->nb_globals
;
995 /* No temps have been previously allocated for size or locality. */
996 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
998 /* No constant temps have been previously allocated. */
999 for (int i
= 0; i
< TCG_TYPE_COUNT
; ++i
) {
1000 if (s
->const_table
[i
]) {
1001 g_hash_table_remove_all(s
->const_table
[i
]);
1007 s
->current_frame_offset
= s
->frame_start
;
1009 #ifdef CONFIG_DEBUG_TCG
1010 s
->goto_tb_issue_mask
= 0;
1013 QTAILQ_INIT(&s
->ops
);
1014 QTAILQ_INIT(&s
->free_ops
);
1015 QSIMPLEQ_INIT(&s
->labels
);
1018 static TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
1020 int n
= s
->nb_temps
++;
1022 if (n
>= TCG_MAX_TEMPS
) {
1023 tcg_raise_tb_overflow(s
);
1025 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
1028 static TCGTemp
*tcg_global_alloc(TCGContext
*s
)
1032 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
1033 tcg_debug_assert(s
->nb_globals
< TCG_MAX_TEMPS
);
1035 ts
= tcg_temp_alloc(s
);
1036 ts
->kind
= TEMP_GLOBAL
;
1041 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
1042 TCGReg reg
, const char *name
)
1046 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
1050 ts
= tcg_global_alloc(s
);
1051 ts
->base_type
= type
;
1053 ts
->kind
= TEMP_FIXED
;
1056 tcg_regset_set_reg(s
->reserved_regs
, reg
);
1061 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
1063 s
->frame_start
= start
;
1064 s
->frame_end
= start
+ size
;
1066 = tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
1069 TCGTemp
*tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
1070 intptr_t offset
, const char *name
)
1072 TCGContext
*s
= tcg_ctx
;
1073 TCGTemp
*base_ts
= tcgv_ptr_temp(base
);
1074 TCGTemp
*ts
= tcg_global_alloc(s
);
1075 int indirect_reg
= 0;
1077 switch (base_ts
->kind
) {
1081 /* We do not support double-indirect registers. */
1082 tcg_debug_assert(!base_ts
->indirect_reg
);
1083 base_ts
->indirect_base
= 1;
1084 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
1089 g_assert_not_reached();
1092 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1093 TCGTemp
*ts2
= tcg_global_alloc(s
);
1096 ts
->base_type
= TCG_TYPE_I64
;
1097 ts
->type
= TCG_TYPE_I32
;
1098 ts
->indirect_reg
= indirect_reg
;
1099 ts
->mem_allocated
= 1;
1100 ts
->mem_base
= base_ts
;
1101 ts
->mem_offset
= offset
;
1102 pstrcpy(buf
, sizeof(buf
), name
);
1103 pstrcat(buf
, sizeof(buf
), "_0");
1104 ts
->name
= strdup(buf
);
1106 tcg_debug_assert(ts2
== ts
+ 1);
1107 ts2
->base_type
= TCG_TYPE_I64
;
1108 ts2
->type
= TCG_TYPE_I32
;
1109 ts2
->indirect_reg
= indirect_reg
;
1110 ts2
->mem_allocated
= 1;
1111 ts2
->mem_base
= base_ts
;
1112 ts2
->mem_offset
= offset
+ 4;
1113 ts2
->temp_subindex
= 1;
1114 pstrcpy(buf
, sizeof(buf
), name
);
1115 pstrcat(buf
, sizeof(buf
), "_1");
1116 ts2
->name
= strdup(buf
);
1118 ts
->base_type
= type
;
1120 ts
->indirect_reg
= indirect_reg
;
1121 ts
->mem_allocated
= 1;
1122 ts
->mem_base
= base_ts
;
1123 ts
->mem_offset
= offset
;
1129 TCGTemp
*tcg_temp_new_internal(TCGType type
, bool temp_local
)
1131 TCGContext
*s
= tcg_ctx
;
1132 TCGTempKind kind
= temp_local
? TEMP_LOCAL
: TEMP_NORMAL
;
1136 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
1137 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
1138 if (idx
< TCG_MAX_TEMPS
) {
1139 /* There is already an available temp with the right type. */
1140 clear_bit(idx
, s
->free_temps
[k
].l
);
1142 ts
= &s
->temps
[idx
];
1143 ts
->temp_allocated
= 1;
1144 tcg_debug_assert(ts
->base_type
== type
);
1145 tcg_debug_assert(ts
->kind
== kind
);
1147 ts
= tcg_temp_alloc(s
);
1148 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1149 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1151 ts
->base_type
= type
;
1152 ts
->type
= TCG_TYPE_I32
;
1153 ts
->temp_allocated
= 1;
1156 tcg_debug_assert(ts2
== ts
+ 1);
1157 ts2
->base_type
= TCG_TYPE_I64
;
1158 ts2
->type
= TCG_TYPE_I32
;
1159 ts2
->temp_allocated
= 1;
1160 ts2
->temp_subindex
= 1;
1163 ts
->base_type
= type
;
1165 ts
->temp_allocated
= 1;
1170 #if defined(CONFIG_DEBUG_TCG)
1176 TCGv_vec
tcg_temp_new_vec(TCGType type
)
1180 #ifdef CONFIG_DEBUG_TCG
1183 assert(TCG_TARGET_HAS_v64
);
1186 assert(TCG_TARGET_HAS_v128
);
1189 assert(TCG_TARGET_HAS_v256
);
1192 g_assert_not_reached();
1196 t
= tcg_temp_new_internal(type
, 0);
1197 return temp_tcgv_vec(t
);
1200 /* Create a new temp of the same type as an existing temp. */
1201 TCGv_vec
tcg_temp_new_vec_matching(TCGv_vec match
)
1203 TCGTemp
*t
= tcgv_vec_temp(match
);
1205 tcg_debug_assert(t
->temp_allocated
!= 0);
1207 t
= tcg_temp_new_internal(t
->base_type
, 0);
1208 return temp_tcgv_vec(t
);
1211 void tcg_temp_free_internal(TCGTemp
*ts
)
1213 TCGContext
*s
= tcg_ctx
;
1219 * In order to simplify users of tcg_constant_*,
1220 * silently ignore free.
1227 g_assert_not_reached();
1230 #if defined(CONFIG_DEBUG_TCG)
1232 if (s
->temps_in_use
< 0) {
1233 fprintf(stderr
, "More temporaries freed than allocated!\n");
1237 tcg_debug_assert(ts
->temp_allocated
!= 0);
1238 ts
->temp_allocated
= 0;
1241 k
= ts
->base_type
+ (ts
->kind
== TEMP_NORMAL
? 0 : TCG_TYPE_COUNT
);
1242 set_bit(idx
, s
->free_temps
[k
].l
);
1245 TCGTemp
*tcg_constant_internal(TCGType type
, int64_t val
)
1247 TCGContext
*s
= tcg_ctx
;
1248 GHashTable
*h
= s
->const_table
[type
];
1252 h
= g_hash_table_new(g_int64_hash
, g_int64_equal
);
1253 s
->const_table
[type
] = h
;
1256 ts
= g_hash_table_lookup(h
, &val
);
1260 ts
= tcg_temp_alloc(s
);
1262 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1263 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1265 tcg_debug_assert(ts2
== ts
+ 1);
1267 ts
->base_type
= TCG_TYPE_I64
;
1268 ts
->type
= TCG_TYPE_I32
;
1269 ts
->kind
= TEMP_CONST
;
1270 ts
->temp_allocated
= 1;
1272 ts2
->base_type
= TCG_TYPE_I64
;
1273 ts2
->type
= TCG_TYPE_I32
;
1274 ts2
->kind
= TEMP_CONST
;
1275 ts2
->temp_allocated
= 1;
1276 ts2
->temp_subindex
= 1;
1279 * Retain the full value of the 64-bit constant in the low
1280 * part, so that the hash table works. Actual uses will
1281 * truncate the value to the low part.
1283 ts
[HOST_BIG_ENDIAN
].val
= val
;
1284 ts
[!HOST_BIG_ENDIAN
].val
= val
>> 32;
1285 val_ptr
= &ts
[HOST_BIG_ENDIAN
].val
;
1287 ts
->base_type
= type
;
1289 ts
->kind
= TEMP_CONST
;
1290 ts
->temp_allocated
= 1;
1294 g_hash_table_insert(h
, val_ptr
, ts
);
1300 TCGv_vec
tcg_constant_vec(TCGType type
, unsigned vece
, int64_t val
)
1302 val
= dup_const(vece
, val
);
1303 return temp_tcgv_vec(tcg_constant_internal(type
, val
));
1306 TCGv_vec
tcg_constant_vec_matching(TCGv_vec match
, unsigned vece
, int64_t val
)
1308 TCGTemp
*t
= tcgv_vec_temp(match
);
1310 tcg_debug_assert(t
->temp_allocated
!= 0);
1311 return tcg_constant_vec(t
->base_type
, vece
, val
);
1314 TCGv_i32
tcg_const_i32(int32_t val
)
1317 t0
= tcg_temp_new_i32();
1318 tcg_gen_movi_i32(t0
, val
);
1322 TCGv_i64
tcg_const_i64(int64_t val
)
1325 t0
= tcg_temp_new_i64();
1326 tcg_gen_movi_i64(t0
, val
);
1330 TCGv_i32
tcg_const_local_i32(int32_t val
)
1333 t0
= tcg_temp_local_new_i32();
1334 tcg_gen_movi_i32(t0
, val
);
1338 TCGv_i64
tcg_const_local_i64(int64_t val
)
1341 t0
= tcg_temp_local_new_i64();
1342 tcg_gen_movi_i64(t0
, val
);
1346 #if defined(CONFIG_DEBUG_TCG)
1347 void tcg_clear_temp_count(void)
1349 TCGContext
*s
= tcg_ctx
;
1350 s
->temps_in_use
= 0;
1353 int tcg_check_temp_count(void)
1355 TCGContext
*s
= tcg_ctx
;
1356 if (s
->temps_in_use
) {
1357 /* Clear the count so that we don't give another
1358 * warning immediately next time around.
1360 s
->temps_in_use
= 0;
1367 /* Return true if OP may appear in the opcode stream.
1368 Test the runtime variable that controls each opcode. */
1369 bool tcg_op_supported(TCGOpcode op
)
1372 = TCG_TARGET_HAS_v64
| TCG_TARGET_HAS_v128
| TCG_TARGET_HAS_v256
;
1375 case INDEX_op_discard
:
1376 case INDEX_op_set_label
:
1380 case INDEX_op_insn_start
:
1381 case INDEX_op_exit_tb
:
1382 case INDEX_op_goto_tb
:
1383 case INDEX_op_goto_ptr
:
1384 case INDEX_op_qemu_ld_i32
:
1385 case INDEX_op_qemu_st_i32
:
1386 case INDEX_op_qemu_ld_i64
:
1387 case INDEX_op_qemu_st_i64
:
1390 case INDEX_op_qemu_st8_i32
:
1391 return TCG_TARGET_HAS_qemu_st8_i32
;
1393 case INDEX_op_mov_i32
:
1394 case INDEX_op_setcond_i32
:
1395 case INDEX_op_brcond_i32
:
1396 case INDEX_op_ld8u_i32
:
1397 case INDEX_op_ld8s_i32
:
1398 case INDEX_op_ld16u_i32
:
1399 case INDEX_op_ld16s_i32
:
1400 case INDEX_op_ld_i32
:
1401 case INDEX_op_st8_i32
:
1402 case INDEX_op_st16_i32
:
1403 case INDEX_op_st_i32
:
1404 case INDEX_op_add_i32
:
1405 case INDEX_op_sub_i32
:
1406 case INDEX_op_mul_i32
:
1407 case INDEX_op_and_i32
:
1408 case INDEX_op_or_i32
:
1409 case INDEX_op_xor_i32
:
1410 case INDEX_op_shl_i32
:
1411 case INDEX_op_shr_i32
:
1412 case INDEX_op_sar_i32
:
1415 case INDEX_op_movcond_i32
:
1416 return TCG_TARGET_HAS_movcond_i32
;
1417 case INDEX_op_div_i32
:
1418 case INDEX_op_divu_i32
:
1419 return TCG_TARGET_HAS_div_i32
;
1420 case INDEX_op_rem_i32
:
1421 case INDEX_op_remu_i32
:
1422 return TCG_TARGET_HAS_rem_i32
;
1423 case INDEX_op_div2_i32
:
1424 case INDEX_op_divu2_i32
:
1425 return TCG_TARGET_HAS_div2_i32
;
1426 case INDEX_op_rotl_i32
:
1427 case INDEX_op_rotr_i32
:
1428 return TCG_TARGET_HAS_rot_i32
;
1429 case INDEX_op_deposit_i32
:
1430 return TCG_TARGET_HAS_deposit_i32
;
1431 case INDEX_op_extract_i32
:
1432 return TCG_TARGET_HAS_extract_i32
;
1433 case INDEX_op_sextract_i32
:
1434 return TCG_TARGET_HAS_sextract_i32
;
1435 case INDEX_op_extract2_i32
:
1436 return TCG_TARGET_HAS_extract2_i32
;
1437 case INDEX_op_add2_i32
:
1438 return TCG_TARGET_HAS_add2_i32
;
1439 case INDEX_op_sub2_i32
:
1440 return TCG_TARGET_HAS_sub2_i32
;
1441 case INDEX_op_mulu2_i32
:
1442 return TCG_TARGET_HAS_mulu2_i32
;
1443 case INDEX_op_muls2_i32
:
1444 return TCG_TARGET_HAS_muls2_i32
;
1445 case INDEX_op_muluh_i32
:
1446 return TCG_TARGET_HAS_muluh_i32
;
1447 case INDEX_op_mulsh_i32
:
1448 return TCG_TARGET_HAS_mulsh_i32
;
1449 case INDEX_op_ext8s_i32
:
1450 return TCG_TARGET_HAS_ext8s_i32
;
1451 case INDEX_op_ext16s_i32
:
1452 return TCG_TARGET_HAS_ext16s_i32
;
1453 case INDEX_op_ext8u_i32
:
1454 return TCG_TARGET_HAS_ext8u_i32
;
1455 case INDEX_op_ext16u_i32
:
1456 return TCG_TARGET_HAS_ext16u_i32
;
1457 case INDEX_op_bswap16_i32
:
1458 return TCG_TARGET_HAS_bswap16_i32
;
1459 case INDEX_op_bswap32_i32
:
1460 return TCG_TARGET_HAS_bswap32_i32
;
1461 case INDEX_op_not_i32
:
1462 return TCG_TARGET_HAS_not_i32
;
1463 case INDEX_op_neg_i32
:
1464 return TCG_TARGET_HAS_neg_i32
;
1465 case INDEX_op_andc_i32
:
1466 return TCG_TARGET_HAS_andc_i32
;
1467 case INDEX_op_orc_i32
:
1468 return TCG_TARGET_HAS_orc_i32
;
1469 case INDEX_op_eqv_i32
:
1470 return TCG_TARGET_HAS_eqv_i32
;
1471 case INDEX_op_nand_i32
:
1472 return TCG_TARGET_HAS_nand_i32
;
1473 case INDEX_op_nor_i32
:
1474 return TCG_TARGET_HAS_nor_i32
;
1475 case INDEX_op_clz_i32
:
1476 return TCG_TARGET_HAS_clz_i32
;
1477 case INDEX_op_ctz_i32
:
1478 return TCG_TARGET_HAS_ctz_i32
;
1479 case INDEX_op_ctpop_i32
:
1480 return TCG_TARGET_HAS_ctpop_i32
;
1482 case INDEX_op_brcond2_i32
:
1483 case INDEX_op_setcond2_i32
:
1484 return TCG_TARGET_REG_BITS
== 32;
1486 case INDEX_op_mov_i64
:
1487 case INDEX_op_setcond_i64
:
1488 case INDEX_op_brcond_i64
:
1489 case INDEX_op_ld8u_i64
:
1490 case INDEX_op_ld8s_i64
:
1491 case INDEX_op_ld16u_i64
:
1492 case INDEX_op_ld16s_i64
:
1493 case INDEX_op_ld32u_i64
:
1494 case INDEX_op_ld32s_i64
:
1495 case INDEX_op_ld_i64
:
1496 case INDEX_op_st8_i64
:
1497 case INDEX_op_st16_i64
:
1498 case INDEX_op_st32_i64
:
1499 case INDEX_op_st_i64
:
1500 case INDEX_op_add_i64
:
1501 case INDEX_op_sub_i64
:
1502 case INDEX_op_mul_i64
:
1503 case INDEX_op_and_i64
:
1504 case INDEX_op_or_i64
:
1505 case INDEX_op_xor_i64
:
1506 case INDEX_op_shl_i64
:
1507 case INDEX_op_shr_i64
:
1508 case INDEX_op_sar_i64
:
1509 case INDEX_op_ext_i32_i64
:
1510 case INDEX_op_extu_i32_i64
:
1511 return TCG_TARGET_REG_BITS
== 64;
1513 case INDEX_op_movcond_i64
:
1514 return TCG_TARGET_HAS_movcond_i64
;
1515 case INDEX_op_div_i64
:
1516 case INDEX_op_divu_i64
:
1517 return TCG_TARGET_HAS_div_i64
;
1518 case INDEX_op_rem_i64
:
1519 case INDEX_op_remu_i64
:
1520 return TCG_TARGET_HAS_rem_i64
;
1521 case INDEX_op_div2_i64
:
1522 case INDEX_op_divu2_i64
:
1523 return TCG_TARGET_HAS_div2_i64
;
1524 case INDEX_op_rotl_i64
:
1525 case INDEX_op_rotr_i64
:
1526 return TCG_TARGET_HAS_rot_i64
;
1527 case INDEX_op_deposit_i64
:
1528 return TCG_TARGET_HAS_deposit_i64
;
1529 case INDEX_op_extract_i64
:
1530 return TCG_TARGET_HAS_extract_i64
;
1531 case INDEX_op_sextract_i64
:
1532 return TCG_TARGET_HAS_sextract_i64
;
1533 case INDEX_op_extract2_i64
:
1534 return TCG_TARGET_HAS_extract2_i64
;
1535 case INDEX_op_extrl_i64_i32
:
1536 return TCG_TARGET_HAS_extrl_i64_i32
;
1537 case INDEX_op_extrh_i64_i32
:
1538 return TCG_TARGET_HAS_extrh_i64_i32
;
1539 case INDEX_op_ext8s_i64
:
1540 return TCG_TARGET_HAS_ext8s_i64
;
1541 case INDEX_op_ext16s_i64
:
1542 return TCG_TARGET_HAS_ext16s_i64
;
1543 case INDEX_op_ext32s_i64
:
1544 return TCG_TARGET_HAS_ext32s_i64
;
1545 case INDEX_op_ext8u_i64
:
1546 return TCG_TARGET_HAS_ext8u_i64
;
1547 case INDEX_op_ext16u_i64
:
1548 return TCG_TARGET_HAS_ext16u_i64
;
1549 case INDEX_op_ext32u_i64
:
1550 return TCG_TARGET_HAS_ext32u_i64
;
1551 case INDEX_op_bswap16_i64
:
1552 return TCG_TARGET_HAS_bswap16_i64
;
1553 case INDEX_op_bswap32_i64
:
1554 return TCG_TARGET_HAS_bswap32_i64
;
1555 case INDEX_op_bswap64_i64
:
1556 return TCG_TARGET_HAS_bswap64_i64
;
1557 case INDEX_op_not_i64
:
1558 return TCG_TARGET_HAS_not_i64
;
1559 case INDEX_op_neg_i64
:
1560 return TCG_TARGET_HAS_neg_i64
;
1561 case INDEX_op_andc_i64
:
1562 return TCG_TARGET_HAS_andc_i64
;
1563 case INDEX_op_orc_i64
:
1564 return TCG_TARGET_HAS_orc_i64
;
1565 case INDEX_op_eqv_i64
:
1566 return TCG_TARGET_HAS_eqv_i64
;
1567 case INDEX_op_nand_i64
:
1568 return TCG_TARGET_HAS_nand_i64
;
1569 case INDEX_op_nor_i64
:
1570 return TCG_TARGET_HAS_nor_i64
;
1571 case INDEX_op_clz_i64
:
1572 return TCG_TARGET_HAS_clz_i64
;
1573 case INDEX_op_ctz_i64
:
1574 return TCG_TARGET_HAS_ctz_i64
;
1575 case INDEX_op_ctpop_i64
:
1576 return TCG_TARGET_HAS_ctpop_i64
;
1577 case INDEX_op_add2_i64
:
1578 return TCG_TARGET_HAS_add2_i64
;
1579 case INDEX_op_sub2_i64
:
1580 return TCG_TARGET_HAS_sub2_i64
;
1581 case INDEX_op_mulu2_i64
:
1582 return TCG_TARGET_HAS_mulu2_i64
;
1583 case INDEX_op_muls2_i64
:
1584 return TCG_TARGET_HAS_muls2_i64
;
1585 case INDEX_op_muluh_i64
:
1586 return TCG_TARGET_HAS_muluh_i64
;
1587 case INDEX_op_mulsh_i64
:
1588 return TCG_TARGET_HAS_mulsh_i64
;
1590 case INDEX_op_mov_vec
:
1591 case INDEX_op_dup_vec
:
1592 case INDEX_op_dupm_vec
:
1593 case INDEX_op_ld_vec
:
1594 case INDEX_op_st_vec
:
1595 case INDEX_op_add_vec
:
1596 case INDEX_op_sub_vec
:
1597 case INDEX_op_and_vec
:
1598 case INDEX_op_or_vec
:
1599 case INDEX_op_xor_vec
:
1600 case INDEX_op_cmp_vec
:
1602 case INDEX_op_dup2_vec
:
1603 return have_vec
&& TCG_TARGET_REG_BITS
== 32;
1604 case INDEX_op_not_vec
:
1605 return have_vec
&& TCG_TARGET_HAS_not_vec
;
1606 case INDEX_op_neg_vec
:
1607 return have_vec
&& TCG_TARGET_HAS_neg_vec
;
1608 case INDEX_op_abs_vec
:
1609 return have_vec
&& TCG_TARGET_HAS_abs_vec
;
1610 case INDEX_op_andc_vec
:
1611 return have_vec
&& TCG_TARGET_HAS_andc_vec
;
1612 case INDEX_op_orc_vec
:
1613 return have_vec
&& TCG_TARGET_HAS_orc_vec
;
1614 case INDEX_op_nand_vec
:
1615 return have_vec
&& TCG_TARGET_HAS_nand_vec
;
1616 case INDEX_op_nor_vec
:
1617 return have_vec
&& TCG_TARGET_HAS_nor_vec
;
1618 case INDEX_op_eqv_vec
:
1619 return have_vec
&& TCG_TARGET_HAS_eqv_vec
;
1620 case INDEX_op_mul_vec
:
1621 return have_vec
&& TCG_TARGET_HAS_mul_vec
;
1622 case INDEX_op_shli_vec
:
1623 case INDEX_op_shri_vec
:
1624 case INDEX_op_sari_vec
:
1625 return have_vec
&& TCG_TARGET_HAS_shi_vec
;
1626 case INDEX_op_shls_vec
:
1627 case INDEX_op_shrs_vec
:
1628 case INDEX_op_sars_vec
:
1629 return have_vec
&& TCG_TARGET_HAS_shs_vec
;
1630 case INDEX_op_shlv_vec
:
1631 case INDEX_op_shrv_vec
:
1632 case INDEX_op_sarv_vec
:
1633 return have_vec
&& TCG_TARGET_HAS_shv_vec
;
1634 case INDEX_op_rotli_vec
:
1635 return have_vec
&& TCG_TARGET_HAS_roti_vec
;
1636 case INDEX_op_rotls_vec
:
1637 return have_vec
&& TCG_TARGET_HAS_rots_vec
;
1638 case INDEX_op_rotlv_vec
:
1639 case INDEX_op_rotrv_vec
:
1640 return have_vec
&& TCG_TARGET_HAS_rotv_vec
;
1641 case INDEX_op_ssadd_vec
:
1642 case INDEX_op_usadd_vec
:
1643 case INDEX_op_sssub_vec
:
1644 case INDEX_op_ussub_vec
:
1645 return have_vec
&& TCG_TARGET_HAS_sat_vec
;
1646 case INDEX_op_smin_vec
:
1647 case INDEX_op_umin_vec
:
1648 case INDEX_op_smax_vec
:
1649 case INDEX_op_umax_vec
:
1650 return have_vec
&& TCG_TARGET_HAS_minmax_vec
;
1651 case INDEX_op_bitsel_vec
:
1652 return have_vec
&& TCG_TARGET_HAS_bitsel_vec
;
1653 case INDEX_op_cmpsel_vec
:
1654 return have_vec
&& TCG_TARGET_HAS_cmpsel_vec
;
1657 tcg_debug_assert(op
> INDEX_op_last_generic
&& op
< NB_OPS
);
1662 static TCGOp
*tcg_op_alloc(TCGOpcode opc
, unsigned nargs
);
1664 void tcg_gen_callN(void *func
, TCGTemp
*ret
, int nargs
, TCGTemp
**args
)
1666 const TCGHelperInfo
*info
;
1667 TCGv_i64 extend_free
[MAX_CALL_IARGS
];
1670 int i
, n
, pi
= 0, total_args
;
1672 info
= g_hash_table_lookup(helper_table
, (gpointer
)func
);
1673 total_args
= info
->nr_out
+ info
->nr_in
+ 2;
1674 op
= tcg_op_alloc(INDEX_op_call
, total_args
);
1676 #ifdef CONFIG_PLUGIN
1677 /* Flag helpers that may affect guest state */
1678 if (tcg_ctx
->plugin_insn
&&
1679 !(info
->flags
& TCG_CALL_PLUGIN
) &&
1680 !(info
->flags
& TCG_CALL_NO_SIDE_EFFECTS
)) {
1681 tcg_ctx
->plugin_insn
->calls_helpers
= true;
1685 TCGOP_CALLO(op
) = n
= info
->nr_out
;
1688 tcg_debug_assert(ret
== NULL
);
1691 tcg_debug_assert(ret
!= NULL
);
1692 op
->args
[pi
++] = temp_arg(ret
);
1695 tcg_debug_assert(ret
!= NULL
);
1696 tcg_debug_assert(ret
->base_type
== ret
->type
+ 1);
1697 tcg_debug_assert(ret
->temp_subindex
== 0);
1698 op
->args
[pi
++] = temp_arg(ret
);
1699 op
->args
[pi
++] = temp_arg(ret
+ 1);
1702 g_assert_not_reached();
1705 TCGOP_CALLI(op
) = n
= info
->nr_in
;
1706 for (i
= 0; i
< n
; i
++) {
1707 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
1708 TCGTemp
*ts
= args
[loc
->arg_idx
] + loc
->tmp_subindex
;
1710 switch (loc
->kind
) {
1711 case TCG_CALL_ARG_NORMAL
:
1712 op
->args
[pi
++] = temp_arg(ts
);
1715 case TCG_CALL_ARG_EXTEND_U
:
1716 case TCG_CALL_ARG_EXTEND_S
:
1718 TCGv_i64 temp
= tcg_temp_new_i64();
1719 TCGv_i32 orig
= temp_tcgv_i32(ts
);
1721 if (loc
->kind
== TCG_CALL_ARG_EXTEND_S
) {
1722 tcg_gen_ext_i32_i64(temp
, orig
);
1724 tcg_gen_extu_i32_i64(temp
, orig
);
1726 op
->args
[pi
++] = tcgv_i64_arg(temp
);
1727 extend_free
[n_extend
++] = temp
;
1732 g_assert_not_reached();
1735 op
->args
[pi
++] = (uintptr_t)func
;
1736 op
->args
[pi
++] = (uintptr_t)info
;
1737 tcg_debug_assert(pi
== total_args
);
1739 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
1741 tcg_debug_assert(n_extend
< ARRAY_SIZE(extend_free
));
1742 for (i
= 0; i
< n_extend
; ++i
) {
1743 tcg_temp_free_i64(extend_free
[i
]);
1747 static void tcg_reg_alloc_start(TCGContext
*s
)
1751 for (i
= 0, n
= s
->nb_temps
; i
< n
; i
++) {
1752 TCGTemp
*ts
= &s
->temps
[i
];
1753 TCGTempVal val
= TEMP_VAL_MEM
;
1757 val
= TEMP_VAL_CONST
;
1766 val
= TEMP_VAL_DEAD
;
1769 ts
->mem_allocated
= 0;
1772 g_assert_not_reached();
1777 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1780 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1783 int idx
= temp_idx(ts
);
1788 pstrcpy(buf
, buf_size
, ts
->name
);
1791 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1794 snprintf(buf
, buf_size
, "ebb%d", idx
- s
->nb_globals
);
1797 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1802 snprintf(buf
, buf_size
, "$0x%x", (int32_t)ts
->val
);
1804 #if TCG_TARGET_REG_BITS > 32
1806 snprintf(buf
, buf_size
, "$0x%" PRIx64
, ts
->val
);
1812 snprintf(buf
, buf_size
, "v%d$0x%" PRIx64
,
1813 64 << (ts
->type
- TCG_TYPE_V64
), ts
->val
);
1816 g_assert_not_reached();
1823 static char *tcg_get_arg_str(TCGContext
*s
, char *buf
,
1824 int buf_size
, TCGArg arg
)
1826 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, arg_temp(arg
));
1829 static const char * const cond_name
[] =
1831 [TCG_COND_NEVER
] = "never",
1832 [TCG_COND_ALWAYS
] = "always",
1833 [TCG_COND_EQ
] = "eq",
1834 [TCG_COND_NE
] = "ne",
1835 [TCG_COND_LT
] = "lt",
1836 [TCG_COND_GE
] = "ge",
1837 [TCG_COND_LE
] = "le",
1838 [TCG_COND_GT
] = "gt",
1839 [TCG_COND_LTU
] = "ltu",
1840 [TCG_COND_GEU
] = "geu",
1841 [TCG_COND_LEU
] = "leu",
1842 [TCG_COND_GTU
] = "gtu"
1845 static const char * const ldst_name
[] =
1861 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1862 #ifdef TARGET_ALIGNED_ONLY
1863 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1864 [MO_ALIGN
>> MO_ASHIFT
] = "",
1866 [MO_UNALN
>> MO_ASHIFT
] = "",
1867 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1869 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1870 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1871 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1872 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1873 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1874 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1877 static const char bswap_flag_name
[][6] = {
1878 [TCG_BSWAP_IZ
] = "iz",
1879 [TCG_BSWAP_OZ
] = "oz",
1880 [TCG_BSWAP_OS
] = "os",
1881 [TCG_BSWAP_IZ
| TCG_BSWAP_OZ
] = "iz,oz",
1882 [TCG_BSWAP_IZ
| TCG_BSWAP_OS
] = "iz,os",
1885 static inline bool tcg_regset_single(TCGRegSet d
)
1887 return (d
& (d
- 1)) == 0;
1890 static inline TCGReg
tcg_regset_first(TCGRegSet d
)
1892 if (TCG_TARGET_NB_REGS
<= 32) {
1899 /* Return only the number of characters output -- no error return. */
1900 #define ne_fprintf(...) \
1901 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1903 static void tcg_dump_ops(TCGContext
*s
, FILE *f
, bool have_prefs
)
1908 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
1909 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1910 const TCGOpDef
*def
;
1915 def
= &tcg_op_defs
[c
];
1917 if (c
== INDEX_op_insn_start
) {
1919 col
+= ne_fprintf(f
, "\n ----");
1921 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
1923 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1924 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
1928 col
+= ne_fprintf(f
, " " TARGET_FMT_lx
, a
);
1930 } else if (c
== INDEX_op_call
) {
1931 const TCGHelperInfo
*info
= tcg_call_info(op
);
1932 void *func
= tcg_call_func(op
);
1934 /* variable number of arguments */
1935 nb_oargs
= TCGOP_CALLO(op
);
1936 nb_iargs
= TCGOP_CALLI(op
);
1937 nb_cargs
= def
->nb_cargs
;
1939 col
+= ne_fprintf(f
, " %s ", def
->name
);
1942 * Print the function name from TCGHelperInfo, if available.
1943 * Note that plugins have a template function for the info,
1944 * but the actual function pointer comes from the plugin.
1946 if (func
== info
->func
) {
1947 col
+= ne_fprintf(f
, "%s", info
->name
);
1949 col
+= ne_fprintf(f
, "plugin(%p)", func
);
1952 col
+= ne_fprintf(f
, ",$0x%x,$%d", info
->flags
, nb_oargs
);
1953 for (i
= 0; i
< nb_oargs
; i
++) {
1954 col
+= ne_fprintf(f
, ",%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
1957 for (i
= 0; i
< nb_iargs
; i
++) {
1958 TCGArg arg
= op
->args
[nb_oargs
+ i
];
1959 const char *t
= tcg_get_arg_str(s
, buf
, sizeof(buf
), arg
);
1960 col
+= ne_fprintf(f
, ",%s", t
);
1963 col
+= ne_fprintf(f
, " %s ", def
->name
);
1965 nb_oargs
= def
->nb_oargs
;
1966 nb_iargs
= def
->nb_iargs
;
1967 nb_cargs
= def
->nb_cargs
;
1969 if (def
->flags
& TCG_OPF_VECTOR
) {
1970 col
+= ne_fprintf(f
, "v%d,e%d,", 64 << TCGOP_VECL(op
),
1971 8 << TCGOP_VECE(op
));
1975 for (i
= 0; i
< nb_oargs
; i
++) {
1976 const char *sep
= k
? "," : "";
1977 col
+= ne_fprintf(f
, "%s%s", sep
,
1978 tcg_get_arg_str(s
, buf
, sizeof(buf
),
1981 for (i
= 0; i
< nb_iargs
; i
++) {
1982 const char *sep
= k
? "," : "";
1983 col
+= ne_fprintf(f
, "%s%s", sep
,
1984 tcg_get_arg_str(s
, buf
, sizeof(buf
),
1988 case INDEX_op_brcond_i32
:
1989 case INDEX_op_setcond_i32
:
1990 case INDEX_op_movcond_i32
:
1991 case INDEX_op_brcond2_i32
:
1992 case INDEX_op_setcond2_i32
:
1993 case INDEX_op_brcond_i64
:
1994 case INDEX_op_setcond_i64
:
1995 case INDEX_op_movcond_i64
:
1996 case INDEX_op_cmp_vec
:
1997 case INDEX_op_cmpsel_vec
:
1998 if (op
->args
[k
] < ARRAY_SIZE(cond_name
)
1999 && cond_name
[op
->args
[k
]]) {
2000 col
+= ne_fprintf(f
, ",%s", cond_name
[op
->args
[k
++]]);
2002 col
+= ne_fprintf(f
, ",$0x%" TCG_PRIlx
, op
->args
[k
++]);
2006 case INDEX_op_qemu_ld_i32
:
2007 case INDEX_op_qemu_st_i32
:
2008 case INDEX_op_qemu_st8_i32
:
2009 case INDEX_op_qemu_ld_i64
:
2010 case INDEX_op_qemu_st_i64
:
2012 MemOpIdx oi
= op
->args
[k
++];
2013 MemOp op
= get_memop(oi
);
2014 unsigned ix
= get_mmuidx(oi
);
2016 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
2017 col
+= ne_fprintf(f
, ",$0x%x,%u", op
, ix
);
2019 const char *s_al
, *s_op
;
2020 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
2021 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
2022 col
+= ne_fprintf(f
, ",%s%s,%u", s_al
, s_op
, ix
);
2027 case INDEX_op_bswap16_i32
:
2028 case INDEX_op_bswap16_i64
:
2029 case INDEX_op_bswap32_i32
:
2030 case INDEX_op_bswap32_i64
:
2031 case INDEX_op_bswap64_i64
:
2033 TCGArg flags
= op
->args
[k
];
2034 const char *name
= NULL
;
2036 if (flags
< ARRAY_SIZE(bswap_flag_name
)) {
2037 name
= bswap_flag_name
[flags
];
2040 col
+= ne_fprintf(f
, ",%s", name
);
2042 col
+= ne_fprintf(f
, ",$0x%" TCG_PRIlx
, flags
);
2052 case INDEX_op_set_label
:
2054 case INDEX_op_brcond_i32
:
2055 case INDEX_op_brcond_i64
:
2056 case INDEX_op_brcond2_i32
:
2057 col
+= ne_fprintf(f
, "%s$L%d", k
? "," : "",
2058 arg_label(op
->args
[k
])->id
);
2064 for (; i
< nb_cargs
; i
++, k
++) {
2065 col
+= ne_fprintf(f
, "%s$0x%" TCG_PRIlx
, k
? "," : "",
2070 if (have_prefs
|| op
->life
) {
2071 for (; col
< 40; ++col
) {
2077 unsigned life
= op
->life
;
2079 if (life
& (SYNC_ARG
* 3)) {
2080 ne_fprintf(f
, " sync:");
2081 for (i
= 0; i
< 2; ++i
) {
2082 if (life
& (SYNC_ARG
<< i
)) {
2083 ne_fprintf(f
, " %d", i
);
2089 ne_fprintf(f
, " dead:");
2090 for (i
= 0; life
; ++i
, life
>>= 1) {
2092 ne_fprintf(f
, " %d", i
);
2099 for (i
= 0; i
< nb_oargs
; ++i
) {
2100 TCGRegSet set
= output_pref(op
, i
);
2103 ne_fprintf(f
, " pref=");
2108 ne_fprintf(f
, "none");
2109 } else if (set
== MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS
)) {
2110 ne_fprintf(f
, "all");
2111 #ifdef CONFIG_DEBUG_TCG
2112 } else if (tcg_regset_single(set
)) {
2113 TCGReg reg
= tcg_regset_first(set
);
2114 ne_fprintf(f
, "%s", tcg_target_reg_names
[reg
]);
2116 } else if (TCG_TARGET_NB_REGS
<= 32) {
2117 ne_fprintf(f
, "0x%x", (uint32_t)set
);
2119 ne_fprintf(f
, "0x%" PRIx64
, (uint64_t)set
);
2128 /* we give more priority to constraints with less registers */
2129 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
2131 const TCGArgConstraint
*arg_ct
= &def
->args_ct
[k
];
2132 int n
= ctpop64(arg_ct
->regs
);
2135 * Sort constraints of a single register first, which includes output
2136 * aliases (which must exactly match the input already allocated).
2138 if (n
== 1 || arg_ct
->oalias
) {
2143 * Sort register pairs next, first then second immediately after.
2144 * Arbitrarily sort multiple pairs by the index of the first reg;
2145 * there shouldn't be many pairs.
2147 switch (arg_ct
->pair
) {
2152 return (arg_ct
->pair_index
+ 1) * 2 - 1;
2155 /* Finally, sort by decreasing register count. */
2160 /* sort from highest priority to lowest */
2161 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
2164 TCGArgConstraint
*a
= def
->args_ct
;
2166 for (i
= 0; i
< n
; i
++) {
2167 a
[start
+ i
].sort_index
= start
+ i
;
2172 for (i
= 0; i
< n
- 1; i
++) {
2173 for (j
= i
+ 1; j
< n
; j
++) {
2174 int p1
= get_constraint_priority(def
, a
[start
+ i
].sort_index
);
2175 int p2
= get_constraint_priority(def
, a
[start
+ j
].sort_index
);
2177 int tmp
= a
[start
+ i
].sort_index
;
2178 a
[start
+ i
].sort_index
= a
[start
+ j
].sort_index
;
2179 a
[start
+ j
].sort_index
= tmp
;
2185 static void process_op_defs(TCGContext
*s
)
2189 for (op
= 0; op
< NB_OPS
; op
++) {
2190 TCGOpDef
*def
= &tcg_op_defs
[op
];
2191 const TCGTargetOpDef
*tdefs
;
2192 bool saw_alias_pair
= false;
2193 int i
, o
, i2
, o2
, nb_args
;
2195 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2199 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
2205 * Macro magic should make it impossible, but double-check that
2206 * the array index is in range. Since the signness of an enum
2207 * is implementation defined, force the result to unsigned.
2209 unsigned con_set
= tcg_target_op_def(op
);
2210 tcg_debug_assert(con_set
< ARRAY_SIZE(constraint_sets
));
2211 tdefs
= &constraint_sets
[con_set
];
2213 for (i
= 0; i
< nb_args
; i
++) {
2214 const char *ct_str
= tdefs
->args_ct_str
[i
];
2215 bool input_p
= i
>= def
->nb_oargs
;
2217 /* Incomplete TCGTargetOpDef entry. */
2218 tcg_debug_assert(ct_str
!= NULL
);
2223 tcg_debug_assert(input_p
);
2224 tcg_debug_assert(o
< def
->nb_oargs
);
2225 tcg_debug_assert(def
->args_ct
[o
].regs
!= 0);
2226 tcg_debug_assert(!def
->args_ct
[o
].oalias
);
2227 def
->args_ct
[i
] = def
->args_ct
[o
];
2228 /* The output sets oalias. */
2229 def
->args_ct
[o
].oalias
= 1;
2230 def
->args_ct
[o
].alias_index
= i
;
2231 /* The input sets ialias. */
2232 def
->args_ct
[i
].ialias
= 1;
2233 def
->args_ct
[i
].alias_index
= o
;
2234 if (def
->args_ct
[i
].pair
) {
2235 saw_alias_pair
= true;
2237 tcg_debug_assert(ct_str
[1] == '\0');
2241 tcg_debug_assert(!input_p
);
2242 def
->args_ct
[i
].newreg
= true;
2246 case 'p': /* plus */
2247 /* Allocate to the register after the previous. */
2248 tcg_debug_assert(i
> (input_p
? def
->nb_oargs
: 0));
2250 tcg_debug_assert(!def
->args_ct
[o
].pair
);
2251 tcg_debug_assert(!def
->args_ct
[o
].ct
);
2252 def
->args_ct
[i
] = (TCGArgConstraint
){
2255 .regs
= def
->args_ct
[o
].regs
<< 1,
2257 def
->args_ct
[o
].pair
= 1;
2258 def
->args_ct
[o
].pair_index
= i
;
2259 tcg_debug_assert(ct_str
[1] == '\0');
2262 case 'm': /* minus */
2263 /* Allocate to the register before the previous. */
2264 tcg_debug_assert(i
> (input_p
? def
->nb_oargs
: 0));
2266 tcg_debug_assert(!def
->args_ct
[o
].pair
);
2267 tcg_debug_assert(!def
->args_ct
[o
].ct
);
2268 def
->args_ct
[i
] = (TCGArgConstraint
){
2271 .regs
= def
->args_ct
[o
].regs
>> 1,
2273 def
->args_ct
[o
].pair
= 2;
2274 def
->args_ct
[o
].pair_index
= i
;
2275 tcg_debug_assert(ct_str
[1] == '\0');
2282 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
2285 /* Include all of the target-specific constraints. */
2288 #define CONST(CASE, MASK) \
2289 case CASE: def->args_ct[i].ct |= MASK; break;
2290 #define REGS(CASE, MASK) \
2291 case CASE: def->args_ct[i].regs |= MASK; break;
2293 #include "tcg-target-con-str.h"
2302 /* Typo in TCGTargetOpDef constraint. */
2303 g_assert_not_reached();
2305 } while (*++ct_str
!= '\0');
2308 /* TCGTargetOpDef entry with too much information? */
2309 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
2312 * Fix up output pairs that are aliased with inputs.
2313 * When we created the alias, we copied pair from the output.
2314 * There are three cases:
2315 * (1a) Pairs of inputs alias pairs of outputs.
2316 * (1b) One input aliases the first of a pair of outputs.
2317 * (2) One input aliases the second of a pair of outputs.
2319 * Case 1a is handled by making sure that the pair_index'es are
2320 * properly updated so that they appear the same as a pair of inputs.
2322 * Case 1b is handled by setting the pair_index of the input to
2323 * itself, simply so it doesn't point to an unrelated argument.
2324 * Since we don't encounter the "second" during the input allocation
2325 * phase, nothing happens with the second half of the input pair.
2327 * Case 2 is handled by setting the second input to pair=3, the
2328 * first output to pair=3, and the pair_index'es to match.
2330 if (saw_alias_pair
) {
2331 for (i
= def
->nb_oargs
; i
< nb_args
; i
++) {
2333 * Since [0-9pm] must be alone in the constraint string,
2334 * the only way they can both be set is if the pair comes
2335 * from the output alias.
2337 if (!def
->args_ct
[i
].ialias
) {
2340 switch (def
->args_ct
[i
].pair
) {
2344 o
= def
->args_ct
[i
].alias_index
;
2345 o2
= def
->args_ct
[o
].pair_index
;
2346 tcg_debug_assert(def
->args_ct
[o
].pair
== 1);
2347 tcg_debug_assert(def
->args_ct
[o2
].pair
== 2);
2348 if (def
->args_ct
[o2
].oalias
) {
2350 i2
= def
->args_ct
[o2
].alias_index
;
2351 tcg_debug_assert(def
->args_ct
[i2
].pair
== 2);
2352 def
->args_ct
[i2
].pair_index
= i
;
2353 def
->args_ct
[i
].pair_index
= i2
;
2356 def
->args_ct
[i
].pair_index
= i
;
2360 o
= def
->args_ct
[i
].alias_index
;
2361 o2
= def
->args_ct
[o
].pair_index
;
2362 tcg_debug_assert(def
->args_ct
[o
].pair
== 2);
2363 tcg_debug_assert(def
->args_ct
[o2
].pair
== 1);
2364 if (def
->args_ct
[o2
].oalias
) {
2366 i2
= def
->args_ct
[o2
].alias_index
;
2367 tcg_debug_assert(def
->args_ct
[i2
].pair
== 1);
2368 def
->args_ct
[i2
].pair_index
= i
;
2369 def
->args_ct
[i
].pair_index
= i2
;
2372 def
->args_ct
[i
].pair
= 3;
2373 def
->args_ct
[o2
].pair
= 3;
2374 def
->args_ct
[i
].pair_index
= o2
;
2375 def
->args_ct
[o2
].pair_index
= i
;
2379 g_assert_not_reached();
2384 /* sort the constraints (XXX: this is just an heuristic) */
2385 sort_constraints(def
, 0, def
->nb_oargs
);
2386 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
2390 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
2396 label
= arg_label(op
->args
[0]);
2399 case INDEX_op_brcond_i32
:
2400 case INDEX_op_brcond_i64
:
2401 label
= arg_label(op
->args
[3]);
2404 case INDEX_op_brcond2_i32
:
2405 label
= arg_label(op
->args
[5]);
2412 QTAILQ_REMOVE(&s
->ops
, op
, link
);
2413 QTAILQ_INSERT_TAIL(&s
->free_ops
, op
, link
);
2416 #ifdef CONFIG_PROFILER
2417 qatomic_set(&s
->prof
.del_op_count
, s
->prof
.del_op_count
+ 1);
2421 void tcg_remove_ops_after(TCGOp
*op
)
2423 TCGContext
*s
= tcg_ctx
;
2426 TCGOp
*last
= tcg_last_op();
2430 tcg_op_remove(s
, last
);
2434 static TCGOp
*tcg_op_alloc(TCGOpcode opc
, unsigned nargs
)
2436 TCGContext
*s
= tcg_ctx
;
2439 if (unlikely(!QTAILQ_EMPTY(&s
->free_ops
))) {
2440 QTAILQ_FOREACH(op
, &s
->free_ops
, link
) {
2441 if (nargs
<= op
->nargs
) {
2442 QTAILQ_REMOVE(&s
->free_ops
, op
, link
);
2449 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2450 nargs
= MAX(4, nargs
);
2451 op
= tcg_malloc(sizeof(TCGOp
) + sizeof(TCGArg
) * nargs
);
2454 memset(op
, 0, offsetof(TCGOp
, link
));
2458 /* Check for bitfield overflow. */
2459 tcg_debug_assert(op
->nargs
== nargs
);
2465 TCGOp
*tcg_emit_op(TCGOpcode opc
, unsigned nargs
)
2467 TCGOp
*op
= tcg_op_alloc(opc
, nargs
);
2468 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
2472 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
,
2473 TCGOpcode opc
, unsigned nargs
)
2475 TCGOp
*new_op
= tcg_op_alloc(opc
, nargs
);
2476 QTAILQ_INSERT_BEFORE(old_op
, new_op
, link
);
2480 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
,
2481 TCGOpcode opc
, unsigned nargs
)
2483 TCGOp
*new_op
= tcg_op_alloc(opc
, nargs
);
2484 QTAILQ_INSERT_AFTER(&s
->ops
, old_op
, new_op
, link
);
2488 /* Reachable analysis : remove unreachable code. */
2489 static void reachable_code_pass(TCGContext
*s
)
2491 TCGOp
*op
, *op_next
;
2494 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2499 case INDEX_op_set_label
:
2500 label
= arg_label(op
->args
[0]);
2501 if (label
->refs
== 0) {
2503 * While there is an occasional backward branch, virtually
2504 * all branches generated by the translators are forward.
2505 * Which means that generally we will have already removed
2506 * all references to the label that will be, and there is
2507 * little to be gained by iterating.
2511 /* Once we see a label, insns become live again. */
2516 * Optimization can fold conditional branches to unconditional.
2517 * If we find a label with one reference which is preceded by
2518 * an unconditional branch to it, remove both. This needed to
2519 * wait until the dead code in between them was removed.
2521 if (label
->refs
== 1) {
2522 TCGOp
*op_prev
= QTAILQ_PREV(op
, link
);
2523 if (op_prev
->opc
== INDEX_op_br
&&
2524 label
== arg_label(op_prev
->args
[0])) {
2525 tcg_op_remove(s
, op_prev
);
2533 case INDEX_op_exit_tb
:
2534 case INDEX_op_goto_ptr
:
2535 /* Unconditional branches; everything following is dead. */
2540 /* Notice noreturn helper calls, raising exceptions. */
2541 if (tcg_call_flags(op
) & TCG_CALL_NO_RETURN
) {
2546 case INDEX_op_insn_start
:
2547 /* Never remove -- we need to keep these for unwind. */
2556 tcg_op_remove(s
, op
);
2564 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2565 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2567 /* For liveness_pass_1, the register preferences for a given temp. */
2568 static inline TCGRegSet
*la_temp_pref(TCGTemp
*ts
)
2570 return ts
->state_ptr
;
2573 /* For liveness_pass_1, reset the preferences for a given temp to the
2574 * maximal regset for its type.
2576 static inline void la_reset_pref(TCGTemp
*ts
)
2579 = (ts
->state
== TS_DEAD
? 0 : tcg_target_available_regs
[ts
->type
]);
2582 /* liveness analysis: end of function: all temps are dead, and globals
2583 should be in memory. */
2584 static void la_func_end(TCGContext
*s
, int ng
, int nt
)
2588 for (i
= 0; i
< ng
; ++i
) {
2589 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2590 la_reset_pref(&s
->temps
[i
]);
2592 for (i
= ng
; i
< nt
; ++i
) {
2593 s
->temps
[i
].state
= TS_DEAD
;
2594 la_reset_pref(&s
->temps
[i
]);
2598 /* liveness analysis: end of basic block: all temps are dead, globals
2599 and local temps should be in memory. */
2600 static void la_bb_end(TCGContext
*s
, int ng
, int nt
)
2604 for (i
= 0; i
< nt
; ++i
) {
2605 TCGTemp
*ts
= &s
->temps
[i
];
2612 state
= TS_DEAD
| TS_MEM
;
2620 g_assert_not_reached();
2627 /* liveness analysis: sync globals back to memory. */
2628 static void la_global_sync(TCGContext
*s
, int ng
)
2632 for (i
= 0; i
< ng
; ++i
) {
2633 int state
= s
->temps
[i
].state
;
2634 s
->temps
[i
].state
= state
| TS_MEM
;
2635 if (state
== TS_DEAD
) {
2636 /* If the global was previously dead, reset prefs. */
2637 la_reset_pref(&s
->temps
[i
]);
2643 * liveness analysis: conditional branch: all temps are dead unless
2644 * explicitly live-across-conditional-branch, globals and local temps
2647 static void la_bb_sync(TCGContext
*s
, int ng
, int nt
)
2649 la_global_sync(s
, ng
);
2651 for (int i
= ng
; i
< nt
; ++i
) {
2652 TCGTemp
*ts
= &s
->temps
[i
];
2658 ts
->state
= state
| TS_MEM
;
2659 if (state
!= TS_DEAD
) {
2664 s
->temps
[i
].state
= TS_DEAD
;
2670 g_assert_not_reached();
2672 la_reset_pref(&s
->temps
[i
]);
2676 /* liveness analysis: sync globals back to memory and kill. */
2677 static void la_global_kill(TCGContext
*s
, int ng
)
2681 for (i
= 0; i
< ng
; i
++) {
2682 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2683 la_reset_pref(&s
->temps
[i
]);
2687 /* liveness analysis: note live globals crossing calls. */
2688 static void la_cross_call(TCGContext
*s
, int nt
)
2690 TCGRegSet mask
= ~tcg_target_call_clobber_regs
;
2693 for (i
= 0; i
< nt
; i
++) {
2694 TCGTemp
*ts
= &s
->temps
[i
];
2695 if (!(ts
->state
& TS_DEAD
)) {
2696 TCGRegSet
*pset
= la_temp_pref(ts
);
2697 TCGRegSet set
= *pset
;
2700 /* If the combination is not possible, restart. */
2702 set
= tcg_target_available_regs
[ts
->type
] & mask
;
2709 /* Liveness analysis : update the opc_arg_life array to tell if a
2710 given input arguments is dead. Instructions updating dead
2711 temporaries are removed. */
2712 static void liveness_pass_1(TCGContext
*s
)
2714 int nb_globals
= s
->nb_globals
;
2715 int nb_temps
= s
->nb_temps
;
2716 TCGOp
*op
, *op_prev
;
2720 prefs
= tcg_malloc(sizeof(TCGRegSet
) * nb_temps
);
2721 for (i
= 0; i
< nb_temps
; ++i
) {
2722 s
->temps
[i
].state_ptr
= prefs
+ i
;
2725 /* ??? Should be redundant with the exit_tb that ends the TB. */
2726 la_func_end(s
, nb_globals
, nb_temps
);
2728 QTAILQ_FOREACH_REVERSE_SAFE(op
, &s
->ops
, link
, op_prev
) {
2729 int nb_iargs
, nb_oargs
;
2730 TCGOpcode opc_new
, opc_new2
;
2732 TCGLifeData arg_life
= 0;
2734 TCGOpcode opc
= op
->opc
;
2735 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2740 const TCGHelperInfo
*info
= tcg_call_info(op
);
2741 int call_flags
= tcg_call_flags(op
);
2743 nb_oargs
= TCGOP_CALLO(op
);
2744 nb_iargs
= TCGOP_CALLI(op
);
2746 /* pure functions can be removed if their result is unused */
2747 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
2748 for (i
= 0; i
< nb_oargs
; i
++) {
2749 ts
= arg_temp(op
->args
[i
]);
2750 if (ts
->state
!= TS_DEAD
) {
2751 goto do_not_remove_call
;
2758 /* Output args are dead. */
2759 for (i
= 0; i
< nb_oargs
; i
++) {
2760 ts
= arg_temp(op
->args
[i
]);
2761 if (ts
->state
& TS_DEAD
) {
2762 arg_life
|= DEAD_ARG
<< i
;
2764 if (ts
->state
& TS_MEM
) {
2765 arg_life
|= SYNC_ARG
<< i
;
2767 ts
->state
= TS_DEAD
;
2771 /* Not used -- it will be tcg_target_call_oarg_reg(). */
2772 memset(op
->output_pref
, 0, sizeof(op
->output_pref
));
2774 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
2775 TCG_CALL_NO_READ_GLOBALS
))) {
2776 la_global_kill(s
, nb_globals
);
2777 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
2778 la_global_sync(s
, nb_globals
);
2781 /* Record arguments that die in this helper. */
2782 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2783 ts
= arg_temp(op
->args
[i
]);
2784 if (ts
->state
& TS_DEAD
) {
2785 arg_life
|= DEAD_ARG
<< i
;
2789 /* For all live registers, remove call-clobbered prefs. */
2790 la_cross_call(s
, nb_temps
);
2793 * Input arguments are live for preceding opcodes.
2795 * For those arguments that die, and will be allocated in
2796 * registers, clear the register set for that arg, to be
2797 * filled in below. For args that will be on the stack,
2798 * reset to any available reg. Process arguments in reverse
2799 * order so that if a temp is used more than once, the stack
2800 * reset to max happens before the register reset to 0.
2802 for (i
= nb_iargs
- 1; i
>= 0; i
--) {
2803 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
2804 ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
2806 if (ts
->state
& TS_DEAD
) {
2807 switch (loc
->kind
) {
2808 case TCG_CALL_ARG_NORMAL
:
2809 case TCG_CALL_ARG_EXTEND_U
:
2810 case TCG_CALL_ARG_EXTEND_S
:
2812 *la_temp_pref(ts
) = 0;
2818 tcg_target_available_regs
[ts
->type
];
2821 ts
->state
&= ~TS_DEAD
;
2826 * For each input argument, add its input register to prefs.
2827 * If a temp is used once, this produces a single set bit;
2828 * if a temp is used multiple times, this produces a set.
2830 for (i
= 0; i
< nb_iargs
; i
++) {
2831 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
2832 ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
2834 switch (loc
->kind
) {
2835 case TCG_CALL_ARG_NORMAL
:
2836 case TCG_CALL_ARG_EXTEND_U
:
2837 case TCG_CALL_ARG_EXTEND_S
:
2839 tcg_regset_set_reg(*la_temp_pref(ts
),
2840 tcg_target_call_iarg_regs
[loc
->arg_slot
]);
2849 case INDEX_op_insn_start
:
2851 case INDEX_op_discard
:
2852 /* mark the temporary as dead */
2853 ts
= arg_temp(op
->args
[0]);
2854 ts
->state
= TS_DEAD
;
2858 case INDEX_op_add2_i32
:
2859 opc_new
= INDEX_op_add_i32
;
2861 case INDEX_op_sub2_i32
:
2862 opc_new
= INDEX_op_sub_i32
;
2864 case INDEX_op_add2_i64
:
2865 opc_new
= INDEX_op_add_i64
;
2867 case INDEX_op_sub2_i64
:
2868 opc_new
= INDEX_op_sub_i64
;
2872 /* Test if the high part of the operation is dead, but not
2873 the low part. The result can be optimized to a simple
2874 add or sub. This happens often for x86_64 guest when the
2875 cpu mode is set to 32 bit. */
2876 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2877 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2880 /* Replace the opcode and adjust the args in place,
2881 leaving 3 unused args at the end. */
2882 op
->opc
= opc
= opc_new
;
2883 op
->args
[1] = op
->args
[2];
2884 op
->args
[2] = op
->args
[4];
2885 /* Fall through and mark the single-word operation live. */
2891 case INDEX_op_mulu2_i32
:
2892 opc_new
= INDEX_op_mul_i32
;
2893 opc_new2
= INDEX_op_muluh_i32
;
2894 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
2896 case INDEX_op_muls2_i32
:
2897 opc_new
= INDEX_op_mul_i32
;
2898 opc_new2
= INDEX_op_mulsh_i32
;
2899 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
2901 case INDEX_op_mulu2_i64
:
2902 opc_new
= INDEX_op_mul_i64
;
2903 opc_new2
= INDEX_op_muluh_i64
;
2904 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
2906 case INDEX_op_muls2_i64
:
2907 opc_new
= INDEX_op_mul_i64
;
2908 opc_new2
= INDEX_op_mulsh_i64
;
2909 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
2914 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2915 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2916 /* Both parts of the operation are dead. */
2919 /* The high part of the operation is dead; generate the low. */
2920 op
->opc
= opc
= opc_new
;
2921 op
->args
[1] = op
->args
[2];
2922 op
->args
[2] = op
->args
[3];
2923 } else if (arg_temp(op
->args
[0])->state
== TS_DEAD
&& have_opc_new2
) {
2924 /* The low part of the operation is dead; generate the high. */
2925 op
->opc
= opc
= opc_new2
;
2926 op
->args
[0] = op
->args
[1];
2927 op
->args
[1] = op
->args
[2];
2928 op
->args
[2] = op
->args
[3];
2932 /* Mark the single-word operation live. */
2937 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2938 nb_iargs
= def
->nb_iargs
;
2939 nb_oargs
= def
->nb_oargs
;
2941 /* Test if the operation can be removed because all
2942 its outputs are dead. We assume that nb_oargs == 0
2943 implies side effects */
2944 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
2945 for (i
= 0; i
< nb_oargs
; i
++) {
2946 if (arg_temp(op
->args
[i
])->state
!= TS_DEAD
) {
2955 tcg_op_remove(s
, op
);
2959 for (i
= 0; i
< nb_oargs
; i
++) {
2960 ts
= arg_temp(op
->args
[i
]);
2962 /* Remember the preference of the uses that followed. */
2963 if (i
< ARRAY_SIZE(op
->output_pref
)) {
2964 op
->output_pref
[i
] = *la_temp_pref(ts
);
2967 /* Output args are dead. */
2968 if (ts
->state
& TS_DEAD
) {
2969 arg_life
|= DEAD_ARG
<< i
;
2971 if (ts
->state
& TS_MEM
) {
2972 arg_life
|= SYNC_ARG
<< i
;
2974 ts
->state
= TS_DEAD
;
2978 /* If end of basic block, update. */
2979 if (def
->flags
& TCG_OPF_BB_EXIT
) {
2980 la_func_end(s
, nb_globals
, nb_temps
);
2981 } else if (def
->flags
& TCG_OPF_COND_BRANCH
) {
2982 la_bb_sync(s
, nb_globals
, nb_temps
);
2983 } else if (def
->flags
& TCG_OPF_BB_END
) {
2984 la_bb_end(s
, nb_globals
, nb_temps
);
2985 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2986 la_global_sync(s
, nb_globals
);
2987 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2988 la_cross_call(s
, nb_temps
);
2992 /* Record arguments that die in this opcode. */
2993 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2994 ts
= arg_temp(op
->args
[i
]);
2995 if (ts
->state
& TS_DEAD
) {
2996 arg_life
|= DEAD_ARG
<< i
;
3000 /* Input arguments are live for preceding opcodes. */
3001 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3002 ts
= arg_temp(op
->args
[i
]);
3003 if (ts
->state
& TS_DEAD
) {
3004 /* For operands that were dead, initially allow
3005 all regs for the type. */
3006 *la_temp_pref(ts
) = tcg_target_available_regs
[ts
->type
];
3007 ts
->state
&= ~TS_DEAD
;
3011 /* Incorporate constraints for this operand. */
3013 case INDEX_op_mov_i32
:
3014 case INDEX_op_mov_i64
:
3015 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3016 have proper constraints. That said, special case
3017 moves to propagate preferences backward. */
3018 if (IS_DEAD_ARG(1)) {
3019 *la_temp_pref(arg_temp(op
->args
[0]))
3020 = *la_temp_pref(arg_temp(op
->args
[1]));
3025 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3026 const TCGArgConstraint
*ct
= &def
->args_ct
[i
];
3027 TCGRegSet set
, *pset
;
3029 ts
= arg_temp(op
->args
[i
]);
3030 pset
= la_temp_pref(ts
);
3035 set
&= output_pref(op
, ct
->alias_index
);
3037 /* If the combination is not possible, restart. */
3047 op
->life
= arg_life
;
3051 /* Liveness analysis: Convert indirect regs to direct temporaries. */
3052 static bool liveness_pass_2(TCGContext
*s
)
3054 int nb_globals
= s
->nb_globals
;
3056 bool changes
= false;
3057 TCGOp
*op
, *op_next
;
3059 /* Create a temporary for each indirect global. */
3060 for (i
= 0; i
< nb_globals
; ++i
) {
3061 TCGTemp
*its
= &s
->temps
[i
];
3062 if (its
->indirect_reg
) {
3063 TCGTemp
*dts
= tcg_temp_alloc(s
);
3064 dts
->type
= its
->type
;
3065 dts
->base_type
= its
->base_type
;
3066 dts
->kind
= TEMP_EBB
;
3067 its
->state_ptr
= dts
;
3069 its
->state_ptr
= NULL
;
3071 /* All globals begin dead. */
3072 its
->state
= TS_DEAD
;
3074 for (nb_temps
= s
->nb_temps
; i
< nb_temps
; ++i
) {
3075 TCGTemp
*its
= &s
->temps
[i
];
3076 its
->state_ptr
= NULL
;
3077 its
->state
= TS_DEAD
;
3080 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
3081 TCGOpcode opc
= op
->opc
;
3082 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
3083 TCGLifeData arg_life
= op
->life
;
3084 int nb_iargs
, nb_oargs
, call_flags
;
3085 TCGTemp
*arg_ts
, *dir_ts
;
3087 if (opc
== INDEX_op_call
) {
3088 nb_oargs
= TCGOP_CALLO(op
);
3089 nb_iargs
= TCGOP_CALLI(op
);
3090 call_flags
= tcg_call_flags(op
);
3092 nb_iargs
= def
->nb_iargs
;
3093 nb_oargs
= def
->nb_oargs
;
3095 /* Set flags similar to how calls require. */
3096 if (def
->flags
& TCG_OPF_COND_BRANCH
) {
3097 /* Like reading globals: sync_globals */
3098 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
3099 } else if (def
->flags
& TCG_OPF_BB_END
) {
3100 /* Like writing globals: save_globals */
3102 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
3103 /* Like reading globals: sync_globals */
3104 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
3106 /* No effect on globals. */
3107 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
3108 TCG_CALL_NO_WRITE_GLOBALS
);
3112 /* Make sure that input arguments are available. */
3113 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3114 arg_ts
= arg_temp(op
->args
[i
]);
3115 dir_ts
= arg_ts
->state_ptr
;
3116 if (dir_ts
&& arg_ts
->state
== TS_DEAD
) {
3117 TCGOpcode lopc
= (arg_ts
->type
== TCG_TYPE_I32
3120 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
, 3);
3122 lop
->args
[0] = temp_arg(dir_ts
);
3123 lop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3124 lop
->args
[2] = arg_ts
->mem_offset
;
3126 /* Loaded, but synced with memory. */
3127 arg_ts
->state
= TS_MEM
;
3131 /* Perform input replacement, and mark inputs that became dead.
3132 No action is required except keeping temp_state up to date
3133 so that we reload when needed. */
3134 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3135 arg_ts
= arg_temp(op
->args
[i
]);
3136 dir_ts
= arg_ts
->state_ptr
;
3138 op
->args
[i
] = temp_arg(dir_ts
);
3140 if (IS_DEAD_ARG(i
)) {
3141 arg_ts
->state
= TS_DEAD
;
3146 /* Liveness analysis should ensure that the following are
3147 all correct, for call sites and basic block end points. */
3148 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
3150 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
3151 for (i
= 0; i
< nb_globals
; ++i
) {
3152 /* Liveness should see that globals are synced back,
3153 that is, either TS_DEAD or TS_MEM. */
3154 arg_ts
= &s
->temps
[i
];
3155 tcg_debug_assert(arg_ts
->state_ptr
== 0
3156 || arg_ts
->state
!= 0);
3159 for (i
= 0; i
< nb_globals
; ++i
) {
3160 /* Liveness should see that globals are saved back,
3161 that is, TS_DEAD, waiting to be reloaded. */
3162 arg_ts
= &s
->temps
[i
];
3163 tcg_debug_assert(arg_ts
->state_ptr
== 0
3164 || arg_ts
->state
== TS_DEAD
);
3168 /* Outputs become available. */
3169 if (opc
== INDEX_op_mov_i32
|| opc
== INDEX_op_mov_i64
) {
3170 arg_ts
= arg_temp(op
->args
[0]);
3171 dir_ts
= arg_ts
->state_ptr
;
3173 op
->args
[0] = temp_arg(dir_ts
);
3176 /* The output is now live and modified. */
3179 if (NEED_SYNC_ARG(0)) {
3180 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
3183 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
3184 TCGTemp
*out_ts
= dir_ts
;
3186 if (IS_DEAD_ARG(0)) {
3187 out_ts
= arg_temp(op
->args
[1]);
3188 arg_ts
->state
= TS_DEAD
;
3189 tcg_op_remove(s
, op
);
3191 arg_ts
->state
= TS_MEM
;
3194 sop
->args
[0] = temp_arg(out_ts
);
3195 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3196 sop
->args
[2] = arg_ts
->mem_offset
;
3198 tcg_debug_assert(!IS_DEAD_ARG(0));
3202 for (i
= 0; i
< nb_oargs
; i
++) {
3203 arg_ts
= arg_temp(op
->args
[i
]);
3204 dir_ts
= arg_ts
->state_ptr
;
3208 op
->args
[i
] = temp_arg(dir_ts
);
3211 /* The output is now live and modified. */
3214 /* Sync outputs upon their last write. */
3215 if (NEED_SYNC_ARG(i
)) {
3216 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
3219 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
3221 sop
->args
[0] = temp_arg(dir_ts
);
3222 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3223 sop
->args
[2] = arg_ts
->mem_offset
;
3225 arg_ts
->state
= TS_MEM
;
3227 /* Drop outputs that are dead. */
3228 if (IS_DEAD_ARG(i
)) {
3229 arg_ts
->state
= TS_DEAD
;
3238 static void temp_allocate_frame(TCGContext
*s
, TCGTemp
*ts
)
3240 int size
= tcg_type_size(ts
->type
);
3254 /* Note that we do not require aligned storage for V256. */
3258 g_assert_not_reached();
3262 * Assume the stack is sufficiently aligned.
3263 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3264 * and do not require 16 byte vector alignment. This seems slightly
3265 * easier than fully parameterizing the above switch statement.
3267 align
= MIN(TCG_TARGET_STACK_ALIGN
, align
);
3268 off
= ROUND_UP(s
->current_frame_offset
, align
);
3270 /* If we've exhausted the stack frame, restart with a smaller TB. */
3271 if (off
+ size
> s
->frame_end
) {
3272 tcg_raise_tb_overflow(s
);
3274 s
->current_frame_offset
= off
+ size
;
3276 ts
->mem_offset
= off
;
3277 #if defined(__sparc__)
3278 ts
->mem_offset
+= TCG_TARGET_STACK_BIAS
;
3280 ts
->mem_base
= s
->frame_temp
;
3281 ts
->mem_allocated
= 1;
3284 /* Assign @reg to @ts, and update reg_to_temp[]. */
3285 static void set_temp_val_reg(TCGContext
*s
, TCGTemp
*ts
, TCGReg reg
)
3287 if (ts
->val_type
== TEMP_VAL_REG
) {
3288 TCGReg old
= ts
->reg
;
3289 tcg_debug_assert(s
->reg_to_temp
[old
] == ts
);
3293 s
->reg_to_temp
[old
] = NULL
;
3295 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
3296 s
->reg_to_temp
[reg
] = ts
;
3297 ts
->val_type
= TEMP_VAL_REG
;
3301 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3302 static void set_temp_val_nonreg(TCGContext
*s
, TCGTemp
*ts
, TCGTempVal type
)
3304 tcg_debug_assert(type
!= TEMP_VAL_REG
);
3305 if (ts
->val_type
== TEMP_VAL_REG
) {
3306 TCGReg reg
= ts
->reg
;
3307 tcg_debug_assert(s
->reg_to_temp
[reg
] == ts
);
3308 s
->reg_to_temp
[reg
] = NULL
;
3310 ts
->val_type
= type
;
3313 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
, TCGRegSet
);
3315 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3316 mark it free; otherwise mark it dead. */
3317 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
3319 TCGTempVal new_type
;
3326 new_type
= TEMP_VAL_MEM
;
3330 new_type
= free_or_dead
< 0 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
;
3333 new_type
= TEMP_VAL_CONST
;
3336 g_assert_not_reached();
3338 set_temp_val_nonreg(s
, ts
, new_type
);
3341 /* Mark a temporary as dead. */
3342 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
3344 temp_free_or_dead(s
, ts
, 1);
3347 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3348 registers needs to be allocated to store a constant. If 'free_or_dead'
3349 is non-zero, subsequently release the temporary; if it is positive, the
3350 temp is dead; if it is negative, the temp is free. */
3351 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
,
3352 TCGRegSet preferred_regs
, int free_or_dead
)
3354 if (!temp_readonly(ts
) && !ts
->mem_coherent
) {
3355 if (!ts
->mem_allocated
) {
3356 temp_allocate_frame(s
, ts
);
3358 switch (ts
->val_type
) {
3359 case TEMP_VAL_CONST
:
3360 /* If we're going to free the temp immediately, then we won't
3361 require it later in a register, so attempt to store the
3362 constant to memory directly. */
3364 && tcg_out_sti(s
, ts
->type
, ts
->val
,
3365 ts
->mem_base
->reg
, ts
->mem_offset
)) {
3368 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3369 allocated_regs
, preferred_regs
);
3373 tcg_out_st(s
, ts
->type
, ts
->reg
,
3374 ts
->mem_base
->reg
, ts
->mem_offset
);
3384 ts
->mem_coherent
= 1;
3387 temp_free_or_dead(s
, ts
, free_or_dead
);
3391 /* free register 'reg' by spilling the corresponding temporary if necessary */
3392 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
3394 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
3396 temp_sync(s
, ts
, allocated_regs
, 0, -1);
3402 * @required_regs: Set of registers in which we must allocate.
3403 * @allocated_regs: Set of registers which must be avoided.
3404 * @preferred_regs: Set of registers we should prefer.
3405 * @rev: True if we search the registers in "indirect" order.
3407 * The allocated register must be in @required_regs & ~@allocated_regs,
3408 * but if we can put it in @preferred_regs we may save a move later.
3410 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet required_regs
,
3411 TCGRegSet allocated_regs
,
3412 TCGRegSet preferred_regs
, bool rev
)
3414 int i
, j
, f
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3415 TCGRegSet reg_ct
[2];
3418 reg_ct
[1] = required_regs
& ~allocated_regs
;
3419 tcg_debug_assert(reg_ct
[1] != 0);
3420 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3422 /* Skip the preferred_regs option if it cannot be satisfied,
3423 or if the preference made no difference. */
3424 f
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3426 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3428 /* Try free registers, preferences first. */
3429 for (j
= f
; j
< 2; j
++) {
3430 TCGRegSet set
= reg_ct
[j
];
3432 if (tcg_regset_single(set
)) {
3433 /* One register in the set. */
3434 TCGReg reg
= tcg_regset_first(set
);
3435 if (s
->reg_to_temp
[reg
] == NULL
) {
3439 for (i
= 0; i
< n
; i
++) {
3440 TCGReg reg
= order
[i
];
3441 if (s
->reg_to_temp
[reg
] == NULL
&&
3442 tcg_regset_test_reg(set
, reg
)) {
3449 /* We must spill something. */
3450 for (j
= f
; j
< 2; j
++) {
3451 TCGRegSet set
= reg_ct
[j
];
3453 if (tcg_regset_single(set
)) {
3454 /* One register in the set. */
3455 TCGReg reg
= tcg_regset_first(set
);
3456 tcg_reg_free(s
, reg
, allocated_regs
);
3459 for (i
= 0; i
< n
; i
++) {
3460 TCGReg reg
= order
[i
];
3461 if (tcg_regset_test_reg(set
, reg
)) {
3462 tcg_reg_free(s
, reg
, allocated_regs
);
3472 static TCGReg
tcg_reg_alloc_pair(TCGContext
*s
, TCGRegSet required_regs
,
3473 TCGRegSet allocated_regs
,
3474 TCGRegSet preferred_regs
, bool rev
)
3476 int i
, j
, k
, fmin
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3477 TCGRegSet reg_ct
[2];
3480 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3481 reg_ct
[1] = required_regs
& ~(allocated_regs
| (allocated_regs
>> 1));
3482 tcg_debug_assert(reg_ct
[1] != 0);
3483 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3485 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3488 * Skip the preferred_regs option if it cannot be satisfied,
3489 * or if the preference made no difference.
3491 k
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3494 * Minimize the number of flushes by looking for 2 free registers first,
3495 * then a single flush, then two flushes.
3497 for (fmin
= 2; fmin
>= 0; fmin
--) {
3498 for (j
= k
; j
< 2; j
++) {
3499 TCGRegSet set
= reg_ct
[j
];
3501 for (i
= 0; i
< n
; i
++) {
3502 TCGReg reg
= order
[i
];
3504 if (tcg_regset_test_reg(set
, reg
)) {
3505 int f
= !s
->reg_to_temp
[reg
] + !s
->reg_to_temp
[reg
+ 1];
3507 tcg_reg_free(s
, reg
, allocated_regs
);
3508 tcg_reg_free(s
, reg
+ 1, allocated_regs
);
3518 /* Make sure the temporary is in a register. If needed, allocate the register
3519 from DESIRED while avoiding ALLOCATED. */
3520 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
3521 TCGRegSet allocated_regs
, TCGRegSet preferred_regs
)
3525 switch (ts
->val_type
) {
3528 case TEMP_VAL_CONST
:
3529 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3530 preferred_regs
, ts
->indirect_base
);
3531 if (ts
->type
<= TCG_TYPE_I64
) {
3532 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
3534 uint64_t val
= ts
->val
;
3538 * Find the minimal vector element that matches the constant.
3539 * The targets will, in general, have to do this search anyway,
3540 * do this generically.
3542 if (val
== dup_const(MO_8
, val
)) {
3544 } else if (val
== dup_const(MO_16
, val
)) {
3546 } else if (val
== dup_const(MO_32
, val
)) {
3550 tcg_out_dupi_vec(s
, ts
->type
, vece
, reg
, ts
->val
);
3552 ts
->mem_coherent
= 0;
3555 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3556 preferred_regs
, ts
->indirect_base
);
3557 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
3558 ts
->mem_coherent
= 1;
3564 set_temp_val_reg(s
, ts
, reg
);
3567 /* Save a temporary to memory. 'allocated_regs' is used in case a
3568 temporary registers needs to be allocated to store a constant. */
3569 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
3571 /* The liveness analysis already ensures that globals are back
3572 in memory. Keep an tcg_debug_assert for safety. */
3573 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| temp_readonly(ts
));
3576 /* save globals to their canonical location and assume they can be
3577 modified be the following code. 'allocated_regs' is used in case a
3578 temporary registers needs to be allocated to store a constant. */
3579 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3583 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3584 temp_save(s
, &s
->temps
[i
], allocated_regs
);
3588 /* sync globals to their canonical location and assume they can be
3589 read by the following code. 'allocated_regs' is used in case a
3590 temporary registers needs to be allocated to store a constant. */
3591 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3595 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3596 TCGTemp
*ts
= &s
->temps
[i
];
3597 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
3598 || ts
->kind
== TEMP_FIXED
3599 || ts
->mem_coherent
);
3603 /* at the end of a basic block, we assume all temporaries are dead and
3604 all globals are stored at their canonical location. */
3605 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
3609 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3610 TCGTemp
*ts
= &s
->temps
[i
];
3614 temp_save(s
, ts
, allocated_regs
);
3618 /* The liveness analysis already ensures that temps are dead.
3619 Keep an tcg_debug_assert for safety. */
3620 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3623 /* Similarly, we should have freed any allocated register. */
3624 tcg_debug_assert(ts
->val_type
== TEMP_VAL_CONST
);
3627 g_assert_not_reached();
3631 save_globals(s
, allocated_regs
);
3635 * At a conditional branch, we assume all temporaries are dead unless
3636 * explicitly live-across-conditional-branch; all globals and local
3637 * temps are synced to their location.
3639 static void tcg_reg_alloc_cbranch(TCGContext
*s
, TCGRegSet allocated_regs
)
3641 sync_globals(s
, allocated_regs
);
3643 for (int i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3644 TCGTemp
*ts
= &s
->temps
[i
];
3646 * The liveness analysis already ensures that temps are dead.
3647 * Keep tcg_debug_asserts for safety.
3651 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
|| ts
->mem_coherent
);
3654 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3660 g_assert_not_reached();
3666 * Specialized code generation for INDEX_op_mov_* with a constant.
3668 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
3669 tcg_target_ulong val
, TCGLifeData arg_life
,
3670 TCGRegSet preferred_regs
)
3672 /* ENV should not be modified. */
3673 tcg_debug_assert(!temp_readonly(ots
));
3675 /* The movi is not explicitly generated here. */
3676 set_temp_val_nonreg(s
, ots
, TEMP_VAL_CONST
);
3678 ots
->mem_coherent
= 0;
3679 if (NEED_SYNC_ARG(0)) {
3680 temp_sync(s
, ots
, s
->reserved_regs
, preferred_regs
, IS_DEAD_ARG(0));
3681 } else if (IS_DEAD_ARG(0)) {
3687 * Specialized code generation for INDEX_op_mov_*.
3689 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOp
*op
)
3691 const TCGLifeData arg_life
= op
->life
;
3692 TCGRegSet allocated_regs
, preferred_regs
;
3694 TCGType otype
, itype
;
3697 allocated_regs
= s
->reserved_regs
;
3698 preferred_regs
= output_pref(op
, 0);
3699 ots
= arg_temp(op
->args
[0]);
3700 ts
= arg_temp(op
->args
[1]);
3702 /* ENV should not be modified. */
3703 tcg_debug_assert(!temp_readonly(ots
));
3705 /* Note that otype != itype for no-op truncation. */
3709 if (ts
->val_type
== TEMP_VAL_CONST
) {
3710 /* propagate constant or generate sti */
3711 tcg_target_ulong val
= ts
->val
;
3712 if (IS_DEAD_ARG(1)) {
3715 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, preferred_regs
);
3719 /* If the source value is in memory we're going to be forced
3720 to have it in a register in order to perform the copy. Copy
3721 the SOURCE value into its own register first, that way we
3722 don't have to reload SOURCE the next time it is used. */
3723 if (ts
->val_type
== TEMP_VAL_MEM
) {
3724 temp_load(s
, ts
, tcg_target_available_regs
[itype
],
3725 allocated_regs
, preferred_regs
);
3727 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
3730 if (IS_DEAD_ARG(0)) {
3731 /* mov to a non-saved dead register makes no sense (even with
3732 liveness analysis disabled). */
3733 tcg_debug_assert(NEED_SYNC_ARG(0));
3734 if (!ots
->mem_allocated
) {
3735 temp_allocate_frame(s
, ots
);
3737 tcg_out_st(s
, otype
, ireg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3738 if (IS_DEAD_ARG(1)) {
3745 if (IS_DEAD_ARG(1) && ts
->kind
!= TEMP_FIXED
) {
3747 * The mov can be suppressed. Kill input first, so that it
3748 * is unlinked from reg_to_temp, then set the output to the
3749 * reg that we saved from the input.
3754 if (ots
->val_type
== TEMP_VAL_REG
) {
3757 /* Make sure to not spill the input register during allocation. */
3758 oreg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
3759 allocated_regs
| ((TCGRegSet
)1 << ireg
),
3760 preferred_regs
, ots
->indirect_base
);
3762 if (!tcg_out_mov(s
, otype
, oreg
, ireg
)) {
3764 * Cross register class move not supported.
3765 * Store the source register into the destination slot
3766 * and leave the destination temp as TEMP_VAL_MEM.
3768 assert(!temp_readonly(ots
));
3769 if (!ts
->mem_allocated
) {
3770 temp_allocate_frame(s
, ots
);
3772 tcg_out_st(s
, ts
->type
, ireg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3773 set_temp_val_nonreg(s
, ts
, TEMP_VAL_MEM
);
3774 ots
->mem_coherent
= 1;
3778 set_temp_val_reg(s
, ots
, oreg
);
3779 ots
->mem_coherent
= 0;
3781 if (NEED_SYNC_ARG(0)) {
3782 temp_sync(s
, ots
, allocated_regs
, 0, 0);
3787 * Specialized code generation for INDEX_op_dup_vec.
3789 static void tcg_reg_alloc_dup(TCGContext
*s
, const TCGOp
*op
)
3791 const TCGLifeData arg_life
= op
->life
;
3792 TCGRegSet dup_out_regs
, dup_in_regs
;
3794 TCGType itype
, vtype
;
3799 ots
= arg_temp(op
->args
[0]);
3800 its
= arg_temp(op
->args
[1]);
3802 /* ENV should not be modified. */
3803 tcg_debug_assert(!temp_readonly(ots
));
3806 vece
= TCGOP_VECE(op
);
3807 vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
3809 if (its
->val_type
== TEMP_VAL_CONST
) {
3810 /* Propagate constant via movi -> dupi. */
3811 tcg_target_ulong val
= its
->val
;
3812 if (IS_DEAD_ARG(1)) {
3815 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, output_pref(op
, 0));
3819 dup_out_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].regs
;
3820 dup_in_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[1].regs
;
3822 /* Allocate the output register now. */
3823 if (ots
->val_type
!= TEMP_VAL_REG
) {
3824 TCGRegSet allocated_regs
= s
->reserved_regs
;
3827 if (!IS_DEAD_ARG(1) && its
->val_type
== TEMP_VAL_REG
) {
3828 /* Make sure to not spill the input register. */
3829 tcg_regset_set_reg(allocated_regs
, its
->reg
);
3831 oreg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
3832 output_pref(op
, 0), ots
->indirect_base
);
3833 set_temp_val_reg(s
, ots
, oreg
);
3836 switch (its
->val_type
) {
3839 * The dup constriaints must be broad, covering all possible VECE.
3840 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3841 * to fail, indicating that extra moves are required for that case.
3843 if (tcg_regset_test_reg(dup_in_regs
, its
->reg
)) {
3844 if (tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, its
->reg
)) {
3847 /* Try again from memory or a vector input register. */
3849 if (!its
->mem_coherent
) {
3851 * The input register is not synced, and so an extra store
3852 * would be required to use memory. Attempt an integer-vector
3853 * register move first. We do not have a TCGRegSet for this.
3855 if (tcg_out_mov(s
, itype
, ots
->reg
, its
->reg
)) {
3858 /* Sync the temp back to its slot and load from there. */
3859 temp_sync(s
, its
, s
->reserved_regs
, 0, 0);
3865 if (HOST_BIG_ENDIAN
) {
3866 lowpart_ofs
= tcg_type_size(itype
) - (1 << vece
);
3868 if (tcg_out_dupm_vec(s
, vtype
, vece
, ots
->reg
, its
->mem_base
->reg
,
3869 its
->mem_offset
+ lowpart_ofs
)) {
3872 /* Load the input into the destination vector register. */
3873 tcg_out_ld(s
, itype
, ots
->reg
, its
->mem_base
->reg
, its
->mem_offset
);
3877 g_assert_not_reached();
3880 /* We now have a vector input register, so dup must succeed. */
3881 ok
= tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, ots
->reg
);
3882 tcg_debug_assert(ok
);
3885 ots
->mem_coherent
= 0;
3886 if (IS_DEAD_ARG(1)) {
3889 if (NEED_SYNC_ARG(0)) {
3890 temp_sync(s
, ots
, s
->reserved_regs
, 0, 0);
3892 if (IS_DEAD_ARG(0)) {
3897 static void tcg_reg_alloc_op(TCGContext
*s
, const TCGOp
*op
)
3899 const TCGLifeData arg_life
= op
->life
;
3900 const TCGOpDef
* const def
= &tcg_op_defs
[op
->opc
];
3901 TCGRegSet i_allocated_regs
;
3902 TCGRegSet o_allocated_regs
;
3903 int i
, k
, nb_iargs
, nb_oargs
;
3906 const TCGArgConstraint
*arg_ct
;
3908 TCGArg new_args
[TCG_MAX_OP_ARGS
];
3909 int const_args
[TCG_MAX_OP_ARGS
];
3911 nb_oargs
= def
->nb_oargs
;
3912 nb_iargs
= def
->nb_iargs
;
3914 /* copy constants */
3915 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
3916 op
->args
+ nb_oargs
+ nb_iargs
,
3917 sizeof(TCGArg
) * def
->nb_cargs
);
3919 i_allocated_regs
= s
->reserved_regs
;
3920 o_allocated_regs
= s
->reserved_regs
;
3922 /* satisfy input constraints */
3923 for (k
= 0; k
< nb_iargs
; k
++) {
3924 TCGRegSet i_preferred_regs
, i_required_regs
;
3925 bool allocate_new_reg
, copyto_new_reg
;
3929 i
= def
->args_ct
[nb_oargs
+ k
].sort_index
;
3931 arg_ct
= &def
->args_ct
[i
];
3934 if (ts
->val_type
== TEMP_VAL_CONST
3935 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
->ct
)) {
3936 /* constant is OK for instruction */
3938 new_args
[i
] = ts
->val
;
3943 i_preferred_regs
= 0;
3944 i_required_regs
= arg_ct
->regs
;
3945 allocate_new_reg
= false;
3946 copyto_new_reg
= false;
3948 switch (arg_ct
->pair
) {
3949 case 0: /* not paired */
3950 if (arg_ct
->ialias
) {
3951 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
3954 * If the input is readonly, then it cannot also be an
3955 * output and aliased to itself. If the input is not
3956 * dead after the instruction, we must allocate a new
3957 * register and move it.
3959 if (temp_readonly(ts
) || !IS_DEAD_ARG(i
)) {
3960 allocate_new_reg
= true;
3961 } else if (ts
->val_type
== TEMP_VAL_REG
) {
3963 * Check if the current register has already been
3964 * allocated for another input.
3967 tcg_regset_test_reg(i_allocated_regs
, reg
);
3970 if (!allocate_new_reg
) {
3971 temp_load(s
, ts
, i_required_regs
, i_allocated_regs
,
3974 allocate_new_reg
= !tcg_regset_test_reg(i_required_regs
, reg
);
3976 if (allocate_new_reg
) {
3978 * Allocate a new register matching the constraint
3979 * and move the temporary register into it.
3981 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3982 i_allocated_regs
, 0);
3983 reg
= tcg_reg_alloc(s
, i_required_regs
, i_allocated_regs
,
3984 i_preferred_regs
, ts
->indirect_base
);
3985 copyto_new_reg
= true;
3990 /* First of an input pair; if i1 == i2, the second is an output. */
3992 i2
= arg_ct
->pair_index
;
3993 ts2
= i1
!= i2
? arg_temp(op
->args
[i2
]) : NULL
;
3996 * It is easier to default to allocating a new pair
3997 * and to identify a few cases where it's not required.
3999 if (arg_ct
->ialias
) {
4000 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
4001 if (IS_DEAD_ARG(i1
) &&
4003 !temp_readonly(ts
) &&
4004 ts
->val_type
== TEMP_VAL_REG
&&
4005 ts
->reg
< TCG_TARGET_NB_REGS
- 1 &&
4006 tcg_regset_test_reg(i_required_regs
, reg
) &&
4007 !tcg_regset_test_reg(i_allocated_regs
, reg
) &&
4008 !tcg_regset_test_reg(i_allocated_regs
, reg
+ 1) &&
4010 ? ts2
->val_type
== TEMP_VAL_REG
&&
4011 ts2
->reg
== reg
+ 1 &&
4013 : s
->reg_to_temp
[reg
+ 1] == NULL
)) {
4017 /* Without aliasing, the pair must also be an input. */
4018 tcg_debug_assert(ts2
);
4019 if (ts
->val_type
== TEMP_VAL_REG
&&
4020 ts2
->val_type
== TEMP_VAL_REG
&&
4021 ts2
->reg
== reg
+ 1 &&
4022 tcg_regset_test_reg(i_required_regs
, reg
)) {
4026 reg
= tcg_reg_alloc_pair(s
, i_required_regs
, i_allocated_regs
,
4027 0, ts
->indirect_base
);
4030 case 2: /* pair second */
4031 reg
= new_args
[arg_ct
->pair_index
] + 1;
4034 case 3: /* ialias with second output, no first input */
4035 tcg_debug_assert(arg_ct
->ialias
);
4036 i_preferred_regs
= output_pref(op
, arg_ct
->alias_index
);
4038 if (IS_DEAD_ARG(i
) &&
4039 !temp_readonly(ts
) &&
4040 ts
->val_type
== TEMP_VAL_REG
&&
4042 s
->reg_to_temp
[reg
- 1] == NULL
&&
4043 tcg_regset_test_reg(i_required_regs
, reg
) &&
4044 !tcg_regset_test_reg(i_allocated_regs
, reg
) &&
4045 !tcg_regset_test_reg(i_allocated_regs
, reg
- 1)) {
4046 tcg_regset_set_reg(i_allocated_regs
, reg
- 1);
4049 reg
= tcg_reg_alloc_pair(s
, i_required_regs
>> 1,
4050 i_allocated_regs
, 0,
4052 tcg_regset_set_reg(i_allocated_regs
, reg
);
4058 * If an aliased input is not dead after the instruction,
4059 * we must allocate a new register and move it.
4061 if (arg_ct
->ialias
&& (!IS_DEAD_ARG(i
) || temp_readonly(ts
))) {
4062 TCGRegSet t_allocated_regs
= i_allocated_regs
;
4065 * Because of the alias, and the continued life, make sure
4066 * that the temp is somewhere *other* than the reg pair,
4067 * and we get a copy in reg.
4069 tcg_regset_set_reg(t_allocated_regs
, reg
);
4070 tcg_regset_set_reg(t_allocated_regs
, reg
+ 1);
4071 if (ts
->val_type
== TEMP_VAL_REG
&& ts
->reg
== reg
) {
4072 /* If ts was already in reg, copy it somewhere else. */
4076 tcg_debug_assert(ts
->kind
!= TEMP_FIXED
);
4077 nr
= tcg_reg_alloc(s
, tcg_target_available_regs
[ts
->type
],
4078 t_allocated_regs
, 0, ts
->indirect_base
);
4079 ok
= tcg_out_mov(s
, ts
->type
, nr
, reg
);
4080 tcg_debug_assert(ok
);
4082 set_temp_val_reg(s
, ts
, nr
);
4084 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
4085 t_allocated_regs
, 0);
4086 copyto_new_reg
= true;
4089 /* Preferably allocate to reg, otherwise copy. */
4090 i_required_regs
= (TCGRegSet
)1 << reg
;
4091 temp_load(s
, ts
, i_required_regs
, i_allocated_regs
,
4093 copyto_new_reg
= ts
->reg
!= reg
;
4098 g_assert_not_reached();
4101 if (copyto_new_reg
) {
4102 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
4104 * Cross register class move not supported. Sync the
4105 * temp back to its slot and load from there.
4107 temp_sync(s
, ts
, i_allocated_regs
, 0, 0);
4108 tcg_out_ld(s
, ts
->type
, reg
,
4109 ts
->mem_base
->reg
, ts
->mem_offset
);
4114 tcg_regset_set_reg(i_allocated_regs
, reg
);
4117 /* mark dead temporaries and free the associated registers */
4118 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
4119 if (IS_DEAD_ARG(i
)) {
4120 temp_dead(s
, arg_temp(op
->args
[i
]));
4124 if (def
->flags
& TCG_OPF_COND_BRANCH
) {
4125 tcg_reg_alloc_cbranch(s
, i_allocated_regs
);
4126 } else if (def
->flags
& TCG_OPF_BB_END
) {
4127 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
4129 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
4130 /* XXX: permit generic clobber register list ? */
4131 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
4132 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
4133 tcg_reg_free(s
, i
, i_allocated_regs
);
4137 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
4138 /* sync globals if the op has side effects and might trigger
4140 sync_globals(s
, i_allocated_regs
);
4143 /* satisfy the output constraints */
4144 for(k
= 0; k
< nb_oargs
; k
++) {
4145 i
= def
->args_ct
[k
].sort_index
;
4147 arg_ct
= &def
->args_ct
[i
];
4150 /* ENV should not be modified. */
4151 tcg_debug_assert(!temp_readonly(ts
));
4153 switch (arg_ct
->pair
) {
4154 case 0: /* not paired */
4155 if (arg_ct
->oalias
&& !const_args
[arg_ct
->alias_index
]) {
4156 reg
= new_args
[arg_ct
->alias_index
];
4157 } else if (arg_ct
->newreg
) {
4158 reg
= tcg_reg_alloc(s
, arg_ct
->regs
,
4159 i_allocated_regs
| o_allocated_regs
,
4160 output_pref(op
, k
), ts
->indirect_base
);
4162 reg
= tcg_reg_alloc(s
, arg_ct
->regs
, o_allocated_regs
,
4163 output_pref(op
, k
), ts
->indirect_base
);
4167 case 1: /* first of pair */
4168 tcg_debug_assert(!arg_ct
->newreg
);
4169 if (arg_ct
->oalias
) {
4170 reg
= new_args
[arg_ct
->alias_index
];
4173 reg
= tcg_reg_alloc_pair(s
, arg_ct
->regs
, o_allocated_regs
,
4174 output_pref(op
, k
), ts
->indirect_base
);
4177 case 2: /* second of pair */
4178 tcg_debug_assert(!arg_ct
->newreg
);
4179 if (arg_ct
->oalias
) {
4180 reg
= new_args
[arg_ct
->alias_index
];
4182 reg
= new_args
[arg_ct
->pair_index
] + 1;
4186 case 3: /* first of pair, aliasing with a second input */
4187 tcg_debug_assert(!arg_ct
->newreg
);
4188 reg
= new_args
[arg_ct
->pair_index
] - 1;
4192 g_assert_not_reached();
4194 tcg_regset_set_reg(o_allocated_regs
, reg
);
4195 set_temp_val_reg(s
, ts
, reg
);
4196 ts
->mem_coherent
= 0;
4201 /* emit instruction */
4202 if (def
->flags
& TCG_OPF_VECTOR
) {
4203 tcg_out_vec_op(s
, op
->opc
, TCGOP_VECL(op
), TCGOP_VECE(op
),
4204 new_args
, const_args
);
4206 tcg_out_op(s
, op
->opc
, new_args
, const_args
);
4209 /* move the outputs in the correct register if needed */
4210 for(i
= 0; i
< nb_oargs
; i
++) {
4211 ts
= arg_temp(op
->args
[i
]);
4213 /* ENV should not be modified. */
4214 tcg_debug_assert(!temp_readonly(ts
));
4216 if (NEED_SYNC_ARG(i
)) {
4217 temp_sync(s
, ts
, o_allocated_regs
, 0, IS_DEAD_ARG(i
));
4218 } else if (IS_DEAD_ARG(i
)) {
4224 static bool tcg_reg_alloc_dup2(TCGContext
*s
, const TCGOp
*op
)
4226 const TCGLifeData arg_life
= op
->life
;
4227 TCGTemp
*ots
, *itsl
, *itsh
;
4228 TCGType vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
4230 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4231 tcg_debug_assert(TCG_TARGET_REG_BITS
== 32);
4232 tcg_debug_assert(TCGOP_VECE(op
) == MO_64
);
4234 ots
= arg_temp(op
->args
[0]);
4235 itsl
= arg_temp(op
->args
[1]);
4236 itsh
= arg_temp(op
->args
[2]);
4238 /* ENV should not be modified. */
4239 tcg_debug_assert(!temp_readonly(ots
));
4241 /* Allocate the output register now. */
4242 if (ots
->val_type
!= TEMP_VAL_REG
) {
4243 TCGRegSet allocated_regs
= s
->reserved_regs
;
4244 TCGRegSet dup_out_regs
=
4245 tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].regs
;
4248 /* Make sure to not spill the input registers. */
4249 if (!IS_DEAD_ARG(1) && itsl
->val_type
== TEMP_VAL_REG
) {
4250 tcg_regset_set_reg(allocated_regs
, itsl
->reg
);
4252 if (!IS_DEAD_ARG(2) && itsh
->val_type
== TEMP_VAL_REG
) {
4253 tcg_regset_set_reg(allocated_regs
, itsh
->reg
);
4256 oreg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
4257 output_pref(op
, 0), ots
->indirect_base
);
4258 set_temp_val_reg(s
, ots
, oreg
);
4261 /* Promote dup2 of immediates to dupi_vec. */
4262 if (itsl
->val_type
== TEMP_VAL_CONST
&& itsh
->val_type
== TEMP_VAL_CONST
) {
4263 uint64_t val
= deposit64(itsl
->val
, 32, 32, itsh
->val
);
4266 if (val
== dup_const(MO_8
, val
)) {
4268 } else if (val
== dup_const(MO_16
, val
)) {
4270 } else if (val
== dup_const(MO_32
, val
)) {
4274 tcg_out_dupi_vec(s
, vtype
, vece
, ots
->reg
, val
);
4278 /* If the two inputs form one 64-bit value, try dupm_vec. */
4279 if (itsl
->temp_subindex
== HOST_BIG_ENDIAN
&&
4280 itsh
->temp_subindex
== !HOST_BIG_ENDIAN
&&
4281 itsl
== itsh
+ (HOST_BIG_ENDIAN
? 1 : -1)) {
4282 TCGTemp
*its
= itsl
- HOST_BIG_ENDIAN
;
4284 temp_sync(s
, its
+ 0, s
->reserved_regs
, 0, 0);
4285 temp_sync(s
, its
+ 1, s
->reserved_regs
, 0, 0);
4287 if (tcg_out_dupm_vec(s
, vtype
, MO_64
, ots
->reg
,
4288 its
->mem_base
->reg
, its
->mem_offset
)) {
4293 /* Fall back to generic expansion. */
4297 ots
->mem_coherent
= 0;
4298 if (IS_DEAD_ARG(1)) {
4301 if (IS_DEAD_ARG(2)) {
4304 if (NEED_SYNC_ARG(0)) {
4305 temp_sync(s
, ots
, s
->reserved_regs
, 0, IS_DEAD_ARG(0));
4306 } else if (IS_DEAD_ARG(0)) {
4312 static void load_arg_reg(TCGContext
*s
, TCGReg reg
, TCGTemp
*ts
,
4313 TCGRegSet allocated_regs
)
4315 if (ts
->val_type
== TEMP_VAL_REG
) {
4316 if (ts
->reg
!= reg
) {
4317 tcg_reg_free(s
, reg
, allocated_regs
);
4318 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
4320 * Cross register class move not supported. Sync the
4321 * temp back to its slot and load from there.
4323 temp_sync(s
, ts
, allocated_regs
, 0, 0);
4324 tcg_out_ld(s
, ts
->type
, reg
,
4325 ts
->mem_base
->reg
, ts
->mem_offset
);
4329 TCGRegSet arg_set
= 0;
4331 tcg_reg_free(s
, reg
, allocated_regs
);
4332 tcg_regset_set_reg(arg_set
, reg
);
4333 temp_load(s
, ts
, arg_set
, allocated_regs
, 0);
4337 static void load_arg_stk(TCGContext
*s
, int stk_slot
, TCGTemp
*ts
,
4338 TCGRegSet allocated_regs
)
4341 * When the destination is on the stack, load up the temp and store.
4342 * If there are many call-saved registers, the temp might live to
4343 * see another use; otherwise it'll be discarded.
4345 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
], allocated_regs
, 0);
4346 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
,
4347 TCG_TARGET_CALL_STACK_OFFSET
+
4348 stk_slot
* sizeof(tcg_target_long
));
4351 static void load_arg_normal(TCGContext
*s
, const TCGCallArgumentLoc
*l
,
4352 TCGTemp
*ts
, TCGRegSet
*allocated_regs
)
4355 TCGReg reg
= tcg_target_call_iarg_regs
[l
->arg_slot
];
4356 load_arg_reg(s
, reg
, ts
, *allocated_regs
);
4357 tcg_regset_set_reg(*allocated_regs
, reg
);
4359 load_arg_stk(s
, l
->arg_slot
- ARRAY_SIZE(tcg_target_call_iarg_regs
),
4360 ts
, *allocated_regs
);
4364 static void tcg_reg_alloc_call(TCGContext
*s
, TCGOp
*op
)
4366 const int nb_oargs
= TCGOP_CALLO(op
);
4367 const int nb_iargs
= TCGOP_CALLI(op
);
4368 const TCGLifeData arg_life
= op
->life
;
4369 const TCGHelperInfo
*info
= tcg_call_info(op
);
4370 TCGRegSet allocated_regs
= s
->reserved_regs
;
4374 * Move inputs into place in reverse order,
4375 * so that we place stacked arguments first.
4377 for (i
= nb_iargs
- 1; i
>= 0; --i
) {
4378 const TCGCallArgumentLoc
*loc
= &info
->in
[i
];
4379 TCGTemp
*ts
= arg_temp(op
->args
[nb_oargs
+ i
]);
4381 switch (loc
->kind
) {
4382 case TCG_CALL_ARG_NORMAL
:
4383 case TCG_CALL_ARG_EXTEND_U
:
4384 case TCG_CALL_ARG_EXTEND_S
:
4385 load_arg_normal(s
, loc
, ts
, &allocated_regs
);
4388 g_assert_not_reached();
4392 /* Mark dead temporaries and free the associated registers. */
4393 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
4394 if (IS_DEAD_ARG(i
)) {
4395 temp_dead(s
, arg_temp(op
->args
[i
]));
4399 /* Clobber call registers. */
4400 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
4401 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
4402 tcg_reg_free(s
, i
, allocated_regs
);
4407 * Save globals if they might be written by the helper,
4408 * sync them if they might be read.
4410 if (info
->flags
& TCG_CALL_NO_READ_GLOBALS
) {
4412 } else if (info
->flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
4413 sync_globals(s
, allocated_regs
);
4415 save_globals(s
, allocated_regs
);
4418 tcg_out_call(s
, tcg_call_func(op
), info
);
4420 /* Assign output registers and emit moves if needed. */
4421 switch (info
->out_kind
) {
4422 case TCG_CALL_RET_NORMAL
:
4423 for (i
= 0; i
< nb_oargs
; i
++) {
4424 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
4425 TCGReg reg
= tcg_target_call_oarg_regs
[i
];
4427 /* ENV should not be modified. */
4428 tcg_debug_assert(!temp_readonly(ts
));
4430 set_temp_val_reg(s
, ts
, reg
);
4431 ts
->mem_coherent
= 0;
4435 g_assert_not_reached();
4438 /* Flush or discard output registers as needed. */
4439 for (i
= 0; i
< nb_oargs
; i
++) {
4440 TCGTemp
*ts
= arg_temp(op
->args
[i
]);
4441 if (NEED_SYNC_ARG(i
)) {
4442 temp_sync(s
, ts
, s
->reserved_regs
, 0, IS_DEAD_ARG(i
));
4443 } else if (IS_DEAD_ARG(i
)) {
4449 #ifdef CONFIG_PROFILER
4451 /* avoid copy/paste errors */
4452 #define PROF_ADD(to, from, field) \
4454 (to)->field += qatomic_read(&((from)->field)); \
4457 #define PROF_MAX(to, from, field) \
4459 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
4460 if (val__ > (to)->field) { \
4461 (to)->field = val__; \
4465 /* Pass in a zero'ed @prof */
4467 void tcg_profile_snapshot(TCGProfile
*prof
, bool counters
, bool table
)
4469 unsigned int n_ctxs
= qatomic_read(&tcg_cur_ctxs
);
4472 for (i
= 0; i
< n_ctxs
; i
++) {
4473 TCGContext
*s
= qatomic_read(&tcg_ctxs
[i
]);
4474 const TCGProfile
*orig
= &s
->prof
;
4477 PROF_ADD(prof
, orig
, cpu_exec_time
);
4478 PROF_ADD(prof
, orig
, tb_count1
);
4479 PROF_ADD(prof
, orig
, tb_count
);
4480 PROF_ADD(prof
, orig
, op_count
);
4481 PROF_MAX(prof
, orig
, op_count_max
);
4482 PROF_ADD(prof
, orig
, temp_count
);
4483 PROF_MAX(prof
, orig
, temp_count_max
);
4484 PROF_ADD(prof
, orig
, del_op_count
);
4485 PROF_ADD(prof
, orig
, code_in_len
);
4486 PROF_ADD(prof
, orig
, code_out_len
);
4487 PROF_ADD(prof
, orig
, search_out_len
);
4488 PROF_ADD(prof
, orig
, interm_time
);
4489 PROF_ADD(prof
, orig
, code_time
);
4490 PROF_ADD(prof
, orig
, la_time
);
4491 PROF_ADD(prof
, orig
, opt_time
);
4492 PROF_ADD(prof
, orig
, restore_count
);
4493 PROF_ADD(prof
, orig
, restore_time
);
4498 for (i
= 0; i
< NB_OPS
; i
++) {
4499 PROF_ADD(prof
, orig
, table_op_count
[i
]);
4508 static void tcg_profile_snapshot_counters(TCGProfile
*prof
)
4510 tcg_profile_snapshot(prof
, true, false);
4513 static void tcg_profile_snapshot_table(TCGProfile
*prof
)
4515 tcg_profile_snapshot(prof
, false, true);
4518 void tcg_dump_op_count(GString
*buf
)
4520 TCGProfile prof
= {};
4523 tcg_profile_snapshot_table(&prof
);
4524 for (i
= 0; i
< NB_OPS
; i
++) {
4525 g_string_append_printf(buf
, "%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
4526 prof
.table_op_count
[i
]);
4530 int64_t tcg_cpu_exec_time(void)
4532 unsigned int n_ctxs
= qatomic_read(&tcg_cur_ctxs
);
4536 for (i
= 0; i
< n_ctxs
; i
++) {
4537 const TCGContext
*s
= qatomic_read(&tcg_ctxs
[i
]);
4538 const TCGProfile
*prof
= &s
->prof
;
4540 ret
+= qatomic_read(&prof
->cpu_exec_time
);
4545 void tcg_dump_op_count(GString
*buf
)
4547 g_string_append_printf(buf
, "[TCG profiler not compiled]\n");
4550 int64_t tcg_cpu_exec_time(void)
4552 error_report("%s: TCG profiler not compiled", __func__
);
4558 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
, target_ulong pc_start
)
4560 #ifdef CONFIG_PROFILER
4561 TCGProfile
*prof
= &s
->prof
;
4566 #ifdef CONFIG_PROFILER
4570 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4573 qatomic_set(&prof
->op_count
, prof
->op_count
+ n
);
4574 if (n
> prof
->op_count_max
) {
4575 qatomic_set(&prof
->op_count_max
, n
);
4579 qatomic_set(&prof
->temp_count
, prof
->temp_count
+ n
);
4580 if (n
> prof
->temp_count_max
) {
4581 qatomic_set(&prof
->temp_count_max
, n
);
4587 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
4588 && qemu_log_in_addr_range(pc_start
))) {
4589 FILE *logfile
= qemu_log_trylock();
4591 fprintf(logfile
, "OP:\n");
4592 tcg_dump_ops(s
, logfile
, false);
4593 fprintf(logfile
, "\n");
4594 qemu_log_unlock(logfile
);
4599 #ifdef CONFIG_DEBUG_TCG
4600 /* Ensure all labels referenced have been emitted. */
4605 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
4606 if (unlikely(!l
->present
) && l
->refs
) {
4607 qemu_log_mask(CPU_LOG_TB_OP
,
4608 "$L%d referenced but not present.\n", l
->id
);
4616 #ifdef CONFIG_PROFILER
4617 qatomic_set(&prof
->opt_time
, prof
->opt_time
- profile_getclock());
4620 #ifdef USE_TCG_OPTIMIZATIONS
4624 #ifdef CONFIG_PROFILER
4625 qatomic_set(&prof
->opt_time
, prof
->opt_time
+ profile_getclock());
4626 qatomic_set(&prof
->la_time
, prof
->la_time
- profile_getclock());
4629 reachable_code_pass(s
);
4632 if (s
->nb_indirects
> 0) {
4634 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
4635 && qemu_log_in_addr_range(pc_start
))) {
4636 FILE *logfile
= qemu_log_trylock();
4638 fprintf(logfile
, "OP before indirect lowering:\n");
4639 tcg_dump_ops(s
, logfile
, false);
4640 fprintf(logfile
, "\n");
4641 qemu_log_unlock(logfile
);
4645 /* Replace indirect temps with direct temps. */
4646 if (liveness_pass_2(s
)) {
4647 /* If changes were made, re-run liveness. */
4652 #ifdef CONFIG_PROFILER
4653 qatomic_set(&prof
->la_time
, prof
->la_time
+ profile_getclock());
4657 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
4658 && qemu_log_in_addr_range(pc_start
))) {
4659 FILE *logfile
= qemu_log_trylock();
4661 fprintf(logfile
, "OP after optimization and liveness analysis:\n");
4662 tcg_dump_ops(s
, logfile
, true);
4663 fprintf(logfile
, "\n");
4664 qemu_log_unlock(logfile
);
4669 /* Initialize goto_tb jump offsets. */
4670 tb
->jmp_reset_offset
[0] = TB_JMP_OFFSET_INVALID
;
4671 tb
->jmp_reset_offset
[1] = TB_JMP_OFFSET_INVALID
;
4672 tb
->jmp_insn_offset
[0] = TB_JMP_OFFSET_INVALID
;
4673 tb
->jmp_insn_offset
[1] = TB_JMP_OFFSET_INVALID
;
4675 tcg_reg_alloc_start(s
);
4678 * Reset the buffer pointers when restarting after overflow.
4679 * TODO: Move this into translate-all.c with the rest of the
4680 * buffer management. Having only this done here is confusing.
4682 s
->code_buf
= tcg_splitwx_to_rw(tb
->tc
.ptr
);
4683 s
->code_ptr
= s
->code_buf
;
4685 #ifdef TCG_TARGET_NEED_LDST_LABELS
4686 QSIMPLEQ_INIT(&s
->ldst_labels
);
4688 #ifdef TCG_TARGET_NEED_POOL_LABELS
4689 s
->pool_labels
= NULL
;
4693 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4694 TCGOpcode opc
= op
->opc
;
4696 #ifdef CONFIG_PROFILER
4697 qatomic_set(&prof
->table_op_count
[opc
], prof
->table_op_count
[opc
] + 1);
4701 case INDEX_op_mov_i32
:
4702 case INDEX_op_mov_i64
:
4703 case INDEX_op_mov_vec
:
4704 tcg_reg_alloc_mov(s
, op
);
4706 case INDEX_op_dup_vec
:
4707 tcg_reg_alloc_dup(s
, op
);
4709 case INDEX_op_insn_start
:
4710 if (num_insns
>= 0) {
4711 size_t off
= tcg_current_code_size(s
);
4712 s
->gen_insn_end_off
[num_insns
] = off
;
4713 /* Assert that we do not overflow our stored offset. */
4714 assert(s
->gen_insn_end_off
[num_insns
] == off
);
4717 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
4719 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4720 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
4724 s
->gen_insn_data
[num_insns
][i
] = a
;
4727 case INDEX_op_discard
:
4728 temp_dead(s
, arg_temp(op
->args
[0]));
4730 case INDEX_op_set_label
:
4731 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
4732 tcg_out_label(s
, arg_label(op
->args
[0]));
4735 tcg_reg_alloc_call(s
, op
);
4737 case INDEX_op_exit_tb
:
4738 tcg_out_exit_tb(s
, op
->args
[0]);
4740 case INDEX_op_goto_tb
:
4741 tcg_out_goto_tb(s
, op
->args
[0]);
4743 case INDEX_op_dup2_vec
:
4744 if (tcg_reg_alloc_dup2(s
, op
)) {
4749 /* Sanity check that we've not introduced any unhandled opcodes. */
4750 tcg_debug_assert(tcg_op_supported(opc
));
4751 /* Note: in order to speed up the code, it would be much
4752 faster to have specialized register allocator functions for
4753 some common argument patterns */
4754 tcg_reg_alloc_op(s
, op
);
4757 /* Test for (pending) buffer overflow. The assumption is that any
4758 one operation beginning below the high water mark cannot overrun
4759 the buffer completely. Thus we can test for overflow after
4760 generating code without having to check during generation. */
4761 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
4764 /* Test for TB overflow, as seen by gen_insn_end_off. */
4765 if (unlikely(tcg_current_code_size(s
) > UINT16_MAX
)) {
4769 tcg_debug_assert(num_insns
>= 0);
4770 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
4772 /* Generate TB finalization at the end of block */
4773 #ifdef TCG_TARGET_NEED_LDST_LABELS
4774 i
= tcg_out_ldst_finalize(s
);
4779 #ifdef TCG_TARGET_NEED_POOL_LABELS
4780 i
= tcg_out_pool_finalize(s
);
4785 if (!tcg_resolve_relocs(s
)) {
4789 #ifndef CONFIG_TCG_INTERPRETER
4790 /* flush instruction cache */
4791 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s
->code_buf
),
4792 (uintptr_t)s
->code_buf
,
4793 tcg_ptr_byte_diff(s
->code_ptr
, s
->code_buf
));
4796 return tcg_current_code_size(s
);
4799 #ifdef CONFIG_PROFILER
4800 void tcg_dump_info(GString
*buf
)
4802 TCGProfile prof
= {};
4803 const TCGProfile
*s
;
4805 int64_t tb_div_count
;
4808 tcg_profile_snapshot_counters(&prof
);
4810 tb_count
= s
->tb_count
;
4811 tb_div_count
= tb_count
? tb_count
: 1;
4812 tot
= s
->interm_time
+ s
->code_time
;
4814 g_string_append_printf(buf
, "JIT cycles %" PRId64
4815 " (%0.3f s at 2.4 GHz)\n",
4817 g_string_append_printf(buf
, "translated TBs %" PRId64
4818 " (aborted=%" PRId64
" %0.1f%%)\n",
4819 tb_count
, s
->tb_count1
- tb_count
,
4820 (double)(s
->tb_count1
- s
->tb_count
)
4821 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
4822 g_string_append_printf(buf
, "avg ops/TB %0.1f max=%d\n",
4823 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
4824 g_string_append_printf(buf
, "deleted ops/TB %0.2f\n",
4825 (double)s
->del_op_count
/ tb_div_count
);
4826 g_string_append_printf(buf
, "avg temps/TB %0.2f max=%d\n",
4827 (double)s
->temp_count
/ tb_div_count
,
4829 g_string_append_printf(buf
, "avg host code/TB %0.1f\n",
4830 (double)s
->code_out_len
/ tb_div_count
);
4831 g_string_append_printf(buf
, "avg search data/TB %0.1f\n",
4832 (double)s
->search_out_len
/ tb_div_count
);
4834 g_string_append_printf(buf
, "cycles/op %0.1f\n",
4835 s
->op_count
? (double)tot
/ s
->op_count
: 0);
4836 g_string_append_printf(buf
, "cycles/in byte %0.1f\n",
4837 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
4838 g_string_append_printf(buf
, "cycles/out byte %0.1f\n",
4839 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
4840 g_string_append_printf(buf
, "cycles/search byte %0.1f\n",
4842 (double)tot
/ s
->search_out_len
: 0);
4846 g_string_append_printf(buf
, " gen_interm time %0.1f%%\n",
4847 (double)s
->interm_time
/ tot
* 100.0);
4848 g_string_append_printf(buf
, " gen_code time %0.1f%%\n",
4849 (double)s
->code_time
/ tot
* 100.0);
4850 g_string_append_printf(buf
, "optim./code time %0.1f%%\n",
4851 (double)s
->opt_time
/ (s
->code_time
?
4854 g_string_append_printf(buf
, "liveness/code time %0.1f%%\n",
4855 (double)s
->la_time
/ (s
->code_time
?
4856 s
->code_time
: 1) * 100.0);
4857 g_string_append_printf(buf
, "cpu_restore count %" PRId64
"\n",
4859 g_string_append_printf(buf
, " avg cycles %0.1f\n",
4861 (double)s
->restore_time
/ s
->restore_count
: 0);
4864 void tcg_dump_info(GString
*buf
)
4866 g_string_append_printf(buf
, "[TCG profiler not compiled]\n");
4870 #ifdef ELF_HOST_MACHINE
4871 /* In order to use this feature, the backend needs to do three things:
4873 (1) Define ELF_HOST_MACHINE to indicate both what value to
4874 put into the ELF image and to indicate support for the feature.
4876 (2) Define tcg_register_jit. This should create a buffer containing
4877 the contents of a .debug_frame section that describes the post-
4878 prologue unwind info for the tcg machine.
4880 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4883 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4890 struct jit_code_entry
{
4891 struct jit_code_entry
*next_entry
;
4892 struct jit_code_entry
*prev_entry
;
4893 const void *symfile_addr
;
4894 uint64_t symfile_size
;
4897 struct jit_descriptor
{
4899 uint32_t action_flag
;
4900 struct jit_code_entry
*relevant_entry
;
4901 struct jit_code_entry
*first_entry
;
4904 void __jit_debug_register_code(void) __attribute__((noinline
));
4905 void __jit_debug_register_code(void)
4910 /* Must statically initialize the version, because GDB may check
4911 the version before we can set it. */
4912 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
4914 /* End GDB interface. */
4916 static int find_string(const char *strtab
, const char *str
)
4918 const char *p
= strtab
+ 1;
4921 if (strcmp(p
, str
) == 0) {
4928 static void tcg_register_jit_int(const void *buf_ptr
, size_t buf_size
,
4929 const void *debug_frame
,
4930 size_t debug_frame_size
)
4932 struct __attribute__((packed
)) DebugInfo
{
4939 uintptr_t cu_low_pc
;
4940 uintptr_t cu_high_pc
;
4943 uintptr_t fn_low_pc
;
4944 uintptr_t fn_high_pc
;
4953 struct DebugInfo di
;
4958 struct ElfImage
*img
;
4960 static const struct ElfImage img_template
= {
4962 .e_ident
[EI_MAG0
] = ELFMAG0
,
4963 .e_ident
[EI_MAG1
] = ELFMAG1
,
4964 .e_ident
[EI_MAG2
] = ELFMAG2
,
4965 .e_ident
[EI_MAG3
] = ELFMAG3
,
4966 .e_ident
[EI_CLASS
] = ELF_CLASS
,
4967 .e_ident
[EI_DATA
] = ELF_DATA
,
4968 .e_ident
[EI_VERSION
] = EV_CURRENT
,
4970 .e_machine
= ELF_HOST_MACHINE
,
4971 .e_version
= EV_CURRENT
,
4972 .e_phoff
= offsetof(struct ElfImage
, phdr
),
4973 .e_shoff
= offsetof(struct ElfImage
, shdr
),
4974 .e_ehsize
= sizeof(ElfW(Shdr
)),
4975 .e_phentsize
= sizeof(ElfW(Phdr
)),
4977 .e_shentsize
= sizeof(ElfW(Shdr
)),
4978 .e_shnum
= ARRAY_SIZE(img
->shdr
),
4979 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
4980 #ifdef ELF_HOST_FLAGS
4981 .e_flags
= ELF_HOST_FLAGS
,
4984 .e_ident
[EI_OSABI
] = ELF_OSABI
,
4992 [0] = { .sh_type
= SHT_NULL
},
4993 /* Trick: The contents of code_gen_buffer are not present in
4994 this fake ELF file; that got allocated elsewhere. Therefore
4995 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4996 will not look for contents. We can record any address. */
4998 .sh_type
= SHT_NOBITS
,
4999 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
5001 [2] = { /* .debug_info */
5002 .sh_type
= SHT_PROGBITS
,
5003 .sh_offset
= offsetof(struct ElfImage
, di
),
5004 .sh_size
= sizeof(struct DebugInfo
),
5006 [3] = { /* .debug_abbrev */
5007 .sh_type
= SHT_PROGBITS
,
5008 .sh_offset
= offsetof(struct ElfImage
, da
),
5009 .sh_size
= sizeof(img
->da
),
5011 [4] = { /* .debug_frame */
5012 .sh_type
= SHT_PROGBITS
,
5013 .sh_offset
= sizeof(struct ElfImage
),
5015 [5] = { /* .symtab */
5016 .sh_type
= SHT_SYMTAB
,
5017 .sh_offset
= offsetof(struct ElfImage
, sym
),
5018 .sh_size
= sizeof(img
->sym
),
5020 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
5021 .sh_entsize
= sizeof(ElfW(Sym
)),
5023 [6] = { /* .strtab */
5024 .sh_type
= SHT_STRTAB
,
5025 .sh_offset
= offsetof(struct ElfImage
, str
),
5026 .sh_size
= sizeof(img
->str
),
5030 [1] = { /* code_gen_buffer */
5031 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
5036 .len
= sizeof(struct DebugInfo
) - 4,
5038 .ptr_size
= sizeof(void *),
5040 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
5042 .fn_name
= "code_gen_buffer"
5045 1, /* abbrev number (the cu) */
5046 0x11, 1, /* DW_TAG_compile_unit, has children */
5047 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
5048 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5049 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5050 0, 0, /* end of abbrev */
5051 2, /* abbrev number (the fn) */
5052 0x2e, 0, /* DW_TAG_subprogram, no children */
5053 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
5054 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
5055 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
5056 0, 0, /* end of abbrev */
5057 0 /* no more abbrev */
5059 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5060 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
5063 /* We only need a single jit entry; statically allocate it. */
5064 static struct jit_code_entry one_entry
;
5066 uintptr_t buf
= (uintptr_t)buf_ptr
;
5067 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
5068 DebugFrameHeader
*dfh
;
5070 img
= g_malloc(img_size
);
5071 *img
= img_template
;
5073 img
->phdr
.p_vaddr
= buf
;
5074 img
->phdr
.p_paddr
= buf
;
5075 img
->phdr
.p_memsz
= buf_size
;
5077 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
5078 img
->shdr
[1].sh_addr
= buf
;
5079 img
->shdr
[1].sh_size
= buf_size
;
5081 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
5082 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
5084 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
5085 img
->shdr
[4].sh_size
= debug_frame_size
;
5087 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
5088 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
5090 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
5091 img
->sym
[1].st_value
= buf
;
5092 img
->sym
[1].st_size
= buf_size
;
5094 img
->di
.cu_low_pc
= buf
;
5095 img
->di
.cu_high_pc
= buf
+ buf_size
;
5096 img
->di
.fn_low_pc
= buf
;
5097 img
->di
.fn_high_pc
= buf
+ buf_size
;
5099 dfh
= (DebugFrameHeader
*)(img
+ 1);
5100 memcpy(dfh
, debug_frame
, debug_frame_size
);
5101 dfh
->fde
.func_start
= buf
;
5102 dfh
->fde
.func_len
= buf_size
;
5105 /* Enable this block to be able to debug the ELF image file creation.
5106 One can use readelf, objdump, or other inspection utilities. */
5108 g_autofree
char *jit
= g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5109 FILE *f
= fopen(jit
, "w+b");
5111 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
5112 /* Avoid stupid unused return value warning for fwrite. */
5119 one_entry
.symfile_addr
= img
;
5120 one_entry
.symfile_size
= img_size
;
5122 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
5123 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
5124 __jit_debug_descriptor
.first_entry
= &one_entry
;
5125 __jit_debug_register_code();
5128 /* No support for the feature. Provide the entry point expected by exec.c,
5129 and implement the internal function we declared earlier. */
5131 static void tcg_register_jit_int(const void *buf
, size_t size
,
5132 const void *debug_frame
,
5133 size_t debug_frame_size
)
5137 void tcg_register_jit(const void *buf
, size_t buf_size
)
5140 #endif /* ELF_HOST_MACHINE */
5142 #if !TCG_TARGET_MAYBE_vec
5143 void tcg_expand_vec_op(TCGOpcode o
, TCGType t
, unsigned e
, TCGArg a0
, ...)
5145 g_assert_not_reached();