2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
39 /* Note: the long term plan is to reduce the dependencies on the QEMU
40 CPU definitions. Currently they are used for qemu_ld/st
42 #define NO_CPU_IO_DEFS
45 #include "exec/exec-all.h"
47 #if !defined(CONFIG_USER_ONLY)
48 #include "hw/boards.h"
51 #include "tcg/tcg-op.h"
53 #if UINTPTR_MAX == UINT32_MAX
54 # define ELF_CLASS ELFCLASS32
56 # define ELF_CLASS ELFCLASS64
58 #ifdef HOST_WORDS_BIGENDIAN
59 # define ELF_DATA ELFDATA2MSB
61 # define ELF_DATA ELFDATA2LSB
66 #include "sysemu/sysemu.h"
68 /* Forward declarations for functions declared in tcg-target.inc.c and
70 static void tcg_target_init(TCGContext
*s
);
71 static const TCGTargetOpDef
*tcg_target_op_def(TCGOpcode
);
72 static void tcg_target_qemu_prologue(TCGContext
*s
);
73 static bool patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
74 intptr_t value
, intptr_t addend
);
76 /* The CIE and FDE header definitions will be common to all hosts. */
78 uint32_t len
__attribute__((aligned((sizeof(void *)))));
84 uint8_t return_column
;
87 typedef struct QEMU_PACKED
{
88 uint32_t len
__attribute__((aligned((sizeof(void *)))));
92 } DebugFrameFDEHeader
;
94 typedef struct QEMU_PACKED
{
96 DebugFrameFDEHeader fde
;
99 static void tcg_register_jit_int(void *buf
, size_t size
,
100 const void *debug_frame
,
101 size_t debug_frame_size
)
102 __attribute__((unused
));
104 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
105 static const char *target_parse_constraint(TCGArgConstraint
*ct
,
106 const char *ct_str
, TCGType type
);
107 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
109 static bool tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
110 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
111 TCGReg ret
, tcg_target_long arg
);
112 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
, const TCGArg
*args
,
113 const int *const_args
);
114 #if TCG_TARGET_MAYBE_vec
115 static bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
116 TCGReg dst
, TCGReg src
);
117 static bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
118 TCGReg dst
, TCGReg base
, intptr_t offset
);
119 static void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
,
120 TCGReg dst
, tcg_target_long arg
);
121 static void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
122 unsigned vece
, const TCGArg
*args
,
123 const int *const_args
);
125 static inline bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
126 TCGReg dst
, TCGReg src
)
128 g_assert_not_reached();
130 static inline bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
131 TCGReg dst
, TCGReg base
, intptr_t offset
)
133 g_assert_not_reached();
135 static inline void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
,
136 TCGReg dst
, tcg_target_long arg
)
138 g_assert_not_reached();
140 static inline void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
141 unsigned vece
, const TCGArg
*args
,
142 const int *const_args
)
144 g_assert_not_reached();
147 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
149 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
150 TCGReg base
, intptr_t ofs
);
151 static void tcg_out_call(TCGContext
*s
, tcg_insn_unit
*target
);
152 static int tcg_target_const_match(tcg_target_long val
, TCGType type
,
153 const TCGArgConstraint
*arg_ct
);
154 #ifdef TCG_TARGET_NEED_LDST_LABELS
155 static int tcg_out_ldst_finalize(TCGContext
*s
);
158 #define TCG_HIGHWATER 1024
160 static TCGContext
**tcg_ctxs
;
161 static unsigned int n_tcg_ctxs
;
162 TCGv_env cpu_env
= 0;
164 struct tcg_region_tree
{
167 /* padding to avoid false sharing is computed at run-time */
171 * We divide code_gen_buffer into equally-sized "regions" that TCG threads
172 * dynamically allocate from as demand dictates. Given appropriate region
173 * sizing, this minimizes flushes even when some TCG threads generate a lot
174 * more code than others.
176 struct tcg_region_state
{
179 /* fields set at init time */
184 size_t size
; /* size of one region */
185 size_t stride
; /* .size + guard size */
187 /* fields protected by the lock */
188 size_t current
; /* current region index */
189 size_t agg_size_full
; /* aggregate size of full regions */
192 static struct tcg_region_state region
;
194 * This is an array of struct tcg_region_tree's, with padding.
195 * We use void * to simplify the computation of region_trees[i]; each
196 * struct is found every tree_size bytes.
198 static void *region_trees
;
199 static size_t tree_size
;
200 static TCGRegSet tcg_target_available_regs
[TCG_TYPE_COUNT
];
201 static TCGRegSet tcg_target_call_clobber_regs
;
203 #if TCG_TARGET_INSN_UNIT_SIZE == 1
204 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
209 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
216 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
217 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
219 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
222 tcg_insn_unit
*p
= s
->code_ptr
;
223 memcpy(p
, &v
, sizeof(v
));
224 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
228 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
231 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
234 memcpy(p
, &v
, sizeof(v
));
239 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
240 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
242 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
245 tcg_insn_unit
*p
= s
->code_ptr
;
246 memcpy(p
, &v
, sizeof(v
));
247 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
251 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
254 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
257 memcpy(p
, &v
, sizeof(v
));
262 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
263 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
265 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
268 tcg_insn_unit
*p
= s
->code_ptr
;
269 memcpy(p
, &v
, sizeof(v
));
270 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
274 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
277 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
280 memcpy(p
, &v
, sizeof(v
));
285 /* label relocation processing */
287 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
288 TCGLabel
*l
, intptr_t addend
)
290 TCGRelocation
*r
= tcg_malloc(sizeof(TCGRelocation
));
295 QSIMPLEQ_INSERT_TAIL(&l
->relocs
, r
, next
);
298 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
, tcg_insn_unit
*ptr
)
300 tcg_debug_assert(!l
->has_value
);
302 l
->u
.value_ptr
= ptr
;
305 TCGLabel
*gen_new_label(void)
307 TCGContext
*s
= tcg_ctx
;
308 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
310 memset(l
, 0, sizeof(TCGLabel
));
311 l
->id
= s
->nb_labels
++;
312 QSIMPLEQ_INIT(&l
->relocs
);
314 QSIMPLEQ_INSERT_TAIL(&s
->labels
, l
, next
);
319 static bool tcg_resolve_relocs(TCGContext
*s
)
323 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
325 uintptr_t value
= l
->u
.value
;
327 QSIMPLEQ_FOREACH(r
, &l
->relocs
, next
) {
328 if (!patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
)) {
336 static void set_jmp_reset_offset(TCGContext
*s
, int which
)
338 size_t off
= tcg_current_code_size(s
);
339 s
->tb_jmp_reset_offset
[which
] = off
;
340 /* Make sure that we didn't overflow the stored offset. */
341 assert(s
->tb_jmp_reset_offset
[which
] == off
);
344 #include "tcg-target.inc.c"
346 /* compare a pointer @ptr and a tb_tc @s */
347 static int ptr_cmp_tb_tc(const void *ptr
, const struct tb_tc
*s
)
349 if (ptr
>= s
->ptr
+ s
->size
) {
351 } else if (ptr
< s
->ptr
) {
357 static gint
tb_tc_cmp(gconstpointer ap
, gconstpointer bp
)
359 const struct tb_tc
*a
= ap
;
360 const struct tb_tc
*b
= bp
;
363 * When both sizes are set, we know this isn't a lookup.
364 * This is the most likely case: every TB must be inserted; lookups
365 * are a lot less frequent.
367 if (likely(a
->size
&& b
->size
)) {
368 if (a
->ptr
> b
->ptr
) {
370 } else if (a
->ptr
< b
->ptr
) {
373 /* a->ptr == b->ptr should happen only on deletions */
374 g_assert(a
->size
== b
->size
);
378 * All lookups have either .size field set to 0.
379 * From the glib sources we see that @ap is always the lookup key. However
380 * the docs provide no guarantee, so we just mark this case as likely.
382 if (likely(a
->size
== 0)) {
383 return ptr_cmp_tb_tc(a
->ptr
, b
);
385 return ptr_cmp_tb_tc(b
->ptr
, a
);
388 static void tcg_region_trees_init(void)
392 tree_size
= ROUND_UP(sizeof(struct tcg_region_tree
), qemu_dcache_linesize
);
393 region_trees
= qemu_memalign(qemu_dcache_linesize
, region
.n
* tree_size
);
394 for (i
= 0; i
< region
.n
; i
++) {
395 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
397 qemu_mutex_init(&rt
->lock
);
398 rt
->tree
= g_tree_new(tb_tc_cmp
);
402 static struct tcg_region_tree
*tc_ptr_to_region_tree(void *p
)
406 if (p
< region
.start_aligned
) {
409 ptrdiff_t offset
= p
- region
.start_aligned
;
411 if (offset
> region
.stride
* (region
.n
- 1)) {
412 region_idx
= region
.n
- 1;
414 region_idx
= offset
/ region
.stride
;
417 return region_trees
+ region_idx
* tree_size
;
420 void tcg_tb_insert(TranslationBlock
*tb
)
422 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
424 qemu_mutex_lock(&rt
->lock
);
425 g_tree_insert(rt
->tree
, &tb
->tc
, tb
);
426 qemu_mutex_unlock(&rt
->lock
);
429 void tcg_tb_remove(TranslationBlock
*tb
)
431 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
433 qemu_mutex_lock(&rt
->lock
);
434 g_tree_remove(rt
->tree
, &tb
->tc
);
435 qemu_mutex_unlock(&rt
->lock
);
439 * Find the TB 'tb' such that
440 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size
441 * Return NULL if not found.
443 TranslationBlock
*tcg_tb_lookup(uintptr_t tc_ptr
)
445 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree((void *)tc_ptr
);
446 TranslationBlock
*tb
;
447 struct tb_tc s
= { .ptr
= (void *)tc_ptr
};
449 qemu_mutex_lock(&rt
->lock
);
450 tb
= g_tree_lookup(rt
->tree
, &s
);
451 qemu_mutex_unlock(&rt
->lock
);
455 static void tcg_region_tree_lock_all(void)
459 for (i
= 0; i
< region
.n
; i
++) {
460 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
462 qemu_mutex_lock(&rt
->lock
);
466 static void tcg_region_tree_unlock_all(void)
470 for (i
= 0; i
< region
.n
; i
++) {
471 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
473 qemu_mutex_unlock(&rt
->lock
);
477 void tcg_tb_foreach(GTraverseFunc func
, gpointer user_data
)
481 tcg_region_tree_lock_all();
482 for (i
= 0; i
< region
.n
; i
++) {
483 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
485 g_tree_foreach(rt
->tree
, func
, user_data
);
487 tcg_region_tree_unlock_all();
490 size_t tcg_nb_tbs(void)
495 tcg_region_tree_lock_all();
496 for (i
= 0; i
< region
.n
; i
++) {
497 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
499 nb_tbs
+= g_tree_nnodes(rt
->tree
);
501 tcg_region_tree_unlock_all();
505 static void tcg_region_tree_reset_all(void)
509 tcg_region_tree_lock_all();
510 for (i
= 0; i
< region
.n
; i
++) {
511 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
513 /* Increment the refcount first so that destroy acts as a reset */
514 g_tree_ref(rt
->tree
);
515 g_tree_destroy(rt
->tree
);
517 tcg_region_tree_unlock_all();
520 static void tcg_region_bounds(size_t curr_region
, void **pstart
, void **pend
)
524 start
= region
.start_aligned
+ curr_region
* region
.stride
;
525 end
= start
+ region
.size
;
527 if (curr_region
== 0) {
528 start
= region
.start
;
530 if (curr_region
== region
.n
- 1) {
538 static void tcg_region_assign(TCGContext
*s
, size_t curr_region
)
542 tcg_region_bounds(curr_region
, &start
, &end
);
544 s
->code_gen_buffer
= start
;
545 s
->code_gen_ptr
= start
;
546 s
->code_gen_buffer_size
= end
- start
;
547 s
->code_gen_highwater
= end
- TCG_HIGHWATER
;
550 static bool tcg_region_alloc__locked(TCGContext
*s
)
552 if (region
.current
== region
.n
) {
555 tcg_region_assign(s
, region
.current
);
561 * Request a new region once the one in use has filled up.
562 * Returns true on error.
564 static bool tcg_region_alloc(TCGContext
*s
)
567 /* read the region size now; alloc__locked will overwrite it on success */
568 size_t size_full
= s
->code_gen_buffer_size
;
570 qemu_mutex_lock(®ion
.lock
);
571 err
= tcg_region_alloc__locked(s
);
573 region
.agg_size_full
+= size_full
- TCG_HIGHWATER
;
575 qemu_mutex_unlock(®ion
.lock
);
580 * Perform a context's first region allocation.
581 * This function does _not_ increment region.agg_size_full.
583 static inline bool tcg_region_initial_alloc__locked(TCGContext
*s
)
585 return tcg_region_alloc__locked(s
);
588 /* Call from a safe-work context */
589 void tcg_region_reset_all(void)
591 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
594 qemu_mutex_lock(®ion
.lock
);
596 region
.agg_size_full
= 0;
598 for (i
= 0; i
< n_ctxs
; i
++) {
599 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
600 bool err
= tcg_region_initial_alloc__locked(s
);
604 qemu_mutex_unlock(®ion
.lock
);
606 tcg_region_tree_reset_all();
609 #ifdef CONFIG_USER_ONLY
610 static size_t tcg_n_regions(void)
616 * It is likely that some vCPUs will translate more code than others, so we
617 * first try to set more regions than max_cpus, with those regions being of
618 * reasonable size. If that's not possible we make do by evenly dividing
619 * the code_gen_buffer among the vCPUs.
621 static size_t tcg_n_regions(void)
625 /* Use a single region if all we have is one vCPU thread */
626 #if !defined(CONFIG_USER_ONLY)
627 MachineState
*ms
= MACHINE(qdev_get_machine());
628 unsigned int max_cpus
= ms
->smp
.max_cpus
;
630 if (max_cpus
== 1 || !qemu_tcg_mttcg_enabled()) {
634 /* Try to have more regions than max_cpus, with each region being >= 2 MB */
635 for (i
= 8; i
> 0; i
--) {
636 size_t regions_per_thread
= i
;
639 region_size
= tcg_init_ctx
.code_gen_buffer_size
;
640 region_size
/= max_cpus
* regions_per_thread
;
642 if (region_size
>= 2 * 1024u * 1024) {
643 return max_cpus
* regions_per_thread
;
646 /* If we can't, then just allocate one region per vCPU thread */
652 * Initializes region partitioning.
654 * Called at init time from the parent thread (i.e. the one calling
655 * tcg_context_init), after the target's TCG globals have been set.
657 * Region partitioning works by splitting code_gen_buffer into separate regions,
658 * and then assigning regions to TCG threads so that the threads can translate
659 * code in parallel without synchronization.
661 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at
662 * least max_cpus regions in MTTCG. In !MTTCG we use a single region.
663 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...])
664 * must have been parsed before calling this function, since it calls
665 * qemu_tcg_mttcg_enabled().
667 * In user-mode we use a single region. Having multiple regions in user-mode
668 * is not supported, because the number of vCPU threads (recall that each thread
669 * spawned by the guest corresponds to a vCPU thread) is only bounded by the
670 * OS, and usually this number is huge (tens of thousands is not uncommon).
671 * Thus, given this large bound on the number of vCPU threads and the fact
672 * that code_gen_buffer is allocated at compile-time, we cannot guarantee
673 * that the availability of at least one region per vCPU thread.
675 * However, this user-mode limitation is unlikely to be a significant problem
676 * in practice. Multi-threaded guests share most if not all of their translated
677 * code, which makes parallel code generation less appealing than in softmmu.
679 void tcg_region_init(void)
681 void *buf
= tcg_init_ctx
.code_gen_buffer
;
683 size_t size
= tcg_init_ctx
.code_gen_buffer_size
;
684 size_t page_size
= qemu_real_host_page_size
;
689 n_regions
= tcg_n_regions();
691 /* The first region will be 'aligned - buf' bytes larger than the others */
692 aligned
= QEMU_ALIGN_PTR_UP(buf
, page_size
);
693 g_assert(aligned
< tcg_init_ctx
.code_gen_buffer
+ size
);
695 * Make region_size a multiple of page_size, using aligned as the start.
696 * As a result of this we might end up with a few extra pages at the end of
697 * the buffer; we will assign those to the last region.
699 region_size
= (size
- (aligned
- buf
)) / n_regions
;
700 region_size
= QEMU_ALIGN_DOWN(region_size
, page_size
);
702 /* A region must have at least 2 pages; one code, one guard */
703 g_assert(region_size
>= 2 * page_size
);
705 /* init the region struct */
706 qemu_mutex_init(®ion
.lock
);
707 region
.n
= n_regions
;
708 region
.size
= region_size
- page_size
;
709 region
.stride
= region_size
;
711 region
.start_aligned
= aligned
;
712 /* page-align the end, since its last page will be a guard page */
713 region
.end
= QEMU_ALIGN_PTR_DOWN(buf
+ size
, page_size
);
714 /* account for that last guard page */
715 region
.end
-= page_size
;
717 /* set guard pages */
718 for (i
= 0; i
< region
.n
; i
++) {
722 tcg_region_bounds(i
, &start
, &end
);
723 rc
= qemu_mprotect_none(end
, page_size
);
727 tcg_region_trees_init();
729 /* In user-mode we support only one ctx, so do the initial allocation now */
730 #ifdef CONFIG_USER_ONLY
732 bool err
= tcg_region_initial_alloc__locked(tcg_ctx
);
739 static void alloc_tcg_plugin_context(TCGContext
*s
)
742 s
->plugin_tb
= g_new0(struct qemu_plugin_tb
, 1);
743 s
->plugin_tb
->insns
=
744 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn
);
749 * All TCG threads except the parent (i.e. the one that called tcg_context_init
750 * and registered the target's TCG globals) must register with this function
751 * before initiating translation.
753 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
754 * of tcg_region_init() for the reasoning behind this.
756 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
757 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
758 * is not used anymore for translation once this function is called.
760 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
761 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
763 #ifdef CONFIG_USER_ONLY
764 void tcg_register_thread(void)
766 tcg_ctx
= &tcg_init_ctx
;
769 void tcg_register_thread(void)
771 MachineState
*ms
= MACHINE(qdev_get_machine());
772 TCGContext
*s
= g_malloc(sizeof(*s
));
778 /* Relink mem_base. */
779 for (i
= 0, n
= tcg_init_ctx
.nb_globals
; i
< n
; ++i
) {
780 if (tcg_init_ctx
.temps
[i
].mem_base
) {
781 ptrdiff_t b
= tcg_init_ctx
.temps
[i
].mem_base
- tcg_init_ctx
.temps
;
782 tcg_debug_assert(b
>= 0 && b
< n
);
783 s
->temps
[i
].mem_base
= &s
->temps
[b
];
787 /* Claim an entry in tcg_ctxs */
788 n
= atomic_fetch_inc(&n_tcg_ctxs
);
789 g_assert(n
< ms
->smp
.max_cpus
);
790 atomic_set(&tcg_ctxs
[n
], s
);
793 alloc_tcg_plugin_context(s
);
797 qemu_mutex_lock(®ion
.lock
);
798 err
= tcg_region_initial_alloc__locked(tcg_ctx
);
800 qemu_mutex_unlock(®ion
.lock
);
802 #endif /* !CONFIG_USER_ONLY */
805 * Returns the size (in bytes) of all translated code (i.e. from all regions)
806 * currently in the cache.
807 * See also: tcg_code_capacity()
808 * Do not confuse with tcg_current_code_size(); that one applies to a single
811 size_t tcg_code_size(void)
813 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
817 qemu_mutex_lock(®ion
.lock
);
818 total
= region
.agg_size_full
;
819 for (i
= 0; i
< n_ctxs
; i
++) {
820 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
823 size
= atomic_read(&s
->code_gen_ptr
) - s
->code_gen_buffer
;
824 g_assert(size
<= s
->code_gen_buffer_size
);
827 qemu_mutex_unlock(®ion
.lock
);
832 * Returns the code capacity (in bytes) of the entire cache, i.e. including all
834 * See also: tcg_code_size()
836 size_t tcg_code_capacity(void)
838 size_t guard_size
, capacity
;
840 /* no need for synchronization; these variables are set at init time */
841 guard_size
= region
.stride
- region
.size
;
842 capacity
= region
.end
+ guard_size
- region
.start
;
843 capacity
-= region
.n
* (guard_size
+ TCG_HIGHWATER
);
847 size_t tcg_tb_phys_invalidate_count(void)
849 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
853 for (i
= 0; i
< n_ctxs
; i
++) {
854 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
856 total
+= atomic_read(&s
->tb_phys_invalidate_count
);
861 /* pool based memory allocation */
862 void *tcg_malloc_internal(TCGContext
*s
, int size
)
867 if (size
> TCG_POOL_CHUNK_SIZE
) {
868 /* big malloc: insert a new pool (XXX: could optimize) */
869 p
= g_malloc(sizeof(TCGPool
) + size
);
871 p
->next
= s
->pool_first_large
;
872 s
->pool_first_large
= p
;
883 pool_size
= TCG_POOL_CHUNK_SIZE
;
884 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
888 s
->pool_current
->next
= p
;
897 s
->pool_cur
= p
->data
+ size
;
898 s
->pool_end
= p
->data
+ p
->size
;
902 void tcg_pool_reset(TCGContext
*s
)
905 for (p
= s
->pool_first_large
; p
; p
= t
) {
909 s
->pool_first_large
= NULL
;
910 s
->pool_cur
= s
->pool_end
= NULL
;
911 s
->pool_current
= NULL
;
914 typedef struct TCGHelperInfo
{
921 #include "exec/helper-proto.h"
923 static const TCGHelperInfo all_helpers
[] = {
924 #include "exec/helper-tcg.h"
926 static GHashTable
*helper_table
;
928 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
929 static void process_op_defs(TCGContext
*s
);
930 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
931 TCGReg reg
, const char *name
);
933 void tcg_context_init(TCGContext
*s
)
935 int op
, total_args
, n
, i
;
937 TCGArgConstraint
*args_ct
;
941 memset(s
, 0, sizeof(*s
));
944 /* Count total number of arguments and allocate the corresponding
947 for(op
= 0; op
< NB_OPS
; op
++) {
948 def
= &tcg_op_defs
[op
];
949 n
= def
->nb_iargs
+ def
->nb_oargs
;
953 args_ct
= g_malloc(sizeof(TCGArgConstraint
) * total_args
);
954 sorted_args
= g_malloc(sizeof(int) * total_args
);
956 for(op
= 0; op
< NB_OPS
; op
++) {
957 def
= &tcg_op_defs
[op
];
958 def
->args_ct
= args_ct
;
959 def
->sorted_args
= sorted_args
;
960 n
= def
->nb_iargs
+ def
->nb_oargs
;
965 /* Register helpers. */
966 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
967 helper_table
= g_hash_table_new(NULL
, NULL
);
969 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
970 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
971 (gpointer
)&all_helpers
[i
]);
977 /* Reverse the order of the saved registers, assuming they're all at
978 the start of tcg_target_reg_alloc_order. */
979 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
980 int r
= tcg_target_reg_alloc_order
[n
];
981 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
985 for (i
= 0; i
< n
; ++i
) {
986 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
988 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
989 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
992 alloc_tcg_plugin_context(s
);
996 * In user-mode we simply share the init context among threads, since we
997 * use a single region. See the documentation tcg_region_init() for the
998 * reasoning behind this.
999 * In softmmu we will have at most max_cpus TCG threads.
1001 #ifdef CONFIG_USER_ONLY
1002 tcg_ctxs
= &tcg_ctx
;
1005 MachineState
*ms
= MACHINE(qdev_get_machine());
1006 unsigned int max_cpus
= ms
->smp
.max_cpus
;
1007 tcg_ctxs
= g_new(TCGContext
*, max_cpus
);
1010 tcg_debug_assert(!tcg_regset_test_reg(s
->reserved_regs
, TCG_AREG0
));
1011 ts
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, TCG_AREG0
, "env");
1012 cpu_env
= temp_tcgv_ptr(ts
);
1016 * Allocate TBs right before their corresponding translated code, making
1017 * sure that TBs and code are on different cache lines.
1019 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
1021 uintptr_t align
= qemu_icache_linesize
;
1022 TranslationBlock
*tb
;
1026 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
1027 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
1029 if (unlikely(next
> s
->code_gen_highwater
)) {
1030 if (tcg_region_alloc(s
)) {
1035 atomic_set(&s
->code_gen_ptr
, next
);
1036 s
->data_gen_ptr
= NULL
;
1040 void tcg_prologue_init(TCGContext
*s
)
1042 size_t prologue_size
, total_size
;
1045 /* Put the prologue at the beginning of code_gen_buffer. */
1046 buf0
= s
->code_gen_buffer
;
1047 total_size
= s
->code_gen_buffer_size
;
1050 s
->data_gen_ptr
= NULL
;
1051 s
->code_gen_prologue
= buf0
;
1053 /* Compute a high-water mark, at which we voluntarily flush the buffer
1054 and start over. The size here is arbitrary, significantly larger
1055 than we expect the code generation for any one opcode to require. */
1056 s
->code_gen_highwater
= s
->code_gen_buffer
+ (total_size
- TCG_HIGHWATER
);
1058 #ifdef TCG_TARGET_NEED_POOL_LABELS
1059 s
->pool_labels
= NULL
;
1062 /* Generate the prologue. */
1063 tcg_target_qemu_prologue(s
);
1065 #ifdef TCG_TARGET_NEED_POOL_LABELS
1066 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1068 int result
= tcg_out_pool_finalize(s
);
1069 tcg_debug_assert(result
== 0);
1074 flush_icache_range((uintptr_t)buf0
, (uintptr_t)buf1
);
1076 /* Deduct the prologue from the buffer. */
1077 prologue_size
= tcg_current_code_size(s
);
1078 s
->code_gen_ptr
= buf1
;
1079 s
->code_gen_buffer
= buf1
;
1081 total_size
-= prologue_size
;
1082 s
->code_gen_buffer_size
= total_size
;
1084 tcg_register_jit(s
->code_gen_buffer
, total_size
);
1087 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
1088 FILE *logfile
= qemu_log_lock();
1089 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size
);
1090 if (s
->data_gen_ptr
) {
1091 size_t code_size
= s
->data_gen_ptr
- buf0
;
1092 size_t data_size
= prologue_size
- code_size
;
1095 log_disas(buf0
, code_size
, NULL
);
1097 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1098 if (sizeof(tcg_target_ulong
) == 8) {
1099 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1100 (uintptr_t)s
->data_gen_ptr
+ i
,
1101 *(uint64_t *)(s
->data_gen_ptr
+ i
));
1103 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1104 (uintptr_t)s
->data_gen_ptr
+ i
,
1105 *(uint32_t *)(s
->data_gen_ptr
+ i
));
1109 log_disas(buf0
, prologue_size
, NULL
);
1113 qemu_log_unlock(logfile
);
1117 /* Assert that goto_ptr is implemented completely. */
1118 if (TCG_TARGET_HAS_goto_ptr
) {
1119 tcg_debug_assert(s
->code_gen_epilogue
!= NULL
);
1123 void tcg_func_start(TCGContext
*s
)
1126 s
->nb_temps
= s
->nb_globals
;
1128 /* No temps have been previously allocated for size or locality. */
1129 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
1133 s
->current_frame_offset
= s
->frame_start
;
1135 #ifdef CONFIG_DEBUG_TCG
1136 s
->goto_tb_issue_mask
= 0;
1139 QTAILQ_INIT(&s
->ops
);
1140 QTAILQ_INIT(&s
->free_ops
);
1141 QSIMPLEQ_INIT(&s
->labels
);
1144 static inline TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
1146 int n
= s
->nb_temps
++;
1147 tcg_debug_assert(n
< TCG_MAX_TEMPS
);
1148 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
1151 static inline TCGTemp
*tcg_global_alloc(TCGContext
*s
)
1155 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
1157 ts
= tcg_temp_alloc(s
);
1158 ts
->temp_global
= 1;
1163 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
1164 TCGReg reg
, const char *name
)
1168 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
1172 ts
= tcg_global_alloc(s
);
1173 ts
->base_type
= type
;
1178 tcg_regset_set_reg(s
->reserved_regs
, reg
);
1183 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
1185 s
->frame_start
= start
;
1186 s
->frame_end
= start
+ size
;
1188 = tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
1191 TCGTemp
*tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
1192 intptr_t offset
, const char *name
)
1194 TCGContext
*s
= tcg_ctx
;
1195 TCGTemp
*base_ts
= tcgv_ptr_temp(base
);
1196 TCGTemp
*ts
= tcg_global_alloc(s
);
1197 int indirect_reg
= 0, bigendian
= 0;
1198 #ifdef HOST_WORDS_BIGENDIAN
1202 if (!base_ts
->fixed_reg
) {
1203 /* We do not support double-indirect registers. */
1204 tcg_debug_assert(!base_ts
->indirect_reg
);
1205 base_ts
->indirect_base
= 1;
1206 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
1211 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1212 TCGTemp
*ts2
= tcg_global_alloc(s
);
1215 ts
->base_type
= TCG_TYPE_I64
;
1216 ts
->type
= TCG_TYPE_I32
;
1217 ts
->indirect_reg
= indirect_reg
;
1218 ts
->mem_allocated
= 1;
1219 ts
->mem_base
= base_ts
;
1220 ts
->mem_offset
= offset
+ bigendian
* 4;
1221 pstrcpy(buf
, sizeof(buf
), name
);
1222 pstrcat(buf
, sizeof(buf
), "_0");
1223 ts
->name
= strdup(buf
);
1225 tcg_debug_assert(ts2
== ts
+ 1);
1226 ts2
->base_type
= TCG_TYPE_I64
;
1227 ts2
->type
= TCG_TYPE_I32
;
1228 ts2
->indirect_reg
= indirect_reg
;
1229 ts2
->mem_allocated
= 1;
1230 ts2
->mem_base
= base_ts
;
1231 ts2
->mem_offset
= offset
+ (1 - bigendian
) * 4;
1232 pstrcpy(buf
, sizeof(buf
), name
);
1233 pstrcat(buf
, sizeof(buf
), "_1");
1234 ts2
->name
= strdup(buf
);
1236 ts
->base_type
= type
;
1238 ts
->indirect_reg
= indirect_reg
;
1239 ts
->mem_allocated
= 1;
1240 ts
->mem_base
= base_ts
;
1241 ts
->mem_offset
= offset
;
1247 TCGTemp
*tcg_temp_new_internal(TCGType type
, bool temp_local
)
1249 TCGContext
*s
= tcg_ctx
;
1253 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
1254 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
1255 if (idx
< TCG_MAX_TEMPS
) {
1256 /* There is already an available temp with the right type. */
1257 clear_bit(idx
, s
->free_temps
[k
].l
);
1259 ts
= &s
->temps
[idx
];
1260 ts
->temp_allocated
= 1;
1261 tcg_debug_assert(ts
->base_type
== type
);
1262 tcg_debug_assert(ts
->temp_local
== temp_local
);
1264 ts
= tcg_temp_alloc(s
);
1265 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1266 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1268 ts
->base_type
= type
;
1269 ts
->type
= TCG_TYPE_I32
;
1270 ts
->temp_allocated
= 1;
1271 ts
->temp_local
= temp_local
;
1273 tcg_debug_assert(ts2
== ts
+ 1);
1274 ts2
->base_type
= TCG_TYPE_I64
;
1275 ts2
->type
= TCG_TYPE_I32
;
1276 ts2
->temp_allocated
= 1;
1277 ts2
->temp_local
= temp_local
;
1279 ts
->base_type
= type
;
1281 ts
->temp_allocated
= 1;
1282 ts
->temp_local
= temp_local
;
1286 #if defined(CONFIG_DEBUG_TCG)
1292 TCGv_vec
tcg_temp_new_vec(TCGType type
)
1296 #ifdef CONFIG_DEBUG_TCG
1299 assert(TCG_TARGET_HAS_v64
);
1302 assert(TCG_TARGET_HAS_v128
);
1305 assert(TCG_TARGET_HAS_v256
);
1308 g_assert_not_reached();
1312 t
= tcg_temp_new_internal(type
, 0);
1313 return temp_tcgv_vec(t
);
1316 /* Create a new temp of the same type as an existing temp. */
1317 TCGv_vec
tcg_temp_new_vec_matching(TCGv_vec match
)
1319 TCGTemp
*t
= tcgv_vec_temp(match
);
1321 tcg_debug_assert(t
->temp_allocated
!= 0);
1323 t
= tcg_temp_new_internal(t
->base_type
, 0);
1324 return temp_tcgv_vec(t
);
1327 void tcg_temp_free_internal(TCGTemp
*ts
)
1329 TCGContext
*s
= tcg_ctx
;
1332 #if defined(CONFIG_DEBUG_TCG)
1334 if (s
->temps_in_use
< 0) {
1335 fprintf(stderr
, "More temporaries freed than allocated!\n");
1339 tcg_debug_assert(ts
->temp_global
== 0);
1340 tcg_debug_assert(ts
->temp_allocated
!= 0);
1341 ts
->temp_allocated
= 0;
1344 k
= ts
->base_type
+ (ts
->temp_local
? TCG_TYPE_COUNT
: 0);
1345 set_bit(idx
, s
->free_temps
[k
].l
);
1348 TCGv_i32
tcg_const_i32(int32_t val
)
1351 t0
= tcg_temp_new_i32();
1352 tcg_gen_movi_i32(t0
, val
);
1356 TCGv_i64
tcg_const_i64(int64_t val
)
1359 t0
= tcg_temp_new_i64();
1360 tcg_gen_movi_i64(t0
, val
);
1364 TCGv_i32
tcg_const_local_i32(int32_t val
)
1367 t0
= tcg_temp_local_new_i32();
1368 tcg_gen_movi_i32(t0
, val
);
1372 TCGv_i64
tcg_const_local_i64(int64_t val
)
1375 t0
= tcg_temp_local_new_i64();
1376 tcg_gen_movi_i64(t0
, val
);
1380 #if defined(CONFIG_DEBUG_TCG)
1381 void tcg_clear_temp_count(void)
1383 TCGContext
*s
= tcg_ctx
;
1384 s
->temps_in_use
= 0;
1387 int tcg_check_temp_count(void)
1389 TCGContext
*s
= tcg_ctx
;
1390 if (s
->temps_in_use
) {
1391 /* Clear the count so that we don't give another
1392 * warning immediately next time around.
1394 s
->temps_in_use
= 0;
1401 /* Return true if OP may appear in the opcode stream.
1402 Test the runtime variable that controls each opcode. */
1403 bool tcg_op_supported(TCGOpcode op
)
1406 = TCG_TARGET_HAS_v64
| TCG_TARGET_HAS_v128
| TCG_TARGET_HAS_v256
;
1409 case INDEX_op_discard
:
1410 case INDEX_op_set_label
:
1414 case INDEX_op_insn_start
:
1415 case INDEX_op_exit_tb
:
1416 case INDEX_op_goto_tb
:
1417 case INDEX_op_qemu_ld_i32
:
1418 case INDEX_op_qemu_st_i32
:
1419 case INDEX_op_qemu_ld_i64
:
1420 case INDEX_op_qemu_st_i64
:
1423 case INDEX_op_goto_ptr
:
1424 return TCG_TARGET_HAS_goto_ptr
;
1426 case INDEX_op_mov_i32
:
1427 case INDEX_op_movi_i32
:
1428 case INDEX_op_setcond_i32
:
1429 case INDEX_op_brcond_i32
:
1430 case INDEX_op_ld8u_i32
:
1431 case INDEX_op_ld8s_i32
:
1432 case INDEX_op_ld16u_i32
:
1433 case INDEX_op_ld16s_i32
:
1434 case INDEX_op_ld_i32
:
1435 case INDEX_op_st8_i32
:
1436 case INDEX_op_st16_i32
:
1437 case INDEX_op_st_i32
:
1438 case INDEX_op_add_i32
:
1439 case INDEX_op_sub_i32
:
1440 case INDEX_op_mul_i32
:
1441 case INDEX_op_and_i32
:
1442 case INDEX_op_or_i32
:
1443 case INDEX_op_xor_i32
:
1444 case INDEX_op_shl_i32
:
1445 case INDEX_op_shr_i32
:
1446 case INDEX_op_sar_i32
:
1449 case INDEX_op_movcond_i32
:
1450 return TCG_TARGET_HAS_movcond_i32
;
1451 case INDEX_op_div_i32
:
1452 case INDEX_op_divu_i32
:
1453 return TCG_TARGET_HAS_div_i32
;
1454 case INDEX_op_rem_i32
:
1455 case INDEX_op_remu_i32
:
1456 return TCG_TARGET_HAS_rem_i32
;
1457 case INDEX_op_div2_i32
:
1458 case INDEX_op_divu2_i32
:
1459 return TCG_TARGET_HAS_div2_i32
;
1460 case INDEX_op_rotl_i32
:
1461 case INDEX_op_rotr_i32
:
1462 return TCG_TARGET_HAS_rot_i32
;
1463 case INDEX_op_deposit_i32
:
1464 return TCG_TARGET_HAS_deposit_i32
;
1465 case INDEX_op_extract_i32
:
1466 return TCG_TARGET_HAS_extract_i32
;
1467 case INDEX_op_sextract_i32
:
1468 return TCG_TARGET_HAS_sextract_i32
;
1469 case INDEX_op_extract2_i32
:
1470 return TCG_TARGET_HAS_extract2_i32
;
1471 case INDEX_op_add2_i32
:
1472 return TCG_TARGET_HAS_add2_i32
;
1473 case INDEX_op_sub2_i32
:
1474 return TCG_TARGET_HAS_sub2_i32
;
1475 case INDEX_op_mulu2_i32
:
1476 return TCG_TARGET_HAS_mulu2_i32
;
1477 case INDEX_op_muls2_i32
:
1478 return TCG_TARGET_HAS_muls2_i32
;
1479 case INDEX_op_muluh_i32
:
1480 return TCG_TARGET_HAS_muluh_i32
;
1481 case INDEX_op_mulsh_i32
:
1482 return TCG_TARGET_HAS_mulsh_i32
;
1483 case INDEX_op_ext8s_i32
:
1484 return TCG_TARGET_HAS_ext8s_i32
;
1485 case INDEX_op_ext16s_i32
:
1486 return TCG_TARGET_HAS_ext16s_i32
;
1487 case INDEX_op_ext8u_i32
:
1488 return TCG_TARGET_HAS_ext8u_i32
;
1489 case INDEX_op_ext16u_i32
:
1490 return TCG_TARGET_HAS_ext16u_i32
;
1491 case INDEX_op_bswap16_i32
:
1492 return TCG_TARGET_HAS_bswap16_i32
;
1493 case INDEX_op_bswap32_i32
:
1494 return TCG_TARGET_HAS_bswap32_i32
;
1495 case INDEX_op_not_i32
:
1496 return TCG_TARGET_HAS_not_i32
;
1497 case INDEX_op_neg_i32
:
1498 return TCG_TARGET_HAS_neg_i32
;
1499 case INDEX_op_andc_i32
:
1500 return TCG_TARGET_HAS_andc_i32
;
1501 case INDEX_op_orc_i32
:
1502 return TCG_TARGET_HAS_orc_i32
;
1503 case INDEX_op_eqv_i32
:
1504 return TCG_TARGET_HAS_eqv_i32
;
1505 case INDEX_op_nand_i32
:
1506 return TCG_TARGET_HAS_nand_i32
;
1507 case INDEX_op_nor_i32
:
1508 return TCG_TARGET_HAS_nor_i32
;
1509 case INDEX_op_clz_i32
:
1510 return TCG_TARGET_HAS_clz_i32
;
1511 case INDEX_op_ctz_i32
:
1512 return TCG_TARGET_HAS_ctz_i32
;
1513 case INDEX_op_ctpop_i32
:
1514 return TCG_TARGET_HAS_ctpop_i32
;
1516 case INDEX_op_brcond2_i32
:
1517 case INDEX_op_setcond2_i32
:
1518 return TCG_TARGET_REG_BITS
== 32;
1520 case INDEX_op_mov_i64
:
1521 case INDEX_op_movi_i64
:
1522 case INDEX_op_setcond_i64
:
1523 case INDEX_op_brcond_i64
:
1524 case INDEX_op_ld8u_i64
:
1525 case INDEX_op_ld8s_i64
:
1526 case INDEX_op_ld16u_i64
:
1527 case INDEX_op_ld16s_i64
:
1528 case INDEX_op_ld32u_i64
:
1529 case INDEX_op_ld32s_i64
:
1530 case INDEX_op_ld_i64
:
1531 case INDEX_op_st8_i64
:
1532 case INDEX_op_st16_i64
:
1533 case INDEX_op_st32_i64
:
1534 case INDEX_op_st_i64
:
1535 case INDEX_op_add_i64
:
1536 case INDEX_op_sub_i64
:
1537 case INDEX_op_mul_i64
:
1538 case INDEX_op_and_i64
:
1539 case INDEX_op_or_i64
:
1540 case INDEX_op_xor_i64
:
1541 case INDEX_op_shl_i64
:
1542 case INDEX_op_shr_i64
:
1543 case INDEX_op_sar_i64
:
1544 case INDEX_op_ext_i32_i64
:
1545 case INDEX_op_extu_i32_i64
:
1546 return TCG_TARGET_REG_BITS
== 64;
1548 case INDEX_op_movcond_i64
:
1549 return TCG_TARGET_HAS_movcond_i64
;
1550 case INDEX_op_div_i64
:
1551 case INDEX_op_divu_i64
:
1552 return TCG_TARGET_HAS_div_i64
;
1553 case INDEX_op_rem_i64
:
1554 case INDEX_op_remu_i64
:
1555 return TCG_TARGET_HAS_rem_i64
;
1556 case INDEX_op_div2_i64
:
1557 case INDEX_op_divu2_i64
:
1558 return TCG_TARGET_HAS_div2_i64
;
1559 case INDEX_op_rotl_i64
:
1560 case INDEX_op_rotr_i64
:
1561 return TCG_TARGET_HAS_rot_i64
;
1562 case INDEX_op_deposit_i64
:
1563 return TCG_TARGET_HAS_deposit_i64
;
1564 case INDEX_op_extract_i64
:
1565 return TCG_TARGET_HAS_extract_i64
;
1566 case INDEX_op_sextract_i64
:
1567 return TCG_TARGET_HAS_sextract_i64
;
1568 case INDEX_op_extract2_i64
:
1569 return TCG_TARGET_HAS_extract2_i64
;
1570 case INDEX_op_extrl_i64_i32
:
1571 return TCG_TARGET_HAS_extrl_i64_i32
;
1572 case INDEX_op_extrh_i64_i32
:
1573 return TCG_TARGET_HAS_extrh_i64_i32
;
1574 case INDEX_op_ext8s_i64
:
1575 return TCG_TARGET_HAS_ext8s_i64
;
1576 case INDEX_op_ext16s_i64
:
1577 return TCG_TARGET_HAS_ext16s_i64
;
1578 case INDEX_op_ext32s_i64
:
1579 return TCG_TARGET_HAS_ext32s_i64
;
1580 case INDEX_op_ext8u_i64
:
1581 return TCG_TARGET_HAS_ext8u_i64
;
1582 case INDEX_op_ext16u_i64
:
1583 return TCG_TARGET_HAS_ext16u_i64
;
1584 case INDEX_op_ext32u_i64
:
1585 return TCG_TARGET_HAS_ext32u_i64
;
1586 case INDEX_op_bswap16_i64
:
1587 return TCG_TARGET_HAS_bswap16_i64
;
1588 case INDEX_op_bswap32_i64
:
1589 return TCG_TARGET_HAS_bswap32_i64
;
1590 case INDEX_op_bswap64_i64
:
1591 return TCG_TARGET_HAS_bswap64_i64
;
1592 case INDEX_op_not_i64
:
1593 return TCG_TARGET_HAS_not_i64
;
1594 case INDEX_op_neg_i64
:
1595 return TCG_TARGET_HAS_neg_i64
;
1596 case INDEX_op_andc_i64
:
1597 return TCG_TARGET_HAS_andc_i64
;
1598 case INDEX_op_orc_i64
:
1599 return TCG_TARGET_HAS_orc_i64
;
1600 case INDEX_op_eqv_i64
:
1601 return TCG_TARGET_HAS_eqv_i64
;
1602 case INDEX_op_nand_i64
:
1603 return TCG_TARGET_HAS_nand_i64
;
1604 case INDEX_op_nor_i64
:
1605 return TCG_TARGET_HAS_nor_i64
;
1606 case INDEX_op_clz_i64
:
1607 return TCG_TARGET_HAS_clz_i64
;
1608 case INDEX_op_ctz_i64
:
1609 return TCG_TARGET_HAS_ctz_i64
;
1610 case INDEX_op_ctpop_i64
:
1611 return TCG_TARGET_HAS_ctpop_i64
;
1612 case INDEX_op_add2_i64
:
1613 return TCG_TARGET_HAS_add2_i64
;
1614 case INDEX_op_sub2_i64
:
1615 return TCG_TARGET_HAS_sub2_i64
;
1616 case INDEX_op_mulu2_i64
:
1617 return TCG_TARGET_HAS_mulu2_i64
;
1618 case INDEX_op_muls2_i64
:
1619 return TCG_TARGET_HAS_muls2_i64
;
1620 case INDEX_op_muluh_i64
:
1621 return TCG_TARGET_HAS_muluh_i64
;
1622 case INDEX_op_mulsh_i64
:
1623 return TCG_TARGET_HAS_mulsh_i64
;
1625 case INDEX_op_mov_vec
:
1626 case INDEX_op_dup_vec
:
1627 case INDEX_op_dupi_vec
:
1628 case INDEX_op_dupm_vec
:
1629 case INDEX_op_ld_vec
:
1630 case INDEX_op_st_vec
:
1631 case INDEX_op_add_vec
:
1632 case INDEX_op_sub_vec
:
1633 case INDEX_op_and_vec
:
1634 case INDEX_op_or_vec
:
1635 case INDEX_op_xor_vec
:
1636 case INDEX_op_cmp_vec
:
1638 case INDEX_op_dup2_vec
:
1639 return have_vec
&& TCG_TARGET_REG_BITS
== 32;
1640 case INDEX_op_not_vec
:
1641 return have_vec
&& TCG_TARGET_HAS_not_vec
;
1642 case INDEX_op_neg_vec
:
1643 return have_vec
&& TCG_TARGET_HAS_neg_vec
;
1644 case INDEX_op_abs_vec
:
1645 return have_vec
&& TCG_TARGET_HAS_abs_vec
;
1646 case INDEX_op_andc_vec
:
1647 return have_vec
&& TCG_TARGET_HAS_andc_vec
;
1648 case INDEX_op_orc_vec
:
1649 return have_vec
&& TCG_TARGET_HAS_orc_vec
;
1650 case INDEX_op_mul_vec
:
1651 return have_vec
&& TCG_TARGET_HAS_mul_vec
;
1652 case INDEX_op_shli_vec
:
1653 case INDEX_op_shri_vec
:
1654 case INDEX_op_sari_vec
:
1655 return have_vec
&& TCG_TARGET_HAS_shi_vec
;
1656 case INDEX_op_shls_vec
:
1657 case INDEX_op_shrs_vec
:
1658 case INDEX_op_sars_vec
:
1659 return have_vec
&& TCG_TARGET_HAS_shs_vec
;
1660 case INDEX_op_shlv_vec
:
1661 case INDEX_op_shrv_vec
:
1662 case INDEX_op_sarv_vec
:
1663 return have_vec
&& TCG_TARGET_HAS_shv_vec
;
1664 case INDEX_op_rotli_vec
:
1665 return have_vec
&& TCG_TARGET_HAS_roti_vec
;
1666 case INDEX_op_rotlv_vec
:
1667 case INDEX_op_rotrv_vec
:
1668 return have_vec
&& TCG_TARGET_HAS_rotv_vec
;
1669 case INDEX_op_ssadd_vec
:
1670 case INDEX_op_usadd_vec
:
1671 case INDEX_op_sssub_vec
:
1672 case INDEX_op_ussub_vec
:
1673 return have_vec
&& TCG_TARGET_HAS_sat_vec
;
1674 case INDEX_op_smin_vec
:
1675 case INDEX_op_umin_vec
:
1676 case INDEX_op_smax_vec
:
1677 case INDEX_op_umax_vec
:
1678 return have_vec
&& TCG_TARGET_HAS_minmax_vec
;
1679 case INDEX_op_bitsel_vec
:
1680 return have_vec
&& TCG_TARGET_HAS_bitsel_vec
;
1681 case INDEX_op_cmpsel_vec
:
1682 return have_vec
&& TCG_TARGET_HAS_cmpsel_vec
;
1685 tcg_debug_assert(op
> INDEX_op_last_generic
&& op
< NB_OPS
);
1690 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1691 and endian swap. Maybe it would be better to do the alignment
1692 and endian swap in tcg_reg_alloc_call(). */
1693 void tcg_gen_callN(void *func
, TCGTemp
*ret
, int nargs
, TCGTemp
**args
)
1695 int i
, real_args
, nb_rets
, pi
;
1696 unsigned sizemask
, flags
;
1697 TCGHelperInfo
*info
;
1700 info
= g_hash_table_lookup(helper_table
, (gpointer
)func
);
1701 flags
= info
->flags
;
1702 sizemask
= info
->sizemask
;
1704 #ifdef CONFIG_PLUGIN
1705 /* detect non-plugin helpers */
1706 if (tcg_ctx
->plugin_insn
&& unlikely(strncmp(info
->name
, "plugin_", 7))) {
1707 tcg_ctx
->plugin_insn
->calls_helpers
= true;
1711 #if defined(__sparc__) && !defined(__arch64__) \
1712 && !defined(CONFIG_TCG_INTERPRETER)
1713 /* We have 64-bit values in one register, but need to pass as two
1714 separate parameters. Split them. */
1715 int orig_sizemask
= sizemask
;
1716 int orig_nargs
= nargs
;
1717 TCGv_i64 retl
, reth
;
1718 TCGTemp
*split_args
[MAX_OPC_PARAM
];
1722 if (sizemask
!= 0) {
1723 for (i
= real_args
= 0; i
< nargs
; ++i
) {
1724 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1726 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1727 TCGv_i32 h
= tcg_temp_new_i32();
1728 TCGv_i32 l
= tcg_temp_new_i32();
1729 tcg_gen_extr_i64_i32(l
, h
, orig
);
1730 split_args
[real_args
++] = tcgv_i32_temp(h
);
1731 split_args
[real_args
++] = tcgv_i32_temp(l
);
1733 split_args
[real_args
++] = args
[i
];
1740 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1741 for (i
= 0; i
< nargs
; ++i
) {
1742 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1743 int is_signed
= sizemask
& (2 << (i
+1)*2);
1745 TCGv_i64 temp
= tcg_temp_new_i64();
1746 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1748 tcg_gen_ext32s_i64(temp
, orig
);
1750 tcg_gen_ext32u_i64(temp
, orig
);
1752 args
[i
] = tcgv_i64_temp(temp
);
1755 #endif /* TCG_TARGET_EXTEND_ARGS */
1757 op
= tcg_emit_op(INDEX_op_call
);
1761 #if defined(__sparc__) && !defined(__arch64__) \
1762 && !defined(CONFIG_TCG_INTERPRETER)
1763 if (orig_sizemask
& 1) {
1764 /* The 32-bit ABI is going to return the 64-bit value in
1765 the %o0/%o1 register pair. Prepare for this by using
1766 two return temporaries, and reassemble below. */
1767 retl
= tcg_temp_new_i64();
1768 reth
= tcg_temp_new_i64();
1769 op
->args
[pi
++] = tcgv_i64_arg(reth
);
1770 op
->args
[pi
++] = tcgv_i64_arg(retl
);
1773 op
->args
[pi
++] = temp_arg(ret
);
1777 if (TCG_TARGET_REG_BITS
< 64 && (sizemask
& 1)) {
1778 #ifdef HOST_WORDS_BIGENDIAN
1779 op
->args
[pi
++] = temp_arg(ret
+ 1);
1780 op
->args
[pi
++] = temp_arg(ret
);
1782 op
->args
[pi
++] = temp_arg(ret
);
1783 op
->args
[pi
++] = temp_arg(ret
+ 1);
1787 op
->args
[pi
++] = temp_arg(ret
);
1794 TCGOP_CALLO(op
) = nb_rets
;
1797 for (i
= 0; i
< nargs
; i
++) {
1798 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1799 if (TCG_TARGET_REG_BITS
< 64 && is_64bit
) {
1800 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1801 /* some targets want aligned 64 bit args */
1802 if (real_args
& 1) {
1803 op
->args
[pi
++] = TCG_CALL_DUMMY_ARG
;
1807 /* If stack grows up, then we will be placing successive
1808 arguments at lower addresses, which means we need to
1809 reverse the order compared to how we would normally
1810 treat either big or little-endian. For those arguments
1811 that will wind up in registers, this still works for
1812 HPPA (the only current STACK_GROWSUP target) since the
1813 argument registers are *also* allocated in decreasing
1814 order. If another such target is added, this logic may
1815 have to get more complicated to differentiate between
1816 stack arguments and register arguments. */
1817 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1818 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1819 op
->args
[pi
++] = temp_arg(args
[i
]);
1821 op
->args
[pi
++] = temp_arg(args
[i
]);
1822 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1828 op
->args
[pi
++] = temp_arg(args
[i
]);
1831 op
->args
[pi
++] = (uintptr_t)func
;
1832 op
->args
[pi
++] = flags
;
1833 TCGOP_CALLI(op
) = real_args
;
1835 /* Make sure the fields didn't overflow. */
1836 tcg_debug_assert(TCGOP_CALLI(op
) == real_args
);
1837 tcg_debug_assert(pi
<= ARRAY_SIZE(op
->args
));
1839 #if defined(__sparc__) && !defined(__arch64__) \
1840 && !defined(CONFIG_TCG_INTERPRETER)
1841 /* Free all of the parts we allocated above. */
1842 for (i
= real_args
= 0; i
< orig_nargs
; ++i
) {
1843 int is_64bit
= orig_sizemask
& (1 << (i
+1)*2);
1845 tcg_temp_free_internal(args
[real_args
++]);
1846 tcg_temp_free_internal(args
[real_args
++]);
1851 if (orig_sizemask
& 1) {
1852 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1853 Note that describing these as TCGv_i64 eliminates an unnecessary
1854 zero-extension that tcg_gen_concat_i32_i64 would create. */
1855 tcg_gen_concat32_i64(temp_tcgv_i64(ret
), retl
, reth
);
1856 tcg_temp_free_i64(retl
);
1857 tcg_temp_free_i64(reth
);
1859 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1860 for (i
= 0; i
< nargs
; ++i
) {
1861 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1863 tcg_temp_free_internal(args
[i
]);
1866 #endif /* TCG_TARGET_EXTEND_ARGS */
1869 static void tcg_reg_alloc_start(TCGContext
*s
)
1874 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
1876 ts
->val_type
= (ts
->fixed_reg
? TEMP_VAL_REG
: TEMP_VAL_MEM
);
1878 for (n
= s
->nb_temps
; i
< n
; i
++) {
1880 ts
->val_type
= (ts
->temp_local
? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
1881 ts
->mem_allocated
= 0;
1885 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1888 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1891 int idx
= temp_idx(ts
);
1893 if (ts
->temp_global
) {
1894 pstrcpy(buf
, buf_size
, ts
->name
);
1895 } else if (ts
->temp_local
) {
1896 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1898 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1903 static char *tcg_get_arg_str(TCGContext
*s
, char *buf
,
1904 int buf_size
, TCGArg arg
)
1906 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, arg_temp(arg
));
1909 /* Find helper name. */
1910 static inline const char *tcg_find_helper(TCGContext
*s
, uintptr_t val
)
1912 const char *ret
= NULL
;
1914 TCGHelperInfo
*info
= g_hash_table_lookup(helper_table
, (gpointer
)val
);
1922 static const char * const cond_name
[] =
1924 [TCG_COND_NEVER
] = "never",
1925 [TCG_COND_ALWAYS
] = "always",
1926 [TCG_COND_EQ
] = "eq",
1927 [TCG_COND_NE
] = "ne",
1928 [TCG_COND_LT
] = "lt",
1929 [TCG_COND_GE
] = "ge",
1930 [TCG_COND_LE
] = "le",
1931 [TCG_COND_GT
] = "gt",
1932 [TCG_COND_LTU
] = "ltu",
1933 [TCG_COND_GEU
] = "geu",
1934 [TCG_COND_LEU
] = "leu",
1935 [TCG_COND_GTU
] = "gtu"
1938 static const char * const ldst_name
[] =
1954 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1955 #ifdef TARGET_ALIGNED_ONLY
1956 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1957 [MO_ALIGN
>> MO_ASHIFT
] = "",
1959 [MO_UNALN
>> MO_ASHIFT
] = "",
1960 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1962 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1963 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1964 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1965 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1966 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1967 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1970 static inline bool tcg_regset_single(TCGRegSet d
)
1972 return (d
& (d
- 1)) == 0;
1975 static inline TCGReg
tcg_regset_first(TCGRegSet d
)
1977 if (TCG_TARGET_NB_REGS
<= 32) {
1984 static void tcg_dump_ops(TCGContext
*s
, bool have_prefs
)
1989 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
1990 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1991 const TCGOpDef
*def
;
1996 def
= &tcg_op_defs
[c
];
1998 if (c
== INDEX_op_insn_start
) {
2000 col
+= qemu_log("\n ----");
2002 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
2004 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2005 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
2009 col
+= qemu_log(" " TARGET_FMT_lx
, a
);
2011 } else if (c
== INDEX_op_call
) {
2012 /* variable number of arguments */
2013 nb_oargs
= TCGOP_CALLO(op
);
2014 nb_iargs
= TCGOP_CALLI(op
);
2015 nb_cargs
= def
->nb_cargs
;
2017 /* function name, flags, out args */
2018 col
+= qemu_log(" %s %s,$0x%" TCG_PRIlx
",$%d", def
->name
,
2019 tcg_find_helper(s
, op
->args
[nb_oargs
+ nb_iargs
]),
2020 op
->args
[nb_oargs
+ nb_iargs
+ 1], nb_oargs
);
2021 for (i
= 0; i
< nb_oargs
; i
++) {
2022 col
+= qemu_log(",%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2025 for (i
= 0; i
< nb_iargs
; i
++) {
2026 TCGArg arg
= op
->args
[nb_oargs
+ i
];
2027 const char *t
= "<dummy>";
2028 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2029 t
= tcg_get_arg_str(s
, buf
, sizeof(buf
), arg
);
2031 col
+= qemu_log(",%s", t
);
2034 col
+= qemu_log(" %s ", def
->name
);
2036 nb_oargs
= def
->nb_oargs
;
2037 nb_iargs
= def
->nb_iargs
;
2038 nb_cargs
= def
->nb_cargs
;
2040 if (def
->flags
& TCG_OPF_VECTOR
) {
2041 col
+= qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op
),
2042 8 << TCGOP_VECE(op
));
2046 for (i
= 0; i
< nb_oargs
; i
++) {
2048 col
+= qemu_log(",");
2050 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2053 for (i
= 0; i
< nb_iargs
; i
++) {
2055 col
+= qemu_log(",");
2057 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2061 case INDEX_op_brcond_i32
:
2062 case INDEX_op_setcond_i32
:
2063 case INDEX_op_movcond_i32
:
2064 case INDEX_op_brcond2_i32
:
2065 case INDEX_op_setcond2_i32
:
2066 case INDEX_op_brcond_i64
:
2067 case INDEX_op_setcond_i64
:
2068 case INDEX_op_movcond_i64
:
2069 case INDEX_op_cmp_vec
:
2070 case INDEX_op_cmpsel_vec
:
2071 if (op
->args
[k
] < ARRAY_SIZE(cond_name
)
2072 && cond_name
[op
->args
[k
]]) {
2073 col
+= qemu_log(",%s", cond_name
[op
->args
[k
++]]);
2075 col
+= qemu_log(",$0x%" TCG_PRIlx
, op
->args
[k
++]);
2079 case INDEX_op_qemu_ld_i32
:
2080 case INDEX_op_qemu_st_i32
:
2081 case INDEX_op_qemu_ld_i64
:
2082 case INDEX_op_qemu_st_i64
:
2084 TCGMemOpIdx oi
= op
->args
[k
++];
2085 MemOp op
= get_memop(oi
);
2086 unsigned ix
= get_mmuidx(oi
);
2088 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
2089 col
+= qemu_log(",$0x%x,%u", op
, ix
);
2091 const char *s_al
, *s_op
;
2092 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
2093 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
2094 col
+= qemu_log(",%s%s,%u", s_al
, s_op
, ix
);
2104 case INDEX_op_set_label
:
2106 case INDEX_op_brcond_i32
:
2107 case INDEX_op_brcond_i64
:
2108 case INDEX_op_brcond2_i32
:
2109 col
+= qemu_log("%s$L%d", k
? "," : "",
2110 arg_label(op
->args
[k
])->id
);
2116 for (; i
< nb_cargs
; i
++, k
++) {
2117 col
+= qemu_log("%s$0x%" TCG_PRIlx
, k
? "," : "", op
->args
[k
]);
2121 if (have_prefs
|| op
->life
) {
2123 QemuLogFile
*logfile
;
2126 logfile
= atomic_rcu_read(&qemu_logfile
);
2128 for (; col
< 40; ++col
) {
2129 putc(' ', logfile
->fd
);
2136 unsigned life
= op
->life
;
2138 if (life
& (SYNC_ARG
* 3)) {
2140 for (i
= 0; i
< 2; ++i
) {
2141 if (life
& (SYNC_ARG
<< i
)) {
2149 for (i
= 0; life
; ++i
, life
>>= 1) {
2158 for (i
= 0; i
< nb_oargs
; ++i
) {
2159 TCGRegSet set
= op
->output_pref
[i
];
2168 } else if (set
== MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS
)) {
2170 #ifdef CONFIG_DEBUG_TCG
2171 } else if (tcg_regset_single(set
)) {
2172 TCGReg reg
= tcg_regset_first(set
);
2173 qemu_log("%s", tcg_target_reg_names
[reg
]);
2175 } else if (TCG_TARGET_NB_REGS
<= 32) {
2176 qemu_log("%#x", (uint32_t)set
);
2178 qemu_log("%#" PRIx64
, (uint64_t)set
);
2187 /* we give more priority to constraints with less registers */
2188 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
2190 const TCGArgConstraint
*arg_ct
;
2193 arg_ct
= &def
->args_ct
[k
];
2194 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
2195 /* an alias is equivalent to a single register */
2198 if (!(arg_ct
->ct
& TCG_CT_REG
))
2201 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2202 if (tcg_regset_test_reg(arg_ct
->u
.regs
, i
))
2206 return TCG_TARGET_NB_REGS
- n
+ 1;
2209 /* sort from highest priority to lowest */
2210 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
2212 int i
, j
, p1
, p2
, tmp
;
2214 for(i
= 0; i
< n
; i
++)
2215 def
->sorted_args
[start
+ i
] = start
+ i
;
2218 for(i
= 0; i
< n
- 1; i
++) {
2219 for(j
= i
+ 1; j
< n
; j
++) {
2220 p1
= get_constraint_priority(def
, def
->sorted_args
[start
+ i
]);
2221 p2
= get_constraint_priority(def
, def
->sorted_args
[start
+ j
]);
2223 tmp
= def
->sorted_args
[start
+ i
];
2224 def
->sorted_args
[start
+ i
] = def
->sorted_args
[start
+ j
];
2225 def
->sorted_args
[start
+ j
] = tmp
;
2231 static void process_op_defs(TCGContext
*s
)
2235 for (op
= 0; op
< NB_OPS
; op
++) {
2236 TCGOpDef
*def
= &tcg_op_defs
[op
];
2237 const TCGTargetOpDef
*tdefs
;
2241 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2245 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
2250 tdefs
= tcg_target_op_def(op
);
2251 /* Missing TCGTargetOpDef entry. */
2252 tcg_debug_assert(tdefs
!= NULL
);
2254 type
= (def
->flags
& TCG_OPF_64BIT
? TCG_TYPE_I64
: TCG_TYPE_I32
);
2255 for (i
= 0; i
< nb_args
; i
++) {
2256 const char *ct_str
= tdefs
->args_ct_str
[i
];
2257 /* Incomplete TCGTargetOpDef entry. */
2258 tcg_debug_assert(ct_str
!= NULL
);
2260 def
->args_ct
[i
].u
.regs
= 0;
2261 def
->args_ct
[i
].ct
= 0;
2262 while (*ct_str
!= '\0') {
2266 int oarg
= *ct_str
- '0';
2267 tcg_debug_assert(ct_str
== tdefs
->args_ct_str
[i
]);
2268 tcg_debug_assert(oarg
< def
->nb_oargs
);
2269 tcg_debug_assert(def
->args_ct
[oarg
].ct
& TCG_CT_REG
);
2270 /* TCG_CT_ALIAS is for the output arguments.
2271 The input is tagged with TCG_CT_IALIAS. */
2272 def
->args_ct
[i
] = def
->args_ct
[oarg
];
2273 def
->args_ct
[oarg
].ct
|= TCG_CT_ALIAS
;
2274 def
->args_ct
[oarg
].alias_index
= i
;
2275 def
->args_ct
[i
].ct
|= TCG_CT_IALIAS
;
2276 def
->args_ct
[i
].alias_index
= oarg
;
2281 def
->args_ct
[i
].ct
|= TCG_CT_NEWREG
;
2285 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
2289 ct_str
= target_parse_constraint(&def
->args_ct
[i
],
2291 /* Typo in TCGTargetOpDef constraint. */
2292 tcg_debug_assert(ct_str
!= NULL
);
2297 /* TCGTargetOpDef entry with too much information? */
2298 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
2300 /* sort the constraints (XXX: this is just an heuristic) */
2301 sort_constraints(def
, 0, def
->nb_oargs
);
2302 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
2306 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
2312 label
= arg_label(op
->args
[0]);
2315 case INDEX_op_brcond_i32
:
2316 case INDEX_op_brcond_i64
:
2317 label
= arg_label(op
->args
[3]);
2320 case INDEX_op_brcond2_i32
:
2321 label
= arg_label(op
->args
[5]);
2328 QTAILQ_REMOVE(&s
->ops
, op
, link
);
2329 QTAILQ_INSERT_TAIL(&s
->free_ops
, op
, link
);
2332 #ifdef CONFIG_PROFILER
2333 atomic_set(&s
->prof
.del_op_count
, s
->prof
.del_op_count
+ 1);
2337 static TCGOp
*tcg_op_alloc(TCGOpcode opc
)
2339 TCGContext
*s
= tcg_ctx
;
2342 if (likely(QTAILQ_EMPTY(&s
->free_ops
))) {
2343 op
= tcg_malloc(sizeof(TCGOp
));
2345 op
= QTAILQ_FIRST(&s
->free_ops
);
2346 QTAILQ_REMOVE(&s
->free_ops
, op
, link
);
2348 memset(op
, 0, offsetof(TCGOp
, link
));
2355 TCGOp
*tcg_emit_op(TCGOpcode opc
)
2357 TCGOp
*op
= tcg_op_alloc(opc
);
2358 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
2362 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
, TCGOpcode opc
)
2364 TCGOp
*new_op
= tcg_op_alloc(opc
);
2365 QTAILQ_INSERT_BEFORE(old_op
, new_op
, link
);
2369 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
, TCGOpcode opc
)
2371 TCGOp
*new_op
= tcg_op_alloc(opc
);
2372 QTAILQ_INSERT_AFTER(&s
->ops
, old_op
, new_op
, link
);
2376 /* Reachable analysis : remove unreachable code. */
2377 static void reachable_code_pass(TCGContext
*s
)
2379 TCGOp
*op
, *op_next
;
2382 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2388 case INDEX_op_set_label
:
2389 label
= arg_label(op
->args
[0]);
2390 if (label
->refs
== 0) {
2392 * While there is an occasional backward branch, virtually
2393 * all branches generated by the translators are forward.
2394 * Which means that generally we will have already removed
2395 * all references to the label that will be, and there is
2396 * little to be gained by iterating.
2400 /* Once we see a label, insns become live again. */
2405 * Optimization can fold conditional branches to unconditional.
2406 * If we find a label with one reference which is preceded by
2407 * an unconditional branch to it, remove both. This needed to
2408 * wait until the dead code in between them was removed.
2410 if (label
->refs
== 1) {
2411 TCGOp
*op_prev
= QTAILQ_PREV(op
, link
);
2412 if (op_prev
->opc
== INDEX_op_br
&&
2413 label
== arg_label(op_prev
->args
[0])) {
2414 tcg_op_remove(s
, op_prev
);
2422 case INDEX_op_exit_tb
:
2423 case INDEX_op_goto_ptr
:
2424 /* Unconditional branches; everything following is dead. */
2429 /* Notice noreturn helper calls, raising exceptions. */
2430 call_flags
= op
->args
[TCGOP_CALLO(op
) + TCGOP_CALLI(op
) + 1];
2431 if (call_flags
& TCG_CALL_NO_RETURN
) {
2436 case INDEX_op_insn_start
:
2437 /* Never remove -- we need to keep these for unwind. */
2446 tcg_op_remove(s
, op
);
2454 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2455 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2457 /* For liveness_pass_1, the register preferences for a given temp. */
2458 static inline TCGRegSet
*la_temp_pref(TCGTemp
*ts
)
2460 return ts
->state_ptr
;
2463 /* For liveness_pass_1, reset the preferences for a given temp to the
2464 * maximal regset for its type.
2466 static inline void la_reset_pref(TCGTemp
*ts
)
2469 = (ts
->state
== TS_DEAD
? 0 : tcg_target_available_regs
[ts
->type
]);
2472 /* liveness analysis: end of function: all temps are dead, and globals
2473 should be in memory. */
2474 static void la_func_end(TCGContext
*s
, int ng
, int nt
)
2478 for (i
= 0; i
< ng
; ++i
) {
2479 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2480 la_reset_pref(&s
->temps
[i
]);
2482 for (i
= ng
; i
< nt
; ++i
) {
2483 s
->temps
[i
].state
= TS_DEAD
;
2484 la_reset_pref(&s
->temps
[i
]);
2488 /* liveness analysis: end of basic block: all temps are dead, globals
2489 and local temps should be in memory. */
2490 static void la_bb_end(TCGContext
*s
, int ng
, int nt
)
2494 for (i
= 0; i
< ng
; ++i
) {
2495 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2496 la_reset_pref(&s
->temps
[i
]);
2498 for (i
= ng
; i
< nt
; ++i
) {
2499 s
->temps
[i
].state
= (s
->temps
[i
].temp_local
2502 la_reset_pref(&s
->temps
[i
]);
2506 /* liveness analysis: sync globals back to memory. */
2507 static void la_global_sync(TCGContext
*s
, int ng
)
2511 for (i
= 0; i
< ng
; ++i
) {
2512 int state
= s
->temps
[i
].state
;
2513 s
->temps
[i
].state
= state
| TS_MEM
;
2514 if (state
== TS_DEAD
) {
2515 /* If the global was previously dead, reset prefs. */
2516 la_reset_pref(&s
->temps
[i
]);
2521 /* liveness analysis: sync globals back to memory and kill. */
2522 static void la_global_kill(TCGContext
*s
, int ng
)
2526 for (i
= 0; i
< ng
; i
++) {
2527 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2528 la_reset_pref(&s
->temps
[i
]);
2532 /* liveness analysis: note live globals crossing calls. */
2533 static void la_cross_call(TCGContext
*s
, int nt
)
2535 TCGRegSet mask
= ~tcg_target_call_clobber_regs
;
2538 for (i
= 0; i
< nt
; i
++) {
2539 TCGTemp
*ts
= &s
->temps
[i
];
2540 if (!(ts
->state
& TS_DEAD
)) {
2541 TCGRegSet
*pset
= la_temp_pref(ts
);
2542 TCGRegSet set
= *pset
;
2545 /* If the combination is not possible, restart. */
2547 set
= tcg_target_available_regs
[ts
->type
] & mask
;
2554 /* Liveness analysis : update the opc_arg_life array to tell if a
2555 given input arguments is dead. Instructions updating dead
2556 temporaries are removed. */
2557 static void liveness_pass_1(TCGContext
*s
)
2559 int nb_globals
= s
->nb_globals
;
2560 int nb_temps
= s
->nb_temps
;
2561 TCGOp
*op
, *op_prev
;
2565 prefs
= tcg_malloc(sizeof(TCGRegSet
) * nb_temps
);
2566 for (i
= 0; i
< nb_temps
; ++i
) {
2567 s
->temps
[i
].state_ptr
= prefs
+ i
;
2570 /* ??? Should be redundant with the exit_tb that ends the TB. */
2571 la_func_end(s
, nb_globals
, nb_temps
);
2573 QTAILQ_FOREACH_REVERSE_SAFE(op
, &s
->ops
, link
, op_prev
) {
2574 int nb_iargs
, nb_oargs
;
2575 TCGOpcode opc_new
, opc_new2
;
2577 TCGLifeData arg_life
= 0;
2579 TCGOpcode opc
= op
->opc
;
2580 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2588 nb_oargs
= TCGOP_CALLO(op
);
2589 nb_iargs
= TCGOP_CALLI(op
);
2590 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2592 /* pure functions can be removed if their result is unused */
2593 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
2594 for (i
= 0; i
< nb_oargs
; i
++) {
2595 ts
= arg_temp(op
->args
[i
]);
2596 if (ts
->state
!= TS_DEAD
) {
2597 goto do_not_remove_call
;
2604 /* Output args are dead. */
2605 for (i
= 0; i
< nb_oargs
; i
++) {
2606 ts
= arg_temp(op
->args
[i
]);
2607 if (ts
->state
& TS_DEAD
) {
2608 arg_life
|= DEAD_ARG
<< i
;
2610 if (ts
->state
& TS_MEM
) {
2611 arg_life
|= SYNC_ARG
<< i
;
2613 ts
->state
= TS_DEAD
;
2616 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2617 op
->output_pref
[i
] = 0;
2620 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
2621 TCG_CALL_NO_READ_GLOBALS
))) {
2622 la_global_kill(s
, nb_globals
);
2623 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
2624 la_global_sync(s
, nb_globals
);
2627 /* Record arguments that die in this helper. */
2628 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2629 ts
= arg_temp(op
->args
[i
]);
2630 if (ts
&& ts
->state
& TS_DEAD
) {
2631 arg_life
|= DEAD_ARG
<< i
;
2635 /* For all live registers, remove call-clobbered prefs. */
2636 la_cross_call(s
, nb_temps
);
2638 nb_call_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
2640 /* Input arguments are live for preceding opcodes. */
2641 for (i
= 0; i
< nb_iargs
; i
++) {
2642 ts
= arg_temp(op
->args
[i
+ nb_oargs
]);
2643 if (ts
&& ts
->state
& TS_DEAD
) {
2644 /* For those arguments that die, and will be allocated
2645 * in registers, clear the register set for that arg,
2646 * to be filled in below. For args that will be on
2647 * the stack, reset to any available reg.
2650 = (i
< nb_call_regs
? 0 :
2651 tcg_target_available_regs
[ts
->type
]);
2652 ts
->state
&= ~TS_DEAD
;
2656 /* For each input argument, add its input register to prefs.
2657 If a temp is used once, this produces a single set bit. */
2658 for (i
= 0; i
< MIN(nb_call_regs
, nb_iargs
); i
++) {
2659 ts
= arg_temp(op
->args
[i
+ nb_oargs
]);
2661 tcg_regset_set_reg(*la_temp_pref(ts
),
2662 tcg_target_call_iarg_regs
[i
]);
2667 case INDEX_op_insn_start
:
2669 case INDEX_op_discard
:
2670 /* mark the temporary as dead */
2671 ts
= arg_temp(op
->args
[0]);
2672 ts
->state
= TS_DEAD
;
2676 case INDEX_op_add2_i32
:
2677 opc_new
= INDEX_op_add_i32
;
2679 case INDEX_op_sub2_i32
:
2680 opc_new
= INDEX_op_sub_i32
;
2682 case INDEX_op_add2_i64
:
2683 opc_new
= INDEX_op_add_i64
;
2685 case INDEX_op_sub2_i64
:
2686 opc_new
= INDEX_op_sub_i64
;
2690 /* Test if the high part of the operation is dead, but not
2691 the low part. The result can be optimized to a simple
2692 add or sub. This happens often for x86_64 guest when the
2693 cpu mode is set to 32 bit. */
2694 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2695 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2698 /* Replace the opcode and adjust the args in place,
2699 leaving 3 unused args at the end. */
2700 op
->opc
= opc
= opc_new
;
2701 op
->args
[1] = op
->args
[2];
2702 op
->args
[2] = op
->args
[4];
2703 /* Fall through and mark the single-word operation live. */
2709 case INDEX_op_mulu2_i32
:
2710 opc_new
= INDEX_op_mul_i32
;
2711 opc_new2
= INDEX_op_muluh_i32
;
2712 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
2714 case INDEX_op_muls2_i32
:
2715 opc_new
= INDEX_op_mul_i32
;
2716 opc_new2
= INDEX_op_mulsh_i32
;
2717 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
2719 case INDEX_op_mulu2_i64
:
2720 opc_new
= INDEX_op_mul_i64
;
2721 opc_new2
= INDEX_op_muluh_i64
;
2722 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
2724 case INDEX_op_muls2_i64
:
2725 opc_new
= INDEX_op_mul_i64
;
2726 opc_new2
= INDEX_op_mulsh_i64
;
2727 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
2732 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2733 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2734 /* Both parts of the operation are dead. */
2737 /* The high part of the operation is dead; generate the low. */
2738 op
->opc
= opc
= opc_new
;
2739 op
->args
[1] = op
->args
[2];
2740 op
->args
[2] = op
->args
[3];
2741 } else if (arg_temp(op
->args
[0])->state
== TS_DEAD
&& have_opc_new2
) {
2742 /* The low part of the operation is dead; generate the high. */
2743 op
->opc
= opc
= opc_new2
;
2744 op
->args
[0] = op
->args
[1];
2745 op
->args
[1] = op
->args
[2];
2746 op
->args
[2] = op
->args
[3];
2750 /* Mark the single-word operation live. */
2755 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2756 nb_iargs
= def
->nb_iargs
;
2757 nb_oargs
= def
->nb_oargs
;
2759 /* Test if the operation can be removed because all
2760 its outputs are dead. We assume that nb_oargs == 0
2761 implies side effects */
2762 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
2763 for (i
= 0; i
< nb_oargs
; i
++) {
2764 if (arg_temp(op
->args
[i
])->state
!= TS_DEAD
) {
2773 tcg_op_remove(s
, op
);
2777 for (i
= 0; i
< nb_oargs
; i
++) {
2778 ts
= arg_temp(op
->args
[i
]);
2780 /* Remember the preference of the uses that followed. */
2781 op
->output_pref
[i
] = *la_temp_pref(ts
);
2783 /* Output args are dead. */
2784 if (ts
->state
& TS_DEAD
) {
2785 arg_life
|= DEAD_ARG
<< i
;
2787 if (ts
->state
& TS_MEM
) {
2788 arg_life
|= SYNC_ARG
<< i
;
2790 ts
->state
= TS_DEAD
;
2794 /* If end of basic block, update. */
2795 if (def
->flags
& TCG_OPF_BB_EXIT
) {
2796 la_func_end(s
, nb_globals
, nb_temps
);
2797 } else if (def
->flags
& TCG_OPF_BB_END
) {
2798 la_bb_end(s
, nb_globals
, nb_temps
);
2799 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2800 la_global_sync(s
, nb_globals
);
2801 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2802 la_cross_call(s
, nb_temps
);
2806 /* Record arguments that die in this opcode. */
2807 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2808 ts
= arg_temp(op
->args
[i
]);
2809 if (ts
->state
& TS_DEAD
) {
2810 arg_life
|= DEAD_ARG
<< i
;
2814 /* Input arguments are live for preceding opcodes. */
2815 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2816 ts
= arg_temp(op
->args
[i
]);
2817 if (ts
->state
& TS_DEAD
) {
2818 /* For operands that were dead, initially allow
2819 all regs for the type. */
2820 *la_temp_pref(ts
) = tcg_target_available_regs
[ts
->type
];
2821 ts
->state
&= ~TS_DEAD
;
2825 /* Incorporate constraints for this operand. */
2827 case INDEX_op_mov_i32
:
2828 case INDEX_op_mov_i64
:
2829 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2830 have proper constraints. That said, special case
2831 moves to propagate preferences backward. */
2832 if (IS_DEAD_ARG(1)) {
2833 *la_temp_pref(arg_temp(op
->args
[0]))
2834 = *la_temp_pref(arg_temp(op
->args
[1]));
2839 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2840 const TCGArgConstraint
*ct
= &def
->args_ct
[i
];
2841 TCGRegSet set
, *pset
;
2843 ts
= arg_temp(op
->args
[i
]);
2844 pset
= la_temp_pref(ts
);
2848 if (ct
->ct
& TCG_CT_IALIAS
) {
2849 set
&= op
->output_pref
[ct
->alias_index
];
2851 /* If the combination is not possible, restart. */
2861 op
->life
= arg_life
;
2865 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2866 static bool liveness_pass_2(TCGContext
*s
)
2868 int nb_globals
= s
->nb_globals
;
2870 bool changes
= false;
2871 TCGOp
*op
, *op_next
;
2873 /* Create a temporary for each indirect global. */
2874 for (i
= 0; i
< nb_globals
; ++i
) {
2875 TCGTemp
*its
= &s
->temps
[i
];
2876 if (its
->indirect_reg
) {
2877 TCGTemp
*dts
= tcg_temp_alloc(s
);
2878 dts
->type
= its
->type
;
2879 dts
->base_type
= its
->base_type
;
2880 its
->state_ptr
= dts
;
2882 its
->state_ptr
= NULL
;
2884 /* All globals begin dead. */
2885 its
->state
= TS_DEAD
;
2887 for (nb_temps
= s
->nb_temps
; i
< nb_temps
; ++i
) {
2888 TCGTemp
*its
= &s
->temps
[i
];
2889 its
->state_ptr
= NULL
;
2890 its
->state
= TS_DEAD
;
2893 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2894 TCGOpcode opc
= op
->opc
;
2895 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2896 TCGLifeData arg_life
= op
->life
;
2897 int nb_iargs
, nb_oargs
, call_flags
;
2898 TCGTemp
*arg_ts
, *dir_ts
;
2900 if (opc
== INDEX_op_call
) {
2901 nb_oargs
= TCGOP_CALLO(op
);
2902 nb_iargs
= TCGOP_CALLI(op
);
2903 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2905 nb_iargs
= def
->nb_iargs
;
2906 nb_oargs
= def
->nb_oargs
;
2908 /* Set flags similar to how calls require. */
2909 if (def
->flags
& TCG_OPF_BB_END
) {
2910 /* Like writing globals: save_globals */
2912 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2913 /* Like reading globals: sync_globals */
2914 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
2916 /* No effect on globals. */
2917 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
2918 TCG_CALL_NO_WRITE_GLOBALS
);
2922 /* Make sure that input arguments are available. */
2923 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2924 arg_ts
= arg_temp(op
->args
[i
]);
2926 dir_ts
= arg_ts
->state_ptr
;
2927 if (dir_ts
&& arg_ts
->state
== TS_DEAD
) {
2928 TCGOpcode lopc
= (arg_ts
->type
== TCG_TYPE_I32
2931 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
);
2933 lop
->args
[0] = temp_arg(dir_ts
);
2934 lop
->args
[1] = temp_arg(arg_ts
->mem_base
);
2935 lop
->args
[2] = arg_ts
->mem_offset
;
2937 /* Loaded, but synced with memory. */
2938 arg_ts
->state
= TS_MEM
;
2943 /* Perform input replacement, and mark inputs that became dead.
2944 No action is required except keeping temp_state up to date
2945 so that we reload when needed. */
2946 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2947 arg_ts
= arg_temp(op
->args
[i
]);
2949 dir_ts
= arg_ts
->state_ptr
;
2951 op
->args
[i
] = temp_arg(dir_ts
);
2953 if (IS_DEAD_ARG(i
)) {
2954 arg_ts
->state
= TS_DEAD
;
2960 /* Liveness analysis should ensure that the following are
2961 all correct, for call sites and basic block end points. */
2962 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
2964 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
2965 for (i
= 0; i
< nb_globals
; ++i
) {
2966 /* Liveness should see that globals are synced back,
2967 that is, either TS_DEAD or TS_MEM. */
2968 arg_ts
= &s
->temps
[i
];
2969 tcg_debug_assert(arg_ts
->state_ptr
== 0
2970 || arg_ts
->state
!= 0);
2973 for (i
= 0; i
< nb_globals
; ++i
) {
2974 /* Liveness should see that globals are saved back,
2975 that is, TS_DEAD, waiting to be reloaded. */
2976 arg_ts
= &s
->temps
[i
];
2977 tcg_debug_assert(arg_ts
->state_ptr
== 0
2978 || arg_ts
->state
== TS_DEAD
);
2982 /* Outputs become available. */
2983 for (i
= 0; i
< nb_oargs
; i
++) {
2984 arg_ts
= arg_temp(op
->args
[i
]);
2985 dir_ts
= arg_ts
->state_ptr
;
2989 op
->args
[i
] = temp_arg(dir_ts
);
2992 /* The output is now live and modified. */
2995 /* Sync outputs upon their last write. */
2996 if (NEED_SYNC_ARG(i
)) {
2997 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
3000 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
);
3002 sop
->args
[0] = temp_arg(dir_ts
);
3003 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3004 sop
->args
[2] = arg_ts
->mem_offset
;
3006 arg_ts
->state
= TS_MEM
;
3008 /* Drop outputs that are dead. */
3009 if (IS_DEAD_ARG(i
)) {
3010 arg_ts
->state
= TS_DEAD
;
3018 #ifdef CONFIG_DEBUG_TCG
3019 static void dump_regs(TCGContext
*s
)
3025 for(i
= 0; i
< s
->nb_temps
; i
++) {
3027 printf(" %10s: ", tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
3028 switch(ts
->val_type
) {
3030 printf("%s", tcg_target_reg_names
[ts
->reg
]);
3033 printf("%d(%s)", (int)ts
->mem_offset
,
3034 tcg_target_reg_names
[ts
->mem_base
->reg
]);
3036 case TEMP_VAL_CONST
:
3037 printf("$0x%" TCG_PRIlx
, ts
->val
);
3049 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3050 if (s
->reg_to_temp
[i
] != NULL
) {
3052 tcg_target_reg_names
[i
],
3053 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), s
->reg_to_temp
[i
]));
3058 static void check_regs(TCGContext
*s
)
3065 for (reg
= 0; reg
< TCG_TARGET_NB_REGS
; reg
++) {
3066 ts
= s
->reg_to_temp
[reg
];
3068 if (ts
->val_type
!= TEMP_VAL_REG
|| ts
->reg
!= reg
) {
3069 printf("Inconsistency for register %s:\n",
3070 tcg_target_reg_names
[reg
]);
3075 for (k
= 0; k
< s
->nb_temps
; k
++) {
3077 if (ts
->val_type
== TEMP_VAL_REG
&& !ts
->fixed_reg
3078 && s
->reg_to_temp
[ts
->reg
] != ts
) {
3079 printf("Inconsistency for temp %s:\n",
3080 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
3082 printf("reg state:\n");
3090 static void temp_allocate_frame(TCGContext
*s
, TCGTemp
*ts
)
3092 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
3093 /* Sparc64 stack is accessed with offset of 2047 */
3094 s
->current_frame_offset
= (s
->current_frame_offset
+
3095 (tcg_target_long
)sizeof(tcg_target_long
) - 1) &
3096 ~(sizeof(tcg_target_long
) - 1);
3098 if (s
->current_frame_offset
+ (tcg_target_long
)sizeof(tcg_target_long
) >
3102 ts
->mem_offset
= s
->current_frame_offset
;
3103 ts
->mem_base
= s
->frame_temp
;
3104 ts
->mem_allocated
= 1;
3105 s
->current_frame_offset
+= sizeof(tcg_target_long
);
3108 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
, TCGRegSet
);
3110 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3111 mark it free; otherwise mark it dead. */
3112 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
3114 if (ts
->fixed_reg
) {
3117 if (ts
->val_type
== TEMP_VAL_REG
) {
3118 s
->reg_to_temp
[ts
->reg
] = NULL
;
3120 ts
->val_type
= (free_or_dead
< 0
3123 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
3126 /* Mark a temporary as dead. */
3127 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
3129 temp_free_or_dead(s
, ts
, 1);
3132 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3133 registers needs to be allocated to store a constant. If 'free_or_dead'
3134 is non-zero, subsequently release the temporary; if it is positive, the
3135 temp is dead; if it is negative, the temp is free. */
3136 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
,
3137 TCGRegSet preferred_regs
, int free_or_dead
)
3139 if (ts
->fixed_reg
) {
3142 if (!ts
->mem_coherent
) {
3143 if (!ts
->mem_allocated
) {
3144 temp_allocate_frame(s
, ts
);
3146 switch (ts
->val_type
) {
3147 case TEMP_VAL_CONST
:
3148 /* If we're going to free the temp immediately, then we won't
3149 require it later in a register, so attempt to store the
3150 constant to memory directly. */
3152 && tcg_out_sti(s
, ts
->type
, ts
->val
,
3153 ts
->mem_base
->reg
, ts
->mem_offset
)) {
3156 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3157 allocated_regs
, preferred_regs
);
3161 tcg_out_st(s
, ts
->type
, ts
->reg
,
3162 ts
->mem_base
->reg
, ts
->mem_offset
);
3172 ts
->mem_coherent
= 1;
3175 temp_free_or_dead(s
, ts
, free_or_dead
);
3179 /* free register 'reg' by spilling the corresponding temporary if necessary */
3180 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
3182 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
3184 temp_sync(s
, ts
, allocated_regs
, 0, -1);
3190 * @required_regs: Set of registers in which we must allocate.
3191 * @allocated_regs: Set of registers which must be avoided.
3192 * @preferred_regs: Set of registers we should prefer.
3193 * @rev: True if we search the registers in "indirect" order.
3195 * The allocated register must be in @required_regs & ~@allocated_regs,
3196 * but if we can put it in @preferred_regs we may save a move later.
3198 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet required_regs
,
3199 TCGRegSet allocated_regs
,
3200 TCGRegSet preferred_regs
, bool rev
)
3202 int i
, j
, f
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3203 TCGRegSet reg_ct
[2];
3206 reg_ct
[1] = required_regs
& ~allocated_regs
;
3207 tcg_debug_assert(reg_ct
[1] != 0);
3208 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3210 /* Skip the preferred_regs option if it cannot be satisfied,
3211 or if the preference made no difference. */
3212 f
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3214 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3216 /* Try free registers, preferences first. */
3217 for (j
= f
; j
< 2; j
++) {
3218 TCGRegSet set
= reg_ct
[j
];
3220 if (tcg_regset_single(set
)) {
3221 /* One register in the set. */
3222 TCGReg reg
= tcg_regset_first(set
);
3223 if (s
->reg_to_temp
[reg
] == NULL
) {
3227 for (i
= 0; i
< n
; i
++) {
3228 TCGReg reg
= order
[i
];
3229 if (s
->reg_to_temp
[reg
] == NULL
&&
3230 tcg_regset_test_reg(set
, reg
)) {
3237 /* We must spill something. */
3238 for (j
= f
; j
< 2; j
++) {
3239 TCGRegSet set
= reg_ct
[j
];
3241 if (tcg_regset_single(set
)) {
3242 /* One register in the set. */
3243 TCGReg reg
= tcg_regset_first(set
);
3244 tcg_reg_free(s
, reg
, allocated_regs
);
3247 for (i
= 0; i
< n
; i
++) {
3248 TCGReg reg
= order
[i
];
3249 if (tcg_regset_test_reg(set
, reg
)) {
3250 tcg_reg_free(s
, reg
, allocated_regs
);
3260 /* Make sure the temporary is in a register. If needed, allocate the register
3261 from DESIRED while avoiding ALLOCATED. */
3262 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
3263 TCGRegSet allocated_regs
, TCGRegSet preferred_regs
)
3267 switch (ts
->val_type
) {
3270 case TEMP_VAL_CONST
:
3271 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3272 preferred_regs
, ts
->indirect_base
);
3273 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
3274 ts
->mem_coherent
= 0;
3277 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3278 preferred_regs
, ts
->indirect_base
);
3279 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
3280 ts
->mem_coherent
= 1;
3287 ts
->val_type
= TEMP_VAL_REG
;
3288 s
->reg_to_temp
[reg
] = ts
;
3291 /* Save a temporary to memory. 'allocated_regs' is used in case a
3292 temporary registers needs to be allocated to store a constant. */
3293 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
3295 /* The liveness analysis already ensures that globals are back
3296 in memory. Keep an tcg_debug_assert for safety. */
3297 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| ts
->fixed_reg
);
3300 /* save globals to their canonical location and assume they can be
3301 modified be the following code. 'allocated_regs' is used in case a
3302 temporary registers needs to be allocated to store a constant. */
3303 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3307 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3308 temp_save(s
, &s
->temps
[i
], allocated_regs
);
3312 /* sync globals to their canonical location and assume they can be
3313 read by the following code. 'allocated_regs' is used in case a
3314 temporary registers needs to be allocated to store a constant. */
3315 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3319 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3320 TCGTemp
*ts
= &s
->temps
[i
];
3321 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
3323 || ts
->mem_coherent
);
3327 /* at the end of a basic block, we assume all temporaries are dead and
3328 all globals are stored at their canonical location. */
3329 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
3333 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3334 TCGTemp
*ts
= &s
->temps
[i
];
3335 if (ts
->temp_local
) {
3336 temp_save(s
, ts
, allocated_regs
);
3338 /* The liveness analysis already ensures that temps are dead.
3339 Keep an tcg_debug_assert for safety. */
3340 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3344 save_globals(s
, allocated_regs
);
3348 * Specialized code generation for INDEX_op_movi_*.
3350 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
3351 tcg_target_ulong val
, TCGLifeData arg_life
,
3352 TCGRegSet preferred_regs
)
3354 /* ENV should not be modified. */
3355 tcg_debug_assert(!ots
->fixed_reg
);
3357 /* The movi is not explicitly generated here. */
3358 if (ots
->val_type
== TEMP_VAL_REG
) {
3359 s
->reg_to_temp
[ots
->reg
] = NULL
;
3361 ots
->val_type
= TEMP_VAL_CONST
;
3363 ots
->mem_coherent
= 0;
3364 if (NEED_SYNC_ARG(0)) {
3365 temp_sync(s
, ots
, s
->reserved_regs
, preferred_regs
, IS_DEAD_ARG(0));
3366 } else if (IS_DEAD_ARG(0)) {
3371 static void tcg_reg_alloc_movi(TCGContext
*s
, const TCGOp
*op
)
3373 TCGTemp
*ots
= arg_temp(op
->args
[0]);
3374 tcg_target_ulong val
= op
->args
[1];
3376 tcg_reg_alloc_do_movi(s
, ots
, val
, op
->life
, op
->output_pref
[0]);
3380 * Specialized code generation for INDEX_op_mov_*.
3382 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOp
*op
)
3384 const TCGLifeData arg_life
= op
->life
;
3385 TCGRegSet allocated_regs
, preferred_regs
;
3387 TCGType otype
, itype
;
3389 allocated_regs
= s
->reserved_regs
;
3390 preferred_regs
= op
->output_pref
[0];
3391 ots
= arg_temp(op
->args
[0]);
3392 ts
= arg_temp(op
->args
[1]);
3394 /* ENV should not be modified. */
3395 tcg_debug_assert(!ots
->fixed_reg
);
3397 /* Note that otype != itype for no-op truncation. */
3401 if (ts
->val_type
== TEMP_VAL_CONST
) {
3402 /* propagate constant or generate sti */
3403 tcg_target_ulong val
= ts
->val
;
3404 if (IS_DEAD_ARG(1)) {
3407 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, preferred_regs
);
3411 /* If the source value is in memory we're going to be forced
3412 to have it in a register in order to perform the copy. Copy
3413 the SOURCE value into its own register first, that way we
3414 don't have to reload SOURCE the next time it is used. */
3415 if (ts
->val_type
== TEMP_VAL_MEM
) {
3416 temp_load(s
, ts
, tcg_target_available_regs
[itype
],
3417 allocated_regs
, preferred_regs
);
3420 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
3421 if (IS_DEAD_ARG(0)) {
3422 /* mov to a non-saved dead register makes no sense (even with
3423 liveness analysis disabled). */
3424 tcg_debug_assert(NEED_SYNC_ARG(0));
3425 if (!ots
->mem_allocated
) {
3426 temp_allocate_frame(s
, ots
);
3428 tcg_out_st(s
, otype
, ts
->reg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3429 if (IS_DEAD_ARG(1)) {
3434 if (IS_DEAD_ARG(1) && !ts
->fixed_reg
) {
3435 /* the mov can be suppressed */
3436 if (ots
->val_type
== TEMP_VAL_REG
) {
3437 s
->reg_to_temp
[ots
->reg
] = NULL
;
3442 if (ots
->val_type
!= TEMP_VAL_REG
) {
3443 /* When allocating a new register, make sure to not spill the
3445 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
3446 ots
->reg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
3447 allocated_regs
, preferred_regs
,
3448 ots
->indirect_base
);
3450 if (!tcg_out_mov(s
, otype
, ots
->reg
, ts
->reg
)) {
3452 * Cross register class move not supported.
3453 * Store the source register into the destination slot
3454 * and leave the destination temp as TEMP_VAL_MEM.
3456 assert(!ots
->fixed_reg
);
3457 if (!ts
->mem_allocated
) {
3458 temp_allocate_frame(s
, ots
);
3460 tcg_out_st(s
, ts
->type
, ts
->reg
,
3461 ots
->mem_base
->reg
, ots
->mem_offset
);
3462 ots
->mem_coherent
= 1;
3463 temp_free_or_dead(s
, ots
, -1);
3467 ots
->val_type
= TEMP_VAL_REG
;
3468 ots
->mem_coherent
= 0;
3469 s
->reg_to_temp
[ots
->reg
] = ots
;
3470 if (NEED_SYNC_ARG(0)) {
3471 temp_sync(s
, ots
, allocated_regs
, 0, 0);
3477 * Specialized code generation for INDEX_op_dup_vec.
3479 static void tcg_reg_alloc_dup(TCGContext
*s
, const TCGOp
*op
)
3481 const TCGLifeData arg_life
= op
->life
;
3482 TCGRegSet dup_out_regs
, dup_in_regs
;
3484 TCGType itype
, vtype
;
3485 intptr_t endian_fixup
;
3489 ots
= arg_temp(op
->args
[0]);
3490 its
= arg_temp(op
->args
[1]);
3492 /* ENV should not be modified. */
3493 tcg_debug_assert(!ots
->fixed_reg
);
3496 vece
= TCGOP_VECE(op
);
3497 vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
3499 if (its
->val_type
== TEMP_VAL_CONST
) {
3500 /* Propagate constant via movi -> dupi. */
3501 tcg_target_ulong val
= its
->val
;
3502 if (IS_DEAD_ARG(1)) {
3505 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, op
->output_pref
[0]);
3509 dup_out_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].u
.regs
;
3510 dup_in_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[1].u
.regs
;
3512 /* Allocate the output register now. */
3513 if (ots
->val_type
!= TEMP_VAL_REG
) {
3514 TCGRegSet allocated_regs
= s
->reserved_regs
;
3516 if (!IS_DEAD_ARG(1) && its
->val_type
== TEMP_VAL_REG
) {
3517 /* Make sure to not spill the input register. */
3518 tcg_regset_set_reg(allocated_regs
, its
->reg
);
3520 ots
->reg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
3521 op
->output_pref
[0], ots
->indirect_base
);
3522 ots
->val_type
= TEMP_VAL_REG
;
3523 ots
->mem_coherent
= 0;
3524 s
->reg_to_temp
[ots
->reg
] = ots
;
3527 switch (its
->val_type
) {
3530 * The dup constriaints must be broad, covering all possible VECE.
3531 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3532 * to fail, indicating that extra moves are required for that case.
3534 if (tcg_regset_test_reg(dup_in_regs
, its
->reg
)) {
3535 if (tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, its
->reg
)) {
3538 /* Try again from memory or a vector input register. */
3540 if (!its
->mem_coherent
) {
3542 * The input register is not synced, and so an extra store
3543 * would be required to use memory. Attempt an integer-vector
3544 * register move first. We do not have a TCGRegSet for this.
3546 if (tcg_out_mov(s
, itype
, ots
->reg
, its
->reg
)) {
3549 /* Sync the temp back to its slot and load from there. */
3550 temp_sync(s
, its
, s
->reserved_regs
, 0, 0);
3555 #ifdef HOST_WORDS_BIGENDIAN
3556 endian_fixup
= itype
== TCG_TYPE_I32
? 4 : 8;
3557 endian_fixup
-= 1 << vece
;
3561 if (tcg_out_dupm_vec(s
, vtype
, vece
, ots
->reg
, its
->mem_base
->reg
,
3562 its
->mem_offset
+ endian_fixup
)) {
3565 tcg_out_ld(s
, itype
, ots
->reg
, its
->mem_base
->reg
, its
->mem_offset
);
3569 g_assert_not_reached();
3572 /* We now have a vector input register, so dup must succeed. */
3573 ok
= tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, ots
->reg
);
3574 tcg_debug_assert(ok
);
3577 if (IS_DEAD_ARG(1)) {
3580 if (NEED_SYNC_ARG(0)) {
3581 temp_sync(s
, ots
, s
->reserved_regs
, 0, 0);
3583 if (IS_DEAD_ARG(0)) {
3588 static void tcg_reg_alloc_op(TCGContext
*s
, const TCGOp
*op
)
3590 const TCGLifeData arg_life
= op
->life
;
3591 const TCGOpDef
* const def
= &tcg_op_defs
[op
->opc
];
3592 TCGRegSet i_allocated_regs
;
3593 TCGRegSet o_allocated_regs
;
3594 int i
, k
, nb_iargs
, nb_oargs
;
3597 const TCGArgConstraint
*arg_ct
;
3599 TCGArg new_args
[TCG_MAX_OP_ARGS
];
3600 int const_args
[TCG_MAX_OP_ARGS
];
3602 nb_oargs
= def
->nb_oargs
;
3603 nb_iargs
= def
->nb_iargs
;
3605 /* copy constants */
3606 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
3607 op
->args
+ nb_oargs
+ nb_iargs
,
3608 sizeof(TCGArg
) * def
->nb_cargs
);
3610 i_allocated_regs
= s
->reserved_regs
;
3611 o_allocated_regs
= s
->reserved_regs
;
3613 /* satisfy input constraints */
3614 for (k
= 0; k
< nb_iargs
; k
++) {
3615 TCGRegSet i_preferred_regs
, o_preferred_regs
;
3617 i
= def
->sorted_args
[nb_oargs
+ k
];
3619 arg_ct
= &def
->args_ct
[i
];
3622 if (ts
->val_type
== TEMP_VAL_CONST
3623 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
)) {
3624 /* constant is OK for instruction */
3626 new_args
[i
] = ts
->val
;
3630 i_preferred_regs
= o_preferred_regs
= 0;
3631 if (arg_ct
->ct
& TCG_CT_IALIAS
) {
3632 o_preferred_regs
= op
->output_pref
[arg_ct
->alias_index
];
3633 if (ts
->fixed_reg
) {
3634 /* if fixed register, we must allocate a new register
3635 if the alias is not the same register */
3636 if (arg
!= op
->args
[arg_ct
->alias_index
]) {
3637 goto allocate_in_reg
;
3640 /* if the input is aliased to an output and if it is
3641 not dead after the instruction, we must allocate
3642 a new register and move it */
3643 if (!IS_DEAD_ARG(i
)) {
3644 goto allocate_in_reg
;
3647 /* check if the current register has already been allocated
3648 for another input aliased to an output */
3649 if (ts
->val_type
== TEMP_VAL_REG
) {
3652 for (k2
= 0 ; k2
< k
; k2
++) {
3653 i2
= def
->sorted_args
[nb_oargs
+ k2
];
3654 if ((def
->args_ct
[i2
].ct
& TCG_CT_IALIAS
) &&
3655 reg
== new_args
[i2
]) {
3656 goto allocate_in_reg
;
3660 i_preferred_regs
= o_preferred_regs
;
3664 temp_load(s
, ts
, arg_ct
->u
.regs
, i_allocated_regs
, i_preferred_regs
);
3667 if (tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
3668 /* nothing to do : the constraint is satisfied */
3671 /* allocate a new register matching the constraint
3672 and move the temporary register into it */
3673 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3674 i_allocated_regs
, 0);
3675 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, i_allocated_regs
,
3676 o_preferred_regs
, ts
->indirect_base
);
3677 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
3679 * Cross register class move not supported. Sync the
3680 * temp back to its slot and load from there.
3682 temp_sync(s
, ts
, i_allocated_regs
, 0, 0);
3683 tcg_out_ld(s
, ts
->type
, reg
,
3684 ts
->mem_base
->reg
, ts
->mem_offset
);
3689 tcg_regset_set_reg(i_allocated_regs
, reg
);
3692 /* mark dead temporaries and free the associated registers */
3693 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3694 if (IS_DEAD_ARG(i
)) {
3695 temp_dead(s
, arg_temp(op
->args
[i
]));
3699 if (def
->flags
& TCG_OPF_BB_END
) {
3700 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
3702 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
3703 /* XXX: permit generic clobber register list ? */
3704 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3705 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3706 tcg_reg_free(s
, i
, i_allocated_regs
);
3710 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
3711 /* sync globals if the op has side effects and might trigger
3713 sync_globals(s
, i_allocated_regs
);
3716 /* satisfy the output constraints */
3717 for(k
= 0; k
< nb_oargs
; k
++) {
3718 i
= def
->sorted_args
[k
];
3720 arg_ct
= &def
->args_ct
[i
];
3723 /* ENV should not be modified. */
3724 tcg_debug_assert(!ts
->fixed_reg
);
3726 if ((arg_ct
->ct
& TCG_CT_ALIAS
)
3727 && !const_args
[arg_ct
->alias_index
]) {
3728 reg
= new_args
[arg_ct
->alias_index
];
3729 } else if (arg_ct
->ct
& TCG_CT_NEWREG
) {
3730 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
,
3731 i_allocated_regs
| o_allocated_regs
,
3732 op
->output_pref
[k
], ts
->indirect_base
);
3734 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, o_allocated_regs
,
3735 op
->output_pref
[k
], ts
->indirect_base
);
3737 tcg_regset_set_reg(o_allocated_regs
, reg
);
3738 if (ts
->val_type
== TEMP_VAL_REG
) {
3739 s
->reg_to_temp
[ts
->reg
] = NULL
;
3741 ts
->val_type
= TEMP_VAL_REG
;
3744 * Temp value is modified, so the value kept in memory is
3745 * potentially not the same.
3747 ts
->mem_coherent
= 0;
3748 s
->reg_to_temp
[reg
] = ts
;
3753 /* emit instruction */
3754 if (def
->flags
& TCG_OPF_VECTOR
) {
3755 tcg_out_vec_op(s
, op
->opc
, TCGOP_VECL(op
), TCGOP_VECE(op
),
3756 new_args
, const_args
);
3758 tcg_out_op(s
, op
->opc
, new_args
, const_args
);
3761 /* move the outputs in the correct register if needed */
3762 for(i
= 0; i
< nb_oargs
; i
++) {
3763 ts
= arg_temp(op
->args
[i
]);
3765 /* ENV should not be modified. */
3766 tcg_debug_assert(!ts
->fixed_reg
);
3768 if (NEED_SYNC_ARG(i
)) {
3769 temp_sync(s
, ts
, o_allocated_regs
, 0, IS_DEAD_ARG(i
));
3770 } else if (IS_DEAD_ARG(i
)) {
3776 #ifdef TCG_TARGET_STACK_GROWSUP
3777 #define STACK_DIR(x) (-(x))
3779 #define STACK_DIR(x) (x)
3782 static void tcg_reg_alloc_call(TCGContext
*s
, TCGOp
*op
)
3784 const int nb_oargs
= TCGOP_CALLO(op
);
3785 const int nb_iargs
= TCGOP_CALLI(op
);
3786 const TCGLifeData arg_life
= op
->life
;
3787 int flags
, nb_regs
, i
;
3791 intptr_t stack_offset
;
3792 size_t call_stack_size
;
3793 tcg_insn_unit
*func_addr
;
3795 TCGRegSet allocated_regs
;
3797 func_addr
= (tcg_insn_unit
*)(intptr_t)op
->args
[nb_oargs
+ nb_iargs
];
3798 flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
3800 nb_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
3801 if (nb_regs
> nb_iargs
) {
3805 /* assign stack slots first */
3806 call_stack_size
= (nb_iargs
- nb_regs
) * sizeof(tcg_target_long
);
3807 call_stack_size
= (call_stack_size
+ TCG_TARGET_STACK_ALIGN
- 1) &
3808 ~(TCG_TARGET_STACK_ALIGN
- 1);
3809 allocate_args
= (call_stack_size
> TCG_STATIC_CALL_ARGS_SIZE
);
3810 if (allocate_args
) {
3811 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3812 preallocate call stack */
3816 stack_offset
= TCG_TARGET_CALL_STACK_OFFSET
;
3817 for (i
= nb_regs
; i
< nb_iargs
; i
++) {
3818 arg
= op
->args
[nb_oargs
+ i
];
3819 #ifdef TCG_TARGET_STACK_GROWSUP
3820 stack_offset
-= sizeof(tcg_target_long
);
3822 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3824 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3825 s
->reserved_regs
, 0);
3826 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
, stack_offset
);
3828 #ifndef TCG_TARGET_STACK_GROWSUP
3829 stack_offset
+= sizeof(tcg_target_long
);
3833 /* assign input registers */
3834 allocated_regs
= s
->reserved_regs
;
3835 for (i
= 0; i
< nb_regs
; i
++) {
3836 arg
= op
->args
[nb_oargs
+ i
];
3837 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3839 reg
= tcg_target_call_iarg_regs
[i
];
3841 if (ts
->val_type
== TEMP_VAL_REG
) {
3842 if (ts
->reg
!= reg
) {
3843 tcg_reg_free(s
, reg
, allocated_regs
);
3844 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
3846 * Cross register class move not supported. Sync the
3847 * temp back to its slot and load from there.
3849 temp_sync(s
, ts
, allocated_regs
, 0, 0);
3850 tcg_out_ld(s
, ts
->type
, reg
,
3851 ts
->mem_base
->reg
, ts
->mem_offset
);
3855 TCGRegSet arg_set
= 0;
3857 tcg_reg_free(s
, reg
, allocated_regs
);
3858 tcg_regset_set_reg(arg_set
, reg
);
3859 temp_load(s
, ts
, arg_set
, allocated_regs
, 0);
3862 tcg_regset_set_reg(allocated_regs
, reg
);
3866 /* mark dead temporaries and free the associated registers */
3867 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3868 if (IS_DEAD_ARG(i
)) {
3869 temp_dead(s
, arg_temp(op
->args
[i
]));
3873 /* clobber call registers */
3874 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3875 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3876 tcg_reg_free(s
, i
, allocated_regs
);
3880 /* Save globals if they might be written by the helper, sync them if
3881 they might be read. */
3882 if (flags
& TCG_CALL_NO_READ_GLOBALS
) {
3884 } else if (flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
3885 sync_globals(s
, allocated_regs
);
3887 save_globals(s
, allocated_regs
);
3890 tcg_out_call(s
, func_addr
);
3892 /* assign output registers and emit moves if needed */
3893 for(i
= 0; i
< nb_oargs
; i
++) {
3897 /* ENV should not be modified. */
3898 tcg_debug_assert(!ts
->fixed_reg
);
3900 reg
= tcg_target_call_oarg_regs
[i
];
3901 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
3902 if (ts
->val_type
== TEMP_VAL_REG
) {
3903 s
->reg_to_temp
[ts
->reg
] = NULL
;
3905 ts
->val_type
= TEMP_VAL_REG
;
3907 ts
->mem_coherent
= 0;
3908 s
->reg_to_temp
[reg
] = ts
;
3909 if (NEED_SYNC_ARG(i
)) {
3910 temp_sync(s
, ts
, allocated_regs
, 0, IS_DEAD_ARG(i
));
3911 } else if (IS_DEAD_ARG(i
)) {
3917 #ifdef CONFIG_PROFILER
3919 /* avoid copy/paste errors */
3920 #define PROF_ADD(to, from, field) \
3922 (to)->field += atomic_read(&((from)->field)); \
3925 #define PROF_MAX(to, from, field) \
3927 typeof((from)->field) val__ = atomic_read(&((from)->field)); \
3928 if (val__ > (to)->field) { \
3929 (to)->field = val__; \
3933 /* Pass in a zero'ed @prof */
3935 void tcg_profile_snapshot(TCGProfile
*prof
, bool counters
, bool table
)
3937 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
3940 for (i
= 0; i
< n_ctxs
; i
++) {
3941 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
3942 const TCGProfile
*orig
= &s
->prof
;
3945 PROF_ADD(prof
, orig
, cpu_exec_time
);
3946 PROF_ADD(prof
, orig
, tb_count1
);
3947 PROF_ADD(prof
, orig
, tb_count
);
3948 PROF_ADD(prof
, orig
, op_count
);
3949 PROF_MAX(prof
, orig
, op_count_max
);
3950 PROF_ADD(prof
, orig
, temp_count
);
3951 PROF_MAX(prof
, orig
, temp_count_max
);
3952 PROF_ADD(prof
, orig
, del_op_count
);
3953 PROF_ADD(prof
, orig
, code_in_len
);
3954 PROF_ADD(prof
, orig
, code_out_len
);
3955 PROF_ADD(prof
, orig
, search_out_len
);
3956 PROF_ADD(prof
, orig
, interm_time
);
3957 PROF_ADD(prof
, orig
, code_time
);
3958 PROF_ADD(prof
, orig
, la_time
);
3959 PROF_ADD(prof
, orig
, opt_time
);
3960 PROF_ADD(prof
, orig
, restore_count
);
3961 PROF_ADD(prof
, orig
, restore_time
);
3966 for (i
= 0; i
< NB_OPS
; i
++) {
3967 PROF_ADD(prof
, orig
, table_op_count
[i
]);
3976 static void tcg_profile_snapshot_counters(TCGProfile
*prof
)
3978 tcg_profile_snapshot(prof
, true, false);
3981 static void tcg_profile_snapshot_table(TCGProfile
*prof
)
3983 tcg_profile_snapshot(prof
, false, true);
3986 void tcg_dump_op_count(void)
3988 TCGProfile prof
= {};
3991 tcg_profile_snapshot_table(&prof
);
3992 for (i
= 0; i
< NB_OPS
; i
++) {
3993 qemu_printf("%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
3994 prof
.table_op_count
[i
]);
3998 int64_t tcg_cpu_exec_time(void)
4000 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
4004 for (i
= 0; i
< n_ctxs
; i
++) {
4005 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
4006 const TCGProfile
*prof
= &s
->prof
;
4008 ret
+= atomic_read(&prof
->cpu_exec_time
);
4013 void tcg_dump_op_count(void)
4015 qemu_printf("[TCG profiler not compiled]\n");
4018 int64_t tcg_cpu_exec_time(void)
4020 error_report("%s: TCG profiler not compiled", __func__
);
4026 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
)
4028 #ifdef CONFIG_PROFILER
4029 TCGProfile
*prof
= &s
->prof
;
4034 #ifdef CONFIG_PROFILER
4038 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4041 atomic_set(&prof
->op_count
, prof
->op_count
+ n
);
4042 if (n
> prof
->op_count_max
) {
4043 atomic_set(&prof
->op_count_max
, n
);
4047 atomic_set(&prof
->temp_count
, prof
->temp_count
+ n
);
4048 if (n
> prof
->temp_count_max
) {
4049 atomic_set(&prof
->temp_count_max
, n
);
4055 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
4056 && qemu_log_in_addr_range(tb
->pc
))) {
4057 FILE *logfile
= qemu_log_lock();
4059 tcg_dump_ops(s
, false);
4061 qemu_log_unlock(logfile
);
4065 #ifdef CONFIG_DEBUG_TCG
4066 /* Ensure all labels referenced have been emitted. */
4071 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
4072 if (unlikely(!l
->present
) && l
->refs
) {
4073 qemu_log_mask(CPU_LOG_TB_OP
,
4074 "$L%d referenced but not present.\n", l
->id
);
4082 #ifdef CONFIG_PROFILER
4083 atomic_set(&prof
->opt_time
, prof
->opt_time
- profile_getclock());
4086 #ifdef USE_TCG_OPTIMIZATIONS
4090 #ifdef CONFIG_PROFILER
4091 atomic_set(&prof
->opt_time
, prof
->opt_time
+ profile_getclock());
4092 atomic_set(&prof
->la_time
, prof
->la_time
- profile_getclock());
4095 reachable_code_pass(s
);
4098 if (s
->nb_indirects
> 0) {
4100 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
4101 && qemu_log_in_addr_range(tb
->pc
))) {
4102 FILE *logfile
= qemu_log_lock();
4103 qemu_log("OP before indirect lowering:\n");
4104 tcg_dump_ops(s
, false);
4106 qemu_log_unlock(logfile
);
4109 /* Replace indirect temps with direct temps. */
4110 if (liveness_pass_2(s
)) {
4111 /* If changes were made, re-run liveness. */
4116 #ifdef CONFIG_PROFILER
4117 atomic_set(&prof
->la_time
, prof
->la_time
+ profile_getclock());
4121 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
4122 && qemu_log_in_addr_range(tb
->pc
))) {
4123 FILE *logfile
= qemu_log_lock();
4124 qemu_log("OP after optimization and liveness analysis:\n");
4125 tcg_dump_ops(s
, true);
4127 qemu_log_unlock(logfile
);
4131 tcg_reg_alloc_start(s
);
4133 s
->code_buf
= tb
->tc
.ptr
;
4134 s
->code_ptr
= tb
->tc
.ptr
;
4136 #ifdef TCG_TARGET_NEED_LDST_LABELS
4137 QSIMPLEQ_INIT(&s
->ldst_labels
);
4139 #ifdef TCG_TARGET_NEED_POOL_LABELS
4140 s
->pool_labels
= NULL
;
4144 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4145 TCGOpcode opc
= op
->opc
;
4147 #ifdef CONFIG_PROFILER
4148 atomic_set(&prof
->table_op_count
[opc
], prof
->table_op_count
[opc
] + 1);
4152 case INDEX_op_mov_i32
:
4153 case INDEX_op_mov_i64
:
4154 case INDEX_op_mov_vec
:
4155 tcg_reg_alloc_mov(s
, op
);
4157 case INDEX_op_movi_i32
:
4158 case INDEX_op_movi_i64
:
4159 case INDEX_op_dupi_vec
:
4160 tcg_reg_alloc_movi(s
, op
);
4162 case INDEX_op_dup_vec
:
4163 tcg_reg_alloc_dup(s
, op
);
4165 case INDEX_op_insn_start
:
4166 if (num_insns
>= 0) {
4167 size_t off
= tcg_current_code_size(s
);
4168 s
->gen_insn_end_off
[num_insns
] = off
;
4169 /* Assert that we do not overflow our stored offset. */
4170 assert(s
->gen_insn_end_off
[num_insns
] == off
);
4173 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
4175 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4176 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
4180 s
->gen_insn_data
[num_insns
][i
] = a
;
4183 case INDEX_op_discard
:
4184 temp_dead(s
, arg_temp(op
->args
[0]));
4186 case INDEX_op_set_label
:
4187 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
4188 tcg_out_label(s
, arg_label(op
->args
[0]), s
->code_ptr
);
4191 tcg_reg_alloc_call(s
, op
);
4194 /* Sanity check that we've not introduced any unhandled opcodes. */
4195 tcg_debug_assert(tcg_op_supported(opc
));
4196 /* Note: in order to speed up the code, it would be much
4197 faster to have specialized register allocator functions for
4198 some common argument patterns */
4199 tcg_reg_alloc_op(s
, op
);
4202 #ifdef CONFIG_DEBUG_TCG
4205 /* Test for (pending) buffer overflow. The assumption is that any
4206 one operation beginning below the high water mark cannot overrun
4207 the buffer completely. Thus we can test for overflow after
4208 generating code without having to check during generation. */
4209 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
4212 /* Test for TB overflow, as seen by gen_insn_end_off. */
4213 if (unlikely(tcg_current_code_size(s
) > UINT16_MAX
)) {
4217 tcg_debug_assert(num_insns
>= 0);
4218 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
4220 /* Generate TB finalization at the end of block */
4221 #ifdef TCG_TARGET_NEED_LDST_LABELS
4222 i
= tcg_out_ldst_finalize(s
);
4227 #ifdef TCG_TARGET_NEED_POOL_LABELS
4228 i
= tcg_out_pool_finalize(s
);
4233 if (!tcg_resolve_relocs(s
)) {
4237 /* flush instruction cache */
4238 flush_icache_range((uintptr_t)s
->code_buf
, (uintptr_t)s
->code_ptr
);
4240 return tcg_current_code_size(s
);
4243 #ifdef CONFIG_PROFILER
4244 void tcg_dump_info(void)
4246 TCGProfile prof
= {};
4247 const TCGProfile
*s
;
4249 int64_t tb_div_count
;
4252 tcg_profile_snapshot_counters(&prof
);
4254 tb_count
= s
->tb_count
;
4255 tb_div_count
= tb_count
? tb_count
: 1;
4256 tot
= s
->interm_time
+ s
->code_time
;
4258 qemu_printf("JIT cycles %" PRId64
" (%0.3f s at 2.4 GHz)\n",
4260 qemu_printf("translated TBs %" PRId64
" (aborted=%" PRId64
4262 tb_count
, s
->tb_count1
- tb_count
,
4263 (double)(s
->tb_count1
- s
->tb_count
)
4264 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
4265 qemu_printf("avg ops/TB %0.1f max=%d\n",
4266 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
4267 qemu_printf("deleted ops/TB %0.2f\n",
4268 (double)s
->del_op_count
/ tb_div_count
);
4269 qemu_printf("avg temps/TB %0.2f max=%d\n",
4270 (double)s
->temp_count
/ tb_div_count
, s
->temp_count_max
);
4271 qemu_printf("avg host code/TB %0.1f\n",
4272 (double)s
->code_out_len
/ tb_div_count
);
4273 qemu_printf("avg search data/TB %0.1f\n",
4274 (double)s
->search_out_len
/ tb_div_count
);
4276 qemu_printf("cycles/op %0.1f\n",
4277 s
->op_count
? (double)tot
/ s
->op_count
: 0);
4278 qemu_printf("cycles/in byte %0.1f\n",
4279 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
4280 qemu_printf("cycles/out byte %0.1f\n",
4281 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
4282 qemu_printf("cycles/search byte %0.1f\n",
4283 s
->search_out_len
? (double)tot
/ s
->search_out_len
: 0);
4287 qemu_printf(" gen_interm time %0.1f%%\n",
4288 (double)s
->interm_time
/ tot
* 100.0);
4289 qemu_printf(" gen_code time %0.1f%%\n",
4290 (double)s
->code_time
/ tot
* 100.0);
4291 qemu_printf("optim./code time %0.1f%%\n",
4292 (double)s
->opt_time
/ (s
->code_time
? s
->code_time
: 1)
4294 qemu_printf("liveness/code time %0.1f%%\n",
4295 (double)s
->la_time
/ (s
->code_time
? s
->code_time
: 1) * 100.0);
4296 qemu_printf("cpu_restore count %" PRId64
"\n",
4298 qemu_printf(" avg cycles %0.1f\n",
4299 s
->restore_count
? (double)s
->restore_time
/ s
->restore_count
: 0);
4302 void tcg_dump_info(void)
4304 qemu_printf("[TCG profiler not compiled]\n");
4308 #ifdef ELF_HOST_MACHINE
4309 /* In order to use this feature, the backend needs to do three things:
4311 (1) Define ELF_HOST_MACHINE to indicate both what value to
4312 put into the ELF image and to indicate support for the feature.
4314 (2) Define tcg_register_jit. This should create a buffer containing
4315 the contents of a .debug_frame section that describes the post-
4316 prologue unwind info for the tcg machine.
4318 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4321 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4328 struct jit_code_entry
{
4329 struct jit_code_entry
*next_entry
;
4330 struct jit_code_entry
*prev_entry
;
4331 const void *symfile_addr
;
4332 uint64_t symfile_size
;
4335 struct jit_descriptor
{
4337 uint32_t action_flag
;
4338 struct jit_code_entry
*relevant_entry
;
4339 struct jit_code_entry
*first_entry
;
4342 void __jit_debug_register_code(void) __attribute__((noinline
));
4343 void __jit_debug_register_code(void)
4348 /* Must statically initialize the version, because GDB may check
4349 the version before we can set it. */
4350 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
4352 /* End GDB interface. */
4354 static int find_string(const char *strtab
, const char *str
)
4356 const char *p
= strtab
+ 1;
4359 if (strcmp(p
, str
) == 0) {
4366 static void tcg_register_jit_int(void *buf_ptr
, size_t buf_size
,
4367 const void *debug_frame
,
4368 size_t debug_frame_size
)
4370 struct __attribute__((packed
)) DebugInfo
{
4377 uintptr_t cu_low_pc
;
4378 uintptr_t cu_high_pc
;
4381 uintptr_t fn_low_pc
;
4382 uintptr_t fn_high_pc
;
4391 struct DebugInfo di
;
4396 struct ElfImage
*img
;
4398 static const struct ElfImage img_template
= {
4400 .e_ident
[EI_MAG0
] = ELFMAG0
,
4401 .e_ident
[EI_MAG1
] = ELFMAG1
,
4402 .e_ident
[EI_MAG2
] = ELFMAG2
,
4403 .e_ident
[EI_MAG3
] = ELFMAG3
,
4404 .e_ident
[EI_CLASS
] = ELF_CLASS
,
4405 .e_ident
[EI_DATA
] = ELF_DATA
,
4406 .e_ident
[EI_VERSION
] = EV_CURRENT
,
4408 .e_machine
= ELF_HOST_MACHINE
,
4409 .e_version
= EV_CURRENT
,
4410 .e_phoff
= offsetof(struct ElfImage
, phdr
),
4411 .e_shoff
= offsetof(struct ElfImage
, shdr
),
4412 .e_ehsize
= sizeof(ElfW(Shdr
)),
4413 .e_phentsize
= sizeof(ElfW(Phdr
)),
4415 .e_shentsize
= sizeof(ElfW(Shdr
)),
4416 .e_shnum
= ARRAY_SIZE(img
->shdr
),
4417 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
4418 #ifdef ELF_HOST_FLAGS
4419 .e_flags
= ELF_HOST_FLAGS
,
4422 .e_ident
[EI_OSABI
] = ELF_OSABI
,
4430 [0] = { .sh_type
= SHT_NULL
},
4431 /* Trick: The contents of code_gen_buffer are not present in
4432 this fake ELF file; that got allocated elsewhere. Therefore
4433 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4434 will not look for contents. We can record any address. */
4436 .sh_type
= SHT_NOBITS
,
4437 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
4439 [2] = { /* .debug_info */
4440 .sh_type
= SHT_PROGBITS
,
4441 .sh_offset
= offsetof(struct ElfImage
, di
),
4442 .sh_size
= sizeof(struct DebugInfo
),
4444 [3] = { /* .debug_abbrev */
4445 .sh_type
= SHT_PROGBITS
,
4446 .sh_offset
= offsetof(struct ElfImage
, da
),
4447 .sh_size
= sizeof(img
->da
),
4449 [4] = { /* .debug_frame */
4450 .sh_type
= SHT_PROGBITS
,
4451 .sh_offset
= sizeof(struct ElfImage
),
4453 [5] = { /* .symtab */
4454 .sh_type
= SHT_SYMTAB
,
4455 .sh_offset
= offsetof(struct ElfImage
, sym
),
4456 .sh_size
= sizeof(img
->sym
),
4458 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
4459 .sh_entsize
= sizeof(ElfW(Sym
)),
4461 [6] = { /* .strtab */
4462 .sh_type
= SHT_STRTAB
,
4463 .sh_offset
= offsetof(struct ElfImage
, str
),
4464 .sh_size
= sizeof(img
->str
),
4468 [1] = { /* code_gen_buffer */
4469 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
4474 .len
= sizeof(struct DebugInfo
) - 4,
4476 .ptr_size
= sizeof(void *),
4478 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
4480 .fn_name
= "code_gen_buffer"
4483 1, /* abbrev number (the cu) */
4484 0x11, 1, /* DW_TAG_compile_unit, has children */
4485 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4486 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4487 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4488 0, 0, /* end of abbrev */
4489 2, /* abbrev number (the fn) */
4490 0x2e, 0, /* DW_TAG_subprogram, no children */
4491 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4492 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4493 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4494 0, 0, /* end of abbrev */
4495 0 /* no more abbrev */
4497 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4498 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
4501 /* We only need a single jit entry; statically allocate it. */
4502 static struct jit_code_entry one_entry
;
4504 uintptr_t buf
= (uintptr_t)buf_ptr
;
4505 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
4506 DebugFrameHeader
*dfh
;
4508 img
= g_malloc(img_size
);
4509 *img
= img_template
;
4511 img
->phdr
.p_vaddr
= buf
;
4512 img
->phdr
.p_paddr
= buf
;
4513 img
->phdr
.p_memsz
= buf_size
;
4515 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
4516 img
->shdr
[1].sh_addr
= buf
;
4517 img
->shdr
[1].sh_size
= buf_size
;
4519 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
4520 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
4522 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
4523 img
->shdr
[4].sh_size
= debug_frame_size
;
4525 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
4526 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
4528 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
4529 img
->sym
[1].st_value
= buf
;
4530 img
->sym
[1].st_size
= buf_size
;
4532 img
->di
.cu_low_pc
= buf
;
4533 img
->di
.cu_high_pc
= buf
+ buf_size
;
4534 img
->di
.fn_low_pc
= buf
;
4535 img
->di
.fn_high_pc
= buf
+ buf_size
;
4537 dfh
= (DebugFrameHeader
*)(img
+ 1);
4538 memcpy(dfh
, debug_frame
, debug_frame_size
);
4539 dfh
->fde
.func_start
= buf
;
4540 dfh
->fde
.func_len
= buf_size
;
4543 /* Enable this block to be able to debug the ELF image file creation.
4544 One can use readelf, objdump, or other inspection utilities. */
4546 FILE *f
= fopen("/tmp/qemu.jit", "w+b");
4548 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
4549 /* Avoid stupid unused return value warning for fwrite. */
4556 one_entry
.symfile_addr
= img
;
4557 one_entry
.symfile_size
= img_size
;
4559 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
4560 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
4561 __jit_debug_descriptor
.first_entry
= &one_entry
;
4562 __jit_debug_register_code();
4565 /* No support for the feature. Provide the entry point expected by exec.c,
4566 and implement the internal function we declared earlier. */
4568 static void tcg_register_jit_int(void *buf
, size_t size
,
4569 const void *debug_frame
,
4570 size_t debug_frame_size
)
4574 void tcg_register_jit(void *buf
, size_t buf_size
)
4577 #endif /* ELF_HOST_MACHINE */
4579 #if !TCG_TARGET_MAYBE_vec
4580 void tcg_expand_vec_op(TCGOpcode o
, TCGType t
, unsigned e
, TCGArg a0
, ...)
4582 g_assert_not_reached();