2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
39 /* Note: the long term plan is to reduce the dependencies on the QEMU
40 CPU definitions. Currently they are used for qemu_ld/st
42 #define NO_CPU_IO_DEFS
45 #include "exec/exec-all.h"
47 #if !defined(CONFIG_USER_ONLY)
48 #include "hw/boards.h"
51 #include "tcg/tcg-op.h"
53 #if UINTPTR_MAX == UINT32_MAX
54 # define ELF_CLASS ELFCLASS32
56 # define ELF_CLASS ELFCLASS64
58 #ifdef HOST_WORDS_BIGENDIAN
59 # define ELF_DATA ELFDATA2MSB
61 # define ELF_DATA ELFDATA2LSB
66 #include "sysemu/sysemu.h"
68 /* Forward declarations for functions declared in tcg-target.inc.c and
70 static void tcg_target_init(TCGContext
*s
);
71 static const TCGTargetOpDef
*tcg_target_op_def(TCGOpcode
);
72 static void tcg_target_qemu_prologue(TCGContext
*s
);
73 static bool patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
74 intptr_t value
, intptr_t addend
);
76 /* The CIE and FDE header definitions will be common to all hosts. */
78 uint32_t len
__attribute__((aligned((sizeof(void *)))));
84 uint8_t return_column
;
87 typedef struct QEMU_PACKED
{
88 uint32_t len
__attribute__((aligned((sizeof(void *)))));
92 } DebugFrameFDEHeader
;
94 typedef struct QEMU_PACKED
{
96 DebugFrameFDEHeader fde
;
99 static void tcg_register_jit_int(void *buf
, size_t size
,
100 const void *debug_frame
,
101 size_t debug_frame_size
)
102 __attribute__((unused
));
104 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
105 static const char *target_parse_constraint(TCGArgConstraint
*ct
,
106 const char *ct_str
, TCGType type
);
107 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
109 static bool tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
110 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
111 TCGReg ret
, tcg_target_long arg
);
112 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
, const TCGArg
*args
,
113 const int *const_args
);
114 #if TCG_TARGET_MAYBE_vec
115 static bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
116 TCGReg dst
, TCGReg src
);
117 static bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
118 TCGReg dst
, TCGReg base
, intptr_t offset
);
119 static void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
,
120 TCGReg dst
, tcg_target_long arg
);
121 static void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
122 unsigned vece
, const TCGArg
*args
,
123 const int *const_args
);
125 static inline bool tcg_out_dup_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
126 TCGReg dst
, TCGReg src
)
128 g_assert_not_reached();
130 static inline bool tcg_out_dupm_vec(TCGContext
*s
, TCGType type
, unsigned vece
,
131 TCGReg dst
, TCGReg base
, intptr_t offset
)
133 g_assert_not_reached();
135 static inline void tcg_out_dupi_vec(TCGContext
*s
, TCGType type
,
136 TCGReg dst
, tcg_target_long arg
)
138 g_assert_not_reached();
140 static inline void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
141 unsigned vece
, const TCGArg
*args
,
142 const int *const_args
)
144 g_assert_not_reached();
147 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
149 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
150 TCGReg base
, intptr_t ofs
);
151 static void tcg_out_call(TCGContext
*s
, tcg_insn_unit
*target
);
152 static int tcg_target_const_match(tcg_target_long val
, TCGType type
,
153 const TCGArgConstraint
*arg_ct
);
154 #ifdef TCG_TARGET_NEED_LDST_LABELS
155 static int tcg_out_ldst_finalize(TCGContext
*s
);
158 #define TCG_HIGHWATER 1024
160 static TCGContext
**tcg_ctxs
;
161 static unsigned int n_tcg_ctxs
;
162 TCGv_env cpu_env
= 0;
164 struct tcg_region_tree
{
167 /* padding to avoid false sharing is computed at run-time */
171 * We divide code_gen_buffer into equally-sized "regions" that TCG threads
172 * dynamically allocate from as demand dictates. Given appropriate region
173 * sizing, this minimizes flushes even when some TCG threads generate a lot
174 * more code than others.
176 struct tcg_region_state
{
179 /* fields set at init time */
184 size_t size
; /* size of one region */
185 size_t stride
; /* .size + guard size */
187 /* fields protected by the lock */
188 size_t current
; /* current region index */
189 size_t agg_size_full
; /* aggregate size of full regions */
192 static struct tcg_region_state region
;
194 * This is an array of struct tcg_region_tree's, with padding.
195 * We use void * to simplify the computation of region_trees[i]; each
196 * struct is found every tree_size bytes.
198 static void *region_trees
;
199 static size_t tree_size
;
200 static TCGRegSet tcg_target_available_regs
[TCG_TYPE_COUNT
];
201 static TCGRegSet tcg_target_call_clobber_regs
;
203 #if TCG_TARGET_INSN_UNIT_SIZE == 1
204 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
209 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
216 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
217 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
219 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
222 tcg_insn_unit
*p
= s
->code_ptr
;
223 memcpy(p
, &v
, sizeof(v
));
224 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
228 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
231 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
234 memcpy(p
, &v
, sizeof(v
));
239 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
240 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
242 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
245 tcg_insn_unit
*p
= s
->code_ptr
;
246 memcpy(p
, &v
, sizeof(v
));
247 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
251 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
254 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
257 memcpy(p
, &v
, sizeof(v
));
262 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
263 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
265 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
268 tcg_insn_unit
*p
= s
->code_ptr
;
269 memcpy(p
, &v
, sizeof(v
));
270 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
274 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
277 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
280 memcpy(p
, &v
, sizeof(v
));
285 /* label relocation processing */
287 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
288 TCGLabel
*l
, intptr_t addend
)
290 TCGRelocation
*r
= tcg_malloc(sizeof(TCGRelocation
));
295 QSIMPLEQ_INSERT_TAIL(&l
->relocs
, r
, next
);
298 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
, tcg_insn_unit
*ptr
)
300 tcg_debug_assert(!l
->has_value
);
302 l
->u
.value_ptr
= ptr
;
305 TCGLabel
*gen_new_label(void)
307 TCGContext
*s
= tcg_ctx
;
308 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
310 memset(l
, 0, sizeof(TCGLabel
));
311 l
->id
= s
->nb_labels
++;
312 QSIMPLEQ_INIT(&l
->relocs
);
314 QSIMPLEQ_INSERT_TAIL(&s
->labels
, l
, next
);
319 static bool tcg_resolve_relocs(TCGContext
*s
)
323 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
325 uintptr_t value
= l
->u
.value
;
327 QSIMPLEQ_FOREACH(r
, &l
->relocs
, next
) {
328 if (!patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
)) {
336 static void set_jmp_reset_offset(TCGContext
*s
, int which
)
338 size_t off
= tcg_current_code_size(s
);
339 s
->tb_jmp_reset_offset
[which
] = off
;
340 /* Make sure that we didn't overflow the stored offset. */
341 assert(s
->tb_jmp_reset_offset
[which
] == off
);
344 #include "tcg-target.inc.c"
346 /* compare a pointer @ptr and a tb_tc @s */
347 static int ptr_cmp_tb_tc(const void *ptr
, const struct tb_tc
*s
)
349 if (ptr
>= s
->ptr
+ s
->size
) {
351 } else if (ptr
< s
->ptr
) {
357 static gint
tb_tc_cmp(gconstpointer ap
, gconstpointer bp
)
359 const struct tb_tc
*a
= ap
;
360 const struct tb_tc
*b
= bp
;
363 * When both sizes are set, we know this isn't a lookup.
364 * This is the most likely case: every TB must be inserted; lookups
365 * are a lot less frequent.
367 if (likely(a
->size
&& b
->size
)) {
368 if (a
->ptr
> b
->ptr
) {
370 } else if (a
->ptr
< b
->ptr
) {
373 /* a->ptr == b->ptr should happen only on deletions */
374 g_assert(a
->size
== b
->size
);
378 * All lookups have either .size field set to 0.
379 * From the glib sources we see that @ap is always the lookup key. However
380 * the docs provide no guarantee, so we just mark this case as likely.
382 if (likely(a
->size
== 0)) {
383 return ptr_cmp_tb_tc(a
->ptr
, b
);
385 return ptr_cmp_tb_tc(b
->ptr
, a
);
388 static void tcg_region_trees_init(void)
392 tree_size
= ROUND_UP(sizeof(struct tcg_region_tree
), qemu_dcache_linesize
);
393 region_trees
= qemu_memalign(qemu_dcache_linesize
, region
.n
* tree_size
);
394 for (i
= 0; i
< region
.n
; i
++) {
395 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
397 qemu_mutex_init(&rt
->lock
);
398 rt
->tree
= g_tree_new(tb_tc_cmp
);
402 static struct tcg_region_tree
*tc_ptr_to_region_tree(void *p
)
406 if (p
< region
.start_aligned
) {
409 ptrdiff_t offset
= p
- region
.start_aligned
;
411 if (offset
> region
.stride
* (region
.n
- 1)) {
412 region_idx
= region
.n
- 1;
414 region_idx
= offset
/ region
.stride
;
417 return region_trees
+ region_idx
* tree_size
;
420 void tcg_tb_insert(TranslationBlock
*tb
)
422 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
424 qemu_mutex_lock(&rt
->lock
);
425 g_tree_insert(rt
->tree
, &tb
->tc
, tb
);
426 qemu_mutex_unlock(&rt
->lock
);
429 void tcg_tb_remove(TranslationBlock
*tb
)
431 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
433 qemu_mutex_lock(&rt
->lock
);
434 g_tree_remove(rt
->tree
, &tb
->tc
);
435 qemu_mutex_unlock(&rt
->lock
);
439 * Find the TB 'tb' such that
440 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size
441 * Return NULL if not found.
443 TranslationBlock
*tcg_tb_lookup(uintptr_t tc_ptr
)
445 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree((void *)tc_ptr
);
446 TranslationBlock
*tb
;
447 struct tb_tc s
= { .ptr
= (void *)tc_ptr
};
449 qemu_mutex_lock(&rt
->lock
);
450 tb
= g_tree_lookup(rt
->tree
, &s
);
451 qemu_mutex_unlock(&rt
->lock
);
455 static void tcg_region_tree_lock_all(void)
459 for (i
= 0; i
< region
.n
; i
++) {
460 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
462 qemu_mutex_lock(&rt
->lock
);
466 static void tcg_region_tree_unlock_all(void)
470 for (i
= 0; i
< region
.n
; i
++) {
471 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
473 qemu_mutex_unlock(&rt
->lock
);
477 void tcg_tb_foreach(GTraverseFunc func
, gpointer user_data
)
481 tcg_region_tree_lock_all();
482 for (i
= 0; i
< region
.n
; i
++) {
483 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
485 g_tree_foreach(rt
->tree
, func
, user_data
);
487 tcg_region_tree_unlock_all();
490 size_t tcg_nb_tbs(void)
495 tcg_region_tree_lock_all();
496 for (i
= 0; i
< region
.n
; i
++) {
497 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
499 nb_tbs
+= g_tree_nnodes(rt
->tree
);
501 tcg_region_tree_unlock_all();
505 static void tcg_region_tree_reset_all(void)
509 tcg_region_tree_lock_all();
510 for (i
= 0; i
< region
.n
; i
++) {
511 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
513 /* Increment the refcount first so that destroy acts as a reset */
514 g_tree_ref(rt
->tree
);
515 g_tree_destroy(rt
->tree
);
517 tcg_region_tree_unlock_all();
520 static void tcg_region_bounds(size_t curr_region
, void **pstart
, void **pend
)
524 start
= region
.start_aligned
+ curr_region
* region
.stride
;
525 end
= start
+ region
.size
;
527 if (curr_region
== 0) {
528 start
= region
.start
;
530 if (curr_region
== region
.n
- 1) {
538 static void tcg_region_assign(TCGContext
*s
, size_t curr_region
)
542 tcg_region_bounds(curr_region
, &start
, &end
);
544 s
->code_gen_buffer
= start
;
545 s
->code_gen_ptr
= start
;
546 s
->code_gen_buffer_size
= end
- start
;
547 s
->code_gen_highwater
= end
- TCG_HIGHWATER
;
550 static bool tcg_region_alloc__locked(TCGContext
*s
)
552 if (region
.current
== region
.n
) {
555 tcg_region_assign(s
, region
.current
);
561 * Request a new region once the one in use has filled up.
562 * Returns true on error.
564 static bool tcg_region_alloc(TCGContext
*s
)
567 /* read the region size now; alloc__locked will overwrite it on success */
568 size_t size_full
= s
->code_gen_buffer_size
;
570 qemu_mutex_lock(®ion
.lock
);
571 err
= tcg_region_alloc__locked(s
);
573 region
.agg_size_full
+= size_full
- TCG_HIGHWATER
;
575 qemu_mutex_unlock(®ion
.lock
);
580 * Perform a context's first region allocation.
581 * This function does _not_ increment region.agg_size_full.
583 static inline bool tcg_region_initial_alloc__locked(TCGContext
*s
)
585 return tcg_region_alloc__locked(s
);
588 /* Call from a safe-work context */
589 void tcg_region_reset_all(void)
591 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
594 qemu_mutex_lock(®ion
.lock
);
596 region
.agg_size_full
= 0;
598 for (i
= 0; i
< n_ctxs
; i
++) {
599 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
600 bool err
= tcg_region_initial_alloc__locked(s
);
604 qemu_mutex_unlock(®ion
.lock
);
606 tcg_region_tree_reset_all();
609 #ifdef CONFIG_USER_ONLY
610 static size_t tcg_n_regions(void)
616 * It is likely that some vCPUs will translate more code than others, so we
617 * first try to set more regions than max_cpus, with those regions being of
618 * reasonable size. If that's not possible we make do by evenly dividing
619 * the code_gen_buffer among the vCPUs.
621 static size_t tcg_n_regions(void)
625 /* Use a single region if all we have is one vCPU thread */
626 #if !defined(CONFIG_USER_ONLY)
627 MachineState
*ms
= MACHINE(qdev_get_machine());
628 unsigned int max_cpus
= ms
->smp
.max_cpus
;
630 if (max_cpus
== 1 || !qemu_tcg_mttcg_enabled()) {
634 /* Try to have more regions than max_cpus, with each region being >= 2 MB */
635 for (i
= 8; i
> 0; i
--) {
636 size_t regions_per_thread
= i
;
639 region_size
= tcg_init_ctx
.code_gen_buffer_size
;
640 region_size
/= max_cpus
* regions_per_thread
;
642 if (region_size
>= 2 * 1024u * 1024) {
643 return max_cpus
* regions_per_thread
;
646 /* If we can't, then just allocate one region per vCPU thread */
652 * Initializes region partitioning.
654 * Called at init time from the parent thread (i.e. the one calling
655 * tcg_context_init), after the target's TCG globals have been set.
657 * Region partitioning works by splitting code_gen_buffer into separate regions,
658 * and then assigning regions to TCG threads so that the threads can translate
659 * code in parallel without synchronization.
661 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at
662 * least max_cpus regions in MTTCG. In !MTTCG we use a single region.
663 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...])
664 * must have been parsed before calling this function, since it calls
665 * qemu_tcg_mttcg_enabled().
667 * In user-mode we use a single region. Having multiple regions in user-mode
668 * is not supported, because the number of vCPU threads (recall that each thread
669 * spawned by the guest corresponds to a vCPU thread) is only bounded by the
670 * OS, and usually this number is huge (tens of thousands is not uncommon).
671 * Thus, given this large bound on the number of vCPU threads and the fact
672 * that code_gen_buffer is allocated at compile-time, we cannot guarantee
673 * that the availability of at least one region per vCPU thread.
675 * However, this user-mode limitation is unlikely to be a significant problem
676 * in practice. Multi-threaded guests share most if not all of their translated
677 * code, which makes parallel code generation less appealing than in softmmu.
679 void tcg_region_init(void)
681 void *buf
= tcg_init_ctx
.code_gen_buffer
;
683 size_t size
= tcg_init_ctx
.code_gen_buffer_size
;
684 size_t page_size
= qemu_real_host_page_size
;
689 n_regions
= tcg_n_regions();
691 /* The first region will be 'aligned - buf' bytes larger than the others */
692 aligned
= QEMU_ALIGN_PTR_UP(buf
, page_size
);
693 g_assert(aligned
< tcg_init_ctx
.code_gen_buffer
+ size
);
695 * Make region_size a multiple of page_size, using aligned as the start.
696 * As a result of this we might end up with a few extra pages at the end of
697 * the buffer; we will assign those to the last region.
699 region_size
= (size
- (aligned
- buf
)) / n_regions
;
700 region_size
= QEMU_ALIGN_DOWN(region_size
, page_size
);
702 /* A region must have at least 2 pages; one code, one guard */
703 g_assert(region_size
>= 2 * page_size
);
705 /* init the region struct */
706 qemu_mutex_init(®ion
.lock
);
707 region
.n
= n_regions
;
708 region
.size
= region_size
- page_size
;
709 region
.stride
= region_size
;
711 region
.start_aligned
= aligned
;
712 /* page-align the end, since its last page will be a guard page */
713 region
.end
= QEMU_ALIGN_PTR_DOWN(buf
+ size
, page_size
);
714 /* account for that last guard page */
715 region
.end
-= page_size
;
717 /* set guard pages */
718 for (i
= 0; i
< region
.n
; i
++) {
722 tcg_region_bounds(i
, &start
, &end
);
723 rc
= qemu_mprotect_none(end
, page_size
);
727 tcg_region_trees_init();
729 /* In user-mode we support only one ctx, so do the initial allocation now */
730 #ifdef CONFIG_USER_ONLY
732 bool err
= tcg_region_initial_alloc__locked(tcg_ctx
);
739 static void alloc_tcg_plugin_context(TCGContext
*s
)
742 s
->plugin_tb
= g_new0(struct qemu_plugin_tb
, 1);
743 s
->plugin_tb
->insns
=
744 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn
);
749 * All TCG threads except the parent (i.e. the one that called tcg_context_init
750 * and registered the target's TCG globals) must register with this function
751 * before initiating translation.
753 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
754 * of tcg_region_init() for the reasoning behind this.
756 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
757 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
758 * is not used anymore for translation once this function is called.
760 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
761 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
763 #ifdef CONFIG_USER_ONLY
764 void tcg_register_thread(void)
766 tcg_ctx
= &tcg_init_ctx
;
769 void tcg_register_thread(void)
771 MachineState
*ms
= MACHINE(qdev_get_machine());
772 TCGContext
*s
= g_malloc(sizeof(*s
));
778 /* Relink mem_base. */
779 for (i
= 0, n
= tcg_init_ctx
.nb_globals
; i
< n
; ++i
) {
780 if (tcg_init_ctx
.temps
[i
].mem_base
) {
781 ptrdiff_t b
= tcg_init_ctx
.temps
[i
].mem_base
- tcg_init_ctx
.temps
;
782 tcg_debug_assert(b
>= 0 && b
< n
);
783 s
->temps
[i
].mem_base
= &s
->temps
[b
];
787 /* Claim an entry in tcg_ctxs */
788 n
= atomic_fetch_inc(&n_tcg_ctxs
);
789 g_assert(n
< ms
->smp
.max_cpus
);
790 atomic_set(&tcg_ctxs
[n
], s
);
793 alloc_tcg_plugin_context(s
);
797 qemu_mutex_lock(®ion
.lock
);
798 err
= tcg_region_initial_alloc__locked(tcg_ctx
);
800 qemu_mutex_unlock(®ion
.lock
);
802 #endif /* !CONFIG_USER_ONLY */
805 * Returns the size (in bytes) of all translated code (i.e. from all regions)
806 * currently in the cache.
807 * See also: tcg_code_capacity()
808 * Do not confuse with tcg_current_code_size(); that one applies to a single
811 size_t tcg_code_size(void)
813 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
817 qemu_mutex_lock(®ion
.lock
);
818 total
= region
.agg_size_full
;
819 for (i
= 0; i
< n_ctxs
; i
++) {
820 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
823 size
= atomic_read(&s
->code_gen_ptr
) - s
->code_gen_buffer
;
824 g_assert(size
<= s
->code_gen_buffer_size
);
827 qemu_mutex_unlock(®ion
.lock
);
832 * Returns the code capacity (in bytes) of the entire cache, i.e. including all
834 * See also: tcg_code_size()
836 size_t tcg_code_capacity(void)
838 size_t guard_size
, capacity
;
840 /* no need for synchronization; these variables are set at init time */
841 guard_size
= region
.stride
- region
.size
;
842 capacity
= region
.end
+ guard_size
- region
.start
;
843 capacity
-= region
.n
* (guard_size
+ TCG_HIGHWATER
);
847 size_t tcg_tb_phys_invalidate_count(void)
849 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
853 for (i
= 0; i
< n_ctxs
; i
++) {
854 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
856 total
+= atomic_read(&s
->tb_phys_invalidate_count
);
861 /* pool based memory allocation */
862 void *tcg_malloc_internal(TCGContext
*s
, int size
)
867 if (size
> TCG_POOL_CHUNK_SIZE
) {
868 /* big malloc: insert a new pool (XXX: could optimize) */
869 p
= g_malloc(sizeof(TCGPool
) + size
);
871 p
->next
= s
->pool_first_large
;
872 s
->pool_first_large
= p
;
883 pool_size
= TCG_POOL_CHUNK_SIZE
;
884 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
888 s
->pool_current
->next
= p
;
897 s
->pool_cur
= p
->data
+ size
;
898 s
->pool_end
= p
->data
+ p
->size
;
902 void tcg_pool_reset(TCGContext
*s
)
905 for (p
= s
->pool_first_large
; p
; p
= t
) {
909 s
->pool_first_large
= NULL
;
910 s
->pool_cur
= s
->pool_end
= NULL
;
911 s
->pool_current
= NULL
;
914 typedef struct TCGHelperInfo
{
921 #include "exec/helper-proto.h"
923 static const TCGHelperInfo all_helpers
[] = {
924 #include "exec/helper-tcg.h"
926 static GHashTable
*helper_table
;
928 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
929 static void process_op_defs(TCGContext
*s
);
930 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
931 TCGReg reg
, const char *name
);
933 void tcg_context_init(TCGContext
*s
)
935 int op
, total_args
, n
, i
;
937 TCGArgConstraint
*args_ct
;
941 memset(s
, 0, sizeof(*s
));
944 /* Count total number of arguments and allocate the corresponding
947 for(op
= 0; op
< NB_OPS
; op
++) {
948 def
= &tcg_op_defs
[op
];
949 n
= def
->nb_iargs
+ def
->nb_oargs
;
953 args_ct
= g_malloc(sizeof(TCGArgConstraint
) * total_args
);
954 sorted_args
= g_malloc(sizeof(int) * total_args
);
956 for(op
= 0; op
< NB_OPS
; op
++) {
957 def
= &tcg_op_defs
[op
];
958 def
->args_ct
= args_ct
;
959 def
->sorted_args
= sorted_args
;
960 n
= def
->nb_iargs
+ def
->nb_oargs
;
965 /* Register helpers. */
966 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
967 helper_table
= g_hash_table_new(NULL
, NULL
);
969 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
970 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
971 (gpointer
)&all_helpers
[i
]);
977 /* Reverse the order of the saved registers, assuming they're all at
978 the start of tcg_target_reg_alloc_order. */
979 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
980 int r
= tcg_target_reg_alloc_order
[n
];
981 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
985 for (i
= 0; i
< n
; ++i
) {
986 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
988 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
989 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
992 alloc_tcg_plugin_context(s
);
996 * In user-mode we simply share the init context among threads, since we
997 * use a single region. See the documentation tcg_region_init() for the
998 * reasoning behind this.
999 * In softmmu we will have at most max_cpus TCG threads.
1001 #ifdef CONFIG_USER_ONLY
1002 tcg_ctxs
= &tcg_ctx
;
1005 MachineState
*ms
= MACHINE(qdev_get_machine());
1006 unsigned int max_cpus
= ms
->smp
.max_cpus
;
1007 tcg_ctxs
= g_new(TCGContext
*, max_cpus
);
1010 tcg_debug_assert(!tcg_regset_test_reg(s
->reserved_regs
, TCG_AREG0
));
1011 ts
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, TCG_AREG0
, "env");
1012 cpu_env
= temp_tcgv_ptr(ts
);
1016 * Allocate TBs right before their corresponding translated code, making
1017 * sure that TBs and code are on different cache lines.
1019 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
1021 uintptr_t align
= qemu_icache_linesize
;
1022 TranslationBlock
*tb
;
1026 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
1027 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
1029 if (unlikely(next
> s
->code_gen_highwater
)) {
1030 if (tcg_region_alloc(s
)) {
1035 atomic_set(&s
->code_gen_ptr
, next
);
1036 s
->data_gen_ptr
= NULL
;
1040 void tcg_prologue_init(TCGContext
*s
)
1042 size_t prologue_size
, total_size
;
1045 /* Put the prologue at the beginning of code_gen_buffer. */
1046 buf0
= s
->code_gen_buffer
;
1047 total_size
= s
->code_gen_buffer_size
;
1050 s
->data_gen_ptr
= NULL
;
1051 s
->code_gen_prologue
= buf0
;
1053 /* Compute a high-water mark, at which we voluntarily flush the buffer
1054 and start over. The size here is arbitrary, significantly larger
1055 than we expect the code generation for any one opcode to require. */
1056 s
->code_gen_highwater
= s
->code_gen_buffer
+ (total_size
- TCG_HIGHWATER
);
1058 #ifdef TCG_TARGET_NEED_POOL_LABELS
1059 s
->pool_labels
= NULL
;
1062 /* Generate the prologue. */
1063 tcg_target_qemu_prologue(s
);
1065 #ifdef TCG_TARGET_NEED_POOL_LABELS
1066 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1068 int result
= tcg_out_pool_finalize(s
);
1069 tcg_debug_assert(result
== 0);
1074 flush_icache_range((uintptr_t)buf0
, (uintptr_t)buf1
);
1076 /* Deduct the prologue from the buffer. */
1077 prologue_size
= tcg_current_code_size(s
);
1078 s
->code_gen_ptr
= buf1
;
1079 s
->code_gen_buffer
= buf1
;
1081 total_size
-= prologue_size
;
1082 s
->code_gen_buffer_size
= total_size
;
1084 tcg_register_jit(s
->code_gen_buffer
, total_size
);
1087 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
1088 FILE *logfile
= qemu_log_lock();
1089 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size
);
1090 if (s
->data_gen_ptr
) {
1091 size_t code_size
= s
->data_gen_ptr
- buf0
;
1092 size_t data_size
= prologue_size
- code_size
;
1095 log_disas(buf0
, code_size
, NULL
);
1097 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1098 if (sizeof(tcg_target_ulong
) == 8) {
1099 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1100 (uintptr_t)s
->data_gen_ptr
+ i
,
1101 *(uint64_t *)(s
->data_gen_ptr
+ i
));
1103 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1104 (uintptr_t)s
->data_gen_ptr
+ i
,
1105 *(uint32_t *)(s
->data_gen_ptr
+ i
));
1109 log_disas(buf0
, prologue_size
, NULL
);
1113 qemu_log_unlock(logfile
);
1117 /* Assert that goto_ptr is implemented completely. */
1118 if (TCG_TARGET_HAS_goto_ptr
) {
1119 tcg_debug_assert(s
->code_gen_epilogue
!= NULL
);
1123 void tcg_func_start(TCGContext
*s
)
1126 s
->nb_temps
= s
->nb_globals
;
1128 /* No temps have been previously allocated for size or locality. */
1129 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
1133 s
->current_frame_offset
= s
->frame_start
;
1135 #ifdef CONFIG_DEBUG_TCG
1136 s
->goto_tb_issue_mask
= 0;
1139 QTAILQ_INIT(&s
->ops
);
1140 QTAILQ_INIT(&s
->free_ops
);
1141 QSIMPLEQ_INIT(&s
->labels
);
1144 static inline TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
1146 int n
= s
->nb_temps
++;
1147 tcg_debug_assert(n
< TCG_MAX_TEMPS
);
1148 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
1151 static inline TCGTemp
*tcg_global_alloc(TCGContext
*s
)
1155 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
1157 ts
= tcg_temp_alloc(s
);
1158 ts
->temp_global
= 1;
1163 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
1164 TCGReg reg
, const char *name
)
1168 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
1172 ts
= tcg_global_alloc(s
);
1173 ts
->base_type
= type
;
1178 tcg_regset_set_reg(s
->reserved_regs
, reg
);
1183 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
1185 s
->frame_start
= start
;
1186 s
->frame_end
= start
+ size
;
1188 = tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
1191 TCGTemp
*tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
1192 intptr_t offset
, const char *name
)
1194 TCGContext
*s
= tcg_ctx
;
1195 TCGTemp
*base_ts
= tcgv_ptr_temp(base
);
1196 TCGTemp
*ts
= tcg_global_alloc(s
);
1197 int indirect_reg
= 0, bigendian
= 0;
1198 #ifdef HOST_WORDS_BIGENDIAN
1202 if (!base_ts
->fixed_reg
) {
1203 /* We do not support double-indirect registers. */
1204 tcg_debug_assert(!base_ts
->indirect_reg
);
1205 base_ts
->indirect_base
= 1;
1206 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
1211 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1212 TCGTemp
*ts2
= tcg_global_alloc(s
);
1215 ts
->base_type
= TCG_TYPE_I64
;
1216 ts
->type
= TCG_TYPE_I32
;
1217 ts
->indirect_reg
= indirect_reg
;
1218 ts
->mem_allocated
= 1;
1219 ts
->mem_base
= base_ts
;
1220 ts
->mem_offset
= offset
+ bigendian
* 4;
1221 pstrcpy(buf
, sizeof(buf
), name
);
1222 pstrcat(buf
, sizeof(buf
), "_0");
1223 ts
->name
= strdup(buf
);
1225 tcg_debug_assert(ts2
== ts
+ 1);
1226 ts2
->base_type
= TCG_TYPE_I64
;
1227 ts2
->type
= TCG_TYPE_I32
;
1228 ts2
->indirect_reg
= indirect_reg
;
1229 ts2
->mem_allocated
= 1;
1230 ts2
->mem_base
= base_ts
;
1231 ts2
->mem_offset
= offset
+ (1 - bigendian
) * 4;
1232 pstrcpy(buf
, sizeof(buf
), name
);
1233 pstrcat(buf
, sizeof(buf
), "_1");
1234 ts2
->name
= strdup(buf
);
1236 ts
->base_type
= type
;
1238 ts
->indirect_reg
= indirect_reg
;
1239 ts
->mem_allocated
= 1;
1240 ts
->mem_base
= base_ts
;
1241 ts
->mem_offset
= offset
;
1247 TCGTemp
*tcg_temp_new_internal(TCGType type
, bool temp_local
)
1249 TCGContext
*s
= tcg_ctx
;
1253 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
1254 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
1255 if (idx
< TCG_MAX_TEMPS
) {
1256 /* There is already an available temp with the right type. */
1257 clear_bit(idx
, s
->free_temps
[k
].l
);
1259 ts
= &s
->temps
[idx
];
1260 ts
->temp_allocated
= 1;
1261 tcg_debug_assert(ts
->base_type
== type
);
1262 tcg_debug_assert(ts
->temp_local
== temp_local
);
1264 ts
= tcg_temp_alloc(s
);
1265 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1266 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1268 ts
->base_type
= type
;
1269 ts
->type
= TCG_TYPE_I32
;
1270 ts
->temp_allocated
= 1;
1271 ts
->temp_local
= temp_local
;
1273 tcg_debug_assert(ts2
== ts
+ 1);
1274 ts2
->base_type
= TCG_TYPE_I64
;
1275 ts2
->type
= TCG_TYPE_I32
;
1276 ts2
->temp_allocated
= 1;
1277 ts2
->temp_local
= temp_local
;
1279 ts
->base_type
= type
;
1281 ts
->temp_allocated
= 1;
1282 ts
->temp_local
= temp_local
;
1286 #if defined(CONFIG_DEBUG_TCG)
1292 TCGv_vec
tcg_temp_new_vec(TCGType type
)
1296 #ifdef CONFIG_DEBUG_TCG
1299 assert(TCG_TARGET_HAS_v64
);
1302 assert(TCG_TARGET_HAS_v128
);
1305 assert(TCG_TARGET_HAS_v256
);
1308 g_assert_not_reached();
1312 t
= tcg_temp_new_internal(type
, 0);
1313 return temp_tcgv_vec(t
);
1316 /* Create a new temp of the same type as an existing temp. */
1317 TCGv_vec
tcg_temp_new_vec_matching(TCGv_vec match
)
1319 TCGTemp
*t
= tcgv_vec_temp(match
);
1321 tcg_debug_assert(t
->temp_allocated
!= 0);
1323 t
= tcg_temp_new_internal(t
->base_type
, 0);
1324 return temp_tcgv_vec(t
);
1327 void tcg_temp_free_internal(TCGTemp
*ts
)
1329 TCGContext
*s
= tcg_ctx
;
1332 #if defined(CONFIG_DEBUG_TCG)
1334 if (s
->temps_in_use
< 0) {
1335 fprintf(stderr
, "More temporaries freed than allocated!\n");
1339 tcg_debug_assert(ts
->temp_global
== 0);
1340 tcg_debug_assert(ts
->temp_allocated
!= 0);
1341 ts
->temp_allocated
= 0;
1344 k
= ts
->base_type
+ (ts
->temp_local
? TCG_TYPE_COUNT
: 0);
1345 set_bit(idx
, s
->free_temps
[k
].l
);
1348 TCGv_i32
tcg_const_i32(int32_t val
)
1351 t0
= tcg_temp_new_i32();
1352 tcg_gen_movi_i32(t0
, val
);
1356 TCGv_i64
tcg_const_i64(int64_t val
)
1359 t0
= tcg_temp_new_i64();
1360 tcg_gen_movi_i64(t0
, val
);
1364 TCGv_i32
tcg_const_local_i32(int32_t val
)
1367 t0
= tcg_temp_local_new_i32();
1368 tcg_gen_movi_i32(t0
, val
);
1372 TCGv_i64
tcg_const_local_i64(int64_t val
)
1375 t0
= tcg_temp_local_new_i64();
1376 tcg_gen_movi_i64(t0
, val
);
1380 #if defined(CONFIG_DEBUG_TCG)
1381 void tcg_clear_temp_count(void)
1383 TCGContext
*s
= tcg_ctx
;
1384 s
->temps_in_use
= 0;
1387 int tcg_check_temp_count(void)
1389 TCGContext
*s
= tcg_ctx
;
1390 if (s
->temps_in_use
) {
1391 /* Clear the count so that we don't give another
1392 * warning immediately next time around.
1394 s
->temps_in_use
= 0;
1401 /* Return true if OP may appear in the opcode stream.
1402 Test the runtime variable that controls each opcode. */
1403 bool tcg_op_supported(TCGOpcode op
)
1406 = TCG_TARGET_HAS_v64
| TCG_TARGET_HAS_v128
| TCG_TARGET_HAS_v256
;
1409 case INDEX_op_discard
:
1410 case INDEX_op_set_label
:
1414 case INDEX_op_insn_start
:
1415 case INDEX_op_exit_tb
:
1416 case INDEX_op_goto_tb
:
1417 case INDEX_op_qemu_ld_i32
:
1418 case INDEX_op_qemu_st_i32
:
1419 case INDEX_op_qemu_ld_i64
:
1420 case INDEX_op_qemu_st_i64
:
1423 case INDEX_op_goto_ptr
:
1424 return TCG_TARGET_HAS_goto_ptr
;
1426 case INDEX_op_mov_i32
:
1427 case INDEX_op_movi_i32
:
1428 case INDEX_op_setcond_i32
:
1429 case INDEX_op_brcond_i32
:
1430 case INDEX_op_ld8u_i32
:
1431 case INDEX_op_ld8s_i32
:
1432 case INDEX_op_ld16u_i32
:
1433 case INDEX_op_ld16s_i32
:
1434 case INDEX_op_ld_i32
:
1435 case INDEX_op_st8_i32
:
1436 case INDEX_op_st16_i32
:
1437 case INDEX_op_st_i32
:
1438 case INDEX_op_add_i32
:
1439 case INDEX_op_sub_i32
:
1440 case INDEX_op_mul_i32
:
1441 case INDEX_op_and_i32
:
1442 case INDEX_op_or_i32
:
1443 case INDEX_op_xor_i32
:
1444 case INDEX_op_shl_i32
:
1445 case INDEX_op_shr_i32
:
1446 case INDEX_op_sar_i32
:
1449 case INDEX_op_movcond_i32
:
1450 return TCG_TARGET_HAS_movcond_i32
;
1451 case INDEX_op_div_i32
:
1452 case INDEX_op_divu_i32
:
1453 return TCG_TARGET_HAS_div_i32
;
1454 case INDEX_op_rem_i32
:
1455 case INDEX_op_remu_i32
:
1456 return TCG_TARGET_HAS_rem_i32
;
1457 case INDEX_op_div2_i32
:
1458 case INDEX_op_divu2_i32
:
1459 return TCG_TARGET_HAS_div2_i32
;
1460 case INDEX_op_rotl_i32
:
1461 case INDEX_op_rotr_i32
:
1462 return TCG_TARGET_HAS_rot_i32
;
1463 case INDEX_op_deposit_i32
:
1464 return TCG_TARGET_HAS_deposit_i32
;
1465 case INDEX_op_extract_i32
:
1466 return TCG_TARGET_HAS_extract_i32
;
1467 case INDEX_op_sextract_i32
:
1468 return TCG_TARGET_HAS_sextract_i32
;
1469 case INDEX_op_extract2_i32
:
1470 return TCG_TARGET_HAS_extract2_i32
;
1471 case INDEX_op_add2_i32
:
1472 return TCG_TARGET_HAS_add2_i32
;
1473 case INDEX_op_sub2_i32
:
1474 return TCG_TARGET_HAS_sub2_i32
;
1475 case INDEX_op_mulu2_i32
:
1476 return TCG_TARGET_HAS_mulu2_i32
;
1477 case INDEX_op_muls2_i32
:
1478 return TCG_TARGET_HAS_muls2_i32
;
1479 case INDEX_op_muluh_i32
:
1480 return TCG_TARGET_HAS_muluh_i32
;
1481 case INDEX_op_mulsh_i32
:
1482 return TCG_TARGET_HAS_mulsh_i32
;
1483 case INDEX_op_ext8s_i32
:
1484 return TCG_TARGET_HAS_ext8s_i32
;
1485 case INDEX_op_ext16s_i32
:
1486 return TCG_TARGET_HAS_ext16s_i32
;
1487 case INDEX_op_ext8u_i32
:
1488 return TCG_TARGET_HAS_ext8u_i32
;
1489 case INDEX_op_ext16u_i32
:
1490 return TCG_TARGET_HAS_ext16u_i32
;
1491 case INDEX_op_bswap16_i32
:
1492 return TCG_TARGET_HAS_bswap16_i32
;
1493 case INDEX_op_bswap32_i32
:
1494 return TCG_TARGET_HAS_bswap32_i32
;
1495 case INDEX_op_not_i32
:
1496 return TCG_TARGET_HAS_not_i32
;
1497 case INDEX_op_neg_i32
:
1498 return TCG_TARGET_HAS_neg_i32
;
1499 case INDEX_op_andc_i32
:
1500 return TCG_TARGET_HAS_andc_i32
;
1501 case INDEX_op_orc_i32
:
1502 return TCG_TARGET_HAS_orc_i32
;
1503 case INDEX_op_eqv_i32
:
1504 return TCG_TARGET_HAS_eqv_i32
;
1505 case INDEX_op_nand_i32
:
1506 return TCG_TARGET_HAS_nand_i32
;
1507 case INDEX_op_nor_i32
:
1508 return TCG_TARGET_HAS_nor_i32
;
1509 case INDEX_op_clz_i32
:
1510 return TCG_TARGET_HAS_clz_i32
;
1511 case INDEX_op_ctz_i32
:
1512 return TCG_TARGET_HAS_ctz_i32
;
1513 case INDEX_op_ctpop_i32
:
1514 return TCG_TARGET_HAS_ctpop_i32
;
1516 case INDEX_op_brcond2_i32
:
1517 case INDEX_op_setcond2_i32
:
1518 return TCG_TARGET_REG_BITS
== 32;
1520 case INDEX_op_mov_i64
:
1521 case INDEX_op_movi_i64
:
1522 case INDEX_op_setcond_i64
:
1523 case INDEX_op_brcond_i64
:
1524 case INDEX_op_ld8u_i64
:
1525 case INDEX_op_ld8s_i64
:
1526 case INDEX_op_ld16u_i64
:
1527 case INDEX_op_ld16s_i64
:
1528 case INDEX_op_ld32u_i64
:
1529 case INDEX_op_ld32s_i64
:
1530 case INDEX_op_ld_i64
:
1531 case INDEX_op_st8_i64
:
1532 case INDEX_op_st16_i64
:
1533 case INDEX_op_st32_i64
:
1534 case INDEX_op_st_i64
:
1535 case INDEX_op_add_i64
:
1536 case INDEX_op_sub_i64
:
1537 case INDEX_op_mul_i64
:
1538 case INDEX_op_and_i64
:
1539 case INDEX_op_or_i64
:
1540 case INDEX_op_xor_i64
:
1541 case INDEX_op_shl_i64
:
1542 case INDEX_op_shr_i64
:
1543 case INDEX_op_sar_i64
:
1544 case INDEX_op_ext_i32_i64
:
1545 case INDEX_op_extu_i32_i64
:
1546 return TCG_TARGET_REG_BITS
== 64;
1548 case INDEX_op_movcond_i64
:
1549 return TCG_TARGET_HAS_movcond_i64
;
1550 case INDEX_op_div_i64
:
1551 case INDEX_op_divu_i64
:
1552 return TCG_TARGET_HAS_div_i64
;
1553 case INDEX_op_rem_i64
:
1554 case INDEX_op_remu_i64
:
1555 return TCG_TARGET_HAS_rem_i64
;
1556 case INDEX_op_div2_i64
:
1557 case INDEX_op_divu2_i64
:
1558 return TCG_TARGET_HAS_div2_i64
;
1559 case INDEX_op_rotl_i64
:
1560 case INDEX_op_rotr_i64
:
1561 return TCG_TARGET_HAS_rot_i64
;
1562 case INDEX_op_deposit_i64
:
1563 return TCG_TARGET_HAS_deposit_i64
;
1564 case INDEX_op_extract_i64
:
1565 return TCG_TARGET_HAS_extract_i64
;
1566 case INDEX_op_sextract_i64
:
1567 return TCG_TARGET_HAS_sextract_i64
;
1568 case INDEX_op_extract2_i64
:
1569 return TCG_TARGET_HAS_extract2_i64
;
1570 case INDEX_op_extrl_i64_i32
:
1571 return TCG_TARGET_HAS_extrl_i64_i32
;
1572 case INDEX_op_extrh_i64_i32
:
1573 return TCG_TARGET_HAS_extrh_i64_i32
;
1574 case INDEX_op_ext8s_i64
:
1575 return TCG_TARGET_HAS_ext8s_i64
;
1576 case INDEX_op_ext16s_i64
:
1577 return TCG_TARGET_HAS_ext16s_i64
;
1578 case INDEX_op_ext32s_i64
:
1579 return TCG_TARGET_HAS_ext32s_i64
;
1580 case INDEX_op_ext8u_i64
:
1581 return TCG_TARGET_HAS_ext8u_i64
;
1582 case INDEX_op_ext16u_i64
:
1583 return TCG_TARGET_HAS_ext16u_i64
;
1584 case INDEX_op_ext32u_i64
:
1585 return TCG_TARGET_HAS_ext32u_i64
;
1586 case INDEX_op_bswap16_i64
:
1587 return TCG_TARGET_HAS_bswap16_i64
;
1588 case INDEX_op_bswap32_i64
:
1589 return TCG_TARGET_HAS_bswap32_i64
;
1590 case INDEX_op_bswap64_i64
:
1591 return TCG_TARGET_HAS_bswap64_i64
;
1592 case INDEX_op_not_i64
:
1593 return TCG_TARGET_HAS_not_i64
;
1594 case INDEX_op_neg_i64
:
1595 return TCG_TARGET_HAS_neg_i64
;
1596 case INDEX_op_andc_i64
:
1597 return TCG_TARGET_HAS_andc_i64
;
1598 case INDEX_op_orc_i64
:
1599 return TCG_TARGET_HAS_orc_i64
;
1600 case INDEX_op_eqv_i64
:
1601 return TCG_TARGET_HAS_eqv_i64
;
1602 case INDEX_op_nand_i64
:
1603 return TCG_TARGET_HAS_nand_i64
;
1604 case INDEX_op_nor_i64
:
1605 return TCG_TARGET_HAS_nor_i64
;
1606 case INDEX_op_clz_i64
:
1607 return TCG_TARGET_HAS_clz_i64
;
1608 case INDEX_op_ctz_i64
:
1609 return TCG_TARGET_HAS_ctz_i64
;
1610 case INDEX_op_ctpop_i64
:
1611 return TCG_TARGET_HAS_ctpop_i64
;
1612 case INDEX_op_add2_i64
:
1613 return TCG_TARGET_HAS_add2_i64
;
1614 case INDEX_op_sub2_i64
:
1615 return TCG_TARGET_HAS_sub2_i64
;
1616 case INDEX_op_mulu2_i64
:
1617 return TCG_TARGET_HAS_mulu2_i64
;
1618 case INDEX_op_muls2_i64
:
1619 return TCG_TARGET_HAS_muls2_i64
;
1620 case INDEX_op_muluh_i64
:
1621 return TCG_TARGET_HAS_muluh_i64
;
1622 case INDEX_op_mulsh_i64
:
1623 return TCG_TARGET_HAS_mulsh_i64
;
1625 case INDEX_op_mov_vec
:
1626 case INDEX_op_dup_vec
:
1627 case INDEX_op_dupi_vec
:
1628 case INDEX_op_dupm_vec
:
1629 case INDEX_op_ld_vec
:
1630 case INDEX_op_st_vec
:
1631 case INDEX_op_add_vec
:
1632 case INDEX_op_sub_vec
:
1633 case INDEX_op_and_vec
:
1634 case INDEX_op_or_vec
:
1635 case INDEX_op_xor_vec
:
1636 case INDEX_op_cmp_vec
:
1638 case INDEX_op_dup2_vec
:
1639 return have_vec
&& TCG_TARGET_REG_BITS
== 32;
1640 case INDEX_op_not_vec
:
1641 return have_vec
&& TCG_TARGET_HAS_not_vec
;
1642 case INDEX_op_neg_vec
:
1643 return have_vec
&& TCG_TARGET_HAS_neg_vec
;
1644 case INDEX_op_abs_vec
:
1645 return have_vec
&& TCG_TARGET_HAS_abs_vec
;
1646 case INDEX_op_andc_vec
:
1647 return have_vec
&& TCG_TARGET_HAS_andc_vec
;
1648 case INDEX_op_orc_vec
:
1649 return have_vec
&& TCG_TARGET_HAS_orc_vec
;
1650 case INDEX_op_mul_vec
:
1651 return have_vec
&& TCG_TARGET_HAS_mul_vec
;
1652 case INDEX_op_shli_vec
:
1653 case INDEX_op_shri_vec
:
1654 case INDEX_op_sari_vec
:
1655 return have_vec
&& TCG_TARGET_HAS_shi_vec
;
1656 case INDEX_op_shls_vec
:
1657 case INDEX_op_shrs_vec
:
1658 case INDEX_op_sars_vec
:
1659 return have_vec
&& TCG_TARGET_HAS_shs_vec
;
1660 case INDEX_op_shlv_vec
:
1661 case INDEX_op_shrv_vec
:
1662 case INDEX_op_sarv_vec
:
1663 return have_vec
&& TCG_TARGET_HAS_shv_vec
;
1664 case INDEX_op_rotli_vec
:
1665 return have_vec
&& TCG_TARGET_HAS_roti_vec
;
1666 case INDEX_op_ssadd_vec
:
1667 case INDEX_op_usadd_vec
:
1668 case INDEX_op_sssub_vec
:
1669 case INDEX_op_ussub_vec
:
1670 return have_vec
&& TCG_TARGET_HAS_sat_vec
;
1671 case INDEX_op_smin_vec
:
1672 case INDEX_op_umin_vec
:
1673 case INDEX_op_smax_vec
:
1674 case INDEX_op_umax_vec
:
1675 return have_vec
&& TCG_TARGET_HAS_minmax_vec
;
1676 case INDEX_op_bitsel_vec
:
1677 return have_vec
&& TCG_TARGET_HAS_bitsel_vec
;
1678 case INDEX_op_cmpsel_vec
:
1679 return have_vec
&& TCG_TARGET_HAS_cmpsel_vec
;
1682 tcg_debug_assert(op
> INDEX_op_last_generic
&& op
< NB_OPS
);
1687 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1688 and endian swap. Maybe it would be better to do the alignment
1689 and endian swap in tcg_reg_alloc_call(). */
1690 void tcg_gen_callN(void *func
, TCGTemp
*ret
, int nargs
, TCGTemp
**args
)
1692 int i
, real_args
, nb_rets
, pi
;
1693 unsigned sizemask
, flags
;
1694 TCGHelperInfo
*info
;
1697 info
= g_hash_table_lookup(helper_table
, (gpointer
)func
);
1698 flags
= info
->flags
;
1699 sizemask
= info
->sizemask
;
1701 #ifdef CONFIG_PLUGIN
1702 /* detect non-plugin helpers */
1703 if (tcg_ctx
->plugin_insn
&& unlikely(strncmp(info
->name
, "plugin_", 7))) {
1704 tcg_ctx
->plugin_insn
->calls_helpers
= true;
1708 #if defined(__sparc__) && !defined(__arch64__) \
1709 && !defined(CONFIG_TCG_INTERPRETER)
1710 /* We have 64-bit values in one register, but need to pass as two
1711 separate parameters. Split them. */
1712 int orig_sizemask
= sizemask
;
1713 int orig_nargs
= nargs
;
1714 TCGv_i64 retl
, reth
;
1715 TCGTemp
*split_args
[MAX_OPC_PARAM
];
1719 if (sizemask
!= 0) {
1720 for (i
= real_args
= 0; i
< nargs
; ++i
) {
1721 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1723 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1724 TCGv_i32 h
= tcg_temp_new_i32();
1725 TCGv_i32 l
= tcg_temp_new_i32();
1726 tcg_gen_extr_i64_i32(l
, h
, orig
);
1727 split_args
[real_args
++] = tcgv_i32_temp(h
);
1728 split_args
[real_args
++] = tcgv_i32_temp(l
);
1730 split_args
[real_args
++] = args
[i
];
1737 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1738 for (i
= 0; i
< nargs
; ++i
) {
1739 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1740 int is_signed
= sizemask
& (2 << (i
+1)*2);
1742 TCGv_i64 temp
= tcg_temp_new_i64();
1743 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1745 tcg_gen_ext32s_i64(temp
, orig
);
1747 tcg_gen_ext32u_i64(temp
, orig
);
1749 args
[i
] = tcgv_i64_temp(temp
);
1752 #endif /* TCG_TARGET_EXTEND_ARGS */
1754 op
= tcg_emit_op(INDEX_op_call
);
1758 #if defined(__sparc__) && !defined(__arch64__) \
1759 && !defined(CONFIG_TCG_INTERPRETER)
1760 if (orig_sizemask
& 1) {
1761 /* The 32-bit ABI is going to return the 64-bit value in
1762 the %o0/%o1 register pair. Prepare for this by using
1763 two return temporaries, and reassemble below. */
1764 retl
= tcg_temp_new_i64();
1765 reth
= tcg_temp_new_i64();
1766 op
->args
[pi
++] = tcgv_i64_arg(reth
);
1767 op
->args
[pi
++] = tcgv_i64_arg(retl
);
1770 op
->args
[pi
++] = temp_arg(ret
);
1774 if (TCG_TARGET_REG_BITS
< 64 && (sizemask
& 1)) {
1775 #ifdef HOST_WORDS_BIGENDIAN
1776 op
->args
[pi
++] = temp_arg(ret
+ 1);
1777 op
->args
[pi
++] = temp_arg(ret
);
1779 op
->args
[pi
++] = temp_arg(ret
);
1780 op
->args
[pi
++] = temp_arg(ret
+ 1);
1784 op
->args
[pi
++] = temp_arg(ret
);
1791 TCGOP_CALLO(op
) = nb_rets
;
1794 for (i
= 0; i
< nargs
; i
++) {
1795 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1796 if (TCG_TARGET_REG_BITS
< 64 && is_64bit
) {
1797 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1798 /* some targets want aligned 64 bit args */
1799 if (real_args
& 1) {
1800 op
->args
[pi
++] = TCG_CALL_DUMMY_ARG
;
1804 /* If stack grows up, then we will be placing successive
1805 arguments at lower addresses, which means we need to
1806 reverse the order compared to how we would normally
1807 treat either big or little-endian. For those arguments
1808 that will wind up in registers, this still works for
1809 HPPA (the only current STACK_GROWSUP target) since the
1810 argument registers are *also* allocated in decreasing
1811 order. If another such target is added, this logic may
1812 have to get more complicated to differentiate between
1813 stack arguments and register arguments. */
1814 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1815 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1816 op
->args
[pi
++] = temp_arg(args
[i
]);
1818 op
->args
[pi
++] = temp_arg(args
[i
]);
1819 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1825 op
->args
[pi
++] = temp_arg(args
[i
]);
1828 op
->args
[pi
++] = (uintptr_t)func
;
1829 op
->args
[pi
++] = flags
;
1830 TCGOP_CALLI(op
) = real_args
;
1832 /* Make sure the fields didn't overflow. */
1833 tcg_debug_assert(TCGOP_CALLI(op
) == real_args
);
1834 tcg_debug_assert(pi
<= ARRAY_SIZE(op
->args
));
1836 #if defined(__sparc__) && !defined(__arch64__) \
1837 && !defined(CONFIG_TCG_INTERPRETER)
1838 /* Free all of the parts we allocated above. */
1839 for (i
= real_args
= 0; i
< orig_nargs
; ++i
) {
1840 int is_64bit
= orig_sizemask
& (1 << (i
+1)*2);
1842 tcg_temp_free_internal(args
[real_args
++]);
1843 tcg_temp_free_internal(args
[real_args
++]);
1848 if (orig_sizemask
& 1) {
1849 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1850 Note that describing these as TCGv_i64 eliminates an unnecessary
1851 zero-extension that tcg_gen_concat_i32_i64 would create. */
1852 tcg_gen_concat32_i64(temp_tcgv_i64(ret
), retl
, reth
);
1853 tcg_temp_free_i64(retl
);
1854 tcg_temp_free_i64(reth
);
1856 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1857 for (i
= 0; i
< nargs
; ++i
) {
1858 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1860 tcg_temp_free_internal(args
[i
]);
1863 #endif /* TCG_TARGET_EXTEND_ARGS */
1866 static void tcg_reg_alloc_start(TCGContext
*s
)
1871 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
1873 ts
->val_type
= (ts
->fixed_reg
? TEMP_VAL_REG
: TEMP_VAL_MEM
);
1875 for (n
= s
->nb_temps
; i
< n
; i
++) {
1877 ts
->val_type
= (ts
->temp_local
? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
1878 ts
->mem_allocated
= 0;
1882 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1885 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1888 int idx
= temp_idx(ts
);
1890 if (ts
->temp_global
) {
1891 pstrcpy(buf
, buf_size
, ts
->name
);
1892 } else if (ts
->temp_local
) {
1893 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1895 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1900 static char *tcg_get_arg_str(TCGContext
*s
, char *buf
,
1901 int buf_size
, TCGArg arg
)
1903 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, arg_temp(arg
));
1906 /* Find helper name. */
1907 static inline const char *tcg_find_helper(TCGContext
*s
, uintptr_t val
)
1909 const char *ret
= NULL
;
1911 TCGHelperInfo
*info
= g_hash_table_lookup(helper_table
, (gpointer
)val
);
1919 static const char * const cond_name
[] =
1921 [TCG_COND_NEVER
] = "never",
1922 [TCG_COND_ALWAYS
] = "always",
1923 [TCG_COND_EQ
] = "eq",
1924 [TCG_COND_NE
] = "ne",
1925 [TCG_COND_LT
] = "lt",
1926 [TCG_COND_GE
] = "ge",
1927 [TCG_COND_LE
] = "le",
1928 [TCG_COND_GT
] = "gt",
1929 [TCG_COND_LTU
] = "ltu",
1930 [TCG_COND_GEU
] = "geu",
1931 [TCG_COND_LEU
] = "leu",
1932 [TCG_COND_GTU
] = "gtu"
1935 static const char * const ldst_name
[] =
1951 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1952 #ifdef TARGET_ALIGNED_ONLY
1953 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1954 [MO_ALIGN
>> MO_ASHIFT
] = "",
1956 [MO_UNALN
>> MO_ASHIFT
] = "",
1957 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1959 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1960 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1961 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1962 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1963 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1964 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1967 static inline bool tcg_regset_single(TCGRegSet d
)
1969 return (d
& (d
- 1)) == 0;
1972 static inline TCGReg
tcg_regset_first(TCGRegSet d
)
1974 if (TCG_TARGET_NB_REGS
<= 32) {
1981 static void tcg_dump_ops(TCGContext
*s
, bool have_prefs
)
1986 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
1987 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1988 const TCGOpDef
*def
;
1993 def
= &tcg_op_defs
[c
];
1995 if (c
== INDEX_op_insn_start
) {
1997 col
+= qemu_log("\n ----");
1999 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
2001 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2002 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
2006 col
+= qemu_log(" " TARGET_FMT_lx
, a
);
2008 } else if (c
== INDEX_op_call
) {
2009 /* variable number of arguments */
2010 nb_oargs
= TCGOP_CALLO(op
);
2011 nb_iargs
= TCGOP_CALLI(op
);
2012 nb_cargs
= def
->nb_cargs
;
2014 /* function name, flags, out args */
2015 col
+= qemu_log(" %s %s,$0x%" TCG_PRIlx
",$%d", def
->name
,
2016 tcg_find_helper(s
, op
->args
[nb_oargs
+ nb_iargs
]),
2017 op
->args
[nb_oargs
+ nb_iargs
+ 1], nb_oargs
);
2018 for (i
= 0; i
< nb_oargs
; i
++) {
2019 col
+= qemu_log(",%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2022 for (i
= 0; i
< nb_iargs
; i
++) {
2023 TCGArg arg
= op
->args
[nb_oargs
+ i
];
2024 const char *t
= "<dummy>";
2025 if (arg
!= TCG_CALL_DUMMY_ARG
) {
2026 t
= tcg_get_arg_str(s
, buf
, sizeof(buf
), arg
);
2028 col
+= qemu_log(",%s", t
);
2031 col
+= qemu_log(" %s ", def
->name
);
2033 nb_oargs
= def
->nb_oargs
;
2034 nb_iargs
= def
->nb_iargs
;
2035 nb_cargs
= def
->nb_cargs
;
2037 if (def
->flags
& TCG_OPF_VECTOR
) {
2038 col
+= qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op
),
2039 8 << TCGOP_VECE(op
));
2043 for (i
= 0; i
< nb_oargs
; i
++) {
2045 col
+= qemu_log(",");
2047 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2050 for (i
= 0; i
< nb_iargs
; i
++) {
2052 col
+= qemu_log(",");
2054 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
2058 case INDEX_op_brcond_i32
:
2059 case INDEX_op_setcond_i32
:
2060 case INDEX_op_movcond_i32
:
2061 case INDEX_op_brcond2_i32
:
2062 case INDEX_op_setcond2_i32
:
2063 case INDEX_op_brcond_i64
:
2064 case INDEX_op_setcond_i64
:
2065 case INDEX_op_movcond_i64
:
2066 case INDEX_op_cmp_vec
:
2067 case INDEX_op_cmpsel_vec
:
2068 if (op
->args
[k
] < ARRAY_SIZE(cond_name
)
2069 && cond_name
[op
->args
[k
]]) {
2070 col
+= qemu_log(",%s", cond_name
[op
->args
[k
++]]);
2072 col
+= qemu_log(",$0x%" TCG_PRIlx
, op
->args
[k
++]);
2076 case INDEX_op_qemu_ld_i32
:
2077 case INDEX_op_qemu_st_i32
:
2078 case INDEX_op_qemu_ld_i64
:
2079 case INDEX_op_qemu_st_i64
:
2081 TCGMemOpIdx oi
= op
->args
[k
++];
2082 MemOp op
= get_memop(oi
);
2083 unsigned ix
= get_mmuidx(oi
);
2085 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
2086 col
+= qemu_log(",$0x%x,%u", op
, ix
);
2088 const char *s_al
, *s_op
;
2089 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
2090 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
2091 col
+= qemu_log(",%s%s,%u", s_al
, s_op
, ix
);
2101 case INDEX_op_set_label
:
2103 case INDEX_op_brcond_i32
:
2104 case INDEX_op_brcond_i64
:
2105 case INDEX_op_brcond2_i32
:
2106 col
+= qemu_log("%s$L%d", k
? "," : "",
2107 arg_label(op
->args
[k
])->id
);
2113 for (; i
< nb_cargs
; i
++, k
++) {
2114 col
+= qemu_log("%s$0x%" TCG_PRIlx
, k
? "," : "", op
->args
[k
]);
2118 if (have_prefs
|| op
->life
) {
2120 QemuLogFile
*logfile
;
2123 logfile
= atomic_rcu_read(&qemu_logfile
);
2125 for (; col
< 40; ++col
) {
2126 putc(' ', logfile
->fd
);
2133 unsigned life
= op
->life
;
2135 if (life
& (SYNC_ARG
* 3)) {
2137 for (i
= 0; i
< 2; ++i
) {
2138 if (life
& (SYNC_ARG
<< i
)) {
2146 for (i
= 0; life
; ++i
, life
>>= 1) {
2155 for (i
= 0; i
< nb_oargs
; ++i
) {
2156 TCGRegSet set
= op
->output_pref
[i
];
2165 } else if (set
== MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS
)) {
2167 #ifdef CONFIG_DEBUG_TCG
2168 } else if (tcg_regset_single(set
)) {
2169 TCGReg reg
= tcg_regset_first(set
);
2170 qemu_log("%s", tcg_target_reg_names
[reg
]);
2172 } else if (TCG_TARGET_NB_REGS
<= 32) {
2173 qemu_log("%#x", (uint32_t)set
);
2175 qemu_log("%#" PRIx64
, (uint64_t)set
);
2184 /* we give more priority to constraints with less registers */
2185 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
2187 const TCGArgConstraint
*arg_ct
;
2190 arg_ct
= &def
->args_ct
[k
];
2191 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
2192 /* an alias is equivalent to a single register */
2195 if (!(arg_ct
->ct
& TCG_CT_REG
))
2198 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2199 if (tcg_regset_test_reg(arg_ct
->u
.regs
, i
))
2203 return TCG_TARGET_NB_REGS
- n
+ 1;
2206 /* sort from highest priority to lowest */
2207 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
2209 int i
, j
, p1
, p2
, tmp
;
2211 for(i
= 0; i
< n
; i
++)
2212 def
->sorted_args
[start
+ i
] = start
+ i
;
2215 for(i
= 0; i
< n
- 1; i
++) {
2216 for(j
= i
+ 1; j
< n
; j
++) {
2217 p1
= get_constraint_priority(def
, def
->sorted_args
[start
+ i
]);
2218 p2
= get_constraint_priority(def
, def
->sorted_args
[start
+ j
]);
2220 tmp
= def
->sorted_args
[start
+ i
];
2221 def
->sorted_args
[start
+ i
] = def
->sorted_args
[start
+ j
];
2222 def
->sorted_args
[start
+ j
] = tmp
;
2228 static void process_op_defs(TCGContext
*s
)
2232 for (op
= 0; op
< NB_OPS
; op
++) {
2233 TCGOpDef
*def
= &tcg_op_defs
[op
];
2234 const TCGTargetOpDef
*tdefs
;
2238 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2242 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
2247 tdefs
= tcg_target_op_def(op
);
2248 /* Missing TCGTargetOpDef entry. */
2249 tcg_debug_assert(tdefs
!= NULL
);
2251 type
= (def
->flags
& TCG_OPF_64BIT
? TCG_TYPE_I64
: TCG_TYPE_I32
);
2252 for (i
= 0; i
< nb_args
; i
++) {
2253 const char *ct_str
= tdefs
->args_ct_str
[i
];
2254 /* Incomplete TCGTargetOpDef entry. */
2255 tcg_debug_assert(ct_str
!= NULL
);
2257 def
->args_ct
[i
].u
.regs
= 0;
2258 def
->args_ct
[i
].ct
= 0;
2259 while (*ct_str
!= '\0') {
2263 int oarg
= *ct_str
- '0';
2264 tcg_debug_assert(ct_str
== tdefs
->args_ct_str
[i
]);
2265 tcg_debug_assert(oarg
< def
->nb_oargs
);
2266 tcg_debug_assert(def
->args_ct
[oarg
].ct
& TCG_CT_REG
);
2267 /* TCG_CT_ALIAS is for the output arguments.
2268 The input is tagged with TCG_CT_IALIAS. */
2269 def
->args_ct
[i
] = def
->args_ct
[oarg
];
2270 def
->args_ct
[oarg
].ct
|= TCG_CT_ALIAS
;
2271 def
->args_ct
[oarg
].alias_index
= i
;
2272 def
->args_ct
[i
].ct
|= TCG_CT_IALIAS
;
2273 def
->args_ct
[i
].alias_index
= oarg
;
2278 def
->args_ct
[i
].ct
|= TCG_CT_NEWREG
;
2282 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
2286 ct_str
= target_parse_constraint(&def
->args_ct
[i
],
2288 /* Typo in TCGTargetOpDef constraint. */
2289 tcg_debug_assert(ct_str
!= NULL
);
2294 /* TCGTargetOpDef entry with too much information? */
2295 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
2297 /* sort the constraints (XXX: this is just an heuristic) */
2298 sort_constraints(def
, 0, def
->nb_oargs
);
2299 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
2303 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
2309 label
= arg_label(op
->args
[0]);
2312 case INDEX_op_brcond_i32
:
2313 case INDEX_op_brcond_i64
:
2314 label
= arg_label(op
->args
[3]);
2317 case INDEX_op_brcond2_i32
:
2318 label
= arg_label(op
->args
[5]);
2325 QTAILQ_REMOVE(&s
->ops
, op
, link
);
2326 QTAILQ_INSERT_TAIL(&s
->free_ops
, op
, link
);
2329 #ifdef CONFIG_PROFILER
2330 atomic_set(&s
->prof
.del_op_count
, s
->prof
.del_op_count
+ 1);
2334 static TCGOp
*tcg_op_alloc(TCGOpcode opc
)
2336 TCGContext
*s
= tcg_ctx
;
2339 if (likely(QTAILQ_EMPTY(&s
->free_ops
))) {
2340 op
= tcg_malloc(sizeof(TCGOp
));
2342 op
= QTAILQ_FIRST(&s
->free_ops
);
2343 QTAILQ_REMOVE(&s
->free_ops
, op
, link
);
2345 memset(op
, 0, offsetof(TCGOp
, link
));
2352 TCGOp
*tcg_emit_op(TCGOpcode opc
)
2354 TCGOp
*op
= tcg_op_alloc(opc
);
2355 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
2359 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
, TCGOpcode opc
)
2361 TCGOp
*new_op
= tcg_op_alloc(opc
);
2362 QTAILQ_INSERT_BEFORE(old_op
, new_op
, link
);
2366 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
, TCGOpcode opc
)
2368 TCGOp
*new_op
= tcg_op_alloc(opc
);
2369 QTAILQ_INSERT_AFTER(&s
->ops
, old_op
, new_op
, link
);
2373 /* Reachable analysis : remove unreachable code. */
2374 static void reachable_code_pass(TCGContext
*s
)
2376 TCGOp
*op
, *op_next
;
2379 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2385 case INDEX_op_set_label
:
2386 label
= arg_label(op
->args
[0]);
2387 if (label
->refs
== 0) {
2389 * While there is an occasional backward branch, virtually
2390 * all branches generated by the translators are forward.
2391 * Which means that generally we will have already removed
2392 * all references to the label that will be, and there is
2393 * little to be gained by iterating.
2397 /* Once we see a label, insns become live again. */
2402 * Optimization can fold conditional branches to unconditional.
2403 * If we find a label with one reference which is preceded by
2404 * an unconditional branch to it, remove both. This needed to
2405 * wait until the dead code in between them was removed.
2407 if (label
->refs
== 1) {
2408 TCGOp
*op_prev
= QTAILQ_PREV(op
, link
);
2409 if (op_prev
->opc
== INDEX_op_br
&&
2410 label
== arg_label(op_prev
->args
[0])) {
2411 tcg_op_remove(s
, op_prev
);
2419 case INDEX_op_exit_tb
:
2420 case INDEX_op_goto_ptr
:
2421 /* Unconditional branches; everything following is dead. */
2426 /* Notice noreturn helper calls, raising exceptions. */
2427 call_flags
= op
->args
[TCGOP_CALLO(op
) + TCGOP_CALLI(op
) + 1];
2428 if (call_flags
& TCG_CALL_NO_RETURN
) {
2433 case INDEX_op_insn_start
:
2434 /* Never remove -- we need to keep these for unwind. */
2443 tcg_op_remove(s
, op
);
2451 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2452 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2454 /* For liveness_pass_1, the register preferences for a given temp. */
2455 static inline TCGRegSet
*la_temp_pref(TCGTemp
*ts
)
2457 return ts
->state_ptr
;
2460 /* For liveness_pass_1, reset the preferences for a given temp to the
2461 * maximal regset for its type.
2463 static inline void la_reset_pref(TCGTemp
*ts
)
2466 = (ts
->state
== TS_DEAD
? 0 : tcg_target_available_regs
[ts
->type
]);
2469 /* liveness analysis: end of function: all temps are dead, and globals
2470 should be in memory. */
2471 static void la_func_end(TCGContext
*s
, int ng
, int nt
)
2475 for (i
= 0; i
< ng
; ++i
) {
2476 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2477 la_reset_pref(&s
->temps
[i
]);
2479 for (i
= ng
; i
< nt
; ++i
) {
2480 s
->temps
[i
].state
= TS_DEAD
;
2481 la_reset_pref(&s
->temps
[i
]);
2485 /* liveness analysis: end of basic block: all temps are dead, globals
2486 and local temps should be in memory. */
2487 static void la_bb_end(TCGContext
*s
, int ng
, int nt
)
2491 for (i
= 0; i
< ng
; ++i
) {
2492 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2493 la_reset_pref(&s
->temps
[i
]);
2495 for (i
= ng
; i
< nt
; ++i
) {
2496 s
->temps
[i
].state
= (s
->temps
[i
].temp_local
2499 la_reset_pref(&s
->temps
[i
]);
2503 /* liveness analysis: sync globals back to memory. */
2504 static void la_global_sync(TCGContext
*s
, int ng
)
2508 for (i
= 0; i
< ng
; ++i
) {
2509 int state
= s
->temps
[i
].state
;
2510 s
->temps
[i
].state
= state
| TS_MEM
;
2511 if (state
== TS_DEAD
) {
2512 /* If the global was previously dead, reset prefs. */
2513 la_reset_pref(&s
->temps
[i
]);
2518 /* liveness analysis: sync globals back to memory and kill. */
2519 static void la_global_kill(TCGContext
*s
, int ng
)
2523 for (i
= 0; i
< ng
; i
++) {
2524 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2525 la_reset_pref(&s
->temps
[i
]);
2529 /* liveness analysis: note live globals crossing calls. */
2530 static void la_cross_call(TCGContext
*s
, int nt
)
2532 TCGRegSet mask
= ~tcg_target_call_clobber_regs
;
2535 for (i
= 0; i
< nt
; i
++) {
2536 TCGTemp
*ts
= &s
->temps
[i
];
2537 if (!(ts
->state
& TS_DEAD
)) {
2538 TCGRegSet
*pset
= la_temp_pref(ts
);
2539 TCGRegSet set
= *pset
;
2542 /* If the combination is not possible, restart. */
2544 set
= tcg_target_available_regs
[ts
->type
] & mask
;
2551 /* Liveness analysis : update the opc_arg_life array to tell if a
2552 given input arguments is dead. Instructions updating dead
2553 temporaries are removed. */
2554 static void liveness_pass_1(TCGContext
*s
)
2556 int nb_globals
= s
->nb_globals
;
2557 int nb_temps
= s
->nb_temps
;
2558 TCGOp
*op
, *op_prev
;
2562 prefs
= tcg_malloc(sizeof(TCGRegSet
) * nb_temps
);
2563 for (i
= 0; i
< nb_temps
; ++i
) {
2564 s
->temps
[i
].state_ptr
= prefs
+ i
;
2567 /* ??? Should be redundant with the exit_tb that ends the TB. */
2568 la_func_end(s
, nb_globals
, nb_temps
);
2570 QTAILQ_FOREACH_REVERSE_SAFE(op
, &s
->ops
, link
, op_prev
) {
2571 int nb_iargs
, nb_oargs
;
2572 TCGOpcode opc_new
, opc_new2
;
2574 TCGLifeData arg_life
= 0;
2576 TCGOpcode opc
= op
->opc
;
2577 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2585 nb_oargs
= TCGOP_CALLO(op
);
2586 nb_iargs
= TCGOP_CALLI(op
);
2587 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2589 /* pure functions can be removed if their result is unused */
2590 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
2591 for (i
= 0; i
< nb_oargs
; i
++) {
2592 ts
= arg_temp(op
->args
[i
]);
2593 if (ts
->state
!= TS_DEAD
) {
2594 goto do_not_remove_call
;
2601 /* Output args are dead. */
2602 for (i
= 0; i
< nb_oargs
; i
++) {
2603 ts
= arg_temp(op
->args
[i
]);
2604 if (ts
->state
& TS_DEAD
) {
2605 arg_life
|= DEAD_ARG
<< i
;
2607 if (ts
->state
& TS_MEM
) {
2608 arg_life
|= SYNC_ARG
<< i
;
2610 ts
->state
= TS_DEAD
;
2613 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2614 op
->output_pref
[i
] = 0;
2617 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
2618 TCG_CALL_NO_READ_GLOBALS
))) {
2619 la_global_kill(s
, nb_globals
);
2620 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
2621 la_global_sync(s
, nb_globals
);
2624 /* Record arguments that die in this helper. */
2625 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2626 ts
= arg_temp(op
->args
[i
]);
2627 if (ts
&& ts
->state
& TS_DEAD
) {
2628 arg_life
|= DEAD_ARG
<< i
;
2632 /* For all live registers, remove call-clobbered prefs. */
2633 la_cross_call(s
, nb_temps
);
2635 nb_call_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
2637 /* Input arguments are live for preceding opcodes. */
2638 for (i
= 0; i
< nb_iargs
; i
++) {
2639 ts
= arg_temp(op
->args
[i
+ nb_oargs
]);
2640 if (ts
&& ts
->state
& TS_DEAD
) {
2641 /* For those arguments that die, and will be allocated
2642 * in registers, clear the register set for that arg,
2643 * to be filled in below. For args that will be on
2644 * the stack, reset to any available reg.
2647 = (i
< nb_call_regs
? 0 :
2648 tcg_target_available_regs
[ts
->type
]);
2649 ts
->state
&= ~TS_DEAD
;
2653 /* For each input argument, add its input register to prefs.
2654 If a temp is used once, this produces a single set bit. */
2655 for (i
= 0; i
< MIN(nb_call_regs
, nb_iargs
); i
++) {
2656 ts
= arg_temp(op
->args
[i
+ nb_oargs
]);
2658 tcg_regset_set_reg(*la_temp_pref(ts
),
2659 tcg_target_call_iarg_regs
[i
]);
2664 case INDEX_op_insn_start
:
2666 case INDEX_op_discard
:
2667 /* mark the temporary as dead */
2668 ts
= arg_temp(op
->args
[0]);
2669 ts
->state
= TS_DEAD
;
2673 case INDEX_op_add2_i32
:
2674 opc_new
= INDEX_op_add_i32
;
2676 case INDEX_op_sub2_i32
:
2677 opc_new
= INDEX_op_sub_i32
;
2679 case INDEX_op_add2_i64
:
2680 opc_new
= INDEX_op_add_i64
;
2682 case INDEX_op_sub2_i64
:
2683 opc_new
= INDEX_op_sub_i64
;
2687 /* Test if the high part of the operation is dead, but not
2688 the low part. The result can be optimized to a simple
2689 add or sub. This happens often for x86_64 guest when the
2690 cpu mode is set to 32 bit. */
2691 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2692 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2695 /* Replace the opcode and adjust the args in place,
2696 leaving 3 unused args at the end. */
2697 op
->opc
= opc
= opc_new
;
2698 op
->args
[1] = op
->args
[2];
2699 op
->args
[2] = op
->args
[4];
2700 /* Fall through and mark the single-word operation live. */
2706 case INDEX_op_mulu2_i32
:
2707 opc_new
= INDEX_op_mul_i32
;
2708 opc_new2
= INDEX_op_muluh_i32
;
2709 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
2711 case INDEX_op_muls2_i32
:
2712 opc_new
= INDEX_op_mul_i32
;
2713 opc_new2
= INDEX_op_mulsh_i32
;
2714 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
2716 case INDEX_op_mulu2_i64
:
2717 opc_new
= INDEX_op_mul_i64
;
2718 opc_new2
= INDEX_op_muluh_i64
;
2719 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
2721 case INDEX_op_muls2_i64
:
2722 opc_new
= INDEX_op_mul_i64
;
2723 opc_new2
= INDEX_op_mulsh_i64
;
2724 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
2729 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2730 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2731 /* Both parts of the operation are dead. */
2734 /* The high part of the operation is dead; generate the low. */
2735 op
->opc
= opc
= opc_new
;
2736 op
->args
[1] = op
->args
[2];
2737 op
->args
[2] = op
->args
[3];
2738 } else if (arg_temp(op
->args
[0])->state
== TS_DEAD
&& have_opc_new2
) {
2739 /* The low part of the operation is dead; generate the high. */
2740 op
->opc
= opc
= opc_new2
;
2741 op
->args
[0] = op
->args
[1];
2742 op
->args
[1] = op
->args
[2];
2743 op
->args
[2] = op
->args
[3];
2747 /* Mark the single-word operation live. */
2752 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2753 nb_iargs
= def
->nb_iargs
;
2754 nb_oargs
= def
->nb_oargs
;
2756 /* Test if the operation can be removed because all
2757 its outputs are dead. We assume that nb_oargs == 0
2758 implies side effects */
2759 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
2760 for (i
= 0; i
< nb_oargs
; i
++) {
2761 if (arg_temp(op
->args
[i
])->state
!= TS_DEAD
) {
2770 tcg_op_remove(s
, op
);
2774 for (i
= 0; i
< nb_oargs
; i
++) {
2775 ts
= arg_temp(op
->args
[i
]);
2777 /* Remember the preference of the uses that followed. */
2778 op
->output_pref
[i
] = *la_temp_pref(ts
);
2780 /* Output args are dead. */
2781 if (ts
->state
& TS_DEAD
) {
2782 arg_life
|= DEAD_ARG
<< i
;
2784 if (ts
->state
& TS_MEM
) {
2785 arg_life
|= SYNC_ARG
<< i
;
2787 ts
->state
= TS_DEAD
;
2791 /* If end of basic block, update. */
2792 if (def
->flags
& TCG_OPF_BB_EXIT
) {
2793 la_func_end(s
, nb_globals
, nb_temps
);
2794 } else if (def
->flags
& TCG_OPF_BB_END
) {
2795 la_bb_end(s
, nb_globals
, nb_temps
);
2796 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2797 la_global_sync(s
, nb_globals
);
2798 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
2799 la_cross_call(s
, nb_temps
);
2803 /* Record arguments that die in this opcode. */
2804 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2805 ts
= arg_temp(op
->args
[i
]);
2806 if (ts
->state
& TS_DEAD
) {
2807 arg_life
|= DEAD_ARG
<< i
;
2811 /* Input arguments are live for preceding opcodes. */
2812 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2813 ts
= arg_temp(op
->args
[i
]);
2814 if (ts
->state
& TS_DEAD
) {
2815 /* For operands that were dead, initially allow
2816 all regs for the type. */
2817 *la_temp_pref(ts
) = tcg_target_available_regs
[ts
->type
];
2818 ts
->state
&= ~TS_DEAD
;
2822 /* Incorporate constraints for this operand. */
2824 case INDEX_op_mov_i32
:
2825 case INDEX_op_mov_i64
:
2826 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2827 have proper constraints. That said, special case
2828 moves to propagate preferences backward. */
2829 if (IS_DEAD_ARG(1)) {
2830 *la_temp_pref(arg_temp(op
->args
[0]))
2831 = *la_temp_pref(arg_temp(op
->args
[1]));
2836 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2837 const TCGArgConstraint
*ct
= &def
->args_ct
[i
];
2838 TCGRegSet set
, *pset
;
2840 ts
= arg_temp(op
->args
[i
]);
2841 pset
= la_temp_pref(ts
);
2845 if (ct
->ct
& TCG_CT_IALIAS
) {
2846 set
&= op
->output_pref
[ct
->alias_index
];
2848 /* If the combination is not possible, restart. */
2858 op
->life
= arg_life
;
2862 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2863 static bool liveness_pass_2(TCGContext
*s
)
2865 int nb_globals
= s
->nb_globals
;
2867 bool changes
= false;
2868 TCGOp
*op
, *op_next
;
2870 /* Create a temporary for each indirect global. */
2871 for (i
= 0; i
< nb_globals
; ++i
) {
2872 TCGTemp
*its
= &s
->temps
[i
];
2873 if (its
->indirect_reg
) {
2874 TCGTemp
*dts
= tcg_temp_alloc(s
);
2875 dts
->type
= its
->type
;
2876 dts
->base_type
= its
->base_type
;
2877 its
->state_ptr
= dts
;
2879 its
->state_ptr
= NULL
;
2881 /* All globals begin dead. */
2882 its
->state
= TS_DEAD
;
2884 for (nb_temps
= s
->nb_temps
; i
< nb_temps
; ++i
) {
2885 TCGTemp
*its
= &s
->temps
[i
];
2886 its
->state_ptr
= NULL
;
2887 its
->state
= TS_DEAD
;
2890 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2891 TCGOpcode opc
= op
->opc
;
2892 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2893 TCGLifeData arg_life
= op
->life
;
2894 int nb_iargs
, nb_oargs
, call_flags
;
2895 TCGTemp
*arg_ts
, *dir_ts
;
2897 if (opc
== INDEX_op_call
) {
2898 nb_oargs
= TCGOP_CALLO(op
);
2899 nb_iargs
= TCGOP_CALLI(op
);
2900 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2902 nb_iargs
= def
->nb_iargs
;
2903 nb_oargs
= def
->nb_oargs
;
2905 /* Set flags similar to how calls require. */
2906 if (def
->flags
& TCG_OPF_BB_END
) {
2907 /* Like writing globals: save_globals */
2909 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2910 /* Like reading globals: sync_globals */
2911 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
2913 /* No effect on globals. */
2914 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
2915 TCG_CALL_NO_WRITE_GLOBALS
);
2919 /* Make sure that input arguments are available. */
2920 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2921 arg_ts
= arg_temp(op
->args
[i
]);
2923 dir_ts
= arg_ts
->state_ptr
;
2924 if (dir_ts
&& arg_ts
->state
== TS_DEAD
) {
2925 TCGOpcode lopc
= (arg_ts
->type
== TCG_TYPE_I32
2928 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
);
2930 lop
->args
[0] = temp_arg(dir_ts
);
2931 lop
->args
[1] = temp_arg(arg_ts
->mem_base
);
2932 lop
->args
[2] = arg_ts
->mem_offset
;
2934 /* Loaded, but synced with memory. */
2935 arg_ts
->state
= TS_MEM
;
2940 /* Perform input replacement, and mark inputs that became dead.
2941 No action is required except keeping temp_state up to date
2942 so that we reload when needed. */
2943 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2944 arg_ts
= arg_temp(op
->args
[i
]);
2946 dir_ts
= arg_ts
->state_ptr
;
2948 op
->args
[i
] = temp_arg(dir_ts
);
2950 if (IS_DEAD_ARG(i
)) {
2951 arg_ts
->state
= TS_DEAD
;
2957 /* Liveness analysis should ensure that the following are
2958 all correct, for call sites and basic block end points. */
2959 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
2961 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
2962 for (i
= 0; i
< nb_globals
; ++i
) {
2963 /* Liveness should see that globals are synced back,
2964 that is, either TS_DEAD or TS_MEM. */
2965 arg_ts
= &s
->temps
[i
];
2966 tcg_debug_assert(arg_ts
->state_ptr
== 0
2967 || arg_ts
->state
!= 0);
2970 for (i
= 0; i
< nb_globals
; ++i
) {
2971 /* Liveness should see that globals are saved back,
2972 that is, TS_DEAD, waiting to be reloaded. */
2973 arg_ts
= &s
->temps
[i
];
2974 tcg_debug_assert(arg_ts
->state_ptr
== 0
2975 || arg_ts
->state
== TS_DEAD
);
2979 /* Outputs become available. */
2980 for (i
= 0; i
< nb_oargs
; i
++) {
2981 arg_ts
= arg_temp(op
->args
[i
]);
2982 dir_ts
= arg_ts
->state_ptr
;
2986 op
->args
[i
] = temp_arg(dir_ts
);
2989 /* The output is now live and modified. */
2992 /* Sync outputs upon their last write. */
2993 if (NEED_SYNC_ARG(i
)) {
2994 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
2997 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
);
2999 sop
->args
[0] = temp_arg(dir_ts
);
3000 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
3001 sop
->args
[2] = arg_ts
->mem_offset
;
3003 arg_ts
->state
= TS_MEM
;
3005 /* Drop outputs that are dead. */
3006 if (IS_DEAD_ARG(i
)) {
3007 arg_ts
->state
= TS_DEAD
;
3015 #ifdef CONFIG_DEBUG_TCG
3016 static void dump_regs(TCGContext
*s
)
3022 for(i
= 0; i
< s
->nb_temps
; i
++) {
3024 printf(" %10s: ", tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
3025 switch(ts
->val_type
) {
3027 printf("%s", tcg_target_reg_names
[ts
->reg
]);
3030 printf("%d(%s)", (int)ts
->mem_offset
,
3031 tcg_target_reg_names
[ts
->mem_base
->reg
]);
3033 case TEMP_VAL_CONST
:
3034 printf("$0x%" TCG_PRIlx
, ts
->val
);
3046 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3047 if (s
->reg_to_temp
[i
] != NULL
) {
3049 tcg_target_reg_names
[i
],
3050 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), s
->reg_to_temp
[i
]));
3055 static void check_regs(TCGContext
*s
)
3062 for (reg
= 0; reg
< TCG_TARGET_NB_REGS
; reg
++) {
3063 ts
= s
->reg_to_temp
[reg
];
3065 if (ts
->val_type
!= TEMP_VAL_REG
|| ts
->reg
!= reg
) {
3066 printf("Inconsistency for register %s:\n",
3067 tcg_target_reg_names
[reg
]);
3072 for (k
= 0; k
< s
->nb_temps
; k
++) {
3074 if (ts
->val_type
== TEMP_VAL_REG
&& !ts
->fixed_reg
3075 && s
->reg_to_temp
[ts
->reg
] != ts
) {
3076 printf("Inconsistency for temp %s:\n",
3077 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
3079 printf("reg state:\n");
3087 static void temp_allocate_frame(TCGContext
*s
, TCGTemp
*ts
)
3089 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
3090 /* Sparc64 stack is accessed with offset of 2047 */
3091 s
->current_frame_offset
= (s
->current_frame_offset
+
3092 (tcg_target_long
)sizeof(tcg_target_long
) - 1) &
3093 ~(sizeof(tcg_target_long
) - 1);
3095 if (s
->current_frame_offset
+ (tcg_target_long
)sizeof(tcg_target_long
) >
3099 ts
->mem_offset
= s
->current_frame_offset
;
3100 ts
->mem_base
= s
->frame_temp
;
3101 ts
->mem_allocated
= 1;
3102 s
->current_frame_offset
+= sizeof(tcg_target_long
);
3105 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
, TCGRegSet
);
3107 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3108 mark it free; otherwise mark it dead. */
3109 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
3111 if (ts
->fixed_reg
) {
3114 if (ts
->val_type
== TEMP_VAL_REG
) {
3115 s
->reg_to_temp
[ts
->reg
] = NULL
;
3117 ts
->val_type
= (free_or_dead
< 0
3120 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
3123 /* Mark a temporary as dead. */
3124 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
3126 temp_free_or_dead(s
, ts
, 1);
3129 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3130 registers needs to be allocated to store a constant. If 'free_or_dead'
3131 is non-zero, subsequently release the temporary; if it is positive, the
3132 temp is dead; if it is negative, the temp is free. */
3133 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
,
3134 TCGRegSet preferred_regs
, int free_or_dead
)
3136 if (ts
->fixed_reg
) {
3139 if (!ts
->mem_coherent
) {
3140 if (!ts
->mem_allocated
) {
3141 temp_allocate_frame(s
, ts
);
3143 switch (ts
->val_type
) {
3144 case TEMP_VAL_CONST
:
3145 /* If we're going to free the temp immediately, then we won't
3146 require it later in a register, so attempt to store the
3147 constant to memory directly. */
3149 && tcg_out_sti(s
, ts
->type
, ts
->val
,
3150 ts
->mem_base
->reg
, ts
->mem_offset
)) {
3153 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3154 allocated_regs
, preferred_regs
);
3158 tcg_out_st(s
, ts
->type
, ts
->reg
,
3159 ts
->mem_base
->reg
, ts
->mem_offset
);
3169 ts
->mem_coherent
= 1;
3172 temp_free_or_dead(s
, ts
, free_or_dead
);
3176 /* free register 'reg' by spilling the corresponding temporary if necessary */
3177 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
3179 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
3181 temp_sync(s
, ts
, allocated_regs
, 0, -1);
3187 * @required_regs: Set of registers in which we must allocate.
3188 * @allocated_regs: Set of registers which must be avoided.
3189 * @preferred_regs: Set of registers we should prefer.
3190 * @rev: True if we search the registers in "indirect" order.
3192 * The allocated register must be in @required_regs & ~@allocated_regs,
3193 * but if we can put it in @preferred_regs we may save a move later.
3195 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet required_regs
,
3196 TCGRegSet allocated_regs
,
3197 TCGRegSet preferred_regs
, bool rev
)
3199 int i
, j
, f
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
3200 TCGRegSet reg_ct
[2];
3203 reg_ct
[1] = required_regs
& ~allocated_regs
;
3204 tcg_debug_assert(reg_ct
[1] != 0);
3205 reg_ct
[0] = reg_ct
[1] & preferred_regs
;
3207 /* Skip the preferred_regs option if it cannot be satisfied,
3208 or if the preference made no difference. */
3209 f
= reg_ct
[0] == 0 || reg_ct
[0] == reg_ct
[1];
3211 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
3213 /* Try free registers, preferences first. */
3214 for (j
= f
; j
< 2; j
++) {
3215 TCGRegSet set
= reg_ct
[j
];
3217 if (tcg_regset_single(set
)) {
3218 /* One register in the set. */
3219 TCGReg reg
= tcg_regset_first(set
);
3220 if (s
->reg_to_temp
[reg
] == NULL
) {
3224 for (i
= 0; i
< n
; i
++) {
3225 TCGReg reg
= order
[i
];
3226 if (s
->reg_to_temp
[reg
] == NULL
&&
3227 tcg_regset_test_reg(set
, reg
)) {
3234 /* We must spill something. */
3235 for (j
= f
; j
< 2; j
++) {
3236 TCGRegSet set
= reg_ct
[j
];
3238 if (tcg_regset_single(set
)) {
3239 /* One register in the set. */
3240 TCGReg reg
= tcg_regset_first(set
);
3241 tcg_reg_free(s
, reg
, allocated_regs
);
3244 for (i
= 0; i
< n
; i
++) {
3245 TCGReg reg
= order
[i
];
3246 if (tcg_regset_test_reg(set
, reg
)) {
3247 tcg_reg_free(s
, reg
, allocated_regs
);
3257 /* Make sure the temporary is in a register. If needed, allocate the register
3258 from DESIRED while avoiding ALLOCATED. */
3259 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
3260 TCGRegSet allocated_regs
, TCGRegSet preferred_regs
)
3264 switch (ts
->val_type
) {
3267 case TEMP_VAL_CONST
:
3268 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3269 preferred_regs
, ts
->indirect_base
);
3270 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
3271 ts
->mem_coherent
= 0;
3274 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
,
3275 preferred_regs
, ts
->indirect_base
);
3276 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
3277 ts
->mem_coherent
= 1;
3284 ts
->val_type
= TEMP_VAL_REG
;
3285 s
->reg_to_temp
[reg
] = ts
;
3288 /* Save a temporary to memory. 'allocated_regs' is used in case a
3289 temporary registers needs to be allocated to store a constant. */
3290 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
3292 /* The liveness analysis already ensures that globals are back
3293 in memory. Keep an tcg_debug_assert for safety. */
3294 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| ts
->fixed_reg
);
3297 /* save globals to their canonical location and assume they can be
3298 modified be the following code. 'allocated_regs' is used in case a
3299 temporary registers needs to be allocated to store a constant. */
3300 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3304 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3305 temp_save(s
, &s
->temps
[i
], allocated_regs
);
3309 /* sync globals to their canonical location and assume they can be
3310 read by the following code. 'allocated_regs' is used in case a
3311 temporary registers needs to be allocated to store a constant. */
3312 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
3316 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
3317 TCGTemp
*ts
= &s
->temps
[i
];
3318 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
3320 || ts
->mem_coherent
);
3324 /* at the end of a basic block, we assume all temporaries are dead and
3325 all globals are stored at their canonical location. */
3326 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
3330 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
3331 TCGTemp
*ts
= &s
->temps
[i
];
3332 if (ts
->temp_local
) {
3333 temp_save(s
, ts
, allocated_regs
);
3335 /* The liveness analysis already ensures that temps are dead.
3336 Keep an tcg_debug_assert for safety. */
3337 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
3341 save_globals(s
, allocated_regs
);
3345 * Specialized code generation for INDEX_op_movi_*.
3347 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
3348 tcg_target_ulong val
, TCGLifeData arg_life
,
3349 TCGRegSet preferred_regs
)
3351 /* ENV should not be modified. */
3352 tcg_debug_assert(!ots
->fixed_reg
);
3354 /* The movi is not explicitly generated here. */
3355 if (ots
->val_type
== TEMP_VAL_REG
) {
3356 s
->reg_to_temp
[ots
->reg
] = NULL
;
3358 ots
->val_type
= TEMP_VAL_CONST
;
3360 ots
->mem_coherent
= 0;
3361 if (NEED_SYNC_ARG(0)) {
3362 temp_sync(s
, ots
, s
->reserved_regs
, preferred_regs
, IS_DEAD_ARG(0));
3363 } else if (IS_DEAD_ARG(0)) {
3368 static void tcg_reg_alloc_movi(TCGContext
*s
, const TCGOp
*op
)
3370 TCGTemp
*ots
= arg_temp(op
->args
[0]);
3371 tcg_target_ulong val
= op
->args
[1];
3373 tcg_reg_alloc_do_movi(s
, ots
, val
, op
->life
, op
->output_pref
[0]);
3377 * Specialized code generation for INDEX_op_mov_*.
3379 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOp
*op
)
3381 const TCGLifeData arg_life
= op
->life
;
3382 TCGRegSet allocated_regs
, preferred_regs
;
3384 TCGType otype
, itype
;
3386 allocated_regs
= s
->reserved_regs
;
3387 preferred_regs
= op
->output_pref
[0];
3388 ots
= arg_temp(op
->args
[0]);
3389 ts
= arg_temp(op
->args
[1]);
3391 /* ENV should not be modified. */
3392 tcg_debug_assert(!ots
->fixed_reg
);
3394 /* Note that otype != itype for no-op truncation. */
3398 if (ts
->val_type
== TEMP_VAL_CONST
) {
3399 /* propagate constant or generate sti */
3400 tcg_target_ulong val
= ts
->val
;
3401 if (IS_DEAD_ARG(1)) {
3404 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, preferred_regs
);
3408 /* If the source value is in memory we're going to be forced
3409 to have it in a register in order to perform the copy. Copy
3410 the SOURCE value into its own register first, that way we
3411 don't have to reload SOURCE the next time it is used. */
3412 if (ts
->val_type
== TEMP_VAL_MEM
) {
3413 temp_load(s
, ts
, tcg_target_available_regs
[itype
],
3414 allocated_regs
, preferred_regs
);
3417 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
3418 if (IS_DEAD_ARG(0)) {
3419 /* mov to a non-saved dead register makes no sense (even with
3420 liveness analysis disabled). */
3421 tcg_debug_assert(NEED_SYNC_ARG(0));
3422 if (!ots
->mem_allocated
) {
3423 temp_allocate_frame(s
, ots
);
3425 tcg_out_st(s
, otype
, ts
->reg
, ots
->mem_base
->reg
, ots
->mem_offset
);
3426 if (IS_DEAD_ARG(1)) {
3431 if (IS_DEAD_ARG(1) && !ts
->fixed_reg
) {
3432 /* the mov can be suppressed */
3433 if (ots
->val_type
== TEMP_VAL_REG
) {
3434 s
->reg_to_temp
[ots
->reg
] = NULL
;
3439 if (ots
->val_type
!= TEMP_VAL_REG
) {
3440 /* When allocating a new register, make sure to not spill the
3442 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
3443 ots
->reg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
3444 allocated_regs
, preferred_regs
,
3445 ots
->indirect_base
);
3447 if (!tcg_out_mov(s
, otype
, ots
->reg
, ts
->reg
)) {
3449 * Cross register class move not supported.
3450 * Store the source register into the destination slot
3451 * and leave the destination temp as TEMP_VAL_MEM.
3453 assert(!ots
->fixed_reg
);
3454 if (!ts
->mem_allocated
) {
3455 temp_allocate_frame(s
, ots
);
3457 tcg_out_st(s
, ts
->type
, ts
->reg
,
3458 ots
->mem_base
->reg
, ots
->mem_offset
);
3459 ots
->mem_coherent
= 1;
3460 temp_free_or_dead(s
, ots
, -1);
3464 ots
->val_type
= TEMP_VAL_REG
;
3465 ots
->mem_coherent
= 0;
3466 s
->reg_to_temp
[ots
->reg
] = ots
;
3467 if (NEED_SYNC_ARG(0)) {
3468 temp_sync(s
, ots
, allocated_regs
, 0, 0);
3474 * Specialized code generation for INDEX_op_dup_vec.
3476 static void tcg_reg_alloc_dup(TCGContext
*s
, const TCGOp
*op
)
3478 const TCGLifeData arg_life
= op
->life
;
3479 TCGRegSet dup_out_regs
, dup_in_regs
;
3481 TCGType itype
, vtype
;
3482 intptr_t endian_fixup
;
3486 ots
= arg_temp(op
->args
[0]);
3487 its
= arg_temp(op
->args
[1]);
3489 /* ENV should not be modified. */
3490 tcg_debug_assert(!ots
->fixed_reg
);
3493 vece
= TCGOP_VECE(op
);
3494 vtype
= TCGOP_VECL(op
) + TCG_TYPE_V64
;
3496 if (its
->val_type
== TEMP_VAL_CONST
) {
3497 /* Propagate constant via movi -> dupi. */
3498 tcg_target_ulong val
= its
->val
;
3499 if (IS_DEAD_ARG(1)) {
3502 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
, op
->output_pref
[0]);
3506 dup_out_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[0].u
.regs
;
3507 dup_in_regs
= tcg_op_defs
[INDEX_op_dup_vec
].args_ct
[1].u
.regs
;
3509 /* Allocate the output register now. */
3510 if (ots
->val_type
!= TEMP_VAL_REG
) {
3511 TCGRegSet allocated_regs
= s
->reserved_regs
;
3513 if (!IS_DEAD_ARG(1) && its
->val_type
== TEMP_VAL_REG
) {
3514 /* Make sure to not spill the input register. */
3515 tcg_regset_set_reg(allocated_regs
, its
->reg
);
3517 ots
->reg
= tcg_reg_alloc(s
, dup_out_regs
, allocated_regs
,
3518 op
->output_pref
[0], ots
->indirect_base
);
3519 ots
->val_type
= TEMP_VAL_REG
;
3520 ots
->mem_coherent
= 0;
3521 s
->reg_to_temp
[ots
->reg
] = ots
;
3524 switch (its
->val_type
) {
3527 * The dup constriaints must be broad, covering all possible VECE.
3528 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3529 * to fail, indicating that extra moves are required for that case.
3531 if (tcg_regset_test_reg(dup_in_regs
, its
->reg
)) {
3532 if (tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, its
->reg
)) {
3535 /* Try again from memory or a vector input register. */
3537 if (!its
->mem_coherent
) {
3539 * The input register is not synced, and so an extra store
3540 * would be required to use memory. Attempt an integer-vector
3541 * register move first. We do not have a TCGRegSet for this.
3543 if (tcg_out_mov(s
, itype
, ots
->reg
, its
->reg
)) {
3546 /* Sync the temp back to its slot and load from there. */
3547 temp_sync(s
, its
, s
->reserved_regs
, 0, 0);
3552 #ifdef HOST_WORDS_BIGENDIAN
3553 endian_fixup
= itype
== TCG_TYPE_I32
? 4 : 8;
3554 endian_fixup
-= 1 << vece
;
3558 if (tcg_out_dupm_vec(s
, vtype
, vece
, ots
->reg
, its
->mem_base
->reg
,
3559 its
->mem_offset
+ endian_fixup
)) {
3562 tcg_out_ld(s
, itype
, ots
->reg
, its
->mem_base
->reg
, its
->mem_offset
);
3566 g_assert_not_reached();
3569 /* We now have a vector input register, so dup must succeed. */
3570 ok
= tcg_out_dup_vec(s
, vtype
, vece
, ots
->reg
, ots
->reg
);
3571 tcg_debug_assert(ok
);
3574 if (IS_DEAD_ARG(1)) {
3577 if (NEED_SYNC_ARG(0)) {
3578 temp_sync(s
, ots
, s
->reserved_regs
, 0, 0);
3580 if (IS_DEAD_ARG(0)) {
3585 static void tcg_reg_alloc_op(TCGContext
*s
, const TCGOp
*op
)
3587 const TCGLifeData arg_life
= op
->life
;
3588 const TCGOpDef
* const def
= &tcg_op_defs
[op
->opc
];
3589 TCGRegSet i_allocated_regs
;
3590 TCGRegSet o_allocated_regs
;
3591 int i
, k
, nb_iargs
, nb_oargs
;
3594 const TCGArgConstraint
*arg_ct
;
3596 TCGArg new_args
[TCG_MAX_OP_ARGS
];
3597 int const_args
[TCG_MAX_OP_ARGS
];
3599 nb_oargs
= def
->nb_oargs
;
3600 nb_iargs
= def
->nb_iargs
;
3602 /* copy constants */
3603 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
3604 op
->args
+ nb_oargs
+ nb_iargs
,
3605 sizeof(TCGArg
) * def
->nb_cargs
);
3607 i_allocated_regs
= s
->reserved_regs
;
3608 o_allocated_regs
= s
->reserved_regs
;
3610 /* satisfy input constraints */
3611 for (k
= 0; k
< nb_iargs
; k
++) {
3612 TCGRegSet i_preferred_regs
, o_preferred_regs
;
3614 i
= def
->sorted_args
[nb_oargs
+ k
];
3616 arg_ct
= &def
->args_ct
[i
];
3619 if (ts
->val_type
== TEMP_VAL_CONST
3620 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
)) {
3621 /* constant is OK for instruction */
3623 new_args
[i
] = ts
->val
;
3627 i_preferred_regs
= o_preferred_regs
= 0;
3628 if (arg_ct
->ct
& TCG_CT_IALIAS
) {
3629 o_preferred_regs
= op
->output_pref
[arg_ct
->alias_index
];
3630 if (ts
->fixed_reg
) {
3631 /* if fixed register, we must allocate a new register
3632 if the alias is not the same register */
3633 if (arg
!= op
->args
[arg_ct
->alias_index
]) {
3634 goto allocate_in_reg
;
3637 /* if the input is aliased to an output and if it is
3638 not dead after the instruction, we must allocate
3639 a new register and move it */
3640 if (!IS_DEAD_ARG(i
)) {
3641 goto allocate_in_reg
;
3644 /* check if the current register has already been allocated
3645 for another input aliased to an output */
3646 if (ts
->val_type
== TEMP_VAL_REG
) {
3649 for (k2
= 0 ; k2
< k
; k2
++) {
3650 i2
= def
->sorted_args
[nb_oargs
+ k2
];
3651 if ((def
->args_ct
[i2
].ct
& TCG_CT_IALIAS
) &&
3652 reg
== new_args
[i2
]) {
3653 goto allocate_in_reg
;
3657 i_preferred_regs
= o_preferred_regs
;
3661 temp_load(s
, ts
, arg_ct
->u
.regs
, i_allocated_regs
, i_preferred_regs
);
3664 if (tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
3665 /* nothing to do : the constraint is satisfied */
3668 /* allocate a new register matching the constraint
3669 and move the temporary register into it */
3670 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3671 i_allocated_regs
, 0);
3672 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, i_allocated_regs
,
3673 o_preferred_regs
, ts
->indirect_base
);
3674 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
3676 * Cross register class move not supported. Sync the
3677 * temp back to its slot and load from there.
3679 temp_sync(s
, ts
, i_allocated_regs
, 0, 0);
3680 tcg_out_ld(s
, ts
->type
, reg
,
3681 ts
->mem_base
->reg
, ts
->mem_offset
);
3686 tcg_regset_set_reg(i_allocated_regs
, reg
);
3689 /* mark dead temporaries and free the associated registers */
3690 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3691 if (IS_DEAD_ARG(i
)) {
3692 temp_dead(s
, arg_temp(op
->args
[i
]));
3696 if (def
->flags
& TCG_OPF_BB_END
) {
3697 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
3699 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
3700 /* XXX: permit generic clobber register list ? */
3701 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3702 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3703 tcg_reg_free(s
, i
, i_allocated_regs
);
3707 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
3708 /* sync globals if the op has side effects and might trigger
3710 sync_globals(s
, i_allocated_regs
);
3713 /* satisfy the output constraints */
3714 for(k
= 0; k
< nb_oargs
; k
++) {
3715 i
= def
->sorted_args
[k
];
3717 arg_ct
= &def
->args_ct
[i
];
3720 /* ENV should not be modified. */
3721 tcg_debug_assert(!ts
->fixed_reg
);
3723 if ((arg_ct
->ct
& TCG_CT_ALIAS
)
3724 && !const_args
[arg_ct
->alias_index
]) {
3725 reg
= new_args
[arg_ct
->alias_index
];
3726 } else if (arg_ct
->ct
& TCG_CT_NEWREG
) {
3727 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
,
3728 i_allocated_regs
| o_allocated_regs
,
3729 op
->output_pref
[k
], ts
->indirect_base
);
3731 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, o_allocated_regs
,
3732 op
->output_pref
[k
], ts
->indirect_base
);
3734 tcg_regset_set_reg(o_allocated_regs
, reg
);
3735 if (ts
->val_type
== TEMP_VAL_REG
) {
3736 s
->reg_to_temp
[ts
->reg
] = NULL
;
3738 ts
->val_type
= TEMP_VAL_REG
;
3741 * Temp value is modified, so the value kept in memory is
3742 * potentially not the same.
3744 ts
->mem_coherent
= 0;
3745 s
->reg_to_temp
[reg
] = ts
;
3750 /* emit instruction */
3751 if (def
->flags
& TCG_OPF_VECTOR
) {
3752 tcg_out_vec_op(s
, op
->opc
, TCGOP_VECL(op
), TCGOP_VECE(op
),
3753 new_args
, const_args
);
3755 tcg_out_op(s
, op
->opc
, new_args
, const_args
);
3758 /* move the outputs in the correct register if needed */
3759 for(i
= 0; i
< nb_oargs
; i
++) {
3760 ts
= arg_temp(op
->args
[i
]);
3762 /* ENV should not be modified. */
3763 tcg_debug_assert(!ts
->fixed_reg
);
3765 if (NEED_SYNC_ARG(i
)) {
3766 temp_sync(s
, ts
, o_allocated_regs
, 0, IS_DEAD_ARG(i
));
3767 } else if (IS_DEAD_ARG(i
)) {
3773 #ifdef TCG_TARGET_STACK_GROWSUP
3774 #define STACK_DIR(x) (-(x))
3776 #define STACK_DIR(x) (x)
3779 static void tcg_reg_alloc_call(TCGContext
*s
, TCGOp
*op
)
3781 const int nb_oargs
= TCGOP_CALLO(op
);
3782 const int nb_iargs
= TCGOP_CALLI(op
);
3783 const TCGLifeData arg_life
= op
->life
;
3784 int flags
, nb_regs
, i
;
3788 intptr_t stack_offset
;
3789 size_t call_stack_size
;
3790 tcg_insn_unit
*func_addr
;
3792 TCGRegSet allocated_regs
;
3794 func_addr
= (tcg_insn_unit
*)(intptr_t)op
->args
[nb_oargs
+ nb_iargs
];
3795 flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
3797 nb_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
3798 if (nb_regs
> nb_iargs
) {
3802 /* assign stack slots first */
3803 call_stack_size
= (nb_iargs
- nb_regs
) * sizeof(tcg_target_long
);
3804 call_stack_size
= (call_stack_size
+ TCG_TARGET_STACK_ALIGN
- 1) &
3805 ~(TCG_TARGET_STACK_ALIGN
- 1);
3806 allocate_args
= (call_stack_size
> TCG_STATIC_CALL_ARGS_SIZE
);
3807 if (allocate_args
) {
3808 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3809 preallocate call stack */
3813 stack_offset
= TCG_TARGET_CALL_STACK_OFFSET
;
3814 for (i
= nb_regs
; i
< nb_iargs
; i
++) {
3815 arg
= op
->args
[nb_oargs
+ i
];
3816 #ifdef TCG_TARGET_STACK_GROWSUP
3817 stack_offset
-= sizeof(tcg_target_long
);
3819 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3821 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3822 s
->reserved_regs
, 0);
3823 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
, stack_offset
);
3825 #ifndef TCG_TARGET_STACK_GROWSUP
3826 stack_offset
+= sizeof(tcg_target_long
);
3830 /* assign input registers */
3831 allocated_regs
= s
->reserved_regs
;
3832 for (i
= 0; i
< nb_regs
; i
++) {
3833 arg
= op
->args
[nb_oargs
+ i
];
3834 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3836 reg
= tcg_target_call_iarg_regs
[i
];
3838 if (ts
->val_type
== TEMP_VAL_REG
) {
3839 if (ts
->reg
!= reg
) {
3840 tcg_reg_free(s
, reg
, allocated_regs
);
3841 if (!tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
)) {
3843 * Cross register class move not supported. Sync the
3844 * temp back to its slot and load from there.
3846 temp_sync(s
, ts
, allocated_regs
, 0, 0);
3847 tcg_out_ld(s
, ts
->type
, reg
,
3848 ts
->mem_base
->reg
, ts
->mem_offset
);
3852 TCGRegSet arg_set
= 0;
3854 tcg_reg_free(s
, reg
, allocated_regs
);
3855 tcg_regset_set_reg(arg_set
, reg
);
3856 temp_load(s
, ts
, arg_set
, allocated_regs
, 0);
3859 tcg_regset_set_reg(allocated_regs
, reg
);
3863 /* mark dead temporaries and free the associated registers */
3864 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3865 if (IS_DEAD_ARG(i
)) {
3866 temp_dead(s
, arg_temp(op
->args
[i
]));
3870 /* clobber call registers */
3871 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3872 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3873 tcg_reg_free(s
, i
, allocated_regs
);
3877 /* Save globals if they might be written by the helper, sync them if
3878 they might be read. */
3879 if (flags
& TCG_CALL_NO_READ_GLOBALS
) {
3881 } else if (flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
3882 sync_globals(s
, allocated_regs
);
3884 save_globals(s
, allocated_regs
);
3887 tcg_out_call(s
, func_addr
);
3889 /* assign output registers and emit moves if needed */
3890 for(i
= 0; i
< nb_oargs
; i
++) {
3894 /* ENV should not be modified. */
3895 tcg_debug_assert(!ts
->fixed_reg
);
3897 reg
= tcg_target_call_oarg_regs
[i
];
3898 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
3899 if (ts
->val_type
== TEMP_VAL_REG
) {
3900 s
->reg_to_temp
[ts
->reg
] = NULL
;
3902 ts
->val_type
= TEMP_VAL_REG
;
3904 ts
->mem_coherent
= 0;
3905 s
->reg_to_temp
[reg
] = ts
;
3906 if (NEED_SYNC_ARG(i
)) {
3907 temp_sync(s
, ts
, allocated_regs
, 0, IS_DEAD_ARG(i
));
3908 } else if (IS_DEAD_ARG(i
)) {
3914 #ifdef CONFIG_PROFILER
3916 /* avoid copy/paste errors */
3917 #define PROF_ADD(to, from, field) \
3919 (to)->field += atomic_read(&((from)->field)); \
3922 #define PROF_MAX(to, from, field) \
3924 typeof((from)->field) val__ = atomic_read(&((from)->field)); \
3925 if (val__ > (to)->field) { \
3926 (to)->field = val__; \
3930 /* Pass in a zero'ed @prof */
3932 void tcg_profile_snapshot(TCGProfile
*prof
, bool counters
, bool table
)
3934 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
3937 for (i
= 0; i
< n_ctxs
; i
++) {
3938 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
3939 const TCGProfile
*orig
= &s
->prof
;
3942 PROF_ADD(prof
, orig
, cpu_exec_time
);
3943 PROF_ADD(prof
, orig
, tb_count1
);
3944 PROF_ADD(prof
, orig
, tb_count
);
3945 PROF_ADD(prof
, orig
, op_count
);
3946 PROF_MAX(prof
, orig
, op_count_max
);
3947 PROF_ADD(prof
, orig
, temp_count
);
3948 PROF_MAX(prof
, orig
, temp_count_max
);
3949 PROF_ADD(prof
, orig
, del_op_count
);
3950 PROF_ADD(prof
, orig
, code_in_len
);
3951 PROF_ADD(prof
, orig
, code_out_len
);
3952 PROF_ADD(prof
, orig
, search_out_len
);
3953 PROF_ADD(prof
, orig
, interm_time
);
3954 PROF_ADD(prof
, orig
, code_time
);
3955 PROF_ADD(prof
, orig
, la_time
);
3956 PROF_ADD(prof
, orig
, opt_time
);
3957 PROF_ADD(prof
, orig
, restore_count
);
3958 PROF_ADD(prof
, orig
, restore_time
);
3963 for (i
= 0; i
< NB_OPS
; i
++) {
3964 PROF_ADD(prof
, orig
, table_op_count
[i
]);
3973 static void tcg_profile_snapshot_counters(TCGProfile
*prof
)
3975 tcg_profile_snapshot(prof
, true, false);
3978 static void tcg_profile_snapshot_table(TCGProfile
*prof
)
3980 tcg_profile_snapshot(prof
, false, true);
3983 void tcg_dump_op_count(void)
3985 TCGProfile prof
= {};
3988 tcg_profile_snapshot_table(&prof
);
3989 for (i
= 0; i
< NB_OPS
; i
++) {
3990 qemu_printf("%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
3991 prof
.table_op_count
[i
]);
3995 int64_t tcg_cpu_exec_time(void)
3997 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
4001 for (i
= 0; i
< n_ctxs
; i
++) {
4002 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
4003 const TCGProfile
*prof
= &s
->prof
;
4005 ret
+= atomic_read(&prof
->cpu_exec_time
);
4010 void tcg_dump_op_count(void)
4012 qemu_printf("[TCG profiler not compiled]\n");
4015 int64_t tcg_cpu_exec_time(void)
4017 error_report("%s: TCG profiler not compiled", __func__
);
4023 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
)
4025 #ifdef CONFIG_PROFILER
4026 TCGProfile
*prof
= &s
->prof
;
4031 #ifdef CONFIG_PROFILER
4035 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4038 atomic_set(&prof
->op_count
, prof
->op_count
+ n
);
4039 if (n
> prof
->op_count_max
) {
4040 atomic_set(&prof
->op_count_max
, n
);
4044 atomic_set(&prof
->temp_count
, prof
->temp_count
+ n
);
4045 if (n
> prof
->temp_count_max
) {
4046 atomic_set(&prof
->temp_count_max
, n
);
4052 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
4053 && qemu_log_in_addr_range(tb
->pc
))) {
4054 FILE *logfile
= qemu_log_lock();
4056 tcg_dump_ops(s
, false);
4058 qemu_log_unlock(logfile
);
4062 #ifdef CONFIG_DEBUG_TCG
4063 /* Ensure all labels referenced have been emitted. */
4068 QSIMPLEQ_FOREACH(l
, &s
->labels
, next
) {
4069 if (unlikely(!l
->present
) && l
->refs
) {
4070 qemu_log_mask(CPU_LOG_TB_OP
,
4071 "$L%d referenced but not present.\n", l
->id
);
4079 #ifdef CONFIG_PROFILER
4080 atomic_set(&prof
->opt_time
, prof
->opt_time
- profile_getclock());
4083 #ifdef USE_TCG_OPTIMIZATIONS
4087 #ifdef CONFIG_PROFILER
4088 atomic_set(&prof
->opt_time
, prof
->opt_time
+ profile_getclock());
4089 atomic_set(&prof
->la_time
, prof
->la_time
- profile_getclock());
4092 reachable_code_pass(s
);
4095 if (s
->nb_indirects
> 0) {
4097 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
4098 && qemu_log_in_addr_range(tb
->pc
))) {
4099 FILE *logfile
= qemu_log_lock();
4100 qemu_log("OP before indirect lowering:\n");
4101 tcg_dump_ops(s
, false);
4103 qemu_log_unlock(logfile
);
4106 /* Replace indirect temps with direct temps. */
4107 if (liveness_pass_2(s
)) {
4108 /* If changes were made, re-run liveness. */
4113 #ifdef CONFIG_PROFILER
4114 atomic_set(&prof
->la_time
, prof
->la_time
+ profile_getclock());
4118 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
4119 && qemu_log_in_addr_range(tb
->pc
))) {
4120 FILE *logfile
= qemu_log_lock();
4121 qemu_log("OP after optimization and liveness analysis:\n");
4122 tcg_dump_ops(s
, true);
4124 qemu_log_unlock(logfile
);
4128 tcg_reg_alloc_start(s
);
4130 s
->code_buf
= tb
->tc
.ptr
;
4131 s
->code_ptr
= tb
->tc
.ptr
;
4133 #ifdef TCG_TARGET_NEED_LDST_LABELS
4134 QSIMPLEQ_INIT(&s
->ldst_labels
);
4136 #ifdef TCG_TARGET_NEED_POOL_LABELS
4137 s
->pool_labels
= NULL
;
4141 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
4142 TCGOpcode opc
= op
->opc
;
4144 #ifdef CONFIG_PROFILER
4145 atomic_set(&prof
->table_op_count
[opc
], prof
->table_op_count
[opc
] + 1);
4149 case INDEX_op_mov_i32
:
4150 case INDEX_op_mov_i64
:
4151 case INDEX_op_mov_vec
:
4152 tcg_reg_alloc_mov(s
, op
);
4154 case INDEX_op_movi_i32
:
4155 case INDEX_op_movi_i64
:
4156 case INDEX_op_dupi_vec
:
4157 tcg_reg_alloc_movi(s
, op
);
4159 case INDEX_op_dup_vec
:
4160 tcg_reg_alloc_dup(s
, op
);
4162 case INDEX_op_insn_start
:
4163 if (num_insns
>= 0) {
4164 size_t off
= tcg_current_code_size(s
);
4165 s
->gen_insn_end_off
[num_insns
] = off
;
4166 /* Assert that we do not overflow our stored offset. */
4167 assert(s
->gen_insn_end_off
[num_insns
] == off
);
4170 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
4172 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4173 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
4177 s
->gen_insn_data
[num_insns
][i
] = a
;
4180 case INDEX_op_discard
:
4181 temp_dead(s
, arg_temp(op
->args
[0]));
4183 case INDEX_op_set_label
:
4184 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
4185 tcg_out_label(s
, arg_label(op
->args
[0]), s
->code_ptr
);
4188 tcg_reg_alloc_call(s
, op
);
4191 /* Sanity check that we've not introduced any unhandled opcodes. */
4192 tcg_debug_assert(tcg_op_supported(opc
));
4193 /* Note: in order to speed up the code, it would be much
4194 faster to have specialized register allocator functions for
4195 some common argument patterns */
4196 tcg_reg_alloc_op(s
, op
);
4199 #ifdef CONFIG_DEBUG_TCG
4202 /* Test for (pending) buffer overflow. The assumption is that any
4203 one operation beginning below the high water mark cannot overrun
4204 the buffer completely. Thus we can test for overflow after
4205 generating code without having to check during generation. */
4206 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
4209 /* Test for TB overflow, as seen by gen_insn_end_off. */
4210 if (unlikely(tcg_current_code_size(s
) > UINT16_MAX
)) {
4214 tcg_debug_assert(num_insns
>= 0);
4215 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
4217 /* Generate TB finalization at the end of block */
4218 #ifdef TCG_TARGET_NEED_LDST_LABELS
4219 i
= tcg_out_ldst_finalize(s
);
4224 #ifdef TCG_TARGET_NEED_POOL_LABELS
4225 i
= tcg_out_pool_finalize(s
);
4230 if (!tcg_resolve_relocs(s
)) {
4234 /* flush instruction cache */
4235 flush_icache_range((uintptr_t)s
->code_buf
, (uintptr_t)s
->code_ptr
);
4237 return tcg_current_code_size(s
);
4240 #ifdef CONFIG_PROFILER
4241 void tcg_dump_info(void)
4243 TCGProfile prof
= {};
4244 const TCGProfile
*s
;
4246 int64_t tb_div_count
;
4249 tcg_profile_snapshot_counters(&prof
);
4251 tb_count
= s
->tb_count
;
4252 tb_div_count
= tb_count
? tb_count
: 1;
4253 tot
= s
->interm_time
+ s
->code_time
;
4255 qemu_printf("JIT cycles %" PRId64
" (%0.3f s at 2.4 GHz)\n",
4257 qemu_printf("translated TBs %" PRId64
" (aborted=%" PRId64
4259 tb_count
, s
->tb_count1
- tb_count
,
4260 (double)(s
->tb_count1
- s
->tb_count
)
4261 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
4262 qemu_printf("avg ops/TB %0.1f max=%d\n",
4263 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
4264 qemu_printf("deleted ops/TB %0.2f\n",
4265 (double)s
->del_op_count
/ tb_div_count
);
4266 qemu_printf("avg temps/TB %0.2f max=%d\n",
4267 (double)s
->temp_count
/ tb_div_count
, s
->temp_count_max
);
4268 qemu_printf("avg host code/TB %0.1f\n",
4269 (double)s
->code_out_len
/ tb_div_count
);
4270 qemu_printf("avg search data/TB %0.1f\n",
4271 (double)s
->search_out_len
/ tb_div_count
);
4273 qemu_printf("cycles/op %0.1f\n",
4274 s
->op_count
? (double)tot
/ s
->op_count
: 0);
4275 qemu_printf("cycles/in byte %0.1f\n",
4276 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
4277 qemu_printf("cycles/out byte %0.1f\n",
4278 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
4279 qemu_printf("cycles/search byte %0.1f\n",
4280 s
->search_out_len
? (double)tot
/ s
->search_out_len
: 0);
4284 qemu_printf(" gen_interm time %0.1f%%\n",
4285 (double)s
->interm_time
/ tot
* 100.0);
4286 qemu_printf(" gen_code time %0.1f%%\n",
4287 (double)s
->code_time
/ tot
* 100.0);
4288 qemu_printf("optim./code time %0.1f%%\n",
4289 (double)s
->opt_time
/ (s
->code_time
? s
->code_time
: 1)
4291 qemu_printf("liveness/code time %0.1f%%\n",
4292 (double)s
->la_time
/ (s
->code_time
? s
->code_time
: 1) * 100.0);
4293 qemu_printf("cpu_restore count %" PRId64
"\n",
4295 qemu_printf(" avg cycles %0.1f\n",
4296 s
->restore_count
? (double)s
->restore_time
/ s
->restore_count
: 0);
4299 void tcg_dump_info(void)
4301 qemu_printf("[TCG profiler not compiled]\n");
4305 #ifdef ELF_HOST_MACHINE
4306 /* In order to use this feature, the backend needs to do three things:
4308 (1) Define ELF_HOST_MACHINE to indicate both what value to
4309 put into the ELF image and to indicate support for the feature.
4311 (2) Define tcg_register_jit. This should create a buffer containing
4312 the contents of a .debug_frame section that describes the post-
4313 prologue unwind info for the tcg machine.
4315 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4318 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4325 struct jit_code_entry
{
4326 struct jit_code_entry
*next_entry
;
4327 struct jit_code_entry
*prev_entry
;
4328 const void *symfile_addr
;
4329 uint64_t symfile_size
;
4332 struct jit_descriptor
{
4334 uint32_t action_flag
;
4335 struct jit_code_entry
*relevant_entry
;
4336 struct jit_code_entry
*first_entry
;
4339 void __jit_debug_register_code(void) __attribute__((noinline
));
4340 void __jit_debug_register_code(void)
4345 /* Must statically initialize the version, because GDB may check
4346 the version before we can set it. */
4347 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
4349 /* End GDB interface. */
4351 static int find_string(const char *strtab
, const char *str
)
4353 const char *p
= strtab
+ 1;
4356 if (strcmp(p
, str
) == 0) {
4363 static void tcg_register_jit_int(void *buf_ptr
, size_t buf_size
,
4364 const void *debug_frame
,
4365 size_t debug_frame_size
)
4367 struct __attribute__((packed
)) DebugInfo
{
4374 uintptr_t cu_low_pc
;
4375 uintptr_t cu_high_pc
;
4378 uintptr_t fn_low_pc
;
4379 uintptr_t fn_high_pc
;
4388 struct DebugInfo di
;
4393 struct ElfImage
*img
;
4395 static const struct ElfImage img_template
= {
4397 .e_ident
[EI_MAG0
] = ELFMAG0
,
4398 .e_ident
[EI_MAG1
] = ELFMAG1
,
4399 .e_ident
[EI_MAG2
] = ELFMAG2
,
4400 .e_ident
[EI_MAG3
] = ELFMAG3
,
4401 .e_ident
[EI_CLASS
] = ELF_CLASS
,
4402 .e_ident
[EI_DATA
] = ELF_DATA
,
4403 .e_ident
[EI_VERSION
] = EV_CURRENT
,
4405 .e_machine
= ELF_HOST_MACHINE
,
4406 .e_version
= EV_CURRENT
,
4407 .e_phoff
= offsetof(struct ElfImage
, phdr
),
4408 .e_shoff
= offsetof(struct ElfImage
, shdr
),
4409 .e_ehsize
= sizeof(ElfW(Shdr
)),
4410 .e_phentsize
= sizeof(ElfW(Phdr
)),
4412 .e_shentsize
= sizeof(ElfW(Shdr
)),
4413 .e_shnum
= ARRAY_SIZE(img
->shdr
),
4414 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
4415 #ifdef ELF_HOST_FLAGS
4416 .e_flags
= ELF_HOST_FLAGS
,
4419 .e_ident
[EI_OSABI
] = ELF_OSABI
,
4427 [0] = { .sh_type
= SHT_NULL
},
4428 /* Trick: The contents of code_gen_buffer are not present in
4429 this fake ELF file; that got allocated elsewhere. Therefore
4430 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4431 will not look for contents. We can record any address. */
4433 .sh_type
= SHT_NOBITS
,
4434 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
4436 [2] = { /* .debug_info */
4437 .sh_type
= SHT_PROGBITS
,
4438 .sh_offset
= offsetof(struct ElfImage
, di
),
4439 .sh_size
= sizeof(struct DebugInfo
),
4441 [3] = { /* .debug_abbrev */
4442 .sh_type
= SHT_PROGBITS
,
4443 .sh_offset
= offsetof(struct ElfImage
, da
),
4444 .sh_size
= sizeof(img
->da
),
4446 [4] = { /* .debug_frame */
4447 .sh_type
= SHT_PROGBITS
,
4448 .sh_offset
= sizeof(struct ElfImage
),
4450 [5] = { /* .symtab */
4451 .sh_type
= SHT_SYMTAB
,
4452 .sh_offset
= offsetof(struct ElfImage
, sym
),
4453 .sh_size
= sizeof(img
->sym
),
4455 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
4456 .sh_entsize
= sizeof(ElfW(Sym
)),
4458 [6] = { /* .strtab */
4459 .sh_type
= SHT_STRTAB
,
4460 .sh_offset
= offsetof(struct ElfImage
, str
),
4461 .sh_size
= sizeof(img
->str
),
4465 [1] = { /* code_gen_buffer */
4466 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
4471 .len
= sizeof(struct DebugInfo
) - 4,
4473 .ptr_size
= sizeof(void *),
4475 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
4477 .fn_name
= "code_gen_buffer"
4480 1, /* abbrev number (the cu) */
4481 0x11, 1, /* DW_TAG_compile_unit, has children */
4482 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4483 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4484 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4485 0, 0, /* end of abbrev */
4486 2, /* abbrev number (the fn) */
4487 0x2e, 0, /* DW_TAG_subprogram, no children */
4488 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4489 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4490 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4491 0, 0, /* end of abbrev */
4492 0 /* no more abbrev */
4494 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4495 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
4498 /* We only need a single jit entry; statically allocate it. */
4499 static struct jit_code_entry one_entry
;
4501 uintptr_t buf
= (uintptr_t)buf_ptr
;
4502 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
4503 DebugFrameHeader
*dfh
;
4505 img
= g_malloc(img_size
);
4506 *img
= img_template
;
4508 img
->phdr
.p_vaddr
= buf
;
4509 img
->phdr
.p_paddr
= buf
;
4510 img
->phdr
.p_memsz
= buf_size
;
4512 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
4513 img
->shdr
[1].sh_addr
= buf
;
4514 img
->shdr
[1].sh_size
= buf_size
;
4516 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
4517 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
4519 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
4520 img
->shdr
[4].sh_size
= debug_frame_size
;
4522 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
4523 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
4525 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
4526 img
->sym
[1].st_value
= buf
;
4527 img
->sym
[1].st_size
= buf_size
;
4529 img
->di
.cu_low_pc
= buf
;
4530 img
->di
.cu_high_pc
= buf
+ buf_size
;
4531 img
->di
.fn_low_pc
= buf
;
4532 img
->di
.fn_high_pc
= buf
+ buf_size
;
4534 dfh
= (DebugFrameHeader
*)(img
+ 1);
4535 memcpy(dfh
, debug_frame
, debug_frame_size
);
4536 dfh
->fde
.func_start
= buf
;
4537 dfh
->fde
.func_len
= buf_size
;
4540 /* Enable this block to be able to debug the ELF image file creation.
4541 One can use readelf, objdump, or other inspection utilities. */
4543 FILE *f
= fopen("/tmp/qemu.jit", "w+b");
4545 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
4546 /* Avoid stupid unused return value warning for fwrite. */
4553 one_entry
.symfile_addr
= img
;
4554 one_entry
.symfile_size
= img_size
;
4556 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
4557 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
4558 __jit_debug_descriptor
.first_entry
= &one_entry
;
4559 __jit_debug_register_code();
4562 /* No support for the feature. Provide the entry point expected by exec.c,
4563 and implement the internal function we declared earlier. */
4565 static void tcg_register_jit_int(void *buf
, size_t size
,
4566 const void *debug_frame
,
4567 size_t debug_frame_size
)
4571 void tcg_register_jit(void *buf
, size_t buf_size
)
4574 #endif /* ELF_HOST_MACHINE */
4576 #if !TCG_TARGET_MAYBE_vec
4577 void tcg_expand_vec_op(TCGOpcode o
, TCGType t
, unsigned e
, TCGArg a0
, ...)
4579 g_assert_not_reached();