2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
28 #include "qemu/osdep.h"
30 /* Define to jump the ELF file used to communicate with GDB. */
33 #include "qemu/cutils.h"
34 #include "qemu/host-utils.h"
35 #include "qemu/timer.h"
37 /* Note: the long term plan is to reduce the dependencies on the QEMU
38 CPU definitions. Currently they are used for qemu_ld/st
40 #define NO_CPU_IO_DEFS
43 #include "exec/cpu-common.h"
44 #include "exec/exec-all.h"
48 #if UINTPTR_MAX == UINT32_MAX
49 # define ELF_CLASS ELFCLASS32
51 # define ELF_CLASS ELFCLASS64
53 #ifdef HOST_WORDS_BIGENDIAN
54 # define ELF_DATA ELFDATA2MSB
56 # define ELF_DATA ELFDATA2LSB
61 #include "sysemu/sysemu.h"
63 /* Forward declarations for functions declared in tcg-target.inc.c and
65 static void tcg_target_init(TCGContext
*s
);
66 static const TCGTargetOpDef
*tcg_target_op_def(TCGOpcode
);
67 static void tcg_target_qemu_prologue(TCGContext
*s
);
68 static void patch_reloc(tcg_insn_unit
*code_ptr
, int type
,
69 intptr_t value
, intptr_t addend
);
71 /* The CIE and FDE header definitions will be common to all hosts. */
73 uint32_t len
__attribute__((aligned((sizeof(void *)))));
79 uint8_t return_column
;
82 typedef struct QEMU_PACKED
{
83 uint32_t len
__attribute__((aligned((sizeof(void *)))));
87 } DebugFrameFDEHeader
;
89 typedef struct QEMU_PACKED
{
91 DebugFrameFDEHeader fde
;
94 static void tcg_register_jit_int(void *buf
, size_t size
,
95 const void *debug_frame
,
96 size_t debug_frame_size
)
97 __attribute__((unused
));
99 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
100 static const char *target_parse_constraint(TCGArgConstraint
*ct
,
101 const char *ct_str
, TCGType type
);
102 static void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg1
,
104 static void tcg_out_mov(TCGContext
*s
, TCGType type
, TCGReg ret
, TCGReg arg
);
105 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
106 TCGReg ret
, tcg_target_long arg
);
107 static void tcg_out_op(TCGContext
*s
, TCGOpcode opc
, const TCGArg
*args
,
108 const int *const_args
);
109 #if TCG_TARGET_MAYBE_vec
110 static void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
111 unsigned vece
, const TCGArg
*args
,
112 const int *const_args
);
114 static inline void tcg_out_vec_op(TCGContext
*s
, TCGOpcode opc
, unsigned vecl
,
115 unsigned vece
, const TCGArg
*args
,
116 const int *const_args
)
118 g_assert_not_reached();
121 static void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
, TCGReg arg1
,
123 static bool tcg_out_sti(TCGContext
*s
, TCGType type
, TCGArg val
,
124 TCGReg base
, intptr_t ofs
);
125 static void tcg_out_call(TCGContext
*s
, tcg_insn_unit
*target
);
126 static int tcg_target_const_match(tcg_target_long val
, TCGType type
,
127 const TCGArgConstraint
*arg_ct
);
128 #ifdef TCG_TARGET_NEED_LDST_LABELS
129 static bool tcg_out_ldst_finalize(TCGContext
*s
);
132 #define TCG_HIGHWATER 1024
134 static TCGContext
**tcg_ctxs
;
135 static unsigned int n_tcg_ctxs
;
136 TCGv_env cpu_env
= 0;
138 struct tcg_region_tree
{
141 /* padding to avoid false sharing is computed at run-time */
145 * We divide code_gen_buffer into equally-sized "regions" that TCG threads
146 * dynamically allocate from as demand dictates. Given appropriate region
147 * sizing, this minimizes flushes even when some TCG threads generate a lot
148 * more code than others.
150 struct tcg_region_state
{
153 /* fields set at init time */
158 size_t size
; /* size of one region */
159 size_t stride
; /* .size + guard size */
161 /* fields protected by the lock */
162 size_t current
; /* current region index */
163 size_t agg_size_full
; /* aggregate size of full regions */
166 static struct tcg_region_state region
;
168 * This is an array of struct tcg_region_tree's, with padding.
169 * We use void * to simplify the computation of region_trees[i]; each
170 * struct is found every tree_size bytes.
172 static void *region_trees
;
173 static size_t tree_size
;
174 static TCGRegSet tcg_target_available_regs
[TCG_TYPE_COUNT
];
175 static TCGRegSet tcg_target_call_clobber_regs
;
177 #if TCG_TARGET_INSN_UNIT_SIZE == 1
178 static __attribute__((unused
)) inline void tcg_out8(TCGContext
*s
, uint8_t v
)
183 static __attribute__((unused
)) inline void tcg_patch8(tcg_insn_unit
*p
,
190 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
191 static __attribute__((unused
)) inline void tcg_out16(TCGContext
*s
, uint16_t v
)
193 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
196 tcg_insn_unit
*p
= s
->code_ptr
;
197 memcpy(p
, &v
, sizeof(v
));
198 s
->code_ptr
= p
+ (2 / TCG_TARGET_INSN_UNIT_SIZE
);
202 static __attribute__((unused
)) inline void tcg_patch16(tcg_insn_unit
*p
,
205 if (TCG_TARGET_INSN_UNIT_SIZE
== 2) {
208 memcpy(p
, &v
, sizeof(v
));
213 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
214 static __attribute__((unused
)) inline void tcg_out32(TCGContext
*s
, uint32_t v
)
216 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
219 tcg_insn_unit
*p
= s
->code_ptr
;
220 memcpy(p
, &v
, sizeof(v
));
221 s
->code_ptr
= p
+ (4 / TCG_TARGET_INSN_UNIT_SIZE
);
225 static __attribute__((unused
)) inline void tcg_patch32(tcg_insn_unit
*p
,
228 if (TCG_TARGET_INSN_UNIT_SIZE
== 4) {
231 memcpy(p
, &v
, sizeof(v
));
236 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
237 static __attribute__((unused
)) inline void tcg_out64(TCGContext
*s
, uint64_t v
)
239 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
242 tcg_insn_unit
*p
= s
->code_ptr
;
243 memcpy(p
, &v
, sizeof(v
));
244 s
->code_ptr
= p
+ (8 / TCG_TARGET_INSN_UNIT_SIZE
);
248 static __attribute__((unused
)) inline void tcg_patch64(tcg_insn_unit
*p
,
251 if (TCG_TARGET_INSN_UNIT_SIZE
== 8) {
254 memcpy(p
, &v
, sizeof(v
));
259 /* label relocation processing */
261 static void tcg_out_reloc(TCGContext
*s
, tcg_insn_unit
*code_ptr
, int type
,
262 TCGLabel
*l
, intptr_t addend
)
267 /* FIXME: This may break relocations on RISC targets that
268 modify instruction fields in place. The caller may not have
269 written the initial value. */
270 patch_reloc(code_ptr
, type
, l
->u
.value
, addend
);
272 /* add a new relocation entry */
273 r
= tcg_malloc(sizeof(TCGRelocation
));
277 r
->next
= l
->u
.first_reloc
;
278 l
->u
.first_reloc
= r
;
282 static void tcg_out_label(TCGContext
*s
, TCGLabel
*l
, tcg_insn_unit
*ptr
)
284 intptr_t value
= (intptr_t)ptr
;
287 tcg_debug_assert(!l
->has_value
);
289 for (r
= l
->u
.first_reloc
; r
!= NULL
; r
= r
->next
) {
290 patch_reloc(r
->ptr
, r
->type
, value
, r
->addend
);
294 l
->u
.value_ptr
= ptr
;
297 TCGLabel
*gen_new_label(void)
299 TCGContext
*s
= tcg_ctx
;
300 TCGLabel
*l
= tcg_malloc(sizeof(TCGLabel
));
309 #include "tcg-target.inc.c"
311 /* compare a pointer @ptr and a tb_tc @s */
312 static int ptr_cmp_tb_tc(const void *ptr
, const struct tb_tc
*s
)
314 if (ptr
>= s
->ptr
+ s
->size
) {
316 } else if (ptr
< s
->ptr
) {
322 static gint
tb_tc_cmp(gconstpointer ap
, gconstpointer bp
)
324 const struct tb_tc
*a
= ap
;
325 const struct tb_tc
*b
= bp
;
328 * When both sizes are set, we know this isn't a lookup.
329 * This is the most likely case: every TB must be inserted; lookups
330 * are a lot less frequent.
332 if (likely(a
->size
&& b
->size
)) {
333 if (a
->ptr
> b
->ptr
) {
335 } else if (a
->ptr
< b
->ptr
) {
338 /* a->ptr == b->ptr should happen only on deletions */
339 g_assert(a
->size
== b
->size
);
343 * All lookups have either .size field set to 0.
344 * From the glib sources we see that @ap is always the lookup key. However
345 * the docs provide no guarantee, so we just mark this case as likely.
347 if (likely(a
->size
== 0)) {
348 return ptr_cmp_tb_tc(a
->ptr
, b
);
350 return ptr_cmp_tb_tc(b
->ptr
, a
);
353 static void tcg_region_trees_init(void)
357 tree_size
= ROUND_UP(sizeof(struct tcg_region_tree
), qemu_dcache_linesize
);
358 region_trees
= qemu_memalign(qemu_dcache_linesize
, region
.n
* tree_size
);
359 for (i
= 0; i
< region
.n
; i
++) {
360 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
362 qemu_mutex_init(&rt
->lock
);
363 rt
->tree
= g_tree_new(tb_tc_cmp
);
367 static struct tcg_region_tree
*tc_ptr_to_region_tree(void *p
)
371 if (p
< region
.start_aligned
) {
374 ptrdiff_t offset
= p
- region
.start_aligned
;
376 if (offset
> region
.stride
* (region
.n
- 1)) {
377 region_idx
= region
.n
- 1;
379 region_idx
= offset
/ region
.stride
;
382 return region_trees
+ region_idx
* tree_size
;
385 void tcg_tb_insert(TranslationBlock
*tb
)
387 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
389 qemu_mutex_lock(&rt
->lock
);
390 g_tree_insert(rt
->tree
, &tb
->tc
, tb
);
391 qemu_mutex_unlock(&rt
->lock
);
394 void tcg_tb_remove(TranslationBlock
*tb
)
396 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree(tb
->tc
.ptr
);
398 qemu_mutex_lock(&rt
->lock
);
399 g_tree_remove(rt
->tree
, &tb
->tc
);
400 qemu_mutex_unlock(&rt
->lock
);
404 * Find the TB 'tb' such that
405 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size
406 * Return NULL if not found.
408 TranslationBlock
*tcg_tb_lookup(uintptr_t tc_ptr
)
410 struct tcg_region_tree
*rt
= tc_ptr_to_region_tree((void *)tc_ptr
);
411 TranslationBlock
*tb
;
412 struct tb_tc s
= { .ptr
= (void *)tc_ptr
};
414 qemu_mutex_lock(&rt
->lock
);
415 tb
= g_tree_lookup(rt
->tree
, &s
);
416 qemu_mutex_unlock(&rt
->lock
);
420 static void tcg_region_tree_lock_all(void)
424 for (i
= 0; i
< region
.n
; i
++) {
425 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
427 qemu_mutex_lock(&rt
->lock
);
431 static void tcg_region_tree_unlock_all(void)
435 for (i
= 0; i
< region
.n
; i
++) {
436 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
438 qemu_mutex_unlock(&rt
->lock
);
442 void tcg_tb_foreach(GTraverseFunc func
, gpointer user_data
)
446 tcg_region_tree_lock_all();
447 for (i
= 0; i
< region
.n
; i
++) {
448 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
450 g_tree_foreach(rt
->tree
, func
, user_data
);
452 tcg_region_tree_unlock_all();
455 size_t tcg_nb_tbs(void)
460 tcg_region_tree_lock_all();
461 for (i
= 0; i
< region
.n
; i
++) {
462 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
464 nb_tbs
+= g_tree_nnodes(rt
->tree
);
466 tcg_region_tree_unlock_all();
470 static void tcg_region_tree_reset_all(void)
474 tcg_region_tree_lock_all();
475 for (i
= 0; i
< region
.n
; i
++) {
476 struct tcg_region_tree
*rt
= region_trees
+ i
* tree_size
;
478 /* Increment the refcount first so that destroy acts as a reset */
479 g_tree_ref(rt
->tree
);
480 g_tree_destroy(rt
->tree
);
482 tcg_region_tree_unlock_all();
485 static void tcg_region_bounds(size_t curr_region
, void **pstart
, void **pend
)
489 start
= region
.start_aligned
+ curr_region
* region
.stride
;
490 end
= start
+ region
.size
;
492 if (curr_region
== 0) {
493 start
= region
.start
;
495 if (curr_region
== region
.n
- 1) {
503 static void tcg_region_assign(TCGContext
*s
, size_t curr_region
)
507 tcg_region_bounds(curr_region
, &start
, &end
);
509 s
->code_gen_buffer
= start
;
510 s
->code_gen_ptr
= start
;
511 s
->code_gen_buffer_size
= end
- start
;
512 s
->code_gen_highwater
= end
- TCG_HIGHWATER
;
515 static bool tcg_region_alloc__locked(TCGContext
*s
)
517 if (region
.current
== region
.n
) {
520 tcg_region_assign(s
, region
.current
);
526 * Request a new region once the one in use has filled up.
527 * Returns true on error.
529 static bool tcg_region_alloc(TCGContext
*s
)
532 /* read the region size now; alloc__locked will overwrite it on success */
533 size_t size_full
= s
->code_gen_buffer_size
;
535 qemu_mutex_lock(®ion
.lock
);
536 err
= tcg_region_alloc__locked(s
);
538 region
.agg_size_full
+= size_full
- TCG_HIGHWATER
;
540 qemu_mutex_unlock(®ion
.lock
);
545 * Perform a context's first region allocation.
546 * This function does _not_ increment region.agg_size_full.
548 static inline bool tcg_region_initial_alloc__locked(TCGContext
*s
)
550 return tcg_region_alloc__locked(s
);
553 /* Call from a safe-work context */
554 void tcg_region_reset_all(void)
556 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
559 qemu_mutex_lock(®ion
.lock
);
561 region
.agg_size_full
= 0;
563 for (i
= 0; i
< n_ctxs
; i
++) {
564 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
565 bool err
= tcg_region_initial_alloc__locked(s
);
569 qemu_mutex_unlock(®ion
.lock
);
571 tcg_region_tree_reset_all();
574 #ifdef CONFIG_USER_ONLY
575 static size_t tcg_n_regions(void)
581 * It is likely that some vCPUs will translate more code than others, so we
582 * first try to set more regions than max_cpus, with those regions being of
583 * reasonable size. If that's not possible we make do by evenly dividing
584 * the code_gen_buffer among the vCPUs.
586 static size_t tcg_n_regions(void)
590 /* Use a single region if all we have is one vCPU thread */
591 if (max_cpus
== 1 || !qemu_tcg_mttcg_enabled()) {
595 /* Try to have more regions than max_cpus, with each region being >= 2 MB */
596 for (i
= 8; i
> 0; i
--) {
597 size_t regions_per_thread
= i
;
600 region_size
= tcg_init_ctx
.code_gen_buffer_size
;
601 region_size
/= max_cpus
* regions_per_thread
;
603 if (region_size
>= 2 * 1024u * 1024) {
604 return max_cpus
* regions_per_thread
;
607 /* If we can't, then just allocate one region per vCPU thread */
613 * Initializes region partitioning.
615 * Called at init time from the parent thread (i.e. the one calling
616 * tcg_context_init), after the target's TCG globals have been set.
618 * Region partitioning works by splitting code_gen_buffer into separate regions,
619 * and then assigning regions to TCG threads so that the threads can translate
620 * code in parallel without synchronization.
622 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at
623 * least max_cpus regions in MTTCG. In !MTTCG we use a single region.
624 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...])
625 * must have been parsed before calling this function, since it calls
626 * qemu_tcg_mttcg_enabled().
628 * In user-mode we use a single region. Having multiple regions in user-mode
629 * is not supported, because the number of vCPU threads (recall that each thread
630 * spawned by the guest corresponds to a vCPU thread) is only bounded by the
631 * OS, and usually this number is huge (tens of thousands is not uncommon).
632 * Thus, given this large bound on the number of vCPU threads and the fact
633 * that code_gen_buffer is allocated at compile-time, we cannot guarantee
634 * that the availability of at least one region per vCPU thread.
636 * However, this user-mode limitation is unlikely to be a significant problem
637 * in practice. Multi-threaded guests share most if not all of their translated
638 * code, which makes parallel code generation less appealing than in softmmu.
640 void tcg_region_init(void)
642 void *buf
= tcg_init_ctx
.code_gen_buffer
;
644 size_t size
= tcg_init_ctx
.code_gen_buffer_size
;
645 size_t page_size
= qemu_real_host_page_size
;
650 n_regions
= tcg_n_regions();
652 /* The first region will be 'aligned - buf' bytes larger than the others */
653 aligned
= QEMU_ALIGN_PTR_UP(buf
, page_size
);
654 g_assert(aligned
< tcg_init_ctx
.code_gen_buffer
+ size
);
656 * Make region_size a multiple of page_size, using aligned as the start.
657 * As a result of this we might end up with a few extra pages at the end of
658 * the buffer; we will assign those to the last region.
660 region_size
= (size
- (aligned
- buf
)) / n_regions
;
661 region_size
= QEMU_ALIGN_DOWN(region_size
, page_size
);
663 /* A region must have at least 2 pages; one code, one guard */
664 g_assert(region_size
>= 2 * page_size
);
666 /* init the region struct */
667 qemu_mutex_init(®ion
.lock
);
668 region
.n
= n_regions
;
669 region
.size
= region_size
- page_size
;
670 region
.stride
= region_size
;
672 region
.start_aligned
= aligned
;
673 /* page-align the end, since its last page will be a guard page */
674 region
.end
= QEMU_ALIGN_PTR_DOWN(buf
+ size
, page_size
);
675 /* account for that last guard page */
676 region
.end
-= page_size
;
678 /* set guard pages */
679 for (i
= 0; i
< region
.n
; i
++) {
683 tcg_region_bounds(i
, &start
, &end
);
684 rc
= qemu_mprotect_none(end
, page_size
);
688 tcg_region_trees_init();
690 /* In user-mode we support only one ctx, so do the initial allocation now */
691 #ifdef CONFIG_USER_ONLY
693 bool err
= tcg_region_initial_alloc__locked(tcg_ctx
);
701 * All TCG threads except the parent (i.e. the one that called tcg_context_init
702 * and registered the target's TCG globals) must register with this function
703 * before initiating translation.
705 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
706 * of tcg_region_init() for the reasoning behind this.
708 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
709 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
710 * is not used anymore for translation once this function is called.
712 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
713 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
715 #ifdef CONFIG_USER_ONLY
716 void tcg_register_thread(void)
718 tcg_ctx
= &tcg_init_ctx
;
721 void tcg_register_thread(void)
723 TCGContext
*s
= g_malloc(sizeof(*s
));
729 /* Relink mem_base. */
730 for (i
= 0, n
= tcg_init_ctx
.nb_globals
; i
< n
; ++i
) {
731 if (tcg_init_ctx
.temps
[i
].mem_base
) {
732 ptrdiff_t b
= tcg_init_ctx
.temps
[i
].mem_base
- tcg_init_ctx
.temps
;
733 tcg_debug_assert(b
>= 0 && b
< n
);
734 s
->temps
[i
].mem_base
= &s
->temps
[b
];
738 /* Claim an entry in tcg_ctxs */
739 n
= atomic_fetch_inc(&n_tcg_ctxs
);
740 g_assert(n
< max_cpus
);
741 atomic_set(&tcg_ctxs
[n
], s
);
744 qemu_mutex_lock(®ion
.lock
);
745 err
= tcg_region_initial_alloc__locked(tcg_ctx
);
747 qemu_mutex_unlock(®ion
.lock
);
749 #endif /* !CONFIG_USER_ONLY */
752 * Returns the size (in bytes) of all translated code (i.e. from all regions)
753 * currently in the cache.
754 * See also: tcg_code_capacity()
755 * Do not confuse with tcg_current_code_size(); that one applies to a single
758 size_t tcg_code_size(void)
760 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
764 qemu_mutex_lock(®ion
.lock
);
765 total
= region
.agg_size_full
;
766 for (i
= 0; i
< n_ctxs
; i
++) {
767 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
770 size
= atomic_read(&s
->code_gen_ptr
) - s
->code_gen_buffer
;
771 g_assert(size
<= s
->code_gen_buffer_size
);
774 qemu_mutex_unlock(®ion
.lock
);
779 * Returns the code capacity (in bytes) of the entire cache, i.e. including all
781 * See also: tcg_code_size()
783 size_t tcg_code_capacity(void)
785 size_t guard_size
, capacity
;
787 /* no need for synchronization; these variables are set at init time */
788 guard_size
= region
.stride
- region
.size
;
789 capacity
= region
.end
+ guard_size
- region
.start
;
790 capacity
-= region
.n
* (guard_size
+ TCG_HIGHWATER
);
794 size_t tcg_tb_phys_invalidate_count(void)
796 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
800 for (i
= 0; i
< n_ctxs
; i
++) {
801 const TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
803 total
+= atomic_read(&s
->tb_phys_invalidate_count
);
808 /* pool based memory allocation */
809 void *tcg_malloc_internal(TCGContext
*s
, int size
)
814 if (size
> TCG_POOL_CHUNK_SIZE
) {
815 /* big malloc: insert a new pool (XXX: could optimize) */
816 p
= g_malloc(sizeof(TCGPool
) + size
);
818 p
->next
= s
->pool_first_large
;
819 s
->pool_first_large
= p
;
830 pool_size
= TCG_POOL_CHUNK_SIZE
;
831 p
= g_malloc(sizeof(TCGPool
) + pool_size
);
835 s
->pool_current
->next
= p
;
844 s
->pool_cur
= p
->data
+ size
;
845 s
->pool_end
= p
->data
+ p
->size
;
849 void tcg_pool_reset(TCGContext
*s
)
852 for (p
= s
->pool_first_large
; p
; p
= t
) {
856 s
->pool_first_large
= NULL
;
857 s
->pool_cur
= s
->pool_end
= NULL
;
858 s
->pool_current
= NULL
;
861 typedef struct TCGHelperInfo
{
868 #include "exec/helper-proto.h"
870 static const TCGHelperInfo all_helpers
[] = {
871 #include "exec/helper-tcg.h"
873 static GHashTable
*helper_table
;
875 static int indirect_reg_alloc_order
[ARRAY_SIZE(tcg_target_reg_alloc_order
)];
876 static void process_op_defs(TCGContext
*s
);
877 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
878 TCGReg reg
, const char *name
);
880 void tcg_context_init(TCGContext
*s
)
882 int op
, total_args
, n
, i
;
884 TCGArgConstraint
*args_ct
;
888 memset(s
, 0, sizeof(*s
));
891 /* Count total number of arguments and allocate the corresponding
894 for(op
= 0; op
< NB_OPS
; op
++) {
895 def
= &tcg_op_defs
[op
];
896 n
= def
->nb_iargs
+ def
->nb_oargs
;
900 args_ct
= g_malloc(sizeof(TCGArgConstraint
) * total_args
);
901 sorted_args
= g_malloc(sizeof(int) * total_args
);
903 for(op
= 0; op
< NB_OPS
; op
++) {
904 def
= &tcg_op_defs
[op
];
905 def
->args_ct
= args_ct
;
906 def
->sorted_args
= sorted_args
;
907 n
= def
->nb_iargs
+ def
->nb_oargs
;
912 /* Register helpers. */
913 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
914 helper_table
= g_hash_table_new(NULL
, NULL
);
916 for (i
= 0; i
< ARRAY_SIZE(all_helpers
); ++i
) {
917 g_hash_table_insert(helper_table
, (gpointer
)all_helpers
[i
].func
,
918 (gpointer
)&all_helpers
[i
]);
924 /* Reverse the order of the saved registers, assuming they're all at
925 the start of tcg_target_reg_alloc_order. */
926 for (n
= 0; n
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++n
) {
927 int r
= tcg_target_reg_alloc_order
[n
];
928 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, r
)) {
932 for (i
= 0; i
< n
; ++i
) {
933 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[n
- 1 - i
];
935 for (; i
< ARRAY_SIZE(tcg_target_reg_alloc_order
); ++i
) {
936 indirect_reg_alloc_order
[i
] = tcg_target_reg_alloc_order
[i
];
941 * In user-mode we simply share the init context among threads, since we
942 * use a single region. See the documentation tcg_region_init() for the
943 * reasoning behind this.
944 * In softmmu we will have at most max_cpus TCG threads.
946 #ifdef CONFIG_USER_ONLY
950 tcg_ctxs
= g_new(TCGContext
*, max_cpus
);
953 tcg_debug_assert(!tcg_regset_test_reg(s
->reserved_regs
, TCG_AREG0
));
954 ts
= tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, TCG_AREG0
, "env");
955 cpu_env
= temp_tcgv_ptr(ts
);
959 * Allocate TBs right before their corresponding translated code, making
960 * sure that TBs and code are on different cache lines.
962 TranslationBlock
*tcg_tb_alloc(TCGContext
*s
)
964 uintptr_t align
= qemu_icache_linesize
;
965 TranslationBlock
*tb
;
969 tb
= (void *)ROUND_UP((uintptr_t)s
->code_gen_ptr
, align
);
970 next
= (void *)ROUND_UP((uintptr_t)(tb
+ 1), align
);
972 if (unlikely(next
> s
->code_gen_highwater
)) {
973 if (tcg_region_alloc(s
)) {
978 atomic_set(&s
->code_gen_ptr
, next
);
979 s
->data_gen_ptr
= NULL
;
983 void tcg_prologue_init(TCGContext
*s
)
985 size_t prologue_size
, total_size
;
988 /* Put the prologue at the beginning of code_gen_buffer. */
989 buf0
= s
->code_gen_buffer
;
990 total_size
= s
->code_gen_buffer_size
;
993 s
->data_gen_ptr
= NULL
;
994 s
->code_gen_prologue
= buf0
;
996 /* Compute a high-water mark, at which we voluntarily flush the buffer
997 and start over. The size here is arbitrary, significantly larger
998 than we expect the code generation for any one opcode to require. */
999 s
->code_gen_highwater
= s
->code_gen_buffer
+ (total_size
- TCG_HIGHWATER
);
1001 #ifdef TCG_TARGET_NEED_POOL_LABELS
1002 s
->pool_labels
= NULL
;
1005 /* Generate the prologue. */
1006 tcg_target_qemu_prologue(s
);
1008 #ifdef TCG_TARGET_NEED_POOL_LABELS
1009 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1011 bool ok
= tcg_out_pool_finalize(s
);
1012 tcg_debug_assert(ok
);
1017 flush_icache_range((uintptr_t)buf0
, (uintptr_t)buf1
);
1019 /* Deduct the prologue from the buffer. */
1020 prologue_size
= tcg_current_code_size(s
);
1021 s
->code_gen_ptr
= buf1
;
1022 s
->code_gen_buffer
= buf1
;
1024 total_size
-= prologue_size
;
1025 s
->code_gen_buffer_size
= total_size
;
1027 tcg_register_jit(s
->code_gen_buffer
, total_size
);
1030 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
1032 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size
);
1033 if (s
->data_gen_ptr
) {
1034 size_t code_size
= s
->data_gen_ptr
- buf0
;
1035 size_t data_size
= prologue_size
- code_size
;
1038 log_disas(buf0
, code_size
);
1040 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1041 if (sizeof(tcg_target_ulong
) == 8) {
1042 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1043 (uintptr_t)s
->data_gen_ptr
+ i
,
1044 *(uint64_t *)(s
->data_gen_ptr
+ i
));
1046 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1047 (uintptr_t)s
->data_gen_ptr
+ i
,
1048 *(uint32_t *)(s
->data_gen_ptr
+ i
));
1052 log_disas(buf0
, prologue_size
);
1060 /* Assert that goto_ptr is implemented completely. */
1061 if (TCG_TARGET_HAS_goto_ptr
) {
1062 tcg_debug_assert(s
->code_gen_epilogue
!= NULL
);
1066 void tcg_func_start(TCGContext
*s
)
1069 s
->nb_temps
= s
->nb_globals
;
1071 /* No temps have been previously allocated for size or locality. */
1072 memset(s
->free_temps
, 0, sizeof(s
->free_temps
));
1076 s
->current_frame_offset
= s
->frame_start
;
1078 #ifdef CONFIG_DEBUG_TCG
1079 s
->goto_tb_issue_mask
= 0;
1082 QTAILQ_INIT(&s
->ops
);
1083 QTAILQ_INIT(&s
->free_ops
);
1086 static inline TCGTemp
*tcg_temp_alloc(TCGContext
*s
)
1088 int n
= s
->nb_temps
++;
1089 tcg_debug_assert(n
< TCG_MAX_TEMPS
);
1090 return memset(&s
->temps
[n
], 0, sizeof(TCGTemp
));
1093 static inline TCGTemp
*tcg_global_alloc(TCGContext
*s
)
1097 tcg_debug_assert(s
->nb_globals
== s
->nb_temps
);
1099 ts
= tcg_temp_alloc(s
);
1100 ts
->temp_global
= 1;
1105 static TCGTemp
*tcg_global_reg_new_internal(TCGContext
*s
, TCGType type
,
1106 TCGReg reg
, const char *name
)
1110 if (TCG_TARGET_REG_BITS
== 32 && type
!= TCG_TYPE_I32
) {
1114 ts
= tcg_global_alloc(s
);
1115 ts
->base_type
= type
;
1120 tcg_regset_set_reg(s
->reserved_regs
, reg
);
1125 void tcg_set_frame(TCGContext
*s
, TCGReg reg
, intptr_t start
, intptr_t size
)
1127 s
->frame_start
= start
;
1128 s
->frame_end
= start
+ size
;
1130 = tcg_global_reg_new_internal(s
, TCG_TYPE_PTR
, reg
, "_frame");
1133 TCGTemp
*tcg_global_mem_new_internal(TCGType type
, TCGv_ptr base
,
1134 intptr_t offset
, const char *name
)
1136 TCGContext
*s
= tcg_ctx
;
1137 TCGTemp
*base_ts
= tcgv_ptr_temp(base
);
1138 TCGTemp
*ts
= tcg_global_alloc(s
);
1139 int indirect_reg
= 0, bigendian
= 0;
1140 #ifdef HOST_WORDS_BIGENDIAN
1144 if (!base_ts
->fixed_reg
) {
1145 /* We do not support double-indirect registers. */
1146 tcg_debug_assert(!base_ts
->indirect_reg
);
1147 base_ts
->indirect_base
= 1;
1148 s
->nb_indirects
+= (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
1153 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1154 TCGTemp
*ts2
= tcg_global_alloc(s
);
1157 ts
->base_type
= TCG_TYPE_I64
;
1158 ts
->type
= TCG_TYPE_I32
;
1159 ts
->indirect_reg
= indirect_reg
;
1160 ts
->mem_allocated
= 1;
1161 ts
->mem_base
= base_ts
;
1162 ts
->mem_offset
= offset
+ bigendian
* 4;
1163 pstrcpy(buf
, sizeof(buf
), name
);
1164 pstrcat(buf
, sizeof(buf
), "_0");
1165 ts
->name
= strdup(buf
);
1167 tcg_debug_assert(ts2
== ts
+ 1);
1168 ts2
->base_type
= TCG_TYPE_I64
;
1169 ts2
->type
= TCG_TYPE_I32
;
1170 ts2
->indirect_reg
= indirect_reg
;
1171 ts2
->mem_allocated
= 1;
1172 ts2
->mem_base
= base_ts
;
1173 ts2
->mem_offset
= offset
+ (1 - bigendian
) * 4;
1174 pstrcpy(buf
, sizeof(buf
), name
);
1175 pstrcat(buf
, sizeof(buf
), "_1");
1176 ts2
->name
= strdup(buf
);
1178 ts
->base_type
= type
;
1180 ts
->indirect_reg
= indirect_reg
;
1181 ts
->mem_allocated
= 1;
1182 ts
->mem_base
= base_ts
;
1183 ts
->mem_offset
= offset
;
1189 TCGTemp
*tcg_temp_new_internal(TCGType type
, bool temp_local
)
1191 TCGContext
*s
= tcg_ctx
;
1195 k
= type
+ (temp_local
? TCG_TYPE_COUNT
: 0);
1196 idx
= find_first_bit(s
->free_temps
[k
].l
, TCG_MAX_TEMPS
);
1197 if (idx
< TCG_MAX_TEMPS
) {
1198 /* There is already an available temp with the right type. */
1199 clear_bit(idx
, s
->free_temps
[k
].l
);
1201 ts
= &s
->temps
[idx
];
1202 ts
->temp_allocated
= 1;
1203 tcg_debug_assert(ts
->base_type
== type
);
1204 tcg_debug_assert(ts
->temp_local
== temp_local
);
1206 ts
= tcg_temp_alloc(s
);
1207 if (TCG_TARGET_REG_BITS
== 32 && type
== TCG_TYPE_I64
) {
1208 TCGTemp
*ts2
= tcg_temp_alloc(s
);
1210 ts
->base_type
= type
;
1211 ts
->type
= TCG_TYPE_I32
;
1212 ts
->temp_allocated
= 1;
1213 ts
->temp_local
= temp_local
;
1215 tcg_debug_assert(ts2
== ts
+ 1);
1216 ts2
->base_type
= TCG_TYPE_I64
;
1217 ts2
->type
= TCG_TYPE_I32
;
1218 ts2
->temp_allocated
= 1;
1219 ts2
->temp_local
= temp_local
;
1221 ts
->base_type
= type
;
1223 ts
->temp_allocated
= 1;
1224 ts
->temp_local
= temp_local
;
1228 #if defined(CONFIG_DEBUG_TCG)
1234 TCGv_vec
tcg_temp_new_vec(TCGType type
)
1238 #ifdef CONFIG_DEBUG_TCG
1241 assert(TCG_TARGET_HAS_v64
);
1244 assert(TCG_TARGET_HAS_v128
);
1247 assert(TCG_TARGET_HAS_v256
);
1250 g_assert_not_reached();
1254 t
= tcg_temp_new_internal(type
, 0);
1255 return temp_tcgv_vec(t
);
1258 /* Create a new temp of the same type as an existing temp. */
1259 TCGv_vec
tcg_temp_new_vec_matching(TCGv_vec match
)
1261 TCGTemp
*t
= tcgv_vec_temp(match
);
1263 tcg_debug_assert(t
->temp_allocated
!= 0);
1265 t
= tcg_temp_new_internal(t
->base_type
, 0);
1266 return temp_tcgv_vec(t
);
1269 void tcg_temp_free_internal(TCGTemp
*ts
)
1271 TCGContext
*s
= tcg_ctx
;
1274 #if defined(CONFIG_DEBUG_TCG)
1276 if (s
->temps_in_use
< 0) {
1277 fprintf(stderr
, "More temporaries freed than allocated!\n");
1281 tcg_debug_assert(ts
->temp_global
== 0);
1282 tcg_debug_assert(ts
->temp_allocated
!= 0);
1283 ts
->temp_allocated
= 0;
1286 k
= ts
->base_type
+ (ts
->temp_local
? TCG_TYPE_COUNT
: 0);
1287 set_bit(idx
, s
->free_temps
[k
].l
);
1290 TCGv_i32
tcg_const_i32(int32_t val
)
1293 t0
= tcg_temp_new_i32();
1294 tcg_gen_movi_i32(t0
, val
);
1298 TCGv_i64
tcg_const_i64(int64_t val
)
1301 t0
= tcg_temp_new_i64();
1302 tcg_gen_movi_i64(t0
, val
);
1306 TCGv_i32
tcg_const_local_i32(int32_t val
)
1309 t0
= tcg_temp_local_new_i32();
1310 tcg_gen_movi_i32(t0
, val
);
1314 TCGv_i64
tcg_const_local_i64(int64_t val
)
1317 t0
= tcg_temp_local_new_i64();
1318 tcg_gen_movi_i64(t0
, val
);
1322 #if defined(CONFIG_DEBUG_TCG)
1323 void tcg_clear_temp_count(void)
1325 TCGContext
*s
= tcg_ctx
;
1326 s
->temps_in_use
= 0;
1329 int tcg_check_temp_count(void)
1331 TCGContext
*s
= tcg_ctx
;
1332 if (s
->temps_in_use
) {
1333 /* Clear the count so that we don't give another
1334 * warning immediately next time around.
1336 s
->temps_in_use
= 0;
1343 /* Return true if OP may appear in the opcode stream.
1344 Test the runtime variable that controls each opcode. */
1345 bool tcg_op_supported(TCGOpcode op
)
1348 = TCG_TARGET_HAS_v64
| TCG_TARGET_HAS_v128
| TCG_TARGET_HAS_v256
;
1351 case INDEX_op_discard
:
1352 case INDEX_op_set_label
:
1356 case INDEX_op_insn_start
:
1357 case INDEX_op_exit_tb
:
1358 case INDEX_op_goto_tb
:
1359 case INDEX_op_qemu_ld_i32
:
1360 case INDEX_op_qemu_st_i32
:
1361 case INDEX_op_qemu_ld_i64
:
1362 case INDEX_op_qemu_st_i64
:
1365 case INDEX_op_goto_ptr
:
1366 return TCG_TARGET_HAS_goto_ptr
;
1368 case INDEX_op_mov_i32
:
1369 case INDEX_op_movi_i32
:
1370 case INDEX_op_setcond_i32
:
1371 case INDEX_op_brcond_i32
:
1372 case INDEX_op_ld8u_i32
:
1373 case INDEX_op_ld8s_i32
:
1374 case INDEX_op_ld16u_i32
:
1375 case INDEX_op_ld16s_i32
:
1376 case INDEX_op_ld_i32
:
1377 case INDEX_op_st8_i32
:
1378 case INDEX_op_st16_i32
:
1379 case INDEX_op_st_i32
:
1380 case INDEX_op_add_i32
:
1381 case INDEX_op_sub_i32
:
1382 case INDEX_op_mul_i32
:
1383 case INDEX_op_and_i32
:
1384 case INDEX_op_or_i32
:
1385 case INDEX_op_xor_i32
:
1386 case INDEX_op_shl_i32
:
1387 case INDEX_op_shr_i32
:
1388 case INDEX_op_sar_i32
:
1391 case INDEX_op_movcond_i32
:
1392 return TCG_TARGET_HAS_movcond_i32
;
1393 case INDEX_op_div_i32
:
1394 case INDEX_op_divu_i32
:
1395 return TCG_TARGET_HAS_div_i32
;
1396 case INDEX_op_rem_i32
:
1397 case INDEX_op_remu_i32
:
1398 return TCG_TARGET_HAS_rem_i32
;
1399 case INDEX_op_div2_i32
:
1400 case INDEX_op_divu2_i32
:
1401 return TCG_TARGET_HAS_div2_i32
;
1402 case INDEX_op_rotl_i32
:
1403 case INDEX_op_rotr_i32
:
1404 return TCG_TARGET_HAS_rot_i32
;
1405 case INDEX_op_deposit_i32
:
1406 return TCG_TARGET_HAS_deposit_i32
;
1407 case INDEX_op_extract_i32
:
1408 return TCG_TARGET_HAS_extract_i32
;
1409 case INDEX_op_sextract_i32
:
1410 return TCG_TARGET_HAS_sextract_i32
;
1411 case INDEX_op_add2_i32
:
1412 return TCG_TARGET_HAS_add2_i32
;
1413 case INDEX_op_sub2_i32
:
1414 return TCG_TARGET_HAS_sub2_i32
;
1415 case INDEX_op_mulu2_i32
:
1416 return TCG_TARGET_HAS_mulu2_i32
;
1417 case INDEX_op_muls2_i32
:
1418 return TCG_TARGET_HAS_muls2_i32
;
1419 case INDEX_op_muluh_i32
:
1420 return TCG_TARGET_HAS_muluh_i32
;
1421 case INDEX_op_mulsh_i32
:
1422 return TCG_TARGET_HAS_mulsh_i32
;
1423 case INDEX_op_ext8s_i32
:
1424 return TCG_TARGET_HAS_ext8s_i32
;
1425 case INDEX_op_ext16s_i32
:
1426 return TCG_TARGET_HAS_ext16s_i32
;
1427 case INDEX_op_ext8u_i32
:
1428 return TCG_TARGET_HAS_ext8u_i32
;
1429 case INDEX_op_ext16u_i32
:
1430 return TCG_TARGET_HAS_ext16u_i32
;
1431 case INDEX_op_bswap16_i32
:
1432 return TCG_TARGET_HAS_bswap16_i32
;
1433 case INDEX_op_bswap32_i32
:
1434 return TCG_TARGET_HAS_bswap32_i32
;
1435 case INDEX_op_not_i32
:
1436 return TCG_TARGET_HAS_not_i32
;
1437 case INDEX_op_neg_i32
:
1438 return TCG_TARGET_HAS_neg_i32
;
1439 case INDEX_op_andc_i32
:
1440 return TCG_TARGET_HAS_andc_i32
;
1441 case INDEX_op_orc_i32
:
1442 return TCG_TARGET_HAS_orc_i32
;
1443 case INDEX_op_eqv_i32
:
1444 return TCG_TARGET_HAS_eqv_i32
;
1445 case INDEX_op_nand_i32
:
1446 return TCG_TARGET_HAS_nand_i32
;
1447 case INDEX_op_nor_i32
:
1448 return TCG_TARGET_HAS_nor_i32
;
1449 case INDEX_op_clz_i32
:
1450 return TCG_TARGET_HAS_clz_i32
;
1451 case INDEX_op_ctz_i32
:
1452 return TCG_TARGET_HAS_ctz_i32
;
1453 case INDEX_op_ctpop_i32
:
1454 return TCG_TARGET_HAS_ctpop_i32
;
1456 case INDEX_op_brcond2_i32
:
1457 case INDEX_op_setcond2_i32
:
1458 return TCG_TARGET_REG_BITS
== 32;
1460 case INDEX_op_mov_i64
:
1461 case INDEX_op_movi_i64
:
1462 case INDEX_op_setcond_i64
:
1463 case INDEX_op_brcond_i64
:
1464 case INDEX_op_ld8u_i64
:
1465 case INDEX_op_ld8s_i64
:
1466 case INDEX_op_ld16u_i64
:
1467 case INDEX_op_ld16s_i64
:
1468 case INDEX_op_ld32u_i64
:
1469 case INDEX_op_ld32s_i64
:
1470 case INDEX_op_ld_i64
:
1471 case INDEX_op_st8_i64
:
1472 case INDEX_op_st16_i64
:
1473 case INDEX_op_st32_i64
:
1474 case INDEX_op_st_i64
:
1475 case INDEX_op_add_i64
:
1476 case INDEX_op_sub_i64
:
1477 case INDEX_op_mul_i64
:
1478 case INDEX_op_and_i64
:
1479 case INDEX_op_or_i64
:
1480 case INDEX_op_xor_i64
:
1481 case INDEX_op_shl_i64
:
1482 case INDEX_op_shr_i64
:
1483 case INDEX_op_sar_i64
:
1484 case INDEX_op_ext_i32_i64
:
1485 case INDEX_op_extu_i32_i64
:
1486 return TCG_TARGET_REG_BITS
== 64;
1488 case INDEX_op_movcond_i64
:
1489 return TCG_TARGET_HAS_movcond_i64
;
1490 case INDEX_op_div_i64
:
1491 case INDEX_op_divu_i64
:
1492 return TCG_TARGET_HAS_div_i64
;
1493 case INDEX_op_rem_i64
:
1494 case INDEX_op_remu_i64
:
1495 return TCG_TARGET_HAS_rem_i64
;
1496 case INDEX_op_div2_i64
:
1497 case INDEX_op_divu2_i64
:
1498 return TCG_TARGET_HAS_div2_i64
;
1499 case INDEX_op_rotl_i64
:
1500 case INDEX_op_rotr_i64
:
1501 return TCG_TARGET_HAS_rot_i64
;
1502 case INDEX_op_deposit_i64
:
1503 return TCG_TARGET_HAS_deposit_i64
;
1504 case INDEX_op_extract_i64
:
1505 return TCG_TARGET_HAS_extract_i64
;
1506 case INDEX_op_sextract_i64
:
1507 return TCG_TARGET_HAS_sextract_i64
;
1508 case INDEX_op_extrl_i64_i32
:
1509 return TCG_TARGET_HAS_extrl_i64_i32
;
1510 case INDEX_op_extrh_i64_i32
:
1511 return TCG_TARGET_HAS_extrh_i64_i32
;
1512 case INDEX_op_ext8s_i64
:
1513 return TCG_TARGET_HAS_ext8s_i64
;
1514 case INDEX_op_ext16s_i64
:
1515 return TCG_TARGET_HAS_ext16s_i64
;
1516 case INDEX_op_ext32s_i64
:
1517 return TCG_TARGET_HAS_ext32s_i64
;
1518 case INDEX_op_ext8u_i64
:
1519 return TCG_TARGET_HAS_ext8u_i64
;
1520 case INDEX_op_ext16u_i64
:
1521 return TCG_TARGET_HAS_ext16u_i64
;
1522 case INDEX_op_ext32u_i64
:
1523 return TCG_TARGET_HAS_ext32u_i64
;
1524 case INDEX_op_bswap16_i64
:
1525 return TCG_TARGET_HAS_bswap16_i64
;
1526 case INDEX_op_bswap32_i64
:
1527 return TCG_TARGET_HAS_bswap32_i64
;
1528 case INDEX_op_bswap64_i64
:
1529 return TCG_TARGET_HAS_bswap64_i64
;
1530 case INDEX_op_not_i64
:
1531 return TCG_TARGET_HAS_not_i64
;
1532 case INDEX_op_neg_i64
:
1533 return TCG_TARGET_HAS_neg_i64
;
1534 case INDEX_op_andc_i64
:
1535 return TCG_TARGET_HAS_andc_i64
;
1536 case INDEX_op_orc_i64
:
1537 return TCG_TARGET_HAS_orc_i64
;
1538 case INDEX_op_eqv_i64
:
1539 return TCG_TARGET_HAS_eqv_i64
;
1540 case INDEX_op_nand_i64
:
1541 return TCG_TARGET_HAS_nand_i64
;
1542 case INDEX_op_nor_i64
:
1543 return TCG_TARGET_HAS_nor_i64
;
1544 case INDEX_op_clz_i64
:
1545 return TCG_TARGET_HAS_clz_i64
;
1546 case INDEX_op_ctz_i64
:
1547 return TCG_TARGET_HAS_ctz_i64
;
1548 case INDEX_op_ctpop_i64
:
1549 return TCG_TARGET_HAS_ctpop_i64
;
1550 case INDEX_op_add2_i64
:
1551 return TCG_TARGET_HAS_add2_i64
;
1552 case INDEX_op_sub2_i64
:
1553 return TCG_TARGET_HAS_sub2_i64
;
1554 case INDEX_op_mulu2_i64
:
1555 return TCG_TARGET_HAS_mulu2_i64
;
1556 case INDEX_op_muls2_i64
:
1557 return TCG_TARGET_HAS_muls2_i64
;
1558 case INDEX_op_muluh_i64
:
1559 return TCG_TARGET_HAS_muluh_i64
;
1560 case INDEX_op_mulsh_i64
:
1561 return TCG_TARGET_HAS_mulsh_i64
;
1563 case INDEX_op_mov_vec
:
1564 case INDEX_op_dup_vec
:
1565 case INDEX_op_dupi_vec
:
1566 case INDEX_op_ld_vec
:
1567 case INDEX_op_st_vec
:
1568 case INDEX_op_add_vec
:
1569 case INDEX_op_sub_vec
:
1570 case INDEX_op_and_vec
:
1571 case INDEX_op_or_vec
:
1572 case INDEX_op_xor_vec
:
1573 case INDEX_op_cmp_vec
:
1575 case INDEX_op_dup2_vec
:
1576 return have_vec
&& TCG_TARGET_REG_BITS
== 32;
1577 case INDEX_op_not_vec
:
1578 return have_vec
&& TCG_TARGET_HAS_not_vec
;
1579 case INDEX_op_neg_vec
:
1580 return have_vec
&& TCG_TARGET_HAS_neg_vec
;
1581 case INDEX_op_andc_vec
:
1582 return have_vec
&& TCG_TARGET_HAS_andc_vec
;
1583 case INDEX_op_orc_vec
:
1584 return have_vec
&& TCG_TARGET_HAS_orc_vec
;
1585 case INDEX_op_mul_vec
:
1586 return have_vec
&& TCG_TARGET_HAS_mul_vec
;
1587 case INDEX_op_shli_vec
:
1588 case INDEX_op_shri_vec
:
1589 case INDEX_op_sari_vec
:
1590 return have_vec
&& TCG_TARGET_HAS_shi_vec
;
1591 case INDEX_op_shls_vec
:
1592 case INDEX_op_shrs_vec
:
1593 case INDEX_op_sars_vec
:
1594 return have_vec
&& TCG_TARGET_HAS_shs_vec
;
1595 case INDEX_op_shlv_vec
:
1596 case INDEX_op_shrv_vec
:
1597 case INDEX_op_sarv_vec
:
1598 return have_vec
&& TCG_TARGET_HAS_shv_vec
;
1601 tcg_debug_assert(op
> INDEX_op_last_generic
&& op
< NB_OPS
);
1606 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1607 and endian swap. Maybe it would be better to do the alignment
1608 and endian swap in tcg_reg_alloc_call(). */
1609 void tcg_gen_callN(void *func
, TCGTemp
*ret
, int nargs
, TCGTemp
**args
)
1611 int i
, real_args
, nb_rets
, pi
;
1612 unsigned sizemask
, flags
;
1613 TCGHelperInfo
*info
;
1616 info
= g_hash_table_lookup(helper_table
, (gpointer
)func
);
1617 flags
= info
->flags
;
1618 sizemask
= info
->sizemask
;
1620 #if defined(__sparc__) && !defined(__arch64__) \
1621 && !defined(CONFIG_TCG_INTERPRETER)
1622 /* We have 64-bit values in one register, but need to pass as two
1623 separate parameters. Split them. */
1624 int orig_sizemask
= sizemask
;
1625 int orig_nargs
= nargs
;
1626 TCGv_i64 retl
, reth
;
1627 TCGTemp
*split_args
[MAX_OPC_PARAM
];
1631 if (sizemask
!= 0) {
1632 for (i
= real_args
= 0; i
< nargs
; ++i
) {
1633 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1635 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1636 TCGv_i32 h
= tcg_temp_new_i32();
1637 TCGv_i32 l
= tcg_temp_new_i32();
1638 tcg_gen_extr_i64_i32(l
, h
, orig
);
1639 split_args
[real_args
++] = tcgv_i32_temp(h
);
1640 split_args
[real_args
++] = tcgv_i32_temp(l
);
1642 split_args
[real_args
++] = args
[i
];
1649 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1650 for (i
= 0; i
< nargs
; ++i
) {
1651 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1652 int is_signed
= sizemask
& (2 << (i
+1)*2);
1654 TCGv_i64 temp
= tcg_temp_new_i64();
1655 TCGv_i64 orig
= temp_tcgv_i64(args
[i
]);
1657 tcg_gen_ext32s_i64(temp
, orig
);
1659 tcg_gen_ext32u_i64(temp
, orig
);
1661 args
[i
] = tcgv_i64_temp(temp
);
1664 #endif /* TCG_TARGET_EXTEND_ARGS */
1666 op
= tcg_emit_op(INDEX_op_call
);
1670 #if defined(__sparc__) && !defined(__arch64__) \
1671 && !defined(CONFIG_TCG_INTERPRETER)
1672 if (orig_sizemask
& 1) {
1673 /* The 32-bit ABI is going to return the 64-bit value in
1674 the %o0/%o1 register pair. Prepare for this by using
1675 two return temporaries, and reassemble below. */
1676 retl
= tcg_temp_new_i64();
1677 reth
= tcg_temp_new_i64();
1678 op
->args
[pi
++] = tcgv_i64_arg(reth
);
1679 op
->args
[pi
++] = tcgv_i64_arg(retl
);
1682 op
->args
[pi
++] = temp_arg(ret
);
1686 if (TCG_TARGET_REG_BITS
< 64 && (sizemask
& 1)) {
1687 #ifdef HOST_WORDS_BIGENDIAN
1688 op
->args
[pi
++] = temp_arg(ret
+ 1);
1689 op
->args
[pi
++] = temp_arg(ret
);
1691 op
->args
[pi
++] = temp_arg(ret
);
1692 op
->args
[pi
++] = temp_arg(ret
+ 1);
1696 op
->args
[pi
++] = temp_arg(ret
);
1703 TCGOP_CALLO(op
) = nb_rets
;
1706 for (i
= 0; i
< nargs
; i
++) {
1707 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1708 if (TCG_TARGET_REG_BITS
< 64 && is_64bit
) {
1709 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1710 /* some targets want aligned 64 bit args */
1711 if (real_args
& 1) {
1712 op
->args
[pi
++] = TCG_CALL_DUMMY_ARG
;
1716 /* If stack grows up, then we will be placing successive
1717 arguments at lower addresses, which means we need to
1718 reverse the order compared to how we would normally
1719 treat either big or little-endian. For those arguments
1720 that will wind up in registers, this still works for
1721 HPPA (the only current STACK_GROWSUP target) since the
1722 argument registers are *also* allocated in decreasing
1723 order. If another such target is added, this logic may
1724 have to get more complicated to differentiate between
1725 stack arguments and register arguments. */
1726 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1727 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1728 op
->args
[pi
++] = temp_arg(args
[i
]);
1730 op
->args
[pi
++] = temp_arg(args
[i
]);
1731 op
->args
[pi
++] = temp_arg(args
[i
] + 1);
1737 op
->args
[pi
++] = temp_arg(args
[i
]);
1740 op
->args
[pi
++] = (uintptr_t)func
;
1741 op
->args
[pi
++] = flags
;
1742 TCGOP_CALLI(op
) = real_args
;
1744 /* Make sure the fields didn't overflow. */
1745 tcg_debug_assert(TCGOP_CALLI(op
) == real_args
);
1746 tcg_debug_assert(pi
<= ARRAY_SIZE(op
->args
));
1748 #if defined(__sparc__) && !defined(__arch64__) \
1749 && !defined(CONFIG_TCG_INTERPRETER)
1750 /* Free all of the parts we allocated above. */
1751 for (i
= real_args
= 0; i
< orig_nargs
; ++i
) {
1752 int is_64bit
= orig_sizemask
& (1 << (i
+1)*2);
1754 tcg_temp_free_internal(args
[real_args
++]);
1755 tcg_temp_free_internal(args
[real_args
++]);
1760 if (orig_sizemask
& 1) {
1761 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1762 Note that describing these as TCGv_i64 eliminates an unnecessary
1763 zero-extension that tcg_gen_concat_i32_i64 would create. */
1764 tcg_gen_concat32_i64(temp_tcgv_i64(ret
), retl
, reth
);
1765 tcg_temp_free_i64(retl
);
1766 tcg_temp_free_i64(reth
);
1768 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1769 for (i
= 0; i
< nargs
; ++i
) {
1770 int is_64bit
= sizemask
& (1 << (i
+1)*2);
1772 tcg_temp_free_internal(args
[i
]);
1775 #endif /* TCG_TARGET_EXTEND_ARGS */
1778 static void tcg_reg_alloc_start(TCGContext
*s
)
1783 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
1785 ts
->val_type
= (ts
->fixed_reg
? TEMP_VAL_REG
: TEMP_VAL_MEM
);
1787 for (n
= s
->nb_temps
; i
< n
; i
++) {
1789 ts
->val_type
= (ts
->temp_local
? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
1790 ts
->mem_allocated
= 0;
1794 memset(s
->reg_to_temp
, 0, sizeof(s
->reg_to_temp
));
1797 static char *tcg_get_arg_str_ptr(TCGContext
*s
, char *buf
, int buf_size
,
1800 int idx
= temp_idx(ts
);
1802 if (ts
->temp_global
) {
1803 pstrcpy(buf
, buf_size
, ts
->name
);
1804 } else if (ts
->temp_local
) {
1805 snprintf(buf
, buf_size
, "loc%d", idx
- s
->nb_globals
);
1807 snprintf(buf
, buf_size
, "tmp%d", idx
- s
->nb_globals
);
1812 static char *tcg_get_arg_str(TCGContext
*s
, char *buf
,
1813 int buf_size
, TCGArg arg
)
1815 return tcg_get_arg_str_ptr(s
, buf
, buf_size
, arg_temp(arg
));
1818 /* Find helper name. */
1819 static inline const char *tcg_find_helper(TCGContext
*s
, uintptr_t val
)
1821 const char *ret
= NULL
;
1823 TCGHelperInfo
*info
= g_hash_table_lookup(helper_table
, (gpointer
)val
);
1831 static const char * const cond_name
[] =
1833 [TCG_COND_NEVER
] = "never",
1834 [TCG_COND_ALWAYS
] = "always",
1835 [TCG_COND_EQ
] = "eq",
1836 [TCG_COND_NE
] = "ne",
1837 [TCG_COND_LT
] = "lt",
1838 [TCG_COND_GE
] = "ge",
1839 [TCG_COND_LE
] = "le",
1840 [TCG_COND_GT
] = "gt",
1841 [TCG_COND_LTU
] = "ltu",
1842 [TCG_COND_GEU
] = "geu",
1843 [TCG_COND_LEU
] = "leu",
1844 [TCG_COND_GTU
] = "gtu"
1847 static const char * const ldst_name
[] =
1863 static const char * const alignment_name
[(MO_AMASK
>> MO_ASHIFT
) + 1] = {
1865 [MO_UNALN
>> MO_ASHIFT
] = "un+",
1866 [MO_ALIGN
>> MO_ASHIFT
] = "",
1868 [MO_UNALN
>> MO_ASHIFT
] = "",
1869 [MO_ALIGN
>> MO_ASHIFT
] = "al+",
1871 [MO_ALIGN_2
>> MO_ASHIFT
] = "al2+",
1872 [MO_ALIGN_4
>> MO_ASHIFT
] = "al4+",
1873 [MO_ALIGN_8
>> MO_ASHIFT
] = "al8+",
1874 [MO_ALIGN_16
>> MO_ASHIFT
] = "al16+",
1875 [MO_ALIGN_32
>> MO_ASHIFT
] = "al32+",
1876 [MO_ALIGN_64
>> MO_ASHIFT
] = "al64+",
1879 void tcg_dump_ops(TCGContext
*s
)
1884 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
1885 int i
, k
, nb_oargs
, nb_iargs
, nb_cargs
;
1886 const TCGOpDef
*def
;
1891 def
= &tcg_op_defs
[c
];
1893 if (c
== INDEX_op_insn_start
) {
1894 col
+= qemu_log("\n ----");
1896 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
1898 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1899 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
1903 col
+= qemu_log(" " TARGET_FMT_lx
, a
);
1905 } else if (c
== INDEX_op_call
) {
1906 /* variable number of arguments */
1907 nb_oargs
= TCGOP_CALLO(op
);
1908 nb_iargs
= TCGOP_CALLI(op
);
1909 nb_cargs
= def
->nb_cargs
;
1911 /* function name, flags, out args */
1912 col
+= qemu_log(" %s %s,$0x%" TCG_PRIlx
",$%d", def
->name
,
1913 tcg_find_helper(s
, op
->args
[nb_oargs
+ nb_iargs
]),
1914 op
->args
[nb_oargs
+ nb_iargs
+ 1], nb_oargs
);
1915 for (i
= 0; i
< nb_oargs
; i
++) {
1916 col
+= qemu_log(",%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
1919 for (i
= 0; i
< nb_iargs
; i
++) {
1920 TCGArg arg
= op
->args
[nb_oargs
+ i
];
1921 const char *t
= "<dummy>";
1922 if (arg
!= TCG_CALL_DUMMY_ARG
) {
1923 t
= tcg_get_arg_str(s
, buf
, sizeof(buf
), arg
);
1925 col
+= qemu_log(",%s", t
);
1928 col
+= qemu_log(" %s ", def
->name
);
1930 nb_oargs
= def
->nb_oargs
;
1931 nb_iargs
= def
->nb_iargs
;
1932 nb_cargs
= def
->nb_cargs
;
1934 if (def
->flags
& TCG_OPF_VECTOR
) {
1935 col
+= qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op
),
1936 8 << TCGOP_VECE(op
));
1940 for (i
= 0; i
< nb_oargs
; i
++) {
1942 col
+= qemu_log(",");
1944 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
1947 for (i
= 0; i
< nb_iargs
; i
++) {
1949 col
+= qemu_log(",");
1951 col
+= qemu_log("%s", tcg_get_arg_str(s
, buf
, sizeof(buf
),
1955 case INDEX_op_brcond_i32
:
1956 case INDEX_op_setcond_i32
:
1957 case INDEX_op_movcond_i32
:
1958 case INDEX_op_brcond2_i32
:
1959 case INDEX_op_setcond2_i32
:
1960 case INDEX_op_brcond_i64
:
1961 case INDEX_op_setcond_i64
:
1962 case INDEX_op_movcond_i64
:
1963 case INDEX_op_cmp_vec
:
1964 if (op
->args
[k
] < ARRAY_SIZE(cond_name
)
1965 && cond_name
[op
->args
[k
]]) {
1966 col
+= qemu_log(",%s", cond_name
[op
->args
[k
++]]);
1968 col
+= qemu_log(",$0x%" TCG_PRIlx
, op
->args
[k
++]);
1972 case INDEX_op_qemu_ld_i32
:
1973 case INDEX_op_qemu_st_i32
:
1974 case INDEX_op_qemu_ld_i64
:
1975 case INDEX_op_qemu_st_i64
:
1977 TCGMemOpIdx oi
= op
->args
[k
++];
1978 TCGMemOp op
= get_memop(oi
);
1979 unsigned ix
= get_mmuidx(oi
);
1981 if (op
& ~(MO_AMASK
| MO_BSWAP
| MO_SSIZE
)) {
1982 col
+= qemu_log(",$0x%x,%u", op
, ix
);
1984 const char *s_al
, *s_op
;
1985 s_al
= alignment_name
[(op
& MO_AMASK
) >> MO_ASHIFT
];
1986 s_op
= ldst_name
[op
& (MO_BSWAP
| MO_SSIZE
)];
1987 col
+= qemu_log(",%s%s,%u", s_al
, s_op
, ix
);
1997 case INDEX_op_set_label
:
1999 case INDEX_op_brcond_i32
:
2000 case INDEX_op_brcond_i64
:
2001 case INDEX_op_brcond2_i32
:
2002 col
+= qemu_log("%s$L%d", k
? "," : "",
2003 arg_label(op
->args
[k
])->id
);
2009 for (; i
< nb_cargs
; i
++, k
++) {
2010 col
+= qemu_log("%s$0x%" TCG_PRIlx
, k
? "," : "", op
->args
[k
]);
2014 unsigned life
= op
->life
;
2016 for (; col
< 48; ++col
) {
2017 putc(' ', qemu_logfile
);
2020 if (life
& (SYNC_ARG
* 3)) {
2022 for (i
= 0; i
< 2; ++i
) {
2023 if (life
& (SYNC_ARG
<< i
)) {
2031 for (i
= 0; life
; ++i
, life
>>= 1) {
2042 /* we give more priority to constraints with less registers */
2043 static int get_constraint_priority(const TCGOpDef
*def
, int k
)
2045 const TCGArgConstraint
*arg_ct
;
2048 arg_ct
= &def
->args_ct
[k
];
2049 if (arg_ct
->ct
& TCG_CT_ALIAS
) {
2050 /* an alias is equivalent to a single register */
2053 if (!(arg_ct
->ct
& TCG_CT_REG
))
2056 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2057 if (tcg_regset_test_reg(arg_ct
->u
.regs
, i
))
2061 return TCG_TARGET_NB_REGS
- n
+ 1;
2064 /* sort from highest priority to lowest */
2065 static void sort_constraints(TCGOpDef
*def
, int start
, int n
)
2067 int i
, j
, p1
, p2
, tmp
;
2069 for(i
= 0; i
< n
; i
++)
2070 def
->sorted_args
[start
+ i
] = start
+ i
;
2073 for(i
= 0; i
< n
- 1; i
++) {
2074 for(j
= i
+ 1; j
< n
; j
++) {
2075 p1
= get_constraint_priority(def
, def
->sorted_args
[start
+ i
]);
2076 p2
= get_constraint_priority(def
, def
->sorted_args
[start
+ j
]);
2078 tmp
= def
->sorted_args
[start
+ i
];
2079 def
->sorted_args
[start
+ i
] = def
->sorted_args
[start
+ j
];
2080 def
->sorted_args
[start
+ j
] = tmp
;
2086 static void process_op_defs(TCGContext
*s
)
2090 for (op
= 0; op
< NB_OPS
; op
++) {
2091 TCGOpDef
*def
= &tcg_op_defs
[op
];
2092 const TCGTargetOpDef
*tdefs
;
2096 if (def
->flags
& TCG_OPF_NOT_PRESENT
) {
2100 nb_args
= def
->nb_iargs
+ def
->nb_oargs
;
2105 tdefs
= tcg_target_op_def(op
);
2106 /* Missing TCGTargetOpDef entry. */
2107 tcg_debug_assert(tdefs
!= NULL
);
2109 type
= (def
->flags
& TCG_OPF_64BIT
? TCG_TYPE_I64
: TCG_TYPE_I32
);
2110 for (i
= 0; i
< nb_args
; i
++) {
2111 const char *ct_str
= tdefs
->args_ct_str
[i
];
2112 /* Incomplete TCGTargetOpDef entry. */
2113 tcg_debug_assert(ct_str
!= NULL
);
2115 def
->args_ct
[i
].u
.regs
= 0;
2116 def
->args_ct
[i
].ct
= 0;
2117 while (*ct_str
!= '\0') {
2121 int oarg
= *ct_str
- '0';
2122 tcg_debug_assert(ct_str
== tdefs
->args_ct_str
[i
]);
2123 tcg_debug_assert(oarg
< def
->nb_oargs
);
2124 tcg_debug_assert(def
->args_ct
[oarg
].ct
& TCG_CT_REG
);
2125 /* TCG_CT_ALIAS is for the output arguments.
2126 The input is tagged with TCG_CT_IALIAS. */
2127 def
->args_ct
[i
] = def
->args_ct
[oarg
];
2128 def
->args_ct
[oarg
].ct
|= TCG_CT_ALIAS
;
2129 def
->args_ct
[oarg
].alias_index
= i
;
2130 def
->args_ct
[i
].ct
|= TCG_CT_IALIAS
;
2131 def
->args_ct
[i
].alias_index
= oarg
;
2136 def
->args_ct
[i
].ct
|= TCG_CT_NEWREG
;
2140 def
->args_ct
[i
].ct
|= TCG_CT_CONST
;
2144 ct_str
= target_parse_constraint(&def
->args_ct
[i
],
2146 /* Typo in TCGTargetOpDef constraint. */
2147 tcg_debug_assert(ct_str
!= NULL
);
2152 /* TCGTargetOpDef entry with too much information? */
2153 tcg_debug_assert(i
== TCG_MAX_OP_ARGS
|| tdefs
->args_ct_str
[i
] == NULL
);
2155 /* sort the constraints (XXX: this is just an heuristic) */
2156 sort_constraints(def
, 0, def
->nb_oargs
);
2157 sort_constraints(def
, def
->nb_oargs
, def
->nb_iargs
);
2161 void tcg_op_remove(TCGContext
*s
, TCGOp
*op
)
2163 QTAILQ_REMOVE(&s
->ops
, op
, link
);
2164 QTAILQ_INSERT_TAIL(&s
->free_ops
, op
, link
);
2167 #ifdef CONFIG_PROFILER
2168 atomic_set(&s
->prof
.del_op_count
, s
->prof
.del_op_count
+ 1);
2172 static TCGOp
*tcg_op_alloc(TCGOpcode opc
)
2174 TCGContext
*s
= tcg_ctx
;
2177 if (likely(QTAILQ_EMPTY(&s
->free_ops
))) {
2178 op
= tcg_malloc(sizeof(TCGOp
));
2180 op
= QTAILQ_FIRST(&s
->free_ops
);
2181 QTAILQ_REMOVE(&s
->free_ops
, op
, link
);
2183 memset(op
, 0, offsetof(TCGOp
, link
));
2190 TCGOp
*tcg_emit_op(TCGOpcode opc
)
2192 TCGOp
*op
= tcg_op_alloc(opc
);
2193 QTAILQ_INSERT_TAIL(&tcg_ctx
->ops
, op
, link
);
2197 TCGOp
*tcg_op_insert_before(TCGContext
*s
, TCGOp
*old_op
,
2198 TCGOpcode opc
, int nargs
)
2200 TCGOp
*new_op
= tcg_op_alloc(opc
);
2201 QTAILQ_INSERT_BEFORE(old_op
, new_op
, link
);
2205 TCGOp
*tcg_op_insert_after(TCGContext
*s
, TCGOp
*old_op
,
2206 TCGOpcode opc
, int nargs
)
2208 TCGOp
*new_op
= tcg_op_alloc(opc
);
2209 QTAILQ_INSERT_AFTER(&s
->ops
, old_op
, new_op
, link
);
2216 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2217 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2219 /* liveness analysis: end of function: all temps are dead, and globals
2220 should be in memory. */
2221 static void tcg_la_func_end(TCGContext
*s
)
2223 int ng
= s
->nb_globals
;
2224 int nt
= s
->nb_temps
;
2227 for (i
= 0; i
< ng
; ++i
) {
2228 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2230 for (i
= ng
; i
< nt
; ++i
) {
2231 s
->temps
[i
].state
= TS_DEAD
;
2235 /* liveness analysis: end of basic block: all temps are dead, globals
2236 and local temps should be in memory. */
2237 static void tcg_la_bb_end(TCGContext
*s
)
2239 int ng
= s
->nb_globals
;
2240 int nt
= s
->nb_temps
;
2243 for (i
= 0; i
< ng
; ++i
) {
2244 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2246 for (i
= ng
; i
< nt
; ++i
) {
2247 s
->temps
[i
].state
= (s
->temps
[i
].temp_local
2253 /* Liveness analysis : update the opc_arg_life array to tell if a
2254 given input arguments is dead. Instructions updating dead
2255 temporaries are removed. */
2256 static void liveness_pass_1(TCGContext
*s
)
2258 int nb_globals
= s
->nb_globals
;
2259 TCGOp
*op
, *op_prev
;
2263 QTAILQ_FOREACH_REVERSE_SAFE(op
, &s
->ops
, TCGOpHead
, link
, op_prev
) {
2264 int i
, nb_iargs
, nb_oargs
;
2265 TCGOpcode opc_new
, opc_new2
;
2267 TCGLifeData arg_life
= 0;
2269 TCGOpcode opc
= op
->opc
;
2270 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2277 nb_oargs
= TCGOP_CALLO(op
);
2278 nb_iargs
= TCGOP_CALLI(op
);
2279 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2281 /* pure functions can be removed if their result is unused */
2282 if (call_flags
& TCG_CALL_NO_SIDE_EFFECTS
) {
2283 for (i
= 0; i
< nb_oargs
; i
++) {
2284 arg_ts
= arg_temp(op
->args
[i
]);
2285 if (arg_ts
->state
!= TS_DEAD
) {
2286 goto do_not_remove_call
;
2293 /* output args are dead */
2294 for (i
= 0; i
< nb_oargs
; i
++) {
2295 arg_ts
= arg_temp(op
->args
[i
]);
2296 if (arg_ts
->state
& TS_DEAD
) {
2297 arg_life
|= DEAD_ARG
<< i
;
2299 if (arg_ts
->state
& TS_MEM
) {
2300 arg_life
|= SYNC_ARG
<< i
;
2302 arg_ts
->state
= TS_DEAD
;
2305 if (!(call_flags
& (TCG_CALL_NO_WRITE_GLOBALS
|
2306 TCG_CALL_NO_READ_GLOBALS
))) {
2307 /* globals should go back to memory */
2308 for (i
= 0; i
< nb_globals
; i
++) {
2309 s
->temps
[i
].state
= TS_DEAD
| TS_MEM
;
2311 } else if (!(call_flags
& TCG_CALL_NO_READ_GLOBALS
)) {
2312 /* globals should be synced to memory */
2313 for (i
= 0; i
< nb_globals
; i
++) {
2314 s
->temps
[i
].state
|= TS_MEM
;
2318 /* record arguments that die in this helper */
2319 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2320 arg_ts
= arg_temp(op
->args
[i
]);
2321 if (arg_ts
&& arg_ts
->state
& TS_DEAD
) {
2322 arg_life
|= DEAD_ARG
<< i
;
2325 /* input arguments are live for preceding opcodes */
2326 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2327 arg_ts
= arg_temp(op
->args
[i
]);
2329 arg_ts
->state
&= ~TS_DEAD
;
2335 case INDEX_op_insn_start
:
2337 case INDEX_op_discard
:
2338 /* mark the temporary as dead */
2339 arg_temp(op
->args
[0])->state
= TS_DEAD
;
2342 case INDEX_op_add2_i32
:
2343 opc_new
= INDEX_op_add_i32
;
2345 case INDEX_op_sub2_i32
:
2346 opc_new
= INDEX_op_sub_i32
;
2348 case INDEX_op_add2_i64
:
2349 opc_new
= INDEX_op_add_i64
;
2351 case INDEX_op_sub2_i64
:
2352 opc_new
= INDEX_op_sub_i64
;
2356 /* Test if the high part of the operation is dead, but not
2357 the low part. The result can be optimized to a simple
2358 add or sub. This happens often for x86_64 guest when the
2359 cpu mode is set to 32 bit. */
2360 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2361 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2364 /* Replace the opcode and adjust the args in place,
2365 leaving 3 unused args at the end. */
2366 op
->opc
= opc
= opc_new
;
2367 op
->args
[1] = op
->args
[2];
2368 op
->args
[2] = op
->args
[4];
2369 /* Fall through and mark the single-word operation live. */
2375 case INDEX_op_mulu2_i32
:
2376 opc_new
= INDEX_op_mul_i32
;
2377 opc_new2
= INDEX_op_muluh_i32
;
2378 have_opc_new2
= TCG_TARGET_HAS_muluh_i32
;
2380 case INDEX_op_muls2_i32
:
2381 opc_new
= INDEX_op_mul_i32
;
2382 opc_new2
= INDEX_op_mulsh_i32
;
2383 have_opc_new2
= TCG_TARGET_HAS_mulsh_i32
;
2385 case INDEX_op_mulu2_i64
:
2386 opc_new
= INDEX_op_mul_i64
;
2387 opc_new2
= INDEX_op_muluh_i64
;
2388 have_opc_new2
= TCG_TARGET_HAS_muluh_i64
;
2390 case INDEX_op_muls2_i64
:
2391 opc_new
= INDEX_op_mul_i64
;
2392 opc_new2
= INDEX_op_mulsh_i64
;
2393 have_opc_new2
= TCG_TARGET_HAS_mulsh_i64
;
2398 if (arg_temp(op
->args
[1])->state
== TS_DEAD
) {
2399 if (arg_temp(op
->args
[0])->state
== TS_DEAD
) {
2400 /* Both parts of the operation are dead. */
2403 /* The high part of the operation is dead; generate the low. */
2404 op
->opc
= opc
= opc_new
;
2405 op
->args
[1] = op
->args
[2];
2406 op
->args
[2] = op
->args
[3];
2407 } else if (arg_temp(op
->args
[0])->state
== TS_DEAD
&& have_opc_new2
) {
2408 /* The low part of the operation is dead; generate the high. */
2409 op
->opc
= opc
= opc_new2
;
2410 op
->args
[0] = op
->args
[1];
2411 op
->args
[1] = op
->args
[2];
2412 op
->args
[2] = op
->args
[3];
2416 /* Mark the single-word operation live. */
2421 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2422 nb_iargs
= def
->nb_iargs
;
2423 nb_oargs
= def
->nb_oargs
;
2425 /* Test if the operation can be removed because all
2426 its outputs are dead. We assume that nb_oargs == 0
2427 implies side effects */
2428 if (!(def
->flags
& TCG_OPF_SIDE_EFFECTS
) && nb_oargs
!= 0) {
2429 for (i
= 0; i
< nb_oargs
; i
++) {
2430 if (arg_temp(op
->args
[i
])->state
!= TS_DEAD
) {
2435 tcg_op_remove(s
, op
);
2438 /* output args are dead */
2439 for (i
= 0; i
< nb_oargs
; i
++) {
2440 arg_ts
= arg_temp(op
->args
[i
]);
2441 if (arg_ts
->state
& TS_DEAD
) {
2442 arg_life
|= DEAD_ARG
<< i
;
2444 if (arg_ts
->state
& TS_MEM
) {
2445 arg_life
|= SYNC_ARG
<< i
;
2447 arg_ts
->state
= TS_DEAD
;
2450 /* if end of basic block, update */
2451 if (def
->flags
& TCG_OPF_BB_END
) {
2453 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2454 /* globals should be synced to memory */
2455 for (i
= 0; i
< nb_globals
; i
++) {
2456 s
->temps
[i
].state
|= TS_MEM
;
2460 /* record arguments that die in this opcode */
2461 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2462 arg_ts
= arg_temp(op
->args
[i
]);
2463 if (arg_ts
->state
& TS_DEAD
) {
2464 arg_life
|= DEAD_ARG
<< i
;
2467 /* input arguments are live for preceding opcodes */
2468 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
2469 arg_temp(op
->args
[i
])->state
&= ~TS_DEAD
;
2474 op
->life
= arg_life
;
2478 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2479 static bool liveness_pass_2(TCGContext
*s
)
2481 int nb_globals
= s
->nb_globals
;
2483 bool changes
= false;
2484 TCGOp
*op
, *op_next
;
2486 /* Create a temporary for each indirect global. */
2487 for (i
= 0; i
< nb_globals
; ++i
) {
2488 TCGTemp
*its
= &s
->temps
[i
];
2489 if (its
->indirect_reg
) {
2490 TCGTemp
*dts
= tcg_temp_alloc(s
);
2491 dts
->type
= its
->type
;
2492 dts
->base_type
= its
->base_type
;
2493 its
->state_ptr
= dts
;
2495 its
->state_ptr
= NULL
;
2497 /* All globals begin dead. */
2498 its
->state
= TS_DEAD
;
2500 for (nb_temps
= s
->nb_temps
; i
< nb_temps
; ++i
) {
2501 TCGTemp
*its
= &s
->temps
[i
];
2502 its
->state_ptr
= NULL
;
2503 its
->state
= TS_DEAD
;
2506 QTAILQ_FOREACH_SAFE(op
, &s
->ops
, link
, op_next
) {
2507 TCGOpcode opc
= op
->opc
;
2508 const TCGOpDef
*def
= &tcg_op_defs
[opc
];
2509 TCGLifeData arg_life
= op
->life
;
2510 int nb_iargs
, nb_oargs
, call_flags
;
2511 TCGTemp
*arg_ts
, *dir_ts
;
2513 if (opc
== INDEX_op_call
) {
2514 nb_oargs
= TCGOP_CALLO(op
);
2515 nb_iargs
= TCGOP_CALLI(op
);
2516 call_flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
2518 nb_iargs
= def
->nb_iargs
;
2519 nb_oargs
= def
->nb_oargs
;
2521 /* Set flags similar to how calls require. */
2522 if (def
->flags
& TCG_OPF_BB_END
) {
2523 /* Like writing globals: save_globals */
2525 } else if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
2526 /* Like reading globals: sync_globals */
2527 call_flags
= TCG_CALL_NO_WRITE_GLOBALS
;
2529 /* No effect on globals. */
2530 call_flags
= (TCG_CALL_NO_READ_GLOBALS
|
2531 TCG_CALL_NO_WRITE_GLOBALS
);
2535 /* Make sure that input arguments are available. */
2536 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2537 arg_ts
= arg_temp(op
->args
[i
]);
2539 dir_ts
= arg_ts
->state_ptr
;
2540 if (dir_ts
&& arg_ts
->state
== TS_DEAD
) {
2541 TCGOpcode lopc
= (arg_ts
->type
== TCG_TYPE_I32
2544 TCGOp
*lop
= tcg_op_insert_before(s
, op
, lopc
, 3);
2546 lop
->args
[0] = temp_arg(dir_ts
);
2547 lop
->args
[1] = temp_arg(arg_ts
->mem_base
);
2548 lop
->args
[2] = arg_ts
->mem_offset
;
2550 /* Loaded, but synced with memory. */
2551 arg_ts
->state
= TS_MEM
;
2556 /* Perform input replacement, and mark inputs that became dead.
2557 No action is required except keeping temp_state up to date
2558 so that we reload when needed. */
2559 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
2560 arg_ts
= arg_temp(op
->args
[i
]);
2562 dir_ts
= arg_ts
->state_ptr
;
2564 op
->args
[i
] = temp_arg(dir_ts
);
2566 if (IS_DEAD_ARG(i
)) {
2567 arg_ts
->state
= TS_DEAD
;
2573 /* Liveness analysis should ensure that the following are
2574 all correct, for call sites and basic block end points. */
2575 if (call_flags
& TCG_CALL_NO_READ_GLOBALS
) {
2577 } else if (call_flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
2578 for (i
= 0; i
< nb_globals
; ++i
) {
2579 /* Liveness should see that globals are synced back,
2580 that is, either TS_DEAD or TS_MEM. */
2581 arg_ts
= &s
->temps
[i
];
2582 tcg_debug_assert(arg_ts
->state_ptr
== 0
2583 || arg_ts
->state
!= 0);
2586 for (i
= 0; i
< nb_globals
; ++i
) {
2587 /* Liveness should see that globals are saved back,
2588 that is, TS_DEAD, waiting to be reloaded. */
2589 arg_ts
= &s
->temps
[i
];
2590 tcg_debug_assert(arg_ts
->state_ptr
== 0
2591 || arg_ts
->state
== TS_DEAD
);
2595 /* Outputs become available. */
2596 for (i
= 0; i
< nb_oargs
; i
++) {
2597 arg_ts
= arg_temp(op
->args
[i
]);
2598 dir_ts
= arg_ts
->state_ptr
;
2602 op
->args
[i
] = temp_arg(dir_ts
);
2605 /* The output is now live and modified. */
2608 /* Sync outputs upon their last write. */
2609 if (NEED_SYNC_ARG(i
)) {
2610 TCGOpcode sopc
= (arg_ts
->type
== TCG_TYPE_I32
2613 TCGOp
*sop
= tcg_op_insert_after(s
, op
, sopc
, 3);
2615 sop
->args
[0] = temp_arg(dir_ts
);
2616 sop
->args
[1] = temp_arg(arg_ts
->mem_base
);
2617 sop
->args
[2] = arg_ts
->mem_offset
;
2619 arg_ts
->state
= TS_MEM
;
2621 /* Drop outputs that are dead. */
2622 if (IS_DEAD_ARG(i
)) {
2623 arg_ts
->state
= TS_DEAD
;
2631 #ifdef CONFIG_DEBUG_TCG
2632 static void dump_regs(TCGContext
*s
)
2638 for(i
= 0; i
< s
->nb_temps
; i
++) {
2640 printf(" %10s: ", tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
2641 switch(ts
->val_type
) {
2643 printf("%s", tcg_target_reg_names
[ts
->reg
]);
2646 printf("%d(%s)", (int)ts
->mem_offset
,
2647 tcg_target_reg_names
[ts
->mem_base
->reg
]);
2649 case TEMP_VAL_CONST
:
2650 printf("$0x%" TCG_PRIlx
, ts
->val
);
2662 for(i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
2663 if (s
->reg_to_temp
[i
] != NULL
) {
2665 tcg_target_reg_names
[i
],
2666 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), s
->reg_to_temp
[i
]));
2671 static void check_regs(TCGContext
*s
)
2678 for (reg
= 0; reg
< TCG_TARGET_NB_REGS
; reg
++) {
2679 ts
= s
->reg_to_temp
[reg
];
2681 if (ts
->val_type
!= TEMP_VAL_REG
|| ts
->reg
!= reg
) {
2682 printf("Inconsistency for register %s:\n",
2683 tcg_target_reg_names
[reg
]);
2688 for (k
= 0; k
< s
->nb_temps
; k
++) {
2690 if (ts
->val_type
== TEMP_VAL_REG
&& !ts
->fixed_reg
2691 && s
->reg_to_temp
[ts
->reg
] != ts
) {
2692 printf("Inconsistency for temp %s:\n",
2693 tcg_get_arg_str_ptr(s
, buf
, sizeof(buf
), ts
));
2695 printf("reg state:\n");
2703 static void temp_allocate_frame(TCGContext
*s
, TCGTemp
*ts
)
2705 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
2706 /* Sparc64 stack is accessed with offset of 2047 */
2707 s
->current_frame_offset
= (s
->current_frame_offset
+
2708 (tcg_target_long
)sizeof(tcg_target_long
) - 1) &
2709 ~(sizeof(tcg_target_long
) - 1);
2711 if (s
->current_frame_offset
+ (tcg_target_long
)sizeof(tcg_target_long
) >
2715 ts
->mem_offset
= s
->current_frame_offset
;
2716 ts
->mem_base
= s
->frame_temp
;
2717 ts
->mem_allocated
= 1;
2718 s
->current_frame_offset
+= sizeof(tcg_target_long
);
2721 static void temp_load(TCGContext
*, TCGTemp
*, TCGRegSet
, TCGRegSet
);
2723 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
2724 mark it free; otherwise mark it dead. */
2725 static void temp_free_or_dead(TCGContext
*s
, TCGTemp
*ts
, int free_or_dead
)
2727 if (ts
->fixed_reg
) {
2730 if (ts
->val_type
== TEMP_VAL_REG
) {
2731 s
->reg_to_temp
[ts
->reg
] = NULL
;
2733 ts
->val_type
= (free_or_dead
< 0
2736 ? TEMP_VAL_MEM
: TEMP_VAL_DEAD
);
2739 /* Mark a temporary as dead. */
2740 static inline void temp_dead(TCGContext
*s
, TCGTemp
*ts
)
2742 temp_free_or_dead(s
, ts
, 1);
2745 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
2746 registers needs to be allocated to store a constant. If 'free_or_dead'
2747 is non-zero, subsequently release the temporary; if it is positive, the
2748 temp is dead; if it is negative, the temp is free. */
2749 static void temp_sync(TCGContext
*s
, TCGTemp
*ts
,
2750 TCGRegSet allocated_regs
, int free_or_dead
)
2752 if (ts
->fixed_reg
) {
2755 if (!ts
->mem_coherent
) {
2756 if (!ts
->mem_allocated
) {
2757 temp_allocate_frame(s
, ts
);
2759 switch (ts
->val_type
) {
2760 case TEMP_VAL_CONST
:
2761 /* If we're going to free the temp immediately, then we won't
2762 require it later in a register, so attempt to store the
2763 constant to memory directly. */
2765 && tcg_out_sti(s
, ts
->type
, ts
->val
,
2766 ts
->mem_base
->reg
, ts
->mem_offset
)) {
2769 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
2774 tcg_out_st(s
, ts
->type
, ts
->reg
,
2775 ts
->mem_base
->reg
, ts
->mem_offset
);
2785 ts
->mem_coherent
= 1;
2788 temp_free_or_dead(s
, ts
, free_or_dead
);
2792 /* free register 'reg' by spilling the corresponding temporary if necessary */
2793 static void tcg_reg_free(TCGContext
*s
, TCGReg reg
, TCGRegSet allocated_regs
)
2795 TCGTemp
*ts
= s
->reg_to_temp
[reg
];
2797 temp_sync(s
, ts
, allocated_regs
, -1);
2801 /* Allocate a register belonging to reg1 & ~reg2 */
2802 static TCGReg
tcg_reg_alloc(TCGContext
*s
, TCGRegSet desired_regs
,
2803 TCGRegSet allocated_regs
, bool rev
)
2805 int i
, n
= ARRAY_SIZE(tcg_target_reg_alloc_order
);
2810 reg_ct
= desired_regs
& ~allocated_regs
;
2811 order
= rev
? indirect_reg_alloc_order
: tcg_target_reg_alloc_order
;
2813 /* first try free registers */
2814 for(i
= 0; i
< n
; i
++) {
2816 if (tcg_regset_test_reg(reg_ct
, reg
) && s
->reg_to_temp
[reg
] == NULL
)
2820 /* XXX: do better spill choice */
2821 for(i
= 0; i
< n
; i
++) {
2823 if (tcg_regset_test_reg(reg_ct
, reg
)) {
2824 tcg_reg_free(s
, reg
, allocated_regs
);
2832 /* Make sure the temporary is in a register. If needed, allocate the register
2833 from DESIRED while avoiding ALLOCATED. */
2834 static void temp_load(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet desired_regs
,
2835 TCGRegSet allocated_regs
)
2839 switch (ts
->val_type
) {
2842 case TEMP_VAL_CONST
:
2843 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
2844 tcg_out_movi(s
, ts
->type
, reg
, ts
->val
);
2845 ts
->mem_coherent
= 0;
2848 reg
= tcg_reg_alloc(s
, desired_regs
, allocated_regs
, ts
->indirect_base
);
2849 tcg_out_ld(s
, ts
->type
, reg
, ts
->mem_base
->reg
, ts
->mem_offset
);
2850 ts
->mem_coherent
= 1;
2857 ts
->val_type
= TEMP_VAL_REG
;
2858 s
->reg_to_temp
[reg
] = ts
;
2861 /* Save a temporary to memory. 'allocated_regs' is used in case a
2862 temporary registers needs to be allocated to store a constant. */
2863 static void temp_save(TCGContext
*s
, TCGTemp
*ts
, TCGRegSet allocated_regs
)
2865 /* The liveness analysis already ensures that globals are back
2866 in memory. Keep an tcg_debug_assert for safety. */
2867 tcg_debug_assert(ts
->val_type
== TEMP_VAL_MEM
|| ts
->fixed_reg
);
2870 /* save globals to their canonical location and assume they can be
2871 modified be the following code. 'allocated_regs' is used in case a
2872 temporary registers needs to be allocated to store a constant. */
2873 static void save_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
2877 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
2878 temp_save(s
, &s
->temps
[i
], allocated_regs
);
2882 /* sync globals to their canonical location and assume they can be
2883 read by the following code. 'allocated_regs' is used in case a
2884 temporary registers needs to be allocated to store a constant. */
2885 static void sync_globals(TCGContext
*s
, TCGRegSet allocated_regs
)
2889 for (i
= 0, n
= s
->nb_globals
; i
< n
; i
++) {
2890 TCGTemp
*ts
= &s
->temps
[i
];
2891 tcg_debug_assert(ts
->val_type
!= TEMP_VAL_REG
2893 || ts
->mem_coherent
);
2897 /* at the end of a basic block, we assume all temporaries are dead and
2898 all globals are stored at their canonical location. */
2899 static void tcg_reg_alloc_bb_end(TCGContext
*s
, TCGRegSet allocated_regs
)
2903 for (i
= s
->nb_globals
; i
< s
->nb_temps
; i
++) {
2904 TCGTemp
*ts
= &s
->temps
[i
];
2905 if (ts
->temp_local
) {
2906 temp_save(s
, ts
, allocated_regs
);
2908 /* The liveness analysis already ensures that temps are dead.
2909 Keep an tcg_debug_assert for safety. */
2910 tcg_debug_assert(ts
->val_type
== TEMP_VAL_DEAD
);
2914 save_globals(s
, allocated_regs
);
2917 static void tcg_reg_alloc_do_movi(TCGContext
*s
, TCGTemp
*ots
,
2918 tcg_target_ulong val
, TCGLifeData arg_life
)
2920 if (ots
->fixed_reg
) {
2921 /* For fixed registers, we do not do any constant propagation. */
2922 tcg_out_movi(s
, ots
->type
, ots
->reg
, val
);
2926 /* The movi is not explicitly generated here. */
2927 if (ots
->val_type
== TEMP_VAL_REG
) {
2928 s
->reg_to_temp
[ots
->reg
] = NULL
;
2930 ots
->val_type
= TEMP_VAL_CONST
;
2932 ots
->mem_coherent
= 0;
2933 if (NEED_SYNC_ARG(0)) {
2934 temp_sync(s
, ots
, s
->reserved_regs
, IS_DEAD_ARG(0));
2935 } else if (IS_DEAD_ARG(0)) {
2940 static void tcg_reg_alloc_movi(TCGContext
*s
, const TCGOp
*op
)
2942 TCGTemp
*ots
= arg_temp(op
->args
[0]);
2943 tcg_target_ulong val
= op
->args
[1];
2945 tcg_reg_alloc_do_movi(s
, ots
, val
, op
->life
);
2948 static void tcg_reg_alloc_mov(TCGContext
*s
, const TCGOp
*op
)
2950 const TCGLifeData arg_life
= op
->life
;
2951 TCGRegSet allocated_regs
;
2953 TCGType otype
, itype
;
2955 allocated_regs
= s
->reserved_regs
;
2956 ots
= arg_temp(op
->args
[0]);
2957 ts
= arg_temp(op
->args
[1]);
2959 /* Note that otype != itype for no-op truncation. */
2963 if (ts
->val_type
== TEMP_VAL_CONST
) {
2964 /* propagate constant or generate sti */
2965 tcg_target_ulong val
= ts
->val
;
2966 if (IS_DEAD_ARG(1)) {
2969 tcg_reg_alloc_do_movi(s
, ots
, val
, arg_life
);
2973 /* If the source value is in memory we're going to be forced
2974 to have it in a register in order to perform the copy. Copy
2975 the SOURCE value into its own register first, that way we
2976 don't have to reload SOURCE the next time it is used. */
2977 if (ts
->val_type
== TEMP_VAL_MEM
) {
2978 temp_load(s
, ts
, tcg_target_available_regs
[itype
], allocated_regs
);
2981 tcg_debug_assert(ts
->val_type
== TEMP_VAL_REG
);
2982 if (IS_DEAD_ARG(0) && !ots
->fixed_reg
) {
2983 /* mov to a non-saved dead register makes no sense (even with
2984 liveness analysis disabled). */
2985 tcg_debug_assert(NEED_SYNC_ARG(0));
2986 if (!ots
->mem_allocated
) {
2987 temp_allocate_frame(s
, ots
);
2989 tcg_out_st(s
, otype
, ts
->reg
, ots
->mem_base
->reg
, ots
->mem_offset
);
2990 if (IS_DEAD_ARG(1)) {
2995 if (IS_DEAD_ARG(1) && !ts
->fixed_reg
&& !ots
->fixed_reg
) {
2996 /* the mov can be suppressed */
2997 if (ots
->val_type
== TEMP_VAL_REG
) {
2998 s
->reg_to_temp
[ots
->reg
] = NULL
;
3003 if (ots
->val_type
!= TEMP_VAL_REG
) {
3004 /* When allocating a new register, make sure to not spill the
3006 tcg_regset_set_reg(allocated_regs
, ts
->reg
);
3007 ots
->reg
= tcg_reg_alloc(s
, tcg_target_available_regs
[otype
],
3008 allocated_regs
, ots
->indirect_base
);
3010 tcg_out_mov(s
, otype
, ots
->reg
, ts
->reg
);
3012 ots
->val_type
= TEMP_VAL_REG
;
3013 ots
->mem_coherent
= 0;
3014 s
->reg_to_temp
[ots
->reg
] = ots
;
3015 if (NEED_SYNC_ARG(0)) {
3016 temp_sync(s
, ots
, allocated_regs
, 0);
3021 static void tcg_reg_alloc_op(TCGContext
*s
, const TCGOp
*op
)
3023 const TCGLifeData arg_life
= op
->life
;
3024 const TCGOpDef
* const def
= &tcg_op_defs
[op
->opc
];
3025 TCGRegSet i_allocated_regs
;
3026 TCGRegSet o_allocated_regs
;
3027 int i
, k
, nb_iargs
, nb_oargs
;
3030 const TCGArgConstraint
*arg_ct
;
3032 TCGArg new_args
[TCG_MAX_OP_ARGS
];
3033 int const_args
[TCG_MAX_OP_ARGS
];
3035 nb_oargs
= def
->nb_oargs
;
3036 nb_iargs
= def
->nb_iargs
;
3038 /* copy constants */
3039 memcpy(new_args
+ nb_oargs
+ nb_iargs
,
3040 op
->args
+ nb_oargs
+ nb_iargs
,
3041 sizeof(TCGArg
) * def
->nb_cargs
);
3043 i_allocated_regs
= s
->reserved_regs
;
3044 o_allocated_regs
= s
->reserved_regs
;
3046 /* satisfy input constraints */
3047 for (k
= 0; k
< nb_iargs
; k
++) {
3048 i
= def
->sorted_args
[nb_oargs
+ k
];
3050 arg_ct
= &def
->args_ct
[i
];
3053 if (ts
->val_type
== TEMP_VAL_CONST
3054 && tcg_target_const_match(ts
->val
, ts
->type
, arg_ct
)) {
3055 /* constant is OK for instruction */
3057 new_args
[i
] = ts
->val
;
3061 temp_load(s
, ts
, arg_ct
->u
.regs
, i_allocated_regs
);
3063 if (arg_ct
->ct
& TCG_CT_IALIAS
) {
3064 if (ts
->fixed_reg
) {
3065 /* if fixed register, we must allocate a new register
3066 if the alias is not the same register */
3067 if (arg
!= op
->args
[arg_ct
->alias_index
])
3068 goto allocate_in_reg
;
3070 /* if the input is aliased to an output and if it is
3071 not dead after the instruction, we must allocate
3072 a new register and move it */
3073 if (!IS_DEAD_ARG(i
)) {
3074 goto allocate_in_reg
;
3076 /* check if the current register has already been allocated
3077 for another input aliased to an output */
3079 for (k2
= 0 ; k2
< k
; k2
++) {
3080 i2
= def
->sorted_args
[nb_oargs
+ k2
];
3081 if ((def
->args_ct
[i2
].ct
& TCG_CT_IALIAS
) &&
3082 (new_args
[i2
] == ts
->reg
)) {
3083 goto allocate_in_reg
;
3089 if (tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
3090 /* nothing to do : the constraint is satisfied */
3093 /* allocate a new register matching the constraint
3094 and move the temporary register into it */
3095 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, i_allocated_regs
,
3097 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
3101 tcg_regset_set_reg(i_allocated_regs
, reg
);
3105 /* mark dead temporaries and free the associated registers */
3106 for (i
= nb_oargs
; i
< nb_oargs
+ nb_iargs
; i
++) {
3107 if (IS_DEAD_ARG(i
)) {
3108 temp_dead(s
, arg_temp(op
->args
[i
]));
3112 if (def
->flags
& TCG_OPF_BB_END
) {
3113 tcg_reg_alloc_bb_end(s
, i_allocated_regs
);
3115 if (def
->flags
& TCG_OPF_CALL_CLOBBER
) {
3116 /* XXX: permit generic clobber register list ? */
3117 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3118 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3119 tcg_reg_free(s
, i
, i_allocated_regs
);
3123 if (def
->flags
& TCG_OPF_SIDE_EFFECTS
) {
3124 /* sync globals if the op has side effects and might trigger
3126 sync_globals(s
, i_allocated_regs
);
3129 /* satisfy the output constraints */
3130 for(k
= 0; k
< nb_oargs
; k
++) {
3131 i
= def
->sorted_args
[k
];
3133 arg_ct
= &def
->args_ct
[i
];
3135 if ((arg_ct
->ct
& TCG_CT_ALIAS
)
3136 && !const_args
[arg_ct
->alias_index
]) {
3137 reg
= new_args
[arg_ct
->alias_index
];
3138 } else if (arg_ct
->ct
& TCG_CT_NEWREG
) {
3139 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
,
3140 i_allocated_regs
| o_allocated_regs
,
3143 /* if fixed register, we try to use it */
3145 if (ts
->fixed_reg
&&
3146 tcg_regset_test_reg(arg_ct
->u
.regs
, reg
)) {
3149 reg
= tcg_reg_alloc(s
, arg_ct
->u
.regs
, o_allocated_regs
,
3152 tcg_regset_set_reg(o_allocated_regs
, reg
);
3153 /* if a fixed register is used, then a move will be done afterwards */
3154 if (!ts
->fixed_reg
) {
3155 if (ts
->val_type
== TEMP_VAL_REG
) {
3156 s
->reg_to_temp
[ts
->reg
] = NULL
;
3158 ts
->val_type
= TEMP_VAL_REG
;
3160 /* temp value is modified, so the value kept in memory is
3161 potentially not the same */
3162 ts
->mem_coherent
= 0;
3163 s
->reg_to_temp
[reg
] = ts
;
3170 /* emit instruction */
3171 if (def
->flags
& TCG_OPF_VECTOR
) {
3172 tcg_out_vec_op(s
, op
->opc
, TCGOP_VECL(op
), TCGOP_VECE(op
),
3173 new_args
, const_args
);
3175 tcg_out_op(s
, op
->opc
, new_args
, const_args
);
3178 /* move the outputs in the correct register if needed */
3179 for(i
= 0; i
< nb_oargs
; i
++) {
3180 ts
= arg_temp(op
->args
[i
]);
3182 if (ts
->fixed_reg
&& ts
->reg
!= reg
) {
3183 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
3185 if (NEED_SYNC_ARG(i
)) {
3186 temp_sync(s
, ts
, o_allocated_regs
, IS_DEAD_ARG(i
));
3187 } else if (IS_DEAD_ARG(i
)) {
3193 #ifdef TCG_TARGET_STACK_GROWSUP
3194 #define STACK_DIR(x) (-(x))
3196 #define STACK_DIR(x) (x)
3199 static void tcg_reg_alloc_call(TCGContext
*s
, TCGOp
*op
)
3201 const int nb_oargs
= TCGOP_CALLO(op
);
3202 const int nb_iargs
= TCGOP_CALLI(op
);
3203 const TCGLifeData arg_life
= op
->life
;
3204 int flags
, nb_regs
, i
;
3208 intptr_t stack_offset
;
3209 size_t call_stack_size
;
3210 tcg_insn_unit
*func_addr
;
3212 TCGRegSet allocated_regs
;
3214 func_addr
= (tcg_insn_unit
*)(intptr_t)op
->args
[nb_oargs
+ nb_iargs
];
3215 flags
= op
->args
[nb_oargs
+ nb_iargs
+ 1];
3217 nb_regs
= ARRAY_SIZE(tcg_target_call_iarg_regs
);
3218 if (nb_regs
> nb_iargs
) {
3222 /* assign stack slots first */
3223 call_stack_size
= (nb_iargs
- nb_regs
) * sizeof(tcg_target_long
);
3224 call_stack_size
= (call_stack_size
+ TCG_TARGET_STACK_ALIGN
- 1) &
3225 ~(TCG_TARGET_STACK_ALIGN
- 1);
3226 allocate_args
= (call_stack_size
> TCG_STATIC_CALL_ARGS_SIZE
);
3227 if (allocate_args
) {
3228 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3229 preallocate call stack */
3233 stack_offset
= TCG_TARGET_CALL_STACK_OFFSET
;
3234 for (i
= nb_regs
; i
< nb_iargs
; i
++) {
3235 arg
= op
->args
[nb_oargs
+ i
];
3236 #ifdef TCG_TARGET_STACK_GROWSUP
3237 stack_offset
-= sizeof(tcg_target_long
);
3239 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3241 temp_load(s
, ts
, tcg_target_available_regs
[ts
->type
],
3243 tcg_out_st(s
, ts
->type
, ts
->reg
, TCG_REG_CALL_STACK
, stack_offset
);
3245 #ifndef TCG_TARGET_STACK_GROWSUP
3246 stack_offset
+= sizeof(tcg_target_long
);
3250 /* assign input registers */
3251 allocated_regs
= s
->reserved_regs
;
3252 for (i
= 0; i
< nb_regs
; i
++) {
3253 arg
= op
->args
[nb_oargs
+ i
];
3254 if (arg
!= TCG_CALL_DUMMY_ARG
) {
3256 reg
= tcg_target_call_iarg_regs
[i
];
3257 tcg_reg_free(s
, reg
, allocated_regs
);
3259 if (ts
->val_type
== TEMP_VAL_REG
) {
3260 if (ts
->reg
!= reg
) {
3261 tcg_out_mov(s
, ts
->type
, reg
, ts
->reg
);
3264 TCGRegSet arg_set
= 0;
3266 tcg_regset_set_reg(arg_set
, reg
);
3267 temp_load(s
, ts
, arg_set
, allocated_regs
);
3270 tcg_regset_set_reg(allocated_regs
, reg
);
3274 /* mark dead temporaries and free the associated registers */
3275 for (i
= nb_oargs
; i
< nb_iargs
+ nb_oargs
; i
++) {
3276 if (IS_DEAD_ARG(i
)) {
3277 temp_dead(s
, arg_temp(op
->args
[i
]));
3281 /* clobber call registers */
3282 for (i
= 0; i
< TCG_TARGET_NB_REGS
; i
++) {
3283 if (tcg_regset_test_reg(tcg_target_call_clobber_regs
, i
)) {
3284 tcg_reg_free(s
, i
, allocated_regs
);
3288 /* Save globals if they might be written by the helper, sync them if
3289 they might be read. */
3290 if (flags
& TCG_CALL_NO_READ_GLOBALS
) {
3292 } else if (flags
& TCG_CALL_NO_WRITE_GLOBALS
) {
3293 sync_globals(s
, allocated_regs
);
3295 save_globals(s
, allocated_regs
);
3298 tcg_out_call(s
, func_addr
);
3300 /* assign output registers and emit moves if needed */
3301 for(i
= 0; i
< nb_oargs
; i
++) {
3304 reg
= tcg_target_call_oarg_regs
[i
];
3305 tcg_debug_assert(s
->reg_to_temp
[reg
] == NULL
);
3307 if (ts
->fixed_reg
) {
3308 if (ts
->reg
!= reg
) {
3309 tcg_out_mov(s
, ts
->type
, ts
->reg
, reg
);
3312 if (ts
->val_type
== TEMP_VAL_REG
) {
3313 s
->reg_to_temp
[ts
->reg
] = NULL
;
3315 ts
->val_type
= TEMP_VAL_REG
;
3317 ts
->mem_coherent
= 0;
3318 s
->reg_to_temp
[reg
] = ts
;
3319 if (NEED_SYNC_ARG(i
)) {
3320 temp_sync(s
, ts
, allocated_regs
, IS_DEAD_ARG(i
));
3321 } else if (IS_DEAD_ARG(i
)) {
3328 #ifdef CONFIG_PROFILER
3330 /* avoid copy/paste errors */
3331 #define PROF_ADD(to, from, field) \
3333 (to)->field += atomic_read(&((from)->field)); \
3336 #define PROF_MAX(to, from, field) \
3338 typeof((from)->field) val__ = atomic_read(&((from)->field)); \
3339 if (val__ > (to)->field) { \
3340 (to)->field = val__; \
3344 /* Pass in a zero'ed @prof */
3346 void tcg_profile_snapshot(TCGProfile
*prof
, bool counters
, bool table
)
3348 unsigned int n_ctxs
= atomic_read(&n_tcg_ctxs
);
3351 for (i
= 0; i
< n_ctxs
; i
++) {
3352 TCGContext
*s
= atomic_read(&tcg_ctxs
[i
]);
3353 const TCGProfile
*orig
= &s
->prof
;
3356 PROF_ADD(prof
, orig
, tb_count1
);
3357 PROF_ADD(prof
, orig
, tb_count
);
3358 PROF_ADD(prof
, orig
, op_count
);
3359 PROF_MAX(prof
, orig
, op_count_max
);
3360 PROF_ADD(prof
, orig
, temp_count
);
3361 PROF_MAX(prof
, orig
, temp_count_max
);
3362 PROF_ADD(prof
, orig
, del_op_count
);
3363 PROF_ADD(prof
, orig
, code_in_len
);
3364 PROF_ADD(prof
, orig
, code_out_len
);
3365 PROF_ADD(prof
, orig
, search_out_len
);
3366 PROF_ADD(prof
, orig
, interm_time
);
3367 PROF_ADD(prof
, orig
, code_time
);
3368 PROF_ADD(prof
, orig
, la_time
);
3369 PROF_ADD(prof
, orig
, opt_time
);
3370 PROF_ADD(prof
, orig
, restore_count
);
3371 PROF_ADD(prof
, orig
, restore_time
);
3376 for (i
= 0; i
< NB_OPS
; i
++) {
3377 PROF_ADD(prof
, orig
, table_op_count
[i
]);
3386 static void tcg_profile_snapshot_counters(TCGProfile
*prof
)
3388 tcg_profile_snapshot(prof
, true, false);
3391 static void tcg_profile_snapshot_table(TCGProfile
*prof
)
3393 tcg_profile_snapshot(prof
, false, true);
3396 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
3398 TCGProfile prof
= {};
3401 tcg_profile_snapshot_table(&prof
);
3402 for (i
= 0; i
< NB_OPS
; i
++) {
3403 cpu_fprintf(f
, "%s %" PRId64
"\n", tcg_op_defs
[i
].name
,
3404 prof
.table_op_count
[i
]);
3408 void tcg_dump_op_count(FILE *f
, fprintf_function cpu_fprintf
)
3410 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
3415 int tcg_gen_code(TCGContext
*s
, TranslationBlock
*tb
)
3417 #ifdef CONFIG_PROFILER
3418 TCGProfile
*prof
= &s
->prof
;
3423 #ifdef CONFIG_PROFILER
3427 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
3430 atomic_set(&prof
->op_count
, prof
->op_count
+ n
);
3431 if (n
> prof
->op_count_max
) {
3432 atomic_set(&prof
->op_count_max
, n
);
3436 atomic_set(&prof
->temp_count
, prof
->temp_count
+ n
);
3437 if (n
> prof
->temp_count_max
) {
3438 atomic_set(&prof
->temp_count_max
, n
);
3444 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP
)
3445 && qemu_log_in_addr_range(tb
->pc
))) {
3454 #ifdef CONFIG_PROFILER
3455 atomic_set(&prof
->opt_time
, prof
->opt_time
- profile_getclock());
3458 #ifdef USE_TCG_OPTIMIZATIONS
3462 #ifdef CONFIG_PROFILER
3463 atomic_set(&prof
->opt_time
, prof
->opt_time
+ profile_getclock());
3464 atomic_set(&prof
->la_time
, prof
->la_time
- profile_getclock());
3469 if (s
->nb_indirects
> 0) {
3471 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND
)
3472 && qemu_log_in_addr_range(tb
->pc
))) {
3474 qemu_log("OP before indirect lowering:\n");
3480 /* Replace indirect temps with direct temps. */
3481 if (liveness_pass_2(s
)) {
3482 /* If changes were made, re-run liveness. */
3487 #ifdef CONFIG_PROFILER
3488 atomic_set(&prof
->la_time
, prof
->la_time
+ profile_getclock());
3492 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT
)
3493 && qemu_log_in_addr_range(tb
->pc
))) {
3495 qemu_log("OP after optimization and liveness analysis:\n");
3502 tcg_reg_alloc_start(s
);
3504 s
->code_buf
= tb
->tc
.ptr
;
3505 s
->code_ptr
= tb
->tc
.ptr
;
3507 #ifdef TCG_TARGET_NEED_LDST_LABELS
3508 QSIMPLEQ_INIT(&s
->ldst_labels
);
3510 #ifdef TCG_TARGET_NEED_POOL_LABELS
3511 s
->pool_labels
= NULL
;
3515 QTAILQ_FOREACH(op
, &s
->ops
, link
) {
3516 TCGOpcode opc
= op
->opc
;
3518 #ifdef CONFIG_PROFILER
3519 atomic_set(&prof
->table_op_count
[opc
], prof
->table_op_count
[opc
] + 1);
3523 case INDEX_op_mov_i32
:
3524 case INDEX_op_mov_i64
:
3525 case INDEX_op_mov_vec
:
3526 tcg_reg_alloc_mov(s
, op
);
3528 case INDEX_op_movi_i32
:
3529 case INDEX_op_movi_i64
:
3530 case INDEX_op_dupi_vec
:
3531 tcg_reg_alloc_movi(s
, op
);
3533 case INDEX_op_insn_start
:
3534 if (num_insns
>= 0) {
3535 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
3538 for (i
= 0; i
< TARGET_INSN_START_WORDS
; ++i
) {
3540 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
3541 a
= deposit64(op
->args
[i
* 2], 32, 32, op
->args
[i
* 2 + 1]);
3545 s
->gen_insn_data
[num_insns
][i
] = a
;
3548 case INDEX_op_discard
:
3549 temp_dead(s
, arg_temp(op
->args
[0]));
3551 case INDEX_op_set_label
:
3552 tcg_reg_alloc_bb_end(s
, s
->reserved_regs
);
3553 tcg_out_label(s
, arg_label(op
->args
[0]), s
->code_ptr
);
3556 tcg_reg_alloc_call(s
, op
);
3559 /* Sanity check that we've not introduced any unhandled opcodes. */
3560 tcg_debug_assert(tcg_op_supported(opc
));
3561 /* Note: in order to speed up the code, it would be much
3562 faster to have specialized register allocator functions for
3563 some common argument patterns */
3564 tcg_reg_alloc_op(s
, op
);
3567 #ifdef CONFIG_DEBUG_TCG
3570 /* Test for (pending) buffer overflow. The assumption is that any
3571 one operation beginning below the high water mark cannot overrun
3572 the buffer completely. Thus we can test for overflow after
3573 generating code without having to check during generation. */
3574 if (unlikely((void *)s
->code_ptr
> s
->code_gen_highwater
)) {
3578 tcg_debug_assert(num_insns
>= 0);
3579 s
->gen_insn_end_off
[num_insns
] = tcg_current_code_size(s
);
3581 /* Generate TB finalization at the end of block */
3582 #ifdef TCG_TARGET_NEED_LDST_LABELS
3583 if (!tcg_out_ldst_finalize(s
)) {
3587 #ifdef TCG_TARGET_NEED_POOL_LABELS
3588 if (!tcg_out_pool_finalize(s
)) {
3593 /* flush instruction cache */
3594 flush_icache_range((uintptr_t)s
->code_buf
, (uintptr_t)s
->code_ptr
);
3596 return tcg_current_code_size(s
);
3599 #ifdef CONFIG_PROFILER
3600 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
3602 TCGProfile prof
= {};
3603 const TCGProfile
*s
;
3605 int64_t tb_div_count
;
3608 tcg_profile_snapshot_counters(&prof
);
3610 tb_count
= s
->tb_count
;
3611 tb_div_count
= tb_count
? tb_count
: 1;
3612 tot
= s
->interm_time
+ s
->code_time
;
3614 cpu_fprintf(f
, "JIT cycles %" PRId64
" (%0.3f s at 2.4 GHz)\n",
3616 cpu_fprintf(f
, "translated TBs %" PRId64
" (aborted=%" PRId64
" %0.1f%%)\n",
3617 tb_count
, s
->tb_count1
- tb_count
,
3618 (double)(s
->tb_count1
- s
->tb_count
)
3619 / (s
->tb_count1
? s
->tb_count1
: 1) * 100.0);
3620 cpu_fprintf(f
, "avg ops/TB %0.1f max=%d\n",
3621 (double)s
->op_count
/ tb_div_count
, s
->op_count_max
);
3622 cpu_fprintf(f
, "deleted ops/TB %0.2f\n",
3623 (double)s
->del_op_count
/ tb_div_count
);
3624 cpu_fprintf(f
, "avg temps/TB %0.2f max=%d\n",
3625 (double)s
->temp_count
/ tb_div_count
, s
->temp_count_max
);
3626 cpu_fprintf(f
, "avg host code/TB %0.1f\n",
3627 (double)s
->code_out_len
/ tb_div_count
);
3628 cpu_fprintf(f
, "avg search data/TB %0.1f\n",
3629 (double)s
->search_out_len
/ tb_div_count
);
3631 cpu_fprintf(f
, "cycles/op %0.1f\n",
3632 s
->op_count
? (double)tot
/ s
->op_count
: 0);
3633 cpu_fprintf(f
, "cycles/in byte %0.1f\n",
3634 s
->code_in_len
? (double)tot
/ s
->code_in_len
: 0);
3635 cpu_fprintf(f
, "cycles/out byte %0.1f\n",
3636 s
->code_out_len
? (double)tot
/ s
->code_out_len
: 0);
3637 cpu_fprintf(f
, "cycles/search byte %0.1f\n",
3638 s
->search_out_len
? (double)tot
/ s
->search_out_len
: 0);
3642 cpu_fprintf(f
, " gen_interm time %0.1f%%\n",
3643 (double)s
->interm_time
/ tot
* 100.0);
3644 cpu_fprintf(f
, " gen_code time %0.1f%%\n",
3645 (double)s
->code_time
/ tot
* 100.0);
3646 cpu_fprintf(f
, "optim./code time %0.1f%%\n",
3647 (double)s
->opt_time
/ (s
->code_time
? s
->code_time
: 1)
3649 cpu_fprintf(f
, "liveness/code time %0.1f%%\n",
3650 (double)s
->la_time
/ (s
->code_time
? s
->code_time
: 1) * 100.0);
3651 cpu_fprintf(f
, "cpu_restore count %" PRId64
"\n",
3653 cpu_fprintf(f
, " avg cycles %0.1f\n",
3654 s
->restore_count
? (double)s
->restore_time
/ s
->restore_count
: 0);
3657 void tcg_dump_info(FILE *f
, fprintf_function cpu_fprintf
)
3659 cpu_fprintf(f
, "[TCG profiler not compiled]\n");
3663 #ifdef ELF_HOST_MACHINE
3664 /* In order to use this feature, the backend needs to do three things:
3666 (1) Define ELF_HOST_MACHINE to indicate both what value to
3667 put into the ELF image and to indicate support for the feature.
3669 (2) Define tcg_register_jit. This should create a buffer containing
3670 the contents of a .debug_frame section that describes the post-
3671 prologue unwind info for the tcg machine.
3673 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
3676 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
3683 struct jit_code_entry
{
3684 struct jit_code_entry
*next_entry
;
3685 struct jit_code_entry
*prev_entry
;
3686 const void *symfile_addr
;
3687 uint64_t symfile_size
;
3690 struct jit_descriptor
{
3692 uint32_t action_flag
;
3693 struct jit_code_entry
*relevant_entry
;
3694 struct jit_code_entry
*first_entry
;
3697 void __jit_debug_register_code(void) __attribute__((noinline
));
3698 void __jit_debug_register_code(void)
3703 /* Must statically initialize the version, because GDB may check
3704 the version before we can set it. */
3705 struct jit_descriptor __jit_debug_descriptor
= { 1, 0, 0, 0 };
3707 /* End GDB interface. */
3709 static int find_string(const char *strtab
, const char *str
)
3711 const char *p
= strtab
+ 1;
3714 if (strcmp(p
, str
) == 0) {
3721 static void tcg_register_jit_int(void *buf_ptr
, size_t buf_size
,
3722 const void *debug_frame
,
3723 size_t debug_frame_size
)
3725 struct __attribute__((packed
)) DebugInfo
{
3732 uintptr_t cu_low_pc
;
3733 uintptr_t cu_high_pc
;
3736 uintptr_t fn_low_pc
;
3737 uintptr_t fn_high_pc
;
3746 struct DebugInfo di
;
3751 struct ElfImage
*img
;
3753 static const struct ElfImage img_template
= {
3755 .e_ident
[EI_MAG0
] = ELFMAG0
,
3756 .e_ident
[EI_MAG1
] = ELFMAG1
,
3757 .e_ident
[EI_MAG2
] = ELFMAG2
,
3758 .e_ident
[EI_MAG3
] = ELFMAG3
,
3759 .e_ident
[EI_CLASS
] = ELF_CLASS
,
3760 .e_ident
[EI_DATA
] = ELF_DATA
,
3761 .e_ident
[EI_VERSION
] = EV_CURRENT
,
3763 .e_machine
= ELF_HOST_MACHINE
,
3764 .e_version
= EV_CURRENT
,
3765 .e_phoff
= offsetof(struct ElfImage
, phdr
),
3766 .e_shoff
= offsetof(struct ElfImage
, shdr
),
3767 .e_ehsize
= sizeof(ElfW(Shdr
)),
3768 .e_phentsize
= sizeof(ElfW(Phdr
)),
3770 .e_shentsize
= sizeof(ElfW(Shdr
)),
3771 .e_shnum
= ARRAY_SIZE(img
->shdr
),
3772 .e_shstrndx
= ARRAY_SIZE(img
->shdr
) - 1,
3773 #ifdef ELF_HOST_FLAGS
3774 .e_flags
= ELF_HOST_FLAGS
,
3777 .e_ident
[EI_OSABI
] = ELF_OSABI
,
3785 [0] = { .sh_type
= SHT_NULL
},
3786 /* Trick: The contents of code_gen_buffer are not present in
3787 this fake ELF file; that got allocated elsewhere. Therefore
3788 we mark .text as SHT_NOBITS (similar to .bss) so that readers
3789 will not look for contents. We can record any address. */
3791 .sh_type
= SHT_NOBITS
,
3792 .sh_flags
= SHF_EXECINSTR
| SHF_ALLOC
,
3794 [2] = { /* .debug_info */
3795 .sh_type
= SHT_PROGBITS
,
3796 .sh_offset
= offsetof(struct ElfImage
, di
),
3797 .sh_size
= sizeof(struct DebugInfo
),
3799 [3] = { /* .debug_abbrev */
3800 .sh_type
= SHT_PROGBITS
,
3801 .sh_offset
= offsetof(struct ElfImage
, da
),
3802 .sh_size
= sizeof(img
->da
),
3804 [4] = { /* .debug_frame */
3805 .sh_type
= SHT_PROGBITS
,
3806 .sh_offset
= sizeof(struct ElfImage
),
3808 [5] = { /* .symtab */
3809 .sh_type
= SHT_SYMTAB
,
3810 .sh_offset
= offsetof(struct ElfImage
, sym
),
3811 .sh_size
= sizeof(img
->sym
),
3813 .sh_link
= ARRAY_SIZE(img
->shdr
) - 1,
3814 .sh_entsize
= sizeof(ElfW(Sym
)),
3816 [6] = { /* .strtab */
3817 .sh_type
= SHT_STRTAB
,
3818 .sh_offset
= offsetof(struct ElfImage
, str
),
3819 .sh_size
= sizeof(img
->str
),
3823 [1] = { /* code_gen_buffer */
3824 .st_info
= ELF_ST_INFO(STB_GLOBAL
, STT_FUNC
),
3829 .len
= sizeof(struct DebugInfo
) - 4,
3831 .ptr_size
= sizeof(void *),
3833 .cu_lang
= 0x8001, /* DW_LANG_Mips_Assembler */
3835 .fn_name
= "code_gen_buffer"
3838 1, /* abbrev number (the cu) */
3839 0x11, 1, /* DW_TAG_compile_unit, has children */
3840 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
3841 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
3842 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
3843 0, 0, /* end of abbrev */
3844 2, /* abbrev number (the fn) */
3845 0x2e, 0, /* DW_TAG_subprogram, no children */
3846 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
3847 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
3848 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
3849 0, 0, /* end of abbrev */
3850 0 /* no more abbrev */
3852 .str
= "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
3853 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
3856 /* We only need a single jit entry; statically allocate it. */
3857 static struct jit_code_entry one_entry
;
3859 uintptr_t buf
= (uintptr_t)buf_ptr
;
3860 size_t img_size
= sizeof(struct ElfImage
) + debug_frame_size
;
3861 DebugFrameHeader
*dfh
;
3863 img
= g_malloc(img_size
);
3864 *img
= img_template
;
3866 img
->phdr
.p_vaddr
= buf
;
3867 img
->phdr
.p_paddr
= buf
;
3868 img
->phdr
.p_memsz
= buf_size
;
3870 img
->shdr
[1].sh_name
= find_string(img
->str
, ".text");
3871 img
->shdr
[1].sh_addr
= buf
;
3872 img
->shdr
[1].sh_size
= buf_size
;
3874 img
->shdr
[2].sh_name
= find_string(img
->str
, ".debug_info");
3875 img
->shdr
[3].sh_name
= find_string(img
->str
, ".debug_abbrev");
3877 img
->shdr
[4].sh_name
= find_string(img
->str
, ".debug_frame");
3878 img
->shdr
[4].sh_size
= debug_frame_size
;
3880 img
->shdr
[5].sh_name
= find_string(img
->str
, ".symtab");
3881 img
->shdr
[6].sh_name
= find_string(img
->str
, ".strtab");
3883 img
->sym
[1].st_name
= find_string(img
->str
, "code_gen_buffer");
3884 img
->sym
[1].st_value
= buf
;
3885 img
->sym
[1].st_size
= buf_size
;
3887 img
->di
.cu_low_pc
= buf
;
3888 img
->di
.cu_high_pc
= buf
+ buf_size
;
3889 img
->di
.fn_low_pc
= buf
;
3890 img
->di
.fn_high_pc
= buf
+ buf_size
;
3892 dfh
= (DebugFrameHeader
*)(img
+ 1);
3893 memcpy(dfh
, debug_frame
, debug_frame_size
);
3894 dfh
->fde
.func_start
= buf
;
3895 dfh
->fde
.func_len
= buf_size
;
3898 /* Enable this block to be able to debug the ELF image file creation.
3899 One can use readelf, objdump, or other inspection utilities. */
3901 FILE *f
= fopen("/tmp/qemu.jit", "w+b");
3903 if (fwrite(img
, img_size
, 1, f
) != img_size
) {
3904 /* Avoid stupid unused return value warning for fwrite. */
3911 one_entry
.symfile_addr
= img
;
3912 one_entry
.symfile_size
= img_size
;
3914 __jit_debug_descriptor
.action_flag
= JIT_REGISTER_FN
;
3915 __jit_debug_descriptor
.relevant_entry
= &one_entry
;
3916 __jit_debug_descriptor
.first_entry
= &one_entry
;
3917 __jit_debug_register_code();
3920 /* No support for the feature. Provide the entry point expected by exec.c,
3921 and implement the internal function we declared earlier. */
3923 static void tcg_register_jit_int(void *buf
, size_t size
,
3924 const void *debug_frame
,
3925 size_t debug_frame_size
)
3929 void tcg_register_jit(void *buf
, size_t buf_size
)
3932 #endif /* ELF_HOST_MACHINE */
3934 #if !TCG_TARGET_MAYBE_vec
3935 void tcg_expand_vec_op(TCGOpcode o
, TCGType t
, unsigned e
, TCGArg a0
, ...)
3937 g_assert_not_reached();