#include "qemu/cacheflush.h"
#include "qemu/cacheinfo.h"
#include "qemu/timer.h"
-
-/* Note: the long term plan is to reduce the dependencies on the QEMU
- CPU definitions. Currently they are used for qemu_ld/st
- instructions */
-#define NO_CPU_IO_DEFS
-
-#include "exec/exec-all.h"
+#include "exec/translation-block.h"
#include "exec/tlb-common.h"
+#include "tcg/startup.h"
#include "tcg/tcg-op-common.h"
#if UINTPTR_MAX == UINT32_MAX
#include "tcg/tcg-ldst.h"
#include "tcg/tcg-temp-internal.h"
#include "tcg-internal.h"
-#include "accel/tcg/perf.h"
+#include "tcg/perf.h"
#ifdef CONFIG_USER_ONLY
-#include "exec/user/guest-base.h"
+#include "user/guest-base.h"
#endif
/* Forward declarations for functions declared in tcg-target.c.inc and
__attribute__((unused));
/* Forward declarations for functions declared and used in tcg-target.c.inc. */
+static void tcg_out_tb_start(TCGContext *s);
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
const TCGHelperInfo *info);
static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
-static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
+static bool tcg_target_const_match(int64_t val, int ct,
+ TCGType type, TCGCond cond, int vece);
#ifdef TCG_TARGET_NEED_LDST_LABELS
static int tcg_out_ldst_finalize(TCGContext *s);
#endif
+#ifndef CONFIG_USER_ONLY
+#define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
+#endif
+
typedef struct TCGLdstHelperParam {
TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
unsigned ntmp;
MemOp host_atom, bool allow_two_ops)
__attribute__((unused));
+#ifdef CONFIG_USER_ONLY
+bool tcg_use_softmmu;
+#endif
+
TCGContext tcg_init_ctx;
__thread TCGContext *tcg_ctx;
TCGContext **tcg_ctxs;
unsigned int tcg_cur_ctxs;
unsigned int tcg_max_ctxs;
-TCGv_env cpu_env = 0;
+TCGv_env tcg_env;
const void *tcg_code_gen_epilogue;
uintptr_t tcg_splitwx_diff;
return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
}
-#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER)
-static int tlb_mask_table_ofs(TCGContext *s, int which)
+static int __attribute__((unused))
+tlb_mask_table_ofs(TCGContext *s, int which)
{
- return s->tlb_fast_offset + which * sizeof(CPUTLBDescFast);
+ return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
+ sizeof(CPUNegativeOffsetState));
}
-#endif
/* Signal overflow, starting over with fewer guest insns. */
static G_NORETURN
#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
+#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
+#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
typedef enum {
#include "tcg-target-con-set.h"
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
+#undef C_N1O1_I1
+#undef C_N2_I1
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
+#undef C_N1_O1_I4
/* Put all of the constraint sets into an array, indexed by the enum. */
#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
+#define C_N1O1_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, #O2, #I1 } },
+#define C_N2_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, "&" #O2, #I1 } },
#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
static const TCGTargetOpDef constraint_sets[] = {
#include "tcg-target-con-set.h"
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
+#undef C_N1O1_I1
+#undef C_N2_I1
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
+#undef C_N1_O1_I4
/* Expand the enumerator to be returned from tcg_target_op_def(). */
#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
+#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
+#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
#include "tcg-target.c.inc"
-static void alloc_tcg_plugin_context(TCGContext *s)
-{
-#ifdef CONFIG_PLUGIN
- s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
- s->plugin_tb->insns =
- g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
+#ifndef CONFIG_TCG_INTERPRETER
+/* Validate CPUTLBDescFast placement. */
+QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
+ sizeof(CPUNegativeOffsetState))
+ < MIN_TLB_MASK_TABLE_OFS);
#endif
-}
/*
* All TCG threads except the parent (i.e. the one that called tcg_context_init
* In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
* of tcg_region_init() for the reasoning behind this.
*
- * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
- * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
+ * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
+ * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
* is not used anymore for translation once this function is called.
*
- * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
- * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
+ * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
+ * iterates over the array (e.g. tcg_code_size() the same for both system/user
+ * modes.
*/
#ifdef CONFIG_USER_ONLY
void tcg_register_thread(void)
qatomic_set(&tcg_ctxs[n], s);
if (n > 0) {
- alloc_tcg_plugin_context(s);
tcg_region_initial_alloc(s);
}
.ref_slot = cum->ref_slot + i,
};
}
- cum->info_in_idx += n;
+ cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
cum->ref_slot += n;
}
indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
}
- alloc_tcg_plugin_context(s);
-
tcg_ctx = s;
/*
* In user-mode we simply share the init context among threads, since we
* use a single region. See the documentation tcg_region_init() for the
* reasoning behind this.
- * In softmmu we will have at most max_cpus TCG threads.
+ * In system-mode we will have at most max_cpus TCG threads.
*/
#ifdef CONFIG_USER_ONLY
tcg_ctxs = &tcg_ctx;
tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
- cpu_env = temp_tcgv_ptr(ts);
+ tcg_env = temp_tcgv_ptr(ts);
}
void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
return tb;
}
-void tcg_prologue_init(TCGContext *s)
+void tcg_prologue_init(void)
{
+ TCGContext *s = tcg_ctx;
size_t prologue_size;
s->code_ptr = s->code_gen_ptr;
QTAILQ_INIT(&s->ops);
QTAILQ_INIT(&s->free_ops);
+ s->emit_before_op = NULL;
QSIMPLEQ_INIT(&s->labels);
tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
s->addr_type == TCG_TYPE_I64);
-#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER)
- tcg_debug_assert(s->tlb_fast_offset < 0);
- tcg_debug_assert(s->tlb_fast_offset >= MIN_TLB_MASK_TABLE_OFS);
-#endif
+ tcg_debug_assert(s->insn_start_words > 0);
}
static TCGTemp *tcg_temp_alloc(TCGContext *s)
= tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
}
-TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
- intptr_t offset, const char *name)
+static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
+ const char *name, TCGType type)
{
TCGContext *s = tcg_ctx;
TCGTemp *base_ts = tcgv_ptr_temp(base);
return ts;
}
+TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
+ return temp_tcgv_i32(ts);
+}
+
+TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
+ return temp_tcgv_i64(ts);
+}
+
+TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
+ return temp_tcgv_ptr(ts);
+}
+
TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
{
TCGContext *s = tcg_ctx;
return ts;
}
+TCGv_i32 tcg_temp_new_i32(void)
+{
+ return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
+}
+
+TCGv_i32 tcg_temp_ebb_new_i32(void)
+{
+ return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
+}
+
+TCGv_i64 tcg_temp_new_i64(void)
+{
+ return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
+}
+
+TCGv_i64 tcg_temp_ebb_new_i64(void)
+{
+ return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
+}
+
+TCGv_ptr tcg_temp_new_ptr(void)
+{
+ return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
+}
+
+TCGv_ptr tcg_temp_ebb_new_ptr(void)
+{
+ return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
+}
+
+TCGv_i128 tcg_temp_new_i128(void)
+{
+ return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
+}
+
+TCGv_i128 tcg_temp_ebb_new_i128(void)
+{
+ return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
+}
+
TCGv_vec tcg_temp_new_vec(TCGType type)
{
TCGTemp *t;
}
}
+void tcg_temp_free_i32(TCGv_i32 arg)
+{
+ tcg_temp_free_internal(tcgv_i32_temp(arg));
+}
+
+void tcg_temp_free_i64(TCGv_i64 arg)
+{
+ tcg_temp_free_internal(tcgv_i64_temp(arg));
+}
+
+void tcg_temp_free_i128(TCGv_i128 arg)
+{
+ tcg_temp_free_internal(tcgv_i128_temp(arg));
+}
+
+void tcg_temp_free_ptr(TCGv_ptr arg)
+{
+ tcg_temp_free_internal(tcgv_ptr_temp(arg));
+}
+
+void tcg_temp_free_vec(TCGv_vec arg)
+{
+ tcg_temp_free_internal(tcgv_vec_temp(arg));
+}
+
TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
{
TCGContext *s = tcg_ctx;
return ts;
}
+TCGv_i32 tcg_constant_i32(int32_t val)
+{
+ return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
+}
+
+TCGv_i64 tcg_constant_i64(int64_t val)
+{
+ return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
+}
+
+TCGv_ptr tcg_constant_ptr_int(intptr_t val)
+{
+ return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
+}
+
TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
{
val = dup_const(vece, val);
case INDEX_op_mov_i32:
case INDEX_op_setcond_i32:
case INDEX_op_brcond_i32:
+ case INDEX_op_movcond_i32:
case INDEX_op_ld8u_i32:
case INDEX_op_ld8s_i32:
case INDEX_op_ld16u_i32:
case INDEX_op_st_i32:
case INDEX_op_add_i32:
case INDEX_op_sub_i32:
+ case INDEX_op_neg_i32:
case INDEX_op_mul_i32:
case INDEX_op_and_i32:
case INDEX_op_or_i32:
case INDEX_op_sar_i32:
return true;
- case INDEX_op_movcond_i32:
- return TCG_TARGET_HAS_movcond_i32;
+ case INDEX_op_negsetcond_i32:
+ return TCG_TARGET_HAS_negsetcond_i32;
case INDEX_op_div_i32:
case INDEX_op_divu_i32:
return TCG_TARGET_HAS_div_i32;
return TCG_TARGET_HAS_bswap32_i32;
case INDEX_op_not_i32:
return TCG_TARGET_HAS_not_i32;
- case INDEX_op_neg_i32:
- return TCG_TARGET_HAS_neg_i32;
case INDEX_op_andc_i32:
return TCG_TARGET_HAS_andc_i32;
case INDEX_op_orc_i32:
case INDEX_op_mov_i64:
case INDEX_op_setcond_i64:
case INDEX_op_brcond_i64:
+ case INDEX_op_movcond_i64:
case INDEX_op_ld8u_i64:
case INDEX_op_ld8s_i64:
case INDEX_op_ld16u_i64:
case INDEX_op_st_i64:
case INDEX_op_add_i64:
case INDEX_op_sub_i64:
+ case INDEX_op_neg_i64:
case INDEX_op_mul_i64:
case INDEX_op_and_i64:
case INDEX_op_or_i64:
case INDEX_op_extu_i32_i64:
return TCG_TARGET_REG_BITS == 64;
- case INDEX_op_movcond_i64:
- return TCG_TARGET_HAS_movcond_i64;
+ case INDEX_op_negsetcond_i64:
+ return TCG_TARGET_HAS_negsetcond_i64;
case INDEX_op_div_i64:
case INDEX_op_divu_i64:
return TCG_TARGET_HAS_div_i64;
case INDEX_op_extract2_i64:
return TCG_TARGET_HAS_extract2_i64;
case INDEX_op_extrl_i64_i32:
- return TCG_TARGET_HAS_extrl_i64_i32;
case INDEX_op_extrh_i64_i32:
- return TCG_TARGET_HAS_extrh_i64_i32;
+ return TCG_TARGET_HAS_extr_i64_i32;
case INDEX_op_ext8s_i64:
return TCG_TARGET_HAS_ext8s_i64;
case INDEX_op_ext16s_i64:
return TCG_TARGET_HAS_bswap64_i64;
case INDEX_op_not_i64:
return TCG_TARGET_HAS_not_i64;
- case INDEX_op_neg_i64:
- return TCG_TARGET_HAS_neg_i64;
case INDEX_op_andc_i64:
return TCG_TARGET_HAS_andc_i64;
case INDEX_op_orc_i64:
static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
-void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, int nargs, TCGTemp **args)
+static void tcg_gen_callN(void *func, TCGHelperInfo *info,
+ TCGTemp *ret, TCGTemp **args)
{
TCGv_i64 extend_free[MAX_CALL_IARGS];
int n_extend = 0;
#ifdef CONFIG_PLUGIN
/* Flag helpers that may affect guest state */
- if (tcg_ctx->plugin_insn &&
- !(info->flags & TCG_CALL_PLUGIN) &&
- !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
+ if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
tcg_ctx->plugin_insn->calls_helpers = true;
}
#endif
g_assert_not_reached();
}
}
- op->args[pi++] = (uintptr_t)info->func;
+ op->args[pi++] = (uintptr_t)func;
op->args[pi++] = (uintptr_t)info;
tcg_debug_assert(pi == total_args);
- QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
+ if (tcg_ctx->emit_before_op) {
+ QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
+ } else {
+ QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
+ }
tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
for (i = 0; i < n_extend; ++i) {
}
}
+void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret)
+{
+ tcg_gen_callN(func, info, ret, NULL);
+}
+
+void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
+{
+ tcg_gen_callN(func, info, ret, &t1);
+}
+
+void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret,
+ TCGTemp *t1, TCGTemp *t2)
+{
+ TCGTemp *args[2] = { t1, t2 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
+void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret,
+ TCGTemp *t1, TCGTemp *t2, TCGTemp *t3)
+{
+ TCGTemp *args[3] = { t1, t2, t3 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
+void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret,
+ TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
+{
+ TCGTemp *args[4] = { t1, t2, t3, t4 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
+void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
+{
+ TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
+void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret,
+ TCGTemp *t1, TCGTemp *t2, TCGTemp *t3,
+ TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
+{
+ TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
+void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
+ TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
+{
+ TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
+ tcg_gen_callN(func, info, ret, args);
+}
+
static void tcg_reg_alloc_start(TCGContext *s)
{
int i, n;
[TCG_COND_LTU] = "ltu",
[TCG_COND_GEU] = "geu",
[TCG_COND_LEU] = "leu",
- [TCG_COND_GTU] = "gtu"
+ [TCG_COND_GTU] = "gtu",
+ [TCG_COND_TSTEQ] = "tsteq",
+ [TCG_COND_TSTNE] = "tstne",
};
static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
[TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
};
+#ifdef CONFIG_PLUGIN
+static const char * const plugin_from_name[] = {
+ "from-tb",
+ "from-insn",
+ "after-insn",
+ "after-tb",
+};
+#endif
+
static inline bool tcg_regset_single(TCGRegSet d)
{
return (d & (d - 1)) == 0;
#define ne_fprintf(...) \
({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
-static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
+void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
{
char buf[128];
TCGOp *op;
nb_oargs = 0;
col += ne_fprintf(f, "\n ----");
- for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+ for (i = 0, k = s->insn_start_words; i < k; ++i) {
col += ne_fprintf(f, " %016" PRIx64,
tcg_get_insn_start_param(op, i));
}
switch (c) {
case INDEX_op_brcond_i32:
case INDEX_op_setcond_i32:
+ case INDEX_op_negsetcond_i32:
case INDEX_op_movcond_i32:
case INDEX_op_brcond2_i32:
case INDEX_op_setcond2_i32:
case INDEX_op_brcond_i64:
case INDEX_op_setcond_i64:
+ case INDEX_op_negsetcond_i64:
case INDEX_op_movcond_i64:
case INDEX_op_cmp_vec:
case INDEX_op_cmpsel_vec:
{
const char *s_al, *s_op, *s_at;
MemOpIdx oi = op->args[k++];
- MemOp op = get_memop(oi);
+ MemOp mop = get_memop(oi);
unsigned ix = get_mmuidx(oi);
- s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
- s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
- s_at = atom_name[(op & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
- op &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
+ s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
+ s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
+ s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
+ mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
/* If all fields are accounted for, print symbolically. */
- if (!op && s_al && s_op && s_at) {
+ if (!mop && s_al && s_op && s_at) {
col += ne_fprintf(f, ",%s%s%s,%u",
s_at, s_al, s_op, ix);
} else {
- op = get_memop(oi);
- col += ne_fprintf(f, ",$0x%x,%u", op, ix);
+ mop = get_memop(oi);
+ col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
}
i = 1;
}
i = k = 1;
}
break;
+#ifdef CONFIG_PLUGIN
+ case INDEX_op_plugin_cb:
+ {
+ TCGArg from = op->args[k++];
+ const char *name = NULL;
+
+ if (from < ARRAY_SIZE(plugin_from_name)) {
+ name = plugin_from_name[from];
+ }
+ if (name) {
+ col += ne_fprintf(f, "%s", name);
+ } else {
+ col += ne_fprintf(f, "$0x%" TCG_PRIlx, from);
+ }
+ i = 1;
+ }
+ break;
+#endif
default:
i = 0;
break;
.pair = 2,
.pair_index = o,
.regs = def->args_ct[o].regs << 1,
+ .newreg = def->args_ct[o].newreg,
};
def->args_ct[o].pair = 1;
def->args_ct[o].pair_index = i;
.pair = 1,
.pair_index = o,
.regs = def->args_ct[o].regs >> 1,
+ .newreg = def->args_ct[o].newreg,
};
def->args_ct[o].pair = 2;
def->args_ct[o].pair_index = i;
QTAILQ_REMOVE(&s->ops, op, link);
QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
s->nb_ops--;
-
-#ifdef CONFIG_PROFILER
- qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
-#endif
}
void tcg_remove_ops_after(TCGOp *op)
TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
{
TCGOp *op = tcg_op_alloc(opc, nargs);
- QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
+
+ if (tcg_ctx->emit_before_op) {
+ QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
+ } else {
+ QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
+ }
return op;
}
TCGTemp *ts;
TCGArg new_args[TCG_MAX_OP_ARGS];
int const_args[TCG_MAX_OP_ARGS];
+ TCGCond op_cond;
nb_oargs = def->nb_oargs;
nb_iargs = def->nb_iargs;
i_allocated_regs = s->reserved_regs;
o_allocated_regs = s->reserved_regs;
+ switch (op->opc) {
+ case INDEX_op_brcond_i32:
+ case INDEX_op_brcond_i64:
+ op_cond = op->args[2];
+ break;
+ case INDEX_op_setcond_i32:
+ case INDEX_op_setcond_i64:
+ case INDEX_op_negsetcond_i32:
+ case INDEX_op_negsetcond_i64:
+ case INDEX_op_cmp_vec:
+ op_cond = op->args[3];
+ break;
+ case INDEX_op_brcond2_i32:
+ op_cond = op->args[4];
+ break;
+ case INDEX_op_movcond_i32:
+ case INDEX_op_movcond_i64:
+ case INDEX_op_setcond2_i32:
+ case INDEX_op_cmpsel_vec:
+ op_cond = op->args[5];
+ break;
+ default:
+ /* No condition within opcode. */
+ op_cond = TCG_COND_ALWAYS;
+ break;
+ }
+
/* satisfy input constraints */
for (k = 0; k < nb_iargs; k++) {
TCGRegSet i_preferred_regs, i_required_regs;
ts = arg_temp(arg);
if (ts->val_type == TEMP_VAL_CONST
- && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
+ && tcg_target_const_match(ts->val, arg_ct->ct, ts->type,
+ op_cond, TCGOP_VECE(op))) {
/* constant is OK for instruction */
const_args[i] = 1;
new_args[i] = ts->val;
* dead after the instruction, we must allocate a new
* register and move it.
*/
- if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
+ if (temp_readonly(ts) || !IS_DEAD_ARG(i)
+ || def->args_ct[arg_ct->alias_index].newreg) {
allocate_new_reg = true;
} else if (ts->val_type == TEMP_VAL_REG) {
/*
break;
case 1: /* first of pair */
- tcg_debug_assert(!arg_ct->newreg);
if (arg_ct->oalias) {
reg = new_args[arg_ct->alias_index];
- break;
+ } else if (arg_ct->newreg) {
+ reg = tcg_reg_alloc_pair(s, arg_ct->regs,
+ i_allocated_regs | o_allocated_regs,
+ output_pref(op, k),
+ ts->indirect_base);
+ } else {
+ reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
+ output_pref(op, k),
+ ts->indirect_base);
}
- reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
- output_pref(op, k), ts->indirect_base);
break;
case 2: /* second of pair */
- tcg_debug_assert(!arg_ct->newreg);
if (arg_ct->oalias) {
reg = new_args[arg_ct->alias_index];
} else {
MemOp align = get_alignment_bits(opc);
MemOp size = opc & MO_SIZE;
MemOp half = size ? size - 1 : 0;
+ MemOp atom = opc & MO_ATOM_MASK;
MemOp atmax;
- MemOp atom;
-
- /* When serialized, no further atomicity required. */
- if (s->gen_tb->cflags & CF_PARALLEL) {
- atom = opc & MO_ATOM_MASK;
- } else {
- atom = MO_ATOM_NONE;
- }
switch (atom) {
case MO_ATOM_NONE:
tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
}
-#ifdef CONFIG_PROFILER
-
-/* avoid copy/paste errors */
-#define PROF_ADD(to, from, field) \
- do { \
- (to)->field += qatomic_read(&((from)->field)); \
- } while (0)
-
-#define PROF_MAX(to, from, field) \
- do { \
- typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
- if (val__ > (to)->field) { \
- (to)->field = val__; \
- } \
- } while (0)
-
-/* Pass in a zero'ed @prof */
-static inline
-void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
-{
- unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
- unsigned int i;
-
- for (i = 0; i < n_ctxs; i++) {
- TCGContext *s = qatomic_read(&tcg_ctxs[i]);
- const TCGProfile *orig = &s->prof;
-
- if (counters) {
- PROF_ADD(prof, orig, cpu_exec_time);
- PROF_ADD(prof, orig, tb_count1);
- PROF_ADD(prof, orig, tb_count);
- PROF_ADD(prof, orig, op_count);
- PROF_MAX(prof, orig, op_count_max);
- PROF_ADD(prof, orig, temp_count);
- PROF_MAX(prof, orig, temp_count_max);
- PROF_ADD(prof, orig, del_op_count);
- PROF_ADD(prof, orig, code_in_len);
- PROF_ADD(prof, orig, code_out_len);
- PROF_ADD(prof, orig, search_out_len);
- PROF_ADD(prof, orig, interm_time);
- PROF_ADD(prof, orig, code_time);
- PROF_ADD(prof, orig, la_time);
- PROF_ADD(prof, orig, opt_time);
- PROF_ADD(prof, orig, restore_count);
- PROF_ADD(prof, orig, restore_time);
- }
- if (table) {
- int i;
-
- for (i = 0; i < NB_OPS; i++) {
- PROF_ADD(prof, orig, table_op_count[i]);
- }
- }
- }
-}
-
-#undef PROF_ADD
-#undef PROF_MAX
-
-static void tcg_profile_snapshot_counters(TCGProfile *prof)
-{
- tcg_profile_snapshot(prof, true, false);
-}
-
-static void tcg_profile_snapshot_table(TCGProfile *prof)
-{
- tcg_profile_snapshot(prof, false, true);
-}
-
-void tcg_dump_op_count(GString *buf)
-{
- TCGProfile prof = {};
- int i;
-
- tcg_profile_snapshot_table(&prof);
- for (i = 0; i < NB_OPS; i++) {
- g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
- prof.table_op_count[i]);
- }
-}
-
-int64_t tcg_cpu_exec_time(void)
-{
- unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
- unsigned int i;
- int64_t ret = 0;
-
- for (i = 0; i < n_ctxs; i++) {
- const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
- const TCGProfile *prof = &s->prof;
-
- ret += qatomic_read(&prof->cpu_exec_time);
- }
- return ret;
-}
-#else
-void tcg_dump_op_count(GString *buf)
-{
- g_string_append_printf(buf, "[TCG profiler not compiled]\n");
-}
-
-int64_t tcg_cpu_exec_time(void)
-{
- error_report("%s: TCG profiler not compiled", __func__);
- exit(EXIT_FAILURE);
-}
-#endif
-
-
int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
{
-#ifdef CONFIG_PROFILER
- TCGProfile *prof = &s->prof;
-#endif
- int i, num_insns;
+ int i, start_words, num_insns;
TCGOp *op;
-#ifdef CONFIG_PROFILER
- {
- int n = 0;
-
- QTAILQ_FOREACH(op, &s->ops, link) {
- n++;
- }
- qatomic_set(&prof->op_count, prof->op_count + n);
- if (n > prof->op_count_max) {
- qatomic_set(&prof->op_count_max, n);
- }
-
- n = s->nb_temps;
- qatomic_set(&prof->temp_count, prof->temp_count + n);
- if (n > prof->temp_count_max) {
- qatomic_set(&prof->temp_count_max, n);
- }
- }
-#endif
-
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
}
#endif
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
-#endif
-
tcg_optimize(s);
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
- qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
-#endif
-
reachable_code_pass(s);
liveness_pass_0(s);
liveness_pass_1(s);
}
}
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
-#endif
-
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
s->pool_labels = NULL;
#endif
+ start_words = s->insn_start_words;
+ s->gen_insn_data =
+ tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words);
+
+ tcg_out_tb_start(s);
+
num_insns = -1;
QTAILQ_FOREACH(op, &s->ops, link) {
TCGOpcode opc = op->opc;
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
-#endif
-
switch (opc) {
case INDEX_op_mov_i32:
case INDEX_op_mov_i64:
assert(s->gen_insn_end_off[num_insns] == off);
}
num_insns++;
- for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
- s->gen_insn_data[num_insns][i] =
+ for (i = 0; i < start_words; ++i) {
+ s->gen_insn_data[num_insns * start_words + i] =
tcg_get_insn_start_param(op, i);
}
break;
return -2;
}
}
- tcg_debug_assert(num_insns >= 0);
+ tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
/* Generate TB finalization at the end of block */
return tcg_current_code_size(s);
}
-#ifdef CONFIG_PROFILER
-void tcg_dump_info(GString *buf)
-{
- TCGProfile prof = {};
- const TCGProfile *s;
- int64_t tb_count;
- int64_t tb_div_count;
- int64_t tot;
-
- tcg_profile_snapshot_counters(&prof);
- s = &prof;
- tb_count = s->tb_count;
- tb_div_count = tb_count ? tb_count : 1;
- tot = s->interm_time + s->code_time;
-
- g_string_append_printf(buf, "JIT cycles %" PRId64
- " (%0.3f s at 2.4 GHz)\n",
- tot, tot / 2.4e9);
- g_string_append_printf(buf, "translated TBs %" PRId64
- " (aborted=%" PRId64 " %0.1f%%)\n",
- tb_count, s->tb_count1 - tb_count,
- (double)(s->tb_count1 - s->tb_count)
- / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
- g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
- (double)s->op_count / tb_div_count, s->op_count_max);
- g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
- (double)s->del_op_count / tb_div_count);
- g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
- (double)s->temp_count / tb_div_count,
- s->temp_count_max);
- g_string_append_printf(buf, "avg host code/TB %0.1f\n",
- (double)s->code_out_len / tb_div_count);
- g_string_append_printf(buf, "avg search data/TB %0.1f\n",
- (double)s->search_out_len / tb_div_count);
-
- g_string_append_printf(buf, "cycles/op %0.1f\n",
- s->op_count ? (double)tot / s->op_count : 0);
- g_string_append_printf(buf, "cycles/in byte %0.1f\n",
- s->code_in_len ? (double)tot / s->code_in_len : 0);
- g_string_append_printf(buf, "cycles/out byte %0.1f\n",
- s->code_out_len ? (double)tot / s->code_out_len : 0);
- g_string_append_printf(buf, "cycles/search byte %0.1f\n",
- s->search_out_len ?
- (double)tot / s->search_out_len : 0);
- if (tot == 0) {
- tot = 1;
- }
- g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
- (double)s->interm_time / tot * 100.0);
- g_string_append_printf(buf, " gen_code time %0.1f%%\n",
- (double)s->code_time / tot * 100.0);
- g_string_append_printf(buf, "optim./code time %0.1f%%\n",
- (double)s->opt_time / (s->code_time ?
- s->code_time : 1)
- * 100.0);
- g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
- (double)s->la_time / (s->code_time ?
- s->code_time : 1) * 100.0);
- g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
- s->restore_count);
- g_string_append_printf(buf, " avg cycles %0.1f\n",
- s->restore_count ?
- (double)s->restore_time / s->restore_count : 0);
-}
-#else
-void tcg_dump_info(GString *buf)
-{
- g_string_append_printf(buf, "[TCG profiler not compiled]\n");
-}
-#endif
-
#ifdef ELF_HOST_MACHINE
/* In order to use this feature, the backend needs to do three things:
projects / mirror_qemu.git / blobdiff