#include "qemu/cacheflush.h"
#include "qemu/cacheinfo.h"
#include "qemu/timer.h"
-
-/* Note: the long term plan is to reduce the dependencies on the QEMU
- CPU definitions. Currently they are used for qemu_ld/st
- instructions */
-#define NO_CPU_IO_DEFS
-
-#include "exec/exec-all.h"
-#include "tcg/tcg-op.h"
+#include "exec/translation-block.h"
+#include "exec/tlb-common.h"
+#include "tcg/startup.h"
+#include "tcg/tcg-op-common.h"
#if UINTPTR_MAX == UINT32_MAX
# define ELF_CLASS ELFCLASS32
#include "tcg/tcg-ldst.h"
#include "tcg/tcg-temp-internal.h"
#include "tcg-internal.h"
-#include "accel/tcg/perf.h"
+#include "tcg/perf.h"
#ifdef CONFIG_USER_ONLY
#include "exec/user/guest-base.h"
#endif
__attribute__((unused));
/* Forward declarations for functions declared and used in tcg-target.c.inc. */
+static void tcg_out_tb_start(TCGContext *s);
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
const TCGHelperInfo *info);
static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
-static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
+static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece);
#ifdef TCG_TARGET_NEED_LDST_LABELS
static int tcg_out_ldst_finalize(TCGContext *s);
#endif
+#ifndef CONFIG_USER_ONLY
+#define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
+#endif
+
typedef struct TCGLdstHelperParam {
TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
unsigned ntmp;
MemOp host_atom, bool allow_two_ops)
__attribute__((unused));
+#ifdef CONFIG_USER_ONLY
+bool tcg_use_softmmu;
+#endif
+
TCGContext tcg_init_ctx;
__thread TCGContext *tcg_ctx;
TCGContext **tcg_ctxs;
unsigned int tcg_cur_ctxs;
unsigned int tcg_max_ctxs;
-TCGv_env cpu_env = 0;
+TCGv_env tcg_env;
const void *tcg_code_gen_epilogue;
uintptr_t tcg_splitwx_diff;
return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
}
+static int __attribute__((unused))
+tlb_mask_table_ofs(TCGContext *s, int which)
+{
+ return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
+ sizeof(CPUNegativeOffsetState));
+}
+
/* Signal overflow, starting over with fewer guest insns. */
static G_NORETURN
void tcg_raise_tb_overflow(TCGContext *s)
#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
+#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
+#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
typedef enum {
#include "tcg-target-con-set.h"
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
+#undef C_N1O1_I1
+#undef C_N2_I1
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
+#undef C_N1_O1_I4
/* Put all of the constraint sets into an array, indexed by the enum. */
#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
+#define C_N1O1_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, #O2, #I1 } },
+#define C_N2_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, "&" #O2, #I1 } },
#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
static const TCGTargetOpDef constraint_sets[] = {
#include "tcg-target-con-set.h"
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
+#undef C_N1O1_I1
+#undef C_N2_I1
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
+#undef C_N1_O1_I4
/* Expand the enumerator to be returned from tcg_target_op_def(). */
#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
+#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
+#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
+#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
#include "tcg-target.c.inc"
+#ifndef CONFIG_TCG_INTERPRETER
+/* Validate CPUTLBDescFast placement. */
+QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
+ sizeof(CPUNegativeOffsetState))
+ < MIN_TLB_MASK_TABLE_OFS);
+#endif
+
static void alloc_tcg_plugin_context(TCGContext *s)
{
#ifdef CONFIG_PLUGIN
* In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
* of tcg_region_init() for the reasoning behind this.
*
- * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
- * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
+ * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
+ * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
* is not used anymore for translation once this function is called.
*
- * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
- * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
+ * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
+ * iterates over the array (e.g. tcg_code_size() the same for both system/user
+ * modes.
*/
#ifdef CONFIG_USER_ONLY
void tcg_register_thread(void)
s->pool_current = NULL;
}
-#include "exec/helper-proto.h"
-
-static TCGHelperInfo all_helpers[] = {
-#include "exec/helper-tcg.h"
-};
-static GHashTable *helper_table;
-
/*
* Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
* akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
g_assert_not_reached();
}
-static void init_ffi_layouts(void)
+static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
{
- /* g_direct_hash/equal for direct comparisons on uint32_t. */
- GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
-
- for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
- TCGHelperInfo *info = &all_helpers[i];
- unsigned typemask = info->typemask;
- gpointer hash = (gpointer)(uintptr_t)typemask;
- struct {
- ffi_cif cif;
- ffi_type *args[];
- } *ca;
- ffi_status status;
- int nargs;
- ffi_cif *cif;
-
- cif = g_hash_table_lookup(ffi_table, hash);
- if (cif) {
- info->cif = cif;
- continue;
- }
-
- /* Ignoring the return type, find the last non-zero field. */
- nargs = 32 - clz32(typemask >> 3);
- nargs = DIV_ROUND_UP(nargs, 3);
- assert(nargs <= MAX_CALL_IARGS);
-
- ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
- ca->cif.rtype = typecode_to_ffi(typemask & 7);
- ca->cif.nargs = nargs;
-
- if (nargs != 0) {
- ca->cif.arg_types = ca->args;
- for (int j = 0; j < nargs; ++j) {
- int typecode = extract32(typemask, (j + 1) * 3, 3);
- ca->args[j] = typecode_to_ffi(typecode);
- }
+ unsigned typemask = info->typemask;
+ struct {
+ ffi_cif cif;
+ ffi_type *args[];
+ } *ca;
+ ffi_status status;
+ int nargs;
+
+ /* Ignoring the return type, find the last non-zero field. */
+ nargs = 32 - clz32(typemask >> 3);
+ nargs = DIV_ROUND_UP(nargs, 3);
+ assert(nargs <= MAX_CALL_IARGS);
+
+ ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
+ ca->cif.rtype = typecode_to_ffi(typemask & 7);
+ ca->cif.nargs = nargs;
+
+ if (nargs != 0) {
+ ca->cif.arg_types = ca->args;
+ for (int j = 0; j < nargs; ++j) {
+ int typecode = extract32(typemask, (j + 1) * 3, 3);
+ ca->args[j] = typecode_to_ffi(typecode);
}
-
- status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
- ca->cif.rtype, ca->cif.arg_types);
- assert(status == FFI_OK);
-
- cif = &ca->cif;
- info->cif = cif;
- g_hash_table_insert(ffi_table, hash, (gpointer)cif);
}
- g_hash_table_destroy(ffi_table);
+ status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
+ ca->cif.rtype, ca->cif.arg_types);
+ assert(status == FFI_OK);
+
+ return &ca->cif;
}
+
+#define HELPER_INFO_INIT(I) (&(I)->cif)
+#define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
+#else
+#define HELPER_INFO_INIT(I) (&(I)->init)
+#define HELPER_INFO_INIT_VAL(I) 1
#endif /* CONFIG_TCG_INTERPRETER */
static inline bool arg_slot_reg_p(unsigned arg_slot)
.ref_slot = cum->ref_slot + i,
};
}
- cum->info_in_idx += n;
+ cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
cum->ref_slot += n;
}
args_ct += n;
}
- /* Register helpers. */
- /* Use g_direct_hash/equal for direct pointer comparisons on func. */
- helper_table = g_hash_table_new(NULL, NULL);
-
- for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
- init_call_layout(&all_helpers[i]);
- g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
- (gpointer)&all_helpers[i]);
- }
-
init_call_layout(&info_helper_ld32_mmu);
init_call_layout(&info_helper_ld64_mmu);
init_call_layout(&info_helper_ld128_mmu);
init_call_layout(&info_helper_st64_mmu);
init_call_layout(&info_helper_st128_mmu);
-#ifdef CONFIG_TCG_INTERPRETER
- init_ffi_layouts();
-#endif
-
tcg_target_init(s);
process_op_defs(s);
* In user-mode we simply share the init context among threads, since we
* use a single region. See the documentation tcg_region_init() for the
* reasoning behind this.
- * In softmmu we will have at most max_cpus TCG threads.
+ * In system-mode we will have at most max_cpus TCG threads.
*/
#ifdef CONFIG_USER_ONLY
tcg_ctxs = &tcg_ctx;
tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
- cpu_env = temp_tcgv_ptr(ts);
+ tcg_env = temp_tcgv_ptr(ts);
}
void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
return tb;
}
-void tcg_prologue_init(TCGContext *s)
+void tcg_prologue_init(void)
{
+ TCGContext *s = tcg_ctx;
size_t prologue_size;
s->code_ptr = s->code_gen_ptr;
tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
s->addr_type == TCG_TYPE_I64);
+
+ tcg_debug_assert(s->insn_start_words > 0);
}
static TCGTemp *tcg_temp_alloc(TCGContext *s)
= tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
}
-TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
- intptr_t offset, const char *name)
+static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
+ const char *name, TCGType type)
{
TCGContext *s = tcg_ctx;
TCGTemp *base_ts = tcgv_ptr_temp(base);
return ts;
}
-TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
+TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
+ return temp_tcgv_i32(ts);
+}
+
+TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
+ return temp_tcgv_i64(ts);
+}
+
+TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
+{
+ TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
+ return temp_tcgv_ptr(ts);
+}
+
+static TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
{
TCGContext *s = tcg_ctx;
TCGTemp *ts;
return ts;
}
+TCGv_i32 tcg_temp_new_i32(void)
+{
+ return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
+}
+
+TCGv_i32 tcg_temp_ebb_new_i32(void)
+{
+ return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
+}
+
+TCGv_i64 tcg_temp_new_i64(void)
+{
+ return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
+}
+
+TCGv_i64 tcg_temp_ebb_new_i64(void)
+{
+ return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
+}
+
+TCGv_ptr tcg_temp_new_ptr(void)
+{
+ return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
+}
+
+TCGv_ptr tcg_temp_ebb_new_ptr(void)
+{
+ return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
+}
+
+TCGv_i128 tcg_temp_new_i128(void)
+{
+ return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
+}
+
+TCGv_i128 tcg_temp_ebb_new_i128(void)
+{
+ return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
+}
+
TCGv_vec tcg_temp_new_vec(TCGType type)
{
TCGTemp *t;
}
}
+void tcg_temp_free_i32(TCGv_i32 arg)
+{
+ tcg_temp_free_internal(tcgv_i32_temp(arg));
+}
+
+void tcg_temp_free_i64(TCGv_i64 arg)
+{
+ tcg_temp_free_internal(tcgv_i64_temp(arg));
+}
+
+void tcg_temp_free_i128(TCGv_i128 arg)
+{
+ tcg_temp_free_internal(tcgv_i128_temp(arg));
+}
+
+void tcg_temp_free_ptr(TCGv_ptr arg)
+{
+ tcg_temp_free_internal(tcgv_ptr_temp(arg));
+}
+
+void tcg_temp_free_vec(TCGv_vec arg)
+{
+ tcg_temp_free_internal(tcgv_vec_temp(arg));
+}
+
TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
{
TCGContext *s = tcg_ctx;
return ts;
}
+TCGv_i32 tcg_constant_i32(int32_t val)
+{
+ return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
+}
+
+TCGv_i64 tcg_constant_i64(int64_t val)
+{
+ return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
+}
+
+TCGv_ptr tcg_constant_ptr_int(intptr_t val)
+{
+ return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
+}
+
TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
{
val = dup_const(vece, val);
return tcg_constant_vec(t->base_type, vece, val);
}
+#ifdef CONFIG_DEBUG_TCG
+size_t temp_idx(TCGTemp *ts)
+{
+ ptrdiff_t n = ts - tcg_ctx->temps;
+ assert(n >= 0 && n < tcg_ctx->nb_temps);
+ return n;
+}
+
+TCGTemp *tcgv_i32_temp(TCGv_i32 v)
+{
+ uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
+
+ assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
+ assert(o % sizeof(TCGTemp) == 0);
+
+ return (void *)tcg_ctx + (uintptr_t)v;
+}
+#endif /* CONFIG_DEBUG_TCG */
+
/* Return true if OP may appear in the opcode stream.
Test the runtime variable that controls each opcode. */
bool tcg_op_supported(TCGOpcode op)
case INDEX_op_mov_i32:
case INDEX_op_setcond_i32:
case INDEX_op_brcond_i32:
+ case INDEX_op_movcond_i32:
case INDEX_op_ld8u_i32:
case INDEX_op_ld8s_i32:
case INDEX_op_ld16u_i32:
case INDEX_op_st_i32:
case INDEX_op_add_i32:
case INDEX_op_sub_i32:
+ case INDEX_op_neg_i32:
case INDEX_op_mul_i32:
case INDEX_op_and_i32:
case INDEX_op_or_i32:
case INDEX_op_sar_i32:
return true;
- case INDEX_op_movcond_i32:
- return TCG_TARGET_HAS_movcond_i32;
+ case INDEX_op_negsetcond_i32:
+ return TCG_TARGET_HAS_negsetcond_i32;
case INDEX_op_div_i32:
case INDEX_op_divu_i32:
return TCG_TARGET_HAS_div_i32;
return TCG_TARGET_HAS_bswap32_i32;
case INDEX_op_not_i32:
return TCG_TARGET_HAS_not_i32;
- case INDEX_op_neg_i32:
- return TCG_TARGET_HAS_neg_i32;
case INDEX_op_andc_i32:
return TCG_TARGET_HAS_andc_i32;
case INDEX_op_orc_i32:
case INDEX_op_mov_i64:
case INDEX_op_setcond_i64:
case INDEX_op_brcond_i64:
+ case INDEX_op_movcond_i64:
case INDEX_op_ld8u_i64:
case INDEX_op_ld8s_i64:
case INDEX_op_ld16u_i64:
case INDEX_op_st_i64:
case INDEX_op_add_i64:
case INDEX_op_sub_i64:
+ case INDEX_op_neg_i64:
case INDEX_op_mul_i64:
case INDEX_op_and_i64:
case INDEX_op_or_i64:
case INDEX_op_extu_i32_i64:
return TCG_TARGET_REG_BITS == 64;
- case INDEX_op_movcond_i64:
- return TCG_TARGET_HAS_movcond_i64;
+ case INDEX_op_negsetcond_i64:
+ return TCG_TARGET_HAS_negsetcond_i64;
case INDEX_op_div_i64:
case INDEX_op_divu_i64:
return TCG_TARGET_HAS_div_i64;
case INDEX_op_extract2_i64:
return TCG_TARGET_HAS_extract2_i64;
case INDEX_op_extrl_i64_i32:
- return TCG_TARGET_HAS_extrl_i64_i32;
case INDEX_op_extrh_i64_i32:
- return TCG_TARGET_HAS_extrh_i64_i32;
+ return TCG_TARGET_HAS_extr_i64_i32;
case INDEX_op_ext8s_i64:
return TCG_TARGET_HAS_ext8s_i64;
case INDEX_op_ext16s_i64:
return TCG_TARGET_HAS_bswap64_i64;
case INDEX_op_not_i64:
return TCG_TARGET_HAS_not_i64;
- case INDEX_op_neg_i64:
- return TCG_TARGET_HAS_neg_i64;
case INDEX_op_andc_i64:
return TCG_TARGET_HAS_andc_i64;
case INDEX_op_orc_i64:
static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
-void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
+static void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, TCGTemp **args)
{
- const TCGHelperInfo *info;
TCGv_i64 extend_free[MAX_CALL_IARGS];
int n_extend = 0;
TCGOp *op;
int i, n, pi = 0, total_args;
- info = g_hash_table_lookup(helper_table, (gpointer)func);
+ if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
+ init_call_layout(info);
+ g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
+ }
+
total_args = info->nr_out + info->nr_in + 2;
op = tcg_op_alloc(INDEX_op_call, total_args);
g_assert_not_reached();
}
}
- op->args[pi++] = (uintptr_t)func;
+ op->args[pi++] = (uintptr_t)info->func;
op->args[pi++] = (uintptr_t)info;
tcg_debug_assert(pi == total_args);
}
}
+void tcg_gen_call0(TCGHelperInfo *info, TCGTemp *ret)
+{
+ tcg_gen_callN(info, ret, NULL);
+}
+
+void tcg_gen_call1(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
+{
+ tcg_gen_callN(info, ret, &t1);
+}
+
+void tcg_gen_call2(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2)
+{
+ TCGTemp *args[2] = { t1, t2 };
+ tcg_gen_callN(info, ret, args);
+}
+
+void tcg_gen_call3(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3)
+{
+ TCGTemp *args[3] = { t1, t2, t3 };
+ tcg_gen_callN(info, ret, args);
+}
+
+void tcg_gen_call4(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
+{
+ TCGTemp *args[4] = { t1, t2, t3, t4 };
+ tcg_gen_callN(info, ret, args);
+}
+
+void tcg_gen_call5(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
+{
+ TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
+ tcg_gen_callN(info, ret, args);
+}
+
+void tcg_gen_call6(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2,
+ TCGTemp *t3, TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
+{
+ TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
+ tcg_gen_callN(info, ret, args);
+}
+
+void tcg_gen_call7(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
+ TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
+ TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
+{
+ TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
+ tcg_gen_callN(info, ret, args);
+}
+
static void tcg_reg_alloc_start(TCGContext *s)
{
int i, n;
nb_oargs = 0;
col += ne_fprintf(f, "\n ----");
- for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+ for (i = 0, k = s->insn_start_words; i < k; ++i) {
col += ne_fprintf(f, " %016" PRIx64,
tcg_get_insn_start_param(op, i));
}
switch (c) {
case INDEX_op_brcond_i32:
case INDEX_op_setcond_i32:
+ case INDEX_op_negsetcond_i32:
case INDEX_op_movcond_i32:
case INDEX_op_brcond2_i32:
case INDEX_op_setcond2_i32:
case INDEX_op_brcond_i64:
case INDEX_op_setcond_i64:
+ case INDEX_op_negsetcond_i64:
case INDEX_op_movcond_i64:
case INDEX_op_cmp_vec:
case INDEX_op_cmpsel_vec:
{
const char *s_al, *s_op, *s_at;
MemOpIdx oi = op->args[k++];
- MemOp op = get_memop(oi);
+ MemOp mop = get_memop(oi);
unsigned ix = get_mmuidx(oi);
- s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
- s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
- s_at = atom_name[(op & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
- op &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
+ s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
+ s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
+ s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
+ mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
/* If all fields are accounted for, print symbolically. */
- if (!op && s_al && s_op && s_at) {
+ if (!mop && s_al && s_op && s_at) {
col += ne_fprintf(f, ",%s%s%s,%u",
s_at, s_al, s_op, ix);
} else {
- op = get_memop(oi);
- col += ne_fprintf(f, ",$0x%x,%u", op, ix);
+ mop = get_memop(oi);
+ col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
}
i = 1;
}
.pair = 2,
.pair_index = o,
.regs = def->args_ct[o].regs << 1,
+ .newreg = def->args_ct[o].newreg,
};
def->args_ct[o].pair = 1;
def->args_ct[o].pair_index = i;
.pair = 1,
.pair_index = o,
.regs = def->args_ct[o].regs >> 1,
+ .newreg = def->args_ct[o].newreg,
};
def->args_ct[o].pair = 2;
def->args_ct[o].pair_index = i;
QTAILQ_REMOVE(&s->ops, op, link);
QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
s->nb_ops--;
-
-#ifdef CONFIG_PROFILER
- qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
-#endif
}
void tcg_remove_ops_after(TCGOp *op)
ts = arg_temp(arg);
if (ts->val_type == TEMP_VAL_CONST
- && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
+ && tcg_target_const_match(ts->val, ts->type, arg_ct->ct, TCGOP_VECE(op))) {
/* constant is OK for instruction */
const_args[i] = 1;
new_args[i] = ts->val;
* dead after the instruction, we must allocate a new
* register and move it.
*/
- if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
+ if (temp_readonly(ts) || !IS_DEAD_ARG(i)
+ || def->args_ct[arg_ct->alias_index].newreg) {
allocate_new_reg = true;
} else if (ts->val_type == TEMP_VAL_REG) {
/*
break;
case 1: /* first of pair */
- tcg_debug_assert(!arg_ct->newreg);
if (arg_ct->oalias) {
reg = new_args[arg_ct->alias_index];
- break;
+ } else if (arg_ct->newreg) {
+ reg = tcg_reg_alloc_pair(s, arg_ct->regs,
+ i_allocated_regs | o_allocated_regs,
+ output_pref(op, k),
+ ts->indirect_base);
+ } else {
+ reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
+ output_pref(op, k),
+ ts->indirect_base);
}
- reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
- output_pref(op, k), ts->indirect_base);
break;
case 2: /* second of pair */
- tcg_debug_assert(!arg_ct->newreg);
if (arg_ct->oalias) {
reg = new_args[arg_ct->alias_index];
} else {
MemOp align = get_alignment_bits(opc);
MemOp size = opc & MO_SIZE;
MemOp half = size ? size - 1 : 0;
+ MemOp atom = opc & MO_ATOM_MASK;
MemOp atmax;
- MemOp atom;
-
- /* When serialized, no further atomicity required. */
- if (s->gen_tb->cflags & CF_PARALLEL) {
- atom = opc & MO_ATOM_MASK;
- } else {
- atom = MO_ATOM_NONE;
- }
switch (atom) {
case MO_ATOM_NONE:
tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
}
-#ifdef CONFIG_PROFILER
-
-/* avoid copy/paste errors */
-#define PROF_ADD(to, from, field) \
- do { \
- (to)->field += qatomic_read(&((from)->field)); \
- } while (0)
-
-#define PROF_MAX(to, from, field) \
- do { \
- typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
- if (val__ > (to)->field) { \
- (to)->field = val__; \
- } \
- } while (0)
-
-/* Pass in a zero'ed @prof */
-static inline
-void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
-{
- unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
- unsigned int i;
-
- for (i = 0; i < n_ctxs; i++) {
- TCGContext *s = qatomic_read(&tcg_ctxs[i]);
- const TCGProfile *orig = &s->prof;
-
- if (counters) {
- PROF_ADD(prof, orig, cpu_exec_time);
- PROF_ADD(prof, orig, tb_count1);
- PROF_ADD(prof, orig, tb_count);
- PROF_ADD(prof, orig, op_count);
- PROF_MAX(prof, orig, op_count_max);
- PROF_ADD(prof, orig, temp_count);
- PROF_MAX(prof, orig, temp_count_max);
- PROF_ADD(prof, orig, del_op_count);
- PROF_ADD(prof, orig, code_in_len);
- PROF_ADD(prof, orig, code_out_len);
- PROF_ADD(prof, orig, search_out_len);
- PROF_ADD(prof, orig, interm_time);
- PROF_ADD(prof, orig, code_time);
- PROF_ADD(prof, orig, la_time);
- PROF_ADD(prof, orig, opt_time);
- PROF_ADD(prof, orig, restore_count);
- PROF_ADD(prof, orig, restore_time);
- }
- if (table) {
- int i;
-
- for (i = 0; i < NB_OPS; i++) {
- PROF_ADD(prof, orig, table_op_count[i]);
- }
- }
- }
-}
-
-#undef PROF_ADD
-#undef PROF_MAX
-
-static void tcg_profile_snapshot_counters(TCGProfile *prof)
-{
- tcg_profile_snapshot(prof, true, false);
-}
-
-static void tcg_profile_snapshot_table(TCGProfile *prof)
-{
- tcg_profile_snapshot(prof, false, true);
-}
-
-void tcg_dump_op_count(GString *buf)
-{
- TCGProfile prof = {};
- int i;
-
- tcg_profile_snapshot_table(&prof);
- for (i = 0; i < NB_OPS; i++) {
- g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
- prof.table_op_count[i]);
- }
-}
-
-int64_t tcg_cpu_exec_time(void)
-{
- unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
- unsigned int i;
- int64_t ret = 0;
-
- for (i = 0; i < n_ctxs; i++) {
- const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
- const TCGProfile *prof = &s->prof;
-
- ret += qatomic_read(&prof->cpu_exec_time);
- }
- return ret;
-}
-#else
-void tcg_dump_op_count(GString *buf)
-{
- g_string_append_printf(buf, "[TCG profiler not compiled]\n");
-}
-
-int64_t tcg_cpu_exec_time(void)
-{
- error_report("%s: TCG profiler not compiled", __func__);
- exit(EXIT_FAILURE);
-}
-#endif
-
-
int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
{
-#ifdef CONFIG_PROFILER
- TCGProfile *prof = &s->prof;
-#endif
- int i, num_insns;
+ int i, start_words, num_insns;
TCGOp *op;
-#ifdef CONFIG_PROFILER
- {
- int n = 0;
-
- QTAILQ_FOREACH(op, &s->ops, link) {
- n++;
- }
- qatomic_set(&prof->op_count, prof->op_count + n);
- if (n > prof->op_count_max) {
- qatomic_set(&prof->op_count_max, n);
- }
-
- n = s->nb_temps;
- qatomic_set(&prof->temp_count, prof->temp_count + n);
- if (n > prof->temp_count_max) {
- qatomic_set(&prof->temp_count_max, n);
- }
- }
-#endif
-
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
}
#endif
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
-#endif
-
tcg_optimize(s);
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
- qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
-#endif
-
reachable_code_pass(s);
liveness_pass_0(s);
liveness_pass_1(s);
}
}
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
-#endif
-
if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
&& qemu_log_in_addr_range(pc_start))) {
FILE *logfile = qemu_log_trylock();
s->pool_labels = NULL;
#endif
+ start_words = s->insn_start_words;
+ s->gen_insn_data =
+ tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words);
+
+ tcg_out_tb_start(s);
+
num_insns = -1;
QTAILQ_FOREACH(op, &s->ops, link) {
TCGOpcode opc = op->opc;
-#ifdef CONFIG_PROFILER
- qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
-#endif
-
switch (opc) {
case INDEX_op_mov_i32:
case INDEX_op_mov_i64:
assert(s->gen_insn_end_off[num_insns] == off);
}
num_insns++;
- for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
- s->gen_insn_data[num_insns][i] =
+ for (i = 0; i < start_words; ++i) {
+ s->gen_insn_data[num_insns * start_words + i] =
tcg_get_insn_start_param(op, i);
}
break;
return -2;
}
}
- tcg_debug_assert(num_insns >= 0);
+ tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
/* Generate TB finalization at the end of block */
return tcg_current_code_size(s);
}
-#ifdef CONFIG_PROFILER
-void tcg_dump_info(GString *buf)
-{
- TCGProfile prof = {};
- const TCGProfile *s;
- int64_t tb_count;
- int64_t tb_div_count;
- int64_t tot;
-
- tcg_profile_snapshot_counters(&prof);
- s = &prof;
- tb_count = s->tb_count;
- tb_div_count = tb_count ? tb_count : 1;
- tot = s->interm_time + s->code_time;
-
- g_string_append_printf(buf, "JIT cycles %" PRId64
- " (%0.3f s at 2.4 GHz)\n",
- tot, tot / 2.4e9);
- g_string_append_printf(buf, "translated TBs %" PRId64
- " (aborted=%" PRId64 " %0.1f%%)\n",
- tb_count, s->tb_count1 - tb_count,
- (double)(s->tb_count1 - s->tb_count)
- / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
- g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
- (double)s->op_count / tb_div_count, s->op_count_max);
- g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
- (double)s->del_op_count / tb_div_count);
- g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
- (double)s->temp_count / tb_div_count,
- s->temp_count_max);
- g_string_append_printf(buf, "avg host code/TB %0.1f\n",
- (double)s->code_out_len / tb_div_count);
- g_string_append_printf(buf, "avg search data/TB %0.1f\n",
- (double)s->search_out_len / tb_div_count);
-
- g_string_append_printf(buf, "cycles/op %0.1f\n",
- s->op_count ? (double)tot / s->op_count : 0);
- g_string_append_printf(buf, "cycles/in byte %0.1f\n",
- s->code_in_len ? (double)tot / s->code_in_len : 0);
- g_string_append_printf(buf, "cycles/out byte %0.1f\n",
- s->code_out_len ? (double)tot / s->code_out_len : 0);
- g_string_append_printf(buf, "cycles/search byte %0.1f\n",
- s->search_out_len ?
- (double)tot / s->search_out_len : 0);
- if (tot == 0) {
- tot = 1;
- }
- g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
- (double)s->interm_time / tot * 100.0);
- g_string_append_printf(buf, " gen_code time %0.1f%%\n",
- (double)s->code_time / tot * 100.0);
- g_string_append_printf(buf, "optim./code time %0.1f%%\n",
- (double)s->opt_time / (s->code_time ?
- s->code_time : 1)
- * 100.0);
- g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
- (double)s->la_time / (s->code_time ?
- s->code_time : 1) * 100.0);
- g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
- s->restore_count);
- g_string_append_printf(buf, " avg cycles %0.1f\n",
- s->restore_count ?
- (double)s->restore_time / s->restore_count : 0);
-}
-#else
-void tcg_dump_info(GString *buf)
-{
- g_string_append_printf(buf, "[TCG profiler not compiled]\n");
-}
-#endif
-
#ifdef ELF_HOST_MACHINE
/* In order to use this feature, the backend needs to do three things:
projects / mirror_qemu.git / blobdiff