#include "cpu.h"
#include "exec/memop.h"
-#include "exec/tb-context.h"
+#include "exec/memopidx.h"
#include "qemu/bitops.h"
#include "qemu/plugin.h"
#include "qemu/queue.h"
#include "tcg/tcg-mo.h"
#include "tcg-target.h"
-#include "qemu/int128.h"
+#include "tcg/tcg-cond.h"
/* XXX: make safe guess about sizes */
#define MAX_OP_PER_INSTR 266
#else
#define MAX_OPC_PARAM_PER_ARG 1
#endif
-#define MAX_OPC_PARAM_IARGS 6
+#define MAX_OPC_PARAM_IARGS 7
#define MAX_OPC_PARAM_OARGS 1
#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
#define CPU_TEMP_BUF_NLONGS 128
+#define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long))
/* Default target word size to pointer size. */
#ifndef TCG_TARGET_REG_BITS
#define TCG_TARGET_HAS_not_vec 0
#define TCG_TARGET_HAS_andc_vec 0
#define TCG_TARGET_HAS_orc_vec 0
+#define TCG_TARGET_HAS_nand_vec 0
+#define TCG_TARGET_HAS_nor_vec 0
+#define TCG_TARGET_HAS_eqv_vec 0
#define TCG_TARGET_HAS_roti_vec 0
#define TCG_TARGET_HAS_rots_vec 0
#define TCG_TARGET_HAS_rotv_vec 0
/* Used to align parameters. See the comment before tcgv_i32_temp. */
#define TCG_CALL_DUMMY_ARG ((TCGArg)0)
-/* Conditions. Note that these are laid out for easy manipulation by
- the functions below:
- bit 0 is used for inverting;
- bit 1 is signed,
- bit 2 is unsigned,
- bit 3 is used with bit 0 for swapping signed/unsigned. */
-typedef enum {
- /* non-signed */
- TCG_COND_NEVER = 0 | 0 | 0 | 0,
- TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
- TCG_COND_EQ = 8 | 0 | 0 | 0,
- TCG_COND_NE = 8 | 0 | 0 | 1,
- /* signed */
- TCG_COND_LT = 0 | 0 | 2 | 0,
- TCG_COND_GE = 0 | 0 | 2 | 1,
- TCG_COND_LE = 8 | 0 | 2 | 0,
- TCG_COND_GT = 8 | 0 | 2 | 1,
- /* unsigned */
- TCG_COND_LTU = 0 | 4 | 0 | 0,
- TCG_COND_GEU = 0 | 4 | 0 | 1,
- TCG_COND_LEU = 8 | 4 | 0 | 0,
- TCG_COND_GTU = 8 | 4 | 0 | 1,
-} TCGCond;
-
-/* Invert the sense of the comparison. */
-static inline TCGCond tcg_invert_cond(TCGCond c)
-{
- return (TCGCond)(c ^ 1);
-}
-
-/* Swap the operands in a comparison. */
-static inline TCGCond tcg_swap_cond(TCGCond c)
-{
- return c & 6 ? (TCGCond)(c ^ 9) : c;
-}
-
-/* Create an "unsigned" version of a "signed" comparison. */
-static inline TCGCond tcg_unsigned_cond(TCGCond c)
-{
- return c & 2 ? (TCGCond)(c ^ 6) : c;
-}
-
-/* Create a "signed" version of an "unsigned" comparison. */
-static inline TCGCond tcg_signed_cond(TCGCond c)
-{
- return c & 4 ? (TCGCond)(c ^ 6) : c;
-}
-
-/* Must a comparison be considered unsigned? */
-static inline bool is_unsigned_cond(TCGCond c)
-{
- return (c & 4) != 0;
-}
-
-/* Create a "high" version of a double-word comparison.
- This removes equality from a LTE or GTE comparison. */
-static inline TCGCond tcg_high_cond(TCGCond c)
-{
- switch (c) {
- case TCG_COND_GE:
- case TCG_COND_LE:
- case TCG_COND_GEU:
- case TCG_COND_LEU:
- return (TCGCond)(c ^ 8);
- default:
- return c;
- }
-}
+/*
+ * Flags for the bswap opcodes.
+ * If IZ, the input is zero-extended, otherwise unknown.
+ * If OZ or OS, the output is zero- or sign-extended respectively,
+ * otherwise the high bits are undefined.
+ */
+enum {
+ TCG_BSWAP_IZ = 1,
+ TCG_BSWAP_OZ = 2,
+ TCG_BSWAP_OS = 4,
+};
typedef enum TCGTempVal {
TEMP_VAL_DEAD,
TEMP_VAL_CONST,
} TCGTempVal;
+typedef enum TCGTempKind {
+ /* Temp is dead at the end of all basic blocks. */
+ TEMP_NORMAL,
+ /* Temp is saved across basic blocks but dead at the end of TBs. */
+ TEMP_LOCAL,
+ /* Temp is saved across both basic blocks and translation blocks. */
+ TEMP_GLOBAL,
+ /* Temp is in a fixed register. */
+ TEMP_FIXED,
+ /* Temp is a fixed constant. */
+ TEMP_CONST,
+} TCGTempKind;
+
typedef struct TCGTemp {
TCGReg reg:8;
TCGTempVal val_type:8;
TCGType base_type:8;
TCGType type:8;
- unsigned int fixed_reg:1;
+ TCGTempKind kind:3;
unsigned int indirect_reg:1;
unsigned int indirect_base:1;
unsigned int mem_coherent:1;
unsigned int mem_allocated:1;
- /* If true, the temp is saved across both basic blocks and
- translation blocks. */
- unsigned int temp_global:1;
- /* If true, the temp is saved across basic blocks but dead
- at the end of translation blocks. If false, the temp is
- dead at the end of basic blocks. */
- unsigned int temp_local:1;
unsigned int temp_allocated:1;
- tcg_target_long val;
+ int64_t val;
struct TCGTemp *mem_base;
intptr_t mem_offset;
const char *name;
/* Next and previous opcodes. */
QTAILQ_ENTRY(TCGOp) link;
-#ifdef CONFIG_PLUGIN
- QSIMPLEQ_ENTRY(TCGOp) plugin_link;
-#endif
/* Arguments for the opcode. */
TCGArg args[MAX_OPC_PARAM];
/* Threshold to flush the translated code buffer. */
void *code_gen_highwater;
- size_t tb_phys_invalidate_count;
-
/* Track which vCPU triggers events */
CPUState *cpu; /* *_trans */
/* descriptor of the instruction being translated */
struct qemu_plugin_insn *plugin_insn;
-
- /* list to quickly access the injected ops */
- QSIMPLEQ_HEAD(, TCGOp) plugin_ops;
#endif
+ GHashTable *const_table[TCG_TYPE_COUNT];
TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
uint16_t gen_insn_end_off[TCG_MAX_INSNS];
target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
+
+ /* Exit to translator on overflow. */
+ sigjmp_buf jmp_trans;
};
-extern TCGContext tcg_init_ctx;
+static inline bool temp_readonly(TCGTemp *ts)
+{
+ return ts->kind >= TEMP_FIXED;
+}
+
extern __thread TCGContext *tcg_ctx;
extern const void *tcg_code_gen_epilogue;
extern uintptr_t tcg_splitwx_diff;
extern TCGv_env cpu_env;
-static inline bool in_code_gen_buffer(const void *p)
-{
- const TCGContext *s = &tcg_init_ctx;
- /*
- * Much like it is valid to have a pointer to the byte past the
- * end of an array (so long as you don't dereference it), allow
- * a pointer to the byte past the end of the code gen buffer.
- */
- return (size_t)(p - s->code_gen_buffer) <= s->code_gen_buffer_size;
-}
+bool in_code_gen_buffer(const void *p);
#ifdef CONFIG_DEBUG_TCG
const void *tcg_splitwx_to_rx(void *rw);
void tcg_pool_reset(TCGContext *s);
TranslationBlock *tcg_tb_alloc(TCGContext *s);
-void tcg_region_init(void);
-void tb_destroy(TranslationBlock *tb);
void tcg_region_reset_all(void);
size_t tcg_code_size(void);
void tcg_tb_insert(TranslationBlock *tb);
void tcg_tb_remove(TranslationBlock *tb);
-size_t tcg_tb_phys_invalidate_count(void);
TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
size_t tcg_nb_tbs(void);
}
}
-void tcg_context_init(TCGContext *s);
+void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus);
void tcg_register_thread(void);
void tcg_prologue_init(TCGContext *s);
void tcg_func_start(TCGContext *s);
#endif
int64_t tcg_cpu_exec_time(void);
-void tcg_dump_info(void);
-void tcg_dump_op_count(void);
+void tcg_dump_info(GString *buf);
+void tcg_dump_op_count(GString *buf);
#define TCG_CT_CONST 1 /* any constant of register size */
TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc);
TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc);
+/**
+ * tcg_remove_ops_after:
+ * @op: target operation
+ *
+ * Discard any opcodes emitted since @op. Expected usage is to save
+ * a starting point with tcg_last_op(), speculatively emit opcodes,
+ * then decide whether or not to keep those opcodes after the fact.
+ */
+void tcg_remove_ops_after(TCGOp *op);
+
void tcg_optimize(TCGContext *s);
+/* Allocate a new temporary and initialize it with a constant. */
TCGv_i32 tcg_const_i32(int32_t val);
TCGv_i64 tcg_const_i64(int64_t val);
TCGv_i32 tcg_const_local_i32(int32_t val);
TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec);
TCGv_vec tcg_const_ones_vec_matching(TCGv_vec);
+/*
+ * Locate or create a read-only temporary that is a constant.
+ * This kind of temporary need not be freed, but for convenience
+ * will be silently ignored by tcg_temp_free_*.
+ */
+TCGTemp *tcg_constant_internal(TCGType type, int64_t val);
+
+static inline TCGv_i32 tcg_constant_i32(int32_t val)
+{
+ return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
+}
+
+static inline TCGv_i64 tcg_constant_i64(int64_t val)
+{
+ return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
+}
+
+TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val);
+TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val);
+
#if UINTPTR_MAX == UINT32_MAX
# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x)))
# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x)))
return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
}
-/* Combine the MemOp and mmu_idx parameters into a single value. */
-typedef uint32_t TCGMemOpIdx;
-
-/**
- * make_memop_idx
- * @op: memory operation
- * @idx: mmu index
- *
- * Encode these values into a single parameter.
- */
-static inline TCGMemOpIdx make_memop_idx(MemOp op, unsigned idx)
-{
- tcg_debug_assert(idx <= 15);
- return (op << 4) | idx;
-}
-
-/**
- * get_memop
- * @oi: combined op/idx parameter
- *
- * Extract the memory operation from the combined value.
- */
-static inline MemOp get_memop(TCGMemOpIdx oi)
-{
- return oi >> 4;
-}
-
-/**
- * get_mmuidx
- * @oi: combined op/idx parameter
- *
- * Extract the mmu index from the combined value.
- */
-static inline unsigned get_mmuidx(TCGMemOpIdx oi)
-{
- return oi & 15;
-}
-
/**
* tcg_qemu_tb_exec:
* @env: pointer to CPUArchState for the CPU
? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
: (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
: (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
- : dup_const(VECE, C)) \
+ : (VECE) == MO_64 ? (uint64_t)(C) \
+ : (qemu_build_not_reached_always(), 0)) \
: dup_const(VECE, C))
-
-/*
- * Memory helpers that will be used by TCG generated code.
- */
-#ifdef CONFIG_SOFTMMU
-/* Value zero-extended to tcg register size. */
-tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-/* Value sign-extended to tcg register size. */
-tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-/* Temporary aliases until backends are converted. */
-#ifdef TARGET_WORDS_BIGENDIAN
-# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
-# define helper_ret_lduw_mmu helper_be_lduw_mmu
-# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
-# define helper_ret_ldul_mmu helper_be_ldul_mmu
-# define helper_ret_ldl_mmu helper_be_ldul_mmu
-# define helper_ret_ldq_mmu helper_be_ldq_mmu
-# define helper_ret_stw_mmu helper_be_stw_mmu
-# define helper_ret_stl_mmu helper_be_stl_mmu
-# define helper_ret_stq_mmu helper_be_stq_mmu
-#else
-# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
-# define helper_ret_lduw_mmu helper_le_lduw_mmu
-# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
-# define helper_ret_ldul_mmu helper_le_ldul_mmu
-# define helper_ret_ldl_mmu helper_le_ldul_mmu
-# define helper_ret_ldq_mmu helper_le_ldq_mmu
-# define helper_ret_stw_mmu helper_le_stw_mmu
-# define helper_ret_stl_mmu helper_le_stl_mmu
-# define helper_ret_stq_mmu helper_le_stq_mmu
-#endif
-
-uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
- uint64_t cmpv, uint64_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
- uint64_t cmpv, uint64_t newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
-TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
- (CPUArchState *env, target_ulong addr, TYPE val, \
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-#ifdef CONFIG_ATOMIC64
-#define GEN_ATOMIC_HELPER_ALL(NAME) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
- GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
- GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
+#if TARGET_LONG_BITS == 64
+# define dup_const_tl dup_const
#else
-#define GEN_ATOMIC_HELPER_ALL(NAME) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
+# define dup_const_tl(VECE, C) \
+ (__builtin_constant_p(VECE) \
+ ? ( (VECE) == MO_8 ? 0x01010101ul * (uint8_t)(C) \
+ : (VECE) == MO_16 ? 0x00010001ul * (uint16_t)(C) \
+ : (VECE) == MO_32 ? 0x00000001ul * (uint32_t)(C) \
+ : (qemu_build_not_reached_always(), 0)) \
+ : (target_long)dup_const(VECE, C))
#endif
-GEN_ATOMIC_HELPER_ALL(fetch_add)
-GEN_ATOMIC_HELPER_ALL(fetch_sub)
-GEN_ATOMIC_HELPER_ALL(fetch_and)
-GEN_ATOMIC_HELPER_ALL(fetch_or)
-GEN_ATOMIC_HELPER_ALL(fetch_xor)
-GEN_ATOMIC_HELPER_ALL(fetch_smin)
-GEN_ATOMIC_HELPER_ALL(fetch_umin)
-GEN_ATOMIC_HELPER_ALL(fetch_smax)
-GEN_ATOMIC_HELPER_ALL(fetch_umax)
-
-GEN_ATOMIC_HELPER_ALL(add_fetch)
-GEN_ATOMIC_HELPER_ALL(sub_fetch)
-GEN_ATOMIC_HELPER_ALL(and_fetch)
-GEN_ATOMIC_HELPER_ALL(or_fetch)
-GEN_ATOMIC_HELPER_ALL(xor_fetch)
-GEN_ATOMIC_HELPER_ALL(smin_fetch)
-GEN_ATOMIC_HELPER_ALL(umin_fetch)
-GEN_ATOMIC_HELPER_ALL(smax_fetch)
-GEN_ATOMIC_HELPER_ALL(umax_fetch)
-
-GEN_ATOMIC_HELPER_ALL(xchg)
-
-#undef GEN_ATOMIC_HELPER_ALL
-#undef GEN_ATOMIC_HELPER
-#endif /* CONFIG_SOFTMMU */
-
-/*
- * These aren't really a "proper" helpers because TCG cannot manage Int128.
- * However, use the same format as the others, for use by the backends.
- *
- * The cmpxchg functions are only defined if HAVE_CMPXCHG128;
- * the ld/st functions are only defined if HAVE_ATOMIC128,
- * as defined by <qemu/atomic128.h>.
- */
-Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
- Int128 cmpv, Int128 newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
- Int128 cmpv, Int128 newv,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
-Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- TCGMemOpIdx oi, uintptr_t retaddr);
-
#ifdef CONFIG_DEBUG_TCG
void tcg_assert_listed_vecop(TCGOpcode);
#else
projects / mirror_qemu.git / blobdiff