#define TCG_REG_GUEST_BASE TCG_REG_X28
#endif
-static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
- tcg_debug_assert(offset == sextract64(offset, 0, 26));
- /* read instruction, mask away previous PC_REL26 parameter contents,
- set the proper offset, then write back the instruction. */
- *code_ptr = deposit32(*code_ptr, 0, 26, offset);
-}
-
-static inline void reloc_pc26_atomic(tcg_insn_unit *code_ptr,
- tcg_insn_unit *target)
-{
- ptrdiff_t offset = target - code_ptr;
- tcg_insn_unit insn;
- tcg_debug_assert(offset == sextract64(offset, 0, 26));
- /* read instruction, mask away previous PC_REL26 parameter contents,
- set the proper offset, then write back the instruction. */
- insn = atomic_read(code_ptr);
- atomic_set(code_ptr, deposit32(insn, 0, 26, offset));
+ if (offset == sextract64(offset, 0, 26)) {
+ /* read instruction, mask away previous PC_REL26 parameter contents,
+ set the proper offset, then write back the instruction. */
+ *code_ptr = deposit32(*code_ptr, 0, 26, offset);
+ return true;
+ }
+ return false;
}
-static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
- tcg_debug_assert(offset == sextract64(offset, 0, 19));
- *code_ptr = deposit32(*code_ptr, 5, 19, offset);
+ if (offset == sextract64(offset, 0, 19)) {
+ *code_ptr = deposit32(*code_ptr, 5, 19, offset);
+ return true;
+ }
+ return false;
}
-static inline void patch_reloc(tcg_insn_unit *code_ptr, int type,
+static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
tcg_debug_assert(addend == 0);
switch (type) {
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
- reloc_pc26(code_ptr, (tcg_insn_unit *)value);
- break;
+ return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
case R_AARCH64_CONDBR19:
- reloc_pc19(code_ptr, (tcg_insn_unit *)value);
- break;
+ return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
default:
- tcg_abort();
+ g_assert_not_reached();
}
}
#define TCG_CT_CONST_LIMM 0x200
#define TCG_CT_CONST_ZERO 0x400
#define TCG_CT_CONST_MONE 0x800
+#define TCG_CT_CONST_ORRI 0x1000
+#define TCG_CT_CONST_ANDI 0x2000
/* parse target specific constraints */
static const char *target_parse_constraint(TCGArgConstraint *ct,
case 'M': /* minus one */
ct->ct |= TCG_CT_CONST_MONE;
break;
+ case 'O': /* vector orr/bic immediate */
+ ct->ct |= TCG_CT_CONST_ORRI;
+ break;
+ case 'N': /* vector orr/bic immediate, inverted */
+ ct->ct |= TCG_CT_CONST_ANDI;
+ break;
case 'Z': /* zero */
ct->ct |= TCG_CT_CONST_ZERO;
break;
return (val & (val - 1)) == 0;
}
-/* Match a constant that is valid for vectors. */
-static bool is_fimm(uint64_t v64, int *op, int *cmode, int *imm8)
+/* Return true if v16 is a valid 16-bit shifted immediate. */
+static bool is_shimm16(uint16_t v16, int *cmode, int *imm8)
{
- int i;
+ if (v16 == (v16 & 0xff)) {
+ *cmode = 0x8;
+ *imm8 = v16 & 0xff;
+ return true;
+ } else if (v16 == (v16 & 0xff00)) {
+ *cmode = 0xa;
+ *imm8 = v16 >> 8;
+ return true;
+ }
+ return false;
+}
- *op = 0;
- /* Match replication across 8 bits. */
- if (v64 == dup_const(MO_8, v64)) {
- *cmode = 0xe;
- *imm8 = v64 & 0xff;
+/* Return true if v32 is a valid 32-bit shifted immediate. */
+static bool is_shimm32(uint32_t v32, int *cmode, int *imm8)
+{
+ if (v32 == (v32 & 0xff)) {
+ *cmode = 0x0;
+ *imm8 = v32 & 0xff;
+ return true;
+ } else if (v32 == (v32 & 0xff00)) {
+ *cmode = 0x2;
+ *imm8 = (v32 >> 8) & 0xff;
+ return true;
+ } else if (v32 == (v32 & 0xff0000)) {
+ *cmode = 0x4;
+ *imm8 = (v32 >> 16) & 0xff;
+ return true;
+ } else if (v32 == (v32 & 0xff000000)) {
+ *cmode = 0x6;
+ *imm8 = v32 >> 24;
return true;
}
- /* Match replication across 16 bits. */
- if (v64 == dup_const(MO_16, v64)) {
- uint16_t v16 = v64;
+ return false;
+}
- if (v16 == (v16 & 0xff)) {
- *cmode = 0x8;
- *imm8 = v16 & 0xff;
- return true;
- } else if (v16 == (v16 & 0xff00)) {
- *cmode = 0xa;
- *imm8 = v16 >> 8;
- return true;
- }
+/* Return true if v32 is a valid 32-bit shifting ones immediate. */
+static bool is_soimm32(uint32_t v32, int *cmode, int *imm8)
+{
+ if ((v32 & 0xffff00ff) == 0xff) {
+ *cmode = 0xc;
+ *imm8 = (v32 >> 8) & 0xff;
+ return true;
+ } else if ((v32 & 0xff00ffff) == 0xffff) {
+ *cmode = 0xd;
+ *imm8 = (v32 >> 16) & 0xff;
+ return true;
}
- /* Match replication across 32 bits. */
- if (v64 == dup_const(MO_32, v64)) {
- uint32_t v32 = v64;
+ return false;
+}
- if (v32 == (v32 & 0xff)) {
- *cmode = 0x0;
- *imm8 = v32 & 0xff;
- return true;
- } else if (v32 == (v32 & 0xff00)) {
- *cmode = 0x2;
- *imm8 = (v32 >> 8) & 0xff;
- return true;
- } else if (v32 == (v32 & 0xff0000)) {
- *cmode = 0x4;
- *imm8 = (v32 >> 16) & 0xff;
- return true;
- } else if (v32 == (v32 & 0xff000000)) {
- *cmode = 0x6;
- *imm8 = v32 >> 24;
- return true;
- } else if ((v32 & 0xffff00ff) == 0xff) {
- *cmode = 0xc;
- *imm8 = (v32 >> 8) & 0xff;
- return true;
- } else if ((v32 & 0xff00ffff) == 0xffff) {
- *cmode = 0xd;
- *imm8 = (v32 >> 16) & 0xff;
- return true;
- }
- /* Match forms of a float32. */
- if (extract32(v32, 0, 19) == 0
- && (extract32(v32, 25, 6) == 0x20
- || extract32(v32, 25, 6) == 0x1f)) {
- *cmode = 0xf;
- *imm8 = (extract32(v32, 31, 1) << 7)
- | (extract32(v32, 25, 1) << 6)
- | extract32(v32, 19, 6);
- return true;
- }
+/* Return true if v32 is a valid float32 immediate. */
+static bool is_fimm32(uint32_t v32, int *cmode, int *imm8)
+{
+ if (extract32(v32, 0, 19) == 0
+ && (extract32(v32, 25, 6) == 0x20
+ || extract32(v32, 25, 6) == 0x1f)) {
+ *cmode = 0xf;
+ *imm8 = (extract32(v32, 31, 1) << 7)
+ | (extract32(v32, 25, 1) << 6)
+ | extract32(v32, 19, 6);
+ return true;
}
- /* Match forms of a float64. */
+ return false;
+}
+
+/* Return true if v64 is a valid float64 immediate. */
+static bool is_fimm64(uint64_t v64, int *cmode, int *imm8)
+{
if (extract64(v64, 0, 48) == 0
&& (extract64(v64, 54, 9) == 0x100
|| extract64(v64, 54, 9) == 0x0ff)) {
*cmode = 0xf;
- *op = 1;
*imm8 = (extract64(v64, 63, 1) << 7)
| (extract64(v64, 54, 1) << 6)
| extract64(v64, 48, 6);
return true;
}
- /* Match bytes of 0x00 and 0xff. */
- for (i = 0; i < 64; i += 8) {
- uint64_t byte = extract64(v64, i, 8);
- if (byte != 0 && byte != 0xff) {
+ return false;
+}
+
+/*
+ * Return non-zero if v32 can be formed by MOVI+ORR.
+ * Place the parameters for MOVI in (cmode, imm8).
+ * Return the cmode for ORR; the imm8 can be had via extraction from v32.
+ */
+static int is_shimm32_pair(uint32_t v32, int *cmode, int *imm8)
+{
+ int i;
+
+ for (i = 6; i > 0; i -= 2) {
+ /* Mask out one byte we can add with ORR. */
+ uint32_t tmp = v32 & ~(0xffu << (i * 4));
+ if (is_shimm32(tmp, cmode, imm8) ||
+ is_soimm32(tmp, cmode, imm8)) {
break;
}
}
- if (i == 64) {
- *cmode = 0xe;
- *op = 1;
- *imm8 = (extract64(v64, 0, 1) << 0)
- | (extract64(v64, 8, 1) << 1)
- | (extract64(v64, 16, 1) << 2)
- | (extract64(v64, 24, 1) << 3)
- | (extract64(v64, 32, 1) << 4)
- | (extract64(v64, 40, 1) << 5)
- | (extract64(v64, 48, 1) << 6)
- | (extract64(v64, 56, 1) << 7);
- return true;
+ return i;
+}
+
+/* Return true if V is a valid 16-bit or 32-bit shifted immediate. */
+static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8)
+{
+ if (v32 == deposit32(v32, 16, 16, v32)) {
+ return is_shimm16(v32, cmode, imm8);
+ } else {
+ return is_shimm32(v32, cmode, imm8);
}
- return false;
}
static int tcg_target_const_match(tcg_target_long val, TCGType type,
return 1;
}
+ switch (ct & (TCG_CT_CONST_ORRI | TCG_CT_CONST_ANDI)) {
+ case 0:
+ break;
+ case TCG_CT_CONST_ANDI:
+ val = ~val;
+ /* fallthru */
+ case TCG_CT_CONST_ORRI:
+ if (val == deposit64(val, 32, 32, val)) {
+ int cmode, imm8;
+ return is_shimm1632(val, &cmode, &imm8);
+ }
+ break;
+ default:
+ /* Both bits should not be set for the same insn. */
+ g_assert_not_reached();
+ }
+
return 0;
}
I3207_BLR = 0xd63f0000,
I3207_RET = 0xd65f0000,
+ /* AdvSIMD load/store single structure. */
+ I3303_LD1R = 0x0d40c000,
+
/* Load literal for loading the address at pc-relative offset */
I3305_LDR = 0x58000000,
I3305_LDR_v64 = 0x5c000000,
I3510_EON = 0x4a200000,
I3510_ANDS = 0x6a000000,
+ /* Logical shifted register instructions (with a shift). */
+ I3502S_AND_LSR = I3510_AND | (1 << 22),
+
/* AdvSIMD copy */
I3605_DUP = 0x0e000400,
I3605_INS = 0x4e001c00,
/* AdvSIMD modified immediate */
I3606_MOVI = 0x0f000400,
+ I3606_MVNI = 0x2f000400,
+ I3606_BIC = 0x2f001400,
+ I3606_ORR = 0x0f001400,
/* AdvSIMD shift by immediate */
I3614_SSHR = 0x0f000400,
I3616_ADD = 0x0e208400,
I3616_AND = 0x0e201c00,
I3616_BIC = 0x0e601c00,
+ I3616_BIF = 0x2ee01c00,
+ I3616_BIT = 0x2ea01c00,
+ I3616_BSL = 0x2e601c00,
I3616_EOR = 0x2e201c00,
I3616_MUL = 0x0e209c00,
I3616_ORR = 0x0ea01c00,
I3616_CMHI = 0x2e203400,
I3616_CMHS = 0x2e203c00,
I3616_CMEQ = 0x2e208c00,
+ I3616_SMAX = 0x0e206400,
+ I3616_SMIN = 0x0e206c00,
+ I3616_SSHL = 0x0e204400,
+ I3616_SQADD = 0x0e200c00,
+ I3616_SQSUB = 0x0e202c00,
+ I3616_UMAX = 0x2e206400,
+ I3616_UMIN = 0x2e206c00,
+ I3616_UQADD = 0x2e200c00,
+ I3616_UQSUB = 0x2e202c00,
+ I3616_USHL = 0x2e204400,
/* AdvSIMD two-reg misc. */
I3617_CMGT0 = 0x0e208800,
I3617_CMGE0 = 0x2e208800,
I3617_CMLE0 = 0x2e20a800,
I3617_NOT = 0x2e205800,
+ I3617_ABS = 0x0e20b800,
I3617_NEG = 0x2e20b800,
/* System instructions. */
#define tcg_out_insn(S, FMT, OP, ...) \
glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
-static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn, int imm19, TCGReg rt)
+static void tcg_out_insn_3303(TCGContext *s, AArch64Insn insn, bool q,
+ TCGReg rt, TCGReg rn, unsigned size)
+{
+ tcg_out32(s, insn | (rt & 0x1f) | (rn << 5) | (size << 10) | (q << 30));
+}
+
+static void tcg_out_insn_3305(TCGContext *s, AArch64Insn insn,
+ int imm19, TCGReg rt)
{
tcg_out32(s, insn | (imm19 & 0x7ffff) << 5 | rt);
}
}
static void tcg_out_dupi_vec(TCGContext *s, TCGType type,
- TCGReg rd, uint64_t v64)
+ TCGReg rd, tcg_target_long v64)
{
- int op, cmode, imm8;
+ bool q = type == TCG_TYPE_V128;
+ int cmode, imm8, i;
- if (is_fimm(v64, &op, &cmode, &imm8)) {
- tcg_out_insn(s, 3606, MOVI, type == TCG_TYPE_V128, rd, op, cmode, imm8);
- } else if (type == TCG_TYPE_V128) {
+ /* Test all bytes equal first. */
+ if (v64 == dup_const(MO_8, v64)) {
+ imm8 = (uint8_t)v64;
+ tcg_out_insn(s, 3606, MOVI, q, rd, 0, 0xe, imm8);
+ return;
+ }
+
+ /*
+ * Test all bytes 0x00 or 0xff second. This can match cases that
+ * might otherwise take 2 or 3 insns for MO_16 or MO_32 below.
+ */
+ for (i = imm8 = 0; i < 8; i++) {
+ uint8_t byte = v64 >> (i * 8);
+ if (byte == 0xff) {
+ imm8 |= 1 << i;
+ } else if (byte != 0) {
+ goto fail_bytes;
+ }
+ }
+ tcg_out_insn(s, 3606, MOVI, q, rd, 1, 0xe, imm8);
+ return;
+ fail_bytes:
+
+ /*
+ * Tests for various replications. For each element width, if we
+ * cannot find an expansion there's no point checking a larger
+ * width because we already know by replication it cannot match.
+ */
+ if (v64 == dup_const(MO_16, v64)) {
+ uint16_t v16 = v64;
+
+ if (is_shimm16(v16, &cmode, &imm8)) {
+ tcg_out_insn(s, 3606, MOVI, q, rd, 0, cmode, imm8);
+ return;
+ }
+ if (is_shimm16(~v16, &cmode, &imm8)) {
+ tcg_out_insn(s, 3606, MVNI, q, rd, 0, cmode, imm8);
+ return;
+ }
+
+ /*
+ * Otherwise, all remaining constants can be loaded in two insns:
+ * rd = v16 & 0xff, rd |= v16 & 0xff00.
+ */
+ tcg_out_insn(s, 3606, MOVI, q, rd, 0, 0x8, v16 & 0xff);
+ tcg_out_insn(s, 3606, ORR, q, rd, 0, 0xa, v16 >> 8);
+ return;
+ } else if (v64 == dup_const(MO_32, v64)) {
+ uint32_t v32 = v64;
+ uint32_t n32 = ~v32;
+
+ if (is_shimm32(v32, &cmode, &imm8) ||
+ is_soimm32(v32, &cmode, &imm8) ||
+ is_fimm32(v32, &cmode, &imm8)) {
+ tcg_out_insn(s, 3606, MOVI, q, rd, 0, cmode, imm8);
+ return;
+ }
+ if (is_shimm32(n32, &cmode, &imm8) ||
+ is_soimm32(n32, &cmode, &imm8)) {
+ tcg_out_insn(s, 3606, MVNI, q, rd, 0, cmode, imm8);
+ return;
+ }
+
+ /*
+ * Restrict the set of constants to those we can load with
+ * two instructions. Others we load from the pool.
+ */
+ i = is_shimm32_pair(v32, &cmode, &imm8);
+ if (i) {
+ tcg_out_insn(s, 3606, MOVI, q, rd, 0, cmode, imm8);
+ tcg_out_insn(s, 3606, ORR, q, rd, 0, i, extract32(v32, i * 4, 8));
+ return;
+ }
+ i = is_shimm32_pair(n32, &cmode, &imm8);
+ if (i) {
+ tcg_out_insn(s, 3606, MVNI, q, rd, 0, cmode, imm8);
+ tcg_out_insn(s, 3606, BIC, q, rd, 0, i, extract32(n32, i * 4, 8));
+ return;
+ }
+ } else if (is_fimm64(v64, &cmode, &imm8)) {
+ tcg_out_insn(s, 3606, MOVI, q, rd, 1, cmode, imm8);
+ return;
+ }
+
+ /*
+ * As a last resort, load from the constant pool. Sadly there
+ * is no LD1R (literal), so store the full 16-byte vector.
+ */
+ if (type == TCG_TYPE_V128) {
new_pool_l2(s, R_AARCH64_CONDBR19, s->code_ptr, 0, v64, v64);
tcg_out_insn(s, 3305, LDR_v128, 0, rd);
} else {
}
}
+static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
+ TCGReg rd, TCGReg rs)
+{
+ int is_q = type - TCG_TYPE_V64;
+ tcg_out_insn(s, 3605, DUP, is_q, rd, rs, 1 << vece, 0);
+ return true;
+}
+
+static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
+ TCGReg r, TCGReg base, intptr_t offset)
+{
+ TCGReg temp = TCG_REG_TMP;
+
+ if (offset < -0xffffff || offset > 0xffffff) {
+ tcg_out_movi(s, TCG_TYPE_PTR, temp, offset);
+ tcg_out_insn(s, 3502, ADD, 1, temp, temp, base);
+ base = temp;
+ } else {
+ AArch64Insn add_insn = I3401_ADDI;
+
+ if (offset < 0) {
+ add_insn = I3401_SUBI;
+ offset = -offset;
+ }
+ if (offset & 0xfff000) {
+ tcg_out_insn_3401(s, add_insn, 1, temp, base, offset & 0xfff000);
+ base = temp;
+ }
+ if (offset & 0xfff) {
+ tcg_out_insn_3401(s, add_insn, 1, temp, base, offset & 0xfff);
+ base = temp;
+ }
+ }
+ tcg_out_insn(s, 3303, LD1R, type == TCG_TYPE_V128, r, base, vece);
+ return true;
+}
+
static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
tcg_target_long value)
{
tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP);
}
-static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
+static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
{
if (ret == arg) {
- return;
+ return true;
}
switch (type) {
case TCG_TYPE_I32:
default:
g_assert_not_reached();
}
+ return true;
}
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
}
}
-static inline void tcg_out_goto_noaddr(TCGContext *s)
-{
- /* We pay attention here to not modify the branch target by reading from
- the buffer. This ensure that caches and memory are kept coherent during
- retranslation. Mask away possible garbage in the high bits for the
- first translation, while keeping the offset bits for retranslation. */
- uint32_t old = tcg_in32(s);
- tcg_out_insn(s, 3206, B, old);
-}
-
-static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
-{
- /* See comments in tcg_out_goto_noaddr. */
- uint32_t old = tcg_in32(s) >> 5;
- tcg_out_insn(s, 3202, B_C, c, old);
-}
-
static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
{
tcg_out_insn(s, 3207, BLR, reg);
{
if (!l->has_value) {
tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0);
- tcg_out_goto_noaddr(s);
+ tcg_out_insn(s, 3206, B, 0);
} else {
tcg_out_goto(s, l->u.value_ptr);
}
tcg_out_insn(s, 3406, ADR, rd, offset);
}
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
TCGMemOpIdx oi = lb->oi;
TCGMemOp opc = get_memop(oi);
TCGMemOp size = opc & MO_SIZE;
- reloc_pc19(lb->label_ptr[0], s->code_ptr);
+ if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ return false;
+ }
tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
}
tcg_out_goto(s, lb->raddr);
+ return true;
}
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
{
TCGMemOpIdx oi = lb->oi;
TCGMemOp opc = get_memop(oi);
TCGMemOp size = opc & MO_SIZE;
- reloc_pc19(lb->label_ptr[0], s->code_ptr);
+ if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ return false;
+ }
tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
tcg_out_adr(s, TCG_REG_X4, lb->raddr);
tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
tcg_out_goto(s, lb->raddr);
+ return true;
}
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
label->label_ptr[0] = label_ptr;
}
+/* We expect to use a 7-bit scaled negative offset from ENV. */
+QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
+QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
+
/* Load and compare a TLB entry, emitting the conditional jump to the
slow path for the failure case, which will be patched later when finalizing
the slow path. Generated code returns the host addend in X1,
tcg_insn_unit **label_ptr, int mem_index,
bool is_read)
{
- int tlb_offset = is_read ?
- offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
- : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+ int fast_ofs = TLB_MASK_TABLE_OFS(mem_index);
+ int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask);
+ int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table);
unsigned a_bits = get_alignment_bits(opc);
unsigned s_bits = opc & MO_SIZE;
unsigned a_mask = (1u << a_bits) - 1;
unsigned s_mask = (1u << s_bits) - 1;
- TCGReg base = TCG_AREG0, x3;
- uint64_t tlb_mask;
+ TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0, x3;
+ TCGType mask_type;
+ uint64_t compare_mask;
+
+ mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
+ ? TCG_TYPE_I64 : TCG_TYPE_I32);
+
+ /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
+ tcg_out_ld(s, mask_type, TCG_REG_X0, mask_base, mask_ofs);
+ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, table_base, table_ofs);
+
+ /* Extract the TLB index from the address into X0. */
+ tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64,
+ TCG_REG_X0, TCG_REG_X0, addr_reg,
+ TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+
+ /* Add the tlb_table pointer, creating the CPUTLBEntry address into X1. */
+ tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);
+
+ /* Load the tlb comparator into X0, and the fast path addend into X1. */
+ tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, is_read
+ ? offsetof(CPUTLBEntry, addr_read)
+ : offsetof(CPUTLBEntry, addr_write));
+ tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
+ offsetof(CPUTLBEntry, addend));
/* For aligned accesses, we check the first byte and include the alignment
bits within the address. For unaligned access, we check that we don't
TCG_REG_X3, addr_reg, s_mask - a_mask);
x3 = TCG_REG_X3;
}
- tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
-
- /* Extract the TLB index from the address into X0.
- X0<CPU_TLB_BITS:0> =
- addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
- tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
- TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
+ compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
/* Store the page mask part of the address into X3. */
tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
- TCG_REG_X3, x3, tlb_mask);
-
- /* Add any "high bits" from the tlb offset to the env address into X2,
- to take advantage of the LSL12 form of the ADDI instruction.
- X2 = env + (tlb_offset & 0xfff000) */
- if (tlb_offset & 0xfff000) {
- tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X2, base,
- tlb_offset & 0xfff000);
- base = TCG_REG_X2;
- }
-
- /* Merge the tlb index contribution into X2.
- X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
- tcg_out_insn(s, 3502S, ADD_LSL, TCG_TYPE_I64, TCG_REG_X2, base,
- TCG_REG_X0, CPU_TLB_ENTRY_BITS);
-
- /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
- X0 = load [X2 + (tlb_offset & 0x000fff)] */
- tcg_out_ldst(s, TARGET_LONG_BITS == 32 ? I3312_LDRW : I3312_LDRX,
- TCG_REG_X0, TCG_REG_X2, tlb_offset & 0xfff,
- TARGET_LONG_BITS == 32 ? 2 : 3);
-
- /* Load the tlb addend. Do that early to avoid stalling.
- X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
- tcg_out_ldst(s, I3312_LDRX, TCG_REG_X1, TCG_REG_X2,
- (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
- (is_read ? offsetof(CPUTLBEntry, addr_read)
- : offsetof(CPUTLBEntry, addr_write)), 3);
+ TCG_REG_X3, x3, compare_mask);
/* Perform the address comparison. */
- tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
+ tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);
/* If not equal, we jump to the slow path. */
*label_ptr = s->code_ptr;
- tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
+ tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
}
#endif /* CONFIG_SOFTMMU */
tcg_out_insn(s, 3305, LDR, offset, TCG_REG_TMP);
}
tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
- s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
+ set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
tcg_out_sbfm(s, ext, a0, a1, a2, a2 + args[3] - 1);
break;
+ case INDEX_op_extract2_i64:
+ case INDEX_op_extract2_i32:
+ tcg_out_extr(s, ext, a0, a1, a2, args[3]);
+ break;
+
case INDEX_op_add2_i32:
tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
(int32_t)args[4], args[5], const_args[4],
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
- case INDEX_op_mov_vec:
case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */
case INDEX_op_movi_i64:
- case INDEX_op_dupi_vec:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
default:
g_assert_not_reached();
TCGType type = vecl + TCG_TYPE_V64;
unsigned is_q = vecl;
- TCGArg a0, a1, a2;
+ TCGArg a0, a1, a2, a3;
+ int cmode, imm8;
a0 = args[0];
a1 = args[1];
case INDEX_op_st_vec:
tcg_out_st(s, type, a0, a1, a2);
break;
+ case INDEX_op_dupm_vec:
+ tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
+ break;
case INDEX_op_add_vec:
tcg_out_insn(s, 3616, ADD, is_q, vece, a0, a1, a2);
break;
case INDEX_op_neg_vec:
tcg_out_insn(s, 3617, NEG, is_q, vece, a0, a1);
break;
+ case INDEX_op_abs_vec:
+ tcg_out_insn(s, 3617, ABS, is_q, vece, a0, a1);
+ break;
case INDEX_op_and_vec:
+ if (const_args[2]) {
+ is_shimm1632(~a2, &cmode, &imm8);
+ if (a0 == a1) {
+ tcg_out_insn(s, 3606, BIC, is_q, a0, 0, cmode, imm8);
+ return;
+ }
+ tcg_out_insn(s, 3606, MVNI, is_q, a0, 0, cmode, imm8);
+ a2 = a0;
+ }
tcg_out_insn(s, 3616, AND, is_q, 0, a0, a1, a2);
break;
case INDEX_op_or_vec:
+ if (const_args[2]) {
+ is_shimm1632(a2, &cmode, &imm8);
+ if (a0 == a1) {
+ tcg_out_insn(s, 3606, ORR, is_q, a0, 0, cmode, imm8);
+ return;
+ }
+ tcg_out_insn(s, 3606, MOVI, is_q, a0, 0, cmode, imm8);
+ a2 = a0;
+ }
tcg_out_insn(s, 3616, ORR, is_q, 0, a0, a1, a2);
break;
- case INDEX_op_xor_vec:
- tcg_out_insn(s, 3616, EOR, is_q, 0, a0, a1, a2);
- break;
case INDEX_op_andc_vec:
+ if (const_args[2]) {
+ is_shimm1632(a2, &cmode, &imm8);
+ if (a0 == a1) {
+ tcg_out_insn(s, 3606, BIC, is_q, a0, 0, cmode, imm8);
+ return;
+ }
+ tcg_out_insn(s, 3606, MOVI, is_q, a0, 0, cmode, imm8);
+ a2 = a0;
+ }
tcg_out_insn(s, 3616, BIC, is_q, 0, a0, a1, a2);
break;
case INDEX_op_orc_vec:
+ if (const_args[2]) {
+ is_shimm1632(~a2, &cmode, &imm8);
+ if (a0 == a1) {
+ tcg_out_insn(s, 3606, ORR, is_q, a0, 0, cmode, imm8);
+ return;
+ }
+ tcg_out_insn(s, 3606, MVNI, is_q, a0, 0, cmode, imm8);
+ a2 = a0;
+ }
tcg_out_insn(s, 3616, ORN, is_q, 0, a0, a1, a2);
break;
+ case INDEX_op_xor_vec:
+ tcg_out_insn(s, 3616, EOR, is_q, 0, a0, a1, a2);
+ break;
+ case INDEX_op_ssadd_vec:
+ tcg_out_insn(s, 3616, SQADD, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_sssub_vec:
+ tcg_out_insn(s, 3616, SQSUB, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_usadd_vec:
+ tcg_out_insn(s, 3616, UQADD, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_ussub_vec:
+ tcg_out_insn(s, 3616, UQSUB, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_smax_vec:
+ tcg_out_insn(s, 3616, SMAX, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_smin_vec:
+ tcg_out_insn(s, 3616, SMIN, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_umax_vec:
+ tcg_out_insn(s, 3616, UMAX, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_umin_vec:
+ tcg_out_insn(s, 3616, UMIN, is_q, vece, a0, a1, a2);
+ break;
case INDEX_op_not_vec:
tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a1);
break;
- case INDEX_op_dup_vec:
- tcg_out_insn(s, 3605, DUP, is_q, a0, a1, 1 << vece, 0);
- break;
case INDEX_op_shli_vec:
tcg_out_insn(s, 3614, SHL, is_q, a0, a1, a2 + (8 << vece));
break;
case INDEX_op_sari_vec:
tcg_out_insn(s, 3614, SSHR, is_q, a0, a1, (16 << vece) - a2);
break;
+ case INDEX_op_shlv_vec:
+ tcg_out_insn(s, 3616, USHL, is_q, vece, a0, a1, a2);
+ break;
+ case INDEX_op_aa64_sshl_vec:
+ tcg_out_insn(s, 3616, SSHL, is_q, vece, a0, a1, a2);
+ break;
case INDEX_op_cmp_vec:
{
TCGCond cond = args[3];
}
}
break;
+
+ case INDEX_op_bitsel_vec:
+ a3 = args[3];
+ if (a0 == a3) {
+ tcg_out_insn(s, 3616, BIT, is_q, 0, a0, a2, a1);
+ } else if (a0 == a2) {
+ tcg_out_insn(s, 3616, BIF, is_q, 0, a0, a3, a1);
+ } else {
+ if (a0 != a1) {
+ tcg_out_mov(s, type, a0, a1);
+ }
+ tcg_out_insn(s, 3616, BSL, is_q, 0, a0, a2, a3);
+ }
+ break;
+
+ case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
+ case INDEX_op_dupi_vec: /* Always emitted via tcg_out_movi. */
+ case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
default:
g_assert_not_reached();
}
switch (opc) {
case INDEX_op_add_vec:
case INDEX_op_sub_vec:
- case INDEX_op_mul_vec:
case INDEX_op_and_vec:
case INDEX_op_or_vec:
case INDEX_op_xor_vec:
case INDEX_op_andc_vec:
case INDEX_op_orc_vec:
case INDEX_op_neg_vec:
+ case INDEX_op_abs_vec:
case INDEX_op_not_vec:
case INDEX_op_cmp_vec:
case INDEX_op_shli_vec:
case INDEX_op_shri_vec:
case INDEX_op_sari_vec:
+ case INDEX_op_ssadd_vec:
+ case INDEX_op_sssub_vec:
+ case INDEX_op_usadd_vec:
+ case INDEX_op_ussub_vec:
+ case INDEX_op_shlv_vec:
+ case INDEX_op_bitsel_vec:
return 1;
+ case INDEX_op_shrv_vec:
+ case INDEX_op_sarv_vec:
+ return -1;
+ case INDEX_op_mul_vec:
+ case INDEX_op_smax_vec:
+ case INDEX_op_smin_vec:
+ case INDEX_op_umax_vec:
+ case INDEX_op_umin_vec:
+ return vece < MO_64;
default:
return 0;
void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
TCGArg a0, ...)
{
+ va_list va;
+ TCGv_vec v0, v1, v2, t1;
+
+ va_start(va, a0);
+ v0 = temp_tcgv_vec(arg_temp(a0));
+ v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
+ v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
+
+ switch (opc) {
+ case INDEX_op_shrv_vec:
+ case INDEX_op_sarv_vec:
+ /* Right shifts are negative left shifts for AArch64. */
+ t1 = tcg_temp_new_vec(type);
+ tcg_gen_neg_vec(vece, t1, v2);
+ opc = (opc == INDEX_op_shrv_vec
+ ? INDEX_op_shlv_vec : INDEX_op_aa64_sshl_vec);
+ vec_gen_3(opc, type, vece, tcgv_vec_arg(v0),
+ tcgv_vec_arg(v1), tcgv_vec_arg(t1));
+ tcg_temp_free_vec(t1);
+ break;
+
+ default:
+ g_assert_not_reached();
+ }
+
+ va_end(va);
}
static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
static const TCGTargetOpDef lZ_l = { .args_ct_str = { "lZ", "l" } };
static const TCGTargetOpDef r_r_r = { .args_ct_str = { "r", "r", "r" } };
static const TCGTargetOpDef w_w_w = { .args_ct_str = { "w", "w", "w" } };
+ static const TCGTargetOpDef w_w_wO = { .args_ct_str = { "w", "w", "wO" } };
+ static const TCGTargetOpDef w_w_wN = { .args_ct_str = { "w", "w", "wN" } };
static const TCGTargetOpDef w_w_wZ = { .args_ct_str = { "w", "w", "wZ" } };
static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } };
static const TCGTargetOpDef r_r_rA = { .args_ct_str = { "r", "r", "rA" } };
= { .args_ct_str = { "r", "r", "rAL" } };
static const TCGTargetOpDef dep
= { .args_ct_str = { "r", "0", "rZ" } };
+ static const TCGTargetOpDef ext2
+ = { .args_ct_str = { "r", "rZ", "rZ" } };
static const TCGTargetOpDef movc
= { .args_ct_str = { "r", "r", "rA", "rZ", "rZ" } };
static const TCGTargetOpDef add2
= { .args_ct_str = { "r", "r", "rZ", "rZ", "rA", "rMZ" } };
+ static const TCGTargetOpDef w_w_w_w
+ = { .args_ct_str = { "w", "w", "w", "w" } };
switch (op) {
case INDEX_op_goto_ptr:
case INDEX_op_deposit_i64:
return &dep;
+ case INDEX_op_extract2_i32:
+ case INDEX_op_extract2_i64:
+ return &ext2;
+
case INDEX_op_add2_i32:
case INDEX_op_add2_i64:
case INDEX_op_sub2_i32:
case INDEX_op_add_vec:
case INDEX_op_sub_vec:
case INDEX_op_mul_vec:
- case INDEX_op_and_vec:
- case INDEX_op_or_vec:
case INDEX_op_xor_vec:
- case INDEX_op_andc_vec:
- case INDEX_op_orc_vec:
+ case INDEX_op_ssadd_vec:
+ case INDEX_op_sssub_vec:
+ case INDEX_op_usadd_vec:
+ case INDEX_op_ussub_vec:
+ case INDEX_op_smax_vec:
+ case INDEX_op_smin_vec:
+ case INDEX_op_umax_vec:
+ case INDEX_op_umin_vec:
+ case INDEX_op_shlv_vec:
+ case INDEX_op_shrv_vec:
+ case INDEX_op_sarv_vec:
+ case INDEX_op_aa64_sshl_vec:
return &w_w_w;
case INDEX_op_not_vec:
case INDEX_op_neg_vec:
+ case INDEX_op_abs_vec:
case INDEX_op_shli_vec:
case INDEX_op_shri_vec:
case INDEX_op_sari_vec:
return &w_w;
case INDEX_op_ld_vec:
case INDEX_op_st_vec:
+ case INDEX_op_dupm_vec:
return &w_r;
case INDEX_op_dup_vec:
return &w_wr;
+ case INDEX_op_or_vec:
+ case INDEX_op_andc_vec:
+ return &w_w_wO;
+ case INDEX_op_and_vec:
+ case INDEX_op_orc_vec:
+ return &w_w_wN;
case INDEX_op_cmp_vec:
return &w_w_wZ;
+ case INDEX_op_bitsel_vec:
+ return &w_w_w_w;
default:
return NULL;
projects / mirror_qemu.git / blobdiff