]> git.proxmox.com Git - mirror_qemu.git/blame - tcg/i386/tcg-target.c.inc
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
hmp: add basic "info stats" implementation
[mirror_qemu.git] / tcg / i386 / tcg-target.c.inc
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
d4a9eb1f 24
b1ee3c67 25#include "../tcg-ldst.c.inc"
139c1837 26#include "../tcg-pool.c.inc"
4e45f239 27
8d8fdbae 28#ifdef CONFIG_DEBUG_TCG
d4a9eb1f 29static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
5d8a4f8f
RH		 30#if TCG_TARGET_REG_BITS == 64
31 "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
5d8a4f8f
RH		 32#else
33 "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
770c2fc7
RH		 34#endif
35 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
36 "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7",
37#if TCG_TARGET_REG_BITS == 64
38 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
39 "%xmm12", "%xmm13", "%xmm14", "%xmm15",
5d8a4f8f 40#endif
c896fe29 41};
d4a9eb1f 42#endif
c896fe29 43
d4a9eb1f 44static const int tcg_target_reg_alloc_order[] = {
5d8a4f8f
RH		 45#if TCG_TARGET_REG_BITS == 64
46 TCG_REG_RBP,
47 TCG_REG_RBX,
48 TCG_REG_R12,
49 TCG_REG_R13,
50 TCG_REG_R14,
51 TCG_REG_R15,
52 TCG_REG_R10,
53 TCG_REG_R11,
54 TCG_REG_R9,
55 TCG_REG_R8,
56 TCG_REG_RCX,
57 TCG_REG_RDX,
58 TCG_REG_RSI,
59 TCG_REG_RDI,
60 TCG_REG_RAX,
61#else
c896fe29
FB		 62 TCG_REG_EBX,
63 TCG_REG_ESI,
64 TCG_REG_EDI,
65 TCG_REG_EBP,
6648e296
RH		 66 TCG_REG_ECX,
67 TCG_REG_EDX,
68 TCG_REG_EAX,
770c2fc7
RH		 69#endif
70 TCG_REG_XMM0,
71 TCG_REG_XMM1,
72 TCG_REG_XMM2,
73 TCG_REG_XMM3,
74 TCG_REG_XMM4,
75 TCG_REG_XMM5,
76#ifndef _WIN64
77 /* The Win64 ABI has xmm6-xmm15 as caller-saves, and we do not save
78 any of them. Therefore only allow xmm0-xmm5 to be allocated. */
79 TCG_REG_XMM6,
80 TCG_REG_XMM7,
81#if TCG_TARGET_REG_BITS == 64
82 TCG_REG_XMM8,
83 TCG_REG_XMM9,
84 TCG_REG_XMM10,
85 TCG_REG_XMM11,
86 TCG_REG_XMM12,
87 TCG_REG_XMM13,
88 TCG_REG_XMM14,
89 TCG_REG_XMM15,
90#endif
5d8a4f8f 91#endif
c896fe29
FB		 92};
93
5d8a4f8f
RH		 94static const int tcg_target_call_iarg_regs[] = {
95#if TCG_TARGET_REG_BITS == 64
8d918718
SW		 96#if defined(_WIN64)
97 TCG_REG_RCX,
98 TCG_REG_RDX,
99#else
5d8a4f8f
RH		 100 TCG_REG_RDI,
101 TCG_REG_RSI,
102 TCG_REG_RDX,
103 TCG_REG_RCX,
8d918718 104#endif
5d8a4f8f
RH		 105 TCG_REG_R8,
106 TCG_REG_R9,
107#else
d73685e3 108 /* 32 bit mode uses stack based calling convention (GCC default). */
5d8a4f8f
RH		 109#endif
110};
111
68af23af 112static const int tcg_target_call_oarg_regs[] = {
5d8a4f8f 113 TCG_REG_EAX,
68af23af 114#if TCG_TARGET_REG_BITS == 32
5d8a4f8f 115 TCG_REG_EDX
68af23af 116#endif
5d8a4f8f 117};
c896fe29 118
a1b29c9a
RH		 119/* Constants we accept. */
120#define TCG_CT_CONST_S32 0x100
121#define TCG_CT_CONST_U32 0x200
9d2eec20 122#define TCG_CT_CONST_I32 0x400
bbf25f90 123#define TCG_CT_CONST_WSZ 0x800
a1b29c9a 124
770c2fc7 125/* Registers used with L constraint, which are the first argument
b18212c6
SW		 126 registers on x86_64, and two random call clobbered registers on
127 i386. */
128#if TCG_TARGET_REG_BITS == 64
129# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
130# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
b18212c6
SW		 131#else
132# define TCG_REG_L0 TCG_REG_EAX
133# define TCG_REG_L1 TCG_REG_EDX
134#endif
135
df903b94
RH		 136#define ALL_BYTEH_REGS 0x0000000fu
137#if TCG_TARGET_REG_BITS == 64
138# define ALL_GENERAL_REGS 0x0000ffffu
139# define ALL_VECTOR_REGS 0xffff0000u
140# define ALL_BYTEL_REGS ALL_GENERAL_REGS
141#else
142# define ALL_GENERAL_REGS 0x000000ffu
143# define ALL_VECTOR_REGS 0x00ff0000u
144# define ALL_BYTEL_REGS ALL_BYTEH_REGS
145#endif
146#ifdef CONFIG_SOFTMMU
147# define SOFTMMU_RESERVE_REGS ((1 << TCG_REG_L0) | (1 << TCG_REG_L1))
148#else
149# define SOFTMMU_RESERVE_REGS 0
150#endif
151
085bb5bb
AJ		 152/* The host compiler should supply <cpuid.h> to enable runtime features
153 detection, as we're not going to go so far as our own inline assembly.
154 If not available, default values will be assumed. */
155#if defined(CONFIG_CPUID_H)
5dd89908 156#include "qemu/cpuid.h"
085bb5bb
AJ		 157#endif
158
5dd89908 159/* For 64-bit, we always know that CMOV is available. */
76a347e1
RH		 160#if TCG_TARGET_REG_BITS == 64
161# define have_cmov 1
5dd89908 162#elif defined(CONFIG_CPUID_H)
76a347e1
RH		 163static bool have_cmov;
164#else
165# define have_cmov 0
166#endif
167
993508e4 168/* We need these symbols in tcg-target.h, and we can't properly conditionalize
9d2eec20
RH		 169 it there. Therefore we always define the variable. */
170bool have_bmi1;
993508e4 171bool have_popcnt;
770c2fc7
RH		 172bool have_avx1;
173bool have_avx2;
ba597b66
RH		 174bool have_avx512bw;
175bool have_avx512dq;
176bool have_avx512vbmi2;
177bool have_avx512vl;
d2ef1b83 178bool have_movbe;
9d2eec20 179
5dd89908 180#ifdef CONFIG_CPUID_H
6399ab33 181static bool have_bmi2;
bbf25f90
RH		 182static bool have_lzcnt;
183#else
5dd89908 184# define have_bmi2 0
bbf25f90
RH		 185# define have_lzcnt 0
186#endif
6399ab33 187
705ed477 188static const tcg_insn_unit *tb_ret_addr;
b03cce8e 189
6ac17786 190static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 191 intptr_t value, intptr_t addend)
c896fe29 192{
f54b3f92 193 value += addend;
c896fe29 194 switch(type) {
c896fe29 195 case R_386_PC32:
705ed477 196 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
5d8a4f8f 197 if (value != (int32_t)value) {
bec3afd5 198 return false;
5d8a4f8f 199 }
770c2fc7
RH		 200 /* FALLTHRU */
201 case R_386_32:
5c53bb81 202 tcg_patch32(code_ptr, value);
c896fe29 203 break;
f75b56c1 204 case R_386_PC8:
705ed477 205 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
f75b56c1 206 if (value != (int8_t)value) {
bec3afd5 207 return false;
f75b56c1 208 }
5c53bb81 209 tcg_patch8(code_ptr, value);
f75b56c1 210 break;
c896fe29
FB		 211 default:
212 tcg_abort();
213 }
6ac17786 214 return true;
c896fe29
FB		 215}
216
c896fe29 217/* test if a constant matches the constraint */
a4fbbd77 218static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
c896fe29 219{
5d8a4f8f 220 if (ct & TCG_CT_CONST) {
c896fe29 221 return 1;
5d8a4f8f 222 }
c7c778b5
RH		 223 if (type == TCG_TYPE_I32) {
224 if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) {
225 return 1;
226 }
227 } else {
228 if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
229 return 1;
230 }
231 if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
232 return 1;
233 }
234 if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) {
235 return 1;
236 }
9d2eec20 237 }
bbf25f90
RH		 238 if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
239 return 1;
240 }
5d8a4f8f 241 return 0;
c896fe29
FB		 242}
243
5d8a4f8f 244# define LOWREGMASK(x) ((x) & 7)
5d8a4f8f 245
96b4cf38 246#define P_EXT 0x100 /* 0x0f opcode prefix */
2a113775
AJ		 247#define P_EXT38 0x200 /* 0x0f 0x38 opcode prefix */
248#define P_DATA16 0x400 /* 0x66 opcode prefix */
fc88a523 249#define P_VEXW 0x1000 /* Set VEX.W = 1 */
5d8a4f8f 250#if TCG_TARGET_REG_BITS == 64
fc88a523 251# define P_REXW P_VEXW /* Set REX.W = 1; match VEXW */
2a113775
AJ		 252# define P_REXB_R 0x2000 /* REG field as byte register */
253# define P_REXB_RM 0x4000 /* R/M field as byte register */
254# define P_GS 0x8000 /* gs segment override */
5d8a4f8f 255#else
5d8a4f8f
RH		 256# define P_REXW 0
257# define P_REXB_R 0
258# define P_REXB_RM 0
44b37ace 259# define P_GS 0
5d8a4f8f 260#endif
770c2fc7
RH		 261#define P_EXT3A 0x10000 /* 0x0f 0x3a opcode prefix */
262#define P_SIMDF3 0x20000 /* 0xf3 opcode prefix */
263#define P_SIMDF2 0x40000 /* 0xf2 opcode prefix */
264#define P_VEXL 0x80000 /* Set VEX.L = 1 */
08b032f7 265#define P_EVEX 0x100000 /* Requires EVEX encoding */
fcb5dac1 266
a369a702
RH		 267#define OPC_ARITH_EvIz (0x81)
268#define OPC_ARITH_EvIb (0x83)
81570a70 269#define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */
9d2eec20 270#define OPC_ANDN (0xf2 | P_EXT38)
81570a70 271#define OPC_ADD_GvEv (OPC_ARITH_GvEv | (ARITH_ADD << 3))
54eaf40b 272#define OPC_AND_GvEv (OPC_ARITH_GvEv | (ARITH_AND << 3))
770c2fc7 273#define OPC_BLENDPS (0x0c | P_EXT3A | P_DATA16)
bbf25f90
RH		 274#define OPC_BSF (0xbc | P_EXT)
275#define OPC_BSR (0xbd | P_EXT)
fcb5dac1 276#define OPC_BSWAP (0xc8 | P_EXT)
aadb21a4 277#define OPC_CALL_Jz (0xe8)
d0a16297 278#define OPC_CMOVCC (0x40 | P_EXT) /* ... plus condition code */
81570a70
RH		 279#define OPC_CMP_GvEv (OPC_ARITH_GvEv | (ARITH_CMP << 3))
280#define OPC_DEC_r32 (0x48)
0566d387
RH		 281#define OPC_IMUL_GvEv (0xaf | P_EXT)
282#define OPC_IMUL_GvEvIb (0x6b)
283#define OPC_IMUL_GvEvIz (0x69)
81570a70 284#define OPC_INC_r32 (0x40)
da441cff
RH		 285#define OPC_JCC_long (0x80 | P_EXT) /* ... plus condition code */
286#define OPC_JCC_short (0x70) /* ... plus condition code */
287#define OPC_JMP_long (0xe9)
288#define OPC_JMP_short (0xeb)
34a6d0b7 289#define OPC_LEA (0x8d)
bbf25f90 290#define OPC_LZCNT (0xbd | P_EXT | P_SIMDF3)
af266089
RH		 291#define OPC_MOVB_EvGv (0x88) /* stores, more or less */
292#define OPC_MOVL_EvGv (0x89) /* stores, more or less */
293#define OPC_MOVL_GvEv (0x8b) /* loads, more or less */
5c2d2a9e 294#define OPC_MOVB_EvIz (0xc6)
5d8a4f8f 295#define OPC_MOVL_EvIz (0xc7)
ef10b106 296#define OPC_MOVL_Iv (0xb8)
085bb5bb
AJ		 297#define OPC_MOVBE_GyMy (0xf0 | P_EXT38)
298#define OPC_MOVBE_MyGy (0xf1 | P_EXT38)
770c2fc7
RH		 299#define OPC_MOVD_VyEy (0x6e | P_EXT | P_DATA16)
300#define OPC_MOVD_EyVy (0x7e | P_EXT | P_DATA16)
7b60ef32 301#define OPC_MOVDDUP (0x12 | P_EXT | P_SIMDF2)
770c2fc7
RH		 302#define OPC_MOVDQA_VxWx (0x6f | P_EXT | P_DATA16)
303#define OPC_MOVDQA_WxVx (0x7f | P_EXT | P_DATA16)
304#define OPC_MOVDQU_VxWx (0x6f | P_EXT | P_SIMDF3)
305#define OPC_MOVDQU_WxVx (0x7f | P_EXT | P_SIMDF3)
306#define OPC_MOVQ_VqWq (0x7e | P_EXT | P_SIMDF3)
307#define OPC_MOVQ_WqVq (0xd6 | P_EXT | P_DATA16)
6817c355
RH		 308#define OPC_MOVSBL (0xbe | P_EXT)
309#define OPC_MOVSWL (0xbf | P_EXT)
5d8a4f8f 310#define OPC_MOVSLQ (0x63 | P_REXW)
55e082a7
RH		 311#define OPC_MOVZBL (0xb6 | P_EXT)
312#define OPC_MOVZWL (0xb7 | P_EXT)
18f9b65f
RH		 313#define OPC_PABSB (0x1c | P_EXT38 | P_DATA16)
314#define OPC_PABSW (0x1d | P_EXT38 | P_DATA16)
315#define OPC_PABSD (0x1e | P_EXT38 | P_DATA16)
dac1648f 316#define OPC_VPABSQ (0x1f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
770c2fc7
RH		 317#define OPC_PACKSSDW (0x6b | P_EXT | P_DATA16)
318#define OPC_PACKSSWB (0x63 | P_EXT | P_DATA16)
319#define OPC_PACKUSDW (0x2b | P_EXT38 | P_DATA16)
320#define OPC_PACKUSWB (0x67 | P_EXT | P_DATA16)
321#define OPC_PADDB (0xfc | P_EXT | P_DATA16)
322#define OPC_PADDW (0xfd | P_EXT | P_DATA16)
323#define OPC_PADDD (0xfe | P_EXT | P_DATA16)
324#define OPC_PADDQ (0xd4 | P_EXT | P_DATA16)
8ffafbce
RH		 325#define OPC_PADDSB (0xec | P_EXT | P_DATA16)
326#define OPC_PADDSW (0xed | P_EXT | P_DATA16)
327#define OPC_PADDUB (0xdc | P_EXT | P_DATA16)
328#define OPC_PADDUW (0xdd | P_EXT | P_DATA16)
770c2fc7
RH		 329#define OPC_PAND (0xdb | P_EXT | P_DATA16)
330#define OPC_PANDN (0xdf | P_EXT | P_DATA16)
331#define OPC_PBLENDW (0x0e | P_EXT3A | P_DATA16)
332#define OPC_PCMPEQB (0x74 | P_EXT | P_DATA16)
333#define OPC_PCMPEQW (0x75 | P_EXT | P_DATA16)
334#define OPC_PCMPEQD (0x76 | P_EXT | P_DATA16)
335#define OPC_PCMPEQQ (0x29 | P_EXT38 | P_DATA16)
336#define OPC_PCMPGTB (0x64 | P_EXT | P_DATA16)
337#define OPC_PCMPGTW (0x65 | P_EXT | P_DATA16)
338#define OPC_PCMPGTD (0x66 | P_EXT | P_DATA16)
339#define OPC_PCMPGTQ (0x37 | P_EXT38 | P_DATA16)
bc37faf4
RH		 340#define OPC_PMAXSB (0x3c | P_EXT38 | P_DATA16)
341#define OPC_PMAXSW (0xee | P_EXT | P_DATA16)
342#define OPC_PMAXSD (0x3d | P_EXT38 | P_DATA16)
dac1648f 343#define OPC_VPMAXSQ (0x3d | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
bc37faf4
RH		 344#define OPC_PMAXUB (0xde | P_EXT | P_DATA16)
345#define OPC_PMAXUW (0x3e | P_EXT38 | P_DATA16)
346#define OPC_PMAXUD (0x3f | P_EXT38 | P_DATA16)
dac1648f 347#define OPC_VPMAXUQ (0x3f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
bc37faf4
RH		 348#define OPC_PMINSB (0x38 | P_EXT38 | P_DATA16)
349#define OPC_PMINSW (0xea | P_EXT | P_DATA16)
350#define OPC_PMINSD (0x39 | P_EXT38 | P_DATA16)
dac1648f 351#define OPC_VPMINSQ (0x39 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
bc37faf4
RH		 352#define OPC_PMINUB (0xda | P_EXT | P_DATA16)
353#define OPC_PMINUW (0x3a | P_EXT38 | P_DATA16)
354#define OPC_PMINUD (0x3b | P_EXT38 | P_DATA16)
dac1648f 355#define OPC_VPMINUQ (0x3b | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
770c2fc7
RH		 356#define OPC_PMOVSXBW (0x20 | P_EXT38 | P_DATA16)
357#define OPC_PMOVSXWD (0x23 | P_EXT38 | P_DATA16)
358#define OPC_PMOVSXDQ (0x25 | P_EXT38 | P_DATA16)
359#define OPC_PMOVZXBW (0x30 | P_EXT38 | P_DATA16)
360#define OPC_PMOVZXWD (0x33 | P_EXT38 | P_DATA16)
361#define OPC_PMOVZXDQ (0x35 | P_EXT38 | P_DATA16)
362#define OPC_PMULLW (0xd5 | P_EXT | P_DATA16)
363#define OPC_PMULLD (0x40 | P_EXT38 | P_DATA16)
4c8b9686 364#define OPC_VPMULLQ (0x40 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
770c2fc7
RH		 365#define OPC_POR (0xeb | P_EXT | P_DATA16)
366#define OPC_PSHUFB (0x00 | P_EXT38 | P_DATA16)
367#define OPC_PSHUFD (0x70 | P_EXT | P_DATA16)
368#define OPC_PSHUFLW (0x70 | P_EXT | P_SIMDF2)
369#define OPC_PSHUFHW (0x70 | P_EXT | P_SIMDF3)
370#define OPC_PSHIFTW_Ib (0x71 | P_EXT | P_DATA16) /* /2 /6 /4 */
4e73f842 371#define OPC_PSHIFTD_Ib (0x72 | P_EXT | P_DATA16) /* /1 /2 /6 /4 */
770c2fc7 372#define OPC_PSHIFTQ_Ib (0x73 | P_EXT | P_DATA16) /* /2 /6 /4 */
0a8d7a3b
RH		 373#define OPC_PSLLW (0xf1 | P_EXT | P_DATA16)
374#define OPC_PSLLD (0xf2 | P_EXT | P_DATA16)
375#define OPC_PSLLQ (0xf3 | P_EXT | P_DATA16)
376#define OPC_PSRAW (0xe1 | P_EXT | P_DATA16)
377#define OPC_PSRAD (0xe2 | P_EXT | P_DATA16)
3cc18d18 378#define OPC_VPSRAQ (0xe2 | P_EXT | P_DATA16 | P_VEXW | P_EVEX)
0a8d7a3b
RH		 379#define OPC_PSRLW (0xd1 | P_EXT | P_DATA16)
380#define OPC_PSRLD (0xd2 | P_EXT | P_DATA16)
381#define OPC_PSRLQ (0xd3 | P_EXT | P_DATA16)
770c2fc7
RH		 382#define OPC_PSUBB (0xf8 | P_EXT | P_DATA16)
383#define OPC_PSUBW (0xf9 | P_EXT | P_DATA16)
384#define OPC_PSUBD (0xfa | P_EXT | P_DATA16)
385#define OPC_PSUBQ (0xfb | P_EXT | P_DATA16)
8ffafbce
RH		 386#define OPC_PSUBSB (0xe8 | P_EXT | P_DATA16)
387#define OPC_PSUBSW (0xe9 | P_EXT | P_DATA16)
388#define OPC_PSUBUB (0xd8 | P_EXT | P_DATA16)
389#define OPC_PSUBUW (0xd9 | P_EXT | P_DATA16)
770c2fc7
RH		 390#define OPC_PUNPCKLBW (0x60 | P_EXT | P_DATA16)
391#define OPC_PUNPCKLWD (0x61 | P_EXT | P_DATA16)
392#define OPC_PUNPCKLDQ (0x62 | P_EXT | P_DATA16)
393#define OPC_PUNPCKLQDQ (0x6c | P_EXT | P_DATA16)
394#define OPC_PUNPCKHBW (0x68 | P_EXT | P_DATA16)
395#define OPC_PUNPCKHWD (0x69 | P_EXT | P_DATA16)
396#define OPC_PUNPCKHDQ (0x6a | P_EXT | P_DATA16)
397#define OPC_PUNPCKHQDQ (0x6d | P_EXT | P_DATA16)
398#define OPC_PXOR (0xef | P_EXT | P_DATA16)
6858614e 399#define OPC_POP_r32 (0x58)
993508e4 400#define OPC_POPCNT (0xb8 | P_EXT | P_SIMDF3)
6858614e
RH		 401#define OPC_PUSH_r32 (0x50)
402#define OPC_PUSH_Iv (0x68)
403#define OPC_PUSH_Ib (0x6a)
3c3accc6 404#define OPC_RET (0xc3)
5d8a4f8f 405#define OPC_SETCC (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
f53dba01
RH		 406#define OPC_SHIFT_1 (0xd1)
407#define OPC_SHIFT_Ib (0xc1)
408#define OPC_SHIFT_cl (0xd3)
6399ab33 409#define OPC_SARX (0xf7 | P_EXT38 | P_SIMDF3)
770c2fc7 410#define OPC_SHUFPS (0xc6 | P_EXT)
6399ab33
RH		 411#define OPC_SHLX (0xf7 | P_EXT38 | P_DATA16)
412#define OPC_SHRX (0xf7 | P_EXT38 | P_SIMDF2)
c6fb8c0c 413#define OPC_SHRD_Ib (0xac | P_EXT)
81570a70 414#define OPC_TESTL (0x85)
bbf25f90 415#define OPC_TZCNT (0xbc | P_EXT | P_SIMDF3)
770c2fc7
RH		 416#define OPC_UD2 (0x0b | P_EXT)
417#define OPC_VPBLENDD (0x02 | P_EXT3A | P_DATA16)
418#define OPC_VPBLENDVB (0x4c | P_EXT3A | P_DATA16)
1e262b49
RH		 419#define OPC_VPINSRB (0x20 | P_EXT3A | P_DATA16)
420#define OPC_VPINSRW (0xc4 | P_EXT | P_DATA16)
421#define OPC_VBROADCASTSS (0x18 | P_EXT38 | P_DATA16)
422#define OPC_VBROADCASTSD (0x19 | P_EXT38 | P_DATA16)
770c2fc7
RH		 423#define OPC_VPBROADCASTB (0x78 | P_EXT38 | P_DATA16)
424#define OPC_VPBROADCASTW (0x79 | P_EXT38 | P_DATA16)
425#define OPC_VPBROADCASTD (0x58 | P_EXT38 | P_DATA16)
426#define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16)
fc88a523 427#define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW)
770c2fc7 428#define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL)
102cd35c
RH		 429#define OPC_VPROLVD (0x15 | P_EXT38 | P_DATA16 | P_EVEX)
430#define OPC_VPROLVQ (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
431#define OPC_VPRORVD (0x14 | P_EXT38 | P_DATA16 | P_EVEX)
432#define OPC_VPRORVQ (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
965d5d06
RH		 433#define OPC_VPSHLDW (0x70 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
434#define OPC_VPSHLDD (0x71 | P_EXT3A | P_DATA16 | P_EVEX)
435#define OPC_VPSHLDQ (0x71 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
436#define OPC_VPSHLDVW (0x70 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
437#define OPC_VPSHLDVD (0x71 | P_EXT38 | P_DATA16 | P_EVEX)
438#define OPC_VPSHLDVQ (0x71 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
439#define OPC_VPSHRDVW (0x72 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
440#define OPC_VPSHRDVD (0x73 | P_EXT38 | P_DATA16 | P_EVEX)
441#define OPC_VPSHRDVQ (0x73 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
ef77ce0d 442#define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
a2ce146a 443#define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16)
fc88a523 444#define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW)
ef77ce0d 445#define OPC_VPSRAVW (0x11 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
a2ce146a 446#define OPC_VPSRAVD (0x46 | P_EXT38 | P_DATA16)
ef77ce0d
RH		 447#define OPC_VPSRAVQ (0x46 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
448#define OPC_VPSRLVW (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
a2ce146a 449#define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16)
fc88a523 450#define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW)
3143767b 451#define OPC_VPTERNLOGQ (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
770c2fc7 452#define OPC_VZEROUPPER (0x77 | P_EXT)
b3e66df7 453#define OPC_XCHG_ax_r32 (0x90)
fcb5dac1 454
b1ee3c67
RH		 455#define OPC_GRP3_Eb (0xf6)
456#define OPC_GRP3_Ev (0xf7)
457#define OPC_GRP5 (0xff)
770c2fc7 458#define OPC_GRP14 (0x73 | P_EXT | P_DATA16)
9363dedb
RH		 459
460/* Group 1 opcode extensions for 0x80-0x83.
461 These are also used as modifiers for OPC_ARITH. */
c896fe29
FB		 462#define ARITH_ADD 0
463#define ARITH_OR 1
464#define ARITH_ADC 2
465#define ARITH_SBB 3
466#define ARITH_AND 4
467#define ARITH_SUB 5
468#define ARITH_XOR 6
469#define ARITH_CMP 7
470
da441cff 471/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3. */
9619376c
AJ		 472#define SHIFT_ROL 0
473#define SHIFT_ROR 1
c896fe29
FB		 474#define SHIFT_SHL 4
475#define SHIFT_SHR 5
476#define SHIFT_SAR 7
477
9363dedb 478/* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */
b1ee3c67 479#define EXT3_TESTi 0
9363dedb
RH		 480#define EXT3_NOT 2
481#define EXT3_NEG 3
482#define EXT3_MUL 4
483#define EXT3_IMUL 5
484#define EXT3_DIV 6
485#define EXT3_IDIV 7
486
487/* Group 5 opcode extensions for 0xff. To be used with OPC_GRP5. */
5d8a4f8f
RH		 488#define EXT5_INC_Ev 0
489#define EXT5_DEC_Ev 1
9363dedb
RH		 490#define EXT5_CALLN_Ev 2
491#define EXT5_JMPN_Ev 4
da441cff
RH		 492
493/* Condition codes to be added to OPC_JCC_{long,short}. */
c896fe29
FB		 494#define JCC_JMP (-1)
495#define JCC_JO 0x0
496#define JCC_JNO 0x1
497#define JCC_JB 0x2
498#define JCC_JAE 0x3
499#define JCC_JE 0x4
500#define JCC_JNE 0x5
501#define JCC_JBE 0x6
502#define JCC_JA 0x7
503#define JCC_JS 0x8
504#define JCC_JNS 0x9
505#define JCC_JP 0xa
506#define JCC_JNP 0xb
507#define JCC_JL 0xc
508#define JCC_JGE 0xd
509#define JCC_JLE 0xe
510#define JCC_JG 0xf
511
0aed257f 512static const uint8_t tcg_cond_to_jcc[] = {
c896fe29
FB		 513 [TCG_COND_EQ] = JCC_JE,
514 [TCG_COND_NE] = JCC_JNE,
515 [TCG_COND_LT] = JCC_JL,
516 [TCG_COND_GE] = JCC_JGE,
517 [TCG_COND_LE] = JCC_JLE,
518 [TCG_COND_GT] = JCC_JG,
519 [TCG_COND_LTU] = JCC_JB,
520 [TCG_COND_GEU] = JCC_JAE,
521 [TCG_COND_LEU] = JCC_JBE,
522 [TCG_COND_GTU] = JCC_JA,
523};
524
5d8a4f8f
RH		 525#if TCG_TARGET_REG_BITS == 64
526static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
527{
528 int rex;
529
44b37ace
RH		 530 if (opc & P_GS) {
531 tcg_out8(s, 0x65);
532 }
5d8a4f8f
RH		 533 if (opc & P_DATA16) {
534 /* We should never be asking for both 16 and 64-bit operation. */
eabb7b91 535 tcg_debug_assert((opc & P_REXW) == 0);
5d8a4f8f
RH		 536 tcg_out8(s, 0x66);
537 }
bbf25f90
RH		 538 if (opc & P_SIMDF3) {
539 tcg_out8(s, 0xf3);
540 } else if (opc & P_SIMDF2) {
541 tcg_out8(s, 0xf2);
542 }
5d8a4f8f
RH		 543
544 rex = 0;
c9d78213 545 rex |= (opc & P_REXW) ? 0x8 : 0x0; /* REX.W */
ecc7e843
RH		 546 rex |= (r & 8) >> 1; /* REX.R */
547 rex |= (x & 8) >> 2; /* REX.X */
548 rex |= (rm & 8) >> 3; /* REX.B */
5d8a4f8f
RH		 549
550 /* P_REXB_{R,RM} indicates that the given register is the low byte.
551 For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
552 as otherwise the encoding indicates %[abcd]h. Note that the values
553 that are ORed in merely indicate that the REX byte must be present;
554 those bits get discarded in output. */
555 rex |= opc & (r >= 4 ? P_REXB_R : 0);
556 rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
557
558 if (rex) {
559 tcg_out8(s, (uint8_t)(rex | 0x40));
560 }
561
770c2fc7 562 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) {
5d8a4f8f 563 tcg_out8(s, 0x0f);
2a113775
AJ		 564 if (opc & P_EXT38) {
565 tcg_out8(s, 0x38);
770c2fc7
RH		 566 } else if (opc & P_EXT3A) {
567 tcg_out8(s, 0x3a);
2a113775 568 }
5d8a4f8f 569 }
2a113775 570
5d8a4f8f
RH		 571 tcg_out8(s, opc);
572}
573#else
574static void tcg_out_opc(TCGContext *s, int opc)
c896fe29 575{
96b4cf38
RH		 576 if (opc & P_DATA16) {
577 tcg_out8(s, 0x66);
578 }
bbf25f90
RH		 579 if (opc & P_SIMDF3) {
580 tcg_out8(s, 0xf3);
581 } else if (opc & P_SIMDF2) {
582 tcg_out8(s, 0xf2);
583 }
770c2fc7 584 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) {
c896fe29 585 tcg_out8(s, 0x0f);
2a113775
AJ		 586 if (opc & P_EXT38) {
587 tcg_out8(s, 0x38);
770c2fc7
RH		 588 } else if (opc & P_EXT3A) {
589 tcg_out8(s, 0x3a);
2a113775 590 }
96b4cf38 591 }
c896fe29
FB		 592 tcg_out8(s, opc);
593}
5d8a4f8f
RH		 594/* Discard the register arguments to tcg_out_opc early, so as not to penalize
595 the 32-bit compilation paths. This method works with all versions of gcc,
596 whereas relying on optimization may not be able to exclude them. */
597#define tcg_out_opc(s, opc, r, rm, x) (tcg_out_opc)(s, opc)
598#endif
c896fe29 599
5d8a4f8f 600static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
c896fe29 601{
5d8a4f8f
RH		 602 tcg_out_opc(s, opc, r, rm, 0);
603 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
c896fe29
FB		 604}
605
770c2fc7
RH		 606static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v,
607 int rm, int index)
ecc7e843
RH		 608{
609 int tmp;
610
770c2fc7
RH		 611 /* Use the two byte form if possible, which cannot encode
612 VEX.W, VEX.B, VEX.X, or an m-mmmm field other than P_EXT. */
fc88a523 613 if ((opc & (P_EXT | P_EXT38 | P_EXT3A | P_VEXW)) == P_EXT
770c2fc7
RH		 614 && ((rm | index) & 8) == 0) {
615 /* Two byte VEX prefix. */
616 tcg_out8(s, 0xc5);
617
618 tmp = (r & 8 ? 0 : 0x80); /* VEX.R */
619 } else {
ecc7e843
RH		 620 /* Three byte VEX prefix. */
621 tcg_out8(s, 0xc4);
622
623 /* VEX.m-mmmm */
770c2fc7
RH		 624 if (opc & P_EXT3A) {
625 tmp = 3;
626 } else if (opc & P_EXT38) {
ecc7e843
RH		 627 tmp = 2;
628 } else if (opc & P_EXT) {
629 tmp = 1;
630 } else {
770c2fc7 631 g_assert_not_reached();
ecc7e843 632 }
770c2fc7
RH		 633 tmp |= (r & 8 ? 0 : 0x80); /* VEX.R */
634 tmp |= (index & 8 ? 0 : 0x40); /* VEX.X */
635 tmp |= (rm & 8 ? 0 : 0x20); /* VEX.B */
ecc7e843
RH		 636 tcg_out8(s, tmp);
637
fc88a523 638 tmp = (opc & P_VEXW ? 0x80 : 0); /* VEX.W */
ecc7e843 639 }
770c2fc7
RH		 640
641 tmp |= (opc & P_VEXL ? 0x04 : 0); /* VEX.L */
6399ab33
RH		 642 /* VEX.pp */
643 if (opc & P_DATA16) {
644 tmp |= 1; /* 0x66 */
645 } else if (opc & P_SIMDF3) {
646 tmp |= 2; /* 0xf3 */
647 } else if (opc & P_SIMDF2) {
648 tmp |= 3; /* 0xf2 */
649 }
ecc7e843
RH		 650 tmp |= (~v & 15) << 3; /* VEX.vvvv */
651 tcg_out8(s, tmp);
652 tcg_out8(s, opc);
770c2fc7
RH		 653}
654
08b032f7
RH		 655static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v,
656 int rm, int index)
657{
658 /* The entire 4-byte evex prefix; with R' and V' set. */
659 uint32_t p = 0x08041062;
660 int mm, pp;
661
662 tcg_debug_assert(have_avx512vl);
663
664 /* EVEX.mm */
665 if (opc & P_EXT3A) {
666 mm = 3;
667 } else if (opc & P_EXT38) {
668 mm = 2;
669 } else if (opc & P_EXT) {
670 mm = 1;
671 } else {
672 g_assert_not_reached();
673 }
674
675 /* EVEX.pp */
676 if (opc & P_DATA16) {
677 pp = 1; /* 0x66 */
678 } else if (opc & P_SIMDF3) {
679 pp = 2; /* 0xf3 */
680 } else if (opc & P_SIMDF2) {
681 pp = 3; /* 0xf2 */
682 } else {
683 pp = 0;
684 }
685
686 p = deposit32(p, 8, 2, mm);
687 p = deposit32(p, 13, 1, (rm & 8) == 0); /* EVEX.RXB.B */
688 p = deposit32(p, 14, 1, (index & 8) == 0); /* EVEX.RXB.X */
689 p = deposit32(p, 15, 1, (r & 8) == 0); /* EVEX.RXB.R */
690 p = deposit32(p, 16, 2, pp);
691 p = deposit32(p, 19, 4, ~v);
692 p = deposit32(p, 23, 1, (opc & P_VEXW) != 0);
693 p = deposit32(p, 29, 2, (opc & P_VEXL) != 0);
694
695 tcg_out32(s, p);
696 tcg_out8(s, opc);
697}
698
770c2fc7
RH		 699static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm)
700{
08b032f7
RH		 701 if (opc & P_EVEX) {
702 tcg_out_evex_opc(s, opc, r, v, rm, 0);
703 } else {
704 tcg_out_vex_opc(s, opc, r, v, rm, 0);
705 }
ecc7e843
RH		 706 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
707}
708
34a6d0b7 709/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
5d8a4f8f
RH		 710 We handle either RM and INDEX missing with a negative value. In 64-bit
711 mode for absolute addresses, ~RM is the size of the immediate operand
712 that will follow the instruction. */
34a6d0b7 713
770c2fc7
RH		 714static void tcg_out_sib_offset(TCGContext *s, int r, int rm, int index,
715 int shift, intptr_t offset)
c896fe29 716{
34a6d0b7
RH		 717 int mod, len;
718
5d8a4f8f
RH		 719 if (index < 0 && rm < 0) {
720 if (TCG_TARGET_REG_BITS == 64) {
721 /* Try for a rip-relative addressing mode. This has replaced
722 the 32-bit-mode absolute addressing encoding. */
357e3d8a
RH		 723 intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
724 intptr_t disp = offset - pc;
5d8a4f8f 725 if (disp == (int32_t)disp) {
5d8a4f8f
RH		 726 tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
727 tcg_out32(s, disp);
728 return;
729 }
34a6d0b7 730
5d8a4f8f
RH		 731 /* Try for an absolute address encoding. This requires the
732 use of the MODRM+SIB encoding and is therefore larger than
733 rip-relative addressing. */
734 if (offset == (int32_t)offset) {
5d8a4f8f
RH		 735 tcg_out8(s, (LOWREGMASK(r) << 3) | 4);
736 tcg_out8(s, (4 << 3) | 5);
737 tcg_out32(s, offset);
738 return;
739 }
740
741 /* ??? The memory isn't directly addressable. */
770c2fc7 742 g_assert_not_reached();
5d8a4f8f
RH		 743 } else {
744 /* Absolute address. */
5d8a4f8f
RH		 745 tcg_out8(s, (r << 3) | 5);
746 tcg_out32(s, offset);
747 return;
748 }
749 }
34a6d0b7
RH		 750
751 /* Find the length of the immediate addend. Note that the encoding
752 that would be used for (%ebp) indicates absolute addressing. */
5d8a4f8f 753 if (rm < 0) {
34a6d0b7 754 mod = 0, len = 4, rm = 5;
5d8a4f8f 755 } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
34a6d0b7
RH		 756 mod = 0, len = 0;
757 } else if (offset == (int8_t)offset) {
758 mod = 0x40, len = 1;
c896fe29 759 } else {
34a6d0b7
RH		 760 mod = 0x80, len = 4;
761 }
762
763 /* Use a single byte MODRM format if possible. Note that the encoding
764 that would be used for %esp is the escape to the two byte form. */
5d8a4f8f 765 if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
34a6d0b7 766 /* Single byte MODRM format. */
5d8a4f8f 767 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
34a6d0b7
RH		 768 } else {
769 /* Two byte MODRM+SIB format. */
770
771 /* Note that the encoding that would place %esp into the index
5d8a4f8f
RH		 772 field indicates no index register. In 64-bit mode, the REX.X
773 bit counts, so %r12 can be used as the index. */
774 if (index < 0) {
34a6d0b7 775 index = 4;
c896fe29 776 } else {
eabb7b91 777 tcg_debug_assert(index != TCG_REG_ESP);
c896fe29 778 }
34a6d0b7 779
5d8a4f8f
RH		 780 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);
781 tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));
34a6d0b7
RH		 782 }
783
784 if (len == 1) {
785 tcg_out8(s, offset);
786 } else if (len == 4) {
c896fe29
FB		 787 tcg_out32(s, offset);
788 }
789}
790
770c2fc7
RH		 791static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
792 int index, int shift, intptr_t offset)
793{
794 tcg_out_opc(s, opc, r, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
795 tcg_out_sib_offset(s, r, rm, index, shift, offset);
796}
797
798static void tcg_out_vex_modrm_sib_offset(TCGContext *s, int opc, int r, int v,
799 int rm, int index, int shift,
800 intptr_t offset)
801{
802 tcg_out_vex_opc(s, opc, r, v, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
803 tcg_out_sib_offset(s, r, rm, index, shift, offset);
804}
805
5d8a4f8f
RH		 806/* A simplification of the above with no index or shift. */
807static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
357e3d8a 808 int rm, intptr_t offset)
34a6d0b7
RH		 809{
810 tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
811}
812
770c2fc7
RH		 813static inline void tcg_out_vex_modrm_offset(TCGContext *s, int opc, int r,
814 int v, int rm, intptr_t offset)
815{
816 tcg_out_vex_modrm_sib_offset(s, opc, r, v, rm, -1, 0, offset);
817}
818
819/* Output an opcode with an expected reference to the constant pool. */
820static inline void tcg_out_modrm_pool(TCGContext *s, int opc, int r)
821{
822 tcg_out_opc(s, opc, r, 0, 0);
823 /* Absolute for 32-bit, pc-relative for 64-bit. */
824 tcg_out8(s, LOWREGMASK(r) << 3 | 5);
825 tcg_out32(s, 0);
826}
827
828/* Output an opcode with an expected reference to the constant pool. */
829static inline void tcg_out_vex_modrm_pool(TCGContext *s, int opc, int r)
830{
831 tcg_out_vex_opc(s, opc, r, 0, 0, 0);
832 /* Absolute for 32-bit, pc-relative for 64-bit. */
833 tcg_out8(s, LOWREGMASK(r) << 3 | 5);
834 tcg_out32(s, 0);
835}
836
81570a70
RH		 837/* Generate dest op= src. Uses the same ARITH_* codes as tgen_arithi. */
838static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
839{
5d8a4f8f
RH		 840 /* Propagate an opcode prefix, such as P_REXW. */
841 int ext = subop & ~0x7;
842 subop &= 0x7;
843
844 tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
81570a70
RH		 845}
846
78113e83 847static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
c896fe29 848{
770c2fc7
RH		 849 int rexw = 0;
850
851 if (arg == ret) {
78113e83 852 return true;
770c2fc7
RH		 853 }
854 switch (type) {
855 case TCG_TYPE_I64:
856 rexw = P_REXW;
857 /* fallthru */
858 case TCG_TYPE_I32:
859 if (ret < 16) {
860 if (arg < 16) {
861 tcg_out_modrm(s, OPC_MOVL_GvEv + rexw, ret, arg);
862 } else {
863 tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, arg, 0, ret);
864 }
865 } else {
866 if (arg < 16) {
867 tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, ret, 0, arg);
868 } else {
869 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
870 }
871 }
872 break;
873
874 case TCG_TYPE_V64:
875 tcg_debug_assert(ret >= 16 && arg >= 16);
876 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
877 break;
878 case TCG_TYPE_V128:
879 tcg_debug_assert(ret >= 16 && arg >= 16);
880 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx, ret, 0, arg);
881 break;
882 case TCG_TYPE_V256:
883 tcg_debug_assert(ret >= 16 && arg >= 16);
884 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx | P_VEXL, ret, 0, arg);
885 break;
886
887 default:
888 g_assert_not_reached();
889 }
78113e83 890 return true;
770c2fc7
RH		 891}
892
1e262b49
RH		 893static const int avx2_dup_insn[4] = {
894 OPC_VPBROADCASTB, OPC_VPBROADCASTW,
895 OPC_VPBROADCASTD, OPC_VPBROADCASTQ,
896};
897
e7632cfa 898static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
770c2fc7
RH		 899 TCGReg r, TCGReg a)
900{
901 if (have_avx2) {
770c2fc7 902 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
1e262b49 903 tcg_out_vex_modrm(s, avx2_dup_insn[vece] + vex_l, r, 0, a);
770c2fc7
RH		 904 } else {
905 switch (vece) {
906 case MO_8:
907 /* ??? With zero in a register, use PSHUFB. */
7eb30ef0 908 tcg_out_vex_modrm(s, OPC_PUNPCKLBW, r, a, a);
770c2fc7
RH		 909 a = r;
910 /* FALLTHRU */
911 case MO_16:
7eb30ef0 912 tcg_out_vex_modrm(s, OPC_PUNPCKLWD, r, a, a);
770c2fc7
RH		 913 a = r;
914 /* FALLTHRU */
915 case MO_32:
916 tcg_out_vex_modrm(s, OPC_PSHUFD, r, 0, a);
917 /* imm8 operand: all output lanes selected from input lane 0. */
918 tcg_out8(s, 0);
919 break;
920 case MO_64:
7eb30ef0 921 tcg_out_vex_modrm(s, OPC_PUNPCKLQDQ, r, a, a);
770c2fc7
RH		 922 break;
923 default:
924 g_assert_not_reached();
925 }
926 }
e7632cfa 927 return true;
770c2fc7
RH		 928}
929
d6ecb4a9
RH		 930static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
931 TCGReg r, TCGReg base, intptr_t offset)
932{
1e262b49
RH		 933 if (have_avx2) {
934 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
935 tcg_out_vex_modrm_offset(s, avx2_dup_insn[vece] + vex_l,
936 r, 0, base, offset);
937 } else {
938 switch (vece) {
939 case MO_64:
7b60ef32 940 tcg_out_vex_modrm_offset(s, OPC_MOVDDUP, r, 0, base, offset);
1e262b49
RH		 941 break;
942 case MO_32:
943 tcg_out_vex_modrm_offset(s, OPC_VBROADCASTSS, r, 0, base, offset);
944 break;
945 case MO_16:
946 tcg_out_vex_modrm_offset(s, OPC_VPINSRW, r, r, base, offset);
947 tcg_out8(s, 0); /* imm8 */
948 tcg_out_dup_vec(s, type, vece, r, r);
949 break;
950 case MO_8:
951 tcg_out_vex_modrm_offset(s, OPC_VPINSRB, r, r, base, offset);
952 tcg_out8(s, 0); /* imm8 */
953 tcg_out_dup_vec(s, type, vece, r, r);
954 break;
955 default:
956 g_assert_not_reached();
957 }
958 }
959 return true;
d6ecb4a9
RH		 960}
961
4e186175
RH		 962static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
963 TCGReg ret, int64_t arg)
770c2fc7
RH		 964{
965 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
966
967 if (arg == 0) {
968 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
969 return;
970 }
971 if (arg == -1) {
972 tcg_out_vex_modrm(s, OPC_PCMPEQB + vex_l, ret, ret, ret);
973 return;
974 }
975
4e186175
RH		 976 if (TCG_TARGET_REG_BITS == 32 && vece < MO_64) {
977 if (have_avx2) {
978 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret);
979 } else {
980 tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret);
981 }
982 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0);
983 } else {
770c2fc7
RH		 984 if (type == TCG_TYPE_V64) {
985 tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret);
986 } else if (have_avx2) {
987 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTQ + vex_l, ret);
988 } else {
7b60ef32 989 tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret);
770c2fc7 990 }
4e186175
RH		 991 if (TCG_TARGET_REG_BITS == 64) {
992 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
1e262b49 993 } else {
4e186175 994 new_pool_l2(s, R_386_32, s->code_ptr - 4, 0, arg, arg >> 32);
1e262b49 995 }
af266089 996 }
c896fe29
FB		 997}
998
0a6a8bc8
RH		 999static void tcg_out_movi_vec(TCGContext *s, TCGType type,
1000 TCGReg ret, tcg_target_long arg)
c896fe29 1001{
0a6a8bc8
RH		 1002 if (arg == 0) {
1003 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
770c2fc7 1004 return;
770c2fc7 1005 }
0a6a8bc8
RH		 1006 if (arg == -1) {
1007 tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret);
1008 return;
1009 }
1010
1011 int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW);
1012 tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret);
1013 if (TCG_TARGET_REG_BITS == 64) {
1014 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
1015 } else {
1016 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0);
1017 }
1018}
1019
1020static void tcg_out_movi_int(TCGContext *s, TCGType type,
1021 TCGReg ret, tcg_target_long arg)
1022{
1023 tcg_target_long diff;
770c2fc7 1024
c896fe29 1025 if (arg == 0) {
81570a70 1026 tgen_arithr(s, ARITH_XOR, ret, ret);
5d8a4f8f 1027 return;
8023ccda
RH		 1028 }
1029 if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
5d8a4f8f
RH		 1030 tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
1031 tcg_out32(s, arg);
8023ccda
RH		 1032 return;
1033 }
1034 if (arg == (int32_t)arg) {
5d8a4f8f
RH		 1035 tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
1036 tcg_out32(s, arg);
8023ccda 1037 return;
c896fe29 1038 }
8023ccda
RH		 1039
1040 /* Try a 7 byte pc-relative lea before the 10 byte movq. */
705ed477 1041 diff = tcg_pcrel_diff(s, (const void *)arg) - 7;
8023ccda
RH		 1042 if (diff == (int32_t)diff) {
1043 tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
1044 tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
1045 tcg_out32(s, diff);
1046 return;
1047 }
1048
1049 tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
1050 tcg_out64(s, arg);
c896fe29
FB		 1051}
1052
0a6a8bc8
RH		 1053static void tcg_out_movi(TCGContext *s, TCGType type,
1054 TCGReg ret, tcg_target_long arg)
1055{
1056 switch (type) {
1057 case TCG_TYPE_I32:
1058#if TCG_TARGET_REG_BITS == 64
1059 case TCG_TYPE_I64:
1060#endif
1061 if (ret < 16) {
1062 tcg_out_movi_int(s, type, ret, arg);
1063 } else {
1064 tcg_out_movi_vec(s, type, ret, arg);
1065 }
1066 break;
1067 default:
1068 g_assert_not_reached();
1069 }
1070}
1071
6858614e
RH		 1072static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
1073{
1074 if (val == (int8_t)val) {
5d8a4f8f 1075 tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
6858614e 1076 tcg_out8(s, val);
5d8a4f8f
RH		 1077 } else if (val == (int32_t)val) {
1078 tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
6858614e 1079 tcg_out32(s, val);
5d8a4f8f
RH		 1080 } else {
1081 tcg_abort();
6858614e
RH		 1082 }
1083}
1084
a7d00d4e
PK		 1085static inline void tcg_out_mb(TCGContext *s, TCGArg a0)
1086{
1087 /* Given the strength of x86 memory ordering, we only need care for
1088 store-load ordering. Experimentally, "lock orl $0,0(%esp)" is
1089 faster than "mfence", so don't bother with the sse insn. */
1090 if (a0 & TCG_MO_ST_LD) {
1091 tcg_out8(s, 0xf0);
1092 tcg_out_modrm_offset(s, OPC_ARITH_EvIb, ARITH_OR, TCG_REG_ESP, 0);
1093 tcg_out8(s, 0);
1094 }
1095}
1096
6858614e
RH		 1097static inline void tcg_out_push(TCGContext *s, int reg)
1098{
5d8a4f8f 1099 tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
6858614e
RH		 1100}
1101
1102static inline void tcg_out_pop(TCGContext *s, int reg)
1103{
5d8a4f8f 1104 tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
6858614e
RH		 1105}
1106
770c2fc7
RH		 1107static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1108 TCGReg arg1, intptr_t arg2)
c896fe29 1109{
770c2fc7
RH		 1110 switch (type) {
1111 case TCG_TYPE_I32:
1112 if (ret < 16) {
1113 tcg_out_modrm_offset(s, OPC_MOVL_GvEv, ret, arg1, arg2);
1114 } else {
1115 tcg_out_vex_modrm_offset(s, OPC_MOVD_VyEy, ret, 0, arg1, arg2);
1116 }
1117 break;
1118 case TCG_TYPE_I64:
1119 if (ret < 16) {
1120 tcg_out_modrm_offset(s, OPC_MOVL_GvEv | P_REXW, ret, arg1, arg2);
1121 break;
1122 }
1123 /* FALLTHRU */
1124 case TCG_TYPE_V64:
11e2bfef 1125 /* There is no instruction that can validate 8-byte alignment. */
770c2fc7
RH		 1126 tcg_debug_assert(ret >= 16);
1127 tcg_out_vex_modrm_offset(s, OPC_MOVQ_VqWq, ret, 0, arg1, arg2);
1128 break;
1129 case TCG_TYPE_V128:
11e2bfef
RH		 1130 /*
1131 * The gvec infrastructure is asserts that v128 vector loads
1132 * and stores use a 16-byte aligned offset. Validate that the
1133 * final pointer is aligned by using an insn that will SIGSEGV.
1134 */
770c2fc7 1135 tcg_debug_assert(ret >= 16);
11e2bfef 1136 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_VxWx, ret, 0, arg1, arg2);
770c2fc7
RH		 1137 break;
1138 case TCG_TYPE_V256:
11e2bfef
RH		 1139 /*
1140 * The gvec infrastructure only requires 16-byte alignment,
1141 * so here we must use an unaligned load.
1142 */
770c2fc7
RH		 1143 tcg_debug_assert(ret >= 16);
1144 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_VxWx | P_VEXL,
1145 ret, 0, arg1, arg2);
1146 break;
1147 default:
1148 g_assert_not_reached();
1149 }
c896fe29
FB		 1150}
1151
770c2fc7
RH		 1152static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1153 TCGReg arg1, intptr_t arg2)
c896fe29 1154{
770c2fc7
RH		 1155 switch (type) {
1156 case TCG_TYPE_I32:
1157 if (arg < 16) {
1158 tcg_out_modrm_offset(s, OPC_MOVL_EvGv, arg, arg1, arg2);
1159 } else {
1160 tcg_out_vex_modrm_offset(s, OPC_MOVD_EyVy, arg, 0, arg1, arg2);
1161 }
1162 break;
1163 case TCG_TYPE_I64:
1164 if (arg < 16) {
1165 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_REXW, arg, arg1, arg2);
1166 break;
1167 }
1168 /* FALLTHRU */
1169 case TCG_TYPE_V64:
11e2bfef 1170 /* There is no instruction that can validate 8-byte alignment. */
770c2fc7
RH		 1171 tcg_debug_assert(arg >= 16);
1172 tcg_out_vex_modrm_offset(s, OPC_MOVQ_WqVq, arg, 0, arg1, arg2);
1173 break;
1174 case TCG_TYPE_V128:
11e2bfef
RH		 1175 /*
1176 * The gvec infrastructure is asserts that v128 vector loads
1177 * and stores use a 16-byte aligned offset. Validate that the
1178 * final pointer is aligned by using an insn that will SIGSEGV.
1179 */
770c2fc7 1180 tcg_debug_assert(arg >= 16);
11e2bfef 1181 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_WxVx, arg, 0, arg1, arg2);
770c2fc7
RH		 1182 break;
1183 case TCG_TYPE_V256:
11e2bfef
RH		 1184 /*
1185 * The gvec infrastructure only requires 16-byte alignment,
1186 * so here we must use an unaligned store.
1187 */
770c2fc7
RH		 1188 tcg_debug_assert(arg >= 16);
1189 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_WxVx | P_VEXL,
1190 arg, 0, arg1, arg2);
1191 break;
1192 default:
1193 g_assert_not_reached();
1194 }
c896fe29
FB		 1195}
1196
59d7c14e
RH		 1197static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1198 TCGReg base, intptr_t ofs)
c6f29ff0 1199{
59d7c14e
RH		 1200 int rexw = 0;
1201 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) {
1202 if (val != (int32_t)val) {
1203 return false;
1204 }
1205 rexw = P_REXW;
770c2fc7
RH		 1206 } else if (type != TCG_TYPE_I32) {
1207 return false;
59d7c14e
RH		 1208 }
1209 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs);
c6f29ff0 1210 tcg_out32(s, val);
59d7c14e 1211 return true;
c6f29ff0
RH		 1212}
1213
f53dba01
RH		 1214static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
1215{
96b4cf38
RH		 1216 /* Propagate an opcode prefix, such as P_DATA16. */
1217 int ext = subopc & ~0x7;
1218 subopc &= 0x7;
1219
f53dba01 1220 if (count == 1) {
5d8a4f8f 1221 tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
f53dba01 1222 } else {
5d8a4f8f 1223 tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
f53dba01
RH		 1224 tcg_out8(s, count);
1225 }
1226}
1227
fcb5dac1
RH		 1228static inline void tcg_out_bswap32(TCGContext *s, int reg)
1229{
5d8a4f8f 1230 tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
fcb5dac1
RH		 1231}
1232
1233static inline void tcg_out_rolw_8(TCGContext *s, int reg)
1234{
5d8a4f8f 1235 tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
fcb5dac1
RH		 1236}
1237
55e082a7
RH		 1238static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
1239{
1240 /* movzbl */
eabb7b91 1241 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
5d8a4f8f 1242 tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
55e082a7
RH		 1243}
1244
5d8a4f8f 1245static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
6817c355
RH		 1246{
1247 /* movsbl */
eabb7b91 1248 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
5d8a4f8f 1249 tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
6817c355
RH		 1250}
1251
55e082a7
RH		 1252static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
1253{
1254 /* movzwl */
1255 tcg_out_modrm(s, OPC_MOVZWL, dest, src);
1256}
1257
5d8a4f8f 1258static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
6817c355 1259{
5d8a4f8f
RH		 1260 /* movsw[lq] */
1261 tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
6817c355
RH		 1262}
1263
5d8a4f8f 1264static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
c896fe29 1265{
5d8a4f8f
RH		 1266 /* 32-bit mov zero extends. */
1267 tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
1268}
1269
1270static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
1271{
1272 tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
1273}
1274
1275static inline void tcg_out_bswap64(TCGContext *s, int reg)
1276{
1277 tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
1278}
1279
1280static void tgen_arithi(TCGContext *s, int c, int r0,
1281 tcg_target_long val, int cf)
1282{
1283 int rexw = 0;
1284
1285 if (TCG_TARGET_REG_BITS == 64) {
1286 rexw = c & -8;
1287 c &= 7;
1288 }
1289
81570a70
RH		 1290 /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
1291 partial flags update stalls on Pentium4 and are not recommended
1292 by current Intel optimization manuals. */
1293 if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {
447d681e 1294 int is_inc = (c == ARITH_ADD) ^ (val < 0);
5d8a4f8f
RH		 1295 if (TCG_TARGET_REG_BITS == 64) {
1296 /* The single-byte increment encodings are re-tasked as the
1297 REX prefixes. Use the MODRM encoding. */
1298 tcg_out_modrm(s, OPC_GRP5 + rexw,
1299 (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
1300 } else {
1301 tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);
1302 }
1303 return;
1304 }
1305
1306 if (c == ARITH_AND) {
1307 if (TCG_TARGET_REG_BITS == 64) {
1308 if (val == 0xffffffffu) {
1309 tcg_out_ext32u(s, r0, r0);
1310 return;
1311 }
1312 if (val == (uint32_t)val) {
1313 /* AND with no high bits set can use a 32-bit operation. */
1314 rexw = 0;
1315 }
1316 }
dc397ca3 1317 if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {
5d8a4f8f
RH		 1318 tcg_out_ext8u(s, r0, r0);
1319 return;
1320 }
1321 if (val == 0xffffu) {
1322 tcg_out_ext16u(s, r0, r0);
1323 return;
1324 }
1325 }
1326
1327 if (val == (int8_t)val) {
1328 tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
c896fe29 1329 tcg_out8(s, val);
5d8a4f8f
RH		 1330 return;
1331 }
1332 if (rexw == 0 || val == (int32_t)val) {
1333 tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
c896fe29 1334 tcg_out32(s, val);
5d8a4f8f 1335 return;
c896fe29 1336 }
5d8a4f8f
RH		 1337
1338 tcg_abort();
c896fe29
FB		 1339}
1340
3e9a474e 1341static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
c896fe29 1342{
5d8a4f8f
RH		 1343 if (val != 0) {
1344 tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
1345 }
c896fe29
FB		 1346}
1347
f75b56c1 1348/* Use SMALL != 0 to force a short forward branch. */
bec16311 1349static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
c896fe29
FB		 1350{
1351 int32_t val, val1;
78686523 1352
c896fe29 1353 if (l->has_value) {
f6bff89d 1354 val = tcg_pcrel_diff(s, l->u.value_ptr);
c896fe29
FB		 1355 val1 = val - 2;
1356 if ((int8_t)val1 == val1) {
f75b56c1 1357 if (opc == -1) {
da441cff 1358 tcg_out8(s, OPC_JMP_short);
f75b56c1 1359 } else {
da441cff 1360 tcg_out8(s, OPC_JCC_short + opc);
f75b56c1 1361 }
c896fe29
FB		 1362 tcg_out8(s, val1);
1363 } else {
f75b56c1
RH		 1364 if (small) {
1365 tcg_abort();
1366 }
c896fe29 1367 if (opc == -1) {
da441cff 1368 tcg_out8(s, OPC_JMP_long);
c896fe29
FB		 1369 tcg_out32(s, val - 5);
1370 } else {
5d8a4f8f 1371 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
c896fe29
FB		 1372 tcg_out32(s, val - 6);
1373 }
1374 }
f75b56c1
RH		 1375 } else if (small) {
1376 if (opc == -1) {
da441cff 1377 tcg_out8(s, OPC_JMP_short);
f75b56c1 1378 } else {
da441cff 1379 tcg_out8(s, OPC_JCC_short + opc);
f75b56c1 1380 }
bec16311 1381 tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1);
f75b56c1 1382 s->code_ptr += 1;
c896fe29
FB		 1383 } else {
1384 if (opc == -1) {
da441cff 1385 tcg_out8(s, OPC_JMP_long);
c896fe29 1386 } else {
5d8a4f8f 1387 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
c896fe29 1388 }
bec16311 1389 tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4);
623e265c 1390 s->code_ptr += 4;
c896fe29
FB		 1391 }
1392}
1393
1d2699ae 1394static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,
5d8a4f8f 1395 int const_arg2, int rexw)
c896fe29 1396{
c896fe29
FB		 1397 if (const_arg2) {
1398 if (arg2 == 0) {
c896fe29 1399 /* test r, r */
5d8a4f8f 1400 tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
c896fe29 1401 } else {
5d8a4f8f 1402 tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
c896fe29
FB		 1403 }
1404 } else {
5d8a4f8f 1405 tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
c896fe29 1406 }
1d2699ae
RH		 1407}
1408
5d8a4f8f
RH		 1409static void tcg_out_brcond32(TCGContext *s, TCGCond cond,
1410 TCGArg arg1, TCGArg arg2, int const_arg2,
bec16311 1411 TCGLabel *label, int small)
1d2699ae 1412{
5d8a4f8f 1413 tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
bec16311 1414 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
c896fe29
FB		 1415}
1416
5d8a4f8f
RH		 1417#if TCG_TARGET_REG_BITS == 64
1418static void tcg_out_brcond64(TCGContext *s, TCGCond cond,
1419 TCGArg arg1, TCGArg arg2, int const_arg2,
bec16311 1420 TCGLabel *label, int small)
5d8a4f8f
RH		 1421{
1422 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
bec16311 1423 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
5d8a4f8f
RH		 1424}
1425#else
c896fe29
FB		 1426/* XXX: we implement it at the target level to avoid having to
1427 handle cross basic blocks temporaries */
f75b56c1
RH		 1428static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
1429 const int *const_args, int small)
c896fe29 1430{
bec16311
RH		 1431 TCGLabel *label_next = gen_new_label();
1432 TCGLabel *label_this = arg_label(args[5]);
42a268c2 1433
c896fe29
FB		 1434 switch(args[4]) {
1435 case TCG_COND_EQ:
5d8a4f8f
RH		 1436 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
1437 label_next, 1);
1438 tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],
bec16311 1439 label_this, small);
c896fe29
FB		 1440 break;
1441 case TCG_COND_NE:
5d8a4f8f 1442 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
bec16311 1443 label_this, small);
5d8a4f8f 1444 tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],
bec16311 1445 label_this, small);
c896fe29
FB		 1446 break;
1447 case TCG_COND_LT:
5d8a4f8f 1448 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
bec16311 1449 label_this, small);
f75b56c1 1450 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1451 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
bec16311 1452 label_this, small);
c896fe29
FB		 1453 break;
1454 case TCG_COND_LE:
5d8a4f8f 1455 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
bec16311 1456 label_this, small);
f75b56c1 1457 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1458 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
bec16311 1459 label_this, small);
c896fe29
FB		 1460 break;
1461 case TCG_COND_GT:
5d8a4f8f 1462 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
bec16311 1463 label_this, small);
f75b56c1 1464 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1465 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
bec16311 1466 label_this, small);
c896fe29
FB		 1467 break;
1468 case TCG_COND_GE:
5d8a4f8f 1469 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
bec16311 1470 label_this, small);
f75b56c1 1471 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1472 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
bec16311 1473 label_this, small);
c896fe29
FB		 1474 break;
1475 case TCG_COND_LTU:
5d8a4f8f 1476 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
bec16311 1477 label_this, small);
f75b56c1 1478 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1479 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
bec16311 1480 label_this, small);
c896fe29
FB		 1481 break;
1482 case TCG_COND_LEU:
5d8a4f8f 1483 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
bec16311 1484 label_this, small);
f75b56c1 1485 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1486 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
bec16311 1487 label_this, small);
c896fe29
FB		 1488 break;
1489 case TCG_COND_GTU:
5d8a4f8f 1490 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
bec16311 1491 label_this, small);
f75b56c1 1492 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1493 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
bec16311 1494 label_this, small);
c896fe29
FB		 1495 break;
1496 case TCG_COND_GEU:
5d8a4f8f 1497 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
bec16311 1498 label_this, small);
f75b56c1 1499 tcg_out_jxx(s, JCC_JNE, label_next, 1);
5d8a4f8f 1500 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
bec16311 1501 label_this, small);
c896fe29
FB		 1502 break;
1503 default:
1504 tcg_abort();
1505 }
92ab8e7d 1506 tcg_out_label(s, label_next);
c896fe29 1507}
5d8a4f8f 1508#endif
c896fe29 1509
5d8a4f8f
RH		 1510static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest,
1511 TCGArg arg1, TCGArg arg2, int const_arg2)
1d2699ae 1512{
5d8a4f8f 1513 tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
32a8ffb9 1514 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
a369a702 1515 tcg_out_ext8u(s, dest, dest);
1d2699ae
RH		 1516}
1517
5d8a4f8f
RH		 1518#if TCG_TARGET_REG_BITS == 64
1519static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest,
1520 TCGArg arg1, TCGArg arg2, int const_arg2)
1521{
1522 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
1523 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
1524 tcg_out_ext8u(s, dest, dest);
1525}
1526#else
1d2699ae
RH		 1527static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
1528 const int *const_args)
1529{
1530 TCGArg new_args[6];
bec16311 1531 TCGLabel *label_true, *label_over;
1d2699ae
RH		 1532
1533 memcpy(new_args, args+1, 5*sizeof(TCGArg));
1534
1535 if (args[0] == args[1] || args[0] == args[2]
1536 || (!const_args[3] && args[0] == args[3])
1537 || (!const_args[4] && args[0] == args[4])) {
1538 /* When the destination overlaps with one of the argument
1539 registers, don't do anything tricky. */
bec16311
RH		 1540 label_true = gen_new_label();
1541 label_over = gen_new_label();
1d2699ae 1542
bec16311 1543 new_args[5] = label_arg(label_true);
1d2699ae
RH		 1544 tcg_out_brcond2(s, new_args, const_args+1, 1);
1545
1546 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
1547 tcg_out_jxx(s, JCC_JMP, label_over, 1);
92ab8e7d 1548 tcg_out_label(s, label_true);
1d2699ae
RH		 1549
1550 tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
92ab8e7d 1551 tcg_out_label(s, label_over);
1d2699ae
RH		 1552 } else {
1553 /* When the destination does not overlap one of the arguments,
1554 clear the destination first, jump if cond false, and emit an
1555 increment in the true case. This results in smaller code. */
1556
1557 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
1558
bec16311 1559 label_over = gen_new_label();
1d2699ae 1560 new_args[4] = tcg_invert_cond(new_args[4]);
bec16311 1561 new_args[5] = label_arg(label_over);
1d2699ae
RH		 1562 tcg_out_brcond2(s, new_args, const_args+1, 1);
1563
1564 tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
92ab8e7d 1565 tcg_out_label(s, label_over);
1d2699ae
RH		 1566 }
1567}
5d8a4f8f
RH		 1568#endif
1569
bbf25f90
RH		 1570static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw,
1571 TCGReg dest, TCGReg v1)
d0a16297 1572{
76a347e1 1573 if (have_cmov) {
bbf25f90 1574 tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | rexw, dest, v1);
76a347e1 1575 } else {
bec16311 1576 TCGLabel *over = gen_new_label();
76a347e1
RH		 1577 tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
1578 tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
92ab8e7d 1579 tcg_out_label(s, over);
76a347e1 1580 }
d0a16297
RH		 1581}
1582
bbf25f90
RH		 1583static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGReg dest,
1584 TCGReg c1, TCGArg c2, int const_c2,
1585 TCGReg v1)
1586{
1587 tcg_out_cmp(s, c1, c2, const_c2, 0);
1588 tcg_out_cmov(s, cond, 0, dest, v1);
1589}
1590
d0a16297 1591#if TCG_TARGET_REG_BITS == 64
bbf25f90
RH		 1592static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGReg dest,
1593 TCGReg c1, TCGArg c2, int const_c2,
1594 TCGReg v1)
d0a16297
RH		 1595{
1596 tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
bbf25f90 1597 tcg_out_cmov(s, cond, P_REXW, dest, v1);
d0a16297
RH		 1598}
1599#endif
1600
bbf25f90
RH		 1601static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
1602 TCGArg arg2, bool const_a2)
1603{
39f099ec 1604 if (have_bmi1) {
bbf25f90 1605 tcg_out_modrm(s, OPC_TZCNT + rexw, dest, arg1);
39f099ec
RH		 1606 if (const_a2) {
1607 tcg_debug_assert(arg2 == (rexw ? 64 : 32));
1608 } else {
1609 tcg_debug_assert(dest != arg2);
1610 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2);
1611 }
bbf25f90 1612 } else {
9bf38308 1613 tcg_debug_assert(dest != arg2);
bbf25f90 1614 tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1);
9bf38308 1615 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2);
bbf25f90
RH		 1616 }
1617}
1618
1619static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
1620 TCGArg arg2, bool const_a2)
1621{
1622 if (have_lzcnt) {
1623 tcg_out_modrm(s, OPC_LZCNT + rexw, dest, arg1);
1624 if (const_a2) {
1625 tcg_debug_assert(arg2 == (rexw ? 64 : 32));
1626 } else {
1627 tcg_debug_assert(dest != arg2);
1628 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2);
1629 }
1630 } else {
9bf38308
RH		 1631 tcg_debug_assert(!const_a2);
1632 tcg_debug_assert(dest != arg1);
1633 tcg_debug_assert(dest != arg2);
bbf25f90 1634
9bf38308 1635 /* Recall that the output of BSR is the index not the count. */
bbf25f90 1636 tcg_out_modrm(s, OPC_BSR + rexw, dest, arg1);
9bf38308
RH		 1637 tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0);
1638
1639 /* Since we have destroyed the flags from BSR, we have to re-test. */
1640 tcg_out_cmp(s, arg1, 0, 1, rexw);
1641 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2);
bbf25f90
RH		 1642 }
1643}
1644
2be7d76b 1645static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest)
5d8a4f8f 1646{
f6bff89d 1647 intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
5d8a4f8f
RH		 1648
1649 if (disp == (int32_t)disp) {
1650 tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
1651 tcg_out32(s, disp);
1652 } else {
4e45f239
RH		 1653 /* rip-relative addressing into the constant pool.
1654 This is 6 + 8 = 14 bytes, as compared to using an
1655 an immediate load 10 + 6 = 16 bytes, plus we may
1656 be able to re-use the pool constant for more calls. */
1657 tcg_out_opc(s, OPC_GRP5, 0, 0, 0);
1658 tcg_out8(s, (call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev) << 3 | 5);
1659 new_pool_label(s, (uintptr_t)dest, R_386_PC32, s->code_ptr, -4);
1660 tcg_out32(s, 0);
5d8a4f8f
RH		 1661 }
1662}
1663
2be7d76b 1664static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
5d8a4f8f
RH		 1665{
1666 tcg_out_branch(s, 1, dest);
1667}
1d2699ae 1668
705ed477 1669static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest)
aadb21a4 1670{
5d8a4f8f 1671 tcg_out_branch(s, 0, dest);
aadb21a4
RH		 1672}
1673
0d07abf0
SF		 1674static void tcg_out_nopn(TCGContext *s, int n)
1675{
1676 int i;
1677 /* Emit 1 or 2 operand size prefixes for the standard one byte nop,
1678 * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the
1679 * duplicate prefix, and all of the interesting recent cores can
1680 * decode and discard the duplicates in a single cycle.
1681 */
1682 tcg_debug_assert(n >= 1);
1683 for (i = 1; i < n; ++i) {
1684 tcg_out8(s, 0x66);
1685 }
1686 tcg_out8(s, 0x90);
1687}
1688
c896fe29 1689#if defined(CONFIG_SOFTMMU)
401c227b
RH		 1690/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
1691 * int mmu_idx, uintptr_t ra)
1692 */
4b473e0c 1693static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
8221a267
RH		 1694 [MO_UB] = helper_ret_ldub_mmu,
1695 [MO_LEUW] = helper_le_lduw_mmu,
1696 [MO_LEUL] = helper_le_ldul_mmu,
fc313c64 1697 [MO_LEUQ] = helper_le_ldq_mmu,
8221a267
RH		 1698 [MO_BEUW] = helper_be_lduw_mmu,
1699 [MO_BEUL] = helper_be_ldul_mmu,
fc313c64 1700 [MO_BEUQ] = helper_be_ldq_mmu,
e141ab52
BS		 1701};
1702
401c227b
RH		 1703/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
1704 * uintxx_t val, int mmu_idx, uintptr_t ra)
1705 */
4b473e0c 1706static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
8221a267
RH		 1707 [MO_UB] = helper_ret_stb_mmu,
1708 [MO_LEUW] = helper_le_stw_mmu,
1709 [MO_LEUL] = helper_le_stl_mmu,
fc313c64 1710 [MO_LEUQ] = helper_le_stq_mmu,
8221a267
RH		 1711 [MO_BEUW] = helper_be_stw_mmu,
1712 [MO_BEUL] = helper_be_stl_mmu,
fc313c64 1713 [MO_BEUQ] = helper_be_stq_mmu,
e141ab52 1714};
8516a044
RH		 1715
1716/* Perform the TLB load and compare.
1717
1718 Inputs:
7352ee54 1719 ADDRLO and ADDRHI contain the low and high part of the address.
8516a044
RH		 1720
1721 MEM_INDEX and S_BITS are the memory context and log2 size of the load.
1722
1723 WHICH is the offset into the CPUTLBEntry structure of the slot to read.
1724 This should be offsetof addr_read or addr_write.
1725
1726 Outputs:
1727 LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
1728 positions of the displacements of forward jumps to the TLB miss case.
1729
166792f7 1730 Second argument register is loaded with the low part of the address.
5d8a4f8f
RH		 1731 In the TLB hit case, it has been adjusted as indicated by the TLB
1732 and so is a host address. In the TLB miss case, it continues to
1733 hold a guest address.
8516a044 1734
166792f7 1735 First argument register is clobbered. */
8516a044 1736
7352ee54 1737static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
14776ab5 1738 int mem_index, MemOp opc,
f6bff89d 1739 tcg_insn_unit **label_ptr, int which)
8516a044 1740{
7352ee54
RH		 1741 const TCGReg r0 = TCG_REG_L0;
1742 const TCGReg r1 = TCG_REG_L1;
d5dad3be 1743 TCGType ttype = TCG_TYPE_I32;
08b0b23b
AJ		 1744 TCGType tlbtype = TCG_TYPE_I32;
1745 int trexw = 0, hrexw = 0, tlbrexw = 0;
85aa8081
RH		 1746 unsigned a_bits = get_alignment_bits(opc);
1747 unsigned s_bits = opc & MO_SIZE;
1748 unsigned a_mask = (1 << a_bits) - 1;
1749 unsigned s_mask = (1 << s_bits) - 1;
1f00b27f 1750 target_ulong tlb_mask;
5d8a4f8f 1751
d5dad3be
RH		 1752 if (TCG_TARGET_REG_BITS == 64) {
1753 if (TARGET_LONG_BITS == 64) {
1754 ttype = TCG_TYPE_I64;
1755 trexw = P_REXW;
1756 }
1757 if (TCG_TYPE_PTR == TCG_TYPE_I64) {
d5dad3be 1758 hrexw = P_REXW;
54eaf40b 1759 if (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32) {
08b0b23b
AJ		 1760 tlbtype = TCG_TYPE_I64;
1761 tlbrexw = P_REXW;
1762 }
d5dad3be 1763 }
5d8a4f8f 1764 }
8516a044 1765
08b0b23b 1766 tcg_out_mov(s, tlbtype, r0, addrlo);
54eaf40b
EC		 1767 tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
1768 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1769
1770 tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, r0, TCG_AREG0,
269bd5d8
RH		 1771 TLB_MASK_TABLE_OFS(mem_index) +
1772 offsetof(CPUTLBDescFast, mask));
54eaf40b
EC		 1773
1774 tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r0, TCG_AREG0,
269bd5d8
RH		 1775 TLB_MASK_TABLE_OFS(mem_index) +
1776 offsetof(CPUTLBDescFast, table));
54eaf40b 1777
85aa8081
RH		 1778 /* If the required alignment is at least as large as the access, simply
1779 copy the address and mask. For lesser alignments, check that we don't
1780 cross pages for the complete access. */
1781 if (a_bits >= s_bits) {
8cc580f6
AJ		 1782 tcg_out_mov(s, ttype, r1, addrlo);
1783 } else {
85aa8081 1784 tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask);
8cc580f6 1785 }
ebb90a00 1786 tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask;
1f00b27f 1787 tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0);
8516a044 1788
166792f7 1789 /* cmp 0(r0), r1 */
54eaf40b 1790 tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, which);
8516a044 1791
d5dad3be 1792 /* Prepare for both the fast path add of the tlb addend, and the slow
4810d96f 1793 path function argument setup. */
d5dad3be 1794 tcg_out_mov(s, ttype, r1, addrlo);
8516a044 1795
b76f0d8c
YL		 1796 /* jne slow_path */
1797 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
8516a044 1798 label_ptr[0] = s->code_ptr;
b76f0d8c 1799 s->code_ptr += 4;
8516a044 1800
5d8a4f8f 1801 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
166792f7 1802 /* cmp 4(r0), addrhi */
54eaf40b 1803 tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, which + 4);
8516a044 1804
b76f0d8c
YL		 1805 /* jne slow_path */
1806 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
8516a044 1807 label_ptr[1] = s->code_ptr;
b76f0d8c 1808 s->code_ptr += 4;
8516a044
RH		 1809 }
1810
1811 /* TLB Hit. */
1812
166792f7 1813 /* add addend(r0), r1 */
d5dad3be 1814 tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
54eaf40b 1815 offsetof(CPUTLBEntry, addend));
8516a044 1816}
7352ee54
RH		 1817
1818/*
1819 * Record the context of a call to the out of line helper code for the slow path
1820 * for a load or store, so that we can later generate the correct helper code
1821 */
3dbc8c61 1822static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64,
9002ffcb 1823 MemOpIdx oi,
7352ee54
RH		 1824 TCGReg datalo, TCGReg datahi,
1825 TCGReg addrlo, TCGReg addrhi,
3972ef6f 1826 tcg_insn_unit *raddr,
f6bff89d 1827 tcg_insn_unit **label_ptr)
7352ee54
RH		 1828{
1829 TCGLabelQemuLdst *label = new_ldst_label(s);
1830
1831 label->is_ld = is_ld;
3972ef6f 1832 label->oi = oi;
3dbc8c61 1833 label->type = is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
7352ee54
RH		 1834 label->datalo_reg = datalo;
1835 label->datahi_reg = datahi;
1836 label->addrlo_reg = addrlo;
1837 label->addrhi_reg = addrhi;
e5e2e4c7 1838 label->raddr = tcg_splitwx_to_rx(raddr);
7352ee54
RH		 1839 label->label_ptr[0] = label_ptr[0];
1840 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
1841 label->label_ptr[1] = label_ptr[1];
1842 }
1843}
1844
1845/*
1846 * Generate code for the slow path for a load at the end of block
1847 */
aeee05f5 1848static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
7352ee54 1849{
9002ffcb 1850 MemOpIdx oi = l->oi;
14776ab5 1851 MemOp opc = get_memop(oi);
7352ee54 1852 TCGReg data_reg;
f6bff89d 1853 tcg_insn_unit **label_ptr = &l->label_ptr[0];
3dbc8c61 1854 int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
7352ee54
RH		 1855
1856 /* resolve label address */
5c53bb81 1857 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
7352ee54 1858 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
5c53bb81 1859 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
7352ee54
RH		 1860 }
1861
1862 if (TCG_TARGET_REG_BITS == 32) {
1863 int ofs = 0;
1864
1865 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
1866 ofs += 4;
1867
1868 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
1869 ofs += 4;
1870
1871 if (TARGET_LONG_BITS == 64) {
1872 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
1873 ofs += 4;
1874 }
1875
59d7c14e 1876 tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
7352ee54
RH		 1877 ofs += 4;
1878
59d7c14e 1879 tcg_out_sti(s, TCG_TYPE_PTR, (uintptr_t)l->raddr, TCG_REG_ESP, ofs);
7352ee54
RH		 1880 } else {
1881 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
1882 /* The second argument is already loaded with addrlo. */
3972ef6f 1883 tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi);
7352ee54
RH		 1884 tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
1885 (uintptr_t)l->raddr);
1886 }
1887
2b7ec66f 1888 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
7352ee54
RH		 1889
1890 data_reg = l->datalo_reg;
1891 switch (opc & MO_SSIZE) {
1892 case MO_SB:
3dbc8c61 1893 tcg_out_ext8s(s, data_reg, TCG_REG_EAX, rexw);
7352ee54
RH		 1894 break;
1895 case MO_SW:
3dbc8c61 1896 tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
7352ee54
RH		 1897 break;
1898#if TCG_TARGET_REG_BITS == 64
1899 case MO_SL:
1900 tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
1901 break;
1902#endif
1903 case MO_UB:
1904 case MO_UW:
1905 /* Note that the helpers have zero-extended to tcg_target_long. */
1906 case MO_UL:
1907 tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
1908 break;
fc313c64 1909 case MO_UQ:
7352ee54
RH		 1910 if (TCG_TARGET_REG_BITS == 64) {
1911 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
1912 } else if (data_reg == TCG_REG_EDX) {
1913 /* xchg %edx, %eax */
1914 tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
1915 tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
1916 } else {
1917 tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
1918 tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
1919 }
1920 break;
1921 default:
1922 tcg_abort();
1923 }
1924
1925 /* Jump to the code corresponding to next IR of qemu_st */
f6bff89d 1926 tcg_out_jmp(s, l->raddr);
aeee05f5 1927 return true;
7352ee54
RH		 1928}
1929
1930/*
1931 * Generate code for the slow path for a store at the end of block
1932 */
aeee05f5 1933static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
7352ee54 1934{
9002ffcb 1935 MemOpIdx oi = l->oi;
14776ab5
TN		 1936 MemOp opc = get_memop(oi);
1937 MemOp s_bits = opc & MO_SIZE;
f6bff89d 1938 tcg_insn_unit **label_ptr = &l->label_ptr[0];
7352ee54
RH		 1939 TCGReg retaddr;
1940
1941 /* resolve label address */
5c53bb81 1942 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
7352ee54 1943 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
5c53bb81 1944 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
7352ee54
RH		 1945 }
1946
1947 if (TCG_TARGET_REG_BITS == 32) {
1948 int ofs = 0;
1949
1950 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
1951 ofs += 4;
1952
1953 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
1954 ofs += 4;
1955
1956 if (TARGET_LONG_BITS == 64) {
1957 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
1958 ofs += 4;
1959 }
1960
1961 tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
1962 ofs += 4;
1963
1964 if (s_bits == MO_64) {
1965 tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
1966 ofs += 4;
1967 }
1968
59d7c14e 1969 tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
7352ee54
RH		 1970 ofs += 4;
1971
1972 retaddr = TCG_REG_EAX;
3972ef6f
RH		 1973 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
1974 tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
7352ee54
RH		 1975 } else {
1976 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
1977 /* The second argument is already loaded with addrlo. */
1978 tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
1979 tcg_target_call_iarg_regs[2], l->datalo_reg);
3972ef6f 1980 tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);
7352ee54
RH		 1981
1982 if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
1983 retaddr = tcg_target_call_iarg_regs[4];
1984 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
1985 } else {
1986 retaddr = TCG_REG_RAX;
1987 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
0b919667
RH		 1988 tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP,
1989 TCG_TARGET_CALL_STACK_OFFSET);
7352ee54
RH		 1990 }
1991 }
1992
1993 /* "Tail call" to the helper, with the return address back inline. */
1994 tcg_out_push(s, retaddr);
2b7ec66f 1995 tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
aeee05f5 1996 return true;
7352ee54 1997}
b1ee3c67
RH		 1998#else
1999
2000static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo,
2001 TCGReg addrhi, unsigned a_bits)
2002{
2003 unsigned a_mask = (1 << a_bits) - 1;
2004 TCGLabelQemuLdst *label;
2005
2006 /*
2007 * We are expecting a_bits to max out at 7, so we can usually use testb.
2008 * For i686, we have to use testl for %esi/%edi.
2009 */
2010 if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) {
2011 tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo);
2012 tcg_out8(s, a_mask);
2013 } else {
2014 tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo);
2015 tcg_out32(s, a_mask);
2016 }
2017
2018 /* jne slow_path */
2019 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
2020
2021 label = new_ldst_label(s);
2022 label->is_ld = is_ld;
2023 label->addrlo_reg = addrlo;
2024 label->addrhi_reg = addrhi;
2025 label->raddr = tcg_splitwx_to_rx(s->code_ptr + 4);
2026 label->label_ptr[0] = s->code_ptr;
2027
2028 s->code_ptr += 4;
2029}
2030
2031static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
2032{
2033 /* resolve label address */
2034 tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4);
2035
2036 if (TCG_TARGET_REG_BITS == 32) {
2037 int ofs = 0;
2038
2039 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
2040 ofs += 4;
2041
2042 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
2043 ofs += 4;
2044 if (TARGET_LONG_BITS == 64) {
2045 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
2046 ofs += 4;
2047 }
2048
2049 tcg_out_pushi(s, (uintptr_t)l->raddr);
2050 } else {
2051 tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
2052 l->addrlo_reg);
2053 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
2054
2055 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr);
2056 tcg_out_push(s, TCG_REG_RAX);
2057 }
2058
2059 /* "Tail call" to the helper, with the return address back inline. */
2060 tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld
2061 : helper_unaligned_st));
2062 return true;
2063}
2064
2065static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2066{
2067 return tcg_out_fail_alignment(s, l);
2068}
2069
2070static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2071{
2072 return tcg_out_fail_alignment(s, l);
2073}
2074
2075#if TCG_TARGET_REG_BITS == 32
913c2bdd
RH		 2076# define x86_guest_base_seg 0
2077# define x86_guest_base_index -1
2078# define x86_guest_base_offset guest_base
2079#else
2080static int x86_guest_base_seg;
2081static int x86_guest_base_index = -1;
2082static int32_t x86_guest_base_offset;
2083# if defined(__x86_64__) && defined(__linux__)
2084# include <asm/prctl.h>
2085# include <sys/prctl.h>
44b37ace 2086int arch_prctl(int code, unsigned long addr);
913c2bdd 2087static inline int setup_guest_base_seg(void)
44b37ace 2088{
b76f21a7 2089 if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
913c2bdd 2090 return P_GS;
44b37ace 2091 }
913c2bdd 2092 return 0;
44b37ace 2093}
5785c17f
RH		 2094# elif defined (__FreeBSD__) || defined (__FreeBSD_kernel__)
2095# include <machine/sysarch.h>
2096static inline int setup_guest_base_seg(void)
2097{
2098 if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) {
2099 return P_GS;
2100 }
2101 return 0;
2102}
913c2bdd
RH		 2103# else
2104static inline int setup_guest_base_seg(void)
2105{
2106 return 0;
2107}
2108# endif
b1ee3c67 2109#endif
44b37ace 2110#endif /* SOFTMMU */
c896fe29 2111
37c5d0d5 2112static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
ee8ba9e4 2113 TCGReg base, int index, intptr_t ofs,
14776ab5 2114 int seg, bool is64, MemOp memop)
be5a4eb7 2115{
d2ef1b83 2116 bool use_movbe = false;
1d21d95b 2117 int rexw = is64 * P_REXW;
085bb5bb
AJ		 2118 int movop = OPC_MOVL_GvEv;
2119
d2ef1b83
RH		 2120 /* Do big-endian loads with movbe. */
2121 if (memop & MO_BSWAP) {
2122 tcg_debug_assert(have_movbe);
2123 use_movbe = true;
085bb5bb
AJ		 2124 movop = OPC_MOVBE_GyMy;
2125 }
37c5d0d5
RH		 2126
2127 switch (memop & MO_SSIZE) {
2128 case MO_UB:
ee8ba9e4
RH		 2129 tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo,
2130 base, index, 0, ofs);
be5a4eb7 2131 break;
37c5d0d5 2132 case MO_SB:
1d21d95b 2133 tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + seg, datalo,
ee8ba9e4 2134 base, index, 0, ofs);
be5a4eb7 2135 break;
37c5d0d5 2136 case MO_UW:
d2ef1b83
RH		 2137 if (use_movbe) {
2138 /* There is no extending movbe; only low 16-bits are modified. */
2139 if (datalo != base && datalo != index) {
2140 /* XOR breaks dependency chains. */
2141 tgen_arithr(s, ARITH_XOR, datalo, datalo);
ee8ba9e4
RH		 2142 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2143 datalo, base, index, 0, ofs);
085bb5bb 2144 } else {
d2ef1b83
RH		 2145 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2146 datalo, base, index, 0, ofs);
2147 tcg_out_ext16u(s, datalo, datalo);
085bb5bb 2148 }
d2ef1b83
RH		 2149 } else {
2150 tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
2151 base, index, 0, ofs);
2152 }
2153 break;
2154 case MO_SW:
2155 if (use_movbe) {
2156 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2157 datalo, base, index, 0, ofs);
2158 tcg_out_ext16s(s, datalo, datalo, rexw);
5d8a4f8f 2159 } else {
1d21d95b 2160 tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg,
ee8ba9e4 2161 datalo, base, index, 0, ofs);
be5a4eb7
RH		 2162 }
2163 break;
37c5d0d5 2164 case MO_UL:
ee8ba9e4 2165 tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
be5a4eb7 2166 break;
5d8a4f8f 2167#if TCG_TARGET_REG_BITS == 64
37c5d0d5 2168 case MO_SL:
d2ef1b83
RH		 2169 if (use_movbe) {
2170 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo,
ee8ba9e4 2171 base, index, 0, ofs);
5d8a4f8f 2172 tcg_out_ext32s(s, datalo, datalo);
be5a4eb7 2173 } else {
ee8ba9e4
RH		 2174 tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo,
2175 base, index, 0, ofs);
be5a4eb7 2176 }
5d8a4f8f
RH		 2177 break;
2178#endif
fc313c64 2179 case MO_UQ:
5d8a4f8f 2180 if (TCG_TARGET_REG_BITS == 64) {
ee8ba9e4
RH		 2181 tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
2182 base, index, 0, ofs);
5d8a4f8f 2183 } else {
d2ef1b83
RH		 2184 if (use_movbe) {
2185 TCGReg t = datalo;
5d8a4f8f
RH		 2186 datalo = datahi;
2187 datahi = t;
2188 }
2189 if (base != datalo) {
ee8ba9e4
RH		 2190 tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2191 base, index, 0, ofs);
2192 tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2193 base, index, 0, ofs + 4);
5d8a4f8f 2194 } else {
ee8ba9e4
RH		 2195 tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2196 base, index, 0, ofs + 4);
2197 tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2198 base, index, 0, ofs);
5d8a4f8f 2199 }
be5a4eb7
RH		 2200 }
2201 break;
2202 default:
d2ef1b83 2203 g_assert_not_reached();
be5a4eb7
RH		 2204 }
2205}
379f6698 2206
c896fe29
FB		 2207/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
2208 EAX. It will be useful once fixed registers globals are less
2209 common. */
8221a267 2210static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
c896fe29 2211{
7352ee54 2212 TCGReg datalo, datahi, addrlo;
8221a267 2213 TCGReg addrhi __attribute__((unused));
9002ffcb 2214 MemOpIdx oi;
14776ab5 2215 MemOp opc;
c896fe29 2216#if defined(CONFIG_SOFTMMU)
37c5d0d5 2217 int mem_index;
f6bff89d 2218 tcg_insn_unit *label_ptr[2];
b1ee3c67
RH		 2219#else
2220 unsigned a_bits;
c896fe29
FB		 2221#endif
2222
7352ee54 2223 datalo = *args++;
8221a267 2224 datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
7352ee54 2225 addrlo = *args++;
8221a267 2226 addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
59227d5d
RH		 2227 oi = *args++;
2228 opc = get_memop(oi);
c896fe29
FB		 2229
2230#if defined(CONFIG_SOFTMMU)
59227d5d 2231 mem_index = get_mmuidx(oi);
1a6dc1e4 2232
8cc580f6 2233 tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
8516a044 2234 label_ptr, offsetof(CPUTLBEntry, addr_read));
1a6dc1e4
RH		 2235
2236 /* TLB Hit. */
1d21d95b 2237 tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, is64, opc);
c896fe29 2238
b76f0d8c 2239 /* Record the current context of a load into ldst label */
3dbc8c61 2240 add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi,
3972ef6f 2241 s->code_ptr, label_ptr);
c896fe29 2242#else
b1ee3c67
RH		 2243 a_bits = get_alignment_bits(opc);
2244 if (a_bits) {
2245 tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits);
2246 }
2247
913c2bdd
RH		 2248 tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
2249 x86_guest_base_offset, x86_guest_base_seg,
2250 is64, opc);
c896fe29 2251#endif
be5a4eb7 2252}
c896fe29 2253
37c5d0d5 2254static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
4810d96f 2255 TCGReg base, int index, intptr_t ofs,
14776ab5 2256 int seg, MemOp memop)
be5a4eb7 2257{
d2ef1b83 2258 bool use_movbe = false;
085bb5bb
AJ		 2259 int movop = OPC_MOVL_EvGv;
2260
d2ef1b83
RH		 2261 /*
2262 * Do big-endian stores with movbe or softmmu.
2263 * User-only without movbe will have its swapping done generically.
2264 */
2265 if (memop & MO_BSWAP) {
2266 tcg_debug_assert(have_movbe);
2267 use_movbe = true;
085bb5bb
AJ		 2268 movop = OPC_MOVBE_MyGy;
2269 }
be5a4eb7 2270
37c5d0d5
RH		 2271 switch (memop & MO_SIZE) {
2272 case MO_8:
07ce0b05
RH		 2273 /* This is handled with constraints on INDEX_op_qemu_st8_i32. */
2274 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4);
4810d96f
RH		 2275 tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
2276 datalo, base, index, 0, ofs);
c896fe29 2277 break;
37c5d0d5 2278 case MO_16:
4810d96f
RH		 2279 tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo,
2280 base, index, 0, ofs);
c896fe29 2281 break;
37c5d0d5 2282 case MO_32:
4810d96f 2283 tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
c896fe29 2284 break;
37c5d0d5 2285 case MO_64:
5d8a4f8f 2286 if (TCG_TARGET_REG_BITS == 64) {
4810d96f
RH		 2287 tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
2288 base, index, 0, ofs);
c896fe29 2289 } else {
d2ef1b83
RH		 2290 if (use_movbe) {
2291 TCGReg t = datalo;
085bb5bb
AJ		 2292 datalo = datahi;
2293 datahi = t;
2294 }
4810d96f
RH		 2295 tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2296 base, index, 0, ofs);
2297 tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2298 base, index, 0, ofs + 4);
c896fe29
FB		 2299 }
2300 break;
2301 default:
d2ef1b83 2302 g_assert_not_reached();
c896fe29 2303 }
c896fe29
FB		 2304}
2305
8221a267 2306static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
c896fe29 2307{
7352ee54 2308 TCGReg datalo, datahi, addrlo;
8221a267 2309 TCGReg addrhi __attribute__((unused));
9002ffcb 2310 MemOpIdx oi;
14776ab5 2311 MemOp opc;
c896fe29 2312#if defined(CONFIG_SOFTMMU)
37c5d0d5 2313 int mem_index;
f6bff89d 2314 tcg_insn_unit *label_ptr[2];
b1ee3c67
RH		 2315#else
2316 unsigned a_bits;
c896fe29
FB		 2317#endif
2318
7352ee54 2319 datalo = *args++;
8221a267 2320 datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
7352ee54 2321 addrlo = *args++;
8221a267 2322 addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
59227d5d
RH		 2323 oi = *args++;
2324 opc = get_memop(oi);
c896fe29
FB		 2325
2326#if defined(CONFIG_SOFTMMU)
59227d5d 2327 mem_index = get_mmuidx(oi);
1a6dc1e4 2328
8cc580f6 2329 tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
8516a044 2330 label_ptr, offsetof(CPUTLBEntry, addr_write));
1a6dc1e4
RH		 2331
2332 /* TLB Hit. */
4810d96f 2333 tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc);
c896fe29 2334
b76f0d8c 2335 /* Record the current context of a store into ldst label */
3dbc8c61 2336 add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi,
3972ef6f 2337 s->code_ptr, label_ptr);
b76f0d8c 2338#else
b1ee3c67
RH		 2339 a_bits = get_alignment_bits(opc);
2340 if (a_bits) {
2341 tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits);
2342 }
2343
913c2bdd
RH		 2344 tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
2345 x86_guest_base_offset, x86_guest_base_seg, opc);
b76f0d8c 2346#endif
b76f0d8c 2347}
c896fe29 2348
a9751609 2349static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
5e8892db
MR		 2350 const TCGArg args[TCG_MAX_OP_ARGS],
2351 const int const_args[TCG_MAX_OP_ARGS])
c896fe29 2352{
42d5b514
RH		 2353 TCGArg a0, a1, a2;
2354 int c, const_a2, vexop, rexw = 0;
5d8a4f8f
RH		 2355
2356#if TCG_TARGET_REG_BITS == 64
2357# define OP_32_64(x) \
2358 case glue(glue(INDEX_op_, x), _i64): \
2359 rexw = P_REXW; /* FALLTHRU */ \
2360 case glue(glue(INDEX_op_, x), _i32)
2361#else
2362# define OP_32_64(x) \
2363 case glue(glue(INDEX_op_, x), _i32)
2364#endif
78686523 2365
42d5b514
RH		 2366 /* Hoist the loads of the most common arguments. */
2367 a0 = args[0];
2368 a1 = args[1];
2369 a2 = args[2];
2370 const_a2 = const_args[2];
2371
2372 switch (opc) {
c896fe29 2373 case INDEX_op_exit_tb:
5cb4ef80
EC		 2374 /* Reuse the zeroing that exists for goto_ptr. */
2375 if (a0 == 0) {
8b5c2b62 2376 tcg_out_jmp(s, tcg_code_gen_epilogue);
5cb4ef80
EC		 2377 } else {
2378 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
2379 tcg_out_jmp(s, tb_ret_addr);
2380 }
c896fe29
FB		 2381 break;
2382 case INDEX_op_goto_tb:
f309101c 2383 if (s->tb_jmp_insn_offset) {
c896fe29 2384 /* direct jump method */
0d07abf0
SF		 2385 int gap;
2386 /* jump displacement must be aligned for atomic patching;
2387 * see if we need to add extra nops before jump
2388 */
705ed477 2389 gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
0d07abf0
SF		 2390 if (gap != 1) {
2391 tcg_out_nopn(s, gap - 1);
2392 }
da441cff 2393 tcg_out8(s, OPC_JMP_long); /* jmp im */
42d5b514 2394 s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
c896fe29
FB		 2395 tcg_out32(s, 0);
2396 } else {
2397 /* indirect jump method */
9363dedb 2398 tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
42d5b514 2399 (intptr_t)(s->tb_jmp_target_addr + a0));
c896fe29 2400 }
9f754620 2401 set_jmp_reset_offset(s, a0);
c896fe29 2402 break;
5cb4ef80
EC		 2403 case INDEX_op_goto_ptr:
2404 /* jmp to the given host address (could be epilogue) */
2405 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
2406 break;
c896fe29 2407 case INDEX_op_br:
42d5b514 2408 tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0);
c896fe29 2409 break;
5d8a4f8f
RH		 2410 OP_32_64(ld8u):
2411 /* Note that we can ignore REXW for the zero-extend to 64-bit. */
42d5b514 2412 tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2);
c896fe29 2413 break;
5d8a4f8f 2414 OP_32_64(ld8s):
42d5b514 2415 tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2);
c896fe29 2416 break;
5d8a4f8f
RH		 2417 OP_32_64(ld16u):
2418 /* Note that we can ignore REXW for the zero-extend to 64-bit. */
42d5b514 2419 tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2);
c896fe29 2420 break;
5d8a4f8f 2421 OP_32_64(ld16s):
42d5b514 2422 tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2);
c896fe29 2423 break;
5d8a4f8f
RH		 2424#if TCG_TARGET_REG_BITS == 64
2425 case INDEX_op_ld32u_i64:
2426#endif
c896fe29 2427 case INDEX_op_ld_i32:
42d5b514 2428 tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2);
c896fe29 2429 break;
5d8a4f8f
RH		 2430
2431 OP_32_64(st8):
5c2d2a9e 2432 if (const_args[0]) {
42d5b514
RH		 2433 tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2);
2434 tcg_out8(s, a0);
5c2d2a9e 2435 } else {
42d5b514 2436 tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2);
5c2d2a9e 2437 }
c896fe29 2438 break;
5d8a4f8f 2439 OP_32_64(st16):
5c2d2a9e 2440 if (const_args[0]) {
42d5b514
RH		 2441 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2);
2442 tcg_out16(s, a0);
5c2d2a9e 2443 } else {
42d5b514 2444 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2);
5c2d2a9e 2445 }
c896fe29 2446 break;
5d8a4f8f
RH		 2447#if TCG_TARGET_REG_BITS == 64
2448 case INDEX_op_st32_i64:
2449#endif
c896fe29 2450 case INDEX_op_st_i32:
5c2d2a9e 2451 if (const_args[0]) {
42d5b514
RH		 2452 tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2);
2453 tcg_out32(s, a0);
5c2d2a9e 2454 } else {
42d5b514 2455 tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2);
5c2d2a9e 2456 }
c896fe29 2457 break;
5d8a4f8f
RH		 2458
2459 OP_32_64(add):
5d1e4e85 2460 /* For 3-operand addition, use LEA. */
42d5b514
RH		 2461 if (a0 != a1) {
2462 TCGArg c3 = 0;
2463 if (const_a2) {
5d1e4e85
RH		 2464 c3 = a2, a2 = -1;
2465 } else if (a0 == a2) {
2466 /* Watch out for dest = src + dest, since we've removed
2467 the matching constraint on the add. */
5d8a4f8f 2468 tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
5d1e4e85
RH		 2469 break;
2470 }
2471
5d8a4f8f 2472 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
5d1e4e85
RH		 2473 break;
2474 }
2475 c = ARITH_ADD;
2476 goto gen_arith;
5d8a4f8f 2477 OP_32_64(sub):
c896fe29
FB		 2478 c = ARITH_SUB;
2479 goto gen_arith;
5d8a4f8f 2480 OP_32_64(and):
c896fe29
FB		 2481 c = ARITH_AND;
2482 goto gen_arith;
5d8a4f8f 2483 OP_32_64(or):
c896fe29
FB		 2484 c = ARITH_OR;
2485 goto gen_arith;
5d8a4f8f 2486 OP_32_64(xor):
c896fe29
FB		 2487 c = ARITH_XOR;
2488 goto gen_arith;
c896fe29 2489 gen_arith:
42d5b514
RH		 2490 if (const_a2) {
2491 tgen_arithi(s, c + rexw, a0, a2, 0);
c896fe29 2492 } else {
42d5b514 2493 tgen_arithr(s, c + rexw, a0, a2);
c896fe29
FB		 2494 }
2495 break;
5d8a4f8f 2496
9d2eec20 2497 OP_32_64(andc):
42d5b514
RH		 2498 if (const_a2) {
2499 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
2500 tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0);
9d2eec20 2501 } else {
42d5b514 2502 tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1);
9d2eec20
RH		 2503 }
2504 break;
2505
5d8a4f8f 2506 OP_32_64(mul):
42d5b514 2507 if (const_a2) {
c896fe29 2508 int32_t val;
42d5b514 2509 val = a2;
c896fe29 2510 if (val == (int8_t)val) {
42d5b514 2511 tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0);
c896fe29
FB		 2512 tcg_out8(s, val);
2513 } else {
42d5b514 2514 tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0);
c896fe29
FB		 2515 tcg_out32(s, val);
2516 }
2517 } else {
42d5b514 2518 tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2);
c896fe29
FB		 2519 }
2520 break;
5d8a4f8f
RH		 2521
2522 OP_32_64(div2):
2523 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
c896fe29 2524 break;
5d8a4f8f
RH		 2525 OP_32_64(divu2):
2526 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
c896fe29 2527 break;
5d8a4f8f
RH		 2528
2529 OP_32_64(shl):
6a5aed4b
RH		 2530 /* For small constant 3-operand shift, use LEA. */
2531 if (const_a2 && a0 != a1 && (a2 - 1) < 3) {
2532 if (a2 - 1 == 0) {
2533 /* shl $1,a1,a0 -> lea (a1,a1),a0 */
2534 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0);
2535 } else {
2536 /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */
2537 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0);
2538 }
2539 break;
2540 }
c896fe29 2541 c = SHIFT_SHL;
6399ab33
RH		 2542 vexop = OPC_SHLX;
2543 goto gen_shift_maybe_vex;
5d8a4f8f 2544 OP_32_64(shr):
c896fe29 2545 c = SHIFT_SHR;
6399ab33
RH		 2546 vexop = OPC_SHRX;
2547 goto gen_shift_maybe_vex;
5d8a4f8f 2548 OP_32_64(sar):
c896fe29 2549 c = SHIFT_SAR;
6399ab33
RH		 2550 vexop = OPC_SARX;
2551 goto gen_shift_maybe_vex;
5d8a4f8f 2552 OP_32_64(rotl):
9619376c 2553 c = SHIFT_ROL;
5d8a4f8f
RH		 2554 goto gen_shift;
2555 OP_32_64(rotr):
9619376c 2556 c = SHIFT_ROR;
5d8a4f8f 2557 goto gen_shift;
6399ab33 2558 gen_shift_maybe_vex:
6a5aed4b
RH		 2559 if (have_bmi2) {
2560 if (!const_a2) {
2561 tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1);
2562 break;
2563 }
2564 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
6399ab33
RH		 2565 }
2566 /* FALLTHRU */
5d8a4f8f 2567 gen_shift:
42d5b514
RH		 2568 if (const_a2) {
2569 tcg_out_shifti(s, c + rexw, a0, a2);
81570a70 2570 } else {
42d5b514 2571 tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0);
81570a70 2572 }
c896fe29 2573 break;
5d8a4f8f 2574
bbf25f90
RH		 2575 OP_32_64(ctz):
2576 tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]);
2577 break;
2578 OP_32_64(clz):
2579 tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]);
2580 break;
993508e4
RH		 2581 OP_32_64(ctpop):
2582 tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1);
2583 break;
bbf25f90 2584
c896fe29 2585 case INDEX_op_brcond_i32:
42d5b514 2586 tcg_out_brcond32(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
c896fe29 2587 break;
5d8a4f8f 2588 case INDEX_op_setcond_i32:
42d5b514 2589 tcg_out_setcond32(s, args[3], a0, a1, a2, const_a2);
c896fe29 2590 break;
d0a16297 2591 case INDEX_op_movcond_i32:
42d5b514 2592 tcg_out_movcond32(s, args[5], a0, a1, a2, const_a2, args[3]);
d0a16297 2593 break;
c896fe29 2594
5d8a4f8f 2595 OP_32_64(bswap16):
7335a3d6
RH		 2596 if (a2 & TCG_BSWAP_OS) {
2597 /* Output must be sign-extended. */
2598 if (rexw) {
2599 tcg_out_bswap64(s, a0);
2600 tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48);
2601 } else {
2602 tcg_out_bswap32(s, a0);
2603 tcg_out_shifti(s, SHIFT_SAR, a0, 16);
2604 }
2605 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2606 /* Output must be zero-extended, but input isn't. */
2607 tcg_out_bswap32(s, a0);
2608 tcg_out_shifti(s, SHIFT_SHR, a0, 16);
2609 } else {
2610 tcg_out_rolw_8(s, a0);
2611 }
5d40cd63 2612 break;
5d8a4f8f 2613 OP_32_64(bswap32):
42d5b514 2614 tcg_out_bswap32(s, a0);
7335a3d6
RH		 2615 if (rexw && (a2 & TCG_BSWAP_OS)) {
2616 tcg_out_ext32s(s, a0, a0);
2617 }
9619376c
AJ		 2618 break;
2619
5d8a4f8f 2620 OP_32_64(neg):
42d5b514 2621 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0);
9619376c 2622 break;
5d8a4f8f 2623 OP_32_64(not):
42d5b514 2624 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
9619376c
AJ		 2625 break;
2626
5d8a4f8f 2627 OP_32_64(ext8s):
42d5b514 2628 tcg_out_ext8s(s, a0, a1, rexw);
9619376c 2629 break;
5d8a4f8f 2630 OP_32_64(ext16s):
42d5b514 2631 tcg_out_ext16s(s, a0, a1, rexw);
9619376c 2632 break;
5d8a4f8f 2633 OP_32_64(ext8u):
42d5b514 2634 tcg_out_ext8u(s, a0, a1);
5f0ce17f 2635 break;
5d8a4f8f 2636 OP_32_64(ext16u):
42d5b514 2637 tcg_out_ext16u(s, a0, a1);
5f0ce17f 2638 break;
9619376c 2639
8221a267
RH		 2640 case INDEX_op_qemu_ld_i32:
2641 tcg_out_qemu_ld(s, args, 0);
c896fe29 2642 break;
8221a267
RH		 2643 case INDEX_op_qemu_ld_i64:
2644 tcg_out_qemu_ld(s, args, 1);
c896fe29 2645 break;
8221a267 2646 case INDEX_op_qemu_st_i32:
07ce0b05 2647 case INDEX_op_qemu_st8_i32:
8221a267 2648 tcg_out_qemu_st(s, args, 0);
c896fe29 2649 break;
8221a267
RH		 2650 case INDEX_op_qemu_st_i64:
2651 tcg_out_qemu_st(s, args, 1);
c896fe29
FB		 2652 break;
2653
624988a5
RH		 2654 OP_32_64(mulu2):
2655 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
5d8a4f8f 2656 break;
624988a5
RH		 2657 OP_32_64(muls2):
2658 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]);
2659 break;
2660 OP_32_64(add2):
5d8a4f8f 2661 if (const_args[4]) {
42d5b514 2662 tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1);
5d8a4f8f 2663 } else {
42d5b514 2664 tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]);
5d8a4f8f
RH		 2665 }
2666 if (const_args[5]) {
42d5b514 2667 tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1);
5d8a4f8f 2668 } else {
42d5b514 2669 tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]);
5d8a4f8f
RH		 2670 }
2671 break;
624988a5 2672 OP_32_64(sub2):
5d8a4f8f 2673 if (const_args[4]) {
42d5b514 2674 tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1);
5d8a4f8f 2675 } else {
42d5b514 2676 tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]);
5d8a4f8f
RH		 2677 }
2678 if (const_args[5]) {
42d5b514 2679 tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1);
5d8a4f8f 2680 } else {
42d5b514 2681 tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]);
5d8a4f8f
RH		 2682 }
2683 break;
bbc863bf
RH		 2684
2685#if TCG_TARGET_REG_BITS == 32
2686 case INDEX_op_brcond2_i32:
2687 tcg_out_brcond2(s, args, const_args, 0);
2688 break;
2689 case INDEX_op_setcond2_i32:
2690 tcg_out_setcond2(s, args, const_args);
2691 break;
5d8a4f8f 2692#else /* TCG_TARGET_REG_BITS == 64 */
5d8a4f8f 2693 case INDEX_op_ld32s_i64:
42d5b514 2694 tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2);
5d8a4f8f
RH		 2695 break;
2696 case INDEX_op_ld_i64:
42d5b514 2697 tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2);
5d8a4f8f
RH		 2698 break;
2699 case INDEX_op_st_i64:
5c2d2a9e 2700 if (const_args[0]) {
42d5b514
RH		 2701 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2);
2702 tcg_out32(s, a0);
5c2d2a9e 2703 } else {
42d5b514 2704 tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2);
5c2d2a9e 2705 }
5d8a4f8f 2706 break;
5d8a4f8f
RH		 2707
2708 case INDEX_op_brcond_i64:
42d5b514 2709 tcg_out_brcond64(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
5d8a4f8f
RH		 2710 break;
2711 case INDEX_op_setcond_i64:
42d5b514 2712 tcg_out_setcond64(s, args[3], a0, a1, a2, const_a2);
5d8a4f8f 2713 break;
d0a16297 2714 case INDEX_op_movcond_i64:
42d5b514 2715 tcg_out_movcond64(s, args[5], a0, a1, a2, const_a2, args[3]);
d0a16297 2716 break;
5d8a4f8f
RH		 2717
2718 case INDEX_op_bswap64_i64:
42d5b514 2719 tcg_out_bswap64(s, a0);
5d8a4f8f 2720 break;
4f2331e5 2721 case INDEX_op_extu_i32_i64:
5d8a4f8f 2722 case INDEX_op_ext32u_i64:
75478279 2723 case INDEX_op_extrl_i64_i32:
42d5b514 2724 tcg_out_ext32u(s, a0, a1);
5d8a4f8f 2725 break;
4f2331e5 2726 case INDEX_op_ext_i32_i64:
5d8a4f8f 2727 case INDEX_op_ext32s_i64:
42d5b514 2728 tcg_out_ext32s(s, a0, a1);
5d8a4f8f 2729 break;
75478279
RH		 2730 case INDEX_op_extrh_i64_i32:
2731 tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
2732 break;
5d8a4f8f
RH		 2733#endif
2734
a4773324
JK		 2735 OP_32_64(deposit):
2736 if (args[3] == 0 && args[4] == 8) {
2737 /* load bits 0..7 */
42d5b514 2738 tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0);
a4773324
JK		 2739 } else if (args[3] == 8 && args[4] == 8) {
2740 /* load bits 8..15 */
42d5b514 2741 tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4);
a4773324
JK		 2742 } else if (args[3] == 0 && args[4] == 16) {
2743 /* load bits 0..15 */
42d5b514 2744 tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0);
a4773324
JK		 2745 } else {
2746 tcg_abort();
2747 }
2748 break;
2749
78fdbfb9 2750 case INDEX_op_extract_i64:
42d5b514 2751 if (a2 + args[3] == 32) {
78fdbfb9 2752 /* This is a 32-bit zero-extending right shift. */
42d5b514
RH		 2753 tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
2754 tcg_out_shifti(s, SHIFT_SHR, a0, a2);
78fdbfb9
RH		 2755 break;
2756 }
2757 /* FALLTHRU */
2758 case INDEX_op_extract_i32:
2759 /* On the off-chance that we can use the high-byte registers.
2760 Otherwise we emit the same ext16 + shift pattern that we
2761 would have gotten from the normal tcg-op.c expansion. */
42d5b514
RH		 2762 tcg_debug_assert(a2 == 8 && args[3] == 8);
2763 if (a1 < 4 && a0 < 8) {
2764 tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4);
78fdbfb9 2765 } else {
42d5b514
RH		 2766 tcg_out_ext16u(s, a0, a1);
2767 tcg_out_shifti(s, SHIFT_SHR, a0, 8);
78fdbfb9
RH		 2768 }
2769 break;
2770
2771 case INDEX_op_sextract_i32:
2772 /* We don't implement sextract_i64, as we cannot sign-extend to
2773 64-bits without using the REX prefix that explicitly excludes
2774 access to the high-byte registers. */
42d5b514
RH		 2775 tcg_debug_assert(a2 == 8 && args[3] == 8);
2776 if (a1 < 4 && a0 < 8) {
2777 tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
78fdbfb9 2778 } else {
42d5b514
RH		 2779 tcg_out_ext16s(s, a0, a1, 0);
2780 tcg_out_shifti(s, SHIFT_SAR, a0, 8);
78fdbfb9
RH		 2781 }
2782 break;
2783
c6fb8c0c
RH		 2784 OP_32_64(extract2):
2785 /* Note that SHRD outputs to the r/m operand. */
2786 tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0);
2787 tcg_out8(s, args[3]);
2788 break;
2789
a7d00d4e 2790 case INDEX_op_mb:
42d5b514 2791 tcg_out_mb(s, a0);
a7d00d4e 2792 break;
96d0ee7f
RH		 2793 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
2794 case INDEX_op_mov_i64:
96d0ee7f 2795 case INDEX_op_call: /* Always emitted via tcg_out_call. */
c896fe29
FB		 2796 default:
2797 tcg_abort();
2798 }
5d8a4f8f
RH		 2799
2800#undef OP_32_64
c896fe29
FB		 2801}
2802
770c2fc7
RH		 2803static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2804 unsigned vecl, unsigned vece,
5e8892db
MR		 2805 const TCGArg args[TCG_MAX_OP_ARGS],
2806 const int const_args[TCG_MAX_OP_ARGS])
770c2fc7
RH		 2807{
2808 static int const add_insn[4] = {
2809 OPC_PADDB, OPC_PADDW, OPC_PADDD, OPC_PADDQ
2810 };
8ffafbce
RH		 2811 static int const ssadd_insn[4] = {
2812 OPC_PADDSB, OPC_PADDSW, OPC_UD2, OPC_UD2
2813 };
2814 static int const usadd_insn[4] = {
3115584d 2815 OPC_PADDUB, OPC_PADDUW, OPC_UD2, OPC_UD2
8ffafbce 2816 };
770c2fc7
RH		 2817 static int const sub_insn[4] = {
2818 OPC_PSUBB, OPC_PSUBW, OPC_PSUBD, OPC_PSUBQ
2819 };
8ffafbce
RH		 2820 static int const sssub_insn[4] = {
2821 OPC_PSUBSB, OPC_PSUBSW, OPC_UD2, OPC_UD2
2822 };
2823 static int const ussub_insn[4] = {
3115584d 2824 OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2
8ffafbce 2825 };
770c2fc7 2826 static int const mul_insn[4] = {
4c8b9686 2827 OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ
770c2fc7
RH		 2828 };
2829 static int const shift_imm_insn[4] = {
2830 OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib
2831 };
2832 static int const cmpeq_insn[4] = {
2833 OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ
2834 };
2835 static int const cmpgt_insn[4] = {
2836 OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ
2837 };
2838 static int const punpckl_insn[4] = {
2839 OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ
2840 };
2841 static int const punpckh_insn[4] = {
2842 OPC_PUNPCKHBW, OPC_PUNPCKHWD, OPC_PUNPCKHDQ, OPC_PUNPCKHQDQ
2843 };
2844 static int const packss_insn[4] = {
2845 OPC_PACKSSWB, OPC_PACKSSDW, OPC_UD2, OPC_UD2
2846 };
2847 static int const packus_insn[4] = {
2848 OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2
2849 };
bc37faf4 2850 static int const smin_insn[4] = {
dac1648f 2851 OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ
bc37faf4
RH		 2852 };
2853 static int const smax_insn[4] = {
dac1648f 2854 OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ
bc37faf4
RH		 2855 };
2856 static int const umin_insn[4] = {
dac1648f 2857 OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ
bc37faf4
RH		 2858 };
2859 static int const umax_insn[4] = {
dac1648f 2860 OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ
bc37faf4 2861 };
102cd35c
RH		 2862 static int const rotlv_insn[4] = {
2863 OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ
2864 };
2865 static int const rotrv_insn[4] = {
2866 OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ
2867 };
a2ce146a 2868 static int const shlv_insn[4] = {
ef77ce0d 2869 OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ
a2ce146a
RH		 2870 };
2871 static int const shrv_insn[4] = {
ef77ce0d 2872 OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ
a2ce146a
RH		 2873 };
2874 static int const sarv_insn[4] = {
ef77ce0d 2875 OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ
a2ce146a 2876 };
0a8d7a3b
RH		 2877 static int const shls_insn[4] = {
2878 OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ
2879 };
2880 static int const shrs_insn[4] = {
2881 OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ
2882 };
2883 static int const sars_insn[4] = {
47b331b2 2884 OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ
0a8d7a3b 2885 };
965d5d06
RH		 2886 static int const vpshldi_insn[4] = {
2887 OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ
2888 };
2889 static int const vpshldv_insn[4] = {
2890 OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ
2891 };
2892 static int const vpshrdv_insn[4] = {
2893 OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ
2894 };
18f9b65f 2895 static int const abs_insn[4] = {
dac1648f 2896 OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ
18f9b65f 2897 };
770c2fc7
RH		 2898
2899 TCGType type = vecl + TCG_TYPE_V64;
2900 int insn, sub;
cf320769 2901 TCGArg a0, a1, a2, a3;
770c2fc7
RH		 2902
2903 a0 = args[0];
2904 a1 = args[1];
2905 a2 = args[2];
2906
2907 switch (opc) {
2908 case INDEX_op_add_vec:
2909 insn = add_insn[vece];
2910 goto gen_simd;
8ffafbce
RH		 2911 case INDEX_op_ssadd_vec:
2912 insn = ssadd_insn[vece];
2913 goto gen_simd;
2914 case INDEX_op_usadd_vec:
2915 insn = usadd_insn[vece];
2916 goto gen_simd;
770c2fc7
RH		 2917 case INDEX_op_sub_vec:
2918 insn = sub_insn[vece];
2919 goto gen_simd;
8ffafbce
RH		 2920 case INDEX_op_sssub_vec:
2921 insn = sssub_insn[vece];
2922 goto gen_simd;
2923 case INDEX_op_ussub_vec:
2924 insn = ussub_insn[vece];
2925 goto gen_simd;
770c2fc7
RH		 2926 case INDEX_op_mul_vec:
2927 insn = mul_insn[vece];
2928 goto gen_simd;
2929 case INDEX_op_and_vec:
2930 insn = OPC_PAND;
2931 goto gen_simd;
2932 case INDEX_op_or_vec:
2933 insn = OPC_POR;
2934 goto gen_simd;
2935 case INDEX_op_xor_vec:
2936 insn = OPC_PXOR;
2937 goto gen_simd;
bc37faf4
RH		 2938 case INDEX_op_smin_vec:
2939 insn = smin_insn[vece];
2940 goto gen_simd;
2941 case INDEX_op_umin_vec:
2942 insn = umin_insn[vece];
2943 goto gen_simd;
2944 case INDEX_op_smax_vec:
2945 insn = smax_insn[vece];
2946 goto gen_simd;
2947 case INDEX_op_umax_vec:
2948 insn = umax_insn[vece];
2949 goto gen_simd;
a2ce146a
RH		 2950 case INDEX_op_shlv_vec:
2951 insn = shlv_insn[vece];
2952 goto gen_simd;
2953 case INDEX_op_shrv_vec:
2954 insn = shrv_insn[vece];
2955 goto gen_simd;
2956 case INDEX_op_sarv_vec:
2957 insn = sarv_insn[vece];
2958 goto gen_simd;
102cd35c
RH		 2959 case INDEX_op_rotlv_vec:
2960 insn = rotlv_insn[vece];
2961 goto gen_simd;
2962 case INDEX_op_rotrv_vec:
2963 insn = rotrv_insn[vece];
2964 goto gen_simd;
0a8d7a3b
RH		 2965 case INDEX_op_shls_vec:
2966 insn = shls_insn[vece];
2967 goto gen_simd;
2968 case INDEX_op_shrs_vec:
2969 insn = shrs_insn[vece];
2970 goto gen_simd;
2971 case INDEX_op_sars_vec:
2972 insn = sars_insn[vece];
2973 goto gen_simd;
770c2fc7
RH		 2974 case INDEX_op_x86_punpckl_vec:
2975 insn = punpckl_insn[vece];
2976 goto gen_simd;
2977 case INDEX_op_x86_punpckh_vec:
2978 insn = punpckh_insn[vece];
2979 goto gen_simd;
2980 case INDEX_op_x86_packss_vec:
2981 insn = packss_insn[vece];
2982 goto gen_simd;
2983 case INDEX_op_x86_packus_vec:
2984 insn = packus_insn[vece];
2985 goto gen_simd;
965d5d06
RH		 2986 case INDEX_op_x86_vpshldv_vec:
2987 insn = vpshldv_insn[vece];
2988 a1 = a2;
2989 a2 = args[3];
2990 goto gen_simd;
2991 case INDEX_op_x86_vpshrdv_vec:
2992 insn = vpshrdv_insn[vece];
2993 a1 = a2;
2994 a2 = args[3];
2995 goto gen_simd;
7f34ed4b
RH		 2996#if TCG_TARGET_REG_BITS == 32
2997 case INDEX_op_dup2_vec:
e20cb81d
RH		 2998 /* First merge the two 32-bit inputs to a single 64-bit element. */
2999 tcg_out_vex_modrm(s, OPC_PUNPCKLDQ, a0, a1, a2);
3000 /* Then replicate the 64-bit elements across the rest of the vector. */
3001 if (type != TCG_TYPE_V64) {
3002 tcg_out_dup_vec(s, type, MO_64, a0, a0);
3003 }
3004 break;
7f34ed4b 3005#endif
18f9b65f
RH		 3006 case INDEX_op_abs_vec:
3007 insn = abs_insn[vece];
3008 a2 = a1;
3009 a1 = 0;
3010 goto gen_simd;
770c2fc7
RH		 3011 gen_simd:
3012 tcg_debug_assert(insn != OPC_UD2);
3013 if (type == TCG_TYPE_V256) {
3014 insn |= P_VEXL;
3015 }
3016 tcg_out_vex_modrm(s, insn, a0, a1, a2);
3017 break;
3018
3019 case INDEX_op_cmp_vec:
3020 sub = args[3];
3021 if (sub == TCG_COND_EQ) {
3022 insn = cmpeq_insn[vece];
3023 } else if (sub == TCG_COND_GT) {
3024 insn = cmpgt_insn[vece];
3025 } else {
3026 g_assert_not_reached();
3027 }
3028 goto gen_simd;
3029
3030 case INDEX_op_andc_vec:
3031 insn = OPC_PANDN;
3032 if (type == TCG_TYPE_V256) {
3033 insn |= P_VEXL;
3034 }
3035 tcg_out_vex_modrm(s, insn, a0, a2, a1);
3036 break;
3037
3038 case INDEX_op_shli_vec:
264e4182 3039 insn = shift_imm_insn[vece];
770c2fc7
RH		 3040 sub = 6;
3041 goto gen_shift;
3042 case INDEX_op_shri_vec:
264e4182 3043 insn = shift_imm_insn[vece];
770c2fc7
RH		 3044 sub = 2;
3045 goto gen_shift;
3046 case INDEX_op_sari_vec:
264e4182
RH		 3047 if (vece == MO_64) {
3048 insn = OPC_PSHIFTD_Ib | P_VEXW | P_EVEX;
3049 } else {
3050 insn = shift_imm_insn[vece];
3051 }
770c2fc7 3052 sub = 4;
4e73f842
RH		 3053 goto gen_shift;
3054 case INDEX_op_rotli_vec:
3055 insn = OPC_PSHIFTD_Ib | P_EVEX; /* VPROL[DQ] */
3056 if (vece == MO_64) {
3057 insn |= P_VEXW;
3058 }
3059 sub = 1;
3060 goto gen_shift;
770c2fc7
RH		 3061 gen_shift:
3062 tcg_debug_assert(vece != MO_8);
770c2fc7
RH		 3063 if (type == TCG_TYPE_V256) {
3064 insn |= P_VEXL;
3065 }
3066 tcg_out_vex_modrm(s, insn, sub, a0, a1);
3067 tcg_out8(s, a2);
3068 break;
3069
3070 case INDEX_op_ld_vec:
3071 tcg_out_ld(s, type, a0, a1, a2);
3072 break;
3073 case INDEX_op_st_vec:
3074 tcg_out_st(s, type, a0, a1, a2);
3075 break;
37ee55a0
RH		 3076 case INDEX_op_dupm_vec:
3077 tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3078 break;
770c2fc7
RH		 3079
3080 case INDEX_op_x86_shufps_vec:
3081 insn = OPC_SHUFPS;
3082 sub = args[3];
3083 goto gen_simd_imm8;
3084 case INDEX_op_x86_blend_vec:
3085 if (vece == MO_16) {
3086 insn = OPC_PBLENDW;
3087 } else if (vece == MO_32) {
3088 insn = (have_avx2 ? OPC_VPBLENDD : OPC_BLENDPS);
3089 } else {
3090 g_assert_not_reached();
3091 }
3092 sub = args[3];
3093 goto gen_simd_imm8;
3094 case INDEX_op_x86_vperm2i128_vec:
3095 insn = OPC_VPERM2I128;
3096 sub = args[3];
3097 goto gen_simd_imm8;
965d5d06
RH		 3098 case INDEX_op_x86_vpshldi_vec:
3099 insn = vpshldi_insn[vece];
3100 sub = args[3];
3101 goto gen_simd_imm8;
3143767b
RH		 3102
3103 case INDEX_op_not_vec:
3104 insn = OPC_VPTERNLOGQ;
3105 a2 = a1;
3106 sub = 0x33; /* !B */
3107 goto gen_simd_imm8;
3108 case INDEX_op_nor_vec:
3109 insn = OPC_VPTERNLOGQ;
3110 sub = 0x11; /* norCB */
3111 goto gen_simd_imm8;
3112 case INDEX_op_nand_vec:
3113 insn = OPC_VPTERNLOGQ;
3114 sub = 0x77; /* nandCB */
3115 goto gen_simd_imm8;
3116 case INDEX_op_eqv_vec:
3117 insn = OPC_VPTERNLOGQ;
3118 sub = 0x99; /* xnorCB */
3119 goto gen_simd_imm8;
3120 case INDEX_op_orc_vec:
3121 insn = OPC_VPTERNLOGQ;
3122 sub = 0xdd; /* orB!C */
3123 goto gen_simd_imm8;
3124
cf320769
RH		 3125 case INDEX_op_bitsel_vec:
3126 insn = OPC_VPTERNLOGQ;
3127 a3 = args[3];
3128 if (a0 == a1) {
3129 a1 = a2;
3130 a2 = a3;
3131 sub = 0xca; /* A?B:C */
3132 } else if (a0 == a2) {
3133 a2 = a3;
3134 sub = 0xe2; /* B?A:C */
3135 } else {
3136 tcg_out_mov(s, type, a0, a3);
3137 sub = 0xb8; /* B?C:A */
3138 }
3139 goto gen_simd_imm8;
3140
770c2fc7 3141 gen_simd_imm8:
965d5d06 3142 tcg_debug_assert(insn != OPC_UD2);
770c2fc7
RH		 3143 if (type == TCG_TYPE_V256) {
3144 insn |= P_VEXL;
3145 }
3146 tcg_out_vex_modrm(s, insn, a0, a1, a2);
3147 tcg_out8(s, sub);
3148 break;
3149
3150 case INDEX_op_x86_vpblendvb_vec:
3151 insn = OPC_VPBLENDVB;
3152 if (type == TCG_TYPE_V256) {
3153 insn |= P_VEXL;
3154 }
3155 tcg_out_vex_modrm(s, insn, a0, a1, a2);
3156 tcg_out8(s, args[3] << 4);
3157 break;
3158
3159 case INDEX_op_x86_psrldq_vec:
3160 tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1);
3161 tcg_out8(s, a2);
3162 break;
3163
bab1671f 3164 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
bab1671f 3165 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
770c2fc7
RH		 3166 default:
3167 g_assert_not_reached();
3168 }
3169}
3170
4c22e840
RH		 3171static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
3172{
cd26449a 3173 switch (op) {
5cb4ef80 3174 case INDEX_op_goto_ptr:
4c22e840 3175 return C_O0_I1(r);
5cb4ef80 3176
cd26449a
RH		 3177 case INDEX_op_ld8u_i32:
3178 case INDEX_op_ld8u_i64:
3179 case INDEX_op_ld8s_i32:
3180 case INDEX_op_ld8s_i64:
3181 case INDEX_op_ld16u_i32:
3182 case INDEX_op_ld16u_i64:
3183 case INDEX_op_ld16s_i32:
3184 case INDEX_op_ld16s_i64:
3185 case INDEX_op_ld_i32:
3186 case INDEX_op_ld32u_i64:
3187 case INDEX_op_ld32s_i64:
3188 case INDEX_op_ld_i64:
4c22e840 3189 return C_O1_I1(r, r);
a4773324 3190
cd26449a
RH		 3191 case INDEX_op_st8_i32:
3192 case INDEX_op_st8_i64:
4c22e840
RH		 3193 return C_O0_I2(qi, r);
3194
cd26449a
RH		 3195 case INDEX_op_st16_i32:
3196 case INDEX_op_st16_i64:
3197 case INDEX_op_st_i32:
3198 case INDEX_op_st32_i64:
4c22e840
RH		 3199 return C_O0_I2(ri, r);
3200
cd26449a 3201 case INDEX_op_st_i64:
4c22e840 3202 return C_O0_I2(re, r);
cd26449a
RH		 3203
3204 case INDEX_op_add_i32:
3205 case INDEX_op_add_i64:
4c22e840
RH		 3206 return C_O1_I2(r, r, re);
3207
cd26449a
RH		 3208 case INDEX_op_sub_i32:
3209 case INDEX_op_sub_i64:
3210 case INDEX_op_mul_i32:
3211 case INDEX_op_mul_i64:
3212 case INDEX_op_or_i32:
3213 case INDEX_op_or_i64:
3214 case INDEX_op_xor_i32:
3215 case INDEX_op_xor_i64:
4c22e840 3216 return C_O1_I2(r, 0, re);
cd26449a
RH		 3217
3218 case INDEX_op_and_i32:
3219 case INDEX_op_and_i64:
4c22e840
RH		 3220 return C_O1_I2(r, 0, reZ);
3221
cd26449a
RH		 3222 case INDEX_op_andc_i32:
3223 case INDEX_op_andc_i64:
4c22e840 3224 return C_O1_I2(r, r, rI);
bbc863bf 3225
cd26449a
RH		 3226 case INDEX_op_shl_i32:
3227 case INDEX_op_shl_i64:
3228 case INDEX_op_shr_i32:
3229 case INDEX_op_shr_i64:
3230 case INDEX_op_sar_i32:
3231 case INDEX_op_sar_i64:
4c22e840
RH		 3232 return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci);
3233
cd26449a
RH		 3234 case INDEX_op_rotl_i32:
3235 case INDEX_op_rotl_i64:
3236 case INDEX_op_rotr_i32:
3237 case INDEX_op_rotr_i64:
4c22e840 3238 return C_O1_I2(r, 0, ci);
a7d00d4e 3239
cd26449a
RH		 3240 case INDEX_op_brcond_i32:
3241 case INDEX_op_brcond_i64:
4c22e840 3242 return C_O0_I2(r, re);
1d2699ae 3243
cd26449a
RH		 3244 case INDEX_op_bswap16_i32:
3245 case INDEX_op_bswap16_i64:
3246 case INDEX_op_bswap32_i32:
3247 case INDEX_op_bswap32_i64:
3248 case INDEX_op_bswap64_i64:
3249 case INDEX_op_neg_i32:
3250 case INDEX_op_neg_i64:
3251 case INDEX_op_not_i32:
3252 case INDEX_op_not_i64:
75478279 3253 case INDEX_op_extrh_i64_i32:
4c22e840 3254 return C_O1_I1(r, 0);
cd26449a
RH		 3255
3256 case INDEX_op_ext8s_i32:
3257 case INDEX_op_ext8s_i64:
3258 case INDEX_op_ext8u_i32:
3259 case INDEX_op_ext8u_i64:
4c22e840
RH		 3260 return C_O1_I1(r, q);
3261
cd26449a
RH		 3262 case INDEX_op_ext16s_i32:
3263 case INDEX_op_ext16s_i64:
3264 case INDEX_op_ext16u_i32:
3265 case INDEX_op_ext16u_i64:
3266 case INDEX_op_ext32s_i64:
3267 case INDEX_op_ext32u_i64:
3268 case INDEX_op_ext_i32_i64:
3269 case INDEX_op_extu_i32_i64:
75478279 3270 case INDEX_op_extrl_i64_i32:
cd26449a
RH		 3271 case INDEX_op_extract_i32:
3272 case INDEX_op_extract_i64:
3273 case INDEX_op_sextract_i32:
993508e4
RH		 3274 case INDEX_op_ctpop_i32:
3275 case INDEX_op_ctpop_i64:
4c22e840
RH		 3276 return C_O1_I1(r, r);
3277
c6fb8c0c
RH		 3278 case INDEX_op_extract2_i32:
3279 case INDEX_op_extract2_i64:
4c22e840 3280 return C_O1_I2(r, 0, r);
cd26449a
RH		 3281
3282 case INDEX_op_deposit_i32:
3283 case INDEX_op_deposit_i64:
4c22e840
RH		 3284 return C_O1_I2(Q, 0, Q);
3285
cd26449a
RH		 3286 case INDEX_op_setcond_i32:
3287 case INDEX_op_setcond_i64:
4c22e840
RH		 3288 return C_O1_I2(q, r, re);
3289
cd26449a
RH		 3290 case INDEX_op_movcond_i32:
3291 case INDEX_op_movcond_i64:
4c22e840
RH		 3292 return C_O1_I4(r, r, re, r, 0);
3293
cd26449a
RH		 3294 case INDEX_op_div2_i32:
3295 case INDEX_op_div2_i64:
3296 case INDEX_op_divu2_i32:
3297 case INDEX_op_divu2_i64:
4c22e840
RH		 3298 return C_O2_I3(a, d, 0, 1, r);
3299
cd26449a
RH		 3300 case INDEX_op_mulu2_i32:
3301 case INDEX_op_mulu2_i64:
3302 case INDEX_op_muls2_i32:
3303 case INDEX_op_muls2_i64:
4c22e840
RH		 3304 return C_O2_I2(a, d, a, r);
3305
cd26449a
RH		 3306 case INDEX_op_add2_i32:
3307 case INDEX_op_add2_i64:
3308 case INDEX_op_sub2_i32:
3309 case INDEX_op_sub2_i64:
4c22e840
RH		 3310 return C_O2_I4(r, r, 0, 1, re, re);
3311
bbf25f90
RH		 3312 case INDEX_op_ctz_i32:
3313 case INDEX_op_ctz_i64:
4c22e840
RH		 3314 return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
3315
bbf25f90
RH		 3316 case INDEX_op_clz_i32:
3317 case INDEX_op_clz_i64:
4c22e840 3318 return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
c896fe29 3319
cd26449a 3320 case INDEX_op_qemu_ld_i32:
4c22e840
RH		 3321 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3322 ? C_O1_I1(r, L) : C_O1_I2(r, L, L));
3323
cd26449a 3324 case INDEX_op_qemu_st_i32:
4c22e840
RH		 3325 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3326 ? C_O0_I2(L, L) : C_O0_I3(L, L, L));
07ce0b05 3327 case INDEX_op_qemu_st8_i32:
4c22e840
RH		 3328 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3329 ? C_O0_I2(s, L) : C_O0_I3(s, L, L));
3330
cd26449a 3331 case INDEX_op_qemu_ld_i64:
4c22e840
RH		 3332 return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L)
3333 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L)
3334 : C_O2_I2(r, r, L, L));
3335
cd26449a 3336 case INDEX_op_qemu_st_i64:
4c22e840
RH		 3337 return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L)
3338 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L)
3339 : C_O0_I4(L, L, L, L));
f69d277e 3340
cd26449a 3341 case INDEX_op_brcond2_i32:
4c22e840
RH		 3342 return C_O0_I4(r, r, ri, ri);
3343
cd26449a 3344 case INDEX_op_setcond2_i32:
4c22e840 3345 return C_O1_I4(r, r, r, ri, ri);
cd26449a 3346
770c2fc7 3347 case INDEX_op_ld_vec:
37ee55a0 3348 case INDEX_op_dupm_vec:
4c22e840
RH		 3349 return C_O1_I1(x, r);
3350
3351 case INDEX_op_st_vec:
3352 return C_O0_I2(x, r);
770c2fc7
RH		 3353
3354 case INDEX_op_add_vec:
3355 case INDEX_op_sub_vec:
3356 case INDEX_op_mul_vec:
3357 case INDEX_op_and_vec:
3358 case INDEX_op_or_vec:
3359 case INDEX_op_xor_vec:
3360 case INDEX_op_andc_vec:
3143767b
RH		 3361 case INDEX_op_orc_vec:
3362 case INDEX_op_nand_vec:
3363 case INDEX_op_nor_vec:
3364 case INDEX_op_eqv_vec:
8ffafbce
RH		 3365 case INDEX_op_ssadd_vec:
3366 case INDEX_op_usadd_vec:
3367 case INDEX_op_sssub_vec:
3368 case INDEX_op_ussub_vec:
bc37faf4
RH		 3369 case INDEX_op_smin_vec:
3370 case INDEX_op_umin_vec:
3371 case INDEX_op_smax_vec:
3372 case INDEX_op_umax_vec:
a2ce146a
RH		 3373 case INDEX_op_shlv_vec:
3374 case INDEX_op_shrv_vec:
3375 case INDEX_op_sarv_vec:
102cd35c
RH		 3376 case INDEX_op_rotlv_vec:
3377 case INDEX_op_rotrv_vec:
0a8d7a3b
RH		 3378 case INDEX_op_shls_vec:
3379 case INDEX_op_shrs_vec:
3380 case INDEX_op_sars_vec:
770c2fc7
RH		 3381 case INDEX_op_cmp_vec:
3382 case INDEX_op_x86_shufps_vec:
3383 case INDEX_op_x86_blend_vec:
3384 case INDEX_op_x86_packss_vec:
3385 case INDEX_op_x86_packus_vec:
3386 case INDEX_op_x86_vperm2i128_vec:
3387 case INDEX_op_x86_punpckl_vec:
3388 case INDEX_op_x86_punpckh_vec:
965d5d06 3389 case INDEX_op_x86_vpshldi_vec:
7f34ed4b
RH		 3390#if TCG_TARGET_REG_BITS == 32
3391 case INDEX_op_dup2_vec:
3392#endif
4c22e840
RH		 3393 return C_O1_I2(x, x, x);
3394
18f9b65f 3395 case INDEX_op_abs_vec:
770c2fc7 3396 case INDEX_op_dup_vec:
3143767b 3397 case INDEX_op_not_vec:
770c2fc7
RH		 3398 case INDEX_op_shli_vec:
3399 case INDEX_op_shri_vec:
3400 case INDEX_op_sari_vec:
4e73f842 3401 case INDEX_op_rotli_vec:
770c2fc7 3402 case INDEX_op_x86_psrldq_vec:
4c22e840
RH		 3403 return C_O1_I1(x, x);
3404
965d5d06
RH		 3405 case INDEX_op_x86_vpshldv_vec:
3406 case INDEX_op_x86_vpshrdv_vec:
3407 return C_O1_I3(x, 0, x, x);
3408
cf320769 3409 case INDEX_op_bitsel_vec:
770c2fc7 3410 case INDEX_op_x86_vpblendvb_vec:
4c22e840 3411 return C_O1_I3(x, x, x, x);
770c2fc7 3412
cd26449a 3413 default:
4c22e840 3414 g_assert_not_reached();
f69d277e 3415 }
f69d277e
RH		 3416}
3417
770c2fc7
RH		 3418int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3419{
3420 switch (opc) {
3421 case INDEX_op_add_vec:
3422 case INDEX_op_sub_vec:
3423 case INDEX_op_and_vec:
3424 case INDEX_op_or_vec:
3425 case INDEX_op_xor_vec:
3426 case INDEX_op_andc_vec:
3143767b
RH		 3427 case INDEX_op_orc_vec:
3428 case INDEX_op_nand_vec:
3429 case INDEX_op_nor_vec:
3430 case INDEX_op_eqv_vec:
3431 case INDEX_op_not_vec:
cf320769 3432 case INDEX_op_bitsel_vec:
770c2fc7
RH		 3433 return 1;
3434 case INDEX_op_cmp_vec:
904c5e19 3435 case INDEX_op_cmpsel_vec:
770c2fc7
RH		 3436 return -1;
3437
4e73f842
RH		 3438 case INDEX_op_rotli_vec:
3439 return have_avx512vl && vece >= MO_32 ? 1 : -1;
3440
770c2fc7
RH		 3441 case INDEX_op_shli_vec:
3442 case INDEX_op_shri_vec:
3443 /* We must expand the operation for MO_8. */
3444 return vece == MO_8 ? -1 : 1;
3445
3446 case INDEX_op_sari_vec:
264e4182
RH		 3447 switch (vece) {
3448 case MO_8:
770c2fc7 3449 return -1;
264e4182
RH		 3450 case MO_16:
3451 case MO_32:
3452 return 1;
3453 case MO_64:
3454 if (have_avx512vl) {
3455 return 1;
3456 }
3457 /*
3458 * We can emulate this for MO_64, but it does not pay off
3459 * unless we're producing at least 4 values.
3460 */
770c2fc7
RH		 3461 return type >= TCG_TYPE_V256 ? -1 : 0;
3462 }
264e4182 3463 return 0;
770c2fc7 3464
0a8d7a3b
RH		 3465 case INDEX_op_shls_vec:
3466 case INDEX_op_shrs_vec:
3467 return vece >= MO_16;
3468 case INDEX_op_sars_vec:
47b331b2
RH		 3469 switch (vece) {
3470 case MO_16:
3471 case MO_32:
3472 return 1;
3473 case MO_64:
3474 return have_avx512vl;
3475 }
3476 return 0;
885b1706
RH		 3477 case INDEX_op_rotls_vec:
3478 return vece >= MO_16 ? -1 : 0;
0a8d7a3b 3479
a2ce146a
RH		 3480 case INDEX_op_shlv_vec:
3481 case INDEX_op_shrv_vec:
ef77ce0d
RH		 3482 switch (vece) {
3483 case MO_16:
3484 return have_avx512bw;
3485 case MO_32:
3486 case MO_64:
3487 return have_avx2;
3488 }
3489 return 0;
a2ce146a 3490 case INDEX_op_sarv_vec:
ef77ce0d
RH		 3491 switch (vece) {
3492 case MO_16:
3493 return have_avx512bw;
3494 case MO_32:
3495 return have_avx2;
3496 case MO_64:
3497 return have_avx512vl;
3498 }
3499 return 0;
885b1706
RH		 3500 case INDEX_op_rotlv_vec:
3501 case INDEX_op_rotrv_vec:
102cd35c 3502 switch (vece) {
786c7ef3
RH		 3503 case MO_16:
3504 return have_avx512vbmi2 ? -1 : 0;
102cd35c
RH		 3505 case MO_32:
3506 case MO_64:
3507 return have_avx512vl ? 1 : have_avx2 ? -1 : 0;
3508 }
3509 return 0;
a2ce146a 3510
770c2fc7 3511 case INDEX_op_mul_vec:
4c8b9686
RH		 3512 switch (vece) {
3513 case MO_8:
770c2fc7 3514 return -1;
4c8b9686
RH		 3515 case MO_64:
3516 return have_avx512dq;
770c2fc7
RH		 3517 }
3518 return 1;
3519
8ffafbce
RH		 3520 case INDEX_op_ssadd_vec:
3521 case INDEX_op_usadd_vec:
3522 case INDEX_op_sssub_vec:
3523 case INDEX_op_ussub_vec:
3524 return vece <= MO_16;
bc37faf4
RH		 3525 case INDEX_op_smin_vec:
3526 case INDEX_op_smax_vec:
3527 case INDEX_op_umin_vec:
3528 case INDEX_op_umax_vec:
18f9b65f 3529 case INDEX_op_abs_vec:
dac1648f 3530 return vece <= MO_32 || have_avx512vl;
8ffafbce 3531
770c2fc7
RH		 3532 default:
3533 return 0;
3534 }
3535}
3536
885b1706 3537static void expand_vec_shi(TCGType type, unsigned vece, TCGOpcode opc,
44f1441d
RH		 3538 TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3539{
3540 TCGv_vec t1, t2;
3541
3542 tcg_debug_assert(vece == MO_8);
3543
3544 t1 = tcg_temp_new_vec(type);
3545 t2 = tcg_temp_new_vec(type);
3546
885b1706
RH		 3547 /*
3548 * Unpack to W, shift, and repack. Tricky bits:
3549 * (1) Use punpck*bw x,x to produce DDCCBBAA,
3550 * i.e. duplicate in other half of the 16-bit lane.
3551 * (2) For right-shift, add 8 so that the high half of the lane
3552 * becomes zero. For left-shift, and left-rotate, we must
3553 * shift up and down again.
3554 * (3) Step 2 leaves high half zero such that PACKUSWB
3555 * (pack with unsigned saturation) does not modify
3556 * the quantity.
3557 */
44f1441d
RH		 3558 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
3559 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3560 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
3561 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3562
885b1706
RH		 3563 if (opc != INDEX_op_rotli_vec) {
3564 imm += 8;
3565 }
3566 if (opc == INDEX_op_shri_vec) {
3567 tcg_gen_shri_vec(MO_16, t1, t1, imm);
3568 tcg_gen_shri_vec(MO_16, t2, t2, imm);
44f1441d 3569 } else {
885b1706
RH		 3570 tcg_gen_shli_vec(MO_16, t1, t1, imm);
3571 tcg_gen_shli_vec(MO_16, t2, t2, imm);
44f1441d
RH		 3572 tcg_gen_shri_vec(MO_16, t1, t1, 8);
3573 tcg_gen_shri_vec(MO_16, t2, t2, 8);
3574 }
770c2fc7 3575
44f1441d
RH		 3576 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8,
3577 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2));
3578 tcg_temp_free_vec(t1);
3579 tcg_temp_free_vec(t2);
3580}
770c2fc7 3581
44f1441d
RH		 3582static void expand_vec_sari(TCGType type, unsigned vece,
3583 TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3584{
3585 TCGv_vec t1, t2;
3586
3587 switch (vece) {
3588 case MO_8:
3589 /* Unpack to W, shift, and repack, as in expand_vec_shi. */
770c2fc7
RH		 3590 t1 = tcg_temp_new_vec(type);
3591 t2 = tcg_temp_new_vec(type);
3592 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
44f1441d 3593 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
770c2fc7 3594 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
44f1441d
RH		 3595 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3596 tcg_gen_sari_vec(MO_16, t1, t1, imm + 8);
3597 tcg_gen_sari_vec(MO_16, t2, t2, imm + 8);
3598 vec_gen_3(INDEX_op_x86_packss_vec, type, MO_8,
3599 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2));
770c2fc7
RH		 3600 tcg_temp_free_vec(t1);
3601 tcg_temp_free_vec(t2);
3602 break;
3603
44f1441d
RH		 3604 case MO_64:
3605 if (imm <= 32) {
312b426f
RH		 3606 /*
3607 * We can emulate a small sign extend by performing an arithmetic
44f1441d 3608 * 32-bit shift and overwriting the high half of a 64-bit logical
312b426f
RH		 3609 * shift. Note that the ISA says shift of 32 is valid, but TCG
3610 * does not, so we have to bound the smaller shift -- we get the
3611 * same result in the high half either way.
44f1441d 3612 */
770c2fc7 3613 t1 = tcg_temp_new_vec(type);
312b426f 3614 tcg_gen_sari_vec(MO_32, t1, v1, MIN(imm, 31));
44f1441d 3615 tcg_gen_shri_vec(MO_64, v0, v1, imm);
770c2fc7 3616 vec_gen_4(INDEX_op_x86_blend_vec, type, MO_32,
44f1441d
RH		 3617 tcgv_vec_arg(v0), tcgv_vec_arg(v0),
3618 tcgv_vec_arg(t1), 0xaa);
3619 tcg_temp_free_vec(t1);
3620 } else {
3621 /* Otherwise we will need to use a compare vs 0 to produce
3622 * the sign-extend, shift and merge.
3623 */
3624 t1 = tcg_const_zeros_vec(type);
3625 tcg_gen_cmp_vec(TCG_COND_GT, MO_64, t1, t1, v1);
3626 tcg_gen_shri_vec(MO_64, v0, v1, imm);
3627 tcg_gen_shli_vec(MO_64, t1, t1, 64 - imm);
3628 tcg_gen_or_vec(MO_64, v0, v0, t1);
770c2fc7 3629 tcg_temp_free_vec(t1);
770c2fc7 3630 }
770c2fc7
RH		 3631 break;
3632
44f1441d
RH		 3633 default:
3634 g_assert_not_reached();
3635 }
3636}
770c2fc7 3637
885b1706
RH		 3638static void expand_vec_rotli(TCGType type, unsigned vece,
3639 TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3640{
3641 TCGv_vec t;
3642
3643 if (vece == MO_8) {
3644 expand_vec_shi(type, vece, INDEX_op_rotli_vec, v0, v1, imm);
3645 return;
3646 }
3647
786c7ef3
RH		 3648 if (have_avx512vbmi2) {
3649 vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece,
3650 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v1), imm);
3651 return;
3652 }
3653
885b1706
RH		 3654 t = tcg_temp_new_vec(type);
3655 tcg_gen_shli_vec(vece, t, v1, imm);
3656 tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm);
3657 tcg_gen_or_vec(vece, v0, v0, t);
3658 tcg_temp_free_vec(t);
3659}
3660
885b1706
RH		 3661static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0,
3662 TCGv_vec v1, TCGv_vec sh, bool right)
3663{
786c7ef3 3664 TCGv_vec t;
885b1706 3665
786c7ef3
RH		 3666 if (have_avx512vbmi2) {
3667 vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec,
3668 type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1),
3669 tcgv_vec_arg(v1), tcgv_vec_arg(sh));
3670 return;
3671 }
3672
3673 t = tcg_temp_new_vec(type);
885b1706
RH		 3674 tcg_gen_dupi_vec(vece, t, 8 << vece);
3675 tcg_gen_sub_vec(vece, t, t, sh);
3676 if (right) {
3677 tcg_gen_shlv_vec(vece, t, v1, t);
3678 tcg_gen_shrv_vec(vece, v0, v1, sh);
3679 } else {
3680 tcg_gen_shrv_vec(vece, t, v1, t);
3681 tcg_gen_shlv_vec(vece, v0, v1, sh);
3682 }
3683 tcg_gen_or_vec(vece, v0, v0, t);
3684 tcg_temp_free_vec(t);
3685}
3686
1d442e42
RH		 3687static void expand_vec_rotls(TCGType type, unsigned vece,
3688 TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh)
3689{
3690 TCGv_vec t = tcg_temp_new_vec(type);
3691
3692 tcg_debug_assert(vece != MO_8);
3693
3694 if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) {
3695 tcg_gen_dup_i32_vec(vece, t, lsh);
3696 if (vece >= MO_32) {
3697 tcg_gen_rotlv_vec(vece, v0, v1, t);
3698 } else {
3699 expand_vec_rotv(type, vece, v0, v1, t, false);
3700 }
3701 } else {
3702 TCGv_i32 rsh = tcg_temp_new_i32();
3703
3704 tcg_gen_neg_i32(rsh, lsh);
3705 tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1);
3706 tcg_gen_shls_vec(vece, t, v1, lsh);
3707 tcg_gen_shrs_vec(vece, v0, v1, rsh);
3708 tcg_gen_or_vec(vece, v0, v0, t);
3709
3710 tcg_temp_free_i32(rsh);
3711 }
3712
3713 tcg_temp_free_vec(t);
3714}
3715
44f1441d
RH		 3716static void expand_vec_mul(TCGType type, unsigned vece,
3717 TCGv_vec v0, TCGv_vec v1, TCGv_vec v2)
3718{
9739a052 3719 TCGv_vec t1, t2, t3, t4, zero;
770c2fc7 3720
44f1441d 3721 tcg_debug_assert(vece == MO_8);
770c2fc7 3722
44f1441d
RH		 3723 /*
3724 * Unpack v1 bytes to words, 0 | x.
3725 * Unpack v2 bytes to words, y | 0.
3726 * This leaves the 8-bit result, x * y, with 8 bits of right padding.
3727 * Shift logical right by 8 bits to clear the high 8 bytes before
3728 * using an unsigned saturated pack.
3729 *
3730 * The difference between the V64, V128 and V256 cases is merely how
3731 * we distribute the expansion between temporaries.
3732 */
3733 switch (type) {
3734 case TCG_TYPE_V64:
3735 t1 = tcg_temp_new_vec(TCG_TYPE_V128);
3736 t2 = tcg_temp_new_vec(TCG_TYPE_V128);
9739a052 3737 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
44f1441d 3738 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
9739a052 3739 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
44f1441d 3740 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
9739a052 3741 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
44f1441d
RH		 3742 tcg_gen_mul_vec(MO_16, t1, t1, t2);
3743 tcg_gen_shri_vec(MO_16, t1, t1, 8);
3744 vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8,
3745 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t1));
3746 tcg_temp_free_vec(t1);
3747 tcg_temp_free_vec(t2);
770c2fc7
RH		 3748 break;
3749
44f1441d
RH		 3750 case TCG_TYPE_V128:
3751 case TCG_TYPE_V256:
3752 t1 = tcg_temp_new_vec(type);
3753 t2 = tcg_temp_new_vec(type);
3754 t3 = tcg_temp_new_vec(type);
3755 t4 = tcg_temp_new_vec(type);
9739a052 3756 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
44f1441d 3757 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
9739a052 3758 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
44f1441d 3759 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
9739a052 3760 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
44f1441d 3761 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
9739a052 3762 tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
44f1441d 3763 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
9739a052 3764 tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
44f1441d
RH		 3765 tcg_gen_mul_vec(MO_16, t1, t1, t2);
3766 tcg_gen_mul_vec(MO_16, t3, t3, t4);
3767 tcg_gen_shri_vec(MO_16, t1, t1, 8);
3768 tcg_gen_shri_vec(MO_16, t3, t3, 8);
3769 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8,
3770 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t3));
3771 tcg_temp_free_vec(t1);
3772 tcg_temp_free_vec(t2);
3773 tcg_temp_free_vec(t3);
3774 tcg_temp_free_vec(t4);
3775 break;
770c2fc7 3776
44f1441d
RH		 3777 default:
3778 g_assert_not_reached();
3779 }
3780}
770c2fc7 3781
904c5e19
RH		 3782static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0,
3783 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
44f1441d
RH		 3784{
3785 enum {
ebcfb91a
RH		 3786 NEED_INV = 1,
3787 NEED_SWAP = 2,
3788 NEED_BIAS = 4,
3789 NEED_UMIN = 8,
3790 NEED_UMAX = 16,
44f1441d 3791 };
9739a052 3792 TCGv_vec t1, t2, t3;
44f1441d 3793 uint8_t fixup;
770c2fc7 3794
ebcfb91a
RH		 3795 switch (cond) {
3796 case TCG_COND_EQ:
3797 case TCG_COND_GT:
3798 fixup = 0;
3799 break;
3800 case TCG_COND_NE:
3801 case TCG_COND_LE:
3802 fixup = NEED_INV;
3803 break;
3804 case TCG_COND_LT:
3805 fixup = NEED_SWAP;
3806 break;
3807 case TCG_COND_GE:
3808 fixup = NEED_SWAP | NEED_INV;
3809 break;
3810 case TCG_COND_LEU:
54e2d650 3811 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) {
ebcfb91a
RH		 3812 fixup = NEED_UMIN;
3813 } else {
3814 fixup = NEED_BIAS | NEED_INV;
3815 }
3816 break;
3817 case TCG_COND_GTU:
54e2d650 3818 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) {
ebcfb91a
RH		 3819 fixup = NEED_UMIN | NEED_INV;
3820 } else {
3821 fixup = NEED_BIAS;
3822 }
3823 break;
3824 case TCG_COND_GEU:
54e2d650 3825 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) {
ebcfb91a
RH		 3826 fixup = NEED_UMAX;
3827 } else {
3828 fixup = NEED_BIAS | NEED_SWAP | NEED_INV;
3829 }
3830 break;
3831 case TCG_COND_LTU:
54e2d650 3832 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) {
ebcfb91a
RH		 3833 fixup = NEED_UMAX | NEED_INV;
3834 } else {
3835 fixup = NEED_BIAS | NEED_SWAP;
3836 }
3837 break;
3838 default:
3839 g_assert_not_reached();
3840 }
770c2fc7 3841
44f1441d
RH		 3842 if (fixup & NEED_INV) {
3843 cond = tcg_invert_cond(cond);
3844 }
3845 if (fixup & NEED_SWAP) {
3846 t1 = v1, v1 = v2, v2 = t1;
3847 cond = tcg_swap_cond(cond);
3848 }
3849
3850 t1 = t2 = NULL;
ebcfb91a
RH		 3851 if (fixup & (NEED_UMIN | NEED_UMAX)) {
3852 t1 = tcg_temp_new_vec(type);
3853 if (fixup & NEED_UMIN) {
3854 tcg_gen_umin_vec(vece, t1, v1, v2);
3855 } else {
3856 tcg_gen_umax_vec(vece, t1, v1, v2);
3857 }
3858 v2 = t1;
3859 cond = TCG_COND_EQ;
3860 } else if (fixup & NEED_BIAS) {
44f1441d
RH		 3861 t1 = tcg_temp_new_vec(type);
3862 t2 = tcg_temp_new_vec(type);
9739a052
RH		 3863 t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1));
3864 tcg_gen_sub_vec(vece, t1, v1, t3);
3865 tcg_gen_sub_vec(vece, t2, v2, t3);
44f1441d
RH		 3866 v1 = t1;
3867 v2 = t2;
3868 cond = tcg_signed_cond(cond);
3869 }
770c2fc7 3870
44f1441d
RH		 3871 tcg_debug_assert(cond == TCG_COND_EQ || cond == TCG_COND_GT);
3872 /* Expand directly; do not recurse. */
3873 vec_gen_4(INDEX_op_cmp_vec, type, vece,
3874 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
770c2fc7 3875
44f1441d
RH		 3876 if (t1) {
3877 tcg_temp_free_vec(t1);
3878 if (t2) {
3879 tcg_temp_free_vec(t2);
770c2fc7 3880 }
44f1441d 3881 }
904c5e19
RH		 3882 return fixup & NEED_INV;
3883}
3884
3885static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
3886 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
3887{
3888 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) {
44f1441d
RH		 3889 tcg_gen_not_vec(vece, v0, v0);
3890 }
3891}
3892
904c5e19
RH		 3893static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0,
3894 TCGv_vec c1, TCGv_vec c2,
3895 TCGv_vec v3, TCGv_vec v4, TCGCond cond)
3896{
3897 TCGv_vec t = tcg_temp_new_vec(type);
3898
3899 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) {
3900 /* Invert the sense of the compare by swapping arguments. */
3901 TCGv_vec x;
3902 x = v3, v3 = v4, v4 = x;
3903 }
3904 vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece,
3905 tcgv_vec_arg(v0), tcgv_vec_arg(v4),
3906 tcgv_vec_arg(v3), tcgv_vec_arg(t));
3907 tcg_temp_free_vec(t);
3908}
3909
44f1441d
RH		 3910void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3911 TCGArg a0, ...)
3912{
3913 va_list va;
3914 TCGArg a2;
904c5e19 3915 TCGv_vec v0, v1, v2, v3, v4;
44f1441d
RH		 3916
3917 va_start(va, a0);
3918 v0 = temp_tcgv_vec(arg_temp(a0));
3919 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3920 a2 = va_arg(va, TCGArg);
3921
3922 switch (opc) {
3923 case INDEX_op_shli_vec:
3924 case INDEX_op_shri_vec:
885b1706 3925 expand_vec_shi(type, vece, opc, v0, v1, a2);
44f1441d
RH		 3926 break;
3927
3928 case INDEX_op_sari_vec:
3929 expand_vec_sari(type, vece, v0, v1, a2);
3930 break;
3931
885b1706
RH		 3932 case INDEX_op_rotli_vec:
3933 expand_vec_rotli(type, vece, v0, v1, a2);
3934 break;
3935
3936 case INDEX_op_rotls_vec:
3937 expand_vec_rotls(type, vece, v0, v1, temp_tcgv_i32(arg_temp(a2)));
3938 break;
3939
3940 case INDEX_op_rotlv_vec:
3941 v2 = temp_tcgv_vec(arg_temp(a2));
3942 expand_vec_rotv(type, vece, v0, v1, v2, false);
3943 break;
3944 case INDEX_op_rotrv_vec:
3945 v2 = temp_tcgv_vec(arg_temp(a2));
3946 expand_vec_rotv(type, vece, v0, v1, v2, true);
3947 break;
3948
44f1441d
RH		 3949 case INDEX_op_mul_vec:
3950 v2 = temp_tcgv_vec(arg_temp(a2));
3951 expand_vec_mul(type, vece, v0, v1, v2);
3952 break;
3953
3954 case INDEX_op_cmp_vec:
3955 v2 = temp_tcgv_vec(arg_temp(a2));
3956 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
770c2fc7 3957 break;
bc37faf4 3958
904c5e19
RH		 3959 case INDEX_op_cmpsel_vec:
3960 v2 = temp_tcgv_vec(arg_temp(a2));
3961 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3962 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3963 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg));
3964 break;
770c2fc7
RH		 3965
3966 default:
3967 break;
3968 }
3969
3970 va_end(va);
3971}
3972
e268f4c0 3973static const int tcg_target_callee_save_regs[] = {
5d8a4f8f
RH		 3974#if TCG_TARGET_REG_BITS == 64
3975 TCG_REG_RBP,
3976 TCG_REG_RBX,
8d918718
SW		 3977#if defined(_WIN64)
3978 TCG_REG_RDI,
3979 TCG_REG_RSI,
3980#endif
5d8a4f8f
RH		 3981 TCG_REG_R12,
3982 TCG_REG_R13,
cea5f9a2 3983 TCG_REG_R14, /* Currently used for the global env. */
5d8a4f8f
RH		 3984 TCG_REG_R15,
3985#else
cea5f9a2 3986 TCG_REG_EBP, /* Currently used for the global env. */
b03cce8e
FB		 3987 TCG_REG_EBX,
3988 TCG_REG_ESI,
3989 TCG_REG_EDI,
5d8a4f8f 3990#endif
b03cce8e
FB		 3991};
3992
813da627
RH		 3993/* Compute frame size via macros, to share between tcg_target_qemu_prologue
3994 and tcg_register_jit. */
3995
3996#define PUSH_SIZE \
3997 ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \
3998 * (TCG_TARGET_REG_BITS / 8))
3999
4000#define FRAME_SIZE \
4001 ((PUSH_SIZE \
4002 + TCG_STATIC_CALL_ARGS_SIZE \
4003 + CPU_TEMP_BUF_NLONGS * sizeof(long) \
4004 + TCG_TARGET_STACK_ALIGN - 1) \
4005 & ~(TCG_TARGET_STACK_ALIGN - 1))
4006
b03cce8e 4007/* Generate global QEMU prologue and epilogue code */
e4d58b41 4008static void tcg_target_qemu_prologue(TCGContext *s)
b03cce8e 4009{
813da627 4010 int i, stack_addend;
78686523 4011
b03cce8e 4012 /* TB prologue */
5d8a4f8f 4013
ac0275dc 4014 /* Reserve some stack space, also for TCG temps. */
813da627 4015 stack_addend = FRAME_SIZE - PUSH_SIZE;
ac0275dc
BS		 4016 tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
4017 CPU_TEMP_BUF_NLONGS * sizeof(long));
4018
4019 /* Save all callee saved registers. */
4020 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
4021 tcg_out_push(s, tcg_target_callee_save_regs[i]);
4022 }
4023
6a18ae2d
BS		 4024#if TCG_TARGET_REG_BITS == 32
4025 tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
4026 (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
b18212c6
SW		 4027 tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
4028 /* jmp *tb. */
4029 tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
7d37435b
PB		 4030 (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
4031 + stack_addend);
6a18ae2d 4032#else
913c2bdd
RH		 4033# if !defined(CONFIG_SOFTMMU) && TCG_TARGET_REG_BITS == 64
4034 if (guest_base) {
4035 int seg = setup_guest_base_seg();
4036 if (seg != 0) {
4037 x86_guest_base_seg = seg;
4038 } else if (guest_base == (int32_t)guest_base) {
4039 x86_guest_base_offset = guest_base;
4040 } else {
4041 /* Choose R12 because, as a base, it requires a SIB byte. */
4042 x86_guest_base_index = TCG_REG_R12;
cce743ab 4043 tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base_index, guest_base);
913c2bdd
RH		 4044 tcg_regset_set_reg(s->reserved_regs, x86_guest_base_index);
4045 }
4046 }
4047# endif
cea5f9a2 4048 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
6a18ae2d 4049 tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
5d8a4f8f 4050 /* jmp *tb. */
cea5f9a2 4051 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
b18212c6 4052#endif
78686523 4053
5cb4ef80
EC		 4054 /*
4055 * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
4056 * and fall through to the rest of the epilogue.
4057 */
c8bc1168 4058 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
5cb4ef80
EC		 4059 tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0);
4060
b03cce8e 4061 /* TB epilogue */
705ed477 4062 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
5d8a4f8f 4063
e83c80f7 4064 tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
5d8a4f8f 4065
770c2fc7
RH		 4066 if (have_avx2) {
4067 tcg_out_vex_opc(s, OPC_VZEROUPPER, 0, 0, 0, 0);
4068 }
5d8a4f8f 4069 for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
b03cce8e
FB		 4070 tcg_out_pop(s, tcg_target_callee_save_regs[i]);
4071 }
5d8a4f8f 4072 tcg_out_opc(s, OPC_RET, 0, 0, 0);
b03cce8e
FB		 4073}
4074
4e45f239
RH		 4075static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
4076{
4077 memset(p, 0x90, count);
4078}
4079
e4d58b41 4080static void tcg_target_init(TCGContext *s)
c896fe29 4081{
774d566c 4082#ifdef CONFIG_CPUID_H
ba597b66 4083 unsigned a, b, c, d, b7 = 0, c7 = 0;
2a728de1 4084 unsigned max = __get_cpuid_max(0, 0);
085bb5bb 4085
770c2fc7
RH		 4086 if (max >= 7) {
4087 /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs. */
ba597b66 4088 __cpuid_count(7, 0, a, b7, c7, d);
770c2fc7
RH		 4089 have_bmi1 = (b7 & bit_BMI) != 0;
4090 have_bmi2 = (b7 & bit_BMI2) != 0;
4091 }
4092
9d2eec20
RH		 4093 if (max >= 1) {
4094 __cpuid(1, a, b, c, d);
4095#ifndef have_cmov
085bb5bb
AJ		 4096 /* For 32-bit, 99% certainty that we're running on hardware that
4097 supports cmov, but we still need to check. In case cmov is not
4098 available, we'll use a small forward branch. */
9d2eec20
RH		 4099 have_cmov = (d & bit_CMOV) != 0;
4100#endif
770c2fc7 4101
085bb5bb
AJ		 4102 /* MOVBE is only available on Intel Atom and Haswell CPUs, so we
4103 need to probe for it. */
9d2eec20 4104 have_movbe = (c & bit_MOVBE) != 0;
993508e4 4105 have_popcnt = (c & bit_POPCNT) != 0;
9d2eec20 4106
770c2fc7
RH		 4107 /* There are a number of things we must check before we can be
4108 sure of not hitting invalid opcode. */
4109 if (c & bit_OSXSAVE) {
4110 unsigned xcrl, xcrh;
1019242a
JA		 4111 /* The xgetbv instruction is not available to older versions of
4112 * the assembler, so we encode the instruction manually.
4113 */
4114 asm(".byte 0x0f, 0x01, 0xd0" : "=a" (xcrl), "=d" (xcrh) : "c" (0));
770c2fc7
RH		 4115 if ((xcrl & 6) == 6) {
4116 have_avx1 = (c & bit_AVX) != 0;
4117 have_avx2 = (b7 & bit_AVX2) != 0;
ba597b66
RH		 4118
4119 /*
4120 * There are interesting instructions in AVX512, so long
4121 * as we have AVX512VL, which indicates support for EVEX
4122 * on sizes smaller than 512 bits. We are required to
4123 * check that OPMASK and all extended ZMM state are enabled
4124 * even if we're not using them -- the insns will fault.
4125 */
4126 if ((xcrl & 0xe0) == 0xe0
4127 && (b7 & bit_AVX512F)
4128 && (b7 & bit_AVX512VL)) {
4129 have_avx512vl = true;
4130 have_avx512bw = (b7 & bit_AVX512BW) != 0;
4131 have_avx512dq = (b7 & bit_AVX512DQ) != 0;
4132 have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0;
4133 }
770c2fc7
RH		 4134 }
4135 }
9d2eec20 4136 }
76a347e1 4137
bbf25f90
RH		 4138 max = __get_cpuid_max(0x8000000, 0);
4139 if (max >= 1) {
4140 __cpuid(0x80000001, a, b, c, d);
4141 /* LZCNT was introduced with AMD Barcelona and Intel Haswell CPUs. */
4142 have_lzcnt = (c & bit_LZCNT) != 0;
4143 }
5dd89908 4144#endif /* CONFIG_CPUID_H */
bbf25f90 4145
770c2fc7 4146 tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
5d8a4f8f 4147 if (TCG_TARGET_REG_BITS == 64) {
770c2fc7
RH		 4148 tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
4149 }
4150 if (have_avx1) {
4151 tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
4152 tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
4153 }
4154 if (have_avx2) {
4155 tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS;
5d8a4f8f 4156 }
4ab50ccf 4157
672189cd 4158 tcg_target_call_clobber_regs = ALL_VECTOR_REGS;
4ab50ccf
RH		 4159 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
4160 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
4161 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
5d8a4f8f 4162 if (TCG_TARGET_REG_BITS == 64) {
8d918718 4163#if !defined(_WIN64)
5d8a4f8f
RH		 4164 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
4165 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
8d918718 4166#endif
5d8a4f8f
RH		 4167 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
4168 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
4169 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
4170 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
4171 }
4ab50ccf 4172
ccb1bb66 4173 s->reserved_regs = 0;
e83c80f7 4174 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
c896fe29 4175}
813da627 4176
813da627 4177typedef struct {
e9a9a5b6 4178 DebugFrameHeader h;
497a22eb
RH		 4179 uint8_t fde_def_cfa[4];
4180 uint8_t fde_reg_ofs[14];
813da627
RH		 4181} DebugFrame;
4182
b5cc476d
RH		 4183/* We're expecting a 2 byte uleb128 encoded value. */
4184QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
4185
c170cb66
SW		 4186#if !defined(__ELF__)
4187 /* Host machine without ELF. */
4188#elif TCG_TARGET_REG_BITS == 64
813da627 4189#define ELF_HOST_MACHINE EM_X86_64
e9a9a5b6
RH		 4190static const DebugFrame debug_frame = {
4191 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4192 .h.cie.id = -1,
4193 .h.cie.version = 1,
4194 .h.cie.code_align = 1,
4195 .h.cie.data_align = 0x78, /* sleb128 -8 */
4196 .h.cie.return_column = 16,
813da627 4197
497a22eb 4198 /* Total FDE size does not include the "len" member. */
e9a9a5b6 4199 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
497a22eb
RH		 4200
4201 .fde_def_cfa = {
813da627
RH		 4202 12, 7, /* DW_CFA_def_cfa %rsp, ... */
4203 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
4204 (FRAME_SIZE >> 7)
4205 },
497a22eb 4206 .fde_reg_ofs = {
813da627
RH		 4207 0x90, 1, /* DW_CFA_offset, %rip, -8 */
4208 /* The following ordering must match tcg_target_callee_save_regs. */
4209 0x86, 2, /* DW_CFA_offset, %rbp, -16 */
4210 0x83, 3, /* DW_CFA_offset, %rbx, -24 */
4211 0x8c, 4, /* DW_CFA_offset, %r12, -32 */
4212 0x8d, 5, /* DW_CFA_offset, %r13, -40 */
4213 0x8e, 6, /* DW_CFA_offset, %r14, -48 */
4214 0x8f, 7, /* DW_CFA_offset, %r15, -56 */
4215 }
4216};
4217#else
4218#define ELF_HOST_MACHINE EM_386
e9a9a5b6
RH		 4219static const DebugFrame debug_frame = {
4220 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4221 .h.cie.id = -1,
4222 .h.cie.version = 1,
4223 .h.cie.code_align = 1,
4224 .h.cie.data_align = 0x7c, /* sleb128 -4 */
4225 .h.cie.return_column = 8,
813da627 4226
497a22eb 4227 /* Total FDE size does not include the "len" member. */
e9a9a5b6 4228 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
497a22eb
RH		 4229
4230 .fde_def_cfa = {
813da627
RH		 4231 12, 4, /* DW_CFA_def_cfa %esp, ... */
4232 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
4233 (FRAME_SIZE >> 7)
4234 },
497a22eb 4235 .fde_reg_ofs = {
813da627
RH		 4236 0x88, 1, /* DW_CFA_offset, %eip, -4 */
4237 /* The following ordering must match tcg_target_callee_save_regs. */
4238 0x85, 2, /* DW_CFA_offset, %ebp, -8 */
4239 0x83, 3, /* DW_CFA_offset, %ebx, -12 */
4240 0x86, 4, /* DW_CFA_offset, %esi, -16 */
4241 0x87, 5, /* DW_CFA_offset, %edi, -20 */
4242 }
4243};
4244#endif
4245
c170cb66 4246#if defined(ELF_HOST_MACHINE)
755bf9e5 4247void tcg_register_jit(const void *buf, size_t buf_size)
813da627 4248{
813da627
RH		 4249 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
4250}
c170cb66 4251#endif
QEMU mirror