2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 static const char * const tcg_target_reg_names
[TCG_TARGET_NB_REGS
] = {
27 #if TCG_TARGET_REG_BITS == 64
28 "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
29 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
31 "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
36 static const int tcg_target_reg_alloc_order
[] = {
37 #if TCG_TARGET_REG_BITS == 64
64 static const int tcg_target_call_iarg_regs
[] = {
65 #if TCG_TARGET_REG_BITS == 64
78 /* 32 bit mode uses stack based calling convention (GCC default). */
82 static const int tcg_target_call_oarg_regs
[] = {
84 #if TCG_TARGET_REG_BITS == 32
89 /* Registers used with L constraint, which are the first argument
90 registers on x86_64, and two random call clobbered registers on
92 #if TCG_TARGET_REG_BITS == 64
93 # define TCG_REG_L0 tcg_target_call_iarg_regs[0]
94 # define TCG_REG_L1 tcg_target_call_iarg_regs[1]
96 # define TCG_REG_L0 TCG_REG_EAX
97 # define TCG_REG_L1 TCG_REG_EDX
100 /* For 32-bit, we are going to attempt to determine at runtime whether cmov
101 is available. However, the host compiler must supply <cpuid.h>, as we're
102 not going to go so far as our own inline assembly. */
103 #if TCG_TARGET_REG_BITS == 64
105 #elif defined(CONFIG_CPUID_H)
107 static bool have_cmov
;
112 static uint8_t *tb_ret_addr
;
114 static void patch_reloc(uint8_t *code_ptr
, int type
,
115 tcg_target_long value
, tcg_target_long addend
)
120 value
-= (uintptr_t)code_ptr
;
121 if (value
!= (int32_t)value
) {
124 *(uint32_t *)code_ptr
= value
;
127 value
-= (uintptr_t)code_ptr
;
128 if (value
!= (int8_t)value
) {
131 *(uint8_t *)code_ptr
= value
;
138 /* parse target specific constraints */
139 static int target_parse_constraint(TCGArgConstraint
*ct
, const char **pct_str
)
146 ct
->ct
|= TCG_CT_REG
;
147 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_EAX
);
150 ct
->ct
|= TCG_CT_REG
;
151 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_EBX
);
154 ct
->ct
|= TCG_CT_REG
;
155 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_ECX
);
158 ct
->ct
|= TCG_CT_REG
;
159 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_EDX
);
162 ct
->ct
|= TCG_CT_REG
;
163 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_ESI
);
166 ct
->ct
|= TCG_CT_REG
;
167 tcg_regset_set_reg(ct
->u
.regs
, TCG_REG_EDI
);
170 ct
->ct
|= TCG_CT_REG
;
171 if (TCG_TARGET_REG_BITS
== 64) {
172 tcg_regset_set32(ct
->u
.regs
, 0, 0xffff);
174 tcg_regset_set32(ct
->u
.regs
, 0, 0xf);
178 ct
->ct
|= TCG_CT_REG
;
179 tcg_regset_set32(ct
->u
.regs
, 0, 0xf);
182 ct
->ct
|= TCG_CT_REG
;
183 if (TCG_TARGET_REG_BITS
== 64) {
184 tcg_regset_set32(ct
->u
.regs
, 0, 0xffff);
186 tcg_regset_set32(ct
->u
.regs
, 0, 0xff);
190 /* qemu_ld/st address constraint */
192 ct
->ct
|= TCG_CT_REG
;
193 if (TCG_TARGET_REG_BITS
== 64) {
194 tcg_regset_set32(ct
->u
.regs
, 0, 0xffff);
196 tcg_regset_set32(ct
->u
.regs
, 0, 0xff);
198 tcg_regset_reset_reg(ct
->u
.regs
, TCG_REG_L0
);
199 tcg_regset_reset_reg(ct
->u
.regs
, TCG_REG_L1
);
203 ct
->ct
|= TCG_CT_CONST_S32
;
206 ct
->ct
|= TCG_CT_CONST_U32
;
217 /* test if a constant matches the constraint */
218 static inline int tcg_target_const_match(tcg_target_long val
,
219 const TCGArgConstraint
*arg_ct
)
222 if (ct
& TCG_CT_CONST
) {
225 if ((ct
& TCG_CT_CONST_S32
) && val
== (int32_t)val
) {
228 if ((ct
& TCG_CT_CONST_U32
) && val
== (uint32_t)val
) {
234 #if TCG_TARGET_REG_BITS == 64
235 # define LOWREGMASK(x) ((x) & 7)
237 # define LOWREGMASK(x) (x)
240 #define P_EXT 0x100 /* 0x0f opcode prefix */
241 #define P_DATA16 0x200 /* 0x66 opcode prefix */
242 #if TCG_TARGET_REG_BITS == 64
243 # define P_ADDR32 0x400 /* 0x67 opcode prefix */
244 # define P_REXW 0x800 /* Set REX.W = 1 */
245 # define P_REXB_R 0x1000 /* REG field as byte register */
246 # define P_REXB_RM 0x2000 /* R/M field as byte register */
247 # define P_GS 0x4000 /* gs segment override */
256 #define OPC_ARITH_EvIz (0x81)
257 #define OPC_ARITH_EvIb (0x83)
258 #define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */
259 #define OPC_ADD_GvEv (OPC_ARITH_GvEv | (ARITH_ADD << 3))
260 #define OPC_BSWAP (0xc8 | P_EXT)
261 #define OPC_CALL_Jz (0xe8)
262 #define OPC_CMOVCC (0x40 | P_EXT) /* ... plus condition code */
263 #define OPC_CMP_GvEv (OPC_ARITH_GvEv | (ARITH_CMP << 3))
264 #define OPC_DEC_r32 (0x48)
265 #define OPC_IMUL_GvEv (0xaf | P_EXT)
266 #define OPC_IMUL_GvEvIb (0x6b)
267 #define OPC_IMUL_GvEvIz (0x69)
268 #define OPC_INC_r32 (0x40)
269 #define OPC_JCC_long (0x80 | P_EXT) /* ... plus condition code */
270 #define OPC_JCC_short (0x70) /* ... plus condition code */
271 #define OPC_JMP_long (0xe9)
272 #define OPC_JMP_short (0xeb)
273 #define OPC_LEA (0x8d)
274 #define OPC_MOVB_EvGv (0x88) /* stores, more or less */
275 #define OPC_MOVL_EvGv (0x89) /* stores, more or less */
276 #define OPC_MOVL_GvEv (0x8b) /* loads, more or less */
277 #define OPC_MOVB_EvIz (0xc6)
278 #define OPC_MOVL_EvIz (0xc7)
279 #define OPC_MOVL_Iv (0xb8)
280 #define OPC_MOVSBL (0xbe | P_EXT)
281 #define OPC_MOVSWL (0xbf | P_EXT)
282 #define OPC_MOVSLQ (0x63 | P_REXW)
283 #define OPC_MOVZBL (0xb6 | P_EXT)
284 #define OPC_MOVZWL (0xb7 | P_EXT)
285 #define OPC_POP_r32 (0x58)
286 #define OPC_PUSH_r32 (0x50)
287 #define OPC_PUSH_Iv (0x68)
288 #define OPC_PUSH_Ib (0x6a)
289 #define OPC_RET (0xc3)
290 #define OPC_SETCC (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
291 #define OPC_SHIFT_1 (0xd1)
292 #define OPC_SHIFT_Ib (0xc1)
293 #define OPC_SHIFT_cl (0xd3)
294 #define OPC_TESTL (0x85)
295 #define OPC_XCHG_ax_r32 (0x90)
297 #define OPC_GRP3_Ev (0xf7)
298 #define OPC_GRP5 (0xff)
300 /* Group 1 opcode extensions for 0x80-0x83.
301 These are also used as modifiers for OPC_ARITH. */
311 /* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3. */
318 /* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */
326 /* Group 5 opcode extensions for 0xff. To be used with OPC_GRP5. */
327 #define EXT5_INC_Ev 0
328 #define EXT5_DEC_Ev 1
329 #define EXT5_CALLN_Ev 2
330 #define EXT5_JMPN_Ev 4
332 /* Condition codes to be added to OPC_JCC_{long,short}. */
351 static const uint8_t tcg_cond_to_jcc
[] = {
352 [TCG_COND_EQ
] = JCC_JE
,
353 [TCG_COND_NE
] = JCC_JNE
,
354 [TCG_COND_LT
] = JCC_JL
,
355 [TCG_COND_GE
] = JCC_JGE
,
356 [TCG_COND_LE
] = JCC_JLE
,
357 [TCG_COND_GT
] = JCC_JG
,
358 [TCG_COND_LTU
] = JCC_JB
,
359 [TCG_COND_GEU
] = JCC_JAE
,
360 [TCG_COND_LEU
] = JCC_JBE
,
361 [TCG_COND_GTU
] = JCC_JA
,
364 #if TCG_TARGET_REG_BITS == 64
365 static void tcg_out_opc(TCGContext
*s
, int opc
, int r
, int rm
, int x
)
372 if (opc
& P_DATA16
) {
373 /* We should never be asking for both 16 and 64-bit operation. */
374 assert((opc
& P_REXW
) == 0);
377 if (opc
& P_ADDR32
) {
382 rex
|= (opc
& P_REXW
) >> 8; /* REX.W */
383 rex
|= (r
& 8) >> 1; /* REX.R */
384 rex
|= (x
& 8) >> 2; /* REX.X */
385 rex
|= (rm
& 8) >> 3; /* REX.B */
387 /* P_REXB_{R,RM} indicates that the given register is the low byte.
388 For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
389 as otherwise the encoding indicates %[abcd]h. Note that the values
390 that are ORed in merely indicate that the REX byte must be present;
391 those bits get discarded in output. */
392 rex
|= opc
& (r
>= 4 ? P_REXB_R
: 0);
393 rex
|= opc
& (rm
>= 4 ? P_REXB_RM
: 0);
396 tcg_out8(s
, (uint8_t)(rex
| 0x40));
405 static void tcg_out_opc(TCGContext
*s
, int opc
)
407 if (opc
& P_DATA16
) {
415 /* Discard the register arguments to tcg_out_opc early, so as not to penalize
416 the 32-bit compilation paths. This method works with all versions of gcc,
417 whereas relying on optimization may not be able to exclude them. */
418 #define tcg_out_opc(s, opc, r, rm, x) (tcg_out_opc)(s, opc)
421 static void tcg_out_modrm(TCGContext
*s
, int opc
, int r
, int rm
)
423 tcg_out_opc(s
, opc
, r
, rm
, 0);
424 tcg_out8(s
, 0xc0 | (LOWREGMASK(r
) << 3) | LOWREGMASK(rm
));
427 /* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
428 We handle either RM and INDEX missing with a negative value. In 64-bit
429 mode for absolute addresses, ~RM is the size of the immediate operand
430 that will follow the instruction. */
432 static void tcg_out_modrm_sib_offset(TCGContext
*s
, int opc
, int r
, int rm
,
433 int index
, int shift
,
434 tcg_target_long offset
)
438 if (index
< 0 && rm
< 0) {
439 if (TCG_TARGET_REG_BITS
== 64) {
440 /* Try for a rip-relative addressing mode. This has replaced
441 the 32-bit-mode absolute addressing encoding. */
442 tcg_target_long pc
= (tcg_target_long
)s
->code_ptr
+ 5 + ~rm
;
443 tcg_target_long disp
= offset
- pc
;
444 if (disp
== (int32_t)disp
) {
445 tcg_out_opc(s
, opc
, r
, 0, 0);
446 tcg_out8(s
, (LOWREGMASK(r
) << 3) | 5);
451 /* Try for an absolute address encoding. This requires the
452 use of the MODRM+SIB encoding and is therefore larger than
453 rip-relative addressing. */
454 if (offset
== (int32_t)offset
) {
455 tcg_out_opc(s
, opc
, r
, 0, 0);
456 tcg_out8(s
, (LOWREGMASK(r
) << 3) | 4);
457 tcg_out8(s
, (4 << 3) | 5);
458 tcg_out32(s
, offset
);
462 /* ??? The memory isn't directly addressable. */
465 /* Absolute address. */
466 tcg_out_opc(s
, opc
, r
, 0, 0);
467 tcg_out8(s
, (r
<< 3) | 5);
468 tcg_out32(s
, offset
);
473 /* Find the length of the immediate addend. Note that the encoding
474 that would be used for (%ebp) indicates absolute addressing. */
476 mod
= 0, len
= 4, rm
= 5;
477 } else if (offset
== 0 && LOWREGMASK(rm
) != TCG_REG_EBP
) {
479 } else if (offset
== (int8_t)offset
) {
485 /* Use a single byte MODRM format if possible. Note that the encoding
486 that would be used for %esp is the escape to the two byte form. */
487 if (index
< 0 && LOWREGMASK(rm
) != TCG_REG_ESP
) {
488 /* Single byte MODRM format. */
489 tcg_out_opc(s
, opc
, r
, rm
, 0);
490 tcg_out8(s
, mod
| (LOWREGMASK(r
) << 3) | LOWREGMASK(rm
));
492 /* Two byte MODRM+SIB format. */
494 /* Note that the encoding that would place %esp into the index
495 field indicates no index register. In 64-bit mode, the REX.X
496 bit counts, so %r12 can be used as the index. */
500 assert(index
!= TCG_REG_ESP
);
503 tcg_out_opc(s
, opc
, r
, rm
, index
);
504 tcg_out8(s
, mod
| (LOWREGMASK(r
) << 3) | 4);
505 tcg_out8(s
, (shift
<< 6) | (LOWREGMASK(index
) << 3) | LOWREGMASK(rm
));
510 } else if (len
== 4) {
511 tcg_out32(s
, offset
);
515 /* A simplification of the above with no index or shift. */
516 static inline void tcg_out_modrm_offset(TCGContext
*s
, int opc
, int r
,
517 int rm
, tcg_target_long offset
)
519 tcg_out_modrm_sib_offset(s
, opc
, r
, rm
, -1, 0, offset
);
522 /* Generate dest op= src. Uses the same ARITH_* codes as tgen_arithi. */
523 static inline void tgen_arithr(TCGContext
*s
, int subop
, int dest
, int src
)
525 /* Propagate an opcode prefix, such as P_REXW. */
526 int ext
= subop
& ~0x7;
529 tcg_out_modrm(s
, OPC_ARITH_GvEv
+ (subop
<< 3) + ext
, dest
, src
);
532 static inline void tcg_out_mov(TCGContext
*s
, TCGType type
,
533 TCGReg ret
, TCGReg arg
)
536 int opc
= OPC_MOVL_GvEv
+ (type
== TCG_TYPE_I64
? P_REXW
: 0);
537 tcg_out_modrm(s
, opc
, ret
, arg
);
541 static void tcg_out_movi(TCGContext
*s
, TCGType type
,
542 TCGReg ret
, tcg_target_long arg
)
544 tcg_target_long diff
;
547 tgen_arithr(s
, ARITH_XOR
, ret
, ret
);
550 if (arg
== (uint32_t)arg
|| type
== TCG_TYPE_I32
) {
551 tcg_out_opc(s
, OPC_MOVL_Iv
+ LOWREGMASK(ret
), 0, ret
, 0);
555 if (arg
== (int32_t)arg
) {
556 tcg_out_modrm(s
, OPC_MOVL_EvIz
+ P_REXW
, 0, ret
);
561 /* Try a 7 byte pc-relative lea before the 10 byte movq. */
562 diff
= arg
- ((tcg_target_long
)s
->code_ptr
+ 7);
563 if (diff
== (int32_t)diff
) {
564 tcg_out_opc(s
, OPC_LEA
| P_REXW
, ret
, 0, 0);
565 tcg_out8(s
, (LOWREGMASK(ret
) << 3) | 5);
570 tcg_out_opc(s
, OPC_MOVL_Iv
+ P_REXW
+ LOWREGMASK(ret
), 0, ret
, 0);
574 static inline void tcg_out_pushi(TCGContext
*s
, tcg_target_long val
)
576 if (val
== (int8_t)val
) {
577 tcg_out_opc(s
, OPC_PUSH_Ib
, 0, 0, 0);
579 } else if (val
== (int32_t)val
) {
580 tcg_out_opc(s
, OPC_PUSH_Iv
, 0, 0, 0);
587 static inline void tcg_out_push(TCGContext
*s
, int reg
)
589 tcg_out_opc(s
, OPC_PUSH_r32
+ LOWREGMASK(reg
), 0, reg
, 0);
592 static inline void tcg_out_pop(TCGContext
*s
, int reg
)
594 tcg_out_opc(s
, OPC_POP_r32
+ LOWREGMASK(reg
), 0, reg
, 0);
597 static inline void tcg_out_ld(TCGContext
*s
, TCGType type
, TCGReg ret
,
598 TCGReg arg1
, tcg_target_long arg2
)
600 int opc
= OPC_MOVL_GvEv
+ (type
== TCG_TYPE_I64
? P_REXW
: 0);
601 tcg_out_modrm_offset(s
, opc
, ret
, arg1
, arg2
);
604 static inline void tcg_out_st(TCGContext
*s
, TCGType type
, TCGReg arg
,
605 TCGReg arg1
, tcg_target_long arg2
)
607 int opc
= OPC_MOVL_EvGv
+ (type
== TCG_TYPE_I64
? P_REXW
: 0);
608 tcg_out_modrm_offset(s
, opc
, arg
, arg1
, arg2
);
611 static inline void tcg_out_sti(TCGContext
*s
, TCGType type
, TCGReg base
,
612 tcg_target_long ofs
, tcg_target_long val
)
614 int opc
= OPC_MOVL_EvIz
+ (type
== TCG_TYPE_I64
? P_REXW
: 0);
615 tcg_out_modrm_offset(s
, opc
, 0, base
, ofs
);
619 static void tcg_out_shifti(TCGContext
*s
, int subopc
, int reg
, int count
)
621 /* Propagate an opcode prefix, such as P_DATA16. */
622 int ext
= subopc
& ~0x7;
626 tcg_out_modrm(s
, OPC_SHIFT_1
+ ext
, subopc
, reg
);
628 tcg_out_modrm(s
, OPC_SHIFT_Ib
+ ext
, subopc
, reg
);
633 static inline void tcg_out_bswap32(TCGContext
*s
, int reg
)
635 tcg_out_opc(s
, OPC_BSWAP
+ LOWREGMASK(reg
), 0, reg
, 0);
638 static inline void tcg_out_rolw_8(TCGContext
*s
, int reg
)
640 tcg_out_shifti(s
, SHIFT_ROL
+ P_DATA16
, reg
, 8);
643 static inline void tcg_out_ext8u(TCGContext
*s
, int dest
, int src
)
646 assert(src
< 4 || TCG_TARGET_REG_BITS
== 64);
647 tcg_out_modrm(s
, OPC_MOVZBL
+ P_REXB_RM
, dest
, src
);
650 static void tcg_out_ext8s(TCGContext
*s
, int dest
, int src
, int rexw
)
653 assert(src
< 4 || TCG_TARGET_REG_BITS
== 64);
654 tcg_out_modrm(s
, OPC_MOVSBL
+ P_REXB_RM
+ rexw
, dest
, src
);
657 static inline void tcg_out_ext16u(TCGContext
*s
, int dest
, int src
)
660 tcg_out_modrm(s
, OPC_MOVZWL
, dest
, src
);
663 static inline void tcg_out_ext16s(TCGContext
*s
, int dest
, int src
, int rexw
)
666 tcg_out_modrm(s
, OPC_MOVSWL
+ rexw
, dest
, src
);
669 static inline void tcg_out_ext32u(TCGContext
*s
, int dest
, int src
)
671 /* 32-bit mov zero extends. */
672 tcg_out_modrm(s
, OPC_MOVL_GvEv
, dest
, src
);
675 static inline void tcg_out_ext32s(TCGContext
*s
, int dest
, int src
)
677 tcg_out_modrm(s
, OPC_MOVSLQ
, dest
, src
);
680 static inline void tcg_out_bswap64(TCGContext
*s
, int reg
)
682 tcg_out_opc(s
, OPC_BSWAP
+ P_REXW
+ LOWREGMASK(reg
), 0, reg
, 0);
685 static void tgen_arithi(TCGContext
*s
, int c
, int r0
,
686 tcg_target_long val
, int cf
)
690 if (TCG_TARGET_REG_BITS
== 64) {
695 /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
696 partial flags update stalls on Pentium4 and are not recommended
697 by current Intel optimization manuals. */
698 if (!cf
&& (c
== ARITH_ADD
|| c
== ARITH_SUB
) && (val
== 1 || val
== -1)) {
699 int is_inc
= (c
== ARITH_ADD
) ^ (val
< 0);
700 if (TCG_TARGET_REG_BITS
== 64) {
701 /* The single-byte increment encodings are re-tasked as the
702 REX prefixes. Use the MODRM encoding. */
703 tcg_out_modrm(s
, OPC_GRP5
+ rexw
,
704 (is_inc
? EXT5_INC_Ev
: EXT5_DEC_Ev
), r0
);
706 tcg_out8(s
, (is_inc
? OPC_INC_r32
: OPC_DEC_r32
) + r0
);
711 if (c
== ARITH_AND
) {
712 if (TCG_TARGET_REG_BITS
== 64) {
713 if (val
== 0xffffffffu
) {
714 tcg_out_ext32u(s
, r0
, r0
);
717 if (val
== (uint32_t)val
) {
718 /* AND with no high bits set can use a 32-bit operation. */
722 if (val
== 0xffu
&& (r0
< 4 || TCG_TARGET_REG_BITS
== 64)) {
723 tcg_out_ext8u(s
, r0
, r0
);
726 if (val
== 0xffffu
) {
727 tcg_out_ext16u(s
, r0
, r0
);
732 if (val
== (int8_t)val
) {
733 tcg_out_modrm(s
, OPC_ARITH_EvIb
+ rexw
, c
, r0
);
737 if (rexw
== 0 || val
== (int32_t)val
) {
738 tcg_out_modrm(s
, OPC_ARITH_EvIz
+ rexw
, c
, r0
);
746 static void tcg_out_addi(TCGContext
*s
, int reg
, tcg_target_long val
)
749 tgen_arithi(s
, ARITH_ADD
+ P_REXW
, reg
, val
, 0);
753 /* Use SMALL != 0 to force a short forward branch. */
754 static void tcg_out_jxx(TCGContext
*s
, int opc
, int label_index
, int small
)
757 TCGLabel
*l
= &s
->labels
[label_index
];
760 val
= l
->u
.value
- (tcg_target_long
)s
->code_ptr
;
762 if ((int8_t)val1
== val1
) {
764 tcg_out8(s
, OPC_JMP_short
);
766 tcg_out8(s
, OPC_JCC_short
+ opc
);
774 tcg_out8(s
, OPC_JMP_long
);
775 tcg_out32(s
, val
- 5);
777 tcg_out_opc(s
, OPC_JCC_long
+ opc
, 0, 0, 0);
778 tcg_out32(s
, val
- 6);
783 tcg_out8(s
, OPC_JMP_short
);
785 tcg_out8(s
, OPC_JCC_short
+ opc
);
787 tcg_out_reloc(s
, s
->code_ptr
, R_386_PC8
, label_index
, -1);
791 tcg_out8(s
, OPC_JMP_long
);
793 tcg_out_opc(s
, OPC_JCC_long
+ opc
, 0, 0, 0);
795 tcg_out_reloc(s
, s
->code_ptr
, R_386_PC32
, label_index
, -4);
800 static void tcg_out_cmp(TCGContext
*s
, TCGArg arg1
, TCGArg arg2
,
801 int const_arg2
, int rexw
)
806 tcg_out_modrm(s
, OPC_TESTL
+ rexw
, arg1
, arg1
);
808 tgen_arithi(s
, ARITH_CMP
+ rexw
, arg1
, arg2
, 0);
811 tgen_arithr(s
, ARITH_CMP
+ rexw
, arg1
, arg2
);
815 static void tcg_out_brcond32(TCGContext
*s
, TCGCond cond
,
816 TCGArg arg1
, TCGArg arg2
, int const_arg2
,
817 int label_index
, int small
)
819 tcg_out_cmp(s
, arg1
, arg2
, const_arg2
, 0);
820 tcg_out_jxx(s
, tcg_cond_to_jcc
[cond
], label_index
, small
);
823 #if TCG_TARGET_REG_BITS == 64
824 static void tcg_out_brcond64(TCGContext
*s
, TCGCond cond
,
825 TCGArg arg1
, TCGArg arg2
, int const_arg2
,
826 int label_index
, int small
)
828 tcg_out_cmp(s
, arg1
, arg2
, const_arg2
, P_REXW
);
829 tcg_out_jxx(s
, tcg_cond_to_jcc
[cond
], label_index
, small
);
832 /* XXX: we implement it at the target level to avoid having to
833 handle cross basic blocks temporaries */
834 static void tcg_out_brcond2(TCGContext
*s
, const TCGArg
*args
,
835 const int *const_args
, int small
)
838 label_next
= gen_new_label();
841 tcg_out_brcond32(s
, TCG_COND_NE
, args
[0], args
[2], const_args
[2],
843 tcg_out_brcond32(s
, TCG_COND_EQ
, args
[1], args
[3], const_args
[3],
847 tcg_out_brcond32(s
, TCG_COND_NE
, args
[0], args
[2], const_args
[2],
849 tcg_out_brcond32(s
, TCG_COND_NE
, args
[1], args
[3], const_args
[3],
853 tcg_out_brcond32(s
, TCG_COND_LT
, args
[1], args
[3], const_args
[3],
855 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
856 tcg_out_brcond32(s
, TCG_COND_LTU
, args
[0], args
[2], const_args
[2],
860 tcg_out_brcond32(s
, TCG_COND_LT
, args
[1], args
[3], const_args
[3],
862 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
863 tcg_out_brcond32(s
, TCG_COND_LEU
, args
[0], args
[2], const_args
[2],
867 tcg_out_brcond32(s
, TCG_COND_GT
, args
[1], args
[3], const_args
[3],
869 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
870 tcg_out_brcond32(s
, TCG_COND_GTU
, args
[0], args
[2], const_args
[2],
874 tcg_out_brcond32(s
, TCG_COND_GT
, args
[1], args
[3], const_args
[3],
876 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
877 tcg_out_brcond32(s
, TCG_COND_GEU
, args
[0], args
[2], const_args
[2],
881 tcg_out_brcond32(s
, TCG_COND_LTU
, args
[1], args
[3], const_args
[3],
883 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
884 tcg_out_brcond32(s
, TCG_COND_LTU
, args
[0], args
[2], const_args
[2],
888 tcg_out_brcond32(s
, TCG_COND_LTU
, args
[1], args
[3], const_args
[3],
890 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
891 tcg_out_brcond32(s
, TCG_COND_LEU
, args
[0], args
[2], const_args
[2],
895 tcg_out_brcond32(s
, TCG_COND_GTU
, args
[1], args
[3], const_args
[3],
897 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
898 tcg_out_brcond32(s
, TCG_COND_GTU
, args
[0], args
[2], const_args
[2],
902 tcg_out_brcond32(s
, TCG_COND_GTU
, args
[1], args
[3], const_args
[3],
904 tcg_out_jxx(s
, JCC_JNE
, label_next
, 1);
905 tcg_out_brcond32(s
, TCG_COND_GEU
, args
[0], args
[2], const_args
[2],
911 tcg_out_label(s
, label_next
, s
->code_ptr
);
915 static void tcg_out_setcond32(TCGContext
*s
, TCGCond cond
, TCGArg dest
,
916 TCGArg arg1
, TCGArg arg2
, int const_arg2
)
918 tcg_out_cmp(s
, arg1
, arg2
, const_arg2
, 0);
919 tcg_out_modrm(s
, OPC_SETCC
| tcg_cond_to_jcc
[cond
], 0, dest
);
920 tcg_out_ext8u(s
, dest
, dest
);
923 #if TCG_TARGET_REG_BITS == 64
924 static void tcg_out_setcond64(TCGContext
*s
, TCGCond cond
, TCGArg dest
,
925 TCGArg arg1
, TCGArg arg2
, int const_arg2
)
927 tcg_out_cmp(s
, arg1
, arg2
, const_arg2
, P_REXW
);
928 tcg_out_modrm(s
, OPC_SETCC
| tcg_cond_to_jcc
[cond
], 0, dest
);
929 tcg_out_ext8u(s
, dest
, dest
);
932 static void tcg_out_setcond2(TCGContext
*s
, const TCGArg
*args
,
933 const int *const_args
)
936 int label_true
, label_over
;
938 memcpy(new_args
, args
+1, 5*sizeof(TCGArg
));
940 if (args
[0] == args
[1] || args
[0] == args
[2]
941 || (!const_args
[3] && args
[0] == args
[3])
942 || (!const_args
[4] && args
[0] == args
[4])) {
943 /* When the destination overlaps with one of the argument
944 registers, don't do anything tricky. */
945 label_true
= gen_new_label();
946 label_over
= gen_new_label();
948 new_args
[5] = label_true
;
949 tcg_out_brcond2(s
, new_args
, const_args
+1, 1);
951 tcg_out_movi(s
, TCG_TYPE_I32
, args
[0], 0);
952 tcg_out_jxx(s
, JCC_JMP
, label_over
, 1);
953 tcg_out_label(s
, label_true
, s
->code_ptr
);
955 tcg_out_movi(s
, TCG_TYPE_I32
, args
[0], 1);
956 tcg_out_label(s
, label_over
, s
->code_ptr
);
958 /* When the destination does not overlap one of the arguments,
959 clear the destination first, jump if cond false, and emit an
960 increment in the true case. This results in smaller code. */
962 tcg_out_movi(s
, TCG_TYPE_I32
, args
[0], 0);
964 label_over
= gen_new_label();
965 new_args
[4] = tcg_invert_cond(new_args
[4]);
966 new_args
[5] = label_over
;
967 tcg_out_brcond2(s
, new_args
, const_args
+1, 1);
969 tgen_arithi(s
, ARITH_ADD
, args
[0], 1, 0);
970 tcg_out_label(s
, label_over
, s
->code_ptr
);
975 static void tcg_out_movcond32(TCGContext
*s
, TCGCond cond
, TCGArg dest
,
976 TCGArg c1
, TCGArg c2
, int const_c2
,
979 tcg_out_cmp(s
, c1
, c2
, const_c2
, 0);
981 tcg_out_modrm(s
, OPC_CMOVCC
| tcg_cond_to_jcc
[cond
], dest
, v1
);
983 int over
= gen_new_label();
984 tcg_out_jxx(s
, tcg_cond_to_jcc
[tcg_invert_cond(cond
)], over
, 1);
985 tcg_out_mov(s
, TCG_TYPE_I32
, dest
, v1
);
986 tcg_out_label(s
, over
, s
->code_ptr
);
990 #if TCG_TARGET_REG_BITS == 64
991 static void tcg_out_movcond64(TCGContext
*s
, TCGCond cond
, TCGArg dest
,
992 TCGArg c1
, TCGArg c2
, int const_c2
,
995 tcg_out_cmp(s
, c1
, c2
, const_c2
, P_REXW
);
996 tcg_out_modrm(s
, OPC_CMOVCC
| tcg_cond_to_jcc
[cond
] | P_REXW
, dest
, v1
);
1000 static void tcg_out_branch(TCGContext
*s
, int call
, tcg_target_long dest
)
1002 tcg_target_long disp
= dest
- (tcg_target_long
)s
->code_ptr
- 5;
1004 if (disp
== (int32_t)disp
) {
1005 tcg_out_opc(s
, call
? OPC_CALL_Jz
: OPC_JMP_long
, 0, 0, 0);
1008 tcg_out_movi(s
, TCG_TYPE_PTR
, TCG_REG_R10
, dest
);
1009 tcg_out_modrm(s
, OPC_GRP5
,
1010 call
? EXT5_CALLN_Ev
: EXT5_JMPN_Ev
, TCG_REG_R10
);
1014 static inline void tcg_out_calli(TCGContext
*s
, tcg_target_long dest
)
1016 tcg_out_branch(s
, 1, dest
);
1019 static void tcg_out_jmp(TCGContext
*s
, tcg_target_long dest
)
1021 tcg_out_branch(s
, 0, dest
);
1024 #if defined(CONFIG_SOFTMMU)
1026 #include "exec/softmmu_defs.h"
1028 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
1029 * int mmu_idx, uintptr_t ra)
1031 static const void * const qemu_ld_helpers
[4] = {
1038 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
1039 * uintxx_t val, int mmu_idx, uintptr_t ra)
1041 static const void * const qemu_st_helpers
[4] = {
1048 static void add_qemu_ldst_label(TCGContext
*s
,
1057 uint8_t **label_ptr
);
1059 /* Perform the TLB load and compare.
1062 ADDRLO_IDX contains the index into ARGS of the low part of the
1063 address; the high part of the address is at ADDR_LOW_IDX+1.
1065 MEM_INDEX and S_BITS are the memory context and log2 size of the load.
1067 WHICH is the offset into the CPUTLBEntry structure of the slot to read.
1068 This should be offsetof addr_read or addr_write.
1071 LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
1072 positions of the displacements of forward jumps to the TLB miss case.
1074 Second argument register is loaded with the low part of the address.
1075 In the TLB hit case, it has been adjusted as indicated by the TLB
1076 and so is a host address. In the TLB miss case, it continues to
1077 hold a guest address.
1079 First argument register is clobbered. */
1081 static inline void tcg_out_tlb_load(TCGContext
*s
, int addrlo_idx
,
1082 int mem_index
, int s_bits
,
1084 uint8_t **label_ptr
, int which
)
1086 const int addrlo
= args
[addrlo_idx
];
1087 const int r0
= TCG_REG_L0
;
1088 const int r1
= TCG_REG_L1
;
1089 TCGType type
= TCG_TYPE_I32
;
1092 if (TCG_TARGET_REG_BITS
== 64 && TARGET_LONG_BITS
== 64) {
1093 type
= TCG_TYPE_I64
;
1097 tcg_out_mov(s
, type
, r0
, addrlo
);
1098 tcg_out_mov(s
, type
, r1
, addrlo
);
1100 tcg_out_shifti(s
, SHIFT_SHR
+ rexw
, r0
,
1101 TARGET_PAGE_BITS
- CPU_TLB_ENTRY_BITS
);
1103 tgen_arithi(s
, ARITH_AND
+ rexw
, r1
,
1104 TARGET_PAGE_MASK
| ((1 << s_bits
) - 1), 0);
1105 tgen_arithi(s
, ARITH_AND
+ rexw
, r0
,
1106 (CPU_TLB_SIZE
- 1) << CPU_TLB_ENTRY_BITS
, 0);
1108 tcg_out_modrm_sib_offset(s
, OPC_LEA
+ P_REXW
, r0
, TCG_AREG0
, r0
, 0,
1109 offsetof(CPUArchState
, tlb_table
[mem_index
][0])
1113 tcg_out_modrm_offset(s
, OPC_CMP_GvEv
+ rexw
, r1
, r0
, 0);
1115 tcg_out_mov(s
, type
, r1
, addrlo
);
1118 tcg_out_opc(s
, OPC_JCC_long
+ JCC_JNE
, 0, 0, 0);
1119 label_ptr
[0] = s
->code_ptr
;
1122 if (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
) {
1123 /* cmp 4(r0), addrhi */
1124 tcg_out_modrm_offset(s
, OPC_CMP_GvEv
, args
[addrlo_idx
+1], r0
, 4);
1127 tcg_out_opc(s
, OPC_JCC_long
+ JCC_JNE
, 0, 0, 0);
1128 label_ptr
[1] = s
->code_ptr
;
1134 /* add addend(r0), r1 */
1135 tcg_out_modrm_offset(s
, OPC_ADD_GvEv
+ P_REXW
, r1
, r0
,
1136 offsetof(CPUTLBEntry
, addend
) - which
);
1138 #elif defined(__x86_64__) && defined(__linux__)
1139 # include <asm/prctl.h>
1140 # include <sys/prctl.h>
1142 int arch_prctl(int code
, unsigned long addr
);
1144 static int guest_base_flags
;
1145 static inline void setup_guest_base_seg(void)
1147 if (arch_prctl(ARCH_SET_GS
, GUEST_BASE
) == 0) {
1148 guest_base_flags
= P_GS
;
1152 # define guest_base_flags 0
1153 static inline void setup_guest_base_seg(void) { }
1154 #endif /* SOFTMMU */
1156 static void tcg_out_qemu_ld_direct(TCGContext
*s
, int datalo
, int datahi
,
1157 int base
, tcg_target_long ofs
, int seg
,
1160 #ifdef TARGET_WORDS_BIGENDIAN
1161 const int bswap
= 1;
1163 const int bswap
= 0;
1167 tcg_out_modrm_offset(s
, OPC_MOVZBL
+ seg
, datalo
, base
, ofs
);
1170 tcg_out_modrm_offset(s
, OPC_MOVSBL
+ P_REXW
+ seg
, datalo
, base
, ofs
);
1173 tcg_out_modrm_offset(s
, OPC_MOVZWL
+ seg
, datalo
, base
, ofs
);
1175 tcg_out_rolw_8(s
, datalo
);
1180 tcg_out_modrm_offset(s
, OPC_MOVZWL
+ seg
, datalo
, base
, ofs
);
1181 tcg_out_rolw_8(s
, datalo
);
1182 tcg_out_modrm(s
, OPC_MOVSWL
+ P_REXW
, datalo
, datalo
);
1184 tcg_out_modrm_offset(s
, OPC_MOVSWL
+ P_REXW
+ seg
,
1189 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
, datalo
, base
, ofs
);
1191 tcg_out_bswap32(s
, datalo
);
1194 #if TCG_TARGET_REG_BITS == 64
1197 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
, datalo
, base
, ofs
);
1198 tcg_out_bswap32(s
, datalo
);
1199 tcg_out_ext32s(s
, datalo
, datalo
);
1201 tcg_out_modrm_offset(s
, OPC_MOVSLQ
+ seg
, datalo
, base
, ofs
);
1206 if (TCG_TARGET_REG_BITS
== 64) {
1207 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ P_REXW
+ seg
,
1210 tcg_out_bswap64(s
, datalo
);
1218 if (base
!= datalo
) {
1219 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
,
1221 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
,
1222 datahi
, base
, ofs
+ 4);
1224 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
,
1225 datahi
, base
, ofs
+ 4);
1226 tcg_out_modrm_offset(s
, OPC_MOVL_GvEv
+ seg
,
1230 tcg_out_bswap32(s
, datalo
);
1231 tcg_out_bswap32(s
, datahi
);
1240 /* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
1241 EAX. It will be useful once fixed registers globals are less
1243 static void tcg_out_qemu_ld(TCGContext
*s
, const TCGArg
*args
,
1246 int data_reg
, data_reg2
= 0;
1248 #if defined(CONFIG_SOFTMMU)
1249 int mem_index
, s_bits
;
1250 uint8_t *label_ptr
[2];
1255 if (TCG_TARGET_REG_BITS
== 32 && opc
== 3) {
1256 data_reg2
= args
[1];
1260 #if defined(CONFIG_SOFTMMU)
1261 mem_index
= args
[addrlo_idx
+ 1 + (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
)];
1264 tcg_out_tlb_load(s
, addrlo_idx
, mem_index
, s_bits
, args
,
1265 label_ptr
, offsetof(CPUTLBEntry
, addr_read
));
1268 tcg_out_qemu_ld_direct(s
, data_reg
, data_reg2
, TCG_REG_L1
, 0, 0, opc
);
1270 /* Record the current context of a load into ldst label */
1271 add_qemu_ldst_label(s
,
1277 args
[addrlo_idx
+ 1],
1283 int32_t offset
= GUEST_BASE
;
1284 int base
= args
[addrlo_idx
];
1287 /* ??? We assume all operations have left us with register contents
1288 that are zero extended. So far this appears to be true. If we
1289 want to enforce this, we can either do an explicit zero-extension
1290 here, or (if GUEST_BASE == 0, or a segment register is in use)
1291 use the ADDR32 prefix. For now, do nothing. */
1292 if (GUEST_BASE
&& guest_base_flags
) {
1293 seg
= guest_base_flags
;
1295 } else if (TCG_TARGET_REG_BITS
== 64 && offset
!= GUEST_BASE
) {
1296 tcg_out_movi(s
, TCG_TYPE_I64
, TCG_REG_L1
, GUEST_BASE
);
1297 tgen_arithr(s
, ARITH_ADD
+ P_REXW
, TCG_REG_L1
, base
);
1302 tcg_out_qemu_ld_direct(s
, data_reg
, data_reg2
, base
, offset
, seg
, opc
);
1307 static void tcg_out_qemu_st_direct(TCGContext
*s
, int datalo
, int datahi
,
1308 int base
, tcg_target_long ofs
, int seg
,
1311 #ifdef TARGET_WORDS_BIGENDIAN
1312 const int bswap
= 1;
1314 const int bswap
= 0;
1316 /* ??? Ideally we wouldn't need a scratch register. For user-only,
1317 we could perform the bswap twice to restore the original value
1318 instead of moving to the scratch. But as it is, the L constraint
1319 means that TCG_REG_L0 is definitely free here. */
1320 const int scratch
= TCG_REG_L0
;
1324 tcg_out_modrm_offset(s
, OPC_MOVB_EvGv
+ P_REXB_R
+ seg
,
1329 tcg_out_mov(s
, TCG_TYPE_I32
, scratch
, datalo
);
1330 tcg_out_rolw_8(s
, scratch
);
1333 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ P_DATA16
+ seg
,
1338 tcg_out_mov(s
, TCG_TYPE_I32
, scratch
, datalo
);
1339 tcg_out_bswap32(s
, scratch
);
1342 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ seg
, datalo
, base
, ofs
);
1345 if (TCG_TARGET_REG_BITS
== 64) {
1347 tcg_out_mov(s
, TCG_TYPE_I64
, scratch
, datalo
);
1348 tcg_out_bswap64(s
, scratch
);
1351 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ P_REXW
+ seg
,
1354 tcg_out_mov(s
, TCG_TYPE_I32
, scratch
, datahi
);
1355 tcg_out_bswap32(s
, scratch
);
1356 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ seg
, scratch
, base
, ofs
);
1357 tcg_out_mov(s
, TCG_TYPE_I32
, scratch
, datalo
);
1358 tcg_out_bswap32(s
, scratch
);
1359 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ seg
, scratch
, base
, ofs
+4);
1361 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ seg
, datalo
, base
, ofs
);
1362 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
+ seg
, datahi
, base
, ofs
+4);
1370 static void tcg_out_qemu_st(TCGContext
*s
, const TCGArg
*args
,
1373 int data_reg
, data_reg2
= 0;
1375 #if defined(CONFIG_SOFTMMU)
1376 int mem_index
, s_bits
;
1377 uint8_t *label_ptr
[2];
1382 if (TCG_TARGET_REG_BITS
== 32 && opc
== 3) {
1383 data_reg2
= args
[1];
1387 #if defined(CONFIG_SOFTMMU)
1388 mem_index
= args
[addrlo_idx
+ 1 + (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
)];
1391 tcg_out_tlb_load(s
, addrlo_idx
, mem_index
, s_bits
, args
,
1392 label_ptr
, offsetof(CPUTLBEntry
, addr_write
));
1395 tcg_out_qemu_st_direct(s
, data_reg
, data_reg2
, TCG_REG_L1
, 0, 0, opc
);
1397 /* Record the current context of a store into ldst label */
1398 add_qemu_ldst_label(s
,
1404 args
[addrlo_idx
+ 1],
1410 int32_t offset
= GUEST_BASE
;
1411 int base
= args
[addrlo_idx
];
1414 /* ??? We assume all operations have left us with register contents
1415 that are zero extended. So far this appears to be true. If we
1416 want to enforce this, we can either do an explicit zero-extension
1417 here, or (if GUEST_BASE == 0, or a segment register is in use)
1418 use the ADDR32 prefix. For now, do nothing. */
1419 if (GUEST_BASE
&& guest_base_flags
) {
1420 seg
= guest_base_flags
;
1422 } else if (TCG_TARGET_REG_BITS
== 64 && offset
!= GUEST_BASE
) {
1423 tcg_out_movi(s
, TCG_TYPE_I64
, TCG_REG_L1
, GUEST_BASE
);
1424 tgen_arithr(s
, ARITH_ADD
+ P_REXW
, TCG_REG_L1
, base
);
1429 tcg_out_qemu_st_direct(s
, data_reg
, data_reg2
, base
, offset
, seg
, opc
);
1434 #if defined(CONFIG_SOFTMMU)
1436 * Record the context of a call to the out of line helper code for the slow path
1437 * for a load or store, so that we can later generate the correct helper code
1439 static void add_qemu_ldst_label(TCGContext
*s
,
1448 uint8_t **label_ptr
)
1451 TCGLabelQemuLdst
*label
;
1453 if (s
->nb_qemu_ldst_labels
>= TCG_MAX_QEMU_LDST
) {
1457 idx
= s
->nb_qemu_ldst_labels
++;
1458 label
= (TCGLabelQemuLdst
*)&s
->qemu_ldst_labels
[idx
];
1459 label
->is_ld
= is_ld
;
1461 label
->datalo_reg
= data_reg
;
1462 label
->datahi_reg
= data_reg2
;
1463 label
->addrlo_reg
= addrlo_reg
;
1464 label
->addrhi_reg
= addrhi_reg
;
1465 label
->mem_index
= mem_index
;
1466 label
->raddr
= raddr
;
1467 label
->label_ptr
[0] = label_ptr
[0];
1468 if (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
) {
1469 label
->label_ptr
[1] = label_ptr
[1];
1473 /* See the GETPC definition in include/exec/exec-all.h. */
1474 static inline uintptr_t do_getpc(uint8_t *raddr
)
1476 return (uintptr_t)raddr
- 1;
1480 * Generate code for the slow path for a load at the end of block
1482 static void tcg_out_qemu_ld_slow_path(TCGContext
*s
, TCGLabelQemuLdst
*l
)
1485 int s_bits
= opc
& 3;
1487 uint8_t **label_ptr
= &l
->label_ptr
[0];
1489 /* resolve label address */
1490 *(uint32_t *)label_ptr
[0] = (uint32_t)(s
->code_ptr
- label_ptr
[0] - 4);
1491 if (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
) {
1492 *(uint32_t *)label_ptr
[1] = (uint32_t)(s
->code_ptr
- label_ptr
[1] - 4);
1495 if (TCG_TARGET_REG_BITS
== 32) {
1498 tcg_out_st(s
, TCG_TYPE_PTR
, TCG_AREG0
, TCG_REG_ESP
, ofs
);
1501 tcg_out_st(s
, TCG_TYPE_I32
, l
->addrlo_reg
, TCG_REG_ESP
, ofs
);
1504 if (TARGET_LONG_BITS
== 64) {
1505 tcg_out_st(s
, TCG_TYPE_I32
, l
->addrhi_reg
, TCG_REG_ESP
, ofs
);
1509 tcg_out_sti(s
, TCG_TYPE_I32
, TCG_REG_ESP
, ofs
, l
->mem_index
);
1512 tcg_out_sti(s
, TCG_TYPE_I32
, TCG_REG_ESP
, ofs
, do_getpc(l
->raddr
));
1514 tcg_out_mov(s
, TCG_TYPE_PTR
, tcg_target_call_iarg_regs
[0], TCG_AREG0
);
1515 /* The second argument is already loaded with addrlo. */
1516 tcg_out_movi(s
, TCG_TYPE_I32
, tcg_target_call_iarg_regs
[2],
1518 tcg_out_movi(s
, TCG_TYPE_PTR
, tcg_target_call_iarg_regs
[3],
1519 do_getpc(l
->raddr
));
1522 tcg_out_calli(s
, (tcg_target_long
)qemu_ld_helpers
[s_bits
]);
1524 data_reg
= l
->datalo_reg
;
1527 tcg_out_ext8s(s
, data_reg
, TCG_REG_EAX
, P_REXW
);
1530 tcg_out_ext16s(s
, data_reg
, TCG_REG_EAX
, P_REXW
);
1533 tcg_out_ext8u(s
, data_reg
, TCG_REG_EAX
);
1536 tcg_out_ext16u(s
, data_reg
, TCG_REG_EAX
);
1539 tcg_out_mov(s
, TCG_TYPE_I32
, data_reg
, TCG_REG_EAX
);
1541 #if TCG_TARGET_REG_BITS == 64
1543 tcg_out_ext32s(s
, data_reg
, TCG_REG_EAX
);
1547 if (TCG_TARGET_REG_BITS
== 64) {
1548 tcg_out_mov(s
, TCG_TYPE_I64
, data_reg
, TCG_REG_RAX
);
1549 } else if (data_reg
== TCG_REG_EDX
) {
1550 /* xchg %edx, %eax */
1551 tcg_out_opc(s
, OPC_XCHG_ax_r32
+ TCG_REG_EDX
, 0, 0, 0);
1552 tcg_out_mov(s
, TCG_TYPE_I32
, l
->datahi_reg
, TCG_REG_EAX
);
1554 tcg_out_mov(s
, TCG_TYPE_I32
, data_reg
, TCG_REG_EAX
);
1555 tcg_out_mov(s
, TCG_TYPE_I32
, l
->datahi_reg
, TCG_REG_EDX
);
1562 /* Jump to the code corresponding to next IR of qemu_st */
1563 tcg_out_jmp(s
, (tcg_target_long
)l
->raddr
);
1567 * Generate code for the slow path for a store at the end of block
1569 static void tcg_out_qemu_st_slow_path(TCGContext
*s
, TCGLabelQemuLdst
*l
)
1572 int s_bits
= opc
& 3;
1573 uint8_t **label_ptr
= &l
->label_ptr
[0];
1575 /* resolve label address */
1576 *(uint32_t *)label_ptr
[0] = (uint32_t)(s
->code_ptr
- label_ptr
[0] - 4);
1577 if (TARGET_LONG_BITS
> TCG_TARGET_REG_BITS
) {
1578 *(uint32_t *)label_ptr
[1] = (uint32_t)(s
->code_ptr
- label_ptr
[1] - 4);
1581 if (TCG_TARGET_REG_BITS
== 32) {
1584 tcg_out_st(s
, TCG_TYPE_PTR
, TCG_AREG0
, TCG_REG_ESP
, ofs
);
1587 tcg_out_st(s
, TCG_TYPE_I32
, l
->addrlo_reg
, TCG_REG_ESP
, ofs
);
1590 if (TARGET_LONG_BITS
== 64) {
1591 tcg_out_st(s
, TCG_TYPE_I32
, l
->addrhi_reg
, TCG_REG_ESP
, ofs
);
1595 tcg_out_st(s
, TCG_TYPE_I32
, l
->datalo_reg
, TCG_REG_ESP
, ofs
);
1599 tcg_out_st(s
, TCG_TYPE_I32
, l
->datahi_reg
, TCG_REG_ESP
, ofs
);
1603 tcg_out_sti(s
, TCG_TYPE_I32
, TCG_REG_ESP
, ofs
, l
->mem_index
);
1606 tcg_out_sti(s
, TCG_TYPE_I32
, TCG_REG_ESP
, ofs
, do_getpc(l
->raddr
));
1610 tcg_out_mov(s
, TCG_TYPE_PTR
, tcg_target_call_iarg_regs
[0], TCG_AREG0
);
1611 /* The second argument is already loaded with addrlo. */
1612 tcg_out_mov(s
, (opc
== 3 ? TCG_TYPE_I64
: TCG_TYPE_I32
),
1613 tcg_target_call_iarg_regs
[2], l
->datalo_reg
);
1614 tcg_out_movi(s
, TCG_TYPE_I32
, tcg_target_call_iarg_regs
[3],
1617 pc
= do_getpc(l
->raddr
);
1618 if (ARRAY_SIZE(tcg_target_call_iarg_regs
) > 4) {
1619 tcg_out_movi(s
, TCG_TYPE_PTR
, tcg_target_call_iarg_regs
[4], pc
);
1620 } else if (pc
== (int32_t)pc
) {
1621 tcg_out_sti(s
, TCG_TYPE_PTR
, TCG_REG_ESP
, 0, pc
);
1623 tcg_out_movi(s
, TCG_TYPE_PTR
, TCG_REG_RAX
, pc
);
1624 tcg_out_st(s
, TCG_TYPE_PTR
, TCG_REG_RAX
, TCG_REG_ESP
, 0);
1628 tcg_out_calli(s
, (tcg_target_long
)qemu_st_helpers
[s_bits
]);
1630 tcg_out_jmp(s
, (tcg_target_long
)l
->raddr
);
1634 * Generate TB finalization at the end of block
1636 void tcg_out_tb_finalize(TCGContext
*s
)
1639 TCGLabelQemuLdst
*label
;
1641 /* qemu_ld/st slow paths */
1642 for (i
= 0; i
< s
->nb_qemu_ldst_labels
; i
++) {
1643 label
= (TCGLabelQemuLdst
*)&s
->qemu_ldst_labels
[i
];
1645 tcg_out_qemu_ld_slow_path(s
, label
);
1647 tcg_out_qemu_st_slow_path(s
, label
);
1651 #endif /* CONFIG_SOFTMMU */
1653 static inline void tcg_out_op(TCGContext
*s
, TCGOpcode opc
,
1654 const TCGArg
*args
, const int *const_args
)
1658 #if TCG_TARGET_REG_BITS == 64
1659 # define OP_32_64(x) \
1660 case glue(glue(INDEX_op_, x), _i64): \
1661 rexw = P_REXW; /* FALLTHRU */ \
1662 case glue(glue(INDEX_op_, x), _i32)
1664 # define OP_32_64(x) \
1665 case glue(glue(INDEX_op_, x), _i32)
1669 case INDEX_op_exit_tb
:
1670 tcg_out_movi(s
, TCG_TYPE_PTR
, TCG_REG_EAX
, args
[0]);
1671 tcg_out_jmp(s
, (tcg_target_long
) tb_ret_addr
);
1673 case INDEX_op_goto_tb
:
1674 if (s
->tb_jmp_offset
) {
1675 /* direct jump method */
1676 tcg_out8(s
, OPC_JMP_long
); /* jmp im */
1677 s
->tb_jmp_offset
[args
[0]] = s
->code_ptr
- s
->code_buf
;
1680 /* indirect jump method */
1681 tcg_out_modrm_offset(s
, OPC_GRP5
, EXT5_JMPN_Ev
, -1,
1682 (tcg_target_long
)(s
->tb_next
+ args
[0]));
1684 s
->tb_next_offset
[args
[0]] = s
->code_ptr
- s
->code_buf
;
1687 if (const_args
[0]) {
1688 tcg_out_calli(s
, args
[0]);
1691 tcg_out_modrm(s
, OPC_GRP5
, EXT5_CALLN_Ev
, args
[0]);
1695 tcg_out_jxx(s
, JCC_JMP
, args
[0], 0);
1697 case INDEX_op_movi_i32
:
1698 tcg_out_movi(s
, TCG_TYPE_I32
, args
[0], args
[1]);
1701 /* Note that we can ignore REXW for the zero-extend to 64-bit. */
1702 tcg_out_modrm_offset(s
, OPC_MOVZBL
, args
[0], args
[1], args
[2]);
1705 tcg_out_modrm_offset(s
, OPC_MOVSBL
+ rexw
, args
[0], args
[1], args
[2]);
1708 /* Note that we can ignore REXW for the zero-extend to 64-bit. */
1709 tcg_out_modrm_offset(s
, OPC_MOVZWL
, args
[0], args
[1], args
[2]);
1712 tcg_out_modrm_offset(s
, OPC_MOVSWL
+ rexw
, args
[0], args
[1], args
[2]);
1714 #if TCG_TARGET_REG_BITS == 64
1715 case INDEX_op_ld32u_i64
:
1717 case INDEX_op_ld_i32
:
1718 tcg_out_ld(s
, TCG_TYPE_I32
, args
[0], args
[1], args
[2]);
1722 if (const_args
[0]) {
1723 tcg_out_modrm_offset(s
, OPC_MOVB_EvIz
,
1724 0, args
[1], args
[2]);
1725 tcg_out8(s
, args
[0]);
1727 tcg_out_modrm_offset(s
, OPC_MOVB_EvGv
| P_REXB_R
,
1728 args
[0], args
[1], args
[2]);
1732 if (const_args
[0]) {
1733 tcg_out_modrm_offset(s
, OPC_MOVL_EvIz
| P_DATA16
,
1734 0, args
[1], args
[2]);
1735 tcg_out16(s
, args
[0]);
1737 tcg_out_modrm_offset(s
, OPC_MOVL_EvGv
| P_DATA16
,
1738 args
[0], args
[1], args
[2]);
1741 #if TCG_TARGET_REG_BITS == 64
1742 case INDEX_op_st32_i64
:
1744 case INDEX_op_st_i32
:
1745 if (const_args
[0]) {
1746 tcg_out_modrm_offset(s
, OPC_MOVL_EvIz
, 0, args
[1], args
[2]);
1747 tcg_out32(s
, args
[0]);
1749 tcg_out_st(s
, TCG_TYPE_I32
, args
[0], args
[1], args
[2]);
1754 /* For 3-operand addition, use LEA. */
1755 if (args
[0] != args
[1]) {
1756 TCGArg a0
= args
[0], a1
= args
[1], a2
= args
[2], c3
= 0;
1758 if (const_args
[2]) {
1760 } else if (a0
== a2
) {
1761 /* Watch out for dest = src + dest, since we've removed
1762 the matching constraint on the add. */
1763 tgen_arithr(s
, ARITH_ADD
+ rexw
, a0
, a1
);
1767 tcg_out_modrm_sib_offset(s
, OPC_LEA
+ rexw
, a0
, a1
, a2
, 0, c3
);
1785 if (const_args
[2]) {
1786 tgen_arithi(s
, c
+ rexw
, args
[0], args
[2], 0);
1788 tgen_arithr(s
, c
+ rexw
, args
[0], args
[2]);
1793 if (const_args
[2]) {
1796 if (val
== (int8_t)val
) {
1797 tcg_out_modrm(s
, OPC_IMUL_GvEvIb
+ rexw
, args
[0], args
[0]);
1800 tcg_out_modrm(s
, OPC_IMUL_GvEvIz
+ rexw
, args
[0], args
[0]);
1804 tcg_out_modrm(s
, OPC_IMUL_GvEv
+ rexw
, args
[0], args
[2]);
1809 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_IDIV
, args
[4]);
1812 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_DIV
, args
[4]);
1831 if (const_args
[2]) {
1832 tcg_out_shifti(s
, c
+ rexw
, args
[0], args
[2]);
1834 tcg_out_modrm(s
, OPC_SHIFT_cl
+ rexw
, c
, args
[0]);
1838 case INDEX_op_brcond_i32
:
1839 tcg_out_brcond32(s
, args
[2], args
[0], args
[1], const_args
[1],
1842 case INDEX_op_setcond_i32
:
1843 tcg_out_setcond32(s
, args
[3], args
[0], args
[1],
1844 args
[2], const_args
[2]);
1846 case INDEX_op_movcond_i32
:
1847 tcg_out_movcond32(s
, args
[5], args
[0], args
[1],
1848 args
[2], const_args
[2], args
[3]);
1852 tcg_out_rolw_8(s
, args
[0]);
1855 tcg_out_bswap32(s
, args
[0]);
1859 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_NEG
, args
[0]);
1862 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_NOT
, args
[0]);
1866 tcg_out_ext8s(s
, args
[0], args
[1], rexw
);
1869 tcg_out_ext16s(s
, args
[0], args
[1], rexw
);
1872 tcg_out_ext8u(s
, args
[0], args
[1]);
1875 tcg_out_ext16u(s
, args
[0], args
[1]);
1878 case INDEX_op_qemu_ld8u
:
1879 tcg_out_qemu_ld(s
, args
, 0);
1881 case INDEX_op_qemu_ld8s
:
1882 tcg_out_qemu_ld(s
, args
, 0 | 4);
1884 case INDEX_op_qemu_ld16u
:
1885 tcg_out_qemu_ld(s
, args
, 1);
1887 case INDEX_op_qemu_ld16s
:
1888 tcg_out_qemu_ld(s
, args
, 1 | 4);
1890 #if TCG_TARGET_REG_BITS == 64
1891 case INDEX_op_qemu_ld32u
:
1893 case INDEX_op_qemu_ld32
:
1894 tcg_out_qemu_ld(s
, args
, 2);
1896 case INDEX_op_qemu_ld64
:
1897 tcg_out_qemu_ld(s
, args
, 3);
1900 case INDEX_op_qemu_st8
:
1901 tcg_out_qemu_st(s
, args
, 0);
1903 case INDEX_op_qemu_st16
:
1904 tcg_out_qemu_st(s
, args
, 1);
1906 case INDEX_op_qemu_st32
:
1907 tcg_out_qemu_st(s
, args
, 2);
1909 case INDEX_op_qemu_st64
:
1910 tcg_out_qemu_st(s
, args
, 3);
1914 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_MUL
, args
[3]);
1917 tcg_out_modrm(s
, OPC_GRP3_Ev
+ rexw
, EXT3_IMUL
, args
[3]);
1920 if (const_args
[4]) {
1921 tgen_arithi(s
, ARITH_ADD
+ rexw
, args
[0], args
[4], 1);
1923 tgen_arithr(s
, ARITH_ADD
+ rexw
, args
[0], args
[4]);
1925 if (const_args
[5]) {
1926 tgen_arithi(s
, ARITH_ADC
+ rexw
, args
[1], args
[5], 1);
1928 tgen_arithr(s
, ARITH_ADC
+ rexw
, args
[1], args
[5]);
1932 if (const_args
[4]) {
1933 tgen_arithi(s
, ARITH_SUB
+ rexw
, args
[0], args
[4], 1);
1935 tgen_arithr(s
, ARITH_SUB
+ rexw
, args
[0], args
[4]);
1937 if (const_args
[5]) {
1938 tgen_arithi(s
, ARITH_SBB
+ rexw
, args
[1], args
[5], 1);
1940 tgen_arithr(s
, ARITH_SBB
+ rexw
, args
[1], args
[5]);
1944 #if TCG_TARGET_REG_BITS == 32
1945 case INDEX_op_brcond2_i32
:
1946 tcg_out_brcond2(s
, args
, const_args
, 0);
1948 case INDEX_op_setcond2_i32
:
1949 tcg_out_setcond2(s
, args
, const_args
);
1951 #else /* TCG_TARGET_REG_BITS == 64 */
1952 case INDEX_op_movi_i64
:
1953 tcg_out_movi(s
, TCG_TYPE_I64
, args
[0], args
[1]);
1955 case INDEX_op_ld32s_i64
:
1956 tcg_out_modrm_offset(s
, OPC_MOVSLQ
, args
[0], args
[1], args
[2]);
1958 case INDEX_op_ld_i64
:
1959 tcg_out_ld(s
, TCG_TYPE_I64
, args
[0], args
[1], args
[2]);
1961 case INDEX_op_st_i64
:
1962 if (const_args
[0]) {
1963 tcg_out_modrm_offset(s
, OPC_MOVL_EvIz
| P_REXW
,
1964 0, args
[1], args
[2]);
1965 tcg_out32(s
, args
[0]);
1967 tcg_out_st(s
, TCG_TYPE_I64
, args
[0], args
[1], args
[2]);
1970 case INDEX_op_qemu_ld32s
:
1971 tcg_out_qemu_ld(s
, args
, 2 | 4);
1974 case INDEX_op_brcond_i64
:
1975 tcg_out_brcond64(s
, args
[2], args
[0], args
[1], const_args
[1],
1978 case INDEX_op_setcond_i64
:
1979 tcg_out_setcond64(s
, args
[3], args
[0], args
[1],
1980 args
[2], const_args
[2]);
1982 case INDEX_op_movcond_i64
:
1983 tcg_out_movcond64(s
, args
[5], args
[0], args
[1],
1984 args
[2], const_args
[2], args
[3]);
1987 case INDEX_op_bswap64_i64
:
1988 tcg_out_bswap64(s
, args
[0]);
1990 case INDEX_op_ext32u_i64
:
1991 tcg_out_ext32u(s
, args
[0], args
[1]);
1993 case INDEX_op_ext32s_i64
:
1994 tcg_out_ext32s(s
, args
[0], args
[1]);
1999 if (args
[3] == 0 && args
[4] == 8) {
2000 /* load bits 0..7 */
2001 tcg_out_modrm(s
, OPC_MOVB_EvGv
| P_REXB_R
| P_REXB_RM
,
2003 } else if (args
[3] == 8 && args
[4] == 8) {
2004 /* load bits 8..15 */
2005 tcg_out_modrm(s
, OPC_MOVB_EvGv
, args
[2], args
[0] + 4);
2006 } else if (args
[3] == 0 && args
[4] == 16) {
2007 /* load bits 0..15 */
2008 tcg_out_modrm(s
, OPC_MOVL_EvGv
| P_DATA16
, args
[2], args
[0]);
2021 static const TCGTargetOpDef x86_op_defs
[] = {
2022 { INDEX_op_exit_tb
, { } },
2023 { INDEX_op_goto_tb
, { } },
2024 { INDEX_op_call
, { "ri" } },
2025 { INDEX_op_br
, { } },
2026 { INDEX_op_mov_i32
, { "r", "r" } },
2027 { INDEX_op_movi_i32
, { "r" } },
2028 { INDEX_op_ld8u_i32
, { "r", "r" } },
2029 { INDEX_op_ld8s_i32
, { "r", "r" } },
2030 { INDEX_op_ld16u_i32
, { "r", "r" } },
2031 { INDEX_op_ld16s_i32
, { "r", "r" } },
2032 { INDEX_op_ld_i32
, { "r", "r" } },
2033 { INDEX_op_st8_i32
, { "qi", "r" } },
2034 { INDEX_op_st16_i32
, { "ri", "r" } },
2035 { INDEX_op_st_i32
, { "ri", "r" } },
2037 { INDEX_op_add_i32
, { "r", "r", "ri" } },
2038 { INDEX_op_sub_i32
, { "r", "0", "ri" } },
2039 { INDEX_op_mul_i32
, { "r", "0", "ri" } },
2040 { INDEX_op_div2_i32
, { "a", "d", "0", "1", "r" } },
2041 { INDEX_op_divu2_i32
, { "a", "d", "0", "1", "r" } },
2042 { INDEX_op_and_i32
, { "r", "0", "ri" } },
2043 { INDEX_op_or_i32
, { "r", "0", "ri" } },
2044 { INDEX_op_xor_i32
, { "r", "0", "ri" } },
2046 { INDEX_op_shl_i32
, { "r", "0", "ci" } },
2047 { INDEX_op_shr_i32
, { "r", "0", "ci" } },
2048 { INDEX_op_sar_i32
, { "r", "0", "ci" } },
2049 { INDEX_op_rotl_i32
, { "r", "0", "ci" } },
2050 { INDEX_op_rotr_i32
, { "r", "0", "ci" } },
2052 { INDEX_op_brcond_i32
, { "r", "ri" } },
2054 { INDEX_op_bswap16_i32
, { "r", "0" } },
2055 { INDEX_op_bswap32_i32
, { "r", "0" } },
2057 { INDEX_op_neg_i32
, { "r", "0" } },
2059 { INDEX_op_not_i32
, { "r", "0" } },
2061 { INDEX_op_ext8s_i32
, { "r", "q" } },
2062 { INDEX_op_ext16s_i32
, { "r", "r" } },
2063 { INDEX_op_ext8u_i32
, { "r", "q" } },
2064 { INDEX_op_ext16u_i32
, { "r", "r" } },
2066 { INDEX_op_setcond_i32
, { "q", "r", "ri" } },
2068 { INDEX_op_deposit_i32
, { "Q", "0", "Q" } },
2069 #if TCG_TARGET_HAS_movcond_i32
2070 { INDEX_op_movcond_i32
, { "r", "r", "ri", "r", "0" } },
2073 { INDEX_op_mulu2_i32
, { "a", "d", "a", "r" } },
2074 { INDEX_op_muls2_i32
, { "a", "d", "a", "r" } },
2075 { INDEX_op_add2_i32
, { "r", "r", "0", "1", "ri", "ri" } },
2076 { INDEX_op_sub2_i32
, { "r", "r", "0", "1", "ri", "ri" } },
2078 #if TCG_TARGET_REG_BITS == 32
2079 { INDEX_op_brcond2_i32
, { "r", "r", "ri", "ri" } },
2080 { INDEX_op_setcond2_i32
, { "r", "r", "r", "ri", "ri" } },
2082 { INDEX_op_mov_i64
, { "r", "r" } },
2083 { INDEX_op_movi_i64
, { "r" } },
2084 { INDEX_op_ld8u_i64
, { "r", "r" } },
2085 { INDEX_op_ld8s_i64
, { "r", "r" } },
2086 { INDEX_op_ld16u_i64
, { "r", "r" } },
2087 { INDEX_op_ld16s_i64
, { "r", "r" } },
2088 { INDEX_op_ld32u_i64
, { "r", "r" } },
2089 { INDEX_op_ld32s_i64
, { "r", "r" } },
2090 { INDEX_op_ld_i64
, { "r", "r" } },
2091 { INDEX_op_st8_i64
, { "ri", "r" } },
2092 { INDEX_op_st16_i64
, { "ri", "r" } },
2093 { INDEX_op_st32_i64
, { "ri", "r" } },
2094 { INDEX_op_st_i64
, { "re", "r" } },
2096 { INDEX_op_add_i64
, { "r", "r", "re" } },
2097 { INDEX_op_mul_i64
, { "r", "0", "re" } },
2098 { INDEX_op_div2_i64
, { "a", "d", "0", "1", "r" } },
2099 { INDEX_op_divu2_i64
, { "a", "d", "0", "1", "r" } },
2100 { INDEX_op_sub_i64
, { "r", "0", "re" } },
2101 { INDEX_op_and_i64
, { "r", "0", "reZ" } },
2102 { INDEX_op_or_i64
, { "r", "0", "re" } },
2103 { INDEX_op_xor_i64
, { "r", "0", "re" } },
2105 { INDEX_op_shl_i64
, { "r", "0", "ci" } },
2106 { INDEX_op_shr_i64
, { "r", "0", "ci" } },
2107 { INDEX_op_sar_i64
, { "r", "0", "ci" } },
2108 { INDEX_op_rotl_i64
, { "r", "0", "ci" } },
2109 { INDEX_op_rotr_i64
, { "r", "0", "ci" } },
2111 { INDEX_op_brcond_i64
, { "r", "re" } },
2112 { INDEX_op_setcond_i64
, { "r", "r", "re" } },
2114 { INDEX_op_bswap16_i64
, { "r", "0" } },
2115 { INDEX_op_bswap32_i64
, { "r", "0" } },
2116 { INDEX_op_bswap64_i64
, { "r", "0" } },
2117 { INDEX_op_neg_i64
, { "r", "0" } },
2118 { INDEX_op_not_i64
, { "r", "0" } },
2120 { INDEX_op_ext8s_i64
, { "r", "r" } },
2121 { INDEX_op_ext16s_i64
, { "r", "r" } },
2122 { INDEX_op_ext32s_i64
, { "r", "r" } },
2123 { INDEX_op_ext8u_i64
, { "r", "r" } },
2124 { INDEX_op_ext16u_i64
, { "r", "r" } },
2125 { INDEX_op_ext32u_i64
, { "r", "r" } },
2127 { INDEX_op_deposit_i64
, { "Q", "0", "Q" } },
2128 { INDEX_op_movcond_i64
, { "r", "r", "re", "r", "0" } },
2130 { INDEX_op_mulu2_i64
, { "a", "d", "a", "r" } },
2131 { INDEX_op_muls2_i64
, { "a", "d", "a", "r" } },
2132 { INDEX_op_add2_i64
, { "r", "r", "0", "1", "re", "re" } },
2133 { INDEX_op_sub2_i64
, { "r", "r", "0", "1", "re", "re" } },
2136 #if TCG_TARGET_REG_BITS == 64
2137 { INDEX_op_qemu_ld8u
, { "r", "L" } },
2138 { INDEX_op_qemu_ld8s
, { "r", "L" } },
2139 { INDEX_op_qemu_ld16u
, { "r", "L" } },
2140 { INDEX_op_qemu_ld16s
, { "r", "L" } },
2141 { INDEX_op_qemu_ld32
, { "r", "L" } },
2142 { INDEX_op_qemu_ld32u
, { "r", "L" } },
2143 { INDEX_op_qemu_ld32s
, { "r", "L" } },
2144 { INDEX_op_qemu_ld64
, { "r", "L" } },
2146 { INDEX_op_qemu_st8
, { "L", "L" } },
2147 { INDEX_op_qemu_st16
, { "L", "L" } },
2148 { INDEX_op_qemu_st32
, { "L", "L" } },
2149 { INDEX_op_qemu_st64
, { "L", "L" } },
2150 #elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
2151 { INDEX_op_qemu_ld8u
, { "r", "L" } },
2152 { INDEX_op_qemu_ld8s
, { "r", "L" } },
2153 { INDEX_op_qemu_ld16u
, { "r", "L" } },
2154 { INDEX_op_qemu_ld16s
, { "r", "L" } },
2155 { INDEX_op_qemu_ld32
, { "r", "L" } },
2156 { INDEX_op_qemu_ld64
, { "r", "r", "L" } },
2158 { INDEX_op_qemu_st8
, { "cb", "L" } },
2159 { INDEX_op_qemu_st16
, { "L", "L" } },
2160 { INDEX_op_qemu_st32
, { "L", "L" } },
2161 { INDEX_op_qemu_st64
, { "L", "L", "L" } },
2163 { INDEX_op_qemu_ld8u
, { "r", "L", "L" } },
2164 { INDEX_op_qemu_ld8s
, { "r", "L", "L" } },
2165 { INDEX_op_qemu_ld16u
, { "r", "L", "L" } },
2166 { INDEX_op_qemu_ld16s
, { "r", "L", "L" } },
2167 { INDEX_op_qemu_ld32
, { "r", "L", "L" } },
2168 { INDEX_op_qemu_ld64
, { "r", "r", "L", "L" } },
2170 { INDEX_op_qemu_st8
, { "cb", "L", "L" } },
2171 { INDEX_op_qemu_st16
, { "L", "L", "L" } },
2172 { INDEX_op_qemu_st32
, { "L", "L", "L" } },
2173 { INDEX_op_qemu_st64
, { "L", "L", "L", "L" } },
2178 static int tcg_target_callee_save_regs
[] = {
2179 #if TCG_TARGET_REG_BITS == 64
2188 TCG_REG_R14
, /* Currently used for the global env. */
2191 TCG_REG_EBP
, /* Currently used for the global env. */
2198 /* Compute frame size via macros, to share between tcg_target_qemu_prologue
2199 and tcg_register_jit. */
2202 ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \
2203 * (TCG_TARGET_REG_BITS / 8))
2205 #define FRAME_SIZE \
2207 + TCG_STATIC_CALL_ARGS_SIZE \
2208 + CPU_TEMP_BUF_NLONGS * sizeof(long) \
2209 + TCG_TARGET_STACK_ALIGN - 1) \
2210 & ~(TCG_TARGET_STACK_ALIGN - 1))
2212 /* Generate global QEMU prologue and epilogue code */
2213 static void tcg_target_qemu_prologue(TCGContext
*s
)
2215 int i
, stack_addend
;
2219 /* Reserve some stack space, also for TCG temps. */
2220 stack_addend
= FRAME_SIZE
- PUSH_SIZE
;
2221 tcg_set_frame(s
, TCG_REG_CALL_STACK
, TCG_STATIC_CALL_ARGS_SIZE
,
2222 CPU_TEMP_BUF_NLONGS
* sizeof(long));
2224 /* Save all callee saved registers. */
2225 for (i
= 0; i
< ARRAY_SIZE(tcg_target_callee_save_regs
); i
++) {
2226 tcg_out_push(s
, tcg_target_callee_save_regs
[i
]);
2229 #if TCG_TARGET_REG_BITS == 32
2230 tcg_out_ld(s
, TCG_TYPE_PTR
, TCG_AREG0
, TCG_REG_ESP
,
2231 (ARRAY_SIZE(tcg_target_callee_save_regs
) + 1) * 4);
2232 tcg_out_addi(s
, TCG_REG_ESP
, -stack_addend
);
2234 tcg_out_modrm_offset(s
, OPC_GRP5
, EXT5_JMPN_Ev
, TCG_REG_ESP
,
2235 (ARRAY_SIZE(tcg_target_callee_save_regs
) + 2) * 4
2238 tcg_out_mov(s
, TCG_TYPE_PTR
, TCG_AREG0
, tcg_target_call_iarg_regs
[0]);
2239 tcg_out_addi(s
, TCG_REG_ESP
, -stack_addend
);
2241 tcg_out_modrm(s
, OPC_GRP5
, EXT5_JMPN_Ev
, tcg_target_call_iarg_regs
[1]);
2245 tb_ret_addr
= s
->code_ptr
;
2247 tcg_out_addi(s
, TCG_REG_CALL_STACK
, stack_addend
);
2249 for (i
= ARRAY_SIZE(tcg_target_callee_save_regs
) - 1; i
>= 0; i
--) {
2250 tcg_out_pop(s
, tcg_target_callee_save_regs
[i
]);
2252 tcg_out_opc(s
, OPC_RET
, 0, 0, 0);
2254 #if !defined(CONFIG_SOFTMMU)
2255 /* Try to set up a segment register to point to GUEST_BASE. */
2257 setup_guest_base_seg();
2262 static void tcg_target_init(TCGContext
*s
)
2264 /* For 32-bit, 99% certainty that we're running on hardware that supports
2265 cmov, but we still need to check. In case cmov is not available, we'll
2266 use a small forward branch. */
2269 unsigned a
, b
, c
, d
;
2270 have_cmov
= (__get_cpuid(1, &a
, &b
, &c
, &d
) && (d
& bit_CMOV
));
2274 if (TCG_TARGET_REG_BITS
== 64) {
2275 tcg_regset_set32(tcg_target_available_regs
[TCG_TYPE_I32
], 0, 0xffff);
2276 tcg_regset_set32(tcg_target_available_regs
[TCG_TYPE_I64
], 0, 0xffff);
2278 tcg_regset_set32(tcg_target_available_regs
[TCG_TYPE_I32
], 0, 0xff);
2281 tcg_regset_clear(tcg_target_call_clobber_regs
);
2282 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_EAX
);
2283 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_EDX
);
2284 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_ECX
);
2285 if (TCG_TARGET_REG_BITS
== 64) {
2286 #if !defined(_WIN64)
2287 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_RDI
);
2288 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_RSI
);
2290 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_R8
);
2291 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_R9
);
2292 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_R10
);
2293 tcg_regset_set_reg(tcg_target_call_clobber_regs
, TCG_REG_R11
);
2296 tcg_regset_clear(s
->reserved_regs
);
2297 tcg_regset_set_reg(s
->reserved_regs
, TCG_REG_CALL_STACK
);
2299 tcg_add_target_add_op_defs(x86_op_defs
);
2304 DebugFrameFDEHeader fde
;
2305 uint8_t fde_def_cfa
[4];
2306 uint8_t fde_reg_ofs
[14];
2309 /* We're expecting a 2 byte uleb128 encoded value. */
2310 QEMU_BUILD_BUG_ON(FRAME_SIZE
>= (1 << 14));
2312 #if !defined(__ELF__)
2313 /* Host machine without ELF. */
2314 #elif TCG_TARGET_REG_BITS == 64
2315 #define ELF_HOST_MACHINE EM_X86_64
2316 static DebugFrame debug_frame
= {
2317 .cie
.len
= sizeof(DebugFrameCIE
)-4, /* length after .len member */
2320 .cie
.code_align
= 1,
2321 .cie
.data_align
= 0x78, /* sleb128 -8 */
2322 .cie
.return_column
= 16,
2324 /* Total FDE size does not include the "len" member. */
2325 .fde
.len
= sizeof(DebugFrame
) - offsetof(DebugFrame
, fde
.cie_offset
),
2328 12, 7, /* DW_CFA_def_cfa %rsp, ... */
2329 (FRAME_SIZE
& 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2333 0x90, 1, /* DW_CFA_offset, %rip, -8 */
2334 /* The following ordering must match tcg_target_callee_save_regs. */
2335 0x86, 2, /* DW_CFA_offset, %rbp, -16 */
2336 0x83, 3, /* DW_CFA_offset, %rbx, -24 */
2337 0x8c, 4, /* DW_CFA_offset, %r12, -32 */
2338 0x8d, 5, /* DW_CFA_offset, %r13, -40 */
2339 0x8e, 6, /* DW_CFA_offset, %r14, -48 */
2340 0x8f, 7, /* DW_CFA_offset, %r15, -56 */
2344 #define ELF_HOST_MACHINE EM_386
2345 static DebugFrame debug_frame
= {
2346 .cie
.len
= sizeof(DebugFrameCIE
)-4, /* length after .len member */
2349 .cie
.code_align
= 1,
2350 .cie
.data_align
= 0x7c, /* sleb128 -4 */
2351 .cie
.return_column
= 8,
2353 /* Total FDE size does not include the "len" member. */
2354 .fde
.len
= sizeof(DebugFrame
) - offsetof(DebugFrame
, fde
.cie_offset
),
2357 12, 4, /* DW_CFA_def_cfa %esp, ... */
2358 (FRAME_SIZE
& 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */
2362 0x88, 1, /* DW_CFA_offset, %eip, -4 */
2363 /* The following ordering must match tcg_target_callee_save_regs. */
2364 0x85, 2, /* DW_CFA_offset, %ebp, -8 */
2365 0x83, 3, /* DW_CFA_offset, %ebx, -12 */
2366 0x86, 4, /* DW_CFA_offset, %esi, -16 */
2367 0x87, 5, /* DW_CFA_offset, %edi, -20 */
2372 #if defined(ELF_HOST_MACHINE)
2373 void tcg_register_jit(void *buf
, size_t buf_size
)
2375 debug_frame
.fde
.func_start
= (tcg_target_long
) buf
;
2376 debug_frame
.fde
.func_len
= buf_size
;
2378 tcg_register_jit_int(buf
, buf_size
, &debug_frame
, sizeof(debug_frame
));