1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
37 #define OpImplicit 1ull /* No generic decode */
38 #define OpReg 2ull /* Register */
39 #define OpMem 3ull /* Memory */
40 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
41 #define OpDI 5ull /* ES:DI/EDI/RDI */
42 #define OpMem64 6ull /* Memory, 64-bit */
43 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
44 #define OpDX 8ull /* DX register */
45 #define OpCL 9ull /* CL register (for shifts) */
46 #define OpImmByte 10ull /* 8-bit sign extended immediate */
47 #define OpOne 11ull /* Implied 1 */
48 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
49 #define OpMem16 13ull /* Memory operand (16-bit). */
50 #define OpMem32 14ull /* Memory operand (32-bit). */
51 #define OpImmU 15ull /* Immediate operand, zero extended */
52 #define OpSI 16ull /* SI/ESI/RSI */
53 #define OpImmFAddr 17ull /* Immediate far address */
54 #define OpMemFAddr 18ull /* Far address in memory */
55 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
56 #define OpES 20ull /* ES */
57 #define OpCS 21ull /* CS */
58 #define OpSS 22ull /* SS */
59 #define OpDS 23ull /* DS */
60 #define OpFS 24ull /* FS */
61 #define OpGS 25ull /* GS */
62 #define OpMem8 26ull /* 8-bit zero extended memory operand */
63 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
66 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
68 #define OpBits 5 /* Width of operand field */
69 #define OpMask ((1ull << OpBits) - 1)
72 * Opcode effective-address decode tables.
73 * Note that we only emulate instructions that have at least one memory
74 * operand (excluding implicit stack references). We assume that stack
75 * references and instruction fetches will never occur in special memory
76 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
80 /* Operand sizes: 8-bit operands or specified/overridden size. */
81 #define ByteOp (1<<0) /* 8-bit operands. */
82 /* Destination operand type. */
84 #define ImplicitOps (OpImplicit << DstShift)
85 #define DstReg (OpReg << DstShift)
86 #define DstMem (OpMem << DstShift)
87 #define DstAcc (OpAcc << DstShift)
88 #define DstDI (OpDI << DstShift)
89 #define DstMem64 (OpMem64 << DstShift)
90 #define DstMem16 (OpMem16 << DstShift)
91 #define DstImmUByte (OpImmUByte << DstShift)
92 #define DstDX (OpDX << DstShift)
93 #define DstAccLo (OpAccLo << DstShift)
94 #define DstMask (OpMask << DstShift)
95 /* Source operand type. */
97 #define SrcNone (OpNone << SrcShift)
98 #define SrcReg (OpReg << SrcShift)
99 #define SrcMem (OpMem << SrcShift)
100 #define SrcMem16 (OpMem16 << SrcShift)
101 #define SrcMem32 (OpMem32 << SrcShift)
102 #define SrcImm (OpImm << SrcShift)
103 #define SrcImmByte (OpImmByte << SrcShift)
104 #define SrcOne (OpOne << SrcShift)
105 #define SrcImmUByte (OpImmUByte << SrcShift)
106 #define SrcImmU (OpImmU << SrcShift)
107 #define SrcSI (OpSI << SrcShift)
108 #define SrcXLat (OpXLat << SrcShift)
109 #define SrcImmFAddr (OpImmFAddr << SrcShift)
110 #define SrcMemFAddr (OpMemFAddr << SrcShift)
111 #define SrcAcc (OpAcc << SrcShift)
112 #define SrcImmU16 (OpImmU16 << SrcShift)
113 #define SrcImm64 (OpImm64 << SrcShift)
114 #define SrcDX (OpDX << SrcShift)
115 #define SrcMem8 (OpMem8 << SrcShift)
116 #define SrcAccHi (OpAccHi << SrcShift)
117 #define SrcMask (OpMask << SrcShift)
118 #define BitOp (1<<11)
119 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
120 #define String (1<<13) /* String instruction (rep capable) */
121 #define Stack (1<<14) /* Stack instruction (push/pop) */
122 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
123 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
124 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
125 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
126 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
127 #define Escape (5<<15) /* Escape to coprocessor instruction */
128 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
129 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
130 #define Sse (1<<18) /* SSE Vector instruction */
131 /* Generic ModRM decode. */
132 #define ModRM (1<<19)
133 /* Destination is only written; never read. */
136 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
137 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
138 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
139 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
140 #define Undefined (1<<25) /* No Such Instruction */
141 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
142 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
144 #define PageTable (1 << 29) /* instruction used to write page table */
145 #define NotImpl (1 << 30) /* instruction is not implemented */
146 /* Source 2 operand type */
147 #define Src2Shift (31)
148 #define Src2None (OpNone << Src2Shift)
149 #define Src2Mem (OpMem << Src2Shift)
150 #define Src2CL (OpCL << Src2Shift)
151 #define Src2ImmByte (OpImmByte << Src2Shift)
152 #define Src2One (OpOne << Src2Shift)
153 #define Src2Imm (OpImm << Src2Shift)
154 #define Src2ES (OpES << Src2Shift)
155 #define Src2CS (OpCS << Src2Shift)
156 #define Src2SS (OpSS << Src2Shift)
157 #define Src2DS (OpDS << Src2Shift)
158 #define Src2FS (OpFS << Src2Shift)
159 #define Src2GS (OpGS << Src2Shift)
160 #define Src2Mask (OpMask << Src2Shift)
161 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
162 #define AlignMask ((u64)7 << 41)
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
166 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
167 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
168 #define NoWrite ((u64)1 << 45) /* No writeback */
169 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
170 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
171 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
172 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
173 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
174 #define NearBranch ((u64)1 << 52) /* Near branches */
175 #define No16 ((u64)1 << 53) /* No 16 bit operand */
176 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
181 #define X2(x...) x, x
182 #define X3(x...) X2(x), x
183 #define X4(x...) X2(x), X2(x)
184 #define X5(x...) X4(x), x
185 #define X6(x...) X4(x), X2(x)
186 #define X7(x...) X4(x), X3(x)
187 #define X8(x...) X4(x), X4(x)
188 #define X16(x...) X8(x), X8(x)
190 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
191 #define FASTOP_SIZE 8
194 * fastop functions have a special calling convention:
199 * flags: rflags (in/out)
200 * ex: rsi (in:fastop pointer, out:zero if exception)
202 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
203 * different operand sizes can be reached by calculation, rather than a jump
204 * table (which would be bigger than the code).
206 * fastop functions are declared as taking a never-defined fastop parameter,
207 * so they can't be called from C directly.
216 int (*execute
)(struct x86_emulate_ctxt
*ctxt
);
217 const struct opcode
*group
;
218 const struct group_dual
*gdual
;
219 const struct gprefix
*gprefix
;
220 const struct escape
*esc
;
221 const struct instr_dual
*idual
;
222 const struct mode_dual
*mdual
;
223 void (*fastop
)(struct fastop
*fake
);
225 int (*check_perm
)(struct x86_emulate_ctxt
*ctxt
);
229 struct opcode mod012
[8];
230 struct opcode mod3
[8];
234 struct opcode pfx_no
;
235 struct opcode pfx_66
;
236 struct opcode pfx_f2
;
237 struct opcode pfx_f3
;
242 struct opcode high
[64];
246 struct opcode mod012
;
251 struct opcode mode32
;
252 struct opcode mode64
;
255 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
257 enum x86_transfer_type
{
259 X86_TRANSFER_CALL_JMP
,
261 X86_TRANSFER_TASK_SWITCH
,
264 static ulong
reg_read(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
266 if (!(ctxt
->regs_valid
& (1 << nr
))) {
267 ctxt
->regs_valid
|= 1 << nr
;
268 ctxt
->_regs
[nr
] = ctxt
->ops
->read_gpr(ctxt
, nr
);
270 return ctxt
->_regs
[nr
];
273 static ulong
*reg_write(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
275 ctxt
->regs_valid
|= 1 << nr
;
276 ctxt
->regs_dirty
|= 1 << nr
;
277 return &ctxt
->_regs
[nr
];
280 static ulong
*reg_rmw(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
283 return reg_write(ctxt
, nr
);
286 static void writeback_registers(struct x86_emulate_ctxt
*ctxt
)
290 for_each_set_bit(reg
, (ulong
*)&ctxt
->regs_dirty
, 16)
291 ctxt
->ops
->write_gpr(ctxt
, reg
, ctxt
->_regs
[reg
]);
294 static void invalidate_registers(struct x86_emulate_ctxt
*ctxt
)
296 ctxt
->regs_dirty
= 0;
297 ctxt
->regs_valid
= 0;
301 * These EFLAGS bits are restored from saved value during emulation, and
302 * any changes are written back to the saved value after emulation.
304 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
305 X86_EFLAGS_PF|X86_EFLAGS_CF)
313 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*));
315 #define FOP_FUNC(name) \
316 ".align " __stringify(FASTOP_SIZE) " \n\t" \
317 ".type " name ", @function \n\t" \
320 #define FOP_RET "ret \n\t"
322 #define FOP_START(op) \
323 extern void em_##op(struct fastop *fake); \
324 asm(".pushsection .text, \"ax\" \n\t" \
325 ".global em_" #op " \n\t" \
332 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
335 #define FOP1E(op, dst) \
336 FOP_FUNC(#op "_" #dst) \
337 "10: " #op " %" #dst " \n\t" FOP_RET
339 #define FOP1EEX(op, dst) \
340 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
342 #define FASTOP1(op) \
347 ON64(FOP1E(op##q, rax)) \
350 /* 1-operand, using src2 (for MUL/DIV r/m) */
351 #define FASTOP1SRC2(op, name) \
356 ON64(FOP1E(op, rcx)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
360 #define FASTOP1SRC2EX(op, name) \
365 ON64(FOP1EEX(op, rcx)) \
368 #define FOP2E(op, dst, src) \
369 FOP_FUNC(#op "_" #dst "_" #src) \
370 #op " %" #src ", %" #dst " \n\t" FOP_RET
372 #define FASTOP2(op) \
374 FOP2E(op##b, al, dl) \
375 FOP2E(op##w, ax, dx) \
376 FOP2E(op##l, eax, edx) \
377 ON64(FOP2E(op##q, rax, rdx)) \
380 /* 2 operand, word only */
381 #define FASTOP2W(op) \
384 FOP2E(op##w, ax, dx) \
385 FOP2E(op##l, eax, edx) \
386 ON64(FOP2E(op##q, rax, rdx)) \
389 /* 2 operand, src is CL */
390 #define FASTOP2CL(op) \
392 FOP2E(op##b, al, cl) \
393 FOP2E(op##w, ax, cl) \
394 FOP2E(op##l, eax, cl) \
395 ON64(FOP2E(op##q, rax, cl)) \
398 /* 2 operand, src and dest are reversed */
399 #define FASTOP2R(op, name) \
401 FOP2E(op##b, dl, al) \
402 FOP2E(op##w, dx, ax) \
403 FOP2E(op##l, edx, eax) \
404 ON64(FOP2E(op##q, rdx, rax)) \
407 #define FOP3E(op, dst, src, src2) \
408 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
409 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
411 /* 3-operand, word-only, src2=cl */
412 #define FASTOP3WCL(op) \
415 FOP3E(op##w, ax, dx, cl) \
416 FOP3E(op##l, eax, edx, cl) \
417 ON64(FOP3E(op##q, rax, rdx, cl)) \
420 /* Special case for SETcc - 1 instruction per cc */
421 #define FOP_SETCC(op) \
423 ".type " #op ", @function \n\t" \
428 asm(".pushsection .fixup, \"ax\"\n"
429 ".global kvm_fastop_exception \n"
430 "kvm_fastop_exception: xor %esi, %esi; ret\n"
452 FOP_START(salc
) "pushf; sbb %al, %al; popf \n\t" FOP_RET
456 * XXX: inoutclob user must know where the argument is being expanded.
457 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
459 #define asm_safe(insn, inoutclob...) \
463 asm volatile("1:" insn "\n" \
465 ".pushsection .fixup, \"ax\"\n" \
466 "3: movl $1, %[_fault]\n" \
469 _ASM_EXTABLE(1b, 3b) \
470 : [_fault] "+qm"(_fault) inoutclob ); \
472 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
475 static int emulator_check_intercept(struct x86_emulate_ctxt
*ctxt
,
476 enum x86_intercept intercept
,
477 enum x86_intercept_stage stage
)
479 struct x86_instruction_info info
= {
480 .intercept
= intercept
,
481 .rep_prefix
= ctxt
->rep_prefix
,
482 .modrm_mod
= ctxt
->modrm_mod
,
483 .modrm_reg
= ctxt
->modrm_reg
,
484 .modrm_rm
= ctxt
->modrm_rm
,
485 .src_val
= ctxt
->src
.val64
,
486 .dst_val
= ctxt
->dst
.val64
,
487 .src_bytes
= ctxt
->src
.bytes
,
488 .dst_bytes
= ctxt
->dst
.bytes
,
489 .ad_bytes
= ctxt
->ad_bytes
,
490 .next_rip
= ctxt
->eip
,
493 return ctxt
->ops
->intercept(ctxt
, &info
, stage
);
496 static void assign_masked(ulong
*dest
, ulong src
, ulong mask
)
498 *dest
= (*dest
& ~mask
) | (src
& mask
);
501 static void assign_register(unsigned long *reg
, u64 val
, int bytes
)
503 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
506 *(u8
*)reg
= (u8
)val
;
509 *(u16
*)reg
= (u16
)val
;
513 break; /* 64b: zero-extend */
520 static inline unsigned long ad_mask(struct x86_emulate_ctxt
*ctxt
)
522 return (1UL << (ctxt
->ad_bytes
<< 3)) - 1;
525 static ulong
stack_mask(struct x86_emulate_ctxt
*ctxt
)
528 struct desc_struct ss
;
530 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
532 ctxt
->ops
->get_segment(ctxt
, &sel
, &ss
, NULL
, VCPU_SREG_SS
);
533 return ~0U >> ((ss
.d
^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
536 static int stack_size(struct x86_emulate_ctxt
*ctxt
)
538 return (__fls(stack_mask(ctxt
)) + 1) >> 3;
541 /* Access/update address held in a register, based on addressing mode. */
542 static inline unsigned long
543 address_mask(struct x86_emulate_ctxt
*ctxt
, unsigned long reg
)
545 if (ctxt
->ad_bytes
== sizeof(unsigned long))
548 return reg
& ad_mask(ctxt
);
551 static inline unsigned long
552 register_address(struct x86_emulate_ctxt
*ctxt
, int reg
)
554 return address_mask(ctxt
, reg_read(ctxt
, reg
));
557 static void masked_increment(ulong
*reg
, ulong mask
, int inc
)
559 assign_masked(reg
, *reg
+ inc
, mask
);
563 register_address_increment(struct x86_emulate_ctxt
*ctxt
, int reg
, int inc
)
565 ulong
*preg
= reg_rmw(ctxt
, reg
);
567 assign_register(preg
, *preg
+ inc
, ctxt
->ad_bytes
);
570 static void rsp_increment(struct x86_emulate_ctxt
*ctxt
, int inc
)
572 masked_increment(reg_rmw(ctxt
, VCPU_REGS_RSP
), stack_mask(ctxt
), inc
);
575 static u32
desc_limit_scaled(struct desc_struct
*desc
)
577 u32 limit
= get_desc_limit(desc
);
579 return desc
->g
? (limit
<< 12) | 0xfff : limit
;
582 static unsigned long seg_base(struct x86_emulate_ctxt
*ctxt
, int seg
)
584 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&& seg
< VCPU_SREG_FS
)
587 return ctxt
->ops
->get_cached_segment_base(ctxt
, seg
);
590 static int emulate_exception(struct x86_emulate_ctxt
*ctxt
, int vec
,
591 u32 error
, bool valid
)
594 ctxt
->exception
.vector
= vec
;
595 ctxt
->exception
.error_code
= error
;
596 ctxt
->exception
.error_code_valid
= valid
;
597 return X86EMUL_PROPAGATE_FAULT
;
600 static int emulate_db(struct x86_emulate_ctxt
*ctxt
)
602 return emulate_exception(ctxt
, DB_VECTOR
, 0, false);
605 static int emulate_gp(struct x86_emulate_ctxt
*ctxt
, int err
)
607 return emulate_exception(ctxt
, GP_VECTOR
, err
, true);
610 static int emulate_ss(struct x86_emulate_ctxt
*ctxt
, int err
)
612 return emulate_exception(ctxt
, SS_VECTOR
, err
, true);
615 static int emulate_ud(struct x86_emulate_ctxt
*ctxt
)
617 return emulate_exception(ctxt
, UD_VECTOR
, 0, false);
620 static int emulate_ts(struct x86_emulate_ctxt
*ctxt
, int err
)
622 return emulate_exception(ctxt
, TS_VECTOR
, err
, true);
625 static int emulate_de(struct x86_emulate_ctxt
*ctxt
)
627 return emulate_exception(ctxt
, DE_VECTOR
, 0, false);
630 static int emulate_nm(struct x86_emulate_ctxt
*ctxt
)
632 return emulate_exception(ctxt
, NM_VECTOR
, 0, false);
635 static u16
get_segment_selector(struct x86_emulate_ctxt
*ctxt
, unsigned seg
)
638 struct desc_struct desc
;
640 ctxt
->ops
->get_segment(ctxt
, &selector
, &desc
, NULL
, seg
);
644 static void set_segment_selector(struct x86_emulate_ctxt
*ctxt
, u16 selector
,
649 struct desc_struct desc
;
651 ctxt
->ops
->get_segment(ctxt
, &dummy
, &desc
, &base3
, seg
);
652 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, seg
);
656 * x86 defines three classes of vector instructions: explicitly
657 * aligned, explicitly unaligned, and the rest, which change behaviour
658 * depending on whether they're AVX encoded or not.
660 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
661 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
662 * 512 bytes of data must be aligned to a 16 byte boundary.
664 static unsigned insn_alignment(struct x86_emulate_ctxt
*ctxt
, unsigned size
)
666 u64 alignment
= ctxt
->d
& AlignMask
;
668 if (likely(size
< 16))
683 static __always_inline
int __linearize(struct x86_emulate_ctxt
*ctxt
,
684 struct segmented_address addr
,
685 unsigned *max_size
, unsigned size
,
686 bool write
, bool fetch
,
687 enum x86emul_mode mode
, ulong
*linear
)
689 struct desc_struct desc
;
696 la
= seg_base(ctxt
, addr
.seg
) + addr
.ea
;
699 case X86EMUL_MODE_PROT64
:
701 va_bits
= ctxt_virt_addr_bits(ctxt
);
702 if (get_canonical(la
, va_bits
) != la
)
705 *max_size
= min_t(u64
, ~0u, (1ull << va_bits
) - la
);
706 if (size
> *max_size
)
710 *linear
= la
= (u32
)la
;
711 usable
= ctxt
->ops
->get_segment(ctxt
, &sel
, &desc
, NULL
,
715 /* code segment in protected mode or read-only data segment */
716 if ((((ctxt
->mode
!= X86EMUL_MODE_REAL
) && (desc
.type
& 8))
717 || !(desc
.type
& 2)) && write
)
719 /* unreadable code segment */
720 if (!fetch
&& (desc
.type
& 8) && !(desc
.type
& 2))
722 lim
= desc_limit_scaled(&desc
);
723 if (!(desc
.type
& 8) && (desc
.type
& 4)) {
724 /* expand-down segment */
727 lim
= desc
.d
? 0xffffffff : 0xffff;
731 if (lim
== 0xffffffff)
734 *max_size
= (u64
)lim
+ 1 - addr
.ea
;
735 if (size
> *max_size
)
740 if (la
& (insn_alignment(ctxt
, size
) - 1))
741 return emulate_gp(ctxt
, 0);
742 return X86EMUL_CONTINUE
;
744 if (addr
.seg
== VCPU_SREG_SS
)
745 return emulate_ss(ctxt
, 0);
747 return emulate_gp(ctxt
, 0);
750 static int linearize(struct x86_emulate_ctxt
*ctxt
,
751 struct segmented_address addr
,
752 unsigned size
, bool write
,
756 return __linearize(ctxt
, addr
, &max_size
, size
, write
, false,
760 static inline int assign_eip(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
761 enum x86emul_mode mode
)
766 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
769 if (ctxt
->op_bytes
!= sizeof(unsigned long))
770 addr
.ea
= dst
& ((1UL << (ctxt
->op_bytes
<< 3)) - 1);
771 rc
= __linearize(ctxt
, addr
, &max_size
, 1, false, true, mode
, &linear
);
772 if (rc
== X86EMUL_CONTINUE
)
773 ctxt
->_eip
= addr
.ea
;
777 static inline int assign_eip_near(struct x86_emulate_ctxt
*ctxt
, ulong dst
)
779 return assign_eip(ctxt
, dst
, ctxt
->mode
);
782 static int assign_eip_far(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
783 const struct desc_struct
*cs_desc
)
785 enum x86emul_mode mode
= ctxt
->mode
;
789 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
) {
793 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
795 mode
= X86EMUL_MODE_PROT64
;
797 mode
= X86EMUL_MODE_PROT32
; /* temporary value */
800 if (mode
== X86EMUL_MODE_PROT16
|| mode
== X86EMUL_MODE_PROT32
)
801 mode
= cs_desc
->d
? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
802 rc
= assign_eip(ctxt
, dst
, mode
);
803 if (rc
== X86EMUL_CONTINUE
)
808 static inline int jmp_rel(struct x86_emulate_ctxt
*ctxt
, int rel
)
810 return assign_eip_near(ctxt
, ctxt
->_eip
+ rel
);
813 static int segmented_read_std(struct x86_emulate_ctxt
*ctxt
,
814 struct segmented_address addr
,
821 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
822 if (rc
!= X86EMUL_CONTINUE
)
824 return ctxt
->ops
->read_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
827 static int segmented_write_std(struct x86_emulate_ctxt
*ctxt
,
828 struct segmented_address addr
,
835 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
836 if (rc
!= X86EMUL_CONTINUE
)
838 return ctxt
->ops
->write_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
842 * Prefetch the remaining bytes of the instruction without crossing page
843 * boundary if they are not in fetch_cache yet.
845 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
, int op_size
)
848 unsigned size
, max_size
;
849 unsigned long linear
;
850 int cur_size
= ctxt
->fetch
.end
- ctxt
->fetch
.data
;
851 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
852 .ea
= ctxt
->eip
+ cur_size
};
855 * We do not know exactly how many bytes will be needed, and
856 * __linearize is expensive, so fetch as much as possible. We
857 * just have to avoid going beyond the 15 byte limit, the end
858 * of the segment, or the end of the page.
860 * __linearize is called with size 0 so that it does not do any
861 * boundary check itself. Instead, we use max_size to check
864 rc
= __linearize(ctxt
, addr
, &max_size
, 0, false, true, ctxt
->mode
,
866 if (unlikely(rc
!= X86EMUL_CONTINUE
))
869 size
= min_t(unsigned, 15UL ^ cur_size
, max_size
);
870 size
= min_t(unsigned, size
, PAGE_SIZE
- offset_in_page(linear
));
873 * One instruction can only straddle two pages,
874 * and one has been loaded at the beginning of
875 * x86_decode_insn. So, if not enough bytes
876 * still, we must have hit the 15-byte boundary.
878 if (unlikely(size
< op_size
))
879 return emulate_gp(ctxt
, 0);
881 rc
= ctxt
->ops
->fetch(ctxt
, linear
, ctxt
->fetch
.end
,
882 size
, &ctxt
->exception
);
883 if (unlikely(rc
!= X86EMUL_CONTINUE
))
885 ctxt
->fetch
.end
+= size
;
886 return X86EMUL_CONTINUE
;
889 static __always_inline
int do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
,
892 unsigned done_size
= ctxt
->fetch
.end
- ctxt
->fetch
.ptr
;
894 if (unlikely(done_size
< size
))
895 return __do_insn_fetch_bytes(ctxt
, size
- done_size
);
897 return X86EMUL_CONTINUE
;
900 /* Fetch next part of the instruction being emulated. */
901 #define insn_fetch(_type, _ctxt) \
904 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
905 if (rc != X86EMUL_CONTINUE) \
907 ctxt->_eip += sizeof(_type); \
908 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
909 ctxt->fetch.ptr += sizeof(_type); \
913 #define insn_fetch_arr(_arr, _size, _ctxt) \
915 rc = do_insn_fetch_bytes(_ctxt, _size); \
916 if (rc != X86EMUL_CONTINUE) \
918 ctxt->_eip += (_size); \
919 memcpy(_arr, ctxt->fetch.ptr, _size); \
920 ctxt->fetch.ptr += (_size); \
924 * Given the 'reg' portion of a ModRM byte, and a register block, return a
925 * pointer into the block that addresses the relevant register.
926 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
928 static void *decode_register(struct x86_emulate_ctxt
*ctxt
, u8 modrm_reg
,
932 int highbyte_regs
= (ctxt
->rex_prefix
== 0) && byteop
;
934 if (highbyte_regs
&& modrm_reg
>= 4 && modrm_reg
< 8)
935 p
= (unsigned char *)reg_rmw(ctxt
, modrm_reg
& 3) + 1;
937 p
= reg_rmw(ctxt
, modrm_reg
);
941 static int read_descriptor(struct x86_emulate_ctxt
*ctxt
,
942 struct segmented_address addr
,
943 u16
*size
, unsigned long *address
, int op_bytes
)
950 rc
= segmented_read_std(ctxt
, addr
, size
, 2);
951 if (rc
!= X86EMUL_CONTINUE
)
954 rc
= segmented_read_std(ctxt
, addr
, address
, op_bytes
);
968 FASTOP1SRC2(mul
, mul_ex
);
969 FASTOP1SRC2(imul
, imul_ex
);
970 FASTOP1SRC2EX(div
, div_ex
);
971 FASTOP1SRC2EX(idiv
, idiv_ex
);
1000 FASTOP2R(cmp
, cmp_r
);
1002 static int em_bsf_c(struct x86_emulate_ctxt
*ctxt
)
1004 /* If src is zero, do not writeback, but update flags */
1005 if (ctxt
->src
.val
== 0)
1006 ctxt
->dst
.type
= OP_NONE
;
1007 return fastop(ctxt
, em_bsf
);
1010 static int em_bsr_c(struct x86_emulate_ctxt
*ctxt
)
1012 /* If src is zero, do not writeback, but update flags */
1013 if (ctxt
->src
.val
== 0)
1014 ctxt
->dst
.type
= OP_NONE
;
1015 return fastop(ctxt
, em_bsr
);
1018 static __always_inline u8
test_cc(unsigned int condition
, unsigned long flags
)
1021 void (*fop
)(void) = (void *)em_setcc
+ 4 * (condition
& 0xf);
1023 flags
= (flags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
1024 asm("push %[flags]; popf; call *%[fastop]"
1025 : "=a"(rc
) : [fastop
]"r"(fop
), [flags
]"r"(flags
));
1029 static void fetch_register_operand(struct operand
*op
)
1031 switch (op
->bytes
) {
1033 op
->val
= *(u8
*)op
->addr
.reg
;
1036 op
->val
= *(u16
*)op
->addr
.reg
;
1039 op
->val
= *(u32
*)op
->addr
.reg
;
1042 op
->val
= *(u64
*)op
->addr
.reg
;
1047 static void read_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
, int reg
)
1049 ctxt
->ops
->get_fpu(ctxt
);
1051 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data
)); break;
1052 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data
)); break;
1053 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data
)); break;
1054 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data
)); break;
1055 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data
)); break;
1056 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data
)); break;
1057 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data
)); break;
1058 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data
)); break;
1059 #ifdef CONFIG_X86_64
1060 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data
)); break;
1061 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data
)); break;
1062 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data
)); break;
1063 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data
)); break;
1064 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data
)); break;
1065 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data
)); break;
1066 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data
)); break;
1067 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data
)); break;
1071 ctxt
->ops
->put_fpu(ctxt
);
1074 static void write_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
,
1077 ctxt
->ops
->get_fpu(ctxt
);
1079 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data
)); break;
1080 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data
)); break;
1081 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data
)); break;
1082 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data
)); break;
1083 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data
)); break;
1084 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data
)); break;
1085 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data
)); break;
1086 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data
)); break;
1087 #ifdef CONFIG_X86_64
1088 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data
)); break;
1089 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data
)); break;
1090 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data
)); break;
1091 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data
)); break;
1092 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data
)); break;
1093 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data
)); break;
1094 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data
)); break;
1095 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data
)); break;
1099 ctxt
->ops
->put_fpu(ctxt
);
1102 static void read_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1104 ctxt
->ops
->get_fpu(ctxt
);
1106 case 0: asm("movq %%mm0, %0" : "=m"(*data
)); break;
1107 case 1: asm("movq %%mm1, %0" : "=m"(*data
)); break;
1108 case 2: asm("movq %%mm2, %0" : "=m"(*data
)); break;
1109 case 3: asm("movq %%mm3, %0" : "=m"(*data
)); break;
1110 case 4: asm("movq %%mm4, %0" : "=m"(*data
)); break;
1111 case 5: asm("movq %%mm5, %0" : "=m"(*data
)); break;
1112 case 6: asm("movq %%mm6, %0" : "=m"(*data
)); break;
1113 case 7: asm("movq %%mm7, %0" : "=m"(*data
)); break;
1116 ctxt
->ops
->put_fpu(ctxt
);
1119 static void write_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1121 ctxt
->ops
->get_fpu(ctxt
);
1123 case 0: asm("movq %0, %%mm0" : : "m"(*data
)); break;
1124 case 1: asm("movq %0, %%mm1" : : "m"(*data
)); break;
1125 case 2: asm("movq %0, %%mm2" : : "m"(*data
)); break;
1126 case 3: asm("movq %0, %%mm3" : : "m"(*data
)); break;
1127 case 4: asm("movq %0, %%mm4" : : "m"(*data
)); break;
1128 case 5: asm("movq %0, %%mm5" : : "m"(*data
)); break;
1129 case 6: asm("movq %0, %%mm6" : : "m"(*data
)); break;
1130 case 7: asm("movq %0, %%mm7" : : "m"(*data
)); break;
1133 ctxt
->ops
->put_fpu(ctxt
);
1136 static int em_fninit(struct x86_emulate_ctxt
*ctxt
)
1138 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1139 return emulate_nm(ctxt
);
1141 ctxt
->ops
->get_fpu(ctxt
);
1142 asm volatile("fninit");
1143 ctxt
->ops
->put_fpu(ctxt
);
1144 return X86EMUL_CONTINUE
;
1147 static int em_fnstcw(struct x86_emulate_ctxt
*ctxt
)
1151 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1152 return emulate_nm(ctxt
);
1154 ctxt
->ops
->get_fpu(ctxt
);
1155 asm volatile("fnstcw %0": "+m"(fcw
));
1156 ctxt
->ops
->put_fpu(ctxt
);
1158 ctxt
->dst
.val
= fcw
;
1160 return X86EMUL_CONTINUE
;
1163 static int em_fnstsw(struct x86_emulate_ctxt
*ctxt
)
1167 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1168 return emulate_nm(ctxt
);
1170 ctxt
->ops
->get_fpu(ctxt
);
1171 asm volatile("fnstsw %0": "+m"(fsw
));
1172 ctxt
->ops
->put_fpu(ctxt
);
1174 ctxt
->dst
.val
= fsw
;
1176 return X86EMUL_CONTINUE
;
1179 static void decode_register_operand(struct x86_emulate_ctxt
*ctxt
,
1182 unsigned reg
= ctxt
->modrm_reg
;
1184 if (!(ctxt
->d
& ModRM
))
1185 reg
= (ctxt
->b
& 7) | ((ctxt
->rex_prefix
& 1) << 3);
1187 if (ctxt
->d
& Sse
) {
1191 read_sse_reg(ctxt
, &op
->vec_val
, reg
);
1194 if (ctxt
->d
& Mmx
) {
1203 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1204 op
->addr
.reg
= decode_register(ctxt
, reg
, ctxt
->d
& ByteOp
);
1206 fetch_register_operand(op
);
1207 op
->orig_val
= op
->val
;
1210 static void adjust_modrm_seg(struct x86_emulate_ctxt
*ctxt
, int base_reg
)
1212 if (base_reg
== VCPU_REGS_RSP
|| base_reg
== VCPU_REGS_RBP
)
1213 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1216 static int decode_modrm(struct x86_emulate_ctxt
*ctxt
,
1220 int index_reg
, base_reg
, scale
;
1221 int rc
= X86EMUL_CONTINUE
;
1224 ctxt
->modrm_reg
= ((ctxt
->rex_prefix
<< 1) & 8); /* REX.R */
1225 index_reg
= (ctxt
->rex_prefix
<< 2) & 8; /* REX.X */
1226 base_reg
= (ctxt
->rex_prefix
<< 3) & 8; /* REX.B */
1228 ctxt
->modrm_mod
= (ctxt
->modrm
& 0xc0) >> 6;
1229 ctxt
->modrm_reg
|= (ctxt
->modrm
& 0x38) >> 3;
1230 ctxt
->modrm_rm
= base_reg
| (ctxt
->modrm
& 0x07);
1231 ctxt
->modrm_seg
= VCPU_SREG_DS
;
1233 if (ctxt
->modrm_mod
== 3 || (ctxt
->d
& NoMod
)) {
1235 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1236 op
->addr
.reg
= decode_register(ctxt
, ctxt
->modrm_rm
,
1238 if (ctxt
->d
& Sse
) {
1241 op
->addr
.xmm
= ctxt
->modrm_rm
;
1242 read_sse_reg(ctxt
, &op
->vec_val
, ctxt
->modrm_rm
);
1245 if (ctxt
->d
& Mmx
) {
1248 op
->addr
.mm
= ctxt
->modrm_rm
& 7;
1251 fetch_register_operand(op
);
1257 if (ctxt
->ad_bytes
== 2) {
1258 unsigned bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
1259 unsigned bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1260 unsigned si
= reg_read(ctxt
, VCPU_REGS_RSI
);
1261 unsigned di
= reg_read(ctxt
, VCPU_REGS_RDI
);
1263 /* 16-bit ModR/M decode. */
1264 switch (ctxt
->modrm_mod
) {
1266 if (ctxt
->modrm_rm
== 6)
1267 modrm_ea
+= insn_fetch(u16
, ctxt
);
1270 modrm_ea
+= insn_fetch(s8
, ctxt
);
1273 modrm_ea
+= insn_fetch(u16
, ctxt
);
1276 switch (ctxt
->modrm_rm
) {
1278 modrm_ea
+= bx
+ si
;
1281 modrm_ea
+= bx
+ di
;
1284 modrm_ea
+= bp
+ si
;
1287 modrm_ea
+= bp
+ di
;
1296 if (ctxt
->modrm_mod
!= 0)
1303 if (ctxt
->modrm_rm
== 2 || ctxt
->modrm_rm
== 3 ||
1304 (ctxt
->modrm_rm
== 6 && ctxt
->modrm_mod
!= 0))
1305 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1306 modrm_ea
= (u16
)modrm_ea
;
1308 /* 32/64-bit ModR/M decode. */
1309 if ((ctxt
->modrm_rm
& 7) == 4) {
1310 sib
= insn_fetch(u8
, ctxt
);
1311 index_reg
|= (sib
>> 3) & 7;
1312 base_reg
|= sib
& 7;
1315 if ((base_reg
& 7) == 5 && ctxt
->modrm_mod
== 0)
1316 modrm_ea
+= insn_fetch(s32
, ctxt
);
1318 modrm_ea
+= reg_read(ctxt
, base_reg
);
1319 adjust_modrm_seg(ctxt
, base_reg
);
1320 /* Increment ESP on POP [ESP] */
1321 if ((ctxt
->d
& IncSP
) &&
1322 base_reg
== VCPU_REGS_RSP
)
1323 modrm_ea
+= ctxt
->op_bytes
;
1326 modrm_ea
+= reg_read(ctxt
, index_reg
) << scale
;
1327 } else if ((ctxt
->modrm_rm
& 7) == 5 && ctxt
->modrm_mod
== 0) {
1328 modrm_ea
+= insn_fetch(s32
, ctxt
);
1329 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
1330 ctxt
->rip_relative
= 1;
1332 base_reg
= ctxt
->modrm_rm
;
1333 modrm_ea
+= reg_read(ctxt
, base_reg
);
1334 adjust_modrm_seg(ctxt
, base_reg
);
1336 switch (ctxt
->modrm_mod
) {
1338 modrm_ea
+= insn_fetch(s8
, ctxt
);
1341 modrm_ea
+= insn_fetch(s32
, ctxt
);
1345 op
->addr
.mem
.ea
= modrm_ea
;
1346 if (ctxt
->ad_bytes
!= 8)
1347 ctxt
->memop
.addr
.mem
.ea
= (u32
)ctxt
->memop
.addr
.mem
.ea
;
1353 static int decode_abs(struct x86_emulate_ctxt
*ctxt
,
1356 int rc
= X86EMUL_CONTINUE
;
1359 switch (ctxt
->ad_bytes
) {
1361 op
->addr
.mem
.ea
= insn_fetch(u16
, ctxt
);
1364 op
->addr
.mem
.ea
= insn_fetch(u32
, ctxt
);
1367 op
->addr
.mem
.ea
= insn_fetch(u64
, ctxt
);
1374 static void fetch_bit_operand(struct x86_emulate_ctxt
*ctxt
)
1378 if (ctxt
->dst
.type
== OP_MEM
&& ctxt
->src
.type
== OP_REG
) {
1379 mask
= ~((long)ctxt
->dst
.bytes
* 8 - 1);
1381 if (ctxt
->src
.bytes
== 2)
1382 sv
= (s16
)ctxt
->src
.val
& (s16
)mask
;
1383 else if (ctxt
->src
.bytes
== 4)
1384 sv
= (s32
)ctxt
->src
.val
& (s32
)mask
;
1386 sv
= (s64
)ctxt
->src
.val
& (s64
)mask
;
1388 ctxt
->dst
.addr
.mem
.ea
= address_mask(ctxt
,
1389 ctxt
->dst
.addr
.mem
.ea
+ (sv
>> 3));
1392 /* only subword offset */
1393 ctxt
->src
.val
&= (ctxt
->dst
.bytes
<< 3) - 1;
1396 static int read_emulated(struct x86_emulate_ctxt
*ctxt
,
1397 unsigned long addr
, void *dest
, unsigned size
)
1400 struct read_cache
*mc
= &ctxt
->mem_read
;
1402 if (mc
->pos
< mc
->end
)
1405 WARN_ON((mc
->end
+ size
) >= sizeof(mc
->data
));
1407 rc
= ctxt
->ops
->read_emulated(ctxt
, addr
, mc
->data
+ mc
->end
, size
,
1409 if (rc
!= X86EMUL_CONTINUE
)
1415 memcpy(dest
, mc
->data
+ mc
->pos
, size
);
1417 return X86EMUL_CONTINUE
;
1420 static int segmented_read(struct x86_emulate_ctxt
*ctxt
,
1421 struct segmented_address addr
,
1428 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
1429 if (rc
!= X86EMUL_CONTINUE
)
1431 return read_emulated(ctxt
, linear
, data
, size
);
1434 static int segmented_write(struct x86_emulate_ctxt
*ctxt
,
1435 struct segmented_address addr
,
1442 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1443 if (rc
!= X86EMUL_CONTINUE
)
1445 return ctxt
->ops
->write_emulated(ctxt
, linear
, data
, size
,
1449 static int segmented_cmpxchg(struct x86_emulate_ctxt
*ctxt
,
1450 struct segmented_address addr
,
1451 const void *orig_data
, const void *data
,
1457 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1458 if (rc
!= X86EMUL_CONTINUE
)
1460 return ctxt
->ops
->cmpxchg_emulated(ctxt
, linear
, orig_data
, data
,
1461 size
, &ctxt
->exception
);
1464 static int pio_in_emulated(struct x86_emulate_ctxt
*ctxt
,
1465 unsigned int size
, unsigned short port
,
1468 struct read_cache
*rc
= &ctxt
->io_read
;
1470 if (rc
->pos
== rc
->end
) { /* refill pio read ahead */
1471 unsigned int in_page
, n
;
1472 unsigned int count
= ctxt
->rep_prefix
?
1473 address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) : 1;
1474 in_page
= (ctxt
->eflags
& X86_EFLAGS_DF
) ?
1475 offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
)) :
1476 PAGE_SIZE
- offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
));
1477 n
= min3(in_page
, (unsigned int)sizeof(rc
->data
) / size
, count
);
1480 rc
->pos
= rc
->end
= 0;
1481 if (!ctxt
->ops
->pio_in_emulated(ctxt
, size
, port
, rc
->data
, n
))
1486 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
) &&
1487 !(ctxt
->eflags
& X86_EFLAGS_DF
)) {
1488 ctxt
->dst
.data
= rc
->data
+ rc
->pos
;
1489 ctxt
->dst
.type
= OP_MEM_STR
;
1490 ctxt
->dst
.count
= (rc
->end
- rc
->pos
) / size
;
1493 memcpy(dest
, rc
->data
+ rc
->pos
, size
);
1499 static int read_interrupt_descriptor(struct x86_emulate_ctxt
*ctxt
,
1500 u16 index
, struct desc_struct
*desc
)
1505 ctxt
->ops
->get_idt(ctxt
, &dt
);
1507 if (dt
.size
< index
* 8 + 7)
1508 return emulate_gp(ctxt
, index
<< 3 | 0x2);
1510 addr
= dt
.address
+ index
* 8;
1511 return ctxt
->ops
->read_std(ctxt
, addr
, desc
, sizeof *desc
,
1515 static void get_descriptor_table_ptr(struct x86_emulate_ctxt
*ctxt
,
1516 u16 selector
, struct desc_ptr
*dt
)
1518 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1521 if (selector
& 1 << 2) {
1522 struct desc_struct desc
;
1525 memset (dt
, 0, sizeof *dt
);
1526 if (!ops
->get_segment(ctxt
, &sel
, &desc
, &base3
,
1530 dt
->size
= desc_limit_scaled(&desc
); /* what if limit > 65535? */
1531 dt
->address
= get_desc_base(&desc
) | ((u64
)base3
<< 32);
1533 ops
->get_gdt(ctxt
, dt
);
1536 static int get_descriptor_ptr(struct x86_emulate_ctxt
*ctxt
,
1537 u16 selector
, ulong
*desc_addr_p
)
1540 u16 index
= selector
>> 3;
1543 get_descriptor_table_ptr(ctxt
, selector
, &dt
);
1545 if (dt
.size
< index
* 8 + 7)
1546 return emulate_gp(ctxt
, selector
& 0xfffc);
1548 addr
= dt
.address
+ index
* 8;
1550 #ifdef CONFIG_X86_64
1551 if (addr
>> 32 != 0) {
1554 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1555 if (!(efer
& EFER_LMA
))
1560 *desc_addr_p
= addr
;
1561 return X86EMUL_CONTINUE
;
1564 /* allowed just for 8 bytes segments */
1565 static int read_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1566 u16 selector
, struct desc_struct
*desc
,
1571 rc
= get_descriptor_ptr(ctxt
, selector
, desc_addr_p
);
1572 if (rc
!= X86EMUL_CONTINUE
)
1575 return ctxt
->ops
->read_std(ctxt
, *desc_addr_p
, desc
, sizeof(*desc
),
1579 /* allowed just for 8 bytes segments */
1580 static int write_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1581 u16 selector
, struct desc_struct
*desc
)
1586 rc
= get_descriptor_ptr(ctxt
, selector
, &addr
);
1587 if (rc
!= X86EMUL_CONTINUE
)
1590 return ctxt
->ops
->write_std(ctxt
, addr
, desc
, sizeof *desc
,
1594 static int __load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1595 u16 selector
, int seg
, u8 cpl
,
1596 enum x86_transfer_type transfer
,
1597 struct desc_struct
*desc
)
1599 struct desc_struct seg_desc
, old_desc
;
1601 unsigned err_vec
= GP_VECTOR
;
1603 bool null_selector
= !(selector
& ~0x3); /* 0000-0003 are null */
1609 memset(&seg_desc
, 0, sizeof seg_desc
);
1611 if (ctxt
->mode
== X86EMUL_MODE_REAL
) {
1612 /* set real mode segment descriptor (keep limit etc. for
1614 ctxt
->ops
->get_segment(ctxt
, &dummy
, &seg_desc
, NULL
, seg
);
1615 set_desc_base(&seg_desc
, selector
<< 4);
1617 } else if (seg
<= VCPU_SREG_GS
&& ctxt
->mode
== X86EMUL_MODE_VM86
) {
1618 /* VM86 needs a clean new segment descriptor */
1619 set_desc_base(&seg_desc
, selector
<< 4);
1620 set_desc_limit(&seg_desc
, 0xffff);
1630 /* TR should be in GDT only */
1631 if (seg
== VCPU_SREG_TR
&& (selector
& (1 << 2)))
1634 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1635 if (null_selector
) {
1636 if (seg
== VCPU_SREG_CS
|| seg
== VCPU_SREG_TR
)
1639 if (seg
== VCPU_SREG_SS
) {
1640 if (ctxt
->mode
!= X86EMUL_MODE_PROT64
|| rpl
!= cpl
)
1644 * ctxt->ops->set_segment expects the CPL to be in
1645 * SS.DPL, so fake an expand-up 32-bit data segment.
1655 /* Skip all following checks */
1659 ret
= read_segment_descriptor(ctxt
, selector
, &seg_desc
, &desc_addr
);
1660 if (ret
!= X86EMUL_CONTINUE
)
1663 err_code
= selector
& 0xfffc;
1664 err_vec
= (transfer
== X86_TRANSFER_TASK_SWITCH
) ? TS_VECTOR
:
1667 /* can't load system descriptor into segment selector */
1668 if (seg
<= VCPU_SREG_GS
&& !seg_desc
.s
) {
1669 if (transfer
== X86_TRANSFER_CALL_JMP
)
1670 return X86EMUL_UNHANDLEABLE
;
1675 err_vec
= (seg
== VCPU_SREG_SS
) ? SS_VECTOR
: NP_VECTOR
;
1684 * segment is not a writable data segment or segment
1685 * selector's RPL != CPL or segment selector's RPL != CPL
1687 if (rpl
!= cpl
|| (seg_desc
.type
& 0xa) != 0x2 || dpl
!= cpl
)
1691 if (!(seg_desc
.type
& 8))
1694 if (seg_desc
.type
& 4) {
1700 if (rpl
> cpl
|| dpl
!= cpl
)
1703 /* in long-mode d/b must be clear if l is set */
1704 if (seg_desc
.d
&& seg_desc
.l
) {
1707 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1708 if (efer
& EFER_LMA
)
1712 /* CS(RPL) <- CPL */
1713 selector
= (selector
& 0xfffc) | cpl
;
1716 if (seg_desc
.s
|| (seg_desc
.type
!= 1 && seg_desc
.type
!= 9))
1718 old_desc
= seg_desc
;
1719 seg_desc
.type
|= 2; /* busy */
1720 ret
= ctxt
->ops
->cmpxchg_emulated(ctxt
, desc_addr
, &old_desc
, &seg_desc
,
1721 sizeof(seg_desc
), &ctxt
->exception
);
1722 if (ret
!= X86EMUL_CONTINUE
)
1725 case VCPU_SREG_LDTR
:
1726 if (seg_desc
.s
|| seg_desc
.type
!= 2)
1729 default: /* DS, ES, FS, or GS */
1731 * segment is not a data or readable code segment or
1732 * ((segment is a data or nonconforming code segment)
1733 * and (both RPL and CPL > DPL))
1735 if ((seg_desc
.type
& 0xa) == 0x8 ||
1736 (((seg_desc
.type
& 0xc) != 0xc) &&
1737 (rpl
> dpl
&& cpl
> dpl
)))
1743 /* mark segment as accessed */
1744 if (!(seg_desc
.type
& 1)) {
1746 ret
= write_segment_descriptor(ctxt
, selector
,
1748 if (ret
!= X86EMUL_CONTINUE
)
1751 } else if (ctxt
->mode
== X86EMUL_MODE_PROT64
) {
1752 ret
= ctxt
->ops
->read_std(ctxt
, desc_addr
+8, &base3
,
1753 sizeof(base3
), &ctxt
->exception
);
1754 if (ret
!= X86EMUL_CONTINUE
)
1756 if (emul_is_noncanonical_address(get_desc_base(&seg_desc
) |
1757 ((u64
)base3
<< 32), ctxt
))
1758 return emulate_gp(ctxt
, 0);
1761 ctxt
->ops
->set_segment(ctxt
, selector
, &seg_desc
, base3
, seg
);
1764 return X86EMUL_CONTINUE
;
1766 return emulate_exception(ctxt
, err_vec
, err_code
, true);
1769 static int load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1770 u16 selector
, int seg
)
1772 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
1775 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1776 * they can load it at CPL<3 (Intel's manual says only LSS can,
1779 * However, the Intel manual says that putting IST=1/DPL=3 in
1780 * an interrupt gate will result in SS=3 (the AMD manual instead
1781 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1782 * and only forbid it here.
1784 if (seg
== VCPU_SREG_SS
&& selector
== 3 &&
1785 ctxt
->mode
== X86EMUL_MODE_PROT64
)
1786 return emulate_exception(ctxt
, GP_VECTOR
, 0, true);
1788 return __load_segment_descriptor(ctxt
, selector
, seg
, cpl
,
1789 X86_TRANSFER_NONE
, NULL
);
1792 static void write_register_operand(struct operand
*op
)
1794 return assign_register(op
->addr
.reg
, op
->val
, op
->bytes
);
1797 static int writeback(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
)
1801 write_register_operand(op
);
1804 if (ctxt
->lock_prefix
)
1805 return segmented_cmpxchg(ctxt
,
1811 return segmented_write(ctxt
,
1817 return segmented_write(ctxt
,
1820 op
->bytes
* op
->count
);
1823 write_sse_reg(ctxt
, &op
->vec_val
, op
->addr
.xmm
);
1826 write_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
1834 return X86EMUL_CONTINUE
;
1837 static int push(struct x86_emulate_ctxt
*ctxt
, void *data
, int bytes
)
1839 struct segmented_address addr
;
1841 rsp_increment(ctxt
, -bytes
);
1842 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1843 addr
.seg
= VCPU_SREG_SS
;
1845 return segmented_write(ctxt
, addr
, data
, bytes
);
1848 static int em_push(struct x86_emulate_ctxt
*ctxt
)
1850 /* Disable writeback. */
1851 ctxt
->dst
.type
= OP_NONE
;
1852 return push(ctxt
, &ctxt
->src
.val
, ctxt
->op_bytes
);
1855 static int emulate_pop(struct x86_emulate_ctxt
*ctxt
,
1856 void *dest
, int len
)
1859 struct segmented_address addr
;
1861 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1862 addr
.seg
= VCPU_SREG_SS
;
1863 rc
= segmented_read(ctxt
, addr
, dest
, len
);
1864 if (rc
!= X86EMUL_CONTINUE
)
1867 rsp_increment(ctxt
, len
);
1871 static int em_pop(struct x86_emulate_ctxt
*ctxt
)
1873 return emulate_pop(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1876 static int emulate_popf(struct x86_emulate_ctxt
*ctxt
,
1877 void *dest
, int len
)
1880 unsigned long val
, change_mask
;
1881 int iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
1882 int cpl
= ctxt
->ops
->cpl(ctxt
);
1884 rc
= emulate_pop(ctxt
, &val
, len
);
1885 if (rc
!= X86EMUL_CONTINUE
)
1888 change_mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
1889 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_OF
|
1890 X86_EFLAGS_TF
| X86_EFLAGS_DF
| X86_EFLAGS_NT
|
1891 X86_EFLAGS_AC
| X86_EFLAGS_ID
;
1893 switch(ctxt
->mode
) {
1894 case X86EMUL_MODE_PROT64
:
1895 case X86EMUL_MODE_PROT32
:
1896 case X86EMUL_MODE_PROT16
:
1898 change_mask
|= X86_EFLAGS_IOPL
;
1900 change_mask
|= X86_EFLAGS_IF
;
1902 case X86EMUL_MODE_VM86
:
1904 return emulate_gp(ctxt
, 0);
1905 change_mask
|= X86_EFLAGS_IF
;
1907 default: /* real mode */
1908 change_mask
|= (X86_EFLAGS_IOPL
| X86_EFLAGS_IF
);
1912 *(unsigned long *)dest
=
1913 (ctxt
->eflags
& ~change_mask
) | (val
& change_mask
);
1918 static int em_popf(struct x86_emulate_ctxt
*ctxt
)
1920 ctxt
->dst
.type
= OP_REG
;
1921 ctxt
->dst
.addr
.reg
= &ctxt
->eflags
;
1922 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
1923 return emulate_popf(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1926 static int em_enter(struct x86_emulate_ctxt
*ctxt
)
1929 unsigned frame_size
= ctxt
->src
.val
;
1930 unsigned nesting_level
= ctxt
->src2
.val
& 31;
1934 return X86EMUL_UNHANDLEABLE
;
1936 rbp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1937 rc
= push(ctxt
, &rbp
, stack_size(ctxt
));
1938 if (rc
!= X86EMUL_CONTINUE
)
1940 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RBP
), reg_read(ctxt
, VCPU_REGS_RSP
),
1942 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
),
1943 reg_read(ctxt
, VCPU_REGS_RSP
) - frame_size
,
1945 return X86EMUL_CONTINUE
;
1948 static int em_leave(struct x86_emulate_ctxt
*ctxt
)
1950 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
), reg_read(ctxt
, VCPU_REGS_RBP
),
1952 return emulate_pop(ctxt
, reg_rmw(ctxt
, VCPU_REGS_RBP
), ctxt
->op_bytes
);
1955 static int em_push_sreg(struct x86_emulate_ctxt
*ctxt
)
1957 int seg
= ctxt
->src2
.val
;
1959 ctxt
->src
.val
= get_segment_selector(ctxt
, seg
);
1960 if (ctxt
->op_bytes
== 4) {
1961 rsp_increment(ctxt
, -2);
1965 return em_push(ctxt
);
1968 static int em_pop_sreg(struct x86_emulate_ctxt
*ctxt
)
1970 int seg
= ctxt
->src2
.val
;
1971 unsigned long selector
;
1974 rc
= emulate_pop(ctxt
, &selector
, 2);
1975 if (rc
!= X86EMUL_CONTINUE
)
1978 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
1979 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
1980 if (ctxt
->op_bytes
> 2)
1981 rsp_increment(ctxt
, ctxt
->op_bytes
- 2);
1983 rc
= load_segment_descriptor(ctxt
, (u16
)selector
, seg
);
1987 static int em_pusha(struct x86_emulate_ctxt
*ctxt
)
1989 unsigned long old_esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
1990 int rc
= X86EMUL_CONTINUE
;
1991 int reg
= VCPU_REGS_RAX
;
1993 while (reg
<= VCPU_REGS_RDI
) {
1994 (reg
== VCPU_REGS_RSP
) ?
1995 (ctxt
->src
.val
= old_esp
) : (ctxt
->src
.val
= reg_read(ctxt
, reg
));
1998 if (rc
!= X86EMUL_CONTINUE
)
2007 static int em_pushf(struct x86_emulate_ctxt
*ctxt
)
2009 ctxt
->src
.val
= (unsigned long)ctxt
->eflags
& ~X86_EFLAGS_VM
;
2010 return em_push(ctxt
);
2013 static int em_popa(struct x86_emulate_ctxt
*ctxt
)
2015 int rc
= X86EMUL_CONTINUE
;
2016 int reg
= VCPU_REGS_RDI
;
2019 while (reg
>= VCPU_REGS_RAX
) {
2020 if (reg
== VCPU_REGS_RSP
) {
2021 rsp_increment(ctxt
, ctxt
->op_bytes
);
2025 rc
= emulate_pop(ctxt
, &val
, ctxt
->op_bytes
);
2026 if (rc
!= X86EMUL_CONTINUE
)
2028 assign_register(reg_rmw(ctxt
, reg
), val
, ctxt
->op_bytes
);
2034 static int __emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2036 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2043 /* TODO: Add limit checks */
2044 ctxt
->src
.val
= ctxt
->eflags
;
2046 if (rc
!= X86EMUL_CONTINUE
)
2049 ctxt
->eflags
&= ~(X86_EFLAGS_IF
| X86_EFLAGS_TF
| X86_EFLAGS_AC
);
2051 ctxt
->src
.val
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2053 if (rc
!= X86EMUL_CONTINUE
)
2056 ctxt
->src
.val
= ctxt
->_eip
;
2058 if (rc
!= X86EMUL_CONTINUE
)
2061 ops
->get_idt(ctxt
, &dt
);
2063 eip_addr
= dt
.address
+ (irq
<< 2);
2064 cs_addr
= dt
.address
+ (irq
<< 2) + 2;
2066 rc
= ops
->read_std(ctxt
, cs_addr
, &cs
, 2, &ctxt
->exception
);
2067 if (rc
!= X86EMUL_CONTINUE
)
2070 rc
= ops
->read_std(ctxt
, eip_addr
, &eip
, 2, &ctxt
->exception
);
2071 if (rc
!= X86EMUL_CONTINUE
)
2074 rc
= load_segment_descriptor(ctxt
, cs
, VCPU_SREG_CS
);
2075 if (rc
!= X86EMUL_CONTINUE
)
2083 int emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2087 invalidate_registers(ctxt
);
2088 rc
= __emulate_int_real(ctxt
, irq
);
2089 if (rc
== X86EMUL_CONTINUE
)
2090 writeback_registers(ctxt
);
2094 static int emulate_int(struct x86_emulate_ctxt
*ctxt
, int irq
)
2096 switch(ctxt
->mode
) {
2097 case X86EMUL_MODE_REAL
:
2098 return __emulate_int_real(ctxt
, irq
);
2099 case X86EMUL_MODE_VM86
:
2100 case X86EMUL_MODE_PROT16
:
2101 case X86EMUL_MODE_PROT32
:
2102 case X86EMUL_MODE_PROT64
:
2104 /* Protected mode interrupts unimplemented yet */
2105 return X86EMUL_UNHANDLEABLE
;
2109 static int emulate_iret_real(struct x86_emulate_ctxt
*ctxt
)
2111 int rc
= X86EMUL_CONTINUE
;
2112 unsigned long temp_eip
= 0;
2113 unsigned long temp_eflags
= 0;
2114 unsigned long cs
= 0;
2115 unsigned long mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
2116 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_TF
|
2117 X86_EFLAGS_IF
| X86_EFLAGS_DF
| X86_EFLAGS_OF
|
2118 X86_EFLAGS_IOPL
| X86_EFLAGS_NT
| X86_EFLAGS_RF
|
2119 X86_EFLAGS_AC
| X86_EFLAGS_ID
|
2121 unsigned long vm86_mask
= X86_EFLAGS_VM
| X86_EFLAGS_VIF
|
2124 /* TODO: Add stack limit check */
2126 rc
= emulate_pop(ctxt
, &temp_eip
, ctxt
->op_bytes
);
2128 if (rc
!= X86EMUL_CONTINUE
)
2131 if (temp_eip
& ~0xffff)
2132 return emulate_gp(ctxt
, 0);
2134 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2136 if (rc
!= X86EMUL_CONTINUE
)
2139 rc
= emulate_pop(ctxt
, &temp_eflags
, ctxt
->op_bytes
);
2141 if (rc
!= X86EMUL_CONTINUE
)
2144 rc
= load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
);
2146 if (rc
!= X86EMUL_CONTINUE
)
2149 ctxt
->_eip
= temp_eip
;
2151 if (ctxt
->op_bytes
== 4)
2152 ctxt
->eflags
= ((temp_eflags
& mask
) | (ctxt
->eflags
& vm86_mask
));
2153 else if (ctxt
->op_bytes
== 2) {
2154 ctxt
->eflags
&= ~0xffff;
2155 ctxt
->eflags
|= temp_eflags
;
2158 ctxt
->eflags
&= ~EFLG_RESERVED_ZEROS_MASK
; /* Clear reserved zeros */
2159 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2160 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2165 static int em_iret(struct x86_emulate_ctxt
*ctxt
)
2167 switch(ctxt
->mode
) {
2168 case X86EMUL_MODE_REAL
:
2169 return emulate_iret_real(ctxt
);
2170 case X86EMUL_MODE_VM86
:
2171 case X86EMUL_MODE_PROT16
:
2172 case X86EMUL_MODE_PROT32
:
2173 case X86EMUL_MODE_PROT64
:
2175 /* iret from protected mode unimplemented yet */
2176 return X86EMUL_UNHANDLEABLE
;
2180 static int em_jmp_far(struct x86_emulate_ctxt
*ctxt
)
2184 struct desc_struct new_desc
;
2185 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
2187 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2189 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
2190 X86_TRANSFER_CALL_JMP
,
2192 if (rc
!= X86EMUL_CONTINUE
)
2195 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
2196 /* Error handling is not implemented. */
2197 if (rc
!= X86EMUL_CONTINUE
)
2198 return X86EMUL_UNHANDLEABLE
;
2203 static int em_jmp_abs(struct x86_emulate_ctxt
*ctxt
)
2205 return assign_eip_near(ctxt
, ctxt
->src
.val
);
2208 static int em_call_near_abs(struct x86_emulate_ctxt
*ctxt
)
2213 old_eip
= ctxt
->_eip
;
2214 rc
= assign_eip_near(ctxt
, ctxt
->src
.val
);
2215 if (rc
!= X86EMUL_CONTINUE
)
2217 ctxt
->src
.val
= old_eip
;
2222 static int em_cmpxchg8b(struct x86_emulate_ctxt
*ctxt
)
2224 u64 old
= ctxt
->dst
.orig_val64
;
2226 if (ctxt
->dst
.bytes
== 16)
2227 return X86EMUL_UNHANDLEABLE
;
2229 if (((u32
) (old
>> 0) != (u32
) reg_read(ctxt
, VCPU_REGS_RAX
)) ||
2230 ((u32
) (old
>> 32) != (u32
) reg_read(ctxt
, VCPU_REGS_RDX
))) {
2231 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
) (old
>> 0);
2232 *reg_write(ctxt
, VCPU_REGS_RDX
) = (u32
) (old
>> 32);
2233 ctxt
->eflags
&= ~X86_EFLAGS_ZF
;
2235 ctxt
->dst
.val64
= ((u64
)reg_read(ctxt
, VCPU_REGS_RCX
) << 32) |
2236 (u32
) reg_read(ctxt
, VCPU_REGS_RBX
);
2238 ctxt
->eflags
|= X86_EFLAGS_ZF
;
2240 return X86EMUL_CONTINUE
;
2243 static int em_ret(struct x86_emulate_ctxt
*ctxt
)
2248 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2249 if (rc
!= X86EMUL_CONTINUE
)
2252 return assign_eip_near(ctxt
, eip
);
2255 static int em_ret_far(struct x86_emulate_ctxt
*ctxt
)
2258 unsigned long eip
, cs
;
2259 int cpl
= ctxt
->ops
->cpl(ctxt
);
2260 struct desc_struct new_desc
;
2262 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2263 if (rc
!= X86EMUL_CONTINUE
)
2265 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2266 if (rc
!= X86EMUL_CONTINUE
)
2268 /* Outer-privilege level return is not implemented */
2269 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
&& (cs
& 3) > cpl
)
2270 return X86EMUL_UNHANDLEABLE
;
2271 rc
= __load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
, cpl
,
2274 if (rc
!= X86EMUL_CONTINUE
)
2276 rc
= assign_eip_far(ctxt
, eip
, &new_desc
);
2277 /* Error handling is not implemented. */
2278 if (rc
!= X86EMUL_CONTINUE
)
2279 return X86EMUL_UNHANDLEABLE
;
2284 static int em_ret_far_imm(struct x86_emulate_ctxt
*ctxt
)
2288 rc
= em_ret_far(ctxt
);
2289 if (rc
!= X86EMUL_CONTINUE
)
2291 rsp_increment(ctxt
, ctxt
->src
.val
);
2292 return X86EMUL_CONTINUE
;
2295 static int em_cmpxchg(struct x86_emulate_ctxt
*ctxt
)
2297 /* Save real source value, then compare EAX against destination. */
2298 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
2299 ctxt
->dst
.val
= reg_read(ctxt
, VCPU_REGS_RAX
);
2300 ctxt
->src
.orig_val
= ctxt
->src
.val
;
2301 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2302 fastop(ctxt
, em_cmp
);
2304 if (ctxt
->eflags
& X86_EFLAGS_ZF
) {
2305 /* Success: write back to memory; no update of EAX */
2306 ctxt
->src
.type
= OP_NONE
;
2307 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
2309 /* Failure: write the value we saw to EAX. */
2310 ctxt
->src
.type
= OP_REG
;
2311 ctxt
->src
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
2312 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2313 /* Create write-cycle to dest by writing the same value */
2314 ctxt
->dst
.val
= ctxt
->dst
.orig_val
;
2316 return X86EMUL_CONTINUE
;
2319 static int em_lseg(struct x86_emulate_ctxt
*ctxt
)
2321 int seg
= ctxt
->src2
.val
;
2325 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2327 rc
= load_segment_descriptor(ctxt
, sel
, seg
);
2328 if (rc
!= X86EMUL_CONTINUE
)
2331 ctxt
->dst
.val
= ctxt
->src
.val
;
2335 static int emulator_has_longmode(struct x86_emulate_ctxt
*ctxt
)
2337 u32 eax
, ebx
, ecx
, edx
;
2341 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, false);
2342 return edx
& bit(X86_FEATURE_LM
);
2345 #define GET_SMSTATE(type, smbase, offset) \
2348 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2350 if (r != X86EMUL_CONTINUE) \
2351 return X86EMUL_UNHANDLEABLE; \
2355 static void rsm_set_desc_flags(struct desc_struct
*desc
, u32 flags
)
2357 desc
->g
= (flags
>> 23) & 1;
2358 desc
->d
= (flags
>> 22) & 1;
2359 desc
->l
= (flags
>> 21) & 1;
2360 desc
->avl
= (flags
>> 20) & 1;
2361 desc
->p
= (flags
>> 15) & 1;
2362 desc
->dpl
= (flags
>> 13) & 3;
2363 desc
->s
= (flags
>> 12) & 1;
2364 desc
->type
= (flags
>> 8) & 15;
2367 static int rsm_load_seg_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2369 struct desc_struct desc
;
2373 selector
= GET_SMSTATE(u32
, smbase
, 0x7fa8 + n
* 4);
2376 offset
= 0x7f84 + n
* 12;
2378 offset
= 0x7f2c + (n
- 3) * 12;
2380 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2381 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2382 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, offset
));
2383 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, n
);
2384 return X86EMUL_CONTINUE
;
2387 static int rsm_load_seg_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2389 struct desc_struct desc
;
2394 offset
= 0x7e00 + n
* 16;
2396 selector
= GET_SMSTATE(u16
, smbase
, offset
);
2397 rsm_set_desc_flags(&desc
, GET_SMSTATE(u16
, smbase
, offset
+ 2) << 8);
2398 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2399 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2400 base3
= GET_SMSTATE(u32
, smbase
, offset
+ 12);
2402 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, n
);
2403 return X86EMUL_CONTINUE
;
2406 static int rsm_enter_protected_mode(struct x86_emulate_ctxt
*ctxt
,
2412 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2413 * Then enable protected mode. However, PCID cannot be enabled
2414 * if EFER.LMA=0, so set it separately.
2416 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2418 return X86EMUL_UNHANDLEABLE
;
2420 bad
= ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
2422 return X86EMUL_UNHANDLEABLE
;
2424 if (cr4
& X86_CR4_PCIDE
) {
2425 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
);
2427 return X86EMUL_UNHANDLEABLE
;
2430 return X86EMUL_CONTINUE
;
2433 static int rsm_load_state_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2435 struct desc_struct desc
;
2441 cr0
= GET_SMSTATE(u32
, smbase
, 0x7ffc);
2442 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u32
, smbase
, 0x7ff8));
2443 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7ff4) | X86_EFLAGS_FIXED
;
2444 ctxt
->_eip
= GET_SMSTATE(u32
, smbase
, 0x7ff0);
2446 for (i
= 0; i
< 8; i
++)
2447 *reg_write(ctxt
, i
) = GET_SMSTATE(u32
, smbase
, 0x7fd0 + i
* 4);
2449 val
= GET_SMSTATE(u32
, smbase
, 0x7fcc);
2450 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2451 val
= GET_SMSTATE(u32
, smbase
, 0x7fc8);
2452 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2454 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc4);
2455 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f64));
2456 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f60));
2457 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f5c));
2458 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_TR
);
2460 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc0);
2461 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f80));
2462 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f7c));
2463 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f78));
2464 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_LDTR
);
2466 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f74);
2467 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f70);
2468 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2470 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f58);
2471 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f54);
2472 ctxt
->ops
->set_idt(ctxt
, &dt
);
2474 for (i
= 0; i
< 6; i
++) {
2475 int r
= rsm_load_seg_32(ctxt
, smbase
, i
);
2476 if (r
!= X86EMUL_CONTINUE
)
2480 cr4
= GET_SMSTATE(u32
, smbase
, 0x7f14);
2482 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7ef8));
2484 return rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2487 static int rsm_load_state_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2489 struct desc_struct desc
;
2496 for (i
= 0; i
< 16; i
++)
2497 *reg_write(ctxt
, i
) = GET_SMSTATE(u64
, smbase
, 0x7ff8 - i
* 8);
2499 ctxt
->_eip
= GET_SMSTATE(u64
, smbase
, 0x7f78);
2500 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7f70) | X86_EFLAGS_FIXED
;
2502 val
= GET_SMSTATE(u32
, smbase
, 0x7f68);
2503 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2504 val
= GET_SMSTATE(u32
, smbase
, 0x7f60);
2505 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2507 cr0
= GET_SMSTATE(u64
, smbase
, 0x7f58);
2508 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u64
, smbase
, 0x7f50));
2509 cr4
= GET_SMSTATE(u64
, smbase
, 0x7f48);
2510 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7f00));
2511 val
= GET_SMSTATE(u64
, smbase
, 0x7ed0);
2512 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, val
& ~EFER_LMA
);
2514 selector
= GET_SMSTATE(u32
, smbase
, 0x7e90);
2515 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e92) << 8);
2516 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e94));
2517 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e98));
2518 base3
= GET_SMSTATE(u32
, smbase
, 0x7e9c);
2519 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_TR
);
2521 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e84);
2522 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e88);
2523 ctxt
->ops
->set_idt(ctxt
, &dt
);
2525 selector
= GET_SMSTATE(u32
, smbase
, 0x7e70);
2526 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e72) << 8);
2527 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e74));
2528 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e78));
2529 base3
= GET_SMSTATE(u32
, smbase
, 0x7e7c);
2530 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_LDTR
);
2532 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e64);
2533 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e68);
2534 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2536 r
= rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2537 if (r
!= X86EMUL_CONTINUE
)
2540 for (i
= 0; i
< 6; i
++) {
2541 r
= rsm_load_seg_64(ctxt
, smbase
, i
);
2542 if (r
!= X86EMUL_CONTINUE
)
2546 return X86EMUL_CONTINUE
;
2549 static int em_rsm(struct x86_emulate_ctxt
*ctxt
)
2551 unsigned long cr0
, cr4
, efer
;
2555 if ((ctxt
->ops
->get_hflags(ctxt
) & X86EMUL_SMM_MASK
) == 0)
2556 return emulate_ud(ctxt
);
2559 * Get back to real mode, to prepare a safe state in which to load
2560 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2561 * supports long mode.
2563 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
2564 if (emulator_has_longmode(ctxt
)) {
2565 struct desc_struct cs_desc
;
2567 /* Zero CR4.PCIDE before CR0.PG. */
2568 if (cr4
& X86_CR4_PCIDE
) {
2569 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2570 cr4
&= ~X86_CR4_PCIDE
;
2573 /* A 32-bit code segment is required to clear EFER.LMA. */
2574 memset(&cs_desc
, 0, sizeof(cs_desc
));
2576 cs_desc
.s
= cs_desc
.g
= cs_desc
.p
= 1;
2577 ctxt
->ops
->set_segment(ctxt
, 0, &cs_desc
, 0, VCPU_SREG_CS
);
2580 /* For the 64-bit case, this will clear EFER.LMA. */
2581 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
2582 if (cr0
& X86_CR0_PE
)
2583 ctxt
->ops
->set_cr(ctxt
, 0, cr0
& ~(X86_CR0_PG
| X86_CR0_PE
));
2585 /* Now clear CR4.PAE (which must be done before clearing EFER.LME). */
2586 if (cr4
& X86_CR4_PAE
)
2587 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PAE
);
2589 /* And finally go back to 32-bit mode. */
2591 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, efer
);
2593 smbase
= ctxt
->ops
->get_smbase(ctxt
);
2596 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2597 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2600 if (ctxt
->ops
->pre_leave_smm(ctxt
, smbase
))
2601 return X86EMUL_UNHANDLEABLE
;
2603 if (emulator_has_longmode(ctxt
))
2604 ret
= rsm_load_state_64(ctxt
, smbase
+ 0x8000);
2606 ret
= rsm_load_state_32(ctxt
, smbase
+ 0x8000);
2608 if (ret
!= X86EMUL_CONTINUE
) {
2609 /* FIXME: should triple fault */
2610 return X86EMUL_UNHANDLEABLE
;
2613 if ((ctxt
->ops
->get_hflags(ctxt
) & X86EMUL_SMM_INSIDE_NMI_MASK
) == 0)
2614 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2616 ctxt
->ops
->set_hflags(ctxt
, ctxt
->ops
->get_hflags(ctxt
) &
2617 ~(X86EMUL_SMM_INSIDE_NMI_MASK
| X86EMUL_SMM_MASK
));
2618 return X86EMUL_CONTINUE
;
2622 setup_syscalls_segments(struct x86_emulate_ctxt
*ctxt
,
2623 struct desc_struct
*cs
, struct desc_struct
*ss
)
2625 cs
->l
= 0; /* will be adjusted later */
2626 set_desc_base(cs
, 0); /* flat segment */
2627 cs
->g
= 1; /* 4kb granularity */
2628 set_desc_limit(cs
, 0xfffff); /* 4GB limit */
2629 cs
->type
= 0x0b; /* Read, Execute, Accessed */
2631 cs
->dpl
= 0; /* will be adjusted later */
2636 set_desc_base(ss
, 0); /* flat segment */
2637 set_desc_limit(ss
, 0xfffff); /* 4GB limit */
2638 ss
->g
= 1; /* 4kb granularity */
2640 ss
->type
= 0x03; /* Read/Write, Accessed */
2641 ss
->d
= 1; /* 32bit stack segment */
2648 static bool vendor_intel(struct x86_emulate_ctxt
*ctxt
)
2650 u32 eax
, ebx
, ecx
, edx
;
2653 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, false);
2654 return ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2655 && ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2656 && edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
;
2659 static bool em_syscall_is_enabled(struct x86_emulate_ctxt
*ctxt
)
2661 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2662 u32 eax
, ebx
, ecx
, edx
;
2665 * syscall should always be enabled in longmode - so only become
2666 * vendor specific (cpuid) if other modes are active...
2668 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2673 ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, false);
2675 * Intel ("GenuineIntel")
2676 * remark: Intel CPUs only support "syscall" in 64bit
2677 * longmode. Also an 64bit guest with a
2678 * 32bit compat-app running will #UD !! While this
2679 * behaviour can be fixed (by emulating) into AMD
2680 * response - CPUs of AMD can't behave like Intel.
2682 if (ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
&&
2683 ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
&&
2684 edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
)
2687 /* AMD ("AuthenticAMD") */
2688 if (ebx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx
&&
2689 ecx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx
&&
2690 edx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_edx
)
2693 /* AMD ("AMDisbetter!") */
2694 if (ebx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx
&&
2695 ecx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx
&&
2696 edx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_edx
)
2699 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2703 static int em_syscall(struct x86_emulate_ctxt
*ctxt
)
2705 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2706 struct desc_struct cs
, ss
;
2711 /* syscall is not available in real mode */
2712 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2713 ctxt
->mode
== X86EMUL_MODE_VM86
)
2714 return emulate_ud(ctxt
);
2716 if (!(em_syscall_is_enabled(ctxt
)))
2717 return emulate_ud(ctxt
);
2719 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2720 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2722 if (!(efer
& EFER_SCE
))
2723 return emulate_ud(ctxt
);
2725 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2727 cs_sel
= (u16
)(msr_data
& 0xfffc);
2728 ss_sel
= (u16
)(msr_data
+ 8);
2730 if (efer
& EFER_LMA
) {
2734 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2735 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2737 *reg_write(ctxt
, VCPU_REGS_RCX
) = ctxt
->_eip
;
2738 if (efer
& EFER_LMA
) {
2739 #ifdef CONFIG_X86_64
2740 *reg_write(ctxt
, VCPU_REGS_R11
) = ctxt
->eflags
;
2743 ctxt
->mode
== X86EMUL_MODE_PROT64
?
2744 MSR_LSTAR
: MSR_CSTAR
, &msr_data
);
2745 ctxt
->_eip
= msr_data
;
2747 ops
->get_msr(ctxt
, MSR_SYSCALL_MASK
, &msr_data
);
2748 ctxt
->eflags
&= ~msr_data
;
2749 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2753 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2754 ctxt
->_eip
= (u32
)msr_data
;
2756 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2759 ctxt
->tf
= (ctxt
->eflags
& X86_EFLAGS_TF
) != 0;
2760 return X86EMUL_CONTINUE
;
2763 static int em_sysenter(struct x86_emulate_ctxt
*ctxt
)
2765 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2766 struct desc_struct cs
, ss
;
2771 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2772 /* inject #GP if in real mode */
2773 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2774 return emulate_gp(ctxt
, 0);
2777 * Not recognized on AMD in compat mode (but is recognized in legacy
2780 if ((ctxt
->mode
!= X86EMUL_MODE_PROT64
) && (efer
& EFER_LMA
)
2781 && !vendor_intel(ctxt
))
2782 return emulate_ud(ctxt
);
2784 /* sysenter/sysexit have not been tested in 64bit mode. */
2785 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2786 return X86EMUL_UNHANDLEABLE
;
2788 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2790 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2791 if ((msr_data
& 0xfffc) == 0x0)
2792 return emulate_gp(ctxt
, 0);
2794 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2795 cs_sel
= (u16
)msr_data
& ~SEGMENT_RPL_MASK
;
2796 ss_sel
= cs_sel
+ 8;
2797 if (efer
& EFER_LMA
) {
2802 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2803 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2805 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_EIP
, &msr_data
);
2806 ctxt
->_eip
= (efer
& EFER_LMA
) ? msr_data
: (u32
)msr_data
;
2808 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_ESP
, &msr_data
);
2809 *reg_write(ctxt
, VCPU_REGS_RSP
) = (efer
& EFER_LMA
) ? msr_data
:
2812 return X86EMUL_CONTINUE
;
2815 static int em_sysexit(struct x86_emulate_ctxt
*ctxt
)
2817 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2818 struct desc_struct cs
, ss
;
2819 u64 msr_data
, rcx
, rdx
;
2821 u16 cs_sel
= 0, ss_sel
= 0;
2823 /* inject #GP if in real mode or Virtual 8086 mode */
2824 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2825 ctxt
->mode
== X86EMUL_MODE_VM86
)
2826 return emulate_gp(ctxt
, 0);
2828 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2830 if ((ctxt
->rex_prefix
& 0x8) != 0x0)
2831 usermode
= X86EMUL_MODE_PROT64
;
2833 usermode
= X86EMUL_MODE_PROT32
;
2835 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2836 rdx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2840 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2842 case X86EMUL_MODE_PROT32
:
2843 cs_sel
= (u16
)(msr_data
+ 16);
2844 if ((msr_data
& 0xfffc) == 0x0)
2845 return emulate_gp(ctxt
, 0);
2846 ss_sel
= (u16
)(msr_data
+ 24);
2850 case X86EMUL_MODE_PROT64
:
2851 cs_sel
= (u16
)(msr_data
+ 32);
2852 if (msr_data
== 0x0)
2853 return emulate_gp(ctxt
, 0);
2854 ss_sel
= cs_sel
+ 8;
2857 if (emul_is_noncanonical_address(rcx
, ctxt
) ||
2858 emul_is_noncanonical_address(rdx
, ctxt
))
2859 return emulate_gp(ctxt
, 0);
2862 cs_sel
|= SEGMENT_RPL_MASK
;
2863 ss_sel
|= SEGMENT_RPL_MASK
;
2865 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2866 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2869 *reg_write(ctxt
, VCPU_REGS_RSP
) = rcx
;
2871 return X86EMUL_CONTINUE
;
2874 static bool emulator_bad_iopl(struct x86_emulate_ctxt
*ctxt
)
2877 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2879 if (ctxt
->mode
== X86EMUL_MODE_VM86
)
2881 iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
2882 return ctxt
->ops
->cpl(ctxt
) > iopl
;
2885 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt
*ctxt
,
2888 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2889 struct desc_struct tr_seg
;
2892 u16 tr
, io_bitmap_ptr
, perm
, bit_idx
= port
& 0x7;
2893 unsigned mask
= (1 << len
) - 1;
2896 ops
->get_segment(ctxt
, &tr
, &tr_seg
, &base3
, VCPU_SREG_TR
);
2899 if (desc_limit_scaled(&tr_seg
) < 103)
2901 base
= get_desc_base(&tr_seg
);
2902 #ifdef CONFIG_X86_64
2903 base
|= ((u64
)base3
) << 32;
2905 r
= ops
->read_std(ctxt
, base
+ 102, &io_bitmap_ptr
, 2, NULL
);
2906 if (r
!= X86EMUL_CONTINUE
)
2908 if (io_bitmap_ptr
+ port
/8 > desc_limit_scaled(&tr_seg
))
2910 r
= ops
->read_std(ctxt
, base
+ io_bitmap_ptr
+ port
/8, &perm
, 2, NULL
);
2911 if (r
!= X86EMUL_CONTINUE
)
2913 if ((perm
>> bit_idx
) & mask
)
2918 static bool emulator_io_permited(struct x86_emulate_ctxt
*ctxt
,
2924 if (emulator_bad_iopl(ctxt
))
2925 if (!emulator_io_port_access_allowed(ctxt
, port
, len
))
2928 ctxt
->perm_ok
= true;
2933 static void string_registers_quirk(struct x86_emulate_ctxt
*ctxt
)
2936 * Intel CPUs mask the counter and pointers in quite strange
2937 * manner when ECX is zero due to REP-string optimizations.
2939 #ifdef CONFIG_X86_64
2940 if (ctxt
->ad_bytes
!= 4 || !vendor_intel(ctxt
))
2943 *reg_write(ctxt
, VCPU_REGS_RCX
) = 0;
2946 case 0xa4: /* movsb */
2947 case 0xa5: /* movsd/w */
2948 *reg_rmw(ctxt
, VCPU_REGS_RSI
) &= (u32
)-1;
2950 case 0xaa: /* stosb */
2951 case 0xab: /* stosd/w */
2952 *reg_rmw(ctxt
, VCPU_REGS_RDI
) &= (u32
)-1;
2957 static void save_state_to_tss16(struct x86_emulate_ctxt
*ctxt
,
2958 struct tss_segment_16
*tss
)
2960 tss
->ip
= ctxt
->_eip
;
2961 tss
->flag
= ctxt
->eflags
;
2962 tss
->ax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2963 tss
->cx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2964 tss
->dx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2965 tss
->bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2966 tss
->sp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2967 tss
->bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2968 tss
->si
= reg_read(ctxt
, VCPU_REGS_RSI
);
2969 tss
->di
= reg_read(ctxt
, VCPU_REGS_RDI
);
2971 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2972 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2973 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2974 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2975 tss
->ldt
= get_segment_selector(ctxt
, VCPU_SREG_LDTR
);
2978 static int load_state_from_tss16(struct x86_emulate_ctxt
*ctxt
,
2979 struct tss_segment_16
*tss
)
2984 ctxt
->_eip
= tss
->ip
;
2985 ctxt
->eflags
= tss
->flag
| 2;
2986 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->ax
;
2987 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->cx
;
2988 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->dx
;
2989 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->bx
;
2990 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->sp
;
2991 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->bp
;
2992 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->si
;
2993 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->di
;
2996 * SDM says that segment selectors are loaded before segment
2999 set_segment_selector(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
);
3000 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
3001 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
3002 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
3003 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
3008 * Now load segment descriptors. If fault happens at this stage
3009 * it is handled in a context of new task
3011 ret
= __load_segment_descriptor(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
, cpl
,
3012 X86_TRANSFER_TASK_SWITCH
, NULL
);
3013 if (ret
!= X86EMUL_CONTINUE
)
3015 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3016 X86_TRANSFER_TASK_SWITCH
, NULL
);
3017 if (ret
!= X86EMUL_CONTINUE
)
3019 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3020 X86_TRANSFER_TASK_SWITCH
, NULL
);
3021 if (ret
!= X86EMUL_CONTINUE
)
3023 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3024 X86_TRANSFER_TASK_SWITCH
, NULL
);
3025 if (ret
!= X86EMUL_CONTINUE
)
3027 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3028 X86_TRANSFER_TASK_SWITCH
, NULL
);
3029 if (ret
!= X86EMUL_CONTINUE
)
3032 return X86EMUL_CONTINUE
;
3035 static int task_switch_16(struct x86_emulate_ctxt
*ctxt
,
3036 u16 tss_selector
, u16 old_tss_sel
,
3037 ulong old_tss_base
, struct desc_struct
*new_desc
)
3039 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3040 struct tss_segment_16 tss_seg
;
3042 u32 new_tss_base
= get_desc_base(new_desc
);
3044 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3046 if (ret
!= X86EMUL_CONTINUE
)
3049 save_state_to_tss16(ctxt
, &tss_seg
);
3051 ret
= ops
->write_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3053 if (ret
!= X86EMUL_CONTINUE
)
3056 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3058 if (ret
!= X86EMUL_CONTINUE
)
3061 if (old_tss_sel
!= 0xffff) {
3062 tss_seg
.prev_task_link
= old_tss_sel
;
3064 ret
= ops
->write_std(ctxt
, new_tss_base
,
3065 &tss_seg
.prev_task_link
,
3066 sizeof tss_seg
.prev_task_link
,
3068 if (ret
!= X86EMUL_CONTINUE
)
3072 return load_state_from_tss16(ctxt
, &tss_seg
);
3075 static void save_state_to_tss32(struct x86_emulate_ctxt
*ctxt
,
3076 struct tss_segment_32
*tss
)
3078 /* CR3 and ldt selector are not saved intentionally */
3079 tss
->eip
= ctxt
->_eip
;
3080 tss
->eflags
= ctxt
->eflags
;
3081 tss
->eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3082 tss
->ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3083 tss
->edx
= reg_read(ctxt
, VCPU_REGS_RDX
);
3084 tss
->ebx
= reg_read(ctxt
, VCPU_REGS_RBX
);
3085 tss
->esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
3086 tss
->ebp
= reg_read(ctxt
, VCPU_REGS_RBP
);
3087 tss
->esi
= reg_read(ctxt
, VCPU_REGS_RSI
);
3088 tss
->edi
= reg_read(ctxt
, VCPU_REGS_RDI
);
3090 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
3091 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
3092 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
3093 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
3094 tss
->fs
= get_segment_selector(ctxt
, VCPU_SREG_FS
);
3095 tss
->gs
= get_segment_selector(ctxt
, VCPU_SREG_GS
);
3098 static int load_state_from_tss32(struct x86_emulate_ctxt
*ctxt
,
3099 struct tss_segment_32
*tss
)
3104 if (ctxt
->ops
->set_cr(ctxt
, 3, tss
->cr3
))
3105 return emulate_gp(ctxt
, 0);
3106 ctxt
->_eip
= tss
->eip
;
3107 ctxt
->eflags
= tss
->eflags
| 2;
3109 /* General purpose registers */
3110 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->eax
;
3111 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->ecx
;
3112 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->edx
;
3113 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->ebx
;
3114 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->esp
;
3115 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->ebp
;
3116 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->esi
;
3117 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->edi
;
3120 * SDM says that segment selectors are loaded before segment
3121 * descriptors. This is important because CPL checks will
3124 set_segment_selector(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
);
3125 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
3126 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
3127 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
3128 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
3129 set_segment_selector(ctxt
, tss
->fs
, VCPU_SREG_FS
);
3130 set_segment_selector(ctxt
, tss
->gs
, VCPU_SREG_GS
);
3133 * If we're switching between Protected Mode and VM86, we need to make
3134 * sure to update the mode before loading the segment descriptors so
3135 * that the selectors are interpreted correctly.
3137 if (ctxt
->eflags
& X86_EFLAGS_VM
) {
3138 ctxt
->mode
= X86EMUL_MODE_VM86
;
3141 ctxt
->mode
= X86EMUL_MODE_PROT32
;
3146 * Now load segment descriptors. If fault happenes at this stage
3147 * it is handled in a context of new task
3149 ret
= __load_segment_descriptor(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
,
3150 cpl
, X86_TRANSFER_TASK_SWITCH
, NULL
);
3151 if (ret
!= X86EMUL_CONTINUE
)
3153 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3154 X86_TRANSFER_TASK_SWITCH
, NULL
);
3155 if (ret
!= X86EMUL_CONTINUE
)
3157 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3158 X86_TRANSFER_TASK_SWITCH
, NULL
);
3159 if (ret
!= X86EMUL_CONTINUE
)
3161 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3162 X86_TRANSFER_TASK_SWITCH
, NULL
);
3163 if (ret
!= X86EMUL_CONTINUE
)
3165 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3166 X86_TRANSFER_TASK_SWITCH
, NULL
);
3167 if (ret
!= X86EMUL_CONTINUE
)
3169 ret
= __load_segment_descriptor(ctxt
, tss
->fs
, VCPU_SREG_FS
, cpl
,
3170 X86_TRANSFER_TASK_SWITCH
, NULL
);
3171 if (ret
!= X86EMUL_CONTINUE
)
3173 ret
= __load_segment_descriptor(ctxt
, tss
->gs
, VCPU_SREG_GS
, cpl
,
3174 X86_TRANSFER_TASK_SWITCH
, NULL
);
3179 static int task_switch_32(struct x86_emulate_ctxt
*ctxt
,
3180 u16 tss_selector
, u16 old_tss_sel
,
3181 ulong old_tss_base
, struct desc_struct
*new_desc
)
3183 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3184 struct tss_segment_32 tss_seg
;
3186 u32 new_tss_base
= get_desc_base(new_desc
);
3187 u32 eip_offset
= offsetof(struct tss_segment_32
, eip
);
3188 u32 ldt_sel_offset
= offsetof(struct tss_segment_32
, ldt_selector
);
3190 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3192 if (ret
!= X86EMUL_CONTINUE
)
3195 save_state_to_tss32(ctxt
, &tss_seg
);
3197 /* Only GP registers and segment selectors are saved */
3198 ret
= ops
->write_std(ctxt
, old_tss_base
+ eip_offset
, &tss_seg
.eip
,
3199 ldt_sel_offset
- eip_offset
, &ctxt
->exception
);
3200 if (ret
!= X86EMUL_CONTINUE
)
3203 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3205 if (ret
!= X86EMUL_CONTINUE
)
3208 if (old_tss_sel
!= 0xffff) {
3209 tss_seg
.prev_task_link
= old_tss_sel
;
3211 ret
= ops
->write_std(ctxt
, new_tss_base
,
3212 &tss_seg
.prev_task_link
,
3213 sizeof tss_seg
.prev_task_link
,
3215 if (ret
!= X86EMUL_CONTINUE
)
3219 return load_state_from_tss32(ctxt
, &tss_seg
);
3222 static int emulator_do_task_switch(struct x86_emulate_ctxt
*ctxt
,
3223 u16 tss_selector
, int idt_index
, int reason
,
3224 bool has_error_code
, u32 error_code
)
3226 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3227 struct desc_struct curr_tss_desc
, next_tss_desc
;
3229 u16 old_tss_sel
= get_segment_selector(ctxt
, VCPU_SREG_TR
);
3230 ulong old_tss_base
=
3231 ops
->get_cached_segment_base(ctxt
, VCPU_SREG_TR
);
3233 ulong desc_addr
, dr7
;
3235 /* FIXME: old_tss_base == ~0 ? */
3237 ret
= read_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
, &desc_addr
);
3238 if (ret
!= X86EMUL_CONTINUE
)
3240 ret
= read_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
, &desc_addr
);
3241 if (ret
!= X86EMUL_CONTINUE
)
3244 /* FIXME: check that next_tss_desc is tss */
3247 * Check privileges. The three cases are task switch caused by...
3249 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3250 * 2. Exception/IRQ/iret: No check is performed
3251 * 3. jmp/call to TSS/task-gate: No check is performed since the
3252 * hardware checks it before exiting.
3254 if (reason
== TASK_SWITCH_GATE
) {
3255 if (idt_index
!= -1) {
3256 /* Software interrupts */
3257 struct desc_struct task_gate_desc
;
3260 ret
= read_interrupt_descriptor(ctxt
, idt_index
,
3262 if (ret
!= X86EMUL_CONTINUE
)
3265 dpl
= task_gate_desc
.dpl
;
3266 if ((tss_selector
& 3) > dpl
|| ops
->cpl(ctxt
) > dpl
)
3267 return emulate_gp(ctxt
, (idt_index
<< 3) | 0x2);
3271 desc_limit
= desc_limit_scaled(&next_tss_desc
);
3272 if (!next_tss_desc
.p
||
3273 ((desc_limit
< 0x67 && (next_tss_desc
.type
& 8)) ||
3274 desc_limit
< 0x2b)) {
3275 return emulate_ts(ctxt
, tss_selector
& 0xfffc);
3278 if (reason
== TASK_SWITCH_IRET
|| reason
== TASK_SWITCH_JMP
) {
3279 curr_tss_desc
.type
&= ~(1 << 1); /* clear busy flag */
3280 write_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
);
3283 if (reason
== TASK_SWITCH_IRET
)
3284 ctxt
->eflags
= ctxt
->eflags
& ~X86_EFLAGS_NT
;
3286 /* set back link to prev task only if NT bit is set in eflags
3287 note that old_tss_sel is not used after this point */
3288 if (reason
!= TASK_SWITCH_CALL
&& reason
!= TASK_SWITCH_GATE
)
3289 old_tss_sel
= 0xffff;
3291 if (next_tss_desc
.type
& 8)
3292 ret
= task_switch_32(ctxt
, tss_selector
, old_tss_sel
,
3293 old_tss_base
, &next_tss_desc
);
3295 ret
= task_switch_16(ctxt
, tss_selector
, old_tss_sel
,
3296 old_tss_base
, &next_tss_desc
);
3297 if (ret
!= X86EMUL_CONTINUE
)
3300 if (reason
== TASK_SWITCH_CALL
|| reason
== TASK_SWITCH_GATE
)
3301 ctxt
->eflags
= ctxt
->eflags
| X86_EFLAGS_NT
;
3303 if (reason
!= TASK_SWITCH_IRET
) {
3304 next_tss_desc
.type
|= (1 << 1); /* set busy flag */
3305 write_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
);
3308 ops
->set_cr(ctxt
, 0, ops
->get_cr(ctxt
, 0) | X86_CR0_TS
);
3309 ops
->set_segment(ctxt
, tss_selector
, &next_tss_desc
, 0, VCPU_SREG_TR
);
3311 if (has_error_code
) {
3312 ctxt
->op_bytes
= ctxt
->ad_bytes
= (next_tss_desc
.type
& 8) ? 4 : 2;
3313 ctxt
->lock_prefix
= 0;
3314 ctxt
->src
.val
= (unsigned long) error_code
;
3315 ret
= em_push(ctxt
);
3318 ops
->get_dr(ctxt
, 7, &dr7
);
3319 ops
->set_dr(ctxt
, 7, dr7
& ~(DR_LOCAL_ENABLE_MASK
| DR_LOCAL_SLOWDOWN
));
3324 int emulator_task_switch(struct x86_emulate_ctxt
*ctxt
,
3325 u16 tss_selector
, int idt_index
, int reason
,
3326 bool has_error_code
, u32 error_code
)
3330 invalidate_registers(ctxt
);
3331 ctxt
->_eip
= ctxt
->eip
;
3332 ctxt
->dst
.type
= OP_NONE
;
3334 rc
= emulator_do_task_switch(ctxt
, tss_selector
, idt_index
, reason
,
3335 has_error_code
, error_code
);
3337 if (rc
== X86EMUL_CONTINUE
) {
3338 ctxt
->eip
= ctxt
->_eip
;
3339 writeback_registers(ctxt
);
3342 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
3345 static void string_addr_inc(struct x86_emulate_ctxt
*ctxt
, int reg
,
3348 int df
= (ctxt
->eflags
& X86_EFLAGS_DF
) ? -op
->count
: op
->count
;
3350 register_address_increment(ctxt
, reg
, df
* op
->bytes
);
3351 op
->addr
.mem
.ea
= register_address(ctxt
, reg
);
3354 static int em_das(struct x86_emulate_ctxt
*ctxt
)
3357 bool af
, cf
, old_cf
;
3359 cf
= ctxt
->eflags
& X86_EFLAGS_CF
;
3365 af
= ctxt
->eflags
& X86_EFLAGS_AF
;
3366 if ((al
& 0x0f) > 9 || af
) {
3368 cf
= old_cf
| (al
>= 250);
3373 if (old_al
> 0x99 || old_cf
) {
3379 /* Set PF, ZF, SF */
3380 ctxt
->src
.type
= OP_IMM
;
3382 ctxt
->src
.bytes
= 1;
3383 fastop(ctxt
, em_or
);
3384 ctxt
->eflags
&= ~(X86_EFLAGS_AF
| X86_EFLAGS_CF
);
3386 ctxt
->eflags
|= X86_EFLAGS_CF
;
3388 ctxt
->eflags
|= X86_EFLAGS_AF
;
3389 return X86EMUL_CONTINUE
;
3392 static int em_aam(struct x86_emulate_ctxt
*ctxt
)
3396 if (ctxt
->src
.val
== 0)
3397 return emulate_de(ctxt
);
3399 al
= ctxt
->dst
.val
& 0xff;
3400 ah
= al
/ ctxt
->src
.val
;
3401 al
%= ctxt
->src
.val
;
3403 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
| (ah
<< 8);
3405 /* Set PF, ZF, SF */
3406 ctxt
->src
.type
= OP_IMM
;
3408 ctxt
->src
.bytes
= 1;
3409 fastop(ctxt
, em_or
);
3411 return X86EMUL_CONTINUE
;
3414 static int em_aad(struct x86_emulate_ctxt
*ctxt
)
3416 u8 al
= ctxt
->dst
.val
& 0xff;
3417 u8 ah
= (ctxt
->dst
.val
>> 8) & 0xff;
3419 al
= (al
+ (ah
* ctxt
->src
.val
)) & 0xff;
3421 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
;
3423 /* Set PF, ZF, SF */
3424 ctxt
->src
.type
= OP_IMM
;
3426 ctxt
->src
.bytes
= 1;
3427 fastop(ctxt
, em_or
);
3429 return X86EMUL_CONTINUE
;
3432 static int em_call(struct x86_emulate_ctxt
*ctxt
)
3435 long rel
= ctxt
->src
.val
;
3437 ctxt
->src
.val
= (unsigned long)ctxt
->_eip
;
3438 rc
= jmp_rel(ctxt
, rel
);
3439 if (rc
!= X86EMUL_CONTINUE
)
3441 return em_push(ctxt
);
3444 static int em_call_far(struct x86_emulate_ctxt
*ctxt
)
3449 struct desc_struct old_desc
, new_desc
;
3450 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3451 int cpl
= ctxt
->ops
->cpl(ctxt
);
3452 enum x86emul_mode prev_mode
= ctxt
->mode
;
3454 old_eip
= ctxt
->_eip
;
3455 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
, VCPU_SREG_CS
);
3457 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
3458 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
3459 X86_TRANSFER_CALL_JMP
, &new_desc
);
3460 if (rc
!= X86EMUL_CONTINUE
)
3463 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
3464 if (rc
!= X86EMUL_CONTINUE
)
3467 ctxt
->src
.val
= old_cs
;
3469 if (rc
!= X86EMUL_CONTINUE
)
3472 ctxt
->src
.val
= old_eip
;
3474 /* If we failed, we tainted the memory, but the very least we should
3476 if (rc
!= X86EMUL_CONTINUE
) {
3477 pr_warn_once("faulting far call emulation tainted memory\n");
3482 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
3483 ctxt
->mode
= prev_mode
;
3488 static int em_ret_near_imm(struct x86_emulate_ctxt
*ctxt
)
3493 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
3494 if (rc
!= X86EMUL_CONTINUE
)
3496 rc
= assign_eip_near(ctxt
, eip
);
3497 if (rc
!= X86EMUL_CONTINUE
)
3499 rsp_increment(ctxt
, ctxt
->src
.val
);
3500 return X86EMUL_CONTINUE
;
3503 static int em_xchg(struct x86_emulate_ctxt
*ctxt
)
3505 /* Write back the register source. */
3506 ctxt
->src
.val
= ctxt
->dst
.val
;
3507 write_register_operand(&ctxt
->src
);
3509 /* Write back the memory destination with implicit LOCK prefix. */
3510 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
3511 ctxt
->lock_prefix
= 1;
3512 return X86EMUL_CONTINUE
;
3515 static int em_imul_3op(struct x86_emulate_ctxt
*ctxt
)
3517 ctxt
->dst
.val
= ctxt
->src2
.val
;
3518 return fastop(ctxt
, em_imul
);
3521 static int em_cwd(struct x86_emulate_ctxt
*ctxt
)
3523 ctxt
->dst
.type
= OP_REG
;
3524 ctxt
->dst
.bytes
= ctxt
->src
.bytes
;
3525 ctxt
->dst
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
3526 ctxt
->dst
.val
= ~((ctxt
->src
.val
>> (ctxt
->src
.bytes
* 8 - 1)) - 1);
3528 return X86EMUL_CONTINUE
;
3531 static int em_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3535 ctxt
->ops
->get_msr(ctxt
, MSR_IA32_TSC
, &tsc
);
3536 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)tsc
;
3537 *reg_write(ctxt
, VCPU_REGS_RDX
) = tsc
>> 32;
3538 return X86EMUL_CONTINUE
;
3541 static int em_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3545 if (ctxt
->ops
->read_pmc(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &pmc
))
3546 return emulate_gp(ctxt
, 0);
3547 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)pmc
;
3548 *reg_write(ctxt
, VCPU_REGS_RDX
) = pmc
>> 32;
3549 return X86EMUL_CONTINUE
;
3552 static int em_mov(struct x86_emulate_ctxt
*ctxt
)
3554 memcpy(ctxt
->dst
.valptr
, ctxt
->src
.valptr
, sizeof(ctxt
->src
.valptr
));
3555 return X86EMUL_CONTINUE
;
3558 #define FFL(x) bit(X86_FEATURE_##x)
3560 static int em_movbe(struct x86_emulate_ctxt
*ctxt
)
3562 u32 ebx
, ecx
, edx
, eax
= 1;
3566 * Check MOVBE is set in the guest-visible CPUID leaf.
3568 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, false);
3569 if (!(ecx
& FFL(MOVBE
)))
3570 return emulate_ud(ctxt
);
3572 switch (ctxt
->op_bytes
) {
3575 * From MOVBE definition: "...When the operand size is 16 bits,
3576 * the upper word of the destination register remains unchanged
3579 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3580 * rules so we have to do the operation almost per hand.
3582 tmp
= (u16
)ctxt
->src
.val
;
3583 ctxt
->dst
.val
&= ~0xffffUL
;
3584 ctxt
->dst
.val
|= (unsigned long)swab16(tmp
);
3587 ctxt
->dst
.val
= swab32((u32
)ctxt
->src
.val
);
3590 ctxt
->dst
.val
= swab64(ctxt
->src
.val
);
3595 return X86EMUL_CONTINUE
;
3598 static int em_cr_write(struct x86_emulate_ctxt
*ctxt
)
3600 if (ctxt
->ops
->set_cr(ctxt
, ctxt
->modrm_reg
, ctxt
->src
.val
))
3601 return emulate_gp(ctxt
, 0);
3603 /* Disable writeback. */
3604 ctxt
->dst
.type
= OP_NONE
;
3605 return X86EMUL_CONTINUE
;
3608 static int em_dr_write(struct x86_emulate_ctxt
*ctxt
)
3612 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3613 val
= ctxt
->src
.val
& ~0ULL;
3615 val
= ctxt
->src
.val
& ~0U;
3617 /* #UD condition is already handled. */
3618 if (ctxt
->ops
->set_dr(ctxt
, ctxt
->modrm_reg
, val
) < 0)
3619 return emulate_gp(ctxt
, 0);
3621 /* Disable writeback. */
3622 ctxt
->dst
.type
= OP_NONE
;
3623 return X86EMUL_CONTINUE
;
3626 static int em_wrmsr(struct x86_emulate_ctxt
*ctxt
)
3630 msr_data
= (u32
)reg_read(ctxt
, VCPU_REGS_RAX
)
3631 | ((u64
)reg_read(ctxt
, VCPU_REGS_RDX
) << 32);
3632 if (ctxt
->ops
->set_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), msr_data
))
3633 return emulate_gp(ctxt
, 0);
3635 return X86EMUL_CONTINUE
;
3638 static int em_rdmsr(struct x86_emulate_ctxt
*ctxt
)
3642 if (ctxt
->ops
->get_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &msr_data
))
3643 return emulate_gp(ctxt
, 0);
3645 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)msr_data
;
3646 *reg_write(ctxt
, VCPU_REGS_RDX
) = msr_data
>> 32;
3647 return X86EMUL_CONTINUE
;
3650 static int em_mov_rm_sreg(struct x86_emulate_ctxt
*ctxt
)
3652 if (ctxt
->modrm_reg
> VCPU_SREG_GS
)
3653 return emulate_ud(ctxt
);
3655 ctxt
->dst
.val
= get_segment_selector(ctxt
, ctxt
->modrm_reg
);
3656 if (ctxt
->dst
.bytes
== 4 && ctxt
->dst
.type
== OP_MEM
)
3657 ctxt
->dst
.bytes
= 2;
3658 return X86EMUL_CONTINUE
;
3661 static int em_mov_sreg_rm(struct x86_emulate_ctxt
*ctxt
)
3663 u16 sel
= ctxt
->src
.val
;
3665 if (ctxt
->modrm_reg
== VCPU_SREG_CS
|| ctxt
->modrm_reg
> VCPU_SREG_GS
)
3666 return emulate_ud(ctxt
);
3668 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
3669 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
3671 /* Disable writeback. */
3672 ctxt
->dst
.type
= OP_NONE
;
3673 return load_segment_descriptor(ctxt
, sel
, ctxt
->modrm_reg
);
3676 static int em_lldt(struct x86_emulate_ctxt
*ctxt
)
3678 u16 sel
= ctxt
->src
.val
;
3680 /* Disable writeback. */
3681 ctxt
->dst
.type
= OP_NONE
;
3682 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_LDTR
);
3685 static int em_ltr(struct x86_emulate_ctxt
*ctxt
)
3687 u16 sel
= ctxt
->src
.val
;
3689 /* Disable writeback. */
3690 ctxt
->dst
.type
= OP_NONE
;
3691 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_TR
);
3694 static int em_invlpg(struct x86_emulate_ctxt
*ctxt
)
3699 rc
= linearize(ctxt
, ctxt
->src
.addr
.mem
, 1, false, &linear
);
3700 if (rc
== X86EMUL_CONTINUE
)
3701 ctxt
->ops
->invlpg(ctxt
, linear
);
3702 /* Disable writeback. */
3703 ctxt
->dst
.type
= OP_NONE
;
3704 return X86EMUL_CONTINUE
;
3707 static int em_clts(struct x86_emulate_ctxt
*ctxt
)
3711 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
3713 ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
3714 return X86EMUL_CONTINUE
;
3717 static int em_hypercall(struct x86_emulate_ctxt
*ctxt
)
3719 int rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3721 if (rc
!= X86EMUL_CONTINUE
)
3724 /* Let the processor re-execute the fixed hypercall */
3725 ctxt
->_eip
= ctxt
->eip
;
3726 /* Disable writeback. */
3727 ctxt
->dst
.type
= OP_NONE
;
3728 return X86EMUL_CONTINUE
;
3731 static int emulate_store_desc_ptr(struct x86_emulate_ctxt
*ctxt
,
3732 void (*get
)(struct x86_emulate_ctxt
*ctxt
,
3733 struct desc_ptr
*ptr
))
3735 struct desc_ptr desc_ptr
;
3737 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3739 get(ctxt
, &desc_ptr
);
3740 if (ctxt
->op_bytes
== 2) {
3742 desc_ptr
.address
&= 0x00ffffff;
3744 /* Disable writeback. */
3745 ctxt
->dst
.type
= OP_NONE
;
3746 return segmented_write_std(ctxt
, ctxt
->dst
.addr
.mem
,
3747 &desc_ptr
, 2 + ctxt
->op_bytes
);
3750 static int em_sgdt(struct x86_emulate_ctxt
*ctxt
)
3752 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_gdt
);
3755 static int em_sidt(struct x86_emulate_ctxt
*ctxt
)
3757 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_idt
);
3760 static int em_lgdt_lidt(struct x86_emulate_ctxt
*ctxt
, bool lgdt
)
3762 struct desc_ptr desc_ptr
;
3765 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3767 rc
= read_descriptor(ctxt
, ctxt
->src
.addr
.mem
,
3768 &desc_ptr
.size
, &desc_ptr
.address
,
3770 if (rc
!= X86EMUL_CONTINUE
)
3772 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&&
3773 emul_is_noncanonical_address(desc_ptr
.address
, ctxt
))
3774 return emulate_gp(ctxt
, 0);
3776 ctxt
->ops
->set_gdt(ctxt
, &desc_ptr
);
3778 ctxt
->ops
->set_idt(ctxt
, &desc_ptr
);
3779 /* Disable writeback. */
3780 ctxt
->dst
.type
= OP_NONE
;
3781 return X86EMUL_CONTINUE
;
3784 static int em_lgdt(struct x86_emulate_ctxt
*ctxt
)
3786 return em_lgdt_lidt(ctxt
, true);
3789 static int em_lidt(struct x86_emulate_ctxt
*ctxt
)
3791 return em_lgdt_lidt(ctxt
, false);
3794 static int em_smsw(struct x86_emulate_ctxt
*ctxt
)
3796 if (ctxt
->dst
.type
== OP_MEM
)
3797 ctxt
->dst
.bytes
= 2;
3798 ctxt
->dst
.val
= ctxt
->ops
->get_cr(ctxt
, 0);
3799 return X86EMUL_CONTINUE
;
3802 static int em_lmsw(struct x86_emulate_ctxt
*ctxt
)
3804 ctxt
->ops
->set_cr(ctxt
, 0, (ctxt
->ops
->get_cr(ctxt
, 0) & ~0x0eul
)
3805 | (ctxt
->src
.val
& 0x0f));
3806 ctxt
->dst
.type
= OP_NONE
;
3807 return X86EMUL_CONTINUE
;
3810 static int em_loop(struct x86_emulate_ctxt
*ctxt
)
3812 int rc
= X86EMUL_CONTINUE
;
3814 register_address_increment(ctxt
, VCPU_REGS_RCX
, -1);
3815 if ((address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) != 0) &&
3816 (ctxt
->b
== 0xe2 || test_cc(ctxt
->b
^ 0x5, ctxt
->eflags
)))
3817 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3822 static int em_jcxz(struct x86_emulate_ctxt
*ctxt
)
3824 int rc
= X86EMUL_CONTINUE
;
3826 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0)
3827 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3832 static int em_in(struct x86_emulate_ctxt
*ctxt
)
3834 if (!pio_in_emulated(ctxt
, ctxt
->dst
.bytes
, ctxt
->src
.val
,
3836 return X86EMUL_IO_NEEDED
;
3838 return X86EMUL_CONTINUE
;
3841 static int em_out(struct x86_emulate_ctxt
*ctxt
)
3843 ctxt
->ops
->pio_out_emulated(ctxt
, ctxt
->src
.bytes
, ctxt
->dst
.val
,
3845 /* Disable writeback. */
3846 ctxt
->dst
.type
= OP_NONE
;
3847 return X86EMUL_CONTINUE
;
3850 static int em_cli(struct x86_emulate_ctxt
*ctxt
)
3852 if (emulator_bad_iopl(ctxt
))
3853 return emulate_gp(ctxt
, 0);
3855 ctxt
->eflags
&= ~X86_EFLAGS_IF
;
3856 return X86EMUL_CONTINUE
;
3859 static int em_sti(struct x86_emulate_ctxt
*ctxt
)
3861 if (emulator_bad_iopl(ctxt
))
3862 return emulate_gp(ctxt
, 0);
3864 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_STI
;
3865 ctxt
->eflags
|= X86_EFLAGS_IF
;
3866 return X86EMUL_CONTINUE
;
3869 static int em_cpuid(struct x86_emulate_ctxt
*ctxt
)
3871 u32 eax
, ebx
, ecx
, edx
;
3874 ctxt
->ops
->get_msr(ctxt
, MSR_MISC_FEATURES_ENABLES
, &msr
);
3875 if (msr
& MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
&&
3876 ctxt
->ops
->cpl(ctxt
)) {
3877 return emulate_gp(ctxt
, 0);
3880 eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3881 ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3882 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, true);
3883 *reg_write(ctxt
, VCPU_REGS_RAX
) = eax
;
3884 *reg_write(ctxt
, VCPU_REGS_RBX
) = ebx
;
3885 *reg_write(ctxt
, VCPU_REGS_RCX
) = ecx
;
3886 *reg_write(ctxt
, VCPU_REGS_RDX
) = edx
;
3887 return X86EMUL_CONTINUE
;
3890 static int em_sahf(struct x86_emulate_ctxt
*ctxt
)
3894 flags
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
| X86_EFLAGS_ZF
|
3896 flags
&= *reg_rmw(ctxt
, VCPU_REGS_RAX
) >> 8;
3898 ctxt
->eflags
&= ~0xffUL
;
3899 ctxt
->eflags
|= flags
| X86_EFLAGS_FIXED
;
3900 return X86EMUL_CONTINUE
;
3903 static int em_lahf(struct x86_emulate_ctxt
*ctxt
)
3905 *reg_rmw(ctxt
, VCPU_REGS_RAX
) &= ~0xff00UL
;
3906 *reg_rmw(ctxt
, VCPU_REGS_RAX
) |= (ctxt
->eflags
& 0xff) << 8;
3907 return X86EMUL_CONTINUE
;
3910 static int em_bswap(struct x86_emulate_ctxt
*ctxt
)
3912 switch (ctxt
->op_bytes
) {
3913 #ifdef CONFIG_X86_64
3915 asm("bswap %0" : "+r"(ctxt
->dst
.val
));
3919 asm("bswap %0" : "+r"(*(u32
*)&ctxt
->dst
.val
));
3922 return X86EMUL_CONTINUE
;
3925 static int em_clflush(struct x86_emulate_ctxt
*ctxt
)
3927 /* emulating clflush regardless of cpuid */
3928 return X86EMUL_CONTINUE
;
3931 static int em_movsxd(struct x86_emulate_ctxt
*ctxt
)
3933 ctxt
->dst
.val
= (s32
) ctxt
->src
.val
;
3934 return X86EMUL_CONTINUE
;
3937 static int check_fxsr(struct x86_emulate_ctxt
*ctxt
)
3939 u32 eax
= 1, ebx
, ecx
= 0, edx
;
3941 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
, false);
3942 if (!(edx
& FFL(FXSR
)))
3943 return emulate_ud(ctxt
);
3945 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
3946 return emulate_nm(ctxt
);
3949 * Don't emulate a case that should never be hit, instead of working
3950 * around a lack of fxsave64/fxrstor64 on old compilers.
3952 if (ctxt
->mode
>= X86EMUL_MODE_PROT64
)
3953 return X86EMUL_UNHANDLEABLE
;
3955 return X86EMUL_CONTINUE
;
3959 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3960 * and restore MXCSR.
3962 static size_t __fxstate_size(int nregs
)
3964 return offsetof(struct fxregs_state
, xmm_space
[0]) + nregs
* 16;
3967 static inline size_t fxstate_size(struct x86_emulate_ctxt
*ctxt
)
3970 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3971 return __fxstate_size(16);
3973 cr4_osfxsr
= ctxt
->ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
;
3974 return __fxstate_size(cr4_osfxsr
? 8 : 0);
3978 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3981 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3982 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3984 * 3) 64-bit mode with REX.W prefix
3985 * - like (2), but XMM 8-15 are being saved and restored
3986 * 4) 64-bit mode without REX.W prefix
3987 * - like (3), but FIP and FDP are 64 bit
3989 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3990 * desired result. (4) is not emulated.
3992 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3993 * and FPU DS) should match.
3995 static int em_fxsave(struct x86_emulate_ctxt
*ctxt
)
3997 struct fxregs_state fx_state
;
4000 rc
= check_fxsr(ctxt
);
4001 if (rc
!= X86EMUL_CONTINUE
)
4004 ctxt
->ops
->get_fpu(ctxt
);
4006 rc
= asm_safe("fxsave %[fx]", , [fx
] "+m"(fx_state
));
4008 ctxt
->ops
->put_fpu(ctxt
);
4010 if (rc
!= X86EMUL_CONTINUE
)
4013 return segmented_write_std(ctxt
, ctxt
->memop
.addr
.mem
, &fx_state
,
4014 fxstate_size(ctxt
));
4017 static int em_fxrstor(struct x86_emulate_ctxt
*ctxt
)
4019 struct fxregs_state fx_state
;
4023 rc
= check_fxsr(ctxt
);
4024 if (rc
!= X86EMUL_CONTINUE
)
4027 ctxt
->ops
->get_fpu(ctxt
);
4029 size
= fxstate_size(ctxt
);
4030 if (size
< __fxstate_size(16)) {
4031 rc
= asm_safe("fxsave %[fx]", , [fx
] "+m"(fx_state
));
4032 if (rc
!= X86EMUL_CONTINUE
)
4036 rc
= segmented_read_std(ctxt
, ctxt
->memop
.addr
.mem
, &fx_state
, size
);
4037 if (rc
!= X86EMUL_CONTINUE
)
4040 if (fx_state
.mxcsr
>> 16) {
4041 rc
= emulate_gp(ctxt
, 0);
4045 if (rc
== X86EMUL_CONTINUE
)
4046 rc
= asm_safe("fxrstor %[fx]", : [fx
] "m"(fx_state
));
4049 ctxt
->ops
->put_fpu(ctxt
);
4054 static bool valid_cr(int nr
)
4066 static int check_cr_read(struct x86_emulate_ctxt
*ctxt
)
4068 if (!valid_cr(ctxt
->modrm_reg
))
4069 return emulate_ud(ctxt
);
4071 return X86EMUL_CONTINUE
;
4074 static int check_cr_write(struct x86_emulate_ctxt
*ctxt
)
4076 u64 new_val
= ctxt
->src
.val64
;
4077 int cr
= ctxt
->modrm_reg
;
4080 static u64 cr_reserved_bits
[] = {
4081 0xffffffff00000000ULL
,
4082 0, 0, 0, /* CR3 checked later */
4089 return emulate_ud(ctxt
);
4091 if (new_val
& cr_reserved_bits
[cr
])
4092 return emulate_gp(ctxt
, 0);
4097 if (((new_val
& X86_CR0_PG
) && !(new_val
& X86_CR0_PE
)) ||
4098 ((new_val
& X86_CR0_NW
) && !(new_val
& X86_CR0_CD
)))
4099 return emulate_gp(ctxt
, 0);
4101 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4102 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4104 if ((new_val
& X86_CR0_PG
) && (efer
& EFER_LME
) &&
4105 !(cr4
& X86_CR4_PAE
))
4106 return emulate_gp(ctxt
, 0);
4113 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4114 if (efer
& EFER_LMA
) {
4116 u32 eax
, ebx
, ecx
, edx
;
4120 if (ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
,
4122 maxphyaddr
= eax
& 0xff;
4125 rsvd
= rsvd_bits(maxphyaddr
, 62);
4129 return emulate_gp(ctxt
, 0);
4134 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4136 if ((efer
& EFER_LMA
) && !(new_val
& X86_CR4_PAE
))
4137 return emulate_gp(ctxt
, 0);
4143 return X86EMUL_CONTINUE
;
4146 static int check_dr7_gd(struct x86_emulate_ctxt
*ctxt
)
4150 ctxt
->ops
->get_dr(ctxt
, 7, &dr7
);
4152 /* Check if DR7.Global_Enable is set */
4153 return dr7
& (1 << 13);
4156 static int check_dr_read(struct x86_emulate_ctxt
*ctxt
)
4158 int dr
= ctxt
->modrm_reg
;
4162 return emulate_ud(ctxt
);
4164 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4165 if ((cr4
& X86_CR4_DE
) && (dr
== 4 || dr
== 5))
4166 return emulate_ud(ctxt
);
4168 if (check_dr7_gd(ctxt
)) {
4171 ctxt
->ops
->get_dr(ctxt
, 6, &dr6
);
4173 dr6
|= DR6_BD
| DR6_RTM
;
4174 ctxt
->ops
->set_dr(ctxt
, 6, dr6
);
4175 return emulate_db(ctxt
);
4178 return X86EMUL_CONTINUE
;
4181 static int check_dr_write(struct x86_emulate_ctxt
*ctxt
)
4183 u64 new_val
= ctxt
->src
.val64
;
4184 int dr
= ctxt
->modrm_reg
;
4186 if ((dr
== 6 || dr
== 7) && (new_val
& 0xffffffff00000000ULL
))
4187 return emulate_gp(ctxt
, 0);
4189 return check_dr_read(ctxt
);
4192 static int check_svme(struct x86_emulate_ctxt
*ctxt
)
4196 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4198 if (!(efer
& EFER_SVME
))
4199 return emulate_ud(ctxt
);
4201 return X86EMUL_CONTINUE
;
4204 static int check_svme_pa(struct x86_emulate_ctxt
*ctxt
)
4206 u64 rax
= reg_read(ctxt
, VCPU_REGS_RAX
);
4208 /* Valid physical address? */
4209 if (rax
& 0xffff000000000000ULL
)
4210 return emulate_gp(ctxt
, 0);
4212 return check_svme(ctxt
);
4215 static int check_rdtsc(struct x86_emulate_ctxt
*ctxt
)
4217 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4219 if (cr4
& X86_CR4_TSD
&& ctxt
->ops
->cpl(ctxt
))
4220 return emulate_ud(ctxt
);
4222 return X86EMUL_CONTINUE
;
4225 static int check_rdpmc(struct x86_emulate_ctxt
*ctxt
)
4227 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4228 u64 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
4230 if ((!(cr4
& X86_CR4_PCE
) && ctxt
->ops
->cpl(ctxt
)) ||
4231 ctxt
->ops
->check_pmc(ctxt
, rcx
))
4232 return emulate_gp(ctxt
, 0);
4234 return X86EMUL_CONTINUE
;
4237 static int check_perm_in(struct x86_emulate_ctxt
*ctxt
)
4239 ctxt
->dst
.bytes
= min(ctxt
->dst
.bytes
, 4u);
4240 if (!emulator_io_permited(ctxt
, ctxt
->src
.val
, ctxt
->dst
.bytes
))
4241 return emulate_gp(ctxt
, 0);
4243 return X86EMUL_CONTINUE
;
4246 static int check_perm_out(struct x86_emulate_ctxt
*ctxt
)
4248 ctxt
->src
.bytes
= min(ctxt
->src
.bytes
, 4u);
4249 if (!emulator_io_permited(ctxt
, ctxt
->dst
.val
, ctxt
->src
.bytes
))
4250 return emulate_gp(ctxt
, 0);
4252 return X86EMUL_CONTINUE
;
4255 #define D(_y) { .flags = (_y) }
4256 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4257 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4258 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4259 #define N D(NotImpl)
4260 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4261 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4262 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4263 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4264 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4265 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4266 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4267 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4268 #define II(_f, _e, _i) \
4269 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4270 #define IIP(_f, _e, _i, _p) \
4271 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4272 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4273 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4275 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4276 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4277 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4278 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4279 #define I2bvIP(_f, _e, _i, _p) \
4280 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4282 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4283 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4284 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4286 static const struct opcode group7_rm0
[] = {
4288 I(SrcNone
| Priv
| EmulateOnUD
, em_hypercall
),
4292 static const struct opcode group7_rm1
[] = {
4293 DI(SrcNone
| Priv
, monitor
),
4294 DI(SrcNone
| Priv
, mwait
),
4298 static const struct opcode group7_rm3
[] = {
4299 DIP(SrcNone
| Prot
| Priv
, vmrun
, check_svme_pa
),
4300 II(SrcNone
| Prot
| EmulateOnUD
, em_hypercall
, vmmcall
),
4301 DIP(SrcNone
| Prot
| Priv
, vmload
, check_svme_pa
),
4302 DIP(SrcNone
| Prot
| Priv
, vmsave
, check_svme_pa
),
4303 DIP(SrcNone
| Prot
| Priv
, stgi
, check_svme
),
4304 DIP(SrcNone
| Prot
| Priv
, clgi
, check_svme
),
4305 DIP(SrcNone
| Prot
| Priv
, skinit
, check_svme
),
4306 DIP(SrcNone
| Prot
| Priv
, invlpga
, check_svme
),
4309 static const struct opcode group7_rm7
[] = {
4311 DIP(SrcNone
, rdtscp
, check_rdtsc
),
4315 static const struct opcode group1
[] = {
4317 F(Lock
| PageTable
, em_or
),
4320 F(Lock
| PageTable
, em_and
),
4326 static const struct opcode group1A
[] = {
4327 I(DstMem
| SrcNone
| Mov
| Stack
| IncSP
| TwoMemOp
, em_pop
), N
, N
, N
, N
, N
, N
, N
,
4330 static const struct opcode group2
[] = {
4331 F(DstMem
| ModRM
, em_rol
),
4332 F(DstMem
| ModRM
, em_ror
),
4333 F(DstMem
| ModRM
, em_rcl
),
4334 F(DstMem
| ModRM
, em_rcr
),
4335 F(DstMem
| ModRM
, em_shl
),
4336 F(DstMem
| ModRM
, em_shr
),
4337 F(DstMem
| ModRM
, em_shl
),
4338 F(DstMem
| ModRM
, em_sar
),
4341 static const struct opcode group3
[] = {
4342 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4343 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4344 F(DstMem
| SrcNone
| Lock
, em_not
),
4345 F(DstMem
| SrcNone
| Lock
, em_neg
),
4346 F(DstXacc
| Src2Mem
, em_mul_ex
),
4347 F(DstXacc
| Src2Mem
, em_imul_ex
),
4348 F(DstXacc
| Src2Mem
, em_div_ex
),
4349 F(DstXacc
| Src2Mem
, em_idiv_ex
),
4352 static const struct opcode group4
[] = {
4353 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_inc
),
4354 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_dec
),
4358 static const struct opcode group5
[] = {
4359 F(DstMem
| SrcNone
| Lock
, em_inc
),
4360 F(DstMem
| SrcNone
| Lock
, em_dec
),
4361 I(SrcMem
| NearBranch
, em_call_near_abs
),
4362 I(SrcMemFAddr
| ImplicitOps
, em_call_far
),
4363 I(SrcMem
| NearBranch
, em_jmp_abs
),
4364 I(SrcMemFAddr
| ImplicitOps
, em_jmp_far
),
4365 I(SrcMem
| Stack
| TwoMemOp
, em_push
), D(Undefined
),
4368 static const struct opcode group6
[] = {
4369 DI(Prot
| DstMem
, sldt
),
4370 DI(Prot
| DstMem
, str
),
4371 II(Prot
| Priv
| SrcMem16
, em_lldt
, lldt
),
4372 II(Prot
| Priv
| SrcMem16
, em_ltr
, ltr
),
4376 static const struct group_dual group7
= { {
4377 II(Mov
| DstMem
, em_sgdt
, sgdt
),
4378 II(Mov
| DstMem
, em_sidt
, sidt
),
4379 II(SrcMem
| Priv
, em_lgdt
, lgdt
),
4380 II(SrcMem
| Priv
, em_lidt
, lidt
),
4381 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4382 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4383 II(SrcMem
| ByteOp
| Priv
| NoAccess
, em_invlpg
, invlpg
),
4387 N
, EXT(0, group7_rm3
),
4388 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4389 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4393 static const struct opcode group8
[] = {
4395 F(DstMem
| SrcImmByte
| NoWrite
, em_bt
),
4396 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_bts
),
4397 F(DstMem
| SrcImmByte
| Lock
, em_btr
),
4398 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_btc
),
4401 static const struct group_dual group9
= { {
4402 N
, I(DstMem64
| Lock
| PageTable
, em_cmpxchg8b
), N
, N
, N
, N
, N
, N
,
4404 N
, N
, N
, N
, N
, N
, N
, N
,
4407 static const struct opcode group11
[] = {
4408 I(DstMem
| SrcImm
| Mov
| PageTable
, em_mov
),
4412 static const struct gprefix pfx_0f_ae_7
= {
4413 I(SrcMem
| ByteOp
, em_clflush
), N
, N
, N
,
4416 static const struct group_dual group15
= { {
4417 I(ModRM
| Aligned16
, em_fxsave
),
4418 I(ModRM
| Aligned16
, em_fxrstor
),
4419 N
, N
, N
, N
, N
, GP(0, &pfx_0f_ae_7
),
4421 N
, N
, N
, N
, N
, N
, N
, N
,
4424 static const struct gprefix pfx_0f_6f_0f_7f
= {
4425 I(Mmx
, em_mov
), I(Sse
| Aligned
, em_mov
), N
, I(Sse
| Unaligned
, em_mov
),
4428 static const struct instr_dual instr_dual_0f_2b
= {
4432 static const struct gprefix pfx_0f_2b
= {
4433 ID(0, &instr_dual_0f_2b
), ID(0, &instr_dual_0f_2b
), N
, N
,
4436 static const struct gprefix pfx_0f_28_0f_29
= {
4437 I(Aligned
, em_mov
), I(Aligned
, em_mov
), N
, N
,
4440 static const struct gprefix pfx_0f_e7
= {
4441 N
, I(Sse
, em_mov
), N
, N
,
4444 static const struct escape escape_d9
= { {
4445 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstcw
),
4448 N
, N
, N
, N
, N
, N
, N
, N
,
4450 N
, N
, N
, N
, N
, N
, N
, N
,
4452 N
, N
, N
, N
, N
, N
, N
, N
,
4454 N
, N
, N
, N
, N
, N
, N
, N
,
4456 N
, N
, N
, N
, N
, N
, N
, N
,
4458 N
, N
, N
, N
, N
, N
, N
, N
,
4460 N
, N
, N
, N
, N
, N
, N
, N
,
4462 N
, N
, N
, N
, N
, N
, N
, N
,
4465 static const struct escape escape_db
= { {
4466 N
, N
, N
, N
, N
, N
, N
, N
,
4469 N
, N
, N
, N
, N
, N
, N
, N
,
4471 N
, N
, N
, N
, N
, N
, N
, N
,
4473 N
, N
, N
, N
, N
, N
, N
, N
,
4475 N
, N
, N
, N
, N
, N
, N
, N
,
4477 N
, N
, N
, I(ImplicitOps
, em_fninit
), N
, N
, N
, N
,
4479 N
, N
, N
, N
, N
, N
, N
, N
,
4481 N
, N
, N
, N
, N
, N
, N
, N
,
4483 N
, N
, N
, N
, N
, N
, N
, N
,
4486 static const struct escape escape_dd
= { {
4487 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstsw
),
4490 N
, N
, N
, N
, N
, N
, N
, N
,
4492 N
, N
, N
, N
, N
, N
, N
, N
,
4494 N
, N
, N
, N
, N
, N
, N
, N
,
4496 N
, N
, N
, N
, N
, N
, N
, N
,
4498 N
, N
, N
, N
, N
, N
, N
, N
,
4500 N
, N
, N
, N
, N
, N
, N
, N
,
4502 N
, N
, N
, N
, N
, N
, N
, N
,
4504 N
, N
, N
, N
, N
, N
, N
, N
,
4507 static const struct instr_dual instr_dual_0f_c3
= {
4508 I(DstMem
| SrcReg
| ModRM
| No16
| Mov
, em_mov
), N
4511 static const struct mode_dual mode_dual_63
= {
4512 N
, I(DstReg
| SrcMem32
| ModRM
| Mov
, em_movsxd
)
4515 static const struct opcode opcode_table
[256] = {
4517 F6ALU(Lock
, em_add
),
4518 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_push_sreg
),
4519 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_pop_sreg
),
4521 F6ALU(Lock
| PageTable
, em_or
),
4522 I(ImplicitOps
| Stack
| No64
| Src2CS
, em_push_sreg
),
4525 F6ALU(Lock
, em_adc
),
4526 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_push_sreg
),
4527 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_pop_sreg
),
4529 F6ALU(Lock
, em_sbb
),
4530 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_push_sreg
),
4531 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_pop_sreg
),
4533 F6ALU(Lock
| PageTable
, em_and
), N
, N
,
4535 F6ALU(Lock
, em_sub
), N
, I(ByteOp
| DstAcc
| No64
, em_das
),
4537 F6ALU(Lock
, em_xor
), N
, N
,
4539 F6ALU(NoWrite
, em_cmp
), N
, N
,
4541 X8(F(DstReg
, em_inc
)), X8(F(DstReg
, em_dec
)),
4543 X8(I(SrcReg
| Stack
, em_push
)),
4545 X8(I(DstReg
| Stack
, em_pop
)),
4547 I(ImplicitOps
| Stack
| No64
, em_pusha
),
4548 I(ImplicitOps
| Stack
| No64
, em_popa
),
4549 N
, MD(ModRM
, &mode_dual_63
),
4552 I(SrcImm
| Mov
| Stack
, em_push
),
4553 I(DstReg
| SrcMem
| ModRM
| Src2Imm
, em_imul_3op
),
4554 I(SrcImmByte
| Mov
| Stack
, em_push
),
4555 I(DstReg
| SrcMem
| ModRM
| Src2ImmByte
, em_imul_3op
),
4556 I2bvIP(DstDI
| SrcDX
| Mov
| String
| Unaligned
, em_in
, ins
, check_perm_in
), /* insb, insw/insd */
4557 I2bvIP(SrcSI
| DstDX
| String
, em_out
, outs
, check_perm_out
), /* outsb, outsw/outsd */
4559 X16(D(SrcImmByte
| NearBranch
)),
4561 G(ByteOp
| DstMem
| SrcImm
, group1
),
4562 G(DstMem
| SrcImm
, group1
),
4563 G(ByteOp
| DstMem
| SrcImm
| No64
, group1
),
4564 G(DstMem
| SrcImmByte
, group1
),
4565 F2bv(DstMem
| SrcReg
| ModRM
| NoWrite
, em_test
),
4566 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
, em_xchg
),
4568 I2bv(DstMem
| SrcReg
| ModRM
| Mov
| PageTable
, em_mov
),
4569 I2bv(DstReg
| SrcMem
| ModRM
| Mov
, em_mov
),
4570 I(DstMem
| SrcNone
| ModRM
| Mov
| PageTable
, em_mov_rm_sreg
),
4571 D(ModRM
| SrcMem
| NoAccess
| DstReg
),
4572 I(ImplicitOps
| SrcMem16
| ModRM
, em_mov_sreg_rm
),
4575 DI(SrcAcc
| DstReg
, pause
), X7(D(SrcAcc
| DstReg
)),
4577 D(DstAcc
| SrcNone
), I(ImplicitOps
| SrcAcc
, em_cwd
),
4578 I(SrcImmFAddr
| No64
, em_call_far
), N
,
4579 II(ImplicitOps
| Stack
, em_pushf
, pushf
),
4580 II(ImplicitOps
| Stack
, em_popf
, popf
),
4581 I(ImplicitOps
, em_sahf
), I(ImplicitOps
, em_lahf
),
4583 I2bv(DstAcc
| SrcMem
| Mov
| MemAbs
, em_mov
),
4584 I2bv(DstMem
| SrcAcc
| Mov
| MemAbs
| PageTable
, em_mov
),
4585 I2bv(SrcSI
| DstDI
| Mov
| String
| TwoMemOp
, em_mov
),
4586 F2bv(SrcSI
| DstDI
| String
| NoWrite
| TwoMemOp
, em_cmp_r
),
4588 F2bv(DstAcc
| SrcImm
| NoWrite
, em_test
),
4589 I2bv(SrcAcc
| DstDI
| Mov
| String
, em_mov
),
4590 I2bv(SrcSI
| DstAcc
| Mov
| String
, em_mov
),
4591 F2bv(SrcAcc
| DstDI
| String
| NoWrite
, em_cmp_r
),
4593 X8(I(ByteOp
| DstReg
| SrcImm
| Mov
, em_mov
)),
4595 X8(I(DstReg
| SrcImm64
| Mov
, em_mov
)),
4597 G(ByteOp
| Src2ImmByte
, group2
), G(Src2ImmByte
, group2
),
4598 I(ImplicitOps
| NearBranch
| SrcImmU16
, em_ret_near_imm
),
4599 I(ImplicitOps
| NearBranch
, em_ret
),
4600 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2ES
, em_lseg
),
4601 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2DS
, em_lseg
),
4602 G(ByteOp
, group11
), G(0, group11
),
4604 I(Stack
| SrcImmU16
| Src2ImmByte
, em_enter
), I(Stack
, em_leave
),
4605 I(ImplicitOps
| SrcImmU16
, em_ret_far_imm
),
4606 I(ImplicitOps
, em_ret_far
),
4607 D(ImplicitOps
), DI(SrcImmByte
, intn
),
4608 D(ImplicitOps
| No64
), II(ImplicitOps
, em_iret
, iret
),
4610 G(Src2One
| ByteOp
, group2
), G(Src2One
, group2
),
4611 G(Src2CL
| ByteOp
, group2
), G(Src2CL
, group2
),
4612 I(DstAcc
| SrcImmUByte
| No64
, em_aam
),
4613 I(DstAcc
| SrcImmUByte
| No64
, em_aad
),
4614 F(DstAcc
| ByteOp
| No64
, em_salc
),
4615 I(DstAcc
| SrcXLat
| ByteOp
, em_mov
),
4617 N
, E(0, &escape_d9
), N
, E(0, &escape_db
), N
, E(0, &escape_dd
), N
, N
,
4619 X3(I(SrcImmByte
| NearBranch
, em_loop
)),
4620 I(SrcImmByte
| NearBranch
, em_jcxz
),
4621 I2bvIP(SrcImmUByte
| DstAcc
, em_in
, in
, check_perm_in
),
4622 I2bvIP(SrcAcc
| DstImmUByte
, em_out
, out
, check_perm_out
),
4624 I(SrcImm
| NearBranch
, em_call
), D(SrcImm
| ImplicitOps
| NearBranch
),
4625 I(SrcImmFAddr
| No64
, em_jmp_far
),
4626 D(SrcImmByte
| ImplicitOps
| NearBranch
),
4627 I2bvIP(SrcDX
| DstAcc
, em_in
, in
, check_perm_in
),
4628 I2bvIP(SrcAcc
| DstDX
, em_out
, out
, check_perm_out
),
4630 N
, DI(ImplicitOps
, icebp
), N
, N
,
4631 DI(ImplicitOps
| Priv
, hlt
), D(ImplicitOps
),
4632 G(ByteOp
, group3
), G(0, group3
),
4634 D(ImplicitOps
), D(ImplicitOps
),
4635 I(ImplicitOps
, em_cli
), I(ImplicitOps
, em_sti
),
4636 D(ImplicitOps
), D(ImplicitOps
), G(0, group4
), G(0, group5
),
4639 static const struct opcode twobyte_table
[256] = {
4641 G(0, group6
), GD(0, &group7
), N
, N
,
4642 N
, I(ImplicitOps
| EmulateOnUD
, em_syscall
),
4643 II(ImplicitOps
| Priv
, em_clts
, clts
), N
,
4644 DI(ImplicitOps
| Priv
, invd
), DI(ImplicitOps
| Priv
, wbinvd
), N
, N
,
4645 N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
), N
, N
,
4647 N
, N
, N
, N
, N
, N
, N
, N
,
4648 D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4649 N
, N
, N
, N
, N
, N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4651 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, cr_read
, check_cr_read
),
4652 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, dr_read
, check_dr_read
),
4653 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_cr_write
, cr_write
,
4655 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_dr_write
, dr_write
,
4658 GP(ModRM
| DstReg
| SrcMem
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4659 GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4660 N
, GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_2b
),
4663 II(ImplicitOps
| Priv
, em_wrmsr
, wrmsr
),
4664 IIP(ImplicitOps
, em_rdtsc
, rdtsc
, check_rdtsc
),
4665 II(ImplicitOps
| Priv
, em_rdmsr
, rdmsr
),
4666 IIP(ImplicitOps
, em_rdpmc
, rdpmc
, check_rdpmc
),
4667 I(ImplicitOps
| EmulateOnUD
, em_sysenter
),
4668 I(ImplicitOps
| Priv
| EmulateOnUD
, em_sysexit
),
4670 N
, N
, N
, N
, N
, N
, N
, N
,
4672 X16(D(DstReg
| SrcMem
| ModRM
)),
4674 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4679 N
, N
, N
, GP(SrcMem
| DstReg
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4684 N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4686 X16(D(SrcImm
| NearBranch
)),
4688 X16(D(ByteOp
| DstMem
| SrcNone
| ModRM
| Mov
)),
4690 I(Stack
| Src2FS
, em_push_sreg
), I(Stack
| Src2FS
, em_pop_sreg
),
4691 II(ImplicitOps
, em_cpuid
, cpuid
),
4692 F(DstMem
| SrcReg
| ModRM
| BitOp
| NoWrite
, em_bt
),
4693 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shld
),
4694 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shld
), N
, N
,
4696 I(Stack
| Src2GS
, em_push_sreg
), I(Stack
| Src2GS
, em_pop_sreg
),
4697 II(EmulateOnUD
| ImplicitOps
, em_rsm
, rsm
),
4698 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_bts
),
4699 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shrd
),
4700 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shrd
),
4701 GD(0, &group15
), F(DstReg
| SrcMem
| ModRM
, em_imul
),
4703 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
| SrcWrite
, em_cmpxchg
),
4704 I(DstReg
| SrcMemFAddr
| ModRM
| Src2SS
, em_lseg
),
4705 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
, em_btr
),
4706 I(DstReg
| SrcMemFAddr
| ModRM
| Src2FS
, em_lseg
),
4707 I(DstReg
| SrcMemFAddr
| ModRM
| Src2GS
, em_lseg
),
4708 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4712 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_btc
),
4713 I(DstReg
| SrcMem
| ModRM
, em_bsf_c
),
4714 I(DstReg
| SrcMem
| ModRM
, em_bsr_c
),
4715 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4717 F2bv(DstMem
| SrcReg
| ModRM
| SrcWrite
| Lock
, em_xadd
),
4718 N
, ID(0, &instr_dual_0f_c3
),
4719 N
, N
, N
, GD(0, &group9
),
4721 X8(I(DstReg
, em_bswap
)),
4723 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4725 N
, N
, N
, N
, N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_e7
),
4726 N
, N
, N
, N
, N
, N
, N
, N
,
4728 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
4731 static const struct instr_dual instr_dual_0f_38_f0
= {
4732 I(DstReg
| SrcMem
| Mov
, em_movbe
), N
4735 static const struct instr_dual instr_dual_0f_38_f1
= {
4736 I(DstMem
| SrcReg
| Mov
, em_movbe
), N
4739 static const struct gprefix three_byte_0f_38_f0
= {
4740 ID(0, &instr_dual_0f_38_f0
), N
, N
, N
4743 static const struct gprefix three_byte_0f_38_f1
= {
4744 ID(0, &instr_dual_0f_38_f1
), N
, N
, N
4748 * Insns below are selected by the prefix which indexed by the third opcode
4751 static const struct opcode opcode_map_0f_38
[256] = {
4753 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4755 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4757 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f0
),
4758 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f1
),
4779 static unsigned imm_size(struct x86_emulate_ctxt
*ctxt
)
4783 size
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4789 static int decode_imm(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4790 unsigned size
, bool sign_extension
)
4792 int rc
= X86EMUL_CONTINUE
;
4796 op
->addr
.mem
.ea
= ctxt
->_eip
;
4797 /* NB. Immediates are sign-extended as necessary. */
4798 switch (op
->bytes
) {
4800 op
->val
= insn_fetch(s8
, ctxt
);
4803 op
->val
= insn_fetch(s16
, ctxt
);
4806 op
->val
= insn_fetch(s32
, ctxt
);
4809 op
->val
= insn_fetch(s64
, ctxt
);
4812 if (!sign_extension
) {
4813 switch (op
->bytes
) {
4821 op
->val
&= 0xffffffff;
4829 static int decode_operand(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4832 int rc
= X86EMUL_CONTINUE
;
4836 decode_register_operand(ctxt
, op
);
4839 rc
= decode_imm(ctxt
, op
, 1, false);
4842 ctxt
->memop
.bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4846 if (ctxt
->d
& BitOp
)
4847 fetch_bit_operand(ctxt
);
4848 op
->orig_val
= op
->val
;
4851 ctxt
->memop
.bytes
= (ctxt
->op_bytes
== 8) ? 16 : 8;
4855 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4856 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4857 fetch_register_operand(op
);
4858 op
->orig_val
= op
->val
;
4862 op
->bytes
= (ctxt
->d
& ByteOp
) ? 2 : ctxt
->op_bytes
;
4863 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4864 fetch_register_operand(op
);
4865 op
->orig_val
= op
->val
;
4868 if (ctxt
->d
& ByteOp
) {
4873 op
->bytes
= ctxt
->op_bytes
;
4874 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4875 fetch_register_operand(op
);
4876 op
->orig_val
= op
->val
;
4880 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4882 register_address(ctxt
, VCPU_REGS_RDI
);
4883 op
->addr
.mem
.seg
= VCPU_SREG_ES
;
4890 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4891 fetch_register_operand(op
);
4896 op
->val
= reg_read(ctxt
, VCPU_REGS_RCX
) & 0xff;
4899 rc
= decode_imm(ctxt
, op
, 1, true);
4907 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), true);
4910 rc
= decode_imm(ctxt
, op
, ctxt
->op_bytes
, true);
4913 ctxt
->memop
.bytes
= 1;
4914 if (ctxt
->memop
.type
== OP_REG
) {
4915 ctxt
->memop
.addr
.reg
= decode_register(ctxt
,
4916 ctxt
->modrm_rm
, true);
4917 fetch_register_operand(&ctxt
->memop
);
4921 ctxt
->memop
.bytes
= 2;
4924 ctxt
->memop
.bytes
= 4;
4927 rc
= decode_imm(ctxt
, op
, 2, false);
4930 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), false);
4934 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4936 register_address(ctxt
, VCPU_REGS_RSI
);
4937 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4943 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4946 reg_read(ctxt
, VCPU_REGS_RBX
) +
4947 (reg_read(ctxt
, VCPU_REGS_RAX
) & 0xff));
4948 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4953 op
->addr
.mem
.ea
= ctxt
->_eip
;
4954 op
->bytes
= ctxt
->op_bytes
+ 2;
4955 insn_fetch_arr(op
->valptr
, op
->bytes
, ctxt
);
4958 ctxt
->memop
.bytes
= ctxt
->op_bytes
+ 2;
4962 op
->val
= VCPU_SREG_ES
;
4966 op
->val
= VCPU_SREG_CS
;
4970 op
->val
= VCPU_SREG_SS
;
4974 op
->val
= VCPU_SREG_DS
;
4978 op
->val
= VCPU_SREG_FS
;
4982 op
->val
= VCPU_SREG_GS
;
4985 /* Special instructions do their own operand decoding. */
4987 op
->type
= OP_NONE
; /* Disable writeback. */
4995 int x86_decode_insn(struct x86_emulate_ctxt
*ctxt
, void *insn
, int insn_len
)
4997 int rc
= X86EMUL_CONTINUE
;
4998 int mode
= ctxt
->mode
;
4999 int def_op_bytes
, def_ad_bytes
, goffset
, simd_prefix
;
5000 bool op_prefix
= false;
5001 bool has_seg_override
= false;
5002 struct opcode opcode
;
5004 ctxt
->memop
.type
= OP_NONE
;
5005 ctxt
->memopp
= NULL
;
5006 ctxt
->_eip
= ctxt
->eip
;
5007 ctxt
->fetch
.ptr
= ctxt
->fetch
.data
;
5008 ctxt
->fetch
.end
= ctxt
->fetch
.data
+ insn_len
;
5009 ctxt
->opcode_len
= 1;
5011 memcpy(ctxt
->fetch
.data
, insn
, insn_len
);
5013 rc
= __do_insn_fetch_bytes(ctxt
, 1);
5014 if (rc
!= X86EMUL_CONTINUE
)
5019 case X86EMUL_MODE_REAL
:
5020 case X86EMUL_MODE_VM86
:
5021 case X86EMUL_MODE_PROT16
:
5022 def_op_bytes
= def_ad_bytes
= 2;
5024 case X86EMUL_MODE_PROT32
:
5025 def_op_bytes
= def_ad_bytes
= 4;
5027 #ifdef CONFIG_X86_64
5028 case X86EMUL_MODE_PROT64
:
5034 return EMULATION_FAILED
;
5037 ctxt
->op_bytes
= def_op_bytes
;
5038 ctxt
->ad_bytes
= def_ad_bytes
;
5040 /* Legacy prefixes. */
5042 switch (ctxt
->b
= insn_fetch(u8
, ctxt
)) {
5043 case 0x66: /* operand-size override */
5045 /* switch between 2/4 bytes */
5046 ctxt
->op_bytes
= def_op_bytes
^ 6;
5048 case 0x67: /* address-size override */
5049 if (mode
== X86EMUL_MODE_PROT64
)
5050 /* switch between 4/8 bytes */
5051 ctxt
->ad_bytes
= def_ad_bytes
^ 12;
5053 /* switch between 2/4 bytes */
5054 ctxt
->ad_bytes
= def_ad_bytes
^ 6;
5056 case 0x26: /* ES override */
5057 case 0x2e: /* CS override */
5058 case 0x36: /* SS override */
5059 case 0x3e: /* DS override */
5060 has_seg_override
= true;
5061 ctxt
->seg_override
= (ctxt
->b
>> 3) & 3;
5063 case 0x64: /* FS override */
5064 case 0x65: /* GS override */
5065 has_seg_override
= true;
5066 ctxt
->seg_override
= ctxt
->b
& 7;
5068 case 0x40 ... 0x4f: /* REX */
5069 if (mode
!= X86EMUL_MODE_PROT64
)
5071 ctxt
->rex_prefix
= ctxt
->b
;
5073 case 0xf0: /* LOCK */
5074 ctxt
->lock_prefix
= 1;
5076 case 0xf2: /* REPNE/REPNZ */
5077 case 0xf3: /* REP/REPE/REPZ */
5078 ctxt
->rep_prefix
= ctxt
->b
;
5084 /* Any legacy prefix after a REX prefix nullifies its effect. */
5086 ctxt
->rex_prefix
= 0;
5092 if (ctxt
->rex_prefix
& 8)
5093 ctxt
->op_bytes
= 8; /* REX.W */
5095 /* Opcode byte(s). */
5096 opcode
= opcode_table
[ctxt
->b
];
5097 /* Two-byte opcode? */
5098 if (ctxt
->b
== 0x0f) {
5099 ctxt
->opcode_len
= 2;
5100 ctxt
->b
= insn_fetch(u8
, ctxt
);
5101 opcode
= twobyte_table
[ctxt
->b
];
5103 /* 0F_38 opcode map */
5104 if (ctxt
->b
== 0x38) {
5105 ctxt
->opcode_len
= 3;
5106 ctxt
->b
= insn_fetch(u8
, ctxt
);
5107 opcode
= opcode_map_0f_38
[ctxt
->b
];
5110 ctxt
->d
= opcode
.flags
;
5112 if (ctxt
->d
& ModRM
)
5113 ctxt
->modrm
= insn_fetch(u8
, ctxt
);
5115 /* vex-prefix instructions are not implemented */
5116 if (ctxt
->opcode_len
== 1 && (ctxt
->b
== 0xc5 || ctxt
->b
== 0xc4) &&
5117 (mode
== X86EMUL_MODE_PROT64
|| (ctxt
->modrm
& 0xc0) == 0xc0)) {
5121 while (ctxt
->d
& GroupMask
) {
5122 switch (ctxt
->d
& GroupMask
) {
5124 goffset
= (ctxt
->modrm
>> 3) & 7;
5125 opcode
= opcode
.u
.group
[goffset
];
5128 goffset
= (ctxt
->modrm
>> 3) & 7;
5129 if ((ctxt
->modrm
>> 6) == 3)
5130 opcode
= opcode
.u
.gdual
->mod3
[goffset
];
5132 opcode
= opcode
.u
.gdual
->mod012
[goffset
];
5135 goffset
= ctxt
->modrm
& 7;
5136 opcode
= opcode
.u
.group
[goffset
];
5139 if (ctxt
->rep_prefix
&& op_prefix
)
5140 return EMULATION_FAILED
;
5141 simd_prefix
= op_prefix
? 0x66 : ctxt
->rep_prefix
;
5142 switch (simd_prefix
) {
5143 case 0x00: opcode
= opcode
.u
.gprefix
->pfx_no
; break;
5144 case 0x66: opcode
= opcode
.u
.gprefix
->pfx_66
; break;
5145 case 0xf2: opcode
= opcode
.u
.gprefix
->pfx_f2
; break;
5146 case 0xf3: opcode
= opcode
.u
.gprefix
->pfx_f3
; break;
5150 if (ctxt
->modrm
> 0xbf)
5151 opcode
= opcode
.u
.esc
->high
[ctxt
->modrm
- 0xc0];
5153 opcode
= opcode
.u
.esc
->op
[(ctxt
->modrm
>> 3) & 7];
5156 if ((ctxt
->modrm
>> 6) == 3)
5157 opcode
= opcode
.u
.idual
->mod3
;
5159 opcode
= opcode
.u
.idual
->mod012
;
5162 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
5163 opcode
= opcode
.u
.mdual
->mode64
;
5165 opcode
= opcode
.u
.mdual
->mode32
;
5168 return EMULATION_FAILED
;
5171 ctxt
->d
&= ~(u64
)GroupMask
;
5172 ctxt
->d
|= opcode
.flags
;
5177 return EMULATION_FAILED
;
5179 ctxt
->execute
= opcode
.u
.execute
;
5181 if (unlikely(ctxt
->ud
) && likely(!(ctxt
->d
& EmulateOnUD
)))
5182 return EMULATION_FAILED
;
5184 if (unlikely(ctxt
->d
&
5185 (NotImpl
|Stack
|Op3264
|Sse
|Mmx
|Intercept
|CheckPerm
|NearBranch
|
5188 * These are copied unconditionally here, and checked unconditionally
5189 * in x86_emulate_insn.
5191 ctxt
->check_perm
= opcode
.check_perm
;
5192 ctxt
->intercept
= opcode
.intercept
;
5194 if (ctxt
->d
& NotImpl
)
5195 return EMULATION_FAILED
;
5197 if (mode
== X86EMUL_MODE_PROT64
) {
5198 if (ctxt
->op_bytes
== 4 && (ctxt
->d
& Stack
))
5200 else if (ctxt
->d
& NearBranch
)
5204 if (ctxt
->d
& Op3264
) {
5205 if (mode
== X86EMUL_MODE_PROT64
)
5211 if ((ctxt
->d
& No16
) && ctxt
->op_bytes
== 2)
5215 ctxt
->op_bytes
= 16;
5216 else if (ctxt
->d
& Mmx
)
5220 /* ModRM and SIB bytes. */
5221 if (ctxt
->d
& ModRM
) {
5222 rc
= decode_modrm(ctxt
, &ctxt
->memop
);
5223 if (!has_seg_override
) {
5224 has_seg_override
= true;
5225 ctxt
->seg_override
= ctxt
->modrm_seg
;
5227 } else if (ctxt
->d
& MemAbs
)
5228 rc
= decode_abs(ctxt
, &ctxt
->memop
);
5229 if (rc
!= X86EMUL_CONTINUE
)
5232 if (!has_seg_override
)
5233 ctxt
->seg_override
= VCPU_SREG_DS
;
5235 ctxt
->memop
.addr
.mem
.seg
= ctxt
->seg_override
;
5238 * Decode and fetch the source operand: register, memory
5241 rc
= decode_operand(ctxt
, &ctxt
->src
, (ctxt
->d
>> SrcShift
) & OpMask
);
5242 if (rc
!= X86EMUL_CONTINUE
)
5246 * Decode and fetch the second source operand: register, memory
5249 rc
= decode_operand(ctxt
, &ctxt
->src2
, (ctxt
->d
>> Src2Shift
) & OpMask
);
5250 if (rc
!= X86EMUL_CONTINUE
)
5253 /* Decode and fetch the destination operand: register or memory. */
5254 rc
= decode_operand(ctxt
, &ctxt
->dst
, (ctxt
->d
>> DstShift
) & OpMask
);
5256 if (ctxt
->rip_relative
&& likely(ctxt
->memopp
))
5257 ctxt
->memopp
->addr
.mem
.ea
= address_mask(ctxt
,
5258 ctxt
->memopp
->addr
.mem
.ea
+ ctxt
->_eip
);
5261 return (rc
!= X86EMUL_CONTINUE
) ? EMULATION_FAILED
: EMULATION_OK
;
5264 bool x86_page_table_writing_insn(struct x86_emulate_ctxt
*ctxt
)
5266 return ctxt
->d
& PageTable
;
5269 static bool string_insn_completed(struct x86_emulate_ctxt
*ctxt
)
5271 /* The second termination condition only applies for REPE
5272 * and REPNE. Test if the repeat string operation prefix is
5273 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5274 * corresponding termination condition according to:
5275 * - if REPE/REPZ and ZF = 0 then done
5276 * - if REPNE/REPNZ and ZF = 1 then done
5278 if (((ctxt
->b
== 0xa6) || (ctxt
->b
== 0xa7) ||
5279 (ctxt
->b
== 0xae) || (ctxt
->b
== 0xaf))
5280 && (((ctxt
->rep_prefix
== REPE_PREFIX
) &&
5281 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == 0))
5282 || ((ctxt
->rep_prefix
== REPNE_PREFIX
) &&
5283 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == X86_EFLAGS_ZF
))))
5289 static int flush_pending_x87_faults(struct x86_emulate_ctxt
*ctxt
)
5293 ctxt
->ops
->get_fpu(ctxt
);
5294 rc
= asm_safe("fwait");
5295 ctxt
->ops
->put_fpu(ctxt
);
5297 if (unlikely(rc
!= X86EMUL_CONTINUE
))
5298 return emulate_exception(ctxt
, MF_VECTOR
, 0, false);
5300 return X86EMUL_CONTINUE
;
5303 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt
*ctxt
,
5306 if (op
->type
== OP_MM
)
5307 read_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
5310 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*))
5312 ulong flags
= (ctxt
->eflags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
5314 if (!(ctxt
->d
& ByteOp
))
5315 fop
+= __ffs(ctxt
->dst
.bytes
) * FASTOP_SIZE
;
5317 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5318 : "+a"(ctxt
->dst
.val
), "+d"(ctxt
->src
.val
), [flags
]"+D"(flags
),
5319 [fastop
]"+S"(fop
), ASM_CALL_CONSTRAINT
5320 : "c"(ctxt
->src2
.val
));
5322 ctxt
->eflags
= (ctxt
->eflags
& ~EFLAGS_MASK
) | (flags
& EFLAGS_MASK
);
5323 if (!fop
) /* exception is returned in fop variable */
5324 return emulate_de(ctxt
);
5325 return X86EMUL_CONTINUE
;
5328 void init_decode_cache(struct x86_emulate_ctxt
*ctxt
)
5330 memset(&ctxt
->rip_relative
, 0,
5331 (void *)&ctxt
->modrm
- (void *)&ctxt
->rip_relative
);
5333 ctxt
->io_read
.pos
= 0;
5334 ctxt
->io_read
.end
= 0;
5335 ctxt
->mem_read
.end
= 0;
5338 int x86_emulate_insn(struct x86_emulate_ctxt
*ctxt
)
5340 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
5341 int rc
= X86EMUL_CONTINUE
;
5342 int saved_dst_type
= ctxt
->dst
.type
;
5343 unsigned emul_flags
;
5345 ctxt
->mem_read
.pos
= 0;
5347 /* LOCK prefix is allowed only with some instructions */
5348 if (ctxt
->lock_prefix
&& (!(ctxt
->d
& Lock
) || ctxt
->dst
.type
!= OP_MEM
)) {
5349 rc
= emulate_ud(ctxt
);
5353 if ((ctxt
->d
& SrcMask
) == SrcMemFAddr
&& ctxt
->src
.type
!= OP_MEM
) {
5354 rc
= emulate_ud(ctxt
);
5358 emul_flags
= ctxt
->ops
->get_hflags(ctxt
);
5359 if (unlikely(ctxt
->d
&
5360 (No64
|Undefined
|Sse
|Mmx
|Intercept
|CheckPerm
|Priv
|Prot
|String
))) {
5361 if ((ctxt
->mode
== X86EMUL_MODE_PROT64
&& (ctxt
->d
& No64
)) ||
5362 (ctxt
->d
& Undefined
)) {
5363 rc
= emulate_ud(ctxt
);
5367 if (((ctxt
->d
& (Sse
|Mmx
)) && ((ops
->get_cr(ctxt
, 0) & X86_CR0_EM
)))
5368 || ((ctxt
->d
& Sse
) && !(ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
))) {
5369 rc
= emulate_ud(ctxt
);
5373 if ((ctxt
->d
& (Sse
|Mmx
)) && (ops
->get_cr(ctxt
, 0) & X86_CR0_TS
)) {
5374 rc
= emulate_nm(ctxt
);
5378 if (ctxt
->d
& Mmx
) {
5379 rc
= flush_pending_x87_faults(ctxt
);
5380 if (rc
!= X86EMUL_CONTINUE
)
5383 * Now that we know the fpu is exception safe, we can fetch
5386 fetch_possible_mmx_operand(ctxt
, &ctxt
->src
);
5387 fetch_possible_mmx_operand(ctxt
, &ctxt
->src2
);
5388 if (!(ctxt
->d
& Mov
))
5389 fetch_possible_mmx_operand(ctxt
, &ctxt
->dst
);
5392 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && ctxt
->intercept
) {
5393 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5394 X86_ICPT_PRE_EXCEPT
);
5395 if (rc
!= X86EMUL_CONTINUE
)
5399 /* Instruction can only be executed in protected mode */
5400 if ((ctxt
->d
& Prot
) && ctxt
->mode
< X86EMUL_MODE_PROT16
) {
5401 rc
= emulate_ud(ctxt
);
5405 /* Privileged instruction can be executed only in CPL=0 */
5406 if ((ctxt
->d
& Priv
) && ops
->cpl(ctxt
)) {
5407 if (ctxt
->d
& PrivUD
)
5408 rc
= emulate_ud(ctxt
);
5410 rc
= emulate_gp(ctxt
, 0);
5414 /* Do instruction specific permission checks */
5415 if (ctxt
->d
& CheckPerm
) {
5416 rc
= ctxt
->check_perm(ctxt
);
5417 if (rc
!= X86EMUL_CONTINUE
)
5421 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5422 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5423 X86_ICPT_POST_EXCEPT
);
5424 if (rc
!= X86EMUL_CONTINUE
)
5428 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5429 /* All REP prefixes have the same first termination condition */
5430 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0) {
5431 string_registers_quirk(ctxt
);
5432 ctxt
->eip
= ctxt
->_eip
;
5433 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5439 if ((ctxt
->src
.type
== OP_MEM
) && !(ctxt
->d
& NoAccess
)) {
5440 rc
= segmented_read(ctxt
, ctxt
->src
.addr
.mem
,
5441 ctxt
->src
.valptr
, ctxt
->src
.bytes
);
5442 if (rc
!= X86EMUL_CONTINUE
)
5444 ctxt
->src
.orig_val64
= ctxt
->src
.val64
;
5447 if (ctxt
->src2
.type
== OP_MEM
) {
5448 rc
= segmented_read(ctxt
, ctxt
->src2
.addr
.mem
,
5449 &ctxt
->src2
.val
, ctxt
->src2
.bytes
);
5450 if (rc
!= X86EMUL_CONTINUE
)
5454 if ((ctxt
->d
& DstMask
) == ImplicitOps
)
5458 if ((ctxt
->dst
.type
== OP_MEM
) && !(ctxt
->d
& Mov
)) {
5459 /* optimisation - avoid slow emulated read if Mov */
5460 rc
= segmented_read(ctxt
, ctxt
->dst
.addr
.mem
,
5461 &ctxt
->dst
.val
, ctxt
->dst
.bytes
);
5462 if (rc
!= X86EMUL_CONTINUE
) {
5463 if (!(ctxt
->d
& NoWrite
) &&
5464 rc
== X86EMUL_PROPAGATE_FAULT
&&
5465 ctxt
->exception
.vector
== PF_VECTOR
)
5466 ctxt
->exception
.error_code
|= PFERR_WRITE_MASK
;
5470 /* Copy full 64-bit value for CMPXCHG8B. */
5471 ctxt
->dst
.orig_val64
= ctxt
->dst
.val64
;
5475 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5476 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5477 X86_ICPT_POST_MEMACCESS
);
5478 if (rc
!= X86EMUL_CONTINUE
)
5482 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5483 ctxt
->eflags
|= X86_EFLAGS_RF
;
5485 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5487 if (ctxt
->execute
) {
5488 if (ctxt
->d
& Fastop
) {
5489 void (*fop
)(struct fastop
*) = (void *)ctxt
->execute
;
5490 rc
= fastop(ctxt
, fop
);
5491 if (rc
!= X86EMUL_CONTINUE
)
5495 rc
= ctxt
->execute(ctxt
);
5496 if (rc
!= X86EMUL_CONTINUE
)
5501 if (ctxt
->opcode_len
== 2)
5503 else if (ctxt
->opcode_len
== 3)
5504 goto threebyte_insn
;
5507 case 0x70 ... 0x7f: /* jcc (short) */
5508 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5509 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5511 case 0x8d: /* lea r16/r32, m */
5512 ctxt
->dst
.val
= ctxt
->src
.addr
.mem
.ea
;
5514 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5515 if (ctxt
->dst
.addr
.reg
== reg_rmw(ctxt
, VCPU_REGS_RAX
))
5516 ctxt
->dst
.type
= OP_NONE
;
5520 case 0x98: /* cbw/cwde/cdqe */
5521 switch (ctxt
->op_bytes
) {
5522 case 2: ctxt
->dst
.val
= (s8
)ctxt
->dst
.val
; break;
5523 case 4: ctxt
->dst
.val
= (s16
)ctxt
->dst
.val
; break;
5524 case 8: ctxt
->dst
.val
= (s32
)ctxt
->dst
.val
; break;
5527 case 0xcc: /* int3 */
5528 rc
= emulate_int(ctxt
, 3);
5530 case 0xcd: /* int n */
5531 rc
= emulate_int(ctxt
, ctxt
->src
.val
);
5533 case 0xce: /* into */
5534 if (ctxt
->eflags
& X86_EFLAGS_OF
)
5535 rc
= emulate_int(ctxt
, 4);
5537 case 0xe9: /* jmp rel */
5538 case 0xeb: /* jmp rel short */
5539 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5540 ctxt
->dst
.type
= OP_NONE
; /* Disable writeback. */
5542 case 0xf4: /* hlt */
5543 ctxt
->ops
->halt(ctxt
);
5545 case 0xf5: /* cmc */
5546 /* complement carry flag from eflags reg */
5547 ctxt
->eflags
^= X86_EFLAGS_CF
;
5549 case 0xf8: /* clc */
5550 ctxt
->eflags
&= ~X86_EFLAGS_CF
;
5552 case 0xf9: /* stc */
5553 ctxt
->eflags
|= X86_EFLAGS_CF
;
5555 case 0xfc: /* cld */
5556 ctxt
->eflags
&= ~X86_EFLAGS_DF
;
5558 case 0xfd: /* std */
5559 ctxt
->eflags
|= X86_EFLAGS_DF
;
5562 goto cannot_emulate
;
5565 if (rc
!= X86EMUL_CONTINUE
)
5569 if (ctxt
->d
& SrcWrite
) {
5570 BUG_ON(ctxt
->src
.type
== OP_MEM
|| ctxt
->src
.type
== OP_MEM_STR
);
5571 rc
= writeback(ctxt
, &ctxt
->src
);
5572 if (rc
!= X86EMUL_CONTINUE
)
5575 if (!(ctxt
->d
& NoWrite
)) {
5576 rc
= writeback(ctxt
, &ctxt
->dst
);
5577 if (rc
!= X86EMUL_CONTINUE
)
5582 * restore dst type in case the decoding will be reused
5583 * (happens for string instruction )
5585 ctxt
->dst
.type
= saved_dst_type
;
5587 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5588 string_addr_inc(ctxt
, VCPU_REGS_RSI
, &ctxt
->src
);
5590 if ((ctxt
->d
& DstMask
) == DstDI
)
5591 string_addr_inc(ctxt
, VCPU_REGS_RDI
, &ctxt
->dst
);
5593 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5595 struct read_cache
*r
= &ctxt
->io_read
;
5596 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5597 count
= ctxt
->src
.count
;
5599 count
= ctxt
->dst
.count
;
5600 register_address_increment(ctxt
, VCPU_REGS_RCX
, -count
);
5602 if (!string_insn_completed(ctxt
)) {
5604 * Re-enter guest when pio read ahead buffer is empty
5605 * or, if it is not used, after each 1024 iteration.
5607 if ((r
->end
!= 0 || reg_read(ctxt
, VCPU_REGS_RCX
) & 0x3ff) &&
5608 (r
->end
== 0 || r
->end
!= r
->pos
)) {
5610 * Reset read cache. Usually happens before
5611 * decode, but since instruction is restarted
5612 * we have to do it here.
5614 ctxt
->mem_read
.end
= 0;
5615 writeback_registers(ctxt
);
5616 return EMULATION_RESTART
;
5618 goto done
; /* skip rip writeback */
5620 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5623 ctxt
->eip
= ctxt
->_eip
;
5626 if (rc
== X86EMUL_PROPAGATE_FAULT
) {
5627 WARN_ON(ctxt
->exception
.vector
> 0x1f);
5628 ctxt
->have_exception
= true;
5630 if (rc
== X86EMUL_INTERCEPTED
)
5631 return EMULATION_INTERCEPTED
;
5633 if (rc
== X86EMUL_CONTINUE
)
5634 writeback_registers(ctxt
);
5636 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
5640 case 0x09: /* wbinvd */
5641 (ctxt
->ops
->wbinvd
)(ctxt
);
5643 case 0x08: /* invd */
5644 case 0x0d: /* GrpP (prefetch) */
5645 case 0x18: /* Grp16 (prefetch/nop) */
5646 case 0x1f: /* nop */
5648 case 0x20: /* mov cr, reg */
5649 ctxt
->dst
.val
= ops
->get_cr(ctxt
, ctxt
->modrm_reg
);
5651 case 0x21: /* mov from dr to reg */
5652 ops
->get_dr(ctxt
, ctxt
->modrm_reg
, &ctxt
->dst
.val
);
5654 case 0x40 ... 0x4f: /* cmov */
5655 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5656 ctxt
->dst
.val
= ctxt
->src
.val
;
5657 else if (ctxt
->op_bytes
!= 4)
5658 ctxt
->dst
.type
= OP_NONE
; /* no writeback */
5660 case 0x80 ... 0x8f: /* jnz rel, etc*/
5661 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5662 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5664 case 0x90 ... 0x9f: /* setcc r/m8 */
5665 ctxt
->dst
.val
= test_cc(ctxt
->b
, ctxt
->eflags
);
5667 case 0xb6 ... 0xb7: /* movzx */
5668 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5669 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (u8
) ctxt
->src
.val
5670 : (u16
) ctxt
->src
.val
;
5672 case 0xbe ... 0xbf: /* movsx */
5673 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5674 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (s8
) ctxt
->src
.val
:
5675 (s16
) ctxt
->src
.val
;
5678 goto cannot_emulate
;
5683 if (rc
!= X86EMUL_CONTINUE
)
5689 return EMULATION_FAILED
;
5692 void emulator_invalidate_register_cache(struct x86_emulate_ctxt
*ctxt
)
5694 invalidate_registers(ctxt
);
5697 void emulator_writeback_register_cache(struct x86_emulate_ctxt
*ctxt
)
5699 writeback_registers(ctxt
);
5702 bool emulator_can_use_gpa(struct x86_emulate_ctxt
*ctxt
)
5704 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5707 if (ctxt
->d
& TwoMemOp
)