1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define AlignMask ((u64)7 << 41)
162 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
163 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
164 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
165 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
166 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
167 #define NoWrite ((u64)1 << 45) /* No writeback */
168 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
169 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
170 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
171 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
172 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch ((u64)1 << 52) /* Near branches */
174 #define No16 ((u64)1 << 53) /* No 16 bit operand */
175 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
179 #define X2(x...) x, x
180 #define X3(x...) X2(x), x
181 #define X4(x...) X2(x), X2(x)
182 #define X5(x...) X4(x), x
183 #define X6(x...) X4(x), X2(x)
184 #define X7(x...) X4(x), X3(x)
185 #define X8(x...) X4(x), X4(x)
186 #define X16(x...) X8(x), X8(x)
188 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
189 #define FASTOP_SIZE 8
192 * fastop functions have a special calling convention:
197 * flags: rflags (in/out)
198 * ex: rsi (in:fastop pointer, out:zero if exception)
200 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
201 * different operand sizes can be reached by calculation, rather than a jump
202 * table (which would be bigger than the code).
204 * fastop functions are declared as taking a never-defined fastop parameter,
205 * so they can't be called from C directly.
214 int (*execute
)(struct x86_emulate_ctxt
*ctxt
);
215 const struct opcode
*group
;
216 const struct group_dual
*gdual
;
217 const struct gprefix
*gprefix
;
218 const struct escape
*esc
;
219 const struct instr_dual
*idual
;
220 const struct mode_dual
*mdual
;
221 void (*fastop
)(struct fastop
*fake
);
223 int (*check_perm
)(struct x86_emulate_ctxt
*ctxt
);
227 struct opcode mod012
[8];
228 struct opcode mod3
[8];
232 struct opcode pfx_no
;
233 struct opcode pfx_66
;
234 struct opcode pfx_f2
;
235 struct opcode pfx_f3
;
240 struct opcode high
[64];
244 struct opcode mod012
;
249 struct opcode mode32
;
250 struct opcode mode64
;
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
255 enum x86_transfer_type
{
257 X86_TRANSFER_CALL_JMP
,
259 X86_TRANSFER_TASK_SWITCH
,
262 static ulong
reg_read(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
264 if (!(ctxt
->regs_valid
& (1 << nr
))) {
265 ctxt
->regs_valid
|= 1 << nr
;
266 ctxt
->_regs
[nr
] = ctxt
->ops
->read_gpr(ctxt
, nr
);
268 return ctxt
->_regs
[nr
];
271 static ulong
*reg_write(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
273 ctxt
->regs_valid
|= 1 << nr
;
274 ctxt
->regs_dirty
|= 1 << nr
;
275 return &ctxt
->_regs
[nr
];
278 static ulong
*reg_rmw(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
281 return reg_write(ctxt
, nr
);
284 static void writeback_registers(struct x86_emulate_ctxt
*ctxt
)
288 for_each_set_bit(reg
, (ulong
*)&ctxt
->regs_dirty
, 16)
289 ctxt
->ops
->write_gpr(ctxt
, reg
, ctxt
->_regs
[reg
]);
292 static void invalidate_registers(struct x86_emulate_ctxt
*ctxt
)
294 ctxt
->regs_dirty
= 0;
295 ctxt
->regs_valid
= 0;
299 * These EFLAGS bits are restored from saved value during emulation, and
300 * any changes are written back to the saved value after emulation.
302 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
303 X86_EFLAGS_PF|X86_EFLAGS_CF)
311 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*));
313 #define FOP_FUNC(name) \
314 ".align " __stringify(FASTOP_SIZE) " \n\t" \
315 ".type " name ", @function \n\t" \
318 #define FOP_RET "ret \n\t"
320 #define FOP_START(op) \
321 extern void em_##op(struct fastop *fake); \
322 asm(".pushsection .text, \"ax\" \n\t" \
323 ".global em_" #op " \n\t" \
330 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
333 #define FOP1E(op, dst) \
334 FOP_FUNC(#op "_" #dst) \
335 "10: " #op " %" #dst " \n\t" FOP_RET
337 #define FOP1EEX(op, dst) \
338 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
340 #define FASTOP1(op) \
345 ON64(FOP1E(op##q, rax)) \
348 /* 1-operand, using src2 (for MUL/DIV r/m) */
349 #define FASTOP1SRC2(op, name) \
354 ON64(FOP1E(op, rcx)) \
357 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
358 #define FASTOP1SRC2EX(op, name) \
363 ON64(FOP1EEX(op, rcx)) \
366 #define FOP2E(op, dst, src) \
367 FOP_FUNC(#op "_" #dst "_" #src) \
368 #op " %" #src ", %" #dst " \n\t" FOP_RET
370 #define FASTOP2(op) \
372 FOP2E(op##b, al, dl) \
373 FOP2E(op##w, ax, dx) \
374 FOP2E(op##l, eax, edx) \
375 ON64(FOP2E(op##q, rax, rdx)) \
378 /* 2 operand, word only */
379 #define FASTOP2W(op) \
382 FOP2E(op##w, ax, dx) \
383 FOP2E(op##l, eax, edx) \
384 ON64(FOP2E(op##q, rax, rdx)) \
387 /* 2 operand, src is CL */
388 #define FASTOP2CL(op) \
390 FOP2E(op##b, al, cl) \
391 FOP2E(op##w, ax, cl) \
392 FOP2E(op##l, eax, cl) \
393 ON64(FOP2E(op##q, rax, cl)) \
396 /* 2 operand, src and dest are reversed */
397 #define FASTOP2R(op, name) \
399 FOP2E(op##b, dl, al) \
400 FOP2E(op##w, dx, ax) \
401 FOP2E(op##l, edx, eax) \
402 ON64(FOP2E(op##q, rdx, rax)) \
405 #define FOP3E(op, dst, src, src2) \
406 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
407 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
409 /* 3-operand, word-only, src2=cl */
410 #define FASTOP3WCL(op) \
413 FOP3E(op##w, ax, dx, cl) \
414 FOP3E(op##l, eax, edx, cl) \
415 ON64(FOP3E(op##q, rax, rdx, cl)) \
418 /* Special case for SETcc - 1 instruction per cc */
419 #define FOP_SETCC(op) \
421 ".type " #op ", @function \n\t" \
426 asm(".global kvm_fastop_exception \n"
427 "kvm_fastop_exception: xor %esi, %esi; ret");
448 FOP_START(salc
) "pushf; sbb %al, %al; popf \n\t" FOP_RET
452 * XXX: inoutclob user must know where the argument is being expanded.
453 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
455 #define asm_safe(insn, inoutclob...) \
459 asm volatile("1:" insn "\n" \
461 ".pushsection .fixup, \"ax\"\n" \
462 "3: movl $1, %[_fault]\n" \
465 _ASM_EXTABLE(1b, 3b) \
466 : [_fault] "+qm"(_fault) inoutclob ); \
468 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
471 static int emulator_check_intercept(struct x86_emulate_ctxt
*ctxt
,
472 enum x86_intercept intercept
,
473 enum x86_intercept_stage stage
)
475 struct x86_instruction_info info
= {
476 .intercept
= intercept
,
477 .rep_prefix
= ctxt
->rep_prefix
,
478 .modrm_mod
= ctxt
->modrm_mod
,
479 .modrm_reg
= ctxt
->modrm_reg
,
480 .modrm_rm
= ctxt
->modrm_rm
,
481 .src_val
= ctxt
->src
.val64
,
482 .dst_val
= ctxt
->dst
.val64
,
483 .src_bytes
= ctxt
->src
.bytes
,
484 .dst_bytes
= ctxt
->dst
.bytes
,
485 .ad_bytes
= ctxt
->ad_bytes
,
486 .next_rip
= ctxt
->eip
,
489 return ctxt
->ops
->intercept(ctxt
, &info
, stage
);
492 static void assign_masked(ulong
*dest
, ulong src
, ulong mask
)
494 *dest
= (*dest
& ~mask
) | (src
& mask
);
497 static void assign_register(unsigned long *reg
, u64 val
, int bytes
)
499 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
502 *(u8
*)reg
= (u8
)val
;
505 *(u16
*)reg
= (u16
)val
;
509 break; /* 64b: zero-extend */
516 static inline unsigned long ad_mask(struct x86_emulate_ctxt
*ctxt
)
518 return (1UL << (ctxt
->ad_bytes
<< 3)) - 1;
521 static ulong
stack_mask(struct x86_emulate_ctxt
*ctxt
)
524 struct desc_struct ss
;
526 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
528 ctxt
->ops
->get_segment(ctxt
, &sel
, &ss
, NULL
, VCPU_SREG_SS
);
529 return ~0U >> ((ss
.d
^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
532 static int stack_size(struct x86_emulate_ctxt
*ctxt
)
534 return (__fls(stack_mask(ctxt
)) + 1) >> 3;
537 /* Access/update address held in a register, based on addressing mode. */
538 static inline unsigned long
539 address_mask(struct x86_emulate_ctxt
*ctxt
, unsigned long reg
)
541 if (ctxt
->ad_bytes
== sizeof(unsigned long))
544 return reg
& ad_mask(ctxt
);
547 static inline unsigned long
548 register_address(struct x86_emulate_ctxt
*ctxt
, int reg
)
550 return address_mask(ctxt
, reg_read(ctxt
, reg
));
553 static void masked_increment(ulong
*reg
, ulong mask
, int inc
)
555 assign_masked(reg
, *reg
+ inc
, mask
);
559 register_address_increment(struct x86_emulate_ctxt
*ctxt
, int reg
, int inc
)
561 ulong
*preg
= reg_rmw(ctxt
, reg
);
563 assign_register(preg
, *preg
+ inc
, ctxt
->ad_bytes
);
566 static void rsp_increment(struct x86_emulate_ctxt
*ctxt
, int inc
)
568 masked_increment(reg_rmw(ctxt
, VCPU_REGS_RSP
), stack_mask(ctxt
), inc
);
571 static u32
desc_limit_scaled(struct desc_struct
*desc
)
573 u32 limit
= get_desc_limit(desc
);
575 return desc
->g
? (limit
<< 12) | 0xfff : limit
;
578 static unsigned long seg_base(struct x86_emulate_ctxt
*ctxt
, int seg
)
580 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&& seg
< VCPU_SREG_FS
)
583 return ctxt
->ops
->get_cached_segment_base(ctxt
, seg
);
586 static int emulate_exception(struct x86_emulate_ctxt
*ctxt
, int vec
,
587 u32 error
, bool valid
)
590 ctxt
->exception
.vector
= vec
;
591 ctxt
->exception
.error_code
= error
;
592 ctxt
->exception
.error_code_valid
= valid
;
593 return X86EMUL_PROPAGATE_FAULT
;
596 static int emulate_db(struct x86_emulate_ctxt
*ctxt
)
598 return emulate_exception(ctxt
, DB_VECTOR
, 0, false);
601 static int emulate_gp(struct x86_emulate_ctxt
*ctxt
, int err
)
603 return emulate_exception(ctxt
, GP_VECTOR
, err
, true);
606 static int emulate_ss(struct x86_emulate_ctxt
*ctxt
, int err
)
608 return emulate_exception(ctxt
, SS_VECTOR
, err
, true);
611 static int emulate_ud(struct x86_emulate_ctxt
*ctxt
)
613 return emulate_exception(ctxt
, UD_VECTOR
, 0, false);
616 static int emulate_ts(struct x86_emulate_ctxt
*ctxt
, int err
)
618 return emulate_exception(ctxt
, TS_VECTOR
, err
, true);
621 static int emulate_de(struct x86_emulate_ctxt
*ctxt
)
623 return emulate_exception(ctxt
, DE_VECTOR
, 0, false);
626 static int emulate_nm(struct x86_emulate_ctxt
*ctxt
)
628 return emulate_exception(ctxt
, NM_VECTOR
, 0, false);
631 static u16
get_segment_selector(struct x86_emulate_ctxt
*ctxt
, unsigned seg
)
634 struct desc_struct desc
;
636 ctxt
->ops
->get_segment(ctxt
, &selector
, &desc
, NULL
, seg
);
640 static void set_segment_selector(struct x86_emulate_ctxt
*ctxt
, u16 selector
,
645 struct desc_struct desc
;
647 ctxt
->ops
->get_segment(ctxt
, &dummy
, &desc
, &base3
, seg
);
648 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, seg
);
652 * x86 defines three classes of vector instructions: explicitly
653 * aligned, explicitly unaligned, and the rest, which change behaviour
654 * depending on whether they're AVX encoded or not.
656 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
657 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
658 * 512 bytes of data must be aligned to a 16 byte boundary.
660 static unsigned insn_alignment(struct x86_emulate_ctxt
*ctxt
, unsigned size
)
662 u64 alignment
= ctxt
->d
& AlignMask
;
664 if (likely(size
< 16))
679 static __always_inline
int __linearize(struct x86_emulate_ctxt
*ctxt
,
680 struct segmented_address addr
,
681 unsigned *max_size
, unsigned size
,
682 bool write
, bool fetch
,
683 enum x86emul_mode mode
, ulong
*linear
)
685 struct desc_struct desc
;
691 la
= seg_base(ctxt
, addr
.seg
) + addr
.ea
;
694 case X86EMUL_MODE_PROT64
:
696 if (is_noncanonical_address(la
))
699 *max_size
= min_t(u64
, ~0u, (1ull << 48) - la
);
700 if (size
> *max_size
)
704 *linear
= la
= (u32
)la
;
705 usable
= ctxt
->ops
->get_segment(ctxt
, &sel
, &desc
, NULL
,
709 /* code segment in protected mode or read-only data segment */
710 if ((((ctxt
->mode
!= X86EMUL_MODE_REAL
) && (desc
.type
& 8))
711 || !(desc
.type
& 2)) && write
)
713 /* unreadable code segment */
714 if (!fetch
&& (desc
.type
& 8) && !(desc
.type
& 2))
716 lim
= desc_limit_scaled(&desc
);
717 if (!(desc
.type
& 8) && (desc
.type
& 4)) {
718 /* expand-down segment */
721 lim
= desc
.d
? 0xffffffff : 0xffff;
725 if (lim
== 0xffffffff)
728 *max_size
= (u64
)lim
+ 1 - addr
.ea
;
729 if (size
> *max_size
)
734 if (la
& (insn_alignment(ctxt
, size
) - 1))
735 return emulate_gp(ctxt
, 0);
736 return X86EMUL_CONTINUE
;
738 if (addr
.seg
== VCPU_SREG_SS
)
739 return emulate_ss(ctxt
, 0);
741 return emulate_gp(ctxt
, 0);
744 static int linearize(struct x86_emulate_ctxt
*ctxt
,
745 struct segmented_address addr
,
746 unsigned size
, bool write
,
750 return __linearize(ctxt
, addr
, &max_size
, size
, write
, false,
754 static inline int assign_eip(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
755 enum x86emul_mode mode
)
760 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
763 if (ctxt
->op_bytes
!= sizeof(unsigned long))
764 addr
.ea
= dst
& ((1UL << (ctxt
->op_bytes
<< 3)) - 1);
765 rc
= __linearize(ctxt
, addr
, &max_size
, 1, false, true, mode
, &linear
);
766 if (rc
== X86EMUL_CONTINUE
)
767 ctxt
->_eip
= addr
.ea
;
771 static inline int assign_eip_near(struct x86_emulate_ctxt
*ctxt
, ulong dst
)
773 return assign_eip(ctxt
, dst
, ctxt
->mode
);
776 static int assign_eip_far(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
777 const struct desc_struct
*cs_desc
)
779 enum x86emul_mode mode
= ctxt
->mode
;
783 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
) {
787 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
789 mode
= X86EMUL_MODE_PROT64
;
791 mode
= X86EMUL_MODE_PROT32
; /* temporary value */
794 if (mode
== X86EMUL_MODE_PROT16
|| mode
== X86EMUL_MODE_PROT32
)
795 mode
= cs_desc
->d
? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
796 rc
= assign_eip(ctxt
, dst
, mode
);
797 if (rc
== X86EMUL_CONTINUE
)
802 static inline int jmp_rel(struct x86_emulate_ctxt
*ctxt
, int rel
)
804 return assign_eip_near(ctxt
, ctxt
->_eip
+ rel
);
807 static int segmented_read_std(struct x86_emulate_ctxt
*ctxt
,
808 struct segmented_address addr
,
815 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
816 if (rc
!= X86EMUL_CONTINUE
)
818 return ctxt
->ops
->read_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
822 * Prefetch the remaining bytes of the instruction without crossing page
823 * boundary if they are not in fetch_cache yet.
825 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
, int op_size
)
828 unsigned size
, max_size
;
829 unsigned long linear
;
830 int cur_size
= ctxt
->fetch
.end
- ctxt
->fetch
.data
;
831 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
832 .ea
= ctxt
->eip
+ cur_size
};
835 * We do not know exactly how many bytes will be needed, and
836 * __linearize is expensive, so fetch as much as possible. We
837 * just have to avoid going beyond the 15 byte limit, the end
838 * of the segment, or the end of the page.
840 * __linearize is called with size 0 so that it does not do any
841 * boundary check itself. Instead, we use max_size to check
844 rc
= __linearize(ctxt
, addr
, &max_size
, 0, false, true, ctxt
->mode
,
846 if (unlikely(rc
!= X86EMUL_CONTINUE
))
849 size
= min_t(unsigned, 15UL ^ cur_size
, max_size
);
850 size
= min_t(unsigned, size
, PAGE_SIZE
- offset_in_page(linear
));
853 * One instruction can only straddle two pages,
854 * and one has been loaded at the beginning of
855 * x86_decode_insn. So, if not enough bytes
856 * still, we must have hit the 15-byte boundary.
858 if (unlikely(size
< op_size
))
859 return emulate_gp(ctxt
, 0);
861 rc
= ctxt
->ops
->fetch(ctxt
, linear
, ctxt
->fetch
.end
,
862 size
, &ctxt
->exception
);
863 if (unlikely(rc
!= X86EMUL_CONTINUE
))
865 ctxt
->fetch
.end
+= size
;
866 return X86EMUL_CONTINUE
;
869 static __always_inline
int do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
,
872 unsigned done_size
= ctxt
->fetch
.end
- ctxt
->fetch
.ptr
;
874 if (unlikely(done_size
< size
))
875 return __do_insn_fetch_bytes(ctxt
, size
- done_size
);
877 return X86EMUL_CONTINUE
;
880 /* Fetch next part of the instruction being emulated. */
881 #define insn_fetch(_type, _ctxt) \
884 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
885 if (rc != X86EMUL_CONTINUE) \
887 ctxt->_eip += sizeof(_type); \
888 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
889 ctxt->fetch.ptr += sizeof(_type); \
893 #define insn_fetch_arr(_arr, _size, _ctxt) \
895 rc = do_insn_fetch_bytes(_ctxt, _size); \
896 if (rc != X86EMUL_CONTINUE) \
898 ctxt->_eip += (_size); \
899 memcpy(_arr, ctxt->fetch.ptr, _size); \
900 ctxt->fetch.ptr += (_size); \
904 * Given the 'reg' portion of a ModRM byte, and a register block, return a
905 * pointer into the block that addresses the relevant register.
906 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
908 static void *decode_register(struct x86_emulate_ctxt
*ctxt
, u8 modrm_reg
,
912 int highbyte_regs
= (ctxt
->rex_prefix
== 0) && byteop
;
914 if (highbyte_regs
&& modrm_reg
>= 4 && modrm_reg
< 8)
915 p
= (unsigned char *)reg_rmw(ctxt
, modrm_reg
& 3) + 1;
917 p
= reg_rmw(ctxt
, modrm_reg
);
921 static int read_descriptor(struct x86_emulate_ctxt
*ctxt
,
922 struct segmented_address addr
,
923 u16
*size
, unsigned long *address
, int op_bytes
)
930 rc
= segmented_read_std(ctxt
, addr
, size
, 2);
931 if (rc
!= X86EMUL_CONTINUE
)
934 rc
= segmented_read_std(ctxt
, addr
, address
, op_bytes
);
948 FASTOP1SRC2(mul
, mul_ex
);
949 FASTOP1SRC2(imul
, imul_ex
);
950 FASTOP1SRC2EX(div
, div_ex
);
951 FASTOP1SRC2EX(idiv
, idiv_ex
);
980 FASTOP2R(cmp
, cmp_r
);
982 static int em_bsf_c(struct x86_emulate_ctxt
*ctxt
)
984 /* If src is zero, do not writeback, but update flags */
985 if (ctxt
->src
.val
== 0)
986 ctxt
->dst
.type
= OP_NONE
;
987 return fastop(ctxt
, em_bsf
);
990 static int em_bsr_c(struct x86_emulate_ctxt
*ctxt
)
992 /* If src is zero, do not writeback, but update flags */
993 if (ctxt
->src
.val
== 0)
994 ctxt
->dst
.type
= OP_NONE
;
995 return fastop(ctxt
, em_bsr
);
998 static __always_inline u8
test_cc(unsigned int condition
, unsigned long flags
)
1001 void (*fop
)(void) = (void *)em_setcc
+ 4 * (condition
& 0xf);
1003 flags
= (flags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
1004 asm("push %[flags]; popf; call *%[fastop]"
1005 : "=a"(rc
) : [fastop
]"r"(fop
), [flags
]"r"(flags
));
1009 static void fetch_register_operand(struct operand
*op
)
1011 switch (op
->bytes
) {
1013 op
->val
= *(u8
*)op
->addr
.reg
;
1016 op
->val
= *(u16
*)op
->addr
.reg
;
1019 op
->val
= *(u32
*)op
->addr
.reg
;
1022 op
->val
= *(u64
*)op
->addr
.reg
;
1027 static void read_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
, int reg
)
1029 ctxt
->ops
->get_fpu(ctxt
);
1031 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data
)); break;
1032 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data
)); break;
1033 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data
)); break;
1034 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data
)); break;
1035 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data
)); break;
1036 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data
)); break;
1037 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data
)); break;
1038 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data
)); break;
1039 #ifdef CONFIG_X86_64
1040 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data
)); break;
1041 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data
)); break;
1042 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data
)); break;
1043 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data
)); break;
1044 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data
)); break;
1045 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data
)); break;
1046 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data
)); break;
1047 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data
)); break;
1051 ctxt
->ops
->put_fpu(ctxt
);
1054 static void write_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
,
1057 ctxt
->ops
->get_fpu(ctxt
);
1059 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data
)); break;
1060 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data
)); break;
1061 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data
)); break;
1062 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data
)); break;
1063 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data
)); break;
1064 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data
)); break;
1065 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data
)); break;
1066 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data
)); break;
1067 #ifdef CONFIG_X86_64
1068 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data
)); break;
1069 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data
)); break;
1070 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data
)); break;
1071 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data
)); break;
1072 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data
)); break;
1073 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data
)); break;
1074 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data
)); break;
1075 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data
)); break;
1079 ctxt
->ops
->put_fpu(ctxt
);
1082 static void read_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1084 ctxt
->ops
->get_fpu(ctxt
);
1086 case 0: asm("movq %%mm0, %0" : "=m"(*data
)); break;
1087 case 1: asm("movq %%mm1, %0" : "=m"(*data
)); break;
1088 case 2: asm("movq %%mm2, %0" : "=m"(*data
)); break;
1089 case 3: asm("movq %%mm3, %0" : "=m"(*data
)); break;
1090 case 4: asm("movq %%mm4, %0" : "=m"(*data
)); break;
1091 case 5: asm("movq %%mm5, %0" : "=m"(*data
)); break;
1092 case 6: asm("movq %%mm6, %0" : "=m"(*data
)); break;
1093 case 7: asm("movq %%mm7, %0" : "=m"(*data
)); break;
1096 ctxt
->ops
->put_fpu(ctxt
);
1099 static void write_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1101 ctxt
->ops
->get_fpu(ctxt
);
1103 case 0: asm("movq %0, %%mm0" : : "m"(*data
)); break;
1104 case 1: asm("movq %0, %%mm1" : : "m"(*data
)); break;
1105 case 2: asm("movq %0, %%mm2" : : "m"(*data
)); break;
1106 case 3: asm("movq %0, %%mm3" : : "m"(*data
)); break;
1107 case 4: asm("movq %0, %%mm4" : : "m"(*data
)); break;
1108 case 5: asm("movq %0, %%mm5" : : "m"(*data
)); break;
1109 case 6: asm("movq %0, %%mm6" : : "m"(*data
)); break;
1110 case 7: asm("movq %0, %%mm7" : : "m"(*data
)); break;
1113 ctxt
->ops
->put_fpu(ctxt
);
1116 static int em_fninit(struct x86_emulate_ctxt
*ctxt
)
1118 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1119 return emulate_nm(ctxt
);
1121 ctxt
->ops
->get_fpu(ctxt
);
1122 asm volatile("fninit");
1123 ctxt
->ops
->put_fpu(ctxt
);
1124 return X86EMUL_CONTINUE
;
1127 static int em_fnstcw(struct x86_emulate_ctxt
*ctxt
)
1131 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1132 return emulate_nm(ctxt
);
1134 ctxt
->ops
->get_fpu(ctxt
);
1135 asm volatile("fnstcw %0": "+m"(fcw
));
1136 ctxt
->ops
->put_fpu(ctxt
);
1138 ctxt
->dst
.val
= fcw
;
1140 return X86EMUL_CONTINUE
;
1143 static int em_fnstsw(struct x86_emulate_ctxt
*ctxt
)
1147 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1148 return emulate_nm(ctxt
);
1150 ctxt
->ops
->get_fpu(ctxt
);
1151 asm volatile("fnstsw %0": "+m"(fsw
));
1152 ctxt
->ops
->put_fpu(ctxt
);
1154 ctxt
->dst
.val
= fsw
;
1156 return X86EMUL_CONTINUE
;
1159 static void decode_register_operand(struct x86_emulate_ctxt
*ctxt
,
1162 unsigned reg
= ctxt
->modrm_reg
;
1164 if (!(ctxt
->d
& ModRM
))
1165 reg
= (ctxt
->b
& 7) | ((ctxt
->rex_prefix
& 1) << 3);
1167 if (ctxt
->d
& Sse
) {
1171 read_sse_reg(ctxt
, &op
->vec_val
, reg
);
1174 if (ctxt
->d
& Mmx
) {
1183 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1184 op
->addr
.reg
= decode_register(ctxt
, reg
, ctxt
->d
& ByteOp
);
1186 fetch_register_operand(op
);
1187 op
->orig_val
= op
->val
;
1190 static void adjust_modrm_seg(struct x86_emulate_ctxt
*ctxt
, int base_reg
)
1192 if (base_reg
== VCPU_REGS_RSP
|| base_reg
== VCPU_REGS_RBP
)
1193 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1196 static int decode_modrm(struct x86_emulate_ctxt
*ctxt
,
1200 int index_reg
, base_reg
, scale
;
1201 int rc
= X86EMUL_CONTINUE
;
1204 ctxt
->modrm_reg
= ((ctxt
->rex_prefix
<< 1) & 8); /* REX.R */
1205 index_reg
= (ctxt
->rex_prefix
<< 2) & 8; /* REX.X */
1206 base_reg
= (ctxt
->rex_prefix
<< 3) & 8; /* REX.B */
1208 ctxt
->modrm_mod
= (ctxt
->modrm
& 0xc0) >> 6;
1209 ctxt
->modrm_reg
|= (ctxt
->modrm
& 0x38) >> 3;
1210 ctxt
->modrm_rm
= base_reg
| (ctxt
->modrm
& 0x07);
1211 ctxt
->modrm_seg
= VCPU_SREG_DS
;
1213 if (ctxt
->modrm_mod
== 3 || (ctxt
->d
& NoMod
)) {
1215 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1216 op
->addr
.reg
= decode_register(ctxt
, ctxt
->modrm_rm
,
1218 if (ctxt
->d
& Sse
) {
1221 op
->addr
.xmm
= ctxt
->modrm_rm
;
1222 read_sse_reg(ctxt
, &op
->vec_val
, ctxt
->modrm_rm
);
1225 if (ctxt
->d
& Mmx
) {
1228 op
->addr
.mm
= ctxt
->modrm_rm
& 7;
1231 fetch_register_operand(op
);
1237 if (ctxt
->ad_bytes
== 2) {
1238 unsigned bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
1239 unsigned bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1240 unsigned si
= reg_read(ctxt
, VCPU_REGS_RSI
);
1241 unsigned di
= reg_read(ctxt
, VCPU_REGS_RDI
);
1243 /* 16-bit ModR/M decode. */
1244 switch (ctxt
->modrm_mod
) {
1246 if (ctxt
->modrm_rm
== 6)
1247 modrm_ea
+= insn_fetch(u16
, ctxt
);
1250 modrm_ea
+= insn_fetch(s8
, ctxt
);
1253 modrm_ea
+= insn_fetch(u16
, ctxt
);
1256 switch (ctxt
->modrm_rm
) {
1258 modrm_ea
+= bx
+ si
;
1261 modrm_ea
+= bx
+ di
;
1264 modrm_ea
+= bp
+ si
;
1267 modrm_ea
+= bp
+ di
;
1276 if (ctxt
->modrm_mod
!= 0)
1283 if (ctxt
->modrm_rm
== 2 || ctxt
->modrm_rm
== 3 ||
1284 (ctxt
->modrm_rm
== 6 && ctxt
->modrm_mod
!= 0))
1285 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1286 modrm_ea
= (u16
)modrm_ea
;
1288 /* 32/64-bit ModR/M decode. */
1289 if ((ctxt
->modrm_rm
& 7) == 4) {
1290 sib
= insn_fetch(u8
, ctxt
);
1291 index_reg
|= (sib
>> 3) & 7;
1292 base_reg
|= sib
& 7;
1295 if ((base_reg
& 7) == 5 && ctxt
->modrm_mod
== 0)
1296 modrm_ea
+= insn_fetch(s32
, ctxt
);
1298 modrm_ea
+= reg_read(ctxt
, base_reg
);
1299 adjust_modrm_seg(ctxt
, base_reg
);
1300 /* Increment ESP on POP [ESP] */
1301 if ((ctxt
->d
& IncSP
) &&
1302 base_reg
== VCPU_REGS_RSP
)
1303 modrm_ea
+= ctxt
->op_bytes
;
1306 modrm_ea
+= reg_read(ctxt
, index_reg
) << scale
;
1307 } else if ((ctxt
->modrm_rm
& 7) == 5 && ctxt
->modrm_mod
== 0) {
1308 modrm_ea
+= insn_fetch(s32
, ctxt
);
1309 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
1310 ctxt
->rip_relative
= 1;
1312 base_reg
= ctxt
->modrm_rm
;
1313 modrm_ea
+= reg_read(ctxt
, base_reg
);
1314 adjust_modrm_seg(ctxt
, base_reg
);
1316 switch (ctxt
->modrm_mod
) {
1318 modrm_ea
+= insn_fetch(s8
, ctxt
);
1321 modrm_ea
+= insn_fetch(s32
, ctxt
);
1325 op
->addr
.mem
.ea
= modrm_ea
;
1326 if (ctxt
->ad_bytes
!= 8)
1327 ctxt
->memop
.addr
.mem
.ea
= (u32
)ctxt
->memop
.addr
.mem
.ea
;
1333 static int decode_abs(struct x86_emulate_ctxt
*ctxt
,
1336 int rc
= X86EMUL_CONTINUE
;
1339 switch (ctxt
->ad_bytes
) {
1341 op
->addr
.mem
.ea
= insn_fetch(u16
, ctxt
);
1344 op
->addr
.mem
.ea
= insn_fetch(u32
, ctxt
);
1347 op
->addr
.mem
.ea
= insn_fetch(u64
, ctxt
);
1354 static void fetch_bit_operand(struct x86_emulate_ctxt
*ctxt
)
1358 if (ctxt
->dst
.type
== OP_MEM
&& ctxt
->src
.type
== OP_REG
) {
1359 mask
= ~((long)ctxt
->dst
.bytes
* 8 - 1);
1361 if (ctxt
->src
.bytes
== 2)
1362 sv
= (s16
)ctxt
->src
.val
& (s16
)mask
;
1363 else if (ctxt
->src
.bytes
== 4)
1364 sv
= (s32
)ctxt
->src
.val
& (s32
)mask
;
1366 sv
= (s64
)ctxt
->src
.val
& (s64
)mask
;
1368 ctxt
->dst
.addr
.mem
.ea
= address_mask(ctxt
,
1369 ctxt
->dst
.addr
.mem
.ea
+ (sv
>> 3));
1372 /* only subword offset */
1373 ctxt
->src
.val
&= (ctxt
->dst
.bytes
<< 3) - 1;
1376 static int read_emulated(struct x86_emulate_ctxt
*ctxt
,
1377 unsigned long addr
, void *dest
, unsigned size
)
1380 struct read_cache
*mc
= &ctxt
->mem_read
;
1382 if (mc
->pos
< mc
->end
)
1385 WARN_ON((mc
->end
+ size
) >= sizeof(mc
->data
));
1387 rc
= ctxt
->ops
->read_emulated(ctxt
, addr
, mc
->data
+ mc
->end
, size
,
1389 if (rc
!= X86EMUL_CONTINUE
)
1395 memcpy(dest
, mc
->data
+ mc
->pos
, size
);
1397 return X86EMUL_CONTINUE
;
1400 static int segmented_read(struct x86_emulate_ctxt
*ctxt
,
1401 struct segmented_address addr
,
1408 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
1409 if (rc
!= X86EMUL_CONTINUE
)
1411 return read_emulated(ctxt
, linear
, data
, size
);
1414 static int segmented_write(struct x86_emulate_ctxt
*ctxt
,
1415 struct segmented_address addr
,
1422 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1423 if (rc
!= X86EMUL_CONTINUE
)
1425 return ctxt
->ops
->write_emulated(ctxt
, linear
, data
, size
,
1429 static int segmented_cmpxchg(struct x86_emulate_ctxt
*ctxt
,
1430 struct segmented_address addr
,
1431 const void *orig_data
, const void *data
,
1437 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1438 if (rc
!= X86EMUL_CONTINUE
)
1440 return ctxt
->ops
->cmpxchg_emulated(ctxt
, linear
, orig_data
, data
,
1441 size
, &ctxt
->exception
);
1444 static int pio_in_emulated(struct x86_emulate_ctxt
*ctxt
,
1445 unsigned int size
, unsigned short port
,
1448 struct read_cache
*rc
= &ctxt
->io_read
;
1450 if (rc
->pos
== rc
->end
) { /* refill pio read ahead */
1451 unsigned int in_page
, n
;
1452 unsigned int count
= ctxt
->rep_prefix
?
1453 address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) : 1;
1454 in_page
= (ctxt
->eflags
& X86_EFLAGS_DF
) ?
1455 offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
)) :
1456 PAGE_SIZE
- offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
));
1457 n
= min3(in_page
, (unsigned int)sizeof(rc
->data
) / size
, count
);
1460 rc
->pos
= rc
->end
= 0;
1461 if (!ctxt
->ops
->pio_in_emulated(ctxt
, size
, port
, rc
->data
, n
))
1466 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
) &&
1467 !(ctxt
->eflags
& X86_EFLAGS_DF
)) {
1468 ctxt
->dst
.data
= rc
->data
+ rc
->pos
;
1469 ctxt
->dst
.type
= OP_MEM_STR
;
1470 ctxt
->dst
.count
= (rc
->end
- rc
->pos
) / size
;
1473 memcpy(dest
, rc
->data
+ rc
->pos
, size
);
1479 static int read_interrupt_descriptor(struct x86_emulate_ctxt
*ctxt
,
1480 u16 index
, struct desc_struct
*desc
)
1485 ctxt
->ops
->get_idt(ctxt
, &dt
);
1487 if (dt
.size
< index
* 8 + 7)
1488 return emulate_gp(ctxt
, index
<< 3 | 0x2);
1490 addr
= dt
.address
+ index
* 8;
1491 return ctxt
->ops
->read_std(ctxt
, addr
, desc
, sizeof *desc
,
1495 static void get_descriptor_table_ptr(struct x86_emulate_ctxt
*ctxt
,
1496 u16 selector
, struct desc_ptr
*dt
)
1498 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1501 if (selector
& 1 << 2) {
1502 struct desc_struct desc
;
1505 memset (dt
, 0, sizeof *dt
);
1506 if (!ops
->get_segment(ctxt
, &sel
, &desc
, &base3
,
1510 dt
->size
= desc_limit_scaled(&desc
); /* what if limit > 65535? */
1511 dt
->address
= get_desc_base(&desc
) | ((u64
)base3
<< 32);
1513 ops
->get_gdt(ctxt
, dt
);
1516 static int get_descriptor_ptr(struct x86_emulate_ctxt
*ctxt
,
1517 u16 selector
, ulong
*desc_addr_p
)
1520 u16 index
= selector
>> 3;
1523 get_descriptor_table_ptr(ctxt
, selector
, &dt
);
1525 if (dt
.size
< index
* 8 + 7)
1526 return emulate_gp(ctxt
, selector
& 0xfffc);
1528 addr
= dt
.address
+ index
* 8;
1530 #ifdef CONFIG_X86_64
1531 if (addr
>> 32 != 0) {
1534 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1535 if (!(efer
& EFER_LMA
))
1540 *desc_addr_p
= addr
;
1541 return X86EMUL_CONTINUE
;
1544 /* allowed just for 8 bytes segments */
1545 static int read_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1546 u16 selector
, struct desc_struct
*desc
,
1551 rc
= get_descriptor_ptr(ctxt
, selector
, desc_addr_p
);
1552 if (rc
!= X86EMUL_CONTINUE
)
1555 return ctxt
->ops
->read_std(ctxt
, *desc_addr_p
, desc
, sizeof(*desc
),
1559 /* allowed just for 8 bytes segments */
1560 static int write_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1561 u16 selector
, struct desc_struct
*desc
)
1566 rc
= get_descriptor_ptr(ctxt
, selector
, &addr
);
1567 if (rc
!= X86EMUL_CONTINUE
)
1570 return ctxt
->ops
->write_std(ctxt
, addr
, desc
, sizeof *desc
,
1574 /* Does not support long mode */
1575 static int __load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1576 u16 selector
, int seg
, u8 cpl
,
1577 enum x86_transfer_type transfer
,
1578 struct desc_struct
*desc
)
1580 struct desc_struct seg_desc
, old_desc
;
1582 unsigned err_vec
= GP_VECTOR
;
1584 bool null_selector
= !(selector
& ~0x3); /* 0000-0003 are null */
1590 memset(&seg_desc
, 0, sizeof seg_desc
);
1592 if (ctxt
->mode
== X86EMUL_MODE_REAL
) {
1593 /* set real mode segment descriptor (keep limit etc. for
1595 ctxt
->ops
->get_segment(ctxt
, &dummy
, &seg_desc
, NULL
, seg
);
1596 set_desc_base(&seg_desc
, selector
<< 4);
1598 } else if (seg
<= VCPU_SREG_GS
&& ctxt
->mode
== X86EMUL_MODE_VM86
) {
1599 /* VM86 needs a clean new segment descriptor */
1600 set_desc_base(&seg_desc
, selector
<< 4);
1601 set_desc_limit(&seg_desc
, 0xffff);
1611 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1612 if ((seg
== VCPU_SREG_CS
1613 || (seg
== VCPU_SREG_SS
1614 && (ctxt
->mode
!= X86EMUL_MODE_PROT64
|| rpl
!= cpl
))
1615 || seg
== VCPU_SREG_TR
)
1619 /* TR should be in GDT only */
1620 if (seg
== VCPU_SREG_TR
&& (selector
& (1 << 2)))
1623 if (null_selector
) /* for NULL selector skip all following checks */
1626 ret
= read_segment_descriptor(ctxt
, selector
, &seg_desc
, &desc_addr
);
1627 if (ret
!= X86EMUL_CONTINUE
)
1630 err_code
= selector
& 0xfffc;
1631 err_vec
= (transfer
== X86_TRANSFER_TASK_SWITCH
) ? TS_VECTOR
:
1634 /* can't load system descriptor into segment selector */
1635 if (seg
<= VCPU_SREG_GS
&& !seg_desc
.s
) {
1636 if (transfer
== X86_TRANSFER_CALL_JMP
)
1637 return X86EMUL_UNHANDLEABLE
;
1642 err_vec
= (seg
== VCPU_SREG_SS
) ? SS_VECTOR
: NP_VECTOR
;
1651 * segment is not a writable data segment or segment
1652 * selector's RPL != CPL or segment selector's RPL != CPL
1654 if (rpl
!= cpl
|| (seg_desc
.type
& 0xa) != 0x2 || dpl
!= cpl
)
1658 if (!(seg_desc
.type
& 8))
1661 if (seg_desc
.type
& 4) {
1667 if (rpl
> cpl
|| dpl
!= cpl
)
1670 /* in long-mode d/b must be clear if l is set */
1671 if (seg_desc
.d
&& seg_desc
.l
) {
1674 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1675 if (efer
& EFER_LMA
)
1679 /* CS(RPL) <- CPL */
1680 selector
= (selector
& 0xfffc) | cpl
;
1683 if (seg_desc
.s
|| (seg_desc
.type
!= 1 && seg_desc
.type
!= 9))
1685 old_desc
= seg_desc
;
1686 seg_desc
.type
|= 2; /* busy */
1687 ret
= ctxt
->ops
->cmpxchg_emulated(ctxt
, desc_addr
, &old_desc
, &seg_desc
,
1688 sizeof(seg_desc
), &ctxt
->exception
);
1689 if (ret
!= X86EMUL_CONTINUE
)
1692 case VCPU_SREG_LDTR
:
1693 if (seg_desc
.s
|| seg_desc
.type
!= 2)
1696 default: /* DS, ES, FS, or GS */
1698 * segment is not a data or readable code segment or
1699 * ((segment is a data or nonconforming code segment)
1700 * and (both RPL and CPL > DPL))
1702 if ((seg_desc
.type
& 0xa) == 0x8 ||
1703 (((seg_desc
.type
& 0xc) != 0xc) &&
1704 (rpl
> dpl
&& cpl
> dpl
)))
1710 /* mark segment as accessed */
1711 if (!(seg_desc
.type
& 1)) {
1713 ret
= write_segment_descriptor(ctxt
, selector
,
1715 if (ret
!= X86EMUL_CONTINUE
)
1718 } else if (ctxt
->mode
== X86EMUL_MODE_PROT64
) {
1719 ret
= ctxt
->ops
->read_std(ctxt
, desc_addr
+8, &base3
,
1720 sizeof(base3
), &ctxt
->exception
);
1721 if (ret
!= X86EMUL_CONTINUE
)
1723 if (is_noncanonical_address(get_desc_base(&seg_desc
) |
1724 ((u64
)base3
<< 32)))
1725 return emulate_gp(ctxt
, 0);
1728 ctxt
->ops
->set_segment(ctxt
, selector
, &seg_desc
, base3
, seg
);
1731 return X86EMUL_CONTINUE
;
1733 return emulate_exception(ctxt
, err_vec
, err_code
, true);
1736 static int load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1737 u16 selector
, int seg
)
1739 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
1740 return __load_segment_descriptor(ctxt
, selector
, seg
, cpl
,
1741 X86_TRANSFER_NONE
, NULL
);
1744 static void write_register_operand(struct operand
*op
)
1746 return assign_register(op
->addr
.reg
, op
->val
, op
->bytes
);
1749 static int writeback(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
)
1753 write_register_operand(op
);
1756 if (ctxt
->lock_prefix
)
1757 return segmented_cmpxchg(ctxt
,
1763 return segmented_write(ctxt
,
1769 return segmented_write(ctxt
,
1772 op
->bytes
* op
->count
);
1775 write_sse_reg(ctxt
, &op
->vec_val
, op
->addr
.xmm
);
1778 write_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
1786 return X86EMUL_CONTINUE
;
1789 static int push(struct x86_emulate_ctxt
*ctxt
, void *data
, int bytes
)
1791 struct segmented_address addr
;
1793 rsp_increment(ctxt
, -bytes
);
1794 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1795 addr
.seg
= VCPU_SREG_SS
;
1797 return segmented_write(ctxt
, addr
, data
, bytes
);
1800 static int em_push(struct x86_emulate_ctxt
*ctxt
)
1802 /* Disable writeback. */
1803 ctxt
->dst
.type
= OP_NONE
;
1804 return push(ctxt
, &ctxt
->src
.val
, ctxt
->op_bytes
);
1807 static int emulate_pop(struct x86_emulate_ctxt
*ctxt
,
1808 void *dest
, int len
)
1811 struct segmented_address addr
;
1813 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1814 addr
.seg
= VCPU_SREG_SS
;
1815 rc
= segmented_read(ctxt
, addr
, dest
, len
);
1816 if (rc
!= X86EMUL_CONTINUE
)
1819 rsp_increment(ctxt
, len
);
1823 static int em_pop(struct x86_emulate_ctxt
*ctxt
)
1825 return emulate_pop(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1828 static int emulate_popf(struct x86_emulate_ctxt
*ctxt
,
1829 void *dest
, int len
)
1832 unsigned long val
, change_mask
;
1833 int iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
1834 int cpl
= ctxt
->ops
->cpl(ctxt
);
1836 rc
= emulate_pop(ctxt
, &val
, len
);
1837 if (rc
!= X86EMUL_CONTINUE
)
1840 change_mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
1841 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_OF
|
1842 X86_EFLAGS_TF
| X86_EFLAGS_DF
| X86_EFLAGS_NT
|
1843 X86_EFLAGS_AC
| X86_EFLAGS_ID
;
1845 switch(ctxt
->mode
) {
1846 case X86EMUL_MODE_PROT64
:
1847 case X86EMUL_MODE_PROT32
:
1848 case X86EMUL_MODE_PROT16
:
1850 change_mask
|= X86_EFLAGS_IOPL
;
1852 change_mask
|= X86_EFLAGS_IF
;
1854 case X86EMUL_MODE_VM86
:
1856 return emulate_gp(ctxt
, 0);
1857 change_mask
|= X86_EFLAGS_IF
;
1859 default: /* real mode */
1860 change_mask
|= (X86_EFLAGS_IOPL
| X86_EFLAGS_IF
);
1864 *(unsigned long *)dest
=
1865 (ctxt
->eflags
& ~change_mask
) | (val
& change_mask
);
1870 static int em_popf(struct x86_emulate_ctxt
*ctxt
)
1872 ctxt
->dst
.type
= OP_REG
;
1873 ctxt
->dst
.addr
.reg
= &ctxt
->eflags
;
1874 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
1875 return emulate_popf(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1878 static int em_enter(struct x86_emulate_ctxt
*ctxt
)
1881 unsigned frame_size
= ctxt
->src
.val
;
1882 unsigned nesting_level
= ctxt
->src2
.val
& 31;
1886 return X86EMUL_UNHANDLEABLE
;
1888 rbp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1889 rc
= push(ctxt
, &rbp
, stack_size(ctxt
));
1890 if (rc
!= X86EMUL_CONTINUE
)
1892 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RBP
), reg_read(ctxt
, VCPU_REGS_RSP
),
1894 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
),
1895 reg_read(ctxt
, VCPU_REGS_RSP
) - frame_size
,
1897 return X86EMUL_CONTINUE
;
1900 static int em_leave(struct x86_emulate_ctxt
*ctxt
)
1902 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
), reg_read(ctxt
, VCPU_REGS_RBP
),
1904 return emulate_pop(ctxt
, reg_rmw(ctxt
, VCPU_REGS_RBP
), ctxt
->op_bytes
);
1907 static int em_push_sreg(struct x86_emulate_ctxt
*ctxt
)
1909 int seg
= ctxt
->src2
.val
;
1911 ctxt
->src
.val
= get_segment_selector(ctxt
, seg
);
1912 if (ctxt
->op_bytes
== 4) {
1913 rsp_increment(ctxt
, -2);
1917 return em_push(ctxt
);
1920 static int em_pop_sreg(struct x86_emulate_ctxt
*ctxt
)
1922 int seg
= ctxt
->src2
.val
;
1923 unsigned long selector
;
1926 rc
= emulate_pop(ctxt
, &selector
, 2);
1927 if (rc
!= X86EMUL_CONTINUE
)
1930 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
1931 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
1932 if (ctxt
->op_bytes
> 2)
1933 rsp_increment(ctxt
, ctxt
->op_bytes
- 2);
1935 rc
= load_segment_descriptor(ctxt
, (u16
)selector
, seg
);
1939 static int em_pusha(struct x86_emulate_ctxt
*ctxt
)
1941 unsigned long old_esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
1942 int rc
= X86EMUL_CONTINUE
;
1943 int reg
= VCPU_REGS_RAX
;
1945 while (reg
<= VCPU_REGS_RDI
) {
1946 (reg
== VCPU_REGS_RSP
) ?
1947 (ctxt
->src
.val
= old_esp
) : (ctxt
->src
.val
= reg_read(ctxt
, reg
));
1950 if (rc
!= X86EMUL_CONTINUE
)
1959 static int em_pushf(struct x86_emulate_ctxt
*ctxt
)
1961 ctxt
->src
.val
= (unsigned long)ctxt
->eflags
& ~X86_EFLAGS_VM
;
1962 return em_push(ctxt
);
1965 static int em_popa(struct x86_emulate_ctxt
*ctxt
)
1967 int rc
= X86EMUL_CONTINUE
;
1968 int reg
= VCPU_REGS_RDI
;
1971 while (reg
>= VCPU_REGS_RAX
) {
1972 if (reg
== VCPU_REGS_RSP
) {
1973 rsp_increment(ctxt
, ctxt
->op_bytes
);
1977 rc
= emulate_pop(ctxt
, &val
, ctxt
->op_bytes
);
1978 if (rc
!= X86EMUL_CONTINUE
)
1980 assign_register(reg_rmw(ctxt
, reg
), val
, ctxt
->op_bytes
);
1986 static int __emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
1988 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1995 /* TODO: Add limit checks */
1996 ctxt
->src
.val
= ctxt
->eflags
;
1998 if (rc
!= X86EMUL_CONTINUE
)
2001 ctxt
->eflags
&= ~(X86_EFLAGS_IF
| X86_EFLAGS_TF
| X86_EFLAGS_AC
);
2003 ctxt
->src
.val
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2005 if (rc
!= X86EMUL_CONTINUE
)
2008 ctxt
->src
.val
= ctxt
->_eip
;
2010 if (rc
!= X86EMUL_CONTINUE
)
2013 ops
->get_idt(ctxt
, &dt
);
2015 eip_addr
= dt
.address
+ (irq
<< 2);
2016 cs_addr
= dt
.address
+ (irq
<< 2) + 2;
2018 rc
= ops
->read_std(ctxt
, cs_addr
, &cs
, 2, &ctxt
->exception
);
2019 if (rc
!= X86EMUL_CONTINUE
)
2022 rc
= ops
->read_std(ctxt
, eip_addr
, &eip
, 2, &ctxt
->exception
);
2023 if (rc
!= X86EMUL_CONTINUE
)
2026 rc
= load_segment_descriptor(ctxt
, cs
, VCPU_SREG_CS
);
2027 if (rc
!= X86EMUL_CONTINUE
)
2035 int emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2039 invalidate_registers(ctxt
);
2040 rc
= __emulate_int_real(ctxt
, irq
);
2041 if (rc
== X86EMUL_CONTINUE
)
2042 writeback_registers(ctxt
);
2046 static int emulate_int(struct x86_emulate_ctxt
*ctxt
, int irq
)
2048 switch(ctxt
->mode
) {
2049 case X86EMUL_MODE_REAL
:
2050 return __emulate_int_real(ctxt
, irq
);
2051 case X86EMUL_MODE_VM86
:
2052 case X86EMUL_MODE_PROT16
:
2053 case X86EMUL_MODE_PROT32
:
2054 case X86EMUL_MODE_PROT64
:
2056 /* Protected mode interrupts unimplemented yet */
2057 return X86EMUL_UNHANDLEABLE
;
2061 static int emulate_iret_real(struct x86_emulate_ctxt
*ctxt
)
2063 int rc
= X86EMUL_CONTINUE
;
2064 unsigned long temp_eip
= 0;
2065 unsigned long temp_eflags
= 0;
2066 unsigned long cs
= 0;
2067 unsigned long mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
2068 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_TF
|
2069 X86_EFLAGS_IF
| X86_EFLAGS_DF
| X86_EFLAGS_OF
|
2070 X86_EFLAGS_IOPL
| X86_EFLAGS_NT
| X86_EFLAGS_RF
|
2071 X86_EFLAGS_AC
| X86_EFLAGS_ID
|
2073 unsigned long vm86_mask
= X86_EFLAGS_VM
| X86_EFLAGS_VIF
|
2076 /* TODO: Add stack limit check */
2078 rc
= emulate_pop(ctxt
, &temp_eip
, ctxt
->op_bytes
);
2080 if (rc
!= X86EMUL_CONTINUE
)
2083 if (temp_eip
& ~0xffff)
2084 return emulate_gp(ctxt
, 0);
2086 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2088 if (rc
!= X86EMUL_CONTINUE
)
2091 rc
= emulate_pop(ctxt
, &temp_eflags
, ctxt
->op_bytes
);
2093 if (rc
!= X86EMUL_CONTINUE
)
2096 rc
= load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
);
2098 if (rc
!= X86EMUL_CONTINUE
)
2101 ctxt
->_eip
= temp_eip
;
2103 if (ctxt
->op_bytes
== 4)
2104 ctxt
->eflags
= ((temp_eflags
& mask
) | (ctxt
->eflags
& vm86_mask
));
2105 else if (ctxt
->op_bytes
== 2) {
2106 ctxt
->eflags
&= ~0xffff;
2107 ctxt
->eflags
|= temp_eflags
;
2110 ctxt
->eflags
&= ~EFLG_RESERVED_ZEROS_MASK
; /* Clear reserved zeros */
2111 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2112 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2117 static int em_iret(struct x86_emulate_ctxt
*ctxt
)
2119 switch(ctxt
->mode
) {
2120 case X86EMUL_MODE_REAL
:
2121 return emulate_iret_real(ctxt
);
2122 case X86EMUL_MODE_VM86
:
2123 case X86EMUL_MODE_PROT16
:
2124 case X86EMUL_MODE_PROT32
:
2125 case X86EMUL_MODE_PROT64
:
2127 /* iret from protected mode unimplemented yet */
2128 return X86EMUL_UNHANDLEABLE
;
2132 static int em_jmp_far(struct x86_emulate_ctxt
*ctxt
)
2135 unsigned short sel
, old_sel
;
2136 struct desc_struct old_desc
, new_desc
;
2137 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2138 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
2140 /* Assignment of RIP may only fail in 64-bit mode */
2141 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2142 ops
->get_segment(ctxt
, &old_sel
, &old_desc
, NULL
,
2145 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2147 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
2148 X86_TRANSFER_CALL_JMP
,
2150 if (rc
!= X86EMUL_CONTINUE
)
2153 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
2154 if (rc
!= X86EMUL_CONTINUE
) {
2155 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2156 /* assigning eip failed; restore the old cs */
2157 ops
->set_segment(ctxt
, old_sel
, &old_desc
, 0, VCPU_SREG_CS
);
2163 static int em_jmp_abs(struct x86_emulate_ctxt
*ctxt
)
2165 return assign_eip_near(ctxt
, ctxt
->src
.val
);
2168 static int em_call_near_abs(struct x86_emulate_ctxt
*ctxt
)
2173 old_eip
= ctxt
->_eip
;
2174 rc
= assign_eip_near(ctxt
, ctxt
->src
.val
);
2175 if (rc
!= X86EMUL_CONTINUE
)
2177 ctxt
->src
.val
= old_eip
;
2182 static int em_cmpxchg8b(struct x86_emulate_ctxt
*ctxt
)
2184 u64 old
= ctxt
->dst
.orig_val64
;
2186 if (ctxt
->dst
.bytes
== 16)
2187 return X86EMUL_UNHANDLEABLE
;
2189 if (((u32
) (old
>> 0) != (u32
) reg_read(ctxt
, VCPU_REGS_RAX
)) ||
2190 ((u32
) (old
>> 32) != (u32
) reg_read(ctxt
, VCPU_REGS_RDX
))) {
2191 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
) (old
>> 0);
2192 *reg_write(ctxt
, VCPU_REGS_RDX
) = (u32
) (old
>> 32);
2193 ctxt
->eflags
&= ~X86_EFLAGS_ZF
;
2195 ctxt
->dst
.val64
= ((u64
)reg_read(ctxt
, VCPU_REGS_RCX
) << 32) |
2196 (u32
) reg_read(ctxt
, VCPU_REGS_RBX
);
2198 ctxt
->eflags
|= X86_EFLAGS_ZF
;
2200 return X86EMUL_CONTINUE
;
2203 static int em_ret(struct x86_emulate_ctxt
*ctxt
)
2208 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2209 if (rc
!= X86EMUL_CONTINUE
)
2212 return assign_eip_near(ctxt
, eip
);
2215 static int em_ret_far(struct x86_emulate_ctxt
*ctxt
)
2218 unsigned long eip
, cs
;
2220 int cpl
= ctxt
->ops
->cpl(ctxt
);
2221 struct desc_struct old_desc
, new_desc
;
2222 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2224 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2225 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
,
2228 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2229 if (rc
!= X86EMUL_CONTINUE
)
2231 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2232 if (rc
!= X86EMUL_CONTINUE
)
2234 /* Outer-privilege level return is not implemented */
2235 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
&& (cs
& 3) > cpl
)
2236 return X86EMUL_UNHANDLEABLE
;
2237 rc
= __load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
, cpl
,
2240 if (rc
!= X86EMUL_CONTINUE
)
2242 rc
= assign_eip_far(ctxt
, eip
, &new_desc
);
2243 if (rc
!= X86EMUL_CONTINUE
) {
2244 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2245 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
2250 static int em_ret_far_imm(struct x86_emulate_ctxt
*ctxt
)
2254 rc
= em_ret_far(ctxt
);
2255 if (rc
!= X86EMUL_CONTINUE
)
2257 rsp_increment(ctxt
, ctxt
->src
.val
);
2258 return X86EMUL_CONTINUE
;
2261 static int em_cmpxchg(struct x86_emulate_ctxt
*ctxt
)
2263 /* Save real source value, then compare EAX against destination. */
2264 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
2265 ctxt
->dst
.val
= reg_read(ctxt
, VCPU_REGS_RAX
);
2266 ctxt
->src
.orig_val
= ctxt
->src
.val
;
2267 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2268 fastop(ctxt
, em_cmp
);
2270 if (ctxt
->eflags
& X86_EFLAGS_ZF
) {
2271 /* Success: write back to memory; no update of EAX */
2272 ctxt
->src
.type
= OP_NONE
;
2273 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
2275 /* Failure: write the value we saw to EAX. */
2276 ctxt
->src
.type
= OP_REG
;
2277 ctxt
->src
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
2278 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2279 /* Create write-cycle to dest by writing the same value */
2280 ctxt
->dst
.val
= ctxt
->dst
.orig_val
;
2282 return X86EMUL_CONTINUE
;
2285 static int em_lseg(struct x86_emulate_ctxt
*ctxt
)
2287 int seg
= ctxt
->src2
.val
;
2291 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2293 rc
= load_segment_descriptor(ctxt
, sel
, seg
);
2294 if (rc
!= X86EMUL_CONTINUE
)
2297 ctxt
->dst
.val
= ctxt
->src
.val
;
2301 static int emulator_has_longmode(struct x86_emulate_ctxt
*ctxt
)
2303 u32 eax
, ebx
, ecx
, edx
;
2307 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2308 return edx
& bit(X86_FEATURE_LM
);
2311 #define GET_SMSTATE(type, smbase, offset) \
2314 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2316 if (r != X86EMUL_CONTINUE) \
2317 return X86EMUL_UNHANDLEABLE; \
2321 static void rsm_set_desc_flags(struct desc_struct
*desc
, u32 flags
)
2323 desc
->g
= (flags
>> 23) & 1;
2324 desc
->d
= (flags
>> 22) & 1;
2325 desc
->l
= (flags
>> 21) & 1;
2326 desc
->avl
= (flags
>> 20) & 1;
2327 desc
->p
= (flags
>> 15) & 1;
2328 desc
->dpl
= (flags
>> 13) & 3;
2329 desc
->s
= (flags
>> 12) & 1;
2330 desc
->type
= (flags
>> 8) & 15;
2333 static int rsm_load_seg_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2335 struct desc_struct desc
;
2339 selector
= GET_SMSTATE(u32
, smbase
, 0x7fa8 + n
* 4);
2342 offset
= 0x7f84 + n
* 12;
2344 offset
= 0x7f2c + (n
- 3) * 12;
2346 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2347 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2348 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, offset
));
2349 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, n
);
2350 return X86EMUL_CONTINUE
;
2353 static int rsm_load_seg_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2355 struct desc_struct desc
;
2360 offset
= 0x7e00 + n
* 16;
2362 selector
= GET_SMSTATE(u16
, smbase
, offset
);
2363 rsm_set_desc_flags(&desc
, GET_SMSTATE(u16
, smbase
, offset
+ 2) << 8);
2364 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2365 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2366 base3
= GET_SMSTATE(u32
, smbase
, offset
+ 12);
2368 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, n
);
2369 return X86EMUL_CONTINUE
;
2372 static int rsm_enter_protected_mode(struct x86_emulate_ctxt
*ctxt
,
2378 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2379 * Then enable protected mode. However, PCID cannot be enabled
2380 * if EFER.LMA=0, so set it separately.
2382 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2384 return X86EMUL_UNHANDLEABLE
;
2386 bad
= ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
2388 return X86EMUL_UNHANDLEABLE
;
2390 if (cr4
& X86_CR4_PCIDE
) {
2391 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
);
2393 return X86EMUL_UNHANDLEABLE
;
2396 return X86EMUL_CONTINUE
;
2399 static int rsm_load_state_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2401 struct desc_struct desc
;
2407 cr0
= GET_SMSTATE(u32
, smbase
, 0x7ffc);
2408 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u32
, smbase
, 0x7ff8));
2409 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7ff4) | X86_EFLAGS_FIXED
;
2410 ctxt
->_eip
= GET_SMSTATE(u32
, smbase
, 0x7ff0);
2412 for (i
= 0; i
< 8; i
++)
2413 *reg_write(ctxt
, i
) = GET_SMSTATE(u32
, smbase
, 0x7fd0 + i
* 4);
2415 val
= GET_SMSTATE(u32
, smbase
, 0x7fcc);
2416 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2417 val
= GET_SMSTATE(u32
, smbase
, 0x7fc8);
2418 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2420 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc4);
2421 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f64));
2422 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f60));
2423 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f5c));
2424 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_TR
);
2426 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc0);
2427 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f80));
2428 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f7c));
2429 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f78));
2430 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_LDTR
);
2432 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f74);
2433 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f70);
2434 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2436 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f58);
2437 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f54);
2438 ctxt
->ops
->set_idt(ctxt
, &dt
);
2440 for (i
= 0; i
< 6; i
++) {
2441 int r
= rsm_load_seg_32(ctxt
, smbase
, i
);
2442 if (r
!= X86EMUL_CONTINUE
)
2446 cr4
= GET_SMSTATE(u32
, smbase
, 0x7f14);
2448 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7ef8));
2450 return rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2453 static int rsm_load_state_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2455 struct desc_struct desc
;
2462 for (i
= 0; i
< 16; i
++)
2463 *reg_write(ctxt
, i
) = GET_SMSTATE(u64
, smbase
, 0x7ff8 - i
* 8);
2465 ctxt
->_eip
= GET_SMSTATE(u64
, smbase
, 0x7f78);
2466 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7f70) | X86_EFLAGS_FIXED
;
2468 val
= GET_SMSTATE(u32
, smbase
, 0x7f68);
2469 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2470 val
= GET_SMSTATE(u32
, smbase
, 0x7f60);
2471 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2473 cr0
= GET_SMSTATE(u64
, smbase
, 0x7f58);
2474 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u64
, smbase
, 0x7f50));
2475 cr4
= GET_SMSTATE(u64
, smbase
, 0x7f48);
2476 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7f00));
2477 val
= GET_SMSTATE(u64
, smbase
, 0x7ed0);
2478 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, val
& ~EFER_LMA
);
2480 selector
= GET_SMSTATE(u32
, smbase
, 0x7e90);
2481 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e92) << 8);
2482 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e94));
2483 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e98));
2484 base3
= GET_SMSTATE(u32
, smbase
, 0x7e9c);
2485 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_TR
);
2487 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e84);
2488 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e88);
2489 ctxt
->ops
->set_idt(ctxt
, &dt
);
2491 selector
= GET_SMSTATE(u32
, smbase
, 0x7e70);
2492 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e72) << 8);
2493 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e74));
2494 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e78));
2495 base3
= GET_SMSTATE(u32
, smbase
, 0x7e7c);
2496 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_LDTR
);
2498 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e64);
2499 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e68);
2500 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2502 r
= rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2503 if (r
!= X86EMUL_CONTINUE
)
2506 for (i
= 0; i
< 6; i
++) {
2507 r
= rsm_load_seg_64(ctxt
, smbase
, i
);
2508 if (r
!= X86EMUL_CONTINUE
)
2512 return X86EMUL_CONTINUE
;
2515 static int em_rsm(struct x86_emulate_ctxt
*ctxt
)
2517 unsigned long cr0
, cr4
, efer
;
2521 if ((ctxt
->emul_flags
& X86EMUL_SMM_MASK
) == 0)
2522 return emulate_ud(ctxt
);
2525 * Get back to real mode, to prepare a safe state in which to load
2526 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2527 * supports long mode.
2529 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
2530 if (emulator_has_longmode(ctxt
)) {
2531 struct desc_struct cs_desc
;
2533 /* Zero CR4.PCIDE before CR0.PG. */
2534 if (cr4
& X86_CR4_PCIDE
) {
2535 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2536 cr4
&= ~X86_CR4_PCIDE
;
2539 /* A 32-bit code segment is required to clear EFER.LMA. */
2540 memset(&cs_desc
, 0, sizeof(cs_desc
));
2542 cs_desc
.s
= cs_desc
.g
= cs_desc
.p
= 1;
2543 ctxt
->ops
->set_segment(ctxt
, 0, &cs_desc
, 0, VCPU_SREG_CS
);
2546 /* For the 64-bit case, this will clear EFER.LMA. */
2547 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
2548 if (cr0
& X86_CR0_PE
)
2549 ctxt
->ops
->set_cr(ctxt
, 0, cr0
& ~(X86_CR0_PG
| X86_CR0_PE
));
2551 /* Now clear CR4.PAE (which must be done before clearing EFER.LME). */
2552 if (cr4
& X86_CR4_PAE
)
2553 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PAE
);
2555 /* And finally go back to 32-bit mode. */
2557 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, efer
);
2559 smbase
= ctxt
->ops
->get_smbase(ctxt
);
2560 if (emulator_has_longmode(ctxt
))
2561 ret
= rsm_load_state_64(ctxt
, smbase
+ 0x8000);
2563 ret
= rsm_load_state_32(ctxt
, smbase
+ 0x8000);
2565 if (ret
!= X86EMUL_CONTINUE
) {
2566 /* FIXME: should triple fault */
2567 return X86EMUL_UNHANDLEABLE
;
2570 if ((ctxt
->emul_flags
& X86EMUL_SMM_INSIDE_NMI_MASK
) == 0)
2571 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2573 ctxt
->emul_flags
&= ~X86EMUL_SMM_INSIDE_NMI_MASK
;
2574 ctxt
->emul_flags
&= ~X86EMUL_SMM_MASK
;
2575 return X86EMUL_CONTINUE
;
2579 setup_syscalls_segments(struct x86_emulate_ctxt
*ctxt
,
2580 struct desc_struct
*cs
, struct desc_struct
*ss
)
2582 cs
->l
= 0; /* will be adjusted later */
2583 set_desc_base(cs
, 0); /* flat segment */
2584 cs
->g
= 1; /* 4kb granularity */
2585 set_desc_limit(cs
, 0xfffff); /* 4GB limit */
2586 cs
->type
= 0x0b; /* Read, Execute, Accessed */
2588 cs
->dpl
= 0; /* will be adjusted later */
2593 set_desc_base(ss
, 0); /* flat segment */
2594 set_desc_limit(ss
, 0xfffff); /* 4GB limit */
2595 ss
->g
= 1; /* 4kb granularity */
2597 ss
->type
= 0x03; /* Read/Write, Accessed */
2598 ss
->d
= 1; /* 32bit stack segment */
2605 static bool vendor_intel(struct x86_emulate_ctxt
*ctxt
)
2607 u32 eax
, ebx
, ecx
, edx
;
2610 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2611 return ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2612 && ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2613 && edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
;
2616 static bool em_syscall_is_enabled(struct x86_emulate_ctxt
*ctxt
)
2618 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2619 u32 eax
, ebx
, ecx
, edx
;
2622 * syscall should always be enabled in longmode - so only become
2623 * vendor specific (cpuid) if other modes are active...
2625 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2630 ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2632 * Intel ("GenuineIntel")
2633 * remark: Intel CPUs only support "syscall" in 64bit
2634 * longmode. Also an 64bit guest with a
2635 * 32bit compat-app running will #UD !! While this
2636 * behaviour can be fixed (by emulating) into AMD
2637 * response - CPUs of AMD can't behave like Intel.
2639 if (ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
&&
2640 ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
&&
2641 edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
)
2644 /* AMD ("AuthenticAMD") */
2645 if (ebx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx
&&
2646 ecx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx
&&
2647 edx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_edx
)
2650 /* AMD ("AMDisbetter!") */
2651 if (ebx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx
&&
2652 ecx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx
&&
2653 edx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_edx
)
2656 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2660 static int em_syscall(struct x86_emulate_ctxt
*ctxt
)
2662 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2663 struct desc_struct cs
, ss
;
2668 /* syscall is not available in real mode */
2669 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2670 ctxt
->mode
== X86EMUL_MODE_VM86
)
2671 return emulate_ud(ctxt
);
2673 if (!(em_syscall_is_enabled(ctxt
)))
2674 return emulate_ud(ctxt
);
2676 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2677 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2679 if (!(efer
& EFER_SCE
))
2680 return emulate_ud(ctxt
);
2682 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2684 cs_sel
= (u16
)(msr_data
& 0xfffc);
2685 ss_sel
= (u16
)(msr_data
+ 8);
2687 if (efer
& EFER_LMA
) {
2691 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2692 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2694 *reg_write(ctxt
, VCPU_REGS_RCX
) = ctxt
->_eip
;
2695 if (efer
& EFER_LMA
) {
2696 #ifdef CONFIG_X86_64
2697 *reg_write(ctxt
, VCPU_REGS_R11
) = ctxt
->eflags
;
2700 ctxt
->mode
== X86EMUL_MODE_PROT64
?
2701 MSR_LSTAR
: MSR_CSTAR
, &msr_data
);
2702 ctxt
->_eip
= msr_data
;
2704 ops
->get_msr(ctxt
, MSR_SYSCALL_MASK
, &msr_data
);
2705 ctxt
->eflags
&= ~msr_data
;
2706 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2710 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2711 ctxt
->_eip
= (u32
)msr_data
;
2713 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2716 return X86EMUL_CONTINUE
;
2719 static int em_sysenter(struct x86_emulate_ctxt
*ctxt
)
2721 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2722 struct desc_struct cs
, ss
;
2727 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2728 /* inject #GP if in real mode */
2729 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2730 return emulate_gp(ctxt
, 0);
2733 * Not recognized on AMD in compat mode (but is recognized in legacy
2736 if ((ctxt
->mode
!= X86EMUL_MODE_PROT64
) && (efer
& EFER_LMA
)
2737 && !vendor_intel(ctxt
))
2738 return emulate_ud(ctxt
);
2740 /* sysenter/sysexit have not been tested in 64bit mode. */
2741 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2742 return X86EMUL_UNHANDLEABLE
;
2744 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2746 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2747 if ((msr_data
& 0xfffc) == 0x0)
2748 return emulate_gp(ctxt
, 0);
2750 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2751 cs_sel
= (u16
)msr_data
& ~SEGMENT_RPL_MASK
;
2752 ss_sel
= cs_sel
+ 8;
2753 if (efer
& EFER_LMA
) {
2758 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2759 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2761 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_EIP
, &msr_data
);
2762 ctxt
->_eip
= (efer
& EFER_LMA
) ? msr_data
: (u32
)msr_data
;
2764 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_ESP
, &msr_data
);
2765 *reg_write(ctxt
, VCPU_REGS_RSP
) = (efer
& EFER_LMA
) ? msr_data
:
2768 return X86EMUL_CONTINUE
;
2771 static int em_sysexit(struct x86_emulate_ctxt
*ctxt
)
2773 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2774 struct desc_struct cs
, ss
;
2775 u64 msr_data
, rcx
, rdx
;
2777 u16 cs_sel
= 0, ss_sel
= 0;
2779 /* inject #GP if in real mode or Virtual 8086 mode */
2780 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2781 ctxt
->mode
== X86EMUL_MODE_VM86
)
2782 return emulate_gp(ctxt
, 0);
2784 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2786 if ((ctxt
->rex_prefix
& 0x8) != 0x0)
2787 usermode
= X86EMUL_MODE_PROT64
;
2789 usermode
= X86EMUL_MODE_PROT32
;
2791 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2792 rdx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2796 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2798 case X86EMUL_MODE_PROT32
:
2799 cs_sel
= (u16
)(msr_data
+ 16);
2800 if ((msr_data
& 0xfffc) == 0x0)
2801 return emulate_gp(ctxt
, 0);
2802 ss_sel
= (u16
)(msr_data
+ 24);
2806 case X86EMUL_MODE_PROT64
:
2807 cs_sel
= (u16
)(msr_data
+ 32);
2808 if (msr_data
== 0x0)
2809 return emulate_gp(ctxt
, 0);
2810 ss_sel
= cs_sel
+ 8;
2813 if (is_noncanonical_address(rcx
) ||
2814 is_noncanonical_address(rdx
))
2815 return emulate_gp(ctxt
, 0);
2818 cs_sel
|= SEGMENT_RPL_MASK
;
2819 ss_sel
|= SEGMENT_RPL_MASK
;
2821 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2822 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2825 *reg_write(ctxt
, VCPU_REGS_RSP
) = rcx
;
2827 return X86EMUL_CONTINUE
;
2830 static bool emulator_bad_iopl(struct x86_emulate_ctxt
*ctxt
)
2833 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2835 if (ctxt
->mode
== X86EMUL_MODE_VM86
)
2837 iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
2838 return ctxt
->ops
->cpl(ctxt
) > iopl
;
2841 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt
*ctxt
,
2844 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2845 struct desc_struct tr_seg
;
2848 u16 tr
, io_bitmap_ptr
, perm
, bit_idx
= port
& 0x7;
2849 unsigned mask
= (1 << len
) - 1;
2852 ops
->get_segment(ctxt
, &tr
, &tr_seg
, &base3
, VCPU_SREG_TR
);
2855 if (desc_limit_scaled(&tr_seg
) < 103)
2857 base
= get_desc_base(&tr_seg
);
2858 #ifdef CONFIG_X86_64
2859 base
|= ((u64
)base3
) << 32;
2861 r
= ops
->read_std(ctxt
, base
+ 102, &io_bitmap_ptr
, 2, NULL
);
2862 if (r
!= X86EMUL_CONTINUE
)
2864 if (io_bitmap_ptr
+ port
/8 > desc_limit_scaled(&tr_seg
))
2866 r
= ops
->read_std(ctxt
, base
+ io_bitmap_ptr
+ port
/8, &perm
, 2, NULL
);
2867 if (r
!= X86EMUL_CONTINUE
)
2869 if ((perm
>> bit_idx
) & mask
)
2874 static bool emulator_io_permited(struct x86_emulate_ctxt
*ctxt
,
2880 if (emulator_bad_iopl(ctxt
))
2881 if (!emulator_io_port_access_allowed(ctxt
, port
, len
))
2884 ctxt
->perm_ok
= true;
2889 static void string_registers_quirk(struct x86_emulate_ctxt
*ctxt
)
2892 * Intel CPUs mask the counter and pointers in quite strange
2893 * manner when ECX is zero due to REP-string optimizations.
2895 #ifdef CONFIG_X86_64
2896 if (ctxt
->ad_bytes
!= 4 || !vendor_intel(ctxt
))
2899 *reg_write(ctxt
, VCPU_REGS_RCX
) = 0;
2902 case 0xa4: /* movsb */
2903 case 0xa5: /* movsd/w */
2904 *reg_rmw(ctxt
, VCPU_REGS_RSI
) &= (u32
)-1;
2906 case 0xaa: /* stosb */
2907 case 0xab: /* stosd/w */
2908 *reg_rmw(ctxt
, VCPU_REGS_RDI
) &= (u32
)-1;
2913 static void save_state_to_tss16(struct x86_emulate_ctxt
*ctxt
,
2914 struct tss_segment_16
*tss
)
2916 tss
->ip
= ctxt
->_eip
;
2917 tss
->flag
= ctxt
->eflags
;
2918 tss
->ax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2919 tss
->cx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2920 tss
->dx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2921 tss
->bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2922 tss
->sp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2923 tss
->bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2924 tss
->si
= reg_read(ctxt
, VCPU_REGS_RSI
);
2925 tss
->di
= reg_read(ctxt
, VCPU_REGS_RDI
);
2927 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2928 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2929 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2930 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2931 tss
->ldt
= get_segment_selector(ctxt
, VCPU_SREG_LDTR
);
2934 static int load_state_from_tss16(struct x86_emulate_ctxt
*ctxt
,
2935 struct tss_segment_16
*tss
)
2940 ctxt
->_eip
= tss
->ip
;
2941 ctxt
->eflags
= tss
->flag
| 2;
2942 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->ax
;
2943 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->cx
;
2944 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->dx
;
2945 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->bx
;
2946 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->sp
;
2947 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->bp
;
2948 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->si
;
2949 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->di
;
2952 * SDM says that segment selectors are loaded before segment
2955 set_segment_selector(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
);
2956 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
2957 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
2958 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
2959 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
2964 * Now load segment descriptors. If fault happens at this stage
2965 * it is handled in a context of new task
2967 ret
= __load_segment_descriptor(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
, cpl
,
2968 X86_TRANSFER_TASK_SWITCH
, NULL
);
2969 if (ret
!= X86EMUL_CONTINUE
)
2971 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
2972 X86_TRANSFER_TASK_SWITCH
, NULL
);
2973 if (ret
!= X86EMUL_CONTINUE
)
2975 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
2976 X86_TRANSFER_TASK_SWITCH
, NULL
);
2977 if (ret
!= X86EMUL_CONTINUE
)
2979 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
2980 X86_TRANSFER_TASK_SWITCH
, NULL
);
2981 if (ret
!= X86EMUL_CONTINUE
)
2983 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
2984 X86_TRANSFER_TASK_SWITCH
, NULL
);
2985 if (ret
!= X86EMUL_CONTINUE
)
2988 return X86EMUL_CONTINUE
;
2991 static int task_switch_16(struct x86_emulate_ctxt
*ctxt
,
2992 u16 tss_selector
, u16 old_tss_sel
,
2993 ulong old_tss_base
, struct desc_struct
*new_desc
)
2995 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2996 struct tss_segment_16 tss_seg
;
2998 u32 new_tss_base
= get_desc_base(new_desc
);
3000 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3002 if (ret
!= X86EMUL_CONTINUE
)
3005 save_state_to_tss16(ctxt
, &tss_seg
);
3007 ret
= ops
->write_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3009 if (ret
!= X86EMUL_CONTINUE
)
3012 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3014 if (ret
!= X86EMUL_CONTINUE
)
3017 if (old_tss_sel
!= 0xffff) {
3018 tss_seg
.prev_task_link
= old_tss_sel
;
3020 ret
= ops
->write_std(ctxt
, new_tss_base
,
3021 &tss_seg
.prev_task_link
,
3022 sizeof tss_seg
.prev_task_link
,
3024 if (ret
!= X86EMUL_CONTINUE
)
3028 return load_state_from_tss16(ctxt
, &tss_seg
);
3031 static void save_state_to_tss32(struct x86_emulate_ctxt
*ctxt
,
3032 struct tss_segment_32
*tss
)
3034 /* CR3 and ldt selector are not saved intentionally */
3035 tss
->eip
= ctxt
->_eip
;
3036 tss
->eflags
= ctxt
->eflags
;
3037 tss
->eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3038 tss
->ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3039 tss
->edx
= reg_read(ctxt
, VCPU_REGS_RDX
);
3040 tss
->ebx
= reg_read(ctxt
, VCPU_REGS_RBX
);
3041 tss
->esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
3042 tss
->ebp
= reg_read(ctxt
, VCPU_REGS_RBP
);
3043 tss
->esi
= reg_read(ctxt
, VCPU_REGS_RSI
);
3044 tss
->edi
= reg_read(ctxt
, VCPU_REGS_RDI
);
3046 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
3047 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
3048 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
3049 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
3050 tss
->fs
= get_segment_selector(ctxt
, VCPU_SREG_FS
);
3051 tss
->gs
= get_segment_selector(ctxt
, VCPU_SREG_GS
);
3054 static int load_state_from_tss32(struct x86_emulate_ctxt
*ctxt
,
3055 struct tss_segment_32
*tss
)
3060 if (ctxt
->ops
->set_cr(ctxt
, 3, tss
->cr3
))
3061 return emulate_gp(ctxt
, 0);
3062 ctxt
->_eip
= tss
->eip
;
3063 ctxt
->eflags
= tss
->eflags
| 2;
3065 /* General purpose registers */
3066 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->eax
;
3067 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->ecx
;
3068 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->edx
;
3069 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->ebx
;
3070 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->esp
;
3071 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->ebp
;
3072 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->esi
;
3073 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->edi
;
3076 * SDM says that segment selectors are loaded before segment
3077 * descriptors. This is important because CPL checks will
3080 set_segment_selector(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
);
3081 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
3082 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
3083 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
3084 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
3085 set_segment_selector(ctxt
, tss
->fs
, VCPU_SREG_FS
);
3086 set_segment_selector(ctxt
, tss
->gs
, VCPU_SREG_GS
);
3089 * If we're switching between Protected Mode and VM86, we need to make
3090 * sure to update the mode before loading the segment descriptors so
3091 * that the selectors are interpreted correctly.
3093 if (ctxt
->eflags
& X86_EFLAGS_VM
) {
3094 ctxt
->mode
= X86EMUL_MODE_VM86
;
3097 ctxt
->mode
= X86EMUL_MODE_PROT32
;
3102 * Now load segment descriptors. If fault happenes at this stage
3103 * it is handled in a context of new task
3105 ret
= __load_segment_descriptor(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
,
3106 cpl
, X86_TRANSFER_TASK_SWITCH
, NULL
);
3107 if (ret
!= X86EMUL_CONTINUE
)
3109 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3110 X86_TRANSFER_TASK_SWITCH
, NULL
);
3111 if (ret
!= X86EMUL_CONTINUE
)
3113 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3114 X86_TRANSFER_TASK_SWITCH
, NULL
);
3115 if (ret
!= X86EMUL_CONTINUE
)
3117 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3118 X86_TRANSFER_TASK_SWITCH
, NULL
);
3119 if (ret
!= X86EMUL_CONTINUE
)
3121 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3122 X86_TRANSFER_TASK_SWITCH
, NULL
);
3123 if (ret
!= X86EMUL_CONTINUE
)
3125 ret
= __load_segment_descriptor(ctxt
, tss
->fs
, VCPU_SREG_FS
, cpl
,
3126 X86_TRANSFER_TASK_SWITCH
, NULL
);
3127 if (ret
!= X86EMUL_CONTINUE
)
3129 ret
= __load_segment_descriptor(ctxt
, tss
->gs
, VCPU_SREG_GS
, cpl
,
3130 X86_TRANSFER_TASK_SWITCH
, NULL
);
3135 static int task_switch_32(struct x86_emulate_ctxt
*ctxt
,
3136 u16 tss_selector
, u16 old_tss_sel
,
3137 ulong old_tss_base
, struct desc_struct
*new_desc
)
3139 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3140 struct tss_segment_32 tss_seg
;
3142 u32 new_tss_base
= get_desc_base(new_desc
);
3143 u32 eip_offset
= offsetof(struct tss_segment_32
, eip
);
3144 u32 ldt_sel_offset
= offsetof(struct tss_segment_32
, ldt_selector
);
3146 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3148 if (ret
!= X86EMUL_CONTINUE
)
3151 save_state_to_tss32(ctxt
, &tss_seg
);
3153 /* Only GP registers and segment selectors are saved */
3154 ret
= ops
->write_std(ctxt
, old_tss_base
+ eip_offset
, &tss_seg
.eip
,
3155 ldt_sel_offset
- eip_offset
, &ctxt
->exception
);
3156 if (ret
!= X86EMUL_CONTINUE
)
3159 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3161 if (ret
!= X86EMUL_CONTINUE
)
3164 if (old_tss_sel
!= 0xffff) {
3165 tss_seg
.prev_task_link
= old_tss_sel
;
3167 ret
= ops
->write_std(ctxt
, new_tss_base
,
3168 &tss_seg
.prev_task_link
,
3169 sizeof tss_seg
.prev_task_link
,
3171 if (ret
!= X86EMUL_CONTINUE
)
3175 return load_state_from_tss32(ctxt
, &tss_seg
);
3178 static int emulator_do_task_switch(struct x86_emulate_ctxt
*ctxt
,
3179 u16 tss_selector
, int idt_index
, int reason
,
3180 bool has_error_code
, u32 error_code
)
3182 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3183 struct desc_struct curr_tss_desc
, next_tss_desc
;
3185 u16 old_tss_sel
= get_segment_selector(ctxt
, VCPU_SREG_TR
);
3186 ulong old_tss_base
=
3187 ops
->get_cached_segment_base(ctxt
, VCPU_SREG_TR
);
3189 ulong desc_addr
, dr7
;
3191 /* FIXME: old_tss_base == ~0 ? */
3193 ret
= read_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
, &desc_addr
);
3194 if (ret
!= X86EMUL_CONTINUE
)
3196 ret
= read_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
, &desc_addr
);
3197 if (ret
!= X86EMUL_CONTINUE
)
3200 /* FIXME: check that next_tss_desc is tss */
3203 * Check privileges. The three cases are task switch caused by...
3205 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3206 * 2. Exception/IRQ/iret: No check is performed
3207 * 3. jmp/call to TSS/task-gate: No check is performed since the
3208 * hardware checks it before exiting.
3210 if (reason
== TASK_SWITCH_GATE
) {
3211 if (idt_index
!= -1) {
3212 /* Software interrupts */
3213 struct desc_struct task_gate_desc
;
3216 ret
= read_interrupt_descriptor(ctxt
, idt_index
,
3218 if (ret
!= X86EMUL_CONTINUE
)
3221 dpl
= task_gate_desc
.dpl
;
3222 if ((tss_selector
& 3) > dpl
|| ops
->cpl(ctxt
) > dpl
)
3223 return emulate_gp(ctxt
, (idt_index
<< 3) | 0x2);
3227 desc_limit
= desc_limit_scaled(&next_tss_desc
);
3228 if (!next_tss_desc
.p
||
3229 ((desc_limit
< 0x67 && (next_tss_desc
.type
& 8)) ||
3230 desc_limit
< 0x2b)) {
3231 return emulate_ts(ctxt
, tss_selector
& 0xfffc);
3234 if (reason
== TASK_SWITCH_IRET
|| reason
== TASK_SWITCH_JMP
) {
3235 curr_tss_desc
.type
&= ~(1 << 1); /* clear busy flag */
3236 write_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
);
3239 if (reason
== TASK_SWITCH_IRET
)
3240 ctxt
->eflags
= ctxt
->eflags
& ~X86_EFLAGS_NT
;
3242 /* set back link to prev task only if NT bit is set in eflags
3243 note that old_tss_sel is not used after this point */
3244 if (reason
!= TASK_SWITCH_CALL
&& reason
!= TASK_SWITCH_GATE
)
3245 old_tss_sel
= 0xffff;
3247 if (next_tss_desc
.type
& 8)
3248 ret
= task_switch_32(ctxt
, tss_selector
, old_tss_sel
,
3249 old_tss_base
, &next_tss_desc
);
3251 ret
= task_switch_16(ctxt
, tss_selector
, old_tss_sel
,
3252 old_tss_base
, &next_tss_desc
);
3253 if (ret
!= X86EMUL_CONTINUE
)
3256 if (reason
== TASK_SWITCH_CALL
|| reason
== TASK_SWITCH_GATE
)
3257 ctxt
->eflags
= ctxt
->eflags
| X86_EFLAGS_NT
;
3259 if (reason
!= TASK_SWITCH_IRET
) {
3260 next_tss_desc
.type
|= (1 << 1); /* set busy flag */
3261 write_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
);
3264 ops
->set_cr(ctxt
, 0, ops
->get_cr(ctxt
, 0) | X86_CR0_TS
);
3265 ops
->set_segment(ctxt
, tss_selector
, &next_tss_desc
, 0, VCPU_SREG_TR
);
3267 if (has_error_code
) {
3268 ctxt
->op_bytes
= ctxt
->ad_bytes
= (next_tss_desc
.type
& 8) ? 4 : 2;
3269 ctxt
->lock_prefix
= 0;
3270 ctxt
->src
.val
= (unsigned long) error_code
;
3271 ret
= em_push(ctxt
);
3274 ops
->get_dr(ctxt
, 7, &dr7
);
3275 ops
->set_dr(ctxt
, 7, dr7
& ~(DR_LOCAL_ENABLE_MASK
| DR_LOCAL_SLOWDOWN
));
3280 int emulator_task_switch(struct x86_emulate_ctxt
*ctxt
,
3281 u16 tss_selector
, int idt_index
, int reason
,
3282 bool has_error_code
, u32 error_code
)
3286 invalidate_registers(ctxt
);
3287 ctxt
->_eip
= ctxt
->eip
;
3288 ctxt
->dst
.type
= OP_NONE
;
3290 rc
= emulator_do_task_switch(ctxt
, tss_selector
, idt_index
, reason
,
3291 has_error_code
, error_code
);
3293 if (rc
== X86EMUL_CONTINUE
) {
3294 ctxt
->eip
= ctxt
->_eip
;
3295 writeback_registers(ctxt
);
3298 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
3301 static void string_addr_inc(struct x86_emulate_ctxt
*ctxt
, int reg
,
3304 int df
= (ctxt
->eflags
& X86_EFLAGS_DF
) ? -op
->count
: op
->count
;
3306 register_address_increment(ctxt
, reg
, df
* op
->bytes
);
3307 op
->addr
.mem
.ea
= register_address(ctxt
, reg
);
3310 static int em_das(struct x86_emulate_ctxt
*ctxt
)
3313 bool af
, cf
, old_cf
;
3315 cf
= ctxt
->eflags
& X86_EFLAGS_CF
;
3321 af
= ctxt
->eflags
& X86_EFLAGS_AF
;
3322 if ((al
& 0x0f) > 9 || af
) {
3324 cf
= old_cf
| (al
>= 250);
3329 if (old_al
> 0x99 || old_cf
) {
3335 /* Set PF, ZF, SF */
3336 ctxt
->src
.type
= OP_IMM
;
3338 ctxt
->src
.bytes
= 1;
3339 fastop(ctxt
, em_or
);
3340 ctxt
->eflags
&= ~(X86_EFLAGS_AF
| X86_EFLAGS_CF
);
3342 ctxt
->eflags
|= X86_EFLAGS_CF
;
3344 ctxt
->eflags
|= X86_EFLAGS_AF
;
3345 return X86EMUL_CONTINUE
;
3348 static int em_aam(struct x86_emulate_ctxt
*ctxt
)
3352 if (ctxt
->src
.val
== 0)
3353 return emulate_de(ctxt
);
3355 al
= ctxt
->dst
.val
& 0xff;
3356 ah
= al
/ ctxt
->src
.val
;
3357 al
%= ctxt
->src
.val
;
3359 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
| (ah
<< 8);
3361 /* Set PF, ZF, SF */
3362 ctxt
->src
.type
= OP_IMM
;
3364 ctxt
->src
.bytes
= 1;
3365 fastop(ctxt
, em_or
);
3367 return X86EMUL_CONTINUE
;
3370 static int em_aad(struct x86_emulate_ctxt
*ctxt
)
3372 u8 al
= ctxt
->dst
.val
& 0xff;
3373 u8 ah
= (ctxt
->dst
.val
>> 8) & 0xff;
3375 al
= (al
+ (ah
* ctxt
->src
.val
)) & 0xff;
3377 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
;
3379 /* Set PF, ZF, SF */
3380 ctxt
->src
.type
= OP_IMM
;
3382 ctxt
->src
.bytes
= 1;
3383 fastop(ctxt
, em_or
);
3385 return X86EMUL_CONTINUE
;
3388 static int em_call(struct x86_emulate_ctxt
*ctxt
)
3391 long rel
= ctxt
->src
.val
;
3393 ctxt
->src
.val
= (unsigned long)ctxt
->_eip
;
3394 rc
= jmp_rel(ctxt
, rel
);
3395 if (rc
!= X86EMUL_CONTINUE
)
3397 return em_push(ctxt
);
3400 static int em_call_far(struct x86_emulate_ctxt
*ctxt
)
3405 struct desc_struct old_desc
, new_desc
;
3406 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3407 int cpl
= ctxt
->ops
->cpl(ctxt
);
3408 enum x86emul_mode prev_mode
= ctxt
->mode
;
3410 old_eip
= ctxt
->_eip
;
3411 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
, VCPU_SREG_CS
);
3413 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
3414 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
3415 X86_TRANSFER_CALL_JMP
, &new_desc
);
3416 if (rc
!= X86EMUL_CONTINUE
)
3419 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
3420 if (rc
!= X86EMUL_CONTINUE
)
3423 ctxt
->src
.val
= old_cs
;
3425 if (rc
!= X86EMUL_CONTINUE
)
3428 ctxt
->src
.val
= old_eip
;
3430 /* If we failed, we tainted the memory, but the very least we should
3432 if (rc
!= X86EMUL_CONTINUE
) {
3433 pr_warn_once("faulting far call emulation tainted memory\n");
3438 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
3439 ctxt
->mode
= prev_mode
;
3444 static int em_ret_near_imm(struct x86_emulate_ctxt
*ctxt
)
3449 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
3450 if (rc
!= X86EMUL_CONTINUE
)
3452 rc
= assign_eip_near(ctxt
, eip
);
3453 if (rc
!= X86EMUL_CONTINUE
)
3455 rsp_increment(ctxt
, ctxt
->src
.val
);
3456 return X86EMUL_CONTINUE
;
3459 static int em_xchg(struct x86_emulate_ctxt
*ctxt
)
3461 /* Write back the register source. */
3462 ctxt
->src
.val
= ctxt
->dst
.val
;
3463 write_register_operand(&ctxt
->src
);
3465 /* Write back the memory destination with implicit LOCK prefix. */
3466 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
3467 ctxt
->lock_prefix
= 1;
3468 return X86EMUL_CONTINUE
;
3471 static int em_imul_3op(struct x86_emulate_ctxt
*ctxt
)
3473 ctxt
->dst
.val
= ctxt
->src2
.val
;
3474 return fastop(ctxt
, em_imul
);
3477 static int em_cwd(struct x86_emulate_ctxt
*ctxt
)
3479 ctxt
->dst
.type
= OP_REG
;
3480 ctxt
->dst
.bytes
= ctxt
->src
.bytes
;
3481 ctxt
->dst
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
3482 ctxt
->dst
.val
= ~((ctxt
->src
.val
>> (ctxt
->src
.bytes
* 8 - 1)) - 1);
3484 return X86EMUL_CONTINUE
;
3487 static int em_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3491 ctxt
->ops
->get_msr(ctxt
, MSR_IA32_TSC
, &tsc
);
3492 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)tsc
;
3493 *reg_write(ctxt
, VCPU_REGS_RDX
) = tsc
>> 32;
3494 return X86EMUL_CONTINUE
;
3497 static int em_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3501 if (ctxt
->ops
->read_pmc(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &pmc
))
3502 return emulate_gp(ctxt
, 0);
3503 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)pmc
;
3504 *reg_write(ctxt
, VCPU_REGS_RDX
) = pmc
>> 32;
3505 return X86EMUL_CONTINUE
;
3508 static int em_mov(struct x86_emulate_ctxt
*ctxt
)
3510 memcpy(ctxt
->dst
.valptr
, ctxt
->src
.valptr
, sizeof(ctxt
->src
.valptr
));
3511 return X86EMUL_CONTINUE
;
3514 #define FFL(x) bit(X86_FEATURE_##x)
3516 static int em_movbe(struct x86_emulate_ctxt
*ctxt
)
3518 u32 ebx
, ecx
, edx
, eax
= 1;
3522 * Check MOVBE is set in the guest-visible CPUID leaf.
3524 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3525 if (!(ecx
& FFL(MOVBE
)))
3526 return emulate_ud(ctxt
);
3528 switch (ctxt
->op_bytes
) {
3531 * From MOVBE definition: "...When the operand size is 16 bits,
3532 * the upper word of the destination register remains unchanged
3535 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3536 * rules so we have to do the operation almost per hand.
3538 tmp
= (u16
)ctxt
->src
.val
;
3539 ctxt
->dst
.val
&= ~0xffffUL
;
3540 ctxt
->dst
.val
|= (unsigned long)swab16(tmp
);
3543 ctxt
->dst
.val
= swab32((u32
)ctxt
->src
.val
);
3546 ctxt
->dst
.val
= swab64(ctxt
->src
.val
);
3551 return X86EMUL_CONTINUE
;
3554 static int em_cr_write(struct x86_emulate_ctxt
*ctxt
)
3556 if (ctxt
->ops
->set_cr(ctxt
, ctxt
->modrm_reg
, ctxt
->src
.val
))
3557 return emulate_gp(ctxt
, 0);
3559 /* Disable writeback. */
3560 ctxt
->dst
.type
= OP_NONE
;
3561 return X86EMUL_CONTINUE
;
3564 static int em_dr_write(struct x86_emulate_ctxt
*ctxt
)
3568 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3569 val
= ctxt
->src
.val
& ~0ULL;
3571 val
= ctxt
->src
.val
& ~0U;
3573 /* #UD condition is already handled. */
3574 if (ctxt
->ops
->set_dr(ctxt
, ctxt
->modrm_reg
, val
) < 0)
3575 return emulate_gp(ctxt
, 0);
3577 /* Disable writeback. */
3578 ctxt
->dst
.type
= OP_NONE
;
3579 return X86EMUL_CONTINUE
;
3582 static int em_wrmsr(struct x86_emulate_ctxt
*ctxt
)
3586 msr_data
= (u32
)reg_read(ctxt
, VCPU_REGS_RAX
)
3587 | ((u64
)reg_read(ctxt
, VCPU_REGS_RDX
) << 32);
3588 if (ctxt
->ops
->set_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), msr_data
))
3589 return emulate_gp(ctxt
, 0);
3591 return X86EMUL_CONTINUE
;
3594 static int em_rdmsr(struct x86_emulate_ctxt
*ctxt
)
3598 if (ctxt
->ops
->get_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &msr_data
))
3599 return emulate_gp(ctxt
, 0);
3601 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)msr_data
;
3602 *reg_write(ctxt
, VCPU_REGS_RDX
) = msr_data
>> 32;
3603 return X86EMUL_CONTINUE
;
3606 static int em_mov_rm_sreg(struct x86_emulate_ctxt
*ctxt
)
3608 if (ctxt
->modrm_reg
> VCPU_SREG_GS
)
3609 return emulate_ud(ctxt
);
3611 ctxt
->dst
.val
= get_segment_selector(ctxt
, ctxt
->modrm_reg
);
3612 if (ctxt
->dst
.bytes
== 4 && ctxt
->dst
.type
== OP_MEM
)
3613 ctxt
->dst
.bytes
= 2;
3614 return X86EMUL_CONTINUE
;
3617 static int em_mov_sreg_rm(struct x86_emulate_ctxt
*ctxt
)
3619 u16 sel
= ctxt
->src
.val
;
3621 if (ctxt
->modrm_reg
== VCPU_SREG_CS
|| ctxt
->modrm_reg
> VCPU_SREG_GS
)
3622 return emulate_ud(ctxt
);
3624 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
3625 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
3627 /* Disable writeback. */
3628 ctxt
->dst
.type
= OP_NONE
;
3629 return load_segment_descriptor(ctxt
, sel
, ctxt
->modrm_reg
);
3632 static int em_lldt(struct x86_emulate_ctxt
*ctxt
)
3634 u16 sel
= ctxt
->src
.val
;
3636 /* Disable writeback. */
3637 ctxt
->dst
.type
= OP_NONE
;
3638 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_LDTR
);
3641 static int em_ltr(struct x86_emulate_ctxt
*ctxt
)
3643 u16 sel
= ctxt
->src
.val
;
3645 /* Disable writeback. */
3646 ctxt
->dst
.type
= OP_NONE
;
3647 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_TR
);
3650 static int em_invlpg(struct x86_emulate_ctxt
*ctxt
)
3655 rc
= linearize(ctxt
, ctxt
->src
.addr
.mem
, 1, false, &linear
);
3656 if (rc
== X86EMUL_CONTINUE
)
3657 ctxt
->ops
->invlpg(ctxt
, linear
);
3658 /* Disable writeback. */
3659 ctxt
->dst
.type
= OP_NONE
;
3660 return X86EMUL_CONTINUE
;
3663 static int em_clts(struct x86_emulate_ctxt
*ctxt
)
3667 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
3669 ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
3670 return X86EMUL_CONTINUE
;
3673 static int em_hypercall(struct x86_emulate_ctxt
*ctxt
)
3675 int rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3677 if (rc
!= X86EMUL_CONTINUE
)
3680 /* Let the processor re-execute the fixed hypercall */
3681 ctxt
->_eip
= ctxt
->eip
;
3682 /* Disable writeback. */
3683 ctxt
->dst
.type
= OP_NONE
;
3684 return X86EMUL_CONTINUE
;
3687 static int emulate_store_desc_ptr(struct x86_emulate_ctxt
*ctxt
,
3688 void (*get
)(struct x86_emulate_ctxt
*ctxt
,
3689 struct desc_ptr
*ptr
))
3691 struct desc_ptr desc_ptr
;
3693 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3695 get(ctxt
, &desc_ptr
);
3696 if (ctxt
->op_bytes
== 2) {
3698 desc_ptr
.address
&= 0x00ffffff;
3700 /* Disable writeback. */
3701 ctxt
->dst
.type
= OP_NONE
;
3702 return segmented_write(ctxt
, ctxt
->dst
.addr
.mem
,
3703 &desc_ptr
, 2 + ctxt
->op_bytes
);
3706 static int em_sgdt(struct x86_emulate_ctxt
*ctxt
)
3708 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_gdt
);
3711 static int em_sidt(struct x86_emulate_ctxt
*ctxt
)
3713 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_idt
);
3716 static int em_lgdt_lidt(struct x86_emulate_ctxt
*ctxt
, bool lgdt
)
3718 struct desc_ptr desc_ptr
;
3721 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3723 rc
= read_descriptor(ctxt
, ctxt
->src
.addr
.mem
,
3724 &desc_ptr
.size
, &desc_ptr
.address
,
3726 if (rc
!= X86EMUL_CONTINUE
)
3728 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&&
3729 is_noncanonical_address(desc_ptr
.address
))
3730 return emulate_gp(ctxt
, 0);
3732 ctxt
->ops
->set_gdt(ctxt
, &desc_ptr
);
3734 ctxt
->ops
->set_idt(ctxt
, &desc_ptr
);
3735 /* Disable writeback. */
3736 ctxt
->dst
.type
= OP_NONE
;
3737 return X86EMUL_CONTINUE
;
3740 static int em_lgdt(struct x86_emulate_ctxt
*ctxt
)
3742 return em_lgdt_lidt(ctxt
, true);
3745 static int em_lidt(struct x86_emulate_ctxt
*ctxt
)
3747 return em_lgdt_lidt(ctxt
, false);
3750 static int em_smsw(struct x86_emulate_ctxt
*ctxt
)
3752 if (ctxt
->dst
.type
== OP_MEM
)
3753 ctxt
->dst
.bytes
= 2;
3754 ctxt
->dst
.val
= ctxt
->ops
->get_cr(ctxt
, 0);
3755 return X86EMUL_CONTINUE
;
3758 static int em_lmsw(struct x86_emulate_ctxt
*ctxt
)
3760 ctxt
->ops
->set_cr(ctxt
, 0, (ctxt
->ops
->get_cr(ctxt
, 0) & ~0x0eul
)
3761 | (ctxt
->src
.val
& 0x0f));
3762 ctxt
->dst
.type
= OP_NONE
;
3763 return X86EMUL_CONTINUE
;
3766 static int em_loop(struct x86_emulate_ctxt
*ctxt
)
3768 int rc
= X86EMUL_CONTINUE
;
3770 register_address_increment(ctxt
, VCPU_REGS_RCX
, -1);
3771 if ((address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) != 0) &&
3772 (ctxt
->b
== 0xe2 || test_cc(ctxt
->b
^ 0x5, ctxt
->eflags
)))
3773 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3778 static int em_jcxz(struct x86_emulate_ctxt
*ctxt
)
3780 int rc
= X86EMUL_CONTINUE
;
3782 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0)
3783 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3788 static int em_in(struct x86_emulate_ctxt
*ctxt
)
3790 if (!pio_in_emulated(ctxt
, ctxt
->dst
.bytes
, ctxt
->src
.val
,
3792 return X86EMUL_IO_NEEDED
;
3794 return X86EMUL_CONTINUE
;
3797 static int em_out(struct x86_emulate_ctxt
*ctxt
)
3799 ctxt
->ops
->pio_out_emulated(ctxt
, ctxt
->src
.bytes
, ctxt
->dst
.val
,
3801 /* Disable writeback. */
3802 ctxt
->dst
.type
= OP_NONE
;
3803 return X86EMUL_CONTINUE
;
3806 static int em_cli(struct x86_emulate_ctxt
*ctxt
)
3808 if (emulator_bad_iopl(ctxt
))
3809 return emulate_gp(ctxt
, 0);
3811 ctxt
->eflags
&= ~X86_EFLAGS_IF
;
3812 return X86EMUL_CONTINUE
;
3815 static int em_sti(struct x86_emulate_ctxt
*ctxt
)
3817 if (emulator_bad_iopl(ctxt
))
3818 return emulate_gp(ctxt
, 0);
3820 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_STI
;
3821 ctxt
->eflags
|= X86_EFLAGS_IF
;
3822 return X86EMUL_CONTINUE
;
3825 static int em_cpuid(struct x86_emulate_ctxt
*ctxt
)
3827 u32 eax
, ebx
, ecx
, edx
;
3829 eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3830 ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3831 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3832 *reg_write(ctxt
, VCPU_REGS_RAX
) = eax
;
3833 *reg_write(ctxt
, VCPU_REGS_RBX
) = ebx
;
3834 *reg_write(ctxt
, VCPU_REGS_RCX
) = ecx
;
3835 *reg_write(ctxt
, VCPU_REGS_RDX
) = edx
;
3836 return X86EMUL_CONTINUE
;
3839 static int em_sahf(struct x86_emulate_ctxt
*ctxt
)
3843 flags
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
| X86_EFLAGS_ZF
|
3845 flags
&= *reg_rmw(ctxt
, VCPU_REGS_RAX
) >> 8;
3847 ctxt
->eflags
&= ~0xffUL
;
3848 ctxt
->eflags
|= flags
| X86_EFLAGS_FIXED
;
3849 return X86EMUL_CONTINUE
;
3852 static int em_lahf(struct x86_emulate_ctxt
*ctxt
)
3854 *reg_rmw(ctxt
, VCPU_REGS_RAX
) &= ~0xff00UL
;
3855 *reg_rmw(ctxt
, VCPU_REGS_RAX
) |= (ctxt
->eflags
& 0xff) << 8;
3856 return X86EMUL_CONTINUE
;
3859 static int em_bswap(struct x86_emulate_ctxt
*ctxt
)
3861 switch (ctxt
->op_bytes
) {
3862 #ifdef CONFIG_X86_64
3864 asm("bswap %0" : "+r"(ctxt
->dst
.val
));
3868 asm("bswap %0" : "+r"(*(u32
*)&ctxt
->dst
.val
));
3871 return X86EMUL_CONTINUE
;
3874 static int em_clflush(struct x86_emulate_ctxt
*ctxt
)
3876 /* emulating clflush regardless of cpuid */
3877 return X86EMUL_CONTINUE
;
3880 static int em_movsxd(struct x86_emulate_ctxt
*ctxt
)
3882 ctxt
->dst
.val
= (s32
) ctxt
->src
.val
;
3883 return X86EMUL_CONTINUE
;
3886 static bool valid_cr(int nr
)
3898 static int check_cr_read(struct x86_emulate_ctxt
*ctxt
)
3900 if (!valid_cr(ctxt
->modrm_reg
))
3901 return emulate_ud(ctxt
);
3903 return X86EMUL_CONTINUE
;
3906 static int check_cr_write(struct x86_emulate_ctxt
*ctxt
)
3908 u64 new_val
= ctxt
->src
.val64
;
3909 int cr
= ctxt
->modrm_reg
;
3912 static u64 cr_reserved_bits
[] = {
3913 0xffffffff00000000ULL
,
3914 0, 0, 0, /* CR3 checked later */
3921 return emulate_ud(ctxt
);
3923 if (new_val
& cr_reserved_bits
[cr
])
3924 return emulate_gp(ctxt
, 0);
3929 if (((new_val
& X86_CR0_PG
) && !(new_val
& X86_CR0_PE
)) ||
3930 ((new_val
& X86_CR0_NW
) && !(new_val
& X86_CR0_CD
)))
3931 return emulate_gp(ctxt
, 0);
3933 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3934 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3936 if ((new_val
& X86_CR0_PG
) && (efer
& EFER_LME
) &&
3937 !(cr4
& X86_CR4_PAE
))
3938 return emulate_gp(ctxt
, 0);
3945 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3946 if (efer
& EFER_LMA
)
3947 rsvd
= CR3_L_MODE_RESERVED_BITS
& ~CR3_PCID_INVD
;
3950 return emulate_gp(ctxt
, 0);
3955 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3957 if ((efer
& EFER_LMA
) && !(new_val
& X86_CR4_PAE
))
3958 return emulate_gp(ctxt
, 0);
3964 return X86EMUL_CONTINUE
;
3967 static int check_dr7_gd(struct x86_emulate_ctxt
*ctxt
)
3971 ctxt
->ops
->get_dr(ctxt
, 7, &dr7
);
3973 /* Check if DR7.Global_Enable is set */
3974 return dr7
& (1 << 13);
3977 static int check_dr_read(struct x86_emulate_ctxt
*ctxt
)
3979 int dr
= ctxt
->modrm_reg
;
3983 return emulate_ud(ctxt
);
3985 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3986 if ((cr4
& X86_CR4_DE
) && (dr
== 4 || dr
== 5))
3987 return emulate_ud(ctxt
);
3989 if (check_dr7_gd(ctxt
)) {
3992 ctxt
->ops
->get_dr(ctxt
, 6, &dr6
);
3994 dr6
|= DR6_BD
| DR6_RTM
;
3995 ctxt
->ops
->set_dr(ctxt
, 6, dr6
);
3996 return emulate_db(ctxt
);
3999 return X86EMUL_CONTINUE
;
4002 static int check_dr_write(struct x86_emulate_ctxt
*ctxt
)
4004 u64 new_val
= ctxt
->src
.val64
;
4005 int dr
= ctxt
->modrm_reg
;
4007 if ((dr
== 6 || dr
== 7) && (new_val
& 0xffffffff00000000ULL
))
4008 return emulate_gp(ctxt
, 0);
4010 return check_dr_read(ctxt
);
4013 static int check_svme(struct x86_emulate_ctxt
*ctxt
)
4017 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4019 if (!(efer
& EFER_SVME
))
4020 return emulate_ud(ctxt
);
4022 return X86EMUL_CONTINUE
;
4025 static int check_svme_pa(struct x86_emulate_ctxt
*ctxt
)
4027 u64 rax
= reg_read(ctxt
, VCPU_REGS_RAX
);
4029 /* Valid physical address? */
4030 if (rax
& 0xffff000000000000ULL
)
4031 return emulate_gp(ctxt
, 0);
4033 return check_svme(ctxt
);
4036 static int check_rdtsc(struct x86_emulate_ctxt
*ctxt
)
4038 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4040 if (cr4
& X86_CR4_TSD
&& ctxt
->ops
->cpl(ctxt
))
4041 return emulate_ud(ctxt
);
4043 return X86EMUL_CONTINUE
;
4046 static int check_rdpmc(struct x86_emulate_ctxt
*ctxt
)
4048 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4049 u64 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
4051 if ((!(cr4
& X86_CR4_PCE
) && ctxt
->ops
->cpl(ctxt
)) ||
4052 ctxt
->ops
->check_pmc(ctxt
, rcx
))
4053 return emulate_gp(ctxt
, 0);
4055 return X86EMUL_CONTINUE
;
4058 static int check_perm_in(struct x86_emulate_ctxt
*ctxt
)
4060 ctxt
->dst
.bytes
= min(ctxt
->dst
.bytes
, 4u);
4061 if (!emulator_io_permited(ctxt
, ctxt
->src
.val
, ctxt
->dst
.bytes
))
4062 return emulate_gp(ctxt
, 0);
4064 return X86EMUL_CONTINUE
;
4067 static int check_perm_out(struct x86_emulate_ctxt
*ctxt
)
4069 ctxt
->src
.bytes
= min(ctxt
->src
.bytes
, 4u);
4070 if (!emulator_io_permited(ctxt
, ctxt
->dst
.val
, ctxt
->src
.bytes
))
4071 return emulate_gp(ctxt
, 0);
4073 return X86EMUL_CONTINUE
;
4076 #define D(_y) { .flags = (_y) }
4077 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4078 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4079 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4080 #define N D(NotImpl)
4081 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4082 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4083 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4084 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4085 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4086 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4087 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4088 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4089 #define II(_f, _e, _i) \
4090 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4091 #define IIP(_f, _e, _i, _p) \
4092 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4093 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4094 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4096 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4097 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4098 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4099 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4100 #define I2bvIP(_f, _e, _i, _p) \
4101 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4103 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4104 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4105 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4107 static const struct opcode group7_rm0
[] = {
4109 I(SrcNone
| Priv
| EmulateOnUD
, em_hypercall
),
4113 static const struct opcode group7_rm1
[] = {
4114 DI(SrcNone
| Priv
, monitor
),
4115 DI(SrcNone
| Priv
, mwait
),
4119 static const struct opcode group7_rm3
[] = {
4120 DIP(SrcNone
| Prot
| Priv
, vmrun
, check_svme_pa
),
4121 II(SrcNone
| Prot
| EmulateOnUD
, em_hypercall
, vmmcall
),
4122 DIP(SrcNone
| Prot
| Priv
, vmload
, check_svme_pa
),
4123 DIP(SrcNone
| Prot
| Priv
, vmsave
, check_svme_pa
),
4124 DIP(SrcNone
| Prot
| Priv
, stgi
, check_svme
),
4125 DIP(SrcNone
| Prot
| Priv
, clgi
, check_svme
),
4126 DIP(SrcNone
| Prot
| Priv
, skinit
, check_svme
),
4127 DIP(SrcNone
| Prot
| Priv
, invlpga
, check_svme
),
4130 static const struct opcode group7_rm7
[] = {
4132 DIP(SrcNone
, rdtscp
, check_rdtsc
),
4136 static const struct opcode group1
[] = {
4138 F(Lock
| PageTable
, em_or
),
4141 F(Lock
| PageTable
, em_and
),
4147 static const struct opcode group1A
[] = {
4148 I(DstMem
| SrcNone
| Mov
| Stack
| IncSP
, em_pop
), N
, N
, N
, N
, N
, N
, N
,
4151 static const struct opcode group2
[] = {
4152 F(DstMem
| ModRM
, em_rol
),
4153 F(DstMem
| ModRM
, em_ror
),
4154 F(DstMem
| ModRM
, em_rcl
),
4155 F(DstMem
| ModRM
, em_rcr
),
4156 F(DstMem
| ModRM
, em_shl
),
4157 F(DstMem
| ModRM
, em_shr
),
4158 F(DstMem
| ModRM
, em_shl
),
4159 F(DstMem
| ModRM
, em_sar
),
4162 static const struct opcode group3
[] = {
4163 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4164 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4165 F(DstMem
| SrcNone
| Lock
, em_not
),
4166 F(DstMem
| SrcNone
| Lock
, em_neg
),
4167 F(DstXacc
| Src2Mem
, em_mul_ex
),
4168 F(DstXacc
| Src2Mem
, em_imul_ex
),
4169 F(DstXacc
| Src2Mem
, em_div_ex
),
4170 F(DstXacc
| Src2Mem
, em_idiv_ex
),
4173 static const struct opcode group4
[] = {
4174 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_inc
),
4175 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_dec
),
4179 static const struct opcode group5
[] = {
4180 F(DstMem
| SrcNone
| Lock
, em_inc
),
4181 F(DstMem
| SrcNone
| Lock
, em_dec
),
4182 I(SrcMem
| NearBranch
, em_call_near_abs
),
4183 I(SrcMemFAddr
| ImplicitOps
, em_call_far
),
4184 I(SrcMem
| NearBranch
, em_jmp_abs
),
4185 I(SrcMemFAddr
| ImplicitOps
, em_jmp_far
),
4186 I(SrcMem
| Stack
, em_push
), D(Undefined
),
4189 static const struct opcode group6
[] = {
4190 DI(Prot
| DstMem
, sldt
),
4191 DI(Prot
| DstMem
, str
),
4192 II(Prot
| Priv
| SrcMem16
, em_lldt
, lldt
),
4193 II(Prot
| Priv
| SrcMem16
, em_ltr
, ltr
),
4197 static const struct group_dual group7
= { {
4198 II(Mov
| DstMem
, em_sgdt
, sgdt
),
4199 II(Mov
| DstMem
, em_sidt
, sidt
),
4200 II(SrcMem
| Priv
, em_lgdt
, lgdt
),
4201 II(SrcMem
| Priv
, em_lidt
, lidt
),
4202 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4203 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4204 II(SrcMem
| ByteOp
| Priv
| NoAccess
, em_invlpg
, invlpg
),
4208 N
, EXT(0, group7_rm3
),
4209 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4210 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4214 static const struct opcode group8
[] = {
4216 F(DstMem
| SrcImmByte
| NoWrite
, em_bt
),
4217 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_bts
),
4218 F(DstMem
| SrcImmByte
| Lock
, em_btr
),
4219 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_btc
),
4222 static const struct group_dual group9
= { {
4223 N
, I(DstMem64
| Lock
| PageTable
, em_cmpxchg8b
), N
, N
, N
, N
, N
, N
,
4225 N
, N
, N
, N
, N
, N
, N
, N
,
4228 static const struct opcode group11
[] = {
4229 I(DstMem
| SrcImm
| Mov
| PageTable
, em_mov
),
4233 static const struct gprefix pfx_0f_ae_7
= {
4234 I(SrcMem
| ByteOp
, em_clflush
), N
, N
, N
,
4237 static const struct group_dual group15
= { {
4238 N
, N
, N
, N
, N
, N
, N
, GP(0, &pfx_0f_ae_7
),
4240 N
, N
, N
, N
, N
, N
, N
, N
,
4243 static const struct gprefix pfx_0f_6f_0f_7f
= {
4244 I(Mmx
, em_mov
), I(Sse
| Aligned
, em_mov
), N
, I(Sse
| Unaligned
, em_mov
),
4247 static const struct instr_dual instr_dual_0f_2b
= {
4251 static const struct gprefix pfx_0f_2b
= {
4252 ID(0, &instr_dual_0f_2b
), ID(0, &instr_dual_0f_2b
), N
, N
,
4255 static const struct gprefix pfx_0f_28_0f_29
= {
4256 I(Aligned
, em_mov
), I(Aligned
, em_mov
), N
, N
,
4259 static const struct gprefix pfx_0f_e7
= {
4260 N
, I(Sse
, em_mov
), N
, N
,
4263 static const struct escape escape_d9
= { {
4264 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstcw
),
4267 N
, N
, N
, N
, N
, N
, N
, N
,
4269 N
, N
, N
, N
, N
, N
, N
, N
,
4271 N
, N
, N
, N
, N
, N
, N
, N
,
4273 N
, N
, N
, N
, N
, N
, N
, N
,
4275 N
, N
, N
, N
, N
, N
, N
, N
,
4277 N
, N
, N
, N
, N
, N
, N
, N
,
4279 N
, N
, N
, N
, N
, N
, N
, N
,
4281 N
, N
, N
, N
, N
, N
, N
, N
,
4284 static const struct escape escape_db
= { {
4285 N
, N
, N
, N
, N
, N
, N
, N
,
4288 N
, N
, N
, N
, N
, N
, N
, N
,
4290 N
, N
, N
, N
, N
, N
, N
, N
,
4292 N
, N
, N
, N
, N
, N
, N
, N
,
4294 N
, N
, N
, N
, N
, N
, N
, N
,
4296 N
, N
, N
, I(ImplicitOps
, em_fninit
), N
, N
, N
, N
,
4298 N
, N
, N
, N
, N
, N
, N
, N
,
4300 N
, N
, N
, N
, N
, N
, N
, N
,
4302 N
, N
, N
, N
, N
, N
, N
, N
,
4305 static const struct escape escape_dd
= { {
4306 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstsw
),
4309 N
, N
, N
, N
, N
, N
, N
, N
,
4311 N
, N
, N
, N
, N
, N
, N
, N
,
4313 N
, N
, N
, N
, N
, N
, N
, N
,
4315 N
, N
, N
, N
, N
, N
, N
, N
,
4317 N
, N
, N
, N
, N
, N
, N
, N
,
4319 N
, N
, N
, N
, N
, N
, N
, N
,
4321 N
, N
, N
, N
, N
, N
, N
, N
,
4323 N
, N
, N
, N
, N
, N
, N
, N
,
4326 static const struct instr_dual instr_dual_0f_c3
= {
4327 I(DstMem
| SrcReg
| ModRM
| No16
| Mov
, em_mov
), N
4330 static const struct mode_dual mode_dual_63
= {
4331 N
, I(DstReg
| SrcMem32
| ModRM
| Mov
, em_movsxd
)
4334 static const struct opcode opcode_table
[256] = {
4336 F6ALU(Lock
, em_add
),
4337 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_push_sreg
),
4338 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_pop_sreg
),
4340 F6ALU(Lock
| PageTable
, em_or
),
4341 I(ImplicitOps
| Stack
| No64
| Src2CS
, em_push_sreg
),
4344 F6ALU(Lock
, em_adc
),
4345 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_push_sreg
),
4346 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_pop_sreg
),
4348 F6ALU(Lock
, em_sbb
),
4349 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_push_sreg
),
4350 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_pop_sreg
),
4352 F6ALU(Lock
| PageTable
, em_and
), N
, N
,
4354 F6ALU(Lock
, em_sub
), N
, I(ByteOp
| DstAcc
| No64
, em_das
),
4356 F6ALU(Lock
, em_xor
), N
, N
,
4358 F6ALU(NoWrite
, em_cmp
), N
, N
,
4360 X8(F(DstReg
, em_inc
)), X8(F(DstReg
, em_dec
)),
4362 X8(I(SrcReg
| Stack
, em_push
)),
4364 X8(I(DstReg
| Stack
, em_pop
)),
4366 I(ImplicitOps
| Stack
| No64
, em_pusha
),
4367 I(ImplicitOps
| Stack
| No64
, em_popa
),
4368 N
, MD(ModRM
, &mode_dual_63
),
4371 I(SrcImm
| Mov
| Stack
, em_push
),
4372 I(DstReg
| SrcMem
| ModRM
| Src2Imm
, em_imul_3op
),
4373 I(SrcImmByte
| Mov
| Stack
, em_push
),
4374 I(DstReg
| SrcMem
| ModRM
| Src2ImmByte
, em_imul_3op
),
4375 I2bvIP(DstDI
| SrcDX
| Mov
| String
| Unaligned
, em_in
, ins
, check_perm_in
), /* insb, insw/insd */
4376 I2bvIP(SrcSI
| DstDX
| String
, em_out
, outs
, check_perm_out
), /* outsb, outsw/outsd */
4378 X16(D(SrcImmByte
| NearBranch
)),
4380 G(ByteOp
| DstMem
| SrcImm
, group1
),
4381 G(DstMem
| SrcImm
, group1
),
4382 G(ByteOp
| DstMem
| SrcImm
| No64
, group1
),
4383 G(DstMem
| SrcImmByte
, group1
),
4384 F2bv(DstMem
| SrcReg
| ModRM
| NoWrite
, em_test
),
4385 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
, em_xchg
),
4387 I2bv(DstMem
| SrcReg
| ModRM
| Mov
| PageTable
, em_mov
),
4388 I2bv(DstReg
| SrcMem
| ModRM
| Mov
, em_mov
),
4389 I(DstMem
| SrcNone
| ModRM
| Mov
| PageTable
, em_mov_rm_sreg
),
4390 D(ModRM
| SrcMem
| NoAccess
| DstReg
),
4391 I(ImplicitOps
| SrcMem16
| ModRM
, em_mov_sreg_rm
),
4394 DI(SrcAcc
| DstReg
, pause
), X7(D(SrcAcc
| DstReg
)),
4396 D(DstAcc
| SrcNone
), I(ImplicitOps
| SrcAcc
, em_cwd
),
4397 I(SrcImmFAddr
| No64
, em_call_far
), N
,
4398 II(ImplicitOps
| Stack
, em_pushf
, pushf
),
4399 II(ImplicitOps
| Stack
, em_popf
, popf
),
4400 I(ImplicitOps
, em_sahf
), I(ImplicitOps
, em_lahf
),
4402 I2bv(DstAcc
| SrcMem
| Mov
| MemAbs
, em_mov
),
4403 I2bv(DstMem
| SrcAcc
| Mov
| MemAbs
| PageTable
, em_mov
),
4404 I2bv(SrcSI
| DstDI
| Mov
| String
, em_mov
),
4405 F2bv(SrcSI
| DstDI
| String
| NoWrite
, em_cmp_r
),
4407 F2bv(DstAcc
| SrcImm
| NoWrite
, em_test
),
4408 I2bv(SrcAcc
| DstDI
| Mov
| String
, em_mov
),
4409 I2bv(SrcSI
| DstAcc
| Mov
| String
, em_mov
),
4410 F2bv(SrcAcc
| DstDI
| String
| NoWrite
, em_cmp_r
),
4412 X8(I(ByteOp
| DstReg
| SrcImm
| Mov
, em_mov
)),
4414 X8(I(DstReg
| SrcImm64
| Mov
, em_mov
)),
4416 G(ByteOp
| Src2ImmByte
, group2
), G(Src2ImmByte
, group2
),
4417 I(ImplicitOps
| NearBranch
| SrcImmU16
, em_ret_near_imm
),
4418 I(ImplicitOps
| NearBranch
, em_ret
),
4419 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2ES
, em_lseg
),
4420 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2DS
, em_lseg
),
4421 G(ByteOp
, group11
), G(0, group11
),
4423 I(Stack
| SrcImmU16
| Src2ImmByte
, em_enter
), I(Stack
, em_leave
),
4424 I(ImplicitOps
| SrcImmU16
, em_ret_far_imm
),
4425 I(ImplicitOps
, em_ret_far
),
4426 D(ImplicitOps
), DI(SrcImmByte
, intn
),
4427 D(ImplicitOps
| No64
), II(ImplicitOps
, em_iret
, iret
),
4429 G(Src2One
| ByteOp
, group2
), G(Src2One
, group2
),
4430 G(Src2CL
| ByteOp
, group2
), G(Src2CL
, group2
),
4431 I(DstAcc
| SrcImmUByte
| No64
, em_aam
),
4432 I(DstAcc
| SrcImmUByte
| No64
, em_aad
),
4433 F(DstAcc
| ByteOp
| No64
, em_salc
),
4434 I(DstAcc
| SrcXLat
| ByteOp
, em_mov
),
4436 N
, E(0, &escape_d9
), N
, E(0, &escape_db
), N
, E(0, &escape_dd
), N
, N
,
4438 X3(I(SrcImmByte
| NearBranch
, em_loop
)),
4439 I(SrcImmByte
| NearBranch
, em_jcxz
),
4440 I2bvIP(SrcImmUByte
| DstAcc
, em_in
, in
, check_perm_in
),
4441 I2bvIP(SrcAcc
| DstImmUByte
, em_out
, out
, check_perm_out
),
4443 I(SrcImm
| NearBranch
, em_call
), D(SrcImm
| ImplicitOps
| NearBranch
),
4444 I(SrcImmFAddr
| No64
, em_jmp_far
),
4445 D(SrcImmByte
| ImplicitOps
| NearBranch
),
4446 I2bvIP(SrcDX
| DstAcc
, em_in
, in
, check_perm_in
),
4447 I2bvIP(SrcAcc
| DstDX
, em_out
, out
, check_perm_out
),
4449 N
, DI(ImplicitOps
, icebp
), N
, N
,
4450 DI(ImplicitOps
| Priv
, hlt
), D(ImplicitOps
),
4451 G(ByteOp
, group3
), G(0, group3
),
4453 D(ImplicitOps
), D(ImplicitOps
),
4454 I(ImplicitOps
, em_cli
), I(ImplicitOps
, em_sti
),
4455 D(ImplicitOps
), D(ImplicitOps
), G(0, group4
), G(0, group5
),
4458 static const struct opcode twobyte_table
[256] = {
4460 G(0, group6
), GD(0, &group7
), N
, N
,
4461 N
, I(ImplicitOps
| EmulateOnUD
, em_syscall
),
4462 II(ImplicitOps
| Priv
, em_clts
, clts
), N
,
4463 DI(ImplicitOps
| Priv
, invd
), DI(ImplicitOps
| Priv
, wbinvd
), N
, N
,
4464 N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
), N
, N
,
4466 N
, N
, N
, N
, N
, N
, N
, N
,
4467 D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4468 N
, N
, N
, N
, N
, N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4470 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, cr_read
, check_cr_read
),
4471 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, dr_read
, check_dr_read
),
4472 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_cr_write
, cr_write
,
4474 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_dr_write
, dr_write
,
4477 GP(ModRM
| DstReg
| SrcMem
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4478 GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4479 N
, GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_2b
),
4482 II(ImplicitOps
| Priv
, em_wrmsr
, wrmsr
),
4483 IIP(ImplicitOps
, em_rdtsc
, rdtsc
, check_rdtsc
),
4484 II(ImplicitOps
| Priv
, em_rdmsr
, rdmsr
),
4485 IIP(ImplicitOps
, em_rdpmc
, rdpmc
, check_rdpmc
),
4486 I(ImplicitOps
| EmulateOnUD
, em_sysenter
),
4487 I(ImplicitOps
| Priv
| EmulateOnUD
, em_sysexit
),
4489 N
, N
, N
, N
, N
, N
, N
, N
,
4491 X16(D(DstReg
| SrcMem
| ModRM
)),
4493 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4498 N
, N
, N
, GP(SrcMem
| DstReg
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4503 N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4505 X16(D(SrcImm
| NearBranch
)),
4507 X16(D(ByteOp
| DstMem
| SrcNone
| ModRM
| Mov
)),
4509 I(Stack
| Src2FS
, em_push_sreg
), I(Stack
| Src2FS
, em_pop_sreg
),
4510 II(ImplicitOps
, em_cpuid
, cpuid
),
4511 F(DstMem
| SrcReg
| ModRM
| BitOp
| NoWrite
, em_bt
),
4512 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shld
),
4513 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shld
), N
, N
,
4515 I(Stack
| Src2GS
, em_push_sreg
), I(Stack
| Src2GS
, em_pop_sreg
),
4516 II(EmulateOnUD
| ImplicitOps
, em_rsm
, rsm
),
4517 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_bts
),
4518 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shrd
),
4519 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shrd
),
4520 GD(0, &group15
), F(DstReg
| SrcMem
| ModRM
, em_imul
),
4522 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
| SrcWrite
, em_cmpxchg
),
4523 I(DstReg
| SrcMemFAddr
| ModRM
| Src2SS
, em_lseg
),
4524 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
, em_btr
),
4525 I(DstReg
| SrcMemFAddr
| ModRM
| Src2FS
, em_lseg
),
4526 I(DstReg
| SrcMemFAddr
| ModRM
| Src2GS
, em_lseg
),
4527 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4531 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_btc
),
4532 I(DstReg
| SrcMem
| ModRM
, em_bsf_c
),
4533 I(DstReg
| SrcMem
| ModRM
, em_bsr_c
),
4534 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4536 F2bv(DstMem
| SrcReg
| ModRM
| SrcWrite
| Lock
, em_xadd
),
4537 N
, ID(0, &instr_dual_0f_c3
),
4538 N
, N
, N
, GD(0, &group9
),
4540 X8(I(DstReg
, em_bswap
)),
4542 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4544 N
, N
, N
, N
, N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_e7
),
4545 N
, N
, N
, N
, N
, N
, N
, N
,
4547 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
4550 static const struct instr_dual instr_dual_0f_38_f0
= {
4551 I(DstReg
| SrcMem
| Mov
, em_movbe
), N
4554 static const struct instr_dual instr_dual_0f_38_f1
= {
4555 I(DstMem
| SrcReg
| Mov
, em_movbe
), N
4558 static const struct gprefix three_byte_0f_38_f0
= {
4559 ID(0, &instr_dual_0f_38_f0
), N
, N
, N
4562 static const struct gprefix three_byte_0f_38_f1
= {
4563 ID(0, &instr_dual_0f_38_f1
), N
, N
, N
4567 * Insns below are selected by the prefix which indexed by the third opcode
4570 static const struct opcode opcode_map_0f_38
[256] = {
4572 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4574 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4576 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f0
),
4577 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f1
),
4598 static unsigned imm_size(struct x86_emulate_ctxt
*ctxt
)
4602 size
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4608 static int decode_imm(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4609 unsigned size
, bool sign_extension
)
4611 int rc
= X86EMUL_CONTINUE
;
4615 op
->addr
.mem
.ea
= ctxt
->_eip
;
4616 /* NB. Immediates are sign-extended as necessary. */
4617 switch (op
->bytes
) {
4619 op
->val
= insn_fetch(s8
, ctxt
);
4622 op
->val
= insn_fetch(s16
, ctxt
);
4625 op
->val
= insn_fetch(s32
, ctxt
);
4628 op
->val
= insn_fetch(s64
, ctxt
);
4631 if (!sign_extension
) {
4632 switch (op
->bytes
) {
4640 op
->val
&= 0xffffffff;
4648 static int decode_operand(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4651 int rc
= X86EMUL_CONTINUE
;
4655 decode_register_operand(ctxt
, op
);
4658 rc
= decode_imm(ctxt
, op
, 1, false);
4661 ctxt
->memop
.bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4665 if (ctxt
->d
& BitOp
)
4666 fetch_bit_operand(ctxt
);
4667 op
->orig_val
= op
->val
;
4670 ctxt
->memop
.bytes
= (ctxt
->op_bytes
== 8) ? 16 : 8;
4674 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4675 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4676 fetch_register_operand(op
);
4677 op
->orig_val
= op
->val
;
4681 op
->bytes
= (ctxt
->d
& ByteOp
) ? 2 : ctxt
->op_bytes
;
4682 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4683 fetch_register_operand(op
);
4684 op
->orig_val
= op
->val
;
4687 if (ctxt
->d
& ByteOp
) {
4692 op
->bytes
= ctxt
->op_bytes
;
4693 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4694 fetch_register_operand(op
);
4695 op
->orig_val
= op
->val
;
4699 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4701 register_address(ctxt
, VCPU_REGS_RDI
);
4702 op
->addr
.mem
.seg
= VCPU_SREG_ES
;
4709 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4710 fetch_register_operand(op
);
4715 op
->val
= reg_read(ctxt
, VCPU_REGS_RCX
) & 0xff;
4718 rc
= decode_imm(ctxt
, op
, 1, true);
4726 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), true);
4729 rc
= decode_imm(ctxt
, op
, ctxt
->op_bytes
, true);
4732 ctxt
->memop
.bytes
= 1;
4733 if (ctxt
->memop
.type
== OP_REG
) {
4734 ctxt
->memop
.addr
.reg
= decode_register(ctxt
,
4735 ctxt
->modrm_rm
, true);
4736 fetch_register_operand(&ctxt
->memop
);
4740 ctxt
->memop
.bytes
= 2;
4743 ctxt
->memop
.bytes
= 4;
4746 rc
= decode_imm(ctxt
, op
, 2, false);
4749 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), false);
4753 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4755 register_address(ctxt
, VCPU_REGS_RSI
);
4756 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4762 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4765 reg_read(ctxt
, VCPU_REGS_RBX
) +
4766 (reg_read(ctxt
, VCPU_REGS_RAX
) & 0xff));
4767 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4772 op
->addr
.mem
.ea
= ctxt
->_eip
;
4773 op
->bytes
= ctxt
->op_bytes
+ 2;
4774 insn_fetch_arr(op
->valptr
, op
->bytes
, ctxt
);
4777 ctxt
->memop
.bytes
= ctxt
->op_bytes
+ 2;
4781 op
->val
= VCPU_SREG_ES
;
4785 op
->val
= VCPU_SREG_CS
;
4789 op
->val
= VCPU_SREG_SS
;
4793 op
->val
= VCPU_SREG_DS
;
4797 op
->val
= VCPU_SREG_FS
;
4801 op
->val
= VCPU_SREG_GS
;
4804 /* Special instructions do their own operand decoding. */
4806 op
->type
= OP_NONE
; /* Disable writeback. */
4814 int x86_decode_insn(struct x86_emulate_ctxt
*ctxt
, void *insn
, int insn_len
)
4816 int rc
= X86EMUL_CONTINUE
;
4817 int mode
= ctxt
->mode
;
4818 int def_op_bytes
, def_ad_bytes
, goffset
, simd_prefix
;
4819 bool op_prefix
= false;
4820 bool has_seg_override
= false;
4821 struct opcode opcode
;
4823 ctxt
->memop
.type
= OP_NONE
;
4824 ctxt
->memopp
= NULL
;
4825 ctxt
->_eip
= ctxt
->eip
;
4826 ctxt
->fetch
.ptr
= ctxt
->fetch
.data
;
4827 ctxt
->fetch
.end
= ctxt
->fetch
.data
+ insn_len
;
4828 ctxt
->opcode_len
= 1;
4830 memcpy(ctxt
->fetch
.data
, insn
, insn_len
);
4832 rc
= __do_insn_fetch_bytes(ctxt
, 1);
4833 if (rc
!= X86EMUL_CONTINUE
)
4838 case X86EMUL_MODE_REAL
:
4839 case X86EMUL_MODE_VM86
:
4840 case X86EMUL_MODE_PROT16
:
4841 def_op_bytes
= def_ad_bytes
= 2;
4843 case X86EMUL_MODE_PROT32
:
4844 def_op_bytes
= def_ad_bytes
= 4;
4846 #ifdef CONFIG_X86_64
4847 case X86EMUL_MODE_PROT64
:
4853 return EMULATION_FAILED
;
4856 ctxt
->op_bytes
= def_op_bytes
;
4857 ctxt
->ad_bytes
= def_ad_bytes
;
4859 /* Legacy prefixes. */
4861 switch (ctxt
->b
= insn_fetch(u8
, ctxt
)) {
4862 case 0x66: /* operand-size override */
4864 /* switch between 2/4 bytes */
4865 ctxt
->op_bytes
= def_op_bytes
^ 6;
4867 case 0x67: /* address-size override */
4868 if (mode
== X86EMUL_MODE_PROT64
)
4869 /* switch between 4/8 bytes */
4870 ctxt
->ad_bytes
= def_ad_bytes
^ 12;
4872 /* switch between 2/4 bytes */
4873 ctxt
->ad_bytes
= def_ad_bytes
^ 6;
4875 case 0x26: /* ES override */
4876 case 0x2e: /* CS override */
4877 case 0x36: /* SS override */
4878 case 0x3e: /* DS override */
4879 has_seg_override
= true;
4880 ctxt
->seg_override
= (ctxt
->b
>> 3) & 3;
4882 case 0x64: /* FS override */
4883 case 0x65: /* GS override */
4884 has_seg_override
= true;
4885 ctxt
->seg_override
= ctxt
->b
& 7;
4887 case 0x40 ... 0x4f: /* REX */
4888 if (mode
!= X86EMUL_MODE_PROT64
)
4890 ctxt
->rex_prefix
= ctxt
->b
;
4892 case 0xf0: /* LOCK */
4893 ctxt
->lock_prefix
= 1;
4895 case 0xf2: /* REPNE/REPNZ */
4896 case 0xf3: /* REP/REPE/REPZ */
4897 ctxt
->rep_prefix
= ctxt
->b
;
4903 /* Any legacy prefix after a REX prefix nullifies its effect. */
4905 ctxt
->rex_prefix
= 0;
4911 if (ctxt
->rex_prefix
& 8)
4912 ctxt
->op_bytes
= 8; /* REX.W */
4914 /* Opcode byte(s). */
4915 opcode
= opcode_table
[ctxt
->b
];
4916 /* Two-byte opcode? */
4917 if (ctxt
->b
== 0x0f) {
4918 ctxt
->opcode_len
= 2;
4919 ctxt
->b
= insn_fetch(u8
, ctxt
);
4920 opcode
= twobyte_table
[ctxt
->b
];
4922 /* 0F_38 opcode map */
4923 if (ctxt
->b
== 0x38) {
4924 ctxt
->opcode_len
= 3;
4925 ctxt
->b
= insn_fetch(u8
, ctxt
);
4926 opcode
= opcode_map_0f_38
[ctxt
->b
];
4929 ctxt
->d
= opcode
.flags
;
4931 if (ctxt
->d
& ModRM
)
4932 ctxt
->modrm
= insn_fetch(u8
, ctxt
);
4934 /* vex-prefix instructions are not implemented */
4935 if (ctxt
->opcode_len
== 1 && (ctxt
->b
== 0xc5 || ctxt
->b
== 0xc4) &&
4936 (mode
== X86EMUL_MODE_PROT64
|| (ctxt
->modrm
& 0xc0) == 0xc0)) {
4940 while (ctxt
->d
& GroupMask
) {
4941 switch (ctxt
->d
& GroupMask
) {
4943 goffset
= (ctxt
->modrm
>> 3) & 7;
4944 opcode
= opcode
.u
.group
[goffset
];
4947 goffset
= (ctxt
->modrm
>> 3) & 7;
4948 if ((ctxt
->modrm
>> 6) == 3)
4949 opcode
= opcode
.u
.gdual
->mod3
[goffset
];
4951 opcode
= opcode
.u
.gdual
->mod012
[goffset
];
4954 goffset
= ctxt
->modrm
& 7;
4955 opcode
= opcode
.u
.group
[goffset
];
4958 if (ctxt
->rep_prefix
&& op_prefix
)
4959 return EMULATION_FAILED
;
4960 simd_prefix
= op_prefix
? 0x66 : ctxt
->rep_prefix
;
4961 switch (simd_prefix
) {
4962 case 0x00: opcode
= opcode
.u
.gprefix
->pfx_no
; break;
4963 case 0x66: opcode
= opcode
.u
.gprefix
->pfx_66
; break;
4964 case 0xf2: opcode
= opcode
.u
.gprefix
->pfx_f2
; break;
4965 case 0xf3: opcode
= opcode
.u
.gprefix
->pfx_f3
; break;
4969 if (ctxt
->modrm
> 0xbf)
4970 opcode
= opcode
.u
.esc
->high
[ctxt
->modrm
- 0xc0];
4972 opcode
= opcode
.u
.esc
->op
[(ctxt
->modrm
>> 3) & 7];
4975 if ((ctxt
->modrm
>> 6) == 3)
4976 opcode
= opcode
.u
.idual
->mod3
;
4978 opcode
= opcode
.u
.idual
->mod012
;
4981 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
4982 opcode
= opcode
.u
.mdual
->mode64
;
4984 opcode
= opcode
.u
.mdual
->mode32
;
4987 return EMULATION_FAILED
;
4990 ctxt
->d
&= ~(u64
)GroupMask
;
4991 ctxt
->d
|= opcode
.flags
;
4996 return EMULATION_FAILED
;
4998 ctxt
->execute
= opcode
.u
.execute
;
5000 if (unlikely(ctxt
->ud
) && likely(!(ctxt
->d
& EmulateOnUD
)))
5001 return EMULATION_FAILED
;
5003 if (unlikely(ctxt
->d
&
5004 (NotImpl
|Stack
|Op3264
|Sse
|Mmx
|Intercept
|CheckPerm
|NearBranch
|
5007 * These are copied unconditionally here, and checked unconditionally
5008 * in x86_emulate_insn.
5010 ctxt
->check_perm
= opcode
.check_perm
;
5011 ctxt
->intercept
= opcode
.intercept
;
5013 if (ctxt
->d
& NotImpl
)
5014 return EMULATION_FAILED
;
5016 if (mode
== X86EMUL_MODE_PROT64
) {
5017 if (ctxt
->op_bytes
== 4 && (ctxt
->d
& Stack
))
5019 else if (ctxt
->d
& NearBranch
)
5023 if (ctxt
->d
& Op3264
) {
5024 if (mode
== X86EMUL_MODE_PROT64
)
5030 if ((ctxt
->d
& No16
) && ctxt
->op_bytes
== 2)
5034 ctxt
->op_bytes
= 16;
5035 else if (ctxt
->d
& Mmx
)
5039 /* ModRM and SIB bytes. */
5040 if (ctxt
->d
& ModRM
) {
5041 rc
= decode_modrm(ctxt
, &ctxt
->memop
);
5042 if (!has_seg_override
) {
5043 has_seg_override
= true;
5044 ctxt
->seg_override
= ctxt
->modrm_seg
;
5046 } else if (ctxt
->d
& MemAbs
)
5047 rc
= decode_abs(ctxt
, &ctxt
->memop
);
5048 if (rc
!= X86EMUL_CONTINUE
)
5051 if (!has_seg_override
)
5052 ctxt
->seg_override
= VCPU_SREG_DS
;
5054 ctxt
->memop
.addr
.mem
.seg
= ctxt
->seg_override
;
5057 * Decode and fetch the source operand: register, memory
5060 rc
= decode_operand(ctxt
, &ctxt
->src
, (ctxt
->d
>> SrcShift
) & OpMask
);
5061 if (rc
!= X86EMUL_CONTINUE
)
5065 * Decode and fetch the second source operand: register, memory
5068 rc
= decode_operand(ctxt
, &ctxt
->src2
, (ctxt
->d
>> Src2Shift
) & OpMask
);
5069 if (rc
!= X86EMUL_CONTINUE
)
5072 /* Decode and fetch the destination operand: register or memory. */
5073 rc
= decode_operand(ctxt
, &ctxt
->dst
, (ctxt
->d
>> DstShift
) & OpMask
);
5075 if (ctxt
->rip_relative
&& likely(ctxt
->memopp
))
5076 ctxt
->memopp
->addr
.mem
.ea
= address_mask(ctxt
,
5077 ctxt
->memopp
->addr
.mem
.ea
+ ctxt
->_eip
);
5080 return (rc
!= X86EMUL_CONTINUE
) ? EMULATION_FAILED
: EMULATION_OK
;
5083 bool x86_page_table_writing_insn(struct x86_emulate_ctxt
*ctxt
)
5085 return ctxt
->d
& PageTable
;
5088 static bool string_insn_completed(struct x86_emulate_ctxt
*ctxt
)
5090 /* The second termination condition only applies for REPE
5091 * and REPNE. Test if the repeat string operation prefix is
5092 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5093 * corresponding termination condition according to:
5094 * - if REPE/REPZ and ZF = 0 then done
5095 * - if REPNE/REPNZ and ZF = 1 then done
5097 if (((ctxt
->b
== 0xa6) || (ctxt
->b
== 0xa7) ||
5098 (ctxt
->b
== 0xae) || (ctxt
->b
== 0xaf))
5099 && (((ctxt
->rep_prefix
== REPE_PREFIX
) &&
5100 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == 0))
5101 || ((ctxt
->rep_prefix
== REPNE_PREFIX
) &&
5102 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == X86_EFLAGS_ZF
))))
5108 static int flush_pending_x87_faults(struct x86_emulate_ctxt
*ctxt
)
5112 ctxt
->ops
->get_fpu(ctxt
);
5113 rc
= asm_safe("fwait");
5114 ctxt
->ops
->put_fpu(ctxt
);
5116 if (unlikely(rc
!= X86EMUL_CONTINUE
))
5117 return emulate_exception(ctxt
, MF_VECTOR
, 0, false);
5119 return X86EMUL_CONTINUE
;
5122 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt
*ctxt
,
5125 if (op
->type
== OP_MM
)
5126 read_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
5129 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*))
5131 register void *__sp
asm(_ASM_SP
);
5132 ulong flags
= (ctxt
->eflags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
5134 if (!(ctxt
->d
& ByteOp
))
5135 fop
+= __ffs(ctxt
->dst
.bytes
) * FASTOP_SIZE
;
5137 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5138 : "+a"(ctxt
->dst
.val
), "+d"(ctxt
->src
.val
), [flags
]"+D"(flags
),
5139 [fastop
]"+S"(fop
), "+r"(__sp
)
5140 : "c"(ctxt
->src2
.val
));
5142 ctxt
->eflags
= (ctxt
->eflags
& ~EFLAGS_MASK
) | (flags
& EFLAGS_MASK
);
5143 if (!fop
) /* exception is returned in fop variable */
5144 return emulate_de(ctxt
);
5145 return X86EMUL_CONTINUE
;
5148 void init_decode_cache(struct x86_emulate_ctxt
*ctxt
)
5150 memset(&ctxt
->rip_relative
, 0,
5151 (void *)&ctxt
->modrm
- (void *)&ctxt
->rip_relative
);
5153 ctxt
->io_read
.pos
= 0;
5154 ctxt
->io_read
.end
= 0;
5155 ctxt
->mem_read
.end
= 0;
5158 int x86_emulate_insn(struct x86_emulate_ctxt
*ctxt
)
5160 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
5161 int rc
= X86EMUL_CONTINUE
;
5162 int saved_dst_type
= ctxt
->dst
.type
;
5164 ctxt
->mem_read
.pos
= 0;
5166 /* LOCK prefix is allowed only with some instructions */
5167 if (ctxt
->lock_prefix
&& (!(ctxt
->d
& Lock
) || ctxt
->dst
.type
!= OP_MEM
)) {
5168 rc
= emulate_ud(ctxt
);
5172 if ((ctxt
->d
& SrcMask
) == SrcMemFAddr
&& ctxt
->src
.type
!= OP_MEM
) {
5173 rc
= emulate_ud(ctxt
);
5177 if (unlikely(ctxt
->d
&
5178 (No64
|Undefined
|Sse
|Mmx
|Intercept
|CheckPerm
|Priv
|Prot
|String
))) {
5179 if ((ctxt
->mode
== X86EMUL_MODE_PROT64
&& (ctxt
->d
& No64
)) ||
5180 (ctxt
->d
& Undefined
)) {
5181 rc
= emulate_ud(ctxt
);
5185 if (((ctxt
->d
& (Sse
|Mmx
)) && ((ops
->get_cr(ctxt
, 0) & X86_CR0_EM
)))
5186 || ((ctxt
->d
& Sse
) && !(ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
))) {
5187 rc
= emulate_ud(ctxt
);
5191 if ((ctxt
->d
& (Sse
|Mmx
)) && (ops
->get_cr(ctxt
, 0) & X86_CR0_TS
)) {
5192 rc
= emulate_nm(ctxt
);
5196 if (ctxt
->d
& Mmx
) {
5197 rc
= flush_pending_x87_faults(ctxt
);
5198 if (rc
!= X86EMUL_CONTINUE
)
5201 * Now that we know the fpu is exception safe, we can fetch
5204 fetch_possible_mmx_operand(ctxt
, &ctxt
->src
);
5205 fetch_possible_mmx_operand(ctxt
, &ctxt
->src2
);
5206 if (!(ctxt
->d
& Mov
))
5207 fetch_possible_mmx_operand(ctxt
, &ctxt
->dst
);
5210 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && ctxt
->intercept
) {
5211 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5212 X86_ICPT_PRE_EXCEPT
);
5213 if (rc
!= X86EMUL_CONTINUE
)
5217 /* Instruction can only be executed in protected mode */
5218 if ((ctxt
->d
& Prot
) && ctxt
->mode
< X86EMUL_MODE_PROT16
) {
5219 rc
= emulate_ud(ctxt
);
5223 /* Privileged instruction can be executed only in CPL=0 */
5224 if ((ctxt
->d
& Priv
) && ops
->cpl(ctxt
)) {
5225 if (ctxt
->d
& PrivUD
)
5226 rc
= emulate_ud(ctxt
);
5228 rc
= emulate_gp(ctxt
, 0);
5232 /* Do instruction specific permission checks */
5233 if (ctxt
->d
& CheckPerm
) {
5234 rc
= ctxt
->check_perm(ctxt
);
5235 if (rc
!= X86EMUL_CONTINUE
)
5239 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5240 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5241 X86_ICPT_POST_EXCEPT
);
5242 if (rc
!= X86EMUL_CONTINUE
)
5246 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5247 /* All REP prefixes have the same first termination condition */
5248 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0) {
5249 string_registers_quirk(ctxt
);
5250 ctxt
->eip
= ctxt
->_eip
;
5251 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5257 if ((ctxt
->src
.type
== OP_MEM
) && !(ctxt
->d
& NoAccess
)) {
5258 rc
= segmented_read(ctxt
, ctxt
->src
.addr
.mem
,
5259 ctxt
->src
.valptr
, ctxt
->src
.bytes
);
5260 if (rc
!= X86EMUL_CONTINUE
)
5262 ctxt
->src
.orig_val64
= ctxt
->src
.val64
;
5265 if (ctxt
->src2
.type
== OP_MEM
) {
5266 rc
= segmented_read(ctxt
, ctxt
->src2
.addr
.mem
,
5267 &ctxt
->src2
.val
, ctxt
->src2
.bytes
);
5268 if (rc
!= X86EMUL_CONTINUE
)
5272 if ((ctxt
->d
& DstMask
) == ImplicitOps
)
5276 if ((ctxt
->dst
.type
== OP_MEM
) && !(ctxt
->d
& Mov
)) {
5277 /* optimisation - avoid slow emulated read if Mov */
5278 rc
= segmented_read(ctxt
, ctxt
->dst
.addr
.mem
,
5279 &ctxt
->dst
.val
, ctxt
->dst
.bytes
);
5280 if (rc
!= X86EMUL_CONTINUE
) {
5281 if (!(ctxt
->d
& NoWrite
) &&
5282 rc
== X86EMUL_PROPAGATE_FAULT
&&
5283 ctxt
->exception
.vector
== PF_VECTOR
)
5284 ctxt
->exception
.error_code
|= PFERR_WRITE_MASK
;
5288 /* Copy full 64-bit value for CMPXCHG8B. */
5289 ctxt
->dst
.orig_val64
= ctxt
->dst
.val64
;
5293 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5294 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5295 X86_ICPT_POST_MEMACCESS
);
5296 if (rc
!= X86EMUL_CONTINUE
)
5300 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5301 ctxt
->eflags
|= X86_EFLAGS_RF
;
5303 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5305 if (ctxt
->execute
) {
5306 if (ctxt
->d
& Fastop
) {
5307 void (*fop
)(struct fastop
*) = (void *)ctxt
->execute
;
5308 rc
= fastop(ctxt
, fop
);
5309 if (rc
!= X86EMUL_CONTINUE
)
5313 rc
= ctxt
->execute(ctxt
);
5314 if (rc
!= X86EMUL_CONTINUE
)
5319 if (ctxt
->opcode_len
== 2)
5321 else if (ctxt
->opcode_len
== 3)
5322 goto threebyte_insn
;
5325 case 0x70 ... 0x7f: /* jcc (short) */
5326 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5327 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5329 case 0x8d: /* lea r16/r32, m */
5330 ctxt
->dst
.val
= ctxt
->src
.addr
.mem
.ea
;
5332 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5333 if (ctxt
->dst
.addr
.reg
== reg_rmw(ctxt
, VCPU_REGS_RAX
))
5334 ctxt
->dst
.type
= OP_NONE
;
5338 case 0x98: /* cbw/cwde/cdqe */
5339 switch (ctxt
->op_bytes
) {
5340 case 2: ctxt
->dst
.val
= (s8
)ctxt
->dst
.val
; break;
5341 case 4: ctxt
->dst
.val
= (s16
)ctxt
->dst
.val
; break;
5342 case 8: ctxt
->dst
.val
= (s32
)ctxt
->dst
.val
; break;
5345 case 0xcc: /* int3 */
5346 rc
= emulate_int(ctxt
, 3);
5348 case 0xcd: /* int n */
5349 rc
= emulate_int(ctxt
, ctxt
->src
.val
);
5351 case 0xce: /* into */
5352 if (ctxt
->eflags
& X86_EFLAGS_OF
)
5353 rc
= emulate_int(ctxt
, 4);
5355 case 0xe9: /* jmp rel */
5356 case 0xeb: /* jmp rel short */
5357 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5358 ctxt
->dst
.type
= OP_NONE
; /* Disable writeback. */
5360 case 0xf4: /* hlt */
5361 ctxt
->ops
->halt(ctxt
);
5363 case 0xf5: /* cmc */
5364 /* complement carry flag from eflags reg */
5365 ctxt
->eflags
^= X86_EFLAGS_CF
;
5367 case 0xf8: /* clc */
5368 ctxt
->eflags
&= ~X86_EFLAGS_CF
;
5370 case 0xf9: /* stc */
5371 ctxt
->eflags
|= X86_EFLAGS_CF
;
5373 case 0xfc: /* cld */
5374 ctxt
->eflags
&= ~X86_EFLAGS_DF
;
5376 case 0xfd: /* std */
5377 ctxt
->eflags
|= X86_EFLAGS_DF
;
5380 goto cannot_emulate
;
5383 if (rc
!= X86EMUL_CONTINUE
)
5387 if (ctxt
->d
& SrcWrite
) {
5388 BUG_ON(ctxt
->src
.type
== OP_MEM
|| ctxt
->src
.type
== OP_MEM_STR
);
5389 rc
= writeback(ctxt
, &ctxt
->src
);
5390 if (rc
!= X86EMUL_CONTINUE
)
5393 if (!(ctxt
->d
& NoWrite
)) {
5394 rc
= writeback(ctxt
, &ctxt
->dst
);
5395 if (rc
!= X86EMUL_CONTINUE
)
5400 * restore dst type in case the decoding will be reused
5401 * (happens for string instruction )
5403 ctxt
->dst
.type
= saved_dst_type
;
5405 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5406 string_addr_inc(ctxt
, VCPU_REGS_RSI
, &ctxt
->src
);
5408 if ((ctxt
->d
& DstMask
) == DstDI
)
5409 string_addr_inc(ctxt
, VCPU_REGS_RDI
, &ctxt
->dst
);
5411 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5413 struct read_cache
*r
= &ctxt
->io_read
;
5414 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5415 count
= ctxt
->src
.count
;
5417 count
= ctxt
->dst
.count
;
5418 register_address_increment(ctxt
, VCPU_REGS_RCX
, -count
);
5420 if (!string_insn_completed(ctxt
)) {
5422 * Re-enter guest when pio read ahead buffer is empty
5423 * or, if it is not used, after each 1024 iteration.
5425 if ((r
->end
!= 0 || reg_read(ctxt
, VCPU_REGS_RCX
) & 0x3ff) &&
5426 (r
->end
== 0 || r
->end
!= r
->pos
)) {
5428 * Reset read cache. Usually happens before
5429 * decode, but since instruction is restarted
5430 * we have to do it here.
5432 ctxt
->mem_read
.end
= 0;
5433 writeback_registers(ctxt
);
5434 return EMULATION_RESTART
;
5436 goto done
; /* skip rip writeback */
5438 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5441 ctxt
->eip
= ctxt
->_eip
;
5444 if (rc
== X86EMUL_PROPAGATE_FAULT
) {
5445 WARN_ON(ctxt
->exception
.vector
> 0x1f);
5446 ctxt
->have_exception
= true;
5448 if (rc
== X86EMUL_INTERCEPTED
)
5449 return EMULATION_INTERCEPTED
;
5451 if (rc
== X86EMUL_CONTINUE
)
5452 writeback_registers(ctxt
);
5454 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
5458 case 0x09: /* wbinvd */
5459 (ctxt
->ops
->wbinvd
)(ctxt
);
5461 case 0x08: /* invd */
5462 case 0x0d: /* GrpP (prefetch) */
5463 case 0x18: /* Grp16 (prefetch/nop) */
5464 case 0x1f: /* nop */
5466 case 0x20: /* mov cr, reg */
5467 ctxt
->dst
.val
= ops
->get_cr(ctxt
, ctxt
->modrm_reg
);
5469 case 0x21: /* mov from dr to reg */
5470 ops
->get_dr(ctxt
, ctxt
->modrm_reg
, &ctxt
->dst
.val
);
5472 case 0x40 ... 0x4f: /* cmov */
5473 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5474 ctxt
->dst
.val
= ctxt
->src
.val
;
5475 else if (ctxt
->op_bytes
!= 4)
5476 ctxt
->dst
.type
= OP_NONE
; /* no writeback */
5478 case 0x80 ... 0x8f: /* jnz rel, etc*/
5479 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5480 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5482 case 0x90 ... 0x9f: /* setcc r/m8 */
5483 ctxt
->dst
.val
= test_cc(ctxt
->b
, ctxt
->eflags
);
5485 case 0xb6 ... 0xb7: /* movzx */
5486 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5487 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (u8
) ctxt
->src
.val
5488 : (u16
) ctxt
->src
.val
;
5490 case 0xbe ... 0xbf: /* movsx */
5491 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5492 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (s8
) ctxt
->src
.val
:
5493 (s16
) ctxt
->src
.val
;
5496 goto cannot_emulate
;
5501 if (rc
!= X86EMUL_CONTINUE
)
5507 return EMULATION_FAILED
;
5510 void emulator_invalidate_register_cache(struct x86_emulate_ctxt
*ctxt
)
5512 invalidate_registers(ctxt
);
5515 void emulator_writeback_register_cache(struct x86_emulate_ctxt
*ctxt
)
5517 writeback_registers(ctxt
);