1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
174 #define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
176 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
178 #define X2(x...) x, x
179 #define X3(x...) X2(x), x
180 #define X4(x...) X2(x), X2(x)
181 #define X5(x...) X4(x), x
182 #define X6(x...) X4(x), X2(x)
183 #define X7(x...) X4(x), X3(x)
184 #define X8(x...) X4(x), X4(x)
185 #define X16(x...) X8(x), X8(x)
187 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
188 #define FASTOP_SIZE 8
191 * fastop functions have a special calling convention:
196 * flags: rflags (in/out)
197 * ex: rsi (in:fastop pointer, out:zero if exception)
199 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
200 * different operand sizes can be reached by calculation, rather than a jump
201 * table (which would be bigger than the code).
203 * fastop functions are declared as taking a never-defined fastop parameter,
204 * so they can't be called from C directly.
213 int (*execute
)(struct x86_emulate_ctxt
*ctxt
);
214 const struct opcode
*group
;
215 const struct group_dual
*gdual
;
216 const struct gprefix
*gprefix
;
217 const struct escape
*esc
;
218 const struct instr_dual
*idual
;
219 const struct mode_dual
*mdual
;
220 void (*fastop
)(struct fastop
*fake
);
222 int (*check_perm
)(struct x86_emulate_ctxt
*ctxt
);
226 struct opcode mod012
[8];
227 struct opcode mod3
[8];
231 struct opcode pfx_no
;
232 struct opcode pfx_66
;
233 struct opcode pfx_f2
;
234 struct opcode pfx_f3
;
239 struct opcode high
[64];
243 struct opcode mod012
;
248 struct opcode mode32
;
249 struct opcode mode64
;
252 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 enum x86_transfer_type
{
256 X86_TRANSFER_CALL_JMP
,
258 X86_TRANSFER_TASK_SWITCH
,
261 static ulong
reg_read(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
263 if (!(ctxt
->regs_valid
& (1 << nr
))) {
264 ctxt
->regs_valid
|= 1 << nr
;
265 ctxt
->_regs
[nr
] = ctxt
->ops
->read_gpr(ctxt
, nr
);
267 return ctxt
->_regs
[nr
];
270 static ulong
*reg_write(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
272 ctxt
->regs_valid
|= 1 << nr
;
273 ctxt
->regs_dirty
|= 1 << nr
;
274 return &ctxt
->_regs
[nr
];
277 static ulong
*reg_rmw(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
280 return reg_write(ctxt
, nr
);
283 static void writeback_registers(struct x86_emulate_ctxt
*ctxt
)
287 for_each_set_bit(reg
, (ulong
*)&ctxt
->regs_dirty
, 16)
288 ctxt
->ops
->write_gpr(ctxt
, reg
, ctxt
->_regs
[reg
]);
291 static void invalidate_registers(struct x86_emulate_ctxt
*ctxt
)
293 ctxt
->regs_dirty
= 0;
294 ctxt
->regs_valid
= 0;
298 * These EFLAGS bits are restored from saved value during emulation, and
299 * any changes are written back to the saved value after emulation.
301 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
302 X86_EFLAGS_PF|X86_EFLAGS_CF)
310 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*));
312 #define FOP_FUNC(name) \
313 ".align " __stringify(FASTOP_SIZE) " \n\t" \
314 ".type " name ", @function \n\t" \
317 #define FOP_RET "ret \n\t"
319 #define FOP_START(op) \
320 extern void em_##op(struct fastop *fake); \
321 asm(".pushsection .text, \"ax\" \n\t" \
322 ".global em_" #op " \n\t" \
329 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
332 #define FOP1E(op, dst) \
333 FOP_FUNC(#op "_" #dst) \
334 "10: " #op " %" #dst " \n\t" FOP_RET
336 #define FOP1EEX(op, dst) \
337 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
339 #define FASTOP1(op) \
344 ON64(FOP1E(op##q, rax)) \
347 /* 1-operand, using src2 (for MUL/DIV r/m) */
348 #define FASTOP1SRC2(op, name) \
353 ON64(FOP1E(op, rcx)) \
356 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
357 #define FASTOP1SRC2EX(op, name) \
362 ON64(FOP1EEX(op, rcx)) \
365 #define FOP2E(op, dst, src) \
366 FOP_FUNC(#op "_" #dst "_" #src) \
367 #op " %" #src ", %" #dst " \n\t" FOP_RET
369 #define FASTOP2(op) \
371 FOP2E(op##b, al, dl) \
372 FOP2E(op##w, ax, dx) \
373 FOP2E(op##l, eax, edx) \
374 ON64(FOP2E(op##q, rax, rdx)) \
377 /* 2 operand, word only */
378 #define FASTOP2W(op) \
381 FOP2E(op##w, ax, dx) \
382 FOP2E(op##l, eax, edx) \
383 ON64(FOP2E(op##q, rax, rdx)) \
386 /* 2 operand, src is CL */
387 #define FASTOP2CL(op) \
389 FOP2E(op##b, al, cl) \
390 FOP2E(op##w, ax, cl) \
391 FOP2E(op##l, eax, cl) \
392 ON64(FOP2E(op##q, rax, cl)) \
395 /* 2 operand, src and dest are reversed */
396 #define FASTOP2R(op, name) \
398 FOP2E(op##b, dl, al) \
399 FOP2E(op##w, dx, ax) \
400 FOP2E(op##l, edx, eax) \
401 ON64(FOP2E(op##q, rdx, rax)) \
404 #define FOP3E(op, dst, src, src2) \
405 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
406 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
408 /* 3-operand, word-only, src2=cl */
409 #define FASTOP3WCL(op) \
412 FOP3E(op##w, ax, dx, cl) \
413 FOP3E(op##l, eax, edx, cl) \
414 ON64(FOP3E(op##q, rax, rdx, cl)) \
417 /* Special case for SETcc - 1 instruction per cc */
418 #define FOP_SETCC(op) \
420 ".type " #op ", @function \n\t" \
425 asm(".global kvm_fastop_exception \n"
426 "kvm_fastop_exception: xor %esi, %esi; ret");
447 FOP_START(salc
) "pushf; sbb %al, %al; popf \n\t" FOP_RET
450 static int emulator_check_intercept(struct x86_emulate_ctxt
*ctxt
,
451 enum x86_intercept intercept
,
452 enum x86_intercept_stage stage
)
454 struct x86_instruction_info info
= {
455 .intercept
= intercept
,
456 .rep_prefix
= ctxt
->rep_prefix
,
457 .modrm_mod
= ctxt
->modrm_mod
,
458 .modrm_reg
= ctxt
->modrm_reg
,
459 .modrm_rm
= ctxt
->modrm_rm
,
460 .src_val
= ctxt
->src
.val64
,
461 .dst_val
= ctxt
->dst
.val64
,
462 .src_bytes
= ctxt
->src
.bytes
,
463 .dst_bytes
= ctxt
->dst
.bytes
,
464 .ad_bytes
= ctxt
->ad_bytes
,
465 .next_rip
= ctxt
->eip
,
468 return ctxt
->ops
->intercept(ctxt
, &info
, stage
);
471 static void assign_masked(ulong
*dest
, ulong src
, ulong mask
)
473 *dest
= (*dest
& ~mask
) | (src
& mask
);
476 static void assign_register(unsigned long *reg
, u64 val
, int bytes
)
478 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
481 *(u8
*)reg
= (u8
)val
;
484 *(u16
*)reg
= (u16
)val
;
488 break; /* 64b: zero-extend */
495 static inline unsigned long ad_mask(struct x86_emulate_ctxt
*ctxt
)
497 return (1UL << (ctxt
->ad_bytes
<< 3)) - 1;
500 static ulong
stack_mask(struct x86_emulate_ctxt
*ctxt
)
503 struct desc_struct ss
;
505 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
507 ctxt
->ops
->get_segment(ctxt
, &sel
, &ss
, NULL
, VCPU_SREG_SS
);
508 return ~0U >> ((ss
.d
^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
511 static int stack_size(struct x86_emulate_ctxt
*ctxt
)
513 return (__fls(stack_mask(ctxt
)) + 1) >> 3;
516 /* Access/update address held in a register, based on addressing mode. */
517 static inline unsigned long
518 address_mask(struct x86_emulate_ctxt
*ctxt
, unsigned long reg
)
520 if (ctxt
->ad_bytes
== sizeof(unsigned long))
523 return reg
& ad_mask(ctxt
);
526 static inline unsigned long
527 register_address(struct x86_emulate_ctxt
*ctxt
, int reg
)
529 return address_mask(ctxt
, reg_read(ctxt
, reg
));
532 static void masked_increment(ulong
*reg
, ulong mask
, int inc
)
534 assign_masked(reg
, *reg
+ inc
, mask
);
538 register_address_increment(struct x86_emulate_ctxt
*ctxt
, int reg
, int inc
)
540 ulong
*preg
= reg_rmw(ctxt
, reg
);
542 assign_register(preg
, *preg
+ inc
, ctxt
->ad_bytes
);
545 static void rsp_increment(struct x86_emulate_ctxt
*ctxt
, int inc
)
547 masked_increment(reg_rmw(ctxt
, VCPU_REGS_RSP
), stack_mask(ctxt
), inc
);
550 static u32
desc_limit_scaled(struct desc_struct
*desc
)
552 u32 limit
= get_desc_limit(desc
);
554 return desc
->g
? (limit
<< 12) | 0xfff : limit
;
557 static unsigned long seg_base(struct x86_emulate_ctxt
*ctxt
, int seg
)
559 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&& seg
< VCPU_SREG_FS
)
562 return ctxt
->ops
->get_cached_segment_base(ctxt
, seg
);
565 static int emulate_exception(struct x86_emulate_ctxt
*ctxt
, int vec
,
566 u32 error
, bool valid
)
569 ctxt
->exception
.vector
= vec
;
570 ctxt
->exception
.error_code
= error
;
571 ctxt
->exception
.error_code_valid
= valid
;
572 return X86EMUL_PROPAGATE_FAULT
;
575 static int emulate_db(struct x86_emulate_ctxt
*ctxt
)
577 return emulate_exception(ctxt
, DB_VECTOR
, 0, false);
580 static int emulate_gp(struct x86_emulate_ctxt
*ctxt
, int err
)
582 return emulate_exception(ctxt
, GP_VECTOR
, err
, true);
585 static int emulate_ss(struct x86_emulate_ctxt
*ctxt
, int err
)
587 return emulate_exception(ctxt
, SS_VECTOR
, err
, true);
590 static int emulate_ud(struct x86_emulate_ctxt
*ctxt
)
592 return emulate_exception(ctxt
, UD_VECTOR
, 0, false);
595 static int emulate_ts(struct x86_emulate_ctxt
*ctxt
, int err
)
597 return emulate_exception(ctxt
, TS_VECTOR
, err
, true);
600 static int emulate_de(struct x86_emulate_ctxt
*ctxt
)
602 return emulate_exception(ctxt
, DE_VECTOR
, 0, false);
605 static int emulate_nm(struct x86_emulate_ctxt
*ctxt
)
607 return emulate_exception(ctxt
, NM_VECTOR
, 0, false);
610 static u16
get_segment_selector(struct x86_emulate_ctxt
*ctxt
, unsigned seg
)
613 struct desc_struct desc
;
615 ctxt
->ops
->get_segment(ctxt
, &selector
, &desc
, NULL
, seg
);
619 static void set_segment_selector(struct x86_emulate_ctxt
*ctxt
, u16 selector
,
624 struct desc_struct desc
;
626 ctxt
->ops
->get_segment(ctxt
, &dummy
, &desc
, &base3
, seg
);
627 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, seg
);
631 * x86 defines three classes of vector instructions: explicitly
632 * aligned, explicitly unaligned, and the rest, which change behaviour
633 * depending on whether they're AVX encoded or not.
635 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
636 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
637 * 512 bytes of data must be aligned to a 16 byte boundary.
639 static unsigned insn_alignment(struct x86_emulate_ctxt
*ctxt
, unsigned size
)
641 if (likely(size
< 16))
644 if (ctxt
->d
& Aligned
)
646 else if (ctxt
->d
& Unaligned
)
648 else if (ctxt
->d
& Avx
)
650 else if (ctxt
->d
& Aligned16
)
656 static __always_inline
int __linearize(struct x86_emulate_ctxt
*ctxt
,
657 struct segmented_address addr
,
658 unsigned *max_size
, unsigned size
,
659 bool write
, bool fetch
,
660 enum x86emul_mode mode
, ulong
*linear
)
662 struct desc_struct desc
;
668 la
= seg_base(ctxt
, addr
.seg
) + addr
.ea
;
671 case X86EMUL_MODE_PROT64
:
673 if (is_noncanonical_address(la
))
676 *max_size
= min_t(u64
, ~0u, (1ull << 48) - la
);
677 if (size
> *max_size
)
681 *linear
= la
= (u32
)la
;
682 usable
= ctxt
->ops
->get_segment(ctxt
, &sel
, &desc
, NULL
,
686 /* code segment in protected mode or read-only data segment */
687 if ((((ctxt
->mode
!= X86EMUL_MODE_REAL
) && (desc
.type
& 8))
688 || !(desc
.type
& 2)) && write
)
690 /* unreadable code segment */
691 if (!fetch
&& (desc
.type
& 8) && !(desc
.type
& 2))
693 lim
= desc_limit_scaled(&desc
);
694 if (!(desc
.type
& 8) && (desc
.type
& 4)) {
695 /* expand-down segment */
698 lim
= desc
.d
? 0xffffffff : 0xffff;
702 if (lim
== 0xffffffff)
705 *max_size
= (u64
)lim
+ 1 - addr
.ea
;
706 if (size
> *max_size
)
711 if (la
& (insn_alignment(ctxt
, size
) - 1))
712 return emulate_gp(ctxt
, 0);
713 return X86EMUL_CONTINUE
;
715 if (addr
.seg
== VCPU_SREG_SS
)
716 return emulate_ss(ctxt
, 0);
718 return emulate_gp(ctxt
, 0);
721 static int linearize(struct x86_emulate_ctxt
*ctxt
,
722 struct segmented_address addr
,
723 unsigned size
, bool write
,
727 return __linearize(ctxt
, addr
, &max_size
, size
, write
, false,
731 static inline int assign_eip(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
732 enum x86emul_mode mode
)
737 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
740 if (ctxt
->op_bytes
!= sizeof(unsigned long))
741 addr
.ea
= dst
& ((1UL << (ctxt
->op_bytes
<< 3)) - 1);
742 rc
= __linearize(ctxt
, addr
, &max_size
, 1, false, true, mode
, &linear
);
743 if (rc
== X86EMUL_CONTINUE
)
744 ctxt
->_eip
= addr
.ea
;
748 static inline int assign_eip_near(struct x86_emulate_ctxt
*ctxt
, ulong dst
)
750 return assign_eip(ctxt
, dst
, ctxt
->mode
);
753 static int assign_eip_far(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
754 const struct desc_struct
*cs_desc
)
756 enum x86emul_mode mode
= ctxt
->mode
;
760 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
) {
764 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
766 mode
= X86EMUL_MODE_PROT64
;
768 mode
= X86EMUL_MODE_PROT32
; /* temporary value */
771 if (mode
== X86EMUL_MODE_PROT16
|| mode
== X86EMUL_MODE_PROT32
)
772 mode
= cs_desc
->d
? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
773 rc
= assign_eip(ctxt
, dst
, mode
);
774 if (rc
== X86EMUL_CONTINUE
)
779 static inline int jmp_rel(struct x86_emulate_ctxt
*ctxt
, int rel
)
781 return assign_eip_near(ctxt
, ctxt
->_eip
+ rel
);
784 static int segmented_read_std(struct x86_emulate_ctxt
*ctxt
,
785 struct segmented_address addr
,
792 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
793 if (rc
!= X86EMUL_CONTINUE
)
795 return ctxt
->ops
->read_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
799 * Prefetch the remaining bytes of the instruction without crossing page
800 * boundary if they are not in fetch_cache yet.
802 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
, int op_size
)
805 unsigned size
, max_size
;
806 unsigned long linear
;
807 int cur_size
= ctxt
->fetch
.end
- ctxt
->fetch
.data
;
808 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
809 .ea
= ctxt
->eip
+ cur_size
};
812 * We do not know exactly how many bytes will be needed, and
813 * __linearize is expensive, so fetch as much as possible. We
814 * just have to avoid going beyond the 15 byte limit, the end
815 * of the segment, or the end of the page.
817 * __linearize is called with size 0 so that it does not do any
818 * boundary check itself. Instead, we use max_size to check
821 rc
= __linearize(ctxt
, addr
, &max_size
, 0, false, true, ctxt
->mode
,
823 if (unlikely(rc
!= X86EMUL_CONTINUE
))
826 size
= min_t(unsigned, 15UL ^ cur_size
, max_size
);
827 size
= min_t(unsigned, size
, PAGE_SIZE
- offset_in_page(linear
));
830 * One instruction can only straddle two pages,
831 * and one has been loaded at the beginning of
832 * x86_decode_insn. So, if not enough bytes
833 * still, we must have hit the 15-byte boundary.
835 if (unlikely(size
< op_size
))
836 return emulate_gp(ctxt
, 0);
838 rc
= ctxt
->ops
->fetch(ctxt
, linear
, ctxt
->fetch
.end
,
839 size
, &ctxt
->exception
);
840 if (unlikely(rc
!= X86EMUL_CONTINUE
))
842 ctxt
->fetch
.end
+= size
;
843 return X86EMUL_CONTINUE
;
846 static __always_inline
int do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
,
849 unsigned done_size
= ctxt
->fetch
.end
- ctxt
->fetch
.ptr
;
851 if (unlikely(done_size
< size
))
852 return __do_insn_fetch_bytes(ctxt
, size
- done_size
);
854 return X86EMUL_CONTINUE
;
857 /* Fetch next part of the instruction being emulated. */
858 #define insn_fetch(_type, _ctxt) \
861 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
862 if (rc != X86EMUL_CONTINUE) \
864 ctxt->_eip += sizeof(_type); \
865 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
866 ctxt->fetch.ptr += sizeof(_type); \
870 #define insn_fetch_arr(_arr, _size, _ctxt) \
872 rc = do_insn_fetch_bytes(_ctxt, _size); \
873 if (rc != X86EMUL_CONTINUE) \
875 ctxt->_eip += (_size); \
876 memcpy(_arr, ctxt->fetch.ptr, _size); \
877 ctxt->fetch.ptr += (_size); \
881 * Given the 'reg' portion of a ModRM byte, and a register block, return a
882 * pointer into the block that addresses the relevant register.
883 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
885 static void *decode_register(struct x86_emulate_ctxt
*ctxt
, u8 modrm_reg
,
889 int highbyte_regs
= (ctxt
->rex_prefix
== 0) && byteop
;
891 if (highbyte_regs
&& modrm_reg
>= 4 && modrm_reg
< 8)
892 p
= (unsigned char *)reg_rmw(ctxt
, modrm_reg
& 3) + 1;
894 p
= reg_rmw(ctxt
, modrm_reg
);
898 static int read_descriptor(struct x86_emulate_ctxt
*ctxt
,
899 struct segmented_address addr
,
900 u16
*size
, unsigned long *address
, int op_bytes
)
907 rc
= segmented_read_std(ctxt
, addr
, size
, 2);
908 if (rc
!= X86EMUL_CONTINUE
)
911 rc
= segmented_read_std(ctxt
, addr
, address
, op_bytes
);
925 FASTOP1SRC2(mul
, mul_ex
);
926 FASTOP1SRC2(imul
, imul_ex
);
927 FASTOP1SRC2EX(div
, div_ex
);
928 FASTOP1SRC2EX(idiv
, idiv_ex
);
957 FASTOP2R(cmp
, cmp_r
);
959 static int em_bsf_c(struct x86_emulate_ctxt
*ctxt
)
961 /* If src is zero, do not writeback, but update flags */
962 if (ctxt
->src
.val
== 0)
963 ctxt
->dst
.type
= OP_NONE
;
964 return fastop(ctxt
, em_bsf
);
967 static int em_bsr_c(struct x86_emulate_ctxt
*ctxt
)
969 /* If src is zero, do not writeback, but update flags */
970 if (ctxt
->src
.val
== 0)
971 ctxt
->dst
.type
= OP_NONE
;
972 return fastop(ctxt
, em_bsr
);
975 static __always_inline u8
test_cc(unsigned int condition
, unsigned long flags
)
978 void (*fop
)(void) = (void *)em_setcc
+ 4 * (condition
& 0xf);
980 flags
= (flags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
981 asm("push %[flags]; popf; call *%[fastop]"
982 : "=a"(rc
) : [fastop
]"r"(fop
), [flags
]"r"(flags
));
986 static void fetch_register_operand(struct operand
*op
)
990 op
->val
= *(u8
*)op
->addr
.reg
;
993 op
->val
= *(u16
*)op
->addr
.reg
;
996 op
->val
= *(u32
*)op
->addr
.reg
;
999 op
->val
= *(u64
*)op
->addr
.reg
;
1004 static void read_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
, int reg
)
1006 ctxt
->ops
->get_fpu(ctxt
);
1008 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data
)); break;
1009 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data
)); break;
1010 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data
)); break;
1011 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data
)); break;
1012 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data
)); break;
1013 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data
)); break;
1014 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data
)); break;
1015 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data
)); break;
1016 #ifdef CONFIG_X86_64
1017 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data
)); break;
1018 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data
)); break;
1019 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data
)); break;
1020 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data
)); break;
1021 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data
)); break;
1022 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data
)); break;
1023 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data
)); break;
1024 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data
)); break;
1028 ctxt
->ops
->put_fpu(ctxt
);
1031 static void write_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
,
1034 ctxt
->ops
->get_fpu(ctxt
);
1036 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data
)); break;
1037 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data
)); break;
1038 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data
)); break;
1039 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data
)); break;
1040 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data
)); break;
1041 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data
)); break;
1042 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data
)); break;
1043 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data
)); break;
1044 #ifdef CONFIG_X86_64
1045 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data
)); break;
1046 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data
)); break;
1047 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data
)); break;
1048 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data
)); break;
1049 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data
)); break;
1050 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data
)); break;
1051 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data
)); break;
1052 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data
)); break;
1056 ctxt
->ops
->put_fpu(ctxt
);
1059 static void read_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1061 ctxt
->ops
->get_fpu(ctxt
);
1063 case 0: asm("movq %%mm0, %0" : "=m"(*data
)); break;
1064 case 1: asm("movq %%mm1, %0" : "=m"(*data
)); break;
1065 case 2: asm("movq %%mm2, %0" : "=m"(*data
)); break;
1066 case 3: asm("movq %%mm3, %0" : "=m"(*data
)); break;
1067 case 4: asm("movq %%mm4, %0" : "=m"(*data
)); break;
1068 case 5: asm("movq %%mm5, %0" : "=m"(*data
)); break;
1069 case 6: asm("movq %%mm6, %0" : "=m"(*data
)); break;
1070 case 7: asm("movq %%mm7, %0" : "=m"(*data
)); break;
1073 ctxt
->ops
->put_fpu(ctxt
);
1076 static void write_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1078 ctxt
->ops
->get_fpu(ctxt
);
1080 case 0: asm("movq %0, %%mm0" : : "m"(*data
)); break;
1081 case 1: asm("movq %0, %%mm1" : : "m"(*data
)); break;
1082 case 2: asm("movq %0, %%mm2" : : "m"(*data
)); break;
1083 case 3: asm("movq %0, %%mm3" : : "m"(*data
)); break;
1084 case 4: asm("movq %0, %%mm4" : : "m"(*data
)); break;
1085 case 5: asm("movq %0, %%mm5" : : "m"(*data
)); break;
1086 case 6: asm("movq %0, %%mm6" : : "m"(*data
)); break;
1087 case 7: asm("movq %0, %%mm7" : : "m"(*data
)); break;
1090 ctxt
->ops
->put_fpu(ctxt
);
1093 static int em_fninit(struct x86_emulate_ctxt
*ctxt
)
1095 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1096 return emulate_nm(ctxt
);
1098 ctxt
->ops
->get_fpu(ctxt
);
1099 asm volatile("fninit");
1100 ctxt
->ops
->put_fpu(ctxt
);
1101 return X86EMUL_CONTINUE
;
1104 static int em_fnstcw(struct x86_emulate_ctxt
*ctxt
)
1108 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1109 return emulate_nm(ctxt
);
1111 ctxt
->ops
->get_fpu(ctxt
);
1112 asm volatile("fnstcw %0": "+m"(fcw
));
1113 ctxt
->ops
->put_fpu(ctxt
);
1115 ctxt
->dst
.val
= fcw
;
1117 return X86EMUL_CONTINUE
;
1120 static int em_fnstsw(struct x86_emulate_ctxt
*ctxt
)
1124 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1125 return emulate_nm(ctxt
);
1127 ctxt
->ops
->get_fpu(ctxt
);
1128 asm volatile("fnstsw %0": "+m"(fsw
));
1129 ctxt
->ops
->put_fpu(ctxt
);
1131 ctxt
->dst
.val
= fsw
;
1133 return X86EMUL_CONTINUE
;
1136 static void decode_register_operand(struct x86_emulate_ctxt
*ctxt
,
1139 unsigned reg
= ctxt
->modrm_reg
;
1141 if (!(ctxt
->d
& ModRM
))
1142 reg
= (ctxt
->b
& 7) | ((ctxt
->rex_prefix
& 1) << 3);
1144 if (ctxt
->d
& Sse
) {
1148 read_sse_reg(ctxt
, &op
->vec_val
, reg
);
1151 if (ctxt
->d
& Mmx
) {
1160 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1161 op
->addr
.reg
= decode_register(ctxt
, reg
, ctxt
->d
& ByteOp
);
1163 fetch_register_operand(op
);
1164 op
->orig_val
= op
->val
;
1167 static void adjust_modrm_seg(struct x86_emulate_ctxt
*ctxt
, int base_reg
)
1169 if (base_reg
== VCPU_REGS_RSP
|| base_reg
== VCPU_REGS_RBP
)
1170 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1173 static int decode_modrm(struct x86_emulate_ctxt
*ctxt
,
1177 int index_reg
, base_reg
, scale
;
1178 int rc
= X86EMUL_CONTINUE
;
1181 ctxt
->modrm_reg
= ((ctxt
->rex_prefix
<< 1) & 8); /* REX.R */
1182 index_reg
= (ctxt
->rex_prefix
<< 2) & 8; /* REX.X */
1183 base_reg
= (ctxt
->rex_prefix
<< 3) & 8; /* REX.B */
1185 ctxt
->modrm_mod
= (ctxt
->modrm
& 0xc0) >> 6;
1186 ctxt
->modrm_reg
|= (ctxt
->modrm
& 0x38) >> 3;
1187 ctxt
->modrm_rm
= base_reg
| (ctxt
->modrm
& 0x07);
1188 ctxt
->modrm_seg
= VCPU_SREG_DS
;
1190 if (ctxt
->modrm_mod
== 3 || (ctxt
->d
& NoMod
)) {
1192 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1193 op
->addr
.reg
= decode_register(ctxt
, ctxt
->modrm_rm
,
1195 if (ctxt
->d
& Sse
) {
1198 op
->addr
.xmm
= ctxt
->modrm_rm
;
1199 read_sse_reg(ctxt
, &op
->vec_val
, ctxt
->modrm_rm
);
1202 if (ctxt
->d
& Mmx
) {
1205 op
->addr
.mm
= ctxt
->modrm_rm
& 7;
1208 fetch_register_operand(op
);
1214 if (ctxt
->ad_bytes
== 2) {
1215 unsigned bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
1216 unsigned bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1217 unsigned si
= reg_read(ctxt
, VCPU_REGS_RSI
);
1218 unsigned di
= reg_read(ctxt
, VCPU_REGS_RDI
);
1220 /* 16-bit ModR/M decode. */
1221 switch (ctxt
->modrm_mod
) {
1223 if (ctxt
->modrm_rm
== 6)
1224 modrm_ea
+= insn_fetch(u16
, ctxt
);
1227 modrm_ea
+= insn_fetch(s8
, ctxt
);
1230 modrm_ea
+= insn_fetch(u16
, ctxt
);
1233 switch (ctxt
->modrm_rm
) {
1235 modrm_ea
+= bx
+ si
;
1238 modrm_ea
+= bx
+ di
;
1241 modrm_ea
+= bp
+ si
;
1244 modrm_ea
+= bp
+ di
;
1253 if (ctxt
->modrm_mod
!= 0)
1260 if (ctxt
->modrm_rm
== 2 || ctxt
->modrm_rm
== 3 ||
1261 (ctxt
->modrm_rm
== 6 && ctxt
->modrm_mod
!= 0))
1262 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1263 modrm_ea
= (u16
)modrm_ea
;
1265 /* 32/64-bit ModR/M decode. */
1266 if ((ctxt
->modrm_rm
& 7) == 4) {
1267 sib
= insn_fetch(u8
, ctxt
);
1268 index_reg
|= (sib
>> 3) & 7;
1269 base_reg
|= sib
& 7;
1272 if ((base_reg
& 7) == 5 && ctxt
->modrm_mod
== 0)
1273 modrm_ea
+= insn_fetch(s32
, ctxt
);
1275 modrm_ea
+= reg_read(ctxt
, base_reg
);
1276 adjust_modrm_seg(ctxt
, base_reg
);
1277 /* Increment ESP on POP [ESP] */
1278 if ((ctxt
->d
& IncSP
) &&
1279 base_reg
== VCPU_REGS_RSP
)
1280 modrm_ea
+= ctxt
->op_bytes
;
1283 modrm_ea
+= reg_read(ctxt
, index_reg
) << scale
;
1284 } else if ((ctxt
->modrm_rm
& 7) == 5 && ctxt
->modrm_mod
== 0) {
1285 modrm_ea
+= insn_fetch(s32
, ctxt
);
1286 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
1287 ctxt
->rip_relative
= 1;
1289 base_reg
= ctxt
->modrm_rm
;
1290 modrm_ea
+= reg_read(ctxt
, base_reg
);
1291 adjust_modrm_seg(ctxt
, base_reg
);
1293 switch (ctxt
->modrm_mod
) {
1295 modrm_ea
+= insn_fetch(s8
, ctxt
);
1298 modrm_ea
+= insn_fetch(s32
, ctxt
);
1302 op
->addr
.mem
.ea
= modrm_ea
;
1303 if (ctxt
->ad_bytes
!= 8)
1304 ctxt
->memop
.addr
.mem
.ea
= (u32
)ctxt
->memop
.addr
.mem
.ea
;
1310 static int decode_abs(struct x86_emulate_ctxt
*ctxt
,
1313 int rc
= X86EMUL_CONTINUE
;
1316 switch (ctxt
->ad_bytes
) {
1318 op
->addr
.mem
.ea
= insn_fetch(u16
, ctxt
);
1321 op
->addr
.mem
.ea
= insn_fetch(u32
, ctxt
);
1324 op
->addr
.mem
.ea
= insn_fetch(u64
, ctxt
);
1331 static void fetch_bit_operand(struct x86_emulate_ctxt
*ctxt
)
1335 if (ctxt
->dst
.type
== OP_MEM
&& ctxt
->src
.type
== OP_REG
) {
1336 mask
= ~((long)ctxt
->dst
.bytes
* 8 - 1);
1338 if (ctxt
->src
.bytes
== 2)
1339 sv
= (s16
)ctxt
->src
.val
& (s16
)mask
;
1340 else if (ctxt
->src
.bytes
== 4)
1341 sv
= (s32
)ctxt
->src
.val
& (s32
)mask
;
1343 sv
= (s64
)ctxt
->src
.val
& (s64
)mask
;
1345 ctxt
->dst
.addr
.mem
.ea
= address_mask(ctxt
,
1346 ctxt
->dst
.addr
.mem
.ea
+ (sv
>> 3));
1349 /* only subword offset */
1350 ctxt
->src
.val
&= (ctxt
->dst
.bytes
<< 3) - 1;
1353 static int read_emulated(struct x86_emulate_ctxt
*ctxt
,
1354 unsigned long addr
, void *dest
, unsigned size
)
1357 struct read_cache
*mc
= &ctxt
->mem_read
;
1359 if (mc
->pos
< mc
->end
)
1362 WARN_ON((mc
->end
+ size
) >= sizeof(mc
->data
));
1364 rc
= ctxt
->ops
->read_emulated(ctxt
, addr
, mc
->data
+ mc
->end
, size
,
1366 if (rc
!= X86EMUL_CONTINUE
)
1372 memcpy(dest
, mc
->data
+ mc
->pos
, size
);
1374 return X86EMUL_CONTINUE
;
1377 static int segmented_read(struct x86_emulate_ctxt
*ctxt
,
1378 struct segmented_address addr
,
1385 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
1386 if (rc
!= X86EMUL_CONTINUE
)
1388 return read_emulated(ctxt
, linear
, data
, size
);
1391 static int segmented_write(struct x86_emulate_ctxt
*ctxt
,
1392 struct segmented_address addr
,
1399 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1400 if (rc
!= X86EMUL_CONTINUE
)
1402 return ctxt
->ops
->write_emulated(ctxt
, linear
, data
, size
,
1406 static int segmented_cmpxchg(struct x86_emulate_ctxt
*ctxt
,
1407 struct segmented_address addr
,
1408 const void *orig_data
, const void *data
,
1414 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1415 if (rc
!= X86EMUL_CONTINUE
)
1417 return ctxt
->ops
->cmpxchg_emulated(ctxt
, linear
, orig_data
, data
,
1418 size
, &ctxt
->exception
);
1421 static int pio_in_emulated(struct x86_emulate_ctxt
*ctxt
,
1422 unsigned int size
, unsigned short port
,
1425 struct read_cache
*rc
= &ctxt
->io_read
;
1427 if (rc
->pos
== rc
->end
) { /* refill pio read ahead */
1428 unsigned int in_page
, n
;
1429 unsigned int count
= ctxt
->rep_prefix
?
1430 address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) : 1;
1431 in_page
= (ctxt
->eflags
& X86_EFLAGS_DF
) ?
1432 offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
)) :
1433 PAGE_SIZE
- offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
));
1434 n
= min3(in_page
, (unsigned int)sizeof(rc
->data
) / size
, count
);
1437 rc
->pos
= rc
->end
= 0;
1438 if (!ctxt
->ops
->pio_in_emulated(ctxt
, size
, port
, rc
->data
, n
))
1443 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
) &&
1444 !(ctxt
->eflags
& X86_EFLAGS_DF
)) {
1445 ctxt
->dst
.data
= rc
->data
+ rc
->pos
;
1446 ctxt
->dst
.type
= OP_MEM_STR
;
1447 ctxt
->dst
.count
= (rc
->end
- rc
->pos
) / size
;
1450 memcpy(dest
, rc
->data
+ rc
->pos
, size
);
1456 static int read_interrupt_descriptor(struct x86_emulate_ctxt
*ctxt
,
1457 u16 index
, struct desc_struct
*desc
)
1462 ctxt
->ops
->get_idt(ctxt
, &dt
);
1464 if (dt
.size
< index
* 8 + 7)
1465 return emulate_gp(ctxt
, index
<< 3 | 0x2);
1467 addr
= dt
.address
+ index
* 8;
1468 return ctxt
->ops
->read_std(ctxt
, addr
, desc
, sizeof *desc
,
1472 static void get_descriptor_table_ptr(struct x86_emulate_ctxt
*ctxt
,
1473 u16 selector
, struct desc_ptr
*dt
)
1475 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1478 if (selector
& 1 << 2) {
1479 struct desc_struct desc
;
1482 memset (dt
, 0, sizeof *dt
);
1483 if (!ops
->get_segment(ctxt
, &sel
, &desc
, &base3
,
1487 dt
->size
= desc_limit_scaled(&desc
); /* what if limit > 65535? */
1488 dt
->address
= get_desc_base(&desc
) | ((u64
)base3
<< 32);
1490 ops
->get_gdt(ctxt
, dt
);
1493 static int get_descriptor_ptr(struct x86_emulate_ctxt
*ctxt
,
1494 u16 selector
, ulong
*desc_addr_p
)
1497 u16 index
= selector
>> 3;
1500 get_descriptor_table_ptr(ctxt
, selector
, &dt
);
1502 if (dt
.size
< index
* 8 + 7)
1503 return emulate_gp(ctxt
, selector
& 0xfffc);
1505 addr
= dt
.address
+ index
* 8;
1507 #ifdef CONFIG_X86_64
1508 if (addr
>> 32 != 0) {
1511 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1512 if (!(efer
& EFER_LMA
))
1517 *desc_addr_p
= addr
;
1518 return X86EMUL_CONTINUE
;
1521 /* allowed just for 8 bytes segments */
1522 static int read_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1523 u16 selector
, struct desc_struct
*desc
,
1528 rc
= get_descriptor_ptr(ctxt
, selector
, desc_addr_p
);
1529 if (rc
!= X86EMUL_CONTINUE
)
1532 return ctxt
->ops
->read_std(ctxt
, *desc_addr_p
, desc
, sizeof(*desc
),
1536 /* allowed just for 8 bytes segments */
1537 static int write_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1538 u16 selector
, struct desc_struct
*desc
)
1543 rc
= get_descriptor_ptr(ctxt
, selector
, &addr
);
1544 if (rc
!= X86EMUL_CONTINUE
)
1547 return ctxt
->ops
->write_std(ctxt
, addr
, desc
, sizeof *desc
,
1551 /* Does not support long mode */
1552 static int __load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1553 u16 selector
, int seg
, u8 cpl
,
1554 enum x86_transfer_type transfer
,
1555 struct desc_struct
*desc
)
1557 struct desc_struct seg_desc
, old_desc
;
1559 unsigned err_vec
= GP_VECTOR
;
1561 bool null_selector
= !(selector
& ~0x3); /* 0000-0003 are null */
1567 memset(&seg_desc
, 0, sizeof seg_desc
);
1569 if (ctxt
->mode
== X86EMUL_MODE_REAL
) {
1570 /* set real mode segment descriptor (keep limit etc. for
1572 ctxt
->ops
->get_segment(ctxt
, &dummy
, &seg_desc
, NULL
, seg
);
1573 set_desc_base(&seg_desc
, selector
<< 4);
1575 } else if (seg
<= VCPU_SREG_GS
&& ctxt
->mode
== X86EMUL_MODE_VM86
) {
1576 /* VM86 needs a clean new segment descriptor */
1577 set_desc_base(&seg_desc
, selector
<< 4);
1578 set_desc_limit(&seg_desc
, 0xffff);
1588 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1589 if ((seg
== VCPU_SREG_CS
1590 || (seg
== VCPU_SREG_SS
1591 && (ctxt
->mode
!= X86EMUL_MODE_PROT64
|| rpl
!= cpl
))
1592 || seg
== VCPU_SREG_TR
)
1596 /* TR should be in GDT only */
1597 if (seg
== VCPU_SREG_TR
&& (selector
& (1 << 2)))
1600 if (null_selector
) /* for NULL selector skip all following checks */
1603 ret
= read_segment_descriptor(ctxt
, selector
, &seg_desc
, &desc_addr
);
1604 if (ret
!= X86EMUL_CONTINUE
)
1607 err_code
= selector
& 0xfffc;
1608 err_vec
= (transfer
== X86_TRANSFER_TASK_SWITCH
) ? TS_VECTOR
:
1611 /* can't load system descriptor into segment selector */
1612 if (seg
<= VCPU_SREG_GS
&& !seg_desc
.s
) {
1613 if (transfer
== X86_TRANSFER_CALL_JMP
)
1614 return X86EMUL_UNHANDLEABLE
;
1619 err_vec
= (seg
== VCPU_SREG_SS
) ? SS_VECTOR
: NP_VECTOR
;
1628 * segment is not a writable data segment or segment
1629 * selector's RPL != CPL or segment selector's RPL != CPL
1631 if (rpl
!= cpl
|| (seg_desc
.type
& 0xa) != 0x2 || dpl
!= cpl
)
1635 if (!(seg_desc
.type
& 8))
1638 if (seg_desc
.type
& 4) {
1644 if (rpl
> cpl
|| dpl
!= cpl
)
1647 /* in long-mode d/b must be clear if l is set */
1648 if (seg_desc
.d
&& seg_desc
.l
) {
1651 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1652 if (efer
& EFER_LMA
)
1656 /* CS(RPL) <- CPL */
1657 selector
= (selector
& 0xfffc) | cpl
;
1660 if (seg_desc
.s
|| (seg_desc
.type
!= 1 && seg_desc
.type
!= 9))
1662 old_desc
= seg_desc
;
1663 seg_desc
.type
|= 2; /* busy */
1664 ret
= ctxt
->ops
->cmpxchg_emulated(ctxt
, desc_addr
, &old_desc
, &seg_desc
,
1665 sizeof(seg_desc
), &ctxt
->exception
);
1666 if (ret
!= X86EMUL_CONTINUE
)
1669 case VCPU_SREG_LDTR
:
1670 if (seg_desc
.s
|| seg_desc
.type
!= 2)
1673 default: /* DS, ES, FS, or GS */
1675 * segment is not a data or readable code segment or
1676 * ((segment is a data or nonconforming code segment)
1677 * and (both RPL and CPL > DPL))
1679 if ((seg_desc
.type
& 0xa) == 0x8 ||
1680 (((seg_desc
.type
& 0xc) != 0xc) &&
1681 (rpl
> dpl
&& cpl
> dpl
)))
1687 /* mark segment as accessed */
1688 if (!(seg_desc
.type
& 1)) {
1690 ret
= write_segment_descriptor(ctxt
, selector
,
1692 if (ret
!= X86EMUL_CONTINUE
)
1695 } else if (ctxt
->mode
== X86EMUL_MODE_PROT64
) {
1696 ret
= ctxt
->ops
->read_std(ctxt
, desc_addr
+8, &base3
,
1697 sizeof(base3
), &ctxt
->exception
);
1698 if (ret
!= X86EMUL_CONTINUE
)
1700 if (is_noncanonical_address(get_desc_base(&seg_desc
) |
1701 ((u64
)base3
<< 32)))
1702 return emulate_gp(ctxt
, 0);
1705 ctxt
->ops
->set_segment(ctxt
, selector
, &seg_desc
, base3
, seg
);
1708 return X86EMUL_CONTINUE
;
1710 return emulate_exception(ctxt
, err_vec
, err_code
, true);
1713 static int load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1714 u16 selector
, int seg
)
1716 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
1717 return __load_segment_descriptor(ctxt
, selector
, seg
, cpl
,
1718 X86_TRANSFER_NONE
, NULL
);
1721 static void write_register_operand(struct operand
*op
)
1723 return assign_register(op
->addr
.reg
, op
->val
, op
->bytes
);
1726 static int writeback(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
)
1730 write_register_operand(op
);
1733 if (ctxt
->lock_prefix
)
1734 return segmented_cmpxchg(ctxt
,
1740 return segmented_write(ctxt
,
1746 return segmented_write(ctxt
,
1749 op
->bytes
* op
->count
);
1752 write_sse_reg(ctxt
, &op
->vec_val
, op
->addr
.xmm
);
1755 write_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
1763 return X86EMUL_CONTINUE
;
1766 static int push(struct x86_emulate_ctxt
*ctxt
, void *data
, int bytes
)
1768 struct segmented_address addr
;
1770 rsp_increment(ctxt
, -bytes
);
1771 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1772 addr
.seg
= VCPU_SREG_SS
;
1774 return segmented_write(ctxt
, addr
, data
, bytes
);
1777 static int em_push(struct x86_emulate_ctxt
*ctxt
)
1779 /* Disable writeback. */
1780 ctxt
->dst
.type
= OP_NONE
;
1781 return push(ctxt
, &ctxt
->src
.val
, ctxt
->op_bytes
);
1784 static int emulate_pop(struct x86_emulate_ctxt
*ctxt
,
1785 void *dest
, int len
)
1788 struct segmented_address addr
;
1790 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1791 addr
.seg
= VCPU_SREG_SS
;
1792 rc
= segmented_read(ctxt
, addr
, dest
, len
);
1793 if (rc
!= X86EMUL_CONTINUE
)
1796 rsp_increment(ctxt
, len
);
1800 static int em_pop(struct x86_emulate_ctxt
*ctxt
)
1802 return emulate_pop(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1805 static int emulate_popf(struct x86_emulate_ctxt
*ctxt
,
1806 void *dest
, int len
)
1809 unsigned long val
, change_mask
;
1810 int iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
1811 int cpl
= ctxt
->ops
->cpl(ctxt
);
1813 rc
= emulate_pop(ctxt
, &val
, len
);
1814 if (rc
!= X86EMUL_CONTINUE
)
1817 change_mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
1818 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_OF
|
1819 X86_EFLAGS_TF
| X86_EFLAGS_DF
| X86_EFLAGS_NT
|
1820 X86_EFLAGS_AC
| X86_EFLAGS_ID
;
1822 switch(ctxt
->mode
) {
1823 case X86EMUL_MODE_PROT64
:
1824 case X86EMUL_MODE_PROT32
:
1825 case X86EMUL_MODE_PROT16
:
1827 change_mask
|= X86_EFLAGS_IOPL
;
1829 change_mask
|= X86_EFLAGS_IF
;
1831 case X86EMUL_MODE_VM86
:
1833 return emulate_gp(ctxt
, 0);
1834 change_mask
|= X86_EFLAGS_IF
;
1836 default: /* real mode */
1837 change_mask
|= (X86_EFLAGS_IOPL
| X86_EFLAGS_IF
);
1841 *(unsigned long *)dest
=
1842 (ctxt
->eflags
& ~change_mask
) | (val
& change_mask
);
1847 static int em_popf(struct x86_emulate_ctxt
*ctxt
)
1849 ctxt
->dst
.type
= OP_REG
;
1850 ctxt
->dst
.addr
.reg
= &ctxt
->eflags
;
1851 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
1852 return emulate_popf(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1855 static int em_enter(struct x86_emulate_ctxt
*ctxt
)
1858 unsigned frame_size
= ctxt
->src
.val
;
1859 unsigned nesting_level
= ctxt
->src2
.val
& 31;
1863 return X86EMUL_UNHANDLEABLE
;
1865 rbp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1866 rc
= push(ctxt
, &rbp
, stack_size(ctxt
));
1867 if (rc
!= X86EMUL_CONTINUE
)
1869 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RBP
), reg_read(ctxt
, VCPU_REGS_RSP
),
1871 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
),
1872 reg_read(ctxt
, VCPU_REGS_RSP
) - frame_size
,
1874 return X86EMUL_CONTINUE
;
1877 static int em_leave(struct x86_emulate_ctxt
*ctxt
)
1879 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
), reg_read(ctxt
, VCPU_REGS_RBP
),
1881 return emulate_pop(ctxt
, reg_rmw(ctxt
, VCPU_REGS_RBP
), ctxt
->op_bytes
);
1884 static int em_push_sreg(struct x86_emulate_ctxt
*ctxt
)
1886 int seg
= ctxt
->src2
.val
;
1888 ctxt
->src
.val
= get_segment_selector(ctxt
, seg
);
1889 if (ctxt
->op_bytes
== 4) {
1890 rsp_increment(ctxt
, -2);
1894 return em_push(ctxt
);
1897 static int em_pop_sreg(struct x86_emulate_ctxt
*ctxt
)
1899 int seg
= ctxt
->src2
.val
;
1900 unsigned long selector
;
1903 rc
= emulate_pop(ctxt
, &selector
, 2);
1904 if (rc
!= X86EMUL_CONTINUE
)
1907 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
1908 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
1909 if (ctxt
->op_bytes
> 2)
1910 rsp_increment(ctxt
, ctxt
->op_bytes
- 2);
1912 rc
= load_segment_descriptor(ctxt
, (u16
)selector
, seg
);
1916 static int em_pusha(struct x86_emulate_ctxt
*ctxt
)
1918 unsigned long old_esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
1919 int rc
= X86EMUL_CONTINUE
;
1920 int reg
= VCPU_REGS_RAX
;
1922 while (reg
<= VCPU_REGS_RDI
) {
1923 (reg
== VCPU_REGS_RSP
) ?
1924 (ctxt
->src
.val
= old_esp
) : (ctxt
->src
.val
= reg_read(ctxt
, reg
));
1927 if (rc
!= X86EMUL_CONTINUE
)
1936 static int em_pushf(struct x86_emulate_ctxt
*ctxt
)
1938 ctxt
->src
.val
= (unsigned long)ctxt
->eflags
& ~X86_EFLAGS_VM
;
1939 return em_push(ctxt
);
1942 static int em_popa(struct x86_emulate_ctxt
*ctxt
)
1944 int rc
= X86EMUL_CONTINUE
;
1945 int reg
= VCPU_REGS_RDI
;
1948 while (reg
>= VCPU_REGS_RAX
) {
1949 if (reg
== VCPU_REGS_RSP
) {
1950 rsp_increment(ctxt
, ctxt
->op_bytes
);
1954 rc
= emulate_pop(ctxt
, &val
, ctxt
->op_bytes
);
1955 if (rc
!= X86EMUL_CONTINUE
)
1957 assign_register(reg_rmw(ctxt
, reg
), val
, ctxt
->op_bytes
);
1963 static int __emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
1965 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1972 /* TODO: Add limit checks */
1973 ctxt
->src
.val
= ctxt
->eflags
;
1975 if (rc
!= X86EMUL_CONTINUE
)
1978 ctxt
->eflags
&= ~(X86_EFLAGS_IF
| X86_EFLAGS_TF
| X86_EFLAGS_AC
);
1980 ctxt
->src
.val
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
1982 if (rc
!= X86EMUL_CONTINUE
)
1985 ctxt
->src
.val
= ctxt
->_eip
;
1987 if (rc
!= X86EMUL_CONTINUE
)
1990 ops
->get_idt(ctxt
, &dt
);
1992 eip_addr
= dt
.address
+ (irq
<< 2);
1993 cs_addr
= dt
.address
+ (irq
<< 2) + 2;
1995 rc
= ops
->read_std(ctxt
, cs_addr
, &cs
, 2, &ctxt
->exception
);
1996 if (rc
!= X86EMUL_CONTINUE
)
1999 rc
= ops
->read_std(ctxt
, eip_addr
, &eip
, 2, &ctxt
->exception
);
2000 if (rc
!= X86EMUL_CONTINUE
)
2003 rc
= load_segment_descriptor(ctxt
, cs
, VCPU_SREG_CS
);
2004 if (rc
!= X86EMUL_CONTINUE
)
2012 int emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2016 invalidate_registers(ctxt
);
2017 rc
= __emulate_int_real(ctxt
, irq
);
2018 if (rc
== X86EMUL_CONTINUE
)
2019 writeback_registers(ctxt
);
2023 static int emulate_int(struct x86_emulate_ctxt
*ctxt
, int irq
)
2025 switch(ctxt
->mode
) {
2026 case X86EMUL_MODE_REAL
:
2027 return __emulate_int_real(ctxt
, irq
);
2028 case X86EMUL_MODE_VM86
:
2029 case X86EMUL_MODE_PROT16
:
2030 case X86EMUL_MODE_PROT32
:
2031 case X86EMUL_MODE_PROT64
:
2033 /* Protected mode interrupts unimplemented yet */
2034 return X86EMUL_UNHANDLEABLE
;
2038 static int emulate_iret_real(struct x86_emulate_ctxt
*ctxt
)
2040 int rc
= X86EMUL_CONTINUE
;
2041 unsigned long temp_eip
= 0;
2042 unsigned long temp_eflags
= 0;
2043 unsigned long cs
= 0;
2044 unsigned long mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
2045 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_TF
|
2046 X86_EFLAGS_IF
| X86_EFLAGS_DF
| X86_EFLAGS_OF
|
2047 X86_EFLAGS_IOPL
| X86_EFLAGS_NT
| X86_EFLAGS_RF
|
2048 X86_EFLAGS_AC
| X86_EFLAGS_ID
|
2050 unsigned long vm86_mask
= X86_EFLAGS_VM
| X86_EFLAGS_VIF
|
2053 /* TODO: Add stack limit check */
2055 rc
= emulate_pop(ctxt
, &temp_eip
, ctxt
->op_bytes
);
2057 if (rc
!= X86EMUL_CONTINUE
)
2060 if (temp_eip
& ~0xffff)
2061 return emulate_gp(ctxt
, 0);
2063 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2065 if (rc
!= X86EMUL_CONTINUE
)
2068 rc
= emulate_pop(ctxt
, &temp_eflags
, ctxt
->op_bytes
);
2070 if (rc
!= X86EMUL_CONTINUE
)
2073 rc
= load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
);
2075 if (rc
!= X86EMUL_CONTINUE
)
2078 ctxt
->_eip
= temp_eip
;
2080 if (ctxt
->op_bytes
== 4)
2081 ctxt
->eflags
= ((temp_eflags
& mask
) | (ctxt
->eflags
& vm86_mask
));
2082 else if (ctxt
->op_bytes
== 2) {
2083 ctxt
->eflags
&= ~0xffff;
2084 ctxt
->eflags
|= temp_eflags
;
2087 ctxt
->eflags
&= ~EFLG_RESERVED_ZEROS_MASK
; /* Clear reserved zeros */
2088 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2089 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2094 static int em_iret(struct x86_emulate_ctxt
*ctxt
)
2096 switch(ctxt
->mode
) {
2097 case X86EMUL_MODE_REAL
:
2098 return emulate_iret_real(ctxt
);
2099 case X86EMUL_MODE_VM86
:
2100 case X86EMUL_MODE_PROT16
:
2101 case X86EMUL_MODE_PROT32
:
2102 case X86EMUL_MODE_PROT64
:
2104 /* iret from protected mode unimplemented yet */
2105 return X86EMUL_UNHANDLEABLE
;
2109 static int em_jmp_far(struct x86_emulate_ctxt
*ctxt
)
2112 unsigned short sel
, old_sel
;
2113 struct desc_struct old_desc
, new_desc
;
2114 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2115 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
2117 /* Assignment of RIP may only fail in 64-bit mode */
2118 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2119 ops
->get_segment(ctxt
, &old_sel
, &old_desc
, NULL
,
2122 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2124 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
2125 X86_TRANSFER_CALL_JMP
,
2127 if (rc
!= X86EMUL_CONTINUE
)
2130 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
2131 if (rc
!= X86EMUL_CONTINUE
) {
2132 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2133 /* assigning eip failed; restore the old cs */
2134 ops
->set_segment(ctxt
, old_sel
, &old_desc
, 0, VCPU_SREG_CS
);
2140 static int em_jmp_abs(struct x86_emulate_ctxt
*ctxt
)
2142 return assign_eip_near(ctxt
, ctxt
->src
.val
);
2145 static int em_call_near_abs(struct x86_emulate_ctxt
*ctxt
)
2150 old_eip
= ctxt
->_eip
;
2151 rc
= assign_eip_near(ctxt
, ctxt
->src
.val
);
2152 if (rc
!= X86EMUL_CONTINUE
)
2154 ctxt
->src
.val
= old_eip
;
2159 static int em_cmpxchg8b(struct x86_emulate_ctxt
*ctxt
)
2161 u64 old
= ctxt
->dst
.orig_val64
;
2163 if (ctxt
->dst
.bytes
== 16)
2164 return X86EMUL_UNHANDLEABLE
;
2166 if (((u32
) (old
>> 0) != (u32
) reg_read(ctxt
, VCPU_REGS_RAX
)) ||
2167 ((u32
) (old
>> 32) != (u32
) reg_read(ctxt
, VCPU_REGS_RDX
))) {
2168 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
) (old
>> 0);
2169 *reg_write(ctxt
, VCPU_REGS_RDX
) = (u32
) (old
>> 32);
2170 ctxt
->eflags
&= ~X86_EFLAGS_ZF
;
2172 ctxt
->dst
.val64
= ((u64
)reg_read(ctxt
, VCPU_REGS_RCX
) << 32) |
2173 (u32
) reg_read(ctxt
, VCPU_REGS_RBX
);
2175 ctxt
->eflags
|= X86_EFLAGS_ZF
;
2177 return X86EMUL_CONTINUE
;
2180 static int em_ret(struct x86_emulate_ctxt
*ctxt
)
2185 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2186 if (rc
!= X86EMUL_CONTINUE
)
2189 return assign_eip_near(ctxt
, eip
);
2192 static int em_ret_far(struct x86_emulate_ctxt
*ctxt
)
2195 unsigned long eip
, cs
;
2197 int cpl
= ctxt
->ops
->cpl(ctxt
);
2198 struct desc_struct old_desc
, new_desc
;
2199 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2201 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2202 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
,
2205 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2206 if (rc
!= X86EMUL_CONTINUE
)
2208 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2209 if (rc
!= X86EMUL_CONTINUE
)
2211 /* Outer-privilege level return is not implemented */
2212 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
&& (cs
& 3) > cpl
)
2213 return X86EMUL_UNHANDLEABLE
;
2214 rc
= __load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
, cpl
,
2217 if (rc
!= X86EMUL_CONTINUE
)
2219 rc
= assign_eip_far(ctxt
, eip
, &new_desc
);
2220 if (rc
!= X86EMUL_CONTINUE
) {
2221 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2222 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
2227 static int em_ret_far_imm(struct x86_emulate_ctxt
*ctxt
)
2231 rc
= em_ret_far(ctxt
);
2232 if (rc
!= X86EMUL_CONTINUE
)
2234 rsp_increment(ctxt
, ctxt
->src
.val
);
2235 return X86EMUL_CONTINUE
;
2238 static int em_cmpxchg(struct x86_emulate_ctxt
*ctxt
)
2240 /* Save real source value, then compare EAX against destination. */
2241 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
2242 ctxt
->dst
.val
= reg_read(ctxt
, VCPU_REGS_RAX
);
2243 ctxt
->src
.orig_val
= ctxt
->src
.val
;
2244 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2245 fastop(ctxt
, em_cmp
);
2247 if (ctxt
->eflags
& X86_EFLAGS_ZF
) {
2248 /* Success: write back to memory; no update of EAX */
2249 ctxt
->src
.type
= OP_NONE
;
2250 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
2252 /* Failure: write the value we saw to EAX. */
2253 ctxt
->src
.type
= OP_REG
;
2254 ctxt
->src
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
2255 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2256 /* Create write-cycle to dest by writing the same value */
2257 ctxt
->dst
.val
= ctxt
->dst
.orig_val
;
2259 return X86EMUL_CONTINUE
;
2262 static int em_lseg(struct x86_emulate_ctxt
*ctxt
)
2264 int seg
= ctxt
->src2
.val
;
2268 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2270 rc
= load_segment_descriptor(ctxt
, sel
, seg
);
2271 if (rc
!= X86EMUL_CONTINUE
)
2274 ctxt
->dst
.val
= ctxt
->src
.val
;
2278 static int emulator_has_longmode(struct x86_emulate_ctxt
*ctxt
)
2280 u32 eax
, ebx
, ecx
, edx
;
2284 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2285 return edx
& bit(X86_FEATURE_LM
);
2288 #define GET_SMSTATE(type, smbase, offset) \
2291 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2293 if (r != X86EMUL_CONTINUE) \
2294 return X86EMUL_UNHANDLEABLE; \
2298 static void rsm_set_desc_flags(struct desc_struct
*desc
, u32 flags
)
2300 desc
->g
= (flags
>> 23) & 1;
2301 desc
->d
= (flags
>> 22) & 1;
2302 desc
->l
= (flags
>> 21) & 1;
2303 desc
->avl
= (flags
>> 20) & 1;
2304 desc
->p
= (flags
>> 15) & 1;
2305 desc
->dpl
= (flags
>> 13) & 3;
2306 desc
->s
= (flags
>> 12) & 1;
2307 desc
->type
= (flags
>> 8) & 15;
2310 static int rsm_load_seg_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2312 struct desc_struct desc
;
2316 selector
= GET_SMSTATE(u32
, smbase
, 0x7fa8 + n
* 4);
2319 offset
= 0x7f84 + n
* 12;
2321 offset
= 0x7f2c + (n
- 3) * 12;
2323 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2324 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2325 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, offset
));
2326 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, n
);
2327 return X86EMUL_CONTINUE
;
2330 static int rsm_load_seg_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2332 struct desc_struct desc
;
2337 offset
= 0x7e00 + n
* 16;
2339 selector
= GET_SMSTATE(u16
, smbase
, offset
);
2340 rsm_set_desc_flags(&desc
, GET_SMSTATE(u16
, smbase
, offset
+ 2) << 8);
2341 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2342 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2343 base3
= GET_SMSTATE(u32
, smbase
, offset
+ 12);
2345 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, n
);
2346 return X86EMUL_CONTINUE
;
2349 static int rsm_enter_protected_mode(struct x86_emulate_ctxt
*ctxt
,
2355 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2356 * Then enable protected mode. However, PCID cannot be enabled
2357 * if EFER.LMA=0, so set it separately.
2359 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2361 return X86EMUL_UNHANDLEABLE
;
2363 bad
= ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
2365 return X86EMUL_UNHANDLEABLE
;
2367 if (cr4
& X86_CR4_PCIDE
) {
2368 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
);
2370 return X86EMUL_UNHANDLEABLE
;
2373 return X86EMUL_CONTINUE
;
2376 static int rsm_load_state_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2378 struct desc_struct desc
;
2384 cr0
= GET_SMSTATE(u32
, smbase
, 0x7ffc);
2385 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u32
, smbase
, 0x7ff8));
2386 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7ff4) | X86_EFLAGS_FIXED
;
2387 ctxt
->_eip
= GET_SMSTATE(u32
, smbase
, 0x7ff0);
2389 for (i
= 0; i
< 8; i
++)
2390 *reg_write(ctxt
, i
) = GET_SMSTATE(u32
, smbase
, 0x7fd0 + i
* 4);
2392 val
= GET_SMSTATE(u32
, smbase
, 0x7fcc);
2393 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2394 val
= GET_SMSTATE(u32
, smbase
, 0x7fc8);
2395 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2397 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc4);
2398 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f64));
2399 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f60));
2400 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f5c));
2401 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_TR
);
2403 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc0);
2404 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f80));
2405 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f7c));
2406 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f78));
2407 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_LDTR
);
2409 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f74);
2410 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f70);
2411 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2413 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f58);
2414 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f54);
2415 ctxt
->ops
->set_idt(ctxt
, &dt
);
2417 for (i
= 0; i
< 6; i
++) {
2418 int r
= rsm_load_seg_32(ctxt
, smbase
, i
);
2419 if (r
!= X86EMUL_CONTINUE
)
2423 cr4
= GET_SMSTATE(u32
, smbase
, 0x7f14);
2425 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7ef8));
2427 return rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2430 static int rsm_load_state_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2432 struct desc_struct desc
;
2439 for (i
= 0; i
< 16; i
++)
2440 *reg_write(ctxt
, i
) = GET_SMSTATE(u64
, smbase
, 0x7ff8 - i
* 8);
2442 ctxt
->_eip
= GET_SMSTATE(u64
, smbase
, 0x7f78);
2443 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7f70) | X86_EFLAGS_FIXED
;
2445 val
= GET_SMSTATE(u32
, smbase
, 0x7f68);
2446 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2447 val
= GET_SMSTATE(u32
, smbase
, 0x7f60);
2448 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2450 cr0
= GET_SMSTATE(u64
, smbase
, 0x7f58);
2451 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u64
, smbase
, 0x7f50));
2452 cr4
= GET_SMSTATE(u64
, smbase
, 0x7f48);
2453 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7f00));
2454 val
= GET_SMSTATE(u64
, smbase
, 0x7ed0);
2455 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, val
& ~EFER_LMA
);
2457 selector
= GET_SMSTATE(u32
, smbase
, 0x7e90);
2458 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e92) << 8);
2459 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e94));
2460 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e98));
2461 base3
= GET_SMSTATE(u32
, smbase
, 0x7e9c);
2462 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_TR
);
2464 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e84);
2465 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e88);
2466 ctxt
->ops
->set_idt(ctxt
, &dt
);
2468 selector
= GET_SMSTATE(u32
, smbase
, 0x7e70);
2469 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e72) << 8);
2470 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e74));
2471 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e78));
2472 base3
= GET_SMSTATE(u32
, smbase
, 0x7e7c);
2473 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_LDTR
);
2475 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e64);
2476 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e68);
2477 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2479 r
= rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2480 if (r
!= X86EMUL_CONTINUE
)
2483 for (i
= 0; i
< 6; i
++) {
2484 r
= rsm_load_seg_64(ctxt
, smbase
, i
);
2485 if (r
!= X86EMUL_CONTINUE
)
2489 return X86EMUL_CONTINUE
;
2492 static int em_rsm(struct x86_emulate_ctxt
*ctxt
)
2494 unsigned long cr0
, cr4
, efer
;
2498 if ((ctxt
->emul_flags
& X86EMUL_SMM_MASK
) == 0)
2499 return emulate_ud(ctxt
);
2502 * Get back to real mode, to prepare a safe state in which to load
2503 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2504 * supports long mode.
2506 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
2507 if (emulator_has_longmode(ctxt
)) {
2508 struct desc_struct cs_desc
;
2510 /* Zero CR4.PCIDE before CR0.PG. */
2511 if (cr4
& X86_CR4_PCIDE
) {
2512 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2513 cr4
&= ~X86_CR4_PCIDE
;
2516 /* A 32-bit code segment is required to clear EFER.LMA. */
2517 memset(&cs_desc
, 0, sizeof(cs_desc
));
2519 cs_desc
.s
= cs_desc
.g
= cs_desc
.p
= 1;
2520 ctxt
->ops
->set_segment(ctxt
, 0, &cs_desc
, 0, VCPU_SREG_CS
);
2523 /* For the 64-bit case, this will clear EFER.LMA. */
2524 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
2525 if (cr0
& X86_CR0_PE
)
2526 ctxt
->ops
->set_cr(ctxt
, 0, cr0
& ~(X86_CR0_PG
| X86_CR0_PE
));
2528 /* Now clear CR4.PAE (which must be done before clearing EFER.LME). */
2529 if (cr4
& X86_CR4_PAE
)
2530 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PAE
);
2532 /* And finally go back to 32-bit mode. */
2534 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, efer
);
2536 smbase
= ctxt
->ops
->get_smbase(ctxt
);
2537 if (emulator_has_longmode(ctxt
))
2538 ret
= rsm_load_state_64(ctxt
, smbase
+ 0x8000);
2540 ret
= rsm_load_state_32(ctxt
, smbase
+ 0x8000);
2542 if (ret
!= X86EMUL_CONTINUE
) {
2543 /* FIXME: should triple fault */
2544 return X86EMUL_UNHANDLEABLE
;
2547 if ((ctxt
->emul_flags
& X86EMUL_SMM_INSIDE_NMI_MASK
) == 0)
2548 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2550 ctxt
->emul_flags
&= ~X86EMUL_SMM_INSIDE_NMI_MASK
;
2551 ctxt
->emul_flags
&= ~X86EMUL_SMM_MASK
;
2552 return X86EMUL_CONTINUE
;
2556 setup_syscalls_segments(struct x86_emulate_ctxt
*ctxt
,
2557 struct desc_struct
*cs
, struct desc_struct
*ss
)
2559 cs
->l
= 0; /* will be adjusted later */
2560 set_desc_base(cs
, 0); /* flat segment */
2561 cs
->g
= 1; /* 4kb granularity */
2562 set_desc_limit(cs
, 0xfffff); /* 4GB limit */
2563 cs
->type
= 0x0b; /* Read, Execute, Accessed */
2565 cs
->dpl
= 0; /* will be adjusted later */
2570 set_desc_base(ss
, 0); /* flat segment */
2571 set_desc_limit(ss
, 0xfffff); /* 4GB limit */
2572 ss
->g
= 1; /* 4kb granularity */
2574 ss
->type
= 0x03; /* Read/Write, Accessed */
2575 ss
->d
= 1; /* 32bit stack segment */
2582 static bool vendor_intel(struct x86_emulate_ctxt
*ctxt
)
2584 u32 eax
, ebx
, ecx
, edx
;
2587 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2588 return ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2589 && ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2590 && edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
;
2593 static bool em_syscall_is_enabled(struct x86_emulate_ctxt
*ctxt
)
2595 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2596 u32 eax
, ebx
, ecx
, edx
;
2599 * syscall should always be enabled in longmode - so only become
2600 * vendor specific (cpuid) if other modes are active...
2602 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2607 ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2609 * Intel ("GenuineIntel")
2610 * remark: Intel CPUs only support "syscall" in 64bit
2611 * longmode. Also an 64bit guest with a
2612 * 32bit compat-app running will #UD !! While this
2613 * behaviour can be fixed (by emulating) into AMD
2614 * response - CPUs of AMD can't behave like Intel.
2616 if (ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
&&
2617 ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
&&
2618 edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
)
2621 /* AMD ("AuthenticAMD") */
2622 if (ebx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx
&&
2623 ecx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx
&&
2624 edx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_edx
)
2627 /* AMD ("AMDisbetter!") */
2628 if (ebx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx
&&
2629 ecx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx
&&
2630 edx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_edx
)
2633 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2637 static int em_syscall(struct x86_emulate_ctxt
*ctxt
)
2639 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2640 struct desc_struct cs
, ss
;
2645 /* syscall is not available in real mode */
2646 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2647 ctxt
->mode
== X86EMUL_MODE_VM86
)
2648 return emulate_ud(ctxt
);
2650 if (!(em_syscall_is_enabled(ctxt
)))
2651 return emulate_ud(ctxt
);
2653 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2654 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2656 if (!(efer
& EFER_SCE
))
2657 return emulate_ud(ctxt
);
2659 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2661 cs_sel
= (u16
)(msr_data
& 0xfffc);
2662 ss_sel
= (u16
)(msr_data
+ 8);
2664 if (efer
& EFER_LMA
) {
2668 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2669 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2671 *reg_write(ctxt
, VCPU_REGS_RCX
) = ctxt
->_eip
;
2672 if (efer
& EFER_LMA
) {
2673 #ifdef CONFIG_X86_64
2674 *reg_write(ctxt
, VCPU_REGS_R11
) = ctxt
->eflags
;
2677 ctxt
->mode
== X86EMUL_MODE_PROT64
?
2678 MSR_LSTAR
: MSR_CSTAR
, &msr_data
);
2679 ctxt
->_eip
= msr_data
;
2681 ops
->get_msr(ctxt
, MSR_SYSCALL_MASK
, &msr_data
);
2682 ctxt
->eflags
&= ~msr_data
;
2683 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2687 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2688 ctxt
->_eip
= (u32
)msr_data
;
2690 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2693 return X86EMUL_CONTINUE
;
2696 static int em_sysenter(struct x86_emulate_ctxt
*ctxt
)
2698 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2699 struct desc_struct cs
, ss
;
2704 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2705 /* inject #GP if in real mode */
2706 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2707 return emulate_gp(ctxt
, 0);
2710 * Not recognized on AMD in compat mode (but is recognized in legacy
2713 if ((ctxt
->mode
!= X86EMUL_MODE_PROT64
) && (efer
& EFER_LMA
)
2714 && !vendor_intel(ctxt
))
2715 return emulate_ud(ctxt
);
2717 /* sysenter/sysexit have not been tested in 64bit mode. */
2718 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2719 return X86EMUL_UNHANDLEABLE
;
2721 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2723 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2724 if ((msr_data
& 0xfffc) == 0x0)
2725 return emulate_gp(ctxt
, 0);
2727 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2728 cs_sel
= (u16
)msr_data
& ~SEGMENT_RPL_MASK
;
2729 ss_sel
= cs_sel
+ 8;
2730 if (efer
& EFER_LMA
) {
2735 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2736 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2738 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_EIP
, &msr_data
);
2739 ctxt
->_eip
= (efer
& EFER_LMA
) ? msr_data
: (u32
)msr_data
;
2741 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_ESP
, &msr_data
);
2742 *reg_write(ctxt
, VCPU_REGS_RSP
) = (efer
& EFER_LMA
) ? msr_data
:
2745 return X86EMUL_CONTINUE
;
2748 static int em_sysexit(struct x86_emulate_ctxt
*ctxt
)
2750 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2751 struct desc_struct cs
, ss
;
2752 u64 msr_data
, rcx
, rdx
;
2754 u16 cs_sel
= 0, ss_sel
= 0;
2756 /* inject #GP if in real mode or Virtual 8086 mode */
2757 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2758 ctxt
->mode
== X86EMUL_MODE_VM86
)
2759 return emulate_gp(ctxt
, 0);
2761 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2763 if ((ctxt
->rex_prefix
& 0x8) != 0x0)
2764 usermode
= X86EMUL_MODE_PROT64
;
2766 usermode
= X86EMUL_MODE_PROT32
;
2768 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2769 rdx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2773 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2775 case X86EMUL_MODE_PROT32
:
2776 cs_sel
= (u16
)(msr_data
+ 16);
2777 if ((msr_data
& 0xfffc) == 0x0)
2778 return emulate_gp(ctxt
, 0);
2779 ss_sel
= (u16
)(msr_data
+ 24);
2783 case X86EMUL_MODE_PROT64
:
2784 cs_sel
= (u16
)(msr_data
+ 32);
2785 if (msr_data
== 0x0)
2786 return emulate_gp(ctxt
, 0);
2787 ss_sel
= cs_sel
+ 8;
2790 if (is_noncanonical_address(rcx
) ||
2791 is_noncanonical_address(rdx
))
2792 return emulate_gp(ctxt
, 0);
2795 cs_sel
|= SEGMENT_RPL_MASK
;
2796 ss_sel
|= SEGMENT_RPL_MASK
;
2798 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2799 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2802 *reg_write(ctxt
, VCPU_REGS_RSP
) = rcx
;
2804 return X86EMUL_CONTINUE
;
2807 static bool emulator_bad_iopl(struct x86_emulate_ctxt
*ctxt
)
2810 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2812 if (ctxt
->mode
== X86EMUL_MODE_VM86
)
2814 iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
2815 return ctxt
->ops
->cpl(ctxt
) > iopl
;
2818 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt
*ctxt
,
2821 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2822 struct desc_struct tr_seg
;
2825 u16 tr
, io_bitmap_ptr
, perm
, bit_idx
= port
& 0x7;
2826 unsigned mask
= (1 << len
) - 1;
2829 ops
->get_segment(ctxt
, &tr
, &tr_seg
, &base3
, VCPU_SREG_TR
);
2832 if (desc_limit_scaled(&tr_seg
) < 103)
2834 base
= get_desc_base(&tr_seg
);
2835 #ifdef CONFIG_X86_64
2836 base
|= ((u64
)base3
) << 32;
2838 r
= ops
->read_std(ctxt
, base
+ 102, &io_bitmap_ptr
, 2, NULL
);
2839 if (r
!= X86EMUL_CONTINUE
)
2841 if (io_bitmap_ptr
+ port
/8 > desc_limit_scaled(&tr_seg
))
2843 r
= ops
->read_std(ctxt
, base
+ io_bitmap_ptr
+ port
/8, &perm
, 2, NULL
);
2844 if (r
!= X86EMUL_CONTINUE
)
2846 if ((perm
>> bit_idx
) & mask
)
2851 static bool emulator_io_permited(struct x86_emulate_ctxt
*ctxt
,
2857 if (emulator_bad_iopl(ctxt
))
2858 if (!emulator_io_port_access_allowed(ctxt
, port
, len
))
2861 ctxt
->perm_ok
= true;
2866 static void string_registers_quirk(struct x86_emulate_ctxt
*ctxt
)
2869 * Intel CPUs mask the counter and pointers in quite strange
2870 * manner when ECX is zero due to REP-string optimizations.
2872 #ifdef CONFIG_X86_64
2873 if (ctxt
->ad_bytes
!= 4 || !vendor_intel(ctxt
))
2876 *reg_write(ctxt
, VCPU_REGS_RCX
) = 0;
2879 case 0xa4: /* movsb */
2880 case 0xa5: /* movsd/w */
2881 *reg_rmw(ctxt
, VCPU_REGS_RSI
) &= (u32
)-1;
2883 case 0xaa: /* stosb */
2884 case 0xab: /* stosd/w */
2885 *reg_rmw(ctxt
, VCPU_REGS_RDI
) &= (u32
)-1;
2890 static void save_state_to_tss16(struct x86_emulate_ctxt
*ctxt
,
2891 struct tss_segment_16
*tss
)
2893 tss
->ip
= ctxt
->_eip
;
2894 tss
->flag
= ctxt
->eflags
;
2895 tss
->ax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2896 tss
->cx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2897 tss
->dx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2898 tss
->bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2899 tss
->sp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2900 tss
->bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2901 tss
->si
= reg_read(ctxt
, VCPU_REGS_RSI
);
2902 tss
->di
= reg_read(ctxt
, VCPU_REGS_RDI
);
2904 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2905 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2906 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2907 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2908 tss
->ldt
= get_segment_selector(ctxt
, VCPU_SREG_LDTR
);
2911 static int load_state_from_tss16(struct x86_emulate_ctxt
*ctxt
,
2912 struct tss_segment_16
*tss
)
2917 ctxt
->_eip
= tss
->ip
;
2918 ctxt
->eflags
= tss
->flag
| 2;
2919 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->ax
;
2920 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->cx
;
2921 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->dx
;
2922 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->bx
;
2923 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->sp
;
2924 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->bp
;
2925 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->si
;
2926 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->di
;
2929 * SDM says that segment selectors are loaded before segment
2932 set_segment_selector(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
);
2933 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
2934 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
2935 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
2936 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
2941 * Now load segment descriptors. If fault happens at this stage
2942 * it is handled in a context of new task
2944 ret
= __load_segment_descriptor(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
, cpl
,
2945 X86_TRANSFER_TASK_SWITCH
, NULL
);
2946 if (ret
!= X86EMUL_CONTINUE
)
2948 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
2949 X86_TRANSFER_TASK_SWITCH
, NULL
);
2950 if (ret
!= X86EMUL_CONTINUE
)
2952 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
2953 X86_TRANSFER_TASK_SWITCH
, NULL
);
2954 if (ret
!= X86EMUL_CONTINUE
)
2956 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
2957 X86_TRANSFER_TASK_SWITCH
, NULL
);
2958 if (ret
!= X86EMUL_CONTINUE
)
2960 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
2961 X86_TRANSFER_TASK_SWITCH
, NULL
);
2962 if (ret
!= X86EMUL_CONTINUE
)
2965 return X86EMUL_CONTINUE
;
2968 static int task_switch_16(struct x86_emulate_ctxt
*ctxt
,
2969 u16 tss_selector
, u16 old_tss_sel
,
2970 ulong old_tss_base
, struct desc_struct
*new_desc
)
2972 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2973 struct tss_segment_16 tss_seg
;
2975 u32 new_tss_base
= get_desc_base(new_desc
);
2977 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
2979 if (ret
!= X86EMUL_CONTINUE
)
2982 save_state_to_tss16(ctxt
, &tss_seg
);
2984 ret
= ops
->write_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
2986 if (ret
!= X86EMUL_CONTINUE
)
2989 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
2991 if (ret
!= X86EMUL_CONTINUE
)
2994 if (old_tss_sel
!= 0xffff) {
2995 tss_seg
.prev_task_link
= old_tss_sel
;
2997 ret
= ops
->write_std(ctxt
, new_tss_base
,
2998 &tss_seg
.prev_task_link
,
2999 sizeof tss_seg
.prev_task_link
,
3001 if (ret
!= X86EMUL_CONTINUE
)
3005 return load_state_from_tss16(ctxt
, &tss_seg
);
3008 static void save_state_to_tss32(struct x86_emulate_ctxt
*ctxt
,
3009 struct tss_segment_32
*tss
)
3011 /* CR3 and ldt selector are not saved intentionally */
3012 tss
->eip
= ctxt
->_eip
;
3013 tss
->eflags
= ctxt
->eflags
;
3014 tss
->eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3015 tss
->ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3016 tss
->edx
= reg_read(ctxt
, VCPU_REGS_RDX
);
3017 tss
->ebx
= reg_read(ctxt
, VCPU_REGS_RBX
);
3018 tss
->esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
3019 tss
->ebp
= reg_read(ctxt
, VCPU_REGS_RBP
);
3020 tss
->esi
= reg_read(ctxt
, VCPU_REGS_RSI
);
3021 tss
->edi
= reg_read(ctxt
, VCPU_REGS_RDI
);
3023 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
3024 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
3025 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
3026 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
3027 tss
->fs
= get_segment_selector(ctxt
, VCPU_SREG_FS
);
3028 tss
->gs
= get_segment_selector(ctxt
, VCPU_SREG_GS
);
3031 static int load_state_from_tss32(struct x86_emulate_ctxt
*ctxt
,
3032 struct tss_segment_32
*tss
)
3037 if (ctxt
->ops
->set_cr(ctxt
, 3, tss
->cr3
))
3038 return emulate_gp(ctxt
, 0);
3039 ctxt
->_eip
= tss
->eip
;
3040 ctxt
->eflags
= tss
->eflags
| 2;
3042 /* General purpose registers */
3043 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->eax
;
3044 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->ecx
;
3045 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->edx
;
3046 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->ebx
;
3047 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->esp
;
3048 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->ebp
;
3049 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->esi
;
3050 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->edi
;
3053 * SDM says that segment selectors are loaded before segment
3054 * descriptors. This is important because CPL checks will
3057 set_segment_selector(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
);
3058 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
3059 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
3060 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
3061 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
3062 set_segment_selector(ctxt
, tss
->fs
, VCPU_SREG_FS
);
3063 set_segment_selector(ctxt
, tss
->gs
, VCPU_SREG_GS
);
3066 * If we're switching between Protected Mode and VM86, we need to make
3067 * sure to update the mode before loading the segment descriptors so
3068 * that the selectors are interpreted correctly.
3070 if (ctxt
->eflags
& X86_EFLAGS_VM
) {
3071 ctxt
->mode
= X86EMUL_MODE_VM86
;
3074 ctxt
->mode
= X86EMUL_MODE_PROT32
;
3079 * Now load segment descriptors. If fault happenes at this stage
3080 * it is handled in a context of new task
3082 ret
= __load_segment_descriptor(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
,
3083 cpl
, X86_TRANSFER_TASK_SWITCH
, NULL
);
3084 if (ret
!= X86EMUL_CONTINUE
)
3086 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3087 X86_TRANSFER_TASK_SWITCH
, NULL
);
3088 if (ret
!= X86EMUL_CONTINUE
)
3090 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3091 X86_TRANSFER_TASK_SWITCH
, NULL
);
3092 if (ret
!= X86EMUL_CONTINUE
)
3094 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3095 X86_TRANSFER_TASK_SWITCH
, NULL
);
3096 if (ret
!= X86EMUL_CONTINUE
)
3098 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3099 X86_TRANSFER_TASK_SWITCH
, NULL
);
3100 if (ret
!= X86EMUL_CONTINUE
)
3102 ret
= __load_segment_descriptor(ctxt
, tss
->fs
, VCPU_SREG_FS
, cpl
,
3103 X86_TRANSFER_TASK_SWITCH
, NULL
);
3104 if (ret
!= X86EMUL_CONTINUE
)
3106 ret
= __load_segment_descriptor(ctxt
, tss
->gs
, VCPU_SREG_GS
, cpl
,
3107 X86_TRANSFER_TASK_SWITCH
, NULL
);
3112 static int task_switch_32(struct x86_emulate_ctxt
*ctxt
,
3113 u16 tss_selector
, u16 old_tss_sel
,
3114 ulong old_tss_base
, struct desc_struct
*new_desc
)
3116 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3117 struct tss_segment_32 tss_seg
;
3119 u32 new_tss_base
= get_desc_base(new_desc
);
3120 u32 eip_offset
= offsetof(struct tss_segment_32
, eip
);
3121 u32 ldt_sel_offset
= offsetof(struct tss_segment_32
, ldt_selector
);
3123 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3125 if (ret
!= X86EMUL_CONTINUE
)
3128 save_state_to_tss32(ctxt
, &tss_seg
);
3130 /* Only GP registers and segment selectors are saved */
3131 ret
= ops
->write_std(ctxt
, old_tss_base
+ eip_offset
, &tss_seg
.eip
,
3132 ldt_sel_offset
- eip_offset
, &ctxt
->exception
);
3133 if (ret
!= X86EMUL_CONTINUE
)
3136 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3138 if (ret
!= X86EMUL_CONTINUE
)
3141 if (old_tss_sel
!= 0xffff) {
3142 tss_seg
.prev_task_link
= old_tss_sel
;
3144 ret
= ops
->write_std(ctxt
, new_tss_base
,
3145 &tss_seg
.prev_task_link
,
3146 sizeof tss_seg
.prev_task_link
,
3148 if (ret
!= X86EMUL_CONTINUE
)
3152 return load_state_from_tss32(ctxt
, &tss_seg
);
3155 static int emulator_do_task_switch(struct x86_emulate_ctxt
*ctxt
,
3156 u16 tss_selector
, int idt_index
, int reason
,
3157 bool has_error_code
, u32 error_code
)
3159 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3160 struct desc_struct curr_tss_desc
, next_tss_desc
;
3162 u16 old_tss_sel
= get_segment_selector(ctxt
, VCPU_SREG_TR
);
3163 ulong old_tss_base
=
3164 ops
->get_cached_segment_base(ctxt
, VCPU_SREG_TR
);
3166 ulong desc_addr
, dr7
;
3168 /* FIXME: old_tss_base == ~0 ? */
3170 ret
= read_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
, &desc_addr
);
3171 if (ret
!= X86EMUL_CONTINUE
)
3173 ret
= read_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
, &desc_addr
);
3174 if (ret
!= X86EMUL_CONTINUE
)
3177 /* FIXME: check that next_tss_desc is tss */
3180 * Check privileges. The three cases are task switch caused by...
3182 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3183 * 2. Exception/IRQ/iret: No check is performed
3184 * 3. jmp/call to TSS/task-gate: No check is performed since the
3185 * hardware checks it before exiting.
3187 if (reason
== TASK_SWITCH_GATE
) {
3188 if (idt_index
!= -1) {
3189 /* Software interrupts */
3190 struct desc_struct task_gate_desc
;
3193 ret
= read_interrupt_descriptor(ctxt
, idt_index
,
3195 if (ret
!= X86EMUL_CONTINUE
)
3198 dpl
= task_gate_desc
.dpl
;
3199 if ((tss_selector
& 3) > dpl
|| ops
->cpl(ctxt
) > dpl
)
3200 return emulate_gp(ctxt
, (idt_index
<< 3) | 0x2);
3204 desc_limit
= desc_limit_scaled(&next_tss_desc
);
3205 if (!next_tss_desc
.p
||
3206 ((desc_limit
< 0x67 && (next_tss_desc
.type
& 8)) ||
3207 desc_limit
< 0x2b)) {
3208 return emulate_ts(ctxt
, tss_selector
& 0xfffc);
3211 if (reason
== TASK_SWITCH_IRET
|| reason
== TASK_SWITCH_JMP
) {
3212 curr_tss_desc
.type
&= ~(1 << 1); /* clear busy flag */
3213 write_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
);
3216 if (reason
== TASK_SWITCH_IRET
)
3217 ctxt
->eflags
= ctxt
->eflags
& ~X86_EFLAGS_NT
;
3219 /* set back link to prev task only if NT bit is set in eflags
3220 note that old_tss_sel is not used after this point */
3221 if (reason
!= TASK_SWITCH_CALL
&& reason
!= TASK_SWITCH_GATE
)
3222 old_tss_sel
= 0xffff;
3224 if (next_tss_desc
.type
& 8)
3225 ret
= task_switch_32(ctxt
, tss_selector
, old_tss_sel
,
3226 old_tss_base
, &next_tss_desc
);
3228 ret
= task_switch_16(ctxt
, tss_selector
, old_tss_sel
,
3229 old_tss_base
, &next_tss_desc
);
3230 if (ret
!= X86EMUL_CONTINUE
)
3233 if (reason
== TASK_SWITCH_CALL
|| reason
== TASK_SWITCH_GATE
)
3234 ctxt
->eflags
= ctxt
->eflags
| X86_EFLAGS_NT
;
3236 if (reason
!= TASK_SWITCH_IRET
) {
3237 next_tss_desc
.type
|= (1 << 1); /* set busy flag */
3238 write_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
);
3241 ops
->set_cr(ctxt
, 0, ops
->get_cr(ctxt
, 0) | X86_CR0_TS
);
3242 ops
->set_segment(ctxt
, tss_selector
, &next_tss_desc
, 0, VCPU_SREG_TR
);
3244 if (has_error_code
) {
3245 ctxt
->op_bytes
= ctxt
->ad_bytes
= (next_tss_desc
.type
& 8) ? 4 : 2;
3246 ctxt
->lock_prefix
= 0;
3247 ctxt
->src
.val
= (unsigned long) error_code
;
3248 ret
= em_push(ctxt
);
3251 ops
->get_dr(ctxt
, 7, &dr7
);
3252 ops
->set_dr(ctxt
, 7, dr7
& ~(DR_LOCAL_ENABLE_MASK
| DR_LOCAL_SLOWDOWN
));
3257 int emulator_task_switch(struct x86_emulate_ctxt
*ctxt
,
3258 u16 tss_selector
, int idt_index
, int reason
,
3259 bool has_error_code
, u32 error_code
)
3263 invalidate_registers(ctxt
);
3264 ctxt
->_eip
= ctxt
->eip
;
3265 ctxt
->dst
.type
= OP_NONE
;
3267 rc
= emulator_do_task_switch(ctxt
, tss_selector
, idt_index
, reason
,
3268 has_error_code
, error_code
);
3270 if (rc
== X86EMUL_CONTINUE
) {
3271 ctxt
->eip
= ctxt
->_eip
;
3272 writeback_registers(ctxt
);
3275 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
3278 static void string_addr_inc(struct x86_emulate_ctxt
*ctxt
, int reg
,
3281 int df
= (ctxt
->eflags
& X86_EFLAGS_DF
) ? -op
->count
: op
->count
;
3283 register_address_increment(ctxt
, reg
, df
* op
->bytes
);
3284 op
->addr
.mem
.ea
= register_address(ctxt
, reg
);
3287 static int em_das(struct x86_emulate_ctxt
*ctxt
)
3290 bool af
, cf
, old_cf
;
3292 cf
= ctxt
->eflags
& X86_EFLAGS_CF
;
3298 af
= ctxt
->eflags
& X86_EFLAGS_AF
;
3299 if ((al
& 0x0f) > 9 || af
) {
3301 cf
= old_cf
| (al
>= 250);
3306 if (old_al
> 0x99 || old_cf
) {
3312 /* Set PF, ZF, SF */
3313 ctxt
->src
.type
= OP_IMM
;
3315 ctxt
->src
.bytes
= 1;
3316 fastop(ctxt
, em_or
);
3317 ctxt
->eflags
&= ~(X86_EFLAGS_AF
| X86_EFLAGS_CF
);
3319 ctxt
->eflags
|= X86_EFLAGS_CF
;
3321 ctxt
->eflags
|= X86_EFLAGS_AF
;
3322 return X86EMUL_CONTINUE
;
3325 static int em_aam(struct x86_emulate_ctxt
*ctxt
)
3329 if (ctxt
->src
.val
== 0)
3330 return emulate_de(ctxt
);
3332 al
= ctxt
->dst
.val
& 0xff;
3333 ah
= al
/ ctxt
->src
.val
;
3334 al
%= ctxt
->src
.val
;
3336 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
| (ah
<< 8);
3338 /* Set PF, ZF, SF */
3339 ctxt
->src
.type
= OP_IMM
;
3341 ctxt
->src
.bytes
= 1;
3342 fastop(ctxt
, em_or
);
3344 return X86EMUL_CONTINUE
;
3347 static int em_aad(struct x86_emulate_ctxt
*ctxt
)
3349 u8 al
= ctxt
->dst
.val
& 0xff;
3350 u8 ah
= (ctxt
->dst
.val
>> 8) & 0xff;
3352 al
= (al
+ (ah
* ctxt
->src
.val
)) & 0xff;
3354 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
;
3356 /* Set PF, ZF, SF */
3357 ctxt
->src
.type
= OP_IMM
;
3359 ctxt
->src
.bytes
= 1;
3360 fastop(ctxt
, em_or
);
3362 return X86EMUL_CONTINUE
;
3365 static int em_call(struct x86_emulate_ctxt
*ctxt
)
3368 long rel
= ctxt
->src
.val
;
3370 ctxt
->src
.val
= (unsigned long)ctxt
->_eip
;
3371 rc
= jmp_rel(ctxt
, rel
);
3372 if (rc
!= X86EMUL_CONTINUE
)
3374 return em_push(ctxt
);
3377 static int em_call_far(struct x86_emulate_ctxt
*ctxt
)
3382 struct desc_struct old_desc
, new_desc
;
3383 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3384 int cpl
= ctxt
->ops
->cpl(ctxt
);
3385 enum x86emul_mode prev_mode
= ctxt
->mode
;
3387 old_eip
= ctxt
->_eip
;
3388 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
, VCPU_SREG_CS
);
3390 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
3391 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
3392 X86_TRANSFER_CALL_JMP
, &new_desc
);
3393 if (rc
!= X86EMUL_CONTINUE
)
3396 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
3397 if (rc
!= X86EMUL_CONTINUE
)
3400 ctxt
->src
.val
= old_cs
;
3402 if (rc
!= X86EMUL_CONTINUE
)
3405 ctxt
->src
.val
= old_eip
;
3407 /* If we failed, we tainted the memory, but the very least we should
3409 if (rc
!= X86EMUL_CONTINUE
) {
3410 pr_warn_once("faulting far call emulation tainted memory\n");
3415 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
3416 ctxt
->mode
= prev_mode
;
3421 static int em_ret_near_imm(struct x86_emulate_ctxt
*ctxt
)
3426 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
3427 if (rc
!= X86EMUL_CONTINUE
)
3429 rc
= assign_eip_near(ctxt
, eip
);
3430 if (rc
!= X86EMUL_CONTINUE
)
3432 rsp_increment(ctxt
, ctxt
->src
.val
);
3433 return X86EMUL_CONTINUE
;
3436 static int em_xchg(struct x86_emulate_ctxt
*ctxt
)
3438 /* Write back the register source. */
3439 ctxt
->src
.val
= ctxt
->dst
.val
;
3440 write_register_operand(&ctxt
->src
);
3442 /* Write back the memory destination with implicit LOCK prefix. */
3443 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
3444 ctxt
->lock_prefix
= 1;
3445 return X86EMUL_CONTINUE
;
3448 static int em_imul_3op(struct x86_emulate_ctxt
*ctxt
)
3450 ctxt
->dst
.val
= ctxt
->src2
.val
;
3451 return fastop(ctxt
, em_imul
);
3454 static int em_cwd(struct x86_emulate_ctxt
*ctxt
)
3456 ctxt
->dst
.type
= OP_REG
;
3457 ctxt
->dst
.bytes
= ctxt
->src
.bytes
;
3458 ctxt
->dst
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
3459 ctxt
->dst
.val
= ~((ctxt
->src
.val
>> (ctxt
->src
.bytes
* 8 - 1)) - 1);
3461 return X86EMUL_CONTINUE
;
3464 static int em_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3468 ctxt
->ops
->get_msr(ctxt
, MSR_IA32_TSC
, &tsc
);
3469 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)tsc
;
3470 *reg_write(ctxt
, VCPU_REGS_RDX
) = tsc
>> 32;
3471 return X86EMUL_CONTINUE
;
3474 static int em_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3478 if (ctxt
->ops
->read_pmc(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &pmc
))
3479 return emulate_gp(ctxt
, 0);
3480 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)pmc
;
3481 *reg_write(ctxt
, VCPU_REGS_RDX
) = pmc
>> 32;
3482 return X86EMUL_CONTINUE
;
3485 static int em_mov(struct x86_emulate_ctxt
*ctxt
)
3487 memcpy(ctxt
->dst
.valptr
, ctxt
->src
.valptr
, sizeof(ctxt
->src
.valptr
));
3488 return X86EMUL_CONTINUE
;
3491 #define FFL(x) bit(X86_FEATURE_##x)
3493 static int em_movbe(struct x86_emulate_ctxt
*ctxt
)
3495 u32 ebx
, ecx
, edx
, eax
= 1;
3499 * Check MOVBE is set in the guest-visible CPUID leaf.
3501 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3502 if (!(ecx
& FFL(MOVBE
)))
3503 return emulate_ud(ctxt
);
3505 switch (ctxt
->op_bytes
) {
3508 * From MOVBE definition: "...When the operand size is 16 bits,
3509 * the upper word of the destination register remains unchanged
3512 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3513 * rules so we have to do the operation almost per hand.
3515 tmp
= (u16
)ctxt
->src
.val
;
3516 ctxt
->dst
.val
&= ~0xffffUL
;
3517 ctxt
->dst
.val
|= (unsigned long)swab16(tmp
);
3520 ctxt
->dst
.val
= swab32((u32
)ctxt
->src
.val
);
3523 ctxt
->dst
.val
= swab64(ctxt
->src
.val
);
3528 return X86EMUL_CONTINUE
;
3531 static int em_cr_write(struct x86_emulate_ctxt
*ctxt
)
3533 if (ctxt
->ops
->set_cr(ctxt
, ctxt
->modrm_reg
, ctxt
->src
.val
))
3534 return emulate_gp(ctxt
, 0);
3536 /* Disable writeback. */
3537 ctxt
->dst
.type
= OP_NONE
;
3538 return X86EMUL_CONTINUE
;
3541 static int em_dr_write(struct x86_emulate_ctxt
*ctxt
)
3545 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3546 val
= ctxt
->src
.val
& ~0ULL;
3548 val
= ctxt
->src
.val
& ~0U;
3550 /* #UD condition is already handled. */
3551 if (ctxt
->ops
->set_dr(ctxt
, ctxt
->modrm_reg
, val
) < 0)
3552 return emulate_gp(ctxt
, 0);
3554 /* Disable writeback. */
3555 ctxt
->dst
.type
= OP_NONE
;
3556 return X86EMUL_CONTINUE
;
3559 static int em_wrmsr(struct x86_emulate_ctxt
*ctxt
)
3563 msr_data
= (u32
)reg_read(ctxt
, VCPU_REGS_RAX
)
3564 | ((u64
)reg_read(ctxt
, VCPU_REGS_RDX
) << 32);
3565 if (ctxt
->ops
->set_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), msr_data
))
3566 return emulate_gp(ctxt
, 0);
3568 return X86EMUL_CONTINUE
;
3571 static int em_rdmsr(struct x86_emulate_ctxt
*ctxt
)
3575 if (ctxt
->ops
->get_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &msr_data
))
3576 return emulate_gp(ctxt
, 0);
3578 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)msr_data
;
3579 *reg_write(ctxt
, VCPU_REGS_RDX
) = msr_data
>> 32;
3580 return X86EMUL_CONTINUE
;
3583 static int em_mov_rm_sreg(struct x86_emulate_ctxt
*ctxt
)
3585 if (ctxt
->modrm_reg
> VCPU_SREG_GS
)
3586 return emulate_ud(ctxt
);
3588 ctxt
->dst
.val
= get_segment_selector(ctxt
, ctxt
->modrm_reg
);
3589 if (ctxt
->dst
.bytes
== 4 && ctxt
->dst
.type
== OP_MEM
)
3590 ctxt
->dst
.bytes
= 2;
3591 return X86EMUL_CONTINUE
;
3594 static int em_mov_sreg_rm(struct x86_emulate_ctxt
*ctxt
)
3596 u16 sel
= ctxt
->src
.val
;
3598 if (ctxt
->modrm_reg
== VCPU_SREG_CS
|| ctxt
->modrm_reg
> VCPU_SREG_GS
)
3599 return emulate_ud(ctxt
);
3601 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
3602 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
3604 /* Disable writeback. */
3605 ctxt
->dst
.type
= OP_NONE
;
3606 return load_segment_descriptor(ctxt
, sel
, ctxt
->modrm_reg
);
3609 static int em_lldt(struct x86_emulate_ctxt
*ctxt
)
3611 u16 sel
= ctxt
->src
.val
;
3613 /* Disable writeback. */
3614 ctxt
->dst
.type
= OP_NONE
;
3615 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_LDTR
);
3618 static int em_ltr(struct x86_emulate_ctxt
*ctxt
)
3620 u16 sel
= ctxt
->src
.val
;
3622 /* Disable writeback. */
3623 ctxt
->dst
.type
= OP_NONE
;
3624 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_TR
);
3627 static int em_invlpg(struct x86_emulate_ctxt
*ctxt
)
3632 rc
= linearize(ctxt
, ctxt
->src
.addr
.mem
, 1, false, &linear
);
3633 if (rc
== X86EMUL_CONTINUE
)
3634 ctxt
->ops
->invlpg(ctxt
, linear
);
3635 /* Disable writeback. */
3636 ctxt
->dst
.type
= OP_NONE
;
3637 return X86EMUL_CONTINUE
;
3640 static int em_clts(struct x86_emulate_ctxt
*ctxt
)
3644 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
3646 ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
3647 return X86EMUL_CONTINUE
;
3650 static int em_hypercall(struct x86_emulate_ctxt
*ctxt
)
3652 int rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3654 if (rc
!= X86EMUL_CONTINUE
)
3657 /* Let the processor re-execute the fixed hypercall */
3658 ctxt
->_eip
= ctxt
->eip
;
3659 /* Disable writeback. */
3660 ctxt
->dst
.type
= OP_NONE
;
3661 return X86EMUL_CONTINUE
;
3664 static int emulate_store_desc_ptr(struct x86_emulate_ctxt
*ctxt
,
3665 void (*get
)(struct x86_emulate_ctxt
*ctxt
,
3666 struct desc_ptr
*ptr
))
3668 struct desc_ptr desc_ptr
;
3670 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3672 get(ctxt
, &desc_ptr
);
3673 if (ctxt
->op_bytes
== 2) {
3675 desc_ptr
.address
&= 0x00ffffff;
3677 /* Disable writeback. */
3678 ctxt
->dst
.type
= OP_NONE
;
3679 return segmented_write(ctxt
, ctxt
->dst
.addr
.mem
,
3680 &desc_ptr
, 2 + ctxt
->op_bytes
);
3683 static int em_sgdt(struct x86_emulate_ctxt
*ctxt
)
3685 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_gdt
);
3688 static int em_sidt(struct x86_emulate_ctxt
*ctxt
)
3690 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_idt
);
3693 static int em_lgdt_lidt(struct x86_emulate_ctxt
*ctxt
, bool lgdt
)
3695 struct desc_ptr desc_ptr
;
3698 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3700 rc
= read_descriptor(ctxt
, ctxt
->src
.addr
.mem
,
3701 &desc_ptr
.size
, &desc_ptr
.address
,
3703 if (rc
!= X86EMUL_CONTINUE
)
3705 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&&
3706 is_noncanonical_address(desc_ptr
.address
))
3707 return emulate_gp(ctxt
, 0);
3709 ctxt
->ops
->set_gdt(ctxt
, &desc_ptr
);
3711 ctxt
->ops
->set_idt(ctxt
, &desc_ptr
);
3712 /* Disable writeback. */
3713 ctxt
->dst
.type
= OP_NONE
;
3714 return X86EMUL_CONTINUE
;
3717 static int em_lgdt(struct x86_emulate_ctxt
*ctxt
)
3719 return em_lgdt_lidt(ctxt
, true);
3722 static int em_lidt(struct x86_emulate_ctxt
*ctxt
)
3724 return em_lgdt_lidt(ctxt
, false);
3727 static int em_smsw(struct x86_emulate_ctxt
*ctxt
)
3729 if (ctxt
->dst
.type
== OP_MEM
)
3730 ctxt
->dst
.bytes
= 2;
3731 ctxt
->dst
.val
= ctxt
->ops
->get_cr(ctxt
, 0);
3732 return X86EMUL_CONTINUE
;
3735 static int em_lmsw(struct x86_emulate_ctxt
*ctxt
)
3737 ctxt
->ops
->set_cr(ctxt
, 0, (ctxt
->ops
->get_cr(ctxt
, 0) & ~0x0eul
)
3738 | (ctxt
->src
.val
& 0x0f));
3739 ctxt
->dst
.type
= OP_NONE
;
3740 return X86EMUL_CONTINUE
;
3743 static int em_loop(struct x86_emulate_ctxt
*ctxt
)
3745 int rc
= X86EMUL_CONTINUE
;
3747 register_address_increment(ctxt
, VCPU_REGS_RCX
, -1);
3748 if ((address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) != 0) &&
3749 (ctxt
->b
== 0xe2 || test_cc(ctxt
->b
^ 0x5, ctxt
->eflags
)))
3750 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3755 static int em_jcxz(struct x86_emulate_ctxt
*ctxt
)
3757 int rc
= X86EMUL_CONTINUE
;
3759 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0)
3760 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3765 static int em_in(struct x86_emulate_ctxt
*ctxt
)
3767 if (!pio_in_emulated(ctxt
, ctxt
->dst
.bytes
, ctxt
->src
.val
,
3769 return X86EMUL_IO_NEEDED
;
3771 return X86EMUL_CONTINUE
;
3774 static int em_out(struct x86_emulate_ctxt
*ctxt
)
3776 ctxt
->ops
->pio_out_emulated(ctxt
, ctxt
->src
.bytes
, ctxt
->dst
.val
,
3778 /* Disable writeback. */
3779 ctxt
->dst
.type
= OP_NONE
;
3780 return X86EMUL_CONTINUE
;
3783 static int em_cli(struct x86_emulate_ctxt
*ctxt
)
3785 if (emulator_bad_iopl(ctxt
))
3786 return emulate_gp(ctxt
, 0);
3788 ctxt
->eflags
&= ~X86_EFLAGS_IF
;
3789 return X86EMUL_CONTINUE
;
3792 static int em_sti(struct x86_emulate_ctxt
*ctxt
)
3794 if (emulator_bad_iopl(ctxt
))
3795 return emulate_gp(ctxt
, 0);
3797 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_STI
;
3798 ctxt
->eflags
|= X86_EFLAGS_IF
;
3799 return X86EMUL_CONTINUE
;
3802 static int em_cpuid(struct x86_emulate_ctxt
*ctxt
)
3804 u32 eax
, ebx
, ecx
, edx
;
3806 eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3807 ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3808 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3809 *reg_write(ctxt
, VCPU_REGS_RAX
) = eax
;
3810 *reg_write(ctxt
, VCPU_REGS_RBX
) = ebx
;
3811 *reg_write(ctxt
, VCPU_REGS_RCX
) = ecx
;
3812 *reg_write(ctxt
, VCPU_REGS_RDX
) = edx
;
3813 return X86EMUL_CONTINUE
;
3816 static int em_sahf(struct x86_emulate_ctxt
*ctxt
)
3820 flags
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
| X86_EFLAGS_ZF
|
3822 flags
&= *reg_rmw(ctxt
, VCPU_REGS_RAX
) >> 8;
3824 ctxt
->eflags
&= ~0xffUL
;
3825 ctxt
->eflags
|= flags
| X86_EFLAGS_FIXED
;
3826 return X86EMUL_CONTINUE
;
3829 static int em_lahf(struct x86_emulate_ctxt
*ctxt
)
3831 *reg_rmw(ctxt
, VCPU_REGS_RAX
) &= ~0xff00UL
;
3832 *reg_rmw(ctxt
, VCPU_REGS_RAX
) |= (ctxt
->eflags
& 0xff) << 8;
3833 return X86EMUL_CONTINUE
;
3836 static int em_bswap(struct x86_emulate_ctxt
*ctxt
)
3838 switch (ctxt
->op_bytes
) {
3839 #ifdef CONFIG_X86_64
3841 asm("bswap %0" : "+r"(ctxt
->dst
.val
));
3845 asm("bswap %0" : "+r"(*(u32
*)&ctxt
->dst
.val
));
3848 return X86EMUL_CONTINUE
;
3851 static int em_clflush(struct x86_emulate_ctxt
*ctxt
)
3853 /* emulating clflush regardless of cpuid */
3854 return X86EMUL_CONTINUE
;
3857 static int em_movsxd(struct x86_emulate_ctxt
*ctxt
)
3859 ctxt
->dst
.val
= (s32
) ctxt
->src
.val
;
3860 return X86EMUL_CONTINUE
;
3863 static bool valid_cr(int nr
)
3875 static int check_cr_read(struct x86_emulate_ctxt
*ctxt
)
3877 if (!valid_cr(ctxt
->modrm_reg
))
3878 return emulate_ud(ctxt
);
3880 return X86EMUL_CONTINUE
;
3883 static int check_cr_write(struct x86_emulate_ctxt
*ctxt
)
3885 u64 new_val
= ctxt
->src
.val64
;
3886 int cr
= ctxt
->modrm_reg
;
3889 static u64 cr_reserved_bits
[] = {
3890 0xffffffff00000000ULL
,
3891 0, 0, 0, /* CR3 checked later */
3898 return emulate_ud(ctxt
);
3900 if (new_val
& cr_reserved_bits
[cr
])
3901 return emulate_gp(ctxt
, 0);
3906 if (((new_val
& X86_CR0_PG
) && !(new_val
& X86_CR0_PE
)) ||
3907 ((new_val
& X86_CR0_NW
) && !(new_val
& X86_CR0_CD
)))
3908 return emulate_gp(ctxt
, 0);
3910 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3911 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3913 if ((new_val
& X86_CR0_PG
) && (efer
& EFER_LME
) &&
3914 !(cr4
& X86_CR4_PAE
))
3915 return emulate_gp(ctxt
, 0);
3922 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3923 if (efer
& EFER_LMA
)
3924 rsvd
= CR3_L_MODE_RESERVED_BITS
& ~CR3_PCID_INVD
;
3927 return emulate_gp(ctxt
, 0);
3932 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3934 if ((efer
& EFER_LMA
) && !(new_val
& X86_CR4_PAE
))
3935 return emulate_gp(ctxt
, 0);
3941 return X86EMUL_CONTINUE
;
3944 static int check_dr7_gd(struct x86_emulate_ctxt
*ctxt
)
3948 ctxt
->ops
->get_dr(ctxt
, 7, &dr7
);
3950 /* Check if DR7.Global_Enable is set */
3951 return dr7
& (1 << 13);
3954 static int check_dr_read(struct x86_emulate_ctxt
*ctxt
)
3956 int dr
= ctxt
->modrm_reg
;
3960 return emulate_ud(ctxt
);
3962 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3963 if ((cr4
& X86_CR4_DE
) && (dr
== 4 || dr
== 5))
3964 return emulate_ud(ctxt
);
3966 if (check_dr7_gd(ctxt
)) {
3969 ctxt
->ops
->get_dr(ctxt
, 6, &dr6
);
3971 dr6
|= DR6_BD
| DR6_RTM
;
3972 ctxt
->ops
->set_dr(ctxt
, 6, dr6
);
3973 return emulate_db(ctxt
);
3976 return X86EMUL_CONTINUE
;
3979 static int check_dr_write(struct x86_emulate_ctxt
*ctxt
)
3981 u64 new_val
= ctxt
->src
.val64
;
3982 int dr
= ctxt
->modrm_reg
;
3984 if ((dr
== 6 || dr
== 7) && (new_val
& 0xffffffff00000000ULL
))
3985 return emulate_gp(ctxt
, 0);
3987 return check_dr_read(ctxt
);
3990 static int check_svme(struct x86_emulate_ctxt
*ctxt
)
3994 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3996 if (!(efer
& EFER_SVME
))
3997 return emulate_ud(ctxt
);
3999 return X86EMUL_CONTINUE
;
4002 static int check_svme_pa(struct x86_emulate_ctxt
*ctxt
)
4004 u64 rax
= reg_read(ctxt
, VCPU_REGS_RAX
);
4006 /* Valid physical address? */
4007 if (rax
& 0xffff000000000000ULL
)
4008 return emulate_gp(ctxt
, 0);
4010 return check_svme(ctxt
);
4013 static int check_rdtsc(struct x86_emulate_ctxt
*ctxt
)
4015 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4017 if (cr4
& X86_CR4_TSD
&& ctxt
->ops
->cpl(ctxt
))
4018 return emulate_ud(ctxt
);
4020 return X86EMUL_CONTINUE
;
4023 static int check_rdpmc(struct x86_emulate_ctxt
*ctxt
)
4025 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4026 u64 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
4028 if ((!(cr4
& X86_CR4_PCE
) && ctxt
->ops
->cpl(ctxt
)) ||
4029 ctxt
->ops
->check_pmc(ctxt
, rcx
))
4030 return emulate_gp(ctxt
, 0);
4032 return X86EMUL_CONTINUE
;
4035 static int check_perm_in(struct x86_emulate_ctxt
*ctxt
)
4037 ctxt
->dst
.bytes
= min(ctxt
->dst
.bytes
, 4u);
4038 if (!emulator_io_permited(ctxt
, ctxt
->src
.val
, ctxt
->dst
.bytes
))
4039 return emulate_gp(ctxt
, 0);
4041 return X86EMUL_CONTINUE
;
4044 static int check_perm_out(struct x86_emulate_ctxt
*ctxt
)
4046 ctxt
->src
.bytes
= min(ctxt
->src
.bytes
, 4u);
4047 if (!emulator_io_permited(ctxt
, ctxt
->dst
.val
, ctxt
->src
.bytes
))
4048 return emulate_gp(ctxt
, 0);
4050 return X86EMUL_CONTINUE
;
4053 #define D(_y) { .flags = (_y) }
4054 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4055 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4056 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4057 #define N D(NotImpl)
4058 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4059 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4060 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4061 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4062 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4063 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4064 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4065 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4066 #define II(_f, _e, _i) \
4067 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4068 #define IIP(_f, _e, _i, _p) \
4069 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4070 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4071 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4073 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4074 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4075 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4076 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4077 #define I2bvIP(_f, _e, _i, _p) \
4078 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4080 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4081 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4082 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4084 static const struct opcode group7_rm0
[] = {
4086 I(SrcNone
| Priv
| EmulateOnUD
, em_hypercall
),
4090 static const struct opcode group7_rm1
[] = {
4091 DI(SrcNone
| Priv
, monitor
),
4092 DI(SrcNone
| Priv
, mwait
),
4096 static const struct opcode group7_rm3
[] = {
4097 DIP(SrcNone
| Prot
| Priv
, vmrun
, check_svme_pa
),
4098 II(SrcNone
| Prot
| EmulateOnUD
, em_hypercall
, vmmcall
),
4099 DIP(SrcNone
| Prot
| Priv
, vmload
, check_svme_pa
),
4100 DIP(SrcNone
| Prot
| Priv
, vmsave
, check_svme_pa
),
4101 DIP(SrcNone
| Prot
| Priv
, stgi
, check_svme
),
4102 DIP(SrcNone
| Prot
| Priv
, clgi
, check_svme
),
4103 DIP(SrcNone
| Prot
| Priv
, skinit
, check_svme
),
4104 DIP(SrcNone
| Prot
| Priv
, invlpga
, check_svme
),
4107 static const struct opcode group7_rm7
[] = {
4109 DIP(SrcNone
, rdtscp
, check_rdtsc
),
4113 static const struct opcode group1
[] = {
4115 F(Lock
| PageTable
, em_or
),
4118 F(Lock
| PageTable
, em_and
),
4124 static const struct opcode group1A
[] = {
4125 I(DstMem
| SrcNone
| Mov
| Stack
| IncSP
, em_pop
), N
, N
, N
, N
, N
, N
, N
,
4128 static const struct opcode group2
[] = {
4129 F(DstMem
| ModRM
, em_rol
),
4130 F(DstMem
| ModRM
, em_ror
),
4131 F(DstMem
| ModRM
, em_rcl
),
4132 F(DstMem
| ModRM
, em_rcr
),
4133 F(DstMem
| ModRM
, em_shl
),
4134 F(DstMem
| ModRM
, em_shr
),
4135 F(DstMem
| ModRM
, em_shl
),
4136 F(DstMem
| ModRM
, em_sar
),
4139 static const struct opcode group3
[] = {
4140 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4141 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4142 F(DstMem
| SrcNone
| Lock
, em_not
),
4143 F(DstMem
| SrcNone
| Lock
, em_neg
),
4144 F(DstXacc
| Src2Mem
, em_mul_ex
),
4145 F(DstXacc
| Src2Mem
, em_imul_ex
),
4146 F(DstXacc
| Src2Mem
, em_div_ex
),
4147 F(DstXacc
| Src2Mem
, em_idiv_ex
),
4150 static const struct opcode group4
[] = {
4151 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_inc
),
4152 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_dec
),
4156 static const struct opcode group5
[] = {
4157 F(DstMem
| SrcNone
| Lock
, em_inc
),
4158 F(DstMem
| SrcNone
| Lock
, em_dec
),
4159 I(SrcMem
| NearBranch
, em_call_near_abs
),
4160 I(SrcMemFAddr
| ImplicitOps
, em_call_far
),
4161 I(SrcMem
| NearBranch
, em_jmp_abs
),
4162 I(SrcMemFAddr
| ImplicitOps
, em_jmp_far
),
4163 I(SrcMem
| Stack
, em_push
), D(Undefined
),
4166 static const struct opcode group6
[] = {
4167 DI(Prot
| DstMem
, sldt
),
4168 DI(Prot
| DstMem
, str
),
4169 II(Prot
| Priv
| SrcMem16
, em_lldt
, lldt
),
4170 II(Prot
| Priv
| SrcMem16
, em_ltr
, ltr
),
4174 static const struct group_dual group7
= { {
4175 II(Mov
| DstMem
, em_sgdt
, sgdt
),
4176 II(Mov
| DstMem
, em_sidt
, sidt
),
4177 II(SrcMem
| Priv
, em_lgdt
, lgdt
),
4178 II(SrcMem
| Priv
, em_lidt
, lidt
),
4179 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4180 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4181 II(SrcMem
| ByteOp
| Priv
| NoAccess
, em_invlpg
, invlpg
),
4185 N
, EXT(0, group7_rm3
),
4186 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4187 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4191 static const struct opcode group8
[] = {
4193 F(DstMem
| SrcImmByte
| NoWrite
, em_bt
),
4194 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_bts
),
4195 F(DstMem
| SrcImmByte
| Lock
, em_btr
),
4196 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_btc
),
4199 static const struct group_dual group9
= { {
4200 N
, I(DstMem64
| Lock
| PageTable
, em_cmpxchg8b
), N
, N
, N
, N
, N
, N
,
4202 N
, N
, N
, N
, N
, N
, N
, N
,
4205 static const struct opcode group11
[] = {
4206 I(DstMem
| SrcImm
| Mov
| PageTable
, em_mov
),
4210 static const struct gprefix pfx_0f_ae_7
= {
4211 I(SrcMem
| ByteOp
, em_clflush
), N
, N
, N
,
4214 static const struct group_dual group15
= { {
4215 N
, N
, N
, N
, N
, N
, N
, GP(0, &pfx_0f_ae_7
),
4217 N
, N
, N
, N
, N
, N
, N
, N
,
4220 static const struct gprefix pfx_0f_6f_0f_7f
= {
4221 I(Mmx
, em_mov
), I(Sse
| Aligned
, em_mov
), N
, I(Sse
| Unaligned
, em_mov
),
4224 static const struct instr_dual instr_dual_0f_2b
= {
4228 static const struct gprefix pfx_0f_2b
= {
4229 ID(0, &instr_dual_0f_2b
), ID(0, &instr_dual_0f_2b
), N
, N
,
4232 static const struct gprefix pfx_0f_28_0f_29
= {
4233 I(Aligned
, em_mov
), I(Aligned
, em_mov
), N
, N
,
4236 static const struct gprefix pfx_0f_e7
= {
4237 N
, I(Sse
, em_mov
), N
, N
,
4240 static const struct escape escape_d9
= { {
4241 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstcw
),
4244 N
, N
, N
, N
, N
, N
, N
, N
,
4246 N
, N
, N
, N
, N
, N
, N
, N
,
4248 N
, N
, N
, N
, N
, N
, N
, N
,
4250 N
, N
, N
, N
, N
, N
, N
, N
,
4252 N
, N
, N
, N
, N
, N
, N
, N
,
4254 N
, N
, N
, N
, N
, N
, N
, N
,
4256 N
, N
, N
, N
, N
, N
, N
, N
,
4258 N
, N
, N
, N
, N
, N
, N
, N
,
4261 static const struct escape escape_db
= { {
4262 N
, N
, N
, N
, N
, N
, N
, N
,
4265 N
, N
, N
, N
, N
, N
, N
, N
,
4267 N
, N
, N
, N
, N
, N
, N
, N
,
4269 N
, N
, N
, N
, N
, N
, N
, N
,
4271 N
, N
, N
, N
, N
, N
, N
, N
,
4273 N
, N
, N
, I(ImplicitOps
, em_fninit
), N
, N
, N
, N
,
4275 N
, N
, N
, N
, N
, N
, N
, N
,
4277 N
, N
, N
, N
, N
, N
, N
, N
,
4279 N
, N
, N
, N
, N
, N
, N
, N
,
4282 static const struct escape escape_dd
= { {
4283 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstsw
),
4286 N
, N
, N
, N
, N
, N
, N
, N
,
4288 N
, N
, N
, N
, N
, N
, N
, N
,
4290 N
, N
, N
, N
, N
, N
, N
, N
,
4292 N
, N
, N
, N
, N
, N
, N
, N
,
4294 N
, N
, N
, N
, N
, N
, N
, N
,
4296 N
, N
, N
, N
, N
, N
, N
, N
,
4298 N
, N
, N
, N
, N
, N
, N
, N
,
4300 N
, N
, N
, N
, N
, N
, N
, N
,
4303 static const struct instr_dual instr_dual_0f_c3
= {
4304 I(DstMem
| SrcReg
| ModRM
| No16
| Mov
, em_mov
), N
4307 static const struct mode_dual mode_dual_63
= {
4308 N
, I(DstReg
| SrcMem32
| ModRM
| Mov
, em_movsxd
)
4311 static const struct opcode opcode_table
[256] = {
4313 F6ALU(Lock
, em_add
),
4314 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_push_sreg
),
4315 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_pop_sreg
),
4317 F6ALU(Lock
| PageTable
, em_or
),
4318 I(ImplicitOps
| Stack
| No64
| Src2CS
, em_push_sreg
),
4321 F6ALU(Lock
, em_adc
),
4322 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_push_sreg
),
4323 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_pop_sreg
),
4325 F6ALU(Lock
, em_sbb
),
4326 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_push_sreg
),
4327 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_pop_sreg
),
4329 F6ALU(Lock
| PageTable
, em_and
), N
, N
,
4331 F6ALU(Lock
, em_sub
), N
, I(ByteOp
| DstAcc
| No64
, em_das
),
4333 F6ALU(Lock
, em_xor
), N
, N
,
4335 F6ALU(NoWrite
, em_cmp
), N
, N
,
4337 X8(F(DstReg
, em_inc
)), X8(F(DstReg
, em_dec
)),
4339 X8(I(SrcReg
| Stack
, em_push
)),
4341 X8(I(DstReg
| Stack
, em_pop
)),
4343 I(ImplicitOps
| Stack
| No64
, em_pusha
),
4344 I(ImplicitOps
| Stack
| No64
, em_popa
),
4345 N
, MD(ModRM
, &mode_dual_63
),
4348 I(SrcImm
| Mov
| Stack
, em_push
),
4349 I(DstReg
| SrcMem
| ModRM
| Src2Imm
, em_imul_3op
),
4350 I(SrcImmByte
| Mov
| Stack
, em_push
),
4351 I(DstReg
| SrcMem
| ModRM
| Src2ImmByte
, em_imul_3op
),
4352 I2bvIP(DstDI
| SrcDX
| Mov
| String
| Unaligned
, em_in
, ins
, check_perm_in
), /* insb, insw/insd */
4353 I2bvIP(SrcSI
| DstDX
| String
, em_out
, outs
, check_perm_out
), /* outsb, outsw/outsd */
4355 X16(D(SrcImmByte
| NearBranch
)),
4357 G(ByteOp
| DstMem
| SrcImm
, group1
),
4358 G(DstMem
| SrcImm
, group1
),
4359 G(ByteOp
| DstMem
| SrcImm
| No64
, group1
),
4360 G(DstMem
| SrcImmByte
, group1
),
4361 F2bv(DstMem
| SrcReg
| ModRM
| NoWrite
, em_test
),
4362 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
, em_xchg
),
4364 I2bv(DstMem
| SrcReg
| ModRM
| Mov
| PageTable
, em_mov
),
4365 I2bv(DstReg
| SrcMem
| ModRM
| Mov
, em_mov
),
4366 I(DstMem
| SrcNone
| ModRM
| Mov
| PageTable
, em_mov_rm_sreg
),
4367 D(ModRM
| SrcMem
| NoAccess
| DstReg
),
4368 I(ImplicitOps
| SrcMem16
| ModRM
, em_mov_sreg_rm
),
4371 DI(SrcAcc
| DstReg
, pause
), X7(D(SrcAcc
| DstReg
)),
4373 D(DstAcc
| SrcNone
), I(ImplicitOps
| SrcAcc
, em_cwd
),
4374 I(SrcImmFAddr
| No64
, em_call_far
), N
,
4375 II(ImplicitOps
| Stack
, em_pushf
, pushf
),
4376 II(ImplicitOps
| Stack
, em_popf
, popf
),
4377 I(ImplicitOps
, em_sahf
), I(ImplicitOps
, em_lahf
),
4379 I2bv(DstAcc
| SrcMem
| Mov
| MemAbs
, em_mov
),
4380 I2bv(DstMem
| SrcAcc
| Mov
| MemAbs
| PageTable
, em_mov
),
4381 I2bv(SrcSI
| DstDI
| Mov
| String
, em_mov
),
4382 F2bv(SrcSI
| DstDI
| String
| NoWrite
, em_cmp_r
),
4384 F2bv(DstAcc
| SrcImm
| NoWrite
, em_test
),
4385 I2bv(SrcAcc
| DstDI
| Mov
| String
, em_mov
),
4386 I2bv(SrcSI
| DstAcc
| Mov
| String
, em_mov
),
4387 F2bv(SrcAcc
| DstDI
| String
| NoWrite
, em_cmp_r
),
4389 X8(I(ByteOp
| DstReg
| SrcImm
| Mov
, em_mov
)),
4391 X8(I(DstReg
| SrcImm64
| Mov
, em_mov
)),
4393 G(ByteOp
| Src2ImmByte
, group2
), G(Src2ImmByte
, group2
),
4394 I(ImplicitOps
| NearBranch
| SrcImmU16
, em_ret_near_imm
),
4395 I(ImplicitOps
| NearBranch
, em_ret
),
4396 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2ES
, em_lseg
),
4397 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2DS
, em_lseg
),
4398 G(ByteOp
, group11
), G(0, group11
),
4400 I(Stack
| SrcImmU16
| Src2ImmByte
, em_enter
), I(Stack
, em_leave
),
4401 I(ImplicitOps
| SrcImmU16
, em_ret_far_imm
),
4402 I(ImplicitOps
, em_ret_far
),
4403 D(ImplicitOps
), DI(SrcImmByte
, intn
),
4404 D(ImplicitOps
| No64
), II(ImplicitOps
, em_iret
, iret
),
4406 G(Src2One
| ByteOp
, group2
), G(Src2One
, group2
),
4407 G(Src2CL
| ByteOp
, group2
), G(Src2CL
, group2
),
4408 I(DstAcc
| SrcImmUByte
| No64
, em_aam
),
4409 I(DstAcc
| SrcImmUByte
| No64
, em_aad
),
4410 F(DstAcc
| ByteOp
| No64
, em_salc
),
4411 I(DstAcc
| SrcXLat
| ByteOp
, em_mov
),
4413 N
, E(0, &escape_d9
), N
, E(0, &escape_db
), N
, E(0, &escape_dd
), N
, N
,
4415 X3(I(SrcImmByte
| NearBranch
, em_loop
)),
4416 I(SrcImmByte
| NearBranch
, em_jcxz
),
4417 I2bvIP(SrcImmUByte
| DstAcc
, em_in
, in
, check_perm_in
),
4418 I2bvIP(SrcAcc
| DstImmUByte
, em_out
, out
, check_perm_out
),
4420 I(SrcImm
| NearBranch
, em_call
), D(SrcImm
| ImplicitOps
| NearBranch
),
4421 I(SrcImmFAddr
| No64
, em_jmp_far
),
4422 D(SrcImmByte
| ImplicitOps
| NearBranch
),
4423 I2bvIP(SrcDX
| DstAcc
, em_in
, in
, check_perm_in
),
4424 I2bvIP(SrcAcc
| DstDX
, em_out
, out
, check_perm_out
),
4426 N
, DI(ImplicitOps
, icebp
), N
, N
,
4427 DI(ImplicitOps
| Priv
, hlt
), D(ImplicitOps
),
4428 G(ByteOp
, group3
), G(0, group3
),
4430 D(ImplicitOps
), D(ImplicitOps
),
4431 I(ImplicitOps
, em_cli
), I(ImplicitOps
, em_sti
),
4432 D(ImplicitOps
), D(ImplicitOps
), G(0, group4
), G(0, group5
),
4435 static const struct opcode twobyte_table
[256] = {
4437 G(0, group6
), GD(0, &group7
), N
, N
,
4438 N
, I(ImplicitOps
| EmulateOnUD
, em_syscall
),
4439 II(ImplicitOps
| Priv
, em_clts
, clts
), N
,
4440 DI(ImplicitOps
| Priv
, invd
), DI(ImplicitOps
| Priv
, wbinvd
), N
, N
,
4441 N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
), N
, N
,
4443 N
, N
, N
, N
, N
, N
, N
, N
,
4444 D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4445 N
, N
, N
, N
, N
, N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4447 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, cr_read
, check_cr_read
),
4448 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, dr_read
, check_dr_read
),
4449 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_cr_write
, cr_write
,
4451 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_dr_write
, dr_write
,
4454 GP(ModRM
| DstReg
| SrcMem
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4455 GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4456 N
, GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_2b
),
4459 II(ImplicitOps
| Priv
, em_wrmsr
, wrmsr
),
4460 IIP(ImplicitOps
, em_rdtsc
, rdtsc
, check_rdtsc
),
4461 II(ImplicitOps
| Priv
, em_rdmsr
, rdmsr
),
4462 IIP(ImplicitOps
, em_rdpmc
, rdpmc
, check_rdpmc
),
4463 I(ImplicitOps
| EmulateOnUD
, em_sysenter
),
4464 I(ImplicitOps
| Priv
| EmulateOnUD
, em_sysexit
),
4466 N
, N
, N
, N
, N
, N
, N
, N
,
4468 X16(D(DstReg
| SrcMem
| ModRM
)),
4470 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4475 N
, N
, N
, GP(SrcMem
| DstReg
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4480 N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4482 X16(D(SrcImm
| NearBranch
)),
4484 X16(D(ByteOp
| DstMem
| SrcNone
| ModRM
| Mov
)),
4486 I(Stack
| Src2FS
, em_push_sreg
), I(Stack
| Src2FS
, em_pop_sreg
),
4487 II(ImplicitOps
, em_cpuid
, cpuid
),
4488 F(DstMem
| SrcReg
| ModRM
| BitOp
| NoWrite
, em_bt
),
4489 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shld
),
4490 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shld
), N
, N
,
4492 I(Stack
| Src2GS
, em_push_sreg
), I(Stack
| Src2GS
, em_pop_sreg
),
4493 II(EmulateOnUD
| ImplicitOps
, em_rsm
, rsm
),
4494 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_bts
),
4495 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shrd
),
4496 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shrd
),
4497 GD(0, &group15
), F(DstReg
| SrcMem
| ModRM
, em_imul
),
4499 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
| SrcWrite
, em_cmpxchg
),
4500 I(DstReg
| SrcMemFAddr
| ModRM
| Src2SS
, em_lseg
),
4501 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
, em_btr
),
4502 I(DstReg
| SrcMemFAddr
| ModRM
| Src2FS
, em_lseg
),
4503 I(DstReg
| SrcMemFAddr
| ModRM
| Src2GS
, em_lseg
),
4504 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4508 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_btc
),
4509 I(DstReg
| SrcMem
| ModRM
, em_bsf_c
),
4510 I(DstReg
| SrcMem
| ModRM
, em_bsr_c
),
4511 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4513 F2bv(DstMem
| SrcReg
| ModRM
| SrcWrite
| Lock
, em_xadd
),
4514 N
, ID(0, &instr_dual_0f_c3
),
4515 N
, N
, N
, GD(0, &group9
),
4517 X8(I(DstReg
, em_bswap
)),
4519 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4521 N
, N
, N
, N
, N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_e7
),
4522 N
, N
, N
, N
, N
, N
, N
, N
,
4524 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
4527 static const struct instr_dual instr_dual_0f_38_f0
= {
4528 I(DstReg
| SrcMem
| Mov
, em_movbe
), N
4531 static const struct instr_dual instr_dual_0f_38_f1
= {
4532 I(DstMem
| SrcReg
| Mov
, em_movbe
), N
4535 static const struct gprefix three_byte_0f_38_f0
= {
4536 ID(0, &instr_dual_0f_38_f0
), N
, N
, N
4539 static const struct gprefix three_byte_0f_38_f1
= {
4540 ID(0, &instr_dual_0f_38_f1
), N
, N
, N
4544 * Insns below are selected by the prefix which indexed by the third opcode
4547 static const struct opcode opcode_map_0f_38
[256] = {
4549 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4551 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4553 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f0
),
4554 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f1
),
4575 static unsigned imm_size(struct x86_emulate_ctxt
*ctxt
)
4579 size
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4585 static int decode_imm(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4586 unsigned size
, bool sign_extension
)
4588 int rc
= X86EMUL_CONTINUE
;
4592 op
->addr
.mem
.ea
= ctxt
->_eip
;
4593 /* NB. Immediates are sign-extended as necessary. */
4594 switch (op
->bytes
) {
4596 op
->val
= insn_fetch(s8
, ctxt
);
4599 op
->val
= insn_fetch(s16
, ctxt
);
4602 op
->val
= insn_fetch(s32
, ctxt
);
4605 op
->val
= insn_fetch(s64
, ctxt
);
4608 if (!sign_extension
) {
4609 switch (op
->bytes
) {
4617 op
->val
&= 0xffffffff;
4625 static int decode_operand(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4628 int rc
= X86EMUL_CONTINUE
;
4632 decode_register_operand(ctxt
, op
);
4635 rc
= decode_imm(ctxt
, op
, 1, false);
4638 ctxt
->memop
.bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4642 if (ctxt
->d
& BitOp
)
4643 fetch_bit_operand(ctxt
);
4644 op
->orig_val
= op
->val
;
4647 ctxt
->memop
.bytes
= (ctxt
->op_bytes
== 8) ? 16 : 8;
4651 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4652 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4653 fetch_register_operand(op
);
4654 op
->orig_val
= op
->val
;
4658 op
->bytes
= (ctxt
->d
& ByteOp
) ? 2 : ctxt
->op_bytes
;
4659 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4660 fetch_register_operand(op
);
4661 op
->orig_val
= op
->val
;
4664 if (ctxt
->d
& ByteOp
) {
4669 op
->bytes
= ctxt
->op_bytes
;
4670 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4671 fetch_register_operand(op
);
4672 op
->orig_val
= op
->val
;
4676 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4678 register_address(ctxt
, VCPU_REGS_RDI
);
4679 op
->addr
.mem
.seg
= VCPU_SREG_ES
;
4686 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4687 fetch_register_operand(op
);
4692 op
->val
= reg_read(ctxt
, VCPU_REGS_RCX
) & 0xff;
4695 rc
= decode_imm(ctxt
, op
, 1, true);
4703 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), true);
4706 rc
= decode_imm(ctxt
, op
, ctxt
->op_bytes
, true);
4709 ctxt
->memop
.bytes
= 1;
4710 if (ctxt
->memop
.type
== OP_REG
) {
4711 ctxt
->memop
.addr
.reg
= decode_register(ctxt
,
4712 ctxt
->modrm_rm
, true);
4713 fetch_register_operand(&ctxt
->memop
);
4717 ctxt
->memop
.bytes
= 2;
4720 ctxt
->memop
.bytes
= 4;
4723 rc
= decode_imm(ctxt
, op
, 2, false);
4726 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), false);
4730 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4732 register_address(ctxt
, VCPU_REGS_RSI
);
4733 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4739 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4742 reg_read(ctxt
, VCPU_REGS_RBX
) +
4743 (reg_read(ctxt
, VCPU_REGS_RAX
) & 0xff));
4744 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4749 op
->addr
.mem
.ea
= ctxt
->_eip
;
4750 op
->bytes
= ctxt
->op_bytes
+ 2;
4751 insn_fetch_arr(op
->valptr
, op
->bytes
, ctxt
);
4754 ctxt
->memop
.bytes
= ctxt
->op_bytes
+ 2;
4758 op
->val
= VCPU_SREG_ES
;
4762 op
->val
= VCPU_SREG_CS
;
4766 op
->val
= VCPU_SREG_SS
;
4770 op
->val
= VCPU_SREG_DS
;
4774 op
->val
= VCPU_SREG_FS
;
4778 op
->val
= VCPU_SREG_GS
;
4781 /* Special instructions do their own operand decoding. */
4783 op
->type
= OP_NONE
; /* Disable writeback. */
4791 int x86_decode_insn(struct x86_emulate_ctxt
*ctxt
, void *insn
, int insn_len
)
4793 int rc
= X86EMUL_CONTINUE
;
4794 int mode
= ctxt
->mode
;
4795 int def_op_bytes
, def_ad_bytes
, goffset
, simd_prefix
;
4796 bool op_prefix
= false;
4797 bool has_seg_override
= false;
4798 struct opcode opcode
;
4800 ctxt
->memop
.type
= OP_NONE
;
4801 ctxt
->memopp
= NULL
;
4802 ctxt
->_eip
= ctxt
->eip
;
4803 ctxt
->fetch
.ptr
= ctxt
->fetch
.data
;
4804 ctxt
->fetch
.end
= ctxt
->fetch
.data
+ insn_len
;
4805 ctxt
->opcode_len
= 1;
4807 memcpy(ctxt
->fetch
.data
, insn
, insn_len
);
4809 rc
= __do_insn_fetch_bytes(ctxt
, 1);
4810 if (rc
!= X86EMUL_CONTINUE
)
4815 case X86EMUL_MODE_REAL
:
4816 case X86EMUL_MODE_VM86
:
4817 case X86EMUL_MODE_PROT16
:
4818 def_op_bytes
= def_ad_bytes
= 2;
4820 case X86EMUL_MODE_PROT32
:
4821 def_op_bytes
= def_ad_bytes
= 4;
4823 #ifdef CONFIG_X86_64
4824 case X86EMUL_MODE_PROT64
:
4830 return EMULATION_FAILED
;
4833 ctxt
->op_bytes
= def_op_bytes
;
4834 ctxt
->ad_bytes
= def_ad_bytes
;
4836 /* Legacy prefixes. */
4838 switch (ctxt
->b
= insn_fetch(u8
, ctxt
)) {
4839 case 0x66: /* operand-size override */
4841 /* switch between 2/4 bytes */
4842 ctxt
->op_bytes
= def_op_bytes
^ 6;
4844 case 0x67: /* address-size override */
4845 if (mode
== X86EMUL_MODE_PROT64
)
4846 /* switch between 4/8 bytes */
4847 ctxt
->ad_bytes
= def_ad_bytes
^ 12;
4849 /* switch between 2/4 bytes */
4850 ctxt
->ad_bytes
= def_ad_bytes
^ 6;
4852 case 0x26: /* ES override */
4853 case 0x2e: /* CS override */
4854 case 0x36: /* SS override */
4855 case 0x3e: /* DS override */
4856 has_seg_override
= true;
4857 ctxt
->seg_override
= (ctxt
->b
>> 3) & 3;
4859 case 0x64: /* FS override */
4860 case 0x65: /* GS override */
4861 has_seg_override
= true;
4862 ctxt
->seg_override
= ctxt
->b
& 7;
4864 case 0x40 ... 0x4f: /* REX */
4865 if (mode
!= X86EMUL_MODE_PROT64
)
4867 ctxt
->rex_prefix
= ctxt
->b
;
4869 case 0xf0: /* LOCK */
4870 ctxt
->lock_prefix
= 1;
4872 case 0xf2: /* REPNE/REPNZ */
4873 case 0xf3: /* REP/REPE/REPZ */
4874 ctxt
->rep_prefix
= ctxt
->b
;
4880 /* Any legacy prefix after a REX prefix nullifies its effect. */
4882 ctxt
->rex_prefix
= 0;
4888 if (ctxt
->rex_prefix
& 8)
4889 ctxt
->op_bytes
= 8; /* REX.W */
4891 /* Opcode byte(s). */
4892 opcode
= opcode_table
[ctxt
->b
];
4893 /* Two-byte opcode? */
4894 if (ctxt
->b
== 0x0f) {
4895 ctxt
->opcode_len
= 2;
4896 ctxt
->b
= insn_fetch(u8
, ctxt
);
4897 opcode
= twobyte_table
[ctxt
->b
];
4899 /* 0F_38 opcode map */
4900 if (ctxt
->b
== 0x38) {
4901 ctxt
->opcode_len
= 3;
4902 ctxt
->b
= insn_fetch(u8
, ctxt
);
4903 opcode
= opcode_map_0f_38
[ctxt
->b
];
4906 ctxt
->d
= opcode
.flags
;
4908 if (ctxt
->d
& ModRM
)
4909 ctxt
->modrm
= insn_fetch(u8
, ctxt
);
4911 /* vex-prefix instructions are not implemented */
4912 if (ctxt
->opcode_len
== 1 && (ctxt
->b
== 0xc5 || ctxt
->b
== 0xc4) &&
4913 (mode
== X86EMUL_MODE_PROT64
|| (ctxt
->modrm
& 0xc0) == 0xc0)) {
4917 while (ctxt
->d
& GroupMask
) {
4918 switch (ctxt
->d
& GroupMask
) {
4920 goffset
= (ctxt
->modrm
>> 3) & 7;
4921 opcode
= opcode
.u
.group
[goffset
];
4924 goffset
= (ctxt
->modrm
>> 3) & 7;
4925 if ((ctxt
->modrm
>> 6) == 3)
4926 opcode
= opcode
.u
.gdual
->mod3
[goffset
];
4928 opcode
= opcode
.u
.gdual
->mod012
[goffset
];
4931 goffset
= ctxt
->modrm
& 7;
4932 opcode
= opcode
.u
.group
[goffset
];
4935 if (ctxt
->rep_prefix
&& op_prefix
)
4936 return EMULATION_FAILED
;
4937 simd_prefix
= op_prefix
? 0x66 : ctxt
->rep_prefix
;
4938 switch (simd_prefix
) {
4939 case 0x00: opcode
= opcode
.u
.gprefix
->pfx_no
; break;
4940 case 0x66: opcode
= opcode
.u
.gprefix
->pfx_66
; break;
4941 case 0xf2: opcode
= opcode
.u
.gprefix
->pfx_f2
; break;
4942 case 0xf3: opcode
= opcode
.u
.gprefix
->pfx_f3
; break;
4946 if (ctxt
->modrm
> 0xbf)
4947 opcode
= opcode
.u
.esc
->high
[ctxt
->modrm
- 0xc0];
4949 opcode
= opcode
.u
.esc
->op
[(ctxt
->modrm
>> 3) & 7];
4952 if ((ctxt
->modrm
>> 6) == 3)
4953 opcode
= opcode
.u
.idual
->mod3
;
4955 opcode
= opcode
.u
.idual
->mod012
;
4958 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
4959 opcode
= opcode
.u
.mdual
->mode64
;
4961 opcode
= opcode
.u
.mdual
->mode32
;
4964 return EMULATION_FAILED
;
4967 ctxt
->d
&= ~(u64
)GroupMask
;
4968 ctxt
->d
|= opcode
.flags
;
4973 return EMULATION_FAILED
;
4975 ctxt
->execute
= opcode
.u
.execute
;
4977 if (unlikely(ctxt
->ud
) && likely(!(ctxt
->d
& EmulateOnUD
)))
4978 return EMULATION_FAILED
;
4980 if (unlikely(ctxt
->d
&
4981 (NotImpl
|Stack
|Op3264
|Sse
|Mmx
|Intercept
|CheckPerm
|NearBranch
|
4984 * These are copied unconditionally here, and checked unconditionally
4985 * in x86_emulate_insn.
4987 ctxt
->check_perm
= opcode
.check_perm
;
4988 ctxt
->intercept
= opcode
.intercept
;
4990 if (ctxt
->d
& NotImpl
)
4991 return EMULATION_FAILED
;
4993 if (mode
== X86EMUL_MODE_PROT64
) {
4994 if (ctxt
->op_bytes
== 4 && (ctxt
->d
& Stack
))
4996 else if (ctxt
->d
& NearBranch
)
5000 if (ctxt
->d
& Op3264
) {
5001 if (mode
== X86EMUL_MODE_PROT64
)
5007 if ((ctxt
->d
& No16
) && ctxt
->op_bytes
== 2)
5011 ctxt
->op_bytes
= 16;
5012 else if (ctxt
->d
& Mmx
)
5016 /* ModRM and SIB bytes. */
5017 if (ctxt
->d
& ModRM
) {
5018 rc
= decode_modrm(ctxt
, &ctxt
->memop
);
5019 if (!has_seg_override
) {
5020 has_seg_override
= true;
5021 ctxt
->seg_override
= ctxt
->modrm_seg
;
5023 } else if (ctxt
->d
& MemAbs
)
5024 rc
= decode_abs(ctxt
, &ctxt
->memop
);
5025 if (rc
!= X86EMUL_CONTINUE
)
5028 if (!has_seg_override
)
5029 ctxt
->seg_override
= VCPU_SREG_DS
;
5031 ctxt
->memop
.addr
.mem
.seg
= ctxt
->seg_override
;
5034 * Decode and fetch the source operand: register, memory
5037 rc
= decode_operand(ctxt
, &ctxt
->src
, (ctxt
->d
>> SrcShift
) & OpMask
);
5038 if (rc
!= X86EMUL_CONTINUE
)
5042 * Decode and fetch the second source operand: register, memory
5045 rc
= decode_operand(ctxt
, &ctxt
->src2
, (ctxt
->d
>> Src2Shift
) & OpMask
);
5046 if (rc
!= X86EMUL_CONTINUE
)
5049 /* Decode and fetch the destination operand: register or memory. */
5050 rc
= decode_operand(ctxt
, &ctxt
->dst
, (ctxt
->d
>> DstShift
) & OpMask
);
5052 if (ctxt
->rip_relative
&& likely(ctxt
->memopp
))
5053 ctxt
->memopp
->addr
.mem
.ea
= address_mask(ctxt
,
5054 ctxt
->memopp
->addr
.mem
.ea
+ ctxt
->_eip
);
5057 return (rc
!= X86EMUL_CONTINUE
) ? EMULATION_FAILED
: EMULATION_OK
;
5060 bool x86_page_table_writing_insn(struct x86_emulate_ctxt
*ctxt
)
5062 return ctxt
->d
& PageTable
;
5065 static bool string_insn_completed(struct x86_emulate_ctxt
*ctxt
)
5067 /* The second termination condition only applies for REPE
5068 * and REPNE. Test if the repeat string operation prefix is
5069 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5070 * corresponding termination condition according to:
5071 * - if REPE/REPZ and ZF = 0 then done
5072 * - if REPNE/REPNZ and ZF = 1 then done
5074 if (((ctxt
->b
== 0xa6) || (ctxt
->b
== 0xa7) ||
5075 (ctxt
->b
== 0xae) || (ctxt
->b
== 0xaf))
5076 && (((ctxt
->rep_prefix
== REPE_PREFIX
) &&
5077 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == 0))
5078 || ((ctxt
->rep_prefix
== REPNE_PREFIX
) &&
5079 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == X86_EFLAGS_ZF
))))
5085 static int flush_pending_x87_faults(struct x86_emulate_ctxt
*ctxt
)
5089 ctxt
->ops
->get_fpu(ctxt
);
5090 asm volatile("1: fwait \n\t"
5092 ".pushsection .fixup,\"ax\" \n\t"
5094 "movb $1, %[fault] \n\t"
5097 _ASM_EXTABLE(1b
, 3b
)
5098 : [fault
]"+qm"(fault
));
5099 ctxt
->ops
->put_fpu(ctxt
);
5101 if (unlikely(fault
))
5102 return emulate_exception(ctxt
, MF_VECTOR
, 0, false);
5104 return X86EMUL_CONTINUE
;
5107 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt
*ctxt
,
5110 if (op
->type
== OP_MM
)
5111 read_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
5114 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*))
5116 register void *__sp
asm(_ASM_SP
);
5117 ulong flags
= (ctxt
->eflags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
5119 if (!(ctxt
->d
& ByteOp
))
5120 fop
+= __ffs(ctxt
->dst
.bytes
) * FASTOP_SIZE
;
5122 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5123 : "+a"(ctxt
->dst
.val
), "+d"(ctxt
->src
.val
), [flags
]"+D"(flags
),
5124 [fastop
]"+S"(fop
), "+r"(__sp
)
5125 : "c"(ctxt
->src2
.val
));
5127 ctxt
->eflags
= (ctxt
->eflags
& ~EFLAGS_MASK
) | (flags
& EFLAGS_MASK
);
5128 if (!fop
) /* exception is returned in fop variable */
5129 return emulate_de(ctxt
);
5130 return X86EMUL_CONTINUE
;
5133 void init_decode_cache(struct x86_emulate_ctxt
*ctxt
)
5135 memset(&ctxt
->rip_relative
, 0,
5136 (void *)&ctxt
->modrm
- (void *)&ctxt
->rip_relative
);
5138 ctxt
->io_read
.pos
= 0;
5139 ctxt
->io_read
.end
= 0;
5140 ctxt
->mem_read
.end
= 0;
5143 int x86_emulate_insn(struct x86_emulate_ctxt
*ctxt
)
5145 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
5146 int rc
= X86EMUL_CONTINUE
;
5147 int saved_dst_type
= ctxt
->dst
.type
;
5149 ctxt
->mem_read
.pos
= 0;
5151 /* LOCK prefix is allowed only with some instructions */
5152 if (ctxt
->lock_prefix
&& (!(ctxt
->d
& Lock
) || ctxt
->dst
.type
!= OP_MEM
)) {
5153 rc
= emulate_ud(ctxt
);
5157 if ((ctxt
->d
& SrcMask
) == SrcMemFAddr
&& ctxt
->src
.type
!= OP_MEM
) {
5158 rc
= emulate_ud(ctxt
);
5162 if (unlikely(ctxt
->d
&
5163 (No64
|Undefined
|Sse
|Mmx
|Intercept
|CheckPerm
|Priv
|Prot
|String
))) {
5164 if ((ctxt
->mode
== X86EMUL_MODE_PROT64
&& (ctxt
->d
& No64
)) ||
5165 (ctxt
->d
& Undefined
)) {
5166 rc
= emulate_ud(ctxt
);
5170 if (((ctxt
->d
& (Sse
|Mmx
)) && ((ops
->get_cr(ctxt
, 0) & X86_CR0_EM
)))
5171 || ((ctxt
->d
& Sse
) && !(ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
))) {
5172 rc
= emulate_ud(ctxt
);
5176 if ((ctxt
->d
& (Sse
|Mmx
)) && (ops
->get_cr(ctxt
, 0) & X86_CR0_TS
)) {
5177 rc
= emulate_nm(ctxt
);
5181 if (ctxt
->d
& Mmx
) {
5182 rc
= flush_pending_x87_faults(ctxt
);
5183 if (rc
!= X86EMUL_CONTINUE
)
5186 * Now that we know the fpu is exception safe, we can fetch
5189 fetch_possible_mmx_operand(ctxt
, &ctxt
->src
);
5190 fetch_possible_mmx_operand(ctxt
, &ctxt
->src2
);
5191 if (!(ctxt
->d
& Mov
))
5192 fetch_possible_mmx_operand(ctxt
, &ctxt
->dst
);
5195 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && ctxt
->intercept
) {
5196 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5197 X86_ICPT_PRE_EXCEPT
);
5198 if (rc
!= X86EMUL_CONTINUE
)
5202 /* Instruction can only be executed in protected mode */
5203 if ((ctxt
->d
& Prot
) && ctxt
->mode
< X86EMUL_MODE_PROT16
) {
5204 rc
= emulate_ud(ctxt
);
5208 /* Privileged instruction can be executed only in CPL=0 */
5209 if ((ctxt
->d
& Priv
) && ops
->cpl(ctxt
)) {
5210 if (ctxt
->d
& PrivUD
)
5211 rc
= emulate_ud(ctxt
);
5213 rc
= emulate_gp(ctxt
, 0);
5217 /* Do instruction specific permission checks */
5218 if (ctxt
->d
& CheckPerm
) {
5219 rc
= ctxt
->check_perm(ctxt
);
5220 if (rc
!= X86EMUL_CONTINUE
)
5224 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5225 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5226 X86_ICPT_POST_EXCEPT
);
5227 if (rc
!= X86EMUL_CONTINUE
)
5231 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5232 /* All REP prefixes have the same first termination condition */
5233 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0) {
5234 string_registers_quirk(ctxt
);
5235 ctxt
->eip
= ctxt
->_eip
;
5236 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5242 if ((ctxt
->src
.type
== OP_MEM
) && !(ctxt
->d
& NoAccess
)) {
5243 rc
= segmented_read(ctxt
, ctxt
->src
.addr
.mem
,
5244 ctxt
->src
.valptr
, ctxt
->src
.bytes
);
5245 if (rc
!= X86EMUL_CONTINUE
)
5247 ctxt
->src
.orig_val64
= ctxt
->src
.val64
;
5250 if (ctxt
->src2
.type
== OP_MEM
) {
5251 rc
= segmented_read(ctxt
, ctxt
->src2
.addr
.mem
,
5252 &ctxt
->src2
.val
, ctxt
->src2
.bytes
);
5253 if (rc
!= X86EMUL_CONTINUE
)
5257 if ((ctxt
->d
& DstMask
) == ImplicitOps
)
5261 if ((ctxt
->dst
.type
== OP_MEM
) && !(ctxt
->d
& Mov
)) {
5262 /* optimisation - avoid slow emulated read if Mov */
5263 rc
= segmented_read(ctxt
, ctxt
->dst
.addr
.mem
,
5264 &ctxt
->dst
.val
, ctxt
->dst
.bytes
);
5265 if (rc
!= X86EMUL_CONTINUE
) {
5266 if (!(ctxt
->d
& NoWrite
) &&
5267 rc
== X86EMUL_PROPAGATE_FAULT
&&
5268 ctxt
->exception
.vector
== PF_VECTOR
)
5269 ctxt
->exception
.error_code
|= PFERR_WRITE_MASK
;
5273 /* Copy full 64-bit value for CMPXCHG8B. */
5274 ctxt
->dst
.orig_val64
= ctxt
->dst
.val64
;
5278 if (unlikely(ctxt
->emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5279 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5280 X86_ICPT_POST_MEMACCESS
);
5281 if (rc
!= X86EMUL_CONTINUE
)
5285 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5286 ctxt
->eflags
|= X86_EFLAGS_RF
;
5288 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5290 if (ctxt
->execute
) {
5291 if (ctxt
->d
& Fastop
) {
5292 void (*fop
)(struct fastop
*) = (void *)ctxt
->execute
;
5293 rc
= fastop(ctxt
, fop
);
5294 if (rc
!= X86EMUL_CONTINUE
)
5298 rc
= ctxt
->execute(ctxt
);
5299 if (rc
!= X86EMUL_CONTINUE
)
5304 if (ctxt
->opcode_len
== 2)
5306 else if (ctxt
->opcode_len
== 3)
5307 goto threebyte_insn
;
5310 case 0x70 ... 0x7f: /* jcc (short) */
5311 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5312 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5314 case 0x8d: /* lea r16/r32, m */
5315 ctxt
->dst
.val
= ctxt
->src
.addr
.mem
.ea
;
5317 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5318 if (ctxt
->dst
.addr
.reg
== reg_rmw(ctxt
, VCPU_REGS_RAX
))
5319 ctxt
->dst
.type
= OP_NONE
;
5323 case 0x98: /* cbw/cwde/cdqe */
5324 switch (ctxt
->op_bytes
) {
5325 case 2: ctxt
->dst
.val
= (s8
)ctxt
->dst
.val
; break;
5326 case 4: ctxt
->dst
.val
= (s16
)ctxt
->dst
.val
; break;
5327 case 8: ctxt
->dst
.val
= (s32
)ctxt
->dst
.val
; break;
5330 case 0xcc: /* int3 */
5331 rc
= emulate_int(ctxt
, 3);
5333 case 0xcd: /* int n */
5334 rc
= emulate_int(ctxt
, ctxt
->src
.val
);
5336 case 0xce: /* into */
5337 if (ctxt
->eflags
& X86_EFLAGS_OF
)
5338 rc
= emulate_int(ctxt
, 4);
5340 case 0xe9: /* jmp rel */
5341 case 0xeb: /* jmp rel short */
5342 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5343 ctxt
->dst
.type
= OP_NONE
; /* Disable writeback. */
5345 case 0xf4: /* hlt */
5346 ctxt
->ops
->halt(ctxt
);
5348 case 0xf5: /* cmc */
5349 /* complement carry flag from eflags reg */
5350 ctxt
->eflags
^= X86_EFLAGS_CF
;
5352 case 0xf8: /* clc */
5353 ctxt
->eflags
&= ~X86_EFLAGS_CF
;
5355 case 0xf9: /* stc */
5356 ctxt
->eflags
|= X86_EFLAGS_CF
;
5358 case 0xfc: /* cld */
5359 ctxt
->eflags
&= ~X86_EFLAGS_DF
;
5361 case 0xfd: /* std */
5362 ctxt
->eflags
|= X86_EFLAGS_DF
;
5365 goto cannot_emulate
;
5368 if (rc
!= X86EMUL_CONTINUE
)
5372 if (ctxt
->d
& SrcWrite
) {
5373 BUG_ON(ctxt
->src
.type
== OP_MEM
|| ctxt
->src
.type
== OP_MEM_STR
);
5374 rc
= writeback(ctxt
, &ctxt
->src
);
5375 if (rc
!= X86EMUL_CONTINUE
)
5378 if (!(ctxt
->d
& NoWrite
)) {
5379 rc
= writeback(ctxt
, &ctxt
->dst
);
5380 if (rc
!= X86EMUL_CONTINUE
)
5385 * restore dst type in case the decoding will be reused
5386 * (happens for string instruction )
5388 ctxt
->dst
.type
= saved_dst_type
;
5390 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5391 string_addr_inc(ctxt
, VCPU_REGS_RSI
, &ctxt
->src
);
5393 if ((ctxt
->d
& DstMask
) == DstDI
)
5394 string_addr_inc(ctxt
, VCPU_REGS_RDI
, &ctxt
->dst
);
5396 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5398 struct read_cache
*r
= &ctxt
->io_read
;
5399 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5400 count
= ctxt
->src
.count
;
5402 count
= ctxt
->dst
.count
;
5403 register_address_increment(ctxt
, VCPU_REGS_RCX
, -count
);
5405 if (!string_insn_completed(ctxt
)) {
5407 * Re-enter guest when pio read ahead buffer is empty
5408 * or, if it is not used, after each 1024 iteration.
5410 if ((r
->end
!= 0 || reg_read(ctxt
, VCPU_REGS_RCX
) & 0x3ff) &&
5411 (r
->end
== 0 || r
->end
!= r
->pos
)) {
5413 * Reset read cache. Usually happens before
5414 * decode, but since instruction is restarted
5415 * we have to do it here.
5417 ctxt
->mem_read
.end
= 0;
5418 writeback_registers(ctxt
);
5419 return EMULATION_RESTART
;
5421 goto done
; /* skip rip writeback */
5423 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5426 ctxt
->eip
= ctxt
->_eip
;
5429 if (rc
== X86EMUL_PROPAGATE_FAULT
) {
5430 WARN_ON(ctxt
->exception
.vector
> 0x1f);
5431 ctxt
->have_exception
= true;
5433 if (rc
== X86EMUL_INTERCEPTED
)
5434 return EMULATION_INTERCEPTED
;
5436 if (rc
== X86EMUL_CONTINUE
)
5437 writeback_registers(ctxt
);
5439 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
5443 case 0x09: /* wbinvd */
5444 (ctxt
->ops
->wbinvd
)(ctxt
);
5446 case 0x08: /* invd */
5447 case 0x0d: /* GrpP (prefetch) */
5448 case 0x18: /* Grp16 (prefetch/nop) */
5449 case 0x1f: /* nop */
5451 case 0x20: /* mov cr, reg */
5452 ctxt
->dst
.val
= ops
->get_cr(ctxt
, ctxt
->modrm_reg
);
5454 case 0x21: /* mov from dr to reg */
5455 ops
->get_dr(ctxt
, ctxt
->modrm_reg
, &ctxt
->dst
.val
);
5457 case 0x40 ... 0x4f: /* cmov */
5458 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5459 ctxt
->dst
.val
= ctxt
->src
.val
;
5460 else if (ctxt
->op_bytes
!= 4)
5461 ctxt
->dst
.type
= OP_NONE
; /* no writeback */
5463 case 0x80 ... 0x8f: /* jnz rel, etc*/
5464 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5465 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5467 case 0x90 ... 0x9f: /* setcc r/m8 */
5468 ctxt
->dst
.val
= test_cc(ctxt
->b
, ctxt
->eflags
);
5470 case 0xb6 ... 0xb7: /* movzx */
5471 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5472 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (u8
) ctxt
->src
.val
5473 : (u16
) ctxt
->src
.val
;
5475 case 0xbe ... 0xbf: /* movsx */
5476 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5477 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (s8
) ctxt
->src
.val
:
5478 (s16
) ctxt
->src
.val
;
5481 goto cannot_emulate
;
5486 if (rc
!= X86EMUL_CONTINUE
)
5492 return EMULATION_FAILED
;
5495 void emulator_invalidate_register_cache(struct x86_emulate_ctxt
*ctxt
)
5497 invalidate_registers(ctxt
);
5500 void emulator_writeback_register_cache(struct x86_emulate_ctxt
*ctxt
)
5502 writeback_registers(ctxt
);