1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute
)(struct x86_emulate_ctxt
*ctxt
);
213 const struct opcode
*group
;
214 const struct group_dual
*gdual
;
215 const struct gprefix
*gprefix
;
216 const struct escape
*esc
;
217 const struct instr_dual
*idual
;
218 const struct mode_dual
*mdual
;
219 void (*fastop
)(struct fastop
*fake
);
221 int (*check_perm
)(struct x86_emulate_ctxt
*ctxt
);
225 struct opcode mod012
[8];
226 struct opcode mod3
[8];
230 struct opcode pfx_no
;
231 struct opcode pfx_66
;
232 struct opcode pfx_f2
;
233 struct opcode pfx_f3
;
238 struct opcode high
[64];
242 struct opcode mod012
;
247 struct opcode mode32
;
248 struct opcode mode64
;
251 /* EFLAGS bit definitions. */
252 #define EFLG_ID (1<<21)
253 #define EFLG_VIP (1<<20)
254 #define EFLG_VIF (1<<19)
255 #define EFLG_AC (1<<18)
256 #define EFLG_VM (1<<17)
257 #define EFLG_RF (1<<16)
258 #define EFLG_IOPL (3<<12)
259 #define EFLG_NT (1<<14)
260 #define EFLG_OF (1<<11)
261 #define EFLG_DF (1<<10)
262 #define EFLG_IF (1<<9)
263 #define EFLG_TF (1<<8)
264 #define EFLG_SF (1<<7)
265 #define EFLG_ZF (1<<6)
266 #define EFLG_AF (1<<4)
267 #define EFLG_PF (1<<2)
268 #define EFLG_CF (1<<0)
270 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271 #define EFLG_RESERVED_ONE_MASK 2
273 enum x86_transfer_type
{
275 X86_TRANSFER_CALL_JMP
,
277 X86_TRANSFER_TASK_SWITCH
,
280 static ulong
reg_read(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
282 if (!(ctxt
->regs_valid
& (1 << nr
))) {
283 ctxt
->regs_valid
|= 1 << nr
;
284 ctxt
->_regs
[nr
] = ctxt
->ops
->read_gpr(ctxt
, nr
);
286 return ctxt
->_regs
[nr
];
289 static ulong
*reg_write(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
291 ctxt
->regs_valid
|= 1 << nr
;
292 ctxt
->regs_dirty
|= 1 << nr
;
293 return &ctxt
->_regs
[nr
];
296 static ulong
*reg_rmw(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
299 return reg_write(ctxt
, nr
);
302 static void writeback_registers(struct x86_emulate_ctxt
*ctxt
)
306 for_each_set_bit(reg
, (ulong
*)&ctxt
->regs_dirty
, 16)
307 ctxt
->ops
->write_gpr(ctxt
, reg
, ctxt
->_regs
[reg
]);
310 static void invalidate_registers(struct x86_emulate_ctxt
*ctxt
)
312 ctxt
->regs_dirty
= 0;
313 ctxt
->regs_valid
= 0;
317 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation.
320 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
328 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*));
330 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
331 #define FOP_RET "ret \n\t"
333 #define FOP_START(op) \
334 extern void em_##op(struct fastop *fake); \
335 asm(".pushsection .text, \"ax\" \n\t" \
336 ".global em_" #op " \n\t" \
343 #define FOPNOP() FOP_ALIGN FOP_RET
345 #define FOP1E(op, dst) \
346 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
348 #define FOP1EEX(op, dst) \
349 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
351 #define FASTOP1(op) \
356 ON64(FOP1E(op##q, rax)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m) */
360 #define FASTOP1SRC2(op, name) \
365 ON64(FOP1E(op, rcx)) \
368 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
369 #define FASTOP1SRC2EX(op, name) \
374 ON64(FOP1EEX(op, rcx)) \
377 #define FOP2E(op, dst, src) \
378 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
418 /* 3-operand, word-only, src2=cl */
419 #define FASTOP3WCL(op) \
422 FOP3E(op##w, ax, dx, cl) \
423 FOP3E(op##l, eax, edx, cl) \
424 ON64(FOP3E(op##q, rax, rdx, cl)) \
427 /* Special case for SETcc - 1 instruction per cc */
428 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
430 asm(".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret");
452 FOP_START(salc
) "pushf; sbb %al, %al; popf \n\t" FOP_RET
455 static int emulator_check_intercept(struct x86_emulate_ctxt
*ctxt
,
456 enum x86_intercept intercept
,
457 enum x86_intercept_stage stage
)
459 struct x86_instruction_info info
= {
460 .intercept
= intercept
,
461 .rep_prefix
= ctxt
->rep_prefix
,
462 .modrm_mod
= ctxt
->modrm_mod
,
463 .modrm_reg
= ctxt
->modrm_reg
,
464 .modrm_rm
= ctxt
->modrm_rm
,
465 .src_val
= ctxt
->src
.val64
,
466 .dst_val
= ctxt
->dst
.val64
,
467 .src_bytes
= ctxt
->src
.bytes
,
468 .dst_bytes
= ctxt
->dst
.bytes
,
469 .ad_bytes
= ctxt
->ad_bytes
,
470 .next_rip
= ctxt
->eip
,
473 return ctxt
->ops
->intercept(ctxt
, &info
, stage
);
476 static void assign_masked(ulong
*dest
, ulong src
, ulong mask
)
478 *dest
= (*dest
& ~mask
) | (src
& mask
);
481 static inline unsigned long ad_mask(struct x86_emulate_ctxt
*ctxt
)
483 return (1UL << (ctxt
->ad_bytes
<< 3)) - 1;
486 static ulong
stack_mask(struct x86_emulate_ctxt
*ctxt
)
489 struct desc_struct ss
;
491 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
493 ctxt
->ops
->get_segment(ctxt
, &sel
, &ss
, NULL
, VCPU_SREG_SS
);
494 return ~0U >> ((ss
.d
^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
497 static int stack_size(struct x86_emulate_ctxt
*ctxt
)
499 return (__fls(stack_mask(ctxt
)) + 1) >> 3;
502 /* Access/update address held in a register, based on addressing mode. */
503 static inline unsigned long
504 address_mask(struct x86_emulate_ctxt
*ctxt
, unsigned long reg
)
506 if (ctxt
->ad_bytes
== sizeof(unsigned long))
509 return reg
& ad_mask(ctxt
);
512 static inline unsigned long
513 register_address(struct x86_emulate_ctxt
*ctxt
, int reg
)
515 return address_mask(ctxt
, reg_read(ctxt
, reg
));
518 static void masked_increment(ulong
*reg
, ulong mask
, int inc
)
520 assign_masked(reg
, *reg
+ inc
, mask
);
524 register_address_increment(struct x86_emulate_ctxt
*ctxt
, int reg
, int inc
)
528 if (ctxt
->ad_bytes
== sizeof(unsigned long))
531 mask
= ad_mask(ctxt
);
532 masked_increment(reg_rmw(ctxt
, reg
), mask
, inc
);
535 static void rsp_increment(struct x86_emulate_ctxt
*ctxt
, int inc
)
537 masked_increment(reg_rmw(ctxt
, VCPU_REGS_RSP
), stack_mask(ctxt
), inc
);
540 static u32
desc_limit_scaled(struct desc_struct
*desc
)
542 u32 limit
= get_desc_limit(desc
);
544 return desc
->g
? (limit
<< 12) | 0xfff : limit
;
547 static unsigned long seg_base(struct x86_emulate_ctxt
*ctxt
, int seg
)
549 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&& seg
< VCPU_SREG_FS
)
552 return ctxt
->ops
->get_cached_segment_base(ctxt
, seg
);
555 static int emulate_exception(struct x86_emulate_ctxt
*ctxt
, int vec
,
556 u32 error
, bool valid
)
559 ctxt
->exception
.vector
= vec
;
560 ctxt
->exception
.error_code
= error
;
561 ctxt
->exception
.error_code_valid
= valid
;
562 return X86EMUL_PROPAGATE_FAULT
;
565 static int emulate_db(struct x86_emulate_ctxt
*ctxt
)
567 return emulate_exception(ctxt
, DB_VECTOR
, 0, false);
570 static int emulate_gp(struct x86_emulate_ctxt
*ctxt
, int err
)
572 return emulate_exception(ctxt
, GP_VECTOR
, err
, true);
575 static int emulate_ss(struct x86_emulate_ctxt
*ctxt
, int err
)
577 return emulate_exception(ctxt
, SS_VECTOR
, err
, true);
580 static int emulate_ud(struct x86_emulate_ctxt
*ctxt
)
582 return emulate_exception(ctxt
, UD_VECTOR
, 0, false);
585 static int emulate_ts(struct x86_emulate_ctxt
*ctxt
, int err
)
587 return emulate_exception(ctxt
, TS_VECTOR
, err
, true);
590 static int emulate_de(struct x86_emulate_ctxt
*ctxt
)
592 return emulate_exception(ctxt
, DE_VECTOR
, 0, false);
595 static int emulate_nm(struct x86_emulate_ctxt
*ctxt
)
597 return emulate_exception(ctxt
, NM_VECTOR
, 0, false);
600 static u16
get_segment_selector(struct x86_emulate_ctxt
*ctxt
, unsigned seg
)
603 struct desc_struct desc
;
605 ctxt
->ops
->get_segment(ctxt
, &selector
, &desc
, NULL
, seg
);
609 static void set_segment_selector(struct x86_emulate_ctxt
*ctxt
, u16 selector
,
614 struct desc_struct desc
;
616 ctxt
->ops
->get_segment(ctxt
, &dummy
, &desc
, &base3
, seg
);
617 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, seg
);
621 * x86 defines three classes of vector instructions: explicitly
622 * aligned, explicitly unaligned, and the rest, which change behaviour
623 * depending on whether they're AVX encoded or not.
625 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
626 * subject to the same check.
628 static bool insn_aligned(struct x86_emulate_ctxt
*ctxt
, unsigned size
)
630 if (likely(size
< 16))
633 if (ctxt
->d
& Aligned
)
635 else if (ctxt
->d
& Unaligned
)
637 else if (ctxt
->d
& Avx
)
643 static __always_inline
int __linearize(struct x86_emulate_ctxt
*ctxt
,
644 struct segmented_address addr
,
645 unsigned *max_size
, unsigned size
,
646 bool write
, bool fetch
,
647 enum x86emul_mode mode
, ulong
*linear
)
649 struct desc_struct desc
;
655 la
= seg_base(ctxt
, addr
.seg
) + addr
.ea
;
658 case X86EMUL_MODE_PROT64
:
659 if (is_noncanonical_address(la
))
662 *max_size
= min_t(u64
, ~0u, (1ull << 48) - la
);
663 if (size
> *max_size
)
667 usable
= ctxt
->ops
->get_segment(ctxt
, &sel
, &desc
, NULL
,
671 /* code segment in protected mode or read-only data segment */
672 if ((((ctxt
->mode
!= X86EMUL_MODE_REAL
) && (desc
.type
& 8))
673 || !(desc
.type
& 2)) && write
)
675 /* unreadable code segment */
676 if (!fetch
&& (desc
.type
& 8) && !(desc
.type
& 2))
678 lim
= desc_limit_scaled(&desc
);
679 if (!(desc
.type
& 8) && (desc
.type
& 4)) {
680 /* expand-down segment */
683 lim
= desc
.d
? 0xffffffff : 0xffff;
687 *max_size
= min_t(u64
, ~0u, (u64
)lim
+ 1 - addr
.ea
);
688 if (size
> *max_size
)
693 if (insn_aligned(ctxt
, size
) && ((la
& (size
- 1)) != 0))
694 return emulate_gp(ctxt
, 0);
696 return X86EMUL_CONTINUE
;
698 if (addr
.seg
== VCPU_SREG_SS
)
699 return emulate_ss(ctxt
, 0);
701 return emulate_gp(ctxt
, 0);
704 static int linearize(struct x86_emulate_ctxt
*ctxt
,
705 struct segmented_address addr
,
706 unsigned size
, bool write
,
710 return __linearize(ctxt
, addr
, &max_size
, size
, write
, false,
714 static inline int assign_eip(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
715 enum x86emul_mode mode
)
720 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
723 if (ctxt
->op_bytes
!= sizeof(unsigned long))
724 addr
.ea
= dst
& ((1UL << (ctxt
->op_bytes
<< 3)) - 1);
725 rc
= __linearize(ctxt
, addr
, &max_size
, 1, false, true, mode
, &linear
);
726 if (rc
== X86EMUL_CONTINUE
)
727 ctxt
->_eip
= addr
.ea
;
731 static inline int assign_eip_near(struct x86_emulate_ctxt
*ctxt
, ulong dst
)
733 return assign_eip(ctxt
, dst
, ctxt
->mode
);
736 static int assign_eip_far(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
737 const struct desc_struct
*cs_desc
)
739 enum x86emul_mode mode
= ctxt
->mode
;
742 if (ctxt
->mode
>= X86EMUL_MODE_PROT32
&& cs_desc
->l
) {
745 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
747 mode
= X86EMUL_MODE_PROT64
;
750 if (mode
== X86EMUL_MODE_PROT16
|| mode
== X86EMUL_MODE_PROT32
)
751 mode
= cs_desc
->d
? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
752 return assign_eip(ctxt
, dst
, mode
);
755 static inline int jmp_rel(struct x86_emulate_ctxt
*ctxt
, int rel
)
757 return assign_eip_near(ctxt
, ctxt
->_eip
+ rel
);
760 static int segmented_read_std(struct x86_emulate_ctxt
*ctxt
,
761 struct segmented_address addr
,
768 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
769 if (rc
!= X86EMUL_CONTINUE
)
771 return ctxt
->ops
->read_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
775 * Prefetch the remaining bytes of the instruction without crossing page
776 * boundary if they are not in fetch_cache yet.
778 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
, int op_size
)
781 unsigned size
, max_size
;
782 unsigned long linear
;
783 int cur_size
= ctxt
->fetch
.end
- ctxt
->fetch
.data
;
784 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
785 .ea
= ctxt
->eip
+ cur_size
};
788 * We do not know exactly how many bytes will be needed, and
789 * __linearize is expensive, so fetch as much as possible. We
790 * just have to avoid going beyond the 15 byte limit, the end
791 * of the segment, or the end of the page.
793 * __linearize is called with size 0 so that it does not do any
794 * boundary check itself. Instead, we use max_size to check
797 rc
= __linearize(ctxt
, addr
, &max_size
, 0, false, true, ctxt
->mode
,
799 if (unlikely(rc
!= X86EMUL_CONTINUE
))
802 size
= min_t(unsigned, 15UL ^ cur_size
, max_size
);
803 size
= min_t(unsigned, size
, PAGE_SIZE
- offset_in_page(linear
));
806 * One instruction can only straddle two pages,
807 * and one has been loaded at the beginning of
808 * x86_decode_insn. So, if not enough bytes
809 * still, we must have hit the 15-byte boundary.
811 if (unlikely(size
< op_size
))
812 return emulate_gp(ctxt
, 0);
814 rc
= ctxt
->ops
->fetch(ctxt
, linear
, ctxt
->fetch
.end
,
815 size
, &ctxt
->exception
);
816 if (unlikely(rc
!= X86EMUL_CONTINUE
))
818 ctxt
->fetch
.end
+= size
;
819 return X86EMUL_CONTINUE
;
822 static __always_inline
int do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
,
825 unsigned done_size
= ctxt
->fetch
.end
- ctxt
->fetch
.ptr
;
827 if (unlikely(done_size
< size
))
828 return __do_insn_fetch_bytes(ctxt
, size
- done_size
);
830 return X86EMUL_CONTINUE
;
833 /* Fetch next part of the instruction being emulated. */
834 #define insn_fetch(_type, _ctxt) \
837 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
838 if (rc != X86EMUL_CONTINUE) \
840 ctxt->_eip += sizeof(_type); \
841 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
842 ctxt->fetch.ptr += sizeof(_type); \
846 #define insn_fetch_arr(_arr, _size, _ctxt) \
848 rc = do_insn_fetch_bytes(_ctxt, _size); \
849 if (rc != X86EMUL_CONTINUE) \
851 ctxt->_eip += (_size); \
852 memcpy(_arr, ctxt->fetch.ptr, _size); \
853 ctxt->fetch.ptr += (_size); \
857 * Given the 'reg' portion of a ModRM byte, and a register block, return a
858 * pointer into the block that addresses the relevant register.
859 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
861 static void *decode_register(struct x86_emulate_ctxt
*ctxt
, u8 modrm_reg
,
865 int highbyte_regs
= (ctxt
->rex_prefix
== 0) && byteop
;
867 if (highbyte_regs
&& modrm_reg
>= 4 && modrm_reg
< 8)
868 p
= (unsigned char *)reg_rmw(ctxt
, modrm_reg
& 3) + 1;
870 p
= reg_rmw(ctxt
, modrm_reg
);
874 static int read_descriptor(struct x86_emulate_ctxt
*ctxt
,
875 struct segmented_address addr
,
876 u16
*size
, unsigned long *address
, int op_bytes
)
883 rc
= segmented_read_std(ctxt
, addr
, size
, 2);
884 if (rc
!= X86EMUL_CONTINUE
)
887 rc
= segmented_read_std(ctxt
, addr
, address
, op_bytes
);
901 FASTOP1SRC2(mul
, mul_ex
);
902 FASTOP1SRC2(imul
, imul_ex
);
903 FASTOP1SRC2EX(div
, div_ex
);
904 FASTOP1SRC2EX(idiv
, idiv_ex
);
933 FASTOP2R(cmp
, cmp_r
);
935 static u8
test_cc(unsigned int condition
, unsigned long flags
)
938 void (*fop
)(void) = (void *)em_setcc
+ 4 * (condition
& 0xf);
940 flags
= (flags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
941 asm("push %[flags]; popf; call *%[fastop]"
942 : "=a"(rc
) : [fastop
]"r"(fop
), [flags
]"r"(flags
));
946 static void fetch_register_operand(struct operand
*op
)
950 op
->val
= *(u8
*)op
->addr
.reg
;
953 op
->val
= *(u16
*)op
->addr
.reg
;
956 op
->val
= *(u32
*)op
->addr
.reg
;
959 op
->val
= *(u64
*)op
->addr
.reg
;
964 static void read_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
, int reg
)
966 ctxt
->ops
->get_fpu(ctxt
);
968 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data
)); break;
969 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data
)); break;
970 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data
)); break;
971 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data
)); break;
972 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data
)); break;
973 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data
)); break;
974 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data
)); break;
975 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data
)); break;
977 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data
)); break;
978 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data
)); break;
979 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data
)); break;
980 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data
)); break;
981 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data
)); break;
982 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data
)); break;
983 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data
)); break;
984 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data
)); break;
988 ctxt
->ops
->put_fpu(ctxt
);
991 static void write_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
,
994 ctxt
->ops
->get_fpu(ctxt
);
996 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data
)); break;
997 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data
)); break;
998 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data
)); break;
999 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data
)); break;
1000 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data
)); break;
1001 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data
)); break;
1002 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data
)); break;
1003 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data
)); break;
1004 #ifdef CONFIG_X86_64
1005 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data
)); break;
1006 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data
)); break;
1007 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data
)); break;
1008 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data
)); break;
1009 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data
)); break;
1010 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data
)); break;
1011 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data
)); break;
1012 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data
)); break;
1016 ctxt
->ops
->put_fpu(ctxt
);
1019 static void read_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1021 ctxt
->ops
->get_fpu(ctxt
);
1023 case 0: asm("movq %%mm0, %0" : "=m"(*data
)); break;
1024 case 1: asm("movq %%mm1, %0" : "=m"(*data
)); break;
1025 case 2: asm("movq %%mm2, %0" : "=m"(*data
)); break;
1026 case 3: asm("movq %%mm3, %0" : "=m"(*data
)); break;
1027 case 4: asm("movq %%mm4, %0" : "=m"(*data
)); break;
1028 case 5: asm("movq %%mm5, %0" : "=m"(*data
)); break;
1029 case 6: asm("movq %%mm6, %0" : "=m"(*data
)); break;
1030 case 7: asm("movq %%mm7, %0" : "=m"(*data
)); break;
1033 ctxt
->ops
->put_fpu(ctxt
);
1036 static void write_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1038 ctxt
->ops
->get_fpu(ctxt
);
1040 case 0: asm("movq %0, %%mm0" : : "m"(*data
)); break;
1041 case 1: asm("movq %0, %%mm1" : : "m"(*data
)); break;
1042 case 2: asm("movq %0, %%mm2" : : "m"(*data
)); break;
1043 case 3: asm("movq %0, %%mm3" : : "m"(*data
)); break;
1044 case 4: asm("movq %0, %%mm4" : : "m"(*data
)); break;
1045 case 5: asm("movq %0, %%mm5" : : "m"(*data
)); break;
1046 case 6: asm("movq %0, %%mm6" : : "m"(*data
)); break;
1047 case 7: asm("movq %0, %%mm7" : : "m"(*data
)); break;
1050 ctxt
->ops
->put_fpu(ctxt
);
1053 static int em_fninit(struct x86_emulate_ctxt
*ctxt
)
1055 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1056 return emulate_nm(ctxt
);
1058 ctxt
->ops
->get_fpu(ctxt
);
1059 asm volatile("fninit");
1060 ctxt
->ops
->put_fpu(ctxt
);
1061 return X86EMUL_CONTINUE
;
1064 static int em_fnstcw(struct x86_emulate_ctxt
*ctxt
)
1068 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1069 return emulate_nm(ctxt
);
1071 ctxt
->ops
->get_fpu(ctxt
);
1072 asm volatile("fnstcw %0": "+m"(fcw
));
1073 ctxt
->ops
->put_fpu(ctxt
);
1075 ctxt
->dst
.val
= fcw
;
1077 return X86EMUL_CONTINUE
;
1080 static int em_fnstsw(struct x86_emulate_ctxt
*ctxt
)
1084 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1085 return emulate_nm(ctxt
);
1087 ctxt
->ops
->get_fpu(ctxt
);
1088 asm volatile("fnstsw %0": "+m"(fsw
));
1089 ctxt
->ops
->put_fpu(ctxt
);
1091 ctxt
->dst
.val
= fsw
;
1093 return X86EMUL_CONTINUE
;
1096 static void decode_register_operand(struct x86_emulate_ctxt
*ctxt
,
1099 unsigned reg
= ctxt
->modrm_reg
;
1101 if (!(ctxt
->d
& ModRM
))
1102 reg
= (ctxt
->b
& 7) | ((ctxt
->rex_prefix
& 1) << 3);
1104 if (ctxt
->d
& Sse
) {
1108 read_sse_reg(ctxt
, &op
->vec_val
, reg
);
1111 if (ctxt
->d
& Mmx
) {
1120 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1121 op
->addr
.reg
= decode_register(ctxt
, reg
, ctxt
->d
& ByteOp
);
1123 fetch_register_operand(op
);
1124 op
->orig_val
= op
->val
;
1127 static void adjust_modrm_seg(struct x86_emulate_ctxt
*ctxt
, int base_reg
)
1129 if (base_reg
== VCPU_REGS_RSP
|| base_reg
== VCPU_REGS_RBP
)
1130 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1133 static int decode_modrm(struct x86_emulate_ctxt
*ctxt
,
1137 int index_reg
, base_reg
, scale
;
1138 int rc
= X86EMUL_CONTINUE
;
1141 ctxt
->modrm_reg
= ((ctxt
->rex_prefix
<< 1) & 8); /* REX.R */
1142 index_reg
= (ctxt
->rex_prefix
<< 2) & 8; /* REX.X */
1143 base_reg
= (ctxt
->rex_prefix
<< 3) & 8; /* REX.B */
1145 ctxt
->modrm_mod
= (ctxt
->modrm
& 0xc0) >> 6;
1146 ctxt
->modrm_reg
|= (ctxt
->modrm
& 0x38) >> 3;
1147 ctxt
->modrm_rm
= base_reg
| (ctxt
->modrm
& 0x07);
1148 ctxt
->modrm_seg
= VCPU_SREG_DS
;
1150 if (ctxt
->modrm_mod
== 3 || (ctxt
->d
& NoMod
)) {
1152 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1153 op
->addr
.reg
= decode_register(ctxt
, ctxt
->modrm_rm
,
1155 if (ctxt
->d
& Sse
) {
1158 op
->addr
.xmm
= ctxt
->modrm_rm
;
1159 read_sse_reg(ctxt
, &op
->vec_val
, ctxt
->modrm_rm
);
1162 if (ctxt
->d
& Mmx
) {
1165 op
->addr
.mm
= ctxt
->modrm_rm
& 7;
1168 fetch_register_operand(op
);
1174 if (ctxt
->ad_bytes
== 2) {
1175 unsigned bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
1176 unsigned bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1177 unsigned si
= reg_read(ctxt
, VCPU_REGS_RSI
);
1178 unsigned di
= reg_read(ctxt
, VCPU_REGS_RDI
);
1180 /* 16-bit ModR/M decode. */
1181 switch (ctxt
->modrm_mod
) {
1183 if (ctxt
->modrm_rm
== 6)
1184 modrm_ea
+= insn_fetch(u16
, ctxt
);
1187 modrm_ea
+= insn_fetch(s8
, ctxt
);
1190 modrm_ea
+= insn_fetch(u16
, ctxt
);
1193 switch (ctxt
->modrm_rm
) {
1195 modrm_ea
+= bx
+ si
;
1198 modrm_ea
+= bx
+ di
;
1201 modrm_ea
+= bp
+ si
;
1204 modrm_ea
+= bp
+ di
;
1213 if (ctxt
->modrm_mod
!= 0)
1220 if (ctxt
->modrm_rm
== 2 || ctxt
->modrm_rm
== 3 ||
1221 (ctxt
->modrm_rm
== 6 && ctxt
->modrm_mod
!= 0))
1222 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1223 modrm_ea
= (u16
)modrm_ea
;
1225 /* 32/64-bit ModR/M decode. */
1226 if ((ctxt
->modrm_rm
& 7) == 4) {
1227 sib
= insn_fetch(u8
, ctxt
);
1228 index_reg
|= (sib
>> 3) & 7;
1229 base_reg
|= sib
& 7;
1232 if ((base_reg
& 7) == 5 && ctxt
->modrm_mod
== 0)
1233 modrm_ea
+= insn_fetch(s32
, ctxt
);
1235 modrm_ea
+= reg_read(ctxt
, base_reg
);
1236 adjust_modrm_seg(ctxt
, base_reg
);
1237 /* Increment ESP on POP [ESP] */
1238 if ((ctxt
->d
& IncSP
) &&
1239 base_reg
== VCPU_REGS_RSP
)
1240 modrm_ea
+= ctxt
->op_bytes
;
1243 modrm_ea
+= reg_read(ctxt
, index_reg
) << scale
;
1244 } else if ((ctxt
->modrm_rm
& 7) == 5 && ctxt
->modrm_mod
== 0) {
1245 modrm_ea
+= insn_fetch(s32
, ctxt
);
1246 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
1247 ctxt
->rip_relative
= 1;
1249 base_reg
= ctxt
->modrm_rm
;
1250 modrm_ea
+= reg_read(ctxt
, base_reg
);
1251 adjust_modrm_seg(ctxt
, base_reg
);
1253 switch (ctxt
->modrm_mod
) {
1255 modrm_ea
+= insn_fetch(s8
, ctxt
);
1258 modrm_ea
+= insn_fetch(s32
, ctxt
);
1262 op
->addr
.mem
.ea
= modrm_ea
;
1263 if (ctxt
->ad_bytes
!= 8)
1264 ctxt
->memop
.addr
.mem
.ea
= (u32
)ctxt
->memop
.addr
.mem
.ea
;
1270 static int decode_abs(struct x86_emulate_ctxt
*ctxt
,
1273 int rc
= X86EMUL_CONTINUE
;
1276 switch (ctxt
->ad_bytes
) {
1278 op
->addr
.mem
.ea
= insn_fetch(u16
, ctxt
);
1281 op
->addr
.mem
.ea
= insn_fetch(u32
, ctxt
);
1284 op
->addr
.mem
.ea
= insn_fetch(u64
, ctxt
);
1291 static void fetch_bit_operand(struct x86_emulate_ctxt
*ctxt
)
1295 if (ctxt
->dst
.type
== OP_MEM
&& ctxt
->src
.type
== OP_REG
) {
1296 mask
= ~((long)ctxt
->dst
.bytes
* 8 - 1);
1298 if (ctxt
->src
.bytes
== 2)
1299 sv
= (s16
)ctxt
->src
.val
& (s16
)mask
;
1300 else if (ctxt
->src
.bytes
== 4)
1301 sv
= (s32
)ctxt
->src
.val
& (s32
)mask
;
1303 sv
= (s64
)ctxt
->src
.val
& (s64
)mask
;
1305 ctxt
->dst
.addr
.mem
.ea
= address_mask(ctxt
,
1306 ctxt
->dst
.addr
.mem
.ea
+ (sv
>> 3));
1309 /* only subword offset */
1310 ctxt
->src
.val
&= (ctxt
->dst
.bytes
<< 3) - 1;
1313 static int read_emulated(struct x86_emulate_ctxt
*ctxt
,
1314 unsigned long addr
, void *dest
, unsigned size
)
1317 struct read_cache
*mc
= &ctxt
->mem_read
;
1319 if (mc
->pos
< mc
->end
)
1322 WARN_ON((mc
->end
+ size
) >= sizeof(mc
->data
));
1324 rc
= ctxt
->ops
->read_emulated(ctxt
, addr
, mc
->data
+ mc
->end
, size
,
1326 if (rc
!= X86EMUL_CONTINUE
)
1332 memcpy(dest
, mc
->data
+ mc
->pos
, size
);
1334 return X86EMUL_CONTINUE
;
1337 static int segmented_read(struct x86_emulate_ctxt
*ctxt
,
1338 struct segmented_address addr
,
1345 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
1346 if (rc
!= X86EMUL_CONTINUE
)
1348 return read_emulated(ctxt
, linear
, data
, size
);
1351 static int segmented_write(struct x86_emulate_ctxt
*ctxt
,
1352 struct segmented_address addr
,
1359 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1360 if (rc
!= X86EMUL_CONTINUE
)
1362 return ctxt
->ops
->write_emulated(ctxt
, linear
, data
, size
,
1366 static int segmented_cmpxchg(struct x86_emulate_ctxt
*ctxt
,
1367 struct segmented_address addr
,
1368 const void *orig_data
, const void *data
,
1374 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1375 if (rc
!= X86EMUL_CONTINUE
)
1377 return ctxt
->ops
->cmpxchg_emulated(ctxt
, linear
, orig_data
, data
,
1378 size
, &ctxt
->exception
);
1381 static int pio_in_emulated(struct x86_emulate_ctxt
*ctxt
,
1382 unsigned int size
, unsigned short port
,
1385 struct read_cache
*rc
= &ctxt
->io_read
;
1387 if (rc
->pos
== rc
->end
) { /* refill pio read ahead */
1388 unsigned int in_page
, n
;
1389 unsigned int count
= ctxt
->rep_prefix
?
1390 address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) : 1;
1391 in_page
= (ctxt
->eflags
& EFLG_DF
) ?
1392 offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
)) :
1393 PAGE_SIZE
- offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
));
1394 n
= min3(in_page
, (unsigned int)sizeof(rc
->data
) / size
, count
);
1397 rc
->pos
= rc
->end
= 0;
1398 if (!ctxt
->ops
->pio_in_emulated(ctxt
, size
, port
, rc
->data
, n
))
1403 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
) &&
1404 !(ctxt
->eflags
& EFLG_DF
)) {
1405 ctxt
->dst
.data
= rc
->data
+ rc
->pos
;
1406 ctxt
->dst
.type
= OP_MEM_STR
;
1407 ctxt
->dst
.count
= (rc
->end
- rc
->pos
) / size
;
1410 memcpy(dest
, rc
->data
+ rc
->pos
, size
);
1416 static int read_interrupt_descriptor(struct x86_emulate_ctxt
*ctxt
,
1417 u16 index
, struct desc_struct
*desc
)
1422 ctxt
->ops
->get_idt(ctxt
, &dt
);
1424 if (dt
.size
< index
* 8 + 7)
1425 return emulate_gp(ctxt
, index
<< 3 | 0x2);
1427 addr
= dt
.address
+ index
* 8;
1428 return ctxt
->ops
->read_std(ctxt
, addr
, desc
, sizeof *desc
,
1432 static void get_descriptor_table_ptr(struct x86_emulate_ctxt
*ctxt
,
1433 u16 selector
, struct desc_ptr
*dt
)
1435 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1438 if (selector
& 1 << 2) {
1439 struct desc_struct desc
;
1442 memset (dt
, 0, sizeof *dt
);
1443 if (!ops
->get_segment(ctxt
, &sel
, &desc
, &base3
,
1447 dt
->size
= desc_limit_scaled(&desc
); /* what if limit > 65535? */
1448 dt
->address
= get_desc_base(&desc
) | ((u64
)base3
<< 32);
1450 ops
->get_gdt(ctxt
, dt
);
1453 static int get_descriptor_ptr(struct x86_emulate_ctxt
*ctxt
,
1454 u16 selector
, ulong
*desc_addr_p
)
1457 u16 index
= selector
>> 3;
1460 get_descriptor_table_ptr(ctxt
, selector
, &dt
);
1462 if (dt
.size
< index
* 8 + 7)
1463 return emulate_gp(ctxt
, selector
& 0xfffc);
1465 addr
= dt
.address
+ index
* 8;
1467 #ifdef CONFIG_X86_64
1468 if (addr
>> 32 != 0) {
1471 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1472 if (!(efer
& EFER_LMA
))
1477 *desc_addr_p
= addr
;
1478 return X86EMUL_CONTINUE
;
1481 /* allowed just for 8 bytes segments */
1482 static int read_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1483 u16 selector
, struct desc_struct
*desc
,
1488 rc
= get_descriptor_ptr(ctxt
, selector
, desc_addr_p
);
1489 if (rc
!= X86EMUL_CONTINUE
)
1492 return ctxt
->ops
->read_std(ctxt
, *desc_addr_p
, desc
, sizeof(*desc
),
1496 /* allowed just for 8 bytes segments */
1497 static int write_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1498 u16 selector
, struct desc_struct
*desc
)
1503 rc
= get_descriptor_ptr(ctxt
, selector
, &addr
);
1504 if (rc
!= X86EMUL_CONTINUE
)
1507 return ctxt
->ops
->write_std(ctxt
, addr
, desc
, sizeof *desc
,
1511 /* Does not support long mode */
1512 static int __load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1513 u16 selector
, int seg
, u8 cpl
,
1514 enum x86_transfer_type transfer
,
1515 struct desc_struct
*desc
)
1517 struct desc_struct seg_desc
, old_desc
;
1519 unsigned err_vec
= GP_VECTOR
;
1521 bool null_selector
= !(selector
& ~0x3); /* 0000-0003 are null */
1527 memset(&seg_desc
, 0, sizeof seg_desc
);
1529 if (ctxt
->mode
== X86EMUL_MODE_REAL
) {
1530 /* set real mode segment descriptor (keep limit etc. for
1532 ctxt
->ops
->get_segment(ctxt
, &dummy
, &seg_desc
, NULL
, seg
);
1533 set_desc_base(&seg_desc
, selector
<< 4);
1535 } else if (seg
<= VCPU_SREG_GS
&& ctxt
->mode
== X86EMUL_MODE_VM86
) {
1536 /* VM86 needs a clean new segment descriptor */
1537 set_desc_base(&seg_desc
, selector
<< 4);
1538 set_desc_limit(&seg_desc
, 0xffff);
1548 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1549 if ((seg
== VCPU_SREG_CS
1550 || (seg
== VCPU_SREG_SS
1551 && (ctxt
->mode
!= X86EMUL_MODE_PROT64
|| rpl
!= cpl
))
1552 || seg
== VCPU_SREG_TR
)
1556 /* TR should be in GDT only */
1557 if (seg
== VCPU_SREG_TR
&& (selector
& (1 << 2)))
1560 if (null_selector
) /* for NULL selector skip all following checks */
1563 ret
= read_segment_descriptor(ctxt
, selector
, &seg_desc
, &desc_addr
);
1564 if (ret
!= X86EMUL_CONTINUE
)
1567 err_code
= selector
& 0xfffc;
1568 err_vec
= (transfer
== X86_TRANSFER_TASK_SWITCH
) ? TS_VECTOR
:
1571 /* can't load system descriptor into segment selector */
1572 if (seg
<= VCPU_SREG_GS
&& !seg_desc
.s
) {
1573 if (transfer
== X86_TRANSFER_CALL_JMP
)
1574 return X86EMUL_UNHANDLEABLE
;
1579 err_vec
= (seg
== VCPU_SREG_SS
) ? SS_VECTOR
: NP_VECTOR
;
1588 * segment is not a writable data segment or segment
1589 * selector's RPL != CPL or segment selector's RPL != CPL
1591 if (rpl
!= cpl
|| (seg_desc
.type
& 0xa) != 0x2 || dpl
!= cpl
)
1595 if (!(seg_desc
.type
& 8))
1598 if (seg_desc
.type
& 4) {
1604 if (rpl
> cpl
|| dpl
!= cpl
)
1607 /* in long-mode d/b must be clear if l is set */
1608 if (seg_desc
.d
&& seg_desc
.l
) {
1611 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1612 if (efer
& EFER_LMA
)
1616 /* CS(RPL) <- CPL */
1617 selector
= (selector
& 0xfffc) | cpl
;
1620 if (seg_desc
.s
|| (seg_desc
.type
!= 1 && seg_desc
.type
!= 9))
1622 old_desc
= seg_desc
;
1623 seg_desc
.type
|= 2; /* busy */
1624 ret
= ctxt
->ops
->cmpxchg_emulated(ctxt
, desc_addr
, &old_desc
, &seg_desc
,
1625 sizeof(seg_desc
), &ctxt
->exception
);
1626 if (ret
!= X86EMUL_CONTINUE
)
1629 case VCPU_SREG_LDTR
:
1630 if (seg_desc
.s
|| seg_desc
.type
!= 2)
1633 default: /* DS, ES, FS, or GS */
1635 * segment is not a data or readable code segment or
1636 * ((segment is a data or nonconforming code segment)
1637 * and (both RPL and CPL > DPL))
1639 if ((seg_desc
.type
& 0xa) == 0x8 ||
1640 (((seg_desc
.type
& 0xc) != 0xc) &&
1641 (rpl
> dpl
&& cpl
> dpl
)))
1647 /* mark segment as accessed */
1648 if (!(seg_desc
.type
& 1)) {
1650 ret
= write_segment_descriptor(ctxt
, selector
,
1652 if (ret
!= X86EMUL_CONTINUE
)
1655 } else if (ctxt
->mode
== X86EMUL_MODE_PROT64
) {
1656 ret
= ctxt
->ops
->read_std(ctxt
, desc_addr
+8, &base3
,
1657 sizeof(base3
), &ctxt
->exception
);
1658 if (ret
!= X86EMUL_CONTINUE
)
1660 if (is_noncanonical_address(get_desc_base(&seg_desc
) |
1661 ((u64
)base3
<< 32)))
1662 return emulate_gp(ctxt
, 0);
1665 ctxt
->ops
->set_segment(ctxt
, selector
, &seg_desc
, base3
, seg
);
1668 return X86EMUL_CONTINUE
;
1670 return emulate_exception(ctxt
, err_vec
, err_code
, true);
1673 static int load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1674 u16 selector
, int seg
)
1676 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
1677 return __load_segment_descriptor(ctxt
, selector
, seg
, cpl
,
1678 X86_TRANSFER_NONE
, NULL
);
1681 static void write_register_operand(struct operand
*op
)
1683 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1684 switch (op
->bytes
) {
1686 *(u8
*)op
->addr
.reg
= (u8
)op
->val
;
1689 *(u16
*)op
->addr
.reg
= (u16
)op
->val
;
1692 *op
->addr
.reg
= (u32
)op
->val
;
1693 break; /* 64b: zero-extend */
1695 *op
->addr
.reg
= op
->val
;
1700 static int writeback(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
)
1704 write_register_operand(op
);
1707 if (ctxt
->lock_prefix
)
1708 return segmented_cmpxchg(ctxt
,
1714 return segmented_write(ctxt
,
1720 return segmented_write(ctxt
,
1723 op
->bytes
* op
->count
);
1726 write_sse_reg(ctxt
, &op
->vec_val
, op
->addr
.xmm
);
1729 write_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
1737 return X86EMUL_CONTINUE
;
1740 static int push(struct x86_emulate_ctxt
*ctxt
, void *data
, int bytes
)
1742 struct segmented_address addr
;
1744 rsp_increment(ctxt
, -bytes
);
1745 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1746 addr
.seg
= VCPU_SREG_SS
;
1748 return segmented_write(ctxt
, addr
, data
, bytes
);
1751 static int em_push(struct x86_emulate_ctxt
*ctxt
)
1753 /* Disable writeback. */
1754 ctxt
->dst
.type
= OP_NONE
;
1755 return push(ctxt
, &ctxt
->src
.val
, ctxt
->op_bytes
);
1758 static int emulate_pop(struct x86_emulate_ctxt
*ctxt
,
1759 void *dest
, int len
)
1762 struct segmented_address addr
;
1764 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1765 addr
.seg
= VCPU_SREG_SS
;
1766 rc
= segmented_read(ctxt
, addr
, dest
, len
);
1767 if (rc
!= X86EMUL_CONTINUE
)
1770 rsp_increment(ctxt
, len
);
1774 static int em_pop(struct x86_emulate_ctxt
*ctxt
)
1776 return emulate_pop(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1779 static int emulate_popf(struct x86_emulate_ctxt
*ctxt
,
1780 void *dest
, int len
)
1783 unsigned long val
, change_mask
;
1784 int iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> IOPL_SHIFT
;
1785 int cpl
= ctxt
->ops
->cpl(ctxt
);
1787 rc
= emulate_pop(ctxt
, &val
, len
);
1788 if (rc
!= X86EMUL_CONTINUE
)
1791 change_mask
= EFLG_CF
| EFLG_PF
| EFLG_AF
| EFLG_ZF
| EFLG_SF
| EFLG_OF
1792 | EFLG_TF
| EFLG_DF
| EFLG_NT
| EFLG_AC
| EFLG_ID
;
1794 switch(ctxt
->mode
) {
1795 case X86EMUL_MODE_PROT64
:
1796 case X86EMUL_MODE_PROT32
:
1797 case X86EMUL_MODE_PROT16
:
1799 change_mask
|= EFLG_IOPL
;
1801 change_mask
|= EFLG_IF
;
1803 case X86EMUL_MODE_VM86
:
1805 return emulate_gp(ctxt
, 0);
1806 change_mask
|= EFLG_IF
;
1808 default: /* real mode */
1809 change_mask
|= (EFLG_IOPL
| EFLG_IF
);
1813 *(unsigned long *)dest
=
1814 (ctxt
->eflags
& ~change_mask
) | (val
& change_mask
);
1819 static int em_popf(struct x86_emulate_ctxt
*ctxt
)
1821 ctxt
->dst
.type
= OP_REG
;
1822 ctxt
->dst
.addr
.reg
= &ctxt
->eflags
;
1823 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
1824 return emulate_popf(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1827 static int em_enter(struct x86_emulate_ctxt
*ctxt
)
1830 unsigned frame_size
= ctxt
->src
.val
;
1831 unsigned nesting_level
= ctxt
->src2
.val
& 31;
1835 return X86EMUL_UNHANDLEABLE
;
1837 rbp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1838 rc
= push(ctxt
, &rbp
, stack_size(ctxt
));
1839 if (rc
!= X86EMUL_CONTINUE
)
1841 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RBP
), reg_read(ctxt
, VCPU_REGS_RSP
),
1843 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
),
1844 reg_read(ctxt
, VCPU_REGS_RSP
) - frame_size
,
1846 return X86EMUL_CONTINUE
;
1849 static int em_leave(struct x86_emulate_ctxt
*ctxt
)
1851 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
), reg_read(ctxt
, VCPU_REGS_RBP
),
1853 return emulate_pop(ctxt
, reg_rmw(ctxt
, VCPU_REGS_RBP
), ctxt
->op_bytes
);
1856 static int em_push_sreg(struct x86_emulate_ctxt
*ctxt
)
1858 int seg
= ctxt
->src2
.val
;
1860 ctxt
->src
.val
= get_segment_selector(ctxt
, seg
);
1861 if (ctxt
->op_bytes
== 4) {
1862 rsp_increment(ctxt
, -2);
1866 return em_push(ctxt
);
1869 static int em_pop_sreg(struct x86_emulate_ctxt
*ctxt
)
1871 int seg
= ctxt
->src2
.val
;
1872 unsigned long selector
;
1875 rc
= emulate_pop(ctxt
, &selector
, 2);
1876 if (rc
!= X86EMUL_CONTINUE
)
1879 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
1880 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
1881 if (ctxt
->op_bytes
> 2)
1882 rsp_increment(ctxt
, ctxt
->op_bytes
- 2);
1884 rc
= load_segment_descriptor(ctxt
, (u16
)selector
, seg
);
1888 static int em_pusha(struct x86_emulate_ctxt
*ctxt
)
1890 unsigned long old_esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
1891 int rc
= X86EMUL_CONTINUE
;
1892 int reg
= VCPU_REGS_RAX
;
1894 while (reg
<= VCPU_REGS_RDI
) {
1895 (reg
== VCPU_REGS_RSP
) ?
1896 (ctxt
->src
.val
= old_esp
) : (ctxt
->src
.val
= reg_read(ctxt
, reg
));
1899 if (rc
!= X86EMUL_CONTINUE
)
1908 static int em_pushf(struct x86_emulate_ctxt
*ctxt
)
1910 ctxt
->src
.val
= (unsigned long)ctxt
->eflags
& ~EFLG_VM
;
1911 return em_push(ctxt
);
1914 static int em_popa(struct x86_emulate_ctxt
*ctxt
)
1916 int rc
= X86EMUL_CONTINUE
;
1917 int reg
= VCPU_REGS_RDI
;
1919 while (reg
>= VCPU_REGS_RAX
) {
1920 if (reg
== VCPU_REGS_RSP
) {
1921 rsp_increment(ctxt
, ctxt
->op_bytes
);
1925 rc
= emulate_pop(ctxt
, reg_rmw(ctxt
, reg
), ctxt
->op_bytes
);
1926 if (rc
!= X86EMUL_CONTINUE
)
1933 static int __emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
1935 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1942 /* TODO: Add limit checks */
1943 ctxt
->src
.val
= ctxt
->eflags
;
1945 if (rc
!= X86EMUL_CONTINUE
)
1948 ctxt
->eflags
&= ~(EFLG_IF
| EFLG_TF
| EFLG_AC
);
1950 ctxt
->src
.val
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
1952 if (rc
!= X86EMUL_CONTINUE
)
1955 ctxt
->src
.val
= ctxt
->_eip
;
1957 if (rc
!= X86EMUL_CONTINUE
)
1960 ops
->get_idt(ctxt
, &dt
);
1962 eip_addr
= dt
.address
+ (irq
<< 2);
1963 cs_addr
= dt
.address
+ (irq
<< 2) + 2;
1965 rc
= ops
->read_std(ctxt
, cs_addr
, &cs
, 2, &ctxt
->exception
);
1966 if (rc
!= X86EMUL_CONTINUE
)
1969 rc
= ops
->read_std(ctxt
, eip_addr
, &eip
, 2, &ctxt
->exception
);
1970 if (rc
!= X86EMUL_CONTINUE
)
1973 rc
= load_segment_descriptor(ctxt
, cs
, VCPU_SREG_CS
);
1974 if (rc
!= X86EMUL_CONTINUE
)
1982 int emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
1986 invalidate_registers(ctxt
);
1987 rc
= __emulate_int_real(ctxt
, irq
);
1988 if (rc
== X86EMUL_CONTINUE
)
1989 writeback_registers(ctxt
);
1993 static int emulate_int(struct x86_emulate_ctxt
*ctxt
, int irq
)
1995 switch(ctxt
->mode
) {
1996 case X86EMUL_MODE_REAL
:
1997 return __emulate_int_real(ctxt
, irq
);
1998 case X86EMUL_MODE_VM86
:
1999 case X86EMUL_MODE_PROT16
:
2000 case X86EMUL_MODE_PROT32
:
2001 case X86EMUL_MODE_PROT64
:
2003 /* Protected mode interrupts unimplemented yet */
2004 return X86EMUL_UNHANDLEABLE
;
2008 static int emulate_iret_real(struct x86_emulate_ctxt
*ctxt
)
2010 int rc
= X86EMUL_CONTINUE
;
2011 unsigned long temp_eip
= 0;
2012 unsigned long temp_eflags
= 0;
2013 unsigned long cs
= 0;
2014 unsigned long mask
= EFLG_CF
| EFLG_PF
| EFLG_AF
| EFLG_ZF
| EFLG_SF
| EFLG_TF
|
2015 EFLG_IF
| EFLG_DF
| EFLG_OF
| EFLG_IOPL
| EFLG_NT
| EFLG_RF
|
2016 EFLG_AC
| EFLG_ID
| (1 << 1); /* Last one is the reserved bit */
2017 unsigned long vm86_mask
= EFLG_VM
| EFLG_VIF
| EFLG_VIP
;
2019 /* TODO: Add stack limit check */
2021 rc
= emulate_pop(ctxt
, &temp_eip
, ctxt
->op_bytes
);
2023 if (rc
!= X86EMUL_CONTINUE
)
2026 if (temp_eip
& ~0xffff)
2027 return emulate_gp(ctxt
, 0);
2029 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2031 if (rc
!= X86EMUL_CONTINUE
)
2034 rc
= emulate_pop(ctxt
, &temp_eflags
, ctxt
->op_bytes
);
2036 if (rc
!= X86EMUL_CONTINUE
)
2039 rc
= load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
);
2041 if (rc
!= X86EMUL_CONTINUE
)
2044 ctxt
->_eip
= temp_eip
;
2047 if (ctxt
->op_bytes
== 4)
2048 ctxt
->eflags
= ((temp_eflags
& mask
) | (ctxt
->eflags
& vm86_mask
));
2049 else if (ctxt
->op_bytes
== 2) {
2050 ctxt
->eflags
&= ~0xffff;
2051 ctxt
->eflags
|= temp_eflags
;
2054 ctxt
->eflags
&= ~EFLG_RESERVED_ZEROS_MASK
; /* Clear reserved zeros */
2055 ctxt
->eflags
|= EFLG_RESERVED_ONE_MASK
;
2056 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2061 static int em_iret(struct x86_emulate_ctxt
*ctxt
)
2063 switch(ctxt
->mode
) {
2064 case X86EMUL_MODE_REAL
:
2065 return emulate_iret_real(ctxt
);
2066 case X86EMUL_MODE_VM86
:
2067 case X86EMUL_MODE_PROT16
:
2068 case X86EMUL_MODE_PROT32
:
2069 case X86EMUL_MODE_PROT64
:
2071 /* iret from protected mode unimplemented yet */
2072 return X86EMUL_UNHANDLEABLE
;
2076 static int em_jmp_far(struct x86_emulate_ctxt
*ctxt
)
2079 unsigned short sel
, old_sel
;
2080 struct desc_struct old_desc
, new_desc
;
2081 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2082 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
2084 /* Assignment of RIP may only fail in 64-bit mode */
2085 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2086 ops
->get_segment(ctxt
, &old_sel
, &old_desc
, NULL
,
2089 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2091 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
2092 X86_TRANSFER_CALL_JMP
,
2094 if (rc
!= X86EMUL_CONTINUE
)
2097 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
2098 if (rc
!= X86EMUL_CONTINUE
) {
2099 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2100 /* assigning eip failed; restore the old cs */
2101 ops
->set_segment(ctxt
, old_sel
, &old_desc
, 0, VCPU_SREG_CS
);
2107 static int em_jmp_abs(struct x86_emulate_ctxt
*ctxt
)
2109 return assign_eip_near(ctxt
, ctxt
->src
.val
);
2112 static int em_call_near_abs(struct x86_emulate_ctxt
*ctxt
)
2117 old_eip
= ctxt
->_eip
;
2118 rc
= assign_eip_near(ctxt
, ctxt
->src
.val
);
2119 if (rc
!= X86EMUL_CONTINUE
)
2121 ctxt
->src
.val
= old_eip
;
2126 static int em_cmpxchg8b(struct x86_emulate_ctxt
*ctxt
)
2128 u64 old
= ctxt
->dst
.orig_val64
;
2130 if (ctxt
->dst
.bytes
== 16)
2131 return X86EMUL_UNHANDLEABLE
;
2133 if (((u32
) (old
>> 0) != (u32
) reg_read(ctxt
, VCPU_REGS_RAX
)) ||
2134 ((u32
) (old
>> 32) != (u32
) reg_read(ctxt
, VCPU_REGS_RDX
))) {
2135 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
) (old
>> 0);
2136 *reg_write(ctxt
, VCPU_REGS_RDX
) = (u32
) (old
>> 32);
2137 ctxt
->eflags
&= ~EFLG_ZF
;
2139 ctxt
->dst
.val64
= ((u64
)reg_read(ctxt
, VCPU_REGS_RCX
) << 32) |
2140 (u32
) reg_read(ctxt
, VCPU_REGS_RBX
);
2142 ctxt
->eflags
|= EFLG_ZF
;
2144 return X86EMUL_CONTINUE
;
2147 static int em_ret(struct x86_emulate_ctxt
*ctxt
)
2152 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2153 if (rc
!= X86EMUL_CONTINUE
)
2156 return assign_eip_near(ctxt
, eip
);
2159 static int em_ret_far(struct x86_emulate_ctxt
*ctxt
)
2162 unsigned long eip
, cs
;
2164 int cpl
= ctxt
->ops
->cpl(ctxt
);
2165 struct desc_struct old_desc
, new_desc
;
2166 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2168 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2169 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
,
2172 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2173 if (rc
!= X86EMUL_CONTINUE
)
2175 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2176 if (rc
!= X86EMUL_CONTINUE
)
2178 /* Outer-privilege level return is not implemented */
2179 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
&& (cs
& 3) > cpl
)
2180 return X86EMUL_UNHANDLEABLE
;
2181 rc
= __load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
, cpl
,
2184 if (rc
!= X86EMUL_CONTINUE
)
2186 rc
= assign_eip_far(ctxt
, eip
, &new_desc
);
2187 if (rc
!= X86EMUL_CONTINUE
) {
2188 WARN_ON(ctxt
->mode
!= X86EMUL_MODE_PROT64
);
2189 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
2194 static int em_ret_far_imm(struct x86_emulate_ctxt
*ctxt
)
2198 rc
= em_ret_far(ctxt
);
2199 if (rc
!= X86EMUL_CONTINUE
)
2201 rsp_increment(ctxt
, ctxt
->src
.val
);
2202 return X86EMUL_CONTINUE
;
2205 static int em_cmpxchg(struct x86_emulate_ctxt
*ctxt
)
2207 /* Save real source value, then compare EAX against destination. */
2208 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
2209 ctxt
->dst
.val
= reg_read(ctxt
, VCPU_REGS_RAX
);
2210 ctxt
->src
.orig_val
= ctxt
->src
.val
;
2211 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2212 fastop(ctxt
, em_cmp
);
2214 if (ctxt
->eflags
& EFLG_ZF
) {
2215 /* Success: write back to memory; no update of EAX */
2216 ctxt
->src
.type
= OP_NONE
;
2217 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
2219 /* Failure: write the value we saw to EAX. */
2220 ctxt
->src
.type
= OP_REG
;
2221 ctxt
->src
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
2222 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2223 /* Create write-cycle to dest by writing the same value */
2224 ctxt
->dst
.val
= ctxt
->dst
.orig_val
;
2226 return X86EMUL_CONTINUE
;
2229 static int em_lseg(struct x86_emulate_ctxt
*ctxt
)
2231 int seg
= ctxt
->src2
.val
;
2235 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2237 rc
= load_segment_descriptor(ctxt
, sel
, seg
);
2238 if (rc
!= X86EMUL_CONTINUE
)
2241 ctxt
->dst
.val
= ctxt
->src
.val
;
2246 setup_syscalls_segments(struct x86_emulate_ctxt
*ctxt
,
2247 struct desc_struct
*cs
, struct desc_struct
*ss
)
2249 cs
->l
= 0; /* will be adjusted later */
2250 set_desc_base(cs
, 0); /* flat segment */
2251 cs
->g
= 1; /* 4kb granularity */
2252 set_desc_limit(cs
, 0xfffff); /* 4GB limit */
2253 cs
->type
= 0x0b; /* Read, Execute, Accessed */
2255 cs
->dpl
= 0; /* will be adjusted later */
2260 set_desc_base(ss
, 0); /* flat segment */
2261 set_desc_limit(ss
, 0xfffff); /* 4GB limit */
2262 ss
->g
= 1; /* 4kb granularity */
2264 ss
->type
= 0x03; /* Read/Write, Accessed */
2265 ss
->d
= 1; /* 32bit stack segment */
2272 static bool vendor_intel(struct x86_emulate_ctxt
*ctxt
)
2274 u32 eax
, ebx
, ecx
, edx
;
2277 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2278 return ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2279 && ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2280 && edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
;
2283 static bool em_syscall_is_enabled(struct x86_emulate_ctxt
*ctxt
)
2285 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2286 u32 eax
, ebx
, ecx
, edx
;
2289 * syscall should always be enabled in longmode - so only become
2290 * vendor specific (cpuid) if other modes are active...
2292 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2297 ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2299 * Intel ("GenuineIntel")
2300 * remark: Intel CPUs only support "syscall" in 64bit
2301 * longmode. Also an 64bit guest with a
2302 * 32bit compat-app running will #UD !! While this
2303 * behaviour can be fixed (by emulating) into AMD
2304 * response - CPUs of AMD can't behave like Intel.
2306 if (ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
&&
2307 ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
&&
2308 edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
)
2311 /* AMD ("AuthenticAMD") */
2312 if (ebx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx
&&
2313 ecx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx
&&
2314 edx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_edx
)
2317 /* AMD ("AMDisbetter!") */
2318 if (ebx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx
&&
2319 ecx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx
&&
2320 edx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_edx
)
2323 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2327 static int em_syscall(struct x86_emulate_ctxt
*ctxt
)
2329 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2330 struct desc_struct cs
, ss
;
2335 /* syscall is not available in real mode */
2336 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2337 ctxt
->mode
== X86EMUL_MODE_VM86
)
2338 return emulate_ud(ctxt
);
2340 if (!(em_syscall_is_enabled(ctxt
)))
2341 return emulate_ud(ctxt
);
2343 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2344 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2346 if (!(efer
& EFER_SCE
))
2347 return emulate_ud(ctxt
);
2349 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2351 cs_sel
= (u16
)(msr_data
& 0xfffc);
2352 ss_sel
= (u16
)(msr_data
+ 8);
2354 if (efer
& EFER_LMA
) {
2358 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2359 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2361 *reg_write(ctxt
, VCPU_REGS_RCX
) = ctxt
->_eip
;
2362 if (efer
& EFER_LMA
) {
2363 #ifdef CONFIG_X86_64
2364 *reg_write(ctxt
, VCPU_REGS_R11
) = ctxt
->eflags
;
2367 ctxt
->mode
== X86EMUL_MODE_PROT64
?
2368 MSR_LSTAR
: MSR_CSTAR
, &msr_data
);
2369 ctxt
->_eip
= msr_data
;
2371 ops
->get_msr(ctxt
, MSR_SYSCALL_MASK
, &msr_data
);
2372 ctxt
->eflags
&= ~msr_data
;
2373 ctxt
->eflags
|= EFLG_RESERVED_ONE_MASK
;
2377 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2378 ctxt
->_eip
= (u32
)msr_data
;
2380 ctxt
->eflags
&= ~(EFLG_VM
| EFLG_IF
);
2383 return X86EMUL_CONTINUE
;
2386 static int em_sysenter(struct x86_emulate_ctxt
*ctxt
)
2388 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2389 struct desc_struct cs
, ss
;
2394 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2395 /* inject #GP if in real mode */
2396 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2397 return emulate_gp(ctxt
, 0);
2400 * Not recognized on AMD in compat mode (but is recognized in legacy
2403 if ((ctxt
->mode
== X86EMUL_MODE_PROT32
) && (efer
& EFER_LMA
)
2404 && !vendor_intel(ctxt
))
2405 return emulate_ud(ctxt
);
2407 /* sysenter/sysexit have not been tested in 64bit mode. */
2408 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2409 return X86EMUL_UNHANDLEABLE
;
2411 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2413 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2414 switch (ctxt
->mode
) {
2415 case X86EMUL_MODE_PROT32
:
2416 if ((msr_data
& 0xfffc) == 0x0)
2417 return emulate_gp(ctxt
, 0);
2419 case X86EMUL_MODE_PROT64
:
2420 if (msr_data
== 0x0)
2421 return emulate_gp(ctxt
, 0);
2427 ctxt
->eflags
&= ~(EFLG_VM
| EFLG_IF
);
2428 cs_sel
= (u16
)msr_data
;
2429 cs_sel
&= ~SELECTOR_RPL_MASK
;
2430 ss_sel
= cs_sel
+ 8;
2431 ss_sel
&= ~SELECTOR_RPL_MASK
;
2432 if (ctxt
->mode
== X86EMUL_MODE_PROT64
|| (efer
& EFER_LMA
)) {
2437 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2438 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2440 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_EIP
, &msr_data
);
2441 ctxt
->_eip
= msr_data
;
2443 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_ESP
, &msr_data
);
2444 *reg_write(ctxt
, VCPU_REGS_RSP
) = msr_data
;
2446 return X86EMUL_CONTINUE
;
2449 static int em_sysexit(struct x86_emulate_ctxt
*ctxt
)
2451 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2452 struct desc_struct cs
, ss
;
2453 u64 msr_data
, rcx
, rdx
;
2455 u16 cs_sel
= 0, ss_sel
= 0;
2457 /* inject #GP if in real mode or Virtual 8086 mode */
2458 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2459 ctxt
->mode
== X86EMUL_MODE_VM86
)
2460 return emulate_gp(ctxt
, 0);
2462 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2464 if ((ctxt
->rex_prefix
& 0x8) != 0x0)
2465 usermode
= X86EMUL_MODE_PROT64
;
2467 usermode
= X86EMUL_MODE_PROT32
;
2469 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2470 rdx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2474 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2476 case X86EMUL_MODE_PROT32
:
2477 cs_sel
= (u16
)(msr_data
+ 16);
2478 if ((msr_data
& 0xfffc) == 0x0)
2479 return emulate_gp(ctxt
, 0);
2480 ss_sel
= (u16
)(msr_data
+ 24);
2484 case X86EMUL_MODE_PROT64
:
2485 cs_sel
= (u16
)(msr_data
+ 32);
2486 if (msr_data
== 0x0)
2487 return emulate_gp(ctxt
, 0);
2488 ss_sel
= cs_sel
+ 8;
2491 if (is_noncanonical_address(rcx
) ||
2492 is_noncanonical_address(rdx
))
2493 return emulate_gp(ctxt
, 0);
2496 cs_sel
|= SELECTOR_RPL_MASK
;
2497 ss_sel
|= SELECTOR_RPL_MASK
;
2499 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2500 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2503 *reg_write(ctxt
, VCPU_REGS_RSP
) = rcx
;
2505 return X86EMUL_CONTINUE
;
2508 static bool emulator_bad_iopl(struct x86_emulate_ctxt
*ctxt
)
2511 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2513 if (ctxt
->mode
== X86EMUL_MODE_VM86
)
2515 iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> IOPL_SHIFT
;
2516 return ctxt
->ops
->cpl(ctxt
) > iopl
;
2519 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt
*ctxt
,
2522 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2523 struct desc_struct tr_seg
;
2526 u16 tr
, io_bitmap_ptr
, perm
, bit_idx
= port
& 0x7;
2527 unsigned mask
= (1 << len
) - 1;
2530 ops
->get_segment(ctxt
, &tr
, &tr_seg
, &base3
, VCPU_SREG_TR
);
2533 if (desc_limit_scaled(&tr_seg
) < 103)
2535 base
= get_desc_base(&tr_seg
);
2536 #ifdef CONFIG_X86_64
2537 base
|= ((u64
)base3
) << 32;
2539 r
= ops
->read_std(ctxt
, base
+ 102, &io_bitmap_ptr
, 2, NULL
);
2540 if (r
!= X86EMUL_CONTINUE
)
2542 if (io_bitmap_ptr
+ port
/8 > desc_limit_scaled(&tr_seg
))
2544 r
= ops
->read_std(ctxt
, base
+ io_bitmap_ptr
+ port
/8, &perm
, 2, NULL
);
2545 if (r
!= X86EMUL_CONTINUE
)
2547 if ((perm
>> bit_idx
) & mask
)
2552 static bool emulator_io_permited(struct x86_emulate_ctxt
*ctxt
,
2558 if (emulator_bad_iopl(ctxt
))
2559 if (!emulator_io_port_access_allowed(ctxt
, port
, len
))
2562 ctxt
->perm_ok
= true;
2567 static void save_state_to_tss16(struct x86_emulate_ctxt
*ctxt
,
2568 struct tss_segment_16
*tss
)
2570 tss
->ip
= ctxt
->_eip
;
2571 tss
->flag
= ctxt
->eflags
;
2572 tss
->ax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2573 tss
->cx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2574 tss
->dx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2575 tss
->bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2576 tss
->sp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2577 tss
->bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2578 tss
->si
= reg_read(ctxt
, VCPU_REGS_RSI
);
2579 tss
->di
= reg_read(ctxt
, VCPU_REGS_RDI
);
2581 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2582 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2583 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2584 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2585 tss
->ldt
= get_segment_selector(ctxt
, VCPU_SREG_LDTR
);
2588 static int load_state_from_tss16(struct x86_emulate_ctxt
*ctxt
,
2589 struct tss_segment_16
*tss
)
2594 ctxt
->_eip
= tss
->ip
;
2595 ctxt
->eflags
= tss
->flag
| 2;
2596 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->ax
;
2597 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->cx
;
2598 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->dx
;
2599 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->bx
;
2600 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->sp
;
2601 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->bp
;
2602 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->si
;
2603 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->di
;
2606 * SDM says that segment selectors are loaded before segment
2609 set_segment_selector(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
);
2610 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
2611 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
2612 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
2613 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
2618 * Now load segment descriptors. If fault happens at this stage
2619 * it is handled in a context of new task
2621 ret
= __load_segment_descriptor(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
, cpl
,
2622 X86_TRANSFER_TASK_SWITCH
, NULL
);
2623 if (ret
!= X86EMUL_CONTINUE
)
2625 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
2626 X86_TRANSFER_TASK_SWITCH
, NULL
);
2627 if (ret
!= X86EMUL_CONTINUE
)
2629 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
2630 X86_TRANSFER_TASK_SWITCH
, NULL
);
2631 if (ret
!= X86EMUL_CONTINUE
)
2633 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
2634 X86_TRANSFER_TASK_SWITCH
, NULL
);
2635 if (ret
!= X86EMUL_CONTINUE
)
2637 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
2638 X86_TRANSFER_TASK_SWITCH
, NULL
);
2639 if (ret
!= X86EMUL_CONTINUE
)
2642 return X86EMUL_CONTINUE
;
2645 static int task_switch_16(struct x86_emulate_ctxt
*ctxt
,
2646 u16 tss_selector
, u16 old_tss_sel
,
2647 ulong old_tss_base
, struct desc_struct
*new_desc
)
2649 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2650 struct tss_segment_16 tss_seg
;
2652 u32 new_tss_base
= get_desc_base(new_desc
);
2654 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
2656 if (ret
!= X86EMUL_CONTINUE
)
2659 save_state_to_tss16(ctxt
, &tss_seg
);
2661 ret
= ops
->write_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
2663 if (ret
!= X86EMUL_CONTINUE
)
2666 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
2668 if (ret
!= X86EMUL_CONTINUE
)
2671 if (old_tss_sel
!= 0xffff) {
2672 tss_seg
.prev_task_link
= old_tss_sel
;
2674 ret
= ops
->write_std(ctxt
, new_tss_base
,
2675 &tss_seg
.prev_task_link
,
2676 sizeof tss_seg
.prev_task_link
,
2678 if (ret
!= X86EMUL_CONTINUE
)
2682 return load_state_from_tss16(ctxt
, &tss_seg
);
2685 static void save_state_to_tss32(struct x86_emulate_ctxt
*ctxt
,
2686 struct tss_segment_32
*tss
)
2688 /* CR3 and ldt selector are not saved intentionally */
2689 tss
->eip
= ctxt
->_eip
;
2690 tss
->eflags
= ctxt
->eflags
;
2691 tss
->eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2692 tss
->ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2693 tss
->edx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2694 tss
->ebx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2695 tss
->esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2696 tss
->ebp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2697 tss
->esi
= reg_read(ctxt
, VCPU_REGS_RSI
);
2698 tss
->edi
= reg_read(ctxt
, VCPU_REGS_RDI
);
2700 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2701 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2702 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2703 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2704 tss
->fs
= get_segment_selector(ctxt
, VCPU_SREG_FS
);
2705 tss
->gs
= get_segment_selector(ctxt
, VCPU_SREG_GS
);
2708 static int load_state_from_tss32(struct x86_emulate_ctxt
*ctxt
,
2709 struct tss_segment_32
*tss
)
2714 if (ctxt
->ops
->set_cr(ctxt
, 3, tss
->cr3
))
2715 return emulate_gp(ctxt
, 0);
2716 ctxt
->_eip
= tss
->eip
;
2717 ctxt
->eflags
= tss
->eflags
| 2;
2719 /* General purpose registers */
2720 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->eax
;
2721 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->ecx
;
2722 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->edx
;
2723 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->ebx
;
2724 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->esp
;
2725 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->ebp
;
2726 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->esi
;
2727 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->edi
;
2730 * SDM says that segment selectors are loaded before segment
2731 * descriptors. This is important because CPL checks will
2734 set_segment_selector(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
);
2735 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
2736 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
2737 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
2738 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
2739 set_segment_selector(ctxt
, tss
->fs
, VCPU_SREG_FS
);
2740 set_segment_selector(ctxt
, tss
->gs
, VCPU_SREG_GS
);
2743 * If we're switching between Protected Mode and VM86, we need to make
2744 * sure to update the mode before loading the segment descriptors so
2745 * that the selectors are interpreted correctly.
2747 if (ctxt
->eflags
& X86_EFLAGS_VM
) {
2748 ctxt
->mode
= X86EMUL_MODE_VM86
;
2751 ctxt
->mode
= X86EMUL_MODE_PROT32
;
2756 * Now load segment descriptors. If fault happenes at this stage
2757 * it is handled in a context of new task
2759 ret
= __load_segment_descriptor(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
,
2760 cpl
, X86_TRANSFER_TASK_SWITCH
, NULL
);
2761 if (ret
!= X86EMUL_CONTINUE
)
2763 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
2764 X86_TRANSFER_TASK_SWITCH
, NULL
);
2765 if (ret
!= X86EMUL_CONTINUE
)
2767 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
2768 X86_TRANSFER_TASK_SWITCH
, NULL
);
2769 if (ret
!= X86EMUL_CONTINUE
)
2771 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
2772 X86_TRANSFER_TASK_SWITCH
, NULL
);
2773 if (ret
!= X86EMUL_CONTINUE
)
2775 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
2776 X86_TRANSFER_TASK_SWITCH
, NULL
);
2777 if (ret
!= X86EMUL_CONTINUE
)
2779 ret
= __load_segment_descriptor(ctxt
, tss
->fs
, VCPU_SREG_FS
, cpl
,
2780 X86_TRANSFER_TASK_SWITCH
, NULL
);
2781 if (ret
!= X86EMUL_CONTINUE
)
2783 ret
= __load_segment_descriptor(ctxt
, tss
->gs
, VCPU_SREG_GS
, cpl
,
2784 X86_TRANSFER_TASK_SWITCH
, NULL
);
2785 if (ret
!= X86EMUL_CONTINUE
)
2788 return X86EMUL_CONTINUE
;
2791 static int task_switch_32(struct x86_emulate_ctxt
*ctxt
,
2792 u16 tss_selector
, u16 old_tss_sel
,
2793 ulong old_tss_base
, struct desc_struct
*new_desc
)
2795 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2796 struct tss_segment_32 tss_seg
;
2798 u32 new_tss_base
= get_desc_base(new_desc
);
2799 u32 eip_offset
= offsetof(struct tss_segment_32
, eip
);
2800 u32 ldt_sel_offset
= offsetof(struct tss_segment_32
, ldt_selector
);
2802 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
2804 if (ret
!= X86EMUL_CONTINUE
)
2807 save_state_to_tss32(ctxt
, &tss_seg
);
2809 /* Only GP registers and segment selectors are saved */
2810 ret
= ops
->write_std(ctxt
, old_tss_base
+ eip_offset
, &tss_seg
.eip
,
2811 ldt_sel_offset
- eip_offset
, &ctxt
->exception
);
2812 if (ret
!= X86EMUL_CONTINUE
)
2815 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
2817 if (ret
!= X86EMUL_CONTINUE
)
2820 if (old_tss_sel
!= 0xffff) {
2821 tss_seg
.prev_task_link
= old_tss_sel
;
2823 ret
= ops
->write_std(ctxt
, new_tss_base
,
2824 &tss_seg
.prev_task_link
,
2825 sizeof tss_seg
.prev_task_link
,
2827 if (ret
!= X86EMUL_CONTINUE
)
2831 return load_state_from_tss32(ctxt
, &tss_seg
);
2834 static int emulator_do_task_switch(struct x86_emulate_ctxt
*ctxt
,
2835 u16 tss_selector
, int idt_index
, int reason
,
2836 bool has_error_code
, u32 error_code
)
2838 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2839 struct desc_struct curr_tss_desc
, next_tss_desc
;
2841 u16 old_tss_sel
= get_segment_selector(ctxt
, VCPU_SREG_TR
);
2842 ulong old_tss_base
=
2843 ops
->get_cached_segment_base(ctxt
, VCPU_SREG_TR
);
2847 /* FIXME: old_tss_base == ~0 ? */
2849 ret
= read_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
, &desc_addr
);
2850 if (ret
!= X86EMUL_CONTINUE
)
2852 ret
= read_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
, &desc_addr
);
2853 if (ret
!= X86EMUL_CONTINUE
)
2856 /* FIXME: check that next_tss_desc is tss */
2859 * Check privileges. The three cases are task switch caused by...
2861 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2862 * 2. Exception/IRQ/iret: No check is performed
2863 * 3. jmp/call to TSS/task-gate: No check is performed since the
2864 * hardware checks it before exiting.
2866 if (reason
== TASK_SWITCH_GATE
) {
2867 if (idt_index
!= -1) {
2868 /* Software interrupts */
2869 struct desc_struct task_gate_desc
;
2872 ret
= read_interrupt_descriptor(ctxt
, idt_index
,
2874 if (ret
!= X86EMUL_CONTINUE
)
2877 dpl
= task_gate_desc
.dpl
;
2878 if ((tss_selector
& 3) > dpl
|| ops
->cpl(ctxt
) > dpl
)
2879 return emulate_gp(ctxt
, (idt_index
<< 3) | 0x2);
2883 desc_limit
= desc_limit_scaled(&next_tss_desc
);
2884 if (!next_tss_desc
.p
||
2885 ((desc_limit
< 0x67 && (next_tss_desc
.type
& 8)) ||
2886 desc_limit
< 0x2b)) {
2887 return emulate_ts(ctxt
, tss_selector
& 0xfffc);
2890 if (reason
== TASK_SWITCH_IRET
|| reason
== TASK_SWITCH_JMP
) {
2891 curr_tss_desc
.type
&= ~(1 << 1); /* clear busy flag */
2892 write_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
);
2895 if (reason
== TASK_SWITCH_IRET
)
2896 ctxt
->eflags
= ctxt
->eflags
& ~X86_EFLAGS_NT
;
2898 /* set back link to prev task only if NT bit is set in eflags
2899 note that old_tss_sel is not used after this point */
2900 if (reason
!= TASK_SWITCH_CALL
&& reason
!= TASK_SWITCH_GATE
)
2901 old_tss_sel
= 0xffff;
2903 if (next_tss_desc
.type
& 8)
2904 ret
= task_switch_32(ctxt
, tss_selector
, old_tss_sel
,
2905 old_tss_base
, &next_tss_desc
);
2907 ret
= task_switch_16(ctxt
, tss_selector
, old_tss_sel
,
2908 old_tss_base
, &next_tss_desc
);
2909 if (ret
!= X86EMUL_CONTINUE
)
2912 if (reason
== TASK_SWITCH_CALL
|| reason
== TASK_SWITCH_GATE
)
2913 ctxt
->eflags
= ctxt
->eflags
| X86_EFLAGS_NT
;
2915 if (reason
!= TASK_SWITCH_IRET
) {
2916 next_tss_desc
.type
|= (1 << 1); /* set busy flag */
2917 write_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
);
2920 ops
->set_cr(ctxt
, 0, ops
->get_cr(ctxt
, 0) | X86_CR0_TS
);
2921 ops
->set_segment(ctxt
, tss_selector
, &next_tss_desc
, 0, VCPU_SREG_TR
);
2923 if (has_error_code
) {
2924 ctxt
->op_bytes
= ctxt
->ad_bytes
= (next_tss_desc
.type
& 8) ? 4 : 2;
2925 ctxt
->lock_prefix
= 0;
2926 ctxt
->src
.val
= (unsigned long) error_code
;
2927 ret
= em_push(ctxt
);
2933 int emulator_task_switch(struct x86_emulate_ctxt
*ctxt
,
2934 u16 tss_selector
, int idt_index
, int reason
,
2935 bool has_error_code
, u32 error_code
)
2939 invalidate_registers(ctxt
);
2940 ctxt
->_eip
= ctxt
->eip
;
2941 ctxt
->dst
.type
= OP_NONE
;
2943 rc
= emulator_do_task_switch(ctxt
, tss_selector
, idt_index
, reason
,
2944 has_error_code
, error_code
);
2946 if (rc
== X86EMUL_CONTINUE
) {
2947 ctxt
->eip
= ctxt
->_eip
;
2948 writeback_registers(ctxt
);
2951 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
2954 static void string_addr_inc(struct x86_emulate_ctxt
*ctxt
, int reg
,
2957 int df
= (ctxt
->eflags
& EFLG_DF
) ? -op
->count
: op
->count
;
2959 register_address_increment(ctxt
, reg
, df
* op
->bytes
);
2960 op
->addr
.mem
.ea
= register_address(ctxt
, reg
);
2963 static int em_das(struct x86_emulate_ctxt
*ctxt
)
2966 bool af
, cf
, old_cf
;
2968 cf
= ctxt
->eflags
& X86_EFLAGS_CF
;
2974 af
= ctxt
->eflags
& X86_EFLAGS_AF
;
2975 if ((al
& 0x0f) > 9 || af
) {
2977 cf
= old_cf
| (al
>= 250);
2982 if (old_al
> 0x99 || old_cf
) {
2988 /* Set PF, ZF, SF */
2989 ctxt
->src
.type
= OP_IMM
;
2991 ctxt
->src
.bytes
= 1;
2992 fastop(ctxt
, em_or
);
2993 ctxt
->eflags
&= ~(X86_EFLAGS_AF
| X86_EFLAGS_CF
);
2995 ctxt
->eflags
|= X86_EFLAGS_CF
;
2997 ctxt
->eflags
|= X86_EFLAGS_AF
;
2998 return X86EMUL_CONTINUE
;
3001 static int em_aam(struct x86_emulate_ctxt
*ctxt
)
3005 if (ctxt
->src
.val
== 0)
3006 return emulate_de(ctxt
);
3008 al
= ctxt
->dst
.val
& 0xff;
3009 ah
= al
/ ctxt
->src
.val
;
3010 al
%= ctxt
->src
.val
;
3012 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
| (ah
<< 8);
3014 /* Set PF, ZF, SF */
3015 ctxt
->src
.type
= OP_IMM
;
3017 ctxt
->src
.bytes
= 1;
3018 fastop(ctxt
, em_or
);
3020 return X86EMUL_CONTINUE
;
3023 static int em_aad(struct x86_emulate_ctxt
*ctxt
)
3025 u8 al
= ctxt
->dst
.val
& 0xff;
3026 u8 ah
= (ctxt
->dst
.val
>> 8) & 0xff;
3028 al
= (al
+ (ah
* ctxt
->src
.val
)) & 0xff;
3030 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
;
3032 /* Set PF, ZF, SF */
3033 ctxt
->src
.type
= OP_IMM
;
3035 ctxt
->src
.bytes
= 1;
3036 fastop(ctxt
, em_or
);
3038 return X86EMUL_CONTINUE
;
3041 static int em_call(struct x86_emulate_ctxt
*ctxt
)
3044 long rel
= ctxt
->src
.val
;
3046 ctxt
->src
.val
= (unsigned long)ctxt
->_eip
;
3047 rc
= jmp_rel(ctxt
, rel
);
3048 if (rc
!= X86EMUL_CONTINUE
)
3050 return em_push(ctxt
);
3053 static int em_call_far(struct x86_emulate_ctxt
*ctxt
)
3058 struct desc_struct old_desc
, new_desc
;
3059 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3060 int cpl
= ctxt
->ops
->cpl(ctxt
);
3062 old_eip
= ctxt
->_eip
;
3063 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
, VCPU_SREG_CS
);
3065 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
3066 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
3067 X86_TRANSFER_CALL_JMP
, &new_desc
);
3068 if (rc
!= X86EMUL_CONTINUE
)
3071 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
3072 if (rc
!= X86EMUL_CONTINUE
)
3075 ctxt
->src
.val
= old_cs
;
3077 if (rc
!= X86EMUL_CONTINUE
)
3080 ctxt
->src
.val
= old_eip
;
3082 /* If we failed, we tainted the memory, but the very least we should
3084 if (rc
!= X86EMUL_CONTINUE
)
3088 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
3093 static int em_ret_near_imm(struct x86_emulate_ctxt
*ctxt
)
3098 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
3099 if (rc
!= X86EMUL_CONTINUE
)
3101 rc
= assign_eip_near(ctxt
, eip
);
3102 if (rc
!= X86EMUL_CONTINUE
)
3104 rsp_increment(ctxt
, ctxt
->src
.val
);
3105 return X86EMUL_CONTINUE
;
3108 static int em_xchg(struct x86_emulate_ctxt
*ctxt
)
3110 /* Write back the register source. */
3111 ctxt
->src
.val
= ctxt
->dst
.val
;
3112 write_register_operand(&ctxt
->src
);
3114 /* Write back the memory destination with implicit LOCK prefix. */
3115 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
3116 ctxt
->lock_prefix
= 1;
3117 return X86EMUL_CONTINUE
;
3120 static int em_imul_3op(struct x86_emulate_ctxt
*ctxt
)
3122 ctxt
->dst
.val
= ctxt
->src2
.val
;
3123 return fastop(ctxt
, em_imul
);
3126 static int em_cwd(struct x86_emulate_ctxt
*ctxt
)
3128 ctxt
->dst
.type
= OP_REG
;
3129 ctxt
->dst
.bytes
= ctxt
->src
.bytes
;
3130 ctxt
->dst
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
3131 ctxt
->dst
.val
= ~((ctxt
->src
.val
>> (ctxt
->src
.bytes
* 8 - 1)) - 1);
3133 return X86EMUL_CONTINUE
;
3136 static int em_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3140 ctxt
->ops
->get_msr(ctxt
, MSR_IA32_TSC
, &tsc
);
3141 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)tsc
;
3142 *reg_write(ctxt
, VCPU_REGS_RDX
) = tsc
>> 32;
3143 return X86EMUL_CONTINUE
;
3146 static int em_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3150 if (ctxt
->ops
->read_pmc(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &pmc
))
3151 return emulate_gp(ctxt
, 0);
3152 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)pmc
;
3153 *reg_write(ctxt
, VCPU_REGS_RDX
) = pmc
>> 32;
3154 return X86EMUL_CONTINUE
;
3157 static int em_mov(struct x86_emulate_ctxt
*ctxt
)
3159 memcpy(ctxt
->dst
.valptr
, ctxt
->src
.valptr
, sizeof(ctxt
->src
.valptr
));
3160 return X86EMUL_CONTINUE
;
3163 #define FFL(x) bit(X86_FEATURE_##x)
3165 static int em_movbe(struct x86_emulate_ctxt
*ctxt
)
3167 u32 ebx
, ecx
, edx
, eax
= 1;
3171 * Check MOVBE is set in the guest-visible CPUID leaf.
3173 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3174 if (!(ecx
& FFL(MOVBE
)))
3175 return emulate_ud(ctxt
);
3177 switch (ctxt
->op_bytes
) {
3180 * From MOVBE definition: "...When the operand size is 16 bits,
3181 * the upper word of the destination register remains unchanged
3184 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3185 * rules so we have to do the operation almost per hand.
3187 tmp
= (u16
)ctxt
->src
.val
;
3188 ctxt
->dst
.val
&= ~0xffffUL
;
3189 ctxt
->dst
.val
|= (unsigned long)swab16(tmp
);
3192 ctxt
->dst
.val
= swab32((u32
)ctxt
->src
.val
);
3195 ctxt
->dst
.val
= swab64(ctxt
->src
.val
);
3200 return X86EMUL_CONTINUE
;
3203 static int em_cr_write(struct x86_emulate_ctxt
*ctxt
)
3205 if (ctxt
->ops
->set_cr(ctxt
, ctxt
->modrm_reg
, ctxt
->src
.val
))
3206 return emulate_gp(ctxt
, 0);
3208 /* Disable writeback. */
3209 ctxt
->dst
.type
= OP_NONE
;
3210 return X86EMUL_CONTINUE
;
3213 static int em_dr_write(struct x86_emulate_ctxt
*ctxt
)
3217 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3218 val
= ctxt
->src
.val
& ~0ULL;
3220 val
= ctxt
->src
.val
& ~0U;
3222 /* #UD condition is already handled. */
3223 if (ctxt
->ops
->set_dr(ctxt
, ctxt
->modrm_reg
, val
) < 0)
3224 return emulate_gp(ctxt
, 0);
3226 /* Disable writeback. */
3227 ctxt
->dst
.type
= OP_NONE
;
3228 return X86EMUL_CONTINUE
;
3231 static int em_wrmsr(struct x86_emulate_ctxt
*ctxt
)
3235 msr_data
= (u32
)reg_read(ctxt
, VCPU_REGS_RAX
)
3236 | ((u64
)reg_read(ctxt
, VCPU_REGS_RDX
) << 32);
3237 if (ctxt
->ops
->set_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), msr_data
))
3238 return emulate_gp(ctxt
, 0);
3240 return X86EMUL_CONTINUE
;
3243 static int em_rdmsr(struct x86_emulate_ctxt
*ctxt
)
3247 if (ctxt
->ops
->get_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &msr_data
))
3248 return emulate_gp(ctxt
, 0);
3250 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)msr_data
;
3251 *reg_write(ctxt
, VCPU_REGS_RDX
) = msr_data
>> 32;
3252 return X86EMUL_CONTINUE
;
3255 static int em_mov_rm_sreg(struct x86_emulate_ctxt
*ctxt
)
3257 if (ctxt
->modrm_reg
> VCPU_SREG_GS
)
3258 return emulate_ud(ctxt
);
3260 ctxt
->dst
.val
= get_segment_selector(ctxt
, ctxt
->modrm_reg
);
3261 if (ctxt
->dst
.bytes
== 4 && ctxt
->dst
.type
== OP_MEM
)
3262 ctxt
->dst
.bytes
= 2;
3263 return X86EMUL_CONTINUE
;
3266 static int em_mov_sreg_rm(struct x86_emulate_ctxt
*ctxt
)
3268 u16 sel
= ctxt
->src
.val
;
3270 if (ctxt
->modrm_reg
== VCPU_SREG_CS
|| ctxt
->modrm_reg
> VCPU_SREG_GS
)
3271 return emulate_ud(ctxt
);
3273 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
3274 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
3276 /* Disable writeback. */
3277 ctxt
->dst
.type
= OP_NONE
;
3278 return load_segment_descriptor(ctxt
, sel
, ctxt
->modrm_reg
);
3281 static int em_lldt(struct x86_emulate_ctxt
*ctxt
)
3283 u16 sel
= ctxt
->src
.val
;
3285 /* Disable writeback. */
3286 ctxt
->dst
.type
= OP_NONE
;
3287 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_LDTR
);
3290 static int em_ltr(struct x86_emulate_ctxt
*ctxt
)
3292 u16 sel
= ctxt
->src
.val
;
3294 /* Disable writeback. */
3295 ctxt
->dst
.type
= OP_NONE
;
3296 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_TR
);
3299 static int em_invlpg(struct x86_emulate_ctxt
*ctxt
)
3304 rc
= linearize(ctxt
, ctxt
->src
.addr
.mem
, 1, false, &linear
);
3305 if (rc
== X86EMUL_CONTINUE
)
3306 ctxt
->ops
->invlpg(ctxt
, linear
);
3307 /* Disable writeback. */
3308 ctxt
->dst
.type
= OP_NONE
;
3309 return X86EMUL_CONTINUE
;
3312 static int em_clts(struct x86_emulate_ctxt
*ctxt
)
3316 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
3318 ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
3319 return X86EMUL_CONTINUE
;
3322 static int em_vmcall(struct x86_emulate_ctxt
*ctxt
)
3324 int rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3326 if (rc
!= X86EMUL_CONTINUE
)
3329 /* Let the processor re-execute the fixed hypercall */
3330 ctxt
->_eip
= ctxt
->eip
;
3331 /* Disable writeback. */
3332 ctxt
->dst
.type
= OP_NONE
;
3333 return X86EMUL_CONTINUE
;
3336 static int emulate_store_desc_ptr(struct x86_emulate_ctxt
*ctxt
,
3337 void (*get
)(struct x86_emulate_ctxt
*ctxt
,
3338 struct desc_ptr
*ptr
))
3340 struct desc_ptr desc_ptr
;
3342 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3344 get(ctxt
, &desc_ptr
);
3345 if (ctxt
->op_bytes
== 2) {
3347 desc_ptr
.address
&= 0x00ffffff;
3349 /* Disable writeback. */
3350 ctxt
->dst
.type
= OP_NONE
;
3351 return segmented_write(ctxt
, ctxt
->dst
.addr
.mem
,
3352 &desc_ptr
, 2 + ctxt
->op_bytes
);
3355 static int em_sgdt(struct x86_emulate_ctxt
*ctxt
)
3357 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_gdt
);
3360 static int em_sidt(struct x86_emulate_ctxt
*ctxt
)
3362 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_idt
);
3365 static int em_lgdt_lidt(struct x86_emulate_ctxt
*ctxt
, bool lgdt
)
3367 struct desc_ptr desc_ptr
;
3370 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3372 rc
= read_descriptor(ctxt
, ctxt
->src
.addr
.mem
,
3373 &desc_ptr
.size
, &desc_ptr
.address
,
3375 if (rc
!= X86EMUL_CONTINUE
)
3377 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&&
3378 is_noncanonical_address(desc_ptr
.address
))
3379 return emulate_gp(ctxt
, 0);
3381 ctxt
->ops
->set_gdt(ctxt
, &desc_ptr
);
3383 ctxt
->ops
->set_idt(ctxt
, &desc_ptr
);
3384 /* Disable writeback. */
3385 ctxt
->dst
.type
= OP_NONE
;
3386 return X86EMUL_CONTINUE
;
3389 static int em_lgdt(struct x86_emulate_ctxt
*ctxt
)
3391 return em_lgdt_lidt(ctxt
, true);
3394 static int em_vmmcall(struct x86_emulate_ctxt
*ctxt
)
3398 rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3400 /* Disable writeback. */
3401 ctxt
->dst
.type
= OP_NONE
;
3405 static int em_lidt(struct x86_emulate_ctxt
*ctxt
)
3407 return em_lgdt_lidt(ctxt
, false);
3410 static int em_smsw(struct x86_emulate_ctxt
*ctxt
)
3412 if (ctxt
->dst
.type
== OP_MEM
)
3413 ctxt
->dst
.bytes
= 2;
3414 ctxt
->dst
.val
= ctxt
->ops
->get_cr(ctxt
, 0);
3415 return X86EMUL_CONTINUE
;
3418 static int em_lmsw(struct x86_emulate_ctxt
*ctxt
)
3420 ctxt
->ops
->set_cr(ctxt
, 0, (ctxt
->ops
->get_cr(ctxt
, 0) & ~0x0eul
)
3421 | (ctxt
->src
.val
& 0x0f));
3422 ctxt
->dst
.type
= OP_NONE
;
3423 return X86EMUL_CONTINUE
;
3426 static int em_loop(struct x86_emulate_ctxt
*ctxt
)
3428 int rc
= X86EMUL_CONTINUE
;
3430 register_address_increment(ctxt
, VCPU_REGS_RCX
, -1);
3431 if ((address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) != 0) &&
3432 (ctxt
->b
== 0xe2 || test_cc(ctxt
->b
^ 0x5, ctxt
->eflags
)))
3433 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3438 static int em_jcxz(struct x86_emulate_ctxt
*ctxt
)
3440 int rc
= X86EMUL_CONTINUE
;
3442 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0)
3443 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3448 static int em_in(struct x86_emulate_ctxt
*ctxt
)
3450 if (!pio_in_emulated(ctxt
, ctxt
->dst
.bytes
, ctxt
->src
.val
,
3452 return X86EMUL_IO_NEEDED
;
3454 return X86EMUL_CONTINUE
;
3457 static int em_out(struct x86_emulate_ctxt
*ctxt
)
3459 ctxt
->ops
->pio_out_emulated(ctxt
, ctxt
->src
.bytes
, ctxt
->dst
.val
,
3461 /* Disable writeback. */
3462 ctxt
->dst
.type
= OP_NONE
;
3463 return X86EMUL_CONTINUE
;
3466 static int em_cli(struct x86_emulate_ctxt
*ctxt
)
3468 if (emulator_bad_iopl(ctxt
))
3469 return emulate_gp(ctxt
, 0);
3471 ctxt
->eflags
&= ~X86_EFLAGS_IF
;
3472 return X86EMUL_CONTINUE
;
3475 static int em_sti(struct x86_emulate_ctxt
*ctxt
)
3477 if (emulator_bad_iopl(ctxt
))
3478 return emulate_gp(ctxt
, 0);
3480 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_STI
;
3481 ctxt
->eflags
|= X86_EFLAGS_IF
;
3482 return X86EMUL_CONTINUE
;
3485 static int em_cpuid(struct x86_emulate_ctxt
*ctxt
)
3487 u32 eax
, ebx
, ecx
, edx
;
3489 eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3490 ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3491 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3492 *reg_write(ctxt
, VCPU_REGS_RAX
) = eax
;
3493 *reg_write(ctxt
, VCPU_REGS_RBX
) = ebx
;
3494 *reg_write(ctxt
, VCPU_REGS_RCX
) = ecx
;
3495 *reg_write(ctxt
, VCPU_REGS_RDX
) = edx
;
3496 return X86EMUL_CONTINUE
;
3499 static int em_sahf(struct x86_emulate_ctxt
*ctxt
)
3503 flags
= EFLG_CF
| EFLG_PF
| EFLG_AF
| EFLG_ZF
| EFLG_SF
;
3504 flags
&= *reg_rmw(ctxt
, VCPU_REGS_RAX
) >> 8;
3506 ctxt
->eflags
&= ~0xffUL
;
3507 ctxt
->eflags
|= flags
| X86_EFLAGS_FIXED
;
3508 return X86EMUL_CONTINUE
;
3511 static int em_lahf(struct x86_emulate_ctxt
*ctxt
)
3513 *reg_rmw(ctxt
, VCPU_REGS_RAX
) &= ~0xff00UL
;
3514 *reg_rmw(ctxt
, VCPU_REGS_RAX
) |= (ctxt
->eflags
& 0xff) << 8;
3515 return X86EMUL_CONTINUE
;
3518 static int em_bswap(struct x86_emulate_ctxt
*ctxt
)
3520 switch (ctxt
->op_bytes
) {
3521 #ifdef CONFIG_X86_64
3523 asm("bswap %0" : "+r"(ctxt
->dst
.val
));
3527 asm("bswap %0" : "+r"(*(u32
*)&ctxt
->dst
.val
));
3530 return X86EMUL_CONTINUE
;
3533 static int em_clflush(struct x86_emulate_ctxt
*ctxt
)
3535 /* emulating clflush regardless of cpuid */
3536 return X86EMUL_CONTINUE
;
3539 static int em_movsxd(struct x86_emulate_ctxt
*ctxt
)
3541 ctxt
->dst
.val
= (s32
) ctxt
->src
.val
;
3542 return X86EMUL_CONTINUE
;
3545 static bool valid_cr(int nr
)
3557 static int check_cr_read(struct x86_emulate_ctxt
*ctxt
)
3559 if (!valid_cr(ctxt
->modrm_reg
))
3560 return emulate_ud(ctxt
);
3562 return X86EMUL_CONTINUE
;
3565 static int check_cr_write(struct x86_emulate_ctxt
*ctxt
)
3567 u64 new_val
= ctxt
->src
.val64
;
3568 int cr
= ctxt
->modrm_reg
;
3571 static u64 cr_reserved_bits
[] = {
3572 0xffffffff00000000ULL
,
3573 0, 0, 0, /* CR3 checked later */
3580 return emulate_ud(ctxt
);
3582 if (new_val
& cr_reserved_bits
[cr
])
3583 return emulate_gp(ctxt
, 0);
3588 if (((new_val
& X86_CR0_PG
) && !(new_val
& X86_CR0_PE
)) ||
3589 ((new_val
& X86_CR0_NW
) && !(new_val
& X86_CR0_CD
)))
3590 return emulate_gp(ctxt
, 0);
3592 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3593 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3595 if ((new_val
& X86_CR0_PG
) && (efer
& EFER_LME
) &&
3596 !(cr4
& X86_CR4_PAE
))
3597 return emulate_gp(ctxt
, 0);
3604 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3605 if (efer
& EFER_LMA
)
3606 rsvd
= CR3_L_MODE_RESERVED_BITS
& ~CR3_PCID_INVD
;
3609 return emulate_gp(ctxt
, 0);
3614 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3616 if ((efer
& EFER_LMA
) && !(new_val
& X86_CR4_PAE
))
3617 return emulate_gp(ctxt
, 0);
3623 return X86EMUL_CONTINUE
;
3626 static int check_dr7_gd(struct x86_emulate_ctxt
*ctxt
)
3630 ctxt
->ops
->get_dr(ctxt
, 7, &dr7
);
3632 /* Check if DR7.Global_Enable is set */
3633 return dr7
& (1 << 13);
3636 static int check_dr_read(struct x86_emulate_ctxt
*ctxt
)
3638 int dr
= ctxt
->modrm_reg
;
3642 return emulate_ud(ctxt
);
3644 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3645 if ((cr4
& X86_CR4_DE
) && (dr
== 4 || dr
== 5))
3646 return emulate_ud(ctxt
);
3648 if (check_dr7_gd(ctxt
)) {
3651 ctxt
->ops
->get_dr(ctxt
, 6, &dr6
);
3653 dr6
|= DR6_BD
| DR6_RTM
;
3654 ctxt
->ops
->set_dr(ctxt
, 6, dr6
);
3655 return emulate_db(ctxt
);
3658 return X86EMUL_CONTINUE
;
3661 static int check_dr_write(struct x86_emulate_ctxt
*ctxt
)
3663 u64 new_val
= ctxt
->src
.val64
;
3664 int dr
= ctxt
->modrm_reg
;
3666 if ((dr
== 6 || dr
== 7) && (new_val
& 0xffffffff00000000ULL
))
3667 return emulate_gp(ctxt
, 0);
3669 return check_dr_read(ctxt
);
3672 static int check_svme(struct x86_emulate_ctxt
*ctxt
)
3676 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
3678 if (!(efer
& EFER_SVME
))
3679 return emulate_ud(ctxt
);
3681 return X86EMUL_CONTINUE
;
3684 static int check_svme_pa(struct x86_emulate_ctxt
*ctxt
)
3686 u64 rax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3688 /* Valid physical address? */
3689 if (rax
& 0xffff000000000000ULL
)
3690 return emulate_gp(ctxt
, 0);
3692 return check_svme(ctxt
);
3695 static int check_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3697 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3699 if (cr4
& X86_CR4_TSD
&& ctxt
->ops
->cpl(ctxt
))
3700 return emulate_ud(ctxt
);
3702 return X86EMUL_CONTINUE
;
3705 static int check_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3707 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
3708 u64 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3710 if ((!(cr4
& X86_CR4_PCE
) && ctxt
->ops
->cpl(ctxt
)) ||
3711 ctxt
->ops
->check_pmc(ctxt
, rcx
))
3712 return emulate_gp(ctxt
, 0);
3714 return X86EMUL_CONTINUE
;
3717 static int check_perm_in(struct x86_emulate_ctxt
*ctxt
)
3719 ctxt
->dst
.bytes
= min(ctxt
->dst
.bytes
, 4u);
3720 if (!emulator_io_permited(ctxt
, ctxt
->src
.val
, ctxt
->dst
.bytes
))
3721 return emulate_gp(ctxt
, 0);
3723 return X86EMUL_CONTINUE
;
3726 static int check_perm_out(struct x86_emulate_ctxt
*ctxt
)
3728 ctxt
->src
.bytes
= min(ctxt
->src
.bytes
, 4u);
3729 if (!emulator_io_permited(ctxt
, ctxt
->dst
.val
, ctxt
->src
.bytes
))
3730 return emulate_gp(ctxt
, 0);
3732 return X86EMUL_CONTINUE
;
3735 #define D(_y) { .flags = (_y) }
3736 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3737 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3738 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3739 #define N D(NotImpl)
3740 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3741 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3742 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3743 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3744 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3745 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3746 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3747 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3748 #define II(_f, _e, _i) \
3749 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3750 #define IIP(_f, _e, _i, _p) \
3751 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3752 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3753 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3755 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3756 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3757 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3758 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3759 #define I2bvIP(_f, _e, _i, _p) \
3760 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3762 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3763 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3764 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3766 static const struct opcode group7_rm0
[] = {
3768 I(SrcNone
| Priv
| EmulateOnUD
, em_vmcall
),
3772 static const struct opcode group7_rm1
[] = {
3773 DI(SrcNone
| Priv
, monitor
),
3774 DI(SrcNone
| Priv
, mwait
),
3778 static const struct opcode group7_rm3
[] = {
3779 DIP(SrcNone
| Prot
| Priv
, vmrun
, check_svme_pa
),
3780 II(SrcNone
| Prot
| EmulateOnUD
, em_vmmcall
, vmmcall
),
3781 DIP(SrcNone
| Prot
| Priv
, vmload
, check_svme_pa
),
3782 DIP(SrcNone
| Prot
| Priv
, vmsave
, check_svme_pa
),
3783 DIP(SrcNone
| Prot
| Priv
, stgi
, check_svme
),
3784 DIP(SrcNone
| Prot
| Priv
, clgi
, check_svme
),
3785 DIP(SrcNone
| Prot
| Priv
, skinit
, check_svme
),
3786 DIP(SrcNone
| Prot
| Priv
, invlpga
, check_svme
),
3789 static const struct opcode group7_rm7
[] = {
3791 DIP(SrcNone
, rdtscp
, check_rdtsc
),
3795 static const struct opcode group1
[] = {
3797 F(Lock
| PageTable
, em_or
),
3800 F(Lock
| PageTable
, em_and
),
3806 static const struct opcode group1A
[] = {
3807 I(DstMem
| SrcNone
| Mov
| Stack
| IncSP
, em_pop
), N
, N
, N
, N
, N
, N
, N
,
3810 static const struct opcode group2
[] = {
3811 F(DstMem
| ModRM
, em_rol
),
3812 F(DstMem
| ModRM
, em_ror
),
3813 F(DstMem
| ModRM
, em_rcl
),
3814 F(DstMem
| ModRM
, em_rcr
),
3815 F(DstMem
| ModRM
, em_shl
),
3816 F(DstMem
| ModRM
, em_shr
),
3817 F(DstMem
| ModRM
, em_shl
),
3818 F(DstMem
| ModRM
, em_sar
),
3821 static const struct opcode group3
[] = {
3822 F(DstMem
| SrcImm
| NoWrite
, em_test
),
3823 F(DstMem
| SrcImm
| NoWrite
, em_test
),
3824 F(DstMem
| SrcNone
| Lock
, em_not
),
3825 F(DstMem
| SrcNone
| Lock
, em_neg
),
3826 F(DstXacc
| Src2Mem
, em_mul_ex
),
3827 F(DstXacc
| Src2Mem
, em_imul_ex
),
3828 F(DstXacc
| Src2Mem
, em_div_ex
),
3829 F(DstXacc
| Src2Mem
, em_idiv_ex
),
3832 static const struct opcode group4
[] = {
3833 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_inc
),
3834 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_dec
),
3838 static const struct opcode group5
[] = {
3839 F(DstMem
| SrcNone
| Lock
, em_inc
),
3840 F(DstMem
| SrcNone
| Lock
, em_dec
),
3841 I(SrcMem
| NearBranch
, em_call_near_abs
),
3842 I(SrcMemFAddr
| ImplicitOps
| Stack
, em_call_far
),
3843 I(SrcMem
| NearBranch
, em_jmp_abs
),
3844 I(SrcMemFAddr
| ImplicitOps
, em_jmp_far
),
3845 I(SrcMem
| Stack
, em_push
), D(Undefined
),
3848 static const struct opcode group6
[] = {
3851 II(Prot
| Priv
| SrcMem16
, em_lldt
, lldt
),
3852 II(Prot
| Priv
| SrcMem16
, em_ltr
, ltr
),
3856 static const struct group_dual group7
= { {
3857 II(Mov
| DstMem
, em_sgdt
, sgdt
),
3858 II(Mov
| DstMem
, em_sidt
, sidt
),
3859 II(SrcMem
| Priv
, em_lgdt
, lgdt
),
3860 II(SrcMem
| Priv
, em_lidt
, lidt
),
3861 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
3862 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
3863 II(SrcMem
| ByteOp
| Priv
| NoAccess
, em_invlpg
, invlpg
),
3867 N
, EXT(0, group7_rm3
),
3868 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
3869 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
3873 static const struct opcode group8
[] = {
3875 F(DstMem
| SrcImmByte
| NoWrite
, em_bt
),
3876 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_bts
),
3877 F(DstMem
| SrcImmByte
| Lock
, em_btr
),
3878 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_btc
),
3881 static const struct group_dual group9
= { {
3882 N
, I(DstMem64
| Lock
| PageTable
, em_cmpxchg8b
), N
, N
, N
, N
, N
, N
,
3884 N
, N
, N
, N
, N
, N
, N
, N
,
3887 static const struct opcode group11
[] = {
3888 I(DstMem
| SrcImm
| Mov
| PageTable
, em_mov
),
3892 static const struct gprefix pfx_0f_ae_7
= {
3893 I(SrcMem
| ByteOp
, em_clflush
), N
, N
, N
,
3896 static const struct group_dual group15
= { {
3897 N
, N
, N
, N
, N
, N
, N
, GP(0, &pfx_0f_ae_7
),
3899 N
, N
, N
, N
, N
, N
, N
, N
,
3902 static const struct gprefix pfx_0f_6f_0f_7f
= {
3903 I(Mmx
, em_mov
), I(Sse
| Aligned
, em_mov
), N
, I(Sse
| Unaligned
, em_mov
),
3906 static const struct instr_dual instr_dual_0f_2b
= {
3910 static const struct gprefix pfx_0f_2b
= {
3911 ID(0, &instr_dual_0f_2b
), ID(0, &instr_dual_0f_2b
), N
, N
,
3914 static const struct gprefix pfx_0f_28_0f_29
= {
3915 I(Aligned
, em_mov
), I(Aligned
, em_mov
), N
, N
,
3918 static const struct gprefix pfx_0f_e7
= {
3919 N
, I(Sse
, em_mov
), N
, N
,
3922 static const struct escape escape_d9
= { {
3923 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstcw
),
3926 N
, N
, N
, N
, N
, N
, N
, N
,
3928 N
, N
, N
, N
, N
, N
, N
, N
,
3930 N
, N
, N
, N
, N
, N
, N
, N
,
3932 N
, N
, N
, N
, N
, N
, N
, N
,
3934 N
, N
, N
, N
, N
, N
, N
, N
,
3936 N
, N
, N
, N
, N
, N
, N
, N
,
3938 N
, N
, N
, N
, N
, N
, N
, N
,
3940 N
, N
, N
, N
, N
, N
, N
, N
,
3943 static const struct escape escape_db
= { {
3944 N
, N
, N
, N
, N
, N
, N
, N
,
3947 N
, N
, N
, N
, N
, N
, N
, N
,
3949 N
, N
, N
, N
, N
, N
, N
, N
,
3951 N
, N
, N
, N
, N
, N
, N
, N
,
3953 N
, N
, N
, N
, N
, N
, N
, N
,
3955 N
, N
, N
, I(ImplicitOps
, em_fninit
), N
, N
, N
, N
,
3957 N
, N
, N
, N
, N
, N
, N
, N
,
3959 N
, N
, N
, N
, N
, N
, N
, N
,
3961 N
, N
, N
, N
, N
, N
, N
, N
,
3964 static const struct escape escape_dd
= { {
3965 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstsw
),
3968 N
, N
, N
, N
, N
, N
, N
, N
,
3970 N
, N
, N
, N
, N
, N
, N
, N
,
3972 N
, N
, N
, N
, N
, N
, N
, N
,
3974 N
, N
, N
, N
, N
, N
, N
, N
,
3976 N
, N
, N
, N
, N
, N
, N
, N
,
3978 N
, N
, N
, N
, N
, N
, N
, N
,
3980 N
, N
, N
, N
, N
, N
, N
, N
,
3982 N
, N
, N
, N
, N
, N
, N
, N
,
3985 static const struct instr_dual instr_dual_0f_c3
= {
3986 I(DstMem
| SrcReg
| ModRM
| No16
| Mov
, em_mov
), N
3989 static const struct mode_dual mode_dual_63
= {
3990 N
, I(DstReg
| SrcMem32
| ModRM
| Mov
, em_movsxd
)
3993 static const struct opcode opcode_table
[256] = {
3995 F6ALU(Lock
, em_add
),
3996 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_push_sreg
),
3997 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_pop_sreg
),
3999 F6ALU(Lock
| PageTable
, em_or
),
4000 I(ImplicitOps
| Stack
| No64
| Src2CS
, em_push_sreg
),
4003 F6ALU(Lock
, em_adc
),
4004 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_push_sreg
),
4005 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_pop_sreg
),
4007 F6ALU(Lock
, em_sbb
),
4008 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_push_sreg
),
4009 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_pop_sreg
),
4011 F6ALU(Lock
| PageTable
, em_and
), N
, N
,
4013 F6ALU(Lock
, em_sub
), N
, I(ByteOp
| DstAcc
| No64
, em_das
),
4015 F6ALU(Lock
, em_xor
), N
, N
,
4017 F6ALU(NoWrite
, em_cmp
), N
, N
,
4019 X8(F(DstReg
, em_inc
)), X8(F(DstReg
, em_dec
)),
4021 X8(I(SrcReg
| Stack
, em_push
)),
4023 X8(I(DstReg
| Stack
, em_pop
)),
4025 I(ImplicitOps
| Stack
| No64
, em_pusha
),
4026 I(ImplicitOps
| Stack
| No64
, em_popa
),
4027 N
, MD(ModRM
, &mode_dual_63
),
4030 I(SrcImm
| Mov
| Stack
, em_push
),
4031 I(DstReg
| SrcMem
| ModRM
| Src2Imm
, em_imul_3op
),
4032 I(SrcImmByte
| Mov
| Stack
, em_push
),
4033 I(DstReg
| SrcMem
| ModRM
| Src2ImmByte
, em_imul_3op
),
4034 I2bvIP(DstDI
| SrcDX
| Mov
| String
| Unaligned
, em_in
, ins
, check_perm_in
), /* insb, insw/insd */
4035 I2bvIP(SrcSI
| DstDX
| String
, em_out
, outs
, check_perm_out
), /* outsb, outsw/outsd */
4037 X16(D(SrcImmByte
| NearBranch
)),
4039 G(ByteOp
| DstMem
| SrcImm
, group1
),
4040 G(DstMem
| SrcImm
, group1
),
4041 G(ByteOp
| DstMem
| SrcImm
| No64
, group1
),
4042 G(DstMem
| SrcImmByte
, group1
),
4043 F2bv(DstMem
| SrcReg
| ModRM
| NoWrite
, em_test
),
4044 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
, em_xchg
),
4046 I2bv(DstMem
| SrcReg
| ModRM
| Mov
| PageTable
, em_mov
),
4047 I2bv(DstReg
| SrcMem
| ModRM
| Mov
, em_mov
),
4048 I(DstMem
| SrcNone
| ModRM
| Mov
| PageTable
, em_mov_rm_sreg
),
4049 D(ModRM
| SrcMem
| NoAccess
| DstReg
),
4050 I(ImplicitOps
| SrcMem16
| ModRM
, em_mov_sreg_rm
),
4053 DI(SrcAcc
| DstReg
, pause
), X7(D(SrcAcc
| DstReg
)),
4055 D(DstAcc
| SrcNone
), I(ImplicitOps
| SrcAcc
, em_cwd
),
4056 I(SrcImmFAddr
| No64
, em_call_far
), N
,
4057 II(ImplicitOps
| Stack
, em_pushf
, pushf
),
4058 II(ImplicitOps
| Stack
, em_popf
, popf
),
4059 I(ImplicitOps
, em_sahf
), I(ImplicitOps
, em_lahf
),
4061 I2bv(DstAcc
| SrcMem
| Mov
| MemAbs
, em_mov
),
4062 I2bv(DstMem
| SrcAcc
| Mov
| MemAbs
| PageTable
, em_mov
),
4063 I2bv(SrcSI
| DstDI
| Mov
| String
, em_mov
),
4064 F2bv(SrcSI
| DstDI
| String
| NoWrite
, em_cmp_r
),
4066 F2bv(DstAcc
| SrcImm
| NoWrite
, em_test
),
4067 I2bv(SrcAcc
| DstDI
| Mov
| String
, em_mov
),
4068 I2bv(SrcSI
| DstAcc
| Mov
| String
, em_mov
),
4069 F2bv(SrcAcc
| DstDI
| String
| NoWrite
, em_cmp_r
),
4071 X8(I(ByteOp
| DstReg
| SrcImm
| Mov
, em_mov
)),
4073 X8(I(DstReg
| SrcImm64
| Mov
, em_mov
)),
4075 G(ByteOp
| Src2ImmByte
, group2
), G(Src2ImmByte
, group2
),
4076 I(ImplicitOps
| NearBranch
| SrcImmU16
, em_ret_near_imm
),
4077 I(ImplicitOps
| NearBranch
, em_ret
),
4078 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2ES
, em_lseg
),
4079 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2DS
, em_lseg
),
4080 G(ByteOp
, group11
), G(0, group11
),
4082 I(Stack
| SrcImmU16
| Src2ImmByte
, em_enter
), I(Stack
, em_leave
),
4083 I(ImplicitOps
| SrcImmU16
, em_ret_far_imm
),
4084 I(ImplicitOps
, em_ret_far
),
4085 D(ImplicitOps
), DI(SrcImmByte
, intn
),
4086 D(ImplicitOps
| No64
), II(ImplicitOps
, em_iret
, iret
),
4088 G(Src2One
| ByteOp
, group2
), G(Src2One
, group2
),
4089 G(Src2CL
| ByteOp
, group2
), G(Src2CL
, group2
),
4090 I(DstAcc
| SrcImmUByte
| No64
, em_aam
),
4091 I(DstAcc
| SrcImmUByte
| No64
, em_aad
),
4092 F(DstAcc
| ByteOp
| No64
, em_salc
),
4093 I(DstAcc
| SrcXLat
| ByteOp
, em_mov
),
4095 N
, E(0, &escape_d9
), N
, E(0, &escape_db
), N
, E(0, &escape_dd
), N
, N
,
4097 X3(I(SrcImmByte
| NearBranch
, em_loop
)),
4098 I(SrcImmByte
| NearBranch
, em_jcxz
),
4099 I2bvIP(SrcImmUByte
| DstAcc
, em_in
, in
, check_perm_in
),
4100 I2bvIP(SrcAcc
| DstImmUByte
, em_out
, out
, check_perm_out
),
4102 I(SrcImm
| NearBranch
, em_call
), D(SrcImm
| ImplicitOps
| NearBranch
),
4103 I(SrcImmFAddr
| No64
, em_jmp_far
),
4104 D(SrcImmByte
| ImplicitOps
| NearBranch
),
4105 I2bvIP(SrcDX
| DstAcc
, em_in
, in
, check_perm_in
),
4106 I2bvIP(SrcAcc
| DstDX
, em_out
, out
, check_perm_out
),
4108 N
, DI(ImplicitOps
, icebp
), N
, N
,
4109 DI(ImplicitOps
| Priv
, hlt
), D(ImplicitOps
),
4110 G(ByteOp
, group3
), G(0, group3
),
4112 D(ImplicitOps
), D(ImplicitOps
),
4113 I(ImplicitOps
, em_cli
), I(ImplicitOps
, em_sti
),
4114 D(ImplicitOps
), D(ImplicitOps
), G(0, group4
), G(0, group5
),
4117 static const struct opcode twobyte_table
[256] = {
4119 G(0, group6
), GD(0, &group7
), N
, N
,
4120 N
, I(ImplicitOps
| EmulateOnUD
, em_syscall
),
4121 II(ImplicitOps
| Priv
, em_clts
, clts
), N
,
4122 DI(ImplicitOps
| Priv
, invd
), DI(ImplicitOps
| Priv
, wbinvd
), N
, N
,
4123 N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
), N
, N
,
4125 N
, N
, N
, N
, N
, N
, N
, N
,
4126 D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4127 N
, N
, N
, N
, N
, N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4129 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, cr_read
, check_cr_read
),
4130 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, dr_read
, check_dr_read
),
4131 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_cr_write
, cr_write
,
4133 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_dr_write
, dr_write
,
4136 GP(ModRM
| DstReg
| SrcMem
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4137 GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4138 N
, GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_2b
),
4141 II(ImplicitOps
| Priv
, em_wrmsr
, wrmsr
),
4142 IIP(ImplicitOps
, em_rdtsc
, rdtsc
, check_rdtsc
),
4143 II(ImplicitOps
| Priv
, em_rdmsr
, rdmsr
),
4144 IIP(ImplicitOps
, em_rdpmc
, rdpmc
, check_rdpmc
),
4145 I(ImplicitOps
| EmulateOnUD
, em_sysenter
),
4146 I(ImplicitOps
| Priv
| EmulateOnUD
, em_sysexit
),
4148 N
, N
, N
, N
, N
, N
, N
, N
,
4150 X16(D(DstReg
| SrcMem
| ModRM
)),
4152 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4157 N
, N
, N
, GP(SrcMem
| DstReg
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4162 N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4164 X16(D(SrcImm
| NearBranch
)),
4166 X16(D(ByteOp
| DstMem
| SrcNone
| ModRM
| Mov
)),
4168 I(Stack
| Src2FS
, em_push_sreg
), I(Stack
| Src2FS
, em_pop_sreg
),
4169 II(ImplicitOps
, em_cpuid
, cpuid
),
4170 F(DstMem
| SrcReg
| ModRM
| BitOp
| NoWrite
, em_bt
),
4171 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shld
),
4172 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shld
), N
, N
,
4174 I(Stack
| Src2GS
, em_push_sreg
), I(Stack
| Src2GS
, em_pop_sreg
),
4175 DI(ImplicitOps
, rsm
),
4176 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_bts
),
4177 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shrd
),
4178 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shrd
),
4179 GD(0, &group15
), F(DstReg
| SrcMem
| ModRM
, em_imul
),
4181 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
| SrcWrite
, em_cmpxchg
),
4182 I(DstReg
| SrcMemFAddr
| ModRM
| Src2SS
, em_lseg
),
4183 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
, em_btr
),
4184 I(DstReg
| SrcMemFAddr
| ModRM
| Src2FS
, em_lseg
),
4185 I(DstReg
| SrcMemFAddr
| ModRM
| Src2GS
, em_lseg
),
4186 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4190 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_btc
),
4191 F(DstReg
| SrcMem
| ModRM
, em_bsf
), F(DstReg
| SrcMem
| ModRM
, em_bsr
),
4192 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4194 F2bv(DstMem
| SrcReg
| ModRM
| SrcWrite
| Lock
, em_xadd
),
4195 N
, ID(0, &instr_dual_0f_c3
),
4196 N
, N
, N
, GD(0, &group9
),
4198 X8(I(DstReg
, em_bswap
)),
4200 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4202 N
, N
, N
, N
, N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_e7
),
4203 N
, N
, N
, N
, N
, N
, N
, N
,
4205 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
4208 static const struct instr_dual instr_dual_0f_38_f0
= {
4209 I(DstReg
| SrcMem
| Mov
, em_movbe
), N
4212 static const struct instr_dual instr_dual_0f_38_f1
= {
4213 I(DstMem
| SrcReg
| Mov
, em_movbe
), N
4216 static const struct gprefix three_byte_0f_38_f0
= {
4217 ID(0, &instr_dual_0f_38_f0
), N
, N
, N
4220 static const struct gprefix three_byte_0f_38_f1
= {
4221 ID(0, &instr_dual_0f_38_f1
), N
, N
, N
4225 * Insns below are selected by the prefix which indexed by the third opcode
4228 static const struct opcode opcode_map_0f_38
[256] = {
4230 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4232 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4234 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f0
),
4235 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f1
),
4256 static unsigned imm_size(struct x86_emulate_ctxt
*ctxt
)
4260 size
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4266 static int decode_imm(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4267 unsigned size
, bool sign_extension
)
4269 int rc
= X86EMUL_CONTINUE
;
4273 op
->addr
.mem
.ea
= ctxt
->_eip
;
4274 /* NB. Immediates are sign-extended as necessary. */
4275 switch (op
->bytes
) {
4277 op
->val
= insn_fetch(s8
, ctxt
);
4280 op
->val
= insn_fetch(s16
, ctxt
);
4283 op
->val
= insn_fetch(s32
, ctxt
);
4286 op
->val
= insn_fetch(s64
, ctxt
);
4289 if (!sign_extension
) {
4290 switch (op
->bytes
) {
4298 op
->val
&= 0xffffffff;
4306 static int decode_operand(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4309 int rc
= X86EMUL_CONTINUE
;
4313 decode_register_operand(ctxt
, op
);
4316 rc
= decode_imm(ctxt
, op
, 1, false);
4319 ctxt
->memop
.bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4323 if (ctxt
->d
& BitOp
)
4324 fetch_bit_operand(ctxt
);
4325 op
->orig_val
= op
->val
;
4328 ctxt
->memop
.bytes
= (ctxt
->op_bytes
== 8) ? 16 : 8;
4332 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4333 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4334 fetch_register_operand(op
);
4335 op
->orig_val
= op
->val
;
4339 op
->bytes
= (ctxt
->d
& ByteOp
) ? 2 : ctxt
->op_bytes
;
4340 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4341 fetch_register_operand(op
);
4342 op
->orig_val
= op
->val
;
4345 if (ctxt
->d
& ByteOp
) {
4350 op
->bytes
= ctxt
->op_bytes
;
4351 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4352 fetch_register_operand(op
);
4353 op
->orig_val
= op
->val
;
4357 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4359 register_address(ctxt
, VCPU_REGS_RDI
);
4360 op
->addr
.mem
.seg
= VCPU_SREG_ES
;
4367 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4368 fetch_register_operand(op
);
4373 op
->val
= reg_read(ctxt
, VCPU_REGS_RCX
) & 0xff;
4376 rc
= decode_imm(ctxt
, op
, 1, true);
4384 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), true);
4387 rc
= decode_imm(ctxt
, op
, ctxt
->op_bytes
, true);
4390 ctxt
->memop
.bytes
= 1;
4391 if (ctxt
->memop
.type
== OP_REG
) {
4392 ctxt
->memop
.addr
.reg
= decode_register(ctxt
,
4393 ctxt
->modrm_rm
, true);
4394 fetch_register_operand(&ctxt
->memop
);
4398 ctxt
->memop
.bytes
= 2;
4401 ctxt
->memop
.bytes
= 4;
4404 rc
= decode_imm(ctxt
, op
, 2, false);
4407 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), false);
4411 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4413 register_address(ctxt
, VCPU_REGS_RSI
);
4414 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4420 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4423 reg_read(ctxt
, VCPU_REGS_RBX
) +
4424 (reg_read(ctxt
, VCPU_REGS_RAX
) & 0xff));
4425 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4430 op
->addr
.mem
.ea
= ctxt
->_eip
;
4431 op
->bytes
= ctxt
->op_bytes
+ 2;
4432 insn_fetch_arr(op
->valptr
, op
->bytes
, ctxt
);
4435 ctxt
->memop
.bytes
= ctxt
->op_bytes
+ 2;
4439 op
->val
= VCPU_SREG_ES
;
4443 op
->val
= VCPU_SREG_CS
;
4447 op
->val
= VCPU_SREG_SS
;
4451 op
->val
= VCPU_SREG_DS
;
4455 op
->val
= VCPU_SREG_FS
;
4459 op
->val
= VCPU_SREG_GS
;
4462 /* Special instructions do their own operand decoding. */
4464 op
->type
= OP_NONE
; /* Disable writeback. */
4472 int x86_decode_insn(struct x86_emulate_ctxt
*ctxt
, void *insn
, int insn_len
)
4474 int rc
= X86EMUL_CONTINUE
;
4475 int mode
= ctxt
->mode
;
4476 int def_op_bytes
, def_ad_bytes
, goffset
, simd_prefix
;
4477 bool op_prefix
= false;
4478 bool has_seg_override
= false;
4479 struct opcode opcode
;
4481 ctxt
->memop
.type
= OP_NONE
;
4482 ctxt
->memopp
= NULL
;
4483 ctxt
->_eip
= ctxt
->eip
;
4484 ctxt
->fetch
.ptr
= ctxt
->fetch
.data
;
4485 ctxt
->fetch
.end
= ctxt
->fetch
.data
+ insn_len
;
4486 ctxt
->opcode_len
= 1;
4488 memcpy(ctxt
->fetch
.data
, insn
, insn_len
);
4490 rc
= __do_insn_fetch_bytes(ctxt
, 1);
4491 if (rc
!= X86EMUL_CONTINUE
)
4496 case X86EMUL_MODE_REAL
:
4497 case X86EMUL_MODE_VM86
:
4498 case X86EMUL_MODE_PROT16
:
4499 def_op_bytes
= def_ad_bytes
= 2;
4501 case X86EMUL_MODE_PROT32
:
4502 def_op_bytes
= def_ad_bytes
= 4;
4504 #ifdef CONFIG_X86_64
4505 case X86EMUL_MODE_PROT64
:
4511 return EMULATION_FAILED
;
4514 ctxt
->op_bytes
= def_op_bytes
;
4515 ctxt
->ad_bytes
= def_ad_bytes
;
4517 /* Legacy prefixes. */
4519 switch (ctxt
->b
= insn_fetch(u8
, ctxt
)) {
4520 case 0x66: /* operand-size override */
4522 /* switch between 2/4 bytes */
4523 ctxt
->op_bytes
= def_op_bytes
^ 6;
4525 case 0x67: /* address-size override */
4526 if (mode
== X86EMUL_MODE_PROT64
)
4527 /* switch between 4/8 bytes */
4528 ctxt
->ad_bytes
= def_ad_bytes
^ 12;
4530 /* switch between 2/4 bytes */
4531 ctxt
->ad_bytes
= def_ad_bytes
^ 6;
4533 case 0x26: /* ES override */
4534 case 0x2e: /* CS override */
4535 case 0x36: /* SS override */
4536 case 0x3e: /* DS override */
4537 has_seg_override
= true;
4538 ctxt
->seg_override
= (ctxt
->b
>> 3) & 3;
4540 case 0x64: /* FS override */
4541 case 0x65: /* GS override */
4542 has_seg_override
= true;
4543 ctxt
->seg_override
= ctxt
->b
& 7;
4545 case 0x40 ... 0x4f: /* REX */
4546 if (mode
!= X86EMUL_MODE_PROT64
)
4548 ctxt
->rex_prefix
= ctxt
->b
;
4550 case 0xf0: /* LOCK */
4551 ctxt
->lock_prefix
= 1;
4553 case 0xf2: /* REPNE/REPNZ */
4554 case 0xf3: /* REP/REPE/REPZ */
4555 ctxt
->rep_prefix
= ctxt
->b
;
4561 /* Any legacy prefix after a REX prefix nullifies its effect. */
4563 ctxt
->rex_prefix
= 0;
4569 if (ctxt
->rex_prefix
& 8)
4570 ctxt
->op_bytes
= 8; /* REX.W */
4572 /* Opcode byte(s). */
4573 opcode
= opcode_table
[ctxt
->b
];
4574 /* Two-byte opcode? */
4575 if (ctxt
->b
== 0x0f) {
4576 ctxt
->opcode_len
= 2;
4577 ctxt
->b
= insn_fetch(u8
, ctxt
);
4578 opcode
= twobyte_table
[ctxt
->b
];
4580 /* 0F_38 opcode map */
4581 if (ctxt
->b
== 0x38) {
4582 ctxt
->opcode_len
= 3;
4583 ctxt
->b
= insn_fetch(u8
, ctxt
);
4584 opcode
= opcode_map_0f_38
[ctxt
->b
];
4587 ctxt
->d
= opcode
.flags
;
4589 if (ctxt
->d
& ModRM
)
4590 ctxt
->modrm
= insn_fetch(u8
, ctxt
);
4592 /* vex-prefix instructions are not implemented */
4593 if (ctxt
->opcode_len
== 1 && (ctxt
->b
== 0xc5 || ctxt
->b
== 0xc4) &&
4594 (mode
== X86EMUL_MODE_PROT64
|| (ctxt
->modrm
& 0xc0) == 0xc0)) {
4598 while (ctxt
->d
& GroupMask
) {
4599 switch (ctxt
->d
& GroupMask
) {
4601 goffset
= (ctxt
->modrm
>> 3) & 7;
4602 opcode
= opcode
.u
.group
[goffset
];
4605 goffset
= (ctxt
->modrm
>> 3) & 7;
4606 if ((ctxt
->modrm
>> 6) == 3)
4607 opcode
= opcode
.u
.gdual
->mod3
[goffset
];
4609 opcode
= opcode
.u
.gdual
->mod012
[goffset
];
4612 goffset
= ctxt
->modrm
& 7;
4613 opcode
= opcode
.u
.group
[goffset
];
4616 if (ctxt
->rep_prefix
&& op_prefix
)
4617 return EMULATION_FAILED
;
4618 simd_prefix
= op_prefix
? 0x66 : ctxt
->rep_prefix
;
4619 switch (simd_prefix
) {
4620 case 0x00: opcode
= opcode
.u
.gprefix
->pfx_no
; break;
4621 case 0x66: opcode
= opcode
.u
.gprefix
->pfx_66
; break;
4622 case 0xf2: opcode
= opcode
.u
.gprefix
->pfx_f2
; break;
4623 case 0xf3: opcode
= opcode
.u
.gprefix
->pfx_f3
; break;
4627 if (ctxt
->modrm
> 0xbf)
4628 opcode
= opcode
.u
.esc
->high
[ctxt
->modrm
- 0xc0];
4630 opcode
= opcode
.u
.esc
->op
[(ctxt
->modrm
>> 3) & 7];
4633 if ((ctxt
->modrm
>> 6) == 3)
4634 opcode
= opcode
.u
.idual
->mod3
;
4636 opcode
= opcode
.u
.idual
->mod012
;
4639 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
4640 opcode
= opcode
.u
.mdual
->mode64
;
4642 opcode
= opcode
.u
.mdual
->mode32
;
4645 return EMULATION_FAILED
;
4648 ctxt
->d
&= ~(u64
)GroupMask
;
4649 ctxt
->d
|= opcode
.flags
;
4654 return EMULATION_FAILED
;
4656 ctxt
->execute
= opcode
.u
.execute
;
4658 if (unlikely(ctxt
->ud
) && likely(!(ctxt
->d
& EmulateOnUD
)))
4659 return EMULATION_FAILED
;
4661 if (unlikely(ctxt
->d
&
4662 (NotImpl
|Stack
|Op3264
|Sse
|Mmx
|Intercept
|CheckPerm
|NearBranch
|
4665 * These are copied unconditionally here, and checked unconditionally
4666 * in x86_emulate_insn.
4668 ctxt
->check_perm
= opcode
.check_perm
;
4669 ctxt
->intercept
= opcode
.intercept
;
4671 if (ctxt
->d
& NotImpl
)
4672 return EMULATION_FAILED
;
4674 if (mode
== X86EMUL_MODE_PROT64
) {
4675 if (ctxt
->op_bytes
== 4 && (ctxt
->d
& Stack
))
4677 else if (ctxt
->d
& NearBranch
)
4681 if (ctxt
->d
& Op3264
) {
4682 if (mode
== X86EMUL_MODE_PROT64
)
4688 if ((ctxt
->d
& No16
) && ctxt
->op_bytes
== 2)
4692 ctxt
->op_bytes
= 16;
4693 else if (ctxt
->d
& Mmx
)
4697 /* ModRM and SIB bytes. */
4698 if (ctxt
->d
& ModRM
) {
4699 rc
= decode_modrm(ctxt
, &ctxt
->memop
);
4700 if (!has_seg_override
) {
4701 has_seg_override
= true;
4702 ctxt
->seg_override
= ctxt
->modrm_seg
;
4704 } else if (ctxt
->d
& MemAbs
)
4705 rc
= decode_abs(ctxt
, &ctxt
->memop
);
4706 if (rc
!= X86EMUL_CONTINUE
)
4709 if (!has_seg_override
)
4710 ctxt
->seg_override
= VCPU_SREG_DS
;
4712 ctxt
->memop
.addr
.mem
.seg
= ctxt
->seg_override
;
4715 * Decode and fetch the source operand: register, memory
4718 rc
= decode_operand(ctxt
, &ctxt
->src
, (ctxt
->d
>> SrcShift
) & OpMask
);
4719 if (rc
!= X86EMUL_CONTINUE
)
4723 * Decode and fetch the second source operand: register, memory
4726 rc
= decode_operand(ctxt
, &ctxt
->src2
, (ctxt
->d
>> Src2Shift
) & OpMask
);
4727 if (rc
!= X86EMUL_CONTINUE
)
4730 /* Decode and fetch the destination operand: register or memory. */
4731 rc
= decode_operand(ctxt
, &ctxt
->dst
, (ctxt
->d
>> DstShift
) & OpMask
);
4733 if (ctxt
->rip_relative
)
4734 ctxt
->memopp
->addr
.mem
.ea
= address_mask(ctxt
,
4735 ctxt
->memopp
->addr
.mem
.ea
+ ctxt
->_eip
);
4738 return (rc
!= X86EMUL_CONTINUE
) ? EMULATION_FAILED
: EMULATION_OK
;
4741 bool x86_page_table_writing_insn(struct x86_emulate_ctxt
*ctxt
)
4743 return ctxt
->d
& PageTable
;
4746 static bool string_insn_completed(struct x86_emulate_ctxt
*ctxt
)
4748 /* The second termination condition only applies for REPE
4749 * and REPNE. Test if the repeat string operation prefix is
4750 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4751 * corresponding termination condition according to:
4752 * - if REPE/REPZ and ZF = 0 then done
4753 * - if REPNE/REPNZ and ZF = 1 then done
4755 if (((ctxt
->b
== 0xa6) || (ctxt
->b
== 0xa7) ||
4756 (ctxt
->b
== 0xae) || (ctxt
->b
== 0xaf))
4757 && (((ctxt
->rep_prefix
== REPE_PREFIX
) &&
4758 ((ctxt
->eflags
& EFLG_ZF
) == 0))
4759 || ((ctxt
->rep_prefix
== REPNE_PREFIX
) &&
4760 ((ctxt
->eflags
& EFLG_ZF
) == EFLG_ZF
))))
4766 static int flush_pending_x87_faults(struct x86_emulate_ctxt
*ctxt
)
4770 ctxt
->ops
->get_fpu(ctxt
);
4771 asm volatile("1: fwait \n\t"
4773 ".pushsection .fixup,\"ax\" \n\t"
4775 "movb $1, %[fault] \n\t"
4778 _ASM_EXTABLE(1b
, 3b
)
4779 : [fault
]"+qm"(fault
));
4780 ctxt
->ops
->put_fpu(ctxt
);
4782 if (unlikely(fault
))
4783 return emulate_exception(ctxt
, MF_VECTOR
, 0, false);
4785 return X86EMUL_CONTINUE
;
4788 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt
*ctxt
,
4791 if (op
->type
== OP_MM
)
4792 read_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
4795 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*))
4797 ulong flags
= (ctxt
->eflags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
4798 if (!(ctxt
->d
& ByteOp
))
4799 fop
+= __ffs(ctxt
->dst
.bytes
) * FASTOP_SIZE
;
4800 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4801 : "+a"(ctxt
->dst
.val
), "+d"(ctxt
->src
.val
), [flags
]"+D"(flags
),
4803 : "c"(ctxt
->src2
.val
));
4804 ctxt
->eflags
= (ctxt
->eflags
& ~EFLAGS_MASK
) | (flags
& EFLAGS_MASK
);
4805 if (!fop
) /* exception is returned in fop variable */
4806 return emulate_de(ctxt
);
4807 return X86EMUL_CONTINUE
;
4810 void init_decode_cache(struct x86_emulate_ctxt
*ctxt
)
4812 memset(&ctxt
->rip_relative
, 0,
4813 (void *)&ctxt
->modrm
- (void *)&ctxt
->rip_relative
);
4815 ctxt
->io_read
.pos
= 0;
4816 ctxt
->io_read
.end
= 0;
4817 ctxt
->mem_read
.end
= 0;
4820 int x86_emulate_insn(struct x86_emulate_ctxt
*ctxt
)
4822 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
4823 int rc
= X86EMUL_CONTINUE
;
4824 int saved_dst_type
= ctxt
->dst
.type
;
4826 ctxt
->mem_read
.pos
= 0;
4828 /* LOCK prefix is allowed only with some instructions */
4829 if (ctxt
->lock_prefix
&& (!(ctxt
->d
& Lock
) || ctxt
->dst
.type
!= OP_MEM
)) {
4830 rc
= emulate_ud(ctxt
);
4834 if ((ctxt
->d
& SrcMask
) == SrcMemFAddr
&& ctxt
->src
.type
!= OP_MEM
) {
4835 rc
= emulate_ud(ctxt
);
4839 if (unlikely(ctxt
->d
&
4840 (No64
|Undefined
|Sse
|Mmx
|Intercept
|CheckPerm
|Priv
|Prot
|String
))) {
4841 if ((ctxt
->mode
== X86EMUL_MODE_PROT64
&& (ctxt
->d
& No64
)) ||
4842 (ctxt
->d
& Undefined
)) {
4843 rc
= emulate_ud(ctxt
);
4847 if (((ctxt
->d
& (Sse
|Mmx
)) && ((ops
->get_cr(ctxt
, 0) & X86_CR0_EM
)))
4848 || ((ctxt
->d
& Sse
) && !(ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
))) {
4849 rc
= emulate_ud(ctxt
);
4853 if ((ctxt
->d
& (Sse
|Mmx
)) && (ops
->get_cr(ctxt
, 0) & X86_CR0_TS
)) {
4854 rc
= emulate_nm(ctxt
);
4858 if (ctxt
->d
& Mmx
) {
4859 rc
= flush_pending_x87_faults(ctxt
);
4860 if (rc
!= X86EMUL_CONTINUE
)
4863 * Now that we know the fpu is exception safe, we can fetch
4866 fetch_possible_mmx_operand(ctxt
, &ctxt
->src
);
4867 fetch_possible_mmx_operand(ctxt
, &ctxt
->src2
);
4868 if (!(ctxt
->d
& Mov
))
4869 fetch_possible_mmx_operand(ctxt
, &ctxt
->dst
);
4872 if (unlikely(ctxt
->guest_mode
) && (ctxt
->d
& Intercept
)) {
4873 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
4874 X86_ICPT_PRE_EXCEPT
);
4875 if (rc
!= X86EMUL_CONTINUE
)
4879 /* Instruction can only be executed in protected mode */
4880 if ((ctxt
->d
& Prot
) && ctxt
->mode
< X86EMUL_MODE_PROT16
) {
4881 rc
= emulate_ud(ctxt
);
4885 /* Privileged instruction can be executed only in CPL=0 */
4886 if ((ctxt
->d
& Priv
) && ops
->cpl(ctxt
)) {
4887 if (ctxt
->d
& PrivUD
)
4888 rc
= emulate_ud(ctxt
);
4890 rc
= emulate_gp(ctxt
, 0);
4894 /* Do instruction specific permission checks */
4895 if (ctxt
->d
& CheckPerm
) {
4896 rc
= ctxt
->check_perm(ctxt
);
4897 if (rc
!= X86EMUL_CONTINUE
)
4901 if (unlikely(ctxt
->guest_mode
) && (ctxt
->d
& Intercept
)) {
4902 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
4903 X86_ICPT_POST_EXCEPT
);
4904 if (rc
!= X86EMUL_CONTINUE
)
4908 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
4909 /* All REP prefixes have the same first termination condition */
4910 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0) {
4911 ctxt
->eip
= ctxt
->_eip
;
4912 ctxt
->eflags
&= ~EFLG_RF
;
4918 if ((ctxt
->src
.type
== OP_MEM
) && !(ctxt
->d
& NoAccess
)) {
4919 rc
= segmented_read(ctxt
, ctxt
->src
.addr
.mem
,
4920 ctxt
->src
.valptr
, ctxt
->src
.bytes
);
4921 if (rc
!= X86EMUL_CONTINUE
)
4923 ctxt
->src
.orig_val64
= ctxt
->src
.val64
;
4926 if (ctxt
->src2
.type
== OP_MEM
) {
4927 rc
= segmented_read(ctxt
, ctxt
->src2
.addr
.mem
,
4928 &ctxt
->src2
.val
, ctxt
->src2
.bytes
);
4929 if (rc
!= X86EMUL_CONTINUE
)
4933 if ((ctxt
->d
& DstMask
) == ImplicitOps
)
4937 if ((ctxt
->dst
.type
== OP_MEM
) && !(ctxt
->d
& Mov
)) {
4938 /* optimisation - avoid slow emulated read if Mov */
4939 rc
= segmented_read(ctxt
, ctxt
->dst
.addr
.mem
,
4940 &ctxt
->dst
.val
, ctxt
->dst
.bytes
);
4941 if (rc
!= X86EMUL_CONTINUE
) {
4942 if (rc
== X86EMUL_PROPAGATE_FAULT
&&
4943 ctxt
->exception
.vector
== PF_VECTOR
)
4944 ctxt
->exception
.error_code
|= PFERR_WRITE_MASK
;
4948 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
4952 if (unlikely(ctxt
->guest_mode
) && (ctxt
->d
& Intercept
)) {
4953 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
4954 X86_ICPT_POST_MEMACCESS
);
4955 if (rc
!= X86EMUL_CONTINUE
)
4959 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
4960 ctxt
->eflags
|= EFLG_RF
;
4962 ctxt
->eflags
&= ~EFLG_RF
;
4964 if (ctxt
->execute
) {
4965 if (ctxt
->d
& Fastop
) {
4966 void (*fop
)(struct fastop
*) = (void *)ctxt
->execute
;
4967 rc
= fastop(ctxt
, fop
);
4968 if (rc
!= X86EMUL_CONTINUE
)
4972 rc
= ctxt
->execute(ctxt
);
4973 if (rc
!= X86EMUL_CONTINUE
)
4978 if (ctxt
->opcode_len
== 2)
4980 else if (ctxt
->opcode_len
== 3)
4981 goto threebyte_insn
;
4984 case 0x70 ... 0x7f: /* jcc (short) */
4985 if (test_cc(ctxt
->b
, ctxt
->eflags
))
4986 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
4988 case 0x8d: /* lea r16/r32, m */
4989 ctxt
->dst
.val
= ctxt
->src
.addr
.mem
.ea
;
4991 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4992 if (ctxt
->dst
.addr
.reg
== reg_rmw(ctxt
, VCPU_REGS_RAX
))
4993 ctxt
->dst
.type
= OP_NONE
;
4997 case 0x98: /* cbw/cwde/cdqe */
4998 switch (ctxt
->op_bytes
) {
4999 case 2: ctxt
->dst
.val
= (s8
)ctxt
->dst
.val
; break;
5000 case 4: ctxt
->dst
.val
= (s16
)ctxt
->dst
.val
; break;
5001 case 8: ctxt
->dst
.val
= (s32
)ctxt
->dst
.val
; break;
5004 case 0xcc: /* int3 */
5005 rc
= emulate_int(ctxt
, 3);
5007 case 0xcd: /* int n */
5008 rc
= emulate_int(ctxt
, ctxt
->src
.val
);
5010 case 0xce: /* into */
5011 if (ctxt
->eflags
& EFLG_OF
)
5012 rc
= emulate_int(ctxt
, 4);
5014 case 0xe9: /* jmp rel */
5015 case 0xeb: /* jmp rel short */
5016 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5017 ctxt
->dst
.type
= OP_NONE
; /* Disable writeback. */
5019 case 0xf4: /* hlt */
5020 ctxt
->ops
->halt(ctxt
);
5022 case 0xf5: /* cmc */
5023 /* complement carry flag from eflags reg */
5024 ctxt
->eflags
^= EFLG_CF
;
5026 case 0xf8: /* clc */
5027 ctxt
->eflags
&= ~EFLG_CF
;
5029 case 0xf9: /* stc */
5030 ctxt
->eflags
|= EFLG_CF
;
5032 case 0xfc: /* cld */
5033 ctxt
->eflags
&= ~EFLG_DF
;
5035 case 0xfd: /* std */
5036 ctxt
->eflags
|= EFLG_DF
;
5039 goto cannot_emulate
;
5042 if (rc
!= X86EMUL_CONTINUE
)
5046 if (ctxt
->d
& SrcWrite
) {
5047 BUG_ON(ctxt
->src
.type
== OP_MEM
|| ctxt
->src
.type
== OP_MEM_STR
);
5048 rc
= writeback(ctxt
, &ctxt
->src
);
5049 if (rc
!= X86EMUL_CONTINUE
)
5052 if (!(ctxt
->d
& NoWrite
)) {
5053 rc
= writeback(ctxt
, &ctxt
->dst
);
5054 if (rc
!= X86EMUL_CONTINUE
)
5059 * restore dst type in case the decoding will be reused
5060 * (happens for string instruction )
5062 ctxt
->dst
.type
= saved_dst_type
;
5064 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5065 string_addr_inc(ctxt
, VCPU_REGS_RSI
, &ctxt
->src
);
5067 if ((ctxt
->d
& DstMask
) == DstDI
)
5068 string_addr_inc(ctxt
, VCPU_REGS_RDI
, &ctxt
->dst
);
5070 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5072 struct read_cache
*r
= &ctxt
->io_read
;
5073 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5074 count
= ctxt
->src
.count
;
5076 count
= ctxt
->dst
.count
;
5077 register_address_increment(ctxt
, VCPU_REGS_RCX
, -count
);
5079 if (!string_insn_completed(ctxt
)) {
5081 * Re-enter guest when pio read ahead buffer is empty
5082 * or, if it is not used, after each 1024 iteration.
5084 if ((r
->end
!= 0 || reg_read(ctxt
, VCPU_REGS_RCX
) & 0x3ff) &&
5085 (r
->end
== 0 || r
->end
!= r
->pos
)) {
5087 * Reset read cache. Usually happens before
5088 * decode, but since instruction is restarted
5089 * we have to do it here.
5091 ctxt
->mem_read
.end
= 0;
5092 writeback_registers(ctxt
);
5093 return EMULATION_RESTART
;
5095 goto done
; /* skip rip writeback */
5097 ctxt
->eflags
&= ~EFLG_RF
;
5100 ctxt
->eip
= ctxt
->_eip
;
5103 if (rc
== X86EMUL_PROPAGATE_FAULT
) {
5104 WARN_ON(ctxt
->exception
.vector
> 0x1f);
5105 ctxt
->have_exception
= true;
5107 if (rc
== X86EMUL_INTERCEPTED
)
5108 return EMULATION_INTERCEPTED
;
5110 if (rc
== X86EMUL_CONTINUE
)
5111 writeback_registers(ctxt
);
5113 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
5117 case 0x09: /* wbinvd */
5118 (ctxt
->ops
->wbinvd
)(ctxt
);
5120 case 0x08: /* invd */
5121 case 0x0d: /* GrpP (prefetch) */
5122 case 0x18: /* Grp16 (prefetch/nop) */
5123 case 0x1f: /* nop */
5125 case 0x20: /* mov cr, reg */
5126 ctxt
->dst
.val
= ops
->get_cr(ctxt
, ctxt
->modrm_reg
);
5128 case 0x21: /* mov from dr to reg */
5129 ops
->get_dr(ctxt
, ctxt
->modrm_reg
, &ctxt
->dst
.val
);
5131 case 0x40 ... 0x4f: /* cmov */
5132 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5133 ctxt
->dst
.val
= ctxt
->src
.val
;
5134 else if (ctxt
->mode
!= X86EMUL_MODE_PROT64
||
5135 ctxt
->op_bytes
!= 4)
5136 ctxt
->dst
.type
= OP_NONE
; /* no writeback */
5138 case 0x80 ... 0x8f: /* jnz rel, etc*/
5139 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5140 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5142 case 0x90 ... 0x9f: /* setcc r/m8 */
5143 ctxt
->dst
.val
= test_cc(ctxt
->b
, ctxt
->eflags
);
5145 case 0xb6 ... 0xb7: /* movzx */
5146 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5147 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (u8
) ctxt
->src
.val
5148 : (u16
) ctxt
->src
.val
;
5150 case 0xbe ... 0xbf: /* movsx */
5151 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5152 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (s8
) ctxt
->src
.val
:
5153 (s16
) ctxt
->src
.val
;
5156 goto cannot_emulate
;
5161 if (rc
!= X86EMUL_CONTINUE
)
5167 return EMULATION_FAILED
;
5170 void emulator_invalidate_register_cache(struct x86_emulate_ctxt
*ctxt
)
5172 invalidate_registers(ctxt
);
5175 void emulator_writeback_register_cache(struct x86_emulate_ctxt
*ctxt
)
5177 writeback_registers(ctxt
);