]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/x86/kvm/vmx/vmx.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: x86: handle wrap around 32-bit address space
[mirror_ubuntu-hirsute-kernel.git] / arch / x86 / kvm / vmx / vmx.c
CommitLineData
20c8ccb1 1// SPDX-License-Identifier: GPL-2.0-only
6aa8b732
AK		 2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
7 *
8 * Copyright (C) 2006 Qumranet, Inc.
9611c187 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
6aa8b732
AK		 10 *
11 * Authors:
12 * Avi Kivity <avi@qumranet.com>
13 * Yaniv Kamay <yaniv@qumranet.com>
6aa8b732
AK		 14 */
15
199b118a
SC		 16#include <linux/frame.h>
17#include <linux/highmem.h>
18#include <linux/hrtimer.h>
19#include <linux/kernel.h>
edf88417 20#include <linux/kvm_host.h>
6aa8b732 21#include <linux/module.h>
c7addb90 22#include <linux/moduleparam.h>
e9bda3b3 23#include <linux/mod_devicetable.h>
199b118a 24#include <linux/mm.h>
199b118a 25#include <linux/sched.h>
b284909a 26#include <linux/sched/smt.h>
5a0e3ad6 27#include <linux/slab.h>
cafd6659 28#include <linux/tboot.h>
199b118a 29#include <linux/trace_events.h>
e495606d 30
199b118a 31#include <asm/apic.h>
fd8ca6da 32#include <asm/asm.h>
28b835d6 33#include <asm/cpu.h>
ba5bade4 34#include <asm/cpu_device_id.h>
199b118a 35#include <asm/debugreg.h>
3b3be0d1 36#include <asm/desc.h>
952f07ec 37#include <asm/fpu/internal.h>
199b118a 38#include <asm/io.h>
efc64404 39#include <asm/irq_remapping.h>
199b118a
SC		 40#include <asm/kexec.h>
41#include <asm/perf_event.h>
42#include <asm/mce.h>
d6e41f11 43#include <asm/mmu_context.h>
773e8a04 44#include <asm/mshyperv.h>
b10c307f 45#include <asm/mwait.h>
199b118a
SC		 46#include <asm/spec-ctrl.h>
47#include <asm/virtext.h>
48#include <asm/vmx.h>
6aa8b732 49
3077c191 50#include "capabilities.h"
199b118a 51#include "cpuid.h"
4cebd747 52#include "evmcs.h"
199b118a
SC		 53#include "irq.h"
54#include "kvm_cache_regs.h"
55#include "lapic.h"
56#include "mmu.h"
55d2375e 57#include "nested.h"
89b0c9f5 58#include "ops.h"
25462f7f 59#include "pmu.h"
199b118a 60#include "trace.h"
cb1d474b 61#include "vmcs.h"
609363cf 62#include "vmcs12.h"
89b0c9f5 63#include "vmx.h"
199b118a 64#include "x86.h"
229456fc 65
6aa8b732
AK		 66MODULE_AUTHOR("Qumranet");
67MODULE_LICENSE("GPL");
68
575b255c 69#ifdef MODULE
e9bda3b3 70static const struct x86_cpu_id vmx_cpu_id[] = {
320debe5 71 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL),
e9bda3b3
JT		 72 {}
73};
74MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
575b255c 75#endif
e9bda3b3 76
2c4fd91d 77bool __read_mostly enable_vpid = 1;
736caefe 78module_param_named(vpid, enable_vpid, bool, 0444);
2384d2b3 79
d02fcf50
PB		 80static bool __read_mostly enable_vnmi = 1;
81module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
82
2c4fd91d 83bool __read_mostly flexpriority_enabled = 1;
736caefe 84module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
4c9fc8ef 85
2c4fd91d 86bool __read_mostly enable_ept = 1;
736caefe 87module_param_named(ept, enable_ept, bool, S_IRUGO);
d56f546d 88
2c4fd91d 89bool __read_mostly enable_unrestricted_guest = 1;
3a624e29
NK		 90module_param_named(unrestricted_guest,
91 enable_unrestricted_guest, bool, S_IRUGO);
92
2c4fd91d 93bool __read_mostly enable_ept_ad_bits = 1;
83c3a331
XH		 94module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
95
a27685c3 96static bool __read_mostly emulate_invalid_guest_state = true;
c1f8bc04 97module_param(emulate_invalid_guest_state, bool, S_IRUGO);
04fa4d32 98
476bc001 99static bool __read_mostly fasteoi = 1;
58fbbf26
KT		 100module_param(fasteoi, bool, S_IRUGO);
101
a4443267 102bool __read_mostly enable_apicv = 1;
01e439be 103module_param(enable_apicv, bool, S_IRUGO);
83d4c286 104
801d3424
NHE		 105/*
106 * If nested=1, nested virtualization is supported, i.e., guests may use
107 * VMX and be a hypervisor for its own guests. If nested=0, guests may not
108 * use VMX instructions.
109 */
1e58e5e5 110static bool __read_mostly nested = 1;
801d3424
NHE		 111module_param(nested, bool, S_IRUGO);
112
2c4fd91d 113bool __read_mostly enable_pml = 1;
843e4330
KH		 114module_param_named(pml, enable_pml, bool, S_IRUGO);
115
6f2f8453
PB		 116static bool __read_mostly dump_invalid_vmcs = 0;
117module_param(dump_invalid_vmcs, bool, 0644);
118
904e14fb
PB		 119#define MSR_BITMAP_MODE_X2APIC 1
120#define MSR_BITMAP_MODE_X2APIC_APICV 2
904e14fb 121
64903d61
HZ		 122#define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL
123
64672c95
YJ		 124/* Guest_tsc -> host_tsc conversion requires 64-bit division. */
125static int __read_mostly cpu_preemption_timer_multi;
126static bool __read_mostly enable_preemption_timer = 1;
127#ifdef CONFIG_X86_64
128module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
129#endif
130
3de6347b 131#define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
1706bd0c
SC		 132#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
133#define KVM_VM_CR0_ALWAYS_ON \
134 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | \
135 X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
4c38609a
AK		 136#define KVM_CR4_GUEST_OWNED_BITS \
137 (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
fd8cb433 138 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
4c38609a 139
5dc1f044 140#define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
cdc0e244
AK		 141#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
142#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
143
78ac8b47
AK		 144#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
145
bf8c55d8
CP		 146#define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \
147 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \
148 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \
149 RTIT_STATUS_BYTECNT))
150
151#define MSR_IA32_RTIT_OUTPUT_BASE_MASK \
152 (~((1UL << cpuid_query_maxphyaddr(vcpu)) - 1) | 0x7f)
153
4b8d54f9
ZE		 154/*
155 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
156 * ple_gap: upper bound on the amount of time between two successive
157 * executions of PAUSE in a loop. Also indicate if ple enabled.
00c25bce 158 * According to test, this time is usually smaller than 128 cycles.
4b8d54f9
ZE		 159 * ple_window: upper bound on the amount of time a guest is allowed to execute
160 * in a PAUSE loop. Tests indicate that most spinlocks are held for
161 * less than 2^12 cycles
162 * Time is measured based on a counter that runs at the same rate as the TSC,
163 * refer SDM volume 3b section 21.6.13 & 22.1.3.
164 */
c8e88717 165static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
a87c99e6 166module_param(ple_gap, uint, 0444);
b4a2d31d 167
7fbc85a5
BM		 168static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
169module_param(ple_window, uint, 0444);
4b8d54f9 170
b4a2d31d 171/* Default doubles per-vcpu window every exit. */
c8e88717 172static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
7fbc85a5 173module_param(ple_window_grow, uint, 0444);
b4a2d31d
RK		 174
175/* Default resets per-vcpu window every exit to ple_window. */
c8e88717 176static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
7fbc85a5 177module_param(ple_window_shrink, uint, 0444);
b4a2d31d
RK		 178
179/* Default is to compute the maximum so we can never overflow. */
7fbc85a5
BM		 180static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
181module_param(ple_window_max, uint, 0444);
b4a2d31d 182
f99e3daf
CP		 183/* Default is SYSTEM mode, 1 for host-guest mode */
184int __read_mostly pt_mode = PT_MODE_SYSTEM;
185module_param(pt_mode, int, S_IRUGO);
186
a399477e 187static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
427362a1 188static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
dd4bfa73 189static DEFINE_MUTEX(vmx_l1d_flush_mutex);
a399477e 190
7db92e16
TG		 191/* Storage for pre module init parameter parsing */
192static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
a399477e
KRW		 193
194static const struct {
195 const char *option;
0027ff2a 196 bool for_parse;
a399477e 197} vmentry_l1d_param[] = {
0027ff2a
PB		 198 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true},
199 [VMENTER_L1D_FLUSH_NEVER] = {"never", true},
200 [VMENTER_L1D_FLUSH_COND] = {"cond", true},
201 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true},
202 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
203 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
a399477e
KRW		 204};
205
7db92e16
TG		 206#define L1D_CACHE_ORDER 4
207static void *vmx_l1d_flush_pages;
208
209static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
a399477e 210{
7db92e16 211 struct page *page;
288d152c 212 unsigned int i;
a399477e 213
19a36d32
WL		 214 if (!boot_cpu_has_bug(X86_BUG_L1TF)) {
215 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
216 return 0;
217 }
218
7db92e16
TG		 219 if (!enable_ept) {
220 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
221 return 0;
a399477e
KRW		 222 }
223
d806afa4
YW		 224 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
225 u64 msr;
226
227 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
228 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
229 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
230 return 0;
231 }
232 }
8e0b2b91 233
d90a7a0e
JK		 234 /* If set to auto use the default l1tf mitigation method */
235 if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
236 switch (l1tf_mitigation) {
237 case L1TF_MITIGATION_OFF:
238 l1tf = VMENTER_L1D_FLUSH_NEVER;
239 break;
240 case L1TF_MITIGATION_FLUSH_NOWARN:
241 case L1TF_MITIGATION_FLUSH:
242 case L1TF_MITIGATION_FLUSH_NOSMT:
243 l1tf = VMENTER_L1D_FLUSH_COND;
244 break;
245 case L1TF_MITIGATION_FULL:
246 case L1TF_MITIGATION_FULL_FORCE:
247 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
248 break;
249 }
250 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
251 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
252 }
253
7db92e16
TG		 254 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
255 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
41836839
BG		 256 /*
257 * This allocation for vmx_l1d_flush_pages is not tied to a VM
258 * lifetime and so should not be charged to a memcg.
259 */
7db92e16
TG		 260 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
261 if (!page)
262 return -ENOMEM;
263 vmx_l1d_flush_pages = page_address(page);
288d152c
NS		 264
265 /*
266 * Initialize each page with a different pattern in
267 * order to protect against KSM in the nested
268 * virtualization case.
269 */
270 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
271 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
272 PAGE_SIZE);
273 }
7db92e16
TG		 274 }
275
276 l1tf_vmx_mitigation = l1tf;
277
895ae47f
TG		 278 if (l1tf != VMENTER_L1D_FLUSH_NEVER)
279 static_branch_enable(&vmx_l1d_should_flush);
280 else
281 static_branch_disable(&vmx_l1d_should_flush);
4c6523ec 282
427362a1
NS		 283 if (l1tf == VMENTER_L1D_FLUSH_COND)
284 static_branch_enable(&vmx_l1d_flush_cond);
895ae47f 285 else
427362a1 286 static_branch_disable(&vmx_l1d_flush_cond);
7db92e16
TG		 287 return 0;
288}
289
290static int vmentry_l1d_flush_parse(const char *s)
291{
292 unsigned int i;
293
294 if (s) {
295 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
0027ff2a
PB		 296 if (vmentry_l1d_param[i].for_parse &&
297 sysfs_streq(s, vmentry_l1d_param[i].option))
298 return i;
7db92e16
TG		 299 }
300 }
a399477e
KRW		 301 return -EINVAL;
302}
303
7db92e16
TG		 304static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
305{
dd4bfa73 306 int l1tf, ret;
7db92e16 307
7db92e16
TG		 308 l1tf = vmentry_l1d_flush_parse(s);
309 if (l1tf < 0)
310 return l1tf;
311
0027ff2a
PB		 312 if (!boot_cpu_has(X86_BUG_L1TF))
313 return 0;
314
7db92e16
TG		 315 /*
316 * Has vmx_init() run already? If not then this is the pre init
317 * parameter parsing. In that case just store the value and let
318 * vmx_init() do the proper setup after enable_ept has been
319 * established.
320 */
321 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
322 vmentry_l1d_flush_param = l1tf;
323 return 0;
324 }
325
dd4bfa73
TG		 326 mutex_lock(&vmx_l1d_flush_mutex);
327 ret = vmx_setup_l1d_flush(l1tf);
328 mutex_unlock(&vmx_l1d_flush_mutex);
329 return ret;
7db92e16
TG		 330}
331
a399477e
KRW		 332static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
333{
0027ff2a
PB		 334 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
335 return sprintf(s, "???\n");
336
7db92e16 337 return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
a399477e
KRW		 338}
339
340static const struct kernel_param_ops vmentry_l1d_flush_ops = {
341 .set = vmentry_l1d_flush_set,
342 .get = vmentry_l1d_flush_get,
343};
895ae47f 344module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
a399477e 345
d99e4152
GN		 346static bool guest_state_valid(struct kvm_vcpu *vcpu);
347static u32 vmx_segment_access_rights(struct kvm_segment *var);
1e4329ee 348static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
15d45071 349 u32 msr, int type);
75880a01 350
453eafbe
SC		 351void vmx_vmexit(void);
352
52a9fcbc
SC		 353#define vmx_insn_failed(fmt...) \
354do { \
355 WARN_ONCE(1, fmt); \
356 pr_warn_ratelimited(fmt); \
357} while (0)
358
6e202097
SC		 359asmlinkage void vmread_error(unsigned long field, bool fault)
360{
361 if (fault)
362 kvm_spurious_fault();
363 else
364 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field);
365}
366
52a9fcbc
SC		 367noinline void vmwrite_error(unsigned long field, unsigned long value)
368{
369 vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n",
370 field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
371}
372
373noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)
374{
375 vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr);
376}
377
378noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)
379{
380 vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr);
381}
382
383noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva)
384{
385 vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n",
386 ext, vpid, gva);
387}
388
389noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa)
390{
391 vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n",
392 ext, eptp, gpa);
393}
394
6aa8b732 395static DEFINE_PER_CPU(struct vmcs *, vmxarea);
75edce8a 396DEFINE_PER_CPU(struct vmcs *, current_vmcs);
d462b819
NHE		 397/*
398 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
399 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
400 */
401static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
6aa8b732 402
bf9f6ac8
FW		 403/*
404 * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
405 * can find which vCPU should be waken up.
406 */
407static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
408static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
409
2384d2b3
SY		 410static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
411static DEFINE_SPINLOCK(vmx_vpid_lock);
412
3077c191
SC		 413struct vmcs_config vmcs_config;
414struct vmx_capability vmx_capability;
d56f546d 415
6aa8b732
AK		 416#define VMX_SEGMENT_FIELD(seg) \
417 [VCPU_SREG_##seg] = { \
418 .selector = GUEST_##seg##_SELECTOR, \
419 .base = GUEST_##seg##_BASE, \
420 .limit = GUEST_##seg##_LIMIT, \
421 .ar_bytes = GUEST_##seg##_AR_BYTES, \
422 }
423
772e0318 424static const struct kvm_vmx_segment_field {
6aa8b732
AK		 425 unsigned selector;
426 unsigned base;
427 unsigned limit;
428 unsigned ar_bytes;
429} kvm_vmx_segment_fields[] = {
430 VMX_SEGMENT_FIELD(CS),
431 VMX_SEGMENT_FIELD(DS),
432 VMX_SEGMENT_FIELD(ES),
433 VMX_SEGMENT_FIELD(FS),
434 VMX_SEGMENT_FIELD(GS),
435 VMX_SEGMENT_FIELD(SS),
436 VMX_SEGMENT_FIELD(TR),
437 VMX_SEGMENT_FIELD(LDTR),
438};
439
ec0241f3
SC		 440static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
441{
442 vmx->segment_cache.bitmask = 0;
443}
444
2342080c 445static unsigned long host_idt_base;
26bb0981 446
4d56c8a7 447/*
898a811f
JM		 448 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm
449 * will emulate SYSCALL in legacy mode if the vendor string in guest
450 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To
451 * support this emulation, IA32_STAR must always be included in
452 * vmx_msr_index[], even in i386 builds.
4d56c8a7 453 */
cf3646eb 454const u32 vmx_msr_index[] = {
05b3e0c2 455#ifdef CONFIG_X86_64
44ea2b17 456 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
6aa8b732 457#endif
8c06585d 458 MSR_EFER, MSR_TSC_AUX, MSR_STAR,
c11f83e0 459 MSR_IA32_TSX_CTRL,
6aa8b732 460};
6aa8b732 461
773e8a04
VK		 462#if IS_ENABLED(CONFIG_HYPERV)
463static bool __read_mostly enlightened_vmcs = true;
464module_param(enlightened_vmcs, bool, 0444);
465
877ad952
TL		 466/* check_ept_pointer() should be under protection of ept_pointer_lock. */
467static void check_ept_pointer_match(struct kvm *kvm)
468{
469 struct kvm_vcpu *vcpu;
470 u64 tmp_eptp = INVALID_PAGE;
471 int i;
472
473 kvm_for_each_vcpu(i, vcpu, kvm) {
474 if (!VALID_PAGE(tmp_eptp)) {
475 tmp_eptp = to_vmx(vcpu)->ept_pointer;
476 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
477 to_kvm_vmx(kvm)->ept_pointers_match
478 = EPT_POINTERS_MISMATCH;
479 return;
480 }
481 }
482
483 to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
484}
485
8997f657 486static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush,
1f3a3e46
LT		 487 void *data)
488{
489 struct kvm_tlb_range *range = data;
490
491 return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
492 range->pages);
493}
494
495static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
496 struct kvm_vcpu *vcpu, struct kvm_tlb_range *range)
497{
498 u64 ept_pointer = to_vmx(vcpu)->ept_pointer;
499
500 /*
501 * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs address
502 * of the base of EPT PML4 table, strip off EPT configuration
503 * information.
504 */
505 if (range)
506 return hyperv_flush_guest_mapping_range(ept_pointer & PAGE_MASK,
507 kvm_fill_hv_flush_list_func, (void *)range);
508 else
509 return hyperv_flush_guest_mapping(ept_pointer & PAGE_MASK);
510}
511
512static int hv_remote_flush_tlb_with_range(struct kvm *kvm,
513 struct kvm_tlb_range *range)
877ad952 514{
a5c214da 515 struct kvm_vcpu *vcpu;
b7c1c226 516 int ret = 0, i;
877ad952
TL		 517
518 spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
519
520 if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
521 check_ept_pointer_match(kvm);
522
523 if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
53963a70 524 kvm_for_each_vcpu(i, vcpu, kvm) {
1f3a3e46
LT		 525 /* If ept_pointer is invalid pointer, bypass flush request. */
526 if (VALID_PAGE(to_vmx(vcpu)->ept_pointer))
527 ret |= __hv_remote_flush_tlb_with_range(
528 kvm, vcpu, range);
53963a70 529 }
a5c214da 530 } else {
1f3a3e46
LT		 531 ret = __hv_remote_flush_tlb_with_range(kvm,
532 kvm_get_vcpu(kvm, 0), range);
877ad952 533 }
877ad952 534
877ad952
TL		 535 spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
536 return ret;
537}
1f3a3e46
LT		 538static int hv_remote_flush_tlb(struct kvm *kvm)
539{
540 return hv_remote_flush_tlb_with_range(kvm, NULL);
541}
542
6f6a657c
VK		 543static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu)
544{
545 struct hv_enlightened_vmcs *evmcs;
546 struct hv_partition_assist_pg **p_hv_pa_pg =
547 &vcpu->kvm->arch.hyperv.hv_pa_pg;
548 /*
549 * Synthetic VM-Exit is not enabled in current code and so All
550 * evmcs in singe VM shares same assist page.
551 */
cab01850 552 if (!*p_hv_pa_pg)
6f6a657c 553 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL);
cab01850
VK		 554
555 if (!*p_hv_pa_pg)
556 return -ENOMEM;
6f6a657c
VK		 557
558 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;
559
560 evmcs->partition_assist_page =
561 __pa(*p_hv_pa_pg);
cab01850 562 evmcs->hv_vm_id = (unsigned long)vcpu->kvm;
6f6a657c
VK		 563 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1;
564
6f6a657c
VK		 565 return 0;
566}
567
773e8a04
VK		 568#endif /* IS_ENABLED(CONFIG_HYPERV) */
569
64672c95
YJ		 570/*
571 * Comment's format: document - errata name - stepping - processor name.
572 * Refer from
573 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
574 */
575static u32 vmx_preemption_cpu_tfms[] = {
576/* 323344.pdf - BA86 - D0 - Xeon 7500 Series */
5770x000206E6,
578/* 323056.pdf - AAX65 - C2 - Xeon L3406 */
579/* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
580/* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
5810x00020652,
582/* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
5830x00020655,
584/* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */
585/* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */
586/*
587 * 320767.pdf - AAP86 - B1 -
588 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
589 */
5900x000106E5,
591/* 321333.pdf - AAM126 - C0 - Xeon 3500 */
5920x000106A0,
593/* 321333.pdf - AAM126 - C1 - Xeon 3500 */
5940x000106A1,
595/* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
5960x000106A4,
597 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
598 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
599 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
6000x000106A5,
3d82c565
WH		 601 /* Xeon E3-1220 V2 */
6020x000306A8,
64672c95
YJ		 603};
604
605static inline bool cpu_has_broken_vmx_preemption_timer(void)
606{
607 u32 eax = cpuid_eax(0x00000001), i;
608
609 /* Clear the reserved bits */
610 eax &= ~(0x3U << 14 | 0xfU << 28);
03f6a22a 611 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
64672c95
YJ		 612 if (eax == vmx_preemption_cpu_tfms[i])
613 return true;
614
615 return false;
616}
617
35754c98 618static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
f78e0e2e 619{
35754c98 620 return flexpriority_enabled && lapic_in_kernel(vcpu);
f78e0e2e
SY		 621}
622
04547156
SY		 623static inline bool report_flexpriority(void)
624{
625 return flexpriority_enabled;
626}
627
97b7ead3 628static inline int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
7725f0ba
AK		 629{
630 int i;
631
a2fa3e9f 632 for (i = 0; i < vmx->nmsrs; ++i)
26bb0981 633 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
a75beee6
ED		 634 return i;
635 return -1;
636}
637
97b7ead3 638struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
a75beee6
ED		 639{
640 int i;
641
8b9cf98c 642 i = __find_msr_index(vmx, msr);
a75beee6 643 if (i >= 0)
a2fa3e9f 644 return &vmx->guest_msrs[i];
8b6d44c7 645 return NULL;
7725f0ba
AK		 646}
647
b07a5c53
PB		 648static int vmx_set_guest_msr(struct vcpu_vmx *vmx, struct shared_msr_entry *msr, u64 data)
649{
650 int ret = 0;
651
652 u64 old_msr_data = msr->data;
653 msr->data = data;
654 if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
655 preempt_disable();
656 ret = kvm_set_shared_msr(msr->index, msr->data,
657 msr->mask);
658 preempt_enable();
659 if (ret)
660 msr->data = old_msr_data;
661 }
662 return ret;
663}
664
2965faa5 665#ifdef CONFIG_KEXEC_CORE
8f536b76
ZY		 666static void crash_vmclear_local_loaded_vmcss(void)
667{
668 int cpu = raw_smp_processor_id();
669 struct loaded_vmcs *v;
670
8f536b76
ZY		 671 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
672 loaded_vmcss_on_cpu_link)
673 vmcs_clear(v->vmcs);
674}
2965faa5 675#endif /* CONFIG_KEXEC_CORE */
8f536b76 676
d462b819 677static void __loaded_vmcs_clear(void *arg)
6aa8b732 678{
d462b819 679 struct loaded_vmcs *loaded_vmcs = arg;
d3b2c338 680 int cpu = raw_smp_processor_id();
6aa8b732 681
d462b819
NHE		 682 if (loaded_vmcs->cpu != cpu)
683 return; /* vcpu migration can race with cpu offline */
684 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
6aa8b732 685 per_cpu(current_vmcs, cpu) = NULL;
31603d4f
SC		 686
687 vmcs_clear(loaded_vmcs->vmcs);
688 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
689 vmcs_clear(loaded_vmcs->shadow_vmcs);
690
d462b819 691 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
5a560f8b
XG		 692
693 /*
31603d4f
SC		 694 * Ensure all writes to loaded_vmcs, including deleting it from its
695 * current percpu list, complete before setting loaded_vmcs->vcpu to
696 * -1, otherwise a different cpu can see vcpu == -1 first and add
697 * loaded_vmcs to its percpu list before it's deleted from this cpu's
698 * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs().
5a560f8b
XG		 699 */
700 smp_wmb();
701
31603d4f
SC		 702 loaded_vmcs->cpu = -1;
703 loaded_vmcs->launched = 0;
6aa8b732
AK		 704}
705
89b0c9f5 706void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
8d0be2b3 707{
e6c7d321
XG		 708 int cpu = loaded_vmcs->cpu;
709
710 if (cpu != -1)
711 smp_call_function_single(cpu,
712 __loaded_vmcs_clear, loaded_vmcs, 1);
8d0be2b3
AK		 713}
714
2fb92db1
AK		 715static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
716 unsigned field)
717{
718 bool ret;
719 u32 mask = 1 << (seg * SEG_FIELD_NR + field);
720
cb3c1e2f
SC		 721 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) {
722 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS);
2fb92db1
AK		 723 vmx->segment_cache.bitmask = 0;
724 }
725 ret = vmx->segment_cache.bitmask & mask;
726 vmx->segment_cache.bitmask |= mask;
727 return ret;
728}
729
730static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
731{
732 u16 *p = &vmx->segment_cache.seg[seg].selector;
733
734 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
735 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
736 return *p;
737}
738
739static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
740{
741 ulong *p = &vmx->segment_cache.seg[seg].base;
742
743 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
744 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
745 return *p;
746}
747
748static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
749{
750 u32 *p = &vmx->segment_cache.seg[seg].limit;
751
752 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
753 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
754 return *p;
755}
756
757static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
758{
759 u32 *p = &vmx->segment_cache.seg[seg].ar;
760
761 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
762 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
763 return *p;
764}
765
97b7ead3 766void update_exception_bitmap(struct kvm_vcpu *vcpu)
abd3f2d6
AK		 767{
768 u32 eb;
769
fd7373cc 770 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
bd7e5b08 771 (1u << DB_VECTOR) | (1u << AC_VECTOR);
9e869480
LA		 772 /*
773 * Guest access to VMware backdoor ports could legitimately
774 * trigger #GP because of TSS I/O permission bitmap.
775 * We intercept those #GP and allow access to them anyway
776 * as VMware does.
777 */
778 if (enable_vmware_backdoor)
779 eb |= (1u << GP_VECTOR);
fd7373cc
JK		 780 if ((vcpu->guest_debug &
781 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
782 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
783 eb |= 1u << BP_VECTOR;
7ffd92c5 784 if (to_vmx(vcpu)->rmode.vm86_active)
abd3f2d6 785 eb = ~0;
089d034e 786 if (enable_ept)
49f933d4 787 eb &= ~(1u << PF_VECTOR);
36cf24e0
NHE		 788
789 /* When we are running a nested L2 guest and L1 specified for it a
790 * certain exception bitmap, we must trap the same exceptions and pass
791 * them to L1. When running L2, we will only handle the exceptions
792 * specified above if L1 did not want them.
793 */
794 if (is_guest_mode(vcpu))
795 eb |= get_vmcs12(vcpu)->exception_bitmap;
796
abd3f2d6
AK		 797 vmcs_write32(EXCEPTION_BITMAP, eb);
798}
799
d28b387f
KA		 800/*
801 * Check if MSR is intercepted for currently loaded MSR bitmap.
802 */
803static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
804{
805 unsigned long *msr_bitmap;
806 int f = sizeof(unsigned long);
807
808 if (!cpu_has_vmx_msr_bitmap())
809 return true;
810
811 msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
812
813 if (msr <= 0x1fff) {
814 return !!test_bit(msr, msr_bitmap + 0x800 / f);
815 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
816 msr &= 0x1fff;
817 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
818 }
819
820 return true;
821}
822
2961e876
GN		 823static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
824 unsigned long entry, unsigned long exit)
8bf00a52 825{
2961e876
GN		 826 vm_entry_controls_clearbit(vmx, entry);
827 vm_exit_controls_clearbit(vmx, exit);
8bf00a52
GN		 828}
829
662f1d1d 830int vmx_find_msr_index(struct vmx_msrs *m, u32 msr)
ca83b4a7
KRW		 831{
832 unsigned int i;
833
834 for (i = 0; i < m->nr; ++i) {
835 if (m->val[i].index == msr)
836 return i;
837 }
838 return -ENOENT;
839}
840
61d2ef2c
AK		 841static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
842{
ca83b4a7 843 int i;
61d2ef2c
AK		 844 struct msr_autoload *m = &vmx->msr_autoload;
845
8bf00a52
GN		 846 switch (msr) {
847 case MSR_EFER:
c73da3fc 848 if (cpu_has_load_ia32_efer()) {
2961e876
GN		 849 clear_atomic_switch_msr_special(vmx,
850 VM_ENTRY_LOAD_IA32_EFER,
8bf00a52
GN		 851 VM_EXIT_LOAD_IA32_EFER);
852 return;
853 }
854 break;
855 case MSR_CORE_PERF_GLOBAL_CTRL:
c73da3fc 856 if (cpu_has_load_perf_global_ctrl()) {
2961e876 857 clear_atomic_switch_msr_special(vmx,
8bf00a52
GN		 858 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
859 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
860 return;
861 }
862 break;
110312c8 863 }
ef0fbcac 864 i = vmx_find_msr_index(&m->guest, msr);
ca83b4a7 865 if (i < 0)
31907093 866 goto skip_guest;
33966dd6 867 --m->guest.nr;
33966dd6 868 m->guest.val[i] = m->guest.val[m->guest.nr];
33966dd6 869 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
110312c8 870
31907093 871skip_guest:
ef0fbcac 872 i = vmx_find_msr_index(&m->host, msr);
31907093 873 if (i < 0)
61d2ef2c 874 return;
31907093
KRW		 875
876 --m->host.nr;
877 m->host.val[i] = m->host.val[m->host.nr];
33966dd6 878 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
61d2ef2c
AK		 879}
880
2961e876
GN		 881static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
882 unsigned long entry, unsigned long exit,
883 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
884 u64 guest_val, u64 host_val)
8bf00a52
GN		 885{
886 vmcs_write64(guest_val_vmcs, guest_val);
5a5e8a15
SC		 887 if (host_val_vmcs != HOST_IA32_EFER)
888 vmcs_write64(host_val_vmcs, host_val);
2961e876
GN		 889 vm_entry_controls_setbit(vmx, entry);
890 vm_exit_controls_setbit(vmx, exit);
8bf00a52
GN		 891}
892
61d2ef2c 893static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
989e3992 894 u64 guest_val, u64 host_val, bool entry_only)
61d2ef2c 895{
989e3992 896 int i, j = 0;
61d2ef2c
AK		 897 struct msr_autoload *m = &vmx->msr_autoload;
898
8bf00a52
GN		 899 switch (msr) {
900 case MSR_EFER:
c73da3fc 901 if (cpu_has_load_ia32_efer()) {
2961e876
GN		 902 add_atomic_switch_msr_special(vmx,
903 VM_ENTRY_LOAD_IA32_EFER,
8bf00a52
GN		 904 VM_EXIT_LOAD_IA32_EFER,
905 GUEST_IA32_EFER,
906 HOST_IA32_EFER,
907 guest_val, host_val);
908 return;
909 }
910 break;
911 case MSR_CORE_PERF_GLOBAL_CTRL:
c73da3fc 912 if (cpu_has_load_perf_global_ctrl()) {
2961e876 913 add_atomic_switch_msr_special(vmx,
8bf00a52
GN		 914 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
915 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
916 GUEST_IA32_PERF_GLOBAL_CTRL,
917 HOST_IA32_PERF_GLOBAL_CTRL,
918 guest_val, host_val);
919 return;
920 }
921 break;
7099e2e1
RK		 922 case MSR_IA32_PEBS_ENABLE:
923 /* PEBS needs a quiescent period after being disabled (to write
924 * a record). Disabling PEBS through VMX MSR swapping doesn't
925 * provide that period, so a CPU could write host's record into
926 * guest's memory.
927 */
928 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
110312c8
AK		 929 }
930
ef0fbcac 931 i = vmx_find_msr_index(&m->guest, msr);
989e3992 932 if (!entry_only)
ef0fbcac 933 j = vmx_find_msr_index(&m->host, msr);
61d2ef2c 934
7cfe0526
AL		 935 if ((i < 0 && m->guest.nr == NR_LOADSTORE_MSRS) ||
936 (j < 0 && m->host.nr == NR_LOADSTORE_MSRS)) {
60266204 937 printk_once(KERN_WARNING "Not enough msr switch entries. "
e7fc6f93
GN		 938 "Can't add msr %x\n", msr);
939 return;
61d2ef2c 940 }
31907093 941 if (i < 0) {
ca83b4a7 942 i = m->guest.nr++;
33966dd6 943 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
31907093 944 }
989e3992
KRW		 945 m->guest.val[i].index = msr;
946 m->guest.val[i].value = guest_val;
947
948 if (entry_only)
949 return;
61d2ef2c 950
31907093
KRW		 951 if (j < 0) {
952 j = m->host.nr++;
33966dd6 953 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
61d2ef2c 954 }
31907093
KRW		 955 m->host.val[j].index = msr;
956 m->host.val[j].value = host_val;
61d2ef2c
AK		 957}
958
92c0d900 959static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
2cc51560 960{
844a5fe2
PB		 961 u64 guest_efer = vmx->vcpu.arch.efer;
962 u64 ignore_bits = 0;
963
9167ab79
PB		 964 /* Shadow paging assumes NX to be available. */
965 if (!enable_ept)
966 guest_efer |= EFER_NX;
3a34a881 967
51c6cf66 968 /*
844a5fe2 969 * LMA and LME handled by hardware; SCE meaningless outside long mode.
51c6cf66 970 */
844a5fe2 971 ignore_bits |= EFER_SCE;
51c6cf66
AK		 972#ifdef CONFIG_X86_64
973 ignore_bits |= EFER_LMA | EFER_LME;
974 /* SCE is meaningful only in long mode on Intel */
975 if (guest_efer & EFER_LMA)
976 ignore_bits &= ~(u64)EFER_SCE;
977#endif
84ad33ef 978
f6577a5f
AL		 979 /*
980 * On EPT, we can't emulate NX, so we must switch EFER atomically.
981 * On CPUs that support "load IA32_EFER", always switch EFER
982 * atomically, since it's faster than switching it manually.
983 */
c73da3fc 984 if (cpu_has_load_ia32_efer() ||
f6577a5f 985 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
84ad33ef
AK		 986 if (!(guest_efer & EFER_LMA))
987 guest_efer &= ~EFER_LME;
54b98bff
AL		 988 if (guest_efer != host_efer)
989 add_atomic_switch_msr(vmx, MSR_EFER,
989e3992 990 guest_efer, host_efer, false);
02343cf2
SC		 991 else
992 clear_atomic_switch_msr(vmx, MSR_EFER);
84ad33ef 993 return false;
844a5fe2 994 } else {
02343cf2
SC		 995 clear_atomic_switch_msr(vmx, MSR_EFER);
996
844a5fe2
PB		 997 guest_efer &= ~ignore_bits;
998 guest_efer |= host_efer & ignore_bits;
999
1000 vmx->guest_msrs[efer_offset].data = guest_efer;
1001 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
84ad33ef 1002
844a5fe2
PB		 1003 return true;
1004 }
51c6cf66
AK		 1005}
1006
e28baead
AL		 1007#ifdef CONFIG_X86_32
1008/*
1009 * On 32-bit kernels, VM exits still load the FS and GS bases from the
1010 * VMCS rather than the segment table. KVM uses this helper to figure
1011 * out the current bases to poke them into the VMCS before entry.
1012 */
2d49ec72
GN		 1013static unsigned long segment_base(u16 selector)
1014{
8c2e41f7 1015 struct desc_struct *table;
2d49ec72
GN		 1016 unsigned long v;
1017
8c2e41f7 1018 if (!(selector & ~SEGMENT_RPL_MASK))
2d49ec72
GN		 1019 return 0;
1020
45fc8757 1021 table = get_current_gdt_ro();
2d49ec72 1022
8c2e41f7 1023 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2d49ec72
GN		 1024 u16 ldt_selector = kvm_read_ldt();
1025
8c2e41f7 1026 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
2d49ec72
GN		 1027 return 0;
1028
8c2e41f7 1029 table = (struct desc_struct *)segment_base(ldt_selector);
2d49ec72 1030 }
8c2e41f7 1031 v = get_desc_base(&table[selector >> 3]);
2d49ec72
GN		 1032 return v;
1033}
e28baead 1034#endif
2d49ec72 1035
e348ac7c
SC		 1036static inline bool pt_can_write_msr(struct vcpu_vmx *vmx)
1037{
2ef7619d 1038 return vmx_pt_mode_is_host_guest() &&
e348ac7c
SC		 1039 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
1040}
1041
2ef444f1
CP		 1042static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
1043{
1044 u32 i;
1045
1046 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1047 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1048 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1049 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1050 for (i = 0; i < addr_range; i++) {
1051 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1052 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1053 }
1054}
1055
1056static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range)
1057{
1058 u32 i;
1059
1060 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1061 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1062 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1063 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1064 for (i = 0; i < addr_range; i++) {
1065 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1066 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1067 }
1068}
1069
1070static void pt_guest_enter(struct vcpu_vmx *vmx)
1071{
2ef7619d 1072 if (vmx_pt_mode_is_system())
2ef444f1
CP		 1073 return;
1074
2ef444f1 1075 /*
b08c2896
CP		 1076 * GUEST_IA32_RTIT_CTL is already set in the VMCS.
1077 * Save host state before VM entry.
2ef444f1 1078 */
b08c2896 1079 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
2ef444f1
CP		 1080 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1081 wrmsrl(MSR_IA32_RTIT_CTL, 0);
1082 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1083 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1084 }
1085}
1086
1087static void pt_guest_exit(struct vcpu_vmx *vmx)
1088{
2ef7619d 1089 if (vmx_pt_mode_is_system())
2ef444f1
CP		 1090 return;
1091
1092 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1093 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1094 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1095 }
1096
1097 /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */
1098 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1099}
1100
13b964a2
SC		 1101void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
1102 unsigned long fs_base, unsigned long gs_base)
1103{
1104 if (unlikely(fs_sel != host->fs_sel)) {
1105 if (!(fs_sel & 7))
1106 vmcs_write16(HOST_FS_SELECTOR, fs_sel);
1107 else
1108 vmcs_write16(HOST_FS_SELECTOR, 0);
1109 host->fs_sel = fs_sel;
1110 }
1111 if (unlikely(gs_sel != host->gs_sel)) {
1112 if (!(gs_sel & 7))
1113 vmcs_write16(HOST_GS_SELECTOR, gs_sel);
1114 else
1115 vmcs_write16(HOST_GS_SELECTOR, 0);
1116 host->gs_sel = gs_sel;
1117 }
1118 if (unlikely(fs_base != host->fs_base)) {
1119 vmcs_writel(HOST_FS_BASE, fs_base);
1120 host->fs_base = fs_base;
1121 }
1122 if (unlikely(gs_base != host->gs_base)) {
1123 vmcs_writel(HOST_GS_BASE, gs_base);
1124 host->gs_base = gs_base;
1125 }
1126}
1127
97b7ead3 1128void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
33ed6329 1129{
04d2cc77 1130 struct vcpu_vmx *vmx = to_vmx(vcpu);
d7ee039e 1131 struct vmcs_host_state *host_state;
51e8a8cc 1132#ifdef CONFIG_X86_64
35060ed6 1133 int cpu = raw_smp_processor_id();
51e8a8cc 1134#endif
e368b875
SC		 1135 unsigned long fs_base, gs_base;
1136 u16 fs_sel, gs_sel;
26bb0981 1137 int i;
04d2cc77 1138
d264ee0c
SC		 1139 vmx->req_immediate_exit = false;
1140
f48b4711
LA		 1141 /*
1142 * Note that guest MSRs to be saved/restored can also be changed
1143 * when guest state is loaded. This happens when guest transitions
1144 * to/from long-mode by setting MSR_EFER.LMA.
1145 */
b464f57e
PB		 1146 if (!vmx->guest_msrs_ready) {
1147 vmx->guest_msrs_ready = true;
f48b4711
LA		 1148 for (i = 0; i < vmx->save_nmsrs; ++i)
1149 kvm_set_shared_msr(vmx->guest_msrs[i].index,
1150 vmx->guest_msrs[i].data,
1151 vmx->guest_msrs[i].mask);
1152
1153 }
c9dfd3fb 1154
1155 if (vmx->nested.need_vmcs12_to_shadow_sync)
1156 nested_sync_vmcs12_to_shadow(vcpu);
1157
b464f57e 1158 if (vmx->guest_state_loaded)
33ed6329
AK		 1159 return;
1160
b464f57e 1161 host_state = &vmx->loaded_vmcs->host_state;
bd9966de 1162
33ed6329
AK		 1163 /*
1164 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
1165 * allow segment selectors with cpl > 0 or ti == 1.
1166 */
d7ee039e 1167 host_state->ldt_sel = kvm_read_ldt();
42b933b5
VK		 1168
1169#ifdef CONFIG_X86_64
d7ee039e
SC		 1170 savesegment(ds, host_state->ds_sel);
1171 savesegment(es, host_state->es_sel);
e368b875
SC		 1172
1173 gs_base = cpu_kernelmode_gs_base(cpu);
b062b794
VK		 1174 if (likely(is_64bit_mm(current->mm))) {
1175 save_fsgs_for_kvm();
e368b875
SC		 1176 fs_sel = current->thread.fsindex;
1177 gs_sel = current->thread.gsindex;
b062b794 1178 fs_base = current->thread.fsbase;
e368b875 1179 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
b062b794 1180 } else {
e368b875
SC		 1181 savesegment(fs, fs_sel);
1182 savesegment(gs, gs_sel);
b062b794 1183 fs_base = read_msr(MSR_FS_BASE);
e368b875 1184 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
33ed6329 1185 }
b2da15ac 1186
4679b61f 1187 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
4fde8d57 1188#else
e368b875
SC		 1189 savesegment(fs, fs_sel);
1190 savesegment(gs, gs_sel);
1191 fs_base = segment_base(fs_sel);
1192 gs_base = segment_base(gs_sel);
707c0874 1193#endif
e368b875 1194
13b964a2 1195 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
b464f57e 1196 vmx->guest_state_loaded = true;
33ed6329
AK		 1197}
1198
6d6095bd 1199static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
33ed6329 1200{
d7ee039e
SC		 1201 struct vmcs_host_state *host_state;
1202
b464f57e 1203 if (!vmx->guest_state_loaded)
33ed6329
AK		 1204 return;
1205
b464f57e 1206 host_state = &vmx->loaded_vmcs->host_state;
bd9966de 1207
e1beb1d3 1208 ++vmx->vcpu.stat.host_state_reload;
bd9966de 1209
c8770e7b 1210#ifdef CONFIG_X86_64
4679b61f 1211 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
c8770e7b 1212#endif
d7ee039e
SC		 1213 if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
1214 kvm_load_ldt(host_state->ldt_sel);
33ed6329 1215#ifdef CONFIG_X86_64
d7ee039e 1216 load_gs_index(host_state->gs_sel);
9581d442 1217#else
d7ee039e 1218 loadsegment(gs, host_state->gs_sel);
33ed6329 1219#endif
33ed6329 1220 }
d7ee039e
SC		 1221 if (host_state->fs_sel & 7)
1222 loadsegment(fs, host_state->fs_sel);
b2da15ac 1223#ifdef CONFIG_X86_64
d7ee039e
SC		 1224 if (unlikely(host_state->ds_sel | host_state->es_sel)) {
1225 loadsegment(ds, host_state->ds_sel);
1226 loadsegment(es, host_state->es_sel);
b2da15ac 1227 }
b2da15ac 1228#endif
b7ffc44d 1229 invalidate_tss_limit();
44ea2b17 1230#ifdef CONFIG_X86_64
c8770e7b 1231 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
44ea2b17 1232#endif
45fc8757 1233 load_fixmap_gdt(raw_smp_processor_id());
b464f57e
PB		 1234 vmx->guest_state_loaded = false;
1235 vmx->guest_msrs_ready = false;
33ed6329
AK		 1236}
1237
678e315e
SC		 1238#ifdef CONFIG_X86_64
1239static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
a9b21b62 1240{
4679b61f 1241 preempt_disable();
b464f57e 1242 if (vmx->guest_state_loaded)
4679b61f
PB		 1243 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1244 preempt_enable();
678e315e 1245 return vmx->msr_guest_kernel_gs_base;
a9b21b62
AK		 1246}
1247
678e315e
SC		 1248static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
1249{
4679b61f 1250 preempt_disable();
b464f57e 1251 if (vmx->guest_state_loaded)
4679b61f
PB		 1252 wrmsrl(MSR_KERNEL_GS_BASE, data);
1253 preempt_enable();
678e315e
SC		 1254 vmx->msr_guest_kernel_gs_base = data;
1255}
1256#endif
1257
28b835d6
FW		 1258static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
1259{
1260 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1261 struct pi_desc old, new;
1262 unsigned int dest;
1263
31afb2ea
PB		 1264 /*
1265 * In case of hot-plug or hot-unplug, we may have to undo
1266 * vmx_vcpu_pi_put even if there is no assigned device. And we
1267 * always keep PI.NDST up to date for simplicity: it makes the
1268 * code easier, and CPU migration is not a fast path.
1269 */
1270 if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
28b835d6
FW		 1271 return;
1272
132194ff
JM		 1273 /*
1274 * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
1275 * PI.NDST: pi_post_block is the one expected to change PID.NDST and the
1276 * wakeup handler expects the vCPU to be on the blocked_vcpu_list that
1277 * matches PI.NDST. Otherwise, a vcpu may not be able to be woken up
1278 * correctly.
1279 */
1280 if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) {
1281 pi_clear_sn(pi_desc);
1282 goto after_clear_sn;
1283 }
1284
31afb2ea 1285 /* The full case. */
28b835d6
FW		 1286 do {
1287 old.control = new.control = pi_desc->control;
1288
31afb2ea 1289 dest = cpu_physical_id(cpu);
28b835d6 1290
31afb2ea
PB		 1291 if (x2apic_enabled())
1292 new.ndst = dest;
1293 else
1294 new.ndst = (dest << 8) & 0xFF00;
28b835d6 1295
28b835d6 1296 new.sn = 0;
c0a1666b
PB		 1297 } while (cmpxchg64(&pi_desc->control, old.control,
1298 new.control) != old.control);
c112b5f5 1299
132194ff
JM		 1300after_clear_sn:
1301
c112b5f5
LK		 1302 /*
1303 * Clear SN before reading the bitmap. The VT-d firmware
1304 * writes the bitmap and reads SN atomically (5.2.3 in the
1305 * spec), so it doesn't really have a memory barrier that
1306 * pairs with this, but we cannot do that and we need one.
1307 */
1308 smp_mb__after_atomic();
1309
29881b6e 1310 if (!pi_is_pir_empty(pi_desc))
c112b5f5 1311 pi_set_on(pi_desc);
28b835d6 1312}
1be0e61c 1313
8ef863e6 1314void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu)
6aa8b732 1315{
a2fa3e9f 1316 struct vcpu_vmx *vmx = to_vmx(vcpu);
b80c76ec 1317 bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
6aa8b732 1318
b80c76ec 1319 if (!already_loaded) {
fe0e80be 1320 loaded_vmcs_clear(vmx->loaded_vmcs);
92fe13be 1321 local_irq_disable();
5a560f8b
XG		 1322
1323 /*
31603d4f
SC		 1324 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to
1325 * this cpu's percpu list, otherwise it may not yet be deleted
1326 * from its previous cpu's percpu list. Pairs with the
1327 * smb_wmb() in __loaded_vmcs_clear().
5a560f8b
XG		 1328 */
1329 smp_rmb();
1330
d462b819
NHE		 1331 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
1332 &per_cpu(loaded_vmcss_on_cpu, cpu));
92fe13be 1333 local_irq_enable();
b80c76ec
JM		 1334 }
1335
1336 if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
1337 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
1338 vmcs_load(vmx->loaded_vmcs->vmcs);
15d45071 1339 indirect_branch_prediction_barrier();
b80c76ec
JM		 1340 }
1341
1342 if (!already_loaded) {
59c58ceb 1343 void *gdt = get_current_gdt_ro();
b80c76ec
JM		 1344 unsigned long sysenter_esp;
1345
eeeb4f67
SC		 1346 /*
1347 * Flush all EPTP/VPID contexts, the new pCPU may have stale
1348 * TLB entries from its previous association with the vCPU.
1349 */
b80c76ec 1350 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
92fe13be 1351
6aa8b732
AK		 1352 /*
1353 * Linux uses per-cpu TSS and GDT, so set these when switching
e0c23063 1354 * processors. See 22.2.4.
6aa8b732 1355 */
e0c23063 1356 vmcs_writel(HOST_TR_BASE,
72f5e08d 1357 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
59c58ceb 1358 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */
6aa8b732
AK		 1359
1360 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
1361 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
ff2c3a18 1362
d462b819 1363 vmx->loaded_vmcs->cpu = cpu;
6aa8b732 1364 }
28b835d6 1365
2680d6da
OH		 1366 /* Setup TSC multiplier */
1367 if (kvm_has_tsc_control &&
c95ba92a
PF		 1368 vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
1369 decache_tsc_multiplier(vmx);
8ef863e6
SC		 1370}
1371
1372/*
1373 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
1374 * vcpu mutex is already taken.
1375 */
1376void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1377{
1378 struct vcpu_vmx *vmx = to_vmx(vcpu);
1379
1380 vmx_vcpu_load_vmcs(vcpu, cpu);
2680d6da 1381
28b835d6 1382 vmx_vcpu_pi_load(vcpu, cpu);
8ef863e6 1383
74c55931 1384 vmx->host_debugctlmsr = get_debugctlmsr();
28b835d6
FW		 1385}
1386
1387static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
1388{
1389 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1390
1391 if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
a0052191
YZ		 1392 !irq_remapping_cap(IRQ_POSTING_CAP) ||
1393 !kvm_vcpu_apicv_active(vcpu))
28b835d6
FW		 1394 return;
1395
1396 /* Set SN when the vCPU is preempted */
1397 if (vcpu->preempted)
1398 pi_set_sn(pi_desc);
6aa8b732
AK		 1399}
1400
13b964a2 1401static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
6aa8b732 1402{
28b835d6
FW		 1403 vmx_vcpu_pi_put(vcpu);
1404
6d6095bd 1405 vmx_prepare_switch_to_host(to_vmx(vcpu));
6aa8b732
AK		 1406}
1407
f244deed
WL		 1408static bool emulation_required(struct kvm_vcpu *vcpu)
1409{
1410 return emulate_invalid_guest_state && !guest_state_valid(vcpu);
1411}
1412
97b7ead3 1413unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
6aa8b732 1414{
e7bddc52 1415 struct vcpu_vmx *vmx = to_vmx(vcpu);
78ac8b47 1416 unsigned long rflags, save_rflags;
345dcaa8 1417
cb3c1e2f
SC		 1418 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) {
1419 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
6de12732 1420 rflags = vmcs_readl(GUEST_RFLAGS);
e7bddc52 1421 if (vmx->rmode.vm86_active) {
6de12732 1422 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
e7bddc52 1423 save_rflags = vmx->rmode.save_rflags;
6de12732
AK		 1424 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
1425 }
e7bddc52 1426 vmx->rflags = rflags;
78ac8b47 1427 }
e7bddc52 1428 return vmx->rflags;
6aa8b732
AK		 1429}
1430
97b7ead3 1431void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
6aa8b732 1432{
e7bddc52 1433 struct vcpu_vmx *vmx = to_vmx(vcpu);
491c1ad1 1434 unsigned long old_rflags;
f244deed 1435
491c1ad1 1436 if (enable_unrestricted_guest) {
cb3c1e2f 1437 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
491c1ad1
SC		 1438 vmx->rflags = rflags;
1439 vmcs_writel(GUEST_RFLAGS, rflags);
1440 return;
1441 }
1442
1443 old_rflags = vmx_get_rflags(vcpu);
e7bddc52
SC		 1444 vmx->rflags = rflags;
1445 if (vmx->rmode.vm86_active) {
1446 vmx->rmode.save_rflags = rflags;
053de044 1447 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
78ac8b47 1448 }
6aa8b732 1449 vmcs_writel(GUEST_RFLAGS, rflags);
f244deed 1450
e7bddc52
SC		 1451 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM)
1452 vmx->emulation_required = emulation_required(vcpu);
6aa8b732
AK		 1453}
1454
97b7ead3 1455u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
2809f5d2
GC		 1456{
1457 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1458 int ret = 0;
1459
1460 if (interruptibility & GUEST_INTR_STATE_STI)
48005f64 1461 ret |= KVM_X86_SHADOW_INT_STI;
2809f5d2 1462 if (interruptibility & GUEST_INTR_STATE_MOV_SS)
48005f64 1463 ret |= KVM_X86_SHADOW_INT_MOV_SS;
2809f5d2 1464
37ccdcbe 1465 return ret;
2809f5d2
GC		 1466}
1467
97b7ead3 1468void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
2809f5d2
GC		 1469{
1470 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1471 u32 interruptibility = interruptibility_old;
1472
1473 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
1474
48005f64 1475 if (mask & KVM_X86_SHADOW_INT_MOV_SS)
2809f5d2 1476 interruptibility |= GUEST_INTR_STATE_MOV_SS;
48005f64 1477 else if (mask & KVM_X86_SHADOW_INT_STI)
2809f5d2
GC		 1478 interruptibility |= GUEST_INTR_STATE_STI;
1479
1480 if ((interruptibility != interruptibility_old))
1481 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
1482}
1483
bf8c55d8
CP		 1484static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
1485{
1486 struct vcpu_vmx *vmx = to_vmx(vcpu);
1487 unsigned long value;
1488
1489 /*
1490 * Any MSR write that attempts to change bits marked reserved will
1491 * case a #GP fault.
1492 */
1493 if (data & vmx->pt_desc.ctl_bitmask)
1494 return 1;
1495
1496 /*
1497 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will
1498 * result in a #GP unless the same write also clears TraceEn.
1499 */
1500 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) &&
1501 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN))
1502 return 1;
1503
1504 /*
1505 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit
1506 * and FabricEn would cause #GP, if
1507 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0
1508 */
1509 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) &&
1510 !(data & RTIT_CTL_FABRIC_EN) &&
1511 !intel_pt_validate_cap(vmx->pt_desc.caps,
1512 PT_CAP_single_range_output))
1513 return 1;
1514
1515 /*
1516 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that
1517 * utilize encodings marked reserved will casue a #GP fault.
1518 */
1519 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods);
1520 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) &&
1521 !test_bit((data & RTIT_CTL_MTC_RANGE) >>
1522 RTIT_CTL_MTC_RANGE_OFFSET, &value))
1523 return 1;
1524 value = intel_pt_validate_cap(vmx->pt_desc.caps,
1525 PT_CAP_cycle_thresholds);
1526 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1527 !test_bit((data & RTIT_CTL_CYC_THRESH) >>
1528 RTIT_CTL_CYC_THRESH_OFFSET, &value))
1529 return 1;
1530 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods);
1531 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1532 !test_bit((data & RTIT_CTL_PSB_FREQ) >>
1533 RTIT_CTL_PSB_FREQ_OFFSET, &value))
1534 return 1;
1535
1536 /*
1537 * If ADDRx_CFG is reserved or the encodings is >2 will
1538 * cause a #GP fault.
1539 */
1540 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET;
1541 if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2))
1542 return 1;
1543 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET;
1544 if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2))
1545 return 1;
1546 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET;
1547 if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2))
1548 return 1;
1549 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET;
1550 if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2))
1551 return 1;
1552
1553 return 0;
1554}
1555
1957aa63 1556static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
6aa8b732 1557{
fede8076 1558 unsigned long rip, orig_rip;
6aa8b732 1559
1957aa63
SC		 1560 /*
1561 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on
1562 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be
1563 * set when EPT misconfig occurs. In practice, real hardware updates
1564 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors
1565 * (namely Hyper-V) don't set it due to it being undefined behavior,
1566 * i.e. we end up advancing IP with some random value.
1567 */
1568 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
1569 to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) {
fede8076
PB		 1570 orig_rip = kvm_rip_read(vcpu);
1571 rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1572#ifdef CONFIG_X86_64
1573 /*
1574 * We need to mask out the high 32 bits of RIP if not in 64-bit
1575 * mode, but just finding out that we are in 64-bit mode is
1576 * quite expensive. Only do it if there was a carry.
1577 */
1578 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu))
1579 rip = (u32)rip;
1580#endif
1957aa63
SC		 1581 kvm_rip_write(vcpu, rip);
1582 } else {
1583 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
1584 return 0;
1585 }
6aa8b732 1586
2809f5d2
GC		 1587 /* skipping an emulated instruction also counts */
1588 vmx_set_interrupt_shadow(vcpu, 0);
f8ea7c60 1589
60fc3d02 1590 return 1;
f8ea7c60
VK		 1591}
1592
5ef8acbd
OU		 1593
1594/*
1595 * Recognizes a pending MTF VM-exit and records the nested state for later
1596 * delivery.
1597 */
1598static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
1599{
1600 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1601 struct vcpu_vmx *vmx = to_vmx(vcpu);
1602
1603 if (!is_guest_mode(vcpu))
1604 return;
1605
1606 /*
1607 * Per the SDM, MTF takes priority over debug-trap exceptions besides
1608 * T-bit traps. As instruction emulation is completed (i.e. at the
1609 * instruction boundary), any #DB exception pending delivery must be a
1610 * debug-trap. Record the pending MTF state to be delivered in
1611 * vmx_check_nested_events().
1612 */
1613 if (nested_cpu_has_mtf(vmcs12) &&
1614 (!vcpu->arch.exception.pending ||
1615 vcpu->arch.exception.nr == DB_VECTOR))
1616 vmx->nested.mtf_pending = true;
1617 else
1618 vmx->nested.mtf_pending = false;
1619}
1620
1621static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
1622{
1623 vmx_update_emulated_instruction(vcpu);
1624 return skip_emulated_instruction(vcpu);
1625}
1626
caa057a2
WL		 1627static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
1628{
1629 /*
1630 * Ensure that we clear the HLT state in the VMCS. We don't need to
1631 * explicitly skip the instruction because if the HLT state is set,
1632 * then the instruction is already executing and RIP has already been
1633 * advanced.
1634 */
1635 if (kvm_hlt_in_guest(vcpu->kvm) &&
1636 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
1637 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
1638}
1639
cfcd20e5 1640static void vmx_queue_exception(struct kvm_vcpu *vcpu)
298101da 1641{
77ab6db0 1642 struct vcpu_vmx *vmx = to_vmx(vcpu);
cfcd20e5
WL		 1643 unsigned nr = vcpu->arch.exception.nr;
1644 bool has_error_code = vcpu->arch.exception.has_error_code;
cfcd20e5 1645 u32 error_code = vcpu->arch.exception.error_code;
8ab2d2e2 1646 u32 intr_info = nr | INTR_INFO_VALID_MASK;
77ab6db0 1647
da998b46
JM		 1648 kvm_deliver_exception_payload(vcpu);
1649
8ab2d2e2 1650 if (has_error_code) {
77ab6db0 1651 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
8ab2d2e2
JK		 1652 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
1653 }
77ab6db0 1654
7ffd92c5 1655 if (vmx->rmode.vm86_active) {
71f9833b
SH		 1656 int inc_eip = 0;
1657 if (kvm_exception_is_soft(nr))
1658 inc_eip = vcpu->arch.event_exit_inst_len;
9497e1f2 1659 kvm_inject_realmode_interrupt(vcpu, nr, inc_eip);
77ab6db0
JK		 1660 return;
1661 }
1662
add5ff7a
SC		 1663 WARN_ON_ONCE(vmx->emulation_required);
1664
66fd3f7f
GN		 1665 if (kvm_exception_is_soft(nr)) {
1666 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1667 vmx->vcpu.arch.event_exit_inst_len);
8ab2d2e2
JK		 1668 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1669 } else
1670 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1671
1672 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
caa057a2
WL		 1673
1674 vmx_clear_hlt(vcpu);
298101da
AK		 1675}
1676
a75beee6
ED		 1677/*
1678 * Swap MSR entry in host/guest MSR entry array.
1679 */
8b9cf98c 1680static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
a75beee6 1681{
26bb0981 1682 struct shared_msr_entry tmp;
a2fa3e9f
GH		 1683
1684 tmp = vmx->guest_msrs[to];
1685 vmx->guest_msrs[to] = vmx->guest_msrs[from];
1686 vmx->guest_msrs[from] = tmp;
a75beee6
ED		 1687}
1688
e38aea3e
AK		 1689/*
1690 * Set up the vmcs to automatically save and restore system
1691 * msrs. Don't touch the 64-bit msrs if the guest is in legacy
1692 * mode, as fiddling with msrs is very expensive.
1693 */
8b9cf98c 1694static void setup_msrs(struct vcpu_vmx *vmx)
e38aea3e 1695{
26bb0981 1696 int save_nmsrs, index;
e38aea3e 1697
a75beee6
ED		 1698 save_nmsrs = 0;
1699#ifdef CONFIG_X86_64
84c8c5b8
JM		 1700 /*
1701 * The SYSCALL MSRs are only needed on long mode guests, and only
1702 * when EFER.SCE is set.
1703 */
1704 if (is_long_mode(&vmx->vcpu) && (vmx->vcpu.arch.efer & EFER_SCE)) {
1705 index = __find_msr_index(vmx, MSR_STAR);
a75beee6 1706 if (index >= 0)
8b9cf98c
RR		 1707 move_msr_up(vmx, index, save_nmsrs++);
1708 index = __find_msr_index(vmx, MSR_LSTAR);
a75beee6 1709 if (index >= 0)
8b9cf98c 1710 move_msr_up(vmx, index, save_nmsrs++);
84c8c5b8
JM		 1711 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1712 if (index >= 0)
8b9cf98c 1713 move_msr_up(vmx, index, save_nmsrs++);
a75beee6
ED		 1714 }
1715#endif
92c0d900
AK		 1716 index = __find_msr_index(vmx, MSR_EFER);
1717 if (index >= 0 && update_transition_efer(vmx, index))
26bb0981 1718 move_msr_up(vmx, index, save_nmsrs++);
0023ef39
JM		 1719 index = __find_msr_index(vmx, MSR_TSC_AUX);
1720 if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
1721 move_msr_up(vmx, index, save_nmsrs++);
c11f83e0
PB		 1722 index = __find_msr_index(vmx, MSR_IA32_TSX_CTRL);
1723 if (index >= 0)
1724 move_msr_up(vmx, index, save_nmsrs++);
e38aea3e 1725
26bb0981 1726 vmx->save_nmsrs = save_nmsrs;
b464f57e 1727 vmx->guest_msrs_ready = false;
5897297b 1728
8d14695f 1729 if (cpu_has_vmx_msr_bitmap())
904e14fb 1730 vmx_update_msr_bitmap(&vmx->vcpu);
e38aea3e
AK		 1731}
1732
e79f245d 1733static u64 vmx_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
6aa8b732 1734{
e79f245d 1735 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6aa8b732 1736
e79f245d 1737 if (is_guest_mode(vcpu) &&
5e3d394f 1738 (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING))
e79f245d
KA		 1739 return vcpu->arch.tsc_offset - vmcs12->tsc_offset;
1740
1741 return vcpu->arch.tsc_offset;
6aa8b732
AK		 1742}
1743
326e7425 1744static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
6aa8b732 1745{
45c3af97
PB		 1746 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1747 u64 g_tsc_offset = 0;
1748
1749 /*
1750 * We're here if L1 chose not to trap WRMSR to TSC. According
1751 * to the spec, this should set L1's TSC; The offset that L1
1752 * set for L2 remains unchanged, and still needs to be added
1753 * to the newly set TSC to get L2's TSC.
1754 */
1755 if (is_guest_mode(vcpu) &&
5e3d394f 1756 (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING))
45c3af97 1757 g_tsc_offset = vmcs12->tsc_offset;
326e7425 1758
45c3af97
PB		 1759 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1760 vcpu->arch.tsc_offset - g_tsc_offset,
1761 offset);
1762 vmcs_write64(TSC_OFFSET, offset + g_tsc_offset);
1763 return offset + g_tsc_offset;
6aa8b732
AK		 1764}
1765
801d3424
NHE		 1766/*
1767 * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
1768 * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
1769 * all guests if the "nested" module option is off, and can also be disabled
1770 * for a single guest by disabling its VMX cpuid bit.
1771 */
7c97fcb3 1772bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
801d3424 1773{
d6321d49 1774 return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
801d3424
NHE		 1775}
1776
55d2375e
SC		 1777static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
1778 uint64_t val)
62cc6b9d 1779{
55d2375e 1780 uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
62cc6b9d 1781
55d2375e 1782 return !(val & ~valid_bits);
62cc6b9d
DM		 1783}
1784
55d2375e 1785static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
62cc6b9d 1786{
55d2375e
SC		 1787 switch (msr->index) {
1788 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1789 if (!nested)
1790 return 1;
1791 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
1792 default:
1793 return 1;
1794 }
62cc6b9d
DM		 1795}
1796
55d2375e
SC		 1797/*
1798 * Reads an msr value (of 'msr_index') into 'pdata'.
1799 * Returns 0 on success, non-0 otherwise.
1800 * Assumes vcpu_load() was already called.
1801 */
1802static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
62cc6b9d 1803{
55d2375e
SC		 1804 struct vcpu_vmx *vmx = to_vmx(vcpu);
1805 struct shared_msr_entry *msr;
bf8c55d8 1806 u32 index;
62cc6b9d 1807
55d2375e
SC		 1808 switch (msr_info->index) {
1809#ifdef CONFIG_X86_64
1810 case MSR_FS_BASE:
1811 msr_info->data = vmcs_readl(GUEST_FS_BASE);
62cc6b9d 1812 break;
55d2375e
SC		 1813 case MSR_GS_BASE:
1814 msr_info->data = vmcs_readl(GUEST_GS_BASE);
62cc6b9d 1815 break;
55d2375e
SC		 1816 case MSR_KERNEL_GS_BASE:
1817 msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
62cc6b9d 1818 break;
55d2375e
SC		 1819#endif
1820 case MSR_EFER:
1821 return kvm_get_msr_common(vcpu, msr_info);
c11f83e0
PB		 1822 case MSR_IA32_TSX_CTRL:
1823 if (!msr_info->host_initiated &&
1824 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
1825 return 1;
1826 goto find_shared_msr;
6e3ba4ab
TX		 1827 case MSR_IA32_UMWAIT_CONTROL:
1828 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
1829 return 1;
1830
1831 msr_info->data = vmx->msr_ia32_umwait_control;
1832 break;
55d2375e
SC		 1833 case MSR_IA32_SPEC_CTRL:
1834 if (!msr_info->host_initiated &&
1835 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
1836 return 1;
1837
1838 msr_info->data = to_vmx(vcpu)->spec_ctrl;
62cc6b9d 1839 break;
6aa8b732 1840 case MSR_IA32_SYSENTER_CS:
609e36d3 1841 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
6aa8b732
AK		 1842 break;
1843 case MSR_IA32_SYSENTER_EIP:
609e36d3 1844 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
6aa8b732
AK		 1845 break;
1846 case MSR_IA32_SYSENTER_ESP:
609e36d3 1847 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
6aa8b732 1848 break;
0dd376e7 1849 case MSR_IA32_BNDCFGS:
691bd434 1850 if (!kvm_mpx_supported() ||
d6321d49
RK		 1851 (!msr_info->host_initiated &&
1852 !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
93c4adc7 1853 return 1;
609e36d3 1854 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
0dd376e7 1855 break;
c45dcc71
AR		 1856 case MSR_IA32_MCG_EXT_CTL:
1857 if (!msr_info->host_initiated &&
a6cb099a 1858 !(vmx->msr_ia32_feature_control &
32ad73db 1859 FEAT_CTL_LMCE_ENABLED))
cae50139 1860 return 1;
c45dcc71
AR		 1861 msr_info->data = vcpu->arch.mcg_ext_ctl;
1862 break;
32ad73db 1863 case MSR_IA32_FEAT_CTL:
a6cb099a 1864 msr_info->data = vmx->msr_ia32_feature_control;
cae50139
JK		 1865 break;
1866 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1867 if (!nested_vmx_allowed(vcpu))
1868 return 1;
31de3d25
VK		 1869 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
1870 &msr_info->data))
1871 return 1;
1872 /*
1873 * Enlightened VMCS v1 doesn't have certain fields, but buggy
1874 * Hyper-V versions are still trying to use corresponding
1875 * features when they are exposed. Filter out the essential
1876 * minimum.
1877 */
1878 if (!msr_info->host_initiated &&
1879 vmx->nested.enlightened_vmcs_enabled)
1880 nested_evmcs_filter_control_msr(msr_info->index,
1881 &msr_info->data);
1882 break;
bf8c55d8 1883 case MSR_IA32_RTIT_CTL:
2ef7619d 1884 if (!vmx_pt_mode_is_host_guest())
bf8c55d8
CP		 1885 return 1;
1886 msr_info->data = vmx->pt_desc.guest.ctl;
1887 break;
1888 case MSR_IA32_RTIT_STATUS:
2ef7619d 1889 if (!vmx_pt_mode_is_host_guest())
bf8c55d8
CP		 1890 return 1;
1891 msr_info->data = vmx->pt_desc.guest.status;
1892 break;
1893 case MSR_IA32_RTIT_CR3_MATCH:
2ef7619d 1894 if (!vmx_pt_mode_is_host_guest() ||
bf8c55d8
CP		 1895 !intel_pt_validate_cap(vmx->pt_desc.caps,
1896 PT_CAP_cr3_filtering))
1897 return 1;
1898 msr_info->data = vmx->pt_desc.guest.cr3_match;
1899 break;
1900 case MSR_IA32_RTIT_OUTPUT_BASE:
2ef7619d 1901 if (!vmx_pt_mode_is_host_guest() ||
bf8c55d8
CP		 1902 (!intel_pt_validate_cap(vmx->pt_desc.caps,
1903 PT_CAP_topa_output) &&
1904 !intel_pt_validate_cap(vmx->pt_desc.caps,
1905 PT_CAP_single_range_output)))
1906 return 1;
1907 msr_info->data = vmx->pt_desc.guest.output_base;
1908 break;
1909 case MSR_IA32_RTIT_OUTPUT_MASK:
2ef7619d 1910 if (!vmx_pt_mode_is_host_guest() ||
bf8c55d8
CP		 1911 (!intel_pt_validate_cap(vmx->pt_desc.caps,
1912 PT_CAP_topa_output) &&
1913 !intel_pt_validate_cap(vmx->pt_desc.caps,
1914 PT_CAP_single_range_output)))
1915 return 1;
1916 msr_info->data = vmx->pt_desc.guest.output_mask;
1917 break;
1918 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
1919 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
2ef7619d 1920 if (!vmx_pt_mode_is_host_guest() ||
bf8c55d8
CP		 1921 (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
1922 PT_CAP_num_address_ranges)))
1923 return 1;
1924 if (index % 2)
1925 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2];
1926 else
1927 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
1928 break;
4e47c7a6 1929 case MSR_TSC_AUX:
d6321d49
RK		 1930 if (!msr_info->host_initiated &&
1931 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
4e47c7a6 1932 return 1;
c11f83e0 1933 goto find_shared_msr;
6aa8b732 1934 default:
c11f83e0 1935 find_shared_msr:
a6cb099a 1936 msr = find_msr_entry(vmx, msr_info->index);
3bab1f5d 1937 if (msr) {
609e36d3 1938 msr_info->data = msr->data;
3bab1f5d 1939 break;
6aa8b732 1940 }
609e36d3 1941 return kvm_get_msr_common(vcpu, msr_info);
6aa8b732
AK		 1942 }
1943
6aa8b732
AK		 1944 return 0;
1945}
1946
1947/*
311497e0 1948 * Writes msr value into the appropriate "register".
6aa8b732
AK		 1949 * Returns 0 on success, non-0 otherwise.
1950 * Assumes vcpu_load() was already called.
1951 */
8fe8ab46 1952static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
6aa8b732 1953{
a2fa3e9f 1954 struct vcpu_vmx *vmx = to_vmx(vcpu);
26bb0981 1955 struct shared_msr_entry *msr;
2cc51560 1956 int ret = 0;
8fe8ab46
WA		 1957 u32 msr_index = msr_info->index;
1958 u64 data = msr_info->data;
bf8c55d8 1959 u32 index;
2cc51560 1960
6aa8b732 1961 switch (msr_index) {
3bab1f5d 1962 case MSR_EFER:
8fe8ab46 1963 ret = kvm_set_msr_common(vcpu, msr_info);
2cc51560 1964 break;
16175a79 1965#ifdef CONFIG_X86_64
6aa8b732 1966 case MSR_FS_BASE:
2fb92db1 1967 vmx_segment_cache_clear(vmx);
6aa8b732
AK		 1968 vmcs_writel(GUEST_FS_BASE, data);
1969 break;
1970 case MSR_GS_BASE:
2fb92db1 1971 vmx_segment_cache_clear(vmx);
6aa8b732
AK		 1972 vmcs_writel(GUEST_GS_BASE, data);
1973 break;
44ea2b17 1974 case MSR_KERNEL_GS_BASE:
678e315e 1975 vmx_write_guest_kernel_gs_base(vmx, data);
44ea2b17 1976 break;
6aa8b732
AK		 1977#endif
1978 case MSR_IA32_SYSENTER_CS:
de70d279
SC		 1979 if (is_guest_mode(vcpu))
1980 get_vmcs12(vcpu)->guest_sysenter_cs = data;
6aa8b732
AK		 1981 vmcs_write32(GUEST_SYSENTER_CS, data);
1982 break;
1983 case MSR_IA32_SYSENTER_EIP:
de70d279
SC		 1984 if (is_guest_mode(vcpu))
1985 get_vmcs12(vcpu)->guest_sysenter_eip = data;
f5b42c33 1986 vmcs_writel(GUEST_SYSENTER_EIP, data);
6aa8b732
AK		 1987 break;
1988 case MSR_IA32_SYSENTER_ESP:
de70d279
SC		 1989 if (is_guest_mode(vcpu))
1990 get_vmcs12(vcpu)->guest_sysenter_esp = data;
f5b42c33 1991 vmcs_writel(GUEST_SYSENTER_ESP, data);
6aa8b732 1992 break;
699a1ac2
SC		 1993 case MSR_IA32_DEBUGCTLMSR:
1994 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
1995 VM_EXIT_SAVE_DEBUG_CONTROLS)
1996 get_vmcs12(vcpu)->guest_ia32_debugctl = data;
1997
1998 ret = kvm_set_msr_common(vcpu, msr_info);
1999 break;
2000
0dd376e7 2001 case MSR_IA32_BNDCFGS:
691bd434 2002 if (!kvm_mpx_supported() ||
d6321d49
RK		 2003 (!msr_info->host_initiated &&
2004 !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
93c4adc7 2005 return 1;
fd8cb433 2006 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
4531662d 2007 (data & MSR_IA32_BNDCFGS_RSVD))
93c4adc7 2008 return 1;
0dd376e7
LJ		 2009 vmcs_write64(GUEST_BNDCFGS, data);
2010 break;
6e3ba4ab
TX		 2011 case MSR_IA32_UMWAIT_CONTROL:
2012 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
2013 return 1;
2014
2015 /* The reserved bit 1 and non-32 bit [63:32] should be zero */
2016 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32)))
2017 return 1;
2018
2019 vmx->msr_ia32_umwait_control = data;
2020 break;
d28b387f
KA		 2021 case MSR_IA32_SPEC_CTRL:
2022 if (!msr_info->host_initiated &&
d28b387f
KA		 2023 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
2024 return 1;
2025
6441fa61 2026 if (data & ~kvm_spec_ctrl_valid_bits(vcpu))
d28b387f
KA		 2027 return 1;
2028
2029 vmx->spec_ctrl = data;
d28b387f
KA		 2030 if (!data)
2031 break;
2032
2033 /*
2034 * For non-nested:
2035 * When it's written (to non-zero) for the first time, pass
2036 * it through.
2037 *
2038 * For nested:
2039 * The handling of the MSR bitmap for L2 guests is done in
4d516fe7 2040 * nested_vmx_prepare_msr_bitmap. We should not touch the
d28b387f
KA		 2041 * vmcs02.msr_bitmap here since it gets completely overwritten
2042 * in the merging. We update the vmcs01 here for L1 as well
2043 * since it will end up touching the MSR anyway now.
2044 */
2045 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
2046 MSR_IA32_SPEC_CTRL,
2047 MSR_TYPE_RW);
2048 break;
c11f83e0
PB		 2049 case MSR_IA32_TSX_CTRL:
2050 if (!msr_info->host_initiated &&
2051 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
2052 return 1;
2053 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR))
2054 return 1;
2055 goto find_shared_msr;
15d45071
AR		 2056 case MSR_IA32_PRED_CMD:
2057 if (!msr_info->host_initiated &&
15d45071
AR		 2058 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
2059 return 1;
2060
2061 if (data & ~PRED_CMD_IBPB)
2062 return 1;
6441fa61
PB		 2063 if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL))
2064 return 1;
15d45071
AR		 2065 if (!data)
2066 break;
2067
2068 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2069
2070 /*
2071 * For non-nested:
2072 * When it's written (to non-zero) for the first time, pass
2073 * it through.
2074 *
2075 * For nested:
2076 * The handling of the MSR bitmap for L2 guests is done in
4d516fe7 2077 * nested_vmx_prepare_msr_bitmap. We should not touch the
15d45071
AR		 2078 * vmcs02.msr_bitmap here since it gets completely overwritten
2079 * in the merging.
2080 */
2081 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
2082 MSR_TYPE_W);
2083 break;
468d472f 2084 case MSR_IA32_CR_PAT:
d28f4290
SC		 2085 if (!kvm_pat_valid(data))
2086 return 1;
2087
142e4be7
SC		 2088 if (is_guest_mode(vcpu) &&
2089 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
2090 get_vmcs12(vcpu)->guest_ia32_pat = data;
2091
468d472f
SY		 2092 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2093 vmcs_write64(GUEST_IA32_PAT, data);
2094 vcpu->arch.pat = data;
2095 break;
2096 }
8fe8ab46 2097 ret = kvm_set_msr_common(vcpu, msr_info);
4e47c7a6 2098 break;
ba904635
WA		 2099 case MSR_IA32_TSC_ADJUST:
2100 ret = kvm_set_msr_common(vcpu, msr_info);
4e47c7a6 2101 break;
c45dcc71
AR		 2102 case MSR_IA32_MCG_EXT_CTL:
2103 if ((!msr_info->host_initiated &&
2104 !(to_vmx(vcpu)->msr_ia32_feature_control &
32ad73db 2105 FEAT_CTL_LMCE_ENABLED)) ||
c45dcc71
AR		 2106 (data & ~MCG_EXT_CTL_LMCE_EN))
2107 return 1;
2108 vcpu->arch.mcg_ext_ctl = data;
2109 break;
32ad73db 2110 case MSR_IA32_FEAT_CTL:
37e4c997 2111 if (!vmx_feature_control_msr_valid(vcpu, data) ||
3b84080b 2112 (to_vmx(vcpu)->msr_ia32_feature_control &
32ad73db 2113 FEAT_CTL_LOCKED && !msr_info->host_initiated))
cae50139 2114 return 1;
3b84080b 2115 vmx->msr_ia32_feature_control = data;
cae50139
JK		 2116 if (msr_info->host_initiated && data == 0)
2117 vmx_leave_nested(vcpu);
2118 break;
2119 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
62cc6b9d
DM		 2120 if (!msr_info->host_initiated)
2121 return 1; /* they are read-only */
2122 if (!nested_vmx_allowed(vcpu))
2123 return 1;
2124 return vmx_set_vmx_msr(vcpu, msr_index, data);
bf8c55d8 2125 case MSR_IA32_RTIT_CTL:
2ef7619d 2126 if (!vmx_pt_mode_is_host_guest() ||
ee85dec2
LK		 2127 vmx_rtit_ctl_check(vcpu, data) ||
2128 vmx->nested.vmxon)
bf8c55d8
CP		 2129 return 1;
2130 vmcs_write64(GUEST_IA32_RTIT_CTL, data);
2131 vmx->pt_desc.guest.ctl = data;
b08c2896 2132 pt_update_intercept_for_msr(vmx);
bf8c55d8
CP		 2133 break;
2134 case MSR_IA32_RTIT_STATUS:
e348ac7c
SC		 2135 if (!pt_can_write_msr(vmx))
2136 return 1;
2137 if (data & MSR_IA32_RTIT_STATUS_MASK)
bf8c55d8
CP		 2138 return 1;
2139 vmx->pt_desc.guest.status = data;
2140 break;
2141 case MSR_IA32_RTIT_CR3_MATCH:
e348ac7c
SC		 2142 if (!pt_can_write_msr(vmx))
2143 return 1;
2144 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2145 PT_CAP_cr3_filtering))
bf8c55d8
CP		 2146 return 1;
2147 vmx->pt_desc.guest.cr3_match = data;
2148 break;
2149 case MSR_IA32_RTIT_OUTPUT_BASE:
e348ac7c
SC		 2150 if (!pt_can_write_msr(vmx))
2151 return 1;
2152 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2153 PT_CAP_topa_output) &&
2154 !intel_pt_validate_cap(vmx->pt_desc.caps,
2155 PT_CAP_single_range_output))
2156 return 1;
2157 if (data & MSR_IA32_RTIT_OUTPUT_BASE_MASK)
bf8c55d8
CP		 2158 return 1;
2159 vmx->pt_desc.guest.output_base = data;
2160 break;
2161 case MSR_IA32_RTIT_OUTPUT_MASK:
e348ac7c
SC		 2162 if (!pt_can_write_msr(vmx))
2163 return 1;
2164 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2165 PT_CAP_topa_output) &&
2166 !intel_pt_validate_cap(vmx->pt_desc.caps,
2167 PT_CAP_single_range_output))
bf8c55d8
CP		 2168 return 1;
2169 vmx->pt_desc.guest.output_mask = data;
2170 break;
2171 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
e348ac7c
SC		 2172 if (!pt_can_write_msr(vmx))
2173 return 1;
bf8c55d8 2174 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
e348ac7c
SC		 2175 if (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
2176 PT_CAP_num_address_ranges))
bf8c55d8 2177 return 1;
fe6ed369 2178 if (is_noncanonical_address(data, vcpu))
bf8c55d8
CP		 2179 return 1;
2180 if (index % 2)
2181 vmx->pt_desc.guest.addr_b[index / 2] = data;
2182 else
2183 vmx->pt_desc.guest.addr_a[index / 2] = data;
2184 break;
4e47c7a6 2185 case MSR_TSC_AUX:
d6321d49
RK		 2186 if (!msr_info->host_initiated &&
2187 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
4e47c7a6
SY		 2188 return 1;
2189 /* Check reserved bit, higher 32 bits should be zero */
2190 if ((data >> 32) != 0)
2191 return 1;
c11f83e0
PB		 2192 goto find_shared_msr;
2193
6aa8b732 2194 default:
c11f83e0 2195 find_shared_msr:
8b9cf98c 2196 msr = find_msr_entry(vmx, msr_index);
b07a5c53
PB		 2197 if (msr)
2198 ret = vmx_set_guest_msr(vmx, msr, data);
2199 else
2200 ret = kvm_set_msr_common(vcpu, msr_info);
6aa8b732
AK		 2201 }
2202
2cc51560 2203 return ret;
6aa8b732
AK		 2204}
2205
5fdbf976 2206static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
6aa8b732 2207{
cb3c1e2f
SC		 2208 kvm_register_mark_available(vcpu, reg);
2209
5fdbf976
MT		 2210 switch (reg) {
2211 case VCPU_REGS_RSP:
2212 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
2213 break;
2214 case VCPU_REGS_RIP:
2215 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
2216 break;
6de4f3ad
AK		 2217 case VCPU_EXREG_PDPTR:
2218 if (enable_ept)
2219 ept_save_pdptrs(vcpu);
2220 break;
34059c25
SC		 2221 case VCPU_EXREG_CR3:
2222 if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu)))
2223 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
2224 break;
5fdbf976 2225 default:
34059c25 2226 WARN_ON_ONCE(1);
5fdbf976
MT		 2227 break;
2228 }
6aa8b732
AK		 2229}
2230
6aa8b732
AK		 2231static __init int cpu_has_kvm_support(void)
2232{
6210e37b 2233 return cpu_has_vmx();
6aa8b732
AK		 2234}
2235
2236static __init int vmx_disabled_by_bios(void)
2237{
a4d0b2fd
SC		 2238 return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2239 !boot_cpu_has(X86_FEATURE_VMX);
6aa8b732
AK		 2240}
2241
4f6ea0a8 2242static int kvm_cpu_vmxon(u64 vmxon_pointer)
7725b894 2243{
4f6ea0a8
SC		 2244 u64 msr;
2245
fe0e80be 2246 cr4_set_bits(X86_CR4_VMXE);
1c5ac21a
AS		 2247 intel_pt_handle_vmx(1);
2248
4f6ea0a8
SC		 2249 asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
2250 _ASM_EXTABLE(1b, %l[fault])
2251 : : [vmxon_pointer] "m"(vmxon_pointer)
2252 : : fault);
2253 return 0;
2254
2255fault:
2256 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
2257 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
2258 intel_pt_handle_vmx(0);
2259 cr4_clear_bits(X86_CR4_VMXE);
2260
2261 return -EFAULT;
7725b894
DX		 2262}
2263
13a34e06 2264static int hardware_enable(void)
6aa8b732
AK		 2265{
2266 int cpu = raw_smp_processor_id();
2267 u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
4f6ea0a8 2268 int r;
6aa8b732 2269
1e02ce4c 2270 if (cr4_read_shadow() & X86_CR4_VMXE)
10474ae8
AG		 2271 return -EBUSY;
2272
773e8a04
VK		 2273 /*
2274 * This can happen if we hot-added a CPU but failed to allocate
2275 * VP assist page for it.
2276 */
2277 if (static_branch_unlikely(&enable_evmcs) &&
2278 !hv_get_vp_assist_page(cpu))
2279 return -EFAULT;
2280
4f6ea0a8
SC		 2281 r = kvm_cpu_vmxon(phys_addr);
2282 if (r)
2283 return r;
8f536b76 2284
fdf288bf
DH		 2285 if (enable_ept)
2286 ept_sync_global();
10474ae8
AG		 2287
2288 return 0;
6aa8b732
AK		 2289}
2290
d462b819 2291static void vmclear_local_loaded_vmcss(void)
543e4243
AK		 2292{
2293 int cpu = raw_smp_processor_id();
d462b819 2294 struct loaded_vmcs *v, *n;
543e4243 2295
d462b819
NHE		 2296 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
2297 loaded_vmcss_on_cpu_link)
2298 __loaded_vmcs_clear(v);
543e4243
AK		 2299}
2300
710ff4a8
EH		 2301
2302/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2303 * tricks.
2304 */
2305static void kvm_cpu_vmxoff(void)
6aa8b732 2306{
4b1e5478 2307 asm volatile (__ex("vmxoff"));
1c5ac21a
AS		 2308
2309 intel_pt_handle_vmx(0);
fe0e80be 2310 cr4_clear_bits(X86_CR4_VMXE);
6aa8b732
AK		 2311}
2312
13a34e06 2313static void hardware_disable(void)
710ff4a8 2314{
fe0e80be
DH		 2315 vmclear_local_loaded_vmcss();
2316 kvm_cpu_vmxoff();
710ff4a8
EH		 2317}
2318
7a57c09b
SC		 2319/*
2320 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID
2321 * directly instead of going through cpu_has(), to ensure KVM is trapping
2322 * ENCLS whenever it's supported in hardware. It does not matter whether
2323 * the host OS supports or has enabled SGX.
2324 */
2325static bool cpu_has_sgx(void)
2326{
2327 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0));
2328}
2329
1c3d14fe 2330static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
d77c26fc 2331 u32 msr, u32 *result)
1c3d14fe
YS		 2332{
2333 u32 vmx_msr_low, vmx_msr_high;
2334 u32 ctl = ctl_min | ctl_opt;
2335
2336 rdmsr(msr, vmx_msr_low, vmx_msr_high);
2337
2338 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
2339 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
2340
2341 /* Ensure minimum (required) set of control bits are supported. */
2342 if (ctl_min & ~ctl)
002c7f7c 2343 return -EIO;
1c3d14fe
YS		 2344
2345 *result = ctl;
2346 return 0;
2347}
2348
7caaa711
SC		 2349static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
2350 struct vmx_capability *vmx_cap)
6aa8b732
AK		 2351{
2352 u32 vmx_msr_low, vmx_msr_high;
d56f546d 2353 u32 min, opt, min2, opt2;
1c3d14fe
YS		 2354 u32 _pin_based_exec_control = 0;
2355 u32 _cpu_based_exec_control = 0;
f78e0e2e 2356 u32 _cpu_based_2nd_exec_control = 0;
1c3d14fe
YS		 2357 u32 _vmexit_control = 0;
2358 u32 _vmentry_control = 0;
2359
1389309c 2360 memset(vmcs_conf, 0, sizeof(*vmcs_conf));
10166744 2361 min = CPU_BASED_HLT_EXITING |
1c3d14fe
YS		 2362#ifdef CONFIG_X86_64
2363 CPU_BASED_CR8_LOAD_EXITING |
2364 CPU_BASED_CR8_STORE_EXITING |
2365#endif
d56f546d
SY		 2366 CPU_BASED_CR3_LOAD_EXITING |
2367 CPU_BASED_CR3_STORE_EXITING |
8eb73e2d 2368 CPU_BASED_UNCOND_IO_EXITING |
1c3d14fe 2369 CPU_BASED_MOV_DR_EXITING |
5e3d394f 2370 CPU_BASED_USE_TSC_OFFSETTING |
4d5422ce
WL		 2371 CPU_BASED_MWAIT_EXITING |
2372 CPU_BASED_MONITOR_EXITING |
fee84b07
AK		 2373 CPU_BASED_INVLPG_EXITING |
2374 CPU_BASED_RDPMC_EXITING;
443381a8 2375
f78e0e2e 2376 opt = CPU_BASED_TPR_SHADOW |
25c5f225 2377 CPU_BASED_USE_MSR_BITMAPS |
f78e0e2e 2378 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1c3d14fe
YS		 2379 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
2380 &_cpu_based_exec_control) < 0)
002c7f7c 2381 return -EIO;
6e5d865c
YS		 2382#ifdef CONFIG_X86_64
2383 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2384 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
2385 ~CPU_BASED_CR8_STORE_EXITING;
2386#endif
f78e0e2e 2387 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
d56f546d
SY		 2388 min2 = 0;
2389 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
8d14695f 2390 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2384d2b3 2391 SECONDARY_EXEC_WBINVD_EXITING |
d56f546d 2392 SECONDARY_EXEC_ENABLE_VPID |
3a624e29 2393 SECONDARY_EXEC_ENABLE_EPT |
4b8d54f9 2394 SECONDARY_EXEC_UNRESTRICTED_GUEST |
4e47c7a6 2395 SECONDARY_EXEC_PAUSE_LOOP_EXITING |
0367f205 2396 SECONDARY_EXEC_DESC |
ad756a16 2397 SECONDARY_EXEC_RDTSCP |
83d4c286 2398 SECONDARY_EXEC_ENABLE_INVPCID |
c7c9c56c 2399 SECONDARY_EXEC_APIC_REGISTER_VIRT |
abc4fc58 2400 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
20300099 2401 SECONDARY_EXEC_SHADOW_VMCS |
843e4330 2402 SECONDARY_EXEC_XSAVES |
736fdf72
DH		 2403 SECONDARY_EXEC_RDSEED_EXITING |
2404 SECONDARY_EXEC_RDRAND_EXITING |
8b3e34e4 2405 SECONDARY_EXEC_ENABLE_PML |
2a499e49 2406 SECONDARY_EXEC_TSC_SCALING |
e69e72fa 2407 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
f99e3daf
CP		 2408 SECONDARY_EXEC_PT_USE_GPA |
2409 SECONDARY_EXEC_PT_CONCEAL_VMX |
7a57c09b
SC		 2410 SECONDARY_EXEC_ENABLE_VMFUNC;
2411 if (cpu_has_sgx())
2412 opt2 |= SECONDARY_EXEC_ENCLS_EXITING;
d56f546d
SY		 2413 if (adjust_vmx_controls(min2, opt2,
2414 MSR_IA32_VMX_PROCBASED_CTLS2,
f78e0e2e
SY		 2415 &_cpu_based_2nd_exec_control) < 0)
2416 return -EIO;
2417 }
2418#ifndef CONFIG_X86_64
2419 if (!(_cpu_based_2nd_exec_control &
2420 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2421 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
2422#endif
83d4c286
YZ		 2423
2424 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2425 _cpu_based_2nd_exec_control &= ~(
8d14695f 2426 SECONDARY_EXEC_APIC_REGISTER_VIRT |
c7c9c56c
YZ		 2427 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2428 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
83d4c286 2429
61f1dd90 2430 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
7caaa711 2431 &vmx_cap->ept, &vmx_cap->vpid);
61f1dd90 2432
d56f546d 2433 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
a7052897
MT		 2434 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
2435 enabled */
5fff7d27
GN		 2436 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
2437 CPU_BASED_CR3_STORE_EXITING |
2438 CPU_BASED_INVLPG_EXITING);
7caaa711
SC		 2439 } else if (vmx_cap->ept) {
2440 vmx_cap->ept = 0;
61f1dd90
WL		 2441 pr_warn_once("EPT CAP should not exist if not support "
2442 "1-setting enable EPT VM-execution control\n");
2443 }
2444 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
7caaa711
SC		 2445 vmx_cap->vpid) {
2446 vmx_cap->vpid = 0;
61f1dd90
WL		 2447 pr_warn_once("VPID CAP should not exist if not support "
2448 "1-setting enable VPID VM-execution control\n");
d56f546d 2449 }
1c3d14fe 2450
91fa0f8e 2451 min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
1c3d14fe
YS		 2452#ifdef CONFIG_X86_64
2453 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
2454#endif
c73da3fc 2455 opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
c73da3fc
SC		 2456 VM_EXIT_LOAD_IA32_PAT |
2457 VM_EXIT_LOAD_IA32_EFER |
f99e3daf
CP		 2458 VM_EXIT_CLEAR_BNDCFGS |
2459 VM_EXIT_PT_CONCEAL_PIP |
2460 VM_EXIT_CLEAR_IA32_RTIT_CTL;
1c3d14fe
YS		 2461 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
2462 &_vmexit_control) < 0)
002c7f7c 2463 return -EIO;
1c3d14fe 2464
8a1b4392
PB		 2465 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2466 opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
2467 PIN_BASED_VMX_PREEMPTION_TIMER;
01e439be
YZ		 2468 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
2469 &_pin_based_exec_control) < 0)
2470 return -EIO;
2471
1c17c3e6
PB		 2472 if (cpu_has_broken_vmx_preemption_timer())
2473 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
01e439be 2474 if (!(_cpu_based_2nd_exec_control &
91fa0f8e 2475 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
01e439be
YZ		 2476 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
2477
c845f9c6 2478 min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
c73da3fc
SC		 2479 opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
2480 VM_ENTRY_LOAD_IA32_PAT |
2481 VM_ENTRY_LOAD_IA32_EFER |
f99e3daf
CP		 2482 VM_ENTRY_LOAD_BNDCFGS |
2483 VM_ENTRY_PT_CONCEAL_PIP |
2484 VM_ENTRY_LOAD_IA32_RTIT_CTL;
1c3d14fe
YS		 2485 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
2486 &_vmentry_control) < 0)
002c7f7c 2487 return -EIO;
6aa8b732 2488
c73da3fc
SC		 2489 /*
2490 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they
2491 * can't be used due to an errata where VM Exit may incorrectly clear
2492 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the
2493 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL.
2494 */
2495 if (boot_cpu_data.x86 == 0x6) {
2496 switch (boot_cpu_data.x86_model) {
2497 case 26: /* AAK155 */
2498 case 30: /* AAP115 */
2499 case 37: /* AAT100 */
2500 case 44: /* BC86,AAY89,BD102 */
2501 case 46: /* BA97 */
85ba2b16 2502 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
c73da3fc
SC		 2503 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
2504 pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
2505 "does not work properly. Using workaround\n");
2506 break;
2507 default:
2508 break;
2509 }
2510 }
2511
2512
c68876fd 2513 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
1c3d14fe
YS		 2514
2515 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
2516 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
002c7f7c 2517 return -EIO;
1c3d14fe
YS		 2518
2519#ifdef CONFIG_X86_64
2520 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
2521 if (vmx_msr_high & (1u<<16))
002c7f7c 2522 return -EIO;
1c3d14fe
YS		 2523#endif
2524
2525 /* Require Write-Back (WB) memory type for VMCS accesses. */
2526 if (((vmx_msr_high >> 18) & 15) != 6)
002c7f7c 2527 return -EIO;
1c3d14fe 2528
002c7f7c 2529 vmcs_conf->size = vmx_msr_high & 0x1fff;
16cb0255 2530 vmcs_conf->order = get_order(vmcs_conf->size);
9ac7e3e8 2531 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
773e8a04 2532
2307af1c 2533 vmcs_conf->revision_id = vmx_msr_low;
1c3d14fe 2534
002c7f7c
YS		 2535 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
2536 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
f78e0e2e 2537 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
002c7f7c
YS		 2538 vmcs_conf->vmexit_ctrl = _vmexit_control;
2539 vmcs_conf->vmentry_ctrl = _vmentry_control;
1c3d14fe 2540
773e8a04
VK		 2541 if (static_branch_unlikely(&enable_evmcs))
2542 evmcs_sanitize_exec_ctrls(vmcs_conf);
2543
1c3d14fe 2544 return 0;
c68876fd 2545}
6aa8b732 2546
41836839 2547struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
6aa8b732
AK		 2548{
2549 int node = cpu_to_node(cpu);
2550 struct page *pages;
2551 struct vmcs *vmcs;
2552
41836839 2553 pages = __alloc_pages_node(node, flags, vmcs_config.order);
6aa8b732
AK		 2554 if (!pages)
2555 return NULL;
2556 vmcs = page_address(pages);
1c3d14fe 2557 memset(vmcs, 0, vmcs_config.size);
2307af1c
LA		 2558
2559 /* KVM supports Enlightened VMCS v1 only */
2560 if (static_branch_unlikely(&enable_evmcs))
392b2f25 2561 vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
2307af1c 2562 else
392b2f25 2563 vmcs->hdr.revision_id = vmcs_config.revision_id;
2307af1c 2564
491a6038
LA		 2565 if (shadow)
2566 vmcs->hdr.shadow_vmcs = 1;
6aa8b732
AK		 2567 return vmcs;
2568}
2569
89b0c9f5 2570void free_vmcs(struct vmcs *vmcs)
6aa8b732 2571{
1c3d14fe 2572 free_pages((unsigned long)vmcs, vmcs_config.order);
6aa8b732
AK		 2573}
2574
d462b819
NHE		 2575/*
2576 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2577 */
89b0c9f5 2578void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
d462b819
NHE		 2579{
2580 if (!loaded_vmcs->vmcs)
2581 return;
2582 loaded_vmcs_clear(loaded_vmcs);
2583 free_vmcs(loaded_vmcs->vmcs);
2584 loaded_vmcs->vmcs = NULL;
904e14fb
PB		 2585 if (loaded_vmcs->msr_bitmap)
2586 free_page((unsigned long)loaded_vmcs->msr_bitmap);
355f4fb1 2587 WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
d462b819
NHE		 2588}
2589
89b0c9f5 2590int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
f21f165e 2591{
491a6038 2592 loaded_vmcs->vmcs = alloc_vmcs(false);
f21f165e
PB		 2593 if (!loaded_vmcs->vmcs)
2594 return -ENOMEM;
2595
d260f9ef
SC		 2596 vmcs_clear(loaded_vmcs->vmcs);
2597
f21f165e 2598 loaded_vmcs->shadow_vmcs = NULL;
804939ea 2599 loaded_vmcs->hv_timer_soft_disabled = false;
d260f9ef
SC		 2600 loaded_vmcs->cpu = -1;
2601 loaded_vmcs->launched = 0;
904e14fb
PB		 2602
2603 if (cpu_has_vmx_msr_bitmap()) {
41836839
BG		 2604 loaded_vmcs->msr_bitmap = (unsigned long *)
2605 __get_free_page(GFP_KERNEL_ACCOUNT);
904e14fb
PB		 2606 if (!loaded_vmcs->msr_bitmap)
2607 goto out_vmcs;
2608 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
ceef7d10 2609
1f008e11
AB		 2610 if (IS_ENABLED(CONFIG_HYPERV) &&
2611 static_branch_unlikely(&enable_evmcs) &&
ceef7d10
VK		 2612 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
2613 struct hv_enlightened_vmcs *evmcs =
2614 (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
2615
2616 evmcs->hv_enlightenments_control.msr_bitmap = 1;
2617 }
904e14fb 2618 }
d7ee039e
SC		 2619
2620 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
3af80fec
SC		 2621 memset(&loaded_vmcs->controls_shadow, 0,
2622 sizeof(struct vmcs_controls_shadow));
d7ee039e 2623
f21f165e 2624 return 0;
904e14fb
PB		 2625
2626out_vmcs:
2627 free_loaded_vmcs(loaded_vmcs);
2628 return -ENOMEM;
f21f165e
PB		 2629}
2630
39959588 2631static void free_kvm_area(void)
6aa8b732
AK		 2632{
2633 int cpu;
2634
3230bb47 2635 for_each_possible_cpu(cpu) {
6aa8b732 2636 free_vmcs(per_cpu(vmxarea, cpu));
3230bb47
ZA		 2637 per_cpu(vmxarea, cpu) = NULL;
2638 }
6aa8b732
AK		 2639}
2640
6aa8b732
AK		 2641static __init int alloc_kvm_area(void)
2642{
2643 int cpu;
2644
3230bb47 2645 for_each_possible_cpu(cpu) {
6aa8b732
AK		 2646 struct vmcs *vmcs;
2647
41836839 2648 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
6aa8b732
AK		 2649 if (!vmcs) {
2650 free_kvm_area();
2651 return -ENOMEM;
2652 }
2653
2307af1c
LA		 2654 /*
2655 * When eVMCS is enabled, alloc_vmcs_cpu() sets
2656 * vmcs->revision_id to KVM_EVMCS_VERSION instead of
2657 * revision_id reported by MSR_IA32_VMX_BASIC.
2658 *
312a4661 2659 * However, even though not explicitly documented by
2307af1c
LA		 2660 * TLFS, VMXArea passed as VMXON argument should
2661 * still be marked with revision_id reported by
2662 * physical CPU.
2663 */
2664 if (static_branch_unlikely(&enable_evmcs))
392b2f25 2665 vmcs->hdr.revision_id = vmcs_config.revision_id;
2307af1c 2666
6aa8b732
AK		 2667 per_cpu(vmxarea, cpu) = vmcs;
2668 }
2669 return 0;
2670}
2671
91b0aa2c 2672static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
d99e4152 2673 struct kvm_segment *save)
6aa8b732 2674{
d99e4152
GN		 2675 if (!emulate_invalid_guest_state) {
2676 /*
2677 * CS and SS RPL should be equal during guest entry according
2678 * to VMX spec, but in reality it is not always so. Since vcpu
2679 * is in the middle of the transition from real mode to
2680 * protected mode it is safe to assume that RPL 0 is a good
2681 * default value.
2682 */
2683 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
b32a9918
NA		 2684 save->selector &= ~SEGMENT_RPL_MASK;
2685 save->dpl = save->selector & SEGMENT_RPL_MASK;
d99e4152 2686 save->s = 1;
6aa8b732 2687 }
d99e4152 2688 vmx_set_segment(vcpu, save, seg);
6aa8b732
AK		 2689}
2690
2691static void enter_pmode(struct kvm_vcpu *vcpu)
2692{
2693 unsigned long flags;
a89a8fb9 2694 struct vcpu_vmx *vmx = to_vmx(vcpu);
6aa8b732 2695
d99e4152
GN		 2696 /*
2697 * Update real mode segment cache. It may be not up-to-date if sement
2698 * register was written while vcpu was in a guest mode.
2699 */
2700 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2701 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2702 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2703 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2704 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2705 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2706
7ffd92c5 2707 vmx->rmode.vm86_active = 0;
6aa8b732 2708
f5f7b2fe 2709 vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
6aa8b732
AK		 2710
2711 flags = vmcs_readl(GUEST_RFLAGS);
78ac8b47
AK		 2712 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2713 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
6aa8b732
AK		 2714 vmcs_writel(GUEST_RFLAGS, flags);
2715
66aee91a
RR		 2716 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
2717 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
6aa8b732
AK		 2718
2719 update_exception_bitmap(vcpu);
2720
91b0aa2c
GN		 2721 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2722 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2723 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2724 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2725 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2726 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
6aa8b732
AK		 2727}
2728
f5f7b2fe 2729static void fix_rmode_seg(int seg, struct kvm_segment *save)
6aa8b732 2730{
772e0318 2731 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
d99e4152
GN		 2732 struct kvm_segment var = *save;
2733
2734 var.dpl = 0x3;
2735 if (seg == VCPU_SREG_CS)
2736 var.type = 0x3;
2737
2738 if (!emulate_invalid_guest_state) {
2739 var.selector = var.base >> 4;
2740 var.base = var.base & 0xffff0;
2741 var.limit = 0xffff;
2742 var.g = 0;
2743 var.db = 0;
2744 var.present = 1;
2745 var.s = 1;
2746 var.l = 0;
2747 var.unusable = 0;
2748 var.type = 0x3;
2749 var.avl = 0;
2750 if (save->base & 0xf)
2751 printk_once(KERN_WARNING "kvm: segment base is not "
2752 "paragraph aligned when entering "
2753 "protected mode (seg=%d)", seg);
2754 }
6aa8b732 2755
d99e4152 2756 vmcs_write16(sf->selector, var.selector);
96794e4e 2757 vmcs_writel(sf->base, var.base);
d99e4152
GN		 2758 vmcs_write32(sf->limit, var.limit);
2759 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
6aa8b732
AK		 2760}
2761
2762static void enter_rmode(struct kvm_vcpu *vcpu)
2763{
2764 unsigned long flags;
a89a8fb9 2765 struct vcpu_vmx *vmx = to_vmx(vcpu);
40bbb9d0 2766 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
6aa8b732 2767
f5f7b2fe
AK		 2768 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2769 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2770 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2771 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2772 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
c6ad1153
GN		 2773 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2774 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
f5f7b2fe 2775
7ffd92c5 2776 vmx->rmode.vm86_active = 1;
6aa8b732 2777
776e58ea
GN		 2778 /*
2779 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
4918c6ca 2780 * vcpu. Warn the user that an update is overdue.
776e58ea 2781 */
40bbb9d0 2782 if (!kvm_vmx->tss_addr)
776e58ea
GN		 2783 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
2784 "called before entering vcpu\n");
776e58ea 2785
2fb92db1
AK		 2786 vmx_segment_cache_clear(vmx);
2787
40bbb9d0 2788 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
6aa8b732 2789 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
6aa8b732
AK		 2790 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2791
2792 flags = vmcs_readl(GUEST_RFLAGS);
78ac8b47 2793 vmx->rmode.save_rflags = flags;
6aa8b732 2794
053de044 2795 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
6aa8b732
AK		 2796
2797 vmcs_writel(GUEST_RFLAGS, flags);
66aee91a 2798 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
6aa8b732
AK		 2799 update_exception_bitmap(vcpu);
2800
d99e4152
GN		 2801 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2802 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2803 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2804 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2805 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2806 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
b246dd5d 2807
8668a3c4 2808 kvm_mmu_reset_context(vcpu);
6aa8b732
AK		 2809}
2810
97b7ead3 2811void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
401d10de
AS		 2812{
2813 struct vcpu_vmx *vmx = to_vmx(vcpu);
26bb0981
AK		 2814 struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
2815
2816 if (!msr)
2817 return;
401d10de 2818
f6801dff 2819 vcpu->arch.efer = efer;
401d10de 2820 if (efer & EFER_LMA) {
2961e876 2821 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
401d10de
AS		 2822 msr->data = efer;
2823 } else {
2961e876 2824 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
401d10de
AS		 2825
2826 msr->data = efer & ~EFER_LME;
2827 }
2828 setup_msrs(vmx);
2829}
2830
05b3e0c2 2831#ifdef CONFIG_X86_64
6aa8b732
AK		 2832
2833static void enter_lmode(struct kvm_vcpu *vcpu)
2834{
2835 u32 guest_tr_ar;
2836
2fb92db1
AK		 2837 vmx_segment_cache_clear(to_vmx(vcpu));
2838
6aa8b732 2839 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
4d283ec9 2840 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
bd80158a
JK		 2841 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
2842 __func__);
6aa8b732 2843 vmcs_write32(GUEST_TR_AR_BYTES,
4d283ec9
AL		 2844 (guest_tr_ar & ~VMX_AR_TYPE_MASK)
2845 | VMX_AR_TYPE_BUSY_64_TSS);
6aa8b732 2846 }
da38f438 2847 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
6aa8b732
AK		 2848}
2849
2850static void exit_lmode(struct kvm_vcpu *vcpu)
2851{
2961e876 2852 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
da38f438 2853 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
6aa8b732
AK		 2854}
2855
2856#endif
2857
7780938c 2858static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
5058b692
SC		 2859{
2860 struct vcpu_vmx *vmx = to_vmx(vcpu);
2861
2862 /*
7780938c
SC		 2863 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as
2864 * the CPU is not required to invalidate guest-physical mappings on
2865 * VM-Entry, even if VPID is disabled. Guest-physical mappings are
2866 * associated with the root EPT structure and not any particular VPID
2867 * (INVVPID also isn't required to invalidate guest-physical mappings).
5058b692
SC		 2868 */
2869 if (enable_ept) {
2870 ept_sync_global();
2871 } else if (enable_vpid) {
2872 if (cpu_has_vmx_invvpid_global()) {
2873 vpid_sync_vcpu_global();
2874 } else {
2875 vpid_sync_vcpu_single(vmx->vpid);
2876 vpid_sync_vcpu_single(vmx->nested.vpid02);
2877 }
2878 }
2879}
2880
33d19ec9
SC		 2881static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
2882{
2883 u64 root_hpa = vcpu->arch.mmu->root_hpa;
2884
2885 /* No flush required if the current context is invalid. */
2886 if (!VALID_PAGE(root_hpa))
2887 return;
2888
2889 if (enable_ept)
2890 ept_sync_context(construct_eptp(vcpu, root_hpa));
2891 else if (!is_guest_mode(vcpu))
2892 vpid_sync_context(to_vmx(vcpu)->vpid);
2893 else
2894 vpid_sync_context(nested_get_vpid02(vcpu));
2895}
2896
faff8758
JS		 2897static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
2898{
faff8758 2899 /*
ad104b5e
SC		 2900 * vpid_sync_vcpu_addr() is a nop if vmx->vpid==0, see the comment in
2901 * vmx_flush_tlb_guest() for an explanation of why this is ok.
faff8758 2902 */
ad104b5e 2903 vpid_sync_vcpu_addr(to_vmx(vcpu)->vpid, addr);
faff8758
JS		 2904}
2905
e64419d9
SC		 2906static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
2907{
2908 /*
2909 * vpid_sync_context() is a nop if vmx->vpid==0, e.g. if enable_vpid==0
2910 * or a vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit
2911 * are required to flush GVA->{G,H}PA mappings from the TLB if vpid is
2912 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed),
2913 * i.e. no explicit INVVPID is necessary.
2914 */
2915 vpid_sync_context(to_vmx(vcpu)->vpid);
2916}
2917
e8467fda
AK		 2918static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2919{
2920 ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
2921
2922 vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
2923 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
2924}
2925
25c4c276 2926static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
399badf3 2927{
fc78f519
AK		 2928 ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
2929
2930 vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
2931 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
399badf3
AK		 2932}
2933
1439442c
SY		 2934static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
2935{
d0d538b9
GN		 2936 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2937
cb3c1e2f 2938 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR))
6de4f3ad
AK		 2939 return;
2940
bf03d4f9 2941 if (is_pae_paging(vcpu)) {
d0d538b9
GN		 2942 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
2943 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
2944 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
2945 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
1439442c
SY		 2946 }
2947}
2948
97b7ead3 2949void ept_save_pdptrs(struct kvm_vcpu *vcpu)
8f5d549f 2950{
d0d538b9
GN		 2951 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2952
9932b49e
SC		 2953 if (WARN_ON_ONCE(!is_pae_paging(vcpu)))
2954 return;
2955
2956 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
2957 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
2958 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
2959 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
6de4f3ad 2960
cb3c1e2f 2961 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
8f5d549f
AK		 2962}
2963
1439442c
SY		 2964static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
2965 unsigned long cr0,
2966 struct kvm_vcpu *vcpu)
2967{
2183f564
SC		 2968 struct vcpu_vmx *vmx = to_vmx(vcpu);
2969
cb3c1e2f 2970 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
34059c25 2971 vmx_cache_reg(vcpu, VCPU_EXREG_CR3);
1439442c
SY		 2972 if (!(cr0 & X86_CR0_PG)) {
2973 /* From paging/starting to nonpaging */
2183f564
SC		 2974 exec_controls_setbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
2975 CPU_BASED_CR3_STORE_EXITING);
1439442c 2976 vcpu->arch.cr0 = cr0;
fc78f519 2977 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1439442c
SY		 2978 } else if (!is_paging(vcpu)) {
2979 /* From nonpaging to paging */
2183f564
SC		 2980 exec_controls_clearbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
2981 CPU_BASED_CR3_STORE_EXITING);
1439442c 2982 vcpu->arch.cr0 = cr0;
fc78f519 2983 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
1439442c 2984 }
95eb84a7
SY		 2985
2986 if (!(cr0 & X86_CR0_WP))
2987 *hw_cr0 &= ~X86_CR0_WP;
1439442c
SY		 2988}
2989
97b7ead3 2990void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
6aa8b732 2991{
7ffd92c5 2992 struct vcpu_vmx *vmx = to_vmx(vcpu);
3a624e29
NK		 2993 unsigned long hw_cr0;
2994
3de6347b 2995 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF);
3a624e29 2996 if (enable_unrestricted_guest)
5037878e 2997 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
218e763f 2998 else {
5037878e 2999 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
1439442c 3000
218e763f
GN		 3001 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
3002 enter_pmode(vcpu);
6aa8b732 3003
218e763f
GN		 3004 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
3005 enter_rmode(vcpu);
3006 }
6aa8b732 3007
05b3e0c2 3008#ifdef CONFIG_X86_64
f6801dff 3009 if (vcpu->arch.efer & EFER_LME) {
707d92fa 3010 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
6aa8b732 3011 enter_lmode(vcpu);
707d92fa 3012 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
6aa8b732
AK		 3013 exit_lmode(vcpu);
3014 }
3015#endif
3016
b4d18517 3017 if (enable_ept && !enable_unrestricted_guest)
1439442c
SY		 3018 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
3019
6aa8b732 3020 vmcs_writel(CR0_READ_SHADOW, cr0);
1439442c 3021 vmcs_writel(GUEST_CR0, hw_cr0);
ad312c7c 3022 vcpu->arch.cr0 = cr0;
14168786
GN		 3023
3024 /* depends on vcpu->arch.cr0 to be set to a new value */
3025 vmx->emulation_required = emulation_required(vcpu);
6aa8b732
AK		 3026}
3027
855feb67
YZ		 3028static int get_ept_level(struct kvm_vcpu *vcpu)
3029{
148d735e 3030 if (is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu)))
ac69dfaa 3031 return vmx_eptp_page_walk_level(nested_ept_get_eptp(vcpu));
855feb67
YZ		 3032 if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48))
3033 return 5;
3034 return 4;
3035}
3036
89b0c9f5 3037u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
1439442c 3038{
855feb67
YZ		 3039 u64 eptp = VMX_EPTP_MT_WB;
3040
3041 eptp |= (get_ept_level(vcpu) == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
1439442c 3042
995f00a6
PF		 3043 if (enable_ept_ad_bits &&
3044 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
bb97a016 3045 eptp |= VMX_EPTP_AD_ENABLE_BIT;
1439442c
SY		 3046 eptp |= (root_hpa & PAGE_MASK);
3047
3048 return eptp;
3049}
3050
be100ef1 3051void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd)
6aa8b732 3052{
877ad952 3053 struct kvm *kvm = vcpu->kvm;
04f11ef4 3054 bool update_guest_cr3 = true;
1439442c
SY		 3055 unsigned long guest_cr3;
3056 u64 eptp;
3057
089d034e 3058 if (enable_ept) {
be100ef1 3059 eptp = construct_eptp(vcpu, pgd);
1439442c 3060 vmcs_write64(EPT_POINTER, eptp);
877ad952 3061
afaf0b2f 3062 if (kvm_x86_ops.tlb_remote_flush) {
877ad952
TL		 3063 spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
3064 to_vmx(vcpu)->ept_pointer = eptp;
3065 to_kvm_vmx(kvm)->ept_pointers_match
3066 = EPT_POINTERS_CHECK;
3067 spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
3068 }
3069
04f11ef4
SC		 3070 /* Loading vmcs02.GUEST_CR3 is handled by nested VM-Enter. */
3071 if (is_guest_mode(vcpu))
3072 update_guest_cr3 = false;
b17b7436 3073 else if (!enable_unrestricted_guest && !is_paging(vcpu))
877ad952 3074 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
b17b7436
SC		 3075 else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3076 guest_cr3 = vcpu->arch.cr3;
3077 else /* vmcs01.GUEST_CR3 is already up-to-date. */
3078 update_guest_cr3 = false;
7c93be44 3079 ept_load_pdptrs(vcpu);
be100ef1
SC		 3080 } else {
3081 guest_cr3 = pgd;
1439442c
SY		 3082 }
3083
04f11ef4
SC		 3084 if (update_guest_cr3)
3085 vmcs_writel(GUEST_CR3, guest_cr3);
6aa8b732
AK		 3086}
3087
97b7ead3 3088int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
6aa8b732 3089{
fe7f895d 3090 struct vcpu_vmx *vmx = to_vmx(vcpu);
085e68ee
BS		 3091 /*
3092 * Pass through host's Machine Check Enable value to hw_cr4, which
3093 * is in force while we are in guest mode. Do not let guests control
3094 * this bit, even if host CR4.MCE == 0.
3095 */
5dc1f044
SC		 3096 unsigned long hw_cr4;
3097
3098 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
3099 if (enable_unrestricted_guest)
3100 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
fe7f895d 3101 else if (vmx->rmode.vm86_active)
5dc1f044
SC		 3102 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
3103 else
3104 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
1439442c 3105
64f7a115
SC		 3106 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
3107 if (cr4 & X86_CR4_UMIP) {
fe7f895d 3108 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC);
64f7a115
SC		 3109 hw_cr4 &= ~X86_CR4_UMIP;
3110 } else if (!is_guest_mode(vcpu) ||
fe7f895d
SC		 3111 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) {
3112 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC);
3113 }
64f7a115 3114 }
0367f205 3115
5e1746d6
NHE		 3116 if (cr4 & X86_CR4_VMXE) {
3117 /*
3118 * To use VMXON (and later other VMX instructions), a guest
3119 * must first be able to turn on cr4.VMXE (see handle_vmon()).
3120 * So basically the check on whether to allow nested VMX
5bea5123
PB		 3121 * is here. We operate under the default treatment of SMM,
3122 * so VMX cannot be enabled under SMM.
5e1746d6 3123 */
5bea5123 3124 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
5e1746d6 3125 return 1;
1a0d74e6 3126 }
3899152c 3127
fe7f895d 3128 if (vmx->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
5e1746d6
NHE		 3129 return 1;
3130
ad312c7c 3131 vcpu->arch.cr4 = cr4;
5dc1f044
SC		 3132
3133 if (!enable_unrestricted_guest) {
3134 if (enable_ept) {
3135 if (!is_paging(vcpu)) {
3136 hw_cr4 &= ~X86_CR4_PAE;
3137 hw_cr4 |= X86_CR4_PSE;
3138 } else if (!(cr4 & X86_CR4_PAE)) {
3139 hw_cr4 &= ~X86_CR4_PAE;
3140 }
bc23008b 3141 }
1439442c 3142
656ec4a4 3143 /*
ddba2628
HH		 3144 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
3145 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs
3146 * to be manually disabled when guest switches to non-paging
3147 * mode.
3148 *
3149 * If !enable_unrestricted_guest, the CPU is always running
3150 * with CR0.PG=1 and CR4 needs to be modified.
3151 * If enable_unrestricted_guest, the CPU automatically
3152 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
656ec4a4 3153 */
5dc1f044
SC		 3154 if (!is_paging(vcpu))
3155 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
3156 }
656ec4a4 3157
1439442c
SY		 3158 vmcs_writel(CR4_READ_SHADOW, cr4);
3159 vmcs_writel(GUEST_CR4, hw_cr4);
5e1746d6 3160 return 0;
6aa8b732
AK		 3161}
3162
97b7ead3 3163void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
6aa8b732 3164{
a9179499 3165 struct vcpu_vmx *vmx = to_vmx(vcpu);
6aa8b732
AK		 3166 u32 ar;
3167
c6ad1153 3168 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
f5f7b2fe 3169 *var = vmx->rmode.segs[seg];
a9179499 3170 if (seg == VCPU_SREG_TR
2fb92db1 3171 || var->selector == vmx_read_guest_seg_selector(vmx, seg))
f5f7b2fe 3172 return;
1390a28b
AK		 3173 var->base = vmx_read_guest_seg_base(vmx, seg);
3174 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3175 return;
a9179499 3176 }
2fb92db1
AK		 3177 var->base = vmx_read_guest_seg_base(vmx, seg);
3178 var->limit = vmx_read_guest_seg_limit(vmx, seg);
3179 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3180 ar = vmx_read_guest_seg_ar(vmx, seg);
03617c18 3181 var->unusable = (ar >> 16) & 1;
6aa8b732
AK		 3182 var->type = ar & 15;
3183 var->s = (ar >> 4) & 1;
3184 var->dpl = (ar >> 5) & 3;
03617c18
GN		 3185 /*
3186 * Some userspaces do not preserve unusable property. Since usable
3187 * segment has to be present according to VMX spec we can use present
3188 * property to amend userspace bug by making unusable segment always
3189 * nonpresent. vmx_segment_access_rights() already marks nonpresent
3190 * segment as unusable.
3191 */
3192 var->present = !var->unusable;
6aa8b732
AK		 3193 var->avl = (ar >> 12) & 1;
3194 var->l = (ar >> 13) & 1;
3195 var->db = (ar >> 14) & 1;
3196 var->g = (ar >> 15) & 1;
6aa8b732
AK		 3197}
3198
a9179499
AK		 3199static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
3200{
a9179499
AK		 3201 struct kvm_segment s;
3202
3203 if (to_vmx(vcpu)->rmode.vm86_active) {
3204 vmx_get_segment(vcpu, &s, seg);
3205 return s.base;
3206 }
2fb92db1 3207 return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
a9179499
AK		 3208}
3209
97b7ead3 3210int vmx_get_cpl(struct kvm_vcpu *vcpu)
2e4d2653 3211{
b09408d0
MT		 3212 struct vcpu_vmx *vmx = to_vmx(vcpu);
3213
ae9fedc7 3214 if (unlikely(vmx->rmode.vm86_active))
2e4d2653 3215 return 0;
ae9fedc7
PB		 3216 else {
3217 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
4d283ec9 3218 return VMX_AR_DPL(ar);
69c73028 3219 }
69c73028
AK		 3220}
3221
653e3108 3222static u32 vmx_segment_access_rights(struct kvm_segment *var)
6aa8b732 3223{
6aa8b732
AK		 3224 u32 ar;
3225
f0495f9b 3226 if (var->unusable || !var->present)
6aa8b732
AK		 3227 ar = 1 << 16;
3228 else {
3229 ar = var->type & 15;
3230 ar |= (var->s & 1) << 4;
3231 ar |= (var->dpl & 3) << 5;
3232 ar |= (var->present & 1) << 7;
3233 ar |= (var->avl & 1) << 12;
3234 ar |= (var->l & 1) << 13;
3235 ar |= (var->db & 1) << 14;
3236 ar |= (var->g & 1) << 15;
3237 }
653e3108
AK		 3238
3239 return ar;
3240}
3241
97b7ead3 3242void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
653e3108 3243{
7ffd92c5 3244 struct vcpu_vmx *vmx = to_vmx(vcpu);
772e0318 3245 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
653e3108 3246
2fb92db1
AK		 3247 vmx_segment_cache_clear(vmx);
3248
1ecd50a9
GN		 3249 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3250 vmx->rmode.segs[seg] = *var;
3251 if (seg == VCPU_SREG_TR)
3252 vmcs_write16(sf->selector, var->selector);
3253 else if (var->s)
3254 fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
d99e4152 3255 goto out;
653e3108 3256 }
1ecd50a9 3257
653e3108
AK		 3258 vmcs_writel(sf->base, var->base);
3259 vmcs_write32(sf->limit, var->limit);
3260 vmcs_write16(sf->selector, var->selector);
3a624e29
NK		 3261
3262 /*
3263 * Fix the "Accessed" bit in AR field of segment registers for older
3264 * qemu binaries.
3265 * IA32 arch specifies that at the time of processor reset the
3266 * "Accessed" bit in the AR field of segment registers is 1. And qemu
0fa06071 3267 * is setting it to 0 in the userland code. This causes invalid guest
3a624e29
NK		 3268 * state vmexit when "unrestricted guest" mode is turned on.
3269 * Fix for this setup issue in cpu_reset is being pushed in the qemu
3270 * tree. Newer qemu binaries with that qemu fix would not need this
3271 * kvm hack.
3272 */
3273 if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
f924d66d 3274 var->type |= 0x1; /* Accessed */
3a624e29 3275
f924d66d 3276 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
d99e4152
GN		 3277
3278out:
98eb2f8b 3279 vmx->emulation_required = emulation_required(vcpu);
6aa8b732
AK		 3280}
3281
6aa8b732
AK		 3282static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3283{
2fb92db1 3284 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
6aa8b732
AK		 3285
3286 *db = (ar >> 14) & 1;
3287 *l = (ar >> 13) & 1;
3288}
3289
89a27f4d 3290static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 3291{
89a27f4d
GN		 3292 dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
3293 dt->address = vmcs_readl(GUEST_IDTR_BASE);
6aa8b732
AK		 3294}
3295
89a27f4d 3296static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 3297{
89a27f4d
GN		 3298 vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
3299 vmcs_writel(GUEST_IDTR_BASE, dt->address);
6aa8b732
AK		 3300}
3301
89a27f4d 3302static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 3303{
89a27f4d
GN		 3304 dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
3305 dt->address = vmcs_readl(GUEST_GDTR_BASE);
6aa8b732
AK		 3306}
3307
89a27f4d 3308static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 3309{
89a27f4d
GN		 3310 vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
3311 vmcs_writel(GUEST_GDTR_BASE, dt->address);
6aa8b732
AK		 3312}
3313
648dfaa7
MG		 3314static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3315{
3316 struct kvm_segment var;
3317 u32 ar;
3318
3319 vmx_get_segment(vcpu, &var, seg);
07f42f5f 3320 var.dpl = 0x3;
0647f4aa
GN		 3321 if (seg == VCPU_SREG_CS)
3322 var.type = 0x3;
648dfaa7
MG		 3323 ar = vmx_segment_access_rights(&var);
3324
3325 if (var.base != (var.selector << 4))
3326 return false;
89efbed0 3327 if (var.limit != 0xffff)
648dfaa7 3328 return false;
07f42f5f 3329 if (ar != 0xf3)
648dfaa7
MG		 3330 return false;
3331
3332 return true;
3333}
3334
3335static bool code_segment_valid(struct kvm_vcpu *vcpu)
3336{
3337 struct kvm_segment cs;
3338 unsigned int cs_rpl;
3339
3340 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
b32a9918 3341 cs_rpl = cs.selector & SEGMENT_RPL_MASK;
648dfaa7 3342
1872a3f4
AK		 3343 if (cs.unusable)
3344 return false;
4d283ec9 3345 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
648dfaa7
MG		 3346 return false;
3347 if (!cs.s)
3348 return false;
4d283ec9 3349 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
648dfaa7
MG		 3350 if (cs.dpl > cs_rpl)
3351 return false;
1872a3f4 3352 } else {
648dfaa7
MG		 3353 if (cs.dpl != cs_rpl)
3354 return false;
3355 }
3356 if (!cs.present)
3357 return false;
3358
3359 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
3360 return true;
3361}
3362
3363static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3364{
3365 struct kvm_segment ss;
3366 unsigned int ss_rpl;
3367
3368 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
b32a9918 3369 ss_rpl = ss.selector & SEGMENT_RPL_MASK;
648dfaa7 3370
1872a3f4
AK		 3371 if (ss.unusable)
3372 return true;
3373 if (ss.type != 3 && ss.type != 7)
648dfaa7
MG		 3374 return false;
3375 if (!ss.s)
3376 return false;
3377 if (ss.dpl != ss_rpl) /* DPL != RPL */
3378 return false;
3379 if (!ss.present)
3380 return false;
3381
3382 return true;
3383}
3384
3385static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3386{
3387 struct kvm_segment var;
3388 unsigned int rpl;
3389
3390 vmx_get_segment(vcpu, &var, seg);
b32a9918 3391 rpl = var.selector & SEGMENT_RPL_MASK;
648dfaa7 3392
1872a3f4
AK		 3393 if (var.unusable)
3394 return true;
648dfaa7
MG		 3395 if (!var.s)
3396 return false;
3397 if (!var.present)
3398 return false;
4d283ec9 3399 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
648dfaa7
MG		 3400 if (var.dpl < rpl) /* DPL < RPL */
3401 return false;
3402 }
3403
3404 /* TODO: Add other members to kvm_segment_field to allow checking for other access
3405 * rights flags
3406 */
3407 return true;
3408}
3409
3410static bool tr_valid(struct kvm_vcpu *vcpu)
3411{
3412 struct kvm_segment tr;
3413
3414 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
3415
1872a3f4
AK		 3416 if (tr.unusable)
3417 return false;
b32a9918 3418 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */
648dfaa7 3419 return false;
1872a3f4 3420 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
648dfaa7
MG		 3421 return false;
3422 if (!tr.present)
3423 return false;
3424
3425 return true;
3426}
3427
3428static bool ldtr_valid(struct kvm_vcpu *vcpu)
3429{
3430 struct kvm_segment ldtr;
3431
3432 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
3433
1872a3f4
AK		 3434 if (ldtr.unusable)
3435 return true;
b32a9918 3436 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */
648dfaa7
MG		 3437 return false;
3438 if (ldtr.type != 2)
3439 return false;
3440 if (!ldtr.present)
3441 return false;
3442
3443 return true;
3444}
3445
3446static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3447{
3448 struct kvm_segment cs, ss;
3449
3450 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3451 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3452
b32a9918
NA		 3453 return ((cs.selector & SEGMENT_RPL_MASK) ==
3454 (ss.selector & SEGMENT_RPL_MASK));
648dfaa7
MG		 3455}
3456
3457/*
3458 * Check if guest state is valid. Returns true if valid, false if
3459 * not.
3460 * We assume that registers are always usable
3461 */
3462static bool guest_state_valid(struct kvm_vcpu *vcpu)
3463{
c5e97c80
GN		 3464 if (enable_unrestricted_guest)
3465 return true;
3466
648dfaa7 3467 /* real mode guest state checks */
f13882d8 3468 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
648dfaa7
MG		 3469 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
3470 return false;
3471 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
3472 return false;
3473 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
3474 return false;
3475 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
3476 return false;
3477 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
3478 return false;
3479 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
3480 return false;
3481 } else {
3482 /* protected mode guest state checks */
3483 if (!cs_ss_rpl_check(vcpu))
3484 return false;
3485 if (!code_segment_valid(vcpu))
3486 return false;
3487 if (!stack_segment_valid(vcpu))
3488 return false;
3489 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
3490 return false;
3491 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
3492 return false;
3493 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
3494 return false;
3495 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
3496 return false;
3497 if (!tr_valid(vcpu))
3498 return false;
3499 if (!ldtr_valid(vcpu))
3500 return false;
3501 }
3502 /* TODO:
3503 * - Add checks on RIP
3504 * - Add checks on RFLAGS
3505 */
3506
3507 return true;
3508}
3509
d77c26fc 3510static int init_rmode_tss(struct kvm *kvm)
6aa8b732 3511{
40dcaa9f 3512 gfn_t fn;
195aefde 3513 u16 data = 0;
1f755a82 3514 int idx, r;
6aa8b732 3515
40dcaa9f 3516 idx = srcu_read_lock(&kvm->srcu);
40bbb9d0 3517 fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
195aefde
IE		 3518 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3519 if (r < 0)
10589a46 3520 goto out;
195aefde 3521 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
464d17c8
SY		 3522 r = kvm_write_guest_page(kvm, fn++, &data,
3523 TSS_IOPB_BASE_OFFSET, sizeof(u16));
195aefde 3524 if (r < 0)
10589a46 3525 goto out;
195aefde
IE		 3526 r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
3527 if (r < 0)
10589a46 3528 goto out;
195aefde
IE		 3529 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3530 if (r < 0)
10589a46 3531 goto out;
195aefde 3532 data = ~0;
10589a46
MT		 3533 r = kvm_write_guest_page(kvm, fn, &data,
3534 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
3535 sizeof(u8));
10589a46 3536out:
40dcaa9f 3537 srcu_read_unlock(&kvm->srcu, idx);
1f755a82 3538 return r;
6aa8b732
AK		 3539}
3540
b7ebfb05
SY		 3541static int init_rmode_identity_map(struct kvm *kvm)
3542{
40bbb9d0 3543 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
2a5755bb 3544 int i, r = 0;
ba049e93 3545 kvm_pfn_t identity_map_pfn;
b7ebfb05
SY		 3546 u32 tmp;
3547
40bbb9d0 3548 /* Protect kvm_vmx->ept_identity_pagetable_done. */
a255d479
TC		 3549 mutex_lock(&kvm->slots_lock);
3550
40bbb9d0 3551 if (likely(kvm_vmx->ept_identity_pagetable_done))
2a5755bb 3552 goto out;
a255d479 3553
40bbb9d0
SC		 3554 if (!kvm_vmx->ept_identity_map_addr)
3555 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
3556 identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT;
a255d479 3557
d8a6e365 3558 r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
40bbb9d0 3559 kvm_vmx->ept_identity_map_addr, PAGE_SIZE);
f51770ed 3560 if (r < 0)
2a5755bb 3561 goto out;
a255d479 3562
b7ebfb05
SY		 3563 r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
3564 if (r < 0)
3565 goto out;
3566 /* Set up identity-mapping pagetable for EPT in real mode */
3567 for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
3568 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
3569 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
3570 r = kvm_write_guest_page(kvm, identity_map_pfn,
3571 &tmp, i * sizeof(tmp), sizeof(tmp));
3572 if (r < 0)
3573 goto out;
3574 }
40bbb9d0 3575 kvm_vmx->ept_identity_pagetable_done = true;
f51770ed 3576
b7ebfb05 3577out:
a255d479 3578 mutex_unlock(&kvm->slots_lock);
f51770ed 3579 return r;
b7ebfb05
SY		 3580}
3581
6aa8b732
AK		 3582static void seg_setup(int seg)
3583{
772e0318 3584 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3a624e29 3585 unsigned int ar;
6aa8b732
AK		 3586
3587 vmcs_write16(sf->selector, 0);
3588 vmcs_writel(sf->base, 0);
3589 vmcs_write32(sf->limit, 0xffff);
d54d07b2
GN		 3590 ar = 0x93;
3591 if (seg == VCPU_SREG_CS)
3592 ar |= 0x08; /* code segment */
3a624e29
NK		 3593
3594 vmcs_write32(sf->ar_bytes, ar);
6aa8b732
AK		 3595}
3596
f78e0e2e
SY		 3597static int alloc_apic_access_page(struct kvm *kvm)
3598{
4484141a 3599 struct page *page;
f78e0e2e
SY		 3600 int r = 0;
3601
79fac95e 3602 mutex_lock(&kvm->slots_lock);
c24ae0dc 3603 if (kvm->arch.apic_access_page_done)
f78e0e2e 3604 goto out;
1d8007bd
PB		 3605 r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
3606 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
f78e0e2e
SY		 3607 if (r)
3608 goto out;
72dc67a6 3609
73a6d941 3610 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
4484141a
XG		 3611 if (is_error_page(page)) {
3612 r = -EFAULT;
3613 goto out;
3614 }
3615
c24ae0dc
TC		 3616 /*
3617 * Do not pin the page in memory, so that memory hot-unplug
3618 * is able to migrate it.
3619 */
3620 put_page(page);
3621 kvm->arch.apic_access_page_done = true;
f78e0e2e 3622out:
79fac95e 3623 mutex_unlock(&kvm->slots_lock);
f78e0e2e
SY		 3624 return r;
3625}
3626
97b7ead3 3627int allocate_vpid(void)
2384d2b3
SY		 3628{
3629 int vpid;
3630
919818ab 3631 if (!enable_vpid)
991e7a0e 3632 return 0;
2384d2b3
SY		 3633 spin_lock(&vmx_vpid_lock);
3634 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
991e7a0e 3635 if (vpid < VMX_NR_VPIDS)
2384d2b3 3636 __set_bit(vpid, vmx_vpid_bitmap);
991e7a0e
WL		 3637 else
3638 vpid = 0;
2384d2b3 3639 spin_unlock(&vmx_vpid_lock);
991e7a0e 3640 return vpid;
2384d2b3
SY		 3641}
3642
97b7ead3 3643void free_vpid(int vpid)
cdbecfc3 3644{
991e7a0e 3645 if (!enable_vpid || vpid == 0)
cdbecfc3
LJ		 3646 return;
3647 spin_lock(&vmx_vpid_lock);
991e7a0e 3648 __clear_bit(vpid, vmx_vpid_bitmap);
cdbecfc3
LJ		 3649 spin_unlock(&vmx_vpid_lock);
3650}
3651
1e4329ee 3652static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
904e14fb 3653 u32 msr, int type)
25c5f225 3654{
3e7c73e9 3655 int f = sizeof(unsigned long);
25c5f225
SY		 3656
3657 if (!cpu_has_vmx_msr_bitmap())
3658 return;
3659
ceef7d10
VK		 3660 if (static_branch_unlikely(&enable_evmcs))
3661 evmcs_touch_msr_bitmap();
3662
25c5f225
SY		 3663 /*
3664 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
3665 * have the write-low and read-high bitmap offsets the wrong way round.
3666 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
3667 */
25c5f225 3668 if (msr <= 0x1fff) {
8d14695f
YZ		 3669 if (type & MSR_TYPE_R)
3670 /* read-low */
3671 __clear_bit(msr, msr_bitmap + 0x000 / f);
3672
3673 if (type & MSR_TYPE_W)
3674 /* write-low */
3675 __clear_bit(msr, msr_bitmap + 0x800 / f);
3676
25c5f225
SY		 3677 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
3678 msr &= 0x1fff;
8d14695f
YZ		 3679 if (type & MSR_TYPE_R)
3680 /* read-high */
3681 __clear_bit(msr, msr_bitmap + 0x400 / f);
3682
3683 if (type & MSR_TYPE_W)
3684 /* write-high */
3685 __clear_bit(msr, msr_bitmap + 0xc00 / f);
3686
3687 }
3688}
3689
1e4329ee 3690static __always_inline void vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
904e14fb
PB		 3691 u32 msr, int type)
3692{
3693 int f = sizeof(unsigned long);
3694
3695 if (!cpu_has_vmx_msr_bitmap())
3696 return;
3697
ceef7d10
VK		 3698 if (static_branch_unlikely(&enable_evmcs))
3699 evmcs_touch_msr_bitmap();
3700
904e14fb
PB		 3701 /*
3702 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
3703 * have the write-low and read-high bitmap offsets the wrong way round.
3704 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
3705 */
3706 if (msr <= 0x1fff) {
3707 if (type & MSR_TYPE_R)
3708 /* read-low */
3709 __set_bit(msr, msr_bitmap + 0x000 / f);
3710
3711 if (type & MSR_TYPE_W)
3712 /* write-low */
3713 __set_bit(msr, msr_bitmap + 0x800 / f);
3714
3715 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
3716 msr &= 0x1fff;
3717 if (type & MSR_TYPE_R)
3718 /* read-high */
3719 __set_bit(msr, msr_bitmap + 0x400 / f);
3720
3721 if (type & MSR_TYPE_W)
3722 /* write-high */
3723 __set_bit(msr, msr_bitmap + 0xc00 / f);
3724
3725 }
3726}
3727
1e4329ee 3728static __always_inline void vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
904e14fb
PB		 3729 u32 msr, int type, bool value)
3730{
3731 if (value)
3732 vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
3733 else
3734 vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
3735}
3736
904e14fb 3737static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
5897297b 3738{
904e14fb
PB		 3739 u8 mode = 0;
3740
3741 if (cpu_has_secondary_exec_ctrls() &&
fe7f895d 3742 (secondary_exec_controls_get(to_vmx(vcpu)) &
904e14fb
PB		 3743 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
3744 mode |= MSR_BITMAP_MODE_X2APIC;
3745 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
3746 mode |= MSR_BITMAP_MODE_X2APIC_APICV;
3747 }
3748
904e14fb 3749 return mode;
8d14695f
YZ		 3750}
3751
904e14fb
PB		 3752static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
3753 u8 mode)
8d14695f 3754{
904e14fb
PB		 3755 int msr;
3756
3757 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
3758 unsigned word = msr / BITS_PER_LONG;
3759 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
3760 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
3761 }
3762
3763 if (mode & MSR_BITMAP_MODE_X2APIC) {
3764 /*
3765 * TPR reads and writes can be virtualized even if virtual interrupt
3766 * delivery is not in use.
3767 */
3768 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
3769 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
3770 vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
3771 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
3772 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
3773 }
f6e90f9e 3774 }
5897297b
AK		 3775}
3776
97b7ead3 3777void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
904e14fb
PB		 3778{
3779 struct vcpu_vmx *vmx = to_vmx(vcpu);
3780 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3781 u8 mode = vmx_msr_bitmap_mode(vcpu);
3782 u8 changed = mode ^ vmx->msr_bitmap_mode;
3783
3784 if (!changed)
3785 return;
3786
904e14fb
PB		 3787 if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
3788 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
3789
3790 vmx->msr_bitmap_mode = mode;
3791}
3792
b08c2896
CP		 3793void pt_update_intercept_for_msr(struct vcpu_vmx *vmx)
3794{
3795 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3796 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
3797 u32 i;
3798
3799 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_STATUS,
3800 MSR_TYPE_RW, flag);
3801 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_BASE,
3802 MSR_TYPE_RW, flag);
3803 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_MASK,
3804 MSR_TYPE_RW, flag);
3805 vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_CR3_MATCH,
3806 MSR_TYPE_RW, flag);
3807 for (i = 0; i < vmx->pt_desc.addr_range; i++) {
3808 vmx_set_intercept_for_msr(msr_bitmap,
3809 MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag);
3810 vmx_set_intercept_for_msr(msr_bitmap,
3811 MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag);
3812 }
3813}
3814
e6c67d8c
LA		 3815static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
3816{
3817 struct vcpu_vmx *vmx = to_vmx(vcpu);
3818 void *vapic_page;
3819 u32 vppr;
3820 int rvi;
3821
3822 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
3823 !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
96c66e87 3824 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn))
e6c67d8c
LA		 3825 return false;
3826
7e712684 3827 rvi = vmx_get_rvi();
e6c67d8c 3828
96c66e87 3829 vapic_page = vmx->nested.virtual_apic_map.hva;
e6c67d8c 3830 vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
e6c67d8c
LA		 3831
3832 return ((rvi & 0xf0) > (vppr & 0xf0));
3833}
3834
06a5524f
WV		 3835static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
3836 bool nested)
21bc8dc5
RK		 3837{
3838#ifdef CONFIG_SMP
06a5524f
WV		 3839 int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
3840
21bc8dc5 3841 if (vcpu->mode == IN_GUEST_MODE) {
28b835d6 3842 /*
5753743f
HZ		 3843 * The vector of interrupt to be delivered to vcpu had
3844 * been set in PIR before this function.
3845 *
3846 * Following cases will be reached in this block, and
3847 * we always send a notification event in all cases as
3848 * explained below.
3849 *
3850 * Case 1: vcpu keeps in non-root mode. Sending a
3851 * notification event posts the interrupt to vcpu.
3852 *
3853 * Case 2: vcpu exits to root mode and is still
3854 * runnable. PIR will be synced to vIRR before the
3855 * next vcpu entry. Sending a notification event in
3856 * this case has no effect, as vcpu is not in root
3857 * mode.
28b835d6 3858 *
5753743f
HZ		 3859 * Case 3: vcpu exits to root mode and is blocked.
3860 * vcpu_block() has already synced PIR to vIRR and
3861 * never blocks vcpu if vIRR is not cleared. Therefore,
3862 * a blocked vcpu here does not wait for any requested
3863 * interrupts in PIR, and sending a notification event
3864 * which has no effect is safe here.
28b835d6 3865 */
28b835d6 3866
06a5524f 3867 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
21bc8dc5
RK		 3868 return true;
3869 }
3870#endif
3871 return false;
3872}
3873
705699a1
WV		 3874static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
3875 int vector)
3876{
3877 struct vcpu_vmx *vmx = to_vmx(vcpu);
3878
3879 if (is_guest_mode(vcpu) &&
3880 vector == vmx->nested.posted_intr_nv) {
705699a1
WV		 3881 /*
3882 * If a posted intr is not recognized by hardware,
3883 * we will accomplish it in the next vmentry.
3884 */
3885 vmx->nested.pi_pending = true;
3886 kvm_make_request(KVM_REQ_EVENT, vcpu);
6b697711
LA		 3887 /* the PIR and ON have been set by L1. */
3888 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
3889 kvm_vcpu_kick(vcpu);
705699a1
WV		 3890 return 0;
3891 }
3892 return -1;
3893}
a20ed54d
YZ		 3894/*
3895 * Send interrupt to vcpu via posted interrupt way.
3896 * 1. If target vcpu is running(non-root mode), send posted interrupt
3897 * notification to vcpu and hardware will sync PIR to vIRR atomically.
3898 * 2. If target vcpu isn't running(root mode), kick it to pick up the
3899 * interrupt from PIR in next vmentry.
3900 */
91a5f413 3901static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
a20ed54d
YZ		 3902{
3903 struct vcpu_vmx *vmx = to_vmx(vcpu);
3904 int r;
3905
705699a1
WV		 3906 r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
3907 if (!r)
91a5f413
VK		 3908 return 0;
3909
3910 if (!vcpu->arch.apicv_active)
3911 return -1;
705699a1 3912
a20ed54d 3913 if (pi_test_and_set_pir(vector, &vmx->pi_desc))
91a5f413 3914 return 0;
a20ed54d 3915
b95234c8
PB		 3916 /* If a previous notification has sent the IPI, nothing to do. */
3917 if (pi_test_and_set_on(&vmx->pi_desc))
91a5f413 3918 return 0;
b95234c8 3919
06a5524f 3920 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
a20ed54d 3921 kvm_vcpu_kick(vcpu);
91a5f413
VK		 3922
3923 return 0;
a20ed54d
YZ		 3924}
3925
a3a8ff8e
NHE		 3926/*
3927 * Set up the vmcs's constant host-state fields, i.e., host-state fields that
3928 * will not change in the lifetime of the guest.
3929 * Note that host-state that does change is set elsewhere. E.g., host-state
3930 * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
3931 */
97b7ead3 3932void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
a3a8ff8e
NHE		 3933{
3934 u32 low32, high32;
3935 unsigned long tmpl;
d6e41f11 3936 unsigned long cr0, cr3, cr4;
a3a8ff8e 3937
04ac88ab
AL		 3938 cr0 = read_cr0();
3939 WARN_ON(cr0 & X86_CR0_TS);
3940 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */
d6e41f11
AL		 3941
3942 /*
3943 * Save the most likely value for this task's CR3 in the VMCS.
3944 * We can't use __get_current_cr3_fast() because we're not atomic.
3945 */
6c690ee1 3946 cr3 = __read_cr3();
d6e41f11 3947 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */
d7ee039e 3948 vmx->loaded_vmcs->host_state.cr3 = cr3;
a3a8ff8e 3949
d974baa3 3950 /* Save the most likely value for this task's CR4 in the VMCS. */
1e02ce4c 3951 cr4 = cr4_read_shadow();
d974baa3 3952 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */
d7ee039e 3953 vmx->loaded_vmcs->host_state.cr4 = cr4;
d974baa3 3954
a3a8ff8e 3955 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
b2da15ac
AK		 3956#ifdef CONFIG_X86_64
3957 /*
3958 * Load null selectors, so we can avoid reloading them in
6d6095bd
SC		 3959 * vmx_prepare_switch_to_host(), in case userspace uses
3960 * the null selectors too (the expected case).
b2da15ac
AK		 3961 */
3962 vmcs_write16(HOST_DS_SELECTOR, 0);
3963 vmcs_write16(HOST_ES_SELECTOR, 0);
3964#else
a3a8ff8e
NHE		 3965 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
3966 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
b2da15ac 3967#endif
a3a8ff8e
NHE		 3968 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
3969 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
3970
2342080c 3971 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */
a3a8ff8e 3972
453eafbe 3973 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */
a3a8ff8e
NHE		 3974
3975 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
3976 vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
3977 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
3978 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */
3979
3980 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
3981 rdmsr(MSR_IA32_CR_PAT, low32, high32);
3982 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
3983 }
5a5e8a15 3984
c73da3fc 3985 if (cpu_has_load_ia32_efer())
5a5e8a15 3986 vmcs_write64(HOST_IA32_EFER, host_efer);
a3a8ff8e
NHE		 3987}
3988
97b7ead3 3989void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
bf8179a0
NHE		 3990{
3991 vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
3992 if (enable_ept)
3993 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
fe3ef05c
NHE		 3994 if (is_guest_mode(&vmx->vcpu))
3995 vmx->vcpu.arch.cr4_guest_owned_bits &=
3996 ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
bf8179a0
NHE		 3997 vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
3998}
3999
c075c3e4 4000u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
01e439be
YZ		 4001{
4002 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
4003
d62caabb 4004 if (!kvm_vcpu_apicv_active(&vmx->vcpu))
01e439be 4005 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
d02fcf50
PB		 4006
4007 if (!enable_vnmi)
4008 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS;
4009
804939ea
SC		 4010 if (!enable_preemption_timer)
4011 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
4012
01e439be
YZ		 4013 return pin_based_exec_ctrl;
4014}
4015
d62caabb
AS		 4016static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
4017{
4018 struct vcpu_vmx *vmx = to_vmx(vcpu);
4019
c5f2c766 4020 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
3ce424e4
RK		 4021 if (cpu_has_secondary_exec_ctrls()) {
4022 if (kvm_vcpu_apicv_active(vcpu))
fe7f895d 4023 secondary_exec_controls_setbit(vmx,
3ce424e4
RK		 4024 SECONDARY_EXEC_APIC_REGISTER_VIRT |
4025 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4026 else
fe7f895d 4027 secondary_exec_controls_clearbit(vmx,
3ce424e4
RK		 4028 SECONDARY_EXEC_APIC_REGISTER_VIRT |
4029 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4030 }
4031
4032 if (cpu_has_vmx_msr_bitmap())
904e14fb 4033 vmx_update_msr_bitmap(vcpu);
d62caabb
AS		 4034}
4035
89b0c9f5
SC		 4036u32 vmx_exec_control(struct vcpu_vmx *vmx)
4037{
4038 u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
4039
4040 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
4041 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
4042
4043 if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
4044 exec_control &= ~CPU_BASED_TPR_SHADOW;
4045#ifdef CONFIG_X86_64
4046 exec_control |= CPU_BASED_CR8_STORE_EXITING |
4047 CPU_BASED_CR8_LOAD_EXITING;
4048#endif
4049 }
4050 if (!enable_ept)
4051 exec_control |= CPU_BASED_CR3_STORE_EXITING |
4052 CPU_BASED_CR3_LOAD_EXITING |
4053 CPU_BASED_INVLPG_EXITING;
4054 if (kvm_mwait_in_guest(vmx->vcpu.kvm))
4055 exec_control &= ~(CPU_BASED_MWAIT_EXITING |
4056 CPU_BASED_MONITOR_EXITING);
4057 if (kvm_hlt_in_guest(vmx->vcpu.kvm))
4058 exec_control &= ~CPU_BASED_HLT_EXITING;
4059 return exec_control;
4060}
4061
4062
80154d77 4063static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
bf8179a0 4064{
80154d77
PB		 4065 struct kvm_vcpu *vcpu = &vmx->vcpu;
4066
bf8179a0 4067 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
0367f205 4068
2ef7619d 4069 if (vmx_pt_mode_is_system())
f99e3daf 4070 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX);
80154d77 4071 if (!cpu_need_virtualize_apic_accesses(vcpu))
bf8179a0
NHE		 4072 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
4073 if (vmx->vpid == 0)
4074 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
4075 if (!enable_ept) {
4076 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
4077 enable_unrestricted_guest = 0;
4078 }
4079 if (!enable_unrestricted_guest)
4080 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
b31c114b 4081 if (kvm_pause_in_guest(vmx->vcpu.kvm))
bf8179a0 4082 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
80154d77 4083 if (!kvm_vcpu_apicv_active(vcpu))
c7c9c56c
YZ		 4084 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
4085 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
8d14695f 4086 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
0367f205
PB		 4087
4088 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
4089 * in vmx_set_cr4. */
4090 exec_control &= ~SECONDARY_EXEC_DESC;
4091
abc4fc58
AG		 4092 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
4093 (handle_vmptrld).
4094 We can NOT enable shadow_vmcs here because we don't have yet
4095 a current VMCS12
4096 */
4097 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
a3eaa864
KH		 4098
4099 if (!enable_pml)
4100 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
843e4330 4101
3db13480
PB		 4102 if (vmx_xsaves_supported()) {
4103 /* Exposing XSAVES only when XSAVE is exposed */
4104 bool xsaves_enabled =
96be4e06 4105 boot_cpu_has(X86_FEATURE_XSAVE) &&
3db13480
PB		 4106 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4107 guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
4108
7204160e
AL		 4109 vcpu->arch.xsaves_enabled = xsaves_enabled;
4110
3db13480
PB		 4111 if (!xsaves_enabled)
4112 exec_control &= ~SECONDARY_EXEC_XSAVES;
4113
4114 if (nested) {
4115 if (xsaves_enabled)
6677f3da 4116 vmx->nested.msrs.secondary_ctls_high |=
3db13480
PB		 4117 SECONDARY_EXEC_XSAVES;
4118 else
6677f3da 4119 vmx->nested.msrs.secondary_ctls_high &=
3db13480
PB		 4120 ~SECONDARY_EXEC_XSAVES;
4121 }
4122 }
4123
a7a200eb 4124 if (cpu_has_vmx_rdtscp()) {
80154d77
PB		 4125 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP);
4126 if (!rdtscp_enabled)
4127 exec_control &= ~SECONDARY_EXEC_RDTSCP;
4128
4129 if (nested) {
4130 if (rdtscp_enabled)
6677f3da 4131 vmx->nested.msrs.secondary_ctls_high |=
80154d77
PB		 4132 SECONDARY_EXEC_RDTSCP;
4133 else
6677f3da 4134 vmx->nested.msrs.secondary_ctls_high &=
80154d77
PB		 4135 ~SECONDARY_EXEC_RDTSCP;
4136 }
4137 }
4138
5ffec6f9 4139 if (cpu_has_vmx_invpcid()) {
80154d77
PB		 4140 /* Exposing INVPCID only when PCID is exposed */
4141 bool invpcid_enabled =
4142 guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) &&
4143 guest_cpuid_has(vcpu, X86_FEATURE_PCID);
4144
4145 if (!invpcid_enabled) {
4146 exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
4147 guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
4148 }
4149
4150 if (nested) {
4151 if (invpcid_enabled)
6677f3da 4152 vmx->nested.msrs.secondary_ctls_high |=
80154d77
PB		 4153 SECONDARY_EXEC_ENABLE_INVPCID;
4154 else
6677f3da 4155 vmx->nested.msrs.secondary_ctls_high &=
80154d77
PB		 4156 ~SECONDARY_EXEC_ENABLE_INVPCID;
4157 }
4158 }
4159
45ec368c
JM		 4160 if (vmx_rdrand_supported()) {
4161 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND);
4162 if (rdrand_enabled)
736fdf72 4163 exec_control &= ~SECONDARY_EXEC_RDRAND_EXITING;
45ec368c
JM		 4164
4165 if (nested) {
4166 if (rdrand_enabled)
6677f3da 4167 vmx->nested.msrs.secondary_ctls_high |=
736fdf72 4168 SECONDARY_EXEC_RDRAND_EXITING;
45ec368c 4169 else
6677f3da 4170 vmx->nested.msrs.secondary_ctls_high &=
736fdf72 4171 ~SECONDARY_EXEC_RDRAND_EXITING;
45ec368c
JM		 4172 }
4173 }
4174
75f4fc8d
JM		 4175 if (vmx_rdseed_supported()) {
4176 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED);
4177 if (rdseed_enabled)
736fdf72 4178 exec_control &= ~SECONDARY_EXEC_RDSEED_EXITING;
75f4fc8d
JM		 4179
4180 if (nested) {
4181 if (rdseed_enabled)
6677f3da 4182 vmx->nested.msrs.secondary_ctls_high |=
736fdf72 4183 SECONDARY_EXEC_RDSEED_EXITING;
75f4fc8d 4184 else
6677f3da 4185 vmx->nested.msrs.secondary_ctls_high &=
736fdf72 4186 ~SECONDARY_EXEC_RDSEED_EXITING;
75f4fc8d
JM		 4187 }
4188 }
4189
e69e72fa
TX		 4190 if (vmx_waitpkg_supported()) {
4191 bool waitpkg_enabled =
4192 guest_cpuid_has(vcpu, X86_FEATURE_WAITPKG);
4193
4194 if (!waitpkg_enabled)
4195 exec_control &= ~SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
4196
4197 if (nested) {
4198 if (waitpkg_enabled)
4199 vmx->nested.msrs.secondary_ctls_high |=
4200 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
4201 else
4202 vmx->nested.msrs.secondary_ctls_high &=
4203 ~SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
4204 }
4205 }
4206
80154d77 4207 vmx->secondary_exec_control = exec_control;
bf8179a0
NHE		 4208}
4209
ce88decf
XG		 4210static void ept_set_mmio_spte_mask(void)
4211{
4212 /*
4213 * EPT Misconfigurations can be generated if the value of bits 2:0
4214 * of an EPT paging-structure entry is 110b (write/execute).
ce88decf 4215 */
dcdca5fe 4216 kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
4af77151 4217 VMX_EPT_MISCONFIG_WX_VALUE, 0);
ce88decf
XG		 4218}
4219
f53cd63c 4220#define VMX_XSS_EXIT_BITMAP 0
6aa8b732 4221
944c3464 4222/*
1b84292b
XL		 4223 * Noting that the initialization of Guest-state Area of VMCS is in
4224 * vmx_vcpu_reset().
944c3464 4225 */
1b84292b 4226static void init_vmcs(struct vcpu_vmx *vmx)
944c3464 4227{
944c3464 4228 if (nested)
1b84292b 4229 nested_vmx_set_vmcs_shadowing_bitmap();
944c3464 4230
25c5f225 4231 if (cpu_has_vmx_msr_bitmap())
904e14fb 4232 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
25c5f225 4233
6aa8b732
AK		 4234 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
4235
6aa8b732 4236 /* Control */
3af80fec 4237 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
6e5d865c 4238
3af80fec 4239 exec_controls_set(vmx, vmx_exec_control(vmx));
6aa8b732 4240
dfa169bb 4241 if (cpu_has_secondary_exec_ctrls()) {
80154d77 4242 vmx_compute_secondary_exec_control(vmx);
3af80fec 4243 secondary_exec_controls_set(vmx, vmx->secondary_exec_control);
dfa169bb 4244 }
f78e0e2e 4245
d62caabb 4246 if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
c7c9c56c
YZ		 4247 vmcs_write64(EOI_EXIT_BITMAP0, 0);
4248 vmcs_write64(EOI_EXIT_BITMAP1, 0);
4249 vmcs_write64(EOI_EXIT_BITMAP2, 0);
4250 vmcs_write64(EOI_EXIT_BITMAP3, 0);
4251
4252 vmcs_write16(GUEST_INTR_STATUS, 0);
01e439be 4253
0bcf261c 4254 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
01e439be 4255 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
c7c9c56c
YZ		 4256 }
4257
b31c114b 4258 if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
4b8d54f9 4259 vmcs_write32(PLE_GAP, ple_gap);
a7653ecd
RK		 4260 vmx->ple_window = ple_window;
4261 vmx->ple_window_dirty = true;
4b8d54f9
ZE		 4262 }
4263
c3707958
XG		 4264 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
4265 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
6aa8b732
AK		 4266 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
4267
9581d442
AK		 4268 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */
4269 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */
a547c6db 4270 vmx_set_constant_host_state(vmx);
6aa8b732
AK		 4271 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
4272 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
6aa8b732 4273
2a499e49
BD		 4274 if (cpu_has_vmx_vmfunc())
4275 vmcs_write64(VM_FUNCTION_CONTROL, 0);
4276
2cc51560
ED		 4277 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
4278 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
33966dd6 4279 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
2cc51560 4280 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
33966dd6 4281 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
6aa8b732 4282
74545705
RK		 4283 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
4284 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
468d472f 4285
3af80fec 4286 vm_exit_controls_set(vmx, vmx_vmexit_ctrl());
6aa8b732
AK		 4287
4288 /* 22.2.1, 20.8.1 */
3af80fec 4289 vm_entry_controls_set(vmx, vmx_vmentry_ctrl());
1c3d14fe 4290
bd7e5b08
PB		 4291 vmx->vcpu.arch.cr0_guest_owned_bits = X86_CR0_TS;
4292 vmcs_writel(CR0_GUEST_HOST_MASK, ~X86_CR0_TS);
4293
bf8179a0 4294 set_cr4_guest_host_mask(vmx);
e00c8cf2 4295
35fbe0d4
XL		 4296 if (vmx->vpid != 0)
4297 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
4298
f53cd63c
WL		 4299 if (vmx_xsaves_supported())
4300 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
4301
4e59516a 4302 if (enable_pml) {
4e59516a
PF		 4303 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
4304 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
4305 }
0b665d30
SC		 4306
4307 if (cpu_has_vmx_encls_vmexit())
4308 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
2ef444f1 4309
2ef7619d 4310 if (vmx_pt_mode_is_host_guest()) {
2ef444f1
CP		 4311 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
4312 /* Bit[6~0] are forced to 1, writes are ignored. */
4313 vmx->pt_desc.guest.output_mask = 0x7F;
4314 vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
4315 }
e00c8cf2
AK		 4316}
4317
d28bc9dd 4318static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
e00c8cf2
AK		 4319{
4320 struct vcpu_vmx *vmx = to_vmx(vcpu);
58cb628d 4321 struct msr_data apic_base_msr;
d28bc9dd 4322 u64 cr0;
e00c8cf2 4323
7ffd92c5 4324 vmx->rmode.vm86_active = 0;
d28b387f 4325 vmx->spec_ctrl = 0;
e00c8cf2 4326
6e3ba4ab
TX		 4327 vmx->msr_ia32_umwait_control = 0;
4328
ad312c7c 4329 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
95c06540 4330 vmx->hv_deadline_tsc = -1;
d28bc9dd
NA		 4331 kvm_set_cr8(vcpu, 0);
4332
4333 if (!init_event) {
4334 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
4335 MSR_IA32_APICBASE_ENABLE;
4336 if (kvm_vcpu_is_reset_bsp(vcpu))
4337 apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
4338 apic_base_msr.host_initiated = true;
4339 kvm_set_apic_base(vcpu, &apic_base_msr);
4340 }
e00c8cf2 4341
2fb92db1
AK		 4342 vmx_segment_cache_clear(vmx);
4343
5706be0d 4344 seg_setup(VCPU_SREG_CS);
66450a21 4345 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
f3531054 4346 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
e00c8cf2
AK		 4347
4348 seg_setup(VCPU_SREG_DS);
4349 seg_setup(VCPU_SREG_ES);
4350 seg_setup(VCPU_SREG_FS);
4351 seg_setup(VCPU_SREG_GS);
4352 seg_setup(VCPU_SREG_SS);
4353
4354 vmcs_write16(GUEST_TR_SELECTOR, 0);
4355 vmcs_writel(GUEST_TR_BASE, 0);
4356 vmcs_write32(GUEST_TR_LIMIT, 0xffff);
4357 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4358
4359 vmcs_write16(GUEST_LDTR_SELECTOR, 0);
4360 vmcs_writel(GUEST_LDTR_BASE, 0);
4361 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
4362 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
4363
d28bc9dd
NA		 4364 if (!init_event) {
4365 vmcs_write32(GUEST_SYSENTER_CS, 0);
4366 vmcs_writel(GUEST_SYSENTER_ESP, 0);
4367 vmcs_writel(GUEST_SYSENTER_EIP, 0);
4368 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4369 }
e00c8cf2 4370
c37c2873 4371 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
66450a21 4372 kvm_rip_write(vcpu, 0xfff0);
e00c8cf2 4373
e00c8cf2
AK		 4374 vmcs_writel(GUEST_GDTR_BASE, 0);
4375 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
4376
4377 vmcs_writel(GUEST_IDTR_BASE, 0);
4378 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
4379
443381a8 4380 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
e00c8cf2 4381 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
f3531054 4382 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
a554d207
WL		 4383 if (kvm_mpx_supported())
4384 vmcs_write64(GUEST_BNDCFGS, 0);
e00c8cf2 4385
e00c8cf2
AK		 4386 setup_msrs(vmx);
4387
6aa8b732
AK		 4388 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
4389
d28bc9dd 4390 if (cpu_has_vmx_tpr_shadow() && !init_event) {
f78e0e2e 4391 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
35754c98 4392 if (cpu_need_tpr_shadow(vcpu))
f78e0e2e 4393 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
d28bc9dd 4394 __pa(vcpu->arch.apic->regs));
f78e0e2e
SY		 4395 vmcs_write32(TPR_THRESHOLD, 0);
4396 }
4397
a73896cb 4398 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
6aa8b732 4399
d28bc9dd 4400 cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
d28bc9dd 4401 vmx->vcpu.arch.cr0 = cr0;
f2463247 4402 vmx_set_cr0(vcpu, cr0); /* enter rmode */
d28bc9dd 4403 vmx_set_cr4(vcpu, 0);
5690891b 4404 vmx_set_efer(vcpu, 0);
bd7e5b08 4405
d28bc9dd 4406 update_exception_bitmap(vcpu);
6aa8b732 4407
dd5f5341 4408 vpid_sync_context(vmx->vpid);
caa057a2
WL		 4409 if (init_event)
4410 vmx_clear_hlt(vcpu);
6aa8b732
AK		 4411}
4412
55d2375e 4413static void enable_irq_window(struct kvm_vcpu *vcpu)
3b86cd99 4414{
9dadc2f9 4415 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
3b86cd99
JK		 4416}
4417
c9a7953f 4418static void enable_nmi_window(struct kvm_vcpu *vcpu)
3b86cd99 4419{
d02fcf50 4420 if (!enable_vnmi ||
8a1b4392 4421 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
c9a7953f
JK		 4422 enable_irq_window(vcpu);
4423 return;
4424 }
3b86cd99 4425
4e2a0bc5 4426 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
3b86cd99
JK		 4427}
4428
66fd3f7f 4429static void vmx_inject_irq(struct kvm_vcpu *vcpu)
85f455f7 4430{
9c8cba37 4431 struct vcpu_vmx *vmx = to_vmx(vcpu);
66fd3f7f
GN		 4432 uint32_t intr;
4433 int irq = vcpu->arch.interrupt.nr;
9c8cba37 4434
229456fc 4435 trace_kvm_inj_virq(irq);
2714d1d3 4436
fa89a817 4437 ++vcpu->stat.irq_injections;
7ffd92c5 4438 if (vmx->rmode.vm86_active) {
71f9833b
SH		 4439 int inc_eip = 0;
4440 if (vcpu->arch.interrupt.soft)
4441 inc_eip = vcpu->arch.event_exit_inst_len;
9497e1f2 4442 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip);
85f455f7
ED		 4443 return;
4444 }
66fd3f7f
GN		 4445 intr = irq | INTR_INFO_VALID_MASK;
4446 if (vcpu->arch.interrupt.soft) {
4447 intr |= INTR_TYPE_SOFT_INTR;
4448 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
4449 vmx->vcpu.arch.event_exit_inst_len);
4450 } else
4451 intr |= INTR_TYPE_EXT_INTR;
4452 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
caa057a2
WL		 4453
4454 vmx_clear_hlt(vcpu);
85f455f7
ED		 4455}
4456
f08864b4
SY		 4457static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
4458{
66a5a347
JK		 4459 struct vcpu_vmx *vmx = to_vmx(vcpu);
4460
d02fcf50 4461 if (!enable_vnmi) {
8a1b4392
PB		 4462 /*
4463 * Tracking the NMI-blocked state in software is built upon
4464 * finding the next open IRQ window. This, in turn, depends on
4465 * well-behaving guests: They have to keep IRQs disabled at
4466 * least as long as the NMI handler runs. Otherwise we may
4467 * cause NMI nesting, maybe breaking the guest. But as this is
4468 * highly unlikely, we can live with the residual risk.
4469 */
4470 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
4471 vmx->loaded_vmcs->vnmi_blocked_time = 0;
4472 }
4473
4c4a6f79
PB		 4474 ++vcpu->stat.nmi_injections;
4475 vmx->loaded_vmcs->nmi_known_unmasked = false;
3b86cd99 4476
7ffd92c5 4477 if (vmx->rmode.vm86_active) {
9497e1f2 4478 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0);
66a5a347
JK		 4479 return;
4480 }
c5a6d5f7 4481
f08864b4
SY		 4482 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
4483 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
caa057a2
WL		 4484
4485 vmx_clear_hlt(vcpu);
f08864b4
SY		 4486}
4487
97b7ead3 4488bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
3cfc3092 4489{
4c4a6f79
PB		 4490 struct vcpu_vmx *vmx = to_vmx(vcpu);
4491 bool masked;
4492
d02fcf50 4493 if (!enable_vnmi)
8a1b4392 4494 return vmx->loaded_vmcs->soft_vnmi_blocked;
4c4a6f79 4495 if (vmx->loaded_vmcs->nmi_known_unmasked)
9d58b931 4496 return false;
4c4a6f79
PB		 4497 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
4498 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4499 return masked;
3cfc3092
JK		 4500}
4501
97b7ead3 4502void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3cfc3092
JK		 4503{
4504 struct vcpu_vmx *vmx = to_vmx(vcpu);
4505
d02fcf50 4506 if (!enable_vnmi) {
8a1b4392
PB		 4507 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
4508 vmx->loaded_vmcs->soft_vnmi_blocked = masked;
4509 vmx->loaded_vmcs->vnmi_blocked_time = 0;
4510 }
4511 } else {
4512 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4513 if (masked)
4514 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
4515 GUEST_INTR_STATE_NMI);
4516 else
4517 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
4518 GUEST_INTR_STATE_NMI);
4519 }
3cfc3092
JK		 4520}
4521
1b660b6b
SC		 4522bool vmx_nmi_blocked(struct kvm_vcpu *vcpu)
4523{
4524 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
4525 return false;
4526
4527 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
4528 return true;
4529
4530 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4531 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI |
4532 GUEST_INTR_STATE_NMI));
4533}
4534
c300ab9f 4535static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
2505dc9f 4536{
b6b8a145 4537 if (to_vmx(vcpu)->nested.nested_run_pending)
88c604b6 4538 return false;
ea8ceb83 4539
c300ab9f
PB		 4540 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */
4541 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
4542 return false;
4543
1b660b6b
SC		 4544 return !vmx_nmi_blocked(vcpu);
4545}
429ab576 4546
1b660b6b
SC		 4547bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu)
4548{
4549 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
88c604b6 4550 return false;
8a1b4392 4551
7ab0abdb 4552 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) ||
1b660b6b
SC		 4553 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4554 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
2505dc9f
JK		 4555}
4556
c300ab9f 4557static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
78646121 4558{
a1c77abb
SC		 4559 if (to_vmx(vcpu)->nested.nested_run_pending)
4560 return false;
4561
c300ab9f
PB		 4562 /*
4563 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
4564 * e.g. if the IRQ arrived asynchronously after checking nested events.
4565 */
4566 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
4567 return false;
4568
1b660b6b 4569 return !vmx_interrupt_blocked(vcpu);
78646121
GN		 4570}
4571
cbc94022
IE		 4572static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
4573{
4574 int ret;
cbc94022 4575
f7eaeb0a
SC		 4576 if (enable_unrestricted_guest)
4577 return 0;
4578
6a3c623b
PX		 4579 mutex_lock(&kvm->slots_lock);
4580 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
4581 PAGE_SIZE * 3);
4582 mutex_unlock(&kvm->slots_lock);
4583
cbc94022
IE		 4584 if (ret)
4585 return ret;
40bbb9d0 4586 to_kvm_vmx(kvm)->tss_addr = addr;
1f755a82 4587 return init_rmode_tss(kvm);
cbc94022
IE		 4588}
4589
2ac52ab8
SC		 4590static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
4591{
40bbb9d0 4592 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
2ac52ab8
SC		 4593 return 0;
4594}
4595
0ca1b4f4 4596static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
6aa8b732 4597{
77ab6db0 4598 switch (vec) {
77ab6db0 4599 case BP_VECTOR:
c573cd22
JK		 4600 /*
4601 * Update instruction length as we may reinject the exception
4602 * from user space while in guest debugging mode.
4603 */
4604 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
4605 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
d0bfb940 4606 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
0ca1b4f4
GN		 4607 return false;
4608 /* fall through */
4609 case DB_VECTOR:
4610 if (vcpu->guest_debug &
4611 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
4612 return false;
d0bfb940
JK		 4613 /* fall through */
4614 case DE_VECTOR:
77ab6db0
JK		 4615 case OF_VECTOR:
4616 case BR_VECTOR:
4617 case UD_VECTOR:
4618 case DF_VECTOR:
4619 case SS_VECTOR:
4620 case GP_VECTOR:
4621 case MF_VECTOR:
0ca1b4f4 4622 return true;
77ab6db0 4623 }
0ca1b4f4
GN		 4624 return false;
4625}
4626
4627static int handle_rmode_exception(struct kvm_vcpu *vcpu,
4628 int vec, u32 err_code)
4629{
4630 /*
4631 * Instruction with address size override prefix opcode 0x67
4632 * Cause the #SS fault with 0 error code in VM86 mode.
4633 */
4634 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
60fc3d02 4635 if (kvm_emulate_instruction(vcpu, 0)) {
0ca1b4f4
GN		 4636 if (vcpu->arch.halt_request) {
4637 vcpu->arch.halt_request = 0;
5cb56059 4638 return kvm_vcpu_halt(vcpu);
0ca1b4f4
GN		 4639 }
4640 return 1;
4641 }
4642 return 0;
4643 }
4644
4645 /*
4646 * Forward all other exceptions that are valid in real mode.
4647 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
4648 * the required debugging infrastructure rework.
4649 */
4650 kvm_queue_exception(vcpu, vec);
4651 return 1;
6aa8b732
AK		 4652}
4653
a0861c02
AK		 4654/*
4655 * Trigger machine check on the host. We assume all the MSRs are already set up
4656 * by the CPU and that we still run on the same CPU as the MCE occurred on.
4657 * We pass a fake environment to the machine check handler because we want
4658 * the guest to be always treated like user space, no matter what context
4659 * it used internally.
4660 */
4661static void kvm_machine_check(void)
4662{
fb56baae 4663#if defined(CONFIG_X86_MCE)
a0861c02
AK		 4664 struct pt_regs regs = {
4665 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
4666 .flags = X86_EFLAGS_IF,
4667 };
4668
4669 do_machine_check(&regs, 0);
4670#endif
4671}
4672
851ba692 4673static int handle_machine_check(struct kvm_vcpu *vcpu)
a0861c02 4674{
95b5a48c 4675 /* handled by vmx_vcpu_run() */
a0861c02
AK		 4676 return 1;
4677}
4678
e6f8b6c1
XL		 4679/*
4680 * If the host has split lock detection disabled, then #AC is
4681 * unconditionally injected into the guest, which is the pre split lock
4682 * detection behaviour.
4683 *
4684 * If the host has split lock detection enabled then #AC is
4685 * only injected into the guest when:
4686 * - Guest CPL == 3 (user mode)
4687 * - Guest has #AC detection enabled in CR0
4688 * - Guest EFLAGS has AC bit set
4689 */
4690static inline bool guest_inject_ac(struct kvm_vcpu *vcpu)
4691{
4692 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
4693 return true;
4694
4695 return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) &&
4696 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC);
4697}
4698
95b5a48c 4699static int handle_exception_nmi(struct kvm_vcpu *vcpu)
6aa8b732 4700{
1155f76a 4701 struct vcpu_vmx *vmx = to_vmx(vcpu);
851ba692 4702 struct kvm_run *kvm_run = vcpu->run;
d0bfb940 4703 u32 intr_info, ex_no, error_code;
42dbaa5a 4704 unsigned long cr2, rip, dr6;
6aa8b732 4705 u32 vect_info;
6aa8b732 4706
1155f76a 4707 vect_info = vmx->idt_vectoring_info;
88786475 4708 intr_info = vmx->exit_intr_info;
6aa8b732 4709
2ea72039 4710 if (is_machine_check(intr_info) || is_nmi(intr_info))
95b5a48c 4711 return 1; /* handled by handle_exception_nmi_irqoff() */
2ab455cc 4712
082d06ed
WL		 4713 if (is_invalid_opcode(intr_info))
4714 return handle_ud(vcpu);
7aa81cc0 4715
6aa8b732 4716 error_code = 0;
2e11384c 4717 if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
6aa8b732 4718 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
bf4ca23e 4719
9e869480
LA		 4720 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
4721 WARN_ON_ONCE(!enable_vmware_backdoor);
a6c6ed1e
SC		 4722
4723 /*
4724 * VMware backdoor emulation on #GP interception only handles
4725 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero
4726 * error code on #GP.
4727 */
4728 if (error_code) {
4729 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
4730 return 1;
4731 }
60fc3d02 4732 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
9e869480
LA		 4733 }
4734
bf4ca23e
XG		 4735 /*
4736 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
4737 * MMIO, it is better to report an internal error.
4738 * See the comments in vmx_handle_exit.
4739 */
4740 if ((vect_info & VECTORING_INFO_VALID_MASK) &&
4741 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
4742 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4743 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
80f0e95d 4744 vcpu->run->internal.ndata = 3;
bf4ca23e
XG		 4745 vcpu->run->internal.data[0] = vect_info;
4746 vcpu->run->internal.data[1] = intr_info;
80f0e95d 4747 vcpu->run->internal.data[2] = error_code;
bf4ca23e
XG		 4748 return 0;
4749 }
4750
6aa8b732 4751 if (is_page_fault(intr_info)) {
5addc235 4752 cr2 = vmx_get_exit_qual(vcpu);
1261bfa3
WL		 4753 /* EPT won't cause page fault directly */
4754 WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept);
d0006530 4755 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
6aa8b732
AK		 4756 }
4757
d0bfb940 4758 ex_no = intr_info & INTR_INFO_VECTOR_MASK;
0ca1b4f4
GN		 4759
4760 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
4761 return handle_rmode_exception(vcpu, ex_no, error_code);
4762
42dbaa5a
JK		 4763 switch (ex_no) {
4764 case DB_VECTOR:
5addc235 4765 dr6 = vmx_get_exit_qual(vcpu);
42dbaa5a
JK		 4766 if (!(vcpu->guest_debug &
4767 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
32d43cd3 4768 if (is_icebp(intr_info))
1957aa63 4769 WARN_ON(!skip_emulated_instruction(vcpu));
fd2a445a 4770
4d5523cf 4771 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
42dbaa5a
JK		 4772 return 1;
4773 }
13196638 4774 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
42dbaa5a
JK		 4775 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4776 /* fall through */
4777 case BP_VECTOR:
c573cd22
JK		 4778 /*
4779 * Update instruction length as we may reinject #BP from
4780 * user space while in guest debugging mode. Reading it for
4781 * #DB as well causes no harm, it is not used in that case.
4782 */
4783 vmx->vcpu.arch.event_exit_inst_len =
4784 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
6aa8b732 4785 kvm_run->exit_reason = KVM_EXIT_DEBUG;
0a434bb2 4786 rip = kvm_rip_read(vcpu);
d0bfb940
JK		 4787 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
4788 kvm_run->debug.arch.exception = ex_no;
42dbaa5a 4789 break;
e6f8b6c1
XL		 4790 case AC_VECTOR:
4791 if (guest_inject_ac(vcpu)) {
4792 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
4793 return 1;
4794 }
4795
4796 /*
4797 * Handle split lock. Depending on detection mode this will
4798 * either warn and disable split lock detection for this
4799 * task or force SIGBUS on it.
4800 */
4801 if (handle_guest_split_lock(kvm_rip_read(vcpu)))
4802 return 1;
4803 fallthrough;
42dbaa5a 4804 default:
d0bfb940
JK		 4805 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
4806 kvm_run->ex.exception = ex_no;
4807 kvm_run->ex.error_code = error_code;
42dbaa5a 4808 break;
6aa8b732 4809 }
6aa8b732
AK		 4810 return 0;
4811}
4812
f399e60c 4813static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu)
6aa8b732 4814{
1165f5fe 4815 ++vcpu->stat.irq_exits;
6aa8b732
AK		 4816 return 1;
4817}
4818
851ba692 4819static int handle_triple_fault(struct kvm_vcpu *vcpu)
988ad74f 4820{
851ba692 4821 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
bbeac283 4822 vcpu->mmio_needed = 0;
988ad74f
AK		 4823 return 0;
4824}
6aa8b732 4825
851ba692 4826static int handle_io(struct kvm_vcpu *vcpu)
6aa8b732 4827{
bfdaab09 4828 unsigned long exit_qualification;
dca7f128 4829 int size, in, string;
039576c0 4830 unsigned port;
6aa8b732 4831
5addc235 4832 exit_qualification = vmx_get_exit_qual(vcpu);
039576c0 4833 string = (exit_qualification & 16) != 0;
e70669ab 4834
cf8f70bf 4835 ++vcpu->stat.io_exits;
e70669ab 4836
432baf60 4837 if (string)
60fc3d02 4838 return kvm_emulate_instruction(vcpu, 0);
e70669ab 4839
cf8f70bf
GN		 4840 port = exit_qualification >> 16;
4841 size = (exit_qualification & 7) + 1;
432baf60 4842 in = (exit_qualification & 8) != 0;
cf8f70bf 4843
dca7f128 4844 return kvm_fast_pio(vcpu, size, port, in);
6aa8b732
AK		 4845}
4846
102d8325
IM		 4847static void
4848vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4849{
4850 /*
4851 * Patch in the VMCALL instruction:
4852 */
4853 hypercall[0] = 0x0f;
4854 hypercall[1] = 0x01;
4855 hypercall[2] = 0xc1;
102d8325
IM		 4856}
4857
0fa06071 4858/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
eeadf9e7
NHE		 4859static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
4860{
eeadf9e7 4861 if (is_guest_mode(vcpu)) {
1a0d74e6
JK		 4862 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4863 unsigned long orig_val = val;
4864
eeadf9e7
NHE		 4865 /*
4866 * We get here when L2 changed cr0 in a way that did not change
4867 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
1a0d74e6
JK		 4868 * but did change L0 shadowed bits. So we first calculate the
4869 * effective cr0 value that L1 would like to write into the
4870 * hardware. It consists of the L2-owned bits from the new
4871 * value combined with the L1-owned bits from L1's guest_cr0.
eeadf9e7 4872 */
1a0d74e6
JK		 4873 val = (val & ~vmcs12->cr0_guest_host_mask) |
4874 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
4875
3899152c 4876 if (!nested_guest_cr0_valid(vcpu, val))
eeadf9e7 4877 return 1;
1a0d74e6
JK		 4878
4879 if (kvm_set_cr0(vcpu, val))
4880 return 1;
4881 vmcs_writel(CR0_READ_SHADOW, orig_val);
eeadf9e7 4882 return 0;
1a0d74e6
JK		 4883 } else {
4884 if (to_vmx(vcpu)->nested.vmxon &&
3899152c 4885 !nested_host_cr0_valid(vcpu, val))
1a0d74e6 4886 return 1;
3899152c 4887
eeadf9e7 4888 return kvm_set_cr0(vcpu, val);
1a0d74e6 4889 }
eeadf9e7
NHE		 4890}
4891
4892static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
4893{
4894 if (is_guest_mode(vcpu)) {
1a0d74e6
JK		 4895 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4896 unsigned long orig_val = val;
4897
4898 /* analogously to handle_set_cr0 */
4899 val = (val & ~vmcs12->cr4_guest_host_mask) |
4900 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
4901 if (kvm_set_cr4(vcpu, val))
eeadf9e7 4902 return 1;
1a0d74e6 4903 vmcs_writel(CR4_READ_SHADOW, orig_val);
eeadf9e7
NHE		 4904 return 0;
4905 } else
4906 return kvm_set_cr4(vcpu, val);
4907}
4908
0367f205
PB		 4909static int handle_desc(struct kvm_vcpu *vcpu)
4910{
4911 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
60fc3d02 4912 return kvm_emulate_instruction(vcpu, 0);
0367f205
PB		 4913}
4914
851ba692 4915static int handle_cr(struct kvm_vcpu *vcpu)
6aa8b732 4916{
229456fc 4917 unsigned long exit_qualification, val;
6aa8b732
AK		 4918 int cr;
4919 int reg;
49a9b07e 4920 int err;
6affcbed 4921 int ret;
6aa8b732 4922
5addc235 4923 exit_qualification = vmx_get_exit_qual(vcpu);
6aa8b732
AK		 4924 cr = exit_qualification & 15;
4925 reg = (exit_qualification >> 8) & 15;
4926 switch ((exit_qualification >> 4) & 3) {
4927 case 0: /* mov to cr */
1e32c079 4928 val = kvm_register_readl(vcpu, reg);
229456fc 4929 trace_kvm_cr_write(cr, val);
6aa8b732
AK		 4930 switch (cr) {
4931 case 0:
eeadf9e7 4932 err = handle_set_cr0(vcpu, val);
6affcbed 4933 return kvm_complete_insn_gp(vcpu, err);
6aa8b732 4934 case 3:
e1de91cc 4935 WARN_ON_ONCE(enable_unrestricted_guest);
2390218b 4936 err = kvm_set_cr3(vcpu, val);
6affcbed 4937 return kvm_complete_insn_gp(vcpu, err);
6aa8b732 4938 case 4:
eeadf9e7 4939 err = handle_set_cr4(vcpu, val);
6affcbed 4940 return kvm_complete_insn_gp(vcpu, err);
0a5fff19
GN		 4941 case 8: {
4942 u8 cr8_prev = kvm_get_cr8(vcpu);
1e32c079 4943 u8 cr8 = (u8)val;
eea1cff9 4944 err = kvm_set_cr8(vcpu, cr8);
6affcbed 4945 ret = kvm_complete_insn_gp(vcpu, err);
35754c98 4946 if (lapic_in_kernel(vcpu))
6affcbed 4947 return ret;
0a5fff19 4948 if (cr8_prev <= cr8)
6affcbed
KH		 4949 return ret;
4950 /*
4951 * TODO: we might be squashing a
4952 * KVM_GUESTDBG_SINGLESTEP-triggered
4953 * KVM_EXIT_DEBUG here.
4954 */
851ba692 4955 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
0a5fff19
GN		 4956 return 0;
4957 }
4b8073e4 4958 }
6aa8b732 4959 break;
25c4c276 4960 case 2: /* clts */
bd7e5b08
PB		 4961 WARN_ONCE(1, "Guest should always own CR0.TS");
4962 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
4d4ec087 4963 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
6affcbed 4964 return kvm_skip_emulated_instruction(vcpu);
6aa8b732
AK		 4965 case 1: /*mov from cr*/
4966 switch (cr) {
4967 case 3:
e1de91cc 4968 WARN_ON_ONCE(enable_unrestricted_guest);
9f8fe504
AK		 4969 val = kvm_read_cr3(vcpu);
4970 kvm_register_write(vcpu, reg, val);
4971 trace_kvm_cr_read(cr, val);
6affcbed 4972 return kvm_skip_emulated_instruction(vcpu);
6aa8b732 4973 case 8:
229456fc
MT		 4974 val = kvm_get_cr8(vcpu);
4975 kvm_register_write(vcpu, reg, val);
4976 trace_kvm_cr_read(cr, val);
6affcbed 4977 return kvm_skip_emulated_instruction(vcpu);
6aa8b732
AK		 4978 }
4979 break;
4980 case 3: /* lmsw */
a1f83a74 4981 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
4d4ec087 4982 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
a1f83a74 4983 kvm_lmsw(vcpu, val);
6aa8b732 4984
6affcbed 4985 return kvm_skip_emulated_instruction(vcpu);
6aa8b732
AK		 4986 default:
4987 break;
4988 }
851ba692 4989 vcpu->run->exit_reason = 0;
a737f256 4990 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
6aa8b732
AK		 4991 (int)(exit_qualification >> 4) & 3, cr);
4992 return 0;
4993}
4994
851ba692 4995static int handle_dr(struct kvm_vcpu *vcpu)
6aa8b732 4996{
bfdaab09 4997 unsigned long exit_qualification;
16f8a6f9
NA		 4998 int dr, dr7, reg;
4999
5addc235 5000 exit_qualification = vmx_get_exit_qual(vcpu);
16f8a6f9
NA		 5001 dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
5002
5003 /* First, if DR does not exist, trigger UD */
5004 if (!kvm_require_dr(vcpu, dr))
5005 return 1;
6aa8b732 5006
f2483415 5007 /* Do not handle if the CPL > 0, will trigger GP on re-entry */
0a79b009
AK		 5008 if (!kvm_require_cpl(vcpu, 0))
5009 return 1;
16f8a6f9
NA		 5010 dr7 = vmcs_readl(GUEST_DR7);
5011 if (dr7 & DR7_GD) {
42dbaa5a
JK		 5012 /*
5013 * As the vm-exit takes precedence over the debug trap, we
5014 * need to emulate the latter, either for the host or the
5015 * guest debugging itself.
5016 */
5017 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
45981ded 5018 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_RTM | DR6_FIXED_1;
16f8a6f9 5019 vcpu->run->debug.arch.dr7 = dr7;
82b32774 5020 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
851ba692
AK		 5021 vcpu->run->debug.arch.exception = DB_VECTOR;
5022 vcpu->run->exit_reason = KVM_EXIT_DEBUG;
42dbaa5a
JK		 5023 return 0;
5024 } else {
4d5523cf 5025 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD);
42dbaa5a
JK		 5026 return 1;
5027 }
5028 }
5029
81908bf4 5030 if (vcpu->guest_debug == 0) {
2183f564 5031 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
81908bf4
PB		 5032
5033 /*
5034 * No more DR vmexits; force a reload of the debug registers
5035 * and reenter on this instruction. The next vmexit will
5036 * retrieve the full state of the debug registers.
5037 */
5038 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
5039 return 1;
5040 }
5041
42dbaa5a
JK		 5042 reg = DEBUG_REG_ACCESS_REG(exit_qualification);
5043 if (exit_qualification & TYPE_MOV_FROM_DR) {
020df079 5044 unsigned long val;
4c4d563b
JK		 5045
5046 if (kvm_get_dr(vcpu, dr, &val))
5047 return 1;
5048 kvm_register_write(vcpu, reg, val);
020df079 5049 } else
5777392e 5050 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
4c4d563b
JK		 5051 return 1;
5052
6affcbed 5053 return kvm_skip_emulated_instruction(vcpu);
6aa8b732
AK		 5054}
5055
81908bf4
PB		 5056static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
5057{
81908bf4
PB		 5058 get_debugreg(vcpu->arch.db[0], 0);
5059 get_debugreg(vcpu->arch.db[1], 1);
5060 get_debugreg(vcpu->arch.db[2], 2);
5061 get_debugreg(vcpu->arch.db[3], 3);
5062 get_debugreg(vcpu->arch.dr6, 6);
5063 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
5064
5065 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2183f564 5066 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
81908bf4
PB		 5067}
5068
020df079
GN		 5069static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
5070{
5071 vmcs_writel(GUEST_DR7, val);
5072}
5073
851ba692 5074static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
6e5d865c 5075{
eb90f341 5076 kvm_apic_update_ppr(vcpu);
6e5d865c
YS		 5077 return 1;
5078}
5079
851ba692 5080static int handle_interrupt_window(struct kvm_vcpu *vcpu)
6aa8b732 5081{
9dadc2f9 5082 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
2714d1d3 5083
3842d135
AK		 5084 kvm_make_request(KVM_REQ_EVENT, vcpu);
5085
a26bf12a 5086 ++vcpu->stat.irq_window_exits;
6aa8b732
AK		 5087 return 1;
5088}
5089
851ba692 5090static int handle_vmcall(struct kvm_vcpu *vcpu)
c21415e8 5091{
0d9c055e 5092 return kvm_emulate_hypercall(vcpu);
c21415e8
IM		 5093}
5094
ec25d5e6
GN		 5095static int handle_invd(struct kvm_vcpu *vcpu)
5096{
60fc3d02 5097 return kvm_emulate_instruction(vcpu, 0);
ec25d5e6
GN		 5098}
5099
851ba692 5100static int handle_invlpg(struct kvm_vcpu *vcpu)
a7052897 5101{
5addc235 5102 unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
a7052897
MT		 5103
5104 kvm_mmu_invlpg(vcpu, exit_qualification);
6affcbed 5105 return kvm_skip_emulated_instruction(vcpu);
a7052897
MT		 5106}
5107
fee84b07
AK		 5108static int handle_rdpmc(struct kvm_vcpu *vcpu)
5109{
5110 int err;
5111
5112 err = kvm_rdpmc(vcpu);
6affcbed 5113 return kvm_complete_insn_gp(vcpu, err);
fee84b07
AK		 5114}
5115
851ba692 5116static int handle_wbinvd(struct kvm_vcpu *vcpu)
e5edaa01 5117{
6affcbed 5118 return kvm_emulate_wbinvd(vcpu);
e5edaa01
ED		 5119}
5120
2acf923e
DC		 5121static int handle_xsetbv(struct kvm_vcpu *vcpu)
5122{
5123 u64 new_bv = kvm_read_edx_eax(vcpu);
de3cd117 5124 u32 index = kvm_rcx_read(vcpu);
2acf923e
DC		 5125
5126 if (kvm_set_xcr(vcpu, index, new_bv) == 0)
6affcbed 5127 return kvm_skip_emulated_instruction(vcpu);
2acf923e
DC		 5128 return 1;
5129}
5130
851ba692 5131static int handle_apic_access(struct kvm_vcpu *vcpu)
f78e0e2e 5132{
58fbbf26 5133 if (likely(fasteoi)) {
5addc235 5134 unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
58fbbf26
KT		 5135 int access_type, offset;
5136
5137 access_type = exit_qualification & APIC_ACCESS_TYPE;
5138 offset = exit_qualification & APIC_ACCESS_OFFSET;
5139 /*
5140 * Sane guest uses MOV to write EOI, with written value
5141 * not cared. So make a short-circuit here by avoiding
5142 * heavy instruction emulation.
5143 */
5144 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
5145 (offset == APIC_EOI)) {
5146 kvm_lapic_set_eoi(vcpu);
6affcbed 5147 return kvm_skip_emulated_instruction(vcpu);
58fbbf26
KT		 5148 }
5149 }
60fc3d02 5150 return kvm_emulate_instruction(vcpu, 0);
f78e0e2e
SY		 5151}
5152
c7c9c56c
YZ		 5153static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
5154{
5addc235 5155 unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
c7c9c56c
YZ		 5156 int vector = exit_qualification & 0xff;
5157
5158 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
5159 kvm_apic_set_eoi_accelerated(vcpu, vector);
5160 return 1;
5161}
5162
83d4c286
YZ		 5163static int handle_apic_write(struct kvm_vcpu *vcpu)
5164{
5addc235 5165 unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
83d4c286
YZ		 5166 u32 offset = exit_qualification & 0xfff;
5167
5168 /* APIC-write VM exit is trap-like and thus no need to adjust IP */
5169 kvm_apic_write_nodecode(vcpu, offset);
5170 return 1;
5171}
5172
851ba692 5173static int handle_task_switch(struct kvm_vcpu *vcpu)
37817f29 5174{
60637aac 5175 struct vcpu_vmx *vmx = to_vmx(vcpu);
37817f29 5176 unsigned long exit_qualification;
e269fb21
JK		 5177 bool has_error_code = false;
5178 u32 error_code = 0;
37817f29 5179 u16 tss_selector;
7f3d35fd 5180 int reason, type, idt_v, idt_index;
64a7ec06
GN		 5181
5182 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
7f3d35fd 5183 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
64a7ec06 5184 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
37817f29 5185
5addc235 5186 exit_qualification = vmx_get_exit_qual(vcpu);
37817f29
IE		 5187
5188 reason = (u32)exit_qualification >> 30;
64a7ec06
GN		 5189 if (reason == TASK_SWITCH_GATE && idt_v) {
5190 switch (type) {
5191 case INTR_TYPE_NMI_INTR:
5192 vcpu->arch.nmi_injected = false;
654f06fc 5193 vmx_set_nmi_mask(vcpu, true);
64a7ec06
GN		 5194 break;
5195 case INTR_TYPE_EXT_INTR:
66fd3f7f 5196 case INTR_TYPE_SOFT_INTR:
64a7ec06
GN		 5197 kvm_clear_interrupt_queue(vcpu);
5198 break;
5199 case INTR_TYPE_HARD_EXCEPTION:
e269fb21
JK		 5200 if (vmx->idt_vectoring_info &
5201 VECTORING_INFO_DELIVER_CODE_MASK) {
5202 has_error_code = true;
5203 error_code =
5204 vmcs_read32(IDT_VECTORING_ERROR_CODE);
5205 }
5206 /* fall through */
64a7ec06
GN		 5207 case INTR_TYPE_SOFT_EXCEPTION:
5208 kvm_clear_exception_queue(vcpu);
5209 break;
5210 default:
5211 break;
5212 }
60637aac 5213 }
37817f29
IE		 5214 tss_selector = exit_qualification;
5215
64a7ec06
GN		 5216 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
5217 type != INTR_TYPE_EXT_INTR &&
5218 type != INTR_TYPE_NMI_INTR))
1957aa63 5219 WARN_ON(!skip_emulated_instruction(vcpu));
64a7ec06 5220
42dbaa5a
JK		 5221 /*
5222 * TODO: What about debug traps on tss switch?
5223 * Are we supposed to inject them and update dr6?
5224 */
1051778f
SC		 5225 return kvm_task_switch(vcpu, tss_selector,
5226 type == INTR_TYPE_SOFT_INTR ? idt_index : -1,
60fc3d02 5227 reason, has_error_code, error_code);
37817f29
IE		 5228}
5229
851ba692 5230static int handle_ept_violation(struct kvm_vcpu *vcpu)
1439442c 5231{
f9c617f6 5232 unsigned long exit_qualification;
1439442c 5233 gpa_t gpa;
eebed243 5234 u64 error_code;
1439442c 5235
5addc235 5236 exit_qualification = vmx_get_exit_qual(vcpu);
1439442c 5237
0be9c7a8
GN		 5238 /*
5239 * EPT violation happened while executing iret from NMI,
5240 * "blocked by NMI" bit has to be set before next VM entry.
5241 * There are errata that may cause this bit to not be set:
5242 * AAK134, BY25.
5243 */
bcd1c294 5244 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
d02fcf50 5245 enable_vnmi &&
bcd1c294 5246 (exit_qualification & INTR_INFO_UNBLOCK_NMI))
0be9c7a8
GN		 5247 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
5248
1439442c 5249 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
229456fc 5250 trace_kvm_page_fault(gpa, exit_qualification);
4f5982a5 5251
27959a44 5252 /* Is it a read fault? */
ab22a473 5253 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
27959a44
JS		 5254 ? PFERR_USER_MASK : 0;
5255 /* Is it a write fault? */
ab22a473 5256 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
27959a44
JS		 5257 ? PFERR_WRITE_MASK : 0;
5258 /* Is it a fetch fault? */
ab22a473 5259 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
27959a44
JS		 5260 ? PFERR_FETCH_MASK : 0;
5261 /* ept page table entry is present? */
5262 error_code |= (exit_qualification &
5263 (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
5264 EPT_VIOLATION_EXECUTABLE))
5265 ? PFERR_PRESENT_MASK : 0;
4f5982a5 5266
eebed243
PB		 5267 error_code |= (exit_qualification & 0x100) != 0 ?
5268 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
25d92081 5269
25d92081 5270 vcpu->arch.exit_qualification = exit_qualification;
4f5982a5 5271 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
1439442c
SY		 5272}
5273
851ba692 5274static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
68f89400 5275{
68f89400
MT		 5276 gpa_t gpa;
5277
9034e6e8
PB		 5278 /*
5279 * A nested guest cannot optimize MMIO vmexits, because we have an
5280 * nGPA here instead of the required GPA.
5281 */
68f89400 5282 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
9034e6e8
PB		 5283 if (!is_guest_mode(vcpu) &&
5284 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
931c33b1 5285 trace_kvm_fast_mmio(gpa);
1957aa63 5286 return kvm_skip_emulated_instruction(vcpu);
68c3b4d1 5287 }
68f89400 5288
c75d0edc 5289 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
68f89400
MT		 5290}
5291
851ba692 5292static int handle_nmi_window(struct kvm_vcpu *vcpu)
f08864b4 5293{
d02fcf50 5294 WARN_ON_ONCE(!enable_vnmi);
4e2a0bc5 5295 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
f08864b4 5296 ++vcpu->stat.nmi_window_exits;
3842d135 5297 kvm_make_request(KVM_REQ_EVENT, vcpu);
f08864b4
SY		 5298
5299 return 1;
5300}
5301
80ced186 5302static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
ea953ef0 5303{
8b3079a5 5304 struct vcpu_vmx *vmx = to_vmx(vcpu);
49e9d557 5305 bool intr_window_requested;
b8405c18 5306 unsigned count = 130;
49e9d557 5307
2183f564 5308 intr_window_requested = exec_controls_get(vmx) &
9dadc2f9 5309 CPU_BASED_INTR_WINDOW_EXITING;
ea953ef0 5310
98eb2f8b 5311 while (vmx->emulation_required && count-- != 0) {
db438592 5312 if (intr_window_requested && !vmx_interrupt_blocked(vcpu))
49e9d557
AK		 5313 return handle_interrupt_window(&vmx->vcpu);
5314
72875d8a 5315 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
de87dcdd
AK		 5316 return 1;
5317
60fc3d02 5318 if (!kvm_emulate_instruction(vcpu, 0))
8fff2710 5319 return 0;
1d5a4d9b 5320
add5ff7a 5321 if (vmx->emulation_required && !vmx->rmode.vm86_active &&
8fff2710
SC		 5322 vcpu->arch.exception.pending) {
5323 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5324 vcpu->run->internal.suberror =
5325 KVM_INTERNAL_ERROR_EMULATION;
5326 vcpu->run->internal.ndata = 0;
5327 return 0;
5328 }
ea953ef0 5329
8d76c49e
GN		 5330 if (vcpu->arch.halt_request) {
5331 vcpu->arch.halt_request = 0;
8fff2710 5332 return kvm_vcpu_halt(vcpu);
8d76c49e
GN		 5333 }
5334
8fff2710
SC		 5335 /*
5336 * Note, return 1 and not 0, vcpu_run() is responsible for
5337 * morphing the pending signal into the proper return code.
5338 */
ea953ef0 5339 if (signal_pending(current))
8fff2710
SC		 5340 return 1;
5341
ea953ef0
MG		 5342 if (need_resched())
5343 schedule();
5344 }
5345
8fff2710 5346 return 1;
b4a2d31d
RK		 5347}
5348
5349static void grow_ple_window(struct kvm_vcpu *vcpu)
5350{
5351 struct vcpu_vmx *vmx = to_vmx(vcpu);
c5c5d6fa 5352 unsigned int old = vmx->ple_window;
b4a2d31d 5353
c8e88717
BM		 5354 vmx->ple_window = __grow_ple_window(old, ple_window,
5355 ple_window_grow,
5356 ple_window_max);
b4a2d31d 5357
4f75bcc3 5358 if (vmx->ple_window != old) {
b4a2d31d 5359 vmx->ple_window_dirty = true;
4f75bcc3
PX		 5360 trace_kvm_ple_window_update(vcpu->vcpu_id,
5361 vmx->ple_window, old);
5362 }
b4a2d31d
RK		 5363}
5364
5365static void shrink_ple_window(struct kvm_vcpu *vcpu)
5366{
5367 struct vcpu_vmx *vmx = to_vmx(vcpu);
c5c5d6fa 5368 unsigned int old = vmx->ple_window;
b4a2d31d 5369
c8e88717
BM		 5370 vmx->ple_window = __shrink_ple_window(old, ple_window,
5371 ple_window_shrink,
5372 ple_window);
b4a2d31d 5373
4f75bcc3 5374 if (vmx->ple_window != old) {
b4a2d31d 5375 vmx->ple_window_dirty = true;
4f75bcc3
PX		 5376 trace_kvm_ple_window_update(vcpu->vcpu_id,
5377 vmx->ple_window, old);
5378 }
b4a2d31d
RK		 5379}
5380
bf9f6ac8
FW		 5381/*
5382 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
5383 */
5384static void wakeup_handler(void)
5385{
5386 struct kvm_vcpu *vcpu;
5387 int cpu = smp_processor_id();
5388
5389 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
5390 list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
5391 blocked_vcpu_list) {
5392 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
5393
5394 if (pi_test_on(pi_desc) == 1)
5395 kvm_vcpu_kick(vcpu);
5396 }
5397 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
5398}
5399
e01bca2f 5400static void vmx_enable_tdp(void)
f160c7b7
JS		 5401{
5402 kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
5403 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
5404 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
5405 0ull, VMX_EPT_EXECUTABLE_MASK,
5406 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
d0ec49d4 5407 VMX_EPT_RWX_MASK, 0ull);
f160c7b7
JS		 5408
5409 ept_set_mmio_spte_mask();
f160c7b7
JS		 5410}
5411
4b8d54f9
ZE		 5412/*
5413 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
5414 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
5415 */
9fb41ba8 5416static int handle_pause(struct kvm_vcpu *vcpu)
4b8d54f9 5417{
b31c114b 5418 if (!kvm_pause_in_guest(vcpu->kvm))
b4a2d31d
RK		 5419 grow_ple_window(vcpu);
5420
de63ad4c
LM		 5421 /*
5422 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
5423 * VM-execution control is ignored if CPL > 0. OTOH, KVM
5424 * never set PAUSE_EXITING and just set PLE if supported,
5425 * so the vcpu must be CPL=0 if it gets a PAUSE exit.
5426 */
5427 kvm_vcpu_on_spin(vcpu, true);
6affcbed 5428 return kvm_skip_emulated_instruction(vcpu);
4b8d54f9
ZE		 5429}
5430
87c00572 5431static int handle_nop(struct kvm_vcpu *vcpu)
59708670 5432{
6affcbed 5433 return kvm_skip_emulated_instruction(vcpu);
59708670
SY		 5434}
5435
87c00572
GS		 5436static int handle_mwait(struct kvm_vcpu *vcpu)
5437{
5438 printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
5439 return handle_nop(vcpu);
5440}
5441
45ec368c
JM		 5442static int handle_invalid_op(struct kvm_vcpu *vcpu)
5443{
5444 kvm_queue_exception(vcpu, UD_VECTOR);
5445 return 1;
5446}
5447
5f3d45e7
MD		 5448static int handle_monitor_trap(struct kvm_vcpu *vcpu)
5449{
5450 return 1;
5451}
5452
87c00572
GS		 5453static int handle_monitor(struct kvm_vcpu *vcpu)
5454{
5455 printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
5456 return handle_nop(vcpu);
5457}
5458
55d2375e 5459static int handle_invpcid(struct kvm_vcpu *vcpu)
19677e32 5460{
55d2375e
SC		 5461 u32 vmx_instruction_info;
5462 unsigned long type;
5463 bool pcid_enabled;
5464 gva_t gva;
5465 struct x86_exception e;
5466 unsigned i;
5467 unsigned long roots_to_free = 0;
5468 struct {
5469 u64 pcid;
5470 u64 gla;
5471 } operand;
f9eb4af6 5472
55d2375e 5473 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
19677e32
BD		 5474 kvm_queue_exception(vcpu, UD_VECTOR);
5475 return 1;
5476 }
5477
55d2375e
SC		 5478 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5479 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
5480
5481 if (type > 3) {
5482 kvm_inject_gp(vcpu, 0);
f9eb4af6
EK		 5483 return 1;
5484 }
5485
55d2375e
SC		 5486 /* According to the Intel instruction reference, the memory operand
5487 * is read even if it isn't needed (e.g., for type==all)
5488 */
5addc235 5489 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
fdb28619
EK		 5490 vmx_instruction_info, false,
5491 sizeof(operand), &gva))
3573e22c
BD		 5492 return 1;
5493
55d2375e 5494 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
ee1fa209 5495 kvm_inject_emulated_page_fault(vcpu, &e);
3573e22c
BD		 5496 return 1;
5497 }
5498
55d2375e
SC		 5499 if (operand.pcid >> 12 != 0) {
5500 kvm_inject_gp(vcpu, 0);
5501 return 1;
abfc52c6 5502 }
e29acc55 5503
55d2375e 5504 pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
e29acc55 5505
55d2375e
SC		 5506 switch (type) {
5507 case INVPCID_TYPE_INDIV_ADDR:
5508 if ((!pcid_enabled && (operand.pcid != 0)) ||
5509 is_noncanonical_address(operand.gla, vcpu)) {
5510 kvm_inject_gp(vcpu, 0);
5511 return 1;
5512 }
5513 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
5514 return kvm_skip_emulated_instruction(vcpu);
61ada748 5515
55d2375e
SC		 5516 case INVPCID_TYPE_SINGLE_CTXT:
5517 if (!pcid_enabled && (operand.pcid != 0)) {
5518 kvm_inject_gp(vcpu, 0);
5519 return 1;
5520 }
e29acc55 5521
55d2375e
SC		 5522 if (kvm_get_active_pcid(vcpu) == operand.pcid) {
5523 kvm_mmu_sync_roots(vcpu);
eeeb4f67 5524 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
55d2375e 5525 }
e29acc55 5526
55d2375e 5527 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
be01e8e2 5528 if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd)
55d2375e
SC		 5529 == operand.pcid)
5530 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
63aff655 5531
55d2375e
SC		 5532 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);
5533 /*
5534 * If neither the current cr3 nor any of the prev_roots use the
5535 * given PCID, then nothing needs to be done here because a
5536 * resync will happen anyway before switching to any other CR3.
5537 */
e29acc55 5538
55d2375e 5539 return kvm_skip_emulated_instruction(vcpu);
61ada748 5540
55d2375e
SC		 5541 case INVPCID_TYPE_ALL_NON_GLOBAL:
5542 /*
5543 * Currently, KVM doesn't mark global entries in the shadow
5544 * page tables, so a non-global flush just degenerates to a
5545 * global flush. If needed, we could optimize this later by
5546 * keeping track of global entries in shadow page tables.
5547 */
e29acc55 5548
55d2375e
SC		 5549 /* fall-through */
5550 case INVPCID_TYPE_ALL_INCL_GLOBAL:
5551 kvm_mmu_unload(vcpu);
5552 return kvm_skip_emulated_instruction(vcpu);
e29acc55 5553
55d2375e
SC		 5554 default:
5555 BUG(); /* We have already checked above that type <= 3 */
5556 }
e29acc55
JM		 5557}
5558
55d2375e 5559static int handle_pml_full(struct kvm_vcpu *vcpu)
ec378aee 5560{
55d2375e 5561 unsigned long exit_qualification;
b3897a49 5562
55d2375e 5563 trace_kvm_pml_full(vcpu->vcpu_id);
b3897a49 5564
5addc235 5565 exit_qualification = vmx_get_exit_qual(vcpu);
cbf71279
RK		 5566
5567 /*
55d2375e
SC		 5568 * PML buffer FULL happened while executing iret from NMI,
5569 * "blocked by NMI" bit has to be set before next VM entry.
cbf71279 5570 */
55d2375e
SC		 5571 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5572 enable_vnmi &&
5573 (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5574 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
5575 GUEST_INTR_STATE_NMI);
e49fcb8b 5576
55d2375e
SC		 5577 /*
5578 * PML buffer already flushed at beginning of VMEXIT. Nothing to do
5579 * here.., and there's no userspace involvement needed for PML.
5580 */
ec378aee
NHE		 5581 return 1;
5582}
5583
55d2375e 5584static int handle_preemption_timer(struct kvm_vcpu *vcpu)
8ca44e88 5585{
804939ea
SC		 5586 struct vcpu_vmx *vmx = to_vmx(vcpu);
5587
5588 if (!vmx->req_immediate_exit &&
5589 !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled))
55d2375e 5590 kvm_lapic_expired_hv_timer(vcpu);
804939ea 5591
55d2375e 5592 return 1;
8ca44e88
DM		 5593}
5594
55d2375e
SC		 5595/*
5596 * When nested=0, all VMX instruction VM Exits filter here. The handlers
5597 * are overwritten by nested_vmx_setup() when nested=1.
5598 */
5599static int handle_vmx_instruction(struct kvm_vcpu *vcpu)
b8bbab92 5600{
55d2375e
SC		 5601 kvm_queue_exception(vcpu, UD_VECTOR);
5602 return 1;
b8bbab92
VK		 5603}
5604
55d2375e 5605static int handle_encls(struct kvm_vcpu *vcpu)
e7953d7f 5606{
55d2375e
SC		 5607 /*
5608 * SGX virtualization is not yet supported. There is no software
5609 * enable bit for SGX, so we have to trap ENCLS and inject a #UD
5610 * to prevent the guest from executing ENCLS.
5611 */
5612 kvm_queue_exception(vcpu, UD_VECTOR);
5613 return 1;
e7953d7f
AG		 5614}
5615
ec378aee 5616/*
55d2375e
SC		 5617 * The exit handlers return 1 if the exit was handled fully and guest execution
5618 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
5619 * to be done to userspace and return 0.
ec378aee 5620 */
55d2375e 5621static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
95b5a48c 5622 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi,
55d2375e
SC		 5623 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
5624 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
5625 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
5626 [EXIT_REASON_IO_INSTRUCTION] = handle_io,
5627 [EXIT_REASON_CR_ACCESS] = handle_cr,
5628 [EXIT_REASON_DR_ACCESS] = handle_dr,
f399e60c
AA		 5629 [EXIT_REASON_CPUID] = kvm_emulate_cpuid,
5630 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr,
5631 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr,
9dadc2f9 5632 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window,
f399e60c 5633 [EXIT_REASON_HLT] = kvm_emulate_halt,
55d2375e
SC		 5634 [EXIT_REASON_INVD] = handle_invd,
5635 [EXIT_REASON_INVLPG] = handle_invlpg,
5636 [EXIT_REASON_RDPMC] = handle_rdpmc,
5637 [EXIT_REASON_VMCALL] = handle_vmcall,
5638 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction,
5639 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction,
5640 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction,
5641 [EXIT_REASON_VMPTRST] = handle_vmx_instruction,
5642 [EXIT_REASON_VMREAD] = handle_vmx_instruction,
5643 [EXIT_REASON_VMRESUME] = handle_vmx_instruction,
5644 [EXIT_REASON_VMWRITE] = handle_vmx_instruction,
5645 [EXIT_REASON_VMOFF] = handle_vmx_instruction,
5646 [EXIT_REASON_VMON] = handle_vmx_instruction,
5647 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
5648 [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
5649 [EXIT_REASON_APIC_WRITE] = handle_apic_write,
5650 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced,
5651 [EXIT_REASON_WBINVD] = handle_wbinvd,
5652 [EXIT_REASON_XSETBV] = handle_xsetbv,
5653 [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
5654 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
5655 [EXIT_REASON_GDTR_IDTR] = handle_desc,
5656 [EXIT_REASON_LDTR_TR] = handle_desc,
5657 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
5658 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig,
5659 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause,
5660 [EXIT_REASON_MWAIT_INSTRUCTION] = handle_mwait,
5661 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap,
5662 [EXIT_REASON_MONITOR_INSTRUCTION] = handle_monitor,
5663 [EXIT_REASON_INVEPT] = handle_vmx_instruction,
5664 [EXIT_REASON_INVVPID] = handle_vmx_instruction,
5665 [EXIT_REASON_RDRAND] = handle_invalid_op,
5666 [EXIT_REASON_RDSEED] = handle_invalid_op,
55d2375e
SC		 5667 [EXIT_REASON_PML_FULL] = handle_pml_full,
5668 [EXIT_REASON_INVPCID] = handle_invpcid,
5669 [EXIT_REASON_VMFUNC] = handle_vmx_instruction,
5670 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
5671 [EXIT_REASON_ENCLS] = handle_encls,
5672};
b8bbab92 5673
55d2375e
SC		 5674static const int kvm_vmx_max_exit_handlers =
5675 ARRAY_SIZE(kvm_vmx_exit_handlers);
ec378aee 5676
55d2375e 5677static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
ec378aee 5678{
5addc235 5679 *info1 = vmx_get_exit_qual(vcpu);
87915858 5680 *info2 = vmx_get_intr_info(vcpu);
ec378aee
NHE		 5681}
5682
55d2375e 5683static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
27d6c865 5684{
55d2375e
SC		 5685 if (vmx->pml_pg) {
5686 __free_page(vmx->pml_pg);
5687 vmx->pml_pg = NULL;
b8bbab92 5688 }
27d6c865
NHE		 5689}
5690
55d2375e 5691static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
cd232ad0 5692{
55d2375e
SC		 5693 struct vcpu_vmx *vmx = to_vmx(vcpu);
5694 u64 *pml_buf;
5695 u16 pml_idx;
cd232ad0 5696
55d2375e 5697 pml_idx = vmcs_read16(GUEST_PML_INDEX);
cd232ad0 5698
55d2375e
SC		 5699 /* Do nothing if PML buffer is empty */
5700 if (pml_idx == (PML_ENTITY_NUM - 1))
5701 return;
cd232ad0 5702
55d2375e
SC		 5703 /* PML index always points to next available PML buffer entity */
5704 if (pml_idx >= PML_ENTITY_NUM)
5705 pml_idx = 0;
5706 else
5707 pml_idx++;
945679e3 5708
55d2375e
SC		 5709 pml_buf = page_address(vmx->pml_pg);
5710 for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
5711 u64 gpa;
945679e3 5712
55d2375e
SC		 5713 gpa = pml_buf[pml_idx];
5714 WARN_ON(gpa & (PAGE_SIZE - 1));
5715 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
945679e3
VK		 5716 }
5717
55d2375e
SC		 5718 /* reset PML index */
5719 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
945679e3
VK		 5720}
5721
f4160e45 5722/*
55d2375e
SC		 5723 * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
5724 * Called before reporting dirty_bitmap to userspace.
f4160e45 5725 */
55d2375e 5726static void kvm_flush_pml_buffers(struct kvm *kvm)
49f705c5 5727{
55d2375e
SC		 5728 int i;
5729 struct kvm_vcpu *vcpu;
49f705c5 5730 /*
55d2375e
SC		 5731 * We only need to kick vcpu out of guest mode here, as PML buffer
5732 * is flushed at beginning of all VMEXITs, and it's obvious that only
5733 * vcpus running in guest are possible to have unflushed GPAs in PML
5734 * buffer.
49f705c5 5735 */
55d2375e
SC		 5736 kvm_for_each_vcpu(i, vcpu, kvm)
5737 kvm_vcpu_kick(vcpu);
49f705c5
NHE		 5738}
5739
55d2375e 5740static void vmx_dump_sel(char *name, uint32_t sel)
49f705c5 5741{
55d2375e
SC		 5742 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
5743 name, vmcs_read16(sel),
5744 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
5745 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
5746 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
49f705c5
NHE		 5747}
5748
55d2375e 5749static void vmx_dump_dtsel(char *name, uint32_t limit)
a8bc284e 5750{
55d2375e
SC		 5751 pr_err("%s limit=0x%08x, base=0x%016lx\n",
5752 name, vmcs_read32(limit),
5753 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
a8bc284e
JM		 5754}
5755
69090810 5756void dump_vmcs(void)
63846663 5757{
6f2f8453
PB		 5758 u32 vmentry_ctl, vmexit_ctl;
5759 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control;
5760 unsigned long cr4;
5761 u64 efer;
63846663 5762
6f2f8453
PB		 5763 if (!dump_invalid_vmcs) {
5764 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n");
5765 return;
5766 }
5767
5768 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
5769 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
5770 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5771 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
5772 cr4 = vmcs_readl(GUEST_CR4);
5773 efer = vmcs_read64(GUEST_IA32_EFER);
5774 secondary_exec_control = 0;
55d2375e
SC		 5775 if (cpu_has_secondary_exec_ctrls())
5776 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
14c07ad8 5777
55d2375e
SC		 5778 pr_err("*** Guest State ***\n");
5779 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5780 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
5781 vmcs_readl(CR0_GUEST_HOST_MASK));
5782 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5783 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
5784 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
5785 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
5786 (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
5787 {
5788 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n",
5789 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
5790 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n",
5791 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
e9ac033e 5792 }
55d2375e
SC		 5793 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n",
5794 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
5795 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n",
5796 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
5797 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5798 vmcs_readl(GUEST_SYSENTER_ESP),
5799 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
5800 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR);
5801 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR);
5802 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR);
5803 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR);
5804 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR);
5805 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR);
5806 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
5807 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
5808 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
5809 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR);
5810 if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
5811 (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
5812 pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
5813 efer, vmcs_read64(GUEST_IA32_PAT));
5814 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n",
5815 vmcs_read64(GUEST_IA32_DEBUGCTL),
5816 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
5817 if (cpu_has_load_perf_global_ctrl() &&
5818 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
5819 pr_err("PerfGlobCtl = 0x%016llx\n",
5820 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
5821 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
5822 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
5823 pr_err("Interruptibility = %08x ActivityState = %08x\n",
5824 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
5825 vmcs_read32(GUEST_ACTIVITY_STATE));
5826 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
5827 pr_err("InterruptStatus = %04x\n",
5828 vmcs_read16(GUEST_INTR_STATUS));
ff651cb6 5829
55d2375e
SC		 5830 pr_err("*** Host State ***\n");
5831 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n",
5832 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
5833 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
5834 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
5835 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
5836 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
5837 vmcs_read16(HOST_TR_SELECTOR));
5838 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
5839 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
5840 vmcs_readl(HOST_TR_BASE));
5841 pr_err("GDTBase=%016lx IDTBase=%016lx\n",
5842 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
5843 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
5844 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
5845 vmcs_readl(HOST_CR4));
5846 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5847 vmcs_readl(HOST_IA32_SYSENTER_ESP),
5848 vmcs_read32(HOST_IA32_SYSENTER_CS),
5849 vmcs_readl(HOST_IA32_SYSENTER_EIP));
5850 if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
5851 pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
5852 vmcs_read64(HOST_IA32_EFER),
5853 vmcs_read64(HOST_IA32_PAT));
5854 if (cpu_has_load_perf_global_ctrl() &&
5855 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
5856 pr_err("PerfGlobCtl = 0x%016llx\n",
5857 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
ff651cb6 5858
55d2375e
SC		 5859 pr_err("*** Control State ***\n");
5860 pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
5861 pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
5862 pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
5863 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
5864 vmcs_read32(EXCEPTION_BITMAP),
5865 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
5866 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
5867 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
5868 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
5869 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
5870 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
5871 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
5872 vmcs_read32(VM_EXIT_INTR_INFO),
5873 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5874 vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
5875 pr_err(" reason=%08x qualification=%016lx\n",
5876 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
5877 pr_err("IDTVectoring: info=%08x errcode=%08x\n",
5878 vmcs_read32(IDT_VECTORING_INFO_FIELD),
5879 vmcs_read32(IDT_VECTORING_ERROR_CODE));
5880 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
5881 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
5882 pr_err("TSC Multiplier = 0x%016llx\n",
5883 vmcs_read64(TSC_MULTIPLIER));
9d609649
PB		 5884 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) {
5885 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
5886 u16 status = vmcs_read16(GUEST_INTR_STATUS);
5887 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff);
5888 }
d6a85c32 5889 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
9d609649
PB		 5890 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
5891 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR));
d6a85c32 5892 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR));
9d609649 5893 }
55d2375e
SC		 5894 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
5895 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
5896 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
5897 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
55d2375e
SC		 5898 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
5899 pr_err("PLE Gap=%08x Window=%08x\n",
5900 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
5901 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
5902 pr_err("Virtual processor ID = 0x%04x\n",
5903 vmcs_read16(VIRTUAL_PROCESSOR_ID));
ff651cb6
WV		 5904}
5905
55d2375e
SC		 5906/*
5907 * The guest has exited. See if we can fix it or if we need userspace
5908 * assistance.
5909 */
1e9e2622
WL		 5910static int vmx_handle_exit(struct kvm_vcpu *vcpu,
5911 enum exit_fastpath_completion exit_fastpath)
ff651cb6 5912{
55d2375e
SC		 5913 struct vcpu_vmx *vmx = to_vmx(vcpu);
5914 u32 exit_reason = vmx->exit_reason;
5915 u32 vectoring_info = vmx->idt_vectoring_info;
ff651cb6 5916
55d2375e 5917 trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
ff651cb6 5918
55d2375e
SC		 5919 /*
5920 * Flush logged GPAs PML buffer, this will make dirty_bitmap more
5921 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
5922 * querying dirty_bitmap, we only need to kick all vcpus out of guest
5923 * mode as if vcpus is in root mode, the PML buffer must has been
5924 * flushed already.
5925 */
5926 if (enable_pml)
5927 vmx_flush_pml_buffer(vcpu);
1dc35dac 5928
db438592
SC		 5929 /*
5930 * We should never reach this point with a pending nested VM-Enter, and
5931 * more specifically emulation of L2 due to invalid guest state (see
5932 * below) should never happen as that means we incorrectly allowed a
5933 * nested VM-Enter with an invalid vmcs12.
5934 */
5935 WARN_ON_ONCE(vmx->nested.nested_run_pending);
5936
55d2375e
SC		 5937 /* If guest state is invalid, start emulating */
5938 if (vmx->emulation_required)
5939 return handle_invalid_guest_state(vcpu);
1dc35dac 5940
96b100cd
PB		 5941 if (is_guest_mode(vcpu)) {
5942 /*
5943 * The host physical addresses of some pages of guest memory
5944 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
5945 * Page). The CPU may write to these pages via their host
5946 * physical address while L2 is running, bypassing any
5947 * address-translation-based dirty tracking (e.g. EPT write
5948 * protection).
5949 *
5950 * Mark them dirty on every exit from L2 to prevent them from
5951 * getting out of sync with dirty tracking.
5952 */
5953 nested_mark_vmcs12_pages_dirty(vcpu);
5954
f47baaed 5955 if (nested_vmx_reflect_vmexit(vcpu))
789afc5c 5956 return 1;
96b100cd 5957 }
9ed38ffa 5958
55d2375e
SC		 5959 if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
5960 dump_vmcs();
5961 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5962 vcpu->run->fail_entry.hardware_entry_failure_reason
5963 = exit_reason;
5964 return 0;
9ed38ffa
LP		 5965 }
5966
55d2375e 5967 if (unlikely(vmx->fail)) {
3b20e03a 5968 dump_vmcs();
55d2375e
SC		 5969 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5970 vcpu->run->fail_entry.hardware_entry_failure_reason
5971 = vmcs_read32(VM_INSTRUCTION_ERROR);
5972 return 0;
5973 }
50c28f21 5974
55d2375e
SC		 5975 /*
5976 * Note:
5977 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
5978 * delivery event since it indicates guest is accessing MMIO.
5979 * The vm-exit can be triggered again after return to guest that
5980 * will cause infinite loop.
5981 */
5982 if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
5983 (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
5984 exit_reason != EXIT_REASON_EPT_VIOLATION &&
5985 exit_reason != EXIT_REASON_PML_FULL &&
5986 exit_reason != EXIT_REASON_TASK_SWITCH)) {
5987 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5988 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
5989 vcpu->run->internal.ndata = 3;
5990 vcpu->run->internal.data[0] = vectoring_info;
5991 vcpu->run->internal.data[1] = exit_reason;
5992 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
5993 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
5994 vcpu->run->internal.ndata++;
5995 vcpu->run->internal.data[3] =
5996 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5997 }
5998 return 0;
5999 }
50c28f21 6000
55d2375e
SC		 6001 if (unlikely(!enable_vnmi &&
6002 vmx->loaded_vmcs->soft_vnmi_blocked)) {
db438592 6003 if (!vmx_interrupt_blocked(vcpu)) {
55d2375e
SC		 6004 vmx->loaded_vmcs->soft_vnmi_blocked = 0;
6005 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
6006 vcpu->arch.nmi_pending) {
6007 /*
6008 * This CPU don't support us in finding the end of an
6009 * NMI-blocked window if the guest runs with IRQs
6010 * disabled. So we pull the trigger after 1 s of
6011 * futile waiting, but inform the user about this.
6012 */
6013 printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
6014 "state on VCPU %d after 1 s timeout\n",
6015 __func__, vcpu->vcpu_id);
6016 vmx->loaded_vmcs->soft_vnmi_blocked = 0;
6017 }
6018 }
50c28f21 6019
1e9e2622
WL		 6020 if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) {
6021 kvm_skip_emulated_instruction(vcpu);
6022 return 1;
c926f2f7
MP		 6023 }
6024
6025 if (exit_reason >= kvm_vmx_max_exit_handlers)
6026 goto unexpected_vmexit;
4289d272 6027#ifdef CONFIG_RETPOLINE
c926f2f7
MP		 6028 if (exit_reason == EXIT_REASON_MSR_WRITE)
6029 return kvm_emulate_wrmsr(vcpu);
6030 else if (exit_reason == EXIT_REASON_PREEMPTION_TIMER)
6031 return handle_preemption_timer(vcpu);
6032 else if (exit_reason == EXIT_REASON_INTERRUPT_WINDOW)
6033 return handle_interrupt_window(vcpu);
6034 else if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
6035 return handle_external_interrupt(vcpu);
6036 else if (exit_reason == EXIT_REASON_HLT)
6037 return kvm_emulate_halt(vcpu);
6038 else if (exit_reason == EXIT_REASON_EPT_MISCONFIG)
6039 return handle_ept_misconfig(vcpu);
4289d272 6040#endif
c926f2f7
MP		 6041
6042 exit_reason = array_index_nospec(exit_reason,
6043 kvm_vmx_max_exit_handlers);
6044 if (!kvm_vmx_exit_handlers[exit_reason])
6045 goto unexpected_vmexit;
6046
6047 return kvm_vmx_exit_handlers[exit_reason](vcpu);
6048
6049unexpected_vmexit:
6050 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", exit_reason);
6051 dump_vmcs();
6052 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6053 vcpu->run->internal.suberror =
7396d337 6054 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
c926f2f7
MP		 6055 vcpu->run->internal.ndata = 1;
6056 vcpu->run->internal.data[0] = exit_reason;
6057 return 0;
9ed38ffa
LP		 6058}
6059
efebf0aa 6060/*
55d2375e
SC		 6061 * Software based L1D cache flush which is used when microcode providing
6062 * the cache control MSR is not loaded.
efebf0aa 6063 *
55d2375e
SC		 6064 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
6065 * flush it is required to read in 64 KiB because the replacement algorithm
6066 * is not exactly LRU. This could be sized at runtime via topology
6067 * information but as all relevant affected CPUs have 32KiB L1D cache size
6068 * there is no point in doing so.
efebf0aa 6069 */
55d2375e 6070static void vmx_l1d_flush(struct kvm_vcpu *vcpu)
fe3ef05c 6071{
55d2375e 6072 int size = PAGE_SIZE << L1D_CACHE_ORDER;
25a2e4fe
PB		 6073
6074 /*
55d2375e
SC		 6075 * This code is only executed when the the flush mode is 'cond' or
6076 * 'always'
25a2e4fe 6077 */
55d2375e
SC		 6078 if (static_branch_likely(&vmx_l1d_flush_cond)) {
6079 bool flush_l1d;
25a2e4fe 6080
55d2375e
SC		 6081 /*
6082 * Clear the per-vcpu flush bit, it gets set again
6083 * either from vcpu_run() or from one of the unsafe
6084 * VMEXIT handlers.
6085 */
6086 flush_l1d = vcpu->arch.l1tf_flush_l1d;
6087 vcpu->arch.l1tf_flush_l1d = false;
25a2e4fe 6088
55d2375e
SC		 6089 /*
6090 * Clear the per-cpu flush bit, it gets set again from
6091 * the interrupt handlers.
6092 */
6093 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
6094 kvm_clear_cpu_l1tf_flush_l1d();
25a2e4fe 6095
55d2375e
SC		 6096 if (!flush_l1d)
6097 return;
6098 }
09abe320 6099
55d2375e 6100 vcpu->stat.l1d_flush++;
25a2e4fe 6101
55d2375e
SC		 6102 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
6103 wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
6104 return;
6105 }
25a2e4fe 6106
55d2375e
SC		 6107 asm volatile(
6108 /* First ensure the pages are in the TLB */
6109 "xorl %%eax, %%eax\n"
6110 ".Lpopulate_tlb:\n\t"
6111 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
6112 "addl $4096, %%eax\n\t"
6113 "cmpl %%eax, %[size]\n\t"
6114 "jne .Lpopulate_tlb\n\t"
6115 "xorl %%eax, %%eax\n\t"
6116 "cpuid\n\t"
6117 /* Now fill the cache */
6118 "xorl %%eax, %%eax\n"
6119 ".Lfill_cache:\n"
6120 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
6121 "addl $64, %%eax\n\t"
6122 "cmpl %%eax, %[size]\n\t"
6123 "jne .Lfill_cache\n\t"
6124 "lfence\n"
6125 :: [flush_pages] "r" (vmx_l1d_flush_pages),
6126 [size] "r" (size)
6127 : "eax", "ebx", "ecx", "edx");
09abe320 6128}
25a2e4fe 6129
55d2375e 6130static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
09abe320 6131{
55d2375e 6132 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
132f4f7e 6133 int tpr_threshold;
09abe320 6134
55d2375e
SC		 6135 if (is_guest_mode(vcpu) &&
6136 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
6137 return;
25a2e4fe 6138
132f4f7e 6139 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr;
02d496cf
LA		 6140 if (is_guest_mode(vcpu))
6141 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold;
6142 else
6143 vmcs_write32(TPR_THRESHOLD, tpr_threshold);
8665c3f9
PB		 6144}
6145
55d2375e 6146void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
8665c3f9 6147{
fe7f895d 6148 struct vcpu_vmx *vmx = to_vmx(vcpu);
55d2375e 6149 u32 sec_exec_control;
8665c3f9 6150
55d2375e
SC		 6151 if (!lapic_in_kernel(vcpu))
6152 return;
9314006d 6153
55d2375e
SC		 6154 if (!flexpriority_enabled &&
6155 !cpu_has_vmx_virtualize_x2apic_mode())
6156 return;
705699a1 6157
55d2375e
SC		 6158 /* Postpone execution until vmcs01 is the current VMCS. */
6159 if (is_guest_mode(vcpu)) {
fe7f895d 6160 vmx->nested.change_vmcs01_virtual_apic_mode = true;
55d2375e 6161 return;
6beb7bd5 6162 }
fe3ef05c 6163
fe7f895d 6164 sec_exec_control = secondary_exec_controls_get(vmx);
55d2375e
SC		 6165 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
6166 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
09abe320 6167
55d2375e
SC		 6168 switch (kvm_get_apic_mode(vcpu)) {
6169 case LAPIC_MODE_INVALID:
6170 WARN_ONCE(true, "Invalid local APIC state");
6171 case LAPIC_MODE_DISABLED:
6172 break;
6173 case LAPIC_MODE_XAPIC:
6174 if (flexpriority_enabled) {
6175 sec_exec_control |=
6176 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
4de1f9d4
SC		 6177 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
6178
6179 /*
6180 * Flush the TLB, reloading the APIC access page will
6181 * only do so if its physical address has changed, but
6182 * the guest may have inserted a non-APIC mapping into
6183 * the TLB while the APIC access page was disabled.
6184 */
6185 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
55d2375e
SC		 6186 }
6187 break;
6188 case LAPIC_MODE_X2APIC:
6189 if (cpu_has_vmx_virtualize_x2apic_mode())
6190 sec_exec_control |=
6191 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
6192 break;
09abe320 6193 }
fe7f895d 6194 secondary_exec_controls_set(vmx, sec_exec_control);
09abe320 6195
55d2375e
SC		 6196 vmx_update_msr_bitmap(vcpu);
6197}
0238ea91 6198
a4148b7c 6199static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
55d2375e 6200{
a4148b7c
SC		 6201 struct page *page;
6202
1196cb97
SC		 6203 /* Defer reload until vmcs01 is the current VMCS. */
6204 if (is_guest_mode(vcpu)) {
6205 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true;
6206 return;
55d2375e 6207 }
1196cb97 6208
4de1f9d4
SC		 6209 if (!(secondary_exec_controls_get(to_vmx(vcpu)) &
6210 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
6211 return;
6212
a4148b7c
SC		 6213 page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
6214 if (is_error_page(page))
6215 return;
6216
6217 vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page));
1196cb97 6218 vmx_flush_tlb_current(vcpu);
a4148b7c
SC		 6219
6220 /*
6221 * Do not pin apic access page in memory, the MMU notifier
6222 * will call us again if it is migrated or swapped out.
6223 */
6224 put_page(page);
55d2375e 6225}
fe3ef05c 6226
55d2375e
SC		 6227static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
6228{
6229 u16 status;
6230 u8 old;
32c7acf0 6231
55d2375e
SC		 6232 if (max_isr == -1)
6233 max_isr = 0;
608406e2 6234
55d2375e
SC		 6235 status = vmcs_read16(GUEST_INTR_STATUS);
6236 old = status >> 8;
6237 if (max_isr != old) {
6238 status &= 0xff;
6239 status |= max_isr << 8;
6240 vmcs_write16(GUEST_INTR_STATUS, status);
6241 }
6242}
6beb7bd5 6243
55d2375e
SC		 6244static void vmx_set_rvi(int vector)
6245{
6246 u16 status;
6247 u8 old;
0b665d30 6248
55d2375e
SC		 6249 if (vector == -1)
6250 vector = 0;
fe3ef05c 6251
55d2375e
SC		 6252 status = vmcs_read16(GUEST_INTR_STATUS);
6253 old = (u8)status & 0xff;
6254 if ((u8)vector != old) {
6255 status &= ~0xff;
6256 status |= (u8)vector;
6257 vmcs_write16(GUEST_INTR_STATUS, status);
09abe320 6258 }
55d2375e 6259}
09abe320 6260
55d2375e
SC		 6261static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
6262{
09abe320 6263 /*
55d2375e
SC		 6264 * When running L2, updating RVI is only relevant when
6265 * vmcs12 virtual-interrupt-delivery enabled.
6266 * However, it can be enabled only when L1 also
6267 * intercepts external-interrupts and in that case
6268 * we should not update vmcs02 RVI but instead intercept
6269 * interrupt. Therefore, do nothing when running L2.
fe3ef05c 6270 */
55d2375e
SC		 6271 if (!is_guest_mode(vcpu))
6272 vmx_set_rvi(max_irr);
6273}
fe3ef05c 6274
55d2375e
SC		 6275static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
6276{
6277 struct vcpu_vmx *vmx = to_vmx(vcpu);
6278 int max_irr;
6279 bool max_irr_updated;
a7c0b07d 6280
55d2375e
SC		 6281 WARN_ON(!vcpu->arch.apicv_active);
6282 if (pi_test_on(&vmx->pi_desc)) {
6283 pi_clear_on(&vmx->pi_desc);
6284 /*
d9ff2744 6285 * IOMMU can write to PID.ON, so the barrier matters even on UP.
55d2375e
SC		 6286 * But on x86 this is just a compiler barrier anyway.
6287 */
6288 smp_mb__after_atomic();
6289 max_irr_updated =
6290 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
c4ebd629
VK		 6291
6292 /*
55d2375e
SC		 6293 * If we are running L2 and L1 has a new pending interrupt
6294 * which can be injected, we should re-evaluate
6295 * what should be done with this new L1 interrupt.
6296 * If L1 intercepts external-interrupts, we should
6297 * exit from L2 to L1. Otherwise, interrupt should be
6298 * delivered directly to L2.
c4ebd629 6299 */
55d2375e
SC		 6300 if (is_guest_mode(vcpu) && max_irr_updated) {
6301 if (nested_exit_on_intr(vcpu))
6302 kvm_vcpu_exiting_guest_mode(vcpu);
6303 else
6304 kvm_make_request(KVM_REQ_EVENT, vcpu);
c4ebd629 6305 }
55d2375e
SC		 6306 } else {
6307 max_irr = kvm_lapic_find_highest_irr(vcpu);
a7c0b07d 6308 }
55d2375e
SC		 6309 vmx_hwapic_irr_update(vcpu, max_irr);
6310 return max_irr;
6311}
a7c0b07d 6312
17e433b5
WL		 6313static bool vmx_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
6314{
9482ae45
JM		 6315 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
6316
6317 return pi_test_on(pi_desc) ||
29881b6e 6318 (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc));
17e433b5
WL		 6319}
6320
55d2375e
SC		 6321static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
6322{
6323 if (!kvm_vcpu_apicv_active(vcpu))
6324 return;
25a2e4fe 6325
55d2375e
SC		 6326 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
6327 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
6328 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
6329 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
8665c3f9
PB		 6330}
6331
55d2375e 6332static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
8665c3f9
PB		 6333{
6334 struct vcpu_vmx *vmx = to_vmx(vcpu);
9d1887ef 6335
55d2375e
SC		 6336 pi_clear_on(&vmx->pi_desc);
6337 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
6338}
8665c3f9 6339
95b5a48c 6340static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx)
55d2375e 6341{
87915858 6342 u32 intr_info = vmx_get_intr_info(&vmx->vcpu);
fe3ef05c 6343
55d2375e 6344 /* if exit due to PF check for async PF */
87915858 6345 if (is_page_fault(intr_info)) {
55d2375e 6346 vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
55d2375e 6347 /* Handle machine checks before interrupts are enabled */
87915858 6348 } else if (is_machine_check(intr_info)) {
55d2375e 6349 kvm_machine_check();
55d2375e 6350 /* We need to handle NMIs before interrupts are enabled */
87915858 6351 } else if (is_nmi(intr_info)) {
55d2375e
SC		 6352 kvm_before_interrupt(&vmx->vcpu);
6353 asm("int $2");
6354 kvm_after_interrupt(&vmx->vcpu);
fe3ef05c 6355 }
55d2375e 6356}
fe3ef05c 6357
95b5a48c 6358static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
55d2375e 6359{
49def500
SC		 6360 unsigned int vector;
6361 unsigned long entry;
55d2375e 6362#ifdef CONFIG_X86_64
49def500 6363 unsigned long tmp;
55d2375e 6364#endif
49def500 6365 gate_desc *desc;
87915858 6366 u32 intr_info = vmx_get_intr_info(vcpu);
fe3ef05c 6367
49def500
SC		 6368 if (WARN_ONCE(!is_external_intr(intr_info),
6369 "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info))
6370 return;
6371
6372 vector = intr_info & INTR_INFO_VECTOR_MASK;
2342080c 6373 desc = (gate_desc *)host_idt_base + vector;
49def500
SC		 6374 entry = gate_offset(desc);
6375
165072b0
SC		 6376 kvm_before_interrupt(vcpu);
6377
49def500 6378 asm volatile(
55d2375e 6379#ifdef CONFIG_X86_64
49def500
SC		 6380 "mov %%" _ASM_SP ", %[sp]\n\t"
6381 "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
6382 "push $%c[ss]\n\t"
6383 "push %[sp]\n\t"
55d2375e 6384#endif
49def500
SC		 6385 "pushf\n\t"
6386 __ASM_SIZE(push) " $%c[cs]\n\t"
6387 CALL_NOSPEC
6388 :
55d2375e 6389#ifdef CONFIG_X86_64
49def500 6390 [sp]"=&r"(tmp),
55d2375e 6391#endif
49def500
SC		 6392 ASM_CALL_CONSTRAINT
6393 :
428b8f1d 6394 [thunk_target]"r"(entry),
49def500
SC		 6395 [ss]"i"(__KERNEL_DS),
6396 [cs]"i"(__KERNEL_CS)
6397 );
165072b0
SC		 6398
6399 kvm_after_interrupt(vcpu);
55d2375e 6400}
95b5a48c
SC		 6401STACK_FRAME_NON_STANDARD(handle_external_interrupt_irqoff);
6402
a9ab13ff 6403static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
95b5a48c
SC		 6404{
6405 struct vcpu_vmx *vmx = to_vmx(vcpu);
6406
6407 if (vmx->exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
6408 handle_external_interrupt_irqoff(vcpu);
6409 else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI)
6410 handle_exception_nmi_irqoff(vmx);
6411}
5a6a9748 6412
55d2375e
SC		 6413static bool vmx_has_emulated_msr(int index)
6414{
6415 switch (index) {
6416 case MSR_IA32_SMBASE:
6417 /*
6418 * We cannot do SMM unless we can run the guest in big
6419 * real mode.
6420 */
6421 return enable_unrestricted_guest || emulate_invalid_guest_state;
95c5c7c7
PB		 6422 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
6423 return nested;
55d2375e
SC		 6424 case MSR_AMD64_VIRT_SPEC_CTRL:
6425 /* This is AMD only. */
6426 return false;
6427 default:
6428 return true;
3184a995 6429 }
55d2375e 6430}
2bb8cafe 6431
55d2375e
SC		 6432static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
6433{
6434 u32 exit_intr_info;
6435 bool unblock_nmi;
6436 u8 vector;
6437 bool idtv_info_valid;
7ca29de2 6438
55d2375e 6439 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
feaf0c7d 6440
55d2375e
SC		 6441 if (enable_vnmi) {
6442 if (vmx->loaded_vmcs->nmi_known_unmasked)
6443 return;
87915858
SC		 6444
6445 exit_intr_info = vmx_get_intr_info(&vmx->vcpu);
55d2375e
SC		 6446 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
6447 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
6448 /*
6449 * SDM 3: 27.7.1.2 (September 2008)
6450 * Re-set bit "block by NMI" before VM entry if vmexit caused by
6451 * a guest IRET fault.
6452 * SDM 3: 23.2.2 (September 2008)
6453 * Bit 12 is undefined in any of the following cases:
6454 * If the VM exit sets the valid bit in the IDT-vectoring
6455 * information field.
6456 * If the VM exit is due to a double fault.
6457 */
6458 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
6459 vector != DF_VECTOR && !idtv_info_valid)
6460 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
6461 GUEST_INTR_STATE_NMI);
6462 else
6463 vmx->loaded_vmcs->nmi_known_unmasked =
6464 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
6465 & GUEST_INTR_STATE_NMI);
6466 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
6467 vmx->loaded_vmcs->vnmi_blocked_time +=
6468 ktime_to_ns(ktime_sub(ktime_get(),
6469 vmx->loaded_vmcs->entry_time));
fe3ef05c
NHE		 6470}
6471
55d2375e
SC		 6472static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
6473 u32 idt_vectoring_info,
6474 int instr_len_field,
6475 int error_code_field)
0c7f650e 6476{
55d2375e
SC		 6477 u8 vector;
6478 int type;
6479 bool idtv_info_valid;
0c7f650e 6480
55d2375e 6481 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
0c7f650e 6482
55d2375e
SC		 6483 vcpu->arch.nmi_injected = false;
6484 kvm_clear_exception_queue(vcpu);
6485 kvm_clear_interrupt_queue(vcpu);
27c42a1b 6486
55d2375e
SC		 6487 if (!idtv_info_valid)
6488 return;
c7c2c709 6489
55d2375e 6490 kvm_make_request(KVM_REQ_EVENT, vcpu);
ca0bde28 6491
55d2375e
SC		 6492 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
6493 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
64a919f7 6494
55d2375e
SC		 6495 switch (type) {
6496 case INTR_TYPE_NMI_INTR:
6497 vcpu->arch.nmi_injected = true;
6498 /*
6499 * SDM 3: 27.7.1.2 (September 2008)
6500 * Clear bit "block by NMI" before VM entry if a NMI
6501 * delivery faulted.
6502 */
6503 vmx_set_nmi_mask(vcpu, false);
6504 break;
6505 case INTR_TYPE_SOFT_EXCEPTION:
6506 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6507 /* fall through */
6508 case INTR_TYPE_HARD_EXCEPTION:
6509 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
6510 u32 err = vmcs_read32(error_code_field);
6511 kvm_requeue_exception_e(vcpu, vector, err);
6512 } else
6513 kvm_requeue_exception(vcpu, vector);
6514 break;
6515 case INTR_TYPE_SOFT_INTR:
6516 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6517 /* fall through */
6518 case INTR_TYPE_EXT_INTR:
6519 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
6520 break;
6521 default:
6522 break;
0447378a 6523 }
ca0bde28
JM		 6524}
6525
55d2375e 6526static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
f145d90d 6527{
55d2375e
SC		 6528 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
6529 VM_EXIT_INSTRUCTION_LEN,
6530 IDT_VECTORING_ERROR_CODE);
f145d90d
LA		 6531}
6532
55d2375e 6533static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
ca0bde28 6534{
55d2375e
SC		 6535 __vmx_complete_interrupts(vcpu,
6536 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
6537 VM_ENTRY_INSTRUCTION_LEN,
6538 VM_ENTRY_EXCEPTION_ERROR_CODE);
f1b026a3 6539
55d2375e 6540 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
ca0bde28
JM		 6541}
6542
55d2375e 6543static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
52017608 6544{
55d2375e
SC		 6545 int i, nr_msrs;
6546 struct perf_guest_switch_msr *msrs;
7c177938 6547
55d2375e 6548 msrs = perf_guest_get_msrs(&nr_msrs);
384bb783 6549
55d2375e
SC		 6550 if (!msrs)
6551 return;
f1b026a3 6552
55d2375e
SC		 6553 for (i = 0; i < nr_msrs; i++)
6554 if (msrs[i].host == msrs[i].guest)
6555 clear_atomic_switch_msr(vmx, msrs[i].msr);
6556 else
6557 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
6558 msrs[i].host, false);
ca0bde28 6559}
52017608 6560
6e3ba4ab
TX		 6561static void atomic_switch_umwait_control_msr(struct vcpu_vmx *vmx)
6562{
6563 u32 host_umwait_control;
6564
6565 if (!vmx_has_waitpkg(vmx))
6566 return;
6567
6568 host_umwait_control = get_umwait_control_msr();
6569
6570 if (vmx->msr_ia32_umwait_control != host_umwait_control)
6571 add_atomic_switch_msr(vmx, MSR_IA32_UMWAIT_CONTROL,
6572 vmx->msr_ia32_umwait_control,
6573 host_umwait_control, false);
6574 else
6575 clear_atomic_switch_msr(vmx, MSR_IA32_UMWAIT_CONTROL);
6576}
6577
55d2375e 6578static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
858e25c0
JM		 6579{
6580 struct vcpu_vmx *vmx = to_vmx(vcpu);
55d2375e
SC		 6581 u64 tscl;
6582 u32 delta_tsc;
52017608 6583
55d2375e 6584 if (vmx->req_immediate_exit) {
804939ea
SC		 6585 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0);
6586 vmx->loaded_vmcs->hv_timer_soft_disabled = false;
6587 } else if (vmx->hv_deadline_tsc != -1) {
55d2375e
SC		 6588 tscl = rdtsc();
6589 if (vmx->hv_deadline_tsc > tscl)
6590 /* set_hv_timer ensures the delta fits in 32-bits */
6591 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
6592 cpu_preemption_timer_multi);
6593 else
6594 delta_tsc = 0;
858e25c0 6595
804939ea
SC		 6596 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
6597 vmx->loaded_vmcs->hv_timer_soft_disabled = false;
6598 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) {
6599 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1);
6600 vmx->loaded_vmcs->hv_timer_soft_disabled = true;
7f7f1ba3 6601 }
858e25c0
JM		 6602}
6603
c09b03eb 6604void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)
ca0bde28 6605{
c09b03eb
SC		 6606 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) {
6607 vmx->loaded_vmcs->host_state.rsp = host_rsp;
6608 vmcs_writel(HOST_RSP, host_rsp);
6609 }
5ad6ece8 6610}
5f3d5799 6611
fc2ba5a2 6612bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched);
5ad6ece8 6613
a9ab13ff 6614static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu)
5ad6ece8 6615{
a9ab13ff 6616 enum exit_fastpath_completion exit_fastpath;
5ad6ece8
SC		 6617 struct vcpu_vmx *vmx = to_vmx(vcpu);
6618 unsigned long cr3, cr4;
6619
6620 /* Record the guest's net vcpu time for enforced NMI injections. */
6621 if (unlikely(!enable_vnmi &&
6622 vmx->loaded_vmcs->soft_vnmi_blocked))
6623 vmx->loaded_vmcs->entry_time = ktime_get();
6624
6625 /* Don't enter VMX if guest state is invalid, let the exit handler
6626 start emulation until we arrive back to a valid state */
6627 if (vmx->emulation_required)
a9ab13ff 6628 return EXIT_FASTPATH_NONE;
5ad6ece8
SC		 6629
6630 if (vmx->ple_window_dirty) {
6631 vmx->ple_window_dirty = false;
6632 vmcs_write32(PLE_WINDOW, vmx->ple_window);
6633 }
6634
c9dfd3fb 6635 /*
6636 * We did this in prepare_switch_to_guest, because it needs to
6637 * be within srcu_read_lock.
6638 */
6639 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync);
5ad6ece8 6640
cb3c1e2f 6641 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP))
5ad6ece8 6642 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
cb3c1e2f 6643 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP))
5ad6ece8
SC		 6644 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
6645
6646 cr3 = __get_current_cr3_fast();
6647 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
6648 vmcs_writel(HOST_CR3, cr3);
6649 vmx->loaded_vmcs->host_state.cr3 = cr3;
6650 }
6651
6652 cr4 = cr4_read_shadow();
6653 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
6654 vmcs_writel(HOST_CR4, cr4);
6655 vmx->loaded_vmcs->host_state.cr4 = cr4;
6656 }
6657
6658 /* When single-stepping over STI and MOV SS, we must clear the
6659 * corresponding interruptibility bits in the guest state. Otherwise
6660 * vmentry fails as it then expects bit 14 (BS) in pending debug
6661 * exceptions being set, but that's not correct for the guest debugging
6662 * case. */
6663 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
6664 vmx_set_interrupt_shadow(vcpu, 0);
6665
139a12cf 6666 kvm_load_guest_xsave_state(vcpu);
1811d979 6667
5ad6ece8
SC		 6668 pt_guest_enter(vmx);
6669
041bc42c
WL		 6670 if (vcpu_to_pmu(vcpu)->version)
6671 atomic_switch_perf_msrs(vmx);
6e3ba4ab 6672 atomic_switch_umwait_control_msr(vmx);
5ad6ece8 6673
804939ea
SC		 6674 if (enable_preemption_timer)
6675 vmx_update_hv_timer(vcpu);
5ad6ece8 6676
b6c4bc65
WL		 6677 if (lapic_in_kernel(vcpu) &&
6678 vcpu->arch.apic->lapic_timer.timer_advance_ns)
6679 kvm_wait_lapic_expire(vcpu);
6680
5ad6ece8
SC		 6681 /*
6682 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
6683 * it's non-zero. Since vmentry is serialising on affected CPUs, there
6684 * is no need to worry about the conditional branch over the wrmsr
6685 * being speculatively taken.
6686 */
6687 x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
6688
fa4bff16 6689 /* L1D Flush includes CPU buffer clear to mitigate MDS */
c823dd5c
SC		 6690 if (static_branch_unlikely(&vmx_l1d_should_flush))
6691 vmx_l1d_flush(vcpu);
fa4bff16
LT		 6692 else if (static_branch_unlikely(&mds_user_clear))
6693 mds_clear_cpu_buffers();
c823dd5c
SC		 6694
6695 if (vcpu->arch.cr2 != read_cr2())
6696 write_cr2(vcpu->arch.cr2);
6697
fc2ba5a2
SC		 6698 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
6699 vmx->loaded_vmcs->launched);
c823dd5c
SC		 6700
6701 vcpu->arch.cr2 = read_cr2();
b6b8a145 6702
55d2375e
SC		 6703 /*
6704 * We do not use IBRS in the kernel. If this vCPU has used the
6705 * SPEC_CTRL MSR it may have left it on; save the value and
6706 * turn it off. This is much more efficient than blindly adding
6707 * it to the atomic save/restore list. Especially as the former
6708 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
6709 *
6710 * For non-nested case:
6711 * If the L01 MSR bitmap does not intercept the MSR, then we need to
6712 * save it.
6713 *
6714 * For nested case:
6715 * If the L02 MSR bitmap does not intercept the MSR, then we need to
6716 * save it.
6717 */
6718 if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
6719 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
b6b8a145 6720
55d2375e 6721 x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
d264ee0c 6722
55d2375e
SC		 6723 /* All fields are clean at this point */
6724 if (static_branch_unlikely(&enable_evmcs))
6725 current_evmcs->hv_clean_fields |=
6726 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
f4124500 6727
6f6a657c
VK		 6728 if (static_branch_unlikely(&enable_evmcs))
6729 current_evmcs->hv_vp_id = vcpu->arch.hyperv.vp_index;
6730
55d2375e
SC		 6731 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6732 if (vmx->host_debugctlmsr)
6733 update_debugctlmsr(vmx->host_debugctlmsr);
f4124500 6734
55d2375e
SC		 6735#ifndef CONFIG_X86_64
6736 /*
6737 * The sysexit path does not restore ds/es, so we must set them to
6738 * a reasonable value ourselves.
6739 *
6740 * We can't defer this to vmx_prepare_switch_to_host() since that
6741 * function may be executed in interrupt context, which saves and
6742 * restore segments around it, nullifying its effect.
6743 */
6744 loadsegment(ds, __USER_DS);
6745 loadsegment(es, __USER_DS);
6746#endif
4704d0be 6747
e5d03de5 6748 vmx_register_cache_reset(vcpu);
7854cbca 6749
2ef444f1
CP		 6750 pt_guest_exit(vmx);
6751
139a12cf 6752 kvm_load_host_xsave_state(vcpu);
1811d979 6753
55d2375e
SC		 6754 vmx->nested.nested_run_pending = 0;
6755 vmx->idt_vectoring_info = 0;
119a9c01 6756
873e1da1
SC		 6757 if (unlikely(vmx->fail)) {
6758 vmx->exit_reason = 0xdead;
a9ab13ff 6759 return EXIT_FASTPATH_NONE;
873e1da1
SC		 6760 }
6761
6762 vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
6763 if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY))
beb8d93b
SC		 6764 kvm_machine_check();
6765
873e1da1 6766 if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
a9ab13ff
WL		 6767 return EXIT_FASTPATH_NONE;
6768
6769 if (!is_guest_mode(vcpu) && vmx->exit_reason == EXIT_REASON_MSR_WRITE)
6770 exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
6771 else
6772 exit_fastpath = EXIT_FASTPATH_NONE;
608406e2 6773
55d2375e
SC		 6774 vmx->loaded_vmcs->launched = 1;
6775 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
c18911a2 6776
55d2375e
SC		 6777 vmx_recover_nmi_blocking(vmx);
6778 vmx_complete_interrupts(vmx);
a9ab13ff
WL		 6779
6780 return exit_fastpath;
55d2375e 6781}
2996fca0 6782
55d2375e 6783static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
cf8b84f4 6784{
55d2375e 6785 struct vcpu_vmx *vmx = to_vmx(vcpu);
4704d0be 6786
55d2375e
SC		 6787 if (enable_pml)
6788 vmx_destroy_pml_buffer(vmx);
6789 free_vpid(vmx->vpid);
55d2375e
SC		 6790 nested_vmx_free_vcpu(vcpu);
6791 free_loaded_vmcs(vmx->loaded_vmcs);
55d2375e 6792}
4704d0be 6793
987b2594 6794static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
55d2375e 6795{
41836839 6796 struct vcpu_vmx *vmx;
55d2375e 6797 unsigned long *msr_bitmap;
34109c04 6798 int i, cpu, err;
4704d0be 6799
a9dd6f09
SC		 6800 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0);
6801 vmx = to_vmx(vcpu);
d9a710e5 6802
55d2375e 6803 err = -ENOMEM;
b666a4b6 6804
55d2375e 6805 vmx->vpid = allocate_vpid();
7cdc2d62 6806
5f3d5799 6807 /*
55d2375e
SC		 6808 * If PML is turned on, failure on enabling PML just results in failure
6809 * of creating the vcpu, therefore we can simplify PML logic (by
6810 * avoiding dealing with cases, such as enabling PML partially on vcpus
67b0ae43 6811 * for the guest), etc.
5f3d5799 6812 */
55d2375e 6813 if (enable_pml) {
41836839 6814 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
55d2375e 6815 if (!vmx->pml_pg)
987b2594 6816 goto free_vpid;
55d2375e 6817 }
4704d0be 6818
7d73710d 6819 BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) != NR_SHARED_MSRS);
4704d0be 6820
4be53410
XL		 6821 for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
6822 u32 index = vmx_msr_index[i];
6823 u32 data_low, data_high;
6824 int j = vmx->nmsrs;
6825
6826 if (rdmsr_safe(index, &data_low, &data_high) < 0)
6827 continue;
6828 if (wrmsr_safe(index, data_low, data_high) < 0)
6829 continue;
46f4f0aa 6830
4be53410
XL		 6831 vmx->guest_msrs[j].index = i;
6832 vmx->guest_msrs[j].data = 0;
46f4f0aa
PB		 6833 switch (index) {
6834 case MSR_IA32_TSX_CTRL:
6835 /*
6836 * No need to pass TSX_CTRL_CPUID_CLEAR through, so
6837 * let's avoid changing CPUID bits under the host
6838 * kernel's feet.
6839 */
6840 vmx->guest_msrs[j].mask = ~(u64)TSX_CTRL_CPUID_CLEAR;
6841 break;
6842 default:
6843 vmx->guest_msrs[j].mask = -1ull;
6844 break;
6845 }
4be53410
XL		 6846 ++vmx->nmsrs;
6847 }
6848
55d2375e
SC		 6849 err = alloc_loaded_vmcs(&vmx->vmcs01);
6850 if (err < 0)
7d73710d 6851 goto free_pml;
cb61de2f 6852
55d2375e 6853 msr_bitmap = vmx->vmcs01.msr_bitmap;
788fc1e9 6854 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_TSC, MSR_TYPE_R);
55d2375e
SC		 6855 vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
6856 vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
6857 vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
6858 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
6859 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
6860 vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
987b2594 6861 if (kvm_cstate_in_guest(vcpu->kvm)) {
b5170063
WL		 6862 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C1_RES, MSR_TYPE_R);
6863 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R);
6864 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R);
6865 vmx_disable_intercept_for_msr(msr_bitmap, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R);
6866 }
55d2375e 6867 vmx->msr_bitmap_mode = 0;
4704d0be 6868
55d2375e
SC		 6869 vmx->loaded_vmcs = &vmx->vmcs01;
6870 cpu = get_cpu();
34109c04
SC		 6871 vmx_vcpu_load(vcpu, cpu);
6872 vcpu->cpu = cpu;
1b84292b 6873 init_vmcs(vmx);
34109c04 6874 vmx_vcpu_put(vcpu);
55d2375e 6875 put_cpu();
34109c04 6876 if (cpu_need_virtualize_apic_accesses(vcpu)) {
987b2594 6877 err = alloc_apic_access_page(vcpu->kvm);
55d2375e
SC		 6878 if (err)
6879 goto free_vmcs;
6880 }
6881
6882 if (enable_ept && !enable_unrestricted_guest) {
987b2594 6883 err = init_rmode_identity_map(vcpu->kvm);
55d2375e
SC		 6884 if (err)
6885 goto free_vmcs;
6886 }
4704d0be 6887
55d2375e
SC		 6888 if (nested)
6889 nested_vmx_setup_ctls_msrs(&vmx->nested.msrs,
a4443267 6890 vmx_capability.ept);
55d2375e
SC		 6891 else
6892 memset(&vmx->nested.msrs, 0, sizeof(vmx->nested.msrs));
bd18bffc 6893
55d2375e
SC		 6894 vmx->nested.posted_intr_nv = -1;
6895 vmx->nested.current_vmptr = -1ull;
bd18bffc 6896
bab0c318 6897 vcpu->arch.microcode_version = 0x100000000ULL;
32ad73db 6898 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED;
feaf0c7d 6899
6f1e03bc 6900 /*
55d2375e
SC		 6901 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
6902 * or POSTED_INTR_WAKEUP_VECTOR.
6f1e03bc 6903 */
55d2375e
SC		 6904 vmx->pi_desc.nv = POSTED_INTR_VECTOR;
6905 vmx->pi_desc.sn = 1;
4704d0be 6906
53963a70
LT		 6907 vmx->ept_pointer = INVALID_PAGE;
6908
a9dd6f09 6909 return 0;
4704d0be 6910
55d2375e
SC		 6911free_vmcs:
6912 free_loaded_vmcs(vmx->loaded_vmcs);
55d2375e
SC		 6913free_pml:
6914 vmx_destroy_pml_buffer(vmx);
987b2594 6915free_vpid:
55d2375e 6916 free_vpid(vmx->vpid);
a9dd6f09 6917 return err;
55d2375e 6918}
36be0b9d 6919
65fd4cb6
TG		 6920#define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
6921#define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
21feb4eb 6922
55d2375e
SC		 6923static int vmx_vm_init(struct kvm *kvm)
6924{
6925 spin_lock_init(&to_kvm_vmx(kvm)->ept_pointer_lock);
ff651cb6 6926
55d2375e
SC		 6927 if (!ple_gap)
6928 kvm->arch.pause_in_guest = true;
3af18d9c 6929
55d2375e
SC		 6930 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) {
6931 switch (l1tf_mitigation) {
6932 case L1TF_MITIGATION_OFF:
6933 case L1TF_MITIGATION_FLUSH_NOWARN:
6934 /* 'I explicitly don't care' is set */
6935 break;
6936 case L1TF_MITIGATION_FLUSH:
6937 case L1TF_MITIGATION_FLUSH_NOSMT:
6938 case L1TF_MITIGATION_FULL:
6939 /*
6940 * Warn upon starting the first VM in a potentially
6941 * insecure environment.
6942 */
b284909a 6943 if (sched_smt_active())
55d2375e
SC		 6944 pr_warn_once(L1TF_MSG_SMT);
6945 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
6946 pr_warn_once(L1TF_MSG_L1D);
6947 break;
6948 case L1TF_MITIGATION_FULL_FORCE:
6949 /* Flush is enforced */
6950 break;
6951 }
6952 }
4e19c36f 6953 kvm_apicv_init(kvm, enable_apicv);
55d2375e 6954 return 0;
4704d0be
NHE		 6955}
6956
f257d6dc 6957static int __init vmx_check_processor_compat(void)
bd18bffc 6958{
55d2375e
SC		 6959 struct vmcs_config vmcs_conf;
6960 struct vmx_capability vmx_cap;
bd18bffc 6961
ff10e22e
SC		 6962 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
6963 !this_cpu_has(X86_FEATURE_VMX)) {
6964 pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id());
6965 return -EIO;
6966 }
6967
55d2375e 6968 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0)
f257d6dc 6969 return -EIO;
55d2375e 6970 if (nested)
a4443267 6971 nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept);
55d2375e
SC		 6972 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
6973 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
6974 smp_processor_id());
f257d6dc 6975 return -EIO;
bd18bffc 6976 }
f257d6dc 6977 return 0;
bd18bffc
SC		 6978}
6979
55d2375e 6980static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
bd18bffc 6981{
55d2375e
SC		 6982 u8 cache;
6983 u64 ipat = 0;
bd18bffc 6984
222f06e7
CW		 6985 /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in
6986 * memory aliases with conflicting memory types and sometimes MCEs.
6987 * We have to be careful as to what are honored and when.
6988 *
6989 * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to
6990 * UC. The effective memory type is UC or WC depending on guest PAT.
6991 * This was historically the source of MCEs and we want to be
6992 * conservative.
6993 *
6994 * When there is no need to deal with noncoherent DMA (e.g., no VT-d
6995 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The
6996 * EPT memory type is set to WB. The effective memory type is forced
6997 * WB.
6998 *
6999 * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The
7000 * EPT memory type is used to emulate guest CD/MTRR.
bd18bffc 7001 */
222f06e7 7002
55d2375e
SC		 7003 if (is_mmio) {
7004 cache = MTRR_TYPE_UNCACHABLE;
7005 goto exit;
7006 }
bd18bffc 7007
55d2375e
SC		 7008 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
7009 ipat = VMX_EPT_IPAT_BIT;
7010 cache = MTRR_TYPE_WRBACK;
7011 goto exit;
7012 }
bd18bffc 7013
55d2375e
SC		 7014 if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
7015 ipat = VMX_EPT_IPAT_BIT;
7016 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
7017 cache = MTRR_TYPE_WRBACK;
7018 else
7019 cache = MTRR_TYPE_UNCACHABLE;
7020 goto exit;
7021 }
bd18bffc 7022
55d2375e 7023 cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
bd18bffc 7024
55d2375e
SC		 7025exit:
7026 return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
7027}
bd18bffc 7028
fe7f895d 7029static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx)
55d2375e 7030{
bd18bffc 7031 /*
55d2375e
SC		 7032 * These bits in the secondary execution controls field
7033 * are dynamic, the others are mostly based on the hypervisor
7034 * architecture and the guest's CPUID. Do not touch the
7035 * dynamic bits.
bd18bffc 7036 */
55d2375e
SC		 7037 u32 mask =
7038 SECONDARY_EXEC_SHADOW_VMCS |
7039 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
7040 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
7041 SECONDARY_EXEC_DESC;
bd18bffc 7042
fe7f895d
SC		 7043 u32 new_ctl = vmx->secondary_exec_control;
7044 u32 cur_ctl = secondary_exec_controls_get(vmx);
bd18bffc 7045
fe7f895d 7046 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask));
bd18bffc
SC		 7047}
7048
4704d0be 7049/*
55d2375e
SC		 7050 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
7051 * (indicating "allowed-1") if they are supported in the guest's CPUID.
4704d0be 7052 */
55d2375e 7053static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
4704d0be
NHE		 7054{
7055 struct vcpu_vmx *vmx = to_vmx(vcpu);
55d2375e 7056 struct kvm_cpuid_entry2 *entry;
4704d0be 7057
55d2375e
SC		 7058 vmx->nested.msrs.cr0_fixed1 = 0xffffffff;
7059 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE;
e79f245d 7060
55d2375e
SC		 7061#define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \
7062 if (entry && (entry->_reg & (_cpuid_mask))) \
7063 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \
7064} while (0)
ff651cb6 7065
55d2375e 7066 entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
87382003
SC		 7067 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME));
7068 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME));
7069 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC));
7070 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE));
7071 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE));
7072 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE));
7073 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE));
7074 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE));
7075 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR));
7076 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM));
7077 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX));
7078 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX));
7079 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID));
7080 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE));
61ada748 7081
55d2375e 7082 entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
87382003
SC		 7083 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE));
7084 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP));
7085 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP));
7086 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU));
7087 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP));
7088 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57));
cf3215d9 7089
55d2375e
SC		 7090#undef cr4_fixed1_update
7091}
36c3cc42 7092
55d2375e
SC		 7093static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
7094{
7095 struct vcpu_vmx *vmx = to_vmx(vcpu);
f459a707 7096
55d2375e
SC		 7097 if (kvm_mpx_supported()) {
7098 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX);
4704d0be 7099
55d2375e
SC		 7100 if (mpx_enabled) {
7101 vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
7102 vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
7103 } else {
7104 vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS;
7105 vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS;
7106 }
dccbfcf5 7107 }
55d2375e 7108}
4704d0be 7109
6c0f0bba
LK		 7110static void update_intel_pt_cfg(struct kvm_vcpu *vcpu)
7111{
7112 struct vcpu_vmx *vmx = to_vmx(vcpu);
7113 struct kvm_cpuid_entry2 *best = NULL;
7114 int i;
7115
7116 for (i = 0; i < PT_CPUID_LEAVES; i++) {
7117 best = kvm_find_cpuid_entry(vcpu, 0x14, i);
7118 if (!best)
7119 return;
7120 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax;
7121 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx;
7122 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx;
7123 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx;
7124 }
7125
7126 /* Get the number of configurable Address Ranges for filtering */
7127 vmx->pt_desc.addr_range = intel_pt_validate_cap(vmx->pt_desc.caps,
7128 PT_CAP_num_address_ranges);
7129
7130 /* Initialize and clear the no dependency bits */
7131 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS |
7132 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC);
7133
7134 /*
7135 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise
7136 * will inject an #GP
7137 */
7138 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering))
7139 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN;
7140
7141 /*
7142 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and
7143 * PSBFreq can be set
7144 */
7145 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc))
7146 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC |
7147 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ);
7148
7149 /*
7150 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn BranchEn and
7151 * MTCFreq can be set
7152 */
7153 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc))
7154 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN |
7155 RTIT_CTL_BRANCH_EN | RTIT_CTL_MTC_RANGE);
7156
7157 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */
7158 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite))
7159 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW |
7160 RTIT_CTL_PTW_EN);
7161
7162 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */
7163 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace))
7164 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN;
7165
7166 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */
7167 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output))
7168 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA;
7169
7170 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabircEn can be set */
7171 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys))
7172 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN;
7173
7174 /* unmask address range configure area */
7175 for (i = 0; i < vmx->pt_desc.addr_range; i++)
d14eff1b 7176 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4));
6c0f0bba
LK		 7177}
7178
55d2375e
SC		 7179static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
7180{
7181 struct vcpu_vmx *vmx = to_vmx(vcpu);
4704d0be 7182
7204160e
AL		 7183 /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */
7184 vcpu->arch.xsaves_enabled = false;
7185
55d2375e
SC		 7186 if (cpu_has_secondary_exec_ctrls()) {
7187 vmx_compute_secondary_exec_control(vmx);
fe7f895d 7188 vmcs_set_secondary_exec_control(vmx);
705699a1 7189 }
4704d0be 7190
55d2375e
SC		 7191 if (nested_vmx_allowed(vcpu))
7192 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
32ad73db
SC		 7193 FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
7194 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
55d2375e
SC		 7195 else
7196 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
32ad73db
SC		 7197 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
7198 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX);
4f350c6d 7199
55d2375e
SC		 7200 if (nested_vmx_allowed(vcpu)) {
7201 nested_vmx_cr_fixed1_bits_update(vcpu);
7202 nested_vmx_entry_exit_ctls_update(vcpu);
4f350c6d 7203 }
6c0f0bba
LK		 7204
7205 if (boot_cpu_has(X86_FEATURE_INTEL_PT) &&
7206 guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT))
7207 update_intel_pt_cfg(vcpu);
b07a5c53
PB		 7208
7209 if (boot_cpu_has(X86_FEATURE_RTM)) {
7210 struct shared_msr_entry *msr;
7211 msr = find_msr_entry(vmx, MSR_IA32_TSX_CTRL);
7212 if (msr) {
7213 bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM);
7214 vmx_set_guest_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE);
7215 }
7216 }
55d2375e 7217}
09abb5e3 7218
3ec6fd8c 7219static __init void vmx_set_cpu_caps(void)
55d2375e 7220{
3ec6fd8c
SC		 7221 kvm_set_cpu_caps();
7222
7223 /* CPUID 0x1 */
7224 if (nested)
7225 kvm_cpu_cap_set(X86_FEATURE_VMX);
7226
7227 /* CPUID 0x7 */
8721f5b0
SC		 7228 if (kvm_mpx_supported())
7229 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX);
7230 if (cpu_has_vmx_invpcid())
7231 kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID);
7232 if (vmx_pt_mode_is_host_guest())
7233 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT);
3ec6fd8c
SC		 7234
7235 /* PKU is not yet implemented for shadow paging. */
8721f5b0
SC		 7236 if (enable_ept && boot_cpu_has(X86_FEATURE_OSPKE))
7237 kvm_cpu_cap_check_and_set(X86_FEATURE_PKU);
3ec6fd8c 7238
90d2f60f
SC		 7239 if (vmx_umip_emulated())
7240 kvm_cpu_cap_set(X86_FEATURE_UMIP);
7241
b3d895d5 7242 /* CPUID 0xD.1 */
408e9a31 7243 supported_xss = 0;
b3d895d5
SC		 7244 if (!vmx_xsaves_supported())
7245 kvm_cpu_cap_clear(X86_FEATURE_XSAVES);
7246
3ec6fd8c
SC		 7247 /* CPUID 0x80000001 */
7248 if (!cpu_has_vmx_rdtscp())
7249 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
4704d0be
NHE		 7250}
7251
55d2375e 7252static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
42124925 7253{
55d2375e 7254 to_vmx(vcpu)->req_immediate_exit = true;
7c177938
NHE		 7255}
7256
35a57134
OU		 7257static int vmx_check_intercept_io(struct kvm_vcpu *vcpu,
7258 struct x86_instruction_info *info)
7259{
7260 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7261 unsigned short port;
7262 bool intercept;
7263 int size;
7264
7265 if (info->intercept == x86_intercept_in ||
7266 info->intercept == x86_intercept_ins) {
7267 port = info->src_val;
7268 size = info->dst_bytes;
7269 } else {
7270 port = info->dst_val;
7271 size = info->src_bytes;
7272 }
7273
7274 /*
7275 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction
7276 * VM-exits depend on the 'unconditional IO exiting' VM-execution
7277 * control.
7278 *
7279 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps.
7280 */
7281 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
7282 intercept = nested_cpu_has(vmcs12,
7283 CPU_BASED_UNCOND_IO_EXITING);
7284 else
7285 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size);
7286
86f7e90c 7287 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */
35a57134
OU		 7288 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
7289}
7290
8a76d7f2
JR		 7291static int vmx_check_intercept(struct kvm_vcpu *vcpu,
7292 struct x86_instruction_info *info,
21f1b8f2
SC		 7293 enum x86_intercept_stage stage,
7294 struct x86_exception *exception)
8a76d7f2 7295{
fb6d4d34 7296 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
fb6d4d34 7297
35a57134 7298 switch (info->intercept) {
fb6d4d34
PB		 7299 /*
7300 * RDPID causes #UD if disabled through secondary execution controls.
7301 * Because it is marked as EmulateOnUD, we need to intercept it here.
7302 */
35a57134
OU		 7303 case x86_intercept_rdtscp:
7304 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) {
21f1b8f2
SC		 7305 exception->vector = UD_VECTOR;
7306 exception->error_code_valid = false;
35a57134
OU		 7307 return X86EMUL_PROPAGATE_FAULT;
7308 }
7309 break;
7310
7311 case x86_intercept_in:
7312 case x86_intercept_ins:
7313 case x86_intercept_out:
7314 case x86_intercept_outs:
7315 return vmx_check_intercept_io(vcpu, info);
fb6d4d34 7316
86f7e90c
OU		 7317 case x86_intercept_lgdt:
7318 case x86_intercept_lidt:
7319 case x86_intercept_lldt:
7320 case x86_intercept_ltr:
7321 case x86_intercept_sgdt:
7322 case x86_intercept_sidt:
7323 case x86_intercept_sldt:
7324 case x86_intercept_str:
7325 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC))
7326 return X86EMUL_CONTINUE;
7327
7328 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */
7329 break;
7330
fb6d4d34 7331 /* TODO: check more intercepts... */
35a57134
OU		 7332 default:
7333 break;
7334 }
7335
07721fee 7336 return X86EMUL_UNHANDLEABLE;
8a76d7f2
JR		 7337}
7338
64672c95
YJ		 7339#ifdef CONFIG_X86_64
7340/* (a << shift) / divisor, return 1 if overflow otherwise 0 */
7341static inline int u64_shl_div_u64(u64 a, unsigned int shift,
7342 u64 divisor, u64 *result)
7343{
7344 u64 low = a << shift, high = a >> (64 - shift);
7345
7346 /* To avoid the overflow on divq */
7347 if (high >= divisor)
7348 return 1;
7349
7350 /* Low hold the result, high hold rem which is discarded */
7351 asm("divq %2\n\t" : "=a" (low), "=d" (high) :
7352 "rm" (divisor), "0" (low), "1" (high));
7353 *result = low;
7354
7355 return 0;
7356}
7357
f9927982
SC		 7358static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
7359 bool *expired)
64672c95 7360{
386c6ddb 7361 struct vcpu_vmx *vmx;
c5ce8235 7362 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
39497d76 7363 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer;
386c6ddb 7364
0c5f81da
WL		 7365 if (kvm_mwait_in_guest(vcpu->kvm) ||
7366 kvm_can_post_timer_interrupt(vcpu))
386c6ddb
KA		 7367 return -EOPNOTSUPP;
7368
7369 vmx = to_vmx(vcpu);
7370 tscl = rdtsc();
7371 guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
7372 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
39497d76
SC		 7373 lapic_timer_advance_cycles = nsec_to_cycles(vcpu,
7374 ktimer->timer_advance_ns);
c5ce8235
WL		 7375
7376 if (delta_tsc > lapic_timer_advance_cycles)
7377 delta_tsc -= lapic_timer_advance_cycles;
7378 else
7379 delta_tsc = 0;
64672c95
YJ		 7380
7381 /* Convert to host delta tsc if tsc scaling is enabled */
7382 if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
0967fa1c 7383 delta_tsc && u64_shl_div_u64(delta_tsc,
64672c95 7384 kvm_tsc_scaling_ratio_frac_bits,
0967fa1c 7385 vcpu->arch.tsc_scaling_ratio, &delta_tsc))
64672c95
YJ		 7386 return -ERANGE;
7387
7388 /*
7389 * If the delta tsc can't fit in the 32 bit after the multi shift,
7390 * we can't use the preemption timer.
7391 * It's possible that it fits on later vmentries, but checking
7392 * on every vmentry is costly so we just use an hrtimer.
7393 */
7394 if (delta_tsc >> (cpu_preemption_timer_multi + 32))
7395 return -ERANGE;
7396
7397 vmx->hv_deadline_tsc = tscl + delta_tsc;
f9927982
SC		 7398 *expired = !delta_tsc;
7399 return 0;
64672c95
YJ		 7400}
7401
7402static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
7403{
f459a707 7404 to_vmx(vcpu)->hv_deadline_tsc = -1;
64672c95
YJ		 7405}
7406#endif
7407
48d89b92 7408static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
ae97a3b8 7409{
b31c114b 7410 if (!kvm_pause_in_guest(vcpu->kvm))
b4a2d31d 7411 shrink_ple_window(vcpu);
ae97a3b8
RK		 7412}
7413
843e4330
KH		 7414static void vmx_slot_enable_log_dirty(struct kvm *kvm,
7415 struct kvm_memory_slot *slot)
7416{
3c9bd400
JZ		 7417 if (!kvm_dirty_log_manual_protect_and_init_set(kvm))
7418 kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
843e4330
KH		 7419 kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
7420}
7421
7422static void vmx_slot_disable_log_dirty(struct kvm *kvm,
7423 struct kvm_memory_slot *slot)
7424{
7425 kvm_mmu_slot_set_dirty(kvm, slot);
7426}
7427
7428static void vmx_flush_log_dirty(struct kvm *kvm)
7429{
7430 kvm_flush_pml_buffers(kvm);
7431}
7432
c5f983f6
BD		 7433static int vmx_write_pml_buffer(struct kvm_vcpu *vcpu)
7434{
7435 struct vmcs12 *vmcs12;
7436 struct vcpu_vmx *vmx = to_vmx(vcpu);
3d5f6beb 7437 gpa_t gpa, dst;
c5f983f6
BD		 7438
7439 if (is_guest_mode(vcpu)) {
7440 WARN_ON_ONCE(vmx->nested.pml_full);
7441
7442 /*
7443 * Check if PML is enabled for the nested guest.
7444 * Whether eptp bit 6 is set is already checked
7445 * as part of A/D emulation.
7446 */
7447 vmcs12 = get_vmcs12(vcpu);
7448 if (!nested_cpu_has_pml(vmcs12))
7449 return 0;
7450
4769886b 7451 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
c5f983f6
BD		 7452 vmx->nested.pml_full = true;
7453 return 1;
7454 }
7455
7456 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS) & ~0xFFFull;
3d5f6beb 7457 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index;
c5f983f6 7458
3d5f6beb
KA		 7459 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa,
7460 offset_in_page(dst), sizeof(gpa)))
c5f983f6
BD		 7461 return 0;
7462
3d5f6beb 7463 vmcs12->guest_pml_index--;
c5f983f6
BD		 7464 }
7465
7466 return 0;
7467}
7468
843e4330
KH		 7469static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
7470 struct kvm_memory_slot *memslot,
7471 gfn_t offset, unsigned long mask)
7472{
7473 kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
7474}
7475
cd39e117
PB		 7476static void __pi_post_block(struct kvm_vcpu *vcpu)
7477{
7478 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
7479 struct pi_desc old, new;
7480 unsigned int dest;
cd39e117
PB		 7481
7482 do {
7483 old.control = new.control = pi_desc->control;
8b306e2f
PB		 7484 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
7485 "Wakeup handler not enabled while the VCPU is blocked\n");
cd39e117
PB		 7486
7487 dest = cpu_physical_id(vcpu->cpu);
7488
7489 if (x2apic_enabled())
7490 new.ndst = dest;
7491 else
7492 new.ndst = (dest << 8) & 0xFF00;
7493
cd39e117
PB		 7494 /* set 'NV' to 'notification vector' */
7495 new.nv = POSTED_INTR_VECTOR;
c0a1666b
PB		 7496 } while (cmpxchg64(&pi_desc->control, old.control,
7497 new.control) != old.control);
cd39e117 7498
8b306e2f
PB		 7499 if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
7500 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
cd39e117 7501 list_del(&vcpu->blocked_vcpu_list);
8b306e2f 7502 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
cd39e117
PB		 7503 vcpu->pre_pcpu = -1;
7504 }
7505}
7506
bf9f6ac8
FW		 7507/*
7508 * This routine does the following things for vCPU which is going
7509 * to be blocked if VT-d PI is enabled.
7510 * - Store the vCPU to the wakeup list, so when interrupts happen
7511 * we can find the right vCPU to wake up.
7512 * - Change the Posted-interrupt descriptor as below:
7513 * 'NDST' <-- vcpu->pre_pcpu
7514 * 'NV' <-- POSTED_INTR_WAKEUP_VECTOR
7515 * - If 'ON' is set during this process, which means at least one
7516 * interrupt is posted for this vCPU, we cannot block it, in
7517 * this case, return 1, otherwise, return 0.
7518 *
7519 */
bc22512b 7520static int pi_pre_block(struct kvm_vcpu *vcpu)
bf9f6ac8 7521{
bf9f6ac8
FW		 7522 unsigned int dest;
7523 struct pi_desc old, new;
7524 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
7525
7526 if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
a0052191
YZ		 7527 !irq_remapping_cap(IRQ_POSTING_CAP) ||
7528 !kvm_vcpu_apicv_active(vcpu))
bf9f6ac8
FW		 7529 return 0;
7530
8b306e2f
PB		 7531 WARN_ON(irqs_disabled());
7532 local_irq_disable();
7533 if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
7534 vcpu->pre_pcpu = vcpu->cpu;
7535 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7536 list_add_tail(&vcpu->blocked_vcpu_list,
7537 &per_cpu(blocked_vcpu_on_cpu,
7538 vcpu->pre_pcpu));
7539 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7540 }
bf9f6ac8
FW		 7541
7542 do {
7543 old.control = new.control = pi_desc->control;
7544
bf9f6ac8
FW		 7545 WARN((pi_desc->sn == 1),
7546 "Warning: SN field of posted-interrupts "
7547 "is set before blocking\n");
7548
7549 /*
7550 * Since vCPU can be preempted during this process,
7551 * vcpu->cpu could be different with pre_pcpu, we
7552 * need to set pre_pcpu as the destination of wakeup
7553 * notification event, then we can find the right vCPU
7554 * to wakeup in wakeup handler if interrupts happen
7555 * when the vCPU is in blocked state.
7556 */
7557 dest = cpu_physical_id(vcpu->pre_pcpu);
7558
7559 if (x2apic_enabled())
7560 new.ndst = dest;
7561 else
7562 new.ndst = (dest << 8) & 0xFF00;
7563
7564 /* set 'NV' to 'wakeup vector' */
7565 new.nv = POSTED_INTR_WAKEUP_VECTOR;
c0a1666b
PB		 7566 } while (cmpxchg64(&pi_desc->control, old.control,
7567 new.control) != old.control);
bf9f6ac8 7568
8b306e2f
PB		 7569 /* We should not block the vCPU if an interrupt is posted for it. */
7570 if (pi_test_on(pi_desc) == 1)
7571 __pi_post_block(vcpu);
7572
7573 local_irq_enable();
7574 return (vcpu->pre_pcpu == -1);
bf9f6ac8
FW		 7575}
7576
bc22512b
YJ		 7577static int vmx_pre_block(struct kvm_vcpu *vcpu)
7578{
7579 if (pi_pre_block(vcpu))
7580 return 1;
7581
64672c95
YJ		 7582 if (kvm_lapic_hv_timer_in_use(vcpu))
7583 kvm_lapic_switch_to_sw_timer(vcpu);
7584
bc22512b
YJ		 7585 return 0;
7586}
7587
7588static void pi_post_block(struct kvm_vcpu *vcpu)
bf9f6ac8 7589{
8b306e2f 7590 if (vcpu->pre_pcpu == -1)
bf9f6ac8
FW		 7591 return;
7592
8b306e2f
PB		 7593 WARN_ON(irqs_disabled());
7594 local_irq_disable();
cd39e117 7595 __pi_post_block(vcpu);
8b306e2f 7596 local_irq_enable();
bf9f6ac8
FW		 7597}
7598
bc22512b
YJ		 7599static void vmx_post_block(struct kvm_vcpu *vcpu)
7600{
afaf0b2f 7601 if (kvm_x86_ops.set_hv_timer)
64672c95
YJ		 7602 kvm_lapic_switch_to_hv_timer(vcpu);
7603
bc22512b
YJ		 7604 pi_post_block(vcpu);
7605}
7606
efc64404
FW		 7607/*
7608 * vmx_update_pi_irte - set IRTE for Posted-Interrupts
7609 *
7610 * @kvm: kvm
7611 * @host_irq: host irq of the interrupt
7612 * @guest_irq: gsi of the interrupt
7613 * @set: set or unset PI
7614 * returns 0 on success, < 0 on failure
7615 */
7616static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
7617 uint32_t guest_irq, bool set)
7618{
7619 struct kvm_kernel_irq_routing_entry *e;
7620 struct kvm_irq_routing_table *irq_rt;
7621 struct kvm_lapic_irq irq;
7622 struct kvm_vcpu *vcpu;
7623 struct vcpu_data vcpu_info;
3a8b0677 7624 int idx, ret = 0;
efc64404
FW		 7625
7626 if (!kvm_arch_has_assigned_device(kvm) ||
a0052191
YZ		 7627 !irq_remapping_cap(IRQ_POSTING_CAP) ||
7628 !kvm_vcpu_apicv_active(kvm->vcpus[0]))
efc64404
FW		 7629 return 0;
7630
7631 idx = srcu_read_lock(&kvm->irq_srcu);
7632 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
3a8b0677
JS		 7633 if (guest_irq >= irq_rt->nr_rt_entries ||
7634 hlist_empty(&irq_rt->map[guest_irq])) {
7635 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
7636 guest_irq, irq_rt->nr_rt_entries);
7637 goto out;
7638 }
efc64404
FW		 7639
7640 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
7641 if (e->type != KVM_IRQ_ROUTING_MSI)
7642 continue;
7643 /*
7644 * VT-d PI cannot support posting multicast/broadcast
7645 * interrupts to a vCPU, we still use interrupt remapping
7646 * for these kind of interrupts.
7647 *
7648 * For lowest-priority interrupts, we only support
7649 * those with single CPU as the destination, e.g. user
7650 * configures the interrupts via /proc/irq or uses
7651 * irqbalance to make the interrupts single-CPU.
7652 *
7653 * We will support full lowest-priority interrupt later.
fdcf7562
AG		 7654 *
7655 * In addition, we can only inject generic interrupts using
7656 * the PI mechanism, refuse to route others through it.
efc64404
FW		 7657 */
7658
37131313 7659 kvm_set_msi_irq(kvm, e, &irq);
fdcf7562
AG		 7660 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
7661 !kvm_irq_is_postable(&irq)) {
23a1c257
FW		 7662 /*
7663 * Make sure the IRTE is in remapped mode if
7664 * we don't handle it in posted mode.
7665 */
7666 ret = irq_set_vcpu_affinity(host_irq, NULL);
7667 if (ret < 0) {
7668 printk(KERN_INFO
7669 "failed to back to remapped mode, irq: %u\n",
7670 host_irq);
7671 goto out;
7672 }
7673
efc64404 7674 continue;
23a1c257 7675 }
efc64404
FW		 7676
7677 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
7678 vcpu_info.vector = irq.vector;
7679
2698d82e 7680 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
efc64404
FW		 7681 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
7682
7683 if (set)
7684 ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
dc91f2eb 7685 else
efc64404 7686 ret = irq_set_vcpu_affinity(host_irq, NULL);
efc64404
FW		 7687
7688 if (ret < 0) {
7689 printk(KERN_INFO "%s: failed to update PI IRTE\n",
7690 __func__);
7691 goto out;
7692 }
7693 }
7694
7695 ret = 0;
7696out:
7697 srcu_read_unlock(&kvm->irq_srcu, idx);
7698 return ret;
7699}
7700
c45dcc71
AR		 7701static void vmx_setup_mce(struct kvm_vcpu *vcpu)
7702{
7703 if (vcpu->arch.mcg_cap & MCG_LMCE_P)
7704 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
32ad73db 7705 FEAT_CTL_LMCE_ENABLED;
c45dcc71
AR		 7706 else
7707 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
32ad73db 7708 ~FEAT_CTL_LMCE_ENABLED;
c45dcc71
AR		 7709}
7710
c300ab9f 7711static bool vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
72d7b374 7712{
72e9cbdb
LP		 7713 /* we need a nested vmexit to enter SMM, postpone if run is pending */
7714 if (to_vmx(vcpu)->nested.nested_run_pending)
88c604b6 7715 return false;
a9fa7cb6 7716 return !is_smm(vcpu);
72d7b374
LP		 7717}
7718
0234bf88
LP		 7719static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
7720{
72e9cbdb
LP		 7721 struct vcpu_vmx *vmx = to_vmx(vcpu);
7722
7723 vmx->nested.smm.guest_mode = is_guest_mode(vcpu);
7724 if (vmx->nested.smm.guest_mode)
7725 nested_vmx_vmexit(vcpu, -1, 0, 0);
7726
7727 vmx->nested.smm.vmxon = vmx->nested.vmxon;
7728 vmx->nested.vmxon = false;
caa057a2 7729 vmx_clear_hlt(vcpu);
0234bf88
LP		 7730 return 0;
7731}
7732
ed19321f 7733static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
0234bf88 7734{
72e9cbdb
LP		 7735 struct vcpu_vmx *vmx = to_vmx(vcpu);
7736 int ret;
7737
7738 if (vmx->nested.smm.vmxon) {
7739 vmx->nested.vmxon = true;
7740 vmx->nested.smm.vmxon = false;
7741 }
7742
7743 if (vmx->nested.smm.guest_mode) {
a633e41e 7744 ret = nested_vmx_enter_non_root_mode(vcpu, false);
72e9cbdb
LP		 7745 if (ret)
7746 return ret;
7747
7748 vmx->nested.smm.guest_mode = false;
7749 }
0234bf88
LP		 7750 return 0;
7751}
7752
cc3d967f
LP		 7753static int enable_smi_window(struct kvm_vcpu *vcpu)
7754{
7755 return 0;
7756}
7757
05d5a486
SB		 7758static bool vmx_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
7759{
9481b7f1 7760 return false;
05d5a486
SB		 7761}
7762
4b9852f4
LA		 7763static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
7764{
7765 return to_vmx(vcpu)->nested.vmxon;
7766}
7767
6e4fd06f 7768static void hardware_unsetup(void)
484014fa
SC		 7769{
7770 if (nested)
7771 nested_vmx_hardware_unsetup();
7772
7773 free_kvm_area();
7774}
7775
7776static bool vmx_check_apicv_inhibit_reasons(ulong bit)
7777{
7778 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
7779 BIT(APICV_INHIBIT_REASON_HYPERV);
7780
7781 return supported & BIT(bit);
7782}
7783
e286ac0e 7784static struct kvm_x86_ops vmx_x86_ops __initdata = {
484014fa
SC		 7785 .hardware_unsetup = hardware_unsetup,
7786
7787 .hardware_enable = hardware_enable,
7788 .hardware_disable = hardware_disable,
7789 .cpu_has_accelerated_tpr = report_flexpriority,
7790 .has_emulated_msr = vmx_has_emulated_msr,
7791
7792 .vm_size = sizeof(struct kvm_vmx),
7793 .vm_init = vmx_vm_init,
7794
7795 .vcpu_create = vmx_create_vcpu,
7796 .vcpu_free = vmx_free_vcpu,
7797 .vcpu_reset = vmx_vcpu_reset,
7798
7799 .prepare_guest_switch = vmx_prepare_switch_to_guest,
7800 .vcpu_load = vmx_vcpu_load,
7801 .vcpu_put = vmx_vcpu_put,
7802
7803 .update_bp_intercept = update_exception_bitmap,
7804 .get_msr_feature = vmx_get_msr_feature,
7805 .get_msr = vmx_get_msr,
7806 .set_msr = vmx_set_msr,
7807 .get_segment_base = vmx_get_segment_base,
7808 .get_segment = vmx_get_segment,
7809 .set_segment = vmx_set_segment,
7810 .get_cpl = vmx_get_cpl,
7811 .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
7812 .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
7813 .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
7814 .set_cr0 = vmx_set_cr0,
7815 .set_cr4 = vmx_set_cr4,
7816 .set_efer = vmx_set_efer,
7817 .get_idt = vmx_get_idt,
7818 .set_idt = vmx_set_idt,
7819 .get_gdt = vmx_get_gdt,
7820 .set_gdt = vmx_set_gdt,
484014fa
SC		 7821 .set_dr7 = vmx_set_dr7,
7822 .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
7823 .cache_reg = vmx_cache_reg,
7824 .get_rflags = vmx_get_rflags,
7825 .set_rflags = vmx_set_rflags,
7826
7780938c 7827 .tlb_flush_all = vmx_flush_tlb_all,
eeeb4f67 7828 .tlb_flush_current = vmx_flush_tlb_current,
484014fa 7829 .tlb_flush_gva = vmx_flush_tlb_gva,
e64419d9 7830 .tlb_flush_guest = vmx_flush_tlb_guest,
484014fa
SC		 7831
7832 .run = vmx_vcpu_run,
7833 .handle_exit = vmx_handle_exit,
7834 .skip_emulated_instruction = vmx_skip_emulated_instruction,
7835 .update_emulated_instruction = vmx_update_emulated_instruction,
7836 .set_interrupt_shadow = vmx_set_interrupt_shadow,
7837 .get_interrupt_shadow = vmx_get_interrupt_shadow,
7838 .patch_hypercall = vmx_patch_hypercall,
7839 .set_irq = vmx_inject_irq,
7840 .set_nmi = vmx_inject_nmi,
7841 .queue_exception = vmx_queue_exception,
7842 .cancel_injection = vmx_cancel_injection,
7843 .interrupt_allowed = vmx_interrupt_allowed,
7844 .nmi_allowed = vmx_nmi_allowed,
7845 .get_nmi_mask = vmx_get_nmi_mask,
7846 .set_nmi_mask = vmx_set_nmi_mask,
7847 .enable_nmi_window = enable_nmi_window,
7848 .enable_irq_window = enable_irq_window,
7849 .update_cr8_intercept = update_cr8_intercept,
7850 .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
7851 .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
7852 .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
7853 .load_eoi_exitmap = vmx_load_eoi_exitmap,
7854 .apicv_post_state_restore = vmx_apicv_post_state_restore,
7855 .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons,
7856 .hwapic_irr_update = vmx_hwapic_irr_update,
7857 .hwapic_isr_update = vmx_hwapic_isr_update,
7858 .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
7859 .sync_pir_to_irr = vmx_sync_pir_to_irr,
7860 .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
7861 .dy_apicv_has_pending_interrupt = vmx_dy_apicv_has_pending_interrupt,
7862
7863 .set_tss_addr = vmx_set_tss_addr,
7864 .set_identity_map_addr = vmx_set_identity_map_addr,
7865 .get_tdp_level = get_ept_level,
7866 .get_mt_mask = vmx_get_mt_mask,
7867
7868 .get_exit_info = vmx_get_exit_info,
7869
7870 .cpuid_update = vmx_cpuid_update,
7871
7872 .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
7873
7874 .read_l1_tsc_offset = vmx_read_l1_tsc_offset,
7875 .write_l1_tsc_offset = vmx_write_l1_tsc_offset,
7876
7877 .load_mmu_pgd = vmx_load_mmu_pgd,
7878
7879 .check_intercept = vmx_check_intercept,
7880 .handle_exit_irqoff = vmx_handle_exit_irqoff,
7881
7882 .request_immediate_exit = vmx_request_immediate_exit,
7883
7884 .sched_in = vmx_sched_in,
7885
7886 .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
7887 .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
7888 .flush_log_dirty = vmx_flush_log_dirty,
7889 .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
7890 .write_log_dirty = vmx_write_pml_buffer,
7891
7892 .pre_block = vmx_pre_block,
7893 .post_block = vmx_post_block,
7894
7895 .pmu_ops = &intel_pmu_ops,
33b22172 7896 .nested_ops = &vmx_nested_ops,
484014fa
SC		 7897
7898 .update_pi_irte = vmx_update_pi_irte,
7899
7900#ifdef CONFIG_X86_64
7901 .set_hv_timer = vmx_set_hv_timer,
7902 .cancel_hv_timer = vmx_cancel_hv_timer,
7903#endif
7904
7905 .setup_mce = vmx_setup_mce,
7906
7907 .smi_allowed = vmx_smi_allowed,
7908 .pre_enter_smm = vmx_pre_enter_smm,
7909 .pre_leave_smm = vmx_pre_leave_smm,
7910 .enable_smi_window = enable_smi_window,
7911
484014fa
SC		 7912 .need_emulation_on_page_fault = vmx_need_emulation_on_page_fault,
7913 .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
7914};
7915
a3203381
SC		 7916static __init int hardware_setup(void)
7917{
7918 unsigned long host_bndcfgs;
2342080c 7919 struct desc_ptr dt;
703c335d 7920 int r, i, ept_lpage_level;
a3203381 7921
2342080c
SC		 7922 store_idt(&dt);
7923 host_idt_base = dt.address;
7924
a3203381
SC		 7925 for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
7926 kvm_define_shared_msr(i, vmx_msr_index[i]);
7927
7928 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
7929 return -EIO;
7930
7931 if (boot_cpu_has(X86_FEATURE_NX))
7932 kvm_enable_efer_bits(EFER_NX);
7933
7934 if (boot_cpu_has(X86_FEATURE_MPX)) {
7935 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs);
7936 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost");
7937 }
7938
7f5581f5 7939 if (!cpu_has_vmx_mpx())
cfc48181
SC		 7940 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
7941 XFEATURE_MASK_BNDCSR);
7942
a3203381
SC		 7943 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
7944 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
7945 enable_vpid = 0;
7946
7947 if (!cpu_has_vmx_ept() ||
7948 !cpu_has_vmx_ept_4levels() ||
7949 !cpu_has_vmx_ept_mt_wb() ||
7950 !cpu_has_vmx_invept_global())
7951 enable_ept = 0;
7952
7953 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
7954 enable_ept_ad_bits = 0;
7955
7956 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
7957 enable_unrestricted_guest = 0;
7958
7959 if (!cpu_has_vmx_flexpriority())
7960 flexpriority_enabled = 0;
7961
7962 if (!cpu_has_virtual_nmis())
7963 enable_vnmi = 0;
7964
7965 /*
7966 * set_apic_access_page_addr() is used to reload apic access
7967 * page upon invalidation. No need to do anything if not
7968 * using the APIC_ACCESS_ADDR VMCS field.
7969 */
7970 if (!flexpriority_enabled)
72b0eaa9 7971 vmx_x86_ops.set_apic_access_page_addr = NULL;
a3203381
SC		 7972
7973 if (!cpu_has_vmx_tpr_shadow())
72b0eaa9 7974 vmx_x86_ops.update_cr8_intercept = NULL;
a3203381
SC		 7975
7976#if IS_ENABLED(CONFIG_HYPERV)
7977 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
1f3a3e46 7978 && enable_ept) {
72b0eaa9
SC		 7979 vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb;
7980 vmx_x86_ops.tlb_remote_flush_with_range =
1f3a3e46
LT		 7981 hv_remote_flush_tlb_with_range;
7982 }
a3203381
SC		 7983#endif
7984
7985 if (!cpu_has_vmx_ple()) {
7986 ple_gap = 0;
7987 ple_window = 0;
7988 ple_window_grow = 0;
7989 ple_window_max = 0;
7990 ple_window_shrink = 0;
7991 }
7992
7993 if (!cpu_has_vmx_apicv()) {
7994 enable_apicv = 0;
72b0eaa9 7995 vmx_x86_ops.sync_pir_to_irr = NULL;
a3203381
SC		 7996 }
7997
7998 if (cpu_has_vmx_tsc_scaling()) {
7999 kvm_has_tsc_control = true;
8000 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
8001 kvm_tsc_scaling_ratio_frac_bits = 48;
8002 }
8003
8004 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
8005
8006 if (enable_ept)
8007 vmx_enable_tdp();
703c335d
SC		 8008
8009 if (!enable_ept)
8010 ept_lpage_level = 0;
8011 else if (cpu_has_vmx_ept_1g_page())
8012 ept_lpage_level = PT_PDPE_LEVEL;
8013 else if (cpu_has_vmx_ept_2m_page())
8014 ept_lpage_level = PT_DIRECTORY_LEVEL;
a3203381 8015 else
703c335d
SC		 8016 ept_lpage_level = PT_PAGE_TABLE_LEVEL;
8017 kvm_configure_mmu(enable_ept, ept_lpage_level);
a3203381 8018
a3203381
SC		 8019 /*
8020 * Only enable PML when hardware supports PML feature, and both EPT
8021 * and EPT A/D bit features are enabled -- PML depends on them to work.
8022 */
8023 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
8024 enable_pml = 0;
8025
8026 if (!enable_pml) {
72b0eaa9
SC		 8027 vmx_x86_ops.slot_enable_log_dirty = NULL;
8028 vmx_x86_ops.slot_disable_log_dirty = NULL;
8029 vmx_x86_ops.flush_log_dirty = NULL;
8030 vmx_x86_ops.enable_log_dirty_pt_masked = NULL;
a3203381
SC		 8031 }
8032
8033 if (!cpu_has_vmx_preemption_timer())
804939ea 8034 enable_preemption_timer = false;
a3203381 8035
804939ea
SC		 8036 if (enable_preemption_timer) {
8037 u64 use_timer_freq = 5000ULL * 1000 * 1000;
a3203381
SC		 8038 u64 vmx_msr;
8039
8040 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
8041 cpu_preemption_timer_multi =
8042 vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
804939ea
SC		 8043
8044 if (tsc_khz)
8045 use_timer_freq = (u64)tsc_khz * 1000;
8046 use_timer_freq >>= cpu_preemption_timer_multi;
8047
8048 /*
8049 * KVM "disables" the preemption timer by setting it to its max
8050 * value. Don't use the timer if it might cause spurious exits
8051 * at a rate faster than 0.1 Hz (of uninterrupted guest time).
8052 */
8053 if (use_timer_freq > 0xffffffffu / 10)
8054 enable_preemption_timer = false;
8055 }
8056
8057 if (!enable_preemption_timer) {
72b0eaa9
SC		 8058 vmx_x86_ops.set_hv_timer = NULL;
8059 vmx_x86_ops.cancel_hv_timer = NULL;
8060 vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
a3203381
SC		 8061 }
8062
a3203381 8063 kvm_set_posted_intr_wakeup_handler(wakeup_handler);
a3203381
SC		 8064
8065 kvm_mce_cap_supported |= MCG_LMCE_P;
8066
f99e3daf
CP		 8067 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST)
8068 return -EINVAL;
8069 if (!enable_ept || !cpu_has_vmx_intel_pt())
8070 pt_mode = PT_MODE_SYSTEM;
8071
a3203381 8072 if (nested) {
3e8eaccc 8073 nested_vmx_setup_ctls_msrs(&vmcs_config.nested,
a4443267 8074 vmx_capability.ept);
3e8eaccc 8075
72b0eaa9
SC		 8076 r = nested_vmx_hardware_setup(&vmx_x86_ops,
8077 kvm_vmx_exit_handlers);
a3203381
SC		 8078 if (r)
8079 return r;
8080 }
8081
3ec6fd8c 8082 vmx_set_cpu_caps();
66a6950f 8083
a3203381
SC		 8084 r = alloc_kvm_area();
8085 if (r)
8086 nested_vmx_hardware_unsetup();
8087 return r;
8088}
8089
d008dfdb 8090static struct kvm_x86_init_ops vmx_init_ops __initdata = {
6aa8b732
AK		 8091 .cpu_has_kvm_support = cpu_has_kvm_support,
8092 .disabled_by_bios = vmx_disabled_by_bios,
002c7f7c 8093 .check_processor_compatibility = vmx_check_processor_compat,
d008dfdb 8094 .hardware_setup = hardware_setup,
57b119da 8095
d008dfdb 8096 .runtime_ops = &vmx_x86_ops,
6aa8b732
AK		 8097};
8098
72c6d2db 8099static void vmx_cleanup_l1d_flush(void)
a47dd5f0
PB		 8100{
8101 if (vmx_l1d_flush_pages) {
8102 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
8103 vmx_l1d_flush_pages = NULL;
8104 }
72c6d2db
TG		 8105 /* Restore state so sysfs ignores VMX */
8106 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
a399477e
KRW		 8107}
8108
a7b9020b
TG		 8109static void vmx_exit(void)
8110{
8111#ifdef CONFIG_KEXEC_CORE
8112 RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
8113 synchronize_rcu();
8114#endif
8115
8116 kvm_exit();
8117
8118#if IS_ENABLED(CONFIG_HYPERV)
8119 if (static_branch_unlikely(&enable_evmcs)) {
8120 int cpu;
8121 struct hv_vp_assist_page *vp_ap;
8122 /*
8123 * Reset everything to support using non-enlightened VMCS
8124 * access later (e.g. when we reload the module with
8125 * enlightened_vmcs=0)
8126 */
8127 for_each_online_cpu(cpu) {
8128 vp_ap = hv_get_vp_assist_page(cpu);
8129
8130 if (!vp_ap)
8131 continue;
8132
6f6a657c 8133 vp_ap->nested_control.features.directhypercall = 0;
a7b9020b
TG		 8134 vp_ap->current_nested_vmcs = 0;
8135 vp_ap->enlighten_vmentry = 0;
8136 }
8137
8138 static_branch_disable(&enable_evmcs);
8139 }
8140#endif
8141 vmx_cleanup_l1d_flush();
8142}
8143module_exit(vmx_exit);
8144
6aa8b732
AK		 8145static int __init vmx_init(void)
8146{
dbef2808 8147 int r, cpu;
773e8a04
VK		 8148
8149#if IS_ENABLED(CONFIG_HYPERV)
8150 /*
8151 * Enlightened VMCS usage should be recommended and the host needs
8152 * to support eVMCS v1 or above. We can also disable eVMCS support
8153 * with module parameter.
8154 */
8155 if (enlightened_vmcs &&
8156 ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED &&
8157 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >=
8158 KVM_EVMCS_VERSION) {
8159 int cpu;
8160
8161 /* Check that we have assist pages on all online CPUs */
8162 for_each_online_cpu(cpu) {
8163 if (!hv_get_vp_assist_page(cpu)) {
8164 enlightened_vmcs = false;
8165 break;
8166 }
8167 }
8168
8169 if (enlightened_vmcs) {
8170 pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n");
8171 static_branch_enable(&enable_evmcs);
8172 }
6f6a657c
VK		 8173
8174 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH)
8175 vmx_x86_ops.enable_direct_tlbflush
8176 = hv_enable_direct_tlbflush;
8177
773e8a04
VK		 8178 } else {
8179 enlightened_vmcs = false;
8180 }
8181#endif
8182
d008dfdb 8183 r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx),
a7b9020b 8184 __alignof__(struct vcpu_vmx), THIS_MODULE);
fdef3ad1 8185 if (r)
34a1cd60 8186 return r;
25c5f225 8187
a7b9020b 8188 /*
7db92e16
TG		 8189 * Must be called after kvm_init() so enable_ept is properly set
8190 * up. Hand the parameter mitigation value in which was stored in
8191 * the pre module init parser. If no parameter was given, it will
8192 * contain 'auto' which will be turned into the default 'cond'
8193 * mitigation mode.
8194 */
19a36d32
WL		 8195 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
8196 if (r) {
8197 vmx_exit();
8198 return r;
a47dd5f0 8199 }
25c5f225 8200
dbef2808
VK		 8201 for_each_possible_cpu(cpu) {
8202 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
8203 INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
8204 spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
8205 }
8206
2965faa5 8207#ifdef CONFIG_KEXEC_CORE
8f536b76
ZY		 8208 rcu_assign_pointer(crash_vmclear_loaded_vmcss,
8209 crash_vmclear_local_loaded_vmcss);
8210#endif
21ebf53b 8211 vmx_check_vmcs12_offsets();
8f536b76 8212
fdef3ad1 8213 return 0;
6aa8b732 8214}
a7b9020b 8215module_init(vmx_init);
Ubuntu Hirsute Linux kernel mirror