1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __KVM_X86_VMX_H
3 #define __KVM_X86_VMX_H
5 #include <linux/kvm_host.h>
8 #include <asm/intel_pt.h>
10 #include "capabilities.h"
11 #include "kvm_cache_regs.h"
12 #include "posted_intr.h"
16 #include "run_flags.h"
22 #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
25 #define MAX_NR_USER_RETURN_MSRS 7
27 #define MAX_NR_USER_RETURN_MSRS 4
30 #define MAX_NR_LOADSTORE_MSRS 8
34 struct vmx_msr_entry val
[MAX_NR_LOADSTORE_MSRS
];
38 bool load_into_hardware
;
43 enum segment_cache_field
{
52 #define RTIT_ADDR_RANGE 4
60 u64 addr_a
[RTIT_ADDR_RANGE
];
61 u64 addr_b
[RTIT_ADDR_RANGE
];
67 u32 caps
[PT_CPUID_REGS_NUM
* PT_CPUID_LEAVES
];
72 union vmx_exit_reason
{
85 u32 bus_lock_detected
: 1;
87 u32 smi_pending_mtf
: 1;
88 u32 smi_from_vmx_root
: 1;
90 u32 failed_vmentry
: 1;
95 #define vcpu_to_lbr_desc(vcpu) (&to_vmx(vcpu)->lbr_desc)
96 #define vcpu_to_lbr_records(vcpu) (&to_vmx(vcpu)->lbr_desc.records)
98 bool intel_pmu_lbr_is_compatible(struct kvm_vcpu
*vcpu
);
99 bool intel_pmu_lbr_is_enabled(struct kvm_vcpu
*vcpu
);
101 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu
*vcpu
);
102 void vmx_passthrough_lbr_msrs(struct kvm_vcpu
*vcpu
);
105 /* Basic info about guest LBR records. */
106 struct x86_pmu_lbr records
;
109 * Emulate LBR feature via passthrough LBR registers when the
110 * per-vcpu guest LBR event is scheduled on the current pcpu.
112 * The records may be inaccurate if the host reclaims the LBR.
114 struct perf_event
*event
;
116 /* True if LBRs are marked as not intercepted in the MSR bitmap */
117 bool msr_passthrough
;
121 * The nested_vmx structure is part of vcpu_vmx, and holds information we need
122 * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
125 /* Has the level1 guest done vmxon? */
130 /* The guest-physical address of the current VMCS L1 keeps for L2 */
133 * Cache of the guest's VMCS, existing outside of guest memory.
134 * Loaded from guest memory during VMPTRLD. Flushed to guest
135 * memory during VMCLEAR and VMPTRLD.
137 struct vmcs12
*cached_vmcs12
;
139 * Cache of the guest's shadow VMCS, existing outside of guest
140 * memory. Loaded from guest memory during VM entry. Flushed
141 * to guest memory during VM exit.
143 struct vmcs12
*cached_shadow_vmcs12
;
146 * Indicates if the shadow vmcs or enlightened vmcs must be updated
147 * with the data held by struct vmcs12.
149 bool need_vmcs12_to_shadow_sync
;
153 * Indicates lazily loaded guest state has not yet been decached from
156 bool need_sync_vmcs02_to_vmcs12_rare
;
159 * vmcs02 has been initialized, i.e. state that is constant for
160 * vmcs02 has been written to the backing VMCS. Initialization
161 * is delayed until L1 actually attempts to run a nested VM.
163 bool vmcs02_initialized
;
165 bool change_vmcs01_virtual_apic_mode
;
166 bool reload_vmcs01_apic_access_page
;
167 bool update_vmcs01_cpu_dirty_logging
;
168 bool update_vmcs01_apicv_status
;
171 * Enlightened VMCS has been enabled. It does not mean that L1 has to
172 * use it. However, VMX features available to L1 will be limited based
173 * on what the enlightened VMCS supports.
175 bool enlightened_vmcs_enabled
;
177 /* L2 must run next, and mustn't decide to exit to L1. */
178 bool nested_run_pending
;
180 /* Pending MTF VM-exit into L1. */
183 struct loaded_vmcs vmcs02
;
186 * Guest pages referred to in the vmcs02 with host-physical
187 * pointers, so we must keep them pinned while L2 runs.
189 struct page
*apic_access_page
;
190 struct kvm_host_map virtual_apic_map
;
191 struct kvm_host_map pi_desc_map
;
193 struct kvm_host_map msr_bitmap_map
;
195 struct pi_desc
*pi_desc
;
199 struct hrtimer preemption_timer
;
200 u64 preemption_timer_deadline
;
201 bool has_preemption_timer_deadline
;
202 bool preemption_timer_expired
;
204 /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
206 u64 vmcs01_guest_bndcfgs
;
208 /* to migrate it to L1 if L2 writes to L1's CR8 directly */
209 int l1_tpr_threshold
;
214 struct nested_vmx_msrs msrs
;
216 /* SMM related state */
218 /* in VMX operation on SMM entry? */
220 /* in guest mode on SMM entry? */
224 gpa_t hv_evmcs_vmptr
;
225 struct kvm_host_map hv_evmcs_map
;
226 struct hv_enlightened_vmcs
*hv_evmcs
;
230 struct kvm_vcpu vcpu
;
232 u8 x2apic_msr_bitmap_mode
;
235 * If true, host state has been stored in vmx->loaded_vmcs for
236 * the CPU registers that only need to be switched when transitioning
237 * to/from the kernel, and the registers have been loaded with guest
238 * values. If false, host state is loaded in the CPU registers
239 * and vmx->loaded_vmcs->host_state is invalid.
241 bool guest_state_loaded
;
243 unsigned long exit_qualification
;
245 u32 idt_vectoring_info
;
249 * User return MSRs are always emulated when enabled in the guest, but
250 * only loaded into hardware when necessary, e.g. SYSCALL #UDs outside
251 * of 64-bit mode or if EFER.SCE=1, thus the SYSCALL MSRs don't need to
252 * be loaded into hardware if those conditions aren't met.
254 struct vmx_uret_msr guest_uret_msrs
[MAX_NR_USER_RETURN_MSRS
];
255 bool guest_uret_msrs_loaded
;
257 u64 msr_host_kernel_gs_base
;
258 u64 msr_guest_kernel_gs_base
;
262 u32 msr_ia32_umwait_control
;
265 * loaded_vmcs points to the VMCS currently used in this vcpu. For a
266 * non-nested (L1) guest, it always points to vmcs01. For a nested
267 * guest (L2), it points to a different VMCS.
269 struct loaded_vmcs vmcs01
;
270 struct loaded_vmcs
*loaded_vmcs
;
272 struct msr_autoload
{
273 struct vmx_msrs guest
;
274 struct vmx_msrs host
;
277 struct msr_autostore
{
278 struct vmx_msrs guest
;
284 struct kvm_segment segs
[8];
287 u32 bitmask
; /* 4 bits per segment (1 bit per field) */
288 struct kvm_save_segment
{
296 bool emulation_required
;
298 union vmx_exit_reason exit_reason
;
300 /* Posted interrupt descriptor */
301 struct pi_desc pi_desc
;
303 /* Support for a guest hypervisor (nested VMX) */
304 struct nested_vmx nested
;
306 /* Dynamic PLE window. */
307 unsigned int ple_window
;
308 bool ple_window_dirty
;
310 bool req_immediate_exit
;
312 /* Support for PML */
313 #define PML_ENTITY_NUM 512
316 /* apic deadline value in host tsc */
319 unsigned long host_debugctlmsr
;
322 * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
323 * msr_ia32_feature_control. FEAT_CTL_LOCKED is always included
324 * in msr_ia32_feature_control_valid_bits.
326 u64 msr_ia32_feature_control
;
327 u64 msr_ia32_feature_control_valid_bits
;
328 /* SGX Launch Control public key hash */
329 u64 msr_ia32_sgxlepubkeyhash
[4];
330 u64 msr_ia32_mcu_opt_ctrl
;
331 bool disable_fb_clear
;
333 struct pt_desc pt_desc
;
334 struct lbr_desc lbr_desc
;
336 /* Save desired MSR intercept (read: pass-through) state */
337 #define MAX_POSSIBLE_PASSTHROUGH_MSRS 13
339 DECLARE_BITMAP(read
, MAX_POSSIBLE_PASSTHROUGH_MSRS
);
340 DECLARE_BITMAP(write
, MAX_POSSIBLE_PASSTHROUGH_MSRS
);
341 } shadow_msr_intercept
;
347 unsigned int tss_addr
;
348 bool ept_identity_pagetable_done
;
349 gpa_t ept_identity_map_addr
;
352 bool nested_vmx_allowed(struct kvm_vcpu
*vcpu
);
353 void vmx_vcpu_load_vmcs(struct kvm_vcpu
*vcpu
, int cpu
,
354 struct loaded_vmcs
*buddy
);
355 int allocate_vpid(void);
356 void free_vpid(int vpid
);
357 void vmx_set_constant_host_state(struct vcpu_vmx
*vmx
);
358 void vmx_prepare_switch_to_guest(struct kvm_vcpu
*vcpu
);
359 void vmx_set_host_fs_gs(struct vmcs_host_state
*host
, u16 fs_sel
, u16 gs_sel
,
360 unsigned long fs_base
, unsigned long gs_base
);
361 int vmx_get_cpl(struct kvm_vcpu
*vcpu
);
362 bool vmx_emulation_required(struct kvm_vcpu
*vcpu
);
363 unsigned long vmx_get_rflags(struct kvm_vcpu
*vcpu
);
364 void vmx_set_rflags(struct kvm_vcpu
*vcpu
, unsigned long rflags
);
365 u32
vmx_get_interrupt_shadow(struct kvm_vcpu
*vcpu
);
366 void vmx_set_interrupt_shadow(struct kvm_vcpu
*vcpu
, int mask
);
367 int vmx_set_efer(struct kvm_vcpu
*vcpu
, u64 efer
);
368 void vmx_set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
);
369 void vmx_set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
);
370 void set_cr4_guest_host_mask(struct vcpu_vmx
*vmx
);
371 void ept_save_pdptrs(struct kvm_vcpu
*vcpu
);
372 void vmx_get_segment(struct kvm_vcpu
*vcpu
, struct kvm_segment
*var
, int seg
);
373 void __vmx_set_segment(struct kvm_vcpu
*vcpu
, struct kvm_segment
*var
, int seg
);
374 u64
construct_eptp(struct kvm_vcpu
*vcpu
, hpa_t root_hpa
, int root_level
);
376 bool vmx_guest_inject_ac(struct kvm_vcpu
*vcpu
);
377 void vmx_update_exception_bitmap(struct kvm_vcpu
*vcpu
);
378 bool vmx_nmi_blocked(struct kvm_vcpu
*vcpu
);
379 bool vmx_interrupt_blocked(struct kvm_vcpu
*vcpu
);
380 bool vmx_get_nmi_mask(struct kvm_vcpu
*vcpu
);
381 void vmx_set_nmi_mask(struct kvm_vcpu
*vcpu
, bool masked
);
382 void vmx_set_virtual_apic_mode(struct kvm_vcpu
*vcpu
);
383 struct vmx_uret_msr
*vmx_find_uret_msr(struct vcpu_vmx
*vmx
, u32 msr
);
384 void pt_update_intercept_for_msr(struct kvm_vcpu
*vcpu
);
385 void vmx_update_host_rsp(struct vcpu_vmx
*vmx
, unsigned long host_rsp
);
386 void vmx_spec_ctrl_restore_host(struct vcpu_vmx
*vmx
, unsigned int flags
);
387 unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx
*vmx
);
388 bool __vmx_vcpu_run(struct vcpu_vmx
*vmx
, unsigned long *regs
,
390 int vmx_find_loadstore_msr_slot(struct vmx_msrs
*m
, u32 msr
);
391 void vmx_ept_load_pdptrs(struct kvm_vcpu
*vcpu
);
393 void vmx_disable_intercept_for_msr(struct kvm_vcpu
*vcpu
, u32 msr
, int type
);
394 void vmx_enable_intercept_for_msr(struct kvm_vcpu
*vcpu
, u32 msr
, int type
);
396 u64
vmx_get_l2_tsc_offset(struct kvm_vcpu
*vcpu
);
397 u64
vmx_get_l2_tsc_multiplier(struct kvm_vcpu
*vcpu
);
399 static inline void vmx_set_intercept_for_msr(struct kvm_vcpu
*vcpu
, u32 msr
,
400 int type
, bool value
)
403 vmx_enable_intercept_for_msr(vcpu
, msr
, type
);
405 vmx_disable_intercept_for_msr(vcpu
, msr
, type
);
408 void vmx_update_cpu_dirty_logging(struct kvm_vcpu
*vcpu
);
410 static inline bool vmx_test_msr_bitmap_read(ulong
*msr_bitmap
, u32 msr
)
412 int f
= sizeof(unsigned long);
415 return test_bit(msr
, msr_bitmap
+ 0x000 / f
);
416 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
417 return test_bit(msr
& 0x1fff, msr_bitmap
+ 0x400 / f
);
421 static inline bool vmx_test_msr_bitmap_write(ulong
*msr_bitmap
, u32 msr
)
423 int f
= sizeof(unsigned long);
426 return test_bit(msr
, msr_bitmap
+ 0x800 / f
);
427 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
428 return test_bit(msr
& 0x1fff, msr_bitmap
+ 0xc00 / f
);
432 static inline void vmx_clear_msr_bitmap_read(ulong
*msr_bitmap
, u32 msr
)
434 int f
= sizeof(unsigned long);
437 __clear_bit(msr
, msr_bitmap
+ 0x000 / f
);
438 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
439 __clear_bit(msr
& 0x1fff, msr_bitmap
+ 0x400 / f
);
442 static inline void vmx_clear_msr_bitmap_write(ulong
*msr_bitmap
, u32 msr
)
444 int f
= sizeof(unsigned long);
447 __clear_bit(msr
, msr_bitmap
+ 0x800 / f
);
448 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
449 __clear_bit(msr
& 0x1fff, msr_bitmap
+ 0xc00 / f
);
452 static inline void vmx_set_msr_bitmap_read(ulong
*msr_bitmap
, u32 msr
)
454 int f
= sizeof(unsigned long);
457 __set_bit(msr
, msr_bitmap
+ 0x000 / f
);
458 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
459 __set_bit(msr
& 0x1fff, msr_bitmap
+ 0x400 / f
);
462 static inline void vmx_set_msr_bitmap_write(ulong
*msr_bitmap
, u32 msr
)
464 int f
= sizeof(unsigned long);
467 __set_bit(msr
, msr_bitmap
+ 0x800 / f
);
468 else if ((msr
>= 0xc0000000) && (msr
<= 0xc0001fff))
469 __set_bit(msr
& 0x1fff, msr_bitmap
+ 0xc00 / f
);
473 static inline u8
vmx_get_rvi(void)
475 return vmcs_read16(GUEST_INTR_STATUS
) & 0xff;
478 #define BUILD_CONTROLS_SHADOW(lname, uname) \
479 static inline void lname##_controls_set(struct vcpu_vmx *vmx, u32 val) \
481 if (vmx->loaded_vmcs->controls_shadow.lname != val) { \
482 vmcs_write32(uname, val); \
483 vmx->loaded_vmcs->controls_shadow.lname = val; \
486 static inline u32 __##lname##_controls_get(struct loaded_vmcs *vmcs) \
488 return vmcs->controls_shadow.lname; \
490 static inline u32 lname##_controls_get(struct vcpu_vmx *vmx) \
492 return __##lname##_controls_get(vmx->loaded_vmcs); \
494 static inline void lname##_controls_setbit(struct vcpu_vmx *vmx, u32 val) \
496 lname##_controls_set(vmx, lname##_controls_get(vmx) | val); \
498 static inline void lname##_controls_clearbit(struct vcpu_vmx *vmx, u32 val) \
500 lname##_controls_set(vmx, lname##_controls_get(vmx) & ~val); \
502 BUILD_CONTROLS_SHADOW(vm_entry
, VM_ENTRY_CONTROLS
)
503 BUILD_CONTROLS_SHADOW(vm_exit
, VM_EXIT_CONTROLS
)
504 BUILD_CONTROLS_SHADOW(pin
, PIN_BASED_VM_EXEC_CONTROL
)
505 BUILD_CONTROLS_SHADOW(exec
, CPU_BASED_VM_EXEC_CONTROL
)
506 BUILD_CONTROLS_SHADOW(secondary_exec
, SECONDARY_VM_EXEC_CONTROL
)
508 static inline void vmx_register_cache_reset(struct kvm_vcpu
*vcpu
)
510 vcpu
->arch
.regs_avail
= ~((1 << VCPU_REGS_RIP
) | (1 << VCPU_REGS_RSP
)
511 | (1 << VCPU_EXREG_RFLAGS
)
512 | (1 << VCPU_EXREG_PDPTR
)
513 | (1 << VCPU_EXREG_SEGMENTS
)
514 | (1 << VCPU_EXREG_CR0
)
515 | (1 << VCPU_EXREG_CR3
)
516 | (1 << VCPU_EXREG_CR4
)
517 | (1 << VCPU_EXREG_EXIT_INFO_1
)
518 | (1 << VCPU_EXREG_EXIT_INFO_2
));
519 vcpu
->arch
.regs_dirty
= 0;
522 static inline struct kvm_vmx
*to_kvm_vmx(struct kvm
*kvm
)
524 return container_of(kvm
, struct kvm_vmx
, kvm
);
527 static inline struct vcpu_vmx
*to_vmx(struct kvm_vcpu
*vcpu
)
529 return container_of(vcpu
, struct vcpu_vmx
, vcpu
);
532 static inline unsigned long vmx_get_exit_qual(struct kvm_vcpu
*vcpu
)
534 struct vcpu_vmx
*vmx
= to_vmx(vcpu
);
536 if (!kvm_register_is_available(vcpu
, VCPU_EXREG_EXIT_INFO_1
)) {
537 kvm_register_mark_available(vcpu
, VCPU_EXREG_EXIT_INFO_1
);
538 vmx
->exit_qualification
= vmcs_readl(EXIT_QUALIFICATION
);
540 return vmx
->exit_qualification
;
543 static inline u32
vmx_get_intr_info(struct kvm_vcpu
*vcpu
)
545 struct vcpu_vmx
*vmx
= to_vmx(vcpu
);
547 if (!kvm_register_is_available(vcpu
, VCPU_EXREG_EXIT_INFO_2
)) {
548 kvm_register_mark_available(vcpu
, VCPU_EXREG_EXIT_INFO_2
);
549 vmx
->exit_intr_info
= vmcs_read32(VM_EXIT_INTR_INFO
);
551 return vmx
->exit_intr_info
;
554 struct vmcs
*alloc_vmcs_cpu(bool shadow
, int cpu
, gfp_t flags
);
555 void free_vmcs(struct vmcs
*vmcs
);
556 int alloc_loaded_vmcs(struct loaded_vmcs
*loaded_vmcs
);
557 void free_loaded_vmcs(struct loaded_vmcs
*loaded_vmcs
);
558 void loaded_vmcs_clear(struct loaded_vmcs
*loaded_vmcs
);
560 static inline struct vmcs
*alloc_vmcs(bool shadow
)
562 return alloc_vmcs_cpu(shadow
, raw_smp_processor_id(),
566 static inline bool vmx_has_waitpkg(struct vcpu_vmx
*vmx
)
568 return secondary_exec_controls_get(vmx
) &
569 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE
;
572 static inline bool vmx_need_pf_intercept(struct kvm_vcpu
*vcpu
)
577 return allow_smaller_maxphyaddr
&& cpuid_maxphyaddr(vcpu
) < boot_cpu_data
.x86_phys_bits
;
580 static inline bool is_unrestricted_guest(struct kvm_vcpu
*vcpu
)
582 return enable_unrestricted_guest
&& (!is_guest_mode(vcpu
) ||
583 (secondary_exec_controls_get(to_vmx(vcpu
)) &
584 SECONDARY_EXEC_UNRESTRICTED_GUEST
));
587 bool __vmx_guest_state_valid(struct kvm_vcpu
*vcpu
);
588 static inline bool vmx_guest_state_valid(struct kvm_vcpu
*vcpu
)
590 return is_unrestricted_guest(vcpu
) || __vmx_guest_state_valid(vcpu
);
593 void dump_vmcs(struct kvm_vcpu
*vcpu
);
595 #endif /* __KVM_X86_VMX_H */