1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Kernel-based Virtual Machine driver for Linux
5 * This header defines architecture specific interfaces, x86 version
8 #ifndef _ASM_X86_KVM_HOST_H
9 #define _ASM_X86_KVM_HOST_H
11 #include <linux/types.h>
13 #include <linux/mmu_notifier.h>
14 #include <linux/tracepoint.h>
15 #include <linux/cpumask.h>
16 #include <linux/irq_work.h>
17 #include <linux/irq.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_para.h>
21 #include <linux/kvm_types.h>
22 #include <linux/perf_event.h>
23 #include <linux/pvclock_gtod.h>
24 #include <linux/clocksource.h>
25 #include <linux/irqbypass.h>
26 #include <linux/hyperv.h>
29 #include <asm/pvclock-abi.h>
32 #include <asm/msr-index.h>
34 #include <asm/kvm_page_track.h>
35 #include <asm/kvm_vcpu_regs.h>
36 #include <asm/hyperv-tlfs.h>
38 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS
40 #define KVM_MAX_VCPUS 288
41 #define KVM_SOFT_MAX_VCPUS 240
42 #define KVM_MAX_VCPU_ID 1023
43 /* memory slots that are not exposed to userspace */
44 #define KVM_PRIVATE_MEM_SLOTS 3
46 #define KVM_HALT_POLL_NS_DEFAULT 200000
48 #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
50 #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
51 KVM_DIRTY_LOG_INITIALLY_SET)
53 #define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF | \
54 KVM_BUS_LOCK_DETECTION_EXIT)
56 /* x86-specific vcpu->requests bit members */
57 #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
58 #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
59 #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
60 #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
61 #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
62 #define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5)
63 #define KVM_REQ_EVENT KVM_ARCH_REQ(6)
64 #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
65 #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
66 #define KVM_REQ_NMI KVM_ARCH_REQ(9)
67 #define KVM_REQ_PMU KVM_ARCH_REQ(10)
68 #define KVM_REQ_PMI KVM_ARCH_REQ(11)
69 #define KVM_REQ_SMI KVM_ARCH_REQ(12)
70 #define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
71 #define KVM_REQ_MCLOCK_INPROGRESS \
72 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
73 #define KVM_REQ_SCAN_IOAPIC \
74 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
75 #define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
76 #define KVM_REQ_APIC_PAGE_RELOAD \
77 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
78 #define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
79 #define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
80 #define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
81 #define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
82 #define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
83 #define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23)
84 #define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24)
85 #define KVM_REQ_APICV_UPDATE \
86 KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
87 #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26)
88 #define KVM_REQ_TLB_FLUSH_GUEST \
89 KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP)
90 #define KVM_REQ_APF_READY KVM_ARCH_REQ(28)
91 #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29)
92 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
93 KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
95 #define CR0_RESERVED_BITS \
96 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
97 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
98 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
100 #define CR4_RESERVED_BITS \
101 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
102 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
103 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
104 | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
105 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
106 | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
108 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
112 #define INVALID_PAGE (~(hpa_t)0)
113 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
115 #define UNMAPPED_GVA (~(gpa_t)0)
116 #define INVALID_GPA (~(gpa_t)0)
118 /* KVM Hugepage definitions for x86 */
119 #define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G
120 #define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1)
121 #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
122 #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
123 #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
124 #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
125 #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
127 static inline gfn_t
gfn_to_index(gfn_t gfn
, gfn_t base_gfn
, int level
)
129 /* KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K) must be 0. */
130 return (gfn
>> KVM_HPAGE_GFN_SHIFT(level
)) -
131 (base_gfn
>> KVM_HPAGE_GFN_SHIFT(level
));
134 #define KVM_PERMILLE_MMU_PAGES 20
135 #define KVM_MIN_ALLOC_MMU_PAGES 64UL
136 #define KVM_MMU_HASH_SHIFT 12
137 #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
138 #define KVM_MIN_FREE_MMU_PAGES 5
139 #define KVM_REFILL_PAGES 25
140 #define KVM_MAX_CPUID_ENTRIES 256
141 #define KVM_NR_FIXED_MTRR_REGION 88
142 #define KVM_NR_VAR_MTRR 8
144 #define ASYNC_PF_PER_VCPU 64
147 VCPU_REGS_RAX
= __VCPU_REGS_RAX
,
148 VCPU_REGS_RCX
= __VCPU_REGS_RCX
,
149 VCPU_REGS_RDX
= __VCPU_REGS_RDX
,
150 VCPU_REGS_RBX
= __VCPU_REGS_RBX
,
151 VCPU_REGS_RSP
= __VCPU_REGS_RSP
,
152 VCPU_REGS_RBP
= __VCPU_REGS_RBP
,
153 VCPU_REGS_RSI
= __VCPU_REGS_RSI
,
154 VCPU_REGS_RDI
= __VCPU_REGS_RDI
,
156 VCPU_REGS_R8
= __VCPU_REGS_R8
,
157 VCPU_REGS_R9
= __VCPU_REGS_R9
,
158 VCPU_REGS_R10
= __VCPU_REGS_R10
,
159 VCPU_REGS_R11
= __VCPU_REGS_R11
,
160 VCPU_REGS_R12
= __VCPU_REGS_R12
,
161 VCPU_REGS_R13
= __VCPU_REGS_R13
,
162 VCPU_REGS_R14
= __VCPU_REGS_R14
,
163 VCPU_REGS_R15
= __VCPU_REGS_R15
,
168 VCPU_EXREG_PDPTR
= NR_VCPU_REGS
,
174 VCPU_EXREG_EXIT_INFO_1
,
175 VCPU_EXREG_EXIT_INFO_2
,
189 enum exit_fastpath_completion
{
191 EXIT_FASTPATH_REENTER_GUEST
,
192 EXIT_FASTPATH_EXIT_HANDLED
,
194 typedef enum exit_fastpath_completion fastpath_t
;
196 struct x86_emulate_ctxt
;
197 struct x86_exception
;
199 enum x86_intercept_stage
;
201 #define KVM_NR_DB_REGS 4
203 #define DR6_BUS_LOCK (1 << 11)
204 #define DR6_BD (1 << 13)
205 #define DR6_BS (1 << 14)
206 #define DR6_BT (1 << 15)
207 #define DR6_RTM (1 << 16)
209 * DR6_ACTIVE_LOW combines fixed-1 and active-low bits.
210 * We can regard all the bits in DR6_FIXED_1 as active_low bits;
211 * they will never be 0 for now, but when they are defined
212 * in the future it will require no code change.
214 * DR6_ACTIVE_LOW is also used as the init/reset value for DR6.
216 #define DR6_ACTIVE_LOW 0xffff0ff0
217 #define DR6_VOLATILE 0x0001e80f
218 #define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE)
220 #define DR7_BP_EN_MASK 0x000000ff
221 #define DR7_GE (1 << 9)
222 #define DR7_GD (1 << 13)
223 #define DR7_FIXED_1 0x00000400
224 #define DR7_VOLATILE 0xffff2bff
226 #define KVM_GUESTDBG_VALID_MASK \
227 (KVM_GUESTDBG_ENABLE | \
228 KVM_GUESTDBG_SINGLESTEP | \
229 KVM_GUESTDBG_USE_HW_BP | \
230 KVM_GUESTDBG_USE_SW_BP | \
231 KVM_GUESTDBG_INJECT_BP | \
232 KVM_GUESTDBG_INJECT_DB)
235 #define PFERR_PRESENT_BIT 0
236 #define PFERR_WRITE_BIT 1
237 #define PFERR_USER_BIT 2
238 #define PFERR_RSVD_BIT 3
239 #define PFERR_FETCH_BIT 4
240 #define PFERR_PK_BIT 5
241 #define PFERR_SGX_BIT 15
242 #define PFERR_GUEST_FINAL_BIT 32
243 #define PFERR_GUEST_PAGE_BIT 33
245 #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
246 #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
247 #define PFERR_USER_MASK (1U << PFERR_USER_BIT)
248 #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
249 #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
250 #define PFERR_PK_MASK (1U << PFERR_PK_BIT)
251 #define PFERR_SGX_MASK (1U << PFERR_SGX_BIT)
252 #define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
253 #define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
255 #define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \
259 /* apic attention bits */
260 #define KVM_APIC_CHECK_VAPIC 0
262 * The following bit is set with PV-EOI, unset on EOI.
263 * We detect PV-EOI changes by guest by comparing
264 * this bit with PV-EOI in guest memory.
265 * See the implementation in apic_update_pv_eoi.
267 #define KVM_APIC_PV_EOI_PENDING 1
269 struct kvm_kernel_irq_routing_entry
;
272 * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page
273 * also includes TDP pages) to determine whether or not a page can be used in
274 * the given MMU context. This is a subset of the overall kvm_mmu_role to
275 * minimize the size of kvm_memory_slot.arch.gfn_track, i.e. allows allocating
276 * 2 bytes per gfn instead of 4 bytes per gfn.
278 * Indirect upper-level shadow pages are tracked for write-protection via
279 * gfn_track. As above, gfn_track is a 16 bit counter, so KVM must not create
280 * more than 2^16-1 upper-level shadow pages at a single gfn, otherwise
281 * gfn_track will overflow and explosions will ensure.
283 * A unique shadow page (SP) for a gfn is created if and only if an existing SP
284 * cannot be reused. The ability to reuse a SP is tracked by its role, which
285 * incorporates various mode bits and properties of the SP. Roughly speaking,
286 * the number of unique SPs that can theoretically be created is 2^n, where n
287 * is the number of bits that are used to compute the role.
289 * But, even though there are 18 bits in the mask below, not all combinations
290 * of modes and flags are possible. The maximum number of possible upper-level
291 * shadow pages for a single gfn is in the neighborhood of 2^13.
293 * - invalid shadow pages are not accounted.
294 * - level is effectively limited to four combinations, not 16 as the number
295 * bits would imply, as 4k SPs are not tracked (allowed to go unsync).
296 * - level is effectively unused for non-PAE paging because there is exactly
297 * one upper level (see 4k SP exception above).
298 * - quadrant is used only for non-PAE paging and is exclusive with
300 * - execonly and ad_disabled are used only for nested EPT, which makes it
301 * exclusive with quadrant.
303 union kvm_mmu_page_role
{
307 unsigned gpte_is_8_bytes
:1;
314 unsigned smep_andnot_wp
:1;
315 unsigned smap_andnot_wp
:1;
316 unsigned ad_disabled
:1;
317 unsigned guest_mode
:1;
321 * This is left at the top of the word so that
322 * kvm_memslots_for_spte_role can extract it with a
323 * simple shift. While there is room, give it a whole
324 * byte so it is also faster to load it from memory.
331 * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties
332 * relevant to the current MMU configuration. When loading CR0, CR4, or EFER,
333 * including on nested transitions, if nothing in the full role changes then
334 * MMU re-configuration can be skipped. @valid bit is set on first usage so we
335 * don't treat all-zero structure as valid data.
337 * The properties that are tracked in the extended role but not the page role
338 * are for things that either (a) do not affect the validity of the shadow page
339 * or (b) are indirectly reflected in the shadow page's role. For example,
340 * CR4.PKE only affects permission checks for software walks of the guest page
341 * tables (because KVM doesn't support Protection Keys with shadow paging), and
342 * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level.
344 * Note, SMEP and SMAP are not redundant with sm*p_andnot_wp in the page role.
345 * If CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMEP and
346 * SMAP, but the MMU's permission checks for software walks need to be SMEP and
347 * SMAP aware regardless of CR0.WP.
349 union kvm_mmu_extended_role
{
352 unsigned int valid
:1;
353 unsigned int execonly
:1;
354 unsigned int cr0_pg
:1;
355 unsigned int cr4_pae
:1;
356 unsigned int cr4_pse
:1;
357 unsigned int cr4_pke
:1;
358 unsigned int cr4_smap
:1;
359 unsigned int cr4_smep
:1;
360 unsigned int cr4_la57
:1;
367 union kvm_mmu_page_role base
;
368 union kvm_mmu_extended_role ext
;
372 struct kvm_rmap_head
{
376 struct kvm_pio_request
{
377 unsigned long linear_rip
;
384 #define PT64_ROOT_MAX_LEVEL 5
386 struct rsvd_bits_validate
{
387 u64 rsvd_bits_mask
[2][PT64_ROOT_MAX_LEVEL
];
391 struct kvm_mmu_root_info
{
396 #define KVM_MMU_ROOT_INFO_INVALID \
397 ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE })
399 #define KVM_MMU_NUM_PREV_ROOTS 3
401 #define KVM_HAVE_MMU_RWLOCK
406 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
407 * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
411 unsigned long (*get_guest_pgd
)(struct kvm_vcpu
*vcpu
);
412 u64 (*get_pdptr
)(struct kvm_vcpu
*vcpu
, int index
);
413 int (*page_fault
)(struct kvm_vcpu
*vcpu
, gpa_t cr2_or_gpa
, u32 err
,
415 void (*inject_page_fault
)(struct kvm_vcpu
*vcpu
,
416 struct x86_exception
*fault
);
417 gpa_t (*gva_to_gpa
)(struct kvm_vcpu
*vcpu
, gpa_t gva_or_gpa
,
418 u32 access
, struct x86_exception
*exception
);
419 gpa_t (*translate_gpa
)(struct kvm_vcpu
*vcpu
, gpa_t gpa
, u32 access
,
420 struct x86_exception
*exception
);
421 int (*sync_page
)(struct kvm_vcpu
*vcpu
,
422 struct kvm_mmu_page
*sp
);
423 void (*invlpg
)(struct kvm_vcpu
*vcpu
, gva_t gva
, hpa_t root_hpa
);
426 union kvm_mmu_role mmu_role
;
428 u8 shadow_root_level
;
431 struct kvm_mmu_root_info prev_roots
[KVM_MMU_NUM_PREV_ROOTS
];
434 * Bitmap; bit set = permission fault
435 * Byte index: page fault error code [4:1]
436 * Bit index: pte permissions in ACC_* format
441 * The pkru_mask indicates if protection key checks are needed. It
442 * consists of 16 domains indexed by page fault error code bits [4:1],
443 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
444 * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
452 * check zero bits on shadow page table entries, these
453 * bits include not only hardware reserved bits but also
454 * the bits spte never used.
456 struct rsvd_bits_validate shadow_zero_check
;
458 struct rsvd_bits_validate guest_rsvd_check
;
460 u64 pdptrs
[4]; /* pae */
463 struct kvm_tlb_range
{
478 struct perf_event
*perf_event
;
479 struct kvm_vcpu
*vcpu
;
481 * eventsel value for general purpose counters,
482 * ctrl value for fixed counters.
488 unsigned nr_arch_gp_counters
;
489 unsigned nr_arch_fixed_counters
;
490 unsigned available_event_types
;
495 u64 counter_bitmask
[2];
496 u64 global_ctrl_mask
;
497 u64 global_ovf_ctrl_mask
;
500 struct kvm_pmc gp_counters
[INTEL_PMC_MAX_GENERIC
];
501 struct kvm_pmc fixed_counters
[INTEL_PMC_MAX_FIXED
];
502 struct irq_work irq_work
;
503 DECLARE_BITMAP(reprogram_pmi
, X86_PMC_IDX_MAX
);
504 DECLARE_BITMAP(all_valid_pmc_idx
, X86_PMC_IDX_MAX
);
505 DECLARE_BITMAP(pmc_in_use
, X86_PMC_IDX_MAX
);
508 * The gate to release perf_events not marked in
509 * pmc_in_use only once in a vcpu time slice.
514 * The total number of programmed perf_events and it helps to avoid
515 * redundant check before cleanup if guest don't use vPMU at all.
523 KVM_DEBUGREG_BP_ENABLED
= 1,
524 KVM_DEBUGREG_WONT_EXIT
= 2,
525 KVM_DEBUGREG_RELOAD
= 4,
528 struct kvm_mtrr_range
{
531 struct list_head node
;
535 struct kvm_mtrr_range var_ranges
[KVM_NR_VAR_MTRR
];
536 mtrr_type fixed_ranges
[KVM_NR_FIXED_MTRR_REGION
];
539 struct list_head head
;
542 /* Hyper-V SynIC timer */
543 struct kvm_vcpu_hv_stimer
{
544 struct hrtimer timer
;
546 union hv_stimer_config config
;
549 struct hv_message msg
;
553 /* Hyper-V synthetic interrupt controller (SynIC)*/
554 struct kvm_vcpu_hv_synic
{
559 atomic64_t sint
[HV_SYNIC_SINT_COUNT
];
560 atomic_t sint_to_gsi
[HV_SYNIC_SINT_COUNT
];
561 DECLARE_BITMAP(auto_eoi_bitmap
, 256);
562 DECLARE_BITMAP(vec_bitmap
, 256);
564 bool dont_zero_synic_pages
;
567 /* Hyper-V per vcpu emulation context */
569 struct kvm_vcpu
*vcpu
;
573 struct kvm_vcpu_hv_synic synic
;
574 struct kvm_hyperv_exit exit
;
575 struct kvm_vcpu_hv_stimer stimer
[HV_SYNIC_STIMER_COUNT
];
576 DECLARE_BITMAP(stimer_pending_bitmap
, HV_SYNIC_STIMER_COUNT
);
580 u32 features_eax
; /* HYPERV_CPUID_FEATURES.EAX */
581 u32 features_ebx
; /* HYPERV_CPUID_FEATURES.EBX */
582 u32 features_edx
; /* HYPERV_CPUID_FEATURES.EDX */
583 u32 enlightenments_eax
; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */
584 u32 enlightenments_ebx
; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */
585 u32 syndbg_cap_eax
; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */
589 /* Xen HVM per vcpu emulation context */
590 struct kvm_vcpu_xen
{
592 u32 current_runstate
;
594 bool vcpu_time_info_set
;
596 struct gfn_to_hva_cache vcpu_info_cache
;
597 struct gfn_to_hva_cache vcpu_time_info_cache
;
598 struct gfn_to_hva_cache runstate_cache
;
600 u64 runstate_entry_time
;
601 u64 runstate_times
[4];
604 struct kvm_vcpu_arch
{
606 * rip and regs accesses must go through
607 * kvm_{register,rip}_{read,write} functions.
609 unsigned long regs
[NR_VCPU_REGS
];
614 unsigned long cr0_guest_owned_bits
;
618 unsigned long cr4_guest_owned_bits
;
619 unsigned long cr4_guest_rsvd_bits
;
626 struct kvm_lapic
*apic
; /* kernel irqchip context */
628 bool load_eoi_exitmap_pending
;
629 DECLARE_BITMAP(ioapic_handled_vectors
, 256);
630 unsigned long apic_attention
;
631 int32_t apic_arb_prio
;
633 u64 ia32_misc_enable_msr
;
636 bool tpr_access_reporting
;
639 u64 microcode_version
;
640 u64 arch_capabilities
;
641 u64 perf_capabilities
;
644 * Paging state of the vcpu
646 * If the vcpu runs in guest mode with two level paging this still saves
647 * the paging mode of the l1 guest. This context is always used to
652 /* Non-nested MMU for L1 */
653 struct kvm_mmu root_mmu
;
655 /* L1 MMU when running nested */
656 struct kvm_mmu guest_mmu
;
659 * Paging state of an L2 guest (used for nested npt)
661 * This context will save all necessary information to walk page tables
662 * of an L2 guest. This context is only initialized for page table
663 * walking and not for faulting since we never handle l2 page faults on
666 struct kvm_mmu nested_mmu
;
669 * Pointer to the mmu context currently used for
670 * gva_to_gpa translations.
672 struct kvm_mmu
*walk_mmu
;
674 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache
;
675 struct kvm_mmu_memory_cache mmu_shadow_page_cache
;
676 struct kvm_mmu_memory_cache mmu_gfn_array_cache
;
677 struct kvm_mmu_memory_cache mmu_page_header_cache
;
680 * QEMU userspace and the guest each have their own FPU state.
681 * In vcpu_run, we switch between the user and guest FPU contexts.
682 * While running a VCPU, the VCPU thread will have the guest FPU
685 * Note that while the PKRU state lives inside the fpu registers,
686 * it is switched out separately at VMENTER and VMEXIT time. The
687 * "guest_fpu" state here contains the guest FPU context, with the
690 struct fpu
*user_fpu
;
691 struct fpu
*guest_fpu
;
694 u64 guest_supported_xcr0
;
696 struct kvm_pio_request pio
;
698 void *guest_ins_data
;
700 u8 event_exit_inst_len
;
702 struct kvm_queued_exception
{
708 unsigned long payload
;
713 struct kvm_queued_interrupt
{
719 int halt_request
; /* real mode on Intel only */
722 struct kvm_cpuid_entry2
*cpuid_entries
;
724 u64 reserved_gpa_bits
;
728 /* emulate context */
730 struct x86_emulate_ctxt
*emulate_ctxt
;
731 bool emulate_regs_need_sync_to_vcpu
;
732 bool emulate_regs_need_sync_from_vcpu
;
733 int (*complete_userspace_io
)(struct kvm_vcpu
*vcpu
);
736 struct pvclock_vcpu_time_info hv_clock
;
737 unsigned int hw_tsc_khz
;
738 struct gfn_to_hva_cache pv_time
;
739 bool pv_time_enabled
;
740 /* set guest stopped flag in pvclock flags field */
741 bool pvclock_set_guest_stopped_request
;
747 struct gfn_to_pfn_cache cache
;
751 u64 tsc_offset
; /* current tsc offset */
754 u64 tsc_offset_adjustment
;
757 u64 this_tsc_generation
;
759 bool tsc_always_catchup
;
760 s8 virtual_tsc_shift
;
761 u32 virtual_tsc_mult
;
763 s64 ia32_tsc_adjust_msr
;
764 u64 msr_ia32_power_ctl
;
765 u64 l1_tsc_scaling_ratio
;
766 u64 tsc_scaling_ratio
; /* current scaling ratio */
768 atomic_t nmi_queued
; /* unprocessed asynchronous NMIs */
769 unsigned nmi_pending
; /* NMI queued after currently running handler */
770 bool nmi_injected
; /* Trying to inject an NMI this entry */
771 bool smi_pending
; /* SMI queued after currently running handler */
773 struct kvm_mtrr mtrr_state
;
776 unsigned switch_db_regs
;
777 unsigned long db
[KVM_NR_DB_REGS
];
780 unsigned long eff_db
[KVM_NR_DB_REGS
];
781 unsigned long guest_debug_dr7
;
782 u64 msr_platform_info
;
783 u64 msr_misc_features_enables
;
791 /* Cache MMIO info */
793 unsigned mmio_access
;
799 /* used for guest single stepping over the given code position */
800 unsigned long singlestep_rip
;
803 struct kvm_vcpu_hv
*hyperv
;
804 struct kvm_vcpu_xen xen
;
806 cpumask_var_t wbinvd_dirty_mask
;
808 unsigned long last_retry_eip
;
809 unsigned long last_retry_addr
;
813 gfn_t gfns
[ASYNC_PF_PER_VCPU
];
814 struct gfn_to_hva_cache data
;
815 u64 msr_en_val
; /* MSR_KVM_ASYNC_PF_EN */
816 u64 msr_int_val
; /* MSR_KVM_ASYNC_PF_INT */
821 unsigned long nested_apf_token
;
822 bool delivery_as_pf_vmexit
;
823 bool pageready_pending
;
826 /* OSVW MSRs (AMD only) */
834 struct gfn_to_hva_cache data
;
837 u64 msr_kvm_poll_control
;
840 * Indicates the guest is trying to write a gfn that contains one or
841 * more of the PTEs used to translate the write itself, i.e. the access
842 * is changing its own translation in the guest page tables. KVM exits
843 * to userspace if emulation of the faulting instruction fails and this
844 * flag is set, as KVM cannot make forward progress.
846 * If emulation fails for a write to guest page tables, KVM unprotects
847 * (zaps) the shadow page for the target gfn and resumes the guest to
848 * retry the non-emulatable instruction (on hardware). Unprotecting the
849 * gfn doesn't allow forward progress for a self-changing access because
850 * doing so also zaps the translation for the gfn, i.e. retrying the
851 * instruction will hit a !PRESENT fault, which results in a new shadow
852 * page and sends KVM back to square one.
854 bool write_fault_to_shadow_pgtable
;
856 /* set at EPT violation at this point */
857 unsigned long exit_qualification
;
859 /* pv related host specific info */
864 int pending_ioapic_eoi
;
865 int pending_external_vector
;
867 /* be preempted when it's in kernel-mode(cpl=0) */
868 bool preempted_in_kernel
;
870 /* Flush the L1 Data cache for L1TF mitigation on VMENTER */
873 /* Host CPU on which VM-entry was most recently attempted */
874 int last_vmentry_cpu
;
876 /* AMD MSRC001_0015 Hardware Configuration */
879 /* pv related cpuid info */
882 * value of the eax register in the KVM_CPUID_FEATURES CPUID
888 * indicates whether pv emulation should be disabled if features
889 * are not present in the guest's cpuid
894 /* Protected Guests */
895 bool guest_state_protected
;
898 * Set when PDPTS were loaded directly by the userspace without
899 * reading the guest memory
901 bool pdptrs_from_userspace
;
903 #if IS_ENABLED(CONFIG_HYPERV)
908 struct kvm_lpage_info
{
912 struct kvm_arch_memory_slot
{
913 struct kvm_rmap_head
*rmap
[KVM_NR_PAGE_SIZES
];
914 struct kvm_lpage_info
*lpage_info
[KVM_NR_PAGE_SIZES
- 1];
915 unsigned short *gfn_track
[KVM_PAGE_TRACK_MAX
];
919 * We use as the mode the number of bits allocated in the LDR for the
920 * logical processor ID. It happens that these are all powers of two.
921 * This makes it is very easy to detect cases where the APICs are
922 * configured for multiple modes; in that case, we cannot use the map and
923 * hence cannot use kvm_irq_delivery_to_apic_fast either.
925 #define KVM_APIC_MODE_XAPIC_CLUSTER 4
926 #define KVM_APIC_MODE_XAPIC_FLAT 8
927 #define KVM_APIC_MODE_X2APIC 16
929 struct kvm_apic_map
{
934 struct kvm_lapic
*xapic_flat_map
[8];
935 struct kvm_lapic
*xapic_cluster_map
[16][4];
937 struct kvm_lapic
*phys_map
[];
940 /* Hyper-V synthetic debugger (SynDbg)*/
941 struct kvm_hv_syndbg
{
952 /* Current state of Hyper-V TSC page clocksource */
953 enum hv_tsc_page_status
{
954 /* TSC page was not set up or disabled */
955 HV_TSC_PAGE_UNSET
= 0,
956 /* TSC page MSR was written by the guest, update pending */
957 HV_TSC_PAGE_GUEST_CHANGED
,
958 /* TSC page MSR was written by KVM userspace, update pending */
959 HV_TSC_PAGE_HOST_CHANGED
,
960 /* TSC page was properly set up and is currently active */
962 /* TSC page is currently being updated and therefore is inactive */
963 HV_TSC_PAGE_UPDATING
,
964 /* TSC page was set up with an inaccessible GPA */
968 /* Hyper-V emulation context */
970 struct mutex hv_lock
;
974 enum hv_tsc_page_status hv_tsc_page_status
;
976 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */
977 u64 hv_crash_param
[HV_X64_MSR_CRASH_PARAMS
];
980 struct ms_hyperv_tsc_page tsc_ref
;
982 struct idr conn_to_evt
;
984 u64 hv_reenlightenment_control
;
985 u64 hv_tsc_emulation_control
;
986 u64 hv_tsc_emulation_status
;
988 /* How many vCPUs have VP index != vCPU index */
989 atomic_t num_mismatched_vp_indexes
;
991 struct hv_partition_assist_pg
*hv_pa_pg
;
992 struct kvm_hv_syndbg hv_syndbg
;
995 struct msr_bitmap_range
{
999 unsigned long *bitmap
;
1002 /* Xen emulation context */
1007 struct gfn_to_hva_cache shinfo_cache
;
1010 enum kvm_irqchip_mode
{
1012 KVM_IRQCHIP_KERNEL
, /* created with KVM_CREATE_IRQCHIP */
1013 KVM_IRQCHIP_SPLIT
, /* created with KVM_CAP_SPLIT_IRQCHIP */
1016 struct kvm_x86_msr_filter
{
1018 bool default_allow
:1;
1019 struct msr_bitmap_range ranges
[16];
1022 #define APICV_INHIBIT_REASON_DISABLE 0
1023 #define APICV_INHIBIT_REASON_HYPERV 1
1024 #define APICV_INHIBIT_REASON_NESTED 2
1025 #define APICV_INHIBIT_REASON_IRQWIN 3
1026 #define APICV_INHIBIT_REASON_PIT_REINJ 4
1027 #define APICV_INHIBIT_REASON_X2APIC 5
1030 unsigned long n_used_mmu_pages
;
1031 unsigned long n_requested_mmu_pages
;
1032 unsigned long n_max_mmu_pages
;
1033 unsigned int indirect_shadow_pages
;
1035 struct hlist_head mmu_page_hash
[KVM_NUM_MMU_PAGES
];
1036 struct list_head active_mmu_pages
;
1037 struct list_head zapped_obsolete_pages
;
1038 struct list_head lpage_disallowed_mmu_pages
;
1039 struct kvm_page_track_notifier_node mmu_sp_tracker
;
1040 struct kvm_page_track_notifier_head track_notifier_head
;
1042 * Protects marking pages unsync during page faults, as TDP MMU page
1043 * faults only take mmu_lock for read. For simplicity, the unsync
1044 * pages lock is always taken when marking pages unsync regardless of
1045 * whether mmu_lock is held for read or write.
1047 spinlock_t mmu_unsync_pages_lock
;
1049 struct list_head assigned_dev_head
;
1050 struct iommu_domain
*iommu_domain
;
1051 bool iommu_noncoherent
;
1052 #define __KVM_HAVE_ARCH_NONCOHERENT_DMA
1053 atomic_t noncoherent_dma_count
;
1054 #define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
1055 atomic_t assigned_device_count
;
1056 struct kvm_pic
*vpic
;
1057 struct kvm_ioapic
*vioapic
;
1058 struct kvm_pit
*vpit
;
1059 atomic_t vapics_in_nmi_mode
;
1060 struct mutex apic_map_lock
;
1061 struct kvm_apic_map __rcu
*apic_map
;
1062 atomic_t apic_map_dirty
;
1064 bool apic_access_memslot_enabled
;
1065 unsigned long apicv_inhibit_reasons
;
1069 bool mwait_in_guest
;
1071 bool pause_in_guest
;
1072 bool cstate_in_guest
;
1074 unsigned long irq_sources_bitmap
;
1075 s64 kvmclock_offset
;
1076 raw_spinlock_t tsc_write_lock
;
1083 u64 cur_tsc_generation
;
1084 int nr_vcpus_matched_tsc
;
1086 spinlock_t pvclock_gtod_sync_lock
;
1087 bool use_master_clock
;
1088 u64 master_kernel_ns
;
1089 u64 master_cycle_now
;
1090 struct delayed_work kvmclock_update_work
;
1091 struct delayed_work kvmclock_sync_work
;
1093 struct kvm_xen_hvm_config xen_hvm_config
;
1095 /* reads protected by irq_srcu, writes by irq_lock */
1096 struct hlist_head mask_notifier_list
;
1098 struct kvm_hv hyperv
;
1101 #ifdef CONFIG_KVM_MMU_AUDIT
1105 bool backwards_tsc_observed
;
1106 bool boot_vcpu_runs_old_kvmclock
;
1109 u64 disabled_quirks
;
1110 int cpu_dirty_logging_count
;
1112 enum kvm_irqchip_mode irqchip_mode
;
1113 u8 nr_reserved_ioapic_pins
;
1115 bool disabled_lapic_found
;
1118 bool x2apic_broadcast_quirk_disabled
;
1120 bool guest_can_read_msr_platform_info
;
1121 bool exception_payload_enabled
;
1123 bool bus_lock_detection_enabled
;
1125 * If exit_on_emulation_error is set, and the in-kernel instruction
1126 * emulator fails to emulate an instruction, allow userspace
1127 * the opportunity to look at it.
1129 bool exit_on_emulation_error
;
1131 /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */
1132 u32 user_space_msr_mask
;
1133 struct kvm_x86_msr_filter __rcu
*msr_filter
;
1135 u32 hypercall_exit_enabled
;
1137 /* Guest can access the SGX PROVISIONKEY. */
1138 bool sgx_provisioning_allowed
;
1140 struct kvm_pmu_event_filter __rcu
*pmu_event_filter
;
1141 struct task_struct
*nx_lpage_recovery_thread
;
1143 #ifdef CONFIG_X86_64
1145 * Whether the TDP MMU is enabled for this VM. This contains a
1146 * snapshot of the TDP MMU module parameter from when the VM was
1147 * created and remains unchanged for the life of the VM. If this is
1148 * true, TDP MMU handler functions will run for various MMU
1151 bool tdp_mmu_enabled
;
1154 * List of struct kvm_mmu_pages being used as roots.
1155 * All struct kvm_mmu_pages in the list should have
1158 * For reads, this list is protected by:
1159 * the MMU lock in read mode + RCU or
1160 * the MMU lock in write mode
1162 * For writes, this list is protected by:
1163 * the MMU lock in read mode + the tdp_mmu_pages_lock or
1164 * the MMU lock in write mode
1166 * Roots will remain in the list until their tdp_mmu_root_count
1167 * drops to zero, at which point the thread that decremented the
1168 * count to zero should removed the root from the list and clean
1169 * it up, freeing the root after an RCU grace period.
1171 struct list_head tdp_mmu_roots
;
1174 * List of struct kvmp_mmu_pages not being used as roots.
1175 * All struct kvm_mmu_pages in the list should have
1176 * tdp_mmu_page set and a tdp_mmu_root_count of 0.
1178 struct list_head tdp_mmu_pages
;
1181 * Protects accesses to the following fields when the MMU lock
1182 * is held in read mode:
1183 * - tdp_mmu_roots (above)
1184 * - tdp_mmu_pages (above)
1185 * - the link field of struct kvm_mmu_pages used by the TDP MMU
1186 * - lpage_disallowed_mmu_pages
1187 * - the lpage_disallowed_link field of struct kvm_mmu_pages used
1189 * It is acceptable, but not necessary, to acquire this lock when
1190 * the thread holds the MMU lock in write mode.
1192 spinlock_t tdp_mmu_pages_lock
;
1193 #endif /* CONFIG_X86_64 */
1196 * If set, rmaps have been allocated for all memslots and should be
1197 * allocated for any newly created or modified memslots.
1199 bool memslots_have_rmaps
;
1201 #if IS_ENABLED(CONFIG_HYPERV)
1203 spinlock_t hv_root_tdp_lock
;
1207 struct kvm_vm_stat
{
1208 struct kvm_vm_stat_generic generic
;
1209 u64 mmu_shadow_zapped
;
1217 u64 nx_lpage_splits
;
1218 u64 max_mmu_page_hash_collisions
;
1221 struct kvm_vcpu_stat
{
1222 struct kvm_vcpu_stat_generic generic
;
1232 u64 irq_window_exits
;
1233 u64 nmi_window_exits
;
1236 u64 request_irq_exits
;
1238 u64 host_state_reload
;
1241 u64 insn_emulation_fail
;
1247 u64 directed_yield_attempted
;
1248 u64 directed_yield_successful
;
1252 struct x86_instruction_info
;
1255 bool host_initiated
;
1260 struct kvm_lapic_irq
{
1268 bool msi_redir_hint
;
1271 static inline u16
kvm_lapic_irq_dest_mode(bool dest_mode_logical
)
1273 return dest_mode_logical
? APIC_DEST_LOGICAL
: APIC_DEST_PHYSICAL
;
1276 struct kvm_x86_ops
{
1277 int (*hardware_enable
)(void);
1278 void (*hardware_disable
)(void);
1279 void (*hardware_unsetup
)(void);
1280 bool (*cpu_has_accelerated_tpr
)(void);
1281 bool (*has_emulated_msr
)(struct kvm
*kvm
, u32 index
);
1282 void (*vcpu_after_set_cpuid
)(struct kvm_vcpu
*vcpu
);
1284 unsigned int vm_size
;
1285 int (*vm_init
)(struct kvm
*kvm
);
1286 void (*vm_destroy
)(struct kvm
*kvm
);
1288 /* Create, but do not attach this VCPU */
1289 int (*vcpu_create
)(struct kvm_vcpu
*vcpu
);
1290 void (*vcpu_free
)(struct kvm_vcpu
*vcpu
);
1291 void (*vcpu_reset
)(struct kvm_vcpu
*vcpu
, bool init_event
);
1293 void (*prepare_guest_switch
)(struct kvm_vcpu
*vcpu
);
1294 void (*vcpu_load
)(struct kvm_vcpu
*vcpu
, int cpu
);
1295 void (*vcpu_put
)(struct kvm_vcpu
*vcpu
);
1297 void (*update_exception_bitmap
)(struct kvm_vcpu
*vcpu
);
1298 int (*get_msr
)(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
);
1299 int (*set_msr
)(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
);
1300 u64 (*get_segment_base
)(struct kvm_vcpu
*vcpu
, int seg
);
1301 void (*get_segment
)(struct kvm_vcpu
*vcpu
,
1302 struct kvm_segment
*var
, int seg
);
1303 int (*get_cpl
)(struct kvm_vcpu
*vcpu
);
1304 void (*set_segment
)(struct kvm_vcpu
*vcpu
,
1305 struct kvm_segment
*var
, int seg
);
1306 void (*get_cs_db_l_bits
)(struct kvm_vcpu
*vcpu
, int *db
, int *l
);
1307 void (*set_cr0
)(struct kvm_vcpu
*vcpu
, unsigned long cr0
);
1308 bool (*is_valid_cr4
)(struct kvm_vcpu
*vcpu
, unsigned long cr0
);
1309 void (*set_cr4
)(struct kvm_vcpu
*vcpu
, unsigned long cr4
);
1310 int (*set_efer
)(struct kvm_vcpu
*vcpu
, u64 efer
);
1311 void (*get_idt
)(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
);
1312 void (*set_idt
)(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
);
1313 void (*get_gdt
)(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
);
1314 void (*set_gdt
)(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
);
1315 void (*sync_dirty_debug_regs
)(struct kvm_vcpu
*vcpu
);
1316 void (*set_dr7
)(struct kvm_vcpu
*vcpu
, unsigned long value
);
1317 void (*cache_reg
)(struct kvm_vcpu
*vcpu
, enum kvm_reg reg
);
1318 unsigned long (*get_rflags
)(struct kvm_vcpu
*vcpu
);
1319 void (*set_rflags
)(struct kvm_vcpu
*vcpu
, unsigned long rflags
);
1321 void (*tlb_flush_all
)(struct kvm_vcpu
*vcpu
);
1322 void (*tlb_flush_current
)(struct kvm_vcpu
*vcpu
);
1323 int (*tlb_remote_flush
)(struct kvm
*kvm
);
1324 int (*tlb_remote_flush_with_range
)(struct kvm
*kvm
,
1325 struct kvm_tlb_range
*range
);
1328 * Flush any TLB entries associated with the given GVA.
1329 * Does not need to flush GPA->HPA mappings.
1330 * Can potentially get non-canonical addresses through INVLPGs, which
1331 * the implementation may choose to ignore if appropriate.
1333 void (*tlb_flush_gva
)(struct kvm_vcpu
*vcpu
, gva_t addr
);
1336 * Flush any TLB entries created by the guest. Like tlb_flush_gva(),
1337 * does not need to flush GPA->HPA mappings.
1339 void (*tlb_flush_guest
)(struct kvm_vcpu
*vcpu
);
1341 enum exit_fastpath_completion (*run
)(struct kvm_vcpu
*vcpu
);
1342 int (*handle_exit
)(struct kvm_vcpu
*vcpu
,
1343 enum exit_fastpath_completion exit_fastpath
);
1344 int (*skip_emulated_instruction
)(struct kvm_vcpu
*vcpu
);
1345 void (*update_emulated_instruction
)(struct kvm_vcpu
*vcpu
);
1346 void (*set_interrupt_shadow
)(struct kvm_vcpu
*vcpu
, int mask
);
1347 u32 (*get_interrupt_shadow
)(struct kvm_vcpu
*vcpu
);
1348 void (*patch_hypercall
)(struct kvm_vcpu
*vcpu
,
1349 unsigned char *hypercall_addr
);
1350 void (*set_irq
)(struct kvm_vcpu
*vcpu
);
1351 void (*set_nmi
)(struct kvm_vcpu
*vcpu
);
1352 void (*queue_exception
)(struct kvm_vcpu
*vcpu
);
1353 void (*cancel_injection
)(struct kvm_vcpu
*vcpu
);
1354 int (*interrupt_allowed
)(struct kvm_vcpu
*vcpu
, bool for_injection
);
1355 int (*nmi_allowed
)(struct kvm_vcpu
*vcpu
, bool for_injection
);
1356 bool (*get_nmi_mask
)(struct kvm_vcpu
*vcpu
);
1357 void (*set_nmi_mask
)(struct kvm_vcpu
*vcpu
, bool masked
);
1358 void (*enable_nmi_window
)(struct kvm_vcpu
*vcpu
);
1359 void (*enable_irq_window
)(struct kvm_vcpu
*vcpu
);
1360 void (*update_cr8_intercept
)(struct kvm_vcpu
*vcpu
, int tpr
, int irr
);
1361 bool (*check_apicv_inhibit_reasons
)(ulong bit
);
1362 void (*pre_update_apicv_exec_ctrl
)(struct kvm
*kvm
, bool activate
);
1363 void (*refresh_apicv_exec_ctrl
)(struct kvm_vcpu
*vcpu
);
1364 void (*hwapic_irr_update
)(struct kvm_vcpu
*vcpu
, int max_irr
);
1365 void (*hwapic_isr_update
)(struct kvm_vcpu
*vcpu
, int isr
);
1366 bool (*guest_apic_has_interrupt
)(struct kvm_vcpu
*vcpu
);
1367 void (*load_eoi_exitmap
)(struct kvm_vcpu
*vcpu
, u64
*eoi_exit_bitmap
);
1368 void (*set_virtual_apic_mode
)(struct kvm_vcpu
*vcpu
);
1369 void (*set_apic_access_page_addr
)(struct kvm_vcpu
*vcpu
);
1370 int (*deliver_posted_interrupt
)(struct kvm_vcpu
*vcpu
, int vector
);
1371 int (*sync_pir_to_irr
)(struct kvm_vcpu
*vcpu
);
1372 int (*set_tss_addr
)(struct kvm
*kvm
, unsigned int addr
);
1373 int (*set_identity_map_addr
)(struct kvm
*kvm
, u64 ident_addr
);
1374 u64 (*get_mt_mask
)(struct kvm_vcpu
*vcpu
, gfn_t gfn
, bool is_mmio
);
1376 void (*load_mmu_pgd
)(struct kvm_vcpu
*vcpu
, hpa_t root_hpa
,
1379 bool (*has_wbinvd_exit
)(void);
1381 u64 (*get_l2_tsc_offset
)(struct kvm_vcpu
*vcpu
);
1382 u64 (*get_l2_tsc_multiplier
)(struct kvm_vcpu
*vcpu
);
1383 void (*write_tsc_offset
)(struct kvm_vcpu
*vcpu
, u64 offset
);
1384 void (*write_tsc_multiplier
)(struct kvm_vcpu
*vcpu
, u64 multiplier
);
1387 * Retrieve somewhat arbitrary exit information. Intended to be used
1388 * only from within tracepoints to avoid VMREADs when tracing is off.
1390 void (*get_exit_info
)(struct kvm_vcpu
*vcpu
, u64
*info1
, u64
*info2
,
1391 u32
*exit_int_info
, u32
*exit_int_info_err_code
);
1393 int (*check_intercept
)(struct kvm_vcpu
*vcpu
,
1394 struct x86_instruction_info
*info
,
1395 enum x86_intercept_stage stage
,
1396 struct x86_exception
*exception
);
1397 void (*handle_exit_irqoff
)(struct kvm_vcpu
*vcpu
);
1399 void (*request_immediate_exit
)(struct kvm_vcpu
*vcpu
);
1401 void (*sched_in
)(struct kvm_vcpu
*kvm
, int cpu
);
1404 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A zero
1405 * value indicates CPU dirty logging is unsupported or disabled.
1407 int cpu_dirty_log_size
;
1408 void (*update_cpu_dirty_logging
)(struct kvm_vcpu
*vcpu
);
1410 /* pmu operations of sub-arch */
1411 const struct kvm_pmu_ops
*pmu_ops
;
1412 const struct kvm_x86_nested_ops
*nested_ops
;
1415 * Architecture specific hooks for vCPU blocking due to
1417 * Returns for .pre_block():
1418 * - 0 means continue to block the vCPU.
1419 * - 1 means we cannot block the vCPU since some event
1420 * happens during this period, such as, 'ON' bit in
1421 * posted-interrupts descriptor is set.
1423 int (*pre_block
)(struct kvm_vcpu
*vcpu
);
1424 void (*post_block
)(struct kvm_vcpu
*vcpu
);
1426 void (*vcpu_blocking
)(struct kvm_vcpu
*vcpu
);
1427 void (*vcpu_unblocking
)(struct kvm_vcpu
*vcpu
);
1429 int (*update_pi_irte
)(struct kvm
*kvm
, unsigned int host_irq
,
1430 uint32_t guest_irq
, bool set
);
1431 void (*start_assignment
)(struct kvm
*kvm
);
1432 void (*apicv_post_state_restore
)(struct kvm_vcpu
*vcpu
);
1433 bool (*dy_apicv_has_pending_interrupt
)(struct kvm_vcpu
*vcpu
);
1435 int (*set_hv_timer
)(struct kvm_vcpu
*vcpu
, u64 guest_deadline_tsc
,
1437 void (*cancel_hv_timer
)(struct kvm_vcpu
*vcpu
);
1439 void (*setup_mce
)(struct kvm_vcpu
*vcpu
);
1441 int (*smi_allowed
)(struct kvm_vcpu
*vcpu
, bool for_injection
);
1442 int (*enter_smm
)(struct kvm_vcpu
*vcpu
, char *smstate
);
1443 int (*leave_smm
)(struct kvm_vcpu
*vcpu
, const char *smstate
);
1444 void (*enable_smi_window
)(struct kvm_vcpu
*vcpu
);
1446 int (*mem_enc_op
)(struct kvm
*kvm
, void __user
*argp
);
1447 int (*mem_enc_reg_region
)(struct kvm
*kvm
, struct kvm_enc_region
*argp
);
1448 int (*mem_enc_unreg_region
)(struct kvm
*kvm
, struct kvm_enc_region
*argp
);
1449 int (*vm_copy_enc_context_from
)(struct kvm
*kvm
, unsigned int source_fd
);
1451 int (*get_msr_feature
)(struct kvm_msr_entry
*entry
);
1453 bool (*can_emulate_instruction
)(struct kvm_vcpu
*vcpu
, void *insn
, int insn_len
);
1455 bool (*apic_init_signal_blocked
)(struct kvm_vcpu
*vcpu
);
1456 int (*enable_direct_tlbflush
)(struct kvm_vcpu
*vcpu
);
1458 void (*migrate_timers
)(struct kvm_vcpu
*vcpu
);
1459 void (*msr_filter_changed
)(struct kvm_vcpu
*vcpu
);
1460 int (*complete_emulated_msr
)(struct kvm_vcpu
*vcpu
, int err
);
1462 void (*vcpu_deliver_sipi_vector
)(struct kvm_vcpu
*vcpu
, u8 vector
);
1465 struct kvm_x86_nested_ops
{
1466 int (*check_events
)(struct kvm_vcpu
*vcpu
);
1467 bool (*hv_timer_pending
)(struct kvm_vcpu
*vcpu
);
1468 void (*triple_fault
)(struct kvm_vcpu
*vcpu
);
1469 int (*get_state
)(struct kvm_vcpu
*vcpu
,
1470 struct kvm_nested_state __user
*user_kvm_nested_state
,
1471 unsigned user_data_size
);
1472 int (*set_state
)(struct kvm_vcpu
*vcpu
,
1473 struct kvm_nested_state __user
*user_kvm_nested_state
,
1474 struct kvm_nested_state
*kvm_state
);
1475 bool (*get_nested_state_pages
)(struct kvm_vcpu
*vcpu
);
1476 int (*write_log_dirty
)(struct kvm_vcpu
*vcpu
, gpa_t l2_gpa
);
1478 int (*enable_evmcs
)(struct kvm_vcpu
*vcpu
,
1479 uint16_t *vmcs_version
);
1480 uint16_t (*get_evmcs_version
)(struct kvm_vcpu
*vcpu
);
1483 struct kvm_x86_init_ops
{
1484 int (*cpu_has_kvm_support
)(void);
1485 int (*disabled_by_bios
)(void);
1486 int (*check_processor_compatibility
)(void);
1487 int (*hardware_setup
)(void);
1489 struct kvm_x86_ops
*runtime_ops
;
1492 struct kvm_arch_async_pf
{
1499 extern u32 __read_mostly kvm_nr_uret_msrs
;
1500 extern u64 __read_mostly host_efer
;
1501 extern bool __read_mostly allow_smaller_maxphyaddr
;
1502 extern bool __read_mostly enable_apicv
;
1503 extern struct kvm_x86_ops kvm_x86_ops
;
1505 #define KVM_X86_OP(func) \
1506 DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func));
1507 #define KVM_X86_OP_NULL KVM_X86_OP
1508 #include <asm/kvm-x86-ops.h>
1510 static inline void kvm_ops_static_call_update(void)
1512 #define KVM_X86_OP(func) \
1513 static_call_update(kvm_x86_##func, kvm_x86_ops.func);
1514 #define KVM_X86_OP_NULL KVM_X86_OP
1515 #include <asm/kvm-x86-ops.h>
1518 #define __KVM_HAVE_ARCH_VM_ALLOC
1519 static inline struct kvm
*kvm_arch_alloc_vm(void)
1521 return __vmalloc(kvm_x86_ops
.vm_size
, GFP_KERNEL_ACCOUNT
| __GFP_ZERO
);
1523 void kvm_arch_free_vm(struct kvm
*kvm
);
1525 #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
1526 static inline int kvm_arch_flush_remote_tlb(struct kvm
*kvm
)
1528 if (kvm_x86_ops
.tlb_remote_flush
&&
1529 !static_call(kvm_x86_tlb_remote_flush
)(kvm
))
1535 int kvm_mmu_module_init(void);
1536 void kvm_mmu_module_exit(void);
1538 void kvm_mmu_destroy(struct kvm_vcpu
*vcpu
);
1539 int kvm_mmu_create(struct kvm_vcpu
*vcpu
);
1540 void kvm_mmu_init_vm(struct kvm
*kvm
);
1541 void kvm_mmu_uninit_vm(struct kvm
*kvm
);
1543 void kvm_mmu_after_set_cpuid(struct kvm_vcpu
*vcpu
);
1544 void kvm_mmu_reset_context(struct kvm_vcpu
*vcpu
);
1545 void kvm_mmu_slot_remove_write_access(struct kvm
*kvm
,
1546 struct kvm_memory_slot
*memslot
,
1548 void kvm_mmu_zap_collapsible_sptes(struct kvm
*kvm
,
1549 const struct kvm_memory_slot
*memslot
);
1550 void kvm_mmu_slot_leaf_clear_dirty(struct kvm
*kvm
,
1551 struct kvm_memory_slot
*memslot
);
1552 void kvm_mmu_zap_all(struct kvm
*kvm
);
1553 void kvm_mmu_invalidate_mmio_sptes(struct kvm
*kvm
, u64 gen
);
1554 unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm
*kvm
);
1555 void kvm_mmu_change_mmu_pages(struct kvm
*kvm
, unsigned long kvm_nr_mmu_pages
);
1557 int load_pdptrs(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
, unsigned long cr3
);
1559 int emulator_write_phys(struct kvm_vcpu
*vcpu
, gpa_t gpa
,
1560 const void *val
, int bytes
);
1562 struct kvm_irq_mask_notifier
{
1563 void (*func
)(struct kvm_irq_mask_notifier
*kimn
, bool masked
);
1565 struct hlist_node link
;
1568 void kvm_register_irq_mask_notifier(struct kvm
*kvm
, int irq
,
1569 struct kvm_irq_mask_notifier
*kimn
);
1570 void kvm_unregister_irq_mask_notifier(struct kvm
*kvm
, int irq
,
1571 struct kvm_irq_mask_notifier
*kimn
);
1572 void kvm_fire_mask_notifiers(struct kvm
*kvm
, unsigned irqchip
, unsigned pin
,
1575 extern bool tdp_enabled
;
1577 u64
vcpu_tsc_khz(struct kvm_vcpu
*vcpu
);
1579 /* control of guest tsc rate supported? */
1580 extern bool kvm_has_tsc_control
;
1581 /* maximum supported tsc_khz for guests */
1582 extern u32 kvm_max_guest_tsc_khz
;
1583 /* number of bits of the fractional part of the TSC scaling ratio */
1584 extern u8 kvm_tsc_scaling_ratio_frac_bits
;
1585 /* maximum allowed value of TSC scaling ratio */
1586 extern u64 kvm_max_tsc_scaling_ratio
;
1587 /* 1ull << kvm_tsc_scaling_ratio_frac_bits */
1588 extern u64 kvm_default_tsc_scaling_ratio
;
1589 /* bus lock detection supported? */
1590 extern bool kvm_has_bus_lock_exit
;
1592 extern u64 kvm_mce_cap_supported
;
1595 * EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing
1596 * userspace I/O) to indicate that the emulation context
1597 * should be reused as is, i.e. skip initialization of
1598 * emulation context, instruction fetch and decode.
1600 * EMULTYPE_TRAP_UD - Set when emulating an intercepted #UD from hardware.
1601 * Indicates that only select instructions (tagged with
1602 * EmulateOnUD) should be emulated (to minimize the emulator
1603 * attack surface). See also EMULTYPE_TRAP_UD_FORCED.
1605 * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to
1606 * decode the instruction length. For use *only* by
1607 * kvm_x86_ops.skip_emulated_instruction() implementations.
1609 * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to
1610 * retry native execution under certain conditions,
1611 * Can only be set in conjunction with EMULTYPE_PF.
1613 * EMULTYPE_TRAP_UD_FORCED - Set when emulating an intercepted #UD that was
1614 * triggered by KVM's magic "force emulation" prefix,
1615 * which is opt in via module param (off by default).
1616 * Bypasses EmulateOnUD restriction despite emulating
1617 * due to an intercepted #UD (see EMULTYPE_TRAP_UD).
1618 * Used to test the full emulator from userspace.
1620 * EMULTYPE_VMWARE_GP - Set when emulating an intercepted #GP for VMware
1621 * backdoor emulation, which is opt in via module param.
1622 * VMware backdoor emulation handles select instructions
1623 * and reinjects the #GP for all other cases.
1625 * EMULTYPE_PF - Set when emulating MMIO by way of an intercepted #PF, in which
1626 * case the CR2/GPA value pass on the stack is valid.
1628 #define EMULTYPE_NO_DECODE (1 << 0)
1629 #define EMULTYPE_TRAP_UD (1 << 1)
1630 #define EMULTYPE_SKIP (1 << 2)
1631 #define EMULTYPE_ALLOW_RETRY_PF (1 << 3)
1632 #define EMULTYPE_TRAP_UD_FORCED (1 << 4)
1633 #define EMULTYPE_VMWARE_GP (1 << 5)
1634 #define EMULTYPE_PF (1 << 6)
1636 int kvm_emulate_instruction(struct kvm_vcpu
*vcpu
, int emulation_type
);
1637 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu
*vcpu
,
1638 void *insn
, int insn_len
);
1640 void kvm_enable_efer_bits(u64
);
1641 bool kvm_valid_efer(struct kvm_vcpu
*vcpu
, u64 efer
);
1642 int __kvm_get_msr(struct kvm_vcpu
*vcpu
, u32 index
, u64
*data
, bool host_initiated
);
1643 int kvm_get_msr(struct kvm_vcpu
*vcpu
, u32 index
, u64
*data
);
1644 int kvm_set_msr(struct kvm_vcpu
*vcpu
, u32 index
, u64 data
);
1645 int kvm_emulate_rdmsr(struct kvm_vcpu
*vcpu
);
1646 int kvm_emulate_wrmsr(struct kvm_vcpu
*vcpu
);
1647 int kvm_emulate_as_nop(struct kvm_vcpu
*vcpu
);
1648 int kvm_emulate_invd(struct kvm_vcpu
*vcpu
);
1649 int kvm_emulate_mwait(struct kvm_vcpu
*vcpu
);
1650 int kvm_handle_invalid_op(struct kvm_vcpu
*vcpu
);
1651 int kvm_emulate_monitor(struct kvm_vcpu
*vcpu
);
1653 int kvm_fast_pio(struct kvm_vcpu
*vcpu
, int size
, unsigned short port
, int in
);
1654 int kvm_emulate_cpuid(struct kvm_vcpu
*vcpu
);
1655 int kvm_emulate_halt(struct kvm_vcpu
*vcpu
);
1656 int kvm_vcpu_halt(struct kvm_vcpu
*vcpu
);
1657 int kvm_emulate_ap_reset_hold(struct kvm_vcpu
*vcpu
);
1658 int kvm_emulate_wbinvd(struct kvm_vcpu
*vcpu
);
1660 void kvm_get_segment(struct kvm_vcpu
*vcpu
, struct kvm_segment
*var
, int seg
);
1661 int kvm_load_segment_descriptor(struct kvm_vcpu
*vcpu
, u16 selector
, int seg
);
1662 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu
*vcpu
, u8 vector
);
1664 int kvm_task_switch(struct kvm_vcpu
*vcpu
, u16 tss_selector
, int idt_index
,
1665 int reason
, bool has_error_code
, u32 error_code
);
1667 void kvm_free_guest_fpu(struct kvm_vcpu
*vcpu
);
1669 void kvm_post_set_cr0(struct kvm_vcpu
*vcpu
, unsigned long old_cr0
, unsigned long cr0
);
1670 void kvm_post_set_cr4(struct kvm_vcpu
*vcpu
, unsigned long old_cr4
, unsigned long cr4
);
1671 int kvm_set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
);
1672 int kvm_set_cr3(struct kvm_vcpu
*vcpu
, unsigned long cr3
);
1673 int kvm_set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
);
1674 int kvm_set_cr8(struct kvm_vcpu
*vcpu
, unsigned long cr8
);
1675 int kvm_set_dr(struct kvm_vcpu
*vcpu
, int dr
, unsigned long val
);
1676 void kvm_get_dr(struct kvm_vcpu
*vcpu
, int dr
, unsigned long *val
);
1677 unsigned long kvm_get_cr8(struct kvm_vcpu
*vcpu
);
1678 void kvm_lmsw(struct kvm_vcpu
*vcpu
, unsigned long msw
);
1679 void kvm_get_cs_db_l_bits(struct kvm_vcpu
*vcpu
, int *db
, int *l
);
1680 int kvm_emulate_xsetbv(struct kvm_vcpu
*vcpu
);
1682 int kvm_get_msr_common(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
);
1683 int kvm_set_msr_common(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
);
1685 unsigned long kvm_get_rflags(struct kvm_vcpu
*vcpu
);
1686 void kvm_set_rflags(struct kvm_vcpu
*vcpu
, unsigned long rflags
);
1687 int kvm_emulate_rdpmc(struct kvm_vcpu
*vcpu
);
1689 void kvm_queue_exception(struct kvm_vcpu
*vcpu
, unsigned nr
);
1690 void kvm_queue_exception_e(struct kvm_vcpu
*vcpu
, unsigned nr
, u32 error_code
);
1691 void kvm_queue_exception_p(struct kvm_vcpu
*vcpu
, unsigned nr
, unsigned long payload
);
1692 void kvm_requeue_exception(struct kvm_vcpu
*vcpu
, unsigned nr
);
1693 void kvm_requeue_exception_e(struct kvm_vcpu
*vcpu
, unsigned nr
, u32 error_code
);
1694 void kvm_inject_page_fault(struct kvm_vcpu
*vcpu
, struct x86_exception
*fault
);
1695 bool kvm_inject_emulated_page_fault(struct kvm_vcpu
*vcpu
,
1696 struct x86_exception
*fault
);
1697 int kvm_read_guest_page_mmu(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
1698 gfn_t gfn
, void *data
, int offset
, int len
,
1700 bool kvm_require_cpl(struct kvm_vcpu
*vcpu
, int required_cpl
);
1701 bool kvm_require_dr(struct kvm_vcpu
*vcpu
, int dr
);
1703 static inline int __kvm_irq_line_state(unsigned long *irq_state
,
1704 int irq_source_id
, int level
)
1706 /* Logical OR for level trig interrupt */
1708 __set_bit(irq_source_id
, irq_state
);
1710 __clear_bit(irq_source_id
, irq_state
);
1712 return !!(*irq_state
);
1715 #define KVM_MMU_ROOT_CURRENT BIT(0)
1716 #define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i)
1717 #define KVM_MMU_ROOTS_ALL (~0UL)
1719 int kvm_pic_set_irq(struct kvm_pic
*pic
, int irq
, int irq_source_id
, int level
);
1720 void kvm_pic_clear_all(struct kvm_pic
*pic
, int irq_source_id
);
1722 void kvm_inject_nmi(struct kvm_vcpu
*vcpu
);
1724 void kvm_update_dr7(struct kvm_vcpu
*vcpu
);
1726 int kvm_mmu_unprotect_page(struct kvm
*kvm
, gfn_t gfn
);
1727 void __kvm_mmu_free_some_pages(struct kvm_vcpu
*vcpu
);
1728 void kvm_mmu_free_roots(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
1729 ulong roots_to_free
);
1730 void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
);
1731 gpa_t
translate_nested_gpa(struct kvm_vcpu
*vcpu
, gpa_t gpa
, u32 access
,
1732 struct x86_exception
*exception
);
1733 gpa_t
kvm_mmu_gva_to_gpa_read(struct kvm_vcpu
*vcpu
, gva_t gva
,
1734 struct x86_exception
*exception
);
1735 gpa_t
kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu
*vcpu
, gva_t gva
,
1736 struct x86_exception
*exception
);
1737 gpa_t
kvm_mmu_gva_to_gpa_write(struct kvm_vcpu
*vcpu
, gva_t gva
,
1738 struct x86_exception
*exception
);
1739 gpa_t
kvm_mmu_gva_to_gpa_system(struct kvm_vcpu
*vcpu
, gva_t gva
,
1740 struct x86_exception
*exception
);
1742 bool kvm_apicv_activated(struct kvm
*kvm
);
1743 void kvm_vcpu_update_apicv(struct kvm_vcpu
*vcpu
);
1744 void kvm_request_apicv_update(struct kvm
*kvm
, bool activate
,
1747 int kvm_emulate_hypercall(struct kvm_vcpu
*vcpu
);
1749 int kvm_mmu_page_fault(struct kvm_vcpu
*vcpu
, gpa_t cr2_or_gpa
, u64 error_code
,
1750 void *insn
, int insn_len
);
1751 void kvm_mmu_invlpg(struct kvm_vcpu
*vcpu
, gva_t gva
);
1752 void kvm_mmu_invalidate_gva(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
1753 gva_t gva
, hpa_t root_hpa
);
1754 void kvm_mmu_invpcid_gva(struct kvm_vcpu
*vcpu
, gva_t gva
, unsigned long pcid
);
1755 void kvm_mmu_new_pgd(struct kvm_vcpu
*vcpu
, gpa_t new_pgd
);
1757 void kvm_configure_mmu(bool enable_tdp
, int tdp_max_root_level
,
1758 int tdp_huge_page_level
);
1760 static inline u16
kvm_read_ldt(void)
1763 asm("sldt %0" : "=g"(ldt
));
1767 static inline void kvm_load_ldt(u16 sel
)
1769 asm("lldt %0" : : "rm"(sel
));
1772 #ifdef CONFIG_X86_64
1773 static inline unsigned long read_msr(unsigned long msr
)
1782 static inline u32
get_rdx_init_val(void)
1784 return 0x600; /* P6 family */
1787 static inline void kvm_inject_gp(struct kvm_vcpu
*vcpu
, u32 error_code
)
1789 kvm_queue_exception_e(vcpu
, GP_VECTOR
, error_code
);
1792 #define TSS_IOPB_BASE_OFFSET 0x66
1793 #define TSS_BASE_SIZE 0x68
1794 #define TSS_IOPB_SIZE (65536 / 8)
1795 #define TSS_REDIRECTION_SIZE (256 / 8)
1796 #define RMODE_TSS_SIZE \
1797 (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
1800 TASK_SWITCH_CALL
= 0,
1801 TASK_SWITCH_IRET
= 1,
1802 TASK_SWITCH_JMP
= 2,
1803 TASK_SWITCH_GATE
= 3,
1806 #define HF_GIF_MASK (1 << 0)
1807 #define HF_NMI_MASK (1 << 3)
1808 #define HF_IRET_MASK (1 << 4)
1809 #define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */
1810 #define HF_SMM_MASK (1 << 6)
1811 #define HF_SMM_INSIDE_NMI_MASK (1 << 7)
1813 #define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
1814 #define KVM_ADDRESS_SPACE_NUM 2
1816 #define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
1817 #define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
1819 asmlinkage
void kvm_spurious_fault(void);
1822 * Hardware virtualization extension instructions may fault if a
1823 * reboot turns off virtualization while processes are running.
1824 * Usually after catching the fault we just panic; during reboot
1825 * instead the instruction is ignored.
1827 #define __kvm_handle_fault_on_reboot(insn) \
1833 ".pushsection .discard.instr_begin \n\t" \
1834 ".long 1b - . \n\t" \
1835 ".popsection \n\t" \
1836 "call kvm_spurious_fault \n\t" \
1838 ".pushsection .discard.instr_end \n\t" \
1839 ".long 1b - . \n\t" \
1840 ".popsection \n\t" \
1842 _ASM_EXTABLE(666b, 667b)
1844 #define KVM_ARCH_WANT_MMU_NOTIFIER
1846 int kvm_cpu_has_injectable_intr(struct kvm_vcpu
*v
);
1847 int kvm_cpu_has_interrupt(struct kvm_vcpu
*vcpu
);
1848 int kvm_cpu_has_extint(struct kvm_vcpu
*v
);
1849 int kvm_arch_interrupt_allowed(struct kvm_vcpu
*vcpu
);
1850 int kvm_cpu_get_interrupt(struct kvm_vcpu
*v
);
1851 void kvm_vcpu_reset(struct kvm_vcpu
*vcpu
, bool init_event
);
1852 void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu
*vcpu
);
1854 int kvm_pv_send_ipi(struct kvm
*kvm
, unsigned long ipi_bitmap_low
,
1855 unsigned long ipi_bitmap_high
, u32 min
,
1856 unsigned long icr
, int op_64_bit
);
1858 int kvm_add_user_return_msr(u32 msr
);
1859 int kvm_find_user_return_msr(u32 msr
);
1860 int kvm_set_user_return_msr(unsigned index
, u64 val
, u64 mask
);
1862 static inline bool kvm_is_supported_user_return_msr(u32 msr
)
1864 return kvm_find_user_return_msr(msr
) >= 0;
1867 u64
kvm_scale_tsc(struct kvm_vcpu
*vcpu
, u64 tsc
, u64 ratio
);
1868 u64
kvm_read_l1_tsc(struct kvm_vcpu
*vcpu
, u64 host_tsc
);
1869 u64
kvm_calc_nested_tsc_offset(u64 l1_offset
, u64 l2_offset
, u64 l2_multiplier
);
1870 u64
kvm_calc_nested_tsc_multiplier(u64 l1_multiplier
, u64 l2_multiplier
);
1872 unsigned long kvm_get_linear_rip(struct kvm_vcpu
*vcpu
);
1873 bool kvm_is_linear_rip(struct kvm_vcpu
*vcpu
, unsigned long linear_rip
);
1875 void kvm_make_mclock_inprogress_request(struct kvm
*kvm
);
1876 void kvm_make_scan_ioapic_request(struct kvm
*kvm
);
1877 void kvm_make_scan_ioapic_request_mask(struct kvm
*kvm
,
1878 unsigned long *vcpu_bitmap
);
1880 bool kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
1881 struct kvm_async_pf
*work
);
1882 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
1883 struct kvm_async_pf
*work
);
1884 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
1885 struct kvm_async_pf
*work
);
1886 void kvm_arch_async_page_present_queued(struct kvm_vcpu
*vcpu
);
1887 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu
*vcpu
);
1888 extern bool kvm_find_async_pf_gfn(struct kvm_vcpu
*vcpu
, gfn_t gfn
);
1890 int kvm_skip_emulated_instruction(struct kvm_vcpu
*vcpu
);
1891 int kvm_complete_insn_gp(struct kvm_vcpu
*vcpu
, int err
);
1892 void __kvm_request_immediate_exit(struct kvm_vcpu
*vcpu
);
1894 int kvm_is_in_guest(void);
1896 void __user
*__x86_set_memory_region(struct kvm
*kvm
, int id
, gpa_t gpa
,
1898 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu
*vcpu
);
1899 bool kvm_vcpu_is_bsp(struct kvm_vcpu
*vcpu
);
1901 bool kvm_intr_is_single_vcpu(struct kvm
*kvm
, struct kvm_lapic_irq
*irq
,
1902 struct kvm_vcpu
**dest_vcpu
);
1904 void kvm_set_msi_irq(struct kvm
*kvm
, struct kvm_kernel_irq_routing_entry
*e
,
1905 struct kvm_lapic_irq
*irq
);
1907 static inline bool kvm_irq_is_postable(struct kvm_lapic_irq
*irq
)
1909 /* We can only post Fixed and LowPrio IRQs */
1910 return (irq
->delivery_mode
== APIC_DM_FIXED
||
1911 irq
->delivery_mode
== APIC_DM_LOWEST
);
1914 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu
*vcpu
)
1916 static_call_cond(kvm_x86_vcpu_blocking
)(vcpu
);
1919 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu
*vcpu
)
1921 static_call_cond(kvm_x86_vcpu_unblocking
)(vcpu
);
1924 static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu
*vcpu
) {}
1926 static inline int kvm_cpu_get_apicid(int mps_cpu
)
1928 #ifdef CONFIG_X86_LOCAL_APIC
1929 return default_cpu_present_to_apicid(mps_cpu
);
1936 #define put_smstate(type, buf, offset, val) \
1937 *(type *)((buf) + (offset) - 0x7e00) = val
1939 #define GET_SMSTATE(type, buf, offset) \
1940 (*(type *)((buf) + (offset) - 0x7e00))
1942 int kvm_cpu_dirty_log_size(void);
1944 int alloc_all_memslots_rmaps(struct kvm
*kvm
);
1946 #endif /* _ASM_X86_KVM_HOST_H */