]> git.proxmox.com Git - mirror_ubuntu-disco-kernel.git/blame - arch/x86/include/asm/kvm_host.h
kvm: x86: Add has_payload and payload to kvm_queued_exception
[mirror_ubuntu-disco-kernel.git] / arch / x86 / include / asm / kvm_host.h
CommitLineData
a656c8ef 1/*
043405e1
CO
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This header defines architecture specific interfaces, x86 version
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
8 *
9 */
10
1965aae3
PA
11#ifndef _ASM_X86_KVM_HOST_H
12#define _ASM_X86_KVM_HOST_H
043405e1 13
34c16eec
ZX
14#include <linux/types.h>
15#include <linux/mm.h>
e930bffe 16#include <linux/mmu_notifier.h>
229456fc 17#include <linux/tracepoint.h>
f5f48ee1 18#include <linux/cpumask.h>
f5132b01 19#include <linux/irq_work.h>
447ae316 20#include <linux/irq.h>
34c16eec
ZX
21
22#include <linux/kvm.h>
23#include <linux/kvm_para.h>
edf88417 24#include <linux/kvm_types.h>
f5132b01 25#include <linux/perf_event.h>
d828199e
MT
26#include <linux/pvclock_gtod.h>
27#include <linux/clocksource.h>
87276880 28#include <linux/irqbypass.h>
5c919412 29#include <linux/hyperv.h>
34c16eec 30
7d669f50 31#include <asm/apic.h>
50d0a0f9 32#include <asm/pvclock-abi.h>
e01a1b57 33#include <asm/desc.h>
0bed3b56 34#include <asm/mtrr.h>
9962d032 35#include <asm/msr-index.h>
3ee89722 36#include <asm/asm.h>
21ebbeda 37#include <asm/kvm_page_track.h>
5a485803 38#include <asm/hyperv-tlfs.h>
e01a1b57 39
682f732e 40#define KVM_MAX_VCPUS 288
757883de 41#define KVM_SOFT_MAX_VCPUS 240
af1bae54 42#define KVM_MAX_VCPU_ID 1023
1d4e7e3c 43#define KVM_USER_MEM_SLOTS 509
0743247f
AW
44/* memory slots that are not exposed to userspace */
45#define KVM_PRIVATE_MEM_SLOTS 3
bbacc0c1 46#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
93a5cef0 47
b401ee0b 48#define KVM_HALT_POLL_NS_DEFAULT 200000
69a9f69b 49
8175e5b7
AG
50#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
51
2860c4b1 52/* x86-specific vcpu->requests bit members */
2387149e
AJ
53#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
54#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
55#define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
56#define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
57#define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
6e42782f 58#define KVM_REQ_LOAD_CR3 KVM_ARCH_REQ(5)
2387149e
AJ
59#define KVM_REQ_EVENT KVM_ARCH_REQ(6)
60#define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
61#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
62#define KVM_REQ_NMI KVM_ARCH_REQ(9)
63#define KVM_REQ_PMU KVM_ARCH_REQ(10)
64#define KVM_REQ_PMI KVM_ARCH_REQ(11)
65#define KVM_REQ_SMI KVM_ARCH_REQ(12)
66#define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
67#define KVM_REQ_MCLOCK_INPROGRESS \
68 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
69#define KVM_REQ_SCAN_IOAPIC \
70 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
71#define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
72#define KVM_REQ_APIC_PAGE_RELOAD \
73 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
74#define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
75#define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
76#define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
77#define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
78#define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
e40ff1d6 79#define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23)
7f7f1ba3 80#define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24)
2860c4b1 81
cfec82cb
JR
82#define CR0_RESERVED_BITS \
83 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
84 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
85 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
86
cfec82cb
JR
87#define CR4_RESERVED_BITS \
88 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
89 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
ad756a16 90 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
afcbf13f 91 | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
fd8cb433 92 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
ae3e61e1 93 | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
cfec82cb
JR
94
95#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
96
97
cd6e8f87 98
cd6e8f87 99#define INVALID_PAGE (~(hpa_t)0)
dd180b3e
XG
100#define VALID_PAGE(x) ((x) != INVALID_PAGE)
101
cd6e8f87
ZX
102#define UNMAPPED_GVA (~(gpa_t)0)
103
ec04b260 104/* KVM Hugepage definitions for x86 */
4fef0f49
WY
105enum {
106 PT_PAGE_TABLE_LEVEL = 1,
107 PT_DIRECTORY_LEVEL = 2,
108 PT_PDPE_LEVEL = 3,
109 /* set max level to the biggest one */
110 PT_MAX_HUGEPAGE_LEVEL = PT_PDPE_LEVEL,
111};
112#define KVM_NR_PAGE_SIZES (PT_MAX_HUGEPAGE_LEVEL - \
113 PT_PAGE_TABLE_LEVEL + 1)
82855413
JR
114#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
115#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
ec04b260
JR
116#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
117#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
118#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
05da4558 119
6d9d41e5
CD
120static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
121{
122 /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */
123 return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
124 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
125}
126
d657a98e
ZX
127#define KVM_PERMILLE_MMU_PAGES 20
128#define KVM_MIN_ALLOC_MMU_PAGES 64
114df303 129#define KVM_MMU_HASH_SHIFT 12
1ae0a13d 130#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
d657a98e
ZX
131#define KVM_MIN_FREE_MMU_PAGES 5
132#define KVM_REFILL_PAGES 25
73c1160c 133#define KVM_MAX_CPUID_ENTRIES 80
0bed3b56 134#define KVM_NR_FIXED_MTRR_REGION 88
0d234daf 135#define KVM_NR_VAR_MTRR 8
d657a98e 136
af585b92
GN
137#define ASYNC_PF_PER_VCPU 64
138
5fdbf976 139enum kvm_reg {
2b3ccfa0
ZX
140 VCPU_REGS_RAX = 0,
141 VCPU_REGS_RCX = 1,
142 VCPU_REGS_RDX = 2,
143 VCPU_REGS_RBX = 3,
144 VCPU_REGS_RSP = 4,
145 VCPU_REGS_RBP = 5,
146 VCPU_REGS_RSI = 6,
147 VCPU_REGS_RDI = 7,
148#ifdef CONFIG_X86_64
149 VCPU_REGS_R8 = 8,
150 VCPU_REGS_R9 = 9,
151 VCPU_REGS_R10 = 10,
152 VCPU_REGS_R11 = 11,
153 VCPU_REGS_R12 = 12,
154 VCPU_REGS_R13 = 13,
155 VCPU_REGS_R14 = 14,
156 VCPU_REGS_R15 = 15,
157#endif
5fdbf976 158 VCPU_REGS_RIP,
2b3ccfa0
ZX
159 NR_VCPU_REGS
160};
161
6de4f3ad
AK
162enum kvm_reg_ex {
163 VCPU_EXREG_PDPTR = NR_VCPU_REGS,
aff48baa 164 VCPU_EXREG_CR3,
6de12732 165 VCPU_EXREG_RFLAGS,
2fb92db1 166 VCPU_EXREG_SEGMENTS,
6de4f3ad
AK
167};
168
2b3ccfa0 169enum {
81609e3e 170 VCPU_SREG_ES,
2b3ccfa0 171 VCPU_SREG_CS,
81609e3e 172 VCPU_SREG_SS,
2b3ccfa0 173 VCPU_SREG_DS,
2b3ccfa0
ZX
174 VCPU_SREG_FS,
175 VCPU_SREG_GS,
2b3ccfa0
ZX
176 VCPU_SREG_TR,
177 VCPU_SREG_LDTR,
178};
179
56e82318 180#include <asm/kvm_emulate.h>
2b3ccfa0 181
d657a98e
ZX
182#define KVM_NR_MEM_OBJS 40
183
42dbaa5a
JK
184#define KVM_NR_DB_REGS 4
185
186#define DR6_BD (1 << 13)
187#define DR6_BS (1 << 14)
cfb634fe 188#define DR6_BT (1 << 15)
6f43ed01
NA
189#define DR6_RTM (1 << 16)
190#define DR6_FIXED_1 0xfffe0ff0
191#define DR6_INIT 0xffff0ff0
192#define DR6_VOLATILE 0x0001e00f
42dbaa5a
JK
193
194#define DR7_BP_EN_MASK 0x000000ff
195#define DR7_GE (1 << 9)
196#define DR7_GD (1 << 13)
197#define DR7_FIXED_1 0x00000400
6f43ed01 198#define DR7_VOLATILE 0xffff2bff
42dbaa5a 199
c205fb7d
NA
200#define PFERR_PRESENT_BIT 0
201#define PFERR_WRITE_BIT 1
202#define PFERR_USER_BIT 2
203#define PFERR_RSVD_BIT 3
204#define PFERR_FETCH_BIT 4
be94f6b7 205#define PFERR_PK_BIT 5
14727754
TL
206#define PFERR_GUEST_FINAL_BIT 32
207#define PFERR_GUEST_PAGE_BIT 33
c205fb7d
NA
208
209#define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
210#define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
211#define PFERR_USER_MASK (1U << PFERR_USER_BIT)
212#define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
213#define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
be94f6b7 214#define PFERR_PK_MASK (1U << PFERR_PK_BIT)
14727754
TL
215#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
216#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
217
218#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \
14727754
TL
219 PFERR_WRITE_MASK | \
220 PFERR_PRESENT_MASK)
c205fb7d 221
37f0e8fe
JS
222/*
223 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or
224 * Access Tracking SPTEs. We use bit 62 instead of bit 63 to avoid conflicting
225 * with the SVE bit in EPT PTEs.
226 */
227#define SPTE_SPECIAL_MASK (1ULL << 62)
228
41383771
GN
229/* apic attention bits */
230#define KVM_APIC_CHECK_VAPIC 0
ae7a2a3f
MT
231/*
232 * The following bit is set with PV-EOI, unset on EOI.
233 * We detect PV-EOI changes by guest by comparing
234 * this bit with PV-EOI in guest memory.
235 * See the implementation in apic_update_pv_eoi.
236 */
237#define KVM_APIC_PV_EOI_PENDING 1
41383771 238
d84f1e07
FW
239struct kvm_kernel_irq_routing_entry;
240
d657a98e
ZX
241/*
242 * We don't want allocation failures within the mmu code, so we preallocate
243 * enough memory for a single page fault in a cache.
244 */
245struct kvm_mmu_memory_cache {
246 int nobjs;
247 void *objects[KVM_NR_MEM_OBJS];
248};
249
21ebbeda
XG
250/*
251 * the pages used as guest page table on soft mmu are tracked by
252 * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used
253 * by indirect shadow page can not be more than 15 bits.
254 *
255 * Currently, we used 14 bits that are @level, @cr4_pae, @quadrant, @access,
256 * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp.
257 */
d657a98e 258union kvm_mmu_page_role {
36d9594d 259 u32 word;
d657a98e 260 struct {
7d76b4d3 261 unsigned level:4;
5b7e0102 262 unsigned cr4_pae:1;
7d76b4d3 263 unsigned quadrant:2;
f6e2c02b 264 unsigned direct:1;
7d76b4d3 265 unsigned access:3;
2e53d63a 266 unsigned invalid:1;
9645bb56 267 unsigned nxe:1;
3dbe1415 268 unsigned cr0_wp:1;
411c588d 269 unsigned smep_andnot_wp:1;
0be0226f 270 unsigned smap_andnot_wp:1;
ac8d57e5 271 unsigned ad_disabled:1;
1313cc2b
JM
272 unsigned guest_mode:1;
273 unsigned :6;
699023e2
PB
274
275 /*
276 * This is left at the top of the word so that
277 * kvm_memslots_for_spte_role can extract it with a
278 * simple shift. While there is room, give it a whole
279 * byte so it is also faster to load it from memory.
280 */
281 unsigned smm:8;
d657a98e
ZX
282 };
283};
284
36d9594d 285union kvm_mmu_extended_role {
a336282d
VK
286/*
287 * This structure complements kvm_mmu_page_role caching everything needed for
288 * MMU configuration. If nothing in both these structures changed, MMU
289 * re-configuration can be skipped. @valid bit is set on first usage so we don't
290 * treat all-zero structure as valid data.
291 */
36d9594d 292 u32 word;
a336282d
VK
293 struct {
294 unsigned int valid:1;
295 unsigned int execonly:1;
7dcd5755 296 unsigned int cr0_pg:1;
a336282d
VK
297 unsigned int cr4_pse:1;
298 unsigned int cr4_pke:1;
299 unsigned int cr4_smap:1;
300 unsigned int cr4_smep:1;
7dcd5755 301 unsigned int cr4_la57:1;
a336282d 302 };
36d9594d
VK
303};
304
305union kvm_mmu_role {
306 u64 as_u64;
307 struct {
308 union kvm_mmu_page_role base;
309 union kvm_mmu_extended_role ext;
310 };
311};
312
018aabb5
TY
313struct kvm_rmap_head {
314 unsigned long val;
315};
316
d657a98e
ZX
317struct kvm_mmu_page {
318 struct list_head link;
319 struct hlist_node hash_link;
3ff519f2 320 bool unsync;
d657a98e
ZX
321
322 /*
323 * The following two entries are used to key the shadow page in the
324 * hash table.
325 */
d657a98e 326 union kvm_mmu_page_role role;
3ff519f2 327 gfn_t gfn;
d657a98e
ZX
328
329 u64 *spt;
330 /* hold the gfn of each spte inside spt */
331 gfn_t *gfns;
0571d366 332 int root_count; /* Currently serving as active root */
60c8aec6 333 unsigned int unsync_children;
018aabb5 334 struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
f6f8adee
XG
335
336 /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */
5304b8d3 337 unsigned long mmu_valid_gen;
f6f8adee 338
0074ff63 339 DECLARE_BITMAP(unsync_child_bitmap, 512);
c2a2ac2b
XG
340
341#ifdef CONFIG_X86_32
accaefe0
XG
342 /*
343 * Used out of the mmu-lock to avoid reading spte values while an
344 * update is in progress; see the comments in __get_spte_lockless().
345 */
c2a2ac2b
XG
346 int clear_spte_count;
347#endif
348
0cbf8e43 349 /* Number of writes since the last time traversal visited this page. */
e5691a81 350 atomic_t write_flooding_count;
d657a98e
ZX
351};
352
1c08364c
AK
353struct kvm_pio_request {
354 unsigned long count;
1c08364c
AK
355 int in;
356 int port;
357 int size;
1c08364c
AK
358};
359
855feb67 360#define PT64_ROOT_MAX_LEVEL 5
2a7266a8 361
a0a64f50 362struct rsvd_bits_validate {
2a7266a8 363 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
a0a64f50
XG
364 u64 bad_mt_xwr;
365};
366
7c390d35
JS
367struct kvm_mmu_root_info {
368 gpa_t cr3;
369 hpa_t hpa;
370};
371
372#define KVM_MMU_ROOT_INFO_INVALID \
373 ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE })
374
b94742c9
JS
375#define KVM_MMU_NUM_PREV_ROOTS 3
376
d657a98e 377/*
855feb67
YZ
378 * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
379 * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
380 * current mmu mode.
d657a98e
ZX
381 */
382struct kvm_mmu {
f43addd4 383 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
5777ed34 384 unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
e4e517b4 385 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
78b2c54a
XG
386 int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
387 bool prefault);
6389ee94
AK
388 void (*inject_page_fault)(struct kvm_vcpu *vcpu,
389 struct x86_exception *fault);
1871c602 390 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
ab9ae313 391 struct x86_exception *exception);
54987b7a
PB
392 gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
393 struct x86_exception *exception);
e8bc217a 394 int (*sync_page)(struct kvm_vcpu *vcpu,
a4a8e6f7 395 struct kvm_mmu_page *sp);
7eb77e9f 396 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
0f53b5b1 397 void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
7c562522 398 u64 *spte, const void *pte);
d657a98e 399 hpa_t root_hpa;
36d9594d 400 union kvm_mmu_role mmu_role;
ae1e2d10
PB
401 u8 root_level;
402 u8 shadow_root_level;
403 u8 ept_ad;
c5a78f2b 404 bool direct_map;
b94742c9 405 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
d657a98e 406
97d64b78
AK
407 /*
408 * Bitmap; bit set = permission fault
409 * Byte index: page fault error code [4:1]
410 * Bit index: pte permissions in ACC_* format
411 */
412 u8 permissions[16];
413
2d344105
HH
414 /*
415 * The pkru_mask indicates if protection key checks are needed. It
416 * consists of 16 domains indexed by page fault error code bits [4:1],
417 * with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
418 * Each domain has 2 bits which are ANDed with AD and WD from PKRU.
419 */
420 u32 pkru_mask;
421
d657a98e 422 u64 *pae_root;
81407ca5 423 u64 *lm_root;
c258b62b
XG
424
425 /*
426 * check zero bits on shadow page table entries, these
427 * bits include not only hardware reserved bits but also
428 * the bits spte never used.
429 */
430 struct rsvd_bits_validate shadow_zero_check;
431
a0a64f50 432 struct rsvd_bits_validate guest_rsvd_check;
ff03a073 433
6bb69c9b
PB
434 /* Can have large pages at levels 2..last_nonleaf_level-1. */
435 u8 last_nonleaf_level;
6fd01b71 436
2d48a985
JR
437 bool nx;
438
ff03a073 439 u64 pdptrs[4]; /* pae */
d657a98e
ZX
440};
441
f5132b01
GN
442enum pmc_type {
443 KVM_PMC_GP = 0,
444 KVM_PMC_FIXED,
445};
446
447struct kvm_pmc {
448 enum pmc_type type;
449 u8 idx;
450 u64 counter;
451 u64 eventsel;
452 struct perf_event *perf_event;
453 struct kvm_vcpu *vcpu;
454};
455
456struct kvm_pmu {
457 unsigned nr_arch_gp_counters;
458 unsigned nr_arch_fixed_counters;
459 unsigned available_event_types;
460 u64 fixed_ctr_ctrl;
461 u64 global_ctrl;
462 u64 global_status;
463 u64 global_ovf_ctrl;
464 u64 counter_bitmask[2];
465 u64 global_ctrl_mask;
103af0a9 466 u64 reserved_bits;
f5132b01 467 u8 version;
15c7ad51
RR
468 struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC];
469 struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED];
f5132b01
GN
470 struct irq_work irq_work;
471 u64 reprogram_pmi;
472};
473
25462f7f
WH
474struct kvm_pmu_ops;
475
360b948d
PB
476enum {
477 KVM_DEBUGREG_BP_ENABLED = 1,
c77fb5fe 478 KVM_DEBUGREG_WONT_EXIT = 2,
ae561ede 479 KVM_DEBUGREG_RELOAD = 4,
360b948d
PB
480};
481
86fd5270
XG
482struct kvm_mtrr_range {
483 u64 base;
484 u64 mask;
19efffa2 485 struct list_head node;
86fd5270
XG
486};
487
70109e7d 488struct kvm_mtrr {
86fd5270 489 struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR];
70109e7d 490 mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION];
10fac2dc 491 u64 deftype;
19efffa2
XG
492
493 struct list_head head;
70109e7d
XG
494};
495
1f4b34f8
AS
496/* Hyper-V SynIC timer */
497struct kvm_vcpu_hv_stimer {
498 struct hrtimer timer;
499 int index;
500 u64 config;
501 u64 count;
502 u64 exp_time;
503 struct hv_message msg;
504 bool msg_pending;
505};
506
5c919412
AS
507/* Hyper-V synthetic interrupt controller (SynIC)*/
508struct kvm_vcpu_hv_synic {
509 u64 version;
510 u64 control;
511 u64 msg_page;
512 u64 evt_page;
513 atomic64_t sint[HV_SYNIC_SINT_COUNT];
514 atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
515 DECLARE_BITMAP(auto_eoi_bitmap, 256);
516 DECLARE_BITMAP(vec_bitmap, 256);
517 bool active;
efc479e6 518 bool dont_zero_synic_pages;
5c919412
AS
519};
520
e83d5887
AS
521/* Hyper-V per vcpu emulation context */
522struct kvm_vcpu_hv {
d3457c87 523 u32 vp_index;
e83d5887 524 u64 hv_vapic;
9eec50b8 525 s64 runtime_offset;
5c919412 526 struct kvm_vcpu_hv_synic synic;
db397571 527 struct kvm_hyperv_exit exit;
1f4b34f8
AS
528 struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
529 DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
e6b6c483 530 cpumask_t tlb_flush;
e83d5887
AS
531};
532
ad312c7c 533struct kvm_vcpu_arch {
5fdbf976
MT
534 /*
535 * rip and regs accesses must go through
536 * kvm_{register,rip}_{read,write} functions.
537 */
538 unsigned long regs[NR_VCPU_REGS];
539 u32 regs_avail;
540 u32 regs_dirty;
34c16eec
ZX
541
542 unsigned long cr0;
e8467fda 543 unsigned long cr0_guest_owned_bits;
34c16eec
ZX
544 unsigned long cr2;
545 unsigned long cr3;
546 unsigned long cr4;
fc78f519 547 unsigned long cr4_guest_owned_bits;
34c16eec 548 unsigned long cr8;
b9dd21e1 549 u32 pkru;
1371d904 550 u32 hflags;
f6801dff 551 u64 efer;
34c16eec
ZX
552 u64 apic_base;
553 struct kvm_lapic *apic; /* kernel irqchip context */
d62caabb 554 bool apicv_active;
e40ff1d6 555 bool load_eoi_exitmap_pending;
6308630b 556 DECLARE_BITMAP(ioapic_handled_vectors, 256);
41383771 557 unsigned long apic_attention;
e1035715 558 int32_t apic_arb_prio;
34c16eec 559 int mp_state;
34c16eec 560 u64 ia32_misc_enable_msr;
64d60670 561 u64 smbase;
52797bf9 562 u64 smi_count;
b209749f 563 bool tpr_access_reporting;
20300099 564 u64 ia32_xss;
518e7b94 565 u64 microcode_version;
34c16eec 566
14dfe855
JR
567 /*
568 * Paging state of the vcpu
569 *
570 * If the vcpu runs in guest mode with two level paging this still saves
571 * the paging mode of the l1 guest. This context is always used to
572 * handle faults.
573 */
44dd3ffa
VK
574 struct kvm_mmu *mmu;
575
576 /* Non-nested MMU for L1 */
577 struct kvm_mmu root_mmu;
8df25a32 578
14c07ad8
VK
579 /* L1 MMU when running nested */
580 struct kvm_mmu guest_mmu;
581
6539e738
JR
582 /*
583 * Paging state of an L2 guest (used for nested npt)
584 *
585 * This context will save all necessary information to walk page tables
586 * of the an L2 guest. This context is only initialized for page table
587 * walking and not for faulting since we never handle l2 page faults on
588 * the host.
589 */
590 struct kvm_mmu nested_mmu;
591
14dfe855
JR
592 /*
593 * Pointer to the mmu context currently used for
594 * gva_to_gpa translations.
595 */
596 struct kvm_mmu *walk_mmu;
597
53c07b18 598 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
34c16eec
ZX
599 struct kvm_mmu_memory_cache mmu_page_cache;
600 struct kvm_mmu_memory_cache mmu_page_header_cache;
601
f775b13e
RR
602 /*
603 * QEMU userspace and the guest each have their own FPU state.
604 * In vcpu_run, we switch between the user and guest FPU contexts.
605 * While running a VCPU, the VCPU thread will have the guest FPU
606 * context.
607 *
608 * Note that while the PKRU state lives inside the fpu registers,
609 * it is switched out separately at VMENTER and VMEXIT time. The
610 * "guest_fpu" state here contains the guest FPU context, with the
611 * host PRKU bits.
612 */
613 struct fpu user_fpu;
98918833 614 struct fpu guest_fpu;
f775b13e 615
2acf923e 616 u64 xcr0;
d7876f1b 617 u64 guest_supported_xcr0;
4344ee98 618 u32 guest_xstate_size;
34c16eec 619
34c16eec
ZX
620 struct kvm_pio_request pio;
621 void *pio_data;
622
66fd3f7f
GN
623 u8 event_exit_inst_len;
624
298101da
AK
625 struct kvm_queued_exception {
626 bool pending;
664f8e26 627 bool injected;
298101da
AK
628 bool has_error_code;
629 u8 nr;
630 u32 error_code;
c851436a
JM
631 unsigned long payload;
632 bool has_payload;
adfe20fb 633 u8 nested_apf;
298101da
AK
634 } exception;
635
937a7eae 636 struct kvm_queued_interrupt {
04140b41 637 bool injected;
66fd3f7f 638 bool soft;
937a7eae
AK
639 u8 nr;
640 } interrupt;
641
34c16eec
ZX
642 int halt_request; /* real mode on Intel only */
643
644 int cpuid_nent;
07716717 645 struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES];
5a4f55cd
EK
646
647 int maxphyaddr;
648
34c16eec
ZX
649 /* emulate context */
650
651 struct x86_emulate_ctxt emulate_ctxt;
7ae441ea
GN
652 bool emulate_regs_need_sync_to_vcpu;
653 bool emulate_regs_need_sync_from_vcpu;
716d51ab 654 int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
18068523
GOC
655
656 gpa_t time;
50d0a0f9 657 struct pvclock_vcpu_time_info hv_clock;
e48672fa 658 unsigned int hw_tsc_khz;
0b79459b
AH
659 struct gfn_to_hva_cache pv_time;
660 bool pv_time_enabled;
51d59c6b
MT
661 /* set guest stopped flag in pvclock flags field */
662 bool pvclock_set_guest_stopped_request;
c9aaa895
GC
663
664 struct {
665 u64 msr_val;
666 u64 last_steal;
c9aaa895
GC
667 struct gfn_to_hva_cache stime;
668 struct kvm_steal_time steal;
669 } st;
670
a545ab6a 671 u64 tsc_offset;
1d5f066e 672 u64 last_guest_tsc;
6f526ec5 673 u64 last_host_tsc;
0dd6a6ed 674 u64 tsc_offset_adjustment;
e26101b1
ZA
675 u64 this_tsc_nsec;
676 u64 this_tsc_write;
0d3da0d2 677 u64 this_tsc_generation;
c285545f 678 bool tsc_catchup;
cc578287
ZA
679 bool tsc_always_catchup;
680 s8 virtual_tsc_shift;
681 u32 virtual_tsc_mult;
682 u32 virtual_tsc_khz;
ba904635 683 s64 ia32_tsc_adjust_msr;
ad721883 684 u64 tsc_scaling_ratio;
3419ffc8 685
7460fb4a
AK
686 atomic_t nmi_queued; /* unprocessed asynchronous NMIs */
687 unsigned nmi_pending; /* NMI queued after currently running handler */
688 bool nmi_injected; /* Trying to inject an NMI this entry */
f077825a 689 bool smi_pending; /* SMI queued after currently running handler */
9ba075a6 690
70109e7d 691 struct kvm_mtrr mtrr_state;
7cb060a9 692 u64 pat;
42dbaa5a 693
360b948d 694 unsigned switch_db_regs;
42dbaa5a
JK
695 unsigned long db[KVM_NR_DB_REGS];
696 unsigned long dr6;
697 unsigned long dr7;
698 unsigned long eff_db[KVM_NR_DB_REGS];
c8639010 699 unsigned long guest_debug_dr7;
db2336a8
KH
700 u64 msr_platform_info;
701 u64 msr_misc_features_enables;
890ca9ae
HY
702
703 u64 mcg_cap;
704 u64 mcg_status;
705 u64 mcg_ctl;
c45dcc71 706 u64 mcg_ext_ctl;
890ca9ae 707 u64 *mce_banks;
94fe45da 708
bebb106a
XG
709 /* Cache MMIO info */
710 u64 mmio_gva;
711 unsigned access;
712 gfn_t mmio_gfn;
56f17dd3 713 u64 mmio_gen;
bebb106a 714
f5132b01
GN
715 struct kvm_pmu pmu;
716
94fe45da 717 /* used for guest single stepping over the given code position */
94fe45da 718 unsigned long singlestep_rip;
f92653ee 719
e83d5887 720 struct kvm_vcpu_hv hyperv;
f5f48ee1
SY
721
722 cpumask_var_t wbinvd_dirty_mask;
af585b92 723
1cb3f3ae
XG
724 unsigned long last_retry_eip;
725 unsigned long last_retry_addr;
726
af585b92
GN
727 struct {
728 bool halted;
729 gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
344d9588
GN
730 struct gfn_to_hva_cache data;
731 u64 msr_val;
7c90705b 732 u32 id;
6adba527 733 bool send_user_only;
1261bfa3 734 u32 host_apf_reason;
adfe20fb 735 unsigned long nested_apf_token;
52a5c155 736 bool delivery_as_pf_vmexit;
af585b92 737 } apf;
2b036c6b
BO
738
739 /* OSVW MSRs (AMD only) */
740 struct {
741 u64 length;
742 u64 status;
743 } osvw;
ae7a2a3f
MT
744
745 struct {
746 u64 msr_val;
747 struct gfn_to_hva_cache data;
748 } pv_eoi;
93c05d3e
XG
749
750 /*
751 * Indicate whether the access faults on its page table in guest
752 * which is set when fix page fault and used to detect unhandeable
753 * instruction.
754 */
755 bool write_fault_to_shadow_pgtable;
25d92081
YZ
756
757 /* set at EPT violation at this point */
758 unsigned long exit_qualification;
6aef266c
SV
759
760 /* pv related host specific info */
761 struct {
762 bool pv_unhalted;
763 } pv;
7543a635
SR
764
765 int pending_ioapic_eoi;
1c1a9ce9 766 int pending_external_vector;
0f89b207 767
618232e2 768 /* GPA available */
0f89b207 769 bool gpa_available;
618232e2 770 gpa_t gpa_val;
de63ad4c
LM
771
772 /* be preempted when it's in kernel-mode(cpl=0) */
773 bool preempted_in_kernel;
c595ceee
PB
774
775 /* Flush the L1 Data cache for L1TF mitigation on VMENTER */
776 bool l1tf_flush_l1d;
34c16eec
ZX
777};
778
db3fe4eb 779struct kvm_lpage_info {
92f94f1e 780 int disallow_lpage;
db3fe4eb
TY
781};
782
783struct kvm_arch_memory_slot {
018aabb5 784 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
db3fe4eb 785 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
21ebbeda 786 unsigned short *gfn_track[KVM_PAGE_TRACK_MAX];
db3fe4eb
TY
787};
788
3548a259
RK
789/*
790 * We use as the mode the number of bits allocated in the LDR for the
791 * logical processor ID. It happens that these are all powers of two.
792 * This makes it is very easy to detect cases where the APICs are
793 * configured for multiple modes; in that case, we cannot use the map and
794 * hence cannot use kvm_irq_delivery_to_apic_fast either.
795 */
796#define KVM_APIC_MODE_XAPIC_CLUSTER 4
797#define KVM_APIC_MODE_XAPIC_FLAT 8
798#define KVM_APIC_MODE_X2APIC 16
799
1e08ec4a
GN
800struct kvm_apic_map {
801 struct rcu_head rcu;
3548a259 802 u8 mode;
0ca52e7b 803 u32 max_apic_id;
e45115b6
RK
804 union {
805 struct kvm_lapic *xapic_flat_map[8];
806 struct kvm_lapic *xapic_cluster_map[16][4];
807 };
0ca52e7b 808 struct kvm_lapic *phys_map[];
1e08ec4a
GN
809};
810
e83d5887
AS
811/* Hyper-V emulation context */
812struct kvm_hv {
3f5ad8be 813 struct mutex hv_lock;
e83d5887
AS
814 u64 hv_guest_os_id;
815 u64 hv_hypercall;
816 u64 hv_tsc_page;
e7d9513b
AS
817
818 /* Hyper-v based guest crash (NT kernel bugcheck) parameters */
819 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS];
820 u64 hv_crash_ctl;
095cf55d
PB
821
822 HV_REFERENCE_TSC_PAGE tsc_ref;
faeb7833
RK
823
824 struct idr conn_to_evt;
a2e164e7
VK
825
826 u64 hv_reenlightenment_control;
827 u64 hv_tsc_emulation_control;
828 u64 hv_tsc_emulation_status;
87ee613d
VK
829
830 /* How many vCPUs have VP index != vCPU index */
831 atomic_t num_mismatched_vp_indexes;
e83d5887
AS
832};
833
49776faf
RK
834enum kvm_irqchip_mode {
835 KVM_IRQCHIP_NONE,
836 KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */
837 KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */
838};
839
fef9cce0 840struct kvm_arch {
49d5ca26 841 unsigned int n_used_mmu_pages;
f05e70ac 842 unsigned int n_requested_mmu_pages;
39de71ec 843 unsigned int n_max_mmu_pages;
332b207d 844 unsigned int indirect_shadow_pages;
5304b8d3 845 unsigned long mmu_valid_gen;
f05e70ac
ZX
846 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
847 /*
848 * Hash table of struct kvm_mmu_page.
849 */
850 struct list_head active_mmu_pages;
365c8868 851 struct list_head zapped_obsolete_pages;
13d268ca 852 struct kvm_page_track_notifier_node mmu_sp_tracker;
0eb05bf2 853 struct kvm_page_track_notifier_head track_notifier_head;
365c8868 854
4d5c5d0f 855 struct list_head assigned_dev_head;
19de40a8 856 struct iommu_domain *iommu_domain;
d96eb2c6 857 bool iommu_noncoherent;
e0f0bbc5
AW
858#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
859 atomic_t noncoherent_dma_count;
5544eb9b
PB
860#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
861 atomic_t assigned_device_count;
d7deeeb0
ZX
862 struct kvm_pic *vpic;
863 struct kvm_ioapic *vioapic;
7837699f 864 struct kvm_pit *vpit;
42720138 865 atomic_t vapics_in_nmi_mode;
1e08ec4a
GN
866 struct mutex apic_map_lock;
867 struct kvm_apic_map *apic_map;
bfc6d222 868
c24ae0dc 869 bool apic_access_page_done;
18068523
GOC
870
871 gpa_t wall_clock;
b7ebfb05 872
4d5422ce 873 bool mwait_in_guest;
caa057a2 874 bool hlt_in_guest;
b31c114b 875 bool pause_in_guest;
4d5422ce 876
5550af4d 877 unsigned long irq_sources_bitmap;
afbcf7ab 878 s64 kvmclock_offset;
038f8c11 879 raw_spinlock_t tsc_write_lock;
f38e098f 880 u64 last_tsc_nsec;
f38e098f 881 u64 last_tsc_write;
5d3cb0f6 882 u32 last_tsc_khz;
e26101b1
ZA
883 u64 cur_tsc_nsec;
884 u64 cur_tsc_write;
885 u64 cur_tsc_offset;
0d3da0d2 886 u64 cur_tsc_generation;
b48aa97e 887 int nr_vcpus_matched_tsc;
ffde22ac 888
d828199e
MT
889 spinlock_t pvclock_gtod_sync_lock;
890 bool use_master_clock;
891 u64 master_kernel_ns;
a5a1d1c2 892 u64 master_cycle_now;
7e44e449 893 struct delayed_work kvmclock_update_work;
332967a3 894 struct delayed_work kvmclock_sync_work;
d828199e 895
ffde22ac 896 struct kvm_xen_hvm_config xen_hvm_config;
55cd8e5a 897
6ef768fa
PB
898 /* reads protected by irq_srcu, writes by irq_lock */
899 struct hlist_head mask_notifier_list;
900
e83d5887 901 struct kvm_hv hyperv;
b034cf01
XG
902
903 #ifdef CONFIG_KVM_MMU_AUDIT
904 int audit_point;
905 #endif
54750f2c 906
a826faf1 907 bool backwards_tsc_observed;
54750f2c 908 bool boot_vcpu_runs_old_kvmclock;
d71ba788 909 u32 bsp_vcpu_id;
90de4a18
NA
910
911 u64 disabled_quirks;
49df6397 912
49776faf 913 enum kvm_irqchip_mode irqchip_mode;
b053b2ae 914 u8 nr_reserved_ioapic_pins;
52004014
FW
915
916 bool disabled_lapic_found;
44a95dae 917
37131313 918 bool x2apic_format;
c519265f 919 bool x2apic_broadcast_quirk_disabled;
6fbbde9a
DS
920
921 bool guest_can_read_msr_platform_info;
d69fb81f
ZX
922};
923
0711456c 924struct kvm_vm_stat {
8a7e75d4
SJS
925 ulong mmu_shadow_zapped;
926 ulong mmu_pte_write;
927 ulong mmu_pte_updated;
928 ulong mmu_pde_zapped;
929 ulong mmu_flooded;
930 ulong mmu_recycled;
931 ulong mmu_cache_miss;
932 ulong mmu_unsync;
933 ulong remote_tlb_flush;
934 ulong lpages;
f3414bc7 935 ulong max_mmu_page_hash_collisions;
0711456c
ZX
936};
937
77b4c255 938struct kvm_vcpu_stat {
8a7e75d4
SJS
939 u64 pf_fixed;
940 u64 pf_guest;
941 u64 tlb_flush;
942 u64 invlpg;
943
944 u64 exits;
945 u64 io_exits;
946 u64 mmio_exits;
947 u64 signal_exits;
948 u64 irq_window_exits;
949 u64 nmi_window_exits;
c595ceee 950 u64 l1d_flush;
8a7e75d4
SJS
951 u64 halt_exits;
952 u64 halt_successful_poll;
953 u64 halt_attempted_poll;
954 u64 halt_poll_invalid;
955 u64 halt_wakeup;
956 u64 request_irq_exits;
957 u64 irq_exits;
958 u64 host_state_reload;
8a7e75d4
SJS
959 u64 fpu_reload;
960 u64 insn_emulation;
961 u64 insn_emulation_fail;
962 u64 hypercalls;
963 u64 irq_injections;
964 u64 nmi_injections;
0f1e261e 965 u64 req_event;
77b4c255 966};
ad312c7c 967
8a76d7f2
JR
968struct x86_instruction_info;
969
8fe8ab46
WA
970struct msr_data {
971 bool host_initiated;
972 u32 index;
973 u64 data;
974};
975
cb5281a5
PB
976struct kvm_lapic_irq {
977 u32 vector;
b7cb2231
PB
978 u16 delivery_mode;
979 u16 dest_mode;
980 bool level;
981 u16 trig_mode;
cb5281a5
PB
982 u32 shorthand;
983 u32 dest_id;
93bbf0b8 984 bool msi_redir_hint;
cb5281a5
PB
985};
986
ea4a5ff8
ZX
987struct kvm_x86_ops {
988 int (*cpu_has_kvm_support)(void); /* __init */
989 int (*disabled_by_bios)(void); /* __init */
13a34e06
RK
990 int (*hardware_enable)(void);
991 void (*hardware_disable)(void);
ea4a5ff8
ZX
992 void (*check_processor_compatibility)(void *rtn);
993 int (*hardware_setup)(void); /* __init */
994 void (*hardware_unsetup)(void); /* __exit */
774ead3a 995 bool (*cpu_has_accelerated_tpr)(void);
bc226f07 996 bool (*has_emulated_msr)(int index);
0e851880 997 void (*cpuid_update)(struct kvm_vcpu *vcpu);
ea4a5ff8 998
434a1e94
SC
999 struct kvm *(*vm_alloc)(void);
1000 void (*vm_free)(struct kvm *);
03543133
SS
1001 int (*vm_init)(struct kvm *kvm);
1002 void (*vm_destroy)(struct kvm *kvm);
1003
ea4a5ff8
ZX
1004 /* Create, but do not attach this VCPU */
1005 struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id);
1006 void (*vcpu_free)(struct kvm_vcpu *vcpu);
d28bc9dd 1007 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event);
ea4a5ff8
ZX
1008
1009 void (*prepare_guest_switch)(struct kvm_vcpu *vcpu);
1010 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
1011 void (*vcpu_put)(struct kvm_vcpu *vcpu);
ea4a5ff8 1012
a96036b8 1013 void (*update_bp_intercept)(struct kvm_vcpu *vcpu);
609e36d3 1014 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
8fe8ab46 1015 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
ea4a5ff8
ZX
1016 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
1017 void (*get_segment)(struct kvm_vcpu *vcpu,
1018 struct kvm_segment *var, int seg);
2e4d2653 1019 int (*get_cpl)(struct kvm_vcpu *vcpu);
ea4a5ff8
ZX
1020 void (*set_segment)(struct kvm_vcpu *vcpu,
1021 struct kvm_segment *var, int seg);
1022 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
e8467fda 1023 void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
aff48baa 1024 void (*decache_cr3)(struct kvm_vcpu *vcpu);
ea4a5ff8
ZX
1025 void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
1026 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
1027 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
5e1746d6 1028 int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
ea4a5ff8 1029 void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
89a27f4d
GN
1030 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1031 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1032 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1033 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
73aaf249
JK
1034 u64 (*get_dr6)(struct kvm_vcpu *vcpu);
1035 void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value);
c77fb5fe 1036 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
020df079 1037 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
5fdbf976 1038 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
ea4a5ff8
ZX
1039 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
1040 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
1041
c2ba05cc 1042 void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa);
b08660e5 1043 int (*tlb_remote_flush)(struct kvm *kvm);
ea4a5ff8 1044
faff8758
JS
1045 /*
1046 * Flush any TLB entries associated with the given GVA.
1047 * Does not need to flush GPA->HPA mappings.
1048 * Can potentially get non-canonical addresses through INVLPGs, which
1049 * the implementation may choose to ignore if appropriate.
1050 */
1051 void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr);
ea4a5ff8 1052
851ba692
AK
1053 void (*run)(struct kvm_vcpu *vcpu);
1054 int (*handle_exit)(struct kvm_vcpu *vcpu);
ea4a5ff8 1055 void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
2809f5d2 1056 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
37ccdcbe 1057 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
ea4a5ff8
ZX
1058 void (*patch_hypercall)(struct kvm_vcpu *vcpu,
1059 unsigned char *hypercall_addr);
66fd3f7f 1060 void (*set_irq)(struct kvm_vcpu *vcpu);
95ba8273 1061 void (*set_nmi)(struct kvm_vcpu *vcpu);
cfcd20e5 1062 void (*queue_exception)(struct kvm_vcpu *vcpu);
b463a6f7 1063 void (*cancel_injection)(struct kvm_vcpu *vcpu);
78646121 1064 int (*interrupt_allowed)(struct kvm_vcpu *vcpu);
95ba8273 1065 int (*nmi_allowed)(struct kvm_vcpu *vcpu);
3cfc3092
JK
1066 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
1067 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
c9a7953f
JK
1068 void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
1069 void (*enable_irq_window)(struct kvm_vcpu *vcpu);
95ba8273 1070 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
b2a05fef 1071 bool (*get_enable_apicv)(struct kvm_vcpu *vcpu);
d62caabb 1072 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
c7c9c56c 1073 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
67c9dddc 1074 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
e6c67d8c 1075 bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu);
6308630b 1076 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
8d860bbe 1077 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
4256f43f 1078 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
a20ed54d 1079 void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
76dfafd5 1080 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
ea4a5ff8 1081 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
2ac52ab8 1082 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
855feb67 1083 int (*get_tdp_level)(struct kvm_vcpu *vcpu);
4b12f0de 1084 u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
17cc3935 1085 int (*get_lpage_level)(void);
4e47c7a6 1086 bool (*rdtscp_supported)(void);
ad756a16 1087 bool (*invpcid_supported)(void);
344f414f 1088
1c97f0a0
JR
1089 void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
1090
d4330ef2
JR
1091 void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry);
1092
f5f48ee1
SY
1093 bool (*has_wbinvd_exit)(void);
1094
e79f245d 1095 u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu);
99e3e30a
ZA
1096 void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
1097
586f9607 1098 void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
8a76d7f2
JR
1099
1100 int (*check_intercept)(struct kvm_vcpu *vcpu,
1101 struct x86_instruction_info *info,
1102 enum x86_intercept_stage stage);
a547c6db 1103 void (*handle_external_intr)(struct kvm_vcpu *vcpu);
da8999d3 1104 bool (*mpx_supported)(void);
55412b2e 1105 bool (*xsaves_supported)(void);
66336cab 1106 bool (*umip_emulated)(void);
b6b8a145
JK
1107
1108 int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
d264ee0c 1109 void (*request_immediate_exit)(struct kvm_vcpu *vcpu);
ae97a3b8
RK
1110
1111 void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
88178fd4
KH
1112
1113 /*
1114 * Arch-specific dirty logging hooks. These hooks are only supposed to
1115 * be valid if the specific arch has hardware-accelerated dirty logging
1116 * mechanism. Currently only for PML on VMX.
1117 *
1118 * - slot_enable_log_dirty:
1119 * called when enabling log dirty mode for the slot.
1120 * - slot_disable_log_dirty:
1121 * called when disabling log dirty mode for the slot.
1122 * also called when slot is created with log dirty disabled.
1123 * - flush_log_dirty:
1124 * called before reporting dirty_bitmap to userspace.
1125 * - enable_log_dirty_pt_masked:
1126 * called when reenabling log dirty for the GFNs in the mask after
1127 * corresponding bits are cleared in slot->dirty_bitmap.
1128 */
1129 void (*slot_enable_log_dirty)(struct kvm *kvm,
1130 struct kvm_memory_slot *slot);
1131 void (*slot_disable_log_dirty)(struct kvm *kvm,
1132 struct kvm_memory_slot *slot);
1133 void (*flush_log_dirty)(struct kvm *kvm);
1134 void (*enable_log_dirty_pt_masked)(struct kvm *kvm,
1135 struct kvm_memory_slot *slot,
1136 gfn_t offset, unsigned long mask);
bab4165e
BD
1137 int (*write_log_dirty)(struct kvm_vcpu *vcpu);
1138
25462f7f
WH
1139 /* pmu operations of sub-arch */
1140 const struct kvm_pmu_ops *pmu_ops;
efc64404 1141
bf9f6ac8
FW
1142 /*
1143 * Architecture specific hooks for vCPU blocking due to
1144 * HLT instruction.
1145 * Returns for .pre_block():
1146 * - 0 means continue to block the vCPU.
1147 * - 1 means we cannot block the vCPU since some event
1148 * happens during this period, such as, 'ON' bit in
1149 * posted-interrupts descriptor is set.
1150 */
1151 int (*pre_block)(struct kvm_vcpu *vcpu);
1152 void (*post_block)(struct kvm_vcpu *vcpu);
d1ed092f
SS
1153
1154 void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
1155 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
1156
efc64404
FW
1157 int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq,
1158 uint32_t guest_irq, bool set);
be8ca170 1159 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
ce7a058a
YJ
1160
1161 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc);
1162 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu);
c45dcc71
AR
1163
1164 void (*setup_mce)(struct kvm_vcpu *vcpu);
0234bf88 1165
8fcc4b59
JM
1166 int (*get_nested_state)(struct kvm_vcpu *vcpu,
1167 struct kvm_nested_state __user *user_kvm_nested_state,
1168 unsigned user_data_size);
1169 int (*set_nested_state)(struct kvm_vcpu *vcpu,
1170 struct kvm_nested_state __user *user_kvm_nested_state,
1171 struct kvm_nested_state *kvm_state);
7f7f1ba3
PB
1172 void (*get_vmcs12_pages)(struct kvm_vcpu *vcpu);
1173
72d7b374 1174 int (*smi_allowed)(struct kvm_vcpu *vcpu);
0234bf88
LP
1175 int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate);
1176 int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase);
cc3d967f 1177 int (*enable_smi_window)(struct kvm_vcpu *vcpu);
5acc5c06
BS
1178
1179 int (*mem_enc_op)(struct kvm *kvm, void __user *argp);
69eaedee
BS
1180 int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
1181 int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
801e459a
TL
1182
1183 int (*get_msr_feature)(struct kvm_msr_entry *entry);
57b119da
VK
1184
1185 int (*nested_enable_evmcs)(struct kvm_vcpu *vcpu,
1186 uint16_t *vmcs_version);
ea4a5ff8
ZX
1187};
1188
af585b92 1189struct kvm_arch_async_pf {
7c90705b 1190 u32 token;
af585b92 1191 gfn_t gfn;
fb67e14f 1192 unsigned long cr3;
c4806acd 1193 bool direct_map;
af585b92
GN
1194};
1195
97896d04
ZX
1196extern struct kvm_x86_ops *kvm_x86_ops;
1197
434a1e94
SC
1198#define __KVM_HAVE_ARCH_VM_ALLOC
1199static inline struct kvm *kvm_arch_alloc_vm(void)
1200{
1201 return kvm_x86_ops->vm_alloc();
1202}
1203
1204static inline void kvm_arch_free_vm(struct kvm *kvm)
1205{
1206 return kvm_x86_ops->vm_free(kvm);
1207}
1208
b08660e5
TL
1209#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
1210static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
1211{
1212 if (kvm_x86_ops->tlb_remote_flush &&
1213 !kvm_x86_ops->tlb_remote_flush(kvm))
1214 return 0;
1215 else
1216 return -ENOTSUPP;
1217}
1218
54f1585a
ZX
1219int kvm_mmu_module_init(void);
1220void kvm_mmu_module_exit(void);
1221
1222void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1223int kvm_mmu_create(struct kvm_vcpu *vcpu);
13d268ca
XG
1224void kvm_mmu_init_vm(struct kvm *kvm);
1225void kvm_mmu_uninit_vm(struct kvm *kvm);
7b52345e 1226void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
f160c7b7 1227 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask,
d0ec49d4 1228 u64 acc_track_mask, u64 me_mask);
54f1585a 1229
8a3c1a33 1230void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
1c91cad4
KH
1231void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
1232 struct kvm_memory_slot *memslot);
3ea3b7fa 1233void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
f36f3f28 1234 const struct kvm_memory_slot *memslot);
f4b4b180
KH
1235void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
1236 struct kvm_memory_slot *memslot);
1237void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
1238 struct kvm_memory_slot *memslot);
1239void kvm_mmu_slot_set_dirty(struct kvm *kvm,
1240 struct kvm_memory_slot *memslot);
1241void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1242 struct kvm_memory_slot *slot,
1243 gfn_t gfn_offset, unsigned long mask);
54f1585a 1244void kvm_mmu_zap_all(struct kvm *kvm);
54bf36aa 1245void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots);
3ad82a7e 1246unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
54f1585a
ZX
1247void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages);
1248
ff03a073 1249int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3);
9ed38ffa 1250bool pdptrs_changed(struct kvm_vcpu *vcpu);
cc4b6871 1251
3200f405 1252int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
9f811285 1253 const void *val, int bytes);
2f333bcb 1254
6ef768fa
PB
1255struct kvm_irq_mask_notifier {
1256 void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
1257 int irq;
1258 struct hlist_node link;
1259};
1260
1261void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
1262 struct kvm_irq_mask_notifier *kimn);
1263void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
1264 struct kvm_irq_mask_notifier *kimn);
1265void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
1266 bool mask);
1267
2f333bcb 1268extern bool tdp_enabled;
9f811285 1269
a3e06bbe
LJ
1270u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
1271
92a1f12d
JR
1272/* control of guest tsc rate supported? */
1273extern bool kvm_has_tsc_control;
92a1f12d
JR
1274/* maximum supported tsc_khz for guests */
1275extern u32 kvm_max_guest_tsc_khz;
bc9b961b
HZ
1276/* number of bits of the fractional part of the TSC scaling ratio */
1277extern u8 kvm_tsc_scaling_ratio_frac_bits;
1278/* maximum allowed value of TSC scaling ratio */
1279extern u64 kvm_max_tsc_scaling_ratio;
64672c95
YJ
1280/* 1ull << kvm_tsc_scaling_ratio_frac_bits */
1281extern u64 kvm_default_tsc_scaling_ratio;
92a1f12d 1282
c45dcc71 1283extern u64 kvm_mce_cap_supported;
92a1f12d 1284
54f1585a 1285enum emulation_result {
ac0a48c3
PB
1286 EMULATE_DONE, /* no further processing */
1287 EMULATE_USER_EXIT, /* kvm_run ready for userspace exit */
54f1585a
ZX
1288 EMULATE_FAIL, /* can't emulate this instruction */
1289};
1290
571008da
SY
1291#define EMULTYPE_NO_DECODE (1 << 0)
1292#define EMULTYPE_TRAP_UD (1 << 1)
ba8afb6b 1293#define EMULTYPE_SKIP (1 << 2)
384bf221
SC
1294#define EMULTYPE_ALLOW_RETRY (1 << 3)
1295#define EMULTYPE_NO_UD_ON_FAIL (1 << 4)
1296#define EMULTYPE_VMWARE (1 << 5)
c60658d1
SC
1297int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
1298int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
1299 void *insn, int insn_len);
35be0ade 1300
f2b4b7dd 1301void kvm_enable_efer_bits(u64);
384bb783 1302bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
609e36d3 1303int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
8fe8ab46 1304int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
54f1585a
ZX
1305
1306struct x86_emulate_ctxt;
1307
dca7f128 1308int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
6a908b62 1309int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
54f1585a 1310int kvm_emulate_halt(struct kvm_vcpu *vcpu);
5cb56059 1311int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
f5f48ee1 1312int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
54f1585a 1313
3e6e0aab 1314void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
c697518a 1315int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
2b4a273b 1316void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
3e6e0aab 1317
7f3d35fd
KW
1318int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
1319 int reason, bool has_error_code, u32 error_code);
37817f29 1320
49a9b07e 1321int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
2390218b 1322int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
a83b29c6 1323int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
eea1cff9 1324int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
020df079
GN
1325int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
1326int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
2d3ad1f4
AK
1327unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
1328void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
54f1585a 1329void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
2acf923e 1330int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
54f1585a 1331
609e36d3 1332int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
8fe8ab46 1333int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
54f1585a 1334
91586a3b
JK
1335unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
1336void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
022cd0e8 1337bool kvm_rdpmc(struct kvm_vcpu *vcpu);
91586a3b 1338
298101da
AK
1339void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1340void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
ce7ddec4
JR
1341void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1342void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
6389ee94 1343void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
ec92fe44
JR
1344int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1345 gfn_t gfn, void *data, int offset, int len,
1346 u32 access);
0a79b009 1347bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
16f8a6f9 1348bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
298101da 1349
1a577b72
MT
1350static inline int __kvm_irq_line_state(unsigned long *irq_state,
1351 int irq_source_id, int level)
1352{
1353 /* Logical OR for level trig interrupt */
1354 if (level)
1355 __set_bit(irq_source_id, irq_state);
1356 else
1357 __clear_bit(irq_source_id, irq_state);
1358
1359 return !!(*irq_state);
1360}
1361
b94742c9
JS
1362#define KVM_MMU_ROOT_CURRENT BIT(0)
1363#define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i)
1364#define KVM_MMU_ROOTS_ALL (~0UL)
08fb59d8 1365
1a577b72
MT
1366int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
1367void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
3de42dc0 1368
3419ffc8
SY
1369void kvm_inject_nmi(struct kvm_vcpu *vcpu);
1370
1cb3f3ae 1371int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn);
54f1585a
ZX
1372int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva);
1373void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
1374int kvm_mmu_load(struct kvm_vcpu *vcpu);
1375void kvm_mmu_unload(struct kvm_vcpu *vcpu);
0ba73cda 1376void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
6a82cd1c
VK
1377void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1378 ulong roots_to_free);
54987b7a
PB
1379gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1380 struct x86_exception *exception);
ab9ae313
AK
1381gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
1382 struct x86_exception *exception);
1383gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
1384 struct x86_exception *exception);
1385gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
1386 struct x86_exception *exception);
1387gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
1388 struct x86_exception *exception);
54f1585a 1389
d62caabb
AS
1390void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu);
1391
54f1585a
ZX
1392int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
1393
14727754 1394int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,
dc25e89e 1395 void *insn, int insn_len);
a7052897 1396void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
eb4b248e 1397void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
ade61e28 1398void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush);
34c16eec 1399
18552672 1400void kvm_enable_tdp(void);
5f4cb662 1401void kvm_disable_tdp(void);
18552672 1402
54987b7a
PB
1403static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1404 struct x86_exception *exception)
e459e322
XG
1405{
1406 return gpa;
1407}
1408
ec6d273d
ZX
1409static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
1410{
1411 struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT);
1412
1413 return (struct kvm_mmu_page *)page_private(page);
1414}
1415
d6e88aec 1416static inline u16 kvm_read_ldt(void)
ec6d273d
ZX
1417{
1418 u16 ldt;
1419 asm("sldt %0" : "=g"(ldt));
1420 return ldt;
1421}
1422
d6e88aec 1423static inline void kvm_load_ldt(u16 sel)
ec6d273d
ZX
1424{
1425 asm("lldt %0" : : "rm"(sel));
1426}
ec6d273d 1427
ec6d273d
ZX
1428#ifdef CONFIG_X86_64
1429static inline unsigned long read_msr(unsigned long msr)
1430{
1431 u64 value;
1432
1433 rdmsrl(msr, value);
1434 return value;
1435}
1436#endif
1437
ec6d273d
ZX
1438static inline u32 get_rdx_init_val(void)
1439{
1440 return 0x600; /* P6 family */
1441}
1442
c1a5d4f9
AK
1443static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
1444{
1445 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1446}
1447
ec6d273d
ZX
1448#define TSS_IOPB_BASE_OFFSET 0x66
1449#define TSS_BASE_SIZE 0x68
1450#define TSS_IOPB_SIZE (65536 / 8)
1451#define TSS_REDIRECTION_SIZE (256 / 8)
7d76b4d3
JP
1452#define RMODE_TSS_SIZE \
1453 (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
53e0aa7b 1454
37817f29
IE
1455enum {
1456 TASK_SWITCH_CALL = 0,
1457 TASK_SWITCH_IRET = 1,
1458 TASK_SWITCH_JMP = 2,
1459 TASK_SWITCH_GATE = 3,
1460};
1461
1371d904 1462#define HF_GIF_MASK (1 << 0)
3d6368ef
AG
1463#define HF_HIF_MASK (1 << 1)
1464#define HF_VINTR_MASK (1 << 2)
95ba8273 1465#define HF_NMI_MASK (1 << 3)
44c11430 1466#define HF_IRET_MASK (1 << 4)
ec9e60b2 1467#define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */
f077825a
PB
1468#define HF_SMM_MASK (1 << 6)
1469#define HF_SMM_INSIDE_NMI_MASK (1 << 7)
1371d904 1470
699023e2
PB
1471#define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
1472#define KVM_ADDRESS_SPACE_NUM 2
1473
1474#define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
1475#define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
1371d904 1476
4ecac3fd
AK
1477/*
1478 * Hardware virtualization extension instructions may fault if a
1479 * reboot turns off virtualization while processes are running.
1480 * Trap the fault and ignore the instruction if that happens.
1481 */
b7c4145b 1482asmlinkage void kvm_spurious_fault(void);
4ecac3fd 1483
5e520e62 1484#define ____kvm_handle_fault_on_reboot(insn, cleanup_insn) \
4ecac3fd 1485 "666: " insn "\n\t" \
b7c4145b 1486 "668: \n\t" \
18b13e54 1487 ".pushsection .fixup, \"ax\" \n" \
4ecac3fd 1488 "667: \n\t" \
5e520e62 1489 cleanup_insn "\n\t" \
b7c4145b
AK
1490 "cmpb $0, kvm_rebooting \n\t" \
1491 "jne 668b \n\t" \
8ceed347 1492 __ASM_SIZE(push) " $666b \n\t" \
b7c4145b 1493 "call kvm_spurious_fault \n\t" \
4ecac3fd 1494 ".popsection \n\t" \
3ee89722 1495 _ASM_EXTABLE(666b, 667b)
4ecac3fd 1496
5e520e62
AK
1497#define __kvm_handle_fault_on_reboot(insn) \
1498 ____kvm_handle_fault_on_reboot(insn, "")
1499
e930bffe 1500#define KVM_ARCH_WANT_MMU_NOTIFIER
b3ae2096 1501int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
57128468 1502int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
8ee53820 1503int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
3da0dd43 1504void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
c7c9c56c 1505int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
a1b37100
GN
1506int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1507int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
0b71785d 1508int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
d28bc9dd 1509void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
4256f43f 1510void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
e930bffe 1511
4180bf1b 1512int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
bdf7ffc8 1513 unsigned long ipi_bitmap_high, u32 min,
4180bf1b
WL
1514 unsigned long icr, int op_64_bit);
1515
5b76a3cf 1516u64 kvm_get_arch_capabilities(void);
18863bdd 1517void kvm_define_shared_msr(unsigned index, u32 msr);
8b3c3104 1518int kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
18863bdd 1519
35181e86 1520u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc);
4ba76538 1521u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
35181e86 1522
82b32774 1523unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
f92653ee
JK
1524bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
1525
2860c4b1
PB
1526void kvm_make_mclock_inprogress_request(struct kvm *kvm);
1527void kvm_make_scan_ioapic_request(struct kvm *kvm);
1528
af585b92
GN
1529void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1530 struct kvm_async_pf *work);
1531void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1532 struct kvm_async_pf *work);
56028d08
GN
1533void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1534 struct kvm_async_pf *work);
7c90705b 1535bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
af585b92
GN
1536extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1537
6affcbed
KH
1538int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
1539int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
d264ee0c 1540void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
db8fcefa 1541
f5132b01
GN
1542int kvm_is_in_guest(void);
1543
1d8007bd
PB
1544int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
1545int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size);
d71ba788
PB
1546bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
1547bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
f5132b01 1548
8feb4a04
FW
1549bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
1550 struct kvm_vcpu **dest_vcpu);
1551
37131313 1552void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
d84f1e07 1553 struct kvm_lapic_irq *irq);
197a4f4b 1554
d1ed092f
SS
1555static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1556{
1557 if (kvm_x86_ops->vcpu_blocking)
1558 kvm_x86_ops->vcpu_blocking(vcpu);
1559}
1560
1561static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
1562{
1563 if (kvm_x86_ops->vcpu_unblocking)
1564 kvm_x86_ops->vcpu_unblocking(vcpu);
1565}
1566
3491caf2 1567static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
3217f7c2 1568
7d669f50
SS
1569static inline int kvm_cpu_get_apicid(int mps_cpu)
1570{
1571#ifdef CONFIG_X86_LOCAL_APIC
64063505 1572 return default_cpu_present_to_apicid(mps_cpu);
7d669f50
SS
1573#else
1574 WARN_ON_ONCE(1);
1575 return BAD_APICID;
1576#endif
1577}
1578
05cade71
LP
1579#define put_smstate(type, buf, offset, val) \
1580 *(type *)((buf) + (offset) - 0x7e00) = val
1581
1965aae3 1582#endif /* _ASM_X86_KVM_HOST_H */