2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
9 * Copyright (C) 2006 Qumranet, Inc.
10 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
22 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
23 * so the code in this file is compiled twice, once per pte size.
27 #define pt_element_t u64
28 #define guest_walker guest_walker64
29 #define FNAME(name) paging##64_##name
30 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
31 #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
32 #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
33 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
34 #define PT_LEVEL_BITS PT64_LEVEL_BITS
35 #define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT
36 #define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT
37 #define PT_HAVE_ACCESSED_DIRTY(mmu) true
39 #define PT_MAX_FULL_LEVELS 4
40 #define CMPXCHG cmpxchg
42 #define CMPXCHG cmpxchg64
43 #define PT_MAX_FULL_LEVELS 2
46 #define pt_element_t u32
47 #define guest_walker guest_walker32
48 #define FNAME(name) paging##32_##name
49 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
50 #define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
51 #define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
52 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
53 #define PT_LEVEL_BITS PT32_LEVEL_BITS
54 #define PT_MAX_FULL_LEVELS 2
55 #define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT
56 #define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT
57 #define PT_HAVE_ACCESSED_DIRTY(mmu) true
58 #define CMPXCHG cmpxchg
59 #elif PTTYPE == PTTYPE_EPT
60 #define pt_element_t u64
61 #define guest_walker guest_walkerEPT
62 #define FNAME(name) ept_##name
63 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
64 #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
65 #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
66 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
67 #define PT_LEVEL_BITS PT64_LEVEL_BITS
68 #define PT_GUEST_DIRTY_SHIFT 9
69 #define PT_GUEST_ACCESSED_SHIFT 8
70 #define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad)
71 #define CMPXCHG cmpxchg64
72 #define PT_MAX_FULL_LEVELS 4
74 #error Invalid PTTYPE value
77 #define PT_GUEST_DIRTY_MASK (1 << PT_GUEST_DIRTY_SHIFT)
78 #define PT_GUEST_ACCESSED_MASK (1 << PT_GUEST_ACCESSED_SHIFT)
80 #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
81 #define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
84 * The guest_walker structure emulates the behavior of the hardware page
90 gfn_t table_gfn
[PT_MAX_FULL_LEVELS
];
91 pt_element_t ptes
[PT_MAX_FULL_LEVELS
];
92 pt_element_t prefetch_ptes
[PTE_PREFETCH_NUM
];
93 gpa_t pte_gpa
[PT_MAX_FULL_LEVELS
];
94 pt_element_t __user
*ptep_user
[PT_MAX_FULL_LEVELS
];
95 bool pte_writable
[PT_MAX_FULL_LEVELS
];
99 struct x86_exception fault
;
102 static gfn_t
gpte_to_gfn_lvl(pt_element_t gpte
, int lvl
)
104 return (gpte
& PT_LVL_ADDR_MASK(lvl
)) >> PAGE_SHIFT
;
107 static inline void FNAME(protect_clean_gpte
)(struct kvm_mmu
*mmu
, unsigned *access
,
112 /* dirty bit is not supported, so no need to track it */
113 if (!PT_HAVE_ACCESSED_DIRTY(mmu
))
116 BUILD_BUG_ON(PT_WRITABLE_MASK
!= ACC_WRITE_MASK
);
118 mask
= (unsigned)~ACC_WRITE_MASK
;
119 /* Allow write access to dirty gptes */
120 mask
|= (gpte
>> (PT_GUEST_DIRTY_SHIFT
- PT_WRITABLE_SHIFT
)) &
125 static inline int FNAME(is_present_gpte
)(unsigned long pte
)
127 #if PTTYPE != PTTYPE_EPT
128 return pte
& PT_PRESENT_MASK
;
134 static int FNAME(cmpxchg_gpte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
135 pt_element_t __user
*ptep_user
, unsigned index
,
136 pt_element_t orig_pte
, pt_element_t new_pte
)
143 npages
= get_user_pages_fast((unsigned long)ptep_user
, 1, 1, &page
);
144 /* Check if the user is doing something meaningless. */
145 if (unlikely(npages
!= 1))
148 table
= kmap_atomic(page
);
149 ret
= CMPXCHG(&table
[index
], orig_pte
, new_pte
);
150 kunmap_atomic(table
);
152 kvm_release_page_dirty(page
);
154 return (ret
!= orig_pte
);
157 static bool FNAME(prefetch_invalid_gpte
)(struct kvm_vcpu
*vcpu
,
158 struct kvm_mmu_page
*sp
, u64
*spte
,
161 if (is_rsvd_bits_set(&vcpu
->arch
.mmu
, gpte
, PT_PAGE_TABLE_LEVEL
))
164 if (!FNAME(is_present_gpte
)(gpte
))
167 /* if accessed bit is not supported prefetch non accessed gpte */
168 if (PT_HAVE_ACCESSED_DIRTY(&vcpu
->arch
.mmu
) && !(gpte
& PT_GUEST_ACCESSED_MASK
))
174 drop_spte(vcpu
->kvm
, spte
);
179 * For PTTYPE_EPT, a page table can be executable but not readable
180 * on supported processors. Therefore, set_spte does not automatically
181 * set bit 0 if execute only is supported. Here, we repurpose ACC_USER_MASK
182 * to signify readability since it isn't used in the EPT case
184 static inline unsigned FNAME(gpte_access
)(u64 gpte
)
187 #if PTTYPE == PTTYPE_EPT
188 access
= ((gpte
& VMX_EPT_WRITABLE_MASK
) ? ACC_WRITE_MASK
: 0) |
189 ((gpte
& VMX_EPT_EXECUTABLE_MASK
) ? ACC_EXEC_MASK
: 0) |
190 ((gpte
& VMX_EPT_READABLE_MASK
) ? ACC_USER_MASK
: 0);
192 BUILD_BUG_ON(ACC_EXEC_MASK
!= PT_PRESENT_MASK
);
193 BUILD_BUG_ON(ACC_EXEC_MASK
!= 1);
194 access
= gpte
& (PT_WRITABLE_MASK
| PT_USER_MASK
| PT_PRESENT_MASK
);
195 /* Combine NX with P (which is set here) to get ACC_EXEC_MASK. */
196 access
^= (gpte
>> PT64_NX_SHIFT
);
202 static int FNAME(update_accessed_dirty_bits
)(struct kvm_vcpu
*vcpu
,
204 struct guest_walker
*walker
,
207 unsigned level
, index
;
208 pt_element_t pte
, orig_pte
;
209 pt_element_t __user
*ptep_user
;
213 /* dirty/accessed bits are not supported, so no need to update them */
214 if (!PT_HAVE_ACCESSED_DIRTY(mmu
))
217 for (level
= walker
->max_level
; level
>= walker
->level
; --level
) {
218 pte
= orig_pte
= walker
->ptes
[level
- 1];
219 table_gfn
= walker
->table_gfn
[level
- 1];
220 ptep_user
= walker
->ptep_user
[level
- 1];
221 index
= offset_in_page(ptep_user
) / sizeof(pt_element_t
);
222 if (!(pte
& PT_GUEST_ACCESSED_MASK
)) {
223 trace_kvm_mmu_set_accessed_bit(table_gfn
, index
, sizeof(pte
));
224 pte
|= PT_GUEST_ACCESSED_MASK
;
226 if (level
== walker
->level
&& write_fault
&&
227 !(pte
& PT_GUEST_DIRTY_MASK
)) {
228 trace_kvm_mmu_set_dirty_bit(table_gfn
, index
, sizeof(pte
));
229 #if PTTYPE == PTTYPE_EPT
230 if (kvm_arch_write_log_dirty(vcpu
))
233 pte
|= PT_GUEST_DIRTY_MASK
;
239 * If the slot is read-only, simply do not process the accessed
240 * and dirty bits. This is the correct thing to do if the slot
241 * is ROM, and page tables in read-as-ROM/write-as-MMIO slots
242 * are only supported if the accessed and dirty bits are already
243 * set in the ROM (so that MMIO writes are never needed).
245 * Note that NPT does not allow this at all and faults, since
246 * it always wants nested page table entries for the guest
247 * page tables to be writable. And EPT works but will simply
248 * overwrite the read-only memory to set the accessed and dirty
251 if (unlikely(!walker
->pte_writable
[level
- 1]))
254 ret
= FNAME(cmpxchg_gpte
)(vcpu
, mmu
, ptep_user
, index
, orig_pte
, pte
);
258 kvm_vcpu_mark_page_dirty(vcpu
, table_gfn
);
259 walker
->ptes
[level
- 1] = pte
;
264 static inline unsigned FNAME(gpte_pkeys
)(struct kvm_vcpu
*vcpu
, u64 gpte
)
268 pte_t pte
= {.pte
= gpte
};
270 pkeys
= pte_flags_pkey(pte_flags(pte
));
276 * Fetch a guest pte for a guest virtual address, or for an L2's GPA.
278 static int FNAME(walk_addr_generic
)(struct guest_walker
*walker
,
279 struct kvm_vcpu
*vcpu
, struct kvm_mmu
*mmu
,
280 gpa_t addr
, u32 access
)
284 pt_element_t __user
*uninitialized_var(ptep_user
);
286 u64 pt_access
, pte_access
;
287 unsigned index
, accessed_dirty
, pte_pkey
;
288 unsigned nested_access
;
292 u64 walk_nx_mask
= 0;
293 const int write_fault
= access
& PFERR_WRITE_MASK
;
294 const int user_fault
= access
& PFERR_USER_MASK
;
295 const int fetch_fault
= access
& PFERR_FETCH_MASK
;
300 trace_kvm_mmu_pagetable_walk(addr
, access
);
302 walker
->level
= mmu
->root_level
;
303 pte
= mmu
->get_cr3(vcpu
);
304 have_ad
= PT_HAVE_ACCESSED_DIRTY(mmu
);
307 walk_nx_mask
= 1ULL << PT64_NX_SHIFT
;
308 if (walker
->level
== PT32E_ROOT_LEVEL
) {
309 pte
= mmu
->get_pdptr(vcpu
, (addr
>> 30) & 3);
310 trace_kvm_mmu_paging_element(pte
, walker
->level
);
311 if (!FNAME(is_present_gpte
)(pte
))
316 walker
->max_level
= walker
->level
;
317 ASSERT(!(is_long_mode(vcpu
) && !is_pae(vcpu
)));
320 * FIXME: on Intel processors, loads of the PDPTE registers for PAE paging
321 * by the MOV to CR instruction are treated as reads and do not cause the
322 * processor to set the dirty flag in any EPT paging-structure entry.
324 nested_access
= (have_ad
? PFERR_WRITE_MASK
: 0) | PFERR_USER_MASK
;
331 unsigned long host_addr
;
333 pt_access
= pte_access
;
336 index
= PT_INDEX(addr
, walker
->level
);
337 table_gfn
= gpte_to_gfn(pte
);
338 offset
= index
* sizeof(pt_element_t
);
339 pte_gpa
= gfn_to_gpa(table_gfn
) + offset
;
341 BUG_ON(walker
->level
< 1);
342 walker
->table_gfn
[walker
->level
- 1] = table_gfn
;
343 walker
->pte_gpa
[walker
->level
- 1] = pte_gpa
;
345 real_gfn
= mmu
->translate_gpa(vcpu
, gfn_to_gpa(table_gfn
),
350 * FIXME: This can happen if emulation (for of an INS/OUTS
351 * instruction) triggers a nested page fault. The exit
352 * qualification / exit info field will incorrectly have
353 * "guest page access" as the nested page fault's cause,
354 * instead of "guest page structure access". To fix this,
355 * the x86_exception struct should be augmented with enough
356 * information to fix the exit_qualification or exit_info_1
359 if (unlikely(real_gfn
== UNMAPPED_GVA
))
362 real_gfn
= gpa_to_gfn(real_gfn
);
364 host_addr
= kvm_vcpu_gfn_to_hva_prot(vcpu
, real_gfn
,
365 &walker
->pte_writable
[walker
->level
- 1]);
366 if (unlikely(kvm_is_error_hva(host_addr
)))
369 ptep_user
= (pt_element_t __user
*)((void *)host_addr
+ offset
);
370 if (unlikely(__copy_from_user(&pte
, ptep_user
, sizeof(pte
))))
372 walker
->ptep_user
[walker
->level
- 1] = ptep_user
;
374 trace_kvm_mmu_paging_element(pte
, walker
->level
);
377 * Inverting the NX it lets us AND it like other
380 pte_access
= pt_access
& (pte
^ walk_nx_mask
);
382 if (unlikely(!FNAME(is_present_gpte
)(pte
)))
385 if (unlikely(is_rsvd_bits_set(mmu
, pte
, walker
->level
))) {
386 errcode
= PFERR_RSVD_MASK
| PFERR_PRESENT_MASK
;
390 walker
->ptes
[walker
->level
- 1] = pte
;
391 } while (!is_last_gpte(mmu
, walker
->level
, pte
));
393 pte_pkey
= FNAME(gpte_pkeys
)(vcpu
, pte
);
394 accessed_dirty
= have_ad
? pte_access
& PT_GUEST_ACCESSED_MASK
: 0;
396 /* Convert to ACC_*_MASK flags for struct guest_walker. */
397 walker
->pt_access
= FNAME(gpte_access
)(pt_access
^ walk_nx_mask
);
398 walker
->pte_access
= FNAME(gpte_access
)(pte_access
^ walk_nx_mask
);
399 errcode
= permission_fault(vcpu
, mmu
, walker
->pte_access
, pte_pkey
, access
);
400 if (unlikely(errcode
))
403 gfn
= gpte_to_gfn_lvl(pte
, walker
->level
);
404 gfn
+= (addr
& PT_LVL_OFFSET_MASK(walker
->level
)) >> PAGE_SHIFT
;
406 if (PTTYPE
== 32 && walker
->level
== PT_DIRECTORY_LEVEL
&& is_cpuid_PSE36())
407 gfn
+= pse36_gfn_delta(pte
);
409 real_gpa
= mmu
->translate_gpa(vcpu
, gfn_to_gpa(gfn
), access
, &walker
->fault
);
410 if (real_gpa
== UNMAPPED_GVA
)
413 walker
->gfn
= real_gpa
>> PAGE_SHIFT
;
416 FNAME(protect_clean_gpte
)(mmu
, &walker
->pte_access
, pte
);
419 * On a write fault, fold the dirty bit into accessed_dirty.
420 * For modes without A/D bits support accessed_dirty will be
423 accessed_dirty
&= pte
>>
424 (PT_GUEST_DIRTY_SHIFT
- PT_GUEST_ACCESSED_SHIFT
);
426 if (unlikely(!accessed_dirty
)) {
427 ret
= FNAME(update_accessed_dirty_bits
)(vcpu
, mmu
, walker
, write_fault
);
428 if (unlikely(ret
< 0))
434 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
435 __func__
, (u64
)pte
, walker
->pte_access
, walker
->pt_access
);
439 errcode
|= write_fault
| user_fault
;
440 if (fetch_fault
&& (mmu
->nx
||
441 kvm_read_cr4_bits(vcpu
, X86_CR4_SMEP
)))
442 errcode
|= PFERR_FETCH_MASK
;
444 walker
->fault
.vector
= PF_VECTOR
;
445 walker
->fault
.error_code_valid
= true;
446 walker
->fault
.error_code
= errcode
;
448 #if PTTYPE == PTTYPE_EPT
450 * Use PFERR_RSVD_MASK in error_code to to tell if EPT
451 * misconfiguration requires to be injected. The detection is
452 * done by is_rsvd_bits_set() above.
454 * We set up the value of exit_qualification to inject:
455 * [2:0] - Derive from the access bits. The exit_qualification might be
456 * out of date if it is serving an EPT misconfiguration.
457 * [5:3] - Calculated by the page walk of the guest EPT page tables
458 * [7:8] - Derived from [7:8] of real exit_qualification
460 * The other bits are set to 0.
462 if (!(errcode
& PFERR_RSVD_MASK
)) {
463 vcpu
->arch
.exit_qualification
&= 0x180;
465 vcpu
->arch
.exit_qualification
|= EPT_VIOLATION_ACC_WRITE
;
467 vcpu
->arch
.exit_qualification
|= EPT_VIOLATION_ACC_READ
;
469 vcpu
->arch
.exit_qualification
|= EPT_VIOLATION_ACC_INSTR
;
470 vcpu
->arch
.exit_qualification
|= (pte_access
& 0x7) << 3;
473 walker
->fault
.address
= addr
;
474 walker
->fault
.nested_page_fault
= mmu
!= vcpu
->arch
.walk_mmu
;
476 trace_kvm_mmu_walker_error(walker
->fault
.error_code
);
480 static int FNAME(walk_addr
)(struct guest_walker
*walker
,
481 struct kvm_vcpu
*vcpu
, gpa_t addr
, u32 access
)
483 return FNAME(walk_addr_generic
)(walker
, vcpu
, &vcpu
->arch
.mmu
, addr
,
487 #if PTTYPE != PTTYPE_EPT
488 static int FNAME(walk_addr_nested
)(struct guest_walker
*walker
,
489 struct kvm_vcpu
*vcpu
, gva_t addr
,
492 return FNAME(walk_addr_generic
)(walker
, vcpu
, &vcpu
->arch
.nested_mmu
,
498 FNAME(prefetch_gpte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
,
499 u64
*spte
, pt_element_t gpte
, bool no_dirty_log
)
505 if (FNAME(prefetch_invalid_gpte
)(vcpu
, sp
, spte
, gpte
))
508 pgprintk("%s: gpte %llx spte %p\n", __func__
, (u64
)gpte
, spte
);
510 gfn
= gpte_to_gfn(gpte
);
511 pte_access
= sp
->role
.access
& FNAME(gpte_access
)(gpte
);
512 FNAME(protect_clean_gpte
)(&vcpu
->arch
.mmu
, &pte_access
, gpte
);
513 pfn
= pte_prefetch_gfn_to_pfn(vcpu
, gfn
,
514 no_dirty_log
&& (pte_access
& ACC_WRITE_MASK
));
515 if (is_error_pfn(pfn
))
519 * we call mmu_set_spte() with host_writable = true because
520 * pte_prefetch_gfn_to_pfn always gets a writable pfn.
522 mmu_set_spte(vcpu
, spte
, pte_access
, 0, PT_PAGE_TABLE_LEVEL
, gfn
, pfn
,
525 kvm_release_pfn_clean(pfn
);
529 static void FNAME(update_pte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
,
530 u64
*spte
, const void *pte
)
532 pt_element_t gpte
= *(const pt_element_t
*)pte
;
534 FNAME(prefetch_gpte
)(vcpu
, sp
, spte
, gpte
, false);
537 static bool FNAME(gpte_changed
)(struct kvm_vcpu
*vcpu
,
538 struct guest_walker
*gw
, int level
)
540 pt_element_t curr_pte
;
541 gpa_t base_gpa
, pte_gpa
= gw
->pte_gpa
[level
- 1];
545 if (level
== PT_PAGE_TABLE_LEVEL
) {
546 mask
= PTE_PREFETCH_NUM
* sizeof(pt_element_t
) - 1;
547 base_gpa
= pte_gpa
& ~mask
;
548 index
= (pte_gpa
- base_gpa
) / sizeof(pt_element_t
);
550 r
= kvm_vcpu_read_guest_atomic(vcpu
, base_gpa
,
551 gw
->prefetch_ptes
, sizeof(gw
->prefetch_ptes
));
552 curr_pte
= gw
->prefetch_ptes
[index
];
554 r
= kvm_vcpu_read_guest_atomic(vcpu
, pte_gpa
,
555 &curr_pte
, sizeof(curr_pte
));
557 return r
|| curr_pte
!= gw
->ptes
[level
- 1];
560 static void FNAME(pte_prefetch
)(struct kvm_vcpu
*vcpu
, struct guest_walker
*gw
,
563 struct kvm_mmu_page
*sp
;
564 pt_element_t
*gptep
= gw
->prefetch_ptes
;
568 sp
= page_header(__pa(sptep
));
570 if (sp
->role
.level
> PT_PAGE_TABLE_LEVEL
)
574 return __direct_pte_prefetch(vcpu
, sp
, sptep
);
576 i
= (sptep
- sp
->spt
) & ~(PTE_PREFETCH_NUM
- 1);
579 for (i
= 0; i
< PTE_PREFETCH_NUM
; i
++, spte
++) {
583 if (is_shadow_present_pte(*spte
))
586 if (!FNAME(prefetch_gpte
)(vcpu
, sp
, spte
, gptep
[i
], true))
592 * Fetch a shadow pte for a specific level in the paging hierarchy.
593 * If the guest tries to write a write-protected page, we need to
594 * emulate this operation, return 1 to indicate this case.
596 static int FNAME(fetch
)(struct kvm_vcpu
*vcpu
, gpa_t addr
,
597 struct guest_walker
*gw
,
598 int write_fault
, int hlevel
,
599 kvm_pfn_t pfn
, bool map_writable
, bool prefault
,
600 bool lpage_disallowed
)
602 struct kvm_mmu_page
*sp
= NULL
;
603 struct kvm_shadow_walk_iterator it
;
604 unsigned direct_access
, access
= gw
->pt_access
;
608 direct_access
= gw
->pte_access
;
610 top_level
= vcpu
->arch
.mmu
.root_level
;
611 if (top_level
== PT32E_ROOT_LEVEL
)
612 top_level
= PT32_ROOT_LEVEL
;
614 * Verify that the top-level gpte is still there. Since the page
615 * is a root page, it is either write protected (and cannot be
616 * changed from now on) or it is invalid (in which case, we don't
617 * really care if it changes underneath us after this point).
619 if (FNAME(gpte_changed
)(vcpu
, gw
, top_level
))
620 goto out_gpte_changed
;
622 if (!VALID_PAGE(vcpu
->arch
.mmu
.root_hpa
))
623 goto out_gpte_changed
;
625 for (shadow_walk_init(&it
, vcpu
, addr
);
626 shadow_walk_okay(&it
) && it
.level
> gw
->level
;
627 shadow_walk_next(&it
)) {
630 clear_sp_write_flooding_count(it
.sptep
);
631 drop_large_spte(vcpu
, it
.sptep
);
634 if (!is_shadow_present_pte(*it
.sptep
)) {
635 table_gfn
= gw
->table_gfn
[it
.level
- 2];
636 sp
= kvm_mmu_get_page(vcpu
, table_gfn
, addr
, it
.level
-1,
641 * Verify that the gpte in the page we've just write
642 * protected is still there.
644 if (FNAME(gpte_changed
)(vcpu
, gw
, it
.level
- 1))
645 goto out_gpte_changed
;
648 link_shadow_page(vcpu
, it
.sptep
, sp
);
652 * FNAME(page_fault) might have clobbered the bottom bits of
653 * gw->gfn, restore them from the virtual address.
655 gfn
= gw
->gfn
| ((addr
& PT_LVL_OFFSET_MASK(gw
->level
)) >> PAGE_SHIFT
);
658 trace_kvm_mmu_spte_requested(addr
, gw
->level
, pfn
);
660 for (; shadow_walk_okay(&it
); shadow_walk_next(&it
)) {
661 clear_sp_write_flooding_count(it
.sptep
);
664 * We cannot overwrite existing page tables with an NX
665 * large page, as the leaf could be executable.
667 disallowed_hugepage_adjust(it
, gfn
, &pfn
, &hlevel
);
669 base_gfn
= gfn
& ~(KVM_PAGES_PER_HPAGE(it
.level
) - 1);
670 if (it
.level
== hlevel
)
673 validate_direct_spte(vcpu
, it
.sptep
, direct_access
);
675 drop_large_spte(vcpu
, it
.sptep
);
677 if (!is_shadow_present_pte(*it
.sptep
)) {
678 sp
= kvm_mmu_get_page(vcpu
, base_gfn
, addr
,
679 it
.level
- 1, true, direct_access
);
680 link_shadow_page(vcpu
, it
.sptep
, sp
);
681 if (lpage_disallowed
)
682 account_huge_nx_page(vcpu
->kvm
, sp
);
686 ret
= mmu_set_spte(vcpu
, it
.sptep
, gw
->pte_access
, write_fault
,
687 it
.level
, base_gfn
, pfn
, prefault
, map_writable
);
688 FNAME(pte_prefetch
)(vcpu
, gw
, it
.sptep
);
689 ++vcpu
->stat
.pf_fixed
;
697 * To see whether the mapped gfn can write its page table in the current
700 * It is the helper function of FNAME(page_fault). When guest uses large page
701 * size to map the writable gfn which is used as current page table, we should
702 * force kvm to use small page size to map it because new shadow page will be
703 * created when kvm establishes shadow page table that stop kvm using large
704 * page size. Do it early can avoid unnecessary #PF and emulation.
706 * @write_fault_to_shadow_pgtable will return true if the fault gfn is
707 * currently used as its page table.
709 * Note: the PDPT page table is not checked for PAE-32 bit guest. It is ok
710 * since the PDPT is always shadowed, that means, we can not use large page
711 * size to map the gfn which is used as PDPT.
714 FNAME(is_self_change_mapping
)(struct kvm_vcpu
*vcpu
,
715 struct guest_walker
*walker
, int user_fault
,
716 bool *write_fault_to_shadow_pgtable
)
719 gfn_t mask
= ~(KVM_PAGES_PER_HPAGE(walker
->level
) - 1);
720 bool self_changed
= false;
722 if (!(walker
->pte_access
& ACC_WRITE_MASK
||
723 (!is_write_protection(vcpu
) && !user_fault
)))
726 for (level
= walker
->level
; level
<= walker
->max_level
; level
++) {
727 gfn_t gfn
= walker
->gfn
^ walker
->table_gfn
[level
- 1];
729 self_changed
|= !(gfn
& mask
);
730 *write_fault_to_shadow_pgtable
|= !gfn
;
737 * Page fault handler. There are several causes for a page fault:
738 * - there is no shadow pte for the guest pte
739 * - write access through a shadow pte marked read only so that we can set
741 * - write access to a shadow pte marked read only so we can update the page
742 * dirty bitmap, when userspace requests it
743 * - mmio access; in this case we will never install a present shadow pte
744 * - normal guest page fault due to the guest pte marked not present, not
745 * writable, or not executable
747 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
748 * a negative value on error.
750 static int FNAME(page_fault
)(struct kvm_vcpu
*vcpu
, gpa_t addr
, u32 error_code
,
753 int write_fault
= error_code
& PFERR_WRITE_MASK
;
754 int user_fault
= error_code
& PFERR_USER_MASK
;
755 struct guest_walker walker
;
758 int level
= PT_PAGE_TABLE_LEVEL
;
759 unsigned long mmu_seq
;
760 bool map_writable
, is_self_change_mapping
;
761 bool lpage_disallowed
= (error_code
& PFERR_FETCH_MASK
) &&
762 is_nx_huge_page_enabled();
763 bool force_pt_level
= lpage_disallowed
;
765 pgprintk("%s: addr %lx err %x\n", __func__
, addr
, error_code
);
767 r
= mmu_topup_memory_caches(vcpu
);
772 * If PFEC.RSVD is set, this is a shadow page fault.
773 * The bit needs to be cleared before walking guest page tables.
775 error_code
&= ~PFERR_RSVD_MASK
;
778 * Look up the guest pte for the faulting address.
780 r
= FNAME(walk_addr
)(&walker
, vcpu
, addr
, error_code
);
783 * The page is not mapped by the guest. Let the guest handle it.
786 pgprintk("%s: guest page fault\n", __func__
);
788 inject_page_fault(vcpu
, &walker
.fault
);
793 if (page_fault_handle_page_track(vcpu
, error_code
, walker
.gfn
)) {
794 shadow_page_table_clear_flood(vcpu
, addr
);
795 return RET_PF_EMULATE
;
798 vcpu
->arch
.write_fault_to_shadow_pgtable
= false;
800 is_self_change_mapping
= FNAME(is_self_change_mapping
)(vcpu
,
801 &walker
, user_fault
, &vcpu
->arch
.write_fault_to_shadow_pgtable
);
803 if (walker
.level
>= PT_DIRECTORY_LEVEL
&& !is_self_change_mapping
) {
804 level
= mapping_level(vcpu
, walker
.gfn
, &force_pt_level
);
805 if (likely(!force_pt_level
)) {
806 level
= min(walker
.level
, level
);
807 walker
.gfn
= walker
.gfn
& ~(KVM_PAGES_PER_HPAGE(level
) - 1);
810 force_pt_level
= true;
812 mmu_seq
= vcpu
->kvm
->mmu_notifier_seq
;
815 if (try_async_pf(vcpu
, prefault
, walker
.gfn
, addr
, &pfn
, write_fault
,
819 if (handle_abnormal_pfn(vcpu
, addr
, walker
.gfn
, pfn
, walker
.pte_access
, &r
))
823 * Do not change pte_access if the pfn is a mmio page, otherwise
824 * we will cache the incorrect access into mmio spte.
826 if (write_fault
&& !(walker
.pte_access
& ACC_WRITE_MASK
) &&
827 !is_write_protection(vcpu
) && !user_fault
&&
828 !is_noslot_pfn(pfn
)) {
829 walker
.pte_access
|= ACC_WRITE_MASK
;
830 walker
.pte_access
&= ~ACC_USER_MASK
;
833 * If we converted a user page to a kernel page,
834 * so that the kernel can write to it when cr0.wp=0,
835 * then we should prevent the kernel from executing it
836 * if SMEP is enabled.
838 if (kvm_read_cr4_bits(vcpu
, X86_CR4_SMEP
))
839 walker
.pte_access
&= ~ACC_EXEC_MASK
;
843 spin_lock(&vcpu
->kvm
->mmu_lock
);
844 if (mmu_notifier_retry(vcpu
->kvm
, mmu_seq
))
847 kvm_mmu_audit(vcpu
, AUDIT_PRE_PAGE_FAULT
);
848 if (make_mmu_pages_available(vcpu
) < 0)
851 transparent_hugepage_adjust(vcpu
, walker
.gfn
, &pfn
, &level
);
852 r
= FNAME(fetch
)(vcpu
, addr
, &walker
, write_fault
,
853 level
, pfn
, map_writable
, prefault
, lpage_disallowed
);
854 kvm_mmu_audit(vcpu
, AUDIT_POST_PAGE_FAULT
);
857 spin_unlock(&vcpu
->kvm
->mmu_lock
);
858 kvm_release_pfn_clean(pfn
);
862 static gpa_t
FNAME(get_level1_sp_gpa
)(struct kvm_mmu_page
*sp
)
866 WARN_ON(sp
->role
.level
!= PT_PAGE_TABLE_LEVEL
);
869 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
871 return gfn_to_gpa(sp
->gfn
) + offset
* sizeof(pt_element_t
);
874 static void FNAME(invlpg
)(struct kvm_vcpu
*vcpu
, gva_t gva
)
876 struct kvm_shadow_walk_iterator iterator
;
877 struct kvm_mmu_page
*sp
;
881 vcpu_clear_mmio_info(vcpu
, gva
);
884 * No need to check return value here, rmap_can_add() can
885 * help us to skip pte prefetch later.
887 mmu_topup_memory_caches(vcpu
);
889 if (!VALID_PAGE(vcpu
->arch
.mmu
.root_hpa
)) {
894 spin_lock(&vcpu
->kvm
->mmu_lock
);
895 for_each_shadow_entry(vcpu
, gva
, iterator
) {
896 level
= iterator
.level
;
897 sptep
= iterator
.sptep
;
899 sp
= page_header(__pa(sptep
));
900 if (is_last_spte(*sptep
, level
)) {
907 pte_gpa
= FNAME(get_level1_sp_gpa
)(sp
);
908 pte_gpa
+= (sptep
- sp
->spt
) * sizeof(pt_element_t
);
910 if (mmu_page_zap_pte(vcpu
->kvm
, sp
, sptep
))
911 kvm_flush_remote_tlbs(vcpu
->kvm
);
913 if (!rmap_can_add(vcpu
))
916 if (kvm_vcpu_read_guest_atomic(vcpu
, pte_gpa
, &gpte
,
917 sizeof(pt_element_t
)))
920 FNAME(update_pte
)(vcpu
, sp
, sptep
, &gpte
);
923 if (!is_shadow_present_pte(*sptep
) || !sp
->unsync_children
)
926 spin_unlock(&vcpu
->kvm
->mmu_lock
);
929 /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */
930 static gpa_t
FNAME(gva_to_gpa
)(struct kvm_vcpu
*vcpu
, gpa_t addr
, u32 access
,
931 struct x86_exception
*exception
)
933 struct guest_walker walker
;
934 gpa_t gpa
= UNMAPPED_GVA
;
937 r
= FNAME(walk_addr
)(&walker
, vcpu
, addr
, access
);
940 gpa
= gfn_to_gpa(walker
.gfn
);
941 gpa
|= addr
& ~PAGE_MASK
;
942 } else if (exception
)
943 *exception
= walker
.fault
;
948 #if PTTYPE != PTTYPE_EPT
949 /* Note, gva_to_gpa_nested() is only used to translate L2 GVAs. */
950 static gpa_t
FNAME(gva_to_gpa_nested
)(struct kvm_vcpu
*vcpu
, gpa_t vaddr
,
952 struct x86_exception
*exception
)
954 struct guest_walker walker
;
955 gpa_t gpa
= UNMAPPED_GVA
;
958 #ifndef CONFIG_X86_64
959 /* A 64-bit GVA should be impossible on 32-bit KVM. */
960 WARN_ON_ONCE(vaddr
>> 32);
963 r
= FNAME(walk_addr_nested
)(&walker
, vcpu
, vaddr
, access
);
966 gpa
= gfn_to_gpa(walker
.gfn
);
967 gpa
|= vaddr
& ~PAGE_MASK
;
968 } else if (exception
)
969 *exception
= walker
.fault
;
976 * Using the cached information from sp->gfns is safe because:
977 * - The spte has a reference to the struct page, so the pfn for a given gfn
978 * can't change unless all sptes pointing to it are nuked first.
981 * We should flush all tlbs if spte is dropped even though guest is
982 * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
983 * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
984 * used by guest then tlbs are not flushed, so guest is allowed to access the
986 * And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
988 static int FNAME(sync_page
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
)
990 int i
, nr_present
= 0;
994 /* direct kvm_mmu_page can not be unsync. */
995 BUG_ON(sp
->role
.direct
);
997 first_pte_gpa
= FNAME(get_level1_sp_gpa
)(sp
);
999 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
++) {
1000 unsigned pte_access
;
1008 pte_gpa
= first_pte_gpa
+ i
* sizeof(pt_element_t
);
1010 if (kvm_vcpu_read_guest_atomic(vcpu
, pte_gpa
, &gpte
,
1011 sizeof(pt_element_t
)))
1014 if (FNAME(prefetch_invalid_gpte
)(vcpu
, sp
, &sp
->spt
[i
], gpte
)) {
1016 * Update spte before increasing tlbs_dirty to make
1017 * sure no tlb flush is lost after spte is zapped; see
1018 * the comments in kvm_flush_remote_tlbs().
1021 vcpu
->kvm
->tlbs_dirty
++;
1025 gfn
= gpte_to_gfn(gpte
);
1026 pte_access
= sp
->role
.access
;
1027 pte_access
&= FNAME(gpte_access
)(gpte
);
1028 FNAME(protect_clean_gpte
)(&vcpu
->arch
.mmu
, &pte_access
, gpte
);
1030 if (sync_mmio_spte(vcpu
, &sp
->spt
[i
], gfn
, pte_access
,
1034 if (gfn
!= sp
->gfns
[i
]) {
1035 drop_spte(vcpu
->kvm
, &sp
->spt
[i
]);
1037 * The same as above where we are doing
1038 * prefetch_invalid_gpte().
1041 vcpu
->kvm
->tlbs_dirty
++;
1047 host_writable
= sp
->spt
[i
] & SPTE_HOST_WRITEABLE
;
1049 set_spte(vcpu
, &sp
->spt
[i
], pte_access
,
1050 PT_PAGE_TABLE_LEVEL
, gfn
,
1051 spte_to_pfn(sp
->spt
[i
]), true, false,
1061 #undef PT_BASE_ADDR_MASK
1063 #undef PT_LVL_ADDR_MASK
1064 #undef PT_LVL_OFFSET_MASK
1065 #undef PT_LEVEL_BITS
1066 #undef PT_MAX_FULL_LEVELS
1068 #undef gpte_to_gfn_lvl
1070 #undef PT_GUEST_ACCESSED_MASK
1071 #undef PT_GUEST_DIRTY_MASK
1072 #undef PT_GUEST_DIRTY_SHIFT
1073 #undef PT_GUEST_ACCESSED_SHIFT
1074 #undef PT_HAVE_ACCESSED_DIRTY