]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/kvm/mmu.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] KVM: MMU: Zap shadow page table entries on writes to guest page tables
[mirror_ubuntu-bionic-kernel.git] / drivers / kvm / mmu.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19#include <linux/types.h>
20#include <linux/string.h>
21#include <asm/page.h>
22#include <linux/mm.h>
23#include <linux/highmem.h>
24#include <linux/module.h>
25
26#include "vmx.h"
27#include "kvm.h"
28
cea0f0e7
AK		 29#define pgprintk(x...) do { printk(x); } while (0)
30#define rmap_printk(x...) do { printk(x); } while (0)
6aa8b732
AK		 31
32#define ASSERT(x) \
33 if (!(x)) { \
34 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
35 __FILE__, __LINE__, #x); \
36 }
37
cea0f0e7
AK		 38#define PT64_PT_BITS 9
39#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
40#define PT32_PT_BITS 10
41#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
6aa8b732
AK		 42
43#define PT_WRITABLE_SHIFT 1
44
45#define PT_PRESENT_MASK (1ULL << 0)
46#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
47#define PT_USER_MASK (1ULL << 2)
48#define PT_PWT_MASK (1ULL << 3)
49#define PT_PCD_MASK (1ULL << 4)
50#define PT_ACCESSED_MASK (1ULL << 5)
51#define PT_DIRTY_MASK (1ULL << 6)
52#define PT_PAGE_SIZE_MASK (1ULL << 7)
53#define PT_PAT_MASK (1ULL << 7)
54#define PT_GLOBAL_MASK (1ULL << 8)
55#define PT64_NX_MASK (1ULL << 63)
56
57#define PT_PAT_SHIFT 7
58#define PT_DIR_PAT_SHIFT 12
59#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
60
61#define PT32_DIR_PSE36_SIZE 4
62#define PT32_DIR_PSE36_SHIFT 13
63#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
64
65
66#define PT32_PTE_COPY_MASK \
8c7bb723 67 (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
6aa8b732 68
8c7bb723 69#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
6aa8b732
AK		 70
71#define PT_FIRST_AVAIL_BITS_SHIFT 9
72#define PT64_SECOND_AVAIL_BITS_SHIFT 52
73
74#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
75#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
76
77#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
78#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
79
80#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
81#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
82
83#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
84
85#define VALID_PAGE(x) ((x) != INVALID_PAGE)
86
87#define PT64_LEVEL_BITS 9
88
89#define PT64_LEVEL_SHIFT(level) \
90 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
91
92#define PT64_LEVEL_MASK(level) \
93 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
94
95#define PT64_INDEX(address, level)\
96 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
97
98
99#define PT32_LEVEL_BITS 10
100
101#define PT32_LEVEL_SHIFT(level) \
102 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
103
104#define PT32_LEVEL_MASK(level) \
105 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
106
107#define PT32_INDEX(address, level)\
108 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
109
110
111#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
112#define PT64_DIR_BASE_ADDR_MASK \
113 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
114
115#define PT32_BASE_ADDR_MASK PAGE_MASK
116#define PT32_DIR_BASE_ADDR_MASK \
117 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
118
119
120#define PFERR_PRESENT_MASK (1U << 0)
121#define PFERR_WRITE_MASK (1U << 1)
122#define PFERR_USER_MASK (1U << 2)
123
124#define PT64_ROOT_LEVEL 4
125#define PT32_ROOT_LEVEL 2
126#define PT32E_ROOT_LEVEL 3
127
128#define PT_DIRECTORY_LEVEL 2
129#define PT_PAGE_TABLE_LEVEL 1
130
cd4a4e53
AK		 131#define RMAP_EXT 4
132
133struct kvm_rmap_desc {
134 u64 *shadow_ptes[RMAP_EXT];
135 struct kvm_rmap_desc *more;
136};
137
6aa8b732
AK		 138static int is_write_protection(struct kvm_vcpu *vcpu)
139{
140 return vcpu->cr0 & CR0_WP_MASK;
141}
142
143static int is_cpuid_PSE36(void)
144{
145 return 1;
146}
147
148static int is_present_pte(unsigned long pte)
149{
150 return pte & PT_PRESENT_MASK;
151}
152
153static int is_writeble_pte(unsigned long pte)
154{
155 return pte & PT_WRITABLE_MASK;
156}
157
158static int is_io_pte(unsigned long pte)
159{
160 return pte & PT_SHADOW_IO_MARK;
161}
162
cd4a4e53
AK		 163static int is_rmap_pte(u64 pte)
164{
165 return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
166 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
167}
168
169/*
170 * Reverse mapping data structures:
171 *
172 * If page->private bit zero is zero, then page->private points to the
173 * shadow page table entry that points to page_address(page).
174 *
175 * If page->private bit zero is one, (then page->private & ~1) points
176 * to a struct kvm_rmap_desc containing more mappings.
177 */
178static void rmap_add(struct kvm *kvm, u64 *spte)
179{
180 struct page *page;
181 struct kvm_rmap_desc *desc;
182 int i;
183
184 if (!is_rmap_pte(*spte))
185 return;
186 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
187 if (!page->private) {
188 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
189 page->private = (unsigned long)spte;
190 } else if (!(page->private & 1)) {
191 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
192 desc = kzalloc(sizeof *desc, GFP_NOWAIT);
193 if (!desc)
194 BUG(); /* FIXME: return error */
195 desc->shadow_ptes[0] = (u64 *)page->private;
196 desc->shadow_ptes[1] = spte;
197 page->private = (unsigned long)desc | 1;
198 } else {
199 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
200 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
201 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
202 desc = desc->more;
203 if (desc->shadow_ptes[RMAP_EXT-1]) {
204 desc->more = kzalloc(sizeof *desc->more, GFP_NOWAIT);
205 if (!desc->more)
206 BUG(); /* FIXME: return error */
207 desc = desc->more;
208 }
209 for (i = 0; desc->shadow_ptes[i]; ++i)
210 ;
211 desc->shadow_ptes[i] = spte;
212 }
213}
214
215static void rmap_desc_remove_entry(struct page *page,
216 struct kvm_rmap_desc *desc,
217 int i,
218 struct kvm_rmap_desc *prev_desc)
219{
220 int j;
221
222 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
223 ;
224 desc->shadow_ptes[i] = desc->shadow_ptes[j];
225 desc->shadow_ptes[j] = 0;
226 if (j != 0)
227 return;
228 if (!prev_desc && !desc->more)
229 page->private = (unsigned long)desc->shadow_ptes[0];
230 else
231 if (prev_desc)
232 prev_desc->more = desc->more;
233 else
234 page->private = (unsigned long)desc->more | 1;
235 kfree(desc);
236}
237
238static void rmap_remove(struct kvm *kvm, u64 *spte)
239{
240 struct page *page;
241 struct kvm_rmap_desc *desc;
242 struct kvm_rmap_desc *prev_desc;
243 int i;
244
245 if (!is_rmap_pte(*spte))
246 return;
247 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
248 if (!page->private) {
249 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
250 BUG();
251 } else if (!(page->private & 1)) {
252 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
253 if ((u64 *)page->private != spte) {
254 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
255 spte, *spte);
256 BUG();
257 }
258 page->private = 0;
259 } else {
260 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
261 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
262 prev_desc = NULL;
263 while (desc) {
264 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
265 if (desc->shadow_ptes[i] == spte) {
266 rmap_desc_remove_entry(page, desc, i,
267 prev_desc);
268 return;
269 }
270 prev_desc = desc;
271 desc = desc->more;
272 }
273 BUG();
274 }
275}
276
374cbac0
AK		 277static void rmap_write_protect(struct kvm *kvm, u64 gfn)
278{
279 struct page *page;
280 struct kvm_memory_slot *slot;
281 struct kvm_rmap_desc *desc;
282 u64 *spte;
283
284 slot = gfn_to_memslot(kvm, gfn);
285 BUG_ON(!slot);
286 page = gfn_to_page(slot, gfn);
287
288 while (page->private) {
289 if (!(page->private & 1))
290 spte = (u64 *)page->private;
291 else {
292 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
293 spte = desc->shadow_ptes[0];
294 }
295 BUG_ON(!spte);
296 BUG_ON((*spte & PT64_BASE_ADDR_MASK) !=
297 page_to_pfn(page) << PAGE_SHIFT);
298 BUG_ON(!(*spte & PT_PRESENT_MASK));
299 BUG_ON(!(*spte & PT_WRITABLE_MASK));
300 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
301 rmap_remove(kvm, spte);
302 *spte &= ~(u64)PT_WRITABLE_MASK;
303 }
304}
305
6aa8b732
AK		 306static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
307{
308 struct kvm_mmu_page *page_head = page_header(page_hpa);
309
310 list_del(&page_head->link);
311 page_head->page_hpa = page_hpa;
312 list_add(&page_head->link, &vcpu->free_pages);
313}
314
315static int is_empty_shadow_page(hpa_t page_hpa)
316{
317 u32 *pos;
318 u32 *end;
319 for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32);
320 pos != end; pos++)
321 if (*pos != 0)
322 return 0;
323 return 1;
324}
325
cea0f0e7
AK		 326static unsigned kvm_page_table_hashfn(gfn_t gfn)
327{
328 return gfn;
329}
330
25c0de2c
AK		 331static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
332 u64 *parent_pte)
6aa8b732
AK		 333{
334 struct kvm_mmu_page *page;
335
336 if (list_empty(&vcpu->free_pages))
25c0de2c 337 return NULL;
6aa8b732
AK		 338
339 page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
340 list_del(&page->link);
341 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
342 ASSERT(is_empty_shadow_page(page->page_hpa));
343 page->slot_bitmap = 0;
344 page->global = 1;
cea0f0e7 345 page->multimapped = 0;
6aa8b732 346 page->parent_pte = parent_pte;
25c0de2c 347 return page;
6aa8b732
AK		 348}
349
cea0f0e7
AK		 350static void mmu_page_add_parent_pte(struct kvm_mmu_page *page, u64 *parent_pte)
351{
352 struct kvm_pte_chain *pte_chain;
353 struct hlist_node *node;
354 int i;
355
356 if (!parent_pte)
357 return;
358 if (!page->multimapped) {
359 u64 *old = page->parent_pte;
360
361 if (!old) {
362 page->parent_pte = parent_pte;
363 return;
364 }
365 page->multimapped = 1;
366 pte_chain = kzalloc(sizeof(struct kvm_pte_chain), GFP_NOWAIT);
367 BUG_ON(!pte_chain);
368 INIT_HLIST_HEAD(&page->parent_ptes);
369 hlist_add_head(&pte_chain->link, &page->parent_ptes);
370 pte_chain->parent_ptes[0] = old;
371 }
372 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
373 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
374 continue;
375 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
376 if (!pte_chain->parent_ptes[i]) {
377 pte_chain->parent_ptes[i] = parent_pte;
378 return;
379 }
380 }
381 pte_chain = kzalloc(sizeof(struct kvm_pte_chain), GFP_NOWAIT);
382 BUG_ON(!pte_chain);
383 hlist_add_head(&pte_chain->link, &page->parent_ptes);
384 pte_chain->parent_ptes[0] = parent_pte;
385}
386
387static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page,
388 u64 *parent_pte)
389{
390 struct kvm_pte_chain *pte_chain;
391 struct hlist_node *node;
392 int i;
393
394 if (!page->multimapped) {
395 BUG_ON(page->parent_pte != parent_pte);
396 page->parent_pte = NULL;
397 return;
398 }
399 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
400 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
401 if (!pte_chain->parent_ptes[i])
402 break;
403 if (pte_chain->parent_ptes[i] != parent_pte)
404 continue;
405 while (i + 1 < NR_PTE_CHAIN_ENTRIES) {
406 pte_chain->parent_ptes[i]
407 = pte_chain->parent_ptes[i + 1];
408 ++i;
409 }
410 pte_chain->parent_ptes[i] = NULL;
411 return;
412 }
413 BUG();
414}
415
416static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
417 gfn_t gfn)
418{
419 unsigned index;
420 struct hlist_head *bucket;
421 struct kvm_mmu_page *page;
422 struct hlist_node *node;
423
424 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
425 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
426 bucket = &vcpu->kvm->mmu_page_hash[index];
427 hlist_for_each_entry(page, node, bucket, hash_link)
428 if (page->gfn == gfn && !page->role.metaphysical) {
429 pgprintk("%s: found role %x\n",
430 __FUNCTION__, page->role.word);
431 return page;
432 }
433 return NULL;
434}
435
436static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
437 gfn_t gfn,
438 gva_t gaddr,
439 unsigned level,
440 int metaphysical,
441 u64 *parent_pte)
442{
443 union kvm_mmu_page_role role;
444 unsigned index;
445 unsigned quadrant;
446 struct hlist_head *bucket;
447 struct kvm_mmu_page *page;
448 struct hlist_node *node;
449
450 role.word = 0;
451 role.glevels = vcpu->mmu.root_level;
452 role.level = level;
453 role.metaphysical = metaphysical;
454 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
455 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
456 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
457 role.quadrant = quadrant;
458 }
459 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
460 gfn, role.word);
461 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
462 bucket = &vcpu->kvm->mmu_page_hash[index];
463 hlist_for_each_entry(page, node, bucket, hash_link)
464 if (page->gfn == gfn && page->role.word == role.word) {
465 mmu_page_add_parent_pte(page, parent_pte);
466 pgprintk("%s: found\n", __FUNCTION__);
467 return page;
468 }
469 page = kvm_mmu_alloc_page(vcpu, parent_pte);
470 if (!page)
471 return page;
472 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
473 page->gfn = gfn;
474 page->role = role;
475 hlist_add_head(&page->hash_link, bucket);
374cbac0
AK		 476 if (!metaphysical)
477 rmap_write_protect(vcpu->kvm, gfn);
cea0f0e7
AK		 478 return page;
479}
480
481static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
482 struct kvm_mmu_page *page,
483 u64 *parent_pte)
484{
485 mmu_page_remove_parent_pte(page, parent_pte);
486}
487
6aa8b732
AK		 488static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
489{
490 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
491 struct kvm_mmu_page *page_head = page_header(__pa(pte));
492
493 __set_bit(slot, &page_head->slot_bitmap);
494}
495
496hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
497{
498 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
499
500 return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
501}
502
503hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
504{
505 struct kvm_memory_slot *slot;
506 struct page *page;
507
508 ASSERT((gpa & HPA_ERR_MASK) == 0);
509 slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
510 if (!slot)
511 return gpa | HPA_ERR_MASK;
512 page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
513 return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
514 | (gpa & (PAGE_SIZE-1));
515}
516
517hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
518{
519 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
520
521 if (gpa == UNMAPPED_GVA)
522 return UNMAPPED_GVA;
523 return gpa_to_hpa(vcpu, gpa);
524}
525
526
527static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa,
528 int level)
529{
cd4a4e53
AK		 530 u64 *pos;
531 u64 *end;
532
6aa8b732
AK		 533 ASSERT(vcpu);
534 ASSERT(VALID_PAGE(page_hpa));
535 ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
536
cd4a4e53
AK		 537 for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE;
538 pos != end; pos++) {
539 u64 current_ent = *pos;
6aa8b732 540
cd4a4e53
AK		 541 if (is_present_pte(current_ent)) {
542 if (level != 1)
6aa8b732
AK		 543 release_pt_page_64(vcpu,
544 current_ent &
545 PT64_BASE_ADDR_MASK,
546 level - 1);
cd4a4e53
AK		 547 else
548 rmap_remove(vcpu->kvm, pos);
6aa8b732 549 }
cd4a4e53 550 *pos = 0;
6aa8b732
AK		 551 }
552 kvm_mmu_free_page(vcpu, page_hpa);
553}
554
555static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
556{
557}
558
559static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
560{
561 int level = PT32E_ROOT_LEVEL;
562 hpa_t table_addr = vcpu->mmu.root_hpa;
563
564 for (; ; level--) {
565 u32 index = PT64_INDEX(v, level);
566 u64 *table;
cea0f0e7 567 u64 pte;
6aa8b732
AK		 568
569 ASSERT(VALID_PAGE(table_addr));
570 table = __va(table_addr);
571
572 if (level == 1) {
cea0f0e7
AK		 573 pte = table[index];
574 if (is_present_pte(pte) && is_writeble_pte(pte))
575 return 0;
6aa8b732
AK		 576 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
577 page_header_update_slot(vcpu->kvm, table, v);
578 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
579 PT_USER_MASK;
cd4a4e53 580 rmap_add(vcpu->kvm, &table[index]);
6aa8b732
AK		 581 return 0;
582 }
583
584 if (table[index] == 0) {
25c0de2c 585 struct kvm_mmu_page *new_table;
cea0f0e7 586 gfn_t pseudo_gfn;
6aa8b732 587
cea0f0e7
AK		 588 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
589 >> PAGE_SHIFT;
590 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
591 v, level - 1,
592 1, &table[index]);
25c0de2c 593 if (!new_table) {
6aa8b732
AK		 594 pgprintk("nonpaging_map: ENOMEM\n");
595 return -ENOMEM;
596 }
597
25c0de2c
AK		 598 table[index] = new_table->page_hpa | PT_PRESENT_MASK
599 | PT_WRITABLE_MASK | PT_USER_MASK;
6aa8b732
AK		 600 }
601 table_addr = table[index] & PT64_BASE_ADDR_MASK;
602 }
603}
604
17ac10ad
AK		 605static void mmu_free_roots(struct kvm_vcpu *vcpu)
606{
607 int i;
608
609#ifdef CONFIG_X86_64
610 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
611 hpa_t root = vcpu->mmu.root_hpa;
612
613 ASSERT(VALID_PAGE(root));
17ac10ad
AK		 614 vcpu->mmu.root_hpa = INVALID_PAGE;
615 return;
616 }
617#endif
618 for (i = 0; i < 4; ++i) {
619 hpa_t root = vcpu->mmu.pae_root[i];
620
621 ASSERT(VALID_PAGE(root));
622 root &= PT64_BASE_ADDR_MASK;
17ac10ad
AK		 623 vcpu->mmu.pae_root[i] = INVALID_PAGE;
624 }
625 vcpu->mmu.root_hpa = INVALID_PAGE;
626}
627
628static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
629{
630 int i;
cea0f0e7
AK		 631 gfn_t root_gfn;
632 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
17ac10ad
AK		 633
634#ifdef CONFIG_X86_64
635 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
636 hpa_t root = vcpu->mmu.root_hpa;
637
638 ASSERT(!VALID_PAGE(root));
cea0f0e7
AK		 639 root = kvm_mmu_get_page(vcpu, root_gfn, 0,
640 PT64_ROOT_LEVEL, 0, NULL)->page_hpa;
17ac10ad
AK		 641 vcpu->mmu.root_hpa = root;
642 return;
643 }
644#endif
645 for (i = 0; i < 4; ++i) {
646 hpa_t root = vcpu->mmu.pae_root[i];
647
648 ASSERT(!VALID_PAGE(root));
cea0f0e7
AK		 649 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL)
650 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
651 else if (vcpu->mmu.root_level == 0)
652 root_gfn = 0;
653 root = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
654 PT32_ROOT_LEVEL, !is_paging(vcpu),
655 NULL)->page_hpa;
17ac10ad
AK		 656 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
657 }
658 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
659}
660
6aa8b732
AK		 661static void nonpaging_flush(struct kvm_vcpu *vcpu)
662{
663 hpa_t root = vcpu->mmu.root_hpa;
664
665 ++kvm_stat.tlb_flush;
666 pgprintk("nonpaging_flush\n");
17ac10ad
AK		 667 mmu_free_roots(vcpu);
668 mmu_alloc_roots(vcpu);
6aa8b732
AK		 669 kvm_arch_ops->set_cr3(vcpu, root);
670 kvm_arch_ops->tlb_flush(vcpu);
671}
672
673static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
674{
675 return vaddr;
676}
677
678static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
679 u32 error_code)
680{
681 int ret;
682 gpa_t addr = gva;
683
684 ASSERT(vcpu);
685 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
686
687 for (;;) {
688 hpa_t paddr;
689
690 paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
691
692 if (is_error_hpa(paddr))
693 return 1;
694
695 ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
696 if (ret) {
697 nonpaging_flush(vcpu);
698 continue;
699 }
700 break;
701 }
702 return ret;
703}
704
705static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
706{
707}
708
709static void nonpaging_free(struct kvm_vcpu *vcpu)
710{
17ac10ad 711 mmu_free_roots(vcpu);
6aa8b732
AK		 712}
713
714static int nonpaging_init_context(struct kvm_vcpu *vcpu)
715{
716 struct kvm_mmu *context = &vcpu->mmu;
717
718 context->new_cr3 = nonpaging_new_cr3;
719 context->page_fault = nonpaging_page_fault;
720 context->inval_page = nonpaging_inval_page;
721 context->gva_to_gpa = nonpaging_gva_to_gpa;
722 context->free = nonpaging_free;
cea0f0e7 723 context->root_level = 0;
6aa8b732 724 context->shadow_root_level = PT32E_ROOT_LEVEL;
17ac10ad 725 mmu_alloc_roots(vcpu);
6aa8b732
AK		 726 ASSERT(VALID_PAGE(context->root_hpa));
727 kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
728 return 0;
729}
730
6aa8b732
AK		 731static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
732{
6aa8b732
AK		 733 ++kvm_stat.tlb_flush;
734 kvm_arch_ops->tlb_flush(vcpu);
735}
736
737static void paging_new_cr3(struct kvm_vcpu *vcpu)
738{
374cbac0 739 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
cea0f0e7
AK		 740 mmu_free_roots(vcpu);
741 mmu_alloc_roots(vcpu);
6aa8b732 742 kvm_mmu_flush_tlb(vcpu);
cea0f0e7 743 kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
6aa8b732
AK		 744}
745
746static void mark_pagetable_nonglobal(void *shadow_pte)
747{
748 page_header(__pa(shadow_pte))->global = 0;
749}
750
751static inline void set_pte_common(struct kvm_vcpu *vcpu,
752 u64 *shadow_pte,
753 gpa_t gaddr,
754 int dirty,
815af8d4
AK		 755 u64 access_bits,
756 gfn_t gfn)
6aa8b732
AK		 757{
758 hpa_t paddr;
759
760 *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
761 if (!dirty)
762 access_bits &= ~PT_WRITABLE_MASK;
cea0f0e7 763
374cbac0 764 paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
6aa8b732
AK		 765
766 *shadow_pte |= access_bits;
767
6aa8b732
AK		 768 if (!(*shadow_pte & PT_GLOBAL_MASK))
769 mark_pagetable_nonglobal(shadow_pte);
770
771 if (is_error_hpa(paddr)) {
772 *shadow_pte |= gaddr;
773 *shadow_pte |= PT_SHADOW_IO_MARK;
774 *shadow_pte &= ~PT_PRESENT_MASK;
374cbac0 775 return;
6aa8b732 776 }
374cbac0
AK		 777
778 *shadow_pte |= paddr;
779
780 if (access_bits & PT_WRITABLE_MASK) {
781 struct kvm_mmu_page *shadow;
782
815af8d4 783 shadow = kvm_mmu_lookup_page(vcpu, gfn);
374cbac0
AK		 784 if (shadow) {
785 pgprintk("%s: found shadow page for %lx, marking ro\n",
815af8d4 786 __FUNCTION__, gfn);
374cbac0
AK		 787 access_bits &= ~PT_WRITABLE_MASK;
788 *shadow_pte &= ~PT_WRITABLE_MASK;
789 }
790 }
791
792 if (access_bits & PT_WRITABLE_MASK)
793 mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
794
795 page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
796 rmap_add(vcpu->kvm, shadow_pte);
6aa8b732
AK		 797}
798
799static void inject_page_fault(struct kvm_vcpu *vcpu,
800 u64 addr,
801 u32 err_code)
802{
803 kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
804}
805
806static inline int fix_read_pf(u64 *shadow_ent)
807{
808 if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
809 !(*shadow_ent & PT_USER_MASK)) {
810 /*
811 * If supervisor write protect is disabled, we shadow kernel
812 * pages as user pages so we can trap the write access.
813 */
814 *shadow_ent |= PT_USER_MASK;
815 *shadow_ent &= ~PT_WRITABLE_MASK;
816
817 return 1;
818
819 }
820 return 0;
821}
822
823static int may_access(u64 pte, int write, int user)
824{
825
826 if (user && !(pte & PT_USER_MASK))
827 return 0;
828 if (write && !(pte & PT_WRITABLE_MASK))
829 return 0;
830 return 1;
831}
832
833/*
834 * Remove a shadow pte.
835 */
836static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
837{
838 hpa_t page_addr = vcpu->mmu.root_hpa;
839 int level = vcpu->mmu.shadow_root_level;
840
841 ++kvm_stat.invlpg;
842
843 for (; ; level--) {
844 u32 index = PT64_INDEX(addr, level);
845 u64 *table = __va(page_addr);
846
847 if (level == PT_PAGE_TABLE_LEVEL ) {
cd4a4e53 848 rmap_remove(vcpu->kvm, &table[index]);
6aa8b732
AK		 849 table[index] = 0;
850 return;
851 }
852
853 if (!is_present_pte(table[index]))
854 return;
855
856 page_addr = table[index] & PT64_BASE_ADDR_MASK;
857
858 if (level == PT_DIRECTORY_LEVEL &&
859 (table[index] & PT_SHADOW_PS_MARK)) {
860 table[index] = 0;
861 release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
862
863 kvm_arch_ops->tlb_flush(vcpu);
864 return;
865 }
866 }
867}
868
869static void paging_free(struct kvm_vcpu *vcpu)
870{
871 nonpaging_free(vcpu);
872}
873
874#define PTTYPE 64
875#include "paging_tmpl.h"
876#undef PTTYPE
877
878#define PTTYPE 32
879#include "paging_tmpl.h"
880#undef PTTYPE
881
17ac10ad 882static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
6aa8b732
AK		 883{
884 struct kvm_mmu *context = &vcpu->mmu;
885
886 ASSERT(is_pae(vcpu));
887 context->new_cr3 = paging_new_cr3;
888 context->page_fault = paging64_page_fault;
889 context->inval_page = paging_inval_page;
890 context->gva_to_gpa = paging64_gva_to_gpa;
891 context->free = paging_free;
17ac10ad
AK		 892 context->root_level = level;
893 context->shadow_root_level = level;
894 mmu_alloc_roots(vcpu);
6aa8b732
AK		 895 ASSERT(VALID_PAGE(context->root_hpa));
896 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
897 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
898 return 0;
899}
900
17ac10ad
AK		 901static int paging64_init_context(struct kvm_vcpu *vcpu)
902{
903 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
904}
905
6aa8b732
AK		 906static int paging32_init_context(struct kvm_vcpu *vcpu)
907{
908 struct kvm_mmu *context = &vcpu->mmu;
909
910 context->new_cr3 = paging_new_cr3;
911 context->page_fault = paging32_page_fault;
912 context->inval_page = paging_inval_page;
913 context->gva_to_gpa = paging32_gva_to_gpa;
914 context->free = paging_free;
915 context->root_level = PT32_ROOT_LEVEL;
916 context->shadow_root_level = PT32E_ROOT_LEVEL;
17ac10ad 917 mmu_alloc_roots(vcpu);
6aa8b732
AK		 918 ASSERT(VALID_PAGE(context->root_hpa));
919 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
920 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
921 return 0;
922}
923
924static int paging32E_init_context(struct kvm_vcpu *vcpu)
925{
17ac10ad 926 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
6aa8b732
AK		 927}
928
929static int init_kvm_mmu(struct kvm_vcpu *vcpu)
930{
931 ASSERT(vcpu);
932 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
933
934 if (!is_paging(vcpu))
935 return nonpaging_init_context(vcpu);
a9058ecd 936 else if (is_long_mode(vcpu))
6aa8b732
AK		 937 return paging64_init_context(vcpu);
938 else if (is_pae(vcpu))
939 return paging32E_init_context(vcpu);
940 else
941 return paging32_init_context(vcpu);
942}
943
944static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
945{
946 ASSERT(vcpu);
947 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
948 vcpu->mmu.free(vcpu);
949 vcpu->mmu.root_hpa = INVALID_PAGE;
950 }
951}
952
953int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
954{
955 destroy_kvm_mmu(vcpu);
956 return init_kvm_mmu(vcpu);
957}
958
da4a00f0
AK		 959void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
960{
9b7a0325
AK		 961 gfn_t gfn = gpa >> PAGE_SHIFT;
962 struct kvm_mmu_page *page;
963 struct kvm_mmu_page *child;
964 struct hlist_node *node;
965 struct hlist_head *bucket;
966 unsigned index;
967 u64 *spte;
968 u64 pte;
969 unsigned offset = offset_in_page(gpa);
970 unsigned page_offset;
971 int level;
972
da4a00f0 973 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
9b7a0325
AK		 974 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
975 bucket = &vcpu->kvm->mmu_page_hash[index];
976 hlist_for_each_entry(page, node, bucket, hash_link) {
977 if (page->gfn != gfn || page->role.metaphysical)
978 continue;
979 page_offset = offset;
980 level = page->role.level;
981 if (page->role.glevels == PT32_ROOT_LEVEL) {
982 page_offset <<= 1; /* 32->64 */
983 page_offset &= ~PAGE_MASK;
984 }
985 spte = __va(page->page_hpa);
986 spte += page_offset / sizeof(*spte);
987 pte = *spte;
988 if (is_present_pte(pte)) {
989 if (level == PT_PAGE_TABLE_LEVEL)
990 rmap_remove(vcpu->kvm, spte);
991 else {
992 child = page_header(pte & PT64_BASE_ADDR_MASK);
993 mmu_page_remove_parent_pte(child, spte);
994 }
995 }
996 *spte = 0;
997 }
da4a00f0
AK		 998}
999
1000void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
1001{
1002}
1003
6aa8b732
AK		 1004static void free_mmu_pages(struct kvm_vcpu *vcpu)
1005{
1006 while (!list_empty(&vcpu->free_pages)) {
1007 struct kvm_mmu_page *page;
1008
1009 page = list_entry(vcpu->free_pages.next,
1010 struct kvm_mmu_page, link);
1011 list_del(&page->link);
1012 __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
1013 page->page_hpa = INVALID_PAGE;
1014 }
17ac10ad 1015 free_page((unsigned long)vcpu->mmu.pae_root);
6aa8b732
AK		 1016}
1017
1018static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1019{
17ac10ad 1020 struct page *page;
6aa8b732
AK		 1021 int i;
1022
1023 ASSERT(vcpu);
1024
1025 for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
6aa8b732
AK		 1026 struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
1027
1028 INIT_LIST_HEAD(&page_header->link);
17ac10ad 1029 if ((page = alloc_page(GFP_KERNEL)) == NULL)
6aa8b732
AK		 1030 goto error_1;
1031 page->private = (unsigned long)page_header;
1032 page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
1033 memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
1034 list_add(&page_header->link, &vcpu->free_pages);
1035 }
17ac10ad
AK		 1036
1037 /*
1038 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1039 * Therefore we need to allocate shadow page tables in the first
1040 * 4GB of memory, which happens to fit the DMA32 zone.
1041 */
1042 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1043 if (!page)
1044 goto error_1;
1045 vcpu->mmu.pae_root = page_address(page);
1046 for (i = 0; i < 4; ++i)
1047 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1048
6aa8b732
AK		 1049 return 0;
1050
1051error_1:
1052 free_mmu_pages(vcpu);
1053 return -ENOMEM;
1054}
1055
8018c27b 1056int kvm_mmu_create(struct kvm_vcpu *vcpu)
6aa8b732 1057{
6aa8b732
AK		 1058 ASSERT(vcpu);
1059 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1060 ASSERT(list_empty(&vcpu->free_pages));
1061
8018c27b
IM		 1062 return alloc_mmu_pages(vcpu);
1063}
6aa8b732 1064
8018c27b
IM		 1065int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1066{
1067 ASSERT(vcpu);
1068 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1069 ASSERT(!list_empty(&vcpu->free_pages));
2c264957 1070
8018c27b 1071 return init_kvm_mmu(vcpu);
6aa8b732
AK		 1072}
1073
1074void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1075{
1076 ASSERT(vcpu);
1077
1078 destroy_kvm_mmu(vcpu);
1079 free_mmu_pages(vcpu);
1080}
1081
1082void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
1083{
1084 struct kvm_mmu_page *page;
1085
1086 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1087 int i;
1088 u64 *pt;
1089
1090 if (!test_bit(slot, &page->slot_bitmap))
1091 continue;
1092
1093 pt = __va(page->page_hpa);
1094 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1095 /* avoid RMW */
cd4a4e53
AK		 1096 if (pt[i] & PT_WRITABLE_MASK) {
1097 rmap_remove(kvm, &pt[i]);
6aa8b732 1098 pt[i] &= ~PT_WRITABLE_MASK;
cd4a4e53 1099 }
6aa8b732
AK		 1100 }
1101}
ubuntu-bionic-kernel mirror