4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/module.h>
47 #include <linux/gfp.h>
48 #include <linux/memblock.h>
49 #include <linux/seq_file.h>
50 #include <linux/crash_dump.h>
52 #include <trace/events/xen.h>
54 #include <asm/pgtable.h>
55 #include <asm/tlbflush.h>
56 #include <asm/fixmap.h>
57 #include <asm/mmu_context.h>
58 #include <asm/setup.h>
59 #include <asm/paravirt.h>
61 #include <asm/linkage.h>
67 #include <asm/xen/hypercall.h>
68 #include <asm/xen/hypervisor.h>
72 #include <xen/interface/xen.h>
73 #include <xen/interface/hvm/hvm_op.h>
74 #include <xen/interface/version.h>
75 #include <xen/interface/memory.h>
76 #include <xen/hvc-console.h>
78 #include "multicalls.h"
83 * Protects atomic reservation decrease/increase against concurrent increases.
84 * Also protects non-atomic updates of current_pages and balloon lists.
86 DEFINE_SPINLOCK(xen_reservation_lock
);
90 * Identity map, in addition to plain kernel map. This needs to be
91 * large enough to allocate page table pages to allocate the rest.
92 * Each page can map 2MB.
94 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
95 static RESERVE_BRK_ARRAY(pte_t
, level1_ident_pgt
, LEVEL1_IDENT_ENTRIES
);
98 /* l3 pud for userspace vsyscall mapping */
99 static pud_t level3_user_vsyscall
[PTRS_PER_PUD
] __page_aligned_bss
;
100 #endif /* CONFIG_X86_64 */
103 * Note about cr3 (pagetable base) values:
105 * xen_cr3 contains the current logical cr3 value; it contains the
106 * last set cr3. This may not be the current effective cr3, because
107 * its update may be being lazily deferred. However, a vcpu looking
108 * at its own cr3 can use this value knowing that it everything will
109 * be self-consistent.
111 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
112 * hypercall to set the vcpu cr3 is complete (so it may be a little
113 * out of date, but it will never be set early). If one vcpu is
114 * looking at another vcpu's cr3 value, it should use this variable.
116 DEFINE_PER_CPU(unsigned long, xen_cr3
); /* cr3 stored as physaddr */
117 DEFINE_PER_CPU(unsigned long, xen_current_cr3
); /* actual vcpu cr3 */
121 * Just beyond the highest usermode address. STACK_TOP_MAX has a
122 * redzone above it, so round it up to a PGD boundary.
124 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
126 unsigned long arbitrary_virt_to_mfn(void *vaddr
)
128 xmaddr_t maddr
= arbitrary_virt_to_machine(vaddr
);
130 return PFN_DOWN(maddr
.maddr
);
133 xmaddr_t
arbitrary_virt_to_machine(void *vaddr
)
135 unsigned long address
= (unsigned long)vaddr
;
141 * if the PFN is in the linear mapped vaddr range, we can just use
142 * the (quick) virt_to_machine() p2m lookup
144 if (virt_addr_valid(vaddr
))
145 return virt_to_machine(vaddr
);
147 /* otherwise we have to do a (slower) full page-table walk */
149 pte
= lookup_address(address
, &level
);
151 offset
= address
& ~PAGE_MASK
;
152 return XMADDR(((phys_addr_t
)pte_mfn(*pte
) << PAGE_SHIFT
) + offset
);
154 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine
);
156 void make_lowmem_page_readonly(void *vaddr
)
159 unsigned long address
= (unsigned long)vaddr
;
162 pte
= lookup_address(address
, &level
);
164 return; /* vaddr missing */
166 ptev
= pte_wrprotect(*pte
);
168 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
172 void make_lowmem_page_readwrite(void *vaddr
)
175 unsigned long address
= (unsigned long)vaddr
;
178 pte
= lookup_address(address
, &level
);
180 return; /* vaddr missing */
182 ptev
= pte_mkwrite(*pte
);
184 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
189 static bool xen_page_pinned(void *ptr
)
191 struct page
*page
= virt_to_page(ptr
);
193 return PagePinned(page
);
196 void xen_set_domain_pte(pte_t
*ptep
, pte_t pteval
, unsigned domid
)
198 struct multicall_space mcs
;
199 struct mmu_update
*u
;
201 trace_xen_mmu_set_domain_pte(ptep
, pteval
, domid
);
203 mcs
= xen_mc_entry(sizeof(*u
));
206 /* ptep might be kmapped when using 32-bit HIGHPTE */
207 u
->ptr
= virt_to_machine(ptep
).maddr
;
208 u
->val
= pte_val_ma(pteval
);
210 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, domid
);
212 xen_mc_issue(PARAVIRT_LAZY_MMU
);
214 EXPORT_SYMBOL_GPL(xen_set_domain_pte
);
216 static void xen_extend_mmu_update(const struct mmu_update
*update
)
218 struct multicall_space mcs
;
219 struct mmu_update
*u
;
221 mcs
= xen_mc_extend_args(__HYPERVISOR_mmu_update
, sizeof(*u
));
223 if (mcs
.mc
!= NULL
) {
226 mcs
= __xen_mc_entry(sizeof(*u
));
227 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
234 static void xen_extend_mmuext_op(const struct mmuext_op
*op
)
236 struct multicall_space mcs
;
239 mcs
= xen_mc_extend_args(__HYPERVISOR_mmuext_op
, sizeof(*u
));
241 if (mcs
.mc
!= NULL
) {
244 mcs
= __xen_mc_entry(sizeof(*u
));
245 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
252 static void xen_set_pmd_hyper(pmd_t
*ptr
, pmd_t val
)
260 /* ptr may be ioremapped for 64-bit pagetable setup */
261 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
262 u
.val
= pmd_val_ma(val
);
263 xen_extend_mmu_update(&u
);
265 xen_mc_issue(PARAVIRT_LAZY_MMU
);
270 static void xen_set_pmd(pmd_t
*ptr
, pmd_t val
)
272 trace_xen_mmu_set_pmd(ptr
, val
);
274 /* If page is not pinned, we can just update the entry
276 if (!xen_page_pinned(ptr
)) {
281 xen_set_pmd_hyper(ptr
, val
);
285 * Associate a virtual page frame with a given physical page frame
286 * and protection flags for that frame.
288 void set_pte_mfn(unsigned long vaddr
, unsigned long mfn
, pgprot_t flags
)
290 set_pte_vaddr(vaddr
, mfn_pte(mfn
, flags
));
293 static bool xen_batched_set_pte(pte_t
*ptep
, pte_t pteval
)
297 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU
)
302 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
303 u
.val
= pte_val_ma(pteval
);
304 xen_extend_mmu_update(&u
);
306 xen_mc_issue(PARAVIRT_LAZY_MMU
);
311 static inline void __xen_set_pte(pte_t
*ptep
, pte_t pteval
)
313 if (!xen_batched_set_pte(ptep
, pteval
)) {
315 * Could call native_set_pte() here and trap and
316 * emulate the PTE write but with 32-bit guests this
317 * needs two traps (one for each of the two 32-bit
318 * words in the PTE) so do one hypercall directly
323 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
324 u
.val
= pte_val_ma(pteval
);
325 HYPERVISOR_mmu_update(&u
, 1, NULL
, DOMID_SELF
);
329 static void xen_set_pte(pte_t
*ptep
, pte_t pteval
)
331 trace_xen_mmu_set_pte(ptep
, pteval
);
332 __xen_set_pte(ptep
, pteval
);
335 static void xen_set_pte_at(struct mm_struct
*mm
, unsigned long addr
,
336 pte_t
*ptep
, pte_t pteval
)
338 trace_xen_mmu_set_pte_at(mm
, addr
, ptep
, pteval
);
339 __xen_set_pte(ptep
, pteval
);
342 pte_t
xen_ptep_modify_prot_start(struct mm_struct
*mm
,
343 unsigned long addr
, pte_t
*ptep
)
345 /* Just return the pte as-is. We preserve the bits on commit */
346 trace_xen_mmu_ptep_modify_prot_start(mm
, addr
, ptep
, *ptep
);
350 void xen_ptep_modify_prot_commit(struct mm_struct
*mm
, unsigned long addr
,
351 pte_t
*ptep
, pte_t pte
)
355 trace_xen_mmu_ptep_modify_prot_commit(mm
, addr
, ptep
, pte
);
358 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_PT_UPDATE_PRESERVE_AD
;
359 u
.val
= pte_val_ma(pte
);
360 xen_extend_mmu_update(&u
);
362 xen_mc_issue(PARAVIRT_LAZY_MMU
);
365 /* Assume pteval_t is equivalent to all the other *val_t types. */
366 static pteval_t
pte_mfn_to_pfn(pteval_t val
)
368 if (val
& _PAGE_PRESENT
) {
369 unsigned long mfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
370 unsigned long pfn
= mfn_to_pfn(mfn
);
372 pteval_t flags
= val
& PTE_FLAGS_MASK
;
373 if (unlikely(pfn
== ~0))
374 val
= flags
& ~_PAGE_PRESENT
;
376 val
= ((pteval_t
)pfn
<< PAGE_SHIFT
) | flags
;
382 static pteval_t
pte_pfn_to_mfn(pteval_t val
)
384 if (val
& _PAGE_PRESENT
) {
385 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
386 pteval_t flags
= val
& PTE_FLAGS_MASK
;
389 if (!xen_feature(XENFEAT_auto_translated_physmap
))
390 mfn
= get_phys_to_machine(pfn
);
394 * If there's no mfn for the pfn, then just create an
395 * empty non-present pte. Unfortunately this loses
396 * information about the original pfn, so
397 * pte_mfn_to_pfn is asymmetric.
399 if (unlikely(mfn
== INVALID_P2M_ENTRY
)) {
403 mfn
&= ~(FOREIGN_FRAME_BIT
| IDENTITY_FRAME_BIT
);
404 val
= ((pteval_t
)mfn
<< PAGE_SHIFT
) | flags
;
410 __visible pteval_t
xen_pte_val(pte_t pte
)
412 pteval_t pteval
= pte
.pte
;
414 /* If this is a WC pte, convert back from Xen WC to Linux WC */
415 if ((pteval
& (_PAGE_PAT
| _PAGE_PCD
| _PAGE_PWT
)) == _PAGE_PAT
) {
416 WARN_ON(!pat_enabled
);
417 pteval
= (pteval
& ~_PAGE_PAT
) | _PAGE_PWT
;
420 return pte_mfn_to_pfn(pteval
);
422 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val
);
424 __visible pgdval_t
xen_pgd_val(pgd_t pgd
)
426 return pte_mfn_to_pfn(pgd
.pgd
);
428 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val
);
431 * Xen's PAT setup is part of its ABI, though I assume entries 6 & 7
432 * are reserved for now, to correspond to the Intel-reserved PAT
435 * We expect Linux's PAT set as follows:
437 * Idx PTE flags Linux Xen Default
444 * 6 PAT PCD UC- rsv UC-
445 * 7 PAT PCD PWT UC rsv UC
448 void xen_set_pat(u64 pat
)
450 /* We expect Linux to use a PAT setting of
451 * UC UC- WC WB (ignoring the PAT flag) */
452 WARN_ON(pat
!= 0x0007010600070106ull
);
455 __visible pte_t
xen_make_pte(pteval_t pte
)
458 /* If Linux is trying to set a WC pte, then map to the Xen WC.
459 * If _PAGE_PAT is set, then it probably means it is really
460 * _PAGE_PSE, so avoid fiddling with the PAT mapping and hope
461 * things work out OK...
463 * (We should never see kernel mappings with _PAGE_PSE set,
464 * but we could see hugetlbfs mappings, I think.).
466 if (pat_enabled
&& !WARN_ON(pte
& _PAGE_PAT
)) {
467 if ((pte
& (_PAGE_PCD
| _PAGE_PWT
)) == _PAGE_PWT
)
468 pte
= (pte
& ~(_PAGE_PCD
| _PAGE_PWT
)) | _PAGE_PAT
;
471 pte
= pte_pfn_to_mfn(pte
);
473 return native_make_pte(pte
);
475 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte
);
477 __visible pgd_t
xen_make_pgd(pgdval_t pgd
)
479 pgd
= pte_pfn_to_mfn(pgd
);
480 return native_make_pgd(pgd
);
482 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd
);
484 __visible pmdval_t
xen_pmd_val(pmd_t pmd
)
486 return pte_mfn_to_pfn(pmd
.pmd
);
488 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val
);
490 static void xen_set_pud_hyper(pud_t
*ptr
, pud_t val
)
498 /* ptr may be ioremapped for 64-bit pagetable setup */
499 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
500 u
.val
= pud_val_ma(val
);
501 xen_extend_mmu_update(&u
);
503 xen_mc_issue(PARAVIRT_LAZY_MMU
);
508 static void xen_set_pud(pud_t
*ptr
, pud_t val
)
510 trace_xen_mmu_set_pud(ptr
, val
);
512 /* If page is not pinned, we can just update the entry
514 if (!xen_page_pinned(ptr
)) {
519 xen_set_pud_hyper(ptr
, val
);
522 #ifdef CONFIG_X86_PAE
523 static void xen_set_pte_atomic(pte_t
*ptep
, pte_t pte
)
525 trace_xen_mmu_set_pte_atomic(ptep
, pte
);
526 set_64bit((u64
*)ptep
, native_pte_val(pte
));
529 static void xen_pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
531 trace_xen_mmu_pte_clear(mm
, addr
, ptep
);
532 if (!xen_batched_set_pte(ptep
, native_make_pte(0)))
533 native_pte_clear(mm
, addr
, ptep
);
536 static void xen_pmd_clear(pmd_t
*pmdp
)
538 trace_xen_mmu_pmd_clear(pmdp
);
539 set_pmd(pmdp
, __pmd(0));
541 #endif /* CONFIG_X86_PAE */
543 __visible pmd_t
xen_make_pmd(pmdval_t pmd
)
545 pmd
= pte_pfn_to_mfn(pmd
);
546 return native_make_pmd(pmd
);
548 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd
);
550 #if PAGETABLE_LEVELS == 4
551 __visible pudval_t
xen_pud_val(pud_t pud
)
553 return pte_mfn_to_pfn(pud
.pud
);
555 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val
);
557 __visible pud_t
xen_make_pud(pudval_t pud
)
559 pud
= pte_pfn_to_mfn(pud
);
561 return native_make_pud(pud
);
563 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud
);
565 static pgd_t
*xen_get_user_pgd(pgd_t
*pgd
)
567 pgd_t
*pgd_page
= (pgd_t
*)(((unsigned long)pgd
) & PAGE_MASK
);
568 unsigned offset
= pgd
- pgd_page
;
569 pgd_t
*user_ptr
= NULL
;
571 if (offset
< pgd_index(USER_LIMIT
)) {
572 struct page
*page
= virt_to_page(pgd_page
);
573 user_ptr
= (pgd_t
*)page
->private;
581 static void __xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
585 u
.ptr
= virt_to_machine(ptr
).maddr
;
586 u
.val
= pgd_val_ma(val
);
587 xen_extend_mmu_update(&u
);
591 * Raw hypercall-based set_pgd, intended for in early boot before
592 * there's a page structure. This implies:
593 * 1. The only existing pagetable is the kernel's
594 * 2. It is always pinned
595 * 3. It has no user pagetable attached to it
597 static void __init
xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
603 __xen_set_pgd_hyper(ptr
, val
);
605 xen_mc_issue(PARAVIRT_LAZY_MMU
);
610 static void xen_set_pgd(pgd_t
*ptr
, pgd_t val
)
612 pgd_t
*user_ptr
= xen_get_user_pgd(ptr
);
614 trace_xen_mmu_set_pgd(ptr
, user_ptr
, val
);
616 /* If page is not pinned, we can just update the entry
618 if (!xen_page_pinned(ptr
)) {
621 WARN_ON(xen_page_pinned(user_ptr
));
627 /* If it's pinned, then we can at least batch the kernel and
628 user updates together. */
631 __xen_set_pgd_hyper(ptr
, val
);
633 __xen_set_pgd_hyper(user_ptr
, val
);
635 xen_mc_issue(PARAVIRT_LAZY_MMU
);
637 #endif /* PAGETABLE_LEVELS == 4 */
640 * (Yet another) pagetable walker. This one is intended for pinning a
641 * pagetable. This means that it walks a pagetable and calls the
642 * callback function on each page it finds making up the page table,
643 * at every level. It walks the entire pagetable, but it only bothers
644 * pinning pte pages which are below limit. In the normal case this
645 * will be STACK_TOP_MAX, but at boot we need to pin up to
648 * For 32-bit the important bit is that we don't pin beyond there,
649 * because then we start getting into Xen's ptes.
651 * For 64-bit, we must skip the Xen hole in the middle of the address
652 * space, just after the big x86-64 virtual hole.
654 static int __xen_pgd_walk(struct mm_struct
*mm
, pgd_t
*pgd
,
655 int (*func
)(struct mm_struct
*mm
, struct page
*,
660 unsigned hole_low
, hole_high
;
661 unsigned pgdidx_limit
, pudidx_limit
, pmdidx_limit
;
662 unsigned pgdidx
, pudidx
, pmdidx
;
664 /* The limit is the last byte to be touched */
666 BUG_ON(limit
>= FIXADDR_TOP
);
668 if (xen_feature(XENFEAT_auto_translated_physmap
))
672 * 64-bit has a great big hole in the middle of the address
673 * space, which contains the Xen mappings. On 32-bit these
674 * will end up making a zero-sized hole and so is a no-op.
676 hole_low
= pgd_index(USER_LIMIT
);
677 hole_high
= pgd_index(PAGE_OFFSET
);
679 pgdidx_limit
= pgd_index(limit
);
681 pudidx_limit
= pud_index(limit
);
686 pmdidx_limit
= pmd_index(limit
);
691 for (pgdidx
= 0; pgdidx
<= pgdidx_limit
; pgdidx
++) {
694 if (pgdidx
>= hole_low
&& pgdidx
< hole_high
)
697 if (!pgd_val(pgd
[pgdidx
]))
700 pud
= pud_offset(&pgd
[pgdidx
], 0);
702 if (PTRS_PER_PUD
> 1) /* not folded */
703 flush
|= (*func
)(mm
, virt_to_page(pud
), PT_PUD
);
705 for (pudidx
= 0; pudidx
< PTRS_PER_PUD
; pudidx
++) {
708 if (pgdidx
== pgdidx_limit
&&
709 pudidx
> pudidx_limit
)
712 if (pud_none(pud
[pudidx
]))
715 pmd
= pmd_offset(&pud
[pudidx
], 0);
717 if (PTRS_PER_PMD
> 1) /* not folded */
718 flush
|= (*func
)(mm
, virt_to_page(pmd
), PT_PMD
);
720 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
; pmdidx
++) {
723 if (pgdidx
== pgdidx_limit
&&
724 pudidx
== pudidx_limit
&&
725 pmdidx
> pmdidx_limit
)
728 if (pmd_none(pmd
[pmdidx
]))
731 pte
= pmd_page(pmd
[pmdidx
]);
732 flush
|= (*func
)(mm
, pte
, PT_PTE
);
738 /* Do the top level last, so that the callbacks can use it as
739 a cue to do final things like tlb flushes. */
740 flush
|= (*func
)(mm
, virt_to_page(pgd
), PT_PGD
);
745 static int xen_pgd_walk(struct mm_struct
*mm
,
746 int (*func
)(struct mm_struct
*mm
, struct page
*,
750 return __xen_pgd_walk(mm
, mm
->pgd
, func
, limit
);
753 /* If we're using split pte locks, then take the page's lock and
754 return a pointer to it. Otherwise return NULL. */
755 static spinlock_t
*xen_pte_lock(struct page
*page
, struct mm_struct
*mm
)
757 spinlock_t
*ptl
= NULL
;
759 #if USE_SPLIT_PTE_PTLOCKS
760 ptl
= ptlock_ptr(page
);
761 spin_lock_nest_lock(ptl
, &mm
->page_table_lock
);
767 static void xen_pte_unlock(void *v
)
773 static void xen_do_pin(unsigned level
, unsigned long pfn
)
778 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
780 xen_extend_mmuext_op(&op
);
783 static int xen_pin_page(struct mm_struct
*mm
, struct page
*page
,
786 unsigned pgfl
= TestSetPagePinned(page
);
790 flush
= 0; /* already pinned */
791 else if (PageHighMem(page
))
792 /* kmaps need flushing if we found an unpinned
796 void *pt
= lowmem_page_address(page
);
797 unsigned long pfn
= page_to_pfn(page
);
798 struct multicall_space mcs
= __xen_mc_entry(0);
804 * We need to hold the pagetable lock between the time
805 * we make the pagetable RO and when we actually pin
806 * it. If we don't, then other users may come in and
807 * attempt to update the pagetable by writing it,
808 * which will fail because the memory is RO but not
809 * pinned, so Xen won't do the trap'n'emulate.
811 * If we're using split pte locks, we can't hold the
812 * entire pagetable's worth of locks during the
813 * traverse, because we may wrap the preempt count (8
814 * bits). The solution is to mark RO and pin each PTE
815 * page while holding the lock. This means the number
816 * of locks we end up holding is never more than a
817 * batch size (~32 entries, at present).
819 * If we're not using split pte locks, we needn't pin
820 * the PTE pages independently, because we're
821 * protected by the overall pagetable lock.
825 ptl
= xen_pte_lock(page
, mm
);
827 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
828 pfn_pte(pfn
, PAGE_KERNEL_RO
),
829 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
832 xen_do_pin(MMUEXT_PIN_L1_TABLE
, pfn
);
834 /* Queue a deferred unlock for when this batch
836 xen_mc_callback(xen_pte_unlock
, ptl
);
843 /* This is called just after a mm has been created, but it has not
844 been used yet. We need to make sure that its pagetable is all
845 read-only, and can be pinned. */
846 static void __xen_pgd_pin(struct mm_struct
*mm
, pgd_t
*pgd
)
848 trace_xen_mmu_pgd_pin(mm
, pgd
);
852 if (__xen_pgd_walk(mm
, pgd
, xen_pin_page
, USER_LIMIT
)) {
853 /* re-enable interrupts for flushing */
863 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
865 xen_do_pin(MMUEXT_PIN_L4_TABLE
, PFN_DOWN(__pa(pgd
)));
868 xen_pin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
869 xen_do_pin(MMUEXT_PIN_L4_TABLE
,
870 PFN_DOWN(__pa(user_pgd
)));
873 #else /* CONFIG_X86_32 */
874 #ifdef CONFIG_X86_PAE
875 /* Need to make sure unshared kernel PMD is pinnable */
876 xen_pin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
879 xen_do_pin(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(__pa(pgd
)));
880 #endif /* CONFIG_X86_64 */
884 static void xen_pgd_pin(struct mm_struct
*mm
)
886 __xen_pgd_pin(mm
, mm
->pgd
);
890 * On save, we need to pin all pagetables to make sure they get their
891 * mfns turned into pfns. Search the list for any unpinned pgds and pin
892 * them (unpinned pgds are not currently in use, probably because the
893 * process is under construction or destruction).
895 * Expected to be called in stop_machine() ("equivalent to taking
896 * every spinlock in the system"), so the locking doesn't really
897 * matter all that much.
899 void xen_mm_pin_all(void)
903 spin_lock(&pgd_lock
);
905 list_for_each_entry(page
, &pgd_list
, lru
) {
906 if (!PagePinned(page
)) {
907 __xen_pgd_pin(&init_mm
, (pgd_t
*)page_address(page
));
908 SetPageSavePinned(page
);
912 spin_unlock(&pgd_lock
);
916 * The init_mm pagetable is really pinned as soon as its created, but
917 * that's before we have page structures to store the bits. So do all
918 * the book-keeping now.
920 static int __init
xen_mark_pinned(struct mm_struct
*mm
, struct page
*page
,
927 static void __init
xen_mark_init_mm_pinned(void)
929 xen_pgd_walk(&init_mm
, xen_mark_pinned
, FIXADDR_TOP
);
932 static int xen_unpin_page(struct mm_struct
*mm
, struct page
*page
,
935 unsigned pgfl
= TestClearPagePinned(page
);
937 if (pgfl
&& !PageHighMem(page
)) {
938 void *pt
= lowmem_page_address(page
);
939 unsigned long pfn
= page_to_pfn(page
);
940 spinlock_t
*ptl
= NULL
;
941 struct multicall_space mcs
;
944 * Do the converse to pin_page. If we're using split
945 * pte locks, we must be holding the lock for while
946 * the pte page is unpinned but still RO to prevent
947 * concurrent updates from seeing it in this
948 * partially-pinned state.
950 if (level
== PT_PTE
) {
951 ptl
= xen_pte_lock(page
, mm
);
954 xen_do_pin(MMUEXT_UNPIN_TABLE
, pfn
);
957 mcs
= __xen_mc_entry(0);
959 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
960 pfn_pte(pfn
, PAGE_KERNEL
),
961 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
964 /* unlock when batch completed */
965 xen_mc_callback(xen_pte_unlock
, ptl
);
969 return 0; /* never need to flush on unpin */
972 /* Release a pagetables pages back as normal RW */
973 static void __xen_pgd_unpin(struct mm_struct
*mm
, pgd_t
*pgd
)
975 trace_xen_mmu_pgd_unpin(mm
, pgd
);
979 xen_do_pin(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
983 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
986 xen_do_pin(MMUEXT_UNPIN_TABLE
,
987 PFN_DOWN(__pa(user_pgd
)));
988 xen_unpin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
993 #ifdef CONFIG_X86_PAE
994 /* Need to make sure unshared kernel PMD is unpinned */
995 xen_unpin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
999 __xen_pgd_walk(mm
, pgd
, xen_unpin_page
, USER_LIMIT
);
1004 static void xen_pgd_unpin(struct mm_struct
*mm
)
1006 __xen_pgd_unpin(mm
, mm
->pgd
);
1010 * On resume, undo any pinning done at save, so that the rest of the
1011 * kernel doesn't see any unexpected pinned pagetables.
1013 void xen_mm_unpin_all(void)
1017 spin_lock(&pgd_lock
);
1019 list_for_each_entry(page
, &pgd_list
, lru
) {
1020 if (PageSavePinned(page
)) {
1021 BUG_ON(!PagePinned(page
));
1022 __xen_pgd_unpin(&init_mm
, (pgd_t
*)page_address(page
));
1023 ClearPageSavePinned(page
);
1027 spin_unlock(&pgd_lock
);
1030 static void xen_activate_mm(struct mm_struct
*prev
, struct mm_struct
*next
)
1032 spin_lock(&next
->page_table_lock
);
1034 spin_unlock(&next
->page_table_lock
);
1037 static void xen_dup_mmap(struct mm_struct
*oldmm
, struct mm_struct
*mm
)
1039 spin_lock(&mm
->page_table_lock
);
1041 spin_unlock(&mm
->page_table_lock
);
1046 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1047 we need to repoint it somewhere else before we can unpin it. */
1048 static void drop_other_mm_ref(void *info
)
1050 struct mm_struct
*mm
= info
;
1051 struct mm_struct
*active_mm
;
1053 active_mm
= this_cpu_read(cpu_tlbstate
.active_mm
);
1055 if (active_mm
== mm
&& this_cpu_read(cpu_tlbstate
.state
) != TLBSTATE_OK
)
1056 leave_mm(smp_processor_id());
1058 /* If this cpu still has a stale cr3 reference, then make sure
1059 it has been flushed. */
1060 if (this_cpu_read(xen_current_cr3
) == __pa(mm
->pgd
))
1061 load_cr3(swapper_pg_dir
);
1064 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1069 if (current
->active_mm
== mm
) {
1070 if (current
->mm
== mm
)
1071 load_cr3(swapper_pg_dir
);
1073 leave_mm(smp_processor_id());
1076 /* Get the "official" set of cpus referring to our pagetable. */
1077 if (!alloc_cpumask_var(&mask
, GFP_ATOMIC
)) {
1078 for_each_online_cpu(cpu
) {
1079 if (!cpumask_test_cpu(cpu
, mm_cpumask(mm
))
1080 && per_cpu(xen_current_cr3
, cpu
) != __pa(mm
->pgd
))
1082 smp_call_function_single(cpu
, drop_other_mm_ref
, mm
, 1);
1086 cpumask_copy(mask
, mm_cpumask(mm
));
1088 /* It's possible that a vcpu may have a stale reference to our
1089 cr3, because its in lazy mode, and it hasn't yet flushed
1090 its set of pending hypercalls yet. In this case, we can
1091 look at its actual current cr3 value, and force it to flush
1093 for_each_online_cpu(cpu
) {
1094 if (per_cpu(xen_current_cr3
, cpu
) == __pa(mm
->pgd
))
1095 cpumask_set_cpu(cpu
, mask
);
1098 if (!cpumask_empty(mask
))
1099 smp_call_function_many(mask
, drop_other_mm_ref
, mm
, 1);
1100 free_cpumask_var(mask
);
1103 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1105 if (current
->active_mm
== mm
)
1106 load_cr3(swapper_pg_dir
);
1111 * While a process runs, Xen pins its pagetables, which means that the
1112 * hypervisor forces it to be read-only, and it controls all updates
1113 * to it. This means that all pagetable updates have to go via the
1114 * hypervisor, which is moderately expensive.
1116 * Since we're pulling the pagetable down, we switch to use init_mm,
1117 * unpin old process pagetable and mark it all read-write, which
1118 * allows further operations on it to be simple memory accesses.
1120 * The only subtle point is that another CPU may be still using the
1121 * pagetable because of lazy tlb flushing. This means we need need to
1122 * switch all CPUs off this pagetable before we can unpin it.
1124 static void xen_exit_mmap(struct mm_struct
*mm
)
1126 get_cpu(); /* make sure we don't move around */
1127 xen_drop_mm_ref(mm
);
1130 spin_lock(&mm
->page_table_lock
);
1132 /* pgd may not be pinned in the error exit path of execve */
1133 if (xen_page_pinned(mm
->pgd
))
1136 spin_unlock(&mm
->page_table_lock
);
1139 static void xen_post_allocator_init(void);
1141 #ifdef CONFIG_X86_64
1142 static void __init
xen_cleanhighmap(unsigned long vaddr
,
1143 unsigned long vaddr_end
)
1145 unsigned long kernel_end
= roundup((unsigned long)_brk_end
, PMD_SIZE
) - 1;
1146 pmd_t
*pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
1148 /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1149 * We include the PMD passed in on _both_ boundaries. */
1150 for (; vaddr
<= vaddr_end
&& (pmd
< (level2_kernel_pgt
+ PAGE_SIZE
));
1151 pmd
++, vaddr
+= PMD_SIZE
) {
1154 if (vaddr
< (unsigned long) _text
|| vaddr
> kernel_end
)
1155 set_pmd(pmd
, __pmd(0));
1157 /* In case we did something silly, we should crash in this function
1158 * instead of somewhere later and be confusing. */
1161 static void __init
xen_pagetable_p2m_copy(void)
1165 unsigned long new_mfn_list
;
1167 if (xen_feature(XENFEAT_auto_translated_physmap
))
1170 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
1172 new_mfn_list
= xen_revector_p2m_tree();
1173 /* No memory or already called. */
1174 if (!new_mfn_list
|| new_mfn_list
== xen_start_info
->mfn_list
)
1177 /* using __ka address and sticking INVALID_P2M_ENTRY! */
1178 memset((void *)xen_start_info
->mfn_list
, 0xff, size
);
1180 /* We should be in __ka space. */
1181 BUG_ON(xen_start_info
->mfn_list
< __START_KERNEL_map
);
1182 addr
= xen_start_info
->mfn_list
;
1183 /* We roundup to the PMD, which means that if anybody at this stage is
1184 * using the __ka address of xen_start_info or xen_start_info->shared_info
1185 * they are in going to crash. Fortunatly we have already revectored
1186 * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */
1187 size
= roundup(size
, PMD_SIZE
);
1188 xen_cleanhighmap(addr
, addr
+ size
);
1190 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
1191 memblock_free(__pa(xen_start_info
->mfn_list
), size
);
1192 /* And revector! Bye bye old array */
1193 xen_start_info
->mfn_list
= new_mfn_list
;
1195 /* At this stage, cleanup_highmap has already cleaned __ka space
1196 * from _brk_limit way up to the max_pfn_mapped (which is the end of
1197 * the ramdisk). We continue on, erasing PMD entries that point to page
1198 * tables - do note that they are accessible at this stage via __va.
1199 * For good measure we also round up to the PMD - which means that if
1200 * anybody is using __ka address to the initial boot-stack - and try
1201 * to use it - they are going to crash. The xen_start_info has been
1202 * taken care of already in xen_setup_kernel_pagetable. */
1203 addr
= xen_start_info
->pt_base
;
1204 size
= roundup(xen_start_info
->nr_pt_frames
* PAGE_SIZE
, PMD_SIZE
);
1206 xen_cleanhighmap(addr
, addr
+ size
);
1207 xen_start_info
->pt_base
= (unsigned long)__va(__pa(xen_start_info
->pt_base
));
1209 /* This is superflous and is not neccessary, but you know what
1210 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1211 * anything at this stage. */
1212 xen_cleanhighmap(MODULES_VADDR
, roundup(MODULES_VADDR
, PUD_SIZE
) - 1);
1217 static void __init
xen_pagetable_init(void)
1220 xen_setup_shared_info();
1221 #ifdef CONFIG_X86_64
1222 xen_pagetable_p2m_copy();
1224 xen_post_allocator_init();
1226 static void xen_write_cr2(unsigned long cr2
)
1228 this_cpu_read(xen_vcpu
)->arch
.cr2
= cr2
;
1231 static unsigned long xen_read_cr2(void)
1233 return this_cpu_read(xen_vcpu
)->arch
.cr2
;
1236 unsigned long xen_read_cr2_direct(void)
1238 return this_cpu_read(xen_vcpu_info
.arch
.cr2
);
1241 void xen_flush_tlb_all(void)
1243 struct mmuext_op
*op
;
1244 struct multicall_space mcs
;
1246 trace_xen_mmu_flush_tlb_all(0);
1250 mcs
= xen_mc_entry(sizeof(*op
));
1253 op
->cmd
= MMUEXT_TLB_FLUSH_ALL
;
1254 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1256 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1260 static void xen_flush_tlb(void)
1262 struct mmuext_op
*op
;
1263 struct multicall_space mcs
;
1265 trace_xen_mmu_flush_tlb(0);
1269 mcs
= xen_mc_entry(sizeof(*op
));
1272 op
->cmd
= MMUEXT_TLB_FLUSH_LOCAL
;
1273 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1275 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1280 static void xen_flush_tlb_single(unsigned long addr
)
1282 struct mmuext_op
*op
;
1283 struct multicall_space mcs
;
1285 trace_xen_mmu_flush_tlb_single(addr
);
1289 mcs
= xen_mc_entry(sizeof(*op
));
1291 op
->cmd
= MMUEXT_INVLPG_LOCAL
;
1292 op
->arg1
.linear_addr
= addr
& PAGE_MASK
;
1293 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1295 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1300 static void xen_flush_tlb_others(const struct cpumask
*cpus
,
1301 struct mm_struct
*mm
, unsigned long start
,
1305 struct mmuext_op op
;
1307 DECLARE_BITMAP(mask
, num_processors
);
1309 DECLARE_BITMAP(mask
, NR_CPUS
);
1312 struct multicall_space mcs
;
1314 trace_xen_mmu_flush_tlb_others(cpus
, mm
, start
, end
);
1316 if (cpumask_empty(cpus
))
1317 return; /* nothing to do */
1319 mcs
= xen_mc_entry(sizeof(*args
));
1321 args
->op
.arg2
.vcpumask
= to_cpumask(args
->mask
);
1323 /* Remove us, and any offline CPUS. */
1324 cpumask_and(to_cpumask(args
->mask
), cpus
, cpu_online_mask
);
1325 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args
->mask
));
1327 args
->op
.cmd
= MMUEXT_TLB_FLUSH_MULTI
;
1328 if (end
!= TLB_FLUSH_ALL
&& (end
- start
) <= PAGE_SIZE
) {
1329 args
->op
.cmd
= MMUEXT_INVLPG_MULTI
;
1330 args
->op
.arg1
.linear_addr
= start
;
1333 MULTI_mmuext_op(mcs
.mc
, &args
->op
, 1, NULL
, DOMID_SELF
);
1335 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1338 static unsigned long xen_read_cr3(void)
1340 return this_cpu_read(xen_cr3
);
1343 static void set_current_cr3(void *v
)
1345 this_cpu_write(xen_current_cr3
, (unsigned long)v
);
1348 static void __xen_write_cr3(bool kernel
, unsigned long cr3
)
1350 struct mmuext_op op
;
1353 trace_xen_mmu_write_cr3(kernel
, cr3
);
1356 mfn
= pfn_to_mfn(PFN_DOWN(cr3
));
1360 WARN_ON(mfn
== 0 && kernel
);
1362 op
.cmd
= kernel
? MMUEXT_NEW_BASEPTR
: MMUEXT_NEW_USER_BASEPTR
;
1365 xen_extend_mmuext_op(&op
);
1368 this_cpu_write(xen_cr3
, cr3
);
1370 /* Update xen_current_cr3 once the batch has actually
1372 xen_mc_callback(set_current_cr3
, (void *)cr3
);
1375 static void xen_write_cr3(unsigned long cr3
)
1377 BUG_ON(preemptible());
1379 xen_mc_batch(); /* disables interrupts */
1381 /* Update while interrupts are disabled, so its atomic with
1383 this_cpu_write(xen_cr3
, cr3
);
1385 __xen_write_cr3(true, cr3
);
1387 #ifdef CONFIG_X86_64
1389 pgd_t
*user_pgd
= xen_get_user_pgd(__va(cr3
));
1391 __xen_write_cr3(false, __pa(user_pgd
));
1393 __xen_write_cr3(false, 0);
1397 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1400 #ifdef CONFIG_X86_64
1402 * At the start of the day - when Xen launches a guest, it has already
1403 * built pagetables for the guest. We diligently look over them
1404 * in xen_setup_kernel_pagetable and graft as appropiate them in the
1405 * init_level4_pgt and its friends. Then when we are happy we load
1406 * the new init_level4_pgt - and continue on.
1408 * The generic code starts (start_kernel) and 'init_mem_mapping' sets
1409 * up the rest of the pagetables. When it has completed it loads the cr3.
1410 * N.B. that baremetal would start at 'start_kernel' (and the early
1411 * #PF handler would create bootstrap pagetables) - so we are running
1412 * with the same assumptions as what to do when write_cr3 is executed
1415 * Since there are no user-page tables at all, we have two variants
1416 * of xen_write_cr3 - the early bootup (this one), and the late one
1417 * (xen_write_cr3). The reason we have to do that is that in 64-bit
1418 * the Linux kernel and user-space are both in ring 3 while the
1419 * hypervisor is in ring 0.
1421 static void __init
xen_write_cr3_init(unsigned long cr3
)
1423 BUG_ON(preemptible());
1425 xen_mc_batch(); /* disables interrupts */
1427 /* Update while interrupts are disabled, so its atomic with
1429 this_cpu_write(xen_cr3
, cr3
);
1431 __xen_write_cr3(true, cr3
);
1433 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1437 static int xen_pgd_alloc(struct mm_struct
*mm
)
1439 pgd_t
*pgd
= mm
->pgd
;
1442 BUG_ON(PagePinned(virt_to_page(pgd
)));
1444 #ifdef CONFIG_X86_64
1446 struct page
*page
= virt_to_page(pgd
);
1449 BUG_ON(page
->private != 0);
1453 user_pgd
= (pgd_t
*)__get_free_page(GFP_KERNEL
| __GFP_ZERO
);
1454 page
->private = (unsigned long)user_pgd
;
1456 if (user_pgd
!= NULL
) {
1457 user_pgd
[pgd_index(VSYSCALL_ADDR
)] =
1458 __pgd(__pa(level3_user_vsyscall
) | _PAGE_TABLE
);
1462 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd
))));
1469 static void xen_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
1471 #ifdef CONFIG_X86_64
1472 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1475 free_page((unsigned long)user_pgd
);
1479 #ifdef CONFIG_X86_32
1480 static pte_t __init
mask_rw_pte(pte_t
*ptep
, pte_t pte
)
1482 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1483 if (pte_val_ma(*ptep
) & _PAGE_PRESENT
)
1484 pte
= __pte_ma(((pte_val_ma(*ptep
) & _PAGE_RW
) | ~_PAGE_RW
) &
1489 #else /* CONFIG_X86_64 */
1490 static pte_t __init
mask_rw_pte(pte_t
*ptep
, pte_t pte
)
1494 #endif /* CONFIG_X86_64 */
1497 * Init-time set_pte while constructing initial pagetables, which
1498 * doesn't allow RO page table pages to be remapped RW.
1500 * If there is no MFN for this PFN then this page is initially
1501 * ballooned out so clear the PTE (as in decrease_reservation() in
1502 * drivers/xen/balloon.c).
1504 * Many of these PTE updates are done on unpinned and writable pages
1505 * and doing a hypercall for these is unnecessary and expensive. At
1506 * this point it is not possible to tell if a page is pinned or not,
1507 * so always write the PTE directly and rely on Xen trapping and
1508 * emulating any updates as necessary.
1510 static void __init
xen_set_pte_init(pte_t
*ptep
, pte_t pte
)
1512 if (pte_mfn(pte
) != INVALID_P2M_ENTRY
)
1513 pte
= mask_rw_pte(ptep
, pte
);
1517 native_set_pte(ptep
, pte
);
1520 static void pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1522 struct mmuext_op op
;
1524 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
1525 if (HYPERVISOR_mmuext_op(&op
, 1, NULL
, DOMID_SELF
))
1529 /* Early in boot, while setting up the initial pagetable, assume
1530 everything is pinned. */
1531 static void __init
xen_alloc_pte_init(struct mm_struct
*mm
, unsigned long pfn
)
1533 #ifdef CONFIG_FLATMEM
1534 BUG_ON(mem_map
); /* should only be used early */
1536 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1537 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1540 /* Used for pmd and pud */
1541 static void __init
xen_alloc_pmd_init(struct mm_struct
*mm
, unsigned long pfn
)
1543 #ifdef CONFIG_FLATMEM
1544 BUG_ON(mem_map
); /* should only be used early */
1546 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1549 /* Early release_pte assumes that all pts are pinned, since there's
1550 only init_mm and anything attached to that is pinned. */
1551 static void __init
xen_release_pte_init(unsigned long pfn
)
1553 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1554 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1557 static void __init
xen_release_pmd_init(unsigned long pfn
)
1559 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1562 static inline void __pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1564 struct multicall_space mcs
;
1565 struct mmuext_op
*op
;
1567 mcs
= __xen_mc_entry(sizeof(*op
));
1570 op
->arg1
.mfn
= pfn_to_mfn(pfn
);
1572 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
1575 static inline void __set_pfn_prot(unsigned long pfn
, pgprot_t prot
)
1577 struct multicall_space mcs
;
1578 unsigned long addr
= (unsigned long)__va(pfn
<< PAGE_SHIFT
);
1580 mcs
= __xen_mc_entry(0);
1581 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)addr
,
1582 pfn_pte(pfn
, prot
), 0);
1585 /* This needs to make sure the new pte page is pinned iff its being
1586 attached to a pinned pagetable. */
1587 static inline void xen_alloc_ptpage(struct mm_struct
*mm
, unsigned long pfn
,
1590 bool pinned
= PagePinned(virt_to_page(mm
->pgd
));
1592 trace_xen_mmu_alloc_ptpage(mm
, pfn
, level
, pinned
);
1595 struct page
*page
= pfn_to_page(pfn
);
1597 SetPagePinned(page
);
1599 if (!PageHighMem(page
)) {
1602 __set_pfn_prot(pfn
, PAGE_KERNEL_RO
);
1604 if (level
== PT_PTE
&& USE_SPLIT_PTE_PTLOCKS
)
1605 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1607 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1609 /* make sure there are no stray mappings of
1611 kmap_flush_unused();
1616 static void xen_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
)
1618 xen_alloc_ptpage(mm
, pfn
, PT_PTE
);
1621 static void xen_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
)
1623 xen_alloc_ptpage(mm
, pfn
, PT_PMD
);
1626 /* This should never happen until we're OK to use struct page */
1627 static inline void xen_release_ptpage(unsigned long pfn
, unsigned level
)
1629 struct page
*page
= pfn_to_page(pfn
);
1630 bool pinned
= PagePinned(page
);
1632 trace_xen_mmu_release_ptpage(pfn
, level
, pinned
);
1635 if (!PageHighMem(page
)) {
1638 if (level
== PT_PTE
&& USE_SPLIT_PTE_PTLOCKS
)
1639 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1641 __set_pfn_prot(pfn
, PAGE_KERNEL
);
1643 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1645 ClearPagePinned(page
);
1649 static void xen_release_pte(unsigned long pfn
)
1651 xen_release_ptpage(pfn
, PT_PTE
);
1654 static void xen_release_pmd(unsigned long pfn
)
1656 xen_release_ptpage(pfn
, PT_PMD
);
1659 #if PAGETABLE_LEVELS == 4
1660 static void xen_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
)
1662 xen_alloc_ptpage(mm
, pfn
, PT_PUD
);
1665 static void xen_release_pud(unsigned long pfn
)
1667 xen_release_ptpage(pfn
, PT_PUD
);
1671 void __init
xen_reserve_top(void)
1673 #ifdef CONFIG_X86_32
1674 unsigned long top
= HYPERVISOR_VIRT_START
;
1675 struct xen_platform_parameters pp
;
1677 if (HYPERVISOR_xen_version(XENVER_platform_parameters
, &pp
) == 0)
1678 top
= pp
.virt_start
;
1680 reserve_top_address(-top
);
1681 #endif /* CONFIG_X86_32 */
1685 * Like __va(), but returns address in the kernel mapping (which is
1686 * all we have until the physical memory mapping has been set up.
1688 static void *__ka(phys_addr_t paddr
)
1690 #ifdef CONFIG_X86_64
1691 return (void *)(paddr
+ __START_KERNEL_map
);
1697 /* Convert a machine address to physical address */
1698 static unsigned long m2p(phys_addr_t maddr
)
1702 maddr
&= PTE_PFN_MASK
;
1703 paddr
= mfn_to_pfn(maddr
>> PAGE_SHIFT
) << PAGE_SHIFT
;
1708 /* Convert a machine address to kernel virtual */
1709 static void *m2v(phys_addr_t maddr
)
1711 return __ka(m2p(maddr
));
1714 /* Set the page permissions on an identity-mapped pages */
1715 static void set_page_prot_flags(void *addr
, pgprot_t prot
, unsigned long flags
)
1717 unsigned long pfn
= __pa(addr
) >> PAGE_SHIFT
;
1718 pte_t pte
= pfn_pte(pfn
, prot
);
1720 /* For PVH no need to set R/O or R/W to pin them or unpin them. */
1721 if (xen_feature(XENFEAT_auto_translated_physmap
))
1724 if (HYPERVISOR_update_va_mapping((unsigned long)addr
, pte
, flags
))
1727 static void set_page_prot(void *addr
, pgprot_t prot
)
1729 return set_page_prot_flags(addr
, prot
, UVMF_NONE
);
1731 #ifdef CONFIG_X86_32
1732 static void __init
xen_map_identity_early(pmd_t
*pmd
, unsigned long max_pfn
)
1734 unsigned pmdidx
, pteidx
;
1738 level1_ident_pgt
= extend_brk(sizeof(pte_t
) * LEVEL1_IDENT_ENTRIES
,
1743 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
&& pfn
< max_pfn
; pmdidx
++) {
1746 /* Reuse or allocate a page of ptes */
1747 if (pmd_present(pmd
[pmdidx
]))
1748 pte_page
= m2v(pmd
[pmdidx
].pmd
);
1750 /* Check for free pte pages */
1751 if (ident_pte
== LEVEL1_IDENT_ENTRIES
)
1754 pte_page
= &level1_ident_pgt
[ident_pte
];
1755 ident_pte
+= PTRS_PER_PTE
;
1757 pmd
[pmdidx
] = __pmd(__pa(pte_page
) | _PAGE_TABLE
);
1760 /* Install mappings */
1761 for (pteidx
= 0; pteidx
< PTRS_PER_PTE
; pteidx
++, pfn
++) {
1764 #ifdef CONFIG_X86_32
1765 if (pfn
> max_pfn_mapped
)
1766 max_pfn_mapped
= pfn
;
1769 if (!pte_none(pte_page
[pteidx
]))
1772 pte
= pfn_pte(pfn
, PAGE_KERNEL_EXEC
);
1773 pte_page
[pteidx
] = pte
;
1777 for (pteidx
= 0; pteidx
< ident_pte
; pteidx
+= PTRS_PER_PTE
)
1778 set_page_prot(&level1_ident_pgt
[pteidx
], PAGE_KERNEL_RO
);
1780 set_page_prot(pmd
, PAGE_KERNEL_RO
);
1783 void __init
xen_setup_machphys_mapping(void)
1785 struct xen_machphys_mapping mapping
;
1787 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping
, &mapping
) == 0) {
1788 machine_to_phys_mapping
= (unsigned long *)mapping
.v_start
;
1789 machine_to_phys_nr
= mapping
.max_mfn
+ 1;
1791 machine_to_phys_nr
= MACH2PHYS_NR_ENTRIES
;
1793 #ifdef CONFIG_X86_32
1794 WARN_ON((machine_to_phys_mapping
+ (machine_to_phys_nr
- 1))
1795 < machine_to_phys_mapping
);
1799 #ifdef CONFIG_X86_64
1800 static void convert_pfn_mfn(void *v
)
1805 /* All levels are converted the same way, so just treat them
1807 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
1808 pte
[i
] = xen_make_pte(pte
[i
].pte
);
1810 static void __init
check_pt_base(unsigned long *pt_base
, unsigned long *pt_end
,
1813 if (*pt_base
== PFN_DOWN(__pa(addr
))) {
1814 set_page_prot_flags((void *)addr
, PAGE_KERNEL
, UVMF_INVLPG
);
1815 clear_page((void *)addr
);
1818 if (*pt_end
== PFN_DOWN(__pa(addr
))) {
1819 set_page_prot_flags((void *)addr
, PAGE_KERNEL
, UVMF_INVLPG
);
1820 clear_page((void *)addr
);
1825 * Set up the initial kernel pagetable.
1827 * We can construct this by grafting the Xen provided pagetable into
1828 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1829 * level2_ident_pgt, and level2_kernel_pgt. This means that only the
1830 * kernel has a physical mapping to start with - but that's enough to
1831 * get __va working. We need to fill in the rest of the physical
1832 * mapping once some sort of allocator has been set up. NOTE: for
1833 * PVH, the page tables are native.
1835 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
1839 unsigned long addr
[3];
1840 unsigned long pt_base
, pt_end
;
1843 /* max_pfn_mapped is the last pfn mapped in the initial memory
1844 * mappings. Considering that on Xen after the kernel mappings we
1845 * have the mappings of some pages that don't exist in pfn space, we
1846 * set max_pfn_mapped to the last real pfn mapped. */
1847 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->mfn_list
));
1849 pt_base
= PFN_DOWN(__pa(xen_start_info
->pt_base
));
1850 pt_end
= pt_base
+ xen_start_info
->nr_pt_frames
;
1852 /* Zap identity mapping */
1853 init_level4_pgt
[0] = __pgd(0);
1855 if (!xen_feature(XENFEAT_auto_translated_physmap
)) {
1856 /* Pre-constructed entries are in pfn, so convert to mfn */
1857 /* L4[272] -> level3_ident_pgt
1858 * L4[511] -> level3_kernel_pgt */
1859 convert_pfn_mfn(init_level4_pgt
);
1861 /* L3_i[0] -> level2_ident_pgt */
1862 convert_pfn_mfn(level3_ident_pgt
);
1863 /* L3_k[510] -> level2_kernel_pgt
1864 * L3_k[511] -> level2_fixmap_pgt */
1865 convert_pfn_mfn(level3_kernel_pgt
);
1867 /* L3_k[511][506] -> level1_fixmap_pgt */
1868 convert_pfn_mfn(level2_fixmap_pgt
);
1870 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1871 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
)].pgd
);
1872 l2
= m2v(l3
[pud_index(__START_KERNEL_map
)].pud
);
1874 addr
[0] = (unsigned long)pgd
;
1875 addr
[1] = (unsigned long)l3
;
1876 addr
[2] = (unsigned long)l2
;
1877 /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
1878 * Both L4[272][0] and L4[511][510] have entries that point to the same
1879 * L2 (PMD) tables. Meaning that if you modify it in __va space
1880 * it will be also modified in the __ka space! (But if you just
1881 * modify the PMD table to point to other PTE's or none, then you
1882 * are OK - which is what cleanup_highmap does) */
1883 copy_page(level2_ident_pgt
, l2
);
1884 /* Graft it onto L4[511][510] */
1885 copy_page(level2_kernel_pgt
, l2
);
1887 if (!xen_feature(XENFEAT_auto_translated_physmap
)) {
1888 /* Make pagetable pieces RO */
1889 set_page_prot(init_level4_pgt
, PAGE_KERNEL_RO
);
1890 set_page_prot(level3_ident_pgt
, PAGE_KERNEL_RO
);
1891 set_page_prot(level3_kernel_pgt
, PAGE_KERNEL_RO
);
1892 set_page_prot(level3_user_vsyscall
, PAGE_KERNEL_RO
);
1893 set_page_prot(level2_ident_pgt
, PAGE_KERNEL_RO
);
1894 set_page_prot(level2_kernel_pgt
, PAGE_KERNEL_RO
);
1895 set_page_prot(level2_fixmap_pgt
, PAGE_KERNEL_RO
);
1896 set_page_prot(level1_fixmap_pgt
, PAGE_KERNEL_RO
);
1898 /* Pin down new L4 */
1899 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE
,
1900 PFN_DOWN(__pa_symbol(init_level4_pgt
)));
1902 /* Unpin Xen-provided one */
1903 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1906 * At this stage there can be no user pgd, and no page
1907 * structure to attach it to, so make sure we just set kernel
1911 __xen_write_cr3(true, __pa(init_level4_pgt
));
1912 xen_mc_issue(PARAVIRT_LAZY_CPU
);
1914 native_write_cr3(__pa(init_level4_pgt
));
1916 /* We can't that easily rip out L3 and L2, as the Xen pagetables are
1917 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for
1918 * the initial domain. For guests using the toolstack, they are in:
1919 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
1920 * rip out the [L4] (pgd), but for guests we shave off three pages.
1922 for (i
= 0; i
< ARRAY_SIZE(addr
); i
++)
1923 check_pt_base(&pt_base
, &pt_end
, addr
[i
]);
1925 /* Our (by three pages) smaller Xen pagetable that we are using */
1926 memblock_reserve(PFN_PHYS(pt_base
), (pt_end
- pt_base
) * PAGE_SIZE
);
1927 /* Revector the xen_start_info */
1928 xen_start_info
= (struct start_info
*)__va(__pa(xen_start_info
));
1930 #else /* !CONFIG_X86_64 */
1931 static RESERVE_BRK_ARRAY(pmd_t
, initial_kernel_pmd
, PTRS_PER_PMD
);
1932 static RESERVE_BRK_ARRAY(pmd_t
, swapper_kernel_pmd
, PTRS_PER_PMD
);
1934 static void __init
xen_write_cr3_init(unsigned long cr3
)
1936 unsigned long pfn
= PFN_DOWN(__pa(swapper_pg_dir
));
1938 BUG_ON(read_cr3() != __pa(initial_page_table
));
1939 BUG_ON(cr3
!= __pa(swapper_pg_dir
));
1942 * We are switching to swapper_pg_dir for the first time (from
1943 * initial_page_table) and therefore need to mark that page
1944 * read-only and then pin it.
1946 * Xen disallows sharing of kernel PMDs for PAE
1947 * guests. Therefore we must copy the kernel PMD from
1948 * initial_page_table into a new kernel PMD to be used in
1951 swapper_kernel_pmd
=
1952 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
1953 copy_page(swapper_kernel_pmd
, initial_kernel_pmd
);
1954 swapper_pg_dir
[KERNEL_PGD_BOUNDARY
] =
1955 __pgd(__pa(swapper_kernel_pmd
) | _PAGE_PRESENT
);
1956 set_page_prot(swapper_kernel_pmd
, PAGE_KERNEL_RO
);
1958 set_page_prot(swapper_pg_dir
, PAGE_KERNEL_RO
);
1960 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
, pfn
);
1962 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
,
1963 PFN_DOWN(__pa(initial_page_table
)));
1964 set_page_prot(initial_page_table
, PAGE_KERNEL
);
1965 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL
);
1967 pv_mmu_ops
.write_cr3
= &xen_write_cr3
;
1970 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
1974 initial_kernel_pmd
=
1975 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
1977 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->pt_base
) +
1978 xen_start_info
->nr_pt_frames
* PAGE_SIZE
+
1981 kernel_pmd
= m2v(pgd
[KERNEL_PGD_BOUNDARY
].pgd
);
1982 copy_page(initial_kernel_pmd
, kernel_pmd
);
1984 xen_map_identity_early(initial_kernel_pmd
, max_pfn
);
1986 copy_page(initial_page_table
, pgd
);
1987 initial_page_table
[KERNEL_PGD_BOUNDARY
] =
1988 __pgd(__pa(initial_kernel_pmd
) | _PAGE_PRESENT
);
1990 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL_RO
);
1991 set_page_prot(initial_page_table
, PAGE_KERNEL_RO
);
1992 set_page_prot(empty_zero_page
, PAGE_KERNEL_RO
);
1994 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1996 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
,
1997 PFN_DOWN(__pa(initial_page_table
)));
1998 xen_write_cr3(__pa(initial_page_table
));
2000 memblock_reserve(__pa(xen_start_info
->pt_base
),
2001 xen_start_info
->nr_pt_frames
* PAGE_SIZE
);
2003 #endif /* CONFIG_X86_64 */
2005 static unsigned char dummy_mapping
[PAGE_SIZE
] __page_aligned_bss
;
2007 static void xen_set_fixmap(unsigned idx
, phys_addr_t phys
, pgprot_t prot
)
2011 phys
>>= PAGE_SHIFT
;
2014 case FIX_BTMAP_END
... FIX_BTMAP_BEGIN
:
2016 #ifdef CONFIG_X86_32
2018 # ifdef CONFIG_HIGHMEM
2019 case FIX_KMAP_BEGIN
... FIX_KMAP_END
:
2024 case FIX_TEXT_POKE0
:
2025 case FIX_TEXT_POKE1
:
2026 /* All local page mappings */
2027 pte
= pfn_pte(phys
, prot
);
2030 #ifdef CONFIG_X86_LOCAL_APIC
2031 case FIX_APIC_BASE
: /* maps dummy local APIC */
2032 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2036 #ifdef CONFIG_X86_IO_APIC
2037 case FIX_IO_APIC_BASE_0
... FIX_IO_APIC_BASE_END
:
2039 * We just don't map the IO APIC - all access is via
2040 * hypercalls. Keep the address in the pte for reference.
2042 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2046 case FIX_PARAVIRT_BOOTMAP
:
2047 /* This is an MFN, but it isn't an IO mapping from the
2049 pte
= mfn_pte(phys
, prot
);
2053 /* By default, set_fixmap is used for hardware mappings */
2054 pte
= mfn_pte(phys
, prot
);
2058 __native_set_fixmap(idx
, pte
);
2060 #ifdef CONFIG_X86_64
2061 /* Replicate changes to map the vsyscall page into the user
2062 pagetable vsyscall mapping. */
2063 if (idx
== VSYSCALL_PAGE
) {
2064 unsigned long vaddr
= __fix_to_virt(idx
);
2065 set_pte_vaddr_pud(level3_user_vsyscall
, vaddr
, pte
);
2070 static void __init
xen_post_allocator_init(void)
2072 if (xen_feature(XENFEAT_auto_translated_physmap
))
2075 pv_mmu_ops
.set_pte
= xen_set_pte
;
2076 pv_mmu_ops
.set_pmd
= xen_set_pmd
;
2077 pv_mmu_ops
.set_pud
= xen_set_pud
;
2078 #if PAGETABLE_LEVELS == 4
2079 pv_mmu_ops
.set_pgd
= xen_set_pgd
;
2082 /* This will work as long as patching hasn't happened yet
2083 (which it hasn't) */
2084 pv_mmu_ops
.alloc_pte
= xen_alloc_pte
;
2085 pv_mmu_ops
.alloc_pmd
= xen_alloc_pmd
;
2086 pv_mmu_ops
.release_pte
= xen_release_pte
;
2087 pv_mmu_ops
.release_pmd
= xen_release_pmd
;
2088 #if PAGETABLE_LEVELS == 4
2089 pv_mmu_ops
.alloc_pud
= xen_alloc_pud
;
2090 pv_mmu_ops
.release_pud
= xen_release_pud
;
2093 #ifdef CONFIG_X86_64
2094 pv_mmu_ops
.write_cr3
= &xen_write_cr3
;
2095 SetPagePinned(virt_to_page(level3_user_vsyscall
));
2097 xen_mark_init_mm_pinned();
2100 static void xen_leave_lazy_mmu(void)
2104 paravirt_leave_lazy_mmu();
2108 static const struct pv_mmu_ops xen_mmu_ops __initconst
= {
2109 .read_cr2
= xen_read_cr2
,
2110 .write_cr2
= xen_write_cr2
,
2112 .read_cr3
= xen_read_cr3
,
2113 .write_cr3
= xen_write_cr3_init
,
2115 .flush_tlb_user
= xen_flush_tlb
,
2116 .flush_tlb_kernel
= xen_flush_tlb
,
2117 .flush_tlb_single
= xen_flush_tlb_single
,
2118 .flush_tlb_others
= xen_flush_tlb_others
,
2120 .pte_update
= paravirt_nop
,
2121 .pte_update_defer
= paravirt_nop
,
2123 .pgd_alloc
= xen_pgd_alloc
,
2124 .pgd_free
= xen_pgd_free
,
2126 .alloc_pte
= xen_alloc_pte_init
,
2127 .release_pte
= xen_release_pte_init
,
2128 .alloc_pmd
= xen_alloc_pmd_init
,
2129 .release_pmd
= xen_release_pmd_init
,
2131 .set_pte
= xen_set_pte_init
,
2132 .set_pte_at
= xen_set_pte_at
,
2133 .set_pmd
= xen_set_pmd_hyper
,
2135 .ptep_modify_prot_start
= __ptep_modify_prot_start
,
2136 .ptep_modify_prot_commit
= __ptep_modify_prot_commit
,
2138 .pte_val
= PV_CALLEE_SAVE(xen_pte_val
),
2139 .pgd_val
= PV_CALLEE_SAVE(xen_pgd_val
),
2141 .make_pte
= PV_CALLEE_SAVE(xen_make_pte
),
2142 .make_pgd
= PV_CALLEE_SAVE(xen_make_pgd
),
2144 #ifdef CONFIG_X86_PAE
2145 .set_pte_atomic
= xen_set_pte_atomic
,
2146 .pte_clear
= xen_pte_clear
,
2147 .pmd_clear
= xen_pmd_clear
,
2148 #endif /* CONFIG_X86_PAE */
2149 .set_pud
= xen_set_pud_hyper
,
2151 .make_pmd
= PV_CALLEE_SAVE(xen_make_pmd
),
2152 .pmd_val
= PV_CALLEE_SAVE(xen_pmd_val
),
2154 #if PAGETABLE_LEVELS == 4
2155 .pud_val
= PV_CALLEE_SAVE(xen_pud_val
),
2156 .make_pud
= PV_CALLEE_SAVE(xen_make_pud
),
2157 .set_pgd
= xen_set_pgd_hyper
,
2159 .alloc_pud
= xen_alloc_pmd_init
,
2160 .release_pud
= xen_release_pmd_init
,
2161 #endif /* PAGETABLE_LEVELS == 4 */
2163 .activate_mm
= xen_activate_mm
,
2164 .dup_mmap
= xen_dup_mmap
,
2165 .exit_mmap
= xen_exit_mmap
,
2168 .enter
= paravirt_enter_lazy_mmu
,
2169 .leave
= xen_leave_lazy_mmu
,
2170 .flush
= paravirt_flush_lazy_mmu
,
2173 .set_fixmap
= xen_set_fixmap
,
2176 void __init
xen_init_mmu_ops(void)
2178 x86_init
.paging
.pagetable_init
= xen_pagetable_init
;
2180 /* Optimization - we can use the HVM one but it has no idea which
2181 * VCPUs are descheduled - which means that it will needlessly IPI
2182 * them. Xen knows so let it do the job.
2184 if (xen_feature(XENFEAT_auto_translated_physmap
)) {
2185 pv_mmu_ops
.flush_tlb_others
= xen_flush_tlb_others
;
2188 pv_mmu_ops
= xen_mmu_ops
;
2190 memset(dummy_mapping
, 0xff, PAGE_SIZE
);
2193 /* Protected by xen_reservation_lock. */
2194 #define MAX_CONTIG_ORDER 9 /* 2MB */
2195 static unsigned long discontig_frames
[1<<MAX_CONTIG_ORDER
];
2197 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2198 static void xen_zap_pfn_range(unsigned long vaddr
, unsigned int order
,
2199 unsigned long *in_frames
,
2200 unsigned long *out_frames
)
2203 struct multicall_space mcs
;
2206 for (i
= 0; i
< (1UL<<order
); i
++, vaddr
+= PAGE_SIZE
) {
2207 mcs
= __xen_mc_entry(0);
2210 in_frames
[i
] = virt_to_mfn(vaddr
);
2212 MULTI_update_va_mapping(mcs
.mc
, vaddr
, VOID_PTE
, 0);
2213 __set_phys_to_machine(virt_to_pfn(vaddr
), INVALID_P2M_ENTRY
);
2216 out_frames
[i
] = virt_to_pfn(vaddr
);
2222 * Update the pfn-to-mfn mappings for a virtual address range, either to
2223 * point to an array of mfns, or contiguously from a single starting
2226 static void xen_remap_exchanged_ptes(unsigned long vaddr
, int order
,
2227 unsigned long *mfns
,
2228 unsigned long first_mfn
)
2235 limit
= 1u << order
;
2236 for (i
= 0; i
< limit
; i
++, vaddr
+= PAGE_SIZE
) {
2237 struct multicall_space mcs
;
2240 mcs
= __xen_mc_entry(0);
2244 mfn
= first_mfn
+ i
;
2246 if (i
< (limit
- 1))
2250 flags
= UVMF_INVLPG
| UVMF_ALL
;
2252 flags
= UVMF_TLB_FLUSH
| UVMF_ALL
;
2255 MULTI_update_va_mapping(mcs
.mc
, vaddr
,
2256 mfn_pte(mfn
, PAGE_KERNEL
), flags
);
2258 set_phys_to_machine(virt_to_pfn(vaddr
), mfn
);
2265 * Perform the hypercall to exchange a region of our pfns to point to
2266 * memory with the required contiguous alignment. Takes the pfns as
2267 * input, and populates mfns as output.
2269 * Returns a success code indicating whether the hypervisor was able to
2270 * satisfy the request or not.
2272 static int xen_exchange_memory(unsigned long extents_in
, unsigned int order_in
,
2273 unsigned long *pfns_in
,
2274 unsigned long extents_out
,
2275 unsigned int order_out
,
2276 unsigned long *mfns_out
,
2277 unsigned int address_bits
)
2282 struct xen_memory_exchange exchange
= {
2284 .nr_extents
= extents_in
,
2285 .extent_order
= order_in
,
2286 .extent_start
= pfns_in
,
2290 .nr_extents
= extents_out
,
2291 .extent_order
= order_out
,
2292 .extent_start
= mfns_out
,
2293 .address_bits
= address_bits
,
2298 BUG_ON(extents_in
<< order_in
!= extents_out
<< order_out
);
2300 rc
= HYPERVISOR_memory_op(XENMEM_exchange
, &exchange
);
2301 success
= (exchange
.nr_exchanged
== extents_in
);
2303 BUG_ON(!success
&& ((exchange
.nr_exchanged
!= 0) || (rc
== 0)));
2304 BUG_ON(success
&& (rc
!= 0));
2309 int xen_create_contiguous_region(phys_addr_t pstart
, unsigned int order
,
2310 unsigned int address_bits
,
2311 dma_addr_t
*dma_handle
)
2313 unsigned long *in_frames
= discontig_frames
, out_frame
;
2314 unsigned long flags
;
2316 unsigned long vstart
= (unsigned long)phys_to_virt(pstart
);
2319 * Currently an auto-translated guest will not perform I/O, nor will
2320 * it require PAE page directories below 4GB. Therefore any calls to
2321 * this function are redundant and can be ignored.
2324 if (xen_feature(XENFEAT_auto_translated_physmap
))
2327 if (unlikely(order
> MAX_CONTIG_ORDER
))
2330 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2332 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2334 /* 1. Zap current PTEs, remembering MFNs. */
2335 xen_zap_pfn_range(vstart
, order
, in_frames
, NULL
);
2337 /* 2. Get a new contiguous memory extent. */
2338 out_frame
= virt_to_pfn(vstart
);
2339 success
= xen_exchange_memory(1UL << order
, 0, in_frames
,
2340 1, order
, &out_frame
,
2343 /* 3. Map the new extent in place of old pages. */
2345 xen_remap_exchanged_ptes(vstart
, order
, NULL
, out_frame
);
2347 xen_remap_exchanged_ptes(vstart
, order
, in_frames
, 0);
2349 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2351 *dma_handle
= virt_to_machine(vstart
).maddr
;
2352 return success
? 0 : -ENOMEM
;
2354 EXPORT_SYMBOL_GPL(xen_create_contiguous_region
);
2356 void xen_destroy_contiguous_region(phys_addr_t pstart
, unsigned int order
)
2358 unsigned long *out_frames
= discontig_frames
, in_frame
;
2359 unsigned long flags
;
2361 unsigned long vstart
;
2363 if (xen_feature(XENFEAT_auto_translated_physmap
))
2366 if (unlikely(order
> MAX_CONTIG_ORDER
))
2369 vstart
= (unsigned long)phys_to_virt(pstart
);
2370 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2372 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2374 /* 1. Find start MFN of contiguous extent. */
2375 in_frame
= virt_to_mfn(vstart
);
2377 /* 2. Zap current PTEs. */
2378 xen_zap_pfn_range(vstart
, order
, NULL
, out_frames
);
2380 /* 3. Do the exchange for non-contiguous MFNs. */
2381 success
= xen_exchange_memory(1, order
, &in_frame
, 1UL << order
,
2384 /* 4. Map new pages in place of old pages. */
2386 xen_remap_exchanged_ptes(vstart
, order
, out_frames
, 0);
2388 xen_remap_exchanged_ptes(vstart
, order
, NULL
, in_frame
);
2390 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2392 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region
);
2394 #ifdef CONFIG_XEN_PVHVM
2395 #ifdef CONFIG_PROC_VMCORE
2397 * This function is used in two contexts:
2398 * - the kdump kernel has to check whether a pfn of the crashed kernel
2399 * was a ballooned page. vmcore is using this function to decide
2400 * whether to access a pfn of the crashed kernel.
2401 * - the kexec kernel has to check whether a pfn was ballooned by the
2402 * previous kernel. If the pfn is ballooned, handle it properly.
2403 * Returns 0 if the pfn is not backed by a RAM page, the caller may
2404 * handle the pfn special in this case.
2406 static int xen_oldmem_pfn_is_ram(unsigned long pfn
)
2408 struct xen_hvm_get_mem_type a
= {
2409 .domid
= DOMID_SELF
,
2414 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type
, &a
))
2417 switch (a
.mem_type
) {
2418 case HVMMEM_mmio_dm
:
2432 static void xen_hvm_exit_mmap(struct mm_struct
*mm
)
2434 struct xen_hvm_pagetable_dying a
;
2437 a
.domid
= DOMID_SELF
;
2438 a
.gpa
= __pa(mm
->pgd
);
2439 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2440 WARN_ON_ONCE(rc
< 0);
2443 static int is_pagetable_dying_supported(void)
2445 struct xen_hvm_pagetable_dying a
;
2448 a
.domid
= DOMID_SELF
;
2450 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2452 printk(KERN_DEBUG
"HVMOP_pagetable_dying not supported\n");
2458 void __init
xen_hvm_init_mmu_ops(void)
2460 if (is_pagetable_dying_supported())
2461 pv_mmu_ops
.exit_mmap
= xen_hvm_exit_mmap
;
2462 #ifdef CONFIG_PROC_VMCORE
2463 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram
);
2468 #ifdef CONFIG_XEN_PVH
2470 * Map foreign gfn (fgfn), to local pfn (lpfn). This for the user
2471 * space creating new guest on pvh dom0 and needing to map domU pages.
2473 static int xlate_add_to_p2m(unsigned long lpfn
, unsigned long fgfn
,
2477 xen_pfn_t gpfn
= lpfn
;
2478 xen_ulong_t idx
= fgfn
;
2480 struct xen_add_to_physmap_range xatp
= {
2481 .domid
= DOMID_SELF
,
2482 .foreign_domid
= domid
,
2484 .space
= XENMAPSPACE_gmfn_foreign
,
2486 set_xen_guest_handle(xatp
.idxs
, &idx
);
2487 set_xen_guest_handle(xatp
.gpfns
, &gpfn
);
2488 set_xen_guest_handle(xatp
.errs
, &err
);
2490 rc
= HYPERVISOR_memory_op(XENMEM_add_to_physmap_range
, &xatp
);
2496 static int xlate_remove_from_p2m(unsigned long spfn
, int count
)
2498 struct xen_remove_from_physmap xrp
;
2501 for (i
= 0; i
< count
; i
++) {
2502 xrp
.domid
= DOMID_SELF
;
2504 rc
= HYPERVISOR_memory_op(XENMEM_remove_from_physmap
, &xrp
);
2511 struct xlate_remap_data
{
2512 unsigned long fgfn
; /* foreign domain's gfn */
2516 struct page
**pages
;
2519 static int xlate_map_pte_fn(pte_t
*ptep
, pgtable_t token
, unsigned long addr
,
2523 struct xlate_remap_data
*remap
= data
;
2524 unsigned long pfn
= page_to_pfn(remap
->pages
[remap
->index
++]);
2525 pte_t pteval
= pte_mkspecial(pfn_pte(pfn
, remap
->prot
));
2527 rc
= xlate_add_to_p2m(pfn
, remap
->fgfn
, remap
->domid
);
2530 native_set_pte(ptep
, pteval
);
2535 static int xlate_remap_gfn_range(struct vm_area_struct
*vma
,
2536 unsigned long addr
, unsigned long mfn
,
2537 int nr
, pgprot_t prot
, unsigned domid
,
2538 struct page
**pages
)
2541 struct xlate_remap_data pvhdata
;
2546 pvhdata
.prot
= prot
;
2547 pvhdata
.domid
= domid
;
2549 pvhdata
.pages
= pages
;
2550 err
= apply_to_page_range(vma
->vm_mm
, addr
, nr
<< PAGE_SHIFT
,
2551 xlate_map_pte_fn
, &pvhdata
);
2557 #define REMAP_BATCH_SIZE 16
2562 struct mmu_update
*mmu_update
;
2565 static int remap_area_mfn_pte_fn(pte_t
*ptep
, pgtable_t token
,
2566 unsigned long addr
, void *data
)
2568 struct remap_data
*rmd
= data
;
2569 pte_t pte
= pte_mkspecial(mfn_pte(rmd
->mfn
++, rmd
->prot
));
2571 rmd
->mmu_update
->ptr
= virt_to_machine(ptep
).maddr
;
2572 rmd
->mmu_update
->val
= pte_val_ma(pte
);
2578 int xen_remap_domain_mfn_range(struct vm_area_struct
*vma
,
2580 xen_pfn_t mfn
, int nr
,
2581 pgprot_t prot
, unsigned domid
,
2582 struct page
**pages
)
2585 struct remap_data rmd
;
2586 struct mmu_update mmu_update
[REMAP_BATCH_SIZE
];
2588 unsigned long range
;
2591 BUG_ON(!((vma
->vm_flags
& (VM_PFNMAP
| VM_IO
)) == (VM_PFNMAP
| VM_IO
)));
2593 if (xen_feature(XENFEAT_auto_translated_physmap
)) {
2594 #ifdef CONFIG_XEN_PVH
2595 /* We need to update the local page tables and the xen HAP */
2596 return xlate_remap_gfn_range(vma
, addr
, mfn
, nr
, prot
,
2607 batch
= min(REMAP_BATCH_SIZE
, nr
);
2608 range
= (unsigned long)batch
<< PAGE_SHIFT
;
2610 rmd
.mmu_update
= mmu_update
;
2611 err
= apply_to_page_range(vma
->vm_mm
, addr
, range
,
2612 remap_area_mfn_pte_fn
, &rmd
);
2616 err
= HYPERVISOR_mmu_update(mmu_update
, batch
, NULL
, domid
);
2627 xen_flush_tlb_all();
2631 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range
);
2633 /* Returns: 0 success */
2634 int xen_unmap_domain_mfn_range(struct vm_area_struct
*vma
,
2635 int numpgs
, struct page
**pages
)
2637 if (!pages
|| !xen_feature(XENFEAT_auto_translated_physmap
))
2640 #ifdef CONFIG_XEN_PVH
2643 * The mmu has already cleaned up the process mmu
2644 * resources at this point (lookup_address will return
2647 unsigned long pfn
= page_to_pfn(pages
[numpgs
]);
2649 xlate_remove_from_p2m(pfn
, 1);
2652 * We don't need to flush tlbs because as part of
2653 * xlate_remove_from_p2m, the hypervisor will do tlb flushes
2654 * after removing the p2m entries from the EPT/NPT
2661 EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range
);