4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched/mm.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/export.h>
47 #include <linux/init.h>
48 #include <linux/gfp.h>
49 #include <linux/memblock.h>
50 #include <linux/seq_file.h>
51 #include <linux/crash_dump.h>
53 #include <trace/events/xen.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/fixmap.h>
58 #include <asm/mmu_context.h>
59 #include <asm/setup.h>
60 #include <asm/paravirt.h>
62 #include <asm/linkage.h>
68 #include <asm/xen/hypercall.h>
69 #include <asm/xen/hypervisor.h>
73 #include <xen/interface/xen.h>
74 #include <xen/interface/hvm/hvm_op.h>
75 #include <xen/interface/version.h>
76 #include <xen/interface/memory.h>
77 #include <xen/hvc-console.h>
79 #include "multicalls.h"
84 * Protects atomic reservation decrease/increase against concurrent increases.
85 * Also protects non-atomic updates of current_pages and balloon lists.
87 DEFINE_SPINLOCK(xen_reservation_lock
);
91 * Identity map, in addition to plain kernel map. This needs to be
92 * large enough to allocate page table pages to allocate the rest.
93 * Each page can map 2MB.
95 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
96 static RESERVE_BRK_ARRAY(pte_t
, level1_ident_pgt
, LEVEL1_IDENT_ENTRIES
);
99 /* l3 pud for userspace vsyscall mapping */
100 static pud_t level3_user_vsyscall
[PTRS_PER_PUD
] __page_aligned_bss
;
101 #endif /* CONFIG_X86_64 */
104 * Note about cr3 (pagetable base) values:
106 * xen_cr3 contains the current logical cr3 value; it contains the
107 * last set cr3. This may not be the current effective cr3, because
108 * its update may be being lazily deferred. However, a vcpu looking
109 * at its own cr3 can use this value knowing that it everything will
110 * be self-consistent.
112 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
113 * hypercall to set the vcpu cr3 is complete (so it may be a little
114 * out of date, but it will never be set early). If one vcpu is
115 * looking at another vcpu's cr3 value, it should use this variable.
117 DEFINE_PER_CPU(unsigned long, xen_cr3
); /* cr3 stored as physaddr */
118 DEFINE_PER_CPU(unsigned long, xen_current_cr3
); /* actual vcpu cr3 */
120 static phys_addr_t xen_pt_base
, xen_pt_size __initdata
;
123 * Just beyond the highest usermode address. STACK_TOP_MAX has a
124 * redzone above it, so round it up to a PGD boundary.
126 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
128 unsigned long arbitrary_virt_to_mfn(void *vaddr
)
130 xmaddr_t maddr
= arbitrary_virt_to_machine(vaddr
);
132 return PFN_DOWN(maddr
.maddr
);
135 xmaddr_t
arbitrary_virt_to_machine(void *vaddr
)
137 unsigned long address
= (unsigned long)vaddr
;
143 * if the PFN is in the linear mapped vaddr range, we can just use
144 * the (quick) virt_to_machine() p2m lookup
146 if (virt_addr_valid(vaddr
))
147 return virt_to_machine(vaddr
);
149 /* otherwise we have to do a (slower) full page-table walk */
151 pte
= lookup_address(address
, &level
);
153 offset
= address
& ~PAGE_MASK
;
154 return XMADDR(((phys_addr_t
)pte_mfn(*pte
) << PAGE_SHIFT
) + offset
);
156 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine
);
158 void make_lowmem_page_readonly(void *vaddr
)
161 unsigned long address
= (unsigned long)vaddr
;
164 pte
= lookup_address(address
, &level
);
166 return; /* vaddr missing */
168 ptev
= pte_wrprotect(*pte
);
170 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
174 void make_lowmem_page_readwrite(void *vaddr
)
177 unsigned long address
= (unsigned long)vaddr
;
180 pte
= lookup_address(address
, &level
);
182 return; /* vaddr missing */
184 ptev
= pte_mkwrite(*pte
);
186 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
191 static bool xen_page_pinned(void *ptr
)
193 struct page
*page
= virt_to_page(ptr
);
195 return PagePinned(page
);
198 void xen_set_domain_pte(pte_t
*ptep
, pte_t pteval
, unsigned domid
)
200 struct multicall_space mcs
;
201 struct mmu_update
*u
;
203 trace_xen_mmu_set_domain_pte(ptep
, pteval
, domid
);
205 mcs
= xen_mc_entry(sizeof(*u
));
208 /* ptep might be kmapped when using 32-bit HIGHPTE */
209 u
->ptr
= virt_to_machine(ptep
).maddr
;
210 u
->val
= pte_val_ma(pteval
);
212 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, domid
);
214 xen_mc_issue(PARAVIRT_LAZY_MMU
);
216 EXPORT_SYMBOL_GPL(xen_set_domain_pte
);
218 static void xen_extend_mmu_update(const struct mmu_update
*update
)
220 struct multicall_space mcs
;
221 struct mmu_update
*u
;
223 mcs
= xen_mc_extend_args(__HYPERVISOR_mmu_update
, sizeof(*u
));
225 if (mcs
.mc
!= NULL
) {
228 mcs
= __xen_mc_entry(sizeof(*u
));
229 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
236 static void xen_extend_mmuext_op(const struct mmuext_op
*op
)
238 struct multicall_space mcs
;
241 mcs
= xen_mc_extend_args(__HYPERVISOR_mmuext_op
, sizeof(*u
));
243 if (mcs
.mc
!= NULL
) {
246 mcs
= __xen_mc_entry(sizeof(*u
));
247 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
254 static void xen_set_pmd_hyper(pmd_t
*ptr
, pmd_t val
)
262 /* ptr may be ioremapped for 64-bit pagetable setup */
263 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
264 u
.val
= pmd_val_ma(val
);
265 xen_extend_mmu_update(&u
);
267 xen_mc_issue(PARAVIRT_LAZY_MMU
);
272 static void xen_set_pmd(pmd_t
*ptr
, pmd_t val
)
274 trace_xen_mmu_set_pmd(ptr
, val
);
276 /* If page is not pinned, we can just update the entry
278 if (!xen_page_pinned(ptr
)) {
283 xen_set_pmd_hyper(ptr
, val
);
287 * Associate a virtual page frame with a given physical page frame
288 * and protection flags for that frame.
290 void set_pte_mfn(unsigned long vaddr
, unsigned long mfn
, pgprot_t flags
)
292 set_pte_vaddr(vaddr
, mfn_pte(mfn
, flags
));
295 static bool xen_batched_set_pte(pte_t
*ptep
, pte_t pteval
)
299 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU
)
304 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
305 u
.val
= pte_val_ma(pteval
);
306 xen_extend_mmu_update(&u
);
308 xen_mc_issue(PARAVIRT_LAZY_MMU
);
313 static inline void __xen_set_pte(pte_t
*ptep
, pte_t pteval
)
315 if (!xen_batched_set_pte(ptep
, pteval
)) {
317 * Could call native_set_pte() here and trap and
318 * emulate the PTE write but with 32-bit guests this
319 * needs two traps (one for each of the two 32-bit
320 * words in the PTE) so do one hypercall directly
325 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
326 u
.val
= pte_val_ma(pteval
);
327 HYPERVISOR_mmu_update(&u
, 1, NULL
, DOMID_SELF
);
331 static void xen_set_pte(pte_t
*ptep
, pte_t pteval
)
333 trace_xen_mmu_set_pte(ptep
, pteval
);
334 __xen_set_pte(ptep
, pteval
);
337 static void xen_set_pte_at(struct mm_struct
*mm
, unsigned long addr
,
338 pte_t
*ptep
, pte_t pteval
)
340 trace_xen_mmu_set_pte_at(mm
, addr
, ptep
, pteval
);
341 __xen_set_pte(ptep
, pteval
);
344 pte_t
xen_ptep_modify_prot_start(struct mm_struct
*mm
,
345 unsigned long addr
, pte_t
*ptep
)
347 /* Just return the pte as-is. We preserve the bits on commit */
348 trace_xen_mmu_ptep_modify_prot_start(mm
, addr
, ptep
, *ptep
);
352 void xen_ptep_modify_prot_commit(struct mm_struct
*mm
, unsigned long addr
,
353 pte_t
*ptep
, pte_t pte
)
357 trace_xen_mmu_ptep_modify_prot_commit(mm
, addr
, ptep
, pte
);
360 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_PT_UPDATE_PRESERVE_AD
;
361 u
.val
= pte_val_ma(pte
);
362 xen_extend_mmu_update(&u
);
364 xen_mc_issue(PARAVIRT_LAZY_MMU
);
367 /* Assume pteval_t is equivalent to all the other *val_t types. */
368 static pteval_t
pte_mfn_to_pfn(pteval_t val
)
370 if (val
& _PAGE_PRESENT
) {
371 unsigned long mfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
372 unsigned long pfn
= mfn_to_pfn(mfn
);
374 pteval_t flags
= val
& PTE_FLAGS_MASK
;
375 if (unlikely(pfn
== ~0))
376 val
= flags
& ~_PAGE_PRESENT
;
378 val
= ((pteval_t
)pfn
<< PAGE_SHIFT
) | flags
;
384 static pteval_t
pte_pfn_to_mfn(pteval_t val
)
386 if (val
& _PAGE_PRESENT
) {
387 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
388 pteval_t flags
= val
& PTE_FLAGS_MASK
;
391 if (!xen_feature(XENFEAT_auto_translated_physmap
))
392 mfn
= __pfn_to_mfn(pfn
);
396 * If there's no mfn for the pfn, then just create an
397 * empty non-present pte. Unfortunately this loses
398 * information about the original pfn, so
399 * pte_mfn_to_pfn is asymmetric.
401 if (unlikely(mfn
== INVALID_P2M_ENTRY
)) {
405 mfn
&= ~(FOREIGN_FRAME_BIT
| IDENTITY_FRAME_BIT
);
406 val
= ((pteval_t
)mfn
<< PAGE_SHIFT
) | flags
;
412 __visible pteval_t
xen_pte_val(pte_t pte
)
414 pteval_t pteval
= pte
.pte
;
416 return pte_mfn_to_pfn(pteval
);
418 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val
);
420 __visible pgdval_t
xen_pgd_val(pgd_t pgd
)
422 return pte_mfn_to_pfn(pgd
.pgd
);
424 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val
);
426 __visible pte_t
xen_make_pte(pteval_t pte
)
428 pte
= pte_pfn_to_mfn(pte
);
430 return native_make_pte(pte
);
432 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte
);
434 __visible pgd_t
xen_make_pgd(pgdval_t pgd
)
436 pgd
= pte_pfn_to_mfn(pgd
);
437 return native_make_pgd(pgd
);
439 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd
);
441 __visible pmdval_t
xen_pmd_val(pmd_t pmd
)
443 return pte_mfn_to_pfn(pmd
.pmd
);
445 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val
);
447 static void xen_set_pud_hyper(pud_t
*ptr
, pud_t val
)
455 /* ptr may be ioremapped for 64-bit pagetable setup */
456 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
457 u
.val
= pud_val_ma(val
);
458 xen_extend_mmu_update(&u
);
460 xen_mc_issue(PARAVIRT_LAZY_MMU
);
465 static void xen_set_pud(pud_t
*ptr
, pud_t val
)
467 trace_xen_mmu_set_pud(ptr
, val
);
469 /* If page is not pinned, we can just update the entry
471 if (!xen_page_pinned(ptr
)) {
476 xen_set_pud_hyper(ptr
, val
);
479 #ifdef CONFIG_X86_PAE
480 static void xen_set_pte_atomic(pte_t
*ptep
, pte_t pte
)
482 trace_xen_mmu_set_pte_atomic(ptep
, pte
);
483 set_64bit((u64
*)ptep
, native_pte_val(pte
));
486 static void xen_pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
488 trace_xen_mmu_pte_clear(mm
, addr
, ptep
);
489 if (!xen_batched_set_pte(ptep
, native_make_pte(0)))
490 native_pte_clear(mm
, addr
, ptep
);
493 static void xen_pmd_clear(pmd_t
*pmdp
)
495 trace_xen_mmu_pmd_clear(pmdp
);
496 set_pmd(pmdp
, __pmd(0));
498 #endif /* CONFIG_X86_PAE */
500 __visible pmd_t
xen_make_pmd(pmdval_t pmd
)
502 pmd
= pte_pfn_to_mfn(pmd
);
503 return native_make_pmd(pmd
);
505 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd
);
507 #if CONFIG_PGTABLE_LEVELS == 4
508 __visible pudval_t
xen_pud_val(pud_t pud
)
510 return pte_mfn_to_pfn(pud
.pud
);
512 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val
);
514 __visible pud_t
xen_make_pud(pudval_t pud
)
516 pud
= pte_pfn_to_mfn(pud
);
518 return native_make_pud(pud
);
520 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud
);
522 static pgd_t
*xen_get_user_pgd(pgd_t
*pgd
)
524 pgd_t
*pgd_page
= (pgd_t
*)(((unsigned long)pgd
) & PAGE_MASK
);
525 unsigned offset
= pgd
- pgd_page
;
526 pgd_t
*user_ptr
= NULL
;
528 if (offset
< pgd_index(USER_LIMIT
)) {
529 struct page
*page
= virt_to_page(pgd_page
);
530 user_ptr
= (pgd_t
*)page
->private;
538 static void __xen_set_p4d_hyper(p4d_t
*ptr
, p4d_t val
)
542 u
.ptr
= virt_to_machine(ptr
).maddr
;
543 u
.val
= p4d_val_ma(val
);
544 xen_extend_mmu_update(&u
);
548 * Raw hypercall-based set_p4d, intended for in early boot before
549 * there's a page structure. This implies:
550 * 1. The only existing pagetable is the kernel's
551 * 2. It is always pinned
552 * 3. It has no user pagetable attached to it
554 static void __init
xen_set_p4d_hyper(p4d_t
*ptr
, p4d_t val
)
560 __xen_set_p4d_hyper(ptr
, val
);
562 xen_mc_issue(PARAVIRT_LAZY_MMU
);
567 static void xen_set_p4d(p4d_t
*ptr
, p4d_t val
)
569 pgd_t
*user_ptr
= xen_get_user_pgd((pgd_t
*)ptr
);
572 trace_xen_mmu_set_p4d(ptr
, (p4d_t
*)user_ptr
, val
);
574 /* If page is not pinned, we can just update the entry
576 if (!xen_page_pinned(ptr
)) {
579 WARN_ON(xen_page_pinned(user_ptr
));
580 pgd_val
.pgd
= p4d_val_ma(val
);
586 /* If it's pinned, then we can at least batch the kernel and
587 user updates together. */
590 __xen_set_p4d_hyper(ptr
, val
);
592 __xen_set_p4d_hyper((p4d_t
*)user_ptr
, val
);
594 xen_mc_issue(PARAVIRT_LAZY_MMU
);
596 #endif /* CONFIG_PGTABLE_LEVELS == 4 */
598 static int xen_pmd_walk(struct mm_struct
*mm
, pmd_t
*pmd
,
599 int (*func
)(struct mm_struct
*mm
, struct page
*, enum pt_level
),
600 bool last
, unsigned long limit
)
602 int i
, nr
, flush
= 0;
604 nr
= last
? pmd_index(limit
) + 1 : PTRS_PER_PMD
;
605 for (i
= 0; i
< nr
; i
++) {
606 if (!pmd_none(pmd
[i
]))
607 flush
|= (*func
)(mm
, pmd_page(pmd
[i
]), PT_PTE
);
612 static int xen_pud_walk(struct mm_struct
*mm
, pud_t
*pud
,
613 int (*func
)(struct mm_struct
*mm
, struct page
*, enum pt_level
),
614 bool last
, unsigned long limit
)
616 int i
, nr
, flush
= 0;
618 nr
= last
? pud_index(limit
) + 1 : PTRS_PER_PUD
;
619 for (i
= 0; i
< nr
; i
++) {
622 if (pud_none(pud
[i
]))
625 pmd
= pmd_offset(&pud
[i
], 0);
626 if (PTRS_PER_PMD
> 1)
627 flush
|= (*func
)(mm
, virt_to_page(pmd
), PT_PMD
);
628 flush
|= xen_pmd_walk(mm
, pmd
, func
,
629 last
&& i
== nr
- 1, limit
);
634 static int xen_p4d_walk(struct mm_struct
*mm
, p4d_t
*p4d
,
635 int (*func
)(struct mm_struct
*mm
, struct page
*, enum pt_level
),
636 bool last
, unsigned long limit
)
638 int i
, nr
, flush
= 0;
640 nr
= last
? p4d_index(limit
) + 1 : PTRS_PER_P4D
;
641 for (i
= 0; i
< nr
; i
++) {
644 if (p4d_none(p4d
[i
]))
647 pud
= pud_offset(&p4d
[i
], 0);
648 if (PTRS_PER_PUD
> 1)
649 flush
|= (*func
)(mm
, virt_to_page(pud
), PT_PUD
);
650 flush
|= xen_pud_walk(mm
, pud
, func
,
651 last
&& i
== nr
- 1, limit
);
657 * (Yet another) pagetable walker. This one is intended for pinning a
658 * pagetable. This means that it walks a pagetable and calls the
659 * callback function on each page it finds making up the page table,
660 * at every level. It walks the entire pagetable, but it only bothers
661 * pinning pte pages which are below limit. In the normal case this
662 * will be STACK_TOP_MAX, but at boot we need to pin up to
665 * For 32-bit the important bit is that we don't pin beyond there,
666 * because then we start getting into Xen's ptes.
668 * For 64-bit, we must skip the Xen hole in the middle of the address
669 * space, just after the big x86-64 virtual hole.
671 static int __xen_pgd_walk(struct mm_struct
*mm
, pgd_t
*pgd
,
672 int (*func
)(struct mm_struct
*mm
, struct page
*,
676 int i
, nr
, flush
= 0;
677 unsigned hole_low
, hole_high
;
679 /* The limit is the last byte to be touched */
681 BUG_ON(limit
>= FIXADDR_TOP
);
683 if (xen_feature(XENFEAT_auto_translated_physmap
))
687 * 64-bit has a great big hole in the middle of the address
688 * space, which contains the Xen mappings. On 32-bit these
689 * will end up making a zero-sized hole and so is a no-op.
691 hole_low
= pgd_index(USER_LIMIT
);
692 hole_high
= pgd_index(PAGE_OFFSET
);
694 nr
= pgd_index(limit
) + 1;
695 for (i
= 0; i
< nr
; i
++) {
698 if (i
>= hole_low
&& i
< hole_high
)
701 if (pgd_none(pgd
[i
]))
704 p4d
= p4d_offset(&pgd
[i
], 0);
705 if (PTRS_PER_P4D
> 1)
706 flush
|= (*func
)(mm
, virt_to_page(p4d
), PT_P4D
);
707 flush
|= xen_p4d_walk(mm
, p4d
, func
, i
== nr
- 1, limit
);
710 /* Do the top level last, so that the callbacks can use it as
711 a cue to do final things like tlb flushes. */
712 flush
|= (*func
)(mm
, virt_to_page(pgd
), PT_PGD
);
717 static int xen_pgd_walk(struct mm_struct
*mm
,
718 int (*func
)(struct mm_struct
*mm
, struct page
*,
722 return __xen_pgd_walk(mm
, mm
->pgd
, func
, limit
);
725 /* If we're using split pte locks, then take the page's lock and
726 return a pointer to it. Otherwise return NULL. */
727 static spinlock_t
*xen_pte_lock(struct page
*page
, struct mm_struct
*mm
)
729 spinlock_t
*ptl
= NULL
;
731 #if USE_SPLIT_PTE_PTLOCKS
732 ptl
= ptlock_ptr(page
);
733 spin_lock_nest_lock(ptl
, &mm
->page_table_lock
);
739 static void xen_pte_unlock(void *v
)
745 static void xen_do_pin(unsigned level
, unsigned long pfn
)
750 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
752 xen_extend_mmuext_op(&op
);
755 static int xen_pin_page(struct mm_struct
*mm
, struct page
*page
,
758 unsigned pgfl
= TestSetPagePinned(page
);
762 flush
= 0; /* already pinned */
763 else if (PageHighMem(page
))
764 /* kmaps need flushing if we found an unpinned
768 void *pt
= lowmem_page_address(page
);
769 unsigned long pfn
= page_to_pfn(page
);
770 struct multicall_space mcs
= __xen_mc_entry(0);
776 * We need to hold the pagetable lock between the time
777 * we make the pagetable RO and when we actually pin
778 * it. If we don't, then other users may come in and
779 * attempt to update the pagetable by writing it,
780 * which will fail because the memory is RO but not
781 * pinned, so Xen won't do the trap'n'emulate.
783 * If we're using split pte locks, we can't hold the
784 * entire pagetable's worth of locks during the
785 * traverse, because we may wrap the preempt count (8
786 * bits). The solution is to mark RO and pin each PTE
787 * page while holding the lock. This means the number
788 * of locks we end up holding is never more than a
789 * batch size (~32 entries, at present).
791 * If we're not using split pte locks, we needn't pin
792 * the PTE pages independently, because we're
793 * protected by the overall pagetable lock.
797 ptl
= xen_pte_lock(page
, mm
);
799 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
800 pfn_pte(pfn
, PAGE_KERNEL_RO
),
801 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
804 xen_do_pin(MMUEXT_PIN_L1_TABLE
, pfn
);
806 /* Queue a deferred unlock for when this batch
808 xen_mc_callback(xen_pte_unlock
, ptl
);
815 /* This is called just after a mm has been created, but it has not
816 been used yet. We need to make sure that its pagetable is all
817 read-only, and can be pinned. */
818 static void __xen_pgd_pin(struct mm_struct
*mm
, pgd_t
*pgd
)
820 trace_xen_mmu_pgd_pin(mm
, pgd
);
824 if (__xen_pgd_walk(mm
, pgd
, xen_pin_page
, USER_LIMIT
)) {
825 /* re-enable interrupts for flushing */
835 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
837 xen_do_pin(MMUEXT_PIN_L4_TABLE
, PFN_DOWN(__pa(pgd
)));
840 xen_pin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
841 xen_do_pin(MMUEXT_PIN_L4_TABLE
,
842 PFN_DOWN(__pa(user_pgd
)));
845 #else /* CONFIG_X86_32 */
846 #ifdef CONFIG_X86_PAE
847 /* Need to make sure unshared kernel PMD is pinnable */
848 xen_pin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
851 xen_do_pin(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(__pa(pgd
)));
852 #endif /* CONFIG_X86_64 */
856 static void xen_pgd_pin(struct mm_struct
*mm
)
858 __xen_pgd_pin(mm
, mm
->pgd
);
862 * On save, we need to pin all pagetables to make sure they get their
863 * mfns turned into pfns. Search the list for any unpinned pgds and pin
864 * them (unpinned pgds are not currently in use, probably because the
865 * process is under construction or destruction).
867 * Expected to be called in stop_machine() ("equivalent to taking
868 * every spinlock in the system"), so the locking doesn't really
869 * matter all that much.
871 void xen_mm_pin_all(void)
875 spin_lock(&pgd_lock
);
877 list_for_each_entry(page
, &pgd_list
, lru
) {
878 if (!PagePinned(page
)) {
879 __xen_pgd_pin(&init_mm
, (pgd_t
*)page_address(page
));
880 SetPageSavePinned(page
);
884 spin_unlock(&pgd_lock
);
888 * The init_mm pagetable is really pinned as soon as its created, but
889 * that's before we have page structures to store the bits. So do all
890 * the book-keeping now.
892 static int __init
xen_mark_pinned(struct mm_struct
*mm
, struct page
*page
,
899 static void __init
xen_mark_init_mm_pinned(void)
901 xen_pgd_walk(&init_mm
, xen_mark_pinned
, FIXADDR_TOP
);
904 static int xen_unpin_page(struct mm_struct
*mm
, struct page
*page
,
907 unsigned pgfl
= TestClearPagePinned(page
);
909 if (pgfl
&& !PageHighMem(page
)) {
910 void *pt
= lowmem_page_address(page
);
911 unsigned long pfn
= page_to_pfn(page
);
912 spinlock_t
*ptl
= NULL
;
913 struct multicall_space mcs
;
916 * Do the converse to pin_page. If we're using split
917 * pte locks, we must be holding the lock for while
918 * the pte page is unpinned but still RO to prevent
919 * concurrent updates from seeing it in this
920 * partially-pinned state.
922 if (level
== PT_PTE
) {
923 ptl
= xen_pte_lock(page
, mm
);
926 xen_do_pin(MMUEXT_UNPIN_TABLE
, pfn
);
929 mcs
= __xen_mc_entry(0);
931 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
932 pfn_pte(pfn
, PAGE_KERNEL
),
933 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
936 /* unlock when batch completed */
937 xen_mc_callback(xen_pte_unlock
, ptl
);
941 return 0; /* never need to flush on unpin */
944 /* Release a pagetables pages back as normal RW */
945 static void __xen_pgd_unpin(struct mm_struct
*mm
, pgd_t
*pgd
)
947 trace_xen_mmu_pgd_unpin(mm
, pgd
);
951 xen_do_pin(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
955 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
958 xen_do_pin(MMUEXT_UNPIN_TABLE
,
959 PFN_DOWN(__pa(user_pgd
)));
960 xen_unpin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
965 #ifdef CONFIG_X86_PAE
966 /* Need to make sure unshared kernel PMD is unpinned */
967 xen_unpin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
971 __xen_pgd_walk(mm
, pgd
, xen_unpin_page
, USER_LIMIT
);
976 static void xen_pgd_unpin(struct mm_struct
*mm
)
978 __xen_pgd_unpin(mm
, mm
->pgd
);
982 * On resume, undo any pinning done at save, so that the rest of the
983 * kernel doesn't see any unexpected pinned pagetables.
985 void xen_mm_unpin_all(void)
989 spin_lock(&pgd_lock
);
991 list_for_each_entry(page
, &pgd_list
, lru
) {
992 if (PageSavePinned(page
)) {
993 BUG_ON(!PagePinned(page
));
994 __xen_pgd_unpin(&init_mm
, (pgd_t
*)page_address(page
));
995 ClearPageSavePinned(page
);
999 spin_unlock(&pgd_lock
);
1002 static void xen_activate_mm(struct mm_struct
*prev
, struct mm_struct
*next
)
1004 spin_lock(&next
->page_table_lock
);
1006 spin_unlock(&next
->page_table_lock
);
1009 static void xen_dup_mmap(struct mm_struct
*oldmm
, struct mm_struct
*mm
)
1011 spin_lock(&mm
->page_table_lock
);
1013 spin_unlock(&mm
->page_table_lock
);
1018 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1019 we need to repoint it somewhere else before we can unpin it. */
1020 static void drop_other_mm_ref(void *info
)
1022 struct mm_struct
*mm
= info
;
1023 struct mm_struct
*active_mm
;
1025 active_mm
= this_cpu_read(cpu_tlbstate
.active_mm
);
1027 if (active_mm
== mm
&& this_cpu_read(cpu_tlbstate
.state
) != TLBSTATE_OK
)
1028 leave_mm(smp_processor_id());
1030 /* If this cpu still has a stale cr3 reference, then make sure
1031 it has been flushed. */
1032 if (this_cpu_read(xen_current_cr3
) == __pa(mm
->pgd
))
1033 load_cr3(swapper_pg_dir
);
1036 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1041 if (current
->active_mm
== mm
) {
1042 if (current
->mm
== mm
)
1043 load_cr3(swapper_pg_dir
);
1045 leave_mm(smp_processor_id());
1048 /* Get the "official" set of cpus referring to our pagetable. */
1049 if (!alloc_cpumask_var(&mask
, GFP_ATOMIC
)) {
1050 for_each_online_cpu(cpu
) {
1051 if (!cpumask_test_cpu(cpu
, mm_cpumask(mm
))
1052 && per_cpu(xen_current_cr3
, cpu
) != __pa(mm
->pgd
))
1054 smp_call_function_single(cpu
, drop_other_mm_ref
, mm
, 1);
1058 cpumask_copy(mask
, mm_cpumask(mm
));
1060 /* It's possible that a vcpu may have a stale reference to our
1061 cr3, because its in lazy mode, and it hasn't yet flushed
1062 its set of pending hypercalls yet. In this case, we can
1063 look at its actual current cr3 value, and force it to flush
1065 for_each_online_cpu(cpu
) {
1066 if (per_cpu(xen_current_cr3
, cpu
) == __pa(mm
->pgd
))
1067 cpumask_set_cpu(cpu
, mask
);
1070 if (!cpumask_empty(mask
))
1071 smp_call_function_many(mask
, drop_other_mm_ref
, mm
, 1);
1072 free_cpumask_var(mask
);
1075 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1077 if (current
->active_mm
== mm
)
1078 load_cr3(swapper_pg_dir
);
1083 * While a process runs, Xen pins its pagetables, which means that the
1084 * hypervisor forces it to be read-only, and it controls all updates
1085 * to it. This means that all pagetable updates have to go via the
1086 * hypervisor, which is moderately expensive.
1088 * Since we're pulling the pagetable down, we switch to use init_mm,
1089 * unpin old process pagetable and mark it all read-write, which
1090 * allows further operations on it to be simple memory accesses.
1092 * The only subtle point is that another CPU may be still using the
1093 * pagetable because of lazy tlb flushing. This means we need need to
1094 * switch all CPUs off this pagetable before we can unpin it.
1096 static void xen_exit_mmap(struct mm_struct
*mm
)
1098 get_cpu(); /* make sure we don't move around */
1099 xen_drop_mm_ref(mm
);
1102 spin_lock(&mm
->page_table_lock
);
1104 /* pgd may not be pinned in the error exit path of execve */
1105 if (xen_page_pinned(mm
->pgd
))
1108 spin_unlock(&mm
->page_table_lock
);
1111 static void xen_post_allocator_init(void);
1113 static void __init
pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1115 struct mmuext_op op
;
1118 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
1119 if (HYPERVISOR_mmuext_op(&op
, 1, NULL
, DOMID_SELF
))
1123 #ifdef CONFIG_X86_64
1124 static void __init
xen_cleanhighmap(unsigned long vaddr
,
1125 unsigned long vaddr_end
)
1127 unsigned long kernel_end
= roundup((unsigned long)_brk_end
, PMD_SIZE
) - 1;
1128 pmd_t
*pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
1130 /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1131 * We include the PMD passed in on _both_ boundaries. */
1132 for (; vaddr
<= vaddr_end
&& (pmd
< (level2_kernel_pgt
+ PTRS_PER_PMD
));
1133 pmd
++, vaddr
+= PMD_SIZE
) {
1136 if (vaddr
< (unsigned long) _text
|| vaddr
> kernel_end
)
1137 set_pmd(pmd
, __pmd(0));
1139 /* In case we did something silly, we should crash in this function
1140 * instead of somewhere later and be confusing. */
1145 * Make a page range writeable and free it.
1147 static void __init
xen_free_ro_pages(unsigned long paddr
, unsigned long size
)
1149 void *vaddr
= __va(paddr
);
1150 void *vaddr_end
= vaddr
+ size
;
1152 for (; vaddr
< vaddr_end
; vaddr
+= PAGE_SIZE
)
1153 make_lowmem_page_readwrite(vaddr
);
1155 memblock_free(paddr
, size
);
1158 static void __init
xen_cleanmfnmap_free_pgtbl(void *pgtbl
, bool unpin
)
1160 unsigned long pa
= __pa(pgtbl
) & PHYSICAL_PAGE_MASK
;
1163 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(pa
));
1164 ClearPagePinned(virt_to_page(__va(pa
)));
1165 xen_free_ro_pages(pa
, PAGE_SIZE
);
1168 static void __init
xen_cleanmfnmap_pmd(pmd_t
*pmd
, bool unpin
)
1174 if (pmd_large(*pmd
)) {
1175 pa
= pmd_val(*pmd
) & PHYSICAL_PAGE_MASK
;
1176 xen_free_ro_pages(pa
, PMD_SIZE
);
1180 pte_tbl
= pte_offset_kernel(pmd
, 0);
1181 for (i
= 0; i
< PTRS_PER_PTE
; i
++) {
1182 if (pte_none(pte_tbl
[i
]))
1184 pa
= pte_pfn(pte_tbl
[i
]) << PAGE_SHIFT
;
1185 xen_free_ro_pages(pa
, PAGE_SIZE
);
1187 set_pmd(pmd
, __pmd(0));
1188 xen_cleanmfnmap_free_pgtbl(pte_tbl
, unpin
);
1191 static void __init
xen_cleanmfnmap_pud(pud_t
*pud
, bool unpin
)
1197 if (pud_large(*pud
)) {
1198 pa
= pud_val(*pud
) & PHYSICAL_PAGE_MASK
;
1199 xen_free_ro_pages(pa
, PUD_SIZE
);
1203 pmd_tbl
= pmd_offset(pud
, 0);
1204 for (i
= 0; i
< PTRS_PER_PMD
; i
++) {
1205 if (pmd_none(pmd_tbl
[i
]))
1207 xen_cleanmfnmap_pmd(pmd_tbl
+ i
, unpin
);
1209 set_pud(pud
, __pud(0));
1210 xen_cleanmfnmap_free_pgtbl(pmd_tbl
, unpin
);
1213 static void __init
xen_cleanmfnmap_p4d(p4d_t
*p4d
, bool unpin
)
1219 if (p4d_large(*p4d
)) {
1220 pa
= p4d_val(*p4d
) & PHYSICAL_PAGE_MASK
;
1221 xen_free_ro_pages(pa
, P4D_SIZE
);
1225 pud_tbl
= pud_offset(p4d
, 0);
1226 for (i
= 0; i
< PTRS_PER_PUD
; i
++) {
1227 if (pud_none(pud_tbl
[i
]))
1229 xen_cleanmfnmap_pud(pud_tbl
+ i
, unpin
);
1231 set_p4d(p4d
, __p4d(0));
1232 xen_cleanmfnmap_free_pgtbl(pud_tbl
, unpin
);
1236 * Since it is well isolated we can (and since it is perhaps large we should)
1237 * also free the page tables mapping the initial P->M table.
1239 static void __init
xen_cleanmfnmap(unsigned long vaddr
)
1246 unpin
= (vaddr
== 2 * PGDIR_SIZE
);
1248 pgd
= pgd_offset_k(vaddr
);
1249 p4d
= p4d_offset(pgd
, 0);
1250 for (i
= 0; i
< PTRS_PER_P4D
; i
++) {
1251 if (p4d_none(p4d
[i
]))
1253 xen_cleanmfnmap_p4d(p4d
+ i
, unpin
);
1255 if (IS_ENABLED(CONFIG_X86_5LEVEL
)) {
1256 set_pgd(pgd
, __pgd(0));
1257 xen_cleanmfnmap_free_pgtbl(p4d
, unpin
);
1261 static void __init
xen_pagetable_p2m_free(void)
1266 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
1268 /* No memory or already called. */
1269 if ((unsigned long)xen_p2m_addr
== xen_start_info
->mfn_list
)
1272 /* using __ka address and sticking INVALID_P2M_ENTRY! */
1273 memset((void *)xen_start_info
->mfn_list
, 0xff, size
);
1275 addr
= xen_start_info
->mfn_list
;
1277 * We could be in __ka space.
1278 * We roundup to the PMD, which means that if anybody at this stage is
1279 * using the __ka address of xen_start_info or
1280 * xen_start_info->shared_info they are in going to crash. Fortunatly
1281 * we have already revectored in xen_setup_kernel_pagetable and in
1282 * xen_setup_shared_info.
1284 size
= roundup(size
, PMD_SIZE
);
1286 if (addr
>= __START_KERNEL_map
) {
1287 xen_cleanhighmap(addr
, addr
+ size
);
1288 size
= PAGE_ALIGN(xen_start_info
->nr_pages
*
1289 sizeof(unsigned long));
1290 memblock_free(__pa(addr
), size
);
1292 xen_cleanmfnmap(addr
);
1296 static void __init
xen_pagetable_cleanhighmap(void)
1301 /* At this stage, cleanup_highmap has already cleaned __ka space
1302 * from _brk_limit way up to the max_pfn_mapped (which is the end of
1303 * the ramdisk). We continue on, erasing PMD entries that point to page
1304 * tables - do note that they are accessible at this stage via __va.
1305 * For good measure we also round up to the PMD - which means that if
1306 * anybody is using __ka address to the initial boot-stack - and try
1307 * to use it - they are going to crash. The xen_start_info has been
1308 * taken care of already in xen_setup_kernel_pagetable. */
1309 addr
= xen_start_info
->pt_base
;
1310 size
= roundup(xen_start_info
->nr_pt_frames
* PAGE_SIZE
, PMD_SIZE
);
1312 xen_cleanhighmap(addr
, addr
+ size
);
1313 xen_start_info
->pt_base
= (unsigned long)__va(__pa(xen_start_info
->pt_base
));
1315 /* This is superfluous and is not necessary, but you know what
1316 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1317 * anything at this stage. */
1318 xen_cleanhighmap(MODULES_VADDR
, roundup(MODULES_VADDR
, PUD_SIZE
) - 1);
1323 static void __init
xen_pagetable_p2m_setup(void)
1325 if (xen_feature(XENFEAT_auto_translated_physmap
))
1328 xen_vmalloc_p2m_tree();
1330 #ifdef CONFIG_X86_64
1331 xen_pagetable_p2m_free();
1333 xen_pagetable_cleanhighmap();
1335 /* And revector! Bye bye old array */
1336 xen_start_info
->mfn_list
= (unsigned long)xen_p2m_addr
;
1339 static void __init
xen_pagetable_init(void)
1342 xen_post_allocator_init();
1344 xen_pagetable_p2m_setup();
1346 /* Allocate and initialize top and mid mfn levels for p2m structure */
1347 xen_build_mfn_list_list();
1349 /* Remap memory freed due to conflicts with E820 map */
1350 if (!xen_feature(XENFEAT_auto_translated_physmap
))
1353 xen_setup_shared_info();
1355 static void xen_write_cr2(unsigned long cr2
)
1357 this_cpu_read(xen_vcpu
)->arch
.cr2
= cr2
;
1360 static unsigned long xen_read_cr2(void)
1362 return this_cpu_read(xen_vcpu
)->arch
.cr2
;
1365 unsigned long xen_read_cr2_direct(void)
1367 return this_cpu_read(xen_vcpu_info
.arch
.cr2
);
1370 void xen_flush_tlb_all(void)
1372 struct mmuext_op
*op
;
1373 struct multicall_space mcs
;
1375 trace_xen_mmu_flush_tlb_all(0);
1379 mcs
= xen_mc_entry(sizeof(*op
));
1382 op
->cmd
= MMUEXT_TLB_FLUSH_ALL
;
1383 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1385 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1389 static void xen_flush_tlb(void)
1391 struct mmuext_op
*op
;
1392 struct multicall_space mcs
;
1394 trace_xen_mmu_flush_tlb(0);
1398 mcs
= xen_mc_entry(sizeof(*op
));
1401 op
->cmd
= MMUEXT_TLB_FLUSH_LOCAL
;
1402 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1404 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1409 static void xen_flush_tlb_single(unsigned long addr
)
1411 struct mmuext_op
*op
;
1412 struct multicall_space mcs
;
1414 trace_xen_mmu_flush_tlb_single(addr
);
1418 mcs
= xen_mc_entry(sizeof(*op
));
1420 op
->cmd
= MMUEXT_INVLPG_LOCAL
;
1421 op
->arg1
.linear_addr
= addr
& PAGE_MASK
;
1422 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1424 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1429 static void xen_flush_tlb_others(const struct cpumask
*cpus
,
1430 struct mm_struct
*mm
, unsigned long start
,
1434 struct mmuext_op op
;
1436 DECLARE_BITMAP(mask
, num_processors
);
1438 DECLARE_BITMAP(mask
, NR_CPUS
);
1441 struct multicall_space mcs
;
1443 trace_xen_mmu_flush_tlb_others(cpus
, mm
, start
, end
);
1445 if (cpumask_empty(cpus
))
1446 return; /* nothing to do */
1448 mcs
= xen_mc_entry(sizeof(*args
));
1450 args
->op
.arg2
.vcpumask
= to_cpumask(args
->mask
);
1452 /* Remove us, and any offline CPUS. */
1453 cpumask_and(to_cpumask(args
->mask
), cpus
, cpu_online_mask
);
1454 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args
->mask
));
1456 args
->op
.cmd
= MMUEXT_TLB_FLUSH_MULTI
;
1457 if (end
!= TLB_FLUSH_ALL
&& (end
- start
) <= PAGE_SIZE
) {
1458 args
->op
.cmd
= MMUEXT_INVLPG_MULTI
;
1459 args
->op
.arg1
.linear_addr
= start
;
1462 MULTI_mmuext_op(mcs
.mc
, &args
->op
, 1, NULL
, DOMID_SELF
);
1464 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1467 static unsigned long xen_read_cr3(void)
1469 return this_cpu_read(xen_cr3
);
1472 static void set_current_cr3(void *v
)
1474 this_cpu_write(xen_current_cr3
, (unsigned long)v
);
1477 static void __xen_write_cr3(bool kernel
, unsigned long cr3
)
1479 struct mmuext_op op
;
1482 trace_xen_mmu_write_cr3(kernel
, cr3
);
1485 mfn
= pfn_to_mfn(PFN_DOWN(cr3
));
1489 WARN_ON(mfn
== 0 && kernel
);
1491 op
.cmd
= kernel
? MMUEXT_NEW_BASEPTR
: MMUEXT_NEW_USER_BASEPTR
;
1494 xen_extend_mmuext_op(&op
);
1497 this_cpu_write(xen_cr3
, cr3
);
1499 /* Update xen_current_cr3 once the batch has actually
1501 xen_mc_callback(set_current_cr3
, (void *)cr3
);
1504 static void xen_write_cr3(unsigned long cr3
)
1506 BUG_ON(preemptible());
1508 xen_mc_batch(); /* disables interrupts */
1510 /* Update while interrupts are disabled, so its atomic with
1512 this_cpu_write(xen_cr3
, cr3
);
1514 __xen_write_cr3(true, cr3
);
1516 #ifdef CONFIG_X86_64
1518 pgd_t
*user_pgd
= xen_get_user_pgd(__va(cr3
));
1520 __xen_write_cr3(false, __pa(user_pgd
));
1522 __xen_write_cr3(false, 0);
1526 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1529 #ifdef CONFIG_X86_64
1531 * At the start of the day - when Xen launches a guest, it has already
1532 * built pagetables for the guest. We diligently look over them
1533 * in xen_setup_kernel_pagetable and graft as appropriate them in the
1534 * init_level4_pgt and its friends. Then when we are happy we load
1535 * the new init_level4_pgt - and continue on.
1537 * The generic code starts (start_kernel) and 'init_mem_mapping' sets
1538 * up the rest of the pagetables. When it has completed it loads the cr3.
1539 * N.B. that baremetal would start at 'start_kernel' (and the early
1540 * #PF handler would create bootstrap pagetables) - so we are running
1541 * with the same assumptions as what to do when write_cr3 is executed
1544 * Since there are no user-page tables at all, we have two variants
1545 * of xen_write_cr3 - the early bootup (this one), and the late one
1546 * (xen_write_cr3). The reason we have to do that is that in 64-bit
1547 * the Linux kernel and user-space are both in ring 3 while the
1548 * hypervisor is in ring 0.
1550 static void __init
xen_write_cr3_init(unsigned long cr3
)
1552 BUG_ON(preemptible());
1554 xen_mc_batch(); /* disables interrupts */
1556 /* Update while interrupts are disabled, so its atomic with
1558 this_cpu_write(xen_cr3
, cr3
);
1560 __xen_write_cr3(true, cr3
);
1562 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1566 static int xen_pgd_alloc(struct mm_struct
*mm
)
1568 pgd_t
*pgd
= mm
->pgd
;
1571 BUG_ON(PagePinned(virt_to_page(pgd
)));
1573 #ifdef CONFIG_X86_64
1575 struct page
*page
= virt_to_page(pgd
);
1578 BUG_ON(page
->private != 0);
1582 user_pgd
= (pgd_t
*)__get_free_page(GFP_KERNEL
| __GFP_ZERO
);
1583 page
->private = (unsigned long)user_pgd
;
1585 if (user_pgd
!= NULL
) {
1586 #ifdef CONFIG_X86_VSYSCALL_EMULATION
1587 user_pgd
[pgd_index(VSYSCALL_ADDR
)] =
1588 __pgd(__pa(level3_user_vsyscall
) | _PAGE_TABLE
);
1593 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd
))));
1599 static void xen_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
1601 #ifdef CONFIG_X86_64
1602 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1605 free_page((unsigned long)user_pgd
);
1610 * Init-time set_pte while constructing initial pagetables, which
1611 * doesn't allow RO page table pages to be remapped RW.
1613 * If there is no MFN for this PFN then this page is initially
1614 * ballooned out so clear the PTE (as in decrease_reservation() in
1615 * drivers/xen/balloon.c).
1617 * Many of these PTE updates are done on unpinned and writable pages
1618 * and doing a hypercall for these is unnecessary and expensive. At
1619 * this point it is not possible to tell if a page is pinned or not,
1620 * so always write the PTE directly and rely on Xen trapping and
1621 * emulating any updates as necessary.
1623 __visible pte_t
xen_make_pte_init(pteval_t pte
)
1625 #ifdef CONFIG_X86_64
1629 * Pages belonging to the initial p2m list mapped outside the default
1630 * address range must be mapped read-only. This region contains the
1631 * page tables for mapping the p2m list, too, and page tables MUST be
1634 pfn
= (pte
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
1635 if (xen_start_info
->mfn_list
< __START_KERNEL_map
&&
1636 pfn
>= xen_start_info
->first_p2m_pfn
&&
1637 pfn
< xen_start_info
->first_p2m_pfn
+ xen_start_info
->nr_p2m_frames
)
1640 pte
= pte_pfn_to_mfn(pte
);
1641 return native_make_pte(pte
);
1643 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte_init
);
1645 static void __init
xen_set_pte_init(pte_t
*ptep
, pte_t pte
)
1647 #ifdef CONFIG_X86_32
1648 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1649 if (pte_mfn(pte
) != INVALID_P2M_ENTRY
1650 && pte_val_ma(*ptep
) & _PAGE_PRESENT
)
1651 pte
= __pte_ma(((pte_val_ma(*ptep
) & _PAGE_RW
) | ~_PAGE_RW
) &
1654 native_set_pte(ptep
, pte
);
1657 /* Early in boot, while setting up the initial pagetable, assume
1658 everything is pinned. */
1659 static void __init
xen_alloc_pte_init(struct mm_struct
*mm
, unsigned long pfn
)
1661 #ifdef CONFIG_FLATMEM
1662 BUG_ON(mem_map
); /* should only be used early */
1664 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1665 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1668 /* Used for pmd and pud */
1669 static void __init
xen_alloc_pmd_init(struct mm_struct
*mm
, unsigned long pfn
)
1671 #ifdef CONFIG_FLATMEM
1672 BUG_ON(mem_map
); /* should only be used early */
1674 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1677 /* Early release_pte assumes that all pts are pinned, since there's
1678 only init_mm and anything attached to that is pinned. */
1679 static void __init
xen_release_pte_init(unsigned long pfn
)
1681 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1682 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1685 static void __init
xen_release_pmd_init(unsigned long pfn
)
1687 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1690 static inline void __pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1692 struct multicall_space mcs
;
1693 struct mmuext_op
*op
;
1695 mcs
= __xen_mc_entry(sizeof(*op
));
1698 op
->arg1
.mfn
= pfn_to_mfn(pfn
);
1700 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
1703 static inline void __set_pfn_prot(unsigned long pfn
, pgprot_t prot
)
1705 struct multicall_space mcs
;
1706 unsigned long addr
= (unsigned long)__va(pfn
<< PAGE_SHIFT
);
1708 mcs
= __xen_mc_entry(0);
1709 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)addr
,
1710 pfn_pte(pfn
, prot
), 0);
1713 /* This needs to make sure the new pte page is pinned iff its being
1714 attached to a pinned pagetable. */
1715 static inline void xen_alloc_ptpage(struct mm_struct
*mm
, unsigned long pfn
,
1718 bool pinned
= PagePinned(virt_to_page(mm
->pgd
));
1720 trace_xen_mmu_alloc_ptpage(mm
, pfn
, level
, pinned
);
1723 struct page
*page
= pfn_to_page(pfn
);
1725 SetPagePinned(page
);
1727 if (!PageHighMem(page
)) {
1730 __set_pfn_prot(pfn
, PAGE_KERNEL_RO
);
1732 if (level
== PT_PTE
&& USE_SPLIT_PTE_PTLOCKS
)
1733 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1735 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1737 /* make sure there are no stray mappings of
1739 kmap_flush_unused();
1744 static void xen_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
)
1746 xen_alloc_ptpage(mm
, pfn
, PT_PTE
);
1749 static void xen_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
)
1751 xen_alloc_ptpage(mm
, pfn
, PT_PMD
);
1754 /* This should never happen until we're OK to use struct page */
1755 static inline void xen_release_ptpage(unsigned long pfn
, unsigned level
)
1757 struct page
*page
= pfn_to_page(pfn
);
1758 bool pinned
= PagePinned(page
);
1760 trace_xen_mmu_release_ptpage(pfn
, level
, pinned
);
1763 if (!PageHighMem(page
)) {
1766 if (level
== PT_PTE
&& USE_SPLIT_PTE_PTLOCKS
)
1767 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1769 __set_pfn_prot(pfn
, PAGE_KERNEL
);
1771 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1773 ClearPagePinned(page
);
1777 static void xen_release_pte(unsigned long pfn
)
1779 xen_release_ptpage(pfn
, PT_PTE
);
1782 static void xen_release_pmd(unsigned long pfn
)
1784 xen_release_ptpage(pfn
, PT_PMD
);
1787 #if CONFIG_PGTABLE_LEVELS >= 4
1788 static void xen_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
)
1790 xen_alloc_ptpage(mm
, pfn
, PT_PUD
);
1793 static void xen_release_pud(unsigned long pfn
)
1795 xen_release_ptpage(pfn
, PT_PUD
);
1799 void __init
xen_reserve_top(void)
1801 #ifdef CONFIG_X86_32
1802 unsigned long top
= HYPERVISOR_VIRT_START
;
1803 struct xen_platform_parameters pp
;
1805 if (HYPERVISOR_xen_version(XENVER_platform_parameters
, &pp
) == 0)
1806 top
= pp
.virt_start
;
1808 reserve_top_address(-top
);
1809 #endif /* CONFIG_X86_32 */
1813 * Like __va(), but returns address in the kernel mapping (which is
1814 * all we have until the physical memory mapping has been set up.
1816 static void * __init
__ka(phys_addr_t paddr
)
1818 #ifdef CONFIG_X86_64
1819 return (void *)(paddr
+ __START_KERNEL_map
);
1825 /* Convert a machine address to physical address */
1826 static unsigned long __init
m2p(phys_addr_t maddr
)
1830 maddr
&= PTE_PFN_MASK
;
1831 paddr
= mfn_to_pfn(maddr
>> PAGE_SHIFT
) << PAGE_SHIFT
;
1836 /* Convert a machine address to kernel virtual */
1837 static void * __init
m2v(phys_addr_t maddr
)
1839 return __ka(m2p(maddr
));
1842 /* Set the page permissions on an identity-mapped pages */
1843 static void __init
set_page_prot_flags(void *addr
, pgprot_t prot
,
1844 unsigned long flags
)
1846 unsigned long pfn
= __pa(addr
) >> PAGE_SHIFT
;
1847 pte_t pte
= pfn_pte(pfn
, prot
);
1849 if (HYPERVISOR_update_va_mapping((unsigned long)addr
, pte
, flags
))
1852 static void __init
set_page_prot(void *addr
, pgprot_t prot
)
1854 return set_page_prot_flags(addr
, prot
, UVMF_NONE
);
1856 #ifdef CONFIG_X86_32
1857 static void __init
xen_map_identity_early(pmd_t
*pmd
, unsigned long max_pfn
)
1859 unsigned pmdidx
, pteidx
;
1863 level1_ident_pgt
= extend_brk(sizeof(pte_t
) * LEVEL1_IDENT_ENTRIES
,
1868 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
&& pfn
< max_pfn
; pmdidx
++) {
1871 /* Reuse or allocate a page of ptes */
1872 if (pmd_present(pmd
[pmdidx
]))
1873 pte_page
= m2v(pmd
[pmdidx
].pmd
);
1875 /* Check for free pte pages */
1876 if (ident_pte
== LEVEL1_IDENT_ENTRIES
)
1879 pte_page
= &level1_ident_pgt
[ident_pte
];
1880 ident_pte
+= PTRS_PER_PTE
;
1882 pmd
[pmdidx
] = __pmd(__pa(pte_page
) | _PAGE_TABLE
);
1885 /* Install mappings */
1886 for (pteidx
= 0; pteidx
< PTRS_PER_PTE
; pteidx
++, pfn
++) {
1889 if (pfn
> max_pfn_mapped
)
1890 max_pfn_mapped
= pfn
;
1892 if (!pte_none(pte_page
[pteidx
]))
1895 pte
= pfn_pte(pfn
, PAGE_KERNEL_EXEC
);
1896 pte_page
[pteidx
] = pte
;
1900 for (pteidx
= 0; pteidx
< ident_pte
; pteidx
+= PTRS_PER_PTE
)
1901 set_page_prot(&level1_ident_pgt
[pteidx
], PAGE_KERNEL_RO
);
1903 set_page_prot(pmd
, PAGE_KERNEL_RO
);
1906 void __init
xen_setup_machphys_mapping(void)
1908 struct xen_machphys_mapping mapping
;
1910 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping
, &mapping
) == 0) {
1911 machine_to_phys_mapping
= (unsigned long *)mapping
.v_start
;
1912 machine_to_phys_nr
= mapping
.max_mfn
+ 1;
1914 machine_to_phys_nr
= MACH2PHYS_NR_ENTRIES
;
1916 #ifdef CONFIG_X86_32
1917 WARN_ON((machine_to_phys_mapping
+ (machine_to_phys_nr
- 1))
1918 < machine_to_phys_mapping
);
1922 #ifdef CONFIG_X86_64
1923 static void __init
convert_pfn_mfn(void *v
)
1928 /* All levels are converted the same way, so just treat them
1930 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
1931 pte
[i
] = xen_make_pte(pte
[i
].pte
);
1933 static void __init
check_pt_base(unsigned long *pt_base
, unsigned long *pt_end
,
1936 if (*pt_base
== PFN_DOWN(__pa(addr
))) {
1937 set_page_prot_flags((void *)addr
, PAGE_KERNEL
, UVMF_INVLPG
);
1938 clear_page((void *)addr
);
1941 if (*pt_end
== PFN_DOWN(__pa(addr
))) {
1942 set_page_prot_flags((void *)addr
, PAGE_KERNEL
, UVMF_INVLPG
);
1943 clear_page((void *)addr
);
1948 * Set up the initial kernel pagetable.
1950 * We can construct this by grafting the Xen provided pagetable into
1951 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1952 * level2_ident_pgt, and level2_kernel_pgt. This means that only the
1953 * kernel has a physical mapping to start with - but that's enough to
1954 * get __va working. We need to fill in the rest of the physical
1955 * mapping once some sort of allocator has been set up.
1957 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
1961 unsigned long addr
[3];
1962 unsigned long pt_base
, pt_end
;
1965 /* max_pfn_mapped is the last pfn mapped in the initial memory
1966 * mappings. Considering that on Xen after the kernel mappings we
1967 * have the mappings of some pages that don't exist in pfn space, we
1968 * set max_pfn_mapped to the last real pfn mapped. */
1969 if (xen_start_info
->mfn_list
< __START_KERNEL_map
)
1970 max_pfn_mapped
= xen_start_info
->first_p2m_pfn
;
1972 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->mfn_list
));
1974 pt_base
= PFN_DOWN(__pa(xen_start_info
->pt_base
));
1975 pt_end
= pt_base
+ xen_start_info
->nr_pt_frames
;
1977 /* Zap identity mapping */
1978 init_level4_pgt
[0] = __pgd(0);
1980 if (!xen_feature(XENFEAT_auto_translated_physmap
)) {
1981 /* Pre-constructed entries are in pfn, so convert to mfn */
1982 /* L4[272] -> level3_ident_pgt
1983 * L4[511] -> level3_kernel_pgt */
1984 convert_pfn_mfn(init_level4_pgt
);
1986 /* L3_i[0] -> level2_ident_pgt */
1987 convert_pfn_mfn(level3_ident_pgt
);
1988 /* L3_k[510] -> level2_kernel_pgt
1989 * L3_k[511] -> level2_fixmap_pgt */
1990 convert_pfn_mfn(level3_kernel_pgt
);
1992 /* L3_k[511][506] -> level1_fixmap_pgt */
1993 convert_pfn_mfn(level2_fixmap_pgt
);
1995 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1996 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
)].pgd
);
1997 l2
= m2v(l3
[pud_index(__START_KERNEL_map
)].pud
);
1999 addr
[0] = (unsigned long)pgd
;
2000 addr
[1] = (unsigned long)l3
;
2001 addr
[2] = (unsigned long)l2
;
2002 /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
2003 * Both L4[272][0] and L4[511][510] have entries that point to the same
2004 * L2 (PMD) tables. Meaning that if you modify it in __va space
2005 * it will be also modified in the __ka space! (But if you just
2006 * modify the PMD table to point to other PTE's or none, then you
2007 * are OK - which is what cleanup_highmap does) */
2008 copy_page(level2_ident_pgt
, l2
);
2009 /* Graft it onto L4[511][510] */
2010 copy_page(level2_kernel_pgt
, l2
);
2012 /* Copy the initial P->M table mappings if necessary. */
2013 i
= pgd_index(xen_start_info
->mfn_list
);
2014 if (i
&& i
< pgd_index(__START_KERNEL_map
))
2015 init_level4_pgt
[i
] = ((pgd_t
*)xen_start_info
->pt_base
)[i
];
2017 if (!xen_feature(XENFEAT_auto_translated_physmap
)) {
2018 /* Make pagetable pieces RO */
2019 set_page_prot(init_level4_pgt
, PAGE_KERNEL_RO
);
2020 set_page_prot(level3_ident_pgt
, PAGE_KERNEL_RO
);
2021 set_page_prot(level3_kernel_pgt
, PAGE_KERNEL_RO
);
2022 set_page_prot(level3_user_vsyscall
, PAGE_KERNEL_RO
);
2023 set_page_prot(level2_ident_pgt
, PAGE_KERNEL_RO
);
2024 set_page_prot(level2_kernel_pgt
, PAGE_KERNEL_RO
);
2025 set_page_prot(level2_fixmap_pgt
, PAGE_KERNEL_RO
);
2026 set_page_prot(level1_fixmap_pgt
, PAGE_KERNEL_RO
);
2028 /* Pin down new L4 */
2029 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE
,
2030 PFN_DOWN(__pa_symbol(init_level4_pgt
)));
2032 /* Unpin Xen-provided one */
2033 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
2036 * At this stage there can be no user pgd, and no page
2037 * structure to attach it to, so make sure we just set kernel
2041 __xen_write_cr3(true, __pa(init_level4_pgt
));
2042 xen_mc_issue(PARAVIRT_LAZY_CPU
);
2044 native_write_cr3(__pa(init_level4_pgt
));
2046 /* We can't that easily rip out L3 and L2, as the Xen pagetables are
2047 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for
2048 * the initial domain. For guests using the toolstack, they are in:
2049 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
2050 * rip out the [L4] (pgd), but for guests we shave off three pages.
2052 for (i
= 0; i
< ARRAY_SIZE(addr
); i
++)
2053 check_pt_base(&pt_base
, &pt_end
, addr
[i
]);
2055 /* Our (by three pages) smaller Xen pagetable that we are using */
2056 xen_pt_base
= PFN_PHYS(pt_base
);
2057 xen_pt_size
= (pt_end
- pt_base
) * PAGE_SIZE
;
2058 memblock_reserve(xen_pt_base
, xen_pt_size
);
2060 /* Revector the xen_start_info */
2061 xen_start_info
= (struct start_info
*)__va(__pa(xen_start_info
));
2065 * Read a value from a physical address.
2067 static unsigned long __init
xen_read_phys_ulong(phys_addr_t addr
)
2069 unsigned long *vaddr
;
2072 vaddr
= early_memremap_ro(addr
, sizeof(val
));
2074 early_memunmap(vaddr
, sizeof(val
));
2079 * Translate a virtual address to a physical one without relying on mapped
2082 static phys_addr_t __init
xen_early_virt_to_phys(unsigned long vaddr
)
2091 pgd
= native_make_pgd(xen_read_phys_ulong(pa
+ pgd_index(vaddr
) *
2093 if (!pgd_present(pgd
))
2096 pa
= pgd_val(pgd
) & PTE_PFN_MASK
;
2097 pud
= native_make_pud(xen_read_phys_ulong(pa
+ pud_index(vaddr
) *
2099 if (!pud_present(pud
))
2101 pa
= pud_pfn(pud
) << PAGE_SHIFT
;
2103 return pa
+ (vaddr
& ~PUD_MASK
);
2105 pmd
= native_make_pmd(xen_read_phys_ulong(pa
+ pmd_index(vaddr
) *
2107 if (!pmd_present(pmd
))
2109 pa
= pmd_pfn(pmd
) << PAGE_SHIFT
;
2111 return pa
+ (vaddr
& ~PMD_MASK
);
2113 pte
= native_make_pte(xen_read_phys_ulong(pa
+ pte_index(vaddr
) *
2115 if (!pte_present(pte
))
2117 pa
= pte_pfn(pte
) << PAGE_SHIFT
;
2119 return pa
| (vaddr
& ~PAGE_MASK
);
2123 * Find a new area for the hypervisor supplied p2m list and relocate the p2m to
2126 void __init
xen_relocate_p2m(void)
2128 phys_addr_t size
, new_area
, pt_phys
, pmd_phys
, pud_phys
, p4d_phys
;
2129 unsigned long p2m_pfn
, p2m_pfn_end
, n_frames
, pfn
, pfn_end
;
2130 int n_pte
, n_pt
, n_pmd
, n_pud
, n_p4d
, idx_pte
, idx_pt
, idx_pmd
, idx_pud
, idx_p4d
;
2136 unsigned long *new_p2m
;
2139 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
2140 n_pte
= roundup(size
, PAGE_SIZE
) >> PAGE_SHIFT
;
2141 n_pt
= roundup(size
, PMD_SIZE
) >> PMD_SHIFT
;
2142 n_pmd
= roundup(size
, PUD_SIZE
) >> PUD_SHIFT
;
2143 n_pud
= roundup(size
, P4D_SIZE
) >> P4D_SHIFT
;
2144 if (PTRS_PER_P4D
> 1)
2145 n_p4d
= roundup(size
, PGDIR_SIZE
) >> PGDIR_SHIFT
;
2148 n_frames
= n_pte
+ n_pt
+ n_pmd
+ n_pud
+ n_p4d
;
2150 new_area
= xen_find_free_area(PFN_PHYS(n_frames
));
2152 xen_raw_console_write("Can't find new memory area for p2m needed due to E820 map conflict\n");
2157 * Setup the page tables for addressing the new p2m list.
2158 * We have asked the hypervisor to map the p2m list at the user address
2159 * PUD_SIZE. It may have done so, or it may have used a kernel space
2160 * address depending on the Xen version.
2161 * To avoid any possible virtual address collision, just use
2162 * 2 * PUD_SIZE for the new area.
2164 p4d_phys
= new_area
;
2165 pud_phys
= p4d_phys
+ PFN_PHYS(n_p4d
);
2166 pmd_phys
= pud_phys
+ PFN_PHYS(n_pud
);
2167 pt_phys
= pmd_phys
+ PFN_PHYS(n_pmd
);
2168 p2m_pfn
= PFN_DOWN(pt_phys
) + n_pt
;
2170 pgd
= __va(read_cr3());
2171 new_p2m
= (unsigned long *)(2 * PGDIR_SIZE
);
2176 p4d
= early_memremap(p4d_phys
, PAGE_SIZE
);
2178 n_pud
= min(save_pud
, PTRS_PER_P4D
);
2180 for (idx_pud
= 0; idx_pud
< n_pud
; idx_pud
++) {
2181 pud
= early_memremap(pud_phys
, PAGE_SIZE
);
2183 for (idx_pmd
= 0; idx_pmd
< min(n_pmd
, PTRS_PER_PUD
);
2185 pmd
= early_memremap(pmd_phys
, PAGE_SIZE
);
2187 for (idx_pt
= 0; idx_pt
< min(n_pt
, PTRS_PER_PMD
);
2189 pt
= early_memremap(pt_phys
, PAGE_SIZE
);
2192 idx_pte
< min(n_pte
, PTRS_PER_PTE
);
2194 set_pte(pt
+ idx_pte
,
2195 pfn_pte(p2m_pfn
, PAGE_KERNEL
));
2198 n_pte
-= PTRS_PER_PTE
;
2199 early_memunmap(pt
, PAGE_SIZE
);
2200 make_lowmem_page_readonly(__va(pt_phys
));
2201 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
,
2203 set_pmd(pmd
+ idx_pt
,
2204 __pmd(_PAGE_TABLE
| pt_phys
));
2205 pt_phys
+= PAGE_SIZE
;
2207 n_pt
-= PTRS_PER_PMD
;
2208 early_memunmap(pmd
, PAGE_SIZE
);
2209 make_lowmem_page_readonly(__va(pmd_phys
));
2210 pin_pagetable_pfn(MMUEXT_PIN_L2_TABLE
,
2211 PFN_DOWN(pmd_phys
));
2212 set_pud(pud
+ idx_pmd
, __pud(_PAGE_TABLE
| pmd_phys
));
2213 pmd_phys
+= PAGE_SIZE
;
2215 n_pmd
-= PTRS_PER_PUD
;
2216 early_memunmap(pud
, PAGE_SIZE
);
2217 make_lowmem_page_readonly(__va(pud_phys
));
2218 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(pud_phys
));
2220 set_p4d(p4d
+ idx_pud
, __p4d(_PAGE_TABLE
| pud_phys
));
2222 set_pgd(pgd
+ 2 + idx_pud
, __pgd(_PAGE_TABLE
| pud_phys
));
2223 pud_phys
+= PAGE_SIZE
;
2226 save_pud
-= PTRS_PER_P4D
;
2227 early_memunmap(p4d
, PAGE_SIZE
);
2228 make_lowmem_page_readonly(__va(p4d_phys
));
2229 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE
, PFN_DOWN(p4d_phys
));
2230 set_pgd(pgd
+ 2 + idx_p4d
, __pgd(_PAGE_TABLE
| p4d_phys
));
2231 p4d_phys
+= PAGE_SIZE
;
2233 } while (++idx_p4d
< n_p4d
);
2235 /* Now copy the old p2m info to the new area. */
2236 memcpy(new_p2m
, xen_p2m_addr
, size
);
2237 xen_p2m_addr
= new_p2m
;
2239 /* Release the old p2m list and set new list info. */
2240 p2m_pfn
= PFN_DOWN(xen_early_virt_to_phys(xen_start_info
->mfn_list
));
2242 p2m_pfn_end
= p2m_pfn
+ PFN_DOWN(size
);
2244 if (xen_start_info
->mfn_list
< __START_KERNEL_map
) {
2245 pfn
= xen_start_info
->first_p2m_pfn
;
2246 pfn_end
= xen_start_info
->first_p2m_pfn
+
2247 xen_start_info
->nr_p2m_frames
;
2248 set_pgd(pgd
+ 1, __pgd(0));
2251 pfn_end
= p2m_pfn_end
;
2254 memblock_free(PFN_PHYS(pfn
), PAGE_SIZE
* (pfn_end
- pfn
));
2255 while (pfn
< pfn_end
) {
2256 if (pfn
== p2m_pfn
) {
2260 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
2264 xen_start_info
->mfn_list
= (unsigned long)xen_p2m_addr
;
2265 xen_start_info
->first_p2m_pfn
= PFN_DOWN(new_area
);
2266 xen_start_info
->nr_p2m_frames
= n_frames
;
2269 #else /* !CONFIG_X86_64 */
2270 static RESERVE_BRK_ARRAY(pmd_t
, initial_kernel_pmd
, PTRS_PER_PMD
);
2271 static RESERVE_BRK_ARRAY(pmd_t
, swapper_kernel_pmd
, PTRS_PER_PMD
);
2273 static void __init
xen_write_cr3_init(unsigned long cr3
)
2275 unsigned long pfn
= PFN_DOWN(__pa(swapper_pg_dir
));
2277 BUG_ON(read_cr3() != __pa(initial_page_table
));
2278 BUG_ON(cr3
!= __pa(swapper_pg_dir
));
2281 * We are switching to swapper_pg_dir for the first time (from
2282 * initial_page_table) and therefore need to mark that page
2283 * read-only and then pin it.
2285 * Xen disallows sharing of kernel PMDs for PAE
2286 * guests. Therefore we must copy the kernel PMD from
2287 * initial_page_table into a new kernel PMD to be used in
2290 swapper_kernel_pmd
=
2291 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
2292 copy_page(swapper_kernel_pmd
, initial_kernel_pmd
);
2293 swapper_pg_dir
[KERNEL_PGD_BOUNDARY
] =
2294 __pgd(__pa(swapper_kernel_pmd
) | _PAGE_PRESENT
);
2295 set_page_prot(swapper_kernel_pmd
, PAGE_KERNEL_RO
);
2297 set_page_prot(swapper_pg_dir
, PAGE_KERNEL_RO
);
2299 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
, pfn
);
2301 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
,
2302 PFN_DOWN(__pa(initial_page_table
)));
2303 set_page_prot(initial_page_table
, PAGE_KERNEL
);
2304 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL
);
2306 pv_mmu_ops
.write_cr3
= &xen_write_cr3
;
2310 * For 32 bit domains xen_start_info->pt_base is the pgd address which might be
2311 * not the first page table in the page table pool.
2312 * Iterate through the initial page tables to find the real page table base.
2314 static phys_addr_t
xen_find_pt_base(pmd_t
*pmd
)
2316 phys_addr_t pt_base
, paddr
;
2319 pt_base
= min(__pa(xen_start_info
->pt_base
), __pa(pmd
));
2321 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
; pmdidx
++)
2322 if (pmd_present(pmd
[pmdidx
]) && !pmd_large(pmd
[pmdidx
])) {
2323 paddr
= m2p(pmd
[pmdidx
].pmd
);
2324 pt_base
= min(pt_base
, paddr
);
2330 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
2334 kernel_pmd
= m2v(pgd
[KERNEL_PGD_BOUNDARY
].pgd
);
2336 xen_pt_base
= xen_find_pt_base(kernel_pmd
);
2337 xen_pt_size
= xen_start_info
->nr_pt_frames
* PAGE_SIZE
;
2339 initial_kernel_pmd
=
2340 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
2342 max_pfn_mapped
= PFN_DOWN(xen_pt_base
+ xen_pt_size
+ 512 * 1024);
2344 copy_page(initial_kernel_pmd
, kernel_pmd
);
2346 xen_map_identity_early(initial_kernel_pmd
, max_pfn
);
2348 copy_page(initial_page_table
, pgd
);
2349 initial_page_table
[KERNEL_PGD_BOUNDARY
] =
2350 __pgd(__pa(initial_kernel_pmd
) | _PAGE_PRESENT
);
2352 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL_RO
);
2353 set_page_prot(initial_page_table
, PAGE_KERNEL_RO
);
2354 set_page_prot(empty_zero_page
, PAGE_KERNEL_RO
);
2356 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
2358 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
,
2359 PFN_DOWN(__pa(initial_page_table
)));
2360 xen_write_cr3(__pa(initial_page_table
));
2362 memblock_reserve(xen_pt_base
, xen_pt_size
);
2364 #endif /* CONFIG_X86_64 */
2366 void __init
xen_reserve_special_pages(void)
2370 memblock_reserve(__pa(xen_start_info
), PAGE_SIZE
);
2371 if (xen_start_info
->store_mfn
) {
2372 paddr
= PFN_PHYS(mfn_to_pfn(xen_start_info
->store_mfn
));
2373 memblock_reserve(paddr
, PAGE_SIZE
);
2375 if (!xen_initial_domain()) {
2376 paddr
= PFN_PHYS(mfn_to_pfn(xen_start_info
->console
.domU
.mfn
));
2377 memblock_reserve(paddr
, PAGE_SIZE
);
2381 void __init
xen_pt_check_e820(void)
2383 if (xen_is_e820_reserved(xen_pt_base
, xen_pt_size
)) {
2384 xen_raw_console_write("Xen hypervisor allocated page table memory conflicts with E820 map\n");
2389 static unsigned char dummy_mapping
[PAGE_SIZE
] __page_aligned_bss
;
2391 static void xen_set_fixmap(unsigned idx
, phys_addr_t phys
, pgprot_t prot
)
2395 phys
>>= PAGE_SHIFT
;
2398 case FIX_BTMAP_END
... FIX_BTMAP_BEGIN
:
2400 #ifdef CONFIG_X86_32
2402 # ifdef CONFIG_HIGHMEM
2403 case FIX_KMAP_BEGIN
... FIX_KMAP_END
:
2405 #elif defined(CONFIG_X86_VSYSCALL_EMULATION)
2408 case FIX_TEXT_POKE0
:
2409 case FIX_TEXT_POKE1
:
2410 case FIX_GDT_REMAP_BEGIN
... FIX_GDT_REMAP_END
:
2411 /* All local page mappings */
2412 pte
= pfn_pte(phys
, prot
);
2415 #ifdef CONFIG_X86_LOCAL_APIC
2416 case FIX_APIC_BASE
: /* maps dummy local APIC */
2417 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2421 #ifdef CONFIG_X86_IO_APIC
2422 case FIX_IO_APIC_BASE_0
... FIX_IO_APIC_BASE_END
:
2424 * We just don't map the IO APIC - all access is via
2425 * hypercalls. Keep the address in the pte for reference.
2427 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2431 case FIX_PARAVIRT_BOOTMAP
:
2432 /* This is an MFN, but it isn't an IO mapping from the
2434 pte
= mfn_pte(phys
, prot
);
2438 /* By default, set_fixmap is used for hardware mappings */
2439 pte
= mfn_pte(phys
, prot
);
2443 __native_set_fixmap(idx
, pte
);
2445 #ifdef CONFIG_X86_VSYSCALL_EMULATION
2446 /* Replicate changes to map the vsyscall page into the user
2447 pagetable vsyscall mapping. */
2448 if (idx
== VSYSCALL_PAGE
) {
2449 unsigned long vaddr
= __fix_to_virt(idx
);
2450 set_pte_vaddr_pud(level3_user_vsyscall
, vaddr
, pte
);
2455 static void __init
xen_post_allocator_init(void)
2457 if (xen_feature(XENFEAT_auto_translated_physmap
))
2460 pv_mmu_ops
.set_pte
= xen_set_pte
;
2461 pv_mmu_ops
.set_pmd
= xen_set_pmd
;
2462 pv_mmu_ops
.set_pud
= xen_set_pud
;
2463 #if CONFIG_PGTABLE_LEVELS >= 4
2464 pv_mmu_ops
.set_p4d
= xen_set_p4d
;
2467 /* This will work as long as patching hasn't happened yet
2468 (which it hasn't) */
2469 pv_mmu_ops
.alloc_pte
= xen_alloc_pte
;
2470 pv_mmu_ops
.alloc_pmd
= xen_alloc_pmd
;
2471 pv_mmu_ops
.release_pte
= xen_release_pte
;
2472 pv_mmu_ops
.release_pmd
= xen_release_pmd
;
2473 #if CONFIG_PGTABLE_LEVELS >= 4
2474 pv_mmu_ops
.alloc_pud
= xen_alloc_pud
;
2475 pv_mmu_ops
.release_pud
= xen_release_pud
;
2477 pv_mmu_ops
.make_pte
= PV_CALLEE_SAVE(xen_make_pte
);
2479 #ifdef CONFIG_X86_64
2480 pv_mmu_ops
.write_cr3
= &xen_write_cr3
;
2481 SetPagePinned(virt_to_page(level3_user_vsyscall
));
2483 xen_mark_init_mm_pinned();
2486 static void xen_leave_lazy_mmu(void)
2490 paravirt_leave_lazy_mmu();
2494 static const struct pv_mmu_ops xen_mmu_ops __initconst
= {
2495 .read_cr2
= xen_read_cr2
,
2496 .write_cr2
= xen_write_cr2
,
2498 .read_cr3
= xen_read_cr3
,
2499 .write_cr3
= xen_write_cr3_init
,
2501 .flush_tlb_user
= xen_flush_tlb
,
2502 .flush_tlb_kernel
= xen_flush_tlb
,
2503 .flush_tlb_single
= xen_flush_tlb_single
,
2504 .flush_tlb_others
= xen_flush_tlb_others
,
2506 .pte_update
= paravirt_nop
,
2508 .pgd_alloc
= xen_pgd_alloc
,
2509 .pgd_free
= xen_pgd_free
,
2511 .alloc_pte
= xen_alloc_pte_init
,
2512 .release_pte
= xen_release_pte_init
,
2513 .alloc_pmd
= xen_alloc_pmd_init
,
2514 .release_pmd
= xen_release_pmd_init
,
2516 .set_pte
= xen_set_pte_init
,
2517 .set_pte_at
= xen_set_pte_at
,
2518 .set_pmd
= xen_set_pmd_hyper
,
2520 .ptep_modify_prot_start
= __ptep_modify_prot_start
,
2521 .ptep_modify_prot_commit
= __ptep_modify_prot_commit
,
2523 .pte_val
= PV_CALLEE_SAVE(xen_pte_val
),
2524 .pgd_val
= PV_CALLEE_SAVE(xen_pgd_val
),
2526 .make_pte
= PV_CALLEE_SAVE(xen_make_pte_init
),
2527 .make_pgd
= PV_CALLEE_SAVE(xen_make_pgd
),
2529 #ifdef CONFIG_X86_PAE
2530 .set_pte_atomic
= xen_set_pte_atomic
,
2531 .pte_clear
= xen_pte_clear
,
2532 .pmd_clear
= xen_pmd_clear
,
2533 #endif /* CONFIG_X86_PAE */
2534 .set_pud
= xen_set_pud_hyper
,
2536 .make_pmd
= PV_CALLEE_SAVE(xen_make_pmd
),
2537 .pmd_val
= PV_CALLEE_SAVE(xen_pmd_val
),
2539 #if CONFIG_PGTABLE_LEVELS >= 4
2540 .pud_val
= PV_CALLEE_SAVE(xen_pud_val
),
2541 .make_pud
= PV_CALLEE_SAVE(xen_make_pud
),
2542 .set_p4d
= xen_set_p4d_hyper
,
2544 .alloc_pud
= xen_alloc_pmd_init
,
2545 .release_pud
= xen_release_pmd_init
,
2546 #endif /* CONFIG_PGTABLE_LEVELS == 4 */
2548 .activate_mm
= xen_activate_mm
,
2549 .dup_mmap
= xen_dup_mmap
,
2550 .exit_mmap
= xen_exit_mmap
,
2553 .enter
= paravirt_enter_lazy_mmu
,
2554 .leave
= xen_leave_lazy_mmu
,
2555 .flush
= paravirt_flush_lazy_mmu
,
2558 .set_fixmap
= xen_set_fixmap
,
2561 void __init
xen_init_mmu_ops(void)
2563 x86_init
.paging
.pagetable_init
= xen_pagetable_init
;
2565 if (xen_feature(XENFEAT_auto_translated_physmap
))
2568 pv_mmu_ops
= xen_mmu_ops
;
2570 memset(dummy_mapping
, 0xff, PAGE_SIZE
);
2573 /* Protected by xen_reservation_lock. */
2574 #define MAX_CONTIG_ORDER 9 /* 2MB */
2575 static unsigned long discontig_frames
[1<<MAX_CONTIG_ORDER
];
2577 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2578 static void xen_zap_pfn_range(unsigned long vaddr
, unsigned int order
,
2579 unsigned long *in_frames
,
2580 unsigned long *out_frames
)
2583 struct multicall_space mcs
;
2586 for (i
= 0; i
< (1UL<<order
); i
++, vaddr
+= PAGE_SIZE
) {
2587 mcs
= __xen_mc_entry(0);
2590 in_frames
[i
] = virt_to_mfn(vaddr
);
2592 MULTI_update_va_mapping(mcs
.mc
, vaddr
, VOID_PTE
, 0);
2593 __set_phys_to_machine(virt_to_pfn(vaddr
), INVALID_P2M_ENTRY
);
2596 out_frames
[i
] = virt_to_pfn(vaddr
);
2602 * Update the pfn-to-mfn mappings for a virtual address range, either to
2603 * point to an array of mfns, or contiguously from a single starting
2606 static void xen_remap_exchanged_ptes(unsigned long vaddr
, int order
,
2607 unsigned long *mfns
,
2608 unsigned long first_mfn
)
2615 limit
= 1u << order
;
2616 for (i
= 0; i
< limit
; i
++, vaddr
+= PAGE_SIZE
) {
2617 struct multicall_space mcs
;
2620 mcs
= __xen_mc_entry(0);
2624 mfn
= first_mfn
+ i
;
2626 if (i
< (limit
- 1))
2630 flags
= UVMF_INVLPG
| UVMF_ALL
;
2632 flags
= UVMF_TLB_FLUSH
| UVMF_ALL
;
2635 MULTI_update_va_mapping(mcs
.mc
, vaddr
,
2636 mfn_pte(mfn
, PAGE_KERNEL
), flags
);
2638 set_phys_to_machine(virt_to_pfn(vaddr
), mfn
);
2645 * Perform the hypercall to exchange a region of our pfns to point to
2646 * memory with the required contiguous alignment. Takes the pfns as
2647 * input, and populates mfns as output.
2649 * Returns a success code indicating whether the hypervisor was able to
2650 * satisfy the request or not.
2652 static int xen_exchange_memory(unsigned long extents_in
, unsigned int order_in
,
2653 unsigned long *pfns_in
,
2654 unsigned long extents_out
,
2655 unsigned int order_out
,
2656 unsigned long *mfns_out
,
2657 unsigned int address_bits
)
2662 struct xen_memory_exchange exchange
= {
2664 .nr_extents
= extents_in
,
2665 .extent_order
= order_in
,
2666 .extent_start
= pfns_in
,
2670 .nr_extents
= extents_out
,
2671 .extent_order
= order_out
,
2672 .extent_start
= mfns_out
,
2673 .address_bits
= address_bits
,
2678 BUG_ON(extents_in
<< order_in
!= extents_out
<< order_out
);
2680 rc
= HYPERVISOR_memory_op(XENMEM_exchange
, &exchange
);
2681 success
= (exchange
.nr_exchanged
== extents_in
);
2683 BUG_ON(!success
&& ((exchange
.nr_exchanged
!= 0) || (rc
== 0)));
2684 BUG_ON(success
&& (rc
!= 0));
2689 int xen_create_contiguous_region(phys_addr_t pstart
, unsigned int order
,
2690 unsigned int address_bits
,
2691 dma_addr_t
*dma_handle
)
2693 unsigned long *in_frames
= discontig_frames
, out_frame
;
2694 unsigned long flags
;
2696 unsigned long vstart
= (unsigned long)phys_to_virt(pstart
);
2699 * Currently an auto-translated guest will not perform I/O, nor will
2700 * it require PAE page directories below 4GB. Therefore any calls to
2701 * this function are redundant and can be ignored.
2704 if (xen_feature(XENFEAT_auto_translated_physmap
))
2707 if (unlikely(order
> MAX_CONTIG_ORDER
))
2710 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2712 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2714 /* 1. Zap current PTEs, remembering MFNs. */
2715 xen_zap_pfn_range(vstart
, order
, in_frames
, NULL
);
2717 /* 2. Get a new contiguous memory extent. */
2718 out_frame
= virt_to_pfn(vstart
);
2719 success
= xen_exchange_memory(1UL << order
, 0, in_frames
,
2720 1, order
, &out_frame
,
2723 /* 3. Map the new extent in place of old pages. */
2725 xen_remap_exchanged_ptes(vstart
, order
, NULL
, out_frame
);
2727 xen_remap_exchanged_ptes(vstart
, order
, in_frames
, 0);
2729 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2731 *dma_handle
= virt_to_machine(vstart
).maddr
;
2732 return success
? 0 : -ENOMEM
;
2734 EXPORT_SYMBOL_GPL(xen_create_contiguous_region
);
2736 void xen_destroy_contiguous_region(phys_addr_t pstart
, unsigned int order
)
2738 unsigned long *out_frames
= discontig_frames
, in_frame
;
2739 unsigned long flags
;
2741 unsigned long vstart
;
2743 if (xen_feature(XENFEAT_auto_translated_physmap
))
2746 if (unlikely(order
> MAX_CONTIG_ORDER
))
2749 vstart
= (unsigned long)phys_to_virt(pstart
);
2750 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2752 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2754 /* 1. Find start MFN of contiguous extent. */
2755 in_frame
= virt_to_mfn(vstart
);
2757 /* 2. Zap current PTEs. */
2758 xen_zap_pfn_range(vstart
, order
, NULL
, out_frames
);
2760 /* 3. Do the exchange for non-contiguous MFNs. */
2761 success
= xen_exchange_memory(1, order
, &in_frame
, 1UL << order
,
2764 /* 4. Map new pages in place of old pages. */
2766 xen_remap_exchanged_ptes(vstart
, order
, out_frames
, 0);
2768 xen_remap_exchanged_ptes(vstart
, order
, NULL
, in_frame
);
2770 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2772 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region
);
2774 #ifdef CONFIG_XEN_PVHVM
2775 #ifdef CONFIG_PROC_VMCORE
2777 * This function is used in two contexts:
2778 * - the kdump kernel has to check whether a pfn of the crashed kernel
2779 * was a ballooned page. vmcore is using this function to decide
2780 * whether to access a pfn of the crashed kernel.
2781 * - the kexec kernel has to check whether a pfn was ballooned by the
2782 * previous kernel. If the pfn is ballooned, handle it properly.
2783 * Returns 0 if the pfn is not backed by a RAM page, the caller may
2784 * handle the pfn special in this case.
2786 static int xen_oldmem_pfn_is_ram(unsigned long pfn
)
2788 struct xen_hvm_get_mem_type a
= {
2789 .domid
= DOMID_SELF
,
2794 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type
, &a
))
2797 switch (a
.mem_type
) {
2798 case HVMMEM_mmio_dm
:
2812 static void xen_hvm_exit_mmap(struct mm_struct
*mm
)
2814 struct xen_hvm_pagetable_dying a
;
2817 a
.domid
= DOMID_SELF
;
2818 a
.gpa
= __pa(mm
->pgd
);
2819 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2820 WARN_ON_ONCE(rc
< 0);
2823 static int is_pagetable_dying_supported(void)
2825 struct xen_hvm_pagetable_dying a
;
2828 a
.domid
= DOMID_SELF
;
2830 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2832 printk(KERN_DEBUG
"HVMOP_pagetable_dying not supported\n");
2838 void __init
xen_hvm_init_mmu_ops(void)
2840 if (is_pagetable_dying_supported())
2841 pv_mmu_ops
.exit_mmap
= xen_hvm_exit_mmap
;
2842 #ifdef CONFIG_PROC_VMCORE
2843 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram
);
2848 #define REMAP_BATCH_SIZE 16
2854 struct mmu_update
*mmu_update
;
2857 static int remap_area_mfn_pte_fn(pte_t
*ptep
, pgtable_t token
,
2858 unsigned long addr
, void *data
)
2860 struct remap_data
*rmd
= data
;
2861 pte_t pte
= pte_mkspecial(mfn_pte(*rmd
->mfn
, rmd
->prot
));
2863 /* If we have a contiguous range, just update the mfn itself,
2864 else update pointer to be "next mfn". */
2865 if (rmd
->contiguous
)
2870 rmd
->mmu_update
->ptr
= virt_to_machine(ptep
).maddr
;
2871 rmd
->mmu_update
->val
= pte_val_ma(pte
);
2877 static int do_remap_gfn(struct vm_area_struct
*vma
,
2879 xen_pfn_t
*gfn
, int nr
,
2880 int *err_ptr
, pgprot_t prot
,
2882 struct page
**pages
)
2885 struct remap_data rmd
;
2886 struct mmu_update mmu_update
[REMAP_BATCH_SIZE
];
2887 unsigned long range
;
2890 BUG_ON(!((vma
->vm_flags
& (VM_PFNMAP
| VM_IO
)) == (VM_PFNMAP
| VM_IO
)));
2894 /* We use the err_ptr to indicate if there we are doing a contiguous
2895 * mapping or a discontigious mapping. */
2896 rmd
.contiguous
= !err_ptr
;
2901 int batch
= min(REMAP_BATCH_SIZE
, nr
);
2902 int batch_left
= batch
;
2903 range
= (unsigned long)batch
<< PAGE_SHIFT
;
2905 rmd
.mmu_update
= mmu_update
;
2906 err
= apply_to_page_range(vma
->vm_mm
, addr
, range
,
2907 remap_area_mfn_pte_fn
, &rmd
);
2911 /* We record the error for each page that gives an error, but
2912 * continue mapping until the whole set is done */
2916 err
= HYPERVISOR_mmu_update(&mmu_update
[index
],
2917 batch_left
, &done
, domid
);
2920 * @err_ptr may be the same buffer as @gfn, so
2921 * only clear it after each chunk of @gfn is
2925 for (i
= index
; i
< index
+ done
; i
++)
2932 done
++; /* Skip failed frame. */
2937 } while (batch_left
);
2947 xen_flush_tlb_all();
2949 return err
< 0 ? err
: mapped
;
2952 int xen_remap_domain_gfn_range(struct vm_area_struct
*vma
,
2954 xen_pfn_t gfn
, int nr
,
2955 pgprot_t prot
, unsigned domid
,
2956 struct page
**pages
)
2958 return do_remap_gfn(vma
, addr
, &gfn
, nr
, NULL
, prot
, domid
, pages
);
2960 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range
);
2962 int xen_remap_domain_gfn_array(struct vm_area_struct
*vma
,
2964 xen_pfn_t
*gfn
, int nr
,
2965 int *err_ptr
, pgprot_t prot
,
2966 unsigned domid
, struct page
**pages
)
2968 /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
2969 * and the consequences later is quite hard to detect what the actual
2970 * cause of "wrong memory was mapped in".
2972 BUG_ON(err_ptr
== NULL
);
2973 return do_remap_gfn(vma
, addr
, gfn
, nr
, err_ptr
, prot
, domid
, pages
);
2975 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array
);
2978 /* Returns: 0 success */
2979 int xen_unmap_domain_gfn_range(struct vm_area_struct
*vma
,
2980 int numpgs
, struct page
**pages
)
2982 if (!pages
|| !xen_feature(XENFEAT_auto_translated_physmap
))
2987 EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range
);