]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - arch/powerpc/mm/pgtable_64.c
2 * This file contains ioremap and related functions for 64-bit machines.
4 * Derived from arch/ppc64/mm/init.c
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
8 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
9 * Copyright (C) 1996 Paul Mackerras
11 * Derived from "arch/i386/mm/init.c"
12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
14 * Dave Engebretsen <engebret@us.ibm.com>
15 * Rework for PPC64 port.
17 * This program is free software; you can redistribute it and/or
18 * modify it under the terms of the GNU General Public License
19 * as published by the Free Software Foundation; either version
20 * 2 of the License, or (at your option) any later version.
24 #include <linux/signal.h>
25 #include <linux/sched.h>
26 #include <linux/kernel.h>
27 #include <linux/errno.h>
28 #include <linux/string.h>
29 #include <linux/export.h>
30 #include <linux/types.h>
31 #include <linux/mman.h>
33 #include <linux/swap.h>
34 #include <linux/stddef.h>
35 #include <linux/vmalloc.h>
36 #include <linux/memblock.h>
37 #include <linux/slab.h>
38 #include <linux/hugetlb.h>
40 #include <asm/pgalloc.h>
44 #include <asm/mmu_context.h>
45 #include <asm/pgtable.h>
48 #include <asm/machdep.h>
50 #include <asm/processor.h>
51 #include <asm/cputable.h>
52 #include <asm/sections.h>
53 #include <asm/firmware.h>
58 #define CREATE_TRACE_POINTS
59 #include <trace/events/thp.h>
61 #ifdef CONFIG_PPC_STD_MMU_64
62 #if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
63 #error TASK_SIZE_USER64 exceeds user VSID range
67 #ifdef CONFIG_PPC_BOOK3S_64
69 * partition table and process table for ISA 3.0
71 struct prtb_entry
*process_tb
;
72 struct patb_entry
*partition_tb
;
76 unsigned long __pte_index_size
;
77 EXPORT_SYMBOL(__pte_index_size
);
78 unsigned long __pmd_index_size
;
79 EXPORT_SYMBOL(__pmd_index_size
);
80 unsigned long __pud_index_size
;
81 EXPORT_SYMBOL(__pud_index_size
);
82 unsigned long __pgd_index_size
;
83 EXPORT_SYMBOL(__pgd_index_size
);
84 unsigned long __pmd_cache_index
;
85 EXPORT_SYMBOL(__pmd_cache_index
);
86 unsigned long __pte_table_size
;
87 EXPORT_SYMBOL(__pte_table_size
);
88 unsigned long __pmd_table_size
;
89 EXPORT_SYMBOL(__pmd_table_size
);
90 unsigned long __pud_table_size
;
91 EXPORT_SYMBOL(__pud_table_size
);
92 unsigned long __pgd_table_size
;
93 EXPORT_SYMBOL(__pgd_table_size
);
96 unsigned long ioremap_bot
= IOREMAP_BASE
;
99 * __ioremap_at - Low level function to establish the page tables
102 void __iomem
* __ioremap_at(phys_addr_t pa
, void *ea
, unsigned long size
,
107 /* Make sure we have the base flags */
108 if ((flags
& _PAGE_PRESENT
) == 0)
109 flags
|= pgprot_val(PAGE_KERNEL
);
111 /* We don't support the 4K PFN hack with ioremap */
112 if (flags
& H_PAGE_4K_PFN
)
115 WARN_ON(pa
& ~PAGE_MASK
);
116 WARN_ON(((unsigned long)ea
) & ~PAGE_MASK
);
117 WARN_ON(size
& ~PAGE_MASK
);
119 for (i
= 0; i
< size
; i
+= PAGE_SIZE
)
120 if (map_kernel_page((unsigned long)ea
+i
, pa
+i
, flags
))
123 return (void __iomem
*)ea
;
127 * __iounmap_from - Low level function to tear down the page tables
128 * for an IO mapping. This is used for mappings that
129 * are manipulated manually, like partial unmapping of
130 * PCI IOs or ISA space.
132 void __iounmap_at(void *ea
, unsigned long size
)
134 WARN_ON(((unsigned long)ea
) & ~PAGE_MASK
);
135 WARN_ON(size
& ~PAGE_MASK
);
137 unmap_kernel_range((unsigned long)ea
, size
);
140 void __iomem
* __ioremap_caller(phys_addr_t addr
, unsigned long size
,
141 unsigned long flags
, void *caller
)
143 phys_addr_t paligned
;
147 * Choose an address to map it to.
148 * Once the imalloc system is running, we use it.
149 * Before that, we map using addresses going
150 * up from ioremap_bot. imalloc will use
151 * the addresses from ioremap_bot through
155 paligned
= addr
& PAGE_MASK
;
156 size
= PAGE_ALIGN(addr
+ size
) - paligned
;
158 if ((size
== 0) || (paligned
== 0))
161 if (slab_is_available()) {
162 struct vm_struct
*area
;
164 area
= __get_vm_area_caller(size
, VM_IOREMAP
,
165 ioremap_bot
, IOREMAP_END
,
170 area
->phys_addr
= paligned
;
171 ret
= __ioremap_at(paligned
, area
->addr
, size
, flags
);
175 ret
= __ioremap_at(paligned
, (void *)ioremap_bot
, size
, flags
);
181 ret
+= addr
& ~PAGE_MASK
;
185 void __iomem
* __ioremap(phys_addr_t addr
, unsigned long size
,
188 return __ioremap_caller(addr
, size
, flags
, __builtin_return_address(0));
191 void __iomem
* ioremap(phys_addr_t addr
, unsigned long size
)
193 unsigned long flags
= pgprot_val(pgprot_noncached(__pgprot(0)));
194 void *caller
= __builtin_return_address(0);
197 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
198 return __ioremap_caller(addr
, size
, flags
, caller
);
201 void __iomem
* ioremap_wc(phys_addr_t addr
, unsigned long size
)
203 unsigned long flags
= pgprot_val(pgprot_noncached_wc(__pgprot(0)));
204 void *caller
= __builtin_return_address(0);
207 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
208 return __ioremap_caller(addr
, size
, flags
, caller
);
211 void __iomem
* ioremap_prot(phys_addr_t addr
, unsigned long size
,
214 void *caller
= __builtin_return_address(0);
216 /* writeable implies dirty for kernel addresses */
217 if (flags
& _PAGE_WRITE
)
218 flags
|= _PAGE_DIRTY
;
220 /* we don't want to let _PAGE_EXEC leak out */
221 flags
&= ~_PAGE_EXEC
;
223 * Force kernel mapping.
225 #if defined(CONFIG_PPC_BOOK3S_64)
226 flags
|= _PAGE_PRIVILEGED
;
228 flags
&= ~_PAGE_USER
;
233 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
234 * which means that we just cleared supervisor access... oops ;-) This
237 flags
|= _PAGE_BAP_SR
;
241 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
242 return __ioremap_caller(addr
, size
, flags
, caller
);
247 * Unmap an IO region and remove it from imalloc'd list.
248 * Access to IO memory should be serialized by driver.
250 void __iounmap(volatile void __iomem
*token
)
254 if (!slab_is_available())
257 addr
= (void *) ((unsigned long __force
)
258 PCI_FIX_ADDR(token
) & PAGE_MASK
);
259 if ((unsigned long)addr
< ioremap_bot
) {
260 printk(KERN_WARNING
"Attempt to iounmap early bolted mapping"
267 void iounmap(volatile void __iomem
*token
)
270 ppc_md
.iounmap(token
);
275 EXPORT_SYMBOL(ioremap
);
276 EXPORT_SYMBOL(ioremap_wc
);
277 EXPORT_SYMBOL(ioremap_prot
);
278 EXPORT_SYMBOL(__ioremap
);
279 EXPORT_SYMBOL(__ioremap_at
);
280 EXPORT_SYMBOL(iounmap
);
281 EXPORT_SYMBOL(__iounmap
);
282 EXPORT_SYMBOL(__iounmap_at
);
284 #ifndef __PAGETABLE_PUD_FOLDED
285 /* 4 level page table */
286 struct page
*pgd_page(pgd_t pgd
)
289 return pte_page(pgd_pte(pgd
));
290 return virt_to_page(pgd_page_vaddr(pgd
));
294 struct page
*pud_page(pud_t pud
)
297 return pte_page(pud_pte(pud
));
298 return virt_to_page(pud_page_vaddr(pud
));
302 * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
303 * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
305 struct page
*pmd_page(pmd_t pmd
)
307 if (pmd_trans_huge(pmd
) || pmd_huge(pmd
))
308 return pte_page(pmd_pte(pmd
));
309 return virt_to_page(pmd_page_vaddr(pmd
));
312 #ifdef CONFIG_PPC_64K_PAGES
313 static pte_t
*get_from_cache(struct mm_struct
*mm
)
315 void *pte_frag
, *ret
;
317 spin_lock(&mm
->page_table_lock
);
318 ret
= mm
->context
.pte_frag
;
320 pte_frag
= ret
+ PTE_FRAG_SIZE
;
322 * If we have taken up all the fragments mark PTE page NULL
324 if (((unsigned long)pte_frag
& ~PAGE_MASK
) == 0)
326 mm
->context
.pte_frag
= pte_frag
;
328 spin_unlock(&mm
->page_table_lock
);
332 static pte_t
*__alloc_for_cache(struct mm_struct
*mm
, int kernel
)
335 struct page
*page
= alloc_page(GFP_KERNEL
| __GFP_NOTRACK
|
336 __GFP_REPEAT
| __GFP_ZERO
);
339 if (!kernel
&& !pgtable_page_ctor(page
)) {
344 ret
= page_address(page
);
345 spin_lock(&mm
->page_table_lock
);
347 * If we find pgtable_page set, we return
348 * the allocated page with single fragement
351 if (likely(!mm
->context
.pte_frag
)) {
352 set_page_count(page
, PTE_FRAG_NR
);
353 mm
->context
.pte_frag
= ret
+ PTE_FRAG_SIZE
;
355 spin_unlock(&mm
->page_table_lock
);
360 pte_t
*pte_fragment_alloc(struct mm_struct
*mm
, unsigned long vmaddr
, int kernel
)
364 pte
= get_from_cache(mm
);
368 return __alloc_for_cache(mm
, kernel
);
371 void pte_fragment_free(unsigned long *table
, int kernel
)
373 struct page
*page
= virt_to_page(table
);
374 if (put_page_testzero(page
)) {
376 pgtable_page_dtor(page
);
377 free_hot_cold_page(page
, 0);
382 void pgtable_free_tlb(struct mmu_gather
*tlb
, void *table
, int shift
)
384 unsigned long pgf
= (unsigned long)table
;
386 BUG_ON(shift
> MAX_PGTABLE_INDEX_SIZE
);
388 tlb_remove_table(tlb
, (void *)pgf
);
391 void __tlb_remove_table(void *_table
)
393 void *table
= (void *)((unsigned long)_table
& ~MAX_PGTABLE_INDEX_SIZE
);
394 unsigned shift
= (unsigned long)_table
& MAX_PGTABLE_INDEX_SIZE
;
397 /* PTE page needs special handling */
398 pte_fragment_free(table
, 0);
400 BUG_ON(shift
> MAX_PGTABLE_INDEX_SIZE
);
401 kmem_cache_free(PGT_CACHE(shift
), table
);
405 void pgtable_free_tlb(struct mmu_gather
*tlb
, void *table
, int shift
)
408 /* PTE page needs special handling */
409 pte_fragment_free(table
, 0);
411 BUG_ON(shift
> MAX_PGTABLE_INDEX_SIZE
);
412 kmem_cache_free(PGT_CACHE(shift
), table
);
416 #endif /* CONFIG_PPC_64K_PAGES */
418 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
421 * This is called when relaxing access to a hugepage. It's also called in the page
422 * fault path when we don't hit any of the major fault cases, ie, a minor
423 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
424 * handled those two for us, we additionally deal with missing execute
425 * permission here on some processors
427 int pmdp_set_access_flags(struct vm_area_struct
*vma
, unsigned long address
,
428 pmd_t
*pmdp
, pmd_t entry
, int dirty
)
431 #ifdef CONFIG_DEBUG_VM
432 WARN_ON(!pmd_trans_huge(*pmdp
));
433 assert_spin_locked(&vma
->vm_mm
->page_table_lock
);
435 changed
= !pmd_same(*(pmdp
), entry
);
437 __ptep_set_access_flags(pmdp_ptep(pmdp
), pmd_pte(entry
));
439 * Since we are not supporting SW TLB systems, we don't
440 * have any thing similar to flush_tlb_page_nohash()
446 unsigned long pmd_hugepage_update(struct mm_struct
*mm
, unsigned long addr
,
447 pmd_t
*pmdp
, unsigned long clr
,
454 #ifdef CONFIG_DEBUG_VM
455 WARN_ON(!pmd_trans_huge(*pmdp
));
456 assert_spin_locked(&mm
->page_table_lock
);
459 __asm__
__volatile__(
467 : "=&r" (old_be
), "=&r" (tmp
), "=m" (*pmdp
)
468 : "r" (pmdp
), "r" (cpu_to_be64(clr
)), "m" (*pmdp
),
469 "r" (cpu_to_be64(H_PAGE_BUSY
)), "r" (cpu_to_be64(set
))
472 old
= be64_to_cpu(old_be
);
474 trace_hugepage_update(addr
, old
, clr
, set
);
475 if (old
& H_PAGE_HASHPTE
)
476 hpte_do_hugepage_flush(mm
, addr
, pmdp
, old
);
480 pmd_t
pmdp_collapse_flush(struct vm_area_struct
*vma
, unsigned long address
,
485 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
486 VM_BUG_ON(pmd_trans_huge(*pmdp
));
491 * Wait for all pending hash_page to finish. This is needed
492 * in case of subpage collapse. When we collapse normal pages
493 * to hugepage, we first clear the pmd, then invalidate all
494 * the PTE entries. The assumption here is that any low level
495 * page fault will see a none pmd and take the slow path that
496 * will wait on mmap_sem. But we could very well be in a
497 * hash_page with local ptep pointer value. Such a hash page
498 * can result in adding new HPTE entries for normal subpages.
499 * That means we could be modifying the page content as we
500 * copy them to a huge page. So wait for parallel hash_page
501 * to finish before invalidating HPTE entries. We can do this
502 * by sending an IPI to all the cpus and executing a dummy
505 kick_all_cpus_sync();
507 * Now invalidate the hpte entries in the range
508 * covered by pmd. This make sure we take a
509 * fault and will find the pmd as none, which will
510 * result in a major fault which takes mmap_sem and
511 * hence wait for collapse to complete. Without this
512 * the __collapse_huge_page_copy can result in copying
515 flush_tlb_pmd_range(vma
->vm_mm
, &pmd
, address
);
520 * We currently remove entries from the hashtable regardless of whether
521 * the entry was young or dirty.
523 * We should be more intelligent about this but for the moment we override
524 * these functions and force a tlb flush unconditionally
526 int pmdp_test_and_clear_young(struct vm_area_struct
*vma
,
527 unsigned long address
, pmd_t
*pmdp
)
529 return __pmdp_test_and_clear_young(vma
->vm_mm
, address
, pmdp
);
533 * We want to put the pgtable in pmd and use pgtable for tracking
534 * the base page size hptes
536 void pgtable_trans_huge_deposit(struct mm_struct
*mm
, pmd_t
*pmdp
,
539 pgtable_t
*pgtable_slot
;
540 assert_spin_locked(&mm
->page_table_lock
);
542 * we store the pgtable in the second half of PMD
544 pgtable_slot
= (pgtable_t
*)pmdp
+ PTRS_PER_PMD
;
545 *pgtable_slot
= pgtable
;
547 * expose the deposited pgtable to other cpus.
548 * before we set the hugepage PTE at pmd level
549 * hash fault code looks at the deposted pgtable
550 * to store hash index values.
555 pgtable_t
pgtable_trans_huge_withdraw(struct mm_struct
*mm
, pmd_t
*pmdp
)
558 pgtable_t
*pgtable_slot
;
560 assert_spin_locked(&mm
->page_table_lock
);
561 pgtable_slot
= (pgtable_t
*)pmdp
+ PTRS_PER_PMD
;
562 pgtable
= *pgtable_slot
;
564 * Once we withdraw, mark the entry NULL.
566 *pgtable_slot
= NULL
;
568 * We store HPTE information in the deposited PTE fragment.
569 * zero out the content on withdraw.
571 memset(pgtable
, 0, PTE_FRAG_SIZE
);
575 void pmdp_huge_split_prepare(struct vm_area_struct
*vma
,
576 unsigned long address
, pmd_t
*pmdp
)
578 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
579 VM_BUG_ON(REGION_ID(address
) != USER_REGION_ID
);
582 * We can't mark the pmd none here, because that will cause a race
583 * against exit_mmap. We need to continue mark pmd TRANS HUGE, while
584 * we spilt, but at the same time we wan't rest of the ppc64 code
585 * not to insert hash pte on this, because we will be modifying
586 * the deposited pgtable in the caller of this function. Hence
587 * clear the _PAGE_USER so that we move the fault handling to
588 * higher level function and that will serialize against ptl.
589 * We need to flush existing hash pte entries here even though,
590 * the translation is still valid, because we will withdraw
591 * pgtable_t after this.
593 pmd_hugepage_update(vma
->vm_mm
, address
, pmdp
, 0, _PAGE_PRIVILEGED
);
598 * set a new huge pmd. We should not be called for updating
599 * an existing pmd entry. That should go via pmd_hugepage_update.
601 void set_pmd_at(struct mm_struct
*mm
, unsigned long addr
,
602 pmd_t
*pmdp
, pmd_t pmd
)
604 #ifdef CONFIG_DEBUG_VM
605 WARN_ON(pte_present(pmd_pte(*pmdp
)) && !pte_protnone(pmd_pte(*pmdp
)));
606 assert_spin_locked(&mm
->page_table_lock
);
607 WARN_ON(!pmd_trans_huge(pmd
));
609 trace_hugepage_set_pmd(addr
, pmd_val(pmd
));
610 return set_pte_at(mm
, addr
, pmdp_ptep(pmdp
), pmd_pte(pmd
));
614 * We use this to invalidate a pmdp entry before switching from a
615 * hugepte to regular pmd entry.
617 void pmdp_invalidate(struct vm_area_struct
*vma
, unsigned long address
,
620 pmd_hugepage_update(vma
->vm_mm
, address
, pmdp
, _PAGE_PRESENT
, 0);
623 * This ensures that generic code that rely on IRQ disabling
624 * to prevent a parallel THP split work as expected.
626 kick_all_cpus_sync();
630 * A linux hugepage PMD was changed and the corresponding hash table entries
631 * neesd to be flushed.
633 void hpte_do_hugepage_flush(struct mm_struct
*mm
, unsigned long addr
,
634 pmd_t
*pmdp
, unsigned long old_pmd
)
639 unsigned long flags
= 0;
640 const struct cpumask
*tmp
;
642 /* get the base page size,vsid and segment size */
643 #ifdef CONFIG_DEBUG_VM
644 psize
= get_slice_psize(mm
, addr
);
645 BUG_ON(psize
== MMU_PAGE_16M
);
647 if (old_pmd
& H_PAGE_COMBO
)
650 psize
= MMU_PAGE_64K
;
652 if (!is_kernel_addr(addr
)) {
653 ssize
= user_segment_size(addr
);
654 vsid
= get_vsid(mm
->context
.id
, addr
, ssize
);
657 vsid
= get_kernel_vsid(addr
, mmu_kernel_ssize
);
658 ssize
= mmu_kernel_ssize
;
661 tmp
= cpumask_of(smp_processor_id());
662 if (cpumask_equal(mm_cpumask(mm
), tmp
))
663 flags
|= HPTE_LOCAL_UPDATE
;
665 return flush_hash_hugepage(vsid
, addr
, pmdp
, psize
, ssize
, flags
);
668 static pmd_t
pmd_set_protbits(pmd_t pmd
, pgprot_t pgprot
)
670 return __pmd(pmd_val(pmd
) | pgprot_val(pgprot
));
673 pmd_t
pfn_pmd(unsigned long pfn
, pgprot_t pgprot
)
677 pmdv
= (pfn
<< PAGE_SHIFT
) & PTE_RPN_MASK
;
678 return pmd_set_protbits(__pmd(pmdv
), pgprot
);
681 pmd_t
mk_pmd(struct page
*page
, pgprot_t pgprot
)
683 return pfn_pmd(page_to_pfn(page
), pgprot
);
686 pmd_t
pmd_modify(pmd_t pmd
, pgprot_t newprot
)
691 pmdv
&= _HPAGE_CHG_MASK
;
692 return pmd_set_protbits(__pmd(pmdv
), newprot
);
696 * This is called at the end of handling a user page fault, when the
697 * fault has been handled by updating a HUGE PMD entry in the linux page tables.
698 * We use it to preload an HPTE into the hash table corresponding to
699 * the updated linux HUGE PMD entry.
701 void update_mmu_cache_pmd(struct vm_area_struct
*vma
, unsigned long addr
,
707 pmd_t
pmdp_huge_get_and_clear(struct mm_struct
*mm
,
708 unsigned long addr
, pmd_t
*pmdp
)
713 pgtable_t
*pgtable_slot
;
715 old
= pmd_hugepage_update(mm
, addr
, pmdp
, ~0UL, 0);
716 old_pmd
= __pmd(old
);
718 * We have pmd == none and we are holding page_table_lock.
719 * So we can safely go and clear the pgtable hash
722 pgtable_slot
= (pgtable_t
*)pmdp
+ PTRS_PER_PMD
;
723 pgtable
= *pgtable_slot
;
725 * Let's zero out old valid and hash index details
726 * hash fault look at them.
728 memset(pgtable
, 0, PTE_FRAG_SIZE
);
730 * Serialize against find_linux_pte_or_hugepte which does lock-less
731 * lookup in page tables with local interrupts disabled. For huge pages
732 * it casts pmd_t to pte_t. Since format of pte_t is different from
733 * pmd_t we want to prevent transit from pmd pointing to page table
734 * to pmd pointing to huge page (and back) while interrupts are disabled.
735 * We clear pmd to possibly replace it with page table pointer in
736 * different code paths. So make sure we wait for the parallel
737 * find_linux_pte_or_hugepage to finish.
739 kick_all_cpus_sync();
743 int has_transparent_hugepage(void)
746 if (!mmu_has_feature(MMU_FTR_16M_PAGE
))
749 * We support THP only if PMD_SIZE is 16MB.
751 if (mmu_psize_defs
[MMU_PAGE_16M
].shift
!= PMD_SHIFT
)
754 * We need to make sure that we support 16MB hugepage in a segement
755 * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
759 * If we have 64K HPTE, we will be using that by default
761 if (mmu_psize_defs
[MMU_PAGE_64K
].shift
&&
762 (mmu_psize_defs
[MMU_PAGE_64K
].penc
[MMU_PAGE_16M
] == -1))
765 * Ok we only have 4K HPTE
767 if (mmu_psize_defs
[MMU_PAGE_4K
].penc
[MMU_PAGE_16M
] == -1)
772 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */