]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - arch/powerpc/mm/hugetlbpage.c
2 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
10 #include <linux/init.h>
13 #include <linux/hugetlb.h>
14 #include <linux/pagemap.h>
15 #include <linux/slab.h>
16 #include <linux/err.h>
17 #include <linux/sysctl.h>
19 #include <asm/pgalloc.h>
21 #include <asm/tlbflush.h>
22 #include <asm/mmu_context.h>
23 #include <asm/machdep.h>
24 #include <asm/cputable.h>
27 #define PAGE_SHIFT_64K 16
28 #define PAGE_SHIFT_16M 24
29 #define PAGE_SHIFT_16G 34
31 #define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
32 #define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
33 #define MAX_NUMBER_GPAGES 1024
35 /* Tracks the 16G pages after the device tree is scanned and before the
36 * huge_boot_pages list is ready. */
37 static unsigned long gpage_freearray
[MAX_NUMBER_GPAGES
];
38 static unsigned nr_gpages
;
40 /* Array of valid huge page sizes - non-zero value(hugepte_shift) is
41 * stored for the huge page sizes that are valid.
43 unsigned int mmu_huge_psizes
[MMU_PAGE_COUNT
] = { }; /* initialize all to 0 */
45 #define hugepte_shift mmu_huge_psizes
46 #define PTRS_PER_HUGEPTE(psize) (1 << hugepte_shift[psize])
47 #define HUGEPTE_TABLE_SIZE(psize) (sizeof(pte_t) << hugepte_shift[psize])
49 #define HUGEPD_SHIFT(psize) (mmu_psize_to_shift(psize) \
50 + hugepte_shift[psize])
51 #define HUGEPD_SIZE(psize) (1UL << HUGEPD_SHIFT(psize))
52 #define HUGEPD_MASK(psize) (~(HUGEPD_SIZE(psize)-1))
54 /* Subtract one from array size because we don't need a cache for 4K since
55 * is not a huge page size */
56 #define huge_pgtable_cache(psize) (pgtable_cache[HUGEPTE_CACHE_NUM \
58 #define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize])
60 static const char *huge_pgtable_cache_name
[MMU_PAGE_COUNT
] = {
61 "unused_4K", "hugepte_cache_64K", "unused_64K_AP",
62 "hugepte_cache_1M", "hugepte_cache_16M", "hugepte_cache_16G"
65 /* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad()
66 * will choke on pointers to hugepte tables, which is handy for
67 * catching screwups early. */
70 typedef struct { unsigned long pd
; } hugepd_t
;
72 #define hugepd_none(hpd) ((hpd).pd == 0)
74 static inline int shift_to_mmu_psize(unsigned int shift
)
77 #ifndef CONFIG_PPC_64K_PAGES
89 static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize
)
91 if (mmu_psize_defs
[mmu_psize
].shift
)
92 return mmu_psize_defs
[mmu_psize
].shift
;
96 static inline pte_t
*hugepd_page(hugepd_t hpd
)
98 BUG_ON(!(hpd
.pd
& HUGEPD_OK
));
99 return (pte_t
*)(hpd
.pd
& ~HUGEPD_OK
);
102 static inline pte_t
*hugepte_offset(hugepd_t
*hpdp
, unsigned long addr
,
103 struct hstate
*hstate
)
105 unsigned int shift
= huge_page_shift(hstate
);
106 int psize
= shift_to_mmu_psize(shift
);
107 unsigned long idx
= ((addr
>> shift
) & (PTRS_PER_HUGEPTE(psize
)-1));
108 pte_t
*dir
= hugepd_page(*hpdp
);
113 static int __hugepte_alloc(struct mm_struct
*mm
, hugepd_t
*hpdp
,
114 unsigned long address
, unsigned int psize
)
116 pte_t
*new = kmem_cache_alloc(huge_pgtable_cache(psize
),
117 GFP_KERNEL
|__GFP_REPEAT
);
122 spin_lock(&mm
->page_table_lock
);
123 if (!hugepd_none(*hpdp
))
124 kmem_cache_free(huge_pgtable_cache(psize
), new);
126 hpdp
->pd
= (unsigned long)new | HUGEPD_OK
;
127 spin_unlock(&mm
->page_table_lock
);
131 /* Base page size affects how we walk hugetlb page tables */
132 #ifdef CONFIG_PPC_64K_PAGES
133 #define hpmd_offset(pud, addr, h) pmd_offset(pud, addr)
134 #define hpmd_alloc(mm, pud, addr, h) pmd_alloc(mm, pud, addr)
137 pmd_t
*hpmd_offset(pud_t
*pud
, unsigned long addr
, struct hstate
*hstate
)
139 if (huge_page_shift(hstate
) == PAGE_SHIFT_64K
)
140 return pmd_offset(pud
, addr
);
142 return (pmd_t
*) pud
;
145 pmd_t
*hpmd_alloc(struct mm_struct
*mm
, pud_t
*pud
, unsigned long addr
,
146 struct hstate
*hstate
)
148 if (huge_page_shift(hstate
) == PAGE_SHIFT_64K
)
149 return pmd_alloc(mm
, pud
, addr
);
151 return (pmd_t
*) pud
;
155 /* Build list of addresses of gigantic pages. This function is used in early
156 * boot before the buddy or bootmem allocator is setup.
158 void add_gpage(unsigned long addr
, unsigned long page_size
,
159 unsigned long number_of_pages
)
163 while (number_of_pages
> 0) {
164 gpage_freearray
[nr_gpages
] = addr
;
171 /* Moves the gigantic page addresses from the temporary list to the
172 * huge_boot_pages list.
174 int alloc_bootmem_huge_page(struct hstate
*hstate
)
176 struct huge_bootmem_page
*m
;
179 m
= phys_to_virt(gpage_freearray
[--nr_gpages
]);
180 gpage_freearray
[nr_gpages
] = 0;
181 list_add(&m
->list
, &huge_boot_pages
);
187 /* Modelled after find_linux_pte() */
188 pte_t
*huge_pte_offset(struct mm_struct
*mm
, unsigned long addr
)
197 struct hstate
*hstate
;
198 psize
= get_slice_psize(mm
, addr
);
199 shift
= mmu_psize_to_shift(psize
);
200 sz
= ((1UL) << shift
);
201 hstate
= size_to_hstate(sz
);
203 addr
&= hstate
->mask
;
205 pg
= pgd_offset(mm
, addr
);
206 if (!pgd_none(*pg
)) {
207 pu
= pud_offset(pg
, addr
);
208 if (!pud_none(*pu
)) {
209 pm
= hpmd_offset(pu
, addr
, hstate
);
211 return hugepte_offset((hugepd_t
*)pm
, addr
,
219 pte_t
*huge_pte_alloc(struct mm_struct
*mm
,
220 unsigned long addr
, unsigned long sz
)
225 hugepd_t
*hpdp
= NULL
;
226 struct hstate
*hstate
;
228 hstate
= size_to_hstate(sz
);
230 psize
= get_slice_psize(mm
, addr
);
231 BUG_ON(!mmu_huge_psizes
[psize
]);
233 addr
&= hstate
->mask
;
235 pg
= pgd_offset(mm
, addr
);
236 pu
= pud_alloc(mm
, pg
, addr
);
239 pm
= hpmd_alloc(mm
, pu
, addr
, hstate
);
241 hpdp
= (hugepd_t
*)pm
;
247 if (hugepd_none(*hpdp
) && __hugepte_alloc(mm
, hpdp
, addr
, psize
))
250 return hugepte_offset(hpdp
, addr
, hstate
);
253 int huge_pmd_unshare(struct mm_struct
*mm
, unsigned long *addr
, pte_t
*ptep
)
258 static void free_hugepte_range(struct mmu_gather
*tlb
, hugepd_t
*hpdp
,
261 pte_t
*hugepte
= hugepd_page(*hpdp
);
265 pgtable_free_tlb(tlb
, pgtable_free_cache(hugepte
,
266 HUGEPTE_CACHE_NUM
+psize
-1,
270 static void hugetlb_free_pmd_range(struct mmu_gather
*tlb
, pud_t
*pud
,
271 unsigned long addr
, unsigned long end
,
272 unsigned long floor
, unsigned long ceiling
,
280 pmd
= pmd_offset(pud
, addr
);
282 next
= pmd_addr_end(addr
, end
);
285 free_hugepte_range(tlb
, (hugepd_t
*)pmd
, psize
);
286 } while (pmd
++, addr
= next
, addr
!= end
);
296 if (end
- 1 > ceiling
- 1)
299 pmd
= pmd_offset(pud
, start
);
301 pmd_free_tlb(tlb
, pmd
);
304 static void hugetlb_free_pud_range(struct mmu_gather
*tlb
, pgd_t
*pgd
,
305 unsigned long addr
, unsigned long end
,
306 unsigned long floor
, unsigned long ceiling
)
312 unsigned int psize
= get_slice_psize(tlb
->mm
, addr
);
313 shift
= mmu_psize_to_shift(psize
);
316 pud
= pud_offset(pgd
, addr
);
318 next
= pud_addr_end(addr
, end
);
319 #ifdef CONFIG_PPC_64K_PAGES
320 if (pud_none_or_clear_bad(pud
))
322 hugetlb_free_pmd_range(tlb
, pud
, addr
, next
, floor
, ceiling
,
325 if (shift
== PAGE_SHIFT_64K
) {
326 if (pud_none_or_clear_bad(pud
))
328 hugetlb_free_pmd_range(tlb
, pud
, addr
, next
, floor
,
333 free_hugepte_range(tlb
, (hugepd_t
*)pud
, psize
);
336 } while (pud
++, addr
= next
, addr
!= end
);
342 ceiling
&= PGDIR_MASK
;
346 if (end
- 1 > ceiling
- 1)
349 pud
= pud_offset(pgd
, start
);
351 pud_free_tlb(tlb
, pud
);
355 * This function frees user-level page tables of a process.
357 * Must be called with pagetable lock held.
359 void hugetlb_free_pgd_range(struct mmu_gather
*tlb
,
360 unsigned long addr
, unsigned long end
,
361 unsigned long floor
, unsigned long ceiling
)
368 * Comments below take from the normal free_pgd_range(). They
369 * apply here too. The tests against HUGEPD_MASK below are
370 * essential, because we *don't* test for this at the bottom
371 * level. Without them we'll attempt to free a hugepte table
372 * when we unmap just part of it, even if there are other
373 * active mappings using it.
375 * The next few lines have given us lots of grief...
377 * Why are we testing HUGEPD* at this top level? Because
378 * often there will be no work to do at all, and we'd prefer
379 * not to go all the way down to the bottom just to discover
382 * Why all these "- 1"s? Because 0 represents both the bottom
383 * of the address space and the top of it (using -1 for the
384 * top wouldn't help much: the masks would do the wrong thing).
385 * The rule is that addr 0 and floor 0 refer to the bottom of
386 * the address space, but end 0 and ceiling 0 refer to the top
387 * Comparisons need to use "end - 1" and "ceiling - 1" (though
388 * that end 0 case should be mythical).
390 * Wherever addr is brought up or ceiling brought down, we
391 * must be careful to reject "the opposite 0" before it
392 * confuses the subsequent tests. But what about where end is
393 * brought down by HUGEPD_SIZE below? no, end can't go down to
396 * Whereas we round start (addr) and ceiling down, by different
397 * masks at different levels, in order to test whether a table
398 * now has no other vmas using it, so can be freed, we don't
399 * bother to round floor or end up - the tests don't need that.
401 unsigned int psize
= get_slice_psize(tlb
->mm
, addr
);
403 addr
&= HUGEPD_MASK(psize
);
405 addr
+= HUGEPD_SIZE(psize
);
410 ceiling
&= HUGEPD_MASK(psize
);
414 if (end
- 1 > ceiling
- 1)
415 end
-= HUGEPD_SIZE(psize
);
420 pgd
= pgd_offset(tlb
->mm
, addr
);
422 psize
= get_slice_psize(tlb
->mm
, addr
);
423 BUG_ON(!mmu_huge_psizes
[psize
]);
424 next
= pgd_addr_end(addr
, end
);
425 if (pgd_none_or_clear_bad(pgd
))
427 hugetlb_free_pud_range(tlb
, pgd
, addr
, next
, floor
, ceiling
);
428 } while (pgd
++, addr
= next
, addr
!= end
);
431 void set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
432 pte_t
*ptep
, pte_t pte
)
434 if (pte_present(*ptep
)) {
435 /* We open-code pte_clear because we need to pass the right
436 * argument to hpte_need_flush (huge / !huge). Might not be
437 * necessary anymore if we make hpte_need_flush() get the
438 * page size from the slices
440 unsigned int psize
= get_slice_psize(mm
, addr
);
441 unsigned int shift
= mmu_psize_to_shift(psize
);
442 unsigned long sz
= ((1UL) << shift
);
443 struct hstate
*hstate
= size_to_hstate(sz
);
444 pte_update(mm
, addr
& hstate
->mask
, ptep
, ~0UL, 1);
446 *ptep
= __pte(pte_val(pte
) & ~_PAGE_HPTEFLAGS
);
449 pte_t
huge_ptep_get_and_clear(struct mm_struct
*mm
, unsigned long addr
,
452 unsigned long old
= pte_update(mm
, addr
, ptep
, ~0UL, 1);
457 follow_huge_addr(struct mm_struct
*mm
, unsigned long address
, int write
)
461 unsigned int mmu_psize
= get_slice_psize(mm
, address
);
463 /* Verify it is a huge page else bail. */
464 if (!mmu_huge_psizes
[mmu_psize
])
465 return ERR_PTR(-EINVAL
);
467 ptep
= huge_pte_offset(mm
, address
);
468 page
= pte_page(*ptep
);
470 unsigned int shift
= mmu_psize_to_shift(mmu_psize
);
471 unsigned long sz
= ((1UL) << shift
);
472 page
+= (address
% sz
) / PAGE_SIZE
;
478 int pmd_huge(pmd_t pmd
)
483 int pud_huge(pud_t pud
)
489 follow_huge_pmd(struct mm_struct
*mm
, unsigned long address
,
490 pmd_t
*pmd
, int write
)
497 unsigned long hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
498 unsigned long len
, unsigned long pgoff
,
501 struct hstate
*hstate
= hstate_file(file
);
502 int mmu_psize
= shift_to_mmu_psize(huge_page_shift(hstate
));
503 return slice_get_unmapped_area(addr
, len
, flags
, mmu_psize
, 1, 0);
507 * Called by asm hashtable.S for doing lazy icache flush
509 static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags
,
510 pte_t pte
, int trap
, unsigned long sz
)
515 if (!pfn_valid(pte_pfn(pte
)))
518 page
= pte_page(pte
);
521 if (!test_bit(PG_arch_1
, &page
->flags
) && !PageReserved(page
)) {
523 for (i
= 0; i
< (sz
/ PAGE_SIZE
); i
++)
524 __flush_dcache_icache(page_address(page
+i
));
525 set_bit(PG_arch_1
, &page
->flags
);
533 int hash_huge_page(struct mm_struct
*mm
, unsigned long access
,
534 unsigned long ea
, unsigned long vsid
, int local
,
538 unsigned long old_pte
, new_pte
;
539 unsigned long va
, rflags
, pa
, sz
;
542 int ssize
= user_segment_size(ea
);
543 unsigned int mmu_psize
;
545 mmu_psize
= get_slice_psize(mm
, ea
);
547 if (!mmu_huge_psizes
[mmu_psize
])
549 ptep
= huge_pte_offset(mm
, ea
);
551 /* Search the Linux page table for a match with va */
552 va
= hpt_va(ea
, vsid
, ssize
);
555 * If no pte found or not present, send the problem up to
558 if (unlikely(!ptep
|| pte_none(*ptep
)))
562 * Check the user's access rights to the page. If access should be
563 * prevented then send the problem up to do_page_fault.
565 if (unlikely(access
& ~pte_val(*ptep
)))
568 * At this point, we have a pte (old_pte) which can be used to build
569 * or update an HPTE. There are 2 cases:
571 * 1. There is a valid (present) pte with no associated HPTE (this is
572 * the most common case)
573 * 2. There is a valid (present) pte with an associated HPTE. The
574 * current values of the pp bits in the HPTE prevent access
575 * because we are doing software DIRTY bit management and the
576 * page is currently not DIRTY.
581 old_pte
= pte_val(*ptep
);
582 if (old_pte
& _PAGE_BUSY
)
584 new_pte
= old_pte
| _PAGE_BUSY
| _PAGE_ACCESSED
;
585 } while(old_pte
!= __cmpxchg_u64((unsigned long *)ptep
,
588 rflags
= 0x2 | (!(new_pte
& _PAGE_RW
));
589 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
590 rflags
|= ((new_pte
& _PAGE_EXEC
) ? 0 : HPTE_R_N
);
591 shift
= mmu_psize_to_shift(mmu_psize
);
592 sz
= ((1UL) << shift
);
593 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE
))
594 /* No CPU has hugepages but lacks no execute, so we
595 * don't need to worry about that case */
596 rflags
= hash_huge_page_do_lazy_icache(rflags
, __pte(old_pte
),
599 /* Check if pte already has an hpte (case 2) */
600 if (unlikely(old_pte
& _PAGE_HASHPTE
)) {
601 /* There MIGHT be an HPTE for this pte */
602 unsigned long hash
, slot
;
604 hash
= hpt_hash(va
, shift
, ssize
);
605 if (old_pte
& _PAGE_F_SECOND
)
607 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
608 slot
+= (old_pte
& _PAGE_F_GIX
) >> 12;
610 if (ppc_md
.hpte_updatepp(slot
, rflags
, va
, mmu_psize
,
612 old_pte
&= ~_PAGE_HPTEFLAGS
;
615 if (likely(!(old_pte
& _PAGE_HASHPTE
))) {
616 unsigned long hash
= hpt_hash(va
, shift
, ssize
);
617 unsigned long hpte_group
;
619 pa
= pte_pfn(__pte(old_pte
)) << PAGE_SHIFT
;
622 hpte_group
= ((hash
& htab_hash_mask
) *
623 HPTES_PER_GROUP
) & ~0x7UL
;
625 /* clear HPTE slot informations in new PTE */
626 #ifdef CONFIG_PPC_64K_PAGES
627 new_pte
= (new_pte
& ~_PAGE_HPTEFLAGS
) | _PAGE_HPTE_SUB0
;
629 new_pte
= (new_pte
& ~_PAGE_HPTEFLAGS
) | _PAGE_HASHPTE
;
631 /* Add in WIMG bits */
632 rflags
|= (new_pte
& (_PAGE_WRITETHRU
| _PAGE_NO_CACHE
|
633 _PAGE_COHERENT
| _PAGE_GUARDED
));
635 /* Insert into the hash table, primary slot */
636 slot
= ppc_md
.hpte_insert(hpte_group
, va
, pa
, rflags
, 0,
639 /* Primary is full, try the secondary */
640 if (unlikely(slot
== -1)) {
641 hpte_group
= ((~hash
& htab_hash_mask
) *
642 HPTES_PER_GROUP
) & ~0x7UL
;
643 slot
= ppc_md
.hpte_insert(hpte_group
, va
, pa
, rflags
,
648 hpte_group
= ((hash
& htab_hash_mask
) *
649 HPTES_PER_GROUP
)&~0x7UL
;
651 ppc_md
.hpte_remove(hpte_group
);
656 if (unlikely(slot
== -2))
657 panic("hash_huge_page: pte_insert failed\n");
659 new_pte
|= (slot
<< 12) & (_PAGE_F_SECOND
| _PAGE_F_GIX
);
663 * No need to use ldarx/stdcx here
665 *ptep
= __pte(new_pte
& ~_PAGE_BUSY
);
673 void set_huge_psize(int psize
)
675 /* Check that it is a page size supported by the hardware and
676 * that it fits within pagetable limits. */
677 if (mmu_psize_defs
[psize
].shift
&&
678 mmu_psize_defs
[psize
].shift
< SID_SHIFT_1T
&&
679 (mmu_psize_defs
[psize
].shift
> MIN_HUGEPTE_SHIFT
||
680 mmu_psize_defs
[psize
].shift
== PAGE_SHIFT_64K
||
681 mmu_psize_defs
[psize
].shift
== PAGE_SHIFT_16G
)) {
682 /* Return if huge page size has already been setup or is the
683 * same as the base page size. */
684 if (mmu_huge_psizes
[psize
] ||
685 mmu_psize_defs
[psize
].shift
== PAGE_SHIFT
)
687 hugetlb_add_hstate(mmu_psize_defs
[psize
].shift
- PAGE_SHIFT
);
689 switch (mmu_psize_defs
[psize
].shift
) {
691 /* We only allow 64k hpages with 4k base page,
692 * which was checked above, and always put them
694 hugepte_shift
[psize
] = PMD_SHIFT
;
697 /* 16M pages can be at two different levels
698 * of pagestables based on base page size */
699 if (PAGE_SHIFT
== PAGE_SHIFT_64K
)
700 hugepte_shift
[psize
] = PMD_SHIFT
;
701 else /* 4k base page */
702 hugepte_shift
[psize
] = PUD_SHIFT
;
705 /* 16G pages are always at PGD level */
706 hugepte_shift
[psize
] = PGDIR_SHIFT
;
709 hugepte_shift
[psize
] -= mmu_psize_defs
[psize
].shift
;
711 hugepte_shift
[psize
] = 0;
714 static int __init
hugepage_setup_sz(char *str
)
716 unsigned long long size
;
720 size
= memparse(str
, &str
);
723 mmu_psize
= shift_to_mmu_psize(shift
);
724 if (mmu_psize
>= 0 && mmu_psize_defs
[mmu_psize
].shift
)
725 set_huge_psize(mmu_psize
);
727 printk(KERN_WARNING
"Invalid huge page size specified(%llu)\n", size
);
731 __setup("hugepagesz=", hugepage_setup_sz
);
733 static void zero_ctor(struct kmem_cache
*cache
, void *addr
)
735 memset(addr
, 0, kmem_cache_size(cache
));
738 static int __init
hugetlbpage_init(void)
742 if (!cpu_has_feature(CPU_FTR_16M_PAGE
))
744 /* Add supported huge page sizes. Need to change HUGE_MAX_HSTATE
745 * and adjust PTE_NONCACHE_NUM if the number of supported huge page
748 set_huge_psize(MMU_PAGE_16M
);
749 set_huge_psize(MMU_PAGE_64K
);
750 set_huge_psize(MMU_PAGE_16G
);
752 for (psize
= 0; psize
< MMU_PAGE_COUNT
; ++psize
) {
753 if (mmu_huge_psizes
[psize
]) {
754 huge_pgtable_cache(psize
) = kmem_cache_create(
755 HUGEPTE_CACHE_NAME(psize
),
756 HUGEPTE_TABLE_SIZE(psize
),
757 HUGEPTE_TABLE_SIZE(psize
),
760 if (!huge_pgtable_cache(psize
))
761 panic("hugetlbpage_init(): could not create %s"\
762 "\n", HUGEPTE_CACHE_NAME(psize
));
769 module_init(hugetlbpage_init
);