1 // SPDX-License-Identifier: GPL-2.0
3 * SPARC64 Huge TLB page support.
5 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
10 #include <linux/sched/mm.h>
11 #include <linux/hugetlb.h>
12 #include <linux/pagemap.h>
13 #include <linux/sysctl.h>
16 #include <asm/pgalloc.h>
18 #include <asm/tlbflush.h>
19 #include <asm/cacheflush.h>
20 #include <asm/mmu_context.h>
22 /* Slightly simplified from the non-hugepage variant because by
23 * definition we don't have to worry about any page coloring stuff
26 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file
*filp
,
32 struct hstate
*h
= hstate_file(filp
);
33 unsigned long task_size
= TASK_SIZE
;
34 struct vm_unmapped_area_info info
;
36 if (test_thread_flag(TIF_32BIT
))
37 task_size
= STACK_TOP32
;
41 info
.low_limit
= TASK_UNMAPPED_BASE
;
42 info
.high_limit
= min(task_size
, VA_EXCLUDE_START
);
43 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
44 info
.align_offset
= 0;
45 addr
= vm_unmapped_area(&info
);
47 if ((addr
& ~PAGE_MASK
) && task_size
> VA_EXCLUDE_END
) {
48 VM_BUG_ON(addr
!= -ENOMEM
);
49 info
.low_limit
= VA_EXCLUDE_END
;
50 info
.high_limit
= task_size
;
51 addr
= vm_unmapped_area(&info
);
58 hugetlb_get_unmapped_area_topdown(struct file
*filp
, const unsigned long addr0
,
59 const unsigned long len
,
60 const unsigned long pgoff
,
61 const unsigned long flags
)
63 struct hstate
*h
= hstate_file(filp
);
64 struct mm_struct
*mm
= current
->mm
;
65 unsigned long addr
= addr0
;
66 struct vm_unmapped_area_info info
;
68 /* This should only ever run for 32-bit processes. */
69 BUG_ON(!test_thread_flag(TIF_32BIT
));
71 info
.flags
= VM_UNMAPPED_AREA_TOPDOWN
;
73 info
.low_limit
= PAGE_SIZE
;
74 info
.high_limit
= mm
->mmap_base
;
75 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
76 info
.align_offset
= 0;
77 addr
= vm_unmapped_area(&info
);
80 * A failed mmap() very likely causes application failure,
81 * so fall back to the bottom-up function here. This scenario
82 * can happen with large stack limits and large mmap()
85 if (addr
& ~PAGE_MASK
) {
86 VM_BUG_ON(addr
!= -ENOMEM
);
88 info
.low_limit
= TASK_UNMAPPED_BASE
;
89 info
.high_limit
= STACK_TOP32
;
90 addr
= vm_unmapped_area(&info
);
97 hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
98 unsigned long len
, unsigned long pgoff
, unsigned long flags
)
100 struct hstate
*h
= hstate_file(file
);
101 struct mm_struct
*mm
= current
->mm
;
102 struct vm_area_struct
*vma
;
103 unsigned long task_size
= TASK_SIZE
;
105 if (test_thread_flag(TIF_32BIT
))
106 task_size
= STACK_TOP32
;
108 if (len
& ~huge_page_mask(h
))
113 if (flags
& MAP_FIXED
) {
114 if (prepare_hugepage_range(file
, addr
, len
))
120 addr
= ALIGN(addr
, huge_page_size(h
));
121 vma
= find_vma(mm
, addr
);
122 if (task_size
- len
>= addr
&&
123 (!vma
|| addr
+ len
<= vm_start_gap(vma
)))
126 if (mm
->get_unmapped_area
== arch_get_unmapped_area
)
127 return hugetlb_get_unmapped_area_bottomup(file
, addr
, len
,
130 return hugetlb_get_unmapped_area_topdown(file
, addr
, len
,
134 static pte_t
sun4u_hugepage_shift_to_tte(pte_t entry
, unsigned int shift
)
139 static pte_t
sun4v_hugepage_shift_to_tte(pte_t entry
, unsigned int shift
)
141 unsigned long hugepage_size
= _PAGE_SZ4MB_4V
;
143 pte_val(entry
) = pte_val(entry
) & ~_PAGE_SZALL_4V
;
146 case HPAGE_16GB_SHIFT
:
147 hugepage_size
= _PAGE_SZ16GB_4V
;
148 pte_val(entry
) |= _PAGE_PUD_HUGE
;
150 case HPAGE_2GB_SHIFT
:
151 hugepage_size
= _PAGE_SZ2GB_4V
;
152 pte_val(entry
) |= _PAGE_PMD_HUGE
;
154 case HPAGE_256MB_SHIFT
:
155 hugepage_size
= _PAGE_SZ256MB_4V
;
156 pte_val(entry
) |= _PAGE_PMD_HUGE
;
159 pte_val(entry
) |= _PAGE_PMD_HUGE
;
161 case HPAGE_64K_SHIFT
:
162 hugepage_size
= _PAGE_SZ64K_4V
;
165 WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift
);
168 pte_val(entry
) = pte_val(entry
) | hugepage_size
;
172 static pte_t
hugepage_shift_to_tte(pte_t entry
, unsigned int shift
)
174 if (tlb_type
== hypervisor
)
175 return sun4v_hugepage_shift_to_tte(entry
, shift
);
177 return sun4u_hugepage_shift_to_tte(entry
, shift
);
180 pte_t
arch_make_huge_pte(pte_t entry
, struct vm_area_struct
*vma
,
181 struct page
*page
, int writeable
)
183 unsigned int shift
= huge_page_shift(hstate_vma(vma
));
186 pte
= hugepage_shift_to_tte(entry
, shift
);
188 #ifdef CONFIG_SPARC64
189 /* If this vma has ADI enabled on it, turn on TTE.mcd
191 if (vma
->vm_flags
& VM_SPARC_ADI
)
192 return pte_mkmcd(pte
);
194 return pte_mknotmcd(pte
);
200 static unsigned int sun4v_huge_tte_to_shift(pte_t entry
)
202 unsigned long tte_szbits
= pte_val(entry
) & _PAGE_SZALL_4V
;
205 switch (tte_szbits
) {
206 case _PAGE_SZ16GB_4V
:
207 shift
= HPAGE_16GB_SHIFT
;
210 shift
= HPAGE_2GB_SHIFT
;
212 case _PAGE_SZ256MB_4V
:
213 shift
= HPAGE_256MB_SHIFT
;
216 shift
= REAL_HPAGE_SHIFT
;
219 shift
= HPAGE_64K_SHIFT
;
228 static unsigned int sun4u_huge_tte_to_shift(pte_t entry
)
230 unsigned long tte_szbits
= pte_val(entry
) & _PAGE_SZALL_4U
;
233 switch (tte_szbits
) {
234 case _PAGE_SZ256MB_4U
:
235 shift
= HPAGE_256MB_SHIFT
;
238 shift
= REAL_HPAGE_SHIFT
;
241 shift
= HPAGE_64K_SHIFT
;
250 static unsigned long tte_to_shift(pte_t entry
)
252 if (tlb_type
== hypervisor
)
253 return sun4v_huge_tte_to_shift(entry
);
255 return sun4u_huge_tte_to_shift(entry
);
258 static unsigned int huge_tte_to_shift(pte_t entry
)
260 unsigned long shift
= tte_to_shift(entry
);
262 if (shift
== PAGE_SHIFT
)
263 WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
269 static unsigned long huge_tte_to_size(pte_t pte
)
271 unsigned long size
= 1UL << huge_tte_to_shift(pte
);
273 if (size
== REAL_HPAGE_SIZE
)
278 unsigned long pud_leaf_size(pud_t pud
) { return 1UL << tte_to_shift(*(pte_t
*)&pud
); }
279 unsigned long pmd_leaf_size(pmd_t pmd
) { return 1UL << tte_to_shift(*(pte_t
*)&pmd
); }
280 unsigned long pte_leaf_size(pte_t pte
) { return 1UL << tte_to_shift(pte
); }
282 pte_t
*huge_pte_alloc(struct mm_struct
*mm
,
283 unsigned long addr
, unsigned long sz
)
290 pgd
= pgd_offset(mm
, addr
);
291 p4d
= p4d_offset(pgd
, addr
);
292 pud
= pud_alloc(mm
, p4d
, addr
);
297 pmd
= pmd_alloc(mm
, pud
, addr
);
302 return pte_alloc_map(mm
, pmd
, addr
);
305 pte_t
*huge_pte_offset(struct mm_struct
*mm
,
306 unsigned long addr
, unsigned long sz
)
313 pgd
= pgd_offset(mm
, addr
);
316 p4d
= p4d_offset(pgd
, addr
);
319 pud
= pud_offset(p4d
, addr
);
322 if (is_hugetlb_pud(*pud
))
324 pmd
= pmd_offset(pud
, addr
);
327 if (is_hugetlb_pmd(*pmd
))
329 return pte_offset_map(pmd
, addr
);
332 void set_huge_pte_at(struct mm_struct
*mm
, unsigned long addr
,
333 pte_t
*ptep
, pte_t entry
)
335 unsigned int nptes
, orig_shift
, shift
;
336 unsigned long i
, size
;
339 size
= huge_tte_to_size(entry
);
342 if (size
>= PUD_SIZE
)
344 else if (size
>= PMD_SIZE
)
349 nptes
= size
>> shift
;
351 if (!pte_present(*ptep
) && pte_present(entry
))
352 mm
->context
.hugetlb_pte_count
+= nptes
;
356 orig_shift
= pte_none(orig
) ? PAGE_SHIFT
: huge_tte_to_shift(orig
);
358 for (i
= 0; i
< nptes
; i
++)
359 ptep
[i
] = __pte(pte_val(entry
) + (i
<< shift
));
361 maybe_tlb_batch_add(mm
, addr
, ptep
, orig
, 0, orig_shift
);
362 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
363 if (size
== HPAGE_SIZE
)
364 maybe_tlb_batch_add(mm
, addr
+ REAL_HPAGE_SIZE
, ptep
, orig
, 0,
368 pte_t
huge_ptep_get_and_clear(struct mm_struct
*mm
, unsigned long addr
,
371 unsigned int i
, nptes
, orig_shift
, shift
;
376 size
= huge_tte_to_size(entry
);
379 if (size
>= PUD_SIZE
)
381 else if (size
>= PMD_SIZE
)
386 nptes
= size
>> shift
;
387 orig_shift
= pte_none(entry
) ? PAGE_SHIFT
: huge_tte_to_shift(entry
);
389 if (pte_present(entry
))
390 mm
->context
.hugetlb_pte_count
-= nptes
;
393 for (i
= 0; i
< nptes
; i
++)
394 ptep
[i
] = __pte(0UL);
396 maybe_tlb_batch_add(mm
, addr
, ptep
, entry
, 0, orig_shift
);
397 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
398 if (size
== HPAGE_SIZE
)
399 maybe_tlb_batch_add(mm
, addr
+ REAL_HPAGE_SIZE
, ptep
, entry
, 0,
405 int pmd_huge(pmd_t pmd
)
407 return !pmd_none(pmd
) &&
408 (pmd_val(pmd
) & (_PAGE_VALID
|_PAGE_PMD_HUGE
)) != _PAGE_VALID
;
411 int pud_huge(pud_t pud
)
413 return !pud_none(pud
) &&
414 (pud_val(pud
) & (_PAGE_VALID
|_PAGE_PUD_HUGE
)) != _PAGE_VALID
;
417 static void hugetlb_free_pte_range(struct mmu_gather
*tlb
, pmd_t
*pmd
,
420 pgtable_t token
= pmd_pgtable(*pmd
);
423 pte_free_tlb(tlb
, token
, addr
);
424 mm_dec_nr_ptes(tlb
->mm
);
427 static void hugetlb_free_pmd_range(struct mmu_gather
*tlb
, pud_t
*pud
,
428 unsigned long addr
, unsigned long end
,
429 unsigned long floor
, unsigned long ceiling
)
436 pmd
= pmd_offset(pud
, addr
);
438 next
= pmd_addr_end(addr
, end
);
441 if (is_hugetlb_pmd(*pmd
))
444 hugetlb_free_pte_range(tlb
, pmd
, addr
);
445 } while (pmd
++, addr
= next
, addr
!= end
);
455 if (end
- 1 > ceiling
- 1)
458 pmd
= pmd_offset(pud
, start
);
460 pmd_free_tlb(tlb
, pmd
, start
);
461 mm_dec_nr_pmds(tlb
->mm
);
464 static void hugetlb_free_pud_range(struct mmu_gather
*tlb
, p4d_t
*p4d
,
465 unsigned long addr
, unsigned long end
,
466 unsigned long floor
, unsigned long ceiling
)
473 pud
= pud_offset(p4d
, addr
);
475 next
= pud_addr_end(addr
, end
);
476 if (pud_none_or_clear_bad(pud
))
478 if (is_hugetlb_pud(*pud
))
481 hugetlb_free_pmd_range(tlb
, pud
, addr
, next
, floor
,
483 } while (pud
++, addr
= next
, addr
!= end
);
489 ceiling
&= PGDIR_MASK
;
493 if (end
- 1 > ceiling
- 1)
496 pud
= pud_offset(p4d
, start
);
498 pud_free_tlb(tlb
, pud
, start
);
499 mm_dec_nr_puds(tlb
->mm
);
502 void hugetlb_free_pgd_range(struct mmu_gather
*tlb
,
503 unsigned long addr
, unsigned long end
,
504 unsigned long floor
, unsigned long ceiling
)
521 if (end
- 1 > ceiling
- 1)
526 pgd
= pgd_offset(tlb
->mm
, addr
);
527 p4d
= p4d_offset(pgd
, addr
);
529 next
= p4d_addr_end(addr
, end
);
530 if (p4d_none_or_clear_bad(p4d
))
532 hugetlb_free_pud_range(tlb
, p4d
, addr
, next
, floor
, ceiling
);
533 } while (p4d
++, addr
= next
, addr
!= end
);