]>
Commit | Line | Data |
---|---|---|
084bd298 SC |
1 | /* |
2 | * arch/arm64/mm/hugetlbpage.c | |
3 | * | |
4 | * Copyright (C) 2013 Linaro Ltd. | |
5 | * | |
6 | * Based on arch/x86/mm/hugetlbpage.c. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
084bd298 SC |
16 | */ |
17 | ||
18 | #include <linux/init.h> | |
19 | #include <linux/fs.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/hugetlb.h> | |
22 | #include <linux/pagemap.h> | |
23 | #include <linux/err.h> | |
24 | #include <linux/sysctl.h> | |
25 | #include <asm/mman.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/tlbflush.h> | |
28 | #include <asm/pgalloc.h> | |
29 | ||
084bd298 SC |
30 | int pmd_huge(pmd_t pmd) |
31 | { | |
fd28f5d4 | 32 | return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); |
084bd298 SC |
33 | } |
34 | ||
35 | int pud_huge(pud_t pud) | |
36 | { | |
4797ec2d | 37 | #ifndef __PAGETABLE_PMD_FOLDED |
fd28f5d4 | 38 | return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); |
4797ec2d MS |
39 | #else |
40 | return 0; | |
41 | #endif | |
084bd298 SC |
42 | } |
43 | ||
66b3923a DW |
44 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, |
45 | pte_t *ptep, pte_t pte, size_t *pgsize) | |
46 | { | |
47 | pgd_t *pgd = pgd_offset(mm, addr); | |
48 | pud_t *pud; | |
49 | pmd_t *pmd; | |
50 | ||
51 | *pgsize = PAGE_SIZE; | |
52 | if (!pte_cont(pte)) | |
53 | return 1; | |
66b3923a | 54 | pud = pud_offset(pgd, addr); |
66b3923a | 55 | pmd = pmd_offset(pud, addr); |
66b3923a DW |
56 | if ((pte_t *)pmd == ptep) { |
57 | *pgsize = PMD_SIZE; | |
58 | return CONT_PMDS; | |
59 | } | |
60 | return CONT_PTES; | |
61 | } | |
62 | ||
63 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |
64 | pte_t *ptep, pte_t pte) | |
65 | { | |
66 | size_t pgsize; | |
67 | int i; | |
68 | int ncontig = find_num_contig(mm, addr, ptep, pte, &pgsize); | |
69 | unsigned long pfn; | |
70 | pgprot_t hugeprot; | |
71 | ||
72 | if (ncontig == 1) { | |
73 | set_pte_at(mm, addr, ptep, pte); | |
74 | return; | |
75 | } | |
76 | ||
77 | pfn = pte_pfn(pte); | |
78 | hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | |
79 | for (i = 0; i < ncontig; i++) { | |
80 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, | |
81 | pte_val(pfn_pte(pfn, hugeprot))); | |
82 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | |
83 | ptep++; | |
84 | pfn += pgsize >> PAGE_SHIFT; | |
85 | addr += pgsize; | |
86 | } | |
87 | } | |
88 | ||
89 | pte_t *huge_pte_alloc(struct mm_struct *mm, | |
90 | unsigned long addr, unsigned long sz) | |
91 | { | |
92 | pgd_t *pgd; | |
93 | pud_t *pud; | |
94 | pte_t *pte = NULL; | |
95 | ||
96 | pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); | |
97 | pgd = pgd_offset(mm, addr); | |
98 | pud = pud_alloc(mm, pgd, addr); | |
99 | if (!pud) | |
100 | return NULL; | |
101 | ||
102 | if (sz == PUD_SIZE) { | |
103 | pte = (pte_t *)pud; | |
104 | } else if (sz == (PAGE_SIZE * CONT_PTES)) { | |
105 | pmd_t *pmd = pmd_alloc(mm, pud, addr); | |
106 | ||
107 | WARN_ON(addr & (sz - 1)); | |
108 | /* | |
109 | * Note that if this code were ever ported to the | |
110 | * 32-bit arm platform then it will cause trouble in | |
111 | * the case where CONFIG_HIGHPTE is set, since there | |
112 | * will be no pte_unmap() to correspond with this | |
113 | * pte_alloc_map(). | |
114 | */ | |
3ed3a4f0 | 115 | pte = pte_alloc_map(mm, pmd, addr); |
66b3923a DW |
116 | } else if (sz == PMD_SIZE) { |
117 | if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && | |
118 | pud_none(*pud)) | |
119 | pte = huge_pmd_share(mm, addr, pud); | |
120 | else | |
121 | pte = (pte_t *)pmd_alloc(mm, pud, addr); | |
122 | } else if (sz == (PMD_SIZE * CONT_PMDS)) { | |
123 | pmd_t *pmd; | |
124 | ||
125 | pmd = pmd_alloc(mm, pud, addr); | |
126 | WARN_ON(addr & (sz - 1)); | |
127 | return (pte_t *)pmd; | |
128 | } | |
129 | ||
130 | pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, | |
131 | sz, pte, pte_val(*pte)); | |
132 | return pte; | |
133 | } | |
134 | ||
135 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |
136 | { | |
137 | pgd_t *pgd; | |
138 | pud_t *pud; | |
139 | pmd_t *pmd = NULL; | |
140 | pte_t *pte = NULL; | |
141 | ||
142 | pgd = pgd_offset(mm, addr); | |
143 | pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); | |
144 | if (!pgd_present(*pgd)) | |
145 | return NULL; | |
146 | pud = pud_offset(pgd, addr); | |
147 | if (!pud_present(*pud)) | |
148 | return NULL; | |
149 | ||
150 | if (pud_huge(*pud)) | |
151 | return (pte_t *)pud; | |
152 | pmd = pmd_offset(pud, addr); | |
153 | if (!pmd_present(*pmd)) | |
154 | return NULL; | |
155 | ||
156 | if (pte_cont(pmd_pte(*pmd))) { | |
157 | pmd = pmd_offset( | |
158 | pud, (addr & CONT_PMD_MASK)); | |
159 | return (pte_t *)pmd; | |
160 | } | |
161 | if (pmd_huge(*pmd)) | |
162 | return (pte_t *)pmd; | |
163 | pte = pte_offset_kernel(pmd, addr); | |
164 | if (pte_present(*pte) && pte_cont(*pte)) { | |
165 | pte = pte_offset_kernel( | |
166 | pmd, (addr & CONT_PTE_MASK)); | |
167 | return pte; | |
168 | } | |
169 | return NULL; | |
170 | } | |
171 | ||
172 | pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, | |
173 | struct page *page, int writable) | |
174 | { | |
175 | size_t pagesize = huge_page_size(hstate_vma(vma)); | |
176 | ||
177 | if (pagesize == CONT_PTE_SIZE) { | |
178 | entry = pte_mkcont(entry); | |
179 | } else if (pagesize == CONT_PMD_SIZE) { | |
180 | entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); | |
181 | } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { | |
182 | pr_warn("%s: unrecognized huge page size 0x%lx\n", | |
183 | __func__, pagesize); | |
184 | } | |
185 | return entry; | |
186 | } | |
187 | ||
188 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
189 | unsigned long addr, pte_t *ptep) | |
190 | { | |
191 | pte_t pte; | |
192 | ||
193 | if (pte_cont(*ptep)) { | |
194 | int ncontig, i; | |
195 | size_t pgsize; | |
196 | pte_t *cpte; | |
197 | bool is_dirty = false; | |
198 | ||
199 | cpte = huge_pte_offset(mm, addr); | |
200 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
201 | /* save the 1st pte to return */ | |
202 | pte = ptep_get_and_clear(mm, addr, cpte); | |
0c2f0afe | 203 | for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { |
66b3923a DW |
204 | /* |
205 | * If HW_AFDBM is enabled, then the HW could | |
206 | * turn on the dirty bit for any of the page | |
207 | * in the set, so check them all. | |
208 | */ | |
209 | ++cpte; | |
210 | if (pte_dirty(ptep_get_and_clear(mm, addr, cpte))) | |
211 | is_dirty = true; | |
212 | } | |
213 | if (is_dirty) | |
214 | return pte_mkdirty(pte); | |
215 | else | |
216 | return pte; | |
217 | } else { | |
218 | return ptep_get_and_clear(mm, addr, ptep); | |
219 | } | |
220 | } | |
221 | ||
222 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, | |
223 | unsigned long addr, pte_t *ptep, | |
224 | pte_t pte, int dirty) | |
225 | { | |
226 | pte_t *cpte; | |
227 | ||
228 | if (pte_cont(pte)) { | |
229 | int ncontig, i, changed = 0; | |
230 | size_t pgsize = 0; | |
231 | unsigned long pfn = pte_pfn(pte); | |
232 | /* Select all bits except the pfn */ | |
233 | pgprot_t hugeprot = | |
234 | __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ | |
235 | pte_val(pte)); | |
236 | ||
237 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
238 | pfn = pte_pfn(*cpte); | |
239 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
240 | *cpte, &pgsize); | |
0c2f0afe | 241 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) { |
69d01234 | 242 | changed |= ptep_set_access_flags(vma, addr, cpte, |
66b3923a DW |
243 | pfn_pte(pfn, |
244 | hugeprot), | |
245 | dirty); | |
246 | pfn += pgsize >> PAGE_SHIFT; | |
247 | } | |
248 | return changed; | |
249 | } else { | |
250 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); | |
251 | } | |
252 | } | |
253 | ||
254 | void huge_ptep_set_wrprotect(struct mm_struct *mm, | |
255 | unsigned long addr, pte_t *ptep) | |
256 | { | |
257 | if (pte_cont(*ptep)) { | |
258 | int ncontig, i; | |
259 | pte_t *cpte; | |
260 | size_t pgsize = 0; | |
261 | ||
262 | cpte = huge_pte_offset(mm, addr); | |
263 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
0c2f0afe | 264 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) |
66b3923a DW |
265 | ptep_set_wrprotect(mm, addr, cpte); |
266 | } else { | |
267 | ptep_set_wrprotect(mm, addr, ptep); | |
268 | } | |
269 | } | |
270 | ||
271 | void huge_ptep_clear_flush(struct vm_area_struct *vma, | |
272 | unsigned long addr, pte_t *ptep) | |
273 | { | |
274 | if (pte_cont(*ptep)) { | |
275 | int ncontig, i; | |
276 | pte_t *cpte; | |
277 | size_t pgsize = 0; | |
278 | ||
279 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
280 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
281 | *cpte, &pgsize); | |
0c2f0afe | 282 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) |
66b3923a DW |
283 | ptep_clear_flush(vma, addr, cpte); |
284 | } else { | |
285 | ptep_clear_flush(vma, addr, ptep); | |
286 | } | |
287 | } | |
288 | ||
084bd298 SC |
289 | static __init int setup_hugepagesz(char *opt) |
290 | { | |
291 | unsigned long ps = memparse(opt, &opt); | |
66b3923a | 292 | |
084bd298 SC |
293 | if (ps == PMD_SIZE) { |
294 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
295 | } else if (ps == PUD_SIZE) { | |
296 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | |
ab2e1b89 WD |
297 | } else if (ps == (PAGE_SIZE * CONT_PTES)) { |
298 | hugetlb_add_hstate(CONT_PTE_SHIFT); | |
299 | } else if (ps == (PMD_SIZE * CONT_PMDS)) { | |
300 | hugetlb_add_hstate((PMD_SHIFT + CONT_PMD_SHIFT) - PAGE_SHIFT); | |
084bd298 | 301 | } else { |
d77e20ce | 302 | hugetlb_bad_size(); |
66b3923a | 303 | pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); |
084bd298 SC |
304 | return 0; |
305 | } | |
306 | return 1; | |
307 | } | |
308 | __setup("hugepagesz=", setup_hugepagesz); | |
ab2e1b89 WD |
309 | |
310 | #ifdef CONFIG_ARM64_64K_PAGES | |
311 | static __init int add_default_hugepagesz(void) | |
312 | { | |
313 | if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) | |
6ed0038d | 314 | hugetlb_add_hstate(CONT_PTE_SHIFT); |
ab2e1b89 WD |
315 | return 0; |
316 | } | |
317 | arch_initcall(add_default_hugepagesz); | |
318 | #endif |