]>
Commit | Line | Data |
---|---|---|
084bd298 SC |
1 | /* |
2 | * arch/arm64/mm/hugetlbpage.c | |
3 | * | |
4 | * Copyright (C) 2013 Linaro Ltd. | |
5 | * | |
6 | * Based on arch/x86/mm/hugetlbpage.c. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
084bd298 SC |
16 | */ |
17 | ||
18 | #include <linux/init.h> | |
19 | #include <linux/fs.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/hugetlb.h> | |
22 | #include <linux/pagemap.h> | |
23 | #include <linux/err.h> | |
24 | #include <linux/sysctl.h> | |
25 | #include <asm/mman.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/tlbflush.h> | |
28 | #include <asm/pgalloc.h> | |
29 | ||
084bd298 SC |
30 | int pmd_huge(pmd_t pmd) |
31 | { | |
fd28f5d4 | 32 | return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); |
084bd298 SC |
33 | } |
34 | ||
35 | int pud_huge(pud_t pud) | |
36 | { | |
4797ec2d | 37 | #ifndef __PAGETABLE_PMD_FOLDED |
fd28f5d4 | 38 | return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); |
4797ec2d MS |
39 | #else |
40 | return 0; | |
41 | #endif | |
084bd298 SC |
42 | } |
43 | ||
66b3923a DW |
44 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, |
45 | pte_t *ptep, pte_t pte, size_t *pgsize) | |
46 | { | |
47 | pgd_t *pgd = pgd_offset(mm, addr); | |
48 | pud_t *pud; | |
49 | pmd_t *pmd; | |
50 | ||
51 | *pgsize = PAGE_SIZE; | |
52 | if (!pte_cont(pte)) | |
53 | return 1; | |
66b3923a | 54 | pud = pud_offset(pgd, addr); |
66b3923a | 55 | pmd = pmd_offset(pud, addr); |
66b3923a DW |
56 | if ((pte_t *)pmd == ptep) { |
57 | *pgsize = PMD_SIZE; | |
58 | return CONT_PMDS; | |
59 | } | |
60 | return CONT_PTES; | |
61 | } | |
62 | ||
63 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |
64 | pte_t *ptep, pte_t pte) | |
65 | { | |
66 | size_t pgsize; | |
67 | int i; | |
68 | int ncontig = find_num_contig(mm, addr, ptep, pte, &pgsize); | |
69 | unsigned long pfn; | |
70 | pgprot_t hugeprot; | |
71 | ||
72 | if (ncontig == 1) { | |
73 | set_pte_at(mm, addr, ptep, pte); | |
74 | return; | |
75 | } | |
76 | ||
77 | pfn = pte_pfn(pte); | |
78 | hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | |
79 | for (i = 0; i < ncontig; i++) { | |
80 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, | |
81 | pte_val(pfn_pte(pfn, hugeprot))); | |
82 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | |
83 | ptep++; | |
84 | pfn += pgsize >> PAGE_SHIFT; | |
85 | addr += pgsize; | |
86 | } | |
87 | } | |
88 | ||
89 | pte_t *huge_pte_alloc(struct mm_struct *mm, | |
90 | unsigned long addr, unsigned long sz) | |
91 | { | |
92 | pgd_t *pgd; | |
93 | pud_t *pud; | |
94 | pte_t *pte = NULL; | |
95 | ||
96 | pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); | |
97 | pgd = pgd_offset(mm, addr); | |
98 | pud = pud_alloc(mm, pgd, addr); | |
99 | if (!pud) | |
100 | return NULL; | |
101 | ||
102 | if (sz == PUD_SIZE) { | |
103 | pte = (pte_t *)pud; | |
104 | } else if (sz == (PAGE_SIZE * CONT_PTES)) { | |
105 | pmd_t *pmd = pmd_alloc(mm, pud, addr); | |
106 | ||
107 | WARN_ON(addr & (sz - 1)); | |
108 | /* | |
109 | * Note that if this code were ever ported to the | |
110 | * 32-bit arm platform then it will cause trouble in | |
111 | * the case where CONFIG_HIGHPTE is set, since there | |
112 | * will be no pte_unmap() to correspond with this | |
113 | * pte_alloc_map(). | |
114 | */ | |
3ed3a4f0 | 115 | pte = pte_alloc_map(mm, pmd, addr); |
66b3923a DW |
116 | } else if (sz == PMD_SIZE) { |
117 | if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && | |
118 | pud_none(*pud)) | |
119 | pte = huge_pmd_share(mm, addr, pud); | |
120 | else | |
121 | pte = (pte_t *)pmd_alloc(mm, pud, addr); | |
122 | } else if (sz == (PMD_SIZE * CONT_PMDS)) { | |
123 | pmd_t *pmd; | |
124 | ||
125 | pmd = pmd_alloc(mm, pud, addr); | |
126 | WARN_ON(addr & (sz - 1)); | |
127 | return (pte_t *)pmd; | |
128 | } | |
129 | ||
130 | pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, | |
131 | sz, pte, pte_val(*pte)); | |
132 | return pte; | |
133 | } | |
134 | ||
135 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |
136 | { | |
137 | pgd_t *pgd; | |
138 | pud_t *pud; | |
0f3c2bcf | 139 | pmd_t *pmd; |
66b3923a DW |
140 | |
141 | pgd = pgd_offset(mm, addr); | |
142 | pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); | |
143 | if (!pgd_present(*pgd)) | |
144 | return NULL; | |
0f3c2bcf | 145 | |
66b3923a | 146 | pud = pud_offset(pgd, addr); |
0f3c2bcf | 147 | if (pud_none(*pud)) |
66b3923a | 148 | return NULL; |
0f3c2bcf PA |
149 | /* swap or huge page */ |
150 | if (!pud_present(*pud) || pud_huge(*pud)) | |
66b3923a | 151 | return (pte_t *)pud; |
0f3c2bcf PA |
152 | /* table; check the next level */ |
153 | ||
66b3923a | 154 | pmd = pmd_offset(pud, addr); |
0f3c2bcf | 155 | if (pmd_none(*pmd)) |
66b3923a | 156 | return NULL; |
0f3c2bcf | 157 | if (!pmd_present(*pmd) || pmd_huge(*pmd)) |
66b3923a | 158 | return (pte_t *)pmd; |
0f3c2bcf | 159 | |
66b3923a DW |
160 | return NULL; |
161 | } | |
162 | ||
163 | pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, | |
164 | struct page *page, int writable) | |
165 | { | |
166 | size_t pagesize = huge_page_size(hstate_vma(vma)); | |
167 | ||
168 | if (pagesize == CONT_PTE_SIZE) { | |
169 | entry = pte_mkcont(entry); | |
170 | } else if (pagesize == CONT_PMD_SIZE) { | |
171 | entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); | |
172 | } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { | |
173 | pr_warn("%s: unrecognized huge page size 0x%lx\n", | |
174 | __func__, pagesize); | |
175 | } | |
176 | return entry; | |
177 | } | |
178 | ||
179 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
180 | unsigned long addr, pte_t *ptep) | |
181 | { | |
182 | pte_t pte; | |
183 | ||
184 | if (pte_cont(*ptep)) { | |
185 | int ncontig, i; | |
186 | size_t pgsize; | |
187 | pte_t *cpte; | |
188 | bool is_dirty = false; | |
189 | ||
190 | cpte = huge_pte_offset(mm, addr); | |
191 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
192 | /* save the 1st pte to return */ | |
193 | pte = ptep_get_and_clear(mm, addr, cpte); | |
0c2f0afe | 194 | for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { |
66b3923a DW |
195 | /* |
196 | * If HW_AFDBM is enabled, then the HW could | |
197 | * turn on the dirty bit for any of the page | |
198 | * in the set, so check them all. | |
199 | */ | |
200 | ++cpte; | |
201 | if (pte_dirty(ptep_get_and_clear(mm, addr, cpte))) | |
202 | is_dirty = true; | |
203 | } | |
204 | if (is_dirty) | |
205 | return pte_mkdirty(pte); | |
206 | else | |
207 | return pte; | |
208 | } else { | |
209 | return ptep_get_and_clear(mm, addr, ptep); | |
210 | } | |
211 | } | |
212 | ||
213 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, | |
214 | unsigned long addr, pte_t *ptep, | |
215 | pte_t pte, int dirty) | |
216 | { | |
217 | pte_t *cpte; | |
218 | ||
219 | if (pte_cont(pte)) { | |
220 | int ncontig, i, changed = 0; | |
221 | size_t pgsize = 0; | |
222 | unsigned long pfn = pte_pfn(pte); | |
223 | /* Select all bits except the pfn */ | |
224 | pgprot_t hugeprot = | |
225 | __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ | |
226 | pte_val(pte)); | |
227 | ||
228 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
229 | pfn = pte_pfn(*cpte); | |
230 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
231 | *cpte, &pgsize); | |
0c2f0afe | 232 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) { |
69d01234 | 233 | changed |= ptep_set_access_flags(vma, addr, cpte, |
66b3923a DW |
234 | pfn_pte(pfn, |
235 | hugeprot), | |
236 | dirty); | |
237 | pfn += pgsize >> PAGE_SHIFT; | |
238 | } | |
239 | return changed; | |
240 | } else { | |
241 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); | |
242 | } | |
243 | } | |
244 | ||
245 | void huge_ptep_set_wrprotect(struct mm_struct *mm, | |
246 | unsigned long addr, pte_t *ptep) | |
247 | { | |
248 | if (pte_cont(*ptep)) { | |
249 | int ncontig, i; | |
250 | pte_t *cpte; | |
251 | size_t pgsize = 0; | |
252 | ||
253 | cpte = huge_pte_offset(mm, addr); | |
254 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
0c2f0afe | 255 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) |
66b3923a DW |
256 | ptep_set_wrprotect(mm, addr, cpte); |
257 | } else { | |
258 | ptep_set_wrprotect(mm, addr, ptep); | |
259 | } | |
260 | } | |
261 | ||
262 | void huge_ptep_clear_flush(struct vm_area_struct *vma, | |
263 | unsigned long addr, pte_t *ptep) | |
264 | { | |
265 | if (pte_cont(*ptep)) { | |
266 | int ncontig, i; | |
267 | pte_t *cpte; | |
268 | size_t pgsize = 0; | |
269 | ||
270 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
271 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
272 | *cpte, &pgsize); | |
0c2f0afe | 273 | for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) |
66b3923a DW |
274 | ptep_clear_flush(vma, addr, cpte); |
275 | } else { | |
276 | ptep_clear_flush(vma, addr, ptep); | |
277 | } | |
278 | } | |
279 | ||
084bd298 SC |
280 | static __init int setup_hugepagesz(char *opt) |
281 | { | |
282 | unsigned long ps = memparse(opt, &opt); | |
66b3923a | 283 | |
084bd298 SC |
284 | if (ps == PMD_SIZE) { |
285 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
286 | } else if (ps == PUD_SIZE) { | |
287 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | |
ab2e1b89 WD |
288 | } else if (ps == (PAGE_SIZE * CONT_PTES)) { |
289 | hugetlb_add_hstate(CONT_PTE_SHIFT); | |
290 | } else if (ps == (PMD_SIZE * CONT_PMDS)) { | |
291 | hugetlb_add_hstate((PMD_SHIFT + CONT_PMD_SHIFT) - PAGE_SHIFT); | |
084bd298 | 292 | } else { |
d77e20ce | 293 | hugetlb_bad_size(); |
66b3923a | 294 | pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); |
084bd298 SC |
295 | return 0; |
296 | } | |
297 | return 1; | |
298 | } | |
299 | __setup("hugepagesz=", setup_hugepagesz); | |
ab2e1b89 WD |
300 | |
301 | #ifdef CONFIG_ARM64_64K_PAGES | |
302 | static __init int add_default_hugepagesz(void) | |
303 | { | |
304 | if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) | |
6ed0038d | 305 | hugetlb_add_hstate(CONT_PTE_SHIFT); |
ab2e1b89 WD |
306 | return 0; |
307 | } | |
308 | arch_initcall(add_default_hugepagesz); | |
309 | #endif |