]>
Commit | Line | Data |
---|---|---|
dcfdae04 CM |
1 | /* |
2 | * arch/arm/include/asm/pgtable-3level.h | |
3 | * | |
4 | * Copyright (C) 2011 ARM Ltd. | |
5 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program; if not, write to the Free Software | |
18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
19 | */ | |
20 | #ifndef _ASM_PGTABLE_3LEVEL_H | |
21 | #define _ASM_PGTABLE_3LEVEL_H | |
22 | ||
23 | /* | |
24 | * With LPAE, there are 3 levels of page tables. Each level has 512 entries of | |
25 | * 8 bytes each, occupying a 4K page. The first level table covers a range of | |
26 | * 512GB, each entry representing 1GB. Since we are limited to 4GB input | |
27 | * address range, only 4 entries in the PGD are used. | |
28 | * | |
29 | * There are enough spare bits in a page table entry for the kernel specific | |
30 | * state. | |
31 | */ | |
32 | #define PTRS_PER_PTE 512 | |
33 | #define PTRS_PER_PMD 512 | |
34 | #define PTRS_PER_PGD 4 | |
35 | ||
e38a5175 | 36 | #define PTE_HWTABLE_PTRS (0) |
dcfdae04 CM |
37 | #define PTE_HWTABLE_OFF (0) |
38 | #define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64)) | |
39 | ||
40 | /* | |
41 | * PGDIR_SHIFT determines the size a top-level page table entry can map. | |
42 | */ | |
43 | #define PGDIR_SHIFT 30 | |
44 | ||
45 | /* | |
46 | * PMD_SHIFT determines the size a middle-level page table entry can map. | |
47 | */ | |
48 | #define PMD_SHIFT 21 | |
49 | ||
50 | #define PMD_SIZE (1UL << PMD_SHIFT) | |
926edcc7 | 51 | #define PMD_MASK (~((1 << PMD_SHIFT) - 1)) |
dcfdae04 | 52 | #define PGDIR_SIZE (1UL << PGDIR_SHIFT) |
926edcc7 | 53 | #define PGDIR_MASK (~((1 << PGDIR_SHIFT) - 1)) |
dcfdae04 CM |
54 | |
55 | /* | |
56 | * section address mask and size definitions. | |
57 | */ | |
58 | #define SECTION_SHIFT 21 | |
59 | #define SECTION_SIZE (1UL << SECTION_SHIFT) | |
926edcc7 | 60 | #define SECTION_MASK (~((1 << SECTION_SHIFT) - 1)) |
dcfdae04 CM |
61 | |
62 | #define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE) | |
63 | ||
1355e2a6 CM |
64 | /* |
65 | * Hugetlb definitions. | |
66 | */ | |
67 | #define HPAGE_SHIFT PMD_SHIFT | |
68 | #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) | |
69 | #define HPAGE_MASK (~(HPAGE_SIZE - 1)) | |
70 | #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) | |
71 | ||
dcfdae04 CM |
72 | /* |
73 | * "Linux" PTE definitions for LPAE. | |
74 | * | |
75 | * These bits overlap with the hardware bits but the naming is preserved for | |
76 | * consistency with the classic page table format. | |
77 | */ | |
dbf62d50 WD |
78 | #define L_PTE_VALID (_AT(pteval_t, 1) << 0) /* Valid */ |
79 | #define L_PTE_PRESENT (_AT(pteval_t, 3) << 0) /* Present */ | |
dcfdae04 | 80 | #define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */ |
dcfdae04 CM |
81 | #define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */ |
82 | #define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */ | |
83 | #define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */ | |
ded94779 SC |
84 | #define L_PTE_DIRTY (_AT(pteval_t, 1) << 55) |
85 | #define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56) | |
26ffd0d4 | 86 | #define L_PTE_NONE (_AT(pteval_t, 1) << 57) /* PROT_NONE */ |
ded94779 | 87 | #define L_PTE_RDONLY (_AT(pteval_t, 1) << 58) /* READ ONLY */ |
dcfdae04 | 88 | |
ded94779 SC |
89 | #define L_PMD_SECT_VALID (_AT(pmdval_t, 1) << 0) |
90 | #define L_PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55) | |
ded94779 SC |
91 | #define L_PMD_SECT_NONE (_AT(pmdval_t, 1) << 57) |
92 | #define L_PMD_SECT_RDONLY (_AT(pteval_t, 1) << 58) | |
8d962507 | 93 | |
dcfdae04 CM |
94 | /* |
95 | * To be used in assembly code with the upper page attributes. | |
96 | */ | |
97 | #define L_PTE_XN_HIGH (1 << (54 - 32)) | |
98 | #define L_PTE_DIRTY_HIGH (1 << (55 - 32)) | |
99 | ||
100 | /* | |
101 | * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers). | |
102 | */ | |
103 | #define L_PTE_MT_UNCACHED (_AT(pteval_t, 0) << 2) /* strongly ordered */ | |
104 | #define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 1) << 2) /* normal non-cacheable */ | |
105 | #define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 2) << 2) /* normal inner write-through */ | |
106 | #define L_PTE_MT_WRITEBACK (_AT(pteval_t, 3) << 2) /* normal inner write-back */ | |
107 | #define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 7) << 2) /* normal inner write-alloc */ | |
108 | #define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 4) << 2) /* device */ | |
109 | #define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 4) << 2) /* device */ | |
110 | #define L_PTE_MT_DEV_WC (_AT(pteval_t, 1) << 2) /* normal non-cacheable */ | |
111 | #define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */ | |
112 | #define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2) | |
113 | ||
da028779 CM |
114 | /* |
115 | * Software PGD flags. | |
116 | */ | |
117 | #define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */ | |
118 | ||
cc577c26 CD |
119 | /* |
120 | * 2nd stage PTE definitions for LPAE. | |
121 | */ | |
4d9c5b89 CD |
122 | #define L_PTE_S2_MT_UNCACHED (_AT(pteval_t, 0x0) << 2) /* strongly ordered */ |
123 | #define L_PTE_S2_MT_WRITETHROUGH (_AT(pteval_t, 0xa) << 2) /* normal inner write-through */ | |
124 | #define L_PTE_S2_MT_WRITEBACK (_AT(pteval_t, 0xf) << 2) /* normal inner write-back */ | |
125 | #define L_PTE_S2_MT_DEV_SHARED (_AT(pteval_t, 0x1) << 2) /* device */ | |
126 | #define L_PTE_S2_MT_MASK (_AT(pteval_t, 0xf) << 2) | |
cc577c26 | 127 | |
4d9c5b89 CD |
128 | #define L_PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[1] */ |
129 | #define L_PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */ | |
130 | ||
c6473555 | 131 | #define L_PMD_S2_RDONLY (_AT(pmdval_t, 1) << 6) /* HAP[1] */ |
4d9c5b89 | 132 | #define L_PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */ |
ad361f09 | 133 | |
cc577c26 CD |
134 | /* |
135 | * Hyp-mode PL2 PTE definitions for LPAE. | |
136 | */ | |
137 | #define L_PTE_HYP L_PTE_USER | |
138 | ||
da028779 CM |
139 | #ifndef __ASSEMBLY__ |
140 | ||
141 | #define pud_none(pud) (!pud_val(pud)) | |
142 | #define pud_bad(pud) (!(pud_val(pud) & 2)) | |
143 | #define pud_present(pud) (pud_val(pud)) | |
cc577c26 CD |
144 | #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ |
145 | PMD_TYPE_TABLE) | |
146 | #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ | |
147 | PMD_TYPE_SECT) | |
1fd15b87 | 148 | #define pmd_large(pmd) pmd_sect(pmd) |
da028779 CM |
149 | |
150 | #define pud_clear(pudp) \ | |
151 | do { \ | |
152 | *pudp = __pud(0); \ | |
153 | clean_pmd_entry(pudp); \ | |
154 | } while (0) | |
155 | ||
156 | #define set_pud(pudp, pud) \ | |
157 | do { \ | |
158 | *pudp = pud; \ | |
159 | flush_pmd_entry(pudp); \ | |
160 | } while (0) | |
161 | ||
162 | static inline pmd_t *pud_page_vaddr(pud_t pud) | |
163 | { | |
164 | return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); | |
165 | } | |
166 | ||
167 | /* Find an entry in the second-level page table.. */ | |
168 | #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) | |
169 | static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) | |
170 | { | |
171 | return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); | |
172 | } | |
173 | ||
174 | #define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) | |
175 | ||
176 | #define copy_pmd(pmdpd,pmdps) \ | |
177 | do { \ | |
178 | *pmdpd = *pmdps; \ | |
179 | flush_pmd_entry(pmdpd); \ | |
180 | } while (0) | |
181 | ||
182 | #define pmd_clear(pmdp) \ | |
183 | do { \ | |
184 | *pmdp = __pmd(0); \ | |
185 | clean_pmd_entry(pmdp); \ | |
186 | } while (0) | |
187 | ||
dde1b651 SC |
188 | /* |
189 | * For 3 levels of paging the PTE_EXT_NG bit will be set for user address ptes | |
190 | * that are written to a page table but not for ptes created with mk_pte. | |
191 | * | |
192 | * In hugetlb_no_page, a new huge pte (new_pte) is generated and passed to | |
193 | * hugetlb_cow, where it is compared with an entry in a page table. | |
194 | * This comparison test fails erroneously leading ultimately to a memory leak. | |
195 | * | |
196 | * To correct this behaviour, we mask off PTE_EXT_NG for any pte that is | |
197 | * present before running the comparison. | |
198 | */ | |
199 | #define __HAVE_ARCH_PTE_SAME | |
200 | #define pte_same(pte_a,pte_b) ((pte_present(pte_a) ? pte_val(pte_a) & ~PTE_EXT_NG \ | |
201 | : pte_val(pte_a)) \ | |
202 | == (pte_present(pte_b) ? pte_val(pte_b) & ~PTE_EXT_NG \ | |
203 | : pte_val(pte_b))) | |
204 | ||
da028779 CM |
205 | #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext))) |
206 | ||
1355e2a6 CM |
207 | #define pte_huge(pte) (pte_val(pte) && !(pte_val(pte) & PTE_TABLE_BIT)) |
208 | #define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT)) | |
209 | ||
f2950706 SC |
210 | #define pmd_isset(pmd, val) ((u32)(val) == (val) ? pmd_val(pmd) & (val) \ |
211 | : !!(pmd_val(pmd) & (val))) | |
212 | #define pmd_isclear(pmd, val) (!(pmd_val(pmd) & (val))) | |
213 | ||
62453188 | 214 | #define pmd_present(pmd) (pmd_isset((pmd), L_PMD_SECT_VALID)) |
f2950706 | 215 | #define pmd_young(pmd) (pmd_isset((pmd), PMD_SECT_AF)) |
bd951303 SC |
216 | #define pte_special(pte) (pte_isset((pte), L_PTE_SPECIAL)) |
217 | static inline pte_t pte_mkspecial(pte_t pte) | |
218 | { | |
219 | pte_val(pte) |= L_PTE_SPECIAL; | |
220 | return pte; | |
221 | } | |
222 | #define __HAVE_ARCH_PTE_SPECIAL | |
8d962507 CM |
223 | |
224 | #define __HAVE_ARCH_PMD_WRITE | |
ded94779 SC |
225 | #define pmd_write(pmd) (pmd_isclear((pmd), L_PMD_SECT_RDONLY)) |
226 | #define pmd_dirty(pmd) (pmd_isset((pmd), L_PMD_SECT_DIRTY)) | |
b8cd51af SC |
227 | #define pud_page(pud) pmd_page(__pmd(pud_val(pud))) |
228 | #define pud_write(pud) pmd_write(__pmd(pud_val(pud))) | |
8d962507 | 229 | |
a3a9ea65 SC |
230 | #define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd)) |
231 | #define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) | |
232 | ||
8d962507 | 233 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
f2950706 | 234 | #define pmd_trans_huge(pmd) (pmd_val(pmd) && !pmd_table(pmd)) |
8d962507 CM |
235 | #endif |
236 | ||
237 | #define PMD_BIT_FUNC(fn,op) \ | |
238 | static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } | |
239 | ||
ded94779 | 240 | PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY); |
8d962507 | 241 | PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF); |
ded94779 SC |
242 | PMD_BIT_FUNC(mkwrite, &= ~L_PMD_SECT_RDONLY); |
243 | PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY); | |
44842045 | 244 | PMD_BIT_FUNC(mkclean, &= ~L_PMD_SECT_DIRTY); |
8d962507 CM |
245 | PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF); |
246 | ||
247 | #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) | |
248 | ||
249 | #define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT) | |
250 | #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) | |
251 | #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) | |
252 | ||
56530f5d | 253 | /* represent a notpresent pmd by faulting entry, this is used by pmdp_invalidate */ |
4d942466 MG |
254 | static inline pmd_t pmd_mknotpresent(pmd_t pmd) |
255 | { | |
56530f5d | 256 | return __pmd(pmd_val(pmd) & ~L_PMD_SECT_VALID); |
4d942466 | 257 | } |
8d962507 CM |
258 | |
259 | static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) | |
260 | { | |
ded94779 SC |
261 | const pmdval_t mask = PMD_SECT_USER | PMD_SECT_XN | L_PMD_SECT_RDONLY | |
262 | L_PMD_SECT_VALID | L_PMD_SECT_NONE; | |
8d962507 CM |
263 | pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask); |
264 | return pmd; | |
265 | } | |
266 | ||
267 | static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, | |
268 | pmd_t *pmdp, pmd_t pmd) | |
269 | { | |
270 | BUG_ON(addr >= TASK_SIZE); | |
271 | ||
272 | /* create a faulting entry if PROT_NONE protected */ | |
ded94779 SC |
273 | if (pmd_val(pmd) & L_PMD_SECT_NONE) |
274 | pmd_val(pmd) &= ~L_PMD_SECT_VALID; | |
275 | ||
276 | if (pmd_write(pmd) && pmd_dirty(pmd)) | |
277 | pmd_val(pmd) &= ~PMD_SECT_AP2; | |
278 | else | |
279 | pmd_val(pmd) |= PMD_SECT_AP2; | |
8d962507 CM |
280 | |
281 | *pmdp = __pmd(pmd_val(pmd) | PMD_SECT_nG); | |
282 | flush_pmd_entry(pmdp); | |
283 | } | |
284 | ||
da028779 CM |
285 | #endif /* __ASSEMBLY__ */ |
286 | ||
dcfdae04 | 287 | #endif /* _ASM_PGTABLE_3LEVEL_H */ |