]>
Commit | Line | Data |
---|---|---|
305b1523 JG |
1 | /* |
2 | * Copyright IBM Corp. 2011 | |
3 | * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> | |
4 | */ | |
6b70a920 | 5 | #include <linux/hugetlb.h> |
305b1523 | 6 | #include <linux/mm.h> |
638ad34a | 7 | #include <asm/cacheflush.h> |
cfb0b241 | 8 | #include <asm/facility.h> |
305b1523 | 9 | #include <asm/pgtable.h> |
6b70a920 | 10 | #include <asm/page.h> |
e6c7c630 | 11 | #include <asm/set_memory.h> |
6b70a920 | 12 | |
f7f8d7e5 HC |
13 | static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) |
14 | { | |
15 | asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0" | |
16 | : [addr] "+a" (addr) : [skey] "d" (skey)); | |
17 | return addr; | |
18 | } | |
19 | ||
127c1fef | 20 | void __storage_key_init_range(unsigned long start, unsigned long end) |
6b70a920 | 21 | { |
f7f8d7e5 | 22 | unsigned long boundary, size; |
6b70a920 | 23 | |
de3fa841 HC |
24 | if (!PAGE_DEFAULT_KEY) |
25 | return; | |
6b70a920 | 26 | while (start < end) { |
6b70a920 HC |
27 | if (MACHINE_HAS_EDAT1) { |
28 | /* set storage keys for a 1MB frame */ | |
6b70a920 HC |
29 | size = 1UL << 20; |
30 | boundary = (start + size) & ~(size - 1); | |
31 | if (boundary <= end) { | |
32 | do { | |
f7f8d7e5 | 33 | start = sske_frame(start, PAGE_DEFAULT_KEY); |
6b70a920 HC |
34 | } while (start < boundary); |
35 | continue; | |
36 | } | |
37 | } | |
38 | page_set_storage_key(start, PAGE_DEFAULT_KEY, 0); | |
39 | start += PAGE_SIZE; | |
40 | } | |
41 | } | |
305b1523 | 42 | |
37cd944c HC |
43 | #ifdef CONFIG_PROC_FS |
44 | atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; | |
45 | ||
46 | void arch_report_meminfo(struct seq_file *m) | |
47 | { | |
48 | seq_printf(m, "DirectMap4k: %8lu kB\n", | |
49 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2); | |
50 | seq_printf(m, "DirectMap1M: %8lu kB\n", | |
51 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10); | |
52 | seq_printf(m, "DirectMap2G: %8lu kB\n", | |
53 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21); | |
54 | } | |
55 | #endif /* CONFIG_PROC_FS */ | |
56 | ||
e8a97e42 HC |
57 | static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, |
58 | unsigned long dtt) | |
5b1ba9e3 | 59 | { |
e8a97e42 HC |
60 | unsigned long table, mask; |
61 | ||
62 | mask = 0; | |
63 | if (MACHINE_HAS_EDAT2) { | |
64 | switch (dtt) { | |
65 | case CRDTE_DTT_REGION3: | |
66 | mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1); | |
67 | break; | |
68 | case CRDTE_DTT_SEGMENT: | |
69 | mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); | |
70 | break; | |
71 | case CRDTE_DTT_PAGE: | |
72 | mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1); | |
73 | break; | |
74 | } | |
75 | table = (unsigned long)old & mask; | |
76 | crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); | |
77 | } else if (MACHINE_HAS_IDTE) { | |
78 | cspg(old, *old, new); | |
79 | } else { | |
80 | csp((unsigned int *)old + 1, *old, new); | |
81 | } | |
82 | } | |
83 | ||
e8a97e42 | 84 | static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, |
57d7f939 | 85 | unsigned long flags) |
e8a97e42 HC |
86 | { |
87 | pte_t *ptep, new; | |
88 | ||
89 | ptep = pte_offset(pmdp, addr); | |
90 | do { | |
57d7f939 MS |
91 | new = *ptep; |
92 | if (pte_none(new)) | |
e8a97e42 | 93 | return -EINVAL; |
57d7f939 MS |
94 | if (flags & SET_MEMORY_RO) |
95 | new = pte_wrprotect(new); | |
96 | else if (flags & SET_MEMORY_RW) | |
97 | new = pte_mkwrite(pte_mkdirty(new)); | |
1366def3 | 98 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
99 | pte_val(new) |= _PAGE_NOEXEC; |
100 | else if (flags & SET_MEMORY_X) | |
101 | pte_val(new) &= ~_PAGE_NOEXEC; | |
e8a97e42 HC |
102 | pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); |
103 | ptep++; | |
104 | addr += PAGE_SIZE; | |
105 | cond_resched(); | |
106 | } while (addr < end); | |
107 | return 0; | |
108 | } | |
109 | ||
110 | static int split_pmd_page(pmd_t *pmdp, unsigned long addr) | |
111 | { | |
112 | unsigned long pte_addr, prot; | |
113 | pte_t *pt_dir, *ptep; | |
114 | pmd_t new; | |
57d7f939 | 115 | int i, ro, nx; |
e8a97e42 HC |
116 | |
117 | pt_dir = vmem_pte_alloc(); | |
118 | if (!pt_dir) | |
119 | return -ENOMEM; | |
120 | pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; | |
121 | ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); | |
57d7f939 | 122 | nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); |
e8a97e42 | 123 | prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); |
57d7f939 MS |
124 | if (!nx) |
125 | prot &= ~_PAGE_NOEXEC; | |
e8a97e42 HC |
126 | ptep = pt_dir; |
127 | for (i = 0; i < PTRS_PER_PTE; i++) { | |
128 | pte_val(*ptep) = pte_addr | prot; | |
129 | pte_addr += PAGE_SIZE; | |
130 | ptep++; | |
131 | } | |
132 | pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY; | |
133 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); | |
37cd944c HC |
134 | update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE); |
135 | update_page_count(PG_DIRECT_MAP_1M, -1); | |
e8a97e42 HC |
136 | return 0; |
137 | } | |
138 | ||
57d7f939 MS |
139 | static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, |
140 | unsigned long flags) | |
e8a97e42 | 141 | { |
57d7f939 MS |
142 | pmd_t new = *pmdp; |
143 | ||
144 | if (flags & SET_MEMORY_RO) | |
145 | new = pmd_wrprotect(new); | |
146 | else if (flags & SET_MEMORY_RW) | |
147 | new = pmd_mkwrite(pmd_mkdirty(new)); | |
1366def3 | 148 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
149 | pmd_val(new) |= _SEGMENT_ENTRY_NOEXEC; |
150 | else if (flags & SET_MEMORY_X) | |
151 | pmd_val(new) &= ~_SEGMENT_ENTRY_NOEXEC; | |
e8a97e42 HC |
152 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); |
153 | } | |
154 | ||
155 | static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, | |
57d7f939 | 156 | unsigned long flags) |
e8a97e42 HC |
157 | { |
158 | unsigned long next; | |
5b1ba9e3 | 159 | pmd_t *pmdp; |
e8a97e42 | 160 | int rc = 0; |
5b1ba9e3 | 161 | |
5b1ba9e3 | 162 | pmdp = pmd_offset(pudp, addr); |
e8a97e42 HC |
163 | do { |
164 | if (pmd_none(*pmdp)) | |
165 | return -EINVAL; | |
166 | next = pmd_addr_end(addr, end); | |
167 | if (pmd_large(*pmdp)) { | |
168 | if (addr & ~PMD_MASK || addr + PMD_SIZE > next) { | |
169 | rc = split_pmd_page(pmdp, addr); | |
170 | if (rc) | |
171 | return rc; | |
172 | continue; | |
173 | } | |
57d7f939 | 174 | modify_pmd_page(pmdp, addr, flags); |
e8a97e42 | 175 | } else { |
57d7f939 | 176 | rc = walk_pte_level(pmdp, addr, next, flags); |
e8a97e42 HC |
177 | if (rc) |
178 | return rc; | |
179 | } | |
180 | pmdp++; | |
181 | addr = next; | |
182 | cond_resched(); | |
183 | } while (addr < end); | |
184 | return rc; | |
5b1ba9e3 HC |
185 | } |
186 | ||
e8a97e42 | 187 | static int split_pud_page(pud_t *pudp, unsigned long addr) |
305b1523 | 188 | { |
e8a97e42 HC |
189 | unsigned long pmd_addr, prot; |
190 | pmd_t *pm_dir, *pmdp; | |
191 | pud_t new; | |
57d7f939 | 192 | int i, ro, nx; |
305b1523 | 193 | |
e8a97e42 HC |
194 | pm_dir = vmem_pmd_alloc(); |
195 | if (!pm_dir) | |
196 | return -ENOMEM; | |
197 | pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; | |
198 | ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); | |
57d7f939 | 199 | nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); |
e8a97e42 | 200 | prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); |
57d7f939 MS |
201 | if (!nx) |
202 | prot &= ~_SEGMENT_ENTRY_NOEXEC; | |
e8a97e42 HC |
203 | pmdp = pm_dir; |
204 | for (i = 0; i < PTRS_PER_PMD; i++) { | |
205 | pmd_val(*pmdp) = pmd_addr | prot; | |
206 | pmd_addr += PMD_SIZE; | |
207 | pmdp++; | |
305b1523 | 208 | } |
e8a97e42 HC |
209 | pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY; |
210 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); | |
37cd944c HC |
211 | update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD); |
212 | update_page_count(PG_DIRECT_MAP_2G, -1); | |
e8a97e42 HC |
213 | return 0; |
214 | } | |
215 | ||
57d7f939 MS |
216 | static void modify_pud_page(pud_t *pudp, unsigned long addr, |
217 | unsigned long flags) | |
e8a97e42 | 218 | { |
57d7f939 MS |
219 | pud_t new = *pudp; |
220 | ||
221 | if (flags & SET_MEMORY_RO) | |
222 | new = pud_wrprotect(new); | |
223 | else if (flags & SET_MEMORY_RW) | |
224 | new = pud_mkwrite(pud_mkdirty(new)); | |
1366def3 | 225 | if (flags & SET_MEMORY_NX) |
57d7f939 MS |
226 | pud_val(new) |= _REGION_ENTRY_NOEXEC; |
227 | else if (flags & SET_MEMORY_X) | |
228 | pud_val(new) &= ~_REGION_ENTRY_NOEXEC; | |
e8a97e42 HC |
229 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); |
230 | } | |
231 | ||
1aea9b3f | 232 | static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, |
57d7f939 | 233 | unsigned long flags) |
e8a97e42 HC |
234 | { |
235 | unsigned long next; | |
236 | pud_t *pudp; | |
237 | int rc = 0; | |
238 | ||
1aea9b3f | 239 | pudp = pud_offset(p4d, addr); |
e8a97e42 HC |
240 | do { |
241 | if (pud_none(*pudp)) | |
242 | return -EINVAL; | |
243 | next = pud_addr_end(addr, end); | |
244 | if (pud_large(*pudp)) { | |
245 | if (addr & ~PUD_MASK || addr + PUD_SIZE > next) { | |
246 | rc = split_pud_page(pudp, addr); | |
247 | if (rc) | |
248 | break; | |
249 | continue; | |
250 | } | |
57d7f939 | 251 | modify_pud_page(pudp, addr, flags); |
e8a97e42 | 252 | } else { |
57d7f939 | 253 | rc = walk_pmd_level(pudp, addr, next, flags); |
e8a97e42 HC |
254 | } |
255 | pudp++; | |
256 | addr = next; | |
257 | cond_resched(); | |
258 | } while (addr < end && !rc); | |
259 | return rc; | |
260 | } | |
261 | ||
1aea9b3f MS |
262 | static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end, |
263 | unsigned long flags) | |
264 | { | |
265 | unsigned long next; | |
266 | p4d_t *p4dp; | |
267 | int rc = 0; | |
268 | ||
269 | p4dp = p4d_offset(pgd, addr); | |
270 | do { | |
271 | if (p4d_none(*p4dp)) | |
272 | return -EINVAL; | |
273 | next = p4d_addr_end(addr, end); | |
274 | rc = walk_pud_level(p4dp, addr, next, flags); | |
275 | p4dp++; | |
276 | addr = next; | |
277 | cond_resched(); | |
278 | } while (addr < end && !rc); | |
279 | return rc; | |
280 | } | |
281 | ||
e8a97e42 HC |
282 | static DEFINE_MUTEX(cpa_mutex); |
283 | ||
284 | static int change_page_attr(unsigned long addr, unsigned long end, | |
57d7f939 | 285 | unsigned long flags) |
e8a97e42 HC |
286 | { |
287 | unsigned long next; | |
288 | int rc = -EINVAL; | |
289 | pgd_t *pgdp; | |
290 | ||
4d81aaa5 HC |
291 | if (addr == end) |
292 | return 0; | |
e8a97e42 HC |
293 | if (end >= MODULES_END) |
294 | return -EINVAL; | |
295 | mutex_lock(&cpa_mutex); | |
296 | pgdp = pgd_offset_k(addr); | |
297 | do { | |
298 | if (pgd_none(*pgdp)) | |
299 | break; | |
300 | next = pgd_addr_end(addr, end); | |
1aea9b3f | 301 | rc = walk_p4d_level(pgdp, addr, next, flags); |
e8a97e42 HC |
302 | if (rc) |
303 | break; | |
304 | cond_resched(); | |
305 | } while (pgdp++, addr = next, addr < end && !rc); | |
306 | mutex_unlock(&cpa_mutex); | |
307 | return rc; | |
305b1523 JG |
308 | } |
309 | ||
57d7f939 | 310 | int __set_memory(unsigned long addr, int numpages, unsigned long flags) |
305b1523 | 311 | { |
1366def3 HC |
312 | if (!MACHINE_HAS_NX) |
313 | flags &= ~(SET_MEMORY_NX | SET_MEMORY_X); | |
314 | if (!flags) | |
315 | return 0; | |
e8a97e42 | 316 | addr &= PAGE_MASK; |
57d7f939 | 317 | return change_page_attr(addr, addr + numpages * PAGE_SIZE, flags); |
448694a1 | 318 | } |
0a4ccc99 HC |
319 | |
320 | #ifdef CONFIG_DEBUG_PAGEALLOC | |
cfb0b241 HC |
321 | |
322 | static void ipte_range(pte_t *pte, unsigned long address, int nr) | |
323 | { | |
324 | int i; | |
325 | ||
5a79859a | 326 | if (test_facility(13)) { |
34eeaf37 | 327 | __ptep_ipte_range(address, nr - 1, pte, IPTE_GLOBAL); |
cfb0b241 HC |
328 | return; |
329 | } | |
330 | for (i = 0; i < nr; i++) { | |
28c807e5 | 331 | __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL); |
cfb0b241 HC |
332 | address += PAGE_SIZE; |
333 | pte++; | |
334 | } | |
335 | } | |
336 | ||
031bc574 | 337 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
0a4ccc99 HC |
338 | { |
339 | unsigned long address; | |
cfb0b241 | 340 | int nr, i, j; |
0a4ccc99 | 341 | pgd_t *pgd; |
1aea9b3f | 342 | p4d_t *p4d; |
0a4ccc99 HC |
343 | pud_t *pud; |
344 | pmd_t *pmd; | |
345 | pte_t *pte; | |
0a4ccc99 | 346 | |
cfb0b241 | 347 | for (i = 0; i < numpages;) { |
0a4ccc99 HC |
348 | address = page_to_phys(page + i); |
349 | pgd = pgd_offset_k(address); | |
1aea9b3f MS |
350 | p4d = p4d_offset(pgd, address); |
351 | pud = pud_offset(p4d, address); | |
0a4ccc99 HC |
352 | pmd = pmd_offset(pud, address); |
353 | pte = pte_offset_kernel(pmd, address); | |
cfb0b241 HC |
354 | nr = (unsigned long)pte >> ilog2(sizeof(long)); |
355 | nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); | |
356 | nr = min(numpages - i, nr); | |
357 | if (enable) { | |
358 | for (j = 0; j < nr; j++) { | |
57d7f939 | 359 | pte_val(*pte) &= ~_PAGE_INVALID; |
cfb0b241 HC |
360 | address += PAGE_SIZE; |
361 | pte++; | |
362 | } | |
363 | } else { | |
364 | ipte_range(pte, address, nr); | |
0a4ccc99 | 365 | } |
cfb0b241 | 366 | i += nr; |
0a4ccc99 HC |
367 | } |
368 | } | |
369 | ||
370 | #ifdef CONFIG_HIBERNATION | |
371 | bool kernel_page_present(struct page *page) | |
372 | { | |
373 | unsigned long addr; | |
374 | int cc; | |
375 | ||
376 | addr = page_to_phys(page); | |
377 | asm volatile( | |
378 | " lra %1,0(%1)\n" | |
379 | " ipm %0\n" | |
380 | " srl %0,28" | |
381 | : "=d" (cc), "+a" (addr) : : "cc"); | |
382 | return cc == 0; | |
383 | } | |
384 | #endif /* CONFIG_HIBERNATION */ | |
385 | ||
386 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |