]>
Commit | Line | Data |
---|---|---|
926e5392 AV |
1 | /* |
2 | * Debug helper to dump the current kernel pagetables of the system | |
3 | * so that we can see what the various memory ranges are set to. | |
4 | * | |
5 | * (C) Copyright 2008 Intel Corporation | |
6 | * | |
7 | * Author: Arjan van de Ven <arjan@linux.intel.com> | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; version 2 | |
12 | * of the License. | |
13 | */ | |
14 | ||
fe770bf0 | 15 | #include <linux/debugfs.h> |
04b67022 | 16 | #include <linux/kasan.h> |
fe770bf0 | 17 | #include <linux/mm.h> |
84e629b6 | 18 | #include <linux/init.h> |
146fbb76 | 19 | #include <linux/sched.h> |
926e5392 | 20 | #include <linux/seq_file.h> |
d6ef1f19 | 21 | #include <linux/highmem.h> |
c200dac7 | 22 | #include <linux/pci.h> |
926e5392 | 23 | |
c200dac7 | 24 | #include <asm/e820/types.h> |
926e5392 AV |
25 | #include <asm/pgtable.h> |
26 | ||
27 | /* | |
28 | * The dumper groups pagetable entries of the same type into one, and for | |
29 | * that it needs to keep some state when walking, and flush this state | |
30 | * when a "break" in the continuity is found. | |
31 | */ | |
32 | struct pg_state { | |
33 | int level; | |
34 | pgprot_t current_prot; | |
672c0ae0 | 35 | pgprotval_t effective_prot; |
926e5392 AV |
36 | unsigned long start_address; |
37 | unsigned long current_address; | |
fe770bf0 | 38 | const struct addr_marker *marker; |
3891a04a | 39 | unsigned long lines; |
ef6bea6d | 40 | bool to_dmesg; |
e1a58320 SS |
41 | bool check_wx; |
42 | unsigned long wx_pages; | |
926e5392 AV |
43 | }; |
44 | ||
fe770bf0 PA |
45 | struct addr_marker { |
46 | unsigned long start_address; | |
47 | const char *name; | |
3891a04a | 48 | unsigned long max_lines; |
fe770bf0 PA |
49 | }; |
50 | ||
146122e2 TG |
51 | /* Address space markers hints */ |
52 | ||
53 | #ifdef CONFIG_X86_64 | |
54 | ||
92851e2f AS |
55 | enum address_markers_idx { |
56 | USER_SPACE_NR = 0, | |
92851e2f | 57 | KERNEL_SPACE_NR, |
254eb550 | 58 | #ifdef CONFIG_MODIFY_LDT_SYSCALL |
f55f0501 AL |
59 | LDT_NR, |
60 | #endif | |
254eb550 | 61 | LOW_KERNEL_NR, |
92851e2f AS |
62 | VMALLOC_START_NR, |
63 | VMEMMAP_START_NR, | |
025205f8 AR |
64 | #ifdef CONFIG_KASAN |
65 | KASAN_SHADOW_START_NR, | |
66 | KASAN_SHADOW_END_NR, | |
f55f0501 | 67 | #endif |
f2078904 | 68 | CPU_ENTRY_AREA_NR, |
146122e2 | 69 | #ifdef CONFIG_X86_ESPFIX64 |
3891a04a | 70 | ESPFIX_START_NR, |
146122e2 TG |
71 | #endif |
72 | #ifdef CONFIG_EFI | |
73 | EFI_END_NR, | |
74 | #endif | |
92851e2f AS |
75 | HIGH_KERNEL_NR, |
76 | MODULES_VADDR_NR, | |
77 | MODULES_END_NR, | |
146122e2 TG |
78 | FIXADDR_START_NR, |
79 | END_OF_SPACE_NR, | |
80 | }; | |
81 | ||
82 | static struct addr_marker address_markers[] = { | |
83 | [USER_SPACE_NR] = { 0, "User Space" }, | |
84 | [KERNEL_SPACE_NR] = { (1UL << 63), "Kernel Space" }, | |
85 | [LOW_KERNEL_NR] = { 0UL, "Low Kernel Mapping" }, | |
86 | [VMALLOC_START_NR] = { 0UL, "vmalloc() Area" }, | |
87 | [VMEMMAP_START_NR] = { 0UL, "Vmemmap" }, | |
88 | #ifdef CONFIG_KASAN | |
09e61a77 KS |
89 | /* |
90 | * These fields get initialized with the (dynamic) | |
91 | * KASAN_SHADOW_{START,END} values in pt_dump_init(). | |
92 | */ | |
93 | [KASAN_SHADOW_START_NR] = { 0UL, "KASAN shadow" }, | |
94 | [KASAN_SHADOW_END_NR] = { 0UL, "KASAN shadow end" }, | |
f55f0501 AL |
95 | #endif |
96 | #ifdef CONFIG_MODIFY_LDT_SYSCALL | |
5c7919bb | 97 | [LDT_NR] = { 0UL, "LDT remap" }, |
146122e2 | 98 | #endif |
92a0f81d | 99 | [CPU_ENTRY_AREA_NR] = { CPU_ENTRY_AREA_BASE,"CPU entry Area" }, |
146122e2 TG |
100 | #ifdef CONFIG_X86_ESPFIX64 |
101 | [ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 }, | |
102 | #endif | |
103 | #ifdef CONFIG_EFI | |
104 | [EFI_END_NR] = { EFI_VA_END, "EFI Runtime Services" }, | |
105 | #endif | |
106 | [HIGH_KERNEL_NR] = { __START_KERNEL_map, "High Kernel Mapping" }, | |
107 | [MODULES_VADDR_NR] = { MODULES_VADDR, "Modules" }, | |
108 | [MODULES_END_NR] = { MODULES_END, "End Modules" }, | |
109 | [FIXADDR_START_NR] = { FIXADDR_START, "Fixmap Area" }, | |
110 | [END_OF_SPACE_NR] = { -1, NULL } | |
111 | }; | |
112 | ||
4e8537e4 JR |
113 | #define INIT_PGD ((pgd_t *) &init_top_pgt) |
114 | ||
146122e2 TG |
115 | #else /* CONFIG_X86_64 */ |
116 | ||
117 | enum address_markers_idx { | |
118 | USER_SPACE_NR = 0, | |
92851e2f AS |
119 | KERNEL_SPACE_NR, |
120 | VMALLOC_START_NR, | |
121 | VMALLOC_END_NR, | |
146122e2 | 122 | #ifdef CONFIG_HIGHMEM |
92851e2f | 123 | PKMAP_BASE_NR, |
f3e48e54 JR |
124 | #endif |
125 | #ifdef CONFIG_MODIFY_LDT_SYSCALL | |
126 | LDT_NR, | |
92851e2f | 127 | #endif |
92a0f81d | 128 | CPU_ENTRY_AREA_NR, |
146122e2 TG |
129 | FIXADDR_START_NR, |
130 | END_OF_SPACE_NR, | |
92851e2f AS |
131 | }; |
132 | ||
fe770bf0 | 133 | static struct addr_marker address_markers[] = { |
146122e2 TG |
134 | [USER_SPACE_NR] = { 0, "User Space" }, |
135 | [KERNEL_SPACE_NR] = { PAGE_OFFSET, "Kernel Mapping" }, | |
136 | [VMALLOC_START_NR] = { 0UL, "vmalloc() Area" }, | |
137 | [VMALLOC_END_NR] = { 0UL, "vmalloc() End" }, | |
138 | #ifdef CONFIG_HIGHMEM | |
139 | [PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" }, | |
f3e48e54 JR |
140 | #endif |
141 | #ifdef CONFIG_MODIFY_LDT_SYSCALL | |
142 | [LDT_NR] = { 0UL, "LDT remap" }, | |
fe770bf0 | 143 | #endif |
92a0f81d | 144 | [CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" }, |
146122e2 TG |
145 | [FIXADDR_START_NR] = { 0UL, "Fixmap area" }, |
146 | [END_OF_SPACE_NR] = { -1, NULL } | |
fe770bf0 | 147 | }; |
926e5392 | 148 | |
4e8537e4 JR |
149 | #define INIT_PGD (swapper_pg_dir) |
150 | ||
146122e2 TG |
151 | #endif /* !CONFIG_X86_64 */ |
152 | ||
fe770bf0 PA |
153 | /* Multipliers for offsets within the PTEs */ |
154 | #define PTE_LEVEL_MULT (PAGE_SIZE) | |
155 | #define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT) | |
156 | #define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT) | |
fdd3d8ce | 157 | #define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT) |
84bbabc3 | 158 | #define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT) |
926e5392 | 159 | |
ef6bea6d BP |
160 | #define pt_dump_seq_printf(m, to_dmesg, fmt, args...) \ |
161 | ({ \ | |
162 | if (to_dmesg) \ | |
163 | printk(KERN_INFO fmt, ##args); \ | |
164 | else \ | |
165 | if (m) \ | |
166 | seq_printf(m, fmt, ##args); \ | |
167 | }) | |
168 | ||
169 | #define pt_dump_cont_printf(m, to_dmesg, fmt, args...) \ | |
170 | ({ \ | |
171 | if (to_dmesg) \ | |
172 | printk(KERN_CONT fmt, ##args); \ | |
173 | else \ | |
174 | if (m) \ | |
175 | seq_printf(m, fmt, ##args); \ | |
176 | }) | |
177 | ||
926e5392 AV |
178 | /* |
179 | * Print a readable form of a pgprot_t to the seq_file | |
180 | */ | |
ef6bea6d | 181 | static void printk_prot(struct seq_file *m, pgprot_t prot, int level, bool dmsg) |
926e5392 | 182 | { |
fe770bf0 PA |
183 | pgprotval_t pr = pgprot_val(prot); |
184 | static const char * const level_name[] = | |
45dcd209 | 185 | { "cr3", "pgd", "p4d", "pud", "pmd", "pte" }; |
fe770bf0 | 186 | |
c0534494 | 187 | if (!(pr & _PAGE_PRESENT)) { |
fe770bf0 | 188 | /* Not present */ |
f439c429 | 189 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 PA |
190 | } else { |
191 | if (pr & _PAGE_USER) | |
ef6bea6d | 192 | pt_dump_cont_printf(m, dmsg, "USR "); |
926e5392 | 193 | else |
ef6bea6d | 194 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 195 | if (pr & _PAGE_RW) |
ef6bea6d | 196 | pt_dump_cont_printf(m, dmsg, "RW "); |
fe770bf0 | 197 | else |
ef6bea6d | 198 | pt_dump_cont_printf(m, dmsg, "ro "); |
fe770bf0 | 199 | if (pr & _PAGE_PWT) |
ef6bea6d | 200 | pt_dump_cont_printf(m, dmsg, "PWT "); |
fe770bf0 | 201 | else |
ef6bea6d | 202 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 203 | if (pr & _PAGE_PCD) |
ef6bea6d | 204 | pt_dump_cont_printf(m, dmsg, "PCD "); |
926e5392 | 205 | else |
ef6bea6d | 206 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 207 | |
f439c429 | 208 | /* Bit 7 has a different meaning on level 3 vs 4 */ |
45dcd209 | 209 | if (level <= 4 && pr & _PAGE_PSE) |
f439c429 JG |
210 | pt_dump_cont_printf(m, dmsg, "PSE "); |
211 | else | |
212 | pt_dump_cont_printf(m, dmsg, " "); | |
45dcd209 KS |
213 | if ((level == 5 && pr & _PAGE_PAT) || |
214 | ((level == 4 || level == 3) && pr & _PAGE_PAT_LARGE)) | |
da25e628 | 215 | pt_dump_cont_printf(m, dmsg, "PAT "); |
f439c429 JG |
216 | else |
217 | pt_dump_cont_printf(m, dmsg, " "); | |
fe770bf0 | 218 | if (pr & _PAGE_GLOBAL) |
ef6bea6d | 219 | pt_dump_cont_printf(m, dmsg, "GLB "); |
fe770bf0 | 220 | else |
ef6bea6d | 221 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 222 | if (pr & _PAGE_NX) |
ef6bea6d | 223 | pt_dump_cont_printf(m, dmsg, "NX "); |
fe770bf0 | 224 | else |
ef6bea6d | 225 | pt_dump_cont_printf(m, dmsg, "x "); |
926e5392 | 226 | } |
ef6bea6d | 227 | pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]); |
926e5392 AV |
228 | } |
229 | ||
230 | /* | |
fe770bf0 | 231 | * On 64 bits, sign-extend the 48 bit address to 64 bit |
926e5392 | 232 | */ |
fe770bf0 | 233 | static unsigned long normalize_addr(unsigned long u) |
926e5392 | 234 | { |
3a366f79 KS |
235 | int shift; |
236 | if (!IS_ENABLED(CONFIG_X86_64)) | |
237 | return u; | |
238 | ||
239 | shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); | |
240 | return (signed long)(u << shift) >> shift; | |
926e5392 AV |
241 | } |
242 | ||
c200dac7 TG |
243 | static void note_wx(struct pg_state *st) |
244 | { | |
245 | unsigned long npages; | |
246 | ||
247 | npages = (st->current_address - st->start_address) / PAGE_SIZE; | |
248 | ||
249 | #ifdef CONFIG_PCI_BIOS | |
250 | /* | |
251 | * If PCI BIOS is enabled, the PCI BIOS area is forced to WX. | |
252 | * Inform about it, but avoid the warning. | |
253 | */ | |
254 | if (pcibios_enabled && st->start_address >= PAGE_OFFSET + BIOS_BEGIN && | |
255 | st->current_address <= PAGE_OFFSET + BIOS_END) { | |
256 | pr_warn_once("x86/mm: PCI BIOS W+X mapping %lu pages\n", npages); | |
257 | return; | |
258 | } | |
259 | #endif | |
260 | /* Account the WX pages */ | |
261 | st->wx_pages += npages; | |
262 | WARN_ONCE(1, "x86/mm: Found insecure W+X mapping at address %pS\n", | |
263 | (void *)st->start_address); | |
264 | } | |
265 | ||
926e5392 AV |
266 | /* |
267 | * This function gets called on a break in a continuous series | |
268 | * of PTE entries; the next one is different so we need to | |
269 | * print what we collected so far. | |
270 | */ | |
271 | static void note_page(struct seq_file *m, struct pg_state *st, | |
672c0ae0 | 272 | pgprot_t new_prot, pgprotval_t new_eff, int level) |
926e5392 | 273 | { |
672c0ae0 | 274 | pgprotval_t prot, cur, eff; |
3891a04a | 275 | static const char units[] = "BKMGTPE"; |
926e5392 AV |
276 | |
277 | /* | |
278 | * If we have a "break" in the series, we need to flush the state that | |
fe770bf0 PA |
279 | * we have now. "break" is either changing perms, levels or |
280 | * address space marker. | |
926e5392 | 281 | */ |
da25e628 TK |
282 | prot = pgprot_val(new_prot); |
283 | cur = pgprot_val(st->current_prot); | |
672c0ae0 | 284 | eff = st->effective_prot; |
926e5392 | 285 | |
fe770bf0 PA |
286 | if (!st->level) { |
287 | /* First entry */ | |
288 | st->current_prot = new_prot; | |
672c0ae0 | 289 | st->effective_prot = new_eff; |
fe770bf0 PA |
290 | st->level = level; |
291 | st->marker = address_markers; | |
3891a04a | 292 | st->lines = 0; |
ef6bea6d BP |
293 | pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n", |
294 | st->marker->name); | |
672c0ae0 | 295 | } else if (prot != cur || new_eff != eff || level != st->level || |
fe770bf0 PA |
296 | st->current_address >= st->marker[1].start_address) { |
297 | const char *unit = units; | |
926e5392 | 298 | unsigned long delta; |
6424fb38 | 299 | int width = sizeof(unsigned long) * 2; |
e1a58320 | 300 | |
c200dac7 TG |
301 | if (st->check_wx && (eff & _PAGE_RW) && !(eff & _PAGE_NX)) |
302 | note_wx(st); | |
926e5392 | 303 | |
926e5392 AV |
304 | /* |
305 | * Now print the actual finished series | |
306 | */ | |
3891a04a PA |
307 | if (!st->marker->max_lines || |
308 | st->lines < st->marker->max_lines) { | |
309 | pt_dump_seq_printf(m, st->to_dmesg, | |
310 | "0x%0*lx-0x%0*lx ", | |
311 | width, st->start_address, | |
312 | width, st->current_address); | |
926e5392 | 313 | |
3891a04a PA |
314 | delta = st->current_address - st->start_address; |
315 | while (!(delta & 1023) && unit[1]) { | |
316 | delta >>= 10; | |
317 | unit++; | |
318 | } | |
319 | pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ", | |
320 | delta, *unit); | |
321 | printk_prot(m, st->current_prot, st->level, | |
322 | st->to_dmesg); | |
926e5392 | 323 | } |
3891a04a | 324 | st->lines++; |
fe770bf0 PA |
325 | |
326 | /* | |
327 | * We print markers for special areas of address space, | |
328 | * such as the start of vmalloc space etc. | |
329 | * This helps in the interpretation. | |
330 | */ | |
331 | if (st->current_address >= st->marker[1].start_address) { | |
3891a04a PA |
332 | if (st->marker->max_lines && |
333 | st->lines > st->marker->max_lines) { | |
334 | unsigned long nskip = | |
335 | st->lines - st->marker->max_lines; | |
336 | pt_dump_seq_printf(m, st->to_dmesg, | |
337 | "... %lu entr%s skipped ... \n", | |
338 | nskip, | |
339 | nskip == 1 ? "y" : "ies"); | |
340 | } | |
fe770bf0 | 341 | st->marker++; |
3891a04a | 342 | st->lines = 0; |
ef6bea6d BP |
343 | pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n", |
344 | st->marker->name); | |
926e5392 | 345 | } |
fe770bf0 | 346 | |
926e5392 AV |
347 | st->start_address = st->current_address; |
348 | st->current_prot = new_prot; | |
672c0ae0 | 349 | st->effective_prot = new_eff; |
926e5392 | 350 | st->level = level; |
fe770bf0 | 351 | } |
926e5392 AV |
352 | } |
353 | ||
672c0ae0 JB |
354 | static inline pgprotval_t effective_prot(pgprotval_t prot1, pgprotval_t prot2) |
355 | { | |
356 | return (prot1 & prot2 & (_PAGE_USER | _PAGE_RW)) | | |
357 | ((prot1 | prot2) & _PAGE_NX); | |
358 | } | |
359 | ||
360 | static void walk_pte_level(struct seq_file *m, struct pg_state *st, pmd_t addr, | |
361 | pgprotval_t eff_in, unsigned long P) | |
926e5392 AV |
362 | { |
363 | int i; | |
d6ef1f19 | 364 | pte_t *pte; |
672c0ae0 | 365 | pgprotval_t prot, eff; |
926e5392 | 366 | |
926e5392 | 367 | for (i = 0; i < PTRS_PER_PTE; i++) { |
fe770bf0 | 368 | st->current_address = normalize_addr(P + i * PTE_LEVEL_MULT); |
d6ef1f19 JR |
369 | pte = pte_offset_map(&addr, st->current_address); |
370 | prot = pte_flags(*pte); | |
371 | eff = effective_prot(eff_in, prot); | |
672c0ae0 | 372 | note_page(m, st, __pgprot(prot), eff, 5); |
d6ef1f19 | 373 | pte_unmap(pte); |
926e5392 AV |
374 | } |
375 | } | |
04b67022 AR |
376 | #ifdef CONFIG_KASAN |
377 | ||
378 | /* | |
379 | * This is an optimization for KASAN=y case. Since all kasan page tables | |
9577dd74 | 380 | * eventually point to the kasan_early_shadow_page we could call note_page() |
04b67022 AR |
381 | * right away without walking through lower level page tables. This saves |
382 | * us dozens of seconds (minutes for 5-level config) while checking for | |
383 | * W+X mapping or reading kernel_page_tables debugfs file. | |
384 | */ | |
385 | static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st, | |
386 | void *pt) | |
387 | { | |
9577dd74 AK |
388 | if (__pa(pt) == __pa(kasan_early_shadow_pmd) || |
389 | (pgtable_l5_enabled() && | |
390 | __pa(pt) == __pa(kasan_early_shadow_p4d)) || | |
391 | __pa(pt) == __pa(kasan_early_shadow_pud)) { | |
392 | pgprotval_t prot = pte_flags(kasan_early_shadow_pte[0]); | |
672c0ae0 | 393 | note_page(m, st, __pgprot(prot), 0, 5); |
04b67022 AR |
394 | return true; |
395 | } | |
396 | return false; | |
397 | } | |
398 | #else | |
399 | static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st, | |
400 | void *pt) | |
401 | { | |
402 | return false; | |
403 | } | |
404 | #endif | |
926e5392 | 405 | |
fe770bf0 | 406 | #if PTRS_PER_PMD > 1 |
926e5392 | 407 | |
672c0ae0 JB |
408 | static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pud_t addr, |
409 | pgprotval_t eff_in, unsigned long P) | |
926e5392 AV |
410 | { |
411 | int i; | |
04b67022 | 412 | pmd_t *start, *pmd_start; |
672c0ae0 | 413 | pgprotval_t prot, eff; |
926e5392 | 414 | |
04b67022 | 415 | pmd_start = start = (pmd_t *)pud_page_vaddr(addr); |
926e5392 | 416 | for (i = 0; i < PTRS_PER_PMD; i++) { |
fe770bf0 | 417 | st->current_address = normalize_addr(P + i * PMD_LEVEL_MULT); |
926e5392 | 418 | if (!pmd_none(*start)) { |
672c0ae0 JB |
419 | prot = pmd_flags(*start); |
420 | eff = effective_prot(eff_in, prot); | |
da25e628 | 421 | if (pmd_large(*start) || !pmd_present(*start)) { |
672c0ae0 | 422 | note_page(m, st, __pgprot(prot), eff, 4); |
04b67022 | 423 | } else if (!kasan_page_table(m, st, pmd_start)) { |
672c0ae0 | 424 | walk_pte_level(m, st, *start, eff, |
fe770bf0 | 425 | P + i * PMD_LEVEL_MULT); |
da25e628 | 426 | } |
926e5392 | 427 | } else |
672c0ae0 | 428 | note_page(m, st, __pgprot(0), 0, 4); |
926e5392 AV |
429 | start++; |
430 | } | |
431 | } | |
432 | ||
fe770bf0 | 433 | #else |
672c0ae0 | 434 | #define walk_pmd_level(m,s,a,e,p) walk_pte_level(m,s,__pmd(pud_val(a)),e,p) |
fe770bf0 PA |
435 | #define pud_large(a) pmd_large(__pmd(pud_val(a))) |
436 | #define pud_none(a) pmd_none(__pmd(pud_val(a))) | |
437 | #endif | |
926e5392 | 438 | |
fe770bf0 PA |
439 | #if PTRS_PER_PUD > 1 |
440 | ||
672c0ae0 JB |
441 | static void walk_pud_level(struct seq_file *m, struct pg_state *st, p4d_t addr, |
442 | pgprotval_t eff_in, unsigned long P) | |
926e5392 AV |
443 | { |
444 | int i; | |
04b67022 | 445 | pud_t *start, *pud_start; |
672c0ae0 | 446 | pgprotval_t prot, eff; |
926e5392 | 447 | |
04b67022 | 448 | pud_start = start = (pud_t *)p4d_page_vaddr(addr); |
926e5392 AV |
449 | |
450 | for (i = 0; i < PTRS_PER_PUD; i++) { | |
fe770bf0 | 451 | st->current_address = normalize_addr(P + i * PUD_LEVEL_MULT); |
04b67022 | 452 | if (!pud_none(*start)) { |
672c0ae0 JB |
453 | prot = pud_flags(*start); |
454 | eff = effective_prot(eff_in, prot); | |
da25e628 | 455 | if (pud_large(*start) || !pud_present(*start)) { |
672c0ae0 | 456 | note_page(m, st, __pgprot(prot), eff, 3); |
04b67022 | 457 | } else if (!kasan_page_table(m, st, pud_start)) { |
672c0ae0 | 458 | walk_pmd_level(m, st, *start, eff, |
fe770bf0 | 459 | P + i * PUD_LEVEL_MULT); |
da25e628 | 460 | } |
926e5392 | 461 | } else |
672c0ae0 | 462 | note_page(m, st, __pgprot(0), 0, 3); |
926e5392 AV |
463 | |
464 | start++; | |
465 | } | |
466 | } | |
467 | ||
fe770bf0 | 468 | #else |
672c0ae0 | 469 | #define walk_pud_level(m,s,a,e,p) walk_pmd_level(m,s,__pud(p4d_val(a)),e,p) |
fdd3d8ce KS |
470 | #define p4d_large(a) pud_large(__pud(p4d_val(a))) |
471 | #define p4d_none(a) pud_none(__pud(p4d_val(a))) | |
472 | #endif | |
473 | ||
672c0ae0 JB |
474 | static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr, |
475 | pgprotval_t eff_in, unsigned long P) | |
fdd3d8ce KS |
476 | { |
477 | int i; | |
04b67022 | 478 | p4d_t *start, *p4d_start; |
672c0ae0 | 479 | pgprotval_t prot, eff; |
fdd3d8ce | 480 | |
c65e774f | 481 | if (PTRS_PER_P4D == 1) |
672c0ae0 | 482 | return walk_pud_level(m, st, __p4d(pgd_val(addr)), eff_in, P); |
c65e774f | 483 | |
04b67022 | 484 | p4d_start = start = (p4d_t *)pgd_page_vaddr(addr); |
fdd3d8ce KS |
485 | |
486 | for (i = 0; i < PTRS_PER_P4D; i++) { | |
487 | st->current_address = normalize_addr(P + i * P4D_LEVEL_MULT); | |
488 | if (!p4d_none(*start)) { | |
672c0ae0 JB |
489 | prot = p4d_flags(*start); |
490 | eff = effective_prot(eff_in, prot); | |
fdd3d8ce | 491 | if (p4d_large(*start) || !p4d_present(*start)) { |
672c0ae0 | 492 | note_page(m, st, __pgprot(prot), eff, 2); |
04b67022 | 493 | } else if (!kasan_page_table(m, st, p4d_start)) { |
672c0ae0 | 494 | walk_pud_level(m, st, *start, eff, |
fdd3d8ce KS |
495 | P + i * P4D_LEVEL_MULT); |
496 | } | |
497 | } else | |
672c0ae0 | 498 | note_page(m, st, __pgprot(0), 0, 2); |
fdd3d8ce KS |
499 | |
500 | start++; | |
501 | } | |
502 | } | |
503 | ||
ed7588d5 KS |
504 | #define pgd_large(a) (pgtable_l5_enabled() ? pgd_large(a) : p4d_large(__p4d(pgd_val(a)))) |
505 | #define pgd_none(a) (pgtable_l5_enabled() ? pgd_none(a) : p4d_none(__p4d(pgd_val(a)))) | |
fe770bf0 | 506 | |
f4e342c8 BO |
507 | static inline bool is_hypervisor_range(int idx) |
508 | { | |
b176862f | 509 | #ifdef CONFIG_X86_64 |
f4e342c8 | 510 | /* |
16877a55 KS |
511 | * A hole in the beginning of kernel address space reserved |
512 | * for a hypervisor. | |
f4e342c8 | 513 | */ |
16877a55 KS |
514 | return (idx >= pgd_index(GUARD_HOLE_BASE_ADDR)) && |
515 | (idx < pgd_index(GUARD_HOLE_END_ADDR)); | |
f4e342c8 | 516 | #else |
b176862f | 517 | return false; |
f4e342c8 | 518 | #endif |
b176862f | 519 | } |
f4e342c8 | 520 | |
e1a58320 | 521 | static void ptdump_walk_pgd_level_core(struct seq_file *m, pgd_t *pgd, |
b4bf4f92 | 522 | bool checkwx, bool dmesg) |
926e5392 | 523 | { |
4e8537e4 | 524 | pgd_t *start = INIT_PGD; |
672c0ae0 | 525 | pgprotval_t prot, eff; |
926e5392 | 526 | int i; |
ef6bea6d | 527 | struct pg_state st = {}; |
926e5392 | 528 | |
ef6bea6d BP |
529 | if (pgd) { |
530 | start = pgd; | |
b4bf4f92 | 531 | st.to_dmesg = dmesg; |
ef6bea6d | 532 | } |
926e5392 | 533 | |
e1a58320 SS |
534 | st.check_wx = checkwx; |
535 | if (checkwx) | |
536 | st.wx_pages = 0; | |
537 | ||
926e5392 | 538 | for (i = 0; i < PTRS_PER_PGD; i++) { |
fe770bf0 | 539 | st.current_address = normalize_addr(i * PGD_LEVEL_MULT); |
f4e342c8 | 540 | if (!pgd_none(*start) && !is_hypervisor_range(i)) { |
672c0ae0 JB |
541 | prot = pgd_flags(*start); |
542 | #ifdef CONFIG_X86_PAE | |
543 | eff = _PAGE_USER | _PAGE_RW; | |
544 | #else | |
545 | eff = prot; | |
546 | #endif | |
da25e628 | 547 | if (pgd_large(*start) || !pgd_present(*start)) { |
672c0ae0 | 548 | note_page(m, &st, __pgprot(prot), eff, 1); |
da25e628 | 549 | } else { |
672c0ae0 | 550 | walk_p4d_level(m, &st, *start, eff, |
fe770bf0 | 551 | i * PGD_LEVEL_MULT); |
da25e628 | 552 | } |
fe770bf0 | 553 | } else |
672c0ae0 | 554 | note_page(m, &st, __pgprot(0), 0, 1); |
fe770bf0 | 555 | |
146fbb76 | 556 | cond_resched(); |
926e5392 AV |
557 | start++; |
558 | } | |
fe770bf0 PA |
559 | |
560 | /* Flush out the last page */ | |
561 | st.current_address = normalize_addr(PTRS_PER_PGD*PGD_LEVEL_MULT); | |
672c0ae0 | 562 | note_page(m, &st, __pgprot(0), 0, 0); |
e1a58320 SS |
563 | if (!checkwx) |
564 | return; | |
565 | if (st.wx_pages) | |
566 | pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n", | |
567 | st.wx_pages); | |
568 | else | |
569 | pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n"); | |
570 | } | |
571 | ||
572 | void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd) | |
573 | { | |
b4bf4f92 TG |
574 | ptdump_walk_pgd_level_core(m, pgd, false, true); |
575 | } | |
576 | ||
a4b51ef6 | 577 | void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user) |
b4bf4f92 | 578 | { |
a4b51ef6 TG |
579 | #ifdef CONFIG_PAGE_TABLE_ISOLATION |
580 | if (user && static_cpu_has(X86_FEATURE_PTI)) | |
581 | pgd = kernel_to_user_pgdp(pgd); | |
582 | #endif | |
b4bf4f92 TG |
583 | ptdump_walk_pgd_level_core(m, pgd, false, false); |
584 | } | |
585 | EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs); | |
586 | ||
d878efce | 587 | void ptdump_walk_user_pgd_level_checkwx(void) |
b4bf4f92 TG |
588 | { |
589 | #ifdef CONFIG_PAGE_TABLE_ISOLATION | |
4e8537e4 | 590 | pgd_t *pgd = INIT_PGD; |
b4bf4f92 | 591 | |
d878efce JR |
592 | if (!(__supported_pte_mask & _PAGE_NX) || |
593 | !static_cpu_has(X86_FEATURE_PTI)) | |
b4bf4f92 TG |
594 | return; |
595 | ||
596 | pr_info("x86/mm: Checking user space page tables\n"); | |
597 | pgd = kernel_to_user_pgdp(pgd); | |
598 | ptdump_walk_pgd_level_core(NULL, pgd, true, false); | |
599 | #endif | |
926e5392 AV |
600 | } |
601 | ||
e1a58320 SS |
602 | void ptdump_walk_pgd_level_checkwx(void) |
603 | { | |
b4bf4f92 | 604 | ptdump_walk_pgd_level_core(NULL, NULL, true, false); |
e1a58320 SS |
605 | } |
606 | ||
8609d1b5 | 607 | static int __init pt_dump_init(void) |
926e5392 | 608 | { |
0483e1fa TG |
609 | /* |
610 | * Various markers are not compile-time constants, so assign them | |
611 | * here. | |
612 | */ | |
613 | #ifdef CONFIG_X86_64 | |
614 | address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET; | |
615 | address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; | |
616 | address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START; | |
5c7919bb KS |
617 | #ifdef CONFIG_MODIFY_LDT_SYSCALL |
618 | address_markers[LDT_NR].start_address = LDT_BASE_ADDR; | |
619 | #endif | |
09e61a77 KS |
620 | #ifdef CONFIG_KASAN |
621 | address_markers[KASAN_SHADOW_START_NR].start_address = KASAN_SHADOW_START; | |
622 | address_markers[KASAN_SHADOW_END_NR].start_address = KASAN_SHADOW_END; | |
623 | #endif | |
0483e1fa | 624 | #endif |
fe770bf0 | 625 | #ifdef CONFIG_X86_32 |
92851e2f AS |
626 | address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; |
627 | address_markers[VMALLOC_END_NR].start_address = VMALLOC_END; | |
fe770bf0 | 628 | # ifdef CONFIG_HIGHMEM |
92851e2f | 629 | address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE; |
fe770bf0 | 630 | # endif |
92851e2f | 631 | address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; |
92a0f81d | 632 | address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE; |
f3e48e54 JR |
633 | # ifdef CONFIG_MODIFY_LDT_SYSCALL |
634 | address_markers[LDT_NR].start_address = LDT_BASE_ADDR; | |
635 | # endif | |
fe770bf0 | 636 | #endif |
926e5392 AV |
637 | return 0; |
638 | } | |
926e5392 | 639 | __initcall(pt_dump_init); |