]>
Commit | Line | Data |
---|---|---|
30d621f6 SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include <linux/pagewalk.h> | |
4 | #include <linux/ptdump.h> | |
5 | #include <linux/kasan.h> | |
6 | ||
7 | #ifdef CONFIG_KASAN | |
8 | /* | |
9 | * This is an optimization for KASAN=y case. Since all kasan page tables | |
10 | * eventually point to the kasan_early_shadow_page we could call note_page() | |
11 | * right away without walking through lower level page tables. This saves | |
12 | * us dozens of seconds (minutes for 5-level config) while checking for | |
13 | * W+X mapping or reading kernel_page_tables debugfs file. | |
14 | */ | |
15 | static inline int note_kasan_page_table(struct mm_walk *walk, | |
16 | unsigned long addr) | |
17 | { | |
18 | struct ptdump_state *st = walk->private; | |
19 | ||
20 | st->note_page(st, addr, 5, pte_val(kasan_early_shadow_pte[0])); | |
21 | ||
22 | walk->action = ACTION_CONTINUE; | |
23 | ||
24 | return 0; | |
25 | } | |
26 | #endif | |
27 | ||
28 | static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr, | |
29 | unsigned long next, struct mm_walk *walk) | |
30 | { | |
31 | struct ptdump_state *st = walk->private; | |
32 | pgd_t val = READ_ONCE(*pgd); | |
33 | ||
34 | #if CONFIG_PGTABLE_LEVELS > 4 && defined(CONFIG_KASAN) | |
35 | if (pgd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_p4d))) | |
36 | return note_kasan_page_table(walk, addr); | |
37 | #endif | |
38 | ||
39 | if (pgd_leaf(val)) | |
40 | st->note_page(st, addr, 1, pgd_val(val)); | |
41 | ||
42 | return 0; | |
43 | } | |
44 | ||
45 | static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr, | |
46 | unsigned long next, struct mm_walk *walk) | |
47 | { | |
48 | struct ptdump_state *st = walk->private; | |
49 | p4d_t val = READ_ONCE(*p4d); | |
50 | ||
51 | #if CONFIG_PGTABLE_LEVELS > 3 && defined(CONFIG_KASAN) | |
52 | if (p4d_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pud))) | |
53 | return note_kasan_page_table(walk, addr); | |
54 | #endif | |
55 | ||
56 | if (p4d_leaf(val)) | |
57 | st->note_page(st, addr, 2, p4d_val(val)); | |
58 | ||
59 | return 0; | |
60 | } | |
61 | ||
62 | static int ptdump_pud_entry(pud_t *pud, unsigned long addr, | |
63 | unsigned long next, struct mm_walk *walk) | |
64 | { | |
65 | struct ptdump_state *st = walk->private; | |
66 | pud_t val = READ_ONCE(*pud); | |
67 | ||
68 | #if CONFIG_PGTABLE_LEVELS > 2 && defined(CONFIG_KASAN) | |
69 | if (pud_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pmd))) | |
70 | return note_kasan_page_table(walk, addr); | |
71 | #endif | |
72 | ||
73 | if (pud_leaf(val)) | |
74 | st->note_page(st, addr, 3, pud_val(val)); | |
75 | ||
76 | return 0; | |
77 | } | |
78 | ||
79 | static int ptdump_pmd_entry(pmd_t *pmd, unsigned long addr, | |
80 | unsigned long next, struct mm_walk *walk) | |
81 | { | |
82 | struct ptdump_state *st = walk->private; | |
83 | pmd_t val = READ_ONCE(*pmd); | |
84 | ||
85 | #if defined(CONFIG_KASAN) | |
86 | if (pmd_page(val) == virt_to_page(lm_alias(kasan_early_shadow_pte))) | |
87 | return note_kasan_page_table(walk, addr); | |
88 | #endif | |
89 | ||
90 | if (pmd_leaf(val)) | |
91 | st->note_page(st, addr, 4, pmd_val(val)); | |
92 | ||
93 | return 0; | |
94 | } | |
95 | ||
96 | static int ptdump_pte_entry(pte_t *pte, unsigned long addr, | |
97 | unsigned long next, struct mm_walk *walk) | |
98 | { | |
99 | struct ptdump_state *st = walk->private; | |
100 | ||
101 | st->note_page(st, addr, 5, pte_val(READ_ONCE(*pte))); | |
102 | ||
103 | return 0; | |
104 | } | |
105 | ||
106 | static int ptdump_hole(unsigned long addr, unsigned long next, | |
107 | int depth, struct mm_walk *walk) | |
108 | { | |
109 | struct ptdump_state *st = walk->private; | |
110 | ||
111 | st->note_page(st, addr, depth + 1, 0); | |
112 | ||
113 | return 0; | |
114 | } | |
115 | ||
116 | static const struct mm_walk_ops ptdump_ops = { | |
117 | .pgd_entry = ptdump_pgd_entry, | |
118 | .p4d_entry = ptdump_p4d_entry, | |
119 | .pud_entry = ptdump_pud_entry, | |
120 | .pmd_entry = ptdump_pmd_entry, | |
121 | .pte_entry = ptdump_pte_entry, | |
122 | .pte_hole = ptdump_hole, | |
123 | }; | |
124 | ||
125 | void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm) | |
126 | { | |
127 | const struct ptdump_range *range = st->range; | |
128 | ||
129 | down_read(&mm->mmap_sem); | |
130 | while (range->start != range->end) { | |
131 | walk_page_range_novma(mm, range->start, range->end, | |
132 | &ptdump_ops, st); | |
133 | range++; | |
134 | } | |
135 | up_read(&mm->mmap_sem); | |
136 | ||
137 | /* Flush out the last page */ | |
138 | st->note_page(st, 0, 0, 0); | |
139 | } |