]>
Commit | Line | Data |
---|---|---|
69786cdb AR |
1 | /* |
2 | * This file contains some kasan initialization code. | |
3 | * | |
4 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. | |
5 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | */ | |
12 | ||
13 | #include <linux/bootmem.h> | |
14 | #include <linux/init.h> | |
15 | #include <linux/kasan.h> | |
16 | #include <linux/kernel.h> | |
17 | #include <linux/memblock.h> | |
5c6a84a3 | 18 | #include <linux/mm.h> |
69786cdb AR |
19 | #include <linux/pfn.h> |
20 | ||
21 | #include <asm/page.h> | |
22 | #include <asm/pgalloc.h> | |
23 | ||
24 | /* | |
25 | * This page serves two purposes: | |
26 | * - It used as early shadow memory. The entire shadow region populated | |
27 | * with this page, before we will be able to setup normal shadow memory. | |
28 | * - Latter it reused it as zero shadow to cover large ranges of memory | |
29 | * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). | |
30 | */ | |
31 | unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss; | |
32 | ||
c2febafc KS |
33 | #if CONFIG_PGTABLE_LEVELS > 4 |
34 | p4d_t kasan_zero_p4d[PTRS_PER_P4D] __page_aligned_bss; | |
35 | #endif | |
69786cdb AR |
36 | #if CONFIG_PGTABLE_LEVELS > 3 |
37 | pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss; | |
38 | #endif | |
39 | #if CONFIG_PGTABLE_LEVELS > 2 | |
40 | pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
41 | #endif | |
42 | pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss; | |
43 | ||
44 | static __init void *early_alloc(size_t size, int node) | |
45 | { | |
46 | return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), | |
47 | BOOTMEM_ALLOC_ACCESSIBLE, node); | |
48 | } | |
49 | ||
50 | static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr, | |
51 | unsigned long end) | |
52 | { | |
53 | pte_t *pte = pte_offset_kernel(pmd, addr); | |
54 | pte_t zero_pte; | |
55 | ||
5c6a84a3 | 56 | zero_pte = pfn_pte(PFN_DOWN(__pa_symbol(kasan_zero_page)), PAGE_KERNEL); |
69786cdb AR |
57 | zero_pte = pte_wrprotect(zero_pte); |
58 | ||
59 | while (addr + PAGE_SIZE <= end) { | |
60 | set_pte_at(&init_mm, addr, pte, zero_pte); | |
61 | addr += PAGE_SIZE; | |
62 | pte = pte_offset_kernel(pmd, addr); | |
63 | } | |
64 | } | |
65 | ||
66 | static void __init zero_pmd_populate(pud_t *pud, unsigned long addr, | |
67 | unsigned long end) | |
68 | { | |
69 | pmd_t *pmd = pmd_offset(pud, addr); | |
70 | unsigned long next; | |
71 | ||
72 | do { | |
73 | next = pmd_addr_end(addr, end); | |
74 | ||
75 | if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { | |
5c6a84a3 | 76 | pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte)); |
69786cdb AR |
77 | continue; |
78 | } | |
79 | ||
80 | if (pmd_none(*pmd)) { | |
81 | pmd_populate_kernel(&init_mm, pmd, | |
82 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
83 | } | |
84 | zero_pte_populate(pmd, addr, next); | |
85 | } while (pmd++, addr = next, addr != end); | |
86 | } | |
87 | ||
c2febafc | 88 | static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr, |
69786cdb AR |
89 | unsigned long end) |
90 | { | |
c2febafc | 91 | pud_t *pud = pud_offset(p4d, addr); |
69786cdb AR |
92 | unsigned long next; |
93 | ||
94 | do { | |
95 | next = pud_addr_end(addr, end); | |
96 | if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { | |
97 | pmd_t *pmd; | |
98 | ||
5c6a84a3 | 99 | pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd)); |
69786cdb | 100 | pmd = pmd_offset(pud, addr); |
5c6a84a3 | 101 | pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte)); |
69786cdb AR |
102 | continue; |
103 | } | |
104 | ||
105 | if (pud_none(*pud)) { | |
106 | pud_populate(&init_mm, pud, | |
107 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
108 | } | |
109 | zero_pmd_populate(pud, addr, next); | |
110 | } while (pud++, addr = next, addr != end); | |
111 | } | |
112 | ||
c2febafc KS |
113 | static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr, |
114 | unsigned long end) | |
115 | { | |
116 | p4d_t *p4d = p4d_offset(pgd, addr); | |
117 | unsigned long next; | |
118 | ||
119 | do { | |
120 | next = p4d_addr_end(addr, end); | |
121 | ||
122 | if (p4d_none(*p4d)) { | |
123 | p4d_populate(&init_mm, p4d, | |
124 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
125 | } | |
126 | zero_pud_populate(p4d, addr, next); | |
127 | } while (p4d++, addr = next, addr != end); | |
128 | } | |
129 | ||
69786cdb AR |
130 | /** |
131 | * kasan_populate_zero_shadow - populate shadow memory region with | |
132 | * kasan_zero_page | |
133 | * @shadow_start - start of the memory range to populate | |
134 | * @shadow_end - end of the memory range to populate | |
135 | */ | |
136 | void __init kasan_populate_zero_shadow(const void *shadow_start, | |
137 | const void *shadow_end) | |
138 | { | |
139 | unsigned long addr = (unsigned long)shadow_start; | |
140 | unsigned long end = (unsigned long)shadow_end; | |
141 | pgd_t *pgd = pgd_offset_k(addr); | |
142 | unsigned long next; | |
143 | ||
144 | do { | |
145 | next = pgd_addr_end(addr, end); | |
146 | ||
147 | if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { | |
c2febafc | 148 | p4d_t *p4d; |
69786cdb AR |
149 | pud_t *pud; |
150 | pmd_t *pmd; | |
151 | ||
152 | /* | |
153 | * kasan_zero_pud should be populated with pmds | |
154 | * at this moment. | |
155 | * [pud,pmd]_populate*() below needed only for | |
156 | * 3,2 - level page tables where we don't have | |
157 | * puds,pmds, so pgd_populate(), pud_populate() | |
158 | * is noops. | |
c2febafc KS |
159 | * |
160 | * The ifndef is required to avoid build breakage. | |
161 | * | |
162 | * With 5level-fixup.h, pgd_populate() is not nop and | |
163 | * we reference kasan_zero_p4d. It's not defined | |
164 | * unless 5-level paging enabled. | |
165 | * | |
166 | * The ifndef can be dropped once all KASAN-enabled | |
167 | * architectures will switch to pgtable-nop4d.h. | |
69786cdb | 168 | */ |
c2febafc KS |
169 | #ifndef __ARCH_HAS_5LEVEL_HACK |
170 | pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d)); | |
171 | #endif | |
172 | p4d = p4d_offset(pgd, addr); | |
173 | p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud)); | |
174 | pud = pud_offset(p4d, addr); | |
5c6a84a3 | 175 | pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd)); |
69786cdb | 176 | pmd = pmd_offset(pud, addr); |
5c6a84a3 | 177 | pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte)); |
69786cdb AR |
178 | continue; |
179 | } | |
180 | ||
181 | if (pgd_none(*pgd)) { | |
182 | pgd_populate(&init_mm, pgd, | |
183 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
184 | } | |
c2febafc | 185 | zero_p4d_populate(pgd, addr, next); |
69786cdb AR |
186 | } while (pgd++, addr = next, addr != end); |
187 | } |