1 // SPDX-License-Identifier: GPL-2.0
2 #define DISABLE_BRANCH_PROFILING
3 #define pr_fmt(fmt) "kasan: " fmt
4 #include <linux/bootmem.h>
5 #include <linux/kasan.h>
6 #include <linux/kdebug.h>
7 #include <linux/memblock.h>
9 #include <linux/sched.h>
10 #include <linux/sched/task.h>
11 #include <linux/vmalloc.h>
13 #include <asm/e820/types.h>
14 #include <asm/pgalloc.h>
15 #include <asm/tlbflush.h>
16 #include <asm/sections.h>
17 #include <asm/pgtable.h>
19 extern struct range pfn_mapped
[E820_MAX_ENTRIES
];
21 static p4d_t tmp_p4d_table
[PTRS_PER_P4D
] __initdata
__aligned(PAGE_SIZE
);
23 static __init
void *early_alloc(size_t size
, int nid
)
25 return memblock_virt_alloc_try_nid_nopanic(size
, size
,
26 __pa(MAX_DMA_ADDRESS
), BOOTMEM_ALLOC_ACCESSIBLE
, nid
);
29 static void __init
kasan_populate_pmd(pmd_t
*pmd
, unsigned long addr
,
30 unsigned long end
, int nid
)
37 if (boot_cpu_has(X86_FEATURE_PSE
) &&
38 ((end
- addr
) == PMD_SIZE
) &&
39 IS_ALIGNED(addr
, PMD_SIZE
)) {
40 p
= early_alloc(PMD_SIZE
, nid
);
41 if (p
&& pmd_set_huge(pmd
, __pa(p
), PAGE_KERNEL
))
44 memblock_free(__pa(p
), PMD_SIZE
);
47 p
= early_alloc(PAGE_SIZE
, nid
);
48 pmd_populate_kernel(&init_mm
, pmd
, p
);
51 pte
= pte_offset_kernel(pmd
, addr
);
59 p
= early_alloc(PAGE_SIZE
, nid
);
60 entry
= pfn_pte(PFN_DOWN(__pa(p
)), PAGE_KERNEL
);
61 set_pte_at(&init_mm
, addr
, pte
, entry
);
62 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
65 static void __init
kasan_populate_pud(pud_t
*pud
, unsigned long addr
,
66 unsigned long end
, int nid
)
74 if (boot_cpu_has(X86_FEATURE_GBPAGES
) &&
75 ((end
- addr
) == PUD_SIZE
) &&
76 IS_ALIGNED(addr
, PUD_SIZE
)) {
77 p
= early_alloc(PUD_SIZE
, nid
);
78 if (p
&& pud_set_huge(pud
, __pa(p
), PAGE_KERNEL
))
81 memblock_free(__pa(p
), PUD_SIZE
);
84 p
= early_alloc(PAGE_SIZE
, nid
);
85 pud_populate(&init_mm
, pud
, p
);
88 pmd
= pmd_offset(pud
, addr
);
90 next
= pmd_addr_end(addr
, end
);
92 kasan_populate_pmd(pmd
, addr
, next
, nid
);
93 } while (pmd
++, addr
= next
, addr
!= end
);
96 static void __init
kasan_populate_p4d(p4d_t
*p4d
, unsigned long addr
,
97 unsigned long end
, int nid
)
102 if (p4d_none(*p4d
)) {
103 void *p
= early_alloc(PAGE_SIZE
, nid
);
105 p4d_populate(&init_mm
, p4d
, p
);
108 pud
= pud_offset(p4d
, addr
);
110 next
= pud_addr_end(addr
, end
);
111 if (!pud_large(*pud
))
112 kasan_populate_pud(pud
, addr
, next
, nid
);
113 } while (pud
++, addr
= next
, addr
!= end
);
116 static void __init
kasan_populate_pgd(pgd_t
*pgd
, unsigned long addr
,
117 unsigned long end
, int nid
)
123 if (pgd_none(*pgd
)) {
124 p
= early_alloc(PAGE_SIZE
, nid
);
125 pgd_populate(&init_mm
, pgd
, p
);
128 p4d
= p4d_offset(pgd
, addr
);
130 next
= p4d_addr_end(addr
, end
);
131 kasan_populate_p4d(p4d
, addr
, next
, nid
);
132 } while (p4d
++, addr
= next
, addr
!= end
);
135 static void __init
kasan_populate_shadow(unsigned long addr
, unsigned long end
,
141 addr
= addr
& PAGE_MASK
;
142 end
= round_up(end
, PAGE_SIZE
);
143 pgd
= pgd_offset_k(addr
);
145 next
= pgd_addr_end(addr
, end
);
146 kasan_populate_pgd(pgd
, addr
, next
, nid
);
147 } while (pgd
++, addr
= next
, addr
!= end
);
150 static void __init
map_range(struct range
*range
)
155 start
= (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range
->start
));
156 end
= (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range
->end
));
158 kasan_populate_shadow(start
, end
, early_pfn_to_nid(range
->start
));
161 static void __init
clear_pgds(unsigned long start
,
165 /* See comment in kasan_init() */
166 unsigned long pgd_end
= end
& PGDIR_MASK
;
168 for (; start
< pgd_end
; start
+= PGDIR_SIZE
) {
169 pgd
= pgd_offset_k(start
);
171 * With folded p4d, pgd_clear() is nop, use p4d_clear()
174 if (CONFIG_PGTABLE_LEVELS
< 5)
175 p4d_clear(p4d_offset(pgd
, start
));
180 pgd
= pgd_offset_k(start
);
181 for (; start
< end
; start
+= P4D_SIZE
)
182 p4d_clear(p4d_offset(pgd
, start
));
185 static inline p4d_t
*early_p4d_offset(pgd_t
*pgd
, unsigned long addr
)
189 if (!IS_ENABLED(CONFIG_X86_5LEVEL
))
192 p4d
= __pa_nodebug(pgd_val(*pgd
)) & PTE_PFN_MASK
;
193 p4d
+= __START_KERNEL_map
- phys_base
;
194 return (p4d_t
*)p4d
+ p4d_index(addr
);
197 static void __init
kasan_early_p4d_populate(pgd_t
*pgd
,
202 p4d_t
*p4d
, p4d_entry
;
205 if (pgd_none(*pgd
)) {
206 pgd_entry
= __pgd(_KERNPG_TABLE
| __pa_nodebug(kasan_zero_p4d
));
207 set_pgd(pgd
, pgd_entry
);
210 p4d
= early_p4d_offset(pgd
, addr
);
212 next
= p4d_addr_end(addr
, end
);
217 p4d_entry
= __p4d(_KERNPG_TABLE
| __pa_nodebug(kasan_zero_pud
));
218 set_p4d(p4d
, p4d_entry
);
219 } while (p4d
++, addr
= next
, addr
!= end
&& p4d_none(*p4d
));
222 static void __init
kasan_map_early_shadow(pgd_t
*pgd
)
224 /* See comment in kasan_init() */
225 unsigned long addr
= KASAN_SHADOW_START
& PGDIR_MASK
;
226 unsigned long end
= KASAN_SHADOW_END
;
229 pgd
+= pgd_index(addr
);
231 next
= pgd_addr_end(addr
, end
);
232 kasan_early_p4d_populate(pgd
, addr
, next
);
233 } while (pgd
++, addr
= next
, addr
!= end
);
236 #ifdef CONFIG_KASAN_INLINE
237 static int kasan_die_handler(struct notifier_block
*self
,
241 if (val
== DIE_GPF
) {
242 pr_emerg("CONFIG_KASAN_INLINE enabled\n");
243 pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n");
248 static struct notifier_block kasan_die_notifier
= {
249 .notifier_call
= kasan_die_handler
,
253 void __init
kasan_early_init(void)
256 pteval_t pte_val
= __pa_nodebug(kasan_zero_page
) | __PAGE_KERNEL
| _PAGE_ENC
;
257 pmdval_t pmd_val
= __pa_nodebug(kasan_zero_pte
) | _KERNPG_TABLE
;
258 pudval_t pud_val
= __pa_nodebug(kasan_zero_pmd
) | _KERNPG_TABLE
;
259 p4dval_t p4d_val
= __pa_nodebug(kasan_zero_pud
) | _KERNPG_TABLE
;
261 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
262 kasan_zero_pte
[i
] = __pte(pte_val
);
264 for (i
= 0; i
< PTRS_PER_PMD
; i
++)
265 kasan_zero_pmd
[i
] = __pmd(pmd_val
);
267 for (i
= 0; i
< PTRS_PER_PUD
; i
++)
268 kasan_zero_pud
[i
] = __pud(pud_val
);
270 for (i
= 0; IS_ENABLED(CONFIG_X86_5LEVEL
) && i
< PTRS_PER_P4D
; i
++)
271 kasan_zero_p4d
[i
] = __p4d(p4d_val
);
273 kasan_map_early_shadow(early_top_pgt
);
274 kasan_map_early_shadow(init_top_pgt
);
277 void __init
kasan_init(void)
281 #ifdef CONFIG_KASAN_INLINE
282 register_die_notifier(&kasan_die_notifier
);
285 memcpy(early_top_pgt
, init_top_pgt
, sizeof(early_top_pgt
));
288 * We use the same shadow offset for 4- and 5-level paging to
289 * facilitate boot-time switching between paging modes.
290 * As result in 5-level paging mode KASAN_SHADOW_START and
291 * KASAN_SHADOW_END are not aligned to PGD boundary.
293 * KASAN_SHADOW_START doesn't share PGD with anything else.
294 * We claim whole PGD entry to make things easier.
296 * KASAN_SHADOW_END lands in the last PGD entry and it collides with
297 * bunch of things like kernel code, modules, EFI mapping, etc.
298 * We need to take extra steps to not overwrite them.
300 if (IS_ENABLED(CONFIG_X86_5LEVEL
)) {
303 ptr
= (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END
));
304 memcpy(tmp_p4d_table
, (void *)ptr
, sizeof(tmp_p4d_table
));
305 set_pgd(&early_top_pgt
[pgd_index(KASAN_SHADOW_END
)],
306 __pgd(__pa(tmp_p4d_table
) | _KERNPG_TABLE
));
309 load_cr3(early_top_pgt
);
312 clear_pgds(KASAN_SHADOW_START
& PGDIR_MASK
, KASAN_SHADOW_END
);
314 kasan_populate_zero_shadow((void *)(KASAN_SHADOW_START
& PGDIR_MASK
),
315 kasan_mem_to_shadow((void *)PAGE_OFFSET
));
317 for (i
= 0; i
< E820_MAX_ENTRIES
; i
++) {
318 if (pfn_mapped
[i
].end
== 0)
321 map_range(&pfn_mapped
[i
]);
324 kasan_populate_zero_shadow(
325 kasan_mem_to_shadow((void *)PAGE_OFFSET
+ MAXMEM
),
326 kasan_mem_to_shadow((void *)__START_KERNEL_map
));
328 kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext
),
329 (unsigned long)kasan_mem_to_shadow(_end
),
330 early_pfn_to_nid(__pa(_stext
)));
332 kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END
),
333 (void *)KASAN_SHADOW_END
);
335 load_cr3(init_top_pgt
);
339 * kasan_zero_page has been used as early shadow memory, thus it may
340 * contain some garbage. Now we can clear and write protect it, since
341 * after the TLB flush no one should write to it.
343 memset(kasan_zero_page
, 0, PAGE_SIZE
);
344 for (i
= 0; i
< PTRS_PER_PTE
; i
++) {
345 pte_t pte
= __pte(__pa(kasan_zero_page
) | __PAGE_KERNEL_RO
| _PAGE_ENC
);
346 set_pte(&kasan_zero_pte
[i
], pte
);
348 /* Flush TLBs again to be sure that write protection applied. */
351 init_task
.kasan_depth
= 0;
352 pr_info("KernelAddressSanitizer initialized\n");