1 #define DISABLE_BRANCH_PROFILING
2 #define pr_fmt(fmt) "kasan: " fmt
3 #include <linux/bootmem.h>
4 #include <linux/kasan.h>
5 #include <linux/kdebug.h>
7 #include <linux/sched.h>
8 #include <linux/sched/task.h>
9 #include <linux/vmalloc.h>
11 #include <asm/tlbflush.h>
12 #include <asm/sections.h>
14 extern pgd_t early_level4_pgt
[PTRS_PER_PGD
];
15 extern struct range pfn_mapped
[E820_X_MAX
];
17 static int __init
map_range(struct range
*range
)
22 start
= (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range
->start
));
23 end
= (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range
->end
));
26 * end + 1 here is intentional. We check several shadow bytes in advance
27 * to slightly speed up fastpath. In some rare cases we could cross
28 * boundary of mapped shadow, so we just map some more here.
30 return vmemmap_populate(start
, end
+ 1, NUMA_NO_NODE
);
33 static void __init
clear_pgds(unsigned long start
,
38 for (; start
< end
; start
+= PGDIR_SIZE
) {
39 pgd
= pgd_offset_k(start
);
41 * With folded p4d, pgd_clear() is nop, use p4d_clear()
44 if (CONFIG_PGTABLE_LEVELS
< 5)
45 p4d_clear(p4d_offset(pgd
, start
));
51 static void __init
kasan_map_early_shadow(pgd_t
*pgd
)
54 unsigned long start
= KASAN_SHADOW_START
;
55 unsigned long end
= KASAN_SHADOW_END
;
57 for (i
= pgd_index(start
); start
< end
; i
++) {
58 pgd
[i
] = __pgd(__pa_nodebug(kasan_zero_pud
)
64 #ifdef CONFIG_KASAN_INLINE
65 static int kasan_die_handler(struct notifier_block
*self
,
70 pr_emerg("CONFIG_KASAN_INLINE enabled\n");
71 pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n");
76 static struct notifier_block kasan_die_notifier
= {
77 .notifier_call
= kasan_die_handler
,
81 void __init
kasan_early_init(void)
84 pteval_t pte_val
= __pa_nodebug(kasan_zero_page
) | __PAGE_KERNEL
;
85 pmdval_t pmd_val
= __pa_nodebug(kasan_zero_pte
) | _KERNPG_TABLE
;
86 pudval_t pud_val
= __pa_nodebug(kasan_zero_pmd
) | _KERNPG_TABLE
;
88 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
89 kasan_zero_pte
[i
] = __pte(pte_val
);
91 for (i
= 0; i
< PTRS_PER_PMD
; i
++)
92 kasan_zero_pmd
[i
] = __pmd(pmd_val
);
94 for (i
= 0; i
< PTRS_PER_PUD
; i
++)
95 kasan_zero_pud
[i
] = __pud(pud_val
);
97 kasan_map_early_shadow(early_level4_pgt
);
98 kasan_map_early_shadow(init_level4_pgt
);
101 void __init
kasan_init(void)
105 #ifdef CONFIG_KASAN_INLINE
106 register_die_notifier(&kasan_die_notifier
);
109 memcpy(early_level4_pgt
, init_level4_pgt
, sizeof(early_level4_pgt
));
110 load_cr3(early_level4_pgt
);
113 clear_pgds(KASAN_SHADOW_START
, KASAN_SHADOW_END
);
115 kasan_populate_zero_shadow((void *)KASAN_SHADOW_START
,
116 kasan_mem_to_shadow((void *)PAGE_OFFSET
));
118 for (i
= 0; i
< E820_X_MAX
; i
++) {
119 if (pfn_mapped
[i
].end
== 0)
122 if (map_range(&pfn_mapped
[i
]))
123 panic("kasan: unable to allocate shadow!");
125 kasan_populate_zero_shadow(
126 kasan_mem_to_shadow((void *)PAGE_OFFSET
+ MAXMEM
),
127 kasan_mem_to_shadow((void *)__START_KERNEL_map
));
129 vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext
),
130 (unsigned long)kasan_mem_to_shadow(_end
),
133 kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END
),
134 (void *)KASAN_SHADOW_END
);
136 load_cr3(init_level4_pgt
);
140 * kasan_zero_page has been used as early shadow memory, thus it may
141 * contain some garbage. Now we can clear and write protect it, since
142 * after the TLB flush no one should write to it.
144 memset(kasan_zero_page
, 0, PAGE_SIZE
);
145 for (i
= 0; i
< PTRS_PER_PTE
; i
++) {
146 pte_t pte
= __pte(__pa(kasan_zero_page
) | __PAGE_KERNEL_RO
);
147 set_pte(&kasan_zero_pte
[i
], pte
);
149 /* Flush TLBs again to be sure that write protection applied. */
152 init_task
.kasan_depth
= 0;
153 pr_info("KernelAddressSanitizer initialized\n");