]>
git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - arch/arm64/mm/kasan_init.c
2 * This file contains kasan initialization code for ARM64.
4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
13 #define pr_fmt(fmt) "kasan: " fmt
14 #include <linux/kasan.h>
15 #include <linux/kernel.h>
16 #include <linux/sched/task.h>
17 #include <linux/memblock.h>
18 #include <linux/start_kernel.h>
21 #include <asm/mmu_context.h>
22 #include <asm/kernel-pgtable.h>
24 #include <asm/pgalloc.h>
25 #include <asm/pgtable.h>
26 #include <asm/sections.h>
27 #include <asm/tlbflush.h>
29 static pgd_t tmp_pg_dir
[PTRS_PER_PGD
] __initdata
__aligned(PGD_SIZE
);
32 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
33 * directly on kernel symbols (bm_p*d). All the early functions are called too
34 * early to use lm_alias so __p*d_populate functions must be used to populate
35 * with the physical address from __pa_symbol.
38 static phys_addr_t __init
kasan_alloc_zeroed_page(int node
)
40 void *p
= memblock_alloc_try_nid(PAGE_SIZE
, PAGE_SIZE
,
41 __pa(MAX_DMA_ADDRESS
),
42 MEMBLOCK_ALLOC_KASAN
, node
);
46 static phys_addr_t __init
kasan_alloc_raw_page(int node
)
48 void *p
= memblock_alloc_try_nid_raw(PAGE_SIZE
, PAGE_SIZE
,
49 __pa(MAX_DMA_ADDRESS
),
50 MEMBLOCK_ALLOC_KASAN
, node
);
54 static pte_t
*__init
kasan_pte_offset(pmd_t
*pmdp
, unsigned long addr
, int node
,
57 if (pmd_none(READ_ONCE(*pmdp
))) {
58 phys_addr_t pte_phys
= early
?
59 __pa_symbol(kasan_early_shadow_pte
)
60 : kasan_alloc_zeroed_page(node
);
61 __pmd_populate(pmdp
, pte_phys
, PMD_TYPE_TABLE
);
64 return early
? pte_offset_kimg(pmdp
, addr
)
65 : pte_offset_kernel(pmdp
, addr
);
68 static pmd_t
*__init
kasan_pmd_offset(pud_t
*pudp
, unsigned long addr
, int node
,
71 if (pud_none(READ_ONCE(*pudp
))) {
72 phys_addr_t pmd_phys
= early
?
73 __pa_symbol(kasan_early_shadow_pmd
)
74 : kasan_alloc_zeroed_page(node
);
75 __pud_populate(pudp
, pmd_phys
, PMD_TYPE_TABLE
);
78 return early
? pmd_offset_kimg(pudp
, addr
) : pmd_offset(pudp
, addr
);
81 static pud_t
*__init
kasan_pud_offset(pgd_t
*pgdp
, unsigned long addr
, int node
,
84 if (pgd_none(READ_ONCE(*pgdp
))) {
85 phys_addr_t pud_phys
= early
?
86 __pa_symbol(kasan_early_shadow_pud
)
87 : kasan_alloc_zeroed_page(node
);
88 __pgd_populate(pgdp
, pud_phys
, PMD_TYPE_TABLE
);
91 return early
? pud_offset_kimg(pgdp
, addr
) : pud_offset(pgdp
, addr
);
94 static void __init
kasan_pte_populate(pmd_t
*pmdp
, unsigned long addr
,
95 unsigned long end
, int node
, bool early
)
98 pte_t
*ptep
= kasan_pte_offset(pmdp
, addr
, node
, early
);
101 phys_addr_t page_phys
= early
?
102 __pa_symbol(kasan_early_shadow_page
)
103 : kasan_alloc_raw_page(node
);
105 memset(__va(page_phys
), KASAN_SHADOW_INIT
, PAGE_SIZE
);
106 next
= addr
+ PAGE_SIZE
;
107 set_pte(ptep
, pfn_pte(__phys_to_pfn(page_phys
), PAGE_KERNEL
));
108 } while (ptep
++, addr
= next
, addr
!= end
&& pte_none(READ_ONCE(*ptep
)));
111 static void __init
kasan_pmd_populate(pud_t
*pudp
, unsigned long addr
,
112 unsigned long end
, int node
, bool early
)
115 pmd_t
*pmdp
= kasan_pmd_offset(pudp
, addr
, node
, early
);
118 next
= pmd_addr_end(addr
, end
);
119 kasan_pte_populate(pmdp
, addr
, next
, node
, early
);
120 } while (pmdp
++, addr
= next
, addr
!= end
&& pmd_none(READ_ONCE(*pmdp
)));
123 static void __init
kasan_pud_populate(pgd_t
*pgdp
, unsigned long addr
,
124 unsigned long end
, int node
, bool early
)
127 pud_t
*pudp
= kasan_pud_offset(pgdp
, addr
, node
, early
);
130 next
= pud_addr_end(addr
, end
);
131 kasan_pmd_populate(pudp
, addr
, next
, node
, early
);
132 } while (pudp
++, addr
= next
, addr
!= end
&& pud_none(READ_ONCE(*pudp
)));
135 static void __init
kasan_pgd_populate(unsigned long addr
, unsigned long end
,
136 int node
, bool early
)
141 pgdp
= pgd_offset_k(addr
);
143 next
= pgd_addr_end(addr
, end
);
144 kasan_pud_populate(pgdp
, addr
, next
, node
, early
);
145 } while (pgdp
++, addr
= next
, addr
!= end
);
148 /* The early shadow maps everything to a single page of zeroes */
149 asmlinkage
void __init
kasan_early_init(void)
151 BUILD_BUG_ON(KASAN_SHADOW_OFFSET
!=
152 KASAN_SHADOW_END
- (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT
)));
153 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START
, PGDIR_SIZE
));
154 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END
, PGDIR_SIZE
));
155 kasan_pgd_populate(KASAN_SHADOW_START
, KASAN_SHADOW_END
, NUMA_NO_NODE
,
159 /* Set up full kasan mappings, ensuring that the mapped pages are zeroed */
160 static void __init
kasan_map_populate(unsigned long start
, unsigned long end
,
163 kasan_pgd_populate(start
& PAGE_MASK
, PAGE_ALIGN(end
), node
, false);
167 * Copy the current shadow region into a new pgdir.
169 void __init
kasan_copy_shadow(pgd_t
*pgdir
)
171 pgd_t
*pgdp
, *pgdp_new
, *pgdp_end
;
173 pgdp
= pgd_offset_k(KASAN_SHADOW_START
);
174 pgdp_end
= pgd_offset_k(KASAN_SHADOW_END
);
175 pgdp_new
= pgd_offset_raw(pgdir
, KASAN_SHADOW_START
);
177 set_pgd(pgdp_new
, READ_ONCE(*pgdp
));
178 } while (pgdp
++, pgdp_new
++, pgdp
!= pgdp_end
);
181 static void __init
clear_pgds(unsigned long start
,
185 * Remove references to kasan page tables from
186 * swapper_pg_dir. pgd_clear() can't be used
187 * here because it's nop on 2,3-level pagetable setups
189 for (; start
< end
; start
+= PGDIR_SIZE
)
190 set_pgd(pgd_offset_k(start
), __pgd(0));
193 void __init
kasan_init(void)
195 u64 kimg_shadow_start
, kimg_shadow_end
;
196 u64 mod_shadow_start
, mod_shadow_end
;
197 struct memblock_region
*reg
;
200 kimg_shadow_start
= (u64
)kasan_mem_to_shadow(_text
) & PAGE_MASK
;
201 kimg_shadow_end
= PAGE_ALIGN((u64
)kasan_mem_to_shadow(_end
));
203 mod_shadow_start
= (u64
)kasan_mem_to_shadow((void *)MODULES_VADDR
);
204 mod_shadow_end
= (u64
)kasan_mem_to_shadow((void *)MODULES_END
);
207 * We are going to perform proper setup of shadow memory.
208 * At first we should unmap early shadow (clear_pgds() call below).
209 * However, instrumented code couldn't execute without shadow memory.
210 * tmp_pg_dir used to keep early shadow mapped until full shadow
211 * setup will be finished.
213 memcpy(tmp_pg_dir
, swapper_pg_dir
, sizeof(tmp_pg_dir
));
215 cpu_replace_ttbr1(lm_alias(tmp_pg_dir
));
217 clear_pgds(KASAN_SHADOW_START
, KASAN_SHADOW_END
);
219 kasan_map_populate(kimg_shadow_start
, kimg_shadow_end
,
220 early_pfn_to_nid(virt_to_pfn(lm_alias(_text
))));
222 kasan_populate_early_shadow((void *)KASAN_SHADOW_START
,
223 (void *)mod_shadow_start
);
224 kasan_populate_early_shadow((void *)kimg_shadow_end
,
225 kasan_mem_to_shadow((void *)PAGE_OFFSET
));
227 if (kimg_shadow_start
> mod_shadow_end
)
228 kasan_populate_early_shadow((void *)mod_shadow_end
,
229 (void *)kimg_shadow_start
);
231 for_each_memblock(memory
, reg
) {
232 void *start
= (void *)__phys_to_virt(reg
->base
);
233 void *end
= (void *)__phys_to_virt(reg
->base
+ reg
->size
);
238 kasan_map_populate((unsigned long)kasan_mem_to_shadow(start
),
239 (unsigned long)kasan_mem_to_shadow(end
),
240 early_pfn_to_nid(virt_to_pfn(start
)));
244 * KAsan may reuse the contents of kasan_early_shadow_pte directly,
245 * so we should make sure that it maps the zero page read-only.
247 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
248 set_pte(&kasan_early_shadow_pte
[i
],
249 pfn_pte(sym_to_pfn(kasan_early_shadow_page
),
252 memset(kasan_early_shadow_page
, KASAN_SHADOW_INIT
, PAGE_SIZE
);
253 cpu_replace_ttbr1(lm_alias(swapper_pg_dir
));
255 /* At this point kasan is fully initialized. Enable error messages */
256 init_task
.kasan_depth
= 0;
257 pr_info("KernelAddressSanitizer initialized\n");