]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
0b24becc AR |
2 | #ifndef _LINUX_KASAN_H |
3 | #define _LINUX_KASAN_H | |
4 | ||
7a3b8353 | 5 | #include <linux/bug.h> |
2db710cc | 6 | #include <linux/kernel.h> |
34303244 | 7 | #include <linux/static_key.h> |
0b24becc AR |
8 | #include <linux/types.h> |
9 | ||
10 | struct kmem_cache; | |
11 | struct page; | |
a5af5aa8 | 12 | struct vm_struct; |
5be9b730 | 13 | struct task_struct; |
0b24becc AR |
14 | |
15 | #ifdef CONFIG_KASAN | |
16 | ||
d5750edf | 17 | #include <linux/linkage.h> |
65fddcfc | 18 | #include <asm/kasan.h> |
0b24becc | 19 | |
83c4e7a0 PA |
20 | /* kasan_data struct is used in KUnit tests for KASAN expected failures */ |
21 | struct kunit_kasan_expectation { | |
83c4e7a0 PA |
22 | bool report_found; |
23 | }; | |
24 | ||
d5750edf AK |
25 | #endif |
26 | ||
27 | #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) | |
28 | ||
29 | #include <linux/pgtable.h> | |
30 | ||
31 | /* Software KASAN implementations use shadow memory. */ | |
32 | ||
33 | #ifdef CONFIG_KASAN_SW_TAGS | |
a064cb00 AK |
34 | /* This matches KASAN_TAG_INVALID. */ |
35 | #define KASAN_SHADOW_INIT 0xFE | |
d5750edf AK |
36 | #else |
37 | #define KASAN_SHADOW_INIT 0 | |
38 | #endif | |
39 | ||
29970dc2 HL |
40 | #ifndef PTE_HWTABLE_PTRS |
41 | #define PTE_HWTABLE_PTRS 0 | |
42 | #endif | |
43 | ||
9577dd74 | 44 | extern unsigned char kasan_early_shadow_page[PAGE_SIZE]; |
cb32c9c5 DA |
45 | extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS]; |
46 | extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD]; | |
47 | extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD]; | |
9577dd74 | 48 | extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D]; |
69786cdb | 49 | |
9577dd74 | 50 | int kasan_populate_early_shadow(const void *shadow_start, |
69786cdb AR |
51 | const void *shadow_end); |
52 | ||
0b24becc AR |
53 | static inline void *kasan_mem_to_shadow(const void *addr) |
54 | { | |
55 | return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) | |
56 | + KASAN_SHADOW_OFFSET; | |
57 | } | |
58 | ||
d5750edf AK |
59 | int kasan_add_zero_shadow(void *start, unsigned long size); |
60 | void kasan_remove_zero_shadow(void *start, unsigned long size); | |
61 | ||
d73b4936 AK |
62 | /* Enable reporting bugs after kasan_disable_current() */ |
63 | extern void kasan_enable_current(void); | |
64 | ||
65 | /* Disable reporting bugs for current task */ | |
66 | extern void kasan_disable_current(void); | |
67 | ||
d5750edf AK |
68 | #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ |
69 | ||
70 | static inline int kasan_add_zero_shadow(void *start, unsigned long size) | |
71 | { | |
72 | return 0; | |
73 | } | |
74 | static inline void kasan_remove_zero_shadow(void *start, | |
75 | unsigned long size) | |
76 | {} | |
77 | ||
d73b4936 AK |
78 | static inline void kasan_enable_current(void) {} |
79 | static inline void kasan_disable_current(void) {} | |
80 | ||
d5750edf AK |
81 | #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ |
82 | ||
34303244 | 83 | #ifdef CONFIG_KASAN_HW_TAGS |
e86f8b09 | 84 | |
34303244 | 85 | DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); |
e86f8b09 | 86 | |
34303244 AK |
87 | static __always_inline bool kasan_enabled(void) |
88 | { | |
89 | return static_branch_likely(&kasan_flag_enabled); | |
90 | } | |
e86f8b09 | 91 | |
1bb5eab3 AK |
92 | static inline bool kasan_has_integrated_init(void) |
93 | { | |
94 | return kasan_enabled(); | |
95 | } | |
96 | ||
7a3b8353 PC |
97 | void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags); |
98 | void kasan_free_pages(struct page *page, unsigned int order); | |
99 | ||
e86f8b09 AK |
100 | #else /* CONFIG_KASAN_HW_TAGS */ |
101 | ||
34303244 AK |
102 | static inline bool kasan_enabled(void) |
103 | { | |
7a3b8353 | 104 | return IS_ENABLED(CONFIG_KASAN); |
34303244 | 105 | } |
e86f8b09 | 106 | |
1bb5eab3 AK |
107 | static inline bool kasan_has_integrated_init(void) |
108 | { | |
109 | return false; | |
110 | } | |
111 | ||
7a3b8353 PC |
112 | static __always_inline void kasan_alloc_pages(struct page *page, |
113 | unsigned int order, gfp_t flags) | |
114 | { | |
115 | /* Only available for integrated init. */ | |
116 | BUILD_BUG(); | |
117 | } | |
118 | ||
119 | static __always_inline void kasan_free_pages(struct page *page, | |
120 | unsigned int order) | |
121 | { | |
122 | /* Only available for integrated init. */ | |
123 | BUILD_BUG(); | |
124 | } | |
125 | ||
e86f8b09 AK |
126 | #endif /* CONFIG_KASAN_HW_TAGS */ |
127 | ||
7a3b8353 PC |
128 | #ifdef CONFIG_KASAN |
129 | ||
130 | struct kasan_cache { | |
131 | int alloc_meta_offset; | |
132 | int free_meta_offset; | |
133 | bool is_kmalloc; | |
134 | }; | |
135 | ||
e86f8b09 AK |
136 | slab_flags_t __kasan_never_merge(void); |
137 | static __always_inline slab_flags_t kasan_never_merge(void) | |
138 | { | |
139 | if (kasan_enabled()) | |
140 | return __kasan_never_merge(); | |
141 | return 0; | |
142 | } | |
b8c73fc2 | 143 | |
34303244 AK |
144 | void __kasan_unpoison_range(const void *addr, size_t size); |
145 | static __always_inline void kasan_unpoison_range(const void *addr, size_t size) | |
146 | { | |
147 | if (kasan_enabled()) | |
148 | __kasan_unpoison_range(addr, size); | |
149 | } | |
7ed2f9e6 | 150 | |
7a3b8353 PC |
151 | void __kasan_poison_pages(struct page *page, unsigned int order, bool init); |
152 | static __always_inline void kasan_poison_pages(struct page *page, | |
1bb5eab3 | 153 | unsigned int order, bool init) |
34303244 AK |
154 | { |
155 | if (kasan_enabled()) | |
7a3b8353 | 156 | __kasan_poison_pages(page, order, init); |
34303244 | 157 | } |
0316bec2 | 158 | |
7a3b8353 PC |
159 | void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init); |
160 | static __always_inline void kasan_unpoison_pages(struct page *page, | |
161 | unsigned int order, bool init) | |
34303244 AK |
162 | { |
163 | if (kasan_enabled()) | |
7a3b8353 | 164 | __kasan_unpoison_pages(page, order, init); |
34303244 | 165 | } |
0316bec2 | 166 | |
34303244 AK |
167 | void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size, |
168 | slab_flags_t *flags); | |
169 | static __always_inline void kasan_cache_create(struct kmem_cache *cache, | |
170 | unsigned int *size, slab_flags_t *flags) | |
171 | { | |
172 | if (kasan_enabled()) | |
173 | __kasan_cache_create(cache, size, flags); | |
174 | } | |
0316bec2 | 175 | |
92850134 AK |
176 | void __kasan_cache_create_kmalloc(struct kmem_cache *cache); |
177 | static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) | |
178 | { | |
179 | if (kasan_enabled()) | |
180 | __kasan_cache_create_kmalloc(cache); | |
181 | } | |
182 | ||
34303244 AK |
183 | size_t __kasan_metadata_size(struct kmem_cache *cache); |
184 | static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache) | |
185 | { | |
186 | if (kasan_enabled()) | |
187 | return __kasan_metadata_size(cache); | |
188 | return 0; | |
189 | } | |
7ed2f9e6 | 190 | |
34303244 AK |
191 | void __kasan_poison_slab(struct page *page); |
192 | static __always_inline void kasan_poison_slab(struct page *page) | |
193 | { | |
194 | if (kasan_enabled()) | |
195 | __kasan_poison_slab(page); | |
196 | } | |
197 | ||
198 | void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object); | |
199 | static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache, | |
200 | void *object) | |
201 | { | |
202 | if (kasan_enabled()) | |
203 | __kasan_unpoison_object_data(cache, object); | |
204 | } | |
205 | ||
206 | void __kasan_poison_object_data(struct kmem_cache *cache, void *object); | |
207 | static __always_inline void kasan_poison_object_data(struct kmem_cache *cache, | |
208 | void *object) | |
209 | { | |
210 | if (kasan_enabled()) | |
211 | __kasan_poison_object_data(cache, object); | |
212 | } | |
213 | ||
214 | void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache, | |
215 | const void *object); | |
216 | static __always_inline void * __must_check kasan_init_slab_obj( | |
217 | struct kmem_cache *cache, const void *object) | |
218 | { | |
219 | if (kasan_enabled()) | |
220 | return __kasan_init_slab_obj(cache, object); | |
221 | return (void *)object; | |
222 | } | |
223 | ||
d57a964e AK |
224 | bool __kasan_slab_free(struct kmem_cache *s, void *object, |
225 | unsigned long ip, bool init); | |
226 | static __always_inline bool kasan_slab_free(struct kmem_cache *s, | |
227 | void *object, bool init) | |
34303244 AK |
228 | { |
229 | if (kasan_enabled()) | |
d57a964e | 230 | return __kasan_slab_free(s, object, _RET_IP_, init); |
34303244 AK |
231 | return false; |
232 | } | |
233 | ||
200072ce AK |
234 | void __kasan_kfree_large(void *ptr, unsigned long ip); |
235 | static __always_inline void kasan_kfree_large(void *ptr) | |
236 | { | |
237 | if (kasan_enabled()) | |
238 | __kasan_kfree_large(ptr, _RET_IP_); | |
239 | } | |
240 | ||
eeb3160c | 241 | void __kasan_slab_free_mempool(void *ptr, unsigned long ip); |
027b37b5 | 242 | static __always_inline void kasan_slab_free_mempool(void *ptr) |
eeb3160c AK |
243 | { |
244 | if (kasan_enabled()) | |
027b37b5 | 245 | __kasan_slab_free_mempool(ptr, _RET_IP_); |
eeb3160c AK |
246 | } |
247 | ||
34303244 | 248 | void * __must_check __kasan_slab_alloc(struct kmem_cache *s, |
da844b78 | 249 | void *object, gfp_t flags, bool init); |
34303244 | 250 | static __always_inline void * __must_check kasan_slab_alloc( |
da844b78 | 251 | struct kmem_cache *s, void *object, gfp_t flags, bool init) |
34303244 AK |
252 | { |
253 | if (kasan_enabled()) | |
da844b78 | 254 | return __kasan_slab_alloc(s, object, flags, init); |
34303244 AK |
255 | return object; |
256 | } | |
257 | ||
258 | void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object, | |
259 | size_t size, gfp_t flags); | |
260 | static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s, | |
261 | const void *object, size_t size, gfp_t flags) | |
262 | { | |
263 | if (kasan_enabled()) | |
264 | return __kasan_kmalloc(s, object, size, flags); | |
265 | return (void *)object; | |
266 | } | |
267 | ||
268 | void * __must_check __kasan_kmalloc_large(const void *ptr, | |
269 | size_t size, gfp_t flags); | |
270 | static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr, | |
271 | size_t size, gfp_t flags) | |
272 | { | |
273 | if (kasan_enabled()) | |
274 | return __kasan_kmalloc_large(ptr, size, flags); | |
275 | return (void *)ptr; | |
276 | } | |
277 | ||
278 | void * __must_check __kasan_krealloc(const void *object, | |
279 | size_t new_size, gfp_t flags); | |
280 | static __always_inline void * __must_check kasan_krealloc(const void *object, | |
281 | size_t new_size, gfp_t flags) | |
282 | { | |
283 | if (kasan_enabled()) | |
284 | return __kasan_krealloc(object, new_size, flags); | |
285 | return (void *)object; | |
286 | } | |
287 | ||
611806b4 AK |
288 | /* |
289 | * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for | |
290 | * the hardware tag-based mode that doesn't rely on compiler instrumentation. | |
291 | */ | |
292 | bool __kasan_check_byte(const void *addr, unsigned long ip); | |
293 | static __always_inline bool kasan_check_byte(const void *addr) | |
294 | { | |
295 | if (kasan_enabled()) | |
296 | return __kasan_check_byte(addr, _RET_IP_); | |
297 | return true; | |
298 | } | |
299 | ||
300 | ||
b0845ce5 MR |
301 | bool kasan_save_enable_multi_shot(void); |
302 | void kasan_restore_multi_shot(bool enabled); | |
303 | ||
0b24becc AR |
304 | #else /* CONFIG_KASAN */ |
305 | ||
e86f8b09 AK |
306 | static inline slab_flags_t kasan_never_merge(void) |
307 | { | |
308 | return 0; | |
309 | } | |
cebd0eb2 | 310 | static inline void kasan_unpoison_range(const void *address, size_t size) {} |
7a3b8353 PC |
311 | static inline void kasan_poison_pages(struct page *page, unsigned int order, |
312 | bool init) {} | |
313 | static inline void kasan_unpoison_pages(struct page *page, unsigned int order, | |
314 | bool init) {} | |
7ed2f9e6 | 315 | static inline void kasan_cache_create(struct kmem_cache *cache, |
be4a7988 | 316 | unsigned int *size, |
d50112ed | 317 | slab_flags_t *flags) {} |
92850134 | 318 | static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {} |
34303244 | 319 | static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; } |
0316bec2 AR |
320 | static inline void kasan_poison_slab(struct page *page) {} |
321 | static inline void kasan_unpoison_object_data(struct kmem_cache *cache, | |
322 | void *object) {} | |
323 | static inline void kasan_poison_object_data(struct kmem_cache *cache, | |
324 | void *object) {} | |
0116523c AK |
325 | static inline void *kasan_init_slab_obj(struct kmem_cache *cache, |
326 | const void *object) | |
327 | { | |
328 | return (void *)object; | |
329 | } | |
d57a964e | 330 | static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init) |
34303244 AK |
331 | { |
332 | return false; | |
333 | } | |
200072ce | 334 | static inline void kasan_kfree_large(void *ptr) {} |
027b37b5 | 335 | static inline void kasan_slab_free_mempool(void *ptr) {} |
34303244 | 336 | static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object, |
da844b78 | 337 | gfp_t flags, bool init) |
0116523c | 338 | { |
34303244 | 339 | return object; |
0116523c | 340 | } |
0116523c AK |
341 | static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object, |
342 | size_t size, gfp_t flags) | |
343 | { | |
344 | return (void *)object; | |
345 | } | |
34303244 AK |
346 | static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags) |
347 | { | |
348 | return (void *)ptr; | |
349 | } | |
0116523c AK |
350 | static inline void *kasan_krealloc(const void *object, size_t new_size, |
351 | gfp_t flags) | |
352 | { | |
353 | return (void *)object; | |
354 | } | |
611806b4 AK |
355 | static inline bool kasan_check_byte(const void *address) |
356 | { | |
357 | return true; | |
358 | } | |
9b75a867 | 359 | |
0b24becc AR |
360 | #endif /* CONFIG_KASAN */ |
361 | ||
02c58773 | 362 | #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK) |
d56a9ef8 AK |
363 | void kasan_unpoison_task_stack(struct task_struct *task); |
364 | #else | |
365 | static inline void kasan_unpoison_task_stack(struct task_struct *task) {} | |
366 | #endif | |
367 | ||
2bd926b4 AK |
368 | #ifdef CONFIG_KASAN_GENERIC |
369 | ||
370 | void kasan_cache_shrink(struct kmem_cache *cache); | |
371 | void kasan_cache_shutdown(struct kmem_cache *cache); | |
26e760c9 | 372 | void kasan_record_aux_stack(void *ptr); |
2bd926b4 AK |
373 | |
374 | #else /* CONFIG_KASAN_GENERIC */ | |
375 | ||
376 | static inline void kasan_cache_shrink(struct kmem_cache *cache) {} | |
377 | static inline void kasan_cache_shutdown(struct kmem_cache *cache) {} | |
26e760c9 | 378 | static inline void kasan_record_aux_stack(void *ptr) {} |
2bd926b4 AK |
379 | |
380 | #endif /* CONFIG_KASAN_GENERIC */ | |
381 | ||
2e903b91 | 382 | #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) |
3c9e3aa1 | 383 | |
c0054c56 AK |
384 | static inline void *kasan_reset_tag(const void *addr) |
385 | { | |
386 | return (void *)arch_kasan_reset_tag(addr); | |
387 | } | |
3c9e3aa1 | 388 | |
49c6631d VF |
389 | /** |
390 | * kasan_report - print a report about a bad memory access detected by KASAN | |
391 | * @addr: address of the bad access | |
392 | * @size: size of the bad access | |
393 | * @is_write: whether the bad access is a write or a read | |
394 | * @ip: instruction pointer for the accessibility check or the bad access itself | |
395 | */ | |
8cceeff4 | 396 | bool kasan_report(unsigned long addr, size_t size, |
41eea9cd AK |
397 | bool is_write, unsigned long ip); |
398 | ||
2e903b91 | 399 | #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ |
3c9e3aa1 AK |
400 | |
401 | static inline void *kasan_reset_tag(const void *addr) | |
402 | { | |
403 | return (void *)addr; | |
404 | } | |
405 | ||
2e903b91 AK |
406 | #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/ |
407 | ||
8f7b5054 VF |
408 | #ifdef CONFIG_KASAN_HW_TAGS |
409 | ||
410 | void kasan_report_async(void); | |
411 | ||
412 | #endif /* CONFIG_KASAN_HW_TAGS */ | |
413 | ||
2e903b91 AK |
414 | #ifdef CONFIG_KASAN_SW_TAGS |
415 | void __init kasan_init_sw_tags(void); | |
416 | #else | |
417 | static inline void kasan_init_sw_tags(void) { } | |
418 | #endif | |
419 | ||
420 | #ifdef CONFIG_KASAN_HW_TAGS | |
421 | void kasan_init_hw_tags_cpu(void); | |
422 | void __init kasan_init_hw_tags(void); | |
423 | #else | |
424 | static inline void kasan_init_hw_tags_cpu(void) { } | |
425 | static inline void kasan_init_hw_tags(void) { } | |
426 | #endif | |
080eb83f | 427 | |
3c5c3cfb | 428 | #ifdef CONFIG_KASAN_VMALLOC |
3b1a4a86 | 429 | |
d98c9e83 AR |
430 | int kasan_populate_vmalloc(unsigned long addr, unsigned long size); |
431 | void kasan_poison_vmalloc(const void *start, unsigned long size); | |
432 | void kasan_unpoison_vmalloc(const void *start, unsigned long size); | |
3c5c3cfb DA |
433 | void kasan_release_vmalloc(unsigned long start, unsigned long end, |
434 | unsigned long free_region_start, | |
435 | unsigned long free_region_end); | |
3b1a4a86 AK |
436 | |
437 | #else /* CONFIG_KASAN_VMALLOC */ | |
438 | ||
d98c9e83 AR |
439 | static inline int kasan_populate_vmalloc(unsigned long start, |
440 | unsigned long size) | |
3c5c3cfb DA |
441 | { |
442 | return 0; | |
443 | } | |
444 | ||
d98c9e83 AR |
445 | static inline void kasan_poison_vmalloc(const void *start, unsigned long size) |
446 | { } | |
447 | static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size) | |
448 | { } | |
3c5c3cfb DA |
449 | static inline void kasan_release_vmalloc(unsigned long start, |
450 | unsigned long end, | |
451 | unsigned long free_region_start, | |
452 | unsigned long free_region_end) {} | |
3b1a4a86 AK |
453 | |
454 | #endif /* CONFIG_KASAN_VMALLOC */ | |
455 | ||
0fea6e9a AK |
456 | #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ |
457 | !defined(CONFIG_KASAN_VMALLOC) | |
3b1a4a86 AK |
458 | |
459 | /* | |
460 | * These functions provide a special case to support backing module | |
461 | * allocations with real shadow memory. With KASAN vmalloc, the special | |
462 | * case is unnecessary, as the work is handled in the generic case. | |
463 | */ | |
464 | int kasan_module_alloc(void *addr, size_t size); | |
465 | void kasan_free_shadow(const struct vm_struct *vm); | |
466 | ||
0fea6e9a | 467 | #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */ |
3b1a4a86 AK |
468 | |
469 | static inline int kasan_module_alloc(void *addr, size_t size) { return 0; } | |
470 | static inline void kasan_free_shadow(const struct vm_struct *vm) {} | |
471 | ||
0fea6e9a | 472 | #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */ |
3c5c3cfb | 473 | |
2f004eea JH |
474 | #ifdef CONFIG_KASAN_INLINE |
475 | void kasan_non_canonical_hook(unsigned long addr); | |
476 | #else /* CONFIG_KASAN_INLINE */ | |
477 | static inline void kasan_non_canonical_hook(unsigned long addr) { } | |
478 | #endif /* CONFIG_KASAN_INLINE */ | |
479 | ||
0b24becc | 480 | #endif /* LINUX_KASAN_H */ |