1 /* SPDX-License-Identifier: GPL-2.0 */
6 #include <linux/kernel.h>
7 #include <linux/static_key.h>
8 #include <linux/types.h>
17 #include <linux/linkage.h>
18 #include <asm/kasan.h>
20 /* kasan_data struct is used in KUnit tests for KASAN expected failures */
21 struct kunit_kasan_expectation
{
27 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
29 #include <linux/pgtable.h>
31 /* Software KASAN implementations use shadow memory. */
33 #ifdef CONFIG_KASAN_SW_TAGS
34 /* This matches KASAN_TAG_INVALID. */
35 #define KASAN_SHADOW_INIT 0xFE
37 #define KASAN_SHADOW_INIT 0
40 #ifndef PTE_HWTABLE_PTRS
41 #define PTE_HWTABLE_PTRS 0
44 extern unsigned char kasan_early_shadow_page
[PAGE_SIZE
];
45 extern pte_t kasan_early_shadow_pte
[MAX_PTRS_PER_PTE
+ PTE_HWTABLE_PTRS
];
46 extern pmd_t kasan_early_shadow_pmd
[MAX_PTRS_PER_PMD
];
47 extern pud_t kasan_early_shadow_pud
[MAX_PTRS_PER_PUD
];
48 extern p4d_t kasan_early_shadow_p4d
[MAX_PTRS_PER_P4D
];
50 int kasan_populate_early_shadow(const void *shadow_start
,
51 const void *shadow_end
);
53 static inline void *kasan_mem_to_shadow(const void *addr
)
55 return (void *)((unsigned long)addr
>> KASAN_SHADOW_SCALE_SHIFT
)
56 + KASAN_SHADOW_OFFSET
;
59 int kasan_add_zero_shadow(void *start
, unsigned long size
);
60 void kasan_remove_zero_shadow(void *start
, unsigned long size
);
62 /* Enable reporting bugs after kasan_disable_current() */
63 extern void kasan_enable_current(void);
65 /* Disable reporting bugs for current task */
66 extern void kasan_disable_current(void);
68 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
70 static inline int kasan_add_zero_shadow(void *start
, unsigned long size
)
74 static inline void kasan_remove_zero_shadow(void *start
,
78 static inline void kasan_enable_current(void) {}
79 static inline void kasan_disable_current(void) {}
81 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
83 #ifdef CONFIG_KASAN_HW_TAGS
85 DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled
);
87 static __always_inline
bool kasan_enabled(void)
89 return static_branch_likely(&kasan_flag_enabled
);
92 static inline bool kasan_has_integrated_init(void)
94 return kasan_enabled();
97 void kasan_alloc_pages(struct page
*page
, unsigned int order
, gfp_t flags
);
98 void kasan_free_pages(struct page
*page
, unsigned int order
);
100 #else /* CONFIG_KASAN_HW_TAGS */
102 static inline bool kasan_enabled(void)
104 return IS_ENABLED(CONFIG_KASAN
);
107 static inline bool kasan_has_integrated_init(void)
112 static __always_inline
void kasan_alloc_pages(struct page
*page
,
113 unsigned int order
, gfp_t flags
)
115 /* Only available for integrated init. */
119 static __always_inline
void kasan_free_pages(struct page
*page
,
122 /* Only available for integrated init. */
126 #endif /* CONFIG_KASAN_HW_TAGS */
131 int alloc_meta_offset
;
132 int free_meta_offset
;
136 slab_flags_t
__kasan_never_merge(void);
137 static __always_inline slab_flags_t
kasan_never_merge(void)
140 return __kasan_never_merge();
144 void __kasan_unpoison_range(const void *addr
, size_t size
);
145 static __always_inline
void kasan_unpoison_range(const void *addr
, size_t size
)
148 __kasan_unpoison_range(addr
, size
);
151 void __kasan_poison_pages(struct page
*page
, unsigned int order
, bool init
);
152 static __always_inline
void kasan_poison_pages(struct page
*page
,
153 unsigned int order
, bool init
)
156 __kasan_poison_pages(page
, order
, init
);
159 void __kasan_unpoison_pages(struct page
*page
, unsigned int order
, bool init
);
160 static __always_inline
void kasan_unpoison_pages(struct page
*page
,
161 unsigned int order
, bool init
)
164 __kasan_unpoison_pages(page
, order
, init
);
167 void __kasan_cache_create(struct kmem_cache
*cache
, unsigned int *size
,
168 slab_flags_t
*flags
);
169 static __always_inline
void kasan_cache_create(struct kmem_cache
*cache
,
170 unsigned int *size
, slab_flags_t
*flags
)
173 __kasan_cache_create(cache
, size
, flags
);
176 void __kasan_cache_create_kmalloc(struct kmem_cache
*cache
);
177 static __always_inline
void kasan_cache_create_kmalloc(struct kmem_cache
*cache
)
180 __kasan_cache_create_kmalloc(cache
);
183 size_t __kasan_metadata_size(struct kmem_cache
*cache
);
184 static __always_inline
size_t kasan_metadata_size(struct kmem_cache
*cache
)
187 return __kasan_metadata_size(cache
);
191 void __kasan_poison_slab(struct page
*page
);
192 static __always_inline
void kasan_poison_slab(struct page
*page
)
195 __kasan_poison_slab(page
);
198 void __kasan_unpoison_object_data(struct kmem_cache
*cache
, void *object
);
199 static __always_inline
void kasan_unpoison_object_data(struct kmem_cache
*cache
,
203 __kasan_unpoison_object_data(cache
, object
);
206 void __kasan_poison_object_data(struct kmem_cache
*cache
, void *object
);
207 static __always_inline
void kasan_poison_object_data(struct kmem_cache
*cache
,
211 __kasan_poison_object_data(cache
, object
);
214 void * __must_check
__kasan_init_slab_obj(struct kmem_cache
*cache
,
216 static __always_inline
void * __must_check
kasan_init_slab_obj(
217 struct kmem_cache
*cache
, const void *object
)
220 return __kasan_init_slab_obj(cache
, object
);
221 return (void *)object
;
224 bool __kasan_slab_free(struct kmem_cache
*s
, void *object
,
225 unsigned long ip
, bool init
);
226 static __always_inline
bool kasan_slab_free(struct kmem_cache
*s
,
227 void *object
, bool init
)
230 return __kasan_slab_free(s
, object
, _RET_IP_
, init
);
234 void __kasan_kfree_large(void *ptr
, unsigned long ip
);
235 static __always_inline
void kasan_kfree_large(void *ptr
)
238 __kasan_kfree_large(ptr
, _RET_IP_
);
241 void __kasan_slab_free_mempool(void *ptr
, unsigned long ip
);
242 static __always_inline
void kasan_slab_free_mempool(void *ptr
)
245 __kasan_slab_free_mempool(ptr
, _RET_IP_
);
248 void * __must_check
__kasan_slab_alloc(struct kmem_cache
*s
,
249 void *object
, gfp_t flags
, bool init
);
250 static __always_inline
void * __must_check
kasan_slab_alloc(
251 struct kmem_cache
*s
, void *object
, gfp_t flags
, bool init
)
254 return __kasan_slab_alloc(s
, object
, flags
, init
);
258 void * __must_check
__kasan_kmalloc(struct kmem_cache
*s
, const void *object
,
259 size_t size
, gfp_t flags
);
260 static __always_inline
void * __must_check
kasan_kmalloc(struct kmem_cache
*s
,
261 const void *object
, size_t size
, gfp_t flags
)
264 return __kasan_kmalloc(s
, object
, size
, flags
);
265 return (void *)object
;
268 void * __must_check
__kasan_kmalloc_large(const void *ptr
,
269 size_t size
, gfp_t flags
);
270 static __always_inline
void * __must_check
kasan_kmalloc_large(const void *ptr
,
271 size_t size
, gfp_t flags
)
274 return __kasan_kmalloc_large(ptr
, size
, flags
);
278 void * __must_check
__kasan_krealloc(const void *object
,
279 size_t new_size
, gfp_t flags
);
280 static __always_inline
void * __must_check
kasan_krealloc(const void *object
,
281 size_t new_size
, gfp_t flags
)
284 return __kasan_krealloc(object
, new_size
, flags
);
285 return (void *)object
;
289 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
290 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
292 bool __kasan_check_byte(const void *addr
, unsigned long ip
);
293 static __always_inline
bool kasan_check_byte(const void *addr
)
296 return __kasan_check_byte(addr
, _RET_IP_
);
301 bool kasan_save_enable_multi_shot(void);
302 void kasan_restore_multi_shot(bool enabled
);
304 #else /* CONFIG_KASAN */
306 static inline slab_flags_t
kasan_never_merge(void)
310 static inline void kasan_unpoison_range(const void *address
, size_t size
) {}
311 static inline void kasan_poison_pages(struct page
*page
, unsigned int order
,
313 static inline void kasan_unpoison_pages(struct page
*page
, unsigned int order
,
315 static inline void kasan_cache_create(struct kmem_cache
*cache
,
317 slab_flags_t
*flags
) {}
318 static inline void kasan_cache_create_kmalloc(struct kmem_cache
*cache
) {}
319 static inline size_t kasan_metadata_size(struct kmem_cache
*cache
) { return 0; }
320 static inline void kasan_poison_slab(struct page
*page
) {}
321 static inline void kasan_unpoison_object_data(struct kmem_cache
*cache
,
323 static inline void kasan_poison_object_data(struct kmem_cache
*cache
,
325 static inline void *kasan_init_slab_obj(struct kmem_cache
*cache
,
328 return (void *)object
;
330 static inline bool kasan_slab_free(struct kmem_cache
*s
, void *object
, bool init
)
334 static inline void kasan_kfree_large(void *ptr
) {}
335 static inline void kasan_slab_free_mempool(void *ptr
) {}
336 static inline void *kasan_slab_alloc(struct kmem_cache
*s
, void *object
,
337 gfp_t flags
, bool init
)
341 static inline void *kasan_kmalloc(struct kmem_cache
*s
, const void *object
,
342 size_t size
, gfp_t flags
)
344 return (void *)object
;
346 static inline void *kasan_kmalloc_large(const void *ptr
, size_t size
, gfp_t flags
)
350 static inline void *kasan_krealloc(const void *object
, size_t new_size
,
353 return (void *)object
;
355 static inline bool kasan_check_byte(const void *address
)
360 #endif /* CONFIG_KASAN */
362 #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
363 void kasan_unpoison_task_stack(struct task_struct
*task
);
365 static inline void kasan_unpoison_task_stack(struct task_struct
*task
) {}
368 #ifdef CONFIG_KASAN_GENERIC
370 void kasan_cache_shrink(struct kmem_cache
*cache
);
371 void kasan_cache_shutdown(struct kmem_cache
*cache
);
372 void kasan_record_aux_stack(void *ptr
);
374 #else /* CONFIG_KASAN_GENERIC */
376 static inline void kasan_cache_shrink(struct kmem_cache
*cache
) {}
377 static inline void kasan_cache_shutdown(struct kmem_cache
*cache
) {}
378 static inline void kasan_record_aux_stack(void *ptr
) {}
380 #endif /* CONFIG_KASAN_GENERIC */
382 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
384 static inline void *kasan_reset_tag(const void *addr
)
386 return (void *)arch_kasan_reset_tag(addr
);
390 * kasan_report - print a report about a bad memory access detected by KASAN
391 * @addr: address of the bad access
392 * @size: size of the bad access
393 * @is_write: whether the bad access is a write or a read
394 * @ip: instruction pointer for the accessibility check or the bad access itself
396 bool kasan_report(unsigned long addr
, size_t size
,
397 bool is_write
, unsigned long ip
);
399 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
401 static inline void *kasan_reset_tag(const void *addr
)
406 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
408 #ifdef CONFIG_KASAN_HW_TAGS
410 void kasan_report_async(void);
412 #endif /* CONFIG_KASAN_HW_TAGS */
414 #ifdef CONFIG_KASAN_SW_TAGS
415 void __init
kasan_init_sw_tags(void);
417 static inline void kasan_init_sw_tags(void) { }
420 #ifdef CONFIG_KASAN_HW_TAGS
421 void kasan_init_hw_tags_cpu(void);
422 void __init
kasan_init_hw_tags(void);
424 static inline void kasan_init_hw_tags_cpu(void) { }
425 static inline void kasan_init_hw_tags(void) { }
428 #ifdef CONFIG_KASAN_VMALLOC
430 int kasan_populate_vmalloc(unsigned long addr
, unsigned long size
);
431 void kasan_poison_vmalloc(const void *start
, unsigned long size
);
432 void kasan_unpoison_vmalloc(const void *start
, unsigned long size
);
433 void kasan_release_vmalloc(unsigned long start
, unsigned long end
,
434 unsigned long free_region_start
,
435 unsigned long free_region_end
);
437 #else /* CONFIG_KASAN_VMALLOC */
439 static inline int kasan_populate_vmalloc(unsigned long start
,
445 static inline void kasan_poison_vmalloc(const void *start
, unsigned long size
)
447 static inline void kasan_unpoison_vmalloc(const void *start
, unsigned long size
)
449 static inline void kasan_release_vmalloc(unsigned long start
,
451 unsigned long free_region_start
,
452 unsigned long free_region_end
) {}
454 #endif /* CONFIG_KASAN_VMALLOC */
456 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
457 !defined(CONFIG_KASAN_VMALLOC)
460 * These functions provide a special case to support backing module
461 * allocations with real shadow memory. With KASAN vmalloc, the special
462 * case is unnecessary, as the work is handled in the generic case.
464 int kasan_module_alloc(void *addr
, size_t size
);
465 void kasan_free_shadow(const struct vm_struct
*vm
);
467 #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
469 static inline int kasan_module_alloc(void *addr
, size_t size
) { return 0; }
470 static inline void kasan_free_shadow(const struct vm_struct
*vm
) {}
472 #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
474 #ifdef CONFIG_KASAN_INLINE
475 void kasan_non_canonical_hook(unsigned long addr
);
476 #else /* CONFIG_KASAN_INLINE */
477 static inline void kasan_non_canonical_hook(unsigned long addr
) { }
478 #endif /* CONFIG_KASAN_INLINE */
480 #endif /* LINUX_KASAN_H */