[mirror_ubuntu-jammy-kernel.git] / include / linux / kasan.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KASAN_H
#define _LINUX_KASAN_H

#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/static_key.h>
#include <linux/types.h>

struct kmem_cache;
struct page;
struct vm_struct;
struct task_struct;

#ifdef CONFIG_KASAN

#include <linux/linkage.h>
#include <asm/kasan.h>

/* kasan_data struct is used in KUnit tests for KASAN expected failures */
struct kunit_kasan_expectation {
	bool report_found;
};

#endif

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

#include <linux/pgtable.h>

/* Software KASAN implementations use shadow memory. */

#ifdef CONFIG_KASAN_SW_TAGS
/* This matches KASAN_TAG_INVALID. */
#define KASAN_SHADOW_INIT 0xFE
#else
#define KASAN_SHADOW_INIT 0
#endif

#ifndef PTE_HWTABLE_PTRS
#define PTE_HWTABLE_PTRS 0
#endif

extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];

int kasan_populate_early_shadow(const void *shadow_start,
				const void *shadow_end);

static inline void *kasan_mem_to_shadow(const void *addr)
{
	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
		+ KASAN_SHADOW_OFFSET;
}

int kasan_add_zero_shadow(void *start, unsigned long size);
void kasan_remove_zero_shadow(void *start, unsigned long size);

/* Enable reporting bugs after kasan_disable_current() */
extern void kasan_enable_current(void);

/* Disable reporting bugs for current task */
extern void kasan_disable_current(void);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline int kasan_add_zero_shadow(void *start, unsigned long size)
{
	return 0;
}
static inline void kasan_remove_zero_shadow(void *start,
					unsigned long size)
{}

static inline void kasan_enable_current(void) {}
static inline void kasan_disable_current(void) {}

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

#ifdef CONFIG_KASAN_HW_TAGS

DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);

static __always_inline bool kasan_enabled(void)
{
	return static_branch_likely(&kasan_flag_enabled);
}

static inline bool kasan_has_integrated_init(void)
{
	return kasan_enabled();
}

void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags);
void kasan_free_pages(struct page *page, unsigned int order);

#else /* CONFIG_KASAN_HW_TAGS */

static inline bool kasan_enabled(void)
{
	return IS_ENABLED(CONFIG_KASAN);
}

static inline bool kasan_has_integrated_init(void)
{
	return false;
}

static __always_inline void kasan_alloc_pages(struct page *page,
					      unsigned int order, gfp_t flags)
{
	/* Only available for integrated init. */
	BUILD_BUG();
}

static __always_inline void kasan_free_pages(struct page *page,
					     unsigned int order)
{
	/* Only available for integrated init. */
	BUILD_BUG();
}

#endif /* CONFIG_KASAN_HW_TAGS */

#ifdef CONFIG_KASAN

struct kasan_cache {
	int alloc_meta_offset;
	int free_meta_offset;
	bool is_kmalloc;
};

slab_flags_t __kasan_never_merge(void);
static __always_inline slab_flags_t kasan_never_merge(void)
{
	if (kasan_enabled())
		return __kasan_never_merge();
	return 0;
}

void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
{
	if (kasan_enabled())
		__kasan_unpoison_range(addr, size);
}

void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_poison_pages(struct page *page,
						unsigned int order, bool init)
{
	if (kasan_enabled())
		__kasan_poison_pages(page, order, init);
}

void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_unpoison_pages(struct page *page,
						 unsigned int order, bool init)
{
	if (kasan_enabled())
		__kasan_unpoison_pages(page, order, init);
}

void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
				slab_flags_t *flags);
static __always_inline void kasan_cache_create(struct kmem_cache *cache,
				unsigned int *size, slab_flags_t *flags)
{
	if (kasan_enabled())
		__kasan_cache_create(cache, size, flags);
}

void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
{
	if (kasan_enabled())
		__kasan_cache_create_kmalloc(cache);
}

size_t __kasan_metadata_size(struct kmem_cache *cache);
static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
{
	if (kasan_enabled())
		return __kasan_metadata_size(cache);
	return 0;
}

void __kasan_poison_slab(struct page *page);
static __always_inline void kasan_poison_slab(struct page *page)
{
	if (kasan_enabled())
		__kasan_poison_slab(page);
}

void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_unpoison_object_data(cache, object);
}

void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_poison_object_data(cache, object);
}

void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
					  const void *object);
static __always_inline void * __must_check kasan_init_slab_obj(
				struct kmem_cache *cache, const void *object)
{
	if (kasan_enabled())
		return __kasan_init_slab_obj(cache, object);
	return (void *)object;
}

bool __kasan_slab_free(struct kmem_cache *s, void *object,
			unsigned long ip, bool init);
static __always_inline bool kasan_slab_free(struct kmem_cache *s,
						void *object, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_free(s, object, _RET_IP_, init);
	return false;
}

void __kasan_kfree_large(void *ptr, unsigned long ip);
static __always_inline void kasan_kfree_large(void *ptr)
{
	if (kasan_enabled())
		__kasan_kfree_large(ptr, _RET_IP_);
}

void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
static __always_inline void kasan_slab_free_mempool(void *ptr)
{
	if (kasan_enabled())
		__kasan_slab_free_mempool(ptr, _RET_IP_);
}

void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
				       void *object, gfp_t flags, bool init);
static __always_inline void * __must_check kasan_slab_alloc(
		struct kmem_cache *s, void *object, gfp_t flags, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_alloc(s, object, flags, init);
	return object;
}

void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
				    size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
				const void *object, size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc(s, object, size, flags);
	return (void *)object;
}

void * __must_check __kasan_kmalloc_large(const void *ptr,
					  size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
						      size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc_large(ptr, size, flags);
	return (void *)ptr;
}

void * __must_check __kasan_krealloc(const void *object,
				     size_t new_size, gfp_t flags);
static __always_inline void * __must_check kasan_krealloc(const void *object,
						 size_t new_size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_krealloc(object, new_size, flags);
	return (void *)object;
}

/*
 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
 */
bool __kasan_check_byte(const void *addr, unsigned long ip);
static __always_inline bool kasan_check_byte(const void *addr)
{
	if (kasan_enabled())
		return __kasan_check_byte(addr, _RET_IP_);
	return true;
}


bool kasan_save_enable_multi_shot(void);
void kasan_restore_multi_shot(bool enabled);

#else /* CONFIG_KASAN */

static inline slab_flags_t kasan_never_merge(void)
{
	return 0;
}
static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_poison_pages(struct page *page, unsigned int order,
				      bool init) {}
static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
					bool init) {}
static inline void kasan_cache_create(struct kmem_cache *cache,
				      unsigned int *size,
				      slab_flags_t *flags) {}
static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
static inline void kasan_poison_slab(struct page *page) {}
static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void kasan_poison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
				const void *object)
{
	return (void *)object;
}
static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init)
{
	return false;
}
static inline void kasan_kfree_large(void *ptr) {}
static inline void kasan_slab_free_mempool(void *ptr) {}
static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
				   gfp_t flags, bool init)
{
	return object;
}
static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
				size_t size, gfp_t flags)
{
	return (void *)object;
}
static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
	return (void *)ptr;
}
static inline void *kasan_krealloc(const void *object, size_t new_size,
				 gfp_t flags)
{
	return (void *)object;
}
static inline bool kasan_check_byte(const void *address)
{
	return true;
}

#endif /* CONFIG_KASAN */

#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
void kasan_unpoison_task_stack(struct task_struct *task);
#else
static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
#endif

#ifdef CONFIG_KASAN_GENERIC

void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);

#else /* CONFIG_KASAN_GENERIC */

static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}

#endif /* CONFIG_KASAN_GENERIC */

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)arch_kasan_reset_tag(addr);
}

/**
 * kasan_report - print a report about a bad memory access detected by KASAN
 * @addr: address of the bad access
 * @size: size of the bad access
 * @is_write: whether the bad access is a write or a read
 * @ip: instruction pointer for the accessibility check or the bad access itself
 */
bool kasan_report(unsigned long addr, size_t size,
		bool is_write, unsigned long ip);

#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)addr;
}

#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/

#ifdef CONFIG_KASAN_HW_TAGS

void kasan_report_async(void);

#endif /* CONFIG_KASAN_HW_TAGS */

#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#else
static inline void kasan_init_sw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_HW_TAGS
void kasan_init_hw_tags_cpu(void);
void __init kasan_init_hw_tags(void);
#else
static inline void kasan_init_hw_tags_cpu(void) { }
static inline void kasan_init_hw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_VMALLOC

int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
void kasan_poison_vmalloc(const void *start, unsigned long size);
void kasan_unpoison_vmalloc(const void *start, unsigned long size);
void kasan_release_vmalloc(unsigned long start, unsigned long end,
			   unsigned long free_region_start,
			   unsigned long free_region_end);

#else /* CONFIG_KASAN_VMALLOC */

static inline int kasan_populate_vmalloc(unsigned long start,
					unsigned long size)
{
	return 0;
}

static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
{ }
static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size)
{ }
static inline void kasan_release_vmalloc(unsigned long start,
					 unsigned long end,
					 unsigned long free_region_start,
					 unsigned long free_region_end) {}

#endif /* CONFIG_KASAN_VMALLOC */

#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
		!defined(CONFIG_KASAN_VMALLOC)

/*
 * These functions provide a special case to support backing module
 * allocations with real shadow memory. With KASAN vmalloc, the special
 * case is unnecessary, as the work is handled in the generic case.
 */
int kasan_module_alloc(void *addr, size_t size);
void kasan_free_shadow(const struct vm_struct *vm);

#else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
static inline void kasan_free_shadow(const struct vm_struct *vm) {}

#endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

#ifdef CONFIG_KASAN_INLINE
void kasan_non_canonical_hook(unsigned long addr);
#else /* CONFIG_KASAN_INLINE */
static inline void kasan_non_canonical_hook(unsigned long addr) { }
#endif /* CONFIG_KASAN_INLINE */

#endif /* LINUX_KASAN_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
0b24becc AR	2	#ifndef _LINUX_KASAN_H
	3	#define _LINUX_KASAN_H
	4
7a3b8353	5	#include <linux/bug.h>
2db710cc	6	#include <linux/kernel.h>
34303244	7	#include <linux/static_key.h>
0b24becc AR	8	#include <linux/types.h>
	9
	10	struct kmem_cache;
	11	struct page;
a5af5aa8	12	struct vm_struct;
5be9b730	13	struct task_struct;
0b24becc AR	14
	15	#ifdef CONFIG_KASAN
	16
d5750edf	17	#include <linux/linkage.h>
65fddcfc	18	#include <asm/kasan.h>
0b24becc	19
83c4e7a0 PA	20	/* kasan_data struct is used in KUnit tests for KASAN expected failures */
83c4e7a0 PA	21	struct kunit_kasan_expectation {
83c4e7a0 PA	22	bool report_found;
	23	};
	24
d5750edf AK	25	#endif
	26
	27	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
	28
	29	#include <linux/pgtable.h>
	30
	31	/* Software KASAN implementations use shadow memory. */
	32
	33	#ifdef CONFIG_KASAN_SW_TAGS
a064cb00 AK	34	/* This matches KASAN_TAG_INVALID. */
a064cb00 AK	35	#define KASAN_SHADOW_INIT 0xFE
d5750edf AK	36	#else
	37	#define KASAN_SHADOW_INIT 0
	38	#endif
	39
29970dc2 HL	40	#ifndef PTE_HWTABLE_PTRS
	41	#define PTE_HWTABLE_PTRS 0
	42	#endif
	43
9577dd74	44	extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
cb32c9c5 DA	45	extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
	46	extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
	47	extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
9577dd74	48	extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
69786cdb	49
9577dd74	50	int kasan_populate_early_shadow(const void *shadow_start,
69786cdb AR	51	const void *shadow_end);
69786cdb AR	52
0b24becc AR	53	static inline void kasan_mem_to_shadow(const void addr)
	54	{
	55	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
	56	+ KASAN_SHADOW_OFFSET;
	57	}
	58
d5750edf AK	59	int kasan_add_zero_shadow(void *start, unsigned long size);
	60	void kasan_remove_zero_shadow(void *start, unsigned long size);
	61
d73b4936 AK	62	/* Enable reporting bugs after kasan_disable_current() */
	63	extern void kasan_enable_current(void);
	64
	65	/* Disable reporting bugs for current task */
	66	extern void kasan_disable_current(void);
	67
d5750edf AK	68	#else /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
	69
	70	static inline int kasan_add_zero_shadow(void *start, unsigned long size)
	71	{
	72	return 0;
	73	}
	74	static inline void kasan_remove_zero_shadow(void *start,
	75	unsigned long size)
	76	{}
	77
d73b4936 AK	78	static inline void kasan_enable_current(void) {}
	79	static inline void kasan_disable_current(void) {}
	80
d5750edf AK	81	#endif /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
d5750edf AK	82
34303244	83	#ifdef CONFIG_KASAN_HW_TAGS
e86f8b09	84
34303244	85	DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);
e86f8b09	86
34303244 AK	87	static __always_inline bool kasan_enabled(void)
	88	{
	89	return static_branch_likely(&kasan_flag_enabled);
	90	}
e86f8b09	91
1bb5eab3 AK	92	static inline bool kasan_has_integrated_init(void)
	93	{
	94	return kasan_enabled();
	95	}
	96
7a3b8353 PC	97	void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags);
	98	void kasan_free_pages(struct page *page, unsigned int order);
	99
e86f8b09 AK	100	#else /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	101
34303244 AK	102	static inline bool kasan_enabled(void)
34303244 AK	103	{
7a3b8353	104	return IS_ENABLED(CONFIG_KASAN);
34303244	105	}
e86f8b09	106
1bb5eab3 AK	107	static inline bool kasan_has_integrated_init(void)
	108	{
	109	return false;
	110	}
	111
7a3b8353 PC	112	static __always_inline void kasan_alloc_pages(struct page *page,
	113	unsigned int order, gfp_t flags)
	114	{
	115	/* Only available for integrated init. */
	116	BUILD_BUG();
	117	}
	118
	119	static __always_inline void kasan_free_pages(struct page *page,
	120	unsigned int order)
	121	{
	122	/* Only available for integrated init. */
	123	BUILD_BUG();
	124	}
	125
e86f8b09 AK	126	#endif /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	127
7a3b8353 PC	128	#ifdef CONFIG_KASAN
	129
	130	struct kasan_cache {
	131	int alloc_meta_offset;
	132	int free_meta_offset;
	133	bool is_kmalloc;
	134	};
	135
e86f8b09 AK	136	slab_flags_t __kasan_never_merge(void);
	137	static __always_inline slab_flags_t kasan_never_merge(void)
	138	{
	139	if (kasan_enabled())
	140	return __kasan_never_merge();
	141	return 0;
	142	}
b8c73fc2	143
34303244 AK	144	void __kasan_unpoison_range(const void *addr, size_t size);
	145	static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
	146	{
	147	if (kasan_enabled())
	148	__kasan_unpoison_range(addr, size);
	149	}
7ed2f9e6	150
7a3b8353 PC	151	void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
7a3b8353 PC	152	static __always_inline void kasan_poison_pages(struct page *page,
1bb5eab3	153	unsigned int order, bool init)
34303244 AK	154	{
34303244 AK	155	if (kasan_enabled())
7a3b8353	156	__kasan_poison_pages(page, order, init);
34303244	157	}
0316bec2	158
7a3b8353 PC	159	void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
	160	static __always_inline void kasan_unpoison_pages(struct page *page,
	161	unsigned int order, bool init)
34303244 AK	162	{
34303244 AK	163	if (kasan_enabled())
7a3b8353	164	__kasan_unpoison_pages(page, order, init);
34303244	165	}
0316bec2	166
34303244 AK	167	void __kasan_cache_create(struct kmem_cache cache, unsigned int size,
	168	slab_flags_t *flags);
	169	static __always_inline void kasan_cache_create(struct kmem_cache *cache,
	170	unsigned int size, slab_flags_t flags)
	171	{
	172	if (kasan_enabled())
	173	__kasan_cache_create(cache, size, flags);
	174	}
0316bec2	175
92850134 AK	176	void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
	177	static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
	178	{
	179	if (kasan_enabled())
	180	__kasan_cache_create_kmalloc(cache);
	181	}
	182
34303244 AK	183	size_t __kasan_metadata_size(struct kmem_cache *cache);
	184	static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
	185	{
	186	if (kasan_enabled())
	187	return __kasan_metadata_size(cache);
	188	return 0;
	189	}
7ed2f9e6	190
34303244 AK	191	void __kasan_poison_slab(struct page *page);
	192	static __always_inline void kasan_poison_slab(struct page *page)
	193	{
	194	if (kasan_enabled())
	195	__kasan_poison_slab(page);
	196	}
	197
	198	void __kasan_unpoison_object_data(struct kmem_cache cache, void object);
	199	static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	200	void *object)
	201	{
	202	if (kasan_enabled())
	203	__kasan_unpoison_object_data(cache, object);
	204	}
	205
	206	void __kasan_poison_object_data(struct kmem_cache cache, void object);
	207	static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
	208	void *object)
	209	{
	210	if (kasan_enabled())
	211	__kasan_poison_object_data(cache, object);
	212	}
	213
	214	void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
	215	const void *object);
	216	static __always_inline void * __must_check kasan_init_slab_obj(
	217	struct kmem_cache cache, const void object)
	218	{
	219	if (kasan_enabled())
	220	return __kasan_init_slab_obj(cache, object);
	221	return (void *)object;
	222	}
	223
d57a964e AK	224	bool __kasan_slab_free(struct kmem_cache s, void object,
	225	unsigned long ip, bool init);
	226	static __always_inline bool kasan_slab_free(struct kmem_cache *s,
	227	void *object, bool init)
34303244 AK	228	{
34303244 AK	229	if (kasan_enabled())
d57a964e	230	return __kasan_slab_free(s, object, _RET_IP_, init);
34303244 AK	231	return false;
	232	}
	233
200072ce AK	234	void __kasan_kfree_large(void *ptr, unsigned long ip);
	235	static __always_inline void kasan_kfree_large(void *ptr)
	236	{
	237	if (kasan_enabled())
	238	__kasan_kfree_large(ptr, _RET_IP_);
	239	}
	240
eeb3160c	241	void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
027b37b5	242	static __always_inline void kasan_slab_free_mempool(void *ptr)
eeb3160c AK	243	{
eeb3160c AK	244	if (kasan_enabled())
027b37b5	245	__kasan_slab_free_mempool(ptr, _RET_IP_);
eeb3160c AK	246	}
eeb3160c AK	247
34303244	248	void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
da844b78	249	void *object, gfp_t flags, bool init);
34303244	250	static __always_inline void * __must_check kasan_slab_alloc(
da844b78	251	struct kmem_cache s, void object, gfp_t flags, bool init)
34303244 AK	252	{
34303244 AK	253	if (kasan_enabled())
da844b78	254	return __kasan_slab_alloc(s, object, flags, init);
34303244 AK	255	return object;
	256	}
	257
	258	void * __must_check __kasan_kmalloc(struct kmem_cache s, const void object,
	259	size_t size, gfp_t flags);
	260	static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
	261	const void *object, size_t size, gfp_t flags)
	262	{
	263	if (kasan_enabled())
	264	return __kasan_kmalloc(s, object, size, flags);
	265	return (void *)object;
	266	}
	267
	268	void * __must_check __kasan_kmalloc_large(const void *ptr,
	269	size_t size, gfp_t flags);
	270	static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
	271	size_t size, gfp_t flags)
	272	{
	273	if (kasan_enabled())
	274	return __kasan_kmalloc_large(ptr, size, flags);
	275	return (void *)ptr;
	276	}
	277
	278	void * __must_check __kasan_krealloc(const void *object,
	279	size_t new_size, gfp_t flags);
	280	static __always_inline void * __must_check kasan_krealloc(const void *object,
	281	size_t new_size, gfp_t flags)
	282	{
	283	if (kasan_enabled())
	284	return __kasan_krealloc(object, new_size, flags);
	285	return (void *)object;
	286	}
	287
611806b4 AK	288	/*
	289	* Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
	290	* the hardware tag-based mode that doesn't rely on compiler instrumentation.
	291	*/
	292	bool __kasan_check_byte(const void *addr, unsigned long ip);
	293	static __always_inline bool kasan_check_byte(const void *addr)
	294	{
	295	if (kasan_enabled())
	296	return __kasan_check_byte(addr, _RET_IP_);
	297	return true;
	298	}
	299
	300
b0845ce5 MR	301	bool kasan_save_enable_multi_shot(void);
	302	void kasan_restore_multi_shot(bool enabled);
	303
0b24becc AR	304	#else /* CONFIG_KASAN */
0b24becc AR	305
e86f8b09 AK	306	static inline slab_flags_t kasan_never_merge(void)
	307	{
	308	return 0;
	309	}
cebd0eb2	310	static inline void kasan_unpoison_range(const void *address, size_t size) {}
7a3b8353 PC	311	static inline void kasan_poison_pages(struct page *page, unsigned int order,
	312	bool init) {}
	313	static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
	314	bool init) {}
7ed2f9e6	315	static inline void kasan_cache_create(struct kmem_cache *cache,
be4a7988	316	unsigned int *size,
d50112ed	317	slab_flags_t *flags) {}
92850134	318	static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
34303244	319	static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
0316bec2 AR	320	static inline void kasan_poison_slab(struct page *page) {}
	321	static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	322	void *object) {}
	323	static inline void kasan_poison_object_data(struct kmem_cache *cache,
	324	void *object) {}
0116523c AK	325	static inline void kasan_init_slab_obj(struct kmem_cache cache,
	326	const void *object)
	327	{
	328	return (void *)object;
	329	}
d57a964e	330	static inline bool kasan_slab_free(struct kmem_cache s, void object, bool init)
34303244 AK	331	{
	332	return false;
	333	}
200072ce	334	static inline void kasan_kfree_large(void *ptr) {}
027b37b5	335	static inline void kasan_slab_free_mempool(void *ptr) {}
34303244	336	static inline void kasan_slab_alloc(struct kmem_cache s, void *object,
da844b78	337	gfp_t flags, bool init)
0116523c	338	{
34303244	339	return object;
0116523c	340	}
0116523c AK	341	static inline void kasan_kmalloc(struct kmem_cache s, const void *object,
	342	size_t size, gfp_t flags)
	343	{
	344	return (void *)object;
	345	}
34303244 AK	346	static inline void kasan_kmalloc_large(const void ptr, size_t size, gfp_t flags)
	347	{
	348	return (void *)ptr;
	349	}
0116523c AK	350	static inline void kasan_krealloc(const void object, size_t new_size,
	351	gfp_t flags)
	352	{
	353	return (void *)object;
	354	}
611806b4 AK	355	static inline bool kasan_check_byte(const void *address)
	356	{
	357	return true;
	358	}
9b75a867	359
0b24becc AR	360	#endif /* CONFIG_KASAN */
0b24becc AR	361
02c58773	362	#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
d56a9ef8 AK	363	void kasan_unpoison_task_stack(struct task_struct *task);
	364	#else
	365	static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
	366	#endif
	367
2bd926b4 AK	368	#ifdef CONFIG_KASAN_GENERIC
	369
	370	void kasan_cache_shrink(struct kmem_cache *cache);
	371	void kasan_cache_shutdown(struct kmem_cache *cache);
26e760c9	372	void kasan_record_aux_stack(void *ptr);
2bd926b4 AK	373
	374	#else /* CONFIG_KASAN_GENERIC */
	375
	376	static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
	377	static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
26e760c9	378	static inline void kasan_record_aux_stack(void *ptr) {}
2bd926b4 AK	379
	380	#endif /* CONFIG_KASAN_GENERIC */
	381
2e903b91	382	#if defined(CONFIG_KASAN_SW_TAGS) \|\| defined(CONFIG_KASAN_HW_TAGS)
3c9e3aa1	383
c0054c56 AK	384	static inline void kasan_reset_tag(const void addr)
	385	{
	386	return (void *)arch_kasan_reset_tag(addr);
	387	}
3c9e3aa1	388
49c6631d VF	389	/**
	390	* kasan_report - print a report about a bad memory access detected by KASAN
	391	* @addr: address of the bad access
	392	* @size: size of the bad access
	393	* @is_write: whether the bad access is a write or a read
	394	* @ip: instruction pointer for the accessibility check or the bad access itself
	395	*/
8cceeff4	396	bool kasan_report(unsigned long addr, size_t size,
41eea9cd AK	397	bool is_write, unsigned long ip);
41eea9cd AK	398
2e903b91	399	#else /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS */
3c9e3aa1 AK	400
	401	static inline void kasan_reset_tag(const void addr)
	402	{
	403	return (void *)addr;
	404	}
	405
2e903b91 AK	406	#endif /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS*/
2e903b91 AK	407
8f7b5054 VF	408	#ifdef CONFIG_KASAN_HW_TAGS
	409
	410	void kasan_report_async(void);
	411
	412	#endif /* CONFIG_KASAN_HW_TAGS */
	413
2e903b91 AK	414	#ifdef CONFIG_KASAN_SW_TAGS
	415	void __init kasan_init_sw_tags(void);
	416	#else
	417	static inline void kasan_init_sw_tags(void) { }
	418	#endif
	419
	420	#ifdef CONFIG_KASAN_HW_TAGS
	421	void kasan_init_hw_tags_cpu(void);
	422	void __init kasan_init_hw_tags(void);
	423	#else
	424	static inline void kasan_init_hw_tags_cpu(void) { }
	425	static inline void kasan_init_hw_tags(void) { }
	426	#endif
080eb83f	427
3c5c3cfb	428	#ifdef CONFIG_KASAN_VMALLOC
3b1a4a86	429
d98c9e83 AR	430	int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
	431	void kasan_poison_vmalloc(const void *start, unsigned long size);
	432	void kasan_unpoison_vmalloc(const void *start, unsigned long size);
3c5c3cfb DA	433	void kasan_release_vmalloc(unsigned long start, unsigned long end,
	434	unsigned long free_region_start,
	435	unsigned long free_region_end);
3b1a4a86 AK	436
	437	#else /* CONFIG_KASAN_VMALLOC */
	438
d98c9e83 AR	439	static inline int kasan_populate_vmalloc(unsigned long start,
d98c9e83 AR	440	unsigned long size)
3c5c3cfb DA	441	{
	442	return 0;
	443	}
	444
d98c9e83 AR	445	static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
	446	{ }
	447	static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size)
	448	{ }
3c5c3cfb DA	449	static inline void kasan_release_vmalloc(unsigned long start,
	450	unsigned long end,
	451	unsigned long free_region_start,
	452	unsigned long free_region_end) {}
3b1a4a86 AK	453
	454	#endif /* CONFIG_KASAN_VMALLOC */
	455
0fea6e9a AK	456	#if (defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)) && \
0fea6e9a AK	457	!defined(CONFIG_KASAN_VMALLOC)
3b1a4a86 AK	458
	459	/*
	460	* These functions provide a special case to support backing module
	461	* allocations with real shadow memory. With KASAN vmalloc, the special
	462	* case is unnecessary, as the work is handled in the generic case.
	463	*/
	464	int kasan_module_alloc(void *addr, size_t size);
	465	void kasan_free_shadow(const struct vm_struct *vm);
	466
0fea6e9a	467	#else /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3b1a4a86 AK	468
	469	static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
	470	static inline void kasan_free_shadow(const struct vm_struct *vm) {}
	471
0fea6e9a	472	#endif /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3c5c3cfb	473
2f004eea JH	474	#ifdef CONFIG_KASAN_INLINE
	475	void kasan_non_canonical_hook(unsigned long addr);
	476	#else /* CONFIG_KASAN_INLINE */
	477	static inline void kasan_non_canonical_hook(unsigned long addr) { }
	478	#endif /* CONFIG_KASAN_INLINE */
	479
0b24becc	480	#endif /* LINUX_KASAN_H */