[mirror_ubuntu-kernels.git] / arch / s390 / mm / kasan_init.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kasan.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/kasan.h>
#include <asm/mem_detect.h>
#include <asm/processor.h>
#include <asm/sclp.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/uv.h>

static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
static unsigned long pgalloc_low __initdata;
static unsigned long pgalloc_freeable __initdata;
static bool has_edat __initdata;
static bool has_nx __initdata;

#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))

static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);

static void __init kasan_early_panic(const char *reason)
{
	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	sclp_early_printk(reason);
	disabled_wait();
}

static void * __init kasan_early_alloc_segment(void)
{
	segment_pos -= _SEGMENT_SIZE;

	if (segment_pos < segment_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)segment_pos;
}

static void * __init kasan_early_alloc_pages(unsigned int order)
{
	pgalloc_pos -= (PAGE_SIZE << order);

	if (pgalloc_pos < pgalloc_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)pgalloc_pos;
}

static void * __init kasan_early_crst_alloc(unsigned long val)
{
	unsigned long *table;

	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	if (table)
		crst_table_init(table, val);
	return table;
}

static pte_t * __init kasan_early_pte_alloc(void)
{
	static void *pte_leftover;
	pte_t *pte;

	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);

	if (!pte_leftover) {
		pte_leftover = kasan_early_alloc_pages(0);
		pte = pte_leftover + _PAGE_TABLE_SIZE;
	} else {
		pte = pte_leftover;
		pte_leftover = NULL;
	}
	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	return pte;
}

enum populate_mode {
	POPULATE_ONE2ONE,
	POPULATE_MAP,
	POPULATE_ZERO_SHADOW,
	POPULATE_SHALLOW
};
static void __init kasan_early_pgtable_populate(unsigned long address,
						unsigned long end,
						enum populate_mode mode)
{
	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
	pgd_t *pg_dir;
	p4d_t *p4_dir;
	pud_t *pu_dir;
	pmd_t *pm_dir;
	pte_t *pt_dir;

	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
	if (!has_nx)
		pgt_prot_zero &= ~_PAGE_NOEXEC;
	pgt_prot = pgprot_val(PAGE_KERNEL);
	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	if (!has_nx || mode == POPULATE_ONE2ONE) {
		pgt_prot &= ~_PAGE_NOEXEC;
		sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	}

	/*
	 * The first 1MB of 1:1 mapping is mapped with 4KB pages
	 */
	while (address < end) {
		pg_dir = pgd_offset_k(address);
		if (pgd_none(*pg_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PGDIR_SIZE) &&
			    end - address >= PGDIR_SIZE) {
				pgd_populate(&init_mm, pg_dir,
						kasan_early_shadow_p4d);
				address = (address + PGDIR_SIZE) & PGDIR_MASK;
				continue;
			}
			p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
			pgd_populate(&init_mm, pg_dir, p4_dir);
		}

		if (mode == POPULATE_SHALLOW) {
			address = (address + P4D_SIZE) & P4D_MASK;
			continue;
		}

		p4_dir = p4d_offset(pg_dir, address);
		if (p4d_none(*p4_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, P4D_SIZE) &&
			    end - address >= P4D_SIZE) {
				p4d_populate(&init_mm, p4_dir,
						kasan_early_shadow_pud);
				address = (address + P4D_SIZE) & P4D_MASK;
				continue;
			}
			pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
			p4d_populate(&init_mm, p4_dir, pu_dir);
		}

		pu_dir = pud_offset(p4_dir, address);
		if (pud_none(*pu_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PUD_SIZE) &&
			    end - address >= PUD_SIZE) {
				pud_populate(&init_mm, pu_dir,
						kasan_early_shadow_pmd);
				address = (address + PUD_SIZE) & PUD_MASK;
				continue;
			}
			pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
			pud_populate(&init_mm, pu_dir, pm_dir);
		}

		pm_dir = pmd_offset(pu_dir, address);
		if (pmd_none(*pm_dir)) {
			if (IS_ALIGNED(address, PMD_SIZE) &&
			    end - address >= PMD_SIZE) {
				if (mode == POPULATE_ZERO_SHADOW) {
					pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
					address = (address + PMD_SIZE) & PMD_MASK;
					continue;
				} else if (has_edat && address) {
					void *page;

					if (mode == POPULATE_ONE2ONE) {
						page = (void *)address;
					} else {
						page = kasan_early_alloc_segment();
						memset(page, 0, _SEGMENT_SIZE);
					}
					pmd_val(*pm_dir) = __pa(page) | sgt_prot;
					address = (address + PMD_SIZE) & PMD_MASK;
					continue;
				}
			}
			pt_dir = kasan_early_pte_alloc();
			pmd_populate(&init_mm, pm_dir, pt_dir);
		} else if (pmd_large(*pm_dir)) {
			address = (address + PMD_SIZE) & PMD_MASK;
			continue;
		}

		pt_dir = pte_offset_kernel(pm_dir, address);
		if (pte_none(*pt_dir)) {
			void *page;

			switch (mode) {
			case POPULATE_ONE2ONE:
				page = (void *)address;
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_MAP:
				page = kasan_early_alloc_pages(0);
				memset(page, 0, PAGE_SIZE);
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_ZERO_SHADOW:
				page = kasan_early_shadow_page;
				pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
				break;
			case POPULATE_SHALLOW:
				/* should never happen */
				break;
			}
		}
		address += PAGE_SIZE;
	}
}

static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
{
	unsigned long asce_bits;

	asce_bits = asce_type | _ASCE_TABLE_LENGTH;
	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
	S390_lowcore.user_asce = S390_lowcore.kernel_asce;

	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
}

static void __init kasan_enable_dat(void)
{
	psw_t psw;

	psw.mask = __extract_psw();
	psw_bits(psw).dat = 1;
	psw_bits(psw).as = PSW_BITS_AS_HOME;
	__load_psw_mask(psw.mask);
}

static void __init kasan_early_detect_facilities(void)
{
	if (test_facility(8)) {
		has_edat = true;
		__ctl_set_bit(0, 23);
	}
	if (!noexec_disabled && test_facility(130)) {
		has_nx = true;
		__ctl_set_bit(0, 20);
	}
}

void __init kasan_early_init(void)
{
	unsigned long shadow_alloc_size;
	unsigned long initrd_end;
	unsigned long memsize;
	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
	pte_t pte_z;
	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);

	kasan_early_detect_facilities();
	if (!has_nx)
		pgt_prot &= ~_PAGE_NOEXEC;
	pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);

	memsize = get_mem_detect_end();
	if (!memsize)
		kasan_early_panic("cannot detect physical memory size\n");
	/*
	 * Kasan currently supports standby memory but only if it follows
	 * online memory (default allocation), i.e. no memory holes.
	 * - memsize represents end of online memory
	 * - ident_map_size represents online + standby and memory limits
	 *   accounted.
	 * Kasan maps "memsize" right away.
	 * [0, memsize]			- as identity mapping
	 * [__sha(0), __sha(memsize)]	- shadow memory for identity mapping
	 * The rest [memsize, ident_map_size] if memsize < ident_map_size
	 * could be mapped/unmapped dynamically later during memory hotplug.
	 */
	memsize = min(memsize, ident_map_size);

	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
	crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);

	/* init kasan zero shadow */
	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
				p4d_val(p4d_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pud,
				pud_val(pud_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
				pmd_val(pmd_z));
	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);

	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
		initrd_end =
		    round_up(initrd_data.start + initrd_data.size, _SEGMENT_SIZE);
		pgalloc_low = max(pgalloc_low, initrd_end);
	}

	if (pgalloc_low + shadow_alloc_size > memsize)
		kasan_early_panic("out of memory during initialisation\n");

	if (has_edat) {
		segment_pos = round_down(memsize, _SEGMENT_SIZE);
		segment_low = segment_pos - shadow_alloc_size;
		pgalloc_pos = segment_low;
	} else {
		pgalloc_pos = memsize;
	}
	init_mm.pgd = early_pg_dir;
	/*
	 * Current memory layout:
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |		    |
	 * |		     |/    |	kasan	    |
	 * +- shadow start --+	   |	zero	    |
	 * | 1/8 addr space  |	   |	page	    |
	 * +- shadow end    -+	   |	mapping	    |
	 * | ... gap ...     |\    |  (untracked)   |
	 * +- vmalloc area  -+ \   |		    |
	 * | vmalloc_size    |	\  |		    |
	 * +- modules vaddr -+	 \ +----------------+
	 * | 2Gb	     |	  \|	  unmapped  | allocated per module
	 * +-----------------+	   +- shadow end ---+
	 *
	 * Current memory layout (KASAN_VMALLOC):
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |	kasan	    |
	 * |		     |/    |	zero	    |
	 * +- shadow start --+	   |	page	    |
	 * | 1/8 addr space  |	   |	mapping     |
	 * +- shadow end    -+	   |  (untracked)   |
	 * | ... gap ...     |\    |		    |
	 * +- vmalloc area  -+ \   +- vmalloc area -+
	 * | vmalloc_size    |	\  |shallow populate|
	 * +- modules vaddr -+	 \ +- modules area -+
	 * | 2Gb	     |	  \|shallow populate|
	 * +-----------------+	   +- shadow end ---+
	 */
	/* populate kasan shadow (for identity mapping and zero page mapping) */
	kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
		/* shallowly populate kasan shadow for vmalloc and modules */
		kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
					     POPULATE_SHALLOW);
	}
	/* populate kasan shadow for untracked memory */
	kasan_early_pgtable_populate(__sha(ident_map_size),
				     IS_ENABLED(CONFIG_KASAN_VMALLOC) ?
						   __sha(VMALLOC_START) :
						   __sha(MODULES_VADDR),
				     POPULATE_ZERO_SHADOW);
	kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
				     POPULATE_ZERO_SHADOW);
	/* memory allocated for identity mapping structs will be freed later */
	pgalloc_freeable = pgalloc_pos;
	/* populate identity mapping */
	kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
	kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
	kasan_enable_dat();
	/* enable kasan */
	init_task.kasan_depth = 0;
	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	sclp_early_printk("KernelAddressSanitizer initialized\n");
}

void __init kasan_copy_shadow_mapping(void)
{
	/*
	 * At this point we are still running on early pages setup early_pg_dir,
	 * while swapper_pg_dir has just been initialized with identity mapping.
	 * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	 */

	pgd_t *pg_dir_src;
	pgd_t *pg_dir_dst;
	p4d_t *p4_dir_src;
	p4d_t *p4_dir_dst;

	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
	pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
	memcpy(p4_dir_dst, p4_dir_src,
	       (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
}

void __init kasan_free_early_identity(void)
{
	memblock_phys_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
}
Commit	Line	Data
42db5ed8 VG	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kasan.h>
	3	#include <linux/sched/task.h>
	4	#include <linux/memblock.h>
ca5999fd	5	#include <linux/pgtable.h>
65fddcfc	6	#include <asm/pgalloc.h>
42db5ed8	7	#include <asm/kasan.h>
29635239	8	#include <asm/mem_detect.h>
42db5ed8 VG	9	#include <asm/processor.h>
42db5ed8 VG	10	#include <asm/sclp.h>
d58106c3	11	#include <asm/facility.h>
42db5ed8 VG	12	#include <asm/sections.h>
42db5ed8 VG	13	#include <asm/setup.h>
c360c9a2	14	#include <asm/uv.h>
42db5ed8	15
d58106c3 VG	16	static unsigned long segment_pos __initdata;
d58106c3 VG	17	static unsigned long segment_low __initdata;
42db5ed8 VG	18	static unsigned long pgalloc_pos __initdata;
42db5ed8 VG	19	static unsigned long pgalloc_low __initdata;
135ff163	20	static unsigned long pgalloc_freeable __initdata;
d58106c3 VG	21	static bool has_edat __initdata;
d58106c3 VG	22	static bool has_nx __initdata;
42db5ed8 VG	23
	24	#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
	25
	26	static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
	27
	28	static void __init kasan_early_panic(const char *reason)
	29	{
	30	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	31	sclp_early_printk(reason);
184c460f	32	disabled_wait();
42db5ed8 VG	33	}
42db5ed8 VG	34
d58106c3 VG	35	static void * __init kasan_early_alloc_segment(void)
	36	{
	37	segment_pos -= _SEGMENT_SIZE;
	38
	39	if (segment_pos < segment_low)
	40	kasan_early_panic("out of memory during initialisation\n");
	41
	42	return (void *)segment_pos;
	43	}
	44
42db5ed8 VG	45	static void * __init kasan_early_alloc_pages(unsigned int order)
	46	{
	47	pgalloc_pos -= (PAGE_SIZE << order);
	48
	49	if (pgalloc_pos < pgalloc_low)
	50	kasan_early_panic("out of memory during initialisation\n");
	51
	52	return (void *)pgalloc_pos;
	53	}
	54
	55	static void * __init kasan_early_crst_alloc(unsigned long val)
	56	{
	57	unsigned long *table;
	58
	59	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	60	if (table)
	61	crst_table_init(table, val);
	62	return table;
	63	}
	64
	65	static pte_t * __init kasan_early_pte_alloc(void)
	66	{
	67	static void *pte_leftover;
	68	pte_t *pte;
	69
	70	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
	71
	72	if (!pte_leftover) {
	73	pte_leftover = kasan_early_alloc_pages(0);
	74	pte = pte_leftover + _PAGE_TABLE_SIZE;
	75	} else {
	76	pte = pte_leftover;
	77	pte_leftover = NULL;
	78	}
	79	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	80	return pte;
	81	}
	82
	83	enum populate_mode {
	84	POPULATE_ONE2ONE,
	85	POPULATE_MAP,
3e39ce26 VG	86	POPULATE_ZERO_SHADOW,
3e39ce26 VG	87	POPULATE_SHALLOW
42db5ed8	88	};
92bca2fe	89	static void __init kasan_early_pgtable_populate(unsigned long address,
42db5ed8 VG	90	unsigned long end,
	91	enum populate_mode mode)
	92	{
d58106c3	93	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
42db5ed8 VG	94	pgd_t *pg_dir;
	95	p4d_t *p4_dir;
	96	pud_t *pu_dir;
	97	pmd_t *pm_dir;
	98	pte_t *pt_dir;
	99
	100	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
d58106c3 VG	101	if (!has_nx)
d58106c3 VG	102	pgt_prot_zero &= ~_PAGE_NOEXEC;
d411e3c6 VG	103	pgt_prot = pgprot_val(PAGE_KERNEL);
	104	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	105	if (!has_nx \|\| mode == POPULATE_ONE2ONE) {
	106	pgt_prot &= ~_PAGE_NOEXEC;
	107	sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	108	}
42db5ed8	109
ddd63c85 AG	110	/*
	111	* The first 1MB of 1:1 mapping is mapped with 4KB pages
	112	*/
42db5ed8 VG	113	while (address < end) {
	114	pg_dir = pgd_offset_k(address);
	115	if (pgd_none(*pg_dir)) {
	116	if (mode == POPULATE_ZERO_SHADOW &&
	117	IS_ALIGNED(address, PGDIR_SIZE) &&
	118	end - address >= PGDIR_SIZE) {
9577dd74 AK	119	pgd_populate(&init_mm, pg_dir,
9577dd74 AK	120	kasan_early_shadow_p4d);
42db5ed8 VG	121	address = (address + PGDIR_SIZE) & PGDIR_MASK;
	122	continue;
	123	}
	124	p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
	125	pgd_populate(&init_mm, pg_dir, p4_dir);
	126	}
	127
a3453d92	128	if (mode == POPULATE_SHALLOW) {
3e39ce26 VG	129	address = (address + P4D_SIZE) & P4D_MASK;
	130	continue;
	131	}
	132
42db5ed8 VG	133	p4_dir = p4d_offset(pg_dir, address);
	134	if (p4d_none(*p4_dir)) {
	135	if (mode == POPULATE_ZERO_SHADOW &&
	136	IS_ALIGNED(address, P4D_SIZE) &&
	137	end - address >= P4D_SIZE) {
9577dd74 AK	138	p4d_populate(&init_mm, p4_dir,
9577dd74 AK	139	kasan_early_shadow_pud);
42db5ed8 VG	140	address = (address + P4D_SIZE) & P4D_MASK;
	141	continue;
	142	}
	143	pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
	144	p4d_populate(&init_mm, p4_dir, pu_dir);
	145	}
	146
	147	pu_dir = pud_offset(p4_dir, address);
	148	if (pud_none(*pu_dir)) {
	149	if (mode == POPULATE_ZERO_SHADOW &&
	150	IS_ALIGNED(address, PUD_SIZE) &&
	151	end - address >= PUD_SIZE) {
9577dd74 AK	152	pud_populate(&init_mm, pu_dir,
9577dd74 AK	153	kasan_early_shadow_pmd);
42db5ed8 VG	154	address = (address + PUD_SIZE) & PUD_MASK;
	155	continue;
	156	}
	157	pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
	158	pud_populate(&init_mm, pu_dir, pm_dir);
	159	}
	160
	161	pm_dir = pmd_offset(pu_dir, address);
	162	if (pmd_none(*pm_dir)) {
ddd63c85	163	if (IS_ALIGNED(address, PMD_SIZE) &&
42db5ed8	164	end - address >= PMD_SIZE) {
ddd63c85 AG	165	if (mode == POPULATE_ZERO_SHADOW) {
	166	pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
	167	address = (address + PMD_SIZE) & PMD_MASK;
	168	continue;
	169	} else if (has_edat && address) {
	170	void *page;
	171
	172	if (mode == POPULATE_ONE2ONE) {
	173	page = (void *)address;
	174	} else {
	175	page = kasan_early_alloc_segment();
	176	memset(page, 0, _SEGMENT_SIZE);
	177	}
	178	pmd_val(*pm_dir) = __pa(page) \| sgt_prot;
	179	address = (address + PMD_SIZE) & PMD_MASK;
	180	continue;
d58106c3	181	}
d58106c3	182	}
42db5ed8 VG	183	pt_dir = kasan_early_pte_alloc();
42db5ed8 VG	184	pmd_populate(&init_mm, pm_dir, pt_dir);
d58106c3 VG	185	} else if (pmd_large(*pm_dir)) {
	186	address = (address + PMD_SIZE) & PMD_MASK;
	187	continue;
42db5ed8 VG	188	}
	189
	190	pt_dir = pte_offset_kernel(pm_dir, address);
	191	if (pte_none(*pt_dir)) {
	192	void *page;
	193
	194	switch (mode) {
	195	case POPULATE_ONE2ONE:
	196	page = (void *)address;
	197	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	198	break;
	199	case POPULATE_MAP:
	200	page = kasan_early_alloc_pages(0);
	201	memset(page, 0, PAGE_SIZE);
	202	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	203	break;
	204	case POPULATE_ZERO_SHADOW:
9577dd74	205	page = kasan_early_shadow_page;
42db5ed8 VG	206	pte_val(*pt_dir) = __pa(page) \| pgt_prot_zero;
42db5ed8 VG	207	break;
3e39ce26 VG	208	case POPULATE_SHALLOW:
	209	/* should never happen */
	210	break;
42db5ed8 VG	211	}
	212	}
	213	address += PAGE_SIZE;
	214	}
	215	}
	216
	217	static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
	218	{
	219	unsigned long asce_bits;
	220
	221	asce_bits = asce_type \| _ASCE_TABLE_LENGTH;
	222	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) \| asce_bits;
	223	S390_lowcore.user_asce = S390_lowcore.kernel_asce;
	224
	225	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	226	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	227	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
	228	}
	229
	230	static void __init kasan_enable_dat(void)
	231	{
	232	psw_t psw;
	233
	234	psw.mask = __extract_psw();
	235	psw_bits(psw).dat = 1;
	236	psw_bits(psw).as = PSW_BITS_AS_HOME;
	237	__load_psw_mask(psw.mask);
	238	}
	239
d58106c3 VG	240	static void __init kasan_early_detect_facilities(void)
d58106c3 VG	241	{
d58106c3 VG	242	if (test_facility(8)) {
	243	has_edat = true;
	244	__ctl_set_bit(0, 23);
	245	}
	246	if (!noexec_disabled && test_facility(130)) {
	247	has_nx = true;
	248	__ctl_set_bit(0, 20);
	249	}
	250	}
	251
42db5ed8 VG	252	void __init kasan_early_init(void)
42db5ed8 VG	253	{
42db5ed8 VG	254	unsigned long shadow_alloc_size;
42db5ed8 VG	255	unsigned long initrd_end;
42db5ed8	256	unsigned long memsize;
42db5ed8 VG	257	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
42db5ed8 VG	258	pte_t pte_z;
9577dd74 AK	259	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) \| _SEGMENT_ENTRY);
	260	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) \| _REGION3_ENTRY);
	261	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) \| _REGION2_ENTRY);
42db5ed8	262
d58106c3 VG	263	kasan_early_detect_facilities();
	264	if (!has_nx)
	265	pgt_prot &= ~_PAGE_NOEXEC;
9577dd74	266	pte_z = __pte(__pa(kasan_early_shadow_page) \| pgt_prot);
42db5ed8	267
29635239 VG	268	memsize = get_mem_detect_end();
	269	if (!memsize)
	270	kasan_early_panic("cannot detect physical memory size\n");
73045a08 VG	271	/*
	272	* Kasan currently supports standby memory but only if it follows
	273	* online memory (default allocation), i.e. no memory holes.
	274	* - memsize represents end of online memory
	275	* - ident_map_size represents online + standby and memory limits
	276	* accounted.
	277	* Kasan maps "memsize" right away.
	278	* [0, memsize] - as identity mapping
	279	* [__sha(0), __sha(memsize)] - shadow memory for identity mapping
	280	* The rest [memsize, ident_map_size] if memsize < ident_map_size
	281	* could be mapped/unmapped dynamically later during memory hotplug.
	282	*/
	283	memsize = min(memsize, ident_map_size);
29635239	284
a3453d92 VG	285	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
	286	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
	287	crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
42db5ed8 VG	288
42db5ed8 VG	289	/* init kasan zero shadow */
9577dd74 AK	290	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
	291	p4d_val(p4d_z));
	292	crst_table_init((unsigned long *)kasan_early_shadow_pud,
	293	pud_val(pud_z));
	294	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
	295	pmd_val(pmd_z));
	296	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
42db5ed8	297
42db5ed8	298	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
42db5ed8 VG	299	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	300	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
	301	initrd_end =
84733284	302	round_up(initrd_data.start + initrd_data.size, _SEGMENT_SIZE);
42db5ed8 VG	303	pgalloc_low = max(pgalloc_low, initrd_end);
	304	}
	305
	306	if (pgalloc_low + shadow_alloc_size > memsize)
	307	kasan_early_panic("out of memory during initialisation\n");
	308
d58106c3 VG	309	if (has_edat) {
	310	segment_pos = round_down(memsize, _SEGMENT_SIZE);
	311	segment_low = segment_pos - shadow_alloc_size;
	312	pgalloc_pos = segment_low;
	313	} else {
	314	pgalloc_pos = memsize;
	315	}
42db5ed8 VG	316	init_mm.pgd = early_pg_dir;
	317	/*
	318	* Current memory layout:
3e39ce26 VG	319	* +- 0 -------------+ +- shadow start -+
	320	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	321	* \| \| / \| \|
	322	* +- end of ram ----+ / +----------------+
	323	* \| ... gap ... \| / \| \|
	324	* \| \|/ \| kasan \|
	325	* +- shadow start --+ \| zero \|
	326	* \| 1/8 addr space \| \| page \|
	327	* +- shadow end -+ \| mapping \|
	328	* \| ... gap ... \|\ \| (untracked) \|
	329	* +- vmalloc area -+ \ \| \|
	330	* \| vmalloc_size \| \ \| \|
	331	* +- modules vaddr -+ \ +----------------+
	332	* \| 2Gb \| \\| unmapped \| allocated per module
	333	* +-----------------+ +- shadow end ---+
	334	*
	335	* Current memory layout (KASAN_VMALLOC):
	336	* +- 0 -------------+ +- shadow start -+
	337	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	338	* \| \| / \| \|
	339	* +- end of ram ----+ / +----------------+
	340	* \| ... gap ... \| / \| kasan \|
	341	* \| \|/ \| zero \|
	342	* +- shadow start --+ \| page \|
	343	* \| 1/8 addr space \| \| mapping \|
	344	* +- shadow end -+ \| (untracked) \|
	345	* \| ... gap ... \|\ \| \|
	346	* +- vmalloc area -+ \ +- vmalloc area -+
	347	* \| vmalloc_size \| \ \|shallow populate\|
	348	* +- modules vaddr -+ \ +- modules area -+
	349	* \| 2Gb \| \\|shallow populate\|
	350	* +-----------------+ +- shadow end ---+
42db5ed8	351	*/
793213a8	352	/* populate kasan shadow (for identity mapping and zero page mapping) */
92bca2fe	353	kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
3e39ce26	354	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
3e39ce26	355	/* shallowly populate kasan shadow for vmalloc and modules */
0c4f2623	356	kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
92bca2fe	357	POPULATE_SHALLOW);
3e39ce26 VG	358	}
3e39ce26 VG	359	/* populate kasan shadow for untracked memory */
0c4f2623 VG	360	kasan_early_pgtable_populate(__sha(ident_map_size),
	361	IS_ENABLED(CONFIG_KASAN_VMALLOC) ?
	362	__sha(VMALLOC_START) :
	363	__sha(MODULES_VADDR),
42db5ed8	364	POPULATE_ZERO_SHADOW);
0c4f2623	365	kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
c360c9a2	366	POPULATE_ZERO_SHADOW);
135ff163 VG	367	/* memory allocated for identity mapping structs will be freed later */
	368	pgalloc_freeable = pgalloc_pos;
	369	/* populate identity mapping */
92bca2fe	370	kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
a3453d92	371	kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
42db5ed8 VG	372	kasan_enable_dat();
	373	/* enable kasan */
	374	init_task.kasan_depth = 0;
	375	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	376	sclp_early_printk("KernelAddressSanitizer initialized\n");
	377	}
	378
54b52981	379	void __init kasan_copy_shadow_mapping(void)
42db5ed8 VG	380	{
	381	/*
	382	* At this point we are still running on early pages setup early_pg_dir,
	383	* while swapper_pg_dir has just been initialized with identity mapping.
	384	* Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	385	*/
	386
	387	pgd_t *pg_dir_src;
	388	pgd_t *pg_dir_dst;
	389	p4d_t *p4_dir_src;
	390	p4d_t *p4_dir_dst;
42db5ed8 VG	391
42db5ed8 VG	392	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
54b52981	393	pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
42db5ed8 VG	394	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
42db5ed8 VG	395	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
a3453d92 VG	396	memcpy(p4_dir_dst, p4_dir_src,
a3453d92 VG	397	(KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
42db5ed8	398	}
135ff163 VG	399
	400	void __init kasan_free_early_identity(void)
	401	{
3ecc6834	402	memblock_phys_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
135ff163	403	}