[mirror_ubuntu-bionic-kernel.git] / arch / arm64 / mm / kasan_init.c

/*
 * This file contains kasan initialization code for ARM64.
 *
 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

#define pr_fmt(fmt) "kasan: " fmt
#include <linux/bootmem.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/start_kernel.h>
#include <linux/mm.h>

#include <asm/mmu_context.h>
#include <asm/kernel-pgtable.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>

static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);

/*
 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
 * directly on kernel symbols (bm_p*d). All the early functions are called too
 * early to use lm_alias so __p*d_populate functions must be used to populate
 * with the physical address from __pa_symbol.
 */

static phys_addr_t __init kasan_alloc_zeroed_page(int node)
{
	void *p = memblock_virt_alloc_try_nid(PAGE_SIZE, PAGE_SIZE,
					      __pa(MAX_DMA_ADDRESS),
					      MEMBLOCK_ALLOC_ACCESSIBLE, node);
	return __pa(p);
}

static pte_t *__init kasan_pte_offset(pmd_t *pmd, unsigned long addr, int node,
				      bool early)
{
	if (pmd_none(*pmd)) {
		phys_addr_t pte_phys = early ? __pa_symbol(kasan_zero_pte)
					     : kasan_alloc_zeroed_page(node);
		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
	}

	return early ? pte_offset_kimg(pmd, addr)
		     : pte_offset_kernel(pmd, addr);
}

static pmd_t *__init kasan_pmd_offset(pud_t *pud, unsigned long addr, int node,
				      bool early)
{
	if (pud_none(*pud)) {
		phys_addr_t pmd_phys = early ? __pa_symbol(kasan_zero_pmd)
					     : kasan_alloc_zeroed_page(node);
		__pud_populate(pud, pmd_phys, PMD_TYPE_TABLE);
	}

	return early ? pmd_offset_kimg(pud, addr) : pmd_offset(pud, addr);
}

static pud_t *__init kasan_pud_offset(pgd_t *pgd, unsigned long addr, int node,
				      bool early)
{
	if (pgd_none(*pgd)) {
		phys_addr_t pud_phys = early ? __pa_symbol(kasan_zero_pud)
					     : kasan_alloc_zeroed_page(node);
		__pgd_populate(pgd, pud_phys, PMD_TYPE_TABLE);
	}

	return early ? pud_offset_kimg(pgd, addr) : pud_offset(pgd, addr);
}

static void __init kasan_pte_populate(pmd_t *pmd, unsigned long addr,
				      unsigned long end, int node, bool early)
{
	unsigned long next;
	pte_t *pte = kasan_pte_offset(pmd, addr, node, early);

	do {
		phys_addr_t page_phys = early ? __pa_symbol(kasan_zero_page)
					      : kasan_alloc_zeroed_page(node);
		next = addr + PAGE_SIZE;
		set_pte(pte, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
	} while (pte++, addr = next, addr != end && pte_none(*pte));
}

static void __init kasan_pmd_populate(pud_t *pud, unsigned long addr,
				      unsigned long end, int node, bool early)
{
	unsigned long next;
	pmd_t *pmd = kasan_pmd_offset(pud, addr, node, early);

	do {
		next = pmd_addr_end(addr, end);
		kasan_pte_populate(pmd, addr, next, node, early);
	} while (pmd++, addr = next, addr != end && pmd_none(*pmd));
}

static void __init kasan_pud_populate(pgd_t *pgd, unsigned long addr,
				      unsigned long end, int node, bool early)
{
	unsigned long next;
	pud_t *pud = kasan_pud_offset(pgd, addr, node, early);

	do {
		next = pud_addr_end(addr, end);
		kasan_pmd_populate(pud, addr, next, node, early);
	} while (pud++, addr = next, addr != end && pud_none(*pud));
}

static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
				      int node, bool early)
{
	unsigned long next;
	pgd_t *pgd;

	pgd = pgd_offset_k(addr);
	do {
		next = pgd_addr_end(addr, end);
		kasan_pud_populate(pgd, addr, next, node, early);
	} while (pgd++, addr = next, addr != end);
}

/* The early shadow maps everything to a single page of zeroes */
asmlinkage void __init kasan_early_init(void)
{
	BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE,
			   true);
}

/* Set up full kasan mappings, ensuring that the mapped pages are zeroed */
static void __init kasan_map_populate(unsigned long start, unsigned long end,
				      int node)
{
	kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false);
}

/*
 * Copy the current shadow region into a new pgdir.
 */
void __init kasan_copy_shadow(pgd_t *pgdir)
{
	pgd_t *pgd, *pgd_new, *pgd_end;

	pgd = pgd_offset_k(KASAN_SHADOW_START);
	pgd_end = pgd_offset_k(KASAN_SHADOW_END);
	pgd_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START);
	do {
		set_pgd(pgd_new, *pgd);
	} while (pgd++, pgd_new++, pgd != pgd_end);
}

static void __init clear_pgds(unsigned long start,
			unsigned long end)
{
	/*
	 * Remove references to kasan page tables from
	 * swapper_pg_dir. pgd_clear() can't be used
	 * here because it's nop on 2,3-level pagetable setups
	 */
	for (; start < end; start += PGDIR_SIZE)
		set_pgd(pgd_offset_k(start), __pgd(0));
}

void __init kasan_init(void)
{
	u64 kimg_shadow_start, kimg_shadow_end;
	u64 mod_shadow_start, mod_shadow_end;
	struct memblock_region *reg;
	int i;

	kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK;
	kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end));

	mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
	mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END);

	/*
	 * We are going to perform proper setup of shadow memory.
	 * At first we should unmap early shadow (clear_pgds() call bellow).
	 * However, instrumented code couldn't execute without shadow memory.
	 * tmp_pg_dir used to keep early shadow mapped until full shadow
	 * setup will be finished.
	 */
	memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
	dsb(ishst);
	cpu_replace_ttbr1(lm_alias(tmp_pg_dir));

	clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);

	kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
			   early_pfn_to_nid(virt_to_pfn(lm_alias(_text))));

	kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
				   (void *)mod_shadow_start);
	kasan_populate_zero_shadow((void *)kimg_shadow_end,
				   kasan_mem_to_shadow((void *)PAGE_OFFSET));

	if (kimg_shadow_start > mod_shadow_end)
		kasan_populate_zero_shadow((void *)mod_shadow_end,
					   (void *)kimg_shadow_start);

	for_each_memblock(memory, reg) {
		void *start = (void *)__phys_to_virt(reg->base);
		void *end = (void *)__phys_to_virt(reg->base + reg->size);

		if (start >= end)
			break;

		kasan_map_populate((unsigned long)kasan_mem_to_shadow(start),
				   (unsigned long)kasan_mem_to_shadow(end),
				   early_pfn_to_nid(virt_to_pfn(start)));
	}

	/*
	 * KAsan may reuse the contents of kasan_zero_pte directly, so we
	 * should make sure that it maps the zero page read-only.
	 */
	for (i = 0; i < PTRS_PER_PTE; i++)
		set_pte(&kasan_zero_pte[i],
			pfn_pte(sym_to_pfn(kasan_zero_page), PAGE_KERNEL_RO));

	memset(kasan_zero_page, 0, PAGE_SIZE);
	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));

	/* At this point kasan is fully initialized. Enable error messages */
	init_task.kasan_depth = 0;
	pr_info("KernelAddressSanitizer initialized\n");
}
Commit	Line	Data
39d114dd AR	1	/*
	2	* This file contains kasan initialization code for ARM64.
	3	*
	4	* Copyright (c) 2015 Samsung Electronics Co., Ltd.
	5	* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	*/
	12
	13	#define pr_fmt(fmt) "kasan: " fmt
e17d8025	14	#include <linux/bootmem.h>
39d114dd AR	15	#include <linux/kasan.h>
39d114dd AR	16	#include <linux/kernel.h>
9164bb4a	17	#include <linux/sched/task.h>
39d114dd AR	18	#include <linux/memblock.h>
39d114dd AR	19	#include <linux/start_kernel.h>
2077be67	20	#include <linux/mm.h>
39d114dd	21
c1a88e91	22	#include <asm/mmu_context.h>
f9040773	23	#include <asm/kernel-pgtable.h>
39d114dd AR	24	#include <asm/page.h>
	25	#include <asm/pgalloc.h>
	26	#include <asm/pgtable.h>
f9040773	27	#include <asm/sections.h>
39d114dd AR	28	#include <asm/tlbflush.h>
	29
	30	static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
	31
2077be67 LA	32	/*
	33	* The p*d_populate functions call virt_to_phys implicitly so they can't be used
	34	* directly on kernel symbols (bm_p*d). All the early functions are called too
	35	* early to use lm_alias so __p*d_populate functions must be used to populate
	36	* with the physical address from __pa_symbol.
	37	*/
	38
e17d8025	39	static phys_addr_t __init kasan_alloc_zeroed_page(int node)
39d114dd	40	{
e17d8025 WD	41	void *p = memblock_virt_alloc_try_nid(PAGE_SIZE, PAGE_SIZE,
	42	__pa(MAX_DMA_ADDRESS),
	43	MEMBLOCK_ALLOC_ACCESSIBLE, node);
	44	return __pa(p);
	45	}
	46
	47	static pte_t __init kasan_pte_offset(pmd_t pmd, unsigned long addr, int node,
	48	bool early)
	49	{
	50	if (pmd_none(*pmd)) {
	51	phys_addr_t pte_phys = early ? __pa_symbol(kasan_zero_pte)
	52	: kasan_alloc_zeroed_page(node);
	53	__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
	54	}
	55
	56	return early ? pte_offset_kimg(pmd, addr)
	57	: pte_offset_kernel(pmd, addr);
	58	}
39d114dd	59
e17d8025 WD	60	static pmd_t __init kasan_pmd_offset(pud_t pud, unsigned long addr, int node,
	61	bool early)
	62	{
	63	if (pud_none(*pud)) {
	64	phys_addr_t pmd_phys = early ? __pa_symbol(kasan_zero_pmd)
	65	: kasan_alloc_zeroed_page(node);
	66	__pud_populate(pud, pmd_phys, PMD_TYPE_TABLE);
	67	}
	68
	69	return early ? pmd_offset_kimg(pud, addr) : pmd_offset(pud, addr);
	70	}
	71
	72	static pud_t __init kasan_pud_offset(pgd_t pgd, unsigned long addr, int node,
	73	bool early)
	74	{
	75	if (pgd_none(*pgd)) {
	76	phys_addr_t pud_phys = early ? __pa_symbol(kasan_zero_pud)
	77	: kasan_alloc_zeroed_page(node);
	78	__pgd_populate(pgd, pud_phys, PMD_TYPE_TABLE);
	79	}
	80
	81	return early ? pud_offset_kimg(pgd, addr) : pud_offset(pgd, addr);
	82	}
	83
	84	static void __init kasan_pte_populate(pmd_t *pmd, unsigned long addr,
	85	unsigned long end, int node, bool early)
	86	{
	87	unsigned long next;
	88	pte_t *pte = kasan_pte_offset(pmd, addr, node, early);
39d114dd	89
39d114dd	90	do {
e17d8025 WD	91	phys_addr_t page_phys = early ? __pa_symbol(kasan_zero_page)
e17d8025 WD	92	: kasan_alloc_zeroed_page(node);
39d114dd	93	next = addr + PAGE_SIZE;
e17d8025	94	set_pte(pte, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
39d114dd AR	95	} while (pte++, addr = next, addr != end && pte_none(*pte));
	96	}
	97
e17d8025 WD	98	static void __init kasan_pmd_populate(pud_t *pud, unsigned long addr,
e17d8025 WD	99	unsigned long end, int node, bool early)
39d114dd	100	{
39d114dd	101	unsigned long next;
e17d8025	102	pmd_t *pmd = kasan_pmd_offset(pud, addr, node, early);
39d114dd	103
39d114dd AR	104	do {
39d114dd AR	105	next = pmd_addr_end(addr, end);
e17d8025	106	kasan_pte_populate(pmd, addr, next, node, early);
39d114dd AR	107	} while (pmd++, addr = next, addr != end && pmd_none(*pmd));
	108	}
	109
e17d8025 WD	110	static void __init kasan_pud_populate(pgd_t *pgd, unsigned long addr,
e17d8025 WD	111	unsigned long end, int node, bool early)
39d114dd	112	{
39d114dd	113	unsigned long next;
e17d8025	114	pud_t *pud = kasan_pud_offset(pgd, addr, node, early);
39d114dd	115
39d114dd AR	116	do {
39d114dd AR	117	next = pud_addr_end(addr, end);
e17d8025	118	kasan_pmd_populate(pud, addr, next, node, early);
39d114dd AR	119	} while (pud++, addr = next, addr != end && pud_none(*pud));
	120	}
	121
e17d8025 WD	122	static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
e17d8025 WD	123	int node, bool early)
39d114dd	124	{
39d114dd AR	125	unsigned long next;
	126	pgd_t *pgd;
	127
	128	pgd = pgd_offset_k(addr);
	129	do {
	130	next = pgd_addr_end(addr, end);
e17d8025	131	kasan_pud_populate(pgd, addr, next, node, early);
39d114dd AR	132	} while (pgd++, addr = next, addr != end);
	133	}
	134
e17d8025	135	/* The early shadow maps everything to a single page of zeroes */
83040123	136	asmlinkage void __init kasan_early_init(void)
39d114dd AR	137	{
	138	BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
	139	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
	140	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
e17d8025 WD	141	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE,
	142	true);
	143	}
	144
	145	/* Set up full kasan mappings, ensuring that the mapped pages are zeroed */
	146	static void __init kasan_map_populate(unsigned long start, unsigned long end,
	147	int node)
	148	{
	149	kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false);
39d114dd AR	150	}
39d114dd AR	151
068a17a5 MR	152	/*
	153	* Copy the current shadow region into a new pgdir.
	154	*/
	155	void __init kasan_copy_shadow(pgd_t *pgdir)
	156	{
	157	pgd_t pgd, pgd_new, *pgd_end;
	158
	159	pgd = pgd_offset_k(KASAN_SHADOW_START);
	160	pgd_end = pgd_offset_k(KASAN_SHADOW_END);
	161	pgd_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START);
	162	do {
	163	set_pgd(pgd_new, *pgd);
	164	} while (pgd++, pgd_new++, pgd != pgd_end);
	165	}
	166
39d114dd AR	167	static void __init clear_pgds(unsigned long start,
	168	unsigned long end)
	169	{
	170	/*
	171	* Remove references to kasan page tables from
	172	* swapper_pg_dir. pgd_clear() can't be used
	173	* here because it's nop on 2,3-level pagetable setups
	174	*/
	175	for (; start < end; start += PGDIR_SIZE)
	176	set_pgd(pgd_offset_k(start), __pgd(0));
	177	}
	178
39d114dd AR	179	void __init kasan_init(void)
39d114dd AR	180	{
f9040773	181	u64 kimg_shadow_start, kimg_shadow_end;
f80fb3a3	182	u64 mod_shadow_start, mod_shadow_end;
39d114dd	183	struct memblock_region *reg;
7b1af979	184	int i;
39d114dd	185
e17d8025 WD	186	kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK;
e17d8025 WD	187	kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end));
f9040773	188
f80fb3a3 AB	189	mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
	190	mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END);
	191
39d114dd AR	192	/*
	193	* We are going to perform proper setup of shadow memory.
	194	* At first we should unmap early shadow (clear_pgds() call bellow).
	195	* However, instrumented code couldn't execute without shadow memory.
	196	* tmp_pg_dir used to keep early shadow mapped until full shadow
	197	* setup will be finished.
	198	*/
	199	memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
c1a88e91	200	dsb(ishst);
2077be67	201	cpu_replace_ttbr1(lm_alias(tmp_pg_dir));
39d114dd AR	202
	203	clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
	204
e17d8025	205	kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
ecff48d3	206	early_pfn_to_nid(virt_to_pfn(lm_alias(_text))));
f9040773	207
39d114dd	208	kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
f80fb3a3	209	(void *)mod_shadow_start);
f9040773	210	kasan_populate_zero_shadow((void *)kimg_shadow_end,
f80fb3a3 AB	211	kasan_mem_to_shadow((void *)PAGE_OFFSET));
	212
	213	if (kimg_shadow_start > mod_shadow_end)
	214	kasan_populate_zero_shadow((void *)mod_shadow_end,
	215	(void *)kimg_shadow_start);
39d114dd AR	216
	217	for_each_memblock(memory, reg) {
	218	void start = (void )__phys_to_virt(reg->base);
	219	void end = (void )__phys_to_virt(reg->base + reg->size);
	220
	221	if (start >= end)
	222	break;
	223
e17d8025 WD	224	kasan_map_populate((unsigned long)kasan_mem_to_shadow(start),
e17d8025 WD	225	(unsigned long)kasan_mem_to_shadow(end),
ecff48d3	226	early_pfn_to_nid(virt_to_pfn(start)));
39d114dd AR	227	}
39d114dd AR	228
7b1af979 AB	229	/*
	230	* KAsan may reuse the contents of kasan_zero_pte directly, so we
	231	* should make sure that it maps the zero page read-only.
	232	*/
	233	for (i = 0; i < PTRS_PER_PTE; i++)
	234	set_pte(&kasan_zero_pte[i],
2077be67	235	pfn_pte(sym_to_pfn(kasan_zero_page), PAGE_KERNEL_RO));
7b1af979	236
39d114dd	237	memset(kasan_zero_page, 0, PAGE_SIZE);
2077be67	238	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
39d114dd AR	239
	240	/* At this point kasan is fully initialized. Enable error messages */
	241	init_task.kasan_depth = 0;
	242	pr_info("KernelAddressSanitizer initialized\n");
	243	}