[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / head64.c

/*
 *  prepare to run common code
 *
 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
 */

#define DISABLE_BRANCH_PROFILING
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/percpu.h>
#include <linux/start_kernel.h>
#include <linux/io.h>
#include <linux/memblock.h>

#include <asm/processor.h>
#include <asm/proto.h>
#include <asm/smp.h>
#include <asm/setup.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/kdebug.h>
#include <asm/e820/api.h>
#include <asm/bios_ebda.h>
#include <asm/bootparam_utils.h>
#include <asm/microcode.h>
#include <asm/kasan.h>

/*
 * Manage page tables very early on.
 */
extern pgd_t early_top_pgt[PTRS_PER_PGD];
extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);

#define __head	__section(.head.text)

static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
{
	return ptr - (void *)_text + (void *)physaddr;
}

void __head __startup_64(unsigned long physaddr)
{
	unsigned long load_delta, *p;
	pgdval_t *pgd;
	p4dval_t *p4d;
	pudval_t *pud;
	pmdval_t *pmd, pmd_entry;
	int i;

	/* Is the address too large? */
	if (physaddr >> MAX_PHYSMEM_BITS)
		for (;;);

	/*
	 * Compute the delta between the address I am compiled to run at
	 * and the address I am actually running at.
	 */
	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);

	/* Is the address not 2M aligned? */
	if (load_delta & ~PMD_PAGE_MASK)
		for (;;);

	/* Fixup the physical addresses in the page table */

	pgd = fixup_pointer(&early_top_pgt, physaddr);
	pgd[pgd_index(__START_KERNEL_map)] += load_delta;

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
		p4d[511] += load_delta;
	}

	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	pud[510] += load_delta;
	pud[511] += load_delta;

	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	pmd[506] += load_delta;

	/*
	 * Set up the identity mapping for the switchover.  These
	 * entries should *NOT* have the global bit set!  This also
	 * creates a bunch of nonsense entries but that is fine --
	 * it avoids problems around wraparound.
	 */

	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;
		pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;

		i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
		p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
		p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	} else {
		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
		pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	}

	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;
	pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;

	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	pmd_entry +=  physaddr;

	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
		int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
		pmd[idx] = pmd_entry + i * PMD_SIZE;
	}

	/*
	 * Fixup the kernel text+data virtual addresses. Note that
	 * we might write invalid pmds, when the kernel is relocated
	 * cleanup_highmap() fixes this up along with the mappings
	 * beyond _end.
	 */

	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	for (i = 0; i < PTRS_PER_PMD; i++) {
		if (pmd[i] & _PAGE_PRESENT)
			pmd[i] += load_delta;
	}

	/* Fixup phys_base */
	p = fixup_pointer(&phys_base, physaddr);
	*p += load_delta;
}

/* Wipe all early page tables except for the kernel symbol map */
static void __init reset_early_page_tables(void)
{
	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
	next_early_pgt = 0;
	write_cr3(__pa_nodebug(early_top_pgt));
}

/* Create a new PMD entry */
int __init early_make_pgtable(unsigned long address)
{
	unsigned long physaddr = address - __PAGE_OFFSET;
	pgdval_t pgd, *pgd_p;
	p4dval_t p4d, *p4d_p;
	pudval_t pud, *pud_p;
	pmdval_t pmd, *pmd_p;

	/* Invalid address or early pgt is done ?  */
	if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
		return -1;

again:
	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
	pgd = *pgd_p;

	/*
	 * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	 * critical -- __PAGE_OFFSET would point us back into the dynamic
	 * range and we might end up looping forever...
	 */
	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
		p4d_p = pgd_p;
	else if (pgd)
		p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
		*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	p4d_p += p4d_index(address);
	p4d = *p4d_p;

	if (p4d)
		pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
		*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pud_p += pud_index(address);
	pud = *pud_p;

	if (pud)
		pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
		*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
	pmd_p[pmd_index(address)] = pmd;

	return 0;
}

/* Don't add a printk in there. printk relies on the PDA which is not initialized 
   yet. */
static void __init clear_bss(void)
{
	memset(__bss_start, 0,
	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
}

static unsigned long get_cmd_line_ptr(void)
{
	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;

	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;

	return cmd_line_ptr;
}

static void __init copy_bootdata(char *real_mode_data)
{
	char * command_line;
	unsigned long cmd_line_ptr;

	memcpy(&boot_params, real_mode_data, sizeof boot_params);
	sanitize_boot_params(&boot_params);
	cmd_line_ptr = get_cmd_line_ptr();
	if (cmd_line_ptr) {
		command_line = __va(cmd_line_ptr);
		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
	}
}

asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
{
	int i;

	/*
	 * Build-time sanity checks on the kernel image and module
	 * area mappings. (these are purely build-time and produce no code)
	 */
	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
				(__START_KERNEL & PGDIR_MASK)));
	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);

	cr4_init_shadow();

	/* Kill off the identity-map trampoline */
	reset_early_page_tables();

	clear_bss();

	clear_page(init_top_pgt);

	kasan_early_init();

	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
		set_intr_gate(i, early_idt_handler_array[i]);
	load_idt((const struct desc_ptr *)&idt_descr);

	copy_bootdata(__va(real_mode_data));

	/*
	 * Load microcode early on BSP.
	 */
	load_ucode_bsp();

	/* set init_top_pgt kernel high mapping*/
	init_top_pgt[511] = early_top_pgt[511];

	x86_64_start_reservations(real_mode_data);
}

void __init x86_64_start_reservations(char *real_mode_data)
{
	/* version is always not zero if it is copied */
	if (!boot_params.hdr.version)
		copy_bootdata(__va(real_mode_data));

	x86_early_init_platform_quirks();

	switch (boot_params.hdr.hardware_subarch) {
	case X86_SUBARCH_INTEL_MID:
		x86_intel_mid_early_setup();
		break;
	default:
		break;
	}

	start_kernel();
}
Commit	Line	Data
1da177e4	1	/*
835c34a1	2	* prepare to run common code
1da177e4 LT	3	*
1da177e4 LT	4	* Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
1da177e4 LT	5	*/
1da177e4 LT	6
be3606ff	7	#define DISABLE_BRANCH_PROFILING
1da177e4 LT	8	#include <linux/init.h>
	9	#include <linux/linkage.h>
	10	#include <linux/types.h>
	11	#include <linux/kernel.h>
	12	#include <linux/string.h>
	13	#include <linux/percpu.h>
eaf76e8b	14	#include <linux/start_kernel.h>
8b664aa6	15	#include <linux/io.h>
72d7c3b3	16	#include <linux/memblock.h>
1da177e4 LT	17
	18	#include <asm/processor.h>
	19	#include <asm/proto.h>
	20	#include <asm/smp.h>
1da177e4 LT	21	#include <asm/setup.h>
1da177e4 LT	22	#include <asm/desc.h>
f6c2e333	23	#include <asm/pgtable.h>
cfd243d4	24	#include <asm/tlbflush.h>
2bc0414e	25	#include <asm/sections.h>
718fc13b	26	#include <asm/kdebug.h>
66441bd3	27	#include <asm/e820/api.h>
47a3d5da	28	#include <asm/bios_ebda.h>
5dcd14ec	29	#include <asm/bootparam_utils.h>
feddc9de	30	#include <asm/microcode.h>
ef7f0d6a	31	#include <asm/kasan.h>
1da177e4	32
8170e6be PA	33	/*
	34	* Manage page tables very early on.
	35	*/
65ade2f8	36	extern pgd_t early_top_pgt[PTRS_PER_PGD];
8170e6be	37	extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
c88d7150	38	static unsigned int __initdata next_early_pgt;
5e427ec2	39	pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL \| _PAGE_NX);
8170e6be	40
26179670 KS	41	#define __head __section(.head.text)
	42
	43	static void __head fixup_pointer(void ptr, unsigned long physaddr)
c88d7150 KS	44	{
	45	return ptr - (void )_text + (void )physaddr;
	46	}
	47
26179670	48	void __head __startup_64(unsigned long physaddr)
c88d7150 KS	49	{
	50	unsigned long load_delta, *p;
	51	pgdval_t *pgd;
032370b9	52	p4dval_t *p4d;
c88d7150 KS	53	pudval_t *pud;
	54	pmdval_t *pmd, pmd_entry;
	55	int i;
	56
	57	/* Is the address too large? */
	58	if (physaddr >> MAX_PHYSMEM_BITS)
	59	for (;;);
	60
	61	/*
	62	* Compute the delta between the address I am compiled to run at
	63	* and the address I am actually running at.
	64	*/
	65	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
	66
	67	/* Is the address not 2M aligned? */
	68	if (load_delta & ~PMD_PAGE_MASK)
	69	for (;;);
	70
	71	/* Fixup the physical addresses in the page table */
	72
65ade2f8	73	pgd = fixup_pointer(&early_top_pgt, physaddr);
c88d7150 KS	74	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
c88d7150 KS	75
032370b9 KS	76	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	77	p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
	78	p4d[511] += load_delta;
	79	}
	80
c88d7150 KS	81	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	82	pud[510] += load_delta;
	83	pud[511] += load_delta;
	84
	85	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	86	pmd[506] += load_delta;
	87
	88	/*
	89	* Set up the identity mapping for the switchover. These
	90	* entries should NOT have the global bit set! This also
	91	* creates a bunch of nonsense entries but that is fine --
	92	* it avoids problems around wraparound.
	93	*/
	94
	95	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	96	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	97
032370b9 KS	98	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	99	p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	100
	101	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
	102	pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;
	103	pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;
	104
	105	i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
	106	p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
	107	p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	108	} else {
	109	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
	110	pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
	111	pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	112	}
c88d7150 KS	113
	114	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	115	pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;
	116	pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;
	117
	118	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	119	pmd_entry += physaddr;
	120
	121	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
	122	int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
	123	pmd[idx] = pmd_entry + i * PMD_SIZE;
	124	}
	125
	126	/*
	127	* Fixup the kernel text+data virtual addresses. Note that
	128	* we might write invalid pmds, when the kernel is relocated
	129	* cleanup_highmap() fixes this up along with the mappings
	130	* beyond _end.
	131	*/
	132
	133	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	134	for (i = 0; i < PTRS_PER_PMD; i++) {
	135	if (pmd[i] & _PAGE_PRESENT)
	136	pmd[i] += load_delta;
	137	}
	138
	139	/* Fixup phys_base */
	140	p = fixup_pointer(&phys_base, physaddr);
	141	*p += load_delta;
	142	}
	143
8170e6be PA	144	/* Wipe all early page tables except for the kernel symbol map */
8170e6be PA	145	static void __init reset_early_page_tables(void)
cfd243d4	146	{
65ade2f8	147	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
8170e6be	148	next_early_pgt = 0;
65ade2f8	149	write_cr3(__pa_nodebug(early_top_pgt));
8170e6be PA	150	}
	151
	152	/* Create a new PMD entry */
	153	int __init early_make_pgtable(unsigned long address)
	154	{
	155	unsigned long physaddr = address - __PAGE_OFFSET;
8170e6be	156	pgdval_t pgd, *pgd_p;
032370b9	157	p4dval_t p4d, *p4d_p;
6b9c75ac	158	pudval_t pud, *pud_p;
8170e6be PA	159	pmdval_t pmd, *pmd_p;
	160
	161	/* Invalid address or early pgt is done ? */
65ade2f8	162	if (physaddr >= MAXMEM \|\| read_cr3_pa() != __pa_nodebug(early_top_pgt))
8170e6be PA	163	return -1;
8170e6be PA	164
6b9c75ac	165	again:
65ade2f8	166	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
8170e6be PA	167	pgd = *pgd_p;
	168
	169	/*
	170	* The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	171	* critical -- __PAGE_OFFSET would point us back into the dynamic
	172	* range and we might end up looping forever...
	173	*/
032370b9 KS	174	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
	175	p4d_p = pgd_p;
	176	else if (pgd)
	177	p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	178	else {
	179	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	180	reset_early_page_tables();
	181	goto again;
	182	}
	183
	184	p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
	185	memset(p4d_p, 0, sizeof(p4d_p) PTRS_PER_P4D);
	186	*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	187	}
	188	p4d_p += p4d_index(address);
	189	p4d = *p4d_p;
	190
	191	if (p4d)
	192	pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
6b9c75ac YL	193	else {
6b9c75ac YL	194	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
8170e6be	195	reset_early_page_tables();
6b9c75ac YL	196	goto again;
6b9c75ac YL	197	}
8170e6be PA	198
8170e6be PA	199	pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	200	memset(pud_p, 0, sizeof(pud_p) PTRS_PER_PUD);
032370b9	201	*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
8170e6be	202	}
6b9c75ac YL	203	pud_p += pud_index(address);
6b9c75ac YL	204	pud = *pud_p;
8170e6be	205
6b9c75ac YL	206	if (pud)
	207	pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	208	else {
	209	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	210	reset_early_page_tables();
	211	goto again;
	212	}
	213
	214	pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	215	memset(pmd_p, 0, sizeof(pmd_p) PTRS_PER_PMD);
6b9c75ac YL	216	*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
6b9c75ac YL	217	}
78d77df7	218	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
6b9c75ac	219	pmd_p[pmd_index(address)] = pmd;
8170e6be PA	220
8170e6be PA	221	return 0;
cfd243d4 VG	222	}
cfd243d4 VG	223
1da177e4 LT	224	/* Don't add a printk in there. printk relies on the PDA which is not initialized
	225	yet. */
	226	static void __init clear_bss(void)
	227	{
1da177e4	228	memset(__bss_start, 0,
2bc0414e	229	(unsigned long) __bss_stop - (unsigned long) __bss_start);
1da177e4 LT	230	}
1da177e4 LT	231
f1da834c YL	232	static unsigned long get_cmd_line_ptr(void)
	233	{
	234	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
	235
ee92d815 YL	236	cmd_line_ptr \|= (u64)boot_params.ext_cmd_line_ptr << 32;
ee92d815 YL	237
f1da834c YL	238	return cmd_line_ptr;
	239	}
	240
1da177e4 LT	241	static void __init copy_bootdata(char *real_mode_data)
1da177e4 LT	242	{
1da177e4	243	char * command_line;
f1da834c	244	unsigned long cmd_line_ptr;
1da177e4	245
30c82645	246	memcpy(&boot_params, real_mode_data, sizeof boot_params);
5dcd14ec	247	sanitize_boot_params(&boot_params);
f1da834c YL	248	cmd_line_ptr = get_cmd_line_ptr();
	249	if (cmd_line_ptr) {
	250	command_line = __va(cmd_line_ptr);
30c82645	251	memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
1da177e4	252	}
1da177e4 LT	253	}
1da177e4 LT	254
2605fc21	255	asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
1da177e4	256	{
1da177e4 LT	257	int i;
1da177e4 LT	258
b4e0409a IM	259	/*
	260	* Build-time sanity checks on the kernel image and module
	261	* area mappings. (these are purely build-time and produce no code)
	262	*/
8e3c2a8c BP	263	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
8e3c2a8c BP	264	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
b4e0409a	265	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
8e3c2a8c	266	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
b4e0409a IM	267	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	268	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	269	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
	270	(__START_KERNEL & PGDIR_MASK)));
66d4bdf2	271	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
b4e0409a	272
1e02ce4c AL	273	cr4_init_shadow();
1e02ce4c AL	274
8170e6be PA	275	/* Kill off the identity-map trampoline */
	276	reset_early_page_tables();
	277
3df0af0e YL	278	clear_bss();
3df0af0e YL	279
65ade2f8	280	clear_page(init_top_pgt);
d0f77d4d	281
5d5aa3cf AP	282	kasan_early_init();
5d5aa3cf AP	283
ac630dd9	284	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
425be567	285	set_intr_gate(i, early_idt_handler_array[i]);
9d1c6e7c	286	load_idt((const struct desc_ptr *)&idt_descr);
f6c2e333	287
fa2bbce9 YL	288	copy_bootdata(__va(real_mode_data));
fa2bbce9 YL	289
feddc9de FY	290	/*
	291	* Load microcode early on BSP.
	292	*/
	293	load_ucode_bsp();
	294
65ade2f8 KS	295	/* set init_top_pgt kernel high mapping*/
65ade2f8 KS	296	init_top_pgt[511] = early_top_pgt[511];
8170e6be	297
f97013fd JF	298	x86_64_start_reservations(real_mode_data);
	299	}
	300
	301	void __init x86_64_start_reservations(char *real_mode_data)
	302	{
fa2bbce9 YL	303	/* version is always not zero if it is copied */
	304	if (!boot_params.hdr.version)
	305	copy_bootdata(__va(real_mode_data));
9de819fe	306
8d152e7a	307	x86_early_init_platform_quirks();
75175278	308
3fda5bb4 AS	309	switch (boot_params.hdr.hardware_subarch) {
	310	case X86_SUBARCH_INTEL_MID:
	311	x86_intel_mid_early_setup();
	312	break;
	313	default:
	314	break;
	315	}
	316
1da177e4 LT	317	start_kernel();
1da177e4 LT	318	}