[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / head64.c

/*
 *  prepare to run common code
 *
 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
 */

#define DISABLE_BRANCH_PROFILING
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/percpu.h>
#include <linux/start_kernel.h>
#include <linux/io.h>
#include <linux/memblock.h>

#include <asm/processor.h>
#include <asm/proto.h>
#include <asm/smp.h>
#include <asm/setup.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/kdebug.h>
#include <asm/e820/api.h>
#include <asm/bios_ebda.h>
#include <asm/bootparam_utils.h>
#include <asm/microcode.h>
#include <asm/kasan.h>

/*
 * Manage page tables very early on.
 */
extern pgd_t early_top_pgt[PTRS_PER_PGD];
extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);

#define __head	__section(.head.text)

static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
{
	return ptr - (void *)_text + (void *)physaddr;
}

void __head __startup_64(unsigned long physaddr)
{
	unsigned long load_delta, *p;
	pgdval_t *pgd;
	p4dval_t *p4d;
	pudval_t *pud;
	pmdval_t *pmd, pmd_entry;
	int i;
	unsigned int *next_pgt_ptr;

	/* Is the address too large? */
	if (physaddr >> MAX_PHYSMEM_BITS)
		for (;;);

	/*
	 * Compute the delta between the address I am compiled to run at
	 * and the address I am actually running at.
	 */
	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);

	/* Is the address not 2M aligned? */
	if (load_delta & ~PMD_PAGE_MASK)
		for (;;);

	/* Fixup the physical addresses in the page table */

	pgd = fixup_pointer(&early_top_pgt, physaddr);
	pgd[pgd_index(__START_KERNEL_map)] += load_delta;

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
		p4d[511] += load_delta;
	}

	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	pud[510] += load_delta;
	pud[511] += load_delta;

	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	pmd[506] += load_delta;

	/*
	 * Set up the identity mapping for the switchover.  These
	 * entries should *NOT* have the global bit set!  This also
	 * creates a bunch of nonsense entries but that is fine --
	 * it avoids problems around wraparound.
	 */
	next_pgt_ptr = fixup_pointer(&next_early_pgt, physaddr);
	pud = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);
	pmd = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;
		pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;

		i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
		p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
		p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	} else {
		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
		pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	}

	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;
	pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;

	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	pmd_entry +=  physaddr;

	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
		int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
		pmd[idx] = pmd_entry + i * PMD_SIZE;
	}

	/*
	 * Fixup the kernel text+data virtual addresses. Note that
	 * we might write invalid pmds, when the kernel is relocated
	 * cleanup_highmap() fixes this up along with the mappings
	 * beyond _end.
	 */

	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	for (i = 0; i < PTRS_PER_PMD; i++) {
		if (pmd[i] & _PAGE_PRESENT)
			pmd[i] += load_delta;
	}

	/* Fixup phys_base */
	p = fixup_pointer(&phys_base, physaddr);
	*p += load_delta;
}

/* Wipe all early page tables except for the kernel symbol map */
static void __init reset_early_page_tables(void)
{
	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
	next_early_pgt = 0;
	write_cr3(__pa_nodebug(early_top_pgt));
}

/* Create a new PMD entry */
int __init early_make_pgtable(unsigned long address)
{
	unsigned long physaddr = address - __PAGE_OFFSET;
	pgdval_t pgd, *pgd_p;
	p4dval_t p4d, *p4d_p;
	pudval_t pud, *pud_p;
	pmdval_t pmd, *pmd_p;

	/* Invalid address or early pgt is done ?  */
	if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
		return -1;

again:
	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
	pgd = *pgd_p;

	/*
	 * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	 * critical -- __PAGE_OFFSET would point us back into the dynamic
	 * range and we might end up looping forever...
	 */
	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
		p4d_p = pgd_p;
	else if (pgd)
		p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
		*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	p4d_p += p4d_index(address);
	p4d = *p4d_p;

	if (p4d)
		pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
		*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pud_p += pud_index(address);
	pud = *pud_p;

	if (pud)
		pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
		*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
	pmd_p[pmd_index(address)] = pmd;

	return 0;
}

/* Don't add a printk in there. printk relies on the PDA which is not initialized 
   yet. */
static void __init clear_bss(void)
{
	memset(__bss_start, 0,
	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
}

static unsigned long get_cmd_line_ptr(void)
{
	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;

	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;

	return cmd_line_ptr;
}

static void __init copy_bootdata(char *real_mode_data)
{
	char * command_line;
	unsigned long cmd_line_ptr;

	memcpy(&boot_params, real_mode_data, sizeof boot_params);
	sanitize_boot_params(&boot_params);
	cmd_line_ptr = get_cmd_line_ptr();
	if (cmd_line_ptr) {
		command_line = __va(cmd_line_ptr);
		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
	}
}

asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
{
	int i;

	/*
	 * Build-time sanity checks on the kernel image and module
	 * area mappings. (these are purely build-time and produce no code)
	 */
	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
				(__START_KERNEL & PGDIR_MASK)));
	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);

	cr4_init_shadow();

	/* Kill off the identity-map trampoline */
	reset_early_page_tables();

	clear_bss();

	clear_page(init_top_pgt);

	kasan_early_init();

	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
		set_intr_gate(i, early_idt_handler_array[i]);
	load_idt((const struct desc_ptr *)&idt_descr);

	copy_bootdata(__va(real_mode_data));

	/*
	 * Load microcode early on BSP.
	 */
	load_ucode_bsp();

	/* set init_top_pgt kernel high mapping*/
	init_top_pgt[511] = early_top_pgt[511];

	x86_64_start_reservations(real_mode_data);
}

void __init x86_64_start_reservations(char *real_mode_data)
{
	/* version is always not zero if it is copied */
	if (!boot_params.hdr.version)
		copy_bootdata(__va(real_mode_data));

	x86_early_init_platform_quirks();

	switch (boot_params.hdr.hardware_subarch) {
	case X86_SUBARCH_INTEL_MID:
		x86_intel_mid_early_setup();
		break;
	default:
		break;
	}

	start_kernel();
}
Commit	Line	Data
1da177e4	1	/*
835c34a1	2	* prepare to run common code
1da177e4 LT	3	*
1da177e4 LT	4	* Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
1da177e4 LT	5	*/
1da177e4 LT	6
be3606ff	7	#define DISABLE_BRANCH_PROFILING
1da177e4 LT	8	#include <linux/init.h>
	9	#include <linux/linkage.h>
	10	#include <linux/types.h>
	11	#include <linux/kernel.h>
	12	#include <linux/string.h>
	13	#include <linux/percpu.h>
eaf76e8b	14	#include <linux/start_kernel.h>
8b664aa6	15	#include <linux/io.h>
72d7c3b3	16	#include <linux/memblock.h>
1da177e4 LT	17
	18	#include <asm/processor.h>
	19	#include <asm/proto.h>
	20	#include <asm/smp.h>
1da177e4 LT	21	#include <asm/setup.h>
1da177e4 LT	22	#include <asm/desc.h>
f6c2e333	23	#include <asm/pgtable.h>
cfd243d4	24	#include <asm/tlbflush.h>
2bc0414e	25	#include <asm/sections.h>
718fc13b	26	#include <asm/kdebug.h>
66441bd3	27	#include <asm/e820/api.h>
47a3d5da	28	#include <asm/bios_ebda.h>
5dcd14ec	29	#include <asm/bootparam_utils.h>
feddc9de	30	#include <asm/microcode.h>
ef7f0d6a	31	#include <asm/kasan.h>
1da177e4	32
8170e6be PA	33	/*
	34	* Manage page tables very early on.
	35	*/
65ade2f8	36	extern pgd_t early_top_pgt[PTRS_PER_PGD];
8170e6be	37	extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
c88d7150	38	static unsigned int __initdata next_early_pgt;
5e427ec2	39	pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL \| _PAGE_NX);
8170e6be	40
26179670 KS	41	#define __head __section(.head.text)
	42
	43	static void __head fixup_pointer(void ptr, unsigned long physaddr)
c88d7150 KS	44	{
	45	return ptr - (void )_text + (void )physaddr;
	46	}
	47
26179670	48	void __head __startup_64(unsigned long physaddr)
c88d7150 KS	49	{
	50	unsigned long load_delta, *p;
	51	pgdval_t *pgd;
032370b9	52	p4dval_t *p4d;
c88d7150 KS	53	pudval_t *pud;
	54	pmdval_t *pmd, pmd_entry;
	55	int i;
187e91fe	56	unsigned int *next_pgt_ptr;
c88d7150 KS	57
	58	/* Is the address too large? */
	59	if (physaddr >> MAX_PHYSMEM_BITS)
	60	for (;;);
	61
	62	/*
	63	* Compute the delta between the address I am compiled to run at
	64	* and the address I am actually running at.
	65	*/
	66	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
	67
	68	/* Is the address not 2M aligned? */
	69	if (load_delta & ~PMD_PAGE_MASK)
	70	for (;;);
	71
	72	/* Fixup the physical addresses in the page table */
	73
65ade2f8	74	pgd = fixup_pointer(&early_top_pgt, physaddr);
c88d7150 KS	75	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
c88d7150 KS	76
032370b9 KS	77	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	78	p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
	79	p4d[511] += load_delta;
	80	}
	81
c88d7150 KS	82	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	83	pud[510] += load_delta;
	84	pud[511] += load_delta;
	85
	86	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	87	pmd[506] += load_delta;
	88
	89	/*
	90	* Set up the identity mapping for the switchover. These
	91	* entries should NOT have the global bit set! This also
	92	* creates a bunch of nonsense entries but that is fine --
	93	* it avoids problems around wraparound.
	94	*/
187e91fe AP	95	next_pgt_ptr = fixup_pointer(&next_early_pgt, physaddr);
	96	pud = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);
	97	pmd = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);
c88d7150	98
032370b9 KS	99	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	100	p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	101
	102	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
	103	pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;
	104	pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;
	105
	106	i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
	107	p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
	108	p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	109	} else {
	110	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
	111	pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;
	112	pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;
	113	}
c88d7150 KS	114
	115	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	116	pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;
	117	pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;
	118
	119	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	120	pmd_entry += physaddr;
	121
	122	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
	123	int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
	124	pmd[idx] = pmd_entry + i * PMD_SIZE;
	125	}
	126
	127	/*
	128	* Fixup the kernel text+data virtual addresses. Note that
	129	* we might write invalid pmds, when the kernel is relocated
	130	* cleanup_highmap() fixes this up along with the mappings
	131	* beyond _end.
	132	*/
	133
	134	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	135	for (i = 0; i < PTRS_PER_PMD; i++) {
	136	if (pmd[i] & _PAGE_PRESENT)
	137	pmd[i] += load_delta;
	138	}
	139
	140	/* Fixup phys_base */
	141	p = fixup_pointer(&phys_base, physaddr);
	142	*p += load_delta;
	143	}
	144
8170e6be PA	145	/* Wipe all early page tables except for the kernel symbol map */
8170e6be PA	146	static void __init reset_early_page_tables(void)
cfd243d4	147	{
65ade2f8	148	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
8170e6be	149	next_early_pgt = 0;
65ade2f8	150	write_cr3(__pa_nodebug(early_top_pgt));
8170e6be PA	151	}
	152
	153	/* Create a new PMD entry */
	154	int __init early_make_pgtable(unsigned long address)
	155	{
	156	unsigned long physaddr = address - __PAGE_OFFSET;
8170e6be	157	pgdval_t pgd, *pgd_p;
032370b9	158	p4dval_t p4d, *p4d_p;
6b9c75ac	159	pudval_t pud, *pud_p;
8170e6be PA	160	pmdval_t pmd, *pmd_p;
	161
	162	/* Invalid address or early pgt is done ? */
65ade2f8	163	if (physaddr >= MAXMEM \|\| read_cr3_pa() != __pa_nodebug(early_top_pgt))
8170e6be PA	164	return -1;
8170e6be PA	165
6b9c75ac	166	again:
65ade2f8	167	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
8170e6be PA	168	pgd = *pgd_p;
	169
	170	/*
	171	* The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	172	* critical -- __PAGE_OFFSET would point us back into the dynamic
	173	* range and we might end up looping forever...
	174	*/
032370b9 KS	175	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
	176	p4d_p = pgd_p;
	177	else if (pgd)
	178	p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	179	else {
	180	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	181	reset_early_page_tables();
	182	goto again;
	183	}
	184
	185	p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
	186	memset(p4d_p, 0, sizeof(p4d_p) PTRS_PER_P4D);
	187	*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	188	}
	189	p4d_p += p4d_index(address);
	190	p4d = *p4d_p;
	191
	192	if (p4d)
	193	pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
6b9c75ac YL	194	else {
6b9c75ac YL	195	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
8170e6be	196	reset_early_page_tables();
6b9c75ac YL	197	goto again;
6b9c75ac YL	198	}
8170e6be PA	199
8170e6be PA	200	pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	201	memset(pud_p, 0, sizeof(pud_p) PTRS_PER_PUD);
032370b9	202	*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
8170e6be	203	}
6b9c75ac YL	204	pud_p += pud_index(address);
6b9c75ac YL	205	pud = *pud_p;
8170e6be	206
6b9c75ac YL	207	if (pud)
	208	pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	209	else {
	210	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	211	reset_early_page_tables();
	212	goto again;
	213	}
	214
	215	pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	216	memset(pmd_p, 0, sizeof(pmd_p) PTRS_PER_PMD);
6b9c75ac YL	217	*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
6b9c75ac YL	218	}
78d77df7	219	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
6b9c75ac	220	pmd_p[pmd_index(address)] = pmd;
8170e6be PA	221
8170e6be PA	222	return 0;
cfd243d4 VG	223	}
cfd243d4 VG	224
1da177e4 LT	225	/* Don't add a printk in there. printk relies on the PDA which is not initialized
	226	yet. */
	227	static void __init clear_bss(void)
	228	{
1da177e4	229	memset(__bss_start, 0,
2bc0414e	230	(unsigned long) __bss_stop - (unsigned long) __bss_start);
1da177e4 LT	231	}
1da177e4 LT	232
f1da834c YL	233	static unsigned long get_cmd_line_ptr(void)
	234	{
	235	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
	236
ee92d815 YL	237	cmd_line_ptr \|= (u64)boot_params.ext_cmd_line_ptr << 32;
ee92d815 YL	238
f1da834c YL	239	return cmd_line_ptr;
	240	}
	241
1da177e4 LT	242	static void __init copy_bootdata(char *real_mode_data)
1da177e4 LT	243	{
1da177e4	244	char * command_line;
f1da834c	245	unsigned long cmd_line_ptr;
1da177e4	246
30c82645	247	memcpy(&boot_params, real_mode_data, sizeof boot_params);
5dcd14ec	248	sanitize_boot_params(&boot_params);
f1da834c YL	249	cmd_line_ptr = get_cmd_line_ptr();
	250	if (cmd_line_ptr) {
	251	command_line = __va(cmd_line_ptr);
30c82645	252	memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
1da177e4	253	}
1da177e4 LT	254	}
1da177e4 LT	255
2605fc21	256	asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
1da177e4	257	{
1da177e4 LT	258	int i;
1da177e4 LT	259
b4e0409a IM	260	/*
	261	* Build-time sanity checks on the kernel image and module
	262	* area mappings. (these are purely build-time and produce no code)
	263	*/
8e3c2a8c BP	264	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
8e3c2a8c BP	265	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
b4e0409a	266	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
8e3c2a8c	267	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
b4e0409a IM	268	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	269	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	270	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
	271	(__START_KERNEL & PGDIR_MASK)));
66d4bdf2	272	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
b4e0409a	273
1e02ce4c AL	274	cr4_init_shadow();
1e02ce4c AL	275
8170e6be PA	276	/* Kill off the identity-map trampoline */
	277	reset_early_page_tables();
	278
3df0af0e YL	279	clear_bss();
3df0af0e YL	280
65ade2f8	281	clear_page(init_top_pgt);
d0f77d4d	282
5d5aa3cf AP	283	kasan_early_init();
5d5aa3cf AP	284
ac630dd9	285	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
425be567	286	set_intr_gate(i, early_idt_handler_array[i]);
9d1c6e7c	287	load_idt((const struct desc_ptr *)&idt_descr);
f6c2e333	288
fa2bbce9 YL	289	copy_bootdata(__va(real_mode_data));
fa2bbce9 YL	290
feddc9de FY	291	/*
	292	* Load microcode early on BSP.
	293	*/
	294	load_ucode_bsp();
	295
65ade2f8 KS	296	/* set init_top_pgt kernel high mapping*/
65ade2f8 KS	297	init_top_pgt[511] = early_top_pgt[511];
8170e6be	298
f97013fd JF	299	x86_64_start_reservations(real_mode_data);
	300	}
	301
	302	void __init x86_64_start_reservations(char *real_mode_data)
	303	{
fa2bbce9 YL	304	/* version is always not zero if it is copied */
	305	if (!boot_params.hdr.version)
	306	copy_bootdata(__va(real_mode_data));
9de819fe	307
8d152e7a	308	x86_early_init_platform_quirks();
75175278	309
3fda5bb4 AS	310	switch (boot_params.hdr.hardware_subarch) {
	311	case X86_SUBARCH_INTEL_MID:
	312	x86_intel_mid_early_setup();
	313	break;
	314	default:
	315	break;
	316	}
	317
1da177e4 LT	318	start_kernel();
1da177e4 LT	319	}