[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / head64.c

/*
 *  prepare to run common code
 *
 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
 */

#define DISABLE_BRANCH_PROFILING
#include <linux/init.h>
#include <linux/linkage.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/percpu.h>
#include <linux/start_kernel.h>
#include <linux/io.h>
#include <linux/memblock.h>
#include <linux/mem_encrypt.h>

#include <asm/processor.h>
#include <asm/proto.h>
#include <asm/smp.h>
#include <asm/setup.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/kdebug.h>
#include <asm/e820/api.h>
#include <asm/bios_ebda.h>
#include <asm/bootparam_utils.h>
#include <asm/microcode.h>
#include <asm/kasan.h>

/*
 * Manage page tables very early on.
 */
extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);

#define __head	__section(.head.text)

static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
{
	return ptr - (void *)_text + (void *)physaddr;
}

unsigned long __head __startup_64(unsigned long physaddr,
				  struct boot_params *bp)
{
	unsigned long load_delta, *p;
	unsigned long pgtable_flags;
	pgdval_t *pgd;
	p4dval_t *p4d;
	pudval_t *pud;
	pmdval_t *pmd, pmd_entry;
	int i;

	/* Is the address too large? */
	if (physaddr >> MAX_PHYSMEM_BITS)
		for (;;);

	/*
	 * Compute the delta between the address I am compiled to run at
	 * and the address I am actually running at.
	 */
	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);

	/* Is the address not 2M aligned? */
	if (load_delta & ~PMD_PAGE_MASK)
		for (;;);

	/* Activate Secure Memory Encryption (SME) if supported and enabled */
	sme_enable(bp);

	/* Include the SME encryption mask in the fixup value */
	load_delta += sme_get_me_mask();

	/* Fixup the physical addresses in the page table */

	pgd = fixup_pointer(&early_top_pgt, physaddr);
	pgd[pgd_index(__START_KERNEL_map)] += load_delta;

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
		p4d[511] += load_delta;
	}

	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	pud[510] += load_delta;
	pud[511] += load_delta;

	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	pmd[506] += load_delta;

	/*
	 * Set up the identity mapping for the switchover.  These
	 * entries should *NOT* have the global bit set!  This also
	 * creates a bunch of nonsense entries but that is fine --
	 * it avoids problems around wraparound.
	 */

	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();

	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
		p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;

		i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
		p4d[i + 0] = (pgdval_t)pud + pgtable_flags;
		p4d[i + 1] = (pgdval_t)pud + pgtable_flags;
	} else {
		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
	}

	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
	pud[i + 0] = (pudval_t)pmd + pgtable_flags;
	pud[i + 1] = (pudval_t)pmd + pgtable_flags;

	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
	pmd_entry += sme_get_me_mask();
	pmd_entry +=  physaddr;

	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
		int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
		pmd[idx] = pmd_entry + i * PMD_SIZE;
	}

	/*
	 * Fixup the kernel text+data virtual addresses. Note that
	 * we might write invalid pmds, when the kernel is relocated
	 * cleanup_highmap() fixes this up along with the mappings
	 * beyond _end.
	 */

	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	for (i = 0; i < PTRS_PER_PMD; i++) {
		if (pmd[i] & _PAGE_PRESENT)
			pmd[i] += load_delta;
	}

	/*
	 * Fixup phys_base - remove the memory encryption mask to obtain
	 * the true physical address.
	 */
	p = fixup_pointer(&phys_base, physaddr);
	*p += load_delta - sme_get_me_mask();

	/* Encrypt the kernel (if SME is active) */
	sme_encrypt_kernel();

	/*
	 * Return the SME encryption mask (if SME is active) to be used as a
	 * modifier for the initial pgdir entry programmed into CR3.
	 */
	return sme_get_me_mask();
}

unsigned long __startup_secondary_64(void)
{
	/*
	 * Return the SME encryption mask (if SME is active) to be used as a
	 * modifier for the initial pgdir entry programmed into CR3.
	 */
	return sme_get_me_mask();
}

/* Wipe all early page tables except for the kernel symbol map */
static void __init reset_early_page_tables(void)
{
	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
	next_early_pgt = 0;
	write_cr3(__sme_pa_nodebug(early_top_pgt));
}

/* Create a new PMD entry */
int __init __early_make_pgtable(unsigned long address, pmdval_t pmd)
{
	unsigned long physaddr = address - __PAGE_OFFSET;
	pgdval_t pgd, *pgd_p;
	p4dval_t p4d, *p4d_p;
	pudval_t pud, *pud_p;
	pmdval_t *pmd_p;

	/* Invalid address or early pgt is done ?  */
	if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
		return -1;

again:
	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
	pgd = *pgd_p;

	/*
	 * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	 * critical -- __PAGE_OFFSET would point us back into the dynamic
	 * range and we might end up looping forever...
	 */
	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
		p4d_p = pgd_p;
	else if (pgd)
		p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
		*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	p4d_p += p4d_index(address);
	p4d = *p4d_p;

	if (p4d)
		pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
		*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pud_p += pud_index(address);
	pud = *pud_p;

	if (pud)
		pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	else {
		if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
			reset_early_page_tables();
			goto again;
		}

		pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
		memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
		*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	}
	pmd_p[pmd_index(address)] = pmd;

	return 0;
}

int __init early_make_pgtable(unsigned long address)
{
	unsigned long physaddr = address - __PAGE_OFFSET;
	pmdval_t pmd;

	pmd = (physaddr & PMD_MASK) + early_pmd_flags;

	return __early_make_pgtable(address, pmd);
}

/* Don't add a printk in there. printk relies on the PDA which is not initialized 
   yet. */
static void __init clear_bss(void)
{
	memset(__bss_start, 0,
	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
}

static unsigned long get_cmd_line_ptr(void)
{
	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;

	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;

	return cmd_line_ptr;
}

static void __init copy_bootdata(char *real_mode_data)
{
	char * command_line;
	unsigned long cmd_line_ptr;

	/*
	 * If SME is active, this will create decrypted mappings of the
	 * boot data in advance of the copy operations.
	 */
	sme_map_bootdata(real_mode_data);

	memcpy(&boot_params, real_mode_data, sizeof boot_params);
	sanitize_boot_params(&boot_params);
	cmd_line_ptr = get_cmd_line_ptr();
	if (cmd_line_ptr) {
		command_line = __va(cmd_line_ptr);
		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
	}

	/*
	 * The old boot data is no longer needed and won't be reserved,
	 * freeing up that memory for use by the system. If SME is active,
	 * we need to remove the mappings that were created so that the
	 * memory doesn't remain mapped as decrypted.
	 */
	sme_unmap_bootdata(real_mode_data);
}

asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
{
	int i;

	/*
	 * Build-time sanity checks on the kernel image and module
	 * area mappings. (these are purely build-time and produce no code)
	 */
	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
				(__START_KERNEL & PGDIR_MASK)));
	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);

	cr4_init_shadow();

	/* Kill off the identity-map trampoline */
	reset_early_page_tables();

	clear_bss();

	clear_page(init_top_pgt);

	/*
	 * SME support may update early_pmd_flags to include the memory
	 * encryption mask, so it needs to be called before anything
	 * that may generate a page fault.
	 */
	sme_early_init();

	kasan_early_init();

	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
		set_intr_gate(i, early_idt_handler_array[i]);
	load_idt((const struct desc_ptr *)&idt_descr);

	copy_bootdata(__va(real_mode_data));

	/*
	 * Load microcode early on BSP.
	 */
	load_ucode_bsp();

	/* set init_top_pgt kernel high mapping*/
	init_top_pgt[511] = early_top_pgt[511];

	x86_64_start_reservations(real_mode_data);
}

void __init x86_64_start_reservations(char *real_mode_data)
{
	/* version is always not zero if it is copied */
	if (!boot_params.hdr.version)
		copy_bootdata(__va(real_mode_data));

	x86_early_init_platform_quirks();

	switch (boot_params.hdr.hardware_subarch) {
	case X86_SUBARCH_INTEL_MID:
		x86_intel_mid_early_setup();
		break;
	default:
		break;
	}

	start_kernel();
}
Commit	Line	Data
1da177e4	1	/*
835c34a1	2	* prepare to run common code
1da177e4 LT	3	*
1da177e4 LT	4	* Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
1da177e4 LT	5	*/
1da177e4 LT	6
be3606ff	7	#define DISABLE_BRANCH_PROFILING
1da177e4 LT	8	#include <linux/init.h>
	9	#include <linux/linkage.h>
	10	#include <linux/types.h>
	11	#include <linux/kernel.h>
	12	#include <linux/string.h>
	13	#include <linux/percpu.h>
eaf76e8b	14	#include <linux/start_kernel.h>
8b664aa6	15	#include <linux/io.h>
72d7c3b3	16	#include <linux/memblock.h>
5868f365	17	#include <linux/mem_encrypt.h>
1da177e4 LT	18
	19	#include <asm/processor.h>
	20	#include <asm/proto.h>
	21	#include <asm/smp.h>
1da177e4 LT	22	#include <asm/setup.h>
1da177e4 LT	23	#include <asm/desc.h>
f6c2e333	24	#include <asm/pgtable.h>
cfd243d4	25	#include <asm/tlbflush.h>
2bc0414e	26	#include <asm/sections.h>
718fc13b	27	#include <asm/kdebug.h>
66441bd3	28	#include <asm/e820/api.h>
47a3d5da	29	#include <asm/bios_ebda.h>
5dcd14ec	30	#include <asm/bootparam_utils.h>
feddc9de	31	#include <asm/microcode.h>
ef7f0d6a	32	#include <asm/kasan.h>
1da177e4	33
8170e6be PA	34	/*
	35	* Manage page tables very early on.
	36	*/
8170e6be	37	extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
c88d7150	38	static unsigned int __initdata next_early_pgt;
5e427ec2	39	pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL \| _PAGE_NX);
8170e6be	40
26179670 KS	41	#define __head __section(.head.text)
	42
	43	static void __head fixup_pointer(void ptr, unsigned long physaddr)
c88d7150 KS	44	{
	45	return ptr - (void )_text + (void )physaddr;
	46	}
	47
aca20d54 TL	48	unsigned long __head __startup_64(unsigned long physaddr,
aca20d54 TL	49	struct boot_params *bp)
c88d7150 KS	50	{
c88d7150 KS	51	unsigned long load_delta, *p;
5868f365	52	unsigned long pgtable_flags;
c88d7150	53	pgdval_t *pgd;
032370b9	54	p4dval_t *p4d;
c88d7150 KS	55	pudval_t *pud;
	56	pmdval_t *pmd, pmd_entry;
	57	int i;
	58
	59	/* Is the address too large? */
	60	if (physaddr >> MAX_PHYSMEM_BITS)
	61	for (;;);
	62
	63	/*
	64	* Compute the delta between the address I am compiled to run at
	65	* and the address I am actually running at.
	66	*/
	67	load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
	68
	69	/* Is the address not 2M aligned? */
	70	if (load_delta & ~PMD_PAGE_MASK)
	71	for (;;);
	72
5868f365	73	/* Activate Secure Memory Encryption (SME) if supported and enabled */
aca20d54	74	sme_enable(bp);
5868f365 TL	75
	76	/* Include the SME encryption mask in the fixup value */
	77	load_delta += sme_get_me_mask();
	78
c88d7150 KS	79	/* Fixup the physical addresses in the page table */
c88d7150 KS	80
65ade2f8	81	pgd = fixup_pointer(&early_top_pgt, physaddr);
c88d7150 KS	82	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
c88d7150 KS	83
032370b9 KS	84	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	85	p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
	86	p4d[511] += load_delta;
	87	}
	88
c88d7150 KS	89	pud = fixup_pointer(&level3_kernel_pgt, physaddr);
	90	pud[510] += load_delta;
	91	pud[511] += load_delta;
	92
	93	pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
	94	pmd[506] += load_delta;
	95
	96	/*
	97	* Set up the identity mapping for the switchover. These
	98	* entries should NOT have the global bit set! This also
	99	* creates a bunch of nonsense entries but that is fine --
	100	* it avoids problems around wraparound.
	101	*/
	102
	103	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	104	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
21729f81	105	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
c88d7150	106
032370b9 KS	107	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
	108	p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
	109
	110	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
5868f365 TL	111	pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
5868f365 TL	112	pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
032370b9 KS	113
032370b9 KS	114	i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
5868f365 TL	115	p4d[i + 0] = (pgdval_t)pud + pgtable_flags;
5868f365 TL	116	p4d[i + 1] = (pgdval_t)pud + pgtable_flags;
032370b9 KS	117	} else {
032370b9 KS	118	i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
5868f365 TL	119	pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
5868f365 TL	120	pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
032370b9	121	}
c88d7150 KS	122
c88d7150 KS	123	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
5868f365 TL	124	pud[i + 0] = (pudval_t)pmd + pgtable_flags;
5868f365 TL	125	pud[i + 1] = (pudval_t)pmd + pgtable_flags;
c88d7150 KS	126
c88d7150 KS	127	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
5868f365	128	pmd_entry += sme_get_me_mask();
c88d7150 KS	129	pmd_entry += physaddr;
	130
	131	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
	132	int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
	133	pmd[idx] = pmd_entry + i * PMD_SIZE;
	134	}
	135
	136	/*
	137	* Fixup the kernel text+data virtual addresses. Note that
	138	* we might write invalid pmds, when the kernel is relocated
	139	* cleanup_highmap() fixes this up along with the mappings
	140	* beyond _end.
	141	*/
	142
	143	pmd = fixup_pointer(level2_kernel_pgt, physaddr);
	144	for (i = 0; i < PTRS_PER_PMD; i++) {
	145	if (pmd[i] & _PAGE_PRESENT)
	146	pmd[i] += load_delta;
	147	}
	148
5868f365 TL	149	/*
	150	* Fixup phys_base - remove the memory encryption mask to obtain
	151	* the true physical address.
	152	*/
c88d7150	153	p = fixup_pointer(&phys_base, physaddr);
5868f365 TL	154	*p += load_delta - sme_get_me_mask();
	155
	156	/* Encrypt the kernel (if SME is active) */
	157	sme_encrypt_kernel();
	158
	159	/*
	160	* Return the SME encryption mask (if SME is active) to be used as a
	161	* modifier for the initial pgdir entry programmed into CR3.
	162	*/
	163	return sme_get_me_mask();
	164	}
	165
	166	unsigned long __startup_secondary_64(void)
	167	{
	168	/*
	169	* Return the SME encryption mask (if SME is active) to be used as a
	170	* modifier for the initial pgdir entry programmed into CR3.
	171	*/
	172	return sme_get_me_mask();
c88d7150 KS	173	}
c88d7150 KS	174
8170e6be PA	175	/* Wipe all early page tables except for the kernel symbol map */
8170e6be PA	176	static void __init reset_early_page_tables(void)
cfd243d4	177	{
65ade2f8	178	memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
8170e6be	179	next_early_pgt = 0;
21729f81	180	write_cr3(__sme_pa_nodebug(early_top_pgt));
8170e6be PA	181	}
	182
	183	/* Create a new PMD entry */
b9d05200	184	int __init __early_make_pgtable(unsigned long address, pmdval_t pmd)
8170e6be PA	185	{
8170e6be PA	186	unsigned long physaddr = address - __PAGE_OFFSET;
8170e6be	187	pgdval_t pgd, *pgd_p;
032370b9	188	p4dval_t p4d, *p4d_p;
6b9c75ac	189	pudval_t pud, *pud_p;
b9d05200	190	pmdval_t *pmd_p;
8170e6be PA	191
8170e6be PA	192	/* Invalid address or early pgt is done ? */
65ade2f8	193	if (physaddr >= MAXMEM \|\| read_cr3_pa() != __pa_nodebug(early_top_pgt))
8170e6be PA	194	return -1;
8170e6be PA	195
6b9c75ac	196	again:
65ade2f8	197	pgd_p = &early_top_pgt[pgd_index(address)].pgd;
8170e6be PA	198	pgd = *pgd_p;
	199
	200	/*
	201	* The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
	202	* critical -- __PAGE_OFFSET would point us back into the dynamic
	203	* range and we might end up looping forever...
	204	*/
032370b9 KS	205	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
	206	p4d_p = pgd_p;
	207	else if (pgd)
	208	p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	209	else {
	210	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	211	reset_early_page_tables();
	212	goto again;
	213	}
	214
	215	p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
	216	memset(p4d_p, 0, sizeof(p4d_p) PTRS_PER_P4D);
	217	*pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
	218	}
	219	p4d_p += p4d_index(address);
	220	p4d = *p4d_p;
	221
	222	if (p4d)
	223	pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
6b9c75ac YL	224	else {
6b9c75ac YL	225	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
8170e6be	226	reset_early_page_tables();
6b9c75ac YL	227	goto again;
6b9c75ac YL	228	}
8170e6be PA	229
8170e6be PA	230	pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	231	memset(pud_p, 0, sizeof(pud_p) PTRS_PER_PUD);
032370b9	232	*p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
8170e6be	233	}
6b9c75ac YL	234	pud_p += pud_index(address);
6b9c75ac YL	235	pud = *pud_p;
8170e6be	236
6b9c75ac YL	237	if (pud)
	238	pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
	239	else {
	240	if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
	241	reset_early_page_tables();
	242	goto again;
	243	}
	244
	245	pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
a91bbe01	246	memset(pmd_p, 0, sizeof(pmd_p) PTRS_PER_PMD);
6b9c75ac YL	247	*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
6b9c75ac YL	248	}
6b9c75ac	249	pmd_p[pmd_index(address)] = pmd;
8170e6be PA	250
8170e6be PA	251	return 0;
cfd243d4 VG	252	}
cfd243d4 VG	253
b9d05200 TL	254	int __init early_make_pgtable(unsigned long address)
	255	{
	256	unsigned long physaddr = address - __PAGE_OFFSET;
	257	pmdval_t pmd;
	258
	259	pmd = (physaddr & PMD_MASK) + early_pmd_flags;
	260
	261	return __early_make_pgtable(address, pmd);
	262	}
	263
1da177e4 LT	264	/* Don't add a printk in there. printk relies on the PDA which is not initialized
	265	yet. */
	266	static void __init clear_bss(void)
	267	{
1da177e4	268	memset(__bss_start, 0,
2bc0414e	269	(unsigned long) __bss_stop - (unsigned long) __bss_start);
1da177e4 LT	270	}
1da177e4 LT	271
f1da834c YL	272	static unsigned long get_cmd_line_ptr(void)
	273	{
	274	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
	275
ee92d815 YL	276	cmd_line_ptr \|= (u64)boot_params.ext_cmd_line_ptr << 32;
ee92d815 YL	277
f1da834c YL	278	return cmd_line_ptr;
	279	}
	280
1da177e4 LT	281	static void __init copy_bootdata(char *real_mode_data)
1da177e4 LT	282	{
1da177e4	283	char * command_line;
f1da834c	284	unsigned long cmd_line_ptr;
1da177e4	285
b9d05200 TL	286	/*
	287	* If SME is active, this will create decrypted mappings of the
	288	* boot data in advance of the copy operations.
	289	*/
	290	sme_map_bootdata(real_mode_data);
	291
30c82645	292	memcpy(&boot_params, real_mode_data, sizeof boot_params);
5dcd14ec	293	sanitize_boot_params(&boot_params);
f1da834c YL	294	cmd_line_ptr = get_cmd_line_ptr();
	295	if (cmd_line_ptr) {
	296	command_line = __va(cmd_line_ptr);
30c82645	297	memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
1da177e4	298	}
b9d05200 TL	299
	300	/*
	301	* The old boot data is no longer needed and won't be reserved,
	302	* freeing up that memory for use by the system. If SME is active,
	303	* we need to remove the mappings that were created so that the
	304	* memory doesn't remain mapped as decrypted.
	305	*/
	306	sme_unmap_bootdata(real_mode_data);
1da177e4 LT	307	}
1da177e4 LT	308
2605fc21	309	asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
1da177e4	310	{
1da177e4 LT	311	int i;
1da177e4 LT	312
b4e0409a IM	313	/*
	314	* Build-time sanity checks on the kernel image and module
	315	* area mappings. (these are purely build-time and produce no code)
	316	*/
8e3c2a8c BP	317	BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
8e3c2a8c BP	318	BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
b4e0409a	319	BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
8e3c2a8c	320	BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
b4e0409a IM	321	BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
	322	BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
	323	BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
	324	(__START_KERNEL & PGDIR_MASK)));
66d4bdf2	325	BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
b4e0409a	326
1e02ce4c AL	327	cr4_init_shadow();
1e02ce4c AL	328
8170e6be PA	329	/* Kill off the identity-map trampoline */
	330	reset_early_page_tables();
	331
3df0af0e YL	332	clear_bss();
3df0af0e YL	333
65ade2f8	334	clear_page(init_top_pgt);
d0f77d4d	335
21729f81 TL	336	/*
	337	* SME support may update early_pmd_flags to include the memory
	338	* encryption mask, so it needs to be called before anything
	339	* that may generate a page fault.
	340	*/
	341	sme_early_init();
	342
5d5aa3cf AP	343	kasan_early_init();
5d5aa3cf AP	344
ac630dd9	345	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
425be567	346	set_intr_gate(i, early_idt_handler_array[i]);
9d1c6e7c	347	load_idt((const struct desc_ptr *)&idt_descr);
f6c2e333	348
fa2bbce9 YL	349	copy_bootdata(__va(real_mode_data));
fa2bbce9 YL	350
feddc9de FY	351	/*
	352	* Load microcode early on BSP.
	353	*/
	354	load_ucode_bsp();
	355
65ade2f8 KS	356	/* set init_top_pgt kernel high mapping*/
65ade2f8 KS	357	init_top_pgt[511] = early_top_pgt[511];
8170e6be	358
f97013fd JF	359	x86_64_start_reservations(real_mode_data);
	360	}
	361
	362	void __init x86_64_start_reservations(char *real_mode_data)
	363	{
fa2bbce9 YL	364	/* version is always not zero if it is copied */
	365	if (!boot_params.hdr.version)
	366	copy_bootdata(__va(real_mode_data));
9de819fe	367
8d152e7a	368	x86_early_init_platform_quirks();
75175278	369
3fda5bb4 AS	370	switch (boot_params.hdr.hardware_subarch) {
	371	case X86_SUBARCH_INTEL_MID:
	372	x86_intel_mid_early_setup();
	373	break;
	374	default:
	375	break;
	376	}
	377
1da177e4 LT	378	start_kernel();
1da177e4 LT	379	}