[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / relocate_kernel_32.S

/*
 * relocate_kernel.S - put the kernel image in place to boot
 * Copyright (C) 2002-2004 Eric Biederman  <ebiederm@xmission.com>
 *
 * This source code is licensed under the GNU General Public License,
 * Version 2.  See the file COPYING for more details.
 */

#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/kexec.h>
#include <asm/processor-flags.h>
#include <asm/nospec-branch.h>

/*
 * Must be relocatable PIC code callable as a C function
 */

#define PTR(x) (x << 2)

/*
 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
 * ~ control_page + PAGE_SIZE are used as data storage and stack for
 * jumping back
 */
#define DATA(offset)		(KEXEC_CONTROL_CODE_MAX_SIZE+(offset))

/* Minimal CPU state */
#define ESP			DATA(0x0)
#define CR0			DATA(0x4)
#define CR3			DATA(0x8)
#define CR4			DATA(0xc)

/* other data */
#define CP_VA_CONTROL_PAGE	DATA(0x10)
#define CP_PA_PGD		DATA(0x14)
#define CP_PA_SWAP_PAGE		DATA(0x18)
#define CP_PA_BACKUP_PAGES_MAP	DATA(0x1c)

	.text
	.globl relocate_kernel
relocate_kernel:
	/* Save the CPU context, used for jumping back */

	pushl	%ebx
	pushl	%esi
	pushl	%edi
	pushl	%ebp
	pushf

	movl	20+8(%esp), %ebp /* list of pages */
	movl	PTR(VA_CONTROL_PAGE)(%ebp), %edi
	movl	%esp, ESP(%edi)
	movl	%cr0, %eax
	movl	%eax, CR0(%edi)
	movl	%cr3, %eax
	movl	%eax, CR3(%edi)
	movl	%cr4, %eax
	movl	%eax, CR4(%edi)

	/* read the arguments and say goodbye to the stack */
	movl  20+4(%esp), %ebx /* page_list */
	movl  20+8(%esp), %ebp /* list of pages */
	movl  20+12(%esp), %edx /* start address */
	movl  20+16(%esp), %ecx /* cpu_has_pae */
	movl  20+20(%esp), %esi /* preserve_context */

	/* zero out flags, and disable interrupts */
	pushl $0
	popfl

	/* save some information for jumping back */
	movl	PTR(VA_CONTROL_PAGE)(%ebp), %edi
	movl	%edi, CP_VA_CONTROL_PAGE(%edi)
	movl	PTR(PA_PGD)(%ebp), %eax
	movl	%eax, CP_PA_PGD(%edi)
	movl	PTR(PA_SWAP_PAGE)(%ebp), %eax
	movl	%eax, CP_PA_SWAP_PAGE(%edi)
	movl	%ebx, CP_PA_BACKUP_PAGES_MAP(%edi)

	/*
	 * get physical address of control page now
	 * this is impossible after page table switch
	 */
	movl	PTR(PA_CONTROL_PAGE)(%ebp), %edi

	/* switch to new set of page tables */
	movl	PTR(PA_PGD)(%ebp), %eax
	movl	%eax, %cr3

	/* setup a new stack at the end of the physical control page */
	lea	PAGE_SIZE(%edi), %esp

	/* jump to identity mapped page */
	movl    %edi, %eax
	addl    $(identity_mapped - relocate_kernel), %eax
	pushl   %eax
	ret

identity_mapped:
	/* set return address to 0 if not preserving context */
	pushl	$0
	/* store the start address on the stack */
	pushl   %edx

	/*
	 * Set cr0 to a known state:
	 *  - Paging disabled
	 *  - Alignment check disabled
	 *  - Write protect disabled
	 *  - No task switch
	 *  - Don't do FP software emulation.
	 *  - Proctected mode enabled
	 */
	movl	%cr0, %eax
	andl	$~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax
	orl	$(X86_CR0_PE), %eax
	movl	%eax, %cr0

	/* clear cr4 if applicable */
	testl	%ecx, %ecx
	jz	1f
	/*
	 * Set cr4 to a known state:
	 * Setting everything to zero seems safe.
	 */
	xorl	%eax, %eax
	movl	%eax, %cr4

	jmp 1f
1:

	/* Flush the TLB (needed?) */
	xorl	%eax, %eax
	movl	%eax, %cr3

	movl	CP_PA_SWAP_PAGE(%edi), %eax
	pushl	%eax
	pushl	%ebx
	call	swap_pages
	addl	$8, %esp

	/*
	 * To be certain of avoiding problems with self-modifying code
	 * I need to execute a serializing instruction here.
	 * So I flush the TLB, it's handy, and not processor dependent.
	 */
	xorl	%eax, %eax
	movl	%eax, %cr3

	/*
	 * set all of the registers to known values
	 * leave %esp alone
	 */

	testl	%esi, %esi
	jnz 1f
	xorl	%edi, %edi
	xorl	%eax, %eax
	xorl	%ebx, %ebx
	xorl    %ecx, %ecx
	xorl    %edx, %edx
	xorl    %esi, %esi
	xorl    %ebp, %ebp
	ret
1:
	popl	%edx
	movl	CP_PA_SWAP_PAGE(%edi), %esp
	addl	$PAGE_SIZE, %esp
2:
	ANNOTATE_RETPOLINE_SAFE
	call	*%edx

	/* get the re-entry point of the peer system */
	movl	0(%esp), %ebp
	call	1f
1:
	popl	%ebx
	subl	$(1b - relocate_kernel), %ebx
	movl	CP_VA_CONTROL_PAGE(%ebx), %edi
	lea	PAGE_SIZE(%ebx), %esp
	movl	CP_PA_SWAP_PAGE(%ebx), %eax
	movl	CP_PA_BACKUP_PAGES_MAP(%ebx), %edx
	pushl	%eax
	pushl	%edx
	call	swap_pages
	addl	$8, %esp
	movl	CP_PA_PGD(%ebx), %eax
	movl	%eax, %cr3
	movl	%cr0, %eax
	orl	$X86_CR0_PG, %eax
	movl	%eax, %cr0
	lea	PAGE_SIZE(%edi), %esp
	movl	%edi, %eax
	addl	$(virtual_mapped - relocate_kernel), %eax
	pushl	%eax
	ret

virtual_mapped:
	movl	CR4(%edi), %eax
	movl	%eax, %cr4
	movl	CR3(%edi), %eax
	movl	%eax, %cr3
	movl	CR0(%edi), %eax
	movl	%eax, %cr0
	movl	ESP(%edi), %esp
	movl	%ebp, %eax

	popf
	popl	%ebp
	popl	%edi
	popl	%esi
	popl	%ebx
	ret

	/* Do the copies */
swap_pages:
	movl	8(%esp), %edx
	movl	4(%esp), %ecx
	pushl	%ebp
	pushl	%ebx
	pushl	%edi
	pushl	%esi
	movl	%ecx, %ebx
	jmp	1f

0:	/* top, read another word from the indirection page */
	movl	(%ebx), %ecx
	addl	$4, %ebx
1:
	testb	$0x1, %cl     /* is it a destination page */
	jz	2f
	movl	%ecx,	%edi
	andl	$0xfffff000, %edi
	jmp     0b
2:
	testb	$0x2, %cl    /* is it an indirection page */
	jz	2f
	movl	%ecx,	%ebx
	andl	$0xfffff000, %ebx
	jmp     0b
2:
	testb   $0x4, %cl    /* is it the done indicator */
	jz      2f
	jmp     3f
2:
	testb   $0x8, %cl    /* is it the source indicator */
	jz      0b	     /* Ignore it otherwise */
	movl    %ecx,   %esi /* For every source page do a copy */
	andl    $0xfffff000, %esi

	movl	%edi, %eax
	movl	%esi, %ebp

	movl	%edx, %edi
	movl    $1024, %ecx
	rep ; movsl

	movl	%ebp, %edi
	movl	%eax, %esi
	movl	$1024, %ecx
	rep ; movsl

	movl	%eax, %edi
	movl	%edx, %esi
	movl	$1024, %ecx
	rep ; movsl

	lea	PAGE_SIZE(%ebp), %esi
	jmp     0b
3:
	popl	%esi
	popl	%edi
	popl	%ebx
	popl	%ebp
	ret

	.globl kexec_control_code_size
.set kexec_control_code_size, . - relocate_kernel
Commit	Line	Data
5033cba0 EB	1	/*
	2	* relocate_kernel.S - put the kernel image in place to boot
	3	* Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com>
	4	*
	5	* This source code is licensed under the GNU General Public License,
	6	* Version 2. See the file COPYING for more details.
	7	*/
	8
	9	#include <linux/linkage.h>
0341c14d	10	#include <asm/page_types.h>
3566561b	11	#include <asm/kexec.h>
fd3af531	12	#include <asm/processor-flags.h>
b3ef7b48	13	#include <asm/nospec-branch.h>
3566561b MD	14
	15	/*
	16	* Must be relocatable PIC code callable as a C function
	17	*/
	18
	19	#define PTR(x) (x << 2)
3566561b	20
fef3a7a1 HY	21	/*
fef3a7a1 HY	22	* control_page + KEXEC_CONTROL_CODE_MAX_SIZE
fb45daa6 HY	23	* ~ control_page + PAGE_SIZE are used as data storage and stack for
fb45daa6 HY	24	* jumping back
3ab83521	25	*/
fb45daa6	26	#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
3ab83521 HY	27
	28	/* Minimal CPU state */
	29	#define ESP DATA(0x0)
	30	#define CR0 DATA(0x4)
	31	#define CR3 DATA(0x8)
	32	#define CR4 DATA(0xc)
	33
	34	/* other data */
	35	#define CP_VA_CONTROL_PAGE DATA(0x10)
	36	#define CP_PA_PGD DATA(0x14)
	37	#define CP_PA_SWAP_PAGE DATA(0x18)
	38	#define CP_PA_BACKUP_PAGES_MAP DATA(0x1c)
	39
3566561b	40	.text
3566561b MD	41	.globl relocate_kernel
3566561b MD	42	relocate_kernel:
3ab83521 HY	43	/* Save the CPU context, used for jumping back */
	44
	45	pushl %ebx
	46	pushl %esi
	47	pushl %edi
	48	pushl %ebp
	49	pushf
	50
	51	movl 20+8(%esp), %ebp /* list of pages */
	52	movl PTR(VA_CONTROL_PAGE)(%ebp), %edi
	53	movl %esp, ESP(%edi)
	54	movl %cr0, %eax
	55	movl %eax, CR0(%edi)
	56	movl %cr3, %eax
	57	movl %eax, CR3(%edi)
	58	movl %cr4, %eax
	59	movl %eax, CR4(%edi)
3566561b	60
5033cba0	61	/* read the arguments and say goodbye to the stack */
3ab83521 HY	62	movl 20+4(%esp), %ebx /* page_list */
	63	movl 20+8(%esp), %ebp /* list of pages */
	64	movl 20+12(%esp), %edx /* start address */
	65	movl 20+16(%esp), %ecx /* cpu_has_pae */
	66	movl 20+20(%esp), %esi /* preserve_context */
5033cba0 EB	67
	68	/* zero out flags, and disable interrupts */
	69	pushl $0
	70	popfl
	71
3ab83521 HY	72	/* save some information for jumping back */
	73	movl PTR(VA_CONTROL_PAGE)(%ebp), %edi
	74	movl %edi, CP_VA_CONTROL_PAGE(%edi)
	75	movl PTR(PA_PGD)(%ebp), %eax
	76	movl %eax, CP_PA_PGD(%edi)
	77	movl PTR(PA_SWAP_PAGE)(%ebp), %eax
	78	movl %eax, CP_PA_SWAP_PAGE(%edi)
	79	movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi)
	80
fef3a7a1 HY	81	/*
	82	* get physical address of control page now
	83	* this is impossible after page table switch
	84	*/
3566561b	85	movl PTR(PA_CONTROL_PAGE)(%ebp), %edi
5033cba0	86
3566561b MD	87	/* switch to new set of page tables */
	88	movl PTR(PA_PGD)(%ebp), %eax
	89	movl %eax, %cr3
	90
	91	/* setup a new stack at the end of the physical control page */
a7bba17b	92	lea PAGE_SIZE(%edi), %esp
3566561b MD	93
	94	/* jump to identity mapped page */
	95	movl %edi, %eax
	96	addl $(identity_mapped - relocate_kernel), %eax
	97	pushl %eax
	98	ret
	99
	100	identity_mapped:
050438ed HY	101	/* set return address to 0 if not preserving context */
050438ed HY	102	pushl $0
3566561b MD	103	/* store the start address on the stack */
3566561b MD	104	pushl %edx
5033cba0	105
fef3a7a1 HY	106	/*
fef3a7a1 HY	107	* Set cr0 to a known state:
fd3af531	108	* - Paging disabled
	109	* - Alignment check disabled
	110	* - Write protect disabled
	111	* - No task switch
	112	* - Don't do FP software emulation.
	113	* - Proctected mode enabled
5033cba0 EB	114	*/
5033cba0 EB	115	movl %cr0, %eax
fd3af531	116	andl $~(X86_CR0_PG \| X86_CR0_AM \| X86_CR0_WP \| X86_CR0_TS \| X86_CR0_EM), %eax
fd3af531	117	orl $(X86_CR0_PE), %eax
5033cba0 EB	118	movl %eax, %cr0
	119
	120	/* clear cr4 if applicable */
	121	testl %ecx, %ecx
	122	jz 1f
fef3a7a1 HY	123	/*
fef3a7a1 HY	124	* Set cr4 to a known state:
5033cba0 EB	125	* Setting everything to zero seems safe.
5033cba0 EB	126	*/
4039ae53	127	xorl %eax, %eax
5033cba0 EB	128	movl %eax, %cr4
	129
	130	jmp 1f
	131	1:
	132
	133	/* Flush the TLB (needed?) */
	134	xorl %eax, %eax
	135	movl %eax, %cr3
	136
3ab83521 HY	137	movl CP_PA_SWAP_PAGE(%edi), %eax
	138	pushl %eax
	139	pushl %ebx
	140	call swap_pages
	141	addl $8, %esp
	142
fef3a7a1 HY	143	/*
fef3a7a1 HY	144	* To be certain of avoiding problems with self-modifying code
3ab83521 HY	145	* I need to execute a serializing instruction here.
	146	* So I flush the TLB, it's handy, and not processor dependent.
	147	*/
	148	xorl %eax, %eax
	149	movl %eax, %cr3
	150
fef3a7a1 HY	151	/*
	152	* set all of the registers to known values
	153	* leave %esp alone
	154	*/
3ab83521 HY	155
	156	testl %esi, %esi
	157	jnz 1f
	158	xorl %edi, %edi
	159	xorl %eax, %eax
	160	xorl %ebx, %ebx
	161	xorl %ecx, %ecx
	162	xorl %edx, %edx
	163	xorl %esi, %esi
	164	xorl %ebp, %ebp
	165	ret
	166	1:
	167	popl %edx
	168	movl CP_PA_SWAP_PAGE(%edi), %esp
	169	addl $PAGE_SIZE, %esp
	170	2:
b3ef7b48	171	ANNOTATE_RETPOLINE_SAFE
3ab83521 HY	172	call *%edx
	173
	174	/* get the re-entry point of the peer system */
	175	movl 0(%esp), %ebp
	176	call 1f
	177	1:
	178	popl %ebx
	179	subl $(1b - relocate_kernel), %ebx
	180	movl CP_VA_CONTROL_PAGE(%ebx), %edi
	181	lea PAGE_SIZE(%ebx), %esp
	182	movl CP_PA_SWAP_PAGE(%ebx), %eax
	183	movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx
	184	pushl %eax
	185	pushl %edx
	186	call swap_pages
	187	addl $8, %esp
	188	movl CP_PA_PGD(%ebx), %eax
	189	movl %eax, %cr3
	190	movl %cr0, %eax
a3d7b7dd	191	orl $X86_CR0_PG, %eax
3ab83521 HY	192	movl %eax, %cr0
	193	lea PAGE_SIZE(%edi), %esp
	194	movl %edi, %eax
	195	addl $(virtual_mapped - relocate_kernel), %eax
	196	pushl %eax
	197	ret
	198
	199	virtual_mapped:
	200	movl CR4(%edi), %eax
	201	movl %eax, %cr4
	202	movl CR3(%edi), %eax
	203	movl %eax, %cr3
	204	movl CR0(%edi), %eax
	205	movl %eax, %cr0
	206	movl ESP(%edi), %esp
	207	movl %ebp, %eax
	208
	209	popf
	210	popl %ebp
	211	popl %edi
	212	popl %esi
	213	popl %ebx
	214	ret
	215
5033cba0	216	/* Do the copies */
3ab83521 HY	217	swap_pages:
	218	movl 8(%esp), %edx
	219	movl 4(%esp), %ecx
	220	pushl %ebp
	221	pushl %ebx
	222	pushl %edi
	223	pushl %esi
	224	movl %ecx, %ebx
5033cba0 EB	225	jmp 1f
	226
	227	0: /* top, read another word from the indirection page */
	228	movl (%ebx), %ecx
	229	addl $4, %ebx
	230	1:
3e1aa7cb	231	testb $0x1, %cl /* is it a destination page */
5033cba0 EB	232	jz 2f
	233	movl %ecx, %edi
	234	andl $0xfffff000, %edi
	235	jmp 0b
	236	2:
3e1aa7cb	237	testb $0x2, %cl /* is it an indirection page */
5033cba0 EB	238	jz 2f
	239	movl %ecx, %ebx
	240	andl $0xfffff000, %ebx
	241	jmp 0b
	242	2:
3e1aa7cb	243	testb $0x4, %cl /* is it the done indicator */
5033cba0 EB	244	jz 2f
	245	jmp 3f
	246	2:
3e1aa7cb	247	testb $0x8, %cl /* is it the source indicator */
5033cba0 EB	248	jz 0b /* Ignore it otherwise */
	249	movl %ecx, %esi /* For every source page do a copy */
	250	andl $0xfffff000, %esi
	251
3ab83521 HY	252	movl %edi, %eax
	253	movl %esi, %ebp
	254
	255	movl %edx, %edi
5033cba0 EB	256	movl $1024, %ecx
5033cba0 EB	257	rep ; movsl
5033cba0	258
3ab83521 HY	259	movl %ebp, %edi
	260	movl %eax, %esi
	261	movl $1024, %ecx
	262	rep ; movsl
5033cba0	263
3ab83521 HY	264	movl %eax, %edi
	265	movl %edx, %esi
	266	movl $1024, %ecx
	267	rep ; movsl
5033cba0	268
3ab83521 HY	269	lea PAGE_SIZE(%ebp), %esi
	270	jmp 0b
	271	3:
	272	popl %esi
	273	popl %edi
	274	popl %ebx
	275	popl %ebp
5033cba0	276	ret
fb45daa6 HY	277
	278	.globl kexec_control_code_size
	279	.set kexec_control_code_size, . - relocate_kernel