[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / relocate_kernel_64.S

/*
 * relocate_kernel.S - put the kernel image in place to boot
 * Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
 *
 * This source code is licensed under the GNU General Public License,
 * Version 2.  See the file COPYING for more details.
 */

#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/kexec.h>
#include <asm/processor-flags.h>
#include <asm/pgtable_types.h>

/*
 * Must be relocatable PIC code callable as a C function
 */

#define PTR(x) (x << 3)
#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)

/*
 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
 * ~ control_page + PAGE_SIZE are used as data storage and stack for
 * jumping back
 */
#define DATA(offset)		(KEXEC_CONTROL_CODE_MAX_SIZE+(offset))

/* Minimal CPU state */
#define RSP			DATA(0x0)
#define CR0			DATA(0x8)
#define CR3			DATA(0x10)
#define CR4			DATA(0x18)

/* other data */
#define CP_PA_TABLE_PAGE	DATA(0x20)
#define CP_PA_SWAP_PAGE		DATA(0x28)
#define CP_PA_BACKUP_PAGES_MAP	DATA(0x30)

	.text
	.align PAGE_SIZE
	.code64
	.globl relocate_kernel
relocate_kernel:
	/*
	 * %rdi indirection_page
	 * %rsi page_list
	 * %rdx start address
	 * %rcx preserve_context
	 */

	/* Save the CPU context, used for jumping back */
	pushq %rbx
	pushq %rbp
	pushq %r12
	pushq %r13
	pushq %r14
	pushq %r15
	pushf

	movq	PTR(VA_CONTROL_PAGE)(%rsi), %r11
	movq	%rsp, RSP(%r11)
	movq	%cr0, %rax
	movq	%rax, CR0(%r11)
	movq	%cr3, %rax
	movq	%rax, CR3(%r11)
	movq	%cr4, %rax
	movq	%rax, CR4(%r11)

	/* zero out flags, and disable interrupts */
	pushq $0
	popfq

	/*
	 * get physical address of control page now
	 * this is impossible after page table switch
	 */
	movq	PTR(PA_CONTROL_PAGE)(%rsi), %r8

	/* get physical address of page table now too */
	movq	PTR(PA_TABLE_PAGE)(%rsi), %r9

	/* get physical address of swap page now */
	movq	PTR(PA_SWAP_PAGE)(%rsi), %r10

	/* save some information for jumping back */
	movq	%r9, CP_PA_TABLE_PAGE(%r11)
	movq	%r10, CP_PA_SWAP_PAGE(%r11)
	movq	%rdi, CP_PA_BACKUP_PAGES_MAP(%r11)

	/* Switch to the identity mapped page tables */
	movq	%r9, %cr3

	/* setup a new stack at the end of the physical control page */
	lea	PAGE_SIZE(%r8), %rsp

	/* jump to identity mapped page */
	addq	$(identity_mapped - relocate_kernel), %r8
	pushq	%r8
	ret

identity_mapped:
	/* store the start address on the stack */
	pushq   %rdx

	/*
	 * Set cr0 to a known state:
	 *  - Paging enabled
	 *  - Alignment check disabled
	 *  - Write protect disabled
	 *  - No task switch
	 *  - Don't do FP software emulation.
	 *  - Proctected mode enabled
	 */
	movq	%cr0, %rax
	andq	$~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax
	orl	$(X86_CR0_PG | X86_CR0_PE), %eax
	movq	%rax, %cr0

	/*
	 * Set cr4 to a known state:
	 *  - physical address extension enabled
	 */
	movq	$X86_CR4_PAE, %rax
	movq	%rax, %cr4

	jmp 1f
1:

	/* Flush the TLB (needed?) */
	movq	%r9, %cr3

	movq	%rcx, %r11
	call	swap_pages

	/*
	 * To be certain of avoiding problems with self-modifying code
	 * I need to execute a serializing instruction here.
	 * So I flush the TLB by reloading %cr3 here, it's handy,
	 * and not processor dependent.
	 */
	movq	%cr3, %rax
	movq	%rax, %cr3

	/*
	 * set all of the registers to known values
	 * leave %rsp alone
	 */

	testq	%r11, %r11
	jnz 1f
	xorq	%rax, %rax
	xorq	%rbx, %rbx
	xorq    %rcx, %rcx
	xorq    %rdx, %rdx
	xorq    %rsi, %rsi
	xorq    %rdi, %rdi
	xorq    %rbp, %rbp
	xorq	%r8,  %r8
	xorq	%r9,  %r9
	xorq	%r10, %r9
	xorq	%r11, %r11
	xorq	%r12, %r12
	xorq	%r13, %r13
	xorq	%r14, %r14
	xorq	%r15, %r15

	ret

1:
	popq	%rdx
	leaq	PAGE_SIZE(%r10), %rsp
	call	*%rdx

	/* get the re-entry point of the peer system */
	movq	0(%rsp), %rbp
	call	1f
1:
	popq	%r8
	subq	$(1b - relocate_kernel), %r8
	movq	CP_PA_SWAP_PAGE(%r8), %r10
	movq	CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
	movq	CP_PA_TABLE_PAGE(%r8), %rax
	movq	%rax, %cr3
	lea	PAGE_SIZE(%r8), %rsp
	call	swap_pages
	movq	$virtual_mapped, %rax
	pushq	%rax
	ret

virtual_mapped:
	movq	RSP(%r8), %rsp
	movq	CR4(%r8), %rax
	movq	%rax, %cr4
	movq	CR3(%r8), %rax
	movq	CR0(%r8), %r8
	movq	%rax, %cr3
	movq	%r8, %cr0
	movq	%rbp, %rax

	popf
	popq	%r15
	popq	%r14
	popq	%r13
	popq	%r12
	popq	%rbp
	popq	%rbx
	ret

	/* Do the copies */
swap_pages:
	movq	%rdi, %rcx 	/* Put the page_list in %rcx */
	xorq	%rdi, %rdi
	xorq	%rsi, %rsi
	jmp	1f

0:	/* top, read another word for the indirection page */

	movq	(%rbx), %rcx
	addq	$8,	%rbx
1:
	testq	$0x1,	%rcx  /* is it a destination page? */
	jz	2f
	movq	%rcx,	%rdi
	andq	$0xfffffffffffff000, %rdi
	jmp	0b
2:
	testq	$0x2,	%rcx  /* is it an indirection page? */
	jz	2f
	movq	%rcx,   %rbx
	andq	$0xfffffffffffff000, %rbx
	jmp	0b
2:
	testq	$0x4,	%rcx  /* is it the done indicator? */
	jz	2f
	jmp	3f
2:
	testq	$0x8,	%rcx  /* is it the source indicator? */
	jz	0b	      /* Ignore it otherwise */
	movq	%rcx,   %rsi  /* For ever source page do a copy */
	andq	$0xfffffffffffff000, %rsi

	movq	%rdi, %rdx
	movq	%rsi, %rax

	movq	%r10, %rdi
	movq	$512,   %rcx
	rep ; movsq

	movq	%rax, %rdi
	movq	%rdx, %rsi
	movq	$512,   %rcx
	rep ; movsq

	movq	%rdx, %rdi
	movq	%r10, %rsi
	movq	$512,   %rcx
	rep ; movsq

	lea	PAGE_SIZE(%rax), %rsi
	jmp	0b
3:
	ret

	.globl kexec_control_code_size
.set kexec_control_code_size, . - relocate_kernel
Commit	Line	Data
5234f5eb EB	1	/*
	2	* relocate_kernel.S - put the kernel image in place to boot
	3	* Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
	4	*
	5	* This source code is licensed under the GNU General Public License,
	6	* Version 2. See the file COPYING for more details.
	7	*/
	8
	9	#include <linux/linkage.h>
0341c14d	10	#include <asm/page_types.h>
4bfaaef0	11	#include <asm/kexec.h>
fd3af531	12	#include <asm/processor-flags.h>
0341c14d	13	#include <asm/pgtable_types.h>
5234f5eb	14
4bfaaef0 MD	15	/*
	16	* Must be relocatable PIC code callable as a C function
	17	*/
	18
	19	#define PTR(x) (x << 3)
366932de	20	#define PAGE_ATTR (_PAGE_PRESENT \| _PAGE_RW \| _PAGE_ACCESSED \| _PAGE_DIRTY)
4bfaaef0	21
fee7b0d8 HY	22	/*
	23	* control_page + KEXEC_CONTROL_CODE_MAX_SIZE
	24	* ~ control_page + PAGE_SIZE are used as data storage and stack for
	25	* jumping back
	26	*/
	27	#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
	28
	29	/* Minimal CPU state */
	30	#define RSP DATA(0x0)
	31	#define CR0 DATA(0x8)
	32	#define CR3 DATA(0x10)
	33	#define CR4 DATA(0x18)
	34
	35	/* other data */
	36	#define CP_PA_TABLE_PAGE DATA(0x20)
	37	#define CP_PA_SWAP_PAGE DATA(0x28)
	38	#define CP_PA_BACKUP_PAGES_MAP DATA(0x30)
	39
4bfaaef0	40	.text
288621e3	41	.align PAGE_SIZE
5234f5eb	42	.code64
4bfaaef0 MD	43	.globl relocate_kernel
4bfaaef0 MD	44	relocate_kernel:
fef3a7a1 HY	45	/*
fef3a7a1 HY	46	* %rdi indirection_page
4bfaaef0 MD	47	* %rsi page_list
4bfaaef0 MD	48	* %rdx start address
fee7b0d8	49	* %rcx preserve_context
4bfaaef0 MD	50	*/
4bfaaef0 MD	51
fee7b0d8 HY	52	/* Save the CPU context, used for jumping back */
	53	pushq %rbx
	54	pushq %rbp
	55	pushq %r12
	56	pushq %r13
	57	pushq %r14
	58	pushq %r15
	59	pushf
	60
	61	movq PTR(VA_CONTROL_PAGE)(%rsi), %r11
	62	movq %rsp, RSP(%r11)
	63	movq %cr0, %rax
	64	movq %rax, CR0(%r11)
	65	movq %cr3, %rax
	66	movq %rax, CR3(%r11)
	67	movq %cr4, %rax
	68	movq %rax, CR4(%r11)
	69
5234f5eb EB	70	/* zero out flags, and disable interrupts */
	71	pushq $0
	72	popfq
	73
fef3a7a1 HY	74	/*
	75	* get physical address of control page now
	76	* this is impossible after page table switch
	77	*/
4bfaaef0 MD	78	movq PTR(PA_CONTROL_PAGE)(%rsi), %r8
	79
	80	/* get physical address of page table now too */
fee7b0d8 HY	81	movq PTR(PA_TABLE_PAGE)(%rsi), %r9
	82
	83	/* get physical address of swap page now */
	84	movq PTR(PA_SWAP_PAGE)(%rsi), %r10
	85
	86	/* save some information for jumping back */
	87	movq %r9, CP_PA_TABLE_PAGE(%r11)
	88	movq %r10, CP_PA_SWAP_PAGE(%r11)
	89	movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11)
5234f5eb	90
f5deb796	91	/* Switch to the identity mapped page tables */
fee7b0d8	92	movq %r9, %cr3
4bfaaef0 MD	93
4bfaaef0 MD	94	/* setup a new stack at the end of the physical control page */
a7bba17b	95	lea PAGE_SIZE(%r8), %rsp
4bfaaef0 MD	96
	97	/* jump to identity mapped page */
	98	addq $(identity_mapped - relocate_kernel), %r8
	99	pushq %r8
	100	ret
	101
	102	identity_mapped:
	103	/* store the start address on the stack */
	104	pushq %rdx
5234f5eb	105
fef3a7a1 HY	106	/*
fef3a7a1 HY	107	* Set cr0 to a known state:
fd3af531	108	* - Paging enabled
	109	* - Alignment check disabled
	110	* - Write protect disabled
	111	* - No task switch
	112	* - Don't do FP software emulation.
	113	* - Proctected mode enabled
5234f5eb EB	114	*/
5234f5eb EB	115	movq %cr0, %rax
fd3af531	116	andq $~(X86_CR0_AM \| X86_CR0_WP \| X86_CR0_TS \| X86_CR0_EM), %rax
fd3af531	117	orl $(X86_CR0_PG \| X86_CR0_PE), %eax
5234f5eb EB	118	movq %rax, %cr0
5234f5eb EB	119
fef3a7a1 HY	120	/*
fef3a7a1 HY	121	* Set cr4 to a known state:
fd3af531	122	* - physical address extension enabled
5234f5eb	123	*/
fd3af531	124	movq $X86_CR4_PAE, %rax
5234f5eb EB	125	movq %rax, %cr4
	126
	127	jmp 1f
	128	1:
	129
f5deb796	130	/* Flush the TLB (needed?) */
fee7b0d8 HY	131	movq %r9, %cr3
	132
	133	movq %rcx, %r11
	134	call swap_pages
	135
	136	/*
	137	* To be certain of avoiding problems with self-modifying code
	138	* I need to execute a serializing instruction here.
	139	* So I flush the TLB by reloading %cr3 here, it's handy,
	140	* and not processor dependent.
	141	*/
	142	movq %cr3, %rax
	143	movq %rax, %cr3
	144
	145	/*
	146	* set all of the registers to known values
	147	* leave %rsp alone
	148	*/
	149
	150	testq %r11, %r11
	151	jnz 1f
	152	xorq %rax, %rax
	153	xorq %rbx, %rbx
	154	xorq %rcx, %rcx
	155	xorq %rdx, %rdx
	156	xorq %rsi, %rsi
	157	xorq %rdi, %rdi
	158	xorq %rbp, %rbp
	159	xorq %r8, %r8
	160	xorq %r9, %r9
	161	xorq %r10, %r9
	162	xorq %r11, %r11
	163	xorq %r12, %r12
	164	xorq %r13, %r13
	165	xorq %r14, %r14
	166	xorq %r15, %r15
	167
	168	ret
	169
	170	1:
	171	popq %rdx
	172	leaq PAGE_SIZE(%r10), %rsp
	173	call *%rdx
	174
	175	/* get the re-entry point of the peer system */
	176	movq 0(%rsp), %rbp
	177	call 1f
	178	1:
	179	popq %r8
	180	subq $(1b - relocate_kernel), %r8
	181	movq CP_PA_SWAP_PAGE(%r8), %r10
	182	movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
	183	movq CP_PA_TABLE_PAGE(%r8), %rax
	184	movq %rax, %cr3
	185	lea PAGE_SIZE(%r8), %rsp
	186	call swap_pages
	187	movq $virtual_mapped, %rax
	188	pushq %rax
	189	ret
	190
	191	virtual_mapped:
	192	movq RSP(%r8), %rsp
	193	movq CR4(%r8), %rax
	194	movq %rax, %cr4
195	movq CR3(%r8), %rax
196	movq CR0(%r8), %r8
197	movq %rax, %cr3
198	movq %r8, %cr0
199	movq %rbp, %rax
200
201	popf
202	popq %r15
203	popq %r14
204	popq %r13
205	popq %r12
206	popq %rbp
207	popq %rbx
208	ret
5234f5eb EB	209
5234f5eb EB	210	/* Do the copies */
fee7b0d8	211	swap_pages:
5234f5eb EB	212	movq %rdi, %rcx /* Put the page_list in %rcx */
	213	xorq %rdi, %rdi
	214	xorq %rsi, %rsi
	215	jmp 1f
	216
	217	0: /* top, read another word for the indirection page */
	218
	219	movq (%rbx), %rcx
	220	addq $8, %rbx
	221	1:
	222	testq $0x1, %rcx /* is it a destination page? */
	223	jz 2f
	224	movq %rcx, %rdi
	225	andq $0xfffffffffffff000, %rdi
	226	jmp 0b
	227	2:
	228	testq $0x2, %rcx /* is it an indirection page? */
	229	jz 2f
	230	movq %rcx, %rbx
	231	andq $0xfffffffffffff000, %rbx
	232	jmp 0b
	233	2:
	234	testq $0x4, %rcx /* is it the done indicator? */
	235	jz 2f
	236	jmp 3f
	237	2:
	238	testq $0x8, %rcx /* is it the source indicator? */
	239	jz 0b /* Ignore it otherwise */
	240	movq %rcx, %rsi /* For ever source page do a copy */
	241	andq $0xfffffffffffff000, %rsi
	242
fee7b0d8 HY	243	movq %rdi, %rdx
	244	movq %rsi, %rax
	245
	246	movq %r10, %rdi
5234f5eb EB	247	movq $512, %rcx
5234f5eb EB	248	rep ; movsq
5234f5eb	249
fee7b0d8 HY	250	movq %rax, %rdi
	251	movq %rdx, %rsi
	252	movq $512, %rcx
	253	rep ; movsq
5234f5eb	254
fee7b0d8 HY	255	movq %rdx, %rdi
	256	movq %r10, %rsi
	257	movq $512, %rcx
	258	rep ; movsq
5234f5eb	259
fee7b0d8 HY	260	lea PAGE_SIZE(%rax), %rsi
	261	jmp 0b
	262	3:
5234f5eb	263	ret
fee7b0d8 HY	264
	265	.globl kexec_control_code_size
	266	.set kexec_control_code_size, . - relocate_kernel