]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/x86/kernel/head_64.S
Merge tag 'drm-intel-fixes-2013-10-29' of git://people.freedesktop.org/~danvet/drm...
[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / head_64.S
CommitLineData
1da177e4
LT
1/*
2 * linux/arch/x86_64/kernel/head.S -- start in 32bit and switch to 64bit
3 *
4 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
7 * Copyright (C) 2001,2002 Andi Kleen <ak@suse.de>
1ab60e0f 8 * Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
1da177e4
LT
9 */
10
11
12#include <linux/linkage.h>
13#include <linux/threads.h>
f6c2e333 14#include <linux/init.h>
1da177e4 15#include <asm/segment.h>
67dcbb6b 16#include <asm/pgtable.h>
1da177e4
LT
17#include <asm/page.h>
18#include <asm/msr.h>
19#include <asm/cache.h>
369101da 20#include <asm/processor-flags.h>
b12d8db8 21#include <asm/percpu.h>
9900aa2f 22#include <asm/nops.h>
1ab60e0f 23
49a69787
GOC
24#ifdef CONFIG_PARAVIRT
25#include <asm/asm-offsets.h>
26#include <asm/paravirt.h>
ffc4bc9c 27#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
49a69787 28#else
ffc4bc9c 29#define GET_CR2_INTO(reg) movq %cr2, reg
9900aa2f 30#define INTERRUPT_RETURN iretq
49a69787
GOC
31#endif
32
3ad2f3fb 33/* we are not able to switch in one step to the final KERNEL ADDRESS SPACE
1ab60e0f
VG
34 * because we need identity-mapped pages.
35 *
1da177e4
LT
36 */
37
a6523748
EH
38#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
39
40L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET)
41L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET)
42L4_START_KERNEL = pgd_index(__START_KERNEL_map)
43L3_START_KERNEL = pud_index(__START_KERNEL_map)
44
1da177e4 45 .text
4ae59b91 46 __HEAD
1ab60e0f
VG
47 .code64
48 .globl startup_64
49startup_64:
1da177e4 50 /*
1256276c 51 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
1ab60e0f
VG
52 * and someone has loaded an identity mapped page table
53 * for us. These identity mapped page tables map all of the
54 * kernel pages and possibly all of memory.
55 *
8170e6be 56 * %rsi holds a physical pointer to real_mode_data.
1ab60e0f
VG
57 *
58 * We come here either directly from a 64bit bootloader, or from
59 * arch/x86_64/boot/compressed/head.S.
60 *
61 * We only come here initially at boot nothing else comes here.
62 *
63 * Since we may be loaded at an address different from what we were
64 * compiled to run at we first fixup the physical addresses in our page
65 * tables and then reload them.
1da177e4
LT
66 */
67
8170e6be
PA
68 /*
69 * Compute the delta between the address I am compiled to run at and the
1ab60e0f 70 * address I am actually running at.
1da177e4 71 */
1ab60e0f
VG
72 leaq _text(%rip), %rbp
73 subq $_text - __START_KERNEL_map, %rbp
74
75 /* Is the address not 2M aligned? */
76 movq %rbp, %rax
31422c51 77 andl $~PMD_PAGE_MASK, %eax
1ab60e0f
VG
78 testl %eax, %eax
79 jnz bad_address
80
8170e6be
PA
81 /*
82 * Is the address too large?
1da177e4 83 */
8170e6be
PA
84 leaq _text(%rip), %rax
85 shrq $MAX_PHYSMEM_BITS, %rax
86 jnz bad_address
1ab60e0f 87
8170e6be
PA
88 /*
89 * Fixup the physical addresses in the page table
90 */
91 addq %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip)
b1c931e3 92
1ab60e0f 93 addq %rbp, level3_kernel_pgt + (510*8)(%rip)
b1c931e3
EB
94 addq %rbp, level3_kernel_pgt + (511*8)(%rip)
95
96 addq %rbp, level2_fixmap_pgt + (506*8)(%rip)
1ab60e0f 97
8170e6be
PA
98 /*
99 * Set up the identity mapping for the switchover. These
100 * entries should *NOT* have the global bit set! This also
101 * creates a bunch of nonsense entries but that is fine --
102 * it avoids problems around wraparound.
103 */
1ab60e0f 104 leaq _text(%rip), %rdi
8170e6be 105 leaq early_level4_pgt(%rip), %rbx
1ab60e0f
VG
106
107 movq %rdi, %rax
8170e6be 108 shrq $PGDIR_SHIFT, %rax
1ab60e0f 109
8170e6be
PA
110 leaq (4096 + _KERNPG_TABLE)(%rbx), %rdx
111 movq %rdx, 0(%rbx,%rax,8)
112 movq %rdx, 8(%rbx,%rax,8)
1ab60e0f 113
8170e6be 114 addq $4096, %rdx
1ab60e0f 115 movq %rdi, %rax
8170e6be
PA
116 shrq $PUD_SHIFT, %rax
117 andl $(PTRS_PER_PUD-1), %eax
e9d0626e
ZY
118 movq %rdx, 4096(%rbx,%rax,8)
119 incl %eax
120 andl $(PTRS_PER_PUD-1), %eax
121 movq %rdx, 4096(%rbx,%rax,8)
8170e6be
PA
122
123 addq $8192, %rbx
124 movq %rdi, %rax
125 shrq $PMD_SHIFT, %rdi
126 addq $(__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL), %rax
127 leaq (_end - 1)(%rip), %rcx
128 shrq $PMD_SHIFT, %rcx
129 subq %rdi, %rcx
130 incl %ecx
131
1321:
133 andq $(PTRS_PER_PMD - 1), %rdi
134 movq %rax, (%rbx,%rdi,8)
135 incq %rdi
136 addq $PMD_SIZE, %rax
137 decl %ecx
138 jnz 1b
1ab60e0f 139
31eedd82
TG
140 /*
141 * Fixup the kernel text+data virtual addresses. Note that
142 * we might write invalid pmds, when the kernel is relocated
143 * cleanup_highmap() fixes this up along with the mappings
144 * beyond _end.
1ab60e0f
VG
145 */
146 leaq level2_kernel_pgt(%rip), %rdi
147 leaq 4096(%rdi), %r8
148 /* See if it is a valid page table entry */
1491: testq $1, 0(%rdi)
150 jz 2f
151 addq %rbp, 0(%rdi)
152 /* Go to the next page */
1532: addq $8, %rdi
154 cmp %r8, %rdi
155 jne 1b
156
157 /* Fixup phys_base */
158 addq %rbp, phys_base(%rip)
1da177e4 159
8170e6be
PA
160 movq $(early_level4_pgt - __START_KERNEL_map), %rax
161 jmp 1f
90b1c208 162ENTRY(secondary_startup_64)
1ab60e0f 163 /*
1256276c 164 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
1ab60e0f
VG
165 * and someone has loaded a mapped page table.
166 *
8170e6be 167 * %rsi holds a physical pointer to real_mode_data.
1ab60e0f
VG
168 *
169 * We come here either from startup_64 (using physical addresses)
170 * or from trampoline.S (using virtual addresses).
171 *
172 * Using virtual addresses from trampoline.S removes the need
173 * to have any identity mapped pages in the kernel page table
174 * after the boot processor executes this code.
1da177e4
LT
175 */
176
8170e6be
PA
177 movq $(init_level4_pgt - __START_KERNEL_map), %rax
1781:
179
1da177e4 180 /* Enable PAE mode and PGE */
8170e6be
PA
181 movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
182 movq %rcx, %cr4
1da177e4
LT
183
184 /* Setup early boot stage 4 level pagetables. */
1ab60e0f 185 addq phys_base(%rip), %rax
1da177e4
LT
186 movq %rax, %cr3
187
1ab60e0f
VG
188 /* Ensure I am executing from virtual addresses */
189 movq $1f, %rax
190 jmp *%rax
1911:
192
1da177e4
LT
193 /* Check if nx is implemented */
194 movl $0x80000001, %eax
195 cpuid
196 movl %edx,%edi
197
198 /* Setup EFER (Extended Feature Enable Register) */
199 movl $MSR_EFER, %ecx
200 rdmsr
1ab60e0f
VG
201 btsl $_EFER_SCE, %eax /* Enable System Call */
202 btl $20,%edi /* No Execute supported? */
1da177e4
LT
203 jnc 1f
204 btsl $_EFER_NX, %eax
78d77df7 205 btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
1ab60e0f 2061: wrmsr /* Make changes effective */
1da177e4
LT
207
208 /* Setup cr0 */
369101da
CG
209#define CR0_STATE (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \
210 X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \
211 X86_CR0_PG)
212 movl $CR0_STATE, %eax
1da177e4
LT
213 /* Make changes effective */
214 movq %rax, %cr0
215
216 /* Setup a boot time stack */
8170e6be 217 movq stack_start(%rip), %rsp
1da177e4
LT
218
219 /* zero EFLAGS after setting rsp */
220 pushq $0
221 popfq
222
223 /*
224 * We must switch to a new descriptor in kernel space for the GDT
225 * because soon the kernel won't have access anymore to the userspace
226 * addresses where we're currently running on. We have to do that here
227 * because in 32bit we couldn't load a 64bit linear address.
228 */
a939098a 229 lgdt early_gdt_descr(%rip)
1da177e4 230
8ec6993d
BG
231 /* set up data segments */
232 xorl %eax,%eax
ffb60175
ZA
233 movl %eax,%ds
234 movl %eax,%ss
235 movl %eax,%es
236
237 /*
238 * We don't really need to load %fs or %gs, but load them anyway
239 * to kill any stale realmode selectors. This allows execution
240 * under VT hardware.
241 */
242 movl %eax,%fs
243 movl %eax,%gs
244
f32ff538
TH
245 /* Set up %gs.
246 *
947e76cd
BG
247 * The base of %gs always points to the bottom of the irqstack
248 * union. If the stack protector canary is enabled, it is
249 * located at %gs:40. Note that, on SMP, the boot cpu uses
250 * init data section till per cpu areas are set up.
f32ff538 251 */
1da177e4 252 movl $MSR_GS_BASE,%ecx
650fb439
BG
253 movl initial_gs(%rip),%eax
254 movl initial_gs+4(%rip),%edx
1da177e4
LT
255 wrmsr
256
8170e6be 257 /* rsi is pointer to real mode structure with interesting info.
1da177e4 258 pass it to C */
8170e6be 259 movq %rsi, %rdi
1da177e4
LT
260
261 /* Finally jump to run C code and to be on real kernel address
262 * Since we are running on identity-mapped space we have to jump
26374c7b
EB
263 * to the full 64bit address, this is only possible as indirect
264 * jump. In addition we need to ensure %cs is set so we make this
265 * a far return.
8170e6be
PA
266 *
267 * Note: do not change to far jump indirect with 64bit offset.
268 *
269 * AMD does not support far jump indirect with 64bit offset.
270 * AMD64 Architecture Programmer's Manual, Volume 3: states only
271 * JMP FAR mem16:16 FF /5 Far jump indirect,
272 * with the target specified by a far pointer in memory.
273 * JMP FAR mem16:32 FF /5 Far jump indirect,
274 * with the target specified by a far pointer in memory.
275 *
276 * Intel64 does support 64bit offset.
277 * Software Developer Manual Vol 2: states:
278 * FF /5 JMP m16:16 Jump far, absolute indirect,
279 * address given in m16:16
280 * FF /5 JMP m16:32 Jump far, absolute indirect,
281 * address given in m16:32.
282 * REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
283 * address given in m16:64.
1da177e4
LT
284 */
285 movq initial_code(%rip),%rax
26374c7b
EB
286 pushq $0 # fake return address to stop unwinder
287 pushq $__KERNEL_CS # set correct cs
288 pushq %rax # target address in negative space
289 lretq
1da177e4 290
42e78e97
FY
291#ifdef CONFIG_HOTPLUG_CPU
292/*
293 * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
294 * up already except stack. We just set up stack here. Then call
295 * start_secondary().
296 */
297ENTRY(start_cpu0)
298 movq stack_start(%rip),%rsp
299 movq initial_code(%rip),%rax
300 pushq $0 # fake return address to stop unwinder
301 pushq $__KERNEL_CS # set correct cs
302 pushq %rax # target address in negative space
303 lretq
304ENDPROC(start_cpu0)
305#endif
306
e57113bc 307 /* SMP bootup changes these two */
da5968ae 308 __REFDATA
8170e6be
PA
309 .balign 8
310 GLOBAL(initial_code)
1da177e4 311 .quad x86_64_start_kernel
8170e6be 312 GLOBAL(initial_gs)
2add8e23 313 .quad INIT_PER_CPU_VAR(irq_stack_union)
f1fbabb3 314
8170e6be 315 GLOBAL(stack_start)
1da177e4 316 .quad init_thread_union+THREAD_SIZE-8
9cf4f298 317 .word 0
b9af7c0d 318 __FINITDATA
1da177e4 319
1ab60e0f
VG
320bad_address:
321 jmp bad_address
322
8170e6be 323 __INIT
8866cd9d
RM
324 .globl early_idt_handlers
325early_idt_handlers:
9900aa2f
PA
326 # 104(%rsp) %rflags
327 # 96(%rsp) %cs
328 # 88(%rsp) %rip
329 # 80(%rsp) error code
749c970a
AK
330 i = 0
331 .rept NUM_EXCEPTION_VECTORS
9900aa2f
PA
332 .if (EXCEPTION_ERRCODE_MASK >> i) & 1
333 ASM_NOP2
334 .else
335 pushq $0 # Dummy error code, to make stack frame uniform
336 .endif
337 pushq $i # 72(%rsp) Vector number
749c970a
AK
338 jmp early_idt_handler
339 i = i + 1
340 .endr
8866cd9d 341
ac630dd9 342/* This is global to keep gas from relaxing the jumps */
1da177e4 343ENTRY(early_idt_handler)
9900aa2f
PA
344 cld
345
b957591f
AK
346 cmpl $2,early_recursion_flag(%rip)
347 jz 1f
348 incl early_recursion_flag(%rip)
9900aa2f
PA
349
350 pushq %rax # 64(%rsp)
351 pushq %rcx # 56(%rsp)
352 pushq %rdx # 48(%rsp)
353 pushq %rsi # 40(%rsp)
354 pushq %rdi # 32(%rsp)
355 pushq %r8 # 24(%rsp)
356 pushq %r9 # 16(%rsp)
357 pushq %r10 # 8(%rsp)
358 pushq %r11 # 0(%rsp)
359
360 cmpl $__KERNEL_CS,96(%rsp)
8170e6be
PA
361 jne 11f
362
363 cmpl $14,72(%rsp) # Page fault?
364 jnz 10f
365 GET_CR2_INTO(%rdi) # can clobber any volatile register if pv
366 call early_make_pgtable
367 andl %eax,%eax
368 jz 20f # All good
9900aa2f 369
8170e6be 37010:
9900aa2f
PA
371 leaq 88(%rsp),%rdi # Pointer to %rip
372 call early_fixup_exception
373 andl %eax,%eax
374 jnz 20f # Found an exception entry
375
8170e6be 37611:
9900aa2f
PA
377#ifdef CONFIG_EARLY_PRINTK
378 GET_CR2_INTO(%r9) # can clobber any volatile register if pv
379 movl 80(%rsp),%r8d # error code
380 movl 72(%rsp),%esi # vector number
381 movl 96(%rsp),%edx # %cs
382 movq 88(%rsp),%rcx # %rip
8866cd9d 383 xorl %eax,%eax
1da177e4
LT
384 leaq early_idt_msg(%rip),%rdi
385 call early_printk
b957591f
AK
386 cmpl $2,early_recursion_flag(%rip)
387 jz 1f
388 call dump_stack
6574ffd7
AK
389#ifdef CONFIG_KALLSYMS
390 leaq early_idt_ripmsg(%rip),%rdi
9900aa2f 391 movq 40(%rsp),%rsi # %rip again
6574ffd7
AK
392 call __print_symbol
393#endif
076f9776 394#endif /* EARLY_PRINTK */
1da177e4
LT
3951: hlt
396 jmp 1b
076f9776 397
8170e6be 39820: # Exception table entry found or page table generated
9900aa2f
PA
399 popq %r11
400 popq %r10
401 popq %r9
402 popq %r8
403 popq %rdi
404 popq %rsi
405 popq %rdx
406 popq %rcx
407 popq %rax
408 addq $16,%rsp # drop vector number and error code
409 decl early_recursion_flag(%rip)
410 INTERRUPT_RETURN
ac630dd9 411ENDPROC(early_idt_handler)
9900aa2f 412
8170e6be
PA
413 __INITDATA
414
9900aa2f 415 .balign 4
b957591f
AK
416early_recursion_flag:
417 .long 0
1da177e4 418
9900aa2f 419#ifdef CONFIG_EARLY_PRINTK
1da177e4 420early_idt_msg:
8866cd9d 421 .asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n"
6574ffd7
AK
422early_idt_ripmsg:
423 .asciz "RIP %s\n"
076f9776 424#endif /* CONFIG_EARLY_PRINTK */
1da177e4 425
f0cf5d1a 426#define NEXT_PAGE(name) \
67dcbb6b 427 .balign PAGE_SIZE; \
8170e6be 428GLOBAL(name)
f0cf5d1a 429
67dcbb6b 430/* Automate the creation of 1 to 1 mapping pmd entries */
0e192b99
CG
431#define PMDS(START, PERM, COUNT) \
432 i = 0 ; \
433 .rept (COUNT) ; \
434 .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
435 i = i + 1 ; \
67dcbb6b
VG
436 .endr
437
8170e6be
PA
438 __INITDATA
439NEXT_PAGE(early_level4_pgt)
440 .fill 511,8,0
441 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
442
443NEXT_PAGE(early_dynamic_pgts)
444 .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
445
b9af7c0d 446 .data
8170e6be
PA
447
448#ifndef CONFIG_XEN
f0cf5d1a 449NEXT_PAGE(init_level4_pgt)
8170e6be
PA
450 .fill 512,8,0
451#else
452NEXT_PAGE(init_level4_pgt)
453 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
454 .org init_level4_pgt + L4_PAGE_OFFSET*8, 0
455 .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
456 .org init_level4_pgt + L4_START_KERNEL*8, 0
cfd243d4 457 /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
8170e6be 458 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
1da177e4 459
f0cf5d1a 460NEXT_PAGE(level3_ident_pgt)
67dcbb6b 461 .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
8170e6be
PA
462 .fill 511, 8, 0
463NEXT_PAGE(level2_ident_pgt)
464 /* Since I easily can, map the first 1G.
465 * Don't set NX because code runs from these pages.
466 */
467 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
468#endif
1da177e4 469
f0cf5d1a 470NEXT_PAGE(level3_kernel_pgt)
a6523748 471 .fill L3_START_KERNEL,8,0
1da177e4 472 /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
67dcbb6b 473 .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
b1c931e3
EB
474 .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
475
f0cf5d1a 476NEXT_PAGE(level2_kernel_pgt)
88f3aec7 477 /*
85eb69a1 478 * 512 MB kernel mapping. We spend a full page on this pagetable
88f3aec7
IM
479 * anyway.
480 *
481 * The kernel code+data+bss must not be bigger than that.
482 *
85eb69a1 483 * (NOTE: at +512MB starts the module area, see MODULES_VADDR.
88f3aec7
IM
484 * If you want to increase this then increase MODULES_VADDR
485 * too.)
486 */
8490638c 487 PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
d4afe414 488 KERNEL_IMAGE_SIZE/PMD_SIZE)
1da177e4 489
8170e6be
PA
490NEXT_PAGE(level2_fixmap_pgt)
491 .fill 506,8,0
492 .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
493 /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
494 .fill 5,8,0
495
496NEXT_PAGE(level1_fixmap_pgt)
497 .fill 512,8,0
1ab60e0f 498
67dcbb6b 499#undef PMDS
1da177e4 500
f0cf5d1a 501 .data
1da177e4 502 .align 16
a939098a
GC
503 .globl early_gdt_descr
504early_gdt_descr:
505 .word GDT_ENTRIES*8-1
3e5d8f97 506early_gdt_descr_base:
2add8e23 507 .quad INIT_PER_CPU_VAR(gdt_page)
1da177e4 508
1ab60e0f
VG
509ENTRY(phys_base)
510 /* This must match the first entry in level2_kernel_pgt */
511 .quad 0x0000000000000000
512
8c5e5ac3 513#include "../../x86/xen/xen-head.S"
1da177e4 514
02b7da37 515 __PAGE_ALIGNED_BSS
8170e6be 516NEXT_PAGE(empty_zero_page)
e57113bc 517 .skip PAGE_SIZE