3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
15 * This file contains the entry point for the 64-bit kernel along
16 * with some early initialization code common to all 64-bit powerpc
19 * This program is free software; you can redistribute it and/or
20 * modify it under the terms of the GNU General Public License
21 * as published by the Free Software Foundation; either version
22 * 2 of the License, or (at your option) any later version.
25 #include <linux/threads.h>
26 #include <linux/init.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/head-64.h>
32 #include <asm/asm-offsets.h>
34 #include <asm/cputable.h>
35 #include <asm/setup.h>
36 #include <asm/hvcall.h>
37 #include <asm/thread_info.h>
38 #include <asm/firmware.h>
39 #include <asm/page_64.h>
40 #include <asm/irqflags.h>
41 #include <asm/kvm_book3s_asm.h>
42 #include <asm/ptrace.h>
43 #include <asm/hw_irq.h>
44 #include <asm/cputhreads.h>
45 #include <asm/ppc-opcode.h>
46 #include <asm/export.h>
48 /* The physical memory is laid out such that the secondary processor
49 * spin code sits at 0x0000...0x00ff. On server, the vectors follow
50 * using the layout described in exceptions-64s.S
54 * Entering into this code we make the following assumptions:
56 * For pSeries or server processors:
57 * 1. The MMU is off & open firmware is running in real mode.
58 * 2. The kernel is entered at __start
59 * -or- For OPAL entry:
60 * 1. The MMU is off, processor in HV mode, primary CPU enters at 0
61 * with device-tree in gpr3. We also get OPAL base in r8 and
62 * entry in r9 for debugging purposes
63 * 2. Secondary processors enter at 0x60 with PIR in gpr3
65 * For Book3E processors:
66 * 1. The MMU is on running in AS0 in a state defined in ePAPR
67 * 2. The kernel is entered at __start
70 OPEN_FIXED_SECTION(first_256B, 0x0, 0x100)
71 USE_FIXED_SECTION(first_256B)
73 * Offsets are relative from the start of fixed section, and
74 * first_256B starts at 0. Offsets are a bit easier to use here
75 * than the fixed section entry macros.
79 /* NOP this out unconditionally */
82 b __start_initialization_multiplatform
85 /* Catch branch to 0 in real mode */
88 /* Secondary processors spin on this value until it becomes non-zero.
89 * When non-zero, it contains the real address of the function the cpu
93 .globl __secondary_hold_spinloop
94 __secondary_hold_spinloop:
97 /* Secondary processors write this value with their cpu # */
98 /* after they enter the spin loop immediately below. */
99 .globl __secondary_hold_acknowledge
100 __secondary_hold_acknowledge:
103 #ifdef CONFIG_RELOCATABLE
104 /* This flag is set to 1 by a loader if the kernel should run
105 * at the loaded address instead of the linked address. This
106 * is used by kexec-tools to keep the the kdump kernel in the
107 * crash_kernel region. The loader is responsible for
108 * observing the alignment requirement.
110 /* Do not move this variable as kexec-tools knows about it. */
114 DEFINE_FIXED_SYMBOL(__run_at_load)
115 .long 0x72756e30 /* "run0" -- relocate to 0 by default */
120 * The following code is used to hold secondary processors
121 * in a spin loop after they have entered the kernel, but
122 * before the bulk of the kernel has been relocated. This code
123 * is relocated to physical address 0x60 before prom_init is run.
124 * All of it must fit below the first exception vector at 0x100.
125 * Use .globl here not _GLOBAL because we want __secondary_hold
126 * to be the actual text address, not a descriptor.
128 .globl __secondary_hold
131 #ifndef CONFIG_PPC_BOOK3E
134 mtmsrd r24 /* RI on */
136 /* Grab our physical cpu number */
138 /* stash r4 for book3e */
141 /* Tell the master cpu we're here */
142 /* Relocation is off & we are located at an address less */
143 /* than 0x100, so only need to grab low order offset. */
144 std r24,(ABS_ADDR(__secondary_hold_acknowledge))(0)
148 #ifdef CONFIG_PPC_BOOK3E
151 /* All secondary cpus wait here until told to start. */
152 100: ld r12,(ABS_ADDR(__secondary_hold_spinloop))(r26)
156 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
157 #ifdef CONFIG_PPC_BOOK3E
163 * it may be the case that other platforms have r4 right to
164 * begin with, this gives us some safety in case it is not
166 #ifdef CONFIG_PPC_BOOK3E
171 /* Make sure that patched code is visible */
177 CLOSE_FIXED_SECTION(first_256B)
179 /* This value is used to mark exception frames on the stack. */
182 .tc ID_72656773_68657265[TC],0x7265677368657265
186 * On server, we include the exception vectors code here as it
187 * relies on absolute addressing which is only possible within
188 * this compilation unit
190 #ifdef CONFIG_PPC_BOOK3S
191 #include "exceptions-64s.S"
193 OPEN_TEXT_SECTION(0x100)
198 #ifdef CONFIG_PPC_BOOK3E
200 * The booting_thread_hwid holds the thread id we want to boot in cpu
201 * hotplug case. It is set by cpu hotplug code, and is invalid by default.
202 * The thread id is the same as the initial value of SPRN_PIR[THREAD_ID]
205 .globl booting_thread_hwid
207 .long INVALID_THREAD_HWID
210 * start a thread in the same core
212 * r3 = the thread physical id
213 * r4 = the entry point where thread starts
215 _GLOBAL(book3e_start_thread)
216 LOAD_REG_IMMEDIATE(r5, MSR_KERNEL)
221 /* If the thread id is invalid, just exit. */
239 * stop a thread in the same core
241 * r3 = the thread physical id
243 _GLOBAL(book3e_stop_thread)
248 /* If the thread id is invalid, just exit. */
257 _GLOBAL(fsl_secondary_thread_init)
260 /* Enable branch prediction */
262 ori r3,r3,BUCSR_INIT@l
267 * Fix PIR to match the linear numbering in the device tree.
269 * On e6500, the reset value of PIR uses the low three bits for
270 * the thread within a core, and the upper bits for the core
271 * number. There are two threads per core, so shift everything
272 * but the low bit right by two bits so that the cpu numbering is
275 * If the old value of BUCSR is non-zero, this thread has run
276 * before. Thus, we assume we are coming from kexec or a similar
277 * scenario, and PIR is already set to the correct value. This
278 * is a bit of a hack, but there are limited opportunities for
279 * getting information into the thread and the alternatives
280 * seemed like they'd be overkill. We can't tell just by looking
281 * at the old PIR value which state it's in, since the same value
282 * could be valid for one thread out of reset and for a different
289 rlwimi r3, r3, 30, 2, 30
294 _GLOBAL(generic_secondary_thread_init)
297 /* turn on 64-bit mode */
300 /* get a valid TOC pointer, wherever we're mapped at */
304 #ifdef CONFIG_PPC_BOOK3E
305 /* Book3E initialization */
307 bl book3e_secondary_thread_init
309 b generic_secondary_common_init
312 * On pSeries and most other platforms, secondary processors spin
313 * in the following code.
314 * At entry, r3 = this processor's number (physical cpu id)
316 * On Book3E, r4 = 1 to indicate that the initial TLB entry for
317 * this core already exists (setup via some other mechanism such
318 * as SCOM before entry).
320 _GLOBAL(generic_secondary_smp_init)
325 /* turn on 64-bit mode */
328 /* get a valid TOC pointer, wherever we're mapped at */
332 #ifdef CONFIG_PPC_BOOK3E
333 /* Book3E initialization */
336 bl book3e_secondary_core_init
339 * After common core init has finished, check if the current thread is the
340 * one we wanted to boot. If not, start the specified thread and stop the
343 LOAD_REG_ADDR(r4, booting_thread_hwid)
345 li r5, INVALID_THREAD_HWID
350 * The value of booting_thread_hwid has been stored in r3,
351 * so make it invalid.
356 * Get the current thread id and check if it is the one we wanted.
357 * If not, start the one specified in booting_thread_hwid and stop
358 * the current thread.
364 /* start the specified thread */
365 LOAD_REG_ADDR(r5, fsl_secondary_thread_init)
367 bl book3e_start_thread
369 /* stop the current thread */
371 bl book3e_stop_thread
377 generic_secondary_common_init:
378 /* Set up a paca value for this processor. Since we have the
379 * physical cpu id in r24, we need to search the pacas to find
380 * which logical id maps to our physical one.
382 LOAD_REG_ADDR(r13, paca) /* Load paca pointer */
383 ld r13,0(r13) /* Get base vaddr of paca array */
385 addi r13,r13,PACA_SIZE /* know r13 if used accidentally */
386 b kexec_wait /* wait for next kernel if !SMP */
388 LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */
389 lwz r7,0(r7) /* also the max paca allocated */
390 li r5,0 /* logical cpu id */
391 1: lhz r6,PACAHWCPUID(r13) /* Load HW procid from paca */
392 cmpw r6,r24 /* Compare to our id */
394 addi r13,r13,PACA_SIZE /* Loop to next PACA on miss */
396 cmpw r5,r7 /* Check if more pacas exist */
399 mr r3,r24 /* not found, copy phys to r3 */
400 b kexec_wait /* next kernel might do better */
403 #ifdef CONFIG_PPC_BOOK3E
404 addi r12,r13,PACA_EXTLB /* and TLB exc frame in another */
405 mtspr SPRN_SPRG_TLB_EXFRAME,r12
408 /* From now on, r24 is expected to be logical cpuid */
411 /* See if we need to call a cpu state restore handler */
412 LOAD_REG_ADDR(r23, cur_cpu_spec)
414 ld r12,CPU_SPEC_RESTORE(r23)
417 #ifdef PPC64_ELF_ABI_v1
423 3: LOAD_REG_ADDR(r3, spinning_secondaries) /* Decrement spinning_secondaries */
431 lbz r23,PACAPROCSTART(r13) /* Test if this processor should */
434 beq 4b /* Loop until told to go */
436 sync /* order paca.run and cur_cpu_spec */
437 isync /* In case code patching happened */
439 /* Create a temp kernel stack for use before relocation is on. */
440 ld r1,PACAEMERGSP(r13)
441 subi r1,r1,STACK_FRAME_OVERHEAD
448 * Assumes we're mapped EA == RA if the MMU is on.
450 #ifdef CONFIG_PPC_BOOK3S
453 andi. r0,r3,MSR_IR|MSR_DR
461 b . /* prevent speculative execution */
466 * Here is our main kernel entry point. We support currently 2 kind of entries
467 * depending on the value of r5.
469 * r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
472 * r5 == NULL -> kexec style entry. r3 is a physical pointer to the
473 * DT block, r4 is a physical pointer to the kernel itself
476 __start_initialization_multiplatform:
477 /* Make sure we are running in 64 bits mode */
480 /* Get TOC pointer (current runtime address) */
483 /* find out where we are now */
485 0: mflr r26 /* r26 = runtime addr here */
486 addis r26,r26,(_stext - 0b)@ha
487 addi r26,r26,(_stext - 0b)@l /* current runtime base addr */
490 * Are we booted from a PROM Of-type client-interface ?
494 b __boot_from_prom /* yes -> prom */
496 /* Save parameters */
499 #ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
500 /* Save OPAL entry */
505 #ifdef CONFIG_PPC_BOOK3E
506 bl start_initialization_book3e
509 /* Setup some critical 970 SPRs before switching MMU off */
512 cmpwi r0,0x39 /* 970 */
514 cmpwi r0,0x3c /* 970FX */
516 cmpwi r0,0x44 /* 970MP */
518 cmpwi r0,0x45 /* 970GX */
520 1: bl __cpu_preinit_ppc970
523 /* Switch off MMU if not already off */
526 #endif /* CONFIG_PPC_BOOK3E */
529 #ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
530 /* Save parameters */
538 * Align the stack to 16-byte boundary
539 * Depending on the size and layout of the ELF sections in the initial
540 * boot binary, the stack pointer may be unaligned on PowerMac
544 #ifdef CONFIG_RELOCATABLE
545 /* Relocate code for where we are now */
550 /* Restore parameters */
557 /* Do all of the interaction with OF client interface */
560 #endif /* #CONFIG_PPC_OF_BOOT_TRAMPOLINE */
562 /* We never return. We also hit that trap if trying to boot
563 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
567 #ifdef CONFIG_RELOCATABLE
568 /* process relocations for the final address of the kernel */
569 lis r25,PAGE_OFFSET@highest /* compute virtual base of kernel */
571 #if defined(CONFIG_PPC_BOOK3E)
572 tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */
574 lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
575 #if defined(CONFIG_PPC_BOOK3E)
578 cmplwi cr0,r7,1 /* flagged to stay where we are ? */
583 #if defined(CONFIG_PPC_BOOK3E)
584 /* IVPR needs to be set after relocation. */
590 * We need to run with _stext at physical address PHYSICAL_START.
591 * This will leave some code in the first 256B of
592 * real memory, which are reserved for software use.
594 * Note: This process overwrites the OF exception vectors.
596 li r3,0 /* target addr */
597 #ifdef CONFIG_PPC_BOOK3E
598 tovirt(r3,r3) /* on booke, we already run at PAGE_OFFSET */
600 mr. r4,r26 /* In some cases the loader may */
601 #if defined(CONFIG_PPC_BOOK3E)
604 beq 9f /* have already put us at zero */
605 li r6,0x100 /* Start offset, the first 0x100 */
606 /* bytes were copied earlier. */
608 #ifdef CONFIG_RELOCATABLE
610 * Check if the kernel has to be running as relocatable kernel based on the
611 * variable __run_at_load, if it is set the kernel is treated as relocatable
612 * kernel, otherwise it will be moved to PHYSICAL_START
614 #if defined(CONFIG_PPC_BOOK3E)
615 tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */
617 lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
621 #ifdef CONFIG_PPC_BOOK3E
622 LOAD_REG_ADDR(r5, __end_interrupts)
623 LOAD_REG_ADDR(r11, _stext)
626 /* just copy interrupts */
627 LOAD_REG_IMMEDIATE(r5, FIXED_SYMBOL_ABS_ADDR(__end_interrupts))
632 /* # bytes of memory to copy */
633 lis r5,(ABS_ADDR(copy_to_here))@ha
634 addi r5,r5,(ABS_ADDR(copy_to_here))@l
636 bl copy_and_flush /* copy the first n bytes */
637 /* this includes the code being */
639 /* Jump to the copy of this code that we just made */
640 addis r8,r3,(ABS_ADDR(4f))@ha
641 addi r12,r8,(ABS_ADDR(4f))@l
646 p_end: .llong _end - copy_to_here
650 * Now copy the rest of the kernel up to _end, add
651 * _end - copy_to_here to the copy limit and run again.
653 addis r8,r26,(ABS_ADDR(p_end))@ha
654 ld r8,(ABS_ADDR(p_end))@l(r8)
656 5: bl copy_and_flush /* copy the rest */
658 9: b start_here_multiplatform
661 * Copy routine used to copy the kernel to start at physical address 0
662 * and flush and invalidate the caches as needed.
663 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
664 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
666 * Note: this routine *only* clobbers r0, r6 and lr
668 _GLOBAL(copy_and_flush)
671 4: li r0,8 /* Use the smallest common */
672 /* denominator cache line */
673 /* size. This results in */
674 /* extra cache line flushes */
675 /* but operation is correct. */
676 /* Can't get cache line size */
677 /* from NACA as it is being */
680 mtctr r0 /* put # words/line in ctr */
681 3: addi r6,r6,8 /* copy a cache line */
685 dcbst r6,r3 /* write it to memory */
687 icbi r6,r3 /* flush the icache line */
700 #ifdef CONFIG_PPC_PMAC
702 * On PowerMac, secondary processors starts from the reset vector, which
703 * is temporarily turned into a call to one of the functions below.
708 .globl __secondary_start_pmac_0
709 __secondary_start_pmac_0:
710 /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
720 _GLOBAL(pmac_secondary_start)
721 /* turn on 64-bit mode */
726 rldimi r3,r0,40,23 /* clear bit 23 (rm_ci) */
733 /* get TOC pointer (real address) */
737 /* Copy some CPU settings from CPU 0 */
738 bl __restore_cpu_ppc970
740 /* pSeries do that early though I don't think we really need it */
743 mtmsrd r3 /* RI on */
745 /* Set up a paca value for this processor. */
746 LOAD_REG_ADDR(r4,paca) /* Load paca pointer */
747 ld r4,0(r4) /* Get base vaddr of paca array */
748 mulli r13,r24,PACA_SIZE /* Calculate vaddr of right paca */
749 add r13,r13,r4 /* for this processor. */
750 SET_PACA(r13) /* Save vaddr of paca in an SPRG*/
752 /* Mark interrupts soft and hard disabled (they might be enabled
753 * in the PACA when doing hotplug)
756 stb r0,PACASOFTIRQEN(r13)
757 li r0,PACA_IRQ_HARD_DIS
758 stb r0,PACAIRQHAPPENED(r13)
760 /* Create a temp kernel stack for use before relocation is on. */
761 ld r1,PACAEMERGSP(r13)
762 subi r1,r1,STACK_FRAME_OVERHEAD
766 #endif /* CONFIG_PPC_PMAC */
769 * This function is called after the master CPU has released the
770 * secondary processors. The execution environment is relocation off.
771 * The paca for this processor has the following fields initialized at
773 * 1. Processor number
774 * 2. Segment table pointer (virtual address)
775 * On entry the following are set:
776 * r1 = stack pointer (real addr of temp stack)
777 * r24 = cpu# (in Linux terms)
778 * r13 = paca virtual address
779 * SPRG_PACA = paca virtual address
784 .globl __secondary_start
786 /* Set thread priority to MEDIUM */
789 /* Initialize the kernel stack */
790 LOAD_REG_ADDR(r3, current_set)
791 sldi r28,r24,3 /* get current_set[cpu#] */
793 addi r14,r14,THREAD_SIZE-STACK_FRAME_OVERHEAD
794 std r14,PACAKSAVE(r13)
796 /* Do early setup for that CPU (SLB and hash table pointer) */
797 bl early_setup_secondary
800 * setup the new stack pointer, but *don't* use this until
805 /* Clear backchain so we get nice backtraces */
809 /* Mark interrupts soft and hard disabled (they might be enabled
810 * in the PACA when doing hotplug)
812 stb r7,PACASOFTIRQEN(r13)
813 li r0,PACA_IRQ_HARD_DIS
814 stb r0,PACAIRQHAPPENED(r13)
816 /* enable MMU and jump to start_secondary */
817 LOAD_REG_ADDR(r3, start_secondary_prolog)
818 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
823 b . /* prevent speculative execution */
826 * Running with relocation on at this point. All we want to do is
827 * zero the stack back-chain pointer and get the TOC virtual address
828 * before going into C code.
830 start_secondary_prolog:
833 std r3,0(r1) /* Zero the stack frame pointer */
837 * Reset stack pointer and call start_secondary
838 * to continue with online operation when woken up
839 * from cede in cpu offline.
841 _GLOBAL(start_secondary_resume)
842 ld r1,PACAKSAVE(r13) /* Reload kernel stack pointer */
844 std r3,0(r1) /* Zero the stack frame pointer */
850 * This subroutine clobbers r11 and r12
853 mfmsr r11 /* grab the current MSR */
854 #ifdef CONFIG_PPC_BOOK3E
855 oris r11,r11,0x8000 /* CM bit set, we'll set ICM later */
857 #else /* CONFIG_PPC_BOOK3E */
858 li r12,(MSR_64BIT | MSR_ISF)@highest
867 * This puts the TOC pointer into r2, offset by 0x8000 (as expected
868 * by the toolchain). It computes the correct value for wherever we
869 * are running at the moment, using position-independent code.
871 * Note: The compiler constructs pointers using offsets from the
872 * TOC in -mcmodel=medium mode. After we relocate to 0 but before
873 * the MMU is on we need our TOC to be a virtual address otherwise
874 * these pointers will be real addresses which may get stored and
875 * accessed later with the MMU on. We use tovirt() at the call
876 * sites to handle this.
878 _GLOBAL(relative_toc)
882 ld r2,(p_toc - 0b)(r11)
888 p_toc: .llong __toc_start + 0x8000 - 0b
891 * This is where the main kernel code starts.
893 start_here_multiplatform:
898 /* Clear out the BSS. It may have been done in prom_init,
899 * already but that's irrelevant since prom_init will soon
900 * be detached from the kernel completely. Besides, we need
901 * to clear it now for kexec-style entry.
903 LOAD_REG_ADDR(r11,__bss_stop)
904 LOAD_REG_ADDR(r8,__bss_start)
905 sub r11,r11,r8 /* bss size */
906 addi r11,r11,7 /* round up to an even double word */
907 srdi. r11,r11,3 /* shift right by 3 */
911 mtctr r11 /* zero this many doublewords */
916 #ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
917 /* Setup OPAL entry */
918 LOAD_REG_ADDR(r11, opal)
923 #ifndef CONFIG_PPC_BOOK3E
926 mtmsrd r6 /* RI on */
929 #ifdef CONFIG_RELOCATABLE
930 /* Save the physical address we're running at in kernstart_addr */
931 LOAD_REG_ADDR(r4, kernstart_addr)
936 /* The following gets the stack set up with the regs */
937 /* pointing to the real addr of the kernel stack. This is */
938 /* all done to support the C function call below which sets */
939 /* up the htab. This is done because we have relocated the */
940 /* kernel but are still running in real mode. */
942 LOAD_REG_ADDR(r3,init_thread_union)
944 /* set up a stack pointer */
945 addi r1,r3,THREAD_SIZE
947 stdu r0,-STACK_FRAME_OVERHEAD(r1)
950 * Do very early kernel initializations, including initial hash table
951 * and SLB setup before we turn on relocation.
954 /* Restore parameters passed from prom_init/kexec */
956 bl early_setup /* also sets r13 and SPRG_PACA */
958 LOAD_REG_ADDR(r3, start_here_common)
963 b . /* prevent speculative execution */
965 /* This is where all platforms converge execution */
968 /* relocation is on at this point */
969 std r1,PACAKSAVE(r13)
971 /* Load the TOC (virtual address) */
974 /* Mark interrupts soft and hard disabled (they might be enabled
975 * in the PACA when doing hotplug)
978 stb r0,PACASOFTIRQEN(r13)
979 li r0,PACA_IRQ_HARD_DIS
980 stb r0,PACAIRQHAPPENED(r13)
982 /* Generic kernel entry */
989 * We put a few things here that have to be page-aligned.
990 * This stuff goes at the beginning of the bss, which is page-aligned.
994 * pgd dir should be aligned to PGD_TABLE_SIZE which is 64K.
995 * We will need to find a better way to fix this
999 .globl swapper_pg_dir
1001 .space PGD_TABLE_SIZE
1003 .globl empty_zero_page
1006 EXPORT_SYMBOL(empty_zero_page)