]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/powerpc/kernel/entry_64.S
powerpc: Exit user context on notify resume
[mirror_ubuntu-artful-kernel.git] / arch / powerpc / kernel / entry_64.S
CommitLineData
9994a338 1/*
9994a338
PM
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
9994a338
PM
21#include <linux/errno.h>
22#include <asm/unistd.h>
23#include <asm/processor.h>
24#include <asm/page.h>
25#include <asm/mmu.h>
26#include <asm/thread_info.h>
27#include <asm/ppc_asm.h>
28#include <asm/asm-offsets.h>
29#include <asm/cputable.h>
3f639ee8 30#include <asm/firmware.h>
007d88d0 31#include <asm/bug.h>
ec2b36b9 32#include <asm/ptrace.h>
945feb17 33#include <asm/irqflags.h>
395a59d0 34#include <asm/ftrace.h>
7230c564 35#include <asm/hw_irq.h>
9994a338
PM
36
37/*
38 * System calls.
39 */
40 .section ".toc","aw"
41.SYS_CALL_TABLE:
42 .tc .sys_call_table[TC],.sys_call_table
43
44/* This value is used to mark exception frames on the stack. */
45exception_marker:
ec2b36b9 46 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
9994a338
PM
47
48 .section ".text"
49 .align 7
50
51#undef SHOW_SYSCALLS
52
53 .globl system_call_common
54system_call_common:
55 andi. r10,r12,MSR_PR
56 mr r10,r1
57 addi r1,r1,-INT_FRAME_SIZE
58 beq- 1f
59 ld r1,PACAKSAVE(r13)
601: std r10,0(r1)
61 std r11,_NIP(r1)
62 std r12,_MSR(r1)
63 std r0,GPR0(r1)
64 std r10,GPR1(r1)
5d75b264 65 beq 2f /* if from kernel mode */
c6622f63 66 ACCOUNT_CPU_USER_ENTRY(r10, r11)
5d75b264 672: std r2,GPR2(r1)
9994a338 68 std r3,GPR3(r1)
fd6c40f3 69 mfcr r2
9994a338
PM
70 std r4,GPR4(r1)
71 std r5,GPR5(r1)
72 std r6,GPR6(r1)
73 std r7,GPR7(r1)
74 std r8,GPR8(r1)
75 li r11,0
76 std r11,GPR9(r1)
77 std r11,GPR10(r1)
78 std r11,GPR11(r1)
79 std r11,GPR12(r1)
823df435 80 std r11,_XER(r1)
82087414 81 std r11,_CTR(r1)
9994a338 82 std r9,GPR13(r1)
9994a338 83 mflr r10
fd6c40f3
AB
84 /*
85 * This clears CR0.SO (bit 28), which is the error indication on
86 * return from this system call.
87 */
88 rldimi r2,r11,28,(63-28)
9994a338 89 li r11,0xc01
9994a338
PM
90 std r10,_LINK(r1)
91 std r11,_TRAP(r1)
9994a338 92 std r3,ORIG_GPR3(r1)
fd6c40f3 93 std r2,_CCR(r1)
9994a338
PM
94 ld r2,PACATOC(r13)
95 addi r9,r1,STACK_FRAME_OVERHEAD
96 ld r11,exception_marker@toc(r2)
97 std r11,-16(r9) /* "regshere" marker */
abf917cd 98#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
cf9efce0
PM
99BEGIN_FW_FTR_SECTION
100 beq 33f
101 /* if from user, see if there are any DTL entries to process */
102 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
103 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
104 ld r10,LPPACA_DTLIDX(r10) /* get log write index */
105 cmpd cr1,r11,r10
106 beq+ cr1,33f
107 bl .accumulate_stolen_time
108 REST_GPR(0,r1)
109 REST_4GPRS(3,r1)
110 REST_2GPRS(7,r1)
111 addi r9,r1,STACK_FRAME_OVERHEAD
11233:
113END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
abf917cd 114#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
cf9efce0 115
1421ae0b
BH
116 /*
117 * A syscall should always be called with interrupts enabled
118 * so we just unconditionally hard-enable here. When some kind
119 * of irq tracing is used, we additionally check that condition
120 * is correct
121 */
122#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
123 lbz r10,PACASOFTIRQEN(r13)
124 xori r10,r10,1
1251: tdnei r10,0
126 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
127#endif
2d27cfd3 128
2d27cfd3
BH
129#ifdef CONFIG_PPC_BOOK3E
130 wrteei 1
131#else
1421ae0b 132 ld r11,PACAKMSR(r13)
9994a338
PM
133 ori r11,r11,MSR_EE
134 mtmsrd r11,1
2d27cfd3 135#endif /* CONFIG_PPC_BOOK3E */
9994a338 136
1421ae0b
BH
137 /* We do need to set SOFTE in the stack frame or the return
138 * from interrupt will be painful
139 */
140 li r10,1
141 std r10,SOFTE(r1)
142
9994a338
PM
143#ifdef SHOW_SYSCALLS
144 bl .do_show_syscall
145 REST_GPR(0,r1)
146 REST_4GPRS(3,r1)
147 REST_2GPRS(7,r1)
148 addi r9,r1,STACK_FRAME_OVERHEAD
149#endif
9778b696 150 CURRENT_THREAD_INFO(r11, r1)
9994a338 151 ld r10,TI_FLAGS(r11)
9994a338
PM
152 andi. r11,r10,_TIF_SYSCALL_T_OR_A
153 bne- syscall_dotrace
d14299de 154.Lsyscall_dotrace_cont:
9994a338
PM
155 cmpldi 0,r0,NR_syscalls
156 bge- syscall_enosys
157
158system_call: /* label this so stack traces look sane */
159/*
160 * Need to vector to 32 Bit or default sys_call_table here,
161 * based on caller's run-mode / personality.
162 */
163 ld r11,.SYS_CALL_TABLE@toc(2)
164 andi. r10,r10,_TIF_32BIT
165 beq 15f
166 addi r11,r11,8 /* use 32-bit syscall entries */
167 clrldi r3,r3,32
168 clrldi r4,r4,32
169 clrldi r5,r5,32
170 clrldi r6,r6,32
171 clrldi r7,r7,32
172 clrldi r8,r8,32
17315:
174 slwi r0,r0,4
175 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
176 mtctr r10
177 bctrl /* Call handler */
178
179syscall_exit:
401d1f02 180 std r3,RESULT(r1)
9994a338 181#ifdef SHOW_SYSCALLS
9994a338 182 bl .do_show_syscall_exit
401d1f02 183 ld r3,RESULT(r1)
9994a338 184#endif
9778b696 185 CURRENT_THREAD_INFO(r12, r1)
9994a338 186
9994a338 187 ld r8,_MSR(r1)
2d27cfd3
BH
188#ifdef CONFIG_PPC_BOOK3S
189 /* No MSR:RI on BookE */
9994a338
PM
190 andi. r10,r8,MSR_RI
191 beq- unrecov_restore
2d27cfd3 192#endif
1421ae0b
BH
193 /*
194 * Disable interrupts so current_thread_info()->flags can't change,
2d27cfd3
BH
195 * and so that we don't get interrupted after loading SRR0/1.
196 */
197#ifdef CONFIG_PPC_BOOK3E
198 wrteei 0
199#else
1421ae0b 200 ld r10,PACAKMSR(r13)
ac1dc365
AB
201 /*
202 * For performance reasons we clear RI the same time that we
203 * clear EE. We only need to clear RI just before we restore r13
204 * below, but batching it with EE saves us one expensive mtmsrd call.
205 * We have to be careful to restore RI if we branch anywhere from
206 * here (eg syscall_exit_work).
207 */
208 li r9,MSR_RI
209 andc r11,r10,r9
210 mtmsrd r11,1
2d27cfd3
BH
211#endif /* CONFIG_PPC_BOOK3E */
212
9994a338 213 ld r9,TI_FLAGS(r12)
401d1f02 214 li r11,-_LAST_ERRNO
1bd79336 215 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
9994a338 216 bne- syscall_exit_work
401d1f02
DW
217 cmpld r3,r11
218 ld r5,_CCR(r1)
219 bge- syscall_error
d14299de 220.Lsyscall_error_cont:
9994a338 221 ld r7,_NIP(r1)
f89451fb 222BEGIN_FTR_SECTION
9994a338 223 stdcx. r0,0,r1 /* to clear the reservation */
f89451fb 224END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
9994a338
PM
225 andi. r6,r8,MSR_PR
226 ld r4,_LINK(r1)
2d27cfd3 227
c6622f63
PM
228 beq- 1f
229 ACCOUNT_CPU_USER_EXIT(r11, r12)
44e9309f 230 HMT_MEDIUM_LOW_HAS_PPR
c6622f63 231 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
9994a338 2321: ld r2,GPR2(r1)
9994a338
PM
233 ld r1,GPR1(r1)
234 mtlr r4
235 mtcr r5
236 mtspr SPRN_SRR0,r7
237 mtspr SPRN_SRR1,r8
2d27cfd3 238 RFI
9994a338
PM
239 b . /* prevent speculative execution */
240
401d1f02 241syscall_error:
9994a338 242 oris r5,r5,0x1000 /* Set SO bit in CR */
401d1f02 243 neg r3,r3
9994a338 244 std r5,_CCR(r1)
d14299de 245 b .Lsyscall_error_cont
401d1f02 246
9994a338
PM
247/* Traced system call support */
248syscall_dotrace:
249 bl .save_nvgprs
250 addi r3,r1,STACK_FRAME_OVERHEAD
251 bl .do_syscall_trace_enter
4f72c427
RM
252 /*
253 * Restore argument registers possibly just changed.
254 * We use the return value of do_syscall_trace_enter
255 * for the call number to look up in the table (r0).
256 */
257 mr r0,r3
9994a338
PM
258 ld r3,GPR3(r1)
259 ld r4,GPR4(r1)
260 ld r5,GPR5(r1)
261 ld r6,GPR6(r1)
262 ld r7,GPR7(r1)
263 ld r8,GPR8(r1)
264 addi r9,r1,STACK_FRAME_OVERHEAD
9778b696 265 CURRENT_THREAD_INFO(r10, r1)
9994a338 266 ld r10,TI_FLAGS(r10)
d14299de 267 b .Lsyscall_dotrace_cont
9994a338 268
401d1f02
DW
269syscall_enosys:
270 li r3,-ENOSYS
271 b syscall_exit
272
273syscall_exit_work:
ac1dc365
AB
274#ifdef CONFIG_PPC_BOOK3S
275 mtmsrd r10,1 /* Restore RI */
276#endif
401d1f02
DW
277 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
278 If TIF_NOERROR is set, just save r3 as it is. */
279
280 andi. r0,r9,_TIF_RESTOREALL
1bd79336
PM
281 beq+ 0f
282 REST_NVGPRS(r1)
283 b 2f
2840: cmpld r3,r11 /* r10 is -LAST_ERRNO */
401d1f02
DW
285 blt+ 1f
286 andi. r0,r9,_TIF_NOERROR
287 bne- 1f
288 ld r5,_CCR(r1)
289 neg r3,r3
290 oris r5,r5,0x1000 /* Set SO bit in CR */
291 std r5,_CCR(r1)
2921: std r3,GPR3(r1)
2932: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
294 beq 4f
295
1bd79336 296 /* Clear per-syscall TIF flags if any are set. */
401d1f02
DW
297
298 li r11,_TIF_PERSYSCALL_MASK
299 addi r12,r12,TI_FLAGS
3003: ldarx r10,0,r12
301 andc r10,r10,r11
302 stdcx. r10,0,r12
303 bne- 3b
304 subi r12,r12,TI_FLAGS
1bd79336
PM
305
3064: /* Anything else left to do? */
05e38e5d 307 SET_DEFAULT_THREAD_PPR(r3, r10) /* Set thread.ppr = 3 */
1bd79336 308 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
401d1f02
DW
309 beq .ret_from_except_lite
310
311 /* Re-enable interrupts */
2d27cfd3
BH
312#ifdef CONFIG_PPC_BOOK3E
313 wrteei 1
314#else
1421ae0b 315 ld r10,PACAKMSR(r13)
401d1f02
DW
316 ori r10,r10,MSR_EE
317 mtmsrd r10,1
2d27cfd3 318#endif /* CONFIG_PPC_BOOK3E */
401d1f02 319
1bd79336 320 bl .save_nvgprs
9994a338
PM
321 addi r3,r1,STACK_FRAME_OVERHEAD
322 bl .do_syscall_trace_leave
1bd79336 323 b .ret_from_except
9994a338
PM
324
325/* Save non-volatile GPRs, if not already saved. */
326_GLOBAL(save_nvgprs)
327 ld r11,_TRAP(r1)
328 andi. r0,r11,1
329 beqlr-
330 SAVE_NVGPRS(r1)
331 clrrdi r0,r11,1
332 std r0,_TRAP(r1)
333 blr
334
401d1f02 335
9994a338
PM
336/*
337 * The sigsuspend and rt_sigsuspend system calls can call do_signal
338 * and thus put the process into the stopped state where we might
339 * want to examine its user state with ptrace. Therefore we need
340 * to save all the nonvolatile registers (r14 - r31) before calling
341 * the C code. Similarly, fork, vfork and clone need the full
342 * register state on the stack so that it can be copied to the child.
343 */
9994a338
PM
344
345_GLOBAL(ppc_fork)
346 bl .save_nvgprs
347 bl .sys_fork
348 b syscall_exit
349
350_GLOBAL(ppc_vfork)
351 bl .save_nvgprs
352 bl .sys_vfork
353 b syscall_exit
354
355_GLOBAL(ppc_clone)
356 bl .save_nvgprs
357 bl .sys_clone
358 b syscall_exit
359
1bd79336
PM
360_GLOBAL(ppc32_swapcontext)
361 bl .save_nvgprs
362 bl .compat_sys_swapcontext
363 b syscall_exit
364
365_GLOBAL(ppc64_swapcontext)
366 bl .save_nvgprs
367 bl .sys_swapcontext
368 b syscall_exit
369
9994a338
PM
370_GLOBAL(ret_from_fork)
371 bl .schedule_tail
372 REST_NVGPRS(r1)
373 li r3,0
374 b syscall_exit
375
58254e10
AV
376_GLOBAL(ret_from_kernel_thread)
377 bl .schedule_tail
378 REST_NVGPRS(r1)
53b50f94 379 ld r14, 0(r14)
58254e10
AV
380 mtlr r14
381 mr r3,r15
382 blrl
383 li r3,0
be6abfa7
AV
384 b syscall_exit
385
71433285
AB
386 .section ".toc","aw"
387DSCR_DEFAULT:
388 .tc dscr_default[TC],dscr_default
389
390 .section ".text"
391
9994a338
PM
392/*
393 * This routine switches between two different tasks. The process
394 * state of one is saved on its kernel stack. Then the state
395 * of the other is restored from its kernel stack. The memory
396 * management hardware is updated to the second process's state.
397 * Finally, we can return to the second process, via ret_from_except.
398 * On entry, r3 points to the THREAD for the current task, r4
399 * points to the THREAD for the new task.
400 *
401 * Note: there are two ways to get to the "going out" portion
402 * of this code; either by coming in via the entry (_switch)
403 * or via "fork" which must set up an environment equivalent
404 * to the "_switch" path. If you change this you'll have to change
405 * the fork code also.
406 *
407 * The code which creates the new task context is in 'copy_thread'
2ef9481e 408 * in arch/powerpc/kernel/process.c
9994a338
PM
409 */
410 .align 7
411_GLOBAL(_switch)
412 mflr r0
413 std r0,16(r1)
414 stdu r1,-SWITCH_FRAME_SIZE(r1)
415 /* r3-r13 are caller saved -- Cort */
416 SAVE_8GPRS(14, r1)
417 SAVE_10GPRS(22, r1)
418 mflr r20 /* Return to switch caller */
419 mfmsr r22
420 li r0, MSR_FP
ce48b210
MN
421#ifdef CONFIG_VSX
422BEGIN_FTR_SECTION
423 oris r0,r0,MSR_VSX@h /* Disable VSX */
424END_FTR_SECTION_IFSET(CPU_FTR_VSX)
425#endif /* CONFIG_VSX */
9994a338
PM
426#ifdef CONFIG_ALTIVEC
427BEGIN_FTR_SECTION
428 oris r0,r0,MSR_VEC@h /* Disable altivec */
429 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
430 std r24,THREAD_VRSAVE(r3)
431END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
432#endif /* CONFIG_ALTIVEC */
efcac658
AK
433#ifdef CONFIG_PPC64
434BEGIN_FTR_SECTION
435 mfspr r25,SPRN_DSCR
436 std r25,THREAD_DSCR(r3)
437END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
438#endif
9994a338
PM
439 and. r0,r0,r22
440 beq+ 1f
441 andc r22,r22,r0
2d27cfd3 442 MTMSRD(r22)
9994a338
PM
443 isync
4441: std r20,_NIP(r1)
445 mfcr r23
446 std r23,_CCR(r1)
447 std r1,KSP(r3) /* Set old stack pointer */
448
2468dcf6
IM
449#ifdef CONFIG_PPC_BOOK3S_64
450BEGIN_FTR_SECTION
451 /*
452 * Back up the TAR across context switches. Note that the TAR is not
453 * available for use in the kernel. (To provide this, the TAR should
454 * be backed up/restored on exception entry/exit instead, and be in
455 * pt_regs. FIXME, this should be in pt_regs anyway (for debug).)
456 */
457 mfspr r0,SPRN_TAR
458 std r0,THREAD_TAR(r3)
9353374b
ME
459
460 /* Event based branch registers */
461 mfspr r0, SPRN_BESCR
462 std r0, THREAD_BESCR(r3)
463 mfspr r0, SPRN_EBBHR
464 std r0, THREAD_EBBHR(r3)
465 mfspr r0, SPRN_EBBRR
466 std r0, THREAD_EBBRR(r3)
1de2bd4e 467END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2468dcf6
IM
468#endif
469
9994a338
PM
470#ifdef CONFIG_SMP
471 /* We need a sync somewhere here to make sure that if the
472 * previous task gets rescheduled on another CPU, it sees all
473 * stores it has performed on this one.
474 */
475 sync
476#endif /* CONFIG_SMP */
477
f89451fb
AB
478 /*
479 * If we optimise away the clear of the reservation in system
480 * calls because we know the CPU tracks the address of the
481 * reservation, then we need to clear it here to cover the
482 * case that the kernel context switch path has no larx
483 * instructions.
484 */
485BEGIN_FTR_SECTION
486 ldarx r6,0,r1
487END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
488
9994a338
PM
489 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
490 std r6,PACACURRENT(r13) /* Set new 'current' */
491
492 ld r8,KSP(r4) /* new stack pointer */
2d27cfd3 493#ifdef CONFIG_PPC_BOOK3S
1189be65 494BEGIN_FTR_SECTION
c230328d 495 BEGIN_FTR_SECTION_NESTED(95)
9994a338
PM
496 clrrdi r6,r8,28 /* get its ESID */
497 clrrdi r9,r1,28 /* get current sp ESID */
c230328d 498 FTR_SECTION_ELSE_NESTED(95)
1189be65
PM
499 clrrdi r6,r8,40 /* get its 1T ESID */
500 clrrdi r9,r1,40 /* get current sp 1T ESID */
44ae3ab3 501 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
c230328d
ME
502FTR_SECTION_ELSE
503 b 2f
44ae3ab3 504ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
9994a338
PM
505 clrldi. r0,r6,2 /* is new ESID c00000000? */
506 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
507 cror eq,4*cr1+eq,eq
508 beq 2f /* if yes, don't slbie it */
509
510 /* Bolt in the new stack SLB entry */
511 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
512 oris r0,r6,(SLB_ESID_V)@h
513 ori r0,r0,(SLB_NUM_BOLTED-1)@l
1189be65
PM
514BEGIN_FTR_SECTION
515 li r9,MMU_SEGSIZE_1T /* insert B field */
516 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
517 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
44ae3ab3 518END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
2f6093c8 519
00efee7d
MN
520 /* Update the last bolted SLB. No write barriers are needed
521 * here, provided we only update the current CPU's SLB shadow
522 * buffer.
523 */
2f6093c8 524 ld r9,PACA_SLBSHADOWPTR(r13)
11a27ad7
MN
525 li r12,0
526 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
527 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
528 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
2f6093c8 529
44ae3ab3 530 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
f66bce5e
OJ
531 * we have 1TB segments, the only CPUs known to have the errata
532 * only support less than 1TB of system memory and we'll never
533 * actually hit this code path.
534 */
535
9994a338
PM
536 slbie r6
537 slbie r6 /* Workaround POWER5 < DD2.1 issue */
538 slbmte r7,r0
539 isync
9994a338 5402:
2d27cfd3
BH
541#endif /* !CONFIG_PPC_BOOK3S */
542
9778b696 543 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
9994a338
PM
544 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
545 because we don't need to leave the 288-byte ABI gap at the
546 top of the kernel stack. */
547 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
548
549 mr r1,r8 /* start using new stack pointer */
550 std r7,PACAKSAVE(r13)
551
2468dcf6
IM
552#ifdef CONFIG_PPC_BOOK3S_64
553BEGIN_FTR_SECTION
9353374b
ME
554 /* Event based branch registers */
555 ld r0, THREAD_BESCR(r4)
556 mtspr SPRN_BESCR, r0
557 ld r0, THREAD_EBBHR(r4)
558 mtspr SPRN_EBBHR, r0
559 ld r0, THREAD_EBBRR(r4)
560 mtspr SPRN_EBBRR, r0
561
2468dcf6
IM
562 ld r0,THREAD_TAR(r4)
563 mtspr SPRN_TAR,r0
1de2bd4e 564END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2468dcf6
IM
565#endif
566
9994a338
PM
567#ifdef CONFIG_ALTIVEC
568BEGIN_FTR_SECTION
569 ld r0,THREAD_VRSAVE(r4)
570 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
571END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
572#endif /* CONFIG_ALTIVEC */
efcac658
AK
573#ifdef CONFIG_PPC64
574BEGIN_FTR_SECTION
71433285
AB
575 lwz r6,THREAD_DSCR_INHERIT(r4)
576 ld r7,DSCR_DEFAULT@toc(2)
efcac658 577 ld r0,THREAD_DSCR(r4)
71433285
AB
578 cmpwi r6,0
579 bne 1f
580 ld r0,0(r7)
5811: cmpd r0,r25
582 beq 2f
efcac658 583 mtspr SPRN_DSCR,r0
71433285 5842:
efcac658
AK
585END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
586#endif
9994a338 587
71433285
AB
588 ld r6,_CCR(r1)
589 mtcrf 0xFF,r6
590
9994a338
PM
591 /* r3-r13 are destroyed -- Cort */
592 REST_8GPRS(14, r1)
593 REST_10GPRS(22, r1)
594
595 /* convert old thread to its task_struct for return value */
596 addi r3,r3,-THREAD
597 ld r7,_NIP(r1) /* Return to _switch caller in new task */
598 mtlr r7
599 addi r1,r1,SWITCH_FRAME_SIZE
600 blr
601
602 .align 7
603_GLOBAL(ret_from_except)
604 ld r11,_TRAP(r1)
605 andi. r0,r11,1
606 bne .ret_from_except_lite
607 REST_NVGPRS(r1)
608
609_GLOBAL(ret_from_except_lite)
610 /*
611 * Disable interrupts so that current_thread_info()->flags
612 * can't change between when we test it and when we return
613 * from the interrupt.
614 */
2d27cfd3
BH
615#ifdef CONFIG_PPC_BOOK3E
616 wrteei 0
617#else
d9ada91a
BH
618 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
619 mtmsrd r10,1 /* Update machine state */
2d27cfd3 620#endif /* CONFIG_PPC_BOOK3E */
9994a338 621
9778b696 622 CURRENT_THREAD_INFO(r9, r1)
9994a338
PM
623 ld r3,_MSR(r1)
624 ld r4,TI_FLAGS(r9)
9994a338 625 andi. r3,r3,MSR_PR
c58ce2b1 626 beq resume_kernel
9994a338
PM
627
628 /* Check current_thread_info()->flags */
c58ce2b1
TC
629 andi. r0,r4,_TIF_USER_WORK_MASK
630 beq restore
631
632 andi. r0,r4,_TIF_NEED_RESCHED
633 beq 1f
634 bl .restore_interrupts
635 bl .schedule
636 b .ret_from_except_lite
637
6381: bl .save_nvgprs
639 bl .restore_interrupts
640 addi r3,r1,STACK_FRAME_OVERHEAD
641 bl .do_notify_resume
642 b .ret_from_except
643
644resume_kernel:
a9c4e541
TC
645 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
646 CURRENT_THREAD_INFO(r9, r1)
647 ld r8,TI_FLAGS(r9)
648 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
649 beq+ 1f
650
651 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
652
653 lwz r3,GPR1(r1)
654 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
655 mr r4,r1 /* src: current exception frame */
656 mr r1,r3 /* Reroute the trampoline frame to r1 */
657
658 /* Copy from the original to the trampoline. */
659 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
660 li r6,0 /* start offset: 0 */
661 mtctr r5
6622: ldx r0,r6,r4
663 stdx r0,r6,r3
664 addi r6,r6,8
665 bdnz 2b
666
667 /* Do real store operation to complete stwu */
668 lwz r5,GPR1(r1)
669 std r8,0(r5)
670
671 /* Clear _TIF_EMULATE_STACK_STORE flag */
672 lis r11,_TIF_EMULATE_STACK_STORE@h
673 addi r5,r9,TI_FLAGS
d8b92292 6740: ldarx r4,0,r5
a9c4e541
TC
675 andc r4,r4,r11
676 stdcx. r4,0,r5
677 bne- 0b
6781:
679
c58ce2b1
TC
680#ifdef CONFIG_PREEMPT
681 /* Check if we need to preempt */
682 andi. r0,r4,_TIF_NEED_RESCHED
683 beq+ restore
684 /* Check that preempt_count() == 0 and interrupts are enabled */
685 lwz r8,TI_PREEMPT(r9)
686 cmpwi cr1,r8,0
687 ld r0,SOFTE(r1)
688 cmpdi r0,0
689 crandc eq,cr1*4+eq,eq
690 bne restore
691
692 /*
693 * Here we are preempting the current task. We want to make
694 * sure we are soft-disabled first
695 */
696 SOFT_DISABLE_INTS(r3,r4)
6971: bl .preempt_schedule_irq
698
699 /* Re-test flags and eventually loop */
9778b696 700 CURRENT_THREAD_INFO(r9, r1)
9994a338 701 ld r4,TI_FLAGS(r9)
c58ce2b1
TC
702 andi. r0,r4,_TIF_NEED_RESCHED
703 bne 1b
572177d7
TC
704
705 /*
706 * arch_local_irq_restore() from preempt_schedule_irq above may
707 * enable hard interrupt but we really should disable interrupts
708 * when we return from the interrupt, and so that we don't get
709 * interrupted after loading SRR0/1.
710 */
711#ifdef CONFIG_PPC_BOOK3E
712 wrteei 0
713#else
714 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
715 mtmsrd r10,1 /* Update machine state */
716#endif /* CONFIG_PPC_BOOK3E */
c58ce2b1 717#endif /* CONFIG_PREEMPT */
9994a338 718
7230c564
BH
719 .globl fast_exc_return_irq
720fast_exc_return_irq:
9994a338 721restore:
7230c564 722 /*
7c0482e3
BH
723 * This is the main kernel exit path. First we check if we
724 * are about to re-enable interrupts
7230c564 725 */
01f3880d 726 ld r5,SOFTE(r1)
7230c564 727 lbz r6,PACASOFTIRQEN(r13)
7c0482e3
BH
728 cmpwi cr0,r5,0
729 beq restore_irq_off
7230c564 730
7c0482e3
BH
731 /* We are enabling, were we already enabled ? Yes, just return */
732 cmpwi cr0,r6,1
733 beq cr0,do_restore
9994a338 734
7c0482e3 735 /*
7230c564
BH
736 * We are about to soft-enable interrupts (we are hard disabled
737 * at this point). We check if there's anything that needs to
738 * be replayed first.
739 */
740 lbz r0,PACAIRQHAPPENED(r13)
741 cmpwi cr0,r0,0
742 bne- restore_check_irq_replay
e56a6e20 743
7230c564
BH
744 /*
745 * Get here when nothing happened while soft-disabled, just
746 * soft-enable and move-on. We will hard-enable as a side
747 * effect of rfi
748 */
749restore_no_replay:
750 TRACE_ENABLE_INTS
751 li r0,1
752 stb r0,PACASOFTIRQEN(r13);
753
754 /*
755 * Final return path. BookE is handled in a different file
756 */
7c0482e3 757do_restore:
2d27cfd3
BH
758#ifdef CONFIG_PPC_BOOK3E
759 b .exception_return_book3e
760#else
7230c564
BH
761 /*
762 * Clear the reservation. If we know the CPU tracks the address of
763 * the reservation then we can potentially save some cycles and use
764 * a larx. On POWER6 and POWER7 this is significantly faster.
765 */
766BEGIN_FTR_SECTION
767 stdcx. r0,0,r1 /* to clear the reservation */
768FTR_SECTION_ELSE
769 ldarx r4,0,r1
770ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
771
772 /*
773 * Some code path such as load_up_fpu or altivec return directly
774 * here. They run entirely hard disabled and do not alter the
775 * interrupt state. They also don't use lwarx/stwcx. and thus
776 * are known not to leave dangling reservations.
777 */
778 .globl fast_exception_return
779fast_exception_return:
780 ld r3,_MSR(r1)
e56a6e20
PM
781 ld r4,_CTR(r1)
782 ld r0,_LINK(r1)
783 mtctr r4
784 mtlr r0
785 ld r4,_XER(r1)
786 mtspr SPRN_XER,r4
787
788 REST_8GPRS(5, r1)
789
9994a338
PM
790 andi. r0,r3,MSR_RI
791 beq- unrecov_restore
792
e56a6e20
PM
793 /*
794 * Clear RI before restoring r13. If we are returning to
795 * userspace and we take an exception after restoring r13,
796 * we end up corrupting the userspace r13 value.
797 */
d9ada91a
BH
798 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
799 andc r4,r4,r0 /* r0 contains MSR_RI here */
e56a6e20 800 mtmsrd r4,1
9994a338 801
afc07701
MN
802#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
803 /* TM debug */
804 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
805#endif
9994a338
PM
806 /*
807 * r13 is our per cpu area, only restore it if we are returning to
7230c564
BH
808 * userspace the value stored in the stack frame may belong to
809 * another CPU.
9994a338 810 */
e56a6e20 811 andi. r0,r3,MSR_PR
9994a338 812 beq 1f
e56a6e20 813 ACCOUNT_CPU_USER_EXIT(r2, r4)
44e9309f 814 RESTORE_PPR(r2, r4)
9994a338
PM
815 REST_GPR(13, r1)
8161:
e56a6e20 817 mtspr SPRN_SRR1,r3
9994a338
PM
818
819 ld r2,_CCR(r1)
820 mtcrf 0xFF,r2
821 ld r2,_NIP(r1)
822 mtspr SPRN_SRR0,r2
823
824 ld r0,GPR0(r1)
825 ld r2,GPR2(r1)
826 ld r3,GPR3(r1)
827 ld r4,GPR4(r1)
828 ld r1,GPR1(r1)
829
830 rfid
831 b . /* prevent speculative execution */
832
2d27cfd3
BH
833#endif /* CONFIG_PPC_BOOK3E */
834
7c0482e3
BH
835 /*
836 * We are returning to a context with interrupts soft disabled.
837 *
838 * However, we may also about to hard enable, so we need to
839 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
840 * or that bit can get out of sync and bad things will happen
841 */
842restore_irq_off:
843 ld r3,_MSR(r1)
844 lbz r7,PACAIRQHAPPENED(r13)
845 andi. r0,r3,MSR_EE
846 beq 1f
847 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
848 stb r7,PACAIRQHAPPENED(r13)
8491: li r0,0
850 stb r0,PACASOFTIRQEN(r13);
851 TRACE_DISABLE_INTS
852 b do_restore
853
7230c564
BH
854 /*
855 * Something did happen, check if a re-emit is needed
856 * (this also clears paca->irq_happened)
857 */
858restore_check_irq_replay:
859 /* XXX: We could implement a fast path here where we check
860 * for irq_happened being just 0x01, in which case we can
861 * clear it and return. That means that we would potentially
862 * miss a decrementer having wrapped all the way around.
863 *
864 * Still, this might be useful for things like hash_page
865 */
866 bl .__check_irq_replay
867 cmpwi cr0,r3,0
868 beq restore_no_replay
869
870 /*
871 * We need to re-emit an interrupt. We do so by re-using our
872 * existing exception frame. We first change the trap value,
873 * but we need to ensure we preserve the low nibble of it
874 */
875 ld r4,_TRAP(r1)
876 clrldi r4,r4,60
877 or r4,r4,r3
878 std r4,_TRAP(r1)
879
880 /*
881 * Then find the right handler and call it. Interrupts are
882 * still soft-disabled and we keep them that way.
883 */
884 cmpwi cr0,r3,0x500
885 bne 1f
886 addi r3,r1,STACK_FRAME_OVERHEAD;
887 bl .do_IRQ
888 b .ret_from_except
8891: cmpwi cr0,r3,0x900
890 bne 1f
891 addi r3,r1,STACK_FRAME_OVERHEAD;
892 bl .timer_interrupt
893 b .ret_from_except
fe9e1d54
IM
894#ifdef CONFIG_PPC_DOORBELL
8951:
7230c564 896#ifdef CONFIG_PPC_BOOK3E
fe9e1d54
IM
897 cmpwi cr0,r3,0x280
898#else
899 BEGIN_FTR_SECTION
900 cmpwi cr0,r3,0xe80
901 FTR_SECTION_ELSE
902 cmpwi cr0,r3,0xa00
903 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
904#endif /* CONFIG_PPC_BOOK3E */
7230c564
BH
905 bne 1f
906 addi r3,r1,STACK_FRAME_OVERHEAD;
907 bl .doorbell_exception
908 b .ret_from_except
fe9e1d54 909#endif /* CONFIG_PPC_DOORBELL */
7230c564
BH
9101: b .ret_from_except /* What else to do here ? */
911
9994a338
PM
912unrecov_restore:
913 addi r3,r1,STACK_FRAME_OVERHEAD
914 bl .unrecoverable_exception
915 b unrecov_restore
916
917#ifdef CONFIG_PPC_RTAS
918/*
919 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
920 * called with the MMU off.
921 *
922 * In addition, we need to be in 32b mode, at least for now.
923 *
924 * Note: r3 is an input parameter to rtas, so don't trash it...
925 */
926_GLOBAL(enter_rtas)
927 mflr r0
928 std r0,16(r1)
929 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
930
931 /* Because RTAS is running in 32b mode, it clobbers the high order half
932 * of all registers that it saves. We therefore save those registers
933 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
934 */
935 SAVE_GPR(2, r1) /* Save the TOC */
936 SAVE_GPR(13, r1) /* Save paca */
937 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
938 SAVE_10GPRS(22, r1) /* ditto */
939
940 mfcr r4
941 std r4,_CCR(r1)
942 mfctr r5
943 std r5,_CTR(r1)
944 mfspr r6,SPRN_XER
945 std r6,_XER(r1)
946 mfdar r7
947 std r7,_DAR(r1)
948 mfdsisr r8
949 std r8,_DSISR(r1)
9994a338 950
9fe901d1
MK
951 /* Temporary workaround to clear CR until RTAS can be modified to
952 * ignore all bits.
953 */
954 li r0,0
955 mtcr r0
956
007d88d0 957#ifdef CONFIG_BUG
9994a338
PM
958 /* There is no way it is acceptable to get here with interrupts enabled,
959 * check it with the asm equivalent of WARN_ON
960 */
d04c56f7 961 lbz r0,PACASOFTIRQEN(r13)
9994a338 9621: tdnei r0,0
007d88d0
DW
963 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
964#endif
965
d04c56f7
PM
966 /* Hard-disable interrupts */
967 mfmsr r6
968 rldicl r7,r6,48,1
969 rotldi r7,r7,16
970 mtmsrd r7,1
971
9994a338
PM
972 /* Unfortunately, the stack pointer and the MSR are also clobbered,
973 * so they are saved in the PACA which allows us to restore
974 * our original state after RTAS returns.
975 */
976 std r1,PACAR1(r13)
977 std r6,PACASAVEDMSR(r13)
978
979 /* Setup our real return addr */
e58c3495
DG
980 LOAD_REG_ADDR(r4,.rtas_return_loc)
981 clrldi r4,r4,2 /* convert to realmode address */
9994a338
PM
982 mtlr r4
983
984 li r0,0
985 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
986 andc r0,r6,r0
987
988 li r9,1
989 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
44c9f3cc 990 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
9994a338 991 andc r6,r0,r9
9994a338
PM
992 sync /* disable interrupts so SRR0/1 */
993 mtmsrd r0 /* don't get trashed */
994
e58c3495 995 LOAD_REG_ADDR(r4, rtas)
9994a338
PM
996 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
997 ld r4,RTASBASE(r4) /* get the rtas->base value */
998
999 mtspr SPRN_SRR0,r5
1000 mtspr SPRN_SRR1,r6
1001 rfid
1002 b . /* prevent speculative execution */
1003
1004_STATIC(rtas_return_loc)
1005 /* relocation is off at this point */
2dd60d79 1006 GET_PACA(r4)
e58c3495 1007 clrldi r4,r4,2 /* convert to realmode address */
9994a338 1008
e31aa453
PM
1009 bcl 20,31,$+4
10100: mflr r3
1011 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
1012
9994a338
PM
1013 mfmsr r6
1014 li r0,MSR_RI
1015 andc r6,r6,r0
1016 sync
1017 mtmsrd r6
1018
1019 ld r1,PACAR1(r4) /* Restore our SP */
9994a338
PM
1020 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1021
1022 mtspr SPRN_SRR0,r3
1023 mtspr SPRN_SRR1,r4
1024 rfid
1025 b . /* prevent speculative execution */
1026
e31aa453
PM
1027 .align 3
10281: .llong .rtas_restore_regs
1029
9994a338
PM
1030_STATIC(rtas_restore_regs)
1031 /* relocation is on at this point */
1032 REST_GPR(2, r1) /* Restore the TOC */
1033 REST_GPR(13, r1) /* Restore paca */
1034 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1035 REST_10GPRS(22, r1) /* ditto */
1036
2dd60d79 1037 GET_PACA(r13)
9994a338
PM
1038
1039 ld r4,_CCR(r1)
1040 mtcr r4
1041 ld r5,_CTR(r1)
1042 mtctr r5
1043 ld r6,_XER(r1)
1044 mtspr SPRN_XER,r6
1045 ld r7,_DAR(r1)
1046 mtdar r7
1047 ld r8,_DSISR(r1)
1048 mtdsisr r8
9994a338
PM
1049
1050 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1051 ld r0,16(r1) /* get return address */
1052
1053 mtlr r0
1054 blr /* return to caller */
1055
1056#endif /* CONFIG_PPC_RTAS */
1057
9994a338
PM
1058_GLOBAL(enter_prom)
1059 mflr r0
1060 std r0,16(r1)
1061 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1062
1063 /* Because PROM is running in 32b mode, it clobbers the high order half
1064 * of all registers that it saves. We therefore save those registers
1065 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1066 */
6c171994 1067 SAVE_GPR(2, r1)
9994a338
PM
1068 SAVE_GPR(13, r1)
1069 SAVE_8GPRS(14, r1)
1070 SAVE_10GPRS(22, r1)
6c171994 1071 mfcr r10
9994a338 1072 mfmsr r11
6c171994 1073 std r10,_CCR(r1)
9994a338
PM
1074 std r11,_MSR(r1)
1075
1076 /* Get the PROM entrypoint */
6c171994 1077 mtlr r4
9994a338
PM
1078
1079 /* Switch MSR to 32 bits mode
1080 */
2d27cfd3
BH
1081#ifdef CONFIG_PPC_BOOK3E
1082 rlwinm r11,r11,0,1,31
1083 mtmsr r11
1084#else /* CONFIG_PPC_BOOK3E */
9994a338
PM
1085 mfmsr r11
1086 li r12,1
1087 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1088 andc r11,r11,r12
1089 li r12,1
1090 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1091 andc r11,r11,r12
1092 mtmsrd r11
2d27cfd3 1093#endif /* CONFIG_PPC_BOOK3E */
9994a338
PM
1094 isync
1095
6c171994 1096 /* Enter PROM here... */
9994a338
PM
1097 blrl
1098
1099 /* Just make sure that r1 top 32 bits didn't get
1100 * corrupt by OF
1101 */
1102 rldicl r1,r1,0,32
1103
1104 /* Restore the MSR (back to 64 bits) */
1105 ld r0,_MSR(r1)
6c171994 1106 MTMSRD(r0)
9994a338
PM
1107 isync
1108
1109 /* Restore other registers */
1110 REST_GPR(2, r1)
1111 REST_GPR(13, r1)
1112 REST_8GPRS(14, r1)
1113 REST_10GPRS(22, r1)
1114 ld r4,_CCR(r1)
1115 mtcr r4
9994a338
PM
1116
1117 addi r1,r1,PROM_FRAME_SIZE
1118 ld r0,16(r1)
1119 mtlr r0
1120 blr
4e491d14 1121
606576ce 1122#ifdef CONFIG_FUNCTION_TRACER
4e491d14
SR
1123#ifdef CONFIG_DYNAMIC_FTRACE
1124_GLOBAL(mcount)
1125_GLOBAL(_mcount)
4e491d14
SR
1126 blr
1127
1128_GLOBAL(ftrace_caller)
1129 /* Taken from output of objdump from lib64/glibc */
1130 mflr r3
1131 ld r11, 0(r1)
1132 stdu r1, -112(r1)
1133 std r3, 128(r1)
1134 ld r4, 16(r11)
395a59d0 1135 subi r3, r3, MCOUNT_INSN_SIZE
4e491d14
SR
1136.globl ftrace_call
1137ftrace_call:
1138 bl ftrace_stub
1139 nop
46542888
SR
1140#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1141.globl ftrace_graph_call
1142ftrace_graph_call:
1143 b ftrace_graph_stub
1144_GLOBAL(ftrace_graph_stub)
1145#endif
4e491d14
SR
1146 ld r0, 128(r1)
1147 mtlr r0
1148 addi r1, r1, 112
1149_GLOBAL(ftrace_stub)
1150 blr
1151#else
1152_GLOBAL(mcount)
1153 blr
1154
1155_GLOBAL(_mcount)
1156 /* Taken from output of objdump from lib64/glibc */
1157 mflr r3
1158 ld r11, 0(r1)
1159 stdu r1, -112(r1)
1160 std r3, 128(r1)
1161 ld r4, 16(r11)
1162
395a59d0 1163 subi r3, r3, MCOUNT_INSN_SIZE
4e491d14
SR
1164 LOAD_REG_ADDR(r5,ftrace_trace_function)
1165 ld r5,0(r5)
1166 ld r5,0(r5)
1167 mtctr r5
1168 bctrl
4e491d14 1169 nop
6794c782
SR
1170
1171
1172#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1173 b ftrace_graph_caller
1174#endif
4e491d14
SR
1175 ld r0, 128(r1)
1176 mtlr r0
1177 addi r1, r1, 112
1178_GLOBAL(ftrace_stub)
1179 blr
1180
6794c782
SR
1181#endif /* CONFIG_DYNAMIC_FTRACE */
1182
1183#ifdef CONFIG_FUNCTION_GRAPH_TRACER
46542888 1184_GLOBAL(ftrace_graph_caller)
6794c782
SR
1185 /* load r4 with local address */
1186 ld r4, 128(r1)
1187 subi r4, r4, MCOUNT_INSN_SIZE
1188
1189 /* get the parent address */
1190 ld r11, 112(r1)
1191 addi r3, r11, 16
1192
1193 bl .prepare_ftrace_return
1194 nop
1195
1196 ld r0, 128(r1)
1197 mtlr r0
1198 addi r1, r1, 112
1199 blr
1200
1201_GLOBAL(return_to_handler)
bb725340
SR
1202 /* need to save return values */
1203 std r4, -24(r1)
1204 std r3, -16(r1)
1205 std r31, -8(r1)
1206 mr r31, r1
1207 stdu r1, -112(r1)
1208
1209 bl .ftrace_return_to_handler
1210 nop
1211
1212 /* return value has real return address */
1213 mtlr r3
1214
1215 ld r1, 0(r1)
1216 ld r4, -24(r1)
1217 ld r3, -16(r1)
1218 ld r31, -8(r1)
1219
1220 /* Jump back to real return address */
1221 blr
1222
1223_GLOBAL(mod_return_to_handler)
6794c782
SR
1224 /* need to save return values */
1225 std r4, -32(r1)
1226 std r3, -24(r1)
1227 /* save TOC */
1228 std r2, -16(r1)
1229 std r31, -8(r1)
1230 mr r31, r1
1231 stdu r1, -112(r1)
1232
bb725340
SR
1233 /*
1234 * We are in a module using the module's TOC.
1235 * Switch to our TOC to run inside the core kernel.
1236 */
be10ab10 1237 ld r2, PACATOC(r13)
6794c782
SR
1238
1239 bl .ftrace_return_to_handler
1240 nop
1241
1242 /* return value has real return address */
1243 mtlr r3
1244
1245 ld r1, 0(r1)
1246 ld r4, -32(r1)
1247 ld r3, -24(r1)
1248 ld r2, -16(r1)
1249 ld r31, -8(r1)
1250
1251 /* Jump back to real return address */
1252 blr
1253#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1254#endif /* CONFIG_FUNCTION_TRACER */