]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/powerpc/kernel/ftrace.c
2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
8 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
13 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
15 #include <linux/spinlock.h>
16 #include <linux/hardirq.h>
17 #include <linux/uaccess.h>
18 #include <linux/module.h>
19 #include <linux/ftrace.h>
20 #include <linux/percpu.h>
21 #include <linux/init.h>
22 #include <linux/list.h>
24 #include <asm/cacheflush.h>
25 #include <asm/code-patching.h>
26 #include <asm/ftrace.h>
27 #include <asm/syscall.h>
30 #ifdef CONFIG_DYNAMIC_FTRACE
32 ftrace_call_replace(unsigned long ip
, unsigned long addr
, int link
)
36 addr
= ppc_function_entry((void *)addr
);
38 /* if (link) set op to 'bl' else 'b' */
39 op
= create_branch((unsigned int *)ip
, addr
, link
? 1 : 0);
45 ftrace_modify_code(unsigned long ip
, unsigned int old
, unsigned int new)
47 unsigned int replaced
;
51 * We are paranoid about modifying text, as if a bug was to happen, it
52 * could cause us to read or write to someplace that could cause harm.
53 * Carefully read and modify the code with probe_kernel_*(), and make
54 * sure what we read is what we expected it to be before modifying it.
57 /* read the text we want to modify */
58 if (probe_kernel_read(&replaced
, (void *)ip
, MCOUNT_INSN_SIZE
))
61 /* Make sure it is what we expect it to be */
62 if (replaced
!= old
) {
63 pr_err("%p: replaced (%#x) != old (%#x)",
64 (void *)ip
, replaced
, old
);
68 /* replace the text with the new text */
69 if (patch_instruction((unsigned int *)ip
, new))
76 * Helper functions that are the same for both PPC64 and PPC32.
78 static int test_24bit_addr(unsigned long ip
, unsigned long addr
)
80 addr
= ppc_function_entry((void *)addr
);
82 /* use the create_branch to verify that this offset can be branched */
83 return create_branch((unsigned int *)ip
, addr
, 0);
88 static int is_bl_op(unsigned int op
)
90 return (op
& 0xfc000003) == 0x48000001;
93 static unsigned long find_bl_target(unsigned long ip
, unsigned int op
)
97 offset
= (op
& 0x03fffffc);
99 if (offset
& 0x02000000)
100 offset
|= 0xfe000000;
102 return ip
+ (long)offset
;
107 __ftrace_make_nop(struct module
*mod
,
108 struct dyn_ftrace
*rec
, unsigned long addr
)
110 unsigned long entry
, ptr
, tramp
;
111 unsigned long ip
= rec
->ip
;
112 unsigned int op
, pop
;
114 /* read where this goes */
115 if (probe_kernel_read(&op
, (void *)ip
, sizeof(int))) {
116 pr_err("Fetching opcode failed.\n");
120 /* Make sure that that this is still a 24bit jump */
122 pr_err("Not expected bl: opcode is %x\n", op
);
126 /* lets find where the pointer goes */
127 tramp
= find_bl_target(ip
, op
);
129 pr_devel("ip:%lx jumps to %lx", ip
, tramp
);
131 if (module_trampoline_target(mod
, tramp
, &ptr
)) {
132 pr_err("Failed to get trampoline target\n");
136 pr_devel("trampoline target %lx", ptr
);
138 entry
= ppc_global_function_entry((void *)addr
);
139 /* This should match what was called */
141 pr_err("addr %lx does not match expected %lx\n", ptr
, entry
);
145 #ifdef CC_USING_MPROFILE_KERNEL
146 /* When using -mkernel_profile there is no load to jump over */
149 if (probe_kernel_read(&op
, (void *)(ip
- 4), 4)) {
150 pr_err("Fetching instruction at %lx failed.\n", ip
- 4);
154 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
155 if (op
!= PPC_INST_MFLR
&& op
!= PPC_INST_STD_LR
) {
156 pr_err("Unexpected instruction %08x around bl _mcount\n", op
);
161 * Our original call site looks like:
166 * Milton Miller pointed out that we can not simply nop the branch.
167 * If a task was preempted when calling a trace function, the nops
168 * will remove the way to restore the TOC in r2 and the r2 TOC will
171 * Use a b +8 to jump over the load.
174 pop
= PPC_INST_BRANCH
| 8; /* b +8 */
177 * Check what is in the next instruction. We can see ld r2,40(r1), but
178 * on first pass after boot we will see mflr r0.
180 if (probe_kernel_read(&op
, (void *)(ip
+4), MCOUNT_INSN_SIZE
)) {
181 pr_err("Fetching op failed.\n");
185 if (op
!= PPC_INST_LD_TOC
) {
186 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC
, op
);
189 #endif /* CC_USING_MPROFILE_KERNEL */
191 if (patch_instruction((unsigned int *)ip
, pop
)) {
192 pr_err("Patching NOP failed.\n");
201 __ftrace_make_nop(struct module
*mod
,
202 struct dyn_ftrace
*rec
, unsigned long addr
)
206 unsigned long ip
= rec
->ip
;
209 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
212 /* Make sure that that this is still a 24bit jump */
214 pr_err("Not expected bl: opcode is %x\n", op
);
218 /* lets find where the pointer goes */
219 tramp
= find_bl_target(ip
, op
);
222 * On PPC32 the trampoline looks like:
223 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha
224 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l
225 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12
226 * 0x4e, 0x80, 0x04, 0x20 bctr
229 pr_devel("ip:%lx jumps to %lx", ip
, tramp
);
231 /* Find where the trampoline jumps to */
232 if (probe_kernel_read(jmp
, (void *)tramp
, sizeof(jmp
))) {
233 pr_err("Failed to read %lx\n", tramp
);
237 pr_devel(" %08x %08x ", jmp
[0], jmp
[1]);
239 /* verify that this is what we expect it to be */
240 if (((jmp
[0] & 0xffff0000) != 0x3d800000) ||
241 ((jmp
[1] & 0xffff0000) != 0x398c0000) ||
242 (jmp
[2] != 0x7d8903a6) ||
243 (jmp
[3] != 0x4e800420)) {
244 pr_err("Not a trampoline\n");
248 tramp
= (jmp
[1] & 0xffff) |
249 ((jmp
[0] & 0xffff) << 16);
253 pr_devel(" %lx ", tramp
);
256 pr_err("Trampoline location %08lx does not match addr\n",
263 if (patch_instruction((unsigned int *)ip
, op
))
269 #endif /* CONFIG_MODULES */
271 int ftrace_make_nop(struct module
*mod
,
272 struct dyn_ftrace
*rec
, unsigned long addr
)
274 unsigned long ip
= rec
->ip
;
275 unsigned int old
, new;
278 * If the calling address is more that 24 bits away,
279 * then we had to use a trampoline to make the call.
280 * Otherwise just update the call site.
282 if (test_24bit_addr(ip
, addr
)) {
284 old
= ftrace_call_replace(ip
, addr
, 1);
286 return ftrace_modify_code(ip
, old
, new);
289 #ifdef CONFIG_MODULES
291 * Out of range jumps are called from modules.
292 * We should either already have a pointer to the module
293 * or it has been passed in.
295 if (!rec
->arch
.mod
) {
297 pr_err("No module loaded addr=%lx\n", addr
);
302 if (mod
!= rec
->arch
.mod
) {
303 pr_err("Record mod %p not equal to passed in mod %p\n",
307 /* nothing to do if mod == rec->arch.mod */
311 return __ftrace_make_nop(mod
, rec
, addr
);
313 /* We should not get here without modules */
315 #endif /* CONFIG_MODULES */
318 #ifdef CONFIG_MODULES
321 * Examine the existing instructions for __ftrace_make_call.
322 * They should effectively be a NOP, and follow formal constraints,
323 * depending on the ABI. Return false if they don't.
325 #ifndef CC_USING_MPROFILE_KERNEL
327 expected_nop_sequence(void *ip
, unsigned int op0
, unsigned int op1
)
335 * The load offset is different depending on the ABI. For simplicity
336 * just mask it out when doing the compare.
338 if ((op0
!= 0x48000008) || ((op1
& 0xffff0000) != 0xe8410000))
344 expected_nop_sequence(void *ip
, unsigned int op0
, unsigned int op1
)
346 /* look for patched "NOP" on ppc64 with -mprofile-kernel */
347 if (op0
!= PPC_INST_NOP
)
354 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
357 void *ip
= (void *)rec
->ip
;
359 /* read where this goes */
360 if (probe_kernel_read(op
, ip
, sizeof(op
)))
363 if (!expected_nop_sequence(ip
, op
[0], op
[1])) {
364 pr_err("Unexpected call sequence at %p: %x %x\n",
369 /* If we never set up a trampoline to ftrace_caller, then bail */
370 if (!rec
->arch
.mod
->arch
.tramp
) {
371 pr_err("No ftrace trampoline\n");
375 /* Ensure branch is within 24 bits */
376 if (!create_branch(ip
, rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
)) {
377 pr_err("Branch out of range\n");
381 if (patch_branch(ip
, rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
)) {
382 pr_err("REL24 out of range!\n");
389 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
390 int ftrace_modify_call(struct dyn_ftrace
*rec
, unsigned long old_addr
,
393 return ftrace_make_call(rec
, addr
);
397 #else /* !CONFIG_PPC64: */
399 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
402 unsigned long ip
= rec
->ip
;
404 /* read where this goes */
405 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
408 /* It should be pointing to a nop */
409 if (op
!= PPC_INST_NOP
) {
410 pr_err("Expected NOP but have %x\n", op
);
414 /* If we never set up a trampoline to ftrace_caller, then bail */
415 if (!rec
->arch
.mod
->arch
.tramp
) {
416 pr_err("No ftrace trampoline\n");
420 /* create the branch to the trampoline */
421 op
= create_branch((unsigned int *)ip
,
422 rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
);
424 pr_err("REL24 out of range!\n");
428 pr_devel("write to %lx\n", rec
->ip
);
430 if (patch_instruction((unsigned int *)ip
, op
))
435 #endif /* CONFIG_PPC64 */
436 #endif /* CONFIG_MODULES */
438 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
440 unsigned long ip
= rec
->ip
;
441 unsigned int old
, new;
444 * If the calling address is more that 24 bits away,
445 * then we had to use a trampoline to make the call.
446 * Otherwise just update the call site.
448 if (test_24bit_addr(ip
, addr
)) {
451 new = ftrace_call_replace(ip
, addr
, 1);
452 return ftrace_modify_code(ip
, old
, new);
455 #ifdef CONFIG_MODULES
457 * Out of range jumps are called from modules.
458 * Being that we are converting from nop, it had better
459 * already have a module defined.
461 if (!rec
->arch
.mod
) {
462 pr_err("No module loaded\n");
466 return __ftrace_make_call(rec
, addr
);
468 /* We should not get here without modules */
470 #endif /* CONFIG_MODULES */
473 int ftrace_update_ftrace_func(ftrace_func_t func
)
475 unsigned long ip
= (unsigned long)(&ftrace_call
);
476 unsigned int old
, new;
479 old
= *(unsigned int *)&ftrace_call
;
480 new = ftrace_call_replace(ip
, (unsigned long)func
, 1);
481 ret
= ftrace_modify_code(ip
, old
, new);
486 static int __ftrace_replace_code(struct dyn_ftrace
*rec
, int enable
)
488 unsigned long ftrace_addr
= (unsigned long)FTRACE_ADDR
;
491 ret
= ftrace_update_record(rec
, enable
);
494 case FTRACE_UPDATE_IGNORE
:
496 case FTRACE_UPDATE_MAKE_CALL
:
497 return ftrace_make_call(rec
, ftrace_addr
);
498 case FTRACE_UPDATE_MAKE_NOP
:
499 return ftrace_make_nop(NULL
, rec
, ftrace_addr
);
505 void ftrace_replace_code(int enable
)
507 struct ftrace_rec_iter
*iter
;
508 struct dyn_ftrace
*rec
;
511 for (iter
= ftrace_rec_iter_start(); iter
;
512 iter
= ftrace_rec_iter_next(iter
)) {
513 rec
= ftrace_rec_iter_record(iter
);
514 ret
= __ftrace_replace_code(rec
, enable
);
516 ftrace_bug(ret
, rec
);
523 * Use the default ftrace_modify_all_code, but without
526 void arch_ftrace_update_code(int command
)
528 ftrace_modify_all_code(command
);
531 int __init
ftrace_dyn_arch_init(void)
535 #endif /* CONFIG_DYNAMIC_FTRACE */
537 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
539 #ifdef CONFIG_DYNAMIC_FTRACE
540 extern void ftrace_graph_call(void);
541 extern void ftrace_graph_stub(void);
543 int ftrace_enable_ftrace_graph_caller(void)
545 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
546 unsigned long addr
= (unsigned long)(&ftrace_graph_caller
);
547 unsigned long stub
= (unsigned long)(&ftrace_graph_stub
);
548 unsigned int old
, new;
550 old
= ftrace_call_replace(ip
, stub
, 0);
551 new = ftrace_call_replace(ip
, addr
, 0);
553 return ftrace_modify_code(ip
, old
, new);
556 int ftrace_disable_ftrace_graph_caller(void)
558 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
559 unsigned long addr
= (unsigned long)(&ftrace_graph_caller
);
560 unsigned long stub
= (unsigned long)(&ftrace_graph_stub
);
561 unsigned int old
, new;
563 old
= ftrace_call_replace(ip
, addr
, 0);
564 new = ftrace_call_replace(ip
, stub
, 0);
566 return ftrace_modify_code(ip
, old
, new);
568 #endif /* CONFIG_DYNAMIC_FTRACE */
571 * Hook the return address and push it in the stack of return addrs
572 * in current thread info. Return the address we want to divert to.
574 unsigned long prepare_ftrace_return(unsigned long parent
, unsigned long ip
)
576 struct ftrace_graph_ent trace
;
577 unsigned long return_hooker
;
579 if (unlikely(ftrace_graph_is_dead()))
582 if (unlikely(atomic_read(¤t
->tracing_graph_pause
)))
585 return_hooker
= ppc_function_entry(return_to_handler
);
588 trace
.depth
= current
->curr_ret_stack
+ 1;
590 /* Only trace if the calling function expects to */
591 if (!ftrace_graph_entry(&trace
))
594 if (ftrace_push_return_trace(parent
, ip
, &trace
.depth
, 0,
598 parent
= return_hooker
;
602 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
604 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
605 unsigned long __init
arch_syscall_addr(int nr
)
607 return sys_call_table
[nr
*2];
609 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
611 #ifdef PPC64_ELF_ABI_v1
612 char *arch_ftrace_match_adjust(char *str
, const char *search
)
614 if (str
[0] == '.' && search
[0] != '.')
619 #endif /* PPC64_ELF_ABI_v1 */