]>
git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - arch/powerpc/kernel/trace/ftrace.c
1 // SPDX-License-Identifier: GPL-2.0
3 * Code for replacing ftrace calls with jumps.
5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
9 * Added function graph tracer code, taken from x86 that was written
10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
16 #include <linux/spinlock.h>
17 #include <linux/hardirq.h>
18 #include <linux/uaccess.h>
19 #include <linux/module.h>
20 #include <linux/ftrace.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/list.h>
25 #include <asm/asm-prototypes.h>
26 #include <asm/cacheflush.h>
27 #include <asm/code-patching.h>
28 #include <asm/ftrace.h>
29 #include <asm/syscall.h>
32 #ifdef CONFIG_DYNAMIC_FTRACE
34 ftrace_call_replace(unsigned long ip
, unsigned long addr
, int link
)
38 addr
= ppc_function_entry((void *)addr
);
40 /* if (link) set op to 'bl' else 'b' */
41 op
= create_branch((unsigned int *)ip
, addr
, link
? 1 : 0);
47 ftrace_modify_code(unsigned long ip
, unsigned int old
, unsigned int new)
49 unsigned int replaced
;
53 * We are paranoid about modifying text, as if a bug was to happen, it
54 * could cause us to read or write to someplace that could cause harm.
55 * Carefully read and modify the code with probe_kernel_*(), and make
56 * sure what we read is what we expected it to be before modifying it.
59 /* read the text we want to modify */
60 if (probe_kernel_read(&replaced
, (void *)ip
, MCOUNT_INSN_SIZE
))
63 /* Make sure it is what we expect it to be */
64 if (replaced
!= old
) {
65 pr_err("%p: replaced (%#x) != old (%#x)",
66 (void *)ip
, replaced
, old
);
70 /* replace the text with the new text */
71 if (patch_instruction((unsigned int *)ip
, new))
78 * Helper functions that are the same for both PPC64 and PPC32.
80 static int test_24bit_addr(unsigned long ip
, unsigned long addr
)
82 addr
= ppc_function_entry((void *)addr
);
84 /* use the create_branch to verify that this offset can be branched */
85 return create_branch((unsigned int *)ip
, addr
, 0);
90 static int is_bl_op(unsigned int op
)
92 return (op
& 0xfc000003) == 0x48000001;
95 static unsigned long find_bl_target(unsigned long ip
, unsigned int op
)
99 offset
= (op
& 0x03fffffc);
101 if (offset
& 0x02000000)
102 offset
|= 0xfe000000;
104 return ip
+ (long)offset
;
109 __ftrace_make_nop(struct module
*mod
,
110 struct dyn_ftrace
*rec
, unsigned long addr
)
112 unsigned long entry
, ptr
, tramp
;
113 unsigned long ip
= rec
->ip
;
114 unsigned int op
, pop
;
116 /* read where this goes */
117 if (probe_kernel_read(&op
, (void *)ip
, sizeof(int))) {
118 pr_err("Fetching opcode failed.\n");
122 /* Make sure that that this is still a 24bit jump */
124 pr_err("Not expected bl: opcode is %x\n", op
);
128 /* lets find where the pointer goes */
129 tramp
= find_bl_target(ip
, op
);
131 pr_devel("ip:%lx jumps to %lx", ip
, tramp
);
133 if (module_trampoline_target(mod
, tramp
, &ptr
)) {
134 pr_err("Failed to get trampoline target\n");
138 pr_devel("trampoline target %lx", ptr
);
140 entry
= ppc_global_function_entry((void *)addr
);
141 /* This should match what was called */
143 pr_err("addr %lx does not match expected %lx\n", ptr
, entry
);
147 #ifdef CC_USING_MPROFILE_KERNEL
148 /* When using -mkernel_profile there is no load to jump over */
151 if (probe_kernel_read(&op
, (void *)(ip
- 4), 4)) {
152 pr_err("Fetching instruction at %lx failed.\n", ip
- 4);
156 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
157 if (op
!= PPC_INST_MFLR
&& op
!= PPC_INST_STD_LR
) {
158 pr_err("Unexpected instruction %08x around bl _mcount\n", op
);
163 * Our original call site looks like:
168 * Milton Miller pointed out that we can not simply nop the branch.
169 * If a task was preempted when calling a trace function, the nops
170 * will remove the way to restore the TOC in r2 and the r2 TOC will
173 * Use a b +8 to jump over the load.
176 pop
= PPC_INST_BRANCH
| 8; /* b +8 */
179 * Check what is in the next instruction. We can see ld r2,40(r1), but
180 * on first pass after boot we will see mflr r0.
182 if (probe_kernel_read(&op
, (void *)(ip
+4), MCOUNT_INSN_SIZE
)) {
183 pr_err("Fetching op failed.\n");
187 if (op
!= PPC_INST_LD_TOC
) {
188 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC
, op
);
191 #endif /* CC_USING_MPROFILE_KERNEL */
193 if (patch_instruction((unsigned int *)ip
, pop
)) {
194 pr_err("Patching NOP failed.\n");
203 __ftrace_make_nop(struct module
*mod
,
204 struct dyn_ftrace
*rec
, unsigned long addr
)
208 unsigned long ip
= rec
->ip
;
211 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
214 /* Make sure that that this is still a 24bit jump */
216 pr_err("Not expected bl: opcode is %x\n", op
);
220 /* lets find where the pointer goes */
221 tramp
= find_bl_target(ip
, op
);
224 * On PPC32 the trampoline looks like:
225 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha
226 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l
227 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12
228 * 0x4e, 0x80, 0x04, 0x20 bctr
231 pr_devel("ip:%lx jumps to %lx", ip
, tramp
);
233 /* Find where the trampoline jumps to */
234 if (probe_kernel_read(jmp
, (void *)tramp
, sizeof(jmp
))) {
235 pr_err("Failed to read %lx\n", tramp
);
239 pr_devel(" %08x %08x ", jmp
[0], jmp
[1]);
241 /* verify that this is what we expect it to be */
242 if (((jmp
[0] & 0xffff0000) != 0x3d800000) ||
243 ((jmp
[1] & 0xffff0000) != 0x398c0000) ||
244 (jmp
[2] != 0x7d8903a6) ||
245 (jmp
[3] != 0x4e800420)) {
246 pr_err("Not a trampoline\n");
250 tramp
= (jmp
[1] & 0xffff) |
251 ((jmp
[0] & 0xffff) << 16);
255 pr_devel(" %lx ", tramp
);
258 pr_err("Trampoline location %08lx does not match addr\n",
265 if (patch_instruction((unsigned int *)ip
, op
))
271 #endif /* CONFIG_MODULES */
273 int ftrace_make_nop(struct module
*mod
,
274 struct dyn_ftrace
*rec
, unsigned long addr
)
276 unsigned long ip
= rec
->ip
;
277 unsigned int old
, new;
280 * If the calling address is more that 24 bits away,
281 * then we had to use a trampoline to make the call.
282 * Otherwise just update the call site.
284 if (test_24bit_addr(ip
, addr
)) {
286 old
= ftrace_call_replace(ip
, addr
, 1);
288 return ftrace_modify_code(ip
, old
, new);
291 #ifdef CONFIG_MODULES
293 * Out of range jumps are called from modules.
294 * We should either already have a pointer to the module
295 * or it has been passed in.
297 if (!rec
->arch
.mod
) {
299 pr_err("No module loaded addr=%lx\n", addr
);
304 if (mod
!= rec
->arch
.mod
) {
305 pr_err("Record mod %p not equal to passed in mod %p\n",
309 /* nothing to do if mod == rec->arch.mod */
313 return __ftrace_make_nop(mod
, rec
, addr
);
315 /* We should not get here without modules */
317 #endif /* CONFIG_MODULES */
320 #ifdef CONFIG_MODULES
323 * Examine the existing instructions for __ftrace_make_call.
324 * They should effectively be a NOP, and follow formal constraints,
325 * depending on the ABI. Return false if they don't.
327 #ifndef CC_USING_MPROFILE_KERNEL
329 expected_nop_sequence(void *ip
, unsigned int op0
, unsigned int op1
)
337 * The load offset is different depending on the ABI. For simplicity
338 * just mask it out when doing the compare.
340 if ((op0
!= 0x48000008) || ((op1
& 0xffff0000) != 0xe8410000))
346 expected_nop_sequence(void *ip
, unsigned int op0
, unsigned int op1
)
348 /* look for patched "NOP" on ppc64 with -mprofile-kernel */
349 if (op0
!= PPC_INST_NOP
)
356 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
359 void *ip
= (void *)rec
->ip
;
361 /* read where this goes */
362 if (probe_kernel_read(op
, ip
, sizeof(op
)))
365 if (!expected_nop_sequence(ip
, op
[0], op
[1])) {
366 pr_err("Unexpected call sequence at %p: %x %x\n",
371 /* If we never set up a trampoline to ftrace_caller, then bail */
372 if (!rec
->arch
.mod
->arch
.tramp
) {
373 pr_err("No ftrace trampoline\n");
377 /* Ensure branch is within 24 bits */
378 if (!create_branch(ip
, rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
)) {
379 pr_err("Branch out of range\n");
383 if (patch_branch(ip
, rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
)) {
384 pr_err("REL24 out of range!\n");
391 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
392 int ftrace_modify_call(struct dyn_ftrace
*rec
, unsigned long old_addr
,
395 return ftrace_make_call(rec
, addr
);
399 #else /* !CONFIG_PPC64: */
401 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
404 unsigned long ip
= rec
->ip
;
406 /* read where this goes */
407 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
410 /* It should be pointing to a nop */
411 if (op
!= PPC_INST_NOP
) {
412 pr_err("Expected NOP but have %x\n", op
);
416 /* If we never set up a trampoline to ftrace_caller, then bail */
417 if (!rec
->arch
.mod
->arch
.tramp
) {
418 pr_err("No ftrace trampoline\n");
422 /* create the branch to the trampoline */
423 op
= create_branch((unsigned int *)ip
,
424 rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
);
426 pr_err("REL24 out of range!\n");
430 pr_devel("write to %lx\n", rec
->ip
);
432 if (patch_instruction((unsigned int *)ip
, op
))
437 #endif /* CONFIG_PPC64 */
438 #endif /* CONFIG_MODULES */
440 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
442 unsigned long ip
= rec
->ip
;
443 unsigned int old
, new;
446 * If the calling address is more that 24 bits away,
447 * then we had to use a trampoline to make the call.
448 * Otherwise just update the call site.
450 if (test_24bit_addr(ip
, addr
)) {
453 new = ftrace_call_replace(ip
, addr
, 1);
454 return ftrace_modify_code(ip
, old
, new);
457 #ifdef CONFIG_MODULES
459 * Out of range jumps are called from modules.
460 * Being that we are converting from nop, it had better
461 * already have a module defined.
463 if (!rec
->arch
.mod
) {
464 pr_err("No module loaded\n");
468 return __ftrace_make_call(rec
, addr
);
470 /* We should not get here without modules */
472 #endif /* CONFIG_MODULES */
475 int ftrace_update_ftrace_func(ftrace_func_t func
)
477 unsigned long ip
= (unsigned long)(&ftrace_call
);
478 unsigned int old
, new;
481 old
= *(unsigned int *)&ftrace_call
;
482 new = ftrace_call_replace(ip
, (unsigned long)func
, 1);
483 ret
= ftrace_modify_code(ip
, old
, new);
488 static int __ftrace_replace_code(struct dyn_ftrace
*rec
, int enable
)
490 unsigned long ftrace_addr
= (unsigned long)FTRACE_ADDR
;
493 ret
= ftrace_update_record(rec
, enable
);
496 case FTRACE_UPDATE_IGNORE
:
498 case FTRACE_UPDATE_MAKE_CALL
:
499 return ftrace_make_call(rec
, ftrace_addr
);
500 case FTRACE_UPDATE_MAKE_NOP
:
501 return ftrace_make_nop(NULL
, rec
, ftrace_addr
);
507 void ftrace_replace_code(int enable
)
509 struct ftrace_rec_iter
*iter
;
510 struct dyn_ftrace
*rec
;
513 for (iter
= ftrace_rec_iter_start(); iter
;
514 iter
= ftrace_rec_iter_next(iter
)) {
515 rec
= ftrace_rec_iter_record(iter
);
516 ret
= __ftrace_replace_code(rec
, enable
);
518 ftrace_bug(ret
, rec
);
525 * Use the default ftrace_modify_all_code, but without
528 void arch_ftrace_update_code(int command
)
530 ftrace_modify_all_code(command
);
533 int __init
ftrace_dyn_arch_init(void)
537 #endif /* CONFIG_DYNAMIC_FTRACE */
539 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
541 #ifdef CONFIG_DYNAMIC_FTRACE
542 extern void ftrace_graph_call(void);
543 extern void ftrace_graph_stub(void);
545 int ftrace_enable_ftrace_graph_caller(void)
547 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
548 unsigned long addr
= (unsigned long)(&ftrace_graph_caller
);
549 unsigned long stub
= (unsigned long)(&ftrace_graph_stub
);
550 unsigned int old
, new;
552 old
= ftrace_call_replace(ip
, stub
, 0);
553 new = ftrace_call_replace(ip
, addr
, 0);
555 return ftrace_modify_code(ip
, old
, new);
558 int ftrace_disable_ftrace_graph_caller(void)
560 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
561 unsigned long addr
= (unsigned long)(&ftrace_graph_caller
);
562 unsigned long stub
= (unsigned long)(&ftrace_graph_stub
);
563 unsigned int old
, new;
565 old
= ftrace_call_replace(ip
, addr
, 0);
566 new = ftrace_call_replace(ip
, stub
, 0);
568 return ftrace_modify_code(ip
, old
, new);
570 #endif /* CONFIG_DYNAMIC_FTRACE */
573 * Hook the return address and push it in the stack of return addrs
574 * in current thread info. Return the address we want to divert to.
576 unsigned long prepare_ftrace_return(unsigned long parent
, unsigned long ip
)
578 struct ftrace_graph_ent trace
;
579 unsigned long return_hooker
;
581 if (unlikely(ftrace_graph_is_dead()))
584 if (unlikely(atomic_read(¤t
->tracing_graph_pause
)))
587 return_hooker
= ppc_function_entry(return_to_handler
);
590 trace
.depth
= current
->curr_ret_stack
+ 1;
592 /* Only trace if the calling function expects to */
593 if (!ftrace_graph_entry(&trace
))
596 if (ftrace_push_return_trace(parent
, ip
, &trace
.depth
, 0,
600 parent
= return_hooker
;
604 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
606 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
607 unsigned long __init
arch_syscall_addr(int nr
)
609 return sys_call_table
[nr
*2];
611 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
613 #ifdef PPC64_ELF_ABI_v1
614 char *arch_ftrace_match_adjust(char *str
, const char *search
)
616 if (str
[0] == '.' && search
[0] != '.')
621 #endif /* PPC64_ELF_ABI_v1 */