]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dfd9f7ab HC |
2 | /* |
3 | * Dynamic function tracer architecture backend. | |
4 | * | |
3d1e220d | 5 | * Copyright IBM Corp. 2009,2014 |
dfd9f7ab HC |
6 | * |
7 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, | |
4cc9bed0 | 8 | * Martin Schwidefsky <schwidefsky@de.ibm.com> |
dfd9f7ab HC |
9 | */ |
10 | ||
c933146a | 11 | #include <linux/moduleloader.h> |
88dbd203 | 12 | #include <linux/hardirq.h> |
dfd9f7ab HC |
13 | #include <linux/uaccess.h> |
14 | #include <linux/ftrace.h> | |
15 | #include <linux/kernel.h> | |
16 | #include <linux/types.h> | |
4cc9bed0 | 17 | #include <linux/kprobes.h> |
9bf1226b | 18 | #include <trace/syscall.h> |
cbb870c8 | 19 | #include <asm/asm-offsets.h> |
c933146a | 20 | #include <asm/cacheflush.h> |
e6c7c630 | 21 | #include <asm/set_memory.h> |
63df41d6 | 22 | #include "entry.h" |
dfd9f7ab | 23 | |
4cc9bed0 | 24 | /* |
53255c9a | 25 | * The mcount code looks like this: |
4cc9bed0 | 26 | * stg %r14,8(%r15) # offset 0 |
3d1e220d HC |
27 | * larl %r1,<&counter> # offset 6 |
28 | * brasl %r14,_mcount # offset 12 | |
4cc9bed0 | 29 | * lg %r14,8(%r15) # offset 18 |
c933146a HC |
30 | * Total length is 24 bytes. Only the first instruction will be patched |
31 | * by ftrace_make_call / ftrace_make_nop. | |
53255c9a | 32 | * The enabled ftrace code block looks like this: |
c933146a HC |
33 | * > brasl %r0,ftrace_caller # offset 0 |
34 | * larl %r1,<&counter> # offset 6 | |
35 | * brasl %r14,_mcount # offset 12 | |
36 | * lg %r14,8(%r15) # offset 18 | |
3d1e220d HC |
37 | * The ftrace function gets called with a non-standard C function call ABI |
38 | * where r0 contains the return address. It is also expected that the called | |
39 | * function only clobbers r0 and r1, but restores r2-r15. | |
c933146a HC |
40 | * For module code we can't directly jump to ftrace caller, but need a |
41 | * trampoline (ftrace_plt), which clobbers also r1. | |
3d1e220d HC |
42 | * The return point of the ftrace function has offset 24, so execution |
43 | * continues behind the mcount block. | |
c933146a HC |
44 | * The disabled ftrace code block looks like this: |
45 | * > jg .+24 # offset 0 | |
46 | * larl %r1,<&counter> # offset 6 | |
47 | * brasl %r14,_mcount # offset 12 | |
48 | * lg %r14,8(%r15) # offset 18 | |
4cc9bed0 MS |
49 | * The jg instruction branches to offset 24 to skip as many instructions |
50 | * as possible. | |
e6d60b36 HC |
51 | * In case we use gcc's hotpatch feature the original and also the disabled |
52 | * function prologue contains only a single six byte instruction and looks | |
53 | * like this: | |
54 | * > brcl 0,0 # offset 0 | |
55 | * To enable ftrace the code gets patched like above and afterwards looks | |
56 | * like this: | |
57 | * > brasl %r0,ftrace_caller # offset 0 | |
4cc9bed0 | 58 | */ |
dfd9f7ab | 59 | |
c933146a | 60 | unsigned long ftrace_plt; |
dfd9f7ab | 61 | |
20e76ee1 HC |
62 | static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) |
63 | { | |
d983c89c | 64 | #if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) |
20e76ee1 HC |
65 | /* brcl 0,0 */ |
66 | insn->opc = 0xc004; | |
67 | insn->disp = 0; | |
68 | #else | |
69 | /* stg r14,8(r15) */ | |
70 | insn->opc = 0xe3e0; | |
71 | insn->disp = 0xf0080024; | |
72 | #endif | |
73 | } | |
74 | ||
75 | static inline int is_kprobe_on_ftrace(struct ftrace_insn *insn) | |
76 | { | |
77 | #ifdef CONFIG_KPROBES | |
78 | if (insn->opc == BREAKPOINT_INSTRUCTION) | |
79 | return 1; | |
80 | #endif | |
81 | return 0; | |
82 | } | |
83 | ||
84 | static inline void ftrace_generate_kprobe_nop_insn(struct ftrace_insn *insn) | |
85 | { | |
86 | #ifdef CONFIG_KPROBES | |
87 | insn->opc = BREAKPOINT_INSTRUCTION; | |
88 | insn->disp = KPROBE_ON_FTRACE_NOP; | |
89 | #endif | |
90 | } | |
91 | ||
92 | static inline void ftrace_generate_kprobe_call_insn(struct ftrace_insn *insn) | |
93 | { | |
94 | #ifdef CONFIG_KPROBES | |
95 | insn->opc = BREAKPOINT_INSTRUCTION; | |
96 | insn->disp = KPROBE_ON_FTRACE_CALL; | |
97 | #endif | |
98 | } | |
99 | ||
10dec7db HC |
100 | int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
101 | unsigned long addr) | |
102 | { | |
103 | return 0; | |
104 | } | |
dfd9f7ab HC |
105 | |
106 | int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, | |
107 | unsigned long addr) | |
108 | { | |
58498ee3 | 109 | struct ftrace_insn orig, new, old; |
c933146a | 110 | |
58498ee3 | 111 | if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
c933146a | 112 | return -EFAULT; |
58498ee3 | 113 | if (addr == MCOUNT_ADDR) { |
e6d60b36 | 114 | /* Initial code replacement */ |
20e76ee1 | 115 | ftrace_generate_orig_insn(&orig); |
58498ee3 | 116 | ftrace_generate_nop_insn(&new); |
20e76ee1 | 117 | } else if (is_kprobe_on_ftrace(&old)) { |
58498ee3 HC |
118 | /* |
119 | * If we find a breakpoint instruction, a kprobe has been | |
120 | * placed at the beginning of the function. We write the | |
121 | * constant KPROBE_ON_FTRACE_NOP into the remaining four | |
122 | * bytes of the original instruction so that the kprobes | |
123 | * handler can execute a nop, if it reaches this breakpoint. | |
124 | */ | |
20e76ee1 HC |
125 | ftrace_generate_kprobe_call_insn(&orig); |
126 | ftrace_generate_kprobe_nop_insn(&new); | |
58498ee3 HC |
127 | } else { |
128 | /* Replace ftrace call with a nop. */ | |
129 | ftrace_generate_call_insn(&orig, rec->ip); | |
130 | ftrace_generate_nop_insn(&new); | |
3d1e220d | 131 | } |
58498ee3 HC |
132 | /* Verify that the to be replaced code matches what we expect. */ |
133 | if (memcmp(&orig, &old, sizeof(old))) | |
134 | return -EINVAL; | |
8a5d8473 | 135 | s390_kernel_write((void *) rec->ip, &new, sizeof(new)); |
4cc9bed0 | 136 | return 0; |
dfd9f7ab HC |
137 | } |
138 | ||
139 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
140 | { | |
58498ee3 | 141 | struct ftrace_insn orig, new, old; |
c933146a | 142 | |
58498ee3 | 143 | if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
c933146a | 144 | return -EFAULT; |
20e76ee1 | 145 | if (is_kprobe_on_ftrace(&old)) { |
58498ee3 HC |
146 | /* |
147 | * If we find a breakpoint instruction, a kprobe has been | |
148 | * placed at the beginning of the function. We write the | |
149 | * constant KPROBE_ON_FTRACE_CALL into the remaining four | |
150 | * bytes of the original instruction so that the kprobes | |
151 | * handler can execute a brasl if it reaches this breakpoint. | |
152 | */ | |
20e76ee1 HC |
153 | ftrace_generate_kprobe_nop_insn(&orig); |
154 | ftrace_generate_kprobe_call_insn(&new); | |
58498ee3 HC |
155 | } else { |
156 | /* Replace nop with an ftrace call. */ | |
157 | ftrace_generate_nop_insn(&orig); | |
158 | ftrace_generate_call_insn(&new, rec->ip); | |
c933146a | 159 | } |
58498ee3 HC |
160 | /* Verify that the to be replaced code matches what we expect. */ |
161 | if (memcmp(&orig, &old, sizeof(old))) | |
162 | return -EINVAL; | |
8a5d8473 | 163 | s390_kernel_write((void *) rec->ip, &new, sizeof(new)); |
4cc9bed0 | 164 | return 0; |
dfd9f7ab HC |
165 | } |
166 | ||
167 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
168 | { | |
dfd9f7ab HC |
169 | return 0; |
170 | } | |
171 | ||
3a36cb11 | 172 | int __init ftrace_dyn_arch_init(void) |
dfd9f7ab | 173 | { |
dfd9f7ab HC |
174 | return 0; |
175 | } | |
88dbd203 | 176 | |
085b6ba0 HC |
177 | #ifdef CONFIG_MODULES |
178 | ||
c933146a HC |
179 | static int __init ftrace_plt_init(void) |
180 | { | |
181 | unsigned int *ip; | |
182 | ||
183 | ftrace_plt = (unsigned long) module_alloc(PAGE_SIZE); | |
184 | if (!ftrace_plt) | |
185 | panic("cannot allocate ftrace plt\n"); | |
186 | ip = (unsigned int *) ftrace_plt; | |
187 | ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ | |
188 | ip[1] = 0x100a0004; | |
189 | ip[2] = 0x07f10000; | |
190 | ip[3] = FTRACE_ADDR >> 32; | |
191 | ip[4] = FTRACE_ADDR & 0xffffffff; | |
192 | set_memory_ro(ftrace_plt, 1); | |
193 | return 0; | |
194 | } | |
195 | device_initcall(ftrace_plt_init); | |
196 | ||
085b6ba0 HC |
197 | #endif /* CONFIG_MODULES */ |
198 | ||
88dbd203 | 199 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
88dbd203 HC |
200 | /* |
201 | * Hook the return address and push it in the stack of return addresses | |
202 | * in current thread info. | |
203 | */ | |
ec7bf478 MS |
204 | unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, |
205 | unsigned long ip) | |
88dbd203 | 206 | { |
6ed15ea6 HC |
207 | if (unlikely(ftrace_graph_is_dead())) |
208 | goto out; | |
88dbd203 HC |
209 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
210 | goto out; | |
9cb1ccec | 211 | ip -= MCOUNT_INSN_SIZE; |
ec7bf478 MS |
212 | if (!function_graph_enter(ra, ip, 0, (void *) sp)) |
213 | ra = (unsigned long) return_to_handler; | |
88dbd203 | 214 | out: |
ec7bf478 | 215 | return ra; |
88dbd203 | 216 | } |
7a5388de | 217 | NOKPROBE_SYMBOL(prepare_ftrace_return); |
4cc9bed0 | 218 | |
4cc9bed0 MS |
219 | /* |
220 | * Patch the kernel code at ftrace_graph_caller location. The instruction | |
0cccdda8 HC |
221 | * there is branch relative on condition. To enable the ftrace graph code |
222 | * block, we simply patch the mask field of the instruction to zero and | |
223 | * turn the instruction into a nop. | |
224 | * To disable the ftrace graph code the mask field will be patched to | |
225 | * all ones, which turns the instruction into an unconditional branch. | |
4cc9bed0 | 226 | */ |
2481a87b HC |
227 | int ftrace_enable_ftrace_graph_caller(void) |
228 | { | |
0cccdda8 | 229 | u8 op = 0x04; /* set mask field to zero */ |
2481a87b | 230 | |
8a5d8473 HC |
231 | s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); |
232 | return 0; | |
2481a87b HC |
233 | } |
234 | ||
235 | int ftrace_disable_ftrace_graph_caller(void) | |
236 | { | |
0cccdda8 | 237 | u8 op = 0xf4; /* set mask field to all ones */ |
2481a87b | 238 | |
8a5d8473 HC |
239 | s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); |
240 | return 0; | |
2481a87b HC |
241 | } |
242 | ||
88dbd203 | 243 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |