]>
Commit | Line | Data |
---|---|---|
014c257c AS |
1 | /* |
2 | * Dynamic function tracing support. | |
3 | * | |
4 | * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> | |
3b6c223b | 5 | * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> |
014c257c AS |
6 | * |
7 | * For licencing details, see COPYING. | |
8 | * | |
9 | * Defines low-level handling of mcount calls when the kernel | |
10 | * is compiled with the -pg flag. When using dynamic ftrace, the | |
3b6c223b RV |
11 | * mcount call-sites get patched with NOP till they are enabled. |
12 | * All code mutation routines here are called under stop_machine(). | |
014c257c AS |
13 | */ |
14 | ||
15 | #include <linux/ftrace.h> | |
3b6c223b | 16 | #include <linux/uaccess.h> |
395a59d0 | 17 | |
014c257c | 18 | #include <asm/cacheflush.h> |
4394e282 | 19 | #include <asm/opcodes.h> |
395a59d0 | 20 | #include <asm/ftrace.h> |
014c257c | 21 | |
72dc43a9 | 22 | #ifdef CONFIG_THUMB2_KERNEL |
4394e282 | 23 | #define NOP 0xf85deb04 /* pop.w {lr} */ |
72dc43a9 | 24 | #else |
3b6c223b | 25 | #define NOP 0xe8bd4000 /* pop {lr} */ |
72dc43a9 | 26 | #endif |
014c257c | 27 | |
376cfa87 | 28 | #ifdef CONFIG_DYNAMIC_FTRACE |
3b6c223b RV |
29 | #ifdef CONFIG_OLD_MCOUNT |
30 | #define OLD_MCOUNT_ADDR ((unsigned long) mcount) | |
31 | #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) | |
014c257c | 32 | |
3b6c223b RV |
33 | #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ |
34 | ||
35 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) | |
36 | { | |
37 | return rec->arch.old_mcount ? OLD_NOP : NOP; | |
38 | } | |
39 | ||
40 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) | |
41 | { | |
42 | if (!rec->arch.old_mcount) | |
43 | return addr; | |
44 | ||
45 | if (addr == MCOUNT_ADDR) | |
46 | addr = OLD_MCOUNT_ADDR; | |
47 | else if (addr == FTRACE_ADDR) | |
48 | addr = OLD_FTRACE_ADDR; | |
49 | ||
50 | return addr; | |
51 | } | |
52 | #else | |
53 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) | |
54 | { | |
55 | return NOP; | |
56 | } | |
57 | ||
58 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) | |
014c257c | 59 | { |
3b6c223b | 60 | return addr; |
014c257c | 61 | } |
3b6c223b | 62 | #endif |
014c257c | 63 | |
72dc43a9 | 64 | #ifdef CONFIG_THUMB2_KERNEL |
dd686eb1 RV |
65 | static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, |
66 | bool link) | |
72dc43a9 RV |
67 | { |
68 | unsigned long s, j1, j2, i1, i2, imm10, imm11; | |
69 | unsigned long first, second; | |
70 | long offset; | |
71 | ||
72 | offset = (long)addr - (long)(pc + 4); | |
73 | if (offset < -16777216 || offset > 16777214) { | |
74 | WARN_ON_ONCE(1); | |
75 | return 0; | |
76 | } | |
77 | ||
78 | s = (offset >> 24) & 0x1; | |
79 | i1 = (offset >> 23) & 0x1; | |
80 | i2 = (offset >> 22) & 0x1; | |
81 | imm10 = (offset >> 12) & 0x3ff; | |
82 | imm11 = (offset >> 1) & 0x7ff; | |
83 | ||
84 | j1 = (!i1) ^ s; | |
85 | j2 = (!i2) ^ s; | |
86 | ||
87 | first = 0xf000 | (s << 10) | imm10; | |
dd686eb1 RV |
88 | second = 0x9000 | (j1 << 13) | (j2 << 11) | imm11; |
89 | if (link) | |
90 | second |= 1 << 14; | |
72dc43a9 | 91 | |
4394e282 | 92 | return __opcode_thumb32_compose(first, second); |
72dc43a9 RV |
93 | } |
94 | #else | |
dd686eb1 RV |
95 | static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, |
96 | bool link) | |
014c257c | 97 | { |
dd686eb1 | 98 | unsigned long opcode = 0xea000000; |
014c257c AS |
99 | long offset; |
100 | ||
dd686eb1 RV |
101 | if (link) |
102 | opcode |= 1 << 24; | |
103 | ||
3b6c223b | 104 | offset = (long)addr - (long)(pc + 8); |
014c257c AS |
105 | if (unlikely(offset < -33554432 || offset > 33554428)) { |
106 | /* Can't generate branches that far (from ARM ARM). Ftrace | |
395a59d0 | 107 | * doesn't generate branches outside of kernel text. |
014c257c AS |
108 | */ |
109 | WARN_ON_ONCE(1); | |
3b6c223b | 110 | return 0; |
014c257c | 111 | } |
014c257c | 112 | |
3b6c223b | 113 | offset = (offset >> 2) & 0x00ffffff; |
014c257c | 114 | |
dd686eb1 | 115 | return opcode | offset; |
3b6c223b | 116 | } |
72dc43a9 | 117 | #endif |
014c257c | 118 | |
dd686eb1 RV |
119 | static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) |
120 | { | |
121 | return ftrace_gen_branch(pc, addr, true); | |
122 | } | |
123 | ||
3b6c223b | 124 | static int ftrace_modify_code(unsigned long pc, unsigned long old, |
dc283d70 | 125 | unsigned long new, bool validate) |
3b6c223b RV |
126 | { |
127 | unsigned long replaced; | |
014c257c | 128 | |
4394e282 RV |
129 | if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { |
130 | old = __opcode_to_mem_thumb32(old); | |
131 | new = __opcode_to_mem_thumb32(new); | |
132 | } else { | |
133 | old = __opcode_to_mem_arm(old); | |
134 | new = __opcode_to_mem_arm(new); | |
135 | } | |
136 | ||
dc283d70 RV |
137 | if (validate) { |
138 | if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) | |
139 | return -EFAULT; | |
014c257c | 140 | |
dc283d70 RV |
141 | if (replaced != old) |
142 | return -EINVAL; | |
143 | } | |
014c257c | 144 | |
3b6c223b RV |
145 | if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) |
146 | return -EPERM; | |
014c257c | 147 | |
3b6c223b | 148 | flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); |
014c257c | 149 | |
3b6c223b | 150 | return 0; |
014c257c AS |
151 | } |
152 | ||
153 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
154 | { | |
dc283d70 | 155 | unsigned long pc; |
3b6c223b RV |
156 | unsigned long new; |
157 | int ret; | |
014c257c AS |
158 | |
159 | pc = (unsigned long)&ftrace_call; | |
014c257c | 160 | new = ftrace_call_replace(pc, (unsigned long)func); |
3b6c223b | 161 | |
dc283d70 | 162 | ret = ftrace_modify_code(pc, 0, new, false); |
3b6c223b RV |
163 | |
164 | #ifdef CONFIG_OLD_MCOUNT | |
165 | if (!ret) { | |
166 | pc = (unsigned long)&ftrace_call_old; | |
3b6c223b RV |
167 | new = ftrace_call_replace(pc, (unsigned long)func); |
168 | ||
dc283d70 | 169 | ret = ftrace_modify_code(pc, 0, new, false); |
3b6c223b RV |
170 | } |
171 | #endif | |
172 | ||
173 | return ret; | |
174 | } | |
175 | ||
176 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
177 | { | |
178 | unsigned long new, old; | |
179 | unsigned long ip = rec->ip; | |
180 | ||
181 | old = ftrace_nop_replace(rec); | |
182 | new = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
183 | ||
dc283d70 | 184 | return ftrace_modify_code(rec->ip, old, new, true); |
3b6c223b RV |
185 | } |
186 | ||
187 | int ftrace_make_nop(struct module *mod, | |
188 | struct dyn_ftrace *rec, unsigned long addr) | |
189 | { | |
190 | unsigned long ip = rec->ip; | |
191 | unsigned long old; | |
192 | unsigned long new; | |
193 | int ret; | |
194 | ||
195 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
196 | new = ftrace_nop_replace(rec); | |
dc283d70 | 197 | ret = ftrace_modify_code(ip, old, new, true); |
3b6c223b RV |
198 | |
199 | #ifdef CONFIG_OLD_MCOUNT | |
200 | if (ret == -EINVAL && addr == MCOUNT_ADDR) { | |
201 | rec->arch.old_mcount = true; | |
202 | ||
203 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
204 | new = ftrace_nop_replace(rec); | |
dc283d70 | 205 | ret = ftrace_modify_code(ip, old, new, true); |
3b6c223b RV |
206 | } |
207 | #endif | |
208 | ||
014c257c AS |
209 | return ret; |
210 | } | |
211 | ||
014c257c AS |
212 | int __init ftrace_dyn_arch_init(void *data) |
213 | { | |
3b6c223b RV |
214 | *(unsigned long *)data = 0; |
215 | ||
014c257c AS |
216 | return 0; |
217 | } | |
376cfa87 TB |
218 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
219 | ||
220 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
221 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, | |
222 | unsigned long frame_pointer) | |
223 | { | |
224 | unsigned long return_hooker = (unsigned long) &return_to_handler; | |
225 | struct ftrace_graph_ent trace; | |
226 | unsigned long old; | |
227 | int err; | |
228 | ||
229 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) | |
230 | return; | |
231 | ||
232 | old = *parent; | |
233 | *parent = return_hooker; | |
234 | ||
235 | err = ftrace_push_return_trace(old, self_addr, &trace.depth, | |
236 | frame_pointer); | |
237 | if (err == -EBUSY) { | |
238 | *parent = old; | |
239 | return; | |
240 | } | |
241 | ||
242 | trace.func = self_addr; | |
243 | ||
244 | /* Only trace if the calling function expects to */ | |
245 | if (!ftrace_graph_entry(&trace)) { | |
246 | current->curr_ret_stack--; | |
247 | *parent = old; | |
248 | } | |
249 | } | |
dd686eb1 RV |
250 | |
251 | #ifdef CONFIG_DYNAMIC_FTRACE | |
252 | extern unsigned long ftrace_graph_call; | |
253 | extern unsigned long ftrace_graph_call_old; | |
254 | extern void ftrace_graph_caller_old(void); | |
255 | ||
256 | static int __ftrace_modify_caller(unsigned long *callsite, | |
257 | void (*func) (void), bool enable) | |
258 | { | |
259 | unsigned long caller_fn = (unsigned long) func; | |
260 | unsigned long pc = (unsigned long) callsite; | |
261 | unsigned long branch = ftrace_gen_branch(pc, caller_fn, false); | |
262 | unsigned long nop = 0xe1a00000; /* mov r0, r0 */ | |
263 | unsigned long old = enable ? nop : branch; | |
264 | unsigned long new = enable ? branch : nop; | |
265 | ||
dc283d70 | 266 | return ftrace_modify_code(pc, old, new, true); |
dd686eb1 RV |
267 | } |
268 | ||
269 | static int ftrace_modify_graph_caller(bool enable) | |
270 | { | |
271 | int ret; | |
272 | ||
273 | ret = __ftrace_modify_caller(&ftrace_graph_call, | |
274 | ftrace_graph_caller, | |
275 | enable); | |
276 | ||
277 | #ifdef CONFIG_OLD_MCOUNT | |
278 | if (!ret) | |
279 | ret = __ftrace_modify_caller(&ftrace_graph_call_old, | |
280 | ftrace_graph_caller_old, | |
281 | enable); | |
282 | #endif | |
283 | ||
284 | return ret; | |
285 | } | |
286 | ||
287 | int ftrace_enable_ftrace_graph_caller(void) | |
288 | { | |
289 | return ftrace_modify_graph_caller(true); | |
290 | } | |
291 | ||
292 | int ftrace_disable_ftrace_graph_caller(void) | |
293 | { | |
294 | return ftrace_modify_graph_caller(false); | |
295 | } | |
296 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
376cfa87 | 297 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |