]>
Commit | Line | Data |
---|---|---|
dfd9f7ab HC |
1 | /* |
2 | * Dynamic function tracer architecture backend. | |
3 | * | |
3d1e220d | 4 | * Copyright IBM Corp. 2009,2014 |
dfd9f7ab HC |
5 | * |
6 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, | |
4cc9bed0 | 7 | * Martin Schwidefsky <schwidefsky@de.ibm.com> |
dfd9f7ab HC |
8 | */ |
9 | ||
c933146a | 10 | #include <linux/moduleloader.h> |
88dbd203 | 11 | #include <linux/hardirq.h> |
dfd9f7ab HC |
12 | #include <linux/uaccess.h> |
13 | #include <linux/ftrace.h> | |
14 | #include <linux/kernel.h> | |
15 | #include <linux/types.h> | |
4cc9bed0 | 16 | #include <linux/kprobes.h> |
9bf1226b | 17 | #include <trace/syscall.h> |
cbb870c8 | 18 | #include <asm/asm-offsets.h> |
c933146a | 19 | #include <asm/cacheflush.h> |
e6c7c630 | 20 | #include <asm/set_memory.h> |
63df41d6 | 21 | #include "entry.h" |
dfd9f7ab | 22 | |
4cc9bed0 | 23 | /* |
53255c9a | 24 | * The mcount code looks like this: |
4cc9bed0 | 25 | * stg %r14,8(%r15) # offset 0 |
3d1e220d HC |
26 | * larl %r1,<&counter> # offset 6 |
27 | * brasl %r14,_mcount # offset 12 | |
4cc9bed0 | 28 | * lg %r14,8(%r15) # offset 18 |
c933146a HC |
29 | * Total length is 24 bytes. Only the first instruction will be patched |
30 | * by ftrace_make_call / ftrace_make_nop. | |
53255c9a | 31 | * The enabled ftrace code block looks like this: |
c933146a HC |
32 | * > brasl %r0,ftrace_caller # offset 0 |
33 | * larl %r1,<&counter> # offset 6 | |
34 | * brasl %r14,_mcount # offset 12 | |
35 | * lg %r14,8(%r15) # offset 18 | |
3d1e220d HC |
36 | * The ftrace function gets called with a non-standard C function call ABI |
37 | * where r0 contains the return address. It is also expected that the called | |
38 | * function only clobbers r0 and r1, but restores r2-r15. | |
c933146a HC |
39 | * For module code we can't directly jump to ftrace caller, but need a |
40 | * trampoline (ftrace_plt), which clobbers also r1. | |
3d1e220d HC |
41 | * The return point of the ftrace function has offset 24, so execution |
42 | * continues behind the mcount block. | |
c933146a HC |
43 | * The disabled ftrace code block looks like this: |
44 | * > jg .+24 # offset 0 | |
45 | * larl %r1,<&counter> # offset 6 | |
46 | * brasl %r14,_mcount # offset 12 | |
47 | * lg %r14,8(%r15) # offset 18 | |
4cc9bed0 MS |
48 | * The jg instruction branches to offset 24 to skip as many instructions |
49 | * as possible. | |
e6d60b36 HC |
50 | * In case we use gcc's hotpatch feature the original and also the disabled |
51 | * function prologue contains only a single six byte instruction and looks | |
52 | * like this: | |
53 | * > brcl 0,0 # offset 0 | |
54 | * To enable ftrace the code gets patched like above and afterwards looks | |
55 | * like this: | |
56 | * > brasl %r0,ftrace_caller # offset 0 | |
4cc9bed0 | 57 | */ |
dfd9f7ab | 58 | |
c933146a | 59 | unsigned long ftrace_plt; |
dfd9f7ab | 60 | |
20e76ee1 HC |
61 | static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) |
62 | { | |
63 | #ifdef CC_USING_HOTPATCH | |
64 | /* brcl 0,0 */ | |
65 | insn->opc = 0xc004; | |
66 | insn->disp = 0; | |
67 | #else | |
68 | /* stg r14,8(r15) */ | |
69 | insn->opc = 0xe3e0; | |
70 | insn->disp = 0xf0080024; | |
71 | #endif | |
72 | } | |
73 | ||
74 | static inline int is_kprobe_on_ftrace(struct ftrace_insn *insn) | |
75 | { | |
76 | #ifdef CONFIG_KPROBES | |
77 | if (insn->opc == BREAKPOINT_INSTRUCTION) | |
78 | return 1; | |
79 | #endif | |
80 | return 0; | |
81 | } | |
82 | ||
83 | static inline void ftrace_generate_kprobe_nop_insn(struct ftrace_insn *insn) | |
84 | { | |
85 | #ifdef CONFIG_KPROBES | |
86 | insn->opc = BREAKPOINT_INSTRUCTION; | |
87 | insn->disp = KPROBE_ON_FTRACE_NOP; | |
88 | #endif | |
89 | } | |
90 | ||
91 | static inline void ftrace_generate_kprobe_call_insn(struct ftrace_insn *insn) | |
92 | { | |
93 | #ifdef CONFIG_KPROBES | |
94 | insn->opc = BREAKPOINT_INSTRUCTION; | |
95 | insn->disp = KPROBE_ON_FTRACE_CALL; | |
96 | #endif | |
97 | } | |
98 | ||
10dec7db HC |
99 | int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
100 | unsigned long addr) | |
101 | { | |
102 | return 0; | |
103 | } | |
dfd9f7ab HC |
104 | |
105 | int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, | |
106 | unsigned long addr) | |
107 | { | |
58498ee3 | 108 | struct ftrace_insn orig, new, old; |
c933146a | 109 | |
58498ee3 | 110 | if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
c933146a | 111 | return -EFAULT; |
58498ee3 | 112 | if (addr == MCOUNT_ADDR) { |
e6d60b36 | 113 | /* Initial code replacement */ |
20e76ee1 | 114 | ftrace_generate_orig_insn(&orig); |
58498ee3 | 115 | ftrace_generate_nop_insn(&new); |
20e76ee1 | 116 | } else if (is_kprobe_on_ftrace(&old)) { |
58498ee3 HC |
117 | /* |
118 | * If we find a breakpoint instruction, a kprobe has been | |
119 | * placed at the beginning of the function. We write the | |
120 | * constant KPROBE_ON_FTRACE_NOP into the remaining four | |
121 | * bytes of the original instruction so that the kprobes | |
122 | * handler can execute a nop, if it reaches this breakpoint. | |
123 | */ | |
20e76ee1 HC |
124 | ftrace_generate_kprobe_call_insn(&orig); |
125 | ftrace_generate_kprobe_nop_insn(&new); | |
58498ee3 HC |
126 | } else { |
127 | /* Replace ftrace call with a nop. */ | |
128 | ftrace_generate_call_insn(&orig, rec->ip); | |
129 | ftrace_generate_nop_insn(&new); | |
3d1e220d | 130 | } |
58498ee3 HC |
131 | /* Verify that the to be replaced code matches what we expect. */ |
132 | if (memcmp(&orig, &old, sizeof(old))) | |
133 | return -EINVAL; | |
8a5d8473 | 134 | s390_kernel_write((void *) rec->ip, &new, sizeof(new)); |
4cc9bed0 | 135 | return 0; |
dfd9f7ab HC |
136 | } |
137 | ||
138 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
139 | { | |
58498ee3 | 140 | struct ftrace_insn orig, new, old; |
c933146a | 141 | |
58498ee3 | 142 | if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
c933146a | 143 | return -EFAULT; |
20e76ee1 | 144 | if (is_kprobe_on_ftrace(&old)) { |
58498ee3 HC |
145 | /* |
146 | * If we find a breakpoint instruction, a kprobe has been | |
147 | * placed at the beginning of the function. We write the | |
148 | * constant KPROBE_ON_FTRACE_CALL into the remaining four | |
149 | * bytes of the original instruction so that the kprobes | |
150 | * handler can execute a brasl if it reaches this breakpoint. | |
151 | */ | |
20e76ee1 HC |
152 | ftrace_generate_kprobe_nop_insn(&orig); |
153 | ftrace_generate_kprobe_call_insn(&new); | |
58498ee3 HC |
154 | } else { |
155 | /* Replace nop with an ftrace call. */ | |
156 | ftrace_generate_nop_insn(&orig); | |
157 | ftrace_generate_call_insn(&new, rec->ip); | |
c933146a | 158 | } |
58498ee3 HC |
159 | /* Verify that the to be replaced code matches what we expect. */ |
160 | if (memcmp(&orig, &old, sizeof(old))) | |
161 | return -EINVAL; | |
8a5d8473 | 162 | s390_kernel_write((void *) rec->ip, &new, sizeof(new)); |
4cc9bed0 | 163 | return 0; |
dfd9f7ab HC |
164 | } |
165 | ||
166 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
167 | { | |
dfd9f7ab HC |
168 | return 0; |
169 | } | |
170 | ||
3a36cb11 | 171 | int __init ftrace_dyn_arch_init(void) |
dfd9f7ab | 172 | { |
dfd9f7ab HC |
173 | return 0; |
174 | } | |
88dbd203 | 175 | |
c933146a HC |
176 | static int __init ftrace_plt_init(void) |
177 | { | |
178 | unsigned int *ip; | |
179 | ||
180 | ftrace_plt = (unsigned long) module_alloc(PAGE_SIZE); | |
181 | if (!ftrace_plt) | |
182 | panic("cannot allocate ftrace plt\n"); | |
183 | ip = (unsigned int *) ftrace_plt; | |
184 | ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ | |
185 | ip[1] = 0x100a0004; | |
186 | ip[2] = 0x07f10000; | |
187 | ip[3] = FTRACE_ADDR >> 32; | |
188 | ip[4] = FTRACE_ADDR & 0xffffffff; | |
189 | set_memory_ro(ftrace_plt, 1); | |
190 | return 0; | |
191 | } | |
192 | device_initcall(ftrace_plt_init); | |
193 | ||
88dbd203 | 194 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
88dbd203 HC |
195 | /* |
196 | * Hook the return address and push it in the stack of return addresses | |
197 | * in current thread info. | |
198 | */ | |
7a5388de | 199 | unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) |
88dbd203 HC |
200 | { |
201 | struct ftrace_graph_ent trace; | |
202 | ||
6ed15ea6 HC |
203 | if (unlikely(ftrace_graph_is_dead())) |
204 | goto out; | |
88dbd203 HC |
205 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
206 | goto out; | |
9cb1ccec | 207 | ip -= MCOUNT_INSN_SIZE; |
aca91209 | 208 | trace.func = ip; |
05e0baaf | 209 | trace.depth = current->curr_ret_stack + 1; |
88dbd203 | 210 | /* Only trace if the calling function expects to. */ |
05e0baaf HC |
211 | if (!ftrace_graph_entry(&trace)) |
212 | goto out; | |
9a7c348b JP |
213 | if (ftrace_push_return_trace(parent, ip, &trace.depth, 0, |
214 | NULL) == -EBUSY) | |
88dbd203 | 215 | goto out; |
4cc9bed0 | 216 | parent = (unsigned long) return_to_handler; |
88dbd203 HC |
217 | out: |
218 | return parent; | |
219 | } | |
7a5388de | 220 | NOKPROBE_SYMBOL(prepare_ftrace_return); |
4cc9bed0 | 221 | |
4cc9bed0 MS |
222 | /* |
223 | * Patch the kernel code at ftrace_graph_caller location. The instruction | |
0cccdda8 HC |
224 | * there is branch relative on condition. To enable the ftrace graph code |
225 | * block, we simply patch the mask field of the instruction to zero and | |
226 | * turn the instruction into a nop. | |
227 | * To disable the ftrace graph code the mask field will be patched to | |
228 | * all ones, which turns the instruction into an unconditional branch. | |
4cc9bed0 | 229 | */ |
2481a87b HC |
230 | int ftrace_enable_ftrace_graph_caller(void) |
231 | { | |
0cccdda8 | 232 | u8 op = 0x04; /* set mask field to zero */ |
2481a87b | 233 | |
8a5d8473 HC |
234 | s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); |
235 | return 0; | |
2481a87b HC |
236 | } |
237 | ||
238 | int ftrace_disable_ftrace_graph_caller(void) | |
239 | { | |
0cccdda8 | 240 | u8 op = 0xf4; /* set mask field to all ones */ |
2481a87b | 241 | |
8a5d8473 HC |
242 | s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); |
243 | return 0; | |
2481a87b HC |
244 | } |
245 | ||
88dbd203 | 246 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |