]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
3d82c59c SRV |
2 | /* |
3 | * Copyright (C) 2017 Steven Rostedt, VMware Inc. | |
4 | */ | |
5 | ||
6 | #include <linux/linkage.h> | |
7 | #include <asm/page_types.h> | |
8 | #include <asm/segment.h> | |
9 | #include <asm/export.h> | |
10 | #include <asm/ftrace.h> | |
9351803b | 11 | #include <asm/nospec-branch.h> |
3d82c59c | 12 | |
644e0e8d SRV |
13 | #ifdef CC_USING_FENTRY |
14 | # define function_hook __fentry__ | |
15 | EXPORT_SYMBOL(__fentry__) | |
16 | #else | |
17 | # define function_hook mcount | |
18 | EXPORT_SYMBOL(mcount) | |
19 | #endif | |
20 | ||
3d82c59c SRV |
21 | #ifdef CONFIG_DYNAMIC_FTRACE |
22 | ||
644e0e8d SRV |
23 | /* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */ |
24 | #if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER) | |
25 | # define USING_FRAME_POINTER | |
26 | #endif | |
27 | ||
28 | #ifdef USING_FRAME_POINTER | |
29 | # define MCOUNT_FRAME 1 /* using frame = true */ | |
30 | #else | |
31 | # define MCOUNT_FRAME 0 /* using frame = false */ | |
32 | #endif | |
33 | ||
34 | ENTRY(function_hook) | |
3d82c59c | 35 | ret |
644e0e8d | 36 | END(function_hook) |
3d82c59c SRV |
37 | |
38 | ENTRY(ftrace_caller) | |
e6928e58 | 39 | |
644e0e8d SRV |
40 | #ifdef USING_FRAME_POINTER |
41 | # ifdef CC_USING_FENTRY | |
42 | /* | |
43 | * Frame pointers are of ip followed by bp. | |
44 | * Since fentry is an immediate jump, we are left with | |
45 | * parent-ip, function-ip. We need to add a frame with | |
46 | * parent-ip followed by ebp. | |
47 | */ | |
48 | pushl 4(%esp) /* parent ip */ | |
e6928e58 SRV |
49 | pushl %ebp |
50 | movl %esp, %ebp | |
644e0e8d SRV |
51 | pushl 2*4(%esp) /* function ip */ |
52 | # endif | |
53 | /* For mcount, the function ip is directly above */ | |
54 | pushl %ebp | |
55 | movl %esp, %ebp | |
56 | #endif | |
3d82c59c SRV |
57 | pushl %eax |
58 | pushl %ecx | |
59 | pushl %edx | |
60 | pushl $0 /* Pass NULL as regs pointer */ | |
644e0e8d SRV |
61 | |
62 | #ifdef USING_FRAME_POINTER | |
63 | /* Load parent ebp into edx */ | |
e6928e58 | 64 | movl 4*4(%esp), %edx |
644e0e8d SRV |
65 | #else |
66 | /* There's no frame pointer, load the appropriate stack addr instead */ | |
67 | lea 4*4(%esp), %edx | |
68 | #endif | |
69 | ||
70 | movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ | |
e6928e58 | 71 | /* Get the parent ip */ |
644e0e8d SRV |
72 | movl 4(%edx), %edx /* edx has ebp */ |
73 | ||
3d82c59c SRV |
74 | movl function_trace_op, %ecx |
75 | subl $MCOUNT_INSN_SIZE, %eax | |
76 | ||
77 | .globl ftrace_call | |
78 | ftrace_call: | |
79 | call ftrace_stub | |
80 | ||
81 | addl $4, %esp /* skip NULL pointer */ | |
82 | popl %edx | |
83 | popl %ecx | |
84 | popl %eax | |
644e0e8d | 85 | #ifdef USING_FRAME_POINTER |
e6928e58 | 86 | popl %ebp |
644e0e8d SRV |
87 | # ifdef CC_USING_FENTRY |
88 | addl $4,%esp /* skip function ip */ | |
89 | popl %ebp /* this is the orig bp */ | |
90 | addl $4, %esp /* skip parent ip */ | |
91 | # endif | |
92 | #endif | |
3d82c59c SRV |
93 | .Lftrace_ret: |
94 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
95 | .globl ftrace_graph_call | |
96 | ftrace_graph_call: | |
97 | jmp ftrace_stub | |
98 | #endif | |
99 | ||
100 | /* This is weak to keep gas from relaxing the jumps */ | |
101 | WEAK(ftrace_stub) | |
102 | ret | |
103 | END(ftrace_caller) | |
104 | ||
105 | ENTRY(ftrace_regs_caller) | |
3d82c59c SRV |
106 | /* |
107 | * i386 does not save SS and ESP when coming from kernel. | |
108 | * Instead, to get sp, ®s->sp is used (see ptrace.h). | |
109 | * Unfortunately, that means eflags must be at the same location | |
110 | * as the current return ip is. We move the return ip into the | |
ff04b440 | 111 | * regs->ip location, and move flags into the return ip location. |
3d82c59c | 112 | */ |
ff04b440 SRV |
113 | pushl $__KERNEL_CS |
114 | pushl 4(%esp) /* Save the return ip */ | |
3d82c59c SRV |
115 | pushl $0 /* Load 0 into orig_ax */ |
116 | pushl %gs | |
117 | pushl %fs | |
118 | pushl %es | |
119 | pushl %ds | |
120 | pushl %eax | |
ff04b440 SRV |
121 | |
122 | /* Get flags and place them into the return ip slot */ | |
123 | pushf | |
124 | popl %eax | |
125 | movl %eax, 8*4(%esp) | |
126 | ||
3d82c59c SRV |
127 | pushl %ebp |
128 | pushl %edi | |
129 | pushl %esi | |
130 | pushl %edx | |
131 | pushl %ecx | |
132 | pushl %ebx | |
133 | ||
3d82c59c SRV |
134 | movl 12*4(%esp), %eax /* Load ip (1st parameter) */ |
135 | subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ | |
dc912c30 SRV |
136 | #ifdef CC_USING_FENTRY |
137 | movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */ | |
138 | #else | |
3d82c59c | 139 | movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ |
dc912c30 | 140 | #endif |
3d82c59c SRV |
141 | movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ |
142 | pushl %esp /* Save pt_regs as 4th parameter */ | |
143 | ||
144 | GLOBAL(ftrace_regs_call) | |
145 | call ftrace_stub | |
146 | ||
147 | addl $4, %esp /* Skip pt_regs */ | |
ff04b440 SRV |
148 | |
149 | /* restore flags */ | |
150 | push 14*4(%esp) | |
151 | popf | |
152 | ||
153 | /* Move return ip back to its original location */ | |
154 | movl 12*4(%esp), %eax | |
155 | movl %eax, 14*4(%esp) | |
3d82c59c SRV |
156 | |
157 | popl %ebx | |
158 | popl %ecx | |
159 | popl %edx | |
160 | popl %esi | |
161 | popl %edi | |
162 | popl %ebp | |
163 | popl %eax | |
164 | popl %ds | |
165 | popl %es | |
166 | popl %fs | |
167 | popl %gs | |
3d82c59c | 168 | |
ff04b440 SRV |
169 | /* use lea to not affect flags */ |
170 | lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */ | |
171 | ||
172 | jmp .Lftrace_ret | |
3d82c59c SRV |
173 | #else /* ! CONFIG_DYNAMIC_FTRACE */ |
174 | ||
644e0e8d | 175 | ENTRY(function_hook) |
3d82c59c SRV |
176 | cmpl $__PAGE_OFFSET, %esp |
177 | jb ftrace_stub /* Paging not enabled yet? */ | |
178 | ||
179 | cmpl $ftrace_stub, ftrace_trace_function | |
180 | jnz .Ltrace | |
181 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
182 | cmpl $ftrace_stub, ftrace_graph_return | |
183 | jnz ftrace_graph_caller | |
184 | ||
185 | cmpl $ftrace_graph_entry_stub, ftrace_graph_entry | |
186 | jnz ftrace_graph_caller | |
187 | #endif | |
188 | .globl ftrace_stub | |
189 | ftrace_stub: | |
190 | ret | |
191 | ||
192 | /* taken from glibc */ | |
193 | .Ltrace: | |
194 | pushl %eax | |
195 | pushl %ecx | |
196 | pushl %edx | |
197 | movl 0xc(%esp), %eax | |
198 | movl 0x4(%ebp), %edx | |
199 | subl $MCOUNT_INSN_SIZE, %eax | |
200 | ||
9351803b DW |
201 | movl ftrace_trace_function, %ecx |
202 | CALL_NOSPEC %ecx | |
3d82c59c SRV |
203 | |
204 | popl %edx | |
205 | popl %ecx | |
206 | popl %eax | |
207 | jmp ftrace_stub | |
644e0e8d | 208 | END(function_hook) |
3d82c59c | 209 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
3d82c59c SRV |
210 | |
211 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
212 | ENTRY(ftrace_graph_caller) | |
213 | pushl %eax | |
214 | pushl %ecx | |
215 | pushl %edx | |
644e0e8d SRV |
216 | movl 3*4(%esp), %eax |
217 | /* Even with frame pointers, fentry doesn't have one here */ | |
218 | #ifdef CC_USING_FENTRY | |
219 | lea 4*4(%esp), %edx | |
220 | movl $0, %ecx | |
221 | #else | |
3d82c59c SRV |
222 | lea 0x4(%ebp), %edx |
223 | movl (%ebp), %ecx | |
644e0e8d | 224 | #endif |
3d82c59c SRV |
225 | subl $MCOUNT_INSN_SIZE, %eax |
226 | call prepare_ftrace_return | |
227 | popl %edx | |
228 | popl %ecx | |
229 | popl %eax | |
230 | ret | |
231 | END(ftrace_graph_caller) | |
232 | ||
233 | .globl return_to_handler | |
234 | return_to_handler: | |
235 | pushl %eax | |
236 | pushl %edx | |
644e0e8d SRV |
237 | #ifdef CC_USING_FENTRY |
238 | movl $0, %eax | |
239 | #else | |
3d82c59c | 240 | movl %ebp, %eax |
644e0e8d | 241 | #endif |
3d82c59c SRV |
242 | call ftrace_return_to_handler |
243 | movl %eax, %ecx | |
244 | popl %edx | |
245 | popl %eax | |
9351803b | 246 | JMP_NOSPEC %ecx |
3d82c59c | 247 | #endif |