2 * Copyright (C) 2017 Steven Rostedt, VMware Inc.
5 #include <linux/linkage.h>
6 #include <asm/page_types.h>
7 #include <asm/segment.h>
8 #include <asm/export.h>
9 #include <asm/ftrace.h>
10 #include <asm/nospec-branch.h>
12 #ifdef CC_USING_FENTRY
13 # define function_hook __fentry__
14 EXPORT_SYMBOL(__fentry__)
16 # define function_hook mcount
20 #ifdef CONFIG_DYNAMIC_FTRACE
22 /* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
23 #if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
24 # define USING_FRAME_POINTER
27 #ifdef USING_FRAME_POINTER
28 # define MCOUNT_FRAME 1 /* using frame = true */
30 # define MCOUNT_FRAME 0 /* using frame = false */
39 #ifdef USING_FRAME_POINTER
40 # ifdef CC_USING_FENTRY
42 * Frame pointers are of ip followed by bp.
43 * Since fentry is an immediate jump, we are left with
44 * parent-ip, function-ip. We need to add a frame with
45 * parent-ip followed by ebp.
47 pushl 4(%esp) /* parent ip */
50 pushl 2*4(%esp) /* function ip */
52 /* For mcount, the function ip is directly above */
59 pushl $0 /* Pass NULL as regs pointer */
61 #ifdef USING_FRAME_POINTER
62 /* Load parent ebp into edx */
65 /* There's no frame pointer, load the appropriate stack addr instead */
69 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
70 /* Get the parent ip */
71 movl 4(%edx), %edx /* edx has ebp */
73 movl function_trace_op, %ecx
74 subl $MCOUNT_INSN_SIZE, %eax
80 addl $4, %esp /* skip NULL pointer */
84 #ifdef USING_FRAME_POINTER
86 # ifdef CC_USING_FENTRY
87 addl $4,%esp /* skip function ip */
88 popl %ebp /* this is the orig bp */
89 addl $4, %esp /* skip parent ip */
93 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
94 .globl ftrace_graph_call
99 /* This is weak to keep gas from relaxing the jumps */
104 ENTRY(ftrace_regs_caller)
106 * i386 does not save SS and ESP when coming from kernel.
107 * Instead, to get sp, ®s->sp is used (see ptrace.h).
108 * Unfortunately, that means eflags must be at the same location
109 * as the current return ip is. We move the return ip into the
110 * regs->ip location, and move flags into the return ip location.
113 pushl 4(%esp) /* Save the return ip */
114 pushl $0 /* Load 0 into orig_ax */
121 /* Get flags and place them into the return ip slot */
133 movl 12*4(%esp), %eax /* Load ip (1st parameter) */
134 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
135 #ifdef CC_USING_FENTRY
136 movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */
138 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
140 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
141 pushl %esp /* Save pt_regs as 4th parameter */
143 GLOBAL(ftrace_regs_call)
146 addl $4, %esp /* Skip pt_regs */
152 /* Move return ip back to its original location */
153 movl 12*4(%esp), %eax
154 movl %eax, 14*4(%esp)
168 /* use lea to not affect flags */
169 lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */
172 #else /* ! CONFIG_DYNAMIC_FTRACE */
175 cmpl $__PAGE_OFFSET, %esp
176 jb ftrace_stub /* Paging not enabled yet? */
178 cmpl $ftrace_stub, ftrace_trace_function
180 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
181 cmpl $ftrace_stub, ftrace_graph_return
182 jnz ftrace_graph_caller
184 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
185 jnz ftrace_graph_caller
191 /* taken from glibc */
198 subl $MCOUNT_INSN_SIZE, %eax
200 movl ftrace_trace_function, %ecx
208 #endif /* CONFIG_DYNAMIC_FTRACE */
210 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
211 ENTRY(ftrace_graph_caller)
216 /* Even with frame pointers, fentry doesn't have one here */
217 #ifdef CC_USING_FENTRY
224 subl $MCOUNT_INSN_SIZE, %eax
225 call prepare_ftrace_return
230 END(ftrace_graph_caller)
232 .globl return_to_handler
236 #ifdef CC_USING_FENTRY
241 call ftrace_return_to_handler