1 // SPDX-License-Identifier: GPL-2.0
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
8 * Highly modified by Steven Rostedt (VMware).
10 #include <linux/suspend.h>
11 #include <linux/ftrace.h>
12 #include <linux/slab.h>
14 #include <trace/events/sched.h>
16 #include "ftrace_internal.h"
18 #ifdef CONFIG_DYNAMIC_FTRACE
19 #define ASSIGN_OPS_HASH(opsname, val) \
21 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
23 #define ASSIGN_OPS_HASH(opsname, val)
26 static bool kill_ftrace_graph
;
27 int ftrace_graph_active
;
29 /* Both enabled by default (can be cleared by function_graph tracer flags */
30 static bool fgraph_sleep_time
= true;
33 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
35 * ftrace_graph_stop() is called when a severe error is detected in
36 * the function graph tracing. This function is called by the critical
37 * paths of function graph to keep those paths from doing any more harm.
39 bool ftrace_graph_is_dead(void)
41 return kill_ftrace_graph
;
45 * ftrace_graph_stop - set to permanently disable function graph tracing
47 * In case of an error int function graph tracing, this is called
48 * to try to keep function graph tracing from causing any more harm.
49 * Usually this is pretty severe and this is called to try to at least
50 * get a warning out to the user.
52 void ftrace_graph_stop(void)
54 kill_ftrace_graph
= true;
57 /* Add a function return address to the trace stack on thread info.*/
59 ftrace_push_return_trace(unsigned long ret
, unsigned long func
,
60 unsigned long frame_pointer
, unsigned long *retp
)
62 unsigned long long calltime
;
65 if (unlikely(ftrace_graph_is_dead()))
68 if (!current
->ret_stack
)
72 * We must make sure the ret_stack is tested before we read
77 /* The return trace stack is full */
78 if (current
->curr_ret_stack
== FTRACE_RETFUNC_DEPTH
- 1) {
79 atomic_inc(¤t
->trace_overrun
);
83 calltime
= trace_clock_local();
85 index
= ++current
->curr_ret_stack
;
87 current
->ret_stack
[index
].ret
= ret
;
88 current
->ret_stack
[index
].func
= func
;
89 current
->ret_stack
[index
].calltime
= calltime
;
90 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
91 current
->ret_stack
[index
].fp
= frame_pointer
;
93 #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
94 current
->ret_stack
[index
].retp
= retp
;
100 * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct
101 * functions. But those archs currently don't support direct functions
102 * anyway, and ftrace_find_rec_direct() is just a stub for them.
103 * Define MCOUNT_INSN_SIZE to keep those archs compiling.
105 #ifndef MCOUNT_INSN_SIZE
106 /* Make sure this only works without direct calls */
107 # ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
108 # error MCOUNT_INSN_SIZE not defined with direct calls enabled
110 # define MCOUNT_INSN_SIZE 0
113 int function_graph_enter(unsigned long ret
, unsigned long func
,
114 unsigned long frame_pointer
, unsigned long *retp
)
116 struct ftrace_graph_ent trace
;
119 * Skip graph tracing if the return location is served by direct trampoline,
120 * since call sequence and return addresses are unpredictable anyway.
121 * Ex: BPF trampoline may call original function and may skip frame
122 * depending on type of BPF programs attached.
124 if (ftrace_direct_func_count
&&
125 ftrace_find_rec_direct(ret
- MCOUNT_INSN_SIZE
))
128 trace
.depth
= ++current
->curr_ret_depth
;
130 if (ftrace_push_return_trace(ret
, func
, frame_pointer
, retp
))
133 /* Only trace if the calling function expects to */
134 if (!ftrace_graph_entry(&trace
))
139 current
->curr_ret_stack
--;
141 current
->curr_ret_depth
--;
145 /* Retrieve a function return address to the trace stack on thread info.*/
147 ftrace_pop_return_trace(struct ftrace_graph_ret
*trace
, unsigned long *ret
,
148 unsigned long frame_pointer
)
152 index
= current
->curr_ret_stack
;
154 if (unlikely(index
< 0 || index
>= FTRACE_RETFUNC_DEPTH
)) {
157 /* Might as well panic, otherwise we have no where to go */
158 *ret
= (unsigned long)panic
;
162 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
164 * The arch may choose to record the frame pointer used
165 * and check it here to make sure that it is what we expect it
166 * to be. If gcc does not set the place holder of the return
167 * address in the frame pointer, and does a copy instead, then
168 * the function graph trace will fail. This test detects this
171 * Currently, x86_32 with optimize for size (-Os) makes the latest
174 * Note, -mfentry does not use frame pointers, and this test
175 * is not needed if CC_USING_FENTRY is set.
177 if (unlikely(current
->ret_stack
[index
].fp
!= frame_pointer
)) {
179 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
180 " from func %ps return to %lx\n",
181 current
->ret_stack
[index
].fp
,
183 (void *)current
->ret_stack
[index
].func
,
184 current
->ret_stack
[index
].ret
);
185 *ret
= (unsigned long)panic
;
190 *ret
= current
->ret_stack
[index
].ret
;
191 trace
->func
= current
->ret_stack
[index
].func
;
192 trace
->calltime
= current
->ret_stack
[index
].calltime
;
193 trace
->overrun
= atomic_read(¤t
->trace_overrun
);
194 trace
->depth
= current
->curr_ret_depth
--;
196 * We still want to trace interrupts coming in if
197 * max_depth is set to 1. Make sure the decrement is
198 * seen before ftrace_graph_return.
204 * Hibernation protection.
205 * The state of the current task is too much unstable during
206 * suspend/restore to disk. We want to protect against that.
209 ftrace_suspend_notifier_call(struct notifier_block
*bl
, unsigned long state
,
213 case PM_HIBERNATION_PREPARE
:
214 pause_graph_tracing();
217 case PM_POST_HIBERNATION
:
218 unpause_graph_tracing();
224 static struct notifier_block ftrace_suspend_notifier
= {
225 .notifier_call
= ftrace_suspend_notifier_call
,
229 * Send the trace to the ring-buffer.
230 * @return the original return address.
232 unsigned long ftrace_return_to_handler(unsigned long frame_pointer
)
234 struct ftrace_graph_ret trace
;
237 ftrace_pop_return_trace(&trace
, &ret
, frame_pointer
);
238 trace
.rettime
= trace_clock_local();
239 ftrace_graph_return(&trace
);
241 * The ftrace_graph_return() may still access the current
242 * ret_stack structure, we need to make sure the update of
243 * curr_ret_stack is after that.
246 current
->curr_ret_stack
--;
248 if (unlikely(!ret
)) {
251 /* Might as well panic. What else to do? */
252 ret
= (unsigned long)panic
;
259 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
260 * @task: The task to read the shadow stack from
261 * @idx: Index down the shadow stack
263 * Return the ret_struct on the shadow stack of the @task at the
264 * call graph at @idx starting with zero. If @idx is zero, it
265 * will return the last saved ret_stack entry. If it is greater than
266 * zero, it will return the corresponding ret_stack for the depth
267 * of saved return addresses.
269 struct ftrace_ret_stack
*
270 ftrace_graph_get_ret_stack(struct task_struct
*task
, int idx
)
272 idx
= task
->curr_ret_stack
- idx
;
274 if (idx
>= 0 && idx
<= task
->curr_ret_stack
)
275 return &task
->ret_stack
[idx
];
281 * ftrace_graph_ret_addr - convert a potentially modified stack return address
282 * to its original value
284 * This function can be called by stack unwinding code to convert a found stack
285 * return address ('ret') to its original value, in case the function graph
286 * tracer has modified it to be 'return_to_handler'. If the address hasn't
287 * been modified, the unchanged value of 'ret' is returned.
289 * 'idx' is a state variable which should be initialized by the caller to zero
290 * before the first call.
292 * 'retp' is a pointer to the return address on the stack. It's ignored if
293 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
295 #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
296 unsigned long ftrace_graph_ret_addr(struct task_struct
*task
, int *idx
,
297 unsigned long ret
, unsigned long *retp
)
299 int index
= task
->curr_ret_stack
;
302 if (ret
!= (unsigned long)dereference_kernel_function_descriptor(return_to_handler
))
308 for (i
= 0; i
<= index
; i
++)
309 if (task
->ret_stack
[i
].retp
== retp
)
310 return task
->ret_stack
[i
].ret
;
314 #else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
315 unsigned long ftrace_graph_ret_addr(struct task_struct
*task
, int *idx
,
316 unsigned long ret
, unsigned long *retp
)
320 if (ret
!= (unsigned long)dereference_kernel_function_descriptor(return_to_handler
))
323 task_idx
= task
->curr_ret_stack
;
325 if (!task
->ret_stack
|| task_idx
< *idx
)
331 return task
->ret_stack
[task_idx
].ret
;
333 #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
335 static struct ftrace_ops graph_ops
= {
337 .flags
= FTRACE_OPS_FL_INITIALIZED
|
340 #ifdef FTRACE_GRAPH_TRAMP_ADDR
341 .trampoline
= FTRACE_GRAPH_TRAMP_ADDR
,
342 /* trampoline_size is only needed for dynamically allocated tramps */
344 ASSIGN_OPS_HASH(graph_ops
, &global_ops
.local_hash
)
347 void ftrace_graph_sleep_time_control(bool enable
)
349 fgraph_sleep_time
= enable
;
352 int ftrace_graph_entry_stub(struct ftrace_graph_ent
*trace
)
358 * Simply points to ftrace_stub, but with the proper protocol.
359 * Defined by the linker script in linux/vmlinux.lds.h
361 extern void ftrace_stub_graph(struct ftrace_graph_ret
*);
363 /* The callbacks that hook a function */
364 trace_func_graph_ret_t ftrace_graph_return
= ftrace_stub_graph
;
365 trace_func_graph_ent_t ftrace_graph_entry
= ftrace_graph_entry_stub
;
366 static trace_func_graph_ent_t __ftrace_graph_entry
= ftrace_graph_entry_stub
;
368 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
369 static int alloc_retstack_tasklist(struct ftrace_ret_stack
**ret_stack_list
)
373 int start
= 0, end
= FTRACE_RETSTACK_ALLOC_SIZE
;
374 struct task_struct
*g
, *t
;
376 for (i
= 0; i
< FTRACE_RETSTACK_ALLOC_SIZE
; i
++) {
378 kmalloc_array(FTRACE_RETFUNC_DEPTH
,
379 sizeof(struct ftrace_ret_stack
),
381 if (!ret_stack_list
[i
]) {
390 for_each_process_thread(g
, t
) {
396 if (t
->ret_stack
== NULL
) {
397 atomic_set(&t
->trace_overrun
, 0);
398 t
->curr_ret_stack
= -1;
399 t
->curr_ret_depth
= -1;
400 /* Make sure the tasks see the -1 first: */
402 t
->ret_stack
= ret_stack_list
[start
++];
409 for (i
= start
; i
< end
; i
++)
410 kfree(ret_stack_list
[i
]);
415 ftrace_graph_probe_sched_switch(void *ignore
, bool preempt
,
416 struct task_struct
*prev
, struct task_struct
*next
)
418 unsigned long long timestamp
;
422 * Does the user want to count the time a function was asleep.
423 * If so, do not update the time stamps.
425 if (fgraph_sleep_time
)
428 timestamp
= trace_clock_local();
430 prev
->ftrace_timestamp
= timestamp
;
432 /* only process tasks that we timestamped */
433 if (!next
->ftrace_timestamp
)
437 * Update all the counters in next to make up for the
438 * time next was sleeping.
440 timestamp
-= next
->ftrace_timestamp
;
442 for (index
= next
->curr_ret_stack
; index
>= 0; index
--)
443 next
->ret_stack
[index
].calltime
+= timestamp
;
446 static int ftrace_graph_entry_test(struct ftrace_graph_ent
*trace
)
448 if (!ftrace_ops_test(&global_ops
, trace
->func
, NULL
))
450 return __ftrace_graph_entry(trace
);
454 * The function graph tracer should only trace the functions defined
455 * by set_ftrace_filter and set_ftrace_notrace. If another function
456 * tracer ops is registered, the graph tracer requires testing the
457 * function against the global ops, and not just trace any function
458 * that any ftrace_ops registered.
460 void update_function_graph_func(void)
462 struct ftrace_ops
*op
;
463 bool do_test
= false;
466 * The graph and global ops share the same set of functions
467 * to test. If any other ops is on the list, then
468 * the graph tracing needs to test if its the function
471 do_for_each_ftrace_op(op
, ftrace_ops_list
) {
472 if (op
!= &global_ops
&& op
!= &graph_ops
&&
473 op
!= &ftrace_list_end
) {
475 /* in double loop, break out with goto */
478 } while_for_each_ftrace_op(op
);
481 ftrace_graph_entry
= ftrace_graph_entry_test
;
483 ftrace_graph_entry
= __ftrace_graph_entry
;
486 static DEFINE_PER_CPU(struct ftrace_ret_stack
*, idle_ret_stack
);
489 graph_init_task(struct task_struct
*t
, struct ftrace_ret_stack
*ret_stack
)
491 atomic_set(&t
->trace_overrun
, 0);
492 t
->ftrace_timestamp
= 0;
493 /* make curr_ret_stack visible before we add the ret_stack */
495 t
->ret_stack
= ret_stack
;
499 * Allocate a return stack for the idle task. May be the first
500 * time through, or it may be done by CPU hotplug online.
502 void ftrace_graph_init_idle_task(struct task_struct
*t
, int cpu
)
504 t
->curr_ret_stack
= -1;
505 t
->curr_ret_depth
= -1;
507 * The idle task has no parent, it either has its own
508 * stack or no stack at all.
511 WARN_ON(t
->ret_stack
!= per_cpu(idle_ret_stack
, cpu
));
513 if (ftrace_graph_active
) {
514 struct ftrace_ret_stack
*ret_stack
;
516 ret_stack
= per_cpu(idle_ret_stack
, cpu
);
519 kmalloc_array(FTRACE_RETFUNC_DEPTH
,
520 sizeof(struct ftrace_ret_stack
),
524 per_cpu(idle_ret_stack
, cpu
) = ret_stack
;
526 graph_init_task(t
, ret_stack
);
530 /* Allocate a return stack for newly created task */
531 void ftrace_graph_init_task(struct task_struct
*t
)
533 /* Make sure we do not use the parent ret_stack */
535 t
->curr_ret_stack
= -1;
536 t
->curr_ret_depth
= -1;
538 if (ftrace_graph_active
) {
539 struct ftrace_ret_stack
*ret_stack
;
541 ret_stack
= kmalloc_array(FTRACE_RETFUNC_DEPTH
,
542 sizeof(struct ftrace_ret_stack
),
546 graph_init_task(t
, ret_stack
);
550 void ftrace_graph_exit_task(struct task_struct
*t
)
552 struct ftrace_ret_stack
*ret_stack
= t
->ret_stack
;
555 /* NULL must become visible to IRQs before we free it: */
561 /* Allocate a return stack for each task */
562 static int start_graph_tracing(void)
564 struct ftrace_ret_stack
**ret_stack_list
;
567 ret_stack_list
= kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE
,
568 sizeof(struct ftrace_ret_stack
*),
574 /* The cpu_boot init_task->ret_stack will never be freed */
575 for_each_online_cpu(cpu
) {
576 if (!idle_task(cpu
)->ret_stack
)
577 ftrace_graph_init_idle_task(idle_task(cpu
), cpu
);
581 ret
= alloc_retstack_tasklist(ret_stack_list
);
582 } while (ret
== -EAGAIN
);
585 ret
= register_trace_sched_switch(ftrace_graph_probe_sched_switch
, NULL
);
587 pr_info("ftrace_graph: Couldn't activate tracepoint"
588 " probe to kernel_sched_switch\n");
591 kfree(ret_stack_list
);
595 int register_ftrace_graph(struct fgraph_ops
*gops
)
599 mutex_lock(&ftrace_lock
);
601 /* we currently allow only one tracer registered at a time */
602 if (ftrace_graph_active
) {
607 register_pm_notifier(&ftrace_suspend_notifier
);
609 ftrace_graph_active
++;
610 ret
= start_graph_tracing();
612 ftrace_graph_active
--;
616 ftrace_graph_return
= gops
->retfunc
;
619 * Update the indirect function to the entryfunc, and the
620 * function that gets called to the entry_test first. Then
621 * call the update fgraph entry function to determine if
622 * the entryfunc should be called directly or not.
624 __ftrace_graph_entry
= gops
->entryfunc
;
625 ftrace_graph_entry
= ftrace_graph_entry_test
;
626 update_function_graph_func();
628 ret
= ftrace_startup(&graph_ops
, FTRACE_START_FUNC_RET
);
630 mutex_unlock(&ftrace_lock
);
634 void unregister_ftrace_graph(struct fgraph_ops
*gops
)
636 mutex_lock(&ftrace_lock
);
638 if (unlikely(!ftrace_graph_active
))
641 ftrace_graph_active
--;
642 ftrace_graph_return
= ftrace_stub_graph
;
643 ftrace_graph_entry
= ftrace_graph_entry_stub
;
644 __ftrace_graph_entry
= ftrace_graph_entry_stub
;
645 ftrace_shutdown(&graph_ops
, FTRACE_STOP_FUNC_RET
);
646 unregister_pm_notifier(&ftrace_suspend_notifier
);
647 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch
, NULL
);
650 mutex_unlock(&ftrace_lock
);