2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
32 #include <trace/sched.h>
34 #include <asm/ftrace.h>
38 #define FTRACE_WARN_ON(cond) \
44 #define FTRACE_WARN_ON_ONCE(cond) \
46 if (WARN_ON_ONCE(cond)) \
50 /* hash bits for specific function selection */
51 #define FTRACE_HASH_BITS 7
52 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
54 /* ftrace_enabled is a method to turn ftrace on or off */
55 int ftrace_enabled __read_mostly
;
56 static int last_ftrace_enabled
;
58 /* Quick disabling of function tracer. */
59 int function_trace_stop
;
62 * ftrace_disabled is set when an anomaly is discovered.
63 * ftrace_disabled is much stronger than ftrace_enabled.
65 static int ftrace_disabled __read_mostly
;
67 static DEFINE_MUTEX(ftrace_lock
);
69 static struct ftrace_ops ftrace_list_end __read_mostly
=
74 static struct ftrace_ops
*ftrace_list __read_mostly
= &ftrace_list_end
;
75 ftrace_func_t ftrace_trace_function __read_mostly
= ftrace_stub
;
76 ftrace_func_t __ftrace_trace_function __read_mostly
= ftrace_stub
;
77 ftrace_func_t ftrace_pid_function __read_mostly
= ftrace_stub
;
79 static void ftrace_list_func(unsigned long ip
, unsigned long parent_ip
)
81 struct ftrace_ops
*op
= ftrace_list
;
83 /* in case someone actually ports this to alpha! */
84 read_barrier_depends();
86 while (op
!= &ftrace_list_end
) {
88 read_barrier_depends();
89 op
->func(ip
, parent_ip
);
94 static void ftrace_pid_func(unsigned long ip
, unsigned long parent_ip
)
96 if (!test_tsk_trace_trace(current
))
99 ftrace_pid_function(ip
, parent_ip
);
102 static void set_ftrace_pid_function(ftrace_func_t func
)
104 /* do not set ftrace_pid_function to itself! */
105 if (func
!= ftrace_pid_func
)
106 ftrace_pid_function
= func
;
110 * clear_ftrace_function - reset the ftrace function
112 * This NULLs the ftrace function and in essence stops
113 * tracing. There may be lag
115 void clear_ftrace_function(void)
117 ftrace_trace_function
= ftrace_stub
;
118 __ftrace_trace_function
= ftrace_stub
;
119 ftrace_pid_function
= ftrace_stub
;
122 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
124 * For those archs that do not test ftrace_trace_stop in their
125 * mcount call site, we need to do it from C.
127 static void ftrace_test_stop_func(unsigned long ip
, unsigned long parent_ip
)
129 if (function_trace_stop
)
132 __ftrace_trace_function(ip
, parent_ip
);
136 static int __register_ftrace_function(struct ftrace_ops
*ops
)
138 ops
->next
= ftrace_list
;
140 * We are entering ops into the ftrace_list but another
141 * CPU might be walking that list. We need to make sure
142 * the ops->next pointer is valid before another CPU sees
143 * the ops pointer included into the ftrace_list.
148 if (ftrace_enabled
) {
151 if (ops
->next
== &ftrace_list_end
)
154 func
= ftrace_list_func
;
156 if (ftrace_pid_trace
) {
157 set_ftrace_pid_function(func
);
158 func
= ftrace_pid_func
;
162 * For one func, simply call it directly.
163 * For more than one func, call the chain.
165 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
166 ftrace_trace_function
= func
;
168 __ftrace_trace_function
= func
;
169 ftrace_trace_function
= ftrace_test_stop_func
;
176 static int __unregister_ftrace_function(struct ftrace_ops
*ops
)
178 struct ftrace_ops
**p
;
181 * If we are removing the last function, then simply point
182 * to the ftrace_stub.
184 if (ftrace_list
== ops
&& ops
->next
== &ftrace_list_end
) {
185 ftrace_trace_function
= ftrace_stub
;
186 ftrace_list
= &ftrace_list_end
;
190 for (p
= &ftrace_list
; *p
!= &ftrace_list_end
; p
= &(*p
)->next
)
199 if (ftrace_enabled
) {
200 /* If we only have one func left, then call that directly */
201 if (ftrace_list
->next
== &ftrace_list_end
) {
202 ftrace_func_t func
= ftrace_list
->func
;
204 if (ftrace_pid_trace
) {
205 set_ftrace_pid_function(func
);
206 func
= ftrace_pid_func
;
208 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
209 ftrace_trace_function
= func
;
211 __ftrace_trace_function
= func
;
219 static void ftrace_update_pid_func(void)
223 if (ftrace_trace_function
== ftrace_stub
)
226 func
= ftrace_trace_function
;
228 if (ftrace_pid_trace
) {
229 set_ftrace_pid_function(func
);
230 func
= ftrace_pid_func
;
232 if (func
== ftrace_pid_func
)
233 func
= ftrace_pid_function
;
236 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
237 ftrace_trace_function
= func
;
239 __ftrace_trace_function
= func
;
243 /* set when tracing only a pid */
244 struct pid
*ftrace_pid_trace
;
245 static struct pid
* const ftrace_swapper_pid
= &init_struct_pid
;
247 #ifdef CONFIG_DYNAMIC_FTRACE
249 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
250 # error Dynamic ftrace depends on MCOUNT_RECORD
253 static struct hlist_head ftrace_func_hash
[FTRACE_FUNC_HASHSIZE
] __read_mostly
;
255 struct ftrace_func_probe
{
256 struct hlist_node node
;
257 struct ftrace_probe_ops
*ops
;
266 FTRACE_ENABLE_CALLS
= (1 << 0),
267 FTRACE_DISABLE_CALLS
= (1 << 1),
268 FTRACE_UPDATE_TRACE_FUNC
= (1 << 2),
269 FTRACE_ENABLE_MCOUNT
= (1 << 3),
270 FTRACE_DISABLE_MCOUNT
= (1 << 4),
271 FTRACE_START_FUNC_RET
= (1 << 5),
272 FTRACE_STOP_FUNC_RET
= (1 << 6),
275 static int ftrace_filtered
;
277 static struct dyn_ftrace
*ftrace_new_addrs
;
279 static DEFINE_MUTEX(ftrace_regex_lock
);
282 struct ftrace_page
*next
;
284 struct dyn_ftrace records
[];
287 #define ENTRIES_PER_PAGE \
288 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
290 /* estimate from running different kernels */
291 #define NR_TO_INIT 10000
293 static struct ftrace_page
*ftrace_pages_start
;
294 static struct ftrace_page
*ftrace_pages
;
296 static struct dyn_ftrace
*ftrace_free_records
;
299 * This is a double for. Do not use 'break' to break out of the loop,
300 * you must use a goto.
302 #define do_for_each_ftrace_rec(pg, rec) \
303 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
305 for (_____i = 0; _____i < pg->index; _____i++) { \
306 rec = &pg->records[_____i];
308 #define while_for_each_ftrace_rec() \
312 #ifdef CONFIG_KPROBES
314 static int frozen_record_count
;
316 static inline void freeze_record(struct dyn_ftrace
*rec
)
318 if (!(rec
->flags
& FTRACE_FL_FROZEN
)) {
319 rec
->flags
|= FTRACE_FL_FROZEN
;
320 frozen_record_count
++;
324 static inline void unfreeze_record(struct dyn_ftrace
*rec
)
326 if (rec
->flags
& FTRACE_FL_FROZEN
) {
327 rec
->flags
&= ~FTRACE_FL_FROZEN
;
328 frozen_record_count
--;
332 static inline int record_frozen(struct dyn_ftrace
*rec
)
334 return rec
->flags
& FTRACE_FL_FROZEN
;
337 # define freeze_record(rec) ({ 0; })
338 # define unfreeze_record(rec) ({ 0; })
339 # define record_frozen(rec) ({ 0; })
340 #endif /* CONFIG_KPROBES */
342 static void ftrace_free_rec(struct dyn_ftrace
*rec
)
344 rec
->ip
= (unsigned long)ftrace_free_records
;
345 ftrace_free_records
= rec
;
346 rec
->flags
|= FTRACE_FL_FREE
;
349 void ftrace_release(void *start
, unsigned long size
)
351 struct dyn_ftrace
*rec
;
352 struct ftrace_page
*pg
;
353 unsigned long s
= (unsigned long)start
;
354 unsigned long e
= s
+ size
;
356 if (ftrace_disabled
|| !start
)
359 mutex_lock(&ftrace_lock
);
360 do_for_each_ftrace_rec(pg
, rec
) {
361 if ((rec
->ip
>= s
) && (rec
->ip
< e
) &&
362 !(rec
->flags
& FTRACE_FL_FREE
))
363 ftrace_free_rec(rec
);
364 } while_for_each_ftrace_rec();
365 mutex_unlock(&ftrace_lock
);
368 static struct dyn_ftrace
*ftrace_alloc_dyn_node(unsigned long ip
)
370 struct dyn_ftrace
*rec
;
372 /* First check for freed records */
373 if (ftrace_free_records
) {
374 rec
= ftrace_free_records
;
376 if (unlikely(!(rec
->flags
& FTRACE_FL_FREE
))) {
377 FTRACE_WARN_ON_ONCE(1);
378 ftrace_free_records
= NULL
;
382 ftrace_free_records
= (void *)rec
->ip
;
383 memset(rec
, 0, sizeof(*rec
));
387 if (ftrace_pages
->index
== ENTRIES_PER_PAGE
) {
388 if (!ftrace_pages
->next
) {
389 /* allocate another page */
391 (void *)get_zeroed_page(GFP_KERNEL
);
392 if (!ftrace_pages
->next
)
395 ftrace_pages
= ftrace_pages
->next
;
398 return &ftrace_pages
->records
[ftrace_pages
->index
++];
401 static struct dyn_ftrace
*
402 ftrace_record_ip(unsigned long ip
)
404 struct dyn_ftrace
*rec
;
409 rec
= ftrace_alloc_dyn_node(ip
);
414 rec
->flags
= (unsigned long)ftrace_new_addrs
;
415 ftrace_new_addrs
= rec
;
420 static void print_ip_ins(const char *fmt
, unsigned char *p
)
424 printk(KERN_CONT
"%s", fmt
);
426 for (i
= 0; i
< MCOUNT_INSN_SIZE
; i
++)
427 printk(KERN_CONT
"%s%02x", i
? ":" : "", p
[i
]);
430 static void ftrace_bug(int failed
, unsigned long ip
)
434 FTRACE_WARN_ON_ONCE(1);
435 pr_info("ftrace faulted on modifying ");
439 FTRACE_WARN_ON_ONCE(1);
440 pr_info("ftrace failed to modify ");
442 print_ip_ins(" actual: ", (unsigned char *)ip
);
443 printk(KERN_CONT
"\n");
446 FTRACE_WARN_ON_ONCE(1);
447 pr_info("ftrace faulted on writing ");
451 FTRACE_WARN_ON_ONCE(1);
452 pr_info("ftrace faulted on unknown error ");
459 __ftrace_replace_code(struct dyn_ftrace
*rec
, int enable
)
461 unsigned long ftrace_addr
;
462 unsigned long ip
, fl
;
464 ftrace_addr
= (unsigned long)FTRACE_ADDR
;
469 * If this record is not to be traced and
470 * it is not enabled then do nothing.
472 * If this record is not to be traced and
473 * it is enabled then disable it.
476 if (rec
->flags
& FTRACE_FL_NOTRACE
) {
477 if (rec
->flags
& FTRACE_FL_ENABLED
)
478 rec
->flags
&= ~FTRACE_FL_ENABLED
;
482 } else if (ftrace_filtered
&& enable
) {
487 fl
= rec
->flags
& (FTRACE_FL_FILTER
| FTRACE_FL_ENABLED
);
489 /* Record is filtered and enabled, do nothing */
490 if (fl
== (FTRACE_FL_FILTER
| FTRACE_FL_ENABLED
))
493 /* Record is not filtered or enabled, do nothing */
497 /* Record is not filtered but enabled, disable it */
498 if (fl
== FTRACE_FL_ENABLED
)
499 rec
->flags
&= ~FTRACE_FL_ENABLED
;
501 /* Otherwise record is filtered but not enabled, enable it */
502 rec
->flags
|= FTRACE_FL_ENABLED
;
504 /* Disable or not filtered */
507 /* if record is enabled, do nothing */
508 if (rec
->flags
& FTRACE_FL_ENABLED
)
511 rec
->flags
|= FTRACE_FL_ENABLED
;
515 /* if record is not enabled, do nothing */
516 if (!(rec
->flags
& FTRACE_FL_ENABLED
))
519 rec
->flags
&= ~FTRACE_FL_ENABLED
;
523 if (rec
->flags
& FTRACE_FL_ENABLED
)
524 return ftrace_make_call(rec
, ftrace_addr
);
526 return ftrace_make_nop(NULL
, rec
, ftrace_addr
);
529 static void ftrace_replace_code(int enable
)
531 struct dyn_ftrace
*rec
;
532 struct ftrace_page
*pg
;
535 do_for_each_ftrace_rec(pg
, rec
) {
537 * Skip over free records, records that have
538 * failed and not converted.
540 if (rec
->flags
& FTRACE_FL_FREE
||
541 rec
->flags
& FTRACE_FL_FAILED
||
542 !(rec
->flags
& FTRACE_FL_CONVERTED
))
545 /* ignore updates to this record's mcount site */
546 if (get_kprobe((void *)rec
->ip
)) {
550 unfreeze_record(rec
);
553 failed
= __ftrace_replace_code(rec
, enable
);
555 rec
->flags
|= FTRACE_FL_FAILED
;
556 if ((system_state
== SYSTEM_BOOTING
) ||
557 !core_kernel_text(rec
->ip
)) {
558 ftrace_free_rec(rec
);
560 ftrace_bug(failed
, rec
->ip
);
561 /* Stop processing */
565 } while_for_each_ftrace_rec();
569 ftrace_code_disable(struct module
*mod
, struct dyn_ftrace
*rec
)
576 ret
= ftrace_make_nop(mod
, rec
, MCOUNT_ADDR
);
579 rec
->flags
|= FTRACE_FL_FAILED
;
586 * archs can override this function if they must do something
587 * before the modifying code is performed.
589 int __weak
ftrace_arch_code_modify_prepare(void)
595 * archs can override this function if they must do something
596 * after the modifying code is performed.
598 int __weak
ftrace_arch_code_modify_post_process(void)
603 static int __ftrace_modify_code(void *data
)
607 if (*command
& FTRACE_ENABLE_CALLS
)
608 ftrace_replace_code(1);
609 else if (*command
& FTRACE_DISABLE_CALLS
)
610 ftrace_replace_code(0);
612 if (*command
& FTRACE_UPDATE_TRACE_FUNC
)
613 ftrace_update_ftrace_func(ftrace_trace_function
);
615 if (*command
& FTRACE_START_FUNC_RET
)
616 ftrace_enable_ftrace_graph_caller();
617 else if (*command
& FTRACE_STOP_FUNC_RET
)
618 ftrace_disable_ftrace_graph_caller();
623 static void ftrace_run_update_code(int command
)
627 ret
= ftrace_arch_code_modify_prepare();
632 stop_machine(__ftrace_modify_code
, &command
, NULL
);
634 ret
= ftrace_arch_code_modify_post_process();
638 static ftrace_func_t saved_ftrace_func
;
639 static int ftrace_start_up
;
641 static void ftrace_startup_enable(int command
)
643 if (saved_ftrace_func
!= ftrace_trace_function
) {
644 saved_ftrace_func
= ftrace_trace_function
;
645 command
|= FTRACE_UPDATE_TRACE_FUNC
;
648 if (!command
|| !ftrace_enabled
)
651 ftrace_run_update_code(command
);
654 static void ftrace_startup(int command
)
656 if (unlikely(ftrace_disabled
))
660 command
|= FTRACE_ENABLE_CALLS
;
662 ftrace_startup_enable(command
);
665 static void ftrace_shutdown(int command
)
667 if (unlikely(ftrace_disabled
))
671 if (!ftrace_start_up
)
672 command
|= FTRACE_DISABLE_CALLS
;
674 if (saved_ftrace_func
!= ftrace_trace_function
) {
675 saved_ftrace_func
= ftrace_trace_function
;
676 command
|= FTRACE_UPDATE_TRACE_FUNC
;
679 if (!command
|| !ftrace_enabled
)
682 ftrace_run_update_code(command
);
685 static void ftrace_startup_sysctl(void)
687 int command
= FTRACE_ENABLE_MCOUNT
;
689 if (unlikely(ftrace_disabled
))
692 /* Force update next time */
693 saved_ftrace_func
= NULL
;
694 /* ftrace_start_up is true if we want ftrace running */
696 command
|= FTRACE_ENABLE_CALLS
;
698 ftrace_run_update_code(command
);
701 static void ftrace_shutdown_sysctl(void)
703 int command
= FTRACE_DISABLE_MCOUNT
;
705 if (unlikely(ftrace_disabled
))
708 /* ftrace_start_up is true if ftrace is running */
710 command
|= FTRACE_DISABLE_CALLS
;
712 ftrace_run_update_code(command
);
715 static cycle_t ftrace_update_time
;
716 static unsigned long ftrace_update_cnt
;
717 unsigned long ftrace_update_tot_cnt
;
719 static int ftrace_update_code(struct module
*mod
)
721 struct dyn_ftrace
*p
;
724 start
= ftrace_now(raw_smp_processor_id());
725 ftrace_update_cnt
= 0;
727 while (ftrace_new_addrs
) {
729 /* If something went wrong, bail without enabling anything */
730 if (unlikely(ftrace_disabled
))
733 p
= ftrace_new_addrs
;
734 ftrace_new_addrs
= (struct dyn_ftrace
*)p
->flags
;
737 /* convert record (i.e, patch mcount-call with NOP) */
738 if (ftrace_code_disable(mod
, p
)) {
739 p
->flags
|= FTRACE_FL_CONVERTED
;
745 stop
= ftrace_now(raw_smp_processor_id());
746 ftrace_update_time
= stop
- start
;
747 ftrace_update_tot_cnt
+= ftrace_update_cnt
;
752 static int __init
ftrace_dyn_table_alloc(unsigned long num_to_init
)
754 struct ftrace_page
*pg
;
758 /* allocate a few pages */
759 ftrace_pages_start
= (void *)get_zeroed_page(GFP_KERNEL
);
760 if (!ftrace_pages_start
)
764 * Allocate a few more pages.
766 * TODO: have some parser search vmlinux before
767 * final linking to find all calls to ftrace.
769 * a) know how many pages to allocate.
771 * b) set up the table then.
773 * The dynamic code is still necessary for
777 pg
= ftrace_pages
= ftrace_pages_start
;
779 cnt
= num_to_init
/ ENTRIES_PER_PAGE
;
780 pr_info("ftrace: allocating %ld entries in %d pages\n",
781 num_to_init
, cnt
+ 1);
783 for (i
= 0; i
< cnt
; i
++) {
784 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
786 /* If we fail, we'll try later anyway */
797 FTRACE_ITER_FILTER
= (1 << 0),
798 FTRACE_ITER_CONT
= (1 << 1),
799 FTRACE_ITER_NOTRACE
= (1 << 2),
800 FTRACE_ITER_FAILURES
= (1 << 3),
801 FTRACE_ITER_PRINTALL
= (1 << 4),
802 FTRACE_ITER_HASH
= (1 << 5),
805 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
807 struct ftrace_iterator
{
808 struct ftrace_page
*pg
;
812 unsigned char buffer
[FTRACE_BUFF_MAX
+1];
818 t_hash_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
820 struct ftrace_iterator
*iter
= m
->private;
821 struct hlist_node
*hnd
= v
;
822 struct hlist_head
*hhd
;
824 WARN_ON(!(iter
->flags
& FTRACE_ITER_HASH
));
829 if (iter
->hidx
>= FTRACE_FUNC_HASHSIZE
)
832 hhd
= &ftrace_func_hash
[iter
->hidx
];
834 if (hlist_empty(hhd
)) {
853 static void *t_hash_start(struct seq_file
*m
, loff_t
*pos
)
855 struct ftrace_iterator
*iter
= m
->private;
858 iter
->flags
|= FTRACE_ITER_HASH
;
860 return t_hash_next(m
, p
, pos
);
863 static int t_hash_show(struct seq_file
*m
, void *v
)
865 struct ftrace_func_probe
*rec
;
866 struct hlist_node
*hnd
= v
;
867 char str
[KSYM_SYMBOL_LEN
];
869 rec
= hlist_entry(hnd
, struct ftrace_func_probe
, node
);
872 return rec
->ops
->print(m
, rec
->ip
, rec
->ops
, rec
->data
);
874 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
875 seq_printf(m
, "%s:", str
);
877 kallsyms_lookup((unsigned long)rec
->ops
->func
, NULL
, NULL
, NULL
, str
);
878 seq_printf(m
, "%s", str
);
881 seq_printf(m
, ":%p", rec
->data
);
888 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
890 struct ftrace_iterator
*iter
= m
->private;
891 struct dyn_ftrace
*rec
= NULL
;
893 if (iter
->flags
& FTRACE_ITER_HASH
)
894 return t_hash_next(m
, v
, pos
);
898 if (iter
->flags
& FTRACE_ITER_PRINTALL
)
902 if (iter
->idx
>= iter
->pg
->index
) {
903 if (iter
->pg
->next
) {
904 iter
->pg
= iter
->pg
->next
;
911 rec
= &iter
->pg
->records
[iter
->idx
++];
912 if ((rec
->flags
& FTRACE_FL_FREE
) ||
914 (!(iter
->flags
& FTRACE_ITER_FAILURES
) &&
915 (rec
->flags
& FTRACE_FL_FAILED
)) ||
917 ((iter
->flags
& FTRACE_ITER_FAILURES
) &&
918 !(rec
->flags
& FTRACE_FL_FAILED
)) ||
920 ((iter
->flags
& FTRACE_ITER_FILTER
) &&
921 !(rec
->flags
& FTRACE_FL_FILTER
)) ||
923 ((iter
->flags
& FTRACE_ITER_NOTRACE
) &&
924 !(rec
->flags
& FTRACE_FL_NOTRACE
))) {
933 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
935 struct ftrace_iterator
*iter
= m
->private;
938 mutex_lock(&ftrace_lock
);
940 * For set_ftrace_filter reading, if we have the filter
941 * off, we can short cut and just print out that all
942 * functions are enabled.
944 if (iter
->flags
& FTRACE_ITER_FILTER
&& !ftrace_filtered
) {
946 return t_hash_start(m
, pos
);
947 iter
->flags
|= FTRACE_ITER_PRINTALL
;
952 if (iter
->flags
& FTRACE_ITER_HASH
)
953 return t_hash_start(m
, pos
);
962 p
= t_next(m
, p
, pos
);
965 return t_hash_start(m
, pos
);
970 static void t_stop(struct seq_file
*m
, void *p
)
972 mutex_unlock(&ftrace_lock
);
975 static int t_show(struct seq_file
*m
, void *v
)
977 struct ftrace_iterator
*iter
= m
->private;
978 struct dyn_ftrace
*rec
= v
;
979 char str
[KSYM_SYMBOL_LEN
];
981 if (iter
->flags
& FTRACE_ITER_HASH
)
982 return t_hash_show(m
, v
);
984 if (iter
->flags
& FTRACE_ITER_PRINTALL
) {
985 seq_printf(m
, "#### all functions enabled ####\n");
992 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
994 seq_printf(m
, "%s\n", str
);
999 static struct seq_operations show_ftrace_seq_ops
= {
1007 ftrace_avail_open(struct inode
*inode
, struct file
*file
)
1009 struct ftrace_iterator
*iter
;
1012 if (unlikely(ftrace_disabled
))
1015 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1019 iter
->pg
= ftrace_pages_start
;
1021 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1023 struct seq_file
*m
= file
->private_data
;
1033 int ftrace_avail_release(struct inode
*inode
, struct file
*file
)
1035 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
1036 struct ftrace_iterator
*iter
= m
->private;
1038 seq_release(inode
, file
);
1045 ftrace_failures_open(struct inode
*inode
, struct file
*file
)
1049 struct ftrace_iterator
*iter
;
1051 ret
= ftrace_avail_open(inode
, file
);
1053 m
= (struct seq_file
*)file
->private_data
;
1054 iter
= (struct ftrace_iterator
*)m
->private;
1055 iter
->flags
= FTRACE_ITER_FAILURES
;
1062 static void ftrace_filter_reset(int enable
)
1064 struct ftrace_page
*pg
;
1065 struct dyn_ftrace
*rec
;
1066 unsigned long type
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1068 mutex_lock(&ftrace_lock
);
1070 ftrace_filtered
= 0;
1071 do_for_each_ftrace_rec(pg
, rec
) {
1072 if (rec
->flags
& FTRACE_FL_FAILED
)
1074 rec
->flags
&= ~type
;
1075 } while_for_each_ftrace_rec();
1076 mutex_unlock(&ftrace_lock
);
1080 ftrace_regex_open(struct inode
*inode
, struct file
*file
, int enable
)
1082 struct ftrace_iterator
*iter
;
1085 if (unlikely(ftrace_disabled
))
1088 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1092 mutex_lock(&ftrace_regex_lock
);
1093 if ((file
->f_mode
& FMODE_WRITE
) &&
1094 !(file
->f_flags
& O_APPEND
))
1095 ftrace_filter_reset(enable
);
1097 if (file
->f_mode
& FMODE_READ
) {
1098 iter
->pg
= ftrace_pages_start
;
1099 iter
->flags
= enable
? FTRACE_ITER_FILTER
:
1100 FTRACE_ITER_NOTRACE
;
1102 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1104 struct seq_file
*m
= file
->private_data
;
1109 file
->private_data
= iter
;
1110 mutex_unlock(&ftrace_regex_lock
);
1116 ftrace_filter_open(struct inode
*inode
, struct file
*file
)
1118 return ftrace_regex_open(inode
, file
, 1);
1122 ftrace_notrace_open(struct inode
*inode
, struct file
*file
)
1124 return ftrace_regex_open(inode
, file
, 0);
1128 ftrace_regex_lseek(struct file
*file
, loff_t offset
, int origin
)
1132 if (file
->f_mode
& FMODE_READ
)
1133 ret
= seq_lseek(file
, offset
, origin
);
1135 file
->f_pos
= ret
= 1;
1148 * (static function - no need for kernel doc)
1150 * Pass in a buffer containing a glob and this function will
1151 * set search to point to the search part of the buffer and
1152 * return the type of search it is (see enum above).
1153 * This does modify buff.
1155 * Returns enum type.
1156 * search returns the pointer to use for comparison.
1157 * not returns 1 if buff started with a '!'
1161 ftrace_setup_glob(char *buff
, int len
, char **search
, int *not)
1163 int type
= MATCH_FULL
;
1166 if (buff
[0] == '!') {
1175 for (i
= 0; i
< len
; i
++) {
1176 if (buff
[i
] == '*') {
1179 type
= MATCH_END_ONLY
;
1181 if (type
== MATCH_END_ONLY
)
1182 type
= MATCH_MIDDLE_ONLY
;
1184 type
= MATCH_FRONT_ONLY
;
1194 static int ftrace_match(char *str
, char *regex
, int len
, int type
)
1201 if (strcmp(str
, regex
) == 0)
1204 case MATCH_FRONT_ONLY
:
1205 if (strncmp(str
, regex
, len
) == 0)
1208 case MATCH_MIDDLE_ONLY
:
1209 if (strstr(str
, regex
))
1212 case MATCH_END_ONLY
:
1213 ptr
= strstr(str
, regex
);
1214 if (ptr
&& (ptr
[len
] == 0))
1223 ftrace_match_record(struct dyn_ftrace
*rec
, char *regex
, int len
, int type
)
1225 char str
[KSYM_SYMBOL_LEN
];
1227 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1228 return ftrace_match(str
, regex
, len
, type
);
1231 static void ftrace_match_records(char *buff
, int len
, int enable
)
1233 unsigned int search_len
;
1234 struct ftrace_page
*pg
;
1235 struct dyn_ftrace
*rec
;
1241 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1242 type
= ftrace_setup_glob(buff
, len
, &search
, ¬);
1244 search_len
= strlen(search
);
1246 mutex_lock(&ftrace_lock
);
1247 do_for_each_ftrace_rec(pg
, rec
) {
1249 if (rec
->flags
& FTRACE_FL_FAILED
)
1252 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
1254 rec
->flags
&= ~flag
;
1259 * Only enable filtering if we have a function that
1262 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1263 ftrace_filtered
= 1;
1264 } while_for_each_ftrace_rec();
1265 mutex_unlock(&ftrace_lock
);
1269 ftrace_match_module_record(struct dyn_ftrace
*rec
, char *mod
,
1270 char *regex
, int len
, int type
)
1272 char str
[KSYM_SYMBOL_LEN
];
1275 kallsyms_lookup(rec
->ip
, NULL
, NULL
, &modname
, str
);
1277 if (!modname
|| strcmp(modname
, mod
))
1280 /* blank search means to match all funcs in the mod */
1282 return ftrace_match(str
, regex
, len
, type
);
1287 static void ftrace_match_module_records(char *buff
, char *mod
, int enable
)
1289 unsigned search_len
= 0;
1290 struct ftrace_page
*pg
;
1291 struct dyn_ftrace
*rec
;
1292 int type
= MATCH_FULL
;
1293 char *search
= buff
;
1297 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1299 /* blank or '*' mean the same */
1300 if (strcmp(buff
, "*") == 0)
1303 /* handle the case of 'dont filter this module' */
1304 if (strcmp(buff
, "!") == 0 || strcmp(buff
, "!*") == 0) {
1310 type
= ftrace_setup_glob(buff
, strlen(buff
), &search
, ¬);
1311 search_len
= strlen(search
);
1314 mutex_lock(&ftrace_lock
);
1315 do_for_each_ftrace_rec(pg
, rec
) {
1317 if (rec
->flags
& FTRACE_FL_FAILED
)
1320 if (ftrace_match_module_record(rec
, mod
,
1321 search
, search_len
, type
)) {
1323 rec
->flags
&= ~flag
;
1327 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1328 ftrace_filtered
= 1;
1330 } while_for_each_ftrace_rec();
1331 mutex_unlock(&ftrace_lock
);
1335 * We register the module command as a template to show others how
1336 * to register the a command as well.
1340 ftrace_mod_callback(char *func
, char *cmd
, char *param
, int enable
)
1345 * cmd == 'mod' because we only registered this func
1346 * for the 'mod' ftrace_func_command.
1347 * But if you register one func with multiple commands,
1348 * you can tell which command was used by the cmd
1352 /* we must have a module name */
1356 mod
= strsep(¶m
, ":");
1360 ftrace_match_module_records(func
, mod
, enable
);
1364 static struct ftrace_func_command ftrace_mod_cmd
= {
1366 .func
= ftrace_mod_callback
,
1369 static int __init
ftrace_mod_cmd_init(void)
1371 return register_ftrace_command(&ftrace_mod_cmd
);
1373 device_initcall(ftrace_mod_cmd_init
);
1376 function_trace_probe_call(unsigned long ip
, unsigned long parent_ip
)
1378 struct ftrace_func_probe
*entry
;
1379 struct hlist_head
*hhd
;
1380 struct hlist_node
*n
;
1384 key
= hash_long(ip
, FTRACE_HASH_BITS
);
1386 hhd
= &ftrace_func_hash
[key
];
1388 if (hlist_empty(hhd
))
1392 * Disable preemption for these calls to prevent a RCU grace
1393 * period. This syncs the hash iteration and freeing of items
1394 * on the hash. rcu_read_lock is too dangerous here.
1396 resched
= ftrace_preempt_disable();
1397 hlist_for_each_entry_rcu(entry
, n
, hhd
, node
) {
1398 if (entry
->ip
== ip
)
1399 entry
->ops
->func(ip
, parent_ip
, &entry
->data
);
1401 ftrace_preempt_enable(resched
);
1404 static struct ftrace_ops trace_probe_ops __read_mostly
=
1406 .func
= function_trace_probe_call
,
1409 static int ftrace_probe_registered
;
1411 static void __enable_ftrace_function_probe(void)
1415 if (ftrace_probe_registered
)
1418 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1419 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1423 /* Nothing registered? */
1424 if (i
== FTRACE_FUNC_HASHSIZE
)
1427 __register_ftrace_function(&trace_probe_ops
);
1429 ftrace_probe_registered
= 1;
1432 static void __disable_ftrace_function_probe(void)
1436 if (!ftrace_probe_registered
)
1439 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1440 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1445 /* no more funcs left */
1446 __unregister_ftrace_function(&trace_probe_ops
);
1448 ftrace_probe_registered
= 0;
1452 static void ftrace_free_entry_rcu(struct rcu_head
*rhp
)
1454 struct ftrace_func_probe
*entry
=
1455 container_of(rhp
, struct ftrace_func_probe
, rcu
);
1457 if (entry
->ops
->free
)
1458 entry
->ops
->free(&entry
->data
);
1464 register_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
1467 struct ftrace_func_probe
*entry
;
1468 struct ftrace_page
*pg
;
1469 struct dyn_ftrace
*rec
;
1475 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
1476 len
= strlen(search
);
1478 /* we do not support '!' for function probes */
1482 mutex_lock(&ftrace_lock
);
1483 do_for_each_ftrace_rec(pg
, rec
) {
1485 if (rec
->flags
& FTRACE_FL_FAILED
)
1488 if (!ftrace_match_record(rec
, search
, len
, type
))
1491 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
1493 /* If we did not process any, then return error */
1504 * The caller might want to do something special
1505 * for each function we find. We call the callback
1506 * to give the caller an opportunity to do so.
1508 if (ops
->callback
) {
1509 if (ops
->callback(rec
->ip
, &entry
->data
) < 0) {
1510 /* caller does not like this func */
1517 entry
->ip
= rec
->ip
;
1519 key
= hash_long(entry
->ip
, FTRACE_HASH_BITS
);
1520 hlist_add_head_rcu(&entry
->node
, &ftrace_func_hash
[key
]);
1522 } while_for_each_ftrace_rec();
1523 __enable_ftrace_function_probe();
1526 mutex_unlock(&ftrace_lock
);
1532 PROBE_TEST_FUNC
= 1,
1537 __unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
1538 void *data
, int flags
)
1540 struct ftrace_func_probe
*entry
;
1541 struct hlist_node
*n
, *tmp
;
1542 char str
[KSYM_SYMBOL_LEN
];
1543 int type
= MATCH_FULL
;
1547 if (glob
&& (strcmp(glob
, "*") || !strlen(glob
)))
1552 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
1553 len
= strlen(search
);
1555 /* we do not support '!' for function probes */
1560 mutex_lock(&ftrace_lock
);
1561 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1562 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1564 hlist_for_each_entry_safe(entry
, n
, tmp
, hhd
, node
) {
1566 /* break up if statements for readability */
1567 if ((flags
& PROBE_TEST_FUNC
) && entry
->ops
!= ops
)
1570 if ((flags
& PROBE_TEST_DATA
) && entry
->data
!= data
)
1573 /* do this last, since it is the most expensive */
1575 kallsyms_lookup(entry
->ip
, NULL
, NULL
,
1577 if (!ftrace_match(str
, glob
, len
, type
))
1581 hlist_del(&entry
->node
);
1582 call_rcu(&entry
->rcu
, ftrace_free_entry_rcu
);
1585 __disable_ftrace_function_probe();
1586 mutex_unlock(&ftrace_lock
);
1590 unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
1593 __unregister_ftrace_function_probe(glob
, ops
, data
,
1594 PROBE_TEST_FUNC
| PROBE_TEST_DATA
);
1598 unregister_ftrace_function_probe_func(char *glob
, struct ftrace_probe_ops
*ops
)
1600 __unregister_ftrace_function_probe(glob
, ops
, NULL
, PROBE_TEST_FUNC
);
1603 void unregister_ftrace_function_probe_all(char *glob
)
1605 __unregister_ftrace_function_probe(glob
, NULL
, NULL
, 0);
1608 static LIST_HEAD(ftrace_commands
);
1609 static DEFINE_MUTEX(ftrace_cmd_mutex
);
1611 int register_ftrace_command(struct ftrace_func_command
*cmd
)
1613 struct ftrace_func_command
*p
;
1616 mutex_lock(&ftrace_cmd_mutex
);
1617 list_for_each_entry(p
, &ftrace_commands
, list
) {
1618 if (strcmp(cmd
->name
, p
->name
) == 0) {
1623 list_add(&cmd
->list
, &ftrace_commands
);
1625 mutex_unlock(&ftrace_cmd_mutex
);
1630 int unregister_ftrace_command(struct ftrace_func_command
*cmd
)
1632 struct ftrace_func_command
*p
, *n
;
1635 mutex_lock(&ftrace_cmd_mutex
);
1636 list_for_each_entry_safe(p
, n
, &ftrace_commands
, list
) {
1637 if (strcmp(cmd
->name
, p
->name
) == 0) {
1639 list_del_init(&p
->list
);
1644 mutex_unlock(&ftrace_cmd_mutex
);
1649 static int ftrace_process_regex(char *buff
, int len
, int enable
)
1651 char *func
, *command
, *next
= buff
;
1652 struct ftrace_func_command
*p
;
1655 func
= strsep(&next
, ":");
1658 ftrace_match_records(func
, len
, enable
);
1664 command
= strsep(&next
, ":");
1666 mutex_lock(&ftrace_cmd_mutex
);
1667 list_for_each_entry(p
, &ftrace_commands
, list
) {
1668 if (strcmp(p
->name
, command
) == 0) {
1669 ret
= p
->func(func
, command
, next
, enable
);
1674 mutex_unlock(&ftrace_cmd_mutex
);
1680 ftrace_regex_write(struct file
*file
, const char __user
*ubuf
,
1681 size_t cnt
, loff_t
*ppos
, int enable
)
1683 struct ftrace_iterator
*iter
;
1688 if (!cnt
|| cnt
< 0)
1691 mutex_lock(&ftrace_regex_lock
);
1693 if (file
->f_mode
& FMODE_READ
) {
1694 struct seq_file
*m
= file
->private_data
;
1697 iter
= file
->private_data
;
1700 iter
->flags
&= ~FTRACE_ITER_CONT
;
1701 iter
->buffer_idx
= 0;
1704 ret
= get_user(ch
, ubuf
++);
1710 if (!(iter
->flags
& ~FTRACE_ITER_CONT
)) {
1711 /* skip white space */
1712 while (cnt
&& isspace(ch
)) {
1713 ret
= get_user(ch
, ubuf
++);
1721 file
->f_pos
+= read
;
1726 iter
->buffer_idx
= 0;
1729 while (cnt
&& !isspace(ch
)) {
1730 if (iter
->buffer_idx
< FTRACE_BUFF_MAX
)
1731 iter
->buffer
[iter
->buffer_idx
++] = ch
;
1736 ret
= get_user(ch
, ubuf
++);
1745 iter
->buffer
[iter
->buffer_idx
] = 0;
1746 ret
= ftrace_process_regex(iter
->buffer
,
1747 iter
->buffer_idx
, enable
);
1750 iter
->buffer_idx
= 0;
1752 iter
->flags
|= FTRACE_ITER_CONT
;
1755 file
->f_pos
+= read
;
1759 mutex_unlock(&ftrace_regex_lock
);
1765 ftrace_filter_write(struct file
*file
, const char __user
*ubuf
,
1766 size_t cnt
, loff_t
*ppos
)
1768 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 1);
1772 ftrace_notrace_write(struct file
*file
, const char __user
*ubuf
,
1773 size_t cnt
, loff_t
*ppos
)
1775 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 0);
1779 ftrace_set_regex(unsigned char *buf
, int len
, int reset
, int enable
)
1781 if (unlikely(ftrace_disabled
))
1784 mutex_lock(&ftrace_regex_lock
);
1786 ftrace_filter_reset(enable
);
1788 ftrace_match_records(buf
, len
, enable
);
1789 mutex_unlock(&ftrace_regex_lock
);
1793 * ftrace_set_filter - set a function to filter on in ftrace
1794 * @buf - the string that holds the function filter text.
1795 * @len - the length of the string.
1796 * @reset - non zero to reset all filters before applying this filter.
1798 * Filters denote which functions should be enabled when tracing is enabled.
1799 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1801 void ftrace_set_filter(unsigned char *buf
, int len
, int reset
)
1803 ftrace_set_regex(buf
, len
, reset
, 1);
1807 * ftrace_set_notrace - set a function to not trace in ftrace
1808 * @buf - the string that holds the function notrace text.
1809 * @len - the length of the string.
1810 * @reset - non zero to reset all filters before applying this filter.
1812 * Notrace Filters denote which functions should not be enabled when tracing
1813 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1816 void ftrace_set_notrace(unsigned char *buf
, int len
, int reset
)
1818 ftrace_set_regex(buf
, len
, reset
, 0);
1822 ftrace_regex_release(struct inode
*inode
, struct file
*file
, int enable
)
1824 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
1825 struct ftrace_iterator
*iter
;
1827 mutex_lock(&ftrace_regex_lock
);
1828 if (file
->f_mode
& FMODE_READ
) {
1831 seq_release(inode
, file
);
1833 iter
= file
->private_data
;
1835 if (iter
->buffer_idx
) {
1837 iter
->buffer
[iter
->buffer_idx
] = 0;
1838 ftrace_match_records(iter
->buffer
, iter
->buffer_idx
, enable
);
1841 mutex_lock(&ftrace_lock
);
1842 if (ftrace_start_up
&& ftrace_enabled
)
1843 ftrace_run_update_code(FTRACE_ENABLE_CALLS
);
1844 mutex_unlock(&ftrace_lock
);
1847 mutex_unlock(&ftrace_regex_lock
);
1852 ftrace_filter_release(struct inode
*inode
, struct file
*file
)
1854 return ftrace_regex_release(inode
, file
, 1);
1858 ftrace_notrace_release(struct inode
*inode
, struct file
*file
)
1860 return ftrace_regex_release(inode
, file
, 0);
1863 static const struct file_operations ftrace_avail_fops
= {
1864 .open
= ftrace_avail_open
,
1866 .llseek
= seq_lseek
,
1867 .release
= ftrace_avail_release
,
1870 static const struct file_operations ftrace_failures_fops
= {
1871 .open
= ftrace_failures_open
,
1873 .llseek
= seq_lseek
,
1874 .release
= ftrace_avail_release
,
1877 static const struct file_operations ftrace_filter_fops
= {
1878 .open
= ftrace_filter_open
,
1880 .write
= ftrace_filter_write
,
1881 .llseek
= ftrace_regex_lseek
,
1882 .release
= ftrace_filter_release
,
1885 static const struct file_operations ftrace_notrace_fops
= {
1886 .open
= ftrace_notrace_open
,
1888 .write
= ftrace_notrace_write
,
1889 .llseek
= ftrace_regex_lseek
,
1890 .release
= ftrace_notrace_release
,
1893 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1895 static DEFINE_MUTEX(graph_lock
);
1897 int ftrace_graph_count
;
1898 unsigned long ftrace_graph_funcs
[FTRACE_GRAPH_MAX_FUNCS
] __read_mostly
;
1901 g_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1903 unsigned long *array
= m
->private;
1908 if (index
>= ftrace_graph_count
)
1911 return &array
[index
];
1914 static void *g_start(struct seq_file
*m
, loff_t
*pos
)
1918 mutex_lock(&graph_lock
);
1920 /* Nothing, tell g_show to print all functions are enabled */
1921 if (!ftrace_graph_count
&& !*pos
)
1924 p
= g_next(m
, p
, pos
);
1929 static void g_stop(struct seq_file
*m
, void *p
)
1931 mutex_unlock(&graph_lock
);
1934 static int g_show(struct seq_file
*m
, void *v
)
1936 unsigned long *ptr
= v
;
1937 char str
[KSYM_SYMBOL_LEN
];
1942 if (ptr
== (unsigned long *)1) {
1943 seq_printf(m
, "#### all functions enabled ####\n");
1947 kallsyms_lookup(*ptr
, NULL
, NULL
, NULL
, str
);
1949 seq_printf(m
, "%s\n", str
);
1954 static struct seq_operations ftrace_graph_seq_ops
= {
1962 ftrace_graph_open(struct inode
*inode
, struct file
*file
)
1966 if (unlikely(ftrace_disabled
))
1969 mutex_lock(&graph_lock
);
1970 if ((file
->f_mode
& FMODE_WRITE
) &&
1971 !(file
->f_flags
& O_APPEND
)) {
1972 ftrace_graph_count
= 0;
1973 memset(ftrace_graph_funcs
, 0, sizeof(ftrace_graph_funcs
));
1976 if (file
->f_mode
& FMODE_READ
) {
1977 ret
= seq_open(file
, &ftrace_graph_seq_ops
);
1979 struct seq_file
*m
= file
->private_data
;
1980 m
->private = ftrace_graph_funcs
;
1983 file
->private_data
= ftrace_graph_funcs
;
1984 mutex_unlock(&graph_lock
);
1990 ftrace_set_func(unsigned long *array
, int *idx
, char *buffer
)
1992 struct dyn_ftrace
*rec
;
1993 struct ftrace_page
*pg
;
2001 if (ftrace_disabled
)
2005 type
= ftrace_setup_glob(buffer
, strlen(buffer
), &search
, ¬);
2009 search_len
= strlen(search
);
2011 mutex_lock(&ftrace_lock
);
2012 do_for_each_ftrace_rec(pg
, rec
) {
2014 if (*idx
>= FTRACE_GRAPH_MAX_FUNCS
)
2017 if (rec
->flags
& (FTRACE_FL_FAILED
| FTRACE_FL_FREE
))
2020 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
2021 /* ensure it is not already in the array */
2023 for (i
= 0; i
< *idx
; i
++)
2024 if (array
[i
] == rec
->ip
) {
2029 array
[(*idx
)++] = rec
->ip
;
2033 } while_for_each_ftrace_rec();
2035 mutex_unlock(&ftrace_lock
);
2037 return found
? 0 : -EINVAL
;
2041 ftrace_graph_write(struct file
*file
, const char __user
*ubuf
,
2042 size_t cnt
, loff_t
*ppos
)
2044 unsigned char buffer
[FTRACE_BUFF_MAX
+1];
2045 unsigned long *array
;
2051 if (!cnt
|| cnt
< 0)
2054 mutex_lock(&graph_lock
);
2056 if (ftrace_graph_count
>= FTRACE_GRAPH_MAX_FUNCS
) {
2061 if (file
->f_mode
& FMODE_READ
) {
2062 struct seq_file
*m
= file
->private_data
;
2065 array
= file
->private_data
;
2067 ret
= get_user(ch
, ubuf
++);
2073 /* skip white space */
2074 while (cnt
&& isspace(ch
)) {
2075 ret
= get_user(ch
, ubuf
++);
2088 while (cnt
&& !isspace(ch
)) {
2089 if (index
< FTRACE_BUFF_MAX
)
2090 buffer
[index
++] = ch
;
2095 ret
= get_user(ch
, ubuf
++);
2103 /* we allow only one expression at a time */
2104 ret
= ftrace_set_func(array
, &ftrace_graph_count
, buffer
);
2108 file
->f_pos
+= read
;
2112 mutex_unlock(&graph_lock
);
2117 static const struct file_operations ftrace_graph_fops
= {
2118 .open
= ftrace_graph_open
,
2120 .write
= ftrace_graph_write
,
2122 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2124 static __init
int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
)
2126 struct dentry
*entry
;
2128 entry
= debugfs_create_file("available_filter_functions", 0444,
2129 d_tracer
, NULL
, &ftrace_avail_fops
);
2131 pr_warning("Could not create debugfs "
2132 "'available_filter_functions' entry\n");
2134 entry
= debugfs_create_file("failures", 0444,
2135 d_tracer
, NULL
, &ftrace_failures_fops
);
2137 pr_warning("Could not create debugfs 'failures' entry\n");
2139 entry
= debugfs_create_file("set_ftrace_filter", 0644, d_tracer
,
2140 NULL
, &ftrace_filter_fops
);
2142 pr_warning("Could not create debugfs "
2143 "'set_ftrace_filter' entry\n");
2145 entry
= debugfs_create_file("set_ftrace_notrace", 0644, d_tracer
,
2146 NULL
, &ftrace_notrace_fops
);
2148 pr_warning("Could not create debugfs "
2149 "'set_ftrace_notrace' entry\n");
2151 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2152 entry
= debugfs_create_file("set_graph_function", 0444, d_tracer
,
2154 &ftrace_graph_fops
);
2156 pr_warning("Could not create debugfs "
2157 "'set_graph_function' entry\n");
2158 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2163 static int ftrace_convert_nops(struct module
*mod
,
2164 unsigned long *start
,
2169 unsigned long flags
;
2171 mutex_lock(&ftrace_lock
);
2174 addr
= ftrace_call_adjust(*p
++);
2176 * Some architecture linkers will pad between
2177 * the different mcount_loc sections of different
2178 * object files to satisfy alignments.
2179 * Skip any NULL pointers.
2183 ftrace_record_ip(addr
);
2186 /* disable interrupts to prevent kstop machine */
2187 local_irq_save(flags
);
2188 ftrace_update_code(mod
);
2189 local_irq_restore(flags
);
2190 mutex_unlock(&ftrace_lock
);
2195 void ftrace_init_module(struct module
*mod
,
2196 unsigned long *start
, unsigned long *end
)
2198 if (ftrace_disabled
|| start
== end
)
2200 ftrace_convert_nops(mod
, start
, end
);
2203 extern unsigned long __start_mcount_loc
[];
2204 extern unsigned long __stop_mcount_loc
[];
2206 void __init
ftrace_init(void)
2208 unsigned long count
, addr
, flags
;
2211 /* Keep the ftrace pointer to the stub */
2212 addr
= (unsigned long)ftrace_stub
;
2214 local_irq_save(flags
);
2215 ftrace_dyn_arch_init(&addr
);
2216 local_irq_restore(flags
);
2218 /* ftrace_dyn_arch_init places the return code in addr */
2222 count
= __stop_mcount_loc
- __start_mcount_loc
;
2224 ret
= ftrace_dyn_table_alloc(count
);
2228 last_ftrace_enabled
= ftrace_enabled
= 1;
2230 ret
= ftrace_convert_nops(NULL
,
2236 ftrace_disabled
= 1;
2241 static int __init
ftrace_nodyn_init(void)
2246 device_initcall(ftrace_nodyn_init
);
2248 static inline int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
) { return 0; }
2249 static inline void ftrace_startup_enable(int command
) { }
2250 /* Keep as macros so we do not need to define the commands */
2251 # define ftrace_startup(command) do { } while (0)
2252 # define ftrace_shutdown(command) do { } while (0)
2253 # define ftrace_startup_sysctl() do { } while (0)
2254 # define ftrace_shutdown_sysctl() do { } while (0)
2255 #endif /* CONFIG_DYNAMIC_FTRACE */
2258 ftrace_pid_read(struct file
*file
, char __user
*ubuf
,
2259 size_t cnt
, loff_t
*ppos
)
2264 if (ftrace_pid_trace
== ftrace_swapper_pid
)
2265 r
= sprintf(buf
, "swapper tasks\n");
2266 else if (ftrace_pid_trace
)
2267 r
= sprintf(buf
, "%u\n", pid_nr(ftrace_pid_trace
));
2269 r
= sprintf(buf
, "no pid\n");
2271 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
2274 static void clear_ftrace_swapper(void)
2276 struct task_struct
*p
;
2280 for_each_online_cpu(cpu
) {
2282 clear_tsk_trace_trace(p
);
2287 static void set_ftrace_swapper(void)
2289 struct task_struct
*p
;
2293 for_each_online_cpu(cpu
) {
2295 set_tsk_trace_trace(p
);
2300 static void clear_ftrace_pid(struct pid
*pid
)
2302 struct task_struct
*p
;
2305 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2306 clear_tsk_trace_trace(p
);
2307 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2313 static void set_ftrace_pid(struct pid
*pid
)
2315 struct task_struct
*p
;
2318 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2319 set_tsk_trace_trace(p
);
2320 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2324 static void clear_ftrace_pid_task(struct pid
**pid
)
2326 if (*pid
== ftrace_swapper_pid
)
2327 clear_ftrace_swapper();
2329 clear_ftrace_pid(*pid
);
2334 static void set_ftrace_pid_task(struct pid
*pid
)
2336 if (pid
== ftrace_swapper_pid
)
2337 set_ftrace_swapper();
2339 set_ftrace_pid(pid
);
2343 ftrace_pid_write(struct file
*filp
, const char __user
*ubuf
,
2344 size_t cnt
, loff_t
*ppos
)
2351 if (cnt
>= sizeof(buf
))
2354 if (copy_from_user(&buf
, ubuf
, cnt
))
2359 ret
= strict_strtol(buf
, 10, &val
);
2363 mutex_lock(&ftrace_lock
);
2365 /* disable pid tracing */
2366 if (!ftrace_pid_trace
)
2369 clear_ftrace_pid_task(&ftrace_pid_trace
);
2372 /* swapper task is special */
2374 pid
= ftrace_swapper_pid
;
2375 if (pid
== ftrace_pid_trace
)
2378 pid
= find_get_pid(val
);
2380 if (pid
== ftrace_pid_trace
) {
2386 if (ftrace_pid_trace
)
2387 clear_ftrace_pid_task(&ftrace_pid_trace
);
2392 ftrace_pid_trace
= pid
;
2394 set_ftrace_pid_task(ftrace_pid_trace
);
2397 /* update the function call */
2398 ftrace_update_pid_func();
2399 ftrace_startup_enable(0);
2402 mutex_unlock(&ftrace_lock
);
2407 static const struct file_operations ftrace_pid_fops
= {
2408 .read
= ftrace_pid_read
,
2409 .write
= ftrace_pid_write
,
2412 static __init
int ftrace_init_debugfs(void)
2414 struct dentry
*d_tracer
;
2415 struct dentry
*entry
;
2417 d_tracer
= tracing_init_dentry();
2421 ftrace_init_dyn_debugfs(d_tracer
);
2423 entry
= debugfs_create_file("set_ftrace_pid", 0644, d_tracer
,
2424 NULL
, &ftrace_pid_fops
);
2426 pr_warning("Could not create debugfs "
2427 "'set_ftrace_pid' entry\n");
2430 fs_initcall(ftrace_init_debugfs
);
2433 * ftrace_kill - kill ftrace
2435 * This function should be used by panic code. It stops ftrace
2436 * but in a not so nice way. If you need to simply kill ftrace
2437 * from a non-atomic section, use ftrace_kill.
2439 void ftrace_kill(void)
2441 ftrace_disabled
= 1;
2443 clear_ftrace_function();
2447 * register_ftrace_function - register a function for profiling
2448 * @ops - ops structure that holds the function for profiling.
2450 * Register a function to be called by all functions in the
2453 * Note: @ops->func and all the functions it calls must be labeled
2454 * with "notrace", otherwise it will go into a
2457 int register_ftrace_function(struct ftrace_ops
*ops
)
2461 if (unlikely(ftrace_disabled
))
2464 mutex_lock(&ftrace_lock
);
2466 ret
= __register_ftrace_function(ops
);
2469 mutex_unlock(&ftrace_lock
);
2474 * unregister_ftrace_function - unregister a function for profiling.
2475 * @ops - ops structure that holds the function to unregister
2477 * Unregister a function that was added to be called by ftrace profiling.
2479 int unregister_ftrace_function(struct ftrace_ops
*ops
)
2483 mutex_lock(&ftrace_lock
);
2484 ret
= __unregister_ftrace_function(ops
);
2486 mutex_unlock(&ftrace_lock
);
2492 ftrace_enable_sysctl(struct ctl_table
*table
, int write
,
2493 struct file
*file
, void __user
*buffer
, size_t *lenp
,
2498 if (unlikely(ftrace_disabled
))
2501 mutex_lock(&ftrace_lock
);
2503 ret
= proc_dointvec(table
, write
, file
, buffer
, lenp
, ppos
);
2505 if (ret
|| !write
|| (last_ftrace_enabled
== ftrace_enabled
))
2508 last_ftrace_enabled
= ftrace_enabled
;
2510 if (ftrace_enabled
) {
2512 ftrace_startup_sysctl();
2514 /* we are starting ftrace again */
2515 if (ftrace_list
!= &ftrace_list_end
) {
2516 if (ftrace_list
->next
== &ftrace_list_end
)
2517 ftrace_trace_function
= ftrace_list
->func
;
2519 ftrace_trace_function
= ftrace_list_func
;
2523 /* stopping ftrace calls (just send to ftrace_stub) */
2524 ftrace_trace_function
= ftrace_stub
;
2526 ftrace_shutdown_sysctl();
2530 mutex_unlock(&ftrace_lock
);
2534 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2536 static atomic_t ftrace_graph_active
;
2537 static struct notifier_block ftrace_suspend_notifier
;
2539 int ftrace_graph_entry_stub(struct ftrace_graph_ent
*trace
)
2544 /* The callbacks that hook a function */
2545 trace_func_graph_ret_t ftrace_graph_return
=
2546 (trace_func_graph_ret_t
)ftrace_stub
;
2547 trace_func_graph_ent_t ftrace_graph_entry
= ftrace_graph_entry_stub
;
2549 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2550 static int alloc_retstack_tasklist(struct ftrace_ret_stack
**ret_stack_list
)
2554 unsigned long flags
;
2555 int start
= 0, end
= FTRACE_RETSTACK_ALLOC_SIZE
;
2556 struct task_struct
*g
, *t
;
2558 for (i
= 0; i
< FTRACE_RETSTACK_ALLOC_SIZE
; i
++) {
2559 ret_stack_list
[i
] = kmalloc(FTRACE_RETFUNC_DEPTH
2560 * sizeof(struct ftrace_ret_stack
),
2562 if (!ret_stack_list
[i
]) {
2570 read_lock_irqsave(&tasklist_lock
, flags
);
2571 do_each_thread(g
, t
) {
2577 if (t
->ret_stack
== NULL
) {
2578 t
->curr_ret_stack
= -1;
2579 /* Make sure IRQs see the -1 first: */
2581 t
->ret_stack
= ret_stack_list
[start
++];
2582 atomic_set(&t
->tracing_graph_pause
, 0);
2583 atomic_set(&t
->trace_overrun
, 0);
2585 } while_each_thread(g
, t
);
2588 read_unlock_irqrestore(&tasklist_lock
, flags
);
2590 for (i
= start
; i
< end
; i
++)
2591 kfree(ret_stack_list
[i
]);
2596 ftrace_graph_probe_sched_switch(struct rq
*__rq
, struct task_struct
*prev
,
2597 struct task_struct
*next
)
2599 unsigned long long timestamp
;
2602 timestamp
= trace_clock_local();
2604 prev
->ftrace_timestamp
= timestamp
;
2606 /* only process tasks that we timestamped */
2607 if (!next
->ftrace_timestamp
)
2611 * Update all the counters in next to make up for the
2612 * time next was sleeping.
2614 timestamp
-= next
->ftrace_timestamp
;
2616 for (index
= next
->curr_ret_stack
; index
>= 0; index
--)
2617 next
->ret_stack
[index
].calltime
+= timestamp
;
2620 /* Allocate a return stack for each task */
2621 static int start_graph_tracing(void)
2623 struct ftrace_ret_stack
**ret_stack_list
;
2626 ret_stack_list
= kmalloc(FTRACE_RETSTACK_ALLOC_SIZE
*
2627 sizeof(struct ftrace_ret_stack
*),
2630 if (!ret_stack_list
)
2633 /* The cpu_boot init_task->ret_stack will never be freed */
2634 for_each_online_cpu(cpu
)
2635 ftrace_graph_init_task(idle_task(cpu
));
2638 ret
= alloc_retstack_tasklist(ret_stack_list
);
2639 } while (ret
== -EAGAIN
);
2642 ret
= register_trace_sched_switch(ftrace_graph_probe_sched_switch
);
2644 pr_info("ftrace_graph: Couldn't activate tracepoint"
2645 " probe to kernel_sched_switch\n");
2648 kfree(ret_stack_list
);
2653 * Hibernation protection.
2654 * The state of the current task is too much unstable during
2655 * suspend/restore to disk. We want to protect against that.
2658 ftrace_suspend_notifier_call(struct notifier_block
*bl
, unsigned long state
,
2662 case PM_HIBERNATION_PREPARE
:
2663 pause_graph_tracing();
2666 case PM_POST_HIBERNATION
:
2667 unpause_graph_tracing();
2673 int register_ftrace_graph(trace_func_graph_ret_t retfunc
,
2674 trace_func_graph_ent_t entryfunc
)
2678 mutex_lock(&ftrace_lock
);
2680 /* we currently allow only one tracer registered at a time */
2681 if (atomic_read(&ftrace_graph_active
)) {
2686 ftrace_suspend_notifier
.notifier_call
= ftrace_suspend_notifier_call
;
2687 register_pm_notifier(&ftrace_suspend_notifier
);
2689 atomic_inc(&ftrace_graph_active
);
2690 ret
= start_graph_tracing();
2692 atomic_dec(&ftrace_graph_active
);
2696 ftrace_graph_return
= retfunc
;
2697 ftrace_graph_entry
= entryfunc
;
2699 ftrace_startup(FTRACE_START_FUNC_RET
);
2702 mutex_unlock(&ftrace_lock
);
2706 void unregister_ftrace_graph(void)
2708 mutex_lock(&ftrace_lock
);
2710 atomic_dec(&ftrace_graph_active
);
2711 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch
);
2712 ftrace_graph_return
= (trace_func_graph_ret_t
)ftrace_stub
;
2713 ftrace_graph_entry
= ftrace_graph_entry_stub
;
2714 ftrace_shutdown(FTRACE_STOP_FUNC_RET
);
2715 unregister_pm_notifier(&ftrace_suspend_notifier
);
2717 mutex_unlock(&ftrace_lock
);
2720 /* Allocate a return stack for newly created task */
2721 void ftrace_graph_init_task(struct task_struct
*t
)
2723 if (atomic_read(&ftrace_graph_active
)) {
2724 t
->ret_stack
= kmalloc(FTRACE_RETFUNC_DEPTH
2725 * sizeof(struct ftrace_ret_stack
),
2729 t
->curr_ret_stack
= -1;
2730 atomic_set(&t
->tracing_graph_pause
, 0);
2731 atomic_set(&t
->trace_overrun
, 0);
2732 t
->ftrace_timestamp
= 0;
2734 t
->ret_stack
= NULL
;
2737 void ftrace_graph_exit_task(struct task_struct
*t
)
2739 struct ftrace_ret_stack
*ret_stack
= t
->ret_stack
;
2741 t
->ret_stack
= NULL
;
2742 /* NULL must become visible to IRQs before we free it: */
2748 void ftrace_graph_stop(void)