4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace
;
25 static int branch_tracing_enabled __read_mostly
;
26 static DEFINE_MUTEX(branch_tracing_mutex
);
28 static struct trace_array
*branch_tracer
;
31 probe_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
33 struct trace_array
*tr
= branch_tracer
;
34 struct ring_buffer_event
*event
;
35 struct trace_branch
*entry
;
41 * I would love to save just the ftrace_likely_data pointer, but
42 * this code can also be used by modules. Ugly things can happen
43 * if the module is unloaded, and then we go and read the
44 * pointer. This is slower, but much safer.
50 local_irq_save(flags
);
51 cpu
= raw_smp_processor_id();
52 if (atomic_inc_return(&tr
->data
[cpu
]->disabled
) != 1)
56 event
= trace_buffer_lock_reserve(tr
, TRACE_BRANCH
,
57 sizeof(*entry
), flags
, pc
);
61 entry
= ring_buffer_event_data(event
);
63 /* Strip off the path, only save the file */
64 p
= f
->file
+ strlen(f
->file
);
65 while (p
>= f
->file
&& *p
!= '/')
69 strncpy(entry
->func
, f
->func
, TRACE_FUNC_SIZE
);
70 strncpy(entry
->file
, p
, TRACE_FILE_SIZE
);
71 entry
->func
[TRACE_FUNC_SIZE
] = 0;
72 entry
->file
[TRACE_FILE_SIZE
] = 0;
73 entry
->line
= f
->line
;
74 entry
->correct
= val
== expect
;
76 ring_buffer_unlock_commit(tr
->buffer
, event
);
79 atomic_dec(&tr
->data
[cpu
]->disabled
);
80 local_irq_restore(flags
);
84 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
86 if (!branch_tracing_enabled
)
89 probe_likely_condition(f
, val
, expect
);
92 int enable_branch_tracing(struct trace_array
*tr
)
94 mutex_lock(&branch_tracing_mutex
);
97 * Must be seen before enabling. The reader is a condition
98 * where we do not need a matching rmb()
101 branch_tracing_enabled
++;
102 mutex_unlock(&branch_tracing_mutex
);
107 void disable_branch_tracing(void)
109 mutex_lock(&branch_tracing_mutex
);
111 if (!branch_tracing_enabled
)
114 branch_tracing_enabled
--;
117 mutex_unlock(&branch_tracing_mutex
);
120 static void start_branch_trace(struct trace_array
*tr
)
122 enable_branch_tracing(tr
);
125 static void stop_branch_trace(struct trace_array
*tr
)
127 disable_branch_tracing();
130 static int branch_trace_init(struct trace_array
*tr
)
132 start_branch_trace(tr
);
136 static void branch_trace_reset(struct trace_array
*tr
)
138 stop_branch_trace(tr
);
141 static enum print_line_t
trace_branch_print(struct trace_iterator
*iter
,
144 struct trace_branch
*field
;
146 trace_assign_type(field
, iter
->ent
);
148 if (trace_seq_printf(&iter
->seq
, "[%s] %s:%s:%d\n",
149 field
->correct
? " ok " : " MISS ",
153 return TRACE_TYPE_PARTIAL_LINE
;
155 return TRACE_TYPE_HANDLED
;
159 static struct trace_event trace_branch_event
= {
160 .type
= TRACE_BRANCH
,
161 .trace
= trace_branch_print
,
162 .latency_trace
= trace_branch_print
,
165 static struct tracer branch_trace __read_mostly
=
168 .init
= branch_trace_init
,
169 .reset
= branch_trace_reset
,
170 #ifdef CONFIG_FTRACE_SELFTEST
171 .selftest
= trace_selftest_startup_branch
,
172 #endif /* CONFIG_FTRACE_SELFTEST */
175 __init
static int init_branch_tracer(void)
179 ret
= register_ftrace_event(&trace_branch_event
);
181 printk(KERN_WARNING
"Warning: could not register "
185 return register_tracer(&branch_trace
);
187 device_initcall(init_branch_tracer
);
191 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
194 #endif /* CONFIG_BRANCH_TRACER */
196 void ftrace_likely_update(struct ftrace_branch_data
*f
, int val
, int expect
)
199 * I would love to have a trace point here instead, but the
200 * trace point code is so inundated with unlikely and likely
201 * conditions that the recursive nightmare that exists is too
202 * much to try to get working. At least for now.
204 trace_likely_condition(f
, val
, expect
);
206 /* FIXME: Make this atomic! */
212 EXPORT_SYMBOL(ftrace_likely_update
);
214 extern unsigned long __start_annotated_branch_profile
[];
215 extern unsigned long __stop_annotated_branch_profile
[];
217 static int annotated_branch_stat_headers(struct seq_file
*m
)
219 seq_printf(m
, " correct incorrect %% ");
220 seq_printf(m
, " Function "
222 " ------- --------- - "
228 static inline long get_incorrect_percent(struct ftrace_branch_data
*p
)
233 percent
= p
->incorrect
* 100;
234 percent
/= p
->correct
+ p
->incorrect
;
236 percent
= p
->incorrect
? 100 : -1;
241 static int branch_stat_show(struct seq_file
*m
, void *v
)
243 struct ftrace_branch_data
*p
= v
;
247 /* Only print the file, not the path */
248 f
= p
->file
+ strlen(p
->file
);
249 while (f
>= p
->file
&& *f
!= '/')
254 * The miss is overlayed on correct, and hit on incorrect.
256 percent
= get_incorrect_percent(p
);
258 seq_printf(m
, "%8lu %8lu ", p
->correct
, p
->incorrect
);
260 seq_printf(m
, " X ");
262 seq_printf(m
, "%3ld ", percent
);
263 seq_printf(m
, "%-30.30s %-20.20s %d\n", p
->func
, f
, p
->line
);
267 static void *annotated_branch_stat_start(void)
269 return __start_annotated_branch_profile
;
273 annotated_branch_stat_next(void *v
, int idx
)
275 struct ftrace_branch_data
*p
= v
;
279 if ((void *)p
>= (void *)__stop_annotated_branch_profile
)
285 static int annotated_branch_stat_cmp(void *p1
, void *p2
)
287 struct ftrace_branch_data
*a
= p1
;
288 struct ftrace_branch_data
*b
= p2
;
290 long percent_a
, percent_b
;
292 percent_a
= get_incorrect_percent(a
);
293 percent_b
= get_incorrect_percent(b
);
295 if (percent_a
< percent_b
)
297 if (percent_a
> percent_b
)
303 static struct tracer_stat annotated_branch_stats
= {
304 .name
= "branch_annotated",
305 .stat_start
= annotated_branch_stat_start
,
306 .stat_next
= annotated_branch_stat_next
,
307 .stat_cmp
= annotated_branch_stat_cmp
,
308 .stat_headers
= annotated_branch_stat_headers
,
309 .stat_show
= branch_stat_show
312 __init
static int init_annotated_branch_stats(void)
316 ret
= register_stat_tracer(&annotated_branch_stats
);
318 printk(KERN_WARNING
"Warning: could not register "
319 "annotated branches stats\n");
324 fs_initcall(init_annotated_branch_stats
);
326 #ifdef CONFIG_PROFILE_ALL_BRANCHES
328 extern unsigned long __start_branch_profile
[];
329 extern unsigned long __stop_branch_profile
[];
331 static int all_branch_stat_headers(struct seq_file
*m
)
333 seq_printf(m
, " miss hit %% ");
334 seq_printf(m
, " Function "
336 " ------- --------- - "
342 static void *all_branch_stat_start(void)
344 return __start_branch_profile
;
348 all_branch_stat_next(void *v
, int idx
)
350 struct ftrace_branch_data
*p
= v
;
354 if ((void *)p
>= (void *)__stop_branch_profile
)
360 static struct tracer_stat all_branch_stats
= {
361 .name
= "branch_all",
362 .stat_start
= all_branch_stat_start
,
363 .stat_next
= all_branch_stat_next
,
364 .stat_headers
= all_branch_stat_headers
,
365 .stat_show
= branch_stat_show
368 __init
static int all_annotated_branch_stats(void)
372 ret
= register_stat_tracer(&all_branch_stats
);
374 printk(KERN_WARNING
"Warning: could not register "
375 "all branches stats\n");
380 fs_initcall(all_annotated_branch_stats
);
381 #endif /* CONFIG_PROFILE_ALL_BRANCHES */