4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace
;
25 static int branch_tracing_enabled __read_mostly
;
26 static DEFINE_MUTEX(branch_tracing_mutex
);
28 static struct trace_array
*branch_tracer
;
31 probe_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
33 struct ftrace_event_call
*call
= &event_branch
;
34 struct trace_array
*tr
= branch_tracer
;
35 struct ring_buffer_event
*event
;
36 struct trace_branch
*entry
;
42 * I would love to save just the ftrace_likely_data pointer, but
43 * this code can also be used by modules. Ugly things can happen
44 * if the module is unloaded, and then we go and read the
45 * pointer. This is slower, but much safer.
51 local_irq_save(flags
);
52 cpu
= raw_smp_processor_id();
53 if (atomic_inc_return(&tr
->data
[cpu
]->disabled
) != 1)
57 event
= trace_buffer_lock_reserve(tr
, TRACE_BRANCH
,
58 sizeof(*entry
), flags
, pc
);
62 entry
= ring_buffer_event_data(event
);
64 /* Strip off the path, only save the file */
65 p
= f
->file
+ strlen(f
->file
);
66 while (p
>= f
->file
&& *p
!= '/')
70 strncpy(entry
->func
, f
->func
, TRACE_FUNC_SIZE
);
71 strncpy(entry
->file
, p
, TRACE_FILE_SIZE
);
72 entry
->func
[TRACE_FUNC_SIZE
] = 0;
73 entry
->file
[TRACE_FILE_SIZE
] = 0;
74 entry
->line
= f
->line
;
75 entry
->correct
= val
== expect
;
77 filter_check_discard(call
, entry
, event
);
79 ring_buffer_unlock_commit(tr
->buffer
, event
);
82 atomic_dec(&tr
->data
[cpu
]->disabled
);
83 local_irq_restore(flags
);
87 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
89 if (!branch_tracing_enabled
)
92 probe_likely_condition(f
, val
, expect
);
95 int enable_branch_tracing(struct trace_array
*tr
)
97 mutex_lock(&branch_tracing_mutex
);
100 * Must be seen before enabling. The reader is a condition
101 * where we do not need a matching rmb()
104 branch_tracing_enabled
++;
105 mutex_unlock(&branch_tracing_mutex
);
110 void disable_branch_tracing(void)
112 mutex_lock(&branch_tracing_mutex
);
114 if (!branch_tracing_enabled
)
117 branch_tracing_enabled
--;
120 mutex_unlock(&branch_tracing_mutex
);
123 static void start_branch_trace(struct trace_array
*tr
)
125 enable_branch_tracing(tr
);
128 static void stop_branch_trace(struct trace_array
*tr
)
130 disable_branch_tracing();
133 static int branch_trace_init(struct trace_array
*tr
)
135 start_branch_trace(tr
);
139 static void branch_trace_reset(struct trace_array
*tr
)
141 stop_branch_trace(tr
);
144 static enum print_line_t
trace_branch_print(struct trace_iterator
*iter
,
147 struct trace_branch
*field
;
149 trace_assign_type(field
, iter
->ent
);
151 if (trace_seq_printf(&iter
->seq
, "[%s] %s:%s:%d\n",
152 field
->correct
? " ok " : " MISS ",
156 return TRACE_TYPE_PARTIAL_LINE
;
158 return TRACE_TYPE_HANDLED
;
162 static struct trace_event trace_branch_event
= {
163 .type
= TRACE_BRANCH
,
164 .trace
= trace_branch_print
,
167 static struct tracer branch_trace __read_mostly
=
170 .init
= branch_trace_init
,
171 .reset
= branch_trace_reset
,
172 #ifdef CONFIG_FTRACE_SELFTEST
173 .selftest
= trace_selftest_startup_branch
,
174 #endif /* CONFIG_FTRACE_SELFTEST */
177 __init
static int init_branch_tracer(void)
181 ret
= register_ftrace_event(&trace_branch_event
);
183 printk(KERN_WARNING
"Warning: could not register "
187 return register_tracer(&branch_trace
);
189 device_initcall(init_branch_tracer
);
193 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
196 #endif /* CONFIG_BRANCH_TRACER */
198 void ftrace_likely_update(struct ftrace_branch_data
*f
, int val
, int expect
)
201 * I would love to have a trace point here instead, but the
202 * trace point code is so inundated with unlikely and likely
203 * conditions that the recursive nightmare that exists is too
204 * much to try to get working. At least for now.
206 trace_likely_condition(f
, val
, expect
);
208 /* FIXME: Make this atomic! */
214 EXPORT_SYMBOL(ftrace_likely_update
);
216 extern unsigned long __start_annotated_branch_profile
[];
217 extern unsigned long __stop_annotated_branch_profile
[];
219 static int annotated_branch_stat_headers(struct seq_file
*m
)
221 seq_printf(m
, " correct incorrect %% ");
222 seq_printf(m
, " Function "
224 " ------- --------- - "
230 static inline long get_incorrect_percent(struct ftrace_branch_data
*p
)
235 percent
= p
->incorrect
* 100;
236 percent
/= p
->correct
+ p
->incorrect
;
238 percent
= p
->incorrect
? 100 : -1;
243 static int branch_stat_show(struct seq_file
*m
, void *v
)
245 struct ftrace_branch_data
*p
= v
;
249 /* Only print the file, not the path */
250 f
= p
->file
+ strlen(p
->file
);
251 while (f
>= p
->file
&& *f
!= '/')
256 * The miss is overlayed on correct, and hit on incorrect.
258 percent
= get_incorrect_percent(p
);
260 seq_printf(m
, "%8lu %8lu ", p
->correct
, p
->incorrect
);
262 seq_printf(m
, " X ");
264 seq_printf(m
, "%3ld ", percent
);
265 seq_printf(m
, "%-30.30s %-20.20s %d\n", p
->func
, f
, p
->line
);
269 static void *annotated_branch_stat_start(struct tracer_stat
*trace
)
271 return __start_annotated_branch_profile
;
275 annotated_branch_stat_next(void *v
, int idx
)
277 struct ftrace_branch_data
*p
= v
;
281 if ((void *)p
>= (void *)__stop_annotated_branch_profile
)
287 static int annotated_branch_stat_cmp(void *p1
, void *p2
)
289 struct ftrace_branch_data
*a
= p1
;
290 struct ftrace_branch_data
*b
= p2
;
292 long percent_a
, percent_b
;
294 percent_a
= get_incorrect_percent(a
);
295 percent_b
= get_incorrect_percent(b
);
297 if (percent_a
< percent_b
)
299 if (percent_a
> percent_b
)
305 static struct tracer_stat annotated_branch_stats
= {
306 .name
= "branch_annotated",
307 .stat_start
= annotated_branch_stat_start
,
308 .stat_next
= annotated_branch_stat_next
,
309 .stat_cmp
= annotated_branch_stat_cmp
,
310 .stat_headers
= annotated_branch_stat_headers
,
311 .stat_show
= branch_stat_show
314 __init
static int init_annotated_branch_stats(void)
318 ret
= register_stat_tracer(&annotated_branch_stats
);
320 printk(KERN_WARNING
"Warning: could not register "
321 "annotated branches stats\n");
326 fs_initcall(init_annotated_branch_stats
);
328 #ifdef CONFIG_PROFILE_ALL_BRANCHES
330 extern unsigned long __start_branch_profile
[];
331 extern unsigned long __stop_branch_profile
[];
333 static int all_branch_stat_headers(struct seq_file
*m
)
335 seq_printf(m
, " miss hit %% ");
336 seq_printf(m
, " Function "
338 " ------- --------- - "
344 static void *all_branch_stat_start(struct tracer_stat
*trace
)
346 return __start_branch_profile
;
350 all_branch_stat_next(void *v
, int idx
)
352 struct ftrace_branch_data
*p
= v
;
356 if ((void *)p
>= (void *)__stop_branch_profile
)
362 static struct tracer_stat all_branch_stats
= {
363 .name
= "branch_all",
364 .stat_start
= all_branch_stat_start
,
365 .stat_next
= all_branch_stat_next
,
366 .stat_headers
= all_branch_stat_headers
,
367 .stat_show
= branch_stat_show
370 __init
static int all_annotated_branch_stats(void)
374 ret
= register_stat_tracer(&all_branch_stats
);
376 printk(KERN_WARNING
"Warning: could not register "
377 "all branches stats\n");
382 fs_initcall(all_annotated_branch_stats
);
383 #endif /* CONFIG_PROFILE_ALL_BRANCHES */